From 8c171c1d1b79c1763633c208e028f58051e8340c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 28 Feb 2012 05:56:47 +0000
Subject: [PATCH 0001/1540] Branch 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1294469 13f79535-47bb-0310-9956-ffa450edef68

From 164d954d6e280b444d2d8a28cc55cbe50cd29ce6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 28 Feb 2012 19:01:22 +0000
Subject: [PATCH 0002/1540] HBASE-5484 Spelling mistake in error message in
 HMasterCommandLine

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1294789 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/HMasterCommandLine.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index 629d4fc352fa..ae9faf9a3488 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -126,9 +126,10 @@ private int startMaster() {
         zooKeeperCluster.setDefaultClientPort(zkClientPort);
         int clientPort = zooKeeperCluster.startup(zkDataPath);
         if (clientPort != zkClientPort) {
-          String errorMsg = "Couldnt start ZK at requested address of " +
-            zkClientPort + ", instead got: " + clientPort + ". Aborting. Why? " +
-            "Because clients (eg shell) wont be able to find this ZK quorum";
+          String errorMsg = "Could not start ZK at requested port of " +
+            zkClientPort + ".  ZK was started at port: " + clientPort +
+            ".  Aborting as clients (e.g. shell) will not be able to find " +
+            "this ZK quorum.";
           System.err.println(errorMsg);
           throw new IOException(errorMsg);
         }

From e7a3981372a210c24399b502c030b6c83d538c01 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 28 Feb 2012 19:39:51 +0000
Subject: [PATCH 0003/1540] HBASE-5485 LogCleaner refers to non-existant
 SnapshotLogCleaner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1294800 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/LogCleaner.java   |  3 +--
 .../hadoop/hbase/master/LogCleanerDelegate.java      | 12 ++++++------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
index 5a5d9a577c37..b5fc66501939 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
@@ -71,8 +71,7 @@ public LogCleaner(final int p, final Stoppable s,
 
   /*
    * Initialize the chain of log cleaners from the configuration. The default
-   * three LogCleanerDelegates in this chain are: TimeToLiveLogCleaner,
-   * ReplicationLogCleaner and SnapshotLogCleaner.
+   * in this chain are: TimeToLiveLogCleaner and ReplicationLogCleaner.
    */
   private void initLogCleanersChain() {
     String[] logCleaners = conf.getStrings(HBASE_MASTER_LOGCLEANER_PLUGINS);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
index 27ea161b015a..3c8eb0a1c2c1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
@@ -24,12 +24,12 @@
 import org.apache.hadoop.hbase.Stoppable;
 
 /**
- * Interface for the log cleaning function inside the master. By default, three
- * cleaners <code>TimeToLiveLogCleaner</code>,  <code>ReplicationLogCleaner</code>,
- * <code>SnapshotLogCleaner</code> are called in order. So if other effects are
- * needed, implement your own LogCleanerDelegate and add it to the configuration
- * "hbase.master.logcleaner.plugins", which is a comma-separated list of fully
- * qualified class names. LogsCleaner will add it to the chain.
+ * Interface for the log cleaning function inside the master. By default, two 
+ * cleaners: <code>TimeToLiveLogCleaner</code> and 
+ * <code>ReplicationLogCleaner</code> are called in order. So if other 
+ * effects are needed, implement your own LogCleanerDelegate and add it to the
+ * configuration "hbase.master.logcleaner.plugins", which is a comma-separated
+ * list of fully qualified class names. LogsCleaner will add it to the chain.
  *
  * <p>HBase ships with LogsCleaner as the default implementation.
  *

From 8208e292a28d1ef65b990f717a8b0ec188c0bb19 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 29 Feb 2012 22:34:38 +0000
Subject: [PATCH 0004/1540] Set version to 0.94.0-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1295324 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 4460d5d22461..c60b9256eb8d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.93-SNAPSHOT</version>
+  <version>0.94.0-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 50e529baefb25dcb6aa25073638c80db4432b940 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 29 Feb 2012 22:41:36 +0000
Subject: [PATCH 0005/1540] HBASE-5497 Add protobuf as M/R dependency jar
 (mapred)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1295328 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index fb80cbbe5bb5..bd098796c02e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -262,6 +262,7 @@ public static void addDependencyJars(JobConf job) throws IOException {
       job,
       org.apache.zookeeper.ZooKeeper.class,
       com.google.common.base.Function.class,
+      com.google.protobuf.Message.class,
       job.getMapOutputKeyClass(),
       job.getMapOutputValueClass(),
       job.getOutputKeyClass(),

From d0c628c50c41b924a5718909073698c654a846d7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 1 Mar 2012 06:52:35 +0000
Subject: [PATCH 0006/1540] HBASE-5491 Remove HBaseConfiguration.create() call
 from coprocessor.Exec class

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1295431 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/coprocessor/Exec.java   | 1 -
 .../hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java       | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
index 45cc463918dc..75535b5efc42 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
@@ -52,7 +52,6 @@
  * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)
  */
 public class Exec extends Invocation implements Row {
-  private Configuration conf = HBaseConfiguration.create();
   /** Row key used as a reference for any region lookups */
   private byte[] referenceRow;
   private Class<? extends CoprocessorProtocol> protocol;
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
index e1f52c1bace9..e38cdcd4d7c0 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
@@ -200,6 +201,7 @@ public void testExecDeserialization() throws IOException {
     dib.reset(dob.getData(), dob.getLength());
 
     Exec after = new Exec();
+    after.setConf(HBaseConfiguration.create());
     after.readFields(dib);
     // no error thrown
     assertEquals(after.getProtocolName(), protocolName);

From 38f431e156c58ca6b78a527edeca5ea0bc097d04 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 1 Mar 2012 17:36:23 +0000
Subject: [PATCH 0007/1540] HBASE-5454 Refuse operations from Admin before
 master is initialized

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1295693 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/PleaseHoldException.java      |  7 ++++---
 .../apache/hadoop/hbase/master/HMaster.java    | 18 ++++++++++++++++--
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
index e3a9315fd491..309dca413437 100644
--- a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
+++ b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
@@ -22,9 +22,10 @@
 import java.io.IOException;
 
 /**
- * This exception is thrown by the master when a region server was shut down
- * and restarted so fast that the master still hasn't processed the server
- * shutdown of the first instance.
+ * This exception is thrown by the master when a region server was shut down and
+ * restarted so fast that the master still hasn't processed the server shutdown
+ * of the first instance, or when master is initializing and client call admin
+ * operations
  */
 @SuppressWarnings("serial")
 public class PleaseHoldException extends IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index fe102ab3bc94..3e4b4d048cf7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -26,16 +26,15 @@
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.management.ObjectName;
 
@@ -50,6 +49,7 @@
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableDescriptors;
@@ -1178,6 +1178,7 @@ private static boolean isCatalogTable(final byte [] tableName) {
 
   @Override
   public void deleteTable(final byte [] tableName) throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       cpHost.preDeleteTable(tableName);
     }
@@ -1240,6 +1241,7 @@ private Pair<Integer, Integer> getAlterStatusFromSchemaChangeTracker(byte[] tabl
 
   public void addColumn(byte [] tableName, HColumnDescriptor column)
   throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       if (cpHost.preAddColumn(tableName, column)) {
         return;
@@ -1254,6 +1256,7 @@ public void addColumn(byte [] tableName, HColumnDescriptor column)
 
   public void modifyColumn(byte [] tableName, HColumnDescriptor descriptor)
   throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       if (cpHost.preModifyColumn(tableName, descriptor)) {
         return;
@@ -1268,6 +1271,7 @@ public void modifyColumn(byte [] tableName, HColumnDescriptor descriptor)
 
   public void deleteColumn(final byte [] tableName, final byte [] c)
   throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       if (cpHost.preDeleteColumn(tableName, c)) {
         return;
@@ -1281,6 +1285,7 @@ public void deleteColumn(final byte [] tableName, final byte [] c)
   }
 
   public void enableTable(final byte [] tableName) throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       cpHost.preEnableTable(tableName);
     }
@@ -1293,6 +1298,7 @@ public void enableTable(final byte [] tableName) throws IOException {
   }
 
   public void disableTable(final byte [] tableName) throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       cpHost.preDisableTable(tableName);
     }
@@ -1342,6 +1348,7 @@ public boolean processRow(Result data) throws IOException {
   @Override
   public void modifyTable(final byte[] tableName, HTableDescriptor htd)
       throws IOException {
+    checkInitialized();
     if (cpHost != null) {
       cpHost.preModifyTable(tableName, htd);
     }
@@ -1648,6 +1655,11 @@ public boolean isAborted() {
     return this.abort;
   }
   
+  void checkInitialized() throws PleaseHoldException {
+    if (!this.initialized) {
+      throw new PleaseHoldException("Master is initializing");
+    }
+  }
   
   /**
    * Report whether this master is currently the active master or not.
@@ -1683,6 +1695,7 @@ public void assign(final byte[] regionName, final boolean force)
 
   @Override
   public void assign(final byte [] regionName)throws IOException {
+    checkInitialized();
     Pair<HRegionInfo, ServerName> pair =
       MetaReader.getRegion(this.catalogTracker, regionName);
     if (pair == null) throw new UnknownRegionException(Bytes.toString(regionName));
@@ -1706,6 +1719,7 @@ public void assignRegion(HRegionInfo hri) {
   @Override
   public void unassign(final byte [] regionName, final boolean force)
   throws IOException {
+    checkInitialized();
     Pair<HRegionInfo, ServerName> pair =
       MetaReader.getRegion(this.catalogTracker, regionName);
     if (pair == null) throw new UnknownRegionException(Bytes.toString(regionName));

From 95ee56bb36a09bed88776e686487539c8c788c02 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 1 Mar 2012 17:51:17 +0000
Subject: [PATCH 0008/1540] HBASE-5502 region_mover.rb fails to load regions
 back to original server for regions only containing empty tables.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1295708 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/region_mover.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 028f9a89cbe7..94da5d90bd9b 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -364,7 +364,8 @@ def loadRegions(options, hostname)
   for r in regions
     exists = false
     begin
-      exists = isSuccessfulScan(admin, r)
+      isSuccessfulScan(admin, r)
+      exists = true
     rescue org.apache.hadoop.hbase.NotServingRegionException => e
       $LOG.info("Failed scan of " + e.message)
     end

From 9f4022eee366addceb1ba73630962ffe29ae6ece Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 1 Mar 2012 18:43:13 +0000
Subject: [PATCH 0009/1540] HBASE-5489 Add HTable accessor to get regions for a
 key range

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1295728 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTable.java    | 29 +++++++++
 .../hbase/client/TestFromClientSide.java      | 60 +++++++++++++++++++
 2 files changed, 89 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 29b8004b9432..137b1ca2961b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -487,6 +487,35 @@ public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOExcep
     return MetaScanner.allTableRegions(getConfiguration(), getTableName(), false);
   }
 
+  /**
+   * Get the corresponding regions for an arbitrary range of keys.
+   * <p>
+   * @param startRow Starting row in range, inclusive
+   * @param endRow Ending row in range, inclusive
+   * @return A list of HRegionLocations corresponding to the regions that
+   * contain the specified range
+   * @throws IOException if a remote or network exception occurs
+   */
+  public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
+    final byte [] endKey) throws IOException {
+    final boolean endKeyIsEndOfTable = Bytes.equals(endKey,
+                                                    HConstants.EMPTY_END_ROW);
+    if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
+      throw new IllegalArgumentException(
+        "Invalid range: " + Bytes.toStringBinary(startKey) +
+        " > " + Bytes.toStringBinary(endKey));
+    }
+    final List<HRegionLocation> regionList = new ArrayList<HRegionLocation>();
+    byte [] currentKey = startKey;
+    do {
+      HRegionLocation regionLocation = getRegionLocation(currentKey, false);
+      regionList.add(regionLocation);
+      currentKey = regionLocation.getRegionInfo().getEndKey();
+    } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) &&
+             (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
+    return regionList;
+  }
+
   /**
    * Save the passed region information and the table's regions
    * cache.
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index bdeaefefb943..17130448b069 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -57,6 +57,7 @@
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
@@ -4594,6 +4595,65 @@ public void testNonCachedGetRegionLocation() throws Exception {
     assertTrue(addrAfter.getPort() != addrCache.getPort());
     assertEquals(addrAfter.getPort(), addrNoCache.getPort());
   }  
+
+  @Test
+  /**
+   * Tests getRegionsInRange by creating some regions over which a range of
+   * keys spans; then changing the key range.
+   */
+  public void testGetRegionsInRange() throws Exception {
+    // Test Initialization.
+    byte [] startKey = Bytes.toBytes("ddc");
+    byte [] endKey = Bytes.toBytes("mmm");
+    byte [] TABLE = Bytes.toBytes("testGetRegionsInRange");
+    HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
+    int numOfRegions = TEST_UTIL.createMultiRegions(table, FAMILY);
+    assertEquals(25, numOfRegions);
+    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+
+    // Get the regions in this range
+    List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,
+      endKey);
+    assertEquals(10, regionsList.size());
+
+    // Change the start key
+    startKey = Bytes.toBytes("fff");
+    regionsList = table.getRegionsInRange(startKey, endKey);
+    assertEquals(7, regionsList.size());
+
+    // Change the end key
+    endKey = Bytes.toBytes("nnn");
+    regionsList = table.getRegionsInRange(startKey, endKey);
+    assertEquals(8, regionsList.size());
+
+    // Empty start key
+    regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
+    assertEquals(13, regionsList.size());
+
+    // Empty end key
+    regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
+    assertEquals(20, regionsList.size());
+
+    // Both start and end keys empty
+    regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW,
+      HConstants.EMPTY_END_ROW);
+    assertEquals(25, regionsList.size());
+
+    // Change the end key to somewhere in the last block
+    endKey = Bytes.toBytes("yyz");
+    regionsList = table.getRegionsInRange(startKey, endKey);
+    assertEquals(20, regionsList.size());
+
+    // Change the start key to somewhere in the first block
+    startKey = Bytes.toBytes("aac");
+    regionsList = table.getRegionsInRange(startKey, endKey);
+    assertEquals(25, regionsList.size());
+
+    // Make start and end key the same
+    startKey = endKey = Bytes.toBytes("ccc");
+    regionsList = table.getRegionsInRange(startKey, endKey);
+    assertEquals(1, regionsList.size());
+  }
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From d12dbbea73f57fccdfbf6de6e518ea25e9d553d9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 2 Mar 2012 00:25:21 +0000
Subject: [PATCH 0010/1540] HBASE-5489 Addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296010 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HTable.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 137b1ca2961b..5e7106a5475e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -491,7 +491,7 @@ public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOExcep
    * Get the corresponding regions for an arbitrary range of keys.
    * <p>
    * @param startRow Starting row in range, inclusive
-   * @param endRow Ending row in range, inclusive
+   * @param endRow Ending row in range, exclusive
    * @return A list of HRegionLocations corresponding to the regions that
    * contain the specified range
    * @throws IOException if a remote or network exception occurs

From 3713f67c18423c8b32783aec96dbef060ac9b796 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 2 Mar 2012 17:27:14 +0000
Subject: [PATCH 0011/1540] HBASE-5511 More doc on maven release process

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296317 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/pom.xml b/pom.xml
index c60b9256eb8d..e7cd62faeadd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -324,6 +324,20 @@
          can be overwritten here. -->
     <pluginManagement>
       <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-release-plugin</artifactId>
+          <version>${maven-release.version}</version>
+          <configuration>
+            <!--This stops our running tests for each stage of maven release.
+                We may want to disable this because it looks like this might
+                be responsible for our not including test jar in our release
+                to maven repository. Remove in 0.94 and trunk since tests
+                run faster there?
+              -->
+            <arguments>-Dmaven.test.skip</arguments>
+          </configuration>
+        </plugin>
         <plugin>
           <artifactId>maven-compiler-plugin</artifactId>
           <configuration>

From 64cf93a4546e3c608e4a13fe76a88a19baff759a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 2 Mar 2012 18:05:29 +0000
Subject: [PATCH 0012/1540] HBASE-5430 Fix licenses in 0.92.1 -- RAT plugin
 won't pass

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296357 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pom.xml b/pom.xml
index e7cd62faeadd..622d4e10fbdb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -502,6 +502,8 @@
           <configuration>
             <excludes>
               <exclude>**/.*</exclude>
+              <exclude>**/*.tgz</exclude>
+              <exclude>**/8e8ab58dcf39412da19833fcd8f687ac</exclude>
               <exclude>**/.git/**</exclude>
               <exclude>**/target/**</exclude>
               <exclude>**/CHANGES.txt</exclude>

From d3122bf73ace318976d4ee7f7555895d09b62d0b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 2 Mar 2012 19:04:48 +0000
Subject: [PATCH 0013/1540] IGNORE; a nothing commit to confirm I can commit
 here

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296387 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 52d21202fe9c..a49d47097285 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 HBase Change Log
+
 Release 0.93.0 - Unreleased
   *DO NOT ADD ISSUES HERE ON COMMIT ANY MORE.  WE'LL GENERATE THE LIST
   FROM JIRA INSTEAD WHEN WE MAKE A RELEASE*

From d2fb2d5e2494834947799c5f4fbd72955e7fdba1 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 2 Mar 2012 19:11:01 +0000
Subject: [PATCH 0014/1540] HBASE-5508 Add an option to allow test output to
 show on the terminal (Scott Chen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296390 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 622d4e10fbdb..aa92a29c2ea9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -365,7 +365,7 @@
           <configuration>
             <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
             <argLine>-enableassertions -Xmx1900m -Djava.security.egd=file:/dev/./urandom</argLine>
-            <redirectTestOutputToFile>true</redirectTestOutputToFile>
+            <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
           </configuration>
         </plugin>
         <plugin>
@@ -393,7 +393,7 @@
               <exclude>**/*$*</exclude>
               <exclude>${test.exclude.pattern}</exclude>
             </excludes>
-            <redirectTestOutputToFile>true</redirectTestOutputToFile>
+            <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
             <environmentVariables>
               <LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/nativelib</LD_LIBRARY_PATH>
               <DYLD_LIBRARY_PATH>${env.DYLD_LIBRARY_PATH}:${project.build.directory}/nativelib</DYLD_LIBRARY_PATH>
@@ -1008,6 +1008,7 @@
 
     <surefire.firstPartGroups>org.apache.hadoop.hbase.SmallTests</surefire.firstPartGroups>
     <surefire.secondPartGroups>org.apache.hadoop.hbase.MediumTests</surefire.secondPartGroups>
+    <test.output.tofile>true</test.output.tofile>
   </properties>
 
   <!-- Sorted by groups of dependencies then groupId and artifactId -->

From 84c103829e3cd34fe43721b0f9583bb5d4ffefc2 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 3 Mar 2012 16:47:55 +0000
Subject: [PATCH 0015/1540] HBASE-5286 bin/hbase's logic of adding Hadoop jar
 files to the classpath is fragile when presented with split packaged Hadoop
 0.23 installation

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296664 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase                                     | 31 ++++------------
 .../hadoop/hbase/util/GetJavaProperty.java    | 37 +++++++++++++++++++
 2 files changed, 44 insertions(+), 24 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java

diff --git a/bin/hbase b/bin/hbase
index c5692ffb746b..1a7ad3786bbf 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -217,31 +217,14 @@ function append_path() {
 JAVA_PLATFORM=""
 
 #If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
-if [ ! -z $HADOOP_HOME ]; then
-  HADOOPCPPATH=""
-  if [ -z $HADOOP_CONF_DIR ]; then
-    HADOOPCPPATH=$(append_path "${HADOOPCPPATH}" "${HADOOP_HOME}/conf")
-  else
-    HADOOPCPPATH=$(append_path "${HADOOPCPPATH}" "${HADOOP_CONF_DIR}")
-  fi
-  if [ "`echo ${HADOOP_HOME}/hadoop-core*.jar`" != "${HADOOP_HOME}/hadoop-core*.jar" ] ; then
-    HADOOPCPPATH=$(append_path "${HADOOPCPPATH}" `ls ${HADOOP_HOME}/hadoop-core*.jar | head -1`)
-  else
-    HADOOPCPPATH=$(append_path "${HADOOPCPPATH}" `ls ${HADOOP_HOME}/hadoop-common*.jar | head -1`)
-    HADOOPCPPATH=$(append_path "${HADOOPCPPATH}" `ls ${HADOOP_HOME}/hadoop-hdfs*.jar | head -1`)
-    HADOOPCPPATH=$(append_path "${HADOOPCPPATH}" `ls ${HADOOP_HOME}/hadoop-mapred*.jar | head -1`)
-  fi
-  for i in "${HADOOP_HOME}/lib/"*.jar; do
-    HADOOPCPPATH="${HADOOPCPPATH}:$i"
-  done
-  CLASSPATH=$(append_path "${CLASSPATH}" "${HADOOPCPPATH}")
-  
-  if [ -d "${HADOOP_HOME}/lib/native" ]; then
-    JAVA_PLATFORM=`CLASSPATH=${HADOOPCPPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
-    if [ -d "${HADOOP_HOME}/lib/native/${JAVA_PLATFORM}" ]; then
-      JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "${HADOOP_HOME}/lib/native/${JAVA_PLATFORM}")
-    fi
+HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null)
+if [ -f ${HADOOP_IN_PATH} ]; then
+  HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \
+                             org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null)
+  if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then
+    JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH")
   fi
+  CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`)
 fi
 
 if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then
diff --git a/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java b/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java
new file mode 100644
index 000000000000..361334db20f7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2012 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+/**
+ * A generic way for querying Java properties.
+ */
+public class GetJavaProperty {
+  public static void main(String args[]) {
+    if (args.length == 0) {
+      for (Object prop: System.getProperties().keySet()) {
+        System.out.println(prop + "=" + System.getProperty((String)prop, ""));
+      }
+    } else {
+      for (String prop: args) {
+        System.out.println(System.getProperty(prop, ""));
+      }
+    }
+  }
+}

From d8dae66308c82b97e5455e5b9b3fbcbb4eec3f91 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 3 Mar 2012 21:49:43 +0000
Subject: [PATCH 0016/1540] HBASE-5371 Introduce
 AccessControllerProtocol.checkPermissions(Permission[] permissons) API,
 addendum (Enis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1296710 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/security/access/AccessController.java  | 2 +-
 .../hadoop/hbase/security/access/AccessControllerProtocol.java | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 6108de898d3a..c1f20de7b315 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -163,7 +163,7 @@ public static AuthResult deny(String reason, User user,
   /**
    * Version number for AccessControllerProtocol
    */
-  private static final long PROTOCOL_VERSION = 2L;
+  private static final long PROTOCOL_VERSION = 1L;
 
   TableAuthManager authManager = null;
 
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java
index 78cca4f88df4..c11f339653a9 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java
@@ -28,8 +28,7 @@
  */
 public interface AccessControllerProtocol extends CoprocessorProtocol {
 
-  /* V2: Added {@link #checkPermissions(Permission...)}) */
-  public static final long VERSION = 2L;
+  public static final long VERSION = 1L;
 
   /**
    * Grants the given user or group the privilege to perform the given actions

From 41c1ad1ca606a2ad34395f2a5fc1a28b2ed1f2a8 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 5 Mar 2012 23:10:20 +0000
Subject: [PATCH 0017/1540] HBASE-5524 Add a couple of more filters to our rat
 exclusion set

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1297278 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pom.xml b/pom.xml
index aa92a29c2ea9..f4a1c717ddf5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -503,6 +503,8 @@
             <excludes>
               <exclude>**/.*</exclude>
               <exclude>**/*.tgz</exclude>
+              <exclude>**/*.orig</exclude>
+              <exclude>**/test/**</exclude>
               <exclude>**/8e8ab58dcf39412da19833fcd8f687ac</exclude>
               <exclude>**/.git/**</exclude>
               <exclude>**/target/**</exclude>

From c148901119adf0591a4bb5caaa6801df19807af2 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 6 Mar 2012 02:16:14 +0000
Subject: [PATCH 0018/1540] HBASE-5507
 ThriftServerRunner.HbaseHandler.getRegionInfo() and getTableRegions() do not
 use ByteBuffer correctly (Scott chen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1297322 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/HRegionThriftServer.java     | 12 ++++-----
 .../hbase/thrift/ThriftServerRunner.java      | 20 +++++++++++---
 .../hadoop/hbase/thrift/TestThriftServer.java |  8 ++++++
 .../hbase/thrift/TestThriftServerCmdLine.java | 27 +++++++++----------
 4 files changed, 43 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
index 1972f35cac87..7acfe9bd09e2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
@@ -19,6 +19,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import static org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler.toBytes;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -117,8 +119,8 @@ public List<TRowResult> getRowWithColumnsTs(ByteBuffer tableName,
                                                 long timestamp,
       Map<ByteBuffer, ByteBuffer> attributes) throws IOError {
       try {
-        byte [] row = rowb.array();
-        HTable table = getTable(tableName.array());
+        byte[] row = toBytes(rowb);
+        HTable table = getTable(toBytes(tableName));
         HRegionLocation location = table.getRegionLocation(row, false);
         byte[] regionName = location.getRegionInfo().getRegionName();
 
@@ -128,11 +130,9 @@ public List<TRowResult> getRowWithColumnsTs(ByteBuffer tableName,
           Result result = rs.get(regionName, get);
           return ThriftUtilities.rowResultFromHBase(result);
         }
-        ByteBuffer[] columnArr = columns.toArray(
-                                   new ByteBuffer[columns.size()]);
         Get get = new Get(row);
-        for(ByteBuffer column : columnArr) {
-          byte [][] famAndQf = KeyValue.parseColumn(column.array());
+        for(ByteBuffer column : columns) {
+          byte [][] famAndQf = KeyValue.parseColumn(toBytes(column));
           if (famAndQf.length == 1) {
             get.addFamily(famAndQf[0]);
           } else {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index f4c6fa7c28dd..dc509d947a29 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -569,7 +569,8 @@ public List<ByteBuffer> getTableNames() throws IOError {
     public List<TRegionInfo> getTableRegions(ByteBuffer tableName)
     throws IOError {
       try{
-        List<HRegionInfo> hris = this.admin.getTableRegions(tableName.array());
+        List<HRegionInfo> hris =
+            this.admin.getTableRegions(toBytes(tableName));
         List<TRegionInfo> regions = new ArrayList<TRegionInfo>();
 
         if (hris != null) {
@@ -590,6 +591,18 @@ public List<TRegionInfo> getTableRegions(ByteBuffer tableName)
       }
     }
 
+    /**
+     * Convert ByteBuffer to byte array. Note that this cannot be replaced by
+     * Bytes.toBytes().
+     */
+    public static byte[] toBytes(ByteBuffer bb) {
+      byte[] result = new byte[bb.remaining()];
+      // Make a duplicate so the position doesn't change
+      ByteBuffer dup = bb.duplicate();
+      dup.get(result, 0, result.length);
+      return result;
+    }
+
     @Deprecated
     @Override
     public List<TCell> get(
@@ -1321,8 +1334,9 @@ public List<TCell> getRowOrBefore(ByteBuffer tableName, ByteBuffer row,
     public TRegionInfo getRegionInfo(ByteBuffer searchRow) throws IOError {
       try {
         HTable table = getTable(HConstants.META_TABLE_NAME);
+        byte[] row = toBytes(searchRow);
         Result startRowResult = table.getRowOrBefore(
-          searchRow.array(), HConstants.CATALOG_FAMILY);
+          row, HConstants.CATALOG_FAMILY);
 
         if (startRowResult == null) {
           throw new IOException("Cannot find row in .META., row="
@@ -1335,7 +1349,7 @@ public TRegionInfo getRegionInfo(ByteBuffer searchRow) throws IOError {
         if (value == null || value.length == 0) {
           throw new IOException("HRegionInfo REGIONINFO was null or " +
                                 " empty in Meta for row="
-                                + Bytes.toString(searchRow.array()));
+                                + Bytes.toString(row));
         }
         HRegionInfo regionInfo = Writables.getHRegionInfo(value);
         TRegionInfo region = new TRegionInfo();
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 8357e29641a9..be81552d1a32 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -116,6 +116,9 @@ public void testAll() throws Exception {
   public void doTestTableCreateDrop() throws Exception {
     ThriftServerRunner.HBaseHandler handler =
       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
+    doTestTableCreateDrop(handler);
+  }
+  public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception {
     createTestTables(handler);
     dropTestTables(handler);
   }
@@ -420,6 +423,11 @@ public void doTestTableScanners() throws Exception {
   public void doTestGetTableRegions() throws Exception {
     ThriftServerRunner.HBaseHandler handler =
       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
+    doTestGetTableRegions(handler);
+  }
+
+  public static void doTestGetTableRegions(Hbase.Iface handler)
+      throws Exception {
     handler.createTable(tableAname, getColumnDescriptors());
     int regionCount = handler.getTableRegions(tableAname).size();
     assertEquals("empty table should have only 1 region, " +
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index 0e158fb5eca8..6f1872d138e1 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -194,22 +194,19 @@ private void talkToThriftServer() throws Exception {
     }
 
     sock.open();
-    TProtocol prot;
-    if (specifyCompact) {
-      prot = new TCompactProtocol(transport);
-    } else {
-      prot = new TBinaryProtocol(transport);
-    }
-    Hbase.Client client = new Hbase.Client(prot);
-    List<ByteBuffer> tableNames = client.getTableNames();
-    if (tableNames.isEmpty()) {
-      TestThriftServer.createTestTables(client);
-      assertEquals(2, client.getTableNames().size());
-    } else {
-      assertEquals(2, tableNames.size());
-      assertEquals(2, client.getColumnDescriptors(tableNames.get(0)).size());
+    try {
+      TProtocol prot;
+      if (specifyCompact) {
+        prot = new TCompactProtocol(transport);
+      } else {
+        prot = new TBinaryProtocol(transport);
+      }
+      Hbase.Client client = new Hbase.Client(prot);
+      TestThriftServer.doTestTableCreateDrop(client);
+      TestThriftServer.doTestGetTableRegions(client);
+    } finally {
+      sock.close();
     }
-    sock.close();
   }
 
   private void stopCmdLineThread() throws Exception {

From 7cf148cb0a7c09de431c2ab31b65021a0b51a3d2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 6 Mar 2012 06:21:23 +0000
Subject: [PATCH 0019/1540] HBASE-5523 Fix Delete Timerange logic for
 KEEP_DELETED_CELLS

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1297345 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanQueryMatcher.java   |  3 +--
 .../hbase/client/TestFromClientSide.java       |  2 +-
 .../hbase/regionserver/TestKeepDeletes.java    | 18 +++++++++---------
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 52459f0a57d4..9f0cba9eb9c7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -279,8 +279,7 @@ public MatchCode match(KeyValue kv) throws IOException {
         // first ignore delete markers if the scanner can do so, and the
         // range does not include the marker
         boolean includeDeleteMarker = seePastDeleteMarkers ?
-            // +1, to allow a range between a delete and put of same TS
-            tr.withinTimeRange(timestamp+1) :
+            tr.withinTimeRange(timestamp) :
             tr.withinOrAfterTimeRange(timestamp);
         if (includeDeleteMarker) {
           this.deletes.add(bytes, offset, qualLength, timestamp, type);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 17130448b069..56403c710b9c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -178,7 +178,7 @@ public void testKeepDeletedCells() throws Exception {
      p.add(FAMILY, C0, T3);
      h.put(p);
 
-     Delete d = new Delete(T1, ts+2, null);
+     Delete d = new Delete(T1, ts+3, null);
      h.delete(d);
 
      d = new Delete(T1, ts+3, null);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
index 72d9c405a18b..ae5da03e0e63 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
@@ -447,16 +447,16 @@ public void testRanges() throws Exception {
     p.add(c1, c1, T2);
     region.put(p);
 
-    Delete d = new Delete(T1, ts+1, null);
-    d.deleteColumns(c0, c0, ts+1);
+    Delete d = new Delete(T1, ts+2, null);
+    d.deleteColumns(c0, c0, ts+2);
     region.delete(d, null, true);
 
-    d = new Delete(T1, ts+1, null);
-    d.deleteFamily(c1, ts+1);
+    d = new Delete(T1, ts+2, null);
+    d.deleteFamily(c1, ts+2);
     region.delete(d, null, true);
 
-    d = new Delete(T2, ts+1, null);
-    d.deleteFamily(c0, ts+1);
+    d = new Delete(T2, ts+2, null);
+    d.deleteFamily(c0, ts+2);
     region.delete(d, null, true);
 
     // add an older delete, to make sure it is filtered
@@ -464,7 +464,7 @@ public void testRanges() throws Exception {
     d.deleteFamily(c1, ts-10);
     region.delete(d, null, true);
 
-    // ts + 2 does NOT include the delete at ts+1
+    // ts + 2 does NOT include the delete at ts+2
     checkGet(region, T1, c0, c0, ts+2, T2, T1);
     checkGet(region, T1, c0, c1, ts+2, T2, T1);
     checkGet(region, T1, c1, c0, ts+2, T2, T1);
@@ -610,10 +610,10 @@ public void testWithMixedCFs() throws Exception {
     region.put(p);
 
     // family markers are each family
-    Delete d = new Delete(T1, ts, null);
+    Delete d = new Delete(T1, ts+1, null);
     region.delete(d, null, true);
 
-    d = new Delete(T2, ts+1, null);
+    d = new Delete(T2, ts+2, null);
     region.delete(d, null, true);
 
     Scan s = new Scan(T1);

From 99ace7915a48f671d76edc8360a12c6d7982bb6e Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 6 Mar 2012 17:45:54 +0000
Subject: [PATCH 0020/1540] HBASE-5531 Maven hadoop profile (version 23) needs
 to be updated with latest 23 snapshot (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1297595 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f4a1c717ddf5..d07a73a6aeee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1821,7 +1821,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>0.23.1-SNAPSHOT</hadoop.version>
+        <hadoop.version>0.23.2-SNAPSHOT</hadoop.version>
       </properties>
       <dependencies>
         <dependency>

From 9a5fa356c837bfcdc455051385ca8b80b32e3c82 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 6 Mar 2012 20:26:08 +0000
Subject: [PATCH 0021/1540] HBASE-5522 hbase 0.92 test artifacts are missing
 from Maven central

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1297682 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/pom.xml b/pom.xml
index d07a73a6aeee..9e4884bbca5a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -327,15 +327,25 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-release-plugin</artifactId>
-          <version>${maven-release.version}</version>
+          <!--Making a release I've been using mvn 3.0 and specifying the apache-release
+              profile on the command line as follows:
+
+              $ ~/bin/mvn/bin/mvn  -Papache-release release:perform
+              
+              I've also been kiling the release:prepare step mid-way to check the
+              release.properties it generates at the top-level.  Sometimes it refers
+              to HEAD rather than to the svn branch.
+           -->
           <configuration>
+            <!--You need this profile. It'll sign your artifacts.
+                I'm not sure if this config. actually works though.
+                I've been specifying -Papache-release on the command-line
+             -->
+            <releaseProfiles>apache-release</releaseProfiles>
             <!--This stops our running tests for each stage of maven release.
-                We may want to disable this because it looks like this might
-                be responsible for our not including test jar in our release
-                to maven repository. Remove in 0.94 and trunk since tests
-                run faster there?
+                But it builds the test jar.  From SUREFIRE-172.      
               -->
-            <arguments>-Dmaven.test.skip</arguments>
+            <arguments>-Dmaven.test.skip.exec</arguments>
           </configuration>
         </plugin>
         <plugin>

From 30731b08c8b3eca965b3f897ea3effcf3555d09b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 6 Mar 2012 20:46:35 +0000
Subject: [PATCH 0022/1540] HBASE-5519 Incorrect warning in splitlogmanager

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1297708 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/SplitLogManager.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 79862c50165f..c6b355b532f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -485,7 +485,10 @@ private void heartbeat(String path, int new_version,
           new_version, workerName);
       tot_mgr_heartbeat.incrementAndGet();
     } else {
-      LOG.warn("got dup heartbeat for " + path + " ver = " + new_version);
+      // duplicate heartbeats - heartbeats w/o zk node version
+      // changing - are possible. The timeout thread does
+      // getDataSetWatch() just to check whether a node still
+      // exists or not
     }
     return;
   }

From d644df42fcc2f39726349d717569d52be496f00c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 7 Mar 2012 18:47:24 +0000
Subject: [PATCH 0023/1540] HBASE-5537 MXBean shouldn't have a dependence on
 InterfaceStability until 0.96

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298038 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/MXBean.java       | 2 --
 src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/MXBean.java b/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
index 7f44dc2db5e8..41da8fb3de62 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
@@ -19,13 +19,11 @@
 
 import java.util.Map;
 
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.hbase.HServerLoad;
 
 /**
  * This is the JMX management interface for Hbase master information
  */
-@Evolving
 public interface MXBean {
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java
index b0a92c573c07..2d75ab54d844 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-
 /**
  * This is the JMX management interface for HBase Region Server information
  */
-@Evolving
 public interface MXBean {
 
   /**

From c5570926a2a4d0fefcaa15c6391827fdc9edcca9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 8 Mar 2012 04:59:32 +0000
Subject: [PATCH 0024/1540] HBASE-4890 fix possible NPE in HConnectionManager

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298271 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/HBaseClient.java  | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 48a68b06b890..0aed5bf6e26f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -674,7 +674,17 @@ protected void cleanupCalls(long rpcTimeout) {
         Call c = itor.next().getValue();
         long waitTime = System.currentTimeMillis() - c.getStartTime();
         if (waitTime >= rpcTimeout) {
-          c.setException(closeException); // local exception
+          if (this.closeException == null) {
+            // There may be no exception in the case that there are many calls
+            // being multiplexed over this connection and these are succeeding
+            // fine while this Call object is taking a long time to finish
+            // over on the server; e.g. I just asked the regionserver to bulk
+            // open 3k regions or its a big fat multiput into a heavily-loaded
+            // server (Perhaps this only happens at the extremes?)
+            this.closeException = new CallTimeoutException("Call id=" + c.id +
+              ", waitTime=" + waitTime + ", rpcTimetout=" + rpcTimeout);
+          }
+          c.setException(this.closeException);
           synchronized (c) {
             c.notifyAll();
           }
@@ -703,6 +713,15 @@ protected void cleanupCalls(long rpcTimeout) {
     }
   }
 
+  /**
+   * Client-side call timeout
+   */
+  public static class CallTimeoutException extends IOException {
+    public CallTimeoutException(final String msg) {
+      super(msg);
+    }
+  }
+
   /** Call implementation used for parallel calls. */
   protected class ParallelCall extends Call {
     private final ParallelResults results;

From 6ac7d93dada4e766b9874ab389bbbe7c350d7ba6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 8 Mar 2012 20:56:13 +0000
Subject: [PATCH 0025/1540] HBASE-5529 MR test failures becuase
 MALLOC_ARENA_MAX is not set (Gregory Chanan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298575 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 9e4884bbca5a..2cb0239ab6b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -407,6 +407,7 @@
             <environmentVariables>
               <LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/nativelib</LD_LIBRARY_PATH>
               <DYLD_LIBRARY_PATH>${env.DYLD_LIBRARY_PATH}:${project.build.directory}/nativelib</DYLD_LIBRARY_PATH>
+              <MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
             </environmentVariables>
           </configuration>
           <executions>

From bf540d55c2a627c4aa2443084977301da329964e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 8 Mar 2012 23:59:16 +0000
Subject: [PATCH 0026/1540] HBASE-5526 Configurable file and directory based
 umask (Jesse Yates)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298658 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |   5 +
 .../hbase/io/hfile/AbstractHFileWriter.java   |   9 +-
 .../hadoop/hbase/regionserver/HRegion.java    |  10 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java | 100 +++++++++++++++---
 src/main/resources/hbase-default.xml          |  14 +++
 .../apache/hadoop/hbase/util/TestFSUtils.java |  49 ++++++++-
 6 files changed, 164 insertions(+), 23 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 17cb0e3d9fd6..d1a4cc15f33a 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -602,6 +602,11 @@ public static enum Modify {
   /** Host name of the local machine */
   public static final String LOCALHOST = "localhost";
 
+  /** Enable file permission modification from standard hbase */
+  public static final String ENABLE_DATA_FILE_UMASK = "hbase.data.umask.enable";
+  /** File permission umask to use when creating hbase data files */
+  public static final String DATA_FILE_UMASK_KEY = "hbase.data.umask";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
index 52f063f5ec74..32aa2ed805e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
@@ -29,10 +29,12 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.Writable;
 
@@ -264,9 +266,8 @@ public static Compression.Algorithm compressionByName(String algoName) {
   /** A helper method to create HFile output streams in constructors */
   protected static FSDataOutputStream createOutputStream(Configuration conf,
       FileSystem fs, Path path) throws IOException {
-    return fs.create(path, FsPermission.getDefault(), true,
-        fs.getConf().getInt("io.file.buffer.size", 4096),
-        fs.getDefaultReplication(), fs.getDefaultBlockSize(),
-        null);
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf,
+        HConstants.DATA_FILE_UMASK_KEY);
+    return FSUtils.create(fs, path, perms);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 16277280c947..1a8bd6c90c92 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -759,8 +760,15 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     // create but before close.  If we don't successfully close the file,
     // subsequent region reopens will fail the below because create is
     // registered in NN.
+
+    // first check to get the permissions
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf,
+        HConstants.DATA_FILE_UMASK_KEY);
+
+    // and then create the file
     Path tmpPath = new Path(getTmpDir(), REGIONINFO_FILE);
-    FSDataOutputStream out = this.fs.create(tmpPath, true);
+    FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
+
     try {
       this.regionInfo.write(out);
       out.write('\n');
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 7dfbc15a8a69..db9ed9326ca7 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -57,6 +58,9 @@
 public abstract class FSUtils {
   private static final Log LOG = LogFactory.getLog(FSUtils.class);
 
+  /** Full access permissions (starting point for a umask) */
+  private static final String FULL_RWX_PERMISSIONS = "777";
+
   protected FSUtils() {
     super();
   }
@@ -101,21 +105,91 @@ public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
   }
 
   /**
-   * Create file.
-   * @param fs filesystem object
-   * @param p path to create
-   * @return Path
-   * @throws IOException e
+   * Create the specified file on the filesystem. By default, this will:
+   * <ol>
+   * <li>overwrite the file if it exists</li>
+   * <li>apply the umask in the configuration (if it is enabled)</li>
+   * <li>use the fs configured buffer size (or {@value DEFAULT_BUFFER_SIZE} if
+   * not set)</li>
+   * <li>use the default replication</li>
+   * <li>use the default block size</li>
+   * <li>not track progress</li>
+   * </ol>
+   * 
+   * @param fs {@link FileSystem} on which to write the file
+   * @param path {@link Path} to the file to write
+   * @return output stream to the created file
+   * @throws IOException if the file cannot be created
+   */
+  public static FSDataOutputStream create(FileSystem fs, Path path,
+      FsPermission perm) throws IOException {
+    return create(fs, path, perm, true);
+  }
+
+  /**
+   * Create the specified file on the filesystem. By default, this will:
+   * <ol>
+   * <li>apply the umask in the configuration (if it is enabled)</li>
+   * <li>use the fs configured buffer size (or {@value DEFAULT_BUFFER_SIZE} if
+   * not set)</li>
+   * <li>use the default replication</li>
+   * <li>use the default block size</li>
+   * <li>not track progress</li>
+   * </ol>
+   * 
+   * @param fs {@link FileSystem} on which to write the file
+   * @param path {@link Path} to the file to write
+   * @param perm
+   * @param overwrite Whether or not the created file should be overwritten.
+   * @return output stream to the created file
+   * @throws IOException if the file cannot be created
    */
-  public static Path create(final FileSystem fs, final Path p)
-  throws IOException {
-    if (fs.exists(p)) {
-      throw new IOException("File already exists " + p.toString());
-    }
-    if (!fs.createNewFile(p)) {
-      throw new IOException("Failed create of " + p);
+  public static FSDataOutputStream create(FileSystem fs, Path path,
+      FsPermission perm, boolean overwrite) throws IOException {
+    LOG.debug("Creating file:" + path + "with permission:" + perm);
+
+    return fs.create(path, perm, overwrite,
+        fs.getConf().getInt("io.file.buffer.size", 4096),
+        fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+  }
+
+  /**
+   * Get the file permissions specified in the configuration, if they are
+   * enabled.
+   * 
+   * @param fs filesystem that the file will be created on.
+   * @param conf configuration to read for determining if permissions are
+   *          enabled and which to use
+   * @param permssionConfKey property key in the configuration to use when
+   *          finding the permission
+   * @return the permission to use when creating a new file on the fs. If
+   *         special permissions are not specified in the configuration, then
+   *         the default permissions on the the fs will be returned.
+   */
+  public static FsPermission getFilePermissions(final FileSystem fs,
+      final Configuration conf, final String permssionConfKey) {
+    boolean enablePermissions = conf.getBoolean(
+        HConstants.ENABLE_DATA_FILE_UMASK, false);
+
+    if (enablePermissions) {
+      try {
+        FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
+        // make sure that we have a mask, if not, go default.
+        String mask = conf.get(permssionConfKey);
+        if (mask == null)
+          return FsPermission.getDefault();
+        // appy the umask
+        FsPermission umask = new FsPermission(mask);
+        return perm.applyUMask(umask);
+      } catch (IllegalArgumentException e) {
+        LOG.warn(
+            "Incorrect umask attempted to be created: "
+                + conf.get(permssionConfKey)
+                + ", using default file permissions.", e);
+        return FsPermission.getDefault();
+      }
     }
-    return p;
+    return FsPermission.getDefault();
   }
 
   /**
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 9277e0c295fe..e15841bc3e4f 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -869,4 +869,18 @@
      value to 0.
      </description>
    </property>
+   <property>
+    <name>hbase.data.umask.enable</name>
+    <value>false</value>
+    <description>Enable, if true, that file permissions should be assigned
+      to the files written by the regionserver
+    </description>
+  </property>
+  <property>
+    <name>hbase.data.umask</name>
+    <value>000</value>
+    <description>File permissions that should be used to write data
+      files when hbase.data.umask.enable is true
+    </description>
+  </property>
 </configuration>
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
index e2611e607d23..96693906dc91 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -19,14 +19,24 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-import org.apache.hadoop.hbase.*;
+import java.io.File;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -139,13 +149,42 @@ private void WriteDataToHDFS(FileSystem fs, Path file, int dataSize)
       FileStatus status = fs.getFileStatus(testFile);
       HDFSBlocksDistribution blocksDistribution =
         FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
-      assertTrue(blocksDistribution.getTopHosts().size() == 3);
+      assertEquals("Wrong number of hosts distributing blocks.", 3,
+          blocksDistribution.getTopHosts().size());
     } finally {
       htu.shutdownMiniDFSCluster();
     }
-    
   }
-  
+
+  @Test
+  public void testPermMask() throws Exception {
+
+    Configuration conf = HBaseConfiguration.create();
+    conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true);
+    FileSystem fs = FileSystem.get(conf);
+    // first check that we don't crash if we don't have perms set
+    FsPermission defaultPerms = FSUtils.getFilePermissions(fs, conf,
+        HConstants.DATA_FILE_UMASK_KEY);
+    assertEquals(FsPermission.getDefault(), defaultPerms);
+
+    conf.setStrings(HConstants.DATA_FILE_UMASK_KEY, "077");
+    // now check that we get the right perms
+    FsPermission filePerm = FSUtils.getFilePermissions(fs, conf,
+        HConstants.DATA_FILE_UMASK_KEY);
+    assertEquals(new FsPermission("700"), filePerm);
+
+    // then that the correct file is created
+    Path p = new Path("target" + File.separator + UUID.randomUUID().toString());
+    try {
+      FSDataOutputStream out = FSUtils.create(fs, p, filePerm);
+      out.close();
+      FileStatus stat = fs.getFileStatus(p);
+      assertEquals(new FsPermission("700"), stat.getPermission());
+      // and then cleanup
+    } finally {
+      fs.delete(p, true);
+    }
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 49becaba090bf50378300e144f27f707de15b4d5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 9 Mar 2012 00:31:38 +0000
Subject: [PATCH 0027/1540] HBASE-5074 Support checksums in HBase block cache
 (Dhruba)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298666 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  29 +
 .../apache/hadoop/hbase/fs/HFileSystem.java   | 177 ++++
 .../hbase/io/hfile/AbstractHFileReader.java   |  21 +-
 .../hadoop/hbase/io/hfile/ChecksumUtil.java   | 233 +++++
 .../hbase/io/hfile/FixedFileTrailer.java      | 147 +++-
 .../apache/hadoop/hbase/io/hfile/HFile.java   |  79 +-
 .../hadoop/hbase/io/hfile/HFileBlock.java     | 782 +++++++++++++----
 .../hbase/io/hfile/HFileDataBlockEncoder.java |   3 +-
 .../io/hfile/HFileDataBlockEncoderImpl.java   |  18 +-
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |   4 +-
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  52 +-
 .../hadoop/hbase/io/hfile/HFileWriterV1.java  |  19 +-
 .../hadoop/hbase/io/hfile/HFileWriterV2.java  |  20 +-
 .../hbase/io/hfile/NoOpDataBlockEncoder.java  |   2 +-
 .../mapreduce/LoadIncrementalHFiles.java      |   3 +
 .../hadoop/hbase/regionserver/HRegion.java    |   9 +-
 .../hbase/regionserver/HRegionServer.java     |  15 +-
 .../regionserver/RegionServerServices.java    |   7 +-
 .../hadoop/hbase/regionserver/Store.java      |  40 +-
 .../hadoop/hbase/regionserver/StoreFile.java  |  41 +-
 .../metrics/RegionServerMetrics.java          |   9 +
 .../hadoop/hbase/util/ChecksumFactory.java    |  99 +++
 .../hadoop/hbase/util/ChecksumType.java       | 183 ++++
 .../hbase/util/CompoundBloomFilter.java       |   2 +-
 .../hadoop/hbase/HBaseTestingUtility.java     |   7 +-
 .../hadoop/hbase/io/hfile/CacheTestUtils.java |   5 +-
 .../hbase/io/hfile/TestCacheOnWrite.java      |   8 +-
 .../hadoop/hbase/io/hfile/TestChecksum.java   | 290 +++++++
 .../hbase/io/hfile/TestFixedFileTrailer.java  |  10 +-
 .../hadoop/hbase/io/hfile/TestHFileBlock.java | 107 ++-
 .../io/hfile/TestHFileBlockCompatibility.java | 806 ++++++++++++++++++
 .../hbase/io/hfile/TestHFileBlockIndex.java   |   6 +-
 .../io/hfile/TestHFileDataBlockEncoder.java   |  10 +-
 .../hbase/io/hfile/TestHFileReaderV1.java     |   2 +-
 .../hbase/io/hfile/TestHFileWriterV2.java     |   2 +-
 .../regionserver/CreateRandomStoreFile.java   |   2 +
 .../regionserver/HFileReadWriteTest.java      |   2 +
 .../regionserver/TestCompoundBloomFilter.java |   2 +
 .../regionserver/TestFSErrorsExposed.java     |  31 +-
 .../hbase/regionserver/TestHRegion.java       |   6 +-
 .../hbase/regionserver/TestStoreFile.java     |  18 +
 .../handler/TestCloseRegionHandler.java       |   6 +-
 .../hbase/util/MockRegionServerServices.java  |  13 +-
 43 files changed, 3020 insertions(+), 307 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index d1a4cc15f33a..cb4832538da0 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -607,6 +607,35 @@ public static enum Modify {
   /** File permission umask to use when creating hbase data files */
   public static final String DATA_FILE_UMASK_KEY = "hbase.data.umask";
 
+  /** 
+   * If this parameter is set to true, then hbase will read
+   * data and then verify checksums. Checksum verification 
+   * inside hdfs will be switched off.  However, if the hbase-checksum 
+   * verification fails, then it will switch back to using
+   * hdfs checksums for verifiying data that is being read from storage.
+   *
+   * If this parameter is set to false, then hbase will not
+   * verify any checksums, instead it will depend on checksum verification
+   * being done in the hdfs client.
+   */
+  public static final String HBASE_CHECKSUM_VERIFICATION = 
+      "hbase.regionserver.checksum.verify";
+
+  /**
+   * The name of the configuration parameter that specifies
+   * the number of bytes in a newly created checksum chunk.
+   */
+  public static final String BYTES_PER_CHECKSUM =
+      "hbase.hstore.bytes.per.checksum";
+
+  /**
+   * The name of the configuration parameter that specifies
+   * the name of an algorithm that is used to compute checksums
+   * for newly created blocks.
+   */
+  public static final String CHECKSUM_TYPE_NAME =
+      "hbase.hstore.checksum.algorithm";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
new file mode 100644
index 000000000000..d6a47053d3c8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * An encapsulation for the FileSystem object that hbase uses to access
+ * data. This class allows the flexibility of using  
+ * separate filesystem objects for reading and writing hfiles and hlogs.
+ * In future, if we want to make hlogs be in a different filesystem,
+ * this is the place to make it happen.
+ */
+public class HFileSystem extends FilterFileSystem {
+
+  private final FileSystem noChecksumFs;   // read hfile data from storage
+  private final boolean useHBaseChecksum;
+
+  /**
+   * Create a FileSystem object for HBase regionservers.
+   * @param conf The configuration to be used for the filesystem
+   * @param useHBaseChecksums if true, then use
+   *        checksum verfication in hbase, otherwise
+   *        delegate checksum verification to the FileSystem.
+   */
+  public HFileSystem(Configuration conf, boolean useHBaseChecksum)
+    throws IOException {
+
+    // Create the default filesystem with checksum verification switched on.
+    // By default, any operation to this FilterFileSystem occurs on
+    // the underlying filesystem that has checksums switched on.
+    this.fs = FileSystem.get(conf);
+    this.useHBaseChecksum = useHBaseChecksum;
+    
+    fs.initialize(getDefaultUri(conf), conf);
+
+    // If hbase checksum verification is switched on, then create a new
+    // filesystem object that has cksum verification turned off.
+    // We will avoid verifying checksums in the fs client, instead do it
+    // inside of hbase.
+    if (useHBaseChecksum) {
+      this.noChecksumFs = newInstanceFileSystem(conf);
+      this.noChecksumFs.setVerifyChecksum(false);
+    } else {
+      this.noChecksumFs = fs;
+    }
+  }
+
+  /**
+   * Wrap a FileSystem object within a HFileSystem. The noChecksumFs and
+   * writefs are both set to be the same specified fs. 
+   * Do not verify hbase-checksums while reading data from filesystem.
+   * @param fs Set the noChecksumFs and writeFs to this specified filesystem.
+   */
+  public HFileSystem(FileSystem fs) {
+    this.fs = fs;
+    this.noChecksumFs = fs;
+    this.useHBaseChecksum = false;
+  }
+
+  /**
+   * Returns the filesystem that is specially setup for 
+   * doing reads from storage. This object avoids doing 
+   * checksum verifications for reads.
+   * @return The FileSystem object that can be used to read data
+   *         from files.
+   */
+  public FileSystem getNoChecksumFs() {
+    return noChecksumFs;
+  }
+
+  /**
+   * Returns the underlying filesystem
+   * @return The underlying FileSystem for this FilterFileSystem object.
+   */
+  public FileSystem getBackingFs() throws IOException {
+    return fs;
+  }
+
+  /**
+   * Are we verifying checksums in HBase?
+   * @return True, if hbase is configured to verify checksums,
+   *         otherwise false.
+   */
+  public boolean useHBaseChecksum() {
+    return useHBaseChecksum;
+  }
+
+  /**
+   * Close this filesystem object
+   */
+  @Override
+  public void close() throws IOException {
+    super.close();
+    if (this.noChecksumFs != fs) {
+      this.noChecksumFs.close();
+    }
+  }
+
+ /**
+   * Returns a brand new instance of the FileSystem. It does not use
+   * the FileSystem.Cache. In newer versions of HDFS, we can directly
+   * invoke FileSystem.newInstance(Configuration).
+   * 
+   * @param conf Configuration
+   * @return A new instance of the filesystem
+   */
+  private static FileSystem newInstanceFileSystem(Configuration conf)
+    throws IOException {
+    URI uri = FileSystem.getDefaultUri(conf);
+    Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
+    if (clazz == null) {
+      throw new IOException("No FileSystem for scheme: " + uri.getScheme());
+    }
+    FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
+    fs.initialize(uri, conf);
+    return fs;
+  }
+
+  /**
+   * Create a new HFileSystem object, similar to FileSystem.get().
+   * This returns a filesystem object that avoids checksum
+   * verification in the filesystem for hfileblock-reads.
+   * For these blocks, checksum verification is done by HBase.
+   */
+  static public FileSystem get(Configuration conf) throws IOException {
+    return new HFileSystem(conf, true);
+  }
+
+  /**
+   * Wrap a LocalFileSystem within a HFileSystem.
+   */
+  static public FileSystem getLocalFs(Configuration conf) throws IOException {
+    return new HFileSystem(FileSystem.getLocal(conf));
+  }
+
+  /**
+   * The org.apache.hadoop.fs.FilterFileSystem does not yet support 
+   * createNonRecursive. This is a hadoop bug and when it is fixed in Hadoop,
+   * this definition will go away.
+   */
+  public FSDataOutputStream createNonRecursive(Path f,
+      boolean overwrite,
+      int bufferSize, short replication, long blockSize,
+      Progressable progress) throws IOException {
+    return fs.createNonRecursive(f, overwrite, bufferSize, replication,
+                                 blockSize, progress);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
index 3d206d7c1079..f418f905c98b 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
@@ -40,9 +41,13 @@ public abstract class AbstractHFileReader extends SchemaConfigured
   /** Filesystem-level block reader for this HFile format version. */
   protected HFileBlock.FSReader fsBlockReader;
 
-  /** Stream to read from. */
+  /** Stream to read from. Does checksum verifications in file system */
   protected FSDataInputStream istream;
 
+  /** The file system stream of the underlying {@link HFile} that
+   * does not do checksum verification in the file system */
+  protected FSDataInputStream istreamNoFsChecksum;
+
   /**
    * True if we should close the input stream when done. We don't close it if we
    * didn't open it.
@@ -97,10 +102,21 @@ public abstract class AbstractHFileReader extends SchemaConfigured
 
   protected FileInfo fileInfo;
 
+  /** The filesystem used for accesing data */
+  protected HFileSystem hfs;
+
   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
       final FSDataInputStream fsdis, final long fileSize,
       final boolean closeIStream,
       final CacheConfig cacheConf) {
+    this(path, trailer, fsdis, fsdis, fileSize, closeIStream, cacheConf, null);
+  }
+
+  protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
+      final FSDataInputStream fsdis, final FSDataInputStream fsdisNoFsChecksum,
+      final long fileSize,
+      final boolean closeIStream,
+      final CacheConfig cacheConf, final HFileSystem hfs) {
     super(null, path);
     this.trailer = trailer;
     this.compressAlgo = trailer.getCompressionCodec();
@@ -110,6 +126,8 @@ protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
     this.closeIStream = closeIStream;
     this.path = path;
     this.name = path.getName();
+    this.hfs = hfs;
+    this.istreamNoFsChecksum = fsdisNoFsChecksum;
   }
 
   @SuppressWarnings("serial")
@@ -341,5 +359,4 @@ public Path getPath() {
   public DataBlockEncoding getEncodingOnDisk() {
     return dataBlockEncoder.getEncodingOnDisk();
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
new file mode 100644
index 000000000000..4ef1be71998b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+import org.apache.hadoop.hbase.util.ChecksumType;
+
+/**
+ * Utility methods to compute and validate checksums.
+ */
+public class ChecksumUtil {
+
+  /** This is used to reserve space in a byte buffer */
+  private static byte[] DUMMY_VALUE = new byte[128 * HFileBlock.CHECKSUM_SIZE];
+
+  /** 
+   * This is used by unit tests to make checksum failures throw an 
+   * exception instead of returning null. Returning a null value from 
+   * checksum validation will cause the higher layer to retry that 
+   * read with hdfs-level checksums. Instead, we would like checksum 
+   * failures to cause the entire unit test to fail.
+   */
+  private static boolean generateExceptions = false;
+
+  /**
+   * Generates a checksum for all the data in indata. The checksum is
+   * written to outdata.
+   * @param indata input data stream
+   * @param startOffset starting offset in the indata stream from where to
+   *                    compute checkums from
+   * @param endOffset ending offset in the indata stream upto
+   *                   which checksums needs to be computed
+   * @param outData the output buffer where checksum values are written
+   * @param outOffset the starting offset in the outdata where the
+   *                  checksum values are written
+   * @param checksumType type of checksum
+   * @param bytesPerChecksum number of bytes per checksum value
+   */
+  static void generateChecksums(byte[] indata,
+    int startOffset, int endOffset, 
+    byte[] outdata, int outOffset,
+    ChecksumType checksumType,
+    int bytesPerChecksum) throws IOException {
+
+    if (checksumType == ChecksumType.NULL) {
+      return; // No checkums for this block.
+    }
+
+    Checksum checksum = checksumType.getChecksumObject();
+    int bytesLeft = endOffset - startOffset;
+    int chunkNum = 0;
+
+    while (bytesLeft > 0) {
+      // generate the checksum for one chunk
+      checksum.reset();
+      int count = Math.min(bytesLeft, bytesPerChecksum);
+      checksum.update(indata, startOffset, count);
+
+      // write the checksum value to the output buffer.
+      int cksumValue = (int)checksum.getValue();
+      outOffset = Bytes.putInt(outdata, outOffset, cksumValue);
+      chunkNum++;
+      startOffset += count;
+      bytesLeft -= count;
+    }
+  }
+
+  /**
+   * Validates that the data in the specified HFileBlock matches the
+   * checksum.  Generates the checksum for the data and
+   * then validate that it matches the value stored in the header.
+   * If there is a checksum mismatch, then return false. Otherwise
+   * return true.
+   * The header is extracted from the specified HFileBlock while the
+   * data-to-be-verified is extracted from 'data'.
+   */
+  static boolean validateBlockChecksum(Path path, HFileBlock block, 
+    byte[] data, int hdrSize) throws IOException {
+
+    // If this is an older version of the block that does not have
+    // checksums, then return false indicating that checksum verification
+    // did not succeed. Actually, this methiod should never be called
+    // when the minorVersion is 0, thus this is a defensive check for a
+    // cannot-happen case. Since this is a cannot-happen case, it is
+    // better to return false to indicate a checksum validation failure.
+    if (block.getMinorVersion() < HFileBlock.MINOR_VERSION_WITH_CHECKSUM) {
+      return false;
+    }
+
+    // Get a checksum object based on the type of checksum that is
+    // set in the HFileBlock header. A ChecksumType.NULL indicates that 
+    // the caller is not interested in validating checksums, so we
+    // always return true.
+    ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType());
+    if (cktype == ChecksumType.NULL) {
+      return true; // No checkums validations needed for this block.
+    }
+    Checksum checksumObject = cktype.getChecksumObject();
+    checksumObject.reset();
+
+    // read in the stored value of the checksum size from the header.
+    int bytesPerChecksum = block.getBytesPerChecksum();
+
+    // bytesPerChecksum is always larger than the size of the header
+    if (bytesPerChecksum < hdrSize) {
+      String msg = "Unsupported value of bytesPerChecksum. " +
+                   " Minimum is " + hdrSize + 
+                   " but the configured value is " + bytesPerChecksum;
+      HFile.LOG.warn(msg);
+      return false;   // cannot happen case, unable to verify checksum
+    }
+    // Extract the header and compute checksum for the header.
+    ByteBuffer hdr = block.getBufferWithHeader();
+    checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize);
+
+    int off = hdrSize;
+    int consumed = hdrSize;
+    int bytesLeft = block.getOnDiskDataSizeWithHeader() - off;
+    int cksumOffset = block.getOnDiskDataSizeWithHeader();
+    
+    // validate each chunk
+    while (bytesLeft > 0) {
+      int thisChunkSize = bytesPerChecksum - consumed;
+      int count = Math.min(bytesLeft, thisChunkSize);
+      checksumObject.update(data, off, count);
+
+      int storedChecksum = Bytes.toInt(data, cksumOffset);
+      if (storedChecksum != (int)checksumObject.getValue()) {
+        String msg = "File " + path +
+                     " Stored checksum value of " + storedChecksum +
+                     " at offset " + cksumOffset +
+                     " does not match computed checksum " +
+                     checksumObject.getValue() +
+                     ", total data size " + data.length +
+                     " Checksum data range offset " + off + " len " + count +
+                     HFileBlock.toStringHeader(block.getBufferReadOnly());
+        HFile.LOG.warn(msg);
+        if (generateExceptions) {
+          throw new IOException(msg); // this is only for unit tests
+        } else {
+          return false;               // checksum validation failure
+        }
+      }
+      cksumOffset += HFileBlock.CHECKSUM_SIZE;
+      bytesLeft -= count; 
+      off += count;
+      consumed = 0;
+      checksumObject.reset();
+    }
+    return true; // checksum is valid
+  }
+
+  /**
+   * Returns the number of bytes needed to store the checksums for
+   * a specified data size
+   * @param datasize number of bytes of data
+   * @param bytesPerChecksum number of bytes in a checksum chunk
+   * @return The number of bytes needed to store the checksum values
+   */
+  static long numBytes(long datasize, int bytesPerChecksum) {
+    return numChunks(datasize, bytesPerChecksum) * 
+                     HFileBlock.CHECKSUM_SIZE;
+  }
+
+  /**
+   * Returns the number of checksum chunks needed to store the checksums for
+   * a specified data size
+   * @param datasize number of bytes of data
+   * @param bytesPerChecksum number of bytes in a checksum chunk
+   * @return The number of checksum chunks
+   */
+  static long numChunks(long datasize, int bytesPerChecksum) {
+    long numChunks = datasize/bytesPerChecksum;
+    if (datasize % bytesPerChecksum != 0) {
+      numChunks++;
+    }
+    return numChunks;
+  }
+
+  /**
+   * Write dummy checksums to the end of the specified bytes array
+   * to reserve space for writing checksums later
+   * @param baos OutputStream to write dummy checkum values
+   * @param numBytes Number of bytes of data for which dummy checksums
+   *                 need to be generated
+   * @param bytesPerChecksum Number of bytes per checksum value
+   */
+  static void reserveSpaceForChecksums(ByteArrayOutputStream baos,
+    int numBytes, int bytesPerChecksum) throws IOException {
+    long numChunks = numChunks(numBytes, bytesPerChecksum);
+    long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE;
+    while (bytesLeft > 0) {
+      long count = Math.min(bytesLeft, DUMMY_VALUE.length);
+      baos.write(DUMMY_VALUE, 0, (int)count);
+      bytesLeft -= count;
+    }
+  }
+
+  /**
+   * Mechanism to throw an exception in case of hbase checksum
+   * failure. This is used by unit tests only.
+   * @param value Setting this to true will cause hbase checksum
+   *              verification failures to generate exceptions.
+   */
+  public static void generateExceptionForChecksumFailureForTest(boolean value) {
+    generateExceptions = value;
+  }
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
index a3a3b919faa0..8e3c8219eb95 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
@@ -43,6 +43,13 @@
  * variable parts of the file. Also includes basic metadata on this file. The
  * trailer size is fixed within a given {@link HFile} format version only, but
  * we always store the version number as the last four-byte integer of the file.
+ * The version number itself is split into two portions, a major 
+ * version and a minor version. 
+ * The last three bytes of a file is the major
+ * version and a single preceding byte is the minor number. The major version
+ * determines which readers/writers to use to read/write a hfile while a minor
+ * version determines smaller changes in hfile format that do not need a new
+ * reader/writer type.
  */
 public class FixedFileTrailer {
 
@@ -106,12 +113,16 @@ public class FixedFileTrailer {
   /** Raw key comparator class name in version 2 */
   private String comparatorClassName = RawComparator.class.getName();
 
-  /** The {@link HFile} format version. */
-  private final int version;
+  /** The {@link HFile} format major version. */
+  private final int majorVersion;
 
-  FixedFileTrailer(int version) {
-    this.version = version;
-    HFile.checkFormatVersion(version);
+  /** The {@link HFile} format minor version. */
+  private final int minorVersion;
+
+  FixedFileTrailer(int majorVersion, int minorVersion) {
+    this.majorVersion = majorVersion;
+    this.minorVersion = minorVersion;
+    HFile.checkFormatVersion(majorVersion);
   }
 
   private static int[] computeTrailerSizeByVersion() {
@@ -119,7 +130,8 @@ private static int[] computeTrailerSizeByVersion() {
     for (int version = MIN_FORMAT_VERSION;
          version <= MAX_FORMAT_VERSION;
          ++version) {
-      FixedFileTrailer fft = new FixedFileTrailer(version);
+      FixedFileTrailer fft = new FixedFileTrailer(version, 
+                                   HFileBlock.MINOR_VERSION_NO_CHECKSUM);
       DataOutputStream dos = new DataOutputStream(new NullOutputStream());
       try {
         fft.serialize(dos);
@@ -149,7 +161,7 @@ static int getTrailerSize(int version) {
   }
 
   public int getTrailerSize() {
-    return getTrailerSize(version);
+    return getTrailerSize(majorVersion);
   }
 
   /**
@@ -161,7 +173,7 @@ public int getTrailerSize() {
    * @throws IOException
    */
   void serialize(DataOutputStream outputStream) throws IOException {
-    HFile.checkFormatVersion(version);
+    HFile.checkFormatVersion(majorVersion);
 
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutput baosDos = new DataOutputStream(baos);
@@ -171,7 +183,7 @@ void serialize(DataOutputStream outputStream) throws IOException {
     baosDos.writeLong(loadOnOpenDataOffset);
     baosDos.writeInt(dataIndexCount);
 
-    if (version == 1) {
+    if (majorVersion == 1) {
       // This used to be metaIndexOffset, but it was not used in version 1.
       baosDos.writeLong(0);
     } else {
@@ -180,7 +192,7 @@ void serialize(DataOutputStream outputStream) throws IOException {
 
     baosDos.writeInt(metaIndexCount);
     baosDos.writeLong(totalUncompressedBytes);
-    if (version == 1) {
+    if (majorVersion == 1) {
       baosDos.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
     } else {
       // This field is long from version 2 onwards.
@@ -188,14 +200,16 @@ void serialize(DataOutputStream outputStream) throws IOException {
     }
     baosDos.writeInt(compressionCodec.ordinal());
 
-    if (version > 1) {
+    if (majorVersion > 1) {
       baosDos.writeInt(numDataIndexLevels);
       baosDos.writeLong(firstDataBlockOffset);
       baosDos.writeLong(lastDataBlockOffset);
       Bytes.writeStringFixedSize(baosDos, comparatorClassName,
           MAX_COMPARATOR_NAME_LENGTH);
     }
-    baosDos.writeInt(version);
+
+    // serialize the major and minor versions
+    baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
 
     outputStream.write(baos.toByteArray());
   }
@@ -210,7 +224,7 @@ void serialize(DataOutputStream outputStream) throws IOException {
    * @throws IOException
    */
   void deserialize(DataInputStream inputStream) throws IOException {
-    HFile.checkFormatVersion(version);
+    HFile.checkFormatVersion(majorVersion);
 
     BlockType.TRAILER.readAndCheck(inputStream);
 
@@ -218,7 +232,7 @@ void deserialize(DataInputStream inputStream) throws IOException {
     loadOnOpenDataOffset = inputStream.readLong();
     dataIndexCount = inputStream.readInt();
 
-    if (version == 1) {
+    if (majorVersion == 1) {
       inputStream.readLong(); // Read and skip metaIndexOffset.
     } else {
       uncompressedDataIndexSize = inputStream.readLong();
@@ -226,9 +240,9 @@ void deserialize(DataInputStream inputStream) throws IOException {
     metaIndexCount = inputStream.readInt();
 
     totalUncompressedBytes = inputStream.readLong();
-    entryCount = version == 1 ? inputStream.readInt() : inputStream.readLong();
+    entryCount = majorVersion == 1 ? inputStream.readInt() : inputStream.readLong();
     compressionCodec = Compression.Algorithm.values()[inputStream.readInt()];
-    if (version > 1) {
+    if (majorVersion > 1) {
       numDataIndexLevels = inputStream.readInt();
       firstDataBlockOffset = inputStream.readLong();
       lastDataBlockOffset = inputStream.readLong();
@@ -236,7 +250,9 @@ void deserialize(DataInputStream inputStream) throws IOException {
           Bytes.readStringFixedSize(inputStream, MAX_COMPARATOR_NAME_LENGTH);
     }
 
-    expectVersion(inputStream.readInt());
+    int version = inputStream.readInt();
+    expectMajorVersion(extractMajorVersion(version));
+    expectMinorVersion(extractMinorVersion(version));
   }
 
   private void append(StringBuilder sb, String s) {
@@ -255,14 +271,15 @@ public String toString() {
     append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
     append(sb, "entryCount=" + entryCount);
     append(sb, "compressionCodec=" + compressionCodec);
-    if (version == 2) {
+    if (majorVersion == 2) {
       append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
       append(sb, "numDataIndexLevels=" + numDataIndexLevels);
       append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
       append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
       append(sb, "comparatorClassName=" + comparatorClassName);
     }
-    append(sb, "version=" + version);
+    append(sb, "majorVersion=" + majorVersion);
+    append(sb, "minorVersion=" + minorVersion);
 
     return sb.toString();
   }
@@ -299,31 +316,44 @@ public static FixedFileTrailer readFromStream(FSDataInputStream istream,
     buf.position(buf.limit() - Bytes.SIZEOF_INT);
     int version = buf.getInt();
 
+    // Extract the major and minor versions.
+    int majorVersion = extractMajorVersion(version);
+    int minorVersion = extractMinorVersion(version);
+
     try {
-      HFile.checkFormatVersion(version);
+      HFile.checkFormatVersion(majorVersion);
     } catch (IllegalArgumentException iae) {
       // In this context, an invalid version might indicate a corrupt HFile.
       throw new IOException(iae);
     }
 
-    int trailerSize = getTrailerSize(version);
+    int trailerSize = getTrailerSize(majorVersion);
 
-    FixedFileTrailer fft = new FixedFileTrailer(version);
+    FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
     fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
         buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
     return fft;
   }
 
-  public void expectVersion(int expected) {
-    if (version != expected) {
-      throw new IllegalArgumentException("Invalid HFile version: " + version
+  public void expectMajorVersion(int expected) {
+    if (majorVersion != expected) {
+      throw new IllegalArgumentException("Invalid HFile major version: "
+          + majorVersion 
           + " (expected: " + expected + ")");
     }
   }
 
-  public void expectAtLeastVersion(int lowerBound) {
-    if (version < lowerBound) {
-      throw new IllegalArgumentException("Invalid HFile version: " + version
+  public void expectMinorVersion(int expected) {
+    if (minorVersion != expected) {
+      throw new IllegalArgumentException("Invalid HFile minor version: "
+          + minorVersion + " (expected: " + expected + ")");
+    }
+  }
+
+  public void expectAtLeastMajorVersion(int lowerBound) {
+    if (majorVersion < lowerBound) {
+      throw new IllegalArgumentException("Invalid HFile major version: "
+          + majorVersion
           + " (expected: " + lowerBound + " or higher).");
     }
   }
@@ -373,11 +403,11 @@ public long getEntryCount() {
   }
 
   public void setEntryCount(long newEntryCount) {
-    if (version == 1) {
+    if (majorVersion == 1) {
       int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
       if (intEntryCount != newEntryCount) {
         LOG.info("Warning: entry count is " + newEntryCount + " but writing "
-            + intEntryCount + " into the version " + version + " trailer");
+            + intEntryCount + " into the version " + majorVersion + " trailer");
       }
       entryCount = intEntryCount;
       return;
@@ -394,42 +424,52 @@ public void setCompressionCodec(Compression.Algorithm compressionCodec) {
   }
 
   public int getNumDataIndexLevels() {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     return numDataIndexLevels;
   }
 
   public void setNumDataIndexLevels(int numDataIndexLevels) {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     this.numDataIndexLevels = numDataIndexLevels;
   }
 
   public long getLastDataBlockOffset() {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     return lastDataBlockOffset;
   }
 
   public void setLastDataBlockOffset(long lastDataBlockOffset) {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     this.lastDataBlockOffset = lastDataBlockOffset;
   }
 
   public long getFirstDataBlockOffset() {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     return firstDataBlockOffset;
   }
 
   public void setFirstDataBlockOffset(long firstDataBlockOffset) {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     this.firstDataBlockOffset = firstDataBlockOffset;
   }
 
-  public int getVersion() {
-    return version;
+  /**
+   * Returns the major version of this HFile format
+   */
+  public int getMajorVersion() {
+    return majorVersion;
+  }
+
+  /**
+   * Returns the minor version of this HFile format
+   */
+  int getMinorVersion() {
+    return minorVersion;
   }
 
   @SuppressWarnings("rawtypes")
   public void setComparatorClass(Class<? extends RawComparator> klass) {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     comparatorClassName = klass.getName();
   }
 
@@ -456,20 +496,43 @@ public static RawComparator<byte[]> createComparator(
   }
 
   RawComparator<byte[]> createComparator() throws IOException {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     return createComparator(comparatorClassName);
   }
 
   public long getUncompressedDataIndexSize() {
-    if (version == 1)
+    if (majorVersion == 1)
       return 0;
     return uncompressedDataIndexSize;
   }
 
   public void setUncompressedDataIndexSize(
       long uncompressedDataIndexSize) {
-    expectAtLeastVersion(2);
+    expectAtLeastMajorVersion(2);
     this.uncompressedDataIndexSize = uncompressedDataIndexSize;
   }
 
+  /**
+   * Extracts the major version for a 4-byte serialized version data.
+   * The major version is the 3 least significant bytes
+   */
+  private static int extractMajorVersion(int serializedVersion) {
+    return (serializedVersion & 0x00ffffff);
+  }
+
+  /**
+   * Extracts the minor version for a 4-byte serialized version data.
+   * The major version are the 3 the most significant bytes
+   */
+  private static int extractMinorVersion(int serializedVersion) {
+    return (serializedVersion >>> 24);
+  }
+
+  /**
+   * Create a 4 byte serialized version number by combining the
+   * minor and major version numbers.
+   */
+  private static int materializeVersion(int majorVersion, int minorVersion) {
+    return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index e765e77e8bad..31c07a173fcb 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -42,10 +42,12 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.HbaseMapWritable;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -154,6 +156,12 @@ public class HFile {
    */
   public final static int MIN_NUM_HFILE_PATH_LEVELS = 5;
 
+  /**
+   * The number of bytes per checksum.
+   */
+  public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
+  public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32;
+
   // For measuring latency of "sequential" reads and writes
   static final AtomicInteger readOps = new AtomicInteger();
   static final AtomicLong readTimeNano = new AtomicLong();
@@ -164,6 +172,9 @@ public class HFile {
   static final AtomicInteger preadOps = new AtomicInteger();
   static final AtomicLong preadTimeNano = new AtomicLong();
 
+  // For measuring number of checksum failures
+  static final AtomicLong checksumFailures = new AtomicLong();
+
   // for test purpose
   public static volatile AtomicLong dataBlockReadCnt = new AtomicLong(0);
 
@@ -193,6 +204,14 @@ public static final long getWriteTimeMs() {
     return writeTimeNano.getAndSet(0) / 1000000;
   }
 
+  /**
+   * Number of checksum verification failures. It also
+   * clears the counter.
+   */
+  public static final long getChecksumFailuresCount() {
+    return checksumFailures.getAndSet(0);
+  }
+
   /** API required to write an {@link HFile} */
   public interface Writer extends Closeable {
 
@@ -245,6 +264,8 @@ public static abstract class WriterFactory {
         HFile.DEFAULT_COMPRESSION_ALGORITHM;
     protected HFileDataBlockEncoder encoder = NoOpDataBlockEncoder.INSTANCE;
     protected KeyComparator comparator;
+    protected ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+    protected int bytesPerChecksum = DEFAULT_BYTES_PER_CHECKSUM;
 
     WriterFactory(Configuration conf, CacheConfig cacheConf) {
       this.conf = conf;
@@ -294,6 +315,17 @@ public WriterFactory withComparator(KeyComparator comparator) {
       return this;
     }
 
+    public WriterFactory withChecksumType(ChecksumType checksumType) {
+      Preconditions.checkNotNull(checksumType);
+      this.checksumType = checksumType;
+      return this;
+    }
+
+    public WriterFactory withBytesPerChecksum(int bytesPerChecksum) {
+      this.bytesPerChecksum = bytesPerChecksum;
+      return this;
+    }
+
     public Writer create() throws IOException {
       if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
         throw new AssertionError("Please specify exactly one of " +
@@ -303,14 +335,15 @@ public Writer create() throws IOException {
         ostream = AbstractHFileWriter.createOutputStream(conf, fs, path);
       }
       return createWriter(fs, path, ostream, blockSize,
-          compression, encoder, comparator);
+          compression, encoder, comparator, checksumType, bytesPerChecksum);
     }
 
     protected abstract Writer createWriter(FileSystem fs, Path path,
         FSDataOutputStream ostream, int blockSize,
         Compression.Algorithm compress,
         HFileDataBlockEncoder dataBlockEncoder,
-        KeyComparator comparator) throws IOException;
+        KeyComparator comparator, ChecksumType checksumType,
+        int bytesPerChecksum) throws IOException;
   }
 
   /** The configuration key for HFile version to use for new files */
@@ -429,20 +462,22 @@ ByteBuffer getMetaBlock(String metaBlockName,
   }
 
   private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
+      FSDataInputStream fsdisNoFsChecksum,
       long size, boolean closeIStream, CacheConfig cacheConf,
-      DataBlockEncoding preferredEncodingInCache)
+      DataBlockEncoding preferredEncodingInCache, HFileSystem hfs)
       throws IOException {
     FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, size);
-    switch (trailer.getVersion()) {
+    switch (trailer.getMajorVersion()) {
     case 1:
       return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
           cacheConf);
     case 2:
-      return new HFileReaderV2(path, trailer, fsdis, size, closeIStream,
-          cacheConf, preferredEncodingInCache);
+      return new HFileReaderV2(path, trailer, fsdis, fsdisNoFsChecksum,
+          size, closeIStream,
+          cacheConf, preferredEncodingInCache, hfs);
     default:
       throw new IOException("Cannot instantiate reader for HFile version " +
-          trailer.getVersion());
+          trailer.getMajorVersion());
     }
   }
 
@@ -450,9 +485,26 @@ public static Reader createReaderWithEncoding(
       FileSystem fs, Path path, CacheConfig cacheConf,
       DataBlockEncoding preferredEncodingInCache) throws IOException {
     final boolean closeIStream = true;
-    return pickReaderVersion(path, fs.open(path),
+    HFileSystem hfs = null;
+    FSDataInputStream fsdis = fs.open(path);
+    FSDataInputStream fsdisNoFsChecksum = fsdis;
+    // If the fs is not an instance of HFileSystem, then create an 
+    // instance of HFileSystem that wraps over the specified fs.
+    // In this case, we will not be able to avoid checksumming inside
+    // the filesystem.
+    if (!(fs instanceof HFileSystem)) {
+      hfs = new HFileSystem(fs);
+    } else {
+      hfs = (HFileSystem)fs;
+      // open a stream to read data without checksum verification in
+      // the filesystem
+      if (hfs != null) {
+        fsdisNoFsChecksum = hfs.getNoChecksumFs().open(path);
+      }
+    }
+    return pickReaderVersion(path, fsdis, fsdisNoFsChecksum,
         fs.getFileStatus(path).getLen(), closeIStream, cacheConf,
-        preferredEncodingInCache);
+        preferredEncodingInCache, hfs);
   }
 
   public static Reader createReader(
@@ -461,12 +513,15 @@ public static Reader createReader(
         DataBlockEncoding.NONE);
   }
 
-  public static Reader createReaderFromStream(Path path,
+  /**
+   * This factory method is used only by unit tests
+   */
+  static Reader createReaderFromStream(Path path,
       FSDataInputStream fsdis, long size, CacheConfig cacheConf)
       throws IOException {
     final boolean closeIStream = false;
-    return pickReaderVersion(path, fsdis, size, closeIStream, cacheConf,
-        DataBlockEncoding.NONE);
+    return pickReaderVersion(path, fsdis, fsdis, size, closeIStream, cacheConf,
+        DataBlockEncoding.NONE, null);
   }
 
   /*
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index ef4cb9de1219..c99dcbdd7c7b 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -32,11 +32,15 @@
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.regionserver.MemStore;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.CompoundBloomFilter;
 import org.apache.hadoop.hbase.util.Pair;
@@ -65,6 +69,12 @@
  * <li>Uncompressed block size, header not included (4 bytes)
  * <li>The offset of the previous block of the same type (8 bytes). This is
  * used to be able to navigate to the previous block without going to the block
+ * <li>For minorVersions >=1, there is an additional 4 byte field 
+ * bytesPerChecksum that records the number of bytes in a checksum chunk.
+ * <li>For minorVersions >=1, there is a 4 byte value to store the size of
+ * data on disk (excluding the checksums)
+ * <li>For minorVersions >=1, a series of 4 byte checksums, one each for
+ * the number of bytes specified by bytesPerChecksum.
  * index.
  * <li>Compressed data (or uncompressed data if compression is disabled). The
  * compression algorithm is the same for all the blocks in the {@link HFile},
@@ -76,12 +86,32 @@
  */
 public class HFileBlock extends SchemaConfigured implements Cacheable {
 
+  /** Minor versions starting with this number have hbase checksums */
+  static final int MINOR_VERSION_WITH_CHECKSUM = 1;
+
+  /** minor version that does not support checksums */
+  static final int MINOR_VERSION_NO_CHECKSUM = 0;
+
+  /**
+   * On a checksum failure on a Reader, these many suceeding read
+   * requests switch back to using hdfs checksums before auto-reenabling
+   * hbase checksum verification.
+   */
+  static final int CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD = 3;
+
+  /** The size data structures with minor version is 0 */
+  static final int HEADER_SIZE_NO_CHECKSUM = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
+      + Bytes.SIZEOF_LONG;
+
   public static final boolean FILL_HEADER = true;
   public static final boolean DONT_FILL_HEADER = false;
 
-  /** The size of a version 2 {@link HFile} block header */
-  public static final int HEADER_SIZE = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
-      + Bytes.SIZEOF_LONG;
+  /** The size of a version 2 {@link HFile} block header, minor version 1.
+   * There is a 1 byte checksum type, followed by a 4 byte bytesPerChecksum
+   * followed by another 4 byte value to store sizeofDataOnDisk.
+   */
+  static final int HEADER_SIZE = HEADER_SIZE_NO_CHECKSUM + Bytes.SIZEOF_BYTE +
+                                 2 * Bytes.SIZEOF_INT;
 
   /**
    * The size of block header when blockType is {@link BlockType#ENCODED_DATA}.
@@ -91,7 +121,9 @@ public class HFileBlock extends SchemaConfigured implements Cacheable {
       + DataBlockEncoding.ID_SIZE;
 
   /** Just an array of bytes of the right size. */
-  public static final byte[] DUMMY_HEADER = new byte[HEADER_SIZE];
+  static final byte[] DUMMY_HEADER = new byte[HEADER_SIZE];
+  static final byte[] DUMMY_HEADER_NO_CHECKSUM = 
+     new byte[HEADER_SIZE_NO_CHECKSUM];
 
   public static final int BYTE_BUFFER_HEAP_SIZE = (int) ClassSize.estimateBase(
       ByteBuffer.wrap(new byte[0], 0, 0).getClass(), false);
@@ -99,6 +131,11 @@ public class HFileBlock extends SchemaConfigured implements Cacheable {
   static final int EXTRA_SERIALIZATION_SPACE = Bytes.SIZEOF_LONG +
       Bytes.SIZEOF_INT;
 
+  /**
+   * Each checksum value is an integer that can be stored in 4 bytes.
+   */
+  static final int CHECKSUM_SIZE = Bytes.SIZEOF_INT;
+
   private static final CacheableDeserializer<Cacheable> blockDeserializer =
       new CacheableDeserializer<Cacheable>() {
         public HFileBlock deserialize(ByteBuffer buf) throws IOException{
@@ -107,7 +144,8 @@ public HFileBlock deserialize(ByteBuffer buf) throws IOException{
           buf.limit(buf.limit()
               - HFileBlock.EXTRA_SERIALIZATION_SPACE).rewind();
           newByteBuffer.put(buf);
-          HFileBlock ourBuffer = new HFileBlock(newByteBuffer);
+          HFileBlock ourBuffer = new HFileBlock(newByteBuffer, 
+                                   MINOR_VERSION_NO_CHECKSUM);
 
           buf.position(buf.limit());
           buf.limit(buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE);
@@ -118,10 +156,32 @@ public HFileBlock deserialize(ByteBuffer buf) throws IOException{
       };
 
   private BlockType blockType;
+
+  /** Size on disk without the header. It includes checksum data too. */
   private int onDiskSizeWithoutHeader;
+
+  /** Size of pure data. Does not include header or checksums */
   private final int uncompressedSizeWithoutHeader;
+
+  /** The offset of the previous block on disk */
   private final long prevBlockOffset;
+
+  /** The Type of checksum, better to store the byte than an object */
+  private final byte checksumType;
+
+  /** The number of bytes for which a checksum is computed */
+  private final int bytesPerChecksum;
+
+  /** Size on disk of header and data. Does not include checksum data */
+  private final int onDiskDataSizeWithHeader;
+
+  /** The minor version of the hfile. */
+  private final int minorVersion;
+
+  /** The in-memory representation of the hfile block */
   private ByteBuffer buf;
+
+  /** Whether there is a memstore timestamp after every key/value */
   private boolean includesMemstoreTS;
 
   /**
@@ -140,7 +200,7 @@ public HFileBlock deserialize(ByteBuffer buf) throws IOException{
   /**
    * Creates a new {@link HFile} block from the given fields. This constructor
    * is mostly used when the block data has already been read and uncompressed,
-   * and is sitting in a byte buffer.
+   * and is sitting in a byte buffer. 
    *
    * @param blockType the type of this block, see {@link BlockType}
    * @param onDiskSizeWithoutHeader compressed size of the block if compression
@@ -155,10 +215,17 @@ public HFileBlock deserialize(ByteBuffer buf) throws IOException{
    * @param fillHeader true to fill in the first {@link #HEADER_SIZE} bytes of
    *          the buffer based on the header fields provided
    * @param offset the file offset the block was read from
+   * @param minorVersion the minor version of this block
+   * @param bytesPerChecksum the number of bytes per checksum chunk
+   * @param checksumType the checksum algorithm to use
+   * @param onDiskDataSizeWithHeader size of header and data on disk not
+   *        including checksum data
    */
-  public HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,
+  HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,
       int uncompressedSizeWithoutHeader, long prevBlockOffset, ByteBuffer buf,
-      boolean fillHeader, long offset, boolean includesMemstoreTS) {
+      boolean fillHeader, long offset, boolean includesMemstoreTS, 
+      int minorVersion, int bytesPerChecksum, byte checksumType,
+      int onDiskDataSizeWithHeader) {
     this.blockType = blockType;
     this.onDiskSizeWithoutHeader = onDiskSizeWithoutHeader;
     this.uncompressedSizeWithoutHeader = uncompressedSizeWithoutHeader;
@@ -168,20 +235,37 @@ public HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,
       overwriteHeader();
     this.offset = offset;
     this.includesMemstoreTS = includesMemstoreTS;
+    this.minorVersion = minorVersion;
+    this.bytesPerChecksum = bytesPerChecksum;
+    this.checksumType = checksumType;
+    this.onDiskDataSizeWithHeader = onDiskDataSizeWithHeader;
   }
 
   /**
    * Creates a block from an existing buffer starting with a header. Rewinds
    * and takes ownership of the buffer. By definition of rewind, ignores the
    * buffer position, but if you slice the buffer beforehand, it will rewind
-   * to that point.
+   * to that point. The reason this has a minorNumber and not a majorNumber is
+   * because majorNumbers indicate the format of a HFile whereas minorNumbers 
+   * indicate the format inside a HFileBlock.
    */
-  private HFileBlock(ByteBuffer b) throws IOException {
+  HFileBlock(ByteBuffer b, int minorVersion) throws IOException {
     b.rewind();
     blockType = BlockType.read(b);
     onDiskSizeWithoutHeader = b.getInt();
     uncompressedSizeWithoutHeader = b.getInt();
     prevBlockOffset = b.getLong();
+    this.minorVersion = minorVersion;
+    if (minorVersion >= MINOR_VERSION_WITH_CHECKSUM) {
+      this.checksumType = b.get();
+      this.bytesPerChecksum = b.getInt();
+      this.onDiskDataSizeWithHeader = b.getInt();
+    } else {
+      this.checksumType = ChecksumType.NULL.getCode();
+      this.bytesPerChecksum = 0;
+      this.onDiskDataSizeWithHeader = onDiskSizeWithoutHeader +
+                                       HEADER_SIZE_NO_CHECKSUM;
+    }
     buf = b;
     buf.rewind();
   }
@@ -196,25 +280,26 @@ public short getDataBlockEncodingId() {
       throw new IllegalArgumentException("Querying encoder ID of a block " +
           "of type other than " + BlockType.ENCODED_DATA + ": " + blockType);
     }
-    return buf.getShort(HEADER_SIZE);
+    return buf.getShort(headerSize());
   }
 
   /**
-   * @return the on-disk size of the block with header size included
+   * @return the on-disk size of the block with header size included. This
+   * includes the header, the data and the checksum data.
    */
-  public int getOnDiskSizeWithHeader() {
-    return onDiskSizeWithoutHeader + HEADER_SIZE;
+  int getOnDiskSizeWithHeader() {
+    return onDiskSizeWithoutHeader + headerSize();
   }
 
   /**
    * Returns the size of the compressed part of the block in case compression
    * is used, or the uncompressed size of the data part otherwise. Header size
-   * is not included.
+   * and checksum data size is not included.
    *
-   * @return the on-disk size of the data part of the block, header not
-   *         included
+   * @return the on-disk size of the data part of the block, header and
+   *         checksum not included. 
    */
-  public int getOnDiskSizeWithoutHeader() {
+  int getOnDiskSizeWithoutHeader() {
     return onDiskSizeWithoutHeader;
   }
 
@@ -222,7 +307,7 @@ public int getOnDiskSizeWithoutHeader() {
    * @return the uncompressed size of the data part of the block, header not
    *         included
    */
-  public int getUncompressedSizeWithoutHeader() {
+   public int getUncompressedSizeWithoutHeader() {
     return uncompressedSizeWithoutHeader;
   }
 
@@ -249,25 +334,27 @@ private void overwriteHeader() {
   /**
    * Returns a buffer that does not include the header. The array offset points
    * to the start of the block data right after the header. The underlying data
-   * array is not copied.
+   * array is not copied. Checksum data is not included in the returned buffer.
    *
    * @return the buffer with header skipped
    */
-  public ByteBuffer getBufferWithoutHeader() {
-    return ByteBuffer.wrap(buf.array(), buf.arrayOffset() + HEADER_SIZE,
-        buf.limit() - HEADER_SIZE).slice();
+  ByteBuffer getBufferWithoutHeader() {
+    return ByteBuffer.wrap(buf.array(), buf.arrayOffset() + headerSize(),
+        buf.limit() - headerSize() - totalChecksumBytes()).slice();
   }
 
   /**
    * Returns the buffer this block stores internally. The clients must not
    * modify the buffer object. This method has to be public because it is
    * used in {@link CompoundBloomFilter} to avoid object creation on every
-   * Bloom filter lookup, but has to be used with caution.
+   * Bloom filter lookup, but has to be used with caution. Checksum data
+   * is not included in the returned buffer.
    *
    * @return the buffer of this block for read-only operations
    */
   public ByteBuffer getBufferReadOnly() {
-    return buf;
+    return ByteBuffer.wrap(buf.array(), buf.arrayOffset(),
+        buf.limit() - totalChecksumBytes()).slice();
   }
 
   /**
@@ -276,7 +363,7 @@ public ByteBuffer getBufferReadOnly() {
    *
    * @return the byte buffer with header included
    */
-  public ByteBuffer getBufferWithHeader() {
+  ByteBuffer getBufferWithHeader() {
     ByteBuffer dupBuf = buf.duplicate();
     dupBuf.rewind();
     return dupBuf;
@@ -286,11 +373,11 @@ public ByteBuffer getBufferWithHeader() {
    * Deserializes fields of the given writable using the data portion of this
    * block. Does not check that all the block data has been read.
    */
-  public void readInto(Writable w) throws IOException {
+  void readInto(Writable w) throws IOException {
     Preconditions.checkNotNull(w);
 
-    if (Writables.getWritable(buf.array(), buf.arrayOffset() + HEADER_SIZE,
-        buf.limit() - HEADER_SIZE, w) == null) {
+    if (Writables.getWritable(buf.array(), buf.arrayOffset() + headerSize(),
+        buf.limit() - headerSize(), w) == null) {
       throw new IOException("Failed to deserialize block " + this + " into a "
           + w.getClass().getSimpleName());
     }
@@ -328,8 +415,17 @@ void sanityCheck() throws IOException {
         "uncompressedSizeWithoutHeader");
 
     sanityCheckAssertion(buf.getLong(), prevBlockOffset, "prevBlocKOffset");
+    if (minorVersion >= MINOR_VERSION_WITH_CHECKSUM) {
+      sanityCheckAssertion(buf.get(), checksumType, "checksumType");
+      sanityCheckAssertion(buf.getInt(), bytesPerChecksum, "bytesPerChecksum");
+      sanityCheckAssertion(buf.getInt(), onDiskDataSizeWithHeader, 
+                           "onDiskDataSizeWithHeader");
+    }
 
-    int expectedBufLimit = uncompressedSizeWithoutHeader + HEADER_SIZE;
+    int cksumBytes = totalChecksumBytes();
+    int hdrSize = headerSize();
+    int expectedBufLimit = uncompressedSizeWithoutHeader + headerSize() +
+                           cksumBytes;
     if (buf.limit() != expectedBufLimit) {
       throw new AssertionError("Expected buffer limit " + expectedBufLimit
           + ", got " + buf.limit());
@@ -337,11 +433,11 @@ void sanityCheck() throws IOException {
 
     // We might optionally allocate HEADER_SIZE more bytes to read the next
     // block's, header, so there are two sensible values for buffer capacity.
-    if (buf.capacity() != uncompressedSizeWithoutHeader + HEADER_SIZE &&
-        buf.capacity() != uncompressedSizeWithoutHeader + 2 * HEADER_SIZE) {
+    int size = uncompressedSizeWithoutHeader + hdrSize + cksumBytes;
+    if (buf.capacity() != size &&
+        buf.capacity() != size + hdrSize) {
       throw new AssertionError("Invalid buffer capacity: " + buf.capacity() +
-          ", expected " + (uncompressedSizeWithoutHeader + HEADER_SIZE) +
-          " or " + (uncompressedSizeWithoutHeader + 2 * HEADER_SIZE));
+          ", expected " + size + " or " + (size + hdrSize));
     }
   }
 
@@ -356,8 +452,8 @@ public String toString() {
         + ", prevBlockOffset="
         + prevBlockOffset
         + ", dataBeginsWith="
-        + Bytes.toStringBinary(buf.array(), buf.arrayOffset() + HEADER_SIZE,
-            Math.min(32, buf.limit() - buf.arrayOffset() - HEADER_SIZE))
+        + Bytes.toStringBinary(buf.array(), buf.arrayOffset() + headerSize(),
+            Math.min(32, buf.limit() - buf.arrayOffset() - headerSize()))
         + ", fileOffset=" + offset;
   }
 
@@ -377,31 +473,36 @@ private void validateOnDiskSizeWithoutHeader(
 
   /**
    * Always allocates a new buffer of the correct size. Copies header bytes
-   * from the existing buffer. Does not change header fields.
+   * from the existing buffer. Does not change header fields. 
+   * Reserve room to keep checksum bytes too.
    *
    * @param extraBytes whether to reserve room in the buffer to read the next
    *          block's header
    */
   private void allocateBuffer(boolean extraBytes) {
-    int capacityNeeded = HEADER_SIZE + uncompressedSizeWithoutHeader +
-        (extraBytes ? HEADER_SIZE : 0);
+    int cksumBytes = totalChecksumBytes();
+    int capacityNeeded = headerSize() + uncompressedSizeWithoutHeader +
+        cksumBytes +
+        (extraBytes ? headerSize() : 0);
 
     ByteBuffer newBuf = ByteBuffer.allocate(capacityNeeded);
 
     // Copy header bytes.
     System.arraycopy(buf.array(), buf.arrayOffset(), newBuf.array(),
-        newBuf.arrayOffset(), HEADER_SIZE);
+        newBuf.arrayOffset(), headerSize());
 
     buf = newBuf;
-    buf.limit(HEADER_SIZE + uncompressedSizeWithoutHeader);
+    buf.limit(headerSize() + uncompressedSizeWithoutHeader + cksumBytes);
   }
 
   /** An additional sanity-check in case no compression is being used. */
   public void assumeUncompressed() throws IOException {
-    if (onDiskSizeWithoutHeader != uncompressedSizeWithoutHeader) {
+    if (onDiskSizeWithoutHeader != uncompressedSizeWithoutHeader + 
+        totalChecksumBytes()) {
       throw new IOException("Using no compression but "
           + "onDiskSizeWithoutHeader=" + onDiskSizeWithoutHeader + ", "
-          + "uncompressedSizeWithoutHeader=" + uncompressedSizeWithoutHeader);
+          + "uncompressedSizeWithoutHeader=" + uncompressedSizeWithoutHeader
+          + ", numChecksumbytes=" + totalChecksumBytes());
     }
   }
 
@@ -430,7 +531,7 @@ public long getOffset() {
    */
   public DataInputStream getByteStream() {
     return new DataInputStream(new ByteArrayInputStream(buf.array(),
-        buf.arrayOffset() + HEADER_SIZE, buf.limit() - HEADER_SIZE));
+        buf.arrayOffset() + headerSize(), buf.limit() - headerSize()));
   }
 
   @Override
@@ -441,7 +542,10 @@ public long heapSize() {
         // Block type and byte buffer references
         2 * ClassSize.REFERENCE +
         // On-disk size, uncompressed size, and next block's on-disk size
-        3 * Bytes.SIZEOF_INT +
+        // bytePerChecksum,  onDiskDataSize and minorVersion
+        6 * Bytes.SIZEOF_INT +
+        // Checksum type
+        1 * Bytes.SIZEOF_BYTE +
         // This and previous block offset
         2 * Bytes.SIZEOF_LONG +
         // "Include memstore timestamp" flag
@@ -568,14 +672,30 @@ private enum State {
 
     /**
      * Bytes to be written to the file system, including the header. Compressed
-     * if compression is turned on.
+     * if compression is turned on. It also includes the checksum data that 
+     * immediately follows the block data. (header + data + checksums)
      */
     private byte[] onDiskBytesWithHeader;
 
+    /**
+     * The size of the data on disk that does not include the checksums.
+     * (header + data)
+     */
+    private int onDiskDataSizeWithHeader;
+
+    /**
+     * The size of the checksum data on disk. It is used only if data is
+     * not compressed. If data is compressed, then the checksums are already
+     * part of onDiskBytesWithHeader. If data is uncompressed, then this
+     * variable stores the checksum data for this block.
+     */
+    private byte[] onDiskChecksum;
+
     /**
      * Valid in the READY state. Contains the header and the uncompressed (but
      * potentially encoded, if this is a data block) bytes, so the length is
      * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE}.
+     * Does not store checksums.
      */
     private byte[] uncompressedBytesWithHeader;
 
@@ -597,12 +717,19 @@ private enum State {
     /** Whether we are including memstore timestamp after every key/value */
     private boolean includesMemstoreTS;
 
+    /** Checksum settings */
+    private ChecksumType checksumType;
+    private int bytesPerChecksum;
+
     /**
      * @param compressionAlgorithm compression algorithm to use
      * @param dataBlockEncoderAlgo data block encoding algorithm to use
+     * @param checksumType type of checksum
+     * @param bytesPerChecksum bytes per checksum
      */
     public Writer(Compression.Algorithm compressionAlgorithm,
-          HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
+          HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS,
+          ChecksumType checksumType, int bytesPerChecksum) {
       compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
       this.dataBlockEncoder = dataBlockEncoder != null
           ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
@@ -620,12 +747,19 @@ public Writer(Compression.Algorithm compressionAlgorithm,
               "for algorithm " + compressionAlgorithm, e);
         }
       }
+      if (bytesPerChecksum < HEADER_SIZE) {
+        throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
+            " Minimum is " + HEADER_SIZE + " but the configured value is " +
+            bytesPerChecksum);
+      }
       
       prevOffsetByType = new long[BlockType.values().length];
       for (int i = 0; i < prevOffsetByType.length; ++i)
         prevOffsetByType[i] = -1;
 
       this.includesMemstoreTS = includesMemstoreTS;
+      this.checksumType = checksumType;
+      this.bytesPerChecksum = bytesPerChecksum;
     }
 
     /**
@@ -701,16 +835,18 @@ private void finishBlock() throws IOException {
       state = State.BLOCK_READY;
       encodeDataBlockForDisk();
 
-      doCompression();
-      putHeader(uncompressedBytesWithHeader, 0, onDiskBytesWithHeader.length,
-          uncompressedBytesWithHeader.length);
+      doCompressionAndChecksumming();
     }
 
     /**
      * Do compression if it is enabled, or re-use the uncompressed buffer if
      * it is not. Fills in the compressed block's header if doing compression.
+     * Also, compute the checksums. In the case of no-compression, write the
+     * checksums to its own seperate data structure called onDiskChecksum. In
+     * the case when compression is enabled, the checksums are written to the
+     * outputbyte stream 'baos'.
      */
-    private void doCompression() throws IOException {
+    private void doCompressionAndChecksumming() throws IOException {
       // do the compression
       if (compressAlgo != NONE) {
         compressedByteStream.reset();
@@ -724,11 +860,53 @@ private void doCompression() throws IOException {
         compressionStream.flush();
         compressionStream.finish();
 
+        // generate checksums
+        onDiskDataSizeWithHeader = compressedByteStream.size(); // data size
+
+        // reserve space for checksums in the output byte stream
+        ChecksumUtil.reserveSpaceForChecksums(compressedByteStream, 
+          onDiskDataSizeWithHeader, bytesPerChecksum);
+
+
         onDiskBytesWithHeader = compressedByteStream.toByteArray();
         putHeader(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
-            uncompressedBytesWithHeader.length);
+            uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+
+       // generate checksums for header and data. The checksums are
+       // part of onDiskBytesWithHeader itself.
+       ChecksumUtil.generateChecksums(
+         onDiskBytesWithHeader, 0, onDiskDataSizeWithHeader,
+         onDiskBytesWithHeader, onDiskDataSizeWithHeader,
+         checksumType, bytesPerChecksum);
+
+        // Checksums are already part of onDiskBytesWithHeader
+        onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
+
+        //set the header for the uncompressed bytes (for cache-on-write)
+        putHeader(uncompressedBytesWithHeader, 0,
+          onDiskBytesWithHeader.length + onDiskChecksum.length,
+          uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+
       } else {
+        // If we are not using any compression, then the
+        // checksums are written to its own array onDiskChecksum.
         onDiskBytesWithHeader = uncompressedBytesWithHeader;
+
+        onDiskDataSizeWithHeader = onDiskBytesWithHeader.length;
+        int numBytes = (int)ChecksumUtil.numBytes(
+                          uncompressedBytesWithHeader.length,
+                          bytesPerChecksum);
+        onDiskChecksum = new byte[numBytes];
+
+        //set the header for the uncompressed bytes
+        putHeader(uncompressedBytesWithHeader, 0,
+          onDiskBytesWithHeader.length + onDiskChecksum.length,
+          uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+
+        ChecksumUtil.generateChecksums(
+          uncompressedBytesWithHeader, 0, uncompressedBytesWithHeader.length,
+          onDiskChecksum, 0,
+          checksumType, bytesPerChecksum);
       }
     }
 
@@ -747,7 +925,7 @@ private void encodeDataBlockForDisk() throws IOException {
           HEADER_SIZE).slice();
       Pair<ByteBuffer, BlockType> encodingResult =
           dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
-              includesMemstoreTS);
+              includesMemstoreTS, DUMMY_HEADER);
 
       BlockType encodedBlockType = encodingResult.getSecond();
       if (encodedBlockType == BlockType.ENCODED_DATA) {
@@ -770,16 +948,21 @@ private void encodeDataBlockForDisk() throws IOException {
 
     /**
      * Put the header into the given byte array at the given offset.
-     * @param onDiskSize size of the block on disk
+     * @param onDiskSize size of the block on disk header + data + checksum
      * @param uncompressedSize size of the block after decompression (but
-     *          before optional data block decoding)
+     *          before optional data block decoding) including header
+     * @param onDiskDataSize size of the block on disk with header
+     *        and data but not including the checksums
      */
     private void putHeader(byte[] dest, int offset, int onDiskSize,
-        int uncompressedSize) {
+        int uncompressedSize, int onDiskDataSize) {
       offset = blockType.put(dest, offset);
       offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
       offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
-      Bytes.putLong(dest, offset, prevOffset);
+      offset = Bytes.putLong(dest, offset, prevOffset);
+      offset = Bytes.putByte(dest, offset, checksumType.getCode());
+      offset = Bytes.putInt(dest, offset, bytesPerChecksum);
+      offset = Bytes.putInt(dest, offset, onDiskDataSizeWithHeader);
     }
 
     /**
@@ -814,19 +997,45 @@ public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
     private void writeHeaderAndData(DataOutputStream out) throws IOException {
       ensureBlockReady();
       out.write(onDiskBytesWithHeader);
+      if (compressAlgo == NONE) {
+        if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
+          throw new IOException("A " + blockType 
+              + " without compression should have checksums " 
+              + " stored separately.");
+        }
+        out.write(onDiskChecksum);
+      }
     }
 
     /**
      * Returns the header or the compressed data (or uncompressed data when not
      * using compression) as a byte array. Can be called in the "writing" state
      * or in the "block ready" state. If called in the "writing" state,
-     * transitions the writer to the "block ready" state.
+     * transitions the writer to the "block ready" state. This returns
+     * the header + data + checksums stored on disk.
      *
      * @return header and data as they would be stored on disk in a byte array
      * @throws IOException
      */
-    public byte[] getHeaderAndData() throws IOException {
+    byte[] getHeaderAndDataForTest() throws IOException {
       ensureBlockReady();
+      if (compressAlgo == NONE) {
+        if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
+          throw new IOException("A " + blockType 
+              + " without compression should have checksums " 
+              + " stored separately.");
+        }
+        // This is not very optimal, because we are doing an extra copy.
+        // But this method is used only by unit tests.
+        byte[] output = new byte[onDiskBytesWithHeader.length +
+                                 onDiskChecksum.length];
+        System.arraycopy(onDiskBytesWithHeader, 0,
+                         output, 0, onDiskBytesWithHeader.length);
+        System.arraycopy(onDiskChecksum, 0,
+                         output, onDiskBytesWithHeader.length,
+                         onDiskChecksum.length);
+        return output;
+      }
       return onDiskBytesWithHeader;
     }
 
@@ -849,9 +1058,9 @@ public void releaseCompressor() {
      *
      * @return the on-disk size of the block, not including the header.
      */
-    public int getOnDiskSizeWithoutHeader() {
+    int getOnDiskSizeWithoutHeader() {
       expectState(State.BLOCK_READY);
-      return onDiskBytesWithHeader.length - HEADER_SIZE;
+      return onDiskBytesWithHeader.length + onDiskChecksum.length - HEADER_SIZE;
     }
 
     /**
@@ -859,17 +1068,17 @@ public int getOnDiskSizeWithoutHeader() {
      * "block ready" state.
      *
      * @return the on-disk size of the block ready to be written, including the
-     *         header size
+     *         header size, the data and the checksum data.
      */
-    public int getOnDiskSizeWithHeader() {
+    int getOnDiskSizeWithHeader() {
       expectState(State.BLOCK_READY);
-      return onDiskBytesWithHeader.length;
+      return onDiskBytesWithHeader.length + onDiskChecksum.length;
     }
 
     /**
      * The uncompressed size of the block data. Does not include header size.
      */
-    public int getUncompressedSizeWithoutHeader() {
+    int getUncompressedSizeWithoutHeader() {
       expectState(State.BLOCK_READY);
       return uncompressedBytesWithHeader.length - HEADER_SIZE;
     }
@@ -877,7 +1086,7 @@ public int getUncompressedSizeWithoutHeader() {
     /**
      * The uncompressed size of the block data, including header size.
      */
-    public int getUncompressedSizeWithHeader() {
+    int getUncompressedSizeWithHeader() {
       expectState(State.BLOCK_READY);
       return uncompressedBytesWithHeader.length;
     }
@@ -904,13 +1113,13 @@ public int blockSizeWritten() {
      * Returns the header followed by the uncompressed data, even if using
      * compression. This is needed for storing uncompressed blocks in the block
      * cache. Can be called in the "writing" state or the "block ready" state.
+     * Returns only the header and data, does not include checksum data.
      *
      * @return uncompressed block bytes for caching on write
      */
-    private byte[] getUncompressedDataWithHeader() {
+    ByteBuffer getUncompressedBufferWithHeader() {
       expectState(State.BLOCK_READY);
-
-      return uncompressedBytesWithHeader;
+      return ByteBuffer.wrap(uncompressedBytesWithHeader);
     }
 
     private void expectState(State expectedState) {
@@ -920,17 +1129,6 @@ private void expectState(State expectedState) {
       }
     }
 
-    /**
-     * Similar to {@link #getUncompressedBufferWithHeader()} but returns a byte
-     * buffer.
-     *
-     * @return uncompressed block for caching on write in the form of a buffer
-     */
-    public ByteBuffer getUncompressedBufferWithHeader() {
-      byte[] b = getUncompressedDataWithHeader();
-      return ByteBuffer.wrap(b, 0, b.length);
-    }
-
     /**
      * Takes the given {@link BlockWritable} instance, creates a new block of
      * its appropriate type, writes the writable into this block, and flushes
@@ -947,13 +1145,21 @@ public void writeBlock(BlockWritable bw, FSDataOutputStream out)
       writeHeaderAndData(out);
     }
 
+    /**
+     * Creates a new HFileBlock. Checksums have already been validated, so
+     * the byte buffer passed into the constructor of this newly created
+     * block does not have checksum data even though the header minor 
+     * version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a
+     * 0 value in bytesPerChecksum.
+     */
     public HFileBlock getBlockForCaching() {
       return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
           getUncompressedSizeWithoutHeader(), prevOffset,
           getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
-          includesMemstoreTS);
+          includesMemstoreTS, MINOR_VERSION_WITH_CHECKSUM,
+          0, ChecksumType.NULL.getCode(),  // no checksums in cached data
+          onDiskBytesWithHeader.length + onDiskChecksum.length);
     }
-
   }
 
   /** Something that can be written into a block. */
@@ -1022,10 +1228,15 @@ HFileBlock readBlockData(long offset, long onDiskSize,
    * A common implementation of some methods of {@link FSReader} and some
    * tools for implementing HFile format version-specific block readers.
    */
-  public abstract static class AbstractFSReader implements FSReader {
+  private abstract static class AbstractFSReader implements FSReader {
+
+    /** The file system stream of the underlying {@link HFile} that 
+     * does checksum validations in the filesystem */
+    protected final FSDataInputStream istream;
 
-    /** The file system stream of the underlying {@link HFile} */
-    protected FSDataInputStream istream;
+    /** The file system stream of the underlying {@link HFile} that
+     * does not do checksum verification in the file system */
+    protected final FSDataInputStream istreamNoFsChecksum;
 
     /** Compression algorithm used by the {@link HFile} */
     protected Compression.Algorithm compressAlgo;
@@ -1033,14 +1244,34 @@ public abstract static class AbstractFSReader implements FSReader {
     /** The size of the file we are reading from, or -1 if unknown. */
     protected long fileSize;
 
+    /** The minor version of this reader */
+    private int minorVersion;
+
+    /** The size of the header */
+    protected int hdrSize;
+
+    /** The filesystem used to access data */
+    protected HFileSystem hfs;
+
+    /** The path (if any) where this data is coming from */
+    protected Path path;
+
     /** The default buffer size for our buffered streams */
     public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
 
-    public AbstractFSReader(FSDataInputStream istream, Algorithm compressAlgo,
-        long fileSize) {
+    public AbstractFSReader(FSDataInputStream istream, 
+        FSDataInputStream istreamNoFsChecksum,
+        Algorithm compressAlgo,
+        long fileSize, int minorVersion, HFileSystem hfs, Path path) 
+        throws IOException {
       this.istream = istream;
       this.compressAlgo = compressAlgo;
       this.fileSize = fileSize;
+      this.minorVersion = minorVersion;
+      this.hfs = hfs;
+      this.path = path;
+      this.hdrSize = headerSize(minorVersion);
+      this.istreamNoFsChecksum = istreamNoFsChecksum;
     }
 
     @Override
@@ -1081,25 +1312,27 @@ public DataInputStream nextBlockAsStream(BlockType blockType)
      * @param peekIntoNextBlock whether to read the next block's on-disk size
      * @param fileOffset position in the stream to read at
      * @param pread whether we should do a positional read
+     * @param istream The input source of data
      * @return the on-disk size of the next block with header size included, or
      *         -1 if it could not be determined
      * @throws IOException
      */
-    protected int readAtOffset(byte[] dest, int destOffset, int size,
+    protected int readAtOffset(FSDataInputStream istream,
+        byte[] dest, int destOffset, int size,
         boolean peekIntoNextBlock, long fileOffset, boolean pread)
         throws IOException {
       if (peekIntoNextBlock &&
-          destOffset + size + HEADER_SIZE > dest.length) {
+          destOffset + size + hdrSize > dest.length) {
         // We are asked to read the next block's header as well, but there is
         // not enough room in the array.
         throw new IOException("Attempted to read " + size + " bytes and " +
-            HEADER_SIZE + " bytes of next header into a " + dest.length +
+            hdrSize + " bytes of next header into a " + dest.length +
             "-byte array at offset " + destOffset);
       }
 
       if (pread) {
         // Positional read. Better for random reads.
-        int extraSize = peekIntoNextBlock ? HEADER_SIZE : 0;
+        int extraSize = peekIntoNextBlock ? hdrSize : 0;
 
         int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
         if (ret < size) {
@@ -1129,14 +1362,14 @@ protected int readAtOffset(byte[] dest, int destOffset, int size,
           }
 
           // Try to read the next block header.
-          if (!readWithExtra(istream, dest, destOffset, size, HEADER_SIZE))
+          if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
             return -1;
         }
       }
 
       assert peekIntoNextBlock;
       return Bytes.toInt(dest, destOffset + size + BlockType.MAGIC_LENGTH) +
-          HEADER_SIZE;
+          hdrSize;
     }
 
     /**
@@ -1147,14 +1380,12 @@ protected int readAtOffset(byte[] dest, int destOffset, int size,
      * @param bufferedBoundedStream
      *          a stream to read compressed data from, bounded to the exact
      *          amount of compressed data
-     * @param compressedSize
-     *          compressed data size, header not included
      * @param uncompressedSize
      *          uncompressed data size, header not included
      * @throws IOException
      */
     protected void decompress(byte[] dest, int destOffset,
-        InputStream bufferedBoundedStream, int compressedSize,
+        InputStream bufferedBoundedStream,
         int uncompressedSize) throws IOException {
       Decompressor decompressor = null;
       try {
@@ -1187,6 +1418,12 @@ protected InputStream createBufferedBoundedStream(long offset,
           offset, size, pread), Math.min(DEFAULT_BUFFER_SIZE, size));
     }
 
+    /**
+     * @return The minorVersion of this HFile
+     */
+    protected int getMinorVersion() {
+      return minorVersion;
+    }
   }
 
   /**
@@ -1196,14 +1433,15 @@ protected InputStream createBufferedBoundedStream(long offset,
    * reader returns blocks represented in the uniform version 2 format in
    * memory.
    */
-  public static class FSReaderV1 extends AbstractFSReader {
+  static class FSReaderV1 extends AbstractFSReader {
 
     /** Header size difference between version 1 and 2 */
-    private static final int HEADER_DELTA = HEADER_SIZE - MAGIC_LENGTH;
+    private static final int HEADER_DELTA = HEADER_SIZE_NO_CHECKSUM - 
+                                            MAGIC_LENGTH;
 
     public FSReaderV1(FSDataInputStream istream, Algorithm compressAlgo,
-        long fileSize) {
-      super(istream, compressAlgo, fileSize);
+        long fileSize) throws IOException {
+      super(istream, istream, compressAlgo, fileSize, 0, null, null);
     }
 
     /**
@@ -1262,7 +1500,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
 
         // The first MAGIC_LENGTH bytes of what this will read will be
         // overwritten.
-        readAtOffset(buf.array(), buf.arrayOffset() + HEADER_DELTA,
+        readAtOffset(istream, buf.array(), buf.arrayOffset() + HEADER_DELTA,
             onDiskSize, false, offset, pread);
 
         onDiskSizeWithoutHeader = uncompressedSizeWithMagic - MAGIC_LENGTH;
@@ -1270,7 +1508,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
         InputStream bufferedBoundedStream = createBufferedBoundedStream(
             offset, onDiskSize, pread);
         decompress(buf.array(), buf.arrayOffset() + HEADER_DELTA,
-            bufferedBoundedStream, onDiskSize, uncompressedSizeWithMagic);
+            bufferedBoundedStream, uncompressedSizeWithMagic);
 
         // We don't really have a good way to exclude the "magic record" size
         // from the compressed block's size, since it is compressed as well.
@@ -1285,7 +1523,8 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
       // since the magic record gets moved to the header.
       HFileBlock b = new HFileBlock(newBlockType, onDiskSizeWithoutHeader,
           uncompressedSizeWithMagic - MAGIC_LENGTH, -1L, buf, FILL_HEADER,
-          offset, MemStore.NO_PERSISTENT_TS);
+          offset, MemStore.NO_PERSISTENT_TS, 0, 0, ChecksumType.NULL.getCode(),
+          onDiskSizeWithoutHeader + HEADER_SIZE_NO_CHECKSUM);
       return b;
     }
   }
@@ -1301,7 +1540,20 @@ private static class PrefetchedHeader {
   }
 
   /** Reads version 2 blocks from the filesystem. */
-  public static class FSReaderV2 extends AbstractFSReader {
+  static class FSReaderV2 extends AbstractFSReader {
+
+    // The configuration states that we should validate hbase checksums
+    private final boolean useHBaseChecksumConfigured;
+
+    // Record the current state of this reader with respect to
+    // validating checkums in HBase. This is originally set the same
+    // value as useHBaseChecksumConfigured, but can change state as and when
+    // we encounter checksum verification failures.
+    private volatile boolean useHBaseChecksum;
+
+    // In the case of a checksum failure, do these many succeeding
+    // reads without hbase checksum verification.
+    private volatile int checksumOffCount = -1;
 
     /** Whether we include memstore timestamp in data blocks */
     protected boolean includesMemstoreTS;
@@ -1318,9 +1570,40 @@ public PrefetchedHeader initialValue() {
           }
         };
 
-    public FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
-        long fileSize) {
-      super(istream, compressAlgo, fileSize);
+    public FSReaderV2(FSDataInputStream istream, 
+        FSDataInputStream istreamNoFsChecksum, Algorithm compressAlgo,
+        long fileSize, int minorVersion, HFileSystem hfs, Path path) 
+      throws IOException {
+      super(istream, istreamNoFsChecksum, compressAlgo, fileSize, 
+            minorVersion, hfs, path);
+
+      if (hfs != null) {
+        // Check the configuration to determine whether hbase-level
+        // checksum verification is needed or not.
+        useHBaseChecksum = hfs.useHBaseChecksum();
+      } else {
+        // The configuration does not specify anything about hbase checksum
+        // validations. Set it to true here assuming that we will verify
+        // hbase checksums for all reads. For older files that do not have 
+        // stored checksums, this flag will be reset later.
+        useHBaseChecksum = true;
+      }
+
+      // for older versions, hbase did not store checksums.
+      if (getMinorVersion() < MINOR_VERSION_WITH_CHECKSUM) {
+        useHBaseChecksum = false;
+      }
+      this.useHBaseChecksumConfigured = useHBaseChecksum;
+    }
+
+    /**
+     * A constructor that reads files with the latest minor version.
+     * This is used by unit tests only.
+     */
+    FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
+        long fileSize) throws IOException {
+      this(istream, istream, compressAlgo, fileSize, 
+           HFileReaderV2.MAX_MINOR_VERSION, null, null);
     }
 
     /**
@@ -1337,6 +1620,101 @@ public FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
     @Override
     public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
         int uncompressedSize, boolean pread) throws IOException {
+
+      // It is ok to get a reference to the stream here without any
+      // locks because it is marked final.
+      FSDataInputStream is = this.istreamNoFsChecksum;
+
+      // get a copy of the current state of whether to validate
+      // hbase checksums or not for this read call. This is not 
+      // thread-safe but the one constaint is that if we decide 
+      // to skip hbase checksum verification then we are 
+      // guaranteed to use hdfs checksum verification.
+      boolean doVerificationThruHBaseChecksum = this.useHBaseChecksum;
+      if (!doVerificationThruHBaseChecksum) {
+        is = this.istream;
+      }
+                     
+      HFileBlock blk = readBlockDataInternal(is, offset, 
+                         onDiskSizeWithHeaderL, 
+                         uncompressedSize, pread,
+                         doVerificationThruHBaseChecksum);
+      if (blk == null) {
+        HFile.LOG.warn("HBase checksum verification failed for file " +
+                       path + " at offset " +
+                       offset + " filesize " + fileSize +
+                       ". Retrying read with HDFS checksums turned on...");
+
+        if (!doVerificationThruHBaseChecksum) {
+          String msg = "HBase checksum verification failed for file " +
+                       path + " at offset " +
+                       offset + " filesize " + fileSize + 
+                       " but this cannot happen because doVerify is " +
+                       doVerificationThruHBaseChecksum;
+          HFile.LOG.warn(msg);
+          throw new IOException(msg); // cannot happen case here
+        }
+        HFile.checksumFailures.incrementAndGet(); // update metrics
+ 
+        // If we have a checksum failure, we fall back into a mode where
+        // the next few reads use HDFS level checksums. We aim to make the
+        // next CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD reads avoid
+        // hbase checksum verification, but since this value is set without
+        // holding any locks, it can so happen that we might actually do
+        // a few more than precisely this number.
+        this.checksumOffCount = CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD;
+        this.useHBaseChecksum = false;
+        doVerificationThruHBaseChecksum = false;
+        is = this.istream;
+        blk = readBlockDataInternal(is, offset, onDiskSizeWithHeaderL,
+                                    uncompressedSize, pread,
+                                    doVerificationThruHBaseChecksum);
+        if (blk != null) {
+          HFile.LOG.warn("HDFS checksum verification suceeded for file " +
+                         path + " at offset " +
+                         offset + " filesize " + fileSize);
+        }
+      } 
+      if (blk == null && !doVerificationThruHBaseChecksum) {
+        String msg = "readBlockData failed, possibly due to " +
+                     "checksum verification failed for file " + path +
+                     " at offset " + offset + " filesize " + fileSize;
+        HFile.LOG.warn(msg);
+        throw new IOException(msg);
+      }
+
+      // If there is a checksum mismatch earlier, then retry with 
+      // HBase checksums switched off and use HDFS checksum verification.
+      // This triggers HDFS to detect and fix corrupt replicas. The
+      // next checksumOffCount read requests will use HDFS checksums.
+      // The decrementing of this.checksumOffCount is not thread-safe,
+      // but it is harmless because eventually checksumOffCount will be
+      // a negative number.
+      if (!this.useHBaseChecksum && this.useHBaseChecksumConfigured) {
+        if (this.checksumOffCount-- < 0) {
+          this.useHBaseChecksum = true; // auto re-enable hbase checksums
+        }
+      }
+      return blk;
+    }
+
+    /**
+     * Reads a version 2 block. 
+     *
+     * @param offset the offset in the stream to read at
+     * @param onDiskSizeWithHeaderL the on-disk size of the block, including
+     *          the header, or -1 if unknown
+     * @param uncompressedSize the uncompressed size of the the block. Always
+     *          expected to be -1. This parameter is only used in version 1.
+     * @param pread whether to use a positional read
+     * @param verifyChecksum Whether to use HBase checksums. 
+     *        If HBase checksum is switched off, then use HDFS checksum.
+     * @return the HFileBlock or null if there is a HBase checksum mismatch
+     */
+    private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, 
+        long onDiskSizeWithHeaderL,
+        int uncompressedSize, boolean pread, boolean verifyChecksum) 
+        throws IOException {
       if (offset < 0) {
         throw new IOException("Invalid offset=" + offset + " trying to read "
             + "block (onDiskSize=" + onDiskSizeWithHeaderL
@@ -1347,10 +1725,10 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
             "the uncompressed size parameter");
       }
 
-      if ((onDiskSizeWithHeaderL < HEADER_SIZE && onDiskSizeWithHeaderL != -1)
+      if ((onDiskSizeWithHeaderL < hdrSize && onDiskSizeWithHeaderL != -1)
           || onDiskSizeWithHeaderL >= Integer.MAX_VALUE) {
         throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL
-            + ": expected to be at least " + HEADER_SIZE
+            + ": expected to be at least " + hdrSize
             + " and at most " + Integer.MAX_VALUE + ", or -1 (offset="
             + offset + ", uncompressedSize=" + uncompressedSize + ")");
       }
@@ -1367,7 +1745,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
         // block's header (e.g. this block's header) when reading the previous
         // block. This is the faster and more preferable case.
 
-        int onDiskSizeWithoutHeader = onDiskSizeWithHeader - HEADER_SIZE;
+        int onDiskSizeWithoutHeader = onDiskSizeWithHeader - hdrSize;
         assert onDiskSizeWithoutHeader >= 0;
 
         // See if we can avoid reading the header. This is desirable, because
@@ -1378,39 +1756,42 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
             ? prefetchedHeader.header : null;
 
         // Size that we have to skip in case we have already read the header.
-        int preReadHeaderSize = header == null ? 0 : HEADER_SIZE;
+        int preReadHeaderSize = header == null ? 0 : hdrSize;
 
         if (compressAlgo == Compression.Algorithm.NONE) {
           // Just read the whole thing. Allocate enough space to read the
           // next block's header too.
 
           ByteBuffer headerAndData = ByteBuffer.allocate(onDiskSizeWithHeader
-              + HEADER_SIZE);
+              + hdrSize);
           headerAndData.limit(onDiskSizeWithHeader);
 
           if (header != null) {
             System.arraycopy(header, 0, headerAndData.array(), 0,
-                HEADER_SIZE);
+                hdrSize);
           }
 
-          int nextBlockOnDiskSizeWithHeader = readAtOffset(
+          int nextBlockOnDiskSizeWithHeader = readAtOffset(is,
               headerAndData.array(), headerAndData.arrayOffset()
                   + preReadHeaderSize, onDiskSizeWithHeader
                   - preReadHeaderSize, true, offset + preReadHeaderSize,
                   pread);
 
-          b = new HFileBlock(headerAndData);
+          b = new HFileBlock(headerAndData, getMinorVersion());
           b.assumeUncompressed();
           b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
           b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSizeWithHeader;
-
+          if (verifyChecksum &&
+              !validateBlockChecksum(b, headerAndData.array(), hdrSize)) {
+            return null;             // checksum mismatch
+          }
           if (b.nextBlockOnDiskSizeWithHeader > 0)
             setNextBlockHeader(offset, b);
         } else {
           // Allocate enough space to fit the next block's header too.
-          byte[] onDiskBlock = new byte[onDiskSizeWithHeader + HEADER_SIZE];
+          byte[] onDiskBlock = new byte[onDiskSizeWithHeader + hdrSize];
 
-          int nextBlockOnDiskSize = readAtOffset(onDiskBlock,
+          int nextBlockOnDiskSize = readAtOffset(is, onDiskBlock,
               preReadHeaderSize, onDiskSizeWithHeader - preReadHeaderSize,
               true, offset + preReadHeaderSize, pread);
 
@@ -1418,32 +1799,38 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
             header = onDiskBlock;
 
           try {
-            b = new HFileBlock(ByteBuffer.wrap(header, 0, HEADER_SIZE));
+            b = new HFileBlock(ByteBuffer.wrap(header, 0, hdrSize), 
+                               getMinorVersion());
           } catch (IOException ex) {
             // Seen in load testing. Provide comprehensive debug info.
             throw new IOException("Failed to read compressed block at "
                 + offset + ", onDiskSizeWithoutHeader=" + onDiskSizeWithHeader
                 + ", preReadHeaderSize=" + preReadHeaderSize
                 + ", header.length=" + header.length + ", header bytes: "
-                + Bytes.toStringBinary(header, 0, HEADER_SIZE), ex);
+                + Bytes.toStringBinary(header, 0, hdrSize), ex);
           }
           b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
           b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSize;
+          if (verifyChecksum && 
+              !validateBlockChecksum(b, onDiskBlock, hdrSize)) {
+            return null;             // checksum mismatch
+          }
 
           DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
-              onDiskBlock, HEADER_SIZE, onDiskSizeWithoutHeader));
+              onDiskBlock, hdrSize, onDiskSizeWithoutHeader));
 
           // This will allocate a new buffer but keep header bytes.
           b.allocateBuffer(b.nextBlockOnDiskSizeWithHeader > 0);
 
-          decompress(b.buf.array(), b.buf.arrayOffset() + HEADER_SIZE, dis,
-              onDiskSizeWithoutHeader, b.uncompressedSizeWithoutHeader);
+          decompress(b.buf.array(), b.buf.arrayOffset() + hdrSize, dis,
+              b.uncompressedSizeWithoutHeader);
 
           // Copy next block's header bytes into the new block if we have them.
           if (nextBlockOnDiskSize > 0) {
             System.arraycopy(onDiskBlock, onDiskSizeWithHeader, b.buf.array(),
-                b.buf.arrayOffset() + HEADER_SIZE
-                    + b.uncompressedSizeWithoutHeader, HEADER_SIZE);
+                b.buf.arrayOffset() + hdrSize
+                    + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes(), 
+                hdrSize);
 
             setNextBlockHeader(offset, b);
           }
@@ -1465,12 +1852,12 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
         if (headerBuf == null) {
           // Unfortunately, we still have to do a separate read operation to
           // read the header.
-          headerBuf = ByteBuffer.allocate(HEADER_SIZE);;
-          readAtOffset(headerBuf.array(), headerBuf.arrayOffset(), HEADER_SIZE,
+          headerBuf = ByteBuffer.allocate(hdrSize);
+          readAtOffset(is, headerBuf.array(), headerBuf.arrayOffset(), hdrSize,
               false, offset, pread);
         }
 
-        b = new HFileBlock(headerBuf);
+        b = new HFileBlock(headerBuf, getMinorVersion());
 
         // This will also allocate enough room for the next block's header.
         b.allocateBuffer(true);
@@ -1480,10 +1867,15 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
           // Avoid creating bounded streams and using a "codec" that does
           // nothing.
           b.assumeUncompressed();
-          b.nextBlockOnDiskSizeWithHeader = readAtOffset(b.buf.array(),
-              b.buf.arrayOffset() + HEADER_SIZE,
-              b.uncompressedSizeWithoutHeader, true, offset + HEADER_SIZE,
+          b.nextBlockOnDiskSizeWithHeader = readAtOffset(is, b.buf.array(),
+              b.buf.arrayOffset() + hdrSize,
+              b.uncompressedSizeWithoutHeader + b.totalChecksumBytes(), 
+              true, offset + hdrSize,
               pread);
+          if (verifyChecksum && 
+              !validateBlockChecksum(b, b.buf.array(), hdrSize)) {
+            return null;             // checksum mismatch
+          }
 
           if (b.nextBlockOnDiskSizeWithHeader > 0) {
             setNextBlockHeader(offset, b);
@@ -1491,26 +1883,30 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
         } else {
           // Allocate enough space for the block's header and compressed data.
           byte[] compressedBytes = new byte[b.getOnDiskSizeWithHeader()
-              + HEADER_SIZE];
-
-          b.nextBlockOnDiskSizeWithHeader = readAtOffset(compressedBytes,
-              HEADER_SIZE, b.onDiskSizeWithoutHeader, true, offset
-                  + HEADER_SIZE, pread);
+              + hdrSize];
+
+          b.nextBlockOnDiskSizeWithHeader = readAtOffset(is, compressedBytes,
+              hdrSize, b.onDiskSizeWithoutHeader, true, offset
+                  + hdrSize, pread);
+          if (verifyChecksum &&
+              !validateBlockChecksum(b, compressedBytes, hdrSize)) {
+            return null;             // checksum mismatch
+          }
           DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
-              compressedBytes, HEADER_SIZE, b.onDiskSizeWithoutHeader));
+              compressedBytes, hdrSize, b.onDiskSizeWithoutHeader));
 
-          decompress(b.buf.array(), b.buf.arrayOffset() + HEADER_SIZE, dis,
-              b.onDiskSizeWithoutHeader, b.uncompressedSizeWithoutHeader);
+          decompress(b.buf.array(), b.buf.arrayOffset() + hdrSize, dis,
+              b.uncompressedSizeWithoutHeader);
 
           if (b.nextBlockOnDiskSizeWithHeader > 0) {
             // Copy the next block's header into the new block.
-            int nextHeaderOffset = b.buf.arrayOffset() + HEADER_SIZE
-                + b.uncompressedSizeWithoutHeader;
+            int nextHeaderOffset = b.buf.arrayOffset() + hdrSize
+                + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes();
             System.arraycopy(compressedBytes,
-                compressedBytes.length - HEADER_SIZE,
+                compressedBytes.length - hdrSize,
                 b.buf.array(),
                 nextHeaderOffset,
-                HEADER_SIZE);
+                hdrSize);
 
             setNextBlockHeader(offset, b);
           }
@@ -1525,10 +1921,10 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
     private void setNextBlockHeader(long offset, HFileBlock b) {
       PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
       prefetchedHeader.offset = offset + b.getOnDiskSizeWithHeader();
-      int nextHeaderOffset = b.buf.arrayOffset() + HEADER_SIZE
-          + b.uncompressedSizeWithoutHeader;
+      int nextHeaderOffset = b.buf.arrayOffset() + hdrSize
+          + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes();
       System.arraycopy(b.buf.array(), nextHeaderOffset,
-          prefetchedHeader.header, 0, HEADER_SIZE);
+          prefetchedHeader.header, 0, hdrSize);
     }
 
     void setIncludesMemstoreTS(boolean enabled) {
@@ -1538,6 +1934,18 @@ void setIncludesMemstoreTS(boolean enabled) {
     void setDataBlockEncoder(HFileDataBlockEncoder encoder) {
       this.dataBlockEncoder = encoder;
     }
+
+    /**
+     * Generates the checksum for the header as well as the data and
+     * then validates that it matches the value stored in the header.
+     * If there is a checksum mismatch, then return false. Otherwise
+     * return true.
+     */
+    protected boolean validateBlockChecksum(HFileBlock block, 
+      byte[] data, int hdrSize) throws IOException {
+      return ChecksumUtil.validateBlockChecksum(path, block,
+                                                data, hdrSize);
+    }
   }
 
   @Override
@@ -1616,5 +2024,87 @@ public DataBlockEncoding getDataBlockEncoding() {
     return DataBlockEncoding.NONE;
   }
 
+  byte getChecksumType() {
+    return this.checksumType;
+  }
+
+  int getBytesPerChecksum() {
+    return this.bytesPerChecksum;
+  }
+
+  int getOnDiskDataSizeWithHeader() {
+    return this.onDiskDataSizeWithHeader;
+  }
+
+  int getMinorVersion() {
+    return this.minorVersion;
+  }
+
+  /** 
+   * Calcuate the number of bytes required to store all the checksums
+   * for this block. Each checksum value is a 4 byte integer.
+   */
+  int totalChecksumBytes() {
+    // If the hfile block has minorVersion 0, then there are no checksum
+    // data to validate. Similarly, a zero value in this.bytesPerChecksum
+    // indicates that cached blocks do not have checksum data because
+    // checksums were already validated when the block was read from disk.
+    if (minorVersion < MINOR_VERSION_WITH_CHECKSUM || this.bytesPerChecksum == 0) {
+      return 0;
+    }
+    return (int)ChecksumUtil.numBytes(onDiskDataSizeWithHeader, bytesPerChecksum);
+  }
+
+  /**
+   * Returns the size of this block header.
+   */
+  public int headerSize() {
+    return headerSize(this.minorVersion);
+  }
+
+  /**
+   * Maps a minor version to the size of the header.
+   */
+  static private int headerSize(int minorVersion) {
+    if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
+      return HEADER_SIZE_NO_CHECKSUM;
+    }
+    return HEADER_SIZE;
+  }
+
+  /**
+   * Convert the contents of the block header into a human readable string.
+   * This is mostly helpful for debugging. This assumes that the block
+   * has minor version > 0.
+   */
+  static String toStringHeader(ByteBuffer buf) throws IOException {
+    int offset = buf.arrayOffset();
+    byte[] b = buf.array();
+    long magic = Bytes.toLong(b, offset); 
+    BlockType bt = BlockType.read(buf);
+    offset += Bytes.SIZEOF_LONG;
+    int compressedBlockSizeNoHeader = Bytes.toInt(b, offset);
+    offset += Bytes.SIZEOF_INT;
+    int uncompressedBlockSizeNoHeader = Bytes.toInt(b, offset);
+    offset += Bytes.SIZEOF_INT;
+    long prevBlockOffset = Bytes.toLong(b, offset); 
+    offset += Bytes.SIZEOF_LONG;
+    byte cksumtype = b[offset];
+    offset += Bytes.SIZEOF_BYTE;
+    long bytesPerChecksum = Bytes.toInt(b, offset); 
+    offset += Bytes.SIZEOF_INT;
+    long onDiskDataSizeWithHeader = Bytes.toInt(b, offset); 
+    offset += Bytes.SIZEOF_INT;
+    return " Header dump: magic: " + magic +
+                   " blockType " + bt +
+                   " compressedBlockSizeNoHeader " + 
+                   compressedBlockSizeNoHeader +
+                   " uncompressedBlockSizeNoHeader " + 
+                   uncompressedBlockSizeNoHeader +
+                   " prevBlockOffset " + prevBlockOffset +
+                   " checksumType " + ChecksumType.codeToType(cksumtype) +
+                   " bytesPerChecksum " + bytesPerChecksum +
+                   " onDiskDataSizeWithHeader " + onDiskDataSizeWithHeader;
+  }
 }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
index 37b0b7c9ba32..7aa96432c4ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
@@ -49,11 +49,12 @@ public HFileBlock diskToCacheFormat(HFileBlock block,
    * Should be called before an encoded or unencoded data block is written to
    * disk.
    * @param in KeyValues next to each other
+   * @param dummyHeader A dummy header to be written as a placeholder
    * @return a non-null on-heap buffer containing the contents of the
    *         HFileBlock with unfilled header and block type
    */
   public Pair<ByteBuffer, BlockType> beforeWriteToDisk(
-      ByteBuffer in, boolean includesMemstoreTS);
+      ByteBuffer in, boolean includesMemstoreTS, byte[] dummyHeader);
 
   /**
    * Decides whether we should use a scanner over encoded blocks.
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
index 1759cffbce30..7e369a896eba 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
@@ -152,14 +152,14 @@ public HFileBlock diskToCacheFormat(HFileBlock block, boolean isCompaction) {
    */
   @Override
   public Pair<ByteBuffer, BlockType> beforeWriteToDisk(ByteBuffer in,
-      boolean includesMemstoreTS) {
+      boolean includesMemstoreTS, byte[] dummyHeader) {
     if (onDisk == DataBlockEncoding.NONE) {
       // there is no need to encode the block before writing it to disk
       return new Pair<ByteBuffer, BlockType>(in, BlockType.DATA);
     }
 
     ByteBuffer encodedBuffer = encodeBufferToHFileBlockBuffer(in,
-        onDisk, includesMemstoreTS);
+        onDisk, includesMemstoreTS, dummyHeader);
     return new Pair<ByteBuffer, BlockType>(encodedBuffer,
         BlockType.ENCODED_DATA);
   }
@@ -173,12 +173,13 @@ public boolean useEncodedScanner(boolean isCompaction) {
   }
 
   private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in,
-      DataBlockEncoding algo, boolean includesMemstoreTS) {
+      DataBlockEncoding algo, boolean includesMemstoreTS,
+      byte[] dummyHeader) {
     ByteArrayOutputStream encodedStream = new ByteArrayOutputStream();
     DataOutputStream dataOut = new DataOutputStream(encodedStream);
     DataBlockEncoder encoder = algo.getEncoder();
     try {
-      encodedStream.write(HFileBlock.DUMMY_HEADER);
+      encodedStream.write(dummyHeader);
       algo.writeIdInBytes(dataOut);
       encoder.compressKeyValues(dataOut, in,
           includesMemstoreTS);
@@ -192,13 +193,16 @@ private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in,
   private HFileBlock encodeDataBlock(HFileBlock block,
       DataBlockEncoding algo, boolean includesMemstoreTS) {
     ByteBuffer compressedBuffer = encodeBufferToHFileBlockBuffer(
-        block.getBufferWithoutHeader(), algo, includesMemstoreTS);
-    int sizeWithoutHeader = compressedBuffer.limit() - HFileBlock.HEADER_SIZE;
+        block.getBufferWithoutHeader(), algo, includesMemstoreTS,
+        HFileBlock.DUMMY_HEADER);
+    int sizeWithoutHeader = compressedBuffer.limit() - block.headerSize();
     HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA,
         block.getOnDiskSizeWithoutHeader(),
         sizeWithoutHeader, block.getPrevBlockOffset(),
         compressedBuffer, HFileBlock.FILL_HEADER, block.getOffset(),
-        includesMemstoreTS);
+        includesMemstoreTS, block.getMinorVersion(),
+        block.getBytesPerChecksum(), block.getChecksumType(),
+        block.getOnDiskDataSizeWithHeader());
     block.passSchemaMetricsTo(encodedBlock);
     return encodedBlock;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index d8dac00c7d74..e49ca0a243dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -63,10 +63,10 @@ public class HFileReaderV1 extends AbstractHFileReader {
   public HFileReaderV1(Path path, FixedFileTrailer trailer,
       final FSDataInputStream fsdis, final long size,
       final boolean closeIStream,
-      final CacheConfig cacheConf) {
+      final CacheConfig cacheConf) throws IOException {
     super(path, trailer, fsdis, size, closeIStream, cacheConf);
 
-    trailer.expectVersion(1);
+    trailer.expectMajorVersion(1);
     fsBlockReader = new HFileBlock.FSReaderV1(fsdis, compressAlgo, fileSize);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 33203cb0183e..8e7be993d261 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
@@ -71,6 +72,12 @@ private boolean shouldIncludeMemstoreTS() {
    */
   private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
 
+  /** Minimum minor version supported by this HFile format */
+  static final int MIN_MINOR_VERSION = 0;
+
+  /** Maximum minor version supported by this HFile format */
+  static final int MAX_MINOR_VERSION = 1;
+
   /**
    * Opens a HFile. You must load the index before you can use it by calling
    * {@link #loadFileInfo()}.
@@ -87,14 +94,18 @@ private boolean shouldIncludeMemstoreTS() {
    *          still use its on-disk encoding in cache.
    */
   public HFileReaderV2(Path path, FixedFileTrailer trailer,
-      final FSDataInputStream fsdis, final long size,
+      final FSDataInputStream fsdis, final FSDataInputStream fsdisNoFsChecksum,
+      final long size,
       final boolean closeIStream, final CacheConfig cacheConf,
-      DataBlockEncoding preferredEncodingInCache)
+      DataBlockEncoding preferredEncodingInCache, final HFileSystem hfs)
       throws IOException {
-    super(path, trailer, fsdis, size, closeIStream, cacheConf);
-    trailer.expectVersion(2);
+    super(path, trailer, fsdis, fsdisNoFsChecksum, size, 
+          closeIStream, cacheConf, hfs);
+    trailer.expectMajorVersion(2);
+    validateMinorVersion(path, trailer.getMinorVersion());
     HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis,
-        compressAlgo, fileSize);
+        fsdisNoFsChecksum,
+        compressAlgo, fileSize, trailer.getMinorVersion(), hfs, path);
     this.fsBlockReader = fsBlockReaderV2; // upcast
 
     // Comparator class name is stored in the trailer in version 2.
@@ -409,9 +420,15 @@ public void close(boolean evictOnClose) throws IOException {
           + " block(s)");
       }
     }
-    if (closeIStream && istream != null) {
-      istream.close();
-      istream = null;
+    if (closeIStream) {
+      if (istream != istreamNoFsChecksum && istreamNoFsChecksum != null) {
+        istreamNoFsChecksum.close();
+        istreamNoFsChecksum = null;
+      }
+      if (istream != null) {
+        istream.close();
+        istream = null;
+      }
     }
   }
 
@@ -913,9 +930,9 @@ private void updateCurrentBlock(HFileBlock newBlock) {
     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
       ByteBuffer origBlock = newBlock.getBufferReadOnly();
       ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
-          origBlock.arrayOffset() + HFileBlock.HEADER_SIZE +
+          origBlock.arrayOffset() + newBlock.headerSize() +
           DataBlockEncoding.ID_SIZE,
-          origBlock.limit() - HFileBlock.HEADER_SIZE -
+          newBlock.getUncompressedSizeWithoutHeader() -
           DataBlockEncoding.ID_SIZE).slice();
       return encodedBlock;
     }
@@ -1051,4 +1068,19 @@ public boolean isFileInfoLoaded() {
     return true; // We load file info in constructor in version 2.
   }
 
+  /**
+   * Validates that the minor version is within acceptable limits.
+   * Otherwise throws an Runtime exception
+   */
+  private void validateMinorVersion(Path path, int minorVersion) {
+    if (minorVersion < MIN_MINOR_VERSION ||
+        minorVersion > MAX_MINOR_VERSION) {
+      String msg = "Minor version for path " + path + 
+                   " is expected to be between " +
+                   MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
+                   " but is found to be " + minorVersion;
+      LOG.error(msg);
+      throw new RuntimeException(msg);
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
index 080a14c178d6..6abee866ccdf 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
 import org.apache.hadoop.hbase.regionserver.MemStore;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Writable;
@@ -90,8 +91,9 @@ static class WriterFactoryV1 extends HFile.WriterFactory {
     public Writer createWriter(FileSystem fs, Path path,
         FSDataOutputStream ostream, int blockSize,
         Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder,
-        KeyComparator comparator)
-        throws IOException {
+        KeyComparator comparator, final ChecksumType checksumType,
+        final int bytesPerChecksum) throws IOException {
+      // version 1 does not implement checksums
       return new HFileWriterV1(conf, cacheConf, fs, path, ostream, blockSize,
           compressAlgo, dataBlockEncoder, comparator);
     }
@@ -147,7 +149,13 @@ private void finishBlock() throws IOException {
       HFileBlock block = new HFileBlock(BlockType.DATA,
           (int) (outputStream.getPos() - blockBegin), bytes.length, -1,
           ByteBuffer.wrap(bytes, 0, bytes.length), HFileBlock.FILL_HEADER,
-          blockBegin, MemStore.NO_PERSISTENT_TS);
+          blockBegin, MemStore.NO_PERSISTENT_TS, 
+          HFileBlock.MINOR_VERSION_NO_CHECKSUM,        // minor version
+          0,                                         // bytesPerChecksum
+          ChecksumType.NULL.getCode(),               // checksum type
+          (int) (outputStream.getPos() - blockBegin) +
+          HFileBlock.HEADER_SIZE_NO_CHECKSUM);       // onDiskDataSizeWithHeader
+
       block = blockEncoder.diskToCacheFormat(block, false);
       passSchemaMetricsTo(block);
       cacheConf.getBlockCache().cacheBlock(
@@ -172,7 +180,7 @@ private void newBlock() throws IOException {
     if (cacheConf.shouldCacheDataOnWrite()) {
       this.baos = new ByteArrayOutputStream();
       this.baosDos = new DataOutputStream(baos);
-      baosDos.write(HFileBlock.DUMMY_HEADER);
+      baosDos.write(HFileBlock.DUMMY_HEADER_NO_CHECKSUM);
     }
   }
 
@@ -330,7 +338,8 @@ public void close() throws IOException {
 
     finishBlock();
 
-    FixedFileTrailer trailer = new FixedFileTrailer(1);
+    FixedFileTrailer trailer = new FixedFileTrailer(1,
+                                 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
 
     // Write out the metadata blocks if any.
     ArrayList<Long> metaOffsets = null;
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index ae7a1347cdd3..5f5d74502672 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
 import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Writable;
@@ -79,6 +80,10 @@ public class HFileWriterV2 extends AbstractHFileWriter {
   private List<BlockWritable> additionalLoadOnOpenData =
     new ArrayList<BlockWritable>();
 
+  /** Checksum related settings */
+  private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+  private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
+
   private final boolean includeMemstoreTS = true;
   private long maxMemstoreTS = 0;
 
@@ -91,9 +96,10 @@ static class WriterFactoryV2 extends HFile.WriterFactory {
     public Writer createWriter(FileSystem fs, Path path,
         FSDataOutputStream ostream, int blockSize,
         Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder,
-        final KeyComparator comparator) throws IOException {
+        final KeyComparator comparator, final ChecksumType checksumType,
+        final int bytesPerChecksum) throws IOException {
       return new HFileWriterV2(conf, cacheConf, fs, path, ostream, blockSize,
-          compress, blockEncoder, comparator);
+          compress, blockEncoder, comparator, checksumType, bytesPerChecksum);
     }
   }
 
@@ -101,11 +107,14 @@ public Writer createWriter(FileSystem fs, Path path,
   public HFileWriterV2(Configuration conf, CacheConfig cacheConf,
       FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize,
       Compression.Algorithm compressAlgo, HFileDataBlockEncoder blockEncoder,
-      final KeyComparator comparator) throws IOException {
+      final KeyComparator comparator, final ChecksumType checksumType,
+      final int bytesPerChecksum) throws IOException {
     super(cacheConf,
         ostream == null ? createOutputStream(conf, fs, path) : ostream,
         path, blockSize, compressAlgo, blockEncoder, comparator);
     SchemaMetrics.configureGlobally(conf);
+    this.checksumType = checksumType;
+    this.bytesPerChecksum = bytesPerChecksum;
     finishInit(conf);
   }
 
@@ -116,7 +125,7 @@ private void finishInit(final Configuration conf) {
 
     // HFile filesystem-level (non-caching) block writer
     fsBlockWriter = new HFileBlock.Writer(compressAlgo, blockEncoder,
-        includeMemstoreTS);
+        includeMemstoreTS, checksumType, bytesPerChecksum);
 
     // Data block index writer
     boolean cacheIndexesOnWrite = cacheConf.shouldCacheIndexesOnWrite();
@@ -354,7 +363,8 @@ public void close() throws IOException {
     finishBlock();
     writeInlineBlocks(true);
 
-    FixedFileTrailer trailer = new FixedFileTrailer(2);
+    FixedFileTrailer trailer = new FixedFileTrailer(2, 
+                                 HFileReaderV2.MAX_MINOR_VERSION);
 
     // Write out the metadata blocks if any.
     if (!metaNames.isEmpty()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
index bba8b538ba24..73abdc92ef25 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
@@ -44,7 +44,7 @@ public HFileBlock diskToCacheFormat(HFileBlock block, boolean isCompaction) {
 
   @Override
   public Pair<ByteBuffer, BlockType> beforeWriteToDisk(
-      ByteBuffer in, boolean includesMemstoreTS) {
+      ByteBuffer in, boolean includesMemstoreTS, byte[] dummyHeader) {
     return new Pair<ByteBuffer, BlockType>(in, BlockType.DATA);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 1a724fc85768..2aab028107da 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -70,6 +70,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -549,6 +550,8 @@ private static void copyHFileHalf(
               .withCompression(compression)
               .withDataBlockEncoder(dataBlockEncoder)
               .withBloomType(bloomFilterType)
+              .withChecksumType(Store.getChecksumType(conf))
+              .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
               .build();
       HFileScanner scanner = halfReader.getScanner(false, false, false);
       scanner.seekTo();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 1a8bd6c90c92..acb1f7aff097 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3622,7 +3622,14 @@ public static HRegion openHRegion(final HRegionInfo info,
     }
     Path dir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
       info.getTableName());
-    HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
+    FileSystem fs = null;
+    if (rsServices != null) {
+      fs = rsServices.getFileSystem();
+    }
+    if (fs == null) {
+      fs = FileSystem.get(conf);
+    }
+    HRegion r = HRegion.newHRegion(dir, wal, fs, conf, info,
       htd, rsServices);
     return r.openHRegion(reporter);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5309aaf771ab..406e5aecbe25 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -105,6 +105,7 @@
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -193,7 +194,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   protected final Configuration conf;
 
   protected final AtomicBoolean haveRootRegion = new AtomicBoolean(false);
-  private FileSystem fs;
+  private HFileSystem fs;
+  private boolean useHBaseChecksum; // verify hbase checksums?
   private Path rootDir;
   private final Random rand = new Random();
 
@@ -366,6 +368,11 @@ public HRegionServer(Configuration conf)
     this.isOnline = false;
     checkCodecs(this.conf);
 
+    // do we use checksum verfication in the hbase? If hbase checksum verification
+    // is enabled, then we automatically switch off hdfs checksum verification.
+    this.useHBaseChecksum = conf.getBoolean(
+      HConstants.HBASE_CHECKSUM_VERIFICATION, true);
+
     // Config'ed params
     this.numRetries = conf.getInt("hbase.client.retries.number", 10);
     this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
@@ -976,7 +983,7 @@ protected void handleReportForDutyResponse(final MapWritable c)
       // to defaults).
       this.conf.set("fs.defaultFS", this.conf.get("hbase.rootdir"));
       // Get fs instance used by this RS
-      this.fs = FileSystem.get(this.conf);
+      this.fs = new HFileSystem(this.conf, this.useHBaseChecksum);
       this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
       this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true);
       this.hlog = setupWALAndReplication();
@@ -1276,7 +1283,7 @@ private HLog setupWALAndReplication() throws IOException {
    * @throws IOException
    */
   protected HLog instantiateHLog(Path logdir, Path oldLogDir) throws IOException {
-    return new HLog(this.fs, logdir, oldLogDir, this.conf,
+    return new HLog(this.fs.getBackingFs(), logdir, oldLogDir, this.conf,
       getWALActionListeners(), this.serverNameFromMasterPOV.toString());
   }
 
@@ -3163,7 +3170,7 @@ protected Path getRootDir() {
   /**
    * @return Return the fs.
    */
-  protected FileSystem getFileSystem() {
+  public FileSystem getFileSystem() {
     return fs;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 984ef479da40..d7ed4e82f33b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.util.Map;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
@@ -78,5 +79,9 @@ public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
    * @return map of regions in transition in this RS
    */
   public Map<byte[], Boolean> getRegionsInTransitionInRS();
-  
+
+  /**
+   * @return Return the FileSystem object used by the regionserver
+   */
+  public FileSystem getFileSystem();
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 3f2e390563a0..23ea3cd2a2a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -69,6 +69,7 @@
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.CollectionBackedScanner;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -155,6 +156,10 @@ public class Store extends SchemaConfigured implements HeapSize {
   private final Compression.Algorithm compactionCompression;
   private HFileDataBlockEncoder dataBlockEncoder;
 
+  /** Checksum configuration */
+  private ChecksumType checksumType;
+  private int bytesPerChecksum;
+
   // Comparing KeyValues
   final KeyValue.KVComparator comparator;
 
@@ -244,6 +249,35 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
           "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
     }
     this.storefiles = sortAndClone(loadStoreFiles());
+
+    // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
+    this.checksumType = getChecksumType(conf);
+    // initilize bytes per checksum
+    this.bytesPerChecksum = getBytesPerChecksum(conf);
+  }
+
+  /**
+   * Returns the configured bytesPerChecksum value.
+   * @param conf The configuration
+   * @return The bytesPerChecksum that is set in the configuration
+   */
+  public static int getBytesPerChecksum(Configuration conf) {
+    return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
+                       HFile.DEFAULT_BYTES_PER_CHECKSUM);
+  }
+
+  /**
+   * Returns the configured checksum algorithm.
+   * @param conf The configuration
+   * @return The checksum algorithm that is set in the configuration
+   */
+  public static ChecksumType getChecksumType(Configuration conf) {
+    String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
+    if (checksumName == null) {
+      return HFile.DEFAULT_CHECKSUM_TYPE;
+    } else {
+      return ChecksumType.nameToType(checksumName);
+    }
   }
 
   public HColumnDescriptor getFamily() {
@@ -797,6 +831,8 @@ private StoreFile.Writer createWriterInTmp(int maxKeyCount,
             .withComparator(comparator)
             .withBloomType(family.getBloomFilterType())
             .withMaxKeyCount(maxKeyCount)
+            .withChecksumType(checksumType)
+            .withBytesPerChecksum(bytesPerChecksum)
             .build();
     // The store file writer's path does not include the CF name, so we need
     // to configure the HFile writer directly.
@@ -2190,8 +2226,8 @@ public CacheConfig getCacheConfig() {
 
   public static final long FIXED_OVERHEAD = 
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (19 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
-          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+          + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 4a4cc6763fb1..9d807ddbe025 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -59,6 +59,7 @@
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilter;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
@@ -697,6 +698,8 @@ public static class WriterBuilder {
     private long maxKeyCount = 0;
     private Path dir;
     private Path filePath;
+    private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+    private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
 
     public WriterBuilder(Configuration conf, CacheConfig cacheConf,
         FileSystem fs, int blockSize) {
@@ -763,6 +766,24 @@ public WriterBuilder withMaxKeyCount(long maxKeyCount) {
       return this;
     }
 
+    /**
+     * @param checksumType the type of checksum
+     * @return this (for chained invocation)
+     */
+    public WriterBuilder withChecksumType(ChecksumType checksumType) {
+      this.checksumType = checksumType;
+      return this;
+    }
+
+    /**
+     * @param bytesPerChecksum the number of bytes per checksum chunk
+     * @return this (for chained invocation)
+     */
+    public WriterBuilder withBytesPerChecksum(int bytesPerChecksum) {
+      this.bytesPerChecksum = bytesPerChecksum;
+      return this;
+    }
+
     /**
      * Create a store file writer. Client is responsible for closing file when
      * done. If metadata, add BEFORE closing using
@@ -796,7 +817,8 @@ public Writer build() throws IOException {
         comparator = KeyValue.COMPARATOR;
       }
       return new Writer(fs, filePath, blockSize, compressAlgo, dataBlockEncoder,
-          conf, cacheConf, comparator, bloomType, maxKeyCount);
+          conf, cacheConf, comparator, bloomType, maxKeyCount, checksumType,
+          bytesPerChecksum);
     }
   }
 
@@ -894,6 +916,12 @@ public static class Writer {
 
     protected HFileDataBlockEncoder dataBlockEncoder;
 
+    /** Checksum type */
+    protected ChecksumType checksumType;
+
+    /** Bytes per Checksum */
+    protected int bytesPerChecksum;
+    
     TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
     /* isTimeRangeTrackerSet keeps track if the timeRange has already been set
      * When flushing a memstore, we set TimeRange and use this variable to
@@ -916,13 +944,16 @@ public static class Writer {
      * @param bloomType bloom filter setting
      * @param maxKeys the expected maximum number of keys to be added. Was used
      *        for Bloom filter size in {@link HFile} format version 1.
+     * @param checksumType the checksum type
+     * @param bytesPerChecksum the number of bytes per checksum value
      * @throws IOException problem writing to FS
      */
     private Writer(FileSystem fs, Path path, int blocksize,
         Compression.Algorithm compress,
         HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
         CacheConfig cacheConf,
-        final KVComparator comparator, BloomType bloomType, long maxKeys)
+        final KVComparator comparator, BloomType bloomType, long maxKeys,
+        final ChecksumType checksumType, final int bytesPerChecksum)
         throws IOException {
       this.dataBlockEncoder = dataBlockEncoder != null ?
           dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
@@ -932,6 +963,8 @@ private Writer(FileSystem fs, Path path, int blocksize,
           .withCompression(compress)
           .withDataBlockEncoder(dataBlockEncoder)
           .withComparator(comparator.getRawComparator())
+          .withChecksumType(checksumType)
+          .withBytesPerChecksum(bytesPerChecksum)
           .create();
 
       this.kvComparator = comparator;
@@ -962,6 +995,8 @@ private Writer(FileSystem fs, Path path, int blocksize,
         LOG.info("Delete Family Bloom filter type for " + path + ": "
             + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
       }
+      this.checksumType = checksumType;
+      this.bytesPerChecksum = bytesPerChecksum;
     }
 
     /**
@@ -1658,7 +1693,7 @@ public long getTotalBloomSize() {
     }
 
     public int getHFileVersion() {
-      return reader.getTrailer().getVersion();
+      return reader.getTrailer().getMajorVersion();
     }
 
     HFile.Reader getHFileReader() {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index e3fcfdf30623..a5f8b010366b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -246,6 +246,12 @@ public class RegionServerMetrics implements Updater {
   public final MetricsTimeVaryingLong regionSplitFailureCount =
       new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
 
+  /**
+   * Number of times checksum verification failed.
+   */
+  public final MetricsLongValue checksumFailuresCount =
+    new MetricsLongValue("checksumFailuresCount", registry);
+
   public RegionServerMetrics() {
     MetricsContext context = MetricsUtil.getContext("hbase");
     metricsRecord = MetricsUtil.createRecord(context, "regionserver");
@@ -344,6 +350,8 @@ public void doUpdates(MetricsContext caller) {
       // HFile metrics, positional reads
       ops = HFile.getPreadOps(); 
       if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
+      this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
+
       /* NOTE: removed HFile write latency.  2 reasons:
        * 1) Mixing HLog latencies are far higher priority since they're 
        *      on-demand and HFile is used in background (compact/flush)
@@ -364,6 +372,7 @@ public void doUpdates(MetricsContext caller) {
       this.slowHLogAppendCount.pushMetric(this.metricsRecord);
       this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
       this.regionSplitFailureCount.pushMetric(this.metricsRecord);
+      this.checksumFailuresCount.pushMetric(this.metricsRecord);
     }
     this.metricsRecord.update();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java
new file mode 100644
index 000000000000..d61238bd02e3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java
@@ -0,0 +1,99 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.lang.ClassNotFoundException;
+import java.util.zip.Checksum;
+import java.lang.reflect.Constructor;
+
+/**
+ * Utility class that is used to generate a Checksum object.
+ * The Checksum implementation is pluggable and an application
+ * can specify their own class that implements their own
+ * Checksum algorithm.
+ */
+public class ChecksumFactory {
+
+  static private final Class<?>[] EMPTY_ARRAY = new Class[]{};
+
+  /**
+   * Create a new instance of a Checksum object.
+   * @return The newly created Checksum object
+   */
+  static public Checksum newInstance(String className) throws IOException {
+    try {
+      Class<?> clazz = getClassByName(className);
+      return (Checksum)newInstance(clazz);
+    } catch (ClassNotFoundException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Returns a Constructor that can be used to create a Checksum object.
+   * @return The Constructor that can be used to create a 
+   * new Checksum object.
+   * @param theClass classname for which an constructor is created
+   * @return a new Constructor object
+   */
+  static public Constructor<?> newConstructor(String className) 
+    throws IOException {
+    try {
+      Class<?> clazz = getClassByName(className);
+      Constructor<?> ctor = clazz.getDeclaredConstructor(EMPTY_ARRAY);
+      ctor.setAccessible(true);
+      return ctor;
+    } catch (ClassNotFoundException e) {
+      throw new IOException(e);
+    } catch (java.lang.NoSuchMethodException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /** Create an object for the given class and initialize it from conf
+   *
+   * @param theClass class of which an object is created
+   * @return a new object
+   */
+  static private <T> T newInstance(Class<T> theClass) {
+    T result;
+    try {
+      Constructor<T> ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY);
+      ctor.setAccessible(true);
+      result = ctor.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    return result;
+  }
+
+  /**
+   * Load a class by name.
+   * @param name the class name.
+   * @return the class object.
+   * @throws ClassNotFoundException if the class is not found.
+   */
+  static private Class<?> getClassByName(String name) 
+    throws ClassNotFoundException {
+    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+    return Class.forName(name, true, classLoader);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
new file mode 100644
index 000000000000..d2329e10c588
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+
+/**
+ * Checksum types. The Checksum type is a one byte number
+ * that stores a representation of the checksum algorithm
+ * used to encode a hfile. The ordinal of these cannot 
+ * change or else you risk breaking all existing HFiles out there.
+ */
+public enum ChecksumType {
+
+  NULL((byte)0) {
+    @Override
+    public String getName() {
+      return "NULL";
+    }
+    @Override
+    public void initialize() {
+      // do nothing
+    }
+    @Override
+    public Checksum getChecksumObject() throws IOException {
+      return null; // checksums not used
+    }
+  },
+
+  CRC32((byte)1) {
+    private volatile Constructor<?> ctor;
+
+    @Override
+    public String getName() {
+      return "CRC32";
+    }
+
+    @Override
+    public void initialize() {
+      final String PURECRC32 = "org.apache.hadoop.util.PureJavaCrc32";
+      final String JDKCRC = "java.util.zip.CRC32";
+      LOG = LogFactory.getLog(ChecksumType.class);
+
+      // check if hadoop library is available
+      try {
+        ctor = ChecksumFactory.newConstructor(PURECRC32);
+        LOG.info("Checksum using " + PURECRC32);
+      } catch (Exception e) {
+        LOG.info(PURECRC32 + " not available.");
+      }
+      try {
+        // The default checksum class name is java.util.zip.CRC32. 
+        // This is available on all JVMs.
+        if (ctor == null) {
+          ctor = ChecksumFactory.newConstructor(JDKCRC);
+          LOG.info("Checksum can use " + JDKCRC);
+        }
+      } catch (Exception e) {
+        LOG.warn(JDKCRC + " not available. ",  e);
+      }
+    }
+
+    @Override
+    public Checksum getChecksumObject() throws IOException {
+      if (ctor == null) {
+        throw new IOException("Bad constructor for " + getName());
+      }
+      try {
+        return (Checksum)ctor.newInstance();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  },
+
+  CRC32C((byte)2) {
+    private transient Constructor<?> ctor;
+
+    @Override
+    public String getName() {
+      return "CRC32C";
+    }
+
+    @Override
+    public void initialize() {
+      final String PURECRC32C = "org.apache.hadoop.util.PureJavaCrc32C";
+      LOG = LogFactory.getLog(ChecksumType.class);
+      try {
+        ctor = ChecksumFactory.newConstructor(PURECRC32C);
+        LOG.info("Checksum can use " + PURECRC32C);
+      } catch (Exception e) {
+        LOG.info(PURECRC32C + " not available. ");
+      }
+    }
+
+    @Override
+    public Checksum getChecksumObject() throws IOException {
+      if (ctor == null) {
+        throw new IOException("Bad constructor for " + getName());
+      }
+      try {
+        return (Checksum)ctor.newInstance();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  };
+
+  private final byte code;
+  protected Log LOG;
+
+  /** initializes the relevant checksum class object */
+  abstract void initialize();
+
+  /** returns the name of this checksum type */
+  public abstract String getName();
+
+  private ChecksumType(final byte c) {
+    this.code = c;
+    initialize();
+  }
+
+  /** returns a object that can be used to generate/validate checksums */
+  public abstract Checksum getChecksumObject() throws IOException;
+
+  public byte getCode() {
+    return this.code;
+  }
+
+  /**
+   * Cannot rely on enum ordinals . They change if item is removed or moved.
+   * Do our own codes.
+   * @param b
+   * @return Type associated with passed code.
+   */
+  public static ChecksumType codeToType(final byte b) {
+    for (ChecksumType t : ChecksumType.values()) {
+      if (t.getCode() == b) {
+        return t;
+      }
+    }
+    throw new RuntimeException("Unknown checksum type code " + b);
+  }
+
+  /**
+   * Map a checksum name to a specific type.
+   * Do our own names.
+   * @param b
+   * @return Type associated with passed code.
+   */
+  public static ChecksumType nameToType(final String name) {
+    for (ChecksumType t : ChecksumType.values()) {
+      if (t.getName().equals(name)) {
+        return t;
+      }
+    }
+    throw new RuntimeException("Unknown checksum type name " + name);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
index 09e20a46c79b..6c4ddcf279f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
@@ -107,7 +107,7 @@ public boolean contains(byte[] key, int keyOffset, int keyLength,
 
       ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly();
       result = ByteBloomFilter.contains(key, keyOffset, keyLength,
-          bloomBuf.array(), bloomBuf.arrayOffset() + HFileBlock.HEADER_SIZE,
+          bloomBuf.array(), bloomBuf.arrayOffset() + bloomBlock.headerSize(),
           bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount);
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 498075d1bb60..18dfe7c7cc49 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -56,7 +56,9 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -185,6 +187,9 @@ public HBaseTestingUtility() {
 
   public HBaseTestingUtility(Configuration conf) {
     this.conf = conf;
+
+    // a hbase checksum verification failure will cause unit tests to fail
+    ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
   }
 
   /**
@@ -1433,7 +1438,7 @@ public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
   }
 
   public FileSystem getTestFileSystem() throws IOException {
-    return FileSystem.get(conf);
+    return HFileSystem.get(conf);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
index 61ce0775cab6..106056ff0b03 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hbase.MultithreadedTestUtil;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 
 public class CacheTestUtils {
@@ -323,7 +324,9 @@ private static HFileBlockPair[] generateHFileBlocks(int blockSize,
       HFileBlock generated = new HFileBlock(BlockType.DATA,
           onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
           prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
-          blockSize, includesMemstoreTS);
+          blockSize, includesMemstoreTS, HFileBlock.MINOR_VERSION_NO_CHECKSUM,
+          0, ChecksumType.NULL.getCode(),
+          onDiskSizeWithoutHeader + HFileBlock.HEADER_SIZE);
 
       String strKey;
       /* No conflicting keys */
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index 6007d5a1e46a..c8163f3aa755 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -49,6 +50,7 @@
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.junit.After;
 import org.junit.Before;
@@ -88,6 +90,8 @@ public class TestCacheOnWrite {
   private static final int INDEX_BLOCK_SIZE = 512;
   private static final int BLOOM_BLOCK_SIZE = 4096;
   private static final BloomType BLOOM_TYPE = StoreFile.BloomType.ROWCOL;
+  private static final ChecksumType CKTYPE = ChecksumType.CRC32;
+  private static final int CKBYTES = 512;
 
   /** The number of valid key types possible in a store file */
   private static final int NUM_VALID_KEY_TYPES =
@@ -192,7 +196,7 @@ public void setUp() throws IOException {
     conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
         cowType.shouldBeCached(BlockType.BLOOM_CHUNK));
     cowType.modifyConf(conf);
-    fs = FileSystem.get(conf);
+    fs = HFileSystem.get(conf);
     cacheConf = new CacheConfig(conf);
     blockCache = cacheConf.getBlockCache();
   }
@@ -292,6 +296,8 @@ public void writeStoreFile() throws IOException {
             .withComparator(KeyValue.COMPARATOR)
             .withBloomType(BLOOM_TYPE)
             .withMaxKeyCount(NUM_KV)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
 
     final int rowLen = 32;
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
new file mode 100644
index 000000000000..c037d2255d84
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
@@ -0,0 +1,290 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.DataOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.Compressor;
+
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestChecksum {
+  // change this value to activate more logs
+  private static final boolean detailedLogging = true;
+  private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
+
+  private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
+
+  static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
+      NONE, GZ };
+
+  static final int[] BYTES_PER_CHECKSUM = {
+      50, 500, 688, 16*1024, (16*1024+980), 64 * 1024};
+
+  private static final HBaseTestingUtility TEST_UTIL =
+    new HBaseTestingUtility();
+  private FileSystem fs;
+  private HFileSystem hfs;
+
+  @Before
+  public void setUp() throws Exception {
+    fs = HFileSystem.get(TEST_UTIL.getConfiguration());
+    hfs = (HFileSystem)fs;
+  }
+
+  /**
+   * Introduce checksum failures and check that we can still read
+   * the data
+   */
+  @Test
+  public void testChecksumCorruption() throws IOException {
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+        LOG.info("testChecksumCorruption: Compression algorithm: " + algo +
+                   ", pread=" + pread);
+        Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+            + algo);
+        FSDataOutputStream os = fs.create(path);
+        HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
+            true, HFile.DEFAULT_CHECKSUM_TYPE,
+            HFile.DEFAULT_BYTES_PER_CHECKSUM);
+        long totalSize = 0;
+        for (int blockId = 0; blockId < 2; ++blockId) {
+          DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+          for (int i = 0; i < 1234; ++i)
+            dos.writeInt(i);
+          hbw.writeHeaderAndData(os);
+          totalSize += hbw.getOnDiskSizeWithHeader();
+        }
+        os.close();
+
+        // Use hbase checksums. 
+        assertEquals(true, hfs.useHBaseChecksum());
+        assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
+
+        // Do a read that purposely introduces checksum verification failures.
+        FSDataInputStream is = fs.open(path);
+        HFileBlock.FSReader hbr = new FSReaderV2Test(is, algo,
+            totalSize, HFile.MAX_FORMAT_VERSION, fs, path);
+        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+        b.sanityCheck();
+        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+        // read data back from the hfile, exclude header and checksum
+        ByteBuffer bb = b.getBufferWithoutHeader(); // read back data
+        DataInputStream in = new DataInputStream(
+                               new ByteArrayInputStream(
+                                 bb.array(), bb.arrayOffset(), bb.limit()));
+
+        // assert that we encountered hbase checksum verification failures
+        // but still used hdfs checksums and read data successfully.
+        assertEquals(1, HFile.getChecksumFailuresCount());
+        validateData(in);
+
+        // A single instance of hbase checksum failure causes the reader to
+        // switch off hbase checksum verification for the next 100 read
+        // requests. Verify that this is correct.
+        for (int i = 0; i < 
+             HFileBlock.CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD + 1; i++) {
+          b = hbr.readBlockData(0, -1, -1, pread);
+          assertEquals(0, HFile.getChecksumFailuresCount());
+        }
+        // The next read should have hbase checksum verification reanabled,
+        // we verify this by assertng that there was a hbase-checksum failure.
+        b = hbr.readBlockData(0, -1, -1, pread);
+        assertEquals(1, HFile.getChecksumFailuresCount());
+
+        // Since the above encountered a checksum failure, we switch
+        // back to not checking hbase checksums.
+        b = hbr.readBlockData(0, -1, -1, pread);
+        assertEquals(0, HFile.getChecksumFailuresCount());
+        is.close();
+
+        // Now, use a completely new reader. Switch off hbase checksums in 
+        // the configuration. In this case, we should not detect
+        // any retries within hbase. 
+        HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
+        assertEquals(false, newfs.useHBaseChecksum());
+        is = newfs.open(path);
+        hbr = new FSReaderV2Test(is, algo,
+            totalSize, HFile.MAX_FORMAT_VERSION, newfs, path);
+        b = hbr.readBlockData(0, -1, -1, pread);
+        is.close();
+        b.sanityCheck();
+        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+        // read data back from the hfile, exclude header and checksum
+        bb = b.getBufferWithoutHeader(); // read back data
+        in = new DataInputStream(new ByteArrayInputStream(
+                                 bb.array(), bb.arrayOffset(), bb.limit()));
+
+        // assert that we did not encounter hbase checksum verification failures
+        // but still used hdfs checksums and read data successfully.
+        assertEquals(0, HFile.getChecksumFailuresCount());
+        validateData(in);
+      }
+    }
+  }
+
+  /** 
+   * Test different values of bytesPerChecksum
+   */
+  @Test
+  public void testChecksumChunks() throws IOException {
+    Compression.Algorithm algo = NONE;
+    for (boolean pread : new boolean[] { false, true }) {
+      for (int bytesPerChecksum : BYTES_PER_CHECKSUM) {
+        Path path = new Path(TEST_UTIL.getDataTestDir(), "checksumChunk_" + 
+                             algo + bytesPerChecksum);
+        FSDataOutputStream os = fs.create(path);
+        HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
+          true, HFile.DEFAULT_CHECKSUM_TYPE, bytesPerChecksum);
+
+        // write one block. The block has data
+        // that is at least 6 times more than the checksum chunk size
+        long dataSize = 0;
+        DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+        for (; dataSize < 6 * bytesPerChecksum;) {
+          for (int i = 0; i < 1234; ++i) {
+            dos.writeInt(i);
+            dataSize += 4;
+          }
+        }
+        hbw.writeHeaderAndData(os);
+        long totalSize = hbw.getOnDiskSizeWithHeader();
+        os.close();
+
+        long expectedChunks = ChecksumUtil.numChunks(
+                               dataSize + HFileBlock.HEADER_SIZE,
+                               bytesPerChecksum);
+        LOG.info("testChecksumChunks: pread=" + pread +
+                   ", bytesPerChecksum=" + bytesPerChecksum +
+                   ", fileSize=" + totalSize +
+                   ", dataSize=" + dataSize +
+                   ", expectedChunks=" + expectedChunks);
+
+        // Verify hbase checksums. 
+        assertEquals(true, hfs.useHBaseChecksum());
+        assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
+
+        // Read data back from file.
+        FSDataInputStream is = fs.open(path);
+        FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);
+        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, nochecksum, 
+            algo, totalSize, HFile.MAX_FORMAT_VERSION, hfs, path);
+        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+        is.close();
+        b.sanityCheck();
+        assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());
+
+        // verify that we have the expected number of checksum chunks
+        assertEquals(totalSize, HFileBlock.HEADER_SIZE + dataSize + 
+                     expectedChunks * HFileBlock.CHECKSUM_SIZE);
+
+        // assert that we did not encounter hbase checksum verification failures
+        assertEquals(0, HFile.getChecksumFailuresCount());
+      }
+    }
+  }
+
+  /** 
+   * Test to ensure that these is at least one valid checksum implementation
+   */
+  @Test
+  public void testChecksumAlgorithm() throws IOException {
+    ChecksumType type = ChecksumType.CRC32;
+    assertEquals(ChecksumType.nameToType(type.getName()), type);
+    assertEquals(ChecksumType.valueOf(type.toString()), type);
+  }
+
+  private void validateData(DataInputStream in) throws IOException {
+    // validate data
+    for (int i = 0; i < 1234; i++) {
+      int val = in.readInt();
+      if (val != i) {
+        String msg = "testChecksumCorruption: data mismatch at index " +
+                     i + " expected " + i + " found " + val;
+        LOG.warn(msg);
+        assertEquals(i, val);
+      }
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+  /**
+   * A class that introduces hbase-checksum failures while 
+   * reading  data from hfiles. This should trigger the hdfs level
+   * checksum validations.
+   */
+  static private class FSReaderV2Test extends HFileBlock.FSReaderV2 {
+
+    FSReaderV2Test(FSDataInputStream istream, Algorithm algo,
+                   long fileSize, int minorVersion, FileSystem fs,
+                   Path path) throws IOException {
+      super(istream, istream, algo, fileSize, minorVersion, 
+            (HFileSystem)fs, path);
+    }
+
+    @Override
+    protected boolean validateBlockChecksum(HFileBlock block, 
+      byte[] data, int hdrSize) throws IOException {
+      return false;  // checksum validation failure
+    }
+  }
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
index 1d0f2cffd7c9..a9288ce979eb 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
@@ -52,7 +52,7 @@ public class TestFixedFileTrailer {
   private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
 
   /** The number of used fields by version. Indexed by version minus one. */
-  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 8, 13 };
+  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 9, 14 };
 
   private HBaseTestingUtility util = new HBaseTestingUtility();
   private FileSystem fs;
@@ -83,7 +83,8 @@ public void setUp() throws IOException {
 
   @Test
   public void testTrailer() throws IOException {
-    FixedFileTrailer t = new FixedFileTrailer(version);
+    FixedFileTrailer t = new FixedFileTrailer(version, 
+                           HFileBlock.MINOR_VERSION_NO_CHECKSUM);
     t.setDataIndexCount(3);
     t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
 
@@ -121,7 +122,8 @@ public void testTrailer() throws IOException {
     // Finished writing, trying to read.
     {
       DataInputStream dis = new DataInputStream(bais);
-      FixedFileTrailer t2 = new FixedFileTrailer(version);
+      FixedFileTrailer t2 = new FixedFileTrailer(version, 
+                              HFileBlock.MINOR_VERSION_NO_CHECKSUM);
       t2.deserialize(dis);
       assertEquals(-1, bais.read()); // Ensure we have read everything.
       checkLoadedTrailer(version, t, t2);
@@ -191,7 +193,7 @@ private void writeTrailer(Path trailerPath, FixedFileTrailer t,
 
   private void checkLoadedTrailer(int version, FixedFileTrailer expected,
       FixedFileTrailer loaded) throws IOException {
-    assertEquals(version, loaded.getVersion());
+    assertEquals(version, loaded.getMajorVersion());
     assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
 
     assertEquals(Math.min(expected.getEntryCount(),
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index e8b7df07591c..6456ccbee3db 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -48,9 +48,11 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.DoubleOutputStream;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.io.compress.CompressionOutputStream;
@@ -102,16 +104,16 @@ public static Collection<Object[]> parameters() {
 
   @Before
   public void setUp() throws IOException {
-    fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    fs = HFileSystem.get(TEST_UTIL.getConfiguration());
   }
 
-  public void writeTestBlockContents(DataOutputStream dos) throws IOException {
+  static void writeTestBlockContents(DataOutputStream dos) throws IOException {
     // This compresses really well.
     for (int i = 0; i < 1000; ++i)
       dos.writeInt(i / 100);
   }
 
-  private int writeTestKeyValues(OutputStream dos, int seed)
+ static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
       throws IOException {
     List<KeyValue> keyValues = new ArrayList<KeyValue>();
     Random randomizer = new Random(42l + seed); // just any fixed number
@@ -191,22 +193,24 @@ public byte[] createTestV1Block(Compression.Algorithm algo)
     return baos.toByteArray();
   }
 
-  private byte[] createTestV2Block(Compression.Algorithm algo)
-      throws IOException {
+  static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
+      boolean includesMemstoreTS) throws IOException {
     final BlockType blockType = BlockType.DATA;
     HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-        includesMemstoreTS);
+        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        HFile.DEFAULT_BYTES_PER_CHECKSUM);
     DataOutputStream dos = hbw.startWriting(blockType);
     writeTestBlockContents(dos);
-    byte[] headerAndData = hbw.getHeaderAndData();
+    byte[] headerAndData = hbw.getHeaderAndDataForTest();
     assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
     hbw.releaseCompressor();
-    return headerAndData;
+    return hbw;
   }
 
   public String createTestBlockStr(Compression.Algorithm algo,
       int correctLength) throws IOException {
-    byte[] testV2Block = createTestV2Block(algo);
+    HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
+    byte[] testV2Block = hbw.getHeaderAndDataForTest();
     int osOffset = HFileBlock.HEADER_SIZE + 9;
     if (testV2Block.length == correctLength) {
       // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
@@ -221,14 +225,16 @@ public String createTestBlockStr(Compression.Algorithm algo,
 
   @Test
   public void testNoCompression() throws IOException {
-    assertEquals(4000 + HFileBlock.HEADER_SIZE, createTestV2Block(NONE).length);
+    assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
+                 getBlockForCaching().getUncompressedSizeWithoutHeader());
   }
 
   @Test
   public void testGzipCompression() throws IOException {
     final String correctTestBlockStr =
-        "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+        "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
             + "\\xFF\\xFF\\xFF\\xFF"
+            + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
             // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
             + "\\x1F\\x8B"  // gzip magic signature
             + "\\x08"  // Compression method: 8 = "deflate"
@@ -240,8 +246,9 @@ public void testGzipCompression() throws IOException {
             + "\\x03"
             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
-            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
-    final int correctGzipBlockLength = 82;
+            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00"
+            + "\\xAB\\x85g\\x91"; //  4 byte checksum
+    final int correctGzipBlockLength = 95;
     assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
         correctGzipBlockLength));
   }
@@ -285,11 +292,14 @@ public void testReaderV1() throws IOException {
   public void testReaderV2() throws IOException {
     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
       for (boolean pread : new boolean[] { false, true }) {
+          LOG.info("testReaderV2: Compression algorithm: " + algo + 
+                   ", pread=" + pread);
         Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
             + algo);
         FSDataOutputStream os = fs.create(path);
         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-            includesMemstoreTS);
+            includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+            HFile.DEFAULT_BYTES_PER_CHECKSUM);
         long totalSize = 0;
         for (int blockId = 0; blockId < 2; ++blockId) {
           DataOutputStream dos = hbw.startWriting(BlockType.DATA);
@@ -305,16 +315,19 @@ public void testReaderV2() throws IOException {
             totalSize);
         HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
         is.close();
+        assertEquals(0, HFile.getChecksumFailuresCount());
 
         b.sanityCheck();
         assertEquals(4936, b.getUncompressedSizeWithoutHeader());
-        assertEquals(algo == GZ ? 2173 : 4936, b.getOnDiskSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
         String blockStr = b.toString();
 
         if (algo == GZ) {
           is = fs.open(path);
           hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
-          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE, -1, pread);
+          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE +
+                                b.totalChecksumBytes(), -1, pread);
           assertEquals(blockStr, b.toString());
           int wrongCompressedSize = 2172;
           try {
@@ -351,13 +364,15 @@ public void testDataBlockEncoding() throws IOException {
           HFileDataBlockEncoder dataBlockEncoder =
               new HFileDataBlockEncoderImpl(encoding);
           HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
-              includesMemstoreTS);
+              includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+              HFile.DEFAULT_BYTES_PER_CHECKSUM);
           long totalSize = 0;
           final List<Integer> encodedSizes = new ArrayList<Integer>();
           final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
           for (int blockId = 0; blockId < numBlocks; ++blockId) {
-            writeEncodedBlock(encoding, hbw, encodedSizes, encodedBlocks,
-                blockId);
+            DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+            writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
+                blockId, includesMemstoreTS);
 
             hbw.writeHeaderAndData(os);
             totalSize += hbw.getOnDiskSizeWithHeader();
@@ -374,6 +389,7 @@ public void testDataBlockEncoding() throws IOException {
           int pos = 0;
           for (int blockId = 0; blockId < numBlocks; ++blockId) {
             b = hbr.readBlockData(pos, -1, -1, pread);
+            assertEquals(0, HFile.getChecksumFailuresCount());
             b.sanityCheck();
             pos += b.getOnDiskSizeWithHeader();
 
@@ -401,16 +417,16 @@ public void testDataBlockEncoding() throws IOException {
     }
   }
 
-  private void writeEncodedBlock(DataBlockEncoding encoding,
-      HFileBlock.Writer hbw, final List<Integer> encodedSizes,
-      final List<ByteBuffer> encodedBlocks, int blockId) throws IOException {
-    DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+  static void writeEncodedBlock(DataBlockEncoding encoding,
+      DataOutputStream dos, final List<Integer> encodedSizes,
+      final List<ByteBuffer> encodedBlocks, int blockId, 
+      boolean includesMemstoreTS) throws IOException {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DoubleOutputStream doubleOutputStream =
         new DoubleOutputStream(dos, baos);
 
     final int rawBlockSize = writeTestKeyValues(doubleOutputStream,
-        blockId);
+        blockId, includesMemstoreTS);
 
     ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
     rawBuf.rewind();
@@ -434,7 +450,7 @@ private void writeEncodedBlock(DataBlockEncoding encoding,
     encodedBlocks.add(encodedBuf);
   }
 
-  private void assertBuffersEqual(ByteBuffer expectedBuffer,
+  static void assertBuffersEqual(ByteBuffer expectedBuffer,
       ByteBuffer actualBuffer, Compression.Algorithm compression,
       DataBlockEncoding encoding, boolean pread) {
     if (!actualBuffer.equals(expectedBuffer)) {
@@ -471,7 +487,9 @@ public void testPreviousOffset() throws IOException {
       for (boolean pread : BOOLEAN_VALUES) {
         for (boolean cacheOnWrite : BOOLEAN_VALUES) {
           Random rand = defaultRandom();
-          LOG.info("Compression algorithm: " + algo + ", pread=" + pread);
+          LOG.info("testPreviousOffset:Compression algorithm: " + algo + 
+                   ", pread=" + pread +
+                   ", cacheOnWrite=" + cacheOnWrite);
           Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
           List<Long> expectedOffsets = new ArrayList<Long>();
           List<Long> expectedPrevOffsets = new ArrayList<Long>();
@@ -522,17 +540,23 @@ public void testPreviousOffset() throws IOException {
                 b2.getUncompressedSizeWithoutHeader());
             assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
             assertEquals(curOffset, b2.getOffset());
+            assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
+            assertEquals(b.getOnDiskDataSizeWithHeader(), 
+                         b2.getOnDiskDataSizeWithHeader());
+            assertEquals(0, HFile.getChecksumFailuresCount());
 
             curOffset += b.getOnDiskSizeWithHeader();
 
             if (cacheOnWrite) {
               // In the cache-on-write mode we store uncompressed bytes so we
               // can compare them to what was read by the block reader.
-
+              // b's buffer has header + data + checksum while 
+              // expectedContents have header + data only
               ByteBuffer bufRead = b.getBufferWithHeader();
               ByteBuffer bufExpected = expectedContents.get(i);
               boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
-                  bufRead.arrayOffset(), bufRead.limit(),
+                  bufRead.arrayOffset(), 
+                  bufRead.limit() - b.totalChecksumBytes(),
                   bufExpected.array(), bufExpected.arrayOffset(),
                   bufExpected.limit()) == 0;
               String wrongBytesMsg = "";
@@ -541,15 +565,26 @@ public void testPreviousOffset() throws IOException {
                 // Optimization: only construct an error message in case we
                 // will need it.
                 wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
-                    + algo + ", pread=" + pread + "):\n";
+                    + algo + ", pread=" + pread
+                    + ", cacheOnWrite=" + cacheOnWrite + "):\n";
                 wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
                     bufExpected.arrayOffset(), Math.min(32,
                         bufExpected.limit()))
                     + ", actual:\n"
                     + Bytes.toStringBinary(bufRead.array(),
                         bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
+                if (detailedLogging) {
+                  LOG.warn("expected header" + 
+                           HFileBlock.toStringHeader(bufExpected) +
+                           "\nfound    header" + 
+                           HFileBlock.toStringHeader(bufRead));
+                  LOG.warn("bufread offset " + bufRead.arrayOffset() +
+                           " limit " + bufRead.limit() +
+                           " expected offset " + bufExpected.arrayOffset() +
+                           " limit " + bufExpected.limit());
+                  LOG.warn(wrongBytesMsg);
+                }
               }
-
               assertTrue(wrongBytesMsg, bytesAreCorrect);
             }
           }
@@ -672,10 +707,12 @@ private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
     boolean cacheOnWrite = expectedContents != null;
     FSDataOutputStream os = fs.create(path);
     HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
-        includesMemstoreTS);
+        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        HFile.DEFAULT_BYTES_PER_CHECKSUM);
     Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
     long totalSize = 0;
     for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
+      long pos = os.getPos();
       int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
       if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
         blockTypeOrdinal = BlockType.DATA.ordinal();
@@ -706,9 +743,9 @@ private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
         expectedContents.add(hbw.getUncompressedBufferWithHeader());
 
       if (detailedLogging) {
-        LOG.info("Writing block #" + i + " of type " + bt
+        LOG.info("Written block #" + i + " of type " + bt
             + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
-            + " at offset " + os.getPos());
+            + " at offset " + pos);
       }
     }
     os.close();
@@ -730,7 +767,9 @@ public void testBlockHeapSize() {
       byte[] byteArr = new byte[HFileBlock.HEADER_SIZE + size];
       ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
       HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
-          HFileBlock.FILL_HEADER, -1, includesMemstoreTS);
+          HFileBlock.FILL_HEADER, -1, includesMemstoreTS, 
+          HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
+          0);
       long byteBufferExpectedSize =
           ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
               + HFileBlock.HEADER_SIZE + size);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
new file mode 100644
index 000000000000..4d9b1580d4cd
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
@@ -0,0 +1,806 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.DoubleOutputStream;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.Pair;
+import com.google.common.base.Preconditions;
+
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * This class has unit tests to prove that older versions of
+ * HFiles (without checksums) are compatible with current readers.
+ */
+@Category(MediumTests.class)
+@RunWith(Parameterized.class)
+public class TestHFileBlockCompatibility {
+  // change this value to activate more logs
+  private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
+
+  private static final Log LOG = LogFactory.getLog(TestHFileBlockCompatibility.class);
+
+  private static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
+      NONE, GZ };
+
+  // The mnior version for pre-checksum files
+  private static int MINOR_VERSION = 0;
+
+  private static final HBaseTestingUtility TEST_UTIL =
+    new HBaseTestingUtility();
+  private HFileSystem fs;
+  private int uncompressedSizeV1;
+
+  private final boolean includesMemstoreTS;
+
+  public TestHFileBlockCompatibility(boolean includesMemstoreTS) {
+    this.includesMemstoreTS = includesMemstoreTS;
+  }
+
+  @Parameters
+  public static Collection<Object[]> parameters() {
+    return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
+  }
+
+  @Before
+  public void setUp() throws IOException {
+    fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
+  }
+
+  public byte[] createTestV1Block(Compression.Algorithm algo)
+      throws IOException {
+    Compressor compressor = algo.getCompressor();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
+    DataOutputStream dos = new DataOutputStream(os);
+    BlockType.META.write(dos); // Let's make this a meta block.
+    TestHFileBlock.writeTestBlockContents(dos);
+    uncompressedSizeV1 = dos.size();
+    dos.flush();
+    algo.returnCompressor(compressor);
+    return baos.toByteArray();
+  }
+
+  private Writer createTestV2Block(Compression.Algorithm algo)
+      throws IOException {
+    final BlockType blockType = BlockType.DATA;
+    Writer hbw = new Writer(algo, null,
+        includesMemstoreTS);
+    DataOutputStream dos = hbw.startWriting(blockType);
+    TestHFileBlock.writeTestBlockContents(dos);
+    byte[] headerAndData = hbw.getHeaderAndData();
+    assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
+    hbw.releaseCompressor();
+    return hbw;
+  }
+
+ private String createTestBlockStr(Compression.Algorithm algo,
+      int correctLength) throws IOException {
+    Writer hbw = createTestV2Block(algo);
+    byte[] testV2Block = hbw.getHeaderAndData();
+    int osOffset = HFileBlock.HEADER_SIZE_NO_CHECKSUM + 9;
+    if (testV2Block.length == correctLength) {
+      // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
+      // variations across operating systems.
+      // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
+      testV2Block[osOffset] = 3;
+    }   
+    return Bytes.toStringBinary(testV2Block);
+  }
+
+  @Test
+  public void testNoCompression() throws IOException {
+    assertEquals(4000, createTestV2Block(NONE).getBlockForCaching().
+                       getUncompressedSizeWithoutHeader());
+  }
+
+  @Test
+  public void testGzipCompression() throws IOException {
+    final String correctTestBlockStr =
+        "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+            + "\\xFF\\xFF\\xFF\\xFF"
+            // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
+            + "\\x1F\\x8B"  // gzip magic signature
+            + "\\x08"  // Compression method: 8 = "deflate"
+            + "\\x00"  // Flags
+            + "\\x00\\x00\\x00\\x00"  // mtime
+            + "\\x00"  // XFL (extra flags)
+            // OS (0 = FAT filesystems, 3 = Unix). However, this field
+            // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
+            + "\\x03"
+            + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
+            + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
+            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
+    final int correctGzipBlockLength = 82;
+    assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
+        correctGzipBlockLength));
+  }
+
+  @Test
+  public void testReaderV1() throws IOException {
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+        byte[] block = createTestV1Block(algo);
+        Path path = new Path(TEST_UTIL.getDataTestDir(),
+          "blocks_v1_"+ algo);
+        LOG.info("Creating temporary file at " + path);
+        FSDataOutputStream os = fs.create(path);
+        int totalSize = 0;
+        int numBlocks = 50;
+        for (int i = 0; i < numBlocks; ++i) {
+          os.write(block);
+          totalSize += block.length;
+        }
+        os.close();
+
+        FSDataInputStream is = fs.open(path);
+        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo,
+            totalSize);
+        HFileBlock b;
+        int numBlocksRead = 0;
+        long pos = 0;
+        while (pos < totalSize) {
+          b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread);
+          b.sanityCheck();
+          pos += block.length;
+          numBlocksRead++;
+        }
+        assertEquals(numBlocks, numBlocksRead);
+        is.close();
+      }
+    }
+  }
+
+  @Test
+  public void testReaderV2() throws IOException {
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+          LOG.info("testReaderV2: Compression algorithm: " + algo + 
+                   ", pread=" + pread);
+        Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+            + algo);
+        FSDataOutputStream os = fs.create(path);
+        Writer hbw = new Writer(algo, null,
+            includesMemstoreTS);
+        long totalSize = 0;
+        for (int blockId = 0; blockId < 2; ++blockId) {
+          DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+          for (int i = 0; i < 1234; ++i)
+            dos.writeInt(i);
+          hbw.writeHeaderAndData(os);
+          totalSize += hbw.getOnDiskSizeWithHeader();
+        }
+        os.close();
+
+        FSDataInputStream is = fs.open(path);
+        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, is, algo,
+            totalSize, MINOR_VERSION, fs, path);
+        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+        is.close();
+
+        b.sanityCheck();
+        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+        String blockStr = b.toString();
+
+        if (algo == GZ) {
+          is = fs.open(path);
+          hbr = new HFileBlock.FSReaderV2(is, is, algo, totalSize, MINOR_VERSION,
+                                          fs, path);
+          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_NO_CHECKSUM +
+                                b.totalChecksumBytes(), -1, pread);
+          assertEquals(blockStr, b.toString());
+          int wrongCompressedSize = 2172;
+          try {
+            b = hbr.readBlockData(0, wrongCompressedSize
+                + HFileBlock.HEADER_SIZE_NO_CHECKSUM, -1, pread);
+            fail("Exception expected");
+          } catch (IOException ex) {
+            String expectedPrefix = "On-disk size without header provided is "
+                + wrongCompressedSize + ", but block header contains "
+                + b.getOnDiskSizeWithoutHeader() + ".";
+            assertTrue("Invalid exception message: '" + ex.getMessage()
+                + "'.\nMessage is expected to start with: '" + expectedPrefix
+                + "'", ex.getMessage().startsWith(expectedPrefix));
+          }
+          is.close();
+        }
+      }
+    }
+  }
+
+  /**
+   * Test encoding/decoding data blocks.
+   * @throws IOException a bug or a problem with temporary files.
+   */
+  @Test
+  public void testDataBlockEncoding() throws IOException {
+    final int numBlocks = 5;
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+        for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
+          LOG.info("testDataBlockEncoding algo " + algo + 
+                   " pread = " + pread +
+                   " encoding " + encoding);
+          Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+              + algo + "_" + encoding.toString());
+          FSDataOutputStream os = fs.create(path);
+          HFileDataBlockEncoder dataBlockEncoder =
+              new HFileDataBlockEncoderImpl(encoding);
+          Writer hbw = new Writer(algo, dataBlockEncoder,
+              includesMemstoreTS);
+          long totalSize = 0;
+          final List<Integer> encodedSizes = new ArrayList<Integer>();
+          final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
+          for (int blockId = 0; blockId < numBlocks; ++blockId) {
+            DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+            TestHFileBlock.writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
+                blockId, includesMemstoreTS);
+
+            hbw.writeHeaderAndData(os);
+            totalSize += hbw.getOnDiskSizeWithHeader();
+          }
+          os.close();
+
+          FSDataInputStream is = fs.open(path);
+          HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, is, algo,
+              totalSize, MINOR_VERSION, fs, path);
+          hbr.setDataBlockEncoder(dataBlockEncoder);
+          hbr.setIncludesMemstoreTS(includesMemstoreTS);
+
+          HFileBlock b;
+          int pos = 0;
+          for (int blockId = 0; blockId < numBlocks; ++blockId) {
+            b = hbr.readBlockData(pos, -1, -1, pread);
+            b.sanityCheck();
+            pos += b.getOnDiskSizeWithHeader();
+
+            assertEquals((int) encodedSizes.get(blockId),
+                b.getUncompressedSizeWithoutHeader());
+            ByteBuffer actualBuffer = b.getBufferWithoutHeader();
+            if (encoding != DataBlockEncoding.NONE) {
+              // We expect a two-byte big-endian encoding id.
+              assertEquals(0, actualBuffer.get(0));
+              assertEquals(encoding.getId(), actualBuffer.get(1));
+              actualBuffer.position(2);
+              actualBuffer = actualBuffer.slice();
+            }
+
+            ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
+            expectedBuffer.rewind();
+
+            // test if content matches, produce nice message
+            TestHFileBlock.assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
+                pread);
+          }
+          is.close();
+        }
+      }
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+
+  /**
+   * This is the version of the HFileBlock.Writer that is used to 
+   * create V2 blocks with minor version 0. These blocks do not 
+   * have hbase-level checksums. The code is here to test 
+   * backward compatibility. The reason we do not inherit from 
+   * HFileBlock.Writer is because we never ever want to change the code
+   * in this class but the code in HFileBlock.Writer will continually 
+   * evolve.
+   */
+  public static final class Writer {
+
+    // These constants are as they were in minorVersion 0.
+    private static final int HEADER_SIZE = HFileBlock.HEADER_SIZE_NO_CHECKSUM;
+    private static final boolean DONT_FILL_HEADER = HFileBlock.DONT_FILL_HEADER;
+    private static final byte[] DUMMY_HEADER = 
+      HFileBlock.DUMMY_HEADER_NO_CHECKSUM;
+
+    private enum State {
+      INIT,
+      WRITING,
+      BLOCK_READY
+    };
+
+    /** Writer state. Used to ensure the correct usage protocol. */
+    private State state = State.INIT;
+
+    /** Compression algorithm for all blocks this instance writes. */
+    private final Compression.Algorithm compressAlgo;
+
+    /** Data block encoder used for data blocks */
+    private final HFileDataBlockEncoder dataBlockEncoder;
+
+    /**
+     * The stream we use to accumulate data in uncompressed format for each
+     * block. We reset this stream at the end of each block and reuse it. The
+     * header is written as the first {@link #HEADER_SIZE} bytes into this
+     * stream.
+     */
+    private ByteArrayOutputStream baosInMemory;
+
+    /** Compressor, which is also reused between consecutive blocks. */
+    private Compressor compressor;
+
+    /** Compression output stream */
+    private CompressionOutputStream compressionStream;
+
+    /** Underlying stream to write compressed bytes to */
+    private ByteArrayOutputStream compressedByteStream;
+
+    /**
+     * Current block type. Set in {@link #startWriting(BlockType)}. Could be
+     * changed in {@link #encodeDataBlockForDisk()} from {@link BlockType#DATA}
+     * to {@link BlockType#ENCODED_DATA}.
+     */
+    private BlockType blockType;
+
+    /**
+     * A stream that we write uncompressed bytes to, which compresses them and
+     * writes them to {@link #baosInMemory}.
+     */
+    private DataOutputStream userDataStream;
+
+    /**
+     * Bytes to be written to the file system, including the header. Compressed
+     * if compression is turned on.
+     */
+    private byte[] onDiskBytesWithHeader;
+
+    /**
+     * Valid in the READY state. Contains the header and the uncompressed (but
+     * potentially encoded, if this is a data block) bytes, so the length is
+     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE}.
+     */
+    private byte[] uncompressedBytesWithHeader;
+
+    /**
+     * Current block's start offset in the {@link HFile}. Set in
+     * {@link #writeHeaderAndData(FSDataOutputStream)}.
+     */
+    private long startOffset;
+
+    /**
+     * Offset of previous block by block type. Updated when the next block is
+     * started.
+     */
+    private long[] prevOffsetByType;
+
+    /** The offset of the previous block of the same type */
+    private long prevOffset;
+
+    /** Whether we are including memstore timestamp after every key/value */
+    private boolean includesMemstoreTS;
+
+    /**
+     * @param compressionAlgorithm compression algorithm to use
+     * @param dataBlockEncoderAlgo data block encoding algorithm to use
+     */
+    public Writer(Compression.Algorithm compressionAlgorithm,
+          HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
+      compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
+      this.dataBlockEncoder = dataBlockEncoder != null
+          ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
+
+      baosInMemory = new ByteArrayOutputStream();
+      if (compressAlgo != NONE) {
+        compressor = compressionAlgorithm.getCompressor();
+        compressedByteStream = new ByteArrayOutputStream();
+        try {
+          compressionStream =
+              compressionAlgorithm.createPlainCompressionStream(
+                  compressedByteStream, compressor);
+        } catch (IOException e) {
+          throw new RuntimeException("Could not create compression stream " +
+              "for algorithm " + compressionAlgorithm, e);
+        }
+      }
+
+      prevOffsetByType = new long[BlockType.values().length];
+      for (int i = 0; i < prevOffsetByType.length; ++i)
+        prevOffsetByType[i] = -1;
+
+      this.includesMemstoreTS = includesMemstoreTS;
+    }
+
+    /**
+     * Starts writing into the block. The previous block's data is discarded.
+     *
+     * @return the stream the user can write their data into
+     * @throws IOException
+     */
+    public DataOutputStream startWriting(BlockType newBlockType)
+        throws IOException {
+      if (state == State.BLOCK_READY && startOffset != -1) {
+        // We had a previous block that was written to a stream at a specific
+        // offset. Save that offset as the last offset of a block of that type.
+        prevOffsetByType[blockType.getId()] = startOffset;
+      }
+
+      startOffset = -1;
+      blockType = newBlockType;
+
+      baosInMemory.reset();
+      baosInMemory.write(DUMMY_HEADER);
+
+      state = State.WRITING;
+
+      // We will compress it later in finishBlock()
+      userDataStream = new DataOutputStream(baosInMemory);
+      return userDataStream;
+    }
+
+    /**
+     * Returns the stream for the user to write to. The block writer takes care
+     * of handling compression and buffering for caching on write. Can only be
+     * called in the "writing" state.
+     *
+     * @return the data output stream for the user to write to
+     */
+    DataOutputStream getUserDataStream() {
+      expectState(State.WRITING);
+      return userDataStream;
+    }
+
+    /**
+     * Transitions the block writer from the "writing" state to the "block
+     * ready" state.  Does nothing if a block is already finished.
+     */
+    private void ensureBlockReady() throws IOException {
+      Preconditions.checkState(state != State.INIT,
+          "Unexpected state: " + state);
+
+      if (state == State.BLOCK_READY)
+        return;
+
+      // This will set state to BLOCK_READY.
+      finishBlock();
+    }
+
+    /**
+     * An internal method that flushes the compressing stream (if using
+     * compression), serializes the header, and takes care of the separate
+     * uncompressed stream for caching on write, if applicable. Sets block
+     * write state to "block ready".
+     */
+    private void finishBlock() throws IOException {
+      userDataStream.flush();
+
+      // This does an array copy, so it is safe to cache this byte array.
+      uncompressedBytesWithHeader = baosInMemory.toByteArray();
+      LOG.warn("Writer.finishBlock user data size with header before compression " +
+               uncompressedBytesWithHeader.length);
+      prevOffset = prevOffsetByType[blockType.getId()];
+
+      // We need to set state before we can package the block up for
+      // cache-on-write. In a way, the block is ready, but not yet encoded or
+      // compressed.
+      state = State.BLOCK_READY;
+      encodeDataBlockForDisk();
+
+      doCompression();
+      putHeader(uncompressedBytesWithHeader, 0, onDiskBytesWithHeader.length,
+          uncompressedBytesWithHeader.length);
+    }
+
+    /**
+     * Do compression if it is enabled, or re-use the uncompressed buffer if
+     * it is not. Fills in the compressed block's header if doing compression.
+     */
+    private void doCompression() throws IOException {
+      // do the compression
+      if (compressAlgo != NONE) {
+        compressedByteStream.reset();
+        compressedByteStream.write(DUMMY_HEADER);
+
+        compressionStream.resetState();
+
+        compressionStream.write(uncompressedBytesWithHeader, HEADER_SIZE,
+            uncompressedBytesWithHeader.length - HEADER_SIZE);
+
+        compressionStream.flush();
+        compressionStream.finish();
+
+        onDiskBytesWithHeader = compressedByteStream.toByteArray();
+        putHeader(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length);
+      } else {
+        onDiskBytesWithHeader = uncompressedBytesWithHeader;
+      }
+    }
+
+    /**
+     * Encodes this block if it is a data block and encoding is turned on in
+     * {@link #dataBlockEncoder}.
+     */
+    private void encodeDataBlockForDisk() throws IOException {
+      if (blockType != BlockType.DATA) {
+        return; // skip any non-data block
+      }
+
+      // do data block encoding, if data block encoder is set
+      ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader,
+          HEADER_SIZE, uncompressedBytesWithHeader.length -
+          HEADER_SIZE).slice();
+      Pair<ByteBuffer, BlockType> encodingResult =
+          dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
+              includesMemstoreTS, DUMMY_HEADER);
+
+      BlockType encodedBlockType = encodingResult.getSecond();
+      if (encodedBlockType == BlockType.ENCODED_DATA) {
+        uncompressedBytesWithHeader = encodingResult.getFirst().array();
+        blockType = BlockType.ENCODED_DATA;
+      } else {
+        // There is no encoding configured. Do some extra sanity-checking.
+        if (encodedBlockType != BlockType.DATA) {
+          throw new IOException("Unexpected block type coming out of data " +
+              "block encoder: " + encodedBlockType);
+        }
+        if (userDataStream.size() !=
+            uncompressedBytesWithHeader.length - HEADER_SIZE) {
+          throw new IOException("Uncompressed size mismatch: "
+              + userDataStream.size() + " vs. "
+              + (uncompressedBytesWithHeader.length - HEADER_SIZE));
+        }
+      }
+    }
+
+    /**
+     * Put the header into the given byte array at the given offset.
+     * @param onDiskSize size of the block on disk
+     * @param uncompressedSize size of the block after decompression (but
+     *          before optional data block decoding)
+     */
+    private void putHeader(byte[] dest, int offset, int onDiskSize,
+        int uncompressedSize) {
+      offset = blockType.put(dest, offset);
+      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
+      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
+      Bytes.putLong(dest, offset, prevOffset);
+    }
+
+    /**
+     * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
+     * the offset of this block so that it can be referenced in the next block
+     * of the same type.
+     *
+     * @param out
+     * @throws IOException
+     */
+    public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
+      long offset = out.getPos();
+      if (startOffset != -1 && offset != startOffset) {
+        throw new IOException("A " + blockType + " block written to a "
+            + "stream twice, first at offset " + startOffset + ", then at "
+            + offset);
+      }
+      startOffset = offset;
+
+      writeHeaderAndData((DataOutputStream) out);
+    }
+
+    /**
+     * Writes the header and the compressed data of this block (or uncompressed
+     * data when not using compression) into the given stream. Can be called in
+     * the "writing" state or in the "block ready" state. If called in the
+     * "writing" state, transitions the writer to the "block ready" state.
+     *
+     * @param out the output stream to write the
+     * @throws IOException
+     */
+    private void writeHeaderAndData(DataOutputStream out) throws IOException {
+      ensureBlockReady();
+      out.write(onDiskBytesWithHeader);
+    }
+
+    /**
+     * Returns the header or the compressed data (or uncompressed data when not
+     * using compression) as a byte array. Can be called in the "writing" state
+     * or in the "block ready" state. If called in the "writing" state,
+     * transitions the writer to the "block ready" state.
+     *
+     * @return header and data as they would be stored on disk in a byte array
+     * @throws IOException
+     */
+    public byte[] getHeaderAndData() throws IOException {
+      ensureBlockReady();
+      return onDiskBytesWithHeader;
+    }
+
+    /**
+     * Releases the compressor this writer uses to compress blocks into the
+     * compressor pool. Needs to be called before the writer is discarded.
+     */
+    public void releaseCompressor() {
+      if (compressor != null) {
+        compressAlgo.returnCompressor(compressor);
+        compressor = null;
+      }
+    }
+
+    /**
+     * Returns the on-disk size of the data portion of the block. This is the
+     * compressed size if compression is enabled. Can only be called in the
+     * "block ready" state. Header is not compressed, and its size is not
+     * included in the return value.
+     *
+     * @return the on-disk size of the block, not including the header.
+     */
+    public int getOnDiskSizeWithoutHeader() {
+      expectState(State.BLOCK_READY);
+      return onDiskBytesWithHeader.length - HEADER_SIZE;
+    }
+
+    /**
+     * Returns the on-disk size of the block. Can only be called in the
+     * "block ready" state.
+     *
+     * @return the on-disk size of the block ready to be written, including the
+     *         header size
+     */
+    public int getOnDiskSizeWithHeader() {
+      expectState(State.BLOCK_READY);
+      return onDiskBytesWithHeader.length;
+    }
+
+    /**
+     * The uncompressed size of the block data. Does not include header size.
+     */
+    public int getUncompressedSizeWithoutHeader() {
+      expectState(State.BLOCK_READY);
+      return uncompressedBytesWithHeader.length - HEADER_SIZE;
+    }
+
+    /**
+     * The uncompressed size of the block data, including header size.
+     */
+    public int getUncompressedSizeWithHeader() {
+      expectState(State.BLOCK_READY);
+      return uncompressedBytesWithHeader.length;
+    }
+
+    /** @return true if a block is being written  */
+    public boolean isWriting() {
+      return state == State.WRITING;
+    }
+
+    /**
+     * Returns the number of bytes written into the current block so far, or
+     * zero if not writing the block at the moment. Note that this will return
+     * zero in the "block ready" state as well.
+     *
+     * @return the number of bytes written
+     */
+    public int blockSizeWritten() {
+      if (state != State.WRITING)
+        return 0;
+      return userDataStream.size();
+    }
+
+    /**
+     * Returns the header followed by the uncompressed data, even if using
+     * compression. This is needed for storing uncompressed blocks in the block
+     * cache. Can be called in the "writing" state or the "block ready" state.
+     *
+     * @return uncompressed block bytes for caching on write
+     */
+    private byte[] getUncompressedDataWithHeader() {
+      expectState(State.BLOCK_READY);
+
+      return uncompressedBytesWithHeader;
+    }
+
+    private void expectState(State expectedState) {
+      if (state != expectedState) {
+        throw new IllegalStateException("Expected state: " + expectedState +
+            ", actual state: " + state);
+      }
+    }
+
+    /**
+     * Similar to {@link #getUncompressedBufferWithHeader()} but returns a byte
+     * buffer.
+     *
+     * @return uncompressed block for caching on write in the form of a buffer
+     */
+    public ByteBuffer getUncompressedBufferWithHeader() {
+      byte[] b = getUncompressedDataWithHeader();
+      return ByteBuffer.wrap(b, 0, b.length);
+    }
+
+    /**
+     * Takes the given {@link BlockWritable} instance, creates a new block of
+     * its appropriate type, writes the writable into this block, and flushes
+     * the block into the output stream. The writer is instructed not to buffer
+     * uncompressed bytes for cache-on-write.
+     *
+     * @param bw the block-writable object to write as a block
+     * @param out the file system output stream
+     * @throws IOException
+     */
+    public void writeBlock(BlockWritable bw, FSDataOutputStream out)
+        throws IOException {
+      bw.writeToBlock(startWriting(bw.getBlockType()));
+      writeHeaderAndData(out);
+    }
+
+    /**
+     * Creates a new HFileBlock. 
+     */
+    public HFileBlock getBlockForCaching() {
+      return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
+          getUncompressedSizeWithoutHeader(), prevOffset,
+          getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
+          includesMemstoreTS, MINOR_VERSION, 0, ChecksumType.NULL.getCode(),
+          getOnDiskSizeWithoutHeader());
+    }
+  }
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index b7d0665f225f..3f36e64cde25 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -110,7 +111,7 @@ public void setUp() throws IOException {
     // This test requires at least HFile format version 2.
     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
 
-    fs = FileSystem.get(conf);
+    fs = HFileSystem.get(conf);
   }
 
   @Test
@@ -215,7 +216,8 @@ public void readIndex() throws IOException {
   private void writeWholeIndex() throws IOException {
     assertEquals(0, keys.size());
     HFileBlock.Writer hbw = new HFileBlock.Writer(compr, null,
-        includesMemstoreTS);
+        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        HFile.DEFAULT_BYTES_PER_CHECKSUM);
     FSDataOutputStream outputStream = fs.create(path);
     HFileBlockIndex.BlockIndexWriter biw =
         new HFileBlockIndex.BlockIndexWriter(hbw, null, null);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
index e1a57e52d225..613ad7da816a 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hbase.io.encoding.RedundantKVGenerator;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.Pair;
 import org.junit.After;
 import org.junit.Before;
@@ -123,12 +124,14 @@ public void testEncodingWritePath() {
     HFileBlock block = getSampleHFileBlock();
     Pair<ByteBuffer, BlockType> result =
         blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
-            includesMemstoreTS);
+            includesMemstoreTS, HFileBlock.DUMMY_HEADER);
 
     int size = result.getFirst().limit() - HFileBlock.HEADER_SIZE;
     HFileBlock blockOnDisk = new HFileBlock(result.getSecond(),
         size, size, -1, result.getFirst(), HFileBlock.FILL_HEADER, 0,
-        includesMemstoreTS);
+        includesMemstoreTS, block.getMinorVersion(),
+        block.getBytesPerChecksum(), block.getChecksumType(),
+        block.getOnDiskDataSizeWithHeader());
 
     if (blockEncoder.getEncodingOnDisk() !=
         DataBlockEncoding.NONE) {
@@ -158,7 +161,8 @@ private HFileBlock getSampleHFileBlock() {
     keyValues.rewind();
     buf.put(keyValues);
     HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
-        HFileBlock.FILL_HEADER, 0, includesMemstoreTS);
+        HFileBlock.FILL_HEADER, 0, includesMemstoreTS, 
+        HFileReaderV2.MAX_MINOR_VERSION, 0, ChecksumType.NULL.getCode(), 0);
     UNKNOWN_TABLE_AND_CF.passSchemaMetricsTo(b);
     return b;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
index c745ffcb9e1f..d313ddd435d4 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
@@ -76,7 +76,7 @@ public void testReadingExistingVersion1HFile() throws IOException {
 
     assertEquals(N, reader.getEntries());
     assertEquals(N, trailer.getEntryCount());
-    assertEquals(1, trailer.getVersion());
+    assertEquals(1, trailer.getMajorVersion());
     assertEquals(Compression.Algorithm.GZ, trailer.getCompressionCodec());
 
     for (boolean pread : new boolean[] { false, true }) {
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
index 365d9fa471b8..e3b18c91c924 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
@@ -123,7 +123,7 @@ public void testHFileFormatV2() throws IOException {
     FixedFileTrailer trailer =
         FixedFileTrailer.readFromStream(fsdis, fileSize);
 
-    assertEquals(2, trailer.getVersion());
+    assertEquals(2, trailer.getMajorVersion());
     assertEquals(ENTRY_COUNT, trailer.getEntryCount());
 
     HFileBlock.FSReader blockReader =
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
index cbcbffa54b79..f1cd8a02ca5c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
@@ -189,6 +189,8 @@ public boolean run(String[] args) throws IOException {
             .withCompression(compr)
             .withBloomType(bloomType)
             .withMaxKeyCount(numKV)
+            .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
+            .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
             .build();
 
     rand = new Random();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java b/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
index e4d849331cb7..ebc5373e2248 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
@@ -356,6 +356,8 @@ public void runMergeWorkload() throws IOException {
             .withDataBlockEncoder(dataBlockEncoder)
             .withBloomType(bloomType)
             .withMaxKeyCount(maxKeyCount)
+            .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
+            .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
             .build();
 
     StatisticsPrinter statsPrinter = new StatisticsPrinter();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
index 8859793c1bdf..aeaf62550684 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
@@ -298,6 +298,8 @@ private Path writeStoreFile(int t, BloomType bt, List<KeyValue> kvs)
         BLOCK_SIZES[t])
             .withOutputDir(TEST_UTIL.getDataTestDir())
             .withBloomType(bt)
+            .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
+            .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
             .build();
 
     assertTrue(w.hasGeneralBloom());
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
index f93487d6adb4..89dfbf74af8e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
@@ -69,10 +70,12 @@ public void testHFileScannerThrowsErrors() throws IOException {
     Path hfilePath = new Path(new Path(
         util.getDataTestDir("internalScannerExposesErrors"),
         "regionname"), "familyname");
-    FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
+    HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
+    FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
+    FileSystem fs = new HFileSystem(faultyfs);
     CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
     StoreFile.Writer writer = new StoreFile.WriterBuilder(
-        util.getConfiguration(), cacheConf, fs, 2*1024)
+        util.getConfiguration(), cacheConf, hfs, 2*1024)
             .withOutputDir(hfilePath)
             .build();
     TestStoreFile.writeStoreFile(
@@ -85,14 +88,14 @@ public void testHFileScannerThrowsErrors() throws IOException {
     StoreFile.Reader reader = sf.createReader();
     HFileScanner scanner = reader.getScanner(false, true);
 
-    FaultyInputStream inStream = fs.inStreams.get(0).get();
+    FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
     assertNotNull(inStream);
 
     scanner.seekTo();
     // Do at least one successful read
     assertTrue(scanner.next());
 
-    inStream.startFaults();
+    faultyfs.startFaults();
 
     try {
       int scanned=0;
@@ -116,10 +119,12 @@ public void testStoreFileScannerThrowsErrors() throws IOException {
     Path hfilePath = new Path(new Path(
         util.getDataTestDir("internalScannerExposesErrors"),
         "regionname"), "familyname");
-    FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
+    HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
+    FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
+    HFileSystem fs = new HFileSystem(faultyfs);
     CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
     StoreFile.Writer writer = new StoreFile.WriterBuilder(
-        util.getConfiguration(), cacheConf, fs, 2 * 1024)
+        util.getConfiguration(), cacheConf, hfs, 2 * 1024)
             .withOutputDir(hfilePath)
             .build();
     TestStoreFile.writeStoreFile(
@@ -132,14 +137,13 @@ public void testStoreFileScannerThrowsErrors() throws IOException {
         Collections.singletonList(sf), false, true, false);
     KeyValueScanner scanner = scanners.get(0);
 
-    FaultyInputStream inStream = fs.inStreams.get(0).get();
+    FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
     assertNotNull(inStream);
 
     scanner.seek(KeyValue.LOWESTKEY);
     // Do at least one successful read
     assertNotNull(scanner.next());
-
-    inStream.startFaults();
+    faultyfs.startFaults();
 
     try {
       int scanned=0;
@@ -220,6 +224,15 @@ public FSDataInputStream open(Path p, int bufferSize) throws IOException  {
       inStreams.add(new SoftReference<FaultyInputStream>(faulty));
       return faulty;
     }
+
+    /**
+     * Starts to simulate faults on all streams opened so far
+     */
+    public void startFaults() {
+      for (SoftReference<FaultyInputStream> is: inStreams) {
+        is.get().startFaults();
+      }
+    } 
   }
 
   static class FaultyInputStream extends FSDataInputStream {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 8db43a41ae08..e2db6886c882 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3173,7 +3173,7 @@ public void testDeleteRowWithBloomFilter() throws IOException {
 
 
     // set up a cluster with 3 nodes
-    MiniHBaseCluster cluster;
+    MiniHBaseCluster cluster = null;
     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
     int regionServersCount = 3;
 
@@ -3221,7 +3221,9 @@ public void testDeleteRowWithBloomFilter() throws IOException {
 
       ht.close();
       } finally {
-        htu.shutdownMiniCluster();
+        if (cluster != null) {
+          htu.shutdownMiniCluster();
+        }
       }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 5b3b96234649..76c329fa4bff 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
@@ -69,6 +70,9 @@ public class TestStoreFile extends HBaseTestCase {
   private String ROOT_DIR;
   private Map<String, Long> startingMetrics;
 
+  private static final ChecksumType CKTYPE = ChecksumType.CRC32;
+  private static final int CKBYTES = 512;
+
   @Override
   public void setUp() throws Exception {
     super.setUp();
@@ -401,6 +405,8 @@ public void testBloomFilter() throws Exception {
             .withFilePath(f)
             .withBloomType(StoreFile.BloomType.ROW)
             .withMaxKeyCount(2000)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
     bloomWriteRead(writer, fs);
   }
@@ -420,6 +426,8 @@ public void testDeleteFamilyBloomFilter() throws Exception {
         fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
             .withFilePath(f)
             .withMaxKeyCount(2000)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
 
     // add delete family
@@ -490,6 +498,8 @@ public void testBloomTypes() throws Exception {
               .withFilePath(f)
               .withBloomType(bt[x])
               .withMaxKeyCount(expKeys[x])
+              .withChecksumType(CKTYPE)
+              .withBytesPerChecksum(CKBYTES)
               .build();
 
       long now = System.currentTimeMillis();
@@ -565,6 +575,8 @@ public void testBloomEdgeCases() throws Exception {
             .withFilePath(f)
             .withBloomType(StoreFile.BloomType.ROW)
             .withMaxKeyCount(2000)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
     assertFalse(writer.hasGeneralBloom());
     writer.close();
@@ -592,6 +604,8 @@ public void testBloomEdgeCases() throws Exception {
             .withFilePath(f)
             .withBloomType(StoreFile.BloomType.ROW)
             .withMaxKeyCount(Integer.MAX_VALUE)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
     assertFalse(writer.hasGeneralBloom());
     writer.close();
@@ -859,6 +873,8 @@ private StoreFile.Writer writeStoreFile(Configuration conf,
         blockSize)
             .withFilePath(path)
             .withMaxKeyCount(2000)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
     // We'll write N-1 KVs to ensure we don't write an extra block
     kvs.remove(kvs.size()-1);
@@ -890,6 +906,8 @@ public void testDataBlockEncodingMetaData() throws IOException {
             .withFilePath(path)
             .withDataBlockEncoder(dataBlockEncoder)
             .withMaxKeyCount(2000)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
     writer.close();
     
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
index e205acb403e0..ad5511c98675 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
@@ -133,7 +133,8 @@ public void setupHRI() {
      @Test public void testZKClosingNodeVersionMismatch()
      throws IOException, NodeExistsException, KeeperException {
        final Server server = new MockServer(HTU);
-       final RegionServerServices rss = new MockRegionServerServices();
+       final MockRegionServerServices rss = new MockRegionServerServices();
+       rss.setFileSystem(HTU.getTestFileSystem());
    
        HTableDescriptor htd = TEST_HTD;
        final HRegionInfo hri = TEST_HRI;
@@ -169,7 +170,8 @@ public void setupHRI() {
      @Test public void testCloseRegion()
      throws IOException, NodeExistsException, KeeperException {
        final Server server = new MockServer(HTU);
-       final RegionServerServices rss = new MockRegionServerServices();
+       final MockRegionServerServices rss = new MockRegionServerServices();
+       rss.setFileSystem(HTU.getTestFileSystem());
    
        HTableDescriptor htd = TEST_HTD;
        HRegionInfo hri = TEST_HRI;
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
index 967970d3d1f9..bb3ddd728a7c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
@@ -24,8 +24,10 @@
 import java.util.concurrent.ConcurrentSkipListMap;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
 import org.apache.hadoop.hbase.regionserver.FlushRequester;
@@ -45,6 +47,7 @@ public class MockRegionServerServices implements RegionServerServices {
   private boolean stopping = false;
   private final ConcurrentSkipListMap<byte[], Boolean> rit = 
     new ConcurrentSkipListMap<byte[], Boolean>(Bytes.BYTES_COMPARATOR);
+  private HFileSystem hfs = null;
 
   @Override
   public boolean removeFromOnlineRegions(String encodedRegionName) {
@@ -147,5 +150,13 @@ public boolean isStopped() {
   public boolean isAborted() {
     return false;
   }
-  
+
+  @Override
+  public HFileSystem getFileSystem() {
+    return this.hfs;
+  }
+
+  public void setFileSystem(FileSystem hfs) {
+    this.hfs = (HFileSystem)hfs;
+  }
 }

From 96d1bfb9297998225a3541a842d8a2759a38b894 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 9 Mar 2012 00:53:54 +0000
Subject: [PATCH 0028/1540] HBASE-5541 Avoid holding the rowlock during HLog
 sync in HRegion.mutateRowWithLocks

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298680 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 100 ++++++++++++++----
 1 file changed, 77 insertions(+), 23 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index acb1f7aff097..01eae0eb6e7b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4216,6 +4216,10 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
         }
       }
 
+      long txid = 0;
+      boolean walSyncSuccessful = false;
+      boolean locked = false;
+
       // 2. acquire the row lock(s)
       acquiredLocks = new ArrayList<Integer>(rowsToLock.size());
       for (byte[] row : rowsToLock) {
@@ -4230,6 +4234,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
 
       // 3. acquire the region lock
       this.updatesLock.readLock().lock();
+      locked = true;
 
       // 4. Get a mvcc write number
       MultiVersionConsistencyControl.WriteEntry w = mvcc.beginMemstoreInsert();
@@ -4258,10 +4263,12 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
           }
         }
 
-        // 6. append/sync all edits at once
-        // TODO: Do batching as in doMiniBatchPut
-        this.log.append(regionInfo, this.htableDescriptor.getName(), walEdit,
-            HConstants.DEFAULT_CLUSTER_ID, now, this.htableDescriptor);
+        // 6. append all edits at once (don't sync)
+        if (walEdit.size() > 0) {
+          txid = this.log.appendNoSync(regionInfo,
+              this.htableDescriptor.getName(), walEdit,
+              HConstants.DEFAULT_CLUSTER_ID, now, this.htableDescriptor);
+        }
 
         // 7. apply to memstore
         long addedSize = 0;
@@ -4269,32 +4276,79 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
           addedSize += applyFamilyMapToMemstore(m.getFamilyMap(), w);
         }
         flush = isFlushSize(this.addAndGetGlobalMemstoreSize(addedSize));
-      } finally {
-        // 8. roll mvcc forward
-        mvcc.completeMemstoreInsert(w);
 
-        // 9. release region lock
+        // 8. release region and row lock(s)
         this.updatesLock.readLock().unlock();
-      }
-      // 10. run all coprocessor post hooks, after region lock is released
-      if (coprocessorHost != null) {
-        for (Mutation m : mutations) {
-          if (m instanceof Put) {
-            coprocessorHost.postPut((Put) m, walEdit, m.getWriteToWAL());
-          } else if (m instanceof Delete) {
-            coprocessorHost.postDelete((Delete) m, walEdit, m.getWriteToWAL());
+        locked = false;
+        if (acquiredLocks != null) {
+          for (Integer lid : acquiredLocks) {
+            releaseRowLock(lid);
           }
+          acquiredLocks = null;
         }
-      }
-    } finally {
-      if (acquiredLocks != null) {
-        // 11. release the row lock
-        for (Integer lid : acquiredLocks) {
-          releaseRowLock(lid);
+
+        // 9. sync WAL if required
+        if (walEdit.size() > 0 &&
+            (this.regionInfo.isMetaRegion() ||
+             !this.htableDescriptor.isDeferredLogFlush())) {
+          this.log.sync(txid);
+        }
+        walSyncSuccessful = true;
+
+        // 10. advance mvcc
+        mvcc.completeMemstoreInsert(w);
+        w = null;
+
+        // 11. run coprocessor post host hooks
+        // after the WAL is sync'ed and all locks are released
+        // (similar to doMiniBatchPut)
+        if (coprocessorHost != null) {
+          for (Mutation m : mutations) {
+            if (m instanceof Put) {
+              coprocessorHost.postPut((Put) m, walEdit, m.getWriteToWAL());
+            } else if (m instanceof Delete) {
+              coprocessorHost.postDelete((Delete) m, walEdit, m.getWriteToWAL());
+            }
+          }
+        }
+      } finally {
+        // 12. clean up if needed
+        if (!walSyncSuccessful) {
+          int kvsRolledback = 0;
+          for (Mutation m : mutations) {
+            for (Map.Entry<byte[], List<KeyValue>> e : m.getFamilyMap()
+                .entrySet()) {
+              List<KeyValue> kvs = e.getValue();
+              byte[] family = e.getKey();
+              Store store = getStore(family);
+              // roll back each kv
+              for (KeyValue kv : kvs) {
+                store.rollback(kv);
+                kvsRolledback++;
+              }
+            }
+          }
+          LOG.info("mutateRowWithLocks: rolled back " + kvsRolledback
+              + " KeyValues");
+        }
+
+        if (w != null) {
+          mvcc.completeMemstoreInsert(w);
+        }
+
+        if (locked) {
+          this.updatesLock.readLock().unlock();
+        }
+
+        if (acquiredLocks != null) {
+          for (Integer lid : acquiredLocks) {
+            releaseRowLock(lid);
+          }
         }
       }
+    } finally {
       if (flush) {
-        // 12. Flush cache if needed. Do it outside update lock.
+        // 13. Flush cache if needed. Do it outside update lock.
         requestFlush();
       }
       closeRegionOperation();

From b27eb1a8d28271eca83389b1292db8eae818fac9 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 9 Mar 2012 15:35:21 +0000
Subject: [PATCH 0029/1540] HBASE-5213 "hbase master stop" does not bring down
 backup masters (Gregory)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298877 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/ActiveMasterManager.java     |  9 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 11 ++-
 .../hbase/master/TestActiveMasterManager.java | 66 +++++++++----
 .../hbase/master/TestMasterShutdown.java      | 99 +++++++++++++++++++
 4 files changed, 161 insertions(+), 24 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index fe4710b87e37..381c683d045f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 
 /**
  * Handles everything on master-side related to master election.
@@ -126,7 +127,8 @@ private void handleMasterNodeChange() {
    * master was running or if some other problem (zookeeper, stop flag has been
    * set on this Master)
    */
-  boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus) {
+  boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
+      ClusterStatusTracker clusterStatusTracker) {
     startupStatus.setStatus("Trying to register in ZK as active master");
     boolean cleanSetOfActiveMaster = true;
     // Try to become the active master, watch if there is another master.
@@ -201,11 +203,14 @@ boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus) {
           LOG.debug("Interrupted waiting for master to die", e);
         }
       }
+      if (!clusterStatusTracker.isClusterUp()) {
+        this.master.stop("Cluster went down before this master became active");
+      }
       if (this.master.isStopped()) {
         return cleanSetOfActiveMaster;
       }
       // Try to become active master again now that there is no active master
-      blockUntilBecomingActiveMaster(startupStatus);
+      blockUntilBecomingActiveMaster(startupStatus,clusterStatusTracker);
     }
     return cleanSetOfActiveMaster;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3e4b4d048cf7..a892ded28045 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -396,7 +396,14 @@ private boolean becomeActiveMaster(MonitoredTask startupStatus)
         this);
     this.zooKeeper.registerListener(activeMasterManager);
     stallIfBackupMaster(this.conf, this.activeMasterManager);
-    return this.activeMasterManager.blockUntilBecomingActiveMaster(startupStatus);
+
+    // The ClusterStatusTracker is setup before the other
+    // ZKBasedSystemTrackers because it's needed by the activeMasterManager
+    // to check if the cluster should be shutdown.
+    this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
+    this.clusterStatusTracker.start();
+    return this.activeMasterManager.blockUntilBecomingActiveMaster(startupStatus,
+        this.clusterStatusTracker);
   }
 
   /**
@@ -425,8 +432,6 @@ private void initializeZKBasedSystemTrackers() throws IOException,
 
     // Set the cluster as up.  If new RSs, they'll be waiting on this before
     // going ahead with their startup.
-    this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
-    this.clusterStatusTracker.start();
     boolean wasUp = this.clusterStatusTracker.isClusterUp();
     if (!wasUp) this.clusterStatusTracker.setClusterUp();
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
index c00b08c16ec2..05f6b1aec7ad 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -66,30 +67,33 @@ public static void tearDownAfterClass() throws Exception {
       "testActiveMasterManagerFromZK", null, true);
     try {
       ZKUtil.deleteNode(zk, zk.masterAddressZNode);
+      ZKUtil.deleteNode(zk, zk.clusterStateZNode);
     } catch(KeeperException.NoNodeException nne) {}
 
     // Create the master node with a dummy address
     ServerName master = new ServerName("localhost", 1, System.currentTimeMillis());
     // Should not have a master yet
-    DummyMaster dummyMaster = new DummyMaster();
-    ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk,
-      master, dummyMaster);
-    zk.registerListener(activeMasterManager);
+    DummyMaster dummyMaster = new DummyMaster(zk,master);
+    ClusterStatusTracker clusterStatusTracker =
+      dummyMaster.getClusterStatusTracker();
+    ActiveMasterManager activeMasterManager =
+      dummyMaster.getActiveMasterManager();
     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
 
     // First test becoming the active master uninterrupted
     MonitoredTask status = Mockito.mock(MonitoredTask.class);
-    activeMasterManager.blockUntilBecomingActiveMaster(status);
+    clusterStatusTracker.setClusterUp();
+
+    activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, master);
 
     // Now pretend master restart
-    DummyMaster secondDummyMaster = new DummyMaster();
-    ActiveMasterManager secondActiveMasterManager = new ActiveMasterManager(zk,
-      master, secondDummyMaster);
-    zk.registerListener(secondActiveMasterManager);
+    DummyMaster secondDummyMaster = new DummyMaster(zk,master);
+    ActiveMasterManager secondActiveMasterManager =
+      secondDummyMaster.getActiveMasterManager();
     assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
-    activeMasterManager.blockUntilBecomingActiveMaster(status);
+    activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, master);
   }
@@ -105,6 +109,7 @@ public void testActiveMasterManagerFromZK() throws Exception {
       "testActiveMasterManagerFromZK", null, true);
     try {
       ZKUtil.deleteNode(zk, zk.masterAddressZNode);
+      ZKUtil.deleteNode(zk, zk.clusterStateZNode);
     } catch(KeeperException.NoNodeException nne) {}
 
     // Create the master node with a dummy address
@@ -114,21 +119,22 @@ public void testActiveMasterManagerFromZK() throws Exception {
       new ServerName("localhost", 2, System.currentTimeMillis());
 
     // Should not have a master yet
-    DummyMaster ms1 = new DummyMaster();
-    ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk,
-      firstMasterAddress, ms1);
-    zk.registerListener(activeMasterManager);
+    DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
+    ActiveMasterManager activeMasterManager =
+      ms1.getActiveMasterManager();
     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
 
     // First test becoming the active master uninterrupted
+    ClusterStatusTracker clusterStatusTracker =
+      ms1.getClusterStatusTracker();
+    clusterStatusTracker.setClusterUp();
     activeMasterManager.blockUntilBecomingActiveMaster(
-        Mockito.mock(MonitoredTask.class));
+        Mockito.mock(MonitoredTask.class),clusterStatusTracker);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, firstMasterAddress);
 
     // New manager will now try to become the active master in another thread
     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
-    zk.registerListener(t.manager);
     t.start();
     // Wait for this guy to figure out there is another active master
     // Wait for 1 second at most
@@ -193,18 +199,20 @@ private void assertMaster(ZooKeeperWatcher zk,
   public static class WaitToBeMasterThread extends Thread {
 
     ActiveMasterManager manager;
+    DummyMaster dummyMaster;
     boolean isActiveMaster;
 
     public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
-      this.manager = new ActiveMasterManager(zk, address,
-          new DummyMaster());
+      this.dummyMaster = new DummyMaster(zk,address);
+      this.manager = this.dummyMaster.getActiveMasterManager();
       isActiveMaster = false;
     }
 
     @Override
     public void run() {
       manager.blockUntilBecomingActiveMaster(
-          Mockito.mock(MonitoredTask.class));
+          Mockito.mock(MonitoredTask.class),
+          this.dummyMaster.getClusterStatusTracker());
       LOG.info("Second master has become the active master!");
       isActiveMaster = true;
     }
@@ -240,6 +248,18 @@ public void waitForDeletion() throws InterruptedException {
    */
   public static class DummyMaster implements Server {
     private volatile boolean stopped;
+    private ClusterStatusTracker clusterStatusTracker;
+    private ActiveMasterManager activeMasterManager;
+
+    public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
+      this.clusterStatusTracker =
+        new ClusterStatusTracker(zk, this);
+      clusterStatusTracker.start();
+      
+      this.activeMasterManager =
+        new ActiveMasterManager(zk, master, this);
+      zk.registerListener(activeMasterManager);
+    }
 
     @Override
     public void abort(final String msg, final Throwable t) {}
@@ -278,6 +298,14 @@ public void stop(String why) {
     public CatalogTracker getCatalogTracker() {
       return null;
     }
+
+    public ClusterStatusTracker getClusterStatusTracker() {
+      return clusterStatusTracker;
+    }
+
+    public ActiveMasterManager getActiveMasterManager() {
+      return activeMasterManager;
+    }
   }
 
   @org.junit.Rule
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
new file mode 100644
index 000000000000..436b2c9eab48
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
@@ -0,0 +1,99 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestMasterShutdown {
+  private static final Log LOG = LogFactory.getLog(TestMasterShutdown.class);
+
+  /**
+   * Simple test of shutdown.
+   * <p>
+   * Starts with three masters.  Tells the active master to shutdown the cluster.
+   * Verifies that all masters are properly shutdown.
+   * @throws Exception
+   */
+  @Test (timeout=240000)
+  public void testMasterShutdown() throws Exception {
+
+    final int NUM_MASTERS = 3;
+    final int NUM_RS = 3;
+
+    // Create config to use for this cluster
+    Configuration conf = HBaseConfiguration.create();
+
+    // Start the cluster
+    HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+
+    // get all the master threads
+    List<MasterThread> masterThreads = cluster.getMasterThreads();
+
+    // wait for each to come online
+    for (MasterThread mt : masterThreads) {
+      assertTrue(mt.isAlive());
+    }
+
+    // find the active master
+    HMaster active = null;
+    for (int i = 0; i < masterThreads.size(); i++) {
+      if (masterThreads.get(i).getMaster().isActiveMaster()) {
+        active = masterThreads.get(i).getMaster();
+        break;
+      }
+    }
+    assertNotNull(active);
+    // make sure the other two are backup masters
+    ClusterStatus status = active.getClusterStatus();
+    assertEquals(2, status.getBackupMastersSize());
+    assertEquals(2, status.getBackupMasters().size());
+
+    // tell the active master to shutdown the cluster
+    active.shutdown();
+    
+    for (int i = NUM_MASTERS - 1; i >= 0 ;--i) {
+      cluster.waitOnMaster(i);
+    }
+    // make sure all the masters properly shutdown
+    assertEquals(0,masterThreads.size());
+    
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
+

From e6893ba6e61bc4ab0ed6f0d81f38f8d02f38e257 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 9 Mar 2012 16:53:56 +0000
Subject: [PATCH 0030/1540] HBASE-5552 Clean up our jmx view; its a bit of a
 mess

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1298922 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java   | 2 +-
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 2 +-
 .../java/org/apache/hadoop/hbase/metrics/HBaseInfo.java   | 8 +++++---
 .../apache/hadoop/hbase/regionserver/HRegionServer.java   | 2 +-
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
index c9b0257a262c..b9fa0dd21d58 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
@@ -36,7 +36,7 @@ public class HBaseRPCStatistics extends MetricsDynamicMBeanBase {
   @SuppressWarnings({"UnusedDeclaration"})
   public HBaseRPCStatistics(MetricsRegistry registry,
       String hostName, String port) {
-	  super(registry, "HBaseRPCStatistics");
+	  super(registry, "Metrics for RPC server instance");
 
     String name = String.format("RPCStatistics-%s",
         (port != null ? port : "unknown"));
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a892ded28045..0c4e22a5ed53 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1830,7 +1830,7 @@ public static void main(String [] args) throws Exception {
   @SuppressWarnings("deprecation")
   void registerMBean() {
     MXBeanImpl mxBeanInfo = MXBeanImpl.init(this);
-    MBeanUtil.registerMBean("org.apache.hbase", "Master", mxBeanInfo);
+    MBeanUtil.registerMBean("Master", "Master", mxBeanInfo);
     LOG.info("Registered HMaster MXBean");
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java b/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
index fb65a652d3d6..38ed87256cf9 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
@@ -36,9 +36,11 @@ protected static class HBaseInfoMBean extends MetricsMBeanBase {
     private final ObjectName mbeanName;
   
     public HBaseInfoMBean(MetricsRegistry registry, String rsName) {
-      super(registry, "HBaseInfo");
-      mbeanName = MBeanUtil.registerMBean("HBase",
-          "Info", this);
+      super(registry, "HBase cluster information");
+      // The name seems wrong to me; should include clusterid IMO.
+      // That would make it harder to locate and rare we have
+      // two clusters up on single machine. St.Ack 20120309
+      mbeanName = MBeanUtil.registerMBean("HBase", "Info", this);
     }
   
     public void shutdown() {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 406e5aecbe25..0cbe9cf036ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3708,7 +3708,7 @@ public String[] getCoprocessors() {
   @SuppressWarnings("deprecation")
   void registerMBean() {
     MXBeanImpl mxBeanInfo = MXBeanImpl.init(this);
-    mxBean = MBeanUtil.registerMBean("org.apache.hbase", "RegionServer",
+    mxBean = MBeanUtil.registerMBean("RegionServer", "RegionServer",
         mxBeanInfo);
     LOG.info("Registered RegionServer MXBean");
   }

From 28a902dc4d8dba9799d90fb232eba72dd90e44fe Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 9 Mar 2012 22:48:16 +0000
Subject: [PATCH 0031/1540] HBASE-5535 Make the functions in task monitor
 synchronized

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1299089 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
index fc9c8301e470..942b7c112589 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
@@ -64,7 +64,7 @@ public static synchronized TaskMonitor get() {
     return instance;
   }
   
-  public MonitoredTask createStatus(String description) {
+  public synchronized MonitoredTask createStatus(String description) {
     MonitoredTask stat = new MonitoredTaskImpl();
     stat.setDescription(description);
     MonitoredTask proxy = (MonitoredTask) Proxy.newProxyInstance(
@@ -78,7 +78,7 @@ public MonitoredTask createStatus(String description) {
     return proxy;
   }
 
-  public MonitoredRPCHandler createRPCStatus(String description) {
+  public synchronized MonitoredRPCHandler createRPCStatus(String description) {
     MonitoredRPCHandler stat = new MonitoredRPCHandlerImpl();
     stat.setDescription(description);
     MonitoredRPCHandler proxy = (MonitoredRPCHandler) Proxy.newProxyInstance(

From 0ae06d72b6e1c365cd2df93bb1b364ee25118afe Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 12 Mar 2012 20:19:49 +0000
Subject: [PATCH 0032/1540] HBASE-5562 test-patch.sh reports a javadoc warning
 when there are no new javadoc warnings

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1299825 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/test-patch.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index efa1fc795ed6..0f77de4adb0e 100644
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -366,7 +366,7 @@ checkJavadocWarnings () {
   echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."
 
   ### if current warnings greater than OK_JAVADOC_WARNINGS
-  if [[ $javadocWarnings > $OK_JAVADOC_WARNINGS ]] ; then
+  if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
     -1 javadoc.  The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages."

From e61455533a7b2db7d8ae4cc793648d2b657efc0b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 12 Mar 2012 20:47:52 +0000
Subject: [PATCH 0033/1540] Fixups to MultithreadedTableMapper for Hadoop
 0.23.2+ (Andrew)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1299848 13f79535-47bb-0310-9956-ffa450edef68
---
 .../mapreduce/MultithreadedTableMapper.java   | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
index 1bc1abc5a9c3..eb96ac684458 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
+import java.lang.reflect.Constructor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -221,6 +222,10 @@ public void progress() {
     public void setStatus(String status) {
       outer.setStatus(status);
     }
+
+    public float getProgress() {
+      return 0;
+    }
   }
 
   private class MapRunner implements Runnable {
@@ -228,16 +233,32 @@ private class MapRunner implements Runnable {
     private Context subcontext;
     private Throwable throwable;
 
+    @SuppressWarnings({ "rawtypes", "unchecked" })
     MapRunner(Context context) throws IOException, InterruptedException {
       mapper = ReflectionUtils.newInstance(mapClass,
           context.getConfiguration());
-      subcontext = new Context(outer.getConfiguration(), 
+      try {
+        Constructor c = context.getClass().getConstructor(
+          Configuration.class,
+          outer.getTaskAttemptID().getClass(),
+          SubMapRecordReader.class,
+          SubMapRecordWriter.class,
+          context.getOutputCommitter().getClass(),
+          SubMapStatusReporter.class,
+          outer.getInputSplit().getClass());
+        c.setAccessible(true);
+        subcontext = (Context) c.newInstance(
+          outer.getConfiguration(), 
           outer.getTaskAttemptID(),
           new SubMapRecordReader(),
           new SubMapRecordWriter(),
           context.getOutputCommitter(),
           new SubMapStatusReporter(),
           outer.getInputSplit());
+      } catch (Exception e) {
+        // rethrow as IOE
+        throw new IOException(e);
+      }
     }
 
     @Override

From ab2b1acced9d7bd2843fd8edd8b1e8a24ff61131 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 12 Mar 2012 21:49:49 +0000
Subject: [PATCH 0034/1540] HBASE-5567 test-patch.sh has logic error in
 findbugs check

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1299893 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/test-patch.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 0f77de4adb0e..bdef3ab6a4fa 100644
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -545,7 +545,7 @@ $JIRA_COMMENT_FOOTER"
   done
 
   ### if current warnings greater than OK_FINDBUGS_WARNINGS
-  if [[ $findbugsWarnings > $OK_FINDBUGS_WARNINGS ]] ; then
+  if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
     -1 findbugs.  The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version ${findbugs_version}) warnings."

From ff143dd9c44096a4165891a66b75e7f6756c2b12 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 12 Mar 2012 23:39:30 +0000
Subject: [PATCH 0035/1540] HBASE-5514 Compile against hadoop 0.24-SNAPSHOT
 (Mingjie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1299928 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       | 82 +++++++++++++++++++
 .../hbase/regionserver/wal/TestHLog.java      | 14 +++-
 .../hbase/regionserver/wal/TestHLogSplit.java | 18 +++-
 3 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 2cb0239ab6b3..d4c7c1a67b12 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1900,6 +1900,88 @@
       </build>
     </profile>
 
+    <!--
+      profile for building against Hadoop 0.24.0. Activate using:
+       mvn -Dhadoop.profile=24
+    -->
+    <profile>
+      <id>hadoop-0.24</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>24</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>0.24.0-SNAPSHOT</hadoop.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-annotations</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <!-- test deps for hadoop-0.24 profile -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>compile</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-resource</id>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>src/test/resources</directory>
+                      <includes>
+                        <include>hbase-site.xml</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>create-mrapp-generated-classpath</id>
+                <phase>generate-test-resources</phase>
+                <goals>
+                  <goal>build-classpath</goal>
+                </goals>
+                <configuration>
+                  <!-- needed to run the unit test for DS to generate
+                  the required classpath that is required in the env
+                  of the launch container in the mini mr/yarn cluster
+                  -->
+                  <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+
     <!-- profiles for the tests
          See as well the properties of the project for the values
          when no profile is active.     -->
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 9034844f0131..81592f85fe53 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -22,6 +22,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.lang.reflect.Method;
@@ -204,7 +205,18 @@ public void Broken_testSync() throws Exception {
     Path p = new Path(dir, getName() + ".fsdos");
     FSDataOutputStream out = fs.create(p);
     out.write(bytes);
-    out.sync();
+    Method syncMethod = null;
+    try {
+      syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});
+    } catch (NoSuchMethodException e) {
+      try {
+        syncMethod = out.getClass().getMethod("sync", new Class<?> []{});
+      } catch (NoSuchMethodException ex) {
+        fail("This version of Hadoop supports neither Syncable.sync() " +
+            "nor Syncable.hflush().");
+      }
+    }
+    syncMethod.invoke(out, new Object[]{});
     FSDataInputStream in = fs.open(p);
     assertTrue(in.available() > 0);
     byte [] buffer = new byte [1024];
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index f1ea70114b06..d838a68d287d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -27,6 +27,7 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -1266,7 +1267,22 @@ private void closeOrFlush(boolean close, FSDataOutputStream out)
     if (close) {
       out.close();
     } else {
-      out.sync();
+      Method syncMethod = null;
+      try {
+        syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});
+      } catch (NoSuchMethodException e) {
+        try {
+          syncMethod = out.getClass().getMethod("sync", new Class<?> []{});
+        } catch (NoSuchMethodException ex) {
+          throw new IOException("This version of Hadoop supports " +
+              "neither Syncable.sync() nor Syncable.hflush().");
+        }
+      }
+      try {
+        syncMethod.invoke(out, new Object[]{});
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
       // Not in 0out.hflush();
     }
   }

From ffa518256f651c9dfd496676bb96002a1a831587 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 13 Mar 2012 05:04:00 +0000
Subject: [PATCH 0036/1540] HBASE-5499 dev-support/test-patch.sh does not have
 execute perms

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1299975 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/test-patch.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 dev-support/test-patch.sh

diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
old mode 100644
new mode 100755

From aa1d17e2969cd25a2c64aa99593a4e542fcb972d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 13 Mar 2012 16:33:14 +0000
Subject: [PATCH 0037/1540] HBASE-5179 Handle potential data loss due to
 concurrent processing of processFaileOver and ServerShutdownHandler

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1300222 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  59 +++-
 .../apache/hadoop/hbase/master/HMaster.java   |  77 ++++--
 .../hadoop/hbase/master/MasterServices.java   |   7 +-
 .../hadoop/hbase/master/ServerManager.java    |  41 ++-
 .../master/handler/CreateTableHandler.java    |   2 +
 .../hbase/regionserver/HRegionServer.java     |   2 +-
 .../hbase/master/TestAssignmentManager.java   |  20 +-
 .../hbase/master/TestCatalogJanitor.java      |  17 +-
 .../TestRSKilledWhenMasterInitializing.java   | 261 ++++++++++++++++++
 9 files changed, 439 insertions(+), 47 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index bb58026f9fd3..764a0d299975 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -64,7 +64,6 @@
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
-import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
 import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
@@ -326,11 +325,13 @@ void cleanoutUnassigned() throws IOException, KeeperException {
   /**
    * Called on startup.
    * Figures whether a fresh cluster start of we are joining extant running cluster.
+   * @param onlineServers onlined servers when master started
    * @throws IOException
    * @throws KeeperException
    * @throws InterruptedException
    */
-  void joinCluster() throws IOException, KeeperException, InterruptedException {
+  void joinCluster(final Set<ServerName> onlineServers) throws IOException,
+      KeeperException, InterruptedException {
     // Concurrency note: In the below the accesses on regionsInTransition are
     // outside of a synchronization block where usually all accesses to RIT are
     // synchronized.  The presumption is that in this case it is safe since this
@@ -341,7 +342,7 @@ void joinCluster() throws IOException, KeeperException, InterruptedException {
 
     // Scan META to build list of existing regions, servers, and assignment
     // Returns servers who have not checked in (assumed dead) and their regions
-    Map<ServerName,List<Pair<HRegionInfo,Result>>> deadServers = rebuildUserRegions();
+    Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers = rebuildUserRegions(onlineServers);
 
     processDeadServersAndRegionsInTransition(deadServers);
 
@@ -351,6 +352,16 @@ void joinCluster() throws IOException, KeeperException, InterruptedException {
     recoverTableInEnablingState(this.enablingTables, isWatcherCreated);
   }
 
+  /**
+   * Only used for tests
+   * @throws IOException
+   * @throws KeeperException
+   * @throws InterruptedException
+   */
+  void joinCluster() throws IOException, KeeperException, InterruptedException {
+    joinCluster(serverManager.getOnlineServers().keySet());
+  }
+
   /**
    * Process all regions that are in transition up in zookeeper.  Used by
    * master joining an already running cluster.
@@ -396,6 +407,12 @@ void processDeadServersAndRegionsInTransition(
       }
     }
 
+    // Remove regions in RIT, they are possibly being processed by
+    // ServerShutdownHandler.
+    synchronized (regionsInTransition) {
+      nodes.removeAll(regionsInTransition.keySet());
+    }
+
     // If we found user regions out on cluster, its a failover.
     if (this.failover) {
       LOG.info("Found regions out on cluster or in RIT; failover");
@@ -1768,6 +1785,7 @@ RegionPlan getRegionPlan(final RegionState state,
     final List<ServerName> servers = this.serverManager.getOnlineServersList();
     final List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
 
+
     if (serverToExclude != null) servers.remove(serverToExclude);
 
     // Loop through the draining server list and remove them from the server
@@ -1780,6 +1798,11 @@ RegionPlan getRegionPlan(final RegionState state,
       }
     }
 
+    // Remove the deadNotExpired servers from the server list.
+    removeDeadNotExpiredServers(servers);
+
+
+
     if (servers.isEmpty()) return null;
 
     RegionPlan randomPlan = new RegionPlan(state.getRegion(), null,
@@ -1811,7 +1834,7 @@ RegionPlan getRegionPlan(final RegionState state,
         " so generated a random one; " + randomPlan + "; " +
         serverManager.countOfRegionServers() +
                " (online=" + serverManager.getOnlineServers().size() +
-               ", exclude=" + drainingServers.size() + ") available servers");
+               ", available=" + servers.size() + ") available servers");
         return randomPlan;
       }
     LOG.debug("Using pre-existing plan for region " +
@@ -1819,6 +1842,23 @@ RegionPlan getRegionPlan(final RegionState state,
       return existingPlan;
   }
 
+  /**
+   * Loop through the deadNotExpired server list and remove them from the
+   * servers.
+   * @param servers
+   */
+  public void removeDeadNotExpiredServers(List<ServerName> servers) {
+    Set<ServerName> deadNotExpiredServers = this.serverManager
+        .getDeadNotExpiredServers();
+    if (!deadNotExpiredServers.isEmpty()) {
+      for (ServerName server : deadNotExpiredServers) {
+        LOG.debug("Removing dead but not expired server: " + server
+            + " from eligible server pool.");
+        servers.remove(server);
+      }
+    }
+  }
+
   /**
    * Unassign the list of regions. Configuration knobs:
    * hbase.bulk.waitbetween.reopen indicates the number of milliseconds to
@@ -2132,6 +2172,7 @@ public void assignUserRegionsToOnlineServers(List<HRegionInfo> regions)
       throws IOException,
       InterruptedException {
     List<ServerName> servers = this.serverManager.getOnlineServersList();
+    removeDeadNotExpiredServers(servers);
     assignUserRegions(regions, servers);
   }
 
@@ -2171,6 +2212,9 @@ public void assignAllUserRegions() throws IOException, InterruptedException {
     // Get all available servers
     List<ServerName> servers = serverManager.getOnlineServersList();
 
+    // Remove the deadNotExpired servers from the server list.
+    removeDeadNotExpiredServers(servers);
+
     // If there are no servers we need not proceed with region assignment.
     if(servers.isEmpty()) return;
 
@@ -2375,11 +2419,14 @@ boolean waitUntilNoRegionsInTransition(final long timeout, Set<HRegionInfo> regi
    * <p>
    * Returns a map of servers that are not found to be online and the regions
    * they were hosting.
+   * @param onlineServers if one region's location belongs to onlineServers, it
+   *          doesn't need to be assigned.
    * @return map of servers not online to their assigned regions, as stored
    *         in META
    * @throws IOException
    */
-  Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions()
+  Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
+      final Set<ServerName> onlineServers)
   throws IOException, KeeperException {
     // Region assignment from META
     List<Result> results = MetaReader.fullScan(this.catalogTracker);
@@ -2412,7 +2459,7 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions()
         }
         addTheTablesInPartialState(this.disablingTables, this.enablingTables, regionInfo,
             tableName);
-      } else if (!this.serverManager.isServerOnline(regionLocation)) {
+      } else if (!onlineServers.contains(regionLocation)) {
         // Region is located on a server that isn't online
         List<Pair<HRegionInfo, Result>> offlineRegions =
           offlineServers.get(regionLocation);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0c4e22a5ed53..785c6a7aff05 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -26,6 +26,7 @@
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -189,6 +190,8 @@ public class HMaster extends HasThread
   private volatile boolean isActiveMaster = false;
   // flag set after we complete initialization once active (used for testing)
   private volatile boolean initialized = false;
+  // flag set after we complete assignRootAndMeta.
+  private volatile boolean serverShutdownHandlerEnabled = false;
 
   // Instance of the hbase executor service.
   ExecutorService executorService;
@@ -525,13 +528,17 @@ private void finishInitialization(MonitoredTask status)
       }
     }
 
+    Set<ServerName> onlineServers = new HashSet<ServerName>(serverManager
+        .getOnlineServers().keySet());
     // TODO: Should do this in background rather than block master startup
     status.setStatus("Splitting logs after master startup");
-    this.fileSystemManager.
-      splitLogAfterStartup(this.serverManager.getOnlineServers().keySet());
+    splitLogAfterStartup(this.fileSystemManager, onlineServers);
 
     // Make sure root and meta assigned before proceeding.
     assignRootAndMeta(status);
+    serverShutdownHandlerEnabled = true;
+    this.serverManager.expireDeadNotExpiredServers();
+
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
     // before we assign all user regions or else the assignment will fail.
@@ -541,7 +548,7 @@ private void finishInitialization(MonitoredTask status)
 
     // Fixup assignment manager status
     status.setStatus("Starting assignment manager");
-    this.assignmentManager.joinCluster();
+    this.assignmentManager.joinCluster(onlineServers);
 
     this.balancer.setClusterStatus(getClusterStatus());
     this.balancer.setMasterServices(this);
@@ -576,6 +583,16 @@ private void finishInitialization(MonitoredTask status)
     }
   }
 
+  /**
+   * Override to change master's splitLogAfterStartup. Used testing
+   * @param mfs
+   * @param onlineServers
+   */
+  protected void splitLogAfterStartup(final MasterFileSystem mfs,
+      Set<ServerName> onlineServers) {
+    mfs.splitLogAfterStartup(onlineServers);
+  }
+
   /**
    * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,
    * assign them.
@@ -593,17 +610,11 @@ int assignRootAndMeta(MonitoredTask status)
     status.setStatus("Assigning ROOT region");
     boolean rit = this.assignmentManager.
       processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO);
-    ServerName expiredServer = null;
+    ServerName currentRootServer = null;
     if (!catalogTracker.verifyRootRegionLocation(timeout)) {
-      ServerName currentRootServer = this.catalogTracker.getRootLocation();
-      if (expireIfOnline(currentRootServer)) {
-        // We are expiring this server. The processing of expiration will assign
-        // root so don't do it here.
-        expiredServer = currentRootServer;
-      } else {
-        // Root was not on an online server when we failed verification
-        this.assignmentManager.assignRoot();
-      }
+      currentRootServer = this.catalogTracker.getRootLocation();
+      splitLogAndExpireIfOnline(currentRootServer);
+      this.assignmentManager.assignRoot();
       this.catalogTracker.waitForRoot();
       //This guarantees that the transition has completed
       this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);
@@ -623,13 +634,11 @@ int assignRootAndMeta(MonitoredTask status)
     if (!this.catalogTracker.verifyMetaRegionLocation(timeout)) {
       ServerName currentMetaServer =
         this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-      if (currentMetaServer != null && currentMetaServer.equals(expiredServer)) {
-        // We are expiring the server that is carrying meta already.
-        // The expiration processing will take care of reassigning meta.
-        expireIfOnline(currentMetaServer);
-      } else {
-        this.assignmentManager.assignMeta();
+      if (currentMetaServer != null
+          && !currentMetaServer.equals(currentRootServer)) {
+        splitLogAndExpireIfOnline(currentMetaServer);
       }
+      assignmentManager.assignMeta();
       this.catalogTracker.waitForMeta();
       // Above check waits for general meta availability but this does not
       // guarantee that the transition has completed
@@ -680,16 +689,19 @@ public boolean visit(Result r) throws IOException {
   }
 
   /**
-   * Expire a server if we find it is one of the online servers set.
+   * Split a server's log and expire it if we find it is one of the online
+   * servers.
    * @param sn ServerName to check.
-   * @return True if server was online and so we expired it as unreachable.
+   * @throws IOException
    */
-  private boolean expireIfOnline(final ServerName sn) {
-    if (sn == null) return false;
-    if (!this.serverManager.isServerOnline(sn)) return false;
-    LOG.info("Forcing expiration of " + sn);
-    this.serverManager.expireServer(sn);
-    return true;
+  private void splitLogAndExpireIfOnline(final ServerName sn)
+      throws IOException {
+    if (sn == null || !serverManager.isServerOnline(sn)) {
+      return;
+    }
+    LOG.info("Forcing splitLog and expire of " + sn);
+    fileSystemManager.splitLog(sn);
+    serverManager.expireServer(sn);
   }
 
   @Override
@@ -1690,7 +1702,16 @@ public boolean isActiveMaster() {
   public boolean isInitialized() {
     return initialized;
   }
-  
+
+  /**
+   * ServerShutdownHandlerEnabled is set false before completing
+   * assignRootAndMeta to prevent processing of ServerShutdownHandler.
+   * @return true if assignRootAndMeta has completed;
+   */
+  public boolean isServerShutdownHandlerEnabled() {
+    return this.serverShutdownHandlerEnabled;
+  }
+
   @Override
   @Deprecated
   public void assign(final byte[] regionName, final boolean force)
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index c4b42ae5038c..d88db0e91da0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -24,8 +24,6 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableDescriptors;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
@@ -92,4 +90,9 @@ public void createTable(HTableDescriptor desc, byte [][] splitKeys)
    */
   public RegionServerTracker getRegionServerTracker();
 
+  /**
+   * @return true if master enables ServerShutdownHandler;
+   */
+  public boolean isServerShutdownHandlerEnabled();
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 02bebb652dba..e13a94a98e22 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -24,6 +24,8 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -45,10 +47,10 @@
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 
 /**
  * The ServerManager class manages info about region servers.
@@ -95,6 +97,14 @@ public class ServerManager {
 
   private final long maxSkew;
 
+  /**
+   * Set of region servers which are dead but not expired immediately. If one
+   * server died before master enables ServerShutdownHandler, the server will be
+   * added to set and will be expired through calling
+   * {@link ServerManager#expireDeadNotExpiredServers()} by master.
+   */
+  private Set<ServerName> deadNotExpiredServers = new HashSet<ServerName>();
+
   /**
    * Constructor.
    * @param master
@@ -347,6 +357,12 @@ private void excludeRegionServerFromSchemaChanges(final ServerName serverName) {
    * shutdown processing.
    */
   public synchronized void expireServer(final ServerName serverName) {
+    if (!services.isServerShutdownHandlerEnabled()) {
+      LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
+          + "delay expiring server " + serverName);
+      this.deadNotExpiredServers.add(serverName);
+      return;
+    }
     excludeRegionServerFromSchemaChanges(serverName);
     if (!this.onlineServers.containsKey(serverName)) {
       LOG.warn("Received expiration of " + serverName +
@@ -393,6 +409,22 @@ public synchronized void expireServer(final ServerName serverName) {
         carryingRoot + ", meta=" + carryingMeta);
   }
 
+  /**
+   * Expire the servers which died during master's initialization. It will be
+   * called after HMaster#assignRootAndMeta.
+   * @throws IOException
+   * */
+  synchronized void expireDeadNotExpiredServers() throws IOException {
+    if (!services.isServerShutdownHandlerEnabled()) {
+      throw new IOException("Master hasn't enabled ServerShutdownHandler ");
+    }
+    Iterator<ServerName> serverIterator = deadNotExpiredServers.iterator();
+    while (serverIterator.hasNext()) {
+      expireServer(serverIterator.next());
+      serverIterator.remove();
+    }
+  }
+
   /*
    * Remove the server from the drain list.
    */
@@ -604,6 +636,13 @@ public List<ServerName> getDrainingServersList() {
     return new ArrayList<ServerName>(this.drainingServers);
   }
 
+  /**
+   * @return A copy of the internal set of deadNotExpired servers.
+   */
+  Set<ServerName> getDeadNotExpiredServers() {
+    return new HashSet<ServerName>(this.deadNotExpiredServers);
+  }
+
   public boolean isServerOnline(ServerName serverName) {
     return onlineServers.containsKey(serverName);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 4600991cb4e7..95750606df78 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -170,6 +170,8 @@ private void handleCreateTable() throws IOException, KeeperException {
 
     // 4. Trigger immediate assignment of the regions in round-robin fashion
     List<ServerName> servers = serverManager.getOnlineServersList();
+    // Remove the deadNotExpired servers from the server list.
+    assignmentManager.removeDeadNotExpiredServers(servers);
     try {
       this.assignmentManager.assignUserRegions(Arrays.asList(newRegions),
         servers);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 0cbe9cf036ec..44e4891458d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -778,7 +778,7 @@ public void run() {
     // Interrupt catalog tracker here in case any regions being opened out in
     // handlers are stuck waiting on meta or root.
     if (this.catalogTracker != null) this.catalogTracker.stop();
-    if (this.fsOk) {
+    if (!this.killed && this.fsOk) {
       waitOnAllRegionsToClose(abortRequested);
       LOG.info("stopping server " + this.serverNameFromMasterPOV +
         "; all regions closed.");
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index d68ce332ac6a..841649abb3fc 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -22,17 +22,22 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
 import org.apache.hadoop.hbase.client.Result;
@@ -45,6 +50,7 @@
 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -59,10 +65,6 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.client.Get;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 
 /**
@@ -115,10 +117,12 @@ public void before() throws ZooKeeperConnectionException, IOException {
     this.serverManager = Mockito.mock(ServerManager.class);
     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
-    final List<ServerName> onlineServers = new ArrayList<ServerName>();
-    onlineServers.add(SERVERNAME_B);
-    onlineServers.add(SERVERNAME_A);
-    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(onlineServers);
+    final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
+    onlineServers.put(SERVERNAME_B, new HServerLoad());
+    onlineServers.put(SERVERNAME_A, new HServerLoad());
+    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
+        new ArrayList<ServerName>(onlineServers.keySet()));
+    Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
       thenReturn(true);
     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index a5628b95e2fb..227c5f28446a 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -35,7 +35,17 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -260,6 +270,11 @@ public MasterSchemaChangeTracker getSchemaChangeTracker() {
     public RegionServerTracker getRegionServerTracker() {
       return null;
     }
+
+    @Override
+    public boolean isServerShutdownHandlerEnabled() {
+      return true;
+    }
   }
 
   @Test
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
new file mode 100644
index 000000000000..8f8018dc77ca
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.TestMasterFailover;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestRSKilledWhenMasterInitializing {
+  private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
+
+  private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
+  private static final int NUM_MASTERS = 1;
+  private static final int NUM_RS = 4;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // Set it so that this test runs with my custom master
+    TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL,
+        TestingMaster.class, HMaster.class);
+    // Start up the cluster.
+    TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    if (!TESTUTIL.getHBaseCluster().getMaster().isInitialized()) {
+      // master is not initialized and is waiting something forever.
+      for (MasterThread mt : TESTUTIL.getHBaseCluster().getLiveMasterThreads()) {
+        mt.interrupt();
+      }
+    }
+    TESTUTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * An HMaster instance used in this test. If 'TestingMaster.sleep' is set in
+   * the Configuration, then we'll sleep after log is split and we'll also
+   * return a custom RegionServerTracker.
+   */
+  public static class TestingMaster extends HMaster {
+    private boolean logSplit = false;
+
+    public TestingMaster(Configuration conf) throws IOException,
+        KeeperException, InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    protected void splitLogAfterStartup(MasterFileSystem mfs,
+        Set<ServerName> onlineServers) {
+      super.splitLogAfterStartup(mfs, onlineServers);
+      logSplit = true;
+      // If "TestingMaster.sleep" is set, sleep after log split.
+      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
+        int duration = getConfiguration().getInt(
+            "TestingMaster.sleep.duration", 0);
+        Threads.sleep(duration);
+      }
+    }
+
+
+    public boolean isLogSplitAfterStartup() {
+      return logSplit;
+    }
+  }
+
+  @Test(timeout = 120000)
+  public void testCorrectnessWhenMasterFailOver() throws Exception {
+    final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") };
+
+    MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
+
+    HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+    desc.addFamily(new HColumnDescriptor(FAMILY));
+    HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin();
+    hbaseAdmin.createTable(desc, SPLITKEYS);
+
+    assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+
+    HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
+    List<Put> puts = new ArrayList<Put>();
+    Put put1 = new Put(Bytes.toBytes("a"));
+    put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
+    Put put2 = new Put(Bytes.toBytes("h"));
+    put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
+    Put put3 = new Put(Bytes.toBytes("o"));
+    put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
+    puts.add(put1);
+    puts.add(put2);
+    puts.add(put3);
+    table.put(puts);
+    ResultScanner resultScanner = table.getScanner(new Scan());
+    int count = 0;
+    while (resultScanner.next() != null) {
+      count++;
+    }
+    resultScanner.close();
+    table.close();
+    assertEquals(3, count);
+
+    /* Starting test */
+    cluster.getConfiguration().setBoolean("TestingMaster.sleep", true);
+    cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000);
+
+    /* NO.1 .META. region correctness */
+    // First abort master
+    abortMaster(cluster);
+    TestingMaster master = startMasterAndWaitUntilLogSplit(cluster);
+
+    // Second kill meta server
+    int metaServerNum = cluster.getServerWithMeta();
+    int rootServerNum = cluster.getServerWith(HRegionInfo.ROOT_REGIONINFO
+        .getRegionName());
+    HRegionServer metaRS = cluster.getRegionServer(metaServerNum);
+    LOG.debug("Killing metaRS and carryingRoot = "
+        + (metaServerNum == rootServerNum));
+    metaRS.kill();
+    metaRS.join();
+
+    /*
+     * Sleep double time of TestingMaster.sleep.duration, so we can ensure that
+     * master has already assigned ROOTandMETA or is blocking on assigning
+     * ROOTandMETA
+     */
+    Thread.sleep(10000 * 2);
+
+    waitUntilMasterIsInitialized(master);
+
+    // Third check whether data is correct in meta region
+    assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+
+    /*
+     * NO.2 -ROOT- region correctness . If the .META. server killed in the NO.1
+     * is also carrying -ROOT- region, it is not needed
+     */
+    if (rootServerNum != metaServerNum) {
+      // First abort master
+      abortMaster(cluster);
+      master = startMasterAndWaitUntilLogSplit(cluster);
+
+      // Second kill meta server
+      HRegionServer rootRS = cluster.getRegionServer(rootServerNum);
+      LOG.debug("Killing rootRS");
+      rootRS.kill();
+      rootRS.join();
+
+      /*
+       * Sleep double time of TestingMaster.sleep.duration, so we can ensure
+       * that master has already assigned ROOTandMETA or is blocking on
+       * assigning ROOTandMETA
+       */
+      Thread.sleep(10000 * 2);
+      waitUntilMasterIsInitialized(master);
+
+      // Third check whether data is correct in meta region
+      assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+    }
+
+    /* NO.3 data region correctness */
+    ServerManager serverManager = cluster.getMaster().getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
+      Thread.sleep(100);
+    }
+    table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
+    resultScanner = table.getScanner(new Scan());
+    count = 0;
+    while (resultScanner.next() != null) {
+      count++;
+    }
+    resultScanner.close();
+    table.close();
+    assertEquals(3, count);
+  }
+
+  private void abortMaster(MiniHBaseCluster cluster)
+      throws InterruptedException {
+    for (MasterThread mt : cluster.getLiveMasterThreads()) {
+      if (mt.getMaster().isActiveMaster()) {
+        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
+        mt.join();
+        break;
+      }
+    }
+    LOG.debug("Master is aborted");
+  }
+
+  private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
+      throws IOException, InterruptedException {
+    TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
+    while (!master.isLogSplitAfterStartup()) {
+      Thread.sleep(100);
+    }
+    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
+        + master.isInitialized());
+    return master;
+  }
+
+  private void waitUntilMasterIsInitialized(HMaster master)
+      throws InterruptedException {
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    LOG.debug("master isInitialized");
+  }
+  
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+}

From e820dca96f02012782fb115a41b40917ae3fe006 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 13 Mar 2012 19:05:00 +0000
Subject: [PATCH 0038/1540] HBASE-5574 DEFAULT_MAX_FILE_SIZE defaults to a
 negative value

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1300289 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HConstants.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index cb4832538da0..2e2d4f45a686 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -228,7 +228,7 @@ public enum OperationStatusCode {
       "hbase.hregion.max.filesize";
 
   /** Default maximum file size */
-  public static final long DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024;
+  public static final long DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024L;
 
   /**
    * The max number of threads used for opening and closing stores or store

From e4739e0c5cb42bf84aa0907eea19fa61bc0fe24d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 15 Mar 2012 18:49:41 +0000
Subject: [PATCH 0039/1540] HBASE-5569  Do not collect deleted KVs when they
 are still in use by a scanner.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301138 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanQueryMatcher.java   |  3 ++-
 .../regionserver/TestAtomicOperation.java      | 18 +++++++-----------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 9f0cba9eb9c7..221f44694cc9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -306,7 +306,8 @@ public MatchCode match(KeyValue kv) throws IOException {
       }
       // note the following next else if...
       // delete marker are not subject to other delete markers
-    } else if (!this.deletes.isEmpty()) {
+    } else if (!this.deletes.isEmpty()
+        && kv.getMemstoreTS() <= maxReadPointToTrackVersions) {
       DeleteResult deleteResult = deletes.isDeleted(bytes, offset, qualLength,
           timestamp);
       switch (deleteResult) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 1640f296b0fc..c6568703b403 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -40,7 +40,6 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
 import org.junit.experimental.categories.Category;
 
@@ -55,10 +54,7 @@ public class TestAtomicOperation extends HBaseTestCase {
 
   HRegion region = null;
   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final String DIR = TEST_UTIL.getDataTestDir("TestIncrement").toString();
-
-
-  private final int MAX_VERSIONS = 2;
+  private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString();
 
   // Test names
   static final byte[] tableName = Bytes.toBytes("testtable");;
@@ -258,10 +254,10 @@ public void testRowMutationMultiThreads() throws IOException {
     LOG.info("Starting test testRowMutationMultiThreads");
     initHRegion(tableName, getName(), fam1);
 
-    // create 100 threads, each will alternate between adding and
+    // create 50 threads, each will alternate between adding and
     // removing a column
-    int numThreads = 100;
-    int opsPerThread = 1000;
+    int numThreads = 50;
+    int opsPerThread = 500;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
     AtomicLong timeStamps = new AtomicLong(0);
@@ -340,10 +336,10 @@ public void testMultiRowMutationMultiThreads() throws IOException {
     LOG.info("Starting test testMultiRowMutationMultiThreads");
     initHRegion(tableName, getName(), fam1);
 
-    // create 100 threads, each will alternate between adding and
+    // create 50 threads, each will alternate between adding and
     // removing a column
-    int numThreads = 100;
-    int opsPerThread = 1000;
+    int numThreads = 50;
+    int opsPerThread = 500;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
     AtomicLong timeStamps = new AtomicLong(0);

From 8138d76b480337fc90a8aa3fe3054d23f4487e14 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 15 Mar 2012 19:36:04 +0000
Subject: [PATCH 0040/1540] HBASE-4608 HLog Compression

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301167 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |   4 +
 .../regionserver/wal/CompressionContext.java  |  55 +++++
 .../hbase/regionserver/wal/Compressor.java    | 188 +++++++++++++++
 .../hbase/regionserver/wal/Dictionary.java    |  71 ++++++
 .../hadoop/hbase/regionserver/wal/HLog.java   |   9 +
 .../hbase/regionserver/wal/HLogKey.java       |  84 ++++++-
 .../regionserver/wal/KeyValueCompression.java | 126 ++++++++++
 .../hbase/regionserver/wal/LRUDictionary.java | 216 ++++++++++++++++++
 .../wal/SequenceFileLogReader.java            |  43 +++-
 .../wal/SequenceFileLogWriter.java            |  75 +++++-
 .../hbase/regionserver/wal/WALEdit.java       |  24 +-
 .../org/apache/hadoop/hbase/util/Bytes.java   |  12 +
 .../regionserver/wal/TestCompressor.java      |  87 +++++++
 .../wal/TestKeyValueCompression.java          |  81 +++++++
 .../regionserver/wal/TestLRUDictionary.java   | 155 +++++++++++++
 .../hbase/regionserver/wal/TestWALReplay.java |   2 +-
 .../wal/TestWALReplayCompressed.java          |  39 ++++
 17 files changed, 1243 insertions(+), 28 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 2e2d4f45a686..140c0ccf2222 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -636,6 +636,10 @@ public static enum Modify {
   public static final String CHECKSUM_TYPE_NAME =
       "hbase.hstore.checksum.algorithm";
 
+  /** Configuration name of HLog Compression */
+  public static final String ENABLE_WAL_COMPRESSION =
+    "hbase.regionserver.wal.enablecompression";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
new file mode 100644
index 000000000000..5a877af09fa2
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Context that holds the various dictionaries for compression in HLog.
+ */
+@InterfaceAudience.Private
+class CompressionContext {
+  final Dictionary regionDict;
+  final Dictionary tableDict;
+  final Dictionary familyDict;
+  final Dictionary qualifierDict;
+  final Dictionary rowDict;
+
+  public CompressionContext(Class<? extends Dictionary> dictType)
+  throws SecurityException, NoSuchMethodException, InstantiationException,
+      IllegalAccessException, InvocationTargetException {
+    Constructor<? extends Dictionary> dictConstructor =
+        dictType.getConstructor();
+    regionDict = dictConstructor.newInstance();
+    tableDict = dictConstructor.newInstance();
+    familyDict = dictConstructor.newInstance();
+    qualifierDict = dictConstructor.newInstance();
+    rowDict = dictConstructor.newInstance();
+  }
+
+  void clear() {
+    regionDict.clear();
+    tableDict.clear();
+    familyDict.clear();
+    qualifierDict.clear();
+    rowDict.clear();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
new file mode 100644
index 000000000000..8a0dbaf62914
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import com.google.common.base.Preconditions;
+
+/**
+ * A set of static functions for running our custom WAL compression/decompression.
+ * Also contains a command line tool to compress and uncompress HLogs.
+ */
+@InterfaceAudience.Private
+public class Compressor {
+  /**
+   * Command line tool to compress and uncompress WALs.
+   */
+  public static void main(String[] args) throws IOException {
+    if (args.length != 2 || args[0].equals("--help") || args[0].equals("-h")) {
+      printHelp();
+      System.exit(-1);
+    }
+
+    Path inputPath = new Path(args[0]);
+    Path outputPath = new Path(args[1]);
+
+    transformFile(inputPath, outputPath);
+  }
+
+  private static void printHelp() {
+    System.err.println("usage: Compressor <input> <output>");
+    System.err.println("If <input> HLog is compressed, <output> will be decompressed.");
+    System.err.println("If <input> HLog is uncompressed, <output> will be compressed.");
+    return;
+  }
+
+  private static void transformFile(Path input, Path output)
+      throws IOException {
+    SequenceFileLogReader in = new SequenceFileLogReader();
+    SequenceFileLogWriter out = new SequenceFileLogWriter();
+
+    try {
+      Configuration conf = HBaseConfiguration.create();
+
+      FileSystem inFS = input.getFileSystem(conf);
+      FileSystem outFS = output.getFileSystem(conf);
+
+      in.init(inFS, input, conf);
+      boolean compress = in.reader.isWALCompressionEnabled();
+
+      conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
+      out.init(outFS, output, conf);
+
+      Entry e = null;
+      while ((e = in.next()) != null) out.append(e);
+    } finally {
+      in.close();
+      out.close();
+    }
+  }
+
+  /**
+   * Reads the next compressed entry and returns it as a byte array
+   * 
+   * @param in the DataInput to read from
+   * @param dict the dictionary we use for our read.
+   * 
+   * @param the uncompressed array.
+   */
+  static byte[] readCompressed(DataInput in, Dictionary dict)
+      throws IOException {
+    byte status = in.readByte();
+
+    if (status == Dictionary.NOT_IN_DICTIONARY) {
+      int length = WritableUtils.readVInt(in);
+      // if this isn't in the dictionary, we need to add to the dictionary.
+      byte[] arr = new byte[length];
+      in.readFully(arr);
+      if (dict != null) dict.addEntry(arr, 0, length);
+      return arr;
+    } else {
+      // Status here is the higher-order byte of index of the dictionary entry
+      // (when its not Dictionary.NOT_IN_DICTIONARY -- dictionary indices are
+      // shorts).
+      short dictIdx = toShort(status, in.readByte());
+      byte[] entry = dict.getEntry(dictIdx);
+      if (entry == null) {
+        throw new IOException("Missing dictionary entry for index "
+            + dictIdx);
+      }
+      return entry;
+    }
+  }
+
+  /**
+   * Reads a compressed entry into an array.
+   * The output into the array ends up length-prefixed.
+   * 
+   * @param to the array to write into
+   * @param offset array offset to start writing to
+   * @param in the DataInput to read from
+   * @param dict the dictionary to use for compression
+   * 
+   * @return the length of the uncompressed data
+   */
+  static int uncompressIntoArray(byte[] to, int offset, DataInput in,
+      Dictionary dict) throws IOException {
+    byte status = in.readByte();
+
+    if (status == Dictionary.NOT_IN_DICTIONARY) {
+      // status byte indicating that data to be read is not in dictionary.
+      // if this isn't in the dictionary, we need to add to the dictionary.
+      int length = WritableUtils.readVInt(in);
+      in.readFully(to, offset, length);
+      dict.addEntry(to, offset, length);
+      return length;
+    } else {
+      // the status byte also acts as the higher order byte of the dictionary
+      // entry
+      short dictIdx = toShort(status, in.readByte());
+      byte[] entry = dict.getEntry(dictIdx);
+      if (entry == null) {
+        throw new IOException("Missing dictionary entry for index "
+            + dictIdx);
+      }
+      // now we write the uncompressed value.
+      Bytes.putBytes(to, offset, entry, 0, entry.length);
+      return entry.length;
+    }
+  }
+
+  /**
+   * Compresses and writes an array to a DataOutput
+   * 
+   * @param data the array to write.
+   * @param out the DataOutput to write into
+   * @param dict the dictionary to use for compression
+   */
+  static void writeCompressed(byte[] data, int offset, int length,
+      DataOutput out, Dictionary dict)
+      throws IOException {
+    short dictIdx = Dictionary.NOT_IN_DICTIONARY;
+    if (dict != null) {
+      dictIdx = dict.findEntry(data, offset, length);
+    }
+    if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
+      // not in dict
+      out.writeByte(Dictionary.NOT_IN_DICTIONARY);
+      WritableUtils.writeVInt(out, length);
+      out.write(data, offset, length);
+    } else {
+      out.writeShort(dictIdx);
+    }
+  }
+
+  static short toShort(byte hi, byte lo) {
+    short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
+    Preconditions.checkArgument(s >= 0);
+    return s;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java
new file mode 100644
index 000000000000..e1cfed195bb8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Dictionary interface
+ *
+ * Dictionary indexes should be either bytes or shorts, only positive. (The
+ * first bit is reserved for detecting whether something is compressed or not).
+ */
+@InterfaceAudience.Private
+interface Dictionary {
+  static final byte NOT_IN_DICTIONARY = -1;
+
+  /**
+   * Gets an entry from the dictionary.
+   * 
+   * @param idx index of the entry
+   * @return the entry, or null if non existent
+   */
+  public byte[] getEntry(short idx);
+
+  /**
+   * Finds the index of an entry.
+   * If no entry found, we add it.
+   * 
+   * @param data the byte array that we're looking up
+   * @param offset Offset into <code>data</code> to add to Dictionary.
+   * @param length Length beyond <code>offset</code> that comprises entry; must be > 0.
+   * @return the index of the entry, or {@link #NOT_IN_DICTIONARY} if not found
+   */
+  public short findEntry(byte[] data, int offset, int length);
+
+  /**
+   * Adds an entry to the dictionary.
+   * Be careful using this method.  It will add an entry to the
+   * dictionary even if it already has an entry for the same data.
+   * Call {{@link #findEntry(byte[], int, int)}} to add without duplicating
+   * dictionary entries.
+   * 
+   * @param data the entry to add
+   * @param offset Offset into <code>data</code> to add to Dictionary.
+   * @param length Length beyond <code>offset</code> that comprises entry; must be > 0.
+   * @return the index of the entry
+   */
+
+  public short addEntry(byte[] data, int offset, int length);
+
+  /**
+   * Flushes the dictionary, empties all values.
+   */
+  public void clear();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index c945a99f9dc4..6720a5bde75c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1669,6 +1669,15 @@ public HLogKey getKey() {
       return key;
     }
 
+    /**
+     * Set compression context for this entry.
+     * @param compressionContext Compression context
+     */
+    public void setCompressionContext(CompressionContext compressionContext) {
+      edit.setCompressionContext(compressionContext);
+      key.setCompressionContext(compressionContext);
+    }
+
     @Override
     public String toString() {
       return this.key + "=" + this.edit;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
index f067221a2587..95114811c366 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
@@ -44,7 +44,40 @@
  */
 public class HLogKey implements WritableComparable<HLogKey> {
   // should be < 0 (@see #readFields(DataInput))
-  private static final int VERSION = -1;
+  // version 2 supports HLog compression
+  enum Version {
+    UNVERSIONED(0),
+    // Initial number we put on HLogKey when we introduced versioning.
+    INITIAL(-1),
+    // Version -2 introduced a dictionary compression facility.  Only this
+    // dictionary-based compression is available in version -2.
+    COMPRESSED(-2);
+
+    final int code;
+    static final Version[] byCode;
+    static {
+      byCode = Version.values();
+      for (int i = 0; i < byCode.length; i++) {
+        if (byCode[i].code != -1 * i) {
+          throw new AssertionError("Values in this enum should be descending by one");
+        }
+      }
+    }
+
+    Version(int code) {
+      this.code = code;
+    }
+
+    boolean atLeast(Version other) {
+      return code <= other.code;
+    }
+
+    static Version fromCode(int code) {
+      return byCode[code * -1];
+    }
+  }
+
+  private static final Version VERSION = Version.COMPRESSED;
 
   //  The encoded region name.
   private byte [] encodedRegionName;
@@ -55,7 +88,9 @@ public class HLogKey implements WritableComparable<HLogKey> {
 
   private UUID clusterId;
 
-  /** Writable Consructor -- Do not use. */
+  private CompressionContext compressionContext;
+
+  /** Writable Constructor -- Do not use. */
   public HLogKey() {
     this(null, null, 0L, HConstants.LATEST_TIMESTAMP,
         HConstants.DEFAULT_CLUSTER_ID);
@@ -82,6 +117,13 @@ public HLogKey(final byte [] encodedRegionName, final byte [] tablename,
     this.clusterId = clusterId;
   }
 
+  /**
+   * @param compressionContext Compression context to use
+   */
+  public void setCompressionContext(CompressionContext compressionContext) {
+    this.compressionContext = compressionContext;
+  }
+
   /** @return encoded region name */
   public byte [] getEncodedRegionName() {
     return encodedRegionName;
@@ -213,9 +255,17 @@ void internEncodedRegionName(byte []encodedRegionName) {
 
   @Override
   public void write(DataOutput out) throws IOException {
-    WritableUtils.writeVInt(out, VERSION);
-    Bytes.writeByteArray(out, this.encodedRegionName);
-    Bytes.writeByteArray(out, this.tablename);
+    WritableUtils.writeVInt(out, VERSION.code);
+    if (compressionContext == null) {
+      Bytes.writeByteArray(out, this.encodedRegionName);
+      Bytes.writeByteArray(out, this.tablename);
+    } else {
+      Compressor.writeCompressed(this.encodedRegionName, 0,
+          this.encodedRegionName.length, out,
+          compressionContext.regionDict);
+      Compressor.writeCompressed(this.tablename, 0, this.tablename.length, out,
+          compressionContext.tableDict);
+    }
     out.writeLong(this.logSeqNum);
     out.writeLong(this.writeTime);
     // avoid storing 16 bytes when replication is not enabled
@@ -230,7 +280,7 @@ public void write(DataOutput out) throws IOException {
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    int version = 0;
+    Version version = Version.UNVERSIONED;
     // HLogKey was not versioned in the beginning.
     // In order to introduce it now, we make use of the fact
     // that encodedRegionName was written with Bytes.writeByteArray,
@@ -242,16 +292,26 @@ public void readFields(DataInput in) throws IOException {
     int len = WritableUtils.readVInt(in);
     if (len < 0) {
       // what we just read was the version
-      version = len;
-      len = WritableUtils.readVInt(in);
+      version = Version.fromCode(len);
+      // We only compress V2 of HLogkey.
+      // If compression is on, the length is handled by the dictionary
+      if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) {
+        len = WritableUtils.readVInt(in);
+      }
+    }
+    if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) {
+      this.encodedRegionName = new byte[len];
+      in.readFully(this.encodedRegionName);
+      this.tablename = Bytes.readByteArray(in);
+    } else {
+      this.encodedRegionName = Compressor.readCompressed(in, compressionContext.regionDict);
+      this.tablename = Compressor.readCompressed(in, compressionContext.tableDict);
     }
-    this.encodedRegionName = new byte[len];
-    in.readFully(this.encodedRegionName);
-    this.tablename = Bytes.readByteArray(in);
+    
     this.logSeqNum = in.readLong();
     this.writeTime = in.readLong();
     this.clusterId = HConstants.DEFAULT_CLUSTER_ID;
-    if (version < 0) {
+    if (version.atLeast(Version.INITIAL)) {
       if (in.readBoolean()) {
         this.clusterId = new UUID(in.readLong(), in.readLong());
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
new file mode 100644
index 000000000000..0f9743f70a09
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Compression class for {@link KeyValue}s written to the WAL. This is not
+ * synchronized, so synchronization should be handled outside.
+ * 
+ * Class only compresses and uncompresses row keys, family names, and the
+ * qualifier. More may be added depending on use patterns.
+ */
+class KeyValueCompression {
+  /**
+   * Uncompresses a KeyValue from a DataInput and returns it.
+   * 
+   * @param in the DataInput
+   * @param readContext the compressionContext to use.
+   * @return an uncompressed KeyValue
+   * @throws IOException
+   */
+
+  public static KeyValue readKV(DataInput in, CompressionContext readContext)
+      throws IOException {
+    int keylength = WritableUtils.readVInt(in);
+    int vlength = WritableUtils.readVInt(in);
+    int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
+
+    byte[] backingArray = new byte[length];
+    int pos = 0;
+    pos = Bytes.putInt(backingArray, pos, keylength);
+    pos = Bytes.putInt(backingArray, pos, vlength);
+
+    // the row
+    int elemLen = Compressor.uncompressIntoArray(backingArray,
+        pos + Bytes.SIZEOF_SHORT, in, readContext.rowDict);
+    checkLength(elemLen, Short.MAX_VALUE);
+    pos = Bytes.putShort(backingArray, pos, (short)elemLen);
+    pos += elemLen;
+
+    // family
+    elemLen = Compressor.uncompressIntoArray(backingArray,
+        pos + Bytes.SIZEOF_BYTE, in, readContext.familyDict);
+    checkLength(elemLen, Byte.MAX_VALUE);
+    pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
+    pos += elemLen;
+
+    // qualifier
+    elemLen = Compressor.uncompressIntoArray(backingArray, pos, in,
+        readContext.qualifierDict);
+    pos += elemLen;
+
+    // the rest
+    in.readFully(backingArray, pos, length - pos);
+
+    return new KeyValue(backingArray);
+  }
+
+  private static void checkLength(int len, int max) throws IOException {
+    if (len < 0 || len > max) {
+      throw new IOException(
+          "Invalid length for compresesed portion of keyvalue: " + len);
+    }
+  }
+
+  /**
+   * Compresses and writes ourKV to out, a DataOutput.
+   * 
+   * @param out the DataOutput
+   * @param keyVal the KV to compress and write
+   * @param writeContext the compressionContext to use.
+   * @throws IOException
+   */
+  public static void writeKV(final DataOutput out, KeyValue keyVal,
+      CompressionContext writeContext) throws IOException {
+    byte[] backingArray = keyVal.getBuffer();
+    int offset = keyVal.getOffset();
+
+    // we first write the KeyValue infrastructure as VInts.
+    WritableUtils.writeVInt(out, keyVal.getKeyLength());
+    WritableUtils.writeVInt(out, keyVal.getValueLength());
+
+    // now we write the row key, as the row key is likely to be repeated
+    // We save space only if we attempt to compress elements with duplicates
+    Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getRowOffset(),
+        keyVal.getRowLength(), out, writeContext.rowDict);
+
+  
+    // now family, if it exists. if it doesn't, we write a 0 length array.
+    Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getFamilyOffset(),
+        keyVal.getFamilyLength(), out, writeContext.familyDict);
+
+    // qualifier next
+    Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getQualifierOffset(),
+        keyVal.getQualifierLength(), out,
+        writeContext.qualifierDict);
+
+    // now we write the rest uncompressed
+    int pos = keyVal.getTimestampOffset();
+    int remainingLength = keyVal.getLength() + offset - (pos);
+    out.write(backingArray, pos, remainingLength);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java
new file mode 100644
index 000000000000..6e0b20bd65da
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.util.HashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * WALDictionary using an LRU eviction algorithm. Uses a linked list running
+ * through a hashtable.  Currently has max of 2^15 entries.  Will start
+ * evicting if exceeds this number  The maximum memory we expect this dictionary
+ * to take in the worst case is about:
+ * <code>(2 ^ 15) * 5 (Regionname, Row key, CF, Column qual, table) * 100 bytes (these are some big names) = ~16MB</code>.
+ * If you want to get silly, even at 1kb entries, it maxes out at 160 megabytes.
+ */
+@InterfaceAudience.Private
+public class LRUDictionary implements Dictionary {
+  private final BidirectionalLRUMap backingStore = new BidirectionalLRUMap();
+
+  @Override
+  public byte[] getEntry(short idx) {
+    return backingStore.get(idx);
+  }
+
+  @Override
+  public short findEntry(byte[] data, int offset, int length) {
+    short ret = backingStore.findIdx(data, offset, length);
+    if (ret == NOT_IN_DICTIONARY) {
+      addEntry(data, offset, length);
+    }
+    return ret;
+  }
+
+  @Override
+  public short addEntry(byte[] data, int offset, int length) {
+    if (length <= 0) return NOT_IN_DICTIONARY;
+    return backingStore.put(data, offset, length);
+  }
+
+  @Override
+  public void clear() {
+    backingStore.clear();
+  }
+
+  /*
+   * Internal class used to implement LRU eviction and dual lookup (by key and
+   * value).
+   * 
+   * This is not thread safe. Don't use in multi-threaded applications.
+   */
+  static class BidirectionalLRUMap {
+    static final int MAX_SIZE = Short.MAX_VALUE;
+    private int currSize = 0;
+
+    // Head and tail of the LRU list.
+    private Node head;
+    private Node tail;
+
+    private HashMap<Node, Short> nodeToIndex = new HashMap<Node, Short>();
+    private Node[] indexToNode = new Node[MAX_SIZE];
+
+    public BidirectionalLRUMap() {
+      for (int i = 0; i < MAX_SIZE; i++) {
+        indexToNode[i] = new Node();
+      }
+    }
+
+    private short put(byte[] array, int offset, int length) {
+      // We copy the bytes we want, otherwise we might be holding references to
+      // massive arrays in our dictionary (or those arrays might change)
+      byte[] stored = new byte[length];
+      Bytes.putBytes(stored, 0, array, offset, length);
+
+      if (currSize < MAX_SIZE) {
+        // There is space to add without evicting.
+        indexToNode[currSize].setContents(stored, 0, stored.length);
+        setHead(indexToNode[currSize]);
+        short ret = (short) currSize++;
+        nodeToIndex.put(indexToNode[ret], ret);
+        return ret;
+      } else {
+        short s = nodeToIndex.remove(tail);
+        tail.setContents(stored, 0, stored.length);
+        // we need to rehash this.
+        nodeToIndex.put(tail, s);
+        moveToHead(tail);
+        return s;
+      }
+    }
+
+    private short findIdx(byte[] array, int offset, int length) {
+      Short s;
+      final Node comparisonNode = new Node();
+      comparisonNode.setContents(array, offset, length);
+      if ((s = nodeToIndex.get(comparisonNode)) != null) {
+        moveToHead(indexToNode[s]);
+        return s;
+      } else {
+        return -1;
+      }
+    }
+
+    private byte[] get(short idx) {
+      Preconditions.checkElementIndex(idx, currSize);
+      moveToHead(indexToNode[idx]);
+      return indexToNode[idx].container;
+    }
+
+    private void moveToHead(Node n) {
+      if (head == n) {
+        // no-op -- it's already the head.
+        return;
+      }
+      // At this point we definitely have prev, since it's not the head.
+      assert n.prev != null;
+      // Unlink prev.
+      n.prev.next = n.next;
+
+      // Unlink next
+      if (n.next != null) {
+        n.next.prev = n.prev;
+      } else {
+        assert n == tail;
+        tail = n.prev;
+      }
+      // Node is now removed from the list. Re-add it at the head.
+      setHead(n);
+    }
+    
+    private void setHead(Node n) {
+      // assume it's already unlinked from the list at this point.
+      n.prev = null;
+      n.next = head;
+      if (head != null) {
+        assert head.prev == null;
+        head.prev = n;
+      }
+
+      head = n;
+
+      // First entry
+      if (tail == null) {
+        tail = n;
+      }
+    }
+
+    private void clear() {
+      currSize = 0;
+      nodeToIndex.clear();
+      tail = null;
+      head = null;
+
+      for (Node n : indexToNode) {
+        n.container = null;
+      }
+
+      for (int i = 0; i < MAX_SIZE; i++) {
+        indexToNode[i].next = null;
+        indexToNode[i].prev = null;
+      }
+    }
+
+    private static class Node {
+      byte[] container;
+      int offset;
+      int length;
+      Node next; // link towards the tail
+      Node prev; // link towards the head
+
+      public Node() {
+      }
+
+      private void setContents(byte[] container, int offset, int length) {
+        this.container = container;
+        this.offset = offset;
+        this.length = length;
+      }
+
+      @Override
+      public int hashCode() {
+        return Bytes.hashCode(container, offset, length);
+      }
+
+      @Override
+      public boolean equals(Object other) {
+        if (!(other instanceof Node)) {
+          return false;
+        }
+
+        Node casted = (Node) other;
+        return Bytes.equals(container, offset, length, casted.container,
+            casted.offset, casted.length);
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index d9cd6dead27a..f5fb00f1218d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -22,11 +22,8 @@
 
 import java.io.FilterInputStream;
 import java.io.IOException;
-import java.lang.Class;
-import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
-import java.util.Arrays;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -56,7 +53,6 @@ static class WALReader extends SequenceFile.Reader {
     WALReader(final FileSystem fs, final Path p, final Configuration c)
     throws IOException {
       super(fs, p, c);
-
     }
 
     @Override
@@ -67,6 +63,15 @@ protected FSDataInputStream openFile(FileSystem fs, Path file,
         bufferSize, length), length);
     }
 
+    /**
+     * Call this method after init() has been executed
+     * 
+     * @return whether WAL compression is enabled
+     */
+    public boolean isWALCompressionEnabled() {
+      return SequenceFileLogWriter.isWALCompressionEnabled(this.getMetadata());
+    }
+
     /**
      * Override just so can intercept first call to getPos.
      */
@@ -134,10 +139,15 @@ public long getPos() throws IOException {
 
   Configuration conf;
   WALReader reader;
+
   // Needed logging exceptions
   Path path;
   int edit = 0;
   long entryStart = 0;
+  /**
+   * Compression context to use reading.  Can be null if no compression.
+   */
+  private CompressionContext compressionContext = null;
 
   protected Class<? extends HLogKey> keyClass;
 
@@ -157,19 +167,35 @@ public SequenceFileLogReader(Class<? extends HLogKey> keyClass) {
     this.keyClass = keyClass;
   }
 
-
   @Override
   public void init(FileSystem fs, Path path, Configuration conf)
       throws IOException {
     this.conf = conf;
     this.path = path;
     reader = new WALReader(fs, path, conf);
+
+    // If compression is enabled, new dictionaries are created here.
+    boolean compression = reader.isWALCompressionEnabled();
+    if (compression) {
+      try {
+        if (compressionContext == null) {
+          compressionContext = new CompressionContext(LRUDictionary.class);
+        } else {
+          compressionContext.clear();
+        }
+      } catch (Exception e) {
+        throw new IOException("Failed to initialize CompressionContext", e);
+      }
+    }
   }
 
   @Override
   public void close() throws IOException {
     try {
-      reader.close();
+      if (reader != null) {
+        this.reader.close();
+        this.reader = null;
+      }
     } catch (IOException ioe) {
       throw addFileInfoToException(ioe);
     }
@@ -203,6 +229,9 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
     }
     boolean b = false;
     try {
+      if (compressionContext != null) {
+        e.setCompressionContext(compressionContext);
+      }
       b = this.reader.next(e.getKey(), e.getEdit());
     } catch (IOException ioe) {
       throw addFileInfoToException(ioe);
@@ -257,4 +286,4 @@ protected IOException addFileInfoToException(final IOException ioe)
 
     return ioe;
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index bd31ead1af54..c280ae077495 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -25,6 +25,7 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -32,7 +33,9 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.SequenceFile.Metadata;
 import org.apache.hadoop.io.compress.CompressionCodec;
@@ -43,6 +46,16 @@
  * SequenceFile.Writer.
  */
 public class SequenceFileLogWriter implements HLog.Writer {
+  static final Text WAL_VERSION_KEY = new Text("version");
+  // Let the version be 1.  Let absence of a version meta tag be old, version 0.
+  // Set this version '1' to be the version that introduces compression,
+  // the COMPRESSION_VERSION.
+  private static final int COMPRESSION_VERSION = 1;
+  static final int VERSION = COMPRESSION_VERSION;
+  static final Text WAL_VERSION = new Text("" + VERSION);
+  static final Text WAL_COMPRESSION_TYPE_KEY = new Text("compression.type");
+  static final Text DICTIONARY_COMPRESSION_TYPE = new Text("dictionary");
+
   private final Log LOG = LogFactory.getLog(this.getClass());
   // The sequence file we delegate to.
   private SequenceFile.Writer writer;
@@ -52,6 +65,13 @@ public class SequenceFileLogWriter implements HLog.Writer {
 
   private Class<? extends HLogKey> keyClass;
 
+  /**
+   * Context used by our wal dictionary compressor.  Null if we're not to do
+   * our custom dictionary compression.  This custom WAL compression is distinct
+   * from sequencefile native compression.
+   */
+  private CompressionContext compressionContext;
+
   private Method syncFs = null;
   private Method hflush = null;
 
@@ -72,9 +92,56 @@ public SequenceFileLogWriter(Class<? extends HLogKey> keyClass) {
     this.keyClass = keyClass;
   }
 
+  /**
+   * Create sequence file Metadata for our WAL file with version and compression
+   * type (if any).
+   * @param conf
+   * @param compress
+   * @return Metadata instance.
+   */
+  private static Metadata createMetadata(final Configuration conf,
+      final boolean compress) {
+    TreeMap<Text, Text> metaMap = new TreeMap<Text, Text>();
+    metaMap.put(WAL_VERSION_KEY, WAL_VERSION);
+    if (compress) {
+      // Currently we only do one compression type.
+      metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
+    }
+    return new Metadata(metaMap);
+  }
+
+  /**
+   * Call this method after init() has been executed
+   * 
+   * @return whether WAL compression is enabled
+   */
+  static boolean isWALCompressionEnabled(final Metadata metadata) {
+    // Check version is >= VERSION?
+    Text txt = metadata.get(WAL_VERSION_KEY);
+    if (txt == null || Integer.parseInt(txt.toString()) < COMPRESSION_VERSION) {
+      return false;
+    }
+    // Now check that compression type is present.  Currently only one value.
+    txt = metadata.get(WAL_COMPRESSION_TYPE_KEY);
+    return txt != null && txt.equals(DICTIONARY_COMPRESSION_TYPE);
+  }
+
   @Override
   public void init(FileSystem fs, Path path, Configuration conf)
   throws IOException {
+    // Should we do our custom WAL compression?
+    boolean compress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);
+    if (compress) {
+      try {
+        if (this.compressionContext == null) {
+          this.compressionContext = new CompressionContext(LRUDictionary.class);
+        } else {
+          this.compressionContext.clear();
+        }
+      } catch (Exception e) {
+        throw new IOException("Failed to initiate CompressionContext", e);
+      }
+    }
 
     if (null == keyClass) {
       keyClass = HLog.getKeyClass(conf);
@@ -99,7 +166,7 @@ public void init(FileSystem fs, Path path, Configuration conf)
                 fs.getDefaultBlockSize())),
             Boolean.valueOf(false) /*createParent*/,
             SequenceFile.CompressionType.NONE, new DefaultCodec(),
-            new Metadata()
+            createMetadata(conf, compress)
             });
     } catch (InvocationTargetException ite) {
       // function was properly called, but threw it's own exception
@@ -121,7 +188,7 @@ SequenceFile.CompressionType.NONE, new DefaultCodec(),
         SequenceFile.CompressionType.NONE,
         new DefaultCodec(),
         null,
-        new Metadata());
+        createMetadata(conf, compress));
     } else {
       LOG.debug("using new createWriter -- HADOOP-6840");
     }
@@ -131,7 +198,8 @@ SequenceFile.CompressionType.NONE, new DefaultCodec(),
     this.hflush = getHFlush();
     String msg = "Path=" + path +
       ", syncFs=" + (this.syncFs != null) +
-      ", hflush=" + (this.hflush != null);
+      ", hflush=" + (this.hflush != null) +
+      ", compression=" + compress;
     if (this.syncFs != null || this.hflush != null) {
       LOG.debug(msg);
     } else {
@@ -205,6 +273,7 @@ private FSDataOutputStream getSequenceFilePrivateFSDataOutputStreamAccessible()
 
   @Override
   public void append(HLog.Entry entry) throws IOException {
+    entry.setCompressionContext(compressionContext);
     this.writer.append(entry.getKey(), entry.getEdit());
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index e1117ef6ea26..efd5a32cf68e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -74,9 +74,15 @@ public class WALEdit implements Writable, HeapSize {
   private final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
   private NavigableMap<byte[], Integer> scopes;
 
+  private CompressionContext compressionContext;
+
   public WALEdit() {
   }
 
+  public void setCompressionContext(final CompressionContext compressionContext) {
+    this.compressionContext = compressionContext;
+  }
+
   public void add(KeyValue kv) {
     this.kvs.add(kv);
   }
@@ -114,9 +120,13 @@ public void readFields(DataInput in) throws IOException {
       // this is new style HLog entry containing multiple KeyValues.
       int numEdits = in.readInt();
       for (int idx = 0; idx < numEdits; idx++) {
-        KeyValue kv = new KeyValue();
-        kv.readFields(in);
-        this.add(kv);
+        if (compressionContext != null) {
+          this.add(KeyValueCompression.readKV(in, compressionContext));
+        } else {
+          KeyValue kv = new KeyValue();
+          kv.readFields(in);
+          this.add(kv);
+    	  }
       }
       int numFamilies = in.readInt();
       if (numFamilies > 0) {
@@ -131,7 +141,7 @@ public void readFields(DataInput in) throws IOException {
       }
     } else {
       // this is an old style HLog entry. The int that we just
-      // read is actually the length of a single KeyValue.
+      // read is actually the length of a single KeyValue
       KeyValue kv = new KeyValue();
       kv.readFields(versionOrLength, in);
       this.add(kv);
@@ -144,7 +154,11 @@ public void write(DataOutput out) throws IOException {
     out.writeInt(kvs.size());
     // We interleave the two lists for code simplicity
     for (KeyValue kv : kvs) {
-      kv.write(out);
+      if (compressionContext != null) {
+        KeyValueCompression.writeKV(out, kv, compressionContext);
+      } else{
+        kv.write(out);
+      }
     }
     if (scopes == null) {
       out.writeInt(0);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index ead9a3bf2ad5..ece0aae01a43 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -1456,6 +1456,18 @@ public Iterator<byte[]> iterator() {
     };
   }
 
+  /**
+   * @param bytes array to hash
+   * @param offset offset to start from
+   * @param length length to hash
+   * */
+  public static int hashCode(byte[] bytes, int offset, int length) {
+    int hash = 1;
+    for (int i = offset; i < offset + length; i++)
+      hash = (31 * hash) + (int) bytes[i];
+    return hash;
+  }
+
   /**
    * @param t operands
    * @return Array of byte arrays made from passed array of Text
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java
new file mode 100644
index 000000000000..dad681d85d42
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test our compressor class.
+ */
+@Category(SmallTests.class)
+public class TestCompressor {
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+  }
+
+  @Test
+  public void testToShort() {
+    short s = 1;
+    assertEquals(s, Compressor.toShort((byte)0, (byte)1));
+    s <<= 8;
+    assertEquals(s, Compressor.toShort((byte)1, (byte)0));
+  }
+
+  @Test (expected = IllegalArgumentException.class)
+  public void testNegativeToShort() {
+    Compressor.toShort((byte)0xff, (byte)0xff);
+  }
+
+  @Test
+  public void testCompressingWithNullDictionaries() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+    byte [] blahBytes = Bytes.toBytes("blah");
+    Compressor.writeCompressed(blahBytes, 0, blahBytes.length, dos, null);
+    dos.close();
+    byte [] dosbytes = baos.toByteArray();
+    DataInputStream dis =
+      new DataInputStream(new ByteArrayInputStream(dosbytes));
+    byte [] product = Compressor.readCompressed(dis, null);
+    assertTrue(Bytes.equals(blahBytes, product));
+  }
+
+  @Test
+  public void testCompressingWithClearDictionaries() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+    Dictionary dictionary = new LRUDictionary();
+    byte [] blahBytes = Bytes.toBytes("blah");
+    Compressor.writeCompressed(blahBytes, 0, blahBytes.length, dos, dictionary);
+    dos.close();
+    byte [] dosbytes = baos.toByteArray();
+    DataInputStream dis =
+      new DataInputStream(new ByteArrayInputStream(dosbytes));
+    dictionary = new LRUDictionary();
+    byte [] product = Compressor.readCompressed(dis, dictionary);
+    assertTrue(Bytes.equals(blahBytes, product));
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java
new file mode 100644
index 000000000000..8fa7fe883b68
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.util.List;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.*;
+
+import com.google.common.collect.Lists;
+
+@Category(SmallTests.class)
+public class TestKeyValueCompression {
+  private static final byte[] VALUE = Bytes.toBytes("fake value");
+  private static final int BUF_SIZE = 256*1024;
+  
+  @Test
+  public void testCountingKVs() throws Exception {
+    List<KeyValue> kvs = Lists.newArrayList();
+    for (int i = 0; i < 400; i++) {
+      byte[] row = Bytes.toBytes("row" + i);
+      byte[] fam = Bytes.toBytes("fam" + i);
+      byte[] qual = Bytes.toBytes("qual" + i);
+      kvs.add(new KeyValue(row, fam, qual, 12345L, VALUE));
+    }
+    
+    runTestCycle(kvs);
+  }
+  
+  @Test
+  public void testRepeatingKVs() throws Exception {
+    List<KeyValue> kvs = Lists.newArrayList();
+    for (int i = 0; i < 400; i++) {
+      byte[] row = Bytes.toBytes("row" + (i % 10));
+      byte[] fam = Bytes.toBytes("fam" + (i % 127));
+      byte[] qual = Bytes.toBytes("qual" + (i % 128));
+      kvs.add(new KeyValue(row, fam, qual, 12345L, VALUE));
+    }
+    
+    runTestCycle(kvs);
+  }
+
+  private void runTestCycle(List<KeyValue> kvs) throws Exception {
+    CompressionContext ctx = new CompressionContext(LRUDictionary.class);
+    DataOutputBuffer buf = new DataOutputBuffer(BUF_SIZE);
+    for (KeyValue kv : kvs) {
+      KeyValueCompression.writeKV(buf, kv, ctx);
+    }
+
+    ctx.clear();
+    DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+        buf.getData(), 0, buf.getLength()));
+    for (KeyValue kv : kvs) {
+      KeyValue readBack = KeyValueCompression.readKV(in, ctx);
+      assertEquals(kv, readBack);
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java
new file mode 100644
index 000000000000..99983a2f5cc1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.*;
+
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests LRUDictionary
+ */
+@Category(SmallTests.class)
+public class TestLRUDictionary {
+  LRUDictionary testee;
+
+  @Before
+  public void setUp() throws Exception {
+    testee = new LRUDictionary();
+  }
+
+  @Test
+  public void TestContainsNothing() {
+    assertTrue(isDictionaryEmpty(testee));
+  }
+
+  /**
+   * Assert can't add empty array.
+   */
+  @Test
+  public void testPassingEmptyArrayToFindEntry() {
+    assertEquals(Dictionary.NOT_IN_DICTIONARY,
+      testee.findEntry(HConstants.EMPTY_BYTE_ARRAY, 0, 0));
+    assertEquals(Dictionary.NOT_IN_DICTIONARY,
+      testee.addEntry(HConstants.EMPTY_BYTE_ARRAY, 0, 0));
+  }
+
+  @Test
+  public void testPassingSameArrayToAddEntry() {
+    // Add random predefined byte array, in this case a random byte array from
+    // HConstants.  Assert that when we add, we get new index.  Thats how it
+    // works.
+    int len = HConstants.CATALOG_FAMILY.length;
+    int index = testee.addEntry(HConstants.CATALOG_FAMILY, 0, len);
+    assertFalse(index == testee.addEntry(HConstants.CATALOG_FAMILY, 0, len));
+    assertFalse(index == testee.addEntry(HConstants.CATALOG_FAMILY, 0, len));
+  }
+
+  @Test
+  public void testBasic() {
+    Random rand = new Random();
+    byte[] testBytes = new byte[10];
+    rand.nextBytes(testBytes);
+
+    // Verify that our randomly generated array doesn't exist in the dictionary
+    assertEquals(testee.findEntry(testBytes, 0, testBytes.length), -1);
+
+    // now since we looked up an entry, we should have added it to the
+    // dictionary, so it isn't empty
+
+    assertFalse(isDictionaryEmpty(testee));
+
+    // Check if we can find it using findEntry
+    short t = testee.findEntry(testBytes, 0, testBytes.length);
+
+    // Making sure we do find what we're looking for
+    assertTrue(t != -1);
+
+    byte[] testBytesCopy = new byte[20];
+
+    Bytes.putBytes(testBytesCopy, 10, testBytes, 0, testBytes.length);
+
+    // copy byte arrays, make sure that we check that equal byte arrays are
+    // equal without just checking the reference
+    assertEquals(testee.findEntry(testBytesCopy, 10, testBytes.length), t);
+
+    // make sure the entry retrieved is the same as the one put in
+    assertTrue(Arrays.equals(testBytes, testee.getEntry(t)));
+
+    testee.clear();
+
+    // making sure clear clears the dictionary
+    assertTrue(isDictionaryEmpty(testee));
+  }
+
+  @Test
+  public void TestLRUPolicy(){
+    //start by filling the dictionary up with byte arrays
+    for (int i = 0; i < LRUDictionary.BidirectionalLRUMap.MAX_SIZE; i++) {
+      testee.findEntry((BigInteger.valueOf(i)).toByteArray(), 0,
+          (BigInteger.valueOf(i)).toByteArray().length);
+    }
+
+    // check we have the first element added
+    assertTrue(testee.findEntry(BigInteger.ZERO.toByteArray(), 0,
+        BigInteger.ZERO.toByteArray().length) != -1);
+
+    // check for an element we know isn't there
+    assertTrue(testee.findEntry(BigInteger.valueOf(Integer.MAX_VALUE).toByteArray(), 0,
+        BigInteger.valueOf(Integer.MAX_VALUE).toByteArray().length) == -1);
+
+    // since we just checked for this element, it should be there now.
+    assertTrue(testee.findEntry(BigInteger.valueOf(Integer.MAX_VALUE).toByteArray(), 0,
+        BigInteger.valueOf(Integer.MAX_VALUE).toByteArray().length) != -1);
+
+    // test eviction, that the least recently added or looked at element is
+    // evicted.  We looked at ZERO so it should be in the dictionary still.
+    assertTrue(testee.findEntry(BigInteger.ZERO.toByteArray(), 0,
+      BigInteger.ZERO.toByteArray().length) != -1);
+    // Now go from beyond 1 to the end.
+    for(int i = 1; i < LRUDictionary.BidirectionalLRUMap.MAX_SIZE; i++) {
+      assertTrue(testee.findEntry(BigInteger.valueOf(i).toByteArray(), 0,
+          BigInteger.valueOf(i).toByteArray().length) == -1);
+    }
+
+    // check we can find all of these.
+    for (int i = 0; i < LRUDictionary.BidirectionalLRUMap.MAX_SIZE; i++) {
+      assertTrue(testee.findEntry(BigInteger.valueOf(i).toByteArray(), 0,
+          BigInteger.valueOf(i).toByteArray().length) != -1);
+    }
+  }
+
+  static private boolean isDictionaryEmpty(LRUDictionary dict) {
+    try {
+      dict.getEntry((short)0);
+      return false;
+    } catch (IndexOutOfBoundsException ioobe) {
+      return true;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index a11899c6e37e..e5946974eed8 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -61,7 +61,7 @@
 @Category(MediumTests.class)
 public class TestWALReplay {
   public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
   private Path hbaseRootDir = null;
   private Path oldLogDir;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java
new file mode 100644
index 000000000000..7e5735967204
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Enables compression and runs the TestWALReplay tests.
+ */
+@Category(MediumTests.class)
+public class TestWALReplayCompressed extends TestWALReplay {
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TestWALReplay.setUpBeforeClass();
+    Configuration conf = TestWALReplay.TEST_UTIL.getConfiguration();
+    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+  }
+
+}

From 998765257da02b8a6b814e665a55cf97a6c1afee Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 15 Mar 2012 22:11:26 +0000
Subject: [PATCH 0041/1540] HBASE-5579 A Delete Version could mask other values
 (Daniel Ferro)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301240 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanDeleteTracker.java       | 12 +++++++-----
 .../hadoop/hbase/regionserver/TestCompaction.java   |  5 ++---
 .../hbase/regionserver/TestScanDeleteTracker.java   | 13 +++++++++++++
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
index b828669b4bcc..cc26e3a5a438 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
@@ -21,7 +21,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -41,7 +40,8 @@
  */
 public class ScanDeleteTracker implements DeleteTracker {
 
-  private long familyStamp = -1L;
+  private boolean hasFamilyStamp = false;
+  private long familyStamp = 0L;
   private byte [] deleteBuffer = null;
   private int deleteOffset = 0;
   private int deleteLength = 0;
@@ -69,8 +69,9 @@ public ScanDeleteTracker() {
   @Override
   public void add(byte[] buffer, int qualifierOffset, int qualifierLength,
       long timestamp, byte type) {
-    if (timestamp > familyStamp) {
+    if (!hasFamilyStamp || timestamp > familyStamp) {
       if (type == KeyValue.Type.DeleteFamily.getCode()) {
+        hasFamilyStamp = true;
         familyStamp = timestamp;
         return;
       }
@@ -105,7 +106,7 @@ public void add(byte[] buffer, int qualifierOffset, int qualifierLength,
   @Override
   public DeleteResult isDeleted(byte [] buffer, int qualifierOffset,
       int qualifierLength, long timestamp) {
-    if (timestamp <= familyStamp) {
+    if (hasFamilyStamp && timestamp <= familyStamp) {
       return DeleteResult.FAMILY_DELETED;
     }
 
@@ -144,12 +145,13 @@ public DeleteResult isDeleted(byte [] buffer, int qualifierOffset,
 
   @Override
   public boolean isEmpty() {
-    return deleteBuffer == null && familyStamp == 0;
+    return deleteBuffer == null && !hasFamilyStamp;
   }
 
   @Override
   // called between every row.
   public void reset() {
+    hasFamilyStamp = false;
     familyStamp = 0L;
     deleteBuffer = null;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 7d0261c9c8e9..bf186e1bb20e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -377,11 +377,10 @@ public void testMinorCompactionWithDeleteVersion2() throws Exception {
     deleteVersion.deleteColumn(fam2, col2, 1);
     /*
      * the table has 4 versions: 0, 1, 2, and 3.
-     * 0 does not count.
      * We delete 1.
-     * Should have 2 remaining.
+     * Should have 3 remaining.
      */
-    testMinorCompactionWithDelete(deleteVersion, 2);
+    testMinorCompactionWithDelete(deleteVersion, 3);
   }
 
   /*
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
index 13ed753a43a4..765f12b8ff06 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
@@ -110,6 +110,19 @@ public void testDelete_KeepDelete(){
     assertEquals(false ,sdt.isEmpty());
   }
 
+  public void testDelete_KeepVersionZero(){
+    byte [] qualifier = Bytes.toBytes("qualifier");
+    deleteType = KeyValue.Type.Delete.getCode();
+
+    long deleteTimestamp = 10;
+    long valueTimestamp = 0;
+
+    sdt.reset();
+    sdt.add(qualifier, 0, qualifier.length, deleteTimestamp, deleteType);
+    DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, valueTimestamp);
+    assertEquals(DeleteResult.NOT_DELETED, ret);
+  }
+
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From b47957cd4a977397322dba787cc325d8f400a1c7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 15 Mar 2012 22:15:02 +0000
Subject: [PATCH 0042/1540] HBASE-5551  Some functions should not be used by
 customer code and must be deprecated in 0.94 (nkeywal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301243 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HBaseAdmin.java   | 3 +++
 .../java/org/apache/hadoop/hbase/client/HConnection.java  | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 73eca295cf5d..ab09eba2ede7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -202,7 +202,10 @@ public HConnection getConnection() {
    * @return proxy connection to master server for this instance
    * @throws MasterNotRunningException if the master is not running
    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
+   * @deprecated  Master is an implementation detail for HBaseAdmin.
+   * Deprecated in HBase 0.94
    */
+  @Deprecated
   public HMasterInterface getMaster()
   throws MasterNotRunningException, ZooKeeperConnectionException {
     return this.connection.getMaster();
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index c7b730b062d3..576e2d2dede8 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -69,14 +69,22 @@ public interface HConnection extends Abortable, Closeable {
    * @throws IOException if a remote or network exception occurs
    * @deprecated Removed because it was a mistake exposing zookeeper in this
    * interface (ZooKeeper is an implementation detail).
+   * Deprecated in HBase 0.94
    */
+  @Deprecated
   public ZooKeeperWatcher getZooKeeperWatcher() throws IOException;
 
   /**
    * @return proxy connection to master server for this instance
    * @throws MasterNotRunningException if the master is not running
    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
+   * @deprecated Removed because it was a mistake exposing master in this
+   * interface (master is an implementation detail). Master functions are
+   * available from HConnection or HBaseAdmin, without having to use
+   * directly the master.
+   * Deprecated in HBase 0.94
    */
+  @Deprecated
   public HMasterInterface getMaster()
   throws MasterNotRunningException, ZooKeeperConnectionException;
 

From ba29b46e43b5140f970f561e8b34af8f6aeb4f1d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 16 Mar 2012 18:06:50 +0000
Subject: [PATCH 0043/1540] HBASE-5568 Multi concurrent flushcache() for one
 region could cause data loss (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301676 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 28 +++++++++----------
 .../hadoop/hbase/regionserver/TestStore.java  |  2 +-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 01eae0eb6e7b..9dd04ff5cbc9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1304,22 +1304,22 @@ public boolean flushcache() throws IOException {
         status.setStatus("Running coprocessor pre-flush hooks");
         coprocessorHost.preFlush();
       }
-      try {
-        synchronized (writestate) {
-          if (!writestate.flushing && writestate.writesEnabled) {
-            this.writestate.flushing = true;
-          } else {
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("NOT flushing memstore for region " + this +
-                  ", flushing=" +
-                  writestate.flushing + ", writesEnabled=" +
-                  writestate.writesEnabled);
-            }
-            status.abort("Not flushing since " +
-                (writestate.flushing ? "already flushing" : "writes not enabled"));
-            return false;
+      synchronized (writestate) {
+        if (!writestate.flushing && writestate.writesEnabled) {
+          this.writestate.flushing = true;
+        } else {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("NOT flushing memstore for region " + this
+                + ", flushing=" + writestate.flushing + ", writesEnabled="
+                + writestate.writesEnabled);
           }
+          status.abort("Not flushing since "
+              + (writestate.flushing ? "already flushing"
+                  : "writes not enabled"));
+          return false;
         }
+      }
+      try {
         boolean result = internalFlushcache(status);
 
         if (coprocessorHost != null) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index bd7204c968e1..121d2778930c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -152,7 +152,7 @@ private void init(String methodName, Configuration conf,
 
   public void testDeleteExpiredStoreFiles() throws Exception {
     int storeFileNum = 4;
-    int ttl = 1;
+    int ttl = 4;
     
     Configuration conf = HBaseConfiguration.create();
     // Enable the expired store file deletion

From 3f0378402d4e9c27289af369ea4b293d56dae7a8 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 16 Mar 2012 18:47:28 +0000
Subject: [PATCH 0044/1540] HBASE-5581 Creating a table with invalid syntax
 does not give an error message when it fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301690 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/admin.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 601e7e12764f..7f0d6d7b5d7b 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -200,6 +200,7 @@ def create(table_name, *args)
             idx = idx + 1
           end
         elsif arg.kind_of?(Hash) and (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO))
+          raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME)
           raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS)
           raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO)
           raise(ArgumentError, "Number of regions must be geter than 1") unless arg[NUMREGIONS] > 1

From e53fda2103941723ec690fbe6295f310607d0b01 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 16 Mar 2012 19:45:33 +0000
Subject: [PATCH 0045/1540] HBASE-5592 Make it easier to get a table from shell

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301716 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/table.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index 6dcf47e697a8..41dcf21b2936 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -25,6 +25,7 @@
 module Hbase
   class Table
     include HBaseConstants
+    attr_reader :table
 
     def initialize(configuration, table_name, formatter)
       @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name)

From b88c3470d3fcbb3c5d7dfb6e58088458c99da005 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 16 Mar 2012 20:17:42 +0000
Subject: [PATCH 0046/1540] HBASE-5206 port HBASE-5155 to 0.94 (Ashutosh
 Jindal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301737 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 60 +++++++++++++++++--
 .../apache/hadoop/hbase/master/HMaster.java   | 10 ++++
 .../master/handler/DeleteTableHandler.java    |  4 +-
 .../master/handler/ServerShutdownHandler.java | 16 ++++-
 .../hadoop/hbase/zookeeper/ZKTable.java       | 41 +++++++++----
 .../hadoop/hbase/TestDrainingServer.java      |  3 +
 .../apache/hadoop/hbase/client/TestAdmin.java | 17 +++++-
 .../hadoop/hbase/master/TestMaster.java       |  2 +
 .../hbase/master/TestMasterFailover.java      |  5 ++
 .../TestMasterRestartAfterDisablingTable.java |  6 ++
 .../hadoop/hbase/zookeeper/TestZKTable.java   | 15 ++++-
 11 files changed, 160 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 764a0d299975..2eb678fdb567 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1585,6 +1585,7 @@ private void assign(final HRegionInfo region, final RegionState state,
           if (isDisabledorDisablingRegionInRIT(region)) {
             return;
           }
+          setEnabledTable(region);
         }
       }
       
@@ -2199,6 +2200,14 @@ public void assignUserRegions(List<HRegionInfo> regions, List<ServerName> server
     LOG.info("Bulk assigning done");
   }
 
+  private void setEnabledTable(HRegionInfo hri) {
+    String tableName = hri.getTableNameAsString();
+    boolean isTableEnabled = this.zkTable.isEnabledTable(tableName);
+    if (!isTableEnabled) {
+      setEnabledTable(tableName);
+    }    
+  }
+
   /**
    * Assigns all user regions, if any exist.  Used during cluster startup.
    * <p>
@@ -2234,6 +2243,9 @@ public void assignAllUserRegions() throws IOException, InterruptedException {
     } else {
       // assign regions in round-robin fashion
       assignUserRegions(new ArrayList<HRegionInfo>(allRegions.keySet()), servers);
+      for (HRegionInfo hri : allRegions.keySet()) {
+        setEnabledTable(hri);
+      }
       return;
     }
     LOG.info("Bulk assigning " + allRegions.size() + " region(s) across " +
@@ -2242,6 +2254,9 @@ public void assignAllUserRegions() throws IOException, InterruptedException {
     // Use fixed count thread pool assigning.
     BulkAssigner ba = new StartupBulkAssigner(this.master, bulkPlan, this);
     ba.bulkAssign();
+    for (HRegionInfo hri : allRegions.keySet()) {
+      setEnabledTable(hri);
+    }
     LOG.info("Bulk assigning done");
   }
 
@@ -2435,6 +2450,8 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
       new TreeMap<ServerName, List<Pair<HRegionInfo, Result>>>();
     // Iterate regions in META
     for (Result result : results) {
+      boolean disabled = false;
+      boolean disablingOrEnabling = false;
       Pair<HRegionInfo, ServerName> region = MetaReader.parseCatalogResult(result);
       if (region == null) continue;
       HRegionInfo regionInfo = region.getFirst();
@@ -2468,6 +2485,13 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
           offlineServers.put(regionLocation, offlineRegions);
         }
         offlineRegions.add(new Pair<HRegionInfo,Result>(regionInfo, result));
+        disabled = checkIfRegionBelongsToDisabled(regionInfo);
+        disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
+            this.enablingTables, regionInfo, tableName);
+        // need to enable the table if not disabled or disabling or enabling
+        // this will be used in rolling restarts
+        enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
+            disablingOrEnabling, tableName);
       } else {
         // Region is being served and on an active server
         // add only if region not in disabled and enabling table
@@ -2476,21 +2500,37 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
           regions.put(regionInfo, regionLocation);
           addToServers(regionLocation, regionInfo);
         }
-        addTheTablesInPartialState(this.disablingTables, this.enablingTables, regionInfo,
-            tableName);
+        disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
+            this.enablingTables, regionInfo, tableName);
+        disabled = checkIfRegionBelongsToDisabled(regionInfo);
+        // need to enable the table if not disabled or disabling or enabling
+        // this will be used in rolling restarts
+        enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
+            disablingOrEnabling, tableName);
       }
     }
     return offlineServers;
   }
 
-  private void addTheTablesInPartialState(Set<String> disablingTables,
+  private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
+      boolean disablingOrEnabling, String tableName) {
+    if (!disabled && !disablingOrEnabling
+        && !getZKTable().isEnabledTable(tableName)) {
+      setEnabledTable(tableName);
+    }
+  }
+
+  private Boolean addTheTablesInPartialState(Set<String> disablingTables,
       Set<String> enablingTables, HRegionInfo regionInfo,
       String disablingTableName) {
     if (checkIfRegionBelongsToDisabling(regionInfo)) {
       disablingTables.add(disablingTableName);
+      return true;
     } else if (checkIfRegionsBelongsToEnabling(regionInfo)) {
       enablingTables.add(disablingTableName);
-    }
+      return true;
+    } 
+    return false;
   }
 
   /**
@@ -3307,4 +3347,16 @@ public void shutdown() {
       this.threadPoolExecutorService.shutdown();
     }
   }
+
+  protected void setEnabledTable(String tableName) {
+    try {
+      this.zkTable.setEnabledTable(tableName);
+    } catch (KeeperException e) {
+      // here we can abort as it is the start up flow
+      String errorMsg = "Unable to ensure that the table " + tableName
+          + " will be" + " enabled because of a ZooKeeper issue";
+      LOG.error(errorMsg);
+      this.master.abort(errorMsg, e);
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 785c6a7aff05..c59322386447 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -624,6 +624,9 @@ int assignRootAndMeta(MonitoredTask status)
       this.assignmentManager.regionOnline(HRegionInfo.ROOT_REGIONINFO,
         this.catalogTracker.getRootLocation());
     }
+    // Enable the ROOT table if on process fail over the RS containing ROOT
+    // was active.
+    enableCatalogTables(Bytes.toString(HConstants.ROOT_TABLE_NAME));
     LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getRootLocation());
 
@@ -649,12 +652,19 @@ int assignRootAndMeta(MonitoredTask status)
       this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
         this.catalogTracker.getMetaLocation());
     }
+    enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
     LOG.info(".META. assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getMetaLocation());
     status.setStatus("META and ROOT assigned.");
     return assigned;
   }
 
+  private void enableCatalogTables(String catalogTableName) {
+    if (!this.assignmentManager.getZKTable().isEnabledTable(catalogTableName)) {
+      this.assignmentManager.setEnabledTable(catalogTableName);
+    }
+  }
+
   void fixupDaughters(final MonitoredTask status) throws IOException {
     final Map<HRegionInfo, Result> offlineSplitParents =
       new HashMap<HRegionInfo, Result>();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index 17dd2f9c79ec..a79ba088eed6 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -79,8 +79,8 @@ protected void handleTableOperation(List<HRegionInfo> regions)
     this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
 
     // If entry for this table in zk, and up in AssignmentManager, remove it.
-    // Call to undisableTable does this. TODO: Make a more formal purge table.
-    am.getZKTable().setEnabledTable(Bytes.toString(tableName));
+
+    am.getZKTable().setDeletedTable(Bytes.toString(tableName));
   }
   
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 4307d89a4047..4684cc13a677 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -325,13 +325,27 @@ public void process() throws IOException {
   public static boolean processDeadRegion(HRegionInfo hri, Result result,
       AssignmentManager assignmentManager, CatalogTracker catalogTracker)
   throws IOException {
+    boolean tablePresent = assignmentManager.getZKTable().isTablePresent(
+        hri.getTableNameAsString());
+    if (!tablePresent) {
+      LOG.info("The table " + hri.getTableNameAsString()
+          + " was deleted.  Hence not proceeding.");
+      return false;
+    }
     // If table is not disabled but the region is offlined,
     boolean disabled = assignmentManager.getZKTable().isDisabledTable(
         hri.getTableNameAsString());
-    if (disabled) return false;
+    if (disabled){
+      LOG.info("The table " + hri.getTableNameAsString()
+          + " was disabled.  Hence not proceeding.");
+      return false;
+    }
     if (hri.isOffline() && hri.isSplit()) {
       LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
         "; checking daughter presence");
+      if (MetaReader.getRegion(catalogTracker, hri.getRegionName()) == null) {
+        return false;
+      }
       fixupDaughters(result, assignmentManager, catalogTracker);
       return false;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 4f8aea2c328c..b925bb9086c9 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -216,7 +216,7 @@ public boolean checkDisabledAndSetEnablingTable(final String tableName)
   public boolean checkEnabledAndSetDisablingTable(final String tableName)
     throws KeeperException {
     synchronized (this.cache) {
-      if (!isEnabledTable(tableName)) {
+      if (this.cache.get(tableName) != null && !isEnabledTable(tableName)) {
         return false;
       }
       setTableState(tableName, TableState.DISABLING);
@@ -265,10 +265,7 @@ public boolean isEnablingTable(final String tableName) {
   }
 
   public boolean isEnabledTable(String tableName) {
-    synchronized (this.cache) {
-      // No entry in cache means enabled table.
-      return !this.cache.containsKey(tableName);
-    }
+    return isTableState(tableName, TableState.ENABLED);
   }
 
   /**
@@ -283,7 +280,7 @@ public boolean isEnabledTable(String tableName) {
   public static boolean isEnabledTable(final ZooKeeperWatcher zkw,
       final String tableName)
   throws KeeperException {
-    return getTableState(zkw, tableName) == null;
+    return getTableState(zkw, tableName) == TableState.ENABLED;
   }
 
   public boolean isDisablingOrDisabledTable(final String tableName) {
@@ -335,23 +332,47 @@ private static boolean isTableState(final TableState expectedState,
   }
 
   /**
-   * Enables the table in zookeeper.  Fails silently if the
+   * Deletes the table in zookeeper.  Fails silently if the
    * table is not currently disabled in zookeeper.  Sets no watches.
    * @param tableName
    * @throws KeeperException unexpected zookeeper exception
    */
-  public void setEnabledTable(final String tableName)
+  public void setDeletedTable(final String tableName)
   throws KeeperException {
     synchronized (this.cache) {
       if (this.cache.remove(tableName) == null) {
-        LOG.warn("Moving table " + tableName + " state to enabled but was " +
-          "already enabled");
+        LOG.warn("Moving table " + tableName + " state to deleted but was " +
+          "already deleted");
       }
       ZKUtil.deleteNodeFailSilent(this.watcher,
         ZKUtil.joinZNode(this.watcher.tableZNode, tableName));
     }
   }
+  
+  /**
+   * Sets the ENABLED state in the cache and deletes the zookeeper node. Fails
+   * silently if the node is not in enabled in zookeeper
+   * 
+   * @param tableName
+   * @throws KeeperException
+   */
+  public void setEnabledTable(final String tableName) throws KeeperException {
+    setTableState(tableName, TableState.ENABLED);
+  }
 
+  /**
+   * check if table is present .
+   * 
+   * @param tableName
+   * @return true if the table is present
+   */
+  public boolean isTablePresent(final String tableName) {
+    synchronized (this.cache) {
+      TableState state = this.cache.get(tableName);
+      return !(state == null);
+    }
+  }
+  
   /**
    * Gets a list of all the tables set as disabled in zookeeper.
    * @return Set of disabled tables, empty Set if none
diff --git a/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java b/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
index d1b8a1b353b0..a1992c3d5347 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -61,6 +62,7 @@ public class TestDrainingServer {
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(5);
     TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
     HTableDescriptor htd = new HTableDescriptor(TABLENAME);
     htd.addFamily(new HColumnDescriptor(FAMILY));
     TEST_UTIL.createMultiRegionsInMeta(TEST_UTIL.getConfiguration(), htd,
@@ -73,6 +75,7 @@ public static void setUpBeforeClass() throws Exception {
     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
     admin.disableTable(TABLENAME);
     admin.enableTable(TABLENAME);
+    ZKAssign.blockUntilNoRIT(zkw);
     // Assert that every regionserver has some regions on it.
     MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
     for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 327950c118a4..20094003f3e9 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -49,6 +49,8 @@
 import org.apache.hadoop.hbase.regionserver.wal.HLogUtilsForTests;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.*;
 import org.junit.experimental.categories.Category;
 
@@ -222,6 +224,9 @@ public void testDisableAndEnableTable() throws IOException {
     ht.get(get);
 
     this.admin.disableTable(table);
+    assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster()
+        .getMaster().getAssignmentManager().getZKTable().isDisabledTable(
+            Bytes.toString(table)));
 
     // Test that table is disabled
     get = new Get(row);
@@ -236,6 +241,9 @@ public void testDisableAndEnableTable() throws IOException {
     }
     assertTrue(ok);
     this.admin.enableTable(table);
+    assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
+        .getMaster().getAssignmentManager().getZKTable().isEnabledTable(
+            Bytes.toString(table)));
 
     // Test that table is enabled
     try {
@@ -307,6 +315,9 @@ public void testCreateTable() throws IOException {
       HConstants.CATALOG_FAMILY).close();
     tables = this.admin.listTables();
     assertEquals(numTables + 1, tables.length);
+    assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
+        .getMaster().getAssignmentManager().getZKTable().isEnabledTable(
+            "testCreateTable"));
   }
 
   @Test
@@ -952,10 +963,14 @@ public void testInvalidHColumnDescriptor() throws IOException {
      new HColumnDescriptor("/cfamily/name");
   }
 
-  @Test
+  @Test(timeout=36000)
   public void testEnableDisableAddColumnDeleteColumn() throws Exception {
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
     byte [] tableName = Bytes.toBytes("testMasterAdmin");
     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
+    while (!ZKTable.isEnabledTable(zkw, "testMasterAdmin")) {
+      Thread.sleep(10);
+    }
     this.admin.disableTable(tableName);
     try {
       new HTable(TEST_UTIL.getConfiguration(), tableName);
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index b7a8270cf3b5..3122b1592247 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -69,6 +69,8 @@ public void testMasterOpsWhileSplitting() throws Exception {
     HMaster m = cluster.getMaster();
 
     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME);
+    assertTrue(m.assignmentManager.getZKTable().isEnabledTable
+        (Bytes.toString(TABLENAME)));
     TEST_UTIL.loadTable(ht, FAMILYNAME);
     ht.close();
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 63882e199fe4..d3ea94b5b198 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -743,6 +743,8 @@ public boolean isAborted() {
       master.assignRegion(hri);
     }
 
+    assertTrue(" Table must be enabled.", master.getAssignmentManager()
+        .getZKTable().isEnabledTable("enabledTable"));
     // we also need regions assigned out on the dead server
     List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
     enabledAndOnDeadRegions.add(enabledRegions.remove(0));
@@ -788,6 +790,9 @@ public boolean isAborted() {
     ZKTable zktable = new ZKTable(zkw);
     zktable.setDisabledTable(Bytes.toString(disabledTable));
 
+    assertTrue(" The enabled table should be identified on master fail over.",
+        zktable.isEnabledTable("enabledTable"));
+
     /*
      * ZK = CLOSING
      */
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
index 777c67d9bd40..ec08b1740c1d 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.List;
@@ -96,6 +97,9 @@ public void testForCheckingIfEnableAndDisableWorksFineAfterSwitch()
     cluster.hbaseCluster.waitOnMaster(activeMaster);
     cluster.waitForActiveAndReadyMaster();
 
+    assertTrue("The table should not be in enabled state", cluster.getMaster()
+        .getAssignmentManager().getZKTable().isDisablingOrDisabledTable(
+            "tableRestart"));
     log("Enabling table\n");
     // Need a new Admin, the previous one is on the old master
     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
@@ -108,6 +112,8 @@ public void testForCheckingIfEnableAndDisableWorksFineAfterSwitch()
     assertEquals(
         "The assigned regions were not onlined after master switch except for the catalog tables.",
         6, regions.size());
+    assertTrue("The table should be in enabled state", cluster.getMaster()
+        .getAssignmentManager().getZKTable().isEnabledTable("tableRestart"));
     ht.close();
     TEST_UTIL.shutdownMiniCluster();
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
index 3de482bfc778..0a78f4039f32 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
@@ -68,29 +68,42 @@ public boolean isAborted() {
     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
       name, abortable, true);
     ZKTable zkt = new ZKTable(zkw);
-    assertTrue(zkt.isEnabledTable(name));
+    assertFalse(zkt.isEnabledTable(name));
     assertFalse(zkt.isDisablingTable(name));
     assertFalse(zkt.isDisabledTable(name));
     assertFalse(zkt.isEnablingTable(name));
     assertFalse(zkt.isDisablingOrDisabledTable(name));
     assertFalse(zkt.isDisabledOrEnablingTable(name));
+    assertFalse(zkt.isTablePresent(name));
     zkt.setDisablingTable(name);
     assertTrue(zkt.isDisablingTable(name));
     assertTrue(zkt.isDisablingOrDisabledTable(name));
     assertFalse(zkt.getDisabledTables().contains(name));
+    assertTrue(zkt.isTablePresent(name));
     zkt.setDisabledTable(name);
     assertTrue(zkt.isDisabledTable(name));
     assertTrue(zkt.isDisablingOrDisabledTable(name));
     assertFalse(zkt.isDisablingTable(name));
     assertTrue(zkt.getDisabledTables().contains(name));
+    assertTrue(zkt.isTablePresent(name));
     zkt.setEnablingTable(name);
     assertTrue(zkt.isEnablingTable(name));
     assertTrue(zkt.isDisabledOrEnablingTable(name));
     assertFalse(zkt.isDisabledTable(name));
     assertFalse(zkt.getDisabledTables().contains(name));
+    assertTrue(zkt.isTablePresent(name));
     zkt.setEnabledTable(name);
     assertTrue(zkt.isEnabledTable(name));
     assertFalse(zkt.isEnablingTable(name));
+    assertTrue(zkt.isTablePresent(name));
+    zkt.setDeletedTable(name);
+    assertFalse(zkt.isEnabledTable(name));
+    assertFalse(zkt.isDisablingTable(name));
+    assertFalse(zkt.isDisabledTable(name));
+    assertFalse(zkt.isEnablingTable(name));
+    assertFalse(zkt.isDisablingOrDisabledTable(name));
+    assertFalse(zkt.isDisabledOrEnablingTable(name));
+    assertFalse(zkt.isTablePresent(name));
   }
 
   @org.junit.Rule

From 2b5de0611560073505fb9999996f7426ea010f17 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 16 Mar 2012 21:59:22 +0000
Subject: [PATCH 0047/1540] HBASE-5563 Add comparison of regionId to
 HRegionInfo#compareTo (chunhui and jmhsieh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301783 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HRegionInfo.java    |  9 +++++++++
 .../hadoop/hbase/master/AssignmentManager.java       |  4 +++-
 .../hadoop/hbase/regionserver/TestHRegionInfo.java   | 12 ++++++++++++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 74cb8217b351..97ad5fdb3709 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -787,6 +787,15 @@ public int compareTo(HRegionInfo o) {
       }
       return result;
     }
+
+    // regionId is usually milli timestamp -- this defines older stamps
+    // to be "smaller" than newer stamps in sort order.
+    if (this.regionId > o.regionId) {
+      return 1;
+    } else if (this.regionId < o.regionId) {
+      return -1;
+    }
+
     if (this.offLine == o.offLine)
       return 0;
     if (this.offLine == true) return -1;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 2eb678fdb567..6f6523d56ccd 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2778,8 +2778,10 @@ public void waitOnRegionToClearRegionsInTransition(final HRegionInfo hri)
    */
   public List<HRegionInfo> getRegionsOfTable(byte[] tableName) {
     List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
+    // boundary needs to have table's name but regionID 0 so that it is sorted 
+    // before all table's regions.
     HRegionInfo boundary =
-      new HRegionInfo(tableName, null, null);
+      new HRegionInfo(tableName, null, null, false, 0L);
     synchronized (this.regions) {
       for (HRegionInfo regionInfo: this.regions.tailMap(boundary).keySet()) {
         if(Bytes.equals(regionInfo.getTableName(), tableName)) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
index 6e1211bd7d40..6dfba41ca292 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
@@ -130,6 +130,18 @@ public void testMetaTables() {
     assertTrue(HRegionInfo.FIRST_META_REGIONINFO.isMetaTable());
   }
 
+  @Test
+  public void testComparator() {
+    byte[] tablename = Bytes.toBytes("comparatorTablename");
+    byte[] empty = new byte[0];
+    HRegionInfo older = new HRegionInfo(tablename, empty, empty, false, 0L); 
+    HRegionInfo newer = new HRegionInfo(tablename, empty, empty, false, 1L); 
+    assertTrue(older.compareTo(newer) < 0);
+    assertTrue(newer.compareTo(older) > 0);
+    assertTrue(older.compareTo(older) == 0);
+    assertTrue(newer.compareTo(newer) == 0);
+  }
+  
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From f11ccbae0df4e61c746a279720888e0ac05cb62c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 17 Mar 2012 07:00:10 +0000
Subject: [PATCH 0048/1540] Revert HBASE-5569

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301873 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanQueryMatcher.java   |  3 +--
 .../regionserver/TestAtomicOperation.java      | 18 +++++++++++-------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 221f44694cc9..9f0cba9eb9c7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -306,8 +306,7 @@ public MatchCode match(KeyValue kv) throws IOException {
       }
       // note the following next else if...
       // delete marker are not subject to other delete markers
-    } else if (!this.deletes.isEmpty()
-        && kv.getMemstoreTS() <= maxReadPointToTrackVersions) {
+    } else if (!this.deletes.isEmpty()) {
       DeleteResult deleteResult = deletes.isDeleted(bytes, offset, qualLength,
           timestamp);
       switch (deleteResult) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index c6568703b403..1640f296b0fc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
 import org.junit.experimental.categories.Category;
 
@@ -54,7 +55,10 @@ public class TestAtomicOperation extends HBaseTestCase {
 
   HRegion region = null;
   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString();
+  private final String DIR = TEST_UTIL.getDataTestDir("TestIncrement").toString();
+
+
+  private final int MAX_VERSIONS = 2;
 
   // Test names
   static final byte[] tableName = Bytes.toBytes("testtable");;
@@ -254,10 +258,10 @@ public void testRowMutationMultiThreads() throws IOException {
     LOG.info("Starting test testRowMutationMultiThreads");
     initHRegion(tableName, getName(), fam1);
 
-    // create 50 threads, each will alternate between adding and
+    // create 100 threads, each will alternate between adding and
     // removing a column
-    int numThreads = 50;
-    int opsPerThread = 500;
+    int numThreads = 100;
+    int opsPerThread = 1000;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
     AtomicLong timeStamps = new AtomicLong(0);
@@ -336,10 +340,10 @@ public void testMultiRowMutationMultiThreads() throws IOException {
     LOG.info("Starting test testMultiRowMutationMultiThreads");
     initHRegion(tableName, getName(), fam1);
 
-    // create 50 threads, each will alternate between adding and
+    // create 100 threads, each will alternate between adding and
     // removing a column
-    int numThreads = 50;
-    int opsPerThread = 500;
+    int numThreads = 100;
+    int opsPerThread = 1000;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
     AtomicLong timeStamps = new AtomicLong(0);

From 98d60dd747e8c4b1cfae9eac395904ce566fc023 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Sat, 17 Mar 2012 07:33:54 +0000
Subject: [PATCH 0049/1540] HBASE-5588 Deprecate/remove
 AssignmentManager#clearRegionFromTransition

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1301884 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/AssignmentManager.java    | 9 ++++++---
 .../java/org/apache/hadoop/hbase/master/HMaster.java     | 4 ++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 6f6523d56ccd..85b56963567e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -147,7 +147,7 @@ public class AssignmentManager extends ZooKeeperListener {
    * Server to regions assignment map.
    * Contains the set of regions currently assigned to a given server.
    * This Map and {@link #regions} are tied.  Always update this in tandem
-   * with the other under a lock on {@link #regions}
+   * with the other under a lock on {@link #regions}.
    * @see #regions
    */
   private final NavigableMap<ServerName, Set<HRegionInfo>> servers =
@@ -157,7 +157,7 @@ public class AssignmentManager extends ZooKeeperListener {
    * Region to server assignment map.
    * Contains the server a given region is currently assigned to.
    * This Map and {@link #servers} are tied.  Always update this in tandem
-   * with the other under a lock on {@link #regions}
+   * with the other under a lock on {@link #regions}.
    * @see #servers
    */
   private final SortedMap<HRegionInfo, ServerName> regions =
@@ -1117,7 +1117,7 @@ public void nodeDeleted(final String path) {
         if (rs.isSplitting() || rs.isSplit()) {
           LOG.debug("Ephemeral node deleted, regionserver crashed?, " +
             "clearing from RIT; rs=" + rs);
-          clearRegionFromTransition(rs.getRegion());
+          regionOffline(rs.getRegion());
         } else {
           LOG.debug("The znode of region " + regionInfo.getRegionNameAsString()
               + " has been deleted.");
@@ -2720,7 +2720,10 @@ public RegionState isRegionInTransition(final HRegionInfo hri) {
 
   /**
    * Clears the specified region from being in transition.
+   * <p>
    * @param hri Region to remove.
+   * @deprecated This is a dupe of {@link #regionOffline(HRegionInfo)}.
+   *   Please use that method instead.
    */
   public void clearRegionFromTransition(HRegionInfo hri) {
     synchronized (this.regionsInTransition) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c59322386447..6f1e714eef6b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1418,7 +1418,7 @@ private void preCheckTableModifiable(final byte[] tableName)
 
   public void clearFromTransition(HRegionInfo hri) {
     if (this.assignmentManager.isRegionInTransition(hri) != null) {
-      this.assignmentManager.clearRegionFromTransition(hri);
+      this.assignmentManager.regionOffline(hri);
     }
   }
 
@@ -1766,7 +1766,7 @@ public void unassign(final byte [] regionName, final boolean force)
       }
     }
     if (force) {
-      this.assignmentManager.clearRegionFromTransition(hri);
+      this.assignmentManager.regionOffline(hri);
       assignRegion(hri);
     } else {
       this.assignmentManager.unassign(hri, force);

From 7471a06787e861901b0688070e1507d7c0cf7219 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sun, 18 Mar 2012 19:40:59 +0000
Subject: [PATCH 0050/1540] HBASE-5955 Fix NoSuchMethodException in 0.92 when
 running on local filesystem

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1302208 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLog.java   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 6720a5bde75c..672214997e0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -452,26 +452,26 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
    */
   private Method getGetNumCurrentReplicas(final FSDataOutputStream os) {
     Method m = null;
-    Exception exception = null;
     if (os != null) {
+      Class<? extends OutputStream> wrappedStreamClass = os.getWrappedStream()
+          .getClass();
       try {
-        m = os.getWrappedStream().getClass().
-          getDeclaredMethod("getNumCurrentReplicas", new Class<?> []{});
+        m = wrappedStreamClass.getDeclaredMethod("getNumCurrentReplicas",
+            new Class<?>[] {});
         m.setAccessible(true);
       } catch (NoSuchMethodException e) {
-        // Thrown if getNumCurrentReplicas() function isn't available
-        exception = e;
+        LOG.info("FileSystem's output stream doesn't support"
+            + " getNumCurrentReplicas; --HDFS-826 not available; fsOut="
+            + wrappedStreamClass.getName());
       } catch (SecurityException e) {
-        // Thrown if we can't get access to getNumCurrentReplicas()
-        exception = e;
+        LOG.info("Doesn't have access to getNumCurrentReplicas on "
+            + "FileSystems's output stream --HDFS-826 not available; fsOut="
+            + wrappedStreamClass.getName(), e);
         m = null; // could happen on setAccessible()
       }
     }
     if (m != null) {
       LOG.info("Using getNumCurrentReplicas--HDFS-826");
-    } else {
-      LOG.info("getNumCurrentReplicas--HDFS-826 not available; hdfs_out=" +
-        os, exception);
     }
     return m;
   }

From e2fe2f9dabab75688c0c7f889346ec5b4f08cb23 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 20 Mar 2012 17:11:52 +0000
Subject: [PATCH 0051/1540] HBASE-5520 Support reseek() at RegionScanner (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303006 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  17 +++
 .../hbase/regionserver/RegionScanner.java     |  13 +++
 .../coprocessor/TestCoprocessorInterface.java |   5 +
 .../hadoop/hbase/filter/TestFilter.java       | 108 +++++++++++++++++-
 4 files changed, 139 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9dd04ff5cbc9..dc7f246bd5b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3473,6 +3473,23 @@ public synchronized void close() {
     KeyValueHeap getStoreHeapForTesting() {
       return storeHeap;
     }
+
+    @Override
+    public synchronized boolean reseek(byte[] row) throws IOException {
+      if (row == null) {
+        throw new IllegalArgumentException("Row cannot be null.");
+      }
+      startRegionOperation();
+      try {
+        // This could be a new thread from the last time we called next().
+        MultiVersionConsistencyControl.setThreadReadPoint(this.readPt);
+        KeyValue kv = KeyValue.createFirstOnRow(row);
+        // use request seek to make use of the lazy seek option. See HBASE-5520
+        return this.storeHeap.requestSeek(kv, true, true);
+      } finally {
+        closeRegionOperation();
+      }
+    }
   }
 
   // Utility methods
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
index b10aecafa7e6..533775c7697c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -35,4 +35,17 @@ public interface RegionScanner extends InternalScanner {
    *         further rows.
    */
   public boolean isFilterDone();
+
+  /**
+   * Do a reseek to the required row. Should not be used to seek to a key which
+   * may come before the current position. Always seeks to the beginning of a
+   * row boundary.
+   *
+   * @throws IOException
+   * @throws IllegalArgumentException
+   *           if row is null
+   *
+   */
+  public boolean reseek(byte[] row) throws IOException;
+
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 545d107cbb02..99b2dd94661e 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -84,6 +84,11 @@ public boolean isFilterDone() {
       return delegate.isFilterDone();
     }
 
+    @Override
+    public boolean reseek(byte[] row) throws IOException {
+      return false;
+    }
+
   }
 
   public static class CoprocessorImpl extends BaseRegionObserver {
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index 0e19d4d1b4aa..396bba382187 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -24,7 +24,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import junit.framework.Assert;
 import org.apache.commons.logging.Log;
@@ -37,6 +36,7 @@
 import org.apache.hadoop.hbase.filter.FilterList.Operator;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
@@ -63,10 +63,24 @@ public class TestFilter extends HBaseTestCase {
       Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3")
   };
 
+  private static final byte [][] ROWS_THREE = {
+    Bytes.toBytes("testRowThree-0"), Bytes.toBytes("testRowThree-1"),
+    Bytes.toBytes("testRowThree-2"), Bytes.toBytes("testRowThree-3")
+  };
+
+  private static final byte [][] ROWS_FOUR = {
+    Bytes.toBytes("testRowFour-0"), Bytes.toBytes("testRowFour-1"),
+    Bytes.toBytes("testRowFour-2"), Bytes.toBytes("testRowFour-3")
+  };
+
   private static final byte [][] FAMILIES = {
     Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo")
   };
 
+  private static final byte [][] FAMILIES_1 = {
+    Bytes.toBytes("testFamilyThree"), Bytes.toBytes("testFamilyFour")
+  };
+
   private static final byte [][] QUALIFIERS_ONE = {
     Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"),
     Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3")
@@ -77,10 +91,24 @@ public class TestFilter extends HBaseTestCase {
     Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3")
   };
 
+  private static final byte [][] QUALIFIERS_THREE = {
+    Bytes.toBytes("testQualifierThree-0"), Bytes.toBytes("testQualifierThree-1"),
+    Bytes.toBytes("testQualifierThree-2"), Bytes.toBytes("testQualifierThree-3")
+  };
+
+  private static final byte [][] QUALIFIERS_FOUR = {
+    Bytes.toBytes("testQualifierFour-0"), Bytes.toBytes("testQualifierFour-1"),
+    Bytes.toBytes("testQualifierFour-2"), Bytes.toBytes("testQualifierFour-3")
+  };
+
   private static final byte [][] VALUES = {
     Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo")
   };
 
+  byte [][] NEW_FAMILIES = {
+      Bytes.toBytes("f1"), Bytes.toBytes("f2")
+    };
+
   private long numRows = ROWS_ONE.length + ROWS_TWO.length;
   private long colsPerRow = FAMILIES.length * QUALIFIERS_ONE.length;
 
@@ -90,6 +118,11 @@ protected void setUp() throws Exception {
     HTableDescriptor htd = new HTableDescriptor(getName());
     htd.addFamily(new HColumnDescriptor(FAMILIES[0]));
     htd.addFamily(new HColumnDescriptor(FAMILIES[1]));
+    htd.addFamily(new HColumnDescriptor(FAMILIES_1[0]));
+    htd.addFamily(new HColumnDescriptor(FAMILIES_1[1]));
+    htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[0]));
+    htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[1]));
+    htd.addFamily(new HColumnDescriptor(FAMILIES_1[1]));
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     this.region = HRegion.createHRegion(info, this.testDir, this.conf, htd);
 
@@ -170,6 +203,73 @@ protected void tearDown() throws Exception {
     super.tearDown();
   }
 
+
+  public void testRegionScannerReseek() throws Exception {
+    // create new rows and column family to show how reseek works..
+    for (byte[] ROW : ROWS_THREE) {
+      Put p = new Put(ROW);
+      p.setWriteToWAL(true);
+      for (byte[] QUALIFIER : QUALIFIERS_THREE) {
+        p.add(FAMILIES[0], QUALIFIER, VALUES[0]);
+
+      }
+      this.region.put(p);
+    }
+    for (byte[] ROW : ROWS_FOUR) {
+      Put p = new Put(ROW);
+      p.setWriteToWAL(false);
+      for (byte[] QUALIFIER : QUALIFIERS_FOUR) {
+        p.add(FAMILIES[1], QUALIFIER, VALUES[1]);
+      }
+      this.region.put(p);
+    }
+    // Flush
+    this.region.flushcache();
+
+    // Insert second half (reverse families)
+    for (byte[] ROW : ROWS_THREE) {
+      Put p = new Put(ROW);
+      p.setWriteToWAL(false);
+      for (byte[] QUALIFIER : QUALIFIERS_THREE) {
+        p.add(FAMILIES[1], QUALIFIER, VALUES[0]);
+      }
+      this.region.put(p);
+    }
+    for (byte[] ROW : ROWS_FOUR) {
+      Put p = new Put(ROW);
+      p.setWriteToWAL(false);
+      for (byte[] QUALIFIER : QUALIFIERS_FOUR) {
+        p.add(FAMILIES[0], QUALIFIER, VALUES[1]);
+      }
+      this.region.put(p);
+    }
+
+    Scan s = new Scan();
+    // set a start row
+    s.setStartRow(ROWS_FOUR[1]);
+    RegionScanner scanner = region.getScanner(s);
+
+    // reseek to row three.
+    scanner.reseek(ROWS_THREE[1]);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+
+    // the results should belong to ROWS_THREE[1]
+    scanner.next(results);
+    for (KeyValue keyValue : results) {
+      assertEquals("The rows with ROWS_TWO as row key should be appearing.",
+          Bytes.toString(keyValue.getRow()), Bytes.toString(ROWS_THREE[1]));
+    }
+    // again try to reseek to a value before ROWS_THREE[1]
+    scanner.reseek(ROWS_ONE[1]);
+    results = new ArrayList<KeyValue>();
+    // This time no seek would have been done to ROWS_ONE[1]
+    scanner.next(results);
+    for (KeyValue keyValue : results) {
+      assertFalse("Cannot rewind back to a value less than previous reseek.",
+          Bytes.toString(keyValue.getRow()).contains("testRowOne"));
+    }
+  }
+
   public void testNoFilter() throws Exception {
     // No filter
     long expectedRows = this.numRows;
@@ -608,7 +708,7 @@ public void testFamilyFilter() throws IOException {
       verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
 
       // Match all columns in second family
-      // look only in second group of rows        
+      // look only in second group of rows
       expectedRows = this.numRows / 2;
       expectedKeys = this.colsPerRow / 2;
       f = new FamilyFilter(CompareOp.GREATER,
@@ -1345,7 +1445,7 @@ private void verifyScanFullNoValues(Scan s, KeyValue [] kvs, boolean useLen)
         assertFalse("Should not have returned whole value",
             Bytes.equals(kv.getValue(), kvs[idx].getValue()));
         if (useLen) {
-          assertEquals("Value in result is not SIZEOF_INT", 
+          assertEquals("Value in result is not SIZEOF_INT",
                      kv.getValue().length, Bytes.SIZEOF_INT);
           LOG.info("idx = "  + idx + ", len=" + kvs[idx].getValueLength()
               + ", actual=" +  Bytes.toInt(kv.getValue()));
@@ -1353,7 +1453,7 @@ private void verifyScanFullNoValues(Scan s, KeyValue [] kvs, boolean useLen)
                      kvs[idx].getValueLength(), Bytes.toInt(kv.getValue()) );
           LOG.info("good");
         } else {
-          assertEquals("Value in result is not empty", 
+          assertEquals("Value in result is not empty",
                      kv.getValue().length, 0);
         }
         idx++;

From bb7e0fb2084fa5c1bf51903bc7752364ad5a98d7 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 20 Mar 2012 17:22:50 +0000
Subject: [PATCH 0052/1540] HBASE-5520 - 0.94 patch did not apply cleanly . 
 Updated it

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303019 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/RegionScanner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
index 533775c7697c..da95e901a8c9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.IOException;
 import org.apache.hadoop.hbase.HRegionInfo;
 
 /**

From 1949544cbf141c914abf3285a7426f83d86c942e Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Tue, 20 Mar 2012 22:48:21 +0000
Subject: [PATCH 0053/1540] HBASE-5603 rolling-restart.sh script hangs when
 attempting to detect expiration of /hbase/master znode

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303186 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/rolling-restart.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 8c3cc2bf5731..ec4edb5b1fa2 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -97,7 +97,7 @@ else
     if [ "$zmaster" == "null" ]; then zmaster="master"; fi
     zmaster=$zparent/$zmaster
     echo -n "Waiting for Master ZNode ${zmaster} to expire"
-    while bin/hbase zkcli stat $zmaster >/dev/null 2>&1; do
+    while ! bin/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do
       echo -n "."
       sleep 1
     done

From 3c58fdacf1e6c8fe7d602a515289b1be1926cc26 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 20 Mar 2012 23:08:31 +0000
Subject: [PATCH 0054/1540] HBASE-5597 Findbugs check in test-patch.sh always
 fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303197 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/test-patch.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-support/test-patch.properties b/dev-support/test-patch.properties
index 6c3f5e37aaad..afe21d455dad 100644
--- a/dev-support/test-patch.properties
+++ b/dev-support/test-patch.properties
@@ -19,5 +19,5 @@ MAVEN_OPTS="-Xmx3g"
 # Please update the per-module test-patch.properties if you update this file.
 
 OK_RELEASEAUDIT_WARNINGS=84
-OK_FINDBUGS_WARNINGS=607
+OK_FINDBUGS_WARNINGS=768
 OK_JAVADOC_WARNINGS=169

From 1d776ead994be937f3e631e448cddc51e0d4b8ec Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 20 Mar 2012 23:59:23 +0000
Subject: [PATCH 0055/1540] HBASE-5569 Do not collect deleted KVs when they are
 still in use by a scanner.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303222 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     |  2 +-
 .../hbase/regionserver/ScanQueryMatcher.java  |  7 ++++-
 .../org/apache/hadoop/hbase/TestKeyValue.java |  2 +-
 .../regionserver/TestAtomicOperation.java     | 28 ++++++++++---------
 .../hbase/regionserver/TestCompaction.java    |  1 +
 5 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index b8fb08556b2d..0395f953f87d 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -692,7 +692,7 @@ public String toString() {
       return "empty";
     }
     return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) +
-      "/vlen=" + getValueLength();
+      "/vlen=" + getValueLength() + "/ts=" + memstoreTS;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 9f0cba9eb9c7..95b7831c3846 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -278,10 +278,15 @@ public MatchCode match(KeyValue kv) throws IOException {
       if (!keepDeletedCells) {
         // first ignore delete markers if the scanner can do so, and the
         // range does not include the marker
+        //
+        // during flushes and compactions also ignore delete markers newer
+        // than the readpoint of any open scanner, this prevents deleted
+        // rows that could still be seen by a scanner from being collected
         boolean includeDeleteMarker = seePastDeleteMarkers ?
             tr.withinTimeRange(timestamp) :
             tr.withinOrAfterTimeRange(timestamp);
-        if (includeDeleteMarker) {
+        if (includeDeleteMarker
+            && kv.getMemstoreTS() <= maxReadPointToTrackVersions) {
           this.deletes.add(bytes, offset, qualLength, timestamp, type);
         }
         // Can't early out now, because DelFam come before any other keys
diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
index f97d2bab3df0..fae690226c15 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
@@ -404,7 +404,7 @@ public void testCreateKeyValueFromKey() {
     System.err.println("kv=" + kv);
     System.err.println("kvFromKey=" + kvFromKey);
     assertEquals(kvFromKey.toString(),
-        kv.toString().replaceAll("=[0-9]+$", "=0"));
+        kv.toString().replaceAll("=[0-9]+", "=0"));
   }
 
   @org.junit.Rule
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 1640f296b0fc..d320d689f092 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -40,7 +40,6 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
 import org.junit.experimental.categories.Category;
 
@@ -55,11 +54,9 @@ public class TestAtomicOperation extends HBaseTestCase {
 
   HRegion region = null;
   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final String DIR = TEST_UTIL.getDataTestDir("TestIncrement").toString();
+  private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString();
 
 
-  private final int MAX_VERSIONS = 2;
-
   // Test names
   static final byte[] tableName = Bytes.toBytes("testtable");;
   static final byte[] qual1 = Bytes.toBytes("qual1");
@@ -260,8 +257,8 @@ public void testRowMutationMultiThreads() throws IOException {
 
     // create 100 threads, each will alternate between adding and
     // removing a column
-    int numThreads = 100;
-    int opsPerThread = 1000;
+    int numThreads = 10;
+    int opsPerThread = 500;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
     AtomicLong timeStamps = new AtomicLong(0);
@@ -275,9 +272,11 @@ public void run() {
           for (int i=0; i<numOps; i++) {
             try {
               // throw in some flushes
-              if (r.nextFloat() < 0.001) {
-                LOG.debug("flushing");
-                region.flushcache();
+              if (i%10==0) {
+                synchronized(region) {
+                  LOG.debug("flushing");
+                  region.flushcache();
+                }
               }
               long ts = timeStamps.incrementAndGet();
               RowMutations rm = new RowMutations(row);
@@ -342,7 +341,7 @@ public void testMultiRowMutationMultiThreads() throws IOException {
 
     // create 100 threads, each will alternate between adding and
     // removing a column
-    int numThreads = 100;
+    int numThreads = 10;
     int opsPerThread = 1000;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
@@ -358,9 +357,11 @@ public void run() {
           for (int i=0; i<numOps; i++) {
             try {
               // throw in some flushes
-              if (r.nextFloat() < 0.001) {
-                LOG.debug("flushing");
-                region.flushcache();
+              if (i%10==0) {
+                synchronized(region) {
+                  LOG.debug("flushing");
+                  region.flushcache();
+                }
               }
               long ts = timeStamps.incrementAndGet();
               List<Mutation> mrm = new ArrayList<Mutation>();
@@ -386,6 +387,7 @@ public void run() {
               RegionScanner rs = region.getScanner(s);
               List<KeyValue> r = new ArrayList<KeyValue>();
               while(rs.next(r));
+              rs.close();
               if (r.size() != 1) {
                 LOG.debug(r);
                 failures.incrementAndGet();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index bf186e1bb20e..91ac65287123 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -131,6 +131,7 @@ public void testMajorCompactingToNoOutput() throws IOException {
       r.delete(new Delete(results.get(0).getRow()), null, false);
       if (!result) break;
     } while(true);
+    s.close();
     // Flush
     r.flushcache();
     // Major compact.

From f8fc5d3c646e68110e94778567f75de2882207f2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 21 Mar 2012 19:03:10 +0000
Subject: [PATCH 0056/1540] HBASE-4542 add filter info to slow query logging
 (Madhuwanti Vaidya)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303503 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/Get.java   |   3 +
 .../org/apache/hadoop/hbase/client/Scan.java  |   9 +-
 .../hbase/filter/ColumnCountGetFilter.java    |   7 +-
 .../hbase/filter/ColumnPaginationFilter.java  |   8 +-
 .../hbase/filter/ColumnPrefixFilter.java      |   5 +
 .../hbase/filter/ColumnRangeFilter.java       |   8 +
 .../hadoop/hbase/filter/CompareFilter.java    |   9 +
 .../hbase/filter/DependentColumnFilter.java   |  10 +
 .../hadoop/hbase/filter/FilterBase.java       |   7 +
 .../hadoop/hbase/filter/FilterList.java       |  19 +-
 .../hbase/filter/InclusiveStopFilter.java     |   7 +-
 .../filter/MultipleColumnPrefixFilter.java    |  25 ++
 .../hadoop/hbase/filter/PageFilter.java       |   7 +-
 .../hadoop/hbase/filter/PrefixFilter.java     |   7 +-
 .../hbase/filter/SingleColumnValueFilter.java |   8 +
 .../hadoop/hbase/filter/SkipFilter.java       |   5 +
 .../hadoop/hbase/filter/TimestampsFilter.java |  25 ++
 .../hadoop/hbase/filter/WhileMatchFilter.java |   5 +
 .../hadoop/hbase/client/TestOperation.java    | 237 ++++++++++++++++++
 19 files changed, 402 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index 93c9e8961e55..c27b6472ffb9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -368,6 +368,9 @@ public Map<String, Object> toMap(int maxCols) {
       }   
     }   
     map.put("totalColumns", colCount);
+    if (this.filter != null) {
+      map.put("filter", this.filter.toString());
+    }
     return map;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index cc4c6e8d6102..7a53d44e89ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -304,11 +304,11 @@ public Scan setMaxVersions(int maxVersions) {
    * @param batch the maximum number of values
    */
   public void setBatch(int batch) {
-	if(this.hasFilter() && this.filter.hasFilterRow()) {
-	  throw new IncompatibleFilterException(
+    if (this.hasFilter() && this.filter.hasFilterRow()) {
+      throw new IncompatibleFilterException(
         "Cannot set batch on a scan using a filter" +
         " that returns true for filter.hasFilterRow");
-	}
+    }
     this.batch = batch;
   }
 
@@ -529,6 +529,9 @@ public Map<String, Object> toMap(int maxCols) {
       } 
     }       
     map.put("totalColumns", colCount);
+    if (this.filter != null) {
+      map.put("filter", this.filter.toString());
+    }
     return map;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
index 26f11aa029fa..c6946bb5bcb0 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
@@ -88,4 +88,9 @@ public void readFields(DataInput in) throws IOException {
   public void write(DataOutput out) throws IOException {
     out.writeInt(this.limit);
   }
-}
\ No newline at end of file
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + this.limit;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
index 85b0af77673e..d6b128018363 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
@@ -106,4 +106,10 @@ public void write(DataOutput out) throws IOException
     out.writeInt(this.limit);
     out.writeInt(this.offset);
   }
-}
\ No newline at end of file
+
+  @Override
+  public String toString() {
+    return String.format("%s (%d, %d)", this.getClass().getSimpleName(),
+        this.limit, this.offset);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java
index 931c7ad8d169..cb89b7d367e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java
@@ -101,4 +101,9 @@ public KeyValue getNextKeyHint(KeyValue kv) {
         kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
         kv.getFamilyOffset(), kv.getFamilyLength(), prefix, 0, prefix.length);
   }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + Bytes.toStringBinary(this.prefix);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java
index df8786c7b61d..bd5c57357c0f 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java
@@ -202,4 +202,12 @@ public KeyValue getNextKeyHint(KeyValue kv) {
         .getFamilyLength(), this.minColumn, 0, this.minColumn == null ? 0
         : this.minColumn.length);
   }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " "
+        + (this.minColumnInclusive ? "[" : "(") + Bytes.toStringBinary(this.minColumn)
+        + ", " + Bytes.toStringBinary(this.maxColumn)
+        + (this.maxColumnInclusive ? "]" : ")");
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java
index 370906029661..9089f4b559ee 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java
@@ -21,6 +21,7 @@
 package org.apache.hadoop.hbase.filter;
 
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.hbase.util.Bytes;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -155,4 +156,12 @@ public void write(DataOutput out) throws IOException {
     HbaseObjectWritable.writeObject(out, comparator,
       WritableByteArrayComparable.class, null);
   }
+
+  @Override
+  public String toString() {
+    return String.format("%s (%s, %s)",
+        this.getClass().getSimpleName(),
+        this.compareOp.name(),
+        Bytes.toStringBinary(this.comparator.getValue()));
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
index 785bd9e9f6bc..c50c821a2d54 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
@@ -237,4 +237,14 @@ public void write(DataOutput out) throws IOException {
     out.writeBoolean(this.dropDependentColumn);    
   }
 
+  @Override
+  public String toString() {
+    return String.format("%s (%s, %s, %s, %s, %s)",
+        this.getClass().getSimpleName(),
+        Bytes.toStringBinary(this.columnFamily),
+        Bytes.toStringBinary(this.columnQualifier),
+        this.dropDependentColumn,
+        this.compareOp.name(),
+        Bytes.toStringBinary(this.comparator.getValue()));
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
index 0d1b12339c10..48e26c5ebab7 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
@@ -139,4 +139,11 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
   public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments) {
     throw new IllegalArgumentException("This method has not been implemented");
   }
+
+  /**
+   * Return filter's info for debugging and logging purpose.
+   */
+  public String toString() {
+    return this.getClass().getSimpleName();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 216d0dbdc1b1..fee1a41df624 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -51,6 +51,7 @@ public static enum Operator {
   }
 
   private static final Configuration conf = HBaseConfiguration.create();
+  private static final int MAX_LOG_FILTERS = 5;
   private Operator operator = Operator.MUST_PASS_ALL;
   private List<Filter> filters = new ArrayList<Filter>();
 
@@ -310,4 +311,20 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
     }
     return keyHint;
   }
-}
\ No newline at end of file
+
+  @Override
+  public String toString() {
+    return toString(MAX_LOG_FILTERS);
+  }
+
+  protected String toString(int maxFilters) {
+    int endIndex = this.filters.size() < maxFilters
+        ? this.filters.size() : maxFilters;
+    return String.format("%s %s (%d/%d): %s",
+        this.getClass().getSimpleName(),
+        this.operator == Operator.MUST_PASS_ALL ? "AND" : "OR",
+        endIndex,
+        this.filters.size(),
+        this.filters.subList(0, endIndex).toString());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
index 091800c8ebc1..afa31c57e4f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
@@ -89,4 +89,9 @@ public void write(DataOutput out) throws IOException {
   public void readFields(DataInput in) throws IOException {
     this.stopRowKey = Bytes.readByteArray(in);
   }
-}
\ No newline at end of file
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + Bytes.toStringBinary(this.stopRowKey);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java
index 233b295c97ac..11fe6e3877d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java
@@ -36,6 +36,7 @@
 public class MultipleColumnPrefixFilter extends FilterBase {
   protected byte [] hint = null;
   protected TreeSet<byte []> sortedPrefixes = createTreeSet();
+  private final static int MAX_LOG_PREFIXES = 5;
 
   public MultipleColumnPrefixFilter() {
     super();
@@ -136,4 +137,28 @@ public int compare (Object o1, Object o2) {
         }
       });
   }
+
+  @Override
+  public String toString() {
+    return toString(MAX_LOG_PREFIXES);
+  }
+
+  protected String toString(int maxPrefixes) {
+    StringBuilder prefixes = new StringBuilder();
+
+    int count = 0;
+    for (byte[] ba : this.sortedPrefixes) {
+      if (count >= maxPrefixes) {
+        break;
+      }
+      ++count;
+      prefixes.append(Bytes.toStringBinary(ba));
+      if (count < this.sortedPrefixes.size() && count < maxPrefixes) {
+        prefixes.append(", ");
+      }
+    }
+
+    return String.format("%s (%d/%d): [%s]", this.getClass().getSimpleName(),
+        count, this.sortedPrefixes.size(), prefixes.toString());
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java
index 9d66c757ceb2..b00205ab0cd5 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java
@@ -88,4 +88,9 @@ public void readFields(final DataInput in) throws IOException {
   public void write(final DataOutput out) throws IOException {
     out.writeLong(pageSize);
   }
-}
\ No newline at end of file
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + this.pageSize;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
index 9de199b1c0aa..e3c3d39c83b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
@@ -84,4 +84,9 @@ public void write(DataOutput out) throws IOException {
   public void readFields(DataInput in) throws IOException {
     this.prefix = Bytes.readByteArray(in);
   }
-}
\ No newline at end of file
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + Bytes.toStringBinary(this.prefix);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
index 7e4b2aedfaa1..f6e0cb7ed13f 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
@@ -306,4 +306,12 @@ public void write(final DataOutput out) throws IOException {
     out.writeBoolean(filterIfMissing);
     out.writeBoolean(latestVersionOnly);
   }
+
+  @Override
+  public String toString() {
+    return String.format("%s (%s, %s, %s, %s)",
+        this.getClass().getSimpleName(), Bytes.toStringBinary(this.columnFamily),
+        Bytes.toStringBinary(this.columnQualifier), this.compareOp.name(),
+        Bytes.toStringBinary(this.comparator.getValue()));
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
index 8be40eece554..57fa99157944 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
@@ -103,4 +103,9 @@ public void readFields(DataInput in) throws IOException {
       throw new RuntimeException("Failed deserialize.", e);
     }
   }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + this.filter.toString();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java
index 551cbab38b67..5e780b51ec5a 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java
@@ -40,6 +40,7 @@
 public class TimestampsFilter extends FilterBase {
 
   TreeSet<Long> timestamps;
+  private static final int MAX_LOG_TIMESTAMPS = 5;
 
   // Used during scans to hint the scan to stop early
   // once the timestamps fall below the minTimeStamp.
@@ -129,4 +130,28 @@ public void write(DataOutput out) throws IOException {
       out.writeLong(timestamp);
     }
   }
+
+  @Override
+  public String toString() {
+    return toString(MAX_LOG_TIMESTAMPS);
+  }
+
+  protected String toString(int maxTimestamps) {
+    StringBuilder tsList = new StringBuilder();
+
+    int count = 0;
+    for (Long ts : this.timestamps) {
+      if (count >= maxTimestamps) {
+        break;
+      }
+      ++count;
+      tsList.append(ts.toString());
+      if (count < this.timestamps.size() && count < maxTimestamps) {
+        tsList.append(", ");
+      }
+    }
+
+    return String.format("%s (%d/%d): [%s]", this.getClass().getSimpleName(),
+        count, this.timestamps.size(), tsList.toString());
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
index b9fa92787ff0..242e0bdb7215 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
@@ -104,4 +104,9 @@ public void readFields(DataInput in) throws IOException {
       throw new RuntimeException("Failed deserialize.", e);
     }
   }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + this.filter.toString();
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java b/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
index 716022d7d40a..43f856f2d41d 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
@@ -26,10 +26,36 @@
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
+import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.DependentColumnFilter;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
+import org.apache.hadoop.hbase.filter.SkipFilter;
+import org.apache.hadoop.hbase.filter.TimestampsFilter;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.filter.WhileMatchFilter;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import org.codehaus.jackson.map.ObjectMapper;
@@ -48,6 +74,217 @@ public class TestOperation {
 
   private static ObjectMapper mapper = new ObjectMapper();
 
+  private static List<Long> TS_LIST = Arrays.asList(2L, 3L, 5L);
+  private static TimestampsFilter TS_FILTER = new TimestampsFilter(TS_LIST);
+  private static String STR_TS_FILTER =
+      TS_FILTER.getClass().getSimpleName() + " (3/3): [2, 3, 5]";
+
+  private static List<Long> L_TS_LIST =
+      Arrays.asList(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L);
+  private static TimestampsFilter L_TS_FILTER =
+      new TimestampsFilter(L_TS_LIST);
+  private static String STR_L_TS_FILTER =
+      L_TS_FILTER.getClass().getSimpleName() + " (5/11): [0, 1, 2, 3, 4]";
+
+  private static String COL_NAME_1 = "col1";
+  private static ColumnPrefixFilter COL_PRE_FILTER =
+      new ColumnPrefixFilter(COL_NAME_1.getBytes());
+  private static String STR_COL_PRE_FILTER =
+      COL_PRE_FILTER.getClass().getSimpleName() + " " + COL_NAME_1;
+
+  private static String COL_NAME_2 = "col2";
+  private static ColumnRangeFilter CR_FILTER = new ColumnRangeFilter(
+      COL_NAME_1.getBytes(), true, COL_NAME_2.getBytes(), false);
+  private static String STR_CR_FILTER = CR_FILTER.getClass().getSimpleName()
+      + " [" + COL_NAME_1 + ", " + COL_NAME_2 + ")";
+
+  private static int COL_COUNT = 9;
+  private static ColumnCountGetFilter CCG_FILTER =
+      new ColumnCountGetFilter(COL_COUNT);
+  private static String STR_CCG_FILTER =
+      CCG_FILTER.getClass().getSimpleName() + " " + COL_COUNT;
+
+  private static int LIMIT = 3;
+  private static int OFFSET = 4;
+  private static ColumnPaginationFilter CP_FILTER =
+      new ColumnPaginationFilter(LIMIT, OFFSET);
+  private static String STR_CP_FILTER = CP_FILTER.getClass().getSimpleName()
+      + " (" + LIMIT + ", " + OFFSET + ")";
+
+  private static String STOP_ROW_KEY = "stop";
+  private static InclusiveStopFilter IS_FILTER =
+      new InclusiveStopFilter(STOP_ROW_KEY.getBytes());
+  private static String STR_IS_FILTER =
+      IS_FILTER.getClass().getSimpleName() + " " + STOP_ROW_KEY;
+
+  private static String PREFIX = "prefix";
+  private static PrefixFilter PREFIX_FILTER =
+      new PrefixFilter(PREFIX.getBytes());
+  private static String STR_PREFIX_FILTER = "PrefixFilter " + PREFIX;
+
+  private static byte[][] PREFIXES = {
+      "0".getBytes(), "1".getBytes(), "2".getBytes()};
+  private static MultipleColumnPrefixFilter MCP_FILTER =
+      new MultipleColumnPrefixFilter(PREFIXES);
+  private static String STR_MCP_FILTER =
+      MCP_FILTER.getClass().getSimpleName() + " (3/3): [0, 1, 2]";
+
+  private static byte[][] L_PREFIXES = {
+    "0".getBytes(), "1".getBytes(), "2".getBytes(), "3".getBytes(),
+    "4".getBytes(), "5".getBytes(), "6".getBytes(), "7".getBytes()};
+  private static MultipleColumnPrefixFilter L_MCP_FILTER =
+      new MultipleColumnPrefixFilter(L_PREFIXES);
+  private static String STR_L_MCP_FILTER =
+      L_MCP_FILTER.getClass().getSimpleName() + " (5/8): [0, 1, 2, 3, 4]";
+
+  private static int PAGE_SIZE = 9;
+  private static PageFilter PAGE_FILTER = new PageFilter(PAGE_SIZE);
+  private static String STR_PAGE_FILTER =
+      PAGE_FILTER.getClass().getSimpleName() + " " + PAGE_SIZE;
+
+  private static SkipFilter SKIP_FILTER = new SkipFilter(L_TS_FILTER);
+  private static String STR_SKIP_FILTER =
+      SKIP_FILTER.getClass().getSimpleName() + " " + STR_L_TS_FILTER;
+
+  private static WhileMatchFilter WHILE_FILTER =
+      new WhileMatchFilter(L_TS_FILTER);
+  private static String STR_WHILE_FILTER =
+      WHILE_FILTER.getClass().getSimpleName() + " " + STR_L_TS_FILTER;
+
+  private static KeyOnlyFilter KEY_ONLY_FILTER = new KeyOnlyFilter();
+  private static String STR_KEY_ONLY_FILTER =
+      KEY_ONLY_FILTER.getClass().getSimpleName();
+
+  private static FirstKeyOnlyFilter FIRST_KEY_ONLY_FILTER =
+      new FirstKeyOnlyFilter();
+  private static String STR_FIRST_KEY_ONLY_FILTER =
+      FIRST_KEY_ONLY_FILTER.getClass().getSimpleName();
+
+  private static CompareOp CMP_OP = CompareOp.EQUAL;
+  private static byte[] CMP_VALUE = "value".getBytes();
+  private static BinaryComparator BC = new BinaryComparator(CMP_VALUE);
+  private static DependentColumnFilter DC_FILTER =
+      new DependentColumnFilter(FAMILY, QUALIFIER, true, CMP_OP, BC);
+  private static String STR_DC_FILTER = String.format(
+      "%s (%s, %s, %s, %s, %s)", DC_FILTER.getClass().getSimpleName(),
+      Bytes.toStringBinary(FAMILY), Bytes.toStringBinary(QUALIFIER), true,
+      CMP_OP.name(), Bytes.toStringBinary(BC.getValue()));
+
+  private static FamilyFilter FAMILY_FILTER = new FamilyFilter(CMP_OP, BC);
+  private static String STR_FAMILY_FILTER =
+      FAMILY_FILTER.getClass().getSimpleName() + " (EQUAL, value)";
+
+  private static QualifierFilter QUALIFIER_FILTER =
+      new QualifierFilter(CMP_OP, BC);
+  private static String STR_QUALIFIER_FILTER =
+      QUALIFIER_FILTER.getClass().getSimpleName() + " (EQUAL, value)";
+
+  private static RowFilter ROW_FILTER = new RowFilter(CMP_OP, BC);
+  private static String STR_ROW_FILTER =
+      ROW_FILTER.getClass().getSimpleName() + " (EQUAL, value)";
+
+  private static ValueFilter VALUE_FILTER = new ValueFilter(CMP_OP, BC);
+  private static String STR_VALUE_FILTER =
+      VALUE_FILTER.getClass().getSimpleName() + " (EQUAL, value)";
+
+  private static SingleColumnValueFilter SCV_FILTER =
+      new SingleColumnValueFilter(FAMILY, QUALIFIER, CMP_OP, CMP_VALUE);
+  private static String STR_SCV_FILTER = String.format("%s (%s, %s, %s, %s)",
+      SCV_FILTER.getClass().getSimpleName(), Bytes.toStringBinary(FAMILY),
+      Bytes.toStringBinary(QUALIFIER), CMP_OP.name(),
+      Bytes.toStringBinary(CMP_VALUE));
+
+  private static SingleColumnValueExcludeFilter SCVE_FILTER =
+      new SingleColumnValueExcludeFilter(FAMILY, QUALIFIER, CMP_OP, CMP_VALUE);
+  private static String STR_SCVE_FILTER = String.format("%s (%s, %s, %s, %s)",
+      SCVE_FILTER.getClass().getSimpleName(), Bytes.toStringBinary(FAMILY),
+      Bytes.toStringBinary(QUALIFIER), CMP_OP.name(),
+      Bytes.toStringBinary(CMP_VALUE));
+
+  private static FilterList AND_FILTER_LIST = new FilterList(
+      Operator.MUST_PASS_ALL, Arrays.asList((Filter) TS_FILTER, L_TS_FILTER,
+          CR_FILTER));
+  private static String STR_AND_FILTER_LIST = String.format(
+      "%s AND (3/3): [%s, %s, %s]", AND_FILTER_LIST.getClass().getSimpleName(),
+      STR_TS_FILTER, STR_L_TS_FILTER, STR_CR_FILTER);
+
+  private static FilterList OR_FILTER_LIST = new FilterList(
+      Operator.MUST_PASS_ONE, Arrays.asList((Filter) TS_FILTER, L_TS_FILTER,
+          CR_FILTER));
+  private static String STR_OR_FILTER_LIST = String.format(
+      "%s OR (3/3): [%s, %s, %s]", AND_FILTER_LIST.getClass().getSimpleName(),
+      STR_TS_FILTER, STR_L_TS_FILTER, STR_CR_FILTER);
+
+  private static FilterList L_FILTER_LIST = new FilterList(
+      Arrays.asList((Filter) TS_FILTER, L_TS_FILTER, CR_FILTER, COL_PRE_FILTER,
+          CCG_FILTER, CP_FILTER, PREFIX_FILTER, PAGE_FILTER));
+  private static String STR_L_FILTER_LIST = String.format(
+      "%s AND (5/8): [%s, %s, %s, %s, %s]",
+      L_FILTER_LIST.getClass().getSimpleName(), STR_TS_FILTER, STR_L_TS_FILTER,
+      STR_CR_FILTER, STR_COL_PRE_FILTER, STR_CCG_FILTER, STR_CP_FILTER);
+
+  private static Filter[] FILTERS = {
+    TS_FILTER,             // TimestampsFilter
+    L_TS_FILTER,           // TimestampsFilter
+    COL_PRE_FILTER,        // ColumnPrefixFilter
+    CP_FILTER,             // ColumnPaginationFilter
+    CR_FILTER,             // ColumnRangeFilter
+    CCG_FILTER,            // ColumnCountGetFilter
+    IS_FILTER,             // InclusiveStopFilter
+    PREFIX_FILTER,         // PrefixFilter
+    PAGE_FILTER,           // PageFilter
+    SKIP_FILTER,           // SkipFilter
+    WHILE_FILTER,          // WhileMatchFilter
+    KEY_ONLY_FILTER,       // KeyOnlyFilter
+    FIRST_KEY_ONLY_FILTER, // FirstKeyOnlyFilter
+    MCP_FILTER,            // MultipleColumnPrefixFilter
+    L_MCP_FILTER,          // MultipleColumnPrefixFilter
+    DC_FILTER,             // DependentColumnFilter
+    FAMILY_FILTER,         // FamilyFilter
+    QUALIFIER_FILTER,      // QualifierFilter
+    ROW_FILTER,            // RowFilter
+    VALUE_FILTER,          // ValueFilter
+    SCV_FILTER,            // SingleColumnValueFilter
+    SCVE_FILTER,           // SingleColumnValueExcludeFilter
+    AND_FILTER_LIST,       // FilterList
+    OR_FILTER_LIST,        // FilterList
+    L_FILTER_LIST,         // FilterList
+  };
+
+  private static String[] FILTERS_INFO = {
+    STR_TS_FILTER,             // TimestampsFilter
+    STR_L_TS_FILTER,           // TimestampsFilter
+    STR_COL_PRE_FILTER,        // ColumnPrefixFilter
+    STR_CP_FILTER,             // ColumnPaginationFilter
+    STR_CR_FILTER,             // ColumnRangeFilter
+    STR_CCG_FILTER,            // ColumnCountGetFilter
+    STR_IS_FILTER,             // InclusiveStopFilter
+    STR_PREFIX_FILTER,         // PrefixFilter
+    STR_PAGE_FILTER,           // PageFilter
+    STR_SKIP_FILTER,           // SkipFilter
+    STR_WHILE_FILTER,          // WhileMatchFilter
+    STR_KEY_ONLY_FILTER,       // KeyOnlyFilter
+    STR_FIRST_KEY_ONLY_FILTER, // FirstKeyOnlyFilter
+    STR_MCP_FILTER,            // MultipleColumnPrefixFilter
+    STR_L_MCP_FILTER,          // MultipleColumnPrefixFilter
+    STR_DC_FILTER,             // DependentColumnFilter
+    STR_FAMILY_FILTER,         // FamilyFilter
+    STR_QUALIFIER_FILTER,      // QualifierFilter
+    STR_ROW_FILTER,            // RowFilter
+    STR_VALUE_FILTER,          // ValueFilter
+    STR_SCV_FILTER,            // SingleColumnValueFilter
+    STR_SCVE_FILTER,           // SingleColumnValueExcludeFilter
+    STR_AND_FILTER_LIST,       // FilterList
+    STR_OR_FILTER_LIST,        // FilterList
+    STR_L_FILTER_LIST,         // FilterList
+  };
+
+  static {
+    assertEquals("The sizes of static arrays do not match: "
+        + "[FILTERS: %d <=> FILTERS_INFO: %d]",
+        FILTERS.length, FILTERS_INFO.length);
+  }
+
   /**
    * Test the client Operations' JSON encoding to ensure that produced JSON is 
    * parseable and that the details are present and not corrupted.

From 3615ad69b0deacaa048fe8652fbd7eb022bb881b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 21 Mar 2012 19:18:36 +0000
Subject: [PATCH 0057/1540] HBASE-5560 Avoid RegionServer GC caused by
 timed-out calls

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303513 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/HBaseServer.java     | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 5e0040426454..d1039cca17a3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -58,6 +58,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.io.WritableWithSize;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
@@ -194,6 +195,7 @@ public static String getRemoteAddress() {
   protected int socketSendBufferSize;
   protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm
   protected final boolean tcpKeepAlive; // if T then use keepalives
+  protected final long purgeTimeout;    // in milliseconds
 
   volatile protected boolean running = true;         // true while server runs
   protected BlockingQueue<Call> callQueue; // queued calls
@@ -749,8 +751,6 @@ protected class Responder extends Thread {
     private final Selector writeSelector;
     private int pending;         // connections waiting to register
 
-    final static int PURGE_INTERVAL = 900000; // 15mins
-
     Responder() throws IOException {
       this.setName("IPC Server Responder");
       this.setDaemon(true);
@@ -780,7 +780,7 @@ private void doRunLoop() {
       while (running) {
         try {
           waitPending();     // If a channel is being registered, wait.
-          writeSelector.select(PURGE_INTERVAL);
+          writeSelector.select(purgeTimeout);
           Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();
           while (iter.hasNext()) {
             SelectionKey key = iter.next();
@@ -794,7 +794,7 @@ private void doRunLoop() {
             }
           }
           long now = System.currentTimeMillis();
-          if (now < lastPurgeTime + PURGE_INTERVAL) {
+          if (now < lastPurgeTime + purgeTimeout) {
             continue;
           }
           lastPurgeTime = now;
@@ -882,7 +882,7 @@ private void doPurge(Call call, long now) throws IOException {
         Iterator<Call> iter = call.connection.responseQueue.listIterator(0);
         while (iter.hasNext()) {
           Call nextCall = iter.next();
-          if (now > nextCall.timestamp + PURGE_INTERVAL) {
+          if (now > nextCall.timestamp + purgeTimeout) {
             closeConnection(nextCall.connection);
             break;
           }
@@ -1452,6 +1452,8 @@ protected HBaseServer(String bindAddress, int port,
     this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000);
     this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
     this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
+    this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout",
+                                     2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 
     // Start the listener here and let it bind to the port
     listener = new Listener();

From 34bca198f5cfd17fcc1b72b725e7ba8c321a8692 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 21 Mar 2012 19:31:31 +0000
Subject: [PATCH 0058/1540] HBASE-5589 Add of the offline call to the Master
 Interface

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303520 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/ipc/HMasterInterface.java      | 17 +++++++++++++++++
 .../org/apache/hadoop/hbase/master/HMaster.java | 13 +++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index 2ee0c6269498..be83d64d3bb3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -51,6 +51,11 @@ public interface HMasterInterface extends VersionedProtocol {
   // meant all HBase RPC was broke though only one of the three RPC Interfaces
   // had changed.  This has since been undone.
   // 29:  4/3/2010 - changed ClusterStatus serialization
+  // 30: 3/20/2012 - HBASE-5589: Added offline method 
+	
+  // NOTE: Not bumped from 29 to maintain compatibility since this addition is
+  // after the v0.92.0 releases this is applied to.  This is not bumped for
+  // 0.94.0 to maintain rolling restart compatibility with 0.92.x. 
   public static final long VERSION = 29L;
 
   /** @return true if master is available */
@@ -213,6 +218,18 @@ public void assign(final byte [] regionName, final boolean force)
   public void unassign(final byte [] regionName, final boolean force)
   throws IOException;
 
+  
+  /**
+   * Offline a region from the assignment manager's in-memory state.  The
+   * region should be in a closed state and there will be no attempt to
+   * automatically reassign the region as in unassign.   This is a special
+   * method, and should only be used by experts or hbck.
+   * @param regionName Region to offline.  Will clear any existing RegionPlan
+   * if one found.
+   * @throws IOException
+   */
+  public void offline(final byte[] regionName) throws IOException;
+
   /**
    * Run the balancer.  Will run the balancer and if regions to move, it will
    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 6f1e714eef6b..46bd970c4661 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1821,6 +1821,19 @@ public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
   public double getAverageLoad() {
     return this.assignmentManager.getAverageLoad();
   }
+  
+  /**
+   * Special method, only used by hbck.
+   */
+  @Override
+  public void offline(final byte[] regionName) 
+  throws IOException {
+    Pair<HRegionInfo, ServerName> pair =
+      MetaReader.getRegion(this.catalogTracker, regionName);
+    if (pair == null) throw new UnknownRegionException(Bytes.toStringBinary(regionName));
+    HRegionInfo hri = pair.getFirst();
+    this.assignmentManager.regionOffline(hri);    
+  }
 
   /**
    * Utility for constructing an instance of the passed HMaster class.

From ebb949343cb4743954dff18466a28c3857c2f478 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 21 Mar 2012 19:44:47 +0000
Subject: [PATCH 0059/1540] HBASE-5433 [REST] Add metrics to keep track of
 success/failure count (Mubarak Seyed)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303528 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/MultiRowResource.java   |   2 +
 .../hadoop/hbase/rest/RegionsResource.java    |   3 +
 .../hadoop/hbase/rest/RootResource.java       |   2 +
 .../apache/hadoop/hbase/rest/RowResource.java |  10 ++
 .../hbase/rest/ScannerInstanceResource.java   |  22 +++-
 .../hadoop/hbase/rest/ScannerResource.java    |  12 +-
 .../hadoop/hbase/rest/SchemaResource.java     |  10 ++
 .../rest/StorageClusterStatusResource.java    |   2 +
 .../rest/StorageClusterVersionResource.java   |   2 +
 .../hadoop/hbase/rest/VersionResource.java    |   1 +
 .../hbase/rest/metrics/RESTMetrics.java       | 103 ++++++++++++++++++
 11 files changed, 165 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index 6e62150299be..6adfc63f9faa 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -89,8 +89,10 @@ public Response get(final @Context UriInfo uriInfo) {
 
         model.addRow(rowModel);
       }
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return Response.ok(model).build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e,
               Response.Status.SERVICE_UNAVAILABLE);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index bf85bc10d988..d0058aacf5d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -100,10 +100,13 @@ public Response get(final @Context UriInfo uriInfo) {
       }
       ResponseBuilder response = Response.ok(model);
       response.cacheControl(cacheControl);
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (TableNotFoundException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(Response.Status.NOT_FOUND);
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
index 4cf37a81e18e..1931318ce43d 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
@@ -80,8 +80,10 @@ public Response get(final @Context UriInfo uriInfo) {
     try {
       ResponseBuilder response = Response.ok(getTableList());
       response.cacheControl(cacheControl);
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e, 
                   Response.Status.SERVICE_UNAVAILABLE);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index adf1ff621f99..ee1e96dabd09 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -105,8 +105,10 @@ public Response get(final @Context UriInfo uriInfo) {
         value = generator.next();
       } while (value != null);
       model.addRow(rowModel);
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return Response.ok(model).build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     }
@@ -133,8 +135,10 @@ public Response getBinary(final @Context UriInfo uriInfo) {
       KeyValue value = generator.next();
       ResponseBuilder response = Response.ok(value.getValue());
       response.header("X-Timestamp", value.getTimestamp());
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     }
@@ -186,8 +190,10 @@ Response update(final CellSetModel model, final boolean replace) {
       table.put(puts);
       table.flushCommits();
       ResponseBuilder response = Response.ok();
+      servlet.getMetrics().incrementSucessfulPutRequests(1);
       return response.build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     } finally {
@@ -246,8 +252,10 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
       if (LOG.isDebugEnabled()) {
         LOG.debug("PUT " + put.toString());
       }
+      servlet.getMetrics().incrementSucessfulPutRequests(1);
       return Response.ok().build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     } finally {
@@ -338,10 +346,12 @@ public Response delete(final @Context UriInfo uriInfo) {
     try {
       table = pool.getTable(tableResource.getName());
       table.delete(delete);
+      servlet.getMetrics().incrementSucessfulDeleteRequests(1);
       if (LOG.isDebugEnabled()) {
         LOG.debug("DELETE " + delete.toString());
       }
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedDeleteRequests(1);
       throw new WebApplicationException(e, 
                   Response.Status.SERVICE_UNAVAILABLE);
     } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
index 75f106590160..fbd6cfaea667 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
@@ -86,7 +86,12 @@ public Response get(final @Context UriInfo uriInfo,
       try {
         value = generator.next();
       } catch (IllegalStateException e) {
-        ScannerResource.delete(id);
+        if (ScannerResource.delete(id)) {
+          servlet.getMetrics().incrementSucessfulDeleteRequests(1);
+        } else {
+          servlet.getMetrics().incrementFailedDeleteRequests(1);
+        }
+        servlet.getMetrics().incrementFailedGetRequests(1);
         throw new WebApplicationException(Response.Status.GONE);
       }
       if (value == null) {
@@ -122,6 +127,7 @@ public Response get(final @Context UriInfo uriInfo,
     model.addRow(rowModel);
     ResponseBuilder response = Response.ok(model);
     response.cacheControl(cacheControl);
+    servlet.getMetrics().incrementSucessfulGetRequests(1);
     return response.build();
   }
 
@@ -146,9 +152,15 @@ public Response getBinary(final @Context UriInfo uriInfo) {
         Base64.encodeBytes(
           KeyValue.makeColumn(value.getFamily(), value.getQualifier())));
       response.header("X-Timestamp", value.getTimestamp());
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (IllegalStateException e) {
-      ScannerResource.delete(id);
+      if (ScannerResource.delete(id)) {
+        servlet.getMetrics().incrementSucessfulDeleteRequests(1);
+      } else {
+        servlet.getMetrics().incrementFailedDeleteRequests(1);
+      }
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(Response.Status.GONE);
     }
   }
@@ -162,7 +174,11 @@ public Response delete(final @Context UriInfo uriInfo) {
     if (servlet.isReadOnly()) {
       throw new WebApplicationException(Response.Status.FORBIDDEN);
     }
-    ScannerResource.delete(id);
+    if (ScannerResource.delete(id)) {
+      servlet.getMetrics().incrementSucessfulDeleteRequests(1);
+    } else {
+      servlet.getMetrics().incrementFailedDeleteRequests(1);
+    }
     return Response.ok().build();
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
index dad5b4377d6b..be19cb91f533 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
@@ -63,10 +63,13 @@ public ScannerResource(TableResource tableResource)throws IOException {
     this.tableResource = tableResource;
   }
 
-  static void delete(final String id) {
+  static boolean delete(final String id) {
     ScannerInstanceResource instance = scanners.remove(id);
     if (instance != null) {
       instance.generator.close();
+      return true;
+    } else {
+      return false;
     }
   }
 
@@ -94,16 +97,20 @@ Response update(final ScannerModel model, final boolean replace,
       }
       UriBuilder builder = uriInfo.getAbsolutePathBuilder();
       URI uri = builder.path(id).build();
+      servlet.getMetrics().incrementSucessfulPutRequests(1);
       return Response.created(uri).build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
       throw new WebApplicationException(e,
               Response.Status.SERVICE_UNAVAILABLE);
     } catch (RuntimeException e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
       if (e.getCause() instanceof TableNotFoundException) {
         throw new WebApplicationException(e, Response.Status.NOT_FOUND);
       }
       throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
     } catch (Exception e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
       throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
     }
   }
@@ -133,7 +140,10 @@ public ScannerInstanceResource getScannerInstanceResource(
       final @PathParam("scanner") String id) {
     ScannerInstanceResource instance = scanners.get(id);
     if (instance == null) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(Response.Status.NOT_FOUND);
+    } else {
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
     }
     return instance;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index 5e9a5be55250..a31744aaf498 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -96,10 +96,13 @@ public Response get(final @Context UriInfo uriInfo) {
       ResponseBuilder response =
         Response.ok(new TableSchemaModel(getTableSchema()));
       response.cacheControl(cacheControl);
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (TableNotFoundException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(Response.Status.NOT_FOUND);
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     }
@@ -126,8 +129,10 @@ private Response replace(final byte[] name, final TableSchemaModel model,
         admin.disableTable(name);
         admin.modifyTable(name, htd);
         admin.enableTable(name);
+        servlet.getMetrics().incrementSucessfulPutRequests(1);
       } else try {
         admin.createTable(htd);
+        servlet.getMetrics().incrementSucessfulPutRequests(1);
       } catch (TableExistsException e) {
         // race, someone else created a table with the same name
         throw new WebApplicationException(e, Response.Status.NOT_MODIFIED);
@@ -165,6 +170,7 @@ private Response update(final byte[] name, final TableSchemaModel model,
       } finally {
         admin.enableTable(tableResource.getName());
       }
+      servlet.getMetrics().incrementSucessfulPutRequests(1);
       return Response.ok().build();
     } catch (IOException e) {
       throw new WebApplicationException(e,
@@ -183,6 +189,7 @@ private Response update(final TableSchemaModel model, final boolean replace,
         return update(name, model, uriInfo, admin);
       }
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
       throw new WebApplicationException(e, 
             Response.Status.SERVICE_UNAVAILABLE);
     }
@@ -229,10 +236,13 @@ public Response delete(final @Context UriInfo uriInfo) {
         throw new IOException("could not disable table");
       }
       admin.deleteTable(tableResource.getName());
+      servlet.getMetrics().incrementSucessfulDeleteRequests(1);
       return Response.ok().build();
     } catch (TableNotFoundException e) {
+      servlet.getMetrics().incrementFailedDeleteRequests(1);
       throw new WebApplicationException(Response.Status.NOT_FOUND);
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedDeleteRequests(1);
       throw new WebApplicationException(e, 
             Response.Status.SERVICE_UNAVAILABLE);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
index cddade0c76f9..e48562725ee0 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
@@ -93,8 +93,10 @@ public Response get(final @Context UriInfo uriInfo) {
       }
       ResponseBuilder response = Response.ok(model);
       response.cacheControl(cacheControl);
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e, 
                   Response.Status.SERVICE_UNAVAILABLE);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
index 106c9dcb41a1..6768f7d43328 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
@@ -69,8 +69,10 @@ public Response get(final @Context UriInfo uriInfo) {
       model.setVersion(admin.getClusterStatus().getHBaseVersion());
       ResponseBuilder response = Response.ok(model);
       response.cacheControl(cacheControl);
+      servlet.getMetrics().incrementSucessfulGetRequests(1);
       return response.build();
     } catch (IOException e) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
       throw new WebApplicationException(e, 
                   Response.Status.SERVICE_UNAVAILABLE);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
index 3d0a9b3f0b9a..5217ee2c9612 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
@@ -79,6 +79,7 @@ public Response get(final @Context ServletContext context,
     servlet.getMetrics().incrementRequests(1);
     ResponseBuilder response = Response.ok(new VersionModel(context));
     response.cacheControl(cacheControl);
+    servlet.getMetrics().incrementSucessfulGetRequests(1);
     return response.build();
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java
index 284bbc548114..d5abe6fb0a95 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java
@@ -35,6 +35,19 @@ public class RESTMetrics implements Updater {
   private final RESTStatistics restStatistics;
 
   private MetricsRate requests = new MetricsRate("requests", registry);
+  private MetricsRate sucessfulGetCount =
+      new MetricsRate("sucessful.get.count", registry);
+  private MetricsRate sucessfulPutCount =
+      new MetricsRate("sucessful.put.count", registry);
+  private MetricsRate sucessfulDeleteCount =
+      new MetricsRate("sucessful.delete.count", registry);
+  
+  private MetricsRate failedGetCount =
+      new MetricsRate("failed.get.count", registry);
+  private MetricsRate failedPutCount =
+      new MetricsRate("failed.put.count", registry);
+  private MetricsRate failedDeleteCount =
+      new MetricsRate("failed.delete.count", registry);
 
   public RESTMetrics() {
     MetricsContext context = MetricsUtil.getContext("rest");
@@ -62,6 +75,12 @@ public void shutdown() {
   public void doUpdates(MetricsContext unused) {
     synchronized (this) {
       requests.pushMetric(metricsRecord);
+      sucessfulGetCount.pushMetric(metricsRecord);
+      sucessfulPutCount.pushMetric(metricsRecord);
+      sucessfulDeleteCount.pushMetric(metricsRecord);
+      failedGetCount.pushMetric(metricsRecord);
+      failedPutCount.pushMetric(metricsRecord);
+      failedDeleteCount.pushMetric(metricsRecord);
     }
     this.metricsRecord.update();
   }
@@ -83,5 +102,89 @@ public float getRequests() {
   public void incrementRequests(final int inc) {
     requests.inc(inc);
   }
+  
+  /**
+   * @return Count of sucessfulGetCount.
+   */
+  public float getSucessfulGetCount() {
+    return sucessfulGetCount.getPreviousIntervalValue();
+  }
+  
+  /**
+   * @param inc How much to add to sucessfulGetCount.
+   */
+  public void incrementSucessfulGetRequests(final int inc) {
+    sucessfulGetCount.inc(inc);
+  }
+  
+  /**
+   * @return Count of sucessfulGetCount.
+   */
+  public float getSucessfulPutCount() {
+    return sucessfulPutCount.getPreviousIntervalValue();
+  }
+  
+  /**
+   * @param inc How much to add to sucessfulPutCount.
+   */
+  public void incrementSucessfulPutRequests(final int inc) {
+    sucessfulPutCount.inc(inc);
+  }
+  
+  /**
+   * @return Count of failedPutCount.
+   */
+  public float getFailedPutCount() {
+    return failedPutCount.getPreviousIntervalValue();
+  }
+  
+  /**
+   * @param inc How much to add to failedPutCount.
+   */
+  public void incrementFailedPutRequests(final int inc) {
+    failedPutCount.inc(inc);
+  }
+  
+  /**
+   * @return Count of failedGetCount.
+   */
+  public float getFailedGetCount() {
+    return failedGetCount.getPreviousIntervalValue();
+  }
+  
+  /**
+   * @param inc How much to add to failedGetCount.
+   */
+  public void incrementFailedGetRequests(final int inc) {
+    failedGetCount.inc(inc);
+  }
+  
+  /**
+   * @return Count of sucessfulGetCount.
+   */
+  public float getSucessfulDeleteCount() {
+    return sucessfulDeleteCount.getPreviousIntervalValue();
+  }
+  
+  /**
+   * @param inc How much to add to sucessfulDeleteCount.
+   */
+  public void incrementSucessfulDeleteRequests(final int inc) {
+    sucessfulDeleteCount.inc(inc);
+  }
 
+  /**
+   * @return Count of failedDeleteCount.
+   */
+  public float getFailedDeleteCount() {
+    return failedDeleteCount.getPreviousIntervalValue();
+  }
+  
+  /**
+   * @param inc How much to add to failedDeleteCount.
+   */
+  public void incrementFailedDeleteRequests(final int inc) {
+    failedDeleteCount.inc(inc);
+  }
+  
 }

From 051e84c0fd843ecee11f9c0d07497fd6ec0d91cb Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 21 Mar 2012 20:16:09 +0000
Subject: [PATCH 0060/1540] HBASE-4940 hadoop-metrics.properties can include
 configuration of the rest context for ganglia (Mubarak Seyed)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303552 13f79535-47bb-0310-9956-ffa450edef68
---
 conf/hadoop-metrics.properties | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/conf/hadoop-metrics.properties b/conf/hadoop-metrics.properties
index 046a369524da..b348e8b663cf 100644
--- a/conf/hadoop-metrics.properties
+++ b/conf/hadoop-metrics.properties
@@ -7,6 +7,8 @@
 # for the moment.
 #
 # See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia 
+# meta daemon (gmetad) service is running
 
 # Configuration of the "hbase" context for null
 hbase.class=org.apache.hadoop.metrics.spi.NullContext
@@ -56,3 +58,10 @@ rpc.class=org.apache.hadoop.metrics.spi.NullContext
 # rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
 # rpc.period=10
 # rpc.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rest" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rest.period=10
+# rest.servers=GMETADHOST_IP:8649

From dc79f28ddc9cc230c37ba72afcc2db222875d84f Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 22 Mar 2012 18:13:04 +0000
Subject: [PATCH 0061/1540]    HBASE-5586  [replication] NPE in
 ReplicationSource when creating a stream                to an inexistent
 cluster

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1303944 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/ReplicationZookeeper.java     |   6 +-
 .../replication/TestReplicationZookeeper.java | 118 ++++++++++++++++++
 2 files changed, 121 insertions(+), 3 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 8c9cb9b6925d..e2f4771f944e 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -215,11 +215,11 @@ public Map<String,String> listPeers() {
    */
   public List<ServerName> getSlavesAddresses(String peerClusterId) {
     if (this.peerClusters.size() == 0) {
-      return new ArrayList<ServerName>(0);
+      return Collections.emptyList();
     }
     ReplicationPeer peer = this.peerClusters.get(peerClusterId);
     if (peer == null) {
-      return new ArrayList<ServerName>(0);
+      return Collections.emptyList();
     }
     
     List<ServerName> addresses;
@@ -278,7 +278,7 @@ public static List<ServerName> listChildrenAndGetAsServerNames(
   throws KeeperException {
     List<String> children = ZKUtil.listChildrenNoWatch(zkw, znode);
     if(children == null) {
-      return null;
+      return Collections.emptyList();
     }
     List<ServerName> addresses = new ArrayList<ServerName>(children.size());
     for (String child : children) {
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java
new file mode 100644
index 000000000000..553b5cd7d27e
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+
+@Category(MediumTests.class)
+public class TestReplicationZookeeper {
+
+  private static Configuration conf;
+
+  private static HBaseTestingUtility utility;
+
+  private static ZooKeeperWatcher zkw;
+
+  private static ReplicationZookeeper repZk;
+
+  private static String slaveClusterKey;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    utility = new HBaseTestingUtility();
+    utility.startMiniZKCluster();
+    conf = utility.getConfiguration();
+    zkw = HBaseTestingUtility.getZooKeeperWatcher(utility);
+    DummyServer server = new DummyServer();
+    repZk = new ReplicationZookeeper(server, new AtomicBoolean());
+    slaveClusterKey = conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" +
+      conf.get("hbase.zookeeper.property.clientPort") + ":/1";
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    utility.shutdownMiniZKCluster();
+  }
+
+  @Test
+  public void testGetAddressesMissingSlave()
+    throws IOException, KeeperException {
+    repZk.addPeer("1", slaveClusterKey);
+    // HBASE-5586 used to get an NPE
+    assertEquals(0, repZk.getSlavesAddresses("1").size());
+  }
+
+  static class DummyServer implements Server {
+
+    @Override
+    public Configuration getConfiguration() {
+      return conf;
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      return zkw;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+
+    @Override
+    public ServerName getServerName() {
+      return new ServerName("hostname.example.org", 1234, -1L);
+    }
+
+    @Override
+    public void abort(String why, Throwable e) {
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {
+    }
+
+    @Override
+    public boolean isStopped() {
+      return false;
+    }
+  }
+}

From e4c0271bca99a62e9e8e48521cff4d803c376037 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 22 Mar 2012 22:37:51 +0000
Subject: [PATCH 0062/1540] HBASE-4657 Improve the efficiency of our MR jobs
 with a few configurations

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304114 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/mapreduce/CopyTable.java   | 6 +++++-
 src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java | 4 ++++
 src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java | 3 +++
 .../java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java   | 5 ++++-
 .../java/org/apache/hadoop/hbase/mapreduce/RowCounter.java  | 6 +++++-
 5 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 798f22799dfc..01f6d152c9dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -66,6 +66,7 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
     Job job = new Job(conf, NAME + "_" + tableName);
     job.setJarByClass(CopyTable.class);
     Scan scan = new Scan();
+    scan.setCacheBlocks(false);
     if (startTime != 0) {
       scan.setTimeRange(startTime,
           endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
@@ -111,7 +112,7 @@ private static void printUsage(final String errorMsg) {
     if (errorMsg != null && errorMsg.length() > 0) {
       System.err.println("ERROR: " + errorMsg);
     }
-    System.err.println("Usage: CopyTable [--rs.class=CLASS] " +
+    System.err.println("Usage: CopyTable [general options] [--rs.class=CLASS] " +
         "[--rs.impl=IMPL] [--starttime=X] [--endtime=Y] " +
         "[--new.name=NEW] [--peer.adr=ADR] <tablename>");
     System.err.println();
@@ -140,6 +141,9 @@ private static void printUsage(final String errorMsg) {
         "org.apache.hadoop.hbase.mapreduce.CopyTable --rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface " +
         "--rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer --starttime=1265875194289 --endtime=1265878794289 " +
         "--peer.adr=server1,server2,server3:2181:/hbase --families=myOldCf:myNewCf,cf2,cf3 TestTable ");
+    System.err.println("For performance consider the following general options:\n"
+        + "-Dhbase.client.scanner.caching=100\n"
+        + "-Dmapred.map.tasks.speculative.execution=false");
   }
 
   private static boolean doCommandLine(final String[] args) {
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
index 46ab6fd7384c..2c53f6d8161d 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
@@ -166,6 +166,10 @@ private static void usage(final String errorMsg) {
     System.err.println("  to control/limit what is exported..");
     System.err.println("   -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=<familyName>");
     System.err.println("   -D " + RAW_SCAN + "=true");
+    System.err.println("For performance consider the following properties:\n"
+        + "   -Dhbase.client.scanner.caching=100\n"
+        + "   -Dmapred.map.tasks.speculative.execution=false\n"
+        + "   -Dmapred.reduce.tasks.speculative.execution=false");
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index dac1b7560e8f..03d3f8d35962 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -271,6 +271,9 @@ private static void usage(final String errorMsg) {
     System.err.println("By default Import will load data directly into HBase. To instead generate");
     System.err.println("HFiles of data to prepare for a bulk data load, pass the option:");
     System.err.println("  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+    System.err.println("For performance consider the following options:\n"
+        + "  -Dmapred.map.tasks.speculative.execution=false\n"
+        + "  -Dmapred.reduce.tasks.speculative.execution=false");
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index aed7846253af..f30c5082e6c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -266,7 +266,10 @@ private static void usage(final String errorMsg) {
       "  -D" + SKIP_LINES_CONF_KEY + "=false - fail if encountering an invalid line\n" +
       "  '-D" + SEPARATOR_CONF_KEY + "=|' - eg separate on pipes instead of tabs\n" +
       "  -D" + TIMESTAMP_CONF_KEY + "=currentTimeAsLong - use the specified timestamp for the import\n" +
-      "  -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of " + DEFAULT_MAPPER.getName() + "\n";
+      "  -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of " + DEFAULT_MAPPER.getName() + "\n" +
+      "For performance consider the following options:\n" +
+      "  -Dmapred.map.tasks.speculative.execution=false\n" +
+      "  -Dmapred.reduce.tasks.speculative.execution=false";
 
     System.err.println(usage);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
index ebf96978e126..5ebe71232086 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
@@ -109,6 +109,7 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
     Job job = new Job(conf, NAME + "_" + tableName);
     job.setJarByClass(RowCounter.class);
     Scan scan = new Scan();
+    scan.setCacheBlocks(false);
     if (startKey != null && !startKey.equals("")) {
       scan.setStartRow(Bytes.toBytes(startKey));
     }
@@ -145,8 +146,11 @@ private static void printUsage(String errorMessage) {
    * Prints usage without error message
    */
   private static void printUsage() {
-    System.err.println("Usage: RowCounter <tablename> " +
+    System.err.println("Usage: RowCounter [options] <tablename> " +
         "[--range=[startKey],[endKey]] [<column1> <column2>...]");
+    System.err.println("For performance consider the following options:\n"
+        + "-Dhbase.client.scanner.caching=100\n"
+        + "-Dmapred.map.tasks.speculative.execution=false");
   }
 
   /**

From 3d1c31e87e9c69ca8b07639f859e8705a27db88a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 23 Mar 2012 21:19:14 +0000
Subject: [PATCH 0063/1540] HBASE-5624 Aborting regionserver when splitting
 region, may cause daughter region not assigned by ServerShutdownHandler

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304604 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/SplitTransaction.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index cfaffd800c49..e6b125b3520c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -338,9 +338,9 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
     if (stopped || stopping) {
       // add 2nd daughter first (see HBASE-4335)
       MetaEditor.addDaughter(server.getCatalogTracker(),
-          b.getRegionInfo(), null);
+          b.getRegionInfo(), services.getServerName());
       MetaEditor.addDaughter(server.getCatalogTracker(),
-          a.getRegionInfo(), null);
+          a.getRegionInfo(), services.getServerName());
       LOG.info("Not opening daughters " +
           b.getRegionInfo().getRegionNameAsString() +
           " and " +

From cd3934b748295a3d5b337009bee373f74a7558c8 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 23 Mar 2012 21:45:24 +0000
Subject: [PATCH 0064/1540] HBASE-5613 ThriftServer getTableRegions does not
 return serverName and port (Scott Chen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304621 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/thrift/ThriftServerRunner.java      | 48 +++++++++++--------
 .../hadoop/hbase/thrift/TestThriftServer.java | 47 ++++++++++--------
 2 files changed, 56 insertions(+), 39 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index dc509d947a29..d0642907f4d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -27,6 +27,7 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -49,6 +50,8 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -357,7 +360,7 @@ private void setupServer() throws Exception {
           tserver.getClass().getName());
     }
 
-    // login the server principal (if using secure Hadoop)   
+    // login the server principal (if using secure Hadoop)
     if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
       String machineName = Strings.domainNamePointerToHostName(
         DNS.getDefaultHost(conf.get("hbase.thrift.dns.interface", "default"),
@@ -383,7 +386,7 @@ private InetAddress getBindAddress(Configuration conf)
     String bindAddressStr = conf.get(BIND_CONF_KEY, DEFAULT_BIND_ADDR);
     return InetAddress.getByName(bindAddressStr);
   }
-  
+
   /**
    * The HBaseHandler is a glue object that connects Thrift RPC calls to the
    * HBase client API primarily defined in the HBaseAdmin and HTable objects.
@@ -568,23 +571,30 @@ public List<ByteBuffer> getTableNames() throws IOError {
     @Override
     public List<TRegionInfo> getTableRegions(ByteBuffer tableName)
     throws IOError {
-      try{
-        List<HRegionInfo> hris =
-            this.admin.getTableRegions(toBytes(tableName));
-        List<TRegionInfo> regions = new ArrayList<TRegionInfo>();
-
-        if (hris != null) {
-          for (HRegionInfo regionInfo : hris){
-            TRegionInfo region = new TRegionInfo();
-            region.startKey = ByteBuffer.wrap(regionInfo.getStartKey());
-            region.endKey = ByteBuffer.wrap(regionInfo.getEndKey());
-            region.id = regionInfo.getRegionId();
-            region.name = ByteBuffer.wrap(regionInfo.getRegionName());
-            region.version = regionInfo.getVersion();
-            regions.add(region);
-          }
+      try {
+        HTable table = getTable(tableName);
+        Map<HRegionInfo, ServerName> regionLocations =
+            table.getRegionLocations();
+        List<TRegionInfo> results = new ArrayList<TRegionInfo>();
+        for (Map.Entry<HRegionInfo, ServerName> entry :
+            regionLocations.entrySet()) {
+          HRegionInfo info = entry.getKey();
+          ServerName serverName = entry.getValue();
+          TRegionInfo region = new TRegionInfo();
+          region.serverName = ByteBuffer.wrap(
+              Bytes.toBytes(serverName.getHostname()));
+          region.port = serverName.getPort();
+          region.startKey = ByteBuffer.wrap(info.getStartKey());
+          region.endKey = ByteBuffer.wrap(info.getEndKey());
+          region.id = info.getRegionId();
+          region.name = ByteBuffer.wrap(info.getRegionName());
+          region.version = info.getVersion();
+          results.add(region);
         }
-        return regions;
+        return results;
+      } catch (TableNotFoundException e) {
+        // Return empty list for non-existing table
+        return Collections.emptyList();
       } catch (IOException e){
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
@@ -925,7 +935,7 @@ public void deleteTable(ByteBuffer in_tableName) throws IOError {
 
     @Override
     public void mutateRow(ByteBuffer tableName, ByteBuffer row,
-        List<Mutation> mutations, Map<ByteBuffer, ByteBuffer> attributes) 
+        List<Mutation> mutations, Map<ByteBuffer, ByteBuffer> attributes)
         throws IOError, IllegalArgument {
       mutateRowTs(tableName, row, mutations, HConstants.LATEST_TIMESTAMP,
                   attributes);
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index be81552d1a32..0cecbb943ca8 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -28,17 +28,19 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Collection;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.filter.ParseFilter;
-import org.junit.experimental.categories.Category;
-import org.junit.Test;
-import org.junit.BeforeClass;
 import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
 import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
 import org.apache.hadoop.hbase.thrift.generated.Hbase;
 import org.apache.hadoop.hbase.thrift.generated.Mutation;
 import org.apache.hadoop.hbase.thrift.generated.TCell;
+import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.metrics.ContextFactory;
@@ -47,9 +49,9 @@
 import org.apache.hadoop.metrics.spi.NoEmitMetricsContext;
 import org.apache.hadoop.metrics.spi.OutputRecord;
 import org.junit.AfterClass;
-import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.conf.Configuration;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 /**
  * Unit testing for ThriftServerRunner.HBaseHandler, a part of the
@@ -58,6 +60,7 @@
 @Category(MediumTests.class)
 public class TestThriftServer {
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final Log LOG = LogFactory.getLog(TestThriftServer.class);
   protected static final int MAXVERSIONS = 3;
 
   private static ByteBuffer asByteBuffer(String i) {
@@ -158,13 +161,13 @@ private static void setupMetricsContext() throws IOException {
   }
 
   private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue)
-      throws Exception { 
-    MetricsContext context = MetricsUtil.getContext( 
-        ThriftMetrics.CONTEXT_NAME); 
-    metrics.doUpdates(context); 
-    OutputRecord record = context.getAllRecords().get( 
-        ThriftMetrics.CONTEXT_NAME).iterator().next(); 
-    assertEquals(expectValue, record.getMetric(name).intValue()); 
+      throws Exception {
+    MetricsContext context = MetricsUtil.getContext(
+        ThriftMetrics.CONTEXT_NAME);
+    metrics.doUpdates(context);
+    OutputRecord record = context.getAllRecords().get(
+        ThriftMetrics.CONTEXT_NAME).iterator().next();
+    assertEquals(expectValue, record.getMetric(name).intValue());
   }
 
   public static void createTestTables(Hbase.Iface handler) throws Exception {
@@ -240,7 +243,7 @@ public void doTestTableMutations() throws Exception {
     TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0);
     assertEquals(rowBname, rowResult2.row);
     assertEquals(valueCname, rowResult2.columns.get(columnAname).value);
-	  assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
+    assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
 
     // Apply some deletes
     handler.deleteAll(tableAname, rowAname, columnBname, null);
@@ -428,26 +431,30 @@ public void doTestGetTableRegions() throws Exception {
 
   public static void doTestGetTableRegions(Hbase.Iface handler)
       throws Exception {
+    assertEquals(handler.getTableNames().size(), 0);
     handler.createTable(tableAname, getColumnDescriptors());
-    int regionCount = handler.getTableRegions(tableAname).size();
+    assertEquals(handler.getTableNames().size(), 1);
+    List<TRegionInfo> regions = handler.getTableRegions(tableAname);
+    int regionCount = regions.size();
     assertEquals("empty table should have only 1 region, " +
             "but found " + regionCount, regionCount, 1);
+    LOG.info("Region found:" + regions.get(0));
     handler.disableTable(tableAname);
     handler.deleteTable(tableAname);
     regionCount = handler.getTableRegions(tableAname).size();
     assertEquals("non-existing table should have 0 region, " +
             "but found " + regionCount, regionCount, 0);
   }
-  
+
   public void doTestFilterRegistration() throws Exception {
     Configuration conf = UTIL.getConfiguration();
-    
+
     conf.set("hbase.thrift.filters", "MyFilter:filterclass");
 
     ThriftServerRunner.registerFilters(conf);
-    
+
     Map<String, String> registeredFilters = ParseFilter.getAllFilters();
-    
+
     assertEquals("filterclass", registeredFilters.get("MyFilter"));
   }
 

From 9025cd34e519f3c0dd834195337db96dc364ddeb Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Fri, 23 Mar 2012 22:32:46 +0000
Subject: [PATCH 0065/1540] HBASE-5190  Limit the IPC queue size based on
 calls' payload size

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304635 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/HBaseServer.java  | 70 +++++++++++++++----
 1 file changed, 58 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index d1039cca17a3..58b1376f2e9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -75,6 +75,8 @@
 import com.google.common.base.Function;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
+import org.cliffc.high_scale_lib.Counter;
+
 /** An abstract IPC service.  IPC calls take a single {@link Writable} as a
  * parameter, and return a {@link Writable} as their value.  A service runs on
  * a port and is defined by a parameter class and a value class.
@@ -95,7 +97,13 @@ public abstract class HBaseServer implements RpcServer {
   /**
    * How many calls/handler are allowed in the queue.
    */
-  private static final int DEFAULT_MAX_QUEUE_SIZE_PER_HANDLER = 10;
+  private static final int DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER = 10;
+
+  /**
+   * The maximum size that we can hold in the IPC queue
+   */
+  private static final int DEFAULT_MAX_CALLQUEUE_SIZE =
+    1024 * 1024 * 1024;
 
   static final int BUFFER_INITIAL_SIZE = 1024;
 
@@ -191,6 +199,7 @@ public static String getRemoteAddress() {
 
   protected Configuration conf;
 
+  private int maxQueueLength;
   private int maxQueueSize;
   protected int socketSendBufferSize;
   protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm
@@ -199,6 +208,7 @@ public static String getRemoteAddress() {
 
   volatile protected boolean running = true;         // true while server runs
   protected BlockingQueue<Call> callQueue; // queued calls
+  protected final Counter callQueueSize = new Counter();
   protected BlockingQueue<Call> priorityCallQueue;
 
   protected int highPriorityLevel;  // what level a high priority call is at
@@ -257,10 +267,11 @@ protected class Call implements Delayable {
     protected Responder responder;
     protected boolean delayReturnValue;           // if the return value should be
                                                   // set at call completion
+    protected long size;                          // size of current call
     protected boolean isError;
 
     public Call(int id, Writable param, Connection connection,
-        Responder responder) {
+        Responder responder, long size) {
       this.id = id;
       this.param = param;
       this.connection = connection;
@@ -269,6 +280,7 @@ public Call(int id, Writable param, Connection connection,
       this.delayResponse = false;
       this.responder = responder;
       this.isError = false;
+      this.size = size;
     }
 
     @Override
@@ -401,6 +413,10 @@ public synchronized boolean isReturnValueDelayed() {
       return this.delayReturnValue;
     }
 
+    public long getSize() {
+      return this.size;
+    }
+
     /**
      * If we have a response, and delay is not set, then respond
      * immediately.  Otherwise, do not respond to client.  This is
@@ -1197,7 +1213,7 @@ private void setupBadVersionResponse(int clientVersion) throws IOException {
         // we return 0 which will keep the socket up -- bad clients, unless
         // they switch to suit the running server -- will fail later doing
         // getProtocolVersion.
-        Call fakeCall =  new Call(0, null, this, responder);
+        Call fakeCall =  new Call(0, null, this, responder, 0);
         // Versions 3 and greater can interpret this exception
         // response in the same manner
         setupResponse(buffer, fakeCall, Status.FATAL,
@@ -1229,9 +1245,23 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
       DataInputStream dis =
         new DataInputStream(new ByteArrayInputStream(buf));
       int id = dis.readInt();                    // try to read an id
+      long callSize = buf.length;
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(" got call #" + id + ", " + callSize + " bytes");
+      }
 
-      if (LOG.isDebugEnabled())
-        LOG.debug(" got call #" + id + ", " + buf.length + " bytes");
+      // Enforcing the call queue size, this triggers a retry in the client
+      if ((callSize + callQueueSize.get()) > maxQueueSize) {
+        final Call callTooBig =
+          new Call(id, null, this, responder, callSize);
+        ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
+        setupResponse(responseBuffer, callTooBig, Status.FATAL, null,
+            IOException.class.getName(),
+            "Call queue is full, is ipc.server.max.callqueue.size too small?");
+        responder.doRespond(callTooBig);
+        return;
+      }
 
       Writable param;
       try {
@@ -1240,7 +1270,8 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
       } catch (Throwable t) {
         LOG.warn("Unable to read call parameters for client " +
                  getHostAddress(), t);
-        final Call readParamsFailedCall = new Call(id, null, this, responder);
+        final Call readParamsFailedCall =
+          new Call(id, null, this, responder, callSize);
         ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
 
         setupResponse(responseBuffer, readParamsFailedCall, Status.FATAL, null,
@@ -1249,7 +1280,8 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
         responder.doRespond(readParamsFailedCall);
         return;
       }
-      Call call = new Call(id, param, this, responder);
+      Call call = new Call(id, param, this, responder, callSize);
+      callQueueSize.add(callSize);
 
       if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
         priorityCallQueue.put(call);
@@ -1353,7 +1385,7 @@ public void run() {
             RequestContext.clear();
           }
           CurCall.set(null);
-
+          callQueueSize.add(call.getSize() * -1);
           // Set the response for undelayed calls and delayed calls with
           // undelayed responses.
           if (!call.isDelayed() || !call.isReturnValueDelayed()) {
@@ -1436,15 +1468,29 @@ protected HBaseServer(String bindAddress, int port,
     this.handlerCount = handlerCount;
     this.priorityHandlerCount = priorityHandlerCount;
     this.socketSendBufferSize = 0;
+
+    // temporary backward compatibility
+    String oldMaxQueueSize = this.conf.get("ipc.server.max.queue.size");
+    if (oldMaxQueueSize == null) {
+      this.maxQueueLength =
+        this.conf.getInt("ipc.server.max.callqueue.length",
+          handlerCount * DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
+    } else {
+      LOG.warn("ipc.server.max.queue.size was renamed " +
+               "ipc.server.max.callqueue.length, " +
+               "please update your configuration");
+      this.maxQueueLength = Integer.getInteger(oldMaxQueueSize);
+    }
+
     this.maxQueueSize =
-      this.conf.getInt("ipc.server.max.queue.size",
-        handlerCount * DEFAULT_MAX_QUEUE_SIZE_PER_HANDLER);
+      this.conf.getInt("ipc.server.max.callqueue.size",
+        DEFAULT_MAX_CALLQUEUE_SIZE);
      this.readThreads = conf.getInt(
         "ipc.server.read.threadpool.size",
         10);
-    this.callQueue  = new LinkedBlockingQueue<Call>(maxQueueSize);
+    this.callQueue  = new LinkedBlockingQueue<Call>(maxQueueLength);
     if (priorityHandlerCount > 0) {
-      this.priorityCallQueue = new LinkedBlockingQueue<Call>(maxQueueSize); // TODO hack on size
+      this.priorityCallQueue = new LinkedBlockingQueue<Call>(maxQueueLength); // TODO hack on size
     } else {
       this.priorityCallQueue = null;
     }

From 803733d7d161c9140feddc467b8d7a5acb2f2c72 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 23 Mar 2012 23:54:13 +0000
Subject: [PATCH 0066/1540] HBASE-5128 [uber hbck] Online automated repair of
 table integrity and region consistency problems

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304666 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/ipc/HMasterInterface.java    |    1 -
 .../hbase/master/AssignmentManager.java       |    5 +-
 .../apache/hadoop/hbase/master/HMaster.java   |    7 +-
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 1838 ++++++++++++++---
 .../hadoop/hbase/util/HBaseFsckRepair.java    |  158 +-
 .../hbase/util/hbck/OfflineMetaRepair.java    |   13 +-
 .../hadoop/hbase/HBaseTestingUtility.java     |   11 +-
 .../hadoop/hbase/util/TestHBaseFsck.java      |  598 +++++-
 .../hbase/util/TestHBaseFsckComparator.java   |    8 -
 .../hbase/util/hbck/HbckTestingUtil.java      |   20 +-
 .../util/hbck/TestOfflineMetaRebuildBase.java |    2 +-
 .../util/hbck/TestOfflineMetaRebuildHole.java |    3 +-
 .../hbck/TestOfflineMetaRebuildOverlap.java   |    2 +-
 13 files changed, 2236 insertions(+), 430 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index be83d64d3bb3..a6276cbfbb8c 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -218,7 +218,6 @@ public void assign(final byte [] regionName, final boolean force)
   public void unassign(final byte [] regionName, final boolean force)
   throws IOException;
 
-  
   /**
    * Offline a region from the assignment manager's in-memory state.  The
    * region should be in a closed state and there will be no attempt to
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 85b56963567e..a2eb6d6ebf8a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1029,8 +1029,9 @@ private void handleHBCK(RegionTransitionData data) {
           regionInfo = regionState.getRegion();
         } else {
           try {
-            regionInfo = MetaReader.getRegion(catalogTracker,
-                data.getRegionName()).getFirst();
+            byte[] name = data.getRegionName();
+            Pair<HRegionInfo, ServerName> p = MetaReader.getRegion(catalogTracker, name);
+            regionInfo = p.getFirst();
           } catch (IOException e) {
             LOG.info("Exception reading META doing HBCK repair operation", e);
             return;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 46bd970c4661..cbb20918bd48 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1821,18 +1821,17 @@ public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
   public double getAverageLoad() {
     return this.assignmentManager.getAverageLoad();
   }
-  
+
   /**
    * Special method, only used by hbck.
    */
   @Override
-  public void offline(final byte[] regionName) 
-  throws IOException {
+  public void offline(final byte[] regionName) throws IOException {
     Pair<HRegionInfo, ServerName> pair =
       MetaReader.getRegion(this.catalogTracker, regionName);
     if (pair == null) throw new UnknownRegionException(Bytes.toStringBinary(regionName));
     HRegionInfo hri = pair.getFirst();
-    this.assignmentManager.regionOffline(hri);    
+    this.assignmentManager.regionOffline(hri);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index e55b906cba3d..c49181c856b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1,6 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -49,78 +47,163 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.MetaScanner;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
+import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.io.MultipleIOException;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.TreeMultimap;
 
 /**
- * Check consistency among the in-memory states of the master and the
- * region server(s) and the state of data in HDFS.
+ * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
+ * table integrity problems in a corrupted HBase.
+ * <p>
+ * Region consistency checks verify that .META., region deployment on region
+ * servers and the state of data in HDFS (.regioninfo files) all are in
+ * accordance.
+ * <p>
+ * Table integrity checks verify that all possible row keys resolve to exactly
+ * one region of a table.  This means there are no individual degenerate
+ * or backwards regions; no holes between regions; and that there are no
+ * overlapping regions.
+ * <p>
+ * The general repair strategy works in two phases:
+ * <ol>
+ * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
+ * <li> Repair Region Consistency with .META. and assignments
+ * </ol>
+ * <p>
+ * For table integrity repairs, the tables' region directories are scanned
+ * for .regioninfo files.  Each table's integrity is then verified.  If there
+ * are any orphan regions (regions with no .regioninfo files) or holes, new
+ * regions are fabricated.  Backwards regions are sidelined as well as empty
+ * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
+ * a new region is created and all data is merged into the new region.
+ * <p>
+ * Table integrity repairs deal solely with HDFS and could potentially be done
+ * offline -- the hbase region servers or master do not need to be running.
+ * This phase can eventually be used to completely reconstruct the META table in
+ * an offline fashion.
+ * <p>
+ * Region consistency requires three conditions -- 1) valid .regioninfo file
+ * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
+ * and 3) a region is deployed only at the regionserver that was assigned to
+ * with proper state in the master.
+ * <p>
+ * Region consistency repairs require hbase to be online so that hbck can
+ * contact the HBase master and region servers.  The hbck#connect() method must
+ * first be called successfully.  Much of the region consistency information
+ * is transient and less risky to repair.
+ * <p>
+ * If hbck is run from the command line, there are a handful of arguments that
+ * can be used to limit the kinds of repairs hbck will do.  See the code in
+ * {@link #printUsageAndExit()} for more details.
  */
 public class HBaseFsck {
   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
-
+  private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
   private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
+  private static boolean rsSupportsOffline = true;
+  private static final int DEFAULT_MAX_MERGE = 5;
 
+  /**********************
+   * Internal resources
+   **********************/
   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
   private Configuration conf;
-
   private ClusterStatus status;
   private HConnection connection;
-
-  private TreeMap<String, HbckInfo> regionInfo = new TreeMap<String, HbckInfo>();
-  private TreeMap<String, TInfo> tablesInfo = new TreeMap<String, TInfo>();
-  private TreeSet<byte[]> disabledTables =
-    new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
-  ErrorReporter errors = new PrintingErrorReporter();
-
+  private HBaseAdmin admin;
+  private HTable meta;
+  private ThreadPoolExecutor executor; // threads to retrieve data from regionservers
+  private int numThreads = MAX_NUM_THREADS;
+  private long startMillis = System.currentTimeMillis();
+
+  /***********
+   * Options
+   ***********/
   private static boolean details = false; // do we display the full report
   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
-  private boolean fix = false; // do we want to try fixing the errors?
-  private boolean rerun = false; // if we tried to fix something rerun hbck
+  private boolean fixAssignments = false; // fix assignment errors?
+  private boolean fixMeta = false; // fix meta errors?
+  private boolean fixHdfsHoles = false; // fix fs holes?
+  private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
+  private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
+
+  // limit fixes to listed tables, if empty atttempt to fix all
+  private List<byte[]> tablesToFix = new ArrayList<byte[]>();
+  private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
+
+  private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
   private boolean checkMetaOnly = false;
-  
+
+  /*********
+   * State
+   *********/
+  private ErrorReporter errors = new PrintingErrorReporter();
+  int fixes = 0;
+
+  /**
+   * This map contains the state of all hbck items.  It maps from encoded region
+   * name to HbckInfo structure.  The information contained in HbckInfo is used
+   * to detect and correct consistency (hdfs/meta/deployment) problems.
+   */
+  private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
+  private TreeSet<byte[]> disabledTables =
+    new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
   // Empty regioninfo qualifiers in .META.
   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
-  private HBaseAdmin admin;
 
-  ThreadPoolExecutor executor; // threads to retrieve data from regionservers
+  /**
+   * This map from Tablename -> TableInfo contains the structures necessary to
+   * detect table consistency problems (holes, dupes, overlaps).  It is sorted
+   * to prevent dupes.
+   */
+  private TreeMap<String, TableInfo> tablesInfo = new TreeMap<String, TableInfo>();
+
+  /**
+   * When initially looking at HDFS, we attempt to find any orphaned data.
+   */
+  private List<HbckInfo> orphanHdfsDirs = new ArrayList<HbckInfo>();
 
   /**
    * Constructor
    *
    * @param conf Configuration object
    * @throws MasterNotRunningException if the master is not running
-   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
+   * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
    */
   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
       ZooKeeperConnectionException, IOException {
@@ -133,85 +216,33 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
     executor.allowCoreThreadTimeOut(true);
   }
 
-  public void connect() throws MasterNotRunningException,
-      ZooKeeperConnectionException {
+  /**
+   * To repair region consistency, one must call connect() in order to repair
+   * online state.
+   */
+  public void connect() throws IOException {
     admin = new HBaseAdmin(conf);
+    meta = new HTable(conf, HConstants.META_TABLE_NAME);
     status = admin.getMaster().getClusterStatus();
     connection = admin.getConnection();
   }
 
   /**
-   * Contacts the master and prints out cluster-wide information
-   * @throws IOException if a remote or network exception occurs
-   * @return 0 on success, non-zero on failure
-   * @throws KeeperException
-   * @throws InterruptedException
+   * Get deployed regions according to the region servers.
    */
-  public int doWork() throws IOException, KeeperException, InterruptedException {
-    // print hbase server version
-    errors.print("Version: " + status.getHBaseVersion());
-
-    // Make sure regionInfo is empty before starting
-    regionInfo.clear();
-    tablesInfo.clear();
-    emptyRegionInfoQualifiers.clear();
-    disabledTables.clear();
-    errors.clear();
-
-    // get a list of all regions from the master. This involves
-    // scanning the META table
-    if (!recordRootRegion()) {
-      // Will remove later if we can fix it
-      errors.reportError("Encountered fatal error. Exiting...");
-      return -1;
-    }
-    
-    getMetaEntries();
-
-    // Check if .META. is found only once and in the right place
-    if (!checkMetaEntries()) {
-      // Will remove later if we can fix it
-      errors.reportError("Encountered fatal error. Exiting...");
-      return -1;
-    }
-
-    // get a list of all tables that have not changed recently.
-    if (!checkMetaOnly) {
-      AtomicInteger numSkipped = new AtomicInteger(0);
-      HTableDescriptor[] allTables = getTables(numSkipped);
-      errors.print("Number of Tables: " +
-          (allTables == null ? 0 : allTables.length));
-      if (details) {
-        if (numSkipped.get() > 0) {
-          errors.detail("Number of Tables in flux: " + numSkipped.get());
-        }
-        if (allTables != null && allTables.length > 0) {
-          for (HTableDescriptor td : allTables) {
-          String tableName = td.getNameAsString();
-          errors.detail("  Table: " + tableName + "\t" +
-                             (td.isReadOnly() ? "ro" : "rw") + "\t" +
-                             (td.isRootRegion() ? "ROOT" :
-                              (td.isMetaRegion() ? "META" : "    ")) + "\t" +
-                             " families: " + td.getFamilies().size());
-          }
-        }
-      }
-    }
-    
+  private void loadDeployedRegions() throws IOException, InterruptedException {
     // From the master, get a list of all known live region servers
     Collection<ServerName> regionServers = status.getServers();
-    errors.print("Number of live region servers: " +
-                       regionServers.size());
+    errors.print("Number of live region servers: " + regionServers.size());
     if (details) {
       for (ServerName rsinfo: regionServers) {
-        errors.print("  " + rsinfo);
+        errors.print("  " + rsinfo.getServerName());
       }
     }
 
     // From the master, get a list of all dead region servers
     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
-    errors.print("Number of dead region servers: " +
-                       deadRegionServers.size());
+    errors.print("Number of dead region servers: " + deadRegionServers.size());
     if (details) {
       for (ServerName name: deadRegionServers) {
         errors.print("  " + name);
@@ -232,100 +263,390 @@ public int doWork() throws IOException, KeeperException, InterruptedException {
 
     // Determine what's deployed
     processRegionServers(regionServers);
+  }
 
-    // Determine what's on HDFS
-    checkHdfs();
+  /**
+   * Clear the current state of hbck.
+   */
+  private void clearState() {
+    // Make sure regionInfo is empty before starting
+    fixes = 0;
+    regionInfoMap.clear();
+    emptyRegionInfoQualifiers.clear();
+    disabledTables.clear();
+    errors.clear();
+    tablesInfo.clear();
+    orphanHdfsDirs.clear();
+  }
 
-    // Empty cells in .META.?
-    errors.print("Number of empty REGIONINFO_QUALIFIER rows in .META.: " +
-      emptyRegionInfoQualifiers.size());
-    if (details) {
-      for (Result r: emptyRegionInfoQualifiers) {
-        errors.print("  " + r);
+  /**
+   * This repair method analyzes hbase data in hdfs and repairs it to satisfy
+   * the table integrity rules.  HBase doesn't need to be online for this
+   * operation to work.
+   */
+  public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
+    // Initial pass to fix orphans.
+    if (shouldFixHdfsOrphans() || shouldFixHdfsHoles() || shouldFixHdfsOverlaps()) {
+      LOG.info("Loading regioninfos HDFS");
+      // if nothing is happening this should always complete in two iterations.
+      int maxIterations = conf.getInt("hbase.hbck.integrityrepair.iterations.max", 3);
+      int curIter = 0;
+      do {
+        clearState(); // clears hbck state and reset fixes to 0 and.
+        // repair what's on HDFS
+        restoreHdfsIntegrity();
+        curIter++;// limit the number of iterations.
+      } while (fixes > 0 && curIter <= maxIterations);
+
+      // Repairs should be done in the first iteration and verification in the second.
+      // If there are more than 2 passes, something funny has happened.
+      if (curIter > 2) {
+        if (curIter == maxIterations) {
+          LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
+              + "Tables integrity may not be fully repaired!");
+        } else {
+          LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
+        }
       }
     }
+  }
+
+  /**
+   * This repair method requires the cluster to be online since it contacts
+   * region servers and the masters.  It makes each region's state in HDFS, in
+   * .META., and deployments consistent.
+   *
+   * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
+   * error.  If 0, we have a clean hbase.
+   */
+  public int onlineConsistencyRepair() throws IOException, KeeperException,
+    InterruptedException {
+    clearState();
+
+    LOG.info("Loading regionsinfo from the .META. table");
+    boolean success = loadMetaEntries();
+    if (!success) return -1;
+
+    // Check if .META. is found only once and in the right place
+    if (!checkMetaRegion()) {
+      // Will remove later if we can fix it
+      errors.reportError("Encountered fatal error. Exiting...");
+      return -2;
+    }
+
+    // get a list of all tables that have not changed recently.
+    if (!checkMetaOnly) {
+      reportTablesInFlux();
+    }
+
+    // get regions according to what is online on each RegionServer
+    loadDeployedRegions();
+
+    // load regiondirs and regioninfos from HDFS
+    loadHdfsRegionDirs();
+    loadHdfsRegionInfos();
+
+    // Empty cells in .META.?
+    reportEmptyMetaCells();
 
     // Get disabled tables from ZooKeeper
     loadDisabledTables();
 
-    // Check consistency
-    checkConsistency();
+    // Check and fix consistency
+    checkAndFixConsistency();
 
-    // Check integrity
+    // Check integrity (does not fix)
     checkIntegrity();
+    return errors.getErrorList().size();
+  }
 
-    // Print table summary
-    printTableSummary();
+  /**
+   * Contacts the master and prints out cluster-wide information
+   * @return 0 on success, non-zero on failure
+   */
+  public int onlineHbck() throws IOException, KeeperException, InterruptedException {
+    // print hbase server version
+    errors.print("Version: " + status.getHBaseVersion());
+    offlineHdfsIntegrityRepair();
 
+    // turn the balancer off
+    boolean oldBalancer = admin.balanceSwitch(false);
+
+    onlineConsistencyRepair();
+
+    admin.balanceSwitch(oldBalancer);
+
+    // Print table summary
+    printTableSummary(tablesInfo);
     return errors.summarize();
   }
 
+  /**
+   * Iterates through the list of all orphan/invalid regiondirs.
+   */
+  private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
+    for (HbckInfo hi : orphanHdfsDirs) {
+      LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
+      adoptHdfsOrphan(hi);
+    }
+  }
+
+  /**
+   * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
+   * these orphans by creating a new region, and moving the column families,
+   * recovered edits, HLogs, into the new region dir.  We determine the region
+   * startkey and endkeys by looking at all of the hfiles inside the column
+   * families to identify the min and max keys. The resulting region will
+   * likely violate table integrity but will be dealt with by merging
+   * overlapping regions.
+   */
+  private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
+    Path p = hi.getHdfsRegionDir();
+    FileSystem fs = p.getFileSystem(conf);
+    FileStatus[] dirs = fs.listStatus(p);
+
+    String tableName = Bytes.toString(hi.getTableName());
+    TableInfo tableInfo = tablesInfo.get(tableName);
+    Preconditions.checkNotNull("Table " + tableName + "' not present!", tableInfo);
+    HTableDescriptor template = tableInfo.getHTD();
+
+    // find min and max key values
+    Pair<byte[],byte[]> orphanRegionRange = null;
+    for (FileStatus cf : dirs) {
+      String cfName= cf.getPath().getName();
+      // TODO Figure out what the special dirs are
+      if (cfName.startsWith(".") || cfName.equals("splitlog")) continue;
+
+      FileStatus[] hfiles = fs.listStatus(cf.getPath());
+      for (FileStatus hfile : hfiles) {
+        byte[] start, end;
+        HFile.Reader hf = null;
+        try {
+          CacheConfig cacheConf = new CacheConfig(conf);
+          hf = HFile.createReader(fs, hfile.getPath(), cacheConf);
+          hf.loadFileInfo();
+          KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
+          start = startKv.getRow();
+          KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
+          end = endKv.getRow();
+        } catch (IOException ioe) {
+          LOG.warn("Problem reading orphan file " + hfile + ", skipping");
+          continue;
+        } catch (NullPointerException ioe) {
+          LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
+          continue;
+        } finally {
+          if (hf != null) {
+            hf.close();
+          }
+        }
+
+        // expand the range to include the range of all hfiles
+        if (orphanRegionRange == null) {
+          // first range
+          orphanRegionRange = new Pair<byte[], byte[]>(start, end);
+        } else {
+          // TODO add test
+
+          // expand range only if the hfile is wider.
+          if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
+            orphanRegionRange.setFirst(start);
+          }
+          if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
+            orphanRegionRange.setSecond(end);
+          }
+        }
+      }
+    }
+    if (orphanRegionRange == null) {
+      LOG.warn("No data in dir " + p + ", sidelining data");
+      fixes++;
+      sidelineRegionDir(fs, hi);
+      return;
+    }
+    LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
+        Bytes.toString(orphanRegionRange.getSecond()) + ")");
+
+    // create new region on hdfs.  move data into place.
+    HRegionInfo hri = new HRegionInfo(template.getName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
+    LOG.info("Creating new region : " + hri);
+    HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, hri, template);
+    Path target = region.getRegionDir();
+
+    // rename all the data to new region
+    mergeRegionDirs(target, hi);
+    fixes++;
+  }
+
+  /**
+   * This method determines if there are table integrity errors in HDFS.  If
+   * there are errors and the appropriate "fix" options are enabled, the method
+   * will first correct orphan regions making them into legit regiondirs, and
+   * then reload to merge potentially overlapping regions.
+   *
+   * @return number of table integrity errors found
+   */
+  private int restoreHdfsIntegrity() throws IOException, InterruptedException {
+    // Determine what's on HDFS
+    LOG.info("Loading HBase regioninfo from HDFS...");
+    loadHdfsRegionDirs(); // populating regioninfo table.
+
+    int errs = errors.getErrorList().size();
+    // First time just get suggestions.
+    tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
+    checkHdfsIntegrity(false, false);
+
+    if (errors.getErrorList().size() == errs) {
+      LOG.info("No integrity errors.  We are done with this phase. Glorious.");
+      return 0;
+    }
+
+    if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
+      adoptHdfsOrphans(orphanHdfsDirs);
+      // TODO optimize by incrementally adding instead of reloading.
+    }
+
+    // Make sure there are no holes now.
+    if (shouldFixHdfsHoles()) {
+      clearState(); // this also resets # fixes.
+      loadHdfsRegionDirs();
+      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
+      tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
+    }
+
+    // Now we fix overlaps
+    if (shouldFixHdfsOverlaps()) {
+      // second pass we fix overlaps.
+      clearState(); // this also resets # fixes.
+      loadHdfsRegionDirs();
+      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
+      tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
+    }
+
+    return errors.getErrorList().size();
+  }
+
+  /**
+   * TODO -- need to add tests for this.
+   */
+  private void reportEmptyMetaCells() {
+    errors.print("Number of empty REGIONINFO_QUALIFIER rows in .META.: " +
+      emptyRegionInfoQualifiers.size());
+    if (details) {
+      for (Result r: emptyRegionInfoQualifiers) {
+        errors.print("  " + r);
+      }
+    }
+  }
+
+  /**
+   * TODO -- need to add tests for this.
+   */
+  private void reportTablesInFlux() {
+    AtomicInteger numSkipped = new AtomicInteger(0);
+    HTableDescriptor[] allTables = getTables(numSkipped);
+    errors.print("Number of Tables: " + allTables.length);
+    if (details) {
+      if (numSkipped.get() > 0) {
+        errors.detail("Number of Tables in flux: " + numSkipped.get());
+      }
+      for (HTableDescriptor td : allTables) {
+        String tableName = td.getNameAsString();
+        errors.detail("  Table: " + tableName + "\t" +
+                           (td.isReadOnly() ? "ro" : "rw") + "\t" +
+                           (td.isRootRegion() ? "ROOT" :
+                            (td.isMetaRegion() ? "META" : "    ")) + "\t" +
+                           " families: " + td.getFamilies().size());
+      }
+    }
+  }
+
   public ErrorReporter getErrors() {
     return errors;
   }
 
   /**
-   * Populate a specific hbi from regioninfo on file system.
+   * Read the .regioninfo file from the file system.  If there is no
+   * .regioninfo, add it to the orphan hdfs region list.
    */
-  private void loadMetaEntry(HbckInfo hbi) throws IOException {
-    Path regionDir = hbi.foundRegionDir.getPath();
+  private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
+    Path regionDir = hbi.getHdfsRegionDir();
+    if (regionDir == null) {
+      LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
+      return;
+    }
     Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
-    FileSystem fs = FileSystem.get(conf);
+    FileSystem fs = regioninfo.getFileSystem(conf);
+
     FSDataInputStream in = fs.open(regioninfo);
-    byte[] tableName = Bytes.toBytes(hbi.hdfsTableName);
-    HRegionInfo hri = new HRegionInfo(tableName);
+    HRegionInfo hri = new HRegionInfo();
     hri.readFields(in);
     in.close();
     LOG.debug("HRegionInfo read: " + hri.toString());
-    hbi.metaEntry = new MetaEntry(hri, null,
-        hbi.foundRegionDir.getModificationTime());
+    hbi.hdfsEntry.hri = hri;
   }
 
-  public static class RegionInfoLoadException extends IOException {
+  /**
+   * Exception thrown when a integrity repair operation fails in an
+   * unresolvable way.
+   */
+  public static class RegionRepairException extends IOException {
     private static final long serialVersionUID = 1L;
     final IOException ioe;
-    public RegionInfoLoadException(String s, IOException ioe) {
+    public RegionRepairException(String s, IOException ioe) {
       super(s);
       this.ioe = ioe;
     }
   }
 
   /**
-   * Populate hbi's from regionInfos loaded from file system. 
+   * Populate hbi's from regionInfos loaded from file system.
    */
-  private void loadTableInfo() throws IOException {
-    List<IOException> ioes = new ArrayList<IOException>();
+  private TreeMap<String, TableInfo> loadHdfsRegionInfos() throws IOException {
+    tablesInfo.clear(); // regenerating the data
     // generate region split structure
-    for (HbckInfo hbi : regionInfo.values()) {
+    for (HbckInfo hbi : regionInfoMap.values()) {
+
       // only load entries that haven't been loaded yet.
-      if (hbi.metaEntry == null) {
+      if (hbi.getHdfsHRI() == null) {
         try {
-          loadMetaEntry(hbi);
+          loadHdfsRegioninfo(hbi);
         } catch (IOException ioe) {
-          String msg = "Unable to load region info for table " + hbi.hdfsTableName
-            + "!  It may be an invalid format or version file.  You may want to "
-            + "remove " + hbi.foundRegionDir.getPath()
-            + " region from hdfs and retry.";
-          errors.report(msg);
-          LOG.error(msg, ioe);
-          ioes.add(new RegionInfoLoadException(msg, ioe));
+          String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
+            + Bytes.toString(hbi.getTableName()) + " in hdfs dir "
+            + hbi.getHdfsRegionDir()
+            + "!  It may be an invalid format or version file.  Treating as "
+            + "an orphaned regiondir.";
+          errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
+          debugLsr(hbi.getHdfsRegionDir());
+          orphanHdfsDirs.add(hbi);
           continue;
         }
       }
 
       // get table name from hdfs, populate various HBaseFsck tables.
-      String tableName = hbi.hdfsTableName;
-      TInfo modTInfo = tablesInfo.get(tableName);
+      String tableName = Bytes.toString(hbi.getTableName());
+      if (tableName == null) {
+        // There was an entry in META not in the HDFS?
+        LOG.warn("tableName was null for: " + hbi);
+        continue;
+      }
+
+      TableInfo modTInfo = tablesInfo.get(tableName);
       if (modTInfo == null) {
-        modTInfo = new TInfo(tableName);
+        // only executed once per table.
+        modTInfo = new TableInfo(tableName);
+        Path hbaseRoot = new Path(conf.get(HConstants.HBASE_DIR));
+        HTableDescriptor htd =
+          FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
+              hbaseRoot, tableName);
+        modTInfo.htds.add(htd);
       }
       modTInfo.addRegionInfo(hbi);
       tablesInfo.put(tableName, modTInfo);
     }
 
-    if (ioes.size() != 0) {
-      throw MultipleIOException.createIOException(ioes);
-    }
+    return tablesInfo;
   }
 
   /**
@@ -360,10 +681,10 @@ private HRegion createNewRootAndMeta() throws IOException {
    * 
    * @return An array list of puts to do in bulk, null if tables have problems
    */
-  private ArrayList<Put> generatePuts() throws IOException {
+  private ArrayList<Put> generatePuts(TreeMap<String, TableInfo> tablesInfo) throws IOException {
     ArrayList<Put> puts = new ArrayList<Put>();
     boolean hasProblems = false;
-    for (Entry<String, TInfo> e : tablesInfo.entrySet()) {
+    for (Entry<String, TableInfo> e : tablesInfo.entrySet()) {
       String name = e.getKey();
 
       // skip "-ROOT-" and ".META."
@@ -372,7 +693,7 @@ private ArrayList<Put> generatePuts() throws IOException {
         continue;
       }
 
-      TInfo ti = e.getValue();
+      TableInfo ti = e.getValue();
       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
           .entrySet()) {
         Collection<HbckInfo> his = spl.getValue();
@@ -387,7 +708,7 @@ private ArrayList<Put> generatePuts() throws IOException {
 
         // add the row directly to meta.
         HbckInfo hi = his.iterator().next();
-        HRegionInfo hri = hi.metaEntry;
+        HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
         Put p = new Put(hri.getRegionName());
         p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
             Writables.getBytes(hri));
@@ -400,57 +721,68 @@ private ArrayList<Put> generatePuts() throws IOException {
   /**
    * Suggest fixes for each table
    */
-  private void suggestFixes(TreeMap<String, TInfo> tablesInfo) {
-    for (TInfo tInfo : tablesInfo.values()) {
-      tInfo.checkRegionChain();
+  private void suggestFixes(TreeMap<String, TableInfo> tablesInfo) throws IOException {
+    for (TableInfo tInfo : tablesInfo.values()) {
+      TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
+      tInfo.checkRegionChain(handler);
     }
   }
 
-
   /**
    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
    * settings passed into hbck constructor to point to a particular fs/dir.
    * 
+   * @param fix flag that determines if method should attempt to fix holes
    * @return true if successful, false if attempt failed.
    */
-  public boolean rebuildMeta() throws IOException, InterruptedException {
+  public boolean rebuildMeta(boolean fix) throws IOException,
+      InterruptedException {
+
     // TODO check to make sure hbase is offline. (or at least the table
     // currently being worked on is off line)
 
     // Determine what's on HDFS
     LOG.info("Loading HBase regioninfo from HDFS...");
-    checkHdfs(); // populating regioninfo table.
-    loadTableInfo(); // update tableInfos based on region info in fs.
+    loadHdfsRegionDirs(); // populating regioninfo table.
 
-    LOG.info("Checking HBase region split map from HDFS data...");
     int errs = errors.getErrorList().size();
-    for (TInfo tInfo : tablesInfo.values()) {
-      if (!tInfo.checkRegionChain()) {
-        // should dump info as well.
-        errors.report("Found inconsistency in table " + tInfo.getName());
-      }
-    }
+    tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
+    checkHdfsIntegrity(false, false);
 
     // make sure ok.
     if (errors.getErrorList().size() != errs) {
-      suggestFixes(tablesInfo);
-
-      // Not ok, bail out.
-      return false;
+      // While in error state, iterate until no more fixes possible
+      while(true) {
+        fixes = 0;
+        suggestFixes(tablesInfo);
+        errors.clear();
+        loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
+        checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
+
+        int errCount = errors.getErrorList().size();
+
+        if (fixes == 0) {
+          if (errCount > 0) {
+            return false; // failed to fix problems.
+          } else {
+            break; // no fixes and no problems? drop out and fix stuff!
+          }
+        }
+      }
     }
 
     // we can rebuild, move old root and meta out of the way and start
     LOG.info("HDFS regioninfo's seems good.  Sidelining old .META.");
     sidelineOldRootAndMeta();
-    
+
     LOG.info("Creating new .META.");
     HRegion meta = createNewRootAndMeta();
 
     // populate meta
-    List<Put> puts = generatePuts();
+    List<Put> puts = generatePuts(tablesInfo);
     if (puts == null) {
       LOG.fatal("Problem encountered when creating new .META. entries.  " +
-        "You may need to restore the previously sidlined -ROOT- and .META.");
+        "You may need to restore the previously sidelined -ROOT- and .META.");
       return false;
     }
     meta.put(puts.toArray(new Put[0]));
@@ -460,13 +792,113 @@ public boolean rebuildMeta() throws IOException, InterruptedException {
     return true;
   }
 
-  void sidelineTable(FileSystem fs, byte[] table, Path hbaseDir, 
+  private TreeMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
+      boolean fixOverlaps) throws IOException {
+    LOG.info("Checking HBase region split map from HDFS data...");
+    for (TableInfo tInfo : tablesInfo.values()) {
+      TableIntegrityErrorHandler handler;
+      if (fixHoles || fixOverlaps) {
+        if (shouldFixTable(Bytes.toBytes(tInfo.getName()))) {
+          handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
+              fixHoles, fixOverlaps);
+        } else {
+          LOG.info("Table " + tInfo.getName() + " is not in the include table " +
+            "list.  Just suggesting fixes.");
+          handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
+        }
+      } else {
+        handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
+      }
+      if (!tInfo.checkRegionChain(handler)) {
+        // should dump info as well.
+        errors.report("Found inconsistency in table " + tInfo.getName());
+      }
+    }
+    return tablesInfo;
+  }
+
+  private Path getSidelineDir() throws IOException {
+    Path hbaseDir = FSUtils.getRootDir(conf);
+    Path backupDir = new Path(hbaseDir.getParent(), hbaseDir.getName() + "-"
+        + startMillis);
+    return backupDir;
+  }
+
+  /**
+   * Sideline a region dir (instead of deleting it)
+   */
+  void sidelineRegionDir(FileSystem fs, HbckInfo hi)
+    throws IOException {
+    String tableName = Bytes.toString(hi.getTableName());
+    Path regionDir = hi.getHdfsRegionDir();
+
+    if (!fs.exists(regionDir)) {
+      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
+      return;
+    }
+
+    Path sidelineTableDir= new Path(getSidelineDir(), tableName);
+    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
+    fs.mkdirs(sidelineRegionDir);
+    boolean success = false;
+    FileStatus[] cfs =  fs.listStatus(regionDir);
+    if (cfs == null) {
+      LOG.info("Region dir is empty: " + regionDir);
+    } else {
+      for (FileStatus cf : cfs) {
+        Path src = cf.getPath();
+        Path dst =  new Path(sidelineRegionDir, src.getName());
+        if (fs.isFile(src)) {
+          // simple file
+          success = fs.rename(src, dst);
+          if (!success) {
+            String msg = "Unable to rename file " + src +  " to " + dst;
+            LOG.error(msg);
+            throw new IOException(msg);
+          }
+          continue;
+        }
+
+        // is a directory.
+        fs.mkdirs(dst);
+
+        LOG.info("Sidelining files from " + src + " into containing region " + dst);
+        // FileSystem.rename is inconsistent with directories -- if the
+        // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
+        // it moves the src into the dst dir resulting in (foo/a/b).  If
+        // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
+        for (FileStatus hfile : fs.listStatus(src)) {
+          success = fs.rename(hfile.getPath(), dst);
+          if (!success) {
+            String msg = "Unable to rename file " + src +  " to " + dst;
+            LOG.error(msg);
+            throw new IOException(msg);
+          }
+        }
+        LOG.debug("Sideline directory contents:");
+        debugLsr(sidelineRegionDir);
+      }
+    }
+
+    LOG.info("Removing old region dir: " + regionDir);
+    success = fs.delete(regionDir, true);
+    if (!success) {
+      String msg = "Unable to delete dir " + regionDir;
+      LOG.error(msg);
+      throw new IOException(msg);
+    }
+  }
+
+  /**
+   * Side line an entire table.
+   */
+  void sidelineTable(FileSystem fs, byte[] table, Path hbaseDir,
       Path backupHbaseDir) throws IOException {
     String tableName = Bytes.toString(table);
     Path tableDir = new Path(hbaseDir, tableName);
     if (fs.exists(tableDir)) {
       Path backupTableDir= new Path(backupHbaseDir, tableName);
-      boolean success = fs.rename(tableDir, backupTableDir); 
+      boolean success = fs.rename(tableDir, backupTableDir);
       if (!success) {
         throw new IOException("Failed to move  " + tableName + " from " 
             +  tableDir.getName() + " to " + backupTableDir.getName());
@@ -475,18 +907,16 @@ void sidelineTable(FileSystem fs, byte[] table, Path hbaseDir,
       LOG.info("No previous " + tableName +  " exists.  Continuing.");
     }
   }
-  
+
   /**
    * @return Path to backup of original directory
-   * @throws IOException
    */
   Path sidelineOldRootAndMeta() throws IOException {
     // put current -ROOT- and .META. aside.
     Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
     FileSystem fs = hbaseDir.getFileSystem(conf);
-    long now = System.currentTimeMillis();
     Path backupDir = new Path(hbaseDir.getParent(), hbaseDir.getName() + "-"
-        + now);
+        + startMillis);
     fs.mkdirs(backupDir);
 
     sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir);
@@ -535,9 +965,6 @@ public Void connect(HConnection connection) throws IOException {
 
   /**
    * Check if the specified region's table is disabled.
-   * @throws ZooKeeperConnectionException
-   * @throws IOException
-   * @throws KeeperException
    */
   private boolean isTableDisabled(HRegionInfo regionInfo) {
     return disabledTables.contains(regionInfo.getTableName());
@@ -545,9 +972,9 @@ private boolean isTableDisabled(HRegionInfo regionInfo) {
 
   /**
    * Scan HDFS for all regions, recording their information into
-   * regionInfo
+   * regionInfoMap
    */
-  public void checkHdfs() throws IOException, InterruptedException {
+  public void loadHdfsRegionDirs() throws IOException, InterruptedException {
     Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
     FileSystem fs = rootDir.getFileSystem(conf);
 
@@ -576,19 +1003,21 @@ public void checkHdfs() throws IOException, InterruptedException {
     }
 
     // level 1:  <HBASE_DIR>/*
-    WorkItemHdfsDir[] dirs = new WorkItemHdfsDir[tableDirs.size()];  
+    WorkItemHdfsDir[] dirs = new WorkItemHdfsDir[tableDirs.size()];
     int num = 0;
     for (FileStatus tableDir : tableDirs) {
-      dirs[num] = new WorkItemHdfsDir(this, fs, errors, tableDir); 
+      LOG.debug("Loading region dirs from " +tableDir.getPath());
+      dirs[num] = new WorkItemHdfsDir(this, fs, errors, tableDir);
       executor.execute(dirs[num]);
       num++;
     }
 
     // wait for all directories to be done
     for (int i = 0; i < num; i++) {
-      synchronized (dirs[i]) {
-        while (!dirs[i].isDone()) {
-          dirs[i].wait();
+      WorkItemHdfsDir dir = dirs[i];
+      synchronized (dir) {
+        while (!dir.isDone()) {
+          dir.wait();
         }
       }
     }
@@ -599,7 +1028,7 @@ public void checkHdfs() throws IOException, InterruptedException {
    * as if it were in a META table. This is so that we can check
    * deployment of ROOT.
    */
-  boolean recordRootRegion() throws IOException {
+  private boolean recordRootRegion() throws IOException {
     HRegionLocation rootLocation = connection.locateRegion(
       HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW);
 
@@ -619,7 +1048,7 @@ boolean recordRootRegion() throws IOException {
     MetaEntry m =
       new MetaEntry(rootLocation.getRegionInfo(), sn, System.currentTimeMillis());
     HbckInfo hbInfo = new HbckInfo(m);
-    regionInfo.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo);
+    regionInfoMap.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo);
     return true;
   }
 
@@ -654,7 +1083,8 @@ public boolean isAborted(){
    * @throws IOException if a remote or network exception occurs
    */
   void processRegionServers(Collection<ServerName> regionServerList)
-  throws IOException, InterruptedException {
+    throws IOException, InterruptedException {
+
     WorkItemRegion[] work = new WorkItemRegion[regionServerList.size()];
     int num = 0;
 
@@ -677,27 +1107,145 @@ void processRegionServers(Collection<ServerName> regionServerList)
 
   /**
    * Check consistency of all regions that have been found in previous phases.
-   * @throws KeeperException
-   * @throws InterruptedException
    */
-  void checkConsistency()
+  private void checkAndFixConsistency()
   throws IOException, KeeperException, InterruptedException {
-    for (java.util.Map.Entry<String, HbckInfo> e: regionInfo.entrySet()) {
-      doConsistencyCheck(e.getKey(), e.getValue());
+    for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
+      checkRegionConsistency(e.getKey(), e.getValue());
+    }
+  }
+
+  /**
+   * Deletes region from meta table
+   */
+  private void deleteMetaRegion(HbckInfo hi) throws IOException {
+    Delete d = new Delete(hi.metaEntry.getRegionName());
+    meta.delete(d);
+    meta.flushCommits();
+    LOG.info("Deleted " + hi.metaEntry.getRegionNameAsString() + " from META" );
+  }
+
+  /**
+   * This backwards-compatibility wrapper for permanently offlining a region
+   * that should not be alive.  If the region server does not support the
+   * "offline" method, it will use the closest unassign method instead.  This
+   * will basically work until one attempts to disable or delete the affected
+   * table.  The problem has to do with in-memory only master state, so
+   * restarting the HMaster or failing over to another should fix this.
+   */
+  private void offline(byte[] regionName) throws IOException {
+    String regionString = Bytes.toStringBinary(regionName);
+    if (!rsSupportsOffline) {
+      LOG.warn("Using unassign region " + regionString
+          + " instead of using offline method, you should"
+          + " restart HMaster after these repairs");
+      admin.unassign(regionName, true);
+      return;
+    }
+
+    // first time we assume the rs's supports #offline.
+    try {
+      LOG.info("Offlining region " + regionString);
+      admin.getMaster().offline(regionName);
+    } catch (IOException ioe) {
+      String notFoundMsg = "java.lang.NoSuchMethodException: " +
+        "org.apache.hadoop.hbase.master.HMaster.offline([B)";
+      if (ioe.getMessage().contains(notFoundMsg)) {
+        LOG.warn("Using unassign region " + regionString
+            + " instead of using offline method, you should"
+            + " restart HMaster after these repairs");
+        rsSupportsOffline = false; // in the future just use unassign
+        admin.unassign(regionName, true);
+        return;
+      }
+      throw ioe;
+    }
+  }
+
+  private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
+    for (OnlineEntry rse : hi.deployedEntries) {
+      LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
+      try {
+        HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
+        offline(rse.hri.getRegionName());
+      } catch (IOException ioe) {
+        LOG.warn("Got exception when attempting to offline region "
+            + Bytes.toString(rse.hri.getRegionName()), ioe);
+      }
+    }
+  }
+
+  /**
+   * Attempts to undeploy a region from a region server based in information in
+   * META.  Any operations that modify the file system should make sure that
+   * its corresponding region is not deployed to prevent data races.
+   *
+   * A separate call is required to update the master in-memory region state
+   * kept in the AssignementManager.  Because disable uses this state instead of
+   * that found in META, we can't seem to cleanly disable/delete tables that
+   * have been hbck fixed.  When used on a version of HBase that does not have
+   * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
+   * restart or failover may be required.
+   */
+  private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
+    if (hi.metaEntry == null && hi.hdfsEntry == null) {
+      undeployRegions(hi);
+      return;
+    }
+
+    // get assignment info and hregioninfo from meta.
+    Get get = new Get(hi.getRegionName());
+    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
+    Result r = meta.get(get);
+    byte[] value = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+    byte[] startcodeBytes = r.getValue(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
+    if (value == null || startcodeBytes == null) {
+      errors.reportError("Unable to close region "
+          + hi.getRegionNameAsString() +  " because meta does not "
+          + "have handle to reach it.");
+      return;
+    }
+    long startcode = Bytes.toLong(startcodeBytes);
+
+    ServerName hsa = new ServerName(Bytes.toString(value), startcode);
+    byte[] hriVal = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    HRegionInfo hri= Writables.getHRegionInfoOrNull(hriVal);
+    if (hri == null) {
+      LOG.warn("Unable to close region " + hi.getRegionNameAsString()
+          + " because META had invalid or missing "
+          + HConstants.CATALOG_FAMILY_STR + ":"
+          + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
+          + " qualifier value.");
+      return;
+    }
+
+    // close the region -- close files and remove assignment
+    HBaseFsckRepair.closeRegionSilentlyAndWait(admin, hsa, hri);
+  }
+
+  private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
+    KeeperException, InterruptedException {
+    // If we are trying to fix the errors
+    if (shouldFixAssignments()) {
+      errors.print(msg);
+      undeployRegions(hbi);
+      setShouldRerun();
+      HBaseFsckRepair.fixUnassigned(admin, hbi.getHdfsHRI());
+      HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
     }
   }
 
   /**
    * Check a single region for consistency and correct deployment.
-   * @throws KeeperException
-   * @throws InterruptedException
    */
-  void doConsistencyCheck(final String key, final HbckInfo hbi)
+  private void checkRegionConsistency(final String key, final HbckInfo hbi)
   throws IOException, KeeperException, InterruptedException {
     String descriptiveName = hbi.toString();
 
     boolean inMeta = hbi.metaEntry != null;
-    boolean inHdfs = hbi.foundRegionDir != null;
+    boolean inHdfs = hbi.getHdfsRegionDir()!= null;
     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
     boolean isDeployed = !hbi.deployedOn.isEmpty();
     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
@@ -707,18 +1255,21 @@ void doConsistencyCheck(final String key, final HbckInfo hbi)
     boolean splitParent =
       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
-    boolean recentlyModified = hbi.foundRegionDir != null &&
-      hbi.foundRegionDir.getModificationTime() + timelag > System.currentTimeMillis();
+    boolean recentlyModified = hbi.getHdfsRegionDir() != null &&
+      hbi.getModTime() + timelag > System.currentTimeMillis();
 
     // ========== First the healthy cases =============
-    if (hbi.onlyEdits) {
+    if (hbi.containsOnlyHdfsEdits()) {
       return;
     }
     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
       return;
-    } else if (inMeta && !isDeployed && splitParent) {
+    } else if (inMeta && inHdfs && !isDeployed && splitParent) {
+      LOG.warn("Region " + descriptiveName + " is a split parent in META and in HDFS");
       return;
-    } else if (inMeta && !shouldBeDeployed && !isDeployed) {
+    } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
+      LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
+        "tabled that is not deployed");
       return;
     } else if (recentlyModified) {
       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
@@ -732,46 +1283,87 @@ else if (!inMeta && !inHdfs && !isDeployed) {
       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
           + descriptiveName + ", key=" + key + ", not on HDFS or in META but " +
           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
+      if (shouldFixAssignments()) {
+        undeployRegions(hbi);
+      }
+
     } else if (!inMeta && inHdfs && !isDeployed) {
       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
           + descriptiveName + " on HDFS, but not listed in META " +
           "or deployed on any region server");
+      // restore region consistency of an adopted orphan
+      if (shouldFixMeta()) {
+        if (!hbi.isHdfsRegioninfoPresent()) {
+          LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
+              +  " in table integrity repair phase if -fixHdfsOrphans was" +
+              " used.");
+          return;
+        }
+
+        LOG.info("Patching .META. with .regioninfo: " + hbi.getHdfsHRI());
+        HBaseFsckRepair.fixMetaHoleOnline(conf, hbi.getHdfsHRI());
+
+        tryAssignmentRepair(hbi, "Trying to reassign region...");
+      }
+
     } else if (!inMeta && inHdfs && isDeployed) {
       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
+      debugLsr(hbi.getHdfsRegionDir());
+      if (shouldFixMeta()) {
+        if (!hbi.isHdfsRegioninfoPresent()) {
+          LOG.error("This should have been repaired in table integrity repair phase");
+          return;
+        }
+
+        LOG.info("Patching .META. with with .regioninfo: " + hbi.getHdfsHRI());
+        HBaseFsckRepair.fixMetaHoleOnline(conf, hbi.getHdfsHRI());
+
+        tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
+      }
 
     // ========== Cases where the region is in META =============
     } else if (inMeta && !inHdfs && !isDeployed) {
       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
           + descriptiveName + " found in META, but not in HDFS "
           + "or deployed on any region server.");
+      if (shouldFixMeta()) {
+        deleteMetaRegion(hbi);
+      }
     } else if (inMeta && !inHdfs && isDeployed) {
       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
           + " found in META, but not in HDFS, " +
           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
+      // We treat HDFS as ground truth.  Any information in meta is transient
+      // and equivalent data can be regenerated.  So, lets unassign and remove
+      // these problems from META.
+      if (shouldFixAssignments()) {
+        errors.print("Trying to fix unassigned region...");
+        closeRegion(hbi);// Close region will cause RS to abort.
+      }
+      if (shouldFixMeta()) {
+        // wait for it to complete
+        deleteMetaRegion(hbi);
+      }
     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
           + " not deployed on any region server.");
-      // If we are trying to fix the errors
-      if (shouldFix()) {
-        errors.print("Trying to fix unassigned region...");
-        setShouldRerun();
-        HBaseFsckRepair.fixUnassigned(this.admin, hbi.metaEntry);
-      }
+      tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
-      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, "Region "
-          + descriptiveName + " should not be deployed according " +
+      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, "UNHANDLED CASE:" +
+          " Region " + descriptiveName + " should not be deployed according " +
           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
+      // TODO test and handle this case.
     } else if (inMeta && inHdfs && isMultiplyDeployed) {
       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
           + " is listed in META on region server " + hbi.metaEntry.regionServer
           + " but is multiply assigned to region servers " +
           Joiner.on(", ").join(hbi.deployedOn));
       // If we are trying to fix the errors
-      if (shouldFix()) {
+      if (shouldFixAssignments()) {
         errors.print("Trying to fix assignment error...");
         setShouldRerun();
-        HBaseFsckRepair.fixDupeAssignment(this.admin, hbi.metaEntry, hbi.deployedOn);
+        HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
       }
     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
@@ -779,10 +1371,11 @@ else if (!inMeta && !inHdfs && !isDeployed) {
           hbi.metaEntry.regionServer + " but found on region server " +
           hbi.deployedOn.get(0));
       // If we are trying to fix the errors
-      if (shouldFix()) {
+      if (shouldFixAssignments()) {
         errors.print("Trying to fix assignment error...");
         setShouldRerun();
-        HBaseFsckRepair.fixDupeAssignment(this.admin, hbi.metaEntry, hbi.deployedOn);
+        HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
+        HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
       }
     } else {
       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
@@ -800,13 +1393,37 @@ else if (!inMeta && !inHdfs && !isDeployed) {
    * Checks tables integrity. Goes over all regions and scans the tables.
    * Collects all the pieces for each table and checks if there are missing,
    * repeated or overlapping ones.
+   * @throws IOException
    */
-  void checkIntegrity() {
-    for (HbckInfo hbi : regionInfo.values()) {
+  TreeMap<String, TableInfo> checkIntegrity() throws IOException {
+    tablesInfo = new TreeMap<String,TableInfo> ();
+    List<HbckInfo> noHDFSRegionInfos = new ArrayList<HbckInfo>();
+    LOG.debug("There are " + regionInfoMap.size() + " region info entries");
+    for (HbckInfo hbi : regionInfoMap.values()) {
       // Check only valid, working regions
-      if (hbi.metaEntry == null) continue;
-      if (hbi.metaEntry.regionServer == null) continue;
-      if (hbi.onlyEdits) continue;
+      if (hbi.metaEntry == null) {
+        // this assumes that consistency check has run loadMetaEntry
+        noHDFSRegionInfos.add(hbi);
+        Path p = hbi.getHdfsRegionDir();
+        if (p == null) {
+          errors.report("No regioninfo in Meta or HDFS. " + hbi);
+        }
+
+        // TODO test.
+        continue;
+      }
+      if (hbi.metaEntry.regionServer == null) {
+        errors.detail("Skipping region because no region server: " + hbi);
+        continue;
+      }
+      if (hbi.metaEntry.isOffline()) {
+        errors.detail("Skipping region because it is offline: " + hbi);
+        continue;
+      }
+      if (hbi.containsOnlyHdfsEdits()) {
+        errors.detail("Skipping region because it only contains edits" + hbi);
+        continue;
+      }
 
       // Missing regionDir or over-deployment is checked elsewhere. Include
       // these cases in modTInfo, so we can evaluate those regions as part of
@@ -817,9 +1434,9 @@ void checkIntegrity() {
 
       // We should be safe here
       String tableName = hbi.metaEntry.getTableNameAsString();
-      TInfo modTInfo = tablesInfo.get(tableName);
+      TableInfo modTInfo = tablesInfo.get(tableName);
       if (modTInfo == null) {
-        modTInfo = new TInfo(tableName);
+        modTInfo = new TableInfo(tableName);
       }
       for (ServerName server : hbi.deployedOn) {
         modTInfo.addServer(server);
@@ -830,32 +1447,111 @@ void checkIntegrity() {
       tablesInfo.put(tableName, modTInfo);
     }
 
-    for (TInfo tInfo : tablesInfo.values()) {
-      if (!tInfo.checkRegionChain()) {
+    for (TableInfo tInfo : tablesInfo.values()) {
+      TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
+      if (!tInfo.checkRegionChain(handler)) {
         errors.report("Found inconsistency in table " + tInfo.getName());
       }
     }
+    return tablesInfo;
+  }
+
+  /**
+   * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
+   * @return number of file move fixes done to merge regions.
+   */
+  public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
+    int fileMoves = 0;
+
+    LOG.debug("Contained region dir after close and pause");
+    debugLsr(contained.getHdfsRegionDir());
+
+    // rename the contained into the container.
+    FileSystem fs = targetRegionDir.getFileSystem(conf);
+    FileStatus[] dirs = fs.listStatus(contained.getHdfsRegionDir());
+
+    if (dirs == null) {
+      if (!fs.exists(contained.getHdfsRegionDir())) {
+        LOG.warn("HDFS region dir " + contained.getHdfsRegionDir() + " already sidelined.");
+      } else {
+        sidelineRegionDir(fs, contained);
+      }
+      return fileMoves;
+    }
+
+    for (FileStatus cf : dirs) {
+      Path src = cf.getPath();
+      Path dst =  new Path(targetRegionDir, src.getName());
+
+      if (src.getName().equals(HRegion.REGIONINFO_FILE)) {
+        // do not copy the old .regioninfo file.
+        continue;
+      }
+
+      if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
+        // do not copy the .oldlogs files
+        continue;
+      }
+
+      LOG.info("Moving files from " + src + " into containing region " + dst);
+      // FileSystem.rename is inconsistent with directories -- if the
+      // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
+      // it moves the src into the dst dir resulting in (foo/a/b).  If
+      // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
+      for (FileStatus hfile : fs.listStatus(src)) {
+        boolean success = fs.rename(hfile.getPath(), dst);
+        if (success) {
+          fileMoves++;
+        }
+      }
+      LOG.debug("Sideline directory contents:");
+      debugLsr(targetRegionDir);
+    }
+
+    // if all success.
+    sidelineRegionDir(fs, contained);
+    LOG.info("Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
+        getSidelineDir());
+    debugLsr(contained.getHdfsRegionDir());
+
+    return fileMoves;
   }
 
   /**
    * Maintain information about a particular table.
    */
-  private class TInfo {
+  public class TableInfo {
     String tableName;
     TreeSet <ServerName> deployedOn;
 
+    // backwards regions
     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
+
+    // region split calculator
     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
 
+    // Histogram of different HTableDescriptors found.  Ideally there is only one!
+    final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
+
     // key = start split, values = set of splits in problem group
-    final Multimap<byte[], HbckInfo> overlapGroups = 
+    final Multimap<byte[], HbckInfo> overlapGroups =
       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
 
-    TInfo(String name) {
+    TableInfo(String name) {
       this.tableName = name;
       deployedOn = new TreeSet <ServerName>();
     }
 
+    /**
+     * @return descriptor common to all regions.  null if are none or multiple!
+     */
+    private HTableDescriptor getHTD() {
+      if (htds.size() == 1) {
+        return (HTableDescriptor)htds.toArray()[0];
+      }
+      return null;
+    }
+
     public void addRegionInfo(HbckInfo hir) {
       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
         // end key is absolute end key, just add it.
@@ -891,12 +1587,226 @@ public int getNumRegions() {
       return sc.getStarts().size() + backwards.size();
     }
 
+    private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
+      ErrorReporter errors;
+
+      IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
+        this.errors = errors;
+        setTableInfo(ti);
+      }
+
+      @Override
+      public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
+        errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
+            "First region should start with an empty key.  You need to "
+            + " create a new region and regioninfo in HDFS to plug the hole.",
+            getTableInfo(), hi);
+      }
+
+      @Override
+      public void handleDegenerateRegion(HbckInfo hi) throws IOException{
+        errors.reportError(ERROR_CODE.DEGENERATE_REGION,
+            "Region has the same start and end key.", getTableInfo(), hi);
+      }
+
+      @Override
+      public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
+        byte[] key = r1.getStartKey();
+        // dup start key
+        errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
+            "Multiple regions have the same startkey: "
+            + Bytes.toStringBinary(key), getTableInfo(), r1);
+        errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
+            "Multiple regions have the same startkey: "
+            + Bytes.toStringBinary(key), getTableInfo(), r2);
+      }
+
+      @Override
+      public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
+        errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
+            "There is an overlap in the region chain.",
+            getTableInfo(), hi1, hi2);
+      }
+
+      @Override
+      public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
+        errors.reportError(
+            ERROR_CODE.HOLE_IN_REGION_CHAIN,
+            "There is a hole in the region chain between "
+                + Bytes.toStringBinary(holeStart) + " and "
+                + Bytes.toStringBinary(holeStop)
+                + ".  You need to create a new .regioninfo and region "
+                + "dir in hdfs to plug the hole.");
+      }
+    };
+
+    /**
+     * This handler fixes integrity errors from hdfs information.  There are
+     * basically three classes of integrity problems 1) holes, 2) overlaps, and
+     * 3) invalid regions.
+     *
+     * This class overrides methods that fix holes and the overlap group case.
+     * Individual cases of particular overlaps are handled by the general
+     * overlap group merge repair case.
+     *
+     * If hbase is online, this forces regions offline before doing merge
+     * operations.
+     */
+    private class HDFSIntegrityFixer extends IntegrityFixSuggester {
+      Configuration conf;
+
+      boolean fixOverlaps = true;
+
+      HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
+          boolean fixHoles, boolean fixOverlaps) {
+        super(ti, errors);
+        this.conf = conf;
+        this.fixOverlaps = fixOverlaps;
+        // TODO properly use fixHoles
+      }
+
+      /**
+       * This is a special case hole -- when the first region of a table is
+       * missing from META, HBase doesn't acknowledge the existance of the
+       * table.
+       */
+      public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
+        errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
+            "First region should start with an empty key.  Creating a new " +
+            "region and regioninfo in HDFS to plug the hole.",
+            getTableInfo(), next);
+        HTableDescriptor htd = getTableInfo().getHTD();
+        // from special EMPTY_START_ROW to next region's startKey
+        HRegionInfo newRegion = new HRegionInfo(htd.getName(),
+            HConstants.EMPTY_START_ROW, next.getStartKey());
+
+        // TODO test
+        HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
+        LOG.info("Table region start key was not empty.  Created new empty region: "
+            + newRegion + " " +region);
+        fixes++;
+      }
+
+      /**
+       * There is a hole in the hdfs regions that violates the table integrity
+       * rules.  Create a new empty region that patches the hole.
+       */
+      public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
+        errors.reportError(
+            ERROR_CODE.HOLE_IN_REGION_CHAIN,
+            "There is a hole in the region chain between "
+                + Bytes.toStringBinary(holeStartKey) + " and "
+                + Bytes.toStringBinary(holeStopKey)
+                + ".  Creating a new regioninfo and region "
+                + "dir in hdfs to plug the hole.");
+        HTableDescriptor htd = getTableInfo().getHTD();
+        HRegionInfo newRegion = new HRegionInfo(htd.getName(), holeStartKey, holeStopKey);
+        HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
+        LOG.info("Plugged hold by creating new empty region: "+ newRegion + " " +region);
+        fixes++;
+      }
+
+      /**
+       * This takes set of overlapping regions and merges them into a single
+       * region.  This covers cases like degenerate regions, shared start key,
+       * general overlaps, duplicate ranges, and partial overlapping regions.
+       *
+       * Cases:
+       * - Clean regions that overlap
+       * - Only .oldlogs regions (can't find start/stop range, or figure out)
+       */
+      @Override
+      public void handleOverlapGroup(Collection<HbckInfo> overlap)
+          throws IOException {
+        Preconditions.checkNotNull(overlap);
+        Preconditions.checkArgument(overlap.size() >0);
+
+        if (!this.fixOverlaps) {
+          LOG.warn("Not attempting to repair overlaps.");
+          return;
+        }
+
+        if (overlap.size() > maxMerge) {
+          LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
+              "regions which is greater than " + maxMerge + ", the max " +
+              "number of regions to merge.");
+          return;
+        }
+
+        LOG.info("== Merging regions into one region: "
+            + Joiner.on(",").join(overlap));
+        // get the min / max range and close all concerned regions
+        Pair<byte[], byte[]> range = null;
+        for (HbckInfo hi : overlap) {
+          if (range == null) {
+            range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
+          } else {
+            if (RegionSplitCalculator.BYTES_COMPARATOR
+                .compare(hi.getStartKey(), range.getFirst()) < 0) {
+              range.setFirst(hi.getStartKey());
+            }
+            if (RegionSplitCalculator.BYTES_COMPARATOR
+                .compare(hi.getEndKey(), range.getSecond()) > 0) {
+              range.setSecond(hi.getEndKey());
+            }
+          }
+          // need to close files so delete can happen.
+          LOG.debug("Closing region before moving data around: " +  hi);
+          LOG.debug("Contained region dir before close");
+          debugLsr(hi.getHdfsRegionDir());
+          try {
+            closeRegion(hi);
+          } catch (IOException ioe) {
+            // TODO exercise this
+            LOG.warn("Was unable to close region " + hi.getRegionNameAsString()
+                + ".  Just continuing... ");
+          } catch (InterruptedException e) {
+            // TODO exercise this
+            LOG.warn("Was unable to close region " + hi.getRegionNameAsString()
+                + ".  Just continuing... ");
+          }
+
+          try {
+            LOG.info("Offlining region: " + hi);
+            offline(hi.getRegionName());
+          } catch (IOException ioe) {
+            LOG.warn("Unable to offline region from master: " + hi, ioe);
+          }
+        }
+
+        // create new empty container region.
+        HTableDescriptor htd = getTableInfo().getHTD();
+        // from start key to end Key
+        HRegionInfo newRegion = new HRegionInfo(htd.getName(), range.getFirst(),
+            range.getSecond());
+        HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
+        LOG.info("Created new empty container region: " +
+            newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
+        debugLsr(region.getRegionDir());
+
+        // all target regions are closed, should be able to safely cleanup.
+        boolean didFix= false;
+        Path target = region.getRegionDir();
+        for (HbckInfo contained : overlap) {
+          LOG.info("Merging " + contained  + " into " + target );
+          int merges = mergeRegionDirs(target, contained);
+          if (merges > 0) {
+            didFix = true;
+          }
+        }
+        if (didFix) {
+          fixes++;
+        }
+      }
+    };
+
     /**
      * Check the region chain (from META) of this table.  We are looking for
      * holes, overlaps, and cycles.
      * @return false if there are errors
+     * @throws IOException
      */
-    public boolean checkRegionChain() {
+    public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
       int originalErrorsCount = errors.getErrorList().size();
       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
       SortedSet<byte[]> splits = sc.getSplits();
@@ -907,12 +1817,7 @@ public boolean checkRegionChain() {
         Collection<HbckInfo> ranges = regions.get(key);
         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
           for (HbckInfo rng : ranges) {
-            // TODO offline fix region hole.
-
-            errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
-                "First region should start with an empty key.  You need to "
-                + " create a new region and regioninfo in HDFS to plug the hole.",
-                this, rng);
+            handler.handleRegionStartKeyNotEmpty(rng);
           }
         }
 
@@ -922,8 +1827,7 @@ public boolean checkRegionChain() {
           byte[] endKey = rng.getEndKey();
           endKey = (endKey.length == 0) ? null : endKey;
           if (Bytes.equals(rng.getStartKey(),endKey)) {
-            errors.reportError(ERROR_CODE.DEGENERATE_REGION,
-              "Region has the same start and end key.", this, rng);
+            handler.handleDegenerateRegion(rng);
           }
         }
 
@@ -950,18 +1854,10 @@ public boolean checkRegionChain() {
             subRange.remove(r1);
             for (HbckInfo r2 : subRange) {
               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
-                // dup start key
-                errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
-                    "Multiple regions have the same startkey: "
-                    + Bytes.toStringBinary(key), this, r1);
-                errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
-                    "Multiple regions have the same startkey: "
-                    + Bytes.toStringBinary(key), this, r2);
+                handler.handleDuplicateStartKeys(r1,r2);
               } else {
                 // overlap
-                errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
-                    "There is an overlap in the region chain.",
-                    this, r1);
+                handler.handleOverlapInRegionChain(r1, r2);
               }
             }
           }
@@ -976,17 +1872,16 @@ public boolean checkRegionChain() {
           // if higher key is null we reached the top.
           if (holeStopKey != null) {
             // hole
-            errors.reportError(ERROR_CODE.HOLE_IN_REGION_CHAIN,
-                "There is a hole in the region chain between "
-                + Bytes.toStringBinary(key) + " and "
-                + Bytes.toStringBinary(holeStopKey)
-                + ".  You need to create a new regioninfo and region "
-                + "dir in hdfs to plug the hole.");
+            handler.handleHoleInRegionChain(key, holeStopKey);
           }
         }
         prevKey = key;
       }
 
+      for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
+        handler.handleOverlapGroup(overlap);
+      }
+
       if (details) {
         // do full region split map dump
         System.out.println("---- Table '"  +  this.tableName 
@@ -1034,8 +1929,10 @@ public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
     }
   }
 
-  public Multimap<byte[], HbckInfo> getOverlapGroups(String table) {
-    return tablesInfo.get(table).overlapGroups;
+  public Multimap<byte[], HbckInfo> getOverlapGroups(
+      String table) {
+    TableInfo ti = tablesInfo.get(table);
+    return ti.overlapGroups;
   }
 
   /**
@@ -1051,7 +1948,7 @@ HTableDescriptor[] getTables(AtomicInteger numSkipped) {
     List<String> tableNames = new ArrayList<String>();
     long now = System.currentTimeMillis();
 
-    for (HbckInfo hbi : regionInfo.values()) {
+    for (HbckInfo hbi : regionInfoMap.values()) {
       MetaEntry info = hbi.metaEntry;
 
       // if the start key is zero, then we have found the first region of a table.
@@ -1085,10 +1982,10 @@ HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
    * and returned.
    */
   private synchronized HbckInfo getOrCreateInfo(String name) {
-    HbckInfo hbi = regionInfo.get(name);
+    HbckInfo hbi = regionInfoMap.get(name);
     if (hbi == null) {
       hbi = new HbckInfo(null);
-      regionInfo.put(name, hbi);
+      regionInfoMap.put(name, hbi);
     }
     return hbi;
   }
@@ -1102,10 +1999,10 @@ private synchronized HbckInfo getOrCreateInfo(String name) {
    * @throws KeeperException
    * @throws InterruptedException
     */
-  boolean checkMetaEntries()
-  throws IOException, KeeperException, InterruptedException {
+  boolean checkMetaRegion()
+    throws IOException, KeeperException, InterruptedException {
     List <HbckInfo> metaRegions = Lists.newArrayList();
-    for (HbckInfo value : regionInfo.values()) {
+    for (HbckInfo value : regionInfoMap.values()) {
       if (value.metaEntry.isMetaRegion()) {
         metaRegions.add(value);
       }
@@ -1116,22 +2013,23 @@ boolean checkMetaEntries()
       HRegionLocation rootLocation = connection.locateRegion(
         HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW);
       HbckInfo root =
-          regionInfo.get(rootLocation.getRegionInfo().getEncodedName());
+          regionInfoMap.get(rootLocation.getRegionInfo().getEncodedName());
 
       // If there is no region holding .META.
       if (metaRegions.size() == 0) {
         errors.reportError(ERROR_CODE.NO_META_REGION, ".META. is not found on any region.");
-        if (shouldFix()) {
+        if (shouldFixAssignments()) {
           errors.print("Trying to fix a problem with .META...");
           setShouldRerun();
           // try to fix it (treat it as unassigned region)
-          HBaseFsckRepair.fixUnassigned(this.admin, root.metaEntry);
+          HBaseFsckRepair.fixUnassigned(admin, root.metaEntry);
+          HBaseFsckRepair.waitUntilAssigned(admin, root.getHdfsHRI());
         }
       }
       // If there are more than one regions pretending to hold the .META.
       else if (metaRegions.size() > 1) {
         errors.reportError(ERROR_CODE.MULTI_META_REGION, ".META. is found on more than one region.");
-        if (shouldFix()) {
+        if (shouldFixAssignments()) {
           errors.print("Trying to fix a problem with .META...");
           setShouldRerun();
           // try fix it (treat is a dupe assignment)
@@ -1139,7 +2037,7 @@ else if (metaRegions.size() > 1) {
           for (HbckInfo mRegion : metaRegions) {
             deployedOn.add(mRegion.metaEntry.regionServer);
           }
-          HBaseFsckRepair.fixDupeAssignment(this.admin, root.metaEntry, deployedOn);
+          HBaseFsckRepair.fixMultiAssignment(admin, root.metaEntry, deployedOn);
         }
       }
       // rerun hbck with hopefully fixed META
@@ -1153,7 +2051,16 @@ else if (metaRegions.size() > 1) {
    * Scan .META. and -ROOT-, adding all regions found to the regionInfo map.
    * @throws IOException if an error is encountered
    */
-  void getMetaEntries() throws IOException {
+  boolean loadMetaEntries() throws IOException {
+
+    // get a list of all regions from the master. This involves
+    // scanning the META table
+    if (!recordRootRegion()) {
+      // Will remove later if we can fix it
+      errors.reportError("Fatal error: unable to get root region location. Exiting...");
+      return false;
+    }
+
     MetaScannerVisitor visitor = new MetaScannerVisitor() {
       int countRecord = 1;
 
@@ -1180,7 +2087,7 @@ public boolean processRow(Result result) throws IOException {
           }
           MetaEntry m = new MetaEntry(pair.getFirst(), sn, ts);
           HbckInfo hbInfo = new HbckInfo(m);
-          HbckInfo previous = regionInfo.put(pair.getFirst().getEncodedName(), hbInfo);
+          HbckInfo previous = regionInfoMap.put(pair.getFirst().getEncodedName(), hbInfo);
           if (previous != null) {
             throw new IOException("Two entries in META are same " + previous);
           }
@@ -1208,13 +2115,13 @@ public boolean processRow(Result result) throws IOException {
     }
     
     errors.print("");
+    return true;
   }
 
   /**
-   * Stores the entries scanned from META
+   * Stores the regioninfo entries scanned from META
    */
   static class MetaEntry extends HRegionInfo {
-    private static final Log LOG = LogFactory.getLog(HRegionInfo.class);
     ServerName regionServer;   // server hosting this region
     long modTime;          // timestamp of most recent modification metadata
 
@@ -1223,44 +2130,168 @@ public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
       this.regionServer = regionServer;
       this.modTime = modTime;
     }
+
+    public boolean equals(Object o) {
+      boolean superEq = super.equals(o);
+      if (!superEq) {
+        return superEq;
+      }
+
+      MetaEntry me = (MetaEntry) o;
+      if (!regionServer.equals(me.regionServer)) {
+        return false;
+      }
+      return (modTime == me.modTime);
+    }
+  }
+
+  /**
+   * Stores the regioninfo entries from HDFS
+   */
+  static class HdfsEntry {
+    HRegionInfo hri;
+    Path hdfsRegionDir = null;
+    long hdfsRegionDirModTime  = 0;
+    boolean hdfsRegioninfoFilePresent = false;
+    boolean hdfsOnlyEdits = false;
+  }
+
+  /**
+   * Stores the regioninfo retrieved from Online region servers.
+   */
+  static class OnlineEntry {
+    HRegionInfo hri;
+    ServerName hsa;
+
+    public String toString() {
+      return hsa.toString() + ";" + hri.getRegionNameAsString();
+    }
   }
 
   /**
-   * Maintain information about a particular region.
+   * Maintain information about a particular region.  It gathers information
+   * from three places -- HDFS, META, and region servers.
    */
   public static class HbckInfo implements KeyRange {
-    boolean onlyEdits = false;
-    MetaEntry metaEntry = null;
-    FileStatus foundRegionDir = null;
-    List<ServerName> deployedOn = Lists.newArrayList();
-    String hdfsTableName = null; // This is set in the workitem loader.
+    private MetaEntry metaEntry = null; // info in META
+    private HdfsEntry hdfsEntry = null; // info in HDFS
+    private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
+    private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
 
     HbckInfo(MetaEntry metaEntry) {
       this.metaEntry = metaEntry;
     }
 
-    public synchronized void addServer(ServerName server) {
+    public synchronized void addServer(HRegionInfo hri, ServerName server) {
+      OnlineEntry rse = new OnlineEntry() ;
+      rse.hri = hri;
+      rse.hsa = server;
+      this.deployedEntries.add(rse);
       this.deployedOn.add(server);
     }
 
     public synchronized String toString() {
-      if (metaEntry != null) {
-        return metaEntry.getRegionNameAsString();
-      } else if (foundRegionDir != null) {
-        return foundRegionDir.getPath().toString();
-      } else {
-        return "UNKNOWN_REGION on " + Joiner.on(", ").join(deployedOn);
-      }
+      StringBuilder sb = new StringBuilder();
+      sb.append("{ meta => ");
+      sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
+      sb.append( ", hdfs => " + getHdfsRegionDir());
+      sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
+      sb.append(" }");
+      return sb.toString();
     }
 
     @Override
     public byte[] getStartKey() {
-      return this.metaEntry.getStartKey();
+      if (this.metaEntry != null) {
+        return this.metaEntry.getStartKey();
+      } else if (this.hdfsEntry != null) {
+        return this.hdfsEntry.hri.getStartKey();
+      } else {
+        LOG.error("Entry " + this + " has no meta or hdfs region start key.");
+        return null;
+      }
     }
 
     @Override
     public byte[] getEndKey() {
-      return this.metaEntry.getEndKey();
+      if (this.metaEntry != null) {
+        return this.metaEntry.getEndKey();
+      } else if (this.hdfsEntry != null) {
+        return this.hdfsEntry.hri.getEndKey();
+      } else {
+        LOG.error("Entry " + this + " has no meta or hdfs region start key.");
+        return null;
+      }
+    }
+
+    public byte[] getTableName() {
+      if (this.metaEntry != null) {
+        return this.metaEntry.getTableName();
+      } else if (this.hdfsEntry != null) {
+        // we are only guaranteed to have a path and not an HRI for hdfsEntry,
+        // so we get the name from the Path
+        Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
+        return Bytes.toBytes(tableDir.getName());
+      } else {
+        // Currently no code exercises this path, but we could add one for
+        // getting table name from OnlineEntry
+        return null;
+      }
+    }
+
+    public String getRegionNameAsString() {
+      if (metaEntry != null) {
+        return metaEntry.getRegionNameAsString();
+      } else if (hdfsEntry != null) {
+        return hdfsEntry.hri.getRegionNameAsString();
+      } else {
+        return null;
+      }
+    }
+
+    public byte[] getRegionName() {
+      if (metaEntry != null) {
+        return metaEntry.getRegionName();
+      } else if (hdfsEntry != null) {
+        return hdfsEntry.hri.getRegionName();
+      } else {
+        return null;
+      }
+    }
+
+    Path getHdfsRegionDir() {
+      if (hdfsEntry == null) {
+        return null;
+      }
+      return hdfsEntry.hdfsRegionDir;
+    }
+
+    boolean containsOnlyHdfsEdits() {
+      if (hdfsEntry == null) {
+        return false;
+      }
+      return hdfsEntry.hdfsOnlyEdits;
+    }
+
+    boolean isHdfsRegioninfoPresent() {
+      if (hdfsEntry == null) {
+        return false;
+      }
+      return hdfsEntry.hdfsRegioninfoFilePresent;
+    }
+
+    long getModTime() {
+      if (hdfsEntry == null) {
+        return 0;
+      }
+      return hdfsEntry.hdfsRegionDirModTime;
+    }
+
+    HRegionInfo getHdfsHRI() {
+      if (hdfsEntry == null) {
+        return null;
+      }
+      return hdfsEntry.hri;
     }
   }
 
@@ -1273,21 +2304,21 @@ public int compare(HbckInfo l, HbckInfo r) {
       }
 
       int tableCompare = RegionSplitCalculator.BYTES_COMPARATOR.compare(
-          l.metaEntry.getTableName(), r.metaEntry.getTableName());
+          l.getTableName(), r.getTableName());
       if (tableCompare != 0) {
         return tableCompare;
       }
 
       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
-          l.metaEntry.getStartKey(), r.metaEntry.getStartKey());
+          l.getStartKey(), r.getStartKey());
       if (startComparison != 0) {
         return startComparison;
       }
 
       // Special case for absolute endkey
-      byte[] endKey = r.metaEntry.getEndKey();
+      byte[] endKey = r.getEndKey();
       endKey = (endKey.length == 0) ? null : endKey;
-      byte[] endKey2 = l.metaEntry.getEndKey();
+      byte[] endKey2 = l.getEndKey();
       endKey2 = (endKey2.length == 0) ? null : endKey2;
       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
           endKey2,  endKey);
@@ -1296,17 +2327,29 @@ public int compare(HbckInfo l, HbckInfo r) {
         return endComparison;
       }
 
-      // use modTime as tiebreaker.
-      return (int) (l.metaEntry.modTime - r.metaEntry.modTime);
+      // use regionId as tiebreaker.
+      // Null is considered after all possible values so make it bigger.
+      if (l.hdfsEntry == null && r.hdfsEntry == null) {
+        return 0;
+      }
+      if (l.hdfsEntry == null && r.hdfsEntry != null) {
+        return 1;
+      }
+      // l.hdfsEntry must not be null
+      if (r.hdfsEntry == null) {
+        return -1;
+      }
+      // both l.hdfsEntry and r.hdfsEntry must not be null.
+      return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
     }
   };
 
   /**
    * Prints summary of all tables found on the system.
    */
-  private void printTableSummary() {
+  private void printTableSummary(TreeMap<String, TableInfo> tablesInfo) {
     System.out.println("Summary:");
-    for (TInfo tInfo : tablesInfo.values()) {
+    for (TableInfo tInfo : tablesInfo.values()) {
       if (errors.tableHasErrors(tInfo)) {
         System.out.println("Table " + tInfo.getName() + " is inconsistent.");
       } else {
@@ -1327,28 +2370,29 @@ public static enum ERROR_CODE {
       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, DUPE_STARTKEYS,
-      HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION
+      HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
+      ORPHAN_HDFS_REGION
     }
     public void clear();
     public void report(String message);
     public void reportError(String message);
     public void reportError(ERROR_CODE errorCode, String message);
-    public void reportError(ERROR_CODE errorCode, String message, TInfo table, HbckInfo info);
-    public void reportError(ERROR_CODE errorCode, String message, TInfo table, HbckInfo info1, HbckInfo info2);
+    public void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
+    public void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info1, HbckInfo info2);
     public int summarize();
     public void detail(String details);
     public ArrayList<ERROR_CODE> getErrorList();
     public void progress();
     public void print(String message);
     public void resetErrors();
-    public boolean tableHasErrors(TInfo table);
+    public boolean tableHasErrors(TableInfo table);
   }
 
   private static class PrintingErrorReporter implements ErrorReporter {
     public int errorCount = 0;
     private int showProgress;
 
-    Set<TInfo> errorTables = new HashSet<TInfo>();
+    Set<TableInfo> errorTables = new HashSet<TableInfo>();
 
     // for use by unit tests to verify which errors were discovered
     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
@@ -1368,18 +2412,18 @@ public synchronized void reportError(ERROR_CODE errorCode, String message) {
       showProgress = 0;
     }
 
-    public synchronized void reportError(ERROR_CODE errorCode, String message, TInfo table,
+    public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
                                          HbckInfo info) {
       errorTables.add(table);
-      String reference = "(region " + info.metaEntry.getRegionNameAsString() + ")";
+      String reference = "(region " + info.getRegionNameAsString() + ")";
       reportError(errorCode, reference + " " + message);
     }
 
-    public synchronized void reportError(ERROR_CODE errorCode, String message, TInfo table,
+    public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
                                          HbckInfo info1, HbckInfo info2) {
       errorTables.add(table);
-      String reference = "(regions " + info1.metaEntry.getRegionNameAsString()
-          + " and " + info2.metaEntry.getRegionNameAsString() + ")";
+      String reference = "(regions " + info1.getRegionNameAsString()
+          + " and " + info2.getRegionNameAsString() + ")";
       reportError(errorCode, reference + " " + message);
     }
 
@@ -1422,7 +2466,7 @@ public synchronized void print(String message) {
     }
 
     @Override
-    public boolean tableHasErrors(TInfo table) {
+    public boolean tableHasErrors(TableInfo table) {
       return errorTables.contains(table);
     }
 
@@ -1476,7 +2520,8 @@ synchronized boolean isDone() {
     public synchronized void run() {
       errors.progress();
       try {
-        HRegionInterface server = connection.getHRegionConnection(new HServerAddress(rsinfo.getHostname(), rsinfo.getPort()));
+        HRegionInterface server =
+            connection.getHRegionConnection(rsinfo.getHostname(), rsinfo.getPort());
 
         // list all online regions from this region server
         List<HRegionInfo> regions = server.getOnlineRegions();
@@ -1498,7 +2543,7 @@ public synchronized void run() {
         // check to see if the existence of this region matches the region in META
         for (HRegionInfo r:regions) {
           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
-          hbi.addServer(rsinfo);
+          hbi.addServer(r, rsinfo);
         }
       } catch (IOException e) {          // unable to connect to the region server. 
         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
@@ -1521,7 +2566,8 @@ private List<HRegionInfo> filterOnlyMetaRegions(List<HRegionInfo> regions) {
   }
 
   /**
-   * Contact hdfs and get all information about specified table directory.
+   * Contact hdfs and get all information about specified table directory into
+   * regioninfo list.
    */
   static class WorkItemHdfsDir implements Runnable {
     private HBaseFsck hbck;
@@ -1555,36 +2601,44 @@ public synchronized void run() {
         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
         for (FileStatus regionDir : regionDirs) {
           String encodedName = regionDir.getPath().getName();
-
           // ignore directories that aren't hexadecimal
           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) continue;
-  
+
+          LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
-          hbi.hdfsTableName = tableName;
+          HdfsEntry he = new HdfsEntry();
           synchronized (hbi) {
-            if (hbi.foundRegionDir != null) {
+            if (hbi.getHdfsRegionDir() != null) {
               errors.print("Directory " + encodedName + " duplicate??" +
-                           hbi.foundRegionDir);
+                           hbi.getHdfsRegionDir());
             }
-            hbi.foundRegionDir = regionDir;
-        
+
+            he.hdfsRegionDir = regionDir.getPath();
+            he.hdfsRegionDirModTime = regionDir.getModificationTime();
+            Path regioninfoFile = new Path(he.hdfsRegionDir, HRegion.REGIONINFO_FILE);
+            he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
+            // we add to orphan list when we attempt to read .regioninfo
+
             // Set a flag if this region contains only edits
             // This is special case if a region is left after split
-            hbi.onlyEdits = true;
+            he.hdfsOnlyEdits = true;
             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
             Path ePath = HLog.getRegionDirRecoveredEditsDir(regionDir.getPath());
             for (FileStatus subDir : subDirs) {
               String sdName = subDir.getPath().getName();
               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
-                hbi.onlyEdits = false;
+                he.hdfsOnlyEdits = false;
                 break;
               }
             }
+            hbi.hdfsEntry = he;
           }
         }
-      } catch (IOException e) {          // unable to connect to the region server. 
-        errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: " + tableDir.getPath().getName() +
-                      " Unable to fetch region information. " + e);
+      } catch (IOException e) {
+        // unable to connect to the region server.
+        errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
+            + tableDir.getPath().getName()
+            + " Unable to fetch region information. " + e);
       } finally {
         done = true;
         notifyAll();
@@ -1596,7 +2650,7 @@ public synchronized void run() {
    * Display the full report from fsck. This displays all live and dead region
    * servers, and all known regions.
    */
-  public void displayFullReport() {
+  public void setDisplayFullReport() {
     details = true;
   }
 
@@ -1634,12 +2688,74 @@ boolean shouldRerun() {
    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
    * found by fsck utility.
    */
-  public void setFixErrors(boolean shouldFix) {
-    fix = shouldFix;
+  public void setFixAssignments(boolean shouldFix) {
+    fixAssignments = shouldFix;
+  }
+
+  boolean shouldFixAssignments() {
+    return fixAssignments;
+  }
+
+  public void setFixMeta(boolean shouldFix) {
+    fixMeta = shouldFix;
+  }
+
+  boolean shouldFixMeta() {
+    return fixMeta;
+  }
+
+  public void setFixHdfsHoles(boolean shouldFix) {
+    fixHdfsHoles = shouldFix;
+  }
+
+  boolean shouldFixHdfsHoles() {
+    return fixHdfsHoles;
+  }
+
+  public void setFixHdfsOverlaps(boolean shouldFix) {
+    fixHdfsOverlaps = shouldFix;
+  }
+
+  boolean shouldFixHdfsOverlaps() {
+    return fixHdfsOverlaps;
+  }
+
+  public void setFixHdfsOrphans(boolean shouldFix) {
+    fixHdfsOrphans = shouldFix;
+  }
+
+  boolean shouldFixHdfsOrphans() {
+    return fixHdfsOrphans;
+  }
+
+  /**
+   * @param mm maximum number of regions to merge into a single region.
+   */
+  public void setMaxMerge(int mm) {
+    this.maxMerge = mm;
+  }
+
+  public int getMaxMerge() {
+    return maxMerge;
+  }
+
+  /**
+   * Only fix tables specified by the list
+   */
+  boolean shouldFixTable(byte[] table) {
+    if (tablesToFix.size() == 0) {
+      return true;
+    }
+
+    // doing this naively since byte[] equals may not do what we want.
+    for (byte[] t : tablesToFix) {
+      if (Bytes.equals(t, table)) return true;
+    }
+    return false;
   }
 
-  boolean shouldFix() {
-    return fix;
+  void includeTable(byte[] table) {
+    tablesToFix.add(table);
   }
 
   /**
@@ -1652,17 +2768,29 @@ public void setTimeLag(long seconds) {
   }
 
   protected static void printUsageAndExit() {
-    System.err.println("Usage: fsck [opts] ");
+    System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
     System.err.println("   -details Display full report of all regions.");
     System.err.println("   -timelag {timeInSeconds}  Process only regions that " +
                        " have not experienced any metadata updates in the last " +
                        " {{timeInSeconds} seconds.");
-    System.err.println("   -fix Try to fix some of the errors.");
     System.err.println("   -sleepBeforeRerun {timeInSeconds} Sleep this many seconds" +
-                       " before checking if the fix worked if run with -fix");
+        " before checking if the fix worked if run with -fix");
     System.err.println("   -summary Print only summary of the tables and status.");
     System.err.println("   -metaonly Only check the state of ROOT and META tables.");
+
+    System.err.println("  Repair options: (expert features, use with caution!)");
+    System.err.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
+    System.err.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
+    System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
+    System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
+    System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
+    System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
+    System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
+    System.err.println("");
+    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps");
+    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
+
     Runtime.getRuntime().exit(-2);
   }
 
@@ -1683,7 +2811,7 @@ public static void main(String[] args) throws Exception {
     for (int i = 0; i < args.length; i++) {
       String cmd = args[i];
       if (cmd.equals("-details")) {
-        fsck.displayFullReport();
+        fsck.setDisplayFullReport();
       } else if (cmd.equals("-timelag")) {
         if (i == args.length - 1) {
           System.err.println("HBaseFsck: -timelag needs a value.");
@@ -1710,21 +2838,60 @@ public static void main(String[] args) throws Exception {
         }
         i++;
       } else if (cmd.equals("-fix")) {
-        fsck.setFixErrors(true);
+        System.err.println("This option is deprecated, please use " +
+          "-fixAssignments instead.");
+        fsck.setFixAssignments(true);
+      } else if (cmd.equals("-fixAssignments")) {
+        fsck.setFixAssignments(true);
+      } else if (cmd.equals("-fixMeta")) {
+        fsck.setFixMeta(true);
+      } else if (cmd.equals("-fixHdfsHoles")) {
+        fsck.setFixHdfsHoles(true);
+      } else if (cmd.equals("-fixHdfsOrphans")) {
+        fsck.setFixHdfsOrphans(true);
+      } else if (cmd.equals("-fixHdfsOverlaps")) {
+        fsck.setFixHdfsOverlaps(true);
+      } else if (cmd.equals("-repair")) {
+        // this attempts to merge overlapping hdfs regions, needs testing
+        // under load
+        fsck.setFixHdfsHoles(true);
+        fsck.setFixHdfsOrphans(true);
+        fsck.setFixMeta(true);
+        fsck.setFixAssignments(true);
+        fsck.setFixHdfsOverlaps(true);
+      } else if (cmd.equals("-repairHoles")) {
+        // this will make all missing hdfs regions available but may lose data
+        fsck.setFixHdfsHoles(true);
+        fsck.setFixHdfsOrphans(false);
+        fsck.setFixMeta(true);
+        fsck.setFixAssignments(true);
+        fsck.setFixHdfsOverlaps(false);
+      } else if (cmd.equals("-maxMerge")) {
+        if (i == args.length - 1) {
+          System.err.println("-maxMerge needs a numeric value argument.");
+          printUsageAndExit();
+        }
+        try {
+          int maxMerge = Integer.parseInt(args[i+1]);
+          fsck.setMaxMerge(maxMerge);
+        } catch (NumberFormatException e) {
+          System.err.println("-maxMerge needs a numeric value argument.");
+          printUsageAndExit();
+        }
+        i++;
       } else if (cmd.equals("-summary")) {
         fsck.setSummary();
       } else if (cmd.equals("-metaonly")) {
         fsck.setCheckMetaOnly();
       } else {
-        String str = "Unknown command line option : " + cmd;
-        LOG.info(str);
-        System.out.println(str);
-        printUsageAndExit();
+        byte[] table = Bytes.toBytes(cmd);
+        fsck.includeTable(table);
+        System.out.println("Allow fixes for table: " + cmd);
       }
     }
     // do the real work of fsck
     fsck.connect();
-    int code = fsck.doWork();
+    int code = fsck.onlineHbck();
     // If we have changed the HBase state it is better to run fsck again
     // to see if we haven't broken something else in the process.
     // We run it only once more because otherwise we can easily fall into
@@ -1737,11 +2904,48 @@ public static void main(String[] args) throws Exception {
         Runtime.getRuntime().exit(code);
       }
       // Just report
-      fsck.setFixErrors(false);
+      fsck.setFixAssignments(false);
+      fsck.setFixMeta(false);
+      fsck.setFixHdfsHoles(false);
+      fsck.setFixHdfsOverlaps(false);
       fsck.errors.resetErrors();
-      code = fsck.doWork();
+      code = fsck.onlineHbck();
     }
 
     Runtime.getRuntime().exit(code);
   }
+
+  /**
+   * ls -r for debugging purposes
+   */
+  void debugLsr(Path p) throws IOException {
+    debugLsr(conf, p);
+  }
+
+  /**
+   * ls -r for debugging purposes
+   */
+  public static void debugLsr(Configuration conf, Path p) throws IOException {
+    if (!LOG.isDebugEnabled()) {
+      return;
+    }
+    FileSystem fs = p.getFileSystem(conf);
+
+    if (!fs.exists(p)) {
+      // nothing
+      return;
+    }
+    System.out.println(p);
+
+    if (fs.isFile(p)) {
+      return;
+    }
+
+    if (fs.getFileStatus(p).isDir()) {
+      FileStatus[] fss= fs.listStatus(p);
+      for (FileStatus status : fss) {
+        debugLsr(conf, status.getPath());
+      }
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index 77f60a021e70..1741aee6644b 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -21,40 +21,51 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.zookeeper.KeeperException;
 
+/**
+ * This class contains helper methods that repair parts of hbase's filesystem
+ * contents.
+ */
 public class HBaseFsckRepair {
+  public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
 
   /**
-   * Fix dupe assignment by doing silent closes on each RS hosting the region
+   * Fix multiple assignment by doing silent closes on each RS hosting the region
    * and then force ZK unassigned node to OFFLINE to trigger assignment by
    * master.
-   * @param admin
-   * @param region
-   * @param servers
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
+   *
+   * @param admin HBase admin used to undeploy
+   * @param region Region to undeploy
+   * @param servers list of Servers to undeploy from
    */
-  public static void fixDupeAssignment(HBaseAdmin admin, HRegionInfo region,
+  public static void fixMultiAssignment(HBaseAdmin admin, HRegionInfo region,
       List<ServerName> servers)
   throws IOException, KeeperException, InterruptedException {
-
     HRegionInfo actualRegion = new HRegionInfo(region);
 
     // Close region on the servers silently
     for(ServerName server : servers) {
-      closeRegionSilentlyAndWait(admin.getConfiguration(), server, actualRegion);
+      closeRegionSilentlyAndWait(admin, server, actualRegion);
     }
 
     // Force ZK node to OFFLINE so master assigns
@@ -63,58 +74,133 @@ public static void fixDupeAssignment(HBaseAdmin admin, HRegionInfo region,
 
   /**
    * Fix unassigned by creating/transition the unassigned ZK node for this
-   * region to OFFLINE state with a special flag to tell the master that this
-   * is a forced operation by HBCK.
-   * @param admin
+   * region to OFFLINE state with a special flag to tell the master that this is
+   * a forced operation by HBCK.
+   *
+   * This assumes that info is in META.
+   *
+   * @param conf
    * @param region
    * @throws IOException
    * @throws KeeperException
    */
   public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region)
-  throws IOException, KeeperException {
+      throws IOException, KeeperException {
     HRegionInfo actualRegion = new HRegionInfo(region);
 
     // Force ZK node to OFFLINE so master assigns
     forceOfflineInZK(admin, actualRegion);
   }
 
+  /**
+   * In 0.90, this forces an HRI offline by setting the RegionTransitionData
+   * in ZK to have HBCK_CODE_NAME as the server.  This is a special case in
+   * the AssignmentManager that attempts an assign call by the master.
+   *
+   * @see org.apache.hadoop.hbase.master.AssignementManager#handleHBCK
+   *
+   * This doesn't seem to work properly in the updated version of 0.92+'s hbck
+   * so we use assign to force the region into transition.  This has the
+   * side-effect of requiring a HRegionInfo that considers regionId (timestamp)
+   * in comparators that is addressed by HBASE-5563.
+   */
   private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region)
   throws ZooKeeperConnectionException, KeeperException, IOException {
     admin.assign(region.getRegionName());
   }
 
-  private static void closeRegionSilentlyAndWait(Configuration conf,
-      ServerName server, HRegionInfo region) throws IOException,
-      InterruptedException {
-    HConnection connection = HConnectionManager.getConnection(conf);
-    boolean success = false;
+  /*
+   * Should we check all assignments or just not in RIT?
+   */
+  public static void waitUntilAssigned(HBaseAdmin admin,
+      HRegionInfo region) throws IOException, InterruptedException {
+    HConnection connection = admin.getConnection();
+
     try {
-      HRegionInterface rs =
-        connection.getHRegionConnection(server.getHostname(), server.getPort());
-      rs.closeRegion(region, false);
-      long timeout = conf.getLong("hbase.hbck.close.timeout", 120000);
+      long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
       long expiration = timeout + System.currentTimeMillis();
       while (System.currentTimeMillis() < expiration) {
         try {
-          HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName());
-          if (rsRegion == null)
-            throw new NotServingRegionException();
-        } catch (Exception e) {
-          success = true;
-          return;
+          Map<String, RegionState> rits=
+            admin.getClusterStatus().getRegionsInTransition();
+
+          if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
+            // yay! no longer RIT
+            return;
+          }
+          // still in rit
+          LOG.info("Region still in transition, waiting for "
+              + "it to become assigned: " + region);
+        } catch (IOException e) {
+          LOG.warn("Exception when waiting for region to become assigned,"
+              + " retrying", e);
         }
         Thread.sleep(1000);
       }
-      throw new IOException("Region " + region + " failed to close within"
-          + " timeout " + timeout);
+      throw new IOException("Region " + region + " failed to move out of " +
+          "transition within timeout " + timeout + "ms");
     } finally {
       try {
         connection.close();
       } catch (IOException ioe) {
-        if (success) {
-          throw ioe;
-        }
+        throw ioe;
       }
     }
   }
+
+  /**
+   * Contacts a region server and waits up to hbase.hbck.close.timeout ms
+   * (default 120s) to close the region.  This bypasses the active hmaster.
+   */
+  public static void closeRegionSilentlyAndWait(HBaseAdmin admin,
+      ServerName server, HRegionInfo region) throws IOException, InterruptedException {
+    HConnection connection = admin.getConnection();
+    HRegionInterface rs = connection.getHRegionConnection(server.getHostname(),
+        server.getPort());
+    rs.closeRegion(region, false);
+    long timeout = admin.getConfiguration()
+      .getLong("hbase.hbck.close.timeout", 120000);
+    long expiration = timeout + System.currentTimeMillis();
+    while (System.currentTimeMillis() < expiration) {
+      try {
+        HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName());
+        if (rsRegion == null)
+          return;
+      } catch (IOException ioe) {
+        return;
+      }
+      Thread.sleep(1000);
+    }
+    throw new IOException("Region " + region + " failed to close within"
+        + " timeout " + timeout);
+  }
+
+  /**
+   * Puts the specified HRegionInfo into META.
+   */
+  public static void fixMetaHoleOnline(Configuration conf,
+      HRegionInfo hri) throws IOException {
+    Put p = new Put(hri.getRegionName());
+    p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(hri));
+    HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
+    meta.put(p);
+    meta.close();
+  }
+
+  /**
+   * Creates, flushes, and closes a new region.
+   */
+  public static HRegion createHDFSRegionDir(Configuration conf,
+      HRegionInfo hri, HTableDescriptor htd) throws IOException {
+    // Create HRegion
+    Path root = FSUtils.getRootDir(conf);
+    HRegion region = HRegion.createHRegion(hri, root, conf, htd);
+    HLog hlog = region.getLog();
+
+    // Close the new region to flush to disk. Close log file too.
+    region.close();
+    hlog.closeAndDelete();
+    return region;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index 29e8bb2f51e5..d73821a80a7c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -40,7 +40,6 @@
  */
 public class OfflineMetaRepair {
   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
-  HBaseFsck fsck;
 
   protected static void printUsageAndExit() {
     System.err.println("Usage: OfflineMetaRepair [opts] ");
@@ -48,6 +47,8 @@ protected static void printUsageAndExit() {
     System.err
         .println("   -details          Display full report of all regions.");
     System.err.println("   -base <hdfs://>   Base Hbase Data directory");
+    System.err.println("   -fix              Auto fix as many problems as possible");
+    System.err.println("   -fixHoles         Auto fix as region holes");
     Runtime.getRuntime().exit(-2);
   }
 
@@ -63,18 +64,24 @@ public static void main(String[] args) throws Exception {
     Configuration conf = HBaseConfiguration.create();
     conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
     HBaseFsck fsck = new HBaseFsck(conf);
+    boolean fixHoles = false;
 
     // Process command-line args.
     for (int i = 0; i < args.length; i++) {
       String cmd = args[i];
       if (cmd.equals("-details")) {
-        fsck.displayFullReport();
+        fsck.setDisplayFullReport();
       } else if (cmd.equals("-base")) {
         // update hbase root dir to user-specified base
         i++;
         String path = args[i];
         conf.set(HConstants.HBASE_DIR, path);
         conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
+      } else if (cmd.equals("-fixHoles")) {
+        fixHoles = true;
+      } else if (cmd.equals("-fix")) {
+        // make all fix options true
+        fixHoles = true;
       } else {
         String str = "Unknown command line option : " + cmd;
         LOG.info(str);
@@ -87,7 +94,7 @@ public static void main(String[] args) throws Exception {
     // threads cleanly, so we do a System.exit.
     boolean success = false;
     try {
-      success = fsck.rebuildMeta();
+      success = fsck.rebuildMeta(fixHoles);
     } catch (MultipleIOException mioes) {
       for (IOException ioe : mioes.getExceptions()) {
         LOG.error("Bailed out due to:", ioe);
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 18dfe7c7cc49..19fca5803f59 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1204,11 +1204,16 @@ public List<byte[]> getMetaTableRows(byte[] tableName) throws IOException {
     List<byte[]> rows = new ArrayList<byte[]>();
     ResultScanner s = t.getScanner(new Scan());
     for (Result result : s) {
-      HRegionInfo info = Writables.getHRegionInfo(
-          result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
+      byte[] val = result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+      if (val == null) {
+        LOG.error("No region info for row " + Bytes.toString(result.getRow()));
+        // TODO figure out what to do for this new hosed case.
+        continue;
+      }
+      HRegionInfo info = Writables.getHRegionInfo(val);
       if (Bytes.compareTo(info.getTableName(), tableName) == 0) {
         LOG.info("getMetaTableRows: row -> " +
-            Bytes.toStringBinary(result.getRow()));
+            Bytes.toStringBinary(result.getRow()) + info);
         rows.add(result.getRow());
       }
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 937781d56cac..20e2ceea04cd 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -23,8 +23,12 @@
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -32,16 +36,27 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
@@ -54,16 +69,20 @@
  */
 @Category(MediumTests.class)
 public class TestHBaseFsck {
-  final Log LOG = LogFactory.getLog(getClass());
+  final static Log LOG = LogFactory.getLog(TestHBaseFsck.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final static Configuration conf = TEST_UTIL.getConfiguration();
   private final static byte[] FAM = Bytes.toBytes("fam");
 
   // for the instance, reset every test run
   private HTable tbl;
-  private final static byte[][] splits= new byte[][] { Bytes.toBytes("A"), 
+  private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
     Bytes.toBytes("B"), Bytes.toBytes("C") };
-  
+  // one row per region.
+  private final static byte[][] ROWKEYS= new byte[][] {
+    Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
+    Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
+
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.getConfiguration().setBoolean("hbase.master.distributed.log.splitting", false);
@@ -117,8 +136,8 @@ public void testHBaseFsck() throws Exception {
     assertErrors(doFsck(conf, true), new ERROR_CODE[]{
         ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
 
-    // fixing assignements require opening regions is not synchronous.  To make
-    // the test pass consistentyl so for now we bake in some sleep to let it
+    // fixing assignments require opening regions is not synchronous.  To make
+    // the test pass consistently so for now we bake in some sleep to let it
     // finish.  1s seems sufficient.
     Thread.sleep(1000);
 
@@ -135,6 +154,9 @@ public void testHBaseFsck() throws Exception {
     meta.close();
   }
 
+  /**
+   * Create a new region in META.
+   */
   private HRegionInfo createRegion(Configuration conf, final HTableDescriptor
       htd, byte[] startKey, byte[] endKey)
       throws IOException {
@@ -147,47 +169,102 @@ private HRegionInfo createRegion(Configuration conf, final HTableDescriptor
     return hri;
   }
 
-  public void dumpMeta(HTableDescriptor htd) throws IOException {
-    List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getName());
+  /**
+   * Debugging method to dump the contents of meta.
+   */
+  private void dumpMeta(byte[] tableName) throws IOException {
+    List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
     for (byte[] row : metaRows) {
       LOG.info(Bytes.toString(row));
     }
   }
 
-  private void deleteRegion(Configuration conf, final HTableDescriptor htd, 
-      byte[] startKey, byte[] endKey) throws IOException {
-
-    LOG.info("Before delete:");
-    dumpMeta(htd);
+  /**
+   * This method is used to undeploy a region -- close it and attempt to
+   * remove its state from the Master.
+   */
+  private void undeployRegion(HBaseAdmin admin, ServerName sn,
+      HRegionInfo hri) throws IOException, InterruptedException {
+    try {
+      HBaseFsckRepair.closeRegionSilentlyAndWait(admin, sn, hri);
+      admin.getMaster().offline(hri.getRegionName());
+    } catch (IOException ioe) {
+      LOG.warn("Got exception when attempting to offline region "
+          + Bytes.toString(hri.getRegionName()), ioe);
+    }
+  }
+  /**
+   * Delete a region from assignments, meta, or completely from hdfs.
+   * @param unassign if true unassign region if assigned
+   * @param metaRow  if true remove region's row from META
+   * @param hdfs if true remove region's dir in HDFS
+   */
+  private void deleteRegion(Configuration conf, final HTableDescriptor htd,
+      byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
+      boolean hdfs) throws IOException, InterruptedException {
+    deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false);
+  }
 
-    Map<HRegionInfo, HServerAddress> hris = tbl.getRegionsInfo();
-    for (Entry<HRegionInfo, HServerAddress> e: hris.entrySet()) {
+  /**
+   * Delete a region from assignments, meta, or completely from hdfs.
+   * @param unassign if true unassign region if assigned
+   * @param metaRow  if true remove region's row from META
+   * @param hdfs if true remove region's dir in HDFS
+   * @param regionInfoOnly if true remove a region dir's .regioninfo file
+   */
+  private void deleteRegion(Configuration conf, final HTableDescriptor htd,
+      byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
+      boolean hdfs, boolean regionInfoOnly) throws IOException, InterruptedException {
+    LOG.info("** Before delete:");
+    dumpMeta(htd.getName());
+
+    Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
+    for (Entry<HRegionInfo, ServerName> e: hris.entrySet()) {
       HRegionInfo hri = e.getKey();
-      HServerAddress hsa = e.getValue();
-      if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 
+      ServerName hsa = e.getValue();
+      if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
           && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
 
         LOG.info("RegionName: " +hri.getRegionNameAsString());
         byte[] deleteRow = hri.getRegionName();
-        TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
 
-        LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
-        Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
-        FileSystem fs = rootDir.getFileSystem(conf);
-        Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
-        fs.delete(p, true);
+        if (unassign) {
+          LOG.info("Undeploying region " + hri + " from server " + hsa);
+          undeployRegion(new HBaseAdmin(conf), hsa, hri);
+        }
+
+        if (regionInfoOnly) {
+          LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
+          Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+          FileSystem fs = rootDir.getFileSystem(conf);
+          Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
+          Path hriPath = new Path(p, HRegion.REGIONINFO_FILE);
+          fs.delete(hriPath, true);
+        }
+
+        if (hdfs) {
+          LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
+          Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+          FileSystem fs = rootDir.getFileSystem(conf);
+          Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
+          HBaseFsck.debugLsr(conf, p);
+          boolean success = fs.delete(p, true);
+          LOG.info("Deleted " + p + " sucessfully? " + success);
+          HBaseFsck.debugLsr(conf, p);
+        }
 
-        HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
-        Delete delete = new Delete(deleteRow);
-        meta.delete(delete);
+        if (metaRow) {
+          HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
+          Delete delete = new Delete(deleteRow);
+          meta.delete(delete);
+        }
       }
       LOG.info(hri.toString() + hsa.toString());
     }
 
     TEST_UTIL.getMetaTableRows(htd.getName());
-    LOG.info("After delete:");
-    dumpMeta(htd);
-
+    LOG.info("*** After delete:");
+    dumpMeta(htd.getName());
   }
 
   /**
@@ -201,11 +278,32 @@ HTable setupTable(String tablename) throws Exception {
     HTableDescriptor desc = new HTableDescriptor(tablename);
     HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
     desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
-    TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
+    TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS);
     tbl = new HTable(TEST_UTIL.getConfiguration(), tablename);
+
+    List<Put> puts = new ArrayList<Put>();
+    for (byte[] row : ROWKEYS) {
+      Put p = new Put(row);
+      p.add(FAM, Bytes.toBytes("val"), row);
+      puts.add(p);
+    }
+    tbl.put(puts);
+    tbl.flushCommits();
     return tbl;
   }
 
+  /**
+   * Counts the number of row to verify data loss or non-dataloss.
+   */
+  int countRows() throws IOException {
+     Scan s = new Scan();
+     ResultScanner rs = tbl.getScanner(s);
+     int i = 0;
+     while(rs.next() !=null) {
+       i++;
+     }
+     return i;
+  }
 
   /**
    * delete table in preparation for next test
@@ -214,14 +312,21 @@ HTable setupTable(String tablename) throws Exception {
    * @throws IOException
    */
   void deleteTable(String tablename) throws IOException {
-    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    admin.getConnection().clearRegionCache();
     byte[] tbytes = Bytes.toBytes(tablename);
-    admin.disableTable(tbytes);
+    admin.disableTableAsync(tbytes);
+    while (!admin.isTableDisabled(tbytes)) {
+      try {
+        Thread.sleep(250);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+        fail("Interrupted when trying to disable table " + tablename);
+      }
+    }
     admin.deleteTable(tbytes);
   }
 
-
-  
   /**
    * This creates a clean table and confirms that the table is clean.
    */
@@ -234,18 +339,21 @@ public void testHBaseFsckClean() throws Exception {
       assertNoErrors(hbck);
 
       setupTable(table);
-      
+      assertEquals(ROWKEYS.length, countRows());
+
       // We created 1 table, should be fine
       hbck = doFsck(conf, false);
       assertNoErrors(hbck);
       assertEquals(0, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
     } finally {
       deleteTable(table);
     }
   }
 
   /**
-   * This creates a bad table with regions that have a duplicate start key
+   * This create and fixes a bad table with regions that have a duplicate
+   * start key
    */
   @Test
   public void testDupeStartKey() throws Exception {
@@ -253,6 +361,7 @@ public void testDupeStartKey() throws Exception {
     try {
       setupTable(table);
       assertNoErrors(doFsck(conf, false));
+      assertEquals(ROWKEYS.length, countRows());
 
       // Now let's mess it up, by adding a region with a duplicate startkey
       HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
@@ -265,13 +374,112 @@ public void testDupeStartKey() throws Exception {
       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
             ERROR_CODE.DUPE_STARTKEYS});
       assertEquals(2, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won.
+
+      // fix the degenerate region.
+      doFsck(conf,true);
+
+      // check that the degenerate region is gone and no data loss
+      HBaseFsck hbck2 = doFsck(conf,false);
+      assertNoErrors(hbck2);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
     } finally {
       deleteTable(table);
     }
   }
-  
+
+  /**
+   * Get region info from local cluster.
+   */
+  Map<ServerName, List<String>> getDeployedHRIs(HBaseAdmin admin)
+    throws IOException {
+    ClusterStatus status = admin.getMaster().getClusterStatus();
+    Collection<ServerName> regionServers = status.getServers();
+    Map<ServerName, List<String>> mm =
+        new HashMap<ServerName, List<String>>();
+    HConnection connection = admin.getConnection();
+    for (ServerName hsi : regionServers) {
+      HRegionInterface server =
+        connection.getHRegionConnection(hsi.getHostname(), hsi.getPort());
+
+      // list all online regions from this region server
+      List<HRegionInfo> regions = server.getOnlineRegions();
+      List<String> regionNames = new ArrayList<String>();
+      for (HRegionInfo hri : regions) {
+        regionNames.add(hri.getRegionNameAsString());
+      }
+      mm.put(hsi, regionNames);
+    }
+    return mm;
+  }
+
+  /**
+   * Returns the HSI a region info is on.
+   */
+  ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
+    for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
+      if (e.getValue().contains(hri.getRegionNameAsString())) {
+        return e.getKey();
+      }
+    }
+    return null;
+  }
+
   /**
-   * This creates a bad table with regions that has startkey == endkey
+   * This create and fixes a bad table with regions that have a duplicate
+   * start key
+   */
+  @Test
+  public void testDupeRegion() throws Exception {
+    String table = "tableDupeRegion";
+    try {
+      setupTable(table);
+      assertNoErrors(doFsck(conf, false));
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Now let's mess it up, by adding a region with a duplicate startkey
+      HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
+          Bytes.toBytes("A"), Bytes.toBytes("B"));
+
+      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
+      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+          .waitForAssignment(hriDupe);
+
+      // Yikes! The assignment manager can't tell between diff between two
+      // different regions with the same start/endkeys since it doesn't
+      // differentiate on ts/regionId!  We actually need to recheck
+      // deployments!
+      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+      ServerName hsi;
+      while ( (hsi = findDeployedHSI(getDeployedHRIs(admin), hriDupe)) == null) {
+        Thread.sleep(250);
+      }
+
+      LOG.debug("Finished assignment of dupe region");
+
+      // TODO why is dupe region different from dupe start keys?
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
+            ERROR_CODE.DUPE_STARTKEYS});
+      assertEquals(2, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won.
+
+      // fix the degenerate region.
+      doFsck(conf,true);
+
+      // check that the degenerate region is gone and no data loss
+      HBaseFsck hbck2 = doFsck(conf,false);
+      assertNoErrors(hbck2);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates and fixes a bad table with regions that has startkey == endkey
    */
   @Test
   public void testDegenerateRegions() throws Exception {
@@ -279,6 +487,7 @@ public void testDegenerateRegions() throws Exception {
     try {
       setupTable(table);
       assertNoErrors(doFsck(conf,false));
+      assertEquals(ROWKEYS.length, countRows());
 
       // Now let's mess it up, by adding a region with a duplicate startkey
       HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
@@ -291,19 +500,111 @@ public void testDegenerateRegions() throws Exception {
       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION,
           ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS});
       assertEquals(2, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+
+      // fix the degenerate region.
+      doFsck(conf,true);
+
+      // check that the degenerate region is gone and no data loss
+      HBaseFsck hbck2 = doFsck(conf,false);
+      assertNoErrors(hbck2);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
     } finally {
       deleteTable(table);
     }
   }
 
+
   /**
-   * This creates a bad table where a start key contained in another region.
+   * This creates and fixes a bad table where a region is completely contained
+   * by another region.
+   */
+  @Test
+  public void testContainedRegionOverlap() throws Exception {
+    String table = "tableContainedRegionOverlap";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by creating an overlap in the metadata
+      HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
+          Bytes.toBytes("A2"), Bytes.toBytes("B"));
+      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
+      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+          .waitForAssignment(hriOverlap);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+          ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
+      assertEquals(2, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+
+      // fix the problem.
+      doFsck(conf, true);
+
+      // verify that overlaps are fixed
+      HBaseFsck hbck2 = doFsck(conf,false);
+      assertNoErrors(hbck2);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+       deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates and fixes a bad table where a region is completely contained
+   * by another region, and there is a hole (sort of like a bad split)
+   */
+  @Test
+  public void testOverlapAndOrphan() throws Exception {
+    String table = "tableOverlapAndOrphan";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by creating an overlap in the metadata
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
+          Bytes.toBytes("B"), true, true, false, true);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
+          Bytes.toBytes("A2"), Bytes.toBytes("B"));
+      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
+      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+          .waitForAssignment(hriOverlap);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+          ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
+          ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // fix the problem.
+      doFsck(conf, true);
+
+      // verify that overlaps are fixed
+      HBaseFsck hbck2 = doFsck(conf,false);
+      assertNoErrors(hbck2);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+       deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates and fixes a bad table where a region overlaps two regions --
+   * a start key contained in another region and its end key is contained in
+   * yet another region.
    */
   @Test
   public void testCoveredStartKey() throws Exception {
     String table = "tableCoveredStartKey";
     try {
       setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
 
       // Mess it up by creating an overlap in the metadata
       HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
@@ -317,40 +618,239 @@ public void testCoveredStartKey() throws Exception {
           ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
           ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
       assertEquals(3, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+
+      // fix the problem.
+      doFsck(conf, true);
+
+      // verify that overlaps are fixed
+      HBaseFsck hbck2 = doFsck(conf, false);
+      assertErrors(hbck2, new ERROR_CODE[0]);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
     } finally {
       deleteTable(table);
     }
   }
 
   /**
-   * This creates a bad table with a hole in meta.
+   * This creates and fixes a bad table with a missing region -- hole in meta
+   * and data missing in the fs.
    */
   @Test
-  public void testMetaHole() throws Exception {
-    String table = "tableMetaHole";
+  public void testRegionHole() throws Exception {
+    String table = "tableRegionHole";
     try {
       setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by leaving a hole in the assignment, meta, and hdfs data
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+          Bytes.toBytes("C"), true, true, true);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+          ERROR_CODE.HOLE_IN_REGION_CHAIN});
+      // holes are separate from overlap groups
+      assertEquals(0, hbck.getOverlapGroups(table).size());
+
+      // fix hole
+      doFsck(conf, true);
+
+      // check that hole fixed
+      assertNoErrors(doFsck(conf,false));
+      assertEquals(ROWKEYS.length - 2 , countRows()); // lost a region so lost a row
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates and fixes a bad table with a missing region -- hole in meta
+   * and data present but .regioinfino missing (an orphan hdfs region)in the fs.
+   */
+  @Test
+  public void testHDFSRegioninfoMissing() throws Exception {
+    String table = "tableHDFSRegioininfoMissing";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
 
       // Mess it up by leaving a hole in the meta data
-      HRegionInfo hriHole = createRegion(conf, tbl.getTableDescriptor(),
-          Bytes.toBytes("D"), Bytes.toBytes(""));
-      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriHole);
-      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
-          .waitForAssignment(hriHole);
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+          Bytes.toBytes("C"), true, true, false, true);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+          ERROR_CODE.ORPHAN_HDFS_REGION,
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
+          ERROR_CODE.HOLE_IN_REGION_CHAIN});
+      // holes are separate from overlap groups
+      assertEquals(0, hbck.getOverlapGroups(table).size());
+
+      // fix hole
+      doFsck(conf, true);
+
+      // check that hole fixed
+      assertNoErrors(doFsck(conf, false));
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates and fixes a bad table with a region that is missing meta and
+   * not assigned to a region server.
+   */
+  @Test
+  public void testNotInMetaOrDeployedHole() throws Exception {
+    String table = "tableNotInMetaOrDeployedHole";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by leaving a hole in the meta data
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+          Bytes.toBytes("C"), true, true, false); // don't rm from fs
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+      // holes are separate from overlap groups
+      assertEquals(0, hbck.getOverlapGroups(table).size());
+
+      // fix hole
+      assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // check that hole fixed
+      assertNoErrors(doFsck(conf,false));
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates fixes a bad table with a hole in meta.
+   */
+  @Test
+  public void testNotInMetaHole() throws Exception {
+    String table = "tableNotInMetaHole";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
 
+      // Mess it up by leaving a hole in the meta data
       TEST_UTIL.getHBaseAdmin().disableTable(table);
-      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""));
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+          Bytes.toBytes("C"), false, true, false); // don't rm from fs
       TEST_UTIL.getHBaseAdmin().enableTable(table);
 
       HBaseFsck hbck = doFsck(conf, false);
-      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN });
+      assertErrors(hbck, new ERROR_CODE[] {
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
       // holes are separate from overlap groups
       assertEquals(0, hbck.getOverlapGroups(table).size());
+
+      // fix hole
+      assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // check that hole fixed
+      assertNoErrors(doFsck(conf,false));
+      assertEquals(ROWKEYS.length, countRows());
     } finally {
       deleteTable(table);
     }
   }
 
+  /**
+   * This creates and fixes a bad table with a region that is in meta but has
+   * no deployment or data hdfs
+   */
+  @Test
+  public void testNotInHdfs() throws Exception {
+    String table = "tableNotInHdfs";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table);
+
+      // Mess it up by leaving a hole in the hdfs data
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+          Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
+      // holes are separate from overlap groups
+      assertEquals(0, hbck.getOverlapGroups(table).size());
+
+      // fix hole
+      doFsck(conf, true);
+
+      // check that hole fixed
+      assertNoErrors(doFsck(conf,false));
+      assertEquals(ROWKEYS.length - 2, countRows());
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates entries in META with no hdfs data.  This should cleanly
+   * remove the table.
+   */
+  @Test
+  public void testNoHdfsTable() throws Exception {
+    String table = "NoHdfsTable";
+    setupTable(table);
+    assertEquals(ROWKEYS.length, countRows());
+
+    // make sure data in regions, if in hlog only there is no data loss
+    TEST_UTIL.getHBaseAdmin().flush(table);
+
+    // Mess it up by leaving a giant hole in meta
+    deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
+        Bytes.toBytes("A"), false, false, true); // don't rm meta
+    deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
+        Bytes.toBytes("B"), false, false, true); // don't rm meta
+    deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+        Bytes.toBytes("C"), false, false, true); // don't rm meta
+    deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
+        Bytes.toBytes(""), false, false, true); // don't rm meta
+
+    HBaseFsck hbck = doFsck(conf, false);
+    assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
+        ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
+        ERROR_CODE.NOT_IN_HDFS,});
+    // holes are separate from overlap groups
+    assertEquals(0, hbck.getOverlapGroups(table).size());
+
+    // fix hole
+    doFsck(conf, true); // in 0.92+, meta entries auto create regiondirs
+
+    // check that hole fixed
+    assertNoErrors(doFsck(conf,false));
+
+    try {
+      assertEquals(0, countRows());
+    } catch (IOException ioe) {
+      // we've actually deleted the table already. :)
+      return;
+    }
+    fail("Should have failed with IOException");
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
index 0599da13eb45..00d2440181b6 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
@@ -96,14 +96,6 @@ public void testAbsEndKey() {
     assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
   }
 
-  @Test
-  public void testTiebreaker() {
-    HbckInfo hi1 = genHbckInfo(table, keyA, keyC, 0);
-    HbckInfo hi2 = genHbckInfo(table, keyA, keyC, 1);
-    assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0);
-    assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
-  }
-
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index dbb97f8f2bd3..75d24da43432 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -19,6 +19,7 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -28,18 +29,29 @@
 
 public class HbckTestingUtil {
   public static HBaseFsck doFsck(Configuration conf, boolean fix) throws Exception {
+    return doFsck(conf, fix, fix, fix, fix,fix);
+  }
+
+  public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
+      boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
+      boolean fixHdfsOrphans) throws Exception {
     HBaseFsck fsck = new HBaseFsck(conf);
     fsck.connect();
-    fsck.displayFullReport(); // i.e. -details
+    fsck.setDisplayFullReport(); // i.e. -details
     fsck.setTimeLag(0);
-    fsck.setFixErrors(fix);
-    fsck.doWork();
+    fsck.setFixAssignments(fixAssignments);
+    fsck.setFixMeta(fixMeta);
+    fsck.setFixHdfsHoles(fixHdfsHoles);
+    fsck.setFixHdfsOverlaps(fixHdfsOverlaps);
+    fsck.setFixHdfsOrphans(fixHdfsOrphans);
+    fsck.onlineHbck();
     return fsck;
   }
 
+
   public static void assertNoErrors(HBaseFsck fsck) throws Exception {
     List<ERROR_CODE> errs = fsck.getErrors().getErrorList();
-    assertEquals(0, errs.size());
+    assertEquals(new ArrayList<ERROR_CODE>(), errs);
   }
 
   public static void assertErrors(HBaseFsck fsck, ERROR_CODE[] expectedErrors) {
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
index 2b4cac86cb76..bb8b0e7b3ae8 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
@@ -61,7 +61,7 @@ public void testMetaRebuild() throws Exception {
 
     // rebuild meta table from scratch
     HBaseFsck fsck = new HBaseFsck(conf);
-    assertTrue(fsck.rebuildMeta());
+    assertTrue(fsck.rebuildMeta(false));
 
     // bring up the minicluster
     TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
index ebbeeada0d36..d5274986696b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
@@ -21,6 +21,7 @@
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.util.Arrays;
 
@@ -64,7 +65,7 @@ public void testMetaRebuildHoleFail() throws Exception {
 
     // attempt to rebuild meta table from scratch
     HBaseFsck fsck = new HBaseFsck(conf);
-    assertFalse(fsck.rebuildMeta());
+    assertFalse(fsck.rebuildMeta(false));
 
     // bring up the minicluster
     TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
index b17554819a98..67d7c83afed1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
@@ -69,7 +69,7 @@ public void testMetaRebuildOverlapFail() throws Exception {
 
     // attempt to rebuild meta table from scratch
     HBaseFsck fsck = new HBaseFsck(conf);
-    assertFalse(fsck.rebuildMeta());
+    assertFalse(fsck.rebuildMeta(false));
 
     Multimap<byte[], HbckInfo> problems = fsck.getOverlapGroups(table);
     assertEquals(1, problems.keySet().size());

From 0ecb72e084a16c9c0afbd6ef3d69db6f2e647db2 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Sat, 24 Mar 2012 07:40:18 +0000
Subject: [PATCH 0067/1540] HBASE-5128 Addendum adds two missing new files

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304722 13f79535-47bb-0310-9956-ffa450edef68
---
 .../util/hbck/TableIntegrityErrorHandler.java | 92 ++++++++++++++++++
 .../hbck/TableIntegrityErrorHandlerImpl.java  | 96 +++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java

diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java
new file mode 100644
index 000000000000..2d49d01af10b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util.hbck;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
+import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
+
+/**
+ * This interface provides callbacks for handling particular table integrity
+ * invariant violations.  This could probably be boiled down to handling holes
+ * and handling overlaps but currently preserves the older more specific error
+ * condition codes.
+ */
+public interface TableIntegrityErrorHandler {
+
+  TableInfo getTableInfo();
+
+  /**
+   * Set the TableInfo used by all HRegionInfos fabricated by other callbacks
+   */
+  void setTableInfo(TableInfo ti);
+
+  /**
+   * Callback for handling case where a Table has a first region that does not
+   * have an empty start key.
+   *
+   * @param hi An HbckInfo of the second region in a table.  This should have
+   *    a non-empty startkey, and can be used to fabricate a first region that
+   *    has an empty start key.
+   */
+  void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException;
+
+  /**
+   * Callback for handling a region that has the same start and end key.
+   *
+   * @param hi An HbckInfo for a degenerate key.
+   */
+  void handleDegenerateRegion(HbckInfo hi) throws IOException;
+
+  /**
+   * Callback for handling two regions that have the same start key.  This is
+   * a specific case of a region overlap.
+   * @param hi1 one of the overlapping HbckInfo 
+   * @param hi2 the other overlapping HbckInfo
+   */
+  void handleDuplicateStartKeys(HbckInfo hi1, HbckInfo hi2) throws IOException;
+
+  /**
+   * Callback for handling two reigons that overlap in some arbitrary way.
+   * This is a specific case of region overlap, and called for each possible
+   * pair. If two regions have the same start key, the handleDuplicateStartKeys
+   * method is called.
+   * @param hi1 one of the overlapping HbckInfo
+   * @param hi2 the other overlapping HbckInfo
+   */
+  void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2)
+      throws IOException;
+
+  /**
+   * Callback for handling a region hole between two keys.
+   * @param holeStartKey key at the beginning of the region hole
+   * @param holeEndKey key at the end of the region hole
+   
+   */
+  void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeEndKey)
+      throws IOException;
+
+  /**
+   * Callback for handling an group of regions that overlap.
+   * @param overlap Collection of overlapping regions.
+   */
+  void handleOverlapGroup(Collection<HbckInfo> overlap) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java
new file mode 100644
index 000000000000..2c34da806d9e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util.hbck;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
+import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
+
+/**
+ * Simple implementation of TableIntegrityErrorHandler. Can be used as a base
+ * class.
+ */
+abstract public class TableIntegrityErrorHandlerImpl implements
+    TableIntegrityErrorHandler {
+  TableInfo ti;
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public TableInfo getTableInfo() {
+    return ti;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void setTableInfo(TableInfo ti2) {
+    this.ti = ti2;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleDegenerateRegion(HbckInfo hi) throws IOException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleDuplicateStartKeys(HbckInfo hi1, HbckInfo hi2)
+      throws IOException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2)
+      throws IOException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleHoleInRegionChain(byte[] holeStart, byte[] holeEnd)
+      throws IOException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleOverlapGroup(Collection<HbckInfo> overlap)
+      throws IOException {
+  }
+
+}

From 01a8ca5f79c13419ad24471e082c5c57d3e9786c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 24 Mar 2012 21:04:36 +0000
Subject: [PATCH 0068/1540] HBASE-5434 [REST] Include more metrics in cluster
 status request (Mubarak Seyed)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304918 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HServerLoad.java  |  22 +
 .../rest/StorageClusterStatusResource.java    |   6 +-
 .../rest/model/StorageClusterStatusModel.java | 177 +++++++-
 .../StorageClusterStatusMessage.java          | 422 +++++++++++++++++-
 .../apache/hadoop/hbase/rest/XMLSchema.xsd    |   7 +
 .../StorageClusterStatusMessage.proto         |   7 +
 .../model/TestStorageClusterStatusModel.java  |  40 +-
 7 files changed, 657 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
index acded62dbadb..e6d5346ba567 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
@@ -270,7 +270,29 @@ public long getReadRequestsCount() {
     public long getWriteRequestsCount() {
       return writeRequestsCount;
     }
+    
+    /**
+     * @return The current total size of root-level indexes for the region, in KB.
+     */
+    public int getRootIndexSizeKB() {
+      return rootIndexSizeKB;
+    }
+    
+    /**
+     * @return The total size of all index blocks, not just the root level, in KB.
+     */
+    public int getTotalStaticIndexSizeKB() {
+      return totalStaticIndexSizeKB;
+    }
 
+    /**
+     * @return The total size of all Bloom filter blocks, not just loaded into the
+     * block cache, in KB.
+     */
+    public int getTotalStaticBloomSizeKB() {
+      return totalStaticBloomSizeKB;
+    }
+    
     /**
      * @return the total number of kvs in current compaction
      */
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
index e48562725ee0..bba0ae187435 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
@@ -85,7 +85,11 @@ public Response get(final @Context UriInfo uriInfo) {
         for (HServerLoad.RegionLoad region: load.getRegionsLoad().values()) {
           node.addRegion(region.getName(), region.getStores(),
             region.getStorefiles(), region.getStorefileSizeMB(),
-            region.getMemStoreSizeMB(), region.getStorefileIndexSizeMB());
+            region.getMemStoreSizeMB(), region.getStorefileIndexSizeMB(),
+            region.getReadRequestsCount(), region.getWriteRequestsCount(),
+            region.getRootIndexSizeKB(), region.getTotalStaticIndexSizeKB(),
+            region.getTotalStaticBloomSizeKB(), region.getTotalCompactingKVs(),
+            region.getCurrentCompactedKVs());
         }
       }
       for (ServerName name: status.getDeadServerNames()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
index f45e902a6de4..c976c31c1494 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
@@ -82,6 +82,13 @@
  *   &lt;attribute name="storefileSizeMB" type="int"&gt;&lt;/attribute&gt;
  *   &lt;attribute name="memstoreSizeMB" type="int"&gt;&lt;/attribute&gt;
  *   &lt;attribute name="storefileIndexSizeMB" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="readRequestsCount" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="writeRequestsCount" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="rootIndexSizeKB" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="totalStaticIndexSizeKB" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="totalStaticBloomSizeKB" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="totalCompactingKVs" type="int"&gt;&lt;/attribute&gt;
+ *   &lt;attribute name="currentCompactedKVs" type="int"&gt;&lt;/attribute&gt;
  * &lt;/complexType&gt;
  * </pre>
  */
@@ -105,6 +112,13 @@ public static class Region {
 	    private int storefileSizeMB;
 	    private int memstoreSizeMB;
 	    private int storefileIndexSizeMB;
+	    private long readRequestsCount;
+	    private long writeRequestsCount;
+	    private int rootIndexSizeKB;
+	    private int totalStaticIndexSizeKB;
+	    private int totalStaticBloomSizeKB;
+	    private long totalCompactingKVs;
+	    private long currentCompactedKVs;
 
 	    /**
 	     * Default constructor
@@ -129,13 +143,23 @@ public Region(byte[] name) {
 	     * @param storefileIndexSizeMB total size of store file indexes, in MB
 	     */
 	    public Region(byte[] name, int stores, int storefiles,
-          int storefileSizeMB, int memstoreSizeMB, int storefileIndexSizeMB) {
+          int storefileSizeMB, int memstoreSizeMB, int storefileIndexSizeMB,
+          long readRequestsCount, long writeRequestsCount, int rootIndexSizeKB,
+          int totalStaticIndexSizeKB, int totalStaticBloomSizeKB,
+          long totalCompactingKVs, long currentCompactedKVs) {
         this.name = name;
         this.stores = stores;
         this.storefiles = storefiles;
         this.storefileSizeMB = storefileSizeMB;
         this.memstoreSizeMB = memstoreSizeMB;
         this.storefileIndexSizeMB = storefileIndexSizeMB;
+        this.readRequestsCount = readRequestsCount;
+        this.writeRequestsCount = writeRequestsCount;
+        this.rootIndexSizeKB = rootIndexSizeKB;
+        this.totalStaticIndexSizeKB = totalStaticIndexSizeKB;
+        this.totalStaticBloomSizeKB = totalStaticBloomSizeKB;
+        this.totalCompactingKVs = totalCompactingKVs;
+        this.currentCompactedKVs = currentCompactedKVs;
       }
 
       /**
@@ -185,7 +209,117 @@ public int getMemstoreSizeMB() {
       public int getStorefileIndexSizeMB() {
         return storefileIndexSizeMB;
       }
+      
+      /**
+       * @return the current total read requests made to region
+       */
+      @XmlAttribute
+      public long getReadRequestsCount() {
+        return readRequestsCount;
+      }
+
+      /**
+       * @return the current total write requests made to region
+       */
+      @XmlAttribute
+      public long getWriteRequestsCount() {
+        return writeRequestsCount;
+      }
+
+      /**
+       * @return The current total size of root-level indexes for the region, in KB.
+       */
+      @XmlAttribute
+      public int getRootIndexSizeKB() {
+        return rootIndexSizeKB;
+      }
+
+      /**
+       * @return The total size of static index, in KB
+       */
+      @XmlAttribute
+      public int getTotalStaticIndexSizeKB() {
+        return totalStaticIndexSizeKB;
+      }
+
+      /**
+       * @return The total size of static bloom, in KB
+       */
+      @XmlAttribute
+      public int getTotalStaticBloomSizeKB() {
+        return totalStaticBloomSizeKB;
+      }
+      
+      /**
+       * @return The total number of compacting key-values 
+       */
+      @XmlAttribute
+      public long getTotalCompactingKVs() {
+        return totalCompactingKVs;
+      }
 
+      /**
+       * @return The number of current compacted key-values
+       */
+      @XmlAttribute
+      public long getCurrentCompactedKVs() {
+        return currentCompactedKVs;
+      }
+
+      /**
+       * @param readRequestsCount The current total read requests made to region
+       */
+      public void setReadRequestsCount(long readRequestsCount) {
+        this.readRequestsCount = readRequestsCount;
+      }
+      
+      /**
+       * @param rootIndexSizeKB The current total size of root-level indexes
+       *                        for the region, in KB
+       */
+      public void setRootIndexSizeKB(int rootIndexSizeKB) {
+        this.rootIndexSizeKB = rootIndexSizeKB;
+      }
+      
+      /**
+       * @param writeRequestsCount The current total write requests made to region
+       */
+      public void setWriteRequestsCount(long writeRequestsCount) {
+        this.writeRequestsCount = writeRequestsCount;
+      }
+      
+      /**
+       * @param currentCompactedKVs The completed count of key values
+       *                            in currently running compaction
+       */
+      public void setCurrentCompactedKVs(long currentCompactedKVs) {
+        this.currentCompactedKVs = currentCompactedKVs;
+      }
+      
+      /**
+       * @param totalCompactingKVs The total compacting key values
+       *                           in currently running compaction
+       */
+      public void setTotalCompactingKVs(long totalCompactingKVs) {
+        this.totalCompactingKVs = totalCompactingKVs;
+      }
+      
+      /**
+       * @param totalStaticBloomSizeKB The total size of all Bloom filter blocks,
+       *                               not just loaded into the block cache, in KB.
+       */
+      public void setTotalStaticBloomSizeKB(int totalStaticBloomSizeKB) {
+        this.totalStaticBloomSizeKB = totalStaticBloomSizeKB;
+      }
+      
+      /**
+       * @param totalStaticIndexSizeKB The total size of all index blocks,
+       *                               not just the root level, in KB.
+       */
+      public void setTotalStaticIndexSizeKB(int totalStaticIndexSizeKB) {
+        this.totalStaticIndexSizeKB = totalStaticIndexSizeKB;
+      }
+      
       /**
 	     * @param name the region name
 	     */
@@ -241,9 +375,14 @@ public void setStorefileIndexSizeMB(int storefileIndexSizeMB) {
      * @param name the region name
      */
     public void addRegion(byte[] name, int stores, int storefiles,
-        int storefileSizeMB, int memstoreSizeMB, int storefileIndexSizeMB) {
+        int storefileSizeMB, int memstoreSizeMB, int storefileIndexSizeMB,
+        long readRequestsCount, long writeRequestsCount, int rootIndexSizeKB,
+        int totalStaticIndexSizeKB, int totalStaticBloomSizeKB,
+        long totalCompactingKVs, long currentCompactedKVs) { 
       regions.add(new Region(name, stores, storefiles, storefileSizeMB,
-        memstoreSizeMB, storefileIndexSizeMB));
+        memstoreSizeMB, storefileIndexSizeMB, readRequestsCount,
+        writeRequestsCount, rootIndexSizeKB, totalStaticIndexSizeKB,
+        totalStaticBloomSizeKB, totalCompactingKVs, currentCompactedKVs));
     }
 
     /**
@@ -530,6 +669,20 @@ public String toString() {
           sb.append(region.memstoreSizeMB);
           sb.append("\n            storefileIndexSizeMB=");
           sb.append(region.storefileIndexSizeMB);
+          sb.append("\n            readRequestsCount=");
+          sb.append(region.readRequestsCount);
+          sb.append("\n            writeRequestsCount=");
+          sb.append(region.writeRequestsCount);
+          sb.append("\n            rootIndexSizeKB=");
+          sb.append(region.rootIndexSizeKB);
+          sb.append("\n            totalStaticIndexSizeKB=");
+          sb.append(region.totalStaticIndexSizeKB);
+          sb.append("\n            totalStaticBloomSizeKB=");
+          sb.append(region.totalStaticBloomSizeKB);
+          sb.append("\n            totalCompactingKVs=");
+          sb.append(region.totalCompactingKVs);
+          sb.append("\n            currentCompactedKVs=");
+          sb.append(region.currentCompactedKVs);
           sb.append('\n');
         }
         sb.append('\n');
@@ -547,7 +700,7 @@ public String toString() {
     }
 	  return sb.toString();
 	}
-
+	
   @Override
   public byte[] createProtobufOutput() {
     StorageClusterStatus.Builder builder = StorageClusterStatus.newBuilder();
@@ -571,6 +724,13 @@ public byte[] createProtobufOutput() {
         regionBuilder.setStorefileSizeMB(region.storefileSizeMB);
         regionBuilder.setMemstoreSizeMB(region.memstoreSizeMB);
         regionBuilder.setStorefileIndexSizeMB(region.storefileIndexSizeMB);
+        regionBuilder.setReadRequestsCount(region.readRequestsCount);
+        regionBuilder.setWriteRequestsCount(region.writeRequestsCount);
+        regionBuilder.setRootIndexSizeKB(region.rootIndexSizeKB);
+        regionBuilder.setTotalStaticIndexSizeKB(region.totalStaticIndexSizeKB);
+        regionBuilder.setTotalStaticBloomSizeKB(region.totalStaticBloomSizeKB);
+        regionBuilder.setTotalCompactingKVs(region.totalCompactingKVs);
+        regionBuilder.setCurrentCompactedKVs(region.currentCompactedKVs);
         nodeBuilder.addRegions(regionBuilder);
       }
       builder.addLiveNodes(nodeBuilder);
@@ -609,7 +769,14 @@ public ProtobufMessageHandler getObjectFromMessage(byte[] message)
           region.getStorefiles(),
           region.getStorefileSizeMB(),
           region.getMemstoreSizeMB(),
-          region.getStorefileIndexSizeMB());
+          region.getStorefileIndexSizeMB(),
+          region.getReadRequestsCount(),
+          region.getWriteRequestsCount(),
+          region.getRootIndexSizeKB(),
+          region.getTotalStaticIndexSizeKB(),
+          region.getTotalStaticBloomSizeKB(),
+          region.getTotalCompactingKVs(),
+          region.getCurrentCompactedKVs());
       }
     }
     for (String node: builder.getDeadNodesList()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java
index b20d6d4d2fe0..a6023b9eae2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java
@@ -92,6 +92,34 @@ public interface RegionOrBuilder
       // optional int32 storefileIndexSizeMB = 6;
       boolean hasStorefileIndexSizeMB();
       int getStorefileIndexSizeMB();
+      
+      // optional int64 readRequestsCount = 7;
+      boolean hasReadRequestsCount();
+      long getReadRequestsCount();
+      
+      // optional int64 writeRequestsCount = 8;
+      boolean hasWriteRequestsCount();
+      long getWriteRequestsCount();
+      
+      // optional int32 rootIndexSizeKB = 9;
+      boolean hasRootIndexSizeKB();
+      int getRootIndexSizeKB();
+      
+      // optional int32 totalStaticIndexSizeKB = 10;
+      boolean hasTotalStaticIndexSizeKB();
+      int getTotalStaticIndexSizeKB();
+      
+      // optional int32 totalStaticBloomSizeKB = 11;
+      boolean hasTotalStaticBloomSizeKB();
+      int getTotalStaticBloomSizeKB();
+      
+      // optional int64 totalCompactingKVs = 12;
+      boolean hasTotalCompactingKVs();
+      long getTotalCompactingKVs();
+      
+      // optional int64 currentCompactedKVs = 13;
+      boolean hasCurrentCompactedKVs();
+      long getCurrentCompactedKVs();
     }
     public static final class Region extends
         com.google.protobuf.GeneratedMessage
@@ -182,6 +210,76 @@ public int getStorefileIndexSizeMB() {
         return storefileIndexSizeMB_;
       }
       
+      // optional int64 readRequestsCount = 7;
+      public static final int READREQUESTSCOUNT_FIELD_NUMBER = 7;
+      private long readRequestsCount_;
+      public boolean hasReadRequestsCount() {
+        return ((bitField0_ & 0x00000040) == 0x00000040);
+      }
+      public long getReadRequestsCount() {
+        return readRequestsCount_;
+      }
+      
+      // optional int64 writeRequestsCount = 8;
+      public static final int WRITEREQUESTSCOUNT_FIELD_NUMBER = 8;
+      private long writeRequestsCount_;
+      public boolean hasWriteRequestsCount() {
+        return ((bitField0_ & 0x00000080) == 0x00000080);
+      }
+      public long getWriteRequestsCount() {
+        return writeRequestsCount_;
+      }
+      
+      // optional int32 rootIndexSizeKB = 9;
+      public static final int ROOTINDEXSIZEKB_FIELD_NUMBER = 9;
+      private int rootIndexSizeKB_;
+      public boolean hasRootIndexSizeKB() {
+        return ((bitField0_ & 0x00000100) == 0x00000100);
+      }
+      public int getRootIndexSizeKB() {
+        return rootIndexSizeKB_;
+      }
+      
+      // optional int32 totalStaticIndexSizeKB = 10;
+      public static final int TOTALSTATICINDEXSIZEKB_FIELD_NUMBER = 10;
+      private int totalStaticIndexSizeKB_;
+      public boolean hasTotalStaticIndexSizeKB() {
+        return ((bitField0_ & 0x00000200) == 0x00000200);
+      }
+      public int getTotalStaticIndexSizeKB() {
+        return totalStaticIndexSizeKB_;
+      }
+      
+      // optional int32 totalStaticBloomSizeKB = 11;
+      public static final int TOTALSTATICBLOOMSIZEKB_FIELD_NUMBER = 11;
+      private int totalStaticBloomSizeKB_;
+      public boolean hasTotalStaticBloomSizeKB() {
+        return ((bitField0_ & 0x00000400) == 0x00000400);
+      }
+      public int getTotalStaticBloomSizeKB() {
+        return totalStaticBloomSizeKB_;
+      }
+      
+      // optional int64 totalCompactingKVs = 12;
+      public static final int TOTALCOMPACTINGKVS_FIELD_NUMBER = 12;
+      private long totalCompactingKVs_;
+      public boolean hasTotalCompactingKVs() {
+        return ((bitField0_ & 0x00000800) == 0x00000800);
+      }
+      public long getTotalCompactingKVs() {
+        return totalCompactingKVs_;
+      }
+      
+      // optional int64 currentCompactedKVs = 13;
+      public static final int CURRENTCOMPACTEDKVS_FIELD_NUMBER = 13;
+      private long currentCompactedKVs_;
+      public boolean hasCurrentCompactedKVs() {
+        return ((bitField0_ & 0x00001000) == 0x00001000);
+      }
+      public long getCurrentCompactedKVs() {
+        return currentCompactedKVs_;
+      }
+      
       private void initFields() {
         name_ = com.google.protobuf.ByteString.EMPTY;
         stores_ = 0;
@@ -189,6 +287,13 @@ private void initFields() {
         storefileSizeMB_ = 0;
         memstoreSizeMB_ = 0;
         storefileIndexSizeMB_ = 0;
+        readRequestsCount_ = 0L;
+        writeRequestsCount_ = 0L;
+        rootIndexSizeKB_ = 0;
+        totalStaticIndexSizeKB_ = 0;
+        totalStaticBloomSizeKB_ = 0;
+        totalCompactingKVs_ = 0L;
+        currentCompactedKVs_ = 0L;
       }
       private byte memoizedIsInitialized = -1;
       public final boolean isInitialized() {
@@ -224,6 +329,27 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
         if (((bitField0_ & 0x00000020) == 0x00000020)) {
           output.writeInt32(6, storefileIndexSizeMB_);
         }
+        if (((bitField0_ & 0x00000040) == 0x00000040)) {
+          output.writeInt64(7, readRequestsCount_);
+        }
+        if (((bitField0_ & 0x00000080) == 0x00000080)) {
+          output.writeInt64(8, writeRequestsCount_);
+        }
+        if (((bitField0_ & 0x00000100) == 0x00000100)) {
+          output.writeInt32(9, rootIndexSizeKB_);
+        }
+        if (((bitField0_ & 0x00000200) == 0x00000200)) {
+          output.writeInt32(10, totalStaticIndexSizeKB_);
+        }
+        if (((bitField0_ & 0x00000400) == 0x00000400)) {
+          output.writeInt32(11, totalStaticBloomSizeKB_);
+        }
+        if (((bitField0_ & 0x00000800) == 0x00000800)) {
+          output.writeInt64(12, totalCompactingKVs_);
+        }
+        if (((bitField0_ & 0x00001000) == 0x00001000)) {
+          output.writeInt64(13, currentCompactedKVs_);
+        }
         getUnknownFields().writeTo(output);
       }
       
@@ -257,6 +383,34 @@ public int getSerializedSize() {
           size += com.google.protobuf.CodedOutputStream
             .computeInt32Size(6, storefileIndexSizeMB_);
         }
+        if (((bitField0_ & 0x00000040) == 0x00000040)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt64Size(7, readRequestsCount_);
+        }
+        if (((bitField0_ & 0x00000080) == 0x00000080)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt64Size(8, writeRequestsCount_);
+        }
+        if (((bitField0_ & 0x00000100) == 0x00000100)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt32Size(9, rootIndexSizeKB_);
+        }
+        if (((bitField0_ & 0x00000200) == 0x00000200)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt32Size(10, totalStaticIndexSizeKB_);
+        }
+        if (((bitField0_ & 0x00000400) == 0x00000400)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt32Size(11, totalStaticBloomSizeKB_);
+        }
+        if (((bitField0_ & 0x00000800) == 0x00000800)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt64Size(12, totalCompactingKVs_);
+        }
+        if (((bitField0_ & 0x00001000) == 0x00001000)) {
+          size += com.google.protobuf.CodedOutputStream
+            .computeInt64Size(13, currentCompactedKVs_);
+        }
         size += getUnknownFields().getSerializedSize();
         memoizedSerializedSize = size;
         return size;
@@ -393,6 +547,20 @@ public Builder clear() {
           bitField0_ = (bitField0_ & ~0x00000010);
           storefileIndexSizeMB_ = 0;
           bitField0_ = (bitField0_ & ~0x00000020);
+          readRequestsCount_ = 0L;
+          bitField0_ = (bitField0_ & ~0x00000040);
+          writeRequestsCount_ = 0L;
+          bitField0_ = (bitField0_ & ~0x00000080);
+          rootIndexSizeKB_ = 0;
+          bitField0_ = (bitField0_ & ~0x00000100);
+          totalStaticIndexSizeKB_ = 0;
+          bitField0_ = (bitField0_ & ~0x00000200);
+          totalStaticBloomSizeKB_ = 0;
+          bitField0_ = (bitField0_ & ~0x00000400);
+          totalCompactingKVs_ = 0L;
+          bitField0_ = (bitField0_ & ~0x00000800);
+          currentCompactedKVs_ = 0L;
+          bitField0_ = (bitField0_ & ~0x00001000);
           return this;
         }
         
@@ -455,6 +623,34 @@ public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessa
             to_bitField0_ |= 0x00000020;
           }
           result.storefileIndexSizeMB_ = storefileIndexSizeMB_;
+          if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+            to_bitField0_ |= 0x00000040;
+          }
+          result.readRequestsCount_ = readRequestsCount_;
+          if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+            to_bitField0_ |= 0x00000080;
+          }
+          result.writeRequestsCount_ = writeRequestsCount_;
+          if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+            to_bitField0_ |= 0x00000100;
+          }
+          result.rootIndexSizeKB_ = rootIndexSizeKB_;
+          if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+            to_bitField0_ |= 0x00000200;
+          }
+          result.totalStaticIndexSizeKB_ = totalStaticIndexSizeKB_;
+          if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+            to_bitField0_ |= 0x00000400;
+          }
+          result.totalStaticBloomSizeKB_ = totalStaticBloomSizeKB_;
+          if (((from_bitField0_ & 0x00000800) == 0x00000800)) {
+            to_bitField0_ |= 0x00000800;
+          }
+          result.totalCompactingKVs_ = totalCompactingKVs_;
+          if (((from_bitField0_ & 0x00001000) == 0x00001000)) {
+            to_bitField0_ |= 0x00001000;
+          }
+          result.currentCompactedKVs_ = currentCompactedKVs_;
           result.bitField0_ = to_bitField0_;
           onBuilt();
           return result;
@@ -489,6 +685,27 @@ public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.Storage
           if (other.hasStorefileIndexSizeMB()) {
             setStorefileIndexSizeMB(other.getStorefileIndexSizeMB());
           }
+          if (other.hasReadRequestsCount()) {
+            setReadRequestsCount(other.getReadRequestsCount());
+          }
+          if (other.hasWriteRequestsCount()) {
+            setWriteRequestsCount(other.getWriteRequestsCount());
+          }
+          if (other.hasRootIndexSizeKB()) {
+            setRootIndexSizeKB(other.getRootIndexSizeKB());
+          }
+          if (other.hasTotalStaticIndexSizeKB()) {
+            setTotalStaticIndexSizeKB(other.getTotalStaticIndexSizeKB());
+          }
+          if (other.hasTotalStaticBloomSizeKB()) {
+            setTotalStaticBloomSizeKB(other.getTotalStaticBloomSizeKB());
+          }
+          if (other.hasTotalCompactingKVs()) {
+            setTotalCompactingKVs(other.getTotalCompactingKVs());
+          }
+          if (other.hasCurrentCompactedKVs()) {
+            setCurrentCompactedKVs(other.getCurrentCompactedKVs());
+          }
           this.mergeUnknownFields(other.getUnknownFields());
           return this;
         }
@@ -554,6 +771,41 @@ public Builder mergeFrom(
                 storefileIndexSizeMB_ = input.readInt32();
                 break;
               }
+              case 56: {
+                bitField0_ |= 0x00000040;
+                readRequestsCount_ = input.readInt64();
+                break;
+              }
+              case 64: {
+                bitField0_ |= 0x00000080;
+                writeRequestsCount_ = input.readInt64();
+                break;
+              }
+              case 72: {
+                bitField0_ |= 0x00000100;
+                rootIndexSizeKB_ = input.readInt32();
+                break;
+              }
+              case 80: {
+                bitField0_ |= 0x00000200;
+                totalStaticIndexSizeKB_ = input.readInt32();
+                break;
+              }
+              case 88: {
+                bitField0_ |= 0x00000400;
+                totalStaticBloomSizeKB_ = input.readInt32();
+                break;
+              }
+              case 96: {
+                bitField0_ |= 0x00000800;
+                totalCompactingKVs_ = input.readInt64();
+                break;
+              }
+              case 104: {
+                bitField0_ |= 0x00001000;
+                currentCompactedKVs_ = input.readInt64();
+                break;
+              }
             }
           }
         }
@@ -689,6 +941,153 @@ public Builder clearStorefileIndexSizeMB() {
           return this;
         }
         
+        // optional int64 readRequestsCount = 7;
+        private long readRequestsCount_ ;
+        public boolean hasReadRequestsCount() {
+          return ((bitField0_ & 0x00000040) == 0x00000040);
+        }
+        public long getReadRequestsCount() {
+          return readRequestsCount_;
+        }
+        public Builder setReadRequestsCount(long value) {
+          bitField0_ |= 0x00000040;
+          readRequestsCount_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearReadRequestsCount() {
+          bitField0_ = (bitField0_ & ~0x00000040);
+          readRequestsCount_ = 0L;
+          onChanged();
+          return this;
+        }
+        
+        // optional int64 writeRequestsCount = 8;
+        private long writeRequestsCount_ ;
+        public boolean hasWriteRequestsCount() {
+          return ((bitField0_ & 0x00000080) == 0x00000080);
+        }
+        public long getWriteRequestsCount() {
+          return writeRequestsCount_;
+        }
+        public Builder setWriteRequestsCount(long value) {
+          bitField0_ |= 0x00000080;
+          writeRequestsCount_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearWriteRequestsCount() {
+          bitField0_ = (bitField0_ & ~0x00000080);
+          writeRequestsCount_ = 0L;
+          onChanged();
+          return this;
+        }
+        
+        // optional int32 rootIndexSizeKB = 9;
+        private int rootIndexSizeKB_ ;
+        public boolean hasRootIndexSizeKB() {
+          return ((bitField0_ & 0x00000100) == 0x00000100);
+        }
+        public int getRootIndexSizeKB() {
+          return rootIndexSizeKB_;
+        }
+        public Builder setRootIndexSizeKB(int value) {
+          bitField0_ |= 0x00000100;
+          rootIndexSizeKB_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearRootIndexSizeKB() {
+          bitField0_ = (bitField0_ & ~0x00000100);
+          rootIndexSizeKB_ = 0;
+          onChanged();
+          return this;
+        }
+        
+        // optional int32 totalStaticIndexSizeKB = 10;
+        private int totalStaticIndexSizeKB_ ;
+        public boolean hasTotalStaticIndexSizeKB() {
+          return ((bitField0_ & 0x00000200) == 0x00000200);
+        }
+        public int getTotalStaticIndexSizeKB() {
+          return totalStaticIndexSizeKB_;
+        }
+        public Builder setTotalStaticIndexSizeKB(int value) {
+          bitField0_ |= 0x00000200;
+          totalStaticIndexSizeKB_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearTotalStaticIndexSizeKB() {
+          bitField0_ = (bitField0_ & ~0x00000200);
+          totalStaticIndexSizeKB_ = 0;
+          onChanged();
+          return this;
+        }
+        
+        // optional int32 totalStaticBloomSizeKB = 11;
+        private int totalStaticBloomSizeKB_ ;
+        public boolean hasTotalStaticBloomSizeKB() {
+          return ((bitField0_ & 0x00000400) == 0x00000400);
+        }
+        public int getTotalStaticBloomSizeKB() {
+          return totalStaticBloomSizeKB_;
+        }
+        public Builder setTotalStaticBloomSizeKB(int value) {
+          bitField0_ |= 0x00000400;
+          totalStaticBloomSizeKB_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearTotalStaticBloomSizeKB() {
+          bitField0_ = (bitField0_ & ~0x00000400);
+          totalStaticBloomSizeKB_ = 0;
+          onChanged();
+          return this;
+        }
+        
+        // optional int64 totalCompactingKVs = 12;
+        private long totalCompactingKVs_ ;
+        public boolean hasTotalCompactingKVs() {
+          return ((bitField0_ & 0x00000800) == 0x00000800);
+        }
+        public long getTotalCompactingKVs() {
+          return totalCompactingKVs_;
+        }
+        public Builder setTotalCompactingKVs(long value) {
+          bitField0_ |= 0x00000800;
+          totalCompactingKVs_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearTotalCompactingKVs() {
+          bitField0_ = (bitField0_ & ~0x00000800);
+          totalCompactingKVs_ = 0L;
+          onChanged();
+          return this;
+        }
+        
+        // optional int64 currentCompactedKVs = 13;
+        private long currentCompactedKVs_ ;
+        public boolean hasCurrentCompactedKVs() {
+          return ((bitField0_ & 0x00001000) == 0x00001000);
+        }
+        public long getCurrentCompactedKVs() {
+          return currentCompactedKVs_;
+        }
+        public Builder setCurrentCompactedKVs(long value) {
+          bitField0_ |= 0x00001000;
+          currentCompactedKVs_ = value;
+          onChanged();
+          return this;
+        }
+        public Builder clearCurrentCompactedKVs() {
+          bitField0_ = (bitField0_ & ~0x00001000);
+          currentCompactedKVs_ = 0L;
+          onChanged();
+          return this;
+        }
+        
         // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Region)
       }
       
@@ -2412,20 +2811,25 @@ public Builder clearAverageLoad() {
     java.lang.String[] descriptorData = {
       "\n!StorageClusterStatusMessage.proto\022/org" +
       ".apache.hadoop.hbase.rest.protobuf.gener" +
-      "ated\"\222\004\n\024StorageClusterStatus\022]\n\tliveNod" +
+      "ated\"\333\005\n\024StorageClusterStatus\022]\n\tliveNod" +
       "es\030\001 \003(\0132J.org.apache.hadoop.hbase.rest." +
       "protobuf.generated.StorageClusterStatus." +
       "Node\022\021\n\tdeadNodes\030\002 \003(\t\022\017\n\007regions\030\003 \001(\005" +
       "\022\020\n\010requests\030\004 \001(\005\022\023\n\013averageLoad\030\005 \001(\001\032" +
-      "\211\001\n\006Region\022\014\n\004name\030\001 \002(\014\022\016\n\006stores\030\002 \001(\005" +
+      "\322\002\n\006Region\022\014\n\004name\030\001 \002(\014\022\016\n\006stores\030\002 \001(\005" +
       "\022\022\n\nstorefiles\030\003 \001(\005\022\027\n\017storefileSizeMB\030" +
       "\004 \001(\005\022\026\n\016memstoreSizeMB\030\005 \001(\005\022\034\n\024storefi",
-      "leIndexSizeMB\030\006 \001(\005\032\303\001\n\004Node\022\014\n\004name\030\001 \002" +
-      "(\t\022\021\n\tstartCode\030\002 \001(\003\022\020\n\010requests\030\003 \001(\005\022" +
-      "\022\n\nheapSizeMB\030\004 \001(\005\022\025\n\rmaxHeapSizeMB\030\005 \001" +
-      "(\005\022]\n\007regions\030\006 \003(\0132L.org.apache.hadoop." +
-      "hbase.rest.protobuf.generated.StorageClu" +
-      "sterStatus.Region"
+      "leIndexSizeMB\030\006 \001(\005\022\031\n\021readRequestsCount" +
+      "\030\007 \001(\003\022\032\n\022writeRequestsCount\030\010 \001(\003\022\027\n\017ro" +
+      "otIndexSizeKB\030\t \001(\005\022\036\n\026totalStaticIndexS" +
+      "izeKB\030\n \001(\005\022\036\n\026totalStaticBloomSizeKB\030\013 " +
+      "\001(\005\022\032\n\022totalCompactingKVs\030\014 \001(\003\022\033\n\023curre" +
+      "ntCompactedKVs\030\r \001(\003\032\303\001\n\004Node\022\014\n\004name\030\001 " +
+      "\002(\t\022\021\n\tstartCode\030\002 \001(\003\022\020\n\010requests\030\003 \001(\005" +
+      "\022\022\n\nheapSizeMB\030\004 \001(\005\022\025\n\rmaxHeapSizeMB\030\005 " +
+      "\001(\005\022]\n\007regions\030\006 \003(\0132L.org.apache.hadoop" +
+      ".hbase.rest.protobuf.generated.StorageCl",
+      "usterStatus.Region"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -2445,7 +2849,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors(
           internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_descriptor,
-              new java.lang.String[] { "Name", "Stores", "Storefiles", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", },
+              new java.lang.String[] { "Name", "Stores", "Storefiles", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", "ReadRequestsCount", "WriteRequestsCount", "RootIndexSizeKB", "TotalStaticIndexSizeKB", "TotalStaticBloomSizeKB", "TotalCompactingKVs", "CurrentCompactedKVs", },
               org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.class,
               org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder.class);
           internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_descriptor =
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd b/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
index de4fff1761cd..c2df60ddaa7e 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
@@ -166,6 +166,13 @@
         <attribute name="storefileSizeMB" type="int"></attribute>
         <attribute name="memstoreSizeMB" type="int"></attribute>
         <attribute name="storefileIndexSizeMB" type="int"></attribute>
+        <attribute name="readRequestsCount" type="int"></attribute>
+        <attribute name="writeRequestsCount" type="int"></attribute>
+        <attribute name="rootIndexSizeKB" type="int"></attribute>
+        <attribute name="totalStaticIndexSizeKB" type="int"></attribute>
+        <attribute name="totalStaticBloomSizeKB" type="int"></attribute>
+        <attribute name="totalCompactingKVs" type="int"></attribute>
+        <attribute name="currentCompactedKVs" type="int"></attribute>
     </complexType>
 
 </schema>
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto
index 2b032f7f4ac2..46e275d9cc56 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto
@@ -26,6 +26,13 @@ message StorageClusterStatus {
     optional int32 storefileSizeMB = 4;
     optional int32 memstoreSizeMB = 5;
     optional int32 storefileIndexSizeMB = 6;
+    optional int64 readRequestsCount = 7;
+    optional int64 writeRequestsCount = 8;
+    optional int32 rootIndexSizeKB = 9;
+    optional int32 totalStaticIndexSizeKB = 10;
+    optional int32 totalStaticBloomSizeKB = 11;
+    optional int64 totalCompactingKVs = 12;
+    optional int64 currentCompactedKVs = 13;
   }
   message Node {
     required string name = 1;    // name:port
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java
index 32e91d921f75..c44f720484e2 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java
@@ -45,19 +45,25 @@ public class TestStorageClusterStatusModel extends TestCase {
       " name=\"test1\" maxHeapSizeMB=\"1024\" heapSizeMB=\"128\">" + 
         "<Region stores=\"1\" storefiles=\"1\" storefileSizeMB=\"0\"" + 
         " storefileIndexSizeMB=\"0\" name=\"LVJPT1QtLCww\"" + 
-        " memstoreSizeMB=\"0\"/></Node>" + 
+        " memstoreSizeMB=\"0\" readRequestsCount=\"1\"" +
+        " writeRequestsCount=\"2\" rootIndexSizeKB=\"1\"" +
+        " totalStaticIndexSizeKB=\"1\" totalStaticBloomSizeKB=\"1\"" +
+        " totalCompactingKVs=\"1\" currentCompactedKVs=\"1\"/></Node>" + 
       "<Node startCode=\"1245239331198\" requests=\"0\" name=\"test2\"" + 
         " maxHeapSizeMB=\"1024\" heapSizeMB=\"512\">" + 
         "<Region stores=\"1\" storefiles=\"1\" storefileSizeMB=\"0\"" +
         " storefileIndexSizeMB=\"0\" name=\"Lk1FVEEuLCwxMjQ2MDAwMDQzNzI0\"" +
-        " memstoreSizeMB=\"0\"/></Node>"+
+        " memstoreSizeMB=\"0\" readRequestsCount=\"1\"" +
+        " writeRequestsCount=\"2\" rootIndexSizeKB=\"1\"" +
+        " totalStaticIndexSizeKB=\"1\" totalStaticBloomSizeKB=\"1\"" +
+        " totalCompactingKVs=\"1\" currentCompactedKVs=\"1\"/></Node>"+
     "</LiveNodes></ClusterStatus>";
 
-  private static final String AS_PB = 
-"Ci0KBXRlc3QxEOO6i+eeJBgAIIABKIAIMhUKCS1ST09ULSwsMBABGAEgACgAMAAKOQoFdGVzdDIQ"+
-"/pKx8J4kGAAggAQogAgyIQoVLk1FVEEuLCwxMjQ2MDAwMDQzNzI0EAEYASAAKAAwABgCIAApAAAA"+
-"AAAA8D8=";
-
+  private static final String AS_PB =
+  "CjsKBXRlc3QxEOO6i+eeJBgAIIABKIAIMiMKCS1ST09ULSwsMBABGAEgACgAMAA4AUACSAFQAVgB" +
+  "YAFoAQpHCgV0ZXN0MhD+krHwniQYACCABCiACDIvChUuTUVUQS4sLDEyNDYwMDAwNDM3MjQQARgB" +
+  "IAAoADAAOAFAAkgBUAFYAWABaAEYAiAAKQAAAAAAAPA/";
+  
   private JAXBContext context;
 
   public TestStorageClusterStatusModel() throws JAXBException {
@@ -71,9 +77,10 @@ private StorageClusterStatusModel buildTestModel() {
     model.setRequests(0);
     model.setAverageLoad(1.0);
     model.addLiveNode("test1", 1245219839331L, 128, 1024)
-      .addRegion(Bytes.toBytes("-ROOT-,,0"), 1, 1, 0, 0, 0);
+      .addRegion(Bytes.toBytes("-ROOT-,,0"), 1, 1, 0, 0, 0, 1, 2, 1, 1, 1, 1, 1);
     model.addLiveNode("test2", 1245239331198L, 512, 1024)
-      .addRegion(Bytes.toBytes(".META.,,1246000043724"),1, 1, 0, 0, 0);
+      .addRegion(Bytes.toBytes(".META.,,1246000043724"),1, 1, 0, 0, 0,
+          1, 2, 1, 1, 1, 1, 1);
     return model;
   }
 
@@ -119,6 +126,13 @@ private void checkModel(StorageClusterStatusModel model) {
     assertEquals(region.getStorefileSizeMB(), 0);
     assertEquals(region.getMemstoreSizeMB(), 0);
     assertEquals(region.getStorefileIndexSizeMB(), 0);
+    assertEquals(region.getReadRequestsCount(), 1);
+    assertEquals(region.getWriteRequestsCount(), 2);
+    assertEquals(region.getRootIndexSizeKB(), 1);
+    assertEquals(region.getTotalStaticIndexSizeKB(), 1);
+    assertEquals(region.getTotalStaticBloomSizeKB(), 1);
+    assertEquals(region.getTotalCompactingKVs(), 1);
+    assertEquals(region.getCurrentCompactedKVs(), 1);
     assertFalse(regions.hasNext());
     node = nodes.next();
     assertEquals(node.getName(), "test2");
@@ -133,6 +147,14 @@ private void checkModel(StorageClusterStatusModel model) {
     assertEquals(region.getStorefileSizeMB(), 0);
     assertEquals(region.getMemstoreSizeMB(), 0);
     assertEquals(region.getStorefileIndexSizeMB(), 0);
+    assertEquals(region.getReadRequestsCount(), 1);
+    assertEquals(region.getWriteRequestsCount(), 2);
+    assertEquals(region.getRootIndexSizeKB(), 1);
+    assertEquals(region.getTotalStaticIndexSizeKB(), 1);
+    assertEquals(region.getTotalStaticBloomSizeKB(), 1);
+    assertEquals(region.getTotalCompactingKVs(), 1);
+    assertEquals(region.getCurrentCompactedKVs(), 1);
+    
     assertFalse(regions.hasNext());
     assertFalse(nodes.hasNext());
   }

From 61baf348b2939785019d44ff06342e0687caf450 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 24 Mar 2012 21:16:15 +0000
Subject: [PATCH 0069/1540] HBASE-5633 NPE reading ZK config in HBase

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304925 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HConstants.java         | 3 +++
 src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 140c0ccf2222..92b02f74e2e0 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -78,6 +78,9 @@ public enum OperationStatusCode {
   /** Cluster is fully-distributed */
   public static final String CLUSTER_IS_DISTRIBUTED = "true";
 
+  /** Default value for cluster distributed mode */  
+  public static final String DEFAULT_CLUSTER_DISTRIBUTED = CLUSTER_IS_LOCAL;
+
   /** default host address */
   public static final String DEFAULT_HOST = "0.0.0.0";
 
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
index 6d955acfe180..bdbd9bd1ea9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
@@ -159,7 +159,8 @@ public static Properties parseZooCfg(Configuration conf,
       }
       // Special case for 'hbase.cluster.distributed' property being 'true'
       if (key.startsWith("server.")) {
-        if (conf.get(HConstants.CLUSTER_DISTRIBUTED).equals(HConstants.CLUSTER_IS_DISTRIBUTED)
+        if (conf.get(HConstants.CLUSTER_DISTRIBUTED, HConstants.DEFAULT_CLUSTER_DISTRIBUTED).
+              equals(HConstants.CLUSTER_IS_DISTRIBUTED)
             && value.startsWith(HConstants.LOCALHOST)) {
           String msg = "The server in zoo.cfg cannot be set to localhost " +
               "in a fully-distributed setup because it won't be reachable. " +

From e12d610cbef07738787c50c127cb23de78addfdf Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 24 Mar 2012 22:03:04 +0000
Subject: [PATCH 0070/1540] HBASE-4957 Clean up some log messages, code in
 RecoverableZooKeeper (Todd)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1304941 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/RetryCounter.java       |   5 +-
 .../hbase/zookeeper/RecoverableZooKeeper.java | 174 +++++-------------
 2 files changed, 53 insertions(+), 126 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java b/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
index c7b62c9d792d..15e741d1f25a 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
@@ -50,8 +50,7 @@ public int getMaxRetries() {
   public void sleepUntilNextRetry() throws InterruptedException {
     int attempts = getAttemptTimes();
     long sleepTime = (long) (retryIntervalMillis * Math.pow(2, attempts));
-    LOG.info("The " + attempts + " times to retry  after sleeping " + sleepTime
-        + " ms");
+    LOG.info("Sleeping " + sleepTime + "ms before retry #" + attempts + "...");
     timeUnit.sleep(sleepTime);
   }
 
@@ -66,4 +65,4 @@ public void useRetry() {
   public int getAttemptTimes() {
     return maxRetries-retriesRemaining+1;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index a097ec00a971..b862aed06c77 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -71,10 +71,18 @@ public class RecoverableZooKeeper {
   private final byte[] id;
   private int retryIntervalMillis;
 
-  private static final int ID_OFFSET =  Bytes.SIZEOF_INT;
+  // The metadata attached to each piece of data has the
+  // format:
+  //   <magic> 1-byte constant
+  //   <id length> 4-byte big-endian integer (length of next field)
+  //   <id> identifier corresponding uniquely to this process
+  // It is prepended to the data supplied by the user.
+
   // the magic number is to be backward compatible
   private static final byte MAGIC =(byte) 0XFF;
-  private static final int MAGIC_OFFSET = Bytes.SIZEOF_BYTE;
+  private static final int MAGIC_SIZE = Bytes.SIZEOF_BYTE;
+  private static final int ID_LENGTH_OFFSET = MAGIC_SIZE;
+  private static final int ID_LENGTH_SIZE =  Bytes.SIZEOF_INT;
 
   public RecoverableZooKeeper(String quorumServers, int seesionTimeout,
       Watcher watcher, int maxRetries, int retryIntervalMillis) 
@@ -91,12 +99,9 @@ public RecoverableZooKeeper(String quorumServers, int seesionTimeout,
   }
 
   /**
-   * delete is an idempotent operation. Retry before throw out exception.
-   * This function will not throw out NoNodeException if the path is not existed
-   * @param path
-   * @param version
-   * @throws InterruptedException
-   * @throws KeeperException
+   * delete is an idempotent operation. Retry before throwing exception.
+   * This function will not throw NoNodeException if the path does not
+   * exist.
    */
   public void delete(String path, int version)
   throws InterruptedException, KeeperException {
@@ -120,12 +125,7 @@ public void delete(String path, int version)
 
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper delete failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "delete");
             break;
 
           default:
@@ -139,12 +139,8 @@ public void delete(String path, int version)
   }
 
   /**
-   * exists is an idempotent operation. Retry before throw out exception
-   * @param path
-   * @param watcher
+   * exists is an idempotent operation. Retry before throwing exception
    * @return A Stat instance
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public Stat exists(String path, Watcher watcher)
   throws KeeperException, InterruptedException {
@@ -156,12 +152,7 @@ public Stat exists(String path, Watcher watcher)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper exists failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "exists");
             break;
 
           default:
@@ -174,12 +165,8 @@ public Stat exists(String path, Watcher watcher)
   }
 
   /**
-   * exists is an idempotent operation. Retry before throw out exception
-   * @param path
-   * @param watch
+   * exists is an idempotent operation. Retry before throwing exception
    * @return A Stat instance
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public Stat exists(String path, boolean watch)
   throws KeeperException, InterruptedException {
@@ -191,12 +178,7 @@ public Stat exists(String path, boolean watch)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper exists failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "exists");
             break;
 
           default:
@@ -208,13 +190,19 @@ public Stat exists(String path, boolean watch)
     }
   }
 
+  private void retryOrThrow(RetryCounter retryCounter, KeeperException e,
+      String opName) throws KeeperException {
+    LOG.warn("Possibly transient ZooKeeper exception: " + e);
+    if (!retryCounter.shouldRetry()) {
+      LOG.error("ZooKeeper " + opName + " failed after "
+        + retryCounter.getMaxRetries() + " retries");
+      throw e;
+    }
+  }
+
   /**
-   * getChildren is an idempotent operation. Retry before throw out exception
-   * @param path
-   * @param watcher
+   * getChildren is an idempotent operation. Retry before throwing exception
    * @return List of children znodes
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public List<String> getChildren(String path, Watcher watcher)
     throws KeeperException, InterruptedException {
@@ -226,12 +214,7 @@ public List<String> getChildren(String path, Watcher watcher)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper getChildren failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "getChildren");
             break;
 
           default:
@@ -244,12 +227,8 @@ public List<String> getChildren(String path, Watcher watcher)
   }
 
   /**
-   * getChildren is an idempotent operation. Retry before throw out exception
-   * @param path
-   * @param watch
+   * getChildren is an idempotent operation. Retry before throwing exception
    * @return List of children znodes
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public List<String> getChildren(String path, boolean watch)
   throws KeeperException, InterruptedException {
@@ -261,12 +240,7 @@ public List<String> getChildren(String path, boolean watch)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper getChildren failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "getChildren");
             break;
 
           default:
@@ -279,13 +253,8 @@ public List<String> getChildren(String path, boolean watch)
   }
 
   /**
-   * getData is an idempotent operation. Retry before throw out exception
-   * @param path
-   * @param watcher
-   * @param stat
+   * getData is an idempotent operation. Retry before throwing exception
    * @return Data
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public byte[] getData(String path, Watcher watcher, Stat stat)
   throws KeeperException, InterruptedException {
@@ -298,12 +267,7 @@ public byte[] getData(String path, Watcher watcher, Stat stat)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper getData failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "getData");
             break;
 
           default:
@@ -316,13 +280,8 @@ public byte[] getData(String path, Watcher watcher, Stat stat)
   }
 
   /**
-   * getData is an idemnpotent operation. Retry before throw out exception
-   * @param path
-   * @param watch
-   * @param stat
+   * getData is an idemnpotent operation. Retry before throwing exception
    * @return Data
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public byte[] getData(String path, boolean watch, Stat stat)
   throws KeeperException, InterruptedException {
@@ -335,12 +294,7 @@ public byte[] getData(String path, boolean watch, Stat stat)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper getData failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "getData");
             break;
 
           default:
@@ -356,12 +310,7 @@ public byte[] getData(String path, boolean watch, Stat stat)
    * setData is NOT an idempotent operation. Retry may cause BadVersion Exception
    * Adding an identifier field into the data to check whether 
    * badversion is caused by the result of previous correctly setData
-   * @param path
-   * @param data
-   * @param version
    * @return Stat instance
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public Stat setData(String path, byte[] data, int version)
   throws KeeperException, InterruptedException {
@@ -374,33 +323,28 @@ public Stat setData(String path, byte[] data, int version)
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper setData failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "setData");
             break;
           case BADVERSION:
             // try to verify whether the previous setData success or not
             try{
               Stat stat = new Stat();
               byte[] revData = zk.getData(path, false, stat);
-              int idLength = Bytes.toInt(revData, ID_OFFSET);
-              int dataLength = revData.length-ID_OFFSET-idLength;
-              int dataOffset = ID_OFFSET+idLength;
+              int idLength = Bytes.toInt(revData, ID_LENGTH_SIZE);
+              int dataLength = revData.length-ID_LENGTH_SIZE-idLength;
+              int dataOffset = ID_LENGTH_SIZE+idLength;
               
-              if(Bytes.compareTo(revData, ID_OFFSET, id.length, 
+              if(Bytes.compareTo(revData, ID_LENGTH_SIZE, id.length, 
                   revData, dataOffset, dataLength) == 0) {
                 // the bad version is caused by previous successful setData
                 return stat;
               }
             } catch(KeeperException keeperException){
-              // the ZK is not reliable at this moment. just throw out exception
+              // the ZK is not reliable at this moment. just throwing exception
               throw keeperException;
             }            
           
-          // throw out other exceptions and verified bad version exceptions
+          // throw other exceptions and verified bad version exceptions
           default:
             throw e;
         }
@@ -413,8 +357,8 @@ public Stat setData(String path, byte[] data, int version)
   /**
    * <p>
    * NONSEQUENTIAL create is idempotent operation. 
-   * Retry before throw out exceptions.
-   * But this function will not throw out the NodeExist exception back to the
+   * Retry before throwing exceptions.
+   * But this function will not throw the NodeExist exception back to the
    * application.
    * </p>
    * <p>
@@ -423,13 +367,7 @@ public Stat setData(String path, byte[] data, int version)
    * or not.
    * </p>
    * 
-   * @param path
-   * @param data
-   * @param acl
-   * @param createMode
    * @return Path
-   * @throws KeeperException
-   * @throws InterruptedException
    */
   public String create(String path, byte[] data, List<ACL> acl,
       CreateMode createMode)
@@ -481,12 +419,7 @@ private String createNonSequential(String path, byte[] data, List<ACL> acl,
 
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper create failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "create");
             break;
 
           default:
@@ -520,12 +453,7 @@ private String createSequential(String path, byte[] data,
         switch (e.code()) {
           case CONNECTIONLOSS:
           case OPERATIONTIMEOUT:
-            LOG.warn("Possibly transient ZooKeeper exception: " + e);
-            if (!retryCounter.shouldRetry()) {
-              LOG.error("ZooKeeper create failed after "
-                + retryCounter.getMaxRetries() + " retries");
-              throw e;
-            }
+            retryOrThrow(retryCounter, e, "create");
             break;
 
           default:
@@ -566,9 +494,9 @@ public byte[] removeMetaData(byte[] data) {
       return data;
     }
     
-    int idLength = Bytes.toInt(data, MAGIC_OFFSET);
-    int dataLength = data.length-MAGIC_OFFSET-ID_OFFSET-idLength;
-    int dataOffset = MAGIC_OFFSET+ID_OFFSET+idLength;
+    int idLength = Bytes.toInt(data, ID_LENGTH_OFFSET);
+    int dataLength = data.length-MAGIC_SIZE-ID_LENGTH_SIZE-idLength;
+    int dataOffset = MAGIC_SIZE+ID_LENGTH_SIZE+idLength;
 
     byte[] newData = new byte[dataLength];
     System.arraycopy(data, dataOffset, newData, 0, dataLength);
@@ -582,7 +510,7 @@ private byte[] appendMetaData(byte[] data) {
       return data;
     }
     
-    byte[] newData = new byte[MAGIC_OFFSET+ID_OFFSET+id.length+data.length];
+    byte[] newData = new byte[MAGIC_SIZE+ID_LENGTH_SIZE+id.length+data.length];
     int pos = 0;
     pos = Bytes.putByte(newData, pos, MAGIC);
     pos = Bytes.putInt(newData, pos, id.length);

From 320a84e9e04fb465b5bddded9ff4487e7465dd12 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 25 Mar 2012 23:22:32 +0000
Subject: [PATCH 0071/1540] HBASE-5615 the master never does balance because of
 balancing the parent region (Xufeng)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305172 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/AssignmentManager.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index a2eb6d6ebf8a..b20225710118 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2459,6 +2459,7 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
       ServerName regionLocation = region.getSecond();
       if (regionInfo == null) continue;
       String tableName = regionInfo.getTableNameAsString();
+      if (regionInfo.isOffline() && regionInfo.isSplit()) continue;
       if (regionLocation == null) {
         // regionLocation could be null if createTable didn't finish properly.
         // When createTable is in progress, HMaster restarts.

From 0f2585c1359ef5bc9fe28964cbee5dca6a120642 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Mon, 26 Mar 2012 17:39:50 +0000
Subject: [PATCH 0072/1540] HBASE-5190 Limit the IPC queue size based on calls'
 payload size            (Ted's addendum)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305469 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/SecureServer.java    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index 0766f5d23ebb..bf45bb1fa5c9 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -95,8 +95,8 @@ public abstract class SecureServer extends HBaseServer {
 
   protected class SecureCall extends HBaseServer.Call {
     public SecureCall(int id, Writable param, Connection connection,
-        Responder responder) {
-      super(id, param, connection, responder);
+        Responder responder, long size) {
+      super(id, param, connection, responder, size);
     }
 
     @Override
@@ -205,7 +205,7 @@ public class SecureConnection extends HBaseServer.Connection  {
     private final int AUTHORIZATION_FAILED_CALLID = -1;
     // Fake 'call' for SASL context setup
     private static final int SASL_CALLID = -33;
-    private final SecureCall saslCall = new SecureCall(SASL_CALLID, null, this, null);
+    private final SecureCall saslCall = new SecureCall(SASL_CALLID, null, this, null, 0);
 
     private boolean useWrap = false;
 
@@ -414,7 +414,7 @@ public int readAndProcess() throws IOException, InterruptedException {
             AccessControlException ae = new AccessControlException(
                 "Authentication is required");
             SecureCall failedCall = new SecureCall(AUTHORIZATION_FAILED_CALLID, null, this,
-                null);
+                null, 0);
             failedCall.setResponse(null, Status.FATAL, ae.getClass().getName(),
                 ae.getMessage());
             responder.doRespond(failedCall);
@@ -595,7 +595,7 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
       Writable param = ReflectionUtils.newInstance(paramClass, conf);           // read param
       param.readFields(dis);
 
-      SecureCall call = new SecureCall(id, param, this, responder);
+      SecureCall call = new SecureCall(id, param, this, responder, buf.length);
 
       if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
         priorityCallQueue.put(call);
@@ -623,7 +623,7 @@ private boolean authorizeConnection() throws IOException {
         LOG.debug("Connection authorization failed: "+ae.getMessage(), ae);
         rpcMetrics.authorizationFailures.inc();
         SecureCall failedCall = new SecureCall(AUTHORIZATION_FAILED_CALLID, null, this,
-            null);
+            null, 0);
         failedCall.setResponse(null, Status.FATAL, ae.getClass().getName(),
             ae.getMessage());
         responder.doRespond(failedCall);

From 3c2f155561c3cff53d76ce238abbbd9e040ce7c8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 26 Mar 2012 20:39:31 +0000
Subject: [PATCH 0073/1540] HBASE-5623 Race condition when rolling the HLog and
 hlogFlush (Enis Soztutar and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305549 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLog.java   |  34 +++--
 .../wal/SequenceFileLogWriter.java            |  14 +-
 .../wal/TestLogRollingNoCluster.java          | 142 ++++++++++++++++++
 3 files changed, 173 insertions(+), 17 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 672214997e0b..24e75596b1f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -141,7 +141,7 @@ public class HLog implements Syncable {
   private volatile long syncedTillHere = 0;
   private long lastDeferredTxid;
   private final Path oldLogDir;
-  private boolean logRollRunning;
+  private volatile boolean logRollRunning;
 
   private static Class<? extends Writer> logWriterClass;
   private static Class<? extends Reader> logReaderClass;
@@ -1225,10 +1225,8 @@ synchronized List<Entry> getPendingWrites() {
     }
 
     // writes out pending entries to the HLog
-    void hlogFlush(Writer writer) throws IOException {
-      // Atomically fetch all existing pending writes. New writes
-      // will start accumulating in a new list.
-      List<Entry> pending = getPendingWrites();
+    void hlogFlush(Writer writer, List<Entry> pending) throws IOException {
+      if (pending == null) return;
 
       // write out all accumulated Entries to hdfs.
       for (Entry e : pending) {
@@ -1248,8 +1246,10 @@ private void syncer() throws IOException {
 
   // sync all transactions upto the specified txid
   private void syncer(long txid) throws IOException {
+    Writer tempWriter;
     synchronized (this.updateLock) {
       if (this.closed) return;
+      tempWriter = this.writer; // guaranteed non-null
     }
     // if the transaction that we are interested in is already 
     // synced, then return immediately.
@@ -1260,23 +1260,23 @@ private void syncer(long txid) throws IOException {
       long doneUpto = this.unflushedEntries.get();
       long now = System.currentTimeMillis();
       // Done in parallel for all writer threads, thanks to HDFS-895
-      boolean syncSuccessful = true;
+      List<Entry> pending = logSyncerThread.getPendingWrites();
       try {
         // First flush all the pending writes to HDFS. Then 
         // issue the sync to HDFS. If sync is successful, then update
         // syncedTillHere to indicate that transactions till this
         // number has been successfully synced.
-        logSyncerThread.hlogFlush(this.writer);
-        this.writer.sync();
+        logSyncerThread.hlogFlush(tempWriter, pending);
+        pending = null;
+        tempWriter.sync();
         syncBatchSize.addAndGet(doneUpto - this.syncedTillHere);
         this.syncedTillHere = Math.max(this.syncedTillHere, doneUpto);
       } catch(IOException io) {
-        syncSuccessful = false;
-      }
-      if (!syncSuccessful) {
         synchronized (this.updateLock) {
-          // HBASE-4387, retry with updateLock held
-          this.writer.sync();
+          // HBASE-4387, HBASE-5623, retry with updateLock held
+          tempWriter = this.writer;
+          logSyncerThread.hlogFlush(tempWriter, pending);
+          tempWriter.sync();
           syncBatchSize.addAndGet(doneUpto - this.syncedTillHere);
           this.syncedTillHere = doneUpto;
         }
@@ -1286,8 +1286,12 @@ private void syncer(long txid) throws IOException {
       syncTime.inc(System.currentTimeMillis() - now);
       if (!this.logRollRunning) {
         checkLowReplication();
-        if (this.writer.getLength() > this.logrollsize) {
-          requestLogRoll();
+        try {
+          if (tempWriter.getLength() > this.logrollsize) {
+            requestLogRoll();
+          }
+        } catch (IOException x) {
+          LOG.debug("Log roll failed and will be retried. (This is not an error)");
         }
       }
     } catch (IOException e) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index c280ae077495..fbe1ce5ca672 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -274,7 +274,12 @@ private FSDataOutputStream getSequenceFilePrivateFSDataOutputStreamAccessible()
   @Override
   public void append(HLog.Entry entry) throws IOException {
     entry.setCompressionContext(compressionContext);
-    this.writer.append(entry.getKey(), entry.getEdit());
+    try {
+      this.writer.append(entry.getKey(), entry.getEdit());
+    } catch (NullPointerException npe) {
+      // Concurrent close...
+      throw new IOException(npe);
+    }
   }
 
   @Override
@@ -309,7 +314,12 @@ public void sync() throws IOException {
 
   @Override
   public long getLength() throws IOException {
-    return this.writer.getLength();
+    try {
+      return this.writer.getLength();
+    } catch (NullPointerException npe) {
+      // Concurrent close...
+      throw new IOException(npe);
+    }
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java
new file mode 100644
index 000000000000..3f467363bda1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test many concurrent appenders to an {@link #HLog} while rolling the log.
+ */
+@Category(MediumTests.class)
+public class TestLogRollingNoCluster {
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static byte [] EMPTY_1K_ARRAY = new byte[1024];
+  private static final int THREAD_COUNT = 100; // Spin up this many threads
+
+  /**
+   * Spin up a bunch of threads and have them all append to a WAL.  Roll the
+   * WAL frequently to try and trigger NPE.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testContendedLogRolling() throws IOException, InterruptedException {
+    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    Path dir = TEST_UTIL.getDataTestDir();
+    HLog wal = new HLog(fs, new Path(dir, "logs"), new Path(dir, "oldlogs"),
+      TEST_UTIL.getConfiguration());
+    Appender [] appenders = null;
+
+    final int count = THREAD_COUNT;
+    appenders = new Appender[count];
+    try {
+      for (int i = 0; i < count; i++) {
+        // Have each appending thread write 'count' entries
+        appenders[i] = new Appender(wal, i, count);
+      }
+      for (int i = 0; i < count; i++) {
+        appenders[i].start();
+      }
+      for (int i = 0; i < count; i++) {
+        //ensure that all threads are joined before closing the wal
+        appenders[i].join();
+      }
+    } finally {
+      wal.close();
+    }
+    for (int i = 0; i < count; i++) {
+      assertFalse(appenders[i].isException());
+    }
+  }
+
+  /**
+   * Appender thread.  Appends to passed wal file.
+   */
+  static class Appender extends Thread {
+    private final Log log;
+    private final HLog wal;
+    private final int count;
+    private Exception e = null;
+
+    Appender(final HLog wal, final int index, final int count) {
+      super("" + index);
+      this.wal = wal;
+      this.count = count;
+      this.log = LogFactory.getLog("Appender:" + getName());
+    }
+
+    /**
+     * @return Call when the thread is done.
+     */
+    boolean isException() {
+      return !isAlive() && this.e != null;
+    }
+
+    Exception getException() {
+      return this.e;
+    }
+
+    @Override
+    public void run() {
+      this.log.info(getName() +" started");
+      try {
+        for (int i = 0; i < this.count; i++) {
+          long now = System.currentTimeMillis();
+          // Roll every ten edits if the log has anything in it.
+          if (i % 10 == 0 && this.wal.getNumEntries() > 0) {
+            this.wal.rollWriter();
+          }
+          WALEdit edit = new WALEdit();
+          byte[] bytes = Bytes.toBytes(i);
+          edit.add(new KeyValue(bytes, bytes, bytes, now, EMPTY_1K_ARRAY));
+
+          this.wal.append(HRegionInfo.FIRST_META_REGIONINFO,
+              HTableDescriptor.META_TABLEDESC.getName(),
+              edit, now, HTableDescriptor.META_TABLEDESC);
+        }
+        String msg = getName() + " finished";
+        if (isException())
+          this.log.info(msg, getException());
+        else
+          this.log.info(msg);
+      } catch (Exception e) {
+        this.e = e;
+        log.info("Caught exception from Appender:" + getName(), e);
+      }
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
\ No newline at end of file

From 742aa34db9e0648df531378073fed50aaa6b11ab Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 26 Mar 2012 20:56:45 +0000
Subject: [PATCH 0074/1540] HBASE-5533 Add more metrics to HBase (Shaneal
 Manek)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305582 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/hfile/HFile.java   | 101 +++++++-
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  15 +-
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  15 +-
 .../hadoop/hbase/io/hfile/HFileWriterV1.java  |   5 +-
 .../hadoop/hbase/io/hfile/HFileWriterV2.java  |   5 +-
 .../hbase/metrics/ExactCounterMetric.java     | 154 ++++++++++++
 .../ExponentiallyDecayingSample.java          | 212 +++++++++++++++++
 .../metrics/histogram/MetricsHistogram.java   | 225 ++++++++++++++++++
 .../hbase/metrics/histogram/Sample.java       |  49 ++++
 .../hbase/metrics/histogram/Snapshot.java     | 166 +++++++++++++
 .../metrics/histogram/UniformSample.java      | 105 ++++++++
 .../hbase/regionserver/HRegionServer.java     |  23 +-
 .../metrics/RegionServerMetrics.java          | 140 +++++++++--
 .../hadoop/hbase/regionserver/wal/HLog.java   |   2 +-
 .../org/apache/hadoop/hbase/util/Threads.java |  21 ++
 .../hbase/metrics/TestExactCounterMetric.java |  47 ++++
 .../TestExponentiallyDecayingSample.java      |  63 +++++
 .../hbase/metrics/TestMetricsHistogram.java   |  98 ++++++++
 18 files changed, 1392 insertions(+), 54 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 31c07a173fcb..1fe5db9887f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -24,8 +24,12 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -47,14 +51,15 @@
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware;
-import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.Writable;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 
 /**
  * File format for hbase.
@@ -163,18 +168,100 @@ public class HFile {
   public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32;
 
   // For measuring latency of "sequential" reads and writes
-  static final AtomicInteger readOps = new AtomicInteger();
-  static final AtomicLong readTimeNano = new AtomicLong();
-  static final AtomicInteger writeOps = new AtomicInteger();
-  static final AtomicLong writeTimeNano = new AtomicLong();
+  private static final AtomicInteger readOps = new AtomicInteger();
+  private static final AtomicLong readTimeNano = new AtomicLong();
+  private static final AtomicInteger writeOps = new AtomicInteger();
+  private static final AtomicLong writeTimeNano = new AtomicLong();
 
   // For measuring latency of pread
-  static final AtomicInteger preadOps = new AtomicInteger();
-  static final AtomicLong preadTimeNano = new AtomicLong();
+  private static final AtomicInteger preadOps = new AtomicInteger();
+  private static final AtomicLong preadTimeNano = new AtomicLong();
 
   // For measuring number of checksum failures
   static final AtomicLong checksumFailures = new AtomicLong();
 
+  // For getting more detailed stats on FS latencies
+  // If, for some reason, the metrics subsystem stops polling for latencies, 
+  // I don't want data to pile up in a memory leak
+  // so, after LATENCY_BUFFER_SIZE items have been enqueued for processing,
+  // fs latency stats will be dropped (and this behavior will be logged)
+  private static final int LATENCY_BUFFER_SIZE = 5000;
+  private static final BlockingQueue<Long> fsReadLatenciesNanos = 
+      new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
+  private static final BlockingQueue<Long> fsWriteLatenciesNanos = 
+      new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
+  private static final BlockingQueue<Long> fsPreadLatenciesNanos = 
+      new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
+  private static final AtomicLong lastLoggedDataDrop = new AtomicLong(0);
+  
+  // we don't want to fill up the logs with this message, so only log it 
+  // once every 30 seconds at most
+  // I also want to avoid locks on the 'critical path' (the common case will be
+  // uncontended) - hence the CAS
+  private static void logDroppedLatencyStat() {
+    final long now = System.currentTimeMillis();      
+    final long earliestAcceptableLog = now - TimeUnit.SECONDS.toMillis(30L);
+    while (true) {
+      final long lastLog = lastLoggedDataDrop.get();
+      if (lastLog < earliestAcceptableLog) {
+        if (lastLoggedDataDrop.compareAndSet(lastLog, now)) {
+          LOG.warn("Dropping fs latency stats since buffer is full");
+          break;
+        } // otherwise (if the compaseAndSet failed) the while loop retries
+      } else {
+        break;
+      }
+    }    
+  }
+  
+  public static final void offerReadLatency(long latencyNanos, boolean pread) {
+    boolean stored = false;
+    if (pread) {
+      stored = fsPreadLatenciesNanos.offer(latencyNanos);
+      preadOps.incrementAndGet();
+      preadTimeNano.addAndGet(latencyNanos);
+    } else {
+      stored = fsReadLatenciesNanos.offer(latencyNanos);
+      readTimeNano.addAndGet(latencyNanos);
+      readOps.incrementAndGet();
+    }
+
+    if (!stored) { 
+      logDroppedLatencyStat();
+    }    
+  }
+  
+  public static final void offerWriteLatency(long latencyNanos) {
+    final boolean stored = fsWriteLatenciesNanos.offer(latencyNanos);
+    if (!stored) {
+      logDroppedLatencyStat();
+    }
+    
+    writeTimeNano.addAndGet(latencyNanos);
+    writeOps.incrementAndGet();
+  }
+  
+  public static final Collection<Long> getReadLatenciesNanos() {
+    final List<Long> latencies = 
+        Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size());
+    fsReadLatenciesNanos.drainTo(latencies);
+    return latencies;
+  }
+
+  public static final Collection<Long> getPreadLatenciesNanos() {
+    final List<Long> latencies = 
+        Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size());
+    fsPreadLatenciesNanos.drainTo(latencies);
+    return latencies;
+  }
+  
+  public static final Collection<Long> getWriteLatenciesNanos() {
+    final List<Long> latencies = 
+        Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size());
+    fsWriteLatenciesNanos.drainTo(latencies);
+    return latencies;
+  }
+
   // for test purpose
   public static volatile AtomicLong dataBlockReadCnt = new AtomicLong(0);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index e49ca0a243dc..b93c5c483104 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -247,9 +247,8 @@ public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
       passSchemaMetricsTo(hfileBlock);
       hfileBlock.expectType(BlockType.META);
 
-      long delta = System.nanoTime() - startTimeNs;
-      HFile.preadTimeNano.addAndGet(delta);
-      HFile.preadOps.incrementAndGet();
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, true);
       getSchemaMetrics().updateOnCacheMiss(effectiveCategory,
           SchemaMetrics.NO_COMPACTION, delta);
 
@@ -326,14 +325,8 @@ ByteBuffer readBlockBuffer(int block, boolean cacheBlock,
       passSchemaMetricsTo(hfileBlock);
       hfileBlock.expectType(BlockType.DATA);
 
-      long delta = System.nanoTime() - startTimeNs;
-      if (pread) {
-        HFile.preadTimeNano.addAndGet(delta);
-        HFile.preadOps.incrementAndGet();
-      } else {
-        HFile.readTimeNano.addAndGet(delta);
-        HFile.readOps.incrementAndGet();
-      }
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, pread);
       getSchemaMetrics().updateOnCacheMiss(BlockCategory.DATA, isCompaction,
           delta);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 8e7be993d261..7334d0692eb5 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -234,9 +234,8 @@ public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
           blockSize, -1, true);
       passSchemaMetricsTo(metaBlock);
 
-      long delta = System.nanoTime() - startTimeNs;
-      HFile.preadTimeNano.addAndGet(delta);
-      HFile.preadOps.incrementAndGet();
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, true);
       getSchemaMetrics().updateOnCacheMiss(BlockCategory.META, false, delta);
 
       // Cache the block
@@ -333,14 +332,8 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
       passSchemaMetricsTo(hfileBlock);
       BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
 
-      long delta = System.nanoTime() - startTimeNs;
-      if (pread) {
-        HFile.preadTimeNano.addAndGet(delta);
-        HFile.preadOps.incrementAndGet();
-      } else {
-        HFile.readTimeNano.addAndGet(delta);
-        HFile.readOps.incrementAndGet();
-      }
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, pread);
       getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta);
 
       // Cache the block if necessary
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
index 6abee866ccdf..e79c228e71d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
@@ -139,9 +139,8 @@ private void finishBlock() throws IOException {
     blockDataSizes.add(Integer.valueOf(size));
     this.totalUncompressedBytes += size;
 
-    HFile.writeTimeNano.addAndGet(System.nanoTime() - startTimeNs);
-    HFile.writeOps.incrementAndGet();
-
+    HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
+    
     if (cacheConf.shouldCacheDataOnWrite()) {
       baosDos.flush();
       // we do not do data block encoding on disk for HFile v1
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index 5f5d74502672..44bd77eb6c80 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -187,9 +187,8 @@ private void finishBlock() throws IOException {
         onDiskSize);
     totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
 
-    HFile.writeTimeNano.addAndGet(System.nanoTime() - startTimeNs);
-    HFile.writeOps.incrementAndGet();
-
+    HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
+    
     if (cacheConf.shouldCacheDataOnWrite()) {
       doCacheOnWrite(lastDataBlockOffset);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java b/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
new file mode 100644
index 000000000000..40e29eb69ef0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.cliffc.high_scale_lib.Counter;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+
+public class ExactCounterMetric extends MetricsBase {
+
+  private static final int DEFAULT_TOP_N = 5;
+
+  // only publish stats on the topN items (default to DEFAULT_TOP_N)
+  private final int topN;
+  private final Map<String, Counter> counts;
+
+  // all access to the 'counts' map should use this lock.
+  // take a write lock iff you want to guarantee exclusive access
+  // (the map stripes locks internally, so it's already thread safe -
+  // this lock is just so you can take a consistent snapshot of data)
+  private final ReadWriteLock lock;
+  
+  
+  /**
+   * Constructor to create a new counter metric
+   * @param nam         the name to publish this metric under
+   * @param registry    where the metrics object will be registered
+   * @param description metrics description
+   * @param topN        how many 'keys' to publish metrics on 
+   */
+  public ExactCounterMetric(final String nam, final MetricsRegistry registry, 
+      final String description, int topN) {
+    super(nam, description);
+
+    this.counts = new MapMaker().makeComputingMap(
+        new Function<String, Counter>() {
+          @Override
+          public Counter apply(String input) {
+            return new Counter();
+          }    
+        });
+
+    this.lock = new ReentrantReadWriteLock();
+    this.topN = topN;
+
+    if (registry != null) {
+      registry.add(nam, this);      
+    }
+  }
+
+  /**
+   * Constructor creates a new ExactCounterMetric
+   * @param nam       the name of the metrics to be used to publish the metric
+   * @param registry  where the metrics object will be registered
+   */
+  public ExactCounterMetric(final String nam, MetricsRegistry registry) {
+    this(nam, registry, NO_DESCRIPTION, DEFAULT_TOP_N);
+  }
+
+  
+  public void update(String type) {
+    this.lock.readLock().lock();
+    try {
+      this.counts.get(type).increment();
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+  
+  public void update(String type, long count) {
+    this.lock.readLock().lock();
+    try {
+      this.counts.get(type).add(count);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+  
+  public List<Pair<String, Long>> getTop(int n) {
+    final List<Pair<String, Long>> countsSnapshot = 
+        Lists.newArrayListWithCapacity(this.counts.size());
+    
+    // no updates are allowed while I'm holding this lock, so move fast
+    this.lock.writeLock().lock();
+    try {
+      for(Entry<String, Counter> entry : this.counts.entrySet()) {
+        countsSnapshot.add(Pair.newPair(entry.getKey(), 
+            entry.getValue().get()));
+      }
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+    
+    Collections.sort(countsSnapshot, new Comparator<Pair<String, Long>>() {
+      @Override
+      public int compare(Pair<String, Long> a, Pair<String, Long> b) {
+        return b.getSecond().compareTo(a.getSecond());
+      }      
+    });
+    
+    return countsSnapshot.subList(0, Math.min(n, countsSnapshot.size()));
+  }
+  
+  @Override
+  public void pushMetric(MetricsRecord mr) {
+    final List<Pair<String, Long>> topKeys = getTop(Integer.MAX_VALUE);
+    int sum = 0;
+    
+    int counter = 0;
+    for (Pair<String, Long> keyCount : topKeys) {
+      counter++;
+      // only push stats on the topN keys
+      if (counter <= this.topN) {
+        mr.setMetric(getName() + "_" + keyCount.getFirst(), 
+            keyCount.getSecond());        
+      }
+      sum += keyCount.getSecond();
+    }
+    mr.setMetric(getName() + "_map_size", this.counts.size());
+    mr.setMetric(getName() + "_total_count", sum);
+  }
+  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
new file mode 100644
index 000000000000..e615b2123ebc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics.histogram;
+
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hbase.util.Threads;
+
+/**
+ * An exponentially-decaying random sample of {@code long}s. 
+ * Uses Cormode et al's forward-decaying priority reservoir sampling method 
+ * to produce a statistically representative sample, exponentially biased 
+ * towards newer entries.
+ *
+ * see Cormode et al. 
+ * Forward Decay: A Practical Time Decay Model for Streaming Systems. ICDE '09
+ */
+public class ExponentiallyDecayingSample implements Sample {
+
+  private static final Random RANDOM = new Random();  
+  private static final long RESCALE_THRESHOLD = TimeUnit.HOURS.toNanos(1);
+
+  private static final ScheduledExecutorService TICK_SERVICE = 
+      Executors.newScheduledThreadPool(1, 
+          Threads.getNamedThreadFactory("decayingSampleTick"));
+
+  private static volatile long CURRENT_TICK = 
+      TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
+
+  static {
+    // sample at twice our signal's frequency (1Hz) per the Nyquist theorem
+    TICK_SERVICE.scheduleAtFixedRate(new Runnable() {
+      @Override
+      public void run() {
+        CURRENT_TICK = 
+            TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
+      }
+    }, 0, 500, TimeUnit.MILLISECONDS);
+  }
+  
+  private final ConcurrentSkipListMap<Double, Long> values = 
+      new ConcurrentSkipListMap<Double, Long>();
+  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+  private final AtomicLong count = new AtomicLong(0);
+  private final AtomicLong nextScaleTime = new AtomicLong(0);
+
+  private final double alpha;
+  private final int reservoirSize;
+  private volatile long startTime;
+
+  /**
+   * Constructor for an ExponentiallyDecayingSample.
+   *
+   * @param reservoirSize the number of samples to keep in the reservoir
+   * @param alpha         the exponential decay factor; the higher this is, 
+   *                      the more biased the sample will be towards newer 
+   *                      values
+   */
+  public ExponentiallyDecayingSample(int reservoirSize, double alpha) {
+    this.alpha = alpha;
+    this.reservoirSize = reservoirSize;
+    clear();
+  }
+
+  @Override
+  public void clear() {
+    lockForRescale();
+    try {
+      values.clear();
+      count.set(0);
+      this.startTime = CURRENT_TICK;
+      nextScaleTime.set(System.nanoTime() + RESCALE_THRESHOLD);
+    } finally {
+      unlockForRescale();
+    }
+  }
+
+  @Override
+  public int size() {
+    return (int) Math.min(reservoirSize, count.get());
+  }
+
+  @Override
+  public void update(long value) {
+    update(value, CURRENT_TICK);
+  }
+
+  /**
+   * Adds an old value with a fixed timestamp to the sample.
+   *
+   * @param value     the value to be added
+   * @param timestamp the epoch timestamp of {@code value} in seconds
+   */
+  public void update(long value, long timestamp) {
+    lockForRegularUsage();
+    try {
+      final double priority = weight(timestamp - startTime) 
+          / RANDOM.nextDouble();
+      final long newCount = count.incrementAndGet();
+      if (newCount <= reservoirSize) {
+        values.put(priority, value);
+      } else {
+        Double first = values.firstKey();
+        if (first < priority) {
+          if (values.putIfAbsent(priority, value) == null) {
+            // ensure we always remove an item
+            while (values.remove(first) == null) {
+              first = values.firstKey();
+            }
+          }
+        }
+      }
+    } finally {
+      unlockForRegularUsage();
+    }
+
+    final long now = System.nanoTime();
+    final long next = nextScaleTime.get();
+    if (now >= next) {
+      rescale(now, next);
+    }
+  }
+
+  @Override
+  public Snapshot getSnapshot() {
+    lockForRegularUsage();
+    try {
+      return new Snapshot(values.values());
+    } finally {
+      unlockForRegularUsage();
+    }
+  }
+
+  private double weight(long t) {
+    return Math.exp(alpha * t);
+  }
+
+  /* "A common feature of the above techniques—indeed, the key technique that
+   * allows us to track the decayed weights efficiently—is that they maintain
+   * counts and other quantities based on g(ti − L), and only scale by g(t − L)
+   * at query time. But while g(ti −L)/g(t−L) is guaranteed to lie between zero
+   * and one, the intermediate values of g(ti − L) could become very large. For
+   * polynomial functions, these values should not grow too large, and should 
+   * be effectively represented in practice by floating point values without 
+   * loss of precision. For exponential functions, these values could grow 
+   * quite large as new values of (ti − L) become large, and potentially 
+   * exceed the capacity of common floating point types. However, since the 
+   * values stored by the algorithms are linear combinations of g values 
+   * (scaled sums), they can be rescaled relative to a new landmark. That is, 
+   * by the analysis of exponential decay in Section III-A, the choice of L 
+   * does not affect the final result. We can therefore multiply each value 
+   * based on L by a factor of exp(−α(L′ − L)), and obtain the correct value 
+   * as if we had instead computed relative to a new landmark L′ (and then use 
+   * this new L′ at query time). This can be done with a linear pass over 
+   * whatever data structure is being used."
+   */
+  private void rescale(long now, long next) {
+    if (nextScaleTime.compareAndSet(next, now + RESCALE_THRESHOLD)) {
+      lockForRescale();
+      try {
+        final long oldStartTime = startTime;
+        this.startTime = CURRENT_TICK;
+        final ArrayList<Double> keys = new ArrayList<Double>(values.keySet());
+        for (Double key : keys) {
+          final Long value = values.remove(key);
+          values.put(key * Math.exp(-alpha * (startTime - oldStartTime)), 
+              value);
+        }
+      } finally {
+        unlockForRescale();
+      }
+    }
+  }
+
+  private void unlockForRescale() {
+    lock.writeLock().unlock();
+  }
+
+  private void lockForRescale() {
+    lock.writeLock().lock();
+  }
+
+  private void lockForRegularUsage() {
+    lock.readLock().lock();
+  }
+
+  private void unlockForRegularUsage() {
+    lock.readLock().unlock();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
new file mode 100644
index 000000000000..a78b0ce92560
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ 
+
+package org.apache.hadoop.hbase.metrics.histogram;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+public class MetricsHistogram extends MetricsBase {
+  
+  // 1028 items implies 99.9% CI w/ 5% margin of error 
+  // (assuming a normal distribution on the underlying data)
+  private static final int DEFAULT_SAMPLE_SIZE = 1028;
+
+  // the bias towards sampling from more recent data. 
+  // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes
+  private static final double DEFAULT_ALPHA = 0.015;
+
+  /**
+   * Constructor to create a new histogram metric
+   * @param nam           the name to publish the metric under
+   * @param registry      where the metrics object will be registered
+   * @param description   the metric's description
+   * @param forwardBiased true if you want this histogram to give more 
+   *                      weight to recent data, 
+   *                      false if you want all data to have uniform weight
+   */
+  public MetricsHistogram(final String nam, final MetricsRegistry registry, 
+      final String description, boolean forwardBiased) {
+    super(nam, description);
+
+    this.min = new AtomicLong();
+    this.max = new AtomicLong();
+    this.sum = new AtomicLong();
+    this.sample = forwardBiased ? 
+        new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA) 
+    : new UniformSample(DEFAULT_SAMPLE_SIZE);
+
+    this.variance =  new AtomicReference<double[]>(new double[]{-1, 0});
+    this.count = new AtomicLong();
+
+    this.clear();
+
+    if (registry != null) {
+      registry.add(nam, this);      
+    }
+  }
+
+  /**
+   * Constructor create a new (forward biased) histogram metric
+   * @param nam         the name to publish the metric under
+   * @param registry    where the metrics object will be registered
+   * @param description the metric's description
+   */
+  public MetricsHistogram(final String nam, MetricsRegistry registry, 
+      final String description) {
+    this(nam, registry, NO_DESCRIPTION, true);
+  }
+    
+  /**
+   * Constructor - create a new (forward biased) histogram metric
+   * @param nam the name of the metrics to be used to publish the metric
+   * @param registry - where the metrics object will be registered
+   */
+  public MetricsHistogram(final String nam, MetricsRegistry registry) {
+    this(nam, registry, NO_DESCRIPTION);
+  }
+
+  private final Sample sample;
+  private final AtomicLong min;
+  private final AtomicLong max;
+  private final AtomicLong sum;
+
+  // these are for computing a running-variance, 
+  // without letting floating point errors accumulate via Welford's algorithm
+  private final AtomicReference<double[]> variance;
+  private final AtomicLong count;
+
+  /**
+   * Clears all recorded values.
+   */
+  public void clear() {
+    this.sample.clear();
+    this.count.set(0);
+    this.max.set(Long.MIN_VALUE);
+    this.min.set(Long.MAX_VALUE);
+    this.sum.set(0);
+    variance.set(new double[]{-1, 0});
+  }
+
+  public void update(int val) {
+    update((long) val);
+  }
+
+  public void update(final long val) {
+    count.incrementAndGet();
+    sample.update(val);
+    setMax(val);
+    setMin(val);
+    sum.getAndAdd(val);
+    updateVariance(val);
+  }
+
+  private void setMax(final long potentialMax) {
+    boolean done = false;
+    while (!done) {
+      final long currentMax = max.get();
+      done = currentMax >= potentialMax 
+          || max.compareAndSet(currentMax, potentialMax);
+    }
+  }
+
+  private void setMin(long potentialMin) {
+    boolean done = false;
+    while (!done) {
+      final long currentMin = min.get();
+      done = currentMin <= potentialMin 
+          || min.compareAndSet(currentMin, potentialMin);
+    }
+  }
+
+  private void updateVariance(long value) {
+    boolean done = false;
+    while (!done) {
+      final double[] oldValues = variance.get();
+      final double[] newValues = new double[2];
+      if (oldValues[0] == -1) {
+        newValues[0] = value;
+        newValues[1] = 0;
+      } else {
+        final double oldM = oldValues[0];
+        final double oldS = oldValues[1];
+
+        final double newM = oldM + ((value - oldM) / getCount());
+        final double newS = oldS + ((value - oldM) * (value - newM));
+
+        newValues[0] = newM;
+        newValues[1] = newS;
+      }
+      done = variance.compareAndSet(oldValues, newValues);
+    }
+  }
+
+
+  public long getCount() {
+    return count.get();
+  }
+
+  public long getMax() {
+    if (getCount() > 0) {
+      return max.get();
+    }
+    return 0L;
+  }
+
+  public long getMin() {
+    if (getCount() > 0) {
+      return min.get();
+    }
+    return 0L;
+  }
+
+  public double getMean() {
+    if (getCount() > 0) {
+      return sum.get() / (double) getCount();
+    }
+    return 0.0;
+  }
+
+  public double getStdDev() {
+    if (getCount() > 0) {
+      return Math.sqrt(getVariance());
+    }
+    return 0.0;
+  }
+
+  public Snapshot getSnapshot() {
+    return sample.getSnapshot();
+  }
+
+  private double getVariance() {
+    if (getCount() <= 1) {
+      return 0.0;
+    }
+    return variance.get()[1] / (getCount() - 1);
+  }
+
+  @Override
+  public void pushMetric(MetricsRecord mr) {
+    final Snapshot s = this.getSnapshot();
+    mr.setMetric(getName() + "_num_ops", this.getCount());
+    mr.setMetric(getName() + "_min", this.getMin());
+    mr.setMetric(getName() + "_max", this.getMax());
+    
+    mr.setMetric(getName() + "_mean", (float) this.getMean());
+    mr.setMetric(getName() + "_std_dev", (float) this.getStdDev());
+    
+    mr.setMetric(getName() + "_median", (float) s.getMedian());
+    mr.setMetric(getName() + "_75th_percentile", 
+        (float) s.get75thPercentile());
+    mr.setMetric(getName() + "_95th_percentile", 
+        (float) s.get95thPercentile());
+    mr.setMetric(getName() + "_99th_percentile", 
+        (float) s.get99thPercentile());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
new file mode 100644
index 000000000000..55b91d688b21
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics.histogram;
+
+/**
+ * A statistically representative sample of items from a stream.
+ */
+public interface Sample {
+  /**
+   * Clears all recorded values.
+   */
+  void clear();
+
+  /**
+   * Returns the number of values recorded.
+   *
+   * @return the number of values recorded
+   */
+  int size();
+
+  /**
+   * Adds a new recorded value to the sample.
+   *
+   * @param value a new recorded value
+   */
+  void update(long value);
+
+  /**
+   * Returns a snapshot of the sample's values.
+   *
+   * @return a snapshot of the sample's values
+   */
+  Snapshot getSnapshot();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
new file mode 100644
index 000000000000..418b6fb1205d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics.histogram;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * A snapshot of all the information seen in a Sample.
+ */
+public class Snapshot {
+
+  private static final double MEDIAN_Q = 0.5;
+  private static final double P75_Q = 0.75;
+  private static final double P95_Q = 0.95;
+  private static final double P98_Q = 0.98;
+  private static final double P99_Q = 0.99;
+  private static final double P999_Q = 0.999;
+
+  private final double[] values;
+
+  /**
+   * Create a new {@link Snapshot} with the given values.
+   *
+   * @param values    an unordered set of values in the sample
+   */
+  public Snapshot(Collection<Long> values) {
+    final Object[] copy = values.toArray();
+    this.values = new double[copy.length];
+    for (int i = 0; i < copy.length; i++) {
+      this.values[i] = (Long) copy[i];
+    }
+    Arrays.sort(this.values);
+  }
+
+  /**
+   * Create a new {@link Snapshot} with the given values.
+   *
+   * @param values    an unordered set of values in the sample
+   */
+  public Snapshot(double[] values) {
+    this.values = new double[values.length];
+    System.arraycopy(values, 0, this.values, 0, values.length);
+    Arrays.sort(this.values);
+  }
+
+  /**
+   * Returns the value at the given quantile.
+   *
+   * @param quantile    a given quantile, in [0..1]
+   * @return the value in the distribution at quantile
+   */
+  public double getValue(double quantile) {
+    if (quantile < 0.0 || quantile > 1.0) {
+      throw new IllegalArgumentException(quantile + " is not in [0..1]");
+    }
+
+    if (values.length == 0) {
+      return 0.0;
+    }
+
+    final double pos = quantile * (values.length + 1);
+
+    if (pos < 1) {
+      return values[0];
+    }
+
+    if (pos >= values.length) {
+      return values[values.length - 1];
+    }
+
+    final double lower = values[(int) pos - 1];
+    final double upper = values[(int) pos];
+    return lower + (pos - Math.floor(pos)) * (upper - lower);
+  }
+
+  /**
+   * Returns the number of values in the snapshot.
+   *
+   * @return the number of values in the snapshot
+   */
+  public int size() {
+    return values.length;
+  }
+
+  /**
+   * Returns the median value in the distribution.
+   *
+   * @return the median value in the distribution
+   */
+  public double getMedian() {
+    return getValue(MEDIAN_Q);
+  }
+
+  /**
+   * Returns the value at the 75th percentile in the distribution.
+   *
+   * @return the value at the 75th percentile in the distribution
+   */
+  public double get75thPercentile() {
+    return getValue(P75_Q);
+  }
+
+  /**
+   * Returns the value at the 95th percentile in the distribution.
+   *
+   * @return the value at the 95th percentile in the distribution
+   */
+  public double get95thPercentile() {
+    return getValue(P95_Q);
+  }
+
+  /**
+   * Returns the value at the 98th percentile in the distribution.
+   *
+   * @return the value at the 98th percentile in the distribution
+   */
+  public double get98thPercentile() {
+    return getValue(P98_Q);
+  }
+
+  /**
+   * Returns the value at the 99th percentile in the distribution.
+   *
+   * @return the value at the 99th percentile in the distribution
+   */
+  public double get99thPercentile() {
+    return getValue(P99_Q);
+  }
+
+  /**
+   * Returns the value at the 99.9th percentile in the distribution.
+   *
+   * @return the value at the 99.9th percentile in the distribution
+   */
+  public double get999thPercentile() {
+    return getValue(P999_Q);
+  }
+
+  /**
+   * Returns the entire set of values in the snapshot.
+   *
+   * @return the entire set of values in the snapshot
+   */
+  public double[] getValues() {
+    return Arrays.copyOf(values, values.length);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
new file mode 100644
index 000000000000..74dd0f945678
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics.histogram;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+/**
+ * A random sample of a stream of longs. Uses Vitter's Algorithm R to produce a
+ * statistically representative sample.
+ *
+ * see: http://www.cs.umd.edu/~samir/498/vitter.pdf
+ */
+public class UniformSample implements Sample {
+  
+  private static final Random RANDOM = new Random();
+  private static final int BITS_PER_LONG = 63;
+  
+  private final AtomicLong count = new AtomicLong();
+  private final AtomicLongArray values;
+
+  /**
+   * Creates a new UniformSample
+   *
+   * @param reservoirSize the number of samples to keep
+   */
+  public UniformSample(int reservoirSize) {
+    this.values = new AtomicLongArray(reservoirSize);
+    clear();
+  }
+
+  @Override
+  public void clear() {
+    for (int i = 0; i < values.length(); i++) {
+      values.set(i, 0);
+    }
+    count.set(0);
+  }
+
+  @Override
+  public int size() {
+    final long c = count.get();
+    if (c > values.length()) {
+      return values.length();
+    }
+    return (int) c;
+  }
+
+  @Override
+  public void update(long value) {
+    final long c = count.incrementAndGet();
+    if (c <= values.length()) {
+      values.set((int) c - 1, value);
+    } else {
+      final long r = nextLong(c);
+      if (r < values.length()) {
+        values.set((int) r, value);
+      }
+    }
+  }
+
+  /**
+   * Get a pseudo-random long uniformly between 0 and n-1. Stolen from
+   * {@link java.util.Random#nextInt()}.
+   *
+   * @param n the bound
+   * @return a value select randomly from the range {@code [0..n)}.
+   */
+  private static long nextLong(long n) {
+    long bits, val;
+    do {
+      bits = RANDOM.nextLong() & (~(1L << BITS_PER_LONG));
+      val = bits % n;
+    } while (bits - val + (n - 1) < 0L);
+    return val;
+  }
+
+  @Override
+  public Snapshot getSnapshot() {
+    final int s = size();
+    final List<Long> copy = new ArrayList<Long>(s);
+    for (int i = 0; i < s; i++) {
+      copy.add(values.get(i));
+    }
+    return new Snapshot(copy);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 44e4891458d3..ba7fbe1b3065 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1951,12 +1951,15 @@ public Result getClosestRowBefore(final byte[] regionName, final byte[] row,
   /** {@inheritDoc} */
   public Result get(byte[] regionName, Get get) throws IOException {
     checkOpen();
+    final long startTime = System.nanoTime();
     requestCount.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       return region.get(get, getLockFromId(get.getLockId()));
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
+    } finally {
+      this.metrics.getLatencies.update(System.nanoTime() - startTime);
     }
   }
 
@@ -1988,6 +1991,7 @@ public void put(final byte[] regionName, final Put put) throws IOException {
       throw new IllegalArgumentException("update has null row");
     }
 
+    final long startTime = System.nanoTime();
     checkOpen();
     this.requestCount.incrementAndGet();
     HRegion region = getRegion(regionName);
@@ -1999,6 +2003,8 @@ public void put(final byte[] regionName, final Put put) throws IOException {
       region.put(put, getLockFromId(put.getLockId()), writeToWAL);
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
+    } finally {
+      this.metrics.putLatencies.update(System.nanoTime() - startTime);
     }
   }
 
@@ -2006,6 +2012,9 @@ public int put(final byte[] regionName, final List<Put> puts)
       throws IOException {
     checkOpen();
     HRegion region = null;
+    int i = 0;
+
+    final long startTime = System.nanoTime();
     try {
       region = getRegion(regionName);
       if (!region.getRegionInfo().isMetaTable()) {
@@ -2015,7 +2024,6 @@ public int put(final byte[] regionName, final List<Put> puts)
       @SuppressWarnings("unchecked")
       Pair<Put, Integer>[] putsWithLocks = new Pair[puts.size()];
 
-      int i = 0;
       for (Put p : puts) {
         Integer lock = getLockFromId(p.getLockId());
         putsWithLocks[i++] = new Pair<Put, Integer>(p, lock);
@@ -2031,6 +2039,14 @@ public int put(final byte[] regionName, final List<Put> puts)
       return -1;
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
+    } finally {
+      // going to count this as puts.size() PUTs for latency calculations
+      final long totalTime = System.nanoTime() - startTime;
+      final long putCount = i;
+      final long perPutTime = totalTime / putCount;
+      for (int request = 0; request < putCount; request++) {
+        this.metrics.putLatencies.update(perPutTime);
+      }
     }
   }
 
@@ -2492,6 +2508,7 @@ public void leaseExpired() {
   public void delete(final byte[] regionName, final Delete delete)
       throws IOException {
     checkOpen();
+    final long startTime = System.nanoTime();
     try {
       boolean writeToWAL = delete.getWriteToWAL();
       this.requestCount.incrementAndGet();
@@ -2503,6 +2520,8 @@ public void delete(final byte[] regionName, final Delete delete)
       region.delete(delete, lid, writeToWAL);
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
+    } finally {
+      this.metrics.deleteLatencies.update(System.nanoTime() - startTime);
     }
   }
 
@@ -2520,10 +2539,12 @@ public int delete(final byte[] regionName, final List<Delete> deletes)
       int size = deletes.size();
       Integer[] locks = new Integer[size];
       for (Delete delete : deletes) {
+        final long startTime = System.nanoTime();
         this.requestCount.incrementAndGet();
         locks[i] = getLockFromId(delete.getLockId());
         region.delete(delete, locks[i], delete.getWriteToWAL());
         i++;
+        this.metrics.deleteLatencies.update(System.nanoTime() - startTime);
       }
     } catch (WrongRegionException ex) {
       LOG.debug("Batch deletes: " + i, ex);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index a5f8b010366b..4d11d7b5088b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -19,12 +19,19 @@
  */
 package org.apache.hadoop.hbase.regionserver.metrics;
 
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+import java.util.List;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.metrics.HBaseInfo;
 import org.apache.hadoop.hbase.metrics.MetricsRate;
 import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
+import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+import org.apache.hadoop.hbase.metrics.histogram.Snapshot;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Strings;
@@ -41,11 +48,6 @@
 import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
 import org.apache.hadoop.util.StringUtils;
 
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
-import java.util.List;
-
 /**
  * This class is for maintaining the various regionserver statistics
  * and publishing them through the metrics interfaces.
@@ -76,43 +78,51 @@ public class RegionServerMetrics implements Updater {
   /**
    * Block cache size.
    */
-  public final MetricsLongValue blockCacheSize = new MetricsLongValue("blockCacheSize", registry);
+  public final MetricsLongValue blockCacheSize = 
+      new MetricsLongValue("blockCacheSize", registry);
 
   /**
    * Block cache free size.
    */
-  public final MetricsLongValue blockCacheFree = new MetricsLongValue("blockCacheFree", registry);
+  public final MetricsLongValue blockCacheFree = 
+      new MetricsLongValue("blockCacheFree", registry);
 
   /**
    * Block cache item count.
    */
-  public final MetricsLongValue blockCacheCount = new MetricsLongValue("blockCacheCount", registry);
+  public final MetricsLongValue blockCacheCount = 
+      new MetricsLongValue("blockCacheCount", registry);
 
   /**
    * Block cache hit count.
    */
-  public final MetricsLongValue blockCacheHitCount = new MetricsLongValue("blockCacheHitCount", registry);
+  public final MetricsLongValue blockCacheHitCount = 
+      new MetricsLongValue("blockCacheHitCount", registry);
 
   /**
    * Block cache miss count.
    */
-  public final MetricsLongValue blockCacheMissCount = new MetricsLongValue("blockCacheMissCount", registry);
+  public final MetricsLongValue blockCacheMissCount = 
+      new MetricsLongValue("blockCacheMissCount", registry);
 
   /**
    * Block cache evict count.
    */
-  public final MetricsLongValue blockCacheEvictedCount = new MetricsLongValue("blockCacheEvictedCount", registry);
+  public final MetricsLongValue blockCacheEvictedCount = 
+      new MetricsLongValue("blockCacheEvictedCount", registry);
 
   /**
    * Block hit ratio.
    */
-  public final MetricsIntValue blockCacheHitRatio = new MetricsIntValue("blockCacheHitRatio", registry);
+  public final MetricsIntValue blockCacheHitRatio = 
+      new MetricsIntValue("blockCacheHitRatio", registry);
 
   /**
    * Block hit caching ratio.  This only includes the requests to the block
    * cache where caching was turned on.  See HBASE-2253.
    */
-  public final MetricsIntValue blockCacheHitCachingRatio = new MetricsIntValue("blockCacheHitCachingRatio", registry);
+  public final MetricsIntValue blockCacheHitCachingRatio = 
+      new MetricsIntValue("blockCacheHitCachingRatio", registry);
 
   /** Block hit ratio for past N periods. */
   public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry);
@@ -120,6 +130,25 @@ public class RegionServerMetrics implements Updater {
   /** Block hit caching ratio for past N periods */
   public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
 
+  /**
+   * a latency histogram on 'get' requests
+   */
+  public final MetricsHistogram getLatencies = 
+      new MetricsHistogram("getRequestLatency", registry);
+ 
+  /**
+   * a latency histogram on 'delete' requests
+   */
+  public final MetricsHistogram deleteLatencies = 
+      new MetricsHistogram("deleteRequestLatency", registry);
+ 
+  /**
+   * a latency histogram on 'put' requests
+   */
+  public final MetricsHistogram putLatencies = 
+      new MetricsHistogram("putRequestLatency", registry);
+ 
+  
   /*
    * Count of requests to the regionservers since last call to metrics update
    */
@@ -133,17 +162,20 @@ public class RegionServerMetrics implements Updater {
   /**
    * Count of storefiles open on the regionserver.
    */
-  public final MetricsIntValue storefiles = new MetricsIntValue("storefiles", registry);
+  public final MetricsIntValue storefiles = 
+      new MetricsIntValue("storefiles", registry);
 
   /**
    * Count of read requests
    */
-  public final MetricsLongValue readRequestsCount = new MetricsLongValue("readRequestsCount", registry);
+  public final MetricsLongValue readRequestsCount = 
+      new MetricsLongValue("readRequestsCount", registry);
 
   /**
    * Count of write requests
    */
-  public final MetricsLongValue writeRequestsCount = new MetricsLongValue("writeRequestsCount", registry);
+  public final MetricsLongValue writeRequestsCount = 
+      new MetricsLongValue("writeRequestsCount", registry);
 
   /**
    */
@@ -187,7 +219,26 @@ public class RegionServerMetrics implements Updater {
     new MetricsIntValue("flushQueueSize", registry);
 
   /**
-   * filesystem sequential read latency
+   * filesystem sequential read latency distribution
+   */
+  public final MetricsHistogram fsReadLatencyHistogram = 
+      new MetricsHistogram("fsReadLatencyHistogram", registry);
+
+  /**
+   * filesystem pread latency distribution
+   */
+  public final MetricsHistogram fsPreadLatencyHistogram = 
+      new MetricsHistogram("fsPreadLatencyHistogram", registry);
+
+  /**
+   * Metrics on the distribution of filesystem write latencies (improved version of fsWriteLatency)
+   */
+  public final MetricsHistogram fsWriteLatencyHistogram = 
+      new MetricsHistogram("fsWriteLatencyHistogram", registry);
+
+  
+  /**
+   * filesystem read latency
    */
   public final MetricsTimeVaryingRate fsReadLatency =
     new MetricsTimeVaryingRate("fsReadLatency", registry);
@@ -216,6 +267,7 @@ public class RegionServerMetrics implements Updater {
   public final MetricsTimeVaryingRate fsSyncLatency =
     new MetricsTimeVaryingRate("fsSyncLatency", registry);
 
+  
   /**
    * time each scheduled compaction takes
    */
@@ -329,6 +381,10 @@ public void doUpdates(MetricsContext caller) {
       this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
       this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
 
+      this.putLatencies.pushMetric(this.metricsRecord);
+      this.deleteLatencies.pushMetric(this.metricsRecord);
+      this.getLatencies.pushMetric(this.metricsRecord);
+      
       // Mix in HFile and HLog metrics
       // Be careful. Here is code for MTVR from up in hadoop:
       // public synchronized void inc(final int numOps, final long time) {
@@ -359,11 +415,27 @@ public void doUpdates(MetricsContext caller) {
        *      by compaction & flush metrics.
        */
 
+      for(Long latency : HFile.getReadLatenciesNanos()) {
+        this.fsReadLatencyHistogram.update(latency);
+      }
+      for(Long latency : HFile.getPreadLatenciesNanos()) {
+        this.fsPreadLatencyHistogram.update(latency);
+      }
+      for(Long latency : HFile.getWriteLatenciesNanos()) {
+        this.fsWriteLatencyHistogram.update(latency);
+      }
+            
+
       // push the result
       this.fsPreadLatency.pushMetric(this.metricsRecord);
       this.fsReadLatency.pushMetric(this.metricsRecord);
       this.fsWriteLatency.pushMetric(this.metricsRecord);
       this.fsWriteSize.pushMetric(this.metricsRecord);
+      
+      this.fsReadLatencyHistogram.pushMetric(this.metricsRecord);
+      this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord);
+      this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord);
+
       this.fsSyncLatency.pushMetric(this.metricsRecord);
       this.compactionTime.pushMetric(this.metricsRecord);
       this.compactionSize.pushMetric(this.metricsRecord);
@@ -496,6 +568,40 @@ public String toString() {
         Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
     sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
         Long.valueOf(this.slowHLogAppendCount.get()));
+    sb = appendHistogram(sb, this.deleteLatencies);
+    sb = appendHistogram(sb, this.getLatencies);
+    sb = appendHistogram(sb, this.putLatencies);
+    sb = appendHistogram(sb, this.fsReadLatencyHistogram);
+    sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
+    sb = appendHistogram(sb, this.fsWriteLatencyHistogram);
+
     return sb.toString();
   }
+  
+  private StringBuilder appendHistogram(StringBuilder sb, 
+      MetricsHistogram histogram) {
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "Mean", 
+        StringUtils.limitDecimalTo2(histogram.getMean()));
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "Count", 
+        StringUtils.limitDecimalTo2(histogram.getCount()));
+    final Snapshot s = histogram.getSnapshot();
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "Median", 
+        StringUtils.limitDecimalTo2(s.getMedian()));
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "75th", 
+        StringUtils.limitDecimalTo2(s.get75thPercentile()));
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "95th", 
+        StringUtils.limitDecimalTo2(s.get95thPercentile()));
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "99th", 
+        StringUtils.limitDecimalTo2(s.get99thPercentile()));
+    sb = Strings.appendKeyValue(sb, 
+        histogram.getName() + "999th", 
+        StringUtils.limitDecimalTo2(s.get999thPercentile()));
+    return sb;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 24e75596b1f4..7fabbe8c9331 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1879,4 +1879,4 @@ public static void main(String[] args) throws IOException {
       System.exit(-1);
     }
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index ce880a598b55..c1c7ea4c84b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -25,6 +25,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -176,4 +177,24 @@ public static ThreadPoolExecutor getBoundedCachedThreadPool(
     boundedCachedThreadPool.allowCoreThreadTimeOut(true);
     return boundedCachedThreadPool;
   }
+  
+  
+  /**
+   * Returns a {@link java.util.concurrent.ThreadFactory} that names each
+   * created thread uniquely, with a common prefix.
+   * 
+   * @param prefix  The prefix of every created Thread's name
+   * @return a {@link java.util.concurrent.ThreadFactory} that names threads
+   */
+  public static ThreadFactory getNamedThreadFactory(final String prefix) {
+    return new ThreadFactory() {
+
+      private final AtomicInteger threadNumber = new AtomicInteger(1);
+      
+      @Override
+      public Thread newThread(Runnable r) {
+        return new Thread(r, prefix + threadNumber.getAndIncrement());
+      }
+    };
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java
new file mode 100644
index 000000000000..c9e5b6e3af09
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.Test;
+
+public class TestExactCounterMetric {
+
+  @Test
+  public void testBasic() {
+    final ExactCounterMetric counter = new ExactCounterMetric("testCounter", null);
+    for (int i = 1; i <= 10; i++) {
+      for (int j = 0; j < i; j++) {
+        counter.update(i + "");
+      }
+    }
+    
+    List<Pair<String, Long>> topFive = counter.getTop(5);
+    Long i = 10L;
+    for (Pair<String, Long> entry : topFive) {
+      Assert.assertEquals(i + "", entry.getFirst());
+      Assert.assertEquals(i, entry.getSecond());
+      i--;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
new file mode 100644
index 000000000000..8b0153e18781
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hbase.metrics.histogram.ExponentiallyDecayingSample;
+import org.apache.hadoop.hbase.metrics.histogram.Snapshot;
+import org.junit.Test;
+
+public class TestExponentiallyDecayingSample {
+  
+  @Test
+  public void testBasic() {
+      final ExponentiallyDecayingSample sample = 
+          new ExponentiallyDecayingSample(100, 0.99);
+      
+      for (int i = 0; i < 1000; i++) {
+          sample.update(i);
+      }
+      Assert.assertEquals(100, sample.size());
+      
+      final Snapshot snapshot = sample.getSnapshot();
+      Assert.assertEquals(100, snapshot.size());
+
+      for (double i : snapshot.getValues()) {
+        Assert.assertTrue(i >= 0.0 && i < 1000.0);
+      }
+  }
+
+  @Test
+  public void testTooBig() throws Exception {
+      final ExponentiallyDecayingSample sample = 
+          new ExponentiallyDecayingSample(100, 0.99);
+      for (int i = 0; i < 10; i++) {
+          sample.update(i);
+      }
+      Assert.assertEquals(10, sample.size());
+
+      final Snapshot snapshot = sample.getSnapshot();
+      Assert.assertEquals(10, sample.size());
+
+      for (double i : snapshot.getValues()) {
+        Assert.assertTrue(i >= 0.0 && i < 1000.0);
+      }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
new file mode 100644
index 000000000000..4b1ae151d7e7
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+import org.apache.hadoop.hbase.metrics.histogram.Snapshot;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestMetricsHistogram {
+
+  @Test
+  public void testBasicUniform() {
+    MetricsHistogram h = new MetricsHistogram("testHistogram", null);
+
+    for (int i = 0; i < 100; i++) {
+      h.update(i);
+    }
+
+    Assert.assertEquals(100, h.getCount());
+    Assert.assertEquals(0, h.getMin());
+    Assert.assertEquals(99, h.getMax());
+  }
+
+  private static int safeIndex(int i, int len) {
+    if (i < len && i>= 0) {
+      return i;
+    } else if (i >= len) {
+      return len - 1; 
+    } else {
+      return 0;
+    }    
+  }
+  
+  @Test
+  public void testRandom() {
+    final Random r = new Random();
+    final MetricsHistogram h = new MetricsHistogram("testHistogram", null);
+
+    final long[] data = new long[1000];
+
+    for (int i = 0; i < data.length; i++) {
+      data[i] = (long) (r.nextGaussian() * 10000.0);
+      h.update(data[i]);
+    }
+
+    final Snapshot s = h.getSnapshot();
+    Arrays.sort(data);
+
+    // as long as the histogram chooses an item with index N+/-slop, accept it
+    final int slop = 20;
+
+    // make sure the median, 75th percentile and 95th percentile are good
+    final int medianIndex = data.length / 2;
+    final long minAcceptableMedian = data[safeIndex(medianIndex - slop, 
+        data.length)];
+    final long maxAcceptableMedian = data[safeIndex(medianIndex + slop, 
+        data.length)];
+    Assert.assertTrue(s.getMedian() >= minAcceptableMedian 
+        && s.getMedian() <= maxAcceptableMedian);
+
+    final int seventyFifthIndex = (int) (data.length * 0.75);
+    final long minAcceptableseventyFifth = data[safeIndex(seventyFifthIndex 
+        - slop, data.length)];
+    final long maxAcceptableseventyFifth = data[safeIndex(seventyFifthIndex 
+        + slop, data.length)];
+    Assert.assertTrue(s.get75thPercentile() >= minAcceptableseventyFifth 
+        && s.get75thPercentile() <= maxAcceptableseventyFifth);
+
+    final int ninetyFifthIndex = (int) (data.length * 0.95);
+    final long minAcceptableninetyFifth = data[safeIndex(ninetyFifthIndex 
+        - slop, data.length)];
+    final long maxAcceptableninetyFifth = data[safeIndex(ninetyFifthIndex 
+        + slop, data.length)];
+    Assert.assertTrue(s.get95thPercentile() >= minAcceptableninetyFifth 
+        && s.get95thPercentile() <= maxAcceptableninetyFifth);
+
+  }
+}

From 167369c7e408938cbea38c483a68fff25f64805c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 26 Mar 2012 21:02:03 +0000
Subject: [PATCH 0075/1540] Update CHANGES.txt

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305591 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 361 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 358 insertions(+), 3 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index a49d47097285..68c6591ccdcb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,8 +1,363 @@
 HBase Change Log
 
-Release 0.93.0 - Unreleased
-  *DO NOT ADD ISSUES HERE ON COMMIT ANY MORE.  WE'LL GENERATE THE LIST
-  FROM JIRA INSTEAD WHEN WE MAKE A RELEASE*
+Release 0.94.0 - 3/26/2012
+Sub-task
+
+    [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently
+    [HBASE-4345] - Ensure that Scanners that read from the storefiles respect MVCC
+    [HBASE-4346] - Optimise the storage that we use for storing MVCC information.
+    [HBASE-4485] - Eliminate window of missing Data
+    [HBASE-4517] - Document new replication features in 0.92
+    [HBASE-4544] - Rename RWCC to MVCC
+    [HBASE-4594] - Ensure that KV's newer than the oldest-living-scanner is not accounted for the maxVersions during flush/compaction.
+    [HBASE-4661] - Ability to export the list of files for a some or all column families for a given region
+    [HBASE-4682] - Support deleted rows using Import/Export
+    [HBASE-4908] - HBase cluster test tool (port from 0.89-fb)
+    [HBASE-4911] - Clean shutdown
+    [HBASE-4979] - Setting KEEP_DELETE_CELLS fails in shell
+    [HBASE-4981] - add raw scan support to shell
+    [HBASE-4998] - Support deleted rows in CopyTable
+    [HBASE-5005] - Add DEFAULT_MIN_VERSIONS to HColumnDescriptor.DEFAULT_VALUES
+    [HBASE-5058] - Allow HBaseAdmin to use an existing connection
+    [HBASE-5096] - Replication does not handle deletes correctly.
+    [HBASE-5118] - Fix Scan documentation
+    [HBASE-5143] - Fix config typo in pluggable load balancer factory
+    [HBASE-5203] - Group atomic put/delete operation into a single WALEdit to handle region server failures.
+    [HBASE-5266] - Add documentation for ColumnRangeFilter
+    [HBASE-5346] - Fix testColumnFamilyCompression and test_TIMERANGE in TestHFileOutputFormat
+    [HBASE-5368] - Move PrefixSplitKeyPolicy out of the src/test into src, so it is accessible in HBase installs
+    [HBASE-5371] - Introduce AccessControllerProtocol.checkPermissions(Permission[] permissons) API
+    [HBASE-5413] - Rename RowMutation to RowMutations
+    [HBASE-5431] - Improve delete marker handling in Import M/R jobs
+    [HBASE-5460] - Add protobuf as M/R dependency jar
+    [HBASE-5497] - Add protobuf as M/R dependency jar (mapred)
+    [HBASE-5523] - Fix Delete Timerange logic for KEEP_DELETED_CELLS
+    [HBASE-5541] - Avoid holding the rowlock during HLog sync in HRegion.mutateRowWithLocks
+
+Bug
+
+    [HBASE-2856] - TestAcidGuarantee broken on trunk
+    [HBASE-3690] - Option to Exclude Bulk Import Files from Minor Compaction
+    [HBASE-3987] - Fix a NullPointerException on a failure to load Bloom filter data
+    [HBASE-4065] - TableOutputFormat ignores failure to create table instance
+    [HBASE-4078] - Silent Data Offlining During HDFS Flakiness
+    [HBASE-4105] - Stargate does not support Content-Type: application/json and Content-Encoding: gzip in parallel
+    [HBASE-4116] - [stargate] StringIndexOutOfBoundsException in row spec parse
+    [HBASE-4326] - Tests that use HBaseTestingUtility.startMiniCluster(n) should shutdown with HBaseTestingUtility.shutdownMiniCluster.
+    [HBASE-4397] - -ROOT-, .META. tables stay offline for too long in recovery phase after all RSs are shutdown at the same time
+    [HBASE-4476] - Compactions must fail if column tracker gets columns out of order
+    [HBASE-4496] - HFile V2 does not honor setCacheBlocks when scanning.
+    [HBASE-4607] - Split log worker should terminate properly when waiting for znode
+    [HBASE-4609] - ThriftServer.getRegionInfo() is expecting old ServerName format, need to use new Addressing class instead
+    [HBASE-4610] - Port HBASE-3380 (Master failover can split logs of live servers) to 92/trunk (definitely bring in config params, decide if we need to do more to fix the bug)
+    [HBASE-4626] - Filters unnecessarily copy byte arrays...
+    [HBASE-4645] - Edits Log recovery losing data across column families
+    [HBASE-4648] - Bytes.toBigDecimal() doesn't use offset
+    [HBASE-4658] - Put attributes are not exposed via the ThriftServer
+    [HBASE-4673] - NPE in HFileReaderV2.close during major compaction when hfile.block.cache.size is set to 0
+    [HBASE-4679] - Thrift null mutation error
+    [HBASE-4691] - Remove more unnecessary byte[] copies from KeyValues
+    [HBASE-4729] - Clash between region unassign and splitting kills the master
+    [HBASE-4745] - LRU Statistics thread should be daemon
+    [HBASE-4769] - Abort RegionServer Immediately on OOME
+    [HBASE-4776] - HLog.closed should be checked inside of updateLock
+    [HBASE-4778] - Don't ignore corrupt StoreFiles when opening a region
+    [HBASE-4790] - Occasional TestDistributedLogSplitting failure
+    [HBASE-4792] - SplitRegionHandler doesn't care if it deletes the znode or not, leaves the parent region stuck offline
+    [HBASE-4795] - Fix TestHFileBlock when running on a 32-bit JVM
+    [HBASE-4797] - [availability] Skip recovered.edits files with edits we know older than what region currently has
+    [HBASE-4805] - Allow better control of resource consumption in HTable
+    [HBASE-4819] - TestShell broke in trunk; typo
+    [HBASE-4825] - TestRegionServersMetrics and TestZKLeaderManager are not categorized (small/medium/large)
+    [HBASE-4826] - Modify hbasetests.sh to take into account the new pom.xml with surefire
+    [HBASE-4832] - TestRegionServerCoprocessorExceptionWithAbort fails if the region server stops too fast
+    [HBASE-4853] - HBASE-4789 does overzealous pruning of seqids
+    [HBASE-4874] - Run tests with non-secure random, some tests hang otherwise
+    [HBASE-4878] - Master crash when splitting hlog may cause data loss
+    [HBASE-4886] - truncate fails in HBase shell
+    [HBASE-4890] - fix possible NPE in HConnectionManager
+    [HBASE-4932] - Block cache can be mistakenly instantiated by tools
+    [HBASE-4936] - Cached HRegionInterface connections crash when getting UnknownHost exceptions
+    [HBASE-4937] - Error in Quick Start Shell Exercises
+    [HBASE-4942] - HMaster is unable to start of HFile V1 is used
+    [HBASE-4946] - HTable.coprocessorExec (and possibly coprocessorProxy) does not work with dynamically loaded coprocessors (from hdfs or local system), because the RPC system tries to deserialize an unknown class.
+    [HBASE-4993] - Performance regression in minicluster creation
+    [HBASE-5003] - If the master is started with a wrong root dir, it gets stuck and can't be killed
+    [HBASE-5010] - Filter HFiles based on TTL
+    [HBASE-5015] - Remove some leaks in tests due to lack of HTable.close()
+    [HBASE-5026] - Add coprocessor hook to HRegionServer.ScannerListener.leaseExpired()
+    [HBASE-5027] - HConnection.create(final Connection conf) does not clone, it creates a new Configuration reading *.xmls and then does a merge.
+    [HBASE-5038] - Some tests leak connections
+    [HBASE-5041] - Major compaction on non existing table does not throw error
+    [HBASE-5051] - HBaseTestingUtility#getHBaseAdmin() creates a new HBaseAdmin instance at each call
+    [HBASE-5053] - HCM Tests leak connections
+    [HBASE-5055] - Build against hadoop 0.22 broken
+    [HBASE-5068] - RC1 can not build its hadoop-0.23 profile
+    [HBASE-5085] - fix test-patch script from setting the ulimit
+    [HBASE-5088] - A concurrency issue on SoftValueSortedMap
+    [HBASE-5091] - [replication] Update replication doc to reflect current znode structure
+    [HBASE-5099] - ZK event thread waiting for root region assignment may block server shutdown handler for the region sever the root region was on
+    [HBASE-5100] - Rollback of split could cause closed region to be opened again
+    [HBASE-5103] - Fix improper master znode deserialization
+    [HBASE-5120] - Timeout monitor races with table disable handler
+    [HBASE-5121] - MajorCompaction may affect scan's correctness
+    [HBASE-5141] - Memory leak in MonitoredRPCHandlerImpl
+    [HBASE-5152] - Region is on service before completing initialization when doing rollback of split, it will affect read correctness
+    [HBASE-5161] - Compaction algorithm should prioritize reference files
+    [HBASE-5163] - TestLogRolling#testLogRollOnDatanodeDeath fails sometimes on Jenkins or hadoop QA ("The directory is already locked.")
+    [HBASE-5172] - HTableInterface should extend java.io.Closeable
+    [HBASE-5176] - AssignmentManager#getRegion: logging nit adds a redundant '+'
+    [HBASE-5182] - TBoundedThreadPoolServer threadKeepAliveTimeSec is not configured properly
+    [HBASE-5195] - [Coprocessors] preGet hook does not allow overriding or wrapping filter on incoming Get
+    [HBASE-5196] - Failure in region split after PONR could cause region hole
+    [HBASE-5200] - AM.ProcessRegionInTransition() and AM.handleRegion() race thus leaving the region assignment inconsistent
+    [HBASE-5206] - Port HBASE-5155 to 0.92, 0.94, and TRUNK
+    [HBASE-5212] - Fix test TestTableMapReduce against 0.23.
+    [HBASE-5221] - bin/hbase script doesn't look for Hadoop jars in the right place in trunk layout
+    [HBASE-5228] - [REST] Rip out "transform" feature
+    [HBASE-5267] - Add a configuration to disable the slab cache by default
+    [HBASE-5271] - Result.getValue and Result.getColumnLatest return the wrong column.
+    [HBASE-5278] - HBase shell script refers to removed "migrate" functionality
+    [HBASE-5281] - Should a failure in creating an unassigned node abort the master?
+    [HBASE-5282] - Possible file handle leak with truncated HLog file.
+    [HBASE-5283] - Request counters may become negative for heavily loaded regions
+    [HBASE-5286] - bin/hbase's logic of adding Hadoop jar files to the classpath is fragile when presented with split packaged Hadoop 0.23 installation
+    [HBASE-5288] - Security source code dirs missing from 0.92.0 release tarballs.
+    [HBASE-5290] - [FindBugs] Synchronization on boxed primitive
+    [HBASE-5292] - getsize per-CF metric incorrectly counts compaction related reads as well
+    [HBASE-5317] - Fix TestHFileOutputFormat to work against hadoop 0.23
+    [HBASE-5327] - Print a message when an invalid hbase.rootdir is passed
+    [HBASE-5331] - Off by one bug in util.HMerge
+    [HBASE-5345] - CheckAndPut doesn't work when value is empty byte[]
+    [HBASE-5348] - Constraint configuration loaded with bloat
+    [HBASE-5350] - Fix jamon generated package names
+    [HBASE-5351] - hbase completebulkload to a new table fails in a race
+    [HBASE-5364] - Fix source files missing licenses in 0.92 and trunk
+    [HBASE-5384] - Up heap used by hadoopqa
+    [HBASE-5387] - Reuse compression streams in HFileBlock.Writer
+    [HBASE-5398] - HBase shell disable_all/enable_all/drop_all promp wrong tables for confirmation
+    [HBASE-5415] - FSTableDescriptors should handle random folders in hbase.root.dir better
+    [HBASE-5420] - TestImportTsv does not shut down MR Cluster correctly (fails against 0.23 hadoop)
+    [HBASE-5423] - Regionserver may block forever on waitOnAllRegionsToClose when aborting
+    [HBASE-5425] - Punt on the timeout doesn't work in BulkEnabler#waitUntilDone (master's EnableTableHandler)
+    [HBASE-5437] - HRegionThriftServer does not start because of a bug in HbaseHandlerMetricsProxy
+    [HBASE-5466] - Opening a table also opens the metatable and never closes it.
+    [HBASE-5470] - Make DataBlockEncodingTool work correctly with no native compression codecs loaded
+    [HBASE-5473] - Metrics does not push pread time
+    [HBASE-5477] - Cannot build RPM for hbase-0.92.0
+    [HBASE-5480] - Fixups to MultithreadedTableMapper for Hadoop 0.23.2+
+    [HBASE-5481] - Uncaught UnknownHostException prevents HBase from starting
+    [HBASE-5484] - Spelling mistake in error message in HMasterCommandLine
+    [HBASE-5485] - LogCleaner refers to non-existant SnapshotLogCleaner
+    [HBASE-5499] - dev-support/test-patch.sh does not have execute perms
+    [HBASE-5502] - region_mover.rb fails to load regions back to original server for regions only containing empty tables.
+    [HBASE-5507] - ThriftServerRunner.HbaseHandler.getRegionInfo() and getTableRegions() do not use ByteBuffer correctly
+    [HBASE-5514] - Compile against hadoop 0.24-SNAPSHOT
+    [HBASE-5522] - hbase 0.92 test artifacts are missing from Maven central
+    [HBASE-5524] - Add a couple of more filters to our rat exclusion set
+    [HBASE-5529] - MR test failures becuase MALLOC_ARENA_MAX is not set
+    [HBASE-5531] - Maven hadoop profile (version 23) needs to be updated with latest 23 snapshot
+    [HBASE-5535] - Make the functions in task monitor synchronized
+    [HBASE-5537] - MXBean shouldn't have a dependence on InterfaceStability until 0.96
+    [HBASE-5552] - Clean up our jmx view; its a bit of a mess
+    [HBASE-5562] - test-patch.sh reports a javadoc warning when there are no new javadoc warnings
+    [HBASE-5563] - HRegionInfo#compareTo should compare regionId as well
+    [HBASE-5567] - test-patch.sh has logic error in findbugs check
+    [HBASE-5568] - Multi concurrent flushcache() for one region could cause data loss
+    [HBASE-5569] - Do not collect deleted KVs when they are still in use by a scanner.
+    [HBASE-5574] - DEFAULT_MAX_FILE_SIZE defaults to a negative value
+    [HBASE-5579] - A Delete Version could mask other values
+    [HBASE-5581] - Creating a table with invalid syntax does not give an error message when it fails
+    [HBASE-5586] - [replication] NPE in ReplicationSource when creating a stream to an inexistent cluster
+    [HBASE-5597] - Findbugs check in test-patch.sh always fails
+    [HBASE-5603] - rolling-restart.sh script hangs when attempting to detect expiration of /hbase/master znode.
+    [HBASE-5613] - ThriftServer getTableRegions does not return serverName and port
+    [HBASE-5615] - the master never does balance because of balancing the parent region
+    [HBASE-5623] - Race condition when rolling the HLog and hlogFlush
+    [HBASE-5624] - Aborting regionserver when splitting region, may cause daughter region not assigned by ServerShutdownHandler.
+    [HBASE-5633] - NPE reading ZK config in HBase
+
+Improvement
+
+    [HBASE-1744] - Thrift server to match the new java api.
+    [HBASE-2418] - add support for ZooKeeper authentication
+    [HBASE-3373] - Allow regions to be load-balanced by table
+    [HBASE-3433] - Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (why doesn't keyonlyfilter make copies rather than mutate -- HBASE-3211)?
+    [HBASE-3512] - Coprocessors: Shell support for listing currently loaded coprocessor set
+    [HBASE-3565] - Add metrics to keep track of slow HLog appends
+    [HBASE-3763] - Add Bloom Block Index Support
+    [HBASE-3850] - Log more details when a scanner lease expires
+    [HBASE-3924] - Improve Shell's CLI help
+    [HBASE-3949] - Add "Master" link to RegionServer pages
+    [HBASE-4058] - Extend TestHBaseFsck with a complete .META. recovery scenario
+    [HBASE-4062] - Multi-column scanner unit test
+    [HBASE-4070] - [Coprocessors] Improve region server metrics to report loaded coprocessors to master
+    [HBASE-4076] - hbase should pick up HADOOP_CONF_DIR on its classpath
+    [HBASE-4131] - Make the Replication Service pluggable via a standard interface definition
+    [HBASE-4132] - Extend the WALActionsListener API to accomodate log archival
+    [HBASE-4145] - Provide metrics for hbase client
+    [HBASE-4213] - Support for fault tolerant, instant schema updates with out master's intervention (i.e with out enable/disable and bulk assign/unassign) through ZK.
+    [HBASE-4218] - Data Block Encoding of KeyValues (aka delta encoding / prefix compression)
+    [HBASE-4365] - Add a decent heuristic for region size
+    [HBASE-4418] - Show all the hbase configuration in the web ui
+    [HBASE-4439] - Move ClientScanner out of HTable
+    [HBASE-4440] - add an option to presplit table to PerformanceEvaluation
+    [HBASE-4461] - Expose getRowOrBefore via Thrift
+    [HBASE-4463] - Run more aggressive compactions during off peak hours
+    [HBASE-4465] - Lazy-seek optimization for StoreFile scanners
+    [HBASE-4469] - Avoid top row seek by looking up ROWCOL bloomfilter
+    [HBASE-4480] - Testing script to simplify local testing
+    [HBASE-4487] - The increment operation can release the rowlock before sync-ing the Hlog
+    [HBASE-4489] - Better key splitting in RegionSplitter
+    [HBASE-4519] - 25s sleep when expiring sessions in tests
+    [HBASE-4522] - Make hbase-site-custom.xml override the hbase-site.xml
+    [HBASE-4528] - The put operation can release the rowlock before sync-ing the Hlog
+    [HBASE-4532] - Avoid top row seek by dedicated bloom filter for delete family bloom filter
+    [HBASE-4542] - add filter info to slow query logging
+    [HBASE-4554] - Allow set/unset coprocessor table attributes from shell.
+    [HBASE-4568] - Make zk dump jsp response more quickly
+    [HBASE-4585] - Avoid next operations (and instead reseek) when current kv is deleted
+    [HBASE-4591] - TTL for old HLogs should be calculated from last modification time.
+    [HBASE-4612] - Allow ColumnPrefixFilter to support multiple prefixes
+    [HBASE-4627] - Ability to specify a custom start/end to RegionSplitter
+    [HBASE-4628] - Enhance Table Create Presplit Functionality within the HBase Shell
+    [HBASE-4640] - Catch ClosedChannelException and document it
+    [HBASE-4657] - Improve the efficiency of our MR jobs with a few configurations
+    [HBASE-4669] - Add an option of using round-robin assignment for enabling table
+    [HBASE-4696] - HRegionThriftServer' might have to indefinitely do redirtects
+    [HBASE-4704] - A JRuby script for identifying active master
+    [HBASE-4737] - Categorize the tests into small/medium/large; allow small tests to be run in parallel within a single JVM
+    [HBASE-4746] - Use a random ZK client port in unit tests so we can run them in parallel
+    [HBASE-4752] - Don't create an unnecessary LinkedList when evicting from the BlockCache
+    [HBASE-4760] - Add Developer Debug Options to HBase Config
+    [HBASE-4761] - Add Developer Debug Options to HBase Config
+    [HBASE-4764] - naming errors for TestHLogUtils and SoftValueSortedMapTest
+    [HBASE-4779] - TestHTablePool, TestScanWithBloomError, TestRegionSplitCalculator are not tagged and TestPoolMap should not use TestSuite
+    [HBASE-4780] - Lower mini cluster shutdown time in HRegionServer#waitOnAllRegionsToClose and ServerManager#letRegionServersShutdown
+    [HBASE-4781] - Pom update to use the new versions of surefire & junit
+    [HBASE-4783] - Improve RowCounter to count rows in a specific key range.
+    [HBASE-4787] - Make corePool as a configurable parameter in HTable
+    [HBASE-4798] - Sleeps and synchronisation improvements for tests
+    [HBASE-4809] - Per-CF set RPC metrics
+    [HBASE-4820] - Distributed log splitting coding enhancement to make it easier to understand, no semantics change
+    [HBASE-4847] - Activate single jvm for small tests on jenkins
+    [HBASE-4863] - Make Thrift server thread pool bounded and add a command-line UI test
+    [HBASE-4884] - Allow environment overrides for various HBase processes
+    [HBASE-4933] - Ability to calculate the blockcache hit ratio for the last few minutes
+    [HBASE-4938] - Create a HRegion.getScanner public method that allows reading from a specified readPoint
+    [HBASE-4940] - hadoop-metrics.properties can include configuration of the "rest" context for ganglia
+    [HBASE-4957] - Clean up some log messages, code in RecoverableZooKeeper
+    [HBASE-4964] - Add builddate, make less sections in toc, and add header and footer customizations
+    [HBASE-4965] - Monitor the open file descriptors and the threads counters during the unit tests
+    [HBASE-4970] - Add a parameter so that keepAliveTime of Htable thread pool can be changed
+    [HBASE-4971] - Useless sleeps in TestTimestampsFilter and TestMultipleTimestamps
+    [HBASE-4973] - On failure, HBaseAdmin sleeps one time too many
+    [HBASE-4989] - Metrics to measure sequential reads and random reads separately
+    [HBASE-4995] - Increase zk maxClientCnxns to give us some head room
+    [HBASE-5014] - PutSortReducer should adhere to memory limits
+    [HBASE-5017] - Bump the default hfile.block.cache.size because of HFileV2
+    [HBASE-5021] - Enforce upper bound on timestamp
+    [HBASE-5033] - Opening/Closing store in parallel to reduce region open/close time
+    [HBASE-5064] - utilize surefire tests parallelization
+    [HBASE-5072] - Support Max Value for Per-Store Metrics
+    [HBASE-5074] - support checksums in HBase block cache
+    [HBASE-5134] - Remove getRegionServerWithoutRetries and getRegionServerWithRetries from HConnection Interface
+    [HBASE-5166] - MultiThreaded Table Mapper analogous to MultiThreaded Mapper in hadoop
+    [HBASE-5167] - We shouldn't be injecting 'Killing [daemon]' into logs, when we aren't doing that.
+    [HBASE-5186] - Add metrics to ThriftServer
+    [HBASE-5189] - Add metrics to keep track of region-splits in RS
+    [HBASE-5190] - Limit the IPC queue size based on calls' payload size
+    [HBASE-5193] - Use TBoundedThreadPoolServer in HRegionThriftServer
+    [HBASE-5197] - [replication] Handle socket timeouts in ReplicationSource to prevent DDOS
+    [HBASE-5199] - Delete out of TTL store files before compaction selection
+    [HBASE-5201] - Utilize TThreadedSelectorServer and remove redundant code in ThriftServer and HRegionThriftServer
+    [HBASE-5209] - HConnection/HMasterInterface should allow for way to get hostname of currently active master in multi-master HBase setup
+    [HBASE-5246] - Regenerate code with thrift 0.8.0
+    [HBASE-5255] - Use singletons for OperationStatus to save memory
+    [HBASE-5259] - Normalize the RegionLocation in TableInputFormat by the reverse DNS lookup.
+    [HBASE-5297] - Update metrics numOpenConnections and callQueueLen directly in HBaseServer
+    [HBASE-5298] - Add thrift metrics to thrift2
+    [HBASE-5304] - Pluggable split key policy
+    [HBASE-5310] - HConnectionManager server cache key enhancement
+    [HBASE-5325] - Expose basic information about the master-status through jmx beans
+    [HBASE-5332] - Deterministic Compaction Jitter
+    [HBASE-5358] - HBaseObjectWritable should be able to serialize/deserialize generic arrays
+    [HBASE-5363] - Automatically run rat check on mvn release builds
+    [HBASE-5388] - Tune HConnectionManager#getCachedLocation method
+    [HBASE-5393] - Consider splitting after flushing
+    [HBASE-5394] - Add ability to include Protobufs in HbaseObjectWritable
+    [HBASE-5395] - CopyTable needs to use GenericOptionsParser
+    [HBASE-5411] - Add more metrics for ThriftMetrics
+    [HBASE-5421] - use hadoop-client/hadoop-minicluster artifacts for Hadoop 0.23 build
+    [HBASE-5428] - Allow for custom filters to be registered within the Thrift interface
+    [HBASE-5433] - [REST] Add metrics to keep track of success/failure count
+    [HBASE-5434] - [REST] Include more metrics in cluster status request
+    [HBASE-5436] - Right-size the map when reading attributes.
+    [HBASE-5439] - Fix some performance findbugs issues
+    [HBASE-5440] - Allow Import to optionally use HFileOutputFormat
+    [HBASE-5442] - Use builder pattern in StoreFile and HFile
+    [HBASE-5454] - Refuse operations from Admin before master is initialized
+    [HBASE-5464] - Log warning message when thrift calls throw exceptions
+    [HBASE-5483] - Allow configurable host to bind to for starting REST server from commandline
+    [HBASE-5489] - Add HTable accessor to get regions for a key range
+    [HBASE-5508] - Add an option to allow test output to show on the terminal
+    [HBASE-5520] - Support reseek() at RegionScanner
+    [HBASE-5533] - Add more metrics to HBase
+    [HBASE-5551] - Some functions should not be used by customer code and must be deprecated in 0.94
+    [HBASE-5560] - Avoid RegionServer GC caused by timed-out calls
+    [HBASE-5588] - Deprecate/remove AssignmentManager#clearRegionFromTransition
+    [HBASE-5589] - Add of the offline call to the Master Interface
+    [HBASE-5592] - Make it easier to get a table from shell
+
+New Feature
+
+    [HBASE-2947] - MultiIncrement/MultiAppend (MultiGet functionality for increments and appends)
+    [HBASE-3584] - Allow atomic put/delete in one call
+    [HBASE-3856] - Build a tree structure data block index inside of the HFile
+    [HBASE-4102] - atomicAppend: A put that appends to the latest version of a cell; i.e. reads current value then adds the bytes offered by the client to the tail and writes out a new entry
+    [HBASE-4219] - Add Per-Column Family Metrics
+    [HBASE-4256] - Intra-row scanning (part deux)
+    [HBASE-4460] - Support running an embedded ThriftServer within a RegionServer
+    [HBASE-4536] - Allow CF to retain deleted rows
+    [HBASE-4608] - HLog Compression
+    [HBASE-4629] - enable automated patch testing for hbase
+    [HBASE-4683] - Always cache index and bloom blocks
+    [HBASE-4698] - Let the HFile Pretty Printer print all the key values for a specific row.
+    [HBASE-4768] - Per-(table, columnFamily) metrics with configurable table name inclusion
+    [HBASE-5128] - [uber hbck] Online automated repair of table integrity and region consistency problems
+    [HBASE-5177] - HTable needs a non cached version of getRegionLocation
+    [HBASE-5229] - Provide basic building blocks for "multi-row" local transactions.
+    [HBASE-5526] - Configurable file and directory based umask
+
+Task
+
+    [HBASE-4429] - Provide synchronous balanceSwitch()
+    [HBASE-4611] - Add support for Phabricator/Differential as an alternative code review tool
+    [HBASE-4712] - Document rules for writing tests
+    [HBASE-4751] - Make TestAdmin#testEnableTableRoundRobinAssignment friendly to concurrent tests
+    [HBASE-4968] - Add to troubleshooting workaround for direct buffer oome's.
+    [HBASE-5011] - Move test-util.sh from src/test/bin to dev-tools
+    [HBASE-5111] - Upgrade zookeeper to 3.4.2 release
+    [HBASE-5173] - Commit hbase-4480 findHangingTest.sh script under dev-support
+    [HBASE-5264] - Add 0.92.0 upgrade guide
+    [HBASE-5294] - Make sure javadoc is included in tarball bundle when we release
+    [HBASE-5400] - Some tests does not have annotations for (Small|Medium|Large)Tests
+    [HBASE-5427] - Upgrade our zk to 3.4.3
+    [HBASE-5511] - More doc on maven release process
+
+Test
+
+    [HBASE-4516] - HFile-level load tester with compaction and random-read workloads
+    [HBASE-4534] - A new unit test for lazy seek and StoreScanner in general
+    [HBASE-4545] - TestHLog doesn't clean up after itself
+    [HBASE-4772] - Utility to Create StoreFiles
+    [HBASE-4808] - Test to Ensure Expired Deletes Don't Override Puts
+    [HBASE-4864] - TestMasterObserver#testRegionTransitionOperations occasionally fails
+    [HBASE-4868] - TestOfflineMetaRebuildBase#testMetaRebuild occasionally fails
+    [HBASE-5150] - Failure in a thread may not fail a test, clean up log splitting test
+    [HBASE-5223] - TestMetaReaderEditor is missing call to CatalogTracker.stop()
+    [HBASE-5455] - Add test to avoid unintentional reordering of items in HbaseObjectWritable
+
 
 Release 0.92.1 - Unreleased
   BUG FIXES

From d5b5fabf142aea9a6106cbf08845550fb23e4646 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 26 Mar 2012 23:06:16 +0000
Subject: [PATCH 0076/1540] HBASE-5596 Few minor bugs from HBASE-5209 (David S.
 Wang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305662 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ClusterStatus.java   |  7 +++----
 .../java/org/apache/hadoop/hbase/ServerName.java |  7 ++++++-
 .../hadoop/hbase/master/ActiveMasterManager.java |  2 +-
 .../org/apache/hadoop/hbase/master/HMaster.java  | 16 +++++++++++++++-
 4 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
index 7618e30baf53..d962594f8e0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
+++ b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
@@ -67,7 +67,6 @@ public class ClusterStatus extends VersionedWritable {
    */
   private static final byte VERSION_MASTER_BACKUPMASTERS = 2;
   private static final byte VERSION = 2;
-  private static final String UNKNOWN_SERVERNAME = "unknown";
 
   private String hbaseVersion;
   private Map<ServerName, HServerLoad> liveServers;
@@ -173,11 +172,11 @@ public boolean equals(Object o) {
     return (getVersion() == ((ClusterStatus)o).getVersion()) &&
       getHBaseVersion().equals(((ClusterStatus)o).getHBaseVersion()) &&
       this.liveServers.equals(((ClusterStatus)o).liveServers) &&
-      this.deadServers.equals(((ClusterStatus)o).deadServers) &&
+      this.deadServers.containsAll(((ClusterStatus)o).deadServers) &&
       Arrays.equals(this.masterCoprocessors,
                     ((ClusterStatus)o).masterCoprocessors) &&
       this.master.equals(((ClusterStatus)o).master) &&
-      this.backupMasters.equals(((ClusterStatus)o).backupMasters);
+      this.backupMasters.containsAll(((ClusterStatus)o).backupMasters);
   }
 
   /**
@@ -340,7 +339,7 @@ public void readFields(DataInput in) throws IOException {
                                  Bytes.readByteArray(in)));
       }
     } else {
-      this.master = new ServerName(UNKNOWN_SERVERNAME, -1,
+      this.master = new ServerName(ServerName.UNKNOWN_SERVERNAME, -1,
                                    ServerName.NON_STARTCODE);
       this.backupMasters = new ArrayList<ServerName>(0);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/ServerName.java b/src/main/java/org/apache/hadoop/hbase/ServerName.java
index d6d869a6926b..2d6fe4e9e87b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ServerName.java
+++ b/src/main/java/org/apache/hadoop/hbase/ServerName.java
@@ -69,6 +69,11 @@ public class ServerName implements Comparable<ServerName> {
       SERVERNAME_SEPARATOR + Addressing.VALID_PORT_REGEX +
       SERVERNAME_SEPARATOR + Addressing.VALID_PORT_REGEX + "$");
 
+  /**
+   * What to use if server name is unknown.
+   */
+  public static final String UNKNOWN_SERVERNAME = "#unknown#";
+
   private final String servername;
   private final String hostname;
   private final int port;
@@ -287,4 +292,4 @@ public static ServerName parseServerName(final String str) {
     return SERVERNAME_PATTERN.matcher(str).matches()? new ServerName(str):
       new ServerName(str, NON_STARTCODE);
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 381c683d045f..96754d2e003f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -167,7 +167,7 @@ boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
       LOG.info("Adding ZNode for " + backupZNode +
                " in backup master directory");
       ZKUtil.createEphemeralNodeAndWatch(this.watcher, backupZNode,
-          HConstants.EMPTY_BYTE_ARRAY);
+          this.sn.getVersionedBytes());
 
       String msg;
       byte [] bytes =
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index cbb20918bd48..8b3c11111a82 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -25,6 +25,8 @@
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -1438,8 +1440,20 @@ public ClusterStatus getClusterStatus() {
     List<ServerName> backupMasters = new ArrayList<ServerName>(
                                           backupMasterStrings.size());
     for (String s: backupMasterStrings) {
-      backupMasters.add(new ServerName(s));
+      try {
+        byte[] bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(this.zooKeeper.backupMasterAddressesZNode, s));
+        if (bytes != null) {
+          backupMasters.add(ServerName.parseVersionedServerName(bytes));
+        }
+      } catch (KeeperException e) {
+        LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
+                 "backup servers"), e);
+      }
     }
+    Collections.sort(backupMasters, new Comparator<ServerName>() {
+      public int compare(ServerName s1, ServerName s2) {
+        return s1.getServerName().compareTo(s2.getServerName());
+      }});
 
     return new ClusterStatus(VersionInfo.getVersion(),
       this.fileSystemManager.getClusterId(),

From 0c4c75a088745f623a9fa158d7d11210f58a1e17 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 26 Mar 2012 23:41:50 +0000
Subject: [PATCH 0077/1540] Update CHANGES.txt and pom.xml for 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305677 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 2 ++
 pom.xml     | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 68c6591ccdcb..764205adb6a2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -169,6 +169,7 @@ Bug
     [HBASE-5579] - A Delete Version could mask other values
     [HBASE-5581] - Creating a table with invalid syntax does not give an error message when it fails
     [HBASE-5586] - [replication] NPE in ReplicationSource when creating a stream to an inexistent cluster
+    [HBASE-5596] - Few minor bugs from HBASE-5209
     [HBASE-5597] - Findbugs check in test-patch.sh always fails
     [HBASE-5603] - rolling-restart.sh script hangs when attempting to detect expiration of /hbase/master znode.
     [HBASE-5613] - ThriftServer getTableRegions does not return serverName and port
@@ -176,6 +177,7 @@ Bug
     [HBASE-5623] - Race condition when rolling the HLog and hlogFlush
     [HBASE-5624] - Aborting regionserver when splitting region, may cause daughter region not assigned by ServerShutdownHandler.
     [HBASE-5633] - NPE reading ZK config in HBase
+    [HBASE-5639] - The logic used in waiting for region servers during startup is broken
 
 Improvement
 
diff --git a/pom.xml b/pom.xml
index d4c7c1a67b12..54fd55c10e76 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.0-SNAPSHOT</version>
+  <version>0.94.0</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 7e4d8330884980f312a27ee775e53ab38863ae8a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 27 Mar 2012 03:33:21 +0000
Subject: [PATCH 0078/1540] HBASE-5641 decayingSampleTick1 prevents HBase from
 shutting down.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1305721 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                   |  1 +
 .../ExponentiallyDecayingSample.java          | 20 ++++++++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 764205adb6a2..e8708f80a79c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -33,6 +33,7 @@ Sub-task
     [HBASE-5497] - Add protobuf as M/R dependency jar (mapred)
     [HBASE-5523] - Fix Delete Timerange logic for KEEP_DELETED_CELLS
     [HBASE-5541] - Avoid holding the rowlock during HLog sync in HRegion.mutateRowWithLocks
+    [HBASE-5641] - decayingSampleTick1 prevents HBase from shutting down.
 
 Bug
 
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
index e615b2123ebc..fde93e1a5133 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
@@ -22,12 +22,12 @@
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import org.apache.hadoop.hbase.util.Threads;
-
 /**
  * An exponentially-decaying random sample of {@code long}s. 
  * Uses Cormode et al's forward-decaying priority reservoir sampling method 
@@ -44,7 +44,7 @@ public class ExponentiallyDecayingSample implements Sample {
 
   private static final ScheduledExecutorService TICK_SERVICE = 
       Executors.newScheduledThreadPool(1, 
-          Threads.getNamedThreadFactory("decayingSampleTick"));
+          getNamedDaemonThreadFactory("decayingSampleTick"));
 
   private static volatile long CURRENT_TICK = 
       TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
@@ -209,4 +209,18 @@ private void lockForRegularUsage() {
   private void unlockForRegularUsage() {
     lock.readLock().unlock();
   }
+
+  private static ThreadFactory getNamedDaemonThreadFactory(final String prefix) {
+    return new ThreadFactory() {
+
+      private final AtomicInteger threadNumber = new AtomicInteger(1);
+      
+      @Override
+      public Thread newThread(Runnable r) {
+        Thread t= new Thread(r, prefix + threadNumber.getAndIncrement());
+        t.setDaemon(true);
+        return t;
+      }
+    };
+  }
 }

From b404612609cd4eebc9931f04c0b8deece1e92178 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 27 Mar 2012 20:40:07 +0000
Subject: [PATCH 0079/1540] HBASE-5639 The logic used in waiting for region
 servers during startup is broken (J-D and NKeyval)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1306011 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                                 | 6 +++---
 .../java/org/apache/hadoop/hbase/master/ServerManager.java  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index e8708f80a79c..ae76f152f59f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.0 - 3/26/2012
+Release 0.94.0 - 3/27/2012
 Sub-task
 
     [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently
@@ -185,7 +185,7 @@ Improvement
     [HBASE-1744] - Thrift server to match the new java api.
     [HBASE-2418] - add support for ZooKeeper authentication
     [HBASE-3373] - Allow regions to be load-balanced by table
-    [HBASE-3433] - Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (why doesn't keyonlyfilter make copies rather than mutate -- HBASE-3211)?
+    [HBASE-3433] - Remove the KV copy of every KV in Scan; introduced by HBASE-3232
     [HBASE-3512] - Coprocessors: Shell support for listing currently loaded coprocessor set
     [HBASE-3565] - Add metrics to keep track of slow HLog appends
     [HBASE-3763] - Add Bloom Block Index Support
@@ -319,7 +319,6 @@ New Feature
     [HBASE-3856] - Build a tree structure data block index inside of the HFile
     [HBASE-4102] - atomicAppend: A put that appends to the latest version of a cell; i.e. reads current value then adds the bytes offered by the client to the tail and writes out a new entry
     [HBASE-4219] - Add Per-Column Family Metrics
-    [HBASE-4256] - Intra-row scanning (part deux)
     [HBASE-4460] - Support running an embedded ThriftServer within a RegionServer
     [HBASE-4536] - Allow CF to retain deleted rows
     [HBASE-4608] - HLog Compression
@@ -334,6 +333,7 @@ New Feature
 
 Task
 
+    [HBASE-4256] - Intra-row scanning (part deux)
     [HBASE-4429] - Provide synchronous balanceSwitch()
     [HBASE-4611] - Add support for Phabricator/Differential as an alternative code review tool
     [HBASE-4712] - Document rules for writing tests
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index e13a94a98e22..d5c8877d7040 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -585,7 +585,7 @@ public void waitForRegionServers(MonitoredTask status)
       !this.master.isStopped() &&
         slept < timeout &&
         count < maxToStart &&
-        !(lastCountChange+interval > now && count >= minToStart)
+        (lastCountChange+interval > now || count < minToStart)
       ){
 
       // Log some info at every interval time or if there is a change

From 6cab376570c899fbd5bf57b2ca03df0b9456fc11 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 29 Mar 2012 15:30:45 +0000
Subject: [PATCH 0080/1540] AggregationClient fails validation for open stoprow
 scan

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1306919 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/coprocessor/AggregationClient.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
index 865ffb1f4449..05a8d9b826e0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
@@ -119,7 +119,8 @@ private void validateParameters(Scan scan) throws IOException {
     if (scan == null
         || (Bytes.equals(scan.getStartRow(), scan.getStopRow()) && !Bytes
             .equals(scan.getStartRow(), HConstants.EMPTY_START_ROW))
-        || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) > 0) {
+        || ((Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) > 0) &&
+        	!Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW))) {
       throw new IOException(
           "Agg client Exception: Startrow should be smaller than Stoprow");
     } else if (scan.getFamilyMap().size() != 1) {

From 880964cfac98246dd271d962b620719f57890ba7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 29 Mar 2012 15:34:48 +0000
Subject: [PATCH 0081/1540] HBASE-5671 hbase.metrics.showTableName should be
 true by default

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1306924 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/resources/hbase-default.xml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index e15841bc3e4f..341431ace550 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -883,4 +883,15 @@
       files when hbase.data.umask.enable is true
     </description>
   </property>
+
+  <property>
+    <name>hbase.metrics.showTableName</name>
+    <value>true</value>
+    <description>Whether to include the prefix "tbl.tablename" in per-column family metrics.
+	If true, for each metric M, per-cf metrics will be reported for tbl.T.cf.CF.M, if false,
+	per-cf metrics will be aggregated by column-family across tables, and reported for cf.CF.M.
+	In both cases, the aggregated metric M across tables and cfs will be reported.
+    </description>
+  </property>
+
 </configuration>

From dd352996e270ab26396ebf4f0fa4d3eb1c6fb749 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 29 Mar 2012 16:21:15 +0000
Subject: [PATCH 0082/1540] HBASE-5670 Have Mutation implement the Row
 interface.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1306958 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/Append.java      | 3 +--
 src/main/java/org/apache/hadoop/hbase/client/Delete.java      | 2 +-
 src/main/java/org/apache/hadoop/hbase/client/HTable.java      | 4 ++--
 .../java/org/apache/hadoop/hbase/client/HTableInterface.java  | 4 ++--
 src/main/java/org/apache/hadoop/hbase/client/HTablePool.java  | 4 ++--
 src/main/java/org/apache/hadoop/hbase/client/Mutation.java    | 3 ++-
 src/main/java/org/apache/hadoop/hbase/client/Put.java         | 2 +-
 .../org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java  | 4 ++--
 .../org/apache/hadoop/hbase/rest/client/RemoteHTable.java     | 4 ++--
 9 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java
index a47480c7c525..ce435c840ee6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -41,8 +41,7 @@
  * row to append to. At least one column to append must be specified using the
  * {@link #add(byte[], byte[], byte[])} method.
  */
-public class Append extends Mutation implements Row {
-  // TODO: refactor to derive from Put?
+public class Append extends Mutation {
   private static final String RETURN_RESULTS = "_rr_";
   private static final byte APPEND_VERSION = (byte)1;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 7cf1b894a002..a806f8a29d07 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -65,7 +65,7 @@
  * timestamp.  The constructor timestamp is not referenced.
  */
 public class Delete extends Mutation
-  implements Writable, Row, Comparable<Row> {
+  implements Writable, Comparable<Row> {
   private static final byte DELETE_VERSION = (byte)3;
 
   /** Constructor for Writable.  DO NOT USE */
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 5e7106a5475e..f51d49bd0d1b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -690,7 +690,7 @@ public Result[] get(List<Get> gets) throws IOException {
    * {@inheritDoc}
    */
   @Override
-  public synchronized void batch(final List<Row> actions, final Object[] results)
+  public synchronized void batch(final List<?extends Row> actions, final Object[] results)
       throws InterruptedException, IOException {
     connection.processBatch(actions, tableName, pool, results);
   }
@@ -699,7 +699,7 @@ public synchronized void batch(final List<Row> actions, final Object[] results)
    * {@inheritDoc}
    */
   @Override
-  public synchronized Object[] batch(final List<Row> actions) throws InterruptedException, IOException {
+  public synchronized Object[] batch(final List<? extends Row> actions) throws InterruptedException, IOException {
     Object[] results = new Object[actions.size()];
     connection.processBatch(actions, tableName, pool, results);
     return results;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
index 9b3c68b439d9..e7d8cf889453 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
@@ -88,7 +88,7 @@ public interface HTableInterface extends Closeable {
    * @throws IOException
    * @since 0.90.0
    */
-  void batch(final List<Row> actions, final Object[] results) throws IOException, InterruptedException;
+  void batch(final List<?extends Row> actions, final Object[] results) throws IOException, InterruptedException;
 
   /**
    * Same as {@link #batch(List, Object[])}, but returns an array of
@@ -100,7 +100,7 @@ public interface HTableInterface extends Closeable {
    * @throws IOException
    * @since 0.90.0
    */
-  Object[] batch(final List<Row> actions) throws IOException, InterruptedException;
+  Object[] batch(final List<? extends Row> actions) throws IOException, InterruptedException;
 
   /**
    * Extracts certain cells from a given row.
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index a5c198fc2c60..5cc82997916b 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -350,13 +350,13 @@ public boolean exists(Get get) throws IOException {
     }
 
     @Override
-    public void batch(List<Row> actions, Object[] results) throws IOException,
+    public void batch(List<? extends Row> actions, Object[] results) throws IOException,
         InterruptedException {
       table.batch(actions, results);
     }
 
     @Override
-    public Object[] batch(List<Row> actions) throws IOException,
+    public Object[] batch(List<? extends Row> actions) throws IOException,
         InterruptedException {
       return table.batch(actions);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index c246470ed38f..d019ab3b9aaf 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 
-public abstract class Mutation extends OperationWithAttributes {
+public abstract class Mutation extends OperationWithAttributes implements Row {
   // Attribute used in Mutations to indicate the originating cluster.
   private static final String CLUSTER_ID_ATTR = "_c.id_";
 
@@ -148,6 +148,7 @@ public boolean isEmpty() {
    * Method for retrieving the delete's row
    * @return row
    */
+  @Override
   public byte [] getRow() {
     return this.row;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index c09b339d5985..920295827831 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -44,7 +44,7 @@
  * {@link #add(byte[], byte[], long, byte[]) add} if setting the timestamp.
  */
 public class Put extends Mutation
-  implements HeapSize, Writable, Row, Comparable<Row> {
+  implements HeapSize, Writable, Comparable<Row> {
   private static final byte PUT_VERSION = (byte)2;
 
   private static final long OVERHEAD = ClassSize.align(
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index b6412ca67b7f..c08fe2b99507 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -466,13 +466,13 @@ public void unlockRow(RowLock rl) throws IOException {
       }
 
       @Override
-      public void batch(List<Row> actions, Object[] results)
+      public void batch(List<? extends Row> actions, Object[] results)
           throws IOException, InterruptedException {
         table.batch(actions, results);
       }
 
       @Override
-      public Object[] batch(List<Row> actions)
+      public Object[] batch(List<? extends Row> actions)
           throws IOException, InterruptedException {
         return table.batch(actions);
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index 8acc5266fdd7..09d51acb9c10 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -612,12 +612,12 @@ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
   }
 
   @Override
-  public void batch(List<Row> actions, Object[] results) throws IOException {
+  public void batch(List<? extends Row> actions, Object[] results) throws IOException {
     throw new IOException("batch not supported");
   }
 
   @Override
-  public Object[] batch(List<Row> actions) throws IOException {
+  public Object[] batch(List<? extends Row> actions) throws IOException {
     throw new IOException("batch not supported");
   }
 

From b5e62bfbb06b0e4c39debe4bdb3f934df6b12214 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 29 Mar 2012 18:32:52 +0000
Subject: [PATCH 0083/1540] HBASE-5097 RegionObserver implementation whose
 preScannerOpen and postScannerOpen Impl return null can stall the system
 initialization through NPE (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307035 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegionServer.java  | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ba7fbe1b3065..364f933f796b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2324,7 +2324,14 @@ public long openScanner(byte[] regionName, Scan scan) throws IOException {
         s = r.getScanner(scan);
       }
       if (r.getCoprocessorHost() != null) {
-        s = r.getCoprocessorHost().postScannerOpen(scan, s);
+        RegionScanner savedScanner = r.getCoprocessorHost().postScannerOpen(
+            scan, s);
+        if (savedScanner == null) {
+          LOG.warn("PostScannerOpen impl returning null. "
+              + "Check the RegionObserver implementation.");
+        } else {
+          s = savedScanner;
+        }
       }
       return addScanner(s);
     } catch (Throwable t) {

From 28b6a9a40dd54fbd39066f3e380d1d0841d8b66e Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 29 Mar 2012 20:05:22 +0000
Subject: [PATCH 0084/1540] HBASE-5638 Readability improvements on HBASE-5633:
 NPE reading ZK config in HBase (Matteo Bertozzi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307086 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HConstants.java       | 6 +++---
 .../java/org/apache/hadoop/hbase/LocalHBaseCluster.java     | 4 ++--
 .../java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java    | 5 ++---
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 92b02f74e2e0..b6af98a8ef53 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -73,13 +73,13 @@ public enum OperationStatusCode {
   public static final String HBASE_MASTER_LOADBALANCER_CLASS = "hbase.master.loadbalancer.class";
 
   /** Cluster is standalone or pseudo-distributed */
-  public static final String CLUSTER_IS_LOCAL = "false";
+  public static final boolean CLUSTER_IS_LOCAL = false;
 
   /** Cluster is fully-distributed */
-  public static final String CLUSTER_IS_DISTRIBUTED = "true";
+  public static final boolean CLUSTER_IS_DISTRIBUTED = true;
 
   /** Default value for cluster distributed mode */  
-  public static final String DEFAULT_CLUSTER_DISTRIBUTED = CLUSTER_IS_LOCAL;
+  public static final boolean DEFAULT_CLUSTER_DISTRIBUTED = CLUSTER_IS_LOCAL;
 
   /** default host address */
   public static final String DEFAULT_HOST = "0.0.0.0";
diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index 5fcc39c87a00..f49f14c457d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -432,8 +432,8 @@ public void shutdown() {
    * @return True if a 'local' address in hbase.master value.
    */
   public static boolean isLocal(final Configuration c) {
-    final String mode = c.get(HConstants.CLUSTER_DISTRIBUTED);
-    return mode == null || mode.equals(HConstants.CLUSTER_IS_LOCAL);
+    boolean mode = c.getBoolean(HConstants.CLUSTER_DISTRIBUTED, HConstants.DEFAULT_CLUSTER_DISTRIBUTED);
+    return(mode == HConstants.CLUSTER_IS_LOCAL);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
index bdbd9bd1ea9a..4cc9c016d12f 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
@@ -159,9 +159,8 @@ public static Properties parseZooCfg(Configuration conf,
       }
       // Special case for 'hbase.cluster.distributed' property being 'true'
       if (key.startsWith("server.")) {
-        if (conf.get(HConstants.CLUSTER_DISTRIBUTED, HConstants.DEFAULT_CLUSTER_DISTRIBUTED).
-              equals(HConstants.CLUSTER_IS_DISTRIBUTED)
-            && value.startsWith(HConstants.LOCALHOST)) {
+        boolean mode = conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, HConstants.DEFAULT_CLUSTER_DISTRIBUTED);
+        if (mode == HConstants.CLUSTER_IS_DISTRIBUTED && value.startsWith(HConstants.LOCALHOST)) {
           String msg = "The server in zoo.cfg cannot be set to localhost " +
               "in a fully-distributed setup because it won't be reachable. " +
               "See \"Getting Started\" for more information.";

From 8af484601b2083135cb60a94c1e36c41781f6c2c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 29 Mar 2012 21:23:48 +0000
Subject: [PATCH 0085/1540] HBASE-4398 If HRegionPartitioner is used in
 MapReduce, client side configurations are overwritten by hbase-site.xml

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307117 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
index e42d5000e195..363ba51f9b58 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
@@ -116,9 +116,8 @@ public Configuration getConf() {
    */
   @Override
   public void setConf(Configuration configuration) {
-    this.conf = configuration;
+    this.conf = HBaseConfiguration.create(configuration);
     try {
-      HBaseConfiguration.addHbaseResources(conf);
       this.table = new HTable(this.conf,
         configuration.get(TableOutputFormat.OUTPUT_TABLE));
     } catch (IOException e) {

From ff441330272b6418211defbbad0c63285aae1995 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 30 Mar 2012 04:33:43 +0000
Subject: [PATCH 0086/1540] HBASE-5673 The OOM problem of IPC client call cause
 all handle block

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307242 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 0aed5bf6e26f..2b9b6aff45bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -371,7 +371,7 @@ protected synchronized void setupIOstreams()
 
         // start the receiver thread after the socket connection has been set up
         start();
-      } catch (IOException e) {
+      } catch (Throwable e) {
         markClosed(e);
         close();
 

From 2829fab2c6ab0147a56c0c75545fbbd3145bf1ce Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 30 Mar 2012 05:31:16 +0000
Subject: [PATCH 0087/1540] HBASE-5673 The OOM problem of IPC client call cause
 all handle block; REVERT -- APPLIED WRONG PATCH

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307271 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 2b9b6aff45bc..0aed5bf6e26f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -371,7 +371,7 @@ protected synchronized void setupIOstreams()
 
         // start the receiver thread after the socket connection has been set up
         start();
-      } catch (Throwable e) {
+      } catch (IOException e) {
         markClosed(e);
         close();
 

From 322d7e55c180be78648ce4090bfb044d3bd67559 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 30 Mar 2012 05:33:10 +0000
Subject: [PATCH 0088/1540] HBASE-5673 The OOM problem of IPC client call cause
 all handle block; REAPPLY; V2 OF PATCH

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307276 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 0aed5bf6e26f..e4a808509cc1 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -371,11 +371,12 @@ protected synchronized void setupIOstreams()
 
         // start the receiver thread after the socket connection has been set up
         start();
-      } catch (IOException e) {
-        markClosed(e);
+      } catch (Throwable e) {
+        IOException exception = new IOException(e);
+        markClosed(exception);
         close();
 
-        throw e;
+        throw exception;
       }
     }
 

From 9f91fe81e9ed52ee1bce775d714db5e49f3ba512 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 30 Mar 2012 16:33:50 +0000
Subject: [PATCH 0089/1540] HBASE-5673 The OOM problem of IPC client call cause
 all handle block; REVERT OF V2, A SECOND REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307515 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index e4a808509cc1..0aed5bf6e26f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -371,12 +371,11 @@ protected synchronized void setupIOstreams()
 
         // start the receiver thread after the socket connection has been set up
         start();
-      } catch (Throwable e) {
-        IOException exception = new IOException(e);
-        markClosed(exception);
+      } catch (IOException e) {
+        markClosed(e);
         close();
 
-        throw exception;
+        throw e;
       }
     }
 

From 428aef86cfd280c9cfb5b4a444a170b5259891ef Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 31 Mar 2012 00:23:26 +0000
Subject: [PATCH 0090/1540] HBASE-5084 Clow different HTable instances to share
 one ExecutorService

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307659 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTable.java    | 35 ++++++++++++++++--
 .../apache/hadoop/hbase/client/TestHCM.java   | 37 +++++++++++++++----
 2 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index f51d49bd0d1b..6f18f6a6fe9b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -115,7 +115,8 @@ public class HTable implements HTableInterface {
   private boolean closed;
   private int operationTimeout;
   private static final int DOPUT_WB_CHECK = 10;    // i.e., doPut checks the writebuffer every X Puts.
-  private final boolean cleanupOnClose; // close the connection in close()
+  private final boolean cleanupPoolOnClose; // shutdown the pool in close()
+  private final boolean cleanupConnectionOnClose; // close the connection in close()
 
   /**
    * Creates an object to access a HBase table.
@@ -146,7 +147,7 @@ public HTable(Configuration conf, final String tableName)
   public HTable(Configuration conf, final byte [] tableName)
   throws IOException {
     this.tableName = tableName;
-    this.cleanupOnClose = true;
+    this.cleanupPoolOnClose = this.cleanupConnectionOnClose = true;
     if (conf == null) {
       this.connection = null;
       return;
@@ -173,6 +174,30 @@ public HTable(Configuration conf, final byte [] tableName)
     this.finishSetup();
   }
 
+  /**
+   * Creates an object to access a HBase table.
+   * Shares zookeeper connection and other resources with other HTable instances
+   * created with the same <code>conf</code> instance.  Uses already-populated
+   * region cache if one is available, populated by any other HTable instances
+   * sharing this <code>conf</code> instance.
+   * Use this constructor when the ExecutorService is externally managed.
+   * @param conf Configuration object to use.
+   * @param tableName Name of the table.
+   * @param pool ExecutorService to be used.
+   * @throws IOException if a remote or network exception occurs
+   */
+  public HTable(Configuration conf, final byte[] tableName, final ExecutorService pool)
+      throws IOException {
+    this.connection = HConnectionManager.getConnection(conf);
+    this.configuration = conf;
+    this.pool = pool;
+    this.tableName = tableName;
+    this.cleanupPoolOnClose = false;
+    this.cleanupConnectionOnClose = true;
+
+    this.finishSetup();
+  }
+
   /**
    * Creates an object to access a HBase table.
    * Shares zookeeper connection and other resources with other HTable instances
@@ -193,7 +218,7 @@ public HTable(final byte[] tableName, final HConnection connection,
       throw new IllegalArgumentException("Connection is null or closed.");
     }
     this.tableName = tableName;
-    this.cleanupOnClose = false;
+    this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false;
     this.connection = connection;
     this.configuration = connection.getConfiguration();
     this.pool = pool;
@@ -955,8 +980,10 @@ public void close() throws IOException {
       return;
     }
     flushCommits();
-    if (cleanupOnClose) {
+    if (cleanupPoolOnClose) {
       this.pool.shutdown();
+    }
+    if (cleanupConnectionOnClose) {
       if (this.connection != null) {
         this.connection.close();
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 9ac4751bd398..603a4fb8c245 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -23,22 +23,19 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertEquals;
 
-import java.io.IOException;
-import java.lang.reflect.Field;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
-import java.util.Set;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.httpclient.HostConfiguration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.HTable.DaemonThreadFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -54,6 +51,7 @@ public class TestHCM {
   private static final Log LOG = LogFactory.getLog(TestHCM.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final byte[] TABLE_NAME = Bytes.toBytes("test");
+  private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
   private static final byte[] FAM_NAM = Bytes.toBytes("f");
   private static final byte[] ROW = Bytes.toBytes("bbb");
 
@@ -136,10 +134,33 @@ public void testRegionCaching() throws Exception{
     conn.deleteCachedLocation(TABLE_NAME, ROW);
     HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
     assertNull("What is this location?? " + rl, rl);
-    conn.close();
     table.close();
   }
 
+  /**
+   * Test that Connection or Pool are not closed when managed externally
+   * @throws Exception
+   */
+  @Test
+  public void testConnectionManagement() throws Exception{
+    TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
+    HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
+    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
+        60, TimeUnit.SECONDS,
+        new SynchronousQueue<Runnable>(),
+        new DaemonThreadFactory());
+
+    HTable table = new HTable(TABLE_NAME1, conn, pool);
+    table.close();
+    assertFalse(conn.isClosed());
+    assertFalse(pool.isShutdown());
+    table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME1, pool);
+    table.close();
+    assertFalse(pool.isShutdown());
+    conn.close();
+    pool.shutdownNow();
+  }
+
   /**
    * Make sure that {@link HConfiguration} instances that are essentially the
    * same map to the same {@link HConnection} instance.

From f89f6b9bffeb00eada13d27dc4b653877f7f6bcb Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 31 Mar 2012 17:09:44 +0000
Subject: [PATCH 0091/1540] HBASE-5690 compression does not work in Store.java
 of 0.94 (Honghua Zhu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1307851 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 23ea3cd2a2a6..bd2a4c0eb9c2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -833,6 +833,7 @@ private StoreFile.Writer createWriterInTmp(int maxKeyCount,
             .withMaxKeyCount(maxKeyCount)
             .withChecksumType(checksumType)
             .withBytesPerChecksum(bytesPerChecksum)
+            .withCompression(compression)
             .build();
     // The store file writer's path does not include the CF name, so we need
     // to configure the HFile writer directly.

From c8f7e5bee0675ab384d22690555ceca52a862fc0 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 2 Apr 2012 13:52:42 +0000
Subject: [PATCH 0092/1540] HBASE-5663 HBASE-5636 MultithreadedTableMapper
 doesn't work (Takuya Ueshin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1308354 13f79535-47bb-0310-9956-ffa450edef68
---
 .../mapreduce/MultithreadedTableMapper.java   | 47 +++++++++++++++----
 .../hbase/mapred/TestTableMapReduce.java      | 28 ++++++-----
 ...java => TestMultithreadedTableMapper.java} | 23 ++++-----
 .../hbase/mapreduce/TestTableMapReduce.java   | 19 ++++----
 4 files changed, 77 insertions(+), 40 deletions(-)
 rename src/test/java/org/apache/hadoop/hbase/mapreduce/{TestMulitthreadedTableMapper.java => TestMultithreadedTableMapper.java} (93%)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
index eb96ac684458..8d1aaf090753 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -31,11 +32,14 @@
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MapContext;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.StatusReporter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.util.ReflectionUtils;
 
 
@@ -239,15 +243,17 @@ private class MapRunner implements Runnable {
           context.getConfiguration());
       try {
         Constructor c = context.getClass().getConstructor(
+          Mapper.class,
           Configuration.class,
-          outer.getTaskAttemptID().getClass(),
-          SubMapRecordReader.class,
-          SubMapRecordWriter.class,
-          context.getOutputCommitter().getClass(),
-          SubMapStatusReporter.class,
-          outer.getInputSplit().getClass());
+          TaskAttemptID.class,
+          RecordReader.class,
+          RecordWriter.class,
+          OutputCommitter.class,
+          StatusReporter.class,
+          InputSplit.class);
         c.setAccessible(true);
         subcontext = (Context) c.newInstance(
+          mapper,
           outer.getConfiguration(), 
           outer.getTaskAttemptID(),
           new SubMapRecordReader(),
@@ -256,8 +262,31 @@ private class MapRunner implements Runnable {
           new SubMapStatusReporter(),
           outer.getInputSplit());
       } catch (Exception e) {
-        // rethrow as IOE
-        throw new IOException(e);
+        try {
+          Constructor c = Class.forName("org.apache.hadoop.mapreduce.task.MapContextImpl").getConstructor(
+            Configuration.class,
+            TaskAttemptID.class,
+            RecordReader.class,
+            RecordWriter.class,
+            OutputCommitter.class,
+            StatusReporter.class,
+            InputSplit.class);
+          c.setAccessible(true);
+          MapContext mc = (MapContext) c.newInstance(
+            outer.getConfiguration(), 
+            outer.getTaskAttemptID(),
+            new SubMapRecordReader(),
+            new SubMapRecordWriter(),
+            context.getOutputCommitter(),
+            new SubMapStatusReporter(),
+            outer.getInputSplit());
+          Class<?> wrappedMapperClass = Class.forName("org.apache.hadoop.mapreduce.lib.map.WrappedMapper");
+          Method getMapContext = wrappedMapperClass.getMethod("getMapContext", MapContext.class);
+          subcontext = (Context) getMapContext.invoke(wrappedMapperClass.newInstance(), mc);
+        } catch (Exception ee) {
+          // rethrow as IOE
+          throw new IOException(e);
+        }
       }
     }
 
@@ -270,4 +299,4 @@ public void run() {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
index 26a6f610f096..7fe273bf9b46 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
@@ -21,6 +21,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.NavigableMap;
 
@@ -28,7 +29,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -42,11 +42,15 @@
 import org.apache.hadoop.mapred.MapReduceBase;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.RunningJob;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test Map/Reduce job over HBase tables. The map/reduce process we're testing
  * on our tables is simple - take every row in the table, reverse the value of
@@ -58,7 +62,7 @@ public class TestTableMapReduce {
     LogFactory.getLog(TestTableMapReduce.class.getName());
   private static final HBaseTestingUtility UTIL =
     new HBaseTestingUtility();
-  static final String MULTI_REGION_TABLE_NAME = "mrtest";
+  static final byte[] MULTI_REGION_TABLE_NAME = Bytes.toBytes("mrtest");
   static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
   static final byte[] OUTPUT_FAMILY = Bytes.toBytes("text");
 
@@ -69,12 +73,10 @@ public class TestTableMapReduce {
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    HTableDescriptor desc = new HTableDescriptor(MULTI_REGION_TABLE_NAME);
-    desc.addFamily(new HColumnDescriptor(INPUT_FAMILY));
-    desc.addFamily(new HColumnDescriptor(OUTPUT_FAMILY));
     UTIL.startMiniCluster();
-    HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
-    admin.createTable(desc, HBaseTestingUtility.KEYS);
+    HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
+    UTIL.createMultiRegions(table, INPUT_FAMILY);
+    UTIL.loadTable(table, INPUT_FAMILY);
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -150,7 +152,8 @@ private void runTestOnTable(HTable table) throws IOException {
         IdentityTableReduce.class, jobConf);
 
       LOG.info("Started " + Bytes.toString(table.getTableName()));
-      JobClient.runJob(jobConf);
+      RunningJob job = JobClient.runJob(jobConf);
+      assertTrue(job.isSuccessful());
       LOG.info("After map/reduce completion");
 
       // verify map-reduce results
@@ -184,7 +187,7 @@ private void verify(String tableName) throws IOException {
         // continue
       }
     }
-    org.junit.Assert.assertTrue(verified);
+    assertTrue(verified);
   }
 
   /**
@@ -199,7 +202,10 @@ private void verifyAttempt(final HTable table) throws IOException, NullPointerEx
     TableInputFormat.addColumns(scan, columns);
     ResultScanner scanner = table.getScanner(scan);
     try {
-      for (Result r : scanner) {
+      Iterator<Result> itr = scanner.iterator();
+      assertTrue(itr.hasNext());
+      while(itr.hasNext()) {
+        Result r = itr.next();
         if (LOG.isDebugEnabled()) {
           if (r.size() > 2 ) {
             throw new IOException("Too many results, expected 2 got " +
@@ -247,7 +253,7 @@ private void verifyAttempt(final HTable table) throws IOException, NullPointerEx
                 r.getRow() + ", first value=" + first + ", second value=" +
                 second);
           }
-          org.junit.Assert.fail();
+          fail();
         }
       }
     } finally {
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMulitthreadedTableMapper.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
similarity index 93%
rename from src/test/java/org/apache/hadoop/hbase/mapreduce/TestMulitthreadedTableMapper.java
rename to src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
index cc5b1df39c60..f77b7ebe041c 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMulitthreadedTableMapper.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
@@ -19,6 +19,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.NavigableMap;
 
@@ -28,7 +29,6 @@
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -52,23 +52,21 @@
  * a particular cell, and write it back to the table.
  */
 @Category(LargeTests.class)
-public class TestMulitthreadedTableMapper {
-  private static final Log LOG = LogFactory.getLog(TestMulitthreadedTableMapper.class);
+public class TestMultithreadedTableMapper {
+  private static final Log LOG = LogFactory.getLog(TestMultithreadedTableMapper.class);
   private static final HBaseTestingUtility UTIL =
       new HBaseTestingUtility();
-  static final String MULTI_REGION_TABLE_NAME = "mrtest";
+  static final byte[] MULTI_REGION_TABLE_NAME = Bytes.toBytes("mrtest");
   static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
   static final byte[] OUTPUT_FAMILY = Bytes.toBytes("text");
   static final int    NUMBER_OF_THREADS = 10;
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    HTableDescriptor desc = new HTableDescriptor(MULTI_REGION_TABLE_NAME);
-    desc.addFamily(new HColumnDescriptor(INPUT_FAMILY));
-    desc.addFamily(new HColumnDescriptor(OUTPUT_FAMILY));
     UTIL.startMiniCluster();
-    HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
-    admin.createTable(desc, HBaseTestingUtility.KEYS);
+    HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
+    UTIL.createMultiRegions(table, INPUT_FAMILY);
+    UTIL.loadTable(table, INPUT_FAMILY);
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -149,7 +147,7 @@ private void runTestOnTable(HTable table)
           IdentityTableReducer.class, job);
       FileOutputFormat.setOutputPath(job, new Path("test"));
       LOG.info("Started " + Bytes.toString(table.getTableName()));
-      job.waitForCompletion(true);
+      assertTrue(job.waitForCompletion(true));
       LOG.info("After map/reduce completion");
       // verify map-reduce results
       verify(Bytes.toString(table.getTableName()));
@@ -203,7 +201,10 @@ private void verifyAttempt(final HTable table)
     scan.addFamily(OUTPUT_FAMILY);
     ResultScanner scanner = table.getScanner(scan);
     try {
-      for (Result r : scanner) {
+      Iterator<Result> itr = scanner.iterator();
+      assertTrue(itr.hasNext());
+      while(itr.hasNext()) {
+        Result r = itr.next();
         if (LOG.isDebugEnabled()) {
           if (r.size() > 2 ) {
             throw new IOException("Too many results, expected 2 got " +
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index 9268d6d46880..b351444cbeb0 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -21,6 +21,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.NavigableMap;
 
@@ -30,7 +31,6 @@
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -59,18 +59,16 @@ public class TestTableMapReduce {
   private static final Log LOG = LogFactory.getLog(TestTableMapReduce.class);
   private static final HBaseTestingUtility UTIL =
     new HBaseTestingUtility();
-  static final String MULTI_REGION_TABLE_NAME = "mrtest";
+  static final byte[] MULTI_REGION_TABLE_NAME = Bytes.toBytes("mrtest");
   static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
   static final byte[] OUTPUT_FAMILY = Bytes.toBytes("text");
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    HTableDescriptor desc = new HTableDescriptor(MULTI_REGION_TABLE_NAME);
-    desc.addFamily(new HColumnDescriptor(INPUT_FAMILY));
-    desc.addFamily(new HColumnDescriptor(OUTPUT_FAMILY));
     UTIL.startMiniCluster();
-    HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
-    admin.createTable(desc, HBaseTestingUtility.KEYS);
+    HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
+    UTIL.createMultiRegions(table, INPUT_FAMILY);
+    UTIL.loadTable(table, INPUT_FAMILY);
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -150,7 +148,7 @@ private void runTestOnTable(HTable table)
         IdentityTableReducer.class, job);
       FileOutputFormat.setOutputPath(job, new Path("test"));
       LOG.info("Started " + Bytes.toString(table.getTableName()));
-      job.waitForCompletion(true);
+      assertTrue(job.waitForCompletion(true));
       LOG.info("After map/reduce completion");
 
       // verify map-reduce results
@@ -204,7 +202,10 @@ private void verifyAttempt(final HTable table) throws IOException, NullPointerEx
     scan.addFamily(OUTPUT_FAMILY);
     ResultScanner scanner = table.getScanner(scan);
     try {
-      for (Result r : scanner) {
+      Iterator<Result> itr = scanner.iterator();
+      assertTrue(itr.hasNext());
+      while(itr.hasNext()) {
+        Result r = itr.next();
         if (LOG.isDebugEnabled()) {
           if (r.size() > 2 ) {
             throw new IOException("Too many results, expected 2 got " +

From 9668c5e15dcf54e9af9a162ec2cc4c8029fc1062 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 2 Apr 2012 18:00:31 +0000
Subject: [PATCH 0093/1540] HBASE-5694 getRowsWithColumnsTs() in Thrift service
 handles timestamps incorrectly

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1308446 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index d0642907f4d3..859d37bbb025 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -826,8 +826,8 @@ public List<TRowResult> getRowsWithColumnsTs(ByteBuffer tableName,
                 get.addColumn(famAndQf[0], famAndQf[1]);
               }
             }
-            get.setTimeRange(Long.MIN_VALUE, timestamp);
           }
+          get.setTimeRange(Long.MIN_VALUE, timestamp);
           gets.add(get);
         }
         Result[] result = table.get(gets);

From e7d1d73bdee66faee239793a0cc298624c686b48 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 2 Apr 2012 20:50:29 +0000
Subject: [PATCH 0094/1540] HBASE-5665 Repeated split causes HRegionServer
 failures and breaks table

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1308547 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 10 ++++++++
 .../hbase/regionserver/HRegionServer.java     |  5 ++--
 .../hbase/regionserver/SplitTransaction.java  |  2 +-
 .../regionserver/TestSplitTransaction.java    | 24 +++++++++++++++++++
 4 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index dc7f246bd5b5..e4d02584ee5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -823,6 +823,16 @@ public boolean isClosing() {
     return this.closing.get();
   }
 
+  /** @return true if region is available (not closed and not closing) */
+  public boolean isAvailable() {
+    return !isClosed() && !isClosing();
+  }
+
+  /** @return true if region is splittable */
+  public boolean isSplittable() {
+    return isAvailable() && !hasReferences();
+  }
+
   boolean areWritesEnabled() {
     synchronized(this.writestate) {
       return this.writestate.writesEnabled;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 364f933f796b..37bab7540581 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1912,8 +1912,7 @@ void closeUserRegions(final boolean abort) {
     try {
       for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
         HRegion r = e.getValue();
-        if (!r.getRegionInfo().isMetaRegion()) {
-          if (r.isClosed() || r.isClosing()) continue;
+        if (!r.getRegionInfo().isMetaRegion() && r.isAvailable()) {
           // Don't update zk with this close transition; pass false.
           closeRegion(r.getRegionInfo(), abort, false);
         }
@@ -3133,7 +3132,7 @@ protected HRegion getRegion(final byte[] regionName)
   protected HRegionInfo[] getMostLoadedRegions() {
     ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>();
     for (HRegion r : onlineRegions.values()) {
-      if (r.isClosed() || r.isClosing()) {
+      if (!r.isAvailable()) {
         continue;
       }
       if (regions.size() < numRegionsToReport) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index e6b125b3520c..aa8bf7210824 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -161,7 +161,7 @@ public SplitTransaction(final HRegion r, final byte [] splitrow) {
    * <code>false</code> if it is not (e.g. its already closed, etc.).
    */
   public boolean prepare() {
-    if (this.parent.isClosed() || this.parent.isClosing()) return false;
+    if (!this.parent.isSplittable()) return false;
     // Split key can be null if this region is unsplittable; i.e. has refs.
     if (this.splitrow == null) return false;
     HRegionInfo hri = this.parent.getRegionInfo();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index 8dc3d0c05e08..1de88277c2e5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -19,6 +19,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import com.google.common.collect.ImmutableList;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -137,6 +139,28 @@ private SplitTransaction prepareGOOD_SPLIT_ROW() {
     return st;
   }
 
+  /**
+   * Pass a reference store
+   */
+  @Test public void testPrepareWithRegionsWithReference() throws IOException {
+    // create a mock that will act as a reference StoreFile
+    StoreFile storeFileMock  = Mockito.mock(StoreFile.class);
+    when(storeFileMock.isReference()).thenReturn(true);
+
+    // add the mock to the parent stores
+    Store storeMock = Mockito.mock(Store.class);
+    List<StoreFile> storeFileList = new ArrayList<StoreFile>(1);
+    storeFileList.add(storeFileMock);
+    when(storeMock.getStorefiles()).thenReturn(storeFileList);
+    when(storeMock.close()).thenReturn(ImmutableList.copyOf(storeFileList));
+    this.parent.stores.put(Bytes.toBytes(""), storeMock);
+
+    SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW);
+
+    assertFalse("a region should not be splittable if it has instances of store file references",
+                st.prepare());
+  }
+
   /**
    * Pass an unreasonable split row.
    */

From 3529cd3ec180643bf4a2ce0ee357d47ea4717ba6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 2 Apr 2012 22:12:36 +0000
Subject: [PATCH 0095/1540] HBASE-5682 Allow HConnectionImplementation to
 recover from ZK connection loss

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1308596 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 89 ++++++++++---------
 .../apache/hadoop/hbase/TestZooKeeper.java    | 68 +++++++++-----
 2 files changed, 91 insertions(+), 66 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 34b6ba8c9da8..910108dc746f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -32,9 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.NoSuchElementException;
 import java.util.Set;
-import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -498,11 +496,11 @@ static class HConnectionImplementation implements HConnection, Closeable {
     private volatile HMasterInterface master;
     private volatile boolean masterChecked;
     // ZooKeeper reference
-    private ZooKeeperWatcher zooKeeper;
+    private volatile ZooKeeperWatcher zooKeeper;
     // ZooKeeper-based master address tracker
-    private MasterAddressTracker masterAddressTracker;
-    private RootRegionTracker rootRegionTracker;
-    private ClusterId clusterId;
+    private volatile MasterAddressTracker masterAddressTracker;
+    private volatile RootRegionTracker rootRegionTracker;
+    private volatile ClusterId clusterId;
 
     private final Object metaRegionLock = new Object();
 
@@ -574,35 +572,43 @@ public HConnectionImplementation(Configuration conf, boolean managed)
           HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
 
-      setupZookeeperTrackers();
-
       this.master = null;
       this.masterChecked = false;
     }
 
-    private synchronized void setupZookeeperTrackers()
-        throws ZooKeeperConnectionException{
+    private synchronized void ensureZookeeperTrackers()
+        throws ZooKeeperConnectionException {
       // initialize zookeeper and master address manager
-      this.zooKeeper = getZooKeeperWatcher();
-      masterAddressTracker = new MasterAddressTracker(this.zooKeeper, this);
-      masterAddressTracker.start();
-
-      this.rootRegionTracker = new RootRegionTracker(this.zooKeeper, this);
-      this.rootRegionTracker.start();
-
-      this.clusterId = new ClusterId(this.zooKeeper, this);
+      if (zooKeeper == null) {
+        zooKeeper = getZooKeeperWatcher();
+      }
+      if (clusterId == null) {
+        clusterId = new ClusterId(zooKeeper, this);
+      }
+      if (masterAddressTracker == null) {
+        masterAddressTracker = new MasterAddressTracker(zooKeeper, this);
+        masterAddressTracker.start();
+      }
+      if (rootRegionTracker == null) {
+        rootRegionTracker = new RootRegionTracker(zooKeeper, this);
+        rootRegionTracker.start();
+      }
     }
 
-    private synchronized void resetZooKeeperTrackers()
-        throws ZooKeeperConnectionException {
-      LOG.info("Trying to reconnect to zookeeper");
-      masterAddressTracker.stop();
-      masterAddressTracker = null;
-      rootRegionTracker.stop();
-      rootRegionTracker = null;
+    private synchronized void resetZooKeeperTrackers() {
+      if (masterAddressTracker != null) {
+        masterAddressTracker.stop();
+        masterAddressTracker = null;
+      }
+      if (rootRegionTracker != null) {
+        rootRegionTracker.stop();
+        rootRegionTracker = null;
+      }
       clusterId = null;
-      this.zooKeeper = null;
-      setupZookeeperTrackers();
+      if (zooKeeper != null) {
+        zooKeeper.close();
+        zooKeeper = null;
+      }
     }
 
     public Configuration getConfiguration() {
@@ -623,6 +629,7 @@ public HMasterInterface getMaster()
         LOG.info("Exception contacting master. Retrying...", ute.getCause());
       }
 
+      ensureZookeeperTrackers();
       checkIfBaseNodeAvailable();
       ServerName sn = null;
       synchronized (this.masterLock) {
@@ -762,6 +769,7 @@ private boolean testTableOnlineState(byte [] tableName, boolean online)
         // The root region is always enabled
         return online;
       }
+      getZooKeeperWatcher();
       String tableNameStr = Bytes.toString(tableName);
       try {
         if (online) {
@@ -807,11 +815,10 @@ private HRegionLocation locateRegion(final byte [] tableName,
         throw new IllegalArgumentException(
             "table name cannot be null or zero length");
       }
-
+      ensureZookeeperTrackers();
       if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
         try {
-          ServerName servername =
-            this.rootRegionTracker.waitRootRegionLocation(this.rpcTimeout);
+          ServerName servername = this.rootRegionTracker.getRootRegionLocation();
           LOG.debug("Looked up root region location, connection=" + this +
             "; serverName=" + ((servername == null)? "": servername.toString()));
           if (servername == null) return null;
@@ -1255,6 +1262,7 @@ HRegionInterface getHRegionConnection(final String hostname, final int port,
       } else {
         rsName = Addressing.createHostAndPortStr(hostname, port);
       }
+      ensureZookeeperTrackers();
       // See if we already have a connection (common case)
       server = this.servers.get(rsName);
       if (server == null) {
@@ -1642,25 +1650,21 @@ public void prewarmRegionCache(byte[] tableName,
 
     @Override
     public void abort(final String msg, Throwable t) {
-      if (t instanceof KeeperException.SessionExpiredException) {
-        try {
-          LOG.info("This client just lost it's session with ZooKeeper, trying" +
-              " to reconnect.");
-          resetZooKeeperTrackers();
-          LOG.info("Reconnected successfully. This disconnect could have been" +
+      if (t instanceof KeeperException) {
+        LOG.info("This client just lost it's session with ZooKeeper, will"
+            + " automatically reconnect when needed.");
+        if (t instanceof KeeperException.SessionExpiredException) {
+          LOG.info("ZK session expired. This disconnect could have been" +
               " caused by a network partition or a long-running GC pause," +
               " either way it's recommended that you verify your environment.");
-          return;
-        } catch (ZooKeeperConnectionException e) {
-          LOG.error("Could not reconnect to ZooKeeper after session" +
-              " expiration, aborting");
-          t = e;
+          resetZooKeeperTrackers();
         }
+        return;
       }
       if (t != null) LOG.fatal(msg, t);
       else LOG.fatal(msg);
       this.aborted = true;
-      this.closed = true;
+      close();
     }
 
     @Override
@@ -1675,6 +1679,7 @@ public boolean isAborted(){
 
     public int getCurrentNrHRS() throws IOException {
       try {
+        getZooKeeperWatcher();
         // We go to zk rather than to master to get count of regions to avoid
         // HTable having a Master dependency.  See HBase-2828
         return ZKUtil.getNumberOfChildren(this.zooKeeper,
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index d267824b467d..5d3776ade82a 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -26,6 +26,10 @@
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -58,12 +62,17 @@ public class TestZooKeeper {
 
   private final static HBaseTestingUtility
       TEST_UTIL = new HBaseTestingUtility();
+  private static HConnection persistentConnection;
 
   /**
    * @throws java.lang.Exception
    */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
+    // create a connection *before* the cluster is started, to validate that the
+    // connection's ZK trackers are initialized on demand
+    persistentConnection = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
+
     // Test we can first start the ZK cluster by itself
     TEST_UTIL.startMiniZKCluster();
     TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
@@ -75,6 +84,7 @@ public static void setUpBeforeClass() throws Exception {
    */
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
+    persistentConnection.close();
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -93,40 +103,28 @@ public void setUp() throws Exception {
    */
   @Test
   public void testClientSessionExpired()
-  throws IOException, InterruptedException {
+  throws Exception {
     LOG.info("testClientSessionExpired");
     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
     new HTable(c, HConstants.META_TABLE_NAME).close();
-    String quorumServers = ZKConfig.getZKQuorumServersString(c);
-    int sessionTimeout = 5 * 1000; // 5 seconds
     HConnection connection = HConnectionManager.getConnection(c);
     ZooKeeperWatcher connectionZK = connection.getZooKeeperWatcher();
-    long sessionID = connectionZK.getRecoverableZooKeeper().getSessionId();
-    byte[] password = connectionZK.getRecoverableZooKeeper().getSessionPasswd();
-    ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout,
-        EmptyWatcher.instance, sessionID, password);
-    LOG.info("Session timeout=" + zk.getSessionTimeout() +
-      ", original=" + sessionTimeout +
-      ", id=" + zk.getSessionId());
-    zk.close();
-
-    Thread.sleep(sessionTimeout * 3L);
+    TEST_UTIL.expireSession(connectionZK, null);
 
     // provoke session expiration by doing something with ZK
     ZKUtil.dump(connectionZK);
 
     // Check that the old ZK connection is closed, means we did expire
     System.err.println("ZooKeeper should have timed out");
-    String state = connectionZK.getRecoverableZooKeeper().getState().toString();
     LOG.info("state=" + connectionZK.getRecoverableZooKeeper().getState());
     Assert.assertTrue(connectionZK.getRecoverableZooKeeper().getState().
       equals(States.CLOSED));
 
     // Check that the client recovered
     ZooKeeperWatcher newConnectionZK = connection.getZooKeeperWatcher();
-    LOG.info("state=" + newConnectionZK.getRecoverableZooKeeper().getState());
-    Assert.assertTrue(newConnectionZK.getRecoverableZooKeeper().getState().equals(
-      States.CONNECTED));
+    States state = newConnectionZK.getRecoverableZooKeeper().getState();
+    LOG.info("state=" + state);
+    Assert.assertTrue(state.equals(States.CONNECTED) || state.equals(States.CONNECTING));
   }
   
   @Test
@@ -148,21 +146,33 @@ public void testMasterSessionExpired() throws Exception {
    * Make sure we can use the cluster
    * @throws Exception
    */
-  public void testSanity() throws Exception{
-    HBaseAdmin admin =
-      new HBaseAdmin(new Configuration(TEST_UTIL.getConfiguration()));
+  private void testSanity() throws Exception {
+    String tableName = "test"+System.currentTimeMillis();
+    HBaseAdmin admin = new HBaseAdmin(new Configuration(TEST_UTIL.getConfiguration()));
+    testAdminSanity(admin, tableName);
+    HTable table = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
+    testTableSanity(table, tableName);
+  }
+
+  private void testSanity(HConnection conn, ExecutorService pool) throws Exception {
     String tableName = "test"+System.currentTimeMillis();
+    HBaseAdmin admin = new HBaseAdmin(persistentConnection);
+    testAdminSanity(admin, tableName);
+    HTable table = new HTable(Bytes.toBytes(tableName), persistentConnection, pool);
+    testTableSanity(table, tableName);
+
+  }
+  private void testAdminSanity(HBaseAdmin admin, String tableName) throws Exception {
     HTableDescriptor desc = new HTableDescriptor(tableName);
     HColumnDescriptor family = new HColumnDescriptor("fam");
     desc.addFamily(family);
     LOG.info("Creating table " + tableName);
     admin.createTable(desc);
+  }
 
-    HTable table =
-      new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
+  private void testTableSanity(HTable table, String tableName) throws Exception {
     Put put = new Put(Bytes.toBytes("testrow"));
-    put.add(Bytes.toBytes("fam"),
-        Bytes.toBytes("col"), Bytes.toBytes("testdata"));
+    put.add(Bytes.toBytes("fam"), Bytes.toBytes("col"), Bytes.toBytes("testdata"));
     LOG.info("Putting table " + tableName);
     table.put(put);
     table.close();
@@ -229,6 +239,16 @@ public void testClusterKey() throws Exception {
     }
   }
 
+  /**
+   * Test with a connection that existed before the cluster was started
+   */
+  @Test
+  public void testPersistentConnection() throws Exception {
+    ExecutorService pool = new ThreadPoolExecutor(1, 10, 10, TimeUnit.SECONDS,
+        new SynchronousQueue<Runnable>());
+    testSanity(persistentConnection, pool);
+  }
+
   private void testKey(String ensemble, String port, String znode)
       throws IOException {
     Configuration conf = new Configuration();

From 1181e6161c729e0710cbc599adb8c7402ceb026a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 3 Apr 2012 04:16:01 +0000
Subject: [PATCH 0096/1540] HBASE-3134 [replication] Add the ability to
 enable/disable streams (Teruyoshi Zenmyo)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1308676 13f79535-47bb-0310-9956-ffa450edef68
---
 .../client/replication/ReplicationAdmin.java  |  23 +++-
 .../hbase/replication/ReplicationPeer.java    |  51 +++++++++
 .../replication/ReplicationZookeeper.java     | 105 ++++++++++++++++-
 .../regionserver/ReplicationSource.java       |  24 ++--
 .../ReplicationSourceInterface.java           |   5 -
 .../ReplicationSourceManager.java             |   7 +-
 src/main/ruby/hbase/replication_admin.rb      |   6 +
 src/main/ruby/shell/commands/disable_peer.rb  |   2 -
 src/main/ruby/shell/commands/enable_peer.rb   |   2 -
 src/main/ruby/shell/commands/list_peers.rb    |   5 +-
 .../replication/ReplicationSourceDummy.java   |  12 +-
 .../hbase/replication/TestReplication.java    | 108 +++++++++++++++++-
 .../TestReplicationSourceManager.java         |  17 ++-
 13 files changed, 320 insertions(+), 47 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index 61374d94e40e..b4efc3fb62f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -23,7 +23,6 @@
 import java.io.IOException;
 import java.util.Map;
 
-import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HConnection;
@@ -114,16 +113,16 @@ public void removePeer(String id) throws IOException {
    * Restart the replication stream to the specified peer.
    * @param id a short that identifies the cluster
    */
-  public void enablePeer(String id) {
-    throw new NotImplementedException("Not implemented");
+  public void enablePeer(String id) throws IOException {
+    this.replicationZk.enablePeer(id);
   }
 
   /**
    * Stop the replication stream to the specified peer.
    * @param id a short that identifies the cluster
    */
-  public void disablePeer(String id) {
-    throw new NotImplementedException("Not implemented");
+  public void disablePeer(String id) throws IOException {
+    this.replicationZk.disablePeer(id);
   }
 
   /**
@@ -142,6 +141,20 @@ public Map<String, String> listPeers() {
     return this.replicationZk.listPeers();
   }
 
+  /**
+   * Get state of the peer
+   *
+   * @param id peer's identifier
+   * @return current state of the peer
+   */
+  public String getPeerState(String id) throws IOException {
+    try {
+      return this.replicationZk.getPeerState(id).name();
+    } catch (KeeperException e) {
+      throw new IOException("Couldn't get the state of the peer " + id, e);
+    }
+  }
+
   /**
    * Get the current status of the kill switch, if the cluster is replicating
    * or not.
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 6495207bf7e3..0fec27f427be 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -29,7 +29,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper.PeerState;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * This class acts as a wrapper for all the objects used to identify and
@@ -47,6 +52,8 @@ public class ReplicationPeer implements Abortable {
   private ZooKeeperWatcher zkw;
   private final Configuration conf;
 
+  private PeerStateTracker peerStateTracker;
+
   /**
    * Constructor that takes all the objects required to communicate with the
    * specified peer, except for the region server addresses.
@@ -62,6 +69,31 @@ public ReplicationPeer(Configuration conf, String key,
     this.reloadZkWatcher();
   }
 
+  /**
+   * start a state tracker to check whether this peer is enabled or not
+   *
+   * @param zookeeper zk watcher for the local cluster
+   * @param peerStateNode path to zk node which stores peer state
+   * @throws KeeperException
+   */
+  public void startStateTracker(ZooKeeperWatcher zookeeper, String peerStateNode)
+      throws KeeperException {
+    if (ZKUtil.checkExists(zookeeper, peerStateNode) == -1) {
+      ZKUtil.createAndWatch(zookeeper, peerStateNode,
+          Bytes.toBytes(PeerState.ENABLED.name())); // enabled by default
+    }
+    this.peerStateTracker = new PeerStateTracker(peerStateNode, zookeeper,
+        this);
+    this.peerStateTracker.start();
+    this.readPeerStateZnode();
+  }
+
+  private void readPeerStateZnode() {
+    String currentState = Bytes.toString(peerStateTracker.getData(false));
+    this.peerEnabled.set(PeerState.ENABLED.equals(PeerState
+        .valueOf(currentState)));
+  }
+
   /**
    * Get the cluster key of that peer
    * @return string consisting of zk ensemble addresses, client port
@@ -142,4 +174,23 @@ public boolean isAborted() {
     // abort method is called.
     return false;
   }
+
+  /**
+   * Tracker for state of this peer
+   */
+  public class PeerStateTracker extends ZooKeeperNodeTracker {
+
+    public PeerStateTracker(String peerStateZNode, ZooKeeperWatcher watcher,
+        Abortable abortable) {
+      super(watcher, peerStateZNode, abortable);
+    }
+
+    @Override
+    public synchronized void nodeDataChanged(String path) {
+      if (path.equals(node)) {
+        super.nodeDataChanged(path);
+        readPeerStateZnode();
+      }
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index e2f4771f944e..61740589ee5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -48,18 +48,20 @@
 import org.apache.zookeeper.KeeperException.SessionExpiredException;
 
 /**
- * This class serves as a helper for all things related to zookeeper
- * in replication.
+ * This class serves as a helper for all things related to zookeeper in
+ * replication.
  * <p/>
- * The layout looks something like this under zookeeper.znode.parent
- * for the master cluster:
+ * The layout looks something like this under zookeeper.znode.parent for the
+ * master cluster:
  * <p/>
+ *
  * <pre>
  * replication/
  *  state      {contains true or false}
  *  clusterId  {contains a byte}
  *  peers/
  *    1/   {contains a full cluster address}
+ *      peer-state  {contains ENABLED or DISABLED}
  *    2/
  *    ...
  *  rs/ {lists all RS that replicate}
@@ -79,6 +81,12 @@ public class ReplicationZookeeper {
     LogFactory.getLog(ReplicationZookeeper.class);
   // Name of znode we use to lock when failover
   private final static String RS_LOCK_ZNODE = "lock";
+
+  // Values of znode which stores state of a peer
+  public static enum PeerState {
+    ENABLED, DISABLED
+  };
+
   // Our handle on zookeeper
   private final ZooKeeperWatcher zookeeper;
   // Map of peer clusters keyed by their id
@@ -93,6 +101,8 @@ public class ReplicationZookeeper {
   private String rsServerNameZnode;
   // Name node if the replicationState znode
   private String replicationStateNodeName;
+  // Name of zk node which stores peer state
+  private String peerStateNodeName;
   private final Configuration conf;
   // Is this cluster replicating at the moment?
   private AtomicBoolean replicating;
@@ -147,6 +157,8 @@ private void setZNodes(Abortable abortable) throws KeeperException {
         conf.get("zookeeper.znode.replication", "replication");
     String peersZNodeName =
         conf.get("zookeeper.znode.replication.peers", "peers");
+    this.peerStateNodeName = conf.get(
+        "zookeeper.znode.replication.peers.state", "peer-state");
     this.replicationStateNodeName =
         conf.get("zookeeper.znode.replication.state", "state");
     String rsZNodeName =
@@ -336,8 +348,10 @@ public ReplicationPeer getPeer(String peerId) throws IOException, KeeperExceptio
       return null;
     }
 
-    return new ReplicationPeer(otherConf, peerId,
+    ReplicationPeer peer = new ReplicationPeer(otherConf, peerId,
         otherClusterKey);
+    peer.startStateTracker(this.zookeeper, this.getPeerStateNode(peerId));
+    return peer;
   }
 
   /**
@@ -363,7 +377,8 @@ public void removePeer(String id) throws IOException {
       if (!peerExists(id)) {
         throw new IllegalArgumentException("Cannot remove inexisting peer");
       }
-      ZKUtil.deleteNode(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
+      ZKUtil.deleteNodeRecursively(this.zookeeper,
+          ZKUtil.joinZNode(this.peersZNode, id));
     } catch (KeeperException e) {
       throw new IOException("Unable to remove a peer", e);
     }
@@ -385,6 +400,8 @@ public void addPeer(String id, String clusterKey) throws IOException {
       ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
       ZKUtil.createAndWatch(this.zookeeper,
           ZKUtil.joinZNode(this.peersZNode, id), Bytes.toBytes(clusterKey));
+      ZKUtil.createAndWatch(this.zookeeper, getPeerStateNode(id),
+          Bytes.toBytes(PeerState.ENABLED.name())); // enabled by default
     } catch (KeeperException e) {
       throw new IOException("Unable to add peer", e);
     }
@@ -395,6 +412,82 @@ private boolean peerExists(String id) throws KeeperException {
           ZKUtil.joinZNode(this.peersZNode, id)) >= 0;
   }
 
+  /**
+   * Enable replication to the peer
+   *
+   * @param id peer's identifier
+   * @throws IllegalArgumentException
+   *           Thrown when the peer doesn't exist
+   */
+  public void enablePeer(String id) throws IOException {
+    changePeerState(id, PeerState.ENABLED);
+    LOG.info("peer " + id + " is enabled");
+  }
+
+  /**
+   * Disable replication to the peer
+   *
+   * @param id peer's identifier
+   * @throws IllegalArgumentException
+   *           Thrown when the peer doesn't exist
+   */
+  public void disablePeer(String id) throws IOException {
+    changePeerState(id, PeerState.DISABLED);
+    LOG.info("peer " + id + " is disabled");
+  }
+
+  private void changePeerState(String id, PeerState state) throws IOException {
+    try {
+      if (!peerExists(id)) {
+        throw new IllegalArgumentException("peer " + id + " is not registered");
+      }
+      String peerStateZNode = getPeerStateNode(id);
+      if (ZKUtil.checkExists(this.zookeeper, peerStateZNode) != -1) {
+        ZKUtil.setData(this.zookeeper, peerStateZNode,
+          Bytes.toBytes(state.name()));
+      } else {
+        ZKUtil.createAndWatch(zookeeper, peerStateZNode,
+            Bytes.toBytes(state.name()));
+      }
+      LOG.info("state of the peer " + id + " changed to " + state.name());
+    } catch (KeeperException e) {
+      throw new IOException("Unable to change state of the peer " + id, e);
+    }
+  }
+
+  /**
+   * Get state of the peer. This method checks the state by connecting to ZK.
+   *
+   * @param id peer's identifier
+   * @return current state of the peer
+   */
+  public PeerState getPeerState(String id) throws KeeperException {
+    byte[] peerStateBytes = ZKUtil
+        .getData(this.zookeeper, getPeerStateNode(id));
+    return PeerState.valueOf(Bytes.toString(peerStateBytes));
+  }
+
+  /**
+   * Check whether the peer is enabled or not. This method checks the atomic
+   * boolean of ReplicationPeer locally.
+   *
+   * @param id peer identifier
+   * @return true if the peer is enabled, otherwise false
+   * @throws IllegalArgumentException
+   *           Thrown when the peer doesn't exist
+   */
+  public boolean getPeerEnabled(String id) {
+    if (!this.peerClusters.containsKey(id)) {
+      throw new IllegalArgumentException("peer " + id + " is not registered");
+    }
+    return this.peerClusters.get(id).getPeerEnabled().get();
+  }
+
+  private String getPeerStateNode(String id) {
+    return ZKUtil.joinZNode(this.peersZNode,
+        ZKUtil.joinZNode(id, this.peerStateNodeName));
+  }
+
   /**
    * This reads the state znode for replication and sets the atomic boolean
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index ee0cc1286eaa..a9c8ff7cdd58 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -136,9 +136,6 @@ public class ReplicationSource extends Thread
   private volatile boolean running = true;
   // Metrics for this source
   private ReplicationSourceMetrics metrics;
-  // If source is enabled, replication happens. If disabled, nothing will be
-  // replicated but HLogs will still be queued
-  private AtomicBoolean sourceEnabled = new AtomicBoolean();
 
   /**
    * Instantiation method used by region servers
@@ -272,7 +269,7 @@ public void run() {
     // Loop until we close down
     while (isActive()) {
       // Sleep until replication is enabled again
-      if (!this.replicating.get() || !this.sourceEnabled.get()) {
+      if (!isPeerEnabled()) {
         if (sleepForRetries("Replication is disabled", sleepMultiplier)) {
           sleepMultiplier++;
         }
@@ -599,6 +596,12 @@ protected void shipEdits() {
       return;
     }
     while (this.isActive()) {
+      if (!isPeerEnabled()) {
+        if (sleepForRetries("Replication is disabled", sleepMultiplier)) {
+          sleepMultiplier++;
+        }
+        continue;
+      }
       try {
         HRegionInterface rrs = getRS();
         LOG.debug("Replicating " + currentNbEntries);
@@ -657,6 +660,15 @@ protected void shipEdits() {
     }
   }
 
+  /**
+   * check whether the peer is enabled or not
+   *
+   * @return true if the peer is enabled, otherwise false
+   */
+  protected boolean isPeerEnabled() {
+    return this.replicating.get() && this.zkHelper.getPeerEnabled(peerId);
+  }
+
   /**
    * If the queue isn't empty, switch to the next one
    * Else if this is a recovered queue, it means we're done!
@@ -763,10 +775,6 @@ public Path getCurrentPath() {
     return this.currentPath;
   }
 
-  public void setSourceEnabled(boolean status) {
-    this.sourceEnabled.set(status);
-  }
-
   private boolean isActive() {
     return !this.stopper.isStopped() && this.running;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index 62a4bee3518d..6778c43d6f69 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -93,9 +93,4 @@ public void init(final Configuration conf,
    */
   public String getPeerClusterId();
 
-  /**
-   * Set if this source is enabled or disabled
-   * @param status the new status
-   */
-  public void setSourceEnabled(boolean status);
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 152c5f667f98..e2dcf56bb32e 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -35,7 +35,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -47,6 +46,8 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
 /**
  * This class is responsible to manage all the replication
  * sources. There are two classes of sources:
@@ -201,8 +202,6 @@ public void init() throws IOException {
   public ReplicationSourceInterface addSource(String id) throws IOException {
     ReplicationSourceInterface src =
         getReplicationSource(this.conf, this.fs, this, stopper, replicating, id);
-    // TODO set it to what's in ZK
-    src.setSourceEnabled(true);
     synchronized (this.hlogsById) {
       this.sources.add(src);
       this.hlogsById.put(id, new TreeSet<String>());
@@ -583,8 +582,6 @@ public void run() {
           for (String hlog : entry.getValue()) {
             src.enqueueLog(new Path(oldLogDir, hlog));
           }
-          // TODO set it to what's in ZK
-          src.setSourceEnabled(true);
           src.startup();
         } catch (IOException e) {
           // TODO manage it
diff --git a/src/main/ruby/hbase/replication_admin.rb b/src/main/ruby/hbase/replication_admin.rb
index c4be93c299f2..f694f5f678d7 100644
--- a/src/main/ruby/hbase/replication_admin.rb
+++ b/src/main/ruby/hbase/replication_admin.rb
@@ -49,6 +49,12 @@ def list_peers
       @replication_admin.listPeers
     end
 
+    #----------------------------------------------------------------------------------------------
+    # Get peer cluster state
+    def get_peer_state(id)
+      @replication_admin.getPeerState(id)
+    end
+
     #----------------------------------------------------------------------------------------------
     # Restart the replication stream to the specified peer
     def enable_peer(id)
diff --git a/src/main/ruby/shell/commands/disable_peer.rb b/src/main/ruby/shell/commands/disable_peer.rb
index ad1ebbd4c7d4..da9941a9572d 100644
--- a/src/main/ruby/shell/commands/disable_peer.rb
+++ b/src/main/ruby/shell/commands/disable_peer.rb
@@ -26,8 +26,6 @@ def help
 Stops the replication stream to the specified cluster, but still
 keeps track of new edits to replicate.
 
-CURRENTLY UNSUPPORTED
-
 Examples:
 
   hbase> disable_peer '1'
diff --git a/src/main/ruby/shell/commands/enable_peer.rb b/src/main/ruby/shell/commands/enable_peer.rb
index 099f3fd4549a..de459092b58b 100644
--- a/src/main/ruby/shell/commands/enable_peer.rb
+++ b/src/main/ruby/shell/commands/enable_peer.rb
@@ -26,8 +26,6 @@ def help
 Restarts the replication to the specified peer cluster,
 continuing from where it was disabled.
 
-CURRENTLY UNSUPPORTED
-
 Examples:
 
   hbase> enable_peer '1'
diff --git a/src/main/ruby/shell/commands/list_peers.rb b/src/main/ruby/shell/commands/list_peers.rb
index 93a430c87104..2f57592e5876 100644
--- a/src/main/ruby/shell/commands/list_peers.rb
+++ b/src/main/ruby/shell/commands/list_peers.rb
@@ -33,10 +33,11 @@ def command()
         now = Time.now
         peers = replication_admin.list_peers
 
-        formatter.header(["PEER ID", "CLUSTER KEY"])
+        formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE"])
 
         peers.entrySet().each do |e|
-          formatter.row([ e.key, e.value ])
+          state = replication_admin.get_peer_state(e.key)
+          formatter.row([ e.key, e.value, state ])
         end
 
         formatter.footer(now)
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index 9d3e8620ed7d..2daf643ba368 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -19,6 +19,9 @@
  */
 package org.apache.hadoop.hbase.replication;
 
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -26,9 +29,6 @@
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager;
 
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 /**
  * Source that does nothing at all, helpful to test ReplicationSourceManager
  */
@@ -81,10 +81,4 @@ public String getPeerClusterZnode() {
   public String getPeerClusterId() {
     return peerClusterId;
   }
-
-  @Override
-  public void setSourceEnabled(boolean status) {
-
-  }
-
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index a9ae7cac46e0..f61655a6be35 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -26,7 +26,14 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -442,9 +449,108 @@ public void testStartStop() throws Exception {
 
   }
 
+  /**
+   * Test disable/enable replication, trying to insert, make sure nothing's
+   * replicated, enable it, the insert should be replicated
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 300000)
+  public void testDisableEnable() throws Exception {
+
+    // Test disabling replication
+    admin.disablePeer("2");
+
+    byte[] rowkey = Bytes.toBytes("disable enable");
+    Put put = new Put(rowkey);
+    put.add(famName, row, row);
+    htable1.put(put);
+
+    Get get = new Get(rowkey);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      Result res = htable2.get(get);
+      if (res.size() >= 1) {
+        fail("Replication wasn't disabled");
+      } else {
+        LOG.info("Row not replicated, let's wait a bit more...");
+        Thread.sleep(SLEEP_TIME);
+      }
+    }
+
+    // Test enable replication
+    admin.enablePeer("2");
+
+    for (int i = 0; i < NB_RETRIES; i++) {
+      Result res = htable2.get(get);
+      if (res.size() == 0) {
+        LOG.info("Row not available");
+        Thread.sleep(SLEEP_TIME);
+      } else {
+        assertArrayEquals(res.value(), row);
+        return;
+      }
+    }
+    fail("Waited too much time for put replication");
+  }
+
+  /**
+   * Test disabling an inactive peer. Add a peer which is inactive, trying to
+   * insert, disable the peer, then activate the peer and make sure nothing is
+   * replicated. In Addition, enable the peer and check the updates are
+   * replicated.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 600000)
+  public void testDisableInactivePeer() throws Exception {
+
+    // enabling and shutdown the peer
+    admin.enablePeer("2");
+    utility2.shutdownMiniHBaseCluster();
+
+    byte[] rowkey = Bytes.toBytes("disable inactive peer");
+    Put put = new Put(rowkey);
+    put.add(famName, row, row);
+    htable1.put(put);
+
+    // wait for the sleep interval of the master cluster to become long
+    Thread.sleep(SLEEP_TIME * NB_RETRIES);
+
+    // disable and start the peer
+    admin.disablePeer("2");
+    utility2.startMiniHBaseCluster(1, 1);
+    Get get = new Get(rowkey);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      Result res = htable2.get(get);
+      if (res.size() >= 1) {
+        fail("Replication wasn't disabled");
+      } else {
+        LOG.info("Row not replicated, let's wait a bit more...");
+        Thread.sleep(SLEEP_TIME);
+      }
+    }
+
+    // Test enable replication
+    admin.enablePeer("2");
+    // wait since the sleep interval would be long
+    Thread.sleep(SLEEP_TIME * NB_RETRIES);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      Result res = htable2.get(get);
+      if (res.size() == 0) {
+        LOG.info("Row not available");
+        Thread.sleep(SLEEP_TIME * NB_RETRIES);
+      } else {
+        assertArrayEquals(res.value(), row);
+        return;
+      }
+    }
+    fail("Waited too much time for put replication");
+  }
+
   /**
    * Integration test for TestReplicationAdmin, removes and re-add a peer
    * cluster
+   *
    * @throws Exception
    */
   @Test(timeout=300000)
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 07b3f3c30f0f..5828154d08b0 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -30,13 +30,23 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -100,6 +110,9 @@ public static void setUpBeforeClass() throws Exception {
     ZKUtil.setData(zkw, "/hbase/replication/peers/1",
         Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
             + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
+    ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
+    ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
+        Bytes.toBytes(ReplicationZookeeper.PeerState.ENABLED.name()));
     ZKUtil.createWithParents(zkw, "/hbase/replication/state");
     ZKUtil.setData(zkw, "/hbase/replication/state", Bytes.toBytes("true"));
 

From 6a2d0910e4de9e6b0674ee80793201afbbfc8530 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 3 Apr 2012 15:08:17 +0000
Subject: [PATCH 0097/1540] HBASE-5701 Put RegionServerDynamicStatistics under
 RegionServer in MBean hierarchy rather than have it as a peer

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1308970 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/metrics/RegionServerDynamicStatistics.java   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java
index c250d8170531..ea96cf50a506 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java
@@ -36,9 +36,7 @@ public class RegionServerDynamicStatistics extends MetricsDynamicMBeanBase {
 
   public RegionServerDynamicStatistics(MetricsRegistry registry) {
     super(registry, "RegionServerDynamicStatistics");
-    mbeanName = MBeanUtil.registerMBean("RegionServerDynamic",
-                                        "RegionServerDynamicStatistics",
-                                        this);
+    mbeanName = MBeanUtil.registerMBean("RegionServer", "RegionServerDynamicStatistics", this);
   }
 
   public void shutdown() {

From 61250553ceccde31925c2126fce9c518fc9f05fc Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 3 Apr 2012 22:04:15 +0000
Subject: [PATCH 0098/1540] HBASE-5606  SplitLogManger async delete node hangs
 log splitting when ZK connection is lost                (Prakash)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1309172 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/SplitLogManager.java  | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index c6b355b532f4..b72d7f018a93 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -402,6 +402,14 @@ private void getDataSetWatch(String path, Long retry_count) {
     tot_mgr_get_data_queued.incrementAndGet();
   }
 
+  private void tryGetDataSetWatch(String path) {
+    // A negative retry count will lead to ignoring all error processing.
+    this.watcher.getRecoverableZooKeeper().getZooKeeper().
+        getData(path, this.watcher,
+        new GetDataAsyncCallback(), new Long(-1) /* retry count */);
+    tot_mgr_get_data_queued.incrementAndGet();
+  }
+
   private void getDataSetWatchSuccess(String path, byte[] data, int version) {
     if (data == null) {
       if (version == Integer.MIN_VALUE) {
@@ -914,11 +922,13 @@ protected void chore() {
         for (Map.Entry<String, Task> e : tasks.entrySet()) {
           String path = e.getKey();
           Task task = e.getValue();
-          // we have to do this check again because tasks might have
-          // been asynchronously assigned.
-          if (task.isUnassigned()) {
+          // we have to do task.isUnassigned() check again because tasks might
+          // have been asynchronously assigned. There is no locking required
+          // for these checks ... it is OK even if tryGetDataSetWatch() is
+          // called unnecessarily for a task
+          if (task.isUnassigned() && (task.status != FAILURE)) {
             // We just touch the znode to make sure its still there
-            getDataSetWatch(path, zkretries);
+            tryGetDataSetWatch(path);
           }
         }
         createRescanNode(Long.MAX_VALUE);
@@ -988,6 +998,12 @@ public void processResult(int rc, String path, Object ctx, byte[] data,
           return;
         }
         Long retry_count = (Long) ctx;
+
+        if (retry_count < 0) {
+          LOG.warn("getdata rc = " + KeeperException.Code.get(rc) + " " +
+              path + ". Ignoring error. No error handling. No retrying.");
+          return;
+        }
         LOG.warn("getdata rc = " + KeeperException.Code.get(rc) + " " +
             path + " remaining retries=" + retry_count);
         if (retry_count == 0) {

From 85b23613747b9b773f6a0be729bf2a7394f92c0a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 4 Apr 2012 15:36:55 +0000
Subject: [PATCH 0099/1540] HBASE-5706 'Dropping fs latency stats since buffer
 is full' spam

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1309459 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/hfile/HFile.java   | 35 ++-----------------
 1 file changed, 3 insertions(+), 32 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 1fe5db9887f7..5139162ba34f 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -192,50 +192,21 @@ public class HFile {
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
   private static final BlockingQueue<Long> fsPreadLatenciesNanos = 
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  private static final AtomicLong lastLoggedDataDrop = new AtomicLong(0);
-  
-  // we don't want to fill up the logs with this message, so only log it 
-  // once every 30 seconds at most
-  // I also want to avoid locks on the 'critical path' (the common case will be
-  // uncontended) - hence the CAS
-  private static void logDroppedLatencyStat() {
-    final long now = System.currentTimeMillis();      
-    final long earliestAcceptableLog = now - TimeUnit.SECONDS.toMillis(30L);
-    while (true) {
-      final long lastLog = lastLoggedDataDrop.get();
-      if (lastLog < earliestAcceptableLog) {
-        if (lastLoggedDataDrop.compareAndSet(lastLog, now)) {
-          LOG.warn("Dropping fs latency stats since buffer is full");
-          break;
-        } // otherwise (if the compaseAndSet failed) the while loop retries
-      } else {
-        break;
-      }
-    }    
-  }
   
   public static final void offerReadLatency(long latencyNanos, boolean pread) {
-    boolean stored = false;
     if (pread) {
-      stored = fsPreadLatenciesNanos.offer(latencyNanos);
+      fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
       preadOps.incrementAndGet();
       preadTimeNano.addAndGet(latencyNanos);
     } else {
-      stored = fsReadLatenciesNanos.offer(latencyNanos);
+      fsReadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
       readTimeNano.addAndGet(latencyNanos);
       readOps.incrementAndGet();
     }
-
-    if (!stored) { 
-      logDroppedLatencyStat();
-    }    
   }
   
   public static final void offerWriteLatency(long latencyNanos) {
-    final boolean stored = fsWriteLatenciesNanos.offer(latencyNanos);
-    if (!stored) {
-      logDroppedLatencyStat();
-    }
+    fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
     
     writeTimeNano.addAndGet(latencyNanos);
     writeOps.incrementAndGet();

From eab683a833aefa44763618702547141a054efbe4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 5 Apr 2012 17:31:18 +0000
Subject: [PATCH 0100/1540] HBASE-5721 Update bundled hadoop to be 1.0.2 (it
 was just released)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1309964 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 54fd55c10e76..f3156b1708f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1516,7 +1516,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>1.0.0</hadoop.version>
+        <hadoop.version>1.0.2</hadoop.version>
       </properties>
       <dependencies>
         <dependency>
@@ -1587,7 +1587,7 @@
     <profile>
       <id>security</id>
       <properties>
-        <hadoop.version>1.0.0</hadoop.version>
+        <hadoop.version>1.0.2</hadoop.version>
       </properties>
       <build>
         <finalName>${project.artifactId}-${project.version}-security</finalName>

From 6f060a9ac1d12217511a1f4d202a35e82c775508 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 5 Apr 2012 19:13:39 +0000
Subject: [PATCH 0101/1540] HBASE-5715 Revert 'Instant schema alter' for now,
 HBASE-4213

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310017 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/LocalHBaseCluster.java       |  10 +-
 .../hadoop/hbase/executor/EventHandler.java   |  11 +-
 .../hadoop/hbase/ipc/HMasterInterface.java    |   8 -
 .../apache/hadoop/hbase/master/HMaster.java   | 184 +---
 .../hadoop/hbase/master/MasterFileSystem.java |   2 +-
 .../hadoop/hbase/master/MasterServices.java   |  27 +-
 .../hadoop/hbase/master/ServerManager.java    |  10 -
 .../master/handler/DeleteTableHandler.java    |   7 +-
 .../master/handler/ModifyTableHandler.java    |   7 +-
 .../master/handler/TableAddFamilyHandler.java |   7 +-
 .../handler/TableDeleteFamilyHandler.java     |   7 +-
 .../master/handler/TableEventHandler.java     | 164 +---
 .../handler/TableModifyFamilyHandler.java     |   7 +-
 .../regionserver/CompactSplitThread.java      |  17 -
 .../hbase/regionserver/HRegionServer.java     |  94 +-
 .../hbase/regionserver/OnlineRegions.java     |  22 +-
 .../zookeeper/MasterSchemaChangeTracker.java  | 826 ------------------
 .../hbase/zookeeper/SchemaChangeTracker.java  | 476 ----------
 .../hbase/zookeeper/ZooKeeperWatcher.java     |   5 -
 src/main/resources/hbase-default.xml          |  42 +-
 .../client/InstantSchemaChangeTestBase.java   | 169 ----
 .../hbase/client/TestInstantSchemaChange.java | 473 ----------
 .../TestInstantSchemaChangeFailover.java      | 313 -------
 .../client/TestInstantSchemaChangeSplit.java  | 224 -----
 .../hbase/master/TestCatalogJanitor.java      |  21 +-
 .../hbase/util/MockRegionServerServices.java  |   3 -
 26 files changed, 119 insertions(+), 3017 deletions(-)
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/zookeeper/SchemaChangeTracker.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChange.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeFailover.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeSplit.java

diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index f49f14c457d7..134749add43e 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -305,10 +305,12 @@ public HMaster getMaster(int serverNumber) {
    */
   public HMaster getActiveMaster() {
     for (JVMClusterUtil.MasterThread mt : masterThreads) {
-      // Ensure that the current active master is not stopped.
-      // We don't want to return a stopping master as an active master.
-      if (mt.getMaster().isActiveMaster()  && !mt.getMaster().isStopped()) {
-        return mt.getMaster();
+      if (mt.getMaster().isActiveMaster()) {
+        // Ensure that the current active master is not stopped.
+        // We don't want to return a stopping master as an active master.
+        if (mt.getMaster().isActiveMaster()  && !mt.getMaster().isStopped()) {
+          return mt.getMaster();
+        }
       }
     }
     return null;
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
index bdd3cf0d5239..c9acee3ce036 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
@@ -142,12 +142,13 @@ public enum EventType {
      * Constructor
      */
     EventType(int value) {}
-    public boolean isSchemaChangeEvent() {
+    public boolean isOnlineSchemaChangeSupported() {
       return (
-          this.equals(EventType.C_M_ADD_FAMILY) ||
-          this.equals(EventType.C_M_DELETE_FAMILY) ||
-          this.equals(EventType.C_M_MODIFY_FAMILY) ||
-          this.equals(EventType.C_M_MODIFY_TABLE));
+        this.equals(EventType.C_M_ADD_FAMILY) ||
+        this.equals(EventType.C_M_DELETE_FAMILY) ||
+        this.equals(EventType.C_M_MODIFY_FAMILY) ||
+        this.equals(EventType.C_M_MODIFY_TABLE)
+      );
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index a6276cbfbb8c..645b30643c14 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -266,12 +266,4 @@ public void unassign(final byte [] regionName, final boolean force)
    * @return array of HTableDescriptor
    */
   public HTableDescriptor[] getHTableDescriptors(List<String> tableNames);
-
-  /**
-   * Returns the current running status of load balancer.
-   * @return True if LoadBalancer is running now else False.
-   */
-  public boolean isLoadBalancerRunning();
-
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 8b3c11111a82..014b5a6c05f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -66,7 +66,6 @@
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
@@ -85,6 +84,7 @@
 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -101,7 +101,6 @@
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -174,10 +173,7 @@ public class HMaster extends HasThread
   private CatalogTracker catalogTracker;
   // Cluster status zk tracker and local setter
   private ClusterStatusTracker clusterStatusTracker;
-
-  // Schema change tracker
-  private MasterSchemaChangeTracker schemaChangeTracker;
-
+  
   // buffer for "fatal error" notices from region servers
   // in the cluster. This is only used for assisting
   // operations/debugging.
@@ -205,18 +201,12 @@ public class HMaster extends HasThread
 
   private CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
-  private Thread schemaJanitorChore;
 
   private MasterCoprocessorHost cpHost;
   private final ServerName serverName;
 
   private TableDescriptors tableDescriptors;
 
-  // Whether or not schema alter changes go through ZK or not.
-  private boolean supportInstantSchemaChanges = false;
-
-  private volatile boolean loadBalancerRunning = false;
-
   // Time stamps for when a hmaster was started and when it became active
   private long masterStartTime;
   private long masterActiveTime;
@@ -290,17 +280,6 @@ public HMaster(final Configuration conf)
     this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);
     this.rpcServer.startThreads();
     this.metrics = new MasterMetrics(getServerName().toString());
-    // initialize instant schema change settings
-    this.supportInstantSchemaChanges = conf.getBoolean(
-        "hbase.instant.schema.alter.enabled", false);
-    if (supportInstantSchemaChanges) {
-      LOG.info("Instant schema change enabled. All schema alter operations will " +
-          "happen through ZK.");
-    }
-   else {
-      LOG.info("Instant schema change disabled. All schema alter operations will " +
-          "happen normally.");
-    }
   }
 
   /**
@@ -440,12 +419,6 @@ private void initializeZKBasedSystemTrackers() throws IOException,
     boolean wasUp = this.clusterStatusTracker.isClusterUp();
     if (!wasUp) this.clusterStatusTracker.setClusterUp();
 
-    // initialize schema change tracker
-    this.schemaChangeTracker = new MasterSchemaChangeTracker(getZooKeeper(),
-        this, this,
-        conf.getInt("hbase.instant.schema.alter.timeout", 60000));
-    this.schemaChangeTracker.start();
-
     LOG.info("Server active/primary master; " + this.serverName +
         ", sessionid=0x" +
         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
@@ -566,9 +539,6 @@ private void finishInitialization(MonitoredTask status)
     this.catalogJanitorChore = new CatalogJanitor(this, this);
     Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
 
-    // Schema janitor chore.
-    this.schemaJanitorChore = getAndStartSchemaJanitorChore(this);
-
     registerMBean();
 
     status.markComplete("Initialization successful");
@@ -744,15 +714,6 @@ public TableDescriptors getTableDescriptors() {
     return this.tableDescriptors;
   }
 
-  @Override
-  public MasterSchemaChangeTracker getSchemaChangeTracker() {
-    return this.schemaChangeTracker;
-  }
-
-  public RegionServerTracker getRegionServerTracker() {
-    return this.regionServerTracker;
-  }
-
   /** @return InfoServer object. Maybe null.*/
   public InfoServer getInfoServer() {
     return this.infoServer;
@@ -855,28 +816,7 @@ private void stopServiceThreads() {
     if (this.executorService != null) this.executorService.shutdown();
   }
 
-  /**
-   * Start the schema janitor. This Janitor will periodically sweep the failed/expired schema
-   * changes.
-   * @param master
-   * @return
-   */
-  private Thread getAndStartSchemaJanitorChore(final HMaster master) {
-    String name = master.getServerName() + "-SchemaJanitorChore";
-    int schemaJanitorPeriod =
-      master.getConfiguration().getInt("hbase.instant.schema.janitor.period", 120000);
-    // Start up the schema janitor chore
-    Chore chore = new Chore(name, schemaJanitorPeriod, master) {
-      @Override
-      protected void chore() {
-        master.getSchemaChangeTracker().handleFailedOrExpiredSchemaChanges();
-      }
-    };
-    return Threads.setDaemonThreadRunning(chore.getThread());
-  }
-
-
-  private Thread getAndStartBalancerChore(final HMaster master) {
+  private static Thread getAndStartBalancerChore(final HMaster master) {
     String name = master.getServerName() + "-BalancerChore";
     int balancerPeriod =
       master.getConfiguration().getInt("hbase.balancer.period", 300000);
@@ -897,10 +837,6 @@ private void stopChores() {
     if (this.catalogJanitorChore != null) {
       this.catalogJanitorChore.interrupt();
     }
-    if (this.schemaJanitorChore != null) {
-      this.schemaJanitorChore.interrupt();
-    }
-
   }
 
   @Override
@@ -971,15 +907,6 @@ private int getBalancerCutoffTime() {
     return balancerCutoffTime;
   }
 
-
-  /**
-   * Check whether the Load Balancer is currently running.
-   * @return true if the Load balancer is currently running.
-   */
-  public boolean isLoadBalancerRunning() {
-    return loadBalancerRunning;
-  }
-
   @Override
   public boolean balance() {
     // If balance not true, don't run balancer.
@@ -987,33 +914,23 @@ public boolean balance() {
     // Do this call outside of synchronized block.
     int maximumBalanceTime = getBalancerCutoffTime();
     long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
-    boolean balancerRan = false;
+    boolean balancerRan;
     synchronized (this.balancer) {
-      if (loadBalancerRunning) {
-        LOG.debug("Load balancer is currently running. Skipping the current execution.");
-        return false;
-      }
-
       // Only allow one balance run at at time.
       if (this.assignmentManager.isRegionsInTransition()) {
         LOG.debug("Not running balancer because " +
-            this.assignmentManager.getRegionsInTransition().size() +
-            " region(s) in transition: " +
-            org.apache.commons.lang.StringUtils.
+          this.assignmentManager.getRegionsInTransition().size() +
+          " region(s) in transition: " +
+          org.apache.commons.lang.StringUtils.
             abbreviate(this.assignmentManager.getRegionsInTransition().toString(), 256));
         return false;
       }
       if (this.serverManager.areDeadServersInProgress()) {
         LOG.debug("Not running balancer because processing dead regionserver(s): " +
-            this.serverManager.getDeadServers());
+          this.serverManager.getDeadServers());
         return false;
       }
-      if (schemaChangeTracker.isSchemaChangeInProgress()) {
-        LOG.debug("Schema change operation is in progress. Waiting for " +
-        "it to complete before running the load balancer.");
-        return false;
-      }
-      loadBalancerRunning = true;
+
       if (this.cpHost != null) {
         try {
           if (this.cpHost.preBalance()) {
@@ -1048,7 +965,7 @@ public boolean balance() {
               // if performing next balance exceeds cutoff time, exit the loop
               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
-                maximumBalanceTime);
+              maximumBalanceTime);
             break;
           }
         }
@@ -1061,7 +978,6 @@ public boolean balance() {
           LOG.error("Error invoking master coprocessor postBalance()", ioe);
         }
       }
-      loadBalancerRunning = false;
     }
     return balancerRan;
   }
@@ -1211,9 +1127,7 @@ public void deleteTable(final byte [] tableName) throws IOException {
     if (cpHost != null) {
       cpHost.preDeleteTable(tableName);
     }
-    this.executorService.submit(new DeleteTableHandler(tableName, this, this, this,
-        supportInstantSchemaChanges));
-
+    this.executorService.submit(new DeleteTableHandler(tableName, this, this));
     if (cpHost != null) {
       cpHost.postDeleteTable(tableName);
     }
@@ -1225,49 +1139,13 @@ public void deleteTable(final byte [] tableName) throws IOException {
    * @return Pair indicating the number of regions updated Pair.getFirst is the
    *         regions that are yet to be updated Pair.getSecond is the total number
    *         of regions of the table
+   * @throws IOException 
    */
   public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
   throws IOException {
-    if (supportInstantSchemaChanges) {
-      return getAlterStatusFromSchemaChangeTracker(tableName);
-    }
     return this.assignmentManager.getReopenStatus(tableName);
   }
 
-  /**
-   * Used by the client to identify if all regions have the schema updates
-   *
-   * @param tableName
-   * @return Pair indicating the status of the alter command
-   * @throws IOException
-   */
-  private Pair<Integer, Integer> getAlterStatusFromSchemaChangeTracker(byte[] tableName)
-      throws IOException {
-    MasterSchemaChangeTracker.MasterAlterStatus alterStatus = null;
-    try {
-      alterStatus =
-          this.schemaChangeTracker.getMasterAlterStatus(Bytes.toString(tableName));
-    } catch (KeeperException ke) {
-      LOG.error("KeeperException while getting schema alter status for table = "
-      + Bytes.toString(tableName), ke);
-    }
-    if (alterStatus != null) {
-      LOG.debug("Getting AlterStatus from SchemaChangeTracker for table = "
-          + Bytes.toString(tableName) + " Alter Status = "
-          + alterStatus.toString());
-      int numberPending = alterStatus.getNumberOfRegionsToProcess() -
-          alterStatus.getNumberOfRegionsProcessed();
-      return new Pair<Integer, Integer>(alterStatus.getNumberOfRegionsProcessed(),
-          alterStatus.getNumberOfRegionsToProcess());
-    } else {
-      LOG.debug("MasterAlterStatus is NULL for table = "
-          + Bytes.toString(tableName));
-      // should we throw IOException here as it makes more sense?
-      return new Pair<Integer, Integer>(0,0);
-    }
-  }
-
-
   public void addColumn(byte [] tableName, HColumnDescriptor column)
   throws IOException {
     checkInitialized();
@@ -1276,8 +1154,7 @@ public void addColumn(byte [] tableName, HColumnDescriptor column)
         return;
       }
     }
-    new TableAddFamilyHandler(tableName, column, this, this,
-        this, supportInstantSchemaChanges).process();
+    new TableAddFamilyHandler(tableName, column, this, this).process();
     if (cpHost != null) {
       cpHost.postAddColumn(tableName, column);
     }
@@ -1291,8 +1168,7 @@ public void modifyColumn(byte [] tableName, HColumnDescriptor descriptor)
         return;
       }
     }
-    new TableModifyFamilyHandler(tableName, descriptor, this, this,
-        this, supportInstantSchemaChanges).process();
+    new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
     if (cpHost != null) {
       cpHost.postModifyColumn(tableName, descriptor);
     }
@@ -1306,8 +1182,7 @@ public void deleteColumn(final byte [] tableName, final byte [] c)
         return;
       }
     }
-    new TableDeleteFamilyHandler(tableName, c, this, this,
-        this, supportInstantSchemaChanges).process();
+    new TableDeleteFamilyHandler(tableName, c, this, this).process();
     if (cpHost != null) {
       cpHost.postDeleteColumn(tableName, c);
     }
@@ -1319,7 +1194,7 @@ public void enableTable(final byte [] tableName) throws IOException {
       cpHost.preEnableTable(tableName);
     }
     this.executorService.submit(new EnableTableHandler(this, tableName,
-        catalogTracker, assignmentManager, false));
+      catalogTracker, assignmentManager, false));
 
     if (cpHost != null) {
       cpHost.postEnableTable(tableName);
@@ -1381,33 +1256,14 @@ public void modifyTable(final byte[] tableName, HTableDescriptor htd)
     if (cpHost != null) {
       cpHost.preModifyTable(tableName, htd);
     }
-    this.executorService.submit(new ModifyTableHandler(tableName, htd, this,
-      this, this, supportInstantSchemaChanges));
+    this.executorService.submit(new ModifyTableHandler(tableName, htd, this, this));
     if (cpHost != null) {
       cpHost.postModifyTable(tableName, htd);
     }
   }
 
-  private boolean isOnlineSchemaChangeAllowed() {
-    return conf.getBoolean(
-        "hbase.online.schema.update.enable", false);
-  }
-  
   @Override
-  public void checkTableModifiable(final byte [] tableName,
-                                   EventHandler.EventType eventType)
-  throws IOException {
-    preCheckTableModifiable(tableName);
-    if (!eventType.isSchemaChangeEvent() ||
-        !isOnlineSchemaChangeAllowed()) {
-      if (!getAssignmentManager().getZKTable().
-          isDisabledTable(Bytes.toString(tableName))) {
-        throw new TableNotDisabledException(tableName);
-      }
-    }
-  }
-
-  private void preCheckTableModifiable(final byte[] tableName)
+  public void checkTableModifiable(final byte [] tableName)
   throws IOException {
     String tableNameStr = Bytes.toString(tableName);
     if (isCatalogTable(tableName)) {
@@ -1416,6 +1272,10 @@ private void preCheckTableModifiable(final byte[] tableName)
     if (!MetaReader.tableExists(getCatalogTracker(), tableNameStr)) {
       throw new TableNotFoundException(tableNameStr);
     }
+    if (!getAssignmentManager().getZKTable().
+        isDisabledTable(Bytes.toString(tableName))) {
+      throw new TableNotDisabledException(tableName);
+    }
   }
 
   public void clearFromTransition(HRegionInfo hri) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 411023485fbd..d1e7d7eca97b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -512,7 +512,7 @@ public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
    */
   public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
       throws IOException {
-    LOG.debug("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
+    LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
       hcd.toString());
     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
     if (htd == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index d88db0e91da0..6d6d00914c2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 
 /**
@@ -54,15 +53,12 @@ public interface MasterServices extends Server {
   public ExecutorService getExecutorService();
 
   /**
-   * Check table modifiable. i.e not ROOT or META and offlined for all commands except
-   * alter commands
-   * @param tableName
-   * @param eventType
-   * @throws IOException
+   * Check table is modifiable; i.e. exists and is offline.
+   * @param tableName Name of table to check.
+   * @throws TableNotDisabledException
+   * @throws TableNotFoundException 
    */
-  public void checkTableModifiable(final byte [] tableName,
-                                   EventHandler.EventType eventType)
-      throws IOException;
+  public void checkTableModifiable(final byte [] tableName) throws IOException;
 
   /**
    * Create a table using the given table definition.
@@ -78,21 +74,8 @@ public void createTable(HTableDescriptor desc, byte [][] splitKeys)
    */
   public TableDescriptors getTableDescriptors();
 
-  /**
-   * Get Master Schema change tracker
-   * @return
-   */
-  public MasterSchemaChangeTracker getSchemaChangeTracker();
-
-  /**
-   * Return the Region server tracker.
-   * @return RegionServerTracker
-   */
-  public RegionServerTracker getRegionServerTracker();
-
   /**
    * @return true if master enables ServerShutdownHandler;
    */
   public boolean isServerShutdownHandlerEnabled();
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index d5c8877d7040..8281c6ffa786 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -343,15 +343,6 @@ void letRegionServersShutdown() {
     }
   }
 
-  /**
-   * Exclude a RS from any pending schema change process.
-   * @param serverName
-   */
-  private void excludeRegionServerFromSchemaChanges(final ServerName serverName) {
-    this.services.getSchemaChangeTracker()
-        .excludeRegionServerForSchemaChanges(serverName.getServerName());
-  }
-
   /*
    * Expire the passed server.  Add it to list of deadservers and queue a
    * shutdown processing.
@@ -363,7 +354,6 @@ public synchronized void expireServer(final ServerName serverName) {
       this.deadNotExpiredServers.add(serverName);
       return;
     }
-    excludeRegionServerFromSchemaChanges(serverName);
     if (!this.onlineServers.containsKey(serverName)) {
       LOG.warn("Received expiration of " + serverName +
         " but server is not currently online");
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index a79ba088eed6..b299bee4a8fd 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -38,11 +37,9 @@ public class DeleteTableHandler extends TableEventHandler {
   private static final Log LOG = LogFactory.getLog(DeleteTableHandler.class);
 
   public DeleteTableHandler(byte [] tableName, Server server,
-      final MasterServices masterServices, HMasterInterface masterInterface,
-      boolean instantChange)
+      final MasterServices masterServices)
   throws IOException {
-    super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices,
-        masterInterface, instantChange);
+    super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices);
     // The next call fails if no such table.
     getTableDescriptor();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
index c09181648634..a6859f0e6881 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 
 public class ModifyTableHandler extends TableEventHandler {
@@ -33,11 +32,9 @@ public class ModifyTableHandler extends TableEventHandler {
 
   public ModifyTableHandler(final byte [] tableName,
       final HTableDescriptor htd, final Server server,
-      final MasterServices masterServices, final HMasterInterface masterInterface,
-      boolean instantModify)
+      final MasterServices masterServices)
   throws IOException {
-    super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices,
-        masterInterface, instantModify);
+    super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices);
     // Check table exists.
     getTableDescriptor();
     // This is the new schema we are going to write out as this modification.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
index d9933673b650..3fb50b2efeb8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 
 /**
@@ -38,10 +37,8 @@ public class TableAddFamilyHandler extends TableEventHandler {
   private final HColumnDescriptor familyDesc;
 
   public TableAddFamilyHandler(byte[] tableName, HColumnDescriptor familyDesc,
-      Server server, final MasterServices masterServices,
-      HMasterInterface masterInterface, boolean instantChange) throws IOException {
-    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices,
-        masterInterface, instantChange);
+      Server server, final MasterServices masterServices) throws IOException {
+    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
     HTableDescriptor htd = getTableDescriptor();
     if (htd.hasFamily(familyDesc.getName())) {
       throw new InvalidFamilyOperationException("Family '" +
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
index 07f67dd57194..89889316744a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -37,10 +36,8 @@ public class TableDeleteFamilyHandler extends TableEventHandler {
   private final byte [] familyName;
 
   public TableDeleteFamilyHandler(byte[] tableName, byte [] familyName,
-      Server server, final MasterServices masterServices,
-      HMasterInterface masterInterface, boolean instantChange) throws IOException {
-    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices,
-        masterInterface, instantChange);
+      Server server, final MasterServices masterServices) throws IOException {
+    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
     HTableDescriptor htd = getTableDescriptor();
     this.familyName = hasColumnFamily(htd, familyName);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
index af5b96a412a0..4fd784dd9aa9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -35,19 +35,13 @@
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.executor.EventHandler;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.BulkReOpen;
 import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
-import org.apache.hadoop.hbase.zookeeper.ZKAssign;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.collect.Lists;
@@ -63,22 +57,32 @@
 public abstract class TableEventHandler extends EventHandler {
   private static final Log LOG = LogFactory.getLog(TableEventHandler.class);
   protected final MasterServices masterServices;
-  protected HMasterInterface master = null;
   protected final byte [] tableName;
   protected final String tableNameStr;
-  protected boolean instantAction = false;
 
   public TableEventHandler(EventType eventType, byte [] tableName, Server server,
-      MasterServices masterServices, HMasterInterface masterInterface,
-      boolean instantSchemaChange)
+      MasterServices masterServices)
   throws IOException {
     super(server, eventType);
     this.masterServices = masterServices;
     this.tableName = tableName;
-    this.masterServices.checkTableModifiable(tableName, eventType);
+    try {
+      this.masterServices.checkTableModifiable(tableName);
+    } catch (TableNotDisabledException ex)  {
+      if (isOnlineSchemaChangeAllowed()
+          && eventType.isOnlineSchemaChangeSupported()) {
+        LOG.debug("Ignoring table not disabled exception " +
+            "for supporting online schema changes.");
+      }	else {
+        throw ex;
+      }
+    }
     this.tableNameStr = Bytes.toString(this.tableName);
-    this.instantAction = instantSchemaChange;
-    this.master = masterInterface;
+  }
+
+  private boolean isOnlineSchemaChangeAllowed() {
+    return this.server.getConfiguration().getBoolean(
+      "hbase.online.schema.update.enable", false);
   }
 
   @Override
@@ -90,7 +94,16 @@ public void process() {
         MetaReader.getTableRegions(this.server.getCatalogTracker(),
           tableName);
       handleTableOperation(hris);
-      handleSchemaChanges(hris);
+      if (eventType.isOnlineSchemaChangeSupported() && this.masterServices.
+          getAssignmentManager().getZKTable().
+          isEnabledTable(Bytes.toString(tableName))) {
+        if (reOpenAllRegions(hris)) {
+          LOG.info("Completed table operation " + eventType + " on table " +
+              Bytes.toString(tableName));
+        } else {
+          LOG.warn("Error on reopening the regions");
+        }
+      }
     } catch (IOException e) {
       LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
     } catch (KeeperException e) {
@@ -98,47 +111,13 @@ public void process() {
     }
   }
 
-  private void handleSchemaChanges(List<HRegionInfo> regions)
-      throws IOException {
-    if (instantAction && regions != null && !regions.isEmpty()) {
-      handleInstantSchemaChanges(regions);
-    } else {
-      handleRegularSchemaChanges(regions);
-    }
-  }
-
-
-  /**
-   * Perform schema changes only if the table is in enabled state.
-   * @return
-   */
-  private boolean canPerformSchemaChange() {
-    return (eventType.isSchemaChangeEvent() && this.masterServices.
-        getAssignmentManager().getZKTable().
-        isEnabledTable(Bytes.toString(tableName)));
-  }
-
-  private void handleRegularSchemaChanges(List<HRegionInfo> regions)
-      throws IOException {
-    if (canPerformSchemaChange()) {
-      this.masterServices.getAssignmentManager().setRegionsToReopen(regions);
-      if (reOpenAllRegions(regions)) {
-        LOG.info("Completed table operation " + eventType + " on table " +
-            Bytes.toString(tableName));
-      } else {
-        LOG.warn("Error on reopening the regions");
-      }
-    }
-  }
-
   public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
     boolean done = false;
     LOG.info("Bucketing regions by region server...");
     HTable table = new HTable(masterServices.getConfiguration(), tableName);
     TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps
         .newTreeMap();
-    NavigableMap<HRegionInfo, ServerName> hriHserverMapping
-        = table.getRegionLocations();
+    NavigableMap<HRegionInfo, ServerName> hriHserverMapping = table.getRegionLocations();
     List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
     for (HRegionInfo hri : regions) {
       ServerName rsLocation = hriHserverMapping.get(hri);
@@ -180,91 +159,6 @@ public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
     return done;
   }
 
-  /**
-   * Check whether any of the regions from the list of regions is undergoing a split.
-   * We simply check whether there is a unassigned node for any of the region and if so
-   * we return as true.
-   * @param regionInfos
-   * @return
-   */
-  private boolean isSplitInProgress(List<HRegionInfo> regionInfos) {
-    for (HRegionInfo hri : regionInfos) {
-      ZooKeeperWatcher zkw = this.masterServices.getZooKeeper();
-      String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
-      try {
-        if (ZKUtil.checkExists(zkw, node) != -1) {
-          LOG.debug("Region " + hri.getRegionNameAsString() + " is unassigned. Assuming" +
-          " that it is undergoing a split");
-          return true;
-        }
-      } catch (KeeperException ke) {
-        LOG.debug("KeeperException while determining splits in progress.", ke);
-        // Assume no splits happening?
-        return false;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Wait for region split transaction in progress (if any)
-   * @param regions
-   * @param status
-   */
-  private void waitForInflightSplit(List<HRegionInfo> regions, MonitoredTask status) {
-    while (isSplitInProgress(regions)) {
-      try {
-        status.setStatus("Alter Schema is waiting for split region to complete.");
-        Thread.sleep(100);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
-    }
-  }
-
-  protected void handleInstantSchemaChanges(List<HRegionInfo> regions) {
-    if (regions == null || regions.isEmpty()) {
-      LOG.debug("Region size is null or empty. Ignoring alter request.");
-      return;
-    }
-    MonitoredTask status = TaskMonitor.get().createStatus(
-        "Handling alter table request for table = " + tableNameStr);
-    if (canPerformSchemaChange()) {
-      boolean prevBalanceSwitch = false;
-      try {
-        // turn off load balancer synchronously
-        prevBalanceSwitch = master.synchronousBalanceSwitch(false);
-        waitForInflightSplit(regions, status);
-        MasterSchemaChangeTracker masterSchemaChangeTracker =
-          this.masterServices.getSchemaChangeTracker();
-        masterSchemaChangeTracker
-        .createSchemaChangeNode(Bytes.toString(tableName),
-            regions.size());
-        while(!masterSchemaChangeTracker.doesSchemaChangeNodeExists(
-            Bytes.toString(tableName))) {
-          try {
-            Thread.sleep(50);
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-          }
-        }
-        status.markComplete("Created ZK node for handling the alter table request for table = "
-            + tableNameStr);
-      } catch (KeeperException e) {
-        LOG.warn("Instant schema change failed for table " + tableNameStr, e);
-        status.setStatus("Instant schema change failed for table " + tableNameStr
-            + " Cause = " + e.getCause());
-
-      } catch (IOException ioe) {
-        LOG.warn("Instant schema change failed for table " + tableNameStr, ioe);
-        status.setStatus("Instant schema change failed for table " + tableNameStr
-            + " Cause = " + ioe.getCause());
-      } finally {
-        master.synchronousBalanceSwitch(prevBalanceSwitch);
-      }
-    }
-  }
-
   /**
    * @return Table descriptor for this table
    * @throws TableExistsException
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
index c6e292e0bfcc..34560ec674ce 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -39,10 +38,8 @@ public class TableModifyFamilyHandler extends TableEventHandler {
 
   public TableModifyFamilyHandler(byte[] tableName,
       HColumnDescriptor familyDesc, Server server,
-      final MasterServices masterServices,
-      HMasterInterface masterInterface, boolean instantChange) throws IOException {
-    super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices,
-        masterInterface, instantChange);
+      final MasterServices masterServices) throws IOException {
+    super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices);
     HTableDescriptor htd = getTableDescriptor();
     hasColumnFamily(htd, familyDesc.getName());
     this.familyDesc = familyDesc;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 066ae5283485..26e72528c8e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -155,29 +155,12 @@ public synchronized boolean requestSplit(final HRegion r) {
     return false;
   }
 
-  /**
-   * Wait for mid-flight schema alter requests. (if any). We don't want to execute a split
-   * when a schema alter is in progress as we end up in an inconsistent state.
-   * @param tableName
-   */
-  private void waitForInflightSchemaChange(String tableName) {
-    while (this.server.getSchemaChangeTracker()
-        .isSchemaChangeInProgress(tableName)) {
-      try {
-        Thread.sleep(100);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
-    }
-  }
-
   public synchronized void requestSplit(final HRegion r, byte[] midKey) {
     if (midKey == null) {
       LOG.debug("Region " + r.getRegionNameAsString() +
         " not splittable because midkey=null");
       return;
     }
-    waitForInflightSchemaChange(r.getRegionInfo().getTableNameAsString());
     try {
       this.splits.execute(new SplitRequest(r, midKey, this.server));
       if (LOG.isDebugEnabled()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 37bab7540581..bd8fb90c81c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -147,7 +147,6 @@
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
-import org.apache.hadoop.hbase.zookeeper.SchemaChangeTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -292,9 +291,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   // Cluster Status Tracker
   private ClusterStatusTracker clusterStatusTracker;
 
-  // Schema change Tracker
-  private SchemaChangeTracker schemaChangeTracker;
-
   // Log Splitting Worker
   private SplitLogWorker splitLogWorker;
 
@@ -597,11 +593,6 @@ private void initializeZooKeeper() throws IOException, InterruptedException {
     this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,
       this, this.conf.getInt("hbase.regionserver.catalog.timeout", Integer.MAX_VALUE));
     catalogTracker.start();
-
-    // Schema change tracker
-    this.schemaChangeTracker = new SchemaChangeTracker(this.zooKeeper,
-        this, this);
-    this.schemaChangeTracker.start();
   }
 
   /**
@@ -2899,26 +2890,9 @@ public void splitRegion(HRegionInfo regionInfo)
     splitRegion(regionInfo, null);
   }
 
-  /**
-   * Wait for mid-flight schema change requests. (if any)
-   * @param tableName
-   */
-  private void waitForSchemaChange(String tableName) {
-    while (schemaChangeTracker.isSchemaChangeInProgress(tableName)) {
-      try {
-        LOG.debug("Schema alter is inprogress for table = " + tableName
-            + " Waiting for alter to complete before a split");
-        Thread.sleep(100);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
-    }
-  }
-
   @Override
   public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
       throws NotServingRegionException, IOException {
-    waitForSchemaChange(Bytes.toString(regionInfo.getTableName()));
     checkOpen();
     HRegion region = getRegion(regionInfo.getRegionName());
     region.flushcache();
@@ -3670,58 +3644,27 @@ public byte[][] rollHLogWriter() throws IOException, FailedLogCloseException {
   }
 
   /**
-  * Refresh schema changes for given region.
-  * @param hRegion HRegion to refresh
-  * @throws IOException
-  */
- public void refreshRegion(HRegion hRegion) throws IOException {
-
-   if (hRegion != null) {
+   * Gets the online regions of the specified table.
+   * This method looks at the in-memory onlineRegions.  It does not go to <code>.META.</code>.
+   * Only returns <em>online</em> regions.  If a region on this table has been
+   * closed during a disable, etc., it will not be included in the returned list.
+   * So, the returned list may not necessarily be ALL regions in this table, its
+   * all the ONLINE regions in the table.
+   * @param tableName
+   * @return Online regions from <code>tableName</code>
+   */
+   public List<HRegion> getOnlineRegions(byte[] tableName) {
+     List<HRegion> tableRegions = new ArrayList<HRegion>();
      synchronized (this.onlineRegions) {
-       HRegionInfo regionInfo = hRegion.getRegionInfo();
-       // Close the region
-       hRegion.close();
-       // Remove from online regions
-       removeFromOnlineRegions(regionInfo.getEncodedName());
-       // Get new HTD
-       HTableDescriptor htd = this.tableDescriptors.get(regionInfo.getTableName());
-       LOG.debug("HTD for region = " + regionInfo.getRegionNameAsString()
-           + " Is = " + htd );
-       HRegion region =
-         HRegion.openHRegion(hRegion.getRegionInfo(), htd, hlog, conf,
-             this, null);
-       // Add new region to the onlineRegions
-       addToOnlineRegions(region);
+       for (HRegion region: this.onlineRegions.values()) {
+         HRegionInfo regionInfo = region.getRegionInfo();
+         if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+           tableRegions.add(region);
+         }
+       }
      }
+     return tableRegions;
    }
- }
-
- /**
-  * Gets the online regions of the specified table.
-  * This method looks at the in-memory onlineRegions.  It does not go to <code>.META.</code>.
-  * Only returns <em>online</em> regions.  If a region on this table has been
-  * closed during a disable, etc., it will not be included in the returned list.
-  * So, the returned list may not necessarily be ALL regions in this table, its
-  * all the ONLINE regions in the table.
-  * @param tableName
-  * @return Online regions from <code>tableName</code>
-  */
-  public List<HRegion> getOnlineRegions(byte[] tableName) {
-    List<HRegion> tableRegions = new ArrayList<HRegion>();
-    synchronized (this.onlineRegions) {
-      for (HRegion region: this.onlineRegions.values()) {
-        HRegionInfo regionInfo = region.getRegionInfo();
-        if(Bytes.equals(regionInfo.getTableName(), tableName)) {
-          tableRegions.add(region);
-        }
-      }
-    }
-    return tableRegions;
-  }
-
-  public SchemaChangeTracker getSchemaChangeTracker() {
-    return this.schemaChangeTracker;
-  }
 
   // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
   public String[] getCoprocessors() {
@@ -3739,5 +3682,4 @@ void registerMBean() {
         mxBeanInfo);
     LOG.info("Registered RegionServer MXBean");
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
index e587380af422..847efb68d3f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
@@ -52,18 +52,12 @@ interface OnlineRegions extends Server {
    * null if named region is not member of the online regions.
    */
   public HRegion getFromOnlineRegions(String encodedRegionName);
-  /**
-   * Get all online regions of a table in this RS.
-   * @param tableName
-   * @return List of HRegion
-   * @throws java.io.IOException
-   */
-  public List<HRegion> getOnlineRegions(byte[] tableName) throws IOException;
-
-  /**
-   * Refresh a given region updating it with latest HTD info.
-   * @param hRegion
-   */
-  public void refreshRegion(HRegion hRegion) throws IOException;
 
-}
+   /**
+    * Get all online regions of a table in this RS.
+    * @param tableName
+    * @return List of HRegion
+    * @throws java.io.IOException
+    */
+   public List<HRegion> getOnlineRegions(byte[] tableName) throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
deleted file mode 100644
index 39c940e45c01..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
+++ /dev/null
@@ -1,826 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.zookeeper;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.io.Writable;
-import org.apache.zookeeper.KeeperException;
-
-public class MasterSchemaChangeTracker extends ZooKeeperNodeTracker {
-  public static final Log LOG = LogFactory.getLog(MasterSchemaChangeTracker.class);
-  private final MasterServices masterServices;
-  // Used by tests only. Do not change this.
-  private volatile int sleepTimeMillis = 0;
-  // schema changes pending more than this time will be timed out.
-  private long schemaChangeTimeoutMillis = 30000;
-
-  /**
-   * Constructs a new ZK node tracker.
-   * <p/>
-   * <p>After construction, use {@link #start} to kick off tracking.
-   *
-   * @param watcher
-   * @param abortable
-   */
-  public MasterSchemaChangeTracker(ZooKeeperWatcher watcher,
-                                   Abortable abortable, MasterServices masterServices,
-                                   long schemaChangeTimeoutMillis) {
-    super(watcher, watcher.schemaZNode, abortable);
-    this.masterServices = masterServices;
-    this.schemaChangeTimeoutMillis = schemaChangeTimeoutMillis;
-  }
-
-  @Override
-  public void start() {
-    try {
-      watcher.registerListener(this);
-      List<String> tables =
-          ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
-      processCompletedSchemaChanges(tables);
-    } catch (KeeperException e) {
-      LOG.error("MasterSchemaChangeTracker startup failed.", e);
-      abortable.abort("MasterSchemaChangeTracker startup failed", e);
-    }
-  }
-
-  private List<String> getCurrentTables() throws KeeperException {
-    return
-        ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
-  }
-
-  /**
-   * When a primary master crashes and the secondary master takes over
-   * mid-flight during an alter process, the secondary should cleanup any completed
-   * schema changes not handled by the previous master.
-   * @param tables
-   * @throws KeeperException
-   */
-  private void processCompletedSchemaChanges(List<String> tables)
-      throws KeeperException {
-    if (tables == null || tables.isEmpty()) {
-      String msg = "No current schema change in progress. Skipping cleanup";
-      LOG.debug(msg);
-      return;
-    }
-    String msg = "Master seeing following tables undergoing schema change " +
-        "process. Tables = " + tables;
-    MonitoredTask status = TaskMonitor.get().createStatus(msg);
-    LOG.debug(msg);
-    for (String table : tables) {
-      LOG.debug("Processing table = "+ table);
-      status.setStatus("Processing table = "+ table);
-      try {
-        processTableNode(table);
-      } catch (IOException e) {
-        String errmsg = "IOException while processing completed schema changes."
-            + " Cause = " + e.getCause();
-        LOG.error(errmsg, e);
-        status.setStatus(errmsg);
-      }
-    }
-  }
-
-  /**
-   * Get current alter status for a table.
-   * @param tableName
-   * @return MasterAlterStatus
-   * @throws KeeperException
-   * @throws IOException
-   */
-  public MasterAlterStatus getMasterAlterStatus(String tableName)
-      throws KeeperException, IOException {
-    String path = getSchemaChangeNodePathForTable(tableName);
-    byte[] state = ZKUtil.getData(watcher, path);
-    if (state == null || state.length <= 0) {
-      return null;
-    }
-    MasterAlterStatus mas = new MasterAlterStatus();
-    Writables.getWritable(state, mas);
-    return mas;
-  }
-
-  /**
-   * Get RS specific alter status for a table & server
-   * @param tableName
-   * @param serverName
-   * @return Region Server's Schema alter status
-   * @throws KeeperException
-   * @throws IOException
-   */
-  private SchemaChangeTracker.SchemaAlterStatus getRSSchemaAlterStatus(
-      String tableName, String serverName)
-      throws KeeperException, IOException {
-    String childPath =
-        getSchemaChangeNodePathForTableAndServer(tableName, serverName);
-    byte[] childData = ZKUtil.getData(this.watcher, childPath);
-    if (childData == null || childData.length <= 0) {
-      return null;
-    }
-    SchemaChangeTracker.SchemaAlterStatus sas =
-        new SchemaChangeTracker.SchemaAlterStatus();
-    Writables.getWritable(childData, sas);
-    LOG.debug("Schema Status data for server = " + serverName + " table = "
-        + tableName + " == " + sas);
-    return sas;
-  }
-
-  /**
-   * Update the master's alter status based on all region server's response.
-   * @param servers
-   * @param tableName
-   * @throws IOException
-   */
-  private void updateMasterAlterStatus(MasterAlterStatus mas,
-                                       List<String> servers, String tableName)
-      throws IOException, KeeperException {
-    for (String serverName : servers) {
-      SchemaChangeTracker.SchemaAlterStatus sas =
-          getRSSchemaAlterStatus(tableName, serverName);
-      if (sas != null) {
-        mas.update(sas);
-        LOG.debug("processTableNodeWithState:Updated Master Alter Status = "
-            + mas + " for server = " + serverName);
-      } else {
-        LOG.debug("SchemaAlterStatus is NULL for table = " + tableName);
-      }
-    }
-  }
-
-  /**
-   * If schema alter is handled for this table, then delete all the ZK nodes
-   * created for this table.
-   * @param tableName
-   * @throws KeeperException
-   */
-  private void processTableNode(String tableName) throws KeeperException,
-      IOException {
-    LOG.debug("processTableNodeWithState. TableName = " + tableName);
-    List<String> servers =
-      ZKUtil.listChildrenAndWatchThem(watcher,
-          getSchemaChangeNodePathForTable(tableName));
-    MasterAlterStatus mas = getMasterAlterStatus(tableName);
-    if (mas == null) {
-      LOG.debug("MasterAlterStatus is NULL. Table = " + tableName);
-      return;
-    }
-    updateMasterAlterStatus(mas, servers, tableName);
-    LOG.debug("Current Alter status = " + mas);
-    String nodePath = getSchemaChangeNodePathForTable(tableName);
-    ZKUtil.updateExistingNodeData(this.watcher, nodePath,
-        Writables.getBytes(mas), getZKNodeVersion(nodePath));
-    processAlterStatus(mas, tableName, servers);
-  }
-
-  /**
-   * Evaluate the master alter status and determine the current status.
-   * @param alterStatus
-   * @param tableName
-   * @param servers
-   * @param status
-   */
-  private void processAlterStatus(MasterAlterStatus alterStatus,
-                                  String tableName, List<String> servers)
-  throws KeeperException {
-    if (alterStatus.getNumberOfRegionsToProcess()
-        == alterStatus.getNumberOfRegionsProcessed()) {
-      // schema change completed.
-      String msg = "All region servers have successfully processed the " +
-          "schema changes for table = " + tableName
-          + " . Deleting the schema change node for table = "
-          + tableName + " Region servers processed the schema change" +
-          " request = " + alterStatus.getProcessedHosts()
-          + " Total number of regions = " + alterStatus.getNumberOfRegionsToProcess()
-          + " Processed regions = " + alterStatus.getNumberOfRegionsProcessed();
-      MonitoredTask status = TaskMonitor.get().createStatus(
-          "Checking alter schema request status for table = " + tableName);
-      status.markComplete(msg);
-      LOG.debug(msg);
-      cleanProcessedTableNode(getSchemaChangeNodePathForTable(tableName));
-    } else {
-      if (alterStatus.getErrorCause() != null
-          && alterStatus.getErrorCause().trim().length() > 0) {
-        String msg = "Alter schema change failed "
-            + "for table = " + tableName + " Number of online regions = "
-            + alterStatus.getNumberOfRegionsToProcess() + " processed regions count = "
-            + alterStatus.getNumberOfRegionsProcessed()
-            + " Original list = " + alterStatus.hostsToProcess + " Processed servers = "
-            + servers
-            + " Error Cause = " + alterStatus.getErrorCause();
-        MonitoredTask status = TaskMonitor.get().createStatus(
-            "Checking alter schema request status for table = " + tableName);
-        // we have errors.
-        LOG.debug(msg);
-        status.abort(msg);
-      } else {
-        String msg = "Not all region servers have processed the schema changes"
-            + "for table = " + tableName + " Number of online regions = "
-            + alterStatus.getNumberOfRegionsToProcess() + " processed regions count = "
-            + alterStatus.getNumberOfRegionsProcessed()
-            + " Original list = " + alterStatus.hostsToProcess + " Processed servers = "
-            + servers + " Alter STate = "
-            + alterStatus.getCurrentAlterStatus();
-        LOG.debug(msg);
-        // status.setStatus(msg);
-      }
-    }
-  }
-
-  /**
-   * Check whether a in-flight schema change request has expired.
-   * @param tableName
-   * @return true is the schema change request expired.
-   * @throws IOException
-   */
-  private boolean hasSchemaChangeExpiredFor(String tableName)
-      throws IOException, KeeperException {
-    MasterAlterStatus mas = getMasterAlterStatus(tableName);
-    long createdTimeStamp = mas.getStamp();
-    long duration = System.currentTimeMillis() - createdTimeStamp;
-    LOG.debug("Created TimeStamp = " + createdTimeStamp
-        + " duration = " + duration + " Table = " + tableName
-        + " Master Alter Status = " + mas);
-    return (duration > schemaChangeTimeoutMillis);
-  }
-
-  /**
-   * Handle failed and expired schema changes. We simply delete all the
-   * expired/failed schema change attempts. Why we should do this ?
-   * 1) Keeping the failed/expired schema change nodes longer prohibits any
-   *    future schema changes for the table.
-   * 2) Any lingering expired/failed schema change requests will prohibit the
-   *    load balancer from running.
-   */
-  public void handleFailedOrExpiredSchemaChanges() {
-    try {
-      List<String> tables = getCurrentTables();
-      for (String table : tables) {
-        String statmsg = "Cleaning failed or expired schema change requests. " +
-            "current tables undergoing " +
-            "schema change process = " + tables;
-         MonitoredTask status = TaskMonitor.get().createStatus(statmsg);
-        LOG.debug(statmsg);
-        if (hasSchemaChangeExpiredFor(table)) {
-          // time out.. currently, we abandon the in-flight schema change due to
-          // time out.
-          // Here, there are couple of options to consider. One could be to
-          // attempt a retry of the schema change and see if it succeeds, and
-          // another could be to simply rollback the schema change effort and
-          // see if it succeeds.
-          String msg = "Schema change for table = " + table + " has expired."
-              + " Schema change for this table has been in progress for " +
-              + schemaChangeTimeoutMillis +
-              "Deleting the node now.";
-          LOG.debug(msg);
-          ZKUtil.deleteNodeRecursively(this.watcher,
-              getSchemaChangeNodePathForTable(table));
-        } else {
-          String msg = "Schema change request is in progress for " +
-              " table = " + table;
-          LOG.debug(msg);
-          status.setStatus(msg);
-        }
-      }
-    } catch (IOException e) {
-      String msg = "IOException during handleFailedExpiredSchemaChanges."
-            + e.getCause();
-      LOG.error(msg, e);
-      TaskMonitor.get().createStatus(msg);
-    } catch (KeeperException ke) {
-      String msg = "KeeperException during handleFailedExpiredSchemaChanges."
-          + ke.getCause();
-      LOG.error(msg, ke);
-      TaskMonitor.get().createStatus(msg);
-    }
-  }
-
-  /**
-   * Clean the nodes of completed schema change table.
-   * @param path
-   * @throws KeeperException
-   */
-  private void cleanProcessedTableNode(String path) throws KeeperException {
-    if (sleepTimeMillis > 0) {
-      try {
-        LOG.debug("Master schema change tracker sleeping for "
-            + sleepTimeMillis);
-        Thread.sleep(sleepTimeMillis);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
-    }
-    ZKUtil.deleteNodeRecursively(this.watcher, path);
-    LOG.debug("Deleted all nodes for path " + path);
-
-  }
-
-  /**
-   * Exclude a RS from schema change request (if applicable)
-   * We will exclude a RS from schema change request processing if 1) RS
-   * has online regions for the table AND 2) RS went down mid-flight
-   * during schema change process. We don't have to deal with RS going
-   * down mid-flight during a schema change as the online regions from
-   * the dead RS will get reassigned to some other RS and the
-   * process of reassign inherently takes care of the schema change as well.
-   * @param serverName
-   */
-  public void excludeRegionServerForSchemaChanges(String serverName) {
-    try {
-      MonitoredTask status = TaskMonitor.get().createStatus(
-        "Processing schema change exclusion for region server = " + serverName);
-      List<String> tables =
-            ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
-      if (tables == null || tables.isEmpty()) {
-        String msg = "No schema change in progress. Skipping exclusion for " +
-            "server = "+ serverName;
-        LOG.debug(msg);
-        status.setStatus(msg);
-        return ;
-      }
-      for (String tableName : tables) {
-        excludeRegionServer(tableName, serverName, status);
-      }
-    } catch(KeeperException ke) {
-      LOG.error("KeeperException during excludeRegionServerForSchemaChanges", ke);
-    } catch(IOException ioe) {
-      LOG.error("IOException during excludeRegionServerForSchemaChanges", ioe);
-
-    }
-  }
-
-  /**
-   * Check whether a schema change is in progress for a given table on a
-   * given RS.
-   * @param tableName
-   * @param serverName
-   * @return TRUE is this RS is currently processing a schema change request
-   * for the table.
-   * @throws KeeperException
-   */
-  private boolean isSchemaChangeApplicableFor(String tableName,
-                                              String serverName)
-      throws KeeperException {
-    List<String> servers = ZKUtil.listChildrenAndWatchThem(watcher,
-        getSchemaChangeNodePathForTable(tableName));
-    return (servers.contains(serverName));
-  }
-
-  /**
-   * Exclude a region server for a table (if applicable) from schema change processing.
-   * @param tableName
-   * @param serverName
-   * @param status
-   * @throws KeeperException
-   * @throws IOException
-   */
-  private void excludeRegionServer(String tableName, String serverName,
-                                   MonitoredTask status)
-      throws KeeperException, IOException {
-    if (isSchemaChangeApplicableFor(tableName, serverName)) {
-      String msg = "Excluding RS " + serverName + " from schema change process" +
-          " for table = " + tableName;
-      LOG.debug(msg);
-      status.setStatus(msg);
-      SchemaChangeTracker.SchemaAlterStatus sas =
-          getRSSchemaAlterStatus(tableName, serverName);
-      if (sas == null) {
-        LOG.debug("SchemaAlterStatus is NULL for table = " + tableName
-            + " server = " + serverName);
-        return;
-      }
-      // Set the status to IGNORED so we can process it accordingly.
-      sas.setCurrentAlterStatus(
-          SchemaChangeTracker.SchemaAlterStatus.AlterState.IGNORED);
-      LOG.debug("Updating the current schema status to " + sas);
-      String nodePath = getSchemaChangeNodePathForTableAndServer(tableName,
-              serverName);
-      ZKUtil.updateExistingNodeData(this.watcher,
-          nodePath, Writables.getBytes(sas), getZKNodeVersion(nodePath));
-    } else {
-      LOG.debug("Skipping exclusion of RS " + serverName
-          + " from schema change process"
-          + " for table = " + tableName
-          + " as it did not possess any online regions for the table");
-    }
-    processTableNode(tableName);
-  }
-
-  private int getZKNodeVersion(String nodePath) throws KeeperException {
-    return ZKUtil.checkExists(this.watcher, nodePath);
-  }
-
-  /**
-   * Create a new schema change ZK node.
-   * @param tableName Table name that is getting altered
-   * @throws KeeperException
-   */
-  public void createSchemaChangeNode(String tableName,
-                                     int numberOfRegions)
-      throws KeeperException, IOException {
-    MonitoredTask status = TaskMonitor.get().createStatus(
-        "Creating schema change node for table = " + tableName);
-    LOG.debug("Creating schema change node for table = "
-        + tableName + " Path = "
-        + getSchemaChangeNodePathForTable(tableName));
-    if (doesSchemaChangeNodeExists(tableName)) {
-      LOG.debug("Schema change node already exists for table = " + tableName
-          + " Deleting the schema change node.");
-      // If we already see a schema change node for this table we wait till the previous
-      // alter process is complete. Ideally, we need not wait and we could simply delete
-      // existing schema change node for this table and create new one. But then the
-      // RS cloud will not be able to process concurrent schema updates for the same table
-      // as they will be working with same set of online regions for this table. Meaning the
-      // second alter change will not see any online regions (as they were being closed and
-      // re opened by the first change) and will miss the second one.
-      // We either handle this at the RS level using explicit locks while processing a table
-      // or do it here. I prefer doing it here as it seems much simpler and cleaner.
-      while(doesSchemaChangeNodeExists(tableName)) {
-        try {
-          Thread.sleep(50);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-        }
-      }
-    }
-
-    int rsCount = ZKUtil.getNumberOfChildren(this.watcher, watcher.rsZNode);
-    // if number of online RS = 0, we should not do anything!
-    if (rsCount <= 0) {
-      String msg = "Master is not seeing any online region servers. Aborting the " +
-          "schema change processing by region servers.";
-      LOG.debug(msg);
-      status.abort(msg);
-    } else {
-      LOG.debug("Master is seeing " + rsCount + " region servers online before " +
-          "the schema change process.");
-      MasterAlterStatus mas = new MasterAlterStatus(numberOfRegions,
-          getActiveRegionServersAsString());
-      LOG.debug("Master creating the master alter status = " + mas);
-      ZKUtil.createSetData(this.watcher,
-          getSchemaChangeNodePathForTable(tableName), Writables.getBytes(mas));
-      status.markComplete("Created the ZK node for schema change. Current Alter Status = "
-          + mas.toString());
-      ZKUtil.listChildrenAndWatchThem(this.watcher,
-          getSchemaChangeNodePathForTable(tableName));
-    }
-  }
-
-  private String getActiveRegionServersAsString() {
-    StringBuffer sbuf = new StringBuffer();
-    List<ServerName> currentRS =
-        masterServices.getRegionServerTracker().getOnlineServers();
-    for (ServerName serverName : currentRS) {
-      sbuf.append(serverName.getServerName());
-      sbuf.append(" ");
-    }
-    LOG.debug("Current list of RS to process the schema change = "
-        + sbuf.toString());
-    return sbuf.toString();
-  }
-
-  /**
-   * Create a new schema change ZK node.
-   * @param tableName
-   * @throws KeeperException
-   */
-  public boolean doesSchemaChangeNodeExists(String tableName)
-      throws KeeperException {
-    return ZKUtil.checkExists(watcher,
-            getSchemaChangeNodePathForTable(tableName)) != -1;
-  }
-
-  /**
-   * Check whether there are any schema change requests that are in progress now.
-   * We simply assume that a schema change is in progress if we see a ZK schema node for
-   * any table. We may revisit for fine grained checks such as check the current alter status
-   * et al, but it is not required now.
-   * @return
-   */
-  public boolean isSchemaChangeInProgress() {
-    try {
-      int schemaChangeCount = ZKUtil.getNumberOfChildren(this.watcher, watcher.schemaZNode);
-      return schemaChangeCount > 0;
-    } catch (KeeperException ke) {
-      LOG.debug("KeeperException while getting current schema change progress.");
-      // What do we do now??? currently reporting as false.
-    }
-    return false;
-  }
-
-  /**
-   * We get notified when a RS processes/or completed the schema change request.
-   * The path will be of the format /hbase/schema/<table name>
-   * @param path full path of the node whose children have changed
-   */
-  @Override
-  public void nodeChildrenChanged(String path) {
-    String tableName = null;
-    if (path.startsWith(watcher.schemaZNode) &&
-        !path.equals(watcher.schemaZNode)) {
-      try {
-        LOG.debug("NodeChildrenChanged Path = " + path);
-        tableName = path.substring(path.lastIndexOf("/")+1, path.length());
-        processTableNode(tableName);
-      } catch (KeeperException e) {
-        TaskMonitor.get().createStatus(
-        "MasterSchemaChangeTracker: ZK exception while processing " +
-            " nodeChildrenChanged() event for table = " + tableName
-            + " Cause = " + e.getCause());
-        LOG.error("MasterSchemaChangeTracker: Unexpected zk exception getting"
-            + " schema change nodes", e);
-      } catch(IOException ioe) {
-        TaskMonitor.get().createStatus(
-        "MasterSchemaChangeTracker: ZK exception while processing " +
-            " nodeChildrenChanged() event for table = " + tableName
-            + " Cause = " + ioe.getCause());
-        LOG.error("MasterSchemaChangeTracker: Unexpected IO exception getting"
-            + " schema change nodes", ioe);
-      }
-    }
-  }
-
-  /**
-   * We get notified as and when the RS cloud updates their ZK nodes with
-   * progress information. The path will be of the format
-   * /hbase/schema/<table name>/<RS host name>
-   * @param path
-   */
-  @Override
-  public void nodeDataChanged(String path) {
-    String tableName = null;
-    if (path.startsWith(watcher.schemaZNode) &&
-        !path.equals(watcher.schemaZNode)) {
-      try {
-        LOG.debug("NodeDataChanged Path = " + path);
-        String[] paths = path.split("/");
-        tableName = paths[3];
-        processTableNode(tableName);
-      } catch (KeeperException e) {
-        TaskMonitor.get().createStatus(
-        "MasterSchemaChangeTracker: ZK exception while processing " +
-            " nodeDataChanged() event for table = " + tableName
-            + " Cause = " + e.getCause());
-        LOG.error("MasterSchemaChangeTracker: Unexpected zk exception getting"
-            + " schema change nodes", e);
-      } catch(IOException ioe) {
-        TaskMonitor.get().createStatus(
-        "MasterSchemaChangeTracker: IO exception while processing " +
-            " nodeDataChanged() event for table = " + tableName
-            + " Cause = " + ioe.getCause());
-        LOG.error("MasterSchemaChangeTracker: Unexpected IO exception getting"
-            + " schema change nodes", ioe);
-
-      }
-    }
-  }
-
-  public String getSchemaChangeNodePathForTable(String tableName) {
-    return ZKUtil.joinZNode(watcher.schemaZNode, tableName);
-  }
-
-  /**
-   * Used only for tests. Do not use this. See TestInstantSchemaChange for more details
-   * on how this is getting used. This is primarily used to delay the schema complete
-   * processing by master so that we can test some complex scenarios such as
-   * master failover.
-   * @param sleepTimeMillis
-   */
-  public void setSleepTimeMillis(int sleepTimeMillis) {
-    this.sleepTimeMillis = sleepTimeMillis;
-  }
-
-  private String getSchemaChangeNodePathForTableAndServer(
-      String tableName, String regionServerName) {
-    return ZKUtil.joinZNode(getSchemaChangeNodePathForTable(tableName),
-        regionServerName);
-  }
-
-
-  /**
-   * Holds the current alter state for a table. Alter state includes the
-   * current alter status (INPROCESS, FAILURE or SUCCESS (success is not getting
-   * used now.), timestamp of alter request, number of hosts online at the time
-   * of alter request, number of online regions to process for the schema change
-   * request, number of processed regions and a list of region servers that
-   * actually processed the schema change request.
-   *
-   * Master keeps track of schema change requests using the alter status and
-   * periodically updates the alter status based on RS cloud processings.
-   */
-  public static class MasterAlterStatus implements Writable {
-
-    public enum AlterState {
-      INPROCESS,        // Inprocess alter
-      SUCCESS,          // completed alter
-      FAILURE           // failure alter
-    }
-
-    private AlterState currentAlterStatus;
-    // TimeStamp
-    private long stamp;
-    private int numberOfRegionsToProcess;
-    private StringBuffer errorCause = new StringBuffer(" ");
-    private StringBuffer processedHosts = new StringBuffer(" ");
-    private String hostsToProcess;
-    private int numberOfRegionsProcessed = 0;
-
-    public MasterAlterStatus() {
-
-    }
-
-    public MasterAlterStatus(int numberOfRegions, String activeHosts) {
-      this.numberOfRegionsToProcess = numberOfRegions;
-      this.stamp = System.currentTimeMillis();
-      this.currentAlterStatus = AlterState.INPROCESS;
-      //this.rsToProcess = activeHosts;
-      this.hostsToProcess = activeHosts;
-    }
-
-    public AlterState getCurrentAlterStatus() {
-      return currentAlterStatus;
-    }
-
-    public void setCurrentAlterStatus(AlterState currentAlterStatus) {
-      this.currentAlterStatus = currentAlterStatus;
-    }
-
-    public long getStamp() {
-      return stamp;
-    }
-
-    public void setStamp(long stamp) {
-      this.stamp = stamp;
-    }
-
-    public int getNumberOfRegionsToProcess() {
-      return numberOfRegionsToProcess;
-    }
-
-    public void setNumberOfRegionsToProcess(int numberOfRegionsToProcess) {
-      this.numberOfRegionsToProcess = numberOfRegionsToProcess;
-    }
-
-    public int getNumberOfRegionsProcessed() {
-      return numberOfRegionsProcessed;
-    }
-
-    public void setNumberOfRegionsProcessed(int numberOfRegionsProcessed) {
-      this.numberOfRegionsProcessed += numberOfRegionsProcessed;
-    }
-
-    public String getHostsToProcess() {
-      return hostsToProcess;
-    }
-
-    public void setHostsToProcess(String hostsToProcess) {
-      this.hostsToProcess = hostsToProcess;
-    }
-
-    public String getErrorCause() {
-      return errorCause == null ? null : errorCause.toString();
-    }
-
-    public void setErrorCause(String errorCause) {
-      if (errorCause == null || errorCause.trim().length() <= 0) {
-        return;
-      }
-      if (this.errorCause == null) {
-        this.errorCause = new StringBuffer(errorCause);
-      } else {
-        this.errorCause.append(errorCause);
-      }
-    }
-
-    public String getProcessedHosts() {
-      return processedHosts.toString();
-    }
-
-    public void setProcessedHosts(String processedHosts) {
-      if (this.processedHosts == null) {
-        this.processedHosts = new StringBuffer(processedHosts);
-      } else {
-        this.processedHosts.append(" ").append(processedHosts);
-      }
-    }
-
-    /**
-     * Ignore or exempt a RS from schema change processing.
-     * Master will tweak the number of regions to process based on the
-     * number of online regions on the target RS and also remove the
-     * RS from list of hosts to process.
-     * @param schemaAlterStatus
-     */
-    private void ignoreRSForSchemaChange(
-        SchemaChangeTracker.SchemaAlterStatus schemaAlterStatus) {
-      LOG.debug("Removing RS " + schemaAlterStatus.getHostName()
-          + " from schema change process.");
-      hostsToProcess =
-          hostsToProcess.replaceAll(schemaAlterStatus.getHostName(), "");
-      int ignoreRegionsCount = schemaAlterStatus.getNumberOfOnlineRegions();
-      LOG.debug("Current number of regions processed = "
-          + this.numberOfRegionsProcessed + " deducting ignored = "
-          + ignoreRegionsCount
-          + " final = " + (this.numberOfRegionsToProcess-ignoreRegionsCount));
-      if (this.numberOfRegionsToProcess > 0) {
-        this.numberOfRegionsToProcess -= ignoreRegionsCount;
-      } else {
-        LOG.debug("Number of regions to process is less than zero. This is odd");
-      }
-    }
-
-    /**
-     * Update the master alter status for this table based on RS alter status.
-     * @param schemaAlterStatus
-     */
-    public void update(SchemaChangeTracker.SchemaAlterStatus schemaAlterStatus) {
-      this.setProcessedHosts(schemaAlterStatus.getHostName());
-      SchemaChangeTracker.SchemaAlterStatus.AlterState rsState =
-          schemaAlterStatus.getCurrentAlterStatus();
-      switch(rsState) {
-        case FAILURE:
-          LOG.debug("Schema update failure Status = "
-            + schemaAlterStatus);
-          this.setCurrentAlterStatus(
-              MasterAlterStatus.AlterState.FAILURE);
-          this.setNumberOfRegionsProcessed(
-              schemaAlterStatus.getNumberOfRegionsProcessed());
-          this.setErrorCause(schemaAlterStatus.getErrorCause());
-          break;
-        case SUCCESS:
-          LOG.debug("Schema update SUCCESS Status = "
-            + schemaAlterStatus);
-          this.setNumberOfRegionsProcessed(
-              schemaAlterStatus.getNumberOfRegionsProcessed());
-          this.setCurrentAlterStatus(MasterAlterStatus.AlterState.SUCCESS);
-          break;
-        case IGNORED:
-          LOG.debug("Schema update IGNORED Updating regions to " +
-              "process count. Status = "+ schemaAlterStatus);
-          ignoreRSForSchemaChange(schemaAlterStatus);
-          break;
-        default:
-            break;
-      }
-    }
-
-    @Override
-    public void readFields(DataInput in) throws IOException {
-      currentAlterStatus = AlterState.valueOf(in.readUTF());
-      stamp = in.readLong();
-      numberOfRegionsToProcess = in.readInt();
-      hostsToProcess = Bytes.toString(Bytes.readByteArray(in));
-      processedHosts = new StringBuffer(Bytes.toString(Bytes.readByteArray(in)));
-      errorCause = new StringBuffer(Bytes.toString(Bytes.readByteArray(in)));
-    }
-
-    @Override
-    public void write(DataOutput out) throws IOException {
-      out.writeUTF(currentAlterStatus.name());
-      out.writeLong(stamp);
-      out.writeInt(numberOfRegionsToProcess);
-      Bytes.writeByteArray(out, Bytes.toBytes(hostsToProcess));
-      Bytes.writeByteArray(out, Bytes.toBytes(processedHosts.toString()));
-      Bytes.writeByteArray(out, Bytes.toBytes(errorCause.toString()));
-    }
-
-    @Override
-    public String toString() {
-      return
-         " state= " + currentAlterStatus
-        + ", ts= " + stamp
-        + ", number of regions to process = " + numberOfRegionsToProcess
-        + ", number of regions processed = " + numberOfRegionsProcessed
-        + ", hosts = " + hostsToProcess
-        + " , processed hosts = " + processedHosts
-        + " , errorCause = " + errorCause;
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/SchemaChangeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/SchemaChangeTracker.java
deleted file mode 100644
index 9233eee223b9..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/SchemaChangeTracker.java
+++ /dev/null
@@ -1,476 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.zookeeper;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.zookeeper.KeeperException;
-import org.apache.hadoop.hbase.util.Writables;
-
-import org.apache.hadoop.io.Writable;
-
-import java.io.*;
-import java.util.List;
-
-/**
- * Region server schema change tracker. RS uses this tracker to keep track of
- * alter schema requests from master and updates the status once the schema change
- * is complete.
- */
-public class SchemaChangeTracker extends ZooKeeperNodeTracker {
-  public static final Log LOG = LogFactory.getLog(SchemaChangeTracker.class);
-  private RegionServerServices regionServer = null;
-  private volatile int sleepTimeMillis = 0;
-
-
-  /**
-   * Constructs a new ZK node tracker.
-   * <p/>
-   * <p>After construction, use {@link #start} to kick off tracking.
-   *
-   * @param watcher
-   * @param node
-   * @param abortable
-   */
-  public SchemaChangeTracker(ZooKeeperWatcher watcher,
-                             Abortable abortable,
-                             RegionServerServices regionServer) {
-    super(watcher, watcher.schemaZNode, abortable);
-    this.regionServer = regionServer;
-  }
-
-  @Override
-  public void start() {
-    try {
-      watcher.registerListener(this);
-      ZKUtil.listChildrenAndWatchThem(watcher, node);
-      // Clean-up old in-process schema changes for this RS now?
-    } catch (KeeperException e) {
-      LOG.error("RegionServer SchemaChangeTracker startup failed with " +
-          "KeeperException.", e);
-    }
-  }
-
-
-  /**
-   * This event will be triggered whenever new schema change request is processed by the
-   * master. The path will be of the format /hbase/schema/<table name>
-   * @param path full path of the node whose children have changed
-   */
-  @Override
-  public void nodeChildrenChanged(String path) {
-    LOG.debug("NodeChildrenChanged. Path = " + path);
-    if (path.equals(watcher.schemaZNode)) {
-      try {
-        List<String> tables =
-          ZKUtil.listChildrenAndWatchThem(watcher, watcher.schemaZNode);
-        LOG.debug("RS.SchemaChangeTracker: " +
-            "Current list of tables with schema change = " + tables);
-        if (tables != null) {
-          handleSchemaChange(tables);
-        } else {
-          LOG.error("No tables found for schema change event." +
-              " Skipping instant schema refresh");
-        }
-      } catch (KeeperException ke) {
-        String errmsg = "KeeperException while handling nodeChildrenChanged for path = "
-            + path + " Cause = " + ke.getCause();
-        LOG.error(errmsg, ke);
-        TaskMonitor.get().createStatus(errmsg);
-      }
-    }
-  }
-
-  private void handleSchemaChange(List<String> tables) {
-    for (String tableName : tables) {
-      if (tableName != null) {
-        LOG.debug("Processing schema change with status for table = " + tableName);
-        handleSchemaChange(tableName);
-      }
-    }
-  }
-
-  private void handleSchemaChange(String tableName) {
-    int refreshedRegionsCount = 0, onlineRegionsCount = 0;
-    MonitoredTask status = null;
-    try {
-      List<HRegion> onlineRegions =
-          regionServer.getOnlineRegions(Bytes.toBytes(tableName));
-      if (onlineRegions != null && !onlineRegions.isEmpty()) {
-        status = TaskMonitor.get().createStatus("Region server "
-             + regionServer.getServerName().getServerName()
-             + " handling schema change for table = " + tableName
-             + " number of online regions = " + onlineRegions.size());
-        onlineRegionsCount = onlineRegions.size();
-        createStateNode(tableName, onlineRegions.size());
-        for (HRegion hRegion : onlineRegions) {
-          regionServer.refreshRegion(hRegion);
-          refreshedRegionsCount ++;
-        }
-        SchemaAlterStatus alterStatus = getSchemaAlterStatus(tableName);
-        alterStatus.update(SchemaAlterStatus.AlterState.SUCCESS, refreshedRegionsCount);
-        updateSchemaChangeStatus(tableName, alterStatus);
-        String msg = "Refresh schema completed for table name = " + tableName
-        + " server = " + regionServer.getServerName().getServerName()
-        + " online Regions = " + onlineRegions.size()
-        + " refreshed Regions = " + refreshedRegionsCount;
-        LOG.debug(msg);
-        status.setStatus(msg);
-      } else {
-        LOG.debug("Server " + regionServer.getServerName().getServerName()
-         + " has no online regions for table = " + tableName
-         + " Ignoring the schema change request");
-      }
-    } catch (IOException ioe) {
-      reportAndLogSchemaRefreshError(tableName, onlineRegionsCount,
-          refreshedRegionsCount, ioe, status);
-    } catch (KeeperException ke) {
-      reportAndLogSchemaRefreshError(tableName, onlineRegionsCount,
-          refreshedRegionsCount, ke, status);
-    }
-  }
-
-  private int getZKNodeVersion(String nodePath) throws KeeperException {
-    return ZKUtil.checkExists(this.watcher, nodePath);
-  }
-
-  private void reportAndLogSchemaRefreshError(String tableName,
-                                              int onlineRegionsCount,
-                                              int refreshedRegionsCount,
-                                              Throwable exception,
-                                              MonitoredTask status) {
-    try {
-      String errmsg =
-          " Region Server " + regionServer.getServerName().getServerName()
-              + " failed during schema change process. Cause = "
-              + exception.getCause()
-              + " Number of onlineRegions = " + onlineRegionsCount
-              + " Processed regions = " + refreshedRegionsCount;
-      SchemaAlterStatus alterStatus = getSchemaAlterStatus(tableName);
-      alterStatus.update(SchemaAlterStatus.AlterState.FAILURE,
-          refreshedRegionsCount, errmsg);
-      String nodePath = getSchemaChangeNodePathForTableAndServer(tableName,
-              regionServer.getServerName().getServerName());
-      ZKUtil.updateExistingNodeData(this.watcher, nodePath,
-          Writables.getBytes(alterStatus), getZKNodeVersion(nodePath));
-      LOG.info("reportAndLogSchemaRefreshError() " +
-          " Updated child ZKNode with SchemaAlterStatus = "
-          + alterStatus + " for table = " + tableName);
-      if (status == null) {
-        status = TaskMonitor.get().createStatus(errmsg);
-      } else {
-        status.setStatus(errmsg);
-      }
-    } catch (KeeperException e) {
-    // Retry ?
-      String errmsg = "KeeperException while updating the schema change node with "
-        + "error status for table = "
-        + tableName + " server = "
-        + regionServer.getServerName().getServerName()
-        + " Cause = " + e.getCause();
-      LOG.error(errmsg, e);
-      TaskMonitor.get().createStatus(errmsg);
-    } catch(IOException ioe) {
-      // retry ??
-      String errmsg = "IOException while updating the schema change node with "
-        + "server name for table = "
-        + tableName + " server = "
-        + regionServer.getServerName().getServerName()
-        + " Cause = " + ioe.getCause();
-      TaskMonitor.get().createStatus(errmsg);
-      LOG.error(errmsg, ioe);
-    }
-  }
-
-
-  private void createStateNode(String tableName, int numberOfOnlineRegions)
-      throws IOException {
-    SchemaAlterStatus sas =
-        new SchemaAlterStatus(regionServer.getServerName().getServerName(),
-            numberOfOnlineRegions);
-    LOG.debug("Creating Schema Alter State node = " + sas);
-    try {
-      ZKUtil.createSetData(this.watcher,
-          getSchemaChangeNodePathForTableAndServer(tableName,
-                regionServer.getServerName().getServerName()),
-                Writables.getBytes(sas));
-    } catch (KeeperException ke) {
-      String errmsg = "KeeperException while creating the schema change node with "
-          + "server name for table = "
-          + tableName + " server = "
-          + regionServer.getServerName().getServerName()
-          + " Message = " + ke.getCause();
-      LOG.error(errmsg, ke);
-      TaskMonitor.get().createStatus(errmsg);
-    }
-
-  }
-
-  private SchemaAlterStatus getSchemaAlterStatus(String tableName)
-      throws KeeperException, IOException {
-    byte[] statusBytes = ZKUtil.getData(this.watcher,
-        getSchemaChangeNodePathForTableAndServer(tableName,
-            regionServer.getServerName().getServerName()));
-    if (statusBytes == null || statusBytes.length <= 0) {
-      return null;
-    }
-    SchemaAlterStatus sas = new SchemaAlterStatus();
-    Writables.getWritable(statusBytes, sas);
-    return sas;
-  }
-
-  private void updateSchemaChangeStatus(String tableName,
-                                     SchemaAlterStatus schemaAlterStatus)
-      throws KeeperException, IOException {
-    try {
-      if(sleepTimeMillis > 0) {
-        try {
-          LOG.debug("SchemaChangeTracker sleeping for "
-              + sleepTimeMillis);
-          Thread.sleep(sleepTimeMillis);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-        }
-      }
-      ZKUtil.updateExistingNodeData(this.watcher,
-          getSchemaChangeNodePathForTableAndServer(tableName,
-              regionServer.getServerName().getServerName()),
-          Writables.getBytes(schemaAlterStatus), -1);
-      String msg = "Schema change tracker completed for table = " + tableName
-              + " status = " + schemaAlterStatus;
-      LOG.debug(msg);
-      TaskMonitor.get().createStatus(msg);
-    } catch (KeeperException.NoNodeException e) {
-      String errmsg = "KeeperException.NoNodeException while updating the schema "
-          + "change node with server name for table = "
-          + tableName + " server = "
-          + regionServer.getServerName().getServerName()
-          + " Cause = " + e.getCause();
-      TaskMonitor.get().createStatus(errmsg);
-      LOG.error(errmsg, e);
-    } catch (KeeperException e) {
-      // Retry ?
-      String errmsg = "KeeperException while updating the schema change node with "
-          + "server name for table = "
-          + tableName + " server = "
-          + regionServer.getServerName().getServerName()
-          + " Cause = " + e.getCause();
-      LOG.error(errmsg, e);
-      TaskMonitor.get().createStatus(errmsg);
-    } catch(IOException ioe) {
-      String errmsg = "IOException while updating the schema change node with "
-          + "server name for table = "
-          + tableName + " server = "
-          + regionServer.getServerName().getServerName()
-          + " Cause = " + ioe.getCause();
-      LOG.error(errmsg, ioe);
-      TaskMonitor.get().createStatus(errmsg);
-    }
-  }
-
-  private String getSchemaChangeNodePathForTable(String tableName) {
-    return ZKUtil.joinZNode(watcher.schemaZNode, tableName);
-  }
-
-  private String getSchemaChangeNodePathForTableAndServer(
-      String tableName, String regionServerName) {
-    return ZKUtil.joinZNode(getSchemaChangeNodePathForTable(tableName),
-        regionServerName);
-  }
-
-  public int getSleepTimeMillis() {
-    return sleepTimeMillis;
-  }
-
-  /**
-   * Set a sleep time in millis before this RS can update it's progress status.
-   * Used only for test cases to test complex test scenarios such as RS failures and
-   * RS exemption handling.
-   * @param sleepTimeMillis
-   */
-  public void setSleepTimeMillis(int sleepTimeMillis) {
-    this.sleepTimeMillis = sleepTimeMillis;
-  }
-
-  /**
-   * Check whether there are any schema change requests that are in progress now for the given table.
-   * We simply assume that a schema change is in progress if we see a ZK schema node this
-   * any table. We may revisit for fine grained checks such as check the current alter status
-   * et al, but it is not required now.
-   * @return
-   */
-  public boolean isSchemaChangeInProgress(String tableName) {
-    try {
-      List<String> schemaChanges = ZKUtil.listChildrenAndWatchThem(this.watcher,
-          watcher.schemaZNode);
-      if (schemaChanges != null) {
-        for (String alterTableName : schemaChanges) {
-          if (alterTableName.equals(tableName)) {
-            return true;
-          }
-        }
-        return false;
-      }
-    } catch (KeeperException ke) {
-      LOG.debug("isSchemaChangeInProgress. " +
-          "KeeperException while getting current schema change progress.");
-      return false;
-    }
-    return false;
-  }
-
-  /**
-   * Holds the current alter state for a table. Alter state includes the
-   * current alter status (INPROCESS, FAILURE, SUCCESS, or IGNORED, current RS
-   * host name, timestamp of alter request, number of online regions this RS has for
-   * the given table, number of processed regions and an errorCause in case
-   * if the RS failed during the schema change process.
-   *
-   * RS keeps track of schema change requests per table using the alter status and
-   * periodically updates the alter status based on schema change status.
-   */
-  public static class SchemaAlterStatus implements Writable {
-
-    public enum AlterState {
-      INPROCESS,        // Inprocess alter
-      SUCCESS,          // completed alter
-      FAILURE,          // failure alter
-      IGNORED           // Ignore the alter processing.
-    }
-
-    private AlterState currentAlterStatus;
-    // TimeStamp
-    private long stamp;
-    private int numberOfOnlineRegions;
-    private String errorCause = " ";
-    private String hostName;
-    private int numberOfRegionsProcessed = 0;
-
-    public SchemaAlterStatus() {
-
-    }
-
-    public SchemaAlterStatus(String hostName, int numberOfOnlineRegions) {
-      this.numberOfOnlineRegions = numberOfOnlineRegions;
-      this.stamp = System.currentTimeMillis();
-      this.currentAlterStatus = AlterState.INPROCESS;
-      //this.rsToProcess = activeHosts;
-      this.hostName = hostName;
-    }
-
-    public AlterState getCurrentAlterStatus() {
-      return currentAlterStatus;
-    }
-
-    public void setCurrentAlterStatus(AlterState currentAlterStatus) {
-      this.currentAlterStatus = currentAlterStatus;
-    }
-
-    public int getNumberOfOnlineRegions() {
-      return numberOfOnlineRegions;
-    }
-
-    public void setNumberOfOnlineRegions(int numberOfRegions) {
-      this.numberOfOnlineRegions = numberOfRegions;
-    }
-
-    public int getNumberOfRegionsProcessed() {
-      return numberOfRegionsProcessed;
-    }
-
-    public void setNumberOfRegionsProcessed(int numberOfRegionsProcessed) {
-      this.numberOfRegionsProcessed = numberOfRegionsProcessed;
-    }
-
-    public String getErrorCause() {
-      return errorCause;
-    }
-
-    public void setErrorCause(String errorCause) {
-      this.errorCause = errorCause;
-    }
-
-    public String getHostName() {
-      return hostName;
-    }
-
-    public void setHostName(String hostName) {
-      this.hostName = hostName;
-    }
-
-    public void update(AlterState state, int numberOfRegions, String errorCause) {
-      this.currentAlterStatus = state;
-      this.numberOfRegionsProcessed = numberOfRegions;
-      this.errorCause = errorCause;
-    }
-
-    public void update(AlterState state, int numberOfRegions) {
-      this.currentAlterStatus = state;
-      this.numberOfRegionsProcessed = numberOfRegions;
-    }
-
-    public void update(AlterState state) {
-      this.currentAlterStatus = state;
-    }
-
-    public void update(SchemaAlterStatus status) {
-      this.currentAlterStatus = status.getCurrentAlterStatus();
-      this.numberOfRegionsProcessed = status.getNumberOfRegionsProcessed();
-      this.errorCause = status.getErrorCause();
-    }
-
-    @Override
-    public void readFields(DataInput in) throws IOException {
-      currentAlterStatus = AlterState.valueOf(in.readUTF());
-      stamp = in.readLong();
-      numberOfOnlineRegions = in.readInt();
-      hostName = Bytes.toString(Bytes.readByteArray(in));
-      numberOfRegionsProcessed = in.readInt();
-      errorCause = Bytes.toString(Bytes.readByteArray(in));
-    }
-
-    @Override
-    public void write(DataOutput out) throws IOException {
-      out.writeUTF(currentAlterStatus.name());
-      out.writeLong(stamp);
-      out.writeInt(numberOfOnlineRegions);
-      Bytes.writeByteArray(out, Bytes.toBytes(hostName));
-      out.writeInt(numberOfRegionsProcessed);
-      Bytes.writeByteArray(out, Bytes.toBytes(errorCause));
-    }
-
-    @Override
-    public String toString() {
-      return
-         " state= " + currentAlterStatus
-        + ", ts= " + stamp
-        + ", number of online regions = " + numberOfOnlineRegions
-        + ", host= " + hostName + " processed regions = " + numberOfRegionsProcessed
-        + ", errorCause = " + errorCause;
-    }
-  }
-
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 45d7d6a88df5..7805c7e7c7b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -102,8 +102,6 @@ public class ZooKeeperWatcher implements Watcher, Abortable {
   public String clusterIdZNode;
   // znode used for log splitting work assignment
   public String splitLogZNode;
-  // znode used to record table schema changes
-  public String schemaZNode;
 
   // Certain ZooKeeper nodes need to be world-readable
   public static final ArrayList<ACL> CREATOR_ALL_AND_WORLD_READABLE =
@@ -166,7 +164,6 @@ private void createBaseZNodes() throws ZooKeeperConnectionException {
       ZKUtil.createAndFailSilent(this, drainingZNode);
       ZKUtil.createAndFailSilent(this, tableZNode);
       ZKUtil.createAndFailSilent(this, splitLogZNode);
-      ZKUtil.createAndFailSilent(this, schemaZNode);
       ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
     } catch (KeeperException e) {
       throw new ZooKeeperConnectionException(
@@ -219,8 +216,6 @@ private void setNodeNames(Configuration conf) {
         conf.get("zookeeper.znode.clusterId", "hbaseid"));
     splitLogZNode = ZKUtil.joinZNode(baseZNode,
         conf.get("zookeeper.znode.splitlog", HConstants.SPLIT_LOGDIR_NAME));
-    schemaZNode = ZKUtil.joinZNode(baseZNode,
-                conf.get("zookeeper.znode.schema", "schema"));
   }
 
   /**
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 341431ace550..44ee6895ad73 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -780,39 +780,15 @@
     </description>
   </property>
   <property>
-    <name>hbase.coprocessor.abortonerror</name>
-    <value>false</value>
-    <description>
-    Set to true to cause the hosting server (master or regionserver) to
-    abort if a coprocessor throws a Throwable object that is not IOException or
-    a subclass of IOException. Setting it to true might be useful in development
-    environments where one wants to terminate the server as soon as possible to
-    simplify coprocessor failure analysis.
-    </description>
-  </property>
-  <property>
-    <name>hbase.instant.schema.alter.enabled</name>
-    <value>false</value>
-    <description>Whether or not to handle alter schema changes instantly or not.
-    If enabled, all schema change alter operations will be instant, as the master will not
-    explicitly unassign/assign the impacted regions and instead will rely on Region servers to
-    refresh their schema changes. If enabled, the schema alter requests will survive
-    master or RS failures.
-    </description>
-  </property>
-  <property>
-    <name>hbase.instant.schema.janitor.period</name>
-    <value>120000</value>
-    <description>The Schema Janitor process wakes up every millis and sweeps all
-    expired/failed schema change requests.
-    </description>
-  </property>
-  <property>
-    <name>hbase.instant.schema.alter.timeout</name>
-    <value>60000</value>
-    <description>Timeout in millis after which any pending schema alter request will be
-    considered as failed.
-    </description>
+      <name>hbase.coprocessor.abortonerror</name>
+      <value>false</value>
+      <description>
+      Set to true to cause the hosting server (master or regionserver) to
+      abort if a coprocessor throws a Throwable object that is not IOException or
+      a subclass of IOException. Setting it to true might be useful in development
+      environments where one wants to terminate the server as soon as possible to
+      simplify coprocessor failure analysis.
+      </description>
   </property>
   <property>
     <name>hbase.online.schema.update.enable</name>
diff --git a/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java b/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java
index 378c2b47dab2..e69de29bb2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class InstantSchemaChangeTestBase {
-
-  final Log LOG = LogFactory.getLog(getClass());
-  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  protected HBaseAdmin admin;
-  protected static MiniHBaseCluster miniHBaseCluster = null;
-  protected Configuration conf;
-  protected static MasterSchemaChangeTracker msct = null;
-
-  protected final byte [] row = Bytes.toBytes("row");
-  protected final byte [] qualifier = Bytes.toBytes("qualifier");
-  final byte [] value = Bytes.toBytes("value");
-
-  @Before
-  public void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.instant.schema.alter.enabled", true);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
-    TEST_UTIL.getConfiguration().setInt("hbase.instant.schema.janitor.period", 10000);
-    TEST_UTIL.getConfiguration().setInt("hbase.instant.schema.alter.timeout", 30000);
-    //
-    miniHBaseCluster = TEST_UTIL.startMiniCluster(2,5);
-    msct = TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    this.admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
-
-  }
-
-  @After
-  public void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  /**
-   * Find the RS that is currently holding our online region.
-   * @param tableName
-   * @return
-   */
-  protected HRegionServer findRSWithOnlineRegionFor(String tableName) {
-    List<JVMClusterUtil.RegionServerThread> rsThreads =
-        miniHBaseCluster.getLiveRegionServerThreads();
-    for (JVMClusterUtil.RegionServerThread rsT : rsThreads) {
-      HRegionServer rs = rsT.getRegionServer();
-      List<HRegion> regions = rs.getOnlineRegions(Bytes.toBytes(tableName));
-      if (regions != null && !regions.isEmpty()) {
-        return rs;
-      }
-    }
-    return null;
-  }
-
-  protected void waitForSchemaChangeProcess(final String tableName)
-      throws KeeperException, InterruptedException {
-    waitForSchemaChangeProcess(tableName, 10000);
-  }
-
-  /**
-   * This a pretty low cost signalling mechanism. It is quite possible that we will
-   * miss out the ZK node creation signal as in some cases the schema change process
-   * happens rather quickly and our thread waiting for ZK node creation might wait forver.
-   * The fool-proof strategy would be to directly listen for ZK events.
-   * @param tableName
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  protected void waitForSchemaChangeProcess(final String tableName, final long waitTimeMills)
-      throws KeeperException, InterruptedException {
-    LOG.info("Waiting for ZK node creation for table = " + tableName);
-    final MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    final Runnable r = new Runnable() {
-      public void run() {
-        try {
-          while(!msct.doesSchemaChangeNodeExists(tableName)) {
-            try {
-              Thread.sleep(50);
-            } catch (InterruptedException e) {
-              Thread.currentThread().interrupt();
-            }
-          }
-        } catch (KeeperException ke) {
-            ke.printStackTrace();
-        }
-        LOG.info("Waiting for ZK node deletion for table = " + tableName);
-        try {
-          while(msct.doesSchemaChangeNodeExists(tableName)) {
-            try {
-              Thread.sleep(50);
-            } catch (InterruptedException e) {
-               Thread.currentThread().interrupt();
-            }
-          }
-        }  catch (KeeperException ke) {
-            ke.printStackTrace();
-        }
-      }
-    };
-    Thread t = new Thread(r);
-    t.start();
-    if (waitTimeMills > 0) {
-      t.join(waitTimeMills);
-    }  else {
-      t.join(10000);
-    }
-  }
-
-  protected HTable createTableAndValidate(String tableName) throws IOException {
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start createTableAndValidate()");
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    HTableDescriptor[] tables = admin.listTables();
-    int numTables = 0;
-    if (tables != null) {
-      numTables = tables.length;
-    }
-    HTable ht = TEST_UTIL.createTable(Bytes.toBytes(tableName),
-      HConstants.CATALOG_FAMILY);
-    tables = this.admin.listTables();
-    assertEquals(numTables + 1, tables.length);
-    LOG.info("created table = " + tableName);
-    return ht;
-  }
-
-}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChange.java b/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChange.java
deleted file mode 100644
index 4ac28474239d..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChange.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestInstantSchemaChange extends InstantSchemaChangeTestBase {
-
-  final Log LOG = LogFactory.getLog(getClass());
-
-  @Test
-  public void testInstantSchemaChangeForModifyTable() throws IOException,
-      KeeperException, InterruptedException {
-
-    String tableName = "testInstantSchemaChangeForModifyTable";
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start testInstantSchemaChangeForModifyTable()");
-    HTable ht = createTableAndValidate(tableName);
-
-    String newFamily = "newFamily";
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    htd.addFamily(new HColumnDescriptor(newFamily));
-
-    admin.modifyTable(Bytes.toBytes(tableName), htd);
-    waitForSchemaChangeProcess(tableName);
-    assertFalse(msct.doesSchemaChangeNodeExists(tableName));
-
-    Put put1 = new Put(row);
-    put1.add(Bytes.toBytes(newFamily), qualifier, value);
-    ht.put(put1);
-
-    Get get1 = new Get(row);
-    get1.addColumn(Bytes.toBytes(newFamily), qualifier);
-    Result r = ht.get(get1);
-    byte[] tvalue = r.getValue(Bytes.toBytes(newFamily), qualifier);
-    int result = Bytes.compareTo(value, tvalue);
-    assertEquals(result, 0);
-    LOG.info("END testInstantSchemaChangeForModifyTable()");
-    ht.close();
-  }
-
-  @Test
-  public void testInstantSchemaChangeForAddColumn() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("Start testInstantSchemaChangeForAddColumn() ");
-    String tableName = "testSchemachangeForAddColumn";
-    HTable ht = createTableAndValidate(tableName);
-    String newFamily = "newFamily";
-    HColumnDescriptor hcd = new HColumnDescriptor("newFamily");
-
-    admin.addColumn(Bytes.toBytes(tableName), hcd);
-    waitForSchemaChangeProcess(tableName);
-    assertFalse(msct.doesSchemaChangeNodeExists(tableName));
-
-    Put put1 = new Put(row);
-    put1.add(Bytes.toBytes(newFamily), qualifier, value);
-    LOG.info("******** Put into new column family ");
-    ht.put(put1);
-
-    Get get1 = new Get(row);
-    get1.addColumn(Bytes.toBytes(newFamily), qualifier);
-    Result r = ht.get(get1);
-    byte[] tvalue = r.getValue(Bytes.toBytes(newFamily), qualifier);
-    LOG.info(" Value put = " + value + " value from table = " + tvalue);
-    int result = Bytes.compareTo(value, tvalue);
-    assertEquals(result, 0);
-    LOG.info("End testInstantSchemaChangeForAddColumn() ");
-    ht.close();
- }
-
-  @Test
-  public void testInstantSchemaChangeForModifyColumn() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("Start testInstantSchemaChangeForModifyColumn() ");
-    String tableName = "testSchemachangeForModifyColumn";
-    createTableAndValidate(tableName);
-
-    HColumnDescriptor hcd = new HColumnDescriptor(HConstants.CATALOG_FAMILY);
-    hcd.setMaxVersions(99);
-    hcd.setBlockCacheEnabled(false);
-
-    admin.modifyColumn(Bytes.toBytes(tableName), hcd);
-    waitForSchemaChangeProcess(tableName);
-    assertFalse(msct.doesSchemaChangeNodeExists(tableName));
-
-    List<HRegion> onlineRegions
-        = miniHBaseCluster.getRegions(Bytes.toBytes("testSchemachangeForModifyColumn"));
-    for (HRegion onlineRegion : onlineRegions) {
-      HTableDescriptor htd = onlineRegion.getTableDesc();
-      HColumnDescriptor tableHcd = htd.getFamily(HConstants.CATALOG_FAMILY);
-      assertTrue(tableHcd.isBlockCacheEnabled() == false);
-      assertEquals(tableHcd.getMaxVersions(), 99);
-    }
-    LOG.info("End testInstantSchemaChangeForModifyColumn() ");
-
- }
-
-  @Test
-  public void testInstantSchemaChangeForDeleteColumn() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("Start testInstantSchemaChangeForDeleteColumn() ");
-    String tableName = "testSchemachangeForDeleteColumn";
-    int numTables = 0;
-    HTableDescriptor[] tables = admin.listTables();
-    if (tables != null) {
-      numTables = tables.length;
-    }
-
-    byte[][] FAMILIES = new byte[][] {
-      Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
-
-    HTable ht = TEST_UTIL.createTable(Bytes.toBytes(tableName),
-      FAMILIES);
-    tables = this.admin.listTables();
-    assertEquals(numTables + 1, tables.length);
-    LOG.info("Table testSchemachangeForDeleteColumn created");
-
-    admin.deleteColumn(tableName, "C");
-
-    waitForSchemaChangeProcess(tableName);
-    assertFalse(msct.doesSchemaChangeNodeExists(tableName));
-    HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(Bytes.toBytes(tableName));
-    HColumnDescriptor hcd = modifiedHtd.getFamily(Bytes.toBytes("C"));
-    assertTrue(hcd == null);
-    LOG.info("End testInstantSchemaChangeForDeleteColumn() ");
-    ht.close();
- }
-
-  @Test
-  public void testInstantSchemaChangeWhenTableIsNotEnabled() throws IOException,
-      KeeperException {
-    final String tableName = "testInstantSchemaChangeWhenTableIsDisabled";
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start testInstantSchemaChangeWhenTableIsDisabled()");
-    HTable ht = createTableAndValidate(tableName);
-    // Disable table
-    admin.disableTable("testInstantSchemaChangeWhenTableIsDisabled");
-    // perform schema changes
-    HColumnDescriptor hcd = new HColumnDescriptor("newFamily");
-    admin.addColumn(Bytes.toBytes(tableName), hcd);
-    MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    assertTrue(msct.doesSchemaChangeNodeExists(tableName) == false);
-    ht.close();
-  }
-
-  /**
-   * Test that when concurrent alter requests are received for a table we don't miss any.
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testConcurrentInstantSchemaChangeForAddColumn() throws IOException,
-      KeeperException, InterruptedException {
-    final String tableName = "testConcurrentInstantSchemaChangeForModifyTable";
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start testConcurrentInstantSchemaChangeForModifyTable()");
-    HTable ht = createTableAndValidate(tableName);
-
-    Runnable run1 = new Runnable() {
-      public void run() {
-        HColumnDescriptor hcd = new HColumnDescriptor("family1");
-        try {
-          admin.addColumn(Bytes.toBytes(tableName), hcd);
-        } catch (IOException ioe) {
-          ioe.printStackTrace();
-
-        }
-      }
-    };
-    Runnable run2 = new Runnable() {
-      public void run() {
-        HColumnDescriptor hcd = new HColumnDescriptor("family2");
-        try {
-          admin.addColumn(Bytes.toBytes(tableName), hcd);
-        } catch (IOException ioe) {
-          ioe.printStackTrace();
-
-        }
-      }
-    };
-
-    run1.run();
-    // We have to add a sleep here as in concurrent scenarios the HTD update
-    // in HDFS fails and returns with null HTD. This needs to be investigated,
-    // but it doesn't impact the instant alter functionality in any way.
-    Thread.sleep(100);
-    run2.run();
-
-    waitForSchemaChangeProcess(tableName);
-
-    Put put1 = new Put(row);
-    put1.add(Bytes.toBytes("family1"), qualifier, value);
-    ht.put(put1);
-
-    Get get1 = new Get(row);
-    get1.addColumn(Bytes.toBytes("family1"), qualifier);
-    Result r = ht.get(get1);
-    byte[] tvalue = r.getValue(Bytes.toBytes("family1"), qualifier);
-    int result = Bytes.compareTo(value, tvalue);
-    assertEquals(result, 0);
-    Thread.sleep(10000);
-
-    Put put2 = new Put(row);
-    put2.add(Bytes.toBytes("family2"), qualifier, value);
-    ht.put(put2);
-
-    Get get2 = new Get(row);
-    get2.addColumn(Bytes.toBytes("family2"), qualifier);
-    Result r2 = ht.get(get2);
-    byte[] tvalue2 = r2.getValue(Bytes.toBytes("family2"), qualifier);
-    int result2 = Bytes.compareTo(value, tvalue2);
-    assertEquals(result2, 0);
-    LOG.info("END testConcurrentInstantSchemaChangeForModifyTable()");
-    ht.close();
-  }
-
-  /**
-   * The schema change request blocks while a LB run is in progress. This
-   * test validates this behavior.
-   * @throws IOException
-   * @throws InterruptedException
-   * @throws KeeperException
-   */
-  @Test
-  public void testConcurrentInstantSchemaChangeAndLoadBalancerRun() throws IOException,
-      InterruptedException, KeeperException {
-    final String tableName = "testInstantSchemaChangeWithLoadBalancerRunning";
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start testInstantSchemaChangeWithLoadBalancerRunning()");
-    final String newFamily = "newFamily";
-    HTable ht = createTableAndValidate(tableName);
-    final MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-
-
-    Runnable balancer = new Runnable() {
-      public void run() {
-        // run the balancer now.
-        miniHBaseCluster.getMaster().balance();
-      }
-    };
-
-    Runnable schemaChanger = new Runnable() {
-      public void run() {
-        HColumnDescriptor hcd = new HColumnDescriptor(newFamily);
-        try {
-          admin.addColumn(Bytes.toBytes(tableName), hcd);
-        } catch (IOException ioe) {
-          ioe.printStackTrace();
-
-        }
-      }
-    };
-
-    balancer.run();
-    schemaChanger.run();
-    waitForSchemaChangeProcess(tableName, 40000);
-    assertFalse(msct.doesSchemaChangeNodeExists(tableName));
-
-    Put put1 = new Put(row);
-    put1.add(Bytes.toBytes(newFamily), qualifier, value);
-    LOG.info("******** Put into new column family ");
-    ht.put(put1);
-    ht.flushCommits();
-
-    LOG.info("******** Get from new column family ");
-    Get get1 = new Get(row);
-    get1.addColumn(Bytes.toBytes(newFamily), qualifier);
-    Result r = ht.get(get1);
-    byte[] tvalue = r.getValue(Bytes.toBytes(newFamily), qualifier);
-    LOG.info(" Value put = " + value + " value from table = " + tvalue);
-    int result = Bytes.compareTo(value, tvalue);
-    assertEquals(result, 0);
-
-    LOG.info("End testInstantSchemaChangeWithLoadBalancerRunning() ");
-    ht.close();
-  }
-
-
-  /**
-   * This test validates two things. One is that the LoadBalancer does not run when a schema
-   * change process is in progress. The second thing is that it also checks that failed/expired
-   * schema changes are expired to unblock the load balancer run.
-   *
-   */
-  @Test (timeout=70000)
-  public void testLoadBalancerBlocksDuringSchemaChangeRequests() throws KeeperException,
-      IOException, InterruptedException {
-    LOG.info("Start testConcurrentLoadBalancerSchemaChangeRequests() ");
-    final MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    // Test that the load balancer does not run while an in-flight schema
-    // change operation is in progress.
-    // Simulate a new schema change request.
-    msct.createSchemaChangeNode("testLoadBalancerBlocks", 0);
-    // The schema change node is created.
-    assertTrue(msct.doesSchemaChangeNodeExists("testLoadBalancerBlocks"));
-    // Now, request an explicit LB run.
-
-    Runnable balancer1 = new Runnable() {
-      public void run() {
-        // run the balancer now.
-        miniHBaseCluster.getMaster().balance();
-      }
-    };
-    balancer1.run();
-
-    // Load balancer should not run now.
-    assertTrue(miniHBaseCluster.getMaster().isLoadBalancerRunning() == false);
-    LOG.debug("testConcurrentLoadBalancerSchemaChangeRequests Asserted");
-    LOG.info("End testConcurrentLoadBalancerSchemaChangeRequests() ");
-  }
-
-  /**
-   * Test that instant schema change blocks while LB is running.
-   * @throws KeeperException
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  @Test (timeout=10000)
-  public void testInstantSchemaChangeBlocksDuringLoadBalancerRun() throws KeeperException,
-      IOException, InterruptedException {
-    final MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-
-    final String tableName = "testInstantSchemaChangeBlocksDuringLoadBalancerRun";
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start testInstantSchemaChangeBlocksDuringLoadBalancerRun()");
-    final String newFamily = "newFamily";
-    createTableAndValidate(tableName);
-
-    // Test that the schema change request does not run while an in-flight LB run
-    // is in progress.
-    // First, request an explicit LB run.
-
-    Runnable balancer1 = new Runnable() {
-      public void run() {
-        // run the balancer now.
-        miniHBaseCluster.getMaster().balance();
-      }
-    };
-
-    Runnable schemaChanger = new Runnable() {
-      public void run() {
-        HColumnDescriptor hcd = new HColumnDescriptor(newFamily);
-        try {
-          admin.addColumn(Bytes.toBytes(tableName), hcd);
-        } catch (IOException ioe) {
-          ioe.printStackTrace();
-
-        }
-      }
-    };
-
-    Thread t1 = new Thread(balancer1);
-    Thread t2 = new Thread(schemaChanger);
-    t1.start();
-    t2.start();
-
-    // check that they both happen concurrently
-    Runnable balancerCheck = new Runnable() {
-      public void run() {
-        // check whether balancer is running.
-        while(!miniHBaseCluster.getMaster().isLoadBalancerRunning()) {
-          try {
-            Thread.sleep(10);
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-          }
-        }
-        try {
-           assertFalse(msct.doesSchemaChangeNodeExists("testSchemaChangeBlocks"));
-        } catch (KeeperException ke) {
-          ke.printStackTrace();
-        }
-        LOG.debug("Load Balancer is now running or skipped");
-        while(miniHBaseCluster.getMaster().isLoadBalancerRunning()) {
-          try {
-            Thread.sleep(10);
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-          }
-        }
-        assertTrue(miniHBaseCluster.getMaster().isLoadBalancerRunning() == false);
-        try {
-          assertTrue(msct.doesSchemaChangeNodeExists("testSchemaChangeBlocks"));
-        } catch (KeeperException ke) {
-
-        }
-
-      }
-    };
-
-    Thread t = new Thread(balancerCheck);
-    t.start();
-    t.join(1000);
-    // Load balancer should not run now.
-    //assertTrue(miniHBaseCluster.getMaster().isLoadBalancerRunning() == false);
-    // Schema change request node should now exist.
-   // assertTrue(msct.doesSchemaChangeNodeExists("testSchemaChangeBlocks"));
-    LOG.debug("testInstantSchemaChangeBlocksDuringLoadBalancerRun Asserted");
-    LOG.info("End testInstantSchemaChangeBlocksDuringLoadBalancerRun() ");
-  }
-
-  /**
-   * To test the schema janitor (that it cleans expired/failed schema alter attempts) we
-   * simply create a fake table (that doesn't exist, with fake number of online regions) in ZK.
-   * This schema alter request will time out (after 30 seconds) and our janitor will clean it up.
-   * regions
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testInstantSchemaJanitor() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("testInstantSchemaWithFailedExpiredOperations() ");
-    String fakeTableName = "testInstantSchemaWithFailedExpiredOperations";
-    MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    msct.createSchemaChangeNode(fakeTableName, 10);
-    LOG.debug(msct.getSchemaChangeNodePathForTable(fakeTableName)
-        + " created");
-    Thread.sleep(40000);
-    assertFalse(msct.doesSchemaChangeNodeExists(fakeTableName));
-    LOG.debug(msct.getSchemaChangeNodePathForTable(fakeTableName)
-        + " deleted");
-    LOG.info("END testInstantSchemaWithFailedExpiredOperations() ");
-  }
-
-
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeFailover.java b/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeFailover.java
deleted file mode 100644
index c1490eb79ac6..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeFailover.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestInstantSchemaChangeFailover {
-
-  final Log LOG = LogFactory.getLog(getClass());
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private HBaseAdmin admin;
-  private static MiniHBaseCluster miniHBaseCluster = null;
-  private Configuration conf;
-  private ZooKeeperWatcher zkw;
-  private static MasterSchemaChangeTracker msct = null;
-
-  private final byte [] row = Bytes.toBytes("row");
-  private final byte [] qualifier = Bytes.toBytes("qualifier");
-  final byte [] value = Bytes.toBytes("value");
-
-  @Before
-  public void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.instant.schema.alter.enabled", true);
-    TEST_UTIL.getConfiguration().setInt("hbase.instant.schema.janitor.period", 10000);
-    TEST_UTIL.getConfiguration().setInt("hbase.instant.schema.alter.timeout", 30000);
-    //
-    miniHBaseCluster = TEST_UTIL.startMiniCluster(2,5);
-    msct = TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    this.admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
-  }
-
-  @After
-  public void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  /**
-   * This a pretty low cost signalling mechanism. It is quite possible that we will
-   * miss out the ZK node creation signal as in some cases the schema change process
-   * happens rather quickly and our thread waiting for ZK node creation might wait forver.
-   * The fool-proof strategy would be to directly listen for ZK events.
-   * @param tableName
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  private void waitForSchemaChangeProcess(final String tableName)
-      throws KeeperException, InterruptedException {
-    LOG.info("Waiting for ZK node creation for table = " + tableName);
-    final MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    final Runnable r = new Runnable() {
-      public void run() {
-        try {
-          while(!msct.doesSchemaChangeNodeExists(tableName)) {
-            try {
-              Thread.sleep(20);
-            } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            }
-          }
-        } catch (KeeperException ke) {
-            ke.printStackTrace();
-        }
-
-        LOG.info("Waiting for ZK node deletion for table = " + tableName);
-        try {
-          while(msct.doesSchemaChangeNodeExists(tableName)) {
-            try {
-              Thread.sleep(20);
-            } catch (InterruptedException e) {
-              Thread.currentThread().interrupt();
-            }
-          }
-        } catch (KeeperException ke) {
-         ke.printStackTrace();
-        }
-      }
-    };
-    Thread t = new Thread(r);
-    t.start();
-    t.join(10000);
-  }
-
-
-  /**
-   * Kill a random RS and see that the schema change can succeed.
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  @Test (timeout=50000)
-  public void testInstantSchemaChangeWhileRSCrash() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("Start testInstantSchemaChangeWhileRSCrash()");
-    zkw = miniHBaseCluster.getMaster().getZooKeeperWatcher();
-
-    final String tableName = "TestRSCrashDuringSchemaChange";
-    HTable ht = createTableAndValidate(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor("family2");
-    admin.addColumn(Bytes.toBytes(tableName), hcd);
-
-    miniHBaseCluster.getRegionServer(0).abort("Killing while instant schema change");
-    // Let the dust settle down
-    Thread.sleep(10000);
-    waitForSchemaChangeProcess(tableName);
-    Put put2 = new Put(row);
-    put2.add(Bytes.toBytes("family2"), qualifier, value);
-    ht.put(put2);
-
-    Get get2 = new Get(row);
-    get2.addColumn(Bytes.toBytes("family2"), qualifier);
-    Result r2 = ht.get(get2);
-    byte[] tvalue2 = r2.getValue(Bytes.toBytes("family2"), qualifier);
-    int result2 = Bytes.compareTo(value, tvalue2);
-    assertEquals(result2, 0);
-    String nodePath = msct.getSchemaChangeNodePathForTable("TestRSCrashDuringSchemaChange");
-    assertTrue(ZKUtil.checkExists(zkw, nodePath) == -1);
-    LOG.info("result2 = " + result2);
-    LOG.info("end testInstantSchemaChangeWhileRSCrash()");
-    ht.close();
-  }
-
-  /**
-   * Randomly bring down/up RS servers while schema change is in progress. This test
-   * is same as the above one but the only difference is that we intent to kill and start
-   * new RS instances while a schema change is in progress.
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  @Test (timeout=70000)
-  public void testInstantSchemaChangeWhileRandomRSCrashAndStart() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("Start testInstantSchemaChangeWhileRandomRSCrashAndStart()");
-    miniHBaseCluster.getRegionServer(4).abort("Killing RS 4");
-    // Start a new RS before schema change .
-    // Commenting the start RS as it is failing with DFS user permission NPE.
-    //miniHBaseCluster.startRegionServer();
-
-    // Let the dust settle
-    Thread.sleep(10000);
-    final String tableName = "testInstantSchemaChangeWhileRandomRSCrashAndStart";
-    HTable ht = createTableAndValidate(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor("family2");
-    admin.addColumn(Bytes.toBytes(tableName), hcd);
-    // Kill 2 RS now.
-    miniHBaseCluster.getRegionServer(2).abort("Killing RS 2");
-    // Let the dust settle
-    Thread.sleep(10000);
-    // We will be left with only one RS.
-    waitForSchemaChangeProcess(tableName);
-    assertFalse(msct.doesSchemaChangeNodeExists(tableName));
-    Put put2 = new Put(row);
-    put2.add(Bytes.toBytes("family2"), qualifier, value);
-    ht.put(put2);
-
-    Get get2 = new Get(row);
-    get2.addColumn(Bytes.toBytes("family2"), qualifier);
-    Result r2 = ht.get(get2);
-    byte[] tvalue2 = r2.getValue(Bytes.toBytes("family2"), qualifier);
-    int result2 = Bytes.compareTo(value, tvalue2);
-    assertEquals(result2, 0);
-    LOG.info("result2 = " + result2);
-    LOG.info("end testInstantSchemaChangeWhileRandomRSCrashAndStart()");
-    ht.close();
-  }
-
-  /**
-   * Test scenario where primary master is brought down while processing an
-   * alter request. This is harder one as it is very difficult the time this.
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-
-  @Test (timeout=50000)
-  public void testInstantSchemaChangeWhileMasterFailover() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("Start testInstantSchemaChangeWhileMasterFailover()");
-    //Thread.sleep(5000);
-
-    final String tableName = "testInstantSchemaChangeWhileMasterFailover";
-    HTable ht = createTableAndValidate(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor("family2");
-    admin.addColumn(Bytes.toBytes(tableName), hcd);
-    // Kill primary master now.
-    Thread.sleep(50);
-    miniHBaseCluster.getMaster().abort("Aborting master now", new Exception("Schema exception"));
-
-    // It may not be possible for us to check the schema change status
-    // using waitForSchemaChangeProcess as our ZK session in MasterSchemachangeTracker will be
-    // lost when master dies and hence may not be accurate. So relying on old-fashioned
-    // sleep here.
-    Thread.sleep(25000);
-    Put put2 = new Put(row);
-    put2.add(Bytes.toBytes("family2"), qualifier, value);
-    ht.put(put2);
-
-    Get get2 = new Get(row);
-    get2.addColumn(Bytes.toBytes("family2"), qualifier);
-    Result r2 = ht.get(get2);
-    byte[] tvalue2 = r2.getValue(Bytes.toBytes("family2"), qualifier);
-    int result2 = Bytes.compareTo(value, tvalue2);
-    assertEquals(result2, 0);
-    LOG.info("result2 = " + result2);
-    LOG.info("end testInstantSchemaChangeWhileMasterFailover()");
-    ht.close();
-  }
-
-  /**
-   * TEst the master fail over during a schema change request in ZK.
-   * We create a fake schema change request in ZK and abort the primary master
-   * mid-flight to simulate a master fail over scenario during a mid-flight
-   * schema change process. The new master's schema janitor will eventually
-   * cleanup this fake request after time out.
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  @Ignore
-  @Test
-  public void testInstantSchemaOperationsInZKForMasterFailover() throws IOException,
-      KeeperException, InterruptedException {
-    LOG.info("testInstantSchemaOperationsInZKForMasterFailover() ");
-    String tableName = "testInstantSchemaOperationsInZKForMasterFailover";
-
-    conf = TEST_UTIL.getConfiguration();
-    MasterSchemaChangeTracker activesct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    activesct.createSchemaChangeNode(tableName, 10);
-    LOG.debug(activesct.getSchemaChangeNodePathForTable(tableName)
-        + " created");
-    assertTrue(activesct.doesSchemaChangeNodeExists(tableName));
-    // Kill primary master now.
-    miniHBaseCluster.getMaster().abort("Aborting master now", new Exception("Schema exception"));
-    // wait for 50 secs. This is so that our schema janitor from fail-over master will kick-in and
-    // cleanup this failed/expired schema change request.
-    Thread.sleep(50000);
-    MasterSchemaChangeTracker newmsct = miniHBaseCluster.getMaster().getSchemaChangeTracker();
-    assertFalse(newmsct.doesSchemaChangeNodeExists(tableName));
-    LOG.debug(newmsct.getSchemaChangeNodePathForTable(tableName)
-        + " deleted");
-    LOG.info("END testInstantSchemaOperationsInZKForMasterFailover() ");
-  }
-
-  private HTable createTableAndValidate(String tableName) throws IOException {
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start createTableAndValidate()");
-    HTableDescriptor[] tables = admin.listTables();
-    int numTables = 0;
-    if (tables != null) {
-      numTables = tables.length;
-    }
-    HTable ht = TEST_UTIL.createTable(Bytes.toBytes(tableName),
-      HConstants.CATALOG_FAMILY);
-    tables = this.admin.listTables();
-    assertEquals(numTables + 1, tables.length);
-    LOG.info("created table = " + tableName);
-    return ht;
-  }
-
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
-
-
-
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeSplit.java b/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeSplit.java
deleted file mode 100644
index 8f3124b8ea41..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeSplit.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestInstantSchemaChangeSplit extends InstantSchemaChangeTestBase {
-
-  final Log LOG = LogFactory.getLog(getClass());
-
-  /**
-   * The objective of the following test is to validate that schema exclusions happen properly.
-   * When a RS server dies or crashes(?) mid-flight during a schema refresh, we would exclude
-   * all online regions in that RS, as well as the RS itself from schema change process.
-   *
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testInstantSchemaChangeExclusions() throws IOException,
-      KeeperException, InterruptedException {
-    MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-    LOG.info("Start testInstantSchemaChangeExclusions() ");
-    String tableName = "testInstantSchemaChangeExclusions";
-    HTable ht = createTableAndValidate(tableName);
-
-    HColumnDescriptor hcd = new HColumnDescriptor(HConstants.CATALOG_FAMILY);
-    hcd.setMaxVersions(99);
-    hcd.setBlockCacheEnabled(false);
-
-    HRegionServer hrs = findRSWithOnlineRegionFor(tableName);
-    //miniHBaseCluster.getRegionServer(0).abort("killed for test");
-    admin.modifyColumn(Bytes.toBytes(tableName), hcd);
-    hrs.abort("Aborting for tests");
-    hrs.getSchemaChangeTracker().setSleepTimeMillis(20000);
-
-    //admin.modifyColumn(Bytes.toBytes(tableName), hcd);
-    LOG.debug("Waiting for Schema Change process to complete");
-    waitForSchemaChangeProcess(tableName, 15000);
-    assertEquals(msct.doesSchemaChangeNodeExists(tableName), false);
-    // Sleep for some time so that our region is reassigned to some other RS
-    // by master.
-    Thread.sleep(10000);
-    List<HRegion> onlineRegions
-        = miniHBaseCluster.getRegions(Bytes.toBytes("testInstantSchemaChangeExclusions"));
-    assertTrue(!onlineRegions.isEmpty());
-    for (HRegion onlineRegion : onlineRegions) {
-      HTableDescriptor htd = onlineRegion.getTableDesc();
-      HColumnDescriptor tableHcd = htd.getFamily(HConstants.CATALOG_FAMILY);
-      assertTrue(tableHcd.isBlockCacheEnabled() == false);
-      assertEquals(tableHcd.getMaxVersions(), 99);
-    }
-    LOG.info("End testInstantSchemaChangeExclusions() ");
-    ht.close();
- }
-
-  /**
-   * This test validates that when a schema change request fails on the
-   * RS side, we appropriately register the failure in the Master Schema change
-   * tracker's node as well as capture the error cause.
-   *
-   * Currently an alter request fails if RS fails with an IO exception say due to
-   * missing or incorrect codec. With instant schema change the same failure happens
-   * and we register the failure with associated cause and also update the
-   * monitor status appropriately.
-   *
-   * The region(s) will be orphaned in both the cases.
-   *
-   */
-  @Test
-  public void testInstantSchemaChangeWhileRSOpenRegionFailure() throws IOException,
-      KeeperException, InterruptedException {
-    MasterSchemaChangeTracker msct =
-    TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-
-    LOG.info("Start testInstantSchemaChangeWhileRSOpenRegionFailure() ");
-    String tableName = "testInstantSchemaChangeWhileRSOpenRegionFailure";
-    HTable ht = createTableAndValidate(tableName);
-
-    // create now 100 regions
-    TEST_UTIL.createMultiRegions(conf, ht,
-      HConstants.CATALOG_FAMILY, 10);
-
-    // wait for all the regions to be assigned
-    Thread.sleep(10000);
-    List<HRegion> onlineRegions
-        = miniHBaseCluster.getRegions(
-        Bytes.toBytes("testInstantSchemaChangeWhileRSOpenRegionFailure"));
-    int size = onlineRegions.size();
-    // we will not have any online regions
-    LOG.info("Size of online regions = " + onlineRegions.size());
-
-    HColumnDescriptor hcd = new HColumnDescriptor(HConstants.CATALOG_FAMILY);
-    hcd.setMaxVersions(99);
-    hcd.setBlockCacheEnabled(false);
-    hcd.setCompressionType(Compression.Algorithm.SNAPPY);
-
-    admin.modifyColumn(Bytes.toBytes(tableName), hcd);
-    Thread.sleep(100);
-
-    assertEquals(msct.doesSchemaChangeNodeExists(tableName), true);
-    Thread.sleep(10000);
-    // get the current alter status and validate that its failure with appropriate error msg.
-    MasterSchemaChangeTracker.MasterAlterStatus mas = msct.getMasterAlterStatus(tableName);
-    assertTrue(mas != null);
-    assertEquals(mas.getCurrentAlterStatus(),
-        MasterSchemaChangeTracker.MasterAlterStatus.AlterState.FAILURE);
-    assertTrue(mas.getErrorCause() != null);
-    LOG.info("End testInstantSchemaChangeWhileRSOpenRegionFailure() ");
-    ht.close();
- }
-
-  @Test
-  public void testConcurrentInstantSchemaChangeAndSplit() throws IOException,
-  InterruptedException, KeeperException {
-    final String tableName = "testConcurrentInstantSchemaChangeAndSplit";
-    conf = TEST_UTIL.getConfiguration();
-    LOG.info("Start testConcurrentInstantSchemaChangeAndSplit()");
-    final String newFamily = "newFamily";
-    HTable ht = createTableAndValidate(tableName);
-    final MasterSchemaChangeTracker msct =
-      TEST_UTIL.getHBaseCluster().getMaster().getSchemaChangeTracker();
-
-    // create now 10 regions
-    TEST_UTIL.createMultiRegions(conf, ht,
-        HConstants.CATALOG_FAMILY, 4);
-    int rowCount = TEST_UTIL.loadTable(ht, HConstants.CATALOG_FAMILY);
-    //assertRowCount(t, rowCount);
-
-    Runnable splitter = new Runnable() {
-      public void run() {
-        // run the splits now.
-        try {
-          LOG.info("Splitting table now ");
-          admin.split(Bytes.toBytes(tableName));
-        } catch (IOException e) {
-          e.printStackTrace();
-        } catch (InterruptedException e) {
-          e.printStackTrace();
-        }
-      }
-    };
-
-    Runnable schemaChanger = new Runnable() {
-      public void run() {
-        HColumnDescriptor hcd = new HColumnDescriptor(newFamily);
-        try {
-          admin.addColumn(Bytes.toBytes(tableName), hcd);
-        } catch (IOException ioe) {
-          ioe.printStackTrace();
-
-        }
-      }
-    };
-    schemaChanger.run();
-    Thread.sleep(50);
-    splitter.run();
-    waitForSchemaChangeProcess(tableName, 40000);
-
-    Put put1 = new Put(row);
-    put1.add(Bytes.toBytes(newFamily), qualifier, value);
-    LOG.info("******** Put into new column family ");
-    ht.put(put1);
-    ht.flushCommits();
-
-    LOG.info("******** Get from new column family ");
-    Get get1 = new Get(row);
-    get1.addColumn(Bytes.toBytes(newFamily), qualifier);
-    Result r = ht.get(get1);
-    byte[] tvalue = r.getValue(Bytes.toBytes(newFamily), qualifier);
-    LOG.info(" Value put = " + value + " value from table = " + tvalue);
-    int result = Bytes.compareTo(value, tvalue);
-    assertEquals(result, 0);
-    LOG.info("End testConcurrentInstantSchemaChangeAndSplit() ");
-    ht.close();
-  }
-      
-
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
-
-
-
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 227c5f28446a..b4dcb83b9000 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -51,15 +51,12 @@
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
-import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -155,6 +152,11 @@ class MockMasterServices implements MasterServices {
       this.asm = Mockito.mock(AssignmentManager.class);
     }
 
+    @Override
+    public void checkTableModifiable(byte[] tableName) throws IOException {
+      //no-op
+    }
+
     @Override
     public void createTable(HTableDescriptor desc, byte[][] splitKeys)
         throws IOException {
@@ -171,11 +173,6 @@ public ExecutorService getExecutorService() {
       return null;
     }
 
-    public void checkTableModifiable(byte[] tableName,
-                                     EventHandler.EventType eventType)
-        throws IOException {
-    }
-
     @Override
     public MasterFileSystem getMasterFileSystem() {
       return this.mfs;
@@ -263,14 +260,6 @@ public void add(HTableDescriptor htd) throws IOException {
       };
     }
 
-    public MasterSchemaChangeTracker getSchemaChangeTracker() {
-      return null;
-    }
-
-    public RegionServerTracker getRegionServerTracker() {
-      return null;
-    }
-
     @Override
     public boolean isServerShutdownHandlerEnabled() {
       return true;
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
index bb3ddd728a7c..7d0275960cfc 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
@@ -63,9 +63,6 @@ public List<HRegion> getOnlineRegions(byte[] tableName) throws IOException {
     return null;
   }
 
-  public void refreshRegion(HRegion hRegion) throws IOException {
-  }
-
   @Override
   public void addToOnlineRegions(HRegion r) {
     this.regions.put(r.getRegionInfo().getEncodedName(), r);

From a333b784f7f63f1d3d9aadc2df112da56803adf9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 5 Apr 2012 20:21:22 +0000
Subject: [PATCH 0102/1540] HBASE-5711 Tests are failing with incorrect data
 directory permissions.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310051 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 19fca5803f59..3c71d50e201d 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -425,6 +425,9 @@ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
     // It's also deprecated
     System.setProperty("test.cache.data", this.clusterTestDir.toString());
 
+    // Set the permissions for dfs data directories
+    this.conf.set("dfs.datanode.data.dir.perm", "700");
+    
     // Ok, now we can start
     this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
       true, null, null, hosts, null);

From 880c1289b8dd9d1a6b5958f4a2db385a86b5c15c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 5 Apr 2012 20:48:03 +0000
Subject: [PATCH 0103/1540] HBASE-5724 Row cache of KeyValue should be cleared
 in readFields().

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310067 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     |  4 +++-
 .../org/apache/hadoop/hbase/TestKeyValue.java | 22 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 0395f953f87d..277eafaa23c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -27,7 +27,6 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import com.google.common.primitives.Longs;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.io.HeapSize;
@@ -37,6 +36,8 @@
 import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.Writable;
 
+import com.google.common.primitives.Longs;
+
 /**
  * An HBase Key/Value.  This is the fundamental HBase Type.
  *
@@ -2231,6 +2232,7 @@ public long heapSize() {
   // and it expects the length of the KeyValue to be explicitly passed
   // to it.
   public void readFields(int length, final DataInput in) throws IOException {
+    this.rowCache = null;
     this.length = length;
     this.offset = 0;
     this.bytes = new byte[this.length];
diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
index fae690226c15..786d2df52239 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
@@ -19,6 +19,8 @@
  */
 package org.apache.hadoop.hbase;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.IOException;
 import java.util.Set;
 import java.util.TreeSet;
@@ -31,6 +33,7 @@
 import org.apache.hadoop.hbase.KeyValue.MetaComparator;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
 import org.junit.experimental.categories.Category;
 
 @Category(SmallTests.class)
@@ -407,6 +410,25 @@ public void testCreateKeyValueFromKey() {
         kv.toString().replaceAll("=[0-9]+", "=0"));
   }
 
+  /**
+   * The row cache is cleared and re-read for the new value
+   *
+   * @throws IOException
+   */
+  public void testReadFields() throws IOException {
+    KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("cf1"),
+        Bytes.toBytes("qualifier1"), 12345L, Bytes.toBytes("value1"));
+    kv1.getRow(); // set row cache of kv1
+    KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("cf2"),
+        Bytes.toBytes("qualifier2"), 12345L, Bytes.toBytes("value2"));
+    kv1.readFields(new DataInputStream(new ByteArrayInputStream(WritableUtils
+        .toByteArray(kv2))));
+    // check equality
+    assertEquals(kv1, kv2);
+    // check cache state (getRow() return the cached value if the cache is set)
+    assertTrue(Bytes.equals(kv1.getRow(), kv2.getRow()));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From d7f7bdc78622b6716db3a67ecfa6d790a78f44f0 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 5 Apr 2012 22:05:56 +0000
Subject: [PATCH 0104/1540] HBASE-5711 Tests are failing with incorrect data
 directory permissions -- RE-REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310101 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 3c71d50e201d..19fca5803f59 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -425,9 +425,6 @@ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
     // It's also deprecated
     System.setProperty("test.cache.data", this.clusterTestDir.toString());
 
-    // Set the permissions for dfs data directories
-    this.conf.set("dfs.datanode.data.dir.perm", "700");
-    
     // Ok, now we can start
     this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
       true, null, null, hosts, null);

From 8e3845abc157cca259f395f05c45fa4f476a13eb Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 6 Apr 2012 13:41:59 +0000
Subject: [PATCH 0105/1540] HBASE-5615 revert due to race condition in case
 master dies

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310322 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/AssignmentManager.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index b20225710118..a2eb6d6ebf8a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2459,7 +2459,6 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
       ServerName regionLocation = region.getSecond();
       if (regionInfo == null) continue;
       String tableName = regionInfo.getTableNameAsString();
-      if (regionInfo.isOffline() && regionInfo.isSplit()) continue;
       if (regionLocation == null) {
         // regionLocation could be null if createTable didn't finish properly.
         // When createTable is in progress, HMaster restarts.

From 8824147ffee392350e9141b46d83174a0bce1bf4 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 6 Apr 2012 16:06:33 +0000
Subject: [PATCH 0106/1540] HBASE-5680 Improve compatibility warning about
 HBase with Hadoop 0.23.x

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310433 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/HMaster.java | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 014b5a6c05f5..0308f85a084e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -343,7 +343,17 @@ public void run() {
         loop();
       }
     } catch (Throwable t) {
-      abort("Unhandled exception. Starting shutdown.", t);
+      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
+      if (t instanceof NoClassDefFoundError && 
+          t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
+          // improved error message for this special case
+          abort("HBase is having a problem with its Hadoop jars.  You may need to "
+              + "recompile HBase against Hadoop version "
+              +  org.apache.hadoop.util.VersionInfo.getVersion()
+              + " or change your hadoop jars to start properly", t);
+      } else {
+        abort("Unhandled exception. Starting shutdown.", t);
+      }
     } finally {
       startupStatus.cleanup();
       

From 232f89dffc7f1440bfeb721165b65437ac82def0 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 6 Apr 2012 22:42:36 +0000
Subject: [PATCH 0107/1540] HBASE-5734 Change hbck sideline root (Jimmy Xiang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310628 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index c49181c856b2..2f7b4423ea08 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -819,7 +819,8 @@ private TreeMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
 
   private Path getSidelineDir() throws IOException {
     Path hbaseDir = FSUtils.getRootDir(conf);
-    Path backupDir = new Path(hbaseDir.getParent(), hbaseDir.getName() + "-"
+    Path hbckDir = new Path(hbaseDir.getParent(), "hbck");
+    Path backupDir = new Path(hbckDir, hbaseDir.getName() + "-"
         + startMillis);
     return backupDir;
   }

From a875882095e5f0994d346a148edaf53ed4affbc4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 6 Apr 2012 23:38:56 +0000
Subject: [PATCH 0108/1540] HBASE-5736
 ThriftServerRunner.HbaseHandler.mutateRow() does not use ByteBuffer correctly
 (Scott Chen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310638 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift/ThriftServerRunner.java      | 12 ++++++------
 .../apache/hadoop/hbase/thrift/TestThriftServer.java | 10 +++++++---
 .../hadoop/hbase/thrift/TestThriftServerCmdLine.java |  7 ++++---
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 859d37bbb025..02ece7b9bee3 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -971,11 +971,11 @@ public void mutateRowTs(ByteBuffer tableName, ByteBuffer row,
           } else {
             if(famAndQf.length == 1) {
               put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY,
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             } else {
               put.add(famAndQf[0], famAndQf[1],
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             }
             put.setWriteToWAL(m.writeToWAL);
@@ -1029,11 +1029,11 @@ public void mutateRowsTs(
           } else {
             if(famAndQf.length == 1) {
               put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY,
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             } else {
               put.add(famAndQf[0], famAndQf[1],
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             }
             put.setWriteToWAL(m.writeToWAL);
@@ -1398,8 +1398,8 @@ private static void addAttributes(OperationWithAttributes op,
       return;
     }
     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
-      String name = Bytes.toStringBinary(entry.getKey());
-      byte[] value =  Bytes.toBytes(entry.getValue());
+      String name = Bytes.toStringBinary(getBytes(entry.getKey()));
+      byte[] value =  getBytes(entry.getValue());
       op.setAttribute(name, value);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 0cecbb943ca8..1b531eb0a452 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -203,9 +203,13 @@ public static void dropTestTables(Hbase.Iface handler) throws Exception {
    * @throws Exception
    */
   public void doTestTableMutations() throws Exception {
-    // Setup
     ThriftServerRunner.HBaseHandler handler =
       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
+    doTestTableMutations(handler);
+  }
+
+  public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
+    // Setup
     handler.createTable(tableAname, getColumnDescriptors());
 
     // Apply a few Mutations to rowA
@@ -261,7 +265,7 @@ public void doTestTableMutations() throws Exception {
     handler.mutateRow(tableAname, rowAname, mutations, null);
     TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
     assertEquals(rowAname, rowResult3.row);
-    assertEquals(0, rowResult3.columns.get(columnAname).value.array().length);
+    assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
 
     // Teardown
     handler.disableTable(tableAname);
@@ -512,7 +516,7 @@ private List<Mutation> getMutations() {
    * (rowB, columnA): place valueC
    * (rowB, columnB): place valueD
    */
-  private List<BatchMutation> getBatchMutations() {
+  private static List<BatchMutation> getBatchMutations() {
     List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
 
     // Mutations to rowA.  You can't mix delete and put anymore.
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index 6f1872d138e1..f3df9ed62c26 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -59,7 +59,7 @@
 @RunWith(Parameterized.class)
 public class TestThriftServerCmdLine {
 
-  public static final Log LOG = 
+  public static final Log LOG =
       LogFactory.getLog(TestThriftServerCmdLine.class);
 
   private final ImplType implType;
@@ -72,7 +72,7 @@ public class TestThriftServerCmdLine {
 
   private Thread cmdLineThread;
   private volatile Exception cmdLineException;
-  
+
   private Exception clientSideException;
 
   private ThriftServer thriftServer;
@@ -139,7 +139,7 @@ public void run() {
     cmdLineThread.start();
   }
 
-  @Test(timeout=30 * 1000)
+  @Test(timeout=60 * 1000)
   public void testRunThriftServer() throws Exception {
     List<String> args = new ArrayList<String>();
     if (implType != null) {
@@ -204,6 +204,7 @@ private void talkToThriftServer() throws Exception {
       Hbase.Client client = new Hbase.Client(prot);
       TestThriftServer.doTestTableCreateDrop(client);
       TestThriftServer.doTestGetTableRegions(client);
+      TestThriftServer.doTestTableMutations(client);
     } finally {
       sock.close();
     }

From b8f4b86090d31ce70331022de406598e2c27c2ed Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 6 Apr 2012 23:42:42 +0000
Subject: [PATCH 0109/1540] HBASE-5736 revert TestThriftServer doesn't compile

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310643 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift/ThriftServerRunner.java      | 12 ++++++------
 .../apache/hadoop/hbase/thrift/TestThriftServer.java | 10 +++-------
 .../hadoop/hbase/thrift/TestThriftServerCmdLine.java |  7 +++----
 3 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 02ece7b9bee3..859d37bbb025 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -971,11 +971,11 @@ public void mutateRowTs(ByteBuffer tableName, ByteBuffer row,
           } else {
             if(famAndQf.length == 1) {
               put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY,
-                  m.value != null ? getBytes(m.value)
+                  m.value != null ? m.value.array()
                       : HConstants.EMPTY_BYTE_ARRAY);
             } else {
               put.add(famAndQf[0], famAndQf[1],
-                  m.value != null ? getBytes(m.value)
+                  m.value != null ? m.value.array()
                       : HConstants.EMPTY_BYTE_ARRAY);
             }
             put.setWriteToWAL(m.writeToWAL);
@@ -1029,11 +1029,11 @@ public void mutateRowsTs(
           } else {
             if(famAndQf.length == 1) {
               put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY,
-                  m.value != null ? getBytes(m.value)
+                  m.value != null ? m.value.array()
                       : HConstants.EMPTY_BYTE_ARRAY);
             } else {
               put.add(famAndQf[0], famAndQf[1],
-                  m.value != null ? getBytes(m.value)
+                  m.value != null ? m.value.array()
                       : HConstants.EMPTY_BYTE_ARRAY);
             }
             put.setWriteToWAL(m.writeToWAL);
@@ -1398,8 +1398,8 @@ private static void addAttributes(OperationWithAttributes op,
       return;
     }
     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
-      String name = Bytes.toStringBinary(getBytes(entry.getKey()));
-      byte[] value =  getBytes(entry.getValue());
+      String name = Bytes.toStringBinary(entry.getKey());
+      byte[] value =  Bytes.toBytes(entry.getValue());
       op.setAttribute(name, value);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 1b531eb0a452..0cecbb943ca8 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -203,13 +203,9 @@ public static void dropTestTables(Hbase.Iface handler) throws Exception {
    * @throws Exception
    */
   public void doTestTableMutations() throws Exception {
+    // Setup
     ThriftServerRunner.HBaseHandler handler =
       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
-    doTestTableMutations(handler);
-  }
-
-  public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
-    // Setup
     handler.createTable(tableAname, getColumnDescriptors());
 
     // Apply a few Mutations to rowA
@@ -265,7 +261,7 @@ public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
     handler.mutateRow(tableAname, rowAname, mutations, null);
     TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
     assertEquals(rowAname, rowResult3.row);
-    assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
+    assertEquals(0, rowResult3.columns.get(columnAname).value.array().length);
 
     // Teardown
     handler.disableTable(tableAname);
@@ -516,7 +512,7 @@ private List<Mutation> getMutations() {
    * (rowB, columnA): place valueC
    * (rowB, columnB): place valueD
    */
-  private static List<BatchMutation> getBatchMutations() {
+  private List<BatchMutation> getBatchMutations() {
     List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
 
     // Mutations to rowA.  You can't mix delete and put anymore.
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index f3df9ed62c26..6f1872d138e1 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -59,7 +59,7 @@
 @RunWith(Parameterized.class)
 public class TestThriftServerCmdLine {
 
-  public static final Log LOG =
+  public static final Log LOG = 
       LogFactory.getLog(TestThriftServerCmdLine.class);
 
   private final ImplType implType;
@@ -72,7 +72,7 @@ public class TestThriftServerCmdLine {
 
   private Thread cmdLineThread;
   private volatile Exception cmdLineException;
-
+  
   private Exception clientSideException;
 
   private ThriftServer thriftServer;
@@ -139,7 +139,7 @@ public void run() {
     cmdLineThread.start();
   }
 
-  @Test(timeout=60 * 1000)
+  @Test(timeout=30 * 1000)
   public void testRunThriftServer() throws Exception {
     List<String> args = new ArrayList<String>();
     if (implType != null) {
@@ -204,7 +204,6 @@ private void talkToThriftServer() throws Exception {
       Hbase.Client client = new Hbase.Client(prot);
       TestThriftServer.doTestTableCreateDrop(client);
       TestThriftServer.doTestGetTableRegions(client);
-      TestThriftServer.doTestTableMutations(client);
     } finally {
       sock.close();
     }

From b03057d6ccd5adffaadd0d73d165d4ec8777796b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 7 Apr 2012 15:30:19 +0000
Subject: [PATCH 0110/1540] HBASE-5689 Skipping RecoveredEdits may cause data
 loss (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310787 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 26 ++----
 .../hbase/regionserver/wal/HLogSplitter.java  | 48 +++++++---
 .../hbase/regionserver/TestHRegion.java       | 92 +++++++++++++++++++
 3 files changed, 137 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e4d02584ee5f..9a9d14d9658a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2697,7 +2697,6 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
     long seqid = minSeqId;
     NavigableSet<Path> files = HLog.getSplitEditFilesSorted(this.fs, regiondir);
     if (files == null || files.isEmpty()) return seqid;
-    boolean checkSafeToSkip = true;
     for (Path edits: files) {
       if (edits == null || !this.fs.exists(edits)) {
         LOG.warn("Null or non-existent edits file: " + edits);
@@ -2705,22 +2704,15 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
       }
       if (isZeroLengthThenDelete(this.fs, edits)) continue;
 
-      if (checkSafeToSkip) {
-        Path higher = files.higher(edits);
-        long maxSeqId = Long.MAX_VALUE;
-        if (higher != null) {
-          // Edit file name pattern, HLog.EDITFILES_NAME_PATTERN: "-?[0-9]+"
-          String fileName = higher.getName();
-          maxSeqId = Math.abs(Long.parseLong(fileName));
-        }
-        if (maxSeqId <= minSeqId) {
-          String msg = "Maximum possible sequenceid for this log is " + maxSeqId
-              + ", skipped the whole file, path=" + edits;
-          LOG.debug(msg);
-          continue;
-        } else {
-          checkSafeToSkip = false;
-        }
+      long maxSeqId = Long.MAX_VALUE;
+      String fileName = edits.getName();
+      maxSeqId = Math.abs(Long.parseLong(fileName));
+      if (maxSeqId <= minSeqId) {
+        String msg = "Maximum sequenceid for this log is " + maxSeqId
+            + " and minimum sequenceid for the region is " + minSeqId
+            + ", skipped the whole file, path=" + edits;
+        LOG.debug(msg);
+        continue;
       }
 
       try {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 193f15a0a57e..1ddec19577ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -424,6 +424,7 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
           }
         }
         wap.w.append(entry);
+        outputSink.updateRegionMaximumEditLogSeqNum(entry);
         editsCount++;
         // If sufficient edits have passed OR we've opened a few files, check if
         // we should report progress.
@@ -453,7 +454,8 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
       throw e;
     } finally {
       int n = 0;
-      for (Object o : logWriters.values()) {
+      for (Map.Entry<byte[], Object> logWritersEntry : logWriters.entrySet()) {
+        Object o = logWritersEntry.getValue();
         long t1 = EnvironmentEdgeManager.currentTimeMillis();
         if ((t1 - last_report_at) > period) {
           last_report_at = t;
@@ -469,7 +471,8 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
         WriterAndPath wap = (WriterAndPath)o;
         wap.w.close();
         LOG.debug("Closed " + wap.p);
-        Path dst = getCompletedRecoveredEditsFilePath(wap.p);
+        Path dst = getCompletedRecoveredEditsFilePath(wap.p, outputSink
+            .getRegionMaximumEditLogSeqNum(logWritersEntry.getKey()));
         if (!dst.equals(wap.p) && fs.exists(dst)) {
           LOG.warn("Found existing old edits file. It could be the "
               + "result of a previous failed split attempt. Deleting " + dst
@@ -486,6 +489,7 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
           if (!fs.rename(wap.p, dst)) {
             throw new IOException("Failed renaming " + wap.p + " to " + dst);
           }
+          LOG.debug("Rename " + wap.p + " to " + dst);
         }
       }
       String msg = "Processed " + editsCount + " edits across " + n + " regions" +
@@ -679,16 +683,16 @@ static String getTmpRecoveredEditsFileName(String fileName) {
   }
 
   /**
-   * Convert path to a file under RECOVERED_EDITS_DIR directory without
-   * RECOVERED_LOG_TMPFILE_SUFFIX
+   * Get the completed recovered edits file path, renaming it to be by last edit
+   * in the file from its first edit. Then we could use the name to skip
+   * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.
    * @param srcPath
-   * @return dstPath without RECOVERED_LOG_TMPFILE_SUFFIX
+   * @param maximumEditLogSeqNum
+   * @return dstPath take file's last edit log seq num as the name
    */
-  static Path getCompletedRecoveredEditsFilePath(Path srcPath) {
-    String fileName = srcPath.getName();
-    if (fileName.endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
-      fileName = fileName.split(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)[0];
-    }
+  static Path getCompletedRecoveredEditsFilePath(Path srcPath,
+      Long maximumEditLogSeqNum) {
+    String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);
     return new Path(srcPath.getParent(), fileName);
   }
 
@@ -1025,6 +1029,7 @@ private void doRun() throws IOException {
       }
     }
 
+
     private void writeBuffer(RegionEntryBuffer buffer) throws IOException {
       List<Entry> entries = buffer.entryBuffer;
       if (entries.isEmpty()) {
@@ -1048,6 +1053,7 @@ private void writeBuffer(RegionEntryBuffer buffer) throws IOException {
             }
           }
           wap.w.append(logEntry);
+          outputSink.updateRegionMaximumEditLogSeqNum(logEntry);
           editsCount++;
         }
         // Pass along summary statistics
@@ -1136,6 +1142,8 @@ Path convertRegionEditsToTemp(Path rootdir, Path edits, String tmpname) {
   class OutputSink {
     private final Map<byte[], WriterAndPath> logWriters = Collections.synchronizedMap(
           new TreeMap<byte[], WriterAndPath>(Bytes.BYTES_COMPARATOR));
+    private final Map<byte[], Long> regionMaximumEditLogSeqNum = Collections
+        .synchronizedMap(new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR));
     private final List<WriterThread> writerThreads = Lists.newArrayList();
 
     /* Set of regions which we've decided should not output edits */
@@ -1202,8 +1210,11 @@ private List<Path> closeStreams() throws IOException {
       List<Path> paths = new ArrayList<Path>();
       List<IOException> thrown = Lists.newArrayList();
       closeLogWriters(thrown);
-      for (WriterAndPath wap : logWriters.values()) {
-        Path dst = getCompletedRecoveredEditsFilePath(wap.p);
+      for (Map.Entry<byte[], WriterAndPath> logWritersEntry : logWriters
+          .entrySet()) {
+        WriterAndPath wap = logWritersEntry.getValue();
+        Path dst = getCompletedRecoveredEditsFilePath(wap.p,
+            regionMaximumEditLogSeqNum.get(logWritersEntry.getKey()));
         try {
           if (!dst.equals(wap.p) && fs.exists(dst)) {
             LOG.warn("Found existing old edits file. It could be the "
@@ -1221,6 +1232,7 @@ private List<Path> closeStreams() throws IOException {
             if (!fs.rename(wap.p, dst)) {
               throw new IOException("Failed renaming " + wap.p + " to " + dst);
             }
+            LOG.debug("Rename " + wap.p + " to " + dst);
           }
         } catch (IOException ioe) {
           LOG.error("Couldn't rename " + wap.p + " to " + dst, ioe);
@@ -1287,6 +1299,18 @@ WriterAndPath getWriterAndPath(Entry entry) throws IOException {
       return ret;
     }
 
+    /**
+     * Update region's maximum edit log SeqNum.
+     */
+    void updateRegionMaximumEditLogSeqNum(Entry entry) {
+      regionMaximumEditLogSeqNum.put(entry.getKey().getEncodedRegionName(),
+          entry.getKey().getLogSeqNum());
+    }
+
+    Long getRegionMaximumEditLogSeqNum(byte[] region) {
+      return regionMaximumEditLogSeqNum.get(region);
+    }
+
     /**
      * @return a map from encoded region ID to the number of edits written out
      * for that region.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index e2db6886c882..c0ac12ca5e36 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -54,9 +54,11 @@
 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -67,6 +69,7 @@
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
@@ -136,6 +139,95 @@ protected void tearDown() throws Exception {
     SchemaMetrics.validateMetricChanges(startingMetrics);
   }
 
+  public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
+    final int NUM_MASTERS = 1;
+    final int NUM_RS = 3;
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+
+    try {
+      final byte[] TABLENAME = Bytes
+          .toBytes("testDataCorrectnessReplayingRecoveredEdits");
+      final byte[] FAMILY = Bytes.toBytes("family");
+      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+      HMaster master = cluster.getMaster();
+
+      // Create table
+      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+      desc.addFamily(new HColumnDescriptor(FAMILY));
+      HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin();
+      hbaseAdmin.createTable(desc);
+
+      assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+
+      // Put data: r1->v1
+      HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
+      putDataAndVerify(table, "r1", FAMILY, "v1", 1);
+
+      // Move region to target server
+      HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo();
+      int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
+      HRegionServer originServer = cluster.getRegionServer(originServerNum);
+      int targetServerNum = NUM_RS - 1 - originServerNum;
+      HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
+      hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
+          Bytes.toBytes(targetServer.getServerName().getServerName()));
+      do {
+        Thread.sleep(1);
+      } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
+
+      // Put data: r2->v2
+      putDataAndVerify(table, "r2", FAMILY, "v2", 2);
+
+      // Move region to origin server
+      hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
+          Bytes.toBytes(originServer.getServerName().getServerName()));
+      do {
+        Thread.sleep(1);
+      } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);
+
+      // Put data: r3->v3
+      putDataAndVerify(table, "r3", FAMILY, "v3", 3);
+
+      // Kill target server
+      targetServer.kill();
+      cluster.getRegionServerThreads().get(targetServerNum).join();
+      // Wait until finish processing of shutdown
+      while (master.getServerManager().areDeadServersInProgress()) {
+        Thread.sleep(5);
+      }
+      // Kill origin server
+      originServer.kill();
+      cluster.getRegionServerThreads().get(originServerNum).join();
+
+      // Put data: r4->v4
+      putDataAndVerify(table, "r4", FAMILY, "v4", 4);
+
+    } finally {
+      TEST_UTIL.shutdownMiniCluster();
+    }
+  }
+
+  private void putDataAndVerify(HTable table, String row, byte[] family,
+      String value, int verifyNum) throws IOException {
+    System.out.println("=========Putting data :" + row);
+    Put put = new Put(Bytes.toBytes(row));
+    put.add(family, Bytes.toBytes("q1"), Bytes.toBytes(value));
+    table.put(put);
+    ResultScanner resultScanner = table.getScanner(new Scan());
+    List<Result> results = new ArrayList<Result>();
+    while (true) {
+      Result r = resultScanner.next();
+      if (r == null)
+        break;
+      results.add(r);
+    }
+    resultScanner.close();
+    if (results.size() != verifyNum) {
+      System.out.println(results);
+    }
+    assertEquals(verifyNum, results.size());
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   // New tests that doesn't spin up a mini cluster but rather just test the
   // individual code pieces in the HRegion. Putting files locally in

From e19bce4ac76e14ca007be9840e75cb8737c26e7e Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 7 Apr 2012 23:16:43 +0000
Subject: [PATCH 0111/1540] HBASE-5735 Clearer warning message when connecting
 a non-secure HBase client to a secure HBase server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310915 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/SecureServer.java  | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index bf45bb1fa5c9..e6f248d5ad24 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -47,6 +47,8 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 
+import com.google.common.collect.ImmutableSet;
+
 import javax.security.sasl.Sasl;
 import javax.security.sasl.SaslException;
 import javax.security.sasl.SaslServer;
@@ -83,6 +85,7 @@ public abstract class SecureServer extends HBaseServer {
   // 3 : Introduce the protocol into the RPC connection header
   // 4 : Introduced SASL security layer
   public static final byte CURRENT_VERSION = 4;
+  public static final Set<Byte> INSECURE_VERSIONS = ImmutableSet.of((byte) 3);
 
   public static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.SecureServer");
   private static final Log AUDITLOG =
@@ -400,10 +403,17 @@ public int readAndProcess() throws IOException, InterruptedException {
           dataLengthBuffer.flip();
           if (!HEADER.equals(dataLengthBuffer) || version != CURRENT_VERSION) {
             //Warning is ok since this is not supposed to happen.
-            LOG.warn("Incorrect header or version mismatch from " +
-                     hostAddress + ":" + remotePort +
-                     " got version " + version +
-                     " expected version " + CURRENT_VERSION);
+            if (INSECURE_VERSIONS.contains(version)) {
+              LOG.warn("An insecure client (version '" + version + "') is attempting to connect " +
+                  " to this version '" + CURRENT_VERSION + "' secure server from " +
+                  hostAddress + ":" + remotePort);
+            } else {
+              LOG.warn("Incorrect header or version mismatch from " +
+                  hostAddress + ":" + remotePort +
+                  " got version " + version +
+                  " expected version " + CURRENT_VERSION);              
+            }
+            
             return -1;
           }
           dataLengthBuffer.clear();

From cb84dcde8359fed1f783635d7b6c86e6ea11ab0f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 7 Apr 2012 23:32:14 +0000
Subject: [PATCH 0112/1540] HBASE-5618 SplitLogManager - prevent unnecessary
 attempts to resubmits

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310922 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/SplitLogManager.java  | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index b72d7f018a93..84dd61b87ca0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -537,6 +537,7 @@ private boolean resubmit(String path, Task task,
           version) == false) {
         LOG.debug("failed to resubmit task " + path +
             " version changed");
+        task.heartbeatNoDetails(EnvironmentEdgeManager.currentTimeMillis());
         return false;
       }
     } catch (NoNodeException e) {
@@ -544,6 +545,11 @@ private boolean resubmit(String path, Task task,
           " task done (or forced done by removing the znode)");
       getDataSetWatchSuccess(path, null, Integer.MIN_VALUE);
       return false;
+    } catch (KeeperException.BadVersionException e) {
+      LOG.debug("failed to resubmit task " + path +
+          " version changed");
+      task.heartbeatNoDetails(EnvironmentEdgeManager.currentTimeMillis());
+      return false;
     } catch (KeeperException e) {
       tot_mgr_resubmit_failed.incrementAndGet();
       LOG.warn("failed to resubmit " + path, e);
@@ -714,7 +720,12 @@ Task findOrCreateOrphanTask(String path) {
 
   @Override
   public void nodeDataChanged(String path) {
-    if (tasks.get(path) != null || ZKSplitLog.isRescanNode(watcher, path)) {
+    Task task;
+    task = tasks.get(path);
+    if (task != null || ZKSplitLog.isRescanNode(watcher, path)) {
+      if (task != null) {
+        task.heartbeatNoDetails(EnvironmentEdgeManager.currentTimeMillis());
+      }
       getDataSetWatch(path, zkretries);
     }
   }
@@ -810,7 +821,11 @@ public boolean isOrphan() {
     }
 
     public boolean isUnassigned() {
-      return (last_update == -1);
+      return (cur_worker_name == null);
+    }
+
+    public void heartbeatNoDetails(long time) {
+      last_update = time;
     }
 
     public void heartbeat(long time, int version, String worker) {

From 2d8e9d152a32c2ed58b142e651c774b53a2365be Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 8 Apr 2012 00:41:24 +0000
Subject: [PATCH 0113/1540] HBASE-5720 HFileDataBlockEncoderImpl uses wrong
 header size when reading HFiles with no checksums (Matt Corgan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1310932 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileBlock.java     | 17 +++++++++++
 .../io/hfile/HFileDataBlockEncoderImpl.java   | 28 ++++++++++++-------
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index c99dcbdd7c7b..052e49cc1ada 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -2072,6 +2072,23 @@ static private int headerSize(int minorVersion) {
     return HEADER_SIZE;
   }
 
+  /**
+   * Return the appropriate DUMMY_HEADER for the minor version
+   */
+  public byte[] getDummyHeaderForVersion() {
+    return getDummyHeaderForVersion(minorVersion);
+  }
+
+  /**
+   * Return the appropriate DUMMY_HEADER for the minor version
+   */
+  static private byte[] getDummyHeaderForVersion(int minorVersion) {
+    if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
+      return DUMMY_HEADER_NO_CHECKSUM;
+    }
+    return DUMMY_HEADER;
+  }
+
   /**
    * Convert the contents of the block header into a human readable string.
    * This is mostly helpful for debugging. This assumes that the block
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
index 7e369a896eba..01842833dc49 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
@@ -23,6 +23,7 @@
 
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -65,19 +66,26 @@ public HFileDataBlockEncoderImpl(DataBlockEncoding onDisk,
   public static HFileDataBlockEncoder createFromFileInfo(
       FileInfo fileInfo, DataBlockEncoding preferredEncodingInCache)
       throws IOException {
-    byte[] dataBlockEncodingType =
-        fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
-    if (dataBlockEncodingType == null) {
+    
+    boolean hasPreferredCacheEncoding = preferredEncodingInCache != null
+        && preferredEncodingInCache != DataBlockEncoding.NONE;
+
+    byte[] dataBlockEncodingType = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) {
       return NoOpDataBlockEncoder.INSTANCE;
     }
 
-    String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
     DataBlockEncoding onDisk;
-    try {
-      onDisk = DataBlockEncoding.valueOf(dataBlockEncodingStr);
-    } catch (IllegalArgumentException ex) {
-      throw new IOException("Invalid data block encoding type in file info: " +
-          dataBlockEncodingStr, ex);
+    if (dataBlockEncodingType == null) {
+      onDisk = DataBlockEncoding.NONE;
+    }else {
+      String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
+      try {
+        onDisk = DataBlockEncoding.valueOf(dataBlockEncodingStr);
+      } catch (IllegalArgumentException ex) {
+        throw new IOException("Invalid data block encoding type in file info: "
+            + dataBlockEncodingStr, ex);
+      }
     }
 
     DataBlockEncoding inCache;
@@ -194,7 +202,7 @@ private HFileBlock encodeDataBlock(HFileBlock block,
       DataBlockEncoding algo, boolean includesMemstoreTS) {
     ByteBuffer compressedBuffer = encodeBufferToHFileBlockBuffer(
         block.getBufferWithoutHeader(), algo, includesMemstoreTS,
-        HFileBlock.DUMMY_HEADER);
+        block.getDummyHeaderForVersion());
     int sizeWithoutHeader = compressedBuffer.limit() - block.headerSize();
     HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA,
         block.getOnDiskSizeWithoutHeader(),

From f334cca12d14c279824aecd3e17c500385a6cac3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 8 Apr 2012 21:31:02 +0000
Subject: [PATCH 0114/1540] HBASE-5656 LoadIncrementalHFiles createTable should
 detect and set compression algorithm(Cosmin Lehene)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1311104 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 2aab028107da..922e62165143 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -655,6 +655,11 @@ private void createTable(String tableName, String dirPath) throws Exception {
             new CacheConfig(getConf()));
         final byte[] first, last;
         try {
+          if (hcd.getCompressionType() != reader.getCompressionAlgorithm()) {
+            hcd.setCompressionType(reader.getCompressionAlgorithm());
+            LOG.info("Setting compression " + hcd.getCompressionType().name() +
+                     " for family " + hcd.toString());
+          }
           reader.loadFileInfo();
           first = reader.getFirstRowKey();
           last =  reader.getLastRowKey();

From b317497a4370bc82b90244e26c508876d732f220 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 9 Apr 2012 22:28:20 +0000
Subject: [PATCH 0115/1540] HBASE-5748 Enable lib directory in jar file for
 coprocessor

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1311499 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/CoprocessorHost.java    | 19 ++++
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 13 +--
 .../apache/hadoop/hbase/TestZooKeeper.java    | 12 +++
 .../hbase/coprocessor/TestClassLoading.java   | 87 +++++++++++++++++++
 4 files changed, 126 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index c08fe2b99507..6268afb612e9 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -37,12 +37,16 @@
 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.io.IOUtils;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.*;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 
 /**
  * Provides the common setup framework and runtime services for coprocessor
@@ -191,6 +195,21 @@ public E load(Path path, String className, int priority,
       // method which returns URLs for as long as it is available
       List<URL> paths = new ArrayList<URL>();
       paths.add(new File(dst.toString()).getCanonicalFile().toURL());
+
+      JarFile jarFile = new JarFile(dst.toString());
+      Enumeration<JarEntry> entries = jarFile.entries();
+      while (entries.hasMoreElements()) {
+        JarEntry entry = entries.nextElement();
+        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
+          File file = new File(System.getProperty("java.io.tmpdir") +
+              java.io.File.separator +"." + pathPrefix +
+              "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
+          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
+          file.deleteOnExit();
+          paths.add(file.toURL());
+        }
+      }
+
       StringTokenizer st = new StringTokenizer(cp, File.pathSeparator);
       while (st.hasMoreTokens()) {
         paths.add((new File(st.nextToken())).getCanonicalFile().toURL());
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 719a17691abc..898cd047bd6d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -26,6 +26,7 @@
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
 
@@ -594,11 +595,13 @@ public static List<NodeAndData> getChildDataAndWatchForNewChildren(
       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
     List<String> nodes =
       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
-    List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
-    for (String node: nodes) {
-      String nodePath = ZKUtil.joinZNode(baseNode, node);
-      byte [] data = ZKUtil.getDataAndWatch(zkw, nodePath);
-      newNodes.add(new NodeAndData(nodePath, data));
+    List<NodeAndData> newNodes = Collections.emptyList();
+    if (nodes != null) {
+      for (String node : nodes) {
+        String nodePath = ZKUtil.joinZNode(baseNode, node);
+        byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
+        newNodes.add(new NodeAndData(nodePath, data));
+      }
     }
     return newNodes;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 5d3776ade82a..7e5facee828e 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -296,6 +296,18 @@ public void testCreateSilentIsReallySilent() throws InterruptedException,
 
     ZKUtil.createAndFailSilent(zk2, aclZnode);
  }
+  
+  @Test
+  /**
+   * Test should not fail with NPE when getChildDataAndWatchForNewChildren
+   * invoked with wrongNode
+   */
+  public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
+      throws Exception {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+        "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
+    ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index 368a0e524983..1918b1e7e4a0 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -382,6 +382,93 @@ public void testHBase3810() throws Exception {
     assertFalse("Configuration key 'k4' wasn't configured", found5_k4);
   }
 
+  @Test
+  public void testClassLoadingFromLibDirInJar() throws Exception {
+    FileSystem fs = cluster.getFileSystem();
+
+    File innerJarFile1 = buildCoprocessorJar(cpName1);
+    File innerJarFile2 = buildCoprocessorJar(cpName2);
+    File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
+
+    byte buffer[] = new byte[BUFFER_SIZE];
+    // Open archive file
+    FileOutputStream stream = new FileOutputStream(outerJarFile);
+    JarOutputStream out = new JarOutputStream(stream, new Manifest());
+
+    for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) {
+      // Add archive entry
+      JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName());
+      jarAdd.setTime(jarFile.lastModified());
+      out.putNextEntry(jarAdd);
+  
+      // Write file to archive
+      FileInputStream in = new FileInputStream(jarFile);
+      while (true) {
+        int nRead = in.read(buffer, 0, buffer.length);
+        if (nRead <= 0)
+          break;
+        out.write(buffer, 0, nRead);
+      }
+      in.close();
+    }
+    out.close();
+    stream.close();
+    LOG.info("Adding jar file to outer jar file completed");
+
+    // copy the jars into dfs
+    fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
+      new Path(fs.getUri().toString() + Path.SEPARATOR));
+    String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
+      outerJarFile.getName();
+    assertTrue("Copy jar file to HDFS failed.",
+      fs.exists(new Path(jarFileOnHDFS)));
+    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
+
+    // create a table that references the coprocessors
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor("test"));
+      // without configuration values
+    htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
+      "|" + Coprocessor.PRIORITY_USER);
+      // with configuration values
+    htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
+      "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
+    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    if (admin.tableExists(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+    }
+    admin.createTable(htd);
+
+    // verify that the coprocessors were loaded
+    boolean found1 = false, found2 = false, found2_k1 = false,
+        found2_k2 = false, found2_k3 = false;
+    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
+    for (HRegion region:
+        hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
+      if (region.getRegionNameAsString().startsWith(tableName)) {
+        CoprocessorEnvironment env;
+        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
+        if (env != null) {
+          found1 = true;
+        }
+        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
+        if (env != null) {
+          found2 = true;
+          Configuration conf = env.getConfiguration();
+          found2_k1 = conf.get("k1") != null;
+          found2_k2 = conf.get("k2") != null;
+          found2_k3 = conf.get("k3") != null;
+        }
+      }
+    }
+    assertTrue("Class " + cpName1 + " was missing on a region", found1);
+    assertTrue("Class " + cpName2 + " was missing on a region", found2);
+    assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
+    assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
+    assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
+  }
+
   @Test
   public void testRegionServerCoprocessorsReported() throws Exception {
     // HBASE 4070: Improve region server metrics to report loaded coprocessors

From ced0c3f797dd56f25fe35be47cdcd84037119472 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 9 Apr 2012 22:32:30 +0000
Subject: [PATCH 0116/1540] HBASE-5748 Enable lib directory in jar file for
 coprocessor -- UNDO OVERCOMMIT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1311500 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/CoprocessorHost.java    | 19 ----
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 13 ++-
 .../apache/hadoop/hbase/TestZooKeeper.java    | 12 ---
 .../hbase/coprocessor/TestClassLoading.java   | 87 -------------------
 4 files changed, 5 insertions(+), 126 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 6268afb612e9..c08fe2b99507 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -37,16 +37,12 @@
 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.io.IOUtils;
 
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.*;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
 
 /**
  * Provides the common setup framework and runtime services for coprocessor
@@ -195,21 +191,6 @@ public E load(Path path, String className, int priority,
       // method which returns URLs for as long as it is available
       List<URL> paths = new ArrayList<URL>();
       paths.add(new File(dst.toString()).getCanonicalFile().toURL());
-
-      JarFile jarFile = new JarFile(dst.toString());
-      Enumeration<JarEntry> entries = jarFile.entries();
-      while (entries.hasMoreElements()) {
-        JarEntry entry = entries.nextElement();
-        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
-          File file = new File(System.getProperty("java.io.tmpdir") +
-              java.io.File.separator +"." + pathPrefix +
-              "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
-          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
-          file.deleteOnExit();
-          paths.add(file.toURL());
-        }
-      }
-
       StringTokenizer st = new StringTokenizer(cp, File.pathSeparator);
       while (st.hasMoreTokens()) {
         paths.add((new File(st.nextToken())).getCanonicalFile().toURL());
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 898cd047bd6d..719a17691abc 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -26,7 +26,6 @@
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
 
@@ -595,13 +594,11 @@ public static List<NodeAndData> getChildDataAndWatchForNewChildren(
       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
     List<String> nodes =
       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
-    List<NodeAndData> newNodes = Collections.emptyList();
-    if (nodes != null) {
-      for (String node : nodes) {
-        String nodePath = ZKUtil.joinZNode(baseNode, node);
-        byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
-        newNodes.add(new NodeAndData(nodePath, data));
-      }
+    List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
+    for (String node: nodes) {
+      String nodePath = ZKUtil.joinZNode(baseNode, node);
+      byte [] data = ZKUtil.getDataAndWatch(zkw, nodePath);
+      newNodes.add(new NodeAndData(nodePath, data));
     }
     return newNodes;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 7e5facee828e..5d3776ade82a 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -296,18 +296,6 @@ public void testCreateSilentIsReallySilent() throws InterruptedException,
 
     ZKUtil.createAndFailSilent(zk2, aclZnode);
  }
-  
-  @Test
-  /**
-   * Test should not fail with NPE when getChildDataAndWatchForNewChildren
-   * invoked with wrongNode
-   */
-  public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
-      throws Exception {
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
-        "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
-    ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
-  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index 1918b1e7e4a0..368a0e524983 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -382,93 +382,6 @@ public void testHBase3810() throws Exception {
     assertFalse("Configuration key 'k4' wasn't configured", found5_k4);
   }
 
-  @Test
-  public void testClassLoadingFromLibDirInJar() throws Exception {
-    FileSystem fs = cluster.getFileSystem();
-
-    File innerJarFile1 = buildCoprocessorJar(cpName1);
-    File innerJarFile2 = buildCoprocessorJar(cpName2);
-    File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
-
-    byte buffer[] = new byte[BUFFER_SIZE];
-    // Open archive file
-    FileOutputStream stream = new FileOutputStream(outerJarFile);
-    JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
-    for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) {
-      // Add archive entry
-      JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName());
-      jarAdd.setTime(jarFile.lastModified());
-      out.putNextEntry(jarAdd);
-  
-      // Write file to archive
-      FileInputStream in = new FileInputStream(jarFile);
-      while (true) {
-        int nRead = in.read(buffer, 0, buffer.length);
-        if (nRead <= 0)
-          break;
-        out.write(buffer, 0, nRead);
-      }
-      in.close();
-    }
-    out.close();
-    stream.close();
-    LOG.info("Adding jar file to outer jar file completed");
-
-    // copy the jars into dfs
-    fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
-      new Path(fs.getUri().toString() + Path.SEPARATOR));
-    String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
-      outerJarFile.getName();
-    assertTrue("Copy jar file to HDFS failed.",
-      fs.exists(new Path(jarFileOnHDFS)));
-    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
-
-    // create a table that references the coprocessors
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    htd.addFamily(new HColumnDescriptor("test"));
-      // without configuration values
-    htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
-      "|" + Coprocessor.PRIORITY_USER);
-      // with configuration values
-    htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
-      "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
-    HBaseAdmin admin = new HBaseAdmin(this.conf);
-    if (admin.tableExists(tableName)) {
-      admin.disableTable(tableName);
-      admin.deleteTable(tableName);
-    }
-    admin.createTable(htd);
-
-    // verify that the coprocessors were loaded
-    boolean found1 = false, found2 = false, found2_k1 = false,
-        found2_k2 = false, found2_k3 = false;
-    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
-    for (HRegion region:
-        hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
-      if (region.getRegionNameAsString().startsWith(tableName)) {
-        CoprocessorEnvironment env;
-        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
-        if (env != null) {
-          found1 = true;
-        }
-        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
-        if (env != null) {
-          found2 = true;
-          Configuration conf = env.getConfiguration();
-          found2_k1 = conf.get("k1") != null;
-          found2_k2 = conf.get("k2") != null;
-          found2_k3 = conf.get("k3") != null;
-        }
-      }
-    }
-    assertTrue("Class " + cpName1 + " was missing on a region", found1);
-    assertTrue("Class " + cpName2 + " was missing on a region", found2);
-    assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
-    assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
-    assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
-  }
-
   @Test
   public void testRegionServerCoprocessorsReported() throws Exception {
     // HBASE 4070: Improve region server metrics to report loaded coprocessors

From 961c379e536bc52469c91f557a109e64ff2f132f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 9 Apr 2012 22:33:01 +0000
Subject: [PATCH 0117/1540] HBASE-5748 Enable lib directory in jar file for
 coprocessor -- REDO AFTER OVERCOMMIT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1311501 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/CoprocessorHost.java    | 19 ++++
 .../hbase/coprocessor/TestClassLoading.java   | 87 +++++++++++++++++++
 2 files changed, 106 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index c08fe2b99507..6268afb612e9 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -37,12 +37,16 @@
 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.io.IOUtils;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.*;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 
 /**
  * Provides the common setup framework and runtime services for coprocessor
@@ -191,6 +195,21 @@ public E load(Path path, String className, int priority,
       // method which returns URLs for as long as it is available
       List<URL> paths = new ArrayList<URL>();
       paths.add(new File(dst.toString()).getCanonicalFile().toURL());
+
+      JarFile jarFile = new JarFile(dst.toString());
+      Enumeration<JarEntry> entries = jarFile.entries();
+      while (entries.hasMoreElements()) {
+        JarEntry entry = entries.nextElement();
+        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
+          File file = new File(System.getProperty("java.io.tmpdir") +
+              java.io.File.separator +"." + pathPrefix +
+              "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
+          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
+          file.deleteOnExit();
+          paths.add(file.toURL());
+        }
+      }
+
       StringTokenizer st = new StringTokenizer(cp, File.pathSeparator);
       while (st.hasMoreTokens()) {
         paths.add((new File(st.nextToken())).getCanonicalFile().toURL());
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index 368a0e524983..1918b1e7e4a0 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -382,6 +382,93 @@ public void testHBase3810() throws Exception {
     assertFalse("Configuration key 'k4' wasn't configured", found5_k4);
   }
 
+  @Test
+  public void testClassLoadingFromLibDirInJar() throws Exception {
+    FileSystem fs = cluster.getFileSystem();
+
+    File innerJarFile1 = buildCoprocessorJar(cpName1);
+    File innerJarFile2 = buildCoprocessorJar(cpName2);
+    File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
+
+    byte buffer[] = new byte[BUFFER_SIZE];
+    // Open archive file
+    FileOutputStream stream = new FileOutputStream(outerJarFile);
+    JarOutputStream out = new JarOutputStream(stream, new Manifest());
+
+    for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) {
+      // Add archive entry
+      JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName());
+      jarAdd.setTime(jarFile.lastModified());
+      out.putNextEntry(jarAdd);
+  
+      // Write file to archive
+      FileInputStream in = new FileInputStream(jarFile);
+      while (true) {
+        int nRead = in.read(buffer, 0, buffer.length);
+        if (nRead <= 0)
+          break;
+        out.write(buffer, 0, nRead);
+      }
+      in.close();
+    }
+    out.close();
+    stream.close();
+    LOG.info("Adding jar file to outer jar file completed");
+
+    // copy the jars into dfs
+    fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
+      new Path(fs.getUri().toString() + Path.SEPARATOR));
+    String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
+      outerJarFile.getName();
+    assertTrue("Copy jar file to HDFS failed.",
+      fs.exists(new Path(jarFileOnHDFS)));
+    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
+
+    // create a table that references the coprocessors
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor("test"));
+      // without configuration values
+    htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
+      "|" + Coprocessor.PRIORITY_USER);
+      // with configuration values
+    htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
+      "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
+    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    if (admin.tableExists(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+    }
+    admin.createTable(htd);
+
+    // verify that the coprocessors were loaded
+    boolean found1 = false, found2 = false, found2_k1 = false,
+        found2_k2 = false, found2_k3 = false;
+    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
+    for (HRegion region:
+        hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
+      if (region.getRegionNameAsString().startsWith(tableName)) {
+        CoprocessorEnvironment env;
+        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
+        if (env != null) {
+          found1 = true;
+        }
+        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
+        if (env != null) {
+          found2 = true;
+          Configuration conf = env.getConfiguration();
+          found2_k1 = conf.get("k1") != null;
+          found2_k2 = conf.get("k2") != null;
+          found2_k3 = conf.get("k3") != null;
+        }
+      }
+    }
+    assertTrue("Class " + cpName1 + " was missing on a region", found1);
+    assertTrue("Class " + cpName2 + " was missing on a region", found2);
+    assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
+    assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
+    assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
+  }
+
   @Test
   public void testRegionServerCoprocessorsReported() throws Exception {
     // HBASE 4070: Improve region server metrics to report loaded coprocessors

From 07ada7790ecbe004bef55c66c3f0eeed299c2b36 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 10 Apr 2012 16:14:45 +0000
Subject: [PATCH 0118/1540] HBASE-5758 Forward port "HBASE-4109 Hostname
 returned via reverse dns lookup contains trailing period if configured
 interface is not default

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1311822 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java    | 5 +++--
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java  | 5 +++--
 src/main/java/org/apache/hadoop/hbase/util/Strings.java      | 2 +-
 .../java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java  | 5 +++--
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0308f85a084e..54907a0ce742 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -96,6 +96,7 @@
 import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Sleeper;
+import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
@@ -236,9 +237,9 @@ public HMaster(final Configuration conf)
     // Set how many times to retry talking to another server over HConnection.
     HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
     // Server to handle client requests.
-    String hostname = DNS.getDefaultHost(
+    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
       conf.get("hbase.master.dns.interface", "default"),
-      conf.get("hbase.master.dns.nameserver", "default"));
+      conf.get("hbase.master.dns.nameserver", "default")));
     int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
     // Creation of a HSA will force a resolve.
     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index bd8fb90c81c6..db2300864217 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -144,6 +144,7 @@
 import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Sleeper;
+import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
@@ -392,9 +393,9 @@ public HRegionServer(Configuration conf)
     this.stopped = false;
 
     // Server to handle client requests.
-    String hostname = DNS.getDefaultHost(
+    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
       conf.get("hbase.regionserver.dns.interface", "default"),
-      conf.get("hbase.regionserver.dns.nameserver", "default"));
+      conf.get("hbase.regionserver.dns.nameserver", "default")));
     int port = conf.getInt(HConstants.REGIONSERVER_PORT,
       HConstants.DEFAULT_REGIONSERVER_PORT);
     // Creation of a HSA will force a resolve.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Strings.java b/src/main/java/org/apache/hadoop/hbase/util/Strings.java
index 9cbca9e54247..5e98c597aa72 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Strings.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Strings.java
@@ -72,4 +72,4 @@ public static String domainNamePointerToHostName(String dnPtr) {
       return null;
     return dnPtr.endsWith(".") ? dnPtr.substring(0, dnPtr.length()-1) : dnPtr;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
index d551c6fc74a4..84b79a6a927d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
@@ -33,6 +33,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.server.ServerConfig;
@@ -87,9 +88,9 @@ static void writeMyID(Properties properties) throws IOException {
     long myId = -1;
 
     Configuration conf = HBaseConfiguration.create();
-    String myAddress = DNS.getDefaultHost(
+    String myAddress = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
         conf.get("hbase.zookeeper.dns.interface","default"),
-        conf.get("hbase.zookeeper.dns.nameserver","default"));
+        conf.get("hbase.zookeeper.dns.nameserver","default")));
 
     List<String> ips = new ArrayList<String>();
 

From 2c49cb1b68bf767c1b2f25fc0687108b4f50a936 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 11 Apr 2012 17:20:09 +0000
Subject: [PATCH 0119/1540] HBASE-5599 [hbck] handle NO_VERSION_FILE and
 SHOULD_NOT_BE_DEPLOYED inconsistencies (fulin wang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1324880 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  36 +++++-
 .../hadoop/hbase/util/TestHBaseFsck.java      | 104 +++++++++++++++++-
 .../hbase/util/hbck/HbckTestingUtil.java      |   5 +-
 3 files changed, 137 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 2f7b4423ea08..2eb230d8fb5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -160,6 +160,7 @@ public class HBaseFsck {
   private boolean fixHdfsHoles = false; // fix fs holes?
   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
+  private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
 
   // limit fixes to listed tables, if empty atttempt to fix all
   private List<byte[]> tablesToFix = new ArrayList<byte[]>();
@@ -1001,6 +1002,15 @@ public void loadHdfsRegionDirs() throws IOException, InterruptedException {
     if (!foundVersionFile) {
       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
           "Version file does not exist in root dir " + rootDir);
+      if (shouldFixVersionFile()) {
+        LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
+            + " file.");
+        setShouldRerun();
+        FSUtils.setVersion(fs, rootDir, conf.getInt(
+            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), conf.getInt(
+            HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      }
     }
 
     // level 1:  <HBASE_DIR>/*
@@ -1351,10 +1361,14 @@ else if (!inMeta && !inHdfs && !isDeployed) {
           + " not deployed on any region server.");
       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
-      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, "UNHANDLED CASE:" +
-          " Region " + descriptiveName + " should not be deployed according " +
+      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
+          "Region " + descriptiveName + " should not be deployed according " +
           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
-      // TODO test and handle this case.
+      if (shouldFixAssignments()) {
+        errors.print("Trying to close the region " + descriptiveName);
+        setShouldRerun();
+        HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
+      }
     } else if (inMeta && inHdfs && isMultiplyDeployed) {
       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
           + " is listed in META on region server " + hbi.metaEntry.regionServer
@@ -2729,6 +2743,14 @@ boolean shouldFixHdfsOrphans() {
     return fixHdfsOrphans;
   }
 
+  public void setFixVersionFile(boolean shouldFix) {
+    fixVersionFile = shouldFix;
+  }
+
+  public boolean shouldFixVersionFile() {
+    return fixVersionFile;
+  }
+
   /**
    * @param mm maximum number of regions to merge into a single region.
    */
@@ -2787,9 +2809,11 @@ protected static void printUsageAndExit() {
     System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
     System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
     System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
+    System.err.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
     System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
     System.err.println("");
-    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps");
+    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
+        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile");
     System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
 
     Runtime.getRuntime().exit(-2);
@@ -2852,6 +2876,8 @@ public static void main(String[] args) throws Exception {
         fsck.setFixHdfsOrphans(true);
       } else if (cmd.equals("-fixHdfsOverlaps")) {
         fsck.setFixHdfsOverlaps(true);
+      } else if (cmd.equals("-fixVersionFile")) {
+        fsck.setFixVersionFile(true);
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
@@ -2860,6 +2886,7 @@ public static void main(String[] args) throws Exception {
         fsck.setFixMeta(true);
         fsck.setFixAssignments(true);
         fsck.setFixHdfsOverlaps(true);
+        fsck.setFixVersionFile(true);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
         fsck.setFixHdfsHoles(true);
@@ -2909,6 +2936,7 @@ public static void main(String[] args) throws Exception {
       fsck.setFixMeta(false);
       fsck.setFixHdfsHoles(false);
       fsck.setFixHdfsOverlaps(false);
+      fsck.setFixVersionFile(false);
       fsck.errors.resetErrors();
       code = fsck.onlineHbck();
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 20e2ceea04cd..a06942d38a6c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -23,6 +23,7 @@
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
@@ -36,7 +37,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.ClusterStatus;
@@ -45,6 +45,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
@@ -55,9 +56,15 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -73,6 +80,7 @@ public class TestHBaseFsck {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final static Configuration conf = TEST_UTIL.getConfiguration();
   private final static byte[] FAM = Bytes.toBytes("fam");
+  private final static int REGION_ONLINE_TIMEOUT = 300;
 
   // for the instance, reset every test run
   private HTable tbl;
@@ -851,8 +859,100 @@ public void testNoHdfsTable() throws Exception {
     fail("Should have failed with IOException");
   }
 
+  /**
+   * when the hbase.version file missing, It is fix the fault.
+   */
+  @Test
+  public void testNoVersionFile() throws Exception {
+    // delete the hbase.version file
+    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = rootDir.getFileSystem(conf);
+    Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
+    fs.delete(versionFile, true);
+
+    // test
+    HBaseFsck hbck = doFsck(conf, false);
+    assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
+    // fix hbase.version missing
+    doFsck(conf, true);
+
+    // no version file fixed
+    assertNoErrors(doFsck(conf, false));
+  }
+
+  /**
+   * the region is not deployed when the table is disabled.
+   */
+  @Test
+  public void testRegionShouldNotDeployed() throws Exception {
+    String table = "tableRegionShouldNotDeployed";
+    try {
+      LOG.info("Starting testRegionShouldNotDeployed.");
+      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+      assertTrue(cluster.waitForActiveAndReadyMaster());
+
+      // Create a ZKW to use in the test
+      ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
+
+      FileSystem filesystem = FileSystem.get(conf);
+      Path rootdir = filesystem.makeQualified(new Path(conf
+          .get(HConstants.HBASE_DIR)));
+
+      byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
+          Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
+      HTableDescriptor htdDisabled = new HTableDescriptor(Bytes.toBytes(table));
+      htdDisabled.addFamily(new HColumnDescriptor(FAM));
+
+      // Write the .tableinfo
+      FSTableDescriptors
+          .createTableDescriptor(filesystem, rootdir, htdDisabled);
+      List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
+          TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
+
+      // Let's just assign everything to first RS
+      HRegionServer hrs = cluster.getRegionServer(0);
+      ServerName serverName = hrs.getServerName();
+
+      // create region files.
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      // Region of disable table was opened on RS
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      HRegionInfo region = disabledRegions.remove(0);
+      ZKAssign.createNodeOffline(zkw, region, serverName);
+      hrs.openRegion(region);
+
+      int iTimes = 0;
+      while (true) {
+        RegionTransitionData rtd = ZKAssign.getData(zkw,
+            region.getEncodedName());
+        if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+          break;
+        }
+        Thread.sleep(100);
+        iTimes++;
+        if (iTimes >= REGION_ONLINE_TIMEOUT) {
+          break;
+        }
+      }
+      assertTrue(iTimes < REGION_ONLINE_TIMEOUT);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
+
+      // fix this fault
+      doFsck(conf, true);
+
+      // check result
+      assertNoErrors(doFsck(conf, false));
+    } finally {
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+      deleteTable(table);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 75d24da43432..393cbbaed777 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -29,12 +29,12 @@
 
 public class HbckTestingUtil {
   public static HBaseFsck doFsck(Configuration conf, boolean fix) throws Exception {
-    return doFsck(conf, fix, fix, fix, fix,fix);
+    return doFsck(conf, fix, fix, fix, fix,fix, fix);
   }
 
   public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
       boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
-      boolean fixHdfsOrphans) throws Exception {
+      boolean fixHdfsOrphans, boolean fixVersionFile) throws Exception {
     HBaseFsck fsck = new HBaseFsck(conf);
     fsck.connect();
     fsck.setDisplayFullReport(); // i.e. -details
@@ -44,6 +44,7 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
     fsck.setFixHdfsHoles(fixHdfsHoles);
     fsck.setFixHdfsOverlaps(fixHdfsOverlaps);
     fsck.setFixHdfsOrphans(fixHdfsOrphans);
+    fsck.setFixVersionFile(fixVersionFile);
     fsck.onlineHbck();
     return fsck;
   }

From e46c9486a0589a11ed852f5c749c7c87ecce333c Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 11 Apr 2012 23:36:22 +0000
Subject: [PATCH 0120/1540] HBASE-5736
 ThriftServerRunner.HbaseHandler.mutateRow() does not use ByteBuffer correctly
 (Scott Chen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325067 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift/ThriftServerRunner.java      | 12 ++++++------
 .../apache/hadoop/hbase/thrift/TestThriftServer.java | 12 ++++++++----
 .../hadoop/hbase/thrift/TestThriftServerCmdLine.java |  7 ++++---
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 859d37bbb025..02ece7b9bee3 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -971,11 +971,11 @@ public void mutateRowTs(ByteBuffer tableName, ByteBuffer row,
           } else {
             if(famAndQf.length == 1) {
               put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY,
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             } else {
               put.add(famAndQf[0], famAndQf[1],
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             }
             put.setWriteToWAL(m.writeToWAL);
@@ -1029,11 +1029,11 @@ public void mutateRowsTs(
           } else {
             if(famAndQf.length == 1) {
               put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY,
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             } else {
               put.add(famAndQf[0], famAndQf[1],
-                  m.value != null ? m.value.array()
+                  m.value != null ? getBytes(m.value)
                       : HConstants.EMPTY_BYTE_ARRAY);
             }
             put.setWriteToWAL(m.writeToWAL);
@@ -1398,8 +1398,8 @@ private static void addAttributes(OperationWithAttributes op,
       return;
     }
     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
-      String name = Bytes.toStringBinary(entry.getKey());
-      byte[] value =  Bytes.toBytes(entry.getValue());
+      String name = Bytes.toStringBinary(getBytes(entry.getKey()));
+      byte[] value =  getBytes(entry.getValue());
       op.setAttribute(name, value);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 0cecbb943ca8..2ece73ea2080 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -203,9 +203,13 @@ public static void dropTestTables(Hbase.Iface handler) throws Exception {
    * @throws Exception
    */
   public void doTestTableMutations() throws Exception {
-    // Setup
     ThriftServerRunner.HBaseHandler handler =
       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
+    doTestTableMutations(handler);
+  }
+
+  public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
+    // Setup
     handler.createTable(tableAname, getColumnDescriptors());
 
     // Apply a few Mutations to rowA
@@ -261,7 +265,7 @@ public void doTestTableMutations() throws Exception {
     handler.mutateRow(tableAname, rowAname, mutations, null);
     TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
     assertEquals(rowAname, rowResult3.row);
-    assertEquals(0, rowResult3.columns.get(columnAname).value.array().length);
+    assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
 
     // Teardown
     handler.disableTable(tableAname);
@@ -497,7 +501,7 @@ private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) {
    * @return a List of Mutations for a row, with columnA having valueA
    * and columnB having valueB
    */
-  private List<Mutation> getMutations() {
+  private static List<Mutation> getMutations() {
     List<Mutation> mutations = new ArrayList<Mutation>();
     mutations.add(new Mutation(false, columnAname, valueAname, true));
     mutations.add(new Mutation(false, columnBname, valueBname, true));
@@ -512,7 +516,7 @@ private List<Mutation> getMutations() {
    * (rowB, columnA): place valueC
    * (rowB, columnB): place valueD
    */
-  private List<BatchMutation> getBatchMutations() {
+  private static List<BatchMutation> getBatchMutations() {
     List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
 
     // Mutations to rowA.  You can't mix delete and put anymore.
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index 6f1872d138e1..f3df9ed62c26 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -59,7 +59,7 @@
 @RunWith(Parameterized.class)
 public class TestThriftServerCmdLine {
 
-  public static final Log LOG = 
+  public static final Log LOG =
       LogFactory.getLog(TestThriftServerCmdLine.class);
 
   private final ImplType implType;
@@ -72,7 +72,7 @@ public class TestThriftServerCmdLine {
 
   private Thread cmdLineThread;
   private volatile Exception cmdLineException;
-  
+
   private Exception clientSideException;
 
   private ThriftServer thriftServer;
@@ -139,7 +139,7 @@ public void run() {
     cmdLineThread.start();
   }
 
-  @Test(timeout=30 * 1000)
+  @Test(timeout=60 * 1000)
   public void testRunThriftServer() throws Exception {
     List<String> args = new ArrayList<String>();
     if (implType != null) {
@@ -204,6 +204,7 @@ private void talkToThriftServer() throws Exception {
       Hbase.Client client = new Hbase.Client(prot);
       TestThriftServer.doTestTableCreateDrop(client);
       TestThriftServer.doTestGetTableRegions(client);
+      TestThriftServer.doTestTableMutations(client);
     } finally {
       sock.close();
     }

From bb0856d7fa9d94baedc8a37ac9ee1560544a51b2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 12 Apr 2012 16:09:21 +0000
Subject: [PATCH 0121/1540] HBASE-5717 Scanner metrics are only reported if you
 get to the end of a scanner (Ian Varley)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325342 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ClientScanner.java    | 39 ++++++------
 .../hbase/client/TestFromClientSide.java      | 60 ++++++++++++++++---
 2 files changed, 73 insertions(+), 26 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 103e39b733f1..0fbabe5ded5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -224,20 +224,17 @@ protected ScannerCallable getScannerCallable(byte [] localStartKey,
     }
 
     /**
-     * publish the scan metrics
-     * For now, we use scan.setAttribute to pass the metrics for application
-     * or TableInputFormat to consume
-     * Later, we could push it to other systems
-     * We don't use metrics framework because it doesn't support
-     * multi instances of the same metrics on the same machine; for scan/map
-     * reduce scenarios, we will have multiple scans running at the same time
+     * Publish the scan metrics. For now, we use scan.setAttribute to pass the metrics back to the
+     * application or TableInputFormat.Later, we could push it to other systems. We don't use metrics
+     * framework because it doesn't support multi-instances of the same metrics on the same machine;
+     * for scan/map reduce scenarios, we will have multiple scans running at the same time.
+     *
+     * By default, scan metrics are disabled; if the application wants to collect them, this behavior
+     * can be turned on by calling calling:
+     *
+     * scan.setAttribute(SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE))
      */
-    private void writeScanMetrics() throws IOException
-    {
-      // by default, scanMetrics is null
-      // if application wants to collect scanMetrics, it can turn it on by
-      // calling scan.setAttribute(SCAN_ATTRIBUTES_METRICS_ENABLE,
-      // Bytes.toBytes(Boolean.TRUE))
+    private void writeScanMetrics() throws IOException {
       if (this.scanMetrics == null) {
         return;
       }
@@ -247,10 +244,8 @@ private void writeScanMetrics() throws IOException
     }
 
     public Result next() throws IOException {
-      // If the scanner is closed but there is some rows left in the cache,
-      // it will first empty it before returning null
+      // If the scanner is closed and there's nothing left in the cache, next is a no-op.
       if (cache.size() == 0 && this.closed) {
-        writeScanMetrics();
         return null;
       }
       if (cache.size() == 0) {
@@ -312,8 +307,7 @@ public Result next() throws IOException {
           }
           long currentTime = System.currentTimeMillis();
           if (this.scanMetrics != null ) {
-            this.scanMetrics.sumOfMillisSecBetweenNexts.inc(
-              currentTime-lastNext);
+            this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext);
           }
           lastNext = currentTime;
           if (values != null && values.length > 0) {
@@ -333,6 +327,8 @@ public Result next() throws IOException {
       if (cache.size() > 0) {
         return cache.poll();
       }
+
+      // if we exhausted this scanner before calling close, write out the scan metrics
       writeScanMetrics();
       return null;
     }
@@ -370,6 +366,13 @@ public void close() {
           // have since decided that it's not nice for a scanner's close to
           // throw exceptions. Chances are it was just an UnknownScanner
           // exception due to lease time out.
+        } finally {
+          // we want to output the scan metrics even if an error occurred on close
+          try {
+            writeScanMetrics();
+          } catch (IOException e) {
+            // As above, we still don't want the scanner close() method to throw.
+          }
         }
         callable = null;
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 56403c710b9c..e27e520c018b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4414,30 +4414,74 @@ public void testScanMetrics() throws Exception {
 
     // Create multiple regions for this table
     int numOfRegions = TEST_UTIL.createMultiRegions(ht, FAMILY);
+    // Create 3 rows in the table, with rowkeys starting with "z*" so that
+    // scan are forced to hit all the regions.
+    Put put1 = new Put(Bytes.toBytes("z1"));
+    put1.add(FAMILY, QUALIFIER, VALUE);
+    Put put2 = new Put(Bytes.toBytes("z2"));
+    put2.add(FAMILY, QUALIFIER, VALUE);
+    Put put3 = new Put(Bytes.toBytes("z3"));
+    put3.add(FAMILY, QUALIFIER, VALUE);
+    ht.put(Arrays.asList(put1, put2, put3));
 
     Scan scan1 = new Scan();
+    int numRecords = 0;
     for(Result result : ht.getScanner(scan1)) {
+      numRecords++;
     }
+    LOG.info("test data has " + numRecords + " records.");
 
     // by default, scan metrics collection is turned off
-    assertEquals(null, scan1.getAttribute(
-      Scan.SCAN_ATTRIBUTES_METRICS_DATA));
+    assertEquals(null, scan1.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));
 
     // turn on scan metrics
     Scan scan = new Scan();
-    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE,
-      Bytes.toBytes(Boolean.TRUE));
-    for(Result result : ht.getScanner(scan)) {
+    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner scanner = ht.getScanner(scan);
+    // per HBASE-5717, this should still collect even if you don't run all the way to
+    // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
+    for (Result result : scanner.next(numRecords - 1)) {
+    }
+    scanner.close();
+
+    ScanMetrics scanMetrics = getScanMetrics(scan);
+    assertEquals("Did not access all the regions in the table", numOfRegions,
+        scanMetrics.countOfRegions.getCurrentIntervalValue());
+
+    // now, test that the metrics are still collected even if you don't call close, but do 
+    // run past the end of all the records
+    Scan scanWithoutClose = new Scan();
+    scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
+    for (Result result : scannerWithoutClose.next(numRecords + 1)) {
     }
+    ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
+    assertEquals("Did not access all the regions in the table", numOfRegions,
+        scanMetricsWithoutClose.countOfRegions.getCurrentIntervalValue());
+
+    // finally, test that the metrics are collected correctly if you both run past all the records,
+    // AND close the scanner
+    Scan scanWithClose = new Scan();
+    scanWithClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
+    for (Result result : scannerWithClose.next(numRecords + 1)) {
+    }
+    scannerWithClose.close();
+    ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
+    assertEquals("Did not access all the regions in the table", numOfRegions,
+        scanMetricsWithClose.countOfRegions.getCurrentIntervalValue());
 
-    byte[] serializedMetrics = scan.getAttribute(
-      Scan.SCAN_ATTRIBUTES_METRICS_DATA);
+  }
+  
+  private ScanMetrics getScanMetrics(Scan scan) throws Exception {
+    byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
+    assertTrue("Serialized metrics were not found.", serializedMetrics != null);
 
     DataInputBuffer in = new DataInputBuffer();
     in.reset(serializedMetrics, 0, serializedMetrics.length);
     ScanMetrics scanMetrics = new ScanMetrics();
     scanMetrics.readFields(in);
-    assertEquals(numOfRegions, scanMetrics.countOfRegions.getCurrentIntervalValue());
+    return scanMetrics;
   }
 
   /**

From 70ba438053459f1fa05b6da42c309826deb83545 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 12 Apr 2012 16:58:47 +0000
Subject: [PATCH 0122/1540] HBASE-5773 HtablePool constructor not reading
 config files in certain cases

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325381 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HTablePool.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 5cc82997916b..9741dccc9348 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -136,7 +136,7 @@ public HTablePool(final Configuration config, final int maxSize,
       final HTableInterfaceFactory tableFactory, PoolType poolType) {
     // Make a new configuration instance so I can safely cleanup when
     // done with the pool.
-    this.config = config == null ? new Configuration() : config;
+    this.config = config == null ? HBaseConfiguration.create() : config;
     this.maxSize = maxSize;
     this.tableFactory = tableFactory == null ? new HTableFactory()
         : tableFactory;

From 886251eaa3898ea1bbe9054dcb19dfd1cce3406b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 12 Apr 2012 17:20:10 +0000
Subject: [PATCH 0123/1540] HBASE-5770 Add a clock skew warning threshold

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325388 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/ServerManager.java      | 17 ++++++++++++-----
 .../hbase/master/TestClockSkewDetection.java    | 15 ++++++++++++---
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 8281c6ffa786..283713c5d21d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -96,6 +96,7 @@ public class ServerManager {
   private final DeadServer deadservers;
 
   private final long maxSkew;
+  private final long warningSkew;
 
   /**
    * Set of region servers which are dead but not expired immediately. If one
@@ -122,6 +123,7 @@ public ServerManager(final Server master, final MasterServices services)
     this.services = services;
     Configuration c = master.getConfiguration();
     maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
+    warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
     this.deadservers = new DeadServer();
     this.connection = connect ? HConnectionManager.getConnection(c) : null;
   }
@@ -194,14 +196,14 @@ void checkAlreadySameHostPort(final ServerName serverName)
   }
 
   /**
-   * Checks if the clock skew between the server and the master. If the clock
-   * skew is too much it will throw an Exception.
+   * Checks if the clock skew between the server and the master. If the clock skew exceeds the 
+   * configured max, it will throw an exception; if it exceeds the configured warning threshold, 
+   * it will log a warning but start normally.
    * @param serverName Incoming servers's name
    * @param serverCurrentTime
-   * @throws ClockOutOfSyncException
+   * @throws ClockOutOfSyncException if the skew exceeds the configured max value
    */
-  private void checkClockSkew(final ServerName serverName,
-      final long serverCurrentTime)
+  private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
   throws ClockOutOfSyncException {
     long skew = System.currentTimeMillis() - serverCurrentTime;
     if (skew > maxSkew) {
@@ -210,6 +212,11 @@ private void checkClockSkew(final ServerName serverName,
         "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
       LOG.warn(message);
       throw new ClockOutOfSyncException(message);
+    } else if (skew > warningSkew){
+      String message = "Reported time for server " + serverName + " is out of sync with master " +
+        "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " + 
+        "error threshold is " + maxSkew + "ms)";
+      LOG.warn(message);
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
index 977a5c7a7564..7899bf095526 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
@@ -19,6 +19,8 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.fail;
+
 import java.net.InetAddress;
 
 import junit.framework.Assert;
@@ -82,18 +84,25 @@ public void stop(String why) {
     InetAddress ia1 = InetAddress.getLocalHost();
     sm.regionServerStartup(ia1, 1234, -1, System.currentTimeMillis());
 
-    long maxSkew = 30000;
+    final Configuration c = HBaseConfiguration.create();
+    long maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
+    long warningSkew = c.getLong("hbase.master.warningclockskew", 1000);
 
     try {
       LOG.debug("regionServerStartup 2");
       InetAddress ia2 = InetAddress.getLocalHost();
       sm.regionServerStartup(ia2, 1235, -1, System.currentTimeMillis() - maxSkew * 2);
-      Assert.assertTrue("HMaster should have thrown an ClockOutOfSyncException "
-        + "but didn't.", false);
+      fail("HMaster should have thrown an ClockOutOfSyncException but didn't.");
     } catch(ClockOutOfSyncException e) {
       //we want an exception
       LOG.info("Recieved expected exception: "+e);
     }
+    
+    // make sure values above warning threshold but below max threshold don't kill
+    LOG.debug("regionServerStartup 3");
+    InetAddress ia3 = InetAddress.getLocalHost();
+    sm.regionServerStartup(ia3, 1236, -1, System.currentTimeMillis() - warningSkew * 2);
+    
   }
 
   @org.junit.Rule

From 160917b49be3d7207a8c0dd1650ab53520af311b Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 12 Apr 2012 17:43:29 +0000
Subject: [PATCH 0124/1540] HBASE-5719 Enhance hbck to sideline overlapped mega
 regions (Jimmy Xiang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325403 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 145 ++++++++++++++++--
 .../hbase/util/RegionSplitCalculator.java     |  76 ++++++++-
 .../hbase/util/TestRegionSplitCalculator.java |  40 +++++
 3 files changed, 243 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 2eb230d8fb5c..41d05d35c084 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -22,8 +22,10 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedSet;
@@ -135,6 +137,7 @@ public class HBaseFsck {
   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
   private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
   private static boolean rsSupportsOffline = true;
+  private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
 
   /**********************
@@ -147,7 +150,6 @@ public class HBaseFsck {
   private HBaseAdmin admin;
   private HTable meta;
   private ThreadPoolExecutor executor; // threads to retrieve data from regionservers
-  private int numThreads = MAX_NUM_THREADS;
   private long startMillis = System.currentTimeMillis();
 
   /***********
@@ -165,6 +167,8 @@ public class HBaseFsck {
   // limit fixes to listed tables, if empty atttempt to fix all
   private List<byte[]> tablesToFix = new ArrayList<byte[]>();
   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
+  private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
+  private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
 
   private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
@@ -210,7 +214,7 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
       ZooKeeperConnectionException, IOException {
     this.conf = conf;
 
-    int numThreads = conf.getInt("hbasefsck.numthreads", Integer.MAX_VALUE);
+    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
     executor = new ThreadPoolExecutor(1, numThreads,
         THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
         new SynchronousQueue<Runnable>());
@@ -829,14 +833,14 @@ private Path getSidelineDir() throws IOException {
   /**
    * Sideline a region dir (instead of deleting it)
    */
-  void sidelineRegionDir(FileSystem fs, HbckInfo hi)
+  Path sidelineRegionDir(FileSystem fs, HbckInfo hi)
     throws IOException {
     String tableName = Bytes.toString(hi.getTableName());
     Path regionDir = hi.getHdfsRegionDir();
 
     if (!fs.exists(regionDir)) {
       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
-      return;
+      return null;
     }
 
     Path sidelineTableDir= new Path(getSidelineDir(), tableName);
@@ -869,12 +873,15 @@ void sidelineRegionDir(FileSystem fs, HbckInfo hi)
         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
         // it moves the src into the dst dir resulting in (foo/a/b).  If
         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
-        for (FileStatus hfile : fs.listStatus(src)) {
-          success = fs.rename(hfile.getPath(), dst);
-          if (!success) {
-            String msg = "Unable to rename file " + src +  " to " + dst;
-            LOG.error(msg);
-            throw new IOException(msg);
+        FileStatus[] hfiles = fs.listStatus(src);
+        if (hfiles != null && hfiles.length > 0) {
+          for (FileStatus hfile : hfiles) {
+            success = fs.rename(hfile.getPath(), dst);
+            if (!success) {
+              String msg = "Unable to rename file " + src +  " to " + dst;
+              LOG.error(msg);
+              throw new IOException(msg);
+            }
           }
         }
         LOG.debug("Sideline directory contents:");
@@ -889,6 +896,7 @@ void sidelineRegionDir(FileSystem fs, HbckInfo hi)
       LOG.error(msg);
       throw new IOException(msg);
     }
+    return sidelineRegionDir;
   }
 
   /**
@@ -1542,6 +1550,9 @@ public class TableInfo {
     // backwards regions
     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
 
+    // sidelined big overlapped regions
+    final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
+
     // region split calculator
     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
 
@@ -1563,6 +1574,9 @@ public class TableInfo {
     private HTableDescriptor getHTD() {
       if (htds.size() == 1) {
         return (HTableDescriptor)htds.toArray()[0];
+      } else {
+        LOG.error("None/Multiple table descriptors found for table '"
+          + tableName + "' regions: " + htds);
       }
       return null;
     }
@@ -1743,13 +1757,21 @@ public void handleOverlapGroup(Collection<HbckInfo> overlap)
 
         if (overlap.size() > maxMerge) {
           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
-              "regions which is greater than " + maxMerge + ", the max " +
-              "number of regions to merge.");
+            "regions which is greater than " + maxMerge + ", the max number of regions to merge");
+          if (sidelineBigOverlaps) {
+            // we only sideline big overlapped groups that exceeds the max number of regions to merge
+            sidelineBigOverlaps(overlap);
+          }
           return;
         }
 
+        mergeOverlaps(overlap);
+      }
+
+      void mergeOverlaps(Collection<HbckInfo> overlap)
+          throws IOException {
         LOG.info("== Merging regions into one region: "
-            + Joiner.on(",").join(overlap));
+          + Joiner.on(",").join(overlap));
         // get the min / max range and close all concerned regions
         Pair<byte[], byte[]> range = null;
         for (HbckInfo hi : overlap) {
@@ -1813,7 +1835,48 @@ public void handleOverlapGroup(Collection<HbckInfo> overlap)
           fixes++;
         }
       }
-    };
+
+      /**
+       * Sideline some regions in a big overlap group so that it
+       * will have fewer regions, and it is easier to merge them later on.
+       *
+       * @param bigOverlap the overlapped group with regions more than maxMerge
+       * @throws IOException
+       */
+      void sidelineBigOverlaps(
+          Collection<HbckInfo> bigOverlap) throws IOException {
+        int overlapsToSideline = bigOverlap.size() - maxMerge;
+        if (overlapsToSideline > maxOverlapsToSideline) {
+          overlapsToSideline = maxOverlapsToSideline;
+        }
+        List<HbckInfo> regionsToSideline =
+          RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
+        FileSystem fs = FileSystem.get(conf);
+        for (HbckInfo regionToSideline: regionsToSideline) {
+          try {
+            LOG.info("Closing region: " + regionToSideline);
+            closeRegion(regionToSideline);
+          } catch (InterruptedException ie) {
+            LOG.warn("Was unable to close region " + regionToSideline.getRegionNameAsString()
+              + ".  Interrupted.");
+            throw new IOException(ie);
+          }
+
+          LOG.info("Offlining region: " + regionToSideline);
+          offline(regionToSideline.getRegionName());
+
+          LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
+          Path sidelineRegionDir = sidelineRegionDir(fs, regionToSideline);
+          if (sidelineRegionDir != null) {
+            sidelinedRegions.put(sidelineRegionDir, regionToSideline);
+            LOG.info("After sidelined big overlapped region: "
+              + regionToSideline.getRegionNameAsString()
+              + " to " + sidelineRegionDir.toString());
+            fixes++;
+          }
+        }
+      }
+    }
 
     /**
      * Check the region chain (from META) of this table.  We are looking for
@@ -1899,16 +1962,22 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc
 
       if (details) {
         // do full region split map dump
-        System.out.println("---- Table '"  +  this.tableName 
+        System.out.println("---- Table '"  +  this.tableName
             + "': region split map");
         dump(splits, regions);
-        System.out.println("---- Table '"  +  this.tableName 
+        System.out.println("---- Table '"  +  this.tableName
             + "': overlap groups");
         dumpOverlapProblems(overlapGroups);
         System.out.println("There are " + overlapGroups.keySet().size()
             + " overlap groups with " + overlapGroups.size()
             + " overlapping regions");
       }
+      if (!sidelinedRegions.isEmpty()) {
+        LOG.warn("Sidelined big overlapped regions, please bulk load them!");
+        System.out.println("---- Table '"  +  this.tableName
+            + "': sidelined big overlapped regions");
+        dumpSidelinedRegions(sidelinedRegions);
+      }
       return errors.getErrorList().size() == originalErrorsCount;
     }
 
@@ -1944,6 +2013,13 @@ public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
     }
   }
 
+  public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
+    for (Path k : regions.keySet()) {
+      System.out.println("To be bulk loaded sidelined region dir: "
+        + k.toString());
+    }
+  }
+
   public Multimap<byte[], HbckInfo> getOverlapGroups(
       String table) {
     TableInfo ti = tablesInfo.get(table);
@@ -2751,6 +2827,14 @@ public boolean shouldFixVersionFile() {
     return fixVersionFile;
   }
 
+  public void setSidelineBigOverlaps(boolean sbo) {
+    this.sidelineBigOverlaps = sbo;
+  }
+
+  public boolean shouldSidelineBigOverlaps() {
+    return sidelineBigOverlaps;
+  }
+
   /**
    * @param mm maximum number of regions to merge into a single region.
    */
@@ -2762,6 +2846,14 @@ public int getMaxMerge() {
     return maxMerge;
   }
 
+  public void setMaxOverlapsToSideline(int mo) {
+    this.maxOverlapsToSideline = mo;
+  }
+
+  public int getMaxOverlapsToSideline() {
+    return maxOverlapsToSideline;
+  }
+
   /**
    * Only fix tables specified by the list
    */
@@ -2811,9 +2903,11 @@ protected static void printUsageAndExit() {
     System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
     System.err.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
     System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
+    System.err.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
+    System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
     System.err.println("");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
-        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile");
+        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
     System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
 
     Runtime.getRuntime().exit(-2);
@@ -2878,6 +2972,8 @@ public static void main(String[] args) throws Exception {
         fsck.setFixHdfsOverlaps(true);
       } else if (cmd.equals("-fixVersionFile")) {
         fsck.setFixVersionFile(true);
+      } else if (cmd.equals("-sidelineBigOverlaps")) {
+        fsck.setSidelineBigOverlaps(true);
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
@@ -2887,6 +2983,7 @@ public static void main(String[] args) throws Exception {
         fsck.setFixAssignments(true);
         fsck.setFixHdfsOverlaps(true);
         fsck.setFixVersionFile(true);
+        fsck.setSidelineBigOverlaps(true);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
         fsck.setFixHdfsHoles(true);
@@ -2894,6 +2991,20 @@ public static void main(String[] args) throws Exception {
         fsck.setFixMeta(true);
         fsck.setFixAssignments(true);
         fsck.setFixHdfsOverlaps(false);
+        fsck.setSidelineBigOverlaps(false);
+      } else if (cmd.equals("-maxOverlapsToSideline")) {
+        if (i == args.length - 1) {
+          System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+          printUsageAndExit();
+        }
+        try {
+          int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
+          fsck.setMaxOverlapsToSideline(maxOverlapsToSideline);
+        } catch (NumberFormatException e) {
+          System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+          printUsageAndExit();
+        }
+        i++;
       } else if (cmd.equals("-maxMerge")) {
         if (i == args.length - 1) {
           System.err.println("-maxMerge needs a numeric value argument.");
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
index 69a87ab07a37..449e2b002e09 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
@@ -19,9 +19,12 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.List;
 import java.util.Map.Entry;
+import java.util.TreeMap;
 import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
@@ -94,7 +97,7 @@ public int compare(byte[] l, byte[] r) {
    * 
    * @return ENDKEY if end key is empty, else normal endkey.
    */
-  private byte[] specialEndKey(R range) {
+  private static <R extends KeyRange> byte[] specialEndKey(R range) {
     byte[] end = range.getEndKey();
     if (end.length == 0) {
       return ENDKEY;
@@ -159,4 +162,75 @@ public Multimap<byte[], R> getStarts() {
     return starts;
   }
 
+  /**
+   * Find specified number of top ranges in a big overlap group.
+   * It could return less if there are not that many top ranges.
+   * Once these top ranges are excluded, the big overlap group will
+   * be broken into ranges with no overlapping, or smaller overlapped
+   * groups, and most likely some holes.
+   *
+   * @param bigOverlap a list of ranges that overlap with each other
+   * @param count the max number of ranges to find
+   * @return a list of ranges that overlap with most others
+   */
+  public static <R extends KeyRange> List<R>
+      findBigRanges(Collection<R> bigOverlap, int count) {
+    List<R> bigRanges = new ArrayList<R>();
+
+    // The key is the count of overlaps,
+    // The value is a list of ranges that have that many overlaps
+    TreeMap<Integer, List<R>> overlapRangeMap = new TreeMap<Integer, List<R>>();
+    for (R r: bigOverlap) {
+      // Calculates the # of overlaps for each region
+      // and populates rangeOverlapMap
+      byte[] startKey = r.getStartKey();
+      byte[] endKey = specialEndKey(r);
+
+      int overlappedRegions = 0;
+      for (R rr: bigOverlap) {
+        byte[] start = rr.getStartKey();
+        byte[] end = specialEndKey(rr);
+
+        if (BYTES_COMPARATOR.compare(startKey, end) < 0
+            && BYTES_COMPARATOR.compare(endKey, start) > 0) {
+          overlappedRegions++;
+        }
+      }
+
+      // One region always overlaps with itself,
+      // so overlappedRegions should be more than 1
+      // for actual overlaps.
+      if (overlappedRegions > 1) {
+        Integer key = Integer.valueOf(overlappedRegions);
+        List<R> ranges = overlapRangeMap.get(key);
+        if (ranges == null) {
+          ranges = new ArrayList<R>();
+          overlapRangeMap.put(key, ranges);
+        }
+        ranges.add(r);
+      }
+    }
+    int toBeAdded = count;
+    for (Integer key: overlapRangeMap.descendingKeySet()) {
+      List<R> chunk = overlapRangeMap.get(key);
+      int chunkSize = chunk.size();
+      if (chunkSize <= toBeAdded) {
+        bigRanges.addAll(chunk);
+        toBeAdded -= chunkSize;
+        if (toBeAdded > 0) continue;
+      } else {
+        // Try to use the middle chunk in case the overlapping is
+        // chained, for example: [a, c), [b, e), [d, g), [f h)...
+        // In such a case, sideline the middle chunk will break
+        // the group efficiently.
+        int start = (chunkSize - toBeAdded)/2;
+        int end = start + toBeAdded;
+        for (int i = start; i < end; i++) {
+          bigRanges.add(chunk.get(i));
+        }
+      }
+      break;
+    }
+    return bigRanges;
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
index ac3b2250f56a..d0e65d952b8d 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
@@ -20,9 +20,12 @@
 package org.apache.hadoop.hbase.util;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.List;
 import java.util.SortedSet;
 
 import org.apache.commons.logging.Log;
@@ -32,6 +35,7 @@
 
 import com.google.common.collect.ComparisonChain;
 import com.google.common.collect.Multimap;
+
 import org.junit.experimental.categories.Category;
 
 @Category(SmallTests.class)
@@ -351,6 +355,42 @@ public void testBeginEndMarker() {
         + "null:\t\n");
   }
 
+  @Test
+  public void testBigRanges() {
+    SimpleRange ai = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("I"));
+    SimpleRange ae = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E"));
+    SimpleRange ac = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"));
+
+    Collection<SimpleRange> bigOverlap = new ArrayList<SimpleRange>();
+    bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E")));
+    bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")));
+    bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")));
+    bigOverlap.add(new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")));
+    bigOverlap.add(new SimpleRange(Bytes.toBytes("E"), Bytes.toBytes("H")));
+    bigOverlap.add(ai);
+    bigOverlap.add(ae);
+    bigOverlap.add(ac);
+
+    // Expect 1 range to be returned: ai
+    List<SimpleRange> bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 1);
+    assertEquals(1, bigRanges.size());
+    assertEquals(ai, bigRanges.get(0));
+
+    // Expect 3 ranges to be returned: ai, ae and ac
+    bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 3);
+    assertEquals(3, bigRanges.size());
+    assertEquals(ai, bigRanges.get(0));
+
+    SimpleRange r1 = bigRanges.get(1);
+    SimpleRange r2 = bigRanges.get(2);
+    assertEquals(Bytes.toString(r1.start), "A");
+    assertEquals(Bytes.toString(r2.start), "A");
+    String r1e = Bytes.toString(r1.end);
+    String r2e = Bytes.toString(r2.end);
+    assertTrue((r1e.equals("C") && r2e.equals("E"))
+      || (r1e.equals("E") && r2e.equals("C")));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From a5003b9c7d0fe7c79243690216fbb6fb80cedc18 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 12 Apr 2012 19:18:16 +0000
Subject: [PATCH 0125/1540] HBASE-3443 ICV optimization to look in memstore
 first and then store files (HBASE-3082) does not work when deletes are in the
 mix

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325453 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 88 +------------------
 .../hbase/client/TestFromClientSide.java      | 21 +++++
 2 files changed, 23 insertions(+), 86 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9a9d14d9658a..573606dfbed2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -94,7 +94,6 @@
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
-import org.apache.hadoop.hbase.filter.NullComparator;
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.TimeRange;
@@ -4063,89 +4062,6 @@ public Result get(final Get get, final Integer lockid) throws IOException {
     return new Result(results);
   }
 
-  /**
-   * An optimized version of {@link #get(Get)} that checks MemStore first for
-   * the specified query.
-   * <p>
-   * This is intended for use by increment operations where we have the
-   * guarantee that versions are never inserted out-of-order so if a value
-   * exists in MemStore it is the latest value.
-   * <p>
-   * It only makes sense to use this method without a TimeRange and maxVersions
-   * equal to 1.
-   * @param get
-   * @return result
-   * @throws IOException
-   */
-  private List<KeyValue> getLastIncrement(final Get get) throws IOException {
-    InternalScan iscan = new InternalScan(get);
-
-    List<KeyValue> results = new ArrayList<KeyValue>();
-
-    // memstore scan
-    iscan.checkOnlyMemStore();
-    RegionScanner scanner = null;
-    try {
-      scanner = getScanner(iscan);
-      scanner.next(results);
-    } finally {
-      if (scanner != null)
-        scanner.close();
-    }
-
-    // count how many columns we're looking for
-    int expected = 0;
-    Map<byte[], NavigableSet<byte[]>> familyMap = get.getFamilyMap();
-    for (NavigableSet<byte[]> qfs : familyMap.values()) {
-      expected += qfs.size();
-    }
-
-    // found everything we were looking for, done
-    if (results.size() == expected) {
-      return results;
-    }
-
-    // still have more columns to find
-    if (results != null && !results.isEmpty()) {
-      // subtract what was found in memstore
-      for (KeyValue kv : results) {
-        byte [] family = kv.getFamily();
-        NavigableSet<byte[]> qfs = familyMap.get(family);
-        qfs.remove(kv.getQualifier());
-        if (qfs.isEmpty()) familyMap.remove(family);
-        expected--;
-      }
-      // make a new get for just what is left
-      Get newGet = new Get(get.getRow());
-      for (Map.Entry<byte[], NavigableSet<byte[]>> f : familyMap.entrySet()) {
-        byte [] family = f.getKey();
-        for (byte [] qualifier : f.getValue()) {
-          newGet.addColumn(family, qualifier);
-        }
-      }
-      newGet.setTimeRange(get.getTimeRange().getMin(),
-          get.getTimeRange().getMax());
-      iscan = new InternalScan(newGet);
-    }
-
-    // check store files for what is left
-    List<KeyValue> fileResults = new ArrayList<KeyValue>();
-    iscan.checkOnlyStoreFiles();
-    scanner = null;
-    try {
-      scanner = getScanner(iscan);
-      scanner.next(fileResults);
-    } finally {
-      if (scanner != null)
-        scanner.close();
-    }
-
-    // combine and return
-    results.addAll(fileResults);
-    Collections.sort(results, KeyValue.COMPARATOR);
-    return results;
-  }
-
   /*
    * Do a get based on the get parameter.
    * @param withCoprocessor invoke coprocessor or not. We don't want to
@@ -4562,7 +4478,7 @@ public Result increment(Increment increment, Integer lockid,
             get.addColumn(family.getKey(), column.getKey());
           }
           get.setTimeRange(tr.getMin(), tr.getMax());
-          List<KeyValue> results = getLastIncrement(get);
+          List<KeyValue> results = get(get, false);
 
           // Iterate the input columns and update existing values if they were
           // found, otherwise add new column initialized to the increment amount
@@ -4662,7 +4578,7 @@ public long incrementColumnValue(byte [] row, byte [] family,
 
         // we don't want to invoke coprocessor in this case; ICV is wrapped
         // in HRegionServer, so we leave getLastIncrement alone
-        List<KeyValue> results = getLastIncrement(get);
+        List<KeyValue> results = get(get, false);
 
         if (!results.isEmpty()) {
           KeyValue kv = results.get(0);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index e27e520c018b..d614508419de 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4115,6 +4115,27 @@ public void testAppend() throws Exception {
     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
   }
  
+  @Test
+  public void testIncrementWithDeletes() throws Exception {
+    LOG.info("Starting testIncrement");
+    final byte [] TABLENAME = Bytes.toBytes("testIncrementWithDeletes");
+    HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
+    final byte[] COLUMN = Bytes.toBytes("column");
+
+    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
+    TEST_UTIL.flush(TABLENAME);
+
+    Delete del = new Delete(ROW);
+    ht.delete(del);
+
+    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
+
+    Get get = new Get(ROW);
+    Result r = ht.get(get);
+    assertEquals(1, r.size());
+    assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
+  }
+
   @Test
   public void testIncrement() throws Exception {
     LOG.info("Starting testIncrement");

From 53d98dae62da5bc7489396b19c07605c5c5d4c85 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 12 Apr 2012 21:54:02 +0000
Subject: [PATCH 0126/1540] HBASE-5775 ZKUtil doesn't handle deleteRecurisively
 cleanly (Jesse Yates)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325541 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java   | 9 +++++++--
 src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java | 5 +++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 719a17691abc..e1b6bee0f6f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -991,14 +991,19 @@ public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
 
   /**
    * Delete the specified node and all of it's children.
-   *
-   * Sets no watches.  Throws all exceptions besides dealing with deletion of
+   * <p>
+   * If the node does not exist, just returns.
+   * <p>
+   * Sets no watches. Throws all exceptions besides dealing with deletion of
    * children.
    */
   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
   throws KeeperException {
     try {
       List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
+      // the node is already deleted, so we just finish
+      if (children == null) return;
+
       if(!children.isEmpty()) {
         for(String child : children) {
           deleteNodeRecursively(zkw, joinZNode(node, child));
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 5d3776ade82a..ddf8186efb5c 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -223,7 +223,12 @@ public void testZNodeDeletes() throws Exception {
       assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
     }
     ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
+    // make sure it really is deleted
     assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
+
+    // do the same delete again and make sure it doesn't crash
+    ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
+
     ZKUtil.deleteNode(zkw, "/l1");
     assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
   }

From aa0f28a3d48c69d15696caf8f1ea5fe9cb765a58 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 12 Apr 2012 22:30:57 +0000
Subject: [PATCH 0127/1540] HBASE-5604 M/R tool to replay WAL files

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325562 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/ops_mgt.xml                        |  17 +
 .../hbase/mapreduce/HLogInputFormat.java      | 268 +++++++++++++++
 .../hadoop/hbase/mapreduce/WALPlayer.java     | 309 ++++++++++++++++++
 .../hbase/mapreduce/TestHLogRecordReader.java | 240 ++++++++++++++
 .../hadoop/hbase/mapreduce/TestWALPlayer.java | 121 +++++++
 5 files changed, 955 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java

diff --git a/src/docbkx/ops_mgt.xml b/src/docbkx/ops_mgt.xml
index 3dbd718a89c2..0f645a19867e 100644
--- a/src/docbkx/ops_mgt.xml
+++ b/src/docbkx/ops_mgt.xml
@@ -125,6 +125,23 @@
        <title>Import</title>
        <para>Import is a utility that will load data that has been exported back into HBase.  Invoke via:
 <programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.Import &lt;tablename&gt; &lt;inputdir&gt;
+</programlisting>
+       </para>
+    </section>
+    <section xml:id="walplayer">
+       <title>WALPlayer</title>
+       <para>WALPlayer is a utility to replay WAL files into HBase.
+       </para>
+       <para>The WAL can be replayed for a set of tables or all tables, and a timerange can be provided (in milliseconds). The WAL is filtered to this set of tables. The output can optionally be mapped to another set of tables.
+       </para>
+       <para>WALPlayer can also generate HFiles for later bulk importing, in that case only a single table and no mapping can be specified.
+       </para>
+       <para>Invoke via:
+<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.WALPlayer [options] &lt;wal inputdir&gt; &lt;tables&gt; [&lt;tableMappings>]&gt;
+</programlisting>
+       </para>
+       <para>For example:
+<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.WALPlayer /backuplogdir oldTable1,oldTable2 newTable1,newTable2
 </programlisting>
        </para>
     </section>
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
new file mode 100644
index 000000000000..747063c6f584
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Simple {@link InputFormat} for {@link HLog} files.
+ */
+@InterfaceAudience.Public
+public class HLogInputFormat extends InputFormat<HLogKey, WALEdit> {
+  private static Log LOG = LogFactory.getLog(HLogInputFormat.class);
+
+  public static String START_TIME_KEY = "hlog.start.time";
+  public static String END_TIME_KEY = "hlog.end.time";
+
+  /**
+   * {@link InputSplit} for {@link HLog} files. Each split represent
+   * exactly one log file.
+   */
+  static class HLogSplit extends InputSplit implements Writable {
+    private String logFileName;
+    private long fileSize;
+    private long startTime;
+    private long endTime;
+
+    /** for serialization */
+    public HLogSplit() {}
+
+    /**
+     * Represent an HLogSplit, i.e. a single HLog file.
+     * Start- and EndTime are managed by the split, so that HLog files can be
+     * filtered before WALEdits are passed to the mapper(s).
+     * @param logFileName
+     * @param fileSize
+     * @param startTime
+     * @param endTime
+     */
+    public HLogSplit(String logFileName, long fileSize, long startTime, long endTime) {
+      this.logFileName = logFileName;
+      this.fileSize = fileSize;
+      this.startTime = startTime;
+      this.endTime = endTime;
+    }
+
+    @Override
+    public long getLength() throws IOException, InterruptedException {
+      return fileSize;
+    }
+
+    @Override
+    public String[] getLocations() throws IOException, InterruptedException {
+      // TODO: Find the data node with the most blocks for this HLog?
+      return new String[] {};
+    }
+
+    public String getLogFileName() {
+      return logFileName;
+    }
+
+    public long getStartTime() {
+      return startTime;
+    }
+
+    public long getEndTime() {
+      return endTime;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      logFileName = in.readUTF();
+      fileSize = in.readLong();
+      startTime = in.readLong();
+      endTime = in.readLong();
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      out.writeUTF(logFileName);
+      out.writeLong(fileSize);
+      out.writeLong(startTime);
+      out.writeLong(endTime);
+    }
+
+    @Override
+    public String toString() {
+      return logFileName + " (" + startTime + ":" + endTime + ") length:" + fileSize;
+    }
+  }
+
+  /**
+   * {@link RecordReader} for an {@link HLog} file.
+   */
+  static class HLogRecordReader extends RecordReader<HLogKey, WALEdit> {
+    private HLog.Reader reader = null;
+    private HLog.Entry currentEntry = new HLog.Entry();
+    private long startTime;
+    private long endTime;
+
+    @Override
+    public void initialize(InputSplit split, TaskAttemptContext context)
+        throws IOException, InterruptedException {
+      HLogSplit hsplit = (HLogSplit)split;
+      Path logFile = new Path(hsplit.getLogFileName());
+      Configuration conf = context.getConfiguration();
+      LOG.info("Opening reader for "+split);
+      try {
+        this.reader = HLog.getReader(logFile.getFileSystem(conf), logFile, conf);
+      } catch (EOFException x) {
+        LOG.info("Ignoring corrupted HLog file: " + logFile
+            + " (This is normal when a RegionServer crashed.)");
+      }
+      this.startTime = hsplit.getStartTime();
+      this.endTime = hsplit.getEndTime();
+    }
+
+    @Override
+    public boolean nextKeyValue() throws IOException, InterruptedException {
+      if (reader == null) return false;
+
+      HLog.Entry temp;
+      long i = -1;
+      do {
+        // skip older entries
+        try {
+          temp = reader.next(currentEntry);
+          i++;
+        } catch (EOFException x) {
+          LOG.info("Corrupted entry detected. Ignoring the rest of the file."
+              + " (This is normal when a RegionServer crashed.)");
+          return false;
+        }
+      }
+      while(temp != null && temp.getKey().getWriteTime() < startTime);
+
+      if (temp == null) {
+        if (i > 0) LOG.info("Skipped " + i + " entries.");
+        LOG.info("Reached end of file.");
+        return false;
+      } else if (i > 0) {
+        LOG.info("Skipped " + i + " entries, until ts: " + temp.getKey().getWriteTime() + ".");
+      }
+      boolean res = temp.getKey().getWriteTime() <= endTime;
+      if (!res) {
+        LOG.info("Reached ts: " + temp.getKey().getWriteTime() + " ignoring the rest of the file.");
+      }
+      return res;
+    }
+
+    @Override
+    public HLogKey getCurrentKey() throws IOException, InterruptedException {
+      return currentEntry.getKey();
+    }
+
+    @Override
+    public WALEdit getCurrentValue() throws IOException, InterruptedException {
+      return currentEntry.getEdit();
+    }
+
+    @Override
+    public float getProgress() throws IOException, InterruptedException {
+      // N/A depends on total number of entries, which is unknown
+      return 0;
+    }
+
+    @Override
+    public void close() throws IOException {
+      LOG.info("Closing reader");
+      if (reader != null) this.reader.close();
+    }
+  }
+
+  @Override
+  public List<InputSplit> getSplits(JobContext context) throws IOException,
+      InterruptedException {
+    Configuration conf = context.getConfiguration();
+    Path inputDir = new Path(conf.get("mapred.input.dir"));
+
+    long startTime = conf.getLong(START_TIME_KEY, Long.MIN_VALUE);
+    long endTime = conf.getLong(END_TIME_KEY, Long.MAX_VALUE);
+
+    FileSystem fs = inputDir.getFileSystem(conf);
+    List<FileStatus> files = getFiles(fs, inputDir, startTime, endTime);
+
+    List<InputSplit> splits = new ArrayList<InputSplit>(files.size());
+    for (FileStatus file : files) {
+      splits.add(new HLogSplit(file.getPath().toString(), file.getLen(), startTime, endTime));
+    }
+    return splits;
+  }
+
+  private List<FileStatus> getFiles(FileSystem fs, Path dir, long startTime, long endTime)
+      throws IOException {
+    List<FileStatus> result = new ArrayList<FileStatus>();
+    LOG.debug("Scanning " + dir.toString() + " for HLog files");
+
+    FileStatus[] files = fs.listStatus(dir);
+    if (files == null) return Collections.emptyList();
+    for (FileStatus file : files) {
+      if (file.isDir()) {
+        // recurse into sub directories
+        result.addAll(getFiles(fs, file.getPath(), startTime, endTime));
+      } else {
+        String name = file.getPath().toString();
+        int idx = name.lastIndexOf('.');
+        if (idx > 0) {
+          try {
+            long fileStartTime = Long.parseLong(name.substring(idx+1));
+            if (fileStartTime <= endTime) {
+              LOG.info("Found: " + name);
+              result.add(file);
+            }
+          } catch (NumberFormatException x) {
+            idx = 0;
+          }
+        }
+        if (idx == 0) {
+          LOG.warn("File " + name + " does not appear to be an HLog file. Skipping...");
+        }
+      }
+    }
+    return result;
+  }
+
+  @Override
+  public RecordReader<HLogKey, WALEdit> createRecordReader(InputSplit split,
+      TaskAttemptContext context) throws IOException, InterruptedException {
+    return new HLogRecordReader();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
new file mode 100644
index 000000000000..9b1f239b423c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * A tool to replay WAL files as a M/R job.
+ * The WAL can be replayed for a set of tables or all tables,
+ * and a timerange can be provided (in milliseconds).
+ * The WAL is filtered to the passed set of tables and  the output
+ * can optionally be mapped to another set of tables.
+ *
+ * WAL replay can also generate HFiles for later bulk importing,
+ * in that case the WAL is replayed for a single table only.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class WALPlayer extends Configured implements Tool {
+  final static String NAME = "WALPlayer";
+  final static String BULK_OUTPUT_CONF_KEY = "hlog.bulk.output";
+  final static String HLOG_INPUT_KEY = "hlog.input.dir";
+  final static String TABLES_KEY = "hlog.input.tables";
+  final static String TABLE_MAP_KEY = "hlog.input.tablesmap";
+
+  /**
+   * A mapper that just writes out KeyValues.
+   * This one can be used together with {@link KeyValueSortReducer}
+   */
+  static class HLogKeyValueMapper
+  extends Mapper<HLogKey, WALEdit, ImmutableBytesWritable, KeyValue> {
+    private byte[] table;
+
+    @Override
+    public void map(HLogKey key, WALEdit value,
+      Context context)
+    throws IOException {
+      try {
+        // skip all other tables
+        if (Bytes.equals(table, key.getTablename())) {
+          for (KeyValue kv : value.getKeyValues()) {
+            if (HLog.isMetaFamily(kv.getFamily())) continue;
+            context.write(new ImmutableBytesWritable(kv.getRow()), kv);
+          }
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    @Override
+    public void setup(Context context) throws IOException {
+      // only a single table is supported when HFiles are generated with HFileOutputFormat
+      String tables[] = context.getConfiguration().getStrings(TABLES_KEY);
+      if (tables == null || tables.length != 1) {
+        // this can only happen when HLogMapper is used directly by a class other than WALPlayer
+        throw new IOException("Exactly one table must be specified for bulk HFile case.");
+      }
+      table = Bytes.toBytes(tables[0]);
+    }
+  }
+
+  /**
+   * A mapper that writes out {@link Mutation} to be directly applied to
+   * a running HBase instance.
+   */
+  static class HLogMapper
+  extends Mapper<HLogKey, WALEdit, ImmutableBytesWritable, Mutation> {
+    private Map<byte[], byte[]> tables = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
+
+    @Override
+    public void map(HLogKey key, WALEdit value,
+      Context context)
+    throws IOException {
+      try {
+        if (tables.isEmpty() || tables.containsKey(key.getTablename())) {
+          byte[] targetTable = tables.isEmpty() ?
+                key.getTablename() :
+                tables.get(key.getTablename());
+          ImmutableBytesWritable tableOut = new ImmutableBytesWritable(targetTable);
+          Put put = null;
+          Delete del = null;
+          KeyValue lastKV = null;
+          for (KeyValue kv : value.getKeyValues()) {
+            // filtering HLog meta entries, see HLog.completeCacheFlushLogEdit
+            if (HLog.isMetaFamily(kv.getFamily())) continue;
+
+            // A WALEdit may contain multiple operations (HBASE-3584) and/or
+            // multiple rows (HBASE-5229).
+            // Aggregate as much as possible into a single Put/Delete
+            // operation before writing to the context.
+            if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
+              // row or type changed, write out aggregate KVs.
+              if (put != null) context.write(tableOut, put);
+              if (del != null) context.write(tableOut, del);
+
+              if (kv.isDelete()) {
+                del = new Delete(kv.getRow());
+              } else {
+                put = new Put(kv.getRow());
+              }
+            }
+            if (kv.isDelete()) {
+              del.addDeleteMarker(kv);
+            } else {
+              put.add(kv);
+            }
+            lastKV = kv;
+          }
+          // write residual KVs
+          if (put != null) context.write(tableOut, put);
+          if (del != null) context.write(tableOut, del);
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    @Override
+    public void setup(Context context) throws IOException {
+      String[] tableMap = context.getConfiguration().getStrings(TABLE_MAP_KEY);
+      String[] tablesToUse = context.getConfiguration().getStrings(TABLES_KEY);
+      if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) {
+        // this can only happen when HLogMapper is used directly by a class other than WALPlayer
+        throw new IOException("No tables or incorrect table mapping specified.");
+      }
+      int i = 0;
+      for (String table : tablesToUse) {
+        tables.put(Bytes.toBytes(table), Bytes.toBytes(tableMap[i++]));
+      }
+    }
+  }
+
+  /**
+   * @param conf The {@link Configuration} to use.
+   */
+  public WALPlayer(Configuration conf) {
+    super(conf);
+  }
+
+  void setupTime(Configuration conf, String option) throws IOException {
+    String val = conf.get(option);
+    if (val == null) return;
+    long ms;
+    try {
+      // first try to parse in user friendly form
+      ms = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SS").parse(val).getTime();
+    } catch (ParseException pe) {
+      try {
+        // then see if just a number of ms's was specified
+        ms = Long.parseLong(val);
+      } catch (NumberFormatException nfe) {
+        throw new IOException(option
+            + " must be specified either in the form 2001-02-20T16:35:06.99 "
+            + "or as number of milliseconds");
+      }
+    }
+    conf.setLong(option, ms);
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public Job createSubmittableJob(String[] args)
+  throws IOException {
+    Configuration conf = getConf();
+    setupTime(conf, HLogInputFormat.START_TIME_KEY);
+    setupTime(conf, HLogInputFormat.END_TIME_KEY);
+    Path inputDir = new Path(args[0]);
+    String[] tables = args[1].split(",");
+    String[] tableMap;
+    if (args.length > 2) {
+      tableMap = args[2].split(",");
+      if (tableMap.length != tables.length) {
+        throw new IOException("The same number of tables and mapping must be provided.");
+      }
+    } else {
+      // if not mapping is specified map each table to itself
+      tableMap = tables;
+    }
+    conf.setStrings(TABLES_KEY, tables);
+    conf.setStrings(TABLE_MAP_KEY, tableMap);
+    Job job = new Job(conf, NAME + "_" + inputDir);
+    job.setJarByClass(WALPlayer.class);
+    FileInputFormat.setInputPaths(job, inputDir);
+    job.setInputFormatClass(HLogInputFormat.class);
+    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
+    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
+    if (hfileOutPath != null) {
+      // the bulk HFile case
+      if (tables.length != 1) {
+        throw new IOException("Exactly one table must be specified for the bulk export option");
+      }
+      HTable table = new HTable(conf, tables[0]);
+      job.setMapperClass(HLogKeyValueMapper.class);
+      job.setReducerClass(KeyValueSortReducer.class);
+      Path outputDir = new Path(hfileOutPath);
+      FileOutputFormat.setOutputPath(job, outputDir);
+      job.setMapOutputValueClass(KeyValue.class);
+      HFileOutputFormat.configureIncrementalLoad(job, table);
+      TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+          com.google.common.base.Preconditions.class);
+    } else {
+      // output to live cluster
+      job.setMapperClass(HLogMapper.class);
+      job.setOutputFormatClass(MultiTableOutputFormat.class);
+      TableMapReduceUtil.addDependencyJars(job);
+      // No reducers.
+      job.setNumReduceTasks(0);
+    }
+    return job;
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private void usage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    System.err.println("Usage: " + NAME + " [options] <wal inputdir> <tables> [<tableMappings>]");
+    System.err.println("Read all WAL entries for <tables>.");
+    System.err.println("If no tables (\"\") are specific, all tables are imported.");
+    System.err.println("(Careful, even -ROOT- and .META. entries will be imported in that case.)");
+    System.err.println("Otherwise <tables> is a comma separated list of tables.\n");
+    System.err.println("The WAL entries can be mapped to new set of tables via <tableMapping>.");
+    System.err.println("<tableMapping> is a command separated list of targettables.");
+    System.err.println("If specified, each table in <tables> must have a mapping.\n");
+    System.err.println("By default " + NAME + " will load data directly into HBase.");
+    System.err.println("To generate HFiles for a bulk data load instead, pass the option:");
+    System.err.println("  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+    System.err.println("  (Only one table can be specified, and no mapping is allowed!)");
+    System.err.println("Other options: (specify time range to WAL edit to consider)");
+    System.err.println("  -D" + HLogInputFormat.START_TIME_KEY + "=[date|ms]");
+    System.err.println("  -D" + HLogInputFormat.END_TIME_KEY + "=[date|ms]");
+    System.err.println("For performance also consider the following options:\n"
+        + "  -Dmapred.map.tasks.speculative.execution=false\n"
+        + "  -Dmapred.reduce.tasks.speculative.execution=false");
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    int ret = ToolRunner.run(new WALPlayer(HBaseConfiguration.create()), args);
+    System.exit(ret);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
+    if (otherArgs.length < 2) {
+      usage("Wrong number of arguments: " + otherArgs.length);
+      System.exit(-1);
+    }
+    Job job = createSubmittableJob(otherArgs);
+    return job.waitForCompletion(true) ? 0 : 1;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
new file mode 100644
index 000000000000..0b3ba838b659
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.mapreduce.HLogInputFormat.HLogRecordReader;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * JUnit tests for the HLogRecordReader
+ */
+@Category(MediumTests.class)
+public class TestHLogRecordReader {
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static Configuration conf;
+  private static FileSystem fs;
+  private static Path hbaseDir;
+  private static final byte [] tableName = Bytes.toBytes(getName());
+  private static final byte [] rowName = tableName;
+  private static final HRegionInfo info = new HRegionInfo(tableName,
+      Bytes.toBytes(""), Bytes.toBytes(""), false);
+  private static final byte [] family = Bytes.toBytes("column");
+  private static final byte [] value = Bytes.toBytes("value");
+  private static HTableDescriptor htd;
+  private static Path logDir;
+  private static Path oldLogDir;
+
+  private static String getName() {
+    return "TestHLogRecordReader";
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    FileStatus[] entries = fs.listStatus(hbaseDir);
+    for (FileStatus dir : entries) {
+      fs.delete(dir.getPath(), true);
+    }
+
+  }
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // Make block sizes small.
+    conf = TEST_UTIL.getConfiguration();
+    conf.setInt("dfs.blocksize", 1024 * 1024);
+    conf.setInt("dfs.replication", 1);
+    TEST_UTIL.startMiniDFSCluster(1);
+
+    conf = TEST_UTIL.getConfiguration();
+    fs = TEST_UTIL.getDFSCluster().getFileSystem();
+
+    hbaseDir = TEST_UTIL.createRootDir();
+    logDir = new Path(hbaseDir, HConstants.HREGION_LOGDIR_NAME);
+    oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(family));
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Test partial reads from the log based on passed time range
+   * @throws Exception
+   */
+  @Test
+  public void testPartialRead() throws Exception {
+    HLog log = new HLog(fs, logDir, oldLogDir, conf);
+    long ts = System.currentTimeMillis();
+    WALEdit edit = new WALEdit();
+    edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"),
+        ts, value));
+    log.append(info, tableName, edit,
+      ts, htd);
+    edit = new WALEdit();
+    edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"),
+        ts+1, value));
+    log.append(info, tableName, edit,
+        ts+1, htd);
+    log.rollWriter();
+
+    Thread.sleep(1);
+    long ts1 = System.currentTimeMillis();
+
+    edit = new WALEdit();
+    edit.add(new KeyValue(rowName, family, Bytes.toBytes("3"),
+        ts1+1, value));
+    log.append(info, tableName, edit,
+        ts1+1, htd);
+    edit = new WALEdit();
+    edit.add(new KeyValue(rowName, family, Bytes.toBytes("4"),
+        ts1+2, value));
+    log.append(info, tableName, edit,
+        ts1+2, htd);
+    log.close();
+
+    HLogInputFormat input = new HLogInputFormat();
+    Configuration jobConf = new Configuration(conf);
+    jobConf.set("mapred.input.dir", logDir.toString());
+    jobConf.setLong(HLogInputFormat.END_TIME_KEY, ts);
+
+    // only 1st file is considered, and only its 1st entry is used
+    List<InputSplit> splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    assertEquals(1, splits.size());
+    testSplit(splits.get(0), Bytes.toBytes("1"));
+
+    jobConf.setLong(HLogInputFormat.START_TIME_KEY, ts+1);
+    jobConf.setLong(HLogInputFormat.END_TIME_KEY, ts1+1);
+    splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    // both files need to be considered
+    assertEquals(2, splits.size());
+    // only the 2nd entry from the 1st file is used
+    testSplit(splits.get(0), Bytes.toBytes("2"));
+    // only the 1nd entry from the 2nd file is used
+    testSplit(splits.get(1), Bytes.toBytes("3"));
+  }
+
+  /**
+   * Test basic functionality
+   * @throws Exception
+   */
+  @Test
+  public void testHLogRecordReader() throws Exception {
+    HLog log = new HLog(fs, logDir, oldLogDir, conf);
+    byte [] value = Bytes.toBytes("value");
+    WALEdit edit = new WALEdit();
+    edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"),
+        System.currentTimeMillis(), value));
+    log.append(info, tableName, edit,
+      System.currentTimeMillis(), htd);
+
+    Thread.sleep(1); // make sure 2nd log gets a later timestamp
+    long secondTs = System.currentTimeMillis();
+    log.rollWriter();
+
+    edit = new WALEdit();
+    edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"),
+        System.currentTimeMillis(), value));
+    log.append(info, tableName, edit,
+      System.currentTimeMillis(), htd);
+    log.close();
+    long thirdTs = System.currentTimeMillis();
+
+    // should have 2 log files now
+    HLogInputFormat input = new HLogInputFormat();
+    Configuration jobConf = new Configuration(conf);
+    jobConf.set("mapred.input.dir", logDir.toString());
+
+    // make sure both logs are found
+    List<InputSplit> splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    assertEquals(2, splits.size());
+
+    // should return exactly one KV
+    testSplit(splits.get(0), Bytes.toBytes("1"));
+    // same for the 2nd split
+    testSplit(splits.get(1), Bytes.toBytes("2"));
+
+    // now test basic time ranges:
+
+    // set an endtime, the 2nd log file can be ignored completely.
+    jobConf.setLong(HLogInputFormat.END_TIME_KEY, secondTs-1);
+    splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    assertEquals(1, splits.size());
+    testSplit(splits.get(0), Bytes.toBytes("1"));
+
+    // now set a start time
+    jobConf.setLong(HLogInputFormat.END_TIME_KEY, Long.MAX_VALUE);
+    jobConf.setLong(HLogInputFormat.START_TIME_KEY, thirdTs);
+    splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    // both logs need to be considered
+    assertEquals(2, splits.size());
+    // but both readers skip all edits
+    testSplit(splits.get(0));
+    testSplit(splits.get(1));
+  }
+
+  /**
+   * Create a new reader from the split, and match the edits against the passed columns.
+   */
+  private void testSplit(InputSplit split, byte[]... columns) throws Exception {
+    HLogRecordReader reader = new HLogRecordReader();
+    reader.initialize(split, new TaskAttemptContext(conf, new TaskAttemptID()));
+
+    for (byte[] column : columns) {
+      assertTrue(reader.nextKeyValue());
+      assertTrue(Bytes
+          .equals(column, reader.getCurrentValue().getKeyValues().get(0).getQualifier()));
+    }
+    assertFalse(reader.nextKeyValue());
+    reader.close();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
new file mode 100644
index 000000000000..674b1c8da0eb
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Basic test for the WALPlayer M/R tool
+ */
+public class TestWALPlayer {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static MiniHBaseCluster cluster;
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    cluster = TEST_UTIL.startMiniCluster();
+    TEST_UTIL.startMiniMapReduceCluster();
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Simple end-to-end test
+   * @throws Exception
+   */
+  @Test
+  public void testWALPlayer() throws Exception {
+    final byte[] TABLENAME1 = Bytes.toBytes("testWALPlayer1");
+    final byte[] TABLENAME2 = Bytes.toBytes("testWALPlayer2");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[] COLUMN1 = Bytes.toBytes("c1");
+    final byte[] COLUMN2 = Bytes.toBytes("c2");
+    final byte[] ROW = Bytes.toBytes("row");
+    HTable t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
+    HTable t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
+
+    // put a row into the first table
+    Put p = new Put(ROW);
+    p.add(FAMILY, COLUMN1, COLUMN1);
+    p.add(FAMILY, COLUMN2, COLUMN2);
+    t1.put(p);
+    // delete one column
+    Delete d = new Delete(ROW);
+    d.deleteColumns(FAMILY, COLUMN1);
+    t1.delete(d);
+
+    // replay the WAL, map table 1 to table 2
+    HLog log = cluster.getRegionServer(0).getWAL();
+    log.rollWriter();
+    String walInputDir = new Path(cluster.getMaster().getMasterFileSystem()
+        .getRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
+
+    WALPlayer player = new WALPlayer(TEST_UTIL.getConfiguration());
+    assertEquals(0, player.run(new String[] { walInputDir, Bytes.toString(TABLENAME1),
+        Bytes.toString(TABLENAME2) }));
+
+    // verify the WAL was player into table 2
+    Get g = new Get(ROW);
+    Result r = t2.get(g);
+    assertEquals(1, r.size());
+    assertTrue(Bytes.equals(COLUMN2, r.raw()[0].getQualifier()));
+  }
+
+  /**
+   * Simple test for data parsing
+   * @throws Exception
+   */
+  @Test
+  public void testTimeFormat() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    WALPlayer player = new WALPlayer(TEST_UTIL.getConfiguration());
+    player.setupTime(conf, HLogInputFormat.END_TIME_KEY);
+    // make sure if nothing is specified nothing is set
+    assertNull(conf.get(HLogInputFormat.END_TIME_KEY));
+    // test a textual data (including ms)
+    conf.set(HLogInputFormat.END_TIME_KEY, "2012-4-10T14:21:01.01");
+    player.setupTime(conf, HLogInputFormat.END_TIME_KEY);
+    assertEquals(1334092861001L, conf.getLong(HLogInputFormat.END_TIME_KEY, 0));
+    // test with mss as a long
+    conf.set(HLogInputFormat.END_TIME_KEY, "1334092861010");
+    player.setupTime(conf, HLogInputFormat.END_TIME_KEY);
+    assertEquals(1334092861010L, conf.getLong(HLogInputFormat.END_TIME_KEY, 0));
+  }
+}

From 2761be1e72aa247feb86e9cfa2dc43198b9296f3 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 12 Apr 2012 23:00:09 +0000
Subject: [PATCH 0128/1540] HBASE-5778  Turn on WAL compression by default

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325567 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index fbe1ce5ca672..c18414046e57 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -130,7 +130,7 @@ static boolean isWALCompressionEnabled(final Metadata metadata) {
   public void init(FileSystem fs, Path path, Configuration conf)
   throws IOException {
     // Should we do our custom WAL compression?
-    boolean compress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);
+    boolean compress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
     if (compress) {
       try {
         if (this.compressionContext == null) {

From 0c106bb6e1c0cd67bfcdee1297c3ee4c105d3a32 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 13 Apr 2012 04:12:58 +0000
Subject: [PATCH 0129/1540] HBASE-5604 addendum, mark TestWALPlayer as
 LargeTest

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325608 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
index 674b1c8da0eb..212272b75902 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
@@ -36,10 +37,12 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 /**
  * Basic test for the WALPlayer M/R tool
  */
+@Category(LargeTests.class)
 public class TestWALPlayer {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static MiniHBaseCluster cluster;

From c9505c7f468b5f123f9c747efe917b946923e545 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 13 Apr 2012 06:44:42 +0000
Subject: [PATCH 0130/1540] HBASE-5488 OfflineMetaRepair doesn't support hadoop
 0.20's fs.default.name property (gaojinchao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325626 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index d73821a80a7c..80846b3d3c4a 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -62,7 +62,10 @@ public static void main(String[] args) throws Exception {
 
     // create a fsck object
     Configuration conf = HBaseConfiguration.create();
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
     conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
+    conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR));
     HBaseFsck fsck = new HBaseFsck(conf);
     boolean fixHoles = false;
 
@@ -77,6 +80,7 @@ public static void main(String[] args) throws Exception {
         String path = args[i];
         conf.set(HConstants.HBASE_DIR, path);
         conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
+        conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR));
       } else if (cmd.equals("-fixHoles")) {
         fixHoles = true;
       } else if (cmd.equals("-fix")) {

From 216efbe875ffde7660404106badf1570b3094d8b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 13 Apr 2012 15:25:18 +0000
Subject: [PATCH 0131/1540] HBASE-5778 Turn on WAL compression by default:
 REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1325803 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index c18414046e57..fbe1ce5ca672 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -130,7 +130,7 @@ static boolean isWALCompressionEnabled(final Metadata metadata) {
   public void init(FileSystem fs, Path path, Configuration conf)
   throws IOException {
     // Should we do our custom WAL compression?
-    boolean compress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+    boolean compress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);
     if (compress) {
       try {
         if (this.compressionContext == null) {

From 7626839652ca95050d3f04be7c252eae437305f4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 13 Apr 2012 22:49:03 +0000
Subject: [PATCH 0132/1540] HBASE-5604 Addendum. Remove test as per discussion
 on jira.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326001 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/TestWALPlayer.java | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
index 212272b75902..93653afa703d 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -100,25 +100,4 @@ public void testWALPlayer() throws Exception {
     assertEquals(1, r.size());
     assertTrue(Bytes.equals(COLUMN2, r.raw()[0].getQualifier()));
   }
-
-  /**
-   * Simple test for data parsing
-   * @throws Exception
-   */
-  @Test
-  public void testTimeFormat() throws Exception {
-    Configuration conf = TEST_UTIL.getConfiguration();
-    WALPlayer player = new WALPlayer(TEST_UTIL.getConfiguration());
-    player.setupTime(conf, HLogInputFormat.END_TIME_KEY);
-    // make sure if nothing is specified nothing is set
-    assertNull(conf.get(HLogInputFormat.END_TIME_KEY));
-    // test a textual data (including ms)
-    conf.set(HLogInputFormat.END_TIME_KEY, "2012-4-10T14:21:01.01");
-    player.setupTime(conf, HLogInputFormat.END_TIME_KEY);
-    assertEquals(1334092861001L, conf.getLong(HLogInputFormat.END_TIME_KEY, 0));
-    // test with mss as a long
-    conf.set(HLogInputFormat.END_TIME_KEY, "1334092861010");
-    player.setupTime(conf, HLogInputFormat.END_TIME_KEY);
-    assertEquals(1334092861010L, conf.getLong(HLogInputFormat.END_TIME_KEY, 0));
-  }
 }

From 34e8e03808ce16dfa7fd88ed6f401af3579ea969 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 13 Apr 2012 22:54:31 +0000
Subject: [PATCH 0133/1540] Updating CHANGES.txt for 0.94.0RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326003 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 45 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index ae76f152f59f..22bb92193d5f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.0 - 3/27/2012
+Release 0.94.0 - 4/13/2012
 Sub-task
 
     [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently
@@ -33,11 +33,13 @@ Sub-task
     [HBASE-5497] - Add protobuf as M/R dependency jar (mapred)
     [HBASE-5523] - Fix Delete Timerange logic for KEEP_DELETED_CELLS
     [HBASE-5541] - Avoid holding the rowlock during HLog sync in HRegion.mutateRowWithLocks
+    [HBASE-5638] - Backport to 0.90 and 0.92 - NPE reading ZK config in HBase
     [HBASE-5641] - decayingSampleTick1 prevents HBase from shutting down.
 
 Bug
 
     [HBASE-2856] - TestAcidGuarantee broken on trunk
+    [HBASE-3443] - ICV optimization to look in memstore first and then store files (HBASE-3082) does not work when deletes are in the mix
     [HBASE-3690] - Option to Exclude Bulk Import Files from Minor Compaction
     [HBASE-3987] - Fix a NullPointerException on a failure to load Bloom filter data
     [HBASE-4065] - TableOutputFormat ignores failure to create table instance
@@ -46,6 +48,7 @@ Bug
     [HBASE-4116] - [stargate] StringIndexOutOfBoundsException in row spec parse
     [HBASE-4326] - Tests that use HBaseTestingUtility.startMiniCluster(n) should shutdown with HBaseTestingUtility.shutdownMiniCluster.
     [HBASE-4397] - -ROOT-, .META. tables stay offline for too long in recovery phase after all RSs are shutdown at the same time
+    [HBASE-4398] - If HRegionPartitioner is used in MapReduce, client side configurations are overwritten by hbase-site.xml.
     [HBASE-4476] - Compactions must fail if column tracker gets columns out of order
     [HBASE-4496] - HFile V2 does not honor setCacheBlocks when scanning.
     [HBASE-4607] - Split log worker should terminate properly when waiting for znode
@@ -97,6 +100,7 @@ Bug
     [HBASE-5085] - fix test-patch script from setting the ulimit
     [HBASE-5088] - A concurrency issue on SoftValueSortedMap
     [HBASE-5091] - [replication] Update replication doc to reflect current znode structure
+    [HBASE-5097] - RegionObserver implementation whose preScannerOpen and postScannerOpen Impl return null can stall the system initialization through NPE
     [HBASE-5099] - ZK event thread waiting for root region assignment may block server shutdown handler for the region sever the root region was on
     [HBASE-5100] - Rollback of split could cause closed region to be opened again
     [HBASE-5103] - Fix improper master znode deserialization
@@ -114,6 +118,7 @@ Bug
     [HBASE-5200] - AM.ProcessRegionInTransition() and AM.handleRegion() race thus leaving the region assignment inconsistent
     [HBASE-5206] - Port HBASE-5155 to 0.92, 0.94, and TRUNK
     [HBASE-5212] - Fix test TestTableMapReduce against 0.23.
+    [HBASE-5213] - "hbase master stop" does not bring down backup masters
     [HBASE-5221] - bin/hbase script doesn't look for Hadoop jars in the right place in trunk layout
     [HBASE-5228] - [REST] Rip out "transform" feature
     [HBASE-5267] - Add a configuration to disable the slab cache by default
@@ -150,6 +155,7 @@ Bug
     [HBASE-5481] - Uncaught UnknownHostException prevents HBase from starting
     [HBASE-5484] - Spelling mistake in error message in HMasterCommandLine
     [HBASE-5485] - LogCleaner refers to non-existant SnapshotLogCleaner
+    [HBASE-5488] - OfflineMetaRepair doesn't support hadoop 0.20's fs.default.name property
     [HBASE-5499] - dev-support/test-patch.sh does not have execute perms
     [HBASE-5502] - region_mover.rb fails to load regions back to original server for regions only containing empty tables.
     [HBASE-5507] - ThriftServerRunner.HbaseHandler.getRegionInfo() and getTableRegions() do not use ByteBuffer correctly
@@ -173,12 +179,29 @@ Bug
     [HBASE-5596] - Few minor bugs from HBASE-5209
     [HBASE-5597] - Findbugs check in test-patch.sh always fails
     [HBASE-5603] - rolling-restart.sh script hangs when attempting to detect expiration of /hbase/master znode.
+    [HBASE-5606] - SplitLogManger async delete node hangs log splitting when ZK connection is lost
     [HBASE-5613] - ThriftServer getTableRegions does not return serverName and port
-    [HBASE-5615] - the master never does balance because of balancing the parent region
     [HBASE-5623] - Race condition when rolling the HLog and hlogFlush
     [HBASE-5624] - Aborting regionserver when splitting region, may cause daughter region not assigned by ServerShutdownHandler.
     [HBASE-5633] - NPE reading ZK config in HBase
+    [HBASE-5636] - TestTableMapReduce doesn't work properly.
     [HBASE-5639] - The logic used in waiting for region servers during startup is broken
+    [HBASE-5656] - LoadIncrementalHFiles createTable should detect and set compression algorithm
+    [HBASE-5663] - MultithreadedTableMapper doesn't work.
+    [HBASE-5665] - Repeated split causes HRegionServer failures and breaks table
+    [HBASE-5669] - AggregationClient fails validation for open stoprow scan
+    [HBASE-5680] - Improve compatibility warning about HBase with Hadoop 0.23.x
+    [HBASE-5689] - Skipping RecoveredEdits may cause data loss
+    [HBASE-5690] - compression does not work in Store.java of 0.94
+    [HBASE-5694] - getRowsWithColumnsTs() in Thrift service handles timestamps incorrectly
+    [HBASE-5701] - Put RegionServerDynamicStatistics under RegionServer in MBean hierarchy rather than have it as a peer.
+    [HBASE-5717] - Scanner metrics are only reported if you get to the end of a scanner
+    [HBASE-5720] - HFileDataBlockEncoderImpl uses wrong header size when reading HFiles with no checksums
+    [HBASE-5722] - NPE in ZKUtil#getChildDataAndWatchForNewChildren when ZK not available or NW down.
+    [HBASE-5724] - Row cache of KeyValue should be cleared in readFields().
+    [HBASE-5736] - ThriftServerRunner.HbaseHandler.mutateRow() does not use ByteBuffer correctly
+    [HBASE-5743] - Support GIT patches
+    [HBASE-5773] - HtablePool constructor not reading config files in certain cases
 
 Improvement
 
@@ -311,10 +334,21 @@ Improvement
     [HBASE-5588] - Deprecate/remove AssignmentManager#clearRegionFromTransition
     [HBASE-5589] - Add of the offline call to the Master Interface
     [HBASE-5592] - Make it easier to get a table from shell
+    [HBASE-5618] - SplitLogManager - prevent unnecessary attempts to resubmits
+    [HBASE-5670] - Have Mutation implement the Row interface.
+    [HBASE-5671] - hbase.metrics.showTableName should be true by default
+    [HBASE-5682] - Allow HConnectionImplementation to recover from ZK connection loss (for 0.94 only)
+    [HBASE-5706] - "Dropping fs latency stats since buffer is full" spam
+    [HBASE-5734] - Change hbck sideline root
+    [HBASE-5735] - Clearer warning message when connecting a non-secure HBase client to a secure HBase server
+    [HBASE-5748] - Enable lib directory in jar file for coprocessor
+    [HBASE-5770] - Add a clock skew warning threshold
+    [HBASE-5775] - ZKUtil doesn't handle deleteRecurisively cleanly
 
 New Feature
 
     [HBASE-2947] - MultiIncrement/MultiAppend (MultiGet functionality for increments and appends)
+    [HBASE-3134] - [replication] Add the ability to enable/disable streams
     [HBASE-3584] - Allow atomic put/delete in one call
     [HBASE-3856] - Build a tree structure data block index inside of the HFile
     [HBASE-4102] - atomicAppend: A put that appends to the latest version of a cell; i.e. reads current value then adds the bytes offered by the client to the tail and writes out a new entry
@@ -330,6 +364,9 @@ New Feature
     [HBASE-5177] - HTable needs a non cached version of getRegionLocation
     [HBASE-5229] - Provide basic building blocks for "multi-row" local transactions.
     [HBASE-5526] - Configurable file and directory based umask
+    [HBASE-5599] - [hbck] handle NO_VERSION_FILE and SHOULD_NOT_BE_DEPLOYED inconsistencies
+    [HBASE-5604] - M/R tool to replay WAL files
+    [HBASE-5719] - Enhance hbck to sideline overlapped mega regions
 
 Task
 
@@ -340,6 +377,7 @@ Task
     [HBASE-4751] - Make TestAdmin#testEnableTableRoundRobinAssignment friendly to concurrent tests
     [HBASE-4968] - Add to troubleshooting workaround for direct buffer oome's.
     [HBASE-5011] - Move test-util.sh from src/test/bin to dev-tools
+    [HBASE-5084] - Allow different HTable instances to share one ExecutorService
     [HBASE-5111] - Upgrade zookeeper to 3.4.2 release
     [HBASE-5173] - Commit hbase-4480 findHangingTest.sh script under dev-support
     [HBASE-5264] - Add 0.92.0 upgrade guide
@@ -347,6 +385,9 @@ Task
     [HBASE-5400] - Some tests does not have annotations for (Small|Medium|Large)Tests
     [HBASE-5427] - Upgrade our zk to 3.4.3
     [HBASE-5511] - More doc on maven release process
+    [HBASE-5715] - Revert 'Instant schema alter' for now, HBASE-4213
+    [HBASE-5721] - Update bundled hadoop to be 1.0.2 (it was just released)
+    [HBASE-5758] - Forward port "HBASE-4109 Hostname returned via reverse dns lookup contains trailing period if configured interface is not 'default'"
 
 Test
 

From ad6e57645fb804a8746d155b5596a7fa3af8ed49 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 14 Apr 2012 13:28:42 +0000
Subject: [PATCH 0134/1540] HBASE-5780 Fix race in HBase regionserver startup
 vs ZK SASL authentication (Shaneal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326101 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index c6e607ef1334..ea8799097510 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -71,6 +71,12 @@ public ZooKeeperNodeTracker(ZooKeeperWatcher watcher, String node,
    * or {@link #getData(boolean)} to get the data of the node if it is available.
    */
   public synchronized void start() {
+    try {
+      ZKUtil.waitForZKConnectionIfAuthenticating(watcher);
+    } catch (InterruptedException e) {
+      throw new IllegalStateException("ZookeeperNodeTracker on " + this.node 
+          + " interuppted while waiting for SASL Authentication", e);
+    }
     this.watcher.registerListener(this);
     try {
       if(ZKUtil.watchAndCheckExists(watcher, node)) {

From 7ffa120a8bdcb18173dc315358e001d884bb0bf9 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 14 Apr 2012 14:56:00 +0000
Subject: [PATCH 0135/1540] HBASE-5780 revert due to
 TestReplicationPeer#testResetZooKeeperSession failure

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326122 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index ea8799097510..c6e607ef1334 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -71,12 +71,6 @@ public ZooKeeperNodeTracker(ZooKeeperWatcher watcher, String node,
    * or {@link #getData(boolean)} to get the data of the node if it is available.
    */
   public synchronized void start() {
-    try {
-      ZKUtil.waitForZKConnectionIfAuthenticating(watcher);
-    } catch (InterruptedException e) {
-      throw new IllegalStateException("ZookeeperNodeTracker on " + this.node 
-          + " interuppted while waiting for SASL Authentication", e);
-    }
     this.watcher.registerListener(this);
     try {
       if(ZKUtil.watchAndCheckExists(watcher, node)) {

From 7f7bc2b6b62a85721fc059cc7d429f9a0be580d4 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Sun, 15 Apr 2012 08:59:12 +0000
Subject: [PATCH 0136/1540] HBASE-5781 Zookeeper session got closed while
 trying to assign the region to RS using hbck -fix

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326281 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/HBaseFsckRepair.java    | 44 +++++++------------
 1 file changed, 17 insertions(+), 27 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index 1741aee6644b..81ad866105ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -114,38 +114,28 @@ private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region)
    */
   public static void waitUntilAssigned(HBaseAdmin admin,
       HRegionInfo region) throws IOException, InterruptedException {
-    HConnection connection = admin.getConnection();
-
-    try {
-      long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
-      long expiration = timeout + System.currentTimeMillis();
-      while (System.currentTimeMillis() < expiration) {
-        try {
-          Map<String, RegionState> rits=
+    long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
+    long expiration = timeout + System.currentTimeMillis();
+    while (System.currentTimeMillis() < expiration) {
+      try {
+        Map<String, RegionState> rits=
             admin.getClusterStatus().getRegionsInTransition();
 
-          if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
-            // yay! no longer RIT
-            return;
-          }
-          // still in rit
-          LOG.info("Region still in transition, waiting for "
-              + "it to become assigned: " + region);
-        } catch (IOException e) {
-          LOG.warn("Exception when waiting for region to become assigned,"
-              + " retrying", e);
+        if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
+          // yay! no longer RIT
+          return;
         }
-        Thread.sleep(1000);
-      }
-      throw new IOException("Region " + region + " failed to move out of " +
-          "transition within timeout " + timeout + "ms");
-    } finally {
-      try {
-        connection.close();
-      } catch (IOException ioe) {
-        throw ioe;
+        // still in rit
+        LOG.info("Region still in transition, waiting for "
+            + "it to become assigned: " + region);
+      } catch (IOException e) {
+        LOG.warn("Exception when waiting for region to become assigned,"
+            + " retrying", e);
       }
+      Thread.sleep(1000);
     }
+    throw new IOException("Region " + region + " failed to move out of " +
+        "transition within timeout " + timeout + "ms");
   }
 
   /**

From abbe3d9ef370eb27b0b2312e2f48e0e54e79be53 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Sun, 15 Apr 2012 22:43:52 +0000
Subject: [PATCH 0137/1540] HBASE-5793 TestHBaseFsck#testNoHdfsTable tests
 hangs after client retries increased

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326435 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/TestHBaseFsck.java  | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index a06942d38a6c..4ea8ee334ef2 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -23,6 +23,7 @@
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -45,8 +46,8 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -56,12 +57,11 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -849,14 +849,8 @@ public void testNoHdfsTable() throws Exception {
 
     // check that hole fixed
     assertNoErrors(doFsck(conf,false));
-
-    try {
-      assertEquals(0, countRows());
-    } catch (IOException ioe) {
-      // we've actually deleted the table already. :)
-      return;
-    }
-    fail("Should have failed with IOException");
+    assertFalse("Table "+ table + " should have been deleted",
+        TEST_UTIL.getHBaseAdmin().tableExists(table));
   }
 
   /**

From db1ed7ee339dea0bfb783ee5ea1ddb80bf883cdb Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 16 Apr 2012 20:43:07 +0000
Subject: [PATCH 0138/1540] HBASE-5795 HServerLoad$RegionLoad breaks
 0.92<->0.94 compatibility

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326791 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HServerLoad.java  |  38 +-
 .../apache/hadoop/hbase/HServerLoad092.java   | 693 ++++++++++++++++++
 .../hadoop/hbase/TestSerialization.java       |  15 +
 3 files changed, 744 insertions(+), 2 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/HServerLoad092.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
index e6d5346ba567..f683ebed80e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
@@ -374,11 +374,46 @@ public void setCurrentCompactedKVs(long currentCompactedKVs) {
       this.currentCompactedKVs = currentCompactedKVs;
     }
 
+    /**
+     * HBASE-5256 and HBASE-5283 introduced incompatible serialization changes
+     * This method reads the fields in 0.92 serialization format, ex-version field
+     * @param in
+     * @throws IOException
+     */
+    private void readFields92(DataInput in) throws IOException {
+      // in 0.92, the version was actually written twice, consume the second copy
+      int version = in.readByte();
+      int namelen = in.readInt();
+      this.name = new byte[namelen];
+      in.readFully(this.name);
+      this.stores = in.readInt();
+      this.storefiles = in.readInt();
+      this.storeUncompressedSizeMB = in.readInt();
+      this.storefileSizeMB = in.readInt();
+      this.memstoreSizeMB = in.readInt();
+      this.storefileIndexSizeMB = in.readInt();
+      this.readRequestsCount = in.readInt();
+      this.writeRequestsCount = in.readInt();
+      this.rootIndexSizeKB = in.readInt();
+      this.totalStaticIndexSizeKB = in.readInt();
+      this.totalStaticBloomSizeKB = in.readInt();
+      this.totalCompactingKVs = in.readLong();
+      this.currentCompactedKVs = in.readLong();
+      int coprocessorsSize = in.readInt();
+      coprocessors = new TreeSet<String>();
+      for (int i = 0; i < coprocessorsSize; i++) {
+        coprocessors.add(in.readUTF());
+      }
+    }
+    
     // Writable
     public void readFields(DataInput in) throws IOException {
-      super.readFields(in);
       int version = in.readByte();
       if (version > VERSION) throw new IOException("Version mismatch; " + version);
+      if (version == 1) { 
+        readFields92(in);
+        return;
+      }
       int namelen = WritableUtils.readVInt(in);
       this.name = new byte[namelen];
       in.readFully(this.name);
@@ -404,7 +439,6 @@ public void readFields(DataInput in) throws IOException {
 
     public void write(DataOutput out) throws IOException {
       super.write(out);
-      out.writeByte(VERSION);
       WritableUtils.writeVInt(out, name.length);
       out.write(name);
       WritableUtils.writeVInt(out, stores);
diff --git a/src/test/java/org/apache/hadoop/hbase/HServerLoad092.java b/src/test/java/org/apache/hadoop/hbase/HServerLoad092.java
new file mode 100644
index 000000000000..203dcd2fde38
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/HServerLoad092.java
@@ -0,0 +1,693 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Strings;
+import org.apache.hadoop.io.VersionedWritable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * This class is used to export current state of load on a RegionServer.
+ * This is the version of HServerLoad that we had in 0.92.
+ */
+public class HServerLoad092 extends VersionedWritable
+implements WritableComparable<HServerLoad092> {
+  private static final byte VERSION = 2;
+  // Empty load instance.
+  public static final HServerLoad092 EMPTY_HSERVERLOAD = new HServerLoad092();
+
+  /** Number of requests per second since last report.
+   */
+  // TODO: Instead build this up out of region counters.
+  private int numberOfRequests = 0;
+
+  /** Total Number of requests from the start of the region server.
+   */
+  private int totalNumberOfRequests = 0;
+  
+  /** the amount of used heap, in MB */
+  private int usedHeapMB = 0;
+
+  /** the maximum allowable size of the heap, in MB */
+  private int maxHeapMB = 0;
+
+  // Regionserver-level coprocessors, e.g., WALObserver implementations.
+  // Region-level coprocessors, on the other hand, are stored inside RegionLoad
+  // objects.
+  private Set<String> coprocessors =
+      new TreeSet<String>();
+
+  /**
+   * HBASE-4070: Improve region server metrics to report loaded coprocessors.
+   *
+   * @return Returns the set of all coprocessors on this
+   * regionserver, where this set is the union of the
+   * regionserver-level coprocessors on one hand, and all of the region-level
+   * coprocessors, on the other.
+   *
+   * We must iterate through all regions loaded on this regionserver to
+   * obtain all of the region-level coprocessors.
+   */
+  public String[] getCoprocessors() {
+    TreeSet<String> returnValue = new TreeSet<String>(coprocessors);
+    for (Map.Entry<byte[], RegionLoad> rls: getRegionsLoad().entrySet()) {
+      for (String coprocessor: rls.getValue().getCoprocessors()) {
+        returnValue.add(coprocessor);
+      }
+    }
+    return returnValue.toArray(new String[0]);
+  }
+
+  /** per-region load metrics */
+  private Map<byte[], RegionLoad> regionLoad =
+    new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
+
+  /** @return the object version number */
+  public byte getVersion() {
+    return VERSION;
+  }
+
+  /**
+   * Encapsulates per-region loading metrics.
+   */
+  public static class RegionLoad extends VersionedWritable {
+    private static final byte VERSION = 1;
+
+    /** @return the object version number */
+    public byte getVersion() {
+      return VERSION;
+    }
+
+    /** the region name */
+    private byte[] name;
+    /** the number of stores for the region */
+    private int stores;
+    /** the number of storefiles for the region */
+    private int storefiles;
+    /** the total size of the store files for the region, uncompressed, in MB */
+    private int storeUncompressedSizeMB;
+    /** the current total size of the store files for the region, in MB */
+    private int storefileSizeMB;
+    /** the current size of the memstore for the region, in MB */
+    private int memstoreSizeMB;
+
+    /**
+     * The current total size of root-level store file indexes for the region,
+     * in MB. The same as {@link #rootIndexSizeKB} but in MB.
+     */
+    private int storefileIndexSizeMB;
+    /** the current total read requests made to region */
+    private int readRequestsCount;
+    /** the current total write requests made to region */
+    private int writeRequestsCount;
+    /** the total compacting key values in currently running compaction */
+    private long totalCompactingKVs;
+    /** the completed count of key values in currently running compaction */
+    private long currentCompactedKVs;
+
+    /** The current total size of root-level indexes for the region, in KB. */
+    private int rootIndexSizeKB;
+
+    /** The total size of all index blocks, not just the root level, in KB. */
+    private int totalStaticIndexSizeKB;
+
+    /**
+     * The total size of all Bloom filter blocks, not just loaded into the
+     * block cache, in KB.
+     */
+    private int totalStaticBloomSizeKB;
+
+    // Region-level coprocessors.
+    Set<String> coprocessors =
+        new TreeSet<String>();
+
+    /**
+     * Constructor, for Writable
+     */
+    public RegionLoad() {
+        super();
+    }
+
+    /**
+     * @param name
+     * @param stores
+     * @param storefiles
+     * @param storeUncompressedSizeMB
+     * @param storefileSizeMB
+     * @param memstoreSizeMB
+     * @param storefileIndexSizeMB
+     * @param readRequestsCount
+     * @param writeRequestsCount
+     * @param totalCompactingKVs
+     * @param currentCompactedKVs
+     * @param coprocessors
+     */
+    public RegionLoad(final byte[] name, final int stores,
+        final int storefiles, final int storeUncompressedSizeMB,
+        final int storefileSizeMB,
+        final int memstoreSizeMB, final int storefileIndexSizeMB,
+        final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
+        final int totalStaticBloomSizeKB,
+        final int readRequestsCount, final int writeRequestsCount,
+        final long totalCompactingKVs, final long currentCompactedKVs,
+        final Set<String> coprocessors) {
+      this.name = name;
+      this.stores = stores;
+      this.storefiles = storefiles;
+      this.storeUncompressedSizeMB = storeUncompressedSizeMB;
+      this.storefileSizeMB = storefileSizeMB;
+      this.memstoreSizeMB = memstoreSizeMB;
+      this.storefileIndexSizeMB = storefileIndexSizeMB;
+      this.rootIndexSizeKB = rootIndexSizeKB;
+      this.totalStaticIndexSizeKB = totalStaticIndexSizeKB;
+      this.totalStaticBloomSizeKB = totalStaticBloomSizeKB;
+      this.readRequestsCount = readRequestsCount;
+      this.writeRequestsCount = writeRequestsCount;
+      this.totalCompactingKVs = totalCompactingKVs;
+      this.currentCompactedKVs = currentCompactedKVs;
+      this.coprocessors = coprocessors;
+    }
+
+    // Getters
+    private String[] getCoprocessors() {
+      return coprocessors.toArray(new String[0]);
+    }
+
+    /**
+     * @return the region name
+     */
+    public byte[] getName() {
+      return name;
+    }
+
+    /**
+     * @return the region name as a string
+     */
+    public String getNameAsString() {
+      return Bytes.toString(name);
+    }
+
+    /**
+     * @return the number of stores
+     */
+    public int getStores() {
+      return stores;
+    }
+
+    /**
+     * @return the number of storefiles
+     */
+    public int getStorefiles() {
+      return storefiles;
+    }
+
+    /**
+     * @return the total size of the storefiles, in MB
+     */
+    public int getStorefileSizeMB() {
+      return storefileSizeMB;
+    }
+
+    /**
+     * @return the memstore size, in MB
+     */
+    public int getMemStoreSizeMB() {
+      return memstoreSizeMB;
+    }
+
+    /**
+     * @return the approximate size of storefile indexes on the heap, in MB
+     */
+    public int getStorefileIndexSizeMB() {
+      return storefileIndexSizeMB;
+    }
+
+    /**
+     * @return the number of requests made to region
+     */
+    public long getRequestsCount() {
+      return readRequestsCount + writeRequestsCount;
+    }
+
+    /**
+     * @return the number of read requests made to region
+     */
+    public long getReadRequestsCount() {
+      return readRequestsCount;
+    }
+
+    /**
+     * @return the number of read requests made to region
+     */
+    public long getWriteRequestsCount() {
+      return writeRequestsCount;
+    }
+
+    /**
+     * @return the total number of kvs in current compaction
+     */
+    public long getTotalCompactingKVs() {
+      return totalCompactingKVs;
+    }
+
+    /**
+     * @return the number of already compacted kvs in current compaction
+     */
+    public long getCurrentCompactedKVs() {
+      return currentCompactedKVs;
+    }
+
+    // Setters
+
+    /**
+     * @param name the region name
+     */
+    public void setName(byte[] name) {
+      this.name = name;
+    }
+
+    /**
+     * @param stores the number of stores
+     */
+    public void setStores(int stores) {
+      this.stores = stores;
+    }
+
+    /**
+     * @param storefiles the number of storefiles
+     */
+    public void setStorefiles(int storefiles) {
+      this.storefiles = storefiles;
+    }
+
+    /**
+     * @param memstoreSizeMB the memstore size, in MB
+     */
+    public void setMemStoreSizeMB(int memstoreSizeMB) {
+      this.memstoreSizeMB = memstoreSizeMB;
+    }
+
+    /**
+     * @param storefileIndexSizeMB the approximate size of storefile indexes
+     *  on the heap, in MB
+     */
+    public void setStorefileIndexSizeMB(int storefileIndexSizeMB) {
+      this.storefileIndexSizeMB = storefileIndexSizeMB;
+    }
+
+    /**
+     * @param requestsCount the number of read requests to region
+     */
+    public void setReadRequestsCount(int requestsCount) {
+      this.readRequestsCount = requestsCount;
+    }
+
+    /**
+     * @param requestsCount the number of write requests to region
+     */
+    public void setWriteRequestsCount(int requestsCount) {
+      this.writeRequestsCount = requestsCount;
+    }
+
+    /**
+     * @param totalCompactingKVs the number of kvs total in current compaction
+     */
+    public void setTotalCompactingKVs(long totalCompactingKVs) {
+      this.totalCompactingKVs = totalCompactingKVs;
+    }
+
+    /**
+     * @param currentCompactedKVs the number of kvs already compacted in
+     * current compaction
+     */
+    public void setCurrentCompactedKVs(long currentCompactedKVs) {
+      this.currentCompactedKVs = currentCompactedKVs;
+    }
+
+    // Writable
+    public void readFields(DataInput in) throws IOException {
+      super.readFields(in);
+      int version = in.readByte();
+      if (version > VERSION) throw new IOException("Version mismatch; " + version);
+      int namelen = in.readInt();
+      this.name = new byte[namelen];
+      in.readFully(this.name);
+      this.stores = in.readInt();
+      this.storefiles = in.readInt();
+      this.storeUncompressedSizeMB = in.readInt();
+      this.storefileSizeMB = in.readInt();
+      this.memstoreSizeMB = in.readInt();
+      this.storefileIndexSizeMB = in.readInt();
+      this.readRequestsCount = in.readInt();
+      this.writeRequestsCount = in.readInt();
+      this.rootIndexSizeKB = in.readInt();
+      this.totalStaticIndexSizeKB = in.readInt();
+      this.totalStaticBloomSizeKB = in.readInt();
+      this.totalCompactingKVs = in.readLong();
+      this.currentCompactedKVs = in.readLong();
+      int coprocessorsSize = in.readInt();
+      coprocessors = new TreeSet<String>();
+      for (int i = 0; i < coprocessorsSize; i++) {
+        coprocessors.add(in.readUTF());
+      }
+    }
+
+    public void write(DataOutput out) throws IOException {
+      super.write(out);
+      out.writeByte(VERSION);
+      out.writeInt(name.length);
+      out.write(name);
+      out.writeInt(stores);
+      out.writeInt(storefiles);
+      out.writeInt(storeUncompressedSizeMB);
+      out.writeInt(storefileSizeMB);
+      out.writeInt(memstoreSizeMB);
+      out.writeInt(storefileIndexSizeMB);
+      out.writeInt(readRequestsCount);
+      out.writeInt(writeRequestsCount);
+      out.writeInt(rootIndexSizeKB);
+      out.writeInt(totalStaticIndexSizeKB);
+      out.writeInt(totalStaticBloomSizeKB);
+      out.writeLong(totalCompactingKVs);
+      out.writeLong(currentCompactedKVs);
+      out.writeInt(coprocessors.size());
+      for (String coprocessor: coprocessors) {
+        out.writeUTF(coprocessor);
+      }
+    }
+
+    /**
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+      StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "numberOfStores",
+        Integer.valueOf(this.stores));
+      sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
+        Integer.valueOf(this.storefiles));
+      sb = Strings.appendKeyValue(sb, "storefileUncompressedSizeMB",
+        Integer.valueOf(this.storeUncompressedSizeMB));
+      sb = Strings.appendKeyValue(sb, "storefileSizeMB",
+          Integer.valueOf(this.storefileSizeMB));
+      if (this.storeUncompressedSizeMB != 0) {
+        sb = Strings.appendKeyValue(sb, "compressionRatio",
+            String.format("%.4f", (float)this.storefileSizeMB/
+                (float)this.storeUncompressedSizeMB));
+      }
+      sb = Strings.appendKeyValue(sb, "memstoreSizeMB",
+        Integer.valueOf(this.memstoreSizeMB));
+      sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
+        Integer.valueOf(this.storefileIndexSizeMB));
+      sb = Strings.appendKeyValue(sb, "readRequestsCount",
+          Long.valueOf(this.readRequestsCount));
+      sb = Strings.appendKeyValue(sb, "writeRequestsCount",
+          Long.valueOf(this.writeRequestsCount));
+      sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
+          Integer.valueOf(this.rootIndexSizeKB));
+      sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
+          Integer.valueOf(this.totalStaticIndexSizeKB));
+      sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
+        Integer.valueOf(this.totalStaticBloomSizeKB));
+      sb = Strings.appendKeyValue(sb, "totalCompactingKVs",
+          Long.valueOf(this.totalCompactingKVs));
+      sb = Strings.appendKeyValue(sb, "currentCompactedKVs",
+          Long.valueOf(this.currentCompactedKVs));
+      float compactionProgressPct = Float.NaN;
+      if( this.totalCompactingKVs > 0 ) {
+        compactionProgressPct = Float.valueOf(
+            this.currentCompactedKVs / this.totalCompactingKVs);
+      }
+      sb = Strings.appendKeyValue(sb, "compactionProgressPct",
+          compactionProgressPct);
+      String coprocessors = Arrays.toString(getCoprocessors());
+      if (coprocessors != null) {
+        sb = Strings.appendKeyValue(sb, "coprocessors",
+            Arrays.toString(getCoprocessors()));
+      }
+      return sb.toString();
+    }
+  }
+
+  /*
+   * TODO: Other metrics that might be considered when the master is actually
+   * doing load balancing instead of merely trying to decide where to assign
+   * a region:
+   * <ul>
+   *   <li># of CPUs, heap size (to determine the "class" of machine). For
+   *       now, we consider them to be homogeneous.</li>
+   *   <li>#requests per region (Map<{String|HRegionInfo}, Integer>)</li>
+   *   <li>#compactions and/or #splits (churn)</li>
+   *   <li>server death rate (maybe there is something wrong with this server)</li>
+   * </ul>
+   */
+
+  /** default constructor (used by Writable) */
+  public HServerLoad092() {
+    super();
+  }
+
+  /**
+   * Constructor
+   * @param numberOfRequests
+   * @param usedHeapMB
+   * @param maxHeapMB
+   * @param coprocessors : coprocessors loaded at the regionserver-level
+   */
+  public HServerLoad092(final int totalNumberOfRequests,
+      final int numberOfRequests, final int usedHeapMB, final int maxHeapMB,
+      final Map<byte[], RegionLoad> regionLoad,
+      final Set<String> coprocessors) {
+    this.numberOfRequests = numberOfRequests;
+    this.usedHeapMB = usedHeapMB;
+    this.maxHeapMB = maxHeapMB;
+    this.regionLoad = regionLoad;
+    this.totalNumberOfRequests = totalNumberOfRequests;
+    this.coprocessors = coprocessors;
+  }
+
+  /**
+   * Constructor
+   * @param hsl the template HServerLoad
+   */
+  public HServerLoad092(final HServerLoad092 hsl) {
+    this(hsl.totalNumberOfRequests, hsl.numberOfRequests, hsl.usedHeapMB,
+        hsl.maxHeapMB, hsl.getRegionsLoad(), hsl.coprocessors);
+    for (Map.Entry<byte[], RegionLoad> e : hsl.regionLoad.entrySet()) {
+      this.regionLoad.put(e.getKey(), e.getValue());
+    }
+  }
+
+  /**
+   * Originally, this method factored in the effect of requests going to the
+   * server as well. However, this does not interact very well with the current
+   * region rebalancing code, which only factors number of regions. For the
+   * interim, until we can figure out how to make rebalancing use all the info
+   * available, we're just going to make load purely the number of regions.
+   *
+   * @return load factor for this server
+   */
+  public int getLoad() {
+    // int load = numberOfRequests == 0 ? 1 : numberOfRequests;
+    // load *= numberOfRegions == 0 ? 1 : numberOfRegions;
+    // return load;
+    return this.regionLoad.size();
+  }
+
+  /**
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return toString(1);
+  }
+
+  /**
+   * Returns toString() with the number of requests divided by the message
+   * interval in seconds
+   * @param msgInterval
+   * @return The load as a String
+   */
+  public String toString(int msgInterval) {
+    int numberOfRegions = this.regionLoad.size();
+    StringBuilder sb = new StringBuilder();
+    sb = Strings.appendKeyValue(sb, "requestsPerSecond",
+      Integer.valueOf(numberOfRequests/msgInterval));
+    sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions",
+      Integer.valueOf(numberOfRegions));
+    sb = Strings.appendKeyValue(sb, "usedHeapMB",
+      Integer.valueOf(this.usedHeapMB));
+    sb = Strings.appendKeyValue(sb, "maxHeapMB", Integer.valueOf(maxHeapMB));
+    return sb.toString();
+  }
+
+  /**
+   * @see java.lang.Object#equals(java.lang.Object)
+   */
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null) {
+      return false;
+    }
+    if (getClass() != o.getClass()) {
+      return false;
+    }
+    return compareTo((HServerLoad092)o) == 0;
+  }
+
+  // Getters
+
+  /**
+   * @return the numberOfRegions
+   */
+  public int getNumberOfRegions() {
+    return this.regionLoad.size();
+  }
+
+  /**
+   * @return the numberOfRequests per second.
+   */
+  public int getNumberOfRequests() {
+    return numberOfRequests;
+  }
+  
+  /**
+   * @return the numberOfRequests
+   */
+  public int getTotalNumberOfRequests() {
+    return totalNumberOfRequests;
+  }
+
+  /**
+   * @return the amount of heap in use, in MB
+   */
+  public int getUsedHeapMB() {
+    return usedHeapMB;
+  }
+
+  /**
+   * @return the maximum allowable heap size, in MB
+   */
+  public int getMaxHeapMB() {
+    return maxHeapMB;
+  }
+
+  /**
+   * @return region load metrics
+   */
+  public Map<byte[], RegionLoad> getRegionsLoad() {
+    return Collections.unmodifiableMap(regionLoad);
+  }
+
+  /**
+   * @return Count of storefiles on this regionserver
+   */
+  public int getStorefiles() {
+    int count = 0;
+    for (RegionLoad info: regionLoad.values())
+    	count += info.getStorefiles();
+    return count;
+  }
+
+  /**
+   * @return Total size of store files in MB
+   */
+  public int getStorefileSizeInMB() {
+    int count = 0;
+    for (RegionLoad info: regionLoad.values())
+      count += info.getStorefileSizeMB();
+    return count;
+  }
+
+  /**
+   * @return Size of memstores in MB
+   */
+  public int getMemStoreSizeInMB() {
+    int count = 0;
+    for (RegionLoad info: regionLoad.values())
+    	count += info.getMemStoreSizeMB();
+    return count;
+  }
+
+  /**
+   * @return Size of store file indexes in MB
+   */
+  public int getStorefileIndexSizeInMB() {
+    int count = 0;
+    for (RegionLoad info: regionLoad.values())
+    	count += info.getStorefileIndexSizeMB();
+    return count;
+  }
+
+  // Writable
+
+  public void readFields(DataInput in) throws IOException {
+    super.readFields(in);
+    int version = in.readByte();
+    if (version > VERSION) throw new IOException("Version mismatch; " + version);
+    numberOfRequests = in.readInt();
+    usedHeapMB = in.readInt();
+    maxHeapMB = in.readInt();
+    int numberOfRegions = in.readInt();
+    for (int i = 0; i < numberOfRegions; i++) {
+      RegionLoad rl = new RegionLoad();
+      rl.readFields(in);
+      regionLoad.put(rl.getName(), rl);
+    }
+    totalNumberOfRequests = in.readInt();
+    int coprocessorsSize = in.readInt();
+    for(int i = 0; i < coprocessorsSize; i++) {
+      coprocessors.add(in.readUTF());
+    }
+  }
+
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    out.writeByte(VERSION);
+    out.writeInt(numberOfRequests);
+    out.writeInt(usedHeapMB);
+    out.writeInt(maxHeapMB);
+    out.writeInt(this.regionLoad.size());
+    for (RegionLoad rl: regionLoad.values())
+      rl.write(out);
+    out.writeInt(totalNumberOfRequests);
+    out.writeInt(coprocessors.size());
+    for (String coprocessor: coprocessors) {
+      out.writeUTF(coprocessor);
+    }
+  }
+
+  // Comparable
+
+  public int compareTo(HServerLoad092 o) {
+    return this.getLoad() - o.getLoad();
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
index 21c92fff72e9..50cb9d472400 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
@@ -24,12 +24,16 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Set;
+import java.util.TreeMap;
 
+import org.apache.hadoop.hbase.HServerLoad092.RegionLoad;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
@@ -54,6 +58,17 @@
  */
 @Category(SmallTests.class)
 public class TestSerialization {
+  @Test
+  public void testHServerLoadVersioning() throws IOException {
+    Set<String> cps = new HashSet<String>(0);
+    Map<byte [], RegionLoad> regions = new TreeMap<byte [], RegionLoad>(Bytes.BYTES_COMPARATOR);
+    regions.put(HConstants.META_TABLE_NAME,
+      new HServerLoad092.RegionLoad(HConstants.META_TABLE_NAME, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, cps));
+    HServerLoad092 hsl092 = new HServerLoad092(0, 0, 0, 0, regions, cps);
+    byte [] hsl092bytes = Writables.getBytes(hsl092);
+    HServerLoad hsl = (HServerLoad)Writables.getWritable(hsl092bytes, new HServerLoad());
+    // TO BE CONTINUED
+  }
 
   @Test public void testCompareFilter() throws Exception {
     Filter f = new RowFilter(CompareOp.EQUAL,

From ee5f4686456e9ab2b82bb8f1cadf5874afce5eae Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 16 Apr 2012 21:45:34 +0000
Subject: [PATCH 0139/1540] HBASE-5780 Fix race in HBase regionserver startup
 vs ZK SASL authentication (Shaneal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1326810 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index c6e607ef1334..ea8799097510 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -71,6 +71,12 @@ public ZooKeeperNodeTracker(ZooKeeperWatcher watcher, String node,
    * or {@link #getData(boolean)} to get the data of the node if it is available.
    */
   public synchronized void start() {
+    try {
+      ZKUtil.waitForZKConnectionIfAuthenticating(watcher);
+    } catch (InterruptedException e) {
+      throw new IllegalStateException("ZookeeperNodeTracker on " + this.node 
+          + " interuppted while waiting for SASL Authentication", e);
+    }
     this.watcher.registerListener(this);
     try {
       if(ZKUtil.watchAndCheckExists(watcher, node)) {

From 92edf6b4fc1e0c90e32822a2c92674ab75b56deb Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 18 Apr 2012 19:32:06 +0000
Subject: [PATCH 0140/1540] HBASE-5823 Hbck should be able to print help

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327639 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 41d05d35c084..9a8d2bb05419 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2885,6 +2885,7 @@ public void setTimeLag(long seconds) {
   protected static void printUsageAndExit() {
     System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
+    System.err.println("   -help Display help options (this)");
     System.err.println("   -details Display full report of all regions.");
     System.err.println("   -timelag {timeInSeconds}  Process only regions that " +
                        " have not experienced any metadata updates in the last " +
@@ -2929,7 +2930,9 @@ public static void main(String[] args) throws Exception {
     // Process command-line args.
     for (int i = 0; i < args.length; i++) {
       String cmd = args[i];
-      if (cmd.equals("-details")) {
+      if (cmd.equals("-help") || cmd.equals("-h")) {
+        printUsageAndExit();
+      } else if (cmd.equals("-details")) {
         fsck.setDisplayFullReport();
       } else if (cmd.equals("-timelag")) {
         if (i == args.length - 1) {
@@ -3022,6 +3025,9 @@ public static void main(String[] args) throws Exception {
         fsck.setSummary();
       } else if (cmd.equals("-metaonly")) {
         fsck.setCheckMetaOnly();
+      } else if (cmd.startsWith("-")) {
+        System.err.println("Unrecognized option:" + cmd);
+        printUsageAndExit();
       } else {
         byte[] table = Bytes.toBytes(cmd);
         fsck.includeTable(table);

From b00112a028676018115661ab6ead141a485f6c30 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Apr 2012 20:31:18 +0000
Subject: [PATCH 0141/1540] HBASE-5792 HLog Performance Evaluation Tool

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327668 13f79535-47bb-0310-9956-ffa450edef68
---
 .../wal/HLogPerformanceEvaluation.java        | 355 ++++++++++++++++++
 1 file changed, 355 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
new file mode 100644
index 000000000000..0e78d4540071
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.util.Map;
+import java.util.List;
+import java.util.Random;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+/**
+ * This class runs performance benchmarks for {@link HLog}.
+ * See usage for this tool by running:
+ * <code>$ hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation -h</code>
+ */
+@InterfaceAudience.Private
+public final class HLogPerformanceEvaluation extends Configured implements Tool {
+  static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
+
+  private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  static final String TABLE_NAME = "HLogPerformanceEvaluation";
+  static final String QUALIFIER_PREFIX = "q";
+  static final String FAMILY_PREFIX = "cf";
+
+  private int numQualifiers = 1;
+  private int valueSize = 512;
+  private int keySize = 16;
+
+  /**
+   * Perform HLog.append() of Put object, for the number of iterations requested.
+   * Keys and Vaues are generated randomly, the number of column familes,
+   * qualifiers and key/value size is tunable by the user.
+   */
+  class HLogPutBenchmark implements Runnable {
+    private final long numIterations;
+    private final int numFamilies;
+    private final boolean noSync;
+    private final HRegion region;
+    private final HTableDescriptor htd;
+
+    HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
+        final long numIterations, final boolean noSync) {
+      this.numIterations = numIterations;
+      this.noSync = noSync;
+      this.numFamilies = htd.getColumnFamilies().length;
+      this.region = region;
+      this.htd = htd;
+    }
+
+    public void run() {
+      byte[] key = new byte[keySize];
+      byte[] value = new byte[valueSize];
+      Random rand = new Random(Thread.currentThread().getId());
+      HLog hlog = region.getLog();
+
+      try {
+        long startTime = System.currentTimeMillis();
+        for (int i = 0; i < numIterations; ++i) {
+          Put put = setupPut(rand, key, value, numFamilies);
+          long now = System.currentTimeMillis();
+          WALEdit walEdit = new WALEdit();
+          addFamilyMapToWALEdit(put.getFamilyMap(), walEdit);
+          HRegionInfo hri = region.getRegionInfo();
+          if (this.noSync) {
+            hlog.appendNoSync(hri, hri.getTableName(), walEdit,
+                              HConstants.DEFAULT_CLUSTER_ID, now, htd);
+          } else {
+            hlog.append(hri, hri.getTableName(), walEdit, now, htd);
+          }
+        }
+        long totalTime = (System.currentTimeMillis() - startTime);
+        logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
+      } catch (Exception e) {
+        LOG.error(getClass().getSimpleName() + " Thread failed", e);
+      }
+    }
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    Path rootRegionDir = null;
+    int numThreads = 1;
+    long numIterations = 10000;
+    int numFamilies = 1;
+    boolean noSync = false;
+    boolean verify = false;
+    boolean verbose = false;
+    long roll = Long.MAX_VALUE;
+    // Process command line args
+    for (int i = 0; i < args.length; i++) {
+      String cmd = args[i];
+      try {
+        if (cmd.equals("-threads")) {
+          numThreads = Integer.parseInt(args[++i]);
+        } else if (cmd.equals("-iterations")) {
+          numIterations = Long.parseLong(args[++i]);
+        } else if (cmd.equals("-path")) {
+          rootRegionDir = new Path(args[++i]);
+        } else if (cmd.equals("-families")) {
+          numFamilies = Integer.parseInt(args[++i]);
+        } else if (cmd.equals("-qualifiers")) {
+          numQualifiers = Integer.parseInt(args[++i]);
+        } else if (cmd.equals("-keySize")) {
+          keySize = Integer.parseInt(args[++i]);
+        } else if (cmd.equals("-valueSize")) {
+          valueSize = Integer.parseInt(args[++i]);
+        } else if (cmd.equals("-nosync")) {
+          noSync = true;
+        } else if (cmd.equals("-verify")) {
+          verify = true;
+        } else if (cmd.equals("-verbose")) {
+          verbose = true;
+        } else if (cmd.equals("-roll")) {
+          roll = Long.parseLong(args[++i]);
+        } else if (cmd.equals("-h")) {
+          printUsageAndExit();
+        } else if (cmd.equals("--help")) {
+          printUsageAndExit();
+        } else {
+          System.err.println("UNEXPECTED: " + cmd);
+          printUsageAndExit();
+        }
+      } catch (Exception e) {
+        printUsageAndExit();
+      }
+    }
+
+    // Run HLog Performance Evaluation
+    FileSystem fs = FileSystem.get(getConf());
+    LOG.info("" + fs);
+    try {
+      if (rootRegionDir == null) {
+        rootRegionDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation");
+      }
+      rootRegionDir = rootRegionDir.makeQualified(fs);
+      cleanRegionRootDir(fs, rootRegionDir);
+      // Initialize Table Descriptor
+      HTableDescriptor htd = createHTableDescriptor(numFamilies);
+      final long whenToRoll = roll;
+      HLog hlog = new HLog(fs, new Path(rootRegionDir, "wals"),
+          new Path(rootRegionDir, "old.wals"), getConf()) {
+        int appends = 0;
+        protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
+            HTableDescriptor htd)
+        throws IOException {
+          this.appends++;
+          if (this.appends % whenToRoll == 0) {
+            LOG.info("Rolling after " + appends + " edits");
+            rollWriter();
+          }
+          super.doWrite(info, logKey, logEdit, htd);
+        };
+      };
+      hlog.rollWriter();
+      HRegion region = null;
+      try {
+        region = openRegion(fs, rootRegionDir, htd, hlog);
+        long putTime = runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync), numThreads);
+        logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations,
+          numIterations * numThreads, putTime);
+        if (region != null) {
+          closeRegion(region);
+          region = null;
+        }
+        if (verify) {
+          Path dir = hlog.getDir();
+          long editCount = 0;
+          for (FileStatus fss: fs.listStatus(dir)) {
+            editCount += verify(fss.getPath(), verbose);
+          }
+          long expected = numIterations * numThreads;
+          if (editCount != expected) {
+            throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
+          }
+        }
+      } finally {
+        if (region != null) closeRegion(region);
+        // Remove the root dir for this test region
+        cleanRegionRootDir(fs, rootRegionDir);
+      }
+    } finally {
+      fs.close();
+    }
+
+    return(0);
+  }
+
+  private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
+    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
+    for (int i = 0; i < numFamilies; ++i) {
+      HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
+      htd.addFamily(colDef);
+    }
+    return htd;
+  }
+
+  /**
+   * Verify the content of the WAL file.
+   * Verify that sequenceids are ascending and that the file has expected number
+   * of edits.
+   * @param wal
+   * @return Count of edits.
+   * @throws IOException
+   */
+  private long verify(final Path wal, final boolean verbose) throws IOException {
+    HLog.Reader reader = HLog.getReader(wal.getFileSystem(getConf()), wal, getConf());
+    long previousSeqid = -1;
+    long count = 0;
+    try {
+      while (true) {
+        Entry e = reader.next();
+        if (e == null) break;
+        count++;
+        long seqid = e.getKey().getLogSeqNum();
+        if (verbose) LOG.info("seqid=" + seqid);
+        if (previousSeqid >= seqid) {
+          throw new IllegalStateException("wal=" + wal.getName() +
+            ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
+        }
+        previousSeqid = seqid;
+      }
+    } finally {
+      reader.close();
+    }
+    return count;
+  }
+
+  private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
+    float tsec = totalTime / 1000.0f;
+    LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
+  }
+
+  private void printUsageAndExit() {
+    System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
+    System.err.println(" where [options] are:");
+    System.err.println("  -h|-help         Show this help and exit.");
+    System.err.println("  -threads <N>     Number of threads writing on the WAL.");
+    System.err.println("  -iterations <N>  Number of iterations per thread.");
+    System.err.println("  -path <PATH>     Path where region's root directory is created.");
+    System.err.println("  -families <N>    Number of column families to write.");
+    System.err.println("  -qualifiers <N>  Number of qualifiers to write.");
+    System.err.println("  -keySize <N>     Row key size in byte.");
+    System.err.println("  -valueSize <N>   Row/Col value size in byte.");
+    System.err.println("  -nosync          Append without syncing");
+    System.err.println("  -verify          Verify edits written in sequence");
+    System.err.println("  -verbose         Output extra info; e.g. all edit seq ids when verifying");
+    System.err.println("  -roll <N>        Roll the way every N appends");
+    System.err.println("");
+    System.err.println("Examples:");
+    System.err.println("");
+    System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
+    System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
+    System.err.println("    -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
+    System.exit(1);
+  }
+
+  private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
+  throws IOException {
+    // Initialize HRegion
+    HRegionInfo regionInfo = new HRegionInfo(htd.getName());
+    return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
+  }
+
+  private void closeRegion(final HRegion region) throws IOException {
+    if (region != null) {
+      region.close();
+      HLog wal = region.getLog();
+      if (wal != null) wal.close();
+    }
+  }
+
+  private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
+    if (fs.exists(dir)) {
+      fs.delete(dir, true);
+    }
+  }
+
+  private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
+    rand.nextBytes(key);
+    Put put = new Put(key);
+    for (int cf = 0; cf < numFamilies; ++cf) {
+      for (int q = 0; q < numQualifiers; ++q) {
+        rand.nextBytes(value);
+        put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
+      }
+    }
+    return put;
+  }
+
+  private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap, WALEdit walEdit) {
+    for (List<KeyValue> edits : familyMap.values()) {
+      for (KeyValue kv : edits) {
+        walEdit.add(kv);
+      }
+    }
+  }
+
+  private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
+    Thread[] threads = new Thread[numThreads];
+    long startTime = System.currentTimeMillis();
+    for (int i = 0; i < numThreads; ++i) {
+      threads[i] = new Thread(runnable);
+      threads[i].start();
+    }
+    for (Thread t : threads) t.join();
+    long endTime = System.currentTimeMillis();
+    return(endTime - startTime);
+  }
+
+  public static void main(String[] args) throws Exception {
+    int exitCode = ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
+    System.exit(exitCode);
+  }
+}

From 5112babda37b1a48610cf57c170b744f261a5519 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Apr 2012 20:37:25 +0000
Subject: [PATCH 0142/1540] HBASE-5782 Edits can be appended out of seqid order
 since HBASE-4487

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327672 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLog.java   | 49 +++++++++++--------
 .../hbase/regionserver/wal/TestHLog.java      | 14 +++++-
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 7fabbe8c9331..0abf313ca97e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -228,6 +228,7 @@ public interface Writer {
   // during an update
   // locked during appends
   private final Object updateLock = new Object();
+  private final Object flushLock = new Object();
 
   private final boolean enabled;
 
@@ -295,7 +296,6 @@ synchronized Metric get() {
   private static Metric writeSize = new Metric();
   // For measuring latency of syncs
   private static Metric syncTime = new Metric();
-  private static AtomicLong syncBatchSize = new AtomicLong();
   //For measuring slow HLog appends
   private static AtomicLong slowHLogAppendCount = new AtomicLong();
   private static Metric slowHLogAppendTime = new Metric();
@@ -312,10 +312,6 @@ public static Metric getSyncTime() {
     return syncTime.get();
   }
 
-  public static long getSyncBatchSize() {
-    return syncBatchSize.getAndSet(0);
-  }
-  
   public static long getSlowAppendCount() {
     return slowHLogAppendCount.get();
   }
@@ -1257,32 +1253,43 @@ private void syncer(long txid) throws IOException {
       return;
     }
     try {
-      long doneUpto = this.unflushedEntries.get();
+      long doneUpto;
       long now = System.currentTimeMillis();
-      // Done in parallel for all writer threads, thanks to HDFS-895
-      List<Entry> pending = logSyncerThread.getPendingWrites();
+      // First flush all the pending writes to HDFS. Then 
+      // issue the sync to HDFS. If sync is successful, then update
+      // syncedTillHere to indicate that transactions till this
+      // number has been successfully synced.
+      synchronized (flushLock) {
+        if (txid <= this.syncedTillHere) {
+          return;
+        }
+        doneUpto = this.unflushedEntries.get();
+        List<Entry> pending = logSyncerThread.getPendingWrites();
+        try {
+          logSyncerThread.hlogFlush(tempWriter, pending);
+        } catch(IOException io) {
+          synchronized (this.updateLock) {
+            // HBASE-4387, HBASE-5623, retry with updateLock held
+            tempWriter = this.writer;
+            logSyncerThread.hlogFlush(tempWriter, pending);
+          }
+        }
+      }
+      // another thread might have sync'ed avoid double-sync'ing
+      if (txid <= this.syncedTillHere) {
+        return;
+      }
       try {
-        // First flush all the pending writes to HDFS. Then 
-        // issue the sync to HDFS. If sync is successful, then update
-        // syncedTillHere to indicate that transactions till this
-        // number has been successfully synced.
-        logSyncerThread.hlogFlush(tempWriter, pending);
-        pending = null;
         tempWriter.sync();
-        syncBatchSize.addAndGet(doneUpto - this.syncedTillHere);
-        this.syncedTillHere = Math.max(this.syncedTillHere, doneUpto);
       } catch(IOException io) {
         synchronized (this.updateLock) {
           // HBASE-4387, HBASE-5623, retry with updateLock held
           tempWriter = this.writer;
-          logSyncerThread.hlogFlush(tempWriter, pending);
           tempWriter.sync();
-          syncBatchSize.addAndGet(doneUpto - this.syncedTillHere);
-          this.syncedTillHere = doneUpto;
         }
       }
-      // We try to not acquire the updateLock just to update statistics.
-      // Make these statistics as AtomicLong.
+      this.syncedTillHere = Math.max(this.syncedTillHere, doneUpto);
+
       syncTime.inc(System.currentTimeMillis() - now);
       if (!this.logRollRunning) {
         checkLowReplication();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 81592f85fe53..fa5dbf220808 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -51,7 +51,6 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.log4j.Level;
@@ -136,6 +135,19 @@ private static String getName() {
     return "TestHLog";
   }
 
+  /**
+   * Test that with three concurrent threads we still write edits in sequence
+   * edit id order.
+   * @throws Exception
+   */
+  @Test
+  public void testMaintainOrderWithConcurrentWrites() throws Exception {
+    // Run the HPE tool with three threads writing 3000 edits each concurrently.
+    // When done, verify that all edits were written and that the order in the
+    // WALs is of ascending edit sequence ids.
+    HLogPerformanceEvaluation.main(new String [] {"-threads", "3", "-verify", "-iterations", "3000"});
+  }
+
   /**
    * Just write multiple logs then split.  Before fix for HADOOP-2283, this
    * would fail.

From 867272f6efbbf3ed87c8a00e2475c309e02a55b7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Apr 2012 20:53:38 +0000
Subject: [PATCH 0143/1540] HBASE-5545 region can't be opened for a long time.
 Because the creating File failed. (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327676 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  9 ++++++
 .../org/apache/hadoop/hbase/util/FSUtils.java | 26 +++++++++++++++++
 .../apache/hadoop/hbase/util/TestFSUtils.java | 29 +++++++++++++++++++
 3 files changed, 64 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 573606dfbed2..b8565ca422c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -766,6 +766,15 @@ private void checkRegioninfoOnFilesystem() throws IOException {
 
     // and then create the file
     Path tmpPath = new Path(getTmpDir(), REGIONINFO_FILE);
+    
+    // if datanode crashes or if the RS goes down just before the close is called while trying to
+    // close the created regioninfo file in the .tmp directory then on next
+    // creation we will be getting AlreadyCreatedException.
+    // Hence delete and create the file if exists.
+    if (FSUtils.isExists(fs, tmpPath)) {
+      FSUtils.delete(fs, tmpPath, true);
+    }
+    
     FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
 
     try {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index db9ed9326ca7..c7ca432386ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1012,4 +1012,30 @@ public static Map<String, Path> getTableStoreFilePathMap(
     if (status == null || status.length < 1) return null;
     return status;
   }
+  
+  /**
+   * Calls fs.delete() and returns the value returned by the fs.delete()
+   * 
+   * @param fs
+   * @param path
+   * @param recursive
+   * @return
+   * @throws IOException
+   */
+  public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
+      throws IOException {
+    return fs.delete(path, recursive);
+  }
+
+  /**
+   * Calls fs.exists(). Checks if the specified path exists
+   * 
+   * @param fs
+   * @param path
+   * @return
+   * @throws IOException
+   */
+  public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
+    return fs.exists(path);
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
index 96693906dc91..339a1207ee9c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -185,6 +185,35 @@ public void testPermMask() throws Exception {
       fs.delete(p, true);
     }
   }
+  
+  @Test
+  public void testDeleteAndExists() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true);
+    FileSystem fs = FileSystem.get(conf);
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
+    // then that the correct file is created
+    String file = UUID.randomUUID().toString();
+    Path p = new Path("temptarget" + File.separator + file);
+    Path p1 = new Path("temppath" + File.separator + file);
+    try {
+      FSDataOutputStream out = FSUtils.create(fs, p, perms);
+      out.close();
+      assertTrue("The created file should be present", FSUtils.isExists(fs, p));
+      // delete the file with recursion as false. Only the file will be deleted.
+      FSUtils.delete(fs, p, false);
+      // Create another file
+      FSDataOutputStream out1 = FSUtils.create(fs, p1, perms);
+      out1.close();
+      // delete the file with recursion as false. Still the file only will be deleted
+      FSUtils.delete(fs, p1, true);
+      assertFalse("The created file should be present", FSUtils.isExists(fs, p1));
+      // and then cleanup
+    } finally {
+      FSUtils.delete(fs, p, true);
+      FSUtils.delete(fs, p1, true);
+    }
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 2bc438c0309a747d37f86697c3bda18e47304b78 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Apr 2012 20:57:25 +0000
Subject: [PATCH 0144/1540] CHANGES.txt for 0.94.0rc2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327678 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 22bb92193d5f..ca13be91e469 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.0 - 4/13/2012
+Release 0.94.0 - 4/18/2012
 Sub-task
 
     [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently
@@ -35,6 +35,7 @@ Sub-task
     [HBASE-5541] - Avoid holding the rowlock during HLog sync in HRegion.mutateRowWithLocks
     [HBASE-5638] - Backport to 0.90 and 0.92 - NPE reading ZK config in HBase
     [HBASE-5641] - decayingSampleTick1 prevents HBase from shutting down.
+    [HBASE-5793] - TestHBaseFsck#TestNoHdfsTable test hangs after client retries increased
 
 Bug
 
@@ -166,6 +167,7 @@ Bug
     [HBASE-5531] - Maven hadoop profile (version 23) needs to be updated with latest 23 snapshot
     [HBASE-5535] - Make the functions in task monitor synchronized
     [HBASE-5537] - MXBean shouldn't have a dependence on InterfaceStability until 0.96
+    [HBASE-5545] - region can't be opened for a long time. Because the creating File failed.
     [HBASE-5552] - Clean up our jmx view; its a bit of a mess
     [HBASE-5562] - test-patch.sh reports a javadoc warning when there are no new javadoc warnings
     [HBASE-5563] - HRegionInfo#compareTo should compare regionId as well
@@ -202,6 +204,10 @@ Bug
     [HBASE-5736] - ThriftServerRunner.HbaseHandler.mutateRow() does not use ByteBuffer correctly
     [HBASE-5743] - Support GIT patches
     [HBASE-5773] - HtablePool constructor not reading config files in certain cases
+    [HBASE-5780] - Fix race in HBase regionserver startup vs ZK SASL authentication
+    [HBASE-5781] - Zookeeper session got closed while trying to assign the region to RS using hbck -fix
+    [HBASE-5782] - Edits can be appended out of seqid order since HBASE-4487
+    [HBASE-5795] - HServerLoad$RegionLoad breaks 0.92<->0.94 compatibility
 
 Improvement
 
@@ -344,6 +350,7 @@ Improvement
     [HBASE-5748] - Enable lib directory in jar file for coprocessor
     [HBASE-5770] - Add a clock skew warning threshold
     [HBASE-5775] - ZKUtil doesn't handle deleteRecurisively cleanly
+    [HBASE-5823] - Hbck should be able to print help
 
 New Feature
 
@@ -380,6 +387,7 @@ Task
     [HBASE-5084] - Allow different HTable instances to share one ExecutorService
     [HBASE-5111] - Upgrade zookeeper to 3.4.2 release
     [HBASE-5173] - Commit hbase-4480 findHangingTest.sh script under dev-support
+    [HBASE-5256] - Use WritableUtils.readVInt() in RegionLoad.readFields()
     [HBASE-5264] - Add 0.92.0 upgrade guide
     [HBASE-5294] - Make sure javadoc is included in tarball bundle when we release
     [HBASE-5400] - Some tests does not have annotations for (Small|Medium|Large)Tests
@@ -401,6 +409,7 @@ Test
     [HBASE-5150] - Failure in a thread may not fail a test, clean up log splitting test
     [HBASE-5223] - TestMetaReaderEditor is missing call to CatalogTracker.stop()
     [HBASE-5455] - Add test to avoid unintentional reordering of items in HbaseObjectWritable
+    [HBASE-5792] - HLog Performance Evaluation Tool
 
 
 Release 0.92.1 - Unreleased

From 183b567f6f18e3fdff827869f4cf0ebaf5f712a3 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 18 Apr 2012 21:22:06 +0000
Subject: [PATCH 0145/1540] HBASE-5825 TestHLog not running any tests; fix

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327688 13f79535-47bb-0310-9956-ffa450edef68
---
 .../wal/HLogPerformanceEvaluation.java           | 16 +++++++++++++---
 .../hadoop/hbase/regionserver/wal/TestHLog.java  |  4 +++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
index 0e78d4540071..0258cdaef267 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
@@ -348,8 +348,18 @@ private long runBenchmark(Runnable runnable, final int numThreads) throws Interr
     return(endTime - startTime);
   }
 
+  /**
+   * The guts of the {@link #main} method.
+   * Call this method to avoid the {@link #main(String[])} System.exit.
+   * @param args
+   * @return errCode
+   * @throws Exception 
+   */
+  static int innerMain(final String [] args) throws Exception {
+    return ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
+  }
+
   public static void main(String[] args) throws Exception {
-    int exitCode = ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
-    System.exit(exitCode);
+     System.exit(innerMain(args));
   }
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index fa5dbf220808..9fa4480af770 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -145,7 +145,9 @@ public void testMaintainOrderWithConcurrentWrites() throws Exception {
     // Run the HPE tool with three threads writing 3000 edits each concurrently.
     // When done, verify that all edits were written and that the order in the
     // WALs is of ascending edit sequence ids.
-    HLogPerformanceEvaluation.main(new String [] {"-threads", "3", "-verify", "-iterations", "3000"});
+    int errCode =
+      HLogPerformanceEvaluation.innerMain(new String [] {"-threads", "3", "-verify", "-iterations", "3000"});
+    assertEquals(0, errCode);
   }
 
   /**

From a5de517b96aaa9f7befb25b041151f777493523d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 19 Apr 2012 00:43:10 +0000
Subject: [PATCH 0146/1540] HBASE-5787 Table owner can't disable/delete its own
 table (Matteo)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1327757 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 32 ++++++++++++++++---
 .../security/access/TestAccessController.java | 24 +++++++-------
 2 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index c1f20de7b315..e47437bc6df3 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -505,7 +505,11 @@ public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    requirePermission(Permission.Action.CREATE);
+    if (isActiveUserTableOwner(c.getEnvironment(), tableName)) {
+      requirePermission(Permission.Action.CREATE);
+    } else {
+      requirePermission(Permission.Action.ADMIN);
+    }
   }
   @Override
   public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -555,8 +559,11 @@ public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    /* TODO: Allow for users with global CREATE permission and the table owner */
-    requirePermission(Permission.Action.ADMIN);
+    if (isActiveUserTableOwner(c.getEnvironment(), tableName)) {
+      requirePermission(Permission.Action.CREATE);
+    } else {
+      requirePermission(Permission.Action.ADMIN);
+    }
   }
   @Override
   public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -565,8 +572,11 @@ public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    /* TODO: Allow for users with global CREATE permission and the table owner */
-    requirePermission(Permission.Action.ADMIN);
+    if (isActiveUserTableOwner(c.getEnvironment(), tableName)) {
+      requirePermission(Permission.Action.CREATE);
+    } else {
+      requirePermission(Permission.Action.ADMIN);
+    }
   }
   @Override
   public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -1027,4 +1037,16 @@ private byte[] getTableName(RegionCoprocessorEnvironment e) {
     }
     return tableName;
   }
+
+  private String getTableOwner(MasterCoprocessorEnvironment e, 
+      byte[] tableName) throws IOException {
+    HTableDescriptor htd = e.getTable(tableName).getTableDescriptor();
+    return htd.getOwnerString();
+  }
+
+  private boolean isActiveUserTableOwner(MasterCoprocessorEnvironment e,
+      byte[] tableName) throws IOException {
+    String activeUser = getActiveUser().getShortName();
+    return activeUser.equals(getTableOwner(e, tableName));
+  }
 }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index fe04c5a69ba6..ec678b9254fe 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -205,7 +205,7 @@ public Object run() throws Exception {
 
   @Test
   public void testTableModify() throws Exception {
-    PrivilegedExceptionAction disableTable = new PrivilegedExceptionAction() {
+    PrivilegedExceptionAction modifyTable = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
         htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
@@ -216,18 +216,18 @@ public Object run() throws Exception {
     };
 
     // all others should be denied
-    verifyDenied(USER_OWNER, disableTable);
-    verifyDenied(USER_RW, disableTable);
-    verifyDenied(USER_RO, disableTable);
-    verifyDenied(USER_NONE, disableTable);
+    verifyDenied(USER_OWNER, modifyTable);
+    verifyDenied(USER_RW, modifyTable);
+    verifyDenied(USER_RO, modifyTable);
+    verifyDenied(USER_NONE, modifyTable);
 
     // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, disableTable);
+    verifyAllowed(SUPERUSER, modifyTable);
   }
 
   @Test
   public void testTableDelete() throws Exception {
-    PrivilegedExceptionAction disableTable = new PrivilegedExceptionAction() {
+    PrivilegedExceptionAction deleteTable = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         ACCESS_CONTROLLER.preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE);
         return null;
@@ -235,13 +235,13 @@ public Object run() throws Exception {
     };
 
     // all others should be denied
-    verifyDenied(USER_OWNER, disableTable);
-    verifyDenied(USER_RW, disableTable);
-    verifyDenied(USER_RO, disableTable);
-    verifyDenied(USER_NONE, disableTable);
+    verifyDenied(USER_OWNER, deleteTable);
+    verifyDenied(USER_RW, deleteTable);
+    verifyDenied(USER_RO, deleteTable);
+    verifyDenied(USER_NONE, deleteTable);
 
     // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, disableTable);
+    verifyAllowed(SUPERUSER, deleteTable);
   }
 
   @Test

From f6ebb99b0939ef72a1d4f2d649590a1c0893cb0f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 19 Apr 2012 16:19:10 +0000
Subject: [PATCH 0147/1540] HBASE-5821  Incorrect handling of null value in
 Coprocessor aggregation function min() (Maryann Xue)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1328027 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/coprocessor/AggregationClient.java    | 4 ++--
 .../hadoop/hbase/coprocessor/AggregateImplementation.java     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
index 05a8d9b826e0..3a019d38981b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
@@ -101,7 +101,7 @@ R getMax() {
 
       @Override
       public synchronized void update(byte[] region, byte[] row, R result) {
-        max = ci.compare(max, result) < 0 ? result : max;
+        max = (max == null || (result != null && ci.compare(max, result) < 0)) ? result : max;
       }
     }
     MaxCallBack aMaxCallBack = new MaxCallBack();
@@ -151,7 +151,7 @@ public R getMinimum() {
 
       @Override
       public synchronized void update(byte[] region, byte[] row, R result) {
-        min = (min == null || ci.compare(result, min) < 0) ? result : min;
+        min = (min == null || (result != null && ci.compare(result, min) < 0)) ? result : min;
       }
     }
     HTable table = new HTable(conf, tableName);
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
index 579aa5505e11..f6f4b42a3441 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
@@ -68,7 +68,7 @@ public <T, S> T getMax(ColumnInterpreter<T, S> ci, Scan scan)
         hasMoreRows = scanner.next(results);
         for (KeyValue kv : results) {
           temp = ci.getValue(colFamily, qualifier, kv);
-          max = (max == null || ci.compare(temp, max) > 0) ? temp : max;
+          max = (max == null || (temp != null && ci.compare(temp, max) > 0)) ? temp : max;
         }
         results.clear();
       } while (hasMoreRows);
@@ -97,7 +97,7 @@ public <T, S> T getMin(ColumnInterpreter<T, S> ci, Scan scan)
         hasMoreRows = scanner.next(results);
         for (KeyValue kv : results) {
           temp = ci.getValue(colFamily, qualifier, kv);
-          min = (min == null || ci.compare(temp, min) < 0) ? temp : min;
+          min = (min == null || (temp != null && ci.compare(temp, min) < 0)) ? temp : min;
         }
         results.clear();
       } while (hasMoreRows);

From 26fb01429f1b4c161859dcdcb8c89c108ac6c2bd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 19 Apr 2012 16:22:24 +0000
Subject: [PATCH 0148/1540] HBASE-5741 ImportTsv does not check for table
 existence (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1328032 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/ImportTsv.java     | 40 ++++++++++++++++++-
 .../hadoop/hbase/mapreduce/TestImportTsv.java | 27 ++++++++++---
 2 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index f30c5082e6c6..7924cb958411 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -23,11 +23,16 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -61,6 +66,7 @@ public class ImportTsv {
   final static String TIMESTAMP_CONF_KEY = "importtsv.timestamp";
   final static String DEFAULT_SEPARATOR = "\t";
   final static Class DEFAULT_MAPPER = TsvImporterMapper.class;
+  private static HBaseAdmin hbaseAdmin;
 
   static class TsvParser {
     /**
@@ -217,6 +223,9 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
 
     String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
     if (hfileOutPath != null) {
+      if (!doesTableExist(tableName)) {
+        createTable(conf, tableName);
+      }
       HTable table = new HTable(conf, tableName);
       job.setReducerClass(PutSortReducer.class);
       Path outputDir = new Path(hfileOutPath);
@@ -237,6 +246,27 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
     return job;
   }
 
+  private static boolean doesTableExist(String tableName) throws IOException {
+    return hbaseAdmin.tableExists(tableName.getBytes());
+  }
+
+  private static void createTable(Configuration conf, String tableName)
+      throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName.getBytes());
+    String columns[] = conf.getStrings(COLUMNS_CONF_KEY);
+    Set<String> cfSet = new HashSet<String>();
+    for (String aColumn : columns) {
+      if (TsvParser.ROWKEY_COLUMN_SPEC.equals(aColumn)) continue;
+      // we are only concerned with the first one (in case this is a cf:cq)
+      cfSet.add(aColumn.split(":", 2)[0]);
+    }
+    for (String cf : cfSet) {
+      HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf));
+      htd.addFamily(hcd);
+    }
+    hbaseAdmin.createTable(htd);
+  }
+
   /*
    * @param errorMsg Error message.  Can be null.
    */
@@ -274,6 +304,14 @@ private static void usage(final String errorMsg) {
     System.err.println(usage);
   }
 
+  /**
+   * Used only by test method
+   * @param conf
+   */
+  static void createHbaseAdmin(Configuration conf) throws IOException {
+    hbaseAdmin = new HBaseAdmin(conf);
+  }
+
   /**
    * Main entry point.
    *
@@ -311,7 +349,7 @@ public static void main(String[] args) throws Exception {
       usage("One or more columns in addition to the row key are required");
       System.exit(-1);
     }
-
+    hbaseAdmin = new HBaseAdmin(conf);
     Job job = createSubmittableJob(conf, otherArgs);
     System.exit(job.waitForCompletion(true) ? 0 : 1);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index ac30a62677a1..ff1ede3ec3d5 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -204,11 +204,13 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
       final byte[] TAB = Bytes.toBytes(tableName);
       final byte[] QA = Bytes.toBytes("A");
       final byte[] QB = Bytes.toBytes("B");
-
-      HTableDescriptor desc = new HTableDescriptor(TAB);
-      desc.addFamily(new HColumnDescriptor(FAM));
-      new HBaseAdmin(conf).createTable(desc);
-
+      if (conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY) == null) {
+        HTableDescriptor desc = new HTableDescriptor(TAB);
+        desc.addFamily(new HColumnDescriptor(FAM));
+        new HBaseAdmin(conf).createTable(desc);
+      } else { // set the hbaseAdmin as we are not going through main()
+        ImportTsv.createHbaseAdmin(conf);
+      }
       Job job = ImportTsv.createSubmittableJob(conf, args);
       job.waitForCompletion(false);
       assertTrue(job.isSuccessful());
@@ -255,6 +257,21 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
     }
   }
   
+  @Test
+  public void testBulkOutputWithoutAnExistingTable() throws Exception {
+    String TABLE_NAME = "TestTable";
+    String FAMILY = "FAM";
+    String INPUT_FILE = "InputFile2.esv";
+
+    // Prepare the arguments required for the test.
+    String[] args = new String[] {
+        "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B",
+        "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b",
+        "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=output", TABLE_NAME,
+        INPUT_FILE };
+    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, args, 3);
+  }
+
   public static String toU8Str(byte[] bytes) throws UnsupportedEncodingException {
     return new String(bytes, HConstants.UTF8_ENCODING);
   }

From efc32ddc11bce0c58435bd7b2841026b055b9e81 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 19 Apr 2012 18:14:43 +0000
Subject: [PATCH 0149/1540] HBASE-5737 Minor Improvements related to balancer.
 (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1328063 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 11 ++++----
 .../apache/hadoop/hbase/master/HMaster.java   |  4 +--
 .../hbase/master/TestAssignmentManager.java   | 26 ++++++++++++-------
 3 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index a2eb6d6ebf8a..e33e81eabdcc 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -193,8 +193,8 @@ public class AssignmentManager extends ZooKeeperListener {
    * @throws IOException 
    */
   public AssignmentManager(Server master, ServerManager serverManager,
-      CatalogTracker catalogTracker, final ExecutorService service)
-  throws KeeperException, IOException {
+      CatalogTracker catalogTracker, final LoadBalancer balancer,
+      final ExecutorService service) throws KeeperException, IOException {
     super(master.getZooKeeper());
     this.master = master;
     this.serverManager = serverManager;
@@ -212,7 +212,7 @@ public AssignmentManager(Server master, ServerManager serverManager,
     this.zkTable = new ZKTable(this.master.getZooKeeper());
     this.maximumAssignmentAttempts =
       this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
-    this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
+    this.balancer = balancer;
     this.threadPoolExecutorService = Executors.newCachedThreadPool();
   }
 
@@ -1807,8 +1807,7 @@ RegionPlan getRegionPlan(final RegionState state,
 
     if (servers.isEmpty()) return null;
 
-    RegionPlan randomPlan = new RegionPlan(state.getRegion(), null,
-      balancer.randomAssignment(servers));
+    RegionPlan randomPlan = null;
     boolean newPlan = false;
     RegionPlan existingPlan = null;
 
@@ -1826,6 +1825,8 @@ RegionPlan getRegionPlan(final RegionState state,
           || existingPlan.getDestination() == null
           || drainingServers.contains(existingPlan.getDestination())) {
         newPlan = true;
+        randomPlan = new RegionPlan(state.getRegion(), null, balancer
+            .randomAssignment(servers));
         this.regionPlans.put(encodedName, randomPlan);
       }
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 54907a0ce742..2882fa0262ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -412,9 +412,9 @@ private void initializeZKBasedSystemTrackers() throws IOException,
         this, conf.getInt("hbase.master.catalog.timeout", Integer.MAX_VALUE));
     this.catalogTracker.start();
 
-    this.assignmentManager = new AssignmentManager(this, serverManager,
-        this.catalogTracker, this.executorService);
     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
+    this.assignmentManager = new AssignmentManager(this, serverManager,
+        this.catalogTracker, this.balancer, this.executorService);
     zooKeeper.registerListenerFirst(assignmentManager);
 
     this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 841649abb3fc..2215ceb12321 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -306,9 +306,11 @@ public void testBalance()
 
     // We need a mocked catalog tracker.
     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
+        .getConfiguration());
     // Create an AM.
-    AssignmentManager am =
-      new AssignmentManager(this.server, this.serverManager, ct, executor);
+    AssignmentManager am = new AssignmentManager(this.server,
+        this.serverManager, ct, balancer, executor);
     try {
       // Make sure our new AM gets callbacks; once registered, can't unregister.
       // Thats ok because we make a new zk watcher for each test.
@@ -371,9 +373,11 @@ public void testShutdownHandler() throws KeeperException, IOException {
 
     // We need a mocked catalog tracker.
     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
+        .getConfiguration());
     // Create an AM.
     AssignmentManager am =
-      new AssignmentManager(this.server, this.serverManager, ct, executor);
+      new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
     try {
       // Make sure our new AM gets callbacks; once registered, can't unregister.
       // Thats ok because we make a new zk watcher for each test.
@@ -470,9 +474,11 @@ public void testUnassignWithSplitAtSameTime() throws KeeperException, IOExceptio
     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
     // Need a mocked catalog tracker.
     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
+        .getConfiguration());
     // Create an AM.
     AssignmentManager am =
-      new AssignmentManager(this.server, this.serverManager, ct, null);
+      new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
     try {
       // First make sure my mock up basically works.  Unassign a region.
       unassign(am, SERVERNAME_A, hri);
@@ -584,8 +590,10 @@ private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final
     Mockito.when(ct.getConnection()).thenReturn(connection);
     // Create and startup an executor. Used by AM handling zk callbacks.
     ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
-    AssignmentManagerWithExtrasForTesting am =
-      new AssignmentManagerWithExtrasForTesting(server, manager, ct, executor);
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
+        .getConfiguration());
+    AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
+        server, manager, ct, balancer, executor);
     return am;
   }
 
@@ -601,10 +609,10 @@ class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
     AtomicBoolean gate = new AtomicBoolean(true);
 
     public AssignmentManagerWithExtrasForTesting(final Server master,
-        final ServerManager serverManager,
-        final CatalogTracker catalogTracker, final ExecutorService service)
+        final ServerManager serverManager, final CatalogTracker catalogTracker,
+        final LoadBalancer balancer, final ExecutorService service)
     throws KeeperException, IOException {
-      super(master, serverManager, catalogTracker, service);
+      super(master, serverManager, catalogTracker, balancer, service);
       this.es = service;
       this.ct = catalogTracker;
     }

From 49cc89cc60df2dd28c33cc92e4638b23c0d04734 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 23 Apr 2012 16:48:40 +0000
Subject: [PATCH 0150/1540] HBASE-5635 If getTaskList() returns null,
 splitlogWorker would go down and it won't serve any requests (Chinna)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329325 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/SplitLogWorker.java    | 36 +++++++++++--------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
index 0f119e53ad21..89c8ab84da46 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -461,25 +461,31 @@ public void nodeDataChanged(String path) {
 
 
   private List<String> getTaskList() {
-    for (int i = 0; i < zkretries; i++) {
+    List<String> childrenPaths = null;
+    long sleepTime = 1000;
+    // It will be in loop till it gets the list of children or
+    // it will come out if worker thread exited.
+    while (!exitWorker) {
       try {
-        return (ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
-            this.watcher.splitLogZNode));
-      } catch (KeeperException e) {
-        LOG.warn("Could not get children of znode " +
-            this.watcher.splitLogZNode, e);
-        try {
-          Thread.sleep(1000);
-        } catch (InterruptedException e1) {
-          LOG.warn("Interrupted while trying to get task list ...", e1);
-          Thread.currentThread().interrupt();
-          return null;
+        childrenPaths = ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
+            this.watcher.splitLogZNode);
+        if (childrenPaths != null) {
+          return childrenPaths;
         }
+      } catch (KeeperException e) {
+        LOG.warn("Could not get children of znode "
+            + this.watcher.splitLogZNode, e);
+      }
+      try {
+        LOG.debug("Retry listChildren of znode " + this.watcher.splitLogZNode
+            + " after sleep for " + sleepTime + "ms!");
+        Thread.sleep(sleepTime);
+      } catch (InterruptedException e1) {
+        LOG.warn("Interrupted while trying to get task list ...", e1);
+        Thread.currentThread().interrupt();
       }
     }
-    LOG.warn("Tried " + zkretries + " times, still couldn't fetch " +
-        "children of " + watcher.splitLogZNode + " giving up");
-    return null;
+    return childrenPaths;
   }
 
 

From 6819d7dc2e9f4b023e20807a0738dfbd6ed3aece Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 23 Apr 2012 20:13:28 +0000
Subject: [PATCH 0151/1540] HBASE-5833 0.92 build has been failing pretty
 consistently on TestMasterFailover

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329418 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   20 +-
 .../ExponentiallyDecayingSample.java          |    4 +-
 .../hadoop/hbase/regionserver/HRegion.java    |   23 +-
 .../hadoop/hbase/util/JVMClusterUtil.java     |   10 +-
 .../apache/hadoop/hbase/HBaseTestCase.java    |   15 +
 .../hbase/filter/TestColumnPrefixFilter.java  |  174 +-
 .../hbase/master/TestMasterFailover.java      |  163 +-
 .../hbase/master/TestOpenedRegionHandler.java |    6 +-
 .../hbase/regionserver/TestBlocksRead.java    |  389 +-
 .../hbase/regionserver/TestColumnSeeking.java |  153 +-
 .../regionserver/TestCompactSelection.java    |    5 +-
 .../hbase/regionserver/TestHRegion.java       | 3648 +++++++++--------
 .../regionserver/TestHRegionOnCluster.java    |  146 +
 .../hbase/regionserver/TestMinVersions.java   |  619 +--
 .../regionserver/TestResettingCounters.java   |   47 +-
 .../regionserver/TestSplitTransaction.java    |    4 +-
 .../handler/TestCloseRegionHandler.java       |   44 +-
 .../handler/TestOpenRegionHandler.java        |   50 +-
 .../hbase/regionserver/wal/TestWALReplay.java |   14 +-
 .../hadoop/hbase/util/TestMergeTool.java      |    7 +
 20 files changed, 2998 insertions(+), 2543 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index e9a13fdc4e62..7299e0392a7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -49,6 +49,7 @@
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -577,16 +578,17 @@ public long getEvictedCount() {
    */
   private static class EvictionThread extends HasThread {
     private WeakReference<LruBlockCache> cache;
+    private boolean go = true;
 
     public EvictionThread(LruBlockCache cache) {
-      super("LruBlockCache.EvictionThread");
+      super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
       setDaemon(true);
       this.cache = new WeakReference<LruBlockCache>(cache);
     }
 
     @Override
     public void run() {
-      while(true) {
+      while (this.go) {
         synchronized(this) {
           try {
             this.wait();
@@ -597,11 +599,17 @@ public void run() {
         cache.evict();
       }
     }
+
     public void evict() {
       synchronized(this) {
         this.notify(); // FindBugs NN_NAKED_NOTIFY
       }
     }
+
+    void shutdown() {
+      this.go = false;
+      interrupt();
+    }
   }
 
   /*
@@ -725,6 +733,14 @@ private long memorySize() {
 
   public void shutdown() {
     this.scheduleThreadPool.shutdown();
+    for (int i = 0; i < 10; i++) {
+      if (!this.scheduleThreadPool.isShutdown()) Threads.sleep(10);
+    }
+    if (!this.scheduleThreadPool.isShutdown()) {
+      List<Runnable> runnables = this.scheduleThreadPool.shutdownNow();
+      LOG.debug("Still running " + runnables);
+    }
+    this.evictionThread.shutdown();
   }
 
   /** Clears the cache. Used in tests. */
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
index fde93e1a5133..a3a1799b8f18 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
@@ -43,8 +43,8 @@ public class ExponentiallyDecayingSample implements Sample {
   private static final long RESCALE_THRESHOLD = TimeUnit.HOURS.toNanos(1);
 
   private static final ScheduledExecutorService TICK_SERVICE = 
-      Executors.newScheduledThreadPool(1, 
-          getNamedDaemonThreadFactory("decayingSampleTick"));
+    Executors.newScheduledThreadPool(1, 
+      getNamedDaemonThreadFactory(Thread.currentThread().getName() + ".decayingSampleTick."));
 
   private static volatile long CURRENT_TICK = 
       TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b8565ca422c6..3832a03b99ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3548,7 +3548,11 @@ public static HRegion newHRegion(Path tableDir, HLog log, FileSystem fs,
    * bootstrap code in the HMaster constructor.
    * Note, this method creates an {@link HLog} for the created region. It
    * needs to be closed explicitly.  Use {@link HRegion#getLog()} to get
-   * access.
+   * access.  <b>When done with a region created using this method, you will
+   * need to explicitly close the {@link HLog} it created too; it will not be
+   * done for you.  Not closing the log will leave at least a daemon thread
+   * running.</b>  Call {@link #closeHRegion(HRegion)} and it will do
+   * necessary cleanup for you.
    * @param info Info for region to create.
    * @param rootDir Root directory for HBase instance
    * @param conf
@@ -3563,6 +3567,23 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
     return createHRegion(info, rootDir, conf, hTableDescriptor, null);
   }
 
+  /**
+   * This will do the necessary cleanup a call to {@link #createHRegion(HRegionInfo, Path, Configuration, HTableDescriptor)}
+   * requires.  This method will close the region and then close its
+   * associated {@link HLog} file.  You use it if you call the other createHRegion,
+   * the one that takes an {@link HLog} instance but don't be surprised by the
+   * call to the {@link HLog#closeAndDelete()} on the {@link HLog} the
+   * HRegion was carrying.
+   * @param r
+   * @throws IOException
+   */
+  public static void closeHRegion(final HRegion r) throws IOException {
+    if (r == null) return;
+    r.close();
+    if (r.getLog() == null) return;
+    r.getLog().closeAndDelete();
+  }
+
   /**
    * Convenience method creating new HRegions. Used by createTable.
    * The {@link HLog} for the created region needs to be closed explicitly.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
index 6056f73fee49..b079f2e28608 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -221,13 +221,17 @@ public static void shutdown(final List<MasterThread> masters,
       final List<RegionServerThread> regionservers) {
     LOG.debug("Shutting down HBase Cluster");
     if (masters != null) {
+      // Do backups first.
+      JVMClusterUtil.MasterThread activeMaster = null;
       for (JVMClusterUtil.MasterThread t : masters) {
-        if (t.master.isActiveMaster()) {
-          t.master.shutdown();
-        } else {
+        if (!t.master.isActiveMaster()) {
           t.master.stopMaster();
+        } else {
+          activeMaster = t;
         }
       }
+      // Do active after.
+      if (activeMaster != null) activeMaster.master.shutdown();
     }
     // regionServerThreads can never be null because they are initialized when
     // the class is constructed.
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index 5fb9e2c81bce..4a57e4da8255 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -170,6 +170,16 @@ protected Path getUnitTestdir(String testName) {
       );
     }
 
+  /**
+   * You must call close on the returned region and then close on the log file
+   * it created. Do {@link HRegion#close()} followed by {@link HRegion#getLog()}
+   * and on it call close.
+   * @param desc
+   * @param startKey
+   * @param endKey
+   * @return An {@link HRegion}
+   * @throws IOException
+   */
   protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
       byte [] endKey)
   throws IOException {
@@ -673,6 +683,11 @@ public static void shutdownDfs(MiniDFSCluster cluster) {
     }
   }
 
+  /**
+   * You must call {@link #closeRootAndMeta()} when done after calling this
+   * method. It does cleanup.
+   * @throws IOException
+   */
   protected void createRootAndMetaRegions() throws IOException {
     root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir,
         conf, HTableDescriptor.ROOT_TABLEDESC);
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
index 333766c4b052..9f899143b770 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
@@ -50,52 +50,56 @@ public void testColumnPrefixFilter() throws IOException {
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
       getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
-
-    List<String> rows = generateRandomWords(100, "row");
-    List<String> columns = generateRandomWords(10000, "column");
-    long maxTimestamp = 2;
-
-    List<KeyValue> kvList = new ArrayList<KeyValue>();
-
-    Map<String, List<KeyValue>> prefixMap = new HashMap<String,
-        List<KeyValue>>();
-
-    prefixMap.put("p", new ArrayList<KeyValue>());
-    prefixMap.put("s", new ArrayList<KeyValue>());
-
-    String valueString = "ValueString";
-
-    for (String row: rows) {
-      Put p = new Put(Bytes.toBytes(row));
-      p.setWriteToWAL(false);
-      for (String column: columns) {
-        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
-          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
-              valueString);
-          p.add(kv);
-          kvList.add(kv);
-          for (String s: prefixMap.keySet()) {
-            if (column.startsWith(s)) {
-              prefixMap.get(s).add(kv);
+    try {
+      List<String> rows = generateRandomWords(100, "row");
+      List<String> columns = generateRandomWords(10000, "column");
+      long maxTimestamp = 2;
+
+      List<KeyValue> kvList = new ArrayList<KeyValue>();
+
+      Map<String, List<KeyValue>> prefixMap = new HashMap<String,
+          List<KeyValue>>();
+
+      prefixMap.put("p", new ArrayList<KeyValue>());
+      prefixMap.put("s", new ArrayList<KeyValue>());
+
+      String valueString = "ValueString";
+
+      for (String row: rows) {
+        Put p = new Put(Bytes.toBytes(row));
+        p.setWriteToWAL(false);
+        for (String column: columns) {
+          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
+            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
+                valueString);
+            p.add(kv);
+            kvList.add(kv);
+            for (String s: prefixMap.keySet()) {
+              if (column.startsWith(s)) {
+                prefixMap.get(s).add(kv);
+              }
             }
           }
         }
+        region.put(p);
       }
-      region.put(p);
-    }
 
-    ColumnPrefixFilter filter;
-    Scan scan = new Scan();
-    scan.setMaxVersions();
-    for (String s: prefixMap.keySet()) {
-      filter = new ColumnPrefixFilter(Bytes.toBytes(s));
+      ColumnPrefixFilter filter;
+      Scan scan = new Scan();
+      scan.setMaxVersions();
+      for (String s: prefixMap.keySet()) {
+        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
 
-      scan.setFilter(filter);
+        scan.setFilter(filter);
 
-      InternalScanner scanner = region.getScanner(scan);
-      List<KeyValue> results = new ArrayList<KeyValue>();
-      while(scanner.next(results));
-      assertEquals(prefixMap.get(s).size(), results.size());
+        InternalScanner scanner = region.getScanner(scan);
+        List<KeyValue> results = new ArrayList<KeyValue>();
+        while(scanner.next(results));
+        assertEquals(prefixMap.get(s).size(), results.size());
+      }
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
     }
 
     region.close();
@@ -110,55 +114,59 @@ public void testColumnPrefixFilterWithFilterList() throws IOException {
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
       getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
-
-    List<String> rows = generateRandomWords(100, "row");
-    List<String> columns = generateRandomWords(10000, "column");
-    long maxTimestamp = 2;
-
-    List<KeyValue> kvList = new ArrayList<KeyValue>();
-
-    Map<String, List<KeyValue>> prefixMap = new HashMap<String,
-        List<KeyValue>>();
-
-    prefixMap.put("p", new ArrayList<KeyValue>());
-    prefixMap.put("s", new ArrayList<KeyValue>());
-
-    String valueString = "ValueString";
-
-    for (String row: rows) {
-      Put p = new Put(Bytes.toBytes(row));
-      p.setWriteToWAL(false);
-      for (String column: columns) {
-        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
-          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
-              valueString);
-          p.add(kv);
-          kvList.add(kv);
-          for (String s: prefixMap.keySet()) {
-            if (column.startsWith(s)) {
-              prefixMap.get(s).add(kv);
+    try {
+      List<String> rows = generateRandomWords(100, "row");
+      List<String> columns = generateRandomWords(10000, "column");
+      long maxTimestamp = 2;
+
+      List<KeyValue> kvList = new ArrayList<KeyValue>();
+
+      Map<String, List<KeyValue>> prefixMap = new HashMap<String,
+          List<KeyValue>>();
+
+      prefixMap.put("p", new ArrayList<KeyValue>());
+      prefixMap.put("s", new ArrayList<KeyValue>());
+
+      String valueString = "ValueString";
+
+      for (String row: rows) {
+        Put p = new Put(Bytes.toBytes(row));
+        p.setWriteToWAL(false);
+        for (String column: columns) {
+          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
+            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
+                valueString);
+            p.add(kv);
+            kvList.add(kv);
+            for (String s: prefixMap.keySet()) {
+              if (column.startsWith(s)) {
+                prefixMap.get(s).add(kv);
+              }
             }
           }
         }
+        region.put(p);
       }
-      region.put(p);
-    }
 
-    ColumnPrefixFilter filter;
-    Scan scan = new Scan();
-    scan.setMaxVersions();
-    for (String s: prefixMap.keySet()) {
-      filter = new ColumnPrefixFilter(Bytes.toBytes(s));
-
-      //this is how this test differs from the one above
-      FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
-      filterList.addFilter(filter);
-      scan.setFilter(filterList);
-
-      InternalScanner scanner = region.getScanner(scan);
-      List<KeyValue> results = new ArrayList<KeyValue>();
-      while(scanner.next(results));
-      assertEquals(prefixMap.get(s).size(), results.size());
+      ColumnPrefixFilter filter;
+      Scan scan = new Scan();
+      scan.setMaxVersions();
+      for (String s: prefixMap.keySet()) {
+        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
+
+        //this is how this test differs from the one above
+        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
+        filterList.addFilter(filter);
+        scan.setFilter(filterList);
+
+        InternalScanner scanner = region.getScanner(scan);
+        List<KeyValue> results = new ArrayList<KeyValue>();
+        while(scanner.next(results));
+        assertEquals(prefixMap.get(s).size(), results.size());
+      }
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
     }
 
     region.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index d3ea94b5b198..14cdb90a7b34 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -24,6 +24,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -56,6 +57,55 @@
 public class TestMasterFailover {
   private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
 
+  @Test (timeout=180000)
+  public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
+      throws Exception {
+    LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
+    final int NUM_MASTERS = 1;
+    final int NUM_RS = 2;
+
+    Configuration conf = HBaseConfiguration.create();
+    conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
+    conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
+    // Start the cluster
+    HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+
+    // Find regionserver carrying meta.
+    List<RegionServerThread> regionServerThreads =
+      cluster.getRegionServerThreads();
+    int count = -1;
+    HRegion metaRegion = null;
+    for (RegionServerThread regionServerThread : regionServerThreads) {
+      HRegionServer regionServer = regionServerThread.getRegionServer();
+      metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
+      count++;
+      regionServer.abort("");
+      if (null != metaRegion) break;
+    }
+    HRegionServer regionServer = cluster.getRegionServer(count);
+
+    TEST_UTIL.shutdownMiniHBaseCluster();
+
+    // Create a ZKW to use in the test
+    ZooKeeperWatcher zkw = 
+      HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, 
+          metaRegion, regionServer.getServerName());
+
+    LOG.info("Staring cluster for second time");
+    TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
+
+    // Failover should be completed, now wait for no RIT
+    log("Waiting for no more RIT");
+    ZKAssign.blockUntilNoRIT(zkw);
+
+    zkw.close();
+    // Stop the cluster
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
   /**
    * Simple test of master failover.
    * <p>
@@ -101,6 +151,7 @@ public void testSimpleMasterFailover() throws Exception {
     }
     assertEquals(1, numActive);
     assertEquals(NUM_MASTERS, masterThreads.size());
+    LOG.info("Active master " + activeName);
 
     // Check that ClusterStatus reports the correct active and backup masters
     assertNotNull(active);
@@ -110,16 +161,16 @@ public void testSimpleMasterFailover() throws Exception {
     assertEquals(2, status.getBackupMasters().size());
 
     // attempt to stop one of the inactive masters
-    LOG.debug("\n\nStopping a backup master\n");
     int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
+    HMaster master = cluster.getMaster(backupIndex);
+    LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
     cluster.stopMaster(backupIndex, false);
     cluster.waitOnMaster(backupIndex);
 
-    // verify still one active master and it's the same
+    // Verify still one active master and it's the same
     for (int i = 0; i < masterThreads.size(); i++) {
       if (masterThreads.get(i).getMaster().isActiveMaster()) {
-        assertTrue(activeName.equals(
-            masterThreads.get(i).getMaster().getServerName()));
+        assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
         activeIndex = i;
         active = masterThreads.get(activeIndex).getMaster();
       }
@@ -127,7 +178,7 @@ public void testSimpleMasterFailover() throws Exception {
     assertEquals(1, numActive);
     assertEquals(2, masterThreads.size());
     int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
-    LOG.info("Active master managing " + rsCount +  " regions servers");
+    LOG.info("Active master " + active.getServerName() + " managing " + rsCount +  " regions servers");
     assertEquals(3, rsCount);
 
     // Check that ClusterStatus reports the correct active and backup masters
@@ -138,7 +189,7 @@ public void testSimpleMasterFailover() throws Exception {
     assertEquals(1, status.getBackupMasters().size());
 
     // kill the active master
-    LOG.debug("\n\nStopping the active master\n");
+    LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
     cluster.stopMaster(activeIndex, false);
     cluster.waitOnMaster(activeIndex);
 
@@ -159,7 +210,7 @@ public void testSimpleMasterFailover() throws Exception {
     assertEquals(0, status.getBackupMastersSize());
     assertEquals(0, status.getBackupMasters().size());
     int rss = status.getServersSize();
-    LOG.info("Active master " + mastername.getHostname() + " managing " +
+    LOG.info("Active master " + mastername.getServerName() + " managing " +
       rss +  " region servers");
     assertEquals(3, rss);
 
@@ -167,83 +218,6 @@ public void testSimpleMasterFailover() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  @Test
-  public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
-      throws Exception {
-    final int NUM_MASTERS = 1;
-    final int NUM_RS = 2;
-
-    Configuration conf = HBaseConfiguration.create();
-    conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
-    conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
-    // Start the cluster
-    HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-
-    // get all the master threads
-    List<MasterThread> masterThreads = cluster.getMasterThreads();
-
-    // wait for each to come online
-    for (MasterThread mt : masterThreads) {
-      assertTrue(mt.isAlive());
-    }
-
-    // verify only one is the active master and we have right number
-    int numActive = 0;
-    ServerName activeName = null;
-    for (int i = 0; i < masterThreads.size(); i++) {
-      if (masterThreads.get(i).getMaster().isActiveMaster()) {
-        numActive++;
-        activeName = masterThreads.get(i).getMaster().getServerName();
-      }
-    }
-    assertEquals(1, numActive);
-    assertEquals(NUM_MASTERS, masterThreads.size());
-
-    // verify still one active master and it's the same
-    for (int i = 0; i < masterThreads.size(); i++) {
-      if (masterThreads.get(i).getMaster().isActiveMaster()) {
-        assertTrue(activeName.equals(masterThreads.get(i).getMaster()
-            .getServerName()));
-      }
-    }
-    assertEquals(1, numActive);
-    assertEquals(1, masterThreads.size());
-
-    List<RegionServerThread> regionServerThreads = cluster
-        .getRegionServerThreads();
-    int count = -1;
-    HRegion metaRegion = null;
-    for (RegionServerThread regionServerThread : regionServerThreads) {
-      HRegionServer regionServer = regionServerThread.getRegionServer();
-      metaRegion = regionServer
-          .getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
-      count++;
-      regionServer.abort("");
-      if (null != metaRegion) {
-        break;
-      }
-    }
-    HRegionServer regionServer = cluster.getRegionServer(count);
-
-    cluster.shutdown();
-    // Create a ZKW to use in the test
-    ZooKeeperWatcher zkw = 
-      HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, 
-          metaRegion, regionServer.getServerName());
-
-    TEST_UTIL.startMiniHBaseCluster(1, 1);
-
-    // Failover should be completed, now wait for no RIT
-    log("Waiting for no more RIT");
-    ZKAssign.blockUntilNoRIT(zkw);
-
-    // Stop the cluster
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-
   /**
    * Complex test of master failover that tests as many permutations of the
    * different possible states that regions in transition could be in within ZK.
@@ -379,7 +353,7 @@ public void testMasterFailoverWithMockedRIT() throws Exception {
     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled);
 
     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), null, null);
-    HRegion.createHRegion(hriEnabled, rootdir, conf, htdEnabled);
+    createRegion(hriEnabled, rootdir, conf, htdEnabled);
 
     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
@@ -390,7 +364,7 @@ public void testMasterFailoverWithMockedRIT() throws Exception {
     // Write the .tableinfo
     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
-    HRegion.createHRegion(hriDisabled, rootdir, conf, htdDisabled);
+    createRegion(hriDisabled, rootdir, conf, htdDisabled);
     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
 
@@ -692,7 +666,7 @@ public boolean isAborted() {
     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled);
     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(),
         null, null);
-    HRegion.createHRegion(hriEnabled, rootdir, conf, htdEnabled);
+    createRegion(hriEnabled, rootdir, conf, htdEnabled);
 
     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
@@ -703,7 +677,7 @@ public boolean isAborted() {
     // Write the .tableinfo
     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
-    HRegion.createHRegion(hriDisabled, rootdir, conf, htdDisabled);
+    createRegion(hriDisabled, rootdir, conf, htdDisabled);
 
     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
@@ -1024,6 +998,19 @@ public boolean isAborted() {
     TEST_UTIL.shutdownMiniCluster();
   }
 
+  HRegion createRegion(final HRegionInfo  hri, final Path rootdir, final Configuration c,
+      final HTableDescriptor htd)
+  throws IOException {
+    HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
+    // The above call to create a region will create an hlog file.  Each
+    // log file create will also create a running thread to do syncing.  We need
+    // to close out this log else we will have a running thread trying to sync
+    // the file system continuously which is ugly when dfs is taken away at the
+    // end of the test.
+    HRegion.closeHRegion(r);
+    return r;
+  }
+
   // TODO: Next test to add is with testing permutations of the RIT or the RS
   //       killed are hosting ROOT and META regions.
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java
index 6633a20b9fee..f0e7e0c4cbc1 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java
@@ -112,6 +112,7 @@ public void testOpenedRegionHandlerOnMasterRestart() throws Exception {
   @Test
   public void testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches()
       throws Exception {
+    HRegion region = null;
     try {
       int testIndex = 0;
       TEST_UTIL.startMiniZKCluster();
@@ -120,8 +121,7 @@ public void testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches()
           "testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches");
       HRegionInfo hri = new HRegionInfo(htd.getName(),
           Bytes.toBytes(testIndex), Bytes.toBytes(testIndex + 1));
-      HRegion region = HRegion.createHRegion(hri, TEST_UTIL
-          .getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+      region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
       assertNotNull(region);
       AssignmentManager am = Mockito.mock(AssignmentManager.class);
       when(am.isRegionInTransition(hri)).thenReturn(
@@ -160,6 +160,8 @@ public void testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches()
       assertEquals("The region should not be opened successfully.", regionName,
           region.getRegionInfo().getEncodedName());
     } finally {
+      region.close();
+      region.getLog().closeAndDelete();
       TEST_UTIL.shutdownMiniZKCluster();
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
index bd83a7116b4f..3813ea4f6a8c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
@@ -85,7 +85,16 @@ protected void tearDown() throws Exception {
     EnvironmentEdgeManagerTestHelper.reset();
   }
 
-  private void initHRegion(byte[] tableName, String callingMethod,
+  /**
+   * Callers must afterward call {@link HRegion#closeHRegion(HRegion)}
+   * @param tableName
+   * @param callingMethod
+   * @param conf
+   * @param families
+   * @throws IOException
+   * @return created and initialized region.
+   */
+  private HRegion initHRegion(byte[] tableName, String callingMethod,
       HBaseConfiguration conf, String family) throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     HColumnDescriptor familyDesc;
@@ -99,8 +108,9 @@ private void initHRegion(byte[] tableName, String callingMethod,
 
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     Path path = new Path(DIR + callingMethod);
-    region = HRegion.createHRegion(info, path, conf, htd);
+    HRegion r = HRegion.createHRegion(info, path, conf, htd);
     blockCache = new CacheConfig(conf).getBlockCache();
+    return r;
   }
 
   private void putData(String family, String row, String col, long version)
@@ -212,45 +222,50 @@ public void testBlocksRead() throws Exception {
     String FAMILY = "cf1";
     KeyValue kvs[];
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
-
-    putData(FAMILY, "row", "col1", 1);
-    putData(FAMILY, "row", "col2", 2);
-    putData(FAMILY, "row", "col3", 3);
-    putData(FAMILY, "row", "col4", 4);
-    putData(FAMILY, "row", "col5", 5);
-    putData(FAMILY, "row", "col6", 6);
-    putData(FAMILY, "row", "col7", 7);
-    region.flushcache();
-
-    // Expected block reads: 1
-    // The top block has the KV we are
-    // interested. So only 1 seek is needed.
-    kvs = getData(FAMILY, "row", "col1", 1);
-    assertEquals(1, kvs.length);
-    verifyData(kvs[0], "row", "col1", 1);
-
-    // Expected block reads: 2
-    // The top block and next block has the KVs we are
-    // interested. So only 2 seek is needed.
-    kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
-    assertEquals(2, kvs.length);
-    verifyData(kvs[0], "row", "col1", 1);
-    verifyData(kvs[1], "row", "col2", 2);
-
-    // Expected block reads: 3
-    // The first 2 seeks is to find out col2. [HBASE-4443]
-    // One additional seek for col3
-    // So 3 seeks are needed.
-    kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3);
-    assertEquals(2, kvs.length);
-    verifyData(kvs[0], "row", "col2", 2);
-    verifyData(kvs[1], "row", "col3", 3);
-
-    // Expected block reads: 2. [HBASE-4443]
-    kvs = getData(FAMILY, "row", Arrays.asList("col5"), 2);
-    assertEquals(1, kvs.length);
-    verifyData(kvs[0], "row", "col5", 5);
+    this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+
+    try {
+      putData(FAMILY, "row", "col1", 1);
+      putData(FAMILY, "row", "col2", 2);
+      putData(FAMILY, "row", "col3", 3);
+      putData(FAMILY, "row", "col4", 4);
+      putData(FAMILY, "row", "col5", 5);
+      putData(FAMILY, "row", "col6", 6);
+      putData(FAMILY, "row", "col7", 7);
+      region.flushcache();
+
+      // Expected block reads: 1
+      // The top block has the KV we are
+      // interested. So only 1 seek is needed.
+      kvs = getData(FAMILY, "row", "col1", 1);
+      assertEquals(1, kvs.length);
+      verifyData(kvs[0], "row", "col1", 1);
+
+      // Expected block reads: 2
+      // The top block and next block has the KVs we are
+      // interested. So only 2 seek is needed.
+      kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
+      assertEquals(2, kvs.length);
+      verifyData(kvs[0], "row", "col1", 1);
+      verifyData(kvs[1], "row", "col2", 2);
+
+      // Expected block reads: 3
+      // The first 2 seeks is to find out col2. [HBASE-4443]
+      // One additional seek for col3
+      // So 3 seeks are needed.
+      kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3);
+      assertEquals(2, kvs.length);
+      verifyData(kvs[0], "row", "col2", 2);
+      verifyData(kvs[1], "row", "col3", 3);
+
+      // Expected block reads: 2. [HBASE-4443]
+      kvs = getData(FAMILY, "row", Arrays.asList("col5"), 2);
+      assertEquals(1, kvs.length);
+      verifyData(kvs[0], "row", "col5", 5);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   /**
@@ -264,97 +279,102 @@ public void testLazySeekBlocksRead() throws Exception {
     String FAMILY = "cf1";
     KeyValue kvs[];
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
-
-    // File 1
-    putData(FAMILY, "row", "col1", 1);
-    putData(FAMILY, "row", "col2", 2);
-    region.flushcache();
-
-    // File 2
-    putData(FAMILY, "row", "col1", 3);
-    putData(FAMILY, "row", "col2", 4);
-    region.flushcache();
-
-    // Expected blocks read: 1.
-    // File 2's top block is also the KV we are
-    // interested. So only 1 seek is needed.
-    kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1);
-    assertEquals(1, kvs.length);
-    verifyData(kvs[0], "row", "col1", 3);
-
-    // Expected blocks read: 2
-    // File 2's top block has the "col1" KV we are
-    // interested. We also need "col2" which is in a block
-    // of its own. So, we need that block as well.
-    kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
-    assertEquals(2, kvs.length);
-    verifyData(kvs[0], "row", "col1", 3);
-    verifyData(kvs[1], "row", "col2", 4);
-
-    // File 3: Add another column
-    putData(FAMILY, "row", "col3", 5);
-    region.flushcache();
-
-    // Expected blocks read: 1
-    // File 3's top block has the "col3" KV we are
-    // interested. So only 1 seek is needed.
-    kvs = getData(FAMILY, "row", "col3", 1);
-    assertEquals(1, kvs.length);
-    verifyData(kvs[0], "row", "col3", 5);
-
-    // Get a column from older file.
-    // For ROWCOL Bloom filter: Expected blocks read: 1.
-    // For ROW Bloom filter: Expected blocks read: 2.
-    // For NONE Bloom filter: Expected blocks read: 2.
-    kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1, 2, 2);
-    assertEquals(1, kvs.length);
-    verifyData(kvs[0], "row", "col1", 3);
-
-    // File 4: Delete the entire row.
-    deleteFamily(FAMILY, "row", 6);
-    region.flushcache();
-
-    // For ROWCOL Bloom filter: Expected blocks read: 2.
-    // For ROW Bloom filter: Expected blocks read: 3.
-    // For NONE Bloom filter: Expected blocks read: 3.
-    kvs = getData(FAMILY, "row", "col1", 2, 3, 3);
-    assertEquals(0, kvs.length);
-    kvs = getData(FAMILY, "row", "col2", 3, 4, 4);
-    assertEquals(0, kvs.length);
-    kvs = getData(FAMILY, "row", "col3", 2);
-    assertEquals(0, kvs.length);
-    kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 4);
-    assertEquals(0, kvs.length);
-
-    // File 5: Delete
-    deleteFamily(FAMILY, "row", 10);
-    region.flushcache();
-
-    // File 6: some more puts, but with timestamps older than the
-    // previous delete.
-    putData(FAMILY, "row", "col1", 7);
-    putData(FAMILY, "row", "col2", 8);
-    putData(FAMILY, "row", "col3", 9);
-    region.flushcache();
-
-    // Baseline expected blocks read: 8. [HBASE-4532]
-    kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
-    assertEquals(0, kvs.length);
-
-    // File 7: Put back new data
-    putData(FAMILY, "row", "col1", 11);
-    putData(FAMILY, "row", "col2", 12);
-    putData(FAMILY, "row", "col3", 13);
-    region.flushcache();
-
-
-    // Expected blocks read: 5. [HBASE-4585]
-    kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
-    assertEquals(3, kvs.length);
-    verifyData(kvs[0], "row", "col1", 11);
-    verifyData(kvs[1], "row", "col2", 12);
-    verifyData(kvs[2], "row", "col3", 13);
+    this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+
+    try {
+      // File 1
+      putData(FAMILY, "row", "col1", 1);
+      putData(FAMILY, "row", "col2", 2);
+      region.flushcache();
+
+      // File 2
+      putData(FAMILY, "row", "col1", 3);
+      putData(FAMILY, "row", "col2", 4);
+      region.flushcache();
+
+      // Expected blocks read: 1.
+      // File 2's top block is also the KV we are
+      // interested. So only 1 seek is needed.
+      kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1);
+      assertEquals(1, kvs.length);
+      verifyData(kvs[0], "row", "col1", 3);
+
+      // Expected blocks read: 2
+      // File 2's top block has the "col1" KV we are
+      // interested. We also need "col2" which is in a block
+      // of its own. So, we need that block as well.
+      kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
+      assertEquals(2, kvs.length);
+      verifyData(kvs[0], "row", "col1", 3);
+      verifyData(kvs[1], "row", "col2", 4);
+
+      // File 3: Add another column
+      putData(FAMILY, "row", "col3", 5);
+      region.flushcache();
+
+      // Expected blocks read: 1
+      // File 3's top block has the "col3" KV we are
+      // interested. So only 1 seek is needed.
+      kvs = getData(FAMILY, "row", "col3", 1);
+      assertEquals(1, kvs.length);
+      verifyData(kvs[0], "row", "col3", 5);
+
+      // Get a column from older file.
+      // For ROWCOL Bloom filter: Expected blocks read: 1.
+      // For ROW Bloom filter: Expected blocks read: 2.
+      // For NONE Bloom filter: Expected blocks read: 2.
+      kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1, 2, 2);
+      assertEquals(1, kvs.length);
+      verifyData(kvs[0], "row", "col1", 3);
+
+      // File 4: Delete the entire row.
+      deleteFamily(FAMILY, "row", 6);
+      region.flushcache();
+
+      // For ROWCOL Bloom filter: Expected blocks read: 2.
+      // For ROW Bloom filter: Expected blocks read: 3.
+      // For NONE Bloom filter: Expected blocks read: 3.
+      kvs = getData(FAMILY, "row", "col1", 2, 3, 3);
+      assertEquals(0, kvs.length);
+      kvs = getData(FAMILY, "row", "col2", 3, 4, 4);
+      assertEquals(0, kvs.length);
+      kvs = getData(FAMILY, "row", "col3", 2);
+      assertEquals(0, kvs.length);
+      kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 4);
+      assertEquals(0, kvs.length);
+
+      // File 5: Delete
+      deleteFamily(FAMILY, "row", 10);
+      region.flushcache();
+
+      // File 6: some more puts, but with timestamps older than the
+      // previous delete.
+      putData(FAMILY, "row", "col1", 7);
+      putData(FAMILY, "row", "col2", 8);
+      putData(FAMILY, "row", "col3", 9);
+      region.flushcache();
+
+      // Baseline expected blocks read: 8. [HBASE-4532]
+      kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
+      assertEquals(0, kvs.length);
+ 
+      // File 7: Put back new data
+      putData(FAMILY, "row", "col1", 11);
+      putData(FAMILY, "row", "col2", 12);
+      putData(FAMILY, "row", "col3", 13);
+      region.flushcache();
+
+
+      // Expected blocks read: 5. [HBASE-4585]
+      kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
+      assertEquals(3, kvs.length);
+      verifyData(kvs[0], "row", "col1", 11);
+      verifyData(kvs[1], "row", "col2", 12);
+      verifyData(kvs[2], "row", "col3", 13);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   /**
@@ -367,62 +387,71 @@ public void testBlocksStoredWhenCachingDisabled() throws Exception {
     String FAMILY = "cf1";
 
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
-
-    putData(FAMILY, "row", "col1", 1);
-    putData(FAMILY, "row", "col2", 2);
-    region.flushcache();
-
-    // Execute a scan with caching turned off
-    // Expected blocks stored: 0
-    long blocksStart = getBlkCount();
-    Scan scan = new Scan();
-    scan.setCacheBlocks(false);
-    RegionScanner rs = region.getScanner(scan);
-    List<KeyValue> result = new ArrayList<KeyValue>(2);
-    rs.next(result);
-    assertEquals(2 * BLOOM_TYPE.length, result.size());
-    rs.close();
-    long blocksEnd = getBlkCount();
-
-    assertEquals(blocksStart, blocksEnd);
-
-    // Execute with caching turned on
-    // Expected blocks stored: 2
-    blocksStart = blocksEnd;
-    scan.setCacheBlocks(true);
-    rs = region.getScanner(scan);
-    result = new ArrayList<KeyValue>(2);
-    rs.next(result);
-    assertEquals(2 * BLOOM_TYPE.length, result.size());
-    rs.close();
-    blocksEnd = getBlkCount();
+    this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+
+    try {
+      putData(FAMILY, "row", "col1", 1);
+      putData(FAMILY, "row", "col2", 2);
+      region.flushcache();
+
+      // Execute a scan with caching turned off
+      // Expected blocks stored: 0
+      long blocksStart = getBlkCount();
+      Scan scan = new Scan();
+      scan.setCacheBlocks(false);
+      RegionScanner rs = region.getScanner(scan);
+      List<KeyValue> result = new ArrayList<KeyValue>(2);
+      rs.next(result);
+      assertEquals(2 * BLOOM_TYPE.length, result.size());
+      rs.close();
+      long blocksEnd = getBlkCount();
+
+      assertEquals(blocksStart, blocksEnd);
+
+      // Execute with caching turned on
+      // Expected blocks stored: 2
+      blocksStart = blocksEnd;
+      scan.setCacheBlocks(true);
+      rs = region.getScanner(scan);
+      result = new ArrayList<KeyValue>(2);
+      rs.next(result);
+      assertEquals(2 * BLOOM_TYPE.length, result.size());
+      rs.close();
+      blocksEnd = getBlkCount();
     
-    assertEquals(2 * BLOOM_TYPE.length, blocksEnd - blocksStart);
-	}
+      assertEquals(2 * BLOOM_TYPE.length, blocksEnd - blocksStart);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+  }
 
-	@Test
+  @Test
   public void testLazySeekBlocksReadWithDelete() throws Exception {
     byte[] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete");
     String FAMILY = "cf1";
     KeyValue kvs[];
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
-
-    deleteFamily(FAMILY, "row", 200);
-    for (int i = 0; i < 100; i++) {
-      putData(FAMILY, "row", "col" + i, i);
+    this.region = initHRegion(TABLE, getName(), conf, FAMILY);
+    try {
+      deleteFamily(FAMILY, "row", 200);
+      for (int i = 0; i < 100; i++) {
+        putData(FAMILY, "row", "col" + i, i);
+      }
+      putData(FAMILY, "row", "col99", 201);
+      region.flushcache();
+
+      kvs = getData(FAMILY, "row", Arrays.asList("col0"), 2);
+      assertEquals(0, kvs.length);
+
+      kvs = getData(FAMILY, "row", Arrays.asList("col99"), 2);
+      assertEquals(1, kvs.length);
+      verifyData(kvs[0], "row", "col99", 201);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    putData(FAMILY, "row", "col99", 201);
-    region.flushcache();
-
-    kvs = getData(FAMILY, "row", Arrays.asList("col0"), 2);
-    assertEquals(0, kvs.length);
-
-    kvs = getData(FAMILY, "row", Arrays.asList("col99"), 2);
-    assertEquals(1, kvs.length);
-    verifyData(kvs[0], "row", "col99", 201);
-	}
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
index 84296ab6f36b..ab952efb037e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
@@ -62,95 +62,98 @@ public void testDuplicateVersions() throws IOException {
     HRegion region =
         HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
             .getConfiguration(), htd);
-
-    List<String> rows = generateRandomWords(10, "row");
-    List<String> allColumns = generateRandomWords(10, "column");
-    List<String> values = generateRandomWords(100, "value");
-
-    long maxTimestamp = 2;
-    double selectPercent = 0.5;
-    int numberOfTests = 5;
-    double flushPercentage = 0.2;
-    double minorPercentage = 0.2;
-    double majorPercentage = 0.2;
-    double putPercentage = 0.2;
-
-    HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
-
-    HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
-    ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
-
-    for (int i = 0; i < numberOfTests; i++) {
-      kvMaps[i] = new HashMap<String, KeyValue>();
-      columnLists[i] = new ArrayList<String>();
-      for (String column : allColumns) {
-        if (Math.random() < selectPercent) {
-          columnLists[i].add(column);
+    try {
+      List<String> rows = generateRandomWords(10, "row");
+      List<String> allColumns = generateRandomWords(10, "column");
+      List<String> values = generateRandomWords(100, "value");
+
+      long maxTimestamp = 2;
+      double selectPercent = 0.5;
+      int numberOfTests = 5;
+      double flushPercentage = 0.2;
+      double minorPercentage = 0.2;
+      double majorPercentage = 0.2;
+      double putPercentage = 0.2;
+
+      HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
+
+      HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
+      ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
+
+      for (int i = 0; i < numberOfTests; i++) {
+        kvMaps[i] = new HashMap<String, KeyValue>();
+        columnLists[i] = new ArrayList<String>();
+        for (String column : allColumns) {
+          if (Math.random() < selectPercent) {
+            columnLists[i].add(column);
+          }
         }
       }
-    }
 
-    for (String value : values) {
-      for (String row : rows) {
-        Put p = new Put(Bytes.toBytes(row));
-        p.setWriteToWAL(false);
-        for (String column : allColumns) {
-          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
-            KeyValue kv =
-                KeyValueTestUtil.create(row, family, column, timestamp, value);
-            if (Math.random() < putPercentage) {
-              p.add(kv);
-              allKVMap.put(kv.getKeyString(), kv);
-              for (int i = 0; i < numberOfTests; i++) {
-                if (columnLists[i].contains(column)) {
-                  kvMaps[i].put(kv.getKeyString(), kv);
+      for (String value : values) {
+        for (String row : rows) {
+          Put p = new Put(Bytes.toBytes(row));
+          p.setWriteToWAL(false);
+          for (String column : allColumns) {
+            for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
+              KeyValue kv =
+                  KeyValueTestUtil.create(row, family, column, timestamp, value);
+              if (Math.random() < putPercentage) {
+                p.add(kv);
+                allKVMap.put(kv.getKeyString(), kv);
+                for (int i = 0; i < numberOfTests; i++) {
+                  if (columnLists[i].contains(column)) {
+                    kvMaps[i].put(kv.getKeyString(), kv);
+                  }
                 }
               }
             }
           }
-        }
-        region.put(p);
-        if (Math.random() < flushPercentage) {
-          LOG.info("Flushing... ");
-          region.flushcache();
-        }
+          region.put(p);
+          if (Math.random() < flushPercentage) {
+            LOG.info("Flushing... ");
+            region.flushcache();
+          }
 
-        if (Math.random() < minorPercentage) {
-          LOG.info("Minor compacting... ");
-          region.compactStores(false);
-        }
+          if (Math.random() < minorPercentage) {
+            LOG.info("Minor compacting... ");
+            region.compactStores(false);
+          }
 
-        if (Math.random() < majorPercentage) {
-          LOG.info("Major compacting... ");
-          region.compactStores(true);
+          if (Math.random() < majorPercentage) {
+            LOG.info("Major compacting... ");
+            region.compactStores(true);
+          }
         }
       }
-    }
 
-    for (int i = 0; i < numberOfTests + 1; i++) {
-      Collection<KeyValue> kvSet;
-      Scan scan = new Scan();
-      scan.setMaxVersions();
-      if (i < numberOfTests) {
-        kvSet = kvMaps[i].values();
-        for (String column : columnLists[i]) {
-          scan.addColumn(familyBytes, Bytes.toBytes(column));
-        }
-        LOG.info("ExplicitColumns scanner");
-        LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
-            + kvSet.size());
-      } else {
-        kvSet = allKVMap.values();
-        LOG.info("Wildcard scanner");
-        LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
+      for (int i = 0; i < numberOfTests + 1; i++) {
+        Collection<KeyValue> kvSet;
+        Scan scan = new Scan();
+        scan.setMaxVersions();
+        if (i < numberOfTests) {
+          kvSet = kvMaps[i].values();
+          for (String column : columnLists[i]) {
+            scan.addColumn(familyBytes, Bytes.toBytes(column));
+          }
+          LOG.info("ExplicitColumns scanner");
+          LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
+              + kvSet.size());
+        } else {
+          kvSet = allKVMap.values();
+          LOG.info("Wildcard scanner");
+          LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
 
+        }
+        InternalScanner scanner = region.getScanner(scan);
+        List<KeyValue> results = new ArrayList<KeyValue>();
+        while (scanner.next(results))
+          ;
+        assertEquals(kvSet.size(), results.size());
+        assertTrue(results.containsAll(kvSet));
       }
-      InternalScanner scanner = region.getScanner(scan);
-      List<KeyValue> results = new ArrayList<KeyValue>();
-      while (scanner.next(results))
-        ;
-      assertEquals(kvSet.size(), results.size());
-      assertTrue(results.containsAll(kvSet));
+    } finally {
+      HRegion.closeHRegion(region);
     }
 
     region.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
index 80605e755f21..451ada65d489 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
@@ -86,9 +86,10 @@ public void setUp() throws Exception {
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
 
     HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
-    HRegion.createHRegion(info, basedir, conf, htd);
+    HRegion region = HRegion.createHRegion(info, basedir, conf, htd);
+    HRegion.closeHRegion(region);
     Path tableDir = new Path(basedir, Bytes.toString(htd.getName()));
-    HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
+    region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
 
     store = new Store(basedir, region, hcd, fs, conf);
     TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir());
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index c0ac12ca5e36..67b7e523ebf9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -100,13 +100,15 @@
  */
 @Category(MediumTests.class)
 public class TestHRegion extends HBaseTestCase {
+  // Do not spin up clusters in here.  If you need to spin up a cluster, do it
+  // over in TestHRegionOnCluster.
   static final Log LOG = LogFactory.getLog(TestHRegion.class);
 
   private static final String COLUMN_FAMILY = "MyCF";
 
   HRegion region = null;
-  private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
 
   private final int MAX_VERSIONS = 2;
 
@@ -139,95 +141,6 @@ protected void tearDown() throws Exception {
     SchemaMetrics.validateMetricChanges(startingMetrics);
   }
 
-  public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
-    final int NUM_MASTERS = 1;
-    final int NUM_RS = 3;
-    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
-
-    try {
-      final byte[] TABLENAME = Bytes
-          .toBytes("testDataCorrectnessReplayingRecoveredEdits");
-      final byte[] FAMILY = Bytes.toBytes("family");
-      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-      HMaster master = cluster.getMaster();
-
-      // Create table
-      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
-      desc.addFamily(new HColumnDescriptor(FAMILY));
-      HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin();
-      hbaseAdmin.createTable(desc);
-
-      assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
-
-      // Put data: r1->v1
-      HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
-      putDataAndVerify(table, "r1", FAMILY, "v1", 1);
-
-      // Move region to target server
-      HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo();
-      int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
-      HRegionServer originServer = cluster.getRegionServer(originServerNum);
-      int targetServerNum = NUM_RS - 1 - originServerNum;
-      HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
-      hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
-          Bytes.toBytes(targetServer.getServerName().getServerName()));
-      do {
-        Thread.sleep(1);
-      } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
-
-      // Put data: r2->v2
-      putDataAndVerify(table, "r2", FAMILY, "v2", 2);
-
-      // Move region to origin server
-      hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
-          Bytes.toBytes(originServer.getServerName().getServerName()));
-      do {
-        Thread.sleep(1);
-      } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);
-
-      // Put data: r3->v3
-      putDataAndVerify(table, "r3", FAMILY, "v3", 3);
-
-      // Kill target server
-      targetServer.kill();
-      cluster.getRegionServerThreads().get(targetServerNum).join();
-      // Wait until finish processing of shutdown
-      while (master.getServerManager().areDeadServersInProgress()) {
-        Thread.sleep(5);
-      }
-      // Kill origin server
-      originServer.kill();
-      cluster.getRegionServerThreads().get(originServerNum).join();
-
-      // Put data: r4->v4
-      putDataAndVerify(table, "r4", FAMILY, "v4", 4);
-
-    } finally {
-      TEST_UTIL.shutdownMiniCluster();
-    }
-  }
-
-  private void putDataAndVerify(HTable table, String row, byte[] family,
-      String value, int verifyNum) throws IOException {
-    System.out.println("=========Putting data :" + row);
-    Put put = new Put(Bytes.toBytes(row));
-    put.add(family, Bytes.toBytes("q1"), Bytes.toBytes(value));
-    table.put(put);
-    ResultScanner resultScanner = table.getScanner(new Scan());
-    List<Result> results = new ArrayList<Result>();
-    while (true) {
-      Result r = resultScanner.next();
-      if (r == null)
-        break;
-      results.add(r);
-    }
-    resultScanner.close();
-    if (results.size() != verifyNum) {
-      System.out.println(results);
-    }
-    assertEquals(verifyNum, results.size());
-  }
-
   //////////////////////////////////////////////////////////////////////////////
   // New tests that doesn't spin up a mini cluster but rather just test the
   // individual code pieces in the HRegion. Putting files locally in
@@ -240,39 +153,44 @@ public void testSkipRecoveredEditsReplay() throws Exception {
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
     Configuration conf = HBaseConfiguration.create();
-    initHRegion(tableName, method, conf, family);
-    Path regiondir = region.getRegionDir();
-    FileSystem fs = region.getFilesystem();
-    byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
+    this.region = initHRegion(tableName, method, conf, family);
+    try {
+      Path regiondir = region.getRegionDir();
+      FileSystem fs = region.getFilesystem();
+      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
 
-    Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
+      Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
 
-    long maxSeqId = 1050;
-    long minSeqId = 1000;
+      long maxSeqId = 1050;
+      long minSeqId = 1000;
 
-    for (long i = minSeqId; i <= maxSeqId; i += 10) {
-      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
-      fs.create(recoveredEdits);
-      HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf);
+      for (long i = minSeqId; i <= maxSeqId; i += 10) {
+        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
+        fs.create(recoveredEdits);
+        HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf);
 
-      long time = System.nanoTime();
-      WALEdit edit = new WALEdit();
-      edit.add(new KeyValue(row, family, Bytes.toBytes(i),
-          time, KeyValue.Type.Put, Bytes.toBytes(i)));
-      writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
-          i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
+        long time = System.nanoTime();
+        WALEdit edit = new WALEdit();
+        edit.add(new KeyValue(row, family, Bytes.toBytes(i),
+            time, KeyValue.Type.Put, Bytes.toBytes(i)));
+        writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
+            i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
 
-      writer.close();
-    }
-    MonitoredTask status = TaskMonitor.get().createStatus(method);
-    long seqId = region.replayRecoveredEditsIfAny(regiondir, minSeqId-1, null, status);
-    assertEquals(maxSeqId, seqId);
-    Get get = new Get(row);
-    Result result = region.get(get, null);
-    for (long i = minSeqId; i <= maxSeqId; i += 10) {
-      List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
-      assertEquals(1, kvs.size());
-      assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
+        writer.close();
+      }
+      MonitoredTask status = TaskMonitor.get().createStatus(method);
+      long seqId = region.replayRecoveredEditsIfAny(regiondir, minSeqId-1, null, status);
+      assertEquals(maxSeqId, seqId);
+      Get get = new Get(row);
+      Result result = region.get(get, null);
+      for (long i = minSeqId; i <= maxSeqId; i += 10) {
+        List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
+        assertEquals(1, kvs.size());
+        assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -280,44 +198,49 @@ public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
     String method = "testSkipRecoveredEditsReplaySomeIgnored";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    initHRegion(tableName, method, HBaseConfiguration.create(), family);
-    Path regiondir = region.getRegionDir();
-    FileSystem fs = region.getFilesystem();
-    byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
-
-    Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
-
-    long maxSeqId = 1050;
-    long minSeqId = 1000;
-
-    for (long i = minSeqId; i <= maxSeqId; i += 10) {
-      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
-      fs.create(recoveredEdits);
-      HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf);
-
-      long time = System.nanoTime();
-      WALEdit edit = new WALEdit();
-      edit.add(new KeyValue(row, family, Bytes.toBytes(i),
-          time, KeyValue.Type.Put, Bytes.toBytes(i)));
-      writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
-          i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
-
-      writer.close();
-    }
-    long recoverSeqId = 1030;
-    MonitoredTask status = TaskMonitor.get().createStatus(method);
-    long seqId = region.replayRecoveredEditsIfAny(regiondir, recoverSeqId-1, null, status);
-    assertEquals(maxSeqId, seqId);
-    Get get = new Get(row);
-    Result result = region.get(get, null);
-    for (long i = minSeqId; i <= maxSeqId; i += 10) {
-      List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
-      if (i < recoverSeqId) {
-        assertEquals(0, kvs.size());
-      } else {
-        assertEquals(1, kvs.size());
-        assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
+    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    try {
+      Path regiondir = region.getRegionDir();
+      FileSystem fs = region.getFilesystem();
+      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
+
+      Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
+
+      long maxSeqId = 1050;
+      long minSeqId = 1000;
+
+      for (long i = minSeqId; i <= maxSeqId; i += 10) {
+        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
+        fs.create(recoveredEdits);
+        HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf);
+
+        long time = System.nanoTime();
+        WALEdit edit = new WALEdit();
+        edit.add(new KeyValue(row, family, Bytes.toBytes(i),
+            time, KeyValue.Type.Put, Bytes.toBytes(i)));
+        writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
+            i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
+
+        writer.close();
       }
+      long recoverSeqId = 1030;
+      MonitoredTask status = TaskMonitor.get().createStatus(method);
+      long seqId = region.replayRecoveredEditsIfAny(regiondir, recoverSeqId-1, null, status);
+      assertEquals(maxSeqId, seqId);
+      Get get = new Get(row);
+      Result result = region.get(get, null);
+      for (long i = minSeqId; i <= maxSeqId; i += 10) {
+        List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
+        if (i < recoverSeqId) {
+          assertEquals(0, kvs.size());
+        } else {
+          assertEquals(1, kvs.size());
+          assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
+        }
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -325,25 +248,30 @@ public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
     String method = "testSkipRecoveredEditsReplayAllIgnored";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    initHRegion(tableName, method, HBaseConfiguration.create(), family);
-    Path regiondir = region.getRegionDir();
-    FileSystem fs = region.getFilesystem();
-
-    Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
-    for (int i = 1000; i < 1050; i += 10) {
+    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    try {
+      Path regiondir = region.getRegionDir();
+      FileSystem fs = region.getFilesystem();
+
+      Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
+      for (int i = 1000; i < 1050; i += 10) {
+        Path recoveredEdits = new Path(
+            recoveredEditsDir, String.format("%019d", i));
+        FSDataOutputStream dos=  fs.create(recoveredEdits);
+        dos.writeInt(i);
+        dos.close();
+      }
+      long minSeqId = 2000;
       Path recoveredEdits = new Path(
-          recoveredEditsDir, String.format("%019d", i));
+          recoveredEditsDir, String.format("%019d", minSeqId-1));
       FSDataOutputStream dos=  fs.create(recoveredEdits);
-      dos.writeInt(i);
       dos.close();
+      long seqId = region.replayRecoveredEditsIfAny(regiondir, minSeqId, null, null);
+      assertEquals(minSeqId, seqId);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    long minSeqId = 2000;
-    Path recoveredEdits = new Path(
-        recoveredEditsDir, String.format("%019d", minSeqId-1));
-    FSDataOutputStream dos=  fs.create(recoveredEdits);
-    dos.close();
-    long seqId = region.replayRecoveredEditsIfAny(regiondir, minSeqId, null, null);
-    assertEquals(minSeqId, seqId);
   }
 
   public void testGetWhileRegionClose() throws IOException {
@@ -353,52 +281,56 @@ public void testGetWhileRegionClose() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, hc, families);
-
-    // Put data in region
-    final int startRow = 100;
-    putData(startRow, numRows, qual1, families);
-    putData(startRow, numRows, qual2, families);
-    putData(startRow, numRows, qual3, families);
-    // this.region.flushcache();
-    final AtomicBoolean done = new AtomicBoolean(false);
-    final AtomicInteger gets = new AtomicInteger(0);
-    GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
-    try {
-      // Set ten threads running concurrently getting from the region.
-      for (int i = 0; i < threads.length / 2; i++) {
-        threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
-          done, gets);
-        threads[i].setDaemon(true);
-        threads[i].start();
-      }
-      // Artificially make the condition by setting closing flag explicitly.
-      // I can't make the issue happen with a call to region.close().
-      this.region.closing.set(true);
-      for (int i = threads.length / 2; i < threads.length; i++) {
-        threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
-          done, gets);
-        threads[i].setDaemon(true);
-        threads[i].start();
-      }
-    } finally {
-      if (this.region != null) {
-        this.region.close();
-        this.region.getLog().closeAndDelete();
-      }
-    }
-    done.set(true);
-    for (GetTillDoneOrException t: threads) {
+    this.region = initHRegion(tableName, method, hc, families);
+    try {
+      // Put data in region
+      final int startRow = 100;
+      putData(startRow, numRows, qual1, families);
+      putData(startRow, numRows, qual2, families);
+      putData(startRow, numRows, qual3, families);
+      // this.region.flushcache();
+      final AtomicBoolean done = new AtomicBoolean(false);
+      final AtomicInteger gets = new AtomicInteger(0);
+      GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
       try {
-        t.join();
-      } catch (InterruptedException e) {
-        e.printStackTrace();
+        // Set ten threads running concurrently getting from the region.
+        for (int i = 0; i < threads.length / 2; i++) {
+          threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
+              done, gets);
+          threads[i].setDaemon(true);
+          threads[i].start();
+        }
+        // Artificially make the condition by setting closing flag explicitly.
+        // I can't make the issue happen with a call to region.close().
+        this.region.closing.set(true);
+        for (int i = threads.length / 2; i < threads.length; i++) {
+          threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
+              done, gets);
+          threads[i].setDaemon(true);
+          threads[i].start();
+        }
+      } finally {
+        if (this.region != null) {
+          this.region.close();
+          this.region.getLog().closeAndDelete();
+        }
       }
-      if (t.e != null) {
-        LOG.info("Exception=" + t.e);
-        assertFalse("Found a NPE in " + t.getName(),
-          t.e instanceof NullPointerException);
+      done.set(true);
+      for (GetTillDoneOrException t: threads) {
+        try {
+          t.join();
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        if (t.e != null) {
+          LOG.info("Exception=" + t.e);
+          assertFalse("Found a NPE in " + t.getName(),
+              t.e instanceof NullPointerException);
+        }
       }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -442,38 +374,43 @@ public void testWeirdCacheBehaviour() throws Exception {
     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
-    initHRegion(TABLE, getName(), FAMILIES);
-    String value = "this is the value";
-    String value2 = "this is some other value";
-    String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();
-    String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();
-    String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();
-    putRows(this.region, 3, value, keyPrefix1);
-    putRows(this.region, 3, value, keyPrefix2);
-    putRows(this.region, 3, value, keyPrefix3);
-    // this.region.flushCommits();
-    putRows(this.region, 3, value2, keyPrefix1);
-    putRows(this.region, 3, value2, keyPrefix2);
-    putRows(this.region, 3, value2, keyPrefix3);
-    System.out.println("Checking values for key: " + keyPrefix1);
-    assertEquals("Got back incorrect number of rows from scan", 3,
-      getNumberOfRows(keyPrefix1, value2, this.region));
-    System.out.println("Checking values for key: " + keyPrefix2);
-    assertEquals("Got back incorrect number of rows from scan", 3,
-      getNumberOfRows(keyPrefix2, value2, this.region));
-    System.out.println("Checking values for key: " + keyPrefix3);
-    assertEquals("Got back incorrect number of rows from scan", 3,
-      getNumberOfRows(keyPrefix3, value2, this.region));
-    deleteColumns(this.region, value2, keyPrefix1);
-    deleteColumns(this.region, value2, keyPrefix2);
-    deleteColumns(this.region, value2, keyPrefix3);
-    System.out.println("Starting important checks.....");
-    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
-      0, getNumberOfRows(keyPrefix1, value2, this.region));
-    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
-      0, getNumberOfRows(keyPrefix2, value2, this.region));
-    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
-      0, getNumberOfRows(keyPrefix3, value2, this.region));
+    this.region = initHRegion(TABLE, getName(), FAMILIES);
+    try {
+      String value = "this is the value";
+      String value2 = "this is some other value";
+      String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();
+      String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();
+      String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();
+      putRows(this.region, 3, value, keyPrefix1);
+      putRows(this.region, 3, value, keyPrefix2);
+      putRows(this.region, 3, value, keyPrefix3);
+      // this.region.flushCommits();
+      putRows(this.region, 3, value2, keyPrefix1);
+      putRows(this.region, 3, value2, keyPrefix2);
+      putRows(this.region, 3, value2, keyPrefix3);
+      System.out.println("Checking values for key: " + keyPrefix1);
+      assertEquals("Got back incorrect number of rows from scan", 3,
+          getNumberOfRows(keyPrefix1, value2, this.region));
+      System.out.println("Checking values for key: " + keyPrefix2);
+      assertEquals("Got back incorrect number of rows from scan", 3,
+          getNumberOfRows(keyPrefix2, value2, this.region));
+      System.out.println("Checking values for key: " + keyPrefix3);
+      assertEquals("Got back incorrect number of rows from scan", 3,
+          getNumberOfRows(keyPrefix3, value2, this.region));
+      deleteColumns(this.region, value2, keyPrefix1);
+      deleteColumns(this.region, value2, keyPrefix2);
+      deleteColumns(this.region, value2, keyPrefix3);
+      System.out.println("Starting important checks.....");
+      assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
+          0, getNumberOfRows(keyPrefix1, value2, this.region));
+      assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
+          0, getNumberOfRows(keyPrefix2, value2, this.region));
+      assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
+          0, getNumberOfRows(keyPrefix3, value2, this.region));
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   private void deleteColumns(HRegion r, String value, String keyPrefix)
@@ -558,17 +495,22 @@ private void putRows(HRegion r, int numRows, String value, String key)
   public void testFamilyWithAndWithoutColon() throws Exception {
     byte [] b = Bytes.toBytes(getName());
     byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
-    initHRegion(b, getName(), cf);
-    Put p = new Put(b);
-    byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
-    p.add(cfwithcolon, cfwithcolon, cfwithcolon);
-    boolean exception = false;
+    this.region = initHRegion(b, getName(), cf);
     try {
-      this.region.put(p);
-    } catch (DoNotRetryIOException e) {
-      exception = true;
+      Put p = new Put(b);
+      byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
+      p.add(cfwithcolon, cfwithcolon, cfwithcolon);
+      boolean exception = false;
+      try {
+        this.region.put(p);
+      } catch (DoNotRetryIOException e) {
+        exception = true;
+      }
+      assertTrue(exception);
+    } finally {
+       HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    assertTrue(exception);
   }
 
   @SuppressWarnings("unchecked")
@@ -577,96 +519,100 @@ public void testBatchPut() throws Exception {
     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
     byte[] qual = Bytes.toBytes("qual");
     byte[] val = Bytes.toBytes("val");
-    initHRegion(b, getName(), cf);
-
-    HLog.getSyncTime(); // clear counter from prior tests
-    assertEquals(0, HLog.getSyncTime().count);
-
-    LOG.info("First a batch put with all valid puts");
-    final Put[] puts = new Put[10];
-    for (int i = 0; i < 10; i++) {
-      puts[i] = new Put(Bytes.toBytes("row_" + i));
-      puts[i].add(cf, qual, val);
-    }
-
-    OperationStatus[] codes = this.region.put(puts);
-    assertEquals(10, codes.length);
-    for (int i = 0; i < 10; i++) {
-      assertEquals(OperationStatusCode.SUCCESS, codes[i]
-          .getOperationStatusCode());
-    }
-    assertEquals(1, HLog.getSyncTime().count);
-
-    LOG.info("Next a batch put with one invalid family");
-    puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
-    codes = this.region.put(puts);
-    assertEquals(10, codes.length);
-    for (int i = 0; i < 10; i++) {
-      assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
-        OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
-    }
-    assertEquals(1, HLog.getSyncTime().count);
-
-    LOG.info("Next a batch put that has to break into two batches to avoid a lock");
-    Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
-
-    MultithreadedTestUtil.TestContext ctx =
-      new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
-    final AtomicReference<OperationStatus[]> retFromThread =
-      new AtomicReference<OperationStatus[]>();
-    TestThread putter = new TestThread(ctx) {
-      @Override
-      public void doWork() throws IOException {
-        retFromThread.set(region.put(puts));
-      }
-    };
-    LOG.info("...starting put thread while holding lock");
-    ctx.addThread(putter);
-    ctx.startThreads();
-
-    LOG.info("...waiting for put thread to sync first time");
-    long startWait = System.currentTimeMillis();
-    while (HLog.getSyncTime().count == 0) {
-      Thread.sleep(100);
-      if (System.currentTimeMillis() - startWait > 10000) {
-        fail("Timed out waiting for thread to sync first minibatch");
-      }
-    }
-    LOG.info("...releasing row lock, which should let put thread continue");
-    region.releaseRowLock(lockedRow);
-    LOG.info("...joining on thread");
-    ctx.stop();
-    LOG.info("...checking that next batch was synced");
-    assertEquals(1, HLog.getSyncTime().count);
-    codes = retFromThread.get();
-    for (int i = 0; i < 10; i++) {
-      assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
-        OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
-    }
-
-    LOG.info("Nexta, a batch put which uses an already-held lock");
-    lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
-    LOG.info("...obtained row lock");
-    List<Pair<Put, Integer>> putsAndLocks = Lists.newArrayList();
-    for (int i = 0; i < 10; i++) {
-      Pair<Put, Integer> pair = new Pair<Put, Integer>(puts[i], null);
-      if (i == 2) pair.setSecond(lockedRow);
-      putsAndLocks.add(pair);
-    }
-
-    codes = region.put(putsAndLocks.toArray(new Pair[0]));
-    LOG.info("...performed put");
-    for (int i = 0; i < 10; i++) {
-      assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
-        OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
-    }
-    // Make sure we didn't do an extra batch
-    assertEquals(1, HLog.getSyncTime().count);
-
-    // Make sure we still hold lock
-    assertTrue(region.isRowLocked(lockedRow));
-    LOG.info("...releasing lock");
-    region.releaseRowLock(lockedRow);
+    this.region = initHRegion(b, getName(), cf);
+    try {
+      HLog.getSyncTime(); // clear counter from prior tests
+      assertEquals(0, HLog.getSyncTime().count);
+
+      LOG.info("First a batch put with all valid puts");
+      final Put[] puts = new Put[10];
+      for (int i = 0; i < 10; i++) {
+        puts[i] = new Put(Bytes.toBytes("row_" + i));
+        puts[i].add(cf, qual, val);
+      }
+
+      OperationStatus[] codes = this.region.put(puts);
+      assertEquals(10, codes.length);
+      for (int i = 0; i < 10; i++) {
+        assertEquals(OperationStatusCode.SUCCESS, codes[i]
+            .getOperationStatusCode());
+      }
+      assertEquals(1, HLog.getSyncTime().count);
+
+      LOG.info("Next a batch put with one invalid family");
+      puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
+      codes = this.region.put(puts);
+      assertEquals(10, codes.length);
+      for (int i = 0; i < 10; i++) {
+        assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
+          OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
+      }
+      assertEquals(1, HLog.getSyncTime().count);
+
+      LOG.info("Next a batch put that has to break into two batches to avoid a lock");
+      Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
+
+      MultithreadedTestUtil.TestContext ctx =
+        new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+      final AtomicReference<OperationStatus[]> retFromThread =
+        new AtomicReference<OperationStatus[]>();
+      TestThread putter = new TestThread(ctx) {
+        @Override
+        public void doWork() throws IOException {
+          retFromThread.set(region.put(puts));
+        }
+      };
+      LOG.info("...starting put thread while holding lock");
+      ctx.addThread(putter);
+      ctx.startThreads();
+  
+      LOG.info("...waiting for put thread to sync first time");
+      long startWait = System.currentTimeMillis();
+      while (HLog.getSyncTime().count == 0) {
+        Thread.sleep(100);
+        if (System.currentTimeMillis() - startWait > 10000) {
+          fail("Timed out waiting for thread to sync first minibatch");
+        }
+      }
+      LOG.info("...releasing row lock, which should let put thread continue");
+      region.releaseRowLock(lockedRow);
+      LOG.info("...joining on thread");
+      ctx.stop();
+      LOG.info("...checking that next batch was synced");
+      assertEquals(1, HLog.getSyncTime().count);
+      codes = retFromThread.get();
+      for (int i = 0; i < 10; i++) {
+        assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
+          OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
+      }
+  
+      LOG.info("Nexta, a batch put which uses an already-held lock");
+      lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
+      LOG.info("...obtained row lock");
+      List<Pair<Put, Integer>> putsAndLocks = Lists.newArrayList();
+      for (int i = 0; i < 10; i++) {
+        Pair<Put, Integer> pair = new Pair<Put, Integer>(puts[i], null);
+        if (i == 2) pair.setSecond(lockedRow);
+        putsAndLocks.add(pair);
+      }
+  
+      codes = region.put(putsAndLocks.toArray(new Pair[0]));
+      LOG.info("...performed put");
+      for (int i = 0; i < 10; i++) {
+        assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
+          OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
+      }
+      // Make sure we didn't do an extra batch
+      assertEquals(1, HLog.getSyncTime().count);
+  
+      // Make sure we still hold lock
+      assertTrue(region.isRowLocked(lockedRow));
+      LOG.info("...releasing lock");
+      region.releaseRowLock(lockedRow);
+    } finally {
+      HRegion.closeHRegion(this.region);
+       this.region = null;
+    }
   }
 
   //////////////////////////////////////////////////////////////////////////////
@@ -684,65 +630,68 @@ public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    //Putting empty data in key
-    Put put = new Put(row1);
-    put.add(fam1, qf1, emptyVal);
-
-    //checkAndPut with empty value
-    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(emptyVal), put, lockId, true);
-    assertTrue(res);
-    
-    //Putting data in key
-    put = new Put(row1);
-    put.add(fam1, qf1, val1);
-
-    //checkAndPut with correct value
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(emptyVal), put, lockId, true);
-    assertTrue(res);
-
-    // not empty anymore
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(emptyVal), put, lockId, true);
-    assertFalse(res);
-
-    Delete delete = new Delete(row1);
-    delete.deleteColumn(fam1, qf1);
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(emptyVal), delete, lockId, true);
-    assertFalse(res);
-
-    put = new Put(row1);
-    put.add(fam1, qf1, val2);
-    //checkAndPut with correct value
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(val1), put, lockId, true);
-    assertTrue(res);
-
-    //checkAndDelete with correct value
-    delete = new Delete(row1);
-    delete.deleteColumn(fam1, qf1);
-    delete.deleteColumn(fam1, qf1);
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(val2), delete, lockId, true);
-    assertTrue(res);
-
-    delete = new Delete(row1);
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new BinaryComparator(emptyVal), delete, lockId, true);
-    assertTrue(res);
-
-    //checkAndPut looking for a null value
-    put = new Put(row1);
-    put.add(fam1, qf1, val1);
-
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-        new NullComparator(), put, lockId, true);
-    assertTrue(res);
-
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Putting empty data in key
+      Put put = new Put(row1);
+      put.add(fam1, qf1, emptyVal);
+
+      //checkAndPut with empty value
+      boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(emptyVal), put, lockId, true);
+      assertTrue(res);
+
+      //Putting data in key
+      put = new Put(row1);
+      put.add(fam1, qf1, val1);
+
+      //checkAndPut with correct value
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(emptyVal), put, lockId, true);
+      assertTrue(res);
+
+      // not empty anymore
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(emptyVal), put, lockId, true);
+      assertFalse(res);
+
+      Delete delete = new Delete(row1);
+      delete.deleteColumn(fam1, qf1);
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(emptyVal), delete, lockId, true);
+      assertFalse(res);
+
+      put = new Put(row1);
+      put.add(fam1, qf1, val2);
+      //checkAndPut with correct value
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val1), put, lockId, true);
+      assertTrue(res);
+
+      //checkAndDelete with correct value
+      delete = new Delete(row1);
+      delete.deleteColumn(fam1, qf1);
+      delete.deleteColumn(fam1, qf1);
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val2), delete, lockId, true);
+      assertTrue(res);
+
+      delete = new Delete(row1);
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(emptyVal), delete, lockId, true);
+      assertTrue(res);
+
+      //checkAndPut looking for a null value
+      put = new Put(row1);
+      put.add(fam1, qf1, val1);
+
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new NullComparator(), put, lockId, true);
+      assertTrue(res);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testCheckAndMutate_WithWrongValue() throws IOException{
@@ -756,24 +705,28 @@ public void testCheckAndMutate_WithWrongValue() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    //Putting data in key
-    Put put = new Put(row1);
-    put.add(fam1, qf1, val1);
-    region.put(put);
-
-    //checkAndPut with wrong value
-    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val2), put, lockId, true);
-    assertEquals(false, res);
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Putting data in key
+      Put put = new Put(row1);
+      put.add(fam1, qf1, val1);
+      region.put(put);
 
-    //checkAndDelete with wrong value
-    Delete delete = new Delete(row1);
-    delete.deleteFamily(fam1);
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val2), delete, lockId, true);
-    assertEquals(false, res);
+      //checkAndPut with wrong value
+      boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val2), put, lockId, true);
+      assertEquals(false, res);
+
+      //checkAndDelete with wrong value
+      Delete delete = new Delete(row1);
+      delete.deleteFamily(fam1);
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val2), delete, lockId, true);
+      assertEquals(false, res);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testCheckAndMutate_WithCorrectValue() throws IOException{
@@ -786,24 +739,28 @@ public void testCheckAndMutate_WithCorrectValue() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    //Putting data in key
-    Put put = new Put(row1);
-    put.add(fam1, qf1, val1);
-    region.put(put);
-
-    //checkAndPut with correct value
-    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val1), put, lockId, true);
-    assertEquals(true, res);
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Putting data in key
+      Put put = new Put(row1);
+      put.add(fam1, qf1, val1);
+      region.put(put);
 
-    //checkAndDelete with correct value
-    Delete delete = new Delete(row1);
-    delete.deleteColumn(fam1, qf1);
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val1), put, lockId, true);
-    assertEquals(true, res);
+      //checkAndPut with correct value
+      boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val1), put, lockId, true);
+      assertEquals(true, res);
+
+      //checkAndDelete with correct value
+      Delete delete = new Delete(row1);
+      delete.deleteColumn(fam1, qf1);
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val1), put, lockId, true);
+      assertEquals(true, res);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testCheckAndPut_ThatPutWasWritten() throws IOException{
@@ -820,52 +777,59 @@ public void testCheckAndPut_ThatPutWasWritten() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    //Putting data in the key to check
-    Put put = new Put(row1);
-    put.add(fam1, qf1, val1);
-    region.put(put);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Putting data in the key to check
+      Put put = new Put(row1);
+      put.add(fam1, qf1, val1);
+      region.put(put);
 
-    //Creating put to add
-    long ts = System.currentTimeMillis();
-    KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
-    put = new Put(row1);
-    put.add(kv);
+      //Creating put to add
+      long ts = System.currentTimeMillis();
+      KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
+      put = new Put(row1);
+      put.add(kv);
 
-    //checkAndPut with wrong value
-    Store store = region.getStore(fam1);
-    store.memstore.kvset.size();
+      //checkAndPut with wrong value
+      Store store = region.getStore(fam1);
+      store.memstore.kvset.size();
 
-    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val1), put, lockId, true);
-    assertEquals(true, res);
-    store.memstore.kvset.size();
+      boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val1), put, lockId, true);
+      assertEquals(true, res);
+      store.memstore.kvset.size();
 
-    Get get = new Get(row1);
-    get.addColumn(fam2, qf1);
-    KeyValue [] actual = region.get(get, null).raw();
+      Get get = new Get(row1);
+      get.addColumn(fam2, qf1);
+      KeyValue [] actual = region.get(get, null).raw();
 
-    KeyValue [] expected = {kv};
+      KeyValue [] expected = {kv};
 
-    assertEquals(expected.length, actual.length);
-    for(int i=0; i<actual.length; i++) {
-      assertEquals(expected[i], actual[i]);
+      assertEquals(expected.length, actual.length);
+      for(int i=0; i<actual.length; i++) {
+        assertEquals(expected[i], actual[i]);
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-
   }
 
   public void testCheckAndPut_wrongRowInPut() throws IOException {
-    initHRegion(tableName, this.getName(), COLUMNS);
-
-    Put put = new Put(row2);
-    put.add(fam1, qual1, value1);
+    this.region = initHRegion(tableName, this.getName(), COLUMNS);
     try {
-    boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
-        new BinaryComparator(value2), put, null, false);
-      fail();
-    } catch (DoNotRetryIOException expected) {
-      // expected exception.
+      Put put = new Put(row2);
+      put.add(fam1, qual1, value1);
+      try {
+        boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
+            new BinaryComparator(value2), put, null, false);
+        fail();
+      } catch (DoNotRetryIOException expected) {
+        // expected exception.
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -887,60 +851,64 @@ public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Put content
+      Put put = new Put(row1);
+      put.add(fam1, qf1, val1);
+      region.put(put);
+      Threads.sleep(2);
+
+      put = new Put(row1);
+      put.add(fam1, qf1, val2);
+      put.add(fam2, qf1, val3);
+      put.add(fam2, qf2, val2);
+      put.add(fam2, qf3, val1);
+      put.add(fam1, qf3, val1);
+      region.put(put);
 
-    //Put content
-    Put put = new Put(row1);
-    put.add(fam1, qf1, val1);
-    region.put(put);
-    Threads.sleep(2);
-
-    put = new Put(row1);
-    put.add(fam1, qf1, val2);
-    put.add(fam2, qf1, val3);
-    put.add(fam2, qf2, val2);
-    put.add(fam2, qf3, val1);
-    put.add(fam1, qf3, val1);
-    region.put(put);
-
-    //Multi-column delete
-    Delete delete = new Delete(row1);
-    delete.deleteColumn(fam1, qf1);
-    delete.deleteColumn(fam2, qf1);
-    delete.deleteColumn(fam1, qf3);
-    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val2), delete, lockId, true);
-    assertEquals(true, res);
-
-    Get get = new Get(row1);
-    get.addColumn(fam1, qf1);
-    get.addColumn(fam1, qf3);
-    get.addColumn(fam2, qf2);
-    Result r = region.get(get, null);
-    assertEquals(2, r.size());
-    assertEquals(val1, r.getValue(fam1, qf1));
-    assertEquals(val2, r.getValue(fam2, qf2));
-
-    //Family delete
-    delete = new Delete(row1);
-    delete.deleteFamily(fam2);
-    res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL,
-      new BinaryComparator(emptyVal), delete, lockId, true);
-    assertEquals(true, res);
-
-    get = new Get(row1);
-    r = region.get(get, null);
-    assertEquals(1, r.size());
-    assertEquals(val1, r.getValue(fam1, qf1));
-
-    //Row delete
-    delete = new Delete(row1);
-    res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
-      new BinaryComparator(val1), delete, lockId, true);
-    assertEquals(true, res);
-    get = new Get(row1);
-    r = region.get(get, null);
-    assertEquals(0, r.size());
+      //Multi-column delete
+      Delete delete = new Delete(row1);
+      delete.deleteColumn(fam1, qf1);
+      delete.deleteColumn(fam2, qf1);
+      delete.deleteColumn(fam1, qf3);
+      boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val2), delete, lockId, true);
+      assertEquals(true, res);
+
+      Get get = new Get(row1);
+      get.addColumn(fam1, qf1);
+      get.addColumn(fam1, qf3);
+      get.addColumn(fam2, qf2);
+      Result r = region.get(get, null);
+      assertEquals(2, r.size());
+      assertEquals(val1, r.getValue(fam1, qf1));
+      assertEquals(val2, r.getValue(fam2, qf2));
+
+      //Family delete
+      delete = new Delete(row1);
+      delete.deleteFamily(fam2);
+      res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL,
+          new BinaryComparator(emptyVal), delete, lockId, true);
+      assertEquals(true, res);
+
+      get = new Get(row1);
+      r = region.get(get, null);
+      assertEquals(1, r.size());
+      assertEquals(val1, r.getValue(fam1, qf1));
+
+      //Row delete
+      delete = new Delete(row1);
+      res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
+          new BinaryComparator(val1), delete, lockId, true);
+      assertEquals(true, res);
+      get = new Get(row1);
+      r = region.get(get, null);
+      assertEquals(0, r.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   //////////////////////////////////////////////////////////////////////////////
@@ -958,20 +926,24 @@ public void testDelete_multiDeleteColumn() throws IOException {
     put.add(fam1, qual, 2, value);
 
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    region.put(put);
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      region.put(put);
 
-    // We do support deleting more than 1 'latest' version
-    Delete delete = new Delete(row1);
-    delete.deleteColumn(fam1, qual);
-    delete.deleteColumn(fam1, qual);
-    region.delete(delete, null, false);
+      // We do support deleting more than 1 'latest' version
+      Delete delete = new Delete(row1);
+      delete.deleteColumn(fam1, qual);
+      delete.deleteColumn(fam1, qual);
+      region.delete(delete, null, false);
 
-    Get get = new Get(row1);
-    get.addFamily(fam1);
-    Result r = region.get(get, null);
-    assertEquals(0, r.size());
+      Get get = new Get(row1);
+      get.addFamily(fam1);
+      Result r = region.get(get, null);
+      assertEquals(0, r.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testDelete_CheckFamily() throws IOException {
@@ -984,33 +956,37 @@ public void testDelete_CheckFamily() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1, fam2, fam3);
-
-    List<KeyValue> kvs  = new ArrayList<KeyValue>();
-    kvs.add(new KeyValue(row1, fam4, null, null));
+    this.region = initHRegion(tableName, method, fam1, fam2, fam3);
+    try {
+      List<KeyValue> kvs  = new ArrayList<KeyValue>();
+      kvs.add(new KeyValue(row1, fam4, null, null));
 
 
-    //testing existing family
-    byte [] family = fam2;
-    try {
-      Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
-      deleteMap.put(family, kvs);
-      region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
-    } catch (Exception e) {
-      assertTrue("Family " +new String(family)+ " does not exist", false);
-    }
+      //testing existing family
+      byte [] family = fam2;
+      try {
+        Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
+        deleteMap.put(family, kvs);
+        region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
+      } catch (Exception e) {
+        assertTrue("Family " +new String(family)+ " does not exist", false);
+      }
 
-    //testing non existing family
-    boolean ok = false;
-    family = fam4;
-    try {
-      Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
-      deleteMap.put(family, kvs);
-      region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
-    } catch (Exception e) {
-      ok = true;
+      //testing non existing family
+      boolean ok = false;
+      family = fam4;
+      try {
+        Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
+        deleteMap.put(family, kvs);
+        region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
+      } catch (Exception e) {
+        ok = true;
+      }
+      assertEquals("Family " +new String(family)+ " does exist", true, ok);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    assertEquals("Family " +new String(family)+ " does exist", true, ok);
   }
 
   public void testDelete_mixed() throws IOException, InterruptedException {
@@ -1018,62 +994,67 @@ public void testDelete_mixed() throws IOException, InterruptedException {
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    initHRegion(tableName, method, families);
-    EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
+    this.region = initHRegion(tableName, method, families);
+    try {
+      EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
 
-    byte [] row = Bytes.toBytes("table_name");
-    // column names
-    byte [] serverinfo = Bytes.toBytes("serverinfo");
-    byte [] splitA = Bytes.toBytes("splitA");
-    byte [] splitB = Bytes.toBytes("splitB");
+      byte [] row = Bytes.toBytes("table_name");
+      // column names
+      byte [] serverinfo = Bytes.toBytes("serverinfo");
+      byte [] splitA = Bytes.toBytes("splitA");
+      byte [] splitB = Bytes.toBytes("splitB");
 
-    // add some data:
-    Put put = new Put(row);
-    put.add(fam, splitA, Bytes.toBytes("reference_A"));
-    region.put(put);
+      // add some data:
+      Put put = new Put(row);
+      put.add(fam, splitA, Bytes.toBytes("reference_A"));
+      region.put(put);
 
-    put = new Put(row);
-    put.add(fam, splitB, Bytes.toBytes("reference_B"));
-    region.put(put);
+      put = new Put(row);
+      put.add(fam, splitB, Bytes.toBytes("reference_B"));
+      region.put(put);
 
-    put = new Put(row);
-    put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
-    region.put(put);
+      put = new Put(row);
+      put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
+      region.put(put);
 
-    // ok now delete a split:
-    Delete delete = new Delete(row);
-    delete.deleteColumns(fam, splitA);
-    region.delete(delete, null, true);
+      // ok now delete a split:
+      Delete delete = new Delete(row);
+      delete.deleteColumns(fam, splitA);
+      region.delete(delete, null, true);
 
-    // assert some things:
-    Get get = new Get(row).addColumn(fam, serverinfo);
-    Result result = region.get(get, null);
-    assertEquals(1, result.size());
+      // assert some things:
+      Get get = new Get(row).addColumn(fam, serverinfo);
+      Result result = region.get(get, null);
+      assertEquals(1, result.size());
 
-    get = new Get(row).addColumn(fam, splitA);
-    result = region.get(get, null);
-    assertEquals(0, result.size());
+      get = new Get(row).addColumn(fam, splitA);
+      result = region.get(get, null);
+      assertEquals(0, result.size());
 
-    get = new Get(row).addColumn(fam, splitB);
-    result = region.get(get, null);
-    assertEquals(1, result.size());
+      get = new Get(row).addColumn(fam, splitB);
+      result = region.get(get, null);
+      assertEquals(1, result.size());
 
-    // Assert that after a delete, I can put.
-    put = new Put(row);
-    put.add(fam, splitA, Bytes.toBytes("reference_A"));
-    region.put(put);
-    get = new Get(row);
-    result = region.get(get, null);
-    assertEquals(3, result.size());
-
-    // Now delete all... then test I can add stuff back
-    delete = new Delete(row);
-    region.delete(delete, null, false);
-    assertEquals(0, region.get(get, null).size());
-
-    region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
-    result = region.get(get, null);
-    assertEquals(1, result.size());
+      // Assert that after a delete, I can put.
+      put = new Put(row);
+      put.add(fam, splitA, Bytes.toBytes("reference_A"));
+      region.put(put);
+      get = new Get(row);
+      result = region.get(get, null);
+      assertEquals(3, result.size());
+
+      // Now delete all... then test I can add stuff back
+      delete = new Delete(row);
+      region.delete(delete, null, false);
+      assertEquals(0, region.get(get, null).size());
+
+      region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
+      result = region.get(get, null);
+      assertEquals(1, result.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testDeleteRowWithFutureTs() throws IOException {
@@ -1081,34 +1062,38 @@ public void testDeleteRowWithFutureTs() throws IOException {
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      byte [] row = Bytes.toBytes("table_name");
+      // column names
+      byte [] serverinfo = Bytes.toBytes("serverinfo");
 
-    byte [] row = Bytes.toBytes("table_name");
-    // column names
-    byte [] serverinfo = Bytes.toBytes("serverinfo");
+      // add data in the far future
+      Put put = new Put(row);
+      put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
+      region.put(put);
 
-    // add data in the far future
-    Put put = new Put(row);
-    put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
-    region.put(put);
+      // now delete something in the present
+      Delete delete = new Delete(row);
+      region.delete(delete, null, true);
 
-    // now delete something in the present
-    Delete delete = new Delete(row);
-    region.delete(delete, null, true);
+      // make sure we still see our data
+      Get get = new Get(row).addColumn(fam, serverinfo);
+      Result result = region.get(get, null);
+      assertEquals(1, result.size());
 
-    // make sure we still see our data
-    Get get = new Get(row).addColumn(fam, serverinfo);
-    Result result = region.get(get, null);
-    assertEquals(1, result.size());
+      // delete the future row
+      delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null);
+      region.delete(delete, null, true);
 
-    // delete the future row
-    delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null);
-    region.delete(delete, null, true);
-
-    // make sure it is gone
-    get = new Get(row).addColumn(fam, serverinfo);
-    result = region.get(get, null);
-    assertEquals(0, result.size());
+      // make sure it is gone
+      get = new Get(row).addColumn(fam, serverinfo);
+      result = region.get(get, null);
+      assertEquals(0, result.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   /**
@@ -1120,41 +1105,45 @@ public void testPutWithLatestTS() throws IOException {
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    byte [] row = Bytes.toBytes("row1");
-    // column names
-    byte [] qual = Bytes.toBytes("qual");
-
-    // add data with LATEST_TIMESTAMP, put without WAL
-    Put put = new Put(row);
-    put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
-    region.put(put, false);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      byte [] row = Bytes.toBytes("row1");
+      // column names
+      byte [] qual = Bytes.toBytes("qual");
 
-    // Make sure it shows up with an actual timestamp
-    Get get = new Get(row).addColumn(fam, qual);
-    Result result = region.get(get, null);
-    assertEquals(1, result.size());
-    KeyValue kv = result.raw()[0];
-    LOG.info("Got: " + kv);
-    assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
-        kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
-
-    // Check same with WAL enabled (historically these took different
-    // code paths, so check both)
-    row = Bytes.toBytes("row2");
-    put = new Put(row);
-    put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
-    region.put(put, true);
-
-    // Make sure it shows up with an actual timestamp
-    get = new Get(row).addColumn(fam, qual);
-    result = region.get(get, null);
-    assertEquals(1, result.size());
-    kv = result.raw()[0];
-    LOG.info("Got: " + kv);
-    assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
-        kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
+      // add data with LATEST_TIMESTAMP, put without WAL
+      Put put = new Put(row);
+      put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
+      region.put(put, false);
+
+      // Make sure it shows up with an actual timestamp
+      Get get = new Get(row).addColumn(fam, qual);
+      Result result = region.get(get, null);
+      assertEquals(1, result.size());
+      KeyValue kv = result.raw()[0];
+      LOG.info("Got: " + kv);
+      assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
+          kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
+
+      // Check same with WAL enabled (historically these took different
+      // code paths, so check both)
+      row = Bytes.toBytes("row2");
+      put = new Put(row);
+      put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
+      region.put(put, true);
+
+      // Make sure it shows up with an actual timestamp
+      get = new Get(row).addColumn(fam, qual);
+      result = region.get(get, null);
+      assertEquals(1, result.size());
+      kv = result.raw()[0];
+      LOG.info("Got: " + kv);
+      assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
+          kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
 
   }
 
@@ -1172,18 +1161,23 @@ public void testPutWithTsSlop() throws IOException {
 
     // add data with a timestamp that is too recent for range. Ensure assert
     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
-    initHRegion(tableName, method, conf, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
-      // no TS specified == use latest. should not error
-      region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes
-          .toBytes("value")), false);
-      // TS out of range. should error
-      region.put(new Put(row).add(fam, Bytes.toBytes("qual"),
-                 System.currentTimeMillis() + 2000,
-                 Bytes.toBytes("value")), false);
-      fail("Expected IOE for TS out of configured timerange");
-    } catch (DoNotRetryIOException ioe) {
-      LOG.debug("Received expected exception", ioe);
+      try {
+        // no TS specified == use latest. should not error
+        region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes
+            .toBytes("value")), false);
+        // TS out of range. should error
+        region.put(new Put(row).add(fam, Bytes.toBytes("qual"),
+            System.currentTimeMillis() + 2000,
+            Bytes.toBytes("value")), false);
+        fail("Expected IOE for TS out of configured timerange");
+      } catch (DoNotRetryIOException ioe) {
+        LOG.debug("Received expected exception", ioe);
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1191,39 +1185,42 @@ public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
     byte [] tableName = Bytes.toBytes("test_table");
     byte [] fam1 = Bytes.toBytes("columnA");
     byte [] fam2 = Bytes.toBytes("columnB");
-    initHRegion(tableName, getName(), fam1, fam2);
-
-    byte [] rowA = Bytes.toBytes("rowA");
-    byte [] rowB = Bytes.toBytes("rowB");
-
-    byte [] value = Bytes.toBytes("value");
+    this.region = initHRegion(tableName, getName(), fam1, fam2);
+    try {
+      byte [] rowA = Bytes.toBytes("rowA");
+      byte [] rowB = Bytes.toBytes("rowB");
 
-    Delete delete = new Delete(rowA);
-    delete.deleteFamily(fam1);
+      byte [] value = Bytes.toBytes("value");
 
-    region.delete(delete, null, true);
+      Delete delete = new Delete(rowA);
+      delete.deleteFamily(fam1);
 
-    // now create data.
-    Put put = new Put(rowA);
-    put.add(fam2, null, value);
-    region.put(put);
+      region.delete(delete, null, true);
 
-    put = new Put(rowB);
-    put.add(fam1, null, value);
-    put.add(fam2, null, value);
-    region.put(put);
+      // now create data.
+      Put put = new Put(rowA);
+      put.add(fam2, null, value);
+      region.put(put);
 
-    Scan scan = new Scan();
-    scan.addFamily(fam1).addFamily(fam2);
-    InternalScanner s = region.getScanner(scan);
-    List<KeyValue> results = new ArrayList<KeyValue>();
-    s.next(results);
-    assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
+      put = new Put(rowB);
+      put.add(fam1, null, value);
+      put.add(fam2, null, value);
+      region.put(put);
 
-    results.clear();
-    s.next(results);
-    assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
+      Scan scan = new Scan();
+      scan.addFamily(fam1).addFamily(fam2);
+      InternalScanner s = region.getScanner(scan);
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      s.next(results);
+      assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
 
+      results.clear();
+      s.next(results);
+      assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testDeleteColumns_PostInsert() throws IOException,
@@ -1241,47 +1238,50 @@ public void testDeleteFamily_PostInsert() throws IOException, InterruptedExcepti
 
   public void doTestDelete_AndPostInsert(Delete delete)
       throws IOException, InterruptedException {
-    initHRegion(tableName, getName(), fam1);
-    EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
-    Put put = new Put(row);
-    put.add(fam1, qual1, value1);
-    region.put(put);
-
-    // now delete the value:
-    region.delete(delete, null, true);
-
-
-    // ok put data:
-    put = new Put(row);
-    put.add(fam1, qual1, value2);
-    region.put(put);
-
-    // ok get:
-    Get get = new Get(row);
-    get.addColumn(fam1, qual1);
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
+      Put put = new Put(row);
+      put.add(fam1, qual1, value1);
+      region.put(put);
 
-    Result r = region.get(get, null);
-    assertEquals(1, r.size());
-    assertByteEquals(value2, r.getValue(fam1, qual1));
+      // now delete the value:
+      region.delete(delete, null, true);
 
-    // next:
-    Scan scan = new Scan(row);
-    scan.addColumn(fam1, qual1);
-    InternalScanner s = region.getScanner(scan);
 
-    List<KeyValue> results = new ArrayList<KeyValue>();
-    assertEquals(false, s.next(results));
-    assertEquals(1, results.size());
-    KeyValue kv = results.get(0);
+      // ok put data:
+      put = new Put(row);
+      put.add(fam1, qual1, value2);
+      region.put(put);
 
-    assertByteEquals(value2, kv.getValue());
-    assertByteEquals(fam1, kv.getFamily());
-    assertByteEquals(qual1, kv.getQualifier());
-    assertByteEquals(row, kv.getRow());
+      // ok get:
+      Get get = new Get(row);
+      get.addColumn(fam1, qual1);
+
+      Result r = region.get(get, null);
+      assertEquals(1, r.size());
+      assertByteEquals(value2, r.getValue(fam1, qual1));
+
+      // next:
+      Scan scan = new Scan(row);
+      scan.addColumn(fam1, qual1);
+      InternalScanner s = region.getScanner(scan);
+
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      assertEquals(false, s.next(results));
+      assertEquals(1, results.size());
+      KeyValue kv = results.get(0);
+
+      assertByteEquals(value2, kv.getValue());
+      assertByteEquals(fam1, kv.getFamily());
+      assertByteEquals(qual1, kv.getQualifier());
+      assertByteEquals(row, kv.getRow());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
-
-
   public void testDelete_CheckTimestampUpdated()
   throws IOException {
     byte [] row1 = Bytes.toBytes("row1");
@@ -1291,27 +1291,31 @@ public void testDelete_CheckTimestampUpdated()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    //Building checkerList
-    List<KeyValue> kvs  = new ArrayList<KeyValue>();
-    kvs.add(new KeyValue(row1, fam1, col1, null));
-    kvs.add(new KeyValue(row1, fam1, col2, null));
-    kvs.add(new KeyValue(row1, fam1, col3, null));
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Building checkerList
+      List<KeyValue> kvs  = new ArrayList<KeyValue>();
+      kvs.add(new KeyValue(row1, fam1, col1, null));
+      kvs.add(new KeyValue(row1, fam1, col2, null));
+      kvs.add(new KeyValue(row1, fam1, col3, null));
 
-    Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
-    deleteMap.put(fam1, kvs);
-    region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
+      Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
+      deleteMap.put(fam1, kvs);
+      region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
 
-    // extract the key values out the memstore:
-    // This is kinda hacky, but better than nothing...
-    long now = System.currentTimeMillis();
-    KeyValue firstKv = region.getStore(fam1).memstore.kvset.first();
-    assertTrue(firstKv.getTimestamp() <= now);
-    now = firstKv.getTimestamp();
-    for (KeyValue kv: region.getStore(fam1).memstore.kvset) {
-      assertTrue(kv.getTimestamp() <= now);
-      now = kv.getTimestamp();
+      // extract the key values out the memstore:
+      // This is kinda hacky, but better than nothing...
+      long now = System.currentTimeMillis();
+      KeyValue firstKv = region.getStore(fam1).memstore.kvset.first();
+      assertTrue(firstKv.getTimestamp() <= now);
+      now = firstKv.getTimestamp();
+      for (KeyValue kv: region.getStore(fam1).memstore.kvset) {
+        assertTrue(kv.getTimestamp() <= now);
+        now = kv.getTimestamp();
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1327,19 +1331,23 @@ public void testGet_FamilyChecker() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    Get get = new Get(row1);
-    get.addColumn(fam2, col1);
-
-    //Test
+    this.region = initHRegion(tableName, method, fam1);
     try {
-      region.get(get, null);
-    } catch (DoNotRetryIOException e) {
-      assertFalse(false);
-      return;
+      Get get = new Get(row1);
+      get.addColumn(fam2, col1);
+
+      //Test
+      try {
+        region.get(get, null);
+      } catch (DoNotRetryIOException e) {
+        assertFalse(false);
+        return;
+      }
+      assertFalse(true);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    assertFalse(true);
   }
 
   public void testGet_Basic() throws IOException {
@@ -1354,44 +1362,48 @@ public void testGet_Basic() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Add to memstore
+      Put put = new Put(row1);
+      put.add(fam1, col1, null);
+      put.add(fam1, col2, null);
+      put.add(fam1, col3, null);
+      put.add(fam1, col4, null);
+      put.add(fam1, col5, null);
+      region.put(put);
 
-    //Add to memstore
-    Put put = new Put(row1);
-    put.add(fam1, col1, null);
-    put.add(fam1, col2, null);
-    put.add(fam1, col3, null);
-    put.add(fam1, col4, null);
-    put.add(fam1, col5, null);
-    region.put(put);
-
-    Get get = new Get(row1);
-    get.addColumn(fam1, col2);
-    get.addColumn(fam1, col4);
-    //Expected result
-    KeyValue kv1 = new KeyValue(row1, fam1, col2);
-    KeyValue kv2 = new KeyValue(row1, fam1, col4);
-    KeyValue [] expected = {kv1, kv2};
-
-    //Test
-    Result res = region.get(get, null);
-    assertEquals(expected.length, res.size());
-    for(int i=0; i<res.size(); i++){
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
-      assertEquals(0,
-          Bytes.compareTo(
-              expected[i].getQualifier(), res.raw()[i].getQualifier()));
-    }
-
-    // Test using a filter on a Get
-    Get g = new Get(row1);
-    final int count = 2;
-    g.setFilter(new ColumnCountGetFilter(count));
-    res = region.get(g, null);
-    assertEquals(count, res.size());
+      Get get = new Get(row1);
+      get.addColumn(fam1, col2);
+      get.addColumn(fam1, col4);
+      //Expected result
+      KeyValue kv1 = new KeyValue(row1, fam1, col2);
+      KeyValue kv2 = new KeyValue(row1, fam1, col4);
+      KeyValue [] expected = {kv1, kv2};
+
+      //Test
+      Result res = region.get(get, null);
+      assertEquals(expected.length, res.size());
+      for(int i=0; i<res.size(); i++){
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
+        assertEquals(0,
+            Bytes.compareTo(
+                expected[i].getQualifier(), res.raw()[i].getQualifier()));
+      }
+
+      // Test using a filter on a Get
+      Get g = new Get(row1);
+      final int count = 2;
+      g.setFilter(new ColumnCountGetFilter(count));
+      res = region.get(g, null);
+      assertEquals(count, res.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testGet_Empty() throws IOException {
@@ -1400,13 +1412,17 @@ public void testGet_Empty() throws IOException {
     byte [] fam = Bytes.toBytes("fam");
 
     String method = this.getName();
-    initHRegion(tableName, method, fam);
-
-    Get get = new Get(row);
-    get.addFamily(fam);
-    Result r = region.get(get, null);
+    this.region = initHRegion(tableName, method, fam);
+    try {
+      Get get = new Get(row);
+      get.addFamily(fam);
+      Result r = region.get(get, null);
 
-    assertTrue(r.isEmpty());
+      assertTrue(r.isEmpty());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   //Test that checked if there was anything special when reading from the ROOT
@@ -1416,68 +1432,72 @@ public void testGet_Empty() throws IOException {
   public void stestGet_Root() throws IOException {
     //Setting up region
     String method = this.getName();
-    initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
-
-    //Add to memstore
-    Put put = new Put(HConstants.EMPTY_START_ROW);
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
-    region.put(put);
-
-    Get get = new Get(HConstants.EMPTY_START_ROW);
-    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-
-    //Expected result
-    KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
-        HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    KeyValue [] expected = {kv1};
-
-    //Test from memstore
-    Result res = region.get(get, null);
-
-    assertEquals(expected.length, res.size());
-    for(int i=0; i<res.size(); i++){
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
-      assertEquals(0,
-          Bytes.compareTo(
-              expected[i].getQualifier(), res.raw()[i].getQualifier()));
-    }
-
-    //flush
-    region.flushcache();
+    this.region = initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
+    try {
+      //Add to memstore
+      Put put = new Put(HConstants.EMPTY_START_ROW);
+      put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
+      region.put(put);
 
-    //test2
-    res = region.get(get, null);
+      Get get = new Get(HConstants.EMPTY_START_ROW);
+      get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+
+      //Expected result
+      KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
+          HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+      KeyValue [] expected = {kv1};
+
+      //Test from memstore
+      Result res = region.get(get, null);
+
+      assertEquals(expected.length, res.size());
+      for(int i=0; i<res.size(); i++){
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
+        assertEquals(0,
+            Bytes.compareTo(
+                expected[i].getQualifier(), res.raw()[i].getQualifier()));
+      }
 
-    assertEquals(expected.length, res.size());
-    for(int i=0; i<res.size(); i++){
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
-      assertEquals(0,
-          Bytes.compareTo(
-              expected[i].getQualifier(), res.raw()[i].getQualifier()));
-    }
+      //flush
+      region.flushcache();
 
-    //Scan
-    Scan scan = new Scan();
-    scan.addFamily(HConstants.CATALOG_FAMILY);
-    InternalScanner s = region.getScanner(scan);
-    List<KeyValue> result = new ArrayList<KeyValue>();
-    s.next(result);
+      //test2
+      res = region.get(get, null);
+
+      assertEquals(expected.length, res.size());
+      for(int i=0; i<res.size(); i++){
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
+        assertEquals(0,
+            Bytes.compareTo(
+                expected[i].getQualifier(), res.raw()[i].getQualifier()));
+      }
 
-    assertEquals(expected.length, result.size());
-    for(int i=0; i<res.size(); i++){
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
-      assertEquals(0,
-          Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
-      assertEquals(0,
-          Bytes.compareTo(
-              expected[i].getQualifier(), result.get(i).getQualifier()));
+      //Scan
+      Scan scan = new Scan();
+      scan.addFamily(HConstants.CATALOG_FAMILY);
+      InternalScanner s = region.getScanner(scan);
+      List<KeyValue> result = new ArrayList<KeyValue>();
+      s.next(result);
+
+      assertEquals(expected.length, result.size());
+      for(int i=0; i<res.size(); i++){
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
+        assertEquals(0,
+            Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
+        assertEquals(0,
+            Bytes.compareTo(
+                expected[i].getQualifier(), result.get(i).getQualifier()));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1491,58 +1511,62 @@ public void testLocks() throws IOException{
     Configuration hc = initSplit();
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, hc, families);
-
-    final int threadCount = 10;
-    final int lockCount = 10;
-
-    List<Thread>threads = new ArrayList<Thread>(threadCount);
-    for (int i = 0; i < threadCount; i++) {
-      threads.add(new Thread(Integer.toString(i)) {
-        @Override
-        public void run() {
-          Integer [] lockids = new Integer[lockCount];
-          // Get locks.
-          for (int i = 0; i < lockCount; i++) {
-            try {
-              byte [] rowid = Bytes.toBytes(Integer.toString(i));
-              lockids[i] = region.obtainRowLock(rowid);
-              assertEquals(rowid, region.getRowFromLock(lockids[i]));
-              LOG.debug(getName() + " locked " + Bytes.toString(rowid));
-            } catch (IOException e) {
-              e.printStackTrace();
+    this.region = initHRegion(tableName, method, hc, families);
+    try {
+      final int threadCount = 10;
+      final int lockCount = 10;
+
+      List<Thread>threads = new ArrayList<Thread>(threadCount);
+      for (int i = 0; i < threadCount; i++) {
+        threads.add(new Thread(Integer.toString(i)) {
+          @Override
+          public void run() {
+            Integer [] lockids = new Integer[lockCount];
+            // Get locks.
+            for (int i = 0; i < lockCount; i++) {
+              try {
+                byte [] rowid = Bytes.toBytes(Integer.toString(i));
+                lockids[i] = region.obtainRowLock(rowid);
+                assertEquals(rowid, region.getRowFromLock(lockids[i]));
+                LOG.debug(getName() + " locked " + Bytes.toString(rowid));
+              } catch (IOException e) {
+                e.printStackTrace();
+              }
             }
-          }
-          LOG.debug(getName() + " set " +
-              Integer.toString(lockCount) + " locks");
+            LOG.debug(getName() + " set " +
+                Integer.toString(lockCount) + " locks");
 
-          // Abort outstanding locks.
-          for (int i = lockCount - 1; i >= 0; i--) {
-            region.releaseRowLock(lockids[i]);
-            LOG.debug(getName() + " unlocked " + i);
+            // Abort outstanding locks.
+            for (int i = lockCount - 1; i >= 0; i--) {
+              region.releaseRowLock(lockids[i]);
+              LOG.debug(getName() + " unlocked " + i);
+            }
+            LOG.debug(getName() + " released " +
+                Integer.toString(lockCount) + " locks");
           }
-          LOG.debug(getName() + " released " +
-              Integer.toString(lockCount) + " locks");
-        }
-      });
-    }
+        });
+      }
 
-    // Startup all our threads.
-    for (Thread t : threads) {
-      t.start();
-    }
+      // Startup all our threads.
+      for (Thread t : threads) {
+        t.start();
+      }
 
-    // Now wait around till all are done.
-    for (Thread t: threads) {
-      while (t.isAlive()) {
-        try {
-          Thread.sleep(1);
-        } catch (InterruptedException e) {
-          // Go around again.
+      // Now wait around till all are done.
+      for (Thread t: threads) {
+        while (t.isAlive()) {
+          try {
+            Thread.sleep(1);
+          } catch (InterruptedException e) {
+            // Go around again.
+          }
         }
       }
+      LOG.info("locks completed.");
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    LOG.info("locks completed.");
   }
 
   //////////////////////////////////////////////////////////////////////////////
@@ -1554,7 +1578,7 @@ public void testMerge() throws IOException {
     Configuration hc = initSplit();
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, hc, families);
+    this.region = initHRegion(tableName, method, hc, families);
     try {
       LOG.info("" + addContent(region, fam3));
       region.flushcache();
@@ -1591,10 +1615,8 @@ public void testMerge() throws IOException {
         }
       }
     } finally {
-      if (region != null) {
-        region.close();
-        region.getLog().closeAndDelete();
-      }
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1642,15 +1664,19 @@ public void testGetScanner_WithOkFamilies() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    Scan scan = new Scan();
-    scan.addFamily(fam1);
-    scan.addFamily(fam2);
+    this.region = initHRegion(tableName, method, families);
     try {
-      region.getScanner(scan);
-    } catch (Exception e) {
-      assertTrue("Families could not be found in Region", false);
+      Scan scan = new Scan();
+      scan.addFamily(fam1);
+      scan.addFamily(fam2);
+      try {
+        region.getScanner(scan);
+      } catch (Exception e) {
+        assertTrue("Families could not be found in Region", false);
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1663,17 +1689,21 @@ public void testGetScanner_WithNotOkFamilies() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    Scan scan = new Scan();
-    scan.addFamily(fam2);
-    boolean ok = false;
+    this.region = initHRegion(tableName, method, families);
     try {
-      region.getScanner(scan);
-    } catch (Exception e) {
-      ok = true;
+      Scan scan = new Scan();
+      scan.addFamily(fam2);
+      boolean ok = false;
+      try {
+        region.getScanner(scan);
+      } catch (Exception e) {
+        ok = true;
+      }
+      assertTrue("Families could not be found in Region", ok);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    assertTrue("Families could not be found in Region", ok);
   }
 
   public void testGetScanner_WithNoFamilies() throws IOException {
@@ -1688,40 +1718,45 @@ public void testGetScanner_WithNoFamilies() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, families);
+    try {
 
+      //Putting data in Region
+      Put put = new Put(row1);
+      put.add(fam1, null, null);
+      put.add(fam2, null, null);
+      put.add(fam3, null, null);
+      put.add(fam4, null, null);
+      region.put(put);
 
-    //Putting data in Region
-    Put put = new Put(row1);
-    put.add(fam1, null, null);
-    put.add(fam2, null, null);
-    put.add(fam3, null, null);
-    put.add(fam4, null, null);
-    region.put(put);
-
-    Scan scan = null;
-    HRegion.RegionScannerImpl is = null;
-
-    //Testing to see how many scanners that is produced by getScanner, starting
-    //with known number, 2 - current = 1
-    scan = new Scan();
-    scan.addFamily(fam2);
-    scan.addFamily(fam4);
-    is = (RegionScannerImpl) region.getScanner(scan);
-    MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
-    assertEquals(1, ((RegionScannerImpl)is).storeHeap.getHeap().size());
-
-    scan = new Scan();
-    is = (RegionScannerImpl) region.getScanner(scan);
-    MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
-    assertEquals(families.length -1,
-        ((RegionScannerImpl)is).storeHeap.getHeap().size());
+      Scan scan = null;
+      HRegion.RegionScannerImpl is = null;
+
+      //Testing to see how many scanners that is produced by getScanner, starting
+      //with known number, 2 - current = 1
+      scan = new Scan();
+      scan.addFamily(fam2);
+      scan.addFamily(fam4);
+      is = (RegionScannerImpl) region.getScanner(scan);
+      MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
+      assertEquals(1, ((RegionScannerImpl)is).storeHeap.getHeap().size());
+
+      scan = new Scan();
+      is = (RegionScannerImpl) region.getScanner(scan);
+      MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
+      assertEquals(families.length -1,
+          ((RegionScannerImpl)is).storeHeap.getHeap().size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   /**
    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
+   * @throws IOException 
    */
-  public void testGetScanner_WithRegionClosed() {
+  public void testGetScanner_WithRegionClosed() throws IOException {
     byte[] tableName = Bytes.toBytes("testtable");
     byte[] fam1 = Bytes.toBytes("fam1");
     byte[] fam2 = Bytes.toBytes("fam2");
@@ -1731,20 +1766,25 @@ public void testGetScanner_WithRegionClosed() {
     //Setting up region
     String method = this.getName();
     try {
-      initHRegion(tableName, method, families);
+      this.region = initHRegion(tableName, method, families);
     } catch (IOException e) {
       e.printStackTrace();
       fail("Got IOException during initHRegion, " + e.getMessage());
     }
-    region.closed.set(true);
     try {
-      region.getScanner(null);
-      fail("Expected to get an exception during getScanner on a region that is closed");
-    } catch (org.apache.hadoop.hbase.NotServingRegionException e) {
-      //this is the correct exception that is expected
-    } catch (IOException e) {
-      fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
-              + e.getMessage());
+      region.closed.set(true);
+      try {
+        region.getScanner(null);
+        fail("Expected to get an exception during getScanner on a region that is closed");
+      } catch (org.apache.hadoop.hbase.NotServingRegionException e) {
+        //this is the correct exception that is expected
+      } catch (IOException e) {
+        fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
+            + e.getMessage());
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1762,53 +1802,56 @@ public void testRegionScanner_Next() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    //Putting data in Region
-    Put put = null;
-    put = new Put(row1);
-    put.add(fam1, null, ts, null);
-    put.add(fam2, null, ts, null);
-    put.add(fam3, null, ts, null);
-    put.add(fam4, null, ts, null);
-    region.put(put);
-
-    put = new Put(row2);
-    put.add(fam1, null, ts, null);
-    put.add(fam2, null, ts, null);
-    put.add(fam3, null, ts, null);
-    put.add(fam4, null, ts, null);
-    region.put(put);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Putting data in Region
+      Put put = null;
+      put = new Put(row1);
+      put.add(fam1, null, ts, null);
+      put.add(fam2, null, ts, null);
+      put.add(fam3, null, ts, null);
+      put.add(fam4, null, ts, null);
+      region.put(put);
 
-    Scan scan = new Scan();
-    scan.addFamily(fam2);
-    scan.addFamily(fam4);
-    InternalScanner is = region.getScanner(scan);
+      put = new Put(row2);
+      put.add(fam1, null, ts, null);
+      put.add(fam2, null, ts, null);
+      put.add(fam3, null, ts, null);
+      put.add(fam4, null, ts, null);
+      region.put(put);
 
-    List<KeyValue> res = null;
+      Scan scan = new Scan();
+      scan.addFamily(fam2);
+      scan.addFamily(fam4);
+      InternalScanner is = region.getScanner(scan);
 
-    //Result 1
-    List<KeyValue> expected1 = new ArrayList<KeyValue>();
-    expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
-    expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
+      List<KeyValue> res = null;
 
-    res = new ArrayList<KeyValue>();
-    is.next(res);
-    for(int i=0; i<res.size(); i++) {
-      assertEquals(expected1.get(i), res.get(i));
-    }
+      //Result 1
+      List<KeyValue> expected1 = new ArrayList<KeyValue>();
+      expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
+      expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
+
+      res = new ArrayList<KeyValue>();
+      is.next(res);
+      for(int i=0; i<res.size(); i++) {
+        assertEquals(expected1.get(i), res.get(i));
+      }
 
-    //Result 2
-    List<KeyValue> expected2 = new ArrayList<KeyValue>();
-    expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
-    expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
+      //Result 2
+      List<KeyValue> expected2 = new ArrayList<KeyValue>();
+      expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
+      expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
 
-    res = new ArrayList<KeyValue>();
-    is.next(res);
-    for(int i=0; i<res.size(); i++) {
-      assertEquals(expected2.get(i), res.get(i));
+      res = new ArrayList<KeyValue>();
+      is.next(res);
+      for(int i=0; i<res.size(); i++) {
+        assertEquals(expected2.get(i), res.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-
   }
 
   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
@@ -1826,44 +1869,48 @@ public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    //Putting data in Region
-    Put put = null;
-    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
-    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
-    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
-
-    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
-    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
-    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
-
-    put = new Put(row1);
-    put.add(kv13);
-    put.add(kv12);
-    put.add(kv11);
-    put.add(kv23);
-    put.add(kv22);
-    put.add(kv21);
-    region.put(put);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Putting data in Region
+      Put put = null;
+      KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
+      KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
+      KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
+
+      KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
+      KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
+      KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
+
+      put = new Put(row1);
+      put.add(kv13);
+      put.add(kv12);
+      put.add(kv11);
+      put.add(kv23);
+      put.add(kv22);
+      put.add(kv21);
+      region.put(put);
 
-    //Expected
-    List<KeyValue> expected = new ArrayList<KeyValue>();
-    expected.add(kv13);
-    expected.add(kv12);
+      //Expected
+      List<KeyValue> expected = new ArrayList<KeyValue>();
+      expected.add(kv13);
+      expected.add(kv12);
 
-    Scan scan = new Scan(row1);
-    scan.addColumn(fam1, qf1);
-    scan.setMaxVersions(MAX_VERSIONS);
-    List<KeyValue> actual = new ArrayList<KeyValue>();
-    InternalScanner scanner = region.getScanner(scan);
+      Scan scan = new Scan(row1);
+      scan.addColumn(fam1, qf1);
+      scan.setMaxVersions(MAX_VERSIONS);
+      List<KeyValue> actual = new ArrayList<KeyValue>();
+      InternalScanner scanner = region.getScanner(scan);
 
-    boolean hasNext = scanner.next(actual);
-    assertEquals(false, hasNext);
+      boolean hasNext = scanner.next(actual);
+      assertEquals(false, hasNext);
 
-    //Verify result
-    for(int i=0; i<expected.size(); i++) {
-      assertEquals(expected.get(i), actual.get(i));
+      //Verify result
+      for(int i=0; i<expected.size(); i++) {
+        assertEquals(expected.get(i), actual.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1882,48 +1929,52 @@ public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    //Putting data in Region
-    Put put = null;
-    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
-    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
-    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
-
-    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
-    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
-    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
-
-    put = new Put(row1);
-    put.add(kv13);
-    put.add(kv12);
-    put.add(kv11);
-    put.add(kv23);
-    put.add(kv22);
-    put.add(kv21);
-    region.put(put);
-    region.flushcache();
-
-    //Expected
-    List<KeyValue> expected = new ArrayList<KeyValue>();
-    expected.add(kv13);
-    expected.add(kv12);
-    expected.add(kv23);
-    expected.add(kv22);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Putting data in Region
+      Put put = null;
+      KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
+      KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
+      KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
+
+      KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
+      KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
+      KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
+
+      put = new Put(row1);
+      put.add(kv13);
+      put.add(kv12);
+      put.add(kv11);
+      put.add(kv23);
+      put.add(kv22);
+      put.add(kv21);
+      region.put(put);
+      region.flushcache();
 
-    Scan scan = new Scan(row1);
-    scan.addColumn(fam1, qf1);
-    scan.addColumn(fam1, qf2);
-    scan.setMaxVersions(MAX_VERSIONS);
-    List<KeyValue> actual = new ArrayList<KeyValue>();
-    InternalScanner scanner = region.getScanner(scan);
+      //Expected
+      List<KeyValue> expected = new ArrayList<KeyValue>();
+      expected.add(kv13);
+      expected.add(kv12);
+      expected.add(kv23);
+      expected.add(kv22);
+
+      Scan scan = new Scan(row1);
+      scan.addColumn(fam1, qf1);
+      scan.addColumn(fam1, qf2);
+      scan.setMaxVersions(MAX_VERSIONS);
+      List<KeyValue> actual = new ArrayList<KeyValue>();
+      InternalScanner scanner = region.getScanner(scan);
 
-    boolean hasNext = scanner.next(actual);
-    assertEquals(false, hasNext);
+      boolean hasNext = scanner.next(actual);
+      assertEquals(false, hasNext);
 
-    //Verify result
-    for(int i=0; i<expected.size(); i++) {
-      assertEquals(expected.get(i), actual.get(i));
+      //Verify result
+      for(int i=0; i<expected.size(); i++) {
+        assertEquals(expected.get(i), actual.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -1943,66 +1994,70 @@ public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    //Putting data in Region
-    KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
-    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
-    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
-    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
-
-    KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
-    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
-    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
-    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
-
-    Put put = null;
-    put = new Put(row1);
-    put.add(kv14);
-    put.add(kv24);
-    region.put(put);
-    region.flushcache();
-
-    put = new Put(row1);
-    put.add(kv23);
-    put.add(kv13);
-    region.put(put);
-    region.flushcache();
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Putting data in Region
+      KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
+      KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
+      KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
+      KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
+
+      KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
+      KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
+      KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
+      KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
+
+      Put put = null;
+      put = new Put(row1);
+      put.add(kv14);
+      put.add(kv24);
+      region.put(put);
+      region.flushcache();
 
-    put = new Put(row1);
-    put.add(kv22);
-    put.add(kv12);
-    region.put(put);
-    region.flushcache();
+      put = new Put(row1);
+      put.add(kv23);
+      put.add(kv13);
+      region.put(put);
+      region.flushcache();
 
-    put = new Put(row1);
-    put.add(kv21);
-    put.add(kv11);
-    region.put(put);
+      put = new Put(row1);
+      put.add(kv22);
+      put.add(kv12);
+      region.put(put);
+      region.flushcache();
 
-    //Expected
-    List<KeyValue> expected = new ArrayList<KeyValue>();
-    expected.add(kv14);
-    expected.add(kv13);
-    expected.add(kv12);
-    expected.add(kv24);
-    expected.add(kv23);
-    expected.add(kv22);
+      put = new Put(row1);
+      put.add(kv21);
+      put.add(kv11);
+      region.put(put);
 
-    Scan scan = new Scan(row1);
-    scan.addColumn(fam1, qf1);
-    scan.addColumn(fam1, qf2);
-    int versions = 3;
-    scan.setMaxVersions(versions);
-    List<KeyValue> actual = new ArrayList<KeyValue>();
-    InternalScanner scanner = region.getScanner(scan);
+      //Expected
+      List<KeyValue> expected = new ArrayList<KeyValue>();
+      expected.add(kv14);
+      expected.add(kv13);
+      expected.add(kv12);
+      expected.add(kv24);
+      expected.add(kv23);
+      expected.add(kv22);
+
+      Scan scan = new Scan(row1);
+      scan.addColumn(fam1, qf1);
+      scan.addColumn(fam1, qf2);
+      int versions = 3;
+      scan.setMaxVersions(versions);
+      List<KeyValue> actual = new ArrayList<KeyValue>();
+      InternalScanner scanner = region.getScanner(scan);
 
-    boolean hasNext = scanner.next(actual);
-    assertEquals(false, hasNext);
+      boolean hasNext = scanner.next(actual);
+      assertEquals(false, hasNext);
 
-    //Verify result
-    for(int i=0; i<expected.size(); i++) {
-      assertEquals(expected.get(i), actual.get(i));
+      //Verify result
+      for(int i=0; i<expected.size(); i++) {
+        assertEquals(expected.get(i), actual.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -2021,46 +2076,50 @@ public void testScanner_Wildcard_FromMemStore_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, families);
-
-    //Putting data in Region
-    Put put = null;
-    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
-    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
-    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
-
-    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
-    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
-    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
-
-    put = new Put(row1);
-    put.add(kv13);
-    put.add(kv12);
-    put.add(kv11);
-    put.add(kv23);
-    put.add(kv22);
-    put.add(kv21);
-    region.put(put);
-
-    //Expected
-    List<KeyValue> expected = new ArrayList<KeyValue>();
-    expected.add(kv13);
-    expected.add(kv12);
-    expected.add(kv23);
-    expected.add(kv22);
+    this.region = initHRegion(tableName, method, families);
+    try {
+      //Putting data in Region
+      Put put = null;
+      KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
+      KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
+      KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
+
+      KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
+      KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
+      KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
+
+      put = new Put(row1);
+      put.add(kv13);
+      put.add(kv12);
+      put.add(kv11);
+      put.add(kv23);
+      put.add(kv22);
+      put.add(kv21);
+      region.put(put);
 
-    Scan scan = new Scan(row1);
-    scan.addFamily(fam1);
-    scan.setMaxVersions(MAX_VERSIONS);
-    List<KeyValue> actual = new ArrayList<KeyValue>();
-    InternalScanner scanner = region.getScanner(scan);
+      //Expected
+      List<KeyValue> expected = new ArrayList<KeyValue>();
+      expected.add(kv13);
+      expected.add(kv12);
+      expected.add(kv23);
+      expected.add(kv22);
+
+      Scan scan = new Scan(row1);
+      scan.addFamily(fam1);
+      scan.setMaxVersions(MAX_VERSIONS);
+      List<KeyValue> actual = new ArrayList<KeyValue>();
+      InternalScanner scanner = region.getScanner(scan);
 
-    boolean hasNext = scanner.next(actual);
-    assertEquals(false, hasNext);
+      boolean hasNext = scanner.next(actual);
+      assertEquals(false, hasNext);
 
-    //Verify result
-    for(int i=0; i<expected.size(); i++) {
-      assertEquals(expected.get(i), actual.get(i));
+      //Verify result
+      for(int i=0; i<expected.size(); i++) {
+        assertEquals(expected.get(i), actual.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -2078,289 +2137,329 @@ public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    //Putting data in Region
-    Put put = null;
-    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
-    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
-    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
-
-    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
-    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
-    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
-
-    put = new Put(row1);
-    put.add(kv13);
-    put.add(kv12);
-    put.add(kv11);
-    put.add(kv23);
-    put.add(kv22);
-    put.add(kv21);
-    region.put(put);
-    region.flushcache();
-
-    //Expected
-    List<KeyValue> expected = new ArrayList<KeyValue>();
-    expected.add(kv13);
-    expected.add(kv12);
-    expected.add(kv23);
-    expected.add(kv22);
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Putting data in Region
+      Put put = null;
+      KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
+      KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
+      KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
+
+      KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
+      KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
+      KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
+
+      put = new Put(row1);
+      put.add(kv13);
+      put.add(kv12);
+      put.add(kv11);
+      put.add(kv23);
+      put.add(kv22);
+      put.add(kv21);
+      region.put(put);
+      region.flushcache();
 
-    Scan scan = new Scan(row1);
-    scan.addFamily(fam1);
-    scan.setMaxVersions(MAX_VERSIONS);
-    List<KeyValue> actual = new ArrayList<KeyValue>();
-    InternalScanner scanner = region.getScanner(scan);
+      //Expected
+      List<KeyValue> expected = new ArrayList<KeyValue>();
+      expected.add(kv13);
+      expected.add(kv12);
+      expected.add(kv23);
+      expected.add(kv22);
+
+      Scan scan = new Scan(row1);
+      scan.addFamily(fam1);
+      scan.setMaxVersions(MAX_VERSIONS);
+      List<KeyValue> actual = new ArrayList<KeyValue>();
+      InternalScanner scanner = region.getScanner(scan);
 
-    boolean hasNext = scanner.next(actual);
-    assertEquals(false, hasNext);
+      boolean hasNext = scanner.next(actual);
+      assertEquals(false, hasNext);
 
-    //Verify result
-    for(int i=0; i<expected.size(); i++) {
-      assertEquals(expected.get(i), actual.get(i));
+      //Verify result
+      for(int i=0; i<expected.size(); i++) {
+        assertEquals(expected.get(i), actual.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
   public void testScanner_StopRow1542() throws IOException {
     byte [] tableName = Bytes.toBytes("test_table");
     byte [] family = Bytes.toBytes("testFamily");
-    initHRegion(tableName, getName(), family);
-
-    byte [] row1 = Bytes.toBytes("row111");
-    byte [] row2 = Bytes.toBytes("row222");
-    byte [] row3 = Bytes.toBytes("row333");
-    byte [] row4 = Bytes.toBytes("row444");
-    byte [] row5 = Bytes.toBytes("row555");
+    this.region = initHRegion(tableName, getName(), family);
+    try {
+      byte [] row1 = Bytes.toBytes("row111");
+      byte [] row2 = Bytes.toBytes("row222");
+      byte [] row3 = Bytes.toBytes("row333");
+      byte [] row4 = Bytes.toBytes("row444");
+      byte [] row5 = Bytes.toBytes("row555");
 
-    byte [] col1 = Bytes.toBytes("Pub111");
-    byte [] col2 = Bytes.toBytes("Pub222");
+      byte [] col1 = Bytes.toBytes("Pub111");
+      byte [] col2 = Bytes.toBytes("Pub222");
 
 
-    Put put = new Put(row1);
-    put.add(family, col1, Bytes.toBytes(10L));
-    region.put(put);
+      Put put = new Put(row1);
+      put.add(family, col1, Bytes.toBytes(10L));
+      region.put(put);
 
-    put = new Put(row2);
-    put.add(family, col1, Bytes.toBytes(15L));
-    region.put(put);
+      put = new Put(row2);
+      put.add(family, col1, Bytes.toBytes(15L));
+      region.put(put);
 
-    put = new Put(row3);
-    put.add(family, col2, Bytes.toBytes(20L));
-    region.put(put);
+      put = new Put(row3);
+      put.add(family, col2, Bytes.toBytes(20L));
+      region.put(put);
 
-    put = new Put(row4);
-    put.add(family, col2, Bytes.toBytes(30L));
-    region.put(put);
+      put = new Put(row4);
+      put.add(family, col2, Bytes.toBytes(30L));
+      region.put(put);
 
-    put = new Put(row5);
-    put.add(family, col1, Bytes.toBytes(40L));
-    region.put(put);
+      put = new Put(row5);
+      put.add(family, col1, Bytes.toBytes(40L));
+      region.put(put);
 
-    Scan scan = new Scan(row3, row4);
-    scan.setMaxVersions();
-    scan.addColumn(family, col1);
-    InternalScanner s = region.getScanner(scan);
+      Scan scan = new Scan(row3, row4);
+      scan.setMaxVersions();
+      scan.addColumn(family, col1);
+      InternalScanner s = region.getScanner(scan);
 
-    List<KeyValue> results = new ArrayList<KeyValue>();
-    assertEquals(false, s.next(results));
-    assertEquals(0, results.size());
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      assertEquals(false, s.next(results));
+      assertEquals(0, results.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 1L;
-    long amount = 3L;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 1L;
+      long amount = 3L;
 
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    region.put(put);
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      region.put(put);
 
-    long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
+      long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
 
-    assertEquals(value+amount, result);
+      assertEquals(value+amount, result);
 
-    Store store = region.getStore(fam1);
-    // ICV removes any extra values floating around in there.
-    assertEquals(1, store.memstore.kvset.size());
-    assertTrue(store.memstore.snapshot.isEmpty());
+      Store store = region.getStore(fam1);
+      // ICV removes any extra values floating around in there.
+      assertEquals(1, store.memstore.kvset.size());
+      assertTrue(store.memstore.snapshot.isEmpty());
 
-    assertICV(row, fam1, qual1, value+amount);
+      assertICV(row, fam1, qual1, value+amount);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_BumpSnapshot() throws IOException {
     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 42L;
-    long incr = 44L;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 42L;
+      long incr = 44L;
 
-    // first put something in kvset, then snapshot it.
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    region.put(put);
+      // first put something in kvset, then snapshot it.
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      region.put(put);
 
-    // get the store in question:
-    Store s = region.getStore(fam1);
-    s.snapshot(); //bam
+      // get the store in question:
+      Store s = region.getStore(fam1);
+      s.snapshot(); //bam
 
-    // now increment:
-    long newVal = region.incrementColumnValue(row, fam1, qual1,
-        incr, false);
+      // now increment:
+      long newVal = region.incrementColumnValue(row, fam1, qual1,
+          incr, false);
 
-    assertEquals(value+incr, newVal);
+      assertEquals(value+incr, newVal);
 
-    // get both versions:
-    Get get = new Get(row);
-    get.setMaxVersions();
-    get.addColumn(fam1,qual1);
+      // get both versions:
+      Get get = new Get(row);
+      get.setMaxVersions();
+      get.addColumn(fam1,qual1);
 
-    Result r = region.get(get, null);
-    assertEquals(2, r.size());
-    KeyValue first = r.raw()[0];
-    KeyValue second = r.raw()[1];
+      Result r = region.get(get, null);
+      assertEquals(2, r.size());
+      KeyValue first = r.raw()[0];
+      KeyValue second = r.raw()[1];
 
-    assertTrue("ICV failed to upgrade timestamp",
-        first.getTimestamp() != second.getTimestamp());
+      assertTrue("ICV failed to upgrade timestamp",
+          first.getTimestamp() != second.getTimestamp());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 1L;
-    long amount = 3L;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 1L;
+      long amount = 3L;
 
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    region.put(put);
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      region.put(put);
 
-    // now increment during a flush
-    Thread t = new Thread() {
-      public void run() {
-        try {
-          region.flushcache();
-        } catch (IOException e) {
-          LOG.info("test ICV, got IOE during flushcache()");
+      // now increment during a flush
+      Thread t = new Thread() {
+        public void run() {
+          try {
+            region.flushcache();
+          } catch (IOException e) {
+            LOG.info("test ICV, got IOE during flushcache()");
+          }
         }
-      }
-    };
-    t.start();
-    long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
-    assertEquals(value+amount, r);
+      };
+      t.start();
+      long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
+      assertEquals(value+amount, r);
 
-    // this also asserts there is only 1 KeyValue in the set.
-    assertICV(row, fam1, qual1, value+amount);
+      // this also asserts there is only 1 KeyValue in the set.
+      assertICV(row, fam1, qual1, value+amount);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_heapSize() throws IOException {
     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
 
-    initHRegion(tableName, getName(), fam1);
-
-    long byAmount = 1L;
-    long size;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long byAmount = 1L;
+      long size;
 
-    for( int i = 0; i < 1000 ; i++) {
-      region.incrementColumnValue(row, fam1, qual1, byAmount, true);
+      for( int i = 0; i < 1000 ; i++) {
+        region.incrementColumnValue(row, fam1, qual1, byAmount, true);
 
-      size = region.memstoreSize.get();
-      assertTrue("memstore size: " + size, size >= 0);
+        size = region.memstoreSize.get();
+        assertTrue("memstore size: " + size, size >= 0);
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
   public void testIncrementColumnValue_UpdatingInPlace_Negative()
     throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 3L;
-    long amount = -1L;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 3L;
+      long amount = -1L;
 
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    region.put(put);
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      region.put(put);
 
-    long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
-    assertEquals(value+amount, result);
+      long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
+      assertEquals(value+amount, result);
 
-    assertICV(row, fam1, qual1, value+amount);
+      assertICV(row, fam1, qual1, value+amount);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_AddingNew()
     throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 1L;
-    long amount = 3L;
-
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    put.add(fam1, qual2, Bytes.toBytes(value));
-    region.put(put);
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 1L;
+      long amount = 3L;
 
-    long result = region.incrementColumnValue(row, fam1, qual3, amount, true);
-    assertEquals(amount, result);
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      put.add(fam1, qual2, Bytes.toBytes(value));
+      region.put(put);
 
-    Get get = new Get(row);
-    get.addColumn(fam1, qual3);
-    Result rr = region.get(get, null);
-    assertEquals(1, rr.size());
+      long result = region.incrementColumnValue(row, fam1, qual3, amount, true);
+      assertEquals(amount, result);
 
-    // ensure none of the other cols were incremented.
-    assertICV(row, fam1, qual1, value);
-    assertICV(row, fam1, qual2, value);
-    assertICV(row, fam1, qual3, amount);
+      Get get = new Get(row);
+      get.addColumn(fam1, qual3);
+      Result rr = region.get(get, null);
+      assertEquals(1, rr.size());
+
+      // ensure none of the other cols were incremented.
+      assertICV(row, fam1, qual1, value);
+      assertICV(row, fam1, qual2, value);
+      assertICV(row, fam1, qual3, amount);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 1L;
-    long amount = 3L;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 1L;
+      long amount = 3L;
 
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    put.add(fam1, qual2, Bytes.toBytes(value));
-    region.put(put);
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      put.add(fam1, qual2, Bytes.toBytes(value));
+      region.put(put);
 
-    // flush to disk.
-    region.flushcache();
+      // flush to disk.
+      region.flushcache();
 
-    Store store = region.getStore(fam1);
-    assertEquals(0, store.memstore.kvset.size());
+      Store store = region.getStore(fam1);
+      assertEquals(0, store.memstore.kvset.size());
 
-    long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
-    assertEquals(value+amount, r);
+      long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
+      assertEquals(value+amount, r);
 
-    assertICV(row, fam1, qual1, value+amount);
+      assertICV(row, fam1, qual1, value+amount);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_AddingNewAfterSFCheck()
     throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 1L;
-    long amount = 3L;
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 1L;
+      long amount = 3L;
 
-    Put put = new Put(row);
-    put.add(fam1, qual1, Bytes.toBytes(value));
-    put.add(fam1, qual2, Bytes.toBytes(value));
-    region.put(put);
-    region.flushcache();
+      Put put = new Put(row);
+      put.add(fam1, qual1, Bytes.toBytes(value));
+      put.add(fam1, qual2, Bytes.toBytes(value));
+      region.put(put);
+      region.flushcache();
 
-    Store store = region.getStore(fam1);
-    assertEquals(0, store.memstore.kvset.size());
+      Store store = region.getStore(fam1);
+      assertEquals(0, store.memstore.kvset.size());
 
-    long r = region.incrementColumnValue(row, fam1, qual3, amount, true);
-    assertEquals(amount, r);
+      long r = region.incrementColumnValue(row, fam1, qual3, amount, true);
+      assertEquals(amount, r);
 
-    assertICV(row, fam1, qual3, amount);
+      assertICV(row, fam1, qual3, amount);
 
-    region.flushcache();
+      region.flushcache();
 
-    // ensure that this gets to disk.
-    assertICV(row, fam1, qual3, amount);
+      // ensure that this gets to disk.
+      assertICV(row, fam1, qual3, amount);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   /**
@@ -2373,71 +2472,79 @@ public void testIncrementColumnValue_AddingNewAfterSFCheck()
    * @throws IOException
    */
   public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    long value = 1L;
-    long amount = 3L;
-    long now = EnvironmentEdgeManager.currentTimeMillis();
-    ManualEnvironmentEdge mock = new ManualEnvironmentEdge();
-    mock.setValue(now);
-    EnvironmentEdgeManagerTestHelper.injectEdge(mock);
-
-    // verify we catch an ICV on a put with the same timestamp
-    Put put = new Put(row);
-    put.add(fam1, qual1, now, Bytes.toBytes(value));
-    region.put(put);
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      long value = 1L;
+      long amount = 3L;
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      ManualEnvironmentEdge mock = new ManualEnvironmentEdge();
+      mock.setValue(now);
+      EnvironmentEdgeManagerTestHelper.injectEdge(mock);
+
+      // verify we catch an ICV on a put with the same timestamp
+      Put put = new Put(row);
+      put.add(fam1, qual1, now, Bytes.toBytes(value));
+      region.put(put);
 
-    long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
+      long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
 
-    assertEquals(value+amount, result);
+      assertEquals(value+amount, result);
 
-    Store store = region.getStore(fam1);
-    // ICV should update the existing Put with the same timestamp
-    assertEquals(1, store.memstore.kvset.size());
-    assertTrue(store.memstore.snapshot.isEmpty());
+      Store store = region.getStore(fam1);
+      // ICV should update the existing Put with the same timestamp
+      assertEquals(1, store.memstore.kvset.size());
+      assertTrue(store.memstore.snapshot.isEmpty());
 
-    assertICV(row, fam1, qual1, value+amount);
+      assertICV(row, fam1, qual1, value+amount);
 
-    // verify we catch an ICV even when the put ts > now
-    put = new Put(row);
-    put.add(fam1, qual2, now+1, Bytes.toBytes(value));
-    region.put(put);
+      // verify we catch an ICV even when the put ts > now
+      put = new Put(row);
+      put.add(fam1, qual2, now+1, Bytes.toBytes(value));
+      region.put(put);
 
-    result = region.incrementColumnValue(row, fam1, qual2, amount, true);
+      result = region.incrementColumnValue(row, fam1, qual2, amount, true);
 
-    assertEquals(value+amount, result);
+      assertEquals(value+amount, result);
 
-    store = region.getStore(fam1);
-    // ICV should update the existing Put with the same timestamp
-    assertEquals(2, store.memstore.kvset.size());
-    assertTrue(store.memstore.snapshot.isEmpty());
+      store = region.getStore(fam1);
+      // ICV should update the existing Put with the same timestamp
+      assertEquals(2, store.memstore.kvset.size());
+      assertTrue(store.memstore.snapshot.isEmpty());
 
-    assertICV(row, fam1, qual2, value+amount);
-    EnvironmentEdgeManagerTestHelper.reset();
+      assertICV(row, fam1, qual2, value+amount);
+      EnvironmentEdgeManagerTestHelper.reset();
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testIncrementColumnValue_WrongInitialSize() throws IOException {
-    initHRegion(tableName, getName(), fam1);
-
-    byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
-    int row1Field1 = 0;
-    int row1Field2 = 1;
-    Put put1 = new Put(row1);
-    put1.add(fam1, qual1, Bytes.toBytes(row1Field1));
-    put1.add(fam1, qual2, Bytes.toBytes(row1Field2));
-    region.put(put1);
-
-    long result;
+    this.region = initHRegion(tableName, getName(), fam1);
     try {
+      byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
+      int row1Field1 = 0;
+      int row1Field2 = 1;
+      Put put1 = new Put(row1);
+      put1.add(fam1, qual1, Bytes.toBytes(row1Field1));
+      put1.add(fam1, qual2, Bytes.toBytes(row1Field2));
+      region.put(put1);
+
+      long result;
+      try {
         result = region.incrementColumnValue(row1, fam1, qual1, 1, true);
         fail("Expected to fail here");
-    } catch (Exception exception) {
+      } catch (Exception exception) {
         // Expected.
-    }
+      }
 
 
-    assertICV(row1, fam1, qual1, row1Field1);
-    assertICV(row1, fam1, qual2, row1Field2);
+      assertICV(row1, fam1, qual1, row1Field1);
+      assertICV(row1, fam1, qual2, row1Field2);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   private void assertICV(byte [] row,
@@ -2485,64 +2592,68 @@ public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, fam1);
-
-    //Putting data in Region
-    KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
-    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
-    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
-    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
-
-    KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
-    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
-    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
-    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
-
-    Put put = null;
-    put = new Put(row1);
-    put.add(kv14);
-    put.add(kv24);
-    region.put(put);
-    region.flushcache();
-
-    put = new Put(row1);
-    put.add(kv23);
-    put.add(kv13);
-    region.put(put);
-    region.flushcache();
+    this.region = initHRegion(tableName, method, fam1);
+    try {
+      //Putting data in Region
+      KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
+      KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
+      KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
+      KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
+
+      KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
+      KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
+      KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
+      KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
+
+      Put put = null;
+      put = new Put(row1);
+      put.add(kv14);
+      put.add(kv24);
+      region.put(put);
+      region.flushcache();
 
-    put = new Put(row1);
-    put.add(kv22);
-    put.add(kv12);
-    region.put(put);
-    region.flushcache();
+      put = new Put(row1);
+      put.add(kv23);
+      put.add(kv13);
+      region.put(put);
+      region.flushcache();
 
-    put = new Put(row1);
-    put.add(kv21);
-    put.add(kv11);
-    region.put(put);
+      put = new Put(row1);
+      put.add(kv22);
+      put.add(kv12);
+      region.put(put);
+      region.flushcache();
 
-    //Expected
-    List<KeyValue> expected = new ArrayList<KeyValue>();
-    expected.add(kv14);
-    expected.add(kv13);
-    expected.add(kv12);
-    expected.add(kv24);
-    expected.add(kv23);
-    expected.add(kv22);
+      put = new Put(row1);
+      put.add(kv21);
+      put.add(kv11);
+      region.put(put);
 
-    Scan scan = new Scan(row1);
-    int versions = 3;
-    scan.setMaxVersions(versions);
-    List<KeyValue> actual = new ArrayList<KeyValue>();
-    InternalScanner scanner = region.getScanner(scan);
+      //Expected
+      List<KeyValue> expected = new ArrayList<KeyValue>();
+      expected.add(kv14);
+      expected.add(kv13);
+      expected.add(kv12);
+      expected.add(kv24);
+      expected.add(kv23);
+      expected.add(kv22);
+
+      Scan scan = new Scan(row1);
+      int versions = 3;
+      scan.setMaxVersions(versions);
+      List<KeyValue> actual = new ArrayList<KeyValue>();
+      InternalScanner scanner = region.getScanner(scan);
 
-    boolean hasNext = scanner.next(actual);
-    assertEquals(false, hasNext);
+      boolean hasNext = scanner.next(actual);
+      assertEquals(false, hasNext);
 
-    //Verify result
-    for(int i=0; i<expected.size(); i++) {
-      assertEquals(expected.get(i), actual.get(i));
+      //Verify result
+      for(int i=0; i<expected.size(); i++) {
+        assertEquals(expected.get(i), actual.get(i));
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -2560,7 +2671,7 @@ public void testBasicSplit() throws Exception {
     Configuration hc = initSplit();
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, hc, families);
+    this.region = initHRegion(tableName, method, hc, families);
 
     try {
       LOG.info("" + addContent(region, fam3));
@@ -2636,10 +2747,8 @@ public void testBasicSplit() throws Exception {
         }
       }
     } finally {
-      if (region != null) {
-        region.close();
-        region.getLog().closeAndDelete();
-      }
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -2652,7 +2761,7 @@ public void testSplitRegion() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    initHRegion(tableName, method, hc, families);
+    this.region = initHRegion(tableName, method, hc, families);
 
     //Put data in region
     int startRow = 100;
@@ -2677,10 +2786,8 @@ public void testSplitRegion() throws IOException {
       verifyData(regions[1], splitRow, numRows, qualifier, families);
 
     } finally {
-      if (region != null) {
-        region.close();
-        region.getLog().closeAndDelete();
-      }
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -2700,52 +2807,57 @@ public void testFlushCacheWhileScanning() throws IOException, InterruptedExcepti
     int compactInterval = 10 * flushAndScanInterval;
 
     String method = "testFlushCacheWhileScanning";
-    initHRegion(tableName,method, family);
-    FlushThread flushThread = new FlushThread();
-    flushThread.start();
-
-    Scan scan = new Scan();
-    scan.addFamily(family);
-    scan.setFilter(new SingleColumnValueFilter(family, qual1,
-      CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
-
-    int expectedCount = 0;
-    List<KeyValue> res = new ArrayList<KeyValue>();
-
-    boolean toggle=true;
-    for (long i = 0; i < numRows; i++) {
-      Put put = new Put(Bytes.toBytes(i));
-      put.setWriteToWAL(false);
-      put.add(family, qual1, Bytes.toBytes(i % 10));
-      region.put(put);
-
-      if (i != 0 && i % compactInterval == 0) {
-        //System.out.println("iteration = " + i);
-        region.compactStores(true);
-      }
-
-      if (i % 10 == 5L) {
-        expectedCount++;
-      }
+    this.region = initHRegion(tableName,method, family);
+    try {
+      FlushThread flushThread = new FlushThread();
+      flushThread.start();
+
+      Scan scan = new Scan();
+      scan.addFamily(family);
+      scan.setFilter(new SingleColumnValueFilter(family, qual1,
+          CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
+
+      int expectedCount = 0;
+      List<KeyValue> res = new ArrayList<KeyValue>();
+
+      boolean toggle=true;
+      for (long i = 0; i < numRows; i++) {
+        Put put = new Put(Bytes.toBytes(i));
+        put.setWriteToWAL(false);
+        put.add(family, qual1, Bytes.toBytes(i % 10));
+        region.put(put);
+
+        if (i != 0 && i % compactInterval == 0) {
+          //System.out.println("iteration = " + i);
+          region.compactStores(true);
+        }
 
-      if (i != 0 && i % flushAndScanInterval == 0) {
-        res.clear();
-        InternalScanner scanner = region.getScanner(scan);
-        if (toggle) {
-          flushThread.flush();
+        if (i % 10 == 5L) {
+          expectedCount++;
         }
-        while (scanner.next(res)) ;
-        if (!toggle) {
-          flushThread.flush();
+
+        if (i != 0 && i % flushAndScanInterval == 0) {
+          res.clear();
+          InternalScanner scanner = region.getScanner(scan);
+          if (toggle) {
+            flushThread.flush();
+          }
+          while (scanner.next(res)) ;
+          if (!toggle) {
+            flushThread.flush();
+          }
+          assertEquals("i=" + i, expectedCount, res.size());
+          toggle = !toggle;
         }
-        assertEquals("i=" + i, expectedCount, res.size());
-        toggle = !toggle;
       }
-    }
 
-    flushThread.done();
-    flushThread.join();
-    flushThread.checkNoError();
+      flushThread.done();
+      flushThread.join();
+      flushThread.checkNoError();
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   protected class FlushThread extends Thread {
@@ -2826,56 +2938,61 @@ public void testWritesWhileScanning()
     }
 
     String method = "testWritesWhileScanning";
-    initHRegion(tableName, method, families);
-    PutThread putThread = new PutThread(numRows, families, qualifiers);
-    putThread.start();
-    putThread.waitForFirstPut();
+    this.region = initHRegion(tableName, method, families);
+    try {
+      PutThread putThread = new PutThread(numRows, families, qualifiers);
+      putThread.start();
+      putThread.waitForFirstPut();
 
-    FlushThread flushThread = new FlushThread();
-    flushThread.start();
+      FlushThread flushThread = new FlushThread();
+      flushThread.start();
 
-    Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
-//    scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,
-//      new BinaryComparator(Bytes.toBytes("row0"))));
+      Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
+      //    scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,
+      //      new BinaryComparator(Bytes.toBytes("row0"))));
 
-    int expectedCount = numFamilies * numQualifiers;
-    List<KeyValue> res = new ArrayList<KeyValue>();
+      int expectedCount = numFamilies * numQualifiers;
+      List<KeyValue> res = new ArrayList<KeyValue>();
 
-    long prevTimestamp = 0L;
-    for (int i = 0; i < testCount; i++) {
+      long prevTimestamp = 0L;
+      for (int i = 0; i < testCount; i++) {
 
-      if (i != 0 && i % compactInterval == 0) {
-        region.compactStores(true);
-      }
+        if (i != 0 && i % compactInterval == 0) {
+          region.compactStores(true);
+        }
 
-      if (i != 0 && i % flushInterval == 0) {
-        //System.out.println("flush scan iteration = " + i);
-        flushThread.flush();
-      }
+        if (i != 0 && i % flushInterval == 0) {
+          //System.out.println("flush scan iteration = " + i);
+          flushThread.flush();
+        }
 
-      boolean previousEmpty = res.isEmpty();
-      res.clear();
-      InternalScanner scanner = region.getScanner(scan);
-      while (scanner.next(res)) ;
-      if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
-        assertEquals("i=" + i, expectedCount, res.size());
-        long timestamp = res.get(0).getTimestamp();
-        assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
-            timestamp >= prevTimestamp);
-        prevTimestamp = timestamp;
+        boolean previousEmpty = res.isEmpty();
+        res.clear();
+        InternalScanner scanner = region.getScanner(scan);
+        while (scanner.next(res)) ;
+        if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
+          assertEquals("i=" + i, expectedCount, res.size());
+          long timestamp = res.get(0).getTimestamp();
+          assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
+              timestamp >= prevTimestamp);
+          prevTimestamp = timestamp;
+        }
       }
-    }
 
-    putThread.done();
+      putThread.done();
 
-    region.flushcache();
+      region.flushcache();
 
-    putThread.join();
-    putThread.checkNoError();
+      putThread.join();
+      putThread.checkNoError();
 
-    flushThread.done();
-    flushThread.join();
-    flushThread.checkNoError();
+      flushThread.done();
+      flushThread.join();
+      flushThread.checkNoError();
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   protected class PutThread extends Thread {
@@ -2982,95 +3099,105 @@ public void testWritesWhileGetting()
     }
 
     String method = "testWritesWhileGetting";
-    initHRegion(tableName, method, families);
-    PutThread putThread = new PutThread(numRows, families, qualifiers);
-    putThread.start();
-    putThread.waitForFirstPut();
+    this.region = initHRegion(tableName, method, families);
+    try {
+      PutThread putThread = new PutThread(numRows, families, qualifiers);
+      putThread.start();
+      putThread.waitForFirstPut();
 
-    FlushThread flushThread = new FlushThread();
-    flushThread.start();
+      FlushThread flushThread = new FlushThread();
+      flushThread.start();
 
-    Get get = new Get(Bytes.toBytes("row0"));
-    Result result = null;
+      Get get = new Get(Bytes.toBytes("row0"));
+      Result result = null;
 
-    int expectedCount = numFamilies * numQualifiers;
+      int expectedCount = numFamilies * numQualifiers;
 
-    long prevTimestamp = 0L;
-    for (int i = 0; i < testCount; i++) {
+      long prevTimestamp = 0L;
+      for (int i = 0; i < testCount; i++) {
 
-      if (i != 0 && i % compactInterval == 0) {
-        region.compactStores(true);
-      }
+        if (i != 0 && i % compactInterval == 0) {
+          region.compactStores(true);
+        }
 
-      if (i != 0 && i % flushInterval == 0) {
-        //System.out.println("iteration = " + i);
-        flushThread.flush();
-      }
+        if (i != 0 && i % flushInterval == 0) {
+          //System.out.println("iteration = " + i);
+          flushThread.flush();
+        }
 
-      boolean previousEmpty = result == null || result.isEmpty();
-      result = region.get(get, null);
-      if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
-        assertEquals("i=" + i, expectedCount, result.size());
-        // TODO this was removed, now what dangit?!
-        // search looking for the qualifier in question?
-        long timestamp = 0;
-        for (KeyValue kv : result.raw()) {
-          if (Bytes.equals(kv.getFamily(), families[0])
-            && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
-            timestamp = kv.getTimestamp();
+        boolean previousEmpty = result == null || result.isEmpty();
+        result = region.get(get, null);
+        if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
+          assertEquals("i=" + i, expectedCount, result.size());
+          // TODO this was removed, now what dangit?!
+          // search looking for the qualifier in question?
+          long timestamp = 0;
+          for (KeyValue kv : result.raw()) {
+            if (Bytes.equals(kv.getFamily(), families[0])
+                && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
+              timestamp = kv.getTimestamp();
+            }
           }
-        }
-        assertTrue(timestamp >= prevTimestamp);
-        prevTimestamp = timestamp;
-        KeyValue previousKV = null;
-
-        for (KeyValue kv : result.raw()) {
-          byte[] thisValue = kv.getValue();
-          if (previousKV != null) {
-            if (Bytes.compareTo(previousKV.getValue(), thisValue) != 0) {
-              LOG.warn("These two KV should have the same value." +
-                " Previous KV:" +
-                previousKV + "(memStoreTS:" + previousKV.getMemstoreTS() + ")" +
-                ", New KV: " +
-                kv + "(memStoreTS:" + kv.getMemstoreTS() + ")"
-              );
-              assertEquals(previousKV.getValue(), thisValue);
+          assertTrue(timestamp >= prevTimestamp);
+          prevTimestamp = timestamp;
+          KeyValue previousKV = null;
+
+          for (KeyValue kv : result.raw()) {
+            byte[] thisValue = kv.getValue();
+            if (previousKV != null) {
+              if (Bytes.compareTo(previousKV.getValue(), thisValue) != 0) {
+                LOG.warn("These two KV should have the same value." +
+                    " Previous KV:" +
+                    previousKV + "(memStoreTS:" + previousKV.getMemstoreTS() + ")" +
+                    ", New KV: " +
+                    kv + "(memStoreTS:" + kv.getMemstoreTS() + ")"
+                    );
+                assertEquals(previousKV.getValue(), thisValue);
+              }
             }
+            previousKV = kv;
           }
-          previousKV = kv;
         }
       }
-    }
 
-    putThread.done();
+      putThread.done();
 
-    region.flushcache();
+      region.flushcache();
 
-    putThread.join();
-    putThread.checkNoError();
+      putThread.join();
+      putThread.checkNoError();
 
-    flushThread.done();
-    flushThread.join();
-    flushThread.checkNoError();
+      flushThread.done();
+      flushThread.join();
+      flushThread.checkNoError();
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   public void testHolesInMeta() throws Exception {
     String method = "testHolesInMeta";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
+    this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
         HBaseConfiguration.create(), family);
-    byte[] rowNotServed = Bytes.toBytes("a");
-    Get g = new Get(rowNotServed);
     try {
+      byte[] rowNotServed = Bytes.toBytes("a");
+      Get g = new Get(rowNotServed);
+      try {
+        region.get(g, null);
+        fail();
+      } catch (WrongRegionException x) {
+        // OK
+      }
+      byte[] row = Bytes.toBytes("y");
+      g = new Get(row);
       region.get(g, null);
-      fail();
-    } catch (WrongRegionException x) {
-      // OK
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
-    byte[] row = Bytes.toBytes("y");
-    g = new Get(row);
-    region.get(g, null);
   }
 
   public void testIndexesScanWithOneDeletedRow() throws IOException {
@@ -3079,40 +3206,43 @@ public void testIndexesScanWithOneDeletedRow() throws IOException {
 
     //Setting up region
     String method = "testIndexesScanWithOneDeletedRow";
-    initHRegion(tableName, method, HBaseConfiguration.create(), family);
-
-    Put put = new Put(Bytes.toBytes(1L));
-    put.add(family, qual1, 1L, Bytes.toBytes(1L));
-    region.put(put);
-
-    region.flushcache();
-
-    Delete delete = new Delete(Bytes.toBytes(1L), 1L, null);
-    //delete.deleteColumn(family, qual1);
-    region.delete(delete, null, true);
+    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    try {
+      Put put = new Put(Bytes.toBytes(1L));
+      put.add(family, qual1, 1L, Bytes.toBytes(1L));
+      region.put(put);
 
-    put = new Put(Bytes.toBytes(2L));
-    put.add(family, qual1, 2L, Bytes.toBytes(2L));
-    region.put(put);
+      region.flushcache();
 
-    Scan idxScan = new Scan();
-    idxScan.addFamily(family);
-    idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
-      Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
-        CompareOp.GREATER_OR_EQUAL,
-        new BinaryComparator(Bytes.toBytes(0L))),
-        new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL,
-          new BinaryComparator(Bytes.toBytes(3L)))
-      )));
-    InternalScanner scanner = region.getScanner(idxScan);
-    List<KeyValue> res = new ArrayList<KeyValue>();
+      Delete delete = new Delete(Bytes.toBytes(1L), 1L, null);
+      //delete.deleteColumn(family, qual1);
+      region.delete(delete, null, true);
 
-    //long start = System.nanoTime();
-    while (scanner.next(res)) ;
-    //long end = System.nanoTime();
-    //System.out.println("memStoreEmpty=" + memStoreEmpty + ", time=" + (end - start)/1000000D);
-    assertEquals(1L, res.size());
+      put = new Put(Bytes.toBytes(2L));
+      put.add(family, qual1, 2L, Bytes.toBytes(2L));
+      region.put(put);
 
+      Scan idxScan = new Scan();
+      idxScan.addFamily(family);
+      idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
+          Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
+              CompareOp.GREATER_OR_EQUAL,
+              new BinaryComparator(Bytes.toBytes(0L))),
+              new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL,
+                  new BinaryComparator(Bytes.toBytes(3L)))
+              )));
+      InternalScanner scanner = region.getScanner(idxScan);
+      List<KeyValue> res = new ArrayList<KeyValue>();
+
+      //long start = System.nanoTime();
+      while (scanner.next(res)) ;
+      //long end = System.nanoTime();
+      //System.out.println("memStoreEmpty=" + memStoreEmpty + ", time=" + (end - start)/1000000D);
+      assertEquals(1L, res.size());
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   //////////////////////////////////////////////////////////////////////////////
@@ -3133,46 +3263,50 @@ public void testBloomFilterSize() throws IOException {
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     Path path = new Path(DIR + "testBloomFilterSize");
-    region = HRegion.createHRegion(info, path, conf, htd);
-
-    int num_unique_rows = 10;
-    int duplicate_multiplier =2;
-    int num_storefiles = 4;
-
-    int version = 0;
-    for (int f =0 ; f < num_storefiles; f++) {
-      for (int i = 0; i < duplicate_multiplier; i ++) {
-        for (int j = 0; j < num_unique_rows; j++) {
-          Put put = new Put(Bytes.toBytes("row" + j));
-          put.setWriteToWAL(false);
-    			put.add(fam1, qf1, version++, val1);
-          region.put(put);
+    this.region = HRegion.createHRegion(info, path, conf, htd);
+    try {
+      int num_unique_rows = 10;
+      int duplicate_multiplier =2;
+      int num_storefiles = 4;
+
+      int version = 0;
+      for (int f =0 ; f < num_storefiles; f++) {
+        for (int i = 0; i < duplicate_multiplier; i ++) {
+          for (int j = 0; j < num_unique_rows; j++) {
+            Put put = new Put(Bytes.toBytes("row" + j));
+            put.setWriteToWAL(false);
+            put.add(fam1, qf1, version++, val1);
+            region.put(put);
+          }
         }
+        region.flushcache();
+      }
+      //before compaction
+      Store store = region.getStore(fam1);
+      List<StoreFile> storeFiles = store.getStorefiles();
+      for (StoreFile storefile : storeFiles) {
+        StoreFile.Reader reader = storefile.getReader();
+        reader.loadFileInfo();
+        reader.loadBloomfilter();
+        assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries());
+        assertEquals(num_unique_rows, reader.getFilterEntries());
       }
-      region.flushcache();
-    }
-    //before compaction
-    Store store = region.getStore(fam1);
-    List<StoreFile> storeFiles = store.getStorefiles();
-    for (StoreFile storefile : storeFiles) {
-      StoreFile.Reader reader = storefile.getReader();
-      reader.loadFileInfo();
-      reader.loadBloomfilter();
-      assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries());
-      assertEquals(num_unique_rows, reader.getFilterEntries());
-    }
-
-    region.compactStores(true);
 
-    //after compaction
-    storeFiles = store.getStorefiles();
-    for (StoreFile storefile : storeFiles) {
-      StoreFile.Reader reader = storefile.getReader();
-      reader.loadFileInfo();
-      reader.loadBloomfilter();
-      assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles,
-          reader.getEntries());
-      assertEquals(num_unique_rows, reader.getFilterEntries());
+      region.compactStores(true);
+
+      //after compaction
+      storeFiles = store.getStorefiles();
+      for (StoreFile storefile : storeFiles) {
+        StoreFile.Reader reader = storefile.getReader();
+        reader.loadFileInfo();
+        reader.loadBloomfilter();
+        assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles,
+            reader.getEntries());
+        assertEquals(num_unique_rows, reader.getFilterEntries());
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
     }
   }
 
@@ -3188,32 +3322,36 @@ public void testAllColumnsWithBloomFilter() throws IOException {
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     Path path = new Path(DIR + "testAllColumnsWithBloomFilter");
-    region = HRegion.createHRegion(info, path, conf, htd);
-
-    // For row:0, col:0: insert versions 1 through 5.
-    byte row[] = Bytes.toBytes("row:" + 0);
-    byte column[] = Bytes.toBytes("column:" + 0);
-    Put put = new Put(row);
-    put.setWriteToWAL(false);
-    for (long idx = 1; idx <= 4; idx++) {
-      put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
-    }
-    region.put(put);
-
-    //Flush
-    region.flushcache();
+    this.region = HRegion.createHRegion(info, path, conf, htd);
+    try {
+      // For row:0, col:0: insert versions 1 through 5.
+      byte row[] = Bytes.toBytes("row:" + 0);
+      byte column[] = Bytes.toBytes("column:" + 0);
+      Put put = new Put(row);
+      put.setWriteToWAL(false);
+      for (long idx = 1; idx <= 4; idx++) {
+        put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
+      }
+      region.put(put);
 
-    //Get rows
-    Get get = new Get(row);
-    get.setMaxVersions();
-    KeyValue[] kvs = region.get(get, null).raw();
+      //Flush
+      region.flushcache();
 
-    //Check if rows are correct
-    assertEquals(4, kvs.length);
-    checkOneCell(kvs[0], FAMILY, 0, 0, 4);
-    checkOneCell(kvs[1], FAMILY, 0, 0, 3);
-    checkOneCell(kvs[2], FAMILY, 0, 0, 2);
-    checkOneCell(kvs[3], FAMILY, 0, 0, 1);
+      //Get rows
+      Get get = new Get(row);
+      get.setMaxVersions();
+      KeyValue[] kvs = region.get(get, null).raw();
+
+      //Check if rows are correct
+      assertEquals(4, kvs.length);
+      checkOneCell(kvs[0], FAMILY, 0, 0, 4);
+      checkOneCell(kvs[1], FAMILY, 0, 0, 3);
+      checkOneCell(kvs[2], FAMILY, 0, 0, 2);
+      checkOneCell(kvs[3], FAMILY, 0, 0, 1);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   /**
@@ -3234,27 +3372,31 @@ public void testDeleteRowWithBloomFilter() throws IOException {
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
     Path path = new Path(DIR + "TestDeleteRowWithBloomFilter");
-    region = HRegion.createHRegion(info, path, conf, htd);
-
-    // Insert some data
-    byte row[] = Bytes.toBytes("row1");
-    byte col[] = Bytes.toBytes("col1");
+    this.region = HRegion.createHRegion(info, path, conf, htd);
+    try {
+      // Insert some data
+      byte row[] = Bytes.toBytes("row1");
+      byte col[] = Bytes.toBytes("col1");
 
-    Put put = new Put(row);
-    put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
-    region.put(put);
-    region.flushcache();
+      Put put = new Put(row);
+      put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
+      region.put(put);
+      region.flushcache();
 
-    Delete del = new Delete(row);
-    region.delete(del, null, true);
-    region.flushcache();
+      Delete del = new Delete(row);
+      region.delete(del, null, true);
+      region.flushcache();
 
-    // Get remaining rows (should have none)
-    Get get = new Get(row);
-    get.addColumn(familyName, col);
+      // Get remaining rows (should have none)
+      Get get = new Get(row);
+      get.addColumn(familyName, col);
 
-    KeyValue[] keyValues = region.get(get, null).raw();
-    assertTrue(keyValues.length == 0);
+      KeyValue[] keyValues = region.get(get, null).raw();
+      assertTrue(keyValues.length == 0);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
   }
 
   @Test public void testgetHDFSBlocksDistribution() throws Exception {
@@ -3284,17 +3426,17 @@ public void testDeleteRowWithBloomFilter() throws IOException {
       ht.put(put);
 
       HRegion firstRegion = htu.getHBaseCluster().
-        getRegions(Bytes.toBytes(this.getName())).get(0);
+          getRegions(Bytes.toBytes(this.getName())).get(0);
       firstRegion.flushcache();
       HDFSBlocksDistribution blocksDistribution1 =
-        firstRegion.getHDFSBlocksDistribution();
+          firstRegion.getHDFSBlocksDistribution();
 
       // given the default replication factor is 2 and we have 2 HFiles,
       // we will have total of 4 replica of blocks on 3 datanodes; thus there
       // must be at least one host that have replica for 2 HFiles. That host's
       // weight will be equal to the unique block weight.
       long uniqueBlocksWeight1 =
-        blocksDistribution1.getUniqueBlocksTotalWeight();
+          blocksDistribution1.getUniqueBlocksTotalWeight();
 
       String topHost = blocksDistribution1.getTopHosts().get(0);
       long topHostWeight = blocksDistribution1.getWeight(topHost);
@@ -3418,19 +3560,44 @@ private Configuration initSplit() {
     return conf;
   }
 
-  private void initHRegion (byte [] tableName, String callingMethod,
+  /**
+   * @param tableName
+   * @param callingMethod
+   * @param families
+   * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
+   * @throws IOException
+   */
+  private static HRegion initHRegion (byte [] tableName, String callingMethod,
     byte[] ... families)
   throws IOException {
-    initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
+    return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
   }
 
-  private void initHRegion (byte [] tableName, String callingMethod,
+  /**
+   * @param tableName
+   * @param callingMethod
+   * @param conf
+   * @param families
+   * @throws IOException
+   * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
+   */
+  private static HRegion initHRegion (byte [] tableName, String callingMethod,
       Configuration conf, byte [] ... families)
     throws IOException{
-    initHRegion(tableName, null, null, callingMethod, conf, families);
+    return initHRegion(tableName, null, null, callingMethod, conf, families);
   }
 
-  private void initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
+  /**
+   * @param tableName
+   * @param startKey
+   * @param stopKey
+   * @param callingMethod
+   * @param conf
+   * @param families
+   * @throws IOException
+   * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
+   */
+  private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
       String callingMethod, Configuration conf, byte[]... families)
       throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
@@ -3439,12 +3606,13 @@ private void initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
     }
     HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false);
     Path path = new Path(DIR + callingMethod);
+    FileSystem fs = FileSystem.get(conf);
     if (fs.exists(path)) {
       if (!fs.delete(path, true)) {
         throw new IOException("Failed delete of " + path);
       }
     }
-    region = HRegion.createHRegion(info, path, conf, htd);
+    return HRegion.createHRegion(info, path, conf, htd);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
new file mode 100644
index 000000000000..681634c61a2a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.MediumTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests that need to spin up a cluster testing an {@link HRegion}.  Use
+ * {@link TestHRegion} if you don't need a cluster, if you can test w/ a
+ * standalone {@link HRegion}.
+ */
+@Category(MediumTests.class)
+public class TestHRegionOnCluster {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @Test (timeout=180000)
+  public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
+    final int NUM_MASTERS = 1;
+    final int NUM_RS = 3;
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+
+    try {
+      final byte[] TABLENAME = Bytes
+          .toBytes("testDataCorrectnessReplayingRecoveredEdits");
+      final byte[] FAMILY = Bytes.toBytes("family");
+      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+      HMaster master = cluster.getMaster();
+
+      // Create table
+      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+      desc.addFamily(new HColumnDescriptor(FAMILY));
+      HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin();
+      hbaseAdmin.createTable(desc);
+
+      assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+
+      // Put data: r1->v1
+      HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
+      putDataAndVerify(table, "r1", FAMILY, "v1", 1);
+
+      // Move region to target server
+      HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo();
+      int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
+      HRegionServer originServer = cluster.getRegionServer(originServerNum);
+      int targetServerNum = (originServerNum + 1) % NUM_RS;
+      HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
+      assertFalse(originServer.equals(targetServer));
+      hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
+          Bytes.toBytes(targetServer.getServerName().getServerName()));
+      do {
+        Thread.sleep(1);
+      } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
+
+      // Put data: r2->v2
+      putDataAndVerify(table, "r2", FAMILY, "v2", 2);
+
+      // Move region to origin server
+      hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
+          Bytes.toBytes(originServer.getServerName().getServerName()));
+      do {
+        Thread.sleep(1);
+      } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);
+
+      // Put data: r3->v3
+      putDataAndVerify(table, "r3", FAMILY, "v3", 3);
+
+      // Kill target server
+      targetServer.kill();
+      cluster.getRegionServerThreads().get(targetServerNum).join();
+      // Wait until finish processing of shutdown
+      while (master.getServerManager().areDeadServersInProgress()) {
+        Thread.sleep(5);
+      }
+      // Kill origin server
+      originServer.kill();
+      cluster.getRegionServerThreads().get(originServerNum).join();
+
+      // Put data: r4->v4
+      putDataAndVerify(table, "r4", FAMILY, "v4", 4);
+
+    } finally {
+      TEST_UTIL.shutdownMiniCluster();
+    }
+  }
+
+  private void putDataAndVerify(HTable table, String row, byte[] family,
+      String value, int verifyNum) throws IOException {
+    System.out.println("=========Putting data :" + row);
+    Put put = new Put(Bytes.toBytes(row));
+    put.add(family, Bytes.toBytes("q1"), Bytes.toBytes(value));
+    table.put(put);
+    ResultScanner resultScanner = table.getScanner(new Scan());
+    List<Result> results = new ArrayList<Result>();
+    while (true) {
+      Result r = resultScanner.next();
+      if (r == null)
+        break;
+      results.add(r);
+    }
+    resultScanner.close();
+    if (results.size() != verifyNum) {
+      System.out.println(results);
+    }
+    assertEquals(verifyNum, results.size());
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
index 33c78ab7bc9b..631d7706c117 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
@@ -52,40 +52,45 @@ public class TestMinVersions extends HBaseTestCase {
   public void testGetClosestBefore() throws Exception {
     HTableDescriptor htd = createTableDescriptor(getName(), 1, 1000, 1, false);
     HRegion region = createNewHRegion(htd, null, null);
+    try {
 
-    // 2s in the past
-    long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
+      // 2s in the past
+      long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
 
-    Put p = new Put(T1, ts);
-    p.add(c0, c0, T1);
-    region.put(p);
+      Put p = new Put(T1, ts);
+      p.add(c0, c0, T1);
+      region.put(p);
 
-    p = new Put(T1, ts+1);
-    p.add(c0, c0, T4);
-    region.put(p);
+      p = new Put(T1, ts+1);
+      p.add(c0, c0, T4);
+      region.put(p);
 
-    p = new Put(T3, ts);
-    p.add(c0, c0, T3);
-    region.put(p);
+      p = new Put(T3, ts);
+      p.add(c0, c0, T3);
+      region.put(p);
 
-    // now make sure that getClosestBefore(...) get can
-    // rows that would be expired without minVersion.
-    // also make sure it gets the latest version
-    Result r = region.getClosestRowBefore(T1, c0);
-    checkResult(r, c0, T4);
+      // now make sure that getClosestBefore(...) get can
+      // rows that would be expired without minVersion.
+      // also make sure it gets the latest version
+      Result r = region.getClosestRowBefore(T1, c0);
+      checkResult(r, c0, T4);
 
-    r = region.getClosestRowBefore(T2, c0);
-    checkResult(r, c0, T4);
+      r = region.getClosestRowBefore(T2, c0);
+      checkResult(r, c0, T4);
 
-    // now flush/compact
-    region.flushcache();
-    region.compactStores(true);
+      // now flush/compact
+      region.flushcache();
+      region.compactStores(true);
 
-    r = region.getClosestRowBefore(T1, c0);
-    checkResult(r, c0, T4);
+      r = region.getClosestRowBefore(T1, c0);
+      checkResult(r, c0, T4);
 
-    r = region.getClosestRowBefore(T2, c0);
-    checkResult(r, c0, T4);
+      r = region.getClosestRowBefore(T2, c0);
+      checkResult(r, c0, T4);
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
   }
 
   /**
@@ -96,48 +101,52 @@ public void testStoreMemStore() throws Exception {
     // keep 3 versions minimum
     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false);
     HRegion region = createNewHRegion(htd, null, null);
-
     // 2s in the past
     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
 
-    Put p = new Put(T1, ts-1);
-    p.add(c0, c0, T2);
-    region.put(p);
-
-    p = new Put(T1, ts-3);
-    p.add(c0, c0, T0);
-    region.put(p);
-
-    // now flush/compact
-    region.flushcache();
-    region.compactStores(true);
-
-    p = new Put(T1, ts);
-    p.add(c0, c0, T3);
-    region.put(p);
-
-    p = new Put(T1, ts-2);
-    p.add(c0, c0, T1);
-    region.put(p);
-
-    p = new Put(T1, ts-3);
-    p.add(c0, c0, T0);
-    region.put(p);
-
-    // newest version in the memstore
-    // the 2nd oldest in the store file
-    // and the 3rd, 4th oldest also in the memstore
-
-    Get g = new Get(T1);
-    g.setMaxVersions();
-    Result r = region.get(g, null); // this'll use ScanWildcardColumnTracker
-    checkResult(r, c0, T3,T2,T1);
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    g.addColumn(c0, c0);
-    r = region.get(g, null);  // this'll use ExplicitColumnTracker
-    checkResult(r, c0, T3,T2,T1);
+    try {
+      Put p = new Put(T1, ts-1);
+      p.add(c0, c0, T2);
+      region.put(p);
+
+      p = new Put(T1, ts-3);
+      p.add(c0, c0, T0);
+      region.put(p);
+
+      // now flush/compact
+      region.flushcache();
+      region.compactStores(true);
+
+      p = new Put(T1, ts);
+      p.add(c0, c0, T3);
+      region.put(p);
+
+      p = new Put(T1, ts-2);
+      p.add(c0, c0, T1);
+      region.put(p);
+
+      p = new Put(T1, ts-3);
+      p.add(c0, c0, T0);
+      region.put(p);
+
+      // newest version in the memstore
+      // the 2nd oldest in the store file
+      // and the 3rd, 4th oldest also in the memstore
+
+      Get g = new Get(T1);
+      g.setMaxVersions();
+      Result r = region.get(g, null); // this'll use ScanWildcardColumnTracker
+      checkResult(r, c0, T3,T2,T1);
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      g.addColumn(c0, c0);
+      r = region.get(g, null);  // this'll use ExplicitColumnTracker
+      checkResult(r, c0, T3,T2,T1);
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
   }
 
   /**
@@ -150,47 +159,52 @@ public void testDelete() throws Exception {
     // 2s in the past
     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
 
-    Put p = new Put(T1, ts-2);
-    p.add(c0, c0, T1);
-    region.put(p);
-
-    p = new Put(T1, ts-1);
-    p.add(c0, c0, T2);
-    region.put(p);
-
-    p = new Put(T1, ts);
-    p.add(c0, c0, T3);
-    region.put(p);
-
-    Delete d = new Delete(T1, ts-1, null);
-    region.delete(d, null, true);
-
-    Get g = new Get(T1);
-    g.setMaxVersions();
-    Result r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
-    checkResult(r, c0, T3);
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    g.addColumn(c0, c0);
-    r = region.get(g, null);  // this'll use ExplicitColumnTracker
-    checkResult(r, c0, T3);
-
-    // now flush/compact
-    region.flushcache();
-    region.compactStores(true);
-
-    // try again
-    g = new Get(T1);
-    g.setMaxVersions();
-    r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
-    checkResult(r, c0, T3);
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    g.addColumn(c0, c0);
-    r = region.get(g, null);  // this'll use ExplicitColumnTracker
-    checkResult(r, c0, T3);
+    try {
+      Put p = new Put(T1, ts-2);
+      p.add(c0, c0, T1);
+      region.put(p);
+
+      p = new Put(T1, ts-1);
+      p.add(c0, c0, T2);
+      region.put(p);
+
+      p = new Put(T1, ts);
+      p.add(c0, c0, T3);
+      region.put(p);
+
+      Delete d = new Delete(T1, ts-1, null);
+      region.delete(d, null, true);
+
+      Get g = new Get(T1);
+      g.setMaxVersions();
+      Result r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
+      checkResult(r, c0, T3);
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      g.addColumn(c0, c0);
+      r = region.get(g, null);  // this'll use ExplicitColumnTracker
+      checkResult(r, c0, T3);
+
+      // now flush/compact
+      region.flushcache();
+      region.compactStores(true);
+
+      // try again
+      g = new Get(T1);
+      g.setMaxVersions();
+      r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
+      checkResult(r, c0, T3);
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      g.addColumn(c0, c0);
+      r = region.get(g, null);  // this'll use ExplicitColumnTracker
+      checkResult(r, c0, T3);
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
   }
 
   /**
@@ -203,63 +217,68 @@ public void testMemStore() throws Exception {
     // 2s in the past
     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
 
-    // 2nd version
-    Put p = new Put(T1, ts-2);
-    p.add(c0, c0, T2);
-    region.put(p);
-
-    // 3rd version
-    p = new Put(T1, ts-1);
-    p.add(c0, c0, T3);
-    region.put(p);
-
-    // 4th version
-    p = new Put(T1, ts);
-    p.add(c0, c0, T4);
-    region.put(p);
-
-    // now flush/compact
-    region.flushcache();
-    region.compactStores(true);
-
-    // now put the first version (backdated)
-    p = new Put(T1, ts-3);
-    p.add(c0, c0, T1);
-    region.put(p);
-
-    // now the latest change is in the memstore,
-    // but it is not the latest version
-
-    Result r = region.get(new Get(T1), null);
-    checkResult(r, c0, T4);
-
-    Get g = new Get(T1);
-    g.setMaxVersions();
-    r = region.get(g, null); // this'll use ScanWildcardColumnTracker
-    checkResult(r, c0, T4,T3);
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    g.addColumn(c0, c0);
-    r = region.get(g, null);  // this'll use ExplicitColumnTracker
-    checkResult(r, c0, T4,T3);
-
-    p = new Put(T1, ts+1);
-    p.add(c0, c0, T5);
-    region.put(p);
-
-    // now the latest version is in the memstore
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
-    checkResult(r, c0, T5,T4);
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    g.addColumn(c0, c0);
-    r = region.get(g, null);  // this'll use ExplicitColumnTracker
-    checkResult(r, c0, T5,T4);
+    try {
+      // 2nd version
+      Put p = new Put(T1, ts-2);
+      p.add(c0, c0, T2);
+      region.put(p);
+
+      // 3rd version
+      p = new Put(T1, ts-1);
+      p.add(c0, c0, T3);
+      region.put(p);
+
+      // 4th version
+      p = new Put(T1, ts);
+      p.add(c0, c0, T4);
+      region.put(p);
+
+      // now flush/compact
+      region.flushcache();
+      region.compactStores(true);
+
+      // now put the first version (backdated)
+      p = new Put(T1, ts-3);
+      p.add(c0, c0, T1);
+      region.put(p);
+
+      // now the latest change is in the memstore,
+      // but it is not the latest version
+
+      Result r = region.get(new Get(T1), null);
+      checkResult(r, c0, T4);
+
+      Get g = new Get(T1);
+      g.setMaxVersions();
+      r = region.get(g, null); // this'll use ScanWildcardColumnTracker
+      checkResult(r, c0, T4,T3);
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      g.addColumn(c0, c0);
+      r = region.get(g, null);  // this'll use ExplicitColumnTracker
+      checkResult(r, c0, T4,T3);
+
+      p = new Put(T1, ts+1);
+      p.add(c0, c0, T5);
+      region.put(p);
+
+      // now the latest version is in the memstore
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
+      checkResult(r, c0, T5,T4);
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      g.addColumn(c0, c0);
+      r = region.get(g, null);  // this'll use ExplicitColumnTracker
+      checkResult(r, c0, T5,T4);
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
   }
 
   /**
@@ -269,83 +288,88 @@ public void testBaseCase() throws Exception {
     // 1 version minimum, 1000 versions maximum, ttl = 1s
     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
     HRegion region = createNewHRegion(htd, null, null);
-
-    // 2s in the past
-    long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
-
-     // 1st version
-    Put p = new Put(T1, ts-3);
-    p.add(c0, c0, T1);
-    region.put(p);
-
-    // 2nd version
-    p = new Put(T1, ts-2);
-    p.add(c0, c0, T2);
-    region.put(p);
-
-    // 3rd version
-    p = new Put(T1, ts-1);
-    p.add(c0, c0, T3);
-    region.put(p);
-
-    // 4th version
-    p = new Put(T1, ts);
-    p.add(c0, c0, T4);
-    region.put(p);
-
-    Result r = region.get(new Get(T1), null);
-    checkResult(r, c0, T4);
-
-    Get g = new Get(T1);
-    g.setTimeRange(0L, ts+1);
-    r = region.get(g, null);
-    checkResult(r, c0, T4);
-
-    // oldest version still exists
-    g.setTimeRange(0L, ts-2);
-    r = region.get(g, null);
-    checkResult(r, c0, T1);
-
-    // gets see only available versions
-    // even before compactions
-    g = new Get(T1);
-    g.setMaxVersions();
-    r = region.get(g, null); // this'll use ScanWildcardColumnTracker
-    checkResult(r, c0, T4,T3);
-
-    g = new Get(T1);
-    g.setMaxVersions();
-    g.addColumn(c0, c0);
-    r = region.get(g, null);  // this'll use ExplicitColumnTracker
-    checkResult(r, c0, T4,T3);
-
-    // now flush
-    region.flushcache();
-
-    // with HBASE-4241 a flush will eliminate the expired rows
-    g = new Get(T1);
-    g.setTimeRange(0L, ts-2);
-    r = region.get(g, null);
-    assertTrue(r.isEmpty());
-
-    // major compaction
-    region.compactStores(true);
-
-    // after compaction the 4th version is still available
-    g = new Get(T1);
-    g.setTimeRange(0L, ts+1);
-    r = region.get(g, null);
-    checkResult(r, c0, T4);
-
-    // so is the 3rd
-    g.setTimeRange(0L, ts);
-    r = region.get(g, null);
-    checkResult(r, c0, T3);
-
-    // but the 2nd and earlier versions are gone
-    g.setTimeRange(0L, ts-1);
-    r = region.get(g, null);
-    assertTrue(r.isEmpty());
+    try {
+
+      // 2s in the past
+      long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
+
+       // 1st version
+      Put p = new Put(T1, ts-3);
+      p.add(c0, c0, T1);
+      region.put(p);
+
+      // 2nd version
+      p = new Put(T1, ts-2);
+      p.add(c0, c0, T2);
+      region.put(p);
+
+      // 3rd version
+      p = new Put(T1, ts-1);
+      p.add(c0, c0, T3);
+      region.put(p);
+
+      // 4th version
+      p = new Put(T1, ts);
+      p.add(c0, c0, T4);
+      region.put(p);
+
+      Result r = region.get(new Get(T1), null);
+      checkResult(r, c0, T4);
+
+      Get g = new Get(T1);
+      g.setTimeRange(0L, ts+1);
+      r = region.get(g, null);
+      checkResult(r, c0, T4);
+
+  // oldest version still exists
+      g.setTimeRange(0L, ts-2);
+      r = region.get(g, null);
+      checkResult(r, c0, T1);
+
+      // gets see only available versions
+      // even before compactions
+      g = new Get(T1);
+      g.setMaxVersions();
+      r = region.get(g, null); // this'll use ScanWildcardColumnTracker
+      checkResult(r, c0, T4,T3);
+
+      g = new Get(T1);
+      g.setMaxVersions();
+      g.addColumn(c0, c0);
+      r = region.get(g, null);  // this'll use ExplicitColumnTracker
+      checkResult(r, c0, T4,T3);
+
+      // now flush
+      region.flushcache();
+
+      // with HBASE-4241 a flush will eliminate the expired rows
+      g = new Get(T1);
+      g.setTimeRange(0L, ts-2);
+      r = region.get(g, null);
+      assertTrue(r.isEmpty());
+
+      // major compaction
+      region.compactStores(true);
+
+      // after compaction the 4th version is still available
+      g = new Get(T1);
+      g.setTimeRange(0L, ts+1);
+      r = region.get(g, null);
+      checkResult(r, c0, T4);
+
+      // so is the 3rd
+      g.setTimeRange(0L, ts);
+      r = region.get(g, null);
+      checkResult(r, c0, T3);
+
+      // but the 2nd and earlier versions are gone
+      g.setTimeRange(0L, ts-1);
+      r = region.get(g, null);
+      assertTrue(r.isEmpty());
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
   }
 
   /**
@@ -359,62 +383,67 @@ public void testFilters() throws Exception {
 
     // 2s in the past
     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
-
-    Put p = new Put(T1, ts-3);
-    p.add(c0, c0, T0);
-    p.add(c1, c1, T0);
-    region.put(p);
-
-    p = new Put(T1, ts-2);
-    p.add(c0, c0, T1);
-    p.add(c1, c1, T1);
-    region.put(p);
-
-    p = new Put(T1, ts-1);
-    p.add(c0, c0, T2);
-    p.add(c1, c1, T2);
-    region.put(p);
-
-    p = new Put(T1, ts);
-    p.add(c0, c0, T3);
-    p.add(c1, c1, T3);
-    region.put(p);
-
-    List<Long> tss = new ArrayList<Long>();
-    tss.add(ts-1);
-    tss.add(ts-2);
-
-    Get g = new Get(T1);
-    g.addColumn(c1,c1);
-    g.setFilter(new TimestampsFilter(tss));
-    g.setMaxVersions();
-    Result r = region.get(g, null);
-    checkResult(r, c1, T2,T1);
-
-    g = new Get(T1);
-    g.addColumn(c0,c0);
-    g.setFilter(new TimestampsFilter(tss));
-    g.setMaxVersions();
-    r = region.get(g, null);
-    checkResult(r, c0, T2,T1);
-
-    // now flush/compact
-    region.flushcache();
-    region.compactStores(true);
-
-    g = new Get(T1);
-    g.addColumn(c1,c1);
-    g.setFilter(new TimestampsFilter(tss));
-    g.setMaxVersions();
-    r = region.get(g, null);
-    checkResult(r, c1, T2);
-
-    g = new Get(T1);
-    g.addColumn(c0,c0);
-    g.setFilter(new TimestampsFilter(tss));
-    g.setMaxVersions();
-    r = region.get(g, null);
-    checkResult(r, c0, T2);
+    try {
+
+      Put p = new Put(T1, ts-3);
+      p.add(c0, c0, T0);
+      p.add(c1, c1, T0);
+      region.put(p);
+
+      p = new Put(T1, ts-2);
+      p.add(c0, c0, T1);
+      p.add(c1, c1, T1);
+      region.put(p);
+
+      p = new Put(T1, ts-1);
+      p.add(c0, c0, T2);
+      p.add(c1, c1, T2);
+      region.put(p);
+
+      p = new Put(T1, ts);
+      p.add(c0, c0, T3);
+      p.add(c1, c1, T3);
+      region.put(p);
+
+      List<Long> tss = new ArrayList<Long>();
+      tss.add(ts-1);
+      tss.add(ts-2);
+
+      Get g = new Get(T1);
+      g.addColumn(c1,c1);
+      g.setFilter(new TimestampsFilter(tss));
+      g.setMaxVersions();
+      Result r = region.get(g, null);
+      checkResult(r, c1, T2,T1);
+
+      g = new Get(T1);
+      g.addColumn(c0,c0);
+      g.setFilter(new TimestampsFilter(tss));
+      g.setMaxVersions();
+      r = region.get(g, null);
+      checkResult(r, c0, T2,T1);
+
+      // now flush/compact
+      region.flushcache();
+      region.compactStores(true);
+
+      g = new Get(T1);
+      g.addColumn(c1,c1);
+      g.setFilter(new TimestampsFilter(tss));
+      g.setMaxVersions();
+      r = region.get(g, null);
+      checkResult(r, c1, T2);
+
+      g = new Get(T1);
+      g.addColumn(c0,c0);
+      g.setFilter(new TimestampsFilter(tss));
+      g.setMaxVersions();
+      r = region.get(g, null);
+      checkResult(r, c0, T2);
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
   }
 
   private void checkResult(Result r, byte[] col, byte[] ... vals) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
index c98efe50eec1..96afbd2935df 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
@@ -68,31 +68,34 @@ public void testResettingCounters() throws Exception {
       }
     }
     HRegion region = HRegion.createHRegion(hri, path, conf, htd);
+    try {
+      Increment odd = new Increment(rows[0]);
+      Increment even = new Increment(rows[0]);
+      Increment all = new Increment(rows[0]);
+      for (int i=0;i<numQualifiers;i++) {
+        if (i % 2 == 0) even.addColumn(families[0], qualifiers[i], 1);
+        else odd.addColumn(families[0], qualifiers[i], 1);
+        all.addColumn(families[0], qualifiers[i], 1);
+      }
 
-    Increment odd = new Increment(rows[0]);
-    Increment even = new Increment(rows[0]);
-    Increment all = new Increment(rows[0]);
-    for (int i=0;i<numQualifiers;i++) {
-      if (i % 2 == 0) even.addColumn(families[0], qualifiers[i], 1);
-      else odd.addColumn(families[0], qualifiers[i], 1);
-      all.addColumn(families[0], qualifiers[i], 1);
-    }
-
-    // increment odd qualifiers 5 times and flush
-    for (int i=0;i<5;i++) region.increment(odd, null, false);
-    region.flushcache();
+      // increment odd qualifiers 5 times and flush
+      for (int i=0;i<5;i++) region.increment(odd, null, false);
+      region.flushcache();
 
-    // increment even qualifiers 5 times
-    for (int i=0;i<5;i++) region.increment(even, null, false);
+      // increment even qualifiers 5 times
+      for (int i=0;i<5;i++) region.increment(even, null, false);
 
-    // increment all qualifiers, should have value=6 for all
-    Result result = region.increment(all, null, false);
-    assertEquals(numQualifiers, result.size());
-    KeyValue [] kvs = result.raw();
-    for (int i=0;i<kvs.length;i++) {
-      System.out.println(kvs[i].toString());
-      assertTrue(Bytes.equals(kvs[i].getQualifier(), qualifiers[i]));
-      assertEquals(6, Bytes.toLong(kvs[i].getValue()));
+      // increment all qualifiers, should have value=6 for all
+      Result result = region.increment(all, null, false);
+      assertEquals(numQualifiers, result.size());
+      KeyValue [] kvs = result.raw();
+      for (int i=0;i<kvs.length;i++) {
+        System.out.println(kvs[i].toString());
+        assertTrue(Bytes.equals(kvs[i].getQualifier(), qualifiers[i]));
+        assertEquals(6, Bytes.toLong(kvs[i].getValue()));
+      }
+    } finally {
+      HRegion.closeHRegion(region);
     }
     region.close();
     region.getLog().closeAndDelete();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index 1de88277c2e5..b2b7cceb9361 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -319,7 +319,9 @@ HRegion createRegion(final Path testdir, final HLog wal)
     HColumnDescriptor hcd = new HColumnDescriptor(CF);
     htd.addFamily(hcd);
     HRegionInfo hri = new HRegionInfo(htd.getName(), STARTROW, ENDROW);
-    HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
+    HRegion r = HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
+    r.close();
+    r.getLog().closeAndDelete();
     return HRegion.openHRegion(testdir, hri, htd, wal,
       TEST_UTIL.getConfiguration());
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
index ad5511c98675..07f8fc4d7696 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
@@ -99,28 +99,32 @@ public void setupHRI() {
     HRegion region =
       HRegion.createHRegion(hri, HTU.getDataTestDir(),
         HTU.getConfiguration(), htd);
-    assertNotNull(region);
-    // Spy on the region so can throw exception when close is called.
-    HRegion spy = Mockito.spy(region);
-    final boolean abort = false;
-    Mockito.when(spy.close(abort)).
-      thenThrow(new RuntimeException("Mocked failed close!"));
-    // The CloseRegionHandler will try to get an HRegion that corresponds
-    // to the passed hri -- so insert the region into the online region Set.
-    rss.addToOnlineRegions(spy);
-    // Assert the Server is NOT stopped before we call close region.
-    assertFalse(server.isStopped());
-    CloseRegionHandler handler =
-      new CloseRegionHandler(server, rss, hri, false, false, -1);
-    boolean throwable = false;
     try {
-      handler.process();
-    } catch (Throwable t) {
-      throwable = true;
+      assertNotNull(region);
+      // Spy on the region so can throw exception when close is called.
+      HRegion spy = Mockito.spy(region);
+      final boolean abort = false;
+      Mockito.when(spy.close(abort)).
+      thenThrow(new RuntimeException("Mocked failed close!"));
+      // The CloseRegionHandler will try to get an HRegion that corresponds
+      // to the passed hri -- so insert the region into the online region Set.
+      rss.addToOnlineRegions(spy);
+      // Assert the Server is NOT stopped before we call close region.
+      assertFalse(server.isStopped());
+      CloseRegionHandler handler =
+          new CloseRegionHandler(server, rss, hri, false, false, -1);
+      boolean throwable = false;
+      try {
+        handler.process();
+      } catch (Throwable t) {
+        throwable = true;
+      } finally {
+        assertTrue(throwable);
+        // Abort calls stop so stopped flag should be set.
+        assertTrue(server.isStopped());
+      }
     } finally {
-      assertTrue(throwable);
-      // Abort calls stop so stopped flag should be set.
-      assertTrue(server.isStopped());
+      HRegion.closeHRegion(region);
     }
   }
   
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
index 8988f83775e3..55a8c4a54a77 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
@@ -98,30 +98,34 @@ public void setupHRI() {
          HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU
             .getConfiguration(), htd);
     assertNotNull(region);
-    OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd) {
-      HRegion openRegion() {
-        // Open region first, then remove znode as though it'd been hijacked.
-        HRegion region = super.openRegion();
-        
-        // Don't actually open region BUT remove the znode as though it'd
-        // been hijacked on us.
-        ZooKeeperWatcher zkw = this.server.getZooKeeper();
-        String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
-        try {
-          ZKUtil.deleteNodeFailSilent(zkw, node);
-        } catch (KeeperException e) {
-          throw new RuntimeException("Ugh failed delete of " + node, e);
+    try {
+      OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd) {
+        HRegion openRegion() {
+          // Open region first, then remove znode as though it'd been hijacked.
+          HRegion region = super.openRegion();
+
+          // Don't actually open region BUT remove the znode as though it'd
+          // been hijacked on us.
+          ZooKeeperWatcher zkw = this.server.getZooKeeper();
+          String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
+          try {
+            ZKUtil.deleteNodeFailSilent(zkw, node);
+          } catch (KeeperException e) {
+            throw new RuntimeException("Ugh failed delete of " + node, e);
+          }
+          return region;
         }
-        return region;
-      }
-    };
-    // Call process without first creating OFFLINE region in zk, see if
-    // exception or just quiet return (expected).
-    handler.process();
-    ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
-    // Call process again but this time yank the zk znode out from under it
-    // post OPENING; again will expect it to come back w/o NPE or exception.
-    handler.process();
+      };
+      // Call process without first creating OFFLINE region in zk, see if
+      // exception or just quiet return (expected).
+      handler.process();
+      ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
+      // Call process again but this time yank the zk znode out from under it
+      // post OPENING; again will expect it to come back w/o NPE or exception.
+      handler.process();
+    } finally {
+      HRegion.closeHRegion(region);
+    }
   }
   
   @Test
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index e5946974eed8..4fdf34dc4cdc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -133,7 +133,8 @@ public void test2727() throws Exception {
     HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
     HRegion region2 = HRegion.createHRegion(hri,
         hbaseRootDir, this.conf, htd);
-
+    region2.close();
+    region2.getLog().closeAndDelete();
     final byte [] tableName = Bytes.toBytes(tableNameStr);
     final byte [] rowName = tableName;
 
@@ -193,6 +194,8 @@ public void testRegionMadeOfBulkLoadedFilesOnly()
     final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
     HRegion region2 = HRegion.createHRegion(hri,
         hbaseRootDir, this.conf, htd);
+    region2.close();
+    region2.getLog().closeAndDelete();
     HLog wal = createWAL(this.conf);
     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
     Path f =  new Path(basedir, "hfile");
@@ -252,7 +255,8 @@ public void testReplayEditsWrittenViaHRegion()
     final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
     HRegion region3 = HRegion.createHRegion(hri,
             hbaseRootDir, this.conf, htd);
-
+    region3.close();
+    region3.getLog().closeAndDelete();
     // Write countPerFamily edits into the three families.  Do a flush on one
     // of the families during the load of edits so its seqid is not same as
     // others to test we do right thing when different seqids.
@@ -369,7 +373,8 @@ public void testReplayEditsAfterPartialFlush()
     final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
     HRegion region3 = HRegion.createHRegion(hri,
             hbaseRootDir, this.conf, htd);
-
+    region3.close();
+    region3.getLog().closeAndDelete();
     // Write countPerFamily edits into the three families.  Do a flush on one
     // of the families during the load of edits so its seqid is not same as
     // others to test we do right thing when different seqids.
@@ -435,7 +440,8 @@ public void testReplayEditsWrittenIntoWAL() throws Exception {
     final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
     HRegion region2 = HRegion.createHRegion(hri,
             hbaseRootDir, this.conf, htd);
-
+    region2.close();
+    region2.getLog().closeAndDelete();
     final HLog wal = createWAL(this.conf);
     final byte[] tableName = Bytes.toBytes(tableNameStr);
     final byte[] rowName = tableName;
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index 465dc95292d5..f5f0ac7826c7 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -169,6 +169,13 @@ public void setUp() throws Exception {
   @Override
   public void tearDown() throws Exception {
     super.tearDown();
+    for (int i = 0; i < sourceRegions.length; i++) {
+      HRegion r = regions[i];
+      if (r != null) {
+        r.close();
+        r.getLog().closeAndDelete();
+      }
+    }
     TEST_UTIL.shutdownMiniCluster();
   }
 

From cede407b1a0ef892407f8c0dfbe5cbfe25b1034f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 23 Apr 2012 21:54:41 +0000
Subject: [PATCH 0152/1540] HBASE-5857 RIT map in RS not getting cleared while
 region opening

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329471 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     |  2 +-
 .../master/TestZKBasedOpenCloseRegion.java    | 28 +++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index db2300864217..a6f9e5b78e23 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2715,9 +2715,9 @@ public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNod
     }
     LOG.info("Received request to open region: " +
       region.getRegionNameAsString());
+    HTableDescriptor htd = this.tableDescriptors.get(region.getTableName());
     this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(),
         true);
-    HTableDescriptor htd = this.tableDescriptors.get(region.getTableName());
     // Need to pass the expected version in the constructor.
     if (region.isRootRegion()) {
       this.service.submit(new OpenRootHandler(this, this, region, htd,
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
index c8e523f5e56e..8c3f67e1e490 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
@@ -47,9 +47,13 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertFalse;
 
 /**
  * Test open and close of regions using zk.
@@ -308,6 +312,30 @@ public void testRSAlreadyProcessingRegion() throws Exception {
     LOG.info("Done with testCloseRegion");
   }
 
+  /**
+   * If region open fails with IOException in openRegion() while doing tableDescriptors.get()
+   * the region should not add into regionsInTransitionInRS map
+   * @throws Exception
+   */
+  @Test
+  public void testRegionOpenFailsDueToIOException() throws Exception {
+    HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"),
+        HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
+    HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
+    TableDescriptors htd = Mockito.mock(TableDescriptors.class);
+    Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors");
+    Whitebox.setInternalState(regionServer, "tableDescriptors", htd);
+    Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any());
+    try {
+      regionServer.openRegion(REGIONINFO);
+      fail("It should throw IOException ");
+    } catch (IOException e) {
+    }
+    Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState);
+    assertFalse("Region should not be in RIT",
+        regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes()));
+  }
+  
   private static void waitUntilAllRegionsAssigned()
   throws IOException {
     HTable meta = new HTable(TEST_UTIL.getConfiguration(),

From e983d4f0a72caec2d2a2383dfa9bb564c9ad9295 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 01:52:16 +0000
Subject: [PATCH 0153/1540] HBASE-5849 On first cluster startup, RS aborts if
 root znode is not available

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329528 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     |   6 -
 .../hadoop/hbase/TestClusterBootOrder.java    | 118 ++++++++++++++++++
 2 files changed, 118 insertions(+), 6 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a6f9e5b78e23..53f7a9a392df 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -605,12 +605,6 @@ private void initializeZooKeeper() throws IOException, InterruptedException {
    */
   private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
       throws IOException, InterruptedException {
-    if (false == tracker.checkIfBaseNodeAvailable()) {
-      String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. "
-          + "There could be a mismatch with the one configured in the master.";
-      LOG.error(errorMsg);
-      abort(errorMsg);
-    }
     while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
       if (this.stopped) {
         throw new IOException("Received the shutdown message while waiting.");
diff --git a/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
new file mode 100644
index 000000000000..3bbbdf2826fe
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests the boot order indifference between regionserver and master
+ */
+@Category(MediumTests.class)
+public class TestClusterBootOrder {
+
+  private static final long SLEEP_INTERVAL = 1000;
+  private static final long SLEEP_TIME = 4000;
+
+  private HBaseTestingUtility testUtil;
+  private LocalHBaseCluster cluster;
+  private RegionServerThread rs;
+  private MasterThread master;
+
+  @Before
+  public void setUp() throws Exception {
+    testUtil = new HBaseTestingUtility();
+    testUtil.startMiniZKCluster(1);
+    cluster = new LocalHBaseCluster(testUtil.getConfiguration(), 0, 0);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    cluster.shutdown();
+    cluster.join();
+    testUtil.shutdownMiniZKCluster();
+  }
+
+  private void startRegionServer() throws Exception {
+    rs = cluster.addRegionServer();
+    rs.start();
+
+    for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) {
+      //we cannot block on wait for rs at this point , since master is not up.
+      Thread.sleep(SLEEP_INTERVAL);
+      assertTrue(rs.isAlive());
+    }
+  }
+
+  private void startMaster() throws Exception {
+    master = cluster.addMaster();
+    master.start();
+
+    for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) {
+      Thread.sleep(SLEEP_INTERVAL);
+      assertTrue(master.isAlive());
+    }
+  }
+
+  private void waitForClusterOnline() {
+    while (true) {
+      if (master.getMaster().isInitialized()) {
+        break;
+      }
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException ignored) {
+        // Keep waiting
+      }
+    }
+    rs.waitForServerOnline();
+  }
+
+  /**
+   * Tests launching the cluster by first starting regionserver, and then the master
+   * to ensure that it does not matter which is started first.
+   */
+  @Test(timeout = 12000)
+  public void testBootRegionServerFirst() throws Exception {
+    startRegionServer();
+    startMaster();
+    waitForClusterOnline();
+  }
+
+  /**
+   * Tests launching the cluster by first starting master, and then the regionserver
+   * to ensure that it does not matter which is started first.
+   */
+  @Test(timeout = 12000)
+  public void testBootMasterFirst() throws Exception {
+    startMaster();
+    startRegionServer();
+    waitForClusterOnline();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 0dff496795f2faeb4a235b1fdd40fe2bd629d6b1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 24 Apr 2012 05:14:30 +0000
Subject: [PATCH 0154/1540] Refuse operations from Admin before master is
 initialized - fix for all branches

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329556 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 2882fa0262ef..2f2a7f022b0d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1093,6 +1093,7 @@ public void createTable(HTableDescriptor hTableDescriptor,
     }
 
     HRegionInfo [] newRegions = getHRegionInfos(hTableDescriptor, splitKeys);
+    checkInitialized();
     if (cpHost != null) {
       cpHost.preCreateTable(hTableDescriptor, newRegions);
     }

From 8865a88b1168252bf4d90ca9213dd9be7db7461d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 05:56:39 +0000
Subject: [PATCH 0155/1540] HBASE-5849 On first cluster startup, RS aborts if
 root znode is not available; REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329561 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     |   6 +
 .../hadoop/hbase/TestClusterBootOrder.java    | 118 ------------------
 2 files changed, 6 insertions(+), 118 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 53f7a9a392df..a6f9e5b78e23 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -605,6 +605,12 @@ private void initializeZooKeeper() throws IOException, InterruptedException {
    */
   private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
       throws IOException, InterruptedException {
+    if (false == tracker.checkIfBaseNodeAvailable()) {
+      String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. "
+          + "There could be a mismatch with the one configured in the master.";
+      LOG.error(errorMsg);
+      abort(errorMsg);
+    }
     while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
       if (this.stopped) {
         throw new IOException("Received the shutdown message while waiting.");
diff --git a/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
index 3bbbdf2826fe..e69de29bb2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Tests the boot order indifference between regionserver and master
- */
-@Category(MediumTests.class)
-public class TestClusterBootOrder {
-
-  private static final long SLEEP_INTERVAL = 1000;
-  private static final long SLEEP_TIME = 4000;
-
-  private HBaseTestingUtility testUtil;
-  private LocalHBaseCluster cluster;
-  private RegionServerThread rs;
-  private MasterThread master;
-
-  @Before
-  public void setUp() throws Exception {
-    testUtil = new HBaseTestingUtility();
-    testUtil.startMiniZKCluster(1);
-    cluster = new LocalHBaseCluster(testUtil.getConfiguration(), 0, 0);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    cluster.shutdown();
-    cluster.join();
-    testUtil.shutdownMiniZKCluster();
-  }
-
-  private void startRegionServer() throws Exception {
-    rs = cluster.addRegionServer();
-    rs.start();
-
-    for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) {
-      //we cannot block on wait for rs at this point , since master is not up.
-      Thread.sleep(SLEEP_INTERVAL);
-      assertTrue(rs.isAlive());
-    }
-  }
-
-  private void startMaster() throws Exception {
-    master = cluster.addMaster();
-    master.start();
-
-    for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) {
-      Thread.sleep(SLEEP_INTERVAL);
-      assertTrue(master.isAlive());
-    }
-  }
-
-  private void waitForClusterOnline() {
-    while (true) {
-      if (master.getMaster().isInitialized()) {
-        break;
-      }
-      try {
-        Thread.sleep(100);
-      } catch (InterruptedException ignored) {
-        // Keep waiting
-      }
-    }
-    rs.waitForServerOnline();
-  }
-
-  /**
-   * Tests launching the cluster by first starting regionserver, and then the master
-   * to ensure that it does not matter which is started first.
-   */
-  @Test(timeout = 12000)
-  public void testBootRegionServerFirst() throws Exception {
-    startRegionServer();
-    startMaster();
-    waitForClusterOnline();
-  }
-
-  /**
-   * Tests launching the cluster by first starting master, and then the regionserver
-   * to ensure that it does not matter which is started first.
-   */
-  @Test(timeout = 12000)
-  public void testBootMasterFirst() throws Exception {
-    startMaster();
-    startRegionServer();
-    waitForClusterOnline();
-  }
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}

From 89bcd242439498ae8ddddaea875ff0082b8844c3 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 07:22:53 +0000
Subject: [PATCH 0156/1540] HBASE-4393 Implement a canary monitoring program

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329575 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/tools/Canary.java | 253 ++++++++++++++++++
 1 file changed, 253 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/tools/Canary.java

diff --git a/src/main/java/org/apache/hadoop/hbase/tools/Canary.java b/src/main/java/org/apache/hadoop/hbase/tools/Canary.java
new file mode 100644
index 000000000000..0cf5569f7290
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/tools/Canary.java
@@ -0,0 +1,253 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.tool;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableNotFoundException;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+/**
+ * HBase Canary Tool, that that can be used to do
+ * "canary monitoring" of a running HBase cluster.
+ *
+ * Foreach region tries to get one row per column family
+ * and outputs some information about failure or latency.
+ */
+public final class Canary implements Tool {
+  // Sink interface used by the canary to outputs information
+  public interface Sink {
+    public void publishReadFailure(HRegionInfo region);
+    public void publishReadFailure(HRegionInfo region, HColumnDescriptor column);
+    public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
+  }
+
+  // Simple implementation of canary sink that allows to plot on
+  // file or standard output timings or failures.
+  public static class StdOutSink implements Sink {
+    @Override
+    public void publishReadFailure(HRegionInfo region) {
+      LOG.error(String.format("read from region %s failed", region.getRegionNameAsString()));
+    }
+
+    @Override
+    public void publishReadFailure(HRegionInfo region, HColumnDescriptor column) {
+      LOG.error(String.format("read from region %s column family %s failed",
+                region.getRegionNameAsString(), column.getNameAsString()));
+    }
+
+    @Override
+    public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) {
+      LOG.info(String.format("read from region %s column family %s in %dms",
+               region.getRegionNameAsString(), column.getNameAsString(), msTime));
+    }
+  }
+
+  private static final long DEFAULT_INTERVAL = 6000;
+
+  private static final Log LOG = LogFactory.getLog(Canary.class);
+
+  private Configuration conf = null;
+  private HBaseAdmin admin = null;
+  private long interval = 0;
+  private Sink sink = null;
+
+  public Canary() {
+    this(new StdOutSink());
+  }
+
+  public Canary(Sink sink) {
+    this.sink = sink;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    int tables_index = -1;
+
+    // Process command line args
+    for (int i = 0; i < args.length; i++) {
+      String cmd = args[i];
+
+      if (cmd.startsWith("-")) {
+        if (tables_index >= 0) {
+          // command line args must be in the form: [opts] [table 1 [table 2 ...]]
+          System.err.println("Invalid command line options");
+          printUsageAndExit();
+        }
+
+        if (cmd.equals("-help")) {
+          // user asked for help, print the help and quit.
+          printUsageAndExit();
+        } else if (cmd.equals("-daemon") && interval == 0) {
+          // user asked for daemon mode, set a default interval between checks
+          interval = DEFAULT_INTERVAL;
+        } else if (cmd.equals("-interval")) {
+          // user has specified an interval for canary breaths (-interval N)
+          i++;
+
+          if (i == args.length) {
+            System.err.println("-interval needs a numeric value argument.");
+            printUsageAndExit();
+          }
+
+          try {
+            interval = Long.parseLong(args[i]) * 1000;
+          } catch (NumberFormatException e) {
+            System.err.println("-interval needs a numeric value argument.");
+            printUsageAndExit();
+          }
+        } else {
+          // no options match
+          System.err.println(cmd + " options is invalid.");
+          printUsageAndExit();
+        }
+      } else if (tables_index < 0) {
+        // keep track of first table name specified by the user
+        tables_index = i;
+      }
+    }
+
+    // initialize HBase conf and admin
+    if (conf == null) conf = HBaseConfiguration.create();
+    admin = new HBaseAdmin(conf);
+
+    // lets the canary monitor the cluster
+    do {
+      if (admin.isAborted()) {
+        LOG.error("HBaseAdmin aborted");
+        return(1);
+      }
+
+      if (tables_index >= 0) {
+        for (int i = tables_index; i < args.length; i++) {
+          sniff(args[i]);
+        }
+      } else {
+        sniff();
+      }
+
+      Thread.sleep(interval);
+    } while (interval > 0);
+
+    return(0);
+  }
+
+  private void printUsageAndExit() {
+    System.err.printf("Usage: bin/hbase %s [opts] [table 1 [table 2...]]\n", getClass().getName());
+    System.err.println(" where [opts] are:");
+    System.err.println("   -help          Show this help and exit.");
+    System.err.println("   -daemon        Continuous check at defined intervals.");
+    System.err.println("   -interval <N>  Interval between checks (sec)");
+    System.exit(1);
+  }
+
+  /*
+   * canary entry point to monitor all the tables.
+   */
+  private void sniff() throws Exception {
+    for (HTableDescriptor table : admin.listTables()) {
+      sniff(table);
+    }
+  }
+
+  /*
+   * canary entry point to monitor specified table.
+   */
+  private void sniff(String tableName) throws Exception {
+    if (admin.isTableAvailable(tableName)) {
+      sniff(admin.getTableDescriptor(tableName.getBytes()));
+    } else {
+      LOG.warn(String.format("Table %s is not available", tableName));
+    }
+  }
+
+  /*
+   * Loops over regions that owns this table,
+   * and output some information abouts the state.
+   */
+  private void sniff(HTableDescriptor tableDesc) throws Exception {
+    HTable table = null;
+
+    try {
+      table = new HTable(admin.getConfiguration(), tableDesc.getName());
+    } catch (TableNotFoundException e) {
+      return;
+    }
+
+    for (HRegionInfo region : admin.getTableRegions(tableDesc.getName())) {
+      try {
+        sniffRegion(region, table);
+      } catch (Exception e) {
+        sink.publishReadFailure(region);
+      }
+    }
+  }
+
+  /*
+   * For each column family of the region tries to get one row
+   * and outputs the latency, or the failure.
+   */
+  private void sniffRegion(HRegionInfo region, HTable table) throws Exception {
+    HTableDescriptor tableDesc = table.getTableDescriptor();
+    for (HColumnDescriptor column : tableDesc.getColumnFamilies()) {
+      Get get = new Get(region.getStartKey());
+      get.addFamily(column.getName());
+
+      try {
+        long startTime = System.currentTimeMillis();
+        table.get(get);
+        long time = System.currentTimeMillis() - startTime;
+
+        sink.publishReadTiming(region, column, time);
+      } catch (Exception e) {
+        sink.publishReadFailure(region, column);
+      }
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    int exitCode = ToolRunner.run(new Canary(), args);
+    System.exit(exitCode);
+  }
+}
+

From 46d3b63ce76d95023036e36fe338de139865a906 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 15:33:53 +0000
Subject: [PATCH 0157/1540] HBASE-5863 Improve the graceful_stop.sh CLI help
 (especially about reloads)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329794 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/graceful_stop.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh
index cf7bee86ff1d..beac4b15b146 100644
--- a/bin/graceful_stop.sh
+++ b/bin/graceful_stop.sh
@@ -19,16 +19,16 @@
 # * See the License for the specific language governing permissions and
 # * limitations under the License.
 # */
- 
+
 # Move regions off a server then stop it.  Optionally restart and reload.
 # Turn off the balancer before running this script.
 function usage {
-  echo "Usage: graceful_stop.sh [--config <conf-dir>] [--restart] [--reload] [--thrift] [--rest] <hostname>" 
+  echo "Usage: graceful_stop.sh [--config <conf-dir>] [--restart [--reload]] [--thrift] [--rest] <hostname>"
   echo " thrift      If we should stop/start thrift before/after the hbase stop/start"
   echo " rest        If we should stop/start rest before/after the hbase stop/start"
   echo " restart     If we should restart after graceful stop"
-  echo " reload      Move offloaded regions back on to the stopped server"
-  echo " debug       Move offloaded regions back on to the stopped server"
+  echo "   reload      Move offloaded regions back on to the restarted server"
+  echo " debug       Print helpful debug information"
   echo " hostname    Hostname of server we are to stop"
   exit 1
 }

From 2d2c34d539b828bfaabdb3f546b5e2ee1fc2052b Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 24 Apr 2012 16:20:58 +0000
Subject: [PATCH 0158/1540] HBASE-5848 Create table with EMPTY_START_ROW passed
 as splitKey causes the HMaster to abort (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329827 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 15 ++++++---
 .../apache/hadoop/hbase/client/TestAdmin.java | 32 +++++++++++++++++++
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index ab09eba2ede7..81228b7a5adf 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -374,11 +374,13 @@ public void createTable(HTableDescriptor desc, byte [] startKey,
    * Creates a new table with an initial set of empty regions defined by the
    * specified split keys.  The total number of regions created will be the
    * number of split keys plus one. Synchronous operation.
+   * Note : Avoid passing empty split key.
    *
    * @param desc table descriptor for table
    * @param splitKeys array of split keys for the initial regions of the table
    *
-   * @throws IllegalArgumentException if the table name is reserved
+   * @throws IllegalArgumentException if the table name is reserved, if the split keys
+   * are repeated and if the split key has empty byte array.
    * @throws MasterNotRunningException if master is not running
    * @throws TableExistsException if table already exists (If concurrent
    * threads, the table may have been created between test-for-existence
@@ -453,10 +455,11 @@ public boolean processRow(Result rowResult) throws IOException {
    * Asynchronous operation.  To check if the table exists, use
    * {@link: #isTableAvailable} -- it is not safe to create an HTable
    * instance to this table before it is available.
-   *
+   * Note : Avoid passing empty split key.
    * @param desc table descriptor for table
    *
-   * @throws IllegalArgumentException Bad table name.
+   * @throws IllegalArgumentException Bad table name, if the split keys
+   * are repeated and if the split key has empty byte array.
    * @throws MasterNotRunningException if master is not running
    * @throws TableExistsException if table already exists (If concurrent
    * threads, the table may have been created between test-for-existence
@@ -466,11 +469,15 @@ public boolean processRow(Result rowResult) throws IOException {
   public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
   throws IOException {
     HTableDescriptor.isLegalTableName(desc.getName());
-    if(splitKeys != null && splitKeys.length > 1) {
+    if(splitKeys != null && splitKeys.length > 0) {
       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
       // Verify there are no duplicate split keys
       byte [] lastKey = null;
       for(byte [] splitKey : splitKeys) {
+        if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
+          throw new IllegalArgumentException(
+              "Empty split key must not be passed in the split keys.");
+        }
         if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
           throw new IllegalArgumentException("All split keys must be unique, " +
             "found duplicate: " + Bytes.toStringBinary(splitKey) +
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 20094003f3e9..dae300e9543f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -713,6 +713,38 @@ public void testCreateTableWithRegions() throws IOException, InterruptedExceptio
     }
     ladmin.close();
   }
+  
+    
+  @Test
+  public void testCreateTableWithOnlyEmptyStartRow() throws IOException {
+    byte[] tableName = Bytes.toBytes("testCreateTableWithOnlyEmptyStartRow");
+    byte[][] splitKeys = new byte[1][];
+    splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor("col"));
+    try {
+      admin.createTable(desc, splitKeys);
+      fail("Test case should fail as empty split key is passed.");
+    } catch (IllegalArgumentException e) {
+    }
+  }
+    
+  @Test
+  public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException {
+    byte[] tableName = Bytes
+        .toBytes("testCreateTableWithEmptyRowInTheSplitKeys");
+    byte[][] splitKeys = new byte[3][];
+    splitKeys[0] = "region1".getBytes();
+    splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
+    splitKeys[2] = "region2".getBytes();
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor("col"));
+    try {
+      admin.createTable(desc, splitKeys);
+      fail("Test case should fail as empty split key is passed.");
+    } catch (IllegalArgumentException e) {
+    }
+  }
 
   @Test
   public void testTableExist() throws IOException {

From 6971baeabb964cb4af7b3fba467b821feb0a743e Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 18:03:11 +0000
Subject: [PATCH 0159/1540] HBASE-5866 Canary in tool package but says its in
 tools

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329900 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/{tools => tool}/Canary.java | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/java/org/apache/hadoop/hbase/{tools => tool}/Canary.java (100%)

diff --git a/src/main/java/org/apache/hadoop/hbase/tools/Canary.java b/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
similarity index 100%
rename from src/main/java/org/apache/hadoop/hbase/tools/Canary.java
rename to src/main/java/org/apache/hadoop/hbase/tool/Canary.java

From b91d37b2027484ca0d7599096d05be47b438b6f3 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 18:06:27 +0000
Subject: [PATCH 0160/1540] HBASE-5865 test-util.sh broken with unittest
 updates

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329903 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/test-util.sh | 47 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/dev-support/test-util.sh b/dev-support/test-util.sh
index c37ab485acb9..3101e10d03f0 100755
--- a/dev-support/test-util.sh
+++ b/dev-support/test-util.sh
@@ -39,9 +39,13 @@ options:
    -n N        Run each test N times. Default = 1.
    -s N        Print N slowest tests
    -H          Print which tests are hanging (if any)
+   -e          Echo the maven call before running. Default: not enabled
+   -r          Runs remotely, on the build server. Default: not enabled
 EOF
 }
 
+echoUsage=0
+server=0
 testFile=
 doClean=""
 testType=verify
@@ -59,7 +63,7 @@ else
 fi
 testDir=$scriptDir/../../../target/surefire-reports
 
-while getopts "hcHun:s:f:" OPTION
+while getopts "hcerHun:s:f:" OPTION
 do
      case $OPTION in
          h)
@@ -84,6 +88,12 @@ do
          f)
              testFile=$OPTARG
              ;;
+        e)
+             echoUsage=1
+             ;;
+        r)
+            server=1
+            ;;
 	 ?) 
 	     usage
 	     exit 1
@@ -124,15 +134,44 @@ do
         #Now loop through each test
 	for (( j = 0; j < $numTests; j++ ))
 	do
-	    nice -10 mvn $doClean $testType -Dtest=${test[$j]}
-	    if [ $? -ne 0 ]; then
+        # Create the general command
+        cmd="nice -10 mvn $doClean $testType -Dtest=${test[$j]}"
+
+        # Add that is should run locally, if not on the server
+        if [ ${server} -eq 0 ]; then
+            cmd="${cmd} -P localTests"
+        fi
+
+        # Print the command, if we should
+        if [ ${echoUsage} -eq 1 ]; then
+            echo "${cmd}"
+        fi
+
+        # Run the command
+        $cmd
+
+        if [ $? -ne 0 ]; then
 		echo "${test[$j]} failed, iteration: $i"
 		exit 1
 	    fi
 	done
     else
 	echo "EXECUTING ALL TESTS"
-	nice -10 mvn $doClean $testType
+    # Create the general command
+    cmd="nice -10 mvn $doClean $testType"
+
+    # Add that is should run locally, if not on the server
+    if [ ${server} -eq 0 ]; then
+       cmd="${cmd} -P localTests"
+    fi
+
+    # Print the command, if we should
+    if [ ${echoUsage} -eq 1 ]; then
+        echo "${cmd}"
+    fi
+
+    #now run the command
+    $cmd
     fi
 done
 

From d3e44a7307231c5e35b4f9e9ecff23cabe1899f7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Apr 2012 19:28:40 +0000
Subject: [PATCH 0161/1540] HBASE-5836 Backport per region metrics from
 HBASE-3614 to 0.94.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1329958 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 150 +++----------
 .../hbase/regionserver/HRegionServer.java     |   3 +-
 .../hbase/regionserver/StoreScanner.java      |   3 +-
 .../metrics/OperationMetrics.java             | 207 ++++++++++++++++++
 .../metrics/RegionMetricsStorage.java         | 130 +++++++++++
 .../metrics/RegionServerDynamicMetrics.java   |   6 +-
 .../regionserver/metrics/SchemaMetrics.java   |  34 ++-
 .../regionserver/TestRegionServerMetrics.java | 117 +++++++++-
 8 files changed, 518 insertions(+), 132 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 3832a03b99ff..ca9983c7bec1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -38,6 +38,7 @@
 import java.util.NavigableMap;
 import java.util.NavigableSet;
 import java.util.Random;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.Callable;
@@ -104,6 +105,7 @@
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.metrics.OperationMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
@@ -322,83 +324,8 @@ boolean isFlushRequested() {
   public final static String REGIONINFO_FILE = ".regioninfo";
   private HTableDescriptor htableDescriptor = null;
   private RegionSplitPolicy splitPolicy;
+  private final OperationMetrics opMetrics;
 
-  // for simple numeric metrics (# of blocks read from block cache)
-  public static final ConcurrentMap<String, AtomicLong> numericMetrics = new ConcurrentHashMap<String, AtomicLong>();
-
-  // for simple numeric metrics (current block cache size)
-  // These ones are not reset to zero when queried, unlike the previous.
-  public static final ConcurrentMap<String, AtomicLong> numericPersistentMetrics = new ConcurrentHashMap<String, AtomicLong>();
-
-  /**
-   * Used for metrics where we want track a metrics (such as latency) over a
-   * number of operations.
-   */
-  public static final ConcurrentMap<String, Pair<AtomicLong, AtomicInteger>>
-      timeVaryingMetrics = new ConcurrentHashMap<String, 
-          Pair<AtomicLong, AtomicInteger>>();
-
-  public static void incrNumericMetric(String key, long amount) {
-    AtomicLong oldVal = numericMetrics.get(key);
-    if (oldVal == null) {
-      oldVal = numericMetrics.putIfAbsent(key, new AtomicLong(amount));
-      if (oldVal == null)
-        return;
-    }
-    oldVal.addAndGet(amount);
-  }
-
-  public static void setNumericMetric(String key, long amount) {
-    numericMetrics.put(key, new AtomicLong(amount));
-  }
-
-  public static void incrTimeVaryingMetric(String key, long amount) {
-    Pair<AtomicLong, AtomicInteger> oldVal = timeVaryingMetrics.get(key);
-    if (oldVal == null) {
-      oldVal = timeVaryingMetrics.putIfAbsent(key,
-          new Pair<AtomicLong, AtomicInteger>(new AtomicLong(amount),
-              new AtomicInteger(1)));
-      if (oldVal == null)
-        return;
-    }
-    oldVal.getFirst().addAndGet(amount); // total time
-    oldVal.getSecond().incrementAndGet(); // increment ops by 1
-  }
-
-  public static void incrNumericPersistentMetric(String key, long amount) {
-    AtomicLong oldVal = numericPersistentMetrics.get(key);
-    if (oldVal == null) {
-      oldVal = numericPersistentMetrics
-          .putIfAbsent(key, new AtomicLong(amount));
-      if (oldVal == null)
-        return;
-    }
-    oldVal.addAndGet(amount);
-  }
-
-  public static long getNumericMetric(String key) {
-    AtomicLong m = numericMetrics.get(key);
-    if (m == null)
-      return 0;
-    return m.get();
-  }
-
-  public static Pair<Long, Integer> getTimeVaryingMetric(String key) {
-    Pair<AtomicLong, AtomicInteger> pair = timeVaryingMetrics.get(key);
-    if (pair == null) {
-      return new Pair<Long, Integer>(0L, 0);
-    }
-
-    return new Pair<Long, Integer>(pair.getFirst().get(),
-        pair.getSecond().get());
-  }
-
-  static long getNumericPersistentMetric(String key) {
-    AtomicLong m = numericPersistentMetrics.get(key);
-    if (m == null)
-      return 0;
-    return m.get();
-  }
 
   /**
    * Should only be used for testing purposes
@@ -419,6 +346,7 @@ public HRegion(){
     this.threadWakeFrequency = 0L;
     this.coprocessorHost = null;
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+    this.opMetrics = new OperationMetrics();
   }
 
   /**
@@ -463,6 +391,7 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
     setHTableSpecificConf();
     this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+    this.opMetrics = new OperationMetrics(conf, this.regionInfo);
 
     /*
      * timestamp.slop provides a server-side constraint on the timestamp. This
@@ -1851,11 +1780,7 @@ private void internalDelete(Delete delete, UUID clusterId,
       coprocessorHost.postDelete(delete, walEdit, writeToWAL);
     }
     final long after = EnvironmentEdgeManager.currentTimeMillis();
-    final String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
-        getTableDesc().getNameAsString(), familyMap.keySet());
-    if (!metricPrefix.isEmpty()) {
-      HRegion.incrTimeVaryingMetric(metricPrefix + "delete_", after - now);
-    }
+    this.opMetrics.updateDeleteMetrics(familyMap.keySet(), after-now);
 
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
@@ -2001,9 +1926,10 @@ public OperationStatus[] put(
   @SuppressWarnings("unchecked")
   private long doMiniBatchPut(
       BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException {
-    String metricPrefix = null;
     final String tableName = getTableDesc().getNameAsString();
 
+    // The set of columnFamilies first seen.
+    Set<byte[]> cfSet = null;
     // variable to note if all Put items are for the same CF -- metrics related
     boolean cfSetConsistent = true;
     long startTimeMs = EnvironmentEdgeManager.currentTimeMillis();
@@ -2085,19 +2011,13 @@ private long doMiniBatchPut(
         lastIndexExclusive++;
         numReadyToWrite++;
 
-        // If first time around, designate a prefix for metrics based on the CF
-        // set. After that, watch for inconsistencies.
-        final String curMetricPrefix =
-            SchemaMetrics.generateSchemaMetricsPrefix(tableName,
-                put.getFamilyMap().keySet());
-
-        if (metricPrefix == null) {
-          metricPrefix = curMetricPrefix;
-        } else if (cfSetConsistent && !metricPrefix.equals(curMetricPrefix)) {
-          // The column family set for this batch put is undefined.
-          cfSetConsistent = false;
-          metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName,
-              SchemaMetrics.UNKNOWN);
+        // If Column Families stay consistent through out all of the
+        // individual puts then metrics can be reported as a mutliput across
+        // column families in the first put.
+        if (cfSet == null) {
+          cfSet = put.getFamilyMap().keySet();
+        } else {
+          cfSetConsistent = cfSetConsistent && put.getFamilyMap().keySet().equals(cfSet);
         }
       }
 
@@ -2242,11 +2162,12 @@ private long doMiniBatchPut(
 
       // do after lock
       final long endTimeMs = EnvironmentEdgeManager.currentTimeMillis();
-      if (metricPrefix == null) {
-        metricPrefix = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
-      }
-      HRegion.incrTimeVaryingMetric(metricPrefix + "multiput_",
-          endTimeMs - startTimeMs);
+            
+      // See if the column families were consistent through the whole thing.
+      // if they were then keep them. If they were not then pass a null.
+      // null will be treated as unknown.
+      final Set<byte[]> keptCfs = cfSetConsistent ? cfSet : null;
+      this.opMetrics.updateMultiPutMetrics(keptCfs, endTimeMs - startTimeMs);
 
       if (!success) {
         for (int i = firstIndex; i < lastIndexExclusive; i++) {
@@ -2501,12 +2422,8 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
 
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
-    final String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
-        this.getTableDesc().getNameAsString(), familyMap.keySet());
-    if (!metricPrefix.isEmpty()) {
-      HRegion.incrTimeVaryingMetric(metricPrefix + "put_", after - now);
-    }
-
+    this.opMetrics.updatePutMetrics(familyMap.keySet(), after - now);
+    
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -4128,12 +4045,8 @@ private List<KeyValue> get(Get get, boolean withCoprocessor)
 
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
-    final String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
-        this.getTableDesc().getNameAsString(), get.familySet());
-    if (!metricPrefix.isEmpty()) {
-      HRegion.incrTimeVaryingMetric(metricPrefix + "get_", after - now);
-    }
-
+    this.opMetrics.updateGetMetrics(get.familySet(), after - now);
+    
     return results;
   }
 
@@ -4454,6 +4367,10 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
       closeRegionOperation();
     }
 
+    
+    long after = EnvironmentEdgeManager.currentTimeMillis();
+    this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - now);
+    
     if (flush) {
       // Request a cache flush. Do it outside update lock.
       requestFlush();
@@ -4564,6 +4481,9 @@ public Result increment(Increment increment, Integer lockid,
     } finally {
       closeRegionOperation();
     }
+    
+    long after = EnvironmentEdgeManager.currentTimeMillis();
+    this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - now);
 
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
@@ -4661,9 +4581,7 @@ public long incrementColumnValue(byte [] row, byte [] family,
 
     // do after lock
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
-        getTableDesc().getName(), family);
-    HRegion.incrTimeVaryingMetric(metricPrefix + "increment_", after - before);
+    this.opMetrics.updateIncrementColumnValueMetrics(family, after - before);
 
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
@@ -4693,7 +4611,7 @@ private void checkFamily(final byte [] family)
       ClassSize.OBJECT +
       ClassSize.ARRAY +
       30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
-      (5 * Bytes.SIZEOF_LONG) +
+      (6 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a6f9e5b78e23..607427354c76 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -128,6 +128,7 @@
 import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
 import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
 import org.apache.hadoop.hbase.regionserver.handler.OpenRootHandler;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
 import org.apache.hadoop.hbase.regionserver.metrics.RegionServerDynamicMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
@@ -1399,7 +1400,7 @@ protected void metrics() {
     }
 
     for (Entry<String, MutableDouble> e : tempVals.entrySet()) {
-      HRegion.setNumericMetric(e.getKey(), e.getValue().longValue());
+      RegionMetricsStorage.setNumericMetric(e.getKey(), e.getValue().longValue());
     }
 
     this.metrics.stores.set(stores);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index f492f00b238c..44f6df13083c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -367,7 +368,7 @@ public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOE
             this.heap.next();
           }
 
-          HRegion.incrNumericMetric(metricNameGetSize, kv.getLength());
+          RegionMetricsStorage.incrNumericMetric(metricNameGetSize, kv.getLength());
           if (limit > 0 && (results.size() == limit)) {
             break LOOP;
           }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
new file mode 100644
index 000000000000..ac4e57f56c2a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.metrics;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This class provides a simplified interface to expose time varying metrics
+ * about GET/DELETE/PUT/ICV operations on a region and on Column Families. All
+ * metrics are stored in {@link RegionMetricsStorage} and exposed to hadoop
+ * metrics through {@link RegionServerDynamicMetrics}.
+ */
+@InterfaceAudience.Private
+public class OperationMetrics {
+
+  private static final String DELETE_KEY = "delete_";
+  private static final String PUT_KEY = "put_";
+  private static final String GET_KEY = "get_";
+  private static final String ICV_KEY = "incrementColumnValue_";
+  private static final String INCREMENT_KEY = "increment_";
+  private static final String MULTIPUT_KEY = "multiput_";
+  private static final String APPEND_KEY = "append_";
+  
+  /** Conf key controlling whether we should expose metrics.*/
+  private static final String CONF_KEY =
+      "hbase.metrics.exposeOperationTimes";
+
+  private String tableName = null;
+  private String regionName = null;
+  private String regionMetrixPrefix = null;
+  private Configuration conf = null;
+  
+
+  /**
+   * Create a new OperationMetrics
+   * @param conf The Configuration of the HRegion reporting operations coming in.
+   * @param regionInfo The region info
+   */
+  public OperationMetrics(Configuration conf, HRegionInfo regionInfo) {
+    // Configure SchemaMetrics before trying to create a RegionOperationMetrics instance as
+    // RegionOperationMetrics relies on SchemaMetrics to do naming.
+    if (conf != null) {
+      SchemaMetrics.configureGlobally(conf);
+
+      this.conf = conf;
+      if (regionInfo != null) {
+        this.tableName = regionInfo.getTableNameAsString();
+        this.regionName = regionInfo.getEncodedName();
+      } else {
+        this.tableName = SchemaMetrics.UNKNOWN;
+        this.regionName = SchemaMetrics.UNKNOWN;
+      }
+      this.regionMetrixPrefix =
+          SchemaMetrics.generateRegionMetricsPrefix(this.tableName, this.regionName);
+    }
+  }
+  
+  /**
+   * This is used in creating a testing HRegion where the regionInfo is unknown
+   * @param conf
+   */
+  public OperationMetrics() {
+    this(null, null);
+  }
+
+
+  /**
+   * Update the stats associated with {@link HTable#put(java.util.List)}.
+   * 
+   * @param columnFamilies Set of CF's this multiput is associated with
+   * @param value the time
+   */
+  public void updateMultiPutMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, MULTIPUT_KEY, value);
+  }
+
+  /**
+   * Update the metrics associated with a {@link Get}
+   * 
+   * @param columnFamilies
+   *          Set of Column Families in this get.
+   * @param value
+   *          the time
+   */
+  public void updateGetMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, GET_KEY, value);
+  }
+  
+  /**
+   * Update metrics associated with an {@link Increment}
+   * @param columnFamilies
+   * @param value
+   */
+  public void updateIncrementMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, INCREMENT_KEY, value);
+  }
+  
+  
+  /**
+   * Update the metrics associated with an {@link Append}
+   * @param columnFamilies
+   * @param value
+   */
+  public void updateAppendMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, APPEND_KEY, value);
+  }
+
+
+  /**
+   * Update the metrics associated with
+   * {@link HTable#incrementColumnValue(byte[], byte[], byte[], long)}
+   * 
+   * @param columnFamily
+   *          The single column family associated with an ICV
+   * @param value
+   *          the time
+   */
+  public void updateIncrementColumnValueMetrics(byte[] columnFamily, long value) {
+    String cfMetricPrefix =
+        SchemaMetrics.generateSchemaMetricsPrefix(this.tableName, Bytes.toString(columnFamily));
+    doSafeIncTimeVarying(cfMetricPrefix, ICV_KEY, value);
+    doSafeIncTimeVarying(this.regionMetrixPrefix, ICV_KEY, value);
+  }
+
+  /**
+   * update metrics associated with a {@link Put}
+   * 
+   * @param columnFamilies
+   *          Set of column families involved.
+   * @param value
+   *          the time.
+   */
+  public void updatePutMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, PUT_KEY, value);
+  }
+
+  /**
+   * update metrics associated with a {@link Delete}
+   * 
+   * @param columnFamilies
+   * @param value
+   *          the time.
+   */
+  public void updateDeleteMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, DELETE_KEY, value);
+  }
+
+  /**
+   * Method to send updates for cf and region metrics. This is the normal method
+   * used if the naming of stats and CF's are in line with put/delete/multiput.
+   * 
+   * @param columnFamilies
+   *          the set of column families involved.
+   * @param key
+   *          the metric name.
+   * @param value
+   *          the time.
+   */
+  private void doUpdateTimeVarying(Set<byte[]> columnFamilies, String key, long value) {
+    String cfPrefix = null;
+    if (columnFamilies != null) {
+      cfPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName, columnFamilies);
+    } else {
+      cfPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName, SchemaMetrics.UNKNOWN);
+    }
+
+    doSafeIncTimeVarying(cfPrefix, key, value);
+    doSafeIncTimeVarying(this.regionMetrixPrefix, key, value);
+  }
+
+  private void doSafeIncTimeVarying(String prefix, String key, long value) {
+    if (conf.getBoolean(CONF_KEY, true)) {
+      if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) {
+        RegionMetricsStorage.incrTimeVaryingMetric(prefix + key, value);
+      }
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
new file mode 100644
index 000000000000..416e495057d9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.metrics;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * This class if for maintaining the maps used to power metrics for hfiles,
+ * regions, and regionservers. It has methods to mutate and get state of metrics
+ * numbers. These numbers are exposed to Hadoop metrics through
+ * RegionServerDynamicMetrics.
+ */
+@InterfaceAudience.Private
+public class RegionMetricsStorage {
+
+  // for simple numeric metrics (# of blocks read from block cache)
+  private static final ConcurrentMap<String, AtomicLong> numericMetrics =
+      new ConcurrentHashMap<String, AtomicLong>();
+
+  // for simple numeric metrics (current block cache size)
+  // These ones are not reset to zero when queried, unlike the previous.
+  private static final ConcurrentMap<String, AtomicLong> numericPersistentMetrics =
+      new ConcurrentHashMap<String, AtomicLong>();
+
+  /**
+   * Used for metrics where we want track a metrics (such as latency) over a
+   * number of operations.
+   */
+  private static final ConcurrentMap<String, Pair<AtomicLong, AtomicInteger>> timeVaryingMetrics =
+      new ConcurrentHashMap<String, Pair<AtomicLong, AtomicInteger>>();
+
+  public static Map<String, AtomicLong> getNumericMetrics() {
+    return numericMetrics;
+  }
+
+  public static Map<String, AtomicLong> getNumericPersistentMetrics() {
+    return numericPersistentMetrics;
+  }
+
+  public static Map<String, Pair<AtomicLong, AtomicInteger>> getTimeVaryingMetrics() {
+    return timeVaryingMetrics;
+  }
+
+  public static void incrNumericMetric(String key, long amount) {
+    AtomicLong oldVal = numericMetrics.get(key);
+    if (oldVal == null) {
+      oldVal = numericMetrics.putIfAbsent(key, new AtomicLong(amount));
+      if (oldVal == null)
+        return;
+    }
+    oldVal.addAndGet(amount);
+  }
+
+  public static void incrTimeVaryingMetric(String key, long amount) {
+    Pair<AtomicLong, AtomicInteger> oldVal = timeVaryingMetrics.get(key);
+    if (oldVal == null) {
+      oldVal =
+          timeVaryingMetrics.putIfAbsent(key, 
+              new Pair<AtomicLong, AtomicInteger>(
+                  new AtomicLong(amount), 
+                  new AtomicInteger(1)));
+      if (oldVal == null)
+        return;
+    }
+    oldVal.getFirst().addAndGet(amount); // total time
+    oldVal.getSecond().incrementAndGet(); // increment ops by 1
+  }
+
+  public static void incrNumericPersistentMetric(String key, long amount) {
+    AtomicLong oldVal = numericPersistentMetrics.get(key);
+    if (oldVal == null) {
+      oldVal = numericPersistentMetrics.putIfAbsent(key, new AtomicLong(amount));
+      if (oldVal == null)
+        return;
+    }
+    oldVal.addAndGet(amount);
+  }
+
+  public static void setNumericMetric(String key, long amount) {
+    numericMetrics.put(key, new AtomicLong(amount));
+  }
+
+  public static long getNumericMetric(String key) {
+    AtomicLong m = numericMetrics.get(key);
+    if (m == null)
+      return 0;
+    return m.get();
+  }
+
+  public static Pair<Long, Integer> getTimeVaryingMetric(String key) {
+    Pair<AtomicLong, AtomicInteger> pair = timeVaryingMetrics.get(key);
+    if (pair == null) {
+      return new Pair<Long, Integer>(0L, 0);
+    }
+
+    return new Pair<Long, Integer>(pair.getFirst().get(), pair.getSecond().get());
+  }
+
+  public static long getNumericPersistentMetric(String key) {
+    AtomicLong m = numericPersistentMetrics.get(key);
+    if (m == null)
+      return 0;
+    return m.get();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
index c34874c86c10..752b12748fe5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
@@ -131,17 +131,17 @@ public synchronized void incrTimeVaryingMetric(
    */
   public void doUpdates(MetricsContext context) {
     /* get dynamically created numeric metrics, and push the metrics */
-    for (Entry<String, AtomicLong> entry : HRegion.numericMetrics.entrySet()) {
+    for (Entry<String, AtomicLong> entry : RegionMetricsStorage.getNumericMetrics().entrySet()) {
       this.setNumericMetric(entry.getKey(), entry.getValue().getAndSet(0));
     }
     /* get dynamically created numeric metrics, and push the metrics.
      * These ones aren't to be reset; they are cumulative. */
-    for (Entry<String, AtomicLong> entry : HRegion.numericPersistentMetrics.entrySet()) {
+    for (Entry<String, AtomicLong> entry : RegionMetricsStorage.getNumericPersistentMetrics().entrySet()) {
       this.setNumericMetric(entry.getKey(), entry.getValue().get());
     }
     /* get dynamically created time varying metrics, and push the metrics */
     for (Entry<String, Pair<AtomicLong, AtomicInteger>> entry :
-          HRegion.timeVaryingMetrics.entrySet()) {
+        RegionMetricsStorage.getTimeVaryingMetrics().entrySet()) {
       Pair<AtomicLong, AtomicInteger> value = entry.getValue();
       this.incrTimeVaryingMetric(entry.getKey(),
           value.getFirst().getAndSet(0),
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index b70766c38353..08afbbc039a1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -165,9 +165,10 @@ public String toString() {
    */
   public static final String UNKNOWN = "__unknown";
 
-  private static final String TABLE_PREFIX = "tbl.";
+  public static final String TABLE_PREFIX = "tbl.";
   public static final String CF_PREFIX = "cf.";
   public static final String BLOCK_TYPE_PREFIX = "bt.";
+  public static final String REGION_PREFIX = "region.";
 
   public static final String CF_UNKNOWN_PREFIX = CF_PREFIX + UNKNOWN + ".";
   public static final String CF_BAD_FAMILY_PREFIX = CF_PREFIX + "__badfamily.";
@@ -364,7 +365,7 @@ private void incrNumericMetric(BlockCategory blockCategory,
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
-    HRegion.incrNumericMetric(getBlockMetricName(blockCategory,
+    RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
         isCompaction, metricType), 1);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
@@ -375,7 +376,7 @@ private void incrNumericMetric(BlockCategory blockCategory,
 
   private void addToReadTime(BlockCategory blockCategory,
       boolean isCompaction, long timeMs) {
-    HRegion.incrTimeVaryingMetric(getBlockMetricName(blockCategory,
+    RegionMetricsStorage.incrTimeVaryingMetric(getBlockMetricName(blockCategory,
         isCompaction, BlockMetricType.READ_TIME), timeMs);
 
     // Also update the read time aggregated across all block categories
@@ -431,7 +432,7 @@ public String getStoreMetricNameMax(StoreMetricType storeMetricType) {
    */
   public void updatePersistentStoreMetric(StoreMetricType storeMetricType,
       long value) {
-    HRegion.incrNumericPersistentMetric(
+    RegionMetricsStorage.incrNumericPersistentMetric(
         storeMetricNames[storeMetricType.ordinal()], value);
   }
 
@@ -476,7 +477,7 @@ public void addToCacheSize(BlockCategory category, long cacheSizeDelta) {
     if (category == null) {
       category = BlockCategory.ALL_CATEGORIES;
     }
-    HRegion.incrNumericPersistentMetric(getBlockMetricName(category, false,
+    RegionMetricsStorage.incrNumericPersistentMetric(getBlockMetricName(category, false,
         BlockMetricType.CACHE_SIZE), cacheSizeDelta);
 
     if (category != BlockCategory.ALL_CATEGORIES) {
@@ -500,7 +501,7 @@ public void updateOnCachePutOrEvict(BlockCategory blockCategory,
    * positives/negatives as specified by the argument.
    */
   public void updateBloomMetrics(boolean isInBloom) {
-    HRegion.incrNumericMetric(getBloomMetricName(isInBloom), 1);
+    RegionMetricsStorage.incrNumericMetric(getBloomMetricName(isInBloom), 1);
     if (this != ALL_SCHEMA_METRICS) {
       ALL_SCHEMA_METRICS.updateBloomMetrics(isInBloom);
     }
@@ -614,6 +615,23 @@ public static String generateSchemaMetricsPrefix(String tableName,
     return SchemaMetrics.generateSchemaMetricsPrefix(tableName, sb.toString());
   }
 
+  /**
+   * Get the prefix for metrics generated about a single region.
+   * 
+   * @param tableName
+   *          the table name or {@link #TOTAL_KEY} for all tables
+   * @param regionName
+   *          regionName
+   * @return the prefix for this table/region combination.
+   */
+  static String generateRegionMetricsPrefix(String tableName, String regionName) {
+    tableName = getEffectiveTableName(tableName);
+    String schemaMetricPrefix = tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + ".";
+    schemaMetricPrefix += regionName.equals(TOTAL_KEY) ? "" : REGION_PREFIX + regionName + ".";
+
+    return schemaMetricPrefix;
+  }
+  
   /**
    * Sets the flag of whether to use table name in metric names. This flag
    * is specified in configuration and is not expected to change at runtime,
@@ -729,11 +747,11 @@ public static Map<String, Long> getMetricsSnapshot() {
         long metricValue;
         if (isTimeVaryingKey(metricName)) {
           Pair<Long, Integer> totalAndCount =
-              HRegion.getTimeVaryingMetric(stripTimeVaryingSuffix(metricName));
+              RegionMetricsStorage.getTimeVaryingMetric(stripTimeVaryingSuffix(metricName));
           metricValue = metricName.endsWith(TOTAL_SUFFIX) ?
               totalAndCount.getFirst() : totalAndCount.getSecond();
         } else {
-          metricValue = HRegion.getNumericMetric(metricName);
+          metricValue = RegionMetricsStorage.getNumericMetric(metricName);
         }
 
         metricsSnapshot.put(metricName, metricValue);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 6560672abb0d..879f1d65186f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -23,15 +23,25 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.
     StoreMetricType;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -77,6 +87,29 @@ public void tearDown() throws Exception {
     SchemaMetrics.validateMetricChanges(startingMetrics);
   }
 
+  private void assertTimeVaryingMetricCount(int expectedCount, String table, String cf,
+      String regionName, String metricPrefix) {
+
+    Integer expectedCountInteger = new Integer(expectedCount);
+
+    if (cf != null) {
+      String cfKey =
+          SchemaMetrics.TABLE_PREFIX + table + "." + SchemaMetrics.CF_PREFIX + cf + "."
+              + metricPrefix;
+      Pair<Long, Integer> cfPair = RegionMetricsStorage.getTimeVaryingMetric(cfKey);
+      assertEquals(expectedCountInteger, cfPair.getSecond());
+    }
+
+    if (regionName != null) {
+      String rKey =
+          SchemaMetrics.TABLE_PREFIX + table + "." + SchemaMetrics.REGION_PREFIX + regionName + "."
+              + metricPrefix;
+
+      Pair<Long, Integer> regionPair = RegionMetricsStorage.getTimeVaryingMetric(rKey);
+      assertEquals(expectedCountInteger, regionPair.getSecond());
+    }
+  }
+  
   private void assertStoreMetricEquals(long expected,
       SchemaMetrics schemaMetrics, StoreMetricType storeMetricType) {
     final String storeMetricName =
@@ -84,10 +117,88 @@ private void assertStoreMetricEquals(long expected,
     Long startValue = startingMetrics.get(storeMetricName);
     assertEquals("Invalid value for store metric " + storeMetricName
         + " (type " + storeMetricType + ")", expected,
-        HRegion.getNumericMetric(storeMetricName)
+        RegionMetricsStorage.getNumericMetric(storeMetricName)
             - (startValue != null ? startValue : 0));
   }
-
+  
+    @Test
+    public void testOperationMetrics() throws IOException {
+      String cf = "OPCF";
+      String otherCf = "otherCF";
+      String rk = "testRK";
+      String icvCol = "icvCol";
+      String appendCol = "appendCol";
+      String regionName = null;
+      HTable hTable =
+          TEST_UTIL.createTable(TABLE_NAME.getBytes(),
+              new byte[][] { cf.getBytes(), otherCf.getBytes() });
+      Set<HRegionInfo> regionInfos = hTable.getRegionLocations().keySet();
+  
+      regionName = regionInfos.toArray(new HRegionInfo[regionInfos.size()])[0].getEncodedName();
+  
+      //Do a multi put that has one cf.  Since they are in different rk's
+      //The lock will still be obtained and everything will be applied in one multiput.
+      Put pOne = new Put(rk.getBytes());
+      pOne.add(cf.getBytes(), icvCol.getBytes(), Bytes.toBytes(0L));
+      Put pTwo = new Put("ignored1RK".getBytes());
+      pTwo.add(cf.getBytes(), "ignored".getBytes(), Bytes.toBytes(0L));
+      
+      hTable.put(Arrays.asList(new Put[] {pOne, pTwo}));
+  
+      // Do a multiput where the cf doesn't stay consistent.
+      Put pThree = new Put("ignored2RK".getBytes());
+      pThree.add(cf.getBytes(), "ignored".getBytes(), Bytes.toBytes("TEST1"));
+      Put pFour = new Put("ignored3RK".getBytes());
+      pFour.add(otherCf.getBytes(), "ignored".getBytes(), Bytes.toBytes(0L));
+  
+      hTable.put(Arrays.asList(new Put[] { pThree, pFour }));
+  
+      hTable.incrementColumnValue(rk.getBytes(), cf.getBytes(), icvCol.getBytes(), 1L);
+      
+      Increment i = new Increment(rk.getBytes());
+      i.addColumn(cf.getBytes(), icvCol.getBytes(), 1L);
+      hTable.increment(i);
+  
+      Get g = new Get(rk.getBytes());
+      g.addColumn(cf.getBytes(), appendCol.getBytes());
+      hTable.get(g);
+  
+      Append a = new Append(rk.getBytes());
+      a.add(cf.getBytes(), appendCol.getBytes(), Bytes.toBytes("-APPEND"));
+      hTable.append(a);
+  
+      Delete dOne = new Delete(rk.getBytes());
+      dOne.deleteFamily(cf.getBytes());
+      hTable.delete(dOne);
+  
+      Delete dTwo = new Delete(rk.getBytes());
+      hTable.delete(dTwo);
+  
+      // There should be one multi put where the cf is consistent
+      assertTimeVaryingMetricCount(1, TABLE_NAME, cf, null, "multiput_");
+  
+      // There were two multiputs to the cf.
+      assertTimeVaryingMetricCount(2, TABLE_NAME, null, regionName, "multiput_");
+  
+      // There was one multiput where the cf was not consistent.
+      assertTimeVaryingMetricCount(1, TABLE_NAME, "__unknown", null, "multiput_");
+  
+      // One increment and one append
+      assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "incrementColumnValue_");
+      assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "increment_");
+      assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "append_");
+  
+      // One delete where the cf is known
+      assertTimeVaryingMetricCount(1, TABLE_NAME, cf, null, "delete_");
+  
+      // two deletes in the region.
+      assertTimeVaryingMetricCount(2, TABLE_NAME, null, regionName, "delete_");
+  
+      // Three gets. one for gets. One for append. One for increment.
+      assertTimeVaryingMetricCount(4, TABLE_NAME, cf, regionName, "get_");
+  
+    }
+  
   @Test
   public void testMultipleRegions() throws IOException, InterruptedException {
 
@@ -124,7 +235,7 @@ public void testMultipleRegions() throws IOException, InterruptedException {
     final String storeMetricName = ALL_METRICS
         .getStoreMetricNameMax(StoreMetricType.STORE_FILE_COUNT);
     assertEquals("Invalid value for store metric " + storeMetricName,
-        NUM_FLUSHES, HRegion.getNumericMetric(storeMetricName));
+        NUM_FLUSHES, RegionMetricsStorage.getNumericMetric(storeMetricName));
   }
 
 

From 06c229ab498f7c181b1e01e08c79256838ac4082 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 25 Apr 2012 00:07:39 +0000
Subject: [PATCH 0162/1540] HBASE-5861 Hadoop 23 compile broken due to tests
 introduced in HBASE-5064

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330071 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TestHLogRecordReader.java  | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
index 0b3ba838b659..f91187b0a418 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
@@ -18,8 +18,8 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.util.List;
 
@@ -40,9 +40,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.MapReduceTestUtil;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -144,13 +142,13 @@ public void testPartialRead() throws Exception {
     jobConf.setLong(HLogInputFormat.END_TIME_KEY, ts);
 
     // only 1st file is considered, and only its 1st entry is used
-    List<InputSplit> splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    List<InputSplit> splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf));
     assertEquals(1, splits.size());
     testSplit(splits.get(0), Bytes.toBytes("1"));
 
     jobConf.setLong(HLogInputFormat.START_TIME_KEY, ts+1);
     jobConf.setLong(HLogInputFormat.END_TIME_KEY, ts1+1);
-    splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf));
     // both files need to be considered
     assertEquals(2, splits.size());
     // only the 2nd entry from the 1st file is used
@@ -191,7 +189,7 @@ public void testHLogRecordReader() throws Exception {
     jobConf.set("mapred.input.dir", logDir.toString());
 
     // make sure both logs are found
-    List<InputSplit> splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    List<InputSplit> splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf));
     assertEquals(2, splits.size());
 
     // should return exactly one KV
@@ -203,14 +201,14 @@ public void testHLogRecordReader() throws Exception {
 
     // set an endtime, the 2nd log file can be ignored completely.
     jobConf.setLong(HLogInputFormat.END_TIME_KEY, secondTs-1);
-    splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf));
     assertEquals(1, splits.size());
     testSplit(splits.get(0), Bytes.toBytes("1"));
 
     // now set a start time
     jobConf.setLong(HLogInputFormat.END_TIME_KEY, Long.MAX_VALUE);
     jobConf.setLong(HLogInputFormat.START_TIME_KEY, thirdTs);
-    splits = input.getSplits(new JobContext(jobConf, new JobID()));
+    splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf));
     // both logs need to be considered
     assertEquals(2, splits.size());
     // but both readers skip all edits
@@ -223,7 +221,7 @@ public void testHLogRecordReader() throws Exception {
    */
   private void testSplit(InputSplit split, byte[]... columns) throws Exception {
     HLogRecordReader reader = new HLogRecordReader();
-    reader.initialize(split, new TaskAttemptContext(conf, new TaskAttemptID()));
+    reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf));
 
     for (byte[] column : columns) {
       assertTrue(reader.nextKeyValue());

From 3efbb8846ab1f8a85af968e60882a7b7d67451b6 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 25 Apr 2012 01:00:00 +0000
Subject: [PATCH 0163/1540] HBASE-5861 ADDENDUM (missed new
 MapredcueTestingShim.java file)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330077 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/MapreduceTestingShim.java | 88 +++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
new file mode 100644
index 000000000000..76de6a39c804
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+
+/**
+ * This class provides shims for HBase to interact with the Hadoop 1.0.x and the
+ * Hadoop 0.23.x series.
+ * 
+ * NOTE: No testing done against 0.22.x, or 0.21.x.
+ */
+abstract public class MapreduceTestingShim {
+  private static MapreduceTestingShim instance;
+
+  static {
+    try {
+      // This class exists in hadoop 0.22+ but not in Hadoop 20.x/1.x
+      Class c = Class
+          .forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl");
+      instance = new MapreduceV2Shim();
+    } catch (Exception e) {
+      instance = new MapreduceV1Shim();
+    }
+  }
+
+  abstract public JobContext newJobContext(Configuration jobConf)
+      throws IOException;
+
+  public static JobContext createJobContext(Configuration jobConf)
+      throws IOException {
+    return instance.newJobContext(jobConf);
+  }
+
+  private static class MapreduceV1Shim extends MapreduceTestingShim {
+    public JobContext newJobContext(Configuration jobConf) throws IOException {
+      // Implementing:
+      // return new JobContext(jobConf, new JobID());
+      JobID jobId = new JobID();
+      Constructor<JobContext> c;
+      try {
+        c = JobContext.class.getConstructor(Configuration.class, JobID.class);
+        return c.newInstance(jobConf, jobId);
+      } catch (Exception e) {
+        throw new IllegalStateException(
+            "Failed to instantiate new JobContext(jobConf, new JobID())", e);
+      }
+    }
+  };
+
+  private static class MapreduceV2Shim extends MapreduceTestingShim {
+    public JobContext newJobContext(Configuration jobConf) {
+      // Implementing:
+      // return Job.getInstance(jobConf);
+      try {
+        Method m = Job.class.getMethod("getInstance", Configuration.class);
+        return (JobContext) m.invoke(null, jobConf); // static method, then arg
+      } catch (Exception e) {
+        e.printStackTrace();
+        throw new IllegalStateException(
+            "Failed to return from Job.getInstance(jobConf)");
+      }
+    }
+  };
+
+}

From b51b7928324bcfcc460aa4e37ddcbd47a8272071 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 25 Apr 2012 05:07:22 +0000
Subject: [PATCH 0164/1540] HBASE-5848 Addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330106 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/AssignmentManager.java   | 6 +++++-
 .../java/org/apache/hadoop/hbase/TestRegionRebalancing.java | 6 ++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index e33e81eabdcc..58fab1faae1b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1755,7 +1755,11 @@ boolean asyncSetOfflineInZooKeeper(final RegionState state,
       ZKAssign.asyncCreateNodeOffline(master.getZooKeeper(), state.getRegion(),
         this.master.getServerName(), cb, ctx);
     } catch (KeeperException e) {
-      master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
+      if (e instanceof NodeExistsException) {
+        LOG.warn("Node for " + state.getRegion() + " already exists");
+      } else { 
+        master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
+      }
       return false;
     }
     return true;
diff --git a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
index e3b148f3bb9d..5574b7f97ab7 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -78,7 +79,8 @@ public void before() {
   public void testRebalanceOnRegionServerNumberChange()
   throws IOException, InterruptedException {
     HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
-    admin.createTable(this.desc, HBaseTestingUtility.KEYS);
+    admin.createTable(this.desc, Arrays.copyOfRange(HBaseTestingUtility.KEYS,
+        1, HBaseTestingUtility.KEYS.length));
     this.table = new HTable(UTIL.getConfiguration(), this.desc.getName());
     CatalogTracker ct = new CatalogTracker(UTIL.getConfiguration());
     ct.start();
@@ -88,7 +90,7 @@ public void testRebalanceOnRegionServerNumberChange()
       ct.stop();
     }
     assertEquals("Test table should have right number of regions",
-      HBaseTestingUtility.KEYS.length + 1/*One extra to account for start/end keys*/,
+      HBaseTestingUtility.KEYS.length,
       this.table.getStartKeys().length);
 
     // verify that the region assignments are balanced to start out

From 5e03e6e79f2bd40c08d0176a1e63bf91d0df693d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 25 Apr 2012 05:46:59 +0000
Subject: [PATCH 0165/1540] HBASE-5849 On first cluster startup, RS aborts if
 root znode is not available; REAPPLY

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330117 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     |   6 -
 .../hadoop/hbase/TestClusterBootOrder.java    | 121 ++++++++++++++++++
 2 files changed, 121 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 607427354c76..ca5a7c4af77d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -606,12 +606,6 @@ private void initializeZooKeeper() throws IOException, InterruptedException {
    */
   private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
       throws IOException, InterruptedException {
-    if (false == tracker.checkIfBaseNodeAvailable()) {
-      String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. "
-          + "There could be a mismatch with the one configured in the master.";
-      LOG.error(errorMsg);
-      abort(errorMsg);
-    }
     while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
       if (this.stopped) {
         throw new IOException("Received the shutdown message while waiting.");
diff --git a/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
index e69de29bb2d1..b150d135cc54 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests the boot order indifference between regionserver and master
+ */
+@Category(MediumTests.class)
+public class TestClusterBootOrder {
+
+  private static final long SLEEP_INTERVAL = 1000;
+  private static final long SLEEP_TIME = 4000;
+
+  private HBaseTestingUtility testUtil;
+  private LocalHBaseCluster cluster;
+  private RegionServerThread rs;
+  private MasterThread master;
+
+  @Before
+  public void setUp() throws Exception {
+    testUtil = new HBaseTestingUtility();
+    testUtil.startMiniDFSCluster(1);
+    testUtil.startMiniZKCluster(1);
+    testUtil.createRootDir(); //manually setup hbase dir to point to minidfscluster
+    cluster = new LocalHBaseCluster(testUtil.getConfiguration(), 0, 0);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    cluster.shutdown();
+    cluster.join();
+    testUtil.shutdownMiniZKCluster();
+    testUtil.shutdownMiniDFSCluster();
+  }
+
+  private void startRegionServer() throws Exception {
+    rs = cluster.addRegionServer();
+    rs.start();
+
+    for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) {
+      //we cannot block on wait for rs at this point , since master is not up.
+      Thread.sleep(SLEEP_INTERVAL);
+      assertTrue(rs.isAlive());
+    }
+  }
+
+  private void startMaster() throws Exception {
+    master = cluster.addMaster();
+    master.start();
+
+    for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) {
+      Thread.sleep(SLEEP_INTERVAL);
+      assertTrue(master.isAlive());
+    }
+  }
+
+  private void waitForClusterOnline() {
+    while (true) {
+      if (master.getMaster().isInitialized()) {
+        break;
+      }
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException ignored) {
+        // Keep waiting
+      }
+    }
+    rs.waitForServerOnline();
+  }
+
+  /**
+   * Tests launching the cluster by first starting regionserver, and then the master
+   * to ensure that it does not matter which is started first.
+   */
+  @Test
+  public void testBootRegionServerFirst() throws Exception {
+    startRegionServer();
+    startMaster();
+    waitForClusterOnline();
+  }
+
+  /**
+   * Tests launching the cluster by first starting master, and then the regionserver
+   * to ensure that it does not matter which is started first.
+   */
+  @Test
+  public void testBootMasterFirst() throws Exception {
+    startMaster();
+    startRegionServer();
+    waitForClusterOnline();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 5cb34a14cead69d67ebf1b9ac6fc5ce371843517 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 25 Apr 2012 06:02:07 +0000
Subject: [PATCH 0166/1540] HBASE-5871 Usability regression, we don't parse
 compression algos anymore

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330124 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/admin.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 7f0d6d7b5d7b..f761d21d7bc8 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -532,7 +532,6 @@ def hcd(arg, htd)
       family.setScope(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE)
       family.setInMemory(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY)
       family.setTimeToLive(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::TTL])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL)
-      family.setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION)
       family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING)
       family.setEncodeOnDisk(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK)
       family.setBlocksize(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE)

From b85bdb6be135bf168a7a961bc5a3d9b670d40b7d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 25 Apr 2012 20:43:01 +0000
Subject: [PATCH 0167/1540] HBASE-5873 TimeOut Monitor thread should be started
 after atleast one region server registers.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330549 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/AssignmentManager.java  | 7 +++++--
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java  | 1 +
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 58fab1faae1b..14fc66783f55 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -207,14 +207,17 @@ public AssignmentManager(Server master, ServerManager serverManager,
       conf.getInt("hbase.master.assignment.timeoutmonitor.period", 10000),
       master, serverManager,
       conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 1800000));
-    Threads.setDaemonThreadRunning(timeoutMonitor.getThread(),
-      master.getServerName() + ".timeoutMonitor");
     this.zkTable = new ZKTable(this.master.getZooKeeper());
     this.maximumAssignmentAttempts =
       this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
     this.balancer = balancer;
     this.threadPoolExecutorService = Executors.newCachedThreadPool();
   }
+  
+  void startTimeOutMonitor() {
+    Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), master.getServerName()
+        + ".timeoutMonitor");
+  }
 
   /**
    * Compute the average load across all region servers.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 2f2a7f022b0d..20e042680f31 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -514,6 +514,7 @@ private void finishInitialization(MonitoredTask status)
       }
     }
 
+    this.assignmentManager.startTimeOutMonitor();
     Set<ServerName> onlineServers = new HashSet<ServerName>(serverManager
         .getOnlineServers().keySet());
     // TODO: Should do this in background rather than block master startup

From 78717a460f5f9bced8c5ecd5657457a8c32bbeb6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 26 Apr 2012 18:00:42 +0000
Subject: [PATCH 0168/1540] HBASE-5862 After Region Close remove the Operation
 Metrics

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1330998 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  1 +
 .../hbase/regionserver/HRegionServer.java     |  8 +++
 .../metrics/OperationMetrics.java             | 15 +++--
 .../metrics/RegionMetricsStorage.java         |  8 +++
 .../metrics/RegionServerDynamicMetrics.java   | 62 +++++++++++++++++++
 .../regionserver/TestRegionServerMetrics.java | 27 ++++++++
 6 files changed, 117 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index ca9983c7bec1..4bc29a20f244 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -926,6 +926,7 @@ public ImmutableList<StoreFile> call() throws IOException {
         status.setStatus("Running coprocessor post-close hooks");
         this.coprocessorHost.postClose(abort);
       }
+      this.opMetrics.closeMetrics();
       status.markComplete("Closed");
       LOG.info("Closed " + this);
       return result;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ca5a7c4af77d..e78dc63e4a88 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3019,6 +3019,14 @@ public void addToOnlineRegions(HRegion region) {
   public boolean removeFromOnlineRegions(final String encodedName) {
     HRegion toReturn = null;
     toReturn = this.onlineRegions.remove(encodedName);
+    
+    //Clear all of the dynamic metrics as they are now probably useless.
+    //This is a clear because dynamic metrics could include metrics per cf and
+    //per hfile.  Figuring out which cfs, hfiles, and regions are still relevant to
+    //this region server would be an onerous task.  Instead just clear everything
+    //and on the next tick of the metrics everything that is still relevant will be
+    //re-added.
+    this.dynamicMetrics.clear();
     return toReturn != null;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
index ac4e57f56c2a..305dfaba40f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -64,12 +63,12 @@ public class OperationMetrics {
    * @param conf The Configuration of the HRegion reporting operations coming in.
    * @param regionInfo The region info
    */
-  public OperationMetrics(Configuration conf, HRegionInfo regionInfo) {
+  public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { 
     // Configure SchemaMetrics before trying to create a RegionOperationMetrics instance as
     // RegionOperationMetrics relies on SchemaMetrics to do naming.
     if (conf != null) {
       SchemaMetrics.configureGlobally(conf);
-
+      
       this.conf = conf;
       if (regionInfo != null) {
         this.tableName = regionInfo.getTableNameAsString();
@@ -172,6 +171,13 @@ public void updatePutMetrics(Set<byte[]> columnFamilies, long value) {
   public void updateDeleteMetrics(Set<byte[]> columnFamilies, long value) {
     doUpdateTimeVarying(columnFamilies, DELETE_KEY, value);
   }
+  
+  /**
+   * This deletes all old metrics this instance has ever created or updated.
+   */
+  public void closeMetrics() {
+    RegionMetricsStorage.clear();
+  }
 
   /**
    * Method to send updates for cf and region metrics. This is the normal method
@@ -199,7 +205,8 @@ private void doUpdateTimeVarying(Set<byte[]> columnFamilies, String key, long va
   private void doSafeIncTimeVarying(String prefix, String key, long value) {
     if (conf.getBoolean(CONF_KEY, true)) {
       if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) {
-        RegionMetricsStorage.incrTimeVaryingMetric(prefix + key, value);
+        String m = prefix + key;
+        RegionMetricsStorage.incrTimeVaryingMetric(m, value);
       }
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
index 416e495057d9..5d4beffc2e2c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
@@ -127,4 +127,12 @@ public static long getNumericPersistentMetric(String key) {
     return m.get();
   }
 
+  /**
+   * Clear all copies of the metrics this stores.
+   */
+  public static void clear() {
+    timeVaryingMetrics.clear();
+    numericMetrics.clear();
+    numericPersistentMetrics.clear();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
index 752b12748fe5..fd0efa4f2be6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
@@ -20,7 +20,9 @@
 
 package org.apache.hadoop.hbase.regionserver.metrics;
 
+import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -50,12 +52,18 @@
  *
  */
 public class RegionServerDynamicMetrics implements Updater {
+  private static final String UNABLE_TO_CLEAR = "Unable to clear RegionServerDynamicMetrics";
+  
   private MetricsRecord metricsRecord;
   private MetricsContext context;
   private final RegionServerDynamicStatistics rsDynamicStatistics;
   private Method updateMbeanInfoIfMetricsListChanged = null;
   private static final Log LOG =
     LogFactory.getLog(RegionServerDynamicStatistics.class);
+  
+  private boolean reflectionInitialized = false;
+  private Field recordMetricMapField;
+  private Field registryMetricMapField;
 
   /**
    * The metrics variables are public:
@@ -124,6 +132,60 @@ public synchronized void incrTimeVaryingMetric(
       m.inc(numOps, amt);
     }
   }
+  
+  /**
+   * Clear all metrics this exposes. 
+   * Uses reflection to clear them from hadoop metrics side as well.
+   */
+  @SuppressWarnings("rawtypes")
+  public void clear() {
+    
+    // If this is the first clear use reflection to get the two maps that hold copies of our 
+    // metrics on the hadoop metrics side. We have to use reflection because there is not 
+    // remove metrics on the hadoop side. If we can't get them then clearing old metrics 
+    // is not possible and bailing out early is our best option.
+    if (!this.reflectionInitialized) {
+      this.reflectionInitialized = true;
+      try {
+        this.recordMetricMapField = this.metricsRecord.getClass().getDeclaredField("metricTable");
+        this.recordMetricMapField.setAccessible(true);
+      } catch (SecurityException e) {
+        LOG.debug(UNABLE_TO_CLEAR);
+        return;
+      } catch (NoSuchFieldException e) {
+        LOG.debug(UNABLE_TO_CLEAR);
+        return;
+      }
+
+      try {
+        this.registryMetricMapField = this.registry.getClass().getDeclaredField("metricsList");
+        this.registryMetricMapField.setAccessible(true);
+      } catch (SecurityException e) {
+        LOG.debug(UNABLE_TO_CLEAR);
+        return;
+      } catch (NoSuchFieldException e) {
+        LOG.debug(UNABLE_TO_CLEAR);
+        return;
+      } 
+    }
+
+    
+    //If we found both fields then try and clear the maps.
+    if (this.recordMetricMapField != null && this.registryMetricMapField != null) {
+      try {
+        Map recordMap = (Map) this.recordMetricMapField.get(this.metricsRecord);
+        recordMap.clear();
+        Map registryMap = (Map) this.registryMetricMapField.get(this.registry);
+        registryMap.clear();
+      } catch (IllegalArgumentException e) {
+        LOG.debug(UNABLE_TO_CLEAR);
+      } catch (IllegalAccessException e) {
+        LOG.debug(UNABLE_TO_CLEAR);
+      }
+    } else {
+      LOG.debug(UNABLE_TO_CLEAR);
+    }
+  }
 
   /**
    * Push the metrics to the monitoring subsystem on doUpdate() call.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 879f1d65186f..897a00186402 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -47,6 +47,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+
 /**
  * Test metrics incremented on region server operations.
  */
@@ -199,6 +200,32 @@ public void testOperationMetrics() throws IOException {
   
     }
   
+  @Test
+  public void testRemoveRegionMetrics() throws IOException, InterruptedException {
+    String cf = "REMOVECF";
+    HTable hTable = TEST_UTIL.createTable(TABLE_NAME.getBytes(), cf.getBytes());
+    HRegionInfo[] regionInfos =
+        hTable.getRegionLocations().keySet()
+            .toArray(new HRegionInfo[hTable.getRegionLocations().keySet().size()]);
+
+    String regionName = regionInfos[0].getEncodedName();
+
+    // Do some operations so there are metrics.
+    Put pOne = new Put("TEST".getBytes());
+    pOne.add(cf.getBytes(), "test".getBytes(), "test".getBytes());
+    hTable.put(pOne);
+
+    Get g = new Get("TEST".getBytes());
+    g.addFamily(cf.getBytes());
+    hTable.get(g);
+    assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "get_");
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    admin.disableTable(TABLE_NAME.getBytes());
+    admin.deleteTable(TABLE_NAME.getBytes());
+
+    assertTimeVaryingMetricCount(0, TABLE_NAME, cf, regionName, "get_");
+  }
+  
   @Test
   public void testMultipleRegions() throws IOException, InterruptedException {
 

From ea152e2f464933af32778b34d2e0530624c1cd62 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 26 Apr 2012 19:19:47 +0000
Subject: [PATCH 0169/1540] HBASE-5862 After Region Close remove the Operation
 Metrics; ADDENDUM -- missing import

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331040 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/TestRegionServerMetrics.java       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 897a00186402..8d0dc8184c34 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;

From 065a84ae6a47114ae65a220d8df73930d3eecd39 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 26 Apr 2012 20:06:34 +0000
Subject: [PATCH 0170/1540] HBASE-5864 Error while reading from hfile in 0.94
 (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331057 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileBlock.java     |  9 ++--
 .../hbase/io/hfile/HFileBlockIndex.java       | 31 +++++++++---
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  6 +--
 .../hbase/io/hfile/TestHFileBlockIndex.java   |  2 +-
 .../hbase/io/hfile/TestHFileWriterV2.java     | 49 +++++++++++++------
 5 files changed, 68 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 052e49cc1ada..65acd102fa3e 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -1189,10 +1189,9 @@ public interface BlockIterator {
 
     /**
      * Similar to {@link #nextBlock()} but checks block type, throws an
-     * exception if incorrect, and returns the data portion of the block as
-     * an input stream.
+     * exception if incorrect, and returns the HFile block
      */
-    DataInputStream nextBlockAsStream(BlockType blockType) throws IOException;
+    HFileBlock nextBlockWithBlockType(BlockType blockType) throws IOException;
   }
 
   /** A full-fledged reader with iteration ability. */
@@ -1290,14 +1289,14 @@ public HFileBlock nextBlock() throws IOException {
         }
 
         @Override
-        public DataInputStream nextBlockAsStream(BlockType blockType)
+        public HFileBlock nextBlockWithBlockType(BlockType blockType)
             throws IOException {
           HFileBlock blk = nextBlock();
           if (blk.getBlockType() != blockType) {
             throw new IOException("Expected block of type " + blockType
                 + " but found " + blk.getBlockType());
           }
-          return blk.getByteStream();
+          return blk;
         }
       };
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index cae932651943..576c988f6163 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -531,24 +531,43 @@ public void readRootIndex(DataInput in, final int numEntries)
         }
       }
     }
+    
+    /**
+     * Read in the root-level index from the given input stream. Must match
+     * what was written into the root level by
+     * {@link BlockIndexWriter#writeIndexBlocks(FSDataOutputStream)} at the
+     * offset that function returned.
+     *
+     * @param blk the HFile block
+     * @param numEntries the number of root-level index entries
+     * @return the buffered input stream or wrapped byte input stream
+     * @throws IOException
+     */
+    public DataInputStream readRootIndex(HFileBlock blk, final int numEntries) throws IOException {
+      DataInputStream in = blk.getByteStream();
+      readRootIndex(in, numEntries);
+      return in;
+    }
 
     /**
      * Read the root-level metadata of a multi-level block index. Based on
      * {@link #readRootIndex(DataInput, int)}, but also reads metadata
      * necessary to compute the mid-key in a multi-level index.
      *
-     * @param in the buffered or byte input stream to read from
+     * @param blk the HFile block
      * @param numEntries the number of root-level index entries
      * @throws IOException
      */
-    public void readMultiLevelIndexRoot(DataInputStream in,
+    public void readMultiLevelIndexRoot(HFileBlock blk,
         final int numEntries) throws IOException {
-      readRootIndex(in, numEntries);
-      if (in.available() < MID_KEY_METADATA_SIZE) {
+      DataInputStream in = readRootIndex(blk, numEntries);
+      // after reading the root index the checksum bytes have to
+      // be subtracted to know if the mid key exists.
+      int checkSumBytes = blk.totalChecksumBytes();
+      if ((in.available() - checkSumBytes) < MID_KEY_METADATA_SIZE) {
         // No mid-key metadata available.
         return;
       }
-
       midLeafBlockOffset = in.readLong();
       midLeafBlockOnDiskSize = in.readInt();
       midKeyEntry = in.readInt();
@@ -761,7 +780,7 @@ public long writeIndexBlocks(FSDataOutputStream out) throws IOException {
 
       if (LOG.isTraceEnabled()) {
         LOG.trace("Wrote a " + numLevels + "-level index with root level at pos "
-          + out.getPos() + ", " + rootChunk.getNumEntries()
+          + rootLevelIndexPos + ", " + rootChunk.getNumEntries()
           + " root-level entries, " + totalNumEntries + " total entries, "
           + StringUtils.humanReadableInt(this.totalBlockOnDiskSize) +
           " on-disk size, "
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 7334d0692eb5..1f777cbcad64 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -124,17 +124,17 @@ public HFileReaderV2(Path path, FixedFileTrailer trailer,
     // Data index. We also read statistics about the block index written after
     // the root level.
     dataBlockIndexReader.readMultiLevelIndexRoot(
-        blockIter.nextBlockAsStream(BlockType.ROOT_INDEX),
+        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
         trailer.getDataIndexCount());
 
     // Meta index.
     metaBlockIndexReader.readRootIndex(
-        blockIter.nextBlockAsStream(BlockType.ROOT_INDEX),
+        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
         trailer.getMetaIndexCount());
 
     // File info
     fileInfo = new FileInfo();
-    fileInfo.readFields(blockIter.nextBlockAsStream(BlockType.FILE_INFO));
+    fileInfo.readFields(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
     lastKey = fileInfo.get(FileInfo.LASTKEY);
     avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
     avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 3f36e64cde25..4f007553859e 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -176,7 +176,7 @@ public void readIndex() throws IOException {
             Bytes.BYTES_RAWCOMPARATOR, numLevels, brw);
 
     indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset,
-        fileSize).nextBlockAsStream(BlockType.ROOT_INDEX), numRootEntries);
+        fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries);
 
     long prevOffset = -1;
     int i = 0;
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
index e3b18c91c924..cd0852b3ee27 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
@@ -37,8 +37,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
@@ -72,14 +71,30 @@ public void setUp() throws IOException {
   @Test
   public void testHFileFormatV2() throws IOException {
     Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
-        "testHFileFormatV2");
+    "testHFileFormatV2");
+    final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ;
+    final int entryCount = 10000;
+    writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false);
+  }
+  
+  
+  @Test
+  public void testMidKeyInHFile() throws IOException{
+    Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
+    "testMidKeyInHFile");
+    Compression.Algorithm compressAlgo = Compression.Algorithm.NONE;
+    int entryCount = 50000;
+    writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true);
+  }
+
+  private void writeDataAndReadFromHFile(Path hfilePath,
+      Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException {
 
-    final Compression.Algorithm COMPRESS_ALGO = Compression.Algorithm.GZ;
     HFileWriterV2 writer = (HFileWriterV2)
         new HFileWriterV2.WriterFactoryV2(conf, new CacheConfig(conf))
             .withPath(fs, hfilePath)
             .withBlockSize(4096)
-            .withCompression(COMPRESS_ALGO)
+            .withCompression(compressAlgo)
             .withComparator(KeyValue.KEY_COMPARATOR)
             .create();
 
@@ -88,11 +103,10 @@ public void testHFileFormatV2() throws IOException {
 
     Random rand = new Random(9713312); // Just a fixed seed.
 
-    final int ENTRY_COUNT = 10000;
     List<byte[]> keys = new ArrayList<byte[]>();
     List<byte[]> values = new ArrayList<byte[]>();
 
-    for (int i = 0; i < ENTRY_COUNT; ++i) {
+    for (int i = 0; i < entryCount; ++i) {
       byte[] keyBytes = randomOrderedKey(rand, i);
 
       // A random-length random value.
@@ -113,6 +127,7 @@ public void testHFileFormatV2() throws IOException {
     writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris"));
 
     writer.close();
+    
 
     FSDataInputStream fsdis = fs.open(hfilePath);
 
@@ -124,10 +139,10 @@ public void testHFileFormatV2() throws IOException {
         FixedFileTrailer.readFromStream(fsdis, fileSize);
 
     assertEquals(2, trailer.getMajorVersion());
-    assertEquals(ENTRY_COUNT, trailer.getEntryCount());
+    assertEquals(entryCount, trailer.getEntryCount());
 
     HFileBlock.FSReader blockReader =
-        new HFileBlock.FSReaderV2(fsdis, COMPRESS_ALGO, fileSize);
+        new HFileBlock.FSReaderV2(fsdis, compressAlgo, fileSize);
     // Comparator class name is stored in the trailer in version 2.
     RawComparator<byte []> comparator = trailer.createComparator();
     HFileBlockIndex.BlockIndexReader dataBlockIndexReader =
@@ -143,16 +158,21 @@ public void testHFileFormatV2() throws IOException {
     // Data index. We also read statistics about the block index written after
     // the root level.
     dataBlockIndexReader.readMultiLevelIndexRoot(
-        blockIter.nextBlockAsStream(BlockType.ROOT_INDEX),
+        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
         trailer.getDataIndexCount());
-
+    
+    if (findMidKey) {
+      byte[] midkey = dataBlockIndexReader.midkey();
+      assertNotNull("Midkey should not be null", midkey);
+    }
+    
     // Meta index.
     metaBlockIndexReader.readRootIndex(
-        blockIter.nextBlockAsStream(BlockType.ROOT_INDEX),
+        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX).getByteStream(),
         trailer.getMetaIndexCount());
     // File info
     FileInfo fileInfo = new FileInfo();
-    fileInfo.readFields(blockIter.nextBlockAsStream(BlockType.FILE_INFO));
+    fileInfo.readFields(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
     byte [] keyValueFormatVersion = fileInfo.get(
         HFileWriterV2.KEY_VALUE_VERSION);
     boolean includeMemstoreTS = keyValueFormatVersion != null &&
@@ -200,7 +220,7 @@ public void testHFileFormatV2() throws IOException {
     }
     LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead="
         + blocksRead);
-    assertEquals(ENTRY_COUNT, entriesRead);
+    assertEquals(entryCount, entriesRead);
 
     // Meta blocks. We can scan until the load-on-open data offset (which is
     // the root block index offset in version 2) because we are not testing
@@ -226,6 +246,7 @@ public void testHFileFormatV2() throws IOException {
     fsdis.close();
   }
 
+
   // Static stuff used by various HFile v2 unit tests
 
   private static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_";

From c102a23656861a3bdefcc27300aeeebd7bd25fff Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 26 Apr 2012 20:24:39 +0000
Subject: [PATCH 0171/1540] new CHANGES.txt for RC3

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331066 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index ca13be91e469..66e5d8ec19dc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.0 - 4/18/2012
+Release 0.94.0 - 4/26/2012
 Sub-task
 
     [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently
@@ -109,7 +109,6 @@ Bug
     [HBASE-5121] - MajorCompaction may affect scan's correctness
     [HBASE-5141] - Memory leak in MonitoredRPCHandlerImpl
     [HBASE-5152] - Region is on service before completing initialization when doing rollback of split, it will affect read correctness
-    [HBASE-5161] - Compaction algorithm should prioritize reference files
     [HBASE-5163] - TestLogRolling#testLogRollOnDatanodeDeath fails sometimes on Jenkins or hadoop QA ("The directory is already locked.")
     [HBASE-5172] - HTableInterface should extend java.io.Closeable
     [HBASE-5176] - AssignmentManager#getRegion: logging nit adds a redundant '+'
@@ -186,6 +185,7 @@ Bug
     [HBASE-5623] - Race condition when rolling the HLog and hlogFlush
     [HBASE-5624] - Aborting regionserver when splitting region, may cause daughter region not assigned by ServerShutdownHandler.
     [HBASE-5633] - NPE reading ZK config in HBase
+    [HBASE-5635] - If getTaskList() returns null, splitlogWorker would go down and it won't serve any requests
     [HBASE-5636] - TestTableMapReduce doesn't work properly.
     [HBASE-5639] - The logic used in waiting for region servers during startup is broken
     [HBASE-5656] - LoadIncrementalHFiles createTable should detect and set compression algorithm
@@ -207,7 +207,19 @@ Bug
     [HBASE-5780] - Fix race in HBase regionserver startup vs ZK SASL authentication
     [HBASE-5781] - Zookeeper session got closed while trying to assign the region to RS using hbck -fix
     [HBASE-5782] - Edits can be appended out of seqid order since HBASE-4487
+    [HBASE-5787] - Table owner can't disable/delete his/her own table
     [HBASE-5795] - HServerLoad$RegionLoad breaks 0.92<->0.94 compatibility
+    [HBASE-5825] - TestHLog not running any tests; fix
+    [HBASE-5833] - 0.92 build has been failing pretty consistently on TestMasterFailover....
+    [HBASE-5848] - Create table with EMPTY_START_ROW passed as splitKey causes the HMaster to abort
+    [HBASE-5849] - On first cluster startup, RS aborts if root znode is not available
+    [HBASE-5850] - Refuse operations from Admin before master is initialized - fix for all branches.
+    [HBASE-5857] - RIT map in RS not getting cleared while region opening
+    [HBASE-5861] - Hadoop 23 compilation broken due to tests introduced in HBASE-5604
+    [HBASE-5864] - Error while reading from hfile in 0.94
+    [HBASE-5866] - Canary in tool package but says its in tools.
+    [HBASE-5871] - Usability regression, we don't parse compression algos anymore
+    [HBASE-5873] - TimeOut Monitor thread should be started after atleast one region server registers.
 
 Improvement
 
@@ -347,10 +359,12 @@ Improvement
     [HBASE-5706] - "Dropping fs latency stats since buffer is full" spam
     [HBASE-5734] - Change hbck sideline root
     [HBASE-5735] - Clearer warning message when connecting a non-secure HBase client to a secure HBase server
+    [HBASE-5737] - Minor Improvements related to balancer.
     [HBASE-5748] - Enable lib directory in jar file for coprocessor
     [HBASE-5770] - Add a clock skew warning threshold
     [HBASE-5775] - ZKUtil doesn't handle deleteRecurisively cleanly
     [HBASE-5823] - Hbck should be able to print help
+    [HBASE-5862] - After Region Close remove the Operation Metrics.
 
 New Feature
 
@@ -360,6 +374,7 @@ New Feature
     [HBASE-3856] - Build a tree structure data block index inside of the HFile
     [HBASE-4102] - atomicAppend: A put that appends to the latest version of a cell; i.e. reads current value then adds the bytes offered by the client to the tail and writes out a new entry
     [HBASE-4219] - Add Per-Column Family Metrics
+    [HBASE-4393] - Implement a canary monitoring program
     [HBASE-4460] - Support running an embedded ThriftServer within a RegionServer
     [HBASE-4536] - Allow CF to retain deleted rows
     [HBASE-4608] - HLog Compression

From 88117178ce29800523e659c62ad3b0233606cec4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 27 Apr 2012 23:54:51 +0000
Subject: [PATCH 0172/1540] HBASE-5893 Allow spaces in coprocessor conf (aka
 trim() className)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331639 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/coprocessor/CoprocessorHost.java    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 6268afb612e9..3e61aa23a2e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -113,15 +113,15 @@ public Set<String> getCoprocessors() {
   protected void loadSystemCoprocessors(Configuration conf, String confKey) {
     Class<?> implClass = null;
 
-    // load default coprocessors from configure file
-    String defaultCPClasses = conf.get(confKey);
-    if (defaultCPClasses == null || defaultCPClasses.length() == 0)
+    // load default coprocessors from configure file    
+    String[] defaultCPClasses = conf.getStrings(confKey);
+    if (defaultCPClasses == null || defaultCPClasses.length == 0)
       return;
-    StringTokenizer st = new StringTokenizer(defaultCPClasses, ",");
+
     int priority = Coprocessor.PRIORITY_SYSTEM;
     List<E> configured = new ArrayList<E>();
-    while (st.hasMoreTokens()) {
-      String className = st.nextToken();
+    for (String className : defaultCPClasses) {
+      className = className.trim();
       if (findCoprocessor(className) != null) {
         continue;
       }

From b7529e5c7510298f7a085ac81a2977a4a0ec16ca Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 28 Apr 2012 04:03:44 +0000
Subject: [PATCH 0173/1540] HBASE-5885  Invalid HFile block magic on Local file
 System

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331676 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java  | 7 ++++++-
 .../org/apache/hadoop/hbase/io/hfile/TestChecksum.java     | 2 --
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
index d6a47053d3c8..7e3a68cc03f1 100644
--- a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Progressable;
@@ -66,7 +67,11 @@ public HFileSystem(Configuration conf, boolean useHBaseChecksum)
     // filesystem object that has cksum verification turned off.
     // We will avoid verifying checksums in the fs client, instead do it
     // inside of hbase.
-    if (useHBaseChecksum) {
+    // If this is the local file system hadoop has a bug where seeks
+    // do not go to the correct location if setVerifyChecksum(false) is called.
+    // This manifests itself in that incorrect data is read and HFileBlocks won't be able to read
+    // their header magic numbers. See HBASE-5885
+    if (useHBaseChecksum && !(fs instanceof LocalFileSystem)) {
       this.noChecksumFs = newInstanceFileSystem(conf);
       this.noChecksumFs.setVerifyChecksum(false);
     } else {
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
index c037d2255d84..4179725350e1 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
@@ -110,7 +110,6 @@ public void testChecksumCorruption() throws IOException {
 
         // Use hbase checksums. 
         assertEquals(true, hfs.useHBaseChecksum());
-        assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
 
         // Do a read that purposely introduces checksum verification failures.
         FSDataInputStream is = fs.open(path);
@@ -217,7 +216,6 @@ public void testChecksumChunks() throws IOException {
 
         // Verify hbase checksums. 
         assertEquals(true, hfs.useHBaseChecksum());
-        assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
 
         // Read data back from file.
         FSDataInputStream is = fs.open(path);

From b8554c84d41065e0adfda2d801708044654b9e36 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 28 Apr 2012 04:17:39 +0000
Subject: [PATCH 0174/1540] HBASE-5611 Replayed edits from regions that failed
 to open during recovery aren't removed from the global MemStore size
 (Jieshan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331683 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 30 ++++++-----
 .../regionserver/RegionServerAccounting.java  | 52 ++++++++++++++++++-
 .../handler/OpenRegionHandler.java            | 14 ++++-
 3 files changed, 81 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4bc29a20f244..59786e60deab 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -299,6 +299,7 @@ boolean isFlushRequested() {
   final long timestampSlop;
   private volatile long lastFlushTime;
   final RegionServerServices rsServices;
+  private RegionServerAccounting rsAccounting;
   private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
   private long blockingMemStoreSize;
   final long threadWakeFrequency;
@@ -403,9 +404,10 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
         "hbase.hregion.keyvalue.timestamp.slop.millisecs",
         HConstants.LATEST_TIMESTAMP);
 
-    // don't initialize coprocessors if not running within a regionserver
-    // TODO: revisit if coprocessors should load in other cases
     if (rsServices != null) {
+      this.rsAccounting = this.rsServices.getRegionServerAccounting();
+      // don't initialize coprocessors if not running within a regionserver
+      // TODO: revisit if coprocessors should load in other cases
       this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
     }
     if (LOG.isDebugEnabled()) {
@@ -662,14 +664,9 @@ public AtomicLong getMemstoreSize() {
    * @return the size of memstore in this region
    */
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
-    if (this.rsServices != null) {
-      RegionServerAccounting rsAccounting =
-        this.rsServices.getRegionServerAccounting();
-
-      if (rsAccounting != null) {
-        rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-      }
-    }
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
+    }  
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -2653,6 +2650,11 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
           throw e;
         }
       }
+      // The edits size added into rsAccounting during this replaying will not
+      // be required any more. So just clear it.
+      if (this.rsAccounting != null) {
+        this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+      }
     }
     if (seqid > minSeqId) {
       // Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2834,7 +2836,11 @@ private long replayRecoveredEdits(final Path edits,
    * @return True if we should flush.
    */
   protected boolean restoreEdit(final Store s, final KeyValue kv) {
-    return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
+    long kvSize = s.add(kv);
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+    }
+    return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
   }
 
   /*
@@ -4611,7 +4617,7 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      31 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
       (6 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index 05c842ecb4e2..3fb7c0d28e4a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -19,8 +19,12 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.hadoop.hbase.util.Bytes;
+
 /**
  * RegionServerAccounting keeps record of some basic real time information about
  * the Region Server. Currently, it only keeps record the global memstore size. 
@@ -29,6 +33,11 @@ public class RegionServerAccounting {
 
   private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
   
+  // Store the edits size during replaying HLog. Use this to roll back the  
+  // global memstore size once a region opening failed.
+  private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion = 
+    new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
+  
   /**
    * @return the global Memstore size in the RegionServer
    */
@@ -44,5 +53,46 @@ public long getGlobalMemstoreSize() {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
   }
- 
+
+  /***
+   * Add memStoreSize to replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+   * @return the replay edits size for region hri.
+   */
+  public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+    AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+    if (replayEdistsSize == null) {
+      replayEdistsSize = new AtomicLong(0);
+      replayEditsPerRegion.put(regionName, replayEdistsSize);
+    }
+    return replayEdistsSize.addAndGet(memStoreSize);
+  }
+
+  /**
+   * Roll back the global MemStore size for a specified region when this region
+   * can't be opened.
+   * 
+   * @param regionName the region which could not open.
+   * @return the global Memstore size in the RegionServer
+   */
+  public long rollbackRegionReplayEditsSize(byte[] regionName) {
+    AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+    long editsSizeLong = 0L;
+    if (replayEditsSize != null) {
+      editsSizeLong = -replayEditsSize.get();
+      clearRegionReplayEditsSize(regionName);
+    }
+    return addAndGetGlobalMemstoreSize(editsSizeLong);
+  }
+
+  /**
+   * Clear a region from replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   */
+  public void clearRegionReplayEditsSize(byte[] regionName) {
+    replayEditsPerRegion.remove(regionName);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 8ba723d30ba1..0a95e2a2447e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -343,8 +344,17 @@ public boolean progress() {
       // We failed open. Our caller will see the 'null' return value
       // and transition the node back to FAILED_OPEN. If that fails,
       // we rely on the Timeout Monitor in the master to reassign.
-      LOG.error("Failed open of region=" +
-        this.regionInfo.getRegionNameAsString(), t);
+      LOG.error(
+          "Failed open of region=" + this.regionInfo.getRegionNameAsString()
+              + ", starting to roll back the global memstore size.", t);
+      // Decrease the global memstore size.
+      if (this.rsServices != null) {
+        RegionServerAccounting rsAccounting =
+          this.rsServices.getRegionServerAccounting();
+        if (rsAccounting != null) {
+          rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
+        }
+      }
     }
     return region;
   }

From 8c8922da9682f4fbf96de28c53e9bf092993fbcd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 28 Apr 2012 14:05:11 +0000
Subject: [PATCH 0175/1540] HBASE-5611 Addendum fixes test failure in
 TestHeapSize#testSizes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331769 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 31 +++++++++++++------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 59786e60deab..de49ddae5dec 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -299,7 +299,6 @@ boolean isFlushRequested() {
   final long timestampSlop;
   private volatile long lastFlushTime;
   final RegionServerServices rsServices;
-  private RegionServerAccounting rsAccounting;
   private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
   private long blockingMemStoreSize;
   final long threadWakeFrequency;
@@ -405,7 +404,6 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
         HConstants.LATEST_TIMESTAMP);
 
     if (rsServices != null) {
-      this.rsAccounting = this.rsServices.getRegionServerAccounting();
       // don't initialize coprocessors if not running within a regionserver
       // TODO: revisit if coprocessors should load in other cases
       this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
@@ -664,9 +662,14 @@ public AtomicLong getMemstoreSize() {
    * @return the size of memstore in this region
    */
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
-    if (this.rsAccounting != null) {
-      rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-    }  
+    if (this.rsServices != null) {
+      RegionServerAccounting rsAccounting =
+        this.rsServices.getRegionServerAccounting();
+
+      if (rsAccounting != null) {
+        rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
+      }
+    }
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -2652,8 +2655,12 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
       }
       // The edits size added into rsAccounting during this replaying will not
       // be required any more. So just clear it.
-      if (this.rsAccounting != null) {
-        this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+      if (this.rsServices != null) {
+        RegionServerAccounting rsAccounting =
+          this.rsServices.getRegionServerAccounting();
+        if (rsAccounting != null) {
+          rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+        }
       }
     }
     if (seqid > minSeqId) {
@@ -2837,8 +2844,12 @@ private long replayRecoveredEdits(final Path edits,
    */
   protected boolean restoreEdit(final Store s, final KeyValue kv) {
     long kvSize = s.add(kv);
-    if (this.rsAccounting != null) {
-      rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+    if (this.rsServices != null) {
+      RegionServerAccounting rsAccounting =
+        this.rsServices.getRegionServerAccounting();
+      if (rsAccounting != null) {
+        rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+      }
     }
     return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
   }
@@ -4617,7 +4628,7 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      31 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
       (6 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 

From 3f5800a3f1ed390b37db2c944cacc9f15a8cec78 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 28 Apr 2012 20:28:16 +0000
Subject: [PATCH 0176/1540] HBASE-5611 Revert due to consistent
 TestChangingEncoding failure in 0.94 branch

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1331826 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 23 ++------
 .../regionserver/RegionServerAccounting.java  | 52 +------------------
 .../handler/OpenRegionHandler.java            | 14 +----
 3 files changed, 6 insertions(+), 83 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index de49ddae5dec..4bc29a20f244 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -403,9 +403,9 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
         "hbase.hregion.keyvalue.timestamp.slop.millisecs",
         HConstants.LATEST_TIMESTAMP);
 
+    // don't initialize coprocessors if not running within a regionserver
+    // TODO: revisit if coprocessors should load in other cases
     if (rsServices != null) {
-      // don't initialize coprocessors if not running within a regionserver
-      // TODO: revisit if coprocessors should load in other cases
       this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
     }
     if (LOG.isDebugEnabled()) {
@@ -2653,15 +2653,6 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
           throw e;
         }
       }
-      // The edits size added into rsAccounting during this replaying will not
-      // be required any more. So just clear it.
-      if (this.rsServices != null) {
-        RegionServerAccounting rsAccounting =
-          this.rsServices.getRegionServerAccounting();
-        if (rsAccounting != null) {
-          rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
-        }
-      }
     }
     if (seqid > minSeqId) {
       // Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2843,15 +2834,7 @@ private long replayRecoveredEdits(final Path edits,
    * @return True if we should flush.
    */
   protected boolean restoreEdit(final Store s, final KeyValue kv) {
-    long kvSize = s.add(kv);
-    if (this.rsServices != null) {
-      RegionServerAccounting rsAccounting =
-        this.rsServices.getRegionServerAccounting();
-      if (rsAccounting != null) {
-        rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
-      }
-    }
-    return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
+    return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
   }
 
   /*
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index 3fb7c0d28e4a..05c842ecb4e2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -19,12 +19,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.hadoop.hbase.util.Bytes;
-
 /**
  * RegionServerAccounting keeps record of some basic real time information about
  * the Region Server. Currently, it only keeps record the global memstore size. 
@@ -33,11 +29,6 @@ public class RegionServerAccounting {
 
   private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
   
-  // Store the edits size during replaying HLog. Use this to roll back the  
-  // global memstore size once a region opening failed.
-  private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion = 
-    new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
-  
   /**
    * @return the global Memstore size in the RegionServer
    */
@@ -53,46 +44,5 @@ public long getGlobalMemstoreSize() {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
   }
-
-  /***
-   * Add memStoreSize to replayEditsPerRegion.
-   * 
-   * @param regionName region name.
-   * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
-   * @return the replay edits size for region hri.
-   */
-  public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
-    AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
-    if (replayEdistsSize == null) {
-      replayEdistsSize = new AtomicLong(0);
-      replayEditsPerRegion.put(regionName, replayEdistsSize);
-    }
-    return replayEdistsSize.addAndGet(memStoreSize);
-  }
-
-  /**
-   * Roll back the global MemStore size for a specified region when this region
-   * can't be opened.
-   * 
-   * @param regionName the region which could not open.
-   * @return the global Memstore size in the RegionServer
-   */
-  public long rollbackRegionReplayEditsSize(byte[] regionName) {
-    AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
-    long editsSizeLong = 0L;
-    if (replayEditsSize != null) {
-      editsSizeLong = -replayEditsSize.get();
-      clearRegionReplayEditsSize(regionName);
-    }
-    return addAndGetGlobalMemstoreSize(editsSizeLong);
-  }
-
-  /**
-   * Clear a region from replayEditsPerRegion.
-   * 
-   * @param regionName region name.
-   */
-  public void clearRegionReplayEditsSize(byte[] regionName) {
-    replayEditsPerRegion.remove(regionName);
-  }
+ 
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 0a95e2a2447e..8ba723d30ba1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -344,17 +343,8 @@ public boolean progress() {
       // We failed open. Our caller will see the 'null' return value
       // and transition the node back to FAILED_OPEN. If that fails,
       // we rely on the Timeout Monitor in the master to reassign.
-      LOG.error(
-          "Failed open of region=" + this.regionInfo.getRegionNameAsString()
-              + ", starting to roll back the global memstore size.", t);
-      // Decrease the global memstore size.
-      if (this.rsServices != null) {
-        RegionServerAccounting rsAccounting =
-          this.rsServices.getRegionServerAccounting();
-        if (rsAccounting != null) {
-          rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
-        }
-      }
+      LOG.error("Failed open of region=" +
+        this.regionInfo.getRegionNameAsString(), t);
     }
     return region;
   }

From c99f6dc27b4a382d676528ac16dce61d21599efd Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 30 Apr 2012 06:33:55 +0000
Subject: [PATCH 0177/1540] HBASE-5712 Parallelize load of .regioninfo files in
 diagnostic/repair portion of hbck

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332071 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 133 ++++++++++++++----
 .../hadoop/hbase/util/TestHBaseFsck.java      |   6 +-
 2 files changed, 110 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 9a8d2bb05419..83aa3168c336 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -28,9 +28,11 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -196,12 +198,12 @@ public class HBaseFsck {
    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
    * to prevent dupes.
    */
-  private TreeMap<String, TableInfo> tablesInfo = new TreeMap<String, TableInfo>();
+  private SortedMap<String, TableInfo> tablesInfo = new ConcurrentSkipListMap<String,TableInfo>();
 
   /**
    * When initially looking at HDFS, we attempt to find any orphaned data.
    */
-  private List<HbckInfo> orphanHdfsDirs = new ArrayList<HbckInfo>();
+  private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
 
   /**
    * Constructor
@@ -409,6 +411,11 @@ private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
     Path p = hi.getHdfsRegionDir();
     FileSystem fs = p.getFileSystem(conf);
     FileStatus[] dirs = fs.listStatus(p);
+    if (dirs == null) {
+      LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
+          p + ". This dir could probably be deleted.");
+      return ;
+    }
 
     String tableName = Bytes.toString(hi.getTableName());
     TableInfo tableInfo = tablesInfo.get(tableName);
@@ -580,6 +587,12 @@ private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
       return;
     }
+
+    if (hbi.hdfsEntry.hri != null) {
+      // already loaded data
+      return;
+    }
+
     Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
     FileSystem fs = regioninfo.getFileSystem(conf);
 
@@ -607,27 +620,37 @@ public RegionRepairException(String s, IOException ioe) {
   /**
    * Populate hbi's from regionInfos loaded from file system.
    */
-  private TreeMap<String, TableInfo> loadHdfsRegionInfos() throws IOException {
+  private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, InterruptedException {
     tablesInfo.clear(); // regenerating the data
     // generate region split structure
-    for (HbckInfo hbi : regionInfoMap.values()) {
+    Collection<HbckInfo> hbckInfos = regionInfoMap.values();
 
-      // only load entries that haven't been loaded yet.
-      if (hbi.getHdfsHRI() == null) {
-        try {
-          loadHdfsRegioninfo(hbi);
-        } catch (IOException ioe) {
-          String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
-            + Bytes.toString(hbi.getTableName()) + " in hdfs dir "
-            + hbi.getHdfsRegionDir()
-            + "!  It may be an invalid format or version file.  Treating as "
-            + "an orphaned regiondir.";
-          errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
-          debugLsr(hbi.getHdfsRegionDir());
-          orphanHdfsDirs.add(hbi);
-          continue;
+    // Parallelized read of .regioninfo files.
+    WorkItemHdfsRegionInfo[] hbis = new WorkItemHdfsRegionInfo[hbckInfos.size()];
+    int num = 0;
+    for (HbckInfo hbi : hbckInfos) {
+      hbis[num] = new WorkItemHdfsRegionInfo(hbi, this, errors);
+      executor.execute(hbis[num]);
+      num++;
+    }
+
+    for (int i=0; i < num; i++) {
+      WorkItemHdfsRegionInfo hbi = hbis[i];
+      synchronized(hbi) {
+        while (!hbi.isDone()) {
+          hbi.wait();
         }
       }
+    }
+
+    // serialized table info gathering.
+    for (HbckInfo hbi: hbckInfos) {
+
+      if (hbi.getHdfsHRI() == null) {
+        // was an orphan
+        continue;
+      }
+
 
       // get table name from hdfs, populate various HBaseFsck tables.
       String tableName = Bytes.toString(hbi.getTableName());
@@ -642,10 +665,16 @@ private TreeMap<String, TableInfo> loadHdfsRegionInfos() throws IOException {
         // only executed once per table.
         modTInfo = new TableInfo(tableName);
         Path hbaseRoot = new Path(conf.get(HConstants.HBASE_DIR));
-        HTableDescriptor htd =
-          FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
+        try {
+          HTableDescriptor htd =
+              FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
               hbaseRoot, tableName);
-        modTInfo.htds.add(htd);
+          modTInfo.htds.add(htd);
+        } catch (IOException ioe) {
+          LOG.error("Unable to read .tableinfo from " + hbaseRoot, ioe);
+          throw ioe;
+        }
+
       }
       modTInfo.addRegionInfo(hbi);
       tablesInfo.put(tableName, modTInfo);
@@ -686,7 +715,7 @@ private HRegion createNewRootAndMeta() throws IOException {
    * 
    * @return An array list of puts to do in bulk, null if tables have problems
    */
-  private ArrayList<Put> generatePuts(TreeMap<String, TableInfo> tablesInfo) throws IOException {
+  private ArrayList<Put> generatePuts(SortedMap<String, TableInfo> tablesInfo) throws IOException {
     ArrayList<Put> puts = new ArrayList<Put>();
     boolean hasProblems = false;
     for (Entry<String, TableInfo> e : tablesInfo.entrySet()) {
@@ -726,7 +755,7 @@ private ArrayList<Put> generatePuts(TreeMap<String, TableInfo> tablesInfo) throw
   /**
    * Suggest fixes for each table
    */
-  private void suggestFixes(TreeMap<String, TableInfo> tablesInfo) throws IOException {
+  private void suggestFixes(SortedMap<String, TableInfo> tablesInfo) throws IOException {
     for (TableInfo tInfo : tablesInfo.values()) {
       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
       tInfo.checkRegionChain(handler);
@@ -797,7 +826,7 @@ public boolean rebuildMeta(boolean fix) throws IOException,
     return true;
   }
 
-  private TreeMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
+  private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
       boolean fixOverlaps) throws IOException {
     LOG.info("Checking HBase region split map from HDFS data...");
     for (TableInfo tInfo : tablesInfo.values()) {
@@ -1418,7 +1447,7 @@ else if (!inMeta && !inHdfs && !isDeployed) {
    * repeated or overlapping ones.
    * @throws IOException
    */
-  TreeMap<String, TableInfo> checkIntegrity() throws IOException {
+  SortedMap<String, TableInfo> checkIntegrity() throws IOException {
     tablesInfo = new TreeMap<String,TableInfo> ();
     List<HbckInfo> noHDFSRegionInfos = new ArrayList<HbckInfo>();
     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
@@ -2438,7 +2467,7 @@ public int compare(HbckInfo l, HbckInfo r) {
   /**
    * Prints summary of all tables found on the system.
    */
-  private void printTableSummary(TreeMap<String, TableInfo> tablesInfo) {
+  private void printTableSummary(SortedMap<String, TableInfo> tablesInfo) {
     System.out.println("Summary:");
     for (TableInfo tInfo : tablesInfo.values()) {
       if (errors.tableHasErrors(tInfo)) {
@@ -2737,6 +2766,58 @@ public synchronized void run() {
     }
   }
 
+  /**
+   * Contact hdfs and get all information about specified table directory into
+   * regioninfo list.
+   */
+  static class WorkItemHdfsRegionInfo implements Runnable {
+    private HbckInfo hbi;
+    private HBaseFsck hbck;
+    private ErrorReporter errors;
+    private boolean done;
+
+    WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
+      this.hbi = hbi;
+      this.hbck = hbck;
+      this.errors = errors;
+      this.done = false;
+    }
+
+    synchronized boolean isDone() {
+      return done;
+    }
+
+    @Override
+    public synchronized void run() {
+      try {
+        // only load entries that haven't been loaded yet.
+        if (hbi.getHdfsHRI() == null) {
+          try {
+            hbck.loadHdfsRegioninfo(hbi);
+          } catch (IOException ioe) {
+            String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
+                + Bytes.toString(hbi.getTableName()) + " in hdfs dir "
+                + hbi.getHdfsRegionDir()
+                + "!  It may be an invalid format or version file.  Treating as "
+                + "an orphaned regiondir.";
+            errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
+            try {
+              hbck.debugLsr(hbi.getHdfsRegionDir());
+            } catch (IOException ioe2) {
+              LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
+              return; // TODO convert this in to a future
+            }
+            hbck.orphanHdfsDirs.add(hbi);
+            return;
+          }
+        }
+      } finally {
+        done = true;
+        notifyAll();
+      }
+    }
+  };
+
   /**
    * Display the full report from fsck. This displays all live and dead region
    * servers, and all known regions.
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 4ea8ee334ef2..e140d6503dad 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -878,10 +878,10 @@ public void testNoVersionFile() throws Exception {
    * the region is not deployed when the table is disabled.
    */
   @Test
-  public void testRegionShouldNotDeployed() throws Exception {
-    String table = "tableRegionShouldNotDeployed";
+  public void testRegionShouldNotBeDeployed() throws Exception {
+    String table = "tableRegionShouldNotBeDeployed";
     try {
-      LOG.info("Starting testRegionShouldNotDeployed.");
+      LOG.info("Starting testRegionShouldNotBeDeployed.");
       MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
       assertTrue(cluster.waitForActiveAndReadyMaster());
 

From 575aa5e253a4f52308609fbf9415121a192f3dc8 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 30 Apr 2012 18:05:38 +0000
Subject: [PATCH 0178/1540] HBASE-5906 TestChangingEncoding failing
 sporadically in 0.94 build

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332319 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/encoding/TestChangingEncoding.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
index d3825025ae2c..d3418a03d0ab 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
@@ -67,7 +67,7 @@ public class TestChangingEncoding {
       new HBaseTestingUtility();
   private static final Configuration conf = TEST_UTIL.getConfiguration();
 
-  private static final int TIMEOUT_MS = 120000;
+  private static final int TIMEOUT_MS = 240000;
 
   private HBaseAdmin admin;
   private HColumnDescriptor hcd;

From 1ea241364f6f381180cc192ca1abf2206a6e28a8 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 30 Apr 2012 19:15:46 +0000
Subject: [PATCH 0179/1540] HBASE-5611 Replayed edits from regions that failed
 to open during recovery aren't removed from the global MemStore size - v2
 (Jieshan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332344 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 32 +++++++-----
 .../regionserver/RegionServerAccounting.java  | 52 ++++++++++++++++++-
 .../handler/OpenRegionHandler.java            | 14 ++++-
 3 files changed, 82 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4bc29a20f244..c1c6a11ed195 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -299,6 +299,7 @@ boolean isFlushRequested() {
   final long timestampSlop;
   private volatile long lastFlushTime;
   final RegionServerServices rsServices;
+  private RegionServerAccounting rsAccounting;
   private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
   private long blockingMemStoreSize;
   final long threadWakeFrequency;
@@ -403,9 +404,10 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
         "hbase.hregion.keyvalue.timestamp.slop.millisecs",
         HConstants.LATEST_TIMESTAMP);
 
-    // don't initialize coprocessors if not running within a regionserver
-    // TODO: revisit if coprocessors should load in other cases
     if (rsServices != null) {
+      this.rsAccounting = this.rsServices.getRegionServerAccounting();
+      // don't initialize coprocessors if not running within a regionserver
+      // TODO: revisit if coprocessors should load in other cases
       this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
     }
     if (LOG.isDebugEnabled()) {
@@ -662,14 +664,9 @@ public AtomicLong getMemstoreSize() {
    * @return the size of memstore in this region
    */
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
-    if (this.rsServices != null) {
-      RegionServerAccounting rsAccounting =
-        this.rsServices.getRegionServerAccounting();
-
-      if (rsAccounting != null) {
-        rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-      }
-    }
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
+    }  
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -2653,6 +2650,11 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
           throw e;
         }
       }
+      // The edits size added into rsAccounting during this replaying will not
+      // be required any more. So just clear it.
+      if (this.rsAccounting != null) {
+        this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+      }
     }
     if (seqid > minSeqId) {
       // Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2834,7 +2836,11 @@ private long replayRecoveredEdits(final Path edits,
    * @return True if we should flush.
    */
   protected boolean restoreEdit(final Store s, final KeyValue kv) {
-    return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
+    long kvSize = s.add(kv);
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+    }
+    return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
   }
 
   /*
@@ -4611,8 +4617,8 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
-      (6 * Bytes.SIZEOF_LONG) +
+      32 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      (5 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index 05c842ecb4e2..3fb7c0d28e4a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -19,8 +19,12 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.hadoop.hbase.util.Bytes;
+
 /**
  * RegionServerAccounting keeps record of some basic real time information about
  * the Region Server. Currently, it only keeps record the global memstore size. 
@@ -29,6 +33,11 @@ public class RegionServerAccounting {
 
   private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
   
+  // Store the edits size during replaying HLog. Use this to roll back the  
+  // global memstore size once a region opening failed.
+  private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion = 
+    new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
+  
   /**
    * @return the global Memstore size in the RegionServer
    */
@@ -44,5 +53,46 @@ public long getGlobalMemstoreSize() {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
   }
- 
+
+  /***
+   * Add memStoreSize to replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+   * @return the replay edits size for region hri.
+   */
+  public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+    AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+    if (replayEdistsSize == null) {
+      replayEdistsSize = new AtomicLong(0);
+      replayEditsPerRegion.put(regionName, replayEdistsSize);
+    }
+    return replayEdistsSize.addAndGet(memStoreSize);
+  }
+
+  /**
+   * Roll back the global MemStore size for a specified region when this region
+   * can't be opened.
+   * 
+   * @param regionName the region which could not open.
+   * @return the global Memstore size in the RegionServer
+   */
+  public long rollbackRegionReplayEditsSize(byte[] regionName) {
+    AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+    long editsSizeLong = 0L;
+    if (replayEditsSize != null) {
+      editsSizeLong = -replayEditsSize.get();
+      clearRegionReplayEditsSize(regionName);
+    }
+    return addAndGetGlobalMemstoreSize(editsSizeLong);
+  }
+
+  /**
+   * Clear a region from replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   */
+  public void clearRegionReplayEditsSize(byte[] regionName) {
+    replayEditsPerRegion.remove(regionName);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 8ba723d30ba1..0a95e2a2447e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -343,8 +344,17 @@ public boolean progress() {
       // We failed open. Our caller will see the 'null' return value
       // and transition the node back to FAILED_OPEN. If that fails,
       // we rely on the Timeout Monitor in the master to reassign.
-      LOG.error("Failed open of region=" +
-        this.regionInfo.getRegionNameAsString(), t);
+      LOG.error(
+          "Failed open of region=" + this.regionInfo.getRegionNameAsString()
+              + ", starting to roll back the global memstore size.", t);
+      // Decrease the global memstore size.
+      if (this.rsServices != null) {
+        RegionServerAccounting rsAccounting =
+          this.rsServices.getRegionServerAccounting();
+        if (rsAccounting != null) {
+          rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
+        }
+      }
     }
     return region;
   }

From 89a9461b6fd513f57c92ee5bba0295d17e9eef19 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 30 Apr 2012 22:07:46 +0000
Subject: [PATCH 0180/1540] HBASE-5884 MapReduce package info has broken link
 to bulk-loads

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332441 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/mapreduce/package-info.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
index affb94070bad..62d429665251 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
@@ -143,7 +143,7 @@ whichever is smaller (If your job only has two maps, up mapred.map.tasks
 and write your content directly to the filesystem properly formatted as
 HBase data files (HFiles).  Your import will run faster, perhaps an order of
 magnitude faster if not more.  For more on how this mechanism works, see
-<a href="http://hbase.apache.org/docs/current/bulk-loads.html">Bulk Loads</code>
+<a href="http://hbase.apache.org/bulk-loads.html">Bulk Loads</code>
 documentation.
 </p>
 

From fb39f861ba12d60c532e1b95104313136b8c2e95 Mon Sep 17 00:00:00 2001
From: Todd Lipcon <todd@apache.org>
Date: Tue, 1 May 2012 01:38:30 +0000
Subject: [PATCH 0181/1540] HBASE-5908.
 TestHLogSplit.testTralingGarbageCorruptionFileSkipErrorsPasses should not use
 append to corrupt the HLog. Contributed by Gregory Chanan.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332494 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index d838a68d287d..e3cf968aaa03 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -119,8 +119,6 @@ static enum Corruptions {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().
-            setBoolean("dfs.support.append", true);
     TEST_UTIL.getConfiguration().
             setStrings("hbase.rootdir", hbaseDir.toString());
     TEST_UTIL.getConfiguration().
@@ -1227,7 +1225,9 @@ private void corruptHLog(Path path, Corruptions corruption, boolean close,
 
     switch (corruption) {
       case APPEND_GARBAGE:
-        out = fs.append(path);
+        fs.delete(path, false);
+        out = fs.create(path);
+        out.write(corrupted_bytes);
         out.write("-----".getBytes());
         closeOrFlush(close, out);
         break;

From 0f763a584377c3059a07025b8adb32de0fbd08e5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 1 May 2012 20:47:16 +0000
Subject: [PATCH 0182/1540] HBASE-5897 prePut coprocessor hook causing
 substantial CPU usage

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332810 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 52 ++++++++++++++-----
 1 file changed, 38 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c1c6a11ed195..89133e986834 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1861,10 +1861,12 @@ private static class BatchOperationInProgress<T> {
     T[] operations;
     int nextIndexToProcess = 0;
     OperationStatus[] retCodeDetails;
+    WALEdit[] walEditsFromCoprocessors;
 
     public BatchOperationInProgress(T[] operations) {
       this.operations = operations;
       this.retCodeDetails = new OperationStatus[operations.length];
+      this.walEditsFromCoprocessors = new WALEdit[operations.length];
       Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);
     }
 
@@ -1901,14 +1903,21 @@ public OperationStatus[] put(
     BatchOperationInProgress<Pair<Put, Integer>> batchOp =
       new BatchOperationInProgress<Pair<Put,Integer>>(putsAndLocks);
 
+    boolean initialized = false;
+
     while (!batchOp.isDone()) {
       checkReadOnly();
       checkResources();
 
       long newSize;
       startRegionOperation();
-      this.writeRequestsCount.increment();
+
       try {
+        if (!initialized) {
+          this.writeRequestsCount.increment();
+          doPrePutHook(batchOp);
+          initialized = true;
+        }
         long addedSize = doMiniBatchPut(batchOp);
         newSize = this.addAndGetGlobalMemstoreSize(addedSize);
       } finally {
@@ -1921,30 +1930,38 @@ public OperationStatus[] put(
     return batchOp.retCodeDetails;
   }
 
-  @SuppressWarnings("unchecked")
-  private long doMiniBatchPut(
-      BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException {
-    final String tableName = getTableDesc().getNameAsString();
-
-    // The set of columnFamilies first seen.
-    Set<byte[]> cfSet = null;
-    // variable to note if all Put items are for the same CF -- metrics related
-    boolean cfSetConsistent = true;
-    long startTimeMs = EnvironmentEdgeManager.currentTimeMillis();
-
-    WALEdit walEdit = new WALEdit();
+  private void doPrePutHook(BatchOperationInProgress<Pair<Put, Integer>> batchOp)
+      throws IOException {
     /* Run coprocessor pre hook outside of locks to avoid deadlock */
+    WALEdit walEdit = new WALEdit();
     if (coprocessorHost != null) {
       for (int i = 0; i < batchOp.operations.length; i++) {
         Pair<Put, Integer> nextPair = batchOp.operations[i];
         Put put = nextPair.getFirst();
         if (coprocessorHost.prePut(put, walEdit, put.getWriteToWAL())) {
           // pre hook says skip this Put
-          // mark as success and skip below
+          // mark as success and skip in doMiniBatchPut
           batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
         }
+        if (!walEdit.isEmpty()) {
+          batchOp.walEditsFromCoprocessors[i] = walEdit;
+          walEdit = new WALEdit();
+        }
       }
     }
+  }
+
+  @SuppressWarnings("unchecked")
+  private long doMiniBatchPut(
+      BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException {
+
+    // The set of columnFamilies first seen.
+    Set<byte[]> cfSet = null;
+    // variable to note if all Put items are for the same CF -- metrics related
+    boolean cfSetConsistent = true;
+    long startTimeMs = EnvironmentEdgeManager.currentTimeMillis();
+
+    WALEdit walEdit = new WALEdit();
 
     MultiVersionConsistencyControl.WriteEntry w = null;
     long txid = 0;
@@ -2082,6 +2099,13 @@ private long doMiniBatchPut(
 
         Put p = batchOp.operations[i].getFirst();
         if (!p.getWriteToWAL()) continue;
+        // Add WAL edits by CP
+        WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
+        if (fromCP != null) {
+          for (KeyValue kv : fromCP.getKeyValues()) {
+            walEdit.add(kv);
+          }
+        }
         addFamilyMapToWALEdit(familyMaps[i], walEdit);
       }
 

From ca1bab028a77aca67ad754e694ebc85f8db27f27 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 1 May 2012 21:00:53 +0000
Subject: [PATCH 0183/1540] CHANGES.txt for 0.94.0RC3, take 2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1332822 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 66e5d8ec19dc..a42863a6d2f8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.0 - 4/26/2012
+Release 0.94.0 - 5/1/2012
 Sub-task
 
     [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently
@@ -181,6 +181,7 @@ Bug
     [HBASE-5597] - Findbugs check in test-patch.sh always fails
     [HBASE-5603] - rolling-restart.sh script hangs when attempting to detect expiration of /hbase/master znode.
     [HBASE-5606] - SplitLogManger async delete node hangs log splitting when ZK connection is lost
+    [HBASE-5611] - Replayed edits from regions that failed to open during recovery aren't removed from the global MemStore size
     [HBASE-5613] - ThriftServer getTableRegions does not return serverName and port
     [HBASE-5623] - Race condition when rolling the HLog and hlogFlush
     [HBASE-5624] - Aborting regionserver when splitting region, may cause daughter region not assigned by ServerShutdownHandler.
@@ -217,9 +218,15 @@ Bug
     [HBASE-5857] - RIT map in RS not getting cleared while region opening
     [HBASE-5861] - Hadoop 23 compilation broken due to tests introduced in HBASE-5604
     [HBASE-5864] - Error while reading from hfile in 0.94
+    [HBASE-5865] - test-util.sh broken with unittest updates
     [HBASE-5866] - Canary in tool package but says its in tools.
     [HBASE-5871] - Usability regression, we don't parse compression algos anymore
     [HBASE-5873] - TimeOut Monitor thread should be started after atleast one region server registers.
+    [HBASE-5884] - MapReduce package info has broken link to bulk-loads
+    [HBASE-5885] - Invalid HFile block magic on Local file System
+    [HBASE-5893] - Allow spaces in coprocessor conf (aka trim() className)
+    [HBASE-5897] - prePut coprocessor hook causing substantial CPU usage
+    [HBASE-5908] - TestHLogSplit.testTralingGarbageCorruptionFileSkipErrorsPasses should not use append to corrupt the HLog
 
 Improvement
 
@@ -357,6 +364,7 @@ Improvement
     [HBASE-5671] - hbase.metrics.showTableName should be true by default
     [HBASE-5682] - Allow HConnectionImplementation to recover from ZK connection loss (for 0.94 only)
     [HBASE-5706] - "Dropping fs latency stats since buffer is full" spam
+    [HBASE-5712] - Parallelize load of .regioninfo files in diagnostic/repair portion of hbck.
     [HBASE-5734] - Change hbck sideline root
     [HBASE-5735] - Clearer warning message when connecting a non-secure HBase client to a secure HBase server
     [HBASE-5737] - Minor Improvements related to balancer.
@@ -365,6 +373,7 @@ Improvement
     [HBASE-5775] - ZKUtil doesn't handle deleteRecurisively cleanly
     [HBASE-5823] - Hbck should be able to print help
     [HBASE-5862] - After Region Close remove the Operation Metrics.
+    [HBASE-5863] - Improve the graceful_stop.sh CLI help (especially about reloads)
 
 New Feature
 
@@ -411,6 +420,7 @@ Task
     [HBASE-5715] - Revert 'Instant schema alter' for now, HBASE-4213
     [HBASE-5721] - Update bundled hadoop to be 1.0.2 (it was just released)
     [HBASE-5758] - Forward port "HBASE-4109 Hostname returned via reverse dns lookup contains trailing period if configured interface is not 'default'"
+    [HBASE-5836] - Backport per region metrics from HBASE-3614 to 0.94.1
 
 Test
 

From c2de5b7eced125c24a9498b9a8fce1b79ebbcde9 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 2 May 2012 19:08:16 +0000
Subject: [PATCH 0184/1540] HBASE-2214 Do HBASE-1996 -- setting size to return
 in scan rather than count of rows -- properly (Ferdy Galema)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333157 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ClientScanner.java    |  6 +++-
 .../org/apache/hadoop/hbase/client/Scan.java  | 34 +++++++++++++++++--
 .../hadoop/hbase/regionserver/HRegion.java    |  7 ++++
 .../hbase/regionserver/HRegionServer.java     | 10 ++++--
 .../hbase/regionserver/RegionScanner.java     |  5 +++
 .../coprocessor/TestCoprocessorInterface.java |  5 +++
 6 files changed, 60 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 0fbabe5ded5c..166e33c39097 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -95,9 +95,13 @@ public ClientScanner(final Configuration conf, final Scan scan,
       this.tableName = tableName;
       this.lastNext = System.currentTimeMillis();
       this.connection = connection;
-      this.maxScannerResultSize = conf.getLong(
+      if (scan.getMaxResultSize() > 0) {
+        this.maxScannerResultSize = scan.getMaxResultSize();
+      } else {
+        this.maxScannerResultSize = conf.getLong(
           HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
           HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
+      }
       this.scannerTimeout = (int) conf.getLong(
           HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
           HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 7a53d44e89ad..de114f5ebedc 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -51,8 +51,11 @@
  * To scan everything for each row, instantiate a Scan object.
  * <p>
  * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
- * If caching is NOT set, we will use the caching value of the hosting
- * {@link HTable}.  See {@link HTable#setScannerCaching(int)}.
+ * If caching is NOT set, we will use the caching value of the hosting {@link HTable}.  See
+ * {@link HTable#setScannerCaching(int)}. In addition to row caching, it is possible to specify a
+ * maximum result size, using {@link #setMaxResultSize(long)}. When both are used,
+ * single server requests are limited by either number of rows or maximum result size, whichever
+ * limit comes first.
  * <p>
  * To further define the scope of what to get when scanning, perform additional
  * methods as outlined below.
@@ -84,7 +87,7 @@ public class Scan extends OperationWithAttributes implements Writable {
   private static final String RAW_ATTR = "_raw_";
   private static final String ISOLATION_LEVEL = "_isolationlevel_";
 
-  private static final byte SCAN_VERSION = (byte)2;
+  private static final byte SCAN_VERSION = (byte)3;
   private byte [] startRow = HConstants.EMPTY_START_ROW;
   private byte [] stopRow  = HConstants.EMPTY_END_ROW;
   private int maxVersions = 1;
@@ -100,6 +103,7 @@ public class Scan extends OperationWithAttributes implements Writable {
    * -1 means no caching
    */
   private int caching = -1;
+  private long maxResultSize = -1;
   private boolean cacheBlocks = true;
   private Filter filter = null;
   private TimeRange tr = new TimeRange();
@@ -149,6 +153,7 @@ public Scan(Scan scan) throws IOException {
     maxVersions = scan.getMaxVersions();
     batch = scan.getBatch();
     caching = scan.getCaching();
+    maxResultSize = scan.getMaxResultSize();
     cacheBlocks = scan.getCacheBlocks();
     filter = scan.getFilter(); // clone?
     TimeRange ctr = scan.getTimeRange();
@@ -322,6 +327,24 @@ public void setCaching(int caching) {
     this.caching = caching;
   }
 
+  /**
+   * @return the maximum result size in bytes. See {@link #setMaxResultSize(long)}
+   */
+  public long getMaxResultSize() {
+    return maxResultSize;
+  }
+
+  /**
+   * Set the maximum result size. The default is -1; this means that no specific
+   * maximum result size will be set for this scan, and the global configured
+   * value will be used instead. (Defaults to unlimited).
+   *
+   * @param maxResultSize The maximum result size in bytes.
+   */
+  public void setMaxResultSize(long maxResultSize) {
+    this.maxResultSize = maxResultSize;
+  }
+
   /**
    * Apply the specified server-side filter when performing the Scan.
    * @param filter filter to run on the server
@@ -500,6 +523,7 @@ public Map<String, Object> toMap(int maxCols) {
     map.put("maxVersions", this.maxVersions);
     map.put("batch", this.batch);
     map.put("caching", this.caching);
+    map.put("maxResultSize", this.maxResultSize);
     map.put("cacheBlocks", this.cacheBlocks);
     List<Long> timeRange = new ArrayList<Long>();
     timeRange.add(this.tr.getMin());
@@ -582,6 +606,9 @@ public void readFields(final DataInput in)
     if (version > 1) {
       readAttributes(in);
     }
+    if (version > 2) {
+      this.maxResultSize = in.readLong();
+    }
   }
 
   public void write(final DataOutput out)
@@ -615,6 +642,7 @@ public void write(final DataOutput out)
       }
     }
     writeAttributes(out);
+    out.writeLong(maxResultSize);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 89133e986834..a65497cf9972 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3231,6 +3231,7 @@ class RegionScannerImpl implements RegionScanner {
     private int isScan;
     private boolean filterClosed = false;
     private long readPt;
+    private long maxResultSize;
 
     public HRegionInfo getRegionInfo() {
       return regionInfo;
@@ -3238,6 +3239,7 @@ public HRegionInfo getRegionInfo() {
     RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
       //DebugPrint.println("HRegionScanner.<init>");
 
+      this.maxResultSize = scan.getMaxResultSize();
       this.filter = scan.getFilter();
       this.batch = scan.getBatch();
       if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
@@ -3281,6 +3283,11 @@ public HRegionInfo getRegionInfo() {
       this(scan, null);
     }
 
+    @Override
+    public long getMaxResultSize() {
+      return maxResultSize;
+    }
+
     /**
      * Reset both the filter and the old filter.
      */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e78dc63e4a88..838702c1b364 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2385,9 +2385,13 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
               : results.toArray(new Result[0]);
         }
       }
-
-      for (int i = 0; i < nbRows
-          && currentScanResultSize < maxScannerResultSize; i++) {
+      long maxResultSize;
+      if (s.getMaxResultSize() > 0) {
+        maxResultSize = s.getMaxResultSize();
+      } else {
+        maxResultSize = maxScannerResultSize;
+      }
+      for (int i = 0; i < nbRows && currentScanResultSize < maxResultSize; i++) {
         requestCount.incrementAndGet();
         // Collect values to be returned here
         boolean moreRows = s.next(values);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
index da95e901a8c9..d87f70f7ba53 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.Scan;
 
 /**
  * RegionScanner describes iterators over rows in an HRegion.
@@ -49,4 +50,8 @@ public interface RegionScanner extends InternalScanner {
    */
   public boolean reseek(byte[] row) throws IOException;
 
+  /**
+   * @return The preferred max buffersize. See {@link Scan#setMaxResultSize(long)}
+   */
+  public long getMaxResultSize();
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 99b2dd94661e..c30a528aaa12 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -89,6 +89,11 @@ public boolean reseek(byte[] row) throws IOException {
       return false;
     }
 
+    @Override
+    public long getMaxResultSize() {
+      return delegate.getMaxResultSize();
+    }
+
   }
 
   public static class CoprocessorImpl extends BaseRegionObserver {

From b4a1a1df55c7eb114e62d8861d4d0c1dc8b82f87 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 3 May 2012 06:05:17 +0000
Subject: [PATCH 0185/1540] HBASE-5913 Speed up the full scan of META

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333315 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/catalog/MetaReader.java     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index 77a121b33a40..1da3b27724a4 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -696,6 +696,11 @@ static void fullScan(CatalogTracker catalogTracker,
   throws IOException {
     Scan scan = new Scan();
     if (startrow != null) scan.setStartRow(startrow);
+    if (startrow == null && !scanRoot) {
+      int caching = catalogTracker.getConnection().getConfiguration()
+          .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
+      scan.setCaching(caching);
+    }
     scan.addFamily(HConstants.CATALOG_FAMILY);
     HTable metaTable = scanRoot?
       getRootHTable(catalogTracker): getMetaHTable(catalogTracker);

From a8be359f5b214a06dbe2bde3cde095c3eea8d14f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 3 May 2012 16:29:45 +0000
Subject: [PATCH 0186/1540] HBASE-5883 Backup master is going down due to
 connection refused exception (Jieshan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  |  4 +-
 .../org/apache/hadoop/hbase/ipc/HBaseRPC.java | 52 +++++++++++++++----
 2 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 3ba3d0168135..331642af7243 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -583,7 +583,9 @@ private HRegionInterface getCachedConnection(ServerName sn)
       LOG.debug("Unknown host exception connecting to  " + sn);
     } catch (IOException ioe) {
       Throwable cause = ioe.getCause();
-      if (cause != null && cause instanceof EOFException) {
+      if (ioe instanceof ConnectException) {
+        // Catch. Connect refused.
+      } else if (cause != null && cause instanceof EOFException) {
         // Catch. Other end disconnected us.
       } else if (cause != null && cause.getMessage() != null &&
         cause.getMessage().toLowerCase().contains("connection reset")) {
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
index 7eb9e0309f57..1c45c0a2fea1 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -234,21 +234,34 @@ public static VersionedProtocol waitForProxy(Class protocol,
     while (true) {
       try {
         return getProxy(protocol, clientVersion, addr, conf, rpcTimeout);
-      } catch(ConnectException se) {  // namenode has not been started
-        ioe = se;
-        if (maxAttempts >= 0 && ++reconnectAttempts >= maxAttempts) {
-          LOG.info("Server at " + addr + " could not be reached after " +
-            reconnectAttempts + " tries, giving up.");
-          throw new RetriesExhaustedException("Failed setting up proxy " +
-            protocol + " to " + addr.toString() + " after attempts=" +
-            reconnectAttempts, se);
-      }
       } catch(SocketTimeoutException te) {  // namenode is busy
         LOG.info("Problem connecting to server: " + addr);
         ioe = te;
+      } catch (IOException ioex) {
+        // We only handle the ConnectException.
+        ConnectException ce = null;
+        if (ioex instanceof ConnectException) {
+          ce = (ConnectException) ioex;
+          ioe = ce;
+        } else if (ioex.getCause() != null
+            && ioex.getCause() instanceof ConnectException) {
+          ce = (ConnectException) ioex.getCause();
+          ioe = ce;
+        } else if (ioex.getMessage().toLowerCase()
+            .contains("connection refused")) {
+          ce = new ConnectException(ioex.getMessage());
+          ioe = ce;
+        } else {
+          // This is the exception we can't handle.
+          ioe = ioex;
+        }
+        if (ce != null) {
+          handleConnectionException(++reconnectAttempts, maxAttempts, protocol,
+              addr, ce);
+        }
       }
       // check if timed out
-      if (System.currentTimeMillis()-timeout >= startTime) {
+      if (System.currentTimeMillis() - timeout >= startTime) {
         throw ioe;
       }
 
@@ -261,6 +274,25 @@ public static VersionedProtocol waitForProxy(Class protocol,
     }
   }
 
+  /**
+   * @param retries current retried times.
+   * @param maxAttmpts max attempts
+   * @param protocol protocol interface
+   * @param addr address of remote service
+   * @param ce ConnectException
+   * @throws RetriesExhaustedException
+   */
+  private static void handleConnectionException(int retries, int maxAttmpts,
+      Class<?> protocol, InetSocketAddress addr, ConnectException ce)
+      throws RetriesExhaustedException {
+    if (maxAttmpts >= 0 && retries >= maxAttmpts) {
+      LOG.info("Server at " + addr + " could not be reached after "
+          + maxAttmpts + " tries, giving up.");
+      throw new RetriesExhaustedException("Failed setting up proxy " + protocol
+          + " to " + addr.toString() + " after attempts=" + maxAttmpts, ce);
+    }
+  }
+  
   /**
    * Construct a client-side proxy object that implements the named protocol,
    * talking to a server at the named address.

From e211606553580aae935df5eb5ef499324c772a4f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 4 May 2012 00:00:43 +0000
Subject: [PATCH 0187/1540] HBASE-5886 Add new metric for possible data loss
 due to puts without WAL

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333689 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 39 +++++++++++++++++--
 .../hbase/regionserver/HRegionServer.java     |  6 +++
 .../metrics/RegionServerMetrics.java          | 18 +++++++++
 3 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a65497cf9972..32ab1bf39b8a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -214,6 +214,10 @@ public class HRegion implements HeapSize { // , Writable{
 
   final AtomicLong memstoreSize = new AtomicLong(0);
 
+  // Debug possible data loss due to WAL off
+  final AtomicLong numPutsWithoutWAL = new AtomicLong(0);
+  final AtomicLong dataInMemoryWithoutWAL = new AtomicLong(0);
+
   final Counter readRequestsCount = new Counter();
   final Counter writeRequestsCount = new Counter();
 
@@ -1249,6 +1253,10 @@ public boolean flushcache() throws IOException {
         status.setStatus("Running coprocessor pre-flush hooks");
         coprocessorHost.preFlush();
       }
+      if (numPutsWithoutWAL.get() > 0) {
+        numPutsWithoutWAL.set(0);
+        dataInMemoryWithoutWAL.set(0);
+      }
       synchronized (writestate) {
         if (!writestate.flushing && writestate.writesEnabled) {
           this.writestate.flushing = true;
@@ -2098,7 +2106,10 @@ private long doMiniBatchPut(
         batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
 
         Put p = batchOp.operations[i].getFirst();
-        if (!p.getWriteToWAL()) continue;
+        if (!p.getWriteToWAL()) {
+          recordPutWithoutWal(p.getFamilyMap());
+          continue;
+        }
         // Add WAL edits by CP
         WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
         if (fromCP != null) {
@@ -2430,6 +2441,8 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
         addFamilyMapToWALEdit(familyMap, walEdit);
         this.log.append(regionInfo, this.htableDescriptor.getName(),
             walEdit, clusterId, now, this.htableDescriptor);
+      } else {
+        recordPutWithoutWal(familyMap);
       }
 
       long addedSize = applyFamilyMapToMemstore(familyMap, null);
@@ -4648,14 +4661,14 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      32 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      34 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
       (5 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
       ClassSize.OBJECT + // closeLock
       (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing
-      ClassSize.ATOMIC_LONG + // memStoreSize
+      (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL
       ClassSize.ATOMIC_INTEGER + // lockIdGenerator
       (3 * ClassSize.CONCURRENT_HASHMAP) +  // lockedRows, lockIds, scannerReadPoints
       WriteState.HEAP_SIZE + // writestate
@@ -5003,6 +5016,26 @@ private void closeBulkRegionOperation(){
     else lock.readLock().unlock();
   }
 
+  /**
+   * Update counters for numer of puts without wal and the size of possible data loss.
+   * These information are exposed by the region server metrics.
+   */
+  private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
+    if (numPutsWithoutWAL.getAndIncrement() == 0) {
+      LOG.info("writing data to region " + this + 
+               " with WAL disabled. Data may be lost in the event of a crash.");
+    }
+
+    long putSize = 0;
+    for (List<KeyValue> edits : familyMap.values()) {
+      for (KeyValue kv : edits) {
+        putSize += kv.getKeyLength() + kv.getValueLength();
+      }
+    }
+
+    dataInMemoryWithoutWAL.addAndGet(putSize);
+  }
+
   /**
    * A mocked list implementaion - discards all updates.
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 838702c1b364..12a1ce6a288d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1325,6 +1325,8 @@ protected void metrics() {
       new HDFSBlocksDistribution();
     long totalStaticIndexSize = 0;
     long totalStaticBloomSize = 0;
+    long numPutsWithoutWAL = 0;
+    long dataInMemoryWithoutWAL = 0;
 
     // Note that this is a map of Doubles instead of Longs. This is because we
     // do effective integer division, which would perhaps truncate more than it
@@ -1337,6 +1339,8 @@ protected void metrics() {
     for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
       HRegion r = e.getValue();
       memstoreSize += r.memstoreSize.get();
+      numPutsWithoutWAL += r.numPutsWithoutWAL.get();
+      dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
       readRequestsCount += r.readRequestsCount.get();
       writeRequestsCount += r.writeRequestsCount.get();
       synchronized (r.stores) {
@@ -1400,6 +1404,8 @@ protected void metrics() {
     this.metrics.stores.set(stores);
     this.metrics.storefiles.set(storefiles);
     this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
+    this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024)));
+    this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL);
     this.metrics.storefileIndexSizeMB.set(
         (int) (storefileIndexSize / (1024 * 1024)));
     this.metrics.rootIndexSizeKB.set(
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index 4d11d7b5088b..3cb6784f25d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -206,6 +206,18 @@ public class RegionServerMetrics implements Updater {
   public final MetricsIntValue memstoreSizeMB =
     new MetricsIntValue("memstoreSizeMB", registry);
 
+  /**
+   * Number of put with WAL disabled in this regionserver in MB
+   */
+  public final MetricsLongValue numPutsWithoutWAL =
+    new MetricsLongValue("numPutsWithoutWAL", registry);
+
+  /**
+   * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB
+   */
+  public final MetricsIntValue mbInMemoryWithoutWAL =
+    new MetricsIntValue("mbInMemoryWithoutWAL", registry);
+
   /**
    * Size of the compaction queue.
    */
@@ -363,6 +375,8 @@ public void doUpdates(MetricsContext caller) {
       this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
       this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
       this.memstoreSizeMB.pushMetric(this.metricsRecord);
+      this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
+      this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
       this.readRequestsCount.pushMetric(this.metricsRecord);
       this.writeRequestsCount.pushMetric(this.metricsRecord);
       this.regions.pushMetric(this.metricsRecord);
@@ -532,6 +546,10 @@ public String toString() {
         Integer.valueOf(this.totalStaticBloomSizeKB.get()));
     sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
       Integer.valueOf(this.memstoreSizeMB.get()));
+    sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
+      Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
+    sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
+      Long.valueOf(this.numPutsWithoutWAL.get()));
     sb = Strings.appendKeyValue(sb, "readRequestsCount",
         Long.valueOf(this.readRequestsCount.get()));
     sb = Strings.appendKeyValue(sb, "writeRequestsCount",

From c7718ed3546d5ccfc40e50d1a0330cbcfad55453 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 4 May 2012 00:11:27 +0000
Subject: [PATCH 0188/1540] HBASE-5928 Hbck shouldn't npe when there are no
 tables (Elliott Clark)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333692 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HConnectionManager.java | 1 +
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java       | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 910108dc746f..accddae26558 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1782,6 +1782,7 @@ public HTableDescriptor[] listTables() throws IOException {
     }
 
     public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) throws IOException {
+      if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
       if (tableNames == null || tableNames.size() == 0) return null;
       if (this.master == null) {
         this.master = getMaster();
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 83aa3168c336..8217df3dcd05 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2085,7 +2085,7 @@ HTableDescriptor[] getTables(AtomicInteger numSkipped) {
   }
 
    HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
-    HTableDescriptor[] htd = null;
+    HTableDescriptor[] htd = new HTableDescriptor[0];
      try {
        LOG.info("getHTableDescriptors == tableNames => " + tableNames);
        htd = new HBaseAdmin(conf).getTableDescriptors(tableNames);

From 44819df335621c22acf9801838dd353dbb165a79 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 4 May 2012 07:08:18 +0000
Subject: [PATCH 0189/1540] HBASE-5876 TestImportExport has been failing
 against hadoop 0.23 profile

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333777 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/TestImportExport.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index a3faeb3380b1..bc05754b64a1 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
@@ -111,6 +110,7 @@ public void testSimpleCase() throws Exception {
     args = opts.getRemainingArgs();
 
     Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -128,6 +128,7 @@ public void testSimpleCase() throws Exception {
     args = opts.getRemainingArgs();
 
     job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -178,6 +179,7 @@ public void testWithDeletes() throws Exception {
     args = opts.getRemainingArgs();
 
     Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -201,6 +203,7 @@ public void testWithDeletes() throws Exception {
     args = opts.getRemainingArgs();
 
     job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 

From 2d332f17f10017fbe2d9390cc3dc2188208489e0 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 4 May 2012 07:15:21 +0000
Subject: [PATCH 0190/1540] HBASE-5887 Make TestAcidGuarantees usable for
 system testing

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1333784 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/TestAcidGuarantees.java      | 75 +++++++++++++++----
 1 file changed, 60 insertions(+), 15 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
index 4643e6af909d..10039f59de1c 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
@@ -27,8 +27,8 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
@@ -36,11 +36,12 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Ignore;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 import com.google.common.collect.Lists;
-import org.junit.experimental.categories.Category;
 
 /**
  * Test case that uses multiple threads to read and write multifamily rows
@@ -50,7 +51,7 @@
  * a real cluster (eg for testing with failures, region movement, etc)
  */
 @Category(MediumTests.class)
-public class TestAcidGuarantees {
+public class TestAcidGuarantees implements Tool {
   protected static final Log LOG = LogFactory.getLog(TestAcidGuarantees.class);
   public static final byte [] TABLE_NAME = Bytes.toBytes("TestAcidGuarantees");
   public static final byte [] FAMILY_A = Bytes.toBytes("A");
@@ -65,6 +66,9 @@ public class TestAcidGuarantees {
 
   public static int NUM_COLS_TO_CHECK = 50;
 
+  // when run as main
+  private Configuration conf;
+
   private void createTableIfMissing()
     throws IOException {
     try {
@@ -237,6 +241,15 @@ public void runTestAtomicity(long millisToRun,
       int numGetters,
       int numScanners,
       int numUniqueRows) throws Exception {
+      runTestAtomicity(millisToRun, numWriters, numGetters, numScanners,
+		       numUniqueRows, true);
+  }
+
+  public void runTestAtomicity(long millisToRun,
+      int numWriters,
+      int numGetters,
+      int numScanners,
+      int numUniqueRows, boolean useFlusher) throws Exception {
     createTableIfMissing();
     TestContext ctx = new TestContext(util.getConfiguration());
     
@@ -253,11 +266,13 @@ public void runTestAtomicity(long millisToRun,
       ctx.addThread(writer);
     }
     // Add a flusher
-    ctx.addThread(new RepeatingTestThread(ctx) {
-      public void doAnAction() throws Exception {
-        util.flush();
-      }
-    });
+    if (useFlusher) {
+      ctx.addThread(new RepeatingTestThread(ctx) {
+        public void doAnAction() throws Exception {
+          util.flush();
+        }
+      });
+    }
 
     List<AtomicGetReader> getters = Lists.newArrayList();
     for (int i = 0; i < numGetters; i++) {
@@ -323,16 +338,46 @@ public void testMixedAtomicity() throws Exception {
     }    
   }
 
+  ////////////////////////////////////////////////////////////////////////////
+  // Tool interface
+  ////////////////////////////////////////////////////////////////////////////
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration c) {
+    this.conf = c;
+    this.util = new HBaseTestingUtility(c);
+  }
+
+  @Override
+  public int run(String[] arg0) throws Exception {
+    Configuration c = getConf();
+    int millis = c.getInt("millis", 5000);
+    int numWriters = c.getInt("numWriters", 50);
+    int numGetters = c.getInt("numGetters", 2);
+    int numScanners = c.getInt("numScanners", 2);
+    int numUniqueRows = c.getInt("numUniqueRows", 3);
+    // cannot run flusher in real cluster case.
+    runTestAtomicity(millis, numWriters, numGetters, numScanners, numUniqueRows, false);
+    return 0;
+  }
+
   public static void main(String args[]) throws Exception {
     Configuration c = HBaseConfiguration.create();
-    TestAcidGuarantees test = new TestAcidGuarantees();
-    test.setConf(c);
-    test.runTestAtomicity(5000, 50, 2, 2, 3);
+    int status;
+    try {
+      TestAcidGuarantees test = new TestAcidGuarantees();
+      status = ToolRunner.run(c, test, args);
+    } catch (Exception e) {
+      LOG.error("Exiting due to error", e);
+      status = -1;
+    }
+    System.exit(status);
   }
 
-  private void setConf(Configuration c) {
-    util = new HBaseTestingUtility(c);
-  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 7139be463317450f8608f162e70e354be7e71121 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 4 May 2012 15:08:59 +0000
Subject: [PATCH 0191/1540] HBASE-5902 Some scripts are not executable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1334020 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/graceful_stop.sh       | 0
 bin/local-master-backup.sh | 0
 bin/local-regionservers.sh | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 bin/graceful_stop.sh
 mode change 100644 => 100755 bin/local-master-backup.sh
 mode change 100644 => 100755 bin/local-regionservers.sh

diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh
old mode 100644
new mode 100755
diff --git a/bin/local-master-backup.sh b/bin/local-master-backup.sh
old mode 100644
new mode 100755
diff --git a/bin/local-regionservers.sh b/bin/local-regionservers.sh
old mode 100644
new mode 100755

From 4348eb5ecdac569cfb75eb39f7d0ffb8459e5d6f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 5 May 2012 19:59:05 +0000
Subject: [PATCH 0192/1540] HBASE-5894 Table deletion failed but
 HBaseAdmin#deletetable reports it as success (Xufeng)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1334475 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HBaseAdmin.java   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 81228b7a5adf..9f3f62d22768 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -515,6 +515,7 @@ public void deleteTable(final byte [] tableName) throws IOException {
     isMasterRunning();
     HTableDescriptor.isLegalTableName(tableName);
     HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
+    boolean tableExists = true;
     try {
       getMaster().deleteTable(tableName);
     } catch (RemoteException e) {
@@ -538,7 +539,7 @@ public void deleteTable(final byte [] tableName) throws IOException {
         // let us wait until .META. table is updated and
         // HMaster removes the table from its HTableDescriptors
         if (values == null) {
-          boolean tableExists = false;
+          tableExists = false;
           HTableDescriptor[] htds = getMaster().getHTableDescriptors();
           if (htds != null && htds.length > 0) {
             for (HTableDescriptor htd: htds) {
@@ -574,6 +575,11 @@ public void deleteTable(final byte [] tableName) throws IOException {
         // continue
       }
     }
+    
+    if (tableExists) {
+      throw new IOException("Retries exhausted, it took too long to wait"+
+        " for the table " + Bytes.toString(tableName) + " to be deleted.");
+    }
     // Delete cached information to prevent clients from using old locations
     this.connection.clearRegionCache(tableName);
     LOG.info("Deleted " + Bytes.toString(tableName));

From 8d96ec2fe9a8e4f862d978425d42ae5d4d640fea Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 9 May 2012 18:34:30 +0000
Subject: [PATCH 0193/1540] HBASE-5963 ClassCastException:
 FileSystem$Cache$ClientFinalizer cannot be cast to Thread

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1336336 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ShutdownHook.java      | 10 +-
 .../hbase/util/ShutdownHookManager.java       | 97 +++++++++++++++++++
 2 files changed, 104 insertions(+), 3 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
index 4ccf68e7de51..e0363fdf19aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.util.ShutdownHookManager;
 import org.apache.hadoop.hbase.util.Threads;
 
 /**
@@ -81,7 +82,7 @@ public static void install(final Configuration conf, final FileSystem fs,
       final Stoppable stop, final Thread threadToJoin) {
     Thread fsShutdownHook = suppressHdfsShutdownHook(fs);
     Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook);
-    Runtime.getRuntime().addShutdownHook(t);
+    ShutdownHookManager.affixShutdownHook(t, 0);
     LOG.info("Installed shutdown hook thread: " + t.getName());
   }
 
@@ -178,7 +179,10 @@ private static Thread suppressHdfsShutdownHook(final FileSystem fs) {
         Field cacheField = FileSystem.class.getDeclaredField("CACHE");
         cacheField.setAccessible(true);
         Object cacheInstance = cacheField.get(fs);
-        hdfsClientFinalizer = (Thread)field.get(cacheInstance);
+        Runnable finalizerRunnable = (Runnable)field.get(cacheInstance);
+        if (!(finalizerRunnable instanceof Thread)) {
+          hdfsClientFinalizer = new Thread(finalizerRunnable);
+        } else hdfsClientFinalizer = (Thread)finalizerRunnable;
       } else {
         // Then we didnt' find clientFinalizer in Cache.  Presume clean 0.20 hadoop.
         field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD);
@@ -189,7 +193,7 @@ private static Thread suppressHdfsShutdownHook(final FileSystem fs) {
         throw new RuntimeException("Client finalizer is null, can't suppress!");
       }
       if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) &&
-          !Runtime.getRuntime().removeShutdownHook(hdfsClientFinalizer)) {
+          !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) {
         throw new RuntimeException("Failed suppression of fs shutdown hook: " +
           hdfsClientFinalizer);
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java b/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java
new file mode 100644
index 000000000000..ebf308cd554f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+
+/**
+ * This class provides ShutdownHookManager shims for HBase to interact with the Hadoop 1.0.x and the
+ * Hadoop 2.0+ series.
+ * 
+ * NOTE: No testing done against 0.22.x, or 0.21.x.
+ */
+abstract public class ShutdownHookManager {
+  private static ShutdownHookManager instance;
+
+  static Class shutdownHookManagerClass = null;
+  static {
+    try {
+      // This class exists in hadoop 2.0+ but not in Hadoop 20.x/1.x
+      shutdownHookManagerClass = Class.forName("org.apache.hadoop.util.ShutdownHookManager");
+      instance = new ShutdownHookManagerV2();
+    } catch (Exception e) {
+      instance = new ShutdownHookManagerV1();
+    }
+  }
+
+  abstract public void addShutdownHook(Thread shutdownHook, int priority);
+  
+  abstract public boolean removeShutdownHook(Runnable shutdownHook);
+    
+  public static void affixShutdownHook(Thread shutdownHook, int priority) {
+    instance.addShutdownHook(shutdownHook, priority);
+  }
+  
+  public static boolean deleteShutdownHook(Runnable shutdownHook) {
+    return instance.removeShutdownHook(shutdownHook);
+  }
+
+  private static class ShutdownHookManagerV1 extends ShutdownHookManager {
+    // priority is ignored in hadoop versions earlier than 2.0
+    public void addShutdownHook(Thread shutdownHookThread, int priority) {      
+      Runtime.getRuntime().addShutdownHook(shutdownHookThread);
+    }
+    
+    public boolean removeShutdownHook(Runnable shutdownHook) {
+      Thread shutdownHookThread = null;
+      if (!(shutdownHook instanceof Thread)) {
+        shutdownHookThread = new Thread(shutdownHook);
+      } else shutdownHookThread = (Thread) shutdownHook;
+      
+      return Runtime.getRuntime().removeShutdownHook(shutdownHookThread);
+    }
+  };
+
+  private static class ShutdownHookManagerV2 extends ShutdownHookManager {
+    public void addShutdownHook(Thread shutdownHookThread, int priority) {
+      try {
+        Methods.call(shutdownHookManagerClass, 
+            Methods.call(shutdownHookManagerClass, null, "get", null, null),
+            "addShutdownHook",
+            new Class[] { Runnable.class, int.class },
+            new Object[] { shutdownHookThread, priority });
+      } catch (Exception ex) {
+        throw new RuntimeException("we could not use ShutdownHookManager.addShutdownHook", ex);
+      }
+    }
+    
+    public boolean removeShutdownHook(Runnable shutdownHook) {
+      try {
+        return (Boolean)
+        Methods.call(shutdownHookManagerClass, 
+            Methods.call(shutdownHookManagerClass, null, "get", null, null),
+            "removeShutdownHook",
+            new Class[] { Runnable.class },
+            new Object[] { shutdownHook });
+      } catch (Exception ex) {
+        throw new RuntimeException("we could not use ShutdownHookManager", ex);
+      }
+    }
+  };
+
+}

From 4c9b3a6d04dddba0ca6a8624bf7710928ba09ee1 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 9 May 2012 18:47:33 +0000
Subject: [PATCH 0194/1540] HBASE-5964. HFileSystem: "No FileSystem for scheme:
 hdfs"

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1336345 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/fs/HFileSystem.java    | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
index 7e3a68cc03f1..fe72375ad9d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Methods;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Progressable;
 
@@ -141,12 +142,23 @@ public void close() throws IOException {
   private static FileSystem newInstanceFileSystem(Configuration conf)
     throws IOException {
     URI uri = FileSystem.getDefaultUri(conf);
+    FileSystem fs = null;
     Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
-    if (clazz == null) {
+    if (clazz != null) {
+      // This will be true for Hadoop 1.0, or 0.20.
+      fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
+      fs.initialize(uri, conf);
+    } else {
+      // For Hadoop 2.0, we have to go through FileSystem for the filesystem
+      // implementation to be loaded by the service loader in case it has not
+      // been loaded yet.
+      Configuration clone = new Configuration(conf);
+      clone.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true);
+      fs = FileSystem.get(uri, clone);
+    }
+    if (fs == null) {
       throw new IOException("No FileSystem for scheme: " + uri.getScheme());
     }
-    FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
-    fs.initialize(uri, conf);
     return fs;
   }
 

From 6fda76ad35cb8d0bc69a770135ce2a8645ba94f5 Mon Sep 17 00:00:00 2001
From: Todd Lipcon <todd@apache.org>
Date: Thu, 10 May 2012 16:48:33 +0000
Subject: [PATCH 0195/1540] HBASE-5973. Add ability for potentially
 long-running IPC calls to abort if client disconnects. Contributed by Todd
 Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1336788 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/HBaseServer.java  | 19 ++++++++-
 .../apache/hadoop/hbase/ipc/RpcServer.java    |  6 ---
 .../hadoop/hbase/regionserver/HRegion.java    | 11 +++++
 src/main/ruby/hbase/table.rb                  |  6 ++-
 src/main/ruby/shell/commands/scan.rb          |  2 +-
 .../hadoop/hbase/filter/TestFilter.java       | 41 ++++++++++++++++++-
 .../hadoop/hbase/ipc/TestDelayedRpc.java      |  4 +-
 7 files changed, 75 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 58b1376f2e9c..3b1cb7975dfb 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -256,7 +256,7 @@ public static void bind(ServerSocket socket, InetSocketAddress address,
   }
 
   /** A call queued for handling. */
-  protected class Call implements Delayable {
+  protected class Call implements RpcCallContext {
     protected int id;                             // the client's call id
     protected Writable param;                     // the parameter passed
     protected Connection connection;              // connection to client
@@ -412,6 +412,16 @@ public synchronized boolean isDelayed() {
     public synchronized boolean isReturnValueDelayed() {
       return this.delayReturnValue;
     }
+    
+    @Override
+    public void throwExceptionIfCallerDisconnected() throws CallerDisconnectedException {
+      if (!connection.channel.isOpen()) {
+        long afterTime = System.currentTimeMillis() - timestamp;
+        throw new CallerDisconnectedException(
+            "Aborting call " + this + " after " + afterTime + " ms, since " +
+            "caller disconnected");
+      }
+    }
 
     public long getSize() {
       return this.size;
@@ -1777,7 +1787,12 @@ private static int channelIO(ReadableByteChannel readCh,
     return (nBytes > 0) ? nBytes : ret;
   }
 
-  public Delayable getCurrentCall() {
+  /**
+   * Needed for delayed calls.  We need to be able to store the current call
+   * so that we can complete it later.
+   * @return Call the server is currently handling.
+   */
+  public static RpcCallContext getCurrentCall() {
     return CurCall.get();
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 7b2df77c248c..716fa229c409 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -60,12 +60,6 @@ Writable call(Class<? extends VersionedProtocol> protocol,
 
   void startThreads();
 
-  /**
-   * Needed for delayed calls.  We need to be able to store the current call
-   * so that we can complete it later.
-   * @return Call the server is currently handling.
-   */
-  Delayable getCurrentCall();
 
   /**
    * Returns the metrics instance for reporting RPC call statistics
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 32ab1bf39b8a..1a468bf55ed5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -102,6 +102,8 @@
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
+import org.apache.hadoop.hbase.ipc.RpcCallContext;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
@@ -3355,7 +3357,16 @@ public synchronized boolean isFilterDone() {
     }
 
     private boolean nextInternal(int limit) throws IOException {
+      RpcCallContext rpcCall = HBaseServer.getCurrentCall();
       while (true) {
+        if (rpcCall != null) {
+          // If a user specifies a too-restrictive or too-slow scanner, the
+          // client might time out and disconnect while the server side
+          // is still processing the request. We should abort aggressively
+          // in that case.
+          rpcCall.throwExceptionIfCallerDisconnected();
+        }
+
         byte [] currentRow = peekRow();
         if (isStopRow(currentRow)) {
           if (filter != null && filter.hasFilterRow()) {
diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index 41dcf21b2936..5bcbb4dc323e 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -220,7 +220,8 @@ def scan(args = {})
         stoprow = args["STOPROW"]
         timestamp = args["TIMESTAMP"]
         columns = args["COLUMNS"] || args["COLUMN"] || []
-        cache = args["CACHE_BLOCKS"] || true
+        cache_blocks = args["CACHE_BLOCKS"] || true
+        cache = args["CACHE"] || 0
         versions = args["VERSIONS"] || 1
         timerange = args[TIMERANGE]
         raw = args["RAW"] || false
@@ -253,7 +254,8 @@ def scan(args = {})
         end
 
         scan.setTimeStamp(timestamp) if timestamp
-        scan.setCacheBlocks(cache)
+        scan.setCacheBlocks(cache_blocks)
+        scan.setCaching(cache) if cache > 0
         scan.setMaxVersions(versions) if versions > 1
         scan.setTimeRange(timerange[0], timerange[1]) if timerange
         scan.setRaw(raw)
diff --git a/src/main/ruby/shell/commands/scan.rb b/src/main/ruby/shell/commands/scan.rb
index e58aaaceb8ee..518e628d67bd 100644
--- a/src/main/ruby/shell/commands/scan.rb
+++ b/src/main/ruby/shell/commands/scan.rb
@@ -26,7 +26,7 @@ def help
 Scan a table; pass table name and optionally a dictionary of scanner
 specifications.  Scanner specifications may include one or more of:
 TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH,
-or COLUMNS.
+or COLUMNS, CACHE
 
 If no columns are specified, all columns will be scanned.
 To scan all members of a column family, leave the qualifier empty as in
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index 396bba382187..130d84dd9819 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.filter;
 
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -41,12 +43,14 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.base.Throwables;
+
 /**
  * Test filters at the HRegion doorstep.
  */
 @Category(SmallTests.class)
 public class TestFilter extends HBaseTestCase {
-  private final Log LOG = LogFactory.getLog(this.getClass());
+  private final static Log LOG = LogFactory.getLog(TestFilter.class);
   private HRegion region;
 
   //
@@ -1616,6 +1620,41 @@ public void testKeyOnlyFilter() throws Exception {
       verifyScanFullNoValues(s, expectedKVs, useLen);
     }
   }
+  
+  /**
+   * Filter which makes sleeps for a second between each row of a scan.
+   * This can be useful for manual testing of bugs like HBASE-5973. For example:
+   * <code>
+   * create 't1', 'f1'
+   * 1.upto(100)  { |x| put 't1', 'r' + x.to_s, 'f1:q1', 'hi' }
+   * import org.apache.hadoop.hbase.filter.TestFilter
+   * scan 't1', { FILTER => TestFilter::SlowScanFilter.new(), CACHE => 50 }
+   * </code>
+   */
+  public static class SlowScanFilter extends FilterBase {
+    private static Thread ipcHandlerThread = null;
+    
+    @Override
+    public void readFields(DataInput arg0) throws IOException {
+    }
+
+    @Override
+    public void write(DataOutput arg0) throws IOException {
+    }
+
+    @Override
+    public boolean filterRow() {
+      ipcHandlerThread = Thread.currentThread();
+      try {
+        LOG.info("Handler thread " + ipcHandlerThread + " sleeping in filter...");
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        Throwables.propagate(e);
+      }
+      return super.filterRow();
+    }
+  }
+
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
index 5229b6d0500a..9046d99b2fa2 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
@@ -186,7 +186,7 @@ public int test(final boolean delay) {
       if (!delay) {
         return UNDELAYED;
       }
-      final Delayable call = rpcServer.getCurrentCall();
+      final Delayable call = HBaseServer.getCurrentCall();
       call.startDelay(delayReturnValue);
       new Thread() {
         public void run() {
@@ -288,7 +288,7 @@ private static class FaultyTestRpc implements TestRpc {
     public int test(boolean delay) {
       if (!delay)
         return UNDELAYED;
-      Delayable call = rpcServer.getCurrentCall();
+      Delayable call = HBaseServer.getCurrentCall();
       call.startDelay(true);
       try {
         call.endDelayThrowing(new Exception("Something went wrong"));

From 8cda0a122a4dc0ca62ddb70824dccc743f5c25ff Mon Sep 17 00:00:00 2001
From: Todd Lipcon <todd@apache.org>
Date: Thu, 10 May 2012 17:03:44 +0000
Subject: [PATCH 0196/1540] Amend HBASE-5973. Add two missing files from
 previous commit

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1336797 13f79535-47bb-0310-9956-ffa450edef68
---
 .../ipc/CallerDisconnectedException.java      | 35 +++++++++++++++++++
 .../hadoop/hbase/ipc/RpcCallContext.java      | 29 +++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java b/src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java
new file mode 100644
index 000000000000..00e450f4058a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+
+/**
+ * Exception indicating that the remote host making this IPC lost its
+ * IPC connection. This will never be returned back to a client,
+ * but is only used for logging on the server side, etc.
+ */
+public class CallerDisconnectedException extends IOException {
+  public CallerDisconnectedException(String msg) {
+    super(msg);
+  }
+
+  private static final long serialVersionUID = 1L;
+
+  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java
new file mode 100644
index 000000000000..60236d6eb494
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+public interface RpcCallContext extends Delayable {
+
+  /**
+   * Throw an exception if the caller who made this IPC call has disconnected.
+   * If called from outside the context of IPC, this does nothing.
+   * @throws CallerDisconnectedException
+   */
+  void throwExceptionIfCallerDisconnected() throws CallerDisconnectedException;
+
+}

From e18c407d8c3d1ccdb1657b02ed13178a5f928c4f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 10 May 2012 20:16:28 +0000
Subject: [PATCH 0197/1540] HBASE-5975 Failed suppression of fs shutdown hook
 with Hadoop 2.0.0 (Jimmy Xiang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1336876 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ShutdownHook.java      | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
index e0363fdf19aa..5f16535dcc37 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
@@ -57,7 +57,7 @@ public class ShutdownHook {
    * to be executed after the last regionserver referring to a given filesystem
    * stops. We keep track of the # of regionserver references in values of the map.
    */
-  private final static Map<Thread, Integer> fsShutdownHooks = new HashMap<Thread, Integer>();
+  private final static Map<Runnable, Integer> fsShutdownHooks = new HashMap<Runnable, Integer>();
 
   /**
    * Install a shutdown hook that calls stop on the passed Stoppable
@@ -80,7 +80,7 @@ public class ShutdownHook {
    */
   public static void install(final Configuration conf, final FileSystem fs,
       final Stoppable stop, final Thread threadToJoin) {
-    Thread fsShutdownHook = suppressHdfsShutdownHook(fs);
+    Runnable fsShutdownHook = suppressHdfsShutdownHook(fs);
     Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook);
     ShutdownHookManager.affixShutdownHook(t, 0);
     LOG.info("Installed shutdown hook thread: " + t.getName());
@@ -92,11 +92,11 @@ public static void install(final Configuration conf, final FileSystem fs,
   private static class ShutdownHookThread extends Thread {
     private final Stoppable stop;
     private final Thread threadToJoin;
-    private final Thread fsShutdownHook;
+    private final Runnable fsShutdownHook;
     private final Configuration conf;
 
     ShutdownHookThread(final Configuration conf, final Stoppable stop,
-        final Thread threadToJoin, final Thread fsShutdownHook) {
+        final Thread threadToJoin, final Runnable fsShutdownHook) {
       super("Shutdownhook:" + threadToJoin.getName());
       this.stop = stop;
       this.threadToJoin = threadToJoin;
@@ -117,8 +117,10 @@ public void run() {
             int refs = fsShutdownHooks.get(fsShutdownHook);
             if (refs == 1) {
               LOG.info("Starting fs shutdown hook thread.");
-              this.fsShutdownHook.start();
-              Threads.shutdown(this.fsShutdownHook,
+              Thread fsShutdownHookThread = (fsShutdownHook instanceof Thread) ?
+                (Thread)fsShutdownHook : new Thread(fsShutdownHook);
+              fsShutdownHookThread.start();
+              Threads.shutdown(fsShutdownHookThread,
               this.conf.getLong(FS_SHUTDOWN_HOOK_WAIT, 30000));
             }
             if (refs > 0) {
@@ -148,7 +150,7 @@ public void run() {
    * @return The fs shutdown hook
    * @throws RuntimeException if we fail to find or grap the shutdown hook.
    */
-  private static Thread suppressHdfsShutdownHook(final FileSystem fs) {
+  private static Runnable suppressHdfsShutdownHook(final FileSystem fs) {
     try {
       // This introspection has been updated to work for hadoop 0.20, 0.21 and for
       // cloudera 0.20.  0.21 and cloudera 0.20 both have hadoop-4829.  With the
@@ -157,7 +159,7 @@ private static Thread suppressHdfsShutdownHook(final FileSystem fs) {
       // FileSystem and one in the innner class named Cache that actually gets
       // registered as a shutdown hook.  If the latter is present, then we are
       // on 0.21 or cloudera patched 0.20.
-      Thread hdfsClientFinalizer = null;
+      Runnable hdfsClientFinalizer = null;
       // Look into the FileSystem#Cache class for clientFinalizer
       Class<?> [] classes = FileSystem.class.getDeclaredClasses();
       Class<?> cache = null;
@@ -179,15 +181,12 @@ private static Thread suppressHdfsShutdownHook(final FileSystem fs) {
         Field cacheField = FileSystem.class.getDeclaredField("CACHE");
         cacheField.setAccessible(true);
         Object cacheInstance = cacheField.get(fs);
-        Runnable finalizerRunnable = (Runnable)field.get(cacheInstance);
-        if (!(finalizerRunnable instanceof Thread)) {
-          hdfsClientFinalizer = new Thread(finalizerRunnable);
-        } else hdfsClientFinalizer = (Thread)finalizerRunnable;
+        hdfsClientFinalizer = (Runnable)field.get(cacheInstance);
       } else {
         // Then we didnt' find clientFinalizer in Cache.  Presume clean 0.20 hadoop.
         field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD);
         field.setAccessible(true);
-        hdfsClientFinalizer = (Thread)field.get(null);
+        hdfsClientFinalizer = (Runnable)field.get(null);
       }
       if (hdfsClientFinalizer == null) {
         throw new RuntimeException("Client finalizer is null, can't suppress!");

From bd5918d07c9e0765599148acd98f2e0e669b40a4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 11 May 2012 23:27:25 +0000
Subject: [PATCH 0198/1540] HBASE-5922 HalfStoreFileReader seekBefore causes
 StackOverflowError

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1337411 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/HalfStoreFileReader.java  |  3 +-
 .../hbase/io/TestHalfStoreFileReader.java     | 85 +++++++++++++++++++
 2 files changed, 87 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index c2ceb1e23613..b53fa0bca5bb 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -146,9 +146,10 @@ public boolean seekBefore(byte [] key, int offset, int length)
             return false;
           }
         } else {
+          // The equals sign isn't strictly necessary just here to be consistent with seekTo
           if (getComparator().compare(key, offset, length, splitkey, 0,
               splitkey.length) >= 0) {
-            return seekBefore(splitkey, 0, splitkey.length);
+            return this.delegate.seekBefore(splitkey, 0, splitkey.length);
           }
         }
         return this.delegate.seekBefore(key, offset, length);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
index 717432faebc8..3f695b191a93 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.io;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -123,6 +125,89 @@ private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
     halfreader.close(true);
   }
 
+
+  // Tests the scanner on an HFile that is backed by HalfStoreFiles
+  @Test
+  public void testHalfScanner() throws IOException {
+      HBaseTestingUtility test_util = new HBaseTestingUtility();
+      String root_dir = test_util.getDataTestDir("TestHalfStoreFileScanBefore").toString();
+      Path p = new Path(root_dir, "test");
+      Configuration conf = test_util.getConfiguration();
+      FileSystem fs = FileSystem.get(conf);
+      CacheConfig cacheConf = new CacheConfig(conf);
+
+      HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
+              .withPath(fs, p)
+              .withBlockSize(1024)
+              .withComparator(KeyValue.KEY_COMPARATOR)
+              .create();
+
+      // write some things.
+      List<KeyValue> items = genSomeKeys();
+      for (KeyValue kv : items) {
+          w.append(kv);
+      }
+      w.close();
+
+
+      HFile.Reader r = HFile.createReader(fs, p, cacheConf);
+      r.loadFileInfo();
+      byte[] midkey = r.midkey();
+      KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
+      midkey = midKV.getRow();
+
+      Reference bottom = new Reference(midkey, Reference.Range.bottom);
+      Reference top = new Reference(midkey, Reference.Range.top);
+
+      // Ugly code to get the item before the midkey
+      KeyValue beforeMidKey = null;
+      for (KeyValue item : items) {
+          if (item.equals(midKV)) {
+              break;
+          }
+          beforeMidKey = item;
+      }
+
+
+      // Seek on the splitKey, should be in top, not in bottom
+      KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
+      assertEquals(beforeMidKey, foundKeyValue);
+
+      // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
+      foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
+      assertEquals(items.get(items.size() - 2), foundKeyValue);
+
+      foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
+      assertEquals(beforeMidKey, foundKeyValue);
+
+      // Try and seek before something that is in the bottom.
+      foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
+      assertNull(foundKeyValue);
+
+      // Try and seek before the first thing.
+      foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
+      assertNull(foundKeyValue);
+
+      // Try and seek before the second thing in the top and bottom.
+      foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
+      assertNull(foundKeyValue);
+
+      foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
+      assertEquals(items.get(0), foundKeyValue);
+
+    }
+
+  private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
+                                        CacheConfig cacheConfig)
+            throws IOException {
+      final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
+              cacheConfig, bottom, DataBlockEncoding.NONE);
+      halfreader.loadFileInfo();
+      final HFileScanner scanner = halfreader.getScanner(false, false);
+      scanner.seekBefore(seekBefore.getKey());
+      return scanner.getKeyValue();
+  }
+
   private KeyValue getLastOnCol(KeyValue curr) {
     return KeyValue.createLastOnRow(
         curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(),

From 2a53afd03e83e3d3fe7914c41382c3a7a2bde594 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 14 May 2012 18:31:04 +0000
Subject: [PATCH 0199/1540] HBASE-5806 Handle split region related failures on
 master restart and RS restart(Chinna rao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1338331 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/LocalHBaseCluster.java       |   9 +-
 .../hbase/master/AssignmentManager.java       |  13 ++
 .../apache/hadoop/hbase/master/HMaster.java   |  11 +-
 .../master/handler/ServerShutdownHandler.java |   2 +-
 .../TestSplitTransactionOnCluster.java        | 185 ++++++++++++++++++
 5 files changed, 210 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index 134749add43e..888ee77e5fa4 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -67,7 +67,6 @@ public class LocalHBaseCluster {
   /** 'local:' */
   public static final String LOCAL_COLON = LOCAL + ":";
   private final Configuration conf;
-  private final Class<? extends HMaster> masterClass;
   private final Class<? extends HRegionServer> regionServerClass;
 
   /**
@@ -141,9 +140,6 @@ public LocalHBaseCluster(final Configuration conf, final int noMasters,
     conf.set(HConstants.MASTER_PORT, "0");
     conf.set(HConstants.REGIONSERVER_PORT, "0");
     // Start the HMasters.
-    this.masterClass =
-      (Class<? extends HMaster>)conf.getClass(HConstants.MASTER_IMPL,
-          masterClass);
     for (int i = 0; i < noMasters; i++) {
       addMaster(new Configuration(conf), i);
     }
@@ -195,9 +191,8 @@ public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index)
     // Create each master with its own Configuration instance so each has
     // its HConnection instance rather than share (see HBASE_INSTANCES down in
     // the guts of HConnectionManager.
-    JVMClusterUtil.MasterThread mt =
-      JVMClusterUtil.createMasterThread(c,
-        this.masterClass, index);
+    JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c,
+        (Class<? extends HMaster>) c.getClass(HConstants.MASTER_IMPL, HMaster.class), index);
     this.masterThreads.add(mt);
     return mt;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 14fc66783f55..66bb0d9d072b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2502,6 +2502,19 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
         enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
             disablingOrEnabling, tableName);
       } else {
+        // If region is in offline and split state check the ZKNode
+        if (regionInfo.isOffline() && regionInfo.isSplit()) {
+          String node = ZKAssign.getNodeName(this.watcher, regionInfo
+              .getEncodedName());
+          Stat stat = new Stat();
+          byte[] data = ZKUtil.getDataNoWatch(this.watcher, node, stat);
+          // If znode does not exist dont consider this region
+          if (data == null) {
+            LOG.debug("Region "+ regionInfo.getRegionNameAsString() + " split is completed. " 
+                + "Hence need not add to regions list");
+            continue;
+          }
+        }
         // Region is being served and on an active server
         // add only if region not in disabled and enabling table
         if (false == checkIfRegionBelongsToDisabled(regionInfo)
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 20e042680f31..66d618d74db2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -549,8 +549,7 @@ private void finishInitialization(MonitoredTask status)
     status.setStatus("Starting balancer and catalog janitor");
     this.balancerChore = getAndStartBalancerChore(this);
     this.catalogJanitorChore = new CatalogJanitor(this, this);
-    Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
-
+    startCatalogJanitorChore();
     registerMBean();
 
     status.markComplete("Initialization successful");
@@ -566,6 +565,14 @@ private void finishInitialization(MonitoredTask status)
       }
     }
   }
+  
+  /**
+   * Useful for testing purpose also where we have
+   * master restart scenarios.
+   */
+  protected void startCatalogJanitorChore() {
+    Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
+  }
 
   /**
    * Override to change master's splitLogAfterStartup. Used testing
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 4684cc13a677..943e60c4ee2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -266,7 +266,7 @@ public void process() throws IOException {
 
       // Skip regions that were in transition unless CLOSING or PENDING_CLOSE
       for (RegionState rit : regionsInTransition) {
-        if (!rit.isClosing() && !rit.isPendingClose()) {
+        if (!rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()) {
           LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
           " from list of regions to assign because in RIT; region state: " +
           rit.getState());
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 1997abd531cb..f225496dc97e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -28,17 +28,20 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.handler.SplitRegionHandler;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.apache.zookeeper.data.Stat;
@@ -352,6 +355,176 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }
+  
+  /**
+   * Verifies HBASE-5806.  When splitting is partially done and the master goes down
+   * when the SPLIT node is in either SPLIT or SPLITTING state.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws NodeExistsException
+   * @throws KeeperException
+   */
+  @Test(timeout = 300000)
+  public void testMasterRestartWhenSplittingIsPartial()
+      throws IOException, InterruptedException, NodeExistsException,
+      KeeperException {
+    final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
+
+    // Create table then get the single region for our new table.
+    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
+
+    List<HRegion> regions = cluster.getRegions(tableName);
+    HRegionInfo hri = getAndCheckSingleTableRegion(regions);
+
+    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
+
+    // Turn off the meta scanner so it don't remove parent on us.
+    cluster.getMaster().setCatalogJanitorEnabled(false);
+    // Turn off balancer so it doesn't cut in and mess up our placements.
+    this.admin.balanceSwitch(false);
+    
+    try {
+      // Add a bit of load up into the table so splittable.
+      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
+      // Get region pre-split.
+      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
+      printOutRegions(server, "Initial regions: ");
+      int regionCount = server.getOnlineRegions().size();
+      // Now, before we split, set special flag in master, a flag that has
+      // it FAIL the processing of split.
+      SplitRegionHandler.TEST_SKIP = true;
+      // Now try splitting and it should work.
+      split(hri, server, regionCount);
+      // Get daughters
+      List<HRegion> daughters = cluster.getRegions(tableName);
+      assertTrue(daughters.size() >= 2);
+      // Assert the ephemeral node is up in zk.
+      String path = ZKAssign.getNodeName(t.getConnection()
+          .getZooKeeperWatcher(), hri.getEncodedName());
+      Stat stats = t.getConnection().getZooKeeperWatcher()
+          .getRecoverableZooKeeper().exists(path, false);
+      LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
+          + stats);
+      RegionTransitionData rtd = ZKAssign.getData(t.getConnection()
+          .getZooKeeperWatcher(), hri.getEncodedName());
+      // State could be SPLIT or SPLITTING.
+      assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
+          || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
+
+
+      // abort and wait for new master.
+      MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
+
+      this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+
+      // update the hri to be offlined and splitted. 
+      hri.setOffline(true);
+      hri.setSplit(true);
+      ServerName regionServerOfRegion = master.getAssignmentManager()
+          .getRegionServerOfRegion(hri);
+      assertTrue(regionServerOfRegion != null);
+
+    } finally {
+      // Set this flag back.
+      SplitRegionHandler.TEST_SKIP = false;
+      admin.balanceSwitch(true);
+      cluster.getMaster().setCatalogJanitorEnabled(true);
+    }
+  }
+
+
+  /**
+   * Verifies HBASE-5806.  Here the case is that splitting is completed but before the
+   * CJ could remove the parent region the master is killed and restarted.
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws NodeExistsException
+   * @throws KeeperException
+   */
+  @Test (timeout = 300000)
+  public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
+      throws IOException, InterruptedException, NodeExistsException,
+      KeeperException {
+    final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
+
+    // Create table then get the single region for our new table.
+    this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor(HConstants.CATALOG_FAMILY);
+    htd.addFamily(hcd);
+    this.admin.createTable(htd);
+    HTable t = new HTable(TESTING_UTIL.getConfiguration(), tableName);
+
+    List<HRegion> regions = cluster.getRegions(tableName);
+    HRegionInfo hri = getAndCheckSingleTableRegion(regions);
+
+    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
+
+    // Turn off balancer so it doesn't cut in and mess up our placements.
+    this.admin.balanceSwitch(false);
+    // Turn off the meta scanner so it don't remove parent on us.
+    cluster.getMaster().setCatalogJanitorEnabled(false);
+    try {
+      // Add a bit of load up into the table so splittable.
+      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
+      // Get region pre-split.
+      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
+      printOutRegions(server, "Initial regions: ");
+      int regionCount = server.getOnlineRegions().size();
+      
+      split(hri, server, regionCount);
+      // Get daughters
+      List<HRegion> daughters = cluster.getRegions(tableName);
+      assertTrue(daughters.size() >= 2);
+      // Assert the ephemeral node is up in zk.
+      String path = ZKAssign.getNodeName(t.getConnection()
+          .getZooKeeperWatcher(), hri.getEncodedName());
+      Stat stats = t.getConnection().getZooKeeperWatcher()
+          .getRecoverableZooKeeper().exists(path, false);
+      LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
+          + stats);
+      String node = ZKAssign.getNodeName(t.getConnection()
+          .getZooKeeperWatcher(), hri.getEncodedName());
+      Stat stat = new Stat();
+      byte[] data = ZKUtil.getDataNoWatch(t.getConnection()
+          .getZooKeeperWatcher(), node, stat);
+      // ZKUtil.create
+      while (data != null) {
+        Thread.sleep(1000);
+        data = ZKUtil.getDataNoWatch(t.getConnection().getZooKeeperWatcher(),
+            node, stat);
+
+      }
+      
+      MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
+
+      this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+
+      hri.setOffline(true);
+      hri.setSplit(true);
+      ServerName regionServerOfRegion = master.getAssignmentManager()
+          .getRegionServerOfRegion(hri);
+      assertTrue(regionServerOfRegion == null);
+    } finally {
+      // Set this flag back.
+      SplitRegionHandler.TEST_SKIP = false;
+      this.admin.balanceSwitch(true);
+      cluster.getMaster().setCatalogJanitorEnabled(true);
+    }
+  }
+
+  private MockMasterWithoutCatalogJanitor abortAndWaitForMaster() 
+  throws IOException, InterruptedException {
+    cluster.abortMaster(0);
+    cluster.waitOnMaster(0);
+    cluster.getConfiguration().setClass(HConstants.MASTER_IMPL, 
+        MockMasterWithoutCatalogJanitor.class, HMaster.class);
+    MockMasterWithoutCatalogJanitor master = null;
+    master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
+    cluster.waitForActiveAndReadyMaster();
+    return master;
+  }
 
   private void split(final HRegionInfo hri, final HRegionServer server,
       final int regionCount)
@@ -458,5 +631,17 @@ private void waitUntilRegionServerDead() throws InterruptedException {
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+  
+  public static class MockMasterWithoutCatalogJanitor extends HMaster {
+
+    public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
+        InterruptedException {
+      super(conf);
+    }
+
+    protected void startCatalogJanitorChore() {
+      LOG.debug("Customised master executed.");
+    }
+  }
 }
 

From 08b51c09a0ffabe76172f9f12da4358fbc5af437 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 17 May 2012 16:27:16 +0000
Subject: [PATCH 0200/1540] HBASE-6029 HBCK doesn't recover Balance switch if
 exception occurs in onlineHbck() (Maryann)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339688 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/HBaseFsck.java   | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8217df3dcd05..a72ba0eaa521 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -378,10 +378,12 @@ public int onlineHbck() throws IOException, KeeperException, InterruptedExceptio
 
     // turn the balancer off
     boolean oldBalancer = admin.balanceSwitch(false);
-
-    onlineConsistencyRepair();
-
-    admin.balanceSwitch(oldBalancer);
+    try {
+      onlineConsistencyRepair();
+    }
+    finally {
+      admin.balanceSwitch(oldBalancer);
+    }
 
     // Print table summary
     printTableSummary(tablesInfo);

From b4208ce5f3cb128c18b8b6cee8dcc190f62723e3 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 May 2012 18:42:07 +0000
Subject: [PATCH 0201/1540] HBASE-6001. Upgrade slf4j to 1.6.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339790 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f3156b1708f6..5d26793411f8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -988,7 +988,6 @@
     <log4j.version>1.2.16</log4j.version>
     <mockito-all.version>1.8.5</mockito-all.version>
     <protobuf.version>2.4.0a</protobuf.version>
-    <slf4j.version>1.5.8</slf4j.version><!-- newer version available -->
     <stax-api.version>1.0.1</stax-api.version>
     <thrift.version>0.8.0</thrift.version>
     <zookeeper.version>3.4.3</zookeeper.version>
@@ -1517,6 +1516,7 @@
       </activation>
       <properties>
         <hadoop.version>1.0.2</hadoop.version>
+        <slf4j.version>1.4.3</slf4j.version>
       </properties>
       <dependencies>
         <dependency>
@@ -1655,6 +1655,7 @@
       </activation>
       <properties>
         <hadoop.version>0.22.0</hadoop.version>
+        <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencies>
         <dependency>
@@ -1833,6 +1834,7 @@
       </activation>
       <properties>
         <hadoop.version>0.23.2-SNAPSHOT</hadoop.version>
+        <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencies>
         <dependency>
@@ -1914,6 +1916,7 @@
       </activation>
       <properties>
         <hadoop.version>0.24.0-SNAPSHOT</hadoop.version>
+        <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencies>
         <dependency>

From 2a63a5d49f9590ceff7bc396bebac201362000c9 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 May 2012 19:18:17 +0000
Subject: [PATCH 0202/1540] HBASE-6004. Adding more logging to help debugging
 MR job (Jimmy Xiang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339809 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ScannerCallable.java  | 45 ++++++++-
 .../hbase/mapred/TableRecordReaderImpl.java   | 91 ++++++++++++++-----
 .../mapreduce/TableRecordReaderImpl.java      | 89 +++++++++++++-----
 3 files changed, 174 insertions(+), 51 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
index 4a4679c14350..27101509eb2a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
@@ -23,11 +23,14 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.net.DNS;
@@ -37,6 +40,9 @@
  * Used by {@link ResultScanner}s made by {@link HTable}.
  */
 public class ScannerCallable extends ServerCallable<Result[]> {
+  public static final String LOG_SCANNER_LATENCY_CUTOFF
+    = "hbase.client.log.scanner.latency.cutoff";
+  public static final String LOG_SCANNER_ACTIVITY = "hbase.client.log.scanner.activity";
   private static final Log LOG = LogFactory.getLog(ScannerCallable.class);
   private long scannerId = -1L;
   private boolean instantiated = false;
@@ -44,6 +50,8 @@ public class ScannerCallable extends ServerCallable<Result[]> {
   private Scan scan;
   private int caching = 1;
   private ScanMetrics scanMetrics;
+  private boolean logScannerActivity = false;
+  private int logCutOffLatency = 1000;
 
   // indicate if it is a remote server call
   private boolean isRegionServerRemote = true;
@@ -60,6 +68,9 @@ public ScannerCallable (HConnection connection, byte [] tableName, Scan scan,
     super(connection, tableName, scan.getStartRow());
     this.scan = scan;
     this.scanMetrics = scanMetrics;
+    Configuration conf = connection.getConfiguration();
+    logScannerActivity = conf.getBoolean(LOG_SCANNER_ACTIVITY, false);
+    logCutOffLatency = conf.getInt(LOG_SCANNER_LATENCY_CUTOFF, 1000);
   }
 
   /**
@@ -111,14 +122,38 @@ private void checkIfRegionServerIsRemote() throws UnknownHostException {
       Result [] rrs = null;
       try {
         incRPCcallsMetrics();
+        long timestamp = System.currentTimeMillis();
         rrs = server.next(scannerId, caching);
+        if (logScannerActivity) {
+          long now = System.currentTimeMillis();
+          if (now - timestamp > logCutOffLatency) {
+            int rows = rrs == null ? 0 : rrs.length;
+            LOG.info("Took " + (now-timestamp) + "ms to fetch "
+              + rows + " rows from scanner=" + scannerId);
+          }
+        }
         updateResultsMetrics(rrs);
       } catch (IOException e) {
+        if (logScannerActivity) {
+          LOG.info("Got exception in fetching from scanner="
+            + scannerId, e);
+        }
         IOException ioe = null;
         if (e instanceof RemoteException) {
           ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e);
         }
         if (ioe == null) throw new IOException(e);
+        if (logScannerActivity && (ioe instanceof UnknownScannerException)) {
+          try {
+            HRegionLocation location =
+              connection.relocateRegion(tableName, scan.getStartRow());
+            LOG.info("Scanner=" + scannerId
+              + " expired, current region location is " + location.toString()
+              + " ip:" + location.getServerAddress().getBindAddress());
+          } catch (Throwable t) {
+            LOG.info("Failed to relocate region", t);
+          }
+        }
         if (ioe instanceof NotServingRegionException) {
           // Throw a DNRE so that we break out of cycle of calling NSRE
           // when what we need is to open scanner against new location.
@@ -180,8 +215,14 @@ private void close() {
 
   protected long openScanner() throws IOException {
     incRPCcallsMetrics();
-    return this.server.openScanner(this.location.getRegionInfo().getRegionName(),
-      this.scan);
+    long id = this.server.openScanner(this.location.getRegionInfo().getRegionName(),
+       this.scan);
+    if (logScannerActivity) {
+      LOG.info("Open scanner=" + id + " for scan=" + scan.toString()
+        + " on region " + this.location.toString() + " ip:"
+        + this.location.getServerAddress().getBindAddress());
+    }
+    return id;
   }
 
   protected Scan getScan() {
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
index 42569fbdd251..a3639716cc88 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -23,11 +23,13 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ScannerCallable;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
@@ -35,6 +37,7 @@
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.util.StringUtils;
 
+import static org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.LOG_PER_ROW_COUNT;
 
 /**
  * Iterate over an HBase table data, return (Text, RowResult) pairs
@@ -49,6 +52,10 @@ public class TableRecordReaderImpl {
   private ResultScanner scanner;
   private HTable htable;
   private byte [][] trrInputColumns;
+  private long timestamp;
+  private int rowcount;
+  private boolean logScannerActivity = false;
+  private int logPerRowCount = 100;
 
   /**
    * Restart from survivable exceptions by creating a new scanner.
@@ -57,6 +64,7 @@ public class TableRecordReaderImpl {
    * @throws IOException
    */
   public void restart(byte[] firstRow) throws IOException {
+    Scan currentScan;
     if ((endRow != null) && (endRow.length > 0)) {
       if (trrRowFilter != null) {
         Scan scan = new Scan(firstRow, endRow);
@@ -64,6 +72,7 @@ public void restart(byte[] firstRow) throws IOException {
         scan.setFilter(trrRowFilter);
         scan.setCacheBlocks(false);
         this.scanner = this.htable.getScanner(scan);
+        currentScan = scan;
       } else {
         LOG.debug("TIFB.restart, firstRow: " +
             Bytes.toStringBinary(firstRow) + ", endRow: " +
@@ -71,6 +80,7 @@ public void restart(byte[] firstRow) throws IOException {
         Scan scan = new Scan(firstRow, endRow);
         TableInputFormat.addColumns(scan, trrInputColumns);
         this.scanner = this.htable.getScanner(scan);
+        currentScan = scan;
       }
     } else {
       LOG.debug("TIFB.restart, firstRow: " +
@@ -80,6 +90,12 @@ public void restart(byte[] firstRow) throws IOException {
       TableInputFormat.addColumns(scan, trrInputColumns);
       scan.setFilter(trrRowFilter);
       this.scanner = this.htable.getScanner(scan);
+      currentScan = scan;
+    }
+    if (logScannerActivity) {
+      LOG.info("Current scan=" + currentScan.toString());
+      timestamp = System.currentTimeMillis();
+      rowcount = 0;
     }
   }
 
@@ -99,6 +115,10 @@ byte[] getStartRow() {
    * @param htable the {@link HTable} to scan.
    */
   public void setHTable(HTable htable) {
+    Configuration conf = htable.getConfiguration();
+    logScannerActivity = conf.getBoolean(
+      ScannerCallable.LOG_SCANNER_ACTIVITY, false);
+    logPerRowCount = conf.getInt(LOG_PER_ROW_COUNT, 100);
     this.htable = htable;
   }
 
@@ -174,32 +194,55 @@ public boolean next(ImmutableBytesWritable key, Result value)
   throws IOException {
     Result result;
     try {
-      result = this.scanner.next();
-    } catch (DoNotRetryIOException e) {
-      throw e;
-    } catch (IOException e) {
-      LOG.debug("recovered from " + StringUtils.stringifyException(e));
-      if (lastSuccessfulRow == null) {
-        LOG.warn("We are restarting the first next() invocation," +
-            " if your mapper's restarted a few other times like this" +
-            " then you should consider killing this job and investigate" +
-            " why it's taking so long.");
-      }
-      if (lastSuccessfulRow == null) {
-        restart(startRow);
-      } else {
-        restart(lastSuccessfulRow);
-        this.scanner.next();    // skip presumed already mapped row
+      try {
+        result = this.scanner.next();
+        if (logScannerActivity) {
+          rowcount ++;
+          if (rowcount >= logPerRowCount) {
+            long now = System.currentTimeMillis();
+            LOG.info("Mapper took " + (now-timestamp)
+              + "ms to process " + rowcount + " rows");
+            timestamp = now;
+            rowcount = 0;
+          }
+        }
+      } catch (DoNotRetryIOException e) {
+        throw e;
+      } catch (IOException e) {
+        LOG.debug("recovered from " + StringUtils.stringifyException(e));
+        if (lastSuccessfulRow == null) {
+          LOG.warn("We are restarting the first next() invocation," +
+              " if your mapper has restarted a few other times like this" +
+              " then you should consider killing this job and investigate" +
+              " why it's taking so long.");
+        }
+        if (lastSuccessfulRow == null) {
+          restart(startRow);
+        } else {
+          restart(lastSuccessfulRow);
+          this.scanner.next();    // skip presumed already mapped row
+        }
+        result = this.scanner.next();
       }
-      result = this.scanner.next();
-    }
 
-    if (result != null && result.size() > 0) {
-      key.set(result.getRow());
-      lastSuccessfulRow = key.get();
-      Writables.copyWritable(result, value);
-      return true;
+      if (result != null && result.size() > 0) {
+        key.set(result.getRow());
+        lastSuccessfulRow = key.get();
+        Writables.copyWritable(result, value);
+        return true;
+      }
+      return false;
+    } catch (IOException ioe) {
+      if (logScannerActivity) {
+        long now = System.currentTimeMillis();
+        LOG.info("Mapper took " + (now-timestamp)
+          + "ms to process " + rowcount + " rows");
+        LOG.info(ioe);
+        String lastRow = lastSuccessfulRow == null ?
+          "null" : Bytes.toStringBinary(lastSuccessfulRow);
+        LOG.info("lastSuccessfulRow=" + lastRow);
+      }
+      throw ioe;
     }
-    return false;
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index a4068396312d..8c193a97825a 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -22,11 +22,13 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ScannerCallable;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -42,7 +44,8 @@
  * pairs.
  */
 public class TableRecordReaderImpl {
-
+  public static final String LOG_PER_ROW_COUNT
+    = "hbase.mapreduce.log.scanner.rowcount";
 
   static final Log LOG = LogFactory.getLog(TableRecordReader.class);
 
@@ -58,6 +61,10 @@ public class TableRecordReaderImpl {
   private Result value = null;
   private TaskAttemptContext context = null;
   private Method getCounter = null;
+  private long timestamp;
+  private int rowcount;
+  private boolean logScannerActivity = false;
+  private int logPerRowCount = 100;
 
   /**
    * Restart from survivable exceptions by creating a new scanner.
@@ -71,6 +78,11 @@ public void restart(byte[] firstRow) throws IOException {
     currentScan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE,
       Bytes.toBytes(Boolean.TRUE));
     this.scanner = this.htable.getScanner(currentScan);
+    if (logScannerActivity) {
+      LOG.info("Current scan=" + currentScan.toString());
+      timestamp = System.currentTimeMillis();
+      rowcount = 0;
+    }
   }
 
   /**
@@ -99,6 +111,10 @@ private Method retrieveGetCounterWithStringsParams(TaskAttemptContext context)
    * @param htable  The {@link HTable} to scan.
    */
   public void setHTable(HTable htable) {
+    Configuration conf = htable.getConfiguration();
+    logScannerActivity = conf.getBoolean(
+      ScannerCallable.LOG_SCANNER_ACTIVITY, false);
+    logPerRowCount = conf.getInt(LOG_PER_ROW_COUNT, 100);
     this.htable = htable;
   }
 
@@ -170,33 +186,56 @@ public boolean nextKeyValue() throws IOException, InterruptedException {
     if (key == null) key = new ImmutableBytesWritable();
     if (value == null) value = new Result();
     try {
-      value = this.scanner.next();
-    } catch (DoNotRetryIOException e) {
-      throw e;
-    } catch (IOException e) {
-      LOG.info("recovered from " + StringUtils.stringifyException(e));
-      if (lastSuccessfulRow == null) {
-        LOG.warn("We are restarting the first next() invocation," +
-            " if your mapper's restarted a few other times like this" +
-            " then you should consider killing this job and investigate" +
-            " why it's taking so long.");
+      try {
+        value = this.scanner.next();
+        if (logScannerActivity) {
+          rowcount ++;
+          if (rowcount >= logPerRowCount) {
+            long now = System.currentTimeMillis();
+            LOG.info("Mapper took " + (now-timestamp)
+              + "ms to process " + rowcount + " rows");
+            timestamp = now;
+            rowcount = 0;
+          }
+        }
+      } catch (DoNotRetryIOException e) {
+        throw e;
+      } catch (IOException e) {
+        LOG.info("recovered from " + StringUtils.stringifyException(e));
+        if (lastSuccessfulRow == null) {
+          LOG.warn("We are restarting the first next() invocation," +
+              " if your mapper has restarted a few other times like this" +
+              " then you should consider killing this job and investigate" +
+              " why it's taking so long.");
+        }
+        if (lastSuccessfulRow == null) {
+          restart(scan.getStartRow());
+        } else {
+          restart(lastSuccessfulRow);
+          scanner.next();    // skip presumed already mapped row
+        }
+        value = scanner.next();
       }
-      if (lastSuccessfulRow == null) {
-        restart(scan.getStartRow());
-      } else {
-        restart(lastSuccessfulRow);
-        scanner.next();    // skip presumed already mapped row
+      if (value != null && value.size() > 0) {
+        key.set(value.getRow());
+        lastSuccessfulRow = key.get();
+        return true;
       }
-      value = scanner.next();
-    }
-    if (value != null && value.size() > 0) {
-      key.set(value.getRow());
-      lastSuccessfulRow = key.get();
-      return true;
-    }
 
-    updateCounters();
-    return false;
+      updateCounters();
+      return false;
+    } catch (IOException ioe) {
+      if (logScannerActivity) {
+        long now = System.currentTimeMillis();
+        LOG.info("Mapper took " + (now-timestamp)
+          + "ms to process " + rowcount + " rows");
+        LOG.info(ioe);
+        String lastRow = lastSuccessfulRow == null ?
+          "null" : Bytes.toStringBinary(lastSuccessfulRow);
+        LOG.info("lastSuccessfulRow=" + lastRow);
+      }
+      throw ioe;
+    }
   }
 
   /**

From 67a28c27649e46311519ad23c2b1eba0d12bfdb4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 17 May 2012 19:33:46 +0000
Subject: [PATCH 0203/1540] Move the version on past the 0.94 release

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339815 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5d26793411f8..f988313cd7ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.0</version>
+  <version>0.94.1-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need
@@ -977,7 +977,7 @@
     <commons-math.version>2.1</commons-math.version>
     <commons-configuration.version>1.6</commons-configuration.version>
     <guava.version>r09</guava.version>
-    <jackson.version>1.5.5</jackson.version>
+    <jackson.version>1.8.8</jackson.version>
     <jasper.version>5.5.23</jasper.version>
     <jaxb-api.version>2.1</jaxb-api.version>
     <jetty.version>6.1.26</jetty.version>
@@ -1515,7 +1515,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>1.0.2</hadoop.version>
+        <hadoop.version>1.0.3</hadoop.version>
         <slf4j.version>1.4.3</slf4j.version>
       </properties>
       <dependencies>
@@ -1587,7 +1587,7 @@
     <profile>
       <id>security</id>
       <properties>
-        <hadoop.version>1.0.2</hadoop.version>
+        <hadoop.version>1.0.3</hadoop.version>
       </properties>
       <build>
         <finalName>${project.artifactId}-${project.version}-security</finalName>

From 80c9a640fc6f2c4725a6ab18f8be51ffd4e86ff0 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 May 2012 19:37:17 +0000
Subject: [PATCH 0204/1540] HBASE-6010. Security audit logger configuration for
 log4j

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339819 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase             |  7 +++++++
 bin/hbase-daemon.sh   |  1 +
 conf/log4j.properties | 16 ++++++++++++++++
 3 files changed, 24 insertions(+)

diff --git a/bin/hbase b/bin/hbase
index 1a7ad3786bbf..3661ba8f8312 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -325,6 +325,13 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
 fi
 
+# Enable security logging on the master and regionserver only
+if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then
+  HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,DRFAS}"
+else
+  HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}"
+fi
+
 # Exec unless HBASE_NOEXEC is set.
 if [ "${HBASE_NOEXEC}" != "" ]; then
   "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@"
diff --git a/bin/hbase-daemon.sh b/bin/hbase-daemon.sh
index ffae30a49dae..569bbb3a385d 100755
--- a/bin/hbase-daemon.sh
+++ b/bin/hbase-daemon.sh
@@ -117,6 +117,7 @@ JAVA=$JAVA_HOME/bin/java
 export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME
 export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log
 export HBASE_ROOT_LOGGER="INFO,DRFA"
+export HBASE_SECURITY_LOGGER="INFO,DRFAS"
 logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out  
 loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc
 loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}"
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 0010d6a129c9..f6c33051780d 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -1,5 +1,6 @@
 # Define some default values that can be overridden by system properties
 hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
 hbase.log.dir=.
 hbase.log.file=hbase.log
 
@@ -28,6 +29,21 @@ log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 # Debugging Pattern format
 #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
 
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 
+log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
 
 #
 # console

From 0f72bd7f381cfcdbdde28437dc891eed447e6c24 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 17 May 2012 20:02:25 +0000
Subject: [PATCH 0205/1540] HBASE-5802 Change the default metrics class to
 NullContextWithUpdateThread

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339832 13f79535-47bb-0310-9956-ffa450edef68
---
 conf/hadoop-metrics.properties | 17 ++++++++++-------
 src/site/xdoc/metrics.xml      |  4 ++--
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/conf/hadoop-metrics.properties b/conf/hadoop-metrics.properties
index b348e8b663cf..2060e22df5fb 100644
--- a/conf/hadoop-metrics.properties
+++ b/conf/hadoop-metrics.properties
@@ -10,12 +10,15 @@
 # GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia 
 # meta daemon (gmetad) service is running
 
-# Configuration of the "hbase" context for null
-hbase.class=org.apache.hadoop.metrics.spi.NullContext
+# Configuration of the "hbase" context for NullContextWithUpdateThread
+# NullContextWithUpdateThread is a  null context which has a thread calling
+# periodically when monitoring is started. This keeps the data sampled
+# correctly.
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=10
 
 # Configuration of the "hbase" context for file
 # hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
-# hbase.period=10
 # hbase.fileName=/tmp/metrics_hbase.log
 
 # HBase-specific configuration to reset long-running stats (e.g. compactions)
@@ -30,11 +33,11 @@ hbase.extendedperiod = 3600
 # hbase.servers=GMETADHOST_IP:8649
 
 # Configuration of the "jvm" context for null
-jvm.class=org.apache.hadoop.metrics.spi.NullContext
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=10
 
 # Configuration of the "jvm" context for file
 # jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
-# jvm.period=10
 # jvm.fileName=/tmp/metrics_jvm.log
 
 # Configuration of the "jvm" context for ganglia
@@ -45,11 +48,11 @@ jvm.class=org.apache.hadoop.metrics.spi.NullContext
 # jvm.servers=GMETADHOST_IP:8649
 
 # Configuration of the "rpc" context for null
-rpc.class=org.apache.hadoop.metrics.spi.NullContext
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=10
 
 # Configuration of the "rpc" context for file
 # rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
-# rpc.period=10
 # rpc.fileName=/tmp/metrics_rpc.log
 
 # Configuration of the "rpc" context for ganglia
diff --git a/src/site/xdoc/metrics.xml b/src/site/xdoc/metrics.xml
index f7a60a4161f7..9ff14e60d2c1 100644
--- a/src/site/xdoc/metrics.xml
+++ b/src/site/xdoc/metrics.xml
@@ -69,8 +69,8 @@
       <p>
       To enable JMX support in HBase, first edit 
       <code>$HBASE_HOME/conf/hadoop-metrics.properties</code> to support 
-      metrics refreshing. (If you've already configured 
-      <code>hadoop-metrics.properties</code> for another output context, 
+      metrics refreshing. (If you've running 0.94.1 and above, or have already configured 
+      <code>hadoop-metrics.properties</code> for another output context,
       you can skip this step).
       </p>
       <source>

From 30d14a9072172dc4db9e3fae464c5ad6b828fbad Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 17 May 2012 20:24:49 +0000
Subject: [PATCH 0206/1540] HBASE-6022 Include Junit in the libs when packaging
 so that TestAcidGaurntee can run

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339846 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f988313cd7ed..d20ae2cbf47c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1291,7 +1291,7 @@
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <version>${junit.version}</version>
-      <scope>test</scope>
+      <scope>test,runtime</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>

From 20ad08d51e9ab53b35330c98d59127485cf83240 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 17 May 2012 21:53:13 +0000
Subject: [PATCH 0207/1540] HBASE-6018 hbck fails with a
 RejectedExecutionException when >50 regions present

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339877 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  6 +++---
 .../hadoop/hbase/util/TestHBaseFsck.java      | 21 +++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index a72ba0eaa521..1d3e527f3174 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -33,7 +33,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -217,9 +217,9 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
     this.conf = conf;
 
     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
-    executor = new ThreadPoolExecutor(1, numThreads,
+    executor = new ThreadPoolExecutor(numThreads, numThreads,
         THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
-        new SynchronousQueue<Runnable>());
+        new LinkedBlockingQueue<Runnable>());
     executor.allowCoreThreadTimeOut(true);
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index e140d6503dad..80c9fe25d64d 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -359,6 +359,27 @@ public void testHBaseFsckClean() throws Exception {
     }
   }
 
+  /**
+   * Test thread pooling in the case where there are more regions than threads
+   */
+  @Test
+  public void testHbckThreadpooling() throws Exception {
+    String table = "tableDupeStartKey";
+    try {
+      // Create table with 4 regions
+      setupTable(table);
+
+      // limit number of threads to 1.
+      Configuration newconf = new Configuration(conf);
+      newconf.setInt("hbasefsck.numthreads", 1);  
+      assertNoErrors(doFsck(newconf, false));
+      
+      // We should pass without triggering a RejectedExecutionException
+    } finally {
+      deleteTable(table);
+    }    
+  }
+
   /**
    * This create and fixes a bad table with regions that have a duplicate
    * start key

From 2a04220bed8f389fda6e8058d40f09277f197d24 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 May 2012 22:38:04 +0000
Subject: [PATCH 0208/1540] HBASE-6011. Unable to start master in local mode

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339904 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/LocalHBaseCluster.java       |   5 +-
 .../hadoop/hbase/TestLocalHBaseCluster.java   | 128 ++++++++++++++++++
 2 files changed, 132 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java

diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index 888ee77e5fa4..41afb71629f1 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -67,6 +67,7 @@ public class LocalHBaseCluster {
   /** 'local:' */
   public static final String LOCAL_COLON = LOCAL + ":";
   private final Configuration conf;
+  private final Class<? extends HMaster> masterClass;
   private final Class<? extends HRegionServer> regionServerClass;
 
   /**
@@ -139,6 +140,8 @@ public LocalHBaseCluster(final Configuration conf, final int noMasters,
     // clash over default ports.
     conf.set(HConstants.MASTER_PORT, "0");
     conf.set(HConstants.REGIONSERVER_PORT, "0");
+    this.masterClass = (Class<? extends HMaster>)
+      conf.getClass(HConstants.MASTER_IMPL, masterClass);
     // Start the HMasters.
     for (int i = 0; i < noMasters; i++) {
       addMaster(new Configuration(conf), i);
@@ -192,7 +195,7 @@ public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index)
     // its HConnection instance rather than share (see HBASE_INSTANCES down in
     // the guts of HConnectionManager.
     JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c,
-        (Class<? extends HMaster>) c.getClass(HConstants.MASTER_IMPL, HMaster.class), index);
+        this.masterClass, index);
     this.masterThreads.add(mt);
     return mt;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
new file mode 100644
index 000000000000..03464f4523de
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.zookeeper.KeeperException;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestLocalHBaseCluster {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  /**
+   * Check that we can start a local HBase cluster specifying a custom master
+   * and regionserver class and then cast back to those classes; also that
+   * the cluster will launch and terminate cleanly. See HBASE-6011.
+   */
+  @Test
+  public void testLocalHBaseCluster() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    MiniZooKeeperCluster zkCluster = TEST_UTIL.startMiniZKCluster();
+    conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkCluster.getClientPort()));
+    LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1, MyHMaster.class,
+      MyHRegionServer.class);
+    // Can we cast back to our master class?
+    try {
+      ((MyHMaster)cluster.getMaster(0)).setZKCluster(zkCluster);
+    } catch (ClassCastException e) {
+      fail("Could not cast master to our class");
+    }
+    // Can we cast back to our regionserver class?
+    try {
+      ((MyHRegionServer)cluster.getRegionServer(0)).echo(42);
+    } catch (ClassCastException e) {
+      fail("Could not cast regionserver to our class");
+    }
+    // Does the cluster start successfully?
+    try {
+      cluster.startup();
+      waitForClusterUp(conf);
+    } catch (IOException e) {
+      fail("LocalHBaseCluster did not start successfully");
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  private void waitForClusterUp(Configuration conf) throws IOException {
+    HTable t = new HTable(conf, HConstants.META_TABLE_NAME);
+    ResultScanner s = t.getScanner(new Scan());
+    while (s.next() != null) {
+      continue;
+    }
+    s.close();
+    t.close();
+  }
+
+  /**
+   * A private master class similar to that used by HMasterCommandLine when
+   * running in local mode.
+   */
+  public static class MyHMaster extends HMaster {
+    private MiniZooKeeperCluster zkcluster = null;
+
+    public MyHMaster(Configuration conf) throws IOException, KeeperException,
+        InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    public void run() {
+      super.run();
+      if (this.zkcluster != null) {
+        try {
+          this.zkcluster.shutdown();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+    }
+
+    void setZKCluster(final MiniZooKeeperCluster zkcluster) {
+      this.zkcluster = zkcluster;
+    }
+  }
+
+  /**
+   * A private regionserver class with a dummy method for testing casts
+   */
+  public static class MyHRegionServer extends HRegionServer {
+
+    public MyHRegionServer(Configuration conf) throws IOException,
+        InterruptedException {
+      super(conf);
+    }
+
+    public int echo(int val) {
+      return val;
+    }
+  }
+}

From 089f53582b18edbc5c6b362ebdc45f7214b2fc7e Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 May 2012 22:55:19 +0000
Subject: [PATCH 0209/1540] HBASE-6021. NullPointerException when running
 LoadTestTool without specifying compression type

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339911 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index b70ec784d230..9fa7acd4bf9f 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -280,7 +280,7 @@ private void parseColumnFamilyOptions(CommandLine cmd) {
     }
 
     String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
-    compressAlgo = compressStr == null ? null :
+    compressAlgo = compressStr == null ? Compression.Algorithm.NONE :
         Compression.Algorithm.valueOf(compressStr);
 
     String bloomStr = cmd.getOptionValue(OPT_BLOOM);

From a7539726f59ae99f9bd151987a3119d0e4c20f8b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 17 May 2012 22:59:58 +0000
Subject: [PATCH 0210/1540] HBASE-5927 SSH and DisableTableHandler happening
 together does not clear the znode of the region and RIT map (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339915 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |   6 +-
 .../master/handler/ServerShutdownHandler.java |  16 +-
 .../hbase/master/TestAssignmentManager.java   | 140 +++++++++++++-----
 3 files changed, 125 insertions(+), 37 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 66bb0d9d072b..ec3420b59bf1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2077,7 +2077,11 @@ public void unassign(HRegionInfo region, boolean force) {
     }
   }
   
-  private void deleteClosingOrClosedNode(HRegionInfo region) {
+  /**
+   * 
+   * @param region regioninfo of znode to be deleted.
+   */
+  public void deleteClosingOrClosedNode(HRegionInfo region) {
     try {
       if (!ZKAssign.deleteNode(master.getZooKeeper(), region.getEncodedName(),
           EventHandler.EventType.M_ZK_REGION_CLOSING)) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 943e60c4ee2f..a8cba9942d1f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -284,10 +284,10 @@ public void process() throws IOException {
       // Iterate regions that were on this server and assign them
       if (hris != null) {
         for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
+          RegionState rit = this.services.getAssignmentManager().isRegionInTransition(e.getKey());
           if (processDeadRegion(e.getKey(), e.getValue(),
               this.services.getAssignmentManager(),
               this.server.getCatalogTracker())) {
-            RegionState rit = this.services.getAssignmentManager().isRegionInTransition(e.getKey());
             ServerName addressFromAM = this.services.getAssignmentManager()
                 .getRegionServerOfRegion(e.getKey());
             if (rit != null && !rit.isClosing() && !rit.isPendingClose()) {
@@ -304,6 +304,20 @@ public void process() throws IOException {
                 this.services.getAssignmentManager().assign(e.getKey(), true);
               }
           }
+          // If the table was partially disabled and the RS went down, we should clear the RIT
+          // and remove the node for the region.
+          // The rit that we use may be stale in case the table was in DISABLING state
+          // but though we did assign we will not be clearing the znode in CLOSING state.
+          // Doing this will have no harm. See HBASE-5927
+          if (rit != null
+              && (rit.isClosing() || rit.isPendingClose())
+              && this.services.getAssignmentManager().getZKTable()
+                  .isDisablingOrDisabledTable(rit.getRegion().getTableNameAsString())) {
+            HRegionInfo hri = rit.getRegion();
+            AssignmentManager am = this.services.getAssignmentManager();
+            am.deleteClosingOrClosedNode(hri);
+            am.regionOffline(hri);
+          }
         }
       }
     } finally {
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 2215ceb12321..9c4a3f780d21 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -47,6 +48,8 @@
 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -56,6 +59,7 @@
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.junit.After;
@@ -379,42 +383,68 @@ public void testShutdownHandler() throws KeeperException, IOException {
     AssignmentManager am =
       new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
     try {
-      // Make sure our new AM gets callbacks; once registered, can't unregister.
-      // Thats ok because we make a new zk watcher for each test.
-      this.watcher.registerListenerFirst(am);
+      processServerShutdownHandler(ct, am);
+    } finally {
+      executor.shutdown();
+      am.shutdown();
+      // Clean up all znodes
+      ZKAssign.deleteAllNodes(this.watcher);
+    }
+  }
+
+  /**
+   * To test closed region handler to remove rit and delete corresponding znode if region in pending
+   * close or closing while processing shutdown of a region server.(HBASE-5927). 
+   * @throws KeeperException
+   * @throws IOException
+   */
+  @Test
+  public void testSSHWhenDisableTableInProgress()
+      throws KeeperException, IOException {
+    testCaseWithPartiallyDisabledState(TableState.DISABLING);
+    testCaseWithPartiallyDisabledState(TableState.DISABLED);
+  }
+
+  private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperException, IOException, NodeExistsException {
+    // Create and startup an executor. This is used by AssignmentManager
+    // handling zk callbacks.
+    ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
 
-      // Need to set up a fake scan of meta for the servershutdown handler
-      // Make an RS Interface implementation.  Make it so a scanner can go against it.
-      HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
-      // Get a meta row result that has region up on SERVERNAME_A
-      Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
-      Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
-        thenReturn(System.currentTimeMillis());
-      // Return a good result first and then return null to indicate end of scan
-      Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
-        thenReturn(new Result [] {r}, (Result [])null);
-
-      // Get a connection w/ mocked up common methods.
-      HConnection connection =
-        HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
-          implementation, SERVERNAME_B, REGIONINFO);
-
-      // Make it so we can get a catalogtracker from servermanager.. .needed
-      // down in guts of server shutdown handler.
-      Mockito.when(ct.getConnection()).thenReturn(connection);
-      Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
-
-      // Now make a server shutdown handler instance and invoke process.
-      // Have it that SERVERNAME_A died.
-      DeadServer deadServers = new DeadServer();
-      deadServers.add(SERVERNAME_A);
-      // I need a services instance that will return the AM
-      MasterServices services = Mockito.mock(MasterServices.class);
-      Mockito.when(services.getAssignmentManager()).thenReturn(am);
-      ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
-        services, deadServers, SERVERNAME_A, false);
-      handler.process();
-      // The region in r will have been assigned.  It'll be up in zk as unassigned.
+    // We need a mocked catalog tracker.
+    CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
+    // Create an AM.
+    AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
+        executor);
+    // adding region to regions and servers maps.
+    am.regionOnline(REGIONINFO, SERVERNAME_A);
+    // adding region in pending close.
+    am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
+        State.PENDING_CLOSE));
+
+    if (state == TableState.DISABLING) {
+      am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
+    } else {
+      am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
+    }
+    
+    RegionTransitionData data = new RegionTransitionData(EventType.M_ZK_REGION_CLOSING,
+        REGIONINFO.getRegionName(), SERVERNAME_A);
+    String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
+    // create znode in M_ZK_REGION_CLOSING state.
+    ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
+
+    try {
+      processServerShutdownHandler(ct, am);
+      // check znode deleted or not.
+      // In both cases the znode should be deleted.
+      assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
+      // check whether in rit or not.  In the DISABLING case also the below assert will be true
+      // but the piece of code added for HBASE-5927 will not do that.
+      if (state == TableState.DISABLED) {
+        assertTrue("Region state of region in pending close should be removed from rit.",
+            am.regionsInTransition.isEmpty());
+      }
     } finally {
       executor.shutdown();
       am.shutdown();
@@ -423,6 +453,46 @@ public void testShutdownHandler() throws KeeperException, IOException {
     }
   }
 
+  private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am)
+      throws IOException {
+    // Make sure our new AM gets callbacks; once registered, can't unregister.
+    // Thats ok because we make a new zk watcher for each test.
+    this.watcher.registerListenerFirst(am);
+
+    // Need to set up a fake scan of meta for the servershutdown handler
+    // Make an RS Interface implementation.  Make it so a scanner can go against it.
+    HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
+    // Get a meta row result that has region up on SERVERNAME_A
+    Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+    Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
+      thenReturn(System.currentTimeMillis());
+    // Return a good result first and then return null to indicate end of scan
+    Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
+      thenReturn(new Result [] {r}, (Result [])null);
+
+    // Get a connection w/ mocked up common methods.
+    HConnection connection =
+      HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
+        implementation, SERVERNAME_B, REGIONINFO);
+
+    // Make it so we can get a catalogtracker from servermanager.. .needed
+    // down in guts of server shutdown handler.
+    Mockito.when(ct.getConnection()).thenReturn(connection);
+    Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
+
+    // Now make a server shutdown handler instance and invoke process.
+    // Have it that SERVERNAME_A died.
+    DeadServer deadServers = new DeadServer();
+    deadServers.add(SERVERNAME_A);
+    // I need a services instance that will return the AM
+    MasterServices services = Mockito.mock(MasterServices.class);
+    Mockito.when(services.getAssignmentManager()).thenReturn(am);
+    ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
+      services, deadServers, SERVERNAME_A, false);
+    handler.process();
+    // The region in r will have been assigned.  It'll be up in zk as unassigned.
+  }
+
   /**
    * @param sn ServerName to use making startcode and server in meta
    * @param hri Region to serialize into HRegionInfo

From d98944312f9999ced6980767cbda5ffb8df3eb79 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 May 2012 23:01:47 +0000
Subject: [PATCH 0211/1540] HBASE-6023. Normalize security audit logging level
 with Hadoop

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339916 13f79535-47bb-0310-9956-ffa450edef68
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index e6f248d5ad24..c72281987f83 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -339,7 +339,7 @@ public Object run() throws SaslException {
           ticket = getAuthorizedUgi(saslServer.getAuthorizationID());
           LOG.debug("SASL server successfully authenticated client: " + ticket);
           rpcMetrics.authenticationSuccesses.inc();
-          AUDITLOG.trace(AUTH_SUCCESSFUL_FOR + ticket);
+          AUDITLOG.info(AUTH_SUCCESSFUL_FOR + ticket);
           saslContextEstablished = true;
         }
       } else {

From 1e71f845fac0812e942c56b1b49da85d34a6cda5 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 17 May 2012 23:36:49 +0000
Subject: [PATCH 0212/1540] HBASE-5342 Grant/Revoke global permissions (Matteo)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339922 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessControlLists.java   | 158 +++++++++++++-----
 .../security/access/AccessController.java     | 130 ++++++++------
 .../access/AccessControllerProtocol.java      |  27 ++-
 .../security/access/TableAuthManager.java     |  52 ++++--
 .../hbase/security/access/UserPermission.java |  29 ++++
 .../security/access/ZKPermissionWatcher.java  |  16 +-
 .../access/TestAccessControlFilter.java       |  10 +-
 .../security/access/TestAccessController.java |  95 +++++++----
 .../security/access/TestTablePermissions.java |  85 +++++++---
 src/main/ruby/hbase/security.rb               | 121 ++++++++++----
 src/main/ruby/shell.rb                        |   1 +
 src/main/ruby/shell/commands/grant.rb         |   5 +-
 src/main/ruby/shell/commands/revoke.rb        |   8 +-
 .../ruby/shell/commands/user_permission.rb    |   5 +-
 src/main/ruby/shell/commands/whoami.rb        |  37 ++++
 15 files changed, 547 insertions(+), 232 deletions(-)
 create mode 100644 src/main/ruby/shell/commands/whoami.rb

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
index fa001895b3ea..8a100e4fe5df 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
@@ -41,6 +41,9 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.RegexStringComparator;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.io.Text;
@@ -79,6 +82,7 @@ public class AccessControlLists {
   /** Internal storage table for access control lists */
   public static final String ACL_TABLE_NAME_STR = "_acl_";
   public static final byte[] ACL_TABLE_NAME = Bytes.toBytes(ACL_TABLE_NAME_STR);
+  public static final byte[] ACL_GLOBAL_NAME = ACL_TABLE_NAME;
   /** Column family used to store ACL grants */
   public static final String ACL_LIST_FAMILY_STR = "l";
   public static final byte[] ACL_LIST_FAMILY = Bytes.toBytes(ACL_LIST_FAMILY_STR);
@@ -117,31 +121,20 @@ static void init(MasterServices master) throws IOException {
   }
 
   /**
-   * Stores a new table permission grant in the access control lists table.
+   * Stores a new user permission grant in the access control lists table.
    * @param conf the configuration
-   * @param tableName the table to which access is being granted
-   * @param username the user or group being granted the permission
-   * @param perm the details of the permission being granted
+   * @param userPerm the details of the permission to be granted
    * @throws IOException in the case of an error accessing the metadata table
    */
-  static void addTablePermission(Configuration conf,
-      byte[] tableName, String username, TablePermission perm)
-    throws IOException {
+  static void addUserPermission(Configuration conf, UserPermission userPerm)
+      throws IOException {
+    Permission.Action[] actions = userPerm.getActions();
 
-    Put p = new Put(tableName);
-    byte[] key = Bytes.toBytes(username);
-    if (perm.getFamily() != null && perm.getFamily().length > 0) {
-      key = Bytes.add(key,
-          Bytes.add(new byte[]{ACL_KEY_DELIMITER}, perm.getFamily()));
-      if (perm.getQualifier() != null && perm.getQualifier().length > 0) {
-        key = Bytes.add(key,
-            Bytes.add(new byte[]{ACL_KEY_DELIMITER}, perm.getQualifier()));
-      }
-    }
+    Put p = new Put(userPerm.isGlobal() ? ACL_GLOBAL_NAME : userPerm.getTable());
+    byte[] key = userPermissionKey(userPerm);
 
-    TablePermission.Action[] actions = perm.getActions();
     if ((actions == null) || (actions.length == 0)) {
-      LOG.warn("No actions associated with user '"+username+"'");
+      LOG.warn("No actions associated with user '"+Bytes.toString(userPerm.getUser())+"'");
       return;
     }
 
@@ -152,7 +145,7 @@ static void addTablePermission(Configuration conf,
     p.add(ACL_LIST_FAMILY, key, value);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Writing permission for table "+
-          Bytes.toString(tableName)+" "+
+          Bytes.toString(userPerm.getTable())+" "+
           Bytes.toString(key)+": "+Bytes.toStringBinary(value)
       );
     }
@@ -175,34 +168,17 @@ static void addTablePermission(Configuration conf,
    * column qualifier "info:colA") will have no effect.
    *
    * @param conf the configuration
-   * @param tableName the table of the current permission grant
-   * @param userName the user or group currently granted the permission
-   * @param perm the details of the permission to be revoked
+   * @param userPerm the details of the permission to be revoked
    * @throws IOException if there is an error accessing the metadata table
    */
-  static void removeTablePermission(Configuration conf,
-      byte[] tableName, String userName, TablePermission perm)
-    throws IOException {
+  static void removeUserPermission(Configuration conf, UserPermission userPerm)
+      throws IOException {
+
+    Delete d = new Delete(userPerm.isGlobal() ? ACL_GLOBAL_NAME : userPerm.getTable());
+    byte[] key = userPermissionKey(userPerm);
 
-    Delete d = new Delete(tableName);
-    byte[] key = null;
-    if (perm.getFamily() != null && perm.getFamily().length > 0) {
-      key = Bytes.toBytes(userName + ACL_KEY_DELIMITER +
-          Bytes.toString(perm.getFamily()));
-      if (perm.getQualifier() != null && perm.getQualifier().length > 0) {
-       key = Bytes.toBytes(userName + ACL_KEY_DELIMITER +
-          Bytes.toString(perm.getFamily()) + ACL_KEY_DELIMITER +
-          Bytes.toString(perm.getQualifier()));
-      } else {
-        key = Bytes.toBytes(userName + ACL_KEY_DELIMITER +
-          Bytes.toString(perm.getFamily()));
-      }
-    } else {
-      key = Bytes.toBytes(userName);
-    }
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Removing permission for user '" + userName+ "': "+
-          perm.toString());
+      LOG.debug("Removing permission "+ userPerm.toString());
     }
     d.deleteColumns(ACL_LIST_FAMILY, key);
     HTable acls = null;
@@ -214,6 +190,95 @@ static void removeTablePermission(Configuration conf,
     }
   }
 
+  /**
+   * Remove specified table from the _acl_ table.
+   */
+  static void removeTablePermissions(Configuration conf, byte[] tableName)
+      throws IOException{
+    Delete d = new Delete(tableName);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Removing permissions of removed table "+ Bytes.toString(tableName));
+    }
+
+    HTable acls = null;
+    try {
+      acls = new HTable(conf, ACL_TABLE_NAME);
+      acls.delete(d);
+    } finally {
+      if (acls != null) acls.close();
+    }
+  }
+
+  /**
+   * Remove specified table column from the _acl_ table.
+   */
+  static void removeTablePermissions(Configuration conf, byte[] tableName, byte[] column)
+      throws IOException{
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Removing permissions of removed column " + Bytes.toString(column) +
+                " from table "+ Bytes.toString(tableName));
+    }
+
+    HTable acls = null;
+    try {
+      acls = new HTable(conf, ACL_TABLE_NAME);
+
+      Scan scan = new Scan();
+      scan.addFamily(ACL_LIST_FAMILY);
+
+      String columnName = Bytes.toString(column);
+      scan.setFilter(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(
+                     String.format("(%s%s%s)|(%s%s)$",
+                     ACL_KEY_DELIMITER, columnName, ACL_KEY_DELIMITER,
+                     ACL_KEY_DELIMITER, columnName))));
+
+      Set<byte[]> qualifierSet = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+      ResultScanner scanner = acls.getScanner(scan);
+      try {
+        for (Result res : scanner) {
+          for (byte[] q : res.getFamilyMap(ACL_LIST_FAMILY).navigableKeySet()) {
+            qualifierSet.add(q);
+          }
+        }
+      } finally {
+        scanner.close();
+      }
+
+      if (qualifierSet.size() > 0) {
+        Delete d = new Delete(tableName);
+        for (byte[] qualifier : qualifierSet) {
+          d.deleteColumns(ACL_LIST_FAMILY, qualifier);
+        }
+        acls.delete(d);
+      }
+    } finally {
+      if (acls != null) acls.close();
+    }
+  }
+
+  /**
+   * Build qualifier key from user permission:
+   *  username
+   *  username,family
+   *  username,family,qualifier
+   */
+  static byte[] userPermissionKey(UserPermission userPerm) {
+    byte[] qualifier = userPerm.getQualifier();
+    byte[] family = userPerm.getFamily();
+    byte[] key = userPerm.getUser();
+
+    if (family != null && family.length > 0) {
+      key = Bytes.add(key, Bytes.add(new byte[]{ACL_KEY_DELIMITER}, family));
+      if (qualifier != null && qualifier.length > 0) {
+        key = Bytes.add(key, Bytes.add(new byte[]{ACL_KEY_DELIMITER}, qualifier));
+      }
+    }
+
+    return key;
+  }
+
   /**
    * Returns {@code true} if the given region is part of the {@code _acl_}
    * metadata table.
@@ -328,12 +393,13 @@ static Map<byte[],ListMultimap<String,TablePermission>> loadAll(
   static ListMultimap<String,TablePermission> getTablePermissions(
       Configuration conf, byte[] tableName)
   throws IOException {
+    if (tableName == null) tableName = ACL_TABLE_NAME;
+
     /* TODO: -ROOT- and .META. cannot easily be handled because they must be
      * online before _acl_ table.  Can anything be done here?
      */
     if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
-        Bytes.equals(tableName, HConstants.META_TABLE_NAME) ||
-        Bytes.equals(tableName, AccessControlLists.ACL_TABLE_NAME)) {
+        Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
       return ArrayListMultimap.create(0,0);
     }
 
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index e47437bc6df3..260b0f1e51ec 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -18,13 +18,14 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
+import java.util.TreeSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -188,12 +189,10 @@ void initialize(RegionCoprocessorEnvironment e) throws IOException {
     for (Map.Entry<byte[],ListMultimap<String,TablePermission>> t:
       tables.entrySet()) {
       byte[] table = t.getKey();
-      String tableName = Bytes.toString(table);
       ListMultimap<String,TablePermission> perms = t.getValue();
       byte[] serialized = AccessControlLists.writePermissionsAsBytes(perms,
-          e.getRegion().getConf());
-      this.authManager.getZKPermissionWatcher().writeToZookeeper(tableName,
-        serialized);
+          regionEnv.getConfiguration());
+      this.authManager.getZKPermissionWatcher().writeToZookeeper(table, serialized);
     }
   }
 
@@ -204,31 +203,28 @@ void initialize(RegionCoprocessorEnvironment e) throws IOException {
    */
   void updateACL(RegionCoprocessorEnvironment e,
       final Map<byte[], List<KeyValue>> familyMap) {
-    Set<String> tableSet = new HashSet<String>();
+    Set<byte[]> tableSet = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
     for (Map.Entry<byte[], List<KeyValue>> f : familyMap.entrySet()) {
       List<KeyValue> kvs = f.getValue();
       for (KeyValue kv: kvs) {
-        if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(),
+        if (Bytes.equals(kv.getBuffer(), kv.getFamilyOffset(),
             kv.getFamilyLength(), AccessControlLists.ACL_LIST_FAMILY, 0,
-            AccessControlLists.ACL_LIST_FAMILY.length) == 0) {
-          String tableName = Bytes.toString(kv.getRow());
-          tableSet.add(tableName);
+            AccessControlLists.ACL_LIST_FAMILY.length)) {
+          tableSet.add(kv.getRow());
         }
       }
     }
 
-    for (String tableName: tableSet) {
+    ZKPermissionWatcher zkw = this.authManager.getZKPermissionWatcher();
+    Configuration conf = regionEnv.getConfiguration();
+    for (byte[] tableName: tableSet) {
       try {
         ListMultimap<String,TablePermission> perms =
-          AccessControlLists.getTablePermissions(regionEnv.getConfiguration(),
-              Bytes.toBytes(tableName));
-        byte[] serialized = AccessControlLists.writePermissionsAsBytes(
-            perms, e.getRegion().getConf());
-        this.authManager.getZKPermissionWatcher().writeToZookeeper(tableName,
-          serialized);
+          AccessControlLists.getTablePermissions(conf, tableName);
+        byte[] serialized = AccessControlLists.writePermissionsAsBytes(perms, conf);
+        zkw.writeToZookeeper(tableName, serialized);
       } catch (IOException ex) {
-        LOG.error("Failed updating permissions mirror for '" + tableName +
-          "'", ex);
+        LOG.error("Failed updating permissions mirror for '" + tableName + "'", ex);
       }
     }
   }
@@ -256,29 +252,40 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
 
     // 1. All users need read access to .META. and -ROOT- tables.
     // this is a very common operation, so deal with it quickly.
-    if ((hri.isRootRegion() || hri.isMetaRegion()) &&
-        (permRequest == TablePermission.Action.READ)) {
-      return AuthResult.allow("All users allowed", user, permRequest,
-          hri.getTableName());
+    if (hri.isRootRegion() || hri.isMetaRegion()) {
+      if (permRequest == TablePermission.Action.READ) {
+        return AuthResult.allow("All users allowed", user, permRequest, tableName);
+      }
     }
 
     if (user == null) {
-      return AuthResult.deny("No user associated with request!", null,
-          permRequest, hri.getTableName());
+      return AuthResult.deny("No user associated with request!", null, permRequest, tableName);
+    }
+
+    // Users with CREATE/ADMIN rights need to modify .META. and _acl_ table
+    // e.g. When a new table is created a new entry in .META. is added,
+    // so the user need to be allowed to write on it.
+    // e.g. When a table is removed an entry is removed from .META. and _acl_
+    // and the user need to be allowed to write on both tables.
+    if (permRequest == TablePermission.Action.WRITE &&
+       (hri.isRootRegion() || hri.isMetaRegion() ||
+        Bytes.equals(tableName, AccessControlLists.ACL_GLOBAL_NAME)) &&
+       (authManager.authorize(user, Permission.Action.CREATE) ||
+        authManager.authorize(user, Permission.Action.ADMIN)))
+    {
+       return AuthResult.allow("Table permission granted", user, permRequest, tableName);
     }
 
     // 2. The table owner has full privileges
     String owner = htd.getOwnerString();
     if (user.getShortName().equals(owner)) {
       // owner of the table has full access
-      return AuthResult.allow("User is table owner", user, permRequest,
-          hri.getTableName());
+      return AuthResult.allow("User is table owner", user, permRequest, tableName);
     }
 
     // 3. check for the table-level, if successful we can short-circuit
     if (authManager.authorize(user, tableName, (byte[])null, permRequest)) {
-      return AuthResult.allow("Table permission granted", user,
-          permRequest, tableName);
+      return AuthResult.allow("Table permission granted", user, permRequest, tableName);
     }
 
     // 4. check permissions against the requested families
@@ -350,6 +357,7 @@ private User getActiveUser() throws IOException {
       // for non-rpc handling, fallback to system user
       user = User.getCurrent();
     }
+
     return user;
   }
 
@@ -513,8 +521,9 @@ public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
   }
   @Override
   public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName) throws IOException {}
-
+      byte[] tableName) throws IOException {
+    AccessControlLists.removeTablePermissions(c.getEnvironment().getConfiguration(), tableName);
+  }
 
   @Override
   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -535,7 +544,6 @@ public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor column) throws IOException {}
 
-
   @Override
   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor descriptor) throws IOException {
@@ -553,8 +561,10 @@ public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   }
   @Override
   public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, byte[] col) throws IOException {}
-
+      byte[] tableName, byte[] col) throws IOException {
+    AccessControlLists.removeTablePermissions(c.getEnvironment().getConfiguration(),
+                                              tableName, col);
+  }
 
   @Override
   public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -664,7 +674,7 @@ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
     try {
       this.authManager = TableAuthManager.get(
           e.getRegionServerServices().getZooKeeper(),
-          e.getRegion().getConf());
+          regionEnv.getConfiguration());
     } catch (IOException ioe) {
       // pass along as a RuntimeException, so that the coprocessor is unloaded
       throw new RuntimeException("Error obtaining TableAuthManager", ioe);
@@ -892,10 +902,10 @@ public void postScannerClose(final ObserverContext<RegionCoprocessorEnvironment>
   private void requireScannerOwner(InternalScanner s)
       throws AccessDeniedException {
     if (RequestContext.isInRequestContext()) {
+      String requestUserName = RequestContext.getRequestUserName();
       String owner = scannerOwners.get(s);
-      if (owner != null && !owner.equals(RequestContext.getRequestUserName())) {
-        throw new AccessDeniedException("User '"+
-            RequestContext.getRequestUserName()+"' is not the scanner owner!");
+      if (owner != null && !owner.equals(requestUserName)) {
+        throw new AccessDeniedException("User '"+ requestUserName +"' is not the scanner owner!");
       }
     }
   }
@@ -906,24 +916,20 @@ private void requireScannerOwner(InternalScanner s)
    * This will be restricted by both client side and endpoint implementations.
    */
   @Override
-  public void grant(byte[] user, TablePermission permission)
+  public void grant(UserPermission userPermission)
       throws IOException {
     // verify it's only running at .acl.
     if (aclRegion) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Received request to grant access permission to '"
-            + Bytes.toString(user) + "'. "
-            + permission.toString());
+        LOG.debug("Received request to grant access permission " + userPermission.toString());
       }
 
       requirePermission(Permission.Action.ADMIN);
 
-      AccessControlLists.addTablePermission(regionEnv.getConfiguration(),
-          permission.getTable(), Bytes.toString(user), permission);
+      AccessControlLists.addUserPermission(regionEnv.getConfiguration(), userPermission);
       if (AUDITLOG.isTraceEnabled()) {
         // audit log should store permission changes in addition to auth results
-        AUDITLOG.trace("Granted user '" + Bytes.toString(user) + "' permission "
-            + permission.toString());
+        AUDITLOG.trace("Granted permission " + userPermission.toString());
       }
     } else {
       throw new CoprocessorException(AccessController.class, "This method " +
@@ -933,24 +939,29 @@ public void grant(byte[] user, TablePermission permission)
   }
 
   @Override
-  public void revoke(byte[] user, TablePermission permission)
+  @Deprecated
+  public void grant(byte[] user, TablePermission permission)
+      throws IOException {
+    grant(new UserPermission(user, permission.getTable(),
+            permission.getFamily(), permission.getQualifier(),
+            permission.getActions()));
+  }
+
+  @Override
+  public void revoke(UserPermission userPermission)
       throws IOException{
     // only allowed to be called on _acl_ region
     if (aclRegion) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Received request to revoke access permission for '"
-            + Bytes.toString(user) + "'. "
-            + permission.toString());
+        LOG.debug("Received request to revoke access permission " + userPermission.toString());
       }
 
       requirePermission(Permission.Action.ADMIN);
 
-      AccessControlLists.removeTablePermission(regionEnv.getConfiguration(),
-          permission.getTable(), Bytes.toString(user), permission);
+      AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), userPermission);
       if (AUDITLOG.isTraceEnabled()) {
         // audit log should record all permission changes
-        AUDITLOG.trace("Revoked user '" + Bytes.toString(user) + "' permission "
-            + permission.toString());
+        AUDITLOG.trace("Revoked permission " + userPermission.toString());
       }
     } else {
       throw new CoprocessorException(AccessController.class, "This method " +
@@ -959,6 +970,15 @@ public void revoke(byte[] user, TablePermission permission)
     }
   }
 
+  @Override
+  @Deprecated
+  public void revoke(byte[] user, TablePermission permission)
+      throws IOException {
+    revoke(new UserPermission(user, permission.getTable(),
+            permission.getFamily(), permission.getQualifier(),
+            permission.getActions()));
+  }
+
   @Override
   public List<UserPermission> getUserPermissions(final byte[] tableName)
       throws IOException {
@@ -1038,7 +1058,7 @@ private byte[] getTableName(RegionCoprocessorEnvironment e) {
     return tableName;
   }
 
-  private String getTableOwner(MasterCoprocessorEnvironment e, 
+  private String getTableOwner(MasterCoprocessorEnvironment e,
       byte[] tableName) throws IOException {
     HTableDescriptor htd = e.getTable(tableName).getTableDescriptor();
     return htd.getOwnerString();
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java
index c11f339653a9..2ecb60a94bff 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java
@@ -30,6 +30,14 @@ public interface AccessControllerProtocol extends CoprocessorProtocol {
 
   public static final long VERSION = 1L;
 
+  /**
+   * Grants the given user or group the privilege to perform the given actions
+   * @param userPermission the details of the provided user permissions
+   * @throws IOException if the grant could not be applied
+   */
+  public void grant(UserPermission userPermission)
+      throws IOException;
+
   /**
    * Grants the given user or group the privilege to perform the given actions
    * over the specified scope contained in {@link TablePermission}
@@ -37,10 +45,26 @@ public interface AccessControllerProtocol extends CoprocessorProtocol {
    * the grant
    * @param permission the details of the provided permissions
    * @throws IOException if the grant could not be applied
+   * @deprecated Use {@link #revoke(UserPermission userPermission)} instead
    */
+  @Deprecated
   public void grant(byte[] user, TablePermission permission)
       throws IOException;
 
+  /**
+   * Revokes a previously granted privilege from a user or group.
+   * Note that the provided {@link TablePermission} details must exactly match
+   * a stored grant.  For example, if user "bob" has been granted "READ" access
+   * to table "data", over column family and qualifer "info:colA", then the
+   * table, column family and column qualifier must all be specified.
+   * Attempting to revoke permissions over just the "data" table will have
+   * no effect.
+   * @param permission the details of the previously granted permission to revoke
+   * @throws IOException if the revocation could not be performed
+   */
+  public void revoke(UserPermission userPermission)
+      throws IOException;
+
   /**
    * Revokes a previously granted privilege from a user or group.
    * Note that the provided {@link TablePermission} details must exactly match
@@ -53,7 +77,9 @@ public void grant(byte[] user, TablePermission permission)
    * privileges are being revoked
    * @param permission the details of the previously granted permission to revoke
    * @throws IOException if the revocation could not be performed
+   * @deprecated Use {@link #revoke(UserPermission userPermission)} instead
    */
+  @Deprecated
   public void revoke(byte[] user, TablePermission permission)
       throws IOException;
 
@@ -81,5 +107,4 @@ public List<UserPermission> getUserPermissions(byte[] tableName)
    */
   public void checkPermissions(Permission[] permissions)
       throws IOException;
-
 }
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
index 2c3870f932e6..970317bb3ec1 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
@@ -40,8 +40,6 @@
  * Performs authorization checks for a given user's assigned permissions
  */
 public class TableAuthManager {
-  /** Key for the user and group cache maps for globally assigned permissions */
-  private static final String GLOBAL_CACHE_KEY = ".access.";
   private static Log LOG = LogFactory.getLog(TableAuthManager.class);
 
   private static TableAuthManager instance;
@@ -103,14 +101,36 @@ public ZKPermissionWatcher getZKPermissionWatcher() {
 
   public void refreshCacheFromWritable(byte[] table, byte[] data) throws IOException {
     if (data != null && data.length > 0) {
-      DataInput in = new DataInputStream( new ByteArrayInputStream(data) );
+      DataInput in = new DataInputStream(new ByteArrayInputStream(data));
       ListMultimap<String,TablePermission> perms = AccessControlLists.readPermissions(in, conf);
-      cache(table, perms);
+      if (perms != null) {
+        if (Bytes.equals(table, AccessControlLists.ACL_GLOBAL_NAME)) {
+          updateGlobalCache(perms);
+        } else {
+          updateTableCache(table, perms);
+        }
+      }
     } else {
       LOG.debug("Skipping permission cache refresh because writable data is empty");
     }
   }
 
+  /**
+   * Updates the internal global permissions cache
+   *
+   * @param userPerms
+   */
+  private void updateGlobalCache(ListMultimap<String,TablePermission> userPerms) {
+    for (Map.Entry<String,TablePermission> entry : userPerms.entries()) {
+      if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
+        GROUP_CACHE.put(AccessControlLists.getGroupName(entry.getKey()),
+                        new Permission(entry.getValue().getActions()));
+      } else {
+        USER_CACHE.put(entry.getKey(), new Permission(entry.getValue().getActions()));
+      }
+    }
+  }
+
   /**
    * Updates the internal permissions cache for a single table, splitting
    * the permissions listed into separate caches for users and groups to optimize
@@ -119,26 +139,22 @@ public void refreshCacheFromWritable(byte[] table, byte[] data) throws IOExcepti
    * @param table
    * @param tablePerms
    */
-  private void cache(byte[] table,
-      ListMultimap<String,TablePermission> tablePerms) {
+  private void updateTableCache(byte[] table, ListMultimap<String,TablePermission> tablePerms) {
     // split user from group assignments so we don't have to prepend the group
     // prefix every time we query for groups
     ListMultimap<String,TablePermission> userPerms = ArrayListMultimap.create();
     ListMultimap<String,TablePermission> groupPerms = ArrayListMultimap.create();
 
-    if (tablePerms != null) {
-      for (Map.Entry<String,TablePermission> entry : tablePerms.entries()) {
-        if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
-          groupPerms.put(
-              entry.getKey().substring(AccessControlLists.GROUP_PREFIX.length()),
-              entry.getValue());
-        } else {
-          userPerms.put(entry.getKey(), entry.getValue());
-        }
+    for (Map.Entry<String,TablePermission> entry : tablePerms.entries()) {
+      if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
+        groupPerms.put(AccessControlLists.getGroupName(entry.getKey()), entry.getValue());
+      } else {
+        userPerms.put(entry.getKey(), entry.getValue());
       }
-      TABLE_GROUP_CACHE.put(table, groupPerms);
-      TABLE_USER_CACHE.put(table, userPerms);
     }
+
+    TABLE_GROUP_CACHE.put(table, groupPerms);
+    TABLE_USER_CACHE.put(table, userPerms);
   }
 
   private List<TablePermission> getUserPermissions(String username, byte[] table) {
@@ -464,7 +480,7 @@ public void writeToZooKeeper(byte[] table,
       }
     }
     byte[] serialized = AccessControlLists.writePermissionsAsBytes(tmp, conf);
-    zkperms.writeToZookeeper(Bytes.toString(table), serialized);
+    zkperms.writeToZookeeper(table, serialized);
   }
 
   static Map<ZooKeeperWatcher,TableAuthManager> managerMap =
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java
index 8a5c467ab4ad..fd5b755f7d79 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java
@@ -40,6 +40,27 @@ public UserPermission() {
     super();
   }
 
+  /**
+   * Creates a new instance for the given user.
+   * @param user the user
+   * @param assigned the list of allowed actions
+   */
+  public UserPermission(byte[] user, Action... assigned) {
+    super(null, null, null, assigned);
+    this.user = user;
+  }
+
+  /**
+   * Creates a new instance for the given user,
+   * matching the actions with the given codes.
+   * @param user the user
+   * @param actionCodes the list of allowed action codes
+   */
+  public UserPermission(byte[] user, byte[] actionCodes) {
+    super(null, null, null, actionCodes);
+    this.user = user;
+  }
+
   /**
    * Creates a new instance for the given user, table and column family.
    * @param user the user
@@ -92,6 +113,14 @@ public byte[] getUser() {
     return user;
   }
 
+  /**
+   * Returns true if this permission describes a global user permission.
+   */
+  public boolean isGlobal() {
+    byte[] tableName = getTable();
+    return(tableName == null || tableName.length == 0);
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (!(obj instanceof UserPermission)) {
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
index f7e8654abe04..4870bb19651f 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
@@ -146,18 +146,16 @@ private void refreshNodes(List<ZKUtil.NodeAndData> nodes) {
    * @param tableName
    * @param permsData
    */
-  public void writeToZookeeper(String tableName, 
-      byte[] permsData) {
-    String zkNode =
-        ZKUtil.joinZNode(ZKUtil.joinZNode(watcher.baseZNode, ACL_NODE),
-          tableName);
+  public void writeToZookeeper(byte[] tableName, byte[] parmsData) {    
+    String zkNode = ZKUtil.joinZNode(watcher.baseZNode, ACL_NODE);
+    zkNode = ZKUtil.joinZNode(zkNode, Bytes.toString(tableName));
+
     try {
       ZKUtil.createWithParents(watcher, zkNode);
-      ZKUtil.updateExistingNodeData(watcher, zkNode,
-        permsData, -1);
+      ZKUtil.updateExistingNodeData(watcher, zkNode, parmsData, -1);
     } catch (KeeperException e) {
-      LOG.error("Failed updating permissions for table '" + tableName +
-          "'", e);
+      LOG.error("Failed updating permissions for table '" + 
+                Bytes.toString(tableName) + "'", e);
       watcher.abort("Failed writing node "+zkNode+" to zookeeper", e);
     }
   }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
index 0a2cad27c73b..bd423f18a8cb 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
@@ -95,10 +95,12 @@ public Object run() throws Exception {
             AccessControlLists.ACL_TABLE_NAME);
         AccessControllerProtocol acls = aclmeta.coprocessorProxy(
             AccessControllerProtocol.class, Bytes.toBytes("testtable"));
-        TablePermission perm = new TablePermission(TABLE, null, Permission.Action.READ);
-        acls.grant(Bytes.toBytes(READER.getShortName()), perm);
-        perm = new TablePermission(TABLE, FAMILY, PUBLIC_COL, Permission.Action.READ);
-        acls.grant(Bytes.toBytes(LIMITED.getShortName()), perm);
+        UserPermission perm = new UserPermission(Bytes.toBytes(READER.getShortName()), 
+                                                 TABLE, null, Permission.Action.READ);
+        acls.grant(perm);
+        perm = new UserPermission(Bytes.toBytes(LIMITED.getShortName()), 
+                                  TABLE, FAMILY, PUBLIC_COL, Permission.Action.READ);
+        acls.grant(perm);
         return null;
       }
     });
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index ec678b9254fe..5d6351197997 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -72,6 +72,8 @@ public class TestAccessController {
 
   // user with all permissions
   private static User SUPERUSER;
+  // user granted with all global permission
+  private static User USER_ADMIN;
   // table owner user
   private static User USER_OWNER;
   // user with rw permissions
@@ -104,6 +106,7 @@ public static void setupBeforeClass() throws Exception {
 
     // create a set of test users
     SUPERUSER = User.createUserForTesting(conf, "admin", new String[]{"supergroup"});
+    USER_ADMIN = User.createUserForTesting(conf, "admin2", new String[0]);
     USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
     USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
     USER_RO = User.createUserForTesting(conf, "rouser", new String[0]);
@@ -119,12 +122,16 @@ public static void setupBeforeClass() throws Exception {
     HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
     AccessControllerProtocol protocol =
         meta.coprocessorProxy(AccessControllerProtocol.class, TEST_TABLE);
-    protocol.grant(Bytes.toBytes(USER_RW.getShortName()),
-        new TablePermission(TEST_TABLE, TEST_FAMILY, Permission.Action.READ,
-            Permission.Action.WRITE));
 
-    protocol.grant(Bytes.toBytes(USER_RO.getShortName()),
-        new TablePermission(TEST_TABLE, TEST_FAMILY, Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()),
+                      Permission.Action.ADMIN, Permission.Action.CREATE,
+                      Permission.Action.READ, Permission.Action.WRITE));
+
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_RW.getShortName()),
+        TEST_TABLE, TEST_FAMILY, Permission.Action.READ, Permission.Action.WRITE));
+
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()),
+                   TEST_TABLE, TEST_FAMILY, Permission.Action.READ));
   }
 
   @AfterClass
@@ -195,6 +202,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, createTable);
+    verifyAllowed(USER_ADMIN, createTable);
 
     // all others should be denied
     verifyDenied(USER_OWNER, createTable);
@@ -223,6 +231,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, modifyTable);
+    verifyAllowed(USER_ADMIN, modifyTable);
   }
 
   @Test
@@ -242,6 +251,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, deleteTable);
+    verifyAllowed(USER_ADMIN, deleteTable);
   }
 
   @Test
@@ -262,6 +272,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -283,6 +294,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -302,6 +314,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -321,6 +334,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, disableTable);
+    verifyAllowed(USER_ADMIN, disableTable);
   }
 
   @Test
@@ -340,6 +354,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, enableTable);
+    verifyAllowed(USER_ADMIN, enableTable);
   }
 
   @Test
@@ -365,6 +380,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -390,6 +406,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -415,6 +432,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -434,6 +452,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -453,6 +472,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -472,6 +492,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   @Test
@@ -491,6 +512,7 @@ public Object run() throws Exception {
 
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
   }
 
   private void verifyWrite(PrivilegedExceptionAction action) throws Exception {
@@ -500,6 +522,7 @@ private void verifyWrite(PrivilegedExceptionAction action) throws Exception {
 
     // should be allowed
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
     verifyAllowed(USER_OWNER, action);
     verifyAllowed(USER_RW, action);
   }
@@ -510,6 +533,7 @@ private void verifyRead(PrivilegedExceptionAction action) throws Exception {
 
     // should be allowed
     verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
     verifyAllowed(USER_OWNER, action);
     verifyAllowed(USER_RW, action);
     verifyAllowed(USER_RO, action);
@@ -720,8 +744,8 @@ public Object run() throws Exception {
     verifyDenied(user, deleteAction2);
 
     // grant table read permission
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-      new TablePermission(tableName, null, Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, null, Permission.Action.READ));
     Thread.sleep(100);
     // check
     verifyAllowed(user, getActionAll);
@@ -737,8 +761,8 @@ public Object run() throws Exception {
     verifyDenied(user, deleteAction2);
 
     // grant table write permission
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-      new TablePermission(tableName, null, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, null, Permission.Action.WRITE));
     Thread.sleep(100);
     verifyDenied(user, getActionAll);
     verifyDenied(user, getAction1);
@@ -753,12 +777,11 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteAction2);
 
     // revoke table permission
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-      new TablePermission(tableName, null, Permission.Action.READ,
-        Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, null, Permission.Action.READ, Permission.Action.WRITE));
 
-    protocol.revoke(Bytes.toBytes(user.getShortName()),
-        new TablePermission(tableName, null));
+    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
+                    tableName, null));
     Thread.sleep(100);
     verifyDenied(user, getActionAll);
     verifyDenied(user, getAction1);
@@ -773,8 +796,8 @@ public Object run() throws Exception {
     verifyDenied(user, deleteAction2);
 
     // grant column family read permission
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-      new TablePermission(tableName, family1, Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, family1, Permission.Action.READ));
     Thread.sleep(100);
 
     verifyAllowed(user, getActionAll);
@@ -790,8 +813,8 @@ public Object run() throws Exception {
     verifyDenied(user, deleteAction2);
 
     // grant column family write permission
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-      new TablePermission(tableName, family2, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, family2, Permission.Action.WRITE));
     Thread.sleep(100);
 
     verifyAllowed(user, getActionAll);
@@ -807,8 +830,8 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteAction2);
 
     // revoke column family permission
-    protocol.revoke(Bytes.toBytes(user.getShortName()),
-      new TablePermission(tableName, family2));
+    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
+                    tableName, family2));
     Thread.sleep(100);
 
     verifyAllowed(user, getActionAll);
@@ -890,15 +913,14 @@ public Object run() throws Exception {
       }
     };
 
-    protocol.revoke(Bytes.toBytes(user.getShortName()),
-        new TablePermission(tableName, family1));
+    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
+                    tableName, family1));
     verifyDenied(user, getQualifierAction);
     verifyDenied(user, putQualifierAction);
     verifyDenied(user, deleteQualifierAction);
 
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-        new TablePermission(tableName, family1, qualifier,
-            Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, family1, qualifier, Permission.Action.READ));
     Thread.sleep(100);
 
     verifyAllowed(user, getQualifierAction);
@@ -907,9 +929,8 @@ public Object run() throws Exception {
 
     // only grant write permission
     // TODO: comment this portion after HBASE-3583
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-        new TablePermission(tableName, family1, qualifier,
-            Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, family1, qualifier, Permission.Action.WRITE));
     Thread.sleep(100);
 
     verifyDenied(user, getQualifierAction);
@@ -917,9 +938,9 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteQualifierAction);
 
     // grant both read and write permission.
-    protocol.grant(Bytes.toBytes(user.getShortName()),
-        new TablePermission(tableName, family1, qualifier,
-            Permission.Action.READ, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
+                   tableName, family1, qualifier,
+                   Permission.Action.READ, Permission.Action.WRITE));
     Thread.sleep(100);
 
     verifyAllowed(user, getQualifierAction);
@@ -927,8 +948,8 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteQualifierAction);
 
     // revoke family level permission won't impact column level.
-    protocol.revoke(Bytes.toBytes(user.getShortName()),
-        new TablePermission(tableName, family1, qualifier));
+    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
+                    tableName, family1, qualifier));
     Thread.sleep(100);
 
     verifyDenied(user, getQualifierAction);
@@ -974,7 +995,7 @@ public void testPermissionList() throws Exception {
     // grant read permission
     UserPermission upToSet = new UserPermission(user,
         tableName, family1, qualifier, Permission.Action.READ);
-    protocol.grant(user, upToSet);
+    protocol.grant(upToSet);
     perms = protocol.getUserPermissions(tableName);
 
     UserPermission upToVerify = new UserPermission(user,
@@ -990,7 +1011,7 @@ public void testPermissionList() throws Exception {
     // grant read+write
     upToSet = new UserPermission(user, tableName, family1, qualifier,
         Permission.Action.WRITE, Permission.Action.READ);
-    protocol.grant(user, upToSet);
+    protocol.grant(upToSet);
     perms = protocol.getUserPermissions(tableName);
 
     upToVerify = new UserPermission(user, tableName, family1, qualifier,
@@ -998,7 +1019,7 @@ public void testPermissionList() throws Exception {
     assertTrue("User should be granted permission: " + upToVerify.toString(),
             hasFoundUserPermission(upToVerify, perms));
 
-    protocol.revoke(user, upToSet);
+    protocol.revoke(upToSet);
     perms = protocol.getUserPermissions(tableName);
     assertFalse("User should not be granted permission: " + upToVerify.toString(),
       hasFoundUserPermission(upToVerify, perms));
@@ -1053,7 +1074,7 @@ public void checkTablePerms(byte[] table, Permission...perms) throws IOException
 
   public void grant(AccessControllerProtocol protocol, User user, byte[] t, byte[] f,
       byte[] q, Permission.Action... actions) throws IOException {
-    protocol.grant(Bytes.toBytes(user.getShortName()), new TablePermission(t, f, q, actions));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), t, f, q, actions));
   }
 
   @Test
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
index 39fc73e78985..0203dbadd59f 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
@@ -103,15 +103,16 @@ public static void afterClass() throws Exception {
   public void testBasicWrite() throws Exception {
     Configuration conf = UTIL.getConfiguration();
     // add some permissions
-    AccessControlLists.addTablePermission(conf, TEST_TABLE,
-        "george", new TablePermission(TEST_TABLE, null,
-            TablePermission.Action.READ, TablePermission.Action.WRITE));
-    AccessControlLists.addTablePermission(conf, TEST_TABLE,
-        "hubert", new TablePermission(TEST_TABLE, null,
-            TablePermission.Action.READ));
-    AccessControlLists.addTablePermission(conf, TEST_TABLE,
-        "humphrey", new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER,
-            TablePermission.Action.READ));
+    AccessControlLists.addUserPermission(conf,
+            new UserPermission(Bytes.toBytes("george"), TEST_TABLE, null, (byte[])null,
+            UserPermission.Action.READ, UserPermission.Action.WRITE));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("hubert"), TEST_TABLE, null, (byte[])null,
+            UserPermission.Action.READ));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("humphrey"),
+            TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER,
+            UserPermission.Action.READ));
 
     // retrieve the same
     ListMultimap<String,TablePermission> perms =
@@ -165,8 +166,8 @@ public void testBasicWrite() throws Exception {
     assertFalse(actions.contains(TablePermission.Action.WRITE));
 
     // table 2 permissions
-    AccessControlLists.addTablePermission(conf, TEST_TABLE2, "hubert",
-        new TablePermission(TEST_TABLE2, null,
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("hubert"), TEST_TABLE2, null, (byte[])null,
             TablePermission.Action.READ, TablePermission.Action.WRITE));
 
     // check full load
@@ -197,16 +198,21 @@ public void testBasicWrite() throws Exception {
   @Test
   public void testPersistence() throws Exception {
     Configuration conf = UTIL.getConfiguration();
-    AccessControlLists.addTablePermission(conf, TEST_TABLE, "albert",
-        new TablePermission(TEST_TABLE, null, TablePermission.Action.READ));
-    AccessControlLists.addTablePermission(conf, TEST_TABLE, "betty",
-        new TablePermission(TEST_TABLE, null, TablePermission.Action.READ,
-            TablePermission.Action.WRITE));
-    AccessControlLists.addTablePermission(conf, TEST_TABLE, "clark",
-        new TablePermission(TEST_TABLE, TEST_FAMILY, TablePermission.Action.READ));
-    AccessControlLists.addTablePermission(conf, TEST_TABLE, "dwight",
-        new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER,
-            TablePermission.Action.WRITE));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("albert"), TEST_TABLE, null,
+                           (byte[])null, TablePermission.Action.READ));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("betty"), TEST_TABLE, null,
+                           (byte[])null, TablePermission.Action.READ,
+                           TablePermission.Action.WRITE));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("clark"),
+                           TEST_TABLE, TEST_FAMILY,
+                           TablePermission.Action.READ));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("dwight"),
+                           TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER,
+                           TablePermission.Action.WRITE));
 
     // verify permissions survive changes in table metadata
     ListMultimap<String,TablePermission> preperms =
@@ -313,4 +319,41 @@ public void testEquals() throws Exception {
     assertFalse(p1.equals(p2));
     assertFalse(p2.equals(p1));
   }
+
+  @Test
+  public void testGlobalPermission() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+
+    // add some permissions
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("user1"),
+            Permission.Action.READ, Permission.Action.WRITE));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("user2"),
+            Permission.Action.CREATE));
+    AccessControlLists.addUserPermission(conf,
+        new UserPermission(Bytes.toBytes("user3"),
+            Permission.Action.ADMIN, Permission.Action.READ, Permission.Action.CREATE));
+
+    ListMultimap<String,TablePermission> perms = AccessControlLists.getTablePermissions(conf, null);
+    List<TablePermission> user1Perms = perms.get("user1");
+    assertEquals("Should have 1 permission for user1", 1, user1Perms.size());
+    assertEquals("user1 should have WRITE permission",
+                 new Permission.Action[] { Permission.Action.READ, Permission.Action.WRITE },
+                 user1Perms.get(0).getActions());
+
+    List<TablePermission> user2Perms = perms.get("user2");
+    assertEquals("Should have 1 permission for user2", 1, user2Perms.size());
+    assertEquals("user2 should have CREATE permission",
+                 new Permission.Action[] { Permission.Action.CREATE },
+                 user2Perms.get(0).getActions());
+
+    List<TablePermission> user3Perms = perms.get("user3");
+    assertEquals("Should have 1 permission for user3", 1, user3Perms.size());
+    assertEquals("user3 should have ADMIN, READ, CREATE permission",
+                 new Permission.Action[] {
+                    Permission.Action.ADMIN, Permission.Action.READ, Permission.Action.CREATE
+                 },
+                 user3Perms.get(0).getActions());
+  }
 }
diff --git a/src/main/ruby/hbase/security.rb b/src/main/ruby/hbase/security.rb
index 54965ae41cb8..e3b99038ba54 100644
--- a/src/main/ruby/hbase/security.rb
+++ b/src/main/ruby/hbase/security.rb
@@ -31,62 +31,117 @@ def initialize(configuration, formatter)
     end
 
     #----------------------------------------------------------------------------------------------
-    def grant(user, permissions, table_name, family=nil, qualifier=nil)
+    def grant(user, permissions, table_name=nil, family=nil, qualifier=nil)
       security_available?
 
-      # Table should exist
-      raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name)
+      # TODO: need to validate user name
 
-      htd = @admin.getTableDescriptor(table_name.to_java_bytes)
+      if (table_name != nil)
+        # Table should exist
+        raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name)
 
-      if (family != nil)
-        raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes)
-      end
+        htd = @admin.getTableDescriptor(table_name.to_java_bytes)
+
+        if (family != nil)
+          raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes)
+        end
 
-      #TODO: need to validate user name
+        # invoke cp endpoint to perform access controlse
+        fambytes = family.to_java_bytes if (family != nil)
+        qualbytes = qualifier.to_java_bytes if (qualifier != nil)
+        user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new(
+                                                user.to_java_bytes, table_name.to_java_bytes, 
+                                                fambytes, qualbytes, permissions.to_java_bytes)
+      else
+        user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new(
+                                                user.to_java_bytes, permissions.to_java_bytes)
+      end
 
-      # invoke cp endpoint to perform access control
-      fambytes = family.to_java_bytes if (family != nil)
-      qualbytes = qualifier.to_java_bytes if (qualifier != nil)
-      tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, permissions.to_java_bytes)
-      meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
-      protocol = meta_table.coprocessorProxy(org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class,
+      meta_table = org.apache.hadoop.hbase.client.HTable.new(@config,
+                      org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
+      protocol = meta_table.coprocessorProxy(
+                      org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class,
                                              org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW)
-      protocol.grant(user.to_java_bytes, tp)
+      begin
+        protocol.grant(user_permission)
+      rescue java.io.IOException => e
+        if !(e.message.include? "java.lang.NoSuchMethodException")
+          raise e
+        end
+
+        # Server has not the new API, try the old one
+        if (table_name == nil)
+          raise "Global permissions not supported by HBase Server"
+        end
+
+        tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, permissions.to_java_bytes)
+        protocol.grant(user.to_java_bytes, tp)
+      end
     end
 
     #----------------------------------------------------------------------------------------------
-    def revoke(user, table_name, family=nil, qualifier=nil)
+    def revoke(user, table_name=nil, family=nil, qualifier=nil)
       security_available?
 
-      # Table should exist
-      raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name)
+      # TODO: need to validate user name
+
+      if (table_name != nil)
+        # Table should exist
+        raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name)
 
-      htd = @admin.getTableDescriptor(table_name.to_java_bytes)
+        htd = @admin.getTableDescriptor(table_name.to_java_bytes)
 
-      if (family != nil)
-        raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes)
+        if (family != nil)
+          raise(ArgumentError, "Can't find family: #{family}") unless htd.hasFamily(family.to_java_bytes)
+        end
+
+        # invoke cp endpoint to perform access control
+        fambytes = family.to_java_bytes if (family != nil)
+        qualbytes = qualifier.to_java_bytes if (qualifier != nil)
+        user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new(
+                                                    user.to_java_bytes, table_name.to_java_bytes,
+                                                    fambytes, qualbytes, "".to_java_bytes)
+      else
+        user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new(
+                                                    user.to_java_bytes, "".to_java_bytes)
       end
 
-      fambytes = family.to_java_bytes if (family != nil)
-      qualbytes = qualifier.to_java_bytes if (qualifier != nil)
-      tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, "".to_java_bytes)
-      meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
-      protocol = meta_table.coprocessorProxy(org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class,
+      meta_table = org.apache.hadoop.hbase.client.HTable.new(@config,
+                        org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
+      protocol = meta_table.coprocessorProxy(
+                        org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class,
                                              org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW)
-      protocol.revoke(user.to_java_bytes, tp)
+      begin
+        protocol.revoke(user_permission)
+      rescue java.io.IOException => e
+        if !(e.message.include? "java.lang.NoSuchMethodException")
+          raise e
+        end
+
+        # Server has not the new API, try the old one
+        if (table_name == nil)
+          raise "Global permissions not supported by HBase Server"
+        end
+
+        tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, "".to_java_bytes)
+        protocol.revoke(user.to_java_bytes, tp)
+      end
     end
 
     #----------------------------------------------------------------------------------------------
-    def user_permission(table_name)
+    def user_permission(table_name=nil)
       security_available?
 
-      raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name)
+      if (table_name != nil)
+        raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name)
+      end
 
-      meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
-      protocol = meta_table.coprocessorProxy(org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class,
-                                             org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW)
-      perms = protocol.getUserPermissions(table_name.to_java_bytes)
+      meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, 
+                        org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
+      protocol = meta_table.coprocessorProxy(
+                      org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class,
+                      org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW)
+      perms = protocol.getUserPermissions(table_name != nil ? table_name.to_java_bytes : nil)
 
       res = {}
       count  = 0
diff --git a/src/main/ruby/shell.rb b/src/main/ruby/shell.rb
index 53f3de878524..f0d96e6fcc4e 100644
--- a/src/main/ruby/shell.rb
+++ b/src/main/ruby/shell.rb
@@ -211,6 +211,7 @@ def help_footer
   :commands => %w[
     status
     version
+    whoami
   ]
 )
 
diff --git a/src/main/ruby/shell/commands/grant.rb b/src/main/ruby/shell/commands/grant.rb
index 7504f4349bd8..a4e9330126ce 100644
--- a/src/main/ruby/shell/commands/grant.rb
+++ b/src/main/ruby/shell/commands/grant.rb
@@ -21,7 +21,7 @@ module Commands
     class Grant < Command
       def help
         return <<-EOF
-Grant users specific rights to tables.
+Grant users specific rights.
 Syntax : grant <user> <permissions> <table> <column family> <column qualifier>
 
 permissions is either zero or more letters from the set "RWXCA".
@@ -29,11 +29,12 @@ def help
 
 For example:
 
+    hbase> grant 'bobsmith', 'RWXCA'
     hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1'
 EOF
       end
 
-      def command(user, rights, table_name, family=nil, qualifier=nil)
+      def command(user, rights, table_name=nil, family=nil, qualifier=nil)
         format_simple_command do
           security_admin.grant(user, rights, table_name, family, qualifier)
         end
diff --git a/src/main/ruby/shell/commands/revoke.rb b/src/main/ruby/shell/commands/revoke.rb
index e94c3644f772..fe9391c80582 100644
--- a/src/main/ruby/shell/commands/revoke.rb
+++ b/src/main/ruby/shell/commands/revoke.rb
@@ -21,15 +21,15 @@ module Commands
     class Revoke < Command
       def help
         return <<-EOF
-Revoke a user's access rights to tables.
-Syntax : revoke <user> <table> <column family>
+Revoke a user's access rights.
+Syntax : revoke <user> <table> <column family> <column qualifier>
 For example:
 
-    hbase> revoke 'bobsmith', 't1', 'f1'
+    hbase> revoke 'bobsmith', 't1', 'f1', 'col1'
 EOF
       end
 
-      def command(user, table_name, family=nil, qualifier=nil)
+      def command(user, table_name=nil, family=nil, qualifier=nil)
         format_simple_command do
           security_admin.revoke(user, table_name, family, qualifier)
         end
diff --git a/src/main/ruby/shell/commands/user_permission.rb b/src/main/ruby/shell/commands/user_permission.rb
index a4da0824fabb..ad4a7b029e77 100644
--- a/src/main/ruby/shell/commands/user_permission.rb
+++ b/src/main/ruby/shell/commands/user_permission.rb
@@ -21,15 +21,16 @@ module Commands
     class UserPermission < Command
       def help
         return <<-EOF
-Show all table access permissions for the particular user.
+Show all permissions for the particular user.
 Syntax : user_permission <table>
 For example:
 
+    hbase> user_permission
     hbase> user_permission 'table1'
 EOF
       end
 
-      def command(table)
+      def command(table=nil)
         #format_simple_command do
         #admin.user_permission(table)
         now = Time.now
diff --git a/src/main/ruby/shell/commands/whoami.rb b/src/main/ruby/shell/commands/whoami.rb
new file mode 100644
index 000000000000..040ad7e7ef7e
--- /dev/null
+++ b/src/main/ruby/shell/commands/whoami.rb
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class Whoami < Command
+      def help
+        return <<-EOF
+Show the current hbase user.
+Syntax : whoami
+For example:
+
+    hbase> whoami
+EOF
+      end
+
+      def command()
+        puts "#{org.apache.hadoop.hbase.security.User.getCurrent().toString()}"
+      end
+    end
+  end
+end

From ae86a8139cc2a664861a49e419970243c2c3dce6 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 18 May 2012 00:01:21 +0000
Subject: [PATCH 0213/1540] BASE-6013 Polish sharp edges from CopyTable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1339930 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/CopyTable.java  | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 01f6d152c9dc..03947e92d8d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -120,9 +120,9 @@ private static void printUsage(final String errorMsg) {
     System.err.println(" rs.class     hbase.regionserver.class of the peer cluster");
     System.err.println("              specify if different from current cluster");
     System.err.println(" rs.impl      hbase.regionserver.impl of the peer cluster");
-    System.err.println(" starttime    beginning of the time range");
+    System.err.println(" starttime    beginning of the time range (unixtime in millis)");
     System.err.println("              without endtime means from starttime to forever");
-    System.err.println(" endtime      end of the time range");
+    System.err.println(" endtime      end of the time range.  Ignored if no starttime specified.");
     System.err.println(" versions     number of cell versions to copy");
     System.err.println(" new.name     new table's name");
     System.err.println(" peer.adr     Address of the peer cluster given in the format");
@@ -216,6 +216,9 @@ private static boolean doCommandLine(final String[] args) {
 
         if (i == args.length-1) {
           tableName = cmd;
+        } else {
+          printUsage("Invalid argument '" + cmd + "'" );
+          return false;
         }
       }
       if (newTableName == null && peerAddress == null) {
@@ -223,6 +226,10 @@ private static boolean doCommandLine(final String[] args) {
             "peer address must be specified");
         return false;
       }
+      if (startTime > endTime) {
+        printUsage("Invalid time range filter: starttime=" + startTime + " >  endtime=" + endTime);
+        return false;
+      }
     } catch (Exception e) {
       e.printStackTrace();
       printUsage("Can't start because " + e.getMessage());

From 0e4c51dbc58ffdd64545f183023e43521f4fa0ac Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 18 May 2012 16:24:47 +0000
Subject: [PATCH 0214/1540] HBASE-6011. Addendum to support master mocking
 (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1340156 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index 41afb71629f1..fe4897a685d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -195,7 +195,7 @@ public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index)
     // its HConnection instance rather than share (see HBASE_INSTANCES down in
     // the guts of HConnectionManager.
     JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c,
-        this.masterClass, index);
+        (Class<? extends HMaster>) conf.getClass(HConstants.MASTER_IMPL, this.masterClass), index);
     this.masterThreads.add(mt);
     return mt;
   }

From c285fb6b86383d4c383a85d07828abf094e4ca2b Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Fri, 18 May 2012 17:52:58 +0000
Subject: [PATCH 0215/1540] HBASE-5546 Master assigns region in the original
 region server when opening region failed (Ashutosh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1340190 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  3 +
 .../hbase/master/TestAssignmentManager.java   | 75 ++++++++++++++++++-
 2 files changed, 76 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index ec3420b59bf1..652d0c3b5214 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -825,6 +825,9 @@ private void handleRegion(final RegionTransitionData data, int expectedVersion)
           // Handle this the same as if it were opened and then closed.
           regionState.update(RegionState.State.CLOSED,
               data.getStamp(), data.getOrigin());
+          // When there are more than one region server a new RS is selected as the 
+          // destination and the same is updated in the regionplan. (HBASE-5546)
+          getRegionPlan(regionState, sn, true);
           this.executorService.submit(new ClosedRegionHandler(master,
             this, regionState.getRegion()));
           break;
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 9c4a3f780d21..bb100dbf2344 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -70,6 +70,8 @@
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
+import com.google.protobuf.ServiceException;
+
 
 /**
  * Test {@link AssignmentManager}
@@ -89,6 +91,7 @@ public class TestAssignmentManager {
   private Server server;
   private ServerManager serverManager;
   private ZooKeeperWatcher watcher;
+  private LoadBalancer balancer;
 
   @BeforeClass
   public static void beforeClass() throws Exception {
@@ -660,12 +663,80 @@ private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final
     Mockito.when(ct.getConnection()).thenReturn(connection);
     // Create and startup an executor. Used by AM handling zk callbacks.
     ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
-    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
-        .getConfiguration());
+    this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
     AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
         server, manager, ct, balancer, executor);
     return am;
   }
+  
+  /**
+   * TestCase verifies that the regionPlan is updated whenever a region fails to open 
+   * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546).
+   */
+  @Test
+  public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
+      ServiceException, InterruptedException {
+    this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        MockedLoadBalancer.class, LoadBalancer.class);
+    AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
+        this.serverManager);
+    // Boolean variable used for waiting until randomAssignment is called and new
+    // plan is generated.
+    AtomicBoolean gate = new AtomicBoolean(false);
+    if (balancer instanceof MockedLoadBalancer) {
+      ((MockedLoadBalancer) balancer).setGateVariable(gate);
+    }
+    ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
+    int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
+    ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
+        EventType.RS_ZK_REGION_FAILED_OPEN, v);
+    String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
+    RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
+        SERVERNAME_A);
+    am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
+    // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed
+    am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
+    RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
+    List<ServerName> serverList = new ArrayList<ServerName>(2);
+    serverList.add(SERVERNAME_B);
+    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
+    am.nodeDataChanged(path);
+    // here we are waiting until the random assignment in the load balancer is called.
+    while (!gate.get()) {
+      Thread.sleep(10);
+    }
+    // new region plan may take some time to get updated after random assignment is called and 
+    // gate is set to true.
+    RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
+    while (newRegionPlan == null) {
+      Thread.sleep(10);
+      newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
+    }
+    // the new region plan created may contain the same RS as destination but it should
+    // be new plan.
+    assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
+    assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
+        newRegionPlan.getDestination())));
+  }
+  
+  /**
+   * Mocked load balancer class used in the testcase to make sure that the testcase waits until
+   * random assignment is called and the gate variable is set to true.
+   */
+  public static class MockedLoadBalancer extends DefaultLoadBalancer {
+    private AtomicBoolean gate;
+
+    public void setGateVariable(AtomicBoolean gate) {
+      this.gate = gate;
+    }
+
+    @Override
+    public ServerName randomAssignment(List<ServerName> servers) {
+      ServerName randomServerName = super.randomAssignment(servers);
+      this.gate.set(true);
+      return randomServerName;
+    }
+  }
 
   /**
    * An {@link AssignmentManager} with some extra facility used testing

From deaf7c440cb59784c0195653d4321707dd1fc56c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 18 May 2012 22:17:09 +0000
Subject: [PATCH 0216/1540] HBASE-5920 New Compactions Logic can silently
 prevent user-initiated compactions from occurring

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1340283 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/CompactSplitThread.java      |  21 ++--
 .../hbase/regionserver/HRegionServer.java     |   4 +-
 .../hadoop/hbase/regionserver/Store.java      | 101 +++++++++++++-----
 .../hbase/regionserver/TestCompaction.java    |  39 +++++++
 4 files changed, 127 insertions(+), 38 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 26e72528c8e3..15ba981ecabd 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -49,12 +49,6 @@ public class CompactSplitThread implements CompactionRequestor {
   private final ThreadPoolExecutor splits;
   private final long throttleSize;
 
-  /* The default priority for user-specified compaction requests.
-   * The user gets top priority unless we have blocking compactions. (Pri <= 0)
-   */
-  public static final int PRIORITY_USER = 1;
-  public static final int NO_PRIORITY = Integer.MIN_VALUE;
-
   /**
    * Splitting should not take place if the total number of regions exceed this.
    * This is not a hard limit to the number of regions but it is a guideline to
@@ -145,7 +139,7 @@ public String toString() {
 
   public synchronized boolean requestSplit(final HRegion r) {
     // don't split regions that are blocking
-    if (shouldSplitRegion() && r.getCompactPriority() >= PRIORITY_USER) {
+    if (shouldSplitRegion() && r.getCompactPriority() >= Store.PRIORITY_USER) {
       byte[] midKey = r.checkSplit();
       if (midKey != null) {
         requestSplit(r, midKey);
@@ -174,13 +168,13 @@ public synchronized void requestSplit(final HRegion r, byte[] midKey) {
   public synchronized void requestCompaction(final HRegion r,
       final String why) {
     for(Store s : r.getStores().values()) {
-      requestCompaction(r, s, why, NO_PRIORITY);
+      requestCompaction(r, s, why, Store.NO_PRIORITY);
     }
   }
 
   public synchronized void requestCompaction(final HRegion r, final Store s,
       final String why) {
-    requestCompaction(r, s, why, NO_PRIORITY);
+    requestCompaction(r, s, why, Store.NO_PRIORITY);
   }
 
   public synchronized void requestCompaction(final HRegion r, final String why,
@@ -201,10 +195,10 @@ public synchronized void requestCompaction(final HRegion r, final Store s,
     if (this.server.isStopped()) {
       return;
     }
-    CompactionRequest cr = s.requestCompaction();
+    CompactionRequest cr = s.requestCompaction(priority);
     if (cr != null) {
       cr.setServer(server);
-      if (priority != NO_PRIORITY) {
+      if (priority != Store.NO_PRIORITY) {
         cr.setPriority(priority);
       }
       ThreadPoolExecutor pool = largeCompactions;
@@ -222,6 +216,11 @@ public synchronized void requestCompaction(final HRegion r, final Store s,
             + (why != null && !why.isEmpty() ? "; Because: " + why : "")
             + "; " + this);
       }
+    } else {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Not compacting " + r.getRegionNameAsString() + 
+            " because compaction request was cancelled");
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 12a1ce6a288d..ff3d635a62ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2915,9 +2915,11 @@ public void compactRegion(HRegionInfo regionInfo, boolean major)
     if (major) {
       region.triggerMajorCompaction();
     }
+    LOG.trace("User-triggered compaction requested for region " +
+      region.getRegionNameAsString());
     compactSplitThread.requestCompaction(region, "User-triggered "
         + (major ? "major " : "") + "compaction",
-        CompactSplitThread.PRIORITY_USER);
+        Store.PRIORITY_USER);
   }
 
   /** @return the info server */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index bd2a4c0eb9c2..6f32235285ba 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -107,6 +107,7 @@
  */
 public class Store extends SchemaConfigured implements HeapSize {
   static final Log LOG = LogFactory.getLog(Store.class);
+
   protected final MemStore memstore;
   // This stores directory in the filesystem.
   private final Path homedir;
@@ -135,6 +136,12 @@ public class Store extends SchemaConfigured implements HeapSize {
   private final int compactionKVMax;
   private final boolean verifyBulkLoads;
 
+  /* The default priority for user-specified compaction requests.
+   * The user gets top priority unless we have blocking compactions. (Pri <= 0)
+   */
+  public static final int PRIORITY_USER = 1;
+  public static final int NO_PRIORITY = Integer.MIN_VALUE;
+
   // not private for testing
   /* package */ScanInfo scanInfo;
   /*
@@ -170,7 +177,7 @@ public class Store extends SchemaConfigured implements HeapSize {
    * @param region
    * @param family HColumnDescriptor for this column
    * @param fs file system object
-   * @param conf configuration object
+   * @param confParam configuration object
    * failed.  Can be null.
    * @throws IOException
    */
@@ -317,7 +324,7 @@ public static Path getStoreHomedir(final Path tabledir,
   public Path getHomedir() {
     return homedir;
   }
-  
+
   /**
    * @return the data block encoder
    */
@@ -435,7 +442,7 @@ protected long delete(final KeyValue kv) {
 
   /**
    * Removes a kv from the memstore. The KeyValue is removed only
-   * if its key & memstoreTS matches the key & memstoreTS value of the 
+   * if its key & memstoreTS matches the key & memstoreTS value of the
    * kv parameter.
    *
    * @param kv
@@ -521,8 +528,8 @@ void assertBulkLoadHFileOk(Path srcPath) throws IOException {
   }
 
   /**
-   * This method should only be called from HRegion.  It is assumed that the 
-   * ranges of values in the HFile fit within the stores assigned region. 
+   * This method should only be called from HRegion.  It is assumed that the
+   * ranges of values in the HFile fit within the stores assigned region.
    * (assertBulkLoadHFileOk checks this)
    */
   void bulkLoadHFile(String srcPathStr) throws IOException {
@@ -602,7 +609,7 @@ ImmutableList<StoreFile> close() throws IOException {
         ThreadPoolExecutor storeFileCloserThreadPool = this.region
             .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
                 + this.family.getNameAsString());
-  
+
         // close each store file in parallel
         CompletionService<Void> completionService =
           new ExecutorCompletionService<Void>(storeFileCloserThreadPool);
@@ -614,7 +621,7 @@ public Void call() throws IOException {
             }
           });
         }
-  
+
         try {
           for (int i = 0; i < result.size(); i++) {
             Future<Void> future = completionService.take();
@@ -743,7 +750,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
       scanner.close();
     }
     if (LOG.isInfoEnabled()) {
-      LOG.info("Flushed " + 
+      LOG.info("Flushed " +
                ", sequenceid=" + logCacheFlushId +
                ", memsize=" + StringUtils.humanReadableInt(flushed) +
                ", into tmp file " + pathName);
@@ -954,7 +961,7 @@ void deleteChangedReaderObserver(ChangedReadersObserver o) {
    * <p>We don't want to hold the structureLock for the whole time, as a compact()
    * can be lengthy and we want to allow cache-flushes during this period.
    *
-   * @param CompactionRequest
+   * @param cr
    *          compaction details obtained from requestCompaction()
    * @throws IOException
    */
@@ -1187,7 +1194,7 @@ long getNextMajorCompactTime() {
       if (jitterPct > 0) {
         long jitter = Math.round(ret * jitterPct);
         // deterministic jitter avoids a major compaction storm on restart
-        ImmutableList<StoreFile> snapshot = storefiles; 
+        ImmutableList<StoreFile> snapshot = storefiles;
         if (snapshot != null && !snapshot.isEmpty()) {
           String seed = snapshot.get(0).getPath().getName();
           double curRand = new Random(seed.hashCode()).nextDouble();
@@ -1201,6 +1208,10 @@ long getNextMajorCompactTime() {
   }
 
   public CompactionRequest requestCompaction() {
+    return requestCompaction(NO_PRIORITY);
+  }
+
+  public CompactionRequest requestCompaction(int priority) {
     // don't even select for compaction if writes are disabled
     if (!this.region.areWritesEnabled()) {
       return null;
@@ -1231,7 +1242,7 @@ public CompactionRequest requestCompaction() {
           // coprocessor is overriding normal file selection
           filesToCompact = new CompactSelection(conf, candidates);
         } else {
-          filesToCompact = compactSelection(candidates);
+          filesToCompact = compactSelection(candidates, priority);
         }
 
         if (region.getCoprocessorHost() != null) {
@@ -1261,7 +1272,7 @@ public CompactionRequest requestCompaction() {
         }
 
         // everything went better than expected. create a compaction request
-        int pri = getCompactPriority();
+        int pri = getCompactPriority(priority);
         ret = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
       }
     } catch (IOException ex) {
@@ -1280,6 +1291,16 @@ public void finishRequest(CompactionRequest cr) {
     }
   }
 
+  /**
+   * Algorithm to choose which files to compact, see {@link #compactSelection(java.util.List, int)}
+   * @param candidates
+   * @return
+   * @throws IOException
+   */
+  CompactSelection compactSelection(List<StoreFile> candidates) throws IOException {
+    return compactSelection(candidates,NO_PRIORITY);
+  }
+
   /**
    * Algorithm to choose which files to compact
    *
@@ -1299,7 +1320,7 @@ public void finishRequest(CompactionRequest cr) {
    * @return subset copy of candidate list that meets compaction criteria
    * @throws IOException
    */
-  CompactSelection compactSelection(List<StoreFile> candidates)
+  CompactSelection compactSelection(List<StoreFile> candidates, int priority)
       throws IOException {
     // ASSUMPTION!!! filesCompacting is locked when calling this function
 
@@ -1347,10 +1368,16 @@ CompactSelection compactSelection(List<StoreFile> candidates)
       return compactSelection;
     }
 
-    // major compact on user action or age (caveat: we have too many files)
-    boolean majorcompaction =
-      (forcemajor || isMajorCompaction(compactSelection.getFilesToCompact()))
-      && compactSelection.getFilesToCompact().size() < this.maxFilesToCompact;
+    // Force a major compaction if this is a user-requested major compaction,
+    // or if we do not have too many files to compact and this was requested
+    // as a major compaction
+    boolean majorcompaction = (forcemajor && priority == PRIORITY_USER) ||
+      (forcemajor || isMajorCompaction(compactSelection.getFilesToCompact())) &&
+      (compactSelection.getFilesToCompact().size() < this.maxFilesToCompact
+    );
+    LOG.debug(this.getHRegionInfo().getEncodedName() + " - " +
+      this.getColumnFamilyName() + ": Initiating " +
+      (majorcompaction ? "major" : "minor") + "compaction");
 
     if (!majorcompaction &&
         !hasReferences(compactSelection.getFilesToCompact())) {
@@ -1360,6 +1387,11 @@ CompactSelection compactSelection(List<StoreFile> candidates)
 
       // skip selection algorithm if we don't have enough files
       if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("Not compacting files because we only have " +
+            compactSelection.getFilesToCompact().size() +
+            " files ready for compaction.  Need " + this.minFilesToCompact + " to initiate.");
+        }
         compactSelection.emptyFileList();
         return compactSelection;
       }
@@ -1427,11 +1459,18 @@ public boolean apply(StoreFile input) {
         return compactSelection;
       }
     } else {
-      // all files included in this compaction, up to max
-      if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
-        int pastMax =
-          compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
-        compactSelection.clearSubList(0, pastMax);
+      if(majorcompaction) {
+        if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
+          LOG.debug("Warning, compacting more than " + this.maxFilesToCompact +
+            " files, probably because of a user-requested major compaction");
+          if(priority != PRIORITY_USER) {
+            LOG.error("Compacting more than max files on a non user-requested compaction");
+          }
+        }
+      } else if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
+        // all files included in this compaction, up to max
+        int pastMax = compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
+        compactSelection.getFilesToCompact().subList(0, pastMax).clear();
       }
     }
     return compactSelection;
@@ -2093,11 +2132,21 @@ long getMemStoreSize() {
     return this.memstore.heapSize();
   }
 
+  public int getCompactPriority() {
+    return getCompactPriority(NO_PRIORITY);
+  }
+
   /**
    * @return The priority that this store should have in the compaction queue
+   * @param priority
    */
-  public int getCompactPriority() {
-    return this.blockingStoreFileCount - this.storefiles.size();
+  public int getCompactPriority(int priority) {
+    // If this is a user-requested compaction, leave this at the highest priority
+    if(priority == PRIORITY_USER) {
+      return PRIORITY_USER;
+    } else {
+      return this.blockingStoreFileCount - this.storefiles.size();
+    }
   }
 
   HRegion getHRegion() {
@@ -2225,7 +2274,7 @@ public CacheConfig getCacheConfig() {
     return this.cacheConf;
   }
 
-  public static final long FIXED_OVERHEAD = 
+  public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
           + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
           + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
@@ -2303,7 +2352,7 @@ public long getTtl() {
     public boolean getKeepDeletedCells() {
       return keepDeletedCells;
     }
-    
+
     public long getTimeToPurgeDeletes() {
       return timeToPurgeDeletes;
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 91ac65287123..8134f4a93ecb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
@@ -76,6 +77,7 @@ public class TestCompaction extends HBaseTestCase {
   private int compactionThreshold;
   private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
   final private byte[] col1, col2;
+  private static final long MAX_FILES_TO_COMPACT = 10;
 
   /** constructor */
   public TestCompaction() throws Exception {
@@ -612,6 +614,43 @@ public void testCompactionWithCorruptResult() throws Exception {
     fail("testCompactionWithCorruptResult failed since no exception was" +
         "thrown while completing a corrupt file");
   }
+  
+  /**
+   * Test for HBASE-5920 - Test user requested major compactions always occurring
+   */
+  public void testNonUserMajorCompactionRequest() throws Exception {
+    Store store = r.getStore(COLUMN_FAMILY);
+    createStoreFile(r);
+    for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
+      createStoreFile(r);
+    }
+    store.triggerMajorCompaction();
+
+    CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY);
+    assertNotNull("Expected to receive a compaction request", request);
+    assertEquals(
+      "System-requested major compaction should not occur if there are too many store files",
+      false,
+      request.isMajor());
+  }
+
+  /**
+   * Test for HBASE-5920
+   */
+  public void testUserMajorCompactionRequest() throws IOException{
+    Store store = r.getStore(COLUMN_FAMILY);
+    createStoreFile(r);
+    for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
+      createStoreFile(r);
+    }
+    store.triggerMajorCompaction();
+    CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER);
+    assertNotNull("Expected to receive a compaction request", request);
+    assertEquals(
+      "User-requested major compaction should always occur, even if there are too many store files",
+      true, 
+      request.isMajor());
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 94ae47306d7721cd98bd67b90307c06c95735179 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 19 May 2012 05:20:33 +0000
Subject: [PATCH 0217/1540] HBASE-6056 Restore hbase-default version check

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1340345 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 04e99b161166..0477be8be1b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -61,7 +61,6 @@ public HBaseConfiguration(final Configuration c) {
   }
 
   private static void checkDefaultsVersion(Configuration conf) {
-    if (true) return; // REMOVE
     if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
     String defaultsVersion = conf.get("hbase.defaults.for.version");
     String thisVersion = VersionInfo.getVersion();

From 1980ade8b897909a000990b20350a7391d7b02d1 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Sat, 19 May 2012 09:57:48 +0000
Subject: [PATCH 0218/1540] HBASE-5840 Open Region FAILED_OPEN doesn't clear
 the TaskMonitor Status, keeps showing the old status (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1340396 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 18 +++++++-
 .../hbase/regionserver/TestHRegion.java       | 41 +++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 1a468bf55ed5..65037b5bd527 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -453,11 +453,27 @@ public long initialize() throws IOException {
    * @throws IOException e
    */
   public long initialize(final CancelableProgressable reporter)
-  throws IOException {
+      throws IOException {
 
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Initializing region " + this);
 
+    long nextSeqId = -1;
+    try {
+      nextSeqId = initializeRegionInternals(reporter, status);
+      return nextSeqId;
+    } finally {
+      // nextSeqid will be -1 if the initialization fails.
+      // At least it will be 0 otherwise.
+      if (nextSeqId == -1) {
+        status.abort("Exception during region " + this.getRegionNameAsString()
+            + " initialization.");
+      }
+    }
+  }
+
+  private long initializeRegionInternals(final CancelableProgressable reporter,
+      MonitoredTask status) throws IOException, UnsupportedEncodingException {
     if (coprocessorHost != null) {
       status.setStatus("Running coprocessor pre-open hook");
       coprocessorHost.preOpen();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 67b7e523ebf9..a4e225561923 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -70,6 +70,7 @@
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
@@ -88,6 +89,7 @@
 import org.apache.hadoop.hbase.util.Threads;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 import com.google.common.collect.Lists;
 
@@ -3199,6 +3201,45 @@ public void testHolesInMeta() throws Exception {
       this.region = null;
     }
   }
+  
+  /**
+   * Testcase to check state of region initialization task set to ABORTED or not if any exceptions
+   * during initialization
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
+    HRegionInfo info = null;
+    try {
+      FileSystem fs = Mockito.mock(FileSystem.class);
+      Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor("cf"));
+      info = new HRegionInfo(htd.getName(), HConstants.EMPTY_BYTE_ARRAY,
+          HConstants.EMPTY_BYTE_ARRAY, false);
+      Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
+      // no where we are instantiating HStore in this test case so useTableNameGlobally is null. To
+      // avoid NullPointerException we are setting useTableNameGlobally to false.
+      SchemaMetrics.setUseTableNameInTest(false);
+      region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
+      // region initialization throws IOException and set task state to ABORTED.
+      region.initialize();
+      fail("Region initialization should fail due to IOException");
+    } catch (IOException io) {
+      List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
+      for (MonitoredTask monitoredTask : tasks) {
+        if (!(monitoredTask instanceof MonitoredRPCHandler)
+            && monitoredTask.getDescription().contains(region.toString())) {
+          assertTrue("Region state should be ABORTED.",
+              monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
+          break;
+        }
+      }
+    } finally {
+      HRegion.closeHRegion(region);
+    }
+  }
 
   public void testIndexesScanWithOneDeletedRow() throws IOException {
     byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow");

From a01a41692da61515693e99c0e9cb3c52806e4262 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 21 May 2012 18:03:56 +0000
Subject: [PATCH 0219/1540] HBASE-5757 TableInputFormat should handle as many
 errors as possible (Jan Lukavsky)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1341133 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapred/TableRecordReaderImpl.java   |  5 +-
 .../mapreduce/TableRecordReaderImpl.java      |  9 ++-
 .../hbase/mapred/TestTableInputFormat.java    | 78 +++++++++++++++----
 3 files changed, 72 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
index a3639716cc88..5212ccdd85b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -24,7 +24,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -206,9 +205,9 @@ public boolean next(ImmutableBytesWritable key, Result value)
             rowcount = 0;
           }
         }
-      } catch (DoNotRetryIOException e) {
-        throw e;
       } catch (IOException e) {
+        // try to handle all IOExceptions by restarting
+        // the scanner, if the second call fails, it will be rethrown
         LOG.debug("recovered from " + StringUtils.stringifyException(e));
         if (lastSuccessfulRow == null) {
           LOG.warn("We are restarting the first next() invocation," +
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 8c193a97825a..d91b555824ca 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -23,7 +23,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -61,6 +60,7 @@ public class TableRecordReaderImpl {
   private Result value = null;
   private TaskAttemptContext context = null;
   private Method getCounter = null;
+  private long numRestarts = 0;
   private long timestamp;
   private int rowcount;
   private boolean logScannerActivity = false;
@@ -198,9 +198,9 @@ public boolean nextKeyValue() throws IOException, InterruptedException {
             rowcount = 0;
           }
         }
-      } catch (DoNotRetryIOException e) {
-        throw e;
       } catch (IOException e) {
+        // try to handle all IOExceptions by restarting
+        // the scanner, if the second call fails, it will be rethrown
         LOG.info("recovered from " + StringUtils.stringifyException(e));
         if (lastSuccessfulRow == null) {
           LOG.warn("We are restarting the first next() invocation," +
@@ -215,6 +215,7 @@ public boolean nextKeyValue() throws IOException, InterruptedException {
           scanner.next();    // skip presumed already mapped row
         }
         value = scanner.next();
+        numRestarts++;
       }
       if (value != null && value.size() > 0) {
         key.set(value.getRow());
@@ -270,6 +271,8 @@ private void updateCounters() throws IOException {
           HBASE_COUNTER_GROUP_NAME, mlv.getName());
         ct.increment(mlv.getCurrentIntervalValue());
       }
+      ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
+          "NUM_SCANNER_RESTARTS")).increment(numRestarts);
     } catch (Exception e) {
       LOG.debug("can't update counter." + StringUtils.stringifyException(e));
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
index 5956169c20b5..38be7e42f530 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
@@ -195,16 +195,16 @@ static void runTestMapreduce(HTable table) throws IOException,
    * 
    * @throws IOException
    */
-  static HTable createIOEScannerTable(byte[] name) throws IOException {
+  static HTable createIOEScannerTable(byte[] name, final int failCnt)
+      throws IOException {
     // build up a mock scanner stuff to fail the first time
     Answer<ResultScanner> a = new Answer<ResultScanner>() {
-      boolean first = true;
+      int cnt = 0;
 
       @Override
       public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
         // first invocation return the busted mock scanner
-        if (first) {
-          first = false;
+        if (cnt++ < failCnt) {
           // create mock ResultScanner that always fails.
           Scan scan = mock(Scan.class);
           doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
@@ -230,16 +230,16 @@ public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
    * 
    * @throws IOException
    */
-  static HTable createDNRIOEScannerTable(byte[] name) throws IOException {
+  static HTable createDNRIOEScannerTable(byte[] name, final int failCnt)
+      throws IOException {
     // build up a mock scanner stuff to fail the first time
     Answer<ResultScanner> a = new Answer<ResultScanner>() {
-      boolean first = true;
+      int cnt = 0;
 
       @Override
       public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
         // first invocation return the busted mock scanner
-        if (first) {
-          first = false;
+        if (cnt++ < failCnt) {
           // create mock ResultScanner that always fails.
           Scan scan = mock(Scan.class);
           doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
@@ -280,7 +280,18 @@ public void testTableRecordReader() throws IOException {
    */
   @Test
   public void testTableRecordReaderScannerFail() throws IOException {
-    HTable htable = createIOEScannerTable("table2".getBytes());
+    HTable htable = createIOEScannerTable("table2".getBytes(), 1);
+    runTestMapred(htable);
+  }
+
+  /**
+   * Run test assuming Scanner IOException failure using mapred api,
+   * 
+   * @throws IOException
+   */
+  @Test(expected = IOException.class)
+  public void testTableRecordReaderScannerFailTwice() throws IOException {
+    HTable htable = createIOEScannerTable("table3".getBytes(), 2);
     runTestMapred(htable);
   }
 
@@ -290,9 +301,21 @@ public void testTableRecordReaderScannerFail() throws IOException {
    * 
    * @throws DoNotRetryIOException
    */
-  @Test(expected = DoNotRetryIOException.class)
+  @Test
   public void testTableRecordReaderScannerTimeout() throws IOException {
-    HTable htable = createDNRIOEScannerTable("table3".getBytes());
+    HTable htable = createDNRIOEScannerTable("table4".getBytes(), 1);
+    runTestMapred(htable);
+  }
+
+  /**
+   * Run test assuming UnknownScannerException (which is a type of
+   * DoNotRetryIOException) using mapred api.
+   * 
+   * @throws DoNotRetryIOException
+   */
+  @Test(expected = DoNotRetryIOException.class)
+  public void testTableRecordReaderScannerTimeoutTwice() throws IOException {
+    HTable htable = createDNRIOEScannerTable("table5".getBytes(), 2);
     runTestMapred(htable);
   }
 
@@ -318,7 +341,20 @@ public void testTableRecordReaderMapreduce() throws IOException,
   @Test
   public void testTableRecordReaderScannerFailMapreduce() throws IOException,
       InterruptedException {
-    HTable htable = createIOEScannerTable("table2-mr".getBytes());
+    HTable htable = createIOEScannerTable("table2-mr".getBytes(), 1);
+    runTestMapreduce(htable);
+  }
+
+  /**
+   * Run test assuming Scanner IOException failure using newer mapreduce api
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test(expected = IOException.class)
+  public void testTableRecordReaderScannerFailMapreduceTwice() throws IOException,
+      InterruptedException {
+    HTable htable = createIOEScannerTable("table3-mr".getBytes(), 2);
     runTestMapreduce(htable);
   }
 
@@ -329,10 +365,24 @@ public void testTableRecordReaderScannerFailMapreduce() throws IOException,
    * @throws InterruptedException
    * @throws DoNotRetryIOException
    */
-  @Test(expected = DoNotRetryIOException.class)
+  @Test
   public void testTableRecordReaderScannerTimeoutMapreduce()
       throws IOException, InterruptedException {
-    HTable htable = createDNRIOEScannerTable("table3-mr".getBytes());
+    HTable htable = createDNRIOEScannerTable("table4-mr".getBytes(), 1);
+    runTestMapreduce(htable);
+  }
+
+  /**
+   * Run test assuming UnknownScannerException (which is a type of
+   * DoNotRetryIOException) using newer mapreduce api
+   * 
+   * @throws InterruptedException
+   * @throws DoNotRetryIOException
+   */
+  @Test(expected = DoNotRetryIOException.class)
+  public void testTableRecordReaderScannerTimeoutMapreduceTwice()
+      throws IOException, InterruptedException {
+    HTable htable = createDNRIOEScannerTable("table5-mr".getBytes(), 2);
     runTestMapreduce(htable);
   }
 

From 09de7e722f4b93e0d4a0f8d2a7467c3b189c58be Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 21 May 2012 20:56:15 +0000
Subject: [PATCH 0220/1540] HBASE-6044 copytable: remove rs.* parameters

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1341199 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/ops_mgt.xml                        |  6 +----
 .../hadoop/hbase/mapreduce/CopyTable.java     | 22 +++----------------
 2 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/src/docbkx/ops_mgt.xml b/src/docbkx/ops_mgt.xml
index 0f645a19867e..2797049000f4 100644
--- a/src/docbkx/ops_mgt.xml
+++ b/src/docbkx/ops_mgt.xml
@@ -81,14 +81,12 @@
         <title>CopyTable</title>
       <para>
             CopyTable is a utility that can copy part or of all of a table, either to the same cluster or another cluster. The usage is as follows:
-<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.CopyTable [--rs.class=CLASS] [--rs.impl=IMPL] [--starttime=X] [--endtime=Y] [--new.name=NEW] [--peer.adr=ADR] tablename
+<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.CopyTable [--starttime=X] [--endtime=Y] [--new.name=NEW] [--peer.adr=ADR] tablename
 </programlisting>
         </para>
         <para>
         Options:
         <itemizedlist>
-          <listitem><varname>rs.class</varname> hbase.regionserver.class of the peer cluster.  Specify if different from current cluster.</listitem>
-          <listitem><varname>rs.impl</varname>  hbase.regionserver.impl of the peer cluster. </listitem>
           <listitem><varname>starttime</varname>  Beginning of the time range.  Without endtime means starttime to forever.</listitem>
           <listitem><varname>endtime</varname>  End of the time range.  Without endtime means starttime to forever.</listitem>
           <listitem><varname>versions</varname>  Number of cell versions to copy.</listitem>
@@ -104,8 +102,6 @@
         </para>
         <para>Example of copying 'TestTable' to a cluster that uses replication for a 1 hour window:
 <programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.CopyTable
---rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface
---rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer
 --starttime=1265875194289 --endtime=1265878794289
 --peer.adr=server1,server2,server3:2181:/hbase TestTable</programlisting>
         </para>
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 03947e92d8d1..d8e9860c5249 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -39,8 +39,6 @@
 public class CopyTable {
 
   final static String NAME = "copytable";
-  static String rsClass = null;
-  static String rsImpl = null;
   static long startTime = 0;
   static long endTime = 0;
   static int versions = -1;
@@ -100,7 +98,7 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
         Import.Importer.class, null, null, job);
     TableMapReduceUtil.initTableReducerJob(
         newTableName == null ? tableName : newTableName, null, job,
-        null, peerAddress, rsClass, rsImpl);
+        null, peerAddress, null, null);
     job.setNumReduceTasks(0);
     return job;
   }
@@ -112,8 +110,7 @@ private static void printUsage(final String errorMsg) {
     if (errorMsg != null && errorMsg.length() > 0) {
       System.err.println("ERROR: " + errorMsg);
     }
-    System.err.println("Usage: CopyTable [general options] [--rs.class=CLASS] " +
-        "[--rs.impl=IMPL] [--starttime=X] [--endtime=Y] " +
+    System.err.println("Usage: CopyTable [general options] [--starttime=X] [--endtime=Y] " +
         "[--new.name=NEW] [--peer.adr=ADR] <tablename>");
     System.err.println();
     System.err.println("Options:");
@@ -138,8 +135,7 @@ private static void printUsage(final String errorMsg) {
     System.err.println("Examples:");
     System.err.println(" To copy 'TestTable' to a cluster that uses replication for a 1 hour window:");
     System.err.println(" $ bin/hbase " +
-        "org.apache.hadoop.hbase.mapreduce.CopyTable --rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface " +
-        "--rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer --starttime=1265875194289 --endtime=1265878794289 " +
+        "org.apache.hadoop.hbase.mapreduce.CopyTable --starttime=1265875194289 --endtime=1265878794289 " +
         "--peer.adr=server1,server2,server3:2181:/hbase --families=myOldCf:myNewCf,cf2,cf3 TestTable ");
     System.err.println("For performance consider the following general options:\n"
         + "-Dhbase.client.scanner.caching=100\n"
@@ -161,18 +157,6 @@ private static boolean doCommandLine(final String[] args) {
           return false;
         }
 
-        final String rsClassArgKey = "--rs.class=";
-        if (cmd.startsWith(rsClassArgKey)) {
-          rsClass = cmd.substring(rsClassArgKey.length());
-          continue;
-        }
-
-        final String rsImplArgKey = "--rs.impl=";
-        if (cmd.startsWith(rsImplArgKey)) {
-          rsImpl = cmd.substring(rsImplArgKey.length());
-          continue;
-        }
-
         final String startTimeArgKey = "--starttime=";
         if (cmd.startsWith(startTimeArgKey)) {
           startTime = Long.parseLong(cmd.substring(startTimeArgKey.length()));

From 84e8232c8941ed7cb4019872fa34a9e6c24a04c4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 21 May 2012 23:23:35 +0000
Subject: [PATCH 0221/1540] HBASE-6061 Fix ACL "Admin" Table inconsistent
 permission check (Matteo Bertozzi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1341267 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 45 +++++++++++--------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 260b0f1e51ec..b4a90e6d87c7 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -361,6 +361,25 @@ private User getActiveUser() throws IOException {
     return user;
   }
 
+  /**
+   * Authorizes that the current user has "admin" privileges for the given table.
+   * that means he/she can edit/modify/delete the table.
+   * If current user is the table owner, and has CREATE permission,
+   * then he/she has table admin permission. otherwise ADMIN rights are checked.
+   * @param e Master coprocessor environment
+   * @param tableName Table requested
+   * @throws IOException if obtaining the current user fails
+   * @throws AccessDeniedException if authorization is denied
+   */
+  private void requireTableAdminPermission(MasterCoprocessorEnvironment e,
+      byte[] tableName) throws IOException {
+    if (isActiveUserTableOwner(e, tableName)) {
+      requirePermission(Permission.Action.CREATE);
+    } else {
+      requirePermission(Permission.Action.ADMIN);
+    }
+  }
+
   /**
    * Authorizes that the current user has global privileges for the given action.
    * @param perm The action being requested
@@ -513,11 +532,7 @@ public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    if (isActiveUserTableOwner(c.getEnvironment(), tableName)) {
-      requirePermission(Permission.Action.CREATE);
-    } else {
-      requirePermission(Permission.Action.ADMIN);
-    }
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -528,7 +543,7 @@ public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HTableDescriptor htd) throws IOException {
-    requirePermission(Permission.Action.CREATE);
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -538,7 +553,7 @@ public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor column) throws IOException {
-    requirePermission(Permission.Action.CREATE);
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -547,7 +562,7 @@ public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor descriptor) throws IOException {
-    requirePermission(Permission.Action.CREATE);
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -557,7 +572,7 @@ public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, byte[] col) throws IOException {
-    requirePermission(Permission.Action.CREATE);
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -569,11 +584,7 @@ public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    if (isActiveUserTableOwner(c.getEnvironment(), tableName)) {
-      requirePermission(Permission.Action.CREATE);
-    } else {
-      requirePermission(Permission.Action.ADMIN);
-    }
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
@@ -582,11 +593,7 @@ public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    if (isActiveUserTableOwner(c.getEnvironment(), tableName)) {
-      requirePermission(Permission.Action.CREATE);
-    } else {
-      requirePermission(Permission.Action.ADMIN);
-    }
+    requireTableAdminPermission(c.getEnvironment(), tableName);
   }
   @Override
   public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,

From 58917af7350c70432190263480f6e8f3ff5b394a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 23 May 2012 03:39:22 +0000
Subject: [PATCH 0222/1540] HBASE-6047 Put.has() can't determine result
 correctly (Alex Newman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1341740 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/Put.java   | 32 +++++---
 .../hadoop/hbase/client/TestPutDotHas.java    | 76 +++++++++++++++++++
 2 files changed, 98 insertions(+), 10 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index 920295827831..1ec81ff1110a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -252,8 +252,8 @@ public boolean has(byte [] family, byte [] qualifier, long ts, byte [] value) {
    * @return returns true if the given family, qualifier timestamp and value
    * already has an existing KeyValue object in the family map.
    */
-  private boolean has(byte [] family, byte [] qualifier, long ts, byte [] value,
-      boolean ignoreTS, boolean ignoreValue) {
+  private boolean has(byte[] family, byte[] qualifier, long ts, byte[] value,
+                      boolean ignoreTS, boolean ignoreValue) {
     List<KeyValue> list = getKeyValueList(family);
     if (list.size() == 0) {
       return false;
@@ -264,20 +264,32 @@ private boolean has(byte [] family, byte [] qualifier, long ts, byte [] value,
     // F T => 2
     // F F => 1
     if (!ignoreTS && !ignoreValue) {
-      KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
-      return (list.contains(kv));
-    } else if (ignoreValue) {
-      for (KeyValue kv: list) {
+      for (KeyValue kv : list) {
+        if (Arrays.equals(kv.getFamily(), family) &&
+            Arrays.equals(kv.getQualifier(), qualifier) &&
+            Arrays.equals(kv.getValue(), value) &&
+            kv.getTimestamp() == ts) {
+          return true;
+        }
+      }
+    } else if (ignoreValue && !ignoreTS) {
+      for (KeyValue kv : list) {
         if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
             && kv.getTimestamp() == ts) {
           return true;
         }
       }
+    } else if (!ignoreValue && ignoreTS) {
+      for (KeyValue kv : list) {
+        if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
+            && Arrays.equals(kv.getValue(), value)) {
+          return true;
+        }
+      }
     } else {
-      // ignoreTS is always true
-      for (KeyValue kv: list) {
-      if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
-              && Arrays.equals(kv.getValue(), value)) {
+      for (KeyValue kv : list) {
+        if (Arrays.equals(kv.getFamily(), family) &&
+            Arrays.equals(kv.getQualifier(), qualifier)) {
           return true;
         }
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java b/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java
new file mode 100644
index 000000000000..49cfcdc2cc13
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+/**
+ * Addresses HBASE-6047
+ * We test put.has call with all of its polymorphic magic
+ */
+public class TestPutDotHas {
+
+  public static final byte[] ROW_01 = Bytes.toBytes("row-01");
+  public static final byte[] QUALIFIER_01 = Bytes.toBytes("qualifier-01");
+  public static final byte[] VALUE_01 = Bytes.toBytes("value-01");
+  public static final byte[] FAMILY_01 = Bytes.toBytes("family-01");
+  public static final long TS = 1234567L;
+  public Put put = new Put(ROW_01);
+
+  @Before
+  public void setUp() {
+    put.add(FAMILY_01, QUALIFIER_01, TS, VALUE_01);
+  }
+
+  @Test
+  public void testHasIgnoreValueIgnoreTS() {
+    Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01));
+    Assert.assertFalse(put.has(QUALIFIER_01, FAMILY_01));
+  }
+
+  @Test
+  public void testHasIgnoreValue() {
+    Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS));
+    Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1));
+  }
+
+  @Test
+  public void testHasIgnoreTS() {
+    Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, VALUE_01));
+    Assert.assertFalse(put.has(FAMILY_01, VALUE_01, QUALIFIER_01));
+  }
+
+  @Test
+  public void testHas() {
+    Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS, VALUE_01));
+    // Bad TS
+    Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1, VALUE_01));
+    // Bad Value
+    Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS, QUALIFIER_01));
+    // Bad Family
+    Assert.assertFalse(put.has(QUALIFIER_01, QUALIFIER_01, TS, VALUE_01));
+    // Bad Qual
+    Assert.assertFalse(put.has(FAMILY_01, FAMILY_01, TS, VALUE_01));
+  }
+}

From 7db45219e65f2264af5a2ca2269c1c35567b9fd3 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 23 May 2012 15:59:10 +0000
Subject: [PATCH 0223/1540] HBASE-6069
 TableInputFormatBase#createRecordReader() doesn't initialize
 TableRecordReader which causes NPE (Jie Huang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1341919 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/TableInputFormatBase.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index b275e4e0cfe9..a615710333f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -124,6 +125,11 @@ public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
     sc.setStopRow(tSplit.getEndRow());
     trr.setScan(sc);
     trr.setHTable(table);
+    try {
+      trr.initialize(tSplit, context);
+    } catch (InterruptedException e) {
+      throw new InterruptedIOException(e.getMessage());
+    }
     return trr;
   }
 

From 62d689bd9377351ef947d9bcc685d9cfec9d5bfd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 23 May 2012 20:16:19 +0000
Subject: [PATCH 0224/1540] HBASE-6065 Log for flush would append a
 non-sequential edit in the hlog, leading to possible data loss (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342017 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLogSplitter.java   | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 1ddec19577ef..ac29f28e7df0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -1303,8 +1303,15 @@ WriterAndPath getWriterAndPath(Entry entry) throws IOException {
      * Update region's maximum edit log SeqNum.
      */
     void updateRegionMaximumEditLogSeqNum(Entry entry) {
-      regionMaximumEditLogSeqNum.put(entry.getKey().getEncodedRegionName(),
-          entry.getKey().getLogSeqNum());
+      synchronized (regionMaximumEditLogSeqNum) {
+        Long currentMaxSeqNum=regionMaximumEditLogSeqNum.get(entry.getKey().getEncodedRegionName());
+        if (currentMaxSeqNum == null
+            || entry.getKey().getLogSeqNum() > currentMaxSeqNum) {
+          regionMaximumEditLogSeqNum.put(entry.getKey().getEncodedRegionName(),
+              entry.getKey().getLogSeqNum());
+        }
+      }
+
     }
 
     Long getRegionMaximumEditLogSeqNum(byte[] region) {

From 04ab2c6d6e670f4183f642d6763b66437df95b9b Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 24 May 2012 00:32:12 +0000
Subject: [PATCH 0225/1540] HBASE-6077. Document the most common secure RPC
 troubleshooting resolutions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342105 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/troubleshooting.xml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/docbkx/troubleshooting.xml b/src/docbkx/troubleshooting.xml
index a92d9794e925..0b084b58bde8 100644
--- a/src/docbkx/troubleshooting.xml
+++ b/src/docbkx/troubleshooting.xml
@@ -605,6 +605,28 @@ invocation of the admin API.
             </para>
        </section>    
 
+       <section xml:id="trouble.client.security.rpc">
+           <title>Secure Client Cannot Connect ([Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)])</title>
+           <para>
+There can be several causes that produce this symptom.
+           </para>
+           <para>
+First, check that you have a valid Kerberos ticket. One is required in order to set up communication with a secure HBase cluster. Examine the ticket currently in the credential cache, if any, by running the <tt>klist</tt> command line utility. If no ticket is listed, you must obtain a ticket by running the <tt>kinit</tt> command with either a keytab specified, or by interactively entering a password for the desired principal.
+           </para>
+           <para>
+Then, consult the <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jgss/tutorials/Troubleshooting.html">Java Security Guide troubleshooting section</a>. The most common problem addressed there is resolved by setting <tt>javax.security.auth.useSubjectCredsOnly</tt> system property value to <tt>false</tt>.
+           </para>
+           <para>
+Because of a change in the format in which MIT Kerberos writes its credentials cache, there is a bug in the Oracle JDK 6 Update 26 and earlier that causes Java to be unable to read the Kerberos credentials cache created by versions of MIT Kerberos 1.8.1 or higher. If you have this problematic combination of components in your environment, to work around this problem, first log in with <tt>kinit</tt> and then immediately refresh the credential cache with <tt>kinit -R</tt>. The refresh will rewrite the credential cache without the problematic formatting.
+           </para>
+           <para>
+Finally, depending on your Kerberos configuration, you may need to install the <a href="http://docs.oracle.com/javase/1.4.2/docs/guide/security/jce/JCERefGuide.html">Java Cryptography Extension</a>, or JCE. Insure the JCE jars are on the classpath on both server and client systems.
+           </para>
+           <para>
+You may also need to download the <a href="http://www.oracle.com/technetwork/java/javase/downloads/jce-6-download-429243.html">unlimited strength JCE policy files</a>. Uncompress and extract the downloaded file, and install the policy jars into <tt>&lt;java-home&gt;/lib/security</tt>.
+           </para>
+       </section>    
+
     </section>
     
     <section xml:id="trouble.mapreduce">

From 2a6c1bf82f830925089658e15a2d3594f245f052 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 24 May 2012 18:58:38 +0000
Subject: [PATCH 0226/1540] Amend HBASE-6077. Replace HTML formatting that does
 not work with Docbook

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342382 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/troubleshooting.xml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/docbkx/troubleshooting.xml b/src/docbkx/troubleshooting.xml
index 0b084b58bde8..0170b87fae2f 100644
--- a/src/docbkx/troubleshooting.xml
+++ b/src/docbkx/troubleshooting.xml
@@ -611,19 +611,19 @@ invocation of the admin API.
 There can be several causes that produce this symptom.
            </para>
            <para>
-First, check that you have a valid Kerberos ticket. One is required in order to set up communication with a secure HBase cluster. Examine the ticket currently in the credential cache, if any, by running the <tt>klist</tt> command line utility. If no ticket is listed, you must obtain a ticket by running the <tt>kinit</tt> command with either a keytab specified, or by interactively entering a password for the desired principal.
+First, check that you have a valid Kerberos ticket. One is required in order to set up communication with a secure HBase cluster. Examine the ticket currently in the credential cache, if any, by running the klist command line utility. If no ticket is listed, you must obtain a ticket by running the kinit command with either a keytab specified, or by interactively entering a password for the desired principal.
            </para>
            <para>
-Then, consult the <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jgss/tutorials/Troubleshooting.html">Java Security Guide troubleshooting section</a>. The most common problem addressed there is resolved by setting <tt>javax.security.auth.useSubjectCredsOnly</tt> system property value to <tt>false</tt>.
+Then, consult the <link xlink:href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jgss/tutorials/Troubleshooting.html">Java Security Guide troubleshooting section</link>. The most common problem addressed there is resolved by setting javax.security.auth.useSubjectCredsOnly system property value to false.
            </para>
            <para>
-Because of a change in the format in which MIT Kerberos writes its credentials cache, there is a bug in the Oracle JDK 6 Update 26 and earlier that causes Java to be unable to read the Kerberos credentials cache created by versions of MIT Kerberos 1.8.1 or higher. If you have this problematic combination of components in your environment, to work around this problem, first log in with <tt>kinit</tt> and then immediately refresh the credential cache with <tt>kinit -R</tt>. The refresh will rewrite the credential cache without the problematic formatting.
+Because of a change in the format in which MIT Kerberos writes its credentials cache, there is a bug in the Oracle JDK 6 Update 26 and earlier that causes Java to be unable to read the Kerberos credentials cache created by versions of MIT Kerberos 1.8.1 or higher. If you have this problematic combination of components in your environment, to work around this problem, first log in with kinit</tt> and then immediately refresh the credential cache with kinit -R. The refresh will rewrite the credential cache without the problematic formatting.
            </para>
            <para>
-Finally, depending on your Kerberos configuration, you may need to install the <a href="http://docs.oracle.com/javase/1.4.2/docs/guide/security/jce/JCERefGuide.html">Java Cryptography Extension</a>, or JCE. Insure the JCE jars are on the classpath on both server and client systems.
+Finally, depending on your Kerberos configuration, you may need to install the <link xlink:href="http://docs.oracle.com/javase/1.4.2/docs/guide/security/jce/JCERefGuide.html">Java Cryptography Extension</link>, or JCE. Insure the JCE jars are on the classpath on both server and client systems.
            </para>
            <para>
-You may also need to download the <a href="http://www.oracle.com/technetwork/java/javase/downloads/jce-6-download-429243.html">unlimited strength JCE policy files</a>. Uncompress and extract the downloaded file, and install the policy jars into <tt>&lt;java-home&gt;/lib/security</tt>.
+You may also need to download the <link xlink:href="http://www.oracle.com/technetwork/java/javase/downloads/jce-6-download-429243.html">unlimited strength JCE policy files</link>. Uncompress and extract the downloaded file, and install the policy jars into &lt;java-home&gt;/lib/security.
            </para>
        </section>    
 

From 880565348b0004272a1428ec4cf634a55f5802f6 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 25 May 2012 15:59:09 +0000
Subject: [PATCH 0227/1540] Amend HBASE-6077. Remove stray tag

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342705 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/troubleshooting.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/docbkx/troubleshooting.xml b/src/docbkx/troubleshooting.xml
index 0170b87fae2f..a1e382969406 100644
--- a/src/docbkx/troubleshooting.xml
+++ b/src/docbkx/troubleshooting.xml
@@ -617,7 +617,7 @@ First, check that you have a valid Kerberos ticket. One is required in order to
 Then, consult the <link xlink:href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jgss/tutorials/Troubleshooting.html">Java Security Guide troubleshooting section</link>. The most common problem addressed there is resolved by setting javax.security.auth.useSubjectCredsOnly system property value to false.
            </para>
            <para>
-Because of a change in the format in which MIT Kerberos writes its credentials cache, there is a bug in the Oracle JDK 6 Update 26 and earlier that causes Java to be unable to read the Kerberos credentials cache created by versions of MIT Kerberos 1.8.1 or higher. If you have this problematic combination of components in your environment, to work around this problem, first log in with kinit</tt> and then immediately refresh the credential cache with kinit -R. The refresh will rewrite the credential cache without the problematic formatting.
+Because of a change in the format in which MIT Kerberos writes its credentials cache, there is a bug in the Oracle JDK 6 Update 26 and earlier that causes Java to be unable to read the Kerberos credentials cache created by versions of MIT Kerberos 1.8.1 or higher. If you have this problematic combination of components in your environment, to work around this problem, first log in with kinit and then immediately refresh the credential cache with kinit -R. The refresh will rewrite the credential cache without the problematic formatting.
            </para>
            <para>
 Finally, depending on your Kerberos configuration, you may need to install the <link xlink:href="http://docs.oracle.com/javase/1.4.2/docs/guide/security/jce/JCERefGuide.html">Java Cryptography Extension</link>, or JCE. Insure the JCE jars are on the classpath on both server and client systems.

From 685ed7eab45be2f605f57d093996ab75098c87c0 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Fri, 25 May 2012 16:57:16 +0000
Subject: [PATCH 0228/1540] HBASE-6070 AM.nodeDeleted and SSH races creating
 problems for regions under SPLIT (Ramkrishna)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342725 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |   2 +-
 .../master/handler/ServerShutdownHandler.java |  11 +-
 .../hbase/master/TestAssignmentManager.java   | 107 +++++++++++++++++-
 3 files changed, 113 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 652d0c3b5214..69bfcc073d2e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1121,7 +1121,7 @@ public void nodeDeleted(final String path) {
       RegionState rs = this.regionsInTransition.get(regionName);
       if (rs != null) {
         HRegionInfo regionInfo = rs.getRegion();
-        if (rs.isSplitting() || rs.isSplit()) {
+        if (rs.isSplit()) {
           LOG.debug("Ephemeral node deleted, regionserver crashed?, " +
             "clearing from RIT; rs=" + rs);
           regionOffline(rs.getRegion());
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index a8cba9942d1f..0aa62dbcbdeb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -290,7 +290,7 @@ public void process() throws IOException {
               this.server.getCatalogTracker())) {
             ServerName addressFromAM = this.services.getAssignmentManager()
                 .getRegionServerOfRegion(e.getKey());
-            if (rit != null && !rit.isClosing() && !rit.isPendingClose()) {
+            if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()) {
               // Skip regions that were in transition unless CLOSING or
               // PENDING_CLOSE
               LOG.info("Skip assigning region " + rit.toString());
@@ -303,6 +303,15 @@ public void process() throws IOException {
               } else {
                 this.services.getAssignmentManager().assign(e.getKey(), true);
               }
+          } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
+            // This will happen when the RS went down and the call back for the SPLIITING or SPLIT
+            // has not yet happened for node Deleted event.  In that case if the region was actually split
+            // but the RS had gone down before completing the split process then will not try to
+            // assign the parent region again. In that case we should make the region offline and
+            // also delete the region from RIT.
+            HRegionInfo region = rit.getRegion();
+            AssignmentManager am = this.services.getAssignmentManager();
+            am.regionOffline(region);
           }
           // If the table was partially disabled and the RS went down, we should clear the RIT
           // and remove the node for the region.
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index bb100dbf2344..64c8d334e255 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -386,7 +386,7 @@ public void testShutdownHandler() throws KeeperException, IOException {
     AssignmentManager am =
       new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
     try {
-      processServerShutdownHandler(ct, am);
+      processServerShutdownHandler(ct, am, false);
     } finally {
       executor.shutdown();
       am.shutdown();
@@ -407,6 +407,72 @@ public void testSSHWhenDisableTableInProgress()
     testCaseWithPartiallyDisabledState(TableState.DISABLING);
     testCaseWithPartiallyDisabledState(TableState.DISABLED);
   }
+  
+  /**
+   * To test if the split region is removed from RIT if the region was in SPLITTING state
+   * but the RS has actually completed the splitting in META but went down. See HBASE-6070
+   * and also HBASE-5806
+   * @throws KeeperException
+   * @throws IOException
+   */
+  @Test
+  public void testSSHWhenSplitRegionInProgress()
+      throws KeeperException, IOException, Exception {
+    // true indicates the region is split but still in RIT
+    testCaseWithSplitRegionPartial(true);
+    // false indicate the region is not split
+    testCaseWithSplitRegionPartial(false);
+
+  }
+
+  private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
+      NodeExistsException, InterruptedException {
+    // Create and startup an executor. This is used by AssignmentManager
+    // handling zk callbacks.
+    ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
+
+    // We need a mocked catalog tracker.
+    CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+    // Create an AM.
+    AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
+    // adding region to regions and servers maps.
+    am.regionOnline(REGIONINFO, SERVERNAME_A);
+    // adding region in pending close.
+    am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
+        State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
+    am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
+    
+    RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
+        REGIONINFO.getRegionName(), SERVERNAME_A);
+    String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
+    // create znode in M_ZK_REGION_CLOSING state.
+    ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
+
+    try {
+      
+      processServerShutdownHandler(ct, am, regionSplitDone);
+      // check znode deleted or not.
+      // In both cases the znode should be deleted.
+      
+      if(regionSplitDone){
+        assertTrue("Region state of region in SPLITTING should be removed from rit.",
+            am.regionsInTransition.isEmpty());
+      }
+      else{
+        while (!am.assignInvoked) {
+          Thread.sleep(1);
+        }
+        assertTrue("Assign should be invoked.", am.assignInvoked);
+      }
+    } finally {
+      REGIONINFO.setOffline(false);
+      REGIONINFO.setSplit(false);
+      executor.shutdown();
+      am.shutdown();
+      // Clean up all znodes
+      ZKAssign.deleteAllNodes(this.watcher);
+    }
+  }
 
   private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperException, IOException, NodeExistsException {
     // Create and startup an executor. This is used by AssignmentManager
@@ -438,7 +504,7 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
 
     try {
-      processServerShutdownHandler(ct, am);
+      processServerShutdownHandler(ct, am, false);
       // check znode deleted or not.
       // In both cases the znode should be deleted.
       assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
@@ -456,17 +522,21 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     }
   }
 
-  private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am)
+  private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion)
       throws IOException {
     // Make sure our new AM gets callbacks; once registered, can't unregister.
     // Thats ok because we make a new zk watcher for each test.
     this.watcher.registerListenerFirst(am);
-
     // Need to set up a fake scan of meta for the servershutdown handler
     // Make an RS Interface implementation.  Make it so a scanner can go against it.
     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
     // Get a meta row result that has region up on SERVERNAME_A
-    Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+    Result r = null;
+    if (splitRegion) {
+      r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
+    } else {
+      r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+    }
     Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
       thenReturn(System.currentTimeMillis());
     // Return a good result first and then return null to indicate end of scan
@@ -520,6 +590,20 @@ private Result getMetaTableRowResult(final HRegionInfo hri,
       Bytes.toBytes(sn.getStartcode())));
     return new Result(kvs);
   }
+  
+  /**
+   * @param sn ServerName to use making startcode and server in meta
+   * @param hri Region to serialize into HRegionInfo
+   * @return A mocked up Result that fakes a Get on a row in the
+   * <code>.META.</code> table.
+   * @throws IOException 
+   */
+  private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
+      throws IOException {
+    hri.setOffline(true);
+    hri.setSplit(true);
+    return getMetaTableRowResult(hri, sn);
+  }
 
   /**
    * Create and startup executor pools. Start same set as master does (just
@@ -747,6 +831,7 @@ class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
     // Ditto for ct
     private final CatalogTracker ct;
     boolean processRITInvoked = false;
+    boolean assignInvoked = false;
     AtomicBoolean gate = new AtomicBoolean(true);
 
     public AssignmentManagerWithExtrasForTesting(final Server master,
@@ -775,6 +860,18 @@ void processRegionsInTransition(final RegionTransitionData data,
       while (this.gate.get()) Threads.sleep(1);
       super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
     }
+    
+    @Override
+    public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
+        boolean hijack) {
+      assignInvoked = true;
+      super.assign(region, setOfflineInZK, forceNewPlan, hijack);
+    }
+    
+    @Override
+    public ServerName getRegionServerOfRegion(HRegionInfo hri) {
+      return SERVERNAME_A;
+    }
 
     /**
      * @return ExecutorService used by this instance.

From ffe25aa783bbe5671b8172c5b195225b677e0ccf Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 26 May 2012 08:54:18 +0000
Subject: [PATCH 0229/1540] HBASE-5986 Clients can see holes in the META table
 when regions are being split (Enis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342865 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/MetaEditor.java      |  17 +-
 .../hadoop/hbase/client/HBaseAdmin.java       |   3 +-
 .../hbase/client/HConnectionManager.java      |   5 +-
 .../apache/hadoop/hbase/client/HTable.java    |  65 +---
 .../hadoop/hbase/client/MetaScanner.java      | 198 +++++++++-
 .../apache/hadoop/hbase/master/HMaster.java   |   3 +-
 .../apache/hadoop/hbase/util/HBaseFsck.java   |   3 +-
 .../TestEndToEndSplitTransaction.java         | 363 +++++++++++++++++-
 8 files changed, 581 insertions(+), 76 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
index 19fee5c4acdc..9537a60032ba 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.apache.hadoop.hbase.util.Writables;
 
 /**
@@ -132,7 +133,7 @@ static void deleteMetaTable(final CatalogTracker ct, final Delete d)
       t.close();
     }
   }
-  
+
   /**
    * Adds a META row for the specified new region.
    * @param regionInfo region information
@@ -155,7 +156,7 @@ public static void addRegionsToMeta(CatalogTracker catalogTracker,
       List<HRegionInfo> regionInfos)
   throws IOException {
     List<Put> puts = new ArrayList<Put>();
-    for (HRegionInfo regionInfo : regionInfos) { 
+    for (HRegionInfo regionInfo : regionInfos) {
       puts.add(makePutFromRegionInfo(regionInfo));
     }
     putsToMetaTable(catalogTracker, puts);
@@ -304,6 +305,18 @@ public static HRegionInfo getHRegionInfo(
     return info;
   }
 
+  /**
+   * Returns the daughter regions by reading from the corresponding columns of the .META. table
+   * Result. If the region is not a split parent region, it returns PairOfSameType(null, null).
+   */
+  public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
+    HRegionInfo splitA = Writables.getHRegionInfoOrNull(
+        data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER));
+    HRegionInfo splitB = Writables.getHRegionInfoOrNull(
+        data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER));
+    return new PairOfSameType<HRegionInfo>(splitA, splitB);
+  }
+
   private static Put addRegionInfo(final Put p, final HRegionInfo hri)
   throws IOException {
     p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 9f3f62d22768..a77dc223568d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -53,6 +53,7 @@
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
@@ -401,7 +402,7 @@ public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
       ++tries) {
       // Wait for new table to come on-line
       final AtomicInteger actualRegCount = new AtomicInteger(0);
-      MetaScannerVisitor visitor = new MetaScannerVisitor() {
+      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
         @Override
         public boolean processRow(Result rowResult) throws IOException {
           HRegionInfo info = Writables.getHRegionInfoOrNull(
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index accddae26558..5f950583954e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -61,6 +61,7 @@
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.ipc.ExecRPCInvoker;
@@ -736,7 +737,7 @@ public boolean isTableDisabled(byte[] tableName) throws IOException {
     public boolean isTableAvailable(final byte[] tableName) throws IOException {
       final AtomicBoolean available = new AtomicBoolean(true);
       final AtomicInteger regionCount = new AtomicInteger(0);
-      MetaScannerVisitor visitor = new MetaScannerVisitor() {
+      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
         @Override
         public boolean processRow(Result row) throws IOException {
           byte[] value = row.getValue(HConstants.CATALOG_FAMILY,
@@ -847,7 +848,7 @@ private void prefetchRegionCache(final byte[] tableName,
         final byte[] row) {
       // Implement a new visitor for MetaScanner, and use it to walk through
       // the .META.
-      MetaScannerVisitor visitor = new MetaScannerVisitor() {
+      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
         public boolean processRow(Result result) throws IOException {
           try {
             byte[] value = result.getValue(HConstants.CATALOG_FAMILY,
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 6f18f6a6fe9b..24ef4e7597a5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -49,14 +49,12 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
-import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.ipc.ExecRPCInvoker;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.Writables;
 
 /**
  * <p>Used to communicate with a single HBase table.
@@ -435,28 +433,15 @@ public byte[][] getEndKeys() throws IOException {
    * @throws IOException if a remote or network exception occurs
    */
   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
-    final List<byte[]> startKeyList = new ArrayList<byte[]>();
-    final List<byte[]> endKeyList = new ArrayList<byte[]>();
-    MetaScannerVisitor visitor = new MetaScannerVisitor() {
-      public boolean processRow(Result rowResult) throws IOException {
-        byte [] bytes = rowResult.getValue(HConstants.CATALOG_FAMILY,
-          HConstants.REGIONINFO_QUALIFIER);
-        if (bytes == null) {
-          LOG.warn("Null " + HConstants.REGIONINFO_QUALIFIER + " cell in " +
-            rowResult);
-          return true;
-        }
-        HRegionInfo info = Writables.getHRegionInfo(bytes);
-        if (Bytes.equals(info.getTableName(), getTableName())) {
-          if (!(info.isOffline() || info.isSplit())) {
-            startKeyList.add(info.getStartKey());
-            endKeyList.add(info.getEndKey());
-          }
-        }
-        return true;
-      }
-    };
-    MetaScanner.metaScan(configuration, visitor, this.tableName);
+    NavigableMap<HRegionInfo, ServerName> regions = getRegionLocations();
+    final List<byte[]> startKeyList = new ArrayList<byte[]>(regions.size());
+    final List<byte[]> endKeyList = new ArrayList<byte[]>(regions.size());
+
+    for (HRegionInfo region : regions.keySet()) {
+      startKeyList.add(region.getStartKey());
+      endKeyList.add(region.getEndKey());
+    }
+
     return new Pair<byte [][], byte [][]>(
       startKeyList.toArray(new byte[startKeyList.size()][]),
       endKeyList.toArray(new byte[endKeyList.size()][]));
@@ -472,32 +457,18 @@ public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
     final Map<HRegionInfo, HServerAddress> regionMap =
       new TreeMap<HRegionInfo, HServerAddress>();
 
-    MetaScannerVisitor visitor = new MetaScannerVisitor() {
-      public boolean processRow(Result rowResult) throws IOException {
-        HRegionInfo info = Writables.getHRegionInfo(
-            rowResult.getValue(HConstants.CATALOG_FAMILY,
-                HConstants.REGIONINFO_QUALIFIER));
+    final Map<HRegionInfo, ServerName> regionLocations = getRegionLocations();
 
-        if (!(Bytes.equals(info.getTableName(), getTableName()))) {
-          return false;
-        }
-
-        HServerAddress server = new HServerAddress();
-        byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
-            HConstants.SERVER_QUALIFIER);
-        if (value != null && value.length > 0) {
-          String hostAndPort = Bytes.toString(value);
-          server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr(hostAndPort));
-        }
-
-        if (!(info.isOffline() || info.isSplit())) {
-          regionMap.put(new UnmodifyableHRegionInfo(info), server);
-        }
-        return true;
+    for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
+      HServerAddress server = new HServerAddress();
+      ServerName serverName = entry.getValue();
+      if (serverName != null && serverName.getHostAndPort() != null) {
+        server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr(
+            serverName.getHostAndPort()));
       }
+      regionMap.put(entry.getKey(), server);
+    }
 
-    };
-    MetaScanner.metaScan(configuration, visitor, tableName);
     return regionMap;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 2e4aee5c4123..1f1bc2b6d543 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -20,11 +20,13 @@
 
 package org.apache.hadoop.hbase.client;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.NavigableMap;
 import java.util.TreeMap;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -41,9 +43,13 @@
  * Scanner class that contains the <code>.META.</code> table scanning logic
  * and uses a Retryable scanner. Provided visitors will be called
  * for each row.
- * 
+ *
  * Although public visibility, this is not a public-facing API and may evolve in
  * minor releases.
+ *
+ * <p> Note that during concurrent region splits, the scanner might not see
+ * META changes across rows (for parent and daughter entries) consistently.
+ * see HBASE-5986, and {@link BlockingMetaScannerVisitor} for details. </p>
  */
 public class MetaScanner {
   private static final Log LOG = LogFactory.getLog(MetaScanner.class);
@@ -106,7 +112,7 @@ public static void metaScan(Configuration configuration,
    * <code>rowLimit</code> of rows.
    *
    * @param configuration HBase configuration.
-   * @param visitor Visitor object.
+   * @param visitor Visitor object. Closes the visitor before returning.
    * @param tableName User table name in meta table to start scan at.  Pass
    * null if not interested in a particular table.
    * @param row Name of the row at the user table. The scan will start from
@@ -120,14 +126,18 @@ public static void metaScan(Configuration configuration,
       final MetaScannerVisitor visitor, final byte[] tableName,
       final byte[] row, final int rowLimit, final byte[] metaTableName)
       throws IOException {
-    HConnectionManager.execute(new HConnectable<Void>(configuration) {
-      @Override
-      public Void connect(HConnection connection) throws IOException {
-        metaScan(conf, connection, visitor, tableName, row, rowLimit,
-            metaTableName);
-        return null;
-      }
-    });
+    try {
+      HConnectionManager.execute(new HConnectable<Void>(configuration) {
+        @Override
+        public Void connect(HConnection connection) throws IOException {
+          metaScan(conf, connection, visitor, tableName, row, rowLimit,
+              metaTableName);
+          return null;
+        }
+      });
+    } finally {
+      visitor.close();
+    }
   }
 
   private static void metaScan(Configuration configuration, HConnection connection,
@@ -161,7 +171,7 @@ private static void metaScan(Configuration configuration, HConnection connection
             Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
         }
         HRegionInfo regionInfo = Writables.getHRegionInfo(value);
-  
+
         byte[] rowBefore = regionInfo.getStartKey();
         startRow = HRegionInfo.createRegionName(tableName, rowBefore,
             HConstants.ZEROES, false);
@@ -249,9 +259,9 @@ public static List<HRegionInfo> listAllRegions(Configuration conf)
   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
   throws IOException {
     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
-    MetaScannerVisitor visitor = new MetaScannerVisitor() {
+    MetaScannerVisitor visitor = new BlockingMetaScannerVisitor(conf) {
         @Override
-        public boolean processRow(Result result) throws IOException {
+        public boolean processRowInternal(Result result) throws IOException {
           if (result == null || result.isEmpty()) {
             return true;
           }
@@ -280,19 +290,16 @@ public boolean processRow(Result result) throws IOException {
    * @return Map of all user-space regions to servers
    * @throws IOException
    */
-  public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf, final byte [] tablename, final boolean offlined)
-  throws IOException {
+  public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
+      final byte [] tablename, final boolean offlined) throws IOException {
     final NavigableMap<HRegionInfo, ServerName> regions =
       new TreeMap<HRegionInfo, ServerName>();
-    MetaScannerVisitor visitor = new MetaScannerVisitor() {
+    MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) {
       @Override
-      public boolean processRow(Result rowResult) throws IOException {
+      public boolean processRowInternal(Result rowResult) throws IOException {
         HRegionInfo info = Writables.getHRegionInfo(
             rowResult.getValue(HConstants.CATALOG_FAMILY,
                 HConstants.REGIONINFO_QUALIFIER));
-        if (!(Bytes.equals(info.getTableName(), tablename))) {
-          return false;
-        }
         byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
           HConstants.SERVER_QUALIFIER);
         String hostAndPort = null;
@@ -320,7 +327,7 @@ public boolean processRow(Result rowResult) throws IOException {
   /**
    * Visitor class called to process each row of the .META. table
    */
-  public interface MetaScannerVisitor {
+  public interface MetaScannerVisitor extends Closeable {
     /**
      * Visitor method that accepts a RowResult and the meta region location.
      * Implementations can return false to stop the region's loop if it becomes
@@ -332,4 +339,153 @@ public interface MetaScannerVisitor {
      */
     public boolean processRow(Result rowResult) throws IOException;
   }
+
+  public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
+    @Override
+    public void close() throws IOException {
+    }
+  }
+
+  /**
+   * A MetaScannerVisitor that provides a consistent view of the table's
+   * META entries during concurrent splits (see HBASE-5986 for details). This class
+   * does not guarantee ordered traversal of meta entries, and can block until the
+   * META entries for daughters are available during splits.
+   */
+  public static abstract class BlockingMetaScannerVisitor
+    extends MetaScannerVisitorBase {
+
+    private static final int DEFAULT_BLOCKING_TIMEOUT = 10000;
+    private Configuration conf;
+    private TreeSet<byte[]> daughterRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+    private int blockingTimeout;
+    private HTable metaTable;
+
+    public BlockingMetaScannerVisitor(Configuration conf) {
+      this.conf = conf;
+      this.blockingTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
+          DEFAULT_BLOCKING_TIMEOUT);
+    }
+
+    public abstract boolean processRowInternal(Result rowResult) throws IOException;
+
+    @Override
+    public void close() throws IOException {
+      super.close();
+      if (metaTable != null) {
+        metaTable.close();
+        metaTable = null;
+      }
+    }
+
+    public HTable getMetaTable() throws IOException {
+      if (metaTable == null) {
+        metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
+      }
+      return metaTable;
+    }
+
+    @Override
+    public boolean processRow(Result rowResult) throws IOException {
+      HRegionInfo info = Writables.getHRegionInfoOrNull(
+          rowResult.getValue(HConstants.CATALOG_FAMILY,
+              HConstants.REGIONINFO_QUALIFIER));
+      if (info == null) {
+        return true;
+      }
+
+      if (daughterRegions.remove(info.getRegionName())) {
+        return true; //we have already processed this row
+      }
+
+      if (info.isSplitParent()) {
+        /* we have found a parent region which was split. We have to ensure that it's daughters are
+         * seen by this scanner as well, so we block until they are added to the META table. Even
+         * though we are waiting for META entries, ACID semantics in HBase indicates that this
+         * scanner might not see the new rows. So we manually query the daughter rows */
+        HRegionInfo splitA = Writables.getHRegionInfo(rowResult.getValue(HConstants.CATALOG_FAMILY,
+            HConstants.SPLITA_QUALIFIER));
+        HRegionInfo splitB = Writables.getHRegionInfo(rowResult.getValue(HConstants.CATALOG_FAMILY,
+            HConstants.SPLITB_QUALIFIER));
+
+        HTable metaTable = getMetaTable();
+        long start = System.currentTimeMillis();
+        Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
+            splitA.getRegionName());
+        if (resultA != null) {
+          processRow(resultA);
+          daughterRegions.add(splitA.getRegionName());
+        } else {
+          throw new RegionOfflineException("Split daughter region " +
+              splitA.getRegionNameAsString() + " cannot be found in META.");
+        }
+        long rem = blockingTimeout - (System.currentTimeMillis() - start);
+
+        Result resultB = getRegionResultBlocking(metaTable, rem,
+            splitB.getRegionName());
+        if (resultB != null) {
+          processRow(resultB);
+          daughterRegions.add(splitB.getRegionName());
+        } else {
+          throw new RegionOfflineException("Split daughter region " +
+              splitB.getRegionNameAsString() + " cannot be found in META.");
+        }
+      }
+
+      return processRowInternal(rowResult);
+    }
+
+    private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] regionName)
+        throws IOException {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName));
+      }
+      long start = System.currentTimeMillis();
+      while (System.currentTimeMillis() - start < timeout) {
+        Get get = new Get(regionName);
+        Result result = metaTable.get(get);
+        HRegionInfo info = Writables.getHRegionInfoOrNull(
+            result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
+        if (info != null) {
+          return result;
+        }
+        try {
+          Thread.sleep(10);
+        } catch (InterruptedException ex) {
+          Thread.currentThread().interrupt();
+          break;
+        }
+      }
+      return null;
+    }
+  }
+
+  /**
+   * A MetaScannerVisitor for a table. Provides a consistent view of the table's
+   * META entries during concurrent splits (see HBASE-5986 for details). This class
+   * does not guarantee ordered traversal of meta entries, and can block until the
+   * META entries for daughters are available during splits.
+   */
+  public static abstract class TableMetaScannerVisitor extends BlockingMetaScannerVisitor {
+    private byte[] tableName;
+
+    public TableMetaScannerVisitor(Configuration conf, byte[] tableName) {
+      super(conf);
+      this.tableName = tableName;
+    }
+
+    @Override
+    public final boolean processRow(Result rowResult) throws IOException {
+      HRegionInfo info = Writables.getHRegionInfoOrNull(
+          rowResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
+      if (info == null) {
+        return true;
+      }
+      if (!(Bytes.equals(info.getTableName(), tableName))) {
+        return false;
+      }
+      return super.processRow(rowResult);
+    }
+
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 66d618d74db2..d2fd8b423b30 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.MetaScanner;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.executor.ExecutorService;
@@ -1247,7 +1248,7 @@ Pair<HRegionInfo, ServerName> getTableRegionForRow(
       new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
 
     MetaScannerVisitor visitor =
-      new MetaScannerVisitor() {
+      new MetaScannerVisitorBase() {
         @Override
         public boolean processRow(Result data) throws IOException {
           if (data == null || data.size() <= 0) {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 1d3e527f3174..96dd17b2868e 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -66,6 +66,7 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.MetaScanner;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -2183,7 +2184,7 @@ boolean loadMetaEntries() throws IOException {
       return false;
     }
 
-    MetaScannerVisitor visitor = new MetaScannerVisitor() {
+    MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
       int countRecord = 1;
 
       // comparator to sort KeyValues with latest modtime
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 7bfe4cd95831..894b54e14e1a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -17,29 +17,58 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
 
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+
 @Category(LargeTests.class)
 public class TestEndToEndSplitTransaction {
+  private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final Configuration conf = TEST_UTIL.getConfiguration();
 
   @BeforeClass
   public static void beforeAllTests() throws Exception {
@@ -51,7 +80,7 @@ public static void beforeAllTests() throws Exception {
   public static void afterAllTests() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
-  
+
   @Test
   public void testMasterOpsWhileSplitting() throws Exception {
     byte[] tableName = Bytes.toBytes("TestSplit");
@@ -127,6 +156,338 @@ private boolean test(HConnection con, byte[] tableName, byte[] row,
     return true;
   }
 
+  /**
+   * Tests that the client sees meta table changes as atomic during splits
+   */
+  @Test
+  public void testFromClientSideWhileSplitting() throws Throwable {
+    LOG.info("Starting testFromClientSideWhileSplitting");
+    final byte[] TABLENAME = Bytes.toBytes("testFromClientSideWhileSplitting");
+    final byte[] FAMILY = Bytes.toBytes("family");
+
+    //SplitTransaction will update the meta table by offlining the parent region, and adding info
+    //for daughters.
+    HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
+
+    Stoppable stopper = new SimpleStoppable();
+    RegionSplitter regionSplitter = new RegionSplitter(table);
+    RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
+
+    regionChecker.start();
+    regionSplitter.start();
+
+    //wait until the splitter is finished
+    regionSplitter.join();
+    stopper.stop(null);
+
+    if (regionChecker.ex != null) {
+      throw regionChecker.ex;
+    }
+
+    if (regionSplitter.ex != null) {
+      throw regionSplitter.ex;
+    }
+
+    //one final check
+    regionChecker.verify();
+  }
+
+  private static class SimpleStoppable implements Stoppable {
+    volatile boolean stopped = false;
+
+    @Override
+    public void stop(String why) {
+      this.stopped = true;
+    }
+
+    @Override
+    public boolean isStopped() {
+      return stopped;
+    }
+  }
+
+  static class RegionSplitter extends Thread {
+    Throwable ex;
+    HTable table;
+    byte[] tableName, family;
+    HBaseAdmin admin;
+    HTable metaTable;
+    HRegionServer rs;
+
+    RegionSplitter(HTable table) throws IOException {
+      this.table = table;
+      this.tableName = table.getTableName();
+      this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
+      admin = TEST_UTIL.getHBaseAdmin();
+      rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+      metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
+    }
+
+    public void run() {
+      try {
+        Random random = new Random();
+        for (int i=0; i< 5; i++) {
+          NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName, false);
+          if (regions.size() == 0) {
+            continue;
+          }
+          int regionIndex = random.nextInt(regions.size());
+
+          //pick a random region and split it into two
+          HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
+
+          //pick the mid split point
+          int start = 0, end = Integer.MAX_VALUE;
+          if (region.getStartKey().length > 0) {
+            start = Bytes.toInt(region.getStartKey());
+          }
+          if (region.getEndKey().length > 0) {
+            end = Bytes.toInt(region.getEndKey());
+          }
+          int mid = start + ((end - start) / 2);
+          byte[] splitPoint = Bytes.toBytes(mid);
+
+          //put some rows to the regions
+          addData(start);
+          addData(mid);
+
+          flushAndBlockUntilDone(region.getRegionName());
+          compactAndBlockUntilDone(region.getRegionName());
+
+          log("Initiating region split for:" + region.getRegionNameAsString());
+          try {
+            admin.split(region.getRegionName(), splitPoint);
+            //wait until the split is complete
+            blockUntilRegionSplit(50000, region.getRegionName(), true);
+
+          } catch (NotServingRegionException ex) {
+            //ignore
+          }
+        }
+      } catch (Throwable ex) {
+        this.ex = ex;
+      } finally {
+        if (metaTable != null) {
+          IOUtils.closeQuietly(metaTable);
+        }
+      }
+    }
+
+    void addData(int start) throws IOException {
+      for (int i=start; i< start + 100; i++) {
+        Put put = new Put(Bytes.toBytes(i));
+
+        put.add(family, family, Bytes.toBytes(i));
+        table.put(put);
+      }
+      table.flushCommits();
+    }
+
+    void flushAndBlockUntilDone(byte[] regionName) throws IOException, InterruptedException {
+      log("flushing region: " + Bytes.toStringBinary(regionName));
+      admin.flush(regionName);
+      log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
+      Threads.sleepWithoutInterrupt(500);
+      while (rs.cacheFlusher.getFlushQueueSize() > 0) {
+        Threads.sleep(50);
+      }
+    }
+
+    void compactAndBlockUntilDone(byte[] regionName) throws IOException,
+      InterruptedException {
+      log("Compacting region: " + Bytes.toStringBinary(regionName));
+      admin.majorCompact(regionName);
+      log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
+      Threads.sleepWithoutInterrupt(500);
+      while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
+        Threads.sleep(50);
+      }
+    }
+
+    /** bloks until the region split is complete in META and region server opens the daughters */
+    void blockUntilRegionSplit(long timeout, final byte[] regionName, boolean waitForDaughters)
+        throws IOException, InterruptedException {
+      long start = System.currentTimeMillis();
+      log("blocking until region is split:" +  Bytes.toStringBinary(regionName));
+      HRegionInfo daughterA = null, daughterB = null;
+
+      while (System.currentTimeMillis() - start < timeout) {
+        Result result = getRegionRow(regionName);
+        if (result == null) {
+          break;
+        }
+
+        HRegionInfo region = MetaEditor.getHRegionInfo(result);
+        if(region.isSplitParent()) {
+          log("found parent region: " + region.toString());
+          PairOfSameType<HRegionInfo> pair = MetaEditor.getDaughterRegions(result);
+          daughterA = pair.getFirst();
+          daughterB = pair.getSecond();
+          break;
+        }
+        sleep(100);
+      }
+
+      //if we are here, this means the region split is complete or timed out
+      if (waitForDaughters) {
+        long rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsInMeta(rem, daughterA.getRegionName());
+
+        rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsInMeta(rem, daughterB.getRegionName());
+
+        rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsOpenedByRS(rem, daughterA.getRegionName());
+
+        rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsOpenedByRS(rem, daughterB.getRegionName());
+      }
+    }
+
+    Result getRegionRow(byte[] regionName) throws IOException {
+      Get get = new Get(regionName);
+      return metaTable.get(get);
+    }
+
+    void blockUntilRegionIsInMeta(long timeout, byte[] regionName)
+        throws IOException, InterruptedException {
+      log("blocking until region is in META: " + Bytes.toStringBinary(regionName));
+      long start = System.currentTimeMillis();
+      while (System.currentTimeMillis() - start < timeout) {
+        Result result = getRegionRow(regionName);
+        if (result != null) {
+          HRegionInfo info = MetaEditor.getHRegionInfo(result);
+          if (info != null && !info.isOffline()) {
+            log("found region in META: " + Bytes.toStringBinary(regionName));
+            break;
+          }
+        }
+        sleep(10);
+      }
+    }
+
+    void blockUntilRegionIsOpenedByRS(long timeout, byte[] regionName)
+      throws IOException, InterruptedException {
+      log("blocking until region is opened by region server: " + Bytes.toStringBinary(regionName));
+      long start = System.currentTimeMillis();
+      while (System.currentTimeMillis() - start < timeout) {
+        List<HRegion> regions = rs.getOnlineRegions(tableName);
+        for (HRegion region : regions) {
+          if (Bytes.compareTo(region.getRegionName(), regionName) == 0) {
+            log("found region open in RS: " + Bytes.toStringBinary(regionName));
+            return;
+          }
+        }
+        sleep(10);
+      }
+    }
+
+  }
+
+  /**
+   * Checks regions using MetaScanner, MetaReader and HTable methods
+   */
+  static class RegionChecker extends Chore {
+    Configuration conf;
+    byte[] tableName;
+    Throwable ex;
+
+    RegionChecker(Configuration conf, Stoppable stopper, byte[] tableName) {
+      super("RegionChecker", 10, stopper);
+      this.conf = conf;
+      this.tableName = tableName;
+      this.setDaemon(true);
+    }
+
+    /** verify region boundaries obtained from MetaScanner */
+    void verifyRegionsUsingMetaScanner() throws Exception {
+
+      //MetaScanner.allTableRegions()
+      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
+          false);
+      verifyTableRegions(regions.keySet());
+
+      //MetaScanner.listAllRegions()
+      List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, false);
+      verifyTableRegions(Sets.newTreeSet(regionList));
+    }
+
+    /** verify region boundaries obtained from HTable.getStartEndKeys() */
+    void verifyRegionsUsingHTable() throws IOException {
+      HTable table = null;
+      try {
+        //HTable.getStartEndKeys()
+        table = new HTable(conf, tableName);
+        Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
+        verifyStartEndKeys(keys);
+
+        //HTable.getRegionsInfo()
+        Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+        verifyTableRegions(regions.keySet());
+      } finally {
+        IOUtils.closeQuietly(table);
+      }
+    }
+
+    void verify() throws Exception {
+      verifyRegionsUsingMetaScanner();
+      verifyRegionsUsingHTable();
+    }
+
+    void verifyTableRegions(Set<HRegionInfo> regions) {
+      log("Verifying " + regions.size() + " regions");
+
+      byte[][] startKeys = new byte[regions.size()][];
+      byte[][] endKeys = new byte[regions.size()][];
+
+      int i=0;
+      for (HRegionInfo region : regions) {
+        startKeys[i] = region.getStartKey();
+        endKeys[i] = region.getEndKey();
+        i++;
+      }
+
+      Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
+      verifyStartEndKeys(keys);
+    }
+
+    void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
+      byte[][] startKeys = keys.getFirst();
+      byte[][] endKeys = keys.getSecond();
+      assertEquals(startKeys.length, endKeys.length);
+      assertTrue("Found 0 regions for the table", startKeys.length > 0);
+
+      assertArrayEquals("Start key for the first region is not byte[0]",
+          HConstants.EMPTY_START_ROW, startKeys[0]);
+      byte[] prevEndKey = HConstants.EMPTY_START_ROW;
+
+      // ensure that we do not have any gaps
+      for (int i=0; i<startKeys.length; i++) {
+        assertArrayEquals(
+            "Hole in .META. is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
+                + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
+            startKeys[i]);
+        prevEndKey = endKeys[i];
+      }
+      assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
+          endKeys[endKeys.length - 1]);
+    }
+
+    @Override
+    protected void chore() {
+      try {
+        verify();
+      } catch (Throwable ex) {
+        this.ex = ex;
+        stopper.stop("caught exception");
+      }
+    }
+  }
+
+  public static void log(String msg) {
+    LOG.info(msg);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From c3e89b105050996d82a6173c7b3f243e0170e263 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Sat, 26 May 2012 17:14:45 +0000
Subject: [PATCH 0230/1540] HBASE-6002 Possible chance of resource leak in
 HlogSplitter (Chinna rao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342931 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/HLogSplitter.java  | 104 +++++++++++-------
 1 file changed, 65 insertions(+), 39 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index ac29f28e7df0..83a2001452dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -453,51 +453,72 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
       e = RemoteExceptionHandler.checkIOException(e);
       throw e;
     } finally {
-      int n = 0;
-      for (Map.Entry<byte[], Object> logWritersEntry : logWriters.entrySet()) {
-        Object o = logWritersEntry.getValue();
-        long t1 = EnvironmentEdgeManager.currentTimeMillis();
-        if ((t1 - last_report_at) > period) {
-          last_report_at = t;
-          if ((progress_failed == false) && (reporter != null) &&
-              (reporter.progress() == false)) {
-            progress_failed = true;
+      boolean allWritersClosed = false;
+      try {
+        int n = 0;
+        for (Map.Entry<byte[], Object> logWritersEntry : logWriters.entrySet()) {
+          Object o = logWritersEntry.getValue();
+          long t1 = EnvironmentEdgeManager.currentTimeMillis();
+          if ((t1 - last_report_at) > period) {
+            last_report_at = t;
+            if ((progress_failed == false) && (reporter != null) && (reporter.progress() == false)) {
+              progress_failed = true;
+            }
           }
-        }
-        if (o == BAD_WRITER) {
-          continue;
-        }
-        n++;
-        WriterAndPath wap = (WriterAndPath)o;
-        wap.w.close();
-        LOG.debug("Closed " + wap.p);
-        Path dst = getCompletedRecoveredEditsFilePath(wap.p, outputSink
-            .getRegionMaximumEditLogSeqNum(logWritersEntry.getKey()));
-        if (!dst.equals(wap.p) && fs.exists(dst)) {
-          LOG.warn("Found existing old edits file. It could be the "
-              + "result of a previous failed split attempt. Deleting " + dst
-              + ", length=" + fs.getFileStatus(dst).getLen());
-          if (!fs.delete(dst, false)) {
-            LOG.warn("Failed deleting of old " + dst);
-            throw new IOException("Failed deleting of old " + dst);
+          if (o == BAD_WRITER) {
+            continue;
+          }
+          n++;
+          WriterAndPath wap = (WriterAndPath) o;
+          wap.writerClosed = true;
+          wap.w.close();
+          LOG.debug("Closed " + wap.p);
+          Path dst = getCompletedRecoveredEditsFilePath(wap.p,
+              outputSink.getRegionMaximumEditLogSeqNum(logWritersEntry.getKey()));
+          if (!dst.equals(wap.p) && fs.exists(dst)) {
+            LOG.warn("Found existing old edits file. It could be the "
+                + "result of a previous failed split attempt. Deleting " + dst + ", length="
+                + fs.getFileStatus(dst).getLen());
+            if (!fs.delete(dst, false)) {
+              LOG.warn("Failed deleting of old " + dst);
+              throw new IOException("Failed deleting of old " + dst);
+            }
+          }
+          // Skip the unit tests which create a splitter that reads and writes the
+          // data without touching disk. TestHLogSplit#testThreading is an
+          // example.
+          if (fs.exists(wap.p)) {
+            if (!fs.rename(wap.p, dst)) {
+              throw new IOException("Failed renaming " + wap.p + " to " + dst);
+            }
+            LOG.debug("Rename " + wap.p + " to " + dst);
           }
         }
-        // Skip the unit tests which create a splitter that reads and writes the
-        // data without touching disk. TestHLogSplit#testThreading is an
-        // example.
-        if (fs.exists(wap.p)) {
-          if (!fs.rename(wap.p, dst)) {
-            throw new IOException("Failed renaming " + wap.p + " to " + dst);
+        allWritersClosed = true;
+        String msg = "Processed " + editsCount + " edits across " + n + " regions"
+            + " threw away edits for " + (logWriters.size() - n) + " regions" + "; log file="
+            + logPath + " is corrupted = " + isCorrupted + " progress failed = " + progress_failed;
+        LOG.info(msg);
+        status.markComplete(msg);
+      } finally {
+        if (!allWritersClosed) {
+          for (Map.Entry<byte[], Object> logWritersEntry : logWriters.entrySet()) {
+            Object o = logWritersEntry.getValue();
+            if (o != BAD_WRITER) {
+              WriterAndPath wap = (WriterAndPath) o;
+              try {
+                if (!wap.writerClosed) {
+                  wap.writerClosed = true;
+                  wap.w.close();
+                }
+              } catch (IOException e) {
+                LOG.debug("Exception while closing the writer :", e);
+              }
+            }
           }
-          LOG.debug("Rename " + wap.p + " to " + dst);
         }
+        in.close();
       }
-      String msg = "Processed " + editsCount + " edits across " + n + " regions" +
-        " threw away edits for " + (logWriters.size() - n) + " regions" +
-        "; log file=" + logPath + " is corrupted = " + isCorrupted +
-        " progress failed = " + progress_failed;
-      LOG.info(msg);
-      status.markComplete(msg);
     }
     return !progress_failed;
   }
@@ -1349,6 +1370,11 @@ private final static class WriterAndPath {
     long editsWritten = 0;
     /* Number of nanos spent writing to this log */
     long nanosSpent = 0;
+    
+    /* To check whether a close has already been tried on the
+     * writer
+     */
+    boolean writerClosed = false;
 
     WriterAndPath(final Path p, final Writer w) {
       this.p = p;

From 27f945a7178ea80c301e4b6b8bdde0bad9901ed2 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Sat, 26 May 2012 17:32:01 +0000
Subject: [PATCH 0231/1540] HBASE-6050 HLogSplitter renaming recovered.edits
 and CJ removing the parent directory races, making the HBCK to think cluster
 is inconsistent. (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1342934 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/wal/HLogSplitter.java | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 83a2001452dc..440bbb0383c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -570,6 +570,13 @@ public static void moveRecoveredEditsFromTemp(String tmpname,
         if (fs.exists(dst)) {
           fs.delete(dst, false);
         } else {
+          Path regionDir = dst.getParent().getParent();
+          if (!fs.exists(regionDir)) {
+            // See HBASE-6050.
+            LOG.warn("Could not move recovered edits from " + src
+                + " to destination " + regionDir + " as it doesn't exist.");
+            continue;
+          }
           Path dstdir = dst.getParent();
           if (!fs.exists(dstdir)) {
             if (!fs.mkdirs(dstdir)) LOG.warn("mkdir failed on " + dstdir);

From ab125ebe56ea2f40826c63afbb7496134a72c743 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 28 May 2012 17:17:08 +0000
Subject: [PATCH 0232/1540] HBASE-5916 RS restart just before master
 intialization we make the cluster non operative(RajeshBabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1343326 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 27 +++++++++----------
 .../apache/hadoop/hbase/master/HMaster.java   | 16 ++++++-----
 .../hadoop/hbase/master/MasterFileSystem.java |  6 ++++-
 .../hadoop/hbase/master/ServerManager.java    | 24 +++++++++++++++--
 .../TestRSKilledWhenMasterInitializing.java   | 11 +++++---
 5 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 69bfcc073d2e..772c6c857996 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -333,7 +333,7 @@ void cleanoutUnassigned() throws IOException, KeeperException {
    * @throws KeeperException
    * @throws InterruptedException
    */
-  void joinCluster(final Set<ServerName> onlineServers) throws IOException,
+  void joinCluster() throws IOException,
       KeeperException, InterruptedException {
     // Concurrency note: In the below the accesses on regionsInTransition are
     // outside of a synchronization block where usually all accesses to RIT are
@@ -345,7 +345,7 @@ void joinCluster(final Set<ServerName> onlineServers) throws IOException,
 
     // Scan META to build list of existing regions, servers, and assignment
     // Returns servers who have not checked in (assumed dead) and their regions
-    Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers = rebuildUserRegions(onlineServers);
+    Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers = rebuildUserRegions();
 
     processDeadServersAndRegionsInTransition(deadServers);
 
@@ -355,16 +355,6 @@ void joinCluster(final Set<ServerName> onlineServers) throws IOException,
     recoverTableInEnablingState(this.enablingTables, isWatcherCreated);
   }
 
-  /**
-   * Only used for tests
-   * @throws IOException
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  void joinCluster() throws IOException, KeeperException, InterruptedException {
-    joinCluster(serverManager.getOnlineServers().keySet());
-  }
-
   /**
    * Process all regions that are in transition up in zookeeper.  Used by
    * master joining an already running cluster.
@@ -2456,11 +2446,12 @@ boolean waitUntilNoRegionsInTransition(final long timeout, Set<HRegionInfo> regi
    *         in META
    * @throws IOException
    */
-  Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions(
-      final Set<ServerName> onlineServers)
-  throws IOException, KeeperException {
+  Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOException,
+      KeeperException {
     // Region assignment from META
     List<Result> results = MetaReader.fullScan(this.catalogTracker);
+    // Get any new but slow to checkin region server that joined the cluster
+    Set<ServerName> onlineServers = serverManager.getOnlineServers().keySet();    
     // Map of offline servers and their regions to be returned
     Map<ServerName, List<Pair<HRegionInfo,Result>>> offlineServers =
       new TreeMap<ServerName, List<Pair<HRegionInfo, Result>>>();
@@ -2658,8 +2649,14 @@ private void processDeadServersAndRecoverLostRegions(
       Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
       List<String> nodes) throws IOException, KeeperException {
     if (null != deadServers) {
+      Set<ServerName> actualDeadServers = this.serverManager.getDeadServers();
       for (Map.Entry<ServerName, List<Pair<HRegionInfo, Result>>> deadServer : 
         deadServers.entrySet()) {
+        // skip regions of dead servers because SSH will process regions during rs expiration.
+        // see HBASE-5916
+        if (actualDeadServers.contains(deadServer.getKey())) {
+          continue;
+        }
         List<Pair<HRegionInfo, Result>> regions = deadServer.getValue();
         for (Pair<HRegionInfo, Result> region : regions) {
           HRegionInfo regionInfo = region.getFirst();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index d2fd8b423b30..ea0db2284bf0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -516,11 +516,9 @@ private void finishInitialization(MonitoredTask status)
     }
 
     this.assignmentManager.startTimeOutMonitor();
-    Set<ServerName> onlineServers = new HashSet<ServerName>(serverManager
-        .getOnlineServers().keySet());
     // TODO: Should do this in background rather than block master startup
     status.setStatus("Splitting logs after master startup");
-    splitLogAfterStartup(this.fileSystemManager, onlineServers);
+    splitLogAfterStartup(this.fileSystemManager);
 
     // Make sure root and meta assigned before proceeding.
     assignRootAndMeta(status);
@@ -536,7 +534,7 @@ private void finishInitialization(MonitoredTask status)
 
     // Fixup assignment manager status
     status.setStatus("Starting assignment manager");
-    this.assignmentManager.joinCluster(onlineServers);
+    this.assignmentManager.joinCluster();
 
     this.balancer.setClusterStatus(getClusterStatus());
     this.balancer.setMasterServices(this);
@@ -557,6 +555,11 @@ private void finishInitialization(MonitoredTask status)
     LOG.info("Master has completed initialization");
     initialized = true;
 
+    // clear the dead servers with same host name and port of online server because we are not
+    // removing dead server with same hostname and port of rs which is trying to check in before
+    // master initialization. See HBASE-5916.
+    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
+    
     if (this.cpHost != null) {
       // don't let cp initialization errors kill the master
       try {
@@ -580,9 +583,8 @@ protected void startCatalogJanitorChore() {
    * @param mfs
    * @param onlineServers
    */
-  protected void splitLogAfterStartup(final MasterFileSystem mfs,
-      Set<ServerName> onlineServers) {
-    mfs.splitLogAfterStartup(onlineServers);
+  protected void splitLogAfterStartup(final MasterFileSystem mfs) {
+    mfs.splitLogAfterStartup();
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index d1e7d7eca97b..05f06cbde7d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -186,7 +186,7 @@ public String getClusterId() {
    * @param onlineServers Set of online servers keyed by
    * {@link ServerName}
    */
-  void splitLogAfterStartup(final Set<ServerName> onlineServers) {
+  void splitLogAfterStartup() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
         HLog.SPLIT_SKIP_ERRORS_DEFAULT);
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
@@ -195,6 +195,10 @@ void splitLogAfterStartup(final Set<ServerName> onlineServers) {
       try {
         if (!this.fs.exists(logsDirPath)) return;
         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
+        // Get online servers after getting log folders to avoid log folder deletion of newly
+        // checked in region servers . see HBASE-5916
+        Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
+            .keySet();
 
         if (logFolders == null || logFolders.length == 0) {
           LOG.debug("No log files to split, proceeding...");
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 283713c5d21d..0ed32006ce14 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -191,7 +191,10 @@ void checkAlreadySameHostPort(final ServerName serverName)
           existingServer + " looks stale, new server:" + serverName);
         expireServer(existingServer);
       }
-      throw new PleaseHoldException(message);
+      if (services.isServerShutdownHandlerEnabled()) {
+        // master has completed the initialization
+        throw new PleaseHoldException(message);
+      }
     }
   }
 
@@ -239,7 +242,10 @@ private void checkIsDead(final ServerName serverName, final String what)
       throw new YouAreDeadException(message);
     }
 
-    if (this.deadservers.cleanPreviousInstance(serverName)) {
+    // remove dead server with same hostname and port of newly checking in rs after master
+    // initialization.See HBASE-5916 for more information.
+    if ((this.services == null || ((HMaster) this.services).isInitialized())
+        && this.deadservers.cleanPreviousInstance(serverName)) {
       // This server has now become alive after we marked it as dead.
       // We removed it's previous entry from the dead list to reflect it.
       LOG.debug(what + ":" + " Server " + serverName + " came back up," +
@@ -665,4 +671,18 @@ public void stop() {
       }
     }
   }
+    
+  /**
+   * To clear any dead server with same host name and port of any online server
+   */
+  void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
+    ServerName sn = null;
+    for (ServerName serverName : getOnlineServersList()) {
+      while ((sn = ServerName.
+          findServerWithSameHostnamePort(this.deadservers, serverName)) != null) {
+        this.deadservers.remove(sn);
+      }
+    }
+  }
+
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index 8f8018dc77ca..a0ed0bd83a75 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -49,6 +49,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -95,9 +97,8 @@ public TestingMaster(Configuration conf) throws IOException,
     }
 
     @Override
-    protected void splitLogAfterStartup(MasterFileSystem mfs,
-        Set<ServerName> onlineServers) {
-      super.splitLogAfterStartup(mfs, onlineServers);
+    protected void splitLogAfterStartup(MasterFileSystem mfs) {
+      super.splitLogAfterStartup(mfs);
       logSplit = true;
       // If "TestingMaster.sleep" is set, sleep after log split.
       if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
@@ -212,6 +213,10 @@ public void testCorrectnessWhenMasterFailOver() throws Exception {
     while (serverManager.areDeadServersInProgress()) {
       Thread.sleep(100);
     }
+    // Create a ZKW to use in the test
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTUTIL);
+    ZKAssign.blockUntilNoRIT(zkw);
+
     table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
     resultScanner = table.getScanner(new Scan());
     count = 0;

From ecf4f801b7fe3f31052e763b926f7d87f4b12db1 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 28 May 2012 17:50:07 +0000
Subject: [PATCH 0233/1540] HBASE-6118 Add a testcase for HBASE-6065 (Ashutosh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1343336 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/TestWALReplay.java | 87 +++++++++++++++++++
 1 file changed, 87 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 4fdf34dc4cdc..e9517dce4f42 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -31,6 +31,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
@@ -47,6 +48,7 @@
 import org.apache.hadoop.hbase.util.EnvironmentEdge;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Strings;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -523,6 +525,91 @@ protected boolean internalFlushcache(
     });
   }
 
+  @Test
+  public void testSequentialEditLogSeqNum() throws IOException {
+    final String tableNameStr = "testSequentialEditLogSeqNum";
+    final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
+    final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
+    deleteDir(basedir);
+    final byte[] rowName = Bytes.toBytes(tableNameStr);
+    final int countPerFamily = 10;
+    final HTableDescriptor htd = createBasic1FamilyHTD(tableNameStr);
+
+    // Mock the HLog
+    MockHLog wal = createMockWAL(this.conf);
+    
+    HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
+    long seqid = region.initialize();
+    // HRegionServer usually does this. It knows the largest seqid across all
+    // regions.
+    wal.setSequenceNumber(seqid);
+    for (HColumnDescriptor hcd : htd.getFamilies()) {
+      addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
+    }
+    // get the seq no after first set of entries.
+    long sequenceNumber = wal.getSequenceNumber();
+
+    // Let us flush the region
+    // But this time completeflushcache is not yet done
+    region.flushcache();
+    for (HColumnDescriptor hcd : htd.getFamilies()) {
+      addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
+    }
+    long lastestSeqNumber = wal.getSequenceNumber();
+    // get the current seq no
+    wal.doCompleteCacheFlush = true;
+    // allow complete cache flush with the previous seq number got after first
+    // set of edits.
+    wal.completeCacheFlush(hri.getEncodedNameAsBytes(), hri.getTableName(), sequenceNumber, false);
+    wal.close();
+    FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
+    HLogSplitter.splitLogFileToTemp(hbaseRootDir, hbaseRootDir + "/temp", listStatus[0], this.fs,
+        this.conf, null);
+    FileStatus[] listStatus1 = this.fs.listStatus(new Path(hbaseRootDir + "/temp/" + tableNameStr
+        + "/" + hri.getEncodedName() + "/recovered.edits"));
+    int editCount = 0;
+    for (FileStatus fileStatus : listStatus1) {
+      editCount = Integer.parseInt(fileStatus.getPath().getName());
+    }
+    // The sequence number should be same 
+    assertEquals(
+        "The sequence number of the recoverd.edits and the current edit seq should be same",
+        lastestSeqNumber, editCount);
+  }
+  
+   static class MockHLog extends HLog {
+    boolean doCompleteCacheFlush = false;
+
+    public MockHLog(FileSystem fs, Path dir, Path oldLogDir, Configuration conf) throws IOException {
+      super(fs, dir, oldLogDir, conf);
+    }
+
+    @Override
+    public void completeCacheFlush(byte[] encodedRegionName, byte[] tableName, long logSeqId,
+        boolean isMetaRegion) throws IOException {
+      if (!doCompleteCacheFlush) {
+        return;
+      }
+      super.completeCacheFlush(encodedRegionName, tableName, logSeqId, isMetaRegion);
+    }
+  }
+
+  private HTableDescriptor createBasic1FamilyHTD(final String tableName) {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
+    htd.addFamily(a);
+    return htd;
+  }
+  
+  private MockHLog createMockWAL(Configuration conf) throws IOException {
+    MockHLog wal = new MockHLog(FileSystem.get(conf), logDir, oldLogDir, conf);
+    // Set down maximum recovery so we dfsclient doesn't linger retrying something
+    // long gone.
+    HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
+    return wal;
+  }
+  
+    
   // Flusher used in this test.  Keep count of how often we are called and
   // actually run the flush inside here.
   class TestFlusher implements FlushRequester {

From a9a4dbe59c85eb5a82fcad445437eaff9d9f5ea4 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 29 May 2012 16:19:57 +0000
Subject: [PATCH 0234/1540] HBASE-6088 Region splitting not happened for long
 time due to ZK exception while creating RS_ZK_SPLITTING node (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1343818 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/SplitTransaction.java  | 41 +++++++---
 .../TestSplitTransactionOnCluster.java        | 79 +++++++++++++++++++
 2 files changed, 111 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index aa8bf7210824..cbfa1436e7b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -107,6 +107,10 @@ public class SplitTransaction {
    * we need to rollback.
    */
   enum JournalEntry {
+    /**
+     * Before creating node in splitting state.
+     */
+    STARTED_SPLITTING,
     /**
      * Set region as in transition, set it into SPLITTING state.
      */
@@ -232,6 +236,7 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
       server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout",
           this.fileSplitTimeout);
 
+    this.journal.add(JournalEntry.STARTED_SPLITTING);
     // Set ephemeral SPLITTING znode up in zk.  Mocked servers sometimes don't
     // have zookeeper so don't do zk stuff if server or zookeeper is null
     if (server != null && server.getZooKeeper() != null) {
@@ -732,9 +737,16 @@ public boolean rollback(final Server server, final RegionServerServices services
     while (iterator.hasPrevious()) {
       JournalEntry je = iterator.previous();
       switch(je) {
+      
+      case STARTED_SPLITTING:
+        if (server != null && server.getZooKeeper() != null) {
+          cleanZK(server, this.parent.getRegionInfo(), false);
+        }
+        break;
+      
       case SET_SPLITTING_IN_ZK:
         if (server != null && server.getZooKeeper() != null) {
-          cleanZK(server, this.parent.getRegionInfo());
+          cleanZK(server, this.parent.getRegionInfo(), true);
         }
         break;
 
@@ -827,11 +839,15 @@ static void cleanupAnySplitDetritus(final HRegion r) throws IOException {
     LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
   }
 
-  private static void cleanZK(final Server server, final HRegionInfo hri) {
+  private static void cleanZK(final Server server, final HRegionInfo hri, boolean abort) {
     try {
       // Only delete if its in expected state; could have been hijacked.
       ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
         EventType.RS_ZK_REGION_SPLITTING);
+    } catch (KeeperException.NoNodeException nn) {
+      if (abort) {
+        server.abort("Failed cleanup of " + hri.getRegionNameAsString(), nn);
+      }
     } catch (KeeperException e) {
       server.abort("Failed cleanup of " + hri.getRegionNameAsString(), e);
     }
@@ -851,9 +867,8 @@ private static void cleanZK(final Server server, final HRegionInfo hri) {
    * @throws KeeperException 
    * @throws IOException 
    */
-  private static int createNodeSplitting(final ZooKeeperWatcher zkw,
-      final HRegionInfo region, final ServerName serverName)
-  throws KeeperException, IOException {
+  int createNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo region,
+      final ServerName serverName) throws KeeperException, IOException {
     LOG.debug(zkw.prefix("Creating ephemeral node for " +
       region.getEncodedName() + " in SPLITTING state"));
     RegionTransitionData data =
@@ -912,10 +927,18 @@ private static int transitionNodeSplit(ZooKeeperWatcher zkw,
       znodeVersion, payload);
   }
 
-  private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
-      final HRegionInfo parent,
-      final ServerName serverName, final int version)
-  throws KeeperException, IOException {
+  /**
+   * 
+   * @param zkw zk reference
+   * @param parent region to be transitioned to splitting
+   * @param serverName server event originates from
+   * @param version znode version
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException
+   * @throws IOException
+   */
+  int transitionNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo parent,
+      final ServerName serverName, final int version) throws KeeperException, IOException {
     return ZKAssign.transitionNode(zkw, parent, serverName,
       EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index f225496dc97e..195213b2bcf3 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -22,6 +22,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,6 +43,7 @@
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.apache.zookeeper.data.Stat;
@@ -514,6 +516,83 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
     }
   }
 
+  /**
+   * While transitioning node from RS_ZK_REGION_SPLITTING to
+   * RS_ZK_REGION_SPLITTING during region split,if zookeper went down split always
+   * fails for the region. HBASE-6088 fixes this scenario.
+   * This test case is to test the znode is deleted(if created) or not in roll back.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws KeeperException
+   */
+  @Test
+  public void testSplitBeforeSettingSplittingInZK() throws IOException,
+      InterruptedException, KeeperException {
+    testSplitBeforeSettingSplittingInZK(true);
+    testSplitBeforeSettingSplittingInZK(false);
+  }
+
+  private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws IOException,
+      KeeperException {
+    final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
+    HBaseAdmin admin = TESTING_UTIL.getHBaseAdmin();
+    try {
+      // Create table then get the single region for our new table.
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor("cf"));
+      admin.createTable(htd);
+
+      List<HRegion> regions = cluster.getRegions(tableName);
+      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+      SplitTransaction st = null;
+      if (nodeCreated) {
+        st = new MockedSplitTransaction(regions.get(0), null) {
+          @Override
+          int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent,
+              ServerName serverName, int version) throws KeeperException, IOException {
+            throw new IOException();
+          }
+        };
+      } else {
+        st = new MockedSplitTransaction(regions.get(0), null) {
+          @Override
+          int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
+              throws KeeperException, IOException {
+            throw new IOException();
+          }
+        };
+      }
+      try {
+        st.execute(regionServer, regionServer);
+      } catch (IOException e) {
+        String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
+            .getRegionInfo().getEncodedName());
+        if (nodeCreated) {
+          assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+        } else {
+          assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+        }
+        assertTrue(st.rollback(regionServer, regionServer));
+        assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+      }
+    } finally {
+      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+        admin.disableTable(tableName);
+        admin.deleteTable(tableName);
+      }
+    }
+  }
+  
+  public static class MockedSplitTransaction extends SplitTransaction {
+
+    public MockedSplitTransaction(HRegion r, byte[] splitrow) {
+      super(r, splitrow);
+    }
+
+  }
+  
   private MockMasterWithoutCatalogJanitor abortAndWaitForMaster() 
   throws IOException, InterruptedException {
     cluster.abortMaster(0);

From 725dbf9946a6c5fcf304176bc6b6ea868467e06a Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 29 May 2012 16:22:47 +0000
Subject: [PATCH 0235/1540] HBASE-6115 NullPointerException is thrown when root
 and meta table regions are assigning to another RS. (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1343820 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HConnectionManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 5f950583954e..7090d3050611 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -819,7 +819,7 @@ private HRegionLocation locateRegion(final byte [] tableName,
       ensureZookeeperTrackers();
       if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
         try {
-          ServerName servername = this.rootRegionTracker.getRootRegionLocation();
+          ServerName servername = this.rootRegionTracker.waitRootRegionLocation(this.rpcTimeout);
           LOG.debug("Looked up root region location, connection=" + this +
             "; serverName=" + ((servername == null)? "": servername.toString()));
           if (servername == null) return null;

From 677adcee09923b2143a7a8ecdbf4a672bb6e370c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 29 May 2012 16:41:22 +0000
Subject: [PATCH 0236/1540] HBASE-6095 ActiveMasterManager NullPointerException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1343838 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/ActiveMasterManager.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 96754d2e003f..178edb338ba1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -237,7 +237,7 @@ public void stop() {
       byte [] bytes =
         ZKUtil.getDataAndWatch(watcher, watcher.masterAddressZNode);
       // TODO: redo this to make it atomic (only added for tests)
-      ServerName master = ServerName.parseVersionedServerName(bytes);
+      ServerName master = bytes == null ? null : ServerName.parseVersionedServerName(bytes);
       if (master != null &&  master.equals(this.sn)) {
         ZKUtil.deleteNode(watcher, watcher.masterAddressZNode);
       }

From ed21866e2d814476c8daa162e521a6d656a8780b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 29 May 2012 20:51:40 +0000
Subject: [PATCH 0237/1540] HBASE-6126 Fix broke TestLocalHBaseCluster in
 0.92/0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1343977 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
index 03464f4523de..f4ef1cd97050 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hbase;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 
@@ -29,7 +29,6 @@
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
 import org.apache.zookeeper.KeeperException;
-
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -45,6 +44,7 @@ public class TestLocalHBaseCluster {
   @Test
   public void testLocalHBaseCluster() throws Exception {
     Configuration conf = TEST_UTIL.getConfiguration();
+    conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir("hbase.rootdir").toString());
     MiniZooKeeperCluster zkCluster = TEST_UTIL.startMiniZKCluster();
     conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkCluster.getClientPort()));
     LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1, MyHMaster.class,

From 16f58bfebbec7a763afe007cde0a00dfaebf9f5c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 29 May 2012 23:58:13 +0000
Subject: [PATCH 0238/1540] HBASE-6107 Distributed log splitting hangs even
 there is no task under /hbase/splitlog

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344055 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/SplitLogManager.java  | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 84dd61b87ca0..e13bff8bc5ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -321,6 +321,23 @@ private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {
               + " scheduled=" + batch.installed
               + " done=" + batch.done
               + " error=" + batch.error);
+          int remaining = batch.installed - (batch.done + batch.error);
+          int actual = activeTasks(batch);
+          if (remaining != actual) {
+            LOG.warn("Expected " + remaining
+              + " active tasks, but actually there are " + actual);
+          }
+          int remainingInZK = remainingTasksInZK();
+          if (remainingInZK >= 0 && actual > remainingInZK) {
+            LOG.warn("Expected at least" + actual
+              + " tasks in ZK, but actually there are " + remainingInZK);
+          }
+          if (remainingInZK == 0 || actual == 0) {
+            LOG.warn("No more task remaining (ZK or task map), splitting "
+              + "should have completed. Remaining tasks in ZK " + remainingInZK
+              + ", active tasks in map " + actual);
+            return;
+          }
           batch.wait(100);
           if (stopper.isStopped()) {
             LOG.warn("Stopped while waiting for log splits to be completed");
@@ -335,6 +352,35 @@ private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {
     }
   }
 
+  private int activeTasks(final TaskBatch batch) {
+    int count = 0;
+    for (Task t: tasks.values()) {
+      if (t.batch == batch && t.status == TerminationStatus.IN_PROGRESS) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  private int remainingTasksInZK() {
+    int count = 0;
+    try {
+      List<String> tasks =
+        ZKUtil.listChildrenNoWatch(watcher, watcher.splitLogZNode);
+      if (tasks != null) {
+        for (String t: tasks) {
+          if (!ZKSplitLog.isRescanNode(watcher, t)) {
+            count++;
+          }
+        }
+      }
+    } catch (KeeperException ke) {
+      LOG.warn("Failed to check remaining tasks", ke);
+      count = -1;
+    }
+    return count;
+  }
+
   private void setDone(String path, TerminationStatus status) {
     Task task = tasks.get(path);
     if (task == null) {

From 4a80757e6165743687427de296dee18eea7d19e9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 04:43:07 +0000
Subject: [PATCH 0239/1540] HBASE-6133 TestRestartCluster failing in 0.92

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344097 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/TestRestartCluster.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index ed3208c5ea75..3b8b6c6a8cb1 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -22,7 +22,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-import java.io.IOException;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -32,11 +31,11 @@
 import org.apache.hadoop.hbase.client.MetaScanner;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -91,6 +90,9 @@ public class TestRestartCluster {
   @Test (timeout=300000)
   public void testClusterRestart() throws Exception {
     UTIL.startMiniCluster(3);
+    while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
+      Threads.sleep(1);
+    }
     LOG.info("\n\nCreating tables");
     for(byte [] TABLE : TABLES) {
       UTIL.createTable(TABLE, FAMILY);

From f3b67c3f30f0c3fe3a10b8997dddfb7e21ff04a3 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 30 May 2012 05:26:26 +0000
Subject: [PATCH 0240/1540] HBASE-6114. CacheControl flags should be tunable
 per table schema per CF

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344106 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HColumnDescriptor.java       | 116 +++++++-
 .../hadoop/hbase/io/hfile/CacheConfig.java    |  15 +-
 .../hadoop/hbase/io/hfile/HFileBlock.java     |   2 +-
 .../hadoop/hbase/regionserver/Store.java      |   2 +-
 .../TestCacheOnWriteInSchema.java             | 271 ++++++++++++++++++
 5 files changed, 398 insertions(+), 8 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 324ec46f34d5..90701522267e 100644
--- a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -65,7 +65,11 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
   public static final String DATA_BLOCK_ENCODING =
       "DATA_BLOCK_ENCODING";
   public static final String BLOCKCACHE = "BLOCKCACHE";
-  
+  public static final String CACHE_DATA_ON_WRITE = "CACHE_DATA_ON_WRITE";
+  public static final String CACHE_INDEX_ON_WRITE = "CACHE_INDEX_ON_WRITE";
+  public static final String CACHE_BLOOMS_ON_WRITE = "CACHE_BLOOMS_ON_WRITE";
+  public static final String EVICT_BLOCKS_ON_CLOSE = "EVICT_BLOCKS_ON_CLOSE";
+
   /**
    * Size of storefile/hfile 'blocks'.  Default is {@link #DEFAULT_BLOCKSIZE}.
    * Use smaller block sizes for faster random-access at expense of larger
@@ -129,6 +133,18 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
    */
   public static final boolean DEFAULT_BLOCKCACHE = true;
 
+  /**
+   * Default setting for whether to cache data blocks on write if block caching
+   * is enabled.
+   */
+  public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
+  
+  /**
+   * Default setting for whether to cache index blocks on write if block
+   * caching is enabled.
+   */
+  public static final boolean DEFAULT_CACHE_INDEX_ON_WRITE = false;
+
   /**
    * Default size of blocks in files stored to the filesytem (hfiles).
    */
@@ -139,6 +155,12 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
    */
   public static final String DEFAULT_BLOOMFILTER = StoreFile.BloomType.NONE.toString();
 
+  /**
+   * Default setting for whether to cache bloom filter blocks on write if block
+   * caching is enabled.
+   */
+  public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
+
   /**
    * Default time to live of cell contents.
    */
@@ -149,6 +171,12 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
    */
   public static final int DEFAULT_REPLICATION_SCOPE = HConstants.REPLICATION_SCOPE_LOCAL;
 
+  /**
+   * Default setting for whether to evict cached blocks from the blockcache on
+   * close.
+   */
+  public static final boolean DEFAULT_EVICT_BLOCKS_ON_CLOSE = false;
+
   private final static Map<String, String> DEFAULT_VALUES = new HashMap<String, String>();
   static {
       DEFAULT_VALUES.put(BLOOMFILTER, DEFAULT_BLOOMFILTER);
@@ -165,6 +193,14 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
           String.valueOf(DEFAULT_ENCODE_ON_DISK));
       DEFAULT_VALUES.put(DATA_BLOCK_ENCODING,
           String.valueOf(DEFAULT_DATA_BLOCK_ENCODING));
+      DEFAULT_VALUES.put(CACHE_DATA_ON_WRITE,
+          String.valueOf(DEFAULT_CACHE_DATA_ON_WRITE));
+      DEFAULT_VALUES.put(CACHE_INDEX_ON_WRITE,
+          String.valueOf(DEFAULT_CACHE_INDEX_ON_WRITE));
+      DEFAULT_VALUES.put(CACHE_BLOOMS_ON_WRITE,
+          String.valueOf(DEFAULT_CACHE_BLOOMS_ON_WRITE));
+      DEFAULT_VALUES.put(EVICT_BLOCKS_ON_CLOSE,
+          String.valueOf(DEFAULT_EVICT_BLOCKS_ON_CLOSE));
   }
 
   // Column family name
@@ -746,6 +782,84 @@ public HColumnDescriptor setScope(int scope) {
     return setValue(REPLICATION_SCOPE, Integer.toString(scope));
   }
 
+  /**
+   * @return true if we should cache data blocks on write
+   */
+  public boolean shouldCacheDataOnWrite() {
+    String value = getValue(CACHE_DATA_ON_WRITE);
+    if (value != null) {
+      return Boolean.valueOf(value).booleanValue();
+    }
+    return DEFAULT_CACHE_DATA_ON_WRITE;
+  }
+
+  /**
+   * @param value true if we should cache data blocks on write
+   * @return this (for chained invocation)
+   */
+  public HColumnDescriptor setCacheDataOnWrite(boolean value) {
+    return setValue(CACHE_DATA_ON_WRITE, Boolean.toString(value));
+  }
+
+  /**
+   * @return true if we should cache index blocks on write
+   */
+  public boolean shouldCacheIndexesOnWrite() {
+    String value = getValue(CACHE_INDEX_ON_WRITE);
+    if (value != null) {
+      return Boolean.valueOf(value).booleanValue();
+    }
+    return DEFAULT_CACHE_INDEX_ON_WRITE;
+  }
+
+  /**
+   * @param value true if we should cache index blocks on write
+   * @return this (for chained invocation)
+   */
+  public HColumnDescriptor setCacheIndexesOnWrite(boolean value) {
+    return setValue(CACHE_INDEX_ON_WRITE, Boolean.toString(value));
+  }
+
+  /**
+   * @return true if we should cache bloomfilter blocks on write
+   */
+  public boolean shouldCacheBloomsOnWrite() {
+    String value = getValue(CACHE_BLOOMS_ON_WRITE);
+    if (value != null) {
+      return Boolean.valueOf(value).booleanValue();
+    }
+    return DEFAULT_CACHE_BLOOMS_ON_WRITE;
+  }
+
+  /**
+   * @param value true if we should cache bloomfilter blocks on write
+   * @return this (for chained invocation)
+   */
+  public HColumnDescriptor setCacheBloomsOnWrite(boolean value) {
+    return setValue(CACHE_BLOOMS_ON_WRITE, Boolean.toString(value));
+  }
+
+  /**
+   * @return true if we should evict cached blocks from the blockcache on
+   * close
+   */
+  public boolean shouldEvictBlocksOnClose() {
+    String value = getValue(EVICT_BLOCKS_ON_CLOSE);
+    if (value != null) {
+      return Boolean.valueOf(value).booleanValue();
+    }
+    return DEFAULT_EVICT_BLOCKS_ON_CLOSE;
+  }
+
+  /**
+   * @param value true if we should evict cached blocks from the blockcache on
+   * close
+   * @return this (for chained invocation)
+   */
+  public HColumnDescriptor setEvictBlocksOnClose(boolean value) {
+    return setValue(EVICT_BLOCKS_ON_CLOSE, Boolean.toString(value));
+  }
+
   /**
    * @see java.lang.Object#toString()
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 3c2e7229bfb8..70dbbbaf9734 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -115,13 +115,18 @@ public class CacheConfig {
    */
   public CacheConfig(Configuration conf, HColumnDescriptor family) {
     this(CacheConfig.instantiateBlockCache(conf),
-        family.isBlockCacheEnabled(), family.isInMemory(),
-        conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
+        family.isBlockCacheEnabled(),
+        family.isInMemory(),
+        // For the following flags we enable them regardless of per-schema settings
+        // if they are enabled in the global configuration.
+        conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
+            DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
-            DEFAULT_CACHE_INDEXES_ON_WRITE),
+            DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
-            DEFAULT_CACHE_BLOOMS_ON_WRITE),
-        conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
+            DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
+        conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
+            DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE)
      );
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 65acd102fa3e..d0904aaf8fb7 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -287,7 +287,7 @@ public short getDataBlockEncodingId() {
    * @return the on-disk size of the block with header size included. This
    * includes the header, the data and the checksum data.
    */
-  int getOnDiskSizeWithHeader() {
+  public int getOnDiskSizeWithHeader() {
     return onDiskSizeWithoutHeader + headerSize();
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 6f32235285ba..22b3c6970c00 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -820,7 +820,7 @@ private StoreFile.Writer createWriterInTmp(int maxKeyCount)
    * @param isCompaction whether we are creating a new file in a compaction
    * @return Writer for a new StoreFile in the tmp dir.
    */
-  private StoreFile.Writer createWriterInTmp(int maxKeyCount,
+  public StoreFile.Writer createWriterInTmp(int maxKeyCount,
     Compression.Algorithm compression, boolean isCompaction)
   throws IOException {
     final CacheConfig writerCacheConf;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
new file mode 100644
index 000000000000..4e5aff6fc51b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFileReaderV2;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests {@link HFile} cache-on-write functionality for data blocks, non-root
+ * index blocks, and Bloom filter blocks, as specified by the column family. 
+ */
+@RunWith(Parameterized.class)
+@Category(MediumTests.class)
+public class TestCacheOnWriteInSchema {
+
+  private static final Log LOG = LogFactory.getLog(TestCacheOnWriteInSchema.class);
+
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final String DIR = TEST_UTIL.getDataTestDir("TestCacheOnWriteInSchema").toString();
+  private static final byte [] table = Bytes.toBytes("table");
+  private static byte [] family = Bytes.toBytes("family");
+  private static final int NUM_KV = 25000;
+  private static final Random rand = new Random(12983177L);
+  /** The number of valid key types possible in a store file */
+  private static final int NUM_VALID_KEY_TYPES =
+      KeyValue.Type.values().length - 2;
+
+  private static enum CacheOnWriteType {
+    DATA_BLOCKS(BlockType.DATA, BlockType.ENCODED_DATA),
+    BLOOM_BLOCKS(BlockType.BLOOM_CHUNK),
+    INDEX_BLOCKS(BlockType.LEAF_INDEX, BlockType.INTERMEDIATE_INDEX);
+
+    private final BlockType blockType1;
+    private final BlockType blockType2;
+
+    private CacheOnWriteType(BlockType blockType) {
+      this(blockType, blockType);
+    }
+
+    private CacheOnWriteType(BlockType blockType1, BlockType blockType2) {
+      this.blockType1 = blockType1;
+      this.blockType2 = blockType2;
+    }
+
+    public boolean shouldBeCached(BlockType blockType) {
+      return blockType == blockType1 || blockType == blockType2;
+    }
+
+    public void modifyFamilySchema(HColumnDescriptor family) {
+      switch (this) {
+      case DATA_BLOCKS:
+        family.setCacheDataOnWrite(true);
+        break;
+      case BLOOM_BLOCKS:
+        family.setCacheBloomsOnWrite(true);
+        break;
+      case INDEX_BLOCKS:
+        family.setCacheIndexesOnWrite(true);
+        break;
+      }
+    }
+  }
+
+  private final CacheOnWriteType cowType;
+  private Configuration conf;
+  private final String testDescription;
+  private Store store;
+  private FileSystem fs;
+
+  public TestCacheOnWriteInSchema(CacheOnWriteType cowType) {
+    this.cowType = cowType;
+    testDescription = "[cacheOnWrite=" + cowType + "]";
+    System.out.println(testDescription);
+  }
+
+  @Parameters
+  public static Collection<Object[]> getParameters() {
+    List<Object[]> cowTypes = new ArrayList<Object[]>();
+    for (CacheOnWriteType cowType : CacheOnWriteType.values()) {
+      cowTypes.add(new Object[] { cowType });
+    }
+    return cowTypes;
+  }
+
+  @Before
+  public void setUp() throws IOException {
+    conf = TEST_UTIL.getConfiguration();
+    conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
+    conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
+    conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false);
+    conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false);
+
+    fs = HFileSystem.get(conf);
+
+    // Create the schema
+    HColumnDescriptor hcd = new HColumnDescriptor(family);
+    hcd.setBloomFilterType(BloomType.ROWCOL);
+    cowType.modifyFamilySchema(hcd);
+    HTableDescriptor htd = new HTableDescriptor(table);
+    htd.addFamily(hcd);
+
+    // Create a store based on the schema
+    Path basedir = new Path(DIR);
+    Path logdir = new Path(DIR+"/logs");
+    Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME);
+    fs.delete(logdir, true);
+    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+    HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
+    HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null);
+    store = new Store(basedir, region, hcd, fs, conf);
+  }
+
+  @After
+  public void tearDown() {
+    try {
+      fs.delete(new Path(DIR), true);
+    } catch (IOException e) {
+      LOG.error("Could not delete " + DIR, e);
+    }
+  }
+
+  @Test
+  public void testCacheOnWriteInSchema() throws IOException {
+    // Write some random data into the store
+    StoreFile.Writer writer = store.createWriterInTmp(Integer.MAX_VALUE,
+        Compression.Algorithm.NONE, false);
+    writeStoreFile(writer);
+    writer.close();
+    // Verify the block types of interest were cached on write
+    readStoreFile(writer.getPath());
+  }
+
+  private void readStoreFile(Path path) throws IOException {
+    CacheConfig cacheConf = store.getCacheConfig(); 
+    BlockCache cache = cacheConf.getBlockCache();
+    StoreFile sf = new StoreFile(fs, path, conf, cacheConf,
+        BloomType.ROWCOL, null);
+    store.passSchemaMetricsTo(sf);
+    HFileReaderV2 reader = (HFileReaderV2) sf.createReader().getHFileReader();
+    try {
+      // Open a scanner with (on read) caching disabled
+      HFileScanner scanner = reader.getScanner(false, false);
+      assertTrue(testDescription, scanner.seekTo());
+      // Cribbed from io.hfile.TestCacheOnWrite
+      long offset = 0;
+      HFileBlock prevBlock = null;
+      while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
+        long onDiskSize = -1;
+        if (prevBlock != null) {
+          onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
+        }
+        // Flags: don't cache the block, use pread, this is not a compaction.
+        // Also, pass null for expected block type to avoid checking it.
+        HFileBlock block = reader.readBlock(offset, onDiskSize, false, true,
+          false, null);
+        BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
+          offset);
+        boolean isCached = cache.getBlock(blockCacheKey, true) != null;
+        boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
+        if (shouldBeCached != isCached) {
+          throw new AssertionError(
+            "shouldBeCached: " + shouldBeCached+ "\n" +
+            "isCached: " + isCached + "\n" +
+            "Test description: " + testDescription + "\n" +
+            "block: " + block + "\n" +
+            "blockCacheKey: " + blockCacheKey);
+        }
+        prevBlock = block;
+        offset += block.getOnDiskSizeWithHeader();
+      }
+    } finally {
+      reader.close();
+    }
+  }
+
+  private static KeyValue.Type generateKeyType(Random rand) {
+    if (rand.nextBoolean()) {
+      // Let's make half of KVs puts.
+      return KeyValue.Type.Put;
+    } else {
+      KeyValue.Type keyType =
+          KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
+      if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
+      {
+        throw new RuntimeException("Generated an invalid key type: " + keyType
+            + ". " + "Probably the layout of KeyValue.Type has changed.");
+      }
+      return keyType;
+    }
+  }
+
+  private void writeStoreFile(StoreFile.Writer writer) throws IOException {
+    final int rowLen = 32;
+    for (int i = 0; i < NUM_KV; ++i) {
+      byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i);
+      byte[] v = TestHFileWriterV2.randomValue(rand);
+      int cfLen = rand.nextInt(k.length - rowLen + 1);
+      KeyValue kv = new KeyValue(
+          k, 0, rowLen,
+          k, rowLen, cfLen,
+          k, rowLen + cfLen, k.length - rowLen - cfLen,
+          rand.nextLong(),
+          generateKeyType(rand),
+          v, 0, v.length);
+      writer.append(kv);
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
+

From 1869b424ba55dac1dad21a613d970ec83ccc5017 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 30 May 2012 06:32:36 +0000
Subject: [PATCH 0241/1540] HBASE-6088 Addendum fixes
 testSplitBeforeSettingSplittingInZK (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344113 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestSplitTransactionOnCluster.java      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 195213b2bcf3..b7694ef43916 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -536,7 +536,8 @@ public void testSplitBeforeSettingSplittingInZK() throws IOException,
   private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws IOException,
       KeeperException {
     final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
-    HBaseAdmin admin = TESTING_UTIL.getHBaseAdmin();
+    
+    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
     try {
       // Create table then get the single region for our new table.
       HTableDescriptor htd = new HTableDescriptor(tableName);

From fc69447832285ed46ea7974e1531cc1a12a7675a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 15:20:11 +0000
Subject: [PATCH 0242/1540] HBASE-6131 Add attribution for code added by
 HBASE-5533 metrics

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344304 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       |   6 +
 .../ExponentiallyDecayingSample.java          | 226 ------------------
 .../metrics/histogram/MetricsHistogram.java   |   5 +
 .../hbase/metrics/histogram/Sample.java       |  49 ----
 .../hbase/metrics/histogram/Snapshot.java     | 166 -------------
 .../metrics/histogram/UniformSample.java      | 105 --------
 .../metrics/RegionServerMetrics.java          |   2 +-
 .../TestExponentiallyDecayingSample.java      |   4 +-
 .../hbase/metrics/TestMetricsHistogram.java   |   2 +-
 9 files changed, 15 insertions(+), 550 deletions(-)

diff --git a/pom.xml b/pom.xml
index d20ae2cbf47c..05deaba026f7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -976,6 +976,7 @@
     <commons-logging.version>1.1.1</commons-logging.version>
     <commons-math.version>2.1</commons-math.version>
     <commons-configuration.version>1.6</commons-configuration.version>
+    <metrics-core.version>2.1.2</metrics-core.version>
     <guava.version>r09</guava.version>
     <jackson.version>1.8.8</jackson.version>
     <jasper.version>5.5.23</jasper.version>
@@ -1036,6 +1037,11 @@
 
 
     <!-- General dependencies -->
+    <dependency>
+      <groupId>com.yammer.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>${metrics-core.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
index a3a1799b8f18..e69de29bb2d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
@@ -1,226 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.metrics.histogram;
-
-import java.util.ArrayList;
-import java.util.Random;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * An exponentially-decaying random sample of {@code long}s. 
- * Uses Cormode et al's forward-decaying priority reservoir sampling method 
- * to produce a statistically representative sample, exponentially biased 
- * towards newer entries.
- *
- * see Cormode et al. 
- * Forward Decay: A Practical Time Decay Model for Streaming Systems. ICDE '09
- */
-public class ExponentiallyDecayingSample implements Sample {
-
-  private static final Random RANDOM = new Random();  
-  private static final long RESCALE_THRESHOLD = TimeUnit.HOURS.toNanos(1);
-
-  private static final ScheduledExecutorService TICK_SERVICE = 
-    Executors.newScheduledThreadPool(1, 
-      getNamedDaemonThreadFactory(Thread.currentThread().getName() + ".decayingSampleTick."));
-
-  private static volatile long CURRENT_TICK = 
-      TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
-
-  static {
-    // sample at twice our signal's frequency (1Hz) per the Nyquist theorem
-    TICK_SERVICE.scheduleAtFixedRate(new Runnable() {
-      @Override
-      public void run() {
-        CURRENT_TICK = 
-            TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
-      }
-    }, 0, 500, TimeUnit.MILLISECONDS);
-  }
-  
-  private final ConcurrentSkipListMap<Double, Long> values = 
-      new ConcurrentSkipListMap<Double, Long>();
-  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final AtomicLong count = new AtomicLong(0);
-  private final AtomicLong nextScaleTime = new AtomicLong(0);
-
-  private final double alpha;
-  private final int reservoirSize;
-  private volatile long startTime;
-
-  /**
-   * Constructor for an ExponentiallyDecayingSample.
-   *
-   * @param reservoirSize the number of samples to keep in the reservoir
-   * @param alpha         the exponential decay factor; the higher this is, 
-   *                      the more biased the sample will be towards newer 
-   *                      values
-   */
-  public ExponentiallyDecayingSample(int reservoirSize, double alpha) {
-    this.alpha = alpha;
-    this.reservoirSize = reservoirSize;
-    clear();
-  }
-
-  @Override
-  public void clear() {
-    lockForRescale();
-    try {
-      values.clear();
-      count.set(0);
-      this.startTime = CURRENT_TICK;
-      nextScaleTime.set(System.nanoTime() + RESCALE_THRESHOLD);
-    } finally {
-      unlockForRescale();
-    }
-  }
-
-  @Override
-  public int size() {
-    return (int) Math.min(reservoirSize, count.get());
-  }
-
-  @Override
-  public void update(long value) {
-    update(value, CURRENT_TICK);
-  }
-
-  /**
-   * Adds an old value with a fixed timestamp to the sample.
-   *
-   * @param value     the value to be added
-   * @param timestamp the epoch timestamp of {@code value} in seconds
-   */
-  public void update(long value, long timestamp) {
-    lockForRegularUsage();
-    try {
-      final double priority = weight(timestamp - startTime) 
-          / RANDOM.nextDouble();
-      final long newCount = count.incrementAndGet();
-      if (newCount <= reservoirSize) {
-        values.put(priority, value);
-      } else {
-        Double first = values.firstKey();
-        if (first < priority) {
-          if (values.putIfAbsent(priority, value) == null) {
-            // ensure we always remove an item
-            while (values.remove(first) == null) {
-              first = values.firstKey();
-            }
-          }
-        }
-      }
-    } finally {
-      unlockForRegularUsage();
-    }
-
-    final long now = System.nanoTime();
-    final long next = nextScaleTime.get();
-    if (now >= next) {
-      rescale(now, next);
-    }
-  }
-
-  @Override
-  public Snapshot getSnapshot() {
-    lockForRegularUsage();
-    try {
-      return new Snapshot(values.values());
-    } finally {
-      unlockForRegularUsage();
-    }
-  }
-
-  private double weight(long t) {
-    return Math.exp(alpha * t);
-  }
-
-  /* "A common feature of the above techniques—indeed, the key technique that
-   * allows us to track the decayed weights efficiently—is that they maintain
-   * counts and other quantities based on g(ti − L), and only scale by g(t − L)
-   * at query time. But while g(ti −L)/g(t−L) is guaranteed to lie between zero
-   * and one, the intermediate values of g(ti − L) could become very large. For
-   * polynomial functions, these values should not grow too large, and should 
-   * be effectively represented in practice by floating point values without 
-   * loss of precision. For exponential functions, these values could grow 
-   * quite large as new values of (ti − L) become large, and potentially 
-   * exceed the capacity of common floating point types. However, since the 
-   * values stored by the algorithms are linear combinations of g values 
-   * (scaled sums), they can be rescaled relative to a new landmark. That is, 
-   * by the analysis of exponential decay in Section III-A, the choice of L 
-   * does not affect the final result. We can therefore multiply each value 
-   * based on L by a factor of exp(−α(L′ − L)), and obtain the correct value 
-   * as if we had instead computed relative to a new landmark L′ (and then use 
-   * this new L′ at query time). This can be done with a linear pass over 
-   * whatever data structure is being used."
-   */
-  private void rescale(long now, long next) {
-    if (nextScaleTime.compareAndSet(next, now + RESCALE_THRESHOLD)) {
-      lockForRescale();
-      try {
-        final long oldStartTime = startTime;
-        this.startTime = CURRENT_TICK;
-        final ArrayList<Double> keys = new ArrayList<Double>(values.keySet());
-        for (Double key : keys) {
-          final Long value = values.remove(key);
-          values.put(key * Math.exp(-alpha * (startTime - oldStartTime)), 
-              value);
-        }
-      } finally {
-        unlockForRescale();
-      }
-    }
-  }
-
-  private void unlockForRescale() {
-    lock.writeLock().unlock();
-  }
-
-  private void lockForRescale() {
-    lock.writeLock().lock();
-  }
-
-  private void lockForRegularUsage() {
-    lock.readLock().lock();
-  }
-
-  private void unlockForRegularUsage() {
-    lock.readLock().unlock();
-  }
-
-  private static ThreadFactory getNamedDaemonThreadFactory(final String prefix) {
-    return new ThreadFactory() {
-
-      private final AtomicInteger threadNumber = new AtomicInteger(1);
-      
-      @Override
-      public Thread newThread(Runnable r) {
-        Thread t= new Thread(r, prefix + threadNumber.getAndIncrement());
-        t.setDaemon(true);
-        return t;
-      }
-    };
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
index a78b0ce92560..7b65a8fabda4 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
@@ -25,6 +25,11 @@
 import org.apache.hadoop.metrics.util.MetricsBase;
 import org.apache.hadoop.metrics.util.MetricsRegistry;
 
+import com.yammer.metrics.stats.Sample;
+import com.yammer.metrics.stats.Snapshot;
+import com.yammer.metrics.stats.UniformSample;
+import com.yammer.metrics.stats.ExponentiallyDecayingSample;
+
 public class MetricsHistogram extends MetricsBase {
   
   // 1028 items implies 99.9% CI w/ 5% margin of error 
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
index 55b91d688b21..e69de29bb2d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.metrics.histogram;
-
-/**
- * A statistically representative sample of items from a stream.
- */
-public interface Sample {
-  /**
-   * Clears all recorded values.
-   */
-  void clear();
-
-  /**
-   * Returns the number of values recorded.
-   *
-   * @return the number of values recorded
-   */
-  int size();
-
-  /**
-   * Adds a new recorded value to the sample.
-   *
-   * @param value a new recorded value
-   */
-  void update(long value);
-
-  /**
-   * Returns a snapshot of the sample's values.
-   *
-   * @return a snapshot of the sample's values
-   */
-  Snapshot getSnapshot();
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
index 418b6fb1205d..e69de29bb2d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.metrics.histogram;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- * A snapshot of all the information seen in a Sample.
- */
-public class Snapshot {
-
-  private static final double MEDIAN_Q = 0.5;
-  private static final double P75_Q = 0.75;
-  private static final double P95_Q = 0.95;
-  private static final double P98_Q = 0.98;
-  private static final double P99_Q = 0.99;
-  private static final double P999_Q = 0.999;
-
-  private final double[] values;
-
-  /**
-   * Create a new {@link Snapshot} with the given values.
-   *
-   * @param values    an unordered set of values in the sample
-   */
-  public Snapshot(Collection<Long> values) {
-    final Object[] copy = values.toArray();
-    this.values = new double[copy.length];
-    for (int i = 0; i < copy.length; i++) {
-      this.values[i] = (Long) copy[i];
-    }
-    Arrays.sort(this.values);
-  }
-
-  /**
-   * Create a new {@link Snapshot} with the given values.
-   *
-   * @param values    an unordered set of values in the sample
-   */
-  public Snapshot(double[] values) {
-    this.values = new double[values.length];
-    System.arraycopy(values, 0, this.values, 0, values.length);
-    Arrays.sort(this.values);
-  }
-
-  /**
-   * Returns the value at the given quantile.
-   *
-   * @param quantile    a given quantile, in [0..1]
-   * @return the value in the distribution at quantile
-   */
-  public double getValue(double quantile) {
-    if (quantile < 0.0 || quantile > 1.0) {
-      throw new IllegalArgumentException(quantile + " is not in [0..1]");
-    }
-
-    if (values.length == 0) {
-      return 0.0;
-    }
-
-    final double pos = quantile * (values.length + 1);
-
-    if (pos < 1) {
-      return values[0];
-    }
-
-    if (pos >= values.length) {
-      return values[values.length - 1];
-    }
-
-    final double lower = values[(int) pos - 1];
-    final double upper = values[(int) pos];
-    return lower + (pos - Math.floor(pos)) * (upper - lower);
-  }
-
-  /**
-   * Returns the number of values in the snapshot.
-   *
-   * @return the number of values in the snapshot
-   */
-  public int size() {
-    return values.length;
-  }
-
-  /**
-   * Returns the median value in the distribution.
-   *
-   * @return the median value in the distribution
-   */
-  public double getMedian() {
-    return getValue(MEDIAN_Q);
-  }
-
-  /**
-   * Returns the value at the 75th percentile in the distribution.
-   *
-   * @return the value at the 75th percentile in the distribution
-   */
-  public double get75thPercentile() {
-    return getValue(P75_Q);
-  }
-
-  /**
-   * Returns the value at the 95th percentile in the distribution.
-   *
-   * @return the value at the 95th percentile in the distribution
-   */
-  public double get95thPercentile() {
-    return getValue(P95_Q);
-  }
-
-  /**
-   * Returns the value at the 98th percentile in the distribution.
-   *
-   * @return the value at the 98th percentile in the distribution
-   */
-  public double get98thPercentile() {
-    return getValue(P98_Q);
-  }
-
-  /**
-   * Returns the value at the 99th percentile in the distribution.
-   *
-   * @return the value at the 99th percentile in the distribution
-   */
-  public double get99thPercentile() {
-    return getValue(P99_Q);
-  }
-
-  /**
-   * Returns the value at the 99.9th percentile in the distribution.
-   *
-   * @return the value at the 99.9th percentile in the distribution
-   */
-  public double get999thPercentile() {
-    return getValue(P999_Q);
-  }
-
-  /**
-   * Returns the entire set of values in the snapshot.
-   *
-   * @return the entire set of values in the snapshot
-   */
-  public double[] getValues() {
-    return Arrays.copyOf(values, values.length);
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
index 74dd0f945678..e69de29bb2d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.metrics.histogram;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicLongArray;
-
-/**
- * A random sample of a stream of longs. Uses Vitter's Algorithm R to produce a
- * statistically representative sample.
- *
- * see: http://www.cs.umd.edu/~samir/498/vitter.pdf
- */
-public class UniformSample implements Sample {
-  
-  private static final Random RANDOM = new Random();
-  private static final int BITS_PER_LONG = 63;
-  
-  private final AtomicLong count = new AtomicLong();
-  private final AtomicLongArray values;
-
-  /**
-   * Creates a new UniformSample
-   *
-   * @param reservoirSize the number of samples to keep
-   */
-  public UniformSample(int reservoirSize) {
-    this.values = new AtomicLongArray(reservoirSize);
-    clear();
-  }
-
-  @Override
-  public void clear() {
-    for (int i = 0; i < values.length(); i++) {
-      values.set(i, 0);
-    }
-    count.set(0);
-  }
-
-  @Override
-  public int size() {
-    final long c = count.get();
-    if (c > values.length()) {
-      return values.length();
-    }
-    return (int) c;
-  }
-
-  @Override
-  public void update(long value) {
-    final long c = count.incrementAndGet();
-    if (c <= values.length()) {
-      values.set((int) c - 1, value);
-    } else {
-      final long r = nextLong(c);
-      if (r < values.length()) {
-        values.set((int) r, value);
-      }
-    }
-  }
-
-  /**
-   * Get a pseudo-random long uniformly between 0 and n-1. Stolen from
-   * {@link java.util.Random#nextInt()}.
-   *
-   * @param n the bound
-   * @return a value select randomly from the range {@code [0..n)}.
-   */
-  private static long nextLong(long n) {
-    long bits, val;
-    do {
-      bits = RANDOM.nextLong() & (~(1L << BITS_PER_LONG));
-      val = bits % n;
-    } while (bits - val + (n - 1) < 0L);
-    return val;
-  }
-
-  @Override
-  public Snapshot getSnapshot() {
-    final int s = size();
-    final List<Long> copy = new ArrayList<Long>(s);
-    for (int i = 0; i < s; i++) {
-      copy.add(values.get(i));
-    }
-    return new Snapshot(copy);
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index 3cb6784f25d6..b00e4b8c44e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.hbase.metrics.MetricsRate;
 import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
 import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
-import org.apache.hadoop.hbase.metrics.histogram.Snapshot;
+import com.yammer.metrics.stats.Snapshot;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Strings;
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
index 8b0153e18781..b34ec88f032f 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
@@ -20,8 +20,8 @@
 
 import junit.framework.Assert;
 
-import org.apache.hadoop.hbase.metrics.histogram.ExponentiallyDecayingSample;
-import org.apache.hadoop.hbase.metrics.histogram.Snapshot;
+import com.yammer.metrics.stats.ExponentiallyDecayingSample;
+import com.yammer.metrics.stats.Snapshot;
 import org.junit.Test;
 
 public class TestExponentiallyDecayingSample {
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
index 4b1ae151d7e7..57778eee5230 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
@@ -22,7 +22,7 @@
 import java.util.Random;
 
 import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
-import org.apache.hadoop.hbase.metrics.histogram.Snapshot;
+import com.yammer.metrics.stats.Snapshot;
 import org.junit.Assert;
 import org.junit.Test;
 

From 30479ccd7706f0abdc825e3a35bde089b15b8526 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 15:21:25 +0000
Subject: [PATCH 0243/1540] HBASE-6131 Add attribution for code added by
 HBASE-5533 metrics

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344305 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/metrics/histogram/ExponentiallyDecayingSample.java      | 0
 .../java/org/apache/hadoop/hbase/metrics/histogram/Sample.java    | 0
 .../java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java  | 0
 .../org/apache/hadoop/hbase/metrics/histogram/UniformSample.java  | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java

diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/ExponentiallyDecayingSample.java
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Sample.java
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/Snapshot.java
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/UniformSample.java
deleted file mode 100644
index e69de29bb2d1..000000000000

From 9b3a1c9f6ee697d49f11d45c71d36e9c56044ed8 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 30 May 2012 17:01:37 +0000
Subject: [PATCH 0244/1540] HBASE-6122 Backup master does not become Active
 master after ZK exception (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344348 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/ActiveMasterManager.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 178edb338ba1..bf6acfcccfea 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -210,7 +210,8 @@ boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
         return cleanSetOfActiveMaster;
       }
       // Try to become active master again now that there is no active master
-      blockUntilBecomingActiveMaster(startupStatus,clusterStatusTracker);
+      cleanSetOfActiveMaster = blockUntilBecomingActiveMaster(startupStatus,
+          clusterStatusTracker);
     }
     return cleanSetOfActiveMaster;
   }

From b1371723bff7237f60bec70dc619ad6830615f0b Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 30 May 2012 17:06:52 +0000
Subject: [PATCH 0245/1540] HBASE-5722 NPE in
 ZKUtil#getChildDataAndWatchForNewChildren when ZK not available or NW down.
 (Uma)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344351 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/zookeeper/ZKUtil.java    | 10 ++++++----
 .../java/org/apache/hadoop/hbase/TestZooKeeper.java  | 12 ++++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index e1b6bee0f6f5..d4c79422ae9e 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -595,10 +595,12 @@ public static List<NodeAndData> getChildDataAndWatchForNewChildren(
     List<String> nodes =
       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
     List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
-    for (String node: nodes) {
-      String nodePath = ZKUtil.joinZNode(baseNode, node);
-      byte [] data = ZKUtil.getDataAndWatch(zkw, nodePath);
-      newNodes.add(new NodeAndData(nodePath, data));
+    if (nodes != null) {
+      for (String node : nodes) {
+        String nodePath = ZKUtil.joinZNode(baseNode, node);
+        byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
+        newNodes.add(new NodeAndData(nodePath, data));
+      }
     }
     return newNodes;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index ddf8186efb5c..fefa8b838fba 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -301,6 +301,18 @@ public void testCreateSilentIsReallySilent() throws InterruptedException,
 
     ZKUtil.createAndFailSilent(zk2, aclZnode);
  }
+  
+  /**
+   * Test should not fail with NPE when getChildDataAndWatchForNewChildren
+   * invoked with wrongNode
+   */
+  @Test
+  public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
+      throws Exception {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+        "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
+    ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 3408639bbc2e10a6f1ba015fe3b133aee8d37ee4 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 30 May 2012 17:09:20 +0000
Subject: [PATCH 0246/1540] HBASE-5733
 AssignmentManager#processDeadServersAndRegionsInTransition can fail with NPE.
 (Uma)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344352 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  6 +++
 .../hbase/master/TestAssignmentManager.java   | 43 +++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 772c6c857996..88858a0dcbc5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -383,6 +383,12 @@ void processDeadServersAndRegionsInTransition(
   throws KeeperException, IOException, InterruptedException {
     List<String> nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
       watcher.assignmentZNode);
+    
+    if (nodes == null) {
+      String errorMessage = "Failed to get the children from ZK";
+      master.abort(errorMessage, new IOException(errorMessage));
+      return;
+    }
     // Run through all regions.  If they are not assigned and not in RIT, then
     // its a clean cluster startup, else its a failover.
     for (Map.Entry<HRegionInfo, ServerName> e: this.regions.entrySet()) {
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 64c8d334e255..961500d23e16 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -17,9 +17,11 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -28,6 +30,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -56,11 +59,13 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -658,6 +663,39 @@ public void testUnassignWithSplitAtSameTime() throws KeeperException, IOExceptio
       am.shutdown();
     }
   }
+  
+  /**
+   * Tests the processDeadServersAndRegionsInTransition should not fail with NPE
+   * when it failed to get the children. Let's abort the system in this
+   * situation
+   * @throws ServiceException 
+   */
+  @Test(timeout = 5000)
+  public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
+      throws IOException, KeeperException, InterruptedException, ServiceException {
+    final RecoverableZooKeeper recoverableZk = Mockito
+        .mock(RecoverableZooKeeper.class);
+    AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
+        this.server, this.serverManager);
+    Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
+        null) {
+      public RecoverableZooKeeper getRecoverableZooKeeper() {
+        return recoverableZk;
+      }
+    };
+    ((ZooKeeperWatcher) zkw).registerListener(am);
+    Mockito.doThrow(new InterruptedException()).when(recoverableZk)
+        .getChildren("/hbase/unassigned", zkw);
+    am.setWatcher((ZooKeeperWatcher) zkw);
+    try {
+      am.processDeadServersAndRegionsInTransition();
+      fail("Expected to abort");
+    } catch (NullPointerException e) {
+      fail("Should not throw NPE");
+    } catch (RuntimeException e) {
+      assertEquals("Aborted", e.getLocalizedMessage());
+    }
+  }
 
   /**
    * Creates a new ephemeral node in the SPLITTING state for the specified region.
@@ -872,6 +910,11 @@ public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewP
     public ServerName getRegionServerOfRegion(HRegionInfo hri) {
       return SERVERNAME_A;
     }
+    
+    /** reset the watcher */
+    void setWatcher(ZooKeeperWatcher watcher) {
+      this.watcher = watcher;
+    }
 
     /**
      * @return ExecutorService used by this instance.

From 11d3e9366c276149f0fbbe25ed2f993b7846461d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 17:56:42 +0000
Subject: [PATCH 0247/1540] HBASE-6049 Serializing 'List' containing null
 elements will cause NullPointerException in HbaseObjectWritable.writeObject()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344364 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/HbaseObjectWritable.java   | 5 +++--
 .../apache/hadoop/hbase/io/TestHbaseObjectWritable.java   | 8 ++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
index e67a377f27be..d1d199030105 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -472,8 +472,9 @@ public static void writeObject(DataOutput out, Object instance,
       int length = list.size();
       out.writeInt(length);
       for (int i = 0; i < length; i++) {
-        writeObject(out, list.get(i),
-                  list.get(i).getClass(), conf);
+        Object elem = list.get(i);
+        writeObject(out, elem,
+                  elem == null ? Writable.class : elem.getClass(), conf);
       }
     } else if (declClass == String.class) {   // String
       Text.writeString(out, (String)instanceObj);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
index f2f8ee38b6e2..0e467ab2bebb 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
@@ -197,6 +197,14 @@ public void testReadObjectDataInputConfiguration() throws IOException {
     obj = doType(conf, list, List.class);
     assertTrue(obj instanceof List);
     Assert.assertArrayEquals(list.toArray(), ((List)obj).toArray() );
+    //List.class with null values
+    List<String> listWithNulls = new ArrayList<String>();
+    listWithNulls.add("hello");
+    listWithNulls.add("world");
+    listWithNulls.add(null);
+    obj = doType(conf, listWithNulls, List.class);
+    assertTrue(obj instanceof List);
+    Assert.assertArrayEquals(listWithNulls.toArray(), ((List)obj).toArray() );
     //ArrayList.class
     ArrayList<String> arr = new ArrayList<String>();
     arr.add("hello");

From a18e375d1e4cb01f5436b13fd433af428fe7e096 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 18:39:15 +0000
Subject: [PATCH 0248/1540] HBASE-4720 Implement atomic update operations
 (checkAndPut, checkAndDelete) for REST client/server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344399 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/rest/RowResource.java | 164 +++++++++++++++++-
 .../hadoop/hbase/rest/TableResource.java      |   5 +-
 .../hbase/rest/client/RemoteHTable.java       |  72 +++++++-
 .../hadoop/hbase/rest/TestRowResource.java    | 148 +++++++++++++++-
 4 files changed, 376 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index ee1e96dabd09..3cc5da08500b 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -53,8 +53,12 @@
 public class RowResource extends ResourceBase {
   private static final Log LOG = LogFactory.getLog(RowResource.class);
 
+  static final String CHECK_PUT = "put";
+  static final String CHECK_DELETE = "delete";
+
   TableResource tableResource;
   RowSpec rowspec;
+  private String check = null;
 
   /**
    * Constructor
@@ -64,13 +68,14 @@ public class RowResource extends ResourceBase {
    * @throws IOException
    */
   public RowResource(TableResource tableResource, String rowspec,
-      String versions) throws IOException {
+      String versions, String check) throws IOException {
     super();
     this.tableResource = tableResource;
     this.rowspec = new RowSpec(rowspec);
     if (versions != null) {
       this.rowspec.setMaxVersions(Integer.valueOf(versions));
     }
+    this.check = check;
   }
 
   @GET
@@ -149,6 +154,15 @@ Response update(final CellSetModel model, final boolean replace) {
     if (servlet.isReadOnly()) {
       throw new WebApplicationException(Response.Status.FORBIDDEN);
     }
+
+    if (CHECK_PUT.equalsIgnoreCase(check)) {
+      return checkAndPut(model);
+    } else if (CHECK_DELETE.equalsIgnoreCase(check)) {
+      return checkAndDelete(model);
+    } else if (check != null && check.length() > 0) {
+      LOG.warn("Unknown check value: " + check + ", ignored");
+    }
+
     HTablePool pool = servlet.getTablePool();
     HTableInterface table = null;
     try {
@@ -275,7 +289,8 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
   public Response put(final CellSetModel model,
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
-      LOG.debug("PUT " + uriInfo.getAbsolutePath());
+      LOG.debug("PUT " + uriInfo.getAbsolutePath()
+        + " " + uriInfo.getQueryParameters());
     }
     return update(model, true);
   }
@@ -295,7 +310,8 @@ public Response putBinary(final byte[] message,
   public Response post(final CellSetModel model,
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
-      LOG.debug("POST " + uriInfo.getAbsolutePath());
+      LOG.debug("POST " + uriInfo.getAbsolutePath()
+        + " " + uriInfo.getQueryParameters());
     }
     return update(model, false);
   }
@@ -366,4 +382,146 @@ public Response delete(final @Context UriInfo uriInfo) {
     }
     return Response.ok().build();
   }
+
+  /**
+   * Validates the input request parameters, parses columns from CellSetModel,
+   * and invokes checkAndPut on HTable.
+   *
+   * @param model instance of CellSetModel
+   * @return Response 200 OK, 304 Not modified, 400 Bad request
+   */
+  Response checkAndPut(final CellSetModel model) {
+    HTablePool pool = servlet.getTablePool();
+    HTableInterface table = null;
+    try {
+      if (model.getRows().size() != 1) {
+        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      }
+
+      RowModel rowModel = model.getRows().get(0);
+      byte[] key = rowModel.getKey();
+      if (key == null) {
+        key = rowspec.getRow();
+      }
+
+      List<CellModel> cellModels = rowModel.getCells();
+      int cellModelCount = cellModels.size();
+      if (key == null || cellModelCount <= 1) {
+        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      }
+
+      Put put = new Put(key);
+      CellModel valueToCheckCell = cellModels.get(cellModelCount - 1);
+      byte[] valueToCheckColumn = valueToCheckCell.getColumn();
+      byte[][] valueToPutParts = KeyValue.parseColumn(valueToCheckColumn);
+      if (valueToPutParts.length == 2 && valueToPutParts[1].length > 0) {
+        CellModel valueToPutCell = null;
+        for (int i = 0, n = cellModelCount - 1; i < n ; i++) {
+          if(Bytes.equals(cellModels.get(i).getColumn(),
+              valueToCheckCell.getColumn())) {
+            valueToPutCell = cellModels.get(i);
+            break;
+          }
+        }
+        if (valueToPutCell != null) {
+          put.add(valueToPutParts[0], valueToPutParts[1], valueToPutCell
+            .getTimestamp(), valueToPutCell.getValue());
+        } else {
+          throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        }
+      } else {
+        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      }
+
+      table = pool.getTable(this.tableResource.getName());
+      boolean retValue = table.checkAndPut(key, valueToPutParts[0],
+        valueToPutParts[1], valueToCheckCell.getValue(), put);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("CHECK-AND-PUT " + put.toString() + ", returns " + retValue);
+      }
+      table.flushCommits();
+      ResponseBuilder response = Response.ok();
+      if (!retValue) {
+        response = Response.status(304);
+      }
+      return response.build();
+    } catch (IOException e) {
+      throw new WebApplicationException(e, Response.Status.SERVICE_UNAVAILABLE);
+    } finally {
+      try {
+        if(table != null){
+          pool.putTable(table);
+        }
+      } catch (Exception ioe) {
+        throw new WebApplicationException(ioe,
+          Response.Status.SERVICE_UNAVAILABLE);
+      }
+    }
+  }
+
+  /**
+   * Validates the input request parameters, parses columns from CellSetModel,
+   * and invokes checkAndDelete on HTable.
+   *
+   * @param model instance of CellSetModel
+   * @return Response 200 OK, 304 Not modified, 400 Bad request
+   */
+  Response checkAndDelete(final CellSetModel model) {
+    HTablePool pool = servlet.getTablePool();
+    HTableInterface table = null;
+    Delete delete = null;
+    try {
+      if (model.getRows().size() != 1) {
+        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      }
+      RowModel rowModel = model.getRows().get(0);
+      byte[] key = rowModel.getKey();
+      if (key == null) {
+        key = rowspec.getRow();
+      }
+      if (key == null) {
+        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      }
+
+      delete = new Delete(key);
+      CellModel valueToDeleteCell = rowModel.getCells().get(0);
+      byte[] valueToDeleteColumn = valueToDeleteCell.getColumn();
+      if (valueToDeleteColumn == null) {
+        try {
+          valueToDeleteColumn = rowspec.getColumns()[0];
+        } catch (final ArrayIndexOutOfBoundsException e) {
+          throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        }
+      }
+      byte[][] parts = KeyValue.parseColumn(valueToDeleteColumn);
+      if (parts.length == 2 && parts[1].length > 0) {
+        delete.deleteColumns(parts[0], parts[1]);
+      } else {
+        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      }
+
+      table = pool.getTable(tableResource.getName());
+      boolean retValue = table.checkAndDelete(key, parts[0], parts[1],
+        valueToDeleteCell.getValue(), delete);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("CHECK-AND-DELETE " + delete.toString() + ", returns "
+          + retValue);
+      }
+      table.flushCommits();
+      ResponseBuilder response = Response.ok();
+      if (!retValue) {
+        response = Response.status(304);
+      }
+      return response.build();
+    } catch (IOException e) {
+      throw new WebApplicationException(e, Response.Status.SERVICE_UNAVAILABLE);
+    } finally {
+      try {
+        pool.putTable(table);
+      } catch (Exception ioe) {
+        throw new WebApplicationException(ioe,
+          Response.Status.SERVICE_UNAVAILABLE);
+      }
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index 07f5f269488c..f8e0d1827346 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -92,7 +92,8 @@ public RowResource getRowResource(
       // We need the @Encoded decorator so Jersey won't urldecode before
       // the RowSpec constructor has a chance to parse
       final @PathParam("rowspec") @Encoded String rowspec,
-      final @QueryParam("v") String versions) throws IOException {
-    return new RowResource(this, rowspec, versions);
+      final @QueryParam("v") String versions,
+      final @QueryParam("check") String check) throws IOException {
+    return new RowResource(this, rowspec, versions, check);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index 09d51acb9c10..09744ef5f993 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -585,12 +585,80 @@ public void unlockRow(RowLock rl) throws IOException {
 
   public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
       byte[] value, Put put) throws IOException {
-    throw new IOException("checkAndPut not supported");
+    // column to check-the-value
+    put.add(new KeyValue(row, family, qualifier, value));
+
+    CellSetModel model = buildModelFromPut(put);
+    StringBuilder sb = new StringBuilder();
+    sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');
+    }
+    sb.append(Bytes.toStringBinary(name));
+    sb.append('/');
+    sb.append(Bytes.toStringBinary(put.getRow()));
+    sb.append("?check=put");
+
+    for (int i = 0; i < maxRetries; i++) {
+      Response response = client.put(sb.toString(),
+        Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
+      int code = response.getCode();
+      switch (code) {
+      case 200:
+        return true;
+      case 304: // NOT-MODIFIED
+        return false;
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (final InterruptedException e) {
+        }
+        break;
+      default:
+        throw new IOException("checkAndPut request failed with " + code);
+      }
+    }
+    throw new IOException("checkAndPut request timed out");
   }
 
   public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
       byte[] value, Delete delete) throws IOException {
-    throw new IOException("checkAndDelete not supported");
+    Put put = new Put(row);
+    // column to check-the-value
+    put.add(new KeyValue(row, family, qualifier, value));
+    CellSetModel model = buildModelFromPut(put);
+    StringBuilder sb = new StringBuilder();
+    sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');
+    }
+    sb.append(Bytes.toStringBinary(name));
+    sb.append('/');
+    sb.append(Bytes.toStringBinary(row));
+    sb.append("?check=delete");
+
+    for (int i = 0; i < maxRetries; i++) {
+      Response response = client.put(sb.toString(),
+        Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
+      int code = response.getCode();
+      switch (code) {
+      case 200:
+        return true;
+      case 304: // NOT-MODIFIED
+        return false;
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (final InterruptedException e) {
+        }
+        break;
+      default:
+        throw new IOException("checkAndDelete request failed with " + code);
+      }
+    }
+    throw new IOException("checkAndDelete request timed out");
   }
 
   public Result increment(Increment increment) throws IOException {
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
index b59436ca345c..1b67046c4f05 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
@@ -265,11 +265,120 @@ private static void checkValuePB(String table, String row, String column,
     assertEquals(Bytes.toString(cell.getValue()), value);
   }
 
+  private static Response checkAndPutValuePB(String url, String table,
+      String row, String column, String valueToCheck, String valueToPut)
+        throws IOException {
+    RowModel rowModel = new RowModel(row);
+    rowModel.addCell(new CellModel(Bytes.toBytes(column),
+      Bytes.toBytes(valueToPut)));
+    rowModel.addCell(new CellModel(Bytes.toBytes(column),
+      Bytes.toBytes(valueToCheck)));
+    CellSetModel cellSetModel = new CellSetModel();
+    cellSetModel.addRow(rowModel);
+    Response response = client.put(url, Constants.MIMETYPE_PROTOBUF,
+      cellSetModel.createProtobufOutput());
+    Thread.yield();
+    return response;
+  }
+
+  private static Response checkAndPutValuePB(String table, String row,
+      String column, String valueToCheck, String valueToPut) throws IOException {
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    path.append(table);
+    path.append('/');
+    path.append(row);
+    path.append("?check=put");
+    return checkAndPutValuePB(path.toString(), table, row, column,
+      valueToCheck, valueToPut);
+  }
+
+  private static Response checkAndPutValueXML(String url, String table,
+      String row, String column, String valueToCheck, String valueToPut)
+        throws IOException, JAXBException {
+    RowModel rowModel = new RowModel(row);
+    rowModel.addCell(new CellModel(Bytes.toBytes(column),
+      Bytes.toBytes(valueToPut)));
+    rowModel.addCell(new CellModel(Bytes.toBytes(column),
+      Bytes.toBytes(valueToCheck)));
+    CellSetModel cellSetModel = new CellSetModel();
+    cellSetModel.addRow(rowModel);
+    StringWriter writer = new StringWriter();
+    marshaller.marshal(cellSetModel, writer);
+    Response response = client.put(url, Constants.MIMETYPE_XML,
+      Bytes.toBytes(writer.toString()));
+    Thread.yield();
+    return response;
+  }
+
+  private static Response checkAndPutValueXML(String table, String row,
+      String column, String valueToCheck, String valueToPut)
+        throws IOException, JAXBException {
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    path.append(table);
+    path.append('/');
+    path.append(row);
+    path.append("?check=put");
+    return checkAndPutValueXML(path.toString(), table, row, column,
+      valueToCheck, valueToPut);
+  }
+
+  private static Response checkAndDeleteXML(String url, String table,
+      String row, String column, String valueToCheck)
+        throws IOException, JAXBException {
+    RowModel rowModel = new RowModel(row);
+    rowModel.addCell(new CellModel(Bytes.toBytes(column),
+      Bytes.toBytes(valueToCheck)));
+    CellSetModel cellSetModel = new CellSetModel();
+    cellSetModel.addRow(rowModel);
+    StringWriter writer = new StringWriter();
+    marshaller.marshal(cellSetModel, writer);
+    Response response = client.put(url, Constants.MIMETYPE_XML,
+      Bytes.toBytes(writer.toString()));
+    Thread.yield();
+    return response;
+  }
+
+  private static Response checkAndDeleteXML(String table, String row,
+      String column, String valueToCheck) throws IOException, JAXBException {
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    path.append(table);
+    path.append('/');
+    path.append(row);
+    path.append("?check=delete");
+    return checkAndDeleteXML(path.toString(), table, row, column, valueToCheck);
+  }
+
+  private static Response checkAndDeletePB(String table, String row,
+      String column, String value) throws IOException {
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    path.append(table);
+    path.append('/');
+    path.append(row);
+    path.append("?check=delete");
+    return checkAndDeleteValuePB(path.toString(), table, row, column, value);
+  }
+
+  private static Response checkAndDeleteValuePB(String url, String table,
+      String row, String column, String valueToCheck)
+      throws IOException {
+    RowModel rowModel = new RowModel(row);
+    rowModel.addCell(new CellModel(Bytes.toBytes(column), Bytes
+        .toBytes(valueToCheck)));
+    CellSetModel cellSetModel = new CellSetModel();
+    cellSetModel.addRow(rowModel);
+    Response response = client.put(url, Constants.MIMETYPE_PROTOBUF,
+        cellSetModel.createProtobufOutput());
+    Thread.yield();
+    return response;
+  }
+
   @Test
   public void testDelete() throws IOException, JAXBException {
-    Response response;
-    
-    response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1);
+    Response response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1);
     assertEquals(response.getCode(), 200);
     response = putValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2);
     assertEquals(response.getCode(), 200);
@@ -282,6 +391,13 @@ public void testDelete() throws IOException, JAXBException {
     assertEquals(response.getCode(), 404);
     checkValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2);
 
+    response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1);
+    assertEquals(response.getCode(), 200);
+    response = checkAndDeletePB(TABLE, ROW_1, COLUMN_1, VALUE_1);
+    assertEquals(response.getCode(), 200);
+    response = getValueXML(TABLE, ROW_1, COLUMN_1);
+    assertEquals(response.getCode(), 404);
+
     response = deleteRow(TABLE, ROW_1);
     assertEquals(response.getCode(), 200);    
     response = getValueXML(TABLE, ROW_1, COLUMN_1);
@@ -292,16 +408,20 @@ public void testDelete() throws IOException, JAXBException {
 
   @Test
   public void testForbidden() throws IOException, JAXBException {
-    Response response;
-
     conf.set("hbase.rest.readonly", "true");
 
-    response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1);
+    Response response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1);
     assertEquals(response.getCode(), 403);
     response = putValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1);
     assertEquals(response.getCode(), 403);
+    response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1, VALUE_2);
+    assertEquals(response.getCode(), 403);
+    response = checkAndPutValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1, VALUE_2);
+    assertEquals(response.getCode(), 403);
     response = deleteValue(TABLE, ROW_1, COLUMN_1);
     assertEquals(response.getCode(), 403);
+    response = checkAndDeletePB(TABLE, ROW_1, COLUMN_1, VALUE_1);
+    assertEquals(response.getCode(), 403);
     response = deleteRow(TABLE, ROW_1);
     assertEquals(response.getCode(), 403);
 
@@ -311,6 +431,10 @@ public void testForbidden() throws IOException, JAXBException {
     assertEquals(response.getCode(), 200);
     response = putValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1);
     assertEquals(response.getCode(), 200);
+    response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1, VALUE_2);
+    assertEquals(response.getCode(), 200);
+    response = checkAndPutValuePB(TABLE, ROW_1, COLUMN_1, VALUE_2, VALUE_3);
+    assertEquals(response.getCode(), 200);
     response = deleteValue(TABLE, ROW_1, COLUMN_1);
     assertEquals(response.getCode(), 200);
     response = deleteRow(TABLE, ROW_1);
@@ -328,6 +452,11 @@ public void testSingleCellGetPutXML() throws IOException, JAXBException {
     response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2);
     assertEquals(response.getCode(), 200);
     checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2);
+    response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2, VALUE_3);
+    assertEquals(response.getCode(), 200);
+    checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_3);
+    response = checkAndDeleteXML(TABLE, ROW_1, COLUMN_1, VALUE_3);
+    assertEquals(response.getCode(), 200);
 
     response = deleteRow(TABLE, ROW_1);
     assertEquals(response.getCode(), 200);
@@ -349,6 +478,13 @@ public void testSingleCellGetPutPB() throws IOException, JAXBException {
     assertEquals(response.getCode(), 200);
     checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_2);
 
+    response = checkAndPutValuePB(TABLE, ROW_1, COLUMN_1, VALUE_2, VALUE_3);
+    assertEquals(response.getCode(), 200);
+    checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_3);
+    response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_3, VALUE_4);
+    assertEquals(response.getCode(), 200);
+    checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_4);
+
     response = deleteRow(TABLE, ROW_1);
     assertEquals(response.getCode(), 200);
   }

From e6da080386def8035e80d47e0983655248196f28 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 20:02:53 +0000
Subject: [PATCH 0249/1540] HBASE-6129 Backport of Add Increment Coalescing in
 thrift.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344435 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/thrift/IncrementCoalescer.java      |  370 +++
 .../hbase/thrift/IncrementCoalescerMBean.java |   50 +
 .../hbase/thrift/ThriftServerRunner.java      |   46 +-
 .../hadoop/hbase/thrift/ThriftUtilities.java  |   16 +
 .../hadoop/hbase/thrift/generated/Hbase.java  | 2452 ++++++++++++++---
 .../hbase/thrift/generated/TIncrement.java    |  715 +++++
 .../hbase/thrift/generated/TRowResult.java    |    2 +-
 .../apache/hadoop/hbase/thrift/Hbase.thrift   |   27 +
 .../hadoop/hbase/thrift/TestThriftServer.java |   46 +
 9 files changed, 3337 insertions(+), 387 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java
new file mode 100644
index 000000000000..c0205ba54523
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java
@@ -0,0 +1,370 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.thrift;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler;
+import org.apache.hadoop.hbase.thrift.generated.TIncrement;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.thrift.TException;
+
+/**
+ * This class will coalesce increments from a thift server if
+ * hbase.regionserver.thrift.coalesceIncrement is set to true. Turning this
+ * config to true will cause the thrift server to queue increments into an
+ * instance of this class. The thread pool associated with this class will drain
+ * the coalesced increments as the thread is able. This can cause data loss if the
+ * thrift server dies or is shut down before everything in the queue is drained.
+ *
+ */
+public class IncrementCoalescer implements IncrementCoalescerMBean {
+
+  /**
+   * Used to identify a cell that will be incremented.
+   *
+   */
+  static class FullyQualifiedRow {
+    private byte[] table;
+    private byte[] rowKey;
+    private byte[] family;
+    private byte[] qualifier;
+
+    public FullyQualifiedRow(byte[] table, byte[] rowKey, byte[] fam, byte[] qual) {
+      super();
+      this.table = table;
+      this.rowKey = rowKey;
+      this.family = fam;
+      this.qualifier = qual;
+    }
+
+    public byte[] getTable() {
+      return table;
+    }
+
+    public void setTable(byte[] table) {
+      this.table = table;
+    }
+
+    public byte[] getRowKey() {
+      return rowKey;
+    }
+
+    public void setRowKey(byte[] rowKey) {
+      this.rowKey = rowKey;
+    }
+
+    public byte[] getFamily() {
+      return family;
+    }
+
+    public void setFamily(byte[] fam) {
+      this.family = fam;
+    }
+
+    public byte[] getQualifier() {
+      return qualifier;
+    }
+
+    public void setQualifier(byte[] qual) {
+      this.qualifier = qual;
+    }
+
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + Arrays.hashCode(family);
+      result = prime * result + Arrays.hashCode(qualifier);
+      result = prime * result + Arrays.hashCode(rowKey);
+      result = prime * result + Arrays.hashCode(table);
+      return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null) return false;
+      if (getClass() != obj.getClass()) return false;
+      FullyQualifiedRow other = (FullyQualifiedRow) obj;
+      if (!Arrays.equals(family, other.family)) return false;
+      if (!Arrays.equals(qualifier, other.qualifier)) return false;
+      if (!Arrays.equals(rowKey, other.rowKey)) return false;
+      if (!Arrays.equals(table, other.table)) return false;
+      return true;
+    }
+
+  }
+
+  static class DaemonThreadFactory implements ThreadFactory {
+    static final AtomicInteger poolNumber = new AtomicInteger(1);
+    final ThreadGroup group;
+    final AtomicInteger threadNumber = new AtomicInteger(1);
+    final String namePrefix;
+
+    DaemonThreadFactory() {
+      SecurityManager s = System.getSecurityManager();
+      group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
+      namePrefix = "ICV-" + poolNumber.getAndIncrement() + "-thread-";
+    }
+
+    public Thread newThread(Runnable r) {
+      Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
+      if (!t.isDaemon()) t.setDaemon(true);
+      if (t.getPriority() != Thread.NORM_PRIORITY) t.setPriority(Thread.NORM_PRIORITY);
+      return t;
+    }
+  }
+
+  private final AtomicLong failedIncrements = new AtomicLong();
+  private final AtomicLong successfulCoalescings = new AtomicLong();
+  private final AtomicLong totalIncrements = new AtomicLong();
+  private final ConcurrentMap<FullyQualifiedRow, Long> countersMap =
+      new ConcurrentHashMap<FullyQualifiedRow, Long>(100000, 0.75f, 1500);
+  private final ThreadPoolExecutor pool;
+  private final HBaseHandler handler;
+
+  private int maxQueueSize = 500000;
+  private static final int CORE_POOL_SIZE = 1;
+
+  protected final Log LOG = LogFactory.getLog(this.getClass().getName());
+
+  @SuppressWarnings("deprecation")
+  public IncrementCoalescer(HBaseHandler hand) {
+    this.handler = hand;
+    LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
+    pool =
+        new ThreadPoolExecutor(CORE_POOL_SIZE, CORE_POOL_SIZE, 50, TimeUnit.MILLISECONDS, queue,
+            new DaemonThreadFactory());
+
+    MBeanUtil.registerMBean("thrift", "Thrift", this);
+  }
+
+  public boolean queueIncrement(TIncrement inc) throws TException {
+    if (!canQueue()) {
+      failedIncrements.incrementAndGet();
+      return false;
+    }
+    return internalQueueTincrement(inc);
+  }
+
+  public boolean queueIncrements(List<TIncrement> incs) throws TException {
+    if (!canQueue()) {
+      failedIncrements.incrementAndGet();
+      return false;
+    }
+
+    for (TIncrement tinc : incs) {
+      internalQueueTincrement(tinc);
+    }
+    return true;
+
+  }
+
+  private boolean internalQueueTincrement(TIncrement inc) throws TException {
+    byte[][] famAndQf = KeyValue.parseColumn(inc.getColumn());
+    if (famAndQf.length < 1) return false;
+    byte[] qual = famAndQf.length == 1 ? new byte[0] : famAndQf[1];
+
+    return internalQueueIncrement(inc.getTable(), inc.getRow(), famAndQf[0], qual,
+      inc.getAmmount());
+
+  }
+
+  private boolean internalQueueIncrement(byte[] tableName, byte[] rowKey, byte[] fam,
+      byte[] qual, long ammount) throws TException {
+    int countersMapSize = countersMap.size();
+
+
+    //Make sure that the number of threads is scaled.
+    dynamicallySetCoreSize(countersMapSize);
+
+    totalIncrements.incrementAndGet();
+
+    FullyQualifiedRow key = new FullyQualifiedRow(tableName, rowKey, fam, qual);
+
+    long currentAmount = ammount;
+    // Spin until able to insert the value back without collisions
+    while (true) {
+      Long value = countersMap.remove(key);
+      if (value == null) {
+        // There was nothing there, create a new value
+        value = new Long(currentAmount);
+      } else {
+        value += currentAmount;
+        successfulCoalescings.incrementAndGet();
+      }
+      // Try to put the value, only if there was none
+      Long oldValue = countersMap.putIfAbsent(key, value);
+      if (oldValue == null) {
+        // We were able to put it in, we're done
+        break;
+      }
+      // Someone else was able to put a value in, so let's remember our
+      // current value (plus what we picked up) and retry to add it in
+      currentAmount = value;
+    }
+
+    // We limit the size of the queue simply because all we need is a
+    // notification that something needs to be incremented. No need
+    // for millions of callables that mean the same thing.
+    if (pool.getQueue().size() <= 1000) {
+      // queue it up
+      Callable<Integer> callable = createIncCallable();
+      pool.submit(callable);
+    }
+
+    return true;
+  }
+
+  public boolean canQueue() {
+    return countersMap.size() < maxQueueSize;
+  }
+
+  private Callable<Integer> createIncCallable() {
+    return new Callable<Integer>() {
+      @Override
+      public Integer call() throws Exception {
+        int failures = 0;
+        Set<FullyQualifiedRow> keys = countersMap.keySet();
+        for (FullyQualifiedRow row : keys) {
+          Long counter = countersMap.remove(row);
+          if (counter == null) {
+            continue;
+          }
+          try {
+            HTable table = handler.getTable(row.getTable());
+            if (failures > 2) {
+              throw new IOException("Auto-Fail rest of ICVs");
+            }
+            table.incrementColumnValue(row.getRowKey(), row.getFamily(), row.getQualifier(),
+              counter);
+          } catch (IOException e) {
+            // log failure of increment
+            failures++;
+            LOG.error("FAILED_ICV: " + Bytes.toString(row.getTable()) + ", "
+                + Bytes.toStringBinary(row.getRowKey()) + ", "
+                + Bytes.toStringBinary(row.getFamily()) + ", "
+                + Bytes.toStringBinary(row.getQualifier()) + ", " + counter);
+          }
+
+        }
+        return failures;
+      }
+    };
+  }
+
+  /**
+   * This method samples the incoming requests and, if selected, will check if
+   * the corePoolSize should be changed.
+   * @param countersMapSize
+   */
+  private void dynamicallySetCoreSize(int countersMapSize) {
+    // Here we are using countersMapSize as a random number, meaning this
+    // could be a Random object
+    if (countersMapSize % 10 != 0) {
+      return;
+    }
+    double currentRatio = (double) countersMapSize / (double) maxQueueSize;
+    int newValue = 1;
+    if (currentRatio < 0.1) {
+      // it's 1
+    } else if (currentRatio < 0.3) {
+      newValue = 2;
+    } else if (currentRatio < 0.5) {
+      newValue = 4;
+    } else if (currentRatio < 0.7) {
+      newValue = 8;
+    } else if (currentRatio < 0.9) {
+      newValue = 14;
+    } else {
+      newValue = 22;
+    }
+    if (pool.getCorePoolSize() != newValue) {
+      pool.setCorePoolSize(newValue);
+    }
+  }
+
+  // MBean get/set methods
+  public int getQueueSize() {
+    return pool.getQueue().size();
+  }
+  public int getMaxQueueSize() {
+    return this.maxQueueSize;
+  }
+  public void setMaxQueueSize(int newSize) {
+    this.maxQueueSize = newSize;
+  }
+
+  public long getPoolCompletedTaskCount() {
+    return pool.getCompletedTaskCount();
+  }
+  public long getPoolTaskCount() {
+    return pool.getTaskCount();
+  }
+  public int getPoolLargestPoolSize() {
+    return pool.getLargestPoolSize();
+  }
+  public int getCorePoolSize() {
+    return pool.getCorePoolSize();
+  }
+  public void setCorePoolSize(int newCoreSize) {
+    pool.setCorePoolSize(newCoreSize);
+  }
+  public int getMaxPoolSize() {
+    return pool.getMaximumPoolSize();
+  }
+  public void setMaxPoolSize(int newMaxSize) {
+    pool.setMaximumPoolSize(newMaxSize);
+  }
+  public long getFailedIncrements() {
+    return failedIncrements.get();
+  }
+
+  public long getSuccessfulCoalescings() {
+    return successfulCoalescings.get();
+  }
+
+  public long getTotalIncrements() {
+    return totalIncrements.get();
+  }
+
+  public long getCountersMapSize() {
+    return countersMap.size();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java
new file mode 100644
index 000000000000..c3783688dc84
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.thrift;
+
+public interface IncrementCoalescerMBean {
+  public int getQueueSize();
+
+  public int getMaxQueueSize();
+
+  public void setMaxQueueSize(int newSize);
+
+  public long getPoolCompletedTaskCount();
+
+  public long getPoolTaskCount();
+
+  public int getPoolLargestPoolSize();
+
+  public int getCorePoolSize();
+
+  public void setCorePoolSize(int newCoreSize);
+
+  public int getMaxPoolSize();
+
+  public void setMaxPoolSize(int newMaxSize);
+
+  public long getFailedIncrements();
+
+  public long getSuccessfulCoalescings();
+
+  public long getTotalIncrements();
+
+  public long getCountersMapSize();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 02ece7b9bee3..10ded9964877 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.OperationWithAttributes;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -75,6 +76,7 @@
 import org.apache.hadoop.hbase.thrift.generated.IllegalArgument;
 import org.apache.hadoop.hbase.thrift.generated.Mutation;
 import org.apache.hadoop.hbase.thrift.generated.TCell;
+import org.apache.hadoop.hbase.thrift.generated.TIncrement;
 import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
 import org.apache.hadoop.hbase.thrift.generated.TScan;
@@ -116,6 +118,7 @@ public class ThriftServerRunner implements Runnable {
   static final String COMPACT_CONF_KEY = "hbase.regionserver.thrift.compact";
   static final String FRAMED_CONF_KEY = "hbase.regionserver.thrift.framed";
   static final String PORT_CONF_KEY = "hbase.regionserver.thrift.port";
+  static final String COALESCE_INC_KEY = "hbase.regionserver.thrift.coalesceIncrement";
 
   private static final String DEFAULT_BIND_ADDR = "0.0.0.0";
   public static final int DEFAULT_LISTEN_PORT = 9090;
@@ -409,6 +412,8 @@ protected Map<String, HTable> initialValue() {
       }
     };
 
+    IncrementCoalescer coalescer = null;
+
     /**
      * Returns a list of all the column families for a given htable.
      *
@@ -435,7 +440,7 @@ byte[][] getAllColumns(HTable table) throws IOException {
      * @throws IOException
      * @throws IOError
      */
-    protected HTable getTable(final byte[] tableName) throws
+    public HTable getTable(final byte[] tableName) throws
         IOException {
       String table = new String(tableName);
       Map<String, HTable> tables = threadLocalTables.get();
@@ -445,7 +450,7 @@ protected HTable getTable(final byte[] tableName) throws
       return tables.get(table);
     }
 
-    protected HTable getTable(final ByteBuffer tableName) throws IOException {
+    public HTable getTable(final ByteBuffer tableName) throws IOException {
       return getTable(getBytes(tableName));
     }
 
@@ -496,6 +501,7 @@ protected HBaseHandler(final Configuration c) throws IOException {
       this.conf = c;
       admin = new HBaseAdmin(conf);
       scannerMap = new HashMap<Integer, ResultScanner>();
+      this.coalescer = new IncrementCoalescer(this);
     }
 
     @Override
@@ -1388,7 +1394,43 @@ public TRegionInfo getRegionInfo(ByteBuffer searchRow) throws IOError {
     private void initMetrics(ThriftMetrics metrics) {
       this.metrics = metrics;
     }
+
+    @Override
+    public void increment(TIncrement tincrement) throws IOError, TException {
+
+      if (tincrement.getRow().length == 0 || tincrement.getTable().length == 0) {
+        throw new TException("Must supply a table and a row key; can't increment");
+      }
+
+      if (conf.getBoolean(COALESCE_INC_KEY, false)) {
+        this.coalescer.queueIncrement(tincrement);
+        return;
+      }
+
+      try {
+        HTable table = getTable(tincrement.getTable());
+        Increment inc = ThriftUtilities.incrementFromThrift(tincrement);
+        table.increment(inc);
+      } catch (IOException e) {
+        LOG.warn(e.getMessage(), e);
+        throw new IOError(e.getMessage());
+      }
+    }
+
+    @Override
+    public void incrementRows(List<TIncrement> tincrements) throws IOError, TException {
+      if (conf.getBoolean(COALESCE_INC_KEY, false)) {
+        this.coalescer.queueIncrements(tincrements);
+        return;
+      }
+      for (TIncrement tinc : tincrements) {
+        increment(tinc);
+      }
+    }
   }
+
+
+
   /**
    * Adds all the attributes into the Operation object
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
index d7fa95e57963..8313aa8f875e 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
@@ -25,6 +25,7 @@
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -32,6 +33,7 @@
 import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
 import org.apache.hadoop.hbase.thrift.generated.IllegalArgument;
 import org.apache.hadoop.hbase.thrift.generated.TCell;
+import org.apache.hadoop.hbase.thrift.generated.TIncrement;
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -154,4 +156,18 @@ static public List<TRowResult> rowResultFromHBase(Result in) {
     Result [] result = { in };
     return rowResultFromHBase(result);
   }
+
+  /**
+   * From a {@link TIncrement} create an {@link Increment}.
+   * @param tincrement the Thrift version of an increment
+   * @return an increment that the {@link TIncrement} represented.
+   */
+  public static Increment incrementFromThrift(TIncrement tincrement) {
+    Increment inc = new Increment(tincrement.getRow());
+    byte[][] famAndQf = KeyValue.parseColumn(tincrement.getColumn());
+    if (famAndQf.length <1 ) return null;
+    byte[] qual = famAndQf.length == 1 ? new byte[0]: famAndQf[1];
+    inc.addColumn(famAndQf[0], qual, tincrement.getAmmount());
+    return inc;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java
index 6c505c0d01e9..e9ca236327fd 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java
@@ -301,7 +301,7 @@ public interface Iface {
      * 
      * @param mutations list of mutation commands
      * 
-     * @param attributes Put attributes
+     * @param attributes Mutation attributes
      */
     public void mutateRow(ByteBuffer tableName, ByteBuffer row, List<Mutation> mutations, Map<ByteBuffer,ByteBuffer> attributes) throws IOError, IllegalArgument, org.apache.thrift.TException;
 
@@ -319,7 +319,7 @@ public interface Iface {
      * 
      * @param timestamp timestamp
      * 
-     * @param attributes Put attributes
+     * @param attributes Mutation attributes
      */
     public void mutateRowTs(ByteBuffer tableName, ByteBuffer row, List<Mutation> mutations, long timestamp, Map<ByteBuffer,ByteBuffer> attributes) throws IOError, IllegalArgument, org.apache.thrift.TException;
 
@@ -333,7 +333,7 @@ public interface Iface {
      * 
      * @param rowBatches list of row batches
      * 
-     * @param attributes Put attributes
+     * @param attributes Mutation attributes
      */
     public void mutateRows(ByteBuffer tableName, List<BatchMutation> rowBatches, Map<ByteBuffer,ByteBuffer> attributes) throws IOError, IllegalArgument, org.apache.thrift.TException;
 
@@ -349,7 +349,7 @@ public interface Iface {
      * 
      * @param timestamp timestamp
      * 
-     * @param attributes Put attributes
+     * @param attributes Mutation attributes
      */
     public void mutateRowsTs(ByteBuffer tableName, List<BatchMutation> rowBatches, long timestamp, Map<ByteBuffer,ByteBuffer> attributes) throws IOError, IllegalArgument, org.apache.thrift.TException;
 
@@ -406,6 +406,18 @@ public interface Iface {
      */
     public void deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map<ByteBuffer,ByteBuffer> attributes) throws IOError, org.apache.thrift.TException;
 
+    /**
+     * Increment a cell by the ammount.
+     * Increments can be applied async if hbase.regionserver.thrift.coalesceIncrement is set to true.
+     * False is the default.  Turn to true if you need the extra performance and can accept some
+     * data loss if a thrift server dies with increments still in the queue.
+     * 
+     * @param increment The single increment to apply
+     */
+    public void increment(TIncrement increment) throws IOError, org.apache.thrift.TException;
+
+    public void incrementRows(List<TIncrement> increments) throws IOError, org.apache.thrift.TException;
+
     /**
      * Completely delete the row's cells marked with a timestamp
      * equal-to or older than the passed timestamp.
@@ -666,6 +678,10 @@ public interface AsyncIface {
 
     public void deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map<ByteBuffer,ByteBuffer> attributes, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.deleteAllRow_call> resultHandler) throws org.apache.thrift.TException;
 
+    public void increment(TIncrement increment, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.increment_call> resultHandler) throws org.apache.thrift.TException;
+
+    public void incrementRows(List<TIncrement> increments, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.incrementRows_call> resultHandler) throws org.apache.thrift.TException;
+
     public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map<ByteBuffer,ByteBuffer> attributes, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.deleteAllRowTs_call> resultHandler) throws org.apache.thrift.TException;
 
     public void scannerOpenWithScan(ByteBuffer tableName, TScan scan, Map<ByteBuffer,ByteBuffer> attributes, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.scannerOpenWithScan_call> resultHandler) throws org.apache.thrift.TException;
@@ -1508,6 +1524,52 @@ public void recv_deleteAllRow() throws IOError, org.apache.thrift.TException
       return;
     }
 
+    public void increment(TIncrement increment) throws IOError, org.apache.thrift.TException
+    {
+      send_increment(increment);
+      recv_increment();
+    }
+
+    public void send_increment(TIncrement increment) throws org.apache.thrift.TException
+    {
+      increment_args args = new increment_args();
+      args.setIncrement(increment);
+      sendBase("increment", args);
+    }
+
+    public void recv_increment() throws IOError, org.apache.thrift.TException
+    {
+      increment_result result = new increment_result();
+      receiveBase(result, "increment");
+      if (result.io != null) {
+        throw result.io;
+      }
+      return;
+    }
+
+    public void incrementRows(List<TIncrement> increments) throws IOError, org.apache.thrift.TException
+    {
+      send_incrementRows(increments);
+      recv_incrementRows();
+    }
+
+    public void send_incrementRows(List<TIncrement> increments) throws org.apache.thrift.TException
+    {
+      incrementRows_args args = new incrementRows_args();
+      args.setIncrements(increments);
+      sendBase("incrementRows", args);
+    }
+
+    public void recv_incrementRows() throws IOError, org.apache.thrift.TException
+    {
+      incrementRows_result result = new incrementRows_result();
+      receiveBase(result, "incrementRows");
+      if (result.io != null) {
+        throw result.io;
+      }
+      return;
+    }
+
     public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map<ByteBuffer,ByteBuffer> attributes) throws IOError, org.apache.thrift.TException
     {
       send_deleteAllRowTs(tableName, row, timestamp, attributes);
@@ -2976,6 +3038,70 @@ public void getResult() throws IOError, org.apache.thrift.TException {
       }
     }
 
+    public void increment(TIncrement increment, org.apache.thrift.async.AsyncMethodCallback<increment_call> resultHandler) throws org.apache.thrift.TException {
+      checkReady();
+      increment_call method_call = new increment_call(increment, resultHandler, this, ___protocolFactory, ___transport);
+      this.___currentMethod = method_call;
+      ___manager.call(method_call);
+    }
+
+    public static class increment_call extends org.apache.thrift.async.TAsyncMethodCall {
+      private TIncrement increment;
+      public increment_call(TIncrement increment, org.apache.thrift.async.AsyncMethodCallback<increment_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+        super(client, protocolFactory, transport, resultHandler, false);
+        this.increment = increment;
+      }
+
+      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("increment", org.apache.thrift.protocol.TMessageType.CALL, 0));
+        increment_args args = new increment_args();
+        args.setIncrement(increment);
+        args.write(prot);
+        prot.writeMessageEnd();
+      }
+
+      public void getResult() throws IOError, org.apache.thrift.TException {
+        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+          throw new IllegalStateException("Method call not finished!");
+        }
+        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+        (new Client(prot)).recv_increment();
+      }
+    }
+
+    public void incrementRows(List<TIncrement> increments, org.apache.thrift.async.AsyncMethodCallback<incrementRows_call> resultHandler) throws org.apache.thrift.TException {
+      checkReady();
+      incrementRows_call method_call = new incrementRows_call(increments, resultHandler, this, ___protocolFactory, ___transport);
+      this.___currentMethod = method_call;
+      ___manager.call(method_call);
+    }
+
+    public static class incrementRows_call extends org.apache.thrift.async.TAsyncMethodCall {
+      private List<TIncrement> increments;
+      public incrementRows_call(List<TIncrement> increments, org.apache.thrift.async.AsyncMethodCallback<incrementRows_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+        super(client, protocolFactory, transport, resultHandler, false);
+        this.increments = increments;
+      }
+
+      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("incrementRows", org.apache.thrift.protocol.TMessageType.CALL, 0));
+        incrementRows_args args = new incrementRows_args();
+        args.setIncrements(increments);
+        args.write(prot);
+        prot.writeMessageEnd();
+      }
+
+      public void getResult() throws IOError, org.apache.thrift.TException {
+        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+          throw new IllegalStateException("Method call not finished!");
+        }
+        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+        (new Client(prot)).recv_incrementRows();
+      }
+    }
+
     public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map<ByteBuffer,ByteBuffer> attributes, org.apache.thrift.async.AsyncMethodCallback<deleteAllRowTs_call> resultHandler) throws org.apache.thrift.TException {
       checkReady();
       deleteAllRowTs_call method_call = new deleteAllRowTs_call(tableName, row, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport);
@@ -3483,6 +3609,8 @@ protected Processor(I iface, Map<String,  org.apache.thrift.ProcessFunction<I, ?
       processMap.put("deleteAll", new deleteAll());
       processMap.put("deleteAllTs", new deleteAllTs());
       processMap.put("deleteAllRow", new deleteAllRow());
+      processMap.put("increment", new increment());
+      processMap.put("incrementRows", new incrementRows());
       processMap.put("deleteAllRowTs", new deleteAllRowTs());
       processMap.put("scannerOpenWithScan", new scannerOpenWithScan());
       processMap.put("scannerOpen", new scannerOpen());
@@ -4094,6 +4222,46 @@ protected deleteAllRow_result getResult(I iface, deleteAllRow_args args) throws
       }
     }
 
+    private static class increment<I extends Iface> extends org.apache.thrift.ProcessFunction<I, increment_args> {
+      public increment() {
+        super("increment");
+      }
+
+      protected increment_args getEmptyArgsInstance() {
+        return new increment_args();
+      }
+
+      protected increment_result getResult(I iface, increment_args args) throws org.apache.thrift.TException {
+        increment_result result = new increment_result();
+        try {
+          iface.increment(args.increment);
+        } catch (IOError io) {
+          result.io = io;
+        }
+        return result;
+      }
+    }
+
+    private static class incrementRows<I extends Iface> extends org.apache.thrift.ProcessFunction<I, incrementRows_args> {
+      public incrementRows() {
+        super("incrementRows");
+      }
+
+      protected incrementRows_args getEmptyArgsInstance() {
+        return new incrementRows_args();
+      }
+
+      protected incrementRows_result getResult(I iface, incrementRows_args args) throws org.apache.thrift.TException {
+        incrementRows_result result = new incrementRows_result();
+        try {
+          iface.incrementRows(args.increments);
+        } catch (IOError io) {
+          result.io = io;
+        }
+        return result;
+      }
+    }
+
     private static class deleteAllRowTs<I extends Iface> extends org.apache.thrift.ProcessFunction<I, deleteAllRowTs_args> {
       public deleteAllRowTs() {
         super("deleteAllRowTs");
@@ -6520,6 +6688,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -9593,7 +9763,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getColumnDescriptor
                   for (int _i35 = 0; _i35 < _map34.size; ++_i35)
                   {
                     ByteBuffer _key36; // required
-                    ColumnDescriptor _val37; // required
+                    ColumnDescriptor _val37; // optional
                     _key36 = iprot.readBinary();
                     _val37 = new ColumnDescriptor();
                     _val37.read(iprot);
@@ -9699,7 +9869,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptors
             for (int _i41 = 0; _i41 < _map40.size; ++_i41)
             {
               ByteBuffer _key42; // required
-              ColumnDescriptor _val43; // required
+              ColumnDescriptor _val43; // optional
               _key42 = iprot.readBinary();
               _val43 = new ColumnDescriptor();
               _val43.read(iprot);
@@ -13083,7 +13253,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, get_args struct) th
                   for (int _i61 = 0; _i61 < _map60.size; ++_i61)
                   {
                     ByteBuffer _key62; // required
-                    ByteBuffer _val63; // required
+                    ByteBuffer _val63; // optional
                     _key62 = iprot.readBinary();
                     _val63 = iprot.readBinary();
                     struct.attributes.put(_key62, _val63);
@@ -13213,7 +13383,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, get_args struct) thr
             for (int _i67 = 0; _i67 < _map66.size; ++_i67)
             {
               ByteBuffer _key68; // required
-              ByteBuffer _val69; // required
+              ByteBuffer _val69; // optional
               _key68 = iprot.readBinary();
               _val69 = iprot.readBinary();
               struct.attributes.put(_key68, _val69);
@@ -14484,7 +14654,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getVer_args struct)
                   for (int _i79 = 0; _i79 < _map78.size; ++_i79)
                   {
                     ByteBuffer _key80; // required
-                    ByteBuffer _val81; // required
+                    ByteBuffer _val81; // optional
                     _key80 = iprot.readBinary();
                     _val81 = iprot.readBinary();
                     struct.attributes.put(_key80, _val81);
@@ -14627,7 +14797,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getVer_args struct)
             for (int _i85 = 0; _i85 < _map84.size; ++_i85)
             {
               ByteBuffer _key86; // required
-              ByteBuffer _val87; // required
+              ByteBuffer _val87; // optional
               _key86 = iprot.readBinary();
               _val87 = iprot.readBinary();
               struct.attributes.put(_key86, _val87);
@@ -15917,6 +16087,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -15989,7 +16161,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getVerTs_args struc
                   for (int _i97 = 0; _i97 < _map96.size; ++_i97)
                   {
                     ByteBuffer _key98; // required
-                    ByteBuffer _val99; // required
+                    ByteBuffer _val99; // optional
                     _key98 = iprot.readBinary();
                     _val99 = iprot.readBinary();
                     struct.attributes.put(_key98, _val99);
@@ -16145,7 +16317,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getVerTs_args struct
             for (int _i103 = 0; _i103 < _map102.size; ++_i103)
             {
               ByteBuffer _key104; // required
-              ByteBuffer _val105; // required
+              ByteBuffer _val105; // optional
               _key104 = iprot.readBinary();
               _val105 = iprot.readBinary();
               struct.attributes.put(_key104, _val105);
@@ -17211,7 +17383,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRow_args struct)
                   for (int _i115 = 0; _i115 < _map114.size; ++_i115)
                   {
                     ByteBuffer _key116; // required
-                    ByteBuffer _val117; // required
+                    ByteBuffer _val117; // optional
                     _key116 = iprot.readBinary();
                     _val117 = iprot.readBinary();
                     struct.attributes.put(_key116, _val117);
@@ -17326,7 +17498,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRow_args struct)
             for (int _i121 = 0; _i121 < _map120.size; ++_i121)
             {
               ByteBuffer _key122; // required
-              ByteBuffer _val123; // required
+              ByteBuffer _val123; // optional
               _key122 = iprot.readBinary();
               _val123 = iprot.readBinary();
               struct.attributes.put(_key122, _val123);
@@ -18519,7 +18691,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumns_a
                   for (int _i136 = 0; _i136 < _map135.size; ++_i136)
                   {
                     ByteBuffer _key137; // required
-                    ByteBuffer _val138; // required
+                    ByteBuffer _val138; // optional
                     _key137 = iprot.readBinary();
                     _val138 = iprot.readBinary();
                     struct.attributes.put(_key137, _val138);
@@ -18671,7 +18843,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_ar
             for (int _i147 = 0; _i147 < _map146.size; ++_i147)
             {
               ByteBuffer _key148; // required
-              ByteBuffer _val149; // required
+              ByteBuffer _val149; // optional
               _key148 = iprot.readBinary();
               _val149 = iprot.readBinary();
               struct.attributes.put(_key148, _val149);
@@ -19835,7 +20007,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowTs_args struc
                   for (int _i159 = 0; _i159 < _map158.size; ++_i159)
                   {
                     ByteBuffer _key160; // required
-                    ByteBuffer _val161; // required
+                    ByteBuffer _val161; // optional
                     _key160 = iprot.readBinary();
                     _val161 = iprot.readBinary();
                     struct.attributes.put(_key160, _val161);
@@ -19963,7 +20135,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowTs_args struct
             for (int _i165 = 0; _i165 < _map164.size; ++_i165)
             {
               ByteBuffer _key166; // required
-              ByteBuffer _val167; // required
+              ByteBuffer _val167; // optional
               _key166 = iprot.readBinary();
               _val167 = iprot.readBinary();
               struct.attributes.put(_key166, _val167);
@@ -21242,7 +21414,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumnsTs
                   for (int _i180 = 0; _i180 < _map179.size; ++_i180)
                   {
                     ByteBuffer _key181; // required
-                    ByteBuffer _val182; // required
+                    ByteBuffer _val182; // optional
                     _key181 = iprot.readBinary();
                     _val182 = iprot.readBinary();
                     struct.attributes.put(_key181, _val182);
@@ -21407,7 +21579,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_
             for (int _i191 = 0; _i191 < _map190.size; ++_i191)
             {
               ByteBuffer _key192; // required
-              ByteBuffer _val193; // required
+              ByteBuffer _val193; // optional
               _key192 = iprot.readBinary();
               _val193 = iprot.readBinary();
               struct.attributes.put(_key192, _val193);
@@ -22493,7 +22665,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRows_args struct
                   for (int _i206 = 0; _i206 < _map205.size; ++_i206)
                   {
                     ByteBuffer _key207; // required
-                    ByteBuffer _val208; // required
+                    ByteBuffer _val208; // optional
                     _key207 = iprot.readBinary();
                     _val208 = iprot.readBinary();
                     struct.attributes.put(_key207, _val208);
@@ -22630,7 +22802,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRows_args struct)
             for (int _i217 = 0; _i217 < _map216.size; ++_i217)
             {
               ByteBuffer _key218; // required
-              ByteBuffer _val219; // required
+              ByteBuffer _val219; // optional
               _key218 = iprot.readBinary();
               _val219 = iprot.readBinary();
               struct.attributes.put(_key218, _val219);
@@ -23843,7 +24015,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_
                   for (int _i235 = 0; _i235 < _map234.size; ++_i235)
                   {
                     ByteBuffer _key236; // required
-                    ByteBuffer _val237; // required
+                    ByteBuffer _val237; // optional
                     _key236 = iprot.readBinary();
                     _val237 = iprot.readBinary();
                     struct.attributes.put(_key236, _val237);
@@ -24017,7 +24189,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_a
             for (int _i251 = 0; _i251 < _map250.size; ++_i251)
             {
               ByteBuffer _key252; // required
-              ByteBuffer _val253; // required
+              ByteBuffer _val253; // optional
               _key252 = iprot.readBinary();
               _val253 = iprot.readBinary();
               struct.attributes.put(_key252, _val253);
@@ -25201,7 +25373,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsTs_args stru
                   for (int _i266 = 0; _i266 < _map265.size; ++_i266)
                   {
                     ByteBuffer _key267; // required
-                    ByteBuffer _val268; // required
+                    ByteBuffer _val268; // optional
                     _key267 = iprot.readBinary();
                     _val268 = iprot.readBinary();
                     struct.attributes.put(_key267, _val268);
@@ -25351,7 +25523,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args struc
             for (int _i277 = 0; _i277 < _map276.size; ++_i277)
             {
               ByteBuffer _key278; // required
-              ByteBuffer _val279; // required
+              ByteBuffer _val279; // optional
               _key278 = iprot.readBinary();
               _val279 = iprot.readBinary();
               struct.attributes.put(_key278, _val279);
@@ -26650,7 +26822,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsT
                   for (int _i295 = 0; _i295 < _map294.size; ++_i295)
                   {
                     ByteBuffer _key296; // required
-                    ByteBuffer _val297; // required
+                    ByteBuffer _val297; // optional
                     _key296 = iprot.readBinary();
                     _val297 = iprot.readBinary();
                     struct.attributes.put(_key296, _val297);
@@ -26837,7 +27009,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs
             for (int _i311 = 0; _i311 < _map310.size; ++_i311)
             {
               ByteBuffer _key312; // required
-              ByteBuffer _val313; // required
+              ByteBuffer _val313; // optional
               _key312 = iprot.readBinary();
               _val313 = iprot.readBinary();
               struct.attributes.put(_key312, _val313);
@@ -27386,7 +27558,7 @@ public static class mutateRow_args implements org.apache.thrift.TBase<mutateRow_
      */
     public List<Mutation> mutations; // required
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> attributes; // required
 
@@ -27405,7 +27577,7 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
        */
       MUTATIONS((short)3, "mutations"),
       /**
-       * Put attributes
+       * Mutation attributes
        */
       ATTRIBUTES((short)4, "attributes");
 
@@ -27687,14 +27859,14 @@ public void putToAttributes(ByteBuffer key, ByteBuffer val) {
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> getAttributes() {
       return this.attributes;
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public mutateRow_args setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
       this.attributes = attributes;
@@ -28031,7 +28203,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_args stru
                   for (int _i326 = 0; _i326 < _map325.size; ++_i326)
                   {
                     ByteBuffer _key327; // required
-                    ByteBuffer _val328; // required
+                    ByteBuffer _val328; // optional
                     _key327 = iprot.readBinary();
                     _val328 = iprot.readBinary();
                     struct.attributes.put(_key327, _val328);
@@ -28184,7 +28356,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_args struc
             for (int _i337 = 0; _i337 < _map336.size; ++_i337)
             {
               ByteBuffer _key338; // required
-              ByteBuffer _val339; // required
+              ByteBuffer _val339; // optional
               _key338 = iprot.readBinary();
               _val339 = iprot.readBinary();
               struct.attributes.put(_key338, _val339);
@@ -28686,7 +28858,7 @@ public static class mutateRowTs_args implements org.apache.thrift.TBase<mutateRo
      */
     public long timestamp; // required
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> attributes; // required
 
@@ -28709,7 +28881,7 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
        */
       TIMESTAMP((short)4, "timestamp"),
       /**
-       * Put attributes
+       * Mutation attributes
        */
       ATTRIBUTES((short)5, "attributes");
 
@@ -29034,14 +29206,14 @@ public void putToAttributes(ByteBuffer key, ByteBuffer val) {
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> getAttributes() {
       return this.attributes;
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public mutateRowTs_args setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
       this.attributes = attributes;
@@ -29424,7 +29596,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowTs_args st
                   for (int _i344 = 0; _i344 < _map343.size; ++_i344)
                   {
                     ByteBuffer _key345; // required
-                    ByteBuffer _val346; // required
+                    ByteBuffer _val346; // optional
                     _key345 = iprot.readBinary();
                     _val346 = iprot.readBinary();
                     struct.attributes.put(_key345, _val346);
@@ -29590,7 +29762,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args str
             for (int _i355 = 0; _i355 < _map354.size; ++_i355)
             {
               ByteBuffer _key356; // required
-              ByteBuffer _val357; // required
+              ByteBuffer _val357; // optional
               _key356 = iprot.readBinary();
               _val357 = iprot.readBinary();
               struct.attributes.put(_key356, _val357);
@@ -30082,7 +30254,7 @@ public static class mutateRows_args implements org.apache.thrift.TBase<mutateRow
      */
     public List<BatchMutation> rowBatches; // required
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> attributes; // required
 
@@ -30097,7 +30269,7 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
        */
       ROW_BATCHES((short)2, "rowBatches"),
       /**
-       * Put attributes
+       * Mutation attributes
        */
       ATTRIBUTES((short)3, "attributes");
 
@@ -30329,14 +30501,14 @@ public void putToAttributes(ByteBuffer key, ByteBuffer val) {
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> getAttributes() {
       return this.attributes;
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public mutateRows_args setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
       this.attributes = attributes;
@@ -30625,7 +30797,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRows_args str
                   for (int _i362 = 0; _i362 < _map361.size; ++_i362)
                   {
                     ByteBuffer _key363; // required
-                    ByteBuffer _val364; // required
+                    ByteBuffer _val364; // optional
                     _key363 = iprot.readBinary();
                     _val364 = iprot.readBinary();
                     struct.attributes.put(_key363, _val364);
@@ -30763,7 +30935,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRows_args stru
             for (int _i373 = 0; _i373 < _map372.size; ++_i373)
             {
               ByteBuffer _key374; // required
-              ByteBuffer _val375; // required
+              ByteBuffer _val375; // optional
               _key374 = iprot.readBinary();
               _val375 = iprot.readBinary();
               struct.attributes.put(_key374, _val375);
@@ -31260,7 +31432,7 @@ public static class mutateRowsTs_args implements org.apache.thrift.TBase<mutateR
      */
     public long timestamp; // required
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> attributes; // required
 
@@ -31279,7 +31451,7 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
        */
       TIMESTAMP((short)3, "timestamp"),
       /**
-       * Put attributes
+       * Mutation attributes
        */
       ATTRIBUTES((short)4, "attributes");
 
@@ -31554,14 +31726,14 @@ public void putToAttributes(ByteBuffer key, ByteBuffer val) {
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public Map<ByteBuffer,ByteBuffer> getAttributes() {
       return this.attributes;
     }
 
     /**
-     * Put attributes
+     * Mutation attributes
      */
     public mutateRowsTs_args setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
       this.attributes = attributes;
@@ -31894,7 +32066,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowsTs_args s
                   for (int _i380 = 0; _i380 < _map379.size; ++_i380)
                   {
                     ByteBuffer _key381; // required
-                    ByteBuffer _val382; // required
+                    ByteBuffer _val382; // optional
                     _key381 = iprot.readBinary();
                     _val382 = iprot.readBinary();
                     struct.attributes.put(_key381, _val382);
@@ -32045,7 +32217,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args st
             for (int _i391 = 0; _i391 < _map390.size; ++_i391)
             {
               ByteBuffer _key392; // required
-              ByteBuffer _val393; // required
+              ByteBuffer _val393; // optional
               _key392 = iprot.readBinary();
               _val393 = iprot.readBinary();
               struct.attributes.put(_key392, _val393);
@@ -33659,6 +33831,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -34450,7 +34624,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAll_args stru
                   for (int _i395 = 0; _i395 < _map394.size; ++_i395)
                   {
                     ByteBuffer _key396; // required
-                    ByteBuffer _val397; // required
+                    ByteBuffer _val397; // optional
                     _key396 = iprot.readBinary();
                     _val397 = iprot.readBinary();
                     struct.attributes.put(_key396, _val397);
@@ -34580,7 +34754,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAll_args struc
             for (int _i401 = 0; _i401 < _map400.size; ++_i401)
             {
               ByteBuffer _key402; // required
-              ByteBuffer _val403; // required
+              ByteBuffer _val403; // optional
               _key402 = iprot.readBinary();
               _val403 = iprot.readBinary();
               struct.attributes.put(_key402, _val403);
@@ -35697,7 +35871,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllTs_args st
                   for (int _i405 = 0; _i405 < _map404.size; ++_i405)
                   {
                     ByteBuffer _key406; // required
-                    ByteBuffer _val407; // required
+                    ByteBuffer _val407; // optional
                     _key406 = iprot.readBinary();
                     _val407 = iprot.readBinary();
                     struct.attributes.put(_key406, _val407);
@@ -35840,7 +36014,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_args str
             for (int _i411 = 0; _i411 < _map410.size; ++_i411)
             {
               ByteBuffer _key412; // required
-              ByteBuffer _val413; // required
+              ByteBuffer _val413; // optional
               _key412 = iprot.readBinary();
               _val413 = iprot.readBinary();
               struct.attributes.put(_key412, _val413);
@@ -36752,7 +36926,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRow_args s
                   for (int _i415 = 0; _i415 < _map414.size; ++_i415)
                   {
                     ByteBuffer _key416; // required
-                    ByteBuffer _val417; // required
+                    ByteBuffer _val417; // optional
                     _key416 = iprot.readBinary();
                     _val417 = iprot.readBinary();
                     struct.attributes.put(_key416, _val417);
@@ -36867,7 +37041,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_args st
             for (int _i421 = 0; _i421 < _map420.size; ++_i421)
             {
               ByteBuffer _key422; // required
-              ByteBuffer _val423; // required
+              ByteBuffer _val423; // optional
               _key422 = iprot.readBinary();
               _val423 = iprot.readBinary();
               struct.attributes.put(_key422, _val423);
@@ -37235,55 +37409,389 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_result
 
   }
 
-  public static class deleteAllRowTs_args implements org.apache.thrift.TBase<deleteAllRowTs_args, deleteAllRowTs_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllRowTs_args");
+  public static class increment_args implements org.apache.thrift.TBase<increment_args, increment_args._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("increment_args");
 
-    private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1);
-    private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2);
-    private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
-    private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4);
+    private static final org.apache.thrift.protocol.TField INCREMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("increment", org.apache.thrift.protocol.TType.STRUCT, (short)1);
 
     private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
     static {
-      schemes.put(StandardScheme.class, new deleteAllRowTs_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new deleteAllRowTs_argsTupleSchemeFactory());
+      schemes.put(StandardScheme.class, new increment_argsStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new increment_argsTupleSchemeFactory());
     }
 
     /**
-     * name of table
-     */
-    public ByteBuffer tableName; // required
-    /**
-     * key of the row to be completely deleted.
+     * The single increment to apply
      */
-    public ByteBuffer row; // required
-    /**
-     * timestamp
-     */
-    public long timestamp; // required
-    /**
-     * Delete attributes
-     */
-    public Map<ByteBuffer,ByteBuffer> attributes; // required
+    public TIncrement increment; // required
 
     /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
     public enum _Fields implements org.apache.thrift.TFieldIdEnum {
       /**
-       * name of table
+       * The single increment to apply
        */
-      TABLE_NAME((short)1, "tableName"),
+      INCREMENT((short)1, "increment");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
       /**
-       * key of the row to be completely deleted.
+       * Find the _Fields constant that matches fieldId, or null if its not found.
        */
-      ROW((short)2, "row"),
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // INCREMENT
+            return INCREMENT;
+          default:
+            return null;
+        }
+      }
+
       /**
-       * timestamp
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
        */
-      TIMESTAMP((short)3, "timestamp"),
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
       /**
-       * Delete attributes
+       * Find the _Fields constant that matches name, or null if its not found.
        */
-      ATTRIBUTES((short)4, "attributes");
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.INCREMENT, new org.apache.thrift.meta_data.FieldMetaData("increment", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TIncrement.class)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(increment_args.class, metaDataMap);
+    }
+
+    public increment_args() {
+    }
+
+    public increment_args(
+      TIncrement increment)
+    {
+      this();
+      this.increment = increment;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public increment_args(increment_args other) {
+      if (other.isSetIncrement()) {
+        this.increment = new TIncrement(other.increment);
+      }
+    }
+
+    public increment_args deepCopy() {
+      return new increment_args(this);
+    }
+
+    @Override
+    public void clear() {
+      this.increment = null;
+    }
+
+    /**
+     * The single increment to apply
+     */
+    public TIncrement getIncrement() {
+      return this.increment;
+    }
+
+    /**
+     * The single increment to apply
+     */
+    public increment_args setIncrement(TIncrement increment) {
+      this.increment = increment;
+      return this;
+    }
+
+    public void unsetIncrement() {
+      this.increment = null;
+    }
+
+    /** Returns true if field increment is set (has been assigned a value) and false otherwise */
+    public boolean isSetIncrement() {
+      return this.increment != null;
+    }
+
+    public void setIncrementIsSet(boolean value) {
+      if (!value) {
+        this.increment = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case INCREMENT:
+        if (value == null) {
+          unsetIncrement();
+        } else {
+          setIncrement((TIncrement)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case INCREMENT:
+        return getIncrement();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case INCREMENT:
+        return isSetIncrement();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof increment_args)
+        return this.equals((increment_args)that);
+      return false;
+    }
+
+    public boolean equals(increment_args that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_increment = true && this.isSetIncrement();
+      boolean that_present_increment = true && that.isSetIncrement();
+      if (this_present_increment || that_present_increment) {
+        if (!(this_present_increment && that_present_increment))
+          return false;
+        if (!this.increment.equals(that.increment))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(increment_args other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      increment_args typedOther = (increment_args)other;
+
+      lastComparison = Boolean.valueOf(isSetIncrement()).compareTo(typedOther.isSetIncrement());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetIncrement()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.increment, typedOther.increment);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("increment_args(");
+      boolean first = true;
+
+      sb.append("increment:");
+      if (this.increment == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.increment);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class increment_argsStandardSchemeFactory implements SchemeFactory {
+      public increment_argsStandardScheme getScheme() {
+        return new increment_argsStandardScheme();
+      }
+    }
+
+    private static class increment_argsStandardScheme extends StandardScheme<increment_args> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, increment_args struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // INCREMENT
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.increment = new TIncrement();
+                struct.increment.read(iprot);
+                struct.setIncrementIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, increment_args struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.increment != null) {
+          oprot.writeFieldBegin(INCREMENT_FIELD_DESC);
+          struct.increment.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class increment_argsTupleSchemeFactory implements SchemeFactory {
+      public increment_argsTupleScheme getScheme() {
+        return new increment_argsTupleScheme();
+      }
+    }
+
+    private static class increment_argsTupleScheme extends TupleScheme<increment_args> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, increment_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        BitSet optionals = new BitSet();
+        if (struct.isSetIncrement()) {
+          optionals.set(0);
+        }
+        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetIncrement()) {
+          struct.increment.write(oprot);
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, increment_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        BitSet incoming = iprot.readBitSet(1);
+        if (incoming.get(0)) {
+          struct.increment = new TIncrement();
+          struct.increment.read(iprot);
+          struct.setIncrementIsSet(true);
+        }
+      }
+    }
+
+  }
+
+  public static class increment_result implements org.apache.thrift.TBase<increment_result, increment_result._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("increment_result");
+
+    private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new increment_resultStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new increment_resultTupleSchemeFactory());
+    }
+
+    public IOError io; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      IO((short)1, "io");
 
       private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -37298,14 +37806,1176 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
        */
       public static _Fields findByThriftId(int fieldId) {
         switch(fieldId) {
-          case 1: // TABLE_NAME
-            return TABLE_NAME;
-          case 2: // ROW
-            return ROW;
-          case 3: // TIMESTAMP
-            return TIMESTAMP;
-          case 4: // ATTRIBUTES
-            return ATTRIBUTES;
+          case 1: // IO
+            return IO;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(increment_result.class, metaDataMap);
+    }
+
+    public increment_result() {
+    }
+
+    public increment_result(
+      IOError io)
+    {
+      this();
+      this.io = io;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public increment_result(increment_result other) {
+      if (other.isSetIo()) {
+        this.io = new IOError(other.io);
+      }
+    }
+
+    public increment_result deepCopy() {
+      return new increment_result(this);
+    }
+
+    @Override
+    public void clear() {
+      this.io = null;
+    }
+
+    public IOError getIo() {
+      return this.io;
+    }
+
+    public increment_result setIo(IOError io) {
+      this.io = io;
+      return this;
+    }
+
+    public void unsetIo() {
+      this.io = null;
+    }
+
+    /** Returns true if field io is set (has been assigned a value) and false otherwise */
+    public boolean isSetIo() {
+      return this.io != null;
+    }
+
+    public void setIoIsSet(boolean value) {
+      if (!value) {
+        this.io = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case IO:
+        if (value == null) {
+          unsetIo();
+        } else {
+          setIo((IOError)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case IO:
+        return getIo();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case IO:
+        return isSetIo();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof increment_result)
+        return this.equals((increment_result)that);
+      return false;
+    }
+
+    public boolean equals(increment_result that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_io = true && this.isSetIo();
+      boolean that_present_io = true && that.isSetIo();
+      if (this_present_io || that_present_io) {
+        if (!(this_present_io && that_present_io))
+          return false;
+        if (!this.io.equals(that.io))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(increment_result other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      increment_result typedOther = (increment_result)other;
+
+      lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetIo()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+      }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("increment_result(");
+      boolean first = true;
+
+      sb.append("io:");
+      if (this.io == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.io);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class increment_resultStandardSchemeFactory implements SchemeFactory {
+      public increment_resultStandardScheme getScheme() {
+        return new increment_resultStandardScheme();
+      }
+    }
+
+    private static class increment_resultStandardScheme extends StandardScheme<increment_result> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, increment_result struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // IO
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.io = new IOError();
+                struct.io.read(iprot);
+                struct.setIoIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, increment_result struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.io != null) {
+          oprot.writeFieldBegin(IO_FIELD_DESC);
+          struct.io.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class increment_resultTupleSchemeFactory implements SchemeFactory {
+      public increment_resultTupleScheme getScheme() {
+        return new increment_resultTupleScheme();
+      }
+    }
+
+    private static class increment_resultTupleScheme extends TupleScheme<increment_result> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, increment_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        BitSet optionals = new BitSet();
+        if (struct.isSetIo()) {
+          optionals.set(0);
+        }
+        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetIo()) {
+          struct.io.write(oprot);
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, increment_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        BitSet incoming = iprot.readBitSet(1);
+        if (incoming.get(0)) {
+          struct.io = new IOError();
+          struct.io.read(iprot);
+          struct.setIoIsSet(true);
+        }
+      }
+    }
+
+  }
+
+  public static class incrementRows_args implements org.apache.thrift.TBase<incrementRows_args, incrementRows_args._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("incrementRows_args");
+
+    private static final org.apache.thrift.protocol.TField INCREMENTS_FIELD_DESC = new org.apache.thrift.protocol.TField("increments", org.apache.thrift.protocol.TType.LIST, (short)1);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new incrementRows_argsStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new incrementRows_argsTupleSchemeFactory());
+    }
+
+    /**
+     * The list of increments
+     */
+    public List<TIncrement> increments; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      /**
+       * The list of increments
+       */
+      INCREMENTS((short)1, "increments");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // INCREMENTS
+            return INCREMENTS;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.INCREMENTS, new org.apache.thrift.meta_data.FieldMetaData("increments", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+              new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TIncrement.class))));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(incrementRows_args.class, metaDataMap);
+    }
+
+    public incrementRows_args() {
+    }
+
+    public incrementRows_args(
+      List<TIncrement> increments)
+    {
+      this();
+      this.increments = increments;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public incrementRows_args(incrementRows_args other) {
+      if (other.isSetIncrements()) {
+        List<TIncrement> __this__increments = new ArrayList<TIncrement>();
+        for (TIncrement other_element : other.increments) {
+          __this__increments.add(new TIncrement(other_element));
+        }
+        this.increments = __this__increments;
+      }
+    }
+
+    public incrementRows_args deepCopy() {
+      return new incrementRows_args(this);
+    }
+
+    @Override
+    public void clear() {
+      this.increments = null;
+    }
+
+    public int getIncrementsSize() {
+      return (this.increments == null) ? 0 : this.increments.size();
+    }
+
+    public java.util.Iterator<TIncrement> getIncrementsIterator() {
+      return (this.increments == null) ? null : this.increments.iterator();
+    }
+
+    public void addToIncrements(TIncrement elem) {
+      if (this.increments == null) {
+        this.increments = new ArrayList<TIncrement>();
+      }
+      this.increments.add(elem);
+    }
+
+    /**
+     * The list of increments
+     */
+    public List<TIncrement> getIncrements() {
+      return this.increments;
+    }
+
+    /**
+     * The list of increments
+     */
+    public incrementRows_args setIncrements(List<TIncrement> increments) {
+      this.increments = increments;
+      return this;
+    }
+
+    public void unsetIncrements() {
+      this.increments = null;
+    }
+
+    /** Returns true if field increments is set (has been assigned a value) and false otherwise */
+    public boolean isSetIncrements() {
+      return this.increments != null;
+    }
+
+    public void setIncrementsIsSet(boolean value) {
+      if (!value) {
+        this.increments = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case INCREMENTS:
+        if (value == null) {
+          unsetIncrements();
+        } else {
+          setIncrements((List<TIncrement>)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case INCREMENTS:
+        return getIncrements();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case INCREMENTS:
+        return isSetIncrements();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof incrementRows_args)
+        return this.equals((incrementRows_args)that);
+      return false;
+    }
+
+    public boolean equals(incrementRows_args that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_increments = true && this.isSetIncrements();
+      boolean that_present_increments = true && that.isSetIncrements();
+      if (this_present_increments || that_present_increments) {
+        if (!(this_present_increments && that_present_increments))
+          return false;
+        if (!this.increments.equals(that.increments))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(incrementRows_args other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      incrementRows_args typedOther = (incrementRows_args)other;
+
+      lastComparison = Boolean.valueOf(isSetIncrements()).compareTo(typedOther.isSetIncrements());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetIncrements()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.increments, typedOther.increments);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("incrementRows_args(");
+      boolean first = true;
+
+      sb.append("increments:");
+      if (this.increments == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.increments);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class incrementRows_argsStandardSchemeFactory implements SchemeFactory {
+      public incrementRows_argsStandardScheme getScheme() {
+        return new incrementRows_argsStandardScheme();
+      }
+    }
+
+    private static class incrementRows_argsStandardScheme extends StandardScheme<incrementRows_args> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, incrementRows_args struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // INCREMENTS
+              if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+                {
+                  org.apache.thrift.protocol.TList _list424 = iprot.readListBegin();
+                  struct.increments = new ArrayList<TIncrement>(_list424.size);
+                  for (int _i425 = 0; _i425 < _list424.size; ++_i425)
+                  {
+                    TIncrement _elem426; // optional
+                    _elem426 = new TIncrement();
+                    _elem426.read(iprot);
+                    struct.increments.add(_elem426);
+                  }
+                  iprot.readListEnd();
+                }
+                struct.setIncrementsIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, incrementRows_args struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.increments != null) {
+          oprot.writeFieldBegin(INCREMENTS_FIELD_DESC);
+          {
+            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.increments.size()));
+            for (TIncrement _iter427 : struct.increments)
+            {
+              _iter427.write(oprot);
+            }
+            oprot.writeListEnd();
+          }
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class incrementRows_argsTupleSchemeFactory implements SchemeFactory {
+      public incrementRows_argsTupleScheme getScheme() {
+        return new incrementRows_argsTupleScheme();
+      }
+    }
+
+    private static class incrementRows_argsTupleScheme extends TupleScheme<incrementRows_args> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, incrementRows_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        BitSet optionals = new BitSet();
+        if (struct.isSetIncrements()) {
+          optionals.set(0);
+        }
+        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetIncrements()) {
+          {
+            oprot.writeI32(struct.increments.size());
+            for (TIncrement _iter428 : struct.increments)
+            {
+              _iter428.write(oprot);
+            }
+          }
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, incrementRows_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        BitSet incoming = iprot.readBitSet(1);
+        if (incoming.get(0)) {
+          {
+            org.apache.thrift.protocol.TList _list429 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.increments = new ArrayList<TIncrement>(_list429.size);
+            for (int _i430 = 0; _i430 < _list429.size; ++_i430)
+            {
+              TIncrement _elem431; // optional
+              _elem431 = new TIncrement();
+              _elem431.read(iprot);
+              struct.increments.add(_elem431);
+            }
+          }
+          struct.setIncrementsIsSet(true);
+        }
+      }
+    }
+
+  }
+
+  public static class incrementRows_result implements org.apache.thrift.TBase<incrementRows_result, incrementRows_result._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("incrementRows_result");
+
+    private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new incrementRows_resultStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new incrementRows_resultTupleSchemeFactory());
+    }
+
+    public IOError io; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      IO((short)1, "io");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // IO
+            return IO;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(incrementRows_result.class, metaDataMap);
+    }
+
+    public incrementRows_result() {
+    }
+
+    public incrementRows_result(
+      IOError io)
+    {
+      this();
+      this.io = io;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public incrementRows_result(incrementRows_result other) {
+      if (other.isSetIo()) {
+        this.io = new IOError(other.io);
+      }
+    }
+
+    public incrementRows_result deepCopy() {
+      return new incrementRows_result(this);
+    }
+
+    @Override
+    public void clear() {
+      this.io = null;
+    }
+
+    public IOError getIo() {
+      return this.io;
+    }
+
+    public incrementRows_result setIo(IOError io) {
+      this.io = io;
+      return this;
+    }
+
+    public void unsetIo() {
+      this.io = null;
+    }
+
+    /** Returns true if field io is set (has been assigned a value) and false otherwise */
+    public boolean isSetIo() {
+      return this.io != null;
+    }
+
+    public void setIoIsSet(boolean value) {
+      if (!value) {
+        this.io = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case IO:
+        if (value == null) {
+          unsetIo();
+        } else {
+          setIo((IOError)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case IO:
+        return getIo();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case IO:
+        return isSetIo();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof incrementRows_result)
+        return this.equals((incrementRows_result)that);
+      return false;
+    }
+
+    public boolean equals(incrementRows_result that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_io = true && this.isSetIo();
+      boolean that_present_io = true && that.isSetIo();
+      if (this_present_io || that_present_io) {
+        if (!(this_present_io && that_present_io))
+          return false;
+        if (!this.io.equals(that.io))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(incrementRows_result other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      incrementRows_result typedOther = (incrementRows_result)other;
+
+      lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetIo()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+      }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("incrementRows_result(");
+      boolean first = true;
+
+      sb.append("io:");
+      if (this.io == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.io);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class incrementRows_resultStandardSchemeFactory implements SchemeFactory {
+      public incrementRows_resultStandardScheme getScheme() {
+        return new incrementRows_resultStandardScheme();
+      }
+    }
+
+    private static class incrementRows_resultStandardScheme extends StandardScheme<incrementRows_result> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, incrementRows_result struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // IO
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.io = new IOError();
+                struct.io.read(iprot);
+                struct.setIoIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, incrementRows_result struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.io != null) {
+          oprot.writeFieldBegin(IO_FIELD_DESC);
+          struct.io.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class incrementRows_resultTupleSchemeFactory implements SchemeFactory {
+      public incrementRows_resultTupleScheme getScheme() {
+        return new incrementRows_resultTupleScheme();
+      }
+    }
+
+    private static class incrementRows_resultTupleScheme extends TupleScheme<incrementRows_result> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, incrementRows_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        BitSet optionals = new BitSet();
+        if (struct.isSetIo()) {
+          optionals.set(0);
+        }
+        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetIo()) {
+          struct.io.write(oprot);
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, incrementRows_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        BitSet incoming = iprot.readBitSet(1);
+        if (incoming.get(0)) {
+          struct.io = new IOError();
+          struct.io.read(iprot);
+          struct.setIoIsSet(true);
+        }
+      }
+    }
+
+  }
+
+  public static class deleteAllRowTs_args implements org.apache.thrift.TBase<deleteAllRowTs_args, deleteAllRowTs_args._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllRowTs_args");
+
+    private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1);
+    private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2);
+    private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
+    private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new deleteAllRowTs_argsStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new deleteAllRowTs_argsTupleSchemeFactory());
+    }
+
+    /**
+     * name of table
+     */
+    public ByteBuffer tableName; // required
+    /**
+     * key of the row to be completely deleted.
+     */
+    public ByteBuffer row; // required
+    /**
+     * timestamp
+     */
+    public long timestamp; // required
+    /**
+     * Delete attributes
+     */
+    public Map<ByteBuffer,ByteBuffer> attributes; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      /**
+       * name of table
+       */
+      TABLE_NAME((short)1, "tableName"),
+      /**
+       * key of the row to be completely deleted.
+       */
+      ROW((short)2, "row"),
+      /**
+       * timestamp
+       */
+      TIMESTAMP((short)3, "timestamp"),
+      /**
+       * Delete attributes
+       */
+      ATTRIBUTES((short)4, "attributes");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // TABLE_NAME
+            return TABLE_NAME;
+          case 2: // ROW
+            return ROW;
+          case 3: // TIMESTAMP
+            return TIMESTAMP;
+          case 4: // ATTRIBUTES
+            return ATTRIBUTES;
           default:
             return null;
         }
@@ -37872,15 +39542,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRowTs_args
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map424 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map424.size);
-                  for (int _i425 = 0; _i425 < _map424.size; ++_i425)
+                  org.apache.thrift.protocol.TMap _map432 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map432.size);
+                  for (int _i433 = 0; _i433 < _map432.size; ++_i433)
                   {
-                    ByteBuffer _key426; // required
-                    ByteBuffer _val427; // required
-                    _key426 = iprot.readBinary();
-                    _val427 = iprot.readBinary();
-                    struct.attributes.put(_key426, _val427);
+                    ByteBuffer _key434; // required
+                    ByteBuffer _val435; // optional
+                    _key434 = iprot.readBinary();
+                    _val435 = iprot.readBinary();
+                    struct.attributes.put(_key434, _val435);
                   }
                   iprot.readMapEnd();
                 }
@@ -37921,10 +39591,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRowTs_arg
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter428 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter436 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter428.getKey());
-              oprot.writeBinary(_iter428.getValue());
+              oprot.writeBinary(_iter436.getKey());
+              oprot.writeBinary(_iter436.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -37973,10 +39643,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_args
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter429 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter437 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter429.getKey());
-              oprot.writeBinary(_iter429.getValue());
+              oprot.writeBinary(_iter437.getKey());
+              oprot.writeBinary(_iter437.getValue());
             }
           }
         }
@@ -38000,15 +39670,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_args
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map430 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map430.size);
-            for (int _i431 = 0; _i431 < _map430.size; ++_i431)
+            org.apache.thrift.protocol.TMap _map438 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map438.size);
+            for (int _i439 = 0; _i439 < _map438.size; ++_i439)
             {
-              ByteBuffer _key432; // required
-              ByteBuffer _val433; // required
-              _key432 = iprot.readBinary();
-              _val433 = iprot.readBinary();
-              struct.attributes.put(_key432, _val433);
+              ByteBuffer _key440; // required
+              ByteBuffer _val441; // optional
+              _key440 = iprot.readBinary();
+              _val441 = iprot.readBinary();
+              struct.attributes.put(_key440, _val441);
             }
           }
           struct.setAttributesIsSet(true);
@@ -38903,15 +40573,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithScan
             case 3: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map434 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map434.size);
-                  for (int _i435 = 0; _i435 < _map434.size; ++_i435)
+                  org.apache.thrift.protocol.TMap _map442 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map442.size);
+                  for (int _i443 = 0; _i443 < _map442.size; ++_i443)
                   {
-                    ByteBuffer _key436; // required
-                    ByteBuffer _val437; // required
-                    _key436 = iprot.readBinary();
-                    _val437 = iprot.readBinary();
-                    struct.attributes.put(_key436, _val437);
+                    ByteBuffer _key444; // required
+                    ByteBuffer _val445; // optional
+                    _key444 = iprot.readBinary();
+                    _val445 = iprot.readBinary();
+                    struct.attributes.put(_key444, _val445);
                   }
                   iprot.readMapEnd();
                 }
@@ -38949,10 +40619,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSca
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter438 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter446 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter438.getKey());
-              oprot.writeBinary(_iter438.getValue());
+              oprot.writeBinary(_iter446.getKey());
+              oprot.writeBinary(_iter446.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -38995,10 +40665,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter439 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter447 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter439.getKey());
-              oprot.writeBinary(_iter439.getValue());
+              oprot.writeBinary(_iter447.getKey());
+              oprot.writeBinary(_iter447.getValue());
             }
           }
         }
@@ -39019,15 +40689,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan_
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TMap _map440 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map440.size);
-            for (int _i441 = 0; _i441 < _map440.size; ++_i441)
+            org.apache.thrift.protocol.TMap _map448 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map448.size);
+            for (int _i449 = 0; _i449 < _map448.size; ++_i449)
             {
-              ByteBuffer _key442; // required
-              ByteBuffer _val443; // required
-              _key442 = iprot.readBinary();
-              _val443 = iprot.readBinary();
-              struct.attributes.put(_key442, _val443);
+              ByteBuffer _key450; // required
+              ByteBuffer _val451; // optional
+              _key450 = iprot.readBinary();
+              _val451 = iprot.readBinary();
+              struct.attributes.put(_key450, _val451);
             }
           }
           struct.setAttributesIsSet(true);
@@ -39374,6 +41044,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -40149,13 +41821,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpen_args st
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list444 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list444.size);
-                  for (int _i445 = 0; _i445 < _list444.size; ++_i445)
+                  org.apache.thrift.protocol.TList _list452 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list452.size);
+                  for (int _i453 = 0; _i453 < _list452.size; ++_i453)
                   {
-                    ByteBuffer _elem446; // optional
-                    _elem446 = iprot.readBinary();
-                    struct.columns.add(_elem446);
+                    ByteBuffer _elem454; // optional
+                    _elem454 = iprot.readBinary();
+                    struct.columns.add(_elem454);
                   }
                   iprot.readListEnd();
                 }
@@ -40167,15 +41839,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpen_args st
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map447 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map447.size);
-                  for (int _i448 = 0; _i448 < _map447.size; ++_i448)
+                  org.apache.thrift.protocol.TMap _map455 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map455.size);
+                  for (int _i456 = 0; _i456 < _map455.size; ++_i456)
                   {
-                    ByteBuffer _key449; // required
-                    ByteBuffer _val450; // required
-                    _key449 = iprot.readBinary();
-                    _val450 = iprot.readBinary();
-                    struct.attributes.put(_key449, _val450);
+                    ByteBuffer _key457; // required
+                    ByteBuffer _val458; // optional
+                    _key457 = iprot.readBinary();
+                    _val458 = iprot.readBinary();
+                    struct.attributes.put(_key457, _val458);
                   }
                   iprot.readMapEnd();
                 }
@@ -40213,9 +41885,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpen_args s
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter451 : struct.columns)
+            for (ByteBuffer _iter459 : struct.columns)
             {
-              oprot.writeBinary(_iter451);
+              oprot.writeBinary(_iter459);
             }
             oprot.writeListEnd();
           }
@@ -40225,10 +41897,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpen_args s
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter452 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter460 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter452.getKey());
-              oprot.writeBinary(_iter452.getValue());
+              oprot.writeBinary(_iter460.getKey());
+              oprot.writeBinary(_iter460.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -40274,19 +41946,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpen_args st
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter453 : struct.columns)
+            for (ByteBuffer _iter461 : struct.columns)
             {
-              oprot.writeBinary(_iter453);
+              oprot.writeBinary(_iter461);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter454 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter462 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter454.getKey());
-              oprot.writeBinary(_iter454.getValue());
+              oprot.writeBinary(_iter462.getKey());
+              oprot.writeBinary(_iter462.getValue());
             }
           }
         }
@@ -40306,28 +41978,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpen_args str
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list455 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list455.size);
-            for (int _i456 = 0; _i456 < _list455.size; ++_i456)
+            org.apache.thrift.protocol.TList _list463 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list463.size);
+            for (int _i464 = 0; _i464 < _list463.size; ++_i464)
             {
-              ByteBuffer _elem457; // optional
-              _elem457 = iprot.readBinary();
-              struct.columns.add(_elem457);
+              ByteBuffer _elem465; // optional
+              _elem465 = iprot.readBinary();
+              struct.columns.add(_elem465);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map458 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map458.size);
-            for (int _i459 = 0; _i459 < _map458.size; ++_i459)
+            org.apache.thrift.protocol.TMap _map466 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map466.size);
+            for (int _i467 = 0; _i467 < _map466.size; ++_i467)
             {
-              ByteBuffer _key460; // required
-              ByteBuffer _val461; // required
-              _key460 = iprot.readBinary();
-              _val461 = iprot.readBinary();
-              struct.attributes.put(_key460, _val461);
+              ByteBuffer _key468; // required
+              ByteBuffer _val469; // optional
+              _key468 = iprot.readBinary();
+              _val469 = iprot.readBinary();
+              struct.attributes.put(_key468, _val469);
             }
           }
           struct.setAttributesIsSet(true);
@@ -40674,6 +42346,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -41560,13 +43234,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 4: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list462 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list462.size);
-                  for (int _i463 = 0; _i463 < _list462.size; ++_i463)
+                  org.apache.thrift.protocol.TList _list470 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list470.size);
+                  for (int _i471 = 0; _i471 < _list470.size; ++_i471)
                   {
-                    ByteBuffer _elem464; // optional
-                    _elem464 = iprot.readBinary();
-                    struct.columns.add(_elem464);
+                    ByteBuffer _elem472; // optional
+                    _elem472 = iprot.readBinary();
+                    struct.columns.add(_elem472);
                   }
                   iprot.readListEnd();
                 }
@@ -41578,15 +43252,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map465 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map465.size);
-                  for (int _i466 = 0; _i466 < _map465.size; ++_i466)
+                  org.apache.thrift.protocol.TMap _map473 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map473.size);
+                  for (int _i474 = 0; _i474 < _map473.size; ++_i474)
                   {
-                    ByteBuffer _key467; // required
-                    ByteBuffer _val468; // required
-                    _key467 = iprot.readBinary();
-                    _val468 = iprot.readBinary();
-                    struct.attributes.put(_key467, _val468);
+                    ByteBuffer _key475; // required
+                    ByteBuffer _val476; // optional
+                    _key475 = iprot.readBinary();
+                    _val476 = iprot.readBinary();
+                    struct.attributes.put(_key475, _val476);
                   }
                   iprot.readMapEnd();
                 }
@@ -41629,9 +43303,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter469 : struct.columns)
+            for (ByteBuffer _iter477 : struct.columns)
             {
-              oprot.writeBinary(_iter469);
+              oprot.writeBinary(_iter477);
             }
             oprot.writeListEnd();
           }
@@ -41641,10 +43315,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter470 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter478 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter470.getKey());
-              oprot.writeBinary(_iter470.getValue());
+              oprot.writeBinary(_iter478.getKey());
+              oprot.writeBinary(_iter478.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -41696,19 +43370,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter471 : struct.columns)
+            for (ByteBuffer _iter479 : struct.columns)
             {
-              oprot.writeBinary(_iter471);
+              oprot.writeBinary(_iter479);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter472 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter480 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter472.getKey());
-              oprot.writeBinary(_iter472.getValue());
+              oprot.writeBinary(_iter480.getKey());
+              oprot.writeBinary(_iter480.getValue());
             }
           }
         }
@@ -41732,28 +43406,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop_
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TList _list473 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list473.size);
-            for (int _i474 = 0; _i474 < _list473.size; ++_i474)
+            org.apache.thrift.protocol.TList _list481 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list481.size);
+            for (int _i482 = 0; _i482 < _list481.size; ++_i482)
             {
-              ByteBuffer _elem475; // optional
-              _elem475 = iprot.readBinary();
-              struct.columns.add(_elem475);
+              ByteBuffer _elem483; // optional
+              _elem483 = iprot.readBinary();
+              struct.columns.add(_elem483);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map476 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map476.size);
-            for (int _i477 = 0; _i477 < _map476.size; ++_i477)
+            org.apache.thrift.protocol.TMap _map484 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map484.size);
+            for (int _i485 = 0; _i485 < _map484.size; ++_i485)
             {
-              ByteBuffer _key478; // required
-              ByteBuffer _val479; // required
-              _key478 = iprot.readBinary();
-              _val479 = iprot.readBinary();
-              struct.attributes.put(_key478, _val479);
+              ByteBuffer _key486; // required
+              ByteBuffer _val487; // optional
+              _key486 = iprot.readBinary();
+              _val487 = iprot.readBinary();
+              struct.attributes.put(_key486, _val487);
             }
           }
           struct.setAttributesIsSet(true);
@@ -42100,6 +43774,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -42863,13 +44539,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithPref
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list480 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list480.size);
-                  for (int _i481 = 0; _i481 < _list480.size; ++_i481)
+                  org.apache.thrift.protocol.TList _list488 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list488.size);
+                  for (int _i489 = 0; _i489 < _list488.size; ++_i489)
                   {
-                    ByteBuffer _elem482; // optional
-                    _elem482 = iprot.readBinary();
-                    struct.columns.add(_elem482);
+                    ByteBuffer _elem490; // optional
+                    _elem490 = iprot.readBinary();
+                    struct.columns.add(_elem490);
                   }
                   iprot.readListEnd();
                 }
@@ -42881,15 +44557,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithPref
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map483 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map483.size);
-                  for (int _i484 = 0; _i484 < _map483.size; ++_i484)
+                  org.apache.thrift.protocol.TMap _map491 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map491.size);
+                  for (int _i492 = 0; _i492 < _map491.size; ++_i492)
                   {
-                    ByteBuffer _key485; // required
-                    ByteBuffer _val486; // required
-                    _key485 = iprot.readBinary();
-                    _val486 = iprot.readBinary();
-                    struct.attributes.put(_key485, _val486);
+                    ByteBuffer _key493; // required
+                    ByteBuffer _val494; // optional
+                    _key493 = iprot.readBinary();
+                    _val494 = iprot.readBinary();
+                    struct.attributes.put(_key493, _val494);
                   }
                   iprot.readMapEnd();
                 }
@@ -42927,9 +44603,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithPre
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter487 : struct.columns)
+            for (ByteBuffer _iter495 : struct.columns)
             {
-              oprot.writeBinary(_iter487);
+              oprot.writeBinary(_iter495);
             }
             oprot.writeListEnd();
           }
@@ -42939,10 +44615,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithPre
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter488 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter496 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter488.getKey());
-              oprot.writeBinary(_iter488.getValue());
+              oprot.writeBinary(_iter496.getKey());
+              oprot.writeBinary(_iter496.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -42988,19 +44664,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPref
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter489 : struct.columns)
+            for (ByteBuffer _iter497 : struct.columns)
             {
-              oprot.writeBinary(_iter489);
+              oprot.writeBinary(_iter497);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter490 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter498 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter490.getKey());
-              oprot.writeBinary(_iter490.getValue());
+              oprot.writeBinary(_iter498.getKey());
+              oprot.writeBinary(_iter498.getValue());
             }
           }
         }
@@ -43020,28 +44696,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPrefi
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list491 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list491.size);
-            for (int _i492 = 0; _i492 < _list491.size; ++_i492)
+            org.apache.thrift.protocol.TList _list499 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list499.size);
+            for (int _i500 = 0; _i500 < _list499.size; ++_i500)
             {
-              ByteBuffer _elem493; // optional
-              _elem493 = iprot.readBinary();
-              struct.columns.add(_elem493);
+              ByteBuffer _elem501; // optional
+              _elem501 = iprot.readBinary();
+              struct.columns.add(_elem501);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map494 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map494.size);
-            for (int _i495 = 0; _i495 < _map494.size; ++_i495)
+            org.apache.thrift.protocol.TMap _map502 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map502.size);
+            for (int _i503 = 0; _i503 < _map502.size; ++_i503)
             {
-              ByteBuffer _key496; // required
-              ByteBuffer _val497; // required
-              _key496 = iprot.readBinary();
-              _val497 = iprot.readBinary();
-              struct.attributes.put(_key496, _val497);
+              ByteBuffer _key504; // required
+              ByteBuffer _val505; // optional
+              _key504 = iprot.readBinary();
+              _val505 = iprot.readBinary();
+              struct.attributes.put(_key504, _val505);
             }
           }
           struct.setAttributesIsSet(true);
@@ -43388,6 +45064,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -44208,6 +45886,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -44251,13 +45931,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenTs_args
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list498 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list498.size);
-                  for (int _i499 = 0; _i499 < _list498.size; ++_i499)
+                  org.apache.thrift.protocol.TList _list506 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list506.size);
+                  for (int _i507 = 0; _i507 < _list506.size; ++_i507)
                   {
-                    ByteBuffer _elem500; // optional
-                    _elem500 = iprot.readBinary();
-                    struct.columns.add(_elem500);
+                    ByteBuffer _elem508; // optional
+                    _elem508 = iprot.readBinary();
+                    struct.columns.add(_elem508);
                   }
                   iprot.readListEnd();
                 }
@@ -44277,15 +45957,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenTs_args
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map501 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map501.size);
-                  for (int _i502 = 0; _i502 < _map501.size; ++_i502)
+                  org.apache.thrift.protocol.TMap _map509 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map509.size);
+                  for (int _i510 = 0; _i510 < _map509.size; ++_i510)
                   {
-                    ByteBuffer _key503; // required
-                    ByteBuffer _val504; // required
-                    _key503 = iprot.readBinary();
-                    _val504 = iprot.readBinary();
-                    struct.attributes.put(_key503, _val504);
+                    ByteBuffer _key511; // required
+                    ByteBuffer _val512; // optional
+                    _key511 = iprot.readBinary();
+                    _val512 = iprot.readBinary();
+                    struct.attributes.put(_key511, _val512);
                   }
                   iprot.readMapEnd();
                 }
@@ -44323,9 +46003,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenTs_args
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter505 : struct.columns)
+            for (ByteBuffer _iter513 : struct.columns)
             {
-              oprot.writeBinary(_iter505);
+              oprot.writeBinary(_iter513);
             }
             oprot.writeListEnd();
           }
@@ -44338,10 +46018,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenTs_args
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter506 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter514 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter506.getKey());
-              oprot.writeBinary(_iter506.getValue());
+              oprot.writeBinary(_iter514.getKey());
+              oprot.writeBinary(_iter514.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -44390,9 +46070,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter507 : struct.columns)
+            for (ByteBuffer _iter515 : struct.columns)
             {
-              oprot.writeBinary(_iter507);
+              oprot.writeBinary(_iter515);
             }
           }
         }
@@ -44402,10 +46082,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter508 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter516 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter508.getKey());
-              oprot.writeBinary(_iter508.getValue());
+              oprot.writeBinary(_iter516.getKey());
+              oprot.writeBinary(_iter516.getValue());
             }
           }
         }
@@ -44425,13 +46105,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args s
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list509 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list509.size);
-            for (int _i510 = 0; _i510 < _list509.size; ++_i510)
+            org.apache.thrift.protocol.TList _list517 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list517.size);
+            for (int _i518 = 0; _i518 < _list517.size; ++_i518)
             {
-              ByteBuffer _elem511; // optional
-              _elem511 = iprot.readBinary();
-              struct.columns.add(_elem511);
+              ByteBuffer _elem519; // optional
+              _elem519 = iprot.readBinary();
+              struct.columns.add(_elem519);
             }
           }
           struct.setColumnsIsSet(true);
@@ -44442,15 +46122,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args s
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map512 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map512.size);
-            for (int _i513 = 0; _i513 < _map512.size; ++_i513)
+            org.apache.thrift.protocol.TMap _map520 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map520.size);
+            for (int _i521 = 0; _i521 < _map520.size; ++_i521)
             {
-              ByteBuffer _key514; // required
-              ByteBuffer _val515; // required
-              _key514 = iprot.readBinary();
-              _val515 = iprot.readBinary();
-              struct.attributes.put(_key514, _val515);
+              ByteBuffer _key522; // required
+              ByteBuffer _val523; // optional
+              _key522 = iprot.readBinary();
+              _val523 = iprot.readBinary();
+              struct.attributes.put(_key522, _val523);
             }
           }
           struct.setAttributesIsSet(true);
@@ -44797,6 +46477,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -45771,13 +47453,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 4: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list516 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list516.size);
-                  for (int _i517 = 0; _i517 < _list516.size; ++_i517)
+                  org.apache.thrift.protocol.TList _list524 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list524.size);
+                  for (int _i525 = 0; _i525 < _list524.size; ++_i525)
                   {
-                    ByteBuffer _elem518; // optional
-                    _elem518 = iprot.readBinary();
-                    struct.columns.add(_elem518);
+                    ByteBuffer _elem526; // optional
+                    _elem526 = iprot.readBinary();
+                    struct.columns.add(_elem526);
                   }
                   iprot.readListEnd();
                 }
@@ -45797,15 +47479,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 6: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map519 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map519.size);
-                  for (int _i520 = 0; _i520 < _map519.size; ++_i520)
+                  org.apache.thrift.protocol.TMap _map527 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map527.size);
+                  for (int _i528 = 0; _i528 < _map527.size; ++_i528)
                   {
-                    ByteBuffer _key521; // required
-                    ByteBuffer _val522; // required
-                    _key521 = iprot.readBinary();
-                    _val522 = iprot.readBinary();
-                    struct.attributes.put(_key521, _val522);
+                    ByteBuffer _key529; // required
+                    ByteBuffer _val530; // optional
+                    _key529 = iprot.readBinary();
+                    _val530 = iprot.readBinary();
+                    struct.attributes.put(_key529, _val530);
                   }
                   iprot.readMapEnd();
                 }
@@ -45848,9 +47530,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter523 : struct.columns)
+            for (ByteBuffer _iter531 : struct.columns)
             {
-              oprot.writeBinary(_iter523);
+              oprot.writeBinary(_iter531);
             }
             oprot.writeListEnd();
           }
@@ -45863,10 +47545,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter524 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter532 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter524.getKey());
-              oprot.writeBinary(_iter524.getValue());
+              oprot.writeBinary(_iter532.getKey());
+              oprot.writeBinary(_iter532.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -45921,9 +47603,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter525 : struct.columns)
+            for (ByteBuffer _iter533 : struct.columns)
             {
-              oprot.writeBinary(_iter525);
+              oprot.writeBinary(_iter533);
             }
           }
         }
@@ -45933,10 +47615,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter526 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter534 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter526.getKey());
-              oprot.writeBinary(_iter526.getValue());
+              oprot.writeBinary(_iter534.getKey());
+              oprot.writeBinary(_iter534.getValue());
             }
           }
         }
@@ -45960,13 +47642,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopT
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TList _list527 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list527.size);
-            for (int _i528 = 0; _i528 < _list527.size; ++_i528)
+            org.apache.thrift.protocol.TList _list535 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list535.size);
+            for (int _i536 = 0; _i536 < _list535.size; ++_i536)
             {
-              ByteBuffer _elem529; // optional
-              _elem529 = iprot.readBinary();
-              struct.columns.add(_elem529);
+              ByteBuffer _elem537; // optional
+              _elem537 = iprot.readBinary();
+              struct.columns.add(_elem537);
             }
           }
           struct.setColumnsIsSet(true);
@@ -45977,15 +47659,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopT
         }
         if (incoming.get(5)) {
           {
-            org.apache.thrift.protocol.TMap _map530 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map530.size);
-            for (int _i531 = 0; _i531 < _map530.size; ++_i531)
+            org.apache.thrift.protocol.TMap _map538 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map538.size);
+            for (int _i539 = 0; _i539 < _map538.size; ++_i539)
             {
-              ByteBuffer _key532; // required
-              ByteBuffer _val533; // required
-              _key532 = iprot.readBinary();
-              _val533 = iprot.readBinary();
-              struct.attributes.put(_key532, _val533);
+              ByteBuffer _key540; // required
+              ByteBuffer _val541; // optional
+              _key540 = iprot.readBinary();
+              _val541 = iprot.readBinary();
+              struct.attributes.put(_key540, _val541);
             }
           }
           struct.setAttributesIsSet(true);
@@ -46332,6 +48014,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -47273,14 +48957,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGet_result s
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list534 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list534.size);
-                  for (int _i535 = 0; _i535 < _list534.size; ++_i535)
+                  org.apache.thrift.protocol.TList _list542 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list542.size);
+                  for (int _i543 = 0; _i543 < _list542.size; ++_i543)
                   {
-                    TRowResult _elem536; // optional
-                    _elem536 = new TRowResult();
-                    _elem536.read(iprot);
-                    struct.success.add(_elem536);
+                    TRowResult _elem544; // optional
+                    _elem544 = new TRowResult();
+                    _elem544.read(iprot);
+                    struct.success.add(_elem544);
                   }
                   iprot.readListEnd();
                 }
@@ -47326,9 +49010,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGet_result
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter537 : struct.success)
+            for (TRowResult _iter545 : struct.success)
             {
-              _iter537.write(oprot);
+              _iter545.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -47375,9 +49059,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerGet_result s
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter538 : struct.success)
+            for (TRowResult _iter546 : struct.success)
             {
-              _iter538.write(oprot);
+              _iter546.write(oprot);
             }
           }
         }
@@ -47395,14 +49079,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerGet_result st
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list539 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list539.size);
-            for (int _i540 = 0; _i540 < _list539.size; ++_i540)
+            org.apache.thrift.protocol.TList _list547 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list547.size);
+            for (int _i548 = 0; _i548 < _list547.size; ++_i548)
             {
-              TRowResult _elem541; // optional
-              _elem541 = new TRowResult();
-              _elem541.read(iprot);
-              struct.success.add(_elem541);
+              TRowResult _elem549; // optional
+              _elem549 = new TRowResult();
+              _elem549.read(iprot);
+              struct.success.add(_elem549);
             }
           }
           struct.setSuccessIsSet(true);
@@ -48352,14 +50036,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGetList_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list542 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list542.size);
-                  for (int _i543 = 0; _i543 < _list542.size; ++_i543)
+                  org.apache.thrift.protocol.TList _list550 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list550.size);
+                  for (int _i551 = 0; _i551 < _list550.size; ++_i551)
                   {
-                    TRowResult _elem544; // optional
-                    _elem544 = new TRowResult();
-                    _elem544.read(iprot);
-                    struct.success.add(_elem544);
+                    TRowResult _elem552; // optional
+                    _elem552 = new TRowResult();
+                    _elem552.read(iprot);
+                    struct.success.add(_elem552);
                   }
                   iprot.readListEnd();
                 }
@@ -48405,9 +50089,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGetList_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter545 : struct.success)
+            for (TRowResult _iter553 : struct.success)
             {
-              _iter545.write(oprot);
+              _iter553.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -48454,9 +50138,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerGetList_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter546 : struct.success)
+            for (TRowResult _iter554 : struct.success)
             {
-              _iter546.write(oprot);
+              _iter554.write(oprot);
             }
           }
         }
@@ -48474,14 +50158,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerGetList_resul
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list547 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list547.size);
-            for (int _i548 = 0; _i548 < _list547.size; ++_i548)
+            org.apache.thrift.protocol.TList _list555 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list555.size);
+            for (int _i556 = 0; _i556 < _list555.size; ++_i556)
             {
-              TRowResult _elem549; // optional
-              _elem549 = new TRowResult();
-              _elem549.read(iprot);
-              struct.success.add(_elem549);
+              TRowResult _elem557; // optional
+              _elem557 = new TRowResult();
+              _elem557.read(iprot);
+              struct.success.add(_elem557);
             }
           }
           struct.setSuccessIsSet(true);
@@ -50326,14 +52010,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowOrBefore_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list550 = iprot.readListBegin();
-                  struct.success = new ArrayList<TCell>(_list550.size);
-                  for (int _i551 = 0; _i551 < _list550.size; ++_i551)
+                  org.apache.thrift.protocol.TList _list558 = iprot.readListBegin();
+                  struct.success = new ArrayList<TCell>(_list558.size);
+                  for (int _i559 = 0; _i559 < _list558.size; ++_i559)
                   {
-                    TCell _elem552; // optional
-                    _elem552 = new TCell();
-                    _elem552.read(iprot);
-                    struct.success.add(_elem552);
+                    TCell _elem560; // optional
+                    _elem560 = new TCell();
+                    _elem560.read(iprot);
+                    struct.success.add(_elem560);
                   }
                   iprot.readListEnd();
                 }
@@ -50370,9 +52054,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowOrBefore_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TCell _iter553 : struct.success)
+            for (TCell _iter561 : struct.success)
             {
-              _iter553.write(oprot);
+              _iter561.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -50411,9 +52095,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TCell _iter554 : struct.success)
+            for (TCell _iter562 : struct.success)
             {
-              _iter554.write(oprot);
+              _iter562.write(oprot);
             }
           }
         }
@@ -50428,14 +52112,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_resul
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list555 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TCell>(_list555.size);
-            for (int _i556 = 0; _i556 < _list555.size; ++_i556)
+            org.apache.thrift.protocol.TList _list563 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TCell>(_list563.size);
+            for (int _i564 = 0; _i564 < _list563.size; ++_i564)
             {
-              TCell _elem557; // optional
-              _elem557 = new TCell();
-              _elem557.read(iprot);
-              struct.success.add(_elem557);
+              TCell _elem565; // optional
+              _elem565 = new TCell();
+              _elem565.read(iprot);
+              struct.success.add(_elem565);
             }
           }
           struct.setSuccessIsSet(true);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java
new file mode 100644
index 000000000000..6d24aa08c042
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java
@@ -0,0 +1,715 @@
+/**
+ * Autogenerated by Thrift Compiler (0.8.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hbase.thrift.generated;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * For increments that are not incrementColumnValue
+ * equivalents.
+ */
+public class TIncrement implements org.apache.thrift.TBase<TIncrement, TIncrement._Fields>, java.io.Serializable, Cloneable {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIncrement");
+
+  private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1);
+  private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2);
+  private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3);
+  private static final org.apache.thrift.protocol.TField AMMOUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("ammount", org.apache.thrift.protocol.TType.I64, (short)4);
+
+  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+  static {
+    schemes.put(StandardScheme.class, new TIncrementStandardSchemeFactory());
+    schemes.put(TupleScheme.class, new TIncrementTupleSchemeFactory());
+  }
+
+  public ByteBuffer table; // required
+  public ByteBuffer row; // required
+  public ByteBuffer column; // required
+  public long ammount; // required
+
+  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+    TABLE((short)1, "table"),
+    ROW((short)2, "row"),
+    COLUMN((short)3, "column"),
+    AMMOUNT((short)4, "ammount");
+
+    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+    static {
+      for (_Fields field : EnumSet.allOf(_Fields.class)) {
+        byName.put(field.getFieldName(), field);
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, or null if its not found.
+     */
+    public static _Fields findByThriftId(int fieldId) {
+      switch(fieldId) {
+        case 1: // TABLE
+          return TABLE;
+        case 2: // ROW
+          return ROW;
+        case 3: // COLUMN
+          return COLUMN;
+        case 4: // AMMOUNT
+          return AMMOUNT;
+        default:
+          return null;
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, throwing an exception
+     * if it is not found.
+     */
+    public static _Fields findByThriftIdOrThrow(int fieldId) {
+      _Fields fields = findByThriftId(fieldId);
+      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+      return fields;
+    }
+
+    /**
+     * Find the _Fields constant that matches name, or null if its not found.
+     */
+    public static _Fields findByName(String name) {
+      return byName.get(name);
+    }
+
+    private final short _thriftId;
+    private final String _fieldName;
+
+    _Fields(short thriftId, String fieldName) {
+      _thriftId = thriftId;
+      _fieldName = fieldName;
+    }
+
+    public short getThriftFieldId() {
+      return _thriftId;
+    }
+
+    public String getFieldName() {
+      return _fieldName;
+    }
+  }
+
+  // isset id assignments
+  private static final int __AMMOUNT_ISSET_ID = 0;
+  private BitSet __isset_bit_vector = new BitSet(1);
+  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+  static {
+    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+    tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "Text")));
+    tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "Text")));
+    tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "Text")));
+    tmpMap.put(_Fields.AMMOUNT, new org.apache.thrift.meta_data.FieldMetaData("ammount", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+    metaDataMap = Collections.unmodifiableMap(tmpMap);
+    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TIncrement.class, metaDataMap);
+  }
+
+  public TIncrement() {
+  }
+
+  public TIncrement(
+    ByteBuffer table,
+    ByteBuffer row,
+    ByteBuffer column,
+    long ammount)
+  {
+    this();
+    this.table = table;
+    this.row = row;
+    this.column = column;
+    this.ammount = ammount;
+    setAmmountIsSet(true);
+  }
+
+  /**
+   * Performs a deep copy on <i>other</i>.
+   */
+  public TIncrement(TIncrement other) {
+    __isset_bit_vector.clear();
+    __isset_bit_vector.or(other.__isset_bit_vector);
+    if (other.isSetTable()) {
+      this.table = other.table;
+    }
+    if (other.isSetRow()) {
+      this.row = other.row;
+    }
+    if (other.isSetColumn()) {
+      this.column = other.column;
+    }
+    this.ammount = other.ammount;
+  }
+
+  public TIncrement deepCopy() {
+    return new TIncrement(this);
+  }
+
+  @Override
+  public void clear() {
+    this.table = null;
+    this.row = null;
+    this.column = null;
+    setAmmountIsSet(false);
+    this.ammount = 0;
+  }
+
+  public byte[] getTable() {
+    setTable(org.apache.thrift.TBaseHelper.rightSize(table));
+    return table == null ? null : table.array();
+  }
+
+  public ByteBuffer bufferForTable() {
+    return table;
+  }
+
+  public TIncrement setTable(byte[] table) {
+    setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table));
+    return this;
+  }
+
+  public TIncrement setTable(ByteBuffer table) {
+    this.table = table;
+    return this;
+  }
+
+  public void unsetTable() {
+    this.table = null;
+  }
+
+  /** Returns true if field table is set (has been assigned a value) and false otherwise */
+  public boolean isSetTable() {
+    return this.table != null;
+  }
+
+  public void setTableIsSet(boolean value) {
+    if (!value) {
+      this.table = null;
+    }
+  }
+
+  public byte[] getRow() {
+    setRow(org.apache.thrift.TBaseHelper.rightSize(row));
+    return row == null ? null : row.array();
+  }
+
+  public ByteBuffer bufferForRow() {
+    return row;
+  }
+
+  public TIncrement setRow(byte[] row) {
+    setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row));
+    return this;
+  }
+
+  public TIncrement setRow(ByteBuffer row) {
+    this.row = row;
+    return this;
+  }
+
+  public void unsetRow() {
+    this.row = null;
+  }
+
+  /** Returns true if field row is set (has been assigned a value) and false otherwise */
+  public boolean isSetRow() {
+    return this.row != null;
+  }
+
+  public void setRowIsSet(boolean value) {
+    if (!value) {
+      this.row = null;
+    }
+  }
+
+  public byte[] getColumn() {
+    setColumn(org.apache.thrift.TBaseHelper.rightSize(column));
+    return column == null ? null : column.array();
+  }
+
+  public ByteBuffer bufferForColumn() {
+    return column;
+  }
+
+  public TIncrement setColumn(byte[] column) {
+    setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column));
+    return this;
+  }
+
+  public TIncrement setColumn(ByteBuffer column) {
+    this.column = column;
+    return this;
+  }
+
+  public void unsetColumn() {
+    this.column = null;
+  }
+
+  /** Returns true if field column is set (has been assigned a value) and false otherwise */
+  public boolean isSetColumn() {
+    return this.column != null;
+  }
+
+  public void setColumnIsSet(boolean value) {
+    if (!value) {
+      this.column = null;
+    }
+  }
+
+  public long getAmmount() {
+    return this.ammount;
+  }
+
+  public TIncrement setAmmount(long ammount) {
+    this.ammount = ammount;
+    setAmmountIsSet(true);
+    return this;
+  }
+
+  public void unsetAmmount() {
+    __isset_bit_vector.clear(__AMMOUNT_ISSET_ID);
+  }
+
+  /** Returns true if field ammount is set (has been assigned a value) and false otherwise */
+  public boolean isSetAmmount() {
+    return __isset_bit_vector.get(__AMMOUNT_ISSET_ID);
+  }
+
+  public void setAmmountIsSet(boolean value) {
+    __isset_bit_vector.set(__AMMOUNT_ISSET_ID, value);
+  }
+
+  public void setFieldValue(_Fields field, Object value) {
+    switch (field) {
+    case TABLE:
+      if (value == null) {
+        unsetTable();
+      } else {
+        setTable((ByteBuffer)value);
+      }
+      break;
+
+    case ROW:
+      if (value == null) {
+        unsetRow();
+      } else {
+        setRow((ByteBuffer)value);
+      }
+      break;
+
+    case COLUMN:
+      if (value == null) {
+        unsetColumn();
+      } else {
+        setColumn((ByteBuffer)value);
+      }
+      break;
+
+    case AMMOUNT:
+      if (value == null) {
+        unsetAmmount();
+      } else {
+        setAmmount((Long)value);
+      }
+      break;
+
+    }
+  }
+
+  public Object getFieldValue(_Fields field) {
+    switch (field) {
+    case TABLE:
+      return getTable();
+
+    case ROW:
+      return getRow();
+
+    case COLUMN:
+      return getColumn();
+
+    case AMMOUNT:
+      return Long.valueOf(getAmmount());
+
+    }
+    throw new IllegalStateException();
+  }
+
+  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+  public boolean isSet(_Fields field) {
+    if (field == null) {
+      throw new IllegalArgumentException();
+    }
+
+    switch (field) {
+    case TABLE:
+      return isSetTable();
+    case ROW:
+      return isSetRow();
+    case COLUMN:
+      return isSetColumn();
+    case AMMOUNT:
+      return isSetAmmount();
+    }
+    throw new IllegalStateException();
+  }
+
+  @Override
+  public boolean equals(Object that) {
+    if (that == null)
+      return false;
+    if (that instanceof TIncrement)
+      return this.equals((TIncrement)that);
+    return false;
+  }
+
+  public boolean equals(TIncrement that) {
+    if (that == null)
+      return false;
+
+    boolean this_present_table = true && this.isSetTable();
+    boolean that_present_table = true && that.isSetTable();
+    if (this_present_table || that_present_table) {
+      if (!(this_present_table && that_present_table))
+        return false;
+      if (!this.table.equals(that.table))
+        return false;
+    }
+
+    boolean this_present_row = true && this.isSetRow();
+    boolean that_present_row = true && that.isSetRow();
+    if (this_present_row || that_present_row) {
+      if (!(this_present_row && that_present_row))
+        return false;
+      if (!this.row.equals(that.row))
+        return false;
+    }
+
+    boolean this_present_column = true && this.isSetColumn();
+    boolean that_present_column = true && that.isSetColumn();
+    if (this_present_column || that_present_column) {
+      if (!(this_present_column && that_present_column))
+        return false;
+      if (!this.column.equals(that.column))
+        return false;
+    }
+
+    boolean this_present_ammount = true;
+    boolean that_present_ammount = true;
+    if (this_present_ammount || that_present_ammount) {
+      if (!(this_present_ammount && that_present_ammount))
+        return false;
+      if (this.ammount != that.ammount)
+        return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return 0;
+  }
+
+  public int compareTo(TIncrement other) {
+    if (!getClass().equals(other.getClass())) {
+      return getClass().getName().compareTo(other.getClass().getName());
+    }
+
+    int lastComparison = 0;
+    TIncrement typedOther = (TIncrement)other;
+
+    lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetTable()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetRow()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetColumn()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetAmmount()).compareTo(typedOther.isSetAmmount());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetAmmount()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ammount, typedOther.ammount);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    return 0;
+  }
+
+  public _Fields fieldForId(int fieldId) {
+    return _Fields.findByThriftId(fieldId);
+  }
+
+  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+  }
+
+  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder("TIncrement(");
+    boolean first = true;
+
+    sb.append("table:");
+    if (this.table == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.table);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("row:");
+    if (this.row == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.row);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("column:");
+    if (this.column == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.column);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("ammount:");
+    sb.append(this.ammount);
+    first = false;
+    sb.append(")");
+    return sb.toString();
+  }
+
+  public void validate() throws org.apache.thrift.TException {
+    // check for required fields
+  }
+
+  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+    try {
+      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+    try {
+      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+      __isset_bit_vector = new BitSet(1);
+      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private static class TIncrementStandardSchemeFactory implements SchemeFactory {
+    public TIncrementStandardScheme getScheme() {
+      return new TIncrementStandardScheme();
+    }
+  }
+
+  private static class TIncrementStandardScheme extends StandardScheme<TIncrement> {
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot, TIncrement struct) throws org.apache.thrift.TException {
+      org.apache.thrift.protocol.TField schemeField;
+      iprot.readStructBegin();
+      while (true)
+      {
+        schemeField = iprot.readFieldBegin();
+        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+          break;
+        }
+        switch (schemeField.id) {
+          case 1: // TABLE
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.table = iprot.readBinary();
+              struct.setTableIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 2: // ROW
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.row = iprot.readBinary();
+              struct.setRowIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 3: // COLUMN
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.column = iprot.readBinary();
+              struct.setColumnIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 4: // AMMOUNT
+            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+              struct.ammount = iprot.readI64();
+              struct.setAmmountIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          default:
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+        }
+        iprot.readFieldEnd();
+      }
+      iprot.readStructEnd();
+
+      // check for required fields of primitive type, which can't be checked in the validate method
+      struct.validate();
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot, TIncrement struct) throws org.apache.thrift.TException {
+      struct.validate();
+
+      oprot.writeStructBegin(STRUCT_DESC);
+      if (struct.table != null) {
+        oprot.writeFieldBegin(TABLE_FIELD_DESC);
+        oprot.writeBinary(struct.table);
+        oprot.writeFieldEnd();
+      }
+      if (struct.row != null) {
+        oprot.writeFieldBegin(ROW_FIELD_DESC);
+        oprot.writeBinary(struct.row);
+        oprot.writeFieldEnd();
+      }
+      if (struct.column != null) {
+        oprot.writeFieldBegin(COLUMN_FIELD_DESC);
+        oprot.writeBinary(struct.column);
+        oprot.writeFieldEnd();
+      }
+      oprot.writeFieldBegin(AMMOUNT_FIELD_DESC);
+      oprot.writeI64(struct.ammount);
+      oprot.writeFieldEnd();
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+
+  }
+
+  private static class TIncrementTupleSchemeFactory implements SchemeFactory {
+    public TIncrementTupleScheme getScheme() {
+      return new TIncrementTupleScheme();
+    }
+  }
+
+  private static class TIncrementTupleScheme extends TupleScheme<TIncrement> {
+
+    @Override
+    public void write(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) throws org.apache.thrift.TException {
+      TTupleProtocol oprot = (TTupleProtocol) prot;
+      BitSet optionals = new BitSet();
+      if (struct.isSetTable()) {
+        optionals.set(0);
+      }
+      if (struct.isSetRow()) {
+        optionals.set(1);
+      }
+      if (struct.isSetColumn()) {
+        optionals.set(2);
+      }
+      if (struct.isSetAmmount()) {
+        optionals.set(3);
+      }
+      oprot.writeBitSet(optionals, 4);
+      if (struct.isSetTable()) {
+        oprot.writeBinary(struct.table);
+      }
+      if (struct.isSetRow()) {
+        oprot.writeBinary(struct.row);
+      }
+      if (struct.isSetColumn()) {
+        oprot.writeBinary(struct.column);
+      }
+      if (struct.isSetAmmount()) {
+        oprot.writeI64(struct.ammount);
+      }
+    }
+
+    @Override
+    public void read(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) throws org.apache.thrift.TException {
+      TTupleProtocol iprot = (TTupleProtocol) prot;
+      BitSet incoming = iprot.readBitSet(4);
+      if (incoming.get(0)) {
+        struct.table = iprot.readBinary();
+        struct.setTableIsSet(true);
+      }
+      if (incoming.get(1)) {
+        struct.row = iprot.readBinary();
+        struct.setRowIsSet(true);
+      }
+      if (incoming.get(2)) {
+        struct.column = iprot.readBinary();
+        struct.setColumnIsSet(true);
+      }
+      if (incoming.get(3)) {
+        struct.ammount = iprot.readI64();
+        struct.setAmmountIsSet(true);
+      }
+    }
+  }
+
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
index e1709b55f412..44772899c4f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
@@ -443,7 +443,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowResult struct)
                 for (int _i9 = 0; _i9 < _map8.size; ++_i9)
                 {
                   ByteBuffer _key10; // required
-                  TCell _val11; // required
+                  TCell _val11; // optional
                   _key10 = iprot.readBinary();
                   _val11 = new TCell();
                   _val11.read(iprot);
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
index f698a6c8f67f..4b81b1e58531 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
@@ -110,6 +110,16 @@ struct BatchMutation {
   2:list<Mutation> mutations
 }
 
+/**
+ * For increments that are not incrementColumnValue
+ * equivalents.
+ */
+struct TIncrement {
+  1:Text table,
+  2:Text row,
+  3:Text column,
+  4:i64  ammount
+}
 
 /**
  * Holds row name and then a map of columns to cells. 
@@ -627,6 +637,23 @@ service Hbase {
     3:map<Text, Text> attributes
   ) throws (1:IOError io)
 
+  /**
+   * Increment a cell by the ammount.
+   * Increments can be applied async if hbase.regionserver.thrift.coalesceIncrement is set to true.
+   * False is the default.  Turn to true if you need the extra performance and can accept some
+   * data loss if a thrift server dies with increments still in the queue.
+   */
+  void increment(
+    /** The single increment to apply */
+    1:TIncrement increment
+  ) throws (1:IOError io)
+
+
+  void incrementRows(
+    /** The list of increments */
+    1:list<TIncrement> increments
+  ) throws (1:IOError io)
+
   /**
    * Completely delete the row's cells marked with a timestamp
    * equal-to or older than the passed timestamp.
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 2ece73ea2080..81409c4f5b0e 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -42,6 +42,8 @@
 import org.apache.hadoop.hbase.thrift.generated.TCell;
 import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+import org.apache.hadoop.hbase.thrift.generated.TIncrement;
+import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.MetricsContext;
@@ -66,11 +68,15 @@ public class TestThriftServer {
   private static ByteBuffer asByteBuffer(String i) {
     return ByteBuffer.wrap(Bytes.toBytes(i));
   }
+  private static ByteBuffer asByteBuffer(long l) {
+    return ByteBuffer.wrap(Bytes.toBytes(l));
+  }
 
   // Static names for tables, columns, rows, and values
   private static ByteBuffer tableAname = asByteBuffer("tableA");
   private static ByteBuffer tableBname = asByteBuffer("tableB");
   private static ByteBuffer columnAname = asByteBuffer("columnA:");
+  private static ByteBuffer columnAAname = asByteBuffer("columnA:A");
   private static ByteBuffer columnBname = asByteBuffer("columnB:");
   private static ByteBuffer rowAname = asByteBuffer("rowA");
   private static ByteBuffer rowBname = asByteBuffer("rowB");
@@ -78,9 +84,11 @@ private static ByteBuffer asByteBuffer(String i) {
   private static ByteBuffer valueBname = asByteBuffer("valueB");
   private static ByteBuffer valueCname = asByteBuffer("valueC");
   private static ByteBuffer valueDname = asByteBuffer("valueD");
+  private static ByteBuffer valueEname = asByteBuffer(100l);
 
   @BeforeClass
   public static void beforeClass() throws Exception {
+    UTIL.getConfiguration().setBoolean(ThriftServerRunner.COALESCE_INC_KEY, true);
     UTIL.startMiniCluster();
   }
 
@@ -195,6 +203,44 @@ public static void dropTestTables(Hbase.Iface handler) throws Exception {
     handler.deleteTable(tableAname);
   }
 
+  public void doTestIncrements() throws Exception {
+    ThriftServerRunner.HBaseHandler handler =
+        new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
+    createTestTables(handler);
+    doTestIncrements(handler);
+    dropTestTables(handler);
+  }
+
+  public static void doTestIncrements(HBaseHandler handler) throws Exception {
+    List<Mutation> mutations = new ArrayList<Mutation>(1);
+    mutations.add(new Mutation(false, columnAAname, valueEname, true));
+    mutations.add(new Mutation(false, columnAname, valueEname, true));
+    handler.mutateRow(tableAname, rowAname, mutations, null);
+    handler.mutateRow(tableAname, rowBname, mutations, null);
+
+    List<TIncrement> increments = new ArrayList<TIncrement>();
+    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
+    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
+    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
+
+    int numIncrements = 60000;
+    for (int i = 0; i < numIncrements; i++) {
+      handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2));
+      handler.incrementRows(increments);
+    }
+
+    Thread.sleep(1000);
+    long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong();
+    assertEquals((100 + (2 * numIncrements)), lv );
+
+
+    lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong();
+    assertEquals((100 + (3 * 7 * numIncrements)), lv);
+
+    assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0);
+
+  }
+
   /**
    * Tests adding a series of Mutations and BatchMutations, including a
    * delete mutation.  Also tests data retrieval, and getting back multiple

From d6f25c311f1805d0480faa432e0e209c871dd115 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 20:21:28 +0000
Subject: [PATCH 0250/1540] HBASE-6124 Backport HBASE-6033 to 0.90, 0.92 and
 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344445 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       |  88 ++++++++-
 .../hadoop/hbase/ipc/HRegionInterface.java    |  10 ++
 .../hbase/regionserver/HRegionServer.java     |  16 ++
 .../hadoop/hbase/regionserver/Store.java      |   4 +
 .../compactions/CompactionRequest.java        |  73 ++++++++
 .../resources/hbase-webapps/master/table.jsp  |   5 +
 .../regionserver/TestCompactionState.java     | 170 ++++++++++++++++++
 7 files changed, 365 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index a77dc223568d..67e5631e2b58 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -22,7 +22,6 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.lang.reflect.UndeclaredThrowableException;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.LinkedList;
@@ -56,6 +55,7 @@
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -1667,4 +1667,90 @@ public String[] getMasterCoprocessors() {
       return null;
     }
   }
+
+  /**
+   * Get the current compaction state of a table or region.
+   * It could be in a major compaction, a minor compaction, both, or none.
+   *
+   * @param tableNameOrRegionName table or region to major compact
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   * @return the current compaction state
+   */
+  public CompactionState getCompactionState(final String tableNameOrRegionName)
+      throws IOException, InterruptedException {
+    return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
+  }
+
+  /**
+   * Get the current compaction state of a table or region.
+   * It could be in a major compaction, a minor compaction, both, or none.
+   *
+   * @param tableNameOrRegionName table or region to major compact
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   * @return the current compaction state
+   */
+  public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
+      throws IOException, InterruptedException {
+    CompactionState state = CompactionState.NONE;
+    CatalogTracker ct = getCatalogTracker();
+    try {
+      if (isRegionName(tableNameOrRegionName, ct)) {
+        Pair<HRegionInfo, ServerName> pair =
+          MetaReader.getRegion(ct, tableNameOrRegionName);
+        if (pair == null || pair.getSecond() == null) {
+          LOG.info("No server in .META. for " +
+            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+        } else {
+          ServerName sn = pair.getSecond();
+          HRegionInterface rs =
+            this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
+          return CompactionState.valueOf(
+            rs.getCompactionState(pair.getFirst().getRegionName()));
+        }
+      } else {
+        final String tableName = tableNameString(tableNameOrRegionName, ct);
+        List<Pair<HRegionInfo, ServerName>> pairs =
+          MetaReader.getTableRegionsAndLocations(ct, tableName);
+        for (Pair<HRegionInfo, ServerName> pair: pairs) {
+          if (pair.getFirst().isOffline()) continue;
+          if (pair.getSecond() == null) continue;
+          try {
+            ServerName sn = pair.getSecond();
+            HRegionInterface rs =
+              this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
+            switch (CompactionState.valueOf(
+              rs.getCompactionState(pair.getFirst().getRegionName()))) {
+            case MAJOR_AND_MINOR:
+              return CompactionState.MAJOR_AND_MINOR;
+            case MAJOR:
+              if (state == CompactionState.MINOR) {
+                return CompactionState.MAJOR_AND_MINOR;
+              }
+              state = CompactionState.MAJOR;
+              break;
+            case MINOR:
+              if (state == CompactionState.MAJOR) {
+                return CompactionState.MAJOR_AND_MINOR;
+              }
+              state = CompactionState.MINOR;
+              break;
+            case NONE:
+              default: // nothing, continue
+            }
+          } catch (NotServingRegionException e) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Trying to get compaction state of " +
+                pair.getFirst() + ": " +
+                StringUtils.stringifyException(e));
+            }
+          }
+        }
+      }
+    } finally {
+      cleanupCatalogTracker(ct);
+    }
+    return state;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index a3e766957fc9..d5e329ecf6c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.security.TokenInfo;
@@ -621,6 +622,15 @@ public boolean checkAndDelete(final byte[] regionName, final byte[] row,
    */
   public byte[][] rollHLogWriter() throws IOException, FailedLogCloseException;
 
+  /**
+   * Get the current compaction state of the region.
+   *
+   * @param regionName the name of the region to check compaction statte.
+   * @return the compaction state name.
+   * @throws IOException exception
+   */
+  public String getCompactionState(final byte[] regionName) throws IOException;
+
   @Override
   public void stop(String why);
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ff3d635a62ff..3f28b7ac12f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -122,6 +122,7 @@
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
 import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
 import org.apache.hadoop.hbase.regionserver.handler.CloseRootHandler;
@@ -3698,4 +3699,19 @@ void registerMBean() {
         mxBeanInfo);
     LOG.info("Registered RegionServer MXBean");
   }
+
+  /**
+   * Get the current compaction state of the region.
+   *
+   * @param regionName the name of the region to check compaction statte.
+   * @return the compaction state name.
+   * @throws IOException exception
+   */
+  public String getCompactionState(final byte[] regionName) throws IOException {
+      checkOpen();
+      requestCount.incrementAndGet();
+      HRegion region = getRegion(regionName);
+      HRegionInfo info = region.getRegionInfo();
+      return CompactionRequest.getCompactionState(info.getRegionId()).name();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 22b3c6970c00..4bfd0f83893b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1281,10 +1281,14 @@ public CompactionRequest requestCompaction(int priority) {
     } finally {
       this.lock.readLock().unlock();
     }
+    if (ret != null) {
+      CompactionRequest.preRequest(ret);
+    }
     return ret;
   }
 
   public void finishRequest(CompactionRequest cr) {
+    CompactionRequest.postRequest(cr);
     cr.finishRequest();
     synchronized (filesCompacting) {
       filesCompacting.removeAll(cr.getFiles());
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
index 7544b7144fb9..0a931bb6b051 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
@@ -21,12 +21,15 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.RejectedExecutionHandler;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -55,6 +58,14 @@ public class CompactionRequest implements Comparable<CompactionRequest>,
     private final Long timeInNanos;
     private HRegionServer server = null;
 
+    /**
+     * Map to track the number of compaction requested per region (id)
+     */
+    private static final ConcurrentHashMap<Long, AtomicInteger>
+      majorCompactions = new ConcurrentHashMap<Long, AtomicInteger>();
+    private static final ConcurrentHashMap<Long, AtomicInteger>
+      minorCompactions = new ConcurrentHashMap<Long, AtomicInteger>();
+
     public CompactionRequest(HRegion r, Store s,
         CompactSelection files, boolean isMajor, int p) {
       Preconditions.checkNotNull(r);
@@ -73,6 +84,58 @@ public CompactionRequest(HRegion r, Store s,
       this.timeInNanos = System.nanoTime();
     }
 
+    /**
+     * Find out if a given region in compaction now.
+     *
+     * @param regionId
+     * @return
+     */
+    public static CompactionState getCompactionState(
+        final long regionId) {
+      Long key = Long.valueOf(regionId);
+      AtomicInteger major = majorCompactions.get(key);
+      AtomicInteger minor = minorCompactions.get(key);
+      int state = 0;
+      if (minor != null && minor.get() > 0) {
+        state += 1;  // use 1 to indicate minor here
+      }
+      if (major != null && major.get() > 0) {
+        state += 2;  // use 2 to indicate major here
+      }
+      switch (state) {
+      case 3:  // 3 = 2 + 1, so both major and minor
+        return CompactionState.MAJOR_AND_MINOR;
+      case 2:
+        return CompactionState.MAJOR;
+      case 1:
+        return CompactionState.MINOR;
+      default:
+        return CompactionState.NONE;
+      }
+    }
+
+    public static void preRequest(final CompactionRequest cr){
+      Long key = Long.valueOf(cr.getHRegion().getRegionId());
+      ConcurrentHashMap<Long, AtomicInteger> compactions =
+        cr.isMajor() ? majorCompactions : minorCompactions;
+      AtomicInteger count = compactions.get(key);
+      if (count == null) {
+        compactions.putIfAbsent(key, new AtomicInteger(0));
+        count = compactions.get(key);
+      }
+      count.incrementAndGet();
+    }
+
+    public static void postRequest(final CompactionRequest cr){
+      Long key = Long.valueOf(cr.getHRegion().getRegionId());
+      ConcurrentHashMap<Long, AtomicInteger> compactions =
+        cr.isMajor() ? majorCompactions : minorCompactions;
+      AtomicInteger count = compactions.get(key);
+      if (count != null) {
+        count.decrementAndGet();
+      }
+    }
+
     public void finishRequest() {
       this.compactSelection.finishRequest();
     }
@@ -212,6 +275,16 @@ public void run() {
       }
     }
 
+    /**
+     * An enum for the region compaction state
+     */
+    public static enum CompactionState {
+      NONE,
+      MINOR,
+      MAJOR,
+      MAJOR_AND_MINOR;
+    }
+
     /**
      * Cleanup class to use when rejecting a compaction request from the queue.
      */
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 811df46ddc51..b7b0f333dd3a 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -154,6 +154,11 @@
       <td><%= hbadmin.isTableEnabled(table.getTableName()) %></td>
       <td>Is the table enabled</td>
   </tr>
+  <tr>
+      <td>Compaction</td>
+      <td><%= hbadmin.getCompactionState(table.getTableName()) %></td>
+      <td>Is the table compacting</td>
+  </tr>
 <%  if (showFragmentation) { %>
   <tr>
       <td>Fragmentation</td>
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
new file mode 100644
index 000000000000..8fc2f593df97
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/** Unit tests to test retrieving table/region compaction state*/
+@Category(LargeTests.class)
+public class TestCompactionState {
+  final static Log LOG = LogFactory.getLog(TestCompactionState.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static Random random = new Random();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test(timeout=60000)
+  public void testMajorCompaction() throws IOException, InterruptedException {
+    compaction("testMajorCompaction", 8, CompactionState.MAJOR);
+  }
+
+  @Test(timeout=60000)
+  public void testMinorCompaction() throws IOException, InterruptedException {
+    compaction("testMinorCompaction", 15, CompactionState.MINOR);
+  }
+
+  /**
+   * Load data to a table, flush it to disk, trigger compaction,
+   * confirm the compaction state is right and wait till it is done.
+   *
+   * @param tableName
+   * @param flushes
+   * @param expectedState
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  private void compaction(final String tableName, final int flushes,
+      final CompactionState expectedState) throws IOException, InterruptedException {
+    // Create a table with regions
+    byte [] table = Bytes.toBytes(tableName);
+    byte [] family = Bytes.toBytes("family");
+    HTable ht = null;
+    try {
+      ht = TEST_UTIL.createTable(table, family);
+      loadData(ht, family, 3000, flushes);
+      HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+      List<HRegion> regions = rs.getOnlineRegions(table);
+      int countBefore = countStoreFiles(regions, family);
+      assertTrue(countBefore > 0); // there should be some data files
+      HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+      if (expectedState == CompactionState.MINOR) {
+        admin.compact(tableName);
+      } else {
+        admin.majorCompact(table);
+      }
+      long curt = System.currentTimeMillis();
+      long waitTime = 5000;
+      long endt = curt + waitTime;
+      CompactionState state = admin.getCompactionState(table);
+      while (state == CompactionState.NONE && curt < endt) {
+        Thread.sleep(10);
+        state = admin.getCompactionState(table);
+        curt = System.currentTimeMillis();
+      }
+      // Now, should have the right compaction state,
+      // otherwise, the compaction should have already been done
+      if (expectedState != state) {
+        for (HRegion region: regions) {
+          state = CompactionRequest.getCompactionState(region.getRegionId());
+          assertEquals(CompactionState.NONE, state);
+        }
+      } else {
+        curt = System.currentTimeMillis();
+        waitTime = 20000;
+        endt = curt + waitTime;
+        state = admin.getCompactionState(table);
+        while (state != CompactionState.NONE && curt < endt) {
+          Thread.sleep(10);
+          state = admin.getCompactionState(table);
+          curt = System.currentTimeMillis();
+        }
+        // Now, compaction should be done.
+        assertEquals(CompactionState.NONE, state);
+      }
+      int countAfter = countStoreFiles(regions, family);
+      assertTrue(countAfter < countBefore);
+      if (expectedState == CompactionState.MAJOR) assertTrue(1 == countAfter);
+      else assertTrue(1 < countAfter);
+    } finally {
+      if (ht != null) {
+        TEST_UTIL.deleteTable(table);
+      }
+    }
+  }
+
+  private static int countStoreFiles(
+      List<HRegion> regions, final byte[] family) {
+    int count = 0;
+    for (HRegion region: regions) {
+      count += region.getStoreFileList(new byte[][]{family}).size();
+    }
+    return count;
+  }
+
+  private static void loadData(final HTable ht, final byte[] family,
+      final int rows, final int flushes) throws IOException {
+    List<Put> puts = new ArrayList<Put>(rows);
+    byte[] qualifier = Bytes.toBytes("val");
+    for (int i = 0; i < flushes; i++) {
+      for (int k = 0; k < rows; k++) {
+        byte[] row = Bytes.toBytes(random.nextLong());
+        Put p = new Put(row);
+        p.add(family, qualifier, row);
+        puts.add(p);
+      }
+      ht.put(puts);
+      ht.flushCommits();
+      TEST_UTIL.flush();
+      puts.clear();
+    }
+  }
+  
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From c961fe2e6808a5a554ca345b68044a780215da68 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 21:17:51 +0000
Subject: [PATCH 0251/1540] HBASE-6122 Backup master does not become Active
 master after ZK exception: REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344467 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/ActiveMasterManager.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index bf6acfcccfea..178edb338ba1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -210,8 +210,7 @@ boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
         return cleanSetOfActiveMaster;
       }
       // Try to become active master again now that there is no active master
-      cleanSetOfActiveMaster = blockUntilBecomingActiveMaster(startupStatus,
-          clusterStatusTracker);
+      blockUntilBecomingActiveMaster(startupStatus,clusterStatusTracker);
     }
     return cleanSetOfActiveMaster;
   }

From 02e2665462e77f4697420b5f713d8247df62047f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 21:20:41 +0000
Subject: [PATCH 0252/1540] HBASE-6068 Secure HBase cluster : Client not able
 to call some admin APIs

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344472 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/zookeeper/ZKUtil.java     | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index d4c79422ae9e..56d8cb404813 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -717,10 +717,15 @@ public static boolean isSecureZooKeeper(Configuration conf) {
 
   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
     if (isSecureZooKeeper(zkw.getConfiguration())) {
-      // Certain znodes must be readable by non-authenticated clients
-      if ((node.equals(zkw.rootServerZNode) == true) ||
+      // Certain znodes are accessed directly by the client,
+      // so they must be readable by non-authenticated clients
+      if ((node.equals(zkw.baseZNode) == true) ||
+          (node.equals(zkw.rootServerZNode) == true) ||
           (node.equals(zkw.masterAddressZNode) == true) ||
-          (node.equals(zkw.clusterIdZNode) == true)) {
+          (node.equals(zkw.clusterIdZNode) == true) ||
+          (node.equals(zkw.rsZNode) == true) ||
+          (node.equals(zkw.backupMasterAddressesZNode) == true) ||
+          (node.startsWith(zkw.tableZNode) == true)) {
         return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE;
       }
       return Ids.CREATOR_ALL_ACL;

From d71d4f54bbdf643c633d44792c1e82cb7bdb21de Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 30 May 2012 22:28:58 +0000
Subject: [PATCH 0253/1540] HBASE-6062 preCheckAndPut/Delete() checks for READ
 when also a WRITE is performed

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344487 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 10 +++--
 .../security/access/TestAccessController.java | 45 +++++++++++++++++++
 2 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index b4a90e6d87c7..d815be7975c6 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -793,8 +793,9 @@ public boolean preCheckAndPut(final ObserverContext<RegionCoprocessorEnvironment
       final CompareFilter.CompareOp compareOp,
       final WritableByteArrayComparable comparator, final Put put,
       final boolean result) throws IOException {
-    requirePermission(TablePermission.Action.READ, c.getEnvironment(),
-        Arrays.asList(new byte[][]{family}));
+    Collection<byte[]> familyMap = Arrays.asList(new byte[][]{family});
+    requirePermission(TablePermission.Action.READ, c.getEnvironment(), familyMap);
+    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(), familyMap);
     return result;
   }
 
@@ -804,8 +805,9 @@ public boolean preCheckAndDelete(final ObserverContext<RegionCoprocessorEnvironm
       final CompareFilter.CompareOp compareOp,
       final WritableByteArrayComparable comparator, final Delete delete,
       final boolean result) throws IOException {
-    requirePermission(TablePermission.Action.READ, c.getEnvironment(),
-        Arrays.asList( new byte[][] {family}));
+    Collection<byte[]> familyMap = Arrays.asList(new byte[][]{family});
+    requirePermission(TablePermission.Action.READ, c.getEnvironment(), familyMap);
+    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(), familyMap);
     return result;
   }
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 5d6351197997..604fa7dc4631 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -539,6 +539,18 @@ private void verifyRead(PrivilegedExceptionAction action) throws Exception {
     verifyAllowed(USER_RO, action);
   }
 
+  private void verifyReadWrite(PrivilegedExceptionAction action) throws Exception {
+    // should be denied
+    verifyDenied(USER_NONE, action);
+    verifyDenied(USER_RO, action);
+
+    // should be allowed
+    verifyAllowed(SUPERUSER, action);
+    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(USER_OWNER, action);
+    verifyAllowed(USER_RW, action);
+  }
+
   @Test
   public void testRead() throws Exception {
     // get action
@@ -615,6 +627,39 @@ public Object run() throws Exception {
     verifyWrite(incrementAction);
   }
 
+  @Test
+  public void testReadWrite() throws Exception {
+    // action for checkAndDelete
+    PrivilegedExceptionAction checkAndDeleteAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        Delete d = new Delete(Bytes.toBytes("random_row"));
+        d.deleteFamily(TEST_FAMILY);
+
+        HTable t = new HTable(conf, TEST_TABLE);
+        t.checkAndDelete(Bytes.toBytes("random_row"), 
+                         TEST_FAMILY, Bytes.toBytes("q"),
+                         Bytes.toBytes("test_value"), d);
+        return null;
+      }
+    };
+    verifyReadWrite(checkAndDeleteAction);
+
+    // action for checkAndPut()
+    PrivilegedExceptionAction checkAndPut = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        Put p = new Put(Bytes.toBytes("random_row"));
+        p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1));
+
+        HTable t = new HTable(conf, TEST_TABLE);
+        t.checkAndPut(Bytes.toBytes("random_row"), 
+                      TEST_FAMILY, Bytes.toBytes("q"),
+                      Bytes.toBytes("test_value"), p);
+        return null;
+      }
+    };
+    verifyReadWrite(checkAndPut);
+  }
+
   @Test
   public void testGrantRevoke() throws Exception {
     final byte[] tableName = Bytes.toBytes("TempTable");

From a978129d3e8c9944f67c22842f819af4c51f7e9b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 31 May 2012 00:00:04 +0000
Subject: [PATCH 0254/1540] HBASE-6016 ServerShutdownHandler#processDeadRegion
 could return false for disabling table regions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344512 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/handler/ServerShutdownHandler.java   | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 0aa62dbcbdeb..8fad16359eb8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -336,8 +336,8 @@ public void process() throws IOException {
   }
 
   /**
-   * Process a dead region from a dead RS.  Checks if the region is disabled
-   * or if the region has a partially completed split.
+   * Process a dead region from a dead RS. Checks if the region is disabled or
+   * disabling or if the region has a partially completed split.
    * @param hri
    * @param result
    * @param assignmentManager
@@ -372,6 +372,13 @@ public static boolean processDeadRegion(HRegionInfo hri, Result result,
       fixupDaughters(result, assignmentManager, catalogTracker);
       return false;
     }
+    boolean disabling = assignmentManager.getZKTable().isDisablingTable(
+        hri.getTableNameAsString());
+    if (disabling) {
+      LOG.info("The table " + hri.getTableNameAsString()
+          + " is disabled.  Hence not assigning region" + hri.getEncodedName());
+      return false;
+    }
     return true;
   }
 

From 98dcf44edc9f8aa655c48cba7dbc83d658002933 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 31 May 2012 04:27:11 +0000
Subject: [PATCH 0255/1540] HBASE-6040 Use block encoding and HBase handled
 checksum verification in bulk loading using HFileOutputFormat

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344561 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/HFileOutputFormat.java    | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
index a7419d09d272..d2962ee6b9e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
@@ -46,9 +46,14 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -75,6 +80,8 @@ public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable,
   static Log LOG = LogFactory.getLog(HFileOutputFormat.class);
   static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression";
   TimeRangeTracker trt = new TimeRangeTracker();
+  private static final String DATABLOCK_ENCODING_CONF_KEY = 
+     "hbase.mapreduce.hfileoutputformat.datablock.encoding";
 
   public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
   throws IOException, InterruptedException {
@@ -96,6 +103,22 @@ public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final Task
 
     // create a map from column family to the compression algorithm
     final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf);
+    
+    String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
+    final HFileDataBlockEncoder encoder;
+    if (dataBlockEncodingStr == null) {
+      encoder = NoOpDataBlockEncoder.INSTANCE;
+    } else {
+      try {
+        encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding
+            .valueOf(dataBlockEncodingStr));
+      } catch (IllegalArgumentException ex) {
+        throw new RuntimeException(
+            "Invalid data block encoding type configured for the param "
+                + DATABLOCK_ENCODING_CONF_KEY + " : "
+                + dataBlockEncodingStr);
+      }
+    }
 
     return new RecordWriter<ImmutableBytesWritable, KeyValue>() {
       // Map of families to writers and how much has been output on the writer.
@@ -178,6 +201,9 @@ private WriterLength getNewWriter(byte[] family, Configuration conf)
             .withBlockSize(blocksize)
             .withCompression(compression)
             .withComparator(KeyValue.KEY_COMPARATOR)
+            .withDataBlockEncoder(encoder)
+            .withChecksumType(Store.getChecksumType(conf))
+            .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
             .create();
         this.writers.put(family, wl);
         return wl;

From 6cf037046edeb3b635a475adff4027ca9bd7e1ca Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 31 May 2012 16:19:48 +0000
Subject: [PATCH 0256/1540] HBASE-6122 Backup master does not become Active
 master after ZK exception (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344798 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/ActiveMasterManager.java | 2 +-
 .../hadoop/hbase/master/TestMasterZKSessionRecovery.java    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 178edb338ba1..484b7836a139 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -210,7 +210,7 @@ boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
         return cleanSetOfActiveMaster;
       }
       // Try to become active master again now that there is no active master
-      blockUntilBecomingActiveMaster(startupStatus,clusterStatusTracker);
+      cleanSetOfActiveMaster = blockUntilBecomingActiveMaster(startupStatus,clusterStatusTracker);
     }
     return cleanSetOfActiveMaster;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
index 1b9b24ec560b..ba8b725cf2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
@@ -63,8 +63,8 @@ public void tearDown() throws Exception {
    * Negative test of master recovery from zk session expiry.
    * <p>
    * Starts with one master. Fakes the master zk session expired.
-   * Ensures the master cannot recover the expired zk session since
-   * the master zk node is still there.
+   * The master should be able to come up if he is able to create
+   * the node as active master.
    * @throws Exception
    */
   @Test(timeout=10000)
@@ -73,7 +73,7 @@ public void testMasterZKSessionRecoveryFailure() throws Exception {
     HMaster m = cluster.getMaster();
     m.abort("Test recovery from zk session expired",
       new KeeperException.SessionExpiredException());
-    assertTrue(m.isStopped());
+    assertFalse(m.isStopped());
   }
 
   /**

From cdfe69fff133f7251e46bf87b9b93476b096df07 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 31 May 2012 16:38:23 +0000
Subject: [PATCH 0257/1540] HBASE-6089 SSH and AM.joinCluster causes Concurrent
 Modification exception. (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344805 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 35 +++++++++----------
 .../apache/hadoop/hbase/master/HMaster.java   |  1 -
 .../hadoop/hbase/master/MasterFileSystem.java |  2 --
 3 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 88858a0dcbc5..5324a291360f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -328,7 +328,6 @@ void cleanoutUnassigned() throws IOException, KeeperException {
   /**
    * Called on startup.
    * Figures whether a fresh cluster start of we are joining extant running cluster.
-   * @param onlineServers onlined servers when master started
    * @throws IOException
    * @throws KeeperException
    * @throws InterruptedException
@@ -391,18 +390,19 @@ void processDeadServersAndRegionsInTransition(
     }
     // Run through all regions.  If they are not assigned and not in RIT, then
     // its a clean cluster startup, else its a failover.
-    for (Map.Entry<HRegionInfo, ServerName> e: this.regions.entrySet()) {
-      if (!e.getKey().isMetaTable()
-          && e.getValue() != null) {
-        LOG.debug("Found " + e + " out on cluster");
-        this.failover = true;
-        break;
-      }
-      if (nodes.contains(e.getKey().getEncodedName())) {
-        LOG.debug("Found " + e.getKey().getRegionNameAsString() + " in RITs");
-        // Could be a meta region.
-        this.failover = true;
-        break;
+    synchronized (this.regions) {
+      for (Map.Entry<HRegionInfo, ServerName> e : this.regions.entrySet()) {
+        if (!e.getKey().isMetaTable() && e.getValue() != null) {
+          LOG.debug("Found " + e + " out on cluster");
+          this.failover = true;
+          break;
+        }
+        if (nodes.contains(e.getKey().getEncodedName())) {
+          LOG.debug("Found " + e.getKey().getRegionNameAsString() + " in RITs");
+          // Could be a meta region.
+          this.failover = true;
+          break;
+        }
       }
     }
 
@@ -2446,8 +2446,6 @@ boolean waitUntilNoRegionsInTransition(final long timeout, Set<HRegionInfo> regi
    * <p>
    * Returns a map of servers that are not found to be online and the regions
    * they were hosting.
-   * @param onlineServers if one region's location belongs to onlineServers, it
-   *          doesn't need to be assigned.
    * @return map of servers not online to their assigned regions, as stored
    *         in META
    * @throws IOException
@@ -2523,8 +2521,10 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOE
         // add only if region not in disabled and enabling table
         if (false == checkIfRegionBelongsToDisabled(regionInfo)
             && false == checkIfRegionsBelongsToEnabling(regionInfo)) {
-          regions.put(regionInfo, regionLocation);
-          addToServers(regionLocation, regionInfo);
+          synchronized (this.regions) {
+            regions.put(regionInfo, regionLocation);
+            addToServers(regionLocation, regionInfo);
+          }
         }
         disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
             this.enablingTables, regionInfo, tableName);
@@ -3203,7 +3203,6 @@ void balance(final RegionPlan plan) {
    * Run through remaining regionservers and unassign all catalog regions.
    */
   void unassignCatalogRegions() {
-    this.servers.entrySet();
     synchronized (this.regions) {
       for (Map.Entry<ServerName, Set<HRegionInfo>> e: this.servers.entrySet()) {
         Set<HRegionInfo> regions = e.getValue();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index ea0db2284bf0..7ef315a3df2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -581,7 +581,6 @@ protected void startCatalogJanitorChore() {
   /**
    * Override to change master's splitLogAfterStartup. Used testing
    * @param mfs
-   * @param onlineServers
    */
   protected void splitLogAfterStartup(final MasterFileSystem mfs) {
     mfs.splitLogAfterStartup();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 05f06cbde7d2..a57a7b6cd2db 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -183,8 +183,6 @@ public String getClusterId() {
   /**
    * Inspect the log directory to recover any log file without
    * an active region server.
-   * @param onlineServers Set of online servers keyed by
-   * {@link ServerName}
    */
   void splitLogAfterStartup() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",

From aa1ff7c60fe8f04e27736749a3289cb4a254aca7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 31 May 2012 22:10:44 +0000
Subject: [PATCH 0258/1540] HBASE-6141 InterfaceAudience breaks 0.94 on older
 versions of hadoop

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1344927 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/DataOutputOutputStream.java   | 5 -----
 .../org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java   | 2 --
 .../java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java    | 4 ----
 .../hadoop/hbase/regionserver/metrics/OperationMetrics.java  | 2 --
 .../hbase/regionserver/metrics/RegionMetricsStorage.java     | 2 --
 .../hadoop/hbase/regionserver/wal/CompressionContext.java    | 2 --
 .../org/apache/hadoop/hbase/regionserver/wal/Compressor.java | 2 --
 .../org/apache/hadoop/hbase/regionserver/wal/Dictionary.java | 2 --
 .../apache/hadoop/hbase/regionserver/wal/LRUDictionary.java  | 2 --
 .../hbase/regionserver/wal/HLogPerformanceEvaluation.java    | 4 ----
 10 files changed, 27 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/DataOutputOutputStream.java b/src/main/java/org/apache/hadoop/hbase/io/DataOutputOutputStream.java
index 10700c1d1e75..ec61c3a5b154 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/DataOutputOutputStream.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/DataOutputOutputStream.java
@@ -21,14 +21,9 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
 /**
  * OutputStream implementation that wraps a DataOutput.
  */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
 class DataOutputOutputStream extends OutputStream {
 
   private final DataOutput out;
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
index 747063c6f584..bdac1f4ce0bd 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
@@ -27,7 +27,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -45,7 +44,6 @@
 /**
  * Simple {@link InputFormat} for {@link HLog} files.
  */
-@InterfaceAudience.Public
 public class HLogInputFormat extends InputFormat<HLogKey, WALEdit> {
   private static Log LOG = LogFactory.getLog(HLogInputFormat.class);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 9b1f239b423c..b0a7b692ef3e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -23,8 +23,6 @@
 import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
@@ -57,8 +55,6 @@
  * WAL replay can also generate HFiles for later bulk importing,
  * in that case the WAL is replayed for a single table only.
  */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
 public class WALPlayer extends Configured implements Tool {
   final static String NAME = "WALPlayer";
   final static String BULK_OUTPUT_CONF_KEY = "hlog.bulk.output";
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
index 305dfaba40f9..fa91293f067e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
@@ -20,7 +20,6 @@
 
 import java.util.Set;
 
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Append;
@@ -37,7 +36,6 @@
  * metrics are stored in {@link RegionMetricsStorage} and exposed to hadoop
  * metrics through {@link RegionServerDynamicMetrics}.
  */
-@InterfaceAudience.Private
 public class OperationMetrics {
 
   private static final String DELETE_KEY = "delete_";
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
index 5d4beffc2e2c..32395407ca58 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
@@ -25,7 +25,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -34,7 +33,6 @@
  * numbers. These numbers are exposed to Hadoop metrics through
  * RegionServerDynamicMetrics.
  */
-@InterfaceAudience.Private
 public class RegionMetricsStorage {
 
   // for simple numeric metrics (# of blocks read from block cache)
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
index 5a877af09fa2..10aec004b4f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
@@ -20,12 +20,10 @@
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import org.apache.hadoop.classification.InterfaceAudience;
 
 /**
  * Context that holds the various dictionaries for compression in HLog.
  */
-@InterfaceAudience.Private
 class CompressionContext {
   final Dictionary regionDict;
   final Dictionary tableDict;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
index 8a0dbaf62914..8193e1f91161 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
@@ -17,7 +17,6 @@
  */
 
 package org.apache.hadoop.hbase.regionserver.wal;
-import org.apache.hadoop.classification.InterfaceAudience;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -37,7 +36,6 @@
  * A set of static functions for running our custom WAL compression/decompression.
  * Also contains a command line tool to compress and uncompress HLogs.
  */
-@InterfaceAudience.Private
 public class Compressor {
   /**
    * Command line tool to compress and uncompress WALs.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java
index e1cfed195bb8..5dbf3bfc9ec2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hbase.regionserver.wal;
 
-import org.apache.hadoop.classification.InterfaceAudience;
 
 /**
  * Dictionary interface
@@ -26,7 +25,6 @@
  * Dictionary indexes should be either bytes or shorts, only positive. (The
  * first bit is reserved for detecting whether something is compressed or not).
  */
-@InterfaceAudience.Private
 interface Dictionary {
   static final byte NOT_IN_DICTIONARY = -1;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java
index 6e0b20bd65da..5ee18fc7409d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java
@@ -20,7 +20,6 @@
 
 import java.util.HashMap;
 
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import com.google.common.base.Preconditions;
@@ -33,7 +32,6 @@
  * <code>(2 ^ 15) * 5 (Regionname, Row key, CF, Column qual, table) * 100 bytes (these are some big names) = ~16MB</code>.
  * If you want to get silly, even at 1kb entries, it maxes out at 160 megabytes.
  */
-@InterfaceAudience.Private
 public class LRUDictionary implements Dictionary {
   private final BidirectionalLRUMap backingStore = new BidirectionalLRUMap();
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
index 0258cdaef267..e534cc28a752 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.classification.InterfaceAudience;
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -44,16 +43,13 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 
 /**
  * This class runs performance benchmarks for {@link HLog}.
  * See usage for this tool by running:
  * <code>$ hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation -h</code>
  */
-@InterfaceAudience.Private
 public final class HLogPerformanceEvaluation extends Configured implements Tool {
   static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
 

From 26dc2e1d0b034e0ad1ef42b0c3b03bce47c10c07 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 3 Jun 2012 20:02:54 +0000
Subject: [PATCH 0259/1540] HBASE-6067 HBase won't start when hbase.rootdir
 uses ViewFileSystem

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1345753 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLog.java   | 34 +++++++++++++++++--
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 0abf313ca97e..98cc744ed10c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -383,7 +383,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
   public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
       final Configuration conf, final List<WALActionsListener> listeners,
       final boolean failIfLogDirExists, final String prefix)
- throws IOException {
+  throws IOException {
     super();
     this.fs = fs;
     this.dir = dir;
@@ -394,7 +394,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
       }
     }
     this.blocksize = conf.getLong("hbase.regionserver.hlog.blocksize",
-      this.fs.getDefaultBlockSize());
+        getDefaultBlockSize());
     // Roll at 95% of block size.
     float multi = conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f);
     this.logrollsize = (long)(this.blocksize * multi);
@@ -441,6 +441,34 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
         Thread.currentThread().getName() + ".logSyncer");
     coprocessorHost = new WALCoprocessorHost(this, conf);
   }
+  
+  // use reflection to search for getDefaultBlockSize(Path f)
+  // if the method doesn't exist, fall back to using getDefaultBlockSize()
+  private long getDefaultBlockSize() throws IOException {
+    Method m = null;
+    Class<? extends FileSystem> cls = this.fs.getClass();
+    try {
+      m = cls.getDeclaredMethod("getDefaultBlockSize",
+          new Class<?>[] { Path.class });
+      m.setAccessible(true);
+    } catch (NoSuchMethodException e) {
+      LOG.info("FileSystem doesn't support getDefaultBlockSize");
+    } catch (SecurityException e) {
+      LOG.info("Doesn't have access to getDefaultBlockSize on "
+          + "FileSystems", e);
+      m = null; // could happen on setAccessible()
+    }
+    if (null == m) {
+      return this.fs.getDefaultBlockSize();
+    } else {
+      try {
+        Object ret = m.invoke(this.fs, this.dir);
+        return ((Long)ret).longValue();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  }
 
   /**
    * Find the 'getNumCurrentReplicas' on the passed <code>os</code> stream.
@@ -1886,4 +1914,4 @@ public static void main(String[] args) throws IOException {
       System.exit(-1);
     }
   }
-}
\ No newline at end of file
+}

From 4a892cced69ceb10c81d5fbf27b654098b4c9c66 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 4 Jun 2012 09:40:55 +0000
Subject: [PATCH 0260/1540] HBASE-5892 [hbck] Refactor parallel WorkItem* to
 Futures (Andrew Wang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1345889 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 186 ++++++++----------
 1 file changed, 85 insertions(+), 101 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 96dd17b2868e..81547b871217 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -32,10 +32,11 @@
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
@@ -152,7 +153,7 @@ public class HBaseFsck {
   private HConnection connection;
   private HBaseAdmin admin;
   private HTable meta;
-  private ThreadPoolExecutor executor; // threads to retrieve data from regionservers
+  private ScheduledThreadPoolExecutor executor; // threads to retrieve data from regionservers
   private long startMillis = System.currentTimeMillis();
 
   /***********
@@ -218,10 +219,7 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
     this.conf = conf;
 
     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
-    executor = new ThreadPoolExecutor(numThreads, numThreads,
-        THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
-        new LinkedBlockingQueue<Runnable>());
-    executor.allowCoreThreadTimeOut(true);
+    executor = new ScheduledThreadPoolExecutor(numThreads);
   }
 
   /**
@@ -629,20 +627,25 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
 
     // Parallelized read of .regioninfo files.
-    WorkItemHdfsRegionInfo[] hbis = new WorkItemHdfsRegionInfo[hbckInfos.size()];
-    int num = 0;
+    List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
+    List<Future<Void>> hbiFutures;
+
     for (HbckInfo hbi : hbckInfos) {
-      hbis[num] = new WorkItemHdfsRegionInfo(hbi, this, errors);
-      executor.execute(hbis[num]);
-      num++;
+      WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
+      hbis.add(work);
     }
 
-    for (int i=0; i < num; i++) {
-      WorkItemHdfsRegionInfo hbi = hbis[i];
-      synchronized(hbi) {
-        while (!hbi.isDone()) {
-          hbi.wait();
-        }
+    // Submit and wait for completion
+    hbiFutures = executor.invokeAll(hbis);
+
+    for(int i=0; i<hbiFutures.size(); i++) {
+      WorkItemHdfsRegionInfo work = hbis.get(i);
+      Future<Void> f = hbiFutures.get(i);
+      try {
+        f.get();
+      } catch(ExecutionException e) {
+        LOG.warn("Failed to read .regioninfo file for region " +
+              work.hbi.getRegionNameAsString(), e.getCause());
       }
     }
 
@@ -1054,22 +1057,22 @@ public void loadHdfsRegionDirs() throws IOException, InterruptedException {
     }
 
     // level 1:  <HBASE_DIR>/*
-    WorkItemHdfsDir[] dirs = new WorkItemHdfsDir[tableDirs.size()];
-    int num = 0;
+    List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
+    List<Future<Void>> dirsFutures;
+
     for (FileStatus tableDir : tableDirs) {
       LOG.debug("Loading region dirs from " +tableDir.getPath());
-      dirs[num] = new WorkItemHdfsDir(this, fs, errors, tableDir);
-      executor.execute(dirs[num]);
-      num++;
+      dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
     }
 
-    // wait for all directories to be done
-    for (int i = 0; i < num; i++) {
-      WorkItemHdfsDir dir = dirs[i];
-      synchronized (dir) {
-        while (!dir.isDone()) {
-          dir.wait();
-        }
+    // Invoke and wait for Callables to complete
+    dirsFutures = executor.invokeAll(dirs);
+
+    for(Future<Void> f: dirsFutures) {
+      try {
+        f.get();
+      } catch(ExecutionException e) {
+        LOG.warn("Could not load region dir " , e.getCause());
       }
     }
   }
@@ -1136,22 +1139,24 @@ public boolean isAborted(){
   void processRegionServers(Collection<ServerName> regionServerList)
     throws IOException, InterruptedException {
 
-    WorkItemRegion[] work = new WorkItemRegion[regionServerList.size()];
-    int num = 0;
+    List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
+    List<Future<Void>> workFutures;
 
     // loop to contact each region server in parallel
     for (ServerName rsinfo: regionServerList) {
-      work[num] = new WorkItemRegion(this, rsinfo, errors, connection);
-      executor.execute(work[num]);
-      num++;
+      workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
     }
     
-    // wait for all submitted tasks to be done
-    for (int i = 0; i < num; i++) {
-      synchronized (work[i]) {
-        while (!work[i].isDone()) {
-          work[i].wait();
-        }
+    workFutures = executor.invokeAll(workItems);
+
+    for(int i=0; i<workFutures.size(); i++) {
+      WorkItemRegion item = workItems.get(i);
+      Future<Void> f = workFutures.get(i);
+      try {
+        f.get();
+      } catch(ExecutionException e) {
+        LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
+            e.getCause());
       }
     }
   }
@@ -2366,10 +2371,11 @@ public String getRegionNameAsString() {
       if (metaEntry != null) {
         return metaEntry.getRegionNameAsString();
       } else if (hdfsEntry != null) {
-        return hdfsEntry.hri.getRegionNameAsString();
-      } else {
-        return null;
+        if (hdfsEntry.hri != null) {
+          return hdfsEntry.hri.getRegionNameAsString();
+        }
       }
+      return null;
     }
 
     public byte[] getRegionName() {
@@ -2618,12 +2624,11 @@ public synchronized void progress() {
   /**
    * Contact a region server and get all information from it
    */
-  static class WorkItemRegion implements Runnable {
+  static class WorkItemRegion implements Callable<Void> {
     private HBaseFsck hbck;
     private ServerName rsinfo;
     private ErrorReporter errors;
     private HConnection connection;
-    private boolean done;
 
     WorkItemRegion(HBaseFsck hbck, ServerName info,
                    ErrorReporter errors, HConnection connection) {
@@ -2631,16 +2636,10 @@ static class WorkItemRegion implements Runnable {
       this.rsinfo = info;
       this.errors = errors;
       this.connection = connection;
-      this.done = false;
-    }
-
-    // is this task done?
-    synchronized boolean isDone() {
-      return done;
     }
 
     @Override
-    public synchronized void run() {
+    public synchronized Void call() throws IOException {
       errors.progress();
       try {
         HRegionInterface server =
@@ -2671,10 +2670,9 @@ public synchronized void run() {
       } catch (IOException e) {          // unable to connect to the region server. 
         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
           " Unable to fetch region information. " + e);
-      } finally {
-        done = true;
-        notifyAll(); // wakeup anybody waiting for this item to be done
+        throw e;
       }
+      return null;
     }
 
     private List<HRegionInfo> filterOnlyMetaRegions(List<HRegionInfo> regions) {
@@ -2692,12 +2690,11 @@ private List<HRegionInfo> filterOnlyMetaRegions(List<HRegionInfo> regions) {
    * Contact hdfs and get all information about specified table directory into
    * regioninfo list.
    */
-  static class WorkItemHdfsDir implements Runnable {
+  static class WorkItemHdfsDir implements Callable<Void> {
     private HBaseFsck hbck;
     private FileStatus tableDir;
     private ErrorReporter errors;
     private FileSystem fs;
-    private boolean done;
 
     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors, 
                     FileStatus status) {
@@ -2705,27 +2702,25 @@ static class WorkItemHdfsDir implements Runnable {
       this.fs = fs;
       this.tableDir = status;
       this.errors = errors;
-      this.done = false;
     }
 
-    synchronized boolean isDone() {
-      return done;
-    } 
-
     @Override
-    public synchronized void run() {
+    public synchronized Void call() throws IOException {
       try {
         String tableName = tableDir.getPath().getName();
         // ignore hidden files
         if (tableName.startsWith(".") &&
-            !tableName.equals( Bytes.toString(HConstants.META_TABLE_NAME)))
-          return;
+            !tableName.equals( Bytes.toString(HConstants.META_TABLE_NAME))) {
+          return null;
+        }
         // level 2: <HBASE_DIR>/<table>/*
         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
         for (FileStatus regionDir : regionDirs) {
           String encodedName = regionDir.getPath().getName();
           // ignore directories that aren't hexadecimal
-          if (!encodedName.toLowerCase().matches("[0-9a-f]+")) continue;
+          if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
+            continue;
+          }
 
           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
@@ -2762,10 +2757,9 @@ public synchronized void run() {
         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
             + tableDir.getPath().getName()
             + " Unable to fetch region information. " + e);
-      } finally {
-        done = true;
-        notifyAll();
+        throw e;
       }
+      return null;
     }
   }
 
@@ -2773,51 +2767,41 @@ public synchronized void run() {
    * Contact hdfs and get all information about specified table directory into
    * regioninfo list.
    */
-  static class WorkItemHdfsRegionInfo implements Runnable {
+  static class WorkItemHdfsRegionInfo implements Callable<Void> {
     private HbckInfo hbi;
     private HBaseFsck hbck;
     private ErrorReporter errors;
-    private boolean done;
 
     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
       this.hbi = hbi;
       this.hbck = hbck;
       this.errors = errors;
-      this.done = false;
-    }
-
-    synchronized boolean isDone() {
-      return done;
     }
 
     @Override
-    public synchronized void run() {
-      try {
-        // only load entries that haven't been loaded yet.
-        if (hbi.getHdfsHRI() == null) {
+    public synchronized Void call() throws IOException {
+      // only load entries that haven't been loaded yet.
+      if (hbi.getHdfsHRI() == null) {
+        try {
+          hbck.loadHdfsRegioninfo(hbi);
+        } catch (IOException ioe) {
+          String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
+              + Bytes.toString(hbi.getTableName()) + " in hdfs dir "
+              + hbi.getHdfsRegionDir()
+              + "!  It may be an invalid format or version file.  Treating as "
+              + "an orphaned regiondir.";
+          errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
           try {
-            hbck.loadHdfsRegioninfo(hbi);
-          } catch (IOException ioe) {
-            String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
-                + Bytes.toString(hbi.getTableName()) + " in hdfs dir "
-                + hbi.getHdfsRegionDir()
-                + "!  It may be an invalid format or version file.  Treating as "
-                + "an orphaned regiondir.";
-            errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
-            try {
-              hbck.debugLsr(hbi.getHdfsRegionDir());
-            } catch (IOException ioe2) {
-              LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
-              return; // TODO convert this in to a future
-            }
-            hbck.orphanHdfsDirs.add(hbi);
-            return;
+            hbck.debugLsr(hbi.getHdfsRegionDir());
+          } catch (IOException ioe2) {
+            LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
+            throw ioe2;
           }
+          hbck.orphanHdfsDirs.add(hbi);
+          throw ioe;
         }
-      } finally {
-        done = true;
-        notifyAll();
       }
+      return null;
     }
   };
 

From aa3f077dbdcf7bd788ca8e4babc3209f66877745 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 4 Jun 2012 10:06:56 +0000
Subject: [PATCH 0261/1540] HBASE-5874 When 'fs.default.name' not configured,
 the hbck tool and Merge tool throws IllegalArgumentException (fulin wang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1345903 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +++++-
 src/main/java/org/apache/hadoop/hbase/util/Merge.java     | 8 ++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 81547b871217..b47b422e945c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -2991,7 +2992,10 @@ public static void main(String[] args) throws Exception {
 
     // create a fsck object
     Configuration conf = HBaseConfiguration.create();
-    conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
+    Path hbasedir = new Path(conf.get(HConstants.HBASE_DIR));
+    URI defaultFs = hbasedir.getFileSystem(conf).getUri();
+    conf.set("fs.defaultFS", defaultFs.toString());     // for hadoop 0.21+
+    conf.set("fs.default.name", defaultFs.toString());  // for hadoop 0.20
     HBaseFsck fsck = new HBaseFsck(conf);
     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Merge.java b/src/main/java/org/apache/hadoop/hbase/util/Merge.java
index 67d0fda56811..1ed4225bc517 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Merge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Merge.java
@@ -373,8 +373,12 @@ private boolean notInTable(final byte [] tn, final byte [] rn) {
   }
 
   private void usage() {
-    System.err.println(
-        "Usage: bin/hbase merge <table-name> <region-1> <region-2>\n");
+    System.err
+        .println("For hadoop 0.20,  Usage: bin/hbase org.apache.hadoop.hbase.util.Merge "
+            + "[-Dfs.default.name=hdfs://nn:port] <table-name> <region-1> <region-2>\n");
+    System.err
+        .println("For hadoop 0.21+, Usage: bin/hbase org.apache.hadoop.hbase.util.Merge "
+            + "[-Dfs.defaultFS=hdfs://nn:port] <table-name> <region-1> <region-2>\n");
   }
 
   public static void main(String[] args) {

From 6d80be3b2ae464d31f71ece7f887a3e0b732cef5 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 4 Jun 2012 17:54:14 +0000
Subject: [PATCH 0262/1540] HBASE-6067 Addendum adopts suggestions from Daryn

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1346055 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/wal/HLog.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 98cc744ed10c..72f35d7c3fdb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -448,9 +448,8 @@ private long getDefaultBlockSize() throws IOException {
     Method m = null;
     Class<? extends FileSystem> cls = this.fs.getClass();
     try {
-      m = cls.getDeclaredMethod("getDefaultBlockSize",
+      m = cls.getMethod("getDefaultBlockSize",
           new Class<?>[] { Path.class });
-      m.setAccessible(true);
     } catch (NoSuchMethodException e) {
       LOG.info("FileSystem doesn't support getDefaultBlockSize");
     } catch (SecurityException e) {

From 1d43ca0ed8fddf8bbf9de773b485576dd69819ab Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 4 Jun 2012 19:17:19 +0000
Subject: [PATCH 0263/1540] HBASE-6158 Data loss if the words 'merges' or
 'splits' are used as Column Family name

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1346096 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
 .../org/apache/hadoop/hbase/regionserver/SplitTransaction.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 65037b5bd527..d0a858d406f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -172,7 +172,7 @@
  */
 public class HRegion implements HeapSize { // , Writable{
   public static final Log LOG = LogFactory.getLog(HRegion.class);
-  static final String MERGEDIR = "merges";
+  private static final String MERGEDIR = ".merges";
 
   final AtomicBoolean closed = new AtomicBoolean(false);
   /* Closing can take some time; use the closing flag if there is stuff we don't
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index cbfa1436e7b7..c1d6376c0856 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -84,7 +84,7 @@
  */
 public class SplitTransaction {
   private static final Log LOG = LogFactory.getLog(SplitTransaction.class);
-  private static final String SPLITDIR = "splits";
+  private static final String SPLITDIR = ".splits";
 
   /*
    * Region to split

From 9ce6b2a7fcb7e8f8e03d22b0c9823c45dc46471b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 5 Jun 2012 04:08:10 +0000
Subject: [PATCH 0264/1540] HBASE-6161 Log Error when thrift server fails to
 start up

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1346232 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 10ded9964877..e03c7755fe17 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -245,7 +245,7 @@ public void run() {
       setupServer();
       tserver.serve();
     } catch (Exception e) {
-      LOG.fatal("Cannot run ThriftServer");
+      LOG.fatal("Cannot run ThriftServer", e);
       // Crash the process if the ThriftServer is not running
       System.exit(-1);
     }

From 3caccb10bbaeb17dd23f77ef8f0564e5a72a2fc5 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 5 Jun 2012 16:40:47 +0000
Subject: [PATCH 0265/1540] HBASE-6046 Master retry on ZK session expiry causes
 inconsistent region assignments. (Ashutosh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1346461 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   |  75 ++++++------
 .../hadoop/hbase/master/MasterFileSystem.java |   4 +-
 .../hadoop/hbase/master/SplitLogManager.java  |  15 ++-
 .../hbase/master/TestCatalogJanitor.java      |   2 +-
 .../master/TestMasterZKSessionRecovery.java   | 110 +++++++++++++++++-
 5 files changed, 164 insertions(+), 42 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 7ef315a3df2f..89eebd6fe090 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -341,7 +341,7 @@ public void run() {
 
       // We are either the active master or we were asked to shutdown
       if (!this.stopped) {
-        finishInitialization(startupStatus);
+        finishInitialization(startupStatus, false);
         loop();
       }
     } catch (Throwable t) {
@@ -459,12 +459,13 @@ private void loop() {
    * <li>Ensure assignment of root and meta regions<li>
    * <li>Handle either fresh cluster start or master failover</li>
    * </ol>
+   * @param masterRecovery 
    *
    * @throws IOException
    * @throws InterruptedException
    * @throws KeeperException
    */
-  private void finishInitialization(MonitoredTask status)
+  private void finishInitialization(MonitoredTask status, boolean masterRecovery)
   throws IOException, InterruptedException, KeeperException {
 
     isActiveMaster = true;
@@ -478,7 +479,7 @@ private void finishInitialization(MonitoredTask status)
     status.setStatus("Initializing Master file system");
     this.masterActiveTime = System.currentTimeMillis();
     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
-    this.fileSystemManager = new MasterFileSystem(this, this, metrics);
+    this.fileSystemManager = new MasterFileSystem(this, this, metrics, masterRecovery);
 
     this.tableDescriptors =
       new FSTableDescriptors(this.fileSystemManager.getFileSystem(),
@@ -487,21 +488,24 @@ private void finishInitialization(MonitoredTask status)
     // publish cluster ID
     status.setStatus("Publishing Cluster ID in ZooKeeper");
     ClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
+    if (!masterRecovery) {
+      this.executorService = new ExecutorService(getServerName().toString());
+      this.serverManager = new ServerManager(this, this);
+    }
 
-    this.executorService = new ExecutorService(getServerName().toString());
-
-    this.serverManager = new ServerManager(this, this);
 
     status.setStatus("Initializing ZK system trackers");
     initializeZKBasedSystemTrackers();
+    
+    if (!masterRecovery) {
+      // initialize master side coprocessors before we start handling requests
+      status.setStatus("Initializing master coprocessors");
+      this.cpHost = new MasterCoprocessorHost(this, this.conf);
 
-    // initialize master side coprocessors before we start handling requests
-    status.setStatus("Initializing master coprocessors");
-    this.cpHost = new MasterCoprocessorHost(this, this.conf);
-
-    // start up all service threads.
-    status.setStatus("Initializing master service threads");
-    startServiceThreads();
+      // start up all service threads.
+      status.setStatus("Initializing master service threads");
+      startServiceThreads();
+    }
 
     // Wait for region servers to report in.
     this.serverManager.waitForRegionServers(status);
@@ -514,8 +518,9 @@ private void finishInitialization(MonitoredTask status)
         this.serverManager.recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
       }
     }
-
-    this.assignmentManager.startTimeOutMonitor();
+    if (!masterRecovery) {
+      this.assignmentManager.startTimeOutMonitor();
+    }
     // TODO: Should do this in background rather than block master startup
     status.setStatus("Splitting logs after master startup");
     splitLogAfterStartup(this.fileSystemManager);
@@ -543,13 +548,15 @@ private void finishInitialization(MonitoredTask status)
     status.setStatus("Fixing up missing daughters");
     fixupDaughters(status);
 
-    // Start balancer and meta catalog janitor after meta and regions have
-    // been assigned.
-    status.setStatus("Starting balancer and catalog janitor");
-    this.balancerChore = getAndStartBalancerChore(this);
-    this.catalogJanitorChore = new CatalogJanitor(this, this);
-    startCatalogJanitorChore();
-    registerMBean();
+    if (!masterRecovery) {
+      // Start balancer and meta catalog janitor after meta and regions have
+      // been assigned.
+      status.setStatus("Starting balancer and catalog janitor");
+      this.balancerChore = getAndStartBalancerChore(this);
+      this.catalogJanitorChore = new CatalogJanitor(this, this);
+      startCatalogJanitorChore();
+      registerMBean();
+    }
 
     status.markComplete("Initialization successful");
     LOG.info("Master has completed initialization");
@@ -560,12 +567,14 @@ private void finishInitialization(MonitoredTask status)
     // master initialization. See HBASE-5916.
     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
     
-    if (this.cpHost != null) {
-      // don't let cp initialization errors kill the master
-      try {
-        this.cpHost.postStartMaster();
-      } catch (IOException ioe) {
-        LOG.error("Coprocessor postStartMaster() hook failed", ioe);
+    if (!masterRecovery) {
+      if (this.cpHost != null) {
+        // don't let cp initialization errors kill the master
+        try {
+          this.cpHost.postStartMaster();
+        } catch (IOException ioe) {
+          LOG.error("Coprocessor postStartMaster() hook failed", ioe);
+        }
       }
     }
   }
@@ -1433,13 +1442,9 @@ public Boolean call() throws InterruptedException,
           if (!becomeActiveMaster(status)) {
             return Boolean.FALSE;
           }
-          initializeZKBasedSystemTrackers();
-          // Update in-memory structures to reflect our earlier Root/Meta assignment.
-          assignRootAndMeta(status);
-          // process RIT if any
-          // TODO: Why does this not call AssignmentManager.joinCluster?  Otherwise
-          // we are not processing dead servers if any.
-          assignmentManager.processDeadServersAndRegionsInTransition();
+          serverShutdownHandlerEnabled = false;
+          initialized = false;
+          finishInitialization(status, true);
           return Boolean.TRUE;
         } finally {
           status.cleanup();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index a57a7b6cd2db..31ee9df64525 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -81,7 +81,7 @@ public class MasterFileSystem {
   private final MasterServices services;
 
   public MasterFileSystem(Server master, MasterServices services,
-      MasterMetrics metrics)
+      MasterMetrics metrics, boolean masterRecovery)
   throws IOException {
     this.conf = master.getConfiguration();
     this.master = master;
@@ -103,7 +103,7 @@ public MasterFileSystem(Server master, MasterServices services,
     if (this.distributedLogSplitting) {
       this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
           master.getConfiguration(), master, master.getServerName().toString());
-      this.splitLogManager.finishInitialization();
+      this.splitLogManager.finishInitialization(masterRecovery);
     } else {
       this.splitLogManager = null;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index e13bff8bc5ac..2a11756fd02e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -182,9 +182,11 @@ public SplitLogManager(ZooKeeperWatcher zkw, Configuration conf,
         stopper);
   }
 
-  public void finishInitialization() {
-    Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName +
-      ".splitLogManagerTimeoutMonitor");
+  public void finishInitialization(boolean masterRecovery) {
+    if (!masterRecovery) {
+      Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName
+          + ".splitLogManagerTimeoutMonitor");
+    }
     // Watcher can be null during tests with Mock'd servers.
     if (this.watcher != null) {
       this.watcher.registerListener(this);
@@ -1196,4 +1198,11 @@ public String toString() {
       return statusMsg;
     }
   }
+  
+  /**
+   * Completes the initialization
+   */
+  public void finishInitialization() {
+    finishInitialization(false);
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index b4dcb83b9000..a117de171f0f 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -148,7 +148,7 @@ class MockMasterServices implements MasterServices {
     private final AssignmentManager asm;
 
     MockMasterServices(final Server server) throws IOException {
-      this.mfs = new MasterFileSystem(server, this, null);
+      this.mfs = new MasterFileSystem(server, this, null, false);
       this.asm = Mockito.mock(AssignmentManager.class);
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
index ba8b725cf2d1..ef9d368aea8b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
@@ -19,13 +19,31 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 import org.junit.After;
 import org.junit.Before;
@@ -46,6 +64,8 @@ public class TestMasterZKSessionRecovery {
   static {
     Configuration conf = TEST_UTIL.getConfiguration();
     conf.setLong("hbase.master.zksession.recover.timeout", 50000);
+    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        MockLoadBalancer.class, LoadBalancer.class);
   }
 
   @Before
@@ -92,5 +112,93 @@ public void testMasterZKSessionRecoverySuccess() throws Exception {
       new KeeperException.SessionExpiredException());
     assertFalse(m.isStopped());
   }
+  
+  /**
+   * Tests that the master does not call retainAssignment after recovery from
+   * expired zookeeper session. Without the HBASE-6046 fix master always tries
+   * to assign all the user regions by calling retainAssignment.
+   */
+  @Test
+  public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    cluster.startRegionServer();
+    HMaster m = cluster.getMaster();
+    // now the cluster is up. So assign some regions.
+    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
+        Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
+        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
+
+    String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
+    admin.createTable(new HTableDescriptor(tableName), SPLIT_KEYS);
+    ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
+    ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
+    m.getZooKeeperWatcher().close();
+    MockLoadBalancer.retainAssignCalled = false;
+    m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException());
+    assertFalse(m.isStopped());
+    // The recovered master should not call retainAssignment, as it is not a
+    // clean startup.
+    assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
+  }
+
+  static class MockLoadBalancer extends DefaultLoadBalancer {
+    static boolean retainAssignCalled = false;
+
+    @Override
+    public Map<ServerName, List<HRegionInfo>> retainAssignment(
+        Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
+      retainAssignCalled = true;
+      return super.retainAssignment(regions, servers);
+    }
+  }
+
+  /**
+   * Tests whether the logs are split when master recovers from a expired
+   * zookeeper session and an RS goes down.
+   */
+  @Test(timeout = 60000)
+  public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
+      KeeperException, InterruptedException {
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    cluster.startRegionServer();
+    HMaster m = cluster.getMaster();
+    // now the cluster is up. So assign some regions.
+    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
+        Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
+
+    String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor("col");
+    htd.addFamily(hcd);
+    admin.createTable(htd, SPLIT_KEYS);
+    ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
+    ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+
+    Put p = null;
+    int numberOfPuts = 0;
+    for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
+      p = new Put(Bytes.toBytes(numberOfPuts));
+      p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
+      table.put(p);
+    }
+    m.getZooKeeperWatcher().close();
+    m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException());
+    assertFalse(m.isStopped());
+    cluster.getRegionServer(0).abort("Aborting");
+    // Without patch for HBASE-6046 this test case will always timeout
+    // with patch the test case should pass.
+    Scan scan = new Scan();
+    int numberOfRows = 0;
+    ResultScanner scanner = table.getScanner(scan);
+    Result[] result = scanner.next(1);
+    while (result != null && result.length > 0) {
+      numberOfRows++;
+      result = scanner.next(1);
+    }
+    assertEquals("Number of rows should be equal to number of puts.", numberOfPuts, numberOfRows);
+  }
 }
 

From f84b158bfb444e5f53a48d6ba24b1f84727ca4f5 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 5 Jun 2012 18:07:41 +0000
Subject: [PATCH 0266/1540] HBASE-6160 META entries from daughters can be
 deleted before parent entries

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1346503 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/CatalogJanitor.java   | 36 +++++--
 .../hbase/master/TestCatalogJanitor.java      | 95 +++++++++++++++++++
 2 files changed, 124 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 4ee75ee948ee..c1eb02a339ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -22,6 +22,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -97,11 +98,10 @@ protected void chore() {
   }
 
   /**
-   * Run janitorial scan of catalog <code>.META.</code> table looking for
-   * garbage to collect.
-   * @throws IOException
+   * Scans META and returns a number of scanned rows, and
+   * an ordered map of split parents.
    */
-  void scan() throws IOException {
+  Pair<Integer, Map<HRegionInfo, Result>> getSplitParents() throws IOException {
     // TODO: Only works with single .META. region currently.  Fix.
     final AtomicInteger count = new AtomicInteger(0);
     // Keep Map of found split parents.  There are candidates for cleanup.
@@ -123,18 +123,40 @@ public boolean visit(Result r) throws IOException {
     };
     // Run full scan of .META. catalog table passing in our custom visitor
     MetaReader.fullScan(this.server.getCatalogTracker(), visitor);
+
+    return new Pair<Integer, Map<HRegionInfo, Result>>(count.get(), splitParents);
+  }
+
+  /**
+   * Run janitorial scan of catalog <code>.META.</code> table looking for
+   * garbage to collect.
+   * @throws IOException
+   */
+  int scan() throws IOException {
+    Pair<Integer, Map<HRegionInfo, Result>> pair = getSplitParents();
+    int count = pair.getFirst();
+    Map<HRegionInfo, Result> splitParents = pair.getSecond();
+
     // Now work on our list of found parents. See if any we can clean up.
     int cleaned = 0;
+    HashSet<HRegionInfo> parentNotCleaned = new HashSet<HRegionInfo>(); //regions whose parents are still around
     for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
-      if (cleanParent(e.getKey(), e.getValue())) cleaned++;
+      if (!parentNotCleaned.contains(e.getKey()) && cleanParent(e.getKey(), e.getValue())) {
+        cleaned++;
+      } else {
+        // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160)
+        parentNotCleaned.add(getDaughterRegionInfo(e.getValue(), HConstants.SPLITA_QUALIFIER));
+        parentNotCleaned.add(getDaughterRegionInfo(e.getValue(), HConstants.SPLITB_QUALIFIER));
+      }
     }
     if (cleaned != 0) {
-      LOG.info("Scanned " + count.get() + " catalog row(s) and gc'd " + cleaned +
+      LOG.info("Scanned " + count + " catalog row(s) and gc'd " + cleaned +
         " unreferenced parent region(s)");
     } else if (LOG.isDebugEnabled()) {
-      LOG.debug("Scanned " + count.get() + " catalog row(s) and gc'd " + cleaned +
+      LOG.debug("Scanned " + count + " catalog row(s) and gc'd " + cleaned +
       " unreferenced parent region(s)");
     }
+    return cleaned;
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index a117de171f0f..5d76a99eecbf 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -23,6 +23,9 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -53,9 +56,11 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.Test;
@@ -473,6 +478,96 @@ private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
     janitor.join();
   }
 
+  /**
+   * CatalogJanitor.scan() should not clean parent regions if their own
+   * parents are still referencing them. This ensures that grandfather regions
+   * do not point to deleted parent regions.
+   */
+  @Test
+  public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
+    HBaseTestingUtility htu = new HBaseTestingUtility();
+    setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
+    Server server = new MockServer(htu);
+    MasterServices services = new MockMasterServices(server);
+
+    final HTableDescriptor htd = createHTableDescriptor();
+
+    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
+
+    // Parent
+    HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
+      new byte[0], true);
+    // Sleep a second else the encoded name on these regions comes out
+    // same for all with same start key and made in same second.
+    Thread.sleep(1001);
+
+    // Daughter a
+    HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
+      Bytes.toBytes("ccc"), true);
+    Thread.sleep(1001);
+    // Make daughters of daughter a; splitaa and splitab.
+    HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
+      Bytes.toBytes("bbb"), false);
+    HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"),
+      Bytes.toBytes("ccc"), false);
+
+    // Daughter b
+    HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"),
+        new byte[0]);
+    Thread.sleep(1001);
+
+    final Map<HRegionInfo, Result> splitParents =
+        new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
+    splitParents.put(parent, makeResultFromHRegionInfo(parent, splita, splitb));
+    splitParents.put(splita, makeResultFromHRegionInfo(splita, splitaa, splitab));
+
+    CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
+    doReturn(new Pair<Integer, Map<HRegionInfo, Result>>(
+        10, splitParents)).when(janitor).getSplitParents();
+
+    //create ref from splita to parent
+    Path splitaRef =
+        createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
+
+    //parent and A should not be removed
+    assertEquals(0, janitor.scan());
+
+    //now delete the ref
+    FileSystem fs = FileSystem.get(htu.getConfiguration());
+    assertTrue(fs.delete(splitaRef, true));
+
+    //now, both parent, and splita can be deleted
+    assertEquals(2, janitor.scan());
+
+    services.stop("test finished");
+    janitor.join();
+  }
+
+  private Result makeResultFromHRegionInfo(HRegionInfo region, HRegionInfo splita,
+      HRegionInfo splitb) throws IOException {
+    List<KeyValue> kvs = new ArrayList<KeyValue>();
+    kvs.add(new KeyValue(
+        region.getRegionName(),
+        HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(region)));
+
+    if (splita != null) {
+      kvs.add(new KeyValue(
+          region.getRegionName(),
+          HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
+          Writables.getBytes(splita)));
+    }
+
+    if (splitb != null) {
+      kvs.add(new KeyValue(
+          region.getRegionName(),
+          HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
+          Writables.getBytes(splitb)));
+    }
+
+    return new Result(kvs);
+  }
+
   private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
       final String subdir)
   throws IOException {

From 27a0e583cc1a5f3a6c3baa9ebad64bc423248e77 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 5 Jun 2012 18:12:14 +0000
Subject: [PATCH 0267/1540] HBASE-5609 Add the ability to pass additional
 information for slow query logging

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1346509 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/Get.java   |  4 ++
 .../apache/hadoop/hbase/client/Mutation.java  |  4 ++
 .../hbase/client/OperationWithAttributes.java | 26 ++++++++++++
 .../org/apache/hadoop/hbase/client/Scan.java  |  4 ++
 .../hadoop/hbase/client/TestAttributes.java   | 40 +++++++++++++++++++
 5 files changed, 78 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index c27b6472ffb9..3f35caecd999 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -371,6 +371,10 @@ public Map<String, Object> toMap(int maxCols) {
     if (this.filter != null) {
       map.put("filter", this.filter.toString());
     }
+    // add the id if set
+    if (getId() != null) {
+      map.put("id", getId());
+    }
     return map;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index d019ab3b9aaf..79a6d1fd7b2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -102,6 +102,10 @@ public Map<String, Object> toMap(int maxCols) {
       }
     }
     map.put("totalColumns", colCount);
+    // add the id if set
+    if (getId() != null) {
+      map.put("id", getId());
+    }
     return map;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java b/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
index 698b31a5bfd5..97e19b91ede2 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
@@ -35,6 +35,9 @@ public abstract class OperationWithAttributes extends Operation implements Attri
   // a opaque blob of attributes
   private Map<String, byte[]> attributes;
 
+  // used for uniquely identifying an operation
+  static public String ID_ATRIBUTE = "_operation.attributes.id";
+
   public void setAttribute(String name, byte[] value) {
     if (attributes == null && value == null) {
       return;
@@ -104,4 +107,27 @@ protected void readAttributes(final DataInput in) throws IOException {
       }
     }
   }
+
+  /**
+   * This method allows you to set an identifier on an operation. The original
+   * motivation for this was to allow the identifier to be used in slow query
+   * logging, but this could obviously be useful in other places. One use of
+   * this could be to put a class.method identifier in here to see where the
+   * slow query is coming from.
+   * @param id
+   *          id to set for the scan
+   */
+  public void setId(String id) {
+    setAttribute(ID_ATRIBUTE, Bytes.toBytes(id));
+  }
+
+  /**
+   * This method allows you to retrieve the identifier for the operation if one
+   * was set.
+   * @return the id or null if not set
+   */
+  public String getId() {
+    byte[] attr = getAttribute(ID_ATRIBUTE);
+    return attr == null? null: Bytes.toString(attr);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index de114f5ebedc..63cc975e06f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -556,6 +556,10 @@ public Map<String, Object> toMap(int maxCols) {
     if (this.filter != null) {
       map.put("filter", this.filter.toString());
     }
+    // add the id if set
+    if (getId() != null) {
+      map.put("id", getId());
+    }
     return map;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java
index 4821ceaf06ac..e071b769736f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java
@@ -154,6 +154,46 @@ public void testDeleteAttributes() {
     Assert.assertNull(del.getAttributesMap().get("attribute1"));
   }
 
+  @Test
+  public void testGetId() {
+    Get get = new Get();
+    Assert.assertNull("Make sure id is null if unset", get.toMap().get("id"));
+    get.setId("myId");
+    Assert.assertEquals("myId", get.toMap().get("id"));
+  }
+
+  @Test
+  public void testAppendId() {
+    Append append = new Append();
+    Assert.assertNull("Make sure id is null if unset", append.toMap().get("id"));
+    append.setId("myId");
+    Assert.assertEquals("myId", append.toMap().get("id"));
+  }
+
+  @Test
+  public void testDeleteId() {
+    Delete delete = new Delete();
+    Assert.assertNull("Make sure id is null if unset", delete.toMap().get("id"));
+    delete.setId("myId");
+    Assert.assertEquals("myId", delete.toMap().get("id"));
+  }
+
+  @Test
+  public void testPutId() {
+    Put put = new Put();
+    Assert.assertNull("Make sure id is null if unset", put.toMap().get("id"));
+    put.setId("myId");
+    Assert.assertEquals("myId", put.toMap().get("id"));
+  }
+
+  @Test
+  public void testScanId() {
+    Scan scan = new Scan();
+    Assert.assertNull("Make sure id is null if unset", scan.toMap().get("id"));
+    scan.setId("myId");
+    Assert.assertEquals("myId", scan.toMap().get("id"));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 0841e6af3c9d5160433411e058556daf8e06b222 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 7 Jun 2012 04:59:50 +0000
Subject: [PATCH 0268/1540] HBASE-6182 TestStoreFile fails with jdk1.7

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1347390 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/TestStoreFile.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 76c329fa4bff..fbec7614ac0a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -371,7 +371,7 @@ private void bloomWriteRead(StoreFile.Writer writer, FileSystem fs)
     int falseNeg = 0;
     for (int i = 0; i < 2000; i++) {
       String row = String.format(localFormatter, i);
-      TreeSet<byte[]> columns = new TreeSet<byte[]>();
+      TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
       columns.add("family:col".getBytes());
 
       Scan scan = new Scan(row.getBytes(),row.getBytes());
@@ -531,7 +531,7 @@ public void testBloomTypes() throws Exception {
         for (int j = 0; j < colCount*2; ++j) {   // column qualifiers
           String row = String.format(localFormatter, i);
           String col = String.format(localFormatter, j);
-          TreeSet<byte[]> columns = new TreeSet<byte[]>();
+          TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
           columns.add(("col" + col).getBytes());
 
           Scan scan = new Scan(row.getBytes(),row.getBytes());
@@ -715,7 +715,7 @@ public void testMultipleTimestamps() throws IOException {
         StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     StoreFile.Reader reader = hsf.createReader();
     StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
-    TreeSet<byte[]> columns = new TreeSet<byte[]>();
+    TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
     columns.add(qualifier);
 
     scan.setTimeRange(20, 100);

From db98a990774f14c6df2131692f45e144d0341d83 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 7 Jun 2012 17:22:05 +0000
Subject: [PATCH 0269/1540] HBASE-6146 Disabling of Catalog tables should not
 be allowed (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1347718 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HBaseAdmin.java    |  2 ++
 .../org/apache/hadoop/hbase/client/TestAdmin.java | 15 +++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 67e5631e2b58..3e45154daad0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -695,6 +695,7 @@ public void enableTableAsync(final String tableName)
    */
   public void enableTableAsync(final byte [] tableName)
   throws IOException {
+    HTableDescriptor.isLegalTableName(tableName);
     isMasterRunning();
     try {
       getMaster().enableTable(tableName);
@@ -763,6 +764,7 @@ public void disableTableAsync(final String tableName) throws IOException {
    * @since 0.90.0
    */
   public void disableTableAsync(final byte [] tableName) throws IOException {
+    HTableDescriptor.isLegalTableName(tableName);
     isMasterRunning();
     try {
       getMaster().disableTable(tableName);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index dae300e9543f..37d9946dc590 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -1529,6 +1529,21 @@ public void testCheckHBaseAvailableClosesConnection() throws Exception {
     Assert.assertEquals(initialCount, finalCount) ;
   }
 
+  @Test
+  public void testDisableCatalogTable() throws Exception {
+    try {
+      this.admin.disableTable(".META.");
+      fail("Expected to throw IllegalArgumentException");
+    } catch (IllegalArgumentException e) {
+    }
+    // Before the fix for HBASE-6146, the below table creation was failing as the META table
+    // actually getting disabled by the disableTable() call.
+    HTableDescriptor htd = new HTableDescriptor("testDisableCatalogTable".getBytes());
+    HColumnDescriptor hcd = new HColumnDescriptor("cf1".getBytes());
+    htd.addFamily(hcd);
+    TEST_UTIL.getHBaseAdmin().createTable(htd);
+  }
+  
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From d757622d402eb83d071714f7c1ee88463bb67cc2 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 9 Jun 2012 03:34:18 +0000
Subject: [PATCH 0270/1540] HBASE-6173 hbck check specified tables only

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1348306 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                   |  1 +
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 70 +++++++++----------
 .../hadoop/hbase/util/TestHBaseFsck.java      | 45 ++++++++++++
 .../hbase/util/hbck/HbckTestingUtil.java      | 17 +++--
 4 files changed, 91 insertions(+), 42 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index a42863a6d2f8..aa106b739af6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -374,6 +374,7 @@ Improvement
     [HBASE-5823] - Hbck should be able to print help
     [HBASE-5862] - After Region Close remove the Operation Metrics.
     [HBASE-5863] - Improve the graceful_stop.sh CLI help (especially about reloads)
+    [HBASE-6173] - hbck check specified tables only
 
 New Feature
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index b47b422e945c..d1562b08ff72 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -140,7 +140,6 @@ public class HBaseFsck {
   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
-  private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
   private static boolean rsSupportsOffline = true;
   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
@@ -169,8 +168,9 @@ public class HBaseFsck {
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
 
-  // limit fixes to listed tables, if empty atttempt to fix all
-  private List<byte[]> tablesToFix = new ArrayList<byte[]>();
+  // limit checking/fixes to listed tables, if empty attempt to check/fix all
+  // -ROOT- and .META. are always checked
+  private Set<String> tablesIncluded = new HashSet<String>();
   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
@@ -200,6 +200,11 @@ public class HBaseFsck {
    * This map from Tablename -> TableInfo contains the structures necessary to
    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
    * to prevent dupes.
+   *
+   * If tablesIncluded is empty, this map contains all tables.
+   * Otherwise, it contains only meta tables and tables in tablesIncluded,
+   * unless checkMetaOnly is specified, in which case, it contains only
+   * the meta tables (.META. and -ROOT-).
    */
   private SortedMap<String, TableInfo> tablesInfo = new ConcurrentSkipListMap<String,TableInfo>();
 
@@ -672,6 +677,7 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
         // only executed once per table.
         modTInfo = new TableInfo(tableName);
         Path hbaseRoot = new Path(conf.get(HConstants.HBASE_DIR));
+        tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
               FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
@@ -681,10 +687,8 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
           LOG.error("Unable to read .tableinfo from " + hbaseRoot, ioe);
           throw ioe;
         }
-
       }
       modTInfo.addRegionInfo(hbi);
-      tablesInfo.put(tableName, modTInfo);
     }
 
     return tablesInfo;
@@ -839,14 +843,8 @@ private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
     for (TableInfo tInfo : tablesInfo.values()) {
       TableIntegrityErrorHandler handler;
       if (fixHoles || fixOverlaps) {
-        if (shouldFixTable(Bytes.toBytes(tInfo.getName()))) {
-          handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
-              fixHoles, fixOverlaps);
-        } else {
-          LOG.info("Table " + tInfo.getName() + " is not in the include table " +
-            "list.  Just suggesting fixes.");
-          handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
-        }
+        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
+          fixHoles, fixOverlaps);
       } else {
         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
       }
@@ -1034,7 +1032,7 @@ public void loadHdfsRegionDirs() throws IOException, InterruptedException {
       if (dirName.equals(HConstants.VERSION_FILE_NAME)) {
         foundVersionFile = true;
       } else {
-        if (!checkMetaOnly ||
+        if ((!checkMetaOnly && isTableIncluded(dirName)) ||
             dirName.equals("-ROOT-") ||
             dirName.equals(".META.")) {
           tableDirs.add(file);
@@ -2214,9 +2212,14 @@ public boolean processRow(Result result) throws IOException {
           if (pair.getSecond() != null) {
             sn = pair.getSecond();
           }
-          MetaEntry m = new MetaEntry(pair.getFirst(), sn, ts);
+          HRegionInfo hri = pair.getFirst();
+          if (!(isTableIncluded(hri.getTableNameAsString())
+              || hri.isMetaRegion() || hri.isRootRegion())) {
+            return true;
+          }
+          MetaEntry m = new MetaEntry(hri, sn, ts);
           HbckInfo hbInfo = new HbckInfo(m);
-          HbckInfo previous = regionInfoMap.put(pair.getFirst().getEncodedName(), hbInfo);
+          HbckInfo previous = regionInfoMap.put(hri.getEncodedName(), hbInfo);
           if (previous != null) {
             throw new IOException("Two entries in META are same " + previous);
           }
@@ -2242,7 +2245,7 @@ public boolean processRow(Result result) throws IOException {
       // Scan .META. to pick up user regions
       MetaScanner.metaScan(conf, visitor);
     }
-    
+
     errors.print("");
     return true;
   }
@@ -2648,9 +2651,7 @@ public synchronized Void call() throws IOException {
 
         // list all online regions from this region server
         List<HRegionInfo> regions = server.getOnlineRegions();
-        if (hbck.checkMetaOnly) {
-          regions = filterOnlyMetaRegions(regions);
-        }
+        regions = filterRegions(regions);
         if (details) {
           errors.detail("RegionServer: " + rsinfo.getServerName() +
                            " number of regions: " + regions.size());
@@ -2676,10 +2677,11 @@ public synchronized Void call() throws IOException {
       return null;
     }
 
-    private List<HRegionInfo> filterOnlyMetaRegions(List<HRegionInfo> regions) {
+    private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
       List<HRegionInfo> ret = Lists.newArrayList();
       for (HRegionInfo hri : regions) {
-        if (hri.isMetaTable()) {
+        if (hri.isMetaTable() || (!hbck.checkMetaOnly
+            && hbck.isTableIncluded(hri.getTableNameAsString()))) {
           ret.add(hri);
         }
       }
@@ -2924,22 +2926,15 @@ public int getMaxOverlapsToSideline() {
   }
 
   /**
-   * Only fix tables specified by the list
+   * Only check/fix tables specified by the list,
+   * Empty list means all tables are included.
    */
-  boolean shouldFixTable(byte[] table) {
-    if (tablesToFix.size() == 0) {
-      return true;
-    }
-
-    // doing this naively since byte[] equals may not do what we want.
-    for (byte[] t : tablesToFix) {
-      if (Bytes.equals(t, table)) return true;
-    }
-    return false;
+  boolean isTableIncluded(String table) {
+    return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
   }
 
-  void includeTable(byte[] table) {
-    tablesToFix.add(table);
+  public void includeTable(String table) {
+    tablesIncluded.add(table);
   }
 
   /**
@@ -3101,9 +3096,8 @@ public static void main(String[] args) throws Exception {
         System.err.println("Unrecognized option:" + cmd);
         printUsageAndExit();
       } else {
-        byte[] table = Bytes.toBytes(cmd);
-        fsck.includeTable(table);
-        System.out.println("Allow fixes for table: " + cmd);
+        fsck.includeTable(cmd);
+        System.out.println("Allow checking/fixes for table: " + cmd);
       }
     }
     // do the real work of fsck
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 80c9fe25d64d..02e4c027fcbe 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -967,6 +967,51 @@ public void testRegionShouldNotBeDeployed() throws Exception {
     }
   }
 
+  /**
+   * This creates two tables and mess both of them and fix them one by one
+   */
+  @Test
+  public void testFixByTable() throws Exception {
+    String table1 = "testFixByTable1";
+    String table2 = "testFixByTable2";
+    try {
+      setupTable(table1);
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table1);
+      // Mess them up by leaving a hole in the hdfs data
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+        Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+      setupTable(table2);
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table2);
+      // Mess them up by leaving a hole in the hdfs data
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+        Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
+
+      // fix hole in table 1
+      doFsck(conf, true, table1);
+      // check that hole in table 1 fixed
+      assertNoErrors(doFsck(conf, false, table1));
+      // check that hole in table 2 still there
+      assertErrors(doFsck(conf, false, table2),
+        new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
+
+      // fix hole in table 2
+      doFsck(conf, true, table2);
+      // check that hole in both tables fixed
+      assertNoErrors(doFsck(conf, false));
+      assertEquals(ROWKEYS.length - 2, countRows());
+    } finally {
+      deleteTable(table1);
+      deleteTable(table2);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 393cbbaed777..f4c598378966 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -28,13 +28,20 @@
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 
 public class HbckTestingUtil {
-  public static HBaseFsck doFsck(Configuration conf, boolean fix) throws Exception {
-    return doFsck(conf, fix, fix, fix, fix,fix, fix);
+  public static HBaseFsck doFsck(
+      Configuration conf, boolean fix) throws Exception {
+    return doFsck(conf, fix, null);
+  }
+
+  public static HBaseFsck doFsck(
+      Configuration conf, boolean fix, String table) throws Exception {
+    return doFsck(conf, fix, fix, fix, fix,fix, fix, table);
   }
 
   public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
       boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
-      boolean fixHdfsOrphans, boolean fixVersionFile) throws Exception {
+      boolean fixHdfsOrphans, boolean fixVersionFile,
+      String table) throws Exception {
     HBaseFsck fsck = new HBaseFsck(conf);
     fsck.connect();
     fsck.setDisplayFullReport(); // i.e. -details
@@ -45,11 +52,13 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
     fsck.setFixHdfsOverlaps(fixHdfsOverlaps);
     fsck.setFixHdfsOrphans(fixHdfsOrphans);
     fsck.setFixVersionFile(fixVersionFile);
+    if (table != null) {
+      fsck.includeTable(table);
+    }
     fsck.onlineHbck();
     return fsck;
   }
 
-
   public static void assertNoErrors(HBaseFsck fsck) throws Exception {
     List<ERROR_CODE> errs = fsck.getErrors().getErrorList();
     assertEquals(new ArrayList<ERROR_CODE>(), errs);

From c45ecbc4e2793d9c020d8dfdef9c6c6951cf594c Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 9 Jun 2012 18:15:48 +0000
Subject: [PATCH 0271/1540] HBASE-5372. Table mutation operations should check
 table level rights (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1348467 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 31 ++++++++++++++++---
 .../security/access/TestAccessController.java | 13 ++++++++
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index d815be7975c6..4907097eafea 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -371,12 +371,33 @@ private User getActiveUser() throws IOException {
    * @throws IOException if obtaining the current user fails
    * @throws AccessDeniedException if authorization is denied
    */
-  private void requireTableAdminPermission(MasterCoprocessorEnvironment e,
-      byte[] tableName) throws IOException {
-    if (isActiveUserTableOwner(e, tableName)) {
-      requirePermission(Permission.Action.CREATE);
+  private void requireTableAdminPermission(MasterCoprocessorEnvironment e, byte[] tableName)
+      throws IOException {
+    User user = getActiveUser();
+    AuthResult result = null;
+
+    // Table admins are allowed to perform DDL
+    if (authManager.authorize(user, tableName, (byte[]) null, TablePermission.Action.ADMIN)) {
+      result = AuthResult.allow("Table permission granted", user, TablePermission.Action.ADMIN,
+          tableName);
+    } else if (isActiveUserTableOwner(e, tableName)) {
+      // Table owners with Create permission are allowed to perform DDL
+      if (authManager.authorize(user, tableName, (byte[]) null, TablePermission.Action.CREATE)) {
+        result = AuthResult.allow("Owner has table permission", user,
+            TablePermission.Action.CREATE, tableName);
+      } else {
+        // Table owners without Create permission cannot perform DDL
+        result = AuthResult.deny("Insufficient permissions", user, TablePermission.Action.CREATE,
+            tableName);
+      }
     } else {
-      requirePermission(Permission.Action.ADMIN);
+      // rest of the world
+      result = AuthResult.deny("Insufficient permissions", user, TablePermission.Action.ADMIN,
+          tableName);
+    }
+    logResult(result);
+    if (!result.isAllowed()) {
+      throw new AccessDeniedException("Insufficient permissions " + result.toContextString());
     }
   }
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 604fa7dc4631..16b11a035164 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -80,6 +80,8 @@ public class TestAccessController {
   private static User USER_RW;
   // user with read-only permissions
   private static User USER_RO;
+  // user with table admin permissions
+  private static User USER_TBLADM;
   // user with no permissions
   private static User USER_NONE;
 
@@ -110,6 +112,7 @@ public static void setupBeforeClass() throws Exception {
     USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
     USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
     USER_RO = User.createUserForTesting(conf, "rouser", new String[0]);
+    USER_TBLADM = User.createUserForTesting(conf, "tbladm", new String[0]);
     USER_NONE = User.createUserForTesting(conf, "nouser", new String[0]);
 
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
@@ -132,6 +135,9 @@ public static void setupBeforeClass() throws Exception {
 
     protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()),
                    TEST_TABLE, TEST_FAMILY, Permission.Action.READ));
+
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_TBLADM.getShortName()),
+      TEST_TABLE, null, Permission.Action.ADMIN));    
   }
 
   @AfterClass
@@ -232,6 +238,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, modifyTable);
     verifyAllowed(USER_ADMIN, modifyTable);
+    verifyAllowed(USER_TBLADM, modifyTable);
   }
 
   @Test
@@ -252,6 +259,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, deleteTable);
     verifyAllowed(USER_ADMIN, deleteTable);
+    verifyAllowed(USER_TBLADM, deleteTable);
   }
 
   @Test
@@ -273,6 +281,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
     verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(USER_TBLADM, action);
   }
 
   @Test
@@ -295,6 +304,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
     verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(USER_TBLADM, action);
   }
 
   @Test
@@ -315,6 +325,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, action);
     verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(USER_TBLADM, action);
   }
 
   @Test
@@ -335,6 +346,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, disableTable);
     verifyAllowed(USER_ADMIN, disableTable);
+    verifyAllowed(USER_TBLADM, disableTable);
   }
 
   @Test
@@ -355,6 +367,7 @@ public Object run() throws Exception {
     // verify that superuser can create tables
     verifyAllowed(SUPERUSER, enableTable);
     verifyAllowed(USER_ADMIN, enableTable);
+    verifyAllowed(USER_TBLADM, enableTable);
   }
 
   @Test

From 87fb0766fa7c779d2e869619b5bb50fed1ff372c Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 9 Jun 2012 18:21:13 +0000
Subject: [PATCH 0272/1540] HBASE-6157. Revoke of Global permission is not
 taking effect without restart (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1348469 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/TableAuthManager.java     |   8 +
 .../security/access/TestAccessController.java | 212 +++++++++---------
 2 files changed, 110 insertions(+), 110 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
index 970317bb3ec1..a40a57685094 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
@@ -121,6 +121,14 @@ public void refreshCacheFromWritable(byte[] table, byte[] data) throws IOExcepti
    * @param userPerms
    */
   private void updateGlobalCache(ListMultimap<String,TablePermission> userPerms) {
+    USER_CACHE.clear();
+    GROUP_CACHE.clear();
+    try {
+      initGlobal(conf);
+    } catch (IOException e) {
+      // Never happens
+      LOG.error("Error occured while updating the user cache", e);
+    }
     for (Map.Entry<String,TablePermission> entry : userPerms.entries()) {
       if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
         GROUP_CACHE.put(AccessControlLists.getGroupName(entry.getKey()),
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 16b11a035164..a6b615cc33d6 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -145,13 +145,14 @@ public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  public void verifyAllowed(User user, PrivilegedExceptionAction action)
+  public void verifyAllowed(User user, PrivilegedExceptionAction... actions)
     throws Exception {
-    try {
-      user.runAs(action);
-    } catch (AccessDeniedException ade) {
-      fail("Expected action to pass for user '" + user.getShortName() +
-          "' but was denied");
+    for (PrivilegedExceptionAction action : actions) {
+      try {
+        user.runAs(action);
+      } catch (AccessDeniedException ade) {
+        fail("Expected action to pass for user '" + user.getShortName() + "' but was denied");
+      }
     }
   }
 
@@ -162,28 +163,29 @@ public void verifyAllowed(PrivilegedExceptionAction action, User... users)
     }
   }
 
-  public void verifyDenied(User user, PrivilegedExceptionAction action)
+  public void verifyDenied(User user, PrivilegedExceptionAction... actions)
     throws Exception {
-    try {
-      user.runAs(action);
-      fail("Expected AccessDeniedException for user '" + user.getShortName() + "'");
-    } catch (RetriesExhaustedWithDetailsException e) {
-      // in case of batch operations, and put, the client assembles a
-      // RetriesExhaustedWithDetailsException instead of throwing an
-      // AccessDeniedException
-      boolean isAccessDeniedException = false;
-      for ( Throwable ex : e.getCauses()) {
-        if (ex instanceof AccessDeniedException) {
-          isAccessDeniedException = true;
-          break;
+    for (PrivilegedExceptionAction action : actions) {
+      try {
+        user.runAs(action);
+        fail("Expected AccessDeniedException for user '" + user.getShortName() + "'");
+      } catch (RetriesExhaustedWithDetailsException e) {
+        // in case of batch operations, and put, the client assembles a
+        // RetriesExhaustedWithDetailsException instead of throwing an
+        // AccessDeniedException
+        boolean isAccessDeniedException = false;
+        for (Throwable ex : e.getCauses()) {
+          if (ex instanceof AccessDeniedException) {
+            isAccessDeniedException = true;
+            break;
+          }
         }
+        if (!isAccessDeniedException) {
+          fail("Not receiving AccessDeniedException for user '" + user.getShortName() + "'");
+        }
+      } catch (AccessDeniedException ade) {
+        // expected result
       }
-      if (!isAccessDeniedException ) {
-        fail("Not receiving AccessDeniedException for user '" +
-            user.getShortName() + "'");
-      }
-    } catch (AccessDeniedException ade) {
-      // expected result
     }
   }
 
@@ -693,8 +695,8 @@ public void testGrantRevoke() throws Exception {
     admin.createTable(htd);
 
     // create temp users
-    User user = User.createUserForTesting(TEST_UTIL.getConfiguration(),
-        "user", new String[0]);
+    User tblUser = User.createUserForTesting(TEST_UTIL.getConfiguration(), "tbluser", new String[0]);
+    User gblUser = User.createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]);
 
     // perms only stored against the first region
     HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
@@ -789,120 +791,110 @@ public Object run() throws Exception {
     };
 
     // initial check:
-    verifyDenied(user, getActionAll);
-    verifyDenied(user, getAction1);
-    verifyDenied(user, getAction2);
-
-    verifyDenied(user, putActionAll);
-    verifyDenied(user, putAction1);
-    verifyDenied(user, putAction2);
+    verifyDenied(tblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(tblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2);
 
-    verifyDenied(user, deleteActionAll);
-    verifyDenied(user, deleteAction1);
-    verifyDenied(user, deleteAction2);
+    verifyDenied(gblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(gblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant table read permission
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, null, Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+        Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
+
     Thread.sleep(100);
     // check
-    verifyAllowed(user, getActionAll);
-    verifyAllowed(user, getAction1);
-    verifyAllowed(user, getAction2);
-
-    verifyDenied(user, putActionAll);
-    verifyDenied(user, putAction1);
-    verifyDenied(user, putAction2);
+    verifyAllowed(tblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(tblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2);
 
-    verifyDenied(user, deleteActionAll);
-    verifyDenied(user, deleteAction1);
-    verifyDenied(user, deleteAction2);
+    verifyAllowed(gblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(gblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant table write permission
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, null, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+        Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
+        Permission.Action.WRITE));
     Thread.sleep(100);
-    verifyDenied(user, getActionAll);
-    verifyDenied(user, getAction1);
-    verifyDenied(user, getAction2);
 
-    verifyAllowed(user, putActionAll);
-    verifyAllowed(user, putAction1);
-    verifyAllowed(user, putAction2);
+    verifyDenied(tblUser, getActionAll, getAction1, getAction2);
+    verifyAllowed(tblUser, putActionAll, putAction1, putAction2);
+    verifyAllowed(tblUser, deleteActionAll, deleteAction1, deleteAction2);
 
-    verifyAllowed(user, deleteActionAll);
-    verifyAllowed(user, deleteAction1);
-    verifyAllowed(user, deleteAction2);
+    verifyDenied(gblUser, getActionAll, getAction1, getAction2);
+    verifyAllowed(gblUser, putActionAll, putAction1, putAction2);
+    verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // revoke table permission
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, null, Permission.Action.READ, Permission.Action.WRITE));
-
-    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
-                    tableName, null));
+    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+        Permission.Action.READ, Permission.Action.WRITE));
+    protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null));
+    protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName())));
     Thread.sleep(100);
-    verifyDenied(user, getActionAll);
-    verifyDenied(user, getAction1);
-    verifyDenied(user, getAction2);
 
-    verifyDenied(user, putActionAll);
-    verifyDenied(user, putAction1);
-    verifyDenied(user, putAction2);
+    verifyDenied(tblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(tblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2);
 
-    verifyDenied(user, deleteActionAll);
-    verifyDenied(user, deleteAction1);
-    verifyDenied(user, deleteAction2);
+    verifyDenied(gblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(gblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant column family read permission
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, family1, Permission.Action.READ));
-    Thread.sleep(100);
+    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family1,
+        Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
 
-    verifyAllowed(user, getActionAll);
-    verifyAllowed(user, getAction1);
-    verifyDenied(user, getAction2);
+    Thread.sleep(100);
 
-    verifyDenied(user, putActionAll);
-    verifyDenied(user, putAction1);
-    verifyDenied(user, putAction2);
+    // Access should be denied for family2
+    verifyAllowed(tblUser, getActionAll, getAction1);
+    verifyDenied(tblUser, getAction2);
+    verifyDenied(tblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2);
 
-    verifyDenied(user, deleteActionAll);
-    verifyDenied(user, deleteAction1);
-    verifyDenied(user, deleteAction2);
+    verifyAllowed(gblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(gblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant column family write permission
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, family2, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2,
+        Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
+        Permission.Action.WRITE));
     Thread.sleep(100);
 
-    verifyAllowed(user, getActionAll);
-    verifyAllowed(user, getAction1);
-    verifyDenied(user, getAction2);
-
-    verifyDenied(user, putActionAll);
-    verifyDenied(user, putAction1);
-    verifyAllowed(user, putAction2);
+    // READ from family1, WRITE to family2 are allowed
+    verifyAllowed(tblUser, getActionAll, getAction1);
+    verifyAllowed(tblUser, putAction2, deleteAction2);
+    verifyDenied(tblUser, getAction2);
+    verifyDenied(tblUser, putActionAll, putAction1);
+    verifyDenied(tblUser, deleteActionAll, deleteAction1);
 
-    verifyDenied(user, deleteActionAll);
-    verifyDenied(user, deleteAction1);
-    verifyAllowed(user, deleteAction2);
+    verifyDenied(gblUser, getActionAll, getAction1, getAction2);
+    verifyAllowed(gblUser, putActionAll, putAction1, putAction2);
+    verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // revoke column family permission
-    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
-                    tableName, family2));
-    Thread.sleep(100);
+    protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2));
+    protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName())));
 
-    verifyAllowed(user, getActionAll);
-    verifyAllowed(user, getAction1);
-    verifyDenied(user, getAction2);
+    Thread.sleep(100);
 
-    verifyDenied(user, putActionAll);
-    verifyDenied(user, putAction1);
-    verifyDenied(user, putAction2);
+    // Revoke on family2 should not have impact on family1 permissions
+    verifyAllowed(tblUser, getActionAll, getAction1);
+    verifyDenied(tblUser, getAction2);
+    verifyDenied(tblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2);
 
-    verifyDenied(user, deleteActionAll);
-    verifyDenied(user, deleteAction1);
-    verifyDenied(user, deleteAction2);
+    // Should not have access as global permissions are completely revoked
+    verifyDenied(gblUser, getActionAll, getAction1, getAction2);
+    verifyDenied(gblUser, putActionAll, putAction1, putAction2);
+    verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // delete table
     admin.disableTable(tableName);

From 4f1ed002df0bec62bafe7fe97053a336a76168ac Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 13 Jun 2012 00:19:31 +0000
Subject: [PATCH 0273/1540] HBASE-6092. Authorize flush, split, compact
 operations in AccessController (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1349597 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 31 ++++++++--
 .../security/access/TestAccessController.java | 58 +++++++++++++++++++
 .../hbase/coprocessor/BaseRegionObserver.java | 21 ++++---
 .../hbase/coprocessor/RegionObserver.java     | 20 ++++---
 .../regionserver/RegionCoprocessorHost.java   | 30 ++++++----
 5 files changed, 127 insertions(+), 33 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 4907097eafea..35d1da0c96d7 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -53,11 +53,14 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.MapMaker;
@@ -366,12 +369,12 @@ private User getActiveUser() throws IOException {
    * that means he/she can edit/modify/delete the table.
    * If current user is the table owner, and has CREATE permission,
    * then he/she has table admin permission. otherwise ADMIN rights are checked.
-   * @param e Master coprocessor environment
+   * @param e Coprocessor environment
    * @param tableName Table requested
    * @throws IOException if obtaining the current user fails
    * @throws AccessDeniedException if authorization is denied
    */
-  private void requireTableAdminPermission(MasterCoprocessorEnvironment e, byte[] tableName)
+  private void requireTableAdminPermission(CoprocessorEnvironment e, byte[] tableName)
       throws IOException {
     User user = getActiveUser();
     AuthResult result = null;
@@ -720,6 +723,23 @@ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
     }
   }
 
+  @Override
+  public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+    requireTableAdminPermission(e.getEnvironment(), getTableName(e.getEnvironment()));
+  }
+
+  @Override
+  public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+    requireTableAdminPermission(e.getEnvironment(), getTableName(e.getEnvironment()));
+  }
+
+  @Override
+  public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
+      final Store store, final InternalScanner scanner) throws IOException {
+    requireTableAdminPermission(e.getEnvironment(), getTableName(e.getEnvironment()));
+    return scanner;
+  }
+
   @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> c,
       final byte [] row, final byte [] family, final Result result)
@@ -1088,14 +1108,13 @@ private byte[] getTableName(RegionCoprocessorEnvironment e) {
     return tableName;
   }
 
-  private String getTableOwner(MasterCoprocessorEnvironment e,
-      byte[] tableName) throws IOException {
+  private String getTableOwner(CoprocessorEnvironment e, byte[] tableName) throws IOException {
     HTableDescriptor htd = e.getTable(tableName).getTableDescriptor();
     return htd.getOwnerString();
   }
 
-  private boolean isActiveUserTableOwner(MasterCoprocessorEnvironment e,
-      byte[] tableName) throws IOException {
+  private boolean isActiveUserTableOwner(CoprocessorEnvironment e, byte[] tableName)
+      throws IOException {
     String activeUser = getActiveUser().getShortName();
     return activeUser.equals(getTableOwner(e, tableName));
   }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index a6b615cc33d6..2da7c7197b75 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -31,6 +31,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -51,7 +52,10 @@
 import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -89,6 +93,7 @@ public class TestAccessController {
   private static byte[] TEST_FAMILY = Bytes.toBytes("f1");
 
   private static MasterCoprocessorEnvironment CP_ENV;
+  private static RegionCoprocessorEnvironment RCP_ENV;
   private static AccessController ACCESS_CONTROLLER;
 
   @BeforeClass
@@ -126,6 +131,11 @@ public static void setupBeforeClass() throws Exception {
     AccessControllerProtocol protocol =
         meta.coprocessorProxy(AccessControllerProtocol.class, TEST_TABLE);
 
+    HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0);
+    RegionCoprocessorHost rcpHost = region.getCoprocessorHost();
+    RCP_ENV = rcpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER,
+        Coprocessor.PRIORITY_HIGHEST, 1, conf);
+
     protocol.grant(new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()),
                       Permission.Action.ADMIN, Permission.Action.CREATE,
                       Permission.Action.READ, Permission.Action.WRITE));
@@ -542,6 +552,54 @@ private void verifyWrite(PrivilegedExceptionAction action) throws Exception {
     verifyAllowed(USER_RW, action);
   }
 
+  @Test
+  public void testSplit() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preSplit(ObserverContext.createAndPrepare(RCP_ENV, null));
+        return null;
+      }
+    };
+
+    // verify that superuser and admin only can split
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_TBLADM);
+
+    // all others should be denied
+    verifyDenied(action, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+  }
+
+  @Test
+  public void testFlush() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preFlush(ObserverContext.createAndPrepare(RCP_ENV, null));
+        return null;
+      }
+    };
+
+    // verify that superuser and admin only can flush
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_TBLADM);
+
+    // all others should be denied
+    verifyDenied(action, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+  }
+
+  @Test
+  public void testCompact() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preCompact(ObserverContext.createAndPrepare(RCP_ENV, null), null, null);
+        return null;
+      }
+    };
+
+    // verify that superuser and admin only can compact
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_TBLADM);
+
+    // all others should be denied
+    verifyDenied(action, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+  }
+
   private void verifyRead(PrivilegedExceptionAction action) throws Exception {
     // should be denied
     verifyDenied(USER_NONE, action);
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 7a7b89621682..e6043d856d7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -69,17 +69,21 @@ public void postClose(ObserverContext<RegionCoprocessorEnvironment> e,
       boolean abortRequested) { }
 
   @Override
-  public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) { }
+  public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+  }
 
   @Override
-  public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) { }
+  public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+  }
 
   @Override
-  public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) { }
+  public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+  }
 
   @Override
-  public void postSplit(ObserverContext<RegionCoprocessorEnvironment> e,
-      HRegion l, HRegion r) { }
+  public void postSplit(ObserverContext<RegionCoprocessorEnvironment> e, HRegion l, HRegion r)
+      throws IOException {
+  }
 
   @Override
   public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -91,13 +95,14 @@ public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironm
 
   @Override
   public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
-      final Store store, final InternalScanner scanner) {
+      final Store store, final InternalScanner scanner) throws IOException {
     return scanner;
   }
 
   @Override
-  public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
-      final Store store, final StoreFile resultFile) { }
+  public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
+      final StoreFile resultFile) throws IOException {
+  }
 
   @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> e,
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index c0a418427f4e..0e6310946aad 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -62,14 +62,16 @@ public interface RegionObserver extends Coprocessor {
   /**
    * Called before the memstore is flushed to disk.
    * @param c the environment provided by the region server
+   * @throws IOException if an error occurred on the coprocessor
    */
-  void preFlush(final ObserverContext<RegionCoprocessorEnvironment> c);
+  void preFlush(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
 
   /**
    * Called after the memstore is flushed to disk.
    * @param c the environment provided by the region server
+   * @throws IOException if an error occurred on the coprocessor
    */
-  void postFlush(final ObserverContext<RegionCoprocessorEnvironment> c);
+  void postFlush(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
 
   /**
    * Called prior to selecting the {@link StoreFile}s to compact from the list
@@ -114,9 +116,10 @@ void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
    * rewriting
    * @return the scanner to use during compaction.  Should not be {@code null}
    * unless the implementation is writing new store files on its own.
+   * @throws IOException if an error occurred on the coprocessor
    */
   InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
-    final Store store, final InternalScanner scanner);
+      final Store store, final InternalScanner scanner) throws IOException;
 
   /**
    * Called after compaction has completed and the new store file has been
@@ -124,16 +127,18 @@ InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c
    * @param c the environment provided by the region server
    * @param store the store being compacted
    * @param resultFile the new store file written out during compaction
+   * @throws IOException if an error occurred on the coprocessor
    */
-  void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
-    final Store store, StoreFile resultFile);
+  void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      StoreFile resultFile) throws IOException;
 
   /**
    * Called before the region is split.
    * @param c the environment provided by the region server
    * (e.getRegion() returns the parent region)
+   * @throws IOException if an error occurred on the coprocessor
    */
-  void preSplit(final ObserverContext<RegionCoprocessorEnvironment> c);
+  void preSplit(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
 
   /**
    * Called after the region is split.
@@ -141,9 +146,10 @@ void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
    * (e.getRegion() returns the parent region)
    * @param l the left daughter region
    * @param r the right daughter region
+   * @throws IOException if an error occurred on the coprocessor
    */
   void postSplit(final ObserverContext<RegionCoprocessorEnvironment> c, final HRegion l,
-    final HRegion r);
+      final HRegion r) throws IOException;
 
   /**
    * Called before the region is reported as closed to the master.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index a3850e5b09d7..44db31db394d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -355,8 +355,9 @@ public void postCompactSelection(Store store,
    * Called prior to rewriting the store files selected for compaction
    * @param store the store being compacted
    * @param scanner the scanner used to read store data during compaction
+   * @throws IOException 
    */
-  public InternalScanner preCompact(Store store, InternalScanner scanner) {
+  public InternalScanner preCompact(Store store, InternalScanner scanner) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     boolean bypass = false;
     for (RegionEnvironment env: coprocessors) {
@@ -366,7 +367,7 @@ public InternalScanner preCompact(Store store, InternalScanner scanner) {
           scanner = ((RegionObserver)env.getInstance()).preCompact(
               ctx, store, scanner);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env,e);
+          handleCoprocessorThrowable(env,e);
         }
         bypass |= ctx.shouldBypass();
         if (ctx.shouldComplete()) {
@@ -381,8 +382,9 @@ public InternalScanner preCompact(Store store, InternalScanner scanner) {
    * Called after the store compaction has completed.
    * @param store the store being compacted
    * @param resultFile the new store file written during compaction
+   * @throws IOException 
    */
-  public void postCompact(Store store, StoreFile resultFile) {
+  public void postCompact(Store store, StoreFile resultFile) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -390,7 +392,7 @@ public void postCompact(Store store, StoreFile resultFile) {
         try {
           ((RegionObserver)env.getInstance()).postCompact(ctx, store, resultFile);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env, e);
+          handleCoprocessorThrowable(env, e);
         }
         if (ctx.shouldComplete()) {
           break;
@@ -401,8 +403,9 @@ public void postCompact(Store store, StoreFile resultFile) {
 
   /**
    * Invoked before a memstore flush
+   * @throws IOException 
    */
-  public void preFlush() {
+  public void preFlush() throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -410,7 +413,7 @@ public void preFlush() {
         try {
           ((RegionObserver)env.getInstance()).preFlush(ctx);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env, e);
+          handleCoprocessorThrowable(env, e);
         }
         if (ctx.shouldComplete()) {
           break;
@@ -421,8 +424,9 @@ public void preFlush() {
 
   /**
    * Invoked after a memstore flush
+   * @throws IOException 
    */
-  public void postFlush() {
+  public void postFlush() throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -430,7 +434,7 @@ public void postFlush() {
         try {
           ((RegionObserver)env.getInstance()).postFlush(ctx);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env, e);
+          handleCoprocessorThrowable(env, e);
         }
         if (ctx.shouldComplete()) {
           break;
@@ -441,8 +445,9 @@ public void postFlush() {
 
   /**
    * Invoked just before a split
+   * @throws IOException 
    */
-  public void preSplit() {
+  public void preSplit() throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -450,7 +455,7 @@ public void preSplit() {
         try {
           ((RegionObserver)env.getInstance()).preSplit(ctx);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env, e);
+          handleCoprocessorThrowable(env, e);
         }
         if (ctx.shouldComplete()) {
           break;
@@ -463,8 +468,9 @@ public void preSplit() {
    * Invoked just after a split
    * @param l the new left-hand daughter region
    * @param r the new right-hand daughter region
+   * @throws IOException 
    */
-  public void postSplit(HRegion l, HRegion r) {
+  public void postSplit(HRegion l, HRegion r) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -472,7 +478,7 @@ public void postSplit(HRegion l, HRegion r) {
         try {
           ((RegionObserver)env.getInstance()).postSplit(ctx, l, r);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env, e);
+          handleCoprocessorThrowable(env, e);
         }
         if (ctx.shouldComplete()) {
           break;

From 225ccfe2964b20e2b72e27b86a2c5fefc0c8b0e6 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 13 Jun 2012 03:28:35 +0000
Subject: [PATCH 0274/1540] HBASE-6195 Increment data will be lost when the
 memstore is flushed (Xing Shi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1349627 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 22 +++--
 .../hbase/regionserver/TestHRegion.java       | 93 +++++++++++++++++++
 2 files changed, 107 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index d0a858d406f9..ca04430736ba 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4479,8 +4479,8 @@ public Result increment(Increment increment, Integer lockid,
     boolean flush = false;
     WALEdit walEdits = null;
     List<KeyValue> allKVs = new ArrayList<KeyValue>(increment.numColumns());
-    List<KeyValue> kvs = new ArrayList<KeyValue>(increment.numColumns());
-    long now = EnvironmentEdgeManager.currentTimeMillis();
+    Map<Store, List<KeyValue>> tempMemstore = new HashMap<Store, List<KeyValue>>();
+    long before = EnvironmentEdgeManager.currentTimeMillis();
     long size = 0;
     long txid = 0;
 
@@ -4491,11 +4491,13 @@ public Result increment(Increment increment, Integer lockid,
       Integer lid = getLock(lockid, row, true);
       this.updatesLock.readLock().lock();
       try {
+        long now = EnvironmentEdgeManager.currentTimeMillis();
         // Process each family
         for (Map.Entry<byte [], NavigableMap<byte [], Long>> family :
           increment.getFamilyMap().entrySet()) {
 
           Store store = stores.get(family.getKey());
+          List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());
 
           // Get previous values for all columns in this family
           Get get = new Get(row);
@@ -4531,10 +4533,8 @@ public Result increment(Increment increment, Integer lockid,
             }
           }
 
-          // Write the KVs for this family into the store
-          size += store.upsert(kvs);
-          allKVs.addAll(kvs);
-          kvs.clear();
+          //store the kvs to the temporary memstore before writing HLog
+          tempMemstore.put(store, kvs);
         }
 
         // Actually write to WAL now
@@ -4543,10 +4543,16 @@ public Result increment(Increment increment, Integer lockid,
           // cluster. A slave cluster receives the final value (not the delta)
           // as a Put.
           txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
-              walEdits, HConstants.DEFAULT_CLUSTER_ID, now,
+              walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
               this.htableDescriptor);
         }
 
+        //Actually write to Memstore now
+        for (Map.Entry<Store, List<KeyValue>> entry : tempMemstore.entrySet()) {
+          Store store = entry.getKey();
+          size += store.upsert(entry.getValue());
+          allKVs.addAll(entry.getValue());
+        }
         size = this.addAndGetGlobalMemstoreSize(size);
         flush = isFlushSize(size);
       } finally {
@@ -4561,7 +4567,7 @@ public Result increment(Increment increment, Integer lockid,
     }
     
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - now);
+    this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
 
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index a4e225561923..0d2e012c7703 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -3501,6 +3502,98 @@ public void testDeleteRowWithBloomFilter() throws IOException {
         }
       }
   }
+  
+  /**
+   * TestCase for increment
+   *
+   */
+  private static class Incrementer implements Runnable {
+    private HRegion region;
+    private final static byte[] incRow = Bytes.toBytes("incRow");
+    private final static byte[] family = Bytes.toBytes("family");
+    private final static byte[] qualifier = Bytes.toBytes("qualifier");
+    private final static long ONE = 1l;
+    private int incCounter;
+
+    public Incrementer(HRegion region, int incCounter) {
+      this.region = region;
+      this.incCounter = incCounter;
+    }
+
+    @Override
+    public void run() {
+      int count = 0;
+      while (count < incCounter) {
+        Increment inc = new Increment(incRow);
+        inc.addColumn(family, qualifier, ONE);
+        count++;
+        try {
+          region.increment(inc, null, true);
+        } catch (IOException e) {
+          e.printStackTrace();
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Test case to check increment function with memstore flushing
+   * @throws Exception
+   */
+  @Test
+  public void testParallelIncrementWithMemStoreFlush() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String method = "testParallelIncrementWithMemStoreFlush";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Incrementer.family;
+    this.region = initHRegion(tableName, method, conf, family);
+    final HRegion region = this.region;
+    final AtomicBoolean incrementDone = new AtomicBoolean(false);
+    Runnable reader = new Runnable() {
+      @Override
+      public void run() {
+        while (!incrementDone.get()) {
+          try {
+            region.flushcache();
+          } catch (Exception e) {
+            e.printStackTrace();
+          }
+        }
+      }
+    };
+
+    //after all increment finished, the row will increment to 20*100 = 2000
+    int threadNum = 20;
+    int incCounter = 100;
+    long expected = threadNum * incCounter;
+    Thread[] incrementers = new Thread[threadNum];
+    Thread flushThread = new Thread(reader);
+    for (int i = 0; i < threadNum; i++) {
+      incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
+      incrementers[i].start();
+    }
+    flushThread.start();
+    for (int i = 0; i < threadNum; i++) {
+      incrementers[i].join();
+    }
+
+    incrementDone.set(true);
+    flushThread.join();
+
+    Get get = new Get(Incrementer.incRow);
+    get.addColumn(Incrementer.family, Incrementer.qualifier);
+    get.setMaxVersions(1);
+    Result res = this.region.get(get, null);
+    List<KeyValue> kvs = res.getColumn(Incrementer.family,
+        Incrementer.qualifier);
+    
+    //we just got the latest version
+    assertEquals(kvs.size(), 1);
+    KeyValue kv = kvs.get(0);
+    assertEquals(expected, Bytes.toLong(kv.getBuffer(), kv.getValueOffset()));
+    this.region = null;
+  }
 
   private void putData(int startRow, int numRows, byte [] qf,
       byte [] ...families)

From ab8ed3d5b9a582fb4c4b9926283805ac9d65fa82 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 14 Jun 2012 20:33:41 +0000
Subject: [PATCH 0275/1540] HBASE-6185 Update javadoc for
 ConstantSizeRegionSplitPolicy class

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1350389 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ConstantSizeRegionSplitPolicy.java    | 5 ++++-
 .../apache/hadoop/hbase/regionserver/RegionSplitPolicy.java  | 4 +++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index e0c27f099e4d..2098c55cc6dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -23,7 +23,10 @@
  * A {@link RegionSplitPolicy} implementation which splits a region
  * as soon as any of its store files exceeds a maximum configurable
  * size.
- * <p>This is the default split policy.</p>
+ * <p>
+ * This is the default split policy. From 0.94.0 on the default split policy has
+ * changed to {@link IncreasingToUpperBoundRegionSplitPolicy}
+ * </p>
  */
 public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
   private long desiredMaxFileSize;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
index 9957abdcee72..a526544e31f6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
@@ -30,7 +30,9 @@
 
 /**
  * A split policy determines when a region should be split.
- * {@see ConstantSizeRegionSplitPolicy}
+ * @see IncreasingToUpperBoundRegionSplitPolicy Default split policy since
+ *      0.94.0
+ * @see ConstantSizeRegionSplitPolicy Default split policy before 0.94.0
  */
 public abstract class RegionSplitPolicy extends Configured {
   private static final Class<? extends RegionSplitPolicy>

From 1e458fc8458a1deaafef8cc1fb8f10b8f57e71f6 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 16 Jun 2012 02:57:57 +0000
Subject: [PATCH 0276/1540] HBASE-5838. Add an LZ4 compression option to HFile

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1350845 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/Compression.java    | 21 ++++++++++++++++++-
 .../hadoop/hbase/io/hfile/TestHFile.java      |  4 +++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
index b64bbe95a36f..45f8c89793e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
@@ -177,7 +177,26 @@ CompressionCodec getCodec(Configuration conf) {
           }
           return snappyCodec;
         }
-    };
+    },
+    LZ4("lz4") {
+      // Use base type to avoid compile-time dependencies.
+      private transient CompressionCodec lz4Codec;
+
+      @Override
+      CompressionCodec getCodec(Configuration conf) {
+        if (lz4Codec == null) {
+          try {
+            Class<?> externalCodec =
+                getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.Lz4Codec");
+            lz4Codec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
+                conf);
+          } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+          }
+        }
+        return lz4Codec;
+      }
+  };
 
     private final Configuration conf;
     private final String compressName;
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
index bb992b8a66d0..ca2941613934 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
@@ -280,12 +280,14 @@ public void testNullMetaBlocks() throws Exception {
   }
 
   /**
-   * Make sure the orginals for our compression libs doesn't change on us.
+   * Make sure the ordinals for our compression algorithms do not change on us.
    */
   public void testCompressionOrdinance() {
     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
+    assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
+    assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
   }
 
   public void testComparator() throws IOException {

From c7c582ce44236da78a62b2f70ce2b942d9b2d162 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sun, 17 Jun 2012 05:40:03 +0000
Subject: [PATCH 0277/1540] HBASE-6214 Backport HBASE-5998 to 94.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1351052 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 3f28b7ac12f9..d7a779d40e86 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2697,10 +2697,16 @@ public RegionOpeningState openRegion(HRegionInfo region)
   throws IOException {
     return openRegion(region, -1);
   }
+
   @Override
   @QosPriority(priority = HIGH_QOS)
   public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode)
       throws IOException {
+    return openRegion(region, versionOfOfflineNode, null);
+  }
+
+  private RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode,
+      Map<String, HTableDescriptor> htds) throws IOException {
     checkOpen();
     checkIfRegionInTransition(region, OPEN);
     HRegion onlineRegion = this.getFromOnlineRegions(region.getEncodedName());
@@ -2721,7 +2727,16 @@ public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNod
     }
     LOG.info("Received request to open region: " +
       region.getRegionNameAsString());
-    HTableDescriptor htd = this.tableDescriptors.get(region.getTableName());
+    HTableDescriptor htd = null;
+    if (htds == null) {
+      htd = this.tableDescriptors.get(region.getTableName());
+    } else {
+      htd = htds.get(region.getTableNameAsString());
+      if (htd == null) {
+        htd = this.tableDescriptors.get(region.getTableName());
+        htds.put(region.getRegionNameAsString(), htd);
+      }
+    }
     this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(),
         true);
     // Need to pass the expected version in the constructor.
@@ -2757,7 +2772,8 @@ public void openRegions(List<HRegionInfo> regions)
   throws IOException {
     checkOpen();
     LOG.info("Received request to open " + regions.size() + " region(s)");
-    for (HRegionInfo region: regions) openRegion(region);
+    Map<String, HTableDescriptor> htds = new HashMap<String, HTableDescriptor>(regions.size());
+    for (HRegionInfo region : regions) openRegion(region, -1, htds);
   }
 
   @Override

From 1fad1ac2f445ed59782bcc7daa5609af917b923f Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sun, 17 Jun 2012 23:29:16 +0000
Subject: [PATCH 0278/1540] HBASE-5360 [uberhbck] Add options for how to handle
 offline split parents.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1351182 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                   |  1 +
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 51 +++++++++++-
 .../hadoop/hbase/util/TestHBaseFsck.java      | 83 ++++++++++++++++++-
 3 files changed, 130 insertions(+), 5 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index aa106b739af6..82e640293227 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -375,6 +375,7 @@ Improvement
     [HBASE-5862] - After Region Close remove the Operation Metrics.
     [HBASE-5863] - Improve the graceful_stop.sh CLI help (especially about reloads)
     [HBASE-6173] - hbck check specified tables only
+    [HBASE-5360] - [uberhbck] Add options for how to handle offline split parents. 
 
 New Feature
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index d1562b08ff72..ef2d13a123f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -71,6 +71,7 @@
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -167,6 +168,7 @@ public class HBaseFsck {
   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
+  private boolean fixSplitParents = false; // fix lingering split parents
 
   // limit checking/fixes to listed tables, if empty attempt to check/fix all
   // -ROOT- and .META. are always checked
@@ -1180,6 +1182,29 @@ private void deleteMetaRegion(HbckInfo hi) throws IOException {
     LOG.info("Deleted " + hi.metaEntry.getRegionNameAsString() + " from META" );
   }
 
+  /**
+   * Reset the split parent region info in meta table
+   */
+  private void resetSplitParent(HbckInfo hi) throws IOException {
+    RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
+    Delete d = new Delete(hi.metaEntry.getRegionName());
+    d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
+    d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
+    mutations.add(d);
+
+    Put p = new Put(hi.metaEntry.getRegionName());
+    HRegionInfo hri = new HRegionInfo(hi.metaEntry);
+    hri.setOffline(false);
+    hri.setSplit(false);
+    p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+      Writables.getBytes(hri));
+    mutations.add(p);
+
+    meta.mutateRow(mutations);
+    meta.flushCommits();
+    LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
+  }
+
   /**
    * This backwards-compatibility wrapper for permanently offlining a region
    * that should not be alive.  If the region server does not support the
@@ -1319,9 +1344,6 @@ private void checkRegionConsistency(final String key, final HbckInfo hbi)
     }
     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
       return;
-    } else if (inMeta && inHdfs && !isDeployed && splitParent) {
-      LOG.warn("Region " + descriptiveName + " is a split parent in META and in HDFS");
-      return;
     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
         "tabled that is not deployed");
@@ -1378,6 +1400,14 @@ else if (!inMeta && !inHdfs && !isDeployed) {
       }
 
     // ========== Cases where the region is in META =============
+    } else if (inMeta && inHdfs && !isDeployed && splitParent) {
+      errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+          + descriptiveName + " is a split parent in META, in HDFS, "
+          + "and not deployed on any region server. This could be transient.");
+      if (shouldFixSplitParents()) {
+        setShouldRerun();
+        resetSplitParent(hbi);
+      }
     } else if (inMeta && !inHdfs && !isDeployed) {
       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
           + descriptiveName + " found in META, but not in HDFS "
@@ -2504,7 +2534,7 @@ public static enum ERROR_CODE {
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
-      ORPHAN_HDFS_REGION
+      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT
     }
     public void clear();
     public void report(String message);
@@ -2906,6 +2936,14 @@ public boolean shouldSidelineBigOverlaps() {
     return sidelineBigOverlaps;
   }
 
+  public void setFixSplitParents(boolean shouldFix) {
+    fixSplitParents = shouldFix;
+  }
+
+  boolean shouldFixSplitParents() {
+    return fixSplitParents;
+  }
+
   /**
    * @param mm maximum number of regions to merge into a single region.
    */
@@ -2970,6 +3008,7 @@ protected static void printUsageAndExit() {
     System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
     System.err.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
     System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
+    System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
     System.err.println("");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
@@ -3044,6 +3083,8 @@ public static void main(String[] args) throws Exception {
         fsck.setFixVersionFile(true);
       } else if (cmd.equals("-sidelineBigOverlaps")) {
         fsck.setSidelineBigOverlaps(true);
+      } else if (cmd.equals("-fixSplitParents")) {
+        fsck.setFixSplitParents(true);
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
@@ -3054,6 +3095,7 @@ public static void main(String[] args) throws Exception {
         fsck.setFixHdfsOverlaps(true);
         fsck.setFixVersionFile(true);
         fsck.setSidelineBigOverlaps(true);
+        fsck.setFixSplitParents(false);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
         fsck.setFixHdfsHoles(true);
@@ -3062,6 +3104,7 @@ public static void main(String[] args) throws Exception {
         fsck.setFixAssignments(true);
         fsck.setFixHdfsOverlaps(false);
         fsck.setSidelineBigOverlaps(false);
+        fsck.setFixSplitParents(false);
       } else if (cmd.equals("-maxOverlapsToSideline")) {
         if (i == args.length - 1) {
           System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 02e4c027fcbe..2e87c2704b99 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -45,11 +45,13 @@
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTable;
@@ -238,7 +240,7 @@ private void deleteRegion(Configuration conf, final HTableDescriptor htd,
 
         if (unassign) {
           LOG.info("Undeploying region " + hri + " from server " + hsa);
-          undeployRegion(new HBaseAdmin(conf), hsa, hri);
+          undeployRegion(new HBaseAdmin(conf), hsa, new HRegionInfo(hri));
         }
 
         if (regionInfoOnly) {
@@ -1011,6 +1013,85 @@ public void testFixByTable() throws Exception {
       deleteTable(table2);
     }
   }
+  /**
+   * A split parent in meta, in hdfs, and not deployed
+   */
+  @Test
+  public void testLingeringSplitParent() throws Exception {
+    String table = "testLingeringSplitParent";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table);
+      HRegionLocation location = tbl.getRegionLocation("B");
+
+      // Delete one region from meta, but not hdfs, unassign it.
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+        Bytes.toBytes("C"), true, true, false);
+
+      // Create a new meta entry to fake it as a split parent.
+      HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
+      HRegionInfo hri = location.getRegionInfo();
+
+      HRegionInfo a = new HRegionInfo(tbl.getTableName(),
+        Bytes.toBytes("B"), Bytes.toBytes("BM"));
+      HRegionInfo b = new HRegionInfo(tbl.getTableName(),
+        Bytes.toBytes("BM"), Bytes.toBytes("C"));
+      Put p = new Put(hri.getRegionName());
+      hri.setOffline(true);
+      hri.setSplit(true);
+      p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(hri));
+      p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
+        Writables.getBytes(a));
+      p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
+        Writables.getBytes(b));
+      meta.put(p);
+      meta.flushCommits();
+      TEST_UTIL.getHBaseAdmin().flush(HConstants.META_TABLE_NAME);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // regular repair cannot fix lingering split parent
+      hbck = doFsck(conf, true);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+      assertFalse(hbck.shouldRerun());
+      hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // fix lingering split parent
+      hbck = new HBaseFsck(conf);
+      hbck.connect();
+      hbck.setDisplayFullReport(); // i.e. -details
+      hbck.setTimeLag(0);
+      hbck.setFixSplitParents(true);
+      hbck.onlineHbck();
+      assertTrue(hbck.shouldRerun());
+
+      Get get = new Get(hri.getRegionName());
+      Result result = meta.get(get);
+      assertTrue(result.getColumn(HConstants.CATALOG_FAMILY,
+        HConstants.SPLITA_QUALIFIER).isEmpty());
+      assertTrue(result.getColumn(HConstants.CATALOG_FAMILY,
+        HConstants.SPLITB_QUALIFIER).isEmpty());
+      TEST_UTIL.getHBaseAdmin().flush(HConstants.META_TABLE_NAME);
+
+      // fix other issues
+      doFsck(conf, true);
+
+      // check that all are fixed
+      assertNoErrors(doFsck(conf, false));
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+      deleteTable(table);
+    }
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From c2fd92499b2d6aecb7a6343012a8c260cca3a9f8 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 19 Jun 2012 02:24:43 +0000
Subject: [PATCH 0279/1540] HBASE-6188. Remove the concept of table owner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1351557 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessControlLists.java   |   7 +
 .../security/access/AccessController.java     | 119 ++--
 .../security/access/TestAccessController.java | 544 +++++++-----------
 .../apache/hadoop/hbase/HTableDescriptor.java |   6 +-
 4 files changed, 263 insertions(+), 413 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
index 8a100e4fe5df..53181c1aace5 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
@@ -287,6 +287,13 @@ static boolean isAclRegion(HRegion region) {
     return Bytes.equals(ACL_TABLE_NAME, region.getTableDesc().getName());
   }
 
+  /**
+   * Returns {@code true} if the given table is {@code _acl_} metadata table.
+   */
+  static boolean isAclTable(HTableDescriptor desc) {
+    return Bytes.equals(ACL_TABLE_NAME, desc.getName());
+  }
+
   /**
    * Loads all of the permission grants stored in a region of the {@code _acl_}
    * table.
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 35d1da0c96d7..8d0679db81d1 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -18,10 +18,10 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.TreeSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -54,13 +54,12 @@
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
 import org.apache.hadoop.hbase.util.Bytes;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.MapMaker;
@@ -250,7 +249,6 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
       RegionCoprocessorEnvironment e,
       Map<byte [], ? extends Collection<?>> families) {
     HRegionInfo hri = e.getRegion().getRegionInfo();
-    HTableDescriptor htd = e.getRegion().getTableDesc();
     byte[] tableName = hri.getTableName();
 
     // 1. All users need read access to .META. and -ROOT- tables.
@@ -279,19 +277,12 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
        return AuthResult.allow("Table permission granted", user, permRequest, tableName);
     }
 
-    // 2. The table owner has full privileges
-    String owner = htd.getOwnerString();
-    if (user.getShortName().equals(owner)) {
-      // owner of the table has full access
-      return AuthResult.allow("User is table owner", user, permRequest, tableName);
-    }
-
-    // 3. check for the table-level, if successful we can short-circuit
+    // 2. check for the table-level, if successful we can short-circuit
     if (authManager.authorize(user, tableName, (byte[])null, permRequest)) {
       return AuthResult.allow("Table permission granted", user, permRequest, tableName);
     }
 
-    // 4. check permissions against the requested families
+    // 3. check permissions against the requested families
     if (families != null && families.size() > 0) {
       // all families must pass
       for (Map.Entry<byte [], ? extends Collection<?>> family : families.entrySet()) {
@@ -335,7 +326,7 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
           tableName);
     }
 
-    // 5. no families to check and table level access failed
+    // 4. no families to check and table level access failed
     return AuthResult.deny("No families to check and table permission failed",
         user, permRequest, tableName);
   }
@@ -365,38 +356,23 @@ private User getActiveUser() throws IOException {
   }
 
   /**
-   * Authorizes that the current user has "admin" privileges for the given table.
-   * that means he/she can edit/modify/delete the table.
-   * If current user is the table owner, and has CREATE permission,
-   * then he/she has table admin permission. otherwise ADMIN rights are checked.
-   * @param e Coprocessor environment
+   * Authorizes that the current user has any of the given permissions for the given table.
    * @param tableName Table requested
    * @throws IOException if obtaining the current user fails
-   * @throws AccessDeniedException if authorization is denied
+   * @throws AccessDeniedException if user has no authorization
    */
-  private void requireTableAdminPermission(CoprocessorEnvironment e, byte[] tableName)
-      throws IOException {
+  private void requireTablePermission(byte[] tableName, Action... permissions) throws IOException {
     User user = getActiveUser();
     AuthResult result = null;
 
-    // Table admins are allowed to perform DDL
-    if (authManager.authorize(user, tableName, (byte[]) null, TablePermission.Action.ADMIN)) {
-      result = AuthResult.allow("Table permission granted", user, TablePermission.Action.ADMIN,
-          tableName);
-    } else if (isActiveUserTableOwner(e, tableName)) {
-      // Table owners with Create permission are allowed to perform DDL
-      if (authManager.authorize(user, tableName, (byte[]) null, TablePermission.Action.CREATE)) {
-        result = AuthResult.allow("Owner has table permission", user,
-            TablePermission.Action.CREATE, tableName);
+    for (Action permission : permissions) {
+      if (authManager.authorize(user, tableName, (byte[]) null, permission)) {
+        result = AuthResult.allow("Table permission granted", user, permission, tableName);
+        break;
       } else {
-        // Table owners without Create permission cannot perform DDL
-        result = AuthResult.deny("Insufficient permissions", user, TablePermission.Action.CREATE,
-            tableName);
+        // rest of the world
+        result = AuthResult.deny("Insufficient permissions", user, permission, tableName);
       }
-    } else {
-      // rest of the world
-      result = AuthResult.deny("Insufficient permissions", user, TablePermission.Action.ADMIN,
-          tableName);
     }
     logResult(result);
     if (!result.isAllowed()) {
@@ -540,24 +516,27 @@ public void stop(CoprocessorEnvironment env) {
   public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
       HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
     requirePermission(Permission.Action.CREATE);
-
-    // default the table owner if not specified
-    User owner = getActiveUser();
-    if (desc.getOwnerString() == null ||
-        desc.getOwnerString().equals("")) {
-      desc.setOwner(owner);
-    }
   }
 
   @Override
   public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      HTableDescriptor desc, HRegionInfo[] regions) throws IOException {}
+      HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+    if (!AccessControlLists.isAclTable(desc)) {
+      String owner = desc.getOwnerString();
+      // default the table owner to current user, if not specified.
+      if (owner == null) owner = getActiveUser().getShortName();
+      UserPermission userperm = new UserPermission(Bytes.toBytes(owner), desc.getName(), null,
+          Action.values());
+      AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(), userperm);
+    }
+  }
 
   @Override
   public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
@@ -567,18 +546,26 @@ public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HTableDescriptor htd) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, HTableDescriptor htd) throws IOException {}
-
+      byte[] tableName, HTableDescriptor htd) throws IOException {
+    String owner = htd.getOwnerString();
+    // default the table owner to current user, if not specified.
+    if (owner == null) owner = getActiveUser().getShortName();
+    UserPermission userperm = new UserPermission(Bytes.toBytes(owner), htd.getName(), null,
+        Action.values());
+    AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(), userperm);
+  }
 
   @Override
   public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor column) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor column) throws IOException {}
@@ -586,18 +573,19 @@ public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor descriptor) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor descriptor) throws IOException {}
 
-
   @Override
   public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, byte[] col) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, byte[] col) throws IOException {
@@ -608,8 +596,9 @@ public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {}
@@ -617,8 +606,9 @@ public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {
-    requireTableAdminPermission(c.getEnvironment(), tableName);
+    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
   }
+
   @Override
   public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {}
@@ -725,18 +715,18 @@ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
 
   @Override
   public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requireTableAdminPermission(e.getEnvironment(), getTableName(e.getEnvironment()));
+    requireTablePermission(getTableName(e.getEnvironment()), Action.ADMIN);
   }
 
   @Override
   public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requireTableAdminPermission(e.getEnvironment(), getTableName(e.getEnvironment()));
+    requireTablePermission(getTableName(e.getEnvironment()), Action.ADMIN);
   }
 
   @Override
   public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
       final Store store, final InternalScanner scanner) throws IOException {
-    requireTableAdminPermission(e.getEnvironment(), getTableName(e.getEnvironment()));
+    requireTablePermission(getTableName(e.getEnvironment()), Action.ADMIN);
     return scanner;
   }
 
@@ -1107,15 +1097,4 @@ private byte[] getTableName(RegionCoprocessorEnvironment e) {
     }
     return tableName;
   }
-
-  private String getTableOwner(CoprocessorEnvironment e, byte[] tableName) throws IOException {
-    HTableDescriptor htd = e.getTable(tableName).getTableDescriptor();
-    return htd.getOwnerString();
-  }
-
-  private boolean isActiveUserTableOwner(CoprocessorEnvironment e, byte[] tableName)
-      throws IOException {
-    String activeUser = getActiveUser().getShortName();
-    return activeUser.equals(getTableOwner(e, tableName));
-  }
 }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 2da7c7197b75..23aff8080f3d 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -27,11 +27,8 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -58,6 +55,7 @@
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -69,8 +67,8 @@
  * levels of authorized users.
  */
 @Category(LargeTests.class)
+@SuppressWarnings("rawtypes")
 public class TestAccessController {
-  private static Log LOG = LogFactory.getLog(TestAccessController.class);
   private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static Configuration conf;
 
@@ -78,14 +76,14 @@ public class TestAccessController {
   private static User SUPERUSER;
   // user granted with all global permission
   private static User USER_ADMIN;
-  // table owner user
-  private static User USER_OWNER;
   // user with rw permissions
   private static User USER_RW;
   // user with read-only permissions
   private static User USER_RO;
-  // user with table admin permissions
-  private static User USER_TBLADM;
+  // user is table owner. will have all permissions on table
+  private static User USER_OWNER;
+  // user with create table permissions alone
+  private static User USER_CREATE;
   // user with no permissions
   private static User USER_NONE;
 
@@ -103,51 +101,52 @@ public static void setupBeforeClass() throws Exception {
     SecureTestUtil.enableSecurity(conf);
 
     TEST_UTIL.startMiniCluster();
-    MasterCoprocessorHost cpHost = TEST_UTIL.getMiniHBaseCluster()
-        .getMaster().getCoprocessorHost();
+    MasterCoprocessorHost cpHost = TEST_UTIL.getMiniHBaseCluster().getMaster().getCoprocessorHost();
     cpHost.load(AccessController.class, Coprocessor.PRIORITY_HIGHEST, conf);
-    ACCESS_CONTROLLER = (AccessController)cpHost.findCoprocessor(
-        AccessController.class.getName());
+    ACCESS_CONTROLLER = (AccessController) cpHost.findCoprocessor(AccessController.class.getName());
     CP_ENV = cpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER,
-        Coprocessor.PRIORITY_HIGHEST, 1, conf);
+      Coprocessor.PRIORITY_HIGHEST, 1, conf);
+
+    // Wait for the ACL table to become available
+    TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
 
     // create a set of test users
-    SUPERUSER = User.createUserForTesting(conf, "admin", new String[]{"supergroup"});
+    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
     USER_ADMIN = User.createUserForTesting(conf, "admin2", new String[0]);
-    USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
     USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
     USER_RO = User.createUserForTesting(conf, "rouser", new String[0]);
-    USER_TBLADM = User.createUserForTesting(conf, "tbladm", new String[0]);
+    USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
+    USER_CREATE = User.createUserForTesting(conf, "tbl_create", new String[0]);
     USER_NONE = User.createUserForTesting(conf, "nouser", new String[0]);
 
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
-    htd.setOwnerString(USER_OWNER.getShortName());
+    htd.setOwner(USER_OWNER);
     admin.createTable(htd);
 
     // initilize access control
     HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol =
-        meta.coprocessorProxy(AccessControllerProtocol.class, TEST_TABLE);
+    AccessControllerProtocol protocol = meta.coprocessorProxy(AccessControllerProtocol.class,
+      TEST_TABLE);
 
     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0);
     RegionCoprocessorHost rcpHost = region.getCoprocessorHost();
     RCP_ENV = rcpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER,
-        Coprocessor.PRIORITY_HIGHEST, 1, conf);
+      Coprocessor.PRIORITY_HIGHEST, 1, conf);
 
     protocol.grant(new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()),
-                      Permission.Action.ADMIN, Permission.Action.CREATE,
-                      Permission.Action.READ, Permission.Action.WRITE));
+        Permission.Action.ADMIN, Permission.Action.CREATE, Permission.Action.READ,
+        Permission.Action.WRITE));
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_RW.getShortName()),
-        TEST_TABLE, TEST_FAMILY, Permission.Action.READ, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_RW.getShortName()), TEST_TABLE,
+        TEST_FAMILY, Permission.Action.READ, Permission.Action.WRITE));
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()),
-                   TEST_TABLE, TEST_FAMILY, Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
+        TEST_FAMILY, Permission.Action.READ));
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_TBLADM.getShortName()),
-      TEST_TABLE, null, Permission.Action.ADMIN));    
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_CREATE.getShortName()), TEST_TABLE, null,
+        Permission.Action.CREATE));
   }
 
   @AfterClass
@@ -155,8 +154,7 @@ public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  public void verifyAllowed(User user, PrivilegedExceptionAction... actions)
-    throws Exception {
+  public void verifyAllowed(User user, PrivilegedExceptionAction... actions) throws Exception {
     for (PrivilegedExceptionAction action : actions) {
       try {
         user.runAs(action);
@@ -166,15 +164,13 @@ public void verifyAllowed(User user, PrivilegedExceptionAction... actions)
     }
   }
 
-  public void verifyAllowed(PrivilegedExceptionAction action, User... users)
-    throws Exception {
+  public void verifyAllowed(PrivilegedExceptionAction action, User... users) throws Exception {
     for (User user : users) {
       verifyAllowed(user, action);
     }
   }
 
-  public void verifyDenied(User user, PrivilegedExceptionAction... actions)
-    throws Exception {
+  public void verifyDenied(User user, PrivilegedExceptionAction... actions) throws Exception {
     for (PrivilegedExceptionAction action : actions) {
       try {
         user.runAs(action);
@@ -199,12 +195,11 @@ public void verifyDenied(User user, PrivilegedExceptionAction... actions)
     }
   }
 
-  public void verifyDenied(PrivilegedExceptionAction action, User... users)
-      throws Exception {
-      for (User user : users) {
-        verifyDenied(user, action);
-      }
+  public void verifyDenied(PrivilegedExceptionAction action, User... users) throws Exception {
+    for (User user : users) {
+      verifyDenied(user, action);
     }
+  }
 
   @Test
   public void testTableCreate() throws Exception {
@@ -212,21 +207,16 @@ public void testTableCreate() throws Exception {
       public Object run() throws Exception {
         HTableDescriptor htd = new HTableDescriptor("testnewtable");
         htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
-        ACCESS_CONTROLLER.preCreateTable(
-            ObserverContext.createAndPrepare(CP_ENV, null), htd, null);
+        ACCESS_CONTROLLER.preCreateTable(ObserverContext.createAndPrepare(CP_ENV, null), htd, null);
         return null;
       }
     };
 
     // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, createTable);
-    verifyAllowed(USER_ADMIN, createTable);
+    verifyAllowed(createTable, SUPERUSER, USER_ADMIN);
 
     // all others should be denied
-    verifyDenied(USER_OWNER, createTable);
-    verifyDenied(USER_RW, createTable);
-    verifyDenied(USER_RO, createTable);
-    verifyDenied(USER_NONE, createTable);
+    verifyDenied(createTable, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -235,43 +225,29 @@ public void testTableModify() throws Exception {
       public Object run() throws Exception {
         HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
         htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
-        htd.addFamily(new HColumnDescriptor("fam_"+User.getCurrent().getShortName()));
-        ACCESS_CONTROLLER.preModifyTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE, htd);
+        htd.addFamily(new HColumnDescriptor("fam_" + User.getCurrent().getShortName()));
+        ACCESS_CONTROLLER.preModifyTable(ObserverContext.createAndPrepare(CP_ENV, null),
+          TEST_TABLE, htd);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, modifyTable);
-    verifyDenied(USER_RW, modifyTable);
-    verifyDenied(USER_RO, modifyTable);
-    verifyDenied(USER_NONE, modifyTable);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, modifyTable);
-    verifyAllowed(USER_ADMIN, modifyTable);
-    verifyAllowed(USER_TBLADM, modifyTable);
+    verifyAllowed(modifyTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(modifyTable, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testTableDelete() throws Exception {
     PrivilegedExceptionAction deleteTable = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        ACCESS_CONTROLLER.preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE);
+        ACCESS_CONTROLLER
+            .preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, deleteTable);
-    verifyDenied(USER_RW, deleteTable);
-    verifyDenied(USER_RO, deleteTable);
-    verifyDenied(USER_NONE, deleteTable);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, deleteTable);
-    verifyAllowed(USER_ADMIN, deleteTable);
-    verifyAllowed(USER_TBLADM, deleteTable);
+    verifyAllowed(deleteTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(deleteTable, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -279,21 +255,14 @@ public void testAddColumn() throws Exception {
     final HColumnDescriptor hcd = new HColumnDescriptor("fam_new");
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        ACCESS_CONTROLLER.preAddColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE, hcd);
+        ACCESS_CONTROLLER.preAddColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE,
+          hcd);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
-    verifyAllowed(USER_TBLADM, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(action, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -302,162 +271,110 @@ public void testModifyColumn() throws Exception {
     hcd.setMaxVersions(10);
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        ACCESS_CONTROLLER.preModifyColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE, hcd);
+        ACCESS_CONTROLLER.preModifyColumn(ObserverContext.createAndPrepare(CP_ENV, null),
+          TEST_TABLE, hcd);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
-    verifyAllowed(USER_TBLADM, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(action, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testDeleteColumn() throws Exception {
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        ACCESS_CONTROLLER.preDeleteColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE, TEST_FAMILY);
+        ACCESS_CONTROLLER.preDeleteColumn(ObserverContext.createAndPrepare(CP_ENV, null),
+          TEST_TABLE, TEST_FAMILY);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
-    verifyAllowed(USER_TBLADM, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(action, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testTableDisable() throws Exception {
     PrivilegedExceptionAction disableTable = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE);
+        ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null),
+          TEST_TABLE);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, disableTable);
-    verifyDenied(USER_RW, disableTable);
-    verifyDenied(USER_RO, disableTable);
-    verifyDenied(USER_NONE, disableTable);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, disableTable);
-    verifyAllowed(USER_ADMIN, disableTable);
-    verifyAllowed(USER_TBLADM, disableTable);
+    verifyAllowed(disableTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(disableTable, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testTableEnable() throws Exception {
     PrivilegedExceptionAction enableTable = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        ACCESS_CONTROLLER.preEnableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE);
+        ACCESS_CONTROLLER
+            .preEnableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, enableTable);
-    verifyDenied(USER_RW, enableTable);
-    verifyDenied(USER_RO, enableTable);
-    verifyDenied(USER_NONE, enableTable);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, enableTable);
-    verifyAllowed(USER_ADMIN, enableTable);
-    verifyAllowed(USER_TBLADM, enableTable);
+    verifyAllowed(enableTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
+    verifyDenied(enableTable, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testMove() throws Exception {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE);
-    Map<HRegionInfo,HServerAddress> regions = table.getRegionsInfo();
-    final Map.Entry<HRegionInfo,HServerAddress> firstRegion =
-        regions.entrySet().iterator().next();
+    Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+    final Map.Entry<HRegionInfo, HServerAddress> firstRegion = regions.entrySet().iterator().next();
     final ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null),
-            firstRegion.getKey(), server, server);
+          firstRegion.getKey(), server, server);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testAssign() throws Exception {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE);
-    Map<HRegionInfo,HServerAddress> regions = table.getRegionsInfo();
-    final Map.Entry<HRegionInfo,HServerAddress> firstRegion =
-        regions.entrySet().iterator().next();
+    Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+    final Map.Entry<HRegionInfo, HServerAddress> firstRegion = regions.entrySet().iterator().next();
 
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         ACCESS_CONTROLLER.preAssign(ObserverContext.createAndPrepare(CP_ENV, null),
-            firstRegion.getKey());
+          firstRegion.getKey());
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
   public void testUnassign() throws Exception {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE);
-    Map<HRegionInfo,HServerAddress> regions = table.getRegionsInfo();
-    final Map.Entry<HRegionInfo,HServerAddress> firstRegion =
-        regions.entrySet().iterator().next();
+    Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+    final Map.Entry<HRegionInfo, HServerAddress> firstRegion = regions.entrySet().iterator().next();
 
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null),
-            firstRegion.getKey(), false);
+          firstRegion.getKey(), false);
         return null;
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -469,15 +386,8 @@ public Object run() throws Exception {
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -489,15 +399,8 @@ public Object run() throws Exception {
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -509,15 +412,8 @@ public Object run() throws Exception {
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -529,27 +425,13 @@ public Object run() throws Exception {
       }
     };
 
-    // all others should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_RO, action);
-    verifyDenied(USER_NONE, action);
-
-    // verify that superuser can create tables
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
   }
 
   private void verifyWrite(PrivilegedExceptionAction action) throws Exception {
-    // should be denied
-    verifyDenied(USER_NONE, action);
-    verifyDenied(USER_RO, action);
-
-    // should be allowed
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
-    verifyAllowed(USER_OWNER, action);
-    verifyAllowed(USER_RW, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW);
+    verifyDenied(action, USER_NONE, USER_CREATE, USER_RO);
   }
 
   @Test
@@ -561,11 +443,8 @@ public Object run() throws Exception {
       }
     };
 
-    // verify that superuser and admin only can split
-    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_TBLADM);
-
-    // all others should be denied
-    verifyDenied(action, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -577,11 +456,8 @@ public Object run() throws Exception {
       }
     };
 
-    // verify that superuser and admin only can flush
-    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_TBLADM);
-
-    // all others should be denied
-    verifyDenied(action, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -593,35 +469,18 @@ public Object run() throws Exception {
       }
     };
 
-    // verify that superuser and admin only can compact
-    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_TBLADM);
-
-    // all others should be denied
-    verifyDenied(action, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   private void verifyRead(PrivilegedExceptionAction action) throws Exception {
-    // should be denied
-    verifyDenied(USER_NONE, action);
-
-    // should be allowed
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
-    verifyAllowed(USER_OWNER, action);
-    verifyAllowed(USER_RW, action);
-    verifyAllowed(USER_RO, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW, USER_RO);
+    verifyDenied(action, USER_NONE, USER_CREATE);
   }
 
   private void verifyReadWrite(PrivilegedExceptionAction action) throws Exception {
-    // should be denied
-    verifyDenied(USER_NONE, action);
-    verifyDenied(USER_RO, action);
-
-    // should be allowed
-    verifyAllowed(SUPERUSER, action);
-    verifyAllowed(USER_ADMIN, action);
-    verifyAllowed(USER_OWNER, action);
-    verifyAllowed(USER_RW, action);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW);
+    verifyDenied(action, USER_NONE, USER_CREATE, USER_RO);
   }
 
   @Test
@@ -709,9 +568,8 @@ public Object run() throws Exception {
         d.deleteFamily(TEST_FAMILY);
 
         HTable t = new HTable(conf, TEST_TABLE);
-        t.checkAndDelete(Bytes.toBytes("random_row"), 
-                         TEST_FAMILY, Bytes.toBytes("q"),
-                         Bytes.toBytes("test_value"), d);
+        t.checkAndDelete(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
+          Bytes.toBytes("test_value"), d);
         return null;
       }
     };
@@ -724,9 +582,8 @@ public Object run() throws Exception {
         p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1));
 
         HTable t = new HTable(conf, TEST_TABLE);
-        t.checkAndPut(Bytes.toBytes("random_row"), 
-                      TEST_FAMILY, Bytes.toBytes("q"),
-                      Bytes.toBytes("test_value"), p);
+        t.checkAndPut(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
+          Bytes.toBytes("test_value"), p);
         return null;
       }
     };
@@ -749,18 +606,18 @@ public void testGrantRevoke() throws Exception {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.addFamily(new HColumnDescriptor(family1));
     htd.addFamily(new HColumnDescriptor(family2));
-    htd.setOwnerString(USER_OWNER.getShortName());
     admin.createTable(htd);
 
     // create temp users
-    User tblUser = User.createUserForTesting(TEST_UTIL.getConfiguration(), "tbluser", new String[0]);
-    User gblUser = User.createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]);
+    User tblUser = User
+        .createUserForTesting(TEST_UTIL.getConfiguration(), "tbluser", new String[0]);
+    User gblUser = User
+        .createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]);
 
     // perms only stored against the first region
     HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol =
-        acl.coprocessorProxy(AccessControllerProtocol.class,
-            tableName);
+    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+      tableName);
 
     // prepare actions:
     PrivilegedExceptionAction putActionAll = new PrivilegedExceptionAction() {
@@ -860,7 +717,8 @@ public Object run() throws Exception {
     // grant table read permission
     protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
         Permission.Action.READ));
-    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
+    protocol
+        .grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
 
     Thread.sleep(100);
     // check
@@ -905,7 +763,8 @@ public Object run() throws Exception {
     // grant column family read permission
     protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family1,
         Permission.Action.READ));
-    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
+    protocol
+        .grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
 
     Thread.sleep(100);
 
@@ -959,8 +818,7 @@ public Object run() throws Exception {
     admin.deleteTable(tableName);
   }
 
-  private boolean hasFoundUserPermission(UserPermission userPermission,
-                                         List<UserPermission> perms) {
+  private boolean hasFoundUserPermission(UserPermission userPermission, List<UserPermission> perms) {
     return perms.contains(userPermission);
   }
 
@@ -981,16 +839,14 @@ public void testGrantRevokeAtQualifierLevel() throws Exception {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.addFamily(new HColumnDescriptor(family1));
     htd.addFamily(new HColumnDescriptor(family2));
-    htd.setOwnerString(USER_OWNER.getShortName());
     admin.createTable(htd);
 
     // create temp users
-    User user = User.createUserForTesting(TEST_UTIL.getConfiguration(),
-        "user", new String[0]);
+    User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]);
 
     HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol =
-        acl.coprocessorProxy(AccessControllerProtocol.class, tableName);
+    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+      tableName);
 
     PrivilegedExceptionAction getQualifierAction = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
@@ -1014,21 +870,20 @@ public Object run() throws Exception {
       public Object run() throws Exception {
         Delete d = new Delete(Bytes.toBytes("random_row"));
         d.deleteColumn(family1, qualifier);
-        //d.deleteFamily(family1);
+        // d.deleteFamily(family1);
         HTable t = new HTable(conf, tableName);
         t.delete(d);
         return null;
       }
     };
 
-    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
-                    tableName, family1));
+    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1));
     verifyDenied(user, getQualifierAction);
     verifyDenied(user, putQualifierAction);
     verifyDenied(user, deleteQualifierAction);
 
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, family1, qualifier, Permission.Action.READ));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+        qualifier, Permission.Action.READ));
     Thread.sleep(100);
 
     verifyAllowed(user, getQualifierAction);
@@ -1037,8 +892,8 @@ public Object run() throws Exception {
 
     // only grant write permission
     // TODO: comment this portion after HBASE-3583
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, family1, qualifier, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+        qualifier, Permission.Action.WRITE));
     Thread.sleep(100);
 
     verifyDenied(user, getQualifierAction);
@@ -1046,9 +901,8 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteQualifierAction);
 
     // grant both read and write permission.
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()),
-                   tableName, family1, qualifier,
-                   Permission.Action.READ, Permission.Action.WRITE));
+    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+        qualifier, Permission.Action.READ, Permission.Action.WRITE));
     Thread.sleep(100);
 
     verifyAllowed(user, getQualifierAction);
@@ -1056,8 +910,8 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteQualifierAction);
 
     // revoke family level permission won't impact column level.
-    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()),
-                    tableName, family1, qualifier));
+    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+        qualifier));
     Thread.sleep(100);
 
     verifyDenied(user, getQualifierAction);
@@ -1086,76 +940,86 @@ public void testPermissionList() throws Exception {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.addFamily(new HColumnDescriptor(family1));
     htd.addFamily(new HColumnDescriptor(family2));
-    htd.setOwnerString(USER_OWNER.getShortName());
+    htd.setOwner(USER_OWNER);
     admin.createTable(htd);
 
     HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol =
-        acl.coprocessorProxy(AccessControllerProtocol.class, tableName);
+    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+      tableName);
 
     List<UserPermission> perms = protocol.getUserPermissions(tableName);
 
-    UserPermission up = new UserPermission(user,
-        tableName, family1, qualifier, Permission.Action.READ);
+    UserPermission ownerperm = new UserPermission(Bytes.toBytes(USER_OWNER.getName()), tableName,
+        null, Action.values());
+    assertTrue("Owner should have all permissions on table",
+      hasFoundUserPermission(ownerperm, perms));
+
+    UserPermission up = new UserPermission(user, tableName, family1, qualifier,
+        Permission.Action.READ);
     assertFalse("User should not be granted permission: " + up.toString(),
-        hasFoundUserPermission(up, perms));
+      hasFoundUserPermission(up, perms));
 
     // grant read permission
-    UserPermission upToSet = new UserPermission(user,
-        tableName, family1, qualifier, Permission.Action.READ);
+    UserPermission upToSet = new UserPermission(user, tableName, family1, qualifier,
+        Permission.Action.READ);
     protocol.grant(upToSet);
     perms = protocol.getUserPermissions(tableName);
 
-    UserPermission upToVerify = new UserPermission(user,
-        tableName, family1, qualifier, Permission.Action.READ);
+    UserPermission upToVerify = new UserPermission(user, tableName, family1, qualifier,
+        Permission.Action.READ);
     assertTrue("User should be granted permission: " + upToVerify.toString(),
-        hasFoundUserPermission(upToVerify, perms));
+      hasFoundUserPermission(upToVerify, perms));
 
-    upToVerify = new UserPermission(user, tableName, family1, qualifier,
-        Permission.Action.WRITE);
+    upToVerify = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE);
     assertFalse("User should not be granted permission: " + upToVerify.toString(),
-        hasFoundUserPermission(upToVerify, perms));
+      hasFoundUserPermission(upToVerify, perms));
 
     // grant read+write
-    upToSet = new UserPermission(user, tableName, family1, qualifier,
-        Permission.Action.WRITE, Permission.Action.READ);
+    upToSet = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE,
+        Permission.Action.READ);
     protocol.grant(upToSet);
     perms = protocol.getUserPermissions(tableName);
 
-    upToVerify = new UserPermission(user, tableName, family1, qualifier,
-        Permission.Action.WRITE, Permission.Action.READ);
+    upToVerify = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE,
+        Permission.Action.READ);
     assertTrue("User should be granted permission: " + upToVerify.toString(),
-            hasFoundUserPermission(upToVerify, perms));
+      hasFoundUserPermission(upToVerify, perms));
 
     protocol.revoke(upToSet);
     perms = protocol.getUserPermissions(tableName);
     assertFalse("User should not be granted permission: " + upToVerify.toString(),
       hasFoundUserPermission(upToVerify, perms));
 
-    // delete table
+    // disable table before modification
     admin.disableTable(tableName);
+
+    User newOwner = User.createUserForTesting(conf, "new_owner", new String[] {});
+    htd.setOwner(newOwner);
+    admin.modifyTable(tableName, htd);
+    perms = protocol.getUserPermissions(tableName);
+    UserPermission newOwnerperm = new UserPermission(Bytes.toBytes(newOwner.getName()), tableName,
+        null, Action.values());
+    assertTrue("New owner should have all permissions on table",
+      hasFoundUserPermission(newOwnerperm, perms));
+
+    // delete table
     admin.deleteTable(tableName);
   }
 
-  /** global operations*/
+  /** global operations */
   private void verifyGlobal(PrivilegedExceptionAction<?> action) throws Exception {
-    // should be allowed
-    verifyAllowed(SUPERUSER, action);
-
-    // should be denied
-    verifyDenied(USER_OWNER, action);
-    verifyDenied(USER_RW, action);
-    verifyDenied(USER_NONE, action);
-    verifyDenied(USER_RO, action);
+    verifyAllowed(action, SUPERUSER);
+
+    verifyDenied(action, USER_CREATE, USER_RW, USER_NONE, USER_RO);
   }
 
   public void checkGlobalPerms(Permission.Action... actions) throws IOException {
     HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol =
-        acl.coprocessorProxy(AccessControllerProtocol.class, new byte[0]);
+    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+      new byte[0]);
 
     Permission[] perms = new Permission[actions.length];
-    for (int i=0; i < actions.length; i++) {
+    for (int i = 0; i < actions.length; i++) {
       perms[i] = new Permission(actions[i]);
     }
 
@@ -1165,34 +1029,34 @@ public void checkGlobalPerms(Permission.Action... actions) throws IOException {
   public void checkTablePerms(byte[] table, byte[] family, byte[] column,
       Permission.Action... actions) throws IOException {
     Permission[] perms = new Permission[actions.length];
-    for (int i=0; i < actions.length; i++) {
+    for (int i = 0; i < actions.length; i++) {
       perms[i] = new TablePermission(table, family, column, actions[i]);
     }
 
     checkTablePerms(table, perms);
   }
 
-  public void checkTablePerms(byte[] table, Permission...perms) throws IOException {
+  public void checkTablePerms(byte[] table, Permission... perms) throws IOException {
     HTable acl = new HTable(conf, table);
-    AccessControllerProtocol protocol =
-        acl.coprocessorProxy(AccessControllerProtocol.class, new byte[0]);
+    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+      new byte[0]);
 
     protocol.checkPermissions(perms);
   }
 
-  public void grant(AccessControllerProtocol protocol, User user, byte[] t, byte[] f,
-      byte[] q, Permission.Action... actions) throws IOException {
+  public void grant(AccessControllerProtocol protocol, User user, byte[] t, byte[] f, byte[] q,
+      Permission.Action... actions) throws IOException {
     protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), t, f, q, actions));
   }
 
   @Test
   public void testCheckPermissions() throws Exception {
     final HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    final AccessControllerProtocol protocol =
-        acl.coprocessorProxy(AccessControllerProtocol.class, TEST_TABLE);
+    final AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+      TEST_TABLE);
 
-    //--------------------------------------
-    //test global permissions
+    // --------------------------------------
+    // test global permissions
     PrivilegedExceptionAction<Void> globalAdmin = new PrivilegedExceptionAction<Void>() {
       @Override
       public Void run() throws Exception {
@@ -1200,11 +1064,11 @@ public Void run() throws Exception {
         return null;
       }
     };
-    //verify that only superuser can admin
+    // verify that only superuser can admin
     verifyGlobal(globalAdmin);
 
-    //--------------------------------------
-    //test multiple permissions
+    // --------------------------------------
+    // test multiple permissions
     PrivilegedExceptionAction<Void> globalReadWrite = new PrivilegedExceptionAction<Void>() {
       @Override
       public Void run() throws Exception {
@@ -1215,8 +1079,8 @@ public Void run() throws Exception {
 
     verifyGlobal(globalReadWrite);
 
-    //--------------------------------------
-    //table/column/qualifier level permissions
+    // --------------------------------------
+    // table/column/qualifier level permissions
     final byte[] TEST_Q1 = Bytes.toBytes("q1");
     final byte[] TEST_Q2 = Bytes.toBytes("q2");
 
@@ -1256,9 +1120,8 @@ public Void run() throws Exception {
       @Override
       public Void run() throws Exception {
         checkTablePerms(TEST_TABLE, new Permission[] {
-          new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ),
-          new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q2, Permission.Action.READ),
-        });
+            new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ),
+            new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q2, Permission.Action.READ), });
         return null;
       }
     };
@@ -1266,10 +1129,8 @@ public Void run() throws Exception {
     PrivilegedExceptionAction<Void> globalAndTableRead = new PrivilegedExceptionAction<Void>() {
       @Override
       public Void run() throws Exception {
-        checkTablePerms(TEST_TABLE, new Permission[] {
-          new Permission(Permission.Action.READ),
-          new TablePermission(TEST_TABLE, null, (byte[])null, Permission.Action.READ),
-        });
+        checkTablePerms(TEST_TABLE, new Permission[] { new Permission(Permission.Action.READ),
+            new TablePermission(TEST_TABLE, null, (byte[]) null, Permission.Action.READ), });
         return null;
       }
     };
@@ -1298,30 +1159,29 @@ public Void run() throws Exception {
 
     verifyAllowed(noCheck, SUPERUSER, userTable, userColumn, userQualifier);
 
-    //--------------------------------------
-    //test family level multiple permissions
+    // --------------------------------------
+    // test family level multiple permissions
     PrivilegedExceptionAction<Void> familyReadWrite = new PrivilegedExceptionAction<Void>() {
       @Override
       public Void run() throws Exception {
         checkTablePerms(TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ,
-            Permission.Action.WRITE);
+          Permission.Action.WRITE);
         return null;
       }
     };
-    // should be allowed
+
     verifyAllowed(familyReadWrite, SUPERUSER, USER_OWNER, USER_RW);
-    // should be denied
-    verifyDenied(familyReadWrite, USER_NONE, USER_RO);
+    verifyDenied(familyReadWrite, USER_NONE, USER_CREATE, USER_RO);
 
-    //--------------------------------------
-    //check for wrong table region
+    // --------------------------------------
+    // check for wrong table region
     try {
-      //but ask for TablePermissions for TEST_TABLE
-      protocol.checkPermissions(new Permission[] {(Permission) new TablePermission(
-          TEST_TABLE, null, (byte[])null, Permission.Action.CREATE)});
+      // but ask for TablePermissions for TEST_TABLE
+      protocol.checkPermissions(new Permission[] { (Permission) new TablePermission(TEST_TABLE,
+          null, (byte[]) null, Permission.Action.CREATE) });
       fail("this should have thrown CoprocessorException");
-    } catch(CoprocessorException ex) {
-      //expected
+    } catch (CoprocessorException ex) {
+      // expected
     }
 
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index f79739768e94..df63478a7382 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -1134,11 +1134,13 @@ public static Path getTableDir(Path rootdir, final byte [] tableName) {
               .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
       });
 
+  @Deprecated
   public void setOwner(User owner) {
     setOwnerString(owner != null ? owner.getShortName() : null);
   }
 
   // used by admin.rb:alter(table_name,*args) to update owner.
+  @Deprecated
   public void setOwnerString(String ownerString) {
     if (ownerString != null) {
       setValue(OWNER_KEY, Bytes.toBytes(ownerString));
@@ -1147,12 +1149,14 @@ public void setOwnerString(String ownerString) {
     }
   }
 
+  @Deprecated
   public String getOwnerString() {
     if (getValue(OWNER_KEY) != null) {
       return Bytes.toString(getValue(OWNER_KEY));
     }
     // Note that every table should have an owner (i.e. should have OWNER_KEY set).
-    // .META. and -ROOT- should return system user as owner, not null (see MasterFileSystem.java:bootstrap()).
+    // .META. and -ROOT- should return system user as owner, not null (see
+    // MasterFileSystem.java:bootstrap()).
     return null;
   }
 }

From 226006d749e2a2e7b5bb59702c969b8c22621909 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 19 Jun 2012 02:34:25 +0000
Subject: [PATCH 0280/1540] HBASE-6237. Fix race on ACL table creation in
 TestTablePermissions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1351560 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/TestTablePermissions.java    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
index 0203dbadd59f..6407b166c7d6 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
@@ -87,6 +87,10 @@ public static void beforeClass() throws Exception {
     SecureTestUtil.enableSecurity(conf);
 
     UTIL.startMiniCluster();
+
+    // Wait for the ACL table to become available
+    UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+
     ZKW = new ZooKeeperWatcher(UTIL.getConfiguration(),
       "TestTablePermissions", ABORTABLE);
 

From 0c0e5af110edf9a47c976349afbb26311772867b Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 20 Jun 2012 17:34:51 +0000
Subject: [PATCH 0281/1540] HBASE-6164 Correct the bug in block encoding usage
 in bulkload

Submitted by:Anoop
Reviewed by:Ted


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352221 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileDataBlockEncoder.java | 12 +++++++-----
 .../hbase/io/hfile/HFileDataBlockEncoderImpl.java    |  9 +++------
 .../apache/hadoop/hbase/io/hfile/HFileWriterV1.java  |  2 ++
 .../apache/hadoop/hbase/io/hfile/HFileWriterV2.java  |  2 ++
 .../hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java  |  3 +--
 .../apache/hadoop/hbase/regionserver/StoreFile.java  |  7 -------
 .../hadoop/hbase/regionserver/TestStoreFile.java     |  2 +-
 7 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
index 7aa96432c4ad..7be4b1bd2e79 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
@@ -20,7 +20,7 @@
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -29,6 +29,9 @@
  * should just return the unmodified block.
  */
 public interface HFileDataBlockEncoder {
+  /** Type of encoding used for data blocks in HFile. Stored in file info. */
+  public static final byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
+  
   /**
    * Converts a block from the on-disk format to the in-cache format. Called in
    * the following cases:
@@ -64,12 +67,11 @@ public Pair<ByteBuffer, BlockType> beforeWriteToDisk(
   public boolean useEncodedScanner(boolean isCompaction);
 
   /**
-   * Save metadata in StoreFile which will be written to disk
-   * @param storeFileWriter writer for a given StoreFile
+   * Save metadata in HFile which will be written to disk
+   * @param writer writer for a given HFile
    * @exception IOException on disk problems
    */
-  public void saveMetadata(StoreFile.Writer storeFileWriter)
-      throws IOException;
+  public void saveMetadata(HFile.Writer writer) throws IOException;
 
   /** @return the on-disk data block encoding */
   public DataBlockEncoding getEncodingOnDisk();
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
index 01842833dc49..c7b22723f529 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.HFileBlock;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -70,7 +69,7 @@ public static HFileDataBlockEncoder createFromFileInfo(
     boolean hasPreferredCacheEncoding = preferredEncodingInCache != null
         && preferredEncodingInCache != DataBlockEncoding.NONE;
 
-    byte[] dataBlockEncodingType = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
     if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) {
       return NoOpDataBlockEncoder.INSTANCE;
     }
@@ -105,10 +104,8 @@ public static HFileDataBlockEncoder createFromFileInfo(
   }
 
   @Override
-  public void saveMetadata(StoreFile.Writer storeFileWriter)
-      throws IOException {
-    storeFileWriter.appendFileInfo(StoreFile.DATA_BLOCK_ENCODING,
-        onDisk.getNameInBytes());
+  public void saveMetadata(HFile.Writer writer) throws IOException {
+    writer.appendFileInfo(DATA_BLOCK_ENCODING, onDisk.getNameInBytes());
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
index e79c228e71d4..ae276c42944a 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
@@ -332,6 +332,8 @@ public void close() throws IOException {
     if (this.outputStream == null) {
       return;
     }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
     // Write out the end of the data blocks, then write meta data blocks.
     // followed by fileinfo, data block index and meta block index.
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index 44bd77eb6c80..ae9513b3c200 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -356,6 +356,8 @@ public void close() throws IOException {
     if (outputStream == null) {
       return;
     }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
     // Write out the end of the data blocks, then write meta data blocks.
     // followed by fileinfo, data block index and meta block index.
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
index 73abdc92ef25..c69d0875abe4 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
@@ -19,7 +19,6 @@
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -54,7 +53,7 @@ public boolean useEncodedScanner(boolean isCompaction) {
   }
 
   @Override
-  public void saveMetadata(StoreFile.Writer storeFileWriter) {
+  public void saveMetadata(HFile.Writer writer) {
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 9d807ddbe025..d12d0fdb49b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -135,10 +135,6 @@ public static enum BloomType {
   /** Key for timestamp of earliest-put in metadata*/
   public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
 
-  /** Type of encoding used for data blocks in HFile. Stored in file info. */
-  public static final byte[] DATA_BLOCK_ENCODING =
-      Bytes.toBytes("DATA_BLOCK_ENCODING");
-
   // Make default block size for StoreFiles 8k while testing.  TODO: FIX!
   // Need to make it 8k for testing.
   public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
@@ -1205,9 +1201,6 @@ private boolean closeDeleteFamilyBloomFilter() throws IOException {
     }
 
     public void close() throws IOException {
-      // Save data block encoder metadata in the file info.
-      dataBlockEncoder.saveMetadata(this);
-
       boolean hasGeneralBloom = this.closeGeneralBloomFilter();
       boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index fbec7614ac0a..3f63e3d29daf 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -916,7 +916,7 @@ public void testDataBlockEncodingMetaData() throws IOException {
     StoreFile.Reader reader = storeFile.createReader();
     
     Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
-    byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
 
     assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
   }

From 7c47e21a5b0cf95f87030223834e7c6565d04ff9 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 20 Jun 2012 17:44:36 +0000
Subject: [PATCH 0282/1540] HBASE-5918 Master will block forever at startup if
 root server dies between assigning root and assigning meta

Submitted by:Chunhui
Reviewed by:Stack, Ted, Ram


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352229 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/HMaster.java | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 89eebd6fe090..9d3799f9aa14 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -527,8 +527,7 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
 
     // Make sure root and meta assigned before proceeding.
     assignRootAndMeta(status);
-    serverShutdownHandlerEnabled = true;
-    this.serverManager.expireDeadNotExpiredServers();
+    enableServerShutdownHandler();
 
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
@@ -579,6 +578,19 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     }
   }
   
+  /**
+   * If ServerShutdownHandler is disabled, we enable it and expire those dead
+   * but not expired servers.
+   * 
+   * @throws IOException
+   */
+  private void enableServerShutdownHandler() throws IOException {
+    if (!serverShutdownHandlerEnabled) {
+      serverShutdownHandlerEnabled = true;
+      this.serverManager.expireDeadNotExpiredServers();
+    }
+  }
+  
   /**
    * Useful for testing purpose also where we have
    * master restart scenarios.
@@ -644,6 +656,7 @@ int assignRootAndMeta(MonitoredTask status)
         splitLogAndExpireIfOnline(currentMetaServer);
       }
       assignmentManager.assignMeta();
+      enableServerShutdownHandler();
       this.catalogTracker.waitForMeta();
       // Above check waits for general meta availability but this does not
       // guarantee that the transition has completed

From dac5d1d4c3990e9019ae5aa0d2884656f618b744 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 20 Jun 2012 18:46:57 +0000
Subject: [PATCH 0283/1540] HBASE-6229 AM.assign() should not set table state
 to ENABLED directly (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352263 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/AssignmentManager.java       | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 5324a291360f..f39e9d0e7434 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1588,7 +1588,17 @@ private void assign(final HRegionInfo region, final RegionState state,
           if (isDisabledorDisablingRegionInRIT(region)) {
             return;
           }
-          setEnabledTable(region);
+          // In case of assign from EnableTableHandler table state is ENABLING. Any how
+          // EnableTableHandler will set ENABLED after assigning all the table regions. If we
+          // try to set to ENABLED directly then client api may think ENABLE table is completed.
+          // When we have a case like all the regions are added directly into META and we call
+          // assignRegion then we need to make the table ENABLED. Hence in such case the table
+          // will not be in ENABLING or ENABLED state.
+          String tableName = region.getTableNameAsString();
+          if (!zkTable.isEnablingTable(tableName) && !zkTable.isEnabledTable(tableName)) {
+            LOG.debug("Setting table " + tableName + " to ENABLED state.");
+            setEnabledTable(region);
+          }
         }
       }
       

From c796cc4bf7164dab796a002e73c571d344e39093 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 20 Jun 2012 21:14:52 +0000
Subject: [PATCH 0284/1540] HBASE-6244. [REST] Result generators do not need to
 query table schema

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352326 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/RowResultGenerator.java  | 8 --------
 .../apache/hadoop/hbase/rest/ScannerResultGenerator.java  | 6 ------
 2 files changed, 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
index 74f30b4c387f..edff0f347f99 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
@@ -29,14 +29,12 @@
 
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.HTablePool;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 
 public class RowResultGenerator extends ResultGenerator {
   private static final Log LOG = LogFactory.getLog(RowResultGenerator.class);
@@ -59,12 +57,6 @@ public RowResultGenerator(final String tableName, final RowSpec rowspec,
             get.addFamily(split[0]);
           }
         }
-      } else {
-        // rowspec does not explicitly specify columns, return them all
-        for (HColumnDescriptor family: 
-            table.getTableDescriptor().getFamilies()) {
-          get.addFamily(family.getName());
-        }
       }
       get.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
       get.setMaxVersions(rowspec.getMaxVersions());
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
index d4f1dfc8d4f9..15078b73b946 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
@@ -25,7 +25,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.client.HTableInterface;
@@ -78,11 +77,6 @@ public ScannerResultGenerator(final String tableName, final RowSpec rowspec,
             scan.addFamily(split[0]);
           }
         }
-      } else {
-        for (HColumnDescriptor family: 
-            table.getTableDescriptor().getFamilies()) {
-          scan.addFamily(family.getName());
-        }
       }
       scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());          
       scan.setMaxVersions(rowspec.getMaxVersions());

From 073d34bec07475a98673818fca3e37be2da318db Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 20 Jun 2012 21:45:19 +0000
Subject: [PATCH 0285/1540] HBASE-6247. [REST] HTablePool.putTable is
 deprecated

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352349 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/RegionsResource.java    |  2 +-
 .../apache/hadoop/hbase/rest/RowResource.java | 53 ++++++-------------
 .../hadoop/hbase/rest/RowResultGenerator.java |  2 +-
 .../hbase/rest/ScannerResultGenerator.java    |  2 +-
 .../hadoop/hbase/rest/SchemaResource.java     |  2 +-
 5 files changed, 19 insertions(+), 42 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index d0058aacf5d6..d527e3e8ca9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -74,7 +74,7 @@ private Map<HRegionInfo,HServerAddress> getTableRegions()
     try {
       return ((HTable)table).getRegionsInfo();
     } finally {
-      pool.putTable(table);
+      table.close();
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index 3cc5da08500b..97c109360e2b 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -211,14 +211,9 @@ Response update(final CellSetModel model, final boolean replace) {
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     } finally {
-      if (table != null) {
-        try {
-          pool.putTable(table);
-        } catch (IOException ioe) {
-          throw new WebApplicationException(ioe,
-              Response.Status.SERVICE_UNAVAILABLE);
-        }
-      }
+      if (table != null) try {
+        table.close();
+      } catch (IOException ioe) { }
     }
   }
 
@@ -273,14 +268,9 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
       throw new WebApplicationException(e,
                   Response.Status.SERVICE_UNAVAILABLE);
     } finally {
-      if (table != null) {
-        try {
-          pool.putTable(table);
-        } catch (IOException ioe) {
-          throw new WebApplicationException(ioe,
-              Response.Status.SERVICE_UNAVAILABLE);
-        }
-      }
+      if (table != null) try {
+        table.close();
+      } catch (IOException ioe) { }
     }
   }
 
@@ -371,14 +361,9 @@ public Response delete(final @Context UriInfo uriInfo) {
       throw new WebApplicationException(e, 
                   Response.Status.SERVICE_UNAVAILABLE);
     } finally {
-      if (table != null) {
-        try {
-          pool.putTable(table);
-        } catch (IOException ioe) {
-          throw new WebApplicationException(ioe,
-              Response.Status.SERVICE_UNAVAILABLE);
-        }
-      }
+      if (table != null) try {
+        table.close();
+      } catch (IOException ioe) { }
     }
     return Response.ok().build();
   }
@@ -448,14 +433,9 @@ Response checkAndPut(final CellSetModel model) {
     } catch (IOException e) {
       throw new WebApplicationException(e, Response.Status.SERVICE_UNAVAILABLE);
     } finally {
-      try {
-        if(table != null){
-          pool.putTable(table);
-        }
-      } catch (Exception ioe) {
-        throw new WebApplicationException(ioe,
-          Response.Status.SERVICE_UNAVAILABLE);
-      }
+      if (table != null) try {
+        table.close();
+      } catch (IOException ioe) { }
     }
   }
 
@@ -516,12 +496,9 @@ Response checkAndDelete(final CellSetModel model) {
     } catch (IOException e) {
       throw new WebApplicationException(e, Response.Status.SERVICE_UNAVAILABLE);
     } finally {
-      try {
-        pool.putTable(table);
-      } catch (Exception ioe) {
-        throw new WebApplicationException(ioe,
-          Response.Status.SERVICE_UNAVAILABLE);
-      }
+      if (table != null) try {
+        table.close();
+      } catch (IOException ioe) { }
     }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
index edff0f347f99..8d6cfb48513d 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
@@ -76,7 +76,7 @@ public RowResultGenerator(final String tableName, final RowSpec rowspec,
       // the log.
       LOG.warn(StringUtils.stringifyException(e));
     } finally {
-      pool.putTable(table);
+      table.close();
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
index 15078b73b946..e068c2e4edc3 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
@@ -90,7 +90,7 @@ public ScannerResultGenerator(final String tableName, final RowSpec rowspec,
       id = Long.toString(System.currentTimeMillis()) +
              Integer.toHexString(scanner.hashCode());
     } finally {
-      pool.putTable(table);
+      table.close();
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index a31744aaf498..ddc10d1dc950 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -81,7 +81,7 @@ private HTableDescriptor getTableSchema() throws IOException,
     try {
       return table.getTableDescriptor();
     } finally {
-      pool.putTable(table);
+      table.close();
     }
   }
 

From e1eb174504eb4dc48b560abe22b153f9f6a1f18c Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 20 Jun 2012 22:30:10 +0000
Subject: [PATCH 0286/1540] HBASE-6209. ACL Corrections for
 AccessControllerProtocol (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352355 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 124 ++++++++++--------
 .../security/access/TestAccessController.java |  64 ++++++++-
 2 files changed, 130 insertions(+), 58 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 8d0679db81d1..39c29fca0d80 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Increment;
@@ -142,8 +143,12 @@ public String toString() {
           .append(toContextString()).toString();
     }
 
-    public static AuthResult allow(String reason, User user,
-        Permission.Action action, byte[] table) {
+    public static AuthResult allow(String reason, User user, Permission.Action action,
+        byte[] table, byte[] family, byte[] qualifier) {
+      return new AuthResult(true, reason, user, action, table, family, qualifier);
+    }
+
+    public static AuthResult allow(String reason, User user, Permission.Action action, byte[] table) {
       return new AuthResult(true, reason, user, action, table, null, null);
     }
 
@@ -356,22 +361,26 @@ private User getActiveUser() throws IOException {
   }
 
   /**
-   * Authorizes that the current user has any of the given permissions for the given table.
+   * Authorizes that the current user has any of the given permissions for the
+   * given table, column family and column qualifier.
    * @param tableName Table requested
+   * @param family Column family requested
+   * @param qualifier Column qualifier requested
    * @throws IOException if obtaining the current user fails
    * @throws AccessDeniedException if user has no authorization
    */
-  private void requireTablePermission(byte[] tableName, Action... permissions) throws IOException {
+  private void requirePermission(byte[] tableName, byte[] family, byte[] qualifier,
+      Action... permissions) throws IOException {
     User user = getActiveUser();
     AuthResult result = null;
 
     for (Action permission : permissions) {
-      if (authManager.authorize(user, tableName, (byte[]) null, permission)) {
-        result = AuthResult.allow("Table permission granted", user, permission, tableName);
+      if (authManager.authorize(user, tableName, null, null, permission)) {
+        result = AuthResult.allow("Table permission granted", user, permission, tableName, family, qualifier);
         break;
       } else {
         // rest of the world
-        result = AuthResult.deny("Insufficient permissions", user, permission, tableName);
+        result = AuthResult.deny("Insufficient permissions", user, permission, tableName, family, qualifier);
       }
     }
     logResult(result);
@@ -532,9 +541,9 @@ public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
   }
 
   @Override
-  public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName)
+      throws IOException {
+   requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -544,9 +553,9 @@ public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
   }
 
   @Override
-  public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, HTableDescriptor htd) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
+      HTableDescriptor htd) throws IOException {
+    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -561,9 +570,9 @@ public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
   }
 
   @Override
-  public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, HColumnDescriptor column) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
+      HColumnDescriptor column) throws IOException {
+    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -571,9 +580,9 @@ public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor column) throws IOException {}
 
   @Override
-  public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, HColumnDescriptor descriptor) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
+      HColumnDescriptor descriptor) throws IOException {
+    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -581,9 +590,9 @@ public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName, HColumnDescriptor descriptor) throws IOException {}
 
   @Override
-  public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, byte[] col) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
+      byte[] col) throws IOException {
+    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -594,9 +603,9 @@ public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   }
 
   @Override
-  public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName)
+      throws IOException {
+    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -604,9 +613,9 @@ public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {}
 
   @Override
-  public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName) throws IOException {
-    requireTablePermission(tableName, Action.ADMIN, Action.CREATE);
+  public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName)
+      throws IOException {
+    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -715,18 +724,18 @@ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
 
   @Override
   public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requireTablePermission(getTableName(e.getEnvironment()), Action.ADMIN);
+    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
   }
 
   @Override
   public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requireTablePermission(getTableName(e.getEnvironment()), Action.ADMIN);
+    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
   }
 
   @Override
   public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
       final Store store, final InternalScanner scanner) throws IOException {
-    requireTablePermission(getTableName(e.getEnvironment()), Action.ADMIN);
+    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
     return scanner;
   }
 
@@ -852,6 +861,13 @@ public long preIncrementColumnValue(final ObserverContext<RegionCoprocessorEnvir
     return -1;
   }
 
+  @Override
+  public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> c, Append append)
+      throws IOException {
+    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(), append.getFamilyMap());
+    return null;
+  }
+
   @Override
   public Result preIncrement(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Increment increment)
@@ -956,25 +972,23 @@ private void requireScannerOwner(InternalScanner s)
    * This will be restricted by both client side and endpoint implementations.
    */
   @Override
-  public void grant(UserPermission userPermission)
-      throws IOException {
+  public void grant(UserPermission perm) throws IOException {
     // verify it's only running at .acl.
     if (aclRegion) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Received request to grant access permission " + userPermission.toString());
+        LOG.debug("Received request to grant access permission " + perm.toString());
       }
 
-      requirePermission(Permission.Action.ADMIN);
+      requirePermission(perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN);
 
-      AccessControlLists.addUserPermission(regionEnv.getConfiguration(), userPermission);
+      AccessControlLists.addUserPermission(regionEnv.getConfiguration(), perm);
       if (AUDITLOG.isTraceEnabled()) {
         // audit log should store permission changes in addition to auth results
-        AUDITLOG.trace("Granted permission " + userPermission.toString());
+        AUDITLOG.trace("Granted permission " + perm.toString());
       }
     } else {
-      throw new CoprocessorException(AccessController.class, "This method " +
-          "can only execute at " +
-          Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
+      throw new CoprocessorException(AccessController.class, "This method "
+          + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
     }
   }
 
@@ -988,25 +1002,23 @@ public void grant(byte[] user, TablePermission permission)
   }
 
   @Override
-  public void revoke(UserPermission userPermission)
-      throws IOException{
+  public void revoke(UserPermission perm) throws IOException {
     // only allowed to be called on _acl_ region
     if (aclRegion) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Received request to revoke access permission " + userPermission.toString());
+        LOG.debug("Received request to revoke access permission " + perm.toString());
       }
 
-      requirePermission(Permission.Action.ADMIN);
+      requirePermission(perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN);
 
-      AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), userPermission);
+      AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), perm);
       if (AUDITLOG.isTraceEnabled()) {
         // audit log should record all permission changes
-        AUDITLOG.trace("Revoked permission " + userPermission.toString());
+        AUDITLOG.trace("Revoked permission " + perm.toString());
       }
     } else {
-      throw new CoprocessorException(AccessController.class, "This method " +
-          "can only execute at " +
-          Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
+      throw new CoprocessorException(AccessController.class, "This method "
+          + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
     }
   }
 
@@ -1020,19 +1032,17 @@ public void revoke(byte[] user, TablePermission permission)
   }
 
   @Override
-  public List<UserPermission> getUserPermissions(final byte[] tableName)
-      throws IOException {
+  public List<UserPermission> getUserPermissions(final byte[] tableName) throws IOException {
     // only allowed to be called on _acl_ region
     if (aclRegion) {
-      requirePermission(Permission.Action.ADMIN);
+      requirePermission(tableName, null, null, Action.ADMIN);
 
-      List<UserPermission> perms = AccessControlLists.getUserPermissions
-          (regionEnv.getConfiguration(), tableName);
+      List<UserPermission> perms = AccessControlLists.getUserPermissions(
+        regionEnv.getConfiguration(), tableName);
       return perms;
     } else {
-      throw new CoprocessorException(AccessController.class, "This method " +
-          "can only execute at " +
-          Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
+      throw new CoprocessorException(AccessController.class, "This method "
+          + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
     }
   }
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 23aff8080f3d..09709a6597a5 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -590,8 +591,69 @@ public Object run() throws Exception {
     verifyReadWrite(checkAndPut);
   }
 
+  @Test
+  public void testAppend() throws Exception {
+
+    PrivilegedExceptionAction appendAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        Append append = new Append(TEST_TABLE);
+        append.add(TEST_FAMILY, Bytes.toBytes("qualifier"), Bytes.toBytes("value"));
+        ACCESS_CONTROLLER.preAppend(ObserverContext.createAndPrepare(RCP_ENV, null), append);
+        return null;
+      }
+    };
+
+    verifyAllowed(appendAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW);
+    verifyDenied(appendAction, USER_CREATE, USER_RO, USER_NONE);
+  }
+
   @Test
   public void testGrantRevoke() throws Exception {
+
+    PrivilegedExceptionAction grantAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+        AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+          TEST_TABLE);
+        protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
+            TEST_FAMILY, (byte[]) null, Action.READ));
+        return null;
+      }
+    };
+
+    PrivilegedExceptionAction revokeAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+        AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+          TEST_TABLE);
+        protocol.revoke(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
+            TEST_FAMILY, (byte[]) null, Action.READ));
+        return null;
+      }
+    };
+
+    PrivilegedExceptionAction getPermissionsAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+        AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+          TEST_TABLE);
+        protocol.getUserPermissions(TEST_TABLE);
+        return null;
+      }
+    };
+
+    verifyAllowed(grantAction, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(grantAction, USER_CREATE, USER_RW, USER_RO, USER_NONE);
+
+    verifyAllowed(revokeAction, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(revokeAction, USER_CREATE, USER_RW, USER_RO, USER_NONE);
+
+    verifyAllowed(getPermissionsAction, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(getPermissionsAction, USER_CREATE, USER_RW, USER_RO, USER_NONE);
+  }
+
+  @Test
+  public void testPostGrantRevoke() throws Exception {
     final byte[] tableName = Bytes.toBytes("TempTable");
     final byte[] family1 = Bytes.toBytes("f1");
     final byte[] family2 = Bytes.toBytes("f2");
@@ -823,7 +885,7 @@ private boolean hasFoundUserPermission(UserPermission userPermission, List<UserP
   }
 
   @Test
-  public void testGrantRevokeAtQualifierLevel() throws Exception {
+  public void testPostGrantRevokeAtQualifierLevel() throws Exception {
     final byte[] tableName = Bytes.toBytes("testGrantRevokeAtQualifierLevel");
     final byte[] family1 = Bytes.toBytes("f1");
     final byte[] family2 = Bytes.toBytes("f2");

From a1d10f293930c74a80a0ce10c86c0bfa0f9646ca Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 20 Jun 2012 22:35:26 +0000
Subject: [PATCH 0287/1540] HBASE-6238. Grant on META not taking effect
 (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352358 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessControlLists.java   | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
index 53181c1aace5..345fde07fcb0 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
@@ -397,21 +397,12 @@ static Map<byte[],ListMultimap<String,TablePermission>> loadAll(
    * used for storage.
    * </p>
    */
-  static ListMultimap<String,TablePermission> getTablePermissions(
-      Configuration conf, byte[] tableName)
-  throws IOException {
+  static ListMultimap<String, TablePermission> getTablePermissions(Configuration conf,
+      byte[] tableName) throws IOException {
     if (tableName == null) tableName = ACL_TABLE_NAME;
 
-    /* TODO: -ROOT- and .META. cannot easily be handled because they must be
-     * online before _acl_ table.  Can anything be done here?
-     */
-    if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
-        Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
-      return ArrayListMultimap.create(0,0);
-    }
-
     // for normal user tables, we just read the table row from _acl_
-    ListMultimap<String,TablePermission> perms = ArrayListMultimap.create();
+    ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
     HTable acls = null;
     try {
       acls = new HTable(conf, ACL_TABLE_NAME);
@@ -421,8 +412,8 @@ static ListMultimap<String,TablePermission> getTablePermissions(
       if (!row.isEmpty()) {
         perms = parseTablePermissions(tableName, row);
       } else {
-        LOG.info("No permissions found in "+ACL_TABLE_NAME_STR+
-            " for table "+Bytes.toString(tableName));
+        LOG.info("No permissions found in " + ACL_TABLE_NAME_STR + " for table "
+            + Bytes.toString(tableName));
       }
     } finally {
       if (acls != null) acls.close();

From f10c4a0208f3c140d45216c1e6ff78e721eaa49b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 21 Jun 2012 02:17:31 +0000
Subject: [PATCH 0288/1540] HBASE-6248 Jetty init may fail if directory name
 contains "master" (Dave Revell)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352394 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/InfoServer.java     | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
index 552974502bf9..a1698daf252b 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
@@ -123,8 +123,10 @@ protected String getWebAppsPath() throws IOException {
     // web applications.
     final String master = "master";
     String p = getWebAppsPath(master);
-    int index = p.lastIndexOf(master);
     // Now strip master off the end if it is present
-    return index == -1? p: p.substring(0, index);
+    if(p.endsWith(master)) {
+      return p.substring(0, p.lastIndexOf(master));
+    }
+    return p;
   }
-}
\ No newline at end of file
+}

From f64d838a9d78a436418d0def0d9372afe41964f3 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 21 Jun 2012 18:29:54 +0000
Subject: [PATCH 0289/1540] HBASE-6252. TABLE ADMIN should be allowed to
 relocate regions (Laxman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352645 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 24 ++++++++++---------
 .../security/access/TestAccessController.java | 12 +++++-----
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 39c29fca0d80..601bc7bfe352 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -375,7 +375,7 @@ private void requirePermission(byte[] tableName, byte[] family, byte[] qualifier
     AuthResult result = null;
 
     for (Action permission : permissions) {
-      if (authManager.authorize(user, tableName, null, null, permission)) {
+      if (authManager.authorize(user, tableName, family, qualifier, permission)) {
         result = AuthResult.allow("Table permission granted", user, permission, tableName, family, qualifier);
         break;
       } else {
@@ -623,30 +623,32 @@ public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
       byte[] tableName) throws IOException {}
 
   @Override
-  public void preMove(ObserverContext<MasterCoprocessorEnvironment> c,
-      HRegionInfo region, ServerName srcServer, ServerName destServer)
-    throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+  public void preMove(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo region,
+      ServerName srcServer, ServerName destServer) throws IOException {
+    requirePermission(region.getTableName(), null, null, Action.ADMIN);
   }
+
   @Override
   public void postMove(ObserverContext<MasterCoprocessorEnvironment> c,
       HRegionInfo region, ServerName srcServer, ServerName destServer)
     throws IOException {}
 
   @Override
-  public void preAssign(ObserverContext<MasterCoprocessorEnvironment> c,
-      HRegionInfo regionInfo) throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+  public void preAssign(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo)
+      throws IOException {
+    requirePermission(regionInfo.getTableName(), null, null, Action.ADMIN);
   }
+
   @Override
   public void postAssign(ObserverContext<MasterCoprocessorEnvironment> c,
       HRegionInfo regionInfo) throws IOException {}
 
   @Override
-  public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> c,
-       HRegionInfo regionInfo, boolean force) throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+  public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo,
+      boolean force) throws IOException {
+    requirePermission(regionInfo.getTableName(), null, null, Action.ADMIN);
   }
+
   @Override
   public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> c,
       HRegionInfo regionInfo, boolean force) throws IOException {}
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 09709a6597a5..3691635abd4d 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -338,8 +338,8 @@ public Object run() throws Exception {
       }
     };
 
-    verifyAllowed(action, SUPERUSER, USER_ADMIN);
-    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -356,8 +356,8 @@ public Object run() throws Exception {
       }
     };
 
-    verifyAllowed(action, SUPERUSER, USER_ADMIN);
-    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -374,8 +374,8 @@ public Object run() throws Exception {
       }
     };
 
-    verifyAllowed(action, SUPERUSER, USER_ADMIN);
-    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test

From c431b4b44455e0b203dbb618147fbbcb1adb80a1 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 21 Jun 2012 21:56:14 +0000
Subject: [PATCH 0290/1540] HBASE-6207. Add jitter to client retry timer
 (Elliott Clark)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352711 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ConnectionUtils.java  |  8 ++-
 .../hbase/client/TestConnectionUtils.java     | 56 +++++++++++++++++++
 2 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
index f5d774e1b207..0ffae896fcf7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
@@ -19,12 +19,15 @@
 
 import org.apache.hadoop.hbase.HConstants;
 
+import java.util.Random;
 
 /**
  * Utility used by client connections such as {@link HConnection} and
  * {@link ServerCallable}
  */
 public class ConnectionUtils {
+
+  private static final Random RANDOM = new Random();
   /**
    * Calculate pause time.
    * Built on {@link HConstants#RETRY_BACKOFF}.
@@ -37,6 +40,9 @@ public static long getPauseTime(final long pause, final int tries) {
     if (ntries >= HConstants.RETRY_BACKOFF.length) {
       ntries = HConstants.RETRY_BACKOFF.length - 1;
     }
-    return pause * HConstants.RETRY_BACKOFF[ntries];
+
+    long normalPause = pause * HConstants.RETRY_BACKOFF[ntries];
+    long jitter =  (long)(normalPause * RANDOM.nextFloat() * 0.01f); // 1% possible jitter
+    return normalPause + jitter;
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java b/src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java
new file mode 100644
index 000000000000..4120ec332029
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.junit.Assert.assertTrue;
+
+@Category(SmallTests.class)
+public class TestConnectionUtils {
+
+  @Test
+  public void testRetryTimeJitter() {
+    long[] retries = new long[200];
+    long baseTime = 1000000;  //Larger number than reality to help test randomness.
+    long maxTimeExpected = (long) (baseTime * 1.01f);
+    for (int i = 0; i < retries.length; i++) {
+      retries[i] = ConnectionUtils.getPauseTime(baseTime, 0);
+    }
+
+    Set<Long> retyTimeSet = new TreeSet<Long>();
+    for (long l : retries) {
+      /*make sure that there is some jitter but only 1%*/
+      assertTrue(l >= baseTime);
+      assertTrue(l <= maxTimeExpected);
+      // Add the long to the set
+      retyTimeSet.add(l);
+    }
+
+    //Make sure that most are unique.  some overlap will happen
+    assertTrue(retyTimeSet.size() > (retries.length * 0.80));
+  }
+
+}

From 0989123cf4d4df5db55ec364be81367d6cc42766 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 22 Jun 2012 17:39:13 +0000
Subject: [PATCH 0291/1540] HBASE-6246. Admin.move without specifying
 destination does not go through AccessController (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1352981 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   | 49 ++++++++++---------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9d3799f9aa14..3c0284abbda0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1086,35 +1086,36 @@ public void move(final byte[] encodedRegionName, final byte[] destServerName)
       this.assignmentManager.getAssignment(encodedRegionName);
     if (p == null)
       throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
-    HRegionInfo hri = p.getFirst();
     ServerName dest = null;
     if (destServerName == null || destServerName.length == 0) {
-      LOG.info("Passed destination servername is null/empty so " +
-        "choosing a server at random");
-      this.assignmentManager.clearRegionPlan(hri);
-      // Unassign will reassign it elsewhere choosing random server.
-      this.assignmentManager.unassign(hri);
-    } else {
-      dest = new ServerName(Bytes.toString(destServerName));
-      try {
-        if (this.cpHost != null) {
-          if (this.cpHost.preMove(p.getFirst(), p.getSecond(), dest)) {
-            return;
-          }
-        }
-        RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
-        LOG.info("Added move plan " + rp + ", running balancer");
-        this.assignmentManager.balance(rp);
-        if (this.cpHost != null) {
-          this.cpHost.postMove(p.getFirst(), p.getSecond(), dest);
+      LOG.info("Passed destination servername is null or empty so choosing a server at random");
+      List<ServerName> destServers = this.serverManager.getOnlineServersList();
+      destServers.remove(p.getSecond());
+      // If i have only one RS then destination can be null.
+      dest = balancer.randomAssignment(destServers);
+    }
+    
+    // Now we can do the move
+    RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
+    
+    try {
+      if (this.cpHost != null) {
+        if (this.cpHost.preMove(p.getFirst(), p.getSecond(), dest)) {
+          return;
         }
-      } catch (IOException ioe) {
-        UnknownRegionException ure = new UnknownRegionException(
-            Bytes.toStringBinary(encodedRegionName));
-        ure.initCause(ioe);
-        throw ure;
       }
+      LOG.info("Added move plan " + rp + ", running balancer");
+      this.assignmentManager.balance(rp);
+      if (this.cpHost != null) {
+        this.cpHost.postMove(p.getFirst(), p.getSecond(), dest);
+      }
+    } catch (IOException ioe) {
+      UnknownRegionException ure = new UnknownRegionException(
+          Bytes.toStringBinary(encodedRegionName));
+      ure.initCause(ioe);
+      throw ure;
     }
+
   }
 
   public void createTable(HTableDescriptor hTableDescriptor,

From e37e961d79bbe5d32f60ee331a0d66169d65bb5b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 22 Jun 2012 19:58:16 +0000
Subject: [PATCH 0292/1540] HBASE-6236 Offline meta repair fails if the HBase
 base mount point is on a different cluster/volume than its parent in a ViewFS
 or similar FS (Aditya)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353012 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 29 +++++++++++++------
 .../hbase/util/hbck/OfflineMetaRepair.java    |  5 ++++
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index ef2d13a123f9..548e9d59c6bb 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -145,6 +145,9 @@ public class HBaseFsck {
   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
 
+  private static final String DEFAULT_SIDELINE_DIR = ".hbcktmp-" + 
+                                          System.currentTimeMillis();
+  
   /**********************
    * Internal resources
    **********************/
@@ -155,7 +158,6 @@ public class HBaseFsck {
   private HBaseAdmin admin;
   private HTable meta;
   private ScheduledThreadPoolExecutor executor; // threads to retrieve data from regionservers
-  private long startMillis = System.currentTimeMillis();
 
   /***********
    * Options
@@ -176,6 +178,7 @@ public class HBaseFsck {
   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
+  private Path sidelineDir = null;
 
   private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
@@ -820,7 +823,7 @@ public boolean rebuildMeta(boolean fix) throws IOException,
 
     // we can rebuild, move old root and meta out of the way and start
     LOG.info("HDFS regioninfo's seems good.  Sidelining old .META.");
-    sidelineOldRootAndMeta();
+    Path backupDir = sidelineOldRootAndMeta();
 
     LOG.info("Creating new .META.");
     HRegion meta = createNewRootAndMeta();
@@ -836,6 +839,7 @@ public boolean rebuildMeta(boolean fix) throws IOException,
     meta.close();
     meta.getLog().closeAndDelete();
     LOG.info("Success! .META. table rebuilt.");
+    LOG.info("Old -ROOT- and .META. are moved into " + backupDir);
     return true;
   }
 
@@ -859,11 +863,11 @@ private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
   }
 
   private Path getSidelineDir() throws IOException {
-    Path hbaseDir = FSUtils.getRootDir(conf);
-    Path hbckDir = new Path(hbaseDir.getParent(), "hbck");
-    Path backupDir = new Path(hbckDir, hbaseDir.getName() + "-"
-        + startMillis);
-    return backupDir;
+    if (sidelineDir == null) {
+      Path hbaseDir = FSUtils.getRootDir(conf);
+      sidelineDir = new Path(hbaseDir, DEFAULT_SIDELINE_DIR);
+    }
+    return sidelineDir;
   }
 
   /**
@@ -961,8 +965,7 @@ Path sidelineOldRootAndMeta() throws IOException {
     // put current -ROOT- and .META. aside.
     Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
     FileSystem fs = hbaseDir.getFileSystem(conf);
-    Path backupDir = new Path(hbaseDir.getParent(), hbaseDir.getName() + "-"
-        + startMillis);
+    Path backupDir = getSidelineDir();
     fs.mkdirs(backupDir);
 
     sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir);
@@ -2984,6 +2987,14 @@ public void setTimeLag(long seconds) {
     timelag = seconds * 1000; // convert to milliseconds
   }
 
+  /**
+   * 
+   * @param sidelineDir - HDFS path to sideline data
+   */
+  public void setSidelineDir(String sidelineDir) {
+    this.sidelineDir = new Path(sidelineDir);
+  }
+  
   protected static void printUsageAndExit() {
     System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index 80846b3d3c4a..937809ca3ca5 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -47,6 +47,8 @@ protected static void printUsageAndExit() {
     System.err
         .println("   -details          Display full report of all regions.");
     System.err.println("   -base <hdfs://>   Base Hbase Data directory");
+    System.err
+        .println("   -backup <hdfs://> HDFS path to backup existing meta and root.");
     System.err.println("   -fix              Auto fix as many problems as possible");
     System.err.println("   -fixHoles         Auto fix as region holes");
     Runtime.getRuntime().exit(-2);
@@ -81,6 +83,9 @@ public static void main(String[] args) throws Exception {
         conf.set(HConstants.HBASE_DIR, path);
         conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
         conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR));
+      } else if (cmd.equals("-backup")) {
+        i++;
+        fsck.setSidelineDir(args[i]);        
       } else if (cmd.equals("-fixHoles")) {
         fixHoles = true;
       } else if (cmd.equals("-fix")) {

From e69a179e6858ce5bc103b3b143061e7496aaa980 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 22 Jun 2012 20:01:34 +0000
Subject: [PATCH 0293/1540] HBASE-6236 revert upon Jimmy's request.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353014 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 29 ++++++-------------
 .../hbase/util/hbck/OfflineMetaRepair.java    |  5 ----
 2 files changed, 9 insertions(+), 25 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 548e9d59c6bb..ef2d13a123f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -145,9 +145,6 @@ public class HBaseFsck {
   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
 
-  private static final String DEFAULT_SIDELINE_DIR = ".hbcktmp-" + 
-                                          System.currentTimeMillis();
-  
   /**********************
    * Internal resources
    **********************/
@@ -158,6 +155,7 @@ public class HBaseFsck {
   private HBaseAdmin admin;
   private HTable meta;
   private ScheduledThreadPoolExecutor executor; // threads to retrieve data from regionservers
+  private long startMillis = System.currentTimeMillis();
 
   /***********
    * Options
@@ -178,7 +176,6 @@ public class HBaseFsck {
   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
-  private Path sidelineDir = null;
 
   private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
@@ -823,7 +820,7 @@ public boolean rebuildMeta(boolean fix) throws IOException,
 
     // we can rebuild, move old root and meta out of the way and start
     LOG.info("HDFS regioninfo's seems good.  Sidelining old .META.");
-    Path backupDir = sidelineOldRootAndMeta();
+    sidelineOldRootAndMeta();
 
     LOG.info("Creating new .META.");
     HRegion meta = createNewRootAndMeta();
@@ -839,7 +836,6 @@ public boolean rebuildMeta(boolean fix) throws IOException,
     meta.close();
     meta.getLog().closeAndDelete();
     LOG.info("Success! .META. table rebuilt.");
-    LOG.info("Old -ROOT- and .META. are moved into " + backupDir);
     return true;
   }
 
@@ -863,11 +859,11 @@ private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
   }
 
   private Path getSidelineDir() throws IOException {
-    if (sidelineDir == null) {
-      Path hbaseDir = FSUtils.getRootDir(conf);
-      sidelineDir = new Path(hbaseDir, DEFAULT_SIDELINE_DIR);
-    }
-    return sidelineDir;
+    Path hbaseDir = FSUtils.getRootDir(conf);
+    Path hbckDir = new Path(hbaseDir.getParent(), "hbck");
+    Path backupDir = new Path(hbckDir, hbaseDir.getName() + "-"
+        + startMillis);
+    return backupDir;
   }
 
   /**
@@ -965,7 +961,8 @@ Path sidelineOldRootAndMeta() throws IOException {
     // put current -ROOT- and .META. aside.
     Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
     FileSystem fs = hbaseDir.getFileSystem(conf);
-    Path backupDir = getSidelineDir();
+    Path backupDir = new Path(hbaseDir.getParent(), hbaseDir.getName() + "-"
+        + startMillis);
     fs.mkdirs(backupDir);
 
     sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir);
@@ -2987,14 +2984,6 @@ public void setTimeLag(long seconds) {
     timelag = seconds * 1000; // convert to milliseconds
   }
 
-  /**
-   * 
-   * @param sidelineDir - HDFS path to sideline data
-   */
-  public void setSidelineDir(String sidelineDir) {
-    this.sidelineDir = new Path(sidelineDir);
-  }
-  
   protected static void printUsageAndExit() {
     System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index 937809ca3ca5..80846b3d3c4a 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -47,8 +47,6 @@ protected static void printUsageAndExit() {
     System.err
         .println("   -details          Display full report of all regions.");
     System.err.println("   -base <hdfs://>   Base Hbase Data directory");
-    System.err
-        .println("   -backup <hdfs://> HDFS path to backup existing meta and root.");
     System.err.println("   -fix              Auto fix as many problems as possible");
     System.err.println("   -fixHoles         Auto fix as region holes");
     Runtime.getRuntime().exit(-2);
@@ -83,9 +81,6 @@ public static void main(String[] args) throws Exception {
         conf.set(HConstants.HBASE_DIR, path);
         conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
         conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR));
-      } else if (cmd.equals("-backup")) {
-        i++;
-        fsck.setSidelineDir(args[i]);        
       } else if (cmd.equals("-fixHoles")) {
         fixHoles = true;
       } else if (cmd.equals("-fix")) {

From bd7cd8f137bc001d6ba769c982c6d65af625e76c Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 23 Jun 2012 00:40:08 +0000
Subject: [PATCH 0294/1540] HBASE-5630 hbck should disable the balancer using
 synchronousBalanceSwitch (Gregory Chanan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353054 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 16 +++++++++++++
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  4 ++--
 src/main/ruby/hbase/admin.rb                  |  3 ++-
 .../TestSplitTransactionOnCluster.java        | 24 +++++++++----------
 4 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 3e45154daad0..a112f0dac4f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1364,12 +1364,28 @@ public void unassign(final byte [] regionName, final boolean force)
    * Turn the load balancer on or off.
    * @param b If true, enable balancer. If false, disable balancer.
    * @return Previous balancer value
+   * @deprecated use setBalancerRunning(boolean, boolean) instead
    */
+  @Deprecated
   public boolean balanceSwitch(final boolean b)
   throws MasterNotRunningException, ZooKeeperConnectionException {
     return getMaster().balanceSwitch(b);
   }
 
+  /**
+   * Turn the load balancer on or off.
+   * @param on If true, enable balancer. If false, disable balancer.
+   * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
+   * @return Previous balancer value
+   */
+  public boolean setBalancerRunning(final boolean on, final boolean synchronous)
+  throws MasterNotRunningException, ZooKeeperConnectionException {
+    if (synchronous == false) {
+      return balanceSwitch(on);
+    }
+    return getMaster().synchronousBalanceSwitch(on);
+  }
+
   /**
    * Invoke the balancer.  Will run the balancer and if regions to move, it will
    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index ef2d13a123f9..96c51e82fcd2 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -384,12 +384,12 @@ public int onlineHbck() throws IOException, KeeperException, InterruptedExceptio
     offlineHdfsIntegrityRepair();
 
     // turn the balancer off
-    boolean oldBalancer = admin.balanceSwitch(false);
+    boolean oldBalancer = admin.setBalancerRunning(false, true);
     try {
       onlineConsistencyRepair();
     }
     finally {
-      admin.balanceSwitch(oldBalancer);
+      admin.setBalancerRunning(oldBalancer, false);
     }
 
     // Print table summary
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index f761d21d7bc8..a70e05e18053 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -88,7 +88,8 @@ def balancer()
     # Enable/disable balancer
     # Returns previous balancer switch setting.
     def balance_switch(enableDisable)
-      @admin.balanceSwitch(java.lang.Boolean::valueOf(enableDisable))
+      @admin.setBalancerRunning(
+        java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false))
     end
 
     #----------------------------------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index b7694ef43916..1c0806bd222c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -117,7 +117,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
 
     // Turn off balancer so it doesn't cut in and mess up our placements.
-    this.admin.balanceSwitch(false);
+    this.admin.setBalancerRunning(false, true);
     // Turn off the meta scanner so it don't remove parent on us.
     cluster.getMaster().setCatalogJanitorEnabled(false);
     try {
@@ -168,7 +168,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     } finally {
       // Set this flag back.
       SplitRegionHandler.TEST_SKIP = false;
-      admin.balanceSwitch(true);
+      admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }
@@ -187,7 +187,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
 
     // Turn off balancer so it doesn't cut in and mess up our placements.
-    this.admin.balanceSwitch(false);
+    this.admin.setBalancerRunning(false, true);
     // Turn off the meta scanner so it don't remove parent on us.
     cluster.getMaster().setCatalogJanitorEnabled(false);
     try {
@@ -220,7 +220,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       assertTrue(daughters.size() >= 2);
       // OK, so split happened after we cleared the blocking node.
     } finally {
-      admin.balanceSwitch(true);
+      admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }
@@ -246,7 +246,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
 
     // Turn off balancer so it doesn't cut in and mess up our placements.
-    this.admin.balanceSwitch(false);
+    this.admin.setBalancerRunning(false, true);
     // Turn off the meta scanner so it don't remove parent on us.
     cluster.getMaster().setCatalogJanitorEnabled(false);
     try {
@@ -278,7 +278,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
         assertTrue(daughters.contains(r));
       }
     } finally {
-      admin.balanceSwitch(true);
+      admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }
@@ -303,7 +303,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
 
     // Turn off balancer so it doesn't cut in and mess up our placements.
-    this.admin.balanceSwitch(false);
+    this.admin.setBalancerRunning(false, true);
     // Turn off the meta scanner so it don't remove parent on us.
     cluster.getMaster().setCatalogJanitorEnabled(false);
     try {
@@ -353,7 +353,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
         assertTrue(daughters.contains(r));
       }
     } finally {
-      admin.balanceSwitch(true);
+      admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }
@@ -384,7 +384,7 @@ public void testMasterRestartWhenSplittingIsPartial()
     // Turn off the meta scanner so it don't remove parent on us.
     cluster.getMaster().setCatalogJanitorEnabled(false);
     // Turn off balancer so it doesn't cut in and mess up our placements.
-    this.admin.balanceSwitch(false);
+    this.admin.setBalancerRunning(false, true);
     
     try {
       // Add a bit of load up into the table so splittable.
@@ -430,7 +430,7 @@ public void testMasterRestartWhenSplittingIsPartial()
     } finally {
       // Set this flag back.
       SplitRegionHandler.TEST_SKIP = false;
-      admin.balanceSwitch(true);
+      admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }
@@ -464,7 +464,7 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
 
     // Turn off balancer so it doesn't cut in and mess up our placements.
-    this.admin.balanceSwitch(false);
+    this.admin.setBalancerRunning(false, true);
     // Turn off the meta scanner so it don't remove parent on us.
     cluster.getMaster().setCatalogJanitorEnabled(false);
     try {
@@ -511,7 +511,7 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
     } finally {
       // Set this flag back.
       SplitRegionHandler.TEST_SKIP = false;
-      this.admin.balanceSwitch(true);
+      this.admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
     }
   }

From 435d75bf767d8494c186b941aadb72c869ac92fe Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 23 Jun 2012 01:54:12 +0000
Subject: [PATCH 0295/1540] HBASE-6224. Add pre and post coprocessor hooks for
 bulk load (Francis Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353060 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/BaseRegionObserver.java | 12 ++++
 .../hbase/coprocessor/RegionObserver.java     | 25 +++++++++
 .../hbase/regionserver/HRegionServer.java     | 13 ++++-
 .../regionserver/RegionCoprocessorHost.java   | 55 +++++++++++++++++++
 .../hadoop/hbase/regionserver/StoreFile.java  |  2 +-
 .../hbase/regionserver/StoreFileScanner.java  |  2 +-
 .../coprocessor/SimpleRegionObserver.java     | 50 +++++++++++++++++
 .../TestRegionObserverInterface.java          | 55 +++++++++++++++++++
 8 files changed, 211 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index e6043d856d7d..2d95555ac9f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Pair;
 
 import java.io.IOException;
 
@@ -274,4 +275,15 @@ public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRe
   public void postWALRestore(ObserverContext<RegionCoprocessorEnvironment> env,
       HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException {
   }
+
+  @Override
+  public void preBulkLoadHFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    List<Pair<byte[], String>> familyPaths) throws IOException {
+  }
+
+  @Override
+  public boolean postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
+    List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException {
+    return hasLoaded;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 0e6310946aad..5f80ed302421 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 
 import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hbase.util.Pair;
 
 /**
  * Coprocessors implement this interface to observe and mediate client actions
@@ -651,4 +652,28 @@ void preWALRestore(final ObserverContext<RegionCoprocessorEnvironment> ctx,
    */
   void postWALRestore(final ObserverContext<RegionCoprocessorEnvironment> ctx,
       HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException;
+
+  /**
+   * Called before bulkLoadHFile. Users can create a StoreFile instance to
+   * access the contents of a HFile.
+   *
+   * @param ctx
+   * @param familyPaths pairs of { CF, HFile path } submitted for bulk load. Adding
+   * or removing from this list will add or remove HFiles to be bulk loaded.
+   * @throws IOException
+   */
+  void preBulkLoadHFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    List<Pair<byte[], String>> familyPaths) throws IOException;
+
+  /**
+   * Called after bulkLoadHFile.
+   *
+   * @param ctx
+   * @param familyPaths pairs of { CF, HFile path } submitted for bulk load
+   * @param hasLoaded whether the bulkLoad was successful
+   * @return the new value of hasLoaded
+   * @throws IOException
+   */
+  boolean postBulkLoadHFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index d7a779d40e86..91c8aecdacfa 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2662,7 +2662,18 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
       byte[] regionName) throws IOException {
     checkOpen();
     HRegion region = getRegion(regionName);
-    return region.bulkLoadHFiles(familyPaths);
+    boolean bypass = false;
+    if (region.getCoprocessorHost() != null) {
+      bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
+    }
+    boolean loaded = false;
+    if (!bypass) {
+      loaded = region.bulkLoadHFiles(familyPaths);
+    }
+    if (region.getCoprocessorHost() != null) {
+      loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
+    }
+    return loaded;
   }
 
   Map<String, Integer> rowlocks = new ConcurrentHashMap<String, Integer>();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 44db31db394d..a550d139174b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.util.StringUtils;
 
 import java.io.IOException;
@@ -1267,4 +1268,58 @@ public void postWALRestore(HRegionInfo info, HLogKey logKey,
       }
     }
   }
+
+  /**
+   * @param familyPaths pairs of { CF, file path } submitted for bulk load
+   * @return true if the default operation should be bypassed
+   * @throws IOException
+   */
+  public boolean preBulkLoadHFile(List<Pair<byte[], String>> familyPaths) throws IOException {
+    boolean bypass = false;
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preBulkLoadHFile(ctx, familyPaths);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+
+    return bypass;
+  }
+
+  /**
+   * @param familyPaths pairs of { CF, file path } submitted for bulk load
+   * @param hasLoaded whether load was successful or not
+   * @return the possibly modified value of hasLoaded
+   * @throws IOException
+   */
+  public boolean postBulkLoadHFile(List<Pair<byte[], String>> familyPaths, boolean hasLoaded)
+      throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          hasLoaded = ((RegionObserver)env.getInstance()).postBulkLoadHFile(ctx,
+            familyPaths, hasLoaded);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+
+    return hasLoaded;
+  }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index d12d0fdb49b2..76566e565cff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -231,7 +231,7 @@ public void setMaxMemstoreTS(long maxMemstoreTS) {
    * @param dataBlockEncoder data block encoding algorithm.
    * @throws IOException When opening the reader fails.
    */
-  StoreFile(final FileSystem fs,
+  public StoreFile(final FileSystem fs,
             final Path p,
             final Configuration conf,
             final CacheConfig cacheConf,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index 49f8e813c384..363bb62fc14c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -38,7 +38,7 @@
  * KeyValueScanner adaptor over the Reader.  It also provides hooks into
  * bloom filter things.
  */
-class StoreFileScanner implements KeyValueScanner {
+public class StoreFileScanner implements KeyValueScanner {
   static final Log LOG = LogFactory.getLog(Store.class);
 
   // the reader it comes from:
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index dacb9361cb3c..a691bacc4366 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -45,6 +47,7 @@
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 
 /**
  * A sample region observer that tests the RegionObserver interface.
@@ -85,6 +88,8 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   boolean hadPostScannerClose = false;
   boolean hadPreScannerOpen = false;
   boolean hadPostScannerOpen = false;
+  boolean hadPreBulkLoadHFile = false;
+  boolean hadPostBulkLoadHFile = false;
 
   @Override
   public void preOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
@@ -384,6 +389,43 @@ public Result postIncrement(final ObserverContext<RegionCoprocessorEnvironment>
     return result;
   }
 
+  @Override
+  public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
+                               List<Pair<byte[], String>> familyPaths) throws IOException {
+    RegionCoprocessorEnvironment e = ctx.getEnvironment();
+    assertNotNull(e);
+    assertNotNull(e.getRegion());
+    if (Arrays.equals(e.getRegion().getTableDesc().getName(),
+        TestRegionObserverInterface.TEST_TABLE)) {
+      assertNotNull(familyPaths);
+      assertEquals(1,familyPaths.size());
+      assertArrayEquals(familyPaths.get(0).getFirst(), TestRegionObserverInterface.A);
+      String familyPath = familyPaths.get(0).getSecond();
+      String familyName = Bytes.toString(TestRegionObserverInterface.A);
+      assertEquals(familyPath.substring(familyPath.length()-familyName.length()-1),"/"+familyName);
+    }
+    hadPreBulkLoadHFile = true;
+  }
+
+  @Override
+  public boolean postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
+                                   List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException {
+    RegionCoprocessorEnvironment e = ctx.getEnvironment();
+    assertNotNull(e);
+    assertNotNull(e.getRegion());
+    if (Arrays.equals(e.getRegion().getTableDesc().getName(),
+        TestRegionObserverInterface.TEST_TABLE)) {
+      assertNotNull(familyPaths);
+      assertEquals(1,familyPaths.size());
+      assertArrayEquals(familyPaths.get(0).getFirst(), TestRegionObserverInterface.A);
+      String familyPath = familyPaths.get(0).getSecond();
+      String familyName = Bytes.toString(TestRegionObserverInterface.A);
+      assertEquals(familyPath.substring(familyPath.length()-familyName.length()-1),"/"+familyName);
+    }
+    hadPostBulkLoadHFile = true;
+    return hasLoaded;
+  }
+
   public boolean hadPreGet() {
     return hadPreGet;
   }
@@ -430,4 +472,12 @@ public boolean wasScannerOpenCalled() {
   public boolean hadDeleted() {
     return hadPreDeleted && hadPostDeleted;
   }
+
+  public boolean hadPostBulkLoadHFile() {
+    return hadPostBulkLoadHFile;
+  }
+
+  public boolean hadPreBulkLoadHFile() {
+    return hadPreBulkLoadHFile;
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index 1b3b6df3d329..d378ac98ccae 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -29,8 +29,13 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
@@ -416,6 +421,37 @@ public void testCompactionOverride() throws Exception {
     table.close();
   }
 
+  @Test
+  public void bulkLoadHFileTest() throws Exception {
+    String testName = TestRegionObserverInterface.class.getName()+".bulkLoadHFileTest";
+    byte[] tableName = TEST_TABLE;
+    Configuration conf = util.getConfiguration();
+    HTable table = util.createTable(tableName, new byte[][] {A, B, C});
+
+    verifyMethodResult(SimpleRegionObserver.class,
+        new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
+        tableName,
+        new Boolean[] {false, false}
+    );
+
+    FileSystem fs = util.getTestFileSystem();
+    final Path dir = util.getDataTestDir(testName).makeQualified(fs);
+    Path familyDir = new Path(dir, Bytes.toString(A));
+
+    createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);
+
+    //Bulk load
+    new LoadIncrementalHFiles(conf).doBulkLoad(dir, new HTable(conf, tableName));
+
+    verifyMethodResult(SimpleRegionObserver.class,
+        new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
+        tableName,
+        new Boolean[] {true, true}
+    );
+    util.deleteTable(tableName);
+    table.close();
+  }
+
   // check each region whether the coprocessor upcalls are called or not.
   private void verifyMethodResult(Class c, String methodName[], byte[] tableName,
                                   Object value[]) throws IOException {
@@ -444,6 +480,25 @@ private void verifyMethodResult(Class c, String methodName[], byte[] tableName,
     }
   }
 
+  private static void createHFile(
+      Configuration conf,
+      FileSystem fs, Path path,
+      byte[] family, byte[] qualifier) throws IOException {
+    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
+        .withPath(fs, path)
+        .withComparator(KeyValue.KEY_COMPARATOR)
+        .create();
+    long now = System.currentTimeMillis();
+    try {
+      for (int i =1;i<=9;i++) {
+        KeyValue kv = new KeyValue(Bytes.toBytes(i+""), family, qualifier, now, Bytes.toBytes(i+""));
+        writer.append(kv);
+      }
+    } finally {
+      writer.close();
+    }
+  }
+
   private static byte [][] makeN(byte [] base, int n) {
     byte [][] ret = new byte[n][];
     for(int i=0;i<n;i++) {

From e36b94fdcabf6d27188336ece131956ac9254573 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 23 Jun 2012 04:10:46 +0000
Subject: [PATCH 0296/1540] Amend HBASE-6246. Admin.move without specifying
 destination does not go through AccessController

The previous committed change for HBASE-6246 accidentally removed logic
and the omission was caught by TestSplitTransactionOnCluster failures.


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353063 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3c0284abbda0..1bb5c18b37a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1093,6 +1093,8 @@ public void move(final byte[] encodedRegionName, final byte[] destServerName)
       destServers.remove(p.getSecond());
       // If i have only one RS then destination can be null.
       dest = balancer.randomAssignment(destServers);
+    } else {
+      dest = new ServerName(Bytes.toString(destServerName));
     }
     
     // Now we can do the move

From 0e5ea977df92412711be13ea5e2f3fd7dd63e83b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 23 Jun 2012 04:24:55 +0000
Subject: [PATCH 0297/1540] HBASE-6236 Offline meta repair fails if the HBase
 base mount point is on a different cluster/volume than its parent in a ViewFS
 or similar FS (Aditya)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353064 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  6 +++-
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 27 ++++++++++++------
 .../hbase/util/hbck/OfflineMetaRepair.java    | 28 ++++++++++++++-----
 3 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index b6af98a8ef53..4468a973e7f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -223,6 +223,9 @@ public enum OperationStatusCode {
   /** Like the previous, but for old logs that are about to be deleted */
   public static final String HREGION_OLDLOGDIR_NAME = ".oldlogs";
 
+  /** Used by HBCK to sideline backup data */
+  public static final String HBCK_SIDELINEDIR_NAME = ".hbck";
+
   /** Used to construct the name of the compaction directory during compaction */
   public static final String HREGION_COMPACTIONDIR_NAME = "compaction.dir";
 
@@ -586,7 +589,8 @@ public static enum Modify {
   public static final List<String> HBASE_NON_USER_TABLE_DIRS = new ArrayList<String>(
       Arrays.asList(new String[]{ HREGION_LOGDIR_NAME, HREGION_OLDLOGDIR_NAME,
           CORRUPT_DIR_NAME, Bytes.toString(META_TABLE_NAME),
-          Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME }));
+          Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME, 
+          HBCK_SIDELINEDIR_NAME }));
 
   public static final Pattern CP_HTD_ATTR_KEY_PATTERN = Pattern.compile
       ("^coprocessor\\$([0-9]+)$", Pattern.CASE_INSENSITIVE);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 96c51e82fcd2..9f9a3dd35fbd 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -176,6 +176,7 @@ public class HBaseFsck {
   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
+  private Path sidelineDir = null;
 
   private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
@@ -820,7 +821,7 @@ public boolean rebuildMeta(boolean fix) throws IOException,
 
     // we can rebuild, move old root and meta out of the way and start
     LOG.info("HDFS regioninfo's seems good.  Sidelining old .META.");
-    sidelineOldRootAndMeta();
+    Path backupDir = sidelineOldRootAndMeta();
 
     LOG.info("Creating new .META.");
     HRegion meta = createNewRootAndMeta();
@@ -836,6 +837,7 @@ public boolean rebuildMeta(boolean fix) throws IOException,
     meta.close();
     meta.getLog().closeAndDelete();
     LOG.info("Success! .META. table rebuilt.");
+    LOG.info("Old -ROOT- and .META. are moved into " + backupDir);
     return true;
   }
 
@@ -859,11 +861,13 @@ private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
   }
 
   private Path getSidelineDir() throws IOException {
-    Path hbaseDir = FSUtils.getRootDir(conf);
-    Path hbckDir = new Path(hbaseDir.getParent(), "hbck");
-    Path backupDir = new Path(hbckDir, hbaseDir.getName() + "-"
-        + startMillis);
-    return backupDir;
+    if (sidelineDir == null) {
+      Path hbaseDir = FSUtils.getRootDir(conf);
+      Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
+      sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
+          + startMillis);
+    }
+    return sidelineDir;
   }
 
   /**
@@ -961,8 +965,7 @@ Path sidelineOldRootAndMeta() throws IOException {
     // put current -ROOT- and .META. aside.
     Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
     FileSystem fs = hbaseDir.getFileSystem(conf);
-    Path backupDir = new Path(hbaseDir.getParent(), hbaseDir.getName() + "-"
-        + startMillis);
+    Path backupDir = getSidelineDir();
     fs.mkdirs(backupDir);
 
     sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir);
@@ -2984,6 +2987,14 @@ public void setTimeLag(long seconds) {
     timelag = seconds * 1000; // convert to milliseconds
   }
 
+  /**
+   * 
+   * @param sidelineDir - HDFS path to sideline data
+   */
+  public void setSidelineDir(String sidelineDir) {
+    this.sidelineDir = new Path(sidelineDir);
+  }
+  
   protected static void printUsageAndExit() {
     System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index 80846b3d3c4a..67af09f363f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -42,13 +42,15 @@ public class OfflineMetaRepair {
   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
 
   protected static void printUsageAndExit() {
-    System.err.println("Usage: OfflineMetaRepair [opts] ");
-    System.err.println(" where [opts] are:");
-    System.err
-        .println("   -details          Display full report of all regions.");
-    System.err.println("   -base <hdfs://>   Base Hbase Data directory");
-    System.err.println("   -fix              Auto fix as many problems as possible");
-    System.err.println("   -fixHoles         Auto fix as region holes");
+    StringBuilder sb = new StringBuilder();
+    sb.append("Usage: OfflineMetaRepair [opts]\n").
+       append(" where [opts] are:\n").
+       append("   -details               Display full report of all regions.\n").
+       append("   -base <hdfs://>        Base Hbase Data directory.\n").
+       append("   -sidelineDir <hdfs://> HDFS path to backup existing meta and root.\n").
+       append("   -fix                   Auto fix as many problems as possible.\n").
+       append("   -fixHoles              Auto fix as region holes.");
+    System.err.println(sb.toString());
     Runtime.getRuntime().exit(-2);
   }
 
@@ -75,12 +77,24 @@ public static void main(String[] args) throws Exception {
       if (cmd.equals("-details")) {
         fsck.setDisplayFullReport();
       } else if (cmd.equals("-base")) {
+        if (i == args.length - 1) {
+          System.err.println("OfflineMetaRepair: -base needs an HDFS path.");
+          printUsageAndExit();
+        }        
         // update hbase root dir to user-specified base
         i++;
         String path = args[i];
         conf.set(HConstants.HBASE_DIR, path);
         conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR));
         conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR));
+      } else if (cmd.equals("-sidelineDir")) {
+        if (i == args.length - 1) {
+          System.err.println("OfflineMetaRepair: -sidelineDir needs an HDFS path.");
+          printUsageAndExit();
+        }
+        // set the hbck sideline dir to user-specified one
+        i++;
+        fsck.setSidelineDir(args[i]);        
       } else if (cmd.equals("-fixHoles")) {
         fixHoles = true;
       } else if (cmd.equals("-fix")) {

From c4bc86057256045888ef33bbb1ebf5f0bfd0a31b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 23 Jun 2012 04:43:27 +0000
Subject: [PATCH 0298/1540] HBASE-6236 Addendum adds -sidelineDir option to
 hbck (Aditya)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353066 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HConstants.java |  2 +-
 .../org/apache/hadoop/hbase/util/HBaseFsck.java  | 16 ++++++++++++----
 .../hbase/util/hbck/OfflineMetaRepair.java       |  4 ++--
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 4468a973e7f7..7f152b4bbb81 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -589,7 +589,7 @@ public static enum Modify {
   public static final List<String> HBASE_NON_USER_TABLE_DIRS = new ArrayList<String>(
       Arrays.asList(new String[]{ HREGION_LOGDIR_NAME, HREGION_OLDLOGDIR_NAME,
           CORRUPT_DIR_NAME, Bytes.toString(META_TABLE_NAME),
-          Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME, 
+          Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME,
           HBCK_SIDELINEDIR_NAME }));
 
   public static final Pattern CP_HTD_ATTR_KEY_PATTERN = Pattern.compile
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 9f9a3dd35fbd..cf356e43cff1 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2994,19 +2994,20 @@ public void setTimeLag(long seconds) {
   public void setSidelineDir(String sidelineDir) {
     this.sidelineDir = new Path(sidelineDir);
   }
-  
+
   protected static void printUsageAndExit() {
     System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
     System.err.println("   -help Display help options (this)");
     System.err.println("   -details Display full report of all regions.");
-    System.err.println("   -timelag {timeInSeconds}  Process only regions that " +
+    System.err.println("   -timelag <timeInSeconds>  Process only regions that " +
                        " have not experienced any metadata updates in the last " +
-                       " {{timeInSeconds} seconds.");
-    System.err.println("   -sleepBeforeRerun {timeInSeconds} Sleep this many seconds" +
+                       " <timeInSeconds> seconds.");
+    System.err.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
         " before checking if the fix worked if run with -fix");
     System.err.println("   -summary Print only summary of the tables and status.");
     System.err.println("   -metaonly Only check the state of ROOT and META tables.");
+    System.err.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta and root.");
 
     System.err.println("  Repair options: (expert features, use with caution!)");
     System.err.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
@@ -3076,6 +3077,13 @@ public static void main(String[] args) throws Exception {
           printUsageAndExit();
         }
         i++;
+      } else if (cmd.equals("-sidelineDir")) {
+        if (i == args.length - 1) {
+          System.err.println("HBaseFsck: -sidelineDir needs a value.");
+          printUsageAndExit();
+        }
+        i++;
+        fsck.setSidelineDir(args[i]);
       } else if (cmd.equals("-fix")) {
         System.err.println("This option is deprecated, please use " +
           "-fixAssignments instead.");
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index 67af09f363f4..c586fd10d83c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -80,7 +80,7 @@ public static void main(String[] args) throws Exception {
         if (i == args.length - 1) {
           System.err.println("OfflineMetaRepair: -base needs an HDFS path.");
           printUsageAndExit();
-        }        
+        }
         // update hbase root dir to user-specified base
         i++;
         String path = args[i];
@@ -94,7 +94,7 @@ public static void main(String[] args) throws Exception {
         }
         // set the hbck sideline dir to user-specified one
         i++;
-        fsck.setSidelineDir(args[i]);        
+        fsck.setSidelineDir(args[i]);
       } else if (cmd.equals("-fixHoles")) {
         fixHoles = true;
       } else if (cmd.equals("-fix")) {

From e22b13d6cfc61cc964b27fcc9769fd313a79ef90 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 25 Jun 2012 17:04:07 +0000
Subject: [PATCH 0299/1540] HBASE-5360 [uberhbck] Add options for how to handle
 offline split parents, addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353659 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 2e87c2704b99..69be013c8fd3 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1046,7 +1046,7 @@ public void testLingeringSplitParent() throws Exception {
         Writables.getBytes(hri));
       p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
         Writables.getBytes(a));
-      p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
+      p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
         Writables.getBytes(b));
       meta.put(p);
       meta.flushCommits();

From 3c396366f05c60a0e3ce9e1cf9ad16ecf77fcbc3 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 25 Jun 2012 18:09:34 +0000
Subject: [PATCH 0300/1540] HBASE-5875 Process RIT and Master restart may
 remove an online server considering it as a dead server

Submitted by:Rajesh
Reviewed by:Ram, Ted, Stack


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353690 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   | 42 +++++++++++++------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 1bb5c18b37a6..00a94e5bc47c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -625,18 +625,20 @@ int assignRootAndMeta(MonitoredTask status)
     boolean rit = this.assignmentManager.
       processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO);
     ServerName currentRootServer = null;
-    if (!catalogTracker.verifyRootRegionLocation(timeout)) {
+    boolean rootRegionLocation = catalogTracker.verifyRootRegionLocation(timeout);
+    if (!rit && !rootRegionLocation) {
       currentRootServer = this.catalogTracker.getRootLocation();
       splitLogAndExpireIfOnline(currentRootServer);
       this.assignmentManager.assignRoot();
-      this.catalogTracker.waitForRoot();
-      //This guarantees that the transition has completed
-      this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);
+      waitForRootAssignment();
+      assigned++;
+    } else if (rit && !rootRegionLocation) {
+      waitForRootAssignment();
       assigned++;
     } else {
-      // Region already assigned.  We didn't assign it.  Add to in-memory state.
+      // Region already assigned. We didn't assign it. Add to in-memory state.
       this.assignmentManager.regionOnline(HRegionInfo.ROOT_REGIONINFO,
-        this.catalogTracker.getRootLocation());
+          this.catalogTracker.getRootLocation());
     }
     // Enable the ROOT table if on process fail over the RS containing ROOT
     // was active.
@@ -648,7 +650,8 @@ int assignRootAndMeta(MonitoredTask status)
     status.setStatus("Assigning META region");
     rit = this.assignmentManager.
       processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
-    if (!this.catalogTracker.verifyMetaRegionLocation(timeout)) {
+    boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
+    if (!rit && !metaRegionLocation) {
       ServerName currentMetaServer =
         this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
       if (currentMetaServer != null
@@ -656,11 +659,10 @@ int assignRootAndMeta(MonitoredTask status)
         splitLogAndExpireIfOnline(currentMetaServer);
       }
       assignmentManager.assignMeta();
-      enableServerShutdownHandler();
-      this.catalogTracker.waitForMeta();
-      // Above check waits for general meta availability but this does not
-      // guarantee that the transition has completed
-      this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
+      enableSSHandWaitForMeta();
+      assigned++;
+    } else if (rit && !metaRegionLocation) {
+      enableSSHandWaitForMeta();
       assigned++;
     } else {
       // Region already assigned.  We didnt' assign it.  Add to in-memory state.
@@ -674,6 +676,22 @@ int assignRootAndMeta(MonitoredTask status)
     return assigned;
   }
 
+  private void enableSSHandWaitForMeta() throws IOException,
+      InterruptedException {
+    enableServerShutdownHandler();
+    this.catalogTracker.waitForMeta();
+    // Above check waits for general meta availability but this does not
+    // guarantee that the transition has completed
+    this.assignmentManager
+        .waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
+  }
+
+  private void waitForRootAssignment() throws InterruptedException {
+    this.catalogTracker.waitForRoot();
+    // This guarantees that the transition has completed
+    this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);
+  }
+
   private void enableCatalogTables(String catalogTableName) {
     if (!this.assignmentManager.getZKTable().isEnabledTable(catalogTableName)) {
       this.assignmentManager.setEnabledTable(catalogTableName);

From 3658dd92cabe5a3a8f8d632795d4fbd56356cc00 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 26 Jun 2012 04:25:21 +0000
Subject: [PATCH 0301/1540] HBASE-6267. hbase.store.delete.expired.storefile
 should be true by default

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1353813 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/Store.java  | 2 +-
 .../hbase/io/hfile/TestScannerSelectionUsingTTL.java      | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 4bfd0f83893b..256101d24d7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1344,7 +1344,7 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
     boolean forcemajor = this.forceMajor && filesCompacting.isEmpty();
     if (!forcemajor) {
       // Delete the expired store files before the compaction selection.
-      if (conf.getBoolean("hbase.store.delete.expired.storefile", false)
+      if (conf.getBoolean("hbase.store.delete.expired.storefile", true)
           && (ttl != Long.MAX_VALUE) && (this.scanInfo.minVersions == 0)) {
         CompactSelection expiredSelection = compactSelection
             .selectExpiredStoreFilesToCompact(
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
index 17284a3880aa..44aa3e422d5e 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
@@ -27,6 +27,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -38,7 +39,6 @@
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -98,6 +98,8 @@ public TestScannerSelectionUsingTTL(int numFreshFiles,
 
   @Test
   public void testScannerSelection() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setBoolean("hbase.store.delete.expired.storefile", false);
     HColumnDescriptor hcd =
       new HColumnDescriptor(FAMILY_BYTES)
           .setMaxVersions(Integer.MAX_VALUE)
@@ -107,7 +109,7 @@ public void testScannerSelection() throws IOException {
     HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE));
     HRegion region =
         HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(),
-            TEST_UTIL.getConfiguration(), htd);
+            conf, htd);
 
     for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
       if (iFile == NUM_EXPIRED_FILES) {
@@ -127,7 +129,7 @@ public void testScannerSelection() throws IOException {
 
     Scan scan = new Scan();
     scan.setMaxVersions(Integer.MAX_VALUE);
-    CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
+    CacheConfig cacheConf = new CacheConfig(conf);
     LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
     cache.clearCache();
     InternalScanner scanner = region.getScanner(scan);

From 801151036854d64f134aa349fb78f4741424987b Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Tue, 26 Jun 2012 17:17:07 +0000
Subject: [PATCH 0302/1540] HBASE-6240 Race in HCM.getMaster stalls clients

Submitted by:J-D, Ram
Reviewed by:J-D, Ted


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1354116 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HConnectionManager.java    | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 7090d3050611..16f0b5f79f0d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -634,6 +634,14 @@ public HMasterInterface getMaster()
       checkIfBaseNodeAvailable();
       ServerName sn = null;
       synchronized (this.masterLock) {
+        try {
+          if (master != null && master.isMasterRunning()) {
+            return master;
+          }
+        } catch (UndeclaredThrowableException ute) {
+          // log, but ignore, the loop below will attempt to reconnect
+          LOG.info("Exception contacting master. Retrying...", ute.getCause());
+        }
         this.master = null;
 
         for (int tries = 0;

From 5a676a601ec0778110e46f96a909a98ebbff59e9 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 26 Jun 2012 22:02:14 +0000
Subject: [PATCH 0303/1540] HBASE-6276. TestClassLoading is racy

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1354256 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/TestClassLoading.java   | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index 1918b1e7e4a0..dd14e2d3a990 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -33,7 +33,6 @@
 import javax.tools.*;
 import java.io.*;
 import java.util.*;
-import java.util.Arrays;
 import java.util.jar.*;
 
 import org.junit.*;
@@ -51,7 +50,6 @@ public class TestClassLoading {
   private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
-  private static Configuration conf;
   private static MiniDFSCluster cluster;
 
   static final int BUFFER_SIZE = 4096;
@@ -82,7 +80,7 @@ public class TestClassLoading {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    conf = TEST_UTIL.getConfiguration();
+    Configuration conf = TEST_UTIL.getConfiguration();
 
     // regionCoprocessor1 will be loaded on all regionservers, since it is
     // loaded for any tables (user or meta).
@@ -229,12 +227,13 @@ public void testClassLoadingFromHDFS() throws Exception {
       // with configuration values
     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
-    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     if (admin.tableExists(tableName)) {
       admin.disableTable(tableName);
       admin.deleteTable(tableName);
     }
     admin.createTable(htd);
+    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
 
     // verify that the coprocessors were loaded
     boolean found1 = false, found2 = false, found2_k1 = false,
@@ -275,8 +274,9 @@ public void testClassLoadingFromLocalFS() throws Exception {
     htd.addFamily(new HColumnDescriptor("test"));
     htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" +
       Coprocessor.PRIORITY_USER);
-    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     admin.createTable(htd);
+    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
 
     // verify that the coprocessor was loaded
     boolean found = false;
@@ -331,12 +331,13 @@ public void testHBase3810() throws Exception {
     htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()),
         Coprocessor.PRIORITY_USER, kvs);
 
-    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     if (admin.tableExists(tableName)) {
       admin.disableTable(tableName);
       admin.deleteTable(tableName);
     }
     admin.createTable(htd);
+    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
 
     // verify that the coprocessor was loaded
     boolean found_2 = false, found_1 = false, found_3 = false,
@@ -433,12 +434,13 @@ public void testClassLoadingFromLibDirInJar() throws Exception {
       // with configuration values
     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
-    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     if (admin.tableExists(tableName)) {
       admin.disableTable(tableName);
       admin.deleteTable(tableName);
     }
     admin.createTable(htd);
+    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
 
     // verify that the coprocessors were loaded
     boolean found1 = false, found2 = false, found2_k1 = false,
@@ -480,7 +482,7 @@ public void testRegionServerCoprocessorsReported() throws Exception {
     // name "ColumnAggregationEndpoint" will appear before regionCoprocessor2's
     // name "GenericEndpoint" because "C" is before "G" lexicographically.
 
-    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
 
     // disable all user tables, if any are loaded.
     for (HTableDescriptor htd: admin.listTables()) {
@@ -510,6 +512,8 @@ public void testRegionServerCoprocessorsReported() throws Exception {
     String userTable1 = "userTable1";
     HTableDescriptor userTD1 = new HTableDescriptor(userTable1);
     admin.createTable(userTD1);
+    TEST_UTIL.waitTableAvailable(userTD1.getName(), 5000);
+
     // table should be enabled now.
     assertTrue(admin.isTableEnabled(userTable1));
     assertAllRegionServers(regionServerSystemAndUserCoprocessors, userTable1);
@@ -528,6 +532,7 @@ public void testRegionServerCoprocessorsReported() throws Exception {
     htd2.setValue("COPROCESSOR$1", jarFile1.toString() + "|" + userTableCP +
       "|" + Coprocessor.PRIORITY_USER);
     admin.createTable(htd2);
+    TEST_UTIL.waitTableAvailable(htd2.getName(), 5000);
     // table should be enabled now.
     assertTrue(admin.isTableEnabled(userTable2));
 

From bf3697d75031272b66270935f8e1b40e1da44a45 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 26 Jun 2012 23:43:59 +0000
Subject: [PATCH 0304/1540] HBASE-6200 KeyComparator.compareWithoutRow can be
 wrong when families have the same prefix (Jieshan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1354293 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     | 84 +++++++++++--------
 .../org/apache/hadoop/hbase/TestKeyValue.java | 59 +++++++++++++
 2 files changed, 108 insertions(+), 35 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 277eafaa23c6..abbd588978ee 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -2102,33 +2102,29 @@ public int compareIgnoringPrefix(int commonPrefix, byte[] left,
     }
 
     /**
-     * Compare column, timestamp, and key type (everything except the row).
-     * This method is used both in the normal comparator and the "same-prefix"
-     * comparator. Note that we are assuming that row portions of both KVs have
-     * already been parsed and found identical, and we don't validate that
-     * assumption here.
-     * @param commonPrefix the length of the common prefix of the two
-     *          key-values being compared, including row length and row
+     * Compare columnFamily, qualifier, timestamp, and key type (everything
+     * except the row). This method is used both in the normal comparator and
+     * the "same-prefix" comparator. Note that we are assuming that row portions
+     * of both KVs have already been parsed and found identical, and we don't
+     * validate that assumption here.
+     * @param commonPrefix
+     *          the length of the common prefix of the two key-values being
+     *          compared, including row length and row
      */
     private int compareWithoutRow(int commonPrefix, byte[] left, int loffset,
         int llength, byte[] right, int roffset, int rlength, short rowlength) {
-      // Compare column family. Start comparing past row and family length.
-      int lcolumnoffset = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE +
-          rowlength + loffset;
-      int rcolumnoffset = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE +
-          rowlength + roffset;
-      int lcolumnlength = llength - TIMESTAMP_TYPE_SIZE -
-          (lcolumnoffset - loffset);
-      int rcolumnlength = rlength - TIMESTAMP_TYPE_SIZE -
-          (rcolumnoffset - roffset);
-
-      // If row matches, and no column in the 'left' AND put type is 'minimum',
-      // then return that left is larger than right.
-
-      // This supports 'last key on a row' - the magic is if there is no column
-      // in the left operand, and the left operand has a type of '0' - magical
-      // value, then we say the left is bigger.  This will let us seek to the
-      // last key in a row.
+      /***
+       * KeyValue Format and commonLength:
+       * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
+       * ------------------|-------commonLength--------|--------------
+       */
+      int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rowlength;
+
+      // commonLength + TIMESTAMP_TYPE_SIZE
+      int commonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + commonLength;
+      // ColumnFamily + Qualifier length.
+      int lcolumnlength = llength - commonLengthWithTSAndType;
+      int rcolumnlength = rlength - commonLengthWithTSAndType;
 
       byte ltype = left[loffset + (llength - 1)];
       byte rtype = right[roffset + (rlength - 1)];
@@ -2136,7 +2132,7 @@ private int compareWithoutRow(int commonPrefix, byte[] left, int loffset,
       // If the column is not specified, the "minimum" key type appears the
       // latest in the sorted order, regardless of the timestamp. This is used
       // for specifying the last key/value in a given row, because there is no
-      // "lexicographically last column" (it would be infinitely long).  The
+      // "lexicographically last column" (it would be infinitely long). The
       // "maximum" key type does not need this behavior.
       if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) {
         // left is "bigger", i.e. it appears later in the sorted order
@@ -2146,20 +2142,38 @@ private int compareWithoutRow(int commonPrefix, byte[] left, int loffset,
         return -1;
       }
 
+      int lfamilyoffset = commonLength + loffset;
+      int rfamilyoffset = commonLength + roffset;
+
+      // Column family length.
+      int lfamilylength = left[lfamilyoffset - 1];
+      int rfamilylength = right[rfamilyoffset - 1];
+      // If left family size is not equal to right family size, we need not
+      // compare the qualifiers. 
+      boolean sameFamilySize = (lfamilylength == rfamilylength);
       int common = 0;
       if (commonPrefix > 0) {
-        common = Math.max(0, commonPrefix -
-            rowlength - ROW_LENGTH_SIZE - FAMILY_LENGTH_SIZE);
-        common = Math.min(common, Math.min(lcolumnlength, rcolumnlength));
+        common = Math.max(0, commonPrefix - commonLength);
+        if (!sameFamilySize) {
+          // Common should not be larger than Math.min(lfamilylength,
+          // rfamilylength).
+          common = Math.min(common, Math.min(lfamilylength, rfamilylength));
+        } else {
+          common = Math.min(common, Math.min(lcolumnlength, rcolumnlength));
+        }
       }
-
-      final int comparisonResult = Bytes.compareTo(
-          left, lcolumnoffset + common, lcolumnlength - common,
-          right, rcolumnoffset + common, rcolumnlength - common);
-      if (comparisonResult != 0) {
-        return comparisonResult;
+      if (!sameFamilySize) {
+        // comparing column family is enough.
+        return Bytes.compareTo(left, lfamilyoffset + common, lfamilylength
+            - common, right, rfamilyoffset + common, rfamilylength - common);
+      }
+      // Compare family & qualifier together.
+      final int comparison = Bytes.compareTo(left, lfamilyoffset + common,
+          lcolumnlength - common, right, rfamilyoffset + common,
+          rcolumnlength - common);
+      if (comparison != 0) {
+        return comparison;
       }
-
       return compareTimestampAndType(left, loffset, llength, right, roffset,
           rlength, ltype, rtype);
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
index 786d2df52239..e308d718e768 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
@@ -341,6 +341,65 @@ private void assertKVLess(KeyValue.KVComparator c,
     assertTrue(cmp > 0);
   }
 
+  private void assertKVLessWithoutRow(KeyValue.KeyComparator c, int common, KeyValue less,
+      KeyValue greater) {
+    int cmp = c.compareIgnoringPrefix(common, less.getBuffer(), less.getOffset()
+        + KeyValue.ROW_OFFSET, less.getKeyLength(), greater.getBuffer(),
+        greater.getOffset() + KeyValue.ROW_OFFSET, greater.getKeyLength());
+    assertTrue(cmp < 0);
+    cmp = c.compareIgnoringPrefix(common, greater.getBuffer(), greater.getOffset()
+        + KeyValue.ROW_OFFSET, greater.getKeyLength(), less.getBuffer(),
+        less.getOffset() + KeyValue.ROW_OFFSET, less.getKeyLength());
+    assertTrue(cmp > 0);
+  }
+
+  public void testCompareWithoutRow() {
+    final KeyValue.KeyComparator c = KeyValue.KEY_COMPARATOR;
+    byte[] row = Bytes.toBytes("row");
+
+    byte[] fa = Bytes.toBytes("fa");
+    byte[] fami = Bytes.toBytes("fami");
+    byte[] fami1 = Bytes.toBytes("fami1");
+
+    byte[] qual0 = Bytes.toBytes("");
+    byte[] qual1 = Bytes.toBytes("qf1");
+    byte[] qual2 = Bytes.toBytes("qf2");
+    long ts = 1;
+
+    // 'fa:'
+    KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, Type.Put);
+    // 'fami:'
+    KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, Type.Put);
+    // 'fami:qf1'
+    KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, Type.Put);
+    // 'fami:qf2'
+    KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, Type.Put);
+    // 'fami1:'
+    KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, Type.Put);
+
+    // 'fami:qf1' < 'fami:qf2'
+    assertKVLessWithoutRow(c, 0, kv0_1, kv0_2);
+    // 'fami:qf1' < 'fami1:'
+    assertKVLessWithoutRow(c, 0, kv0_1, kv1_0);
+
+    // Test comparison by skipping the same prefix bytes.
+    /***
+     * KeyValue Format and commonLength:
+     * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
+     * ------------------|-------commonLength--------|--------------
+     */
+    int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE
+        + row.length;
+    // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
+    assertKVLessWithoutRow(c, commonLength + 2, kv_0, kv0_0);
+    // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
+    assertKVLessWithoutRow(c, commonLength + 4, kv0_0, kv0_1);
+    // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
+    assertKVLessWithoutRow(c, commonLength + 4, kv0_1, kv1_0);
+    // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
+    assertKVLessWithoutRow(c, commonLength + 6, kv0_1, kv0_2);
+  }
+
   public void testFirstLastOnRow() {
     final KVComparator c = KeyValue.COMPARATOR;
     long ts = 1;

From 92d055ba58c0c0abf0e43cfbec5bee5ad8d453c2 Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Wed, 27 Jun 2012 17:06:40 +0000
Subject: [PATCH 0305/1540] Update details for garyh

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1354632 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 05deaba026f7..d7ee84d57a12 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,8 +129,8 @@
       <name>Gary Helmling</name>
       <email>garyh@apache.org</email>
       <timezone>-8</timezone>
-      <organization>Trend Micro</organization>
-      <organizationUrl>http://www.trendmicro.com</organizationUrl>
+      <organization>Twitter</organization>
+      <organizationUrl>http://www.twitter.com</organizationUrl>
     </developer>
     <developer>
       <id>jdcryans</id>

From 7b17f2e52504e0bb6e28f30a0ff034fedb30c6ba Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 27 Jun 2012 17:15:27 +0000
Subject: [PATCH 0306/1540] HBASE-6227 SSH and cluster startup causes data loss

Submitted by:Chunhui
Reviewed by:Ted, Ram


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1354635 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/AssignmentManager.java  | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index f39e9d0e7434..6a4e4316e651 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -412,6 +412,13 @@ void processDeadServersAndRegionsInTransition(
       nodes.removeAll(regionsInTransition.keySet());
     }
 
+    // If some dead servers are processed by ServerShutdownHandler, we shouldn't
+    // assign all user regions( some would be assigned by
+    // ServerShutdownHandler), consider it as a failover
+    if (!this.serverManager.getDeadServers().isEmpty()) {
+      this.failover = true;
+    }
+
     // If we found user regions out on cluster, its a failover.
     if (this.failover) {
       LOG.info("Found regions out on cluster or in RIT; failover");
@@ -2671,6 +2678,9 @@ private void processDeadServersAndRecoverLostRegions(
         // skip regions of dead servers because SSH will process regions during rs expiration.
         // see HBASE-5916
         if (actualDeadServers.contains(deadServer.getKey())) {
+          for (Pair<HRegionInfo, Result> deadRegion : deadServer.getValue()) {
+            nodes.remove(deadRegion.getFirst().getEncodedName());
+          }
           continue;
         }
         List<Pair<HRegionInfo, Result>> regions = deadServer.getValue();

From 4ef32189c0ebc5aba49987ff332a5c69987306b9 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Jun 2012 04:39:50 +0000
Subject: [PATCH 0307/1540] HBASE-6269 Lazyseek should use the maxSequenseId
 StoreFile's KeyValue as the latest KeyValue (Xing Shi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1354815 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/KeyValueHeap.java      |  2 +-
 .../hbase/regionserver/TestHRegion.java       | 61 +++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 87883a0805b5..659bbdd658ca 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -342,7 +342,7 @@ private KeyValueScanner pollRealKV() throws IOException {
           // Compare the current scanner to the next scanner. We try to avoid
           // putting the current one back into the heap if possible.
           KeyValue nextKV = nextEarliestScanner.peek();
-          if (nextKV == null || comparator.compare(curKV, nextKV) <= 0) {
+          if (nextKV == null || comparator.compare(curKV, nextKV) < 0) {
             // We already have the scanner with the earliest KV, so return it.
             return kvScanner;
           }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 0d2e012c7703..a6f945e0fc61 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3502,6 +3502,67 @@ public void testDeleteRowWithBloomFilter() throws IOException {
         }
       }
   }
+
+  /**
+   * Test case to check put function with memstore flushing for same row, same ts
+   * @throws Exception
+   */
+  public void testPutWithMemStoreFlush() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String method = "testPutWithMemStoreFlush";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Bytes.toBytes("family");;
+    byte[] qualifier = Bytes.toBytes("qualifier");
+    byte[] row = Bytes.toBytes("putRow");
+    byte[] value = null;
+    this.region = initHRegion(tableName, method, conf, family);
+    Put put = null;
+    Get get = null;
+    List<KeyValue> kvs = null;
+    Result res = null;
+
+    put = new Put(row);
+    value = Bytes.toBytes("value0");
+    put.add(family, qualifier, 1234567l, value);
+    region.put(put);
+    get = new Get(row);
+    get.addColumn(family, qualifier);
+    get.setMaxVersions();
+    res = this.region.get(get, null);
+    kvs = res.getColumn(family, qualifier);
+    assertEquals(1, kvs.size());
+    assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
+
+    region.flushcache();
+    get = new Get(row);
+    get.addColumn(family, qualifier);
+    get.setMaxVersions();
+    res = this.region.get(get, null);
+    kvs = res.getColumn(family, qualifier);
+    assertEquals(1, kvs.size());
+    assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
+
+    put = new Put(row);
+    value = Bytes.toBytes("value1");
+    put.add(family, qualifier, 1234567l, value);
+    region.put(put);
+    get = new Get(row);
+    get.addColumn(family, qualifier);
+    get.setMaxVersions();
+    res = this.region.get(get, null);
+    kvs = res.getColumn(family, qualifier);
+    assertEquals(1, kvs.size());
+    assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
+
+    region.flushcache();
+    get = new Get(row);
+    get.addColumn(family, qualifier);
+    get.setMaxVersions();
+    res = this.region.get(get, null);
+    kvs = res.getColumn(family, qualifier);
+    assertEquals(1, kvs.size());
+    assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
+  }
   
   /**
    * TestCase for increment

From e27b702a8eebada06c7d05ae9d138f2172ae67f0 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 28 Jun 2012 17:49:30 +0000
Subject: [PATCH 0308/1540] HBASE-6210  Backport HBASE-6197 to 0.94 and 0.92?

Submitted by:Ram
Reviewed by:Stack


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1355087 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 24 +++--
 .../hbase/regionserver/TestHRegion.java       | 98 +++++++++++++++++++
 2 files changed, 113 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index ca04430736ba..3f87e13f8823 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4334,8 +4334,8 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
     boolean flush = false;
     WALEdit walEdits = null;
     List<KeyValue> allKVs = new ArrayList<KeyValue>(append.size());
-    List<KeyValue> kvs = new ArrayList<KeyValue>(append.size());
-    long now = EnvironmentEdgeManager.currentTimeMillis();
+    Map<Store, List<KeyValue>> tempMemstore = new HashMap<Store, List<KeyValue>>();
+    long before = EnvironmentEdgeManager.currentTimeMillis();
     long size = 0;
     long txid = 0;
 
@@ -4346,11 +4346,13 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
       Integer lid = getLock(lockid, row, true);
       this.updatesLock.readLock().lock();
       try {
+        long now = EnvironmentEdgeManager.currentTimeMillis();
         // Process each family
         for (Map.Entry<byte[], List<KeyValue>> family : append.getFamilyMap()
             .entrySet()) {
 
           Store store = stores.get(family.getKey());
+          List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());
 
           // Get previous values for all columns in this family
           Get get = new Get(row);
@@ -4416,10 +4418,8 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
             }
           }
 
-          // Write the KVs for this family into the store
-          size += store.upsert(kvs);
-          allKVs.addAll(kvs);
-          kvs.clear();
+          // store the kvs to the temporary memstore before writing HLog
+          tempMemstore.put(store, kvs);
         }
 
         // Actually write to WAL now
@@ -4429,9 +4429,15 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
           // as a Put.
           txid = this.log.appendNoSync(regionInfo,
               this.htableDescriptor.getName(), walEdits,
-              HConstants.DEFAULT_CLUSTER_ID, now, this.htableDescriptor);
+              HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
+              this.htableDescriptor);
+        }
+        // Actually write to Memstore now
+        for (Map.Entry<Store, List<KeyValue>> entry : tempMemstore.entrySet()) {
+          Store store = entry.getKey();
+          size += store.upsert(entry.getValue());
+          allKVs.addAll(entry.getValue());
         }
-
         size = this.addAndGetGlobalMemstoreSize(size);
         flush = isFlushSize(size);
       } finally {
@@ -4447,7 +4453,7 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
 
     
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - now);
+    this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - before);
     
     if (flush) {
       // Request a cache flush. Do it outside update lock.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index a6f945e0fc61..797c42be9e47 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MultithreadedTestUtil;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
+import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -3598,6 +3599,103 @@ public void run() {
     }
   }
 
+  /**
+   * TestCase for append
+   * 
+   */
+  private static class Appender implements Runnable {
+    private HRegion region;
+    private final static byte[] appendRow = Bytes.toBytes("appendRow");
+    private final static byte[] family = Bytes.toBytes("family");
+    private final static byte[] qualifier = Bytes.toBytes("qualifier");
+    private final static byte[] CHAR = Bytes.toBytes("a");
+    private int appendCounter;
+
+    public Appender(HRegion region, int appendCounter) {
+      this.region = region;
+      this.appendCounter = appendCounter;
+    }
+
+    @Override
+    public void run() {
+      int count = 0;
+      while (count < appendCounter) {
+        Append app = new Append(appendRow);
+        app.add(family, qualifier, CHAR);
+        count++;
+        try {
+          region.append(app, null, true);
+        } catch (IOException e) {
+          e.printStackTrace();
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Test case to check append function with memstore flushing
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testParallelAppendWithMemStoreFlush() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String method = "testParallelAppendWithMemStoreFlush";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Appender.family;
+    this.region = initHRegion(tableName, method, conf, family);
+    final HRegion region = this.region;
+    final AtomicBoolean appendDone = new AtomicBoolean(false);
+    Runnable flusher = new Runnable() {
+      @Override
+      public void run() {
+        while (!appendDone.get()) {
+          try {
+            region.flushcache();
+          } catch (Exception e) {
+            e.printStackTrace();
+          }
+        }
+      }
+    };
+
+    // after all append finished, the value will append to threadNum * appendCounter Appender.CHAR
+    int threadNum = 20;
+    int appendCounter = 100;
+    byte[] expected = new byte[threadNum * appendCounter];
+    for (int i = 0; i < threadNum * appendCounter; i++) {
+      System.arraycopy(Appender.CHAR, 0, expected, i, 1);
+    }
+    Thread[] appenders = new Thread[threadNum];
+    Thread flushThread = new Thread(flusher);
+    for (int i = 0; i < threadNum; i++) {
+      appenders[i] = new Thread(new Appender(this.region, appendCounter));
+      appenders[i].start();
+    }
+    flushThread.start();
+    for (int i = 0; i < threadNum; i++) {
+      appenders[i].join();
+    }
+
+    appendDone.set(true);
+    flushThread.join();
+
+    Get get = new Get(Appender.appendRow);
+    get.addColumn(Appender.family, Appender.qualifier);
+    get.setMaxVersions(1);
+    Result res = this.region.get(get, null);
+    List<KeyValue> kvs = res.getColumn(Appender.family, Appender.qualifier);
+
+    // we just got the latest version
+    assertEquals(kvs.size(), 1);
+    KeyValue kv = kvs.get(0);
+    byte[] appendResult = new byte[kv.getValueLength()];
+    System.arraycopy(kv.getBuffer(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
+    assertEquals(expected, appendResult);
+    this.region = null;
+  }
+   
   /**
    * Test case to check increment function with memstore flushing
    * @throws Exception

From c0f7d0fe52376ceeb16c32e487f0544bc078c24d Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 30 Jun 2012 21:26:26 +0000
Subject: [PATCH 0309/1540] HBASE-6292. Compact can skip the security access
 control (ShiXing)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1355826 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/security/access/AccessController.java    |  7 +++++++
 .../security/access/TestAccessController.java      | 13 +++++++++++++
 .../hbase/coprocessor/BaseRegionObserver.java      |  2 +-
 .../hadoop/hbase/coprocessor/RegionObserver.java   |  3 ++-
 .../hbase/regionserver/CompactSplitThread.java     |  9 +++++----
 .../hbase/regionserver/CompactionRequestor.java    | 14 ++++++++++----
 .../hadoop/hbase/regionserver/MemStoreFlusher.java |  8 +++++++-
 .../hbase/regionserver/RegionCoprocessorHost.java  | 12 +++++++++---
 .../apache/hadoop/hbase/regionserver/Store.java    |  7 ++-----
 9 files changed, 56 insertions(+), 19 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 601bc7bfe352..c84a93c3fb8b 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
@@ -741,6 +742,12 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
     return scanner;
   }
 
+  @Override
+  public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> e,
+      final Store store, final List<StoreFile> candidates) throws IOException {
+    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+  }
+
   @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> c,
       final byte [] row, final byte [] family, final Result result)
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 3691635abd4d..19d1a070389f 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -474,6 +474,19 @@ public Object run() throws Exception {
     verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
   }
 
+  @Test
+  public void testPreCompactSelection() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preCompactSelection(ObserverContext.createAndPrepare(RCP_ENV, null), null, null);
+        return null;
+      }
+    };
+
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
+  }
+
   private void verifyRead(PrivilegedExceptionAction action) throws Exception {
     verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW, USER_RO);
     verifyDenied(action, USER_NONE, USER_CREATE);
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 2d95555ac9f9..93e53c308d15 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -88,7 +88,7 @@ public void postSplit(ObserverContext<RegionCoprocessorEnvironment> e, HRegion l
 
   @Override
   public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
-      final Store store, final List<StoreFile> candidates) { }
+      final Store store, final List<StoreFile> candidates) throws IOException { }
 
   @Override
   public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 5f80ed302421..889c4af0c81a 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -81,9 +81,10 @@ public interface RegionObserver extends Coprocessor {
    * @param c the environment provided by the region server
    * @param store the store where compaction is being requested
    * @param candidates the store files currently available for compaction
+   * @throws IOException if an error occurred on the coprocessor
    */
   void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
-      final Store store, final List<StoreFile> candidates);
+      final Store store, final List<StoreFile> candidates) throws IOException;
 
   /**
    * Called after the {@link StoreFile}s to compact have been selected from the
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 15ba981ecabd..429b98d0e047 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.IOException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
@@ -166,19 +167,19 @@ public synchronized void requestSplit(final HRegion r, byte[] midKey) {
   }
 
   public synchronized void requestCompaction(final HRegion r,
-      final String why) {
+      final String why) throws IOException {
     for(Store s : r.getStores().values()) {
       requestCompaction(r, s, why, Store.NO_PRIORITY);
     }
   }
 
   public synchronized void requestCompaction(final HRegion r, final Store s,
-      final String why) {
+      final String why) throws IOException {
     requestCompaction(r, s, why, Store.NO_PRIORITY);
   }
 
   public synchronized void requestCompaction(final HRegion r, final String why,
-      int p) {
+      int p) throws IOException {
     for(Store s : r.getStores().values()) {
       requestCompaction(r, s, why, p);
     }
@@ -191,7 +192,7 @@ public synchronized void requestCompaction(final HRegion r, final String why,
    * @param priority override the default priority (NO_PRIORITY == decide)
    */
   public synchronized void requestCompaction(final HRegion r, final Store s,
-      final String why, int priority) {
+      final String why, int priority) throws IOException {
     if (this.server.isStopped()) {
       return;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
index 3fe928663ad3..255c4ecb458b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
@@ -19,34 +19,40 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.IOException;
+
 public interface CompactionRequestor {
   /**
    * @param r Region to compact
    * @param why Why compaction was requested -- used in debug messages
+   * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final String why);
+  public void requestCompaction(final HRegion r, final String why) throws IOException;
 
   /**
    * @param r Region to compact
    * @param s Store within region to compact
    * @param why Why compaction was requested -- used in debug messages
+   * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final Store s, final String why);
+  public void requestCompaction(final HRegion r, final Store s, final String why) throws IOException;
 
   /**
    * @param r Region to compact
    * @param why Why compaction was requested -- used in debug messages
    * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final String why, int pri);
+  public void requestCompaction(final HRegion r, final String why, int pri) throws IOException;
 
   /**
    * @param r Region to compact
    * @param s Store within region to compact
    * @param why Why compaction was requested -- used in debug messages
    * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @throws IOException
    */
   public void requestCompaction(final HRegion r, final Store s,
-      final String why, int pri);
+      final String why, int pri) throws IOException;
 
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index beae44331b66..221ca49cb523 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -360,7 +360,13 @@ private boolean flushRegion(final FlushRegionEntry fqe) {
           LOG.warn("Region " + region.getRegionNameAsString() + " has too many " +
             "store files; delaying flush up to " + this.blockingWaitTime + "ms");
           if (!this.server.compactSplitThread.requestSplit(region)) {
-            this.server.compactSplitThread.requestCompaction(region, getName());
+            try {
+              this.server.compactSplitThread.requestCompaction(region, getName());
+            }  catch (IOException e) {
+              LOG.error("Cache flush failed" +
+                (region != null ? (" for region " + Bytes.toStringBinary(region.getRegionName())) : ""),
+                RemoteExceptionHandler.checkIOException(e));
+            }
           }
         }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index a550d139174b..58afaf439b9f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -309,15 +309,21 @@ public void postClose(boolean abortRequested) {
    * @param store The store where compaction is being requested
    * @param candidates The currently available store files
    * @return If {@code true}, skip the normal selection process and use the current list
+   * @throws IOException
    */
-  public boolean preCompactSelection(Store store, List<StoreFile> candidates) {
+  public boolean preCompactSelection(Store store, List<StoreFile> candidates) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     boolean bypass = false;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
-        ((RegionObserver)env.getInstance()).preCompactSelection(
-            ctx, store, candidates);
+        try {
+          ((RegionObserver)env.getInstance()).preCompactSelection(
+              ctx, store, candidates);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env,e);
+
+        }
         bypass |= ctx.shouldBypass();
         if (ctx.shouldComplete()) {
           break;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 256101d24d7d..f7491fd28a8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1207,11 +1207,11 @@ long getNextMajorCompactTime() {
     return ret;
   }
 
-  public CompactionRequest requestCompaction() {
+  public CompactionRequest requestCompaction() throws IOException {
     return requestCompaction(NO_PRIORITY);
   }
 
-  public CompactionRequest requestCompaction(int priority) {
+  public CompactionRequest requestCompaction(int priority) throws IOException {
     // don't even select for compaction if writes are disabled
     if (!this.region.areWritesEnabled()) {
       return null;
@@ -1275,9 +1275,6 @@ public CompactionRequest requestCompaction(int priority) {
         int pri = getCompactPriority(priority);
         ret = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
       }
-    } catch (IOException ex) {
-      LOG.error("Compaction Request failed for region " + region + ", store "
-          + this, RemoteExceptionHandler.checkIOException(ex));
     } finally {
       this.lock.readLock().unlock();
     }

From 5dfc569b0b06e50d300b15413965be16ab4351cb Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Mon, 2 Jul 2012 08:24:51 +0000
Subject: [PATCH 0310/1540] HBASE-6265 Calling getTimestamp() on a KV in
 cp.prePut() causes KV not to be flushed

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1356105 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/KeyValue.java    |  2 ++
 .../org/apache/hadoop/hbase/TestKeyValue.java     | 15 +++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index abbd588978ee..59443835e992 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -963,6 +963,8 @@ public boolean updateLatestStamp(final byte [] now) {
     if (this.isLatestTimestamp()) {
       int tsOffset = getTimestampOffset();
       System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
+      // clear cache or else getTimestamp() possibly returns an old value
+      timestampCache = -1L;
       return true;
     }
     return false;
diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
index e308d718e768..fe3dfeb2b291 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
@@ -488,6 +488,21 @@ public void testReadFields() throws IOException {
     assertTrue(Bytes.equals(kv1.getRow(), kv2.getRow()));
   }
 
+  /**
+   * Tests that getTimestamp() does always return the proper timestamp, even after updating it.
+   * See HBASE-6265.
+   */
+  public void testGetTimestamp() {
+    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
+      Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP,
+      Bytes.toBytes("myValue"));
+    long time1 = kv.getTimestamp();
+    kv.updateLatestStamp(Bytes.toBytes(12345L));
+    long time2 = kv.getTimestamp();
+    assertEquals(HConstants.LATEST_TIMESTAMP, time1);
+    assertEquals(12345L, time2);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 8689708bd145f9cf911a9f4e615fd331a6e7502b Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Mon, 2 Jul 2012 09:00:05 +0000
Subject: [PATCH 0311/1540] HBASE-6265 Calling getTimestamp() on a KV in
 cp.prePut() causes KV not to be flushed (added CHANGES.txt edit)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1356124 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 82e640293227..e2cf528da37a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -227,6 +227,7 @@ Bug
     [HBASE-5893] - Allow spaces in coprocessor conf (aka trim() className)
     [HBASE-5897] - prePut coprocessor hook causing substantial CPU usage
     [HBASE-5908] - TestHLogSplit.testTralingGarbageCorruptionFileSkipErrorsPasses should not use append to corrupt the HLog
+    [HBASE-6265] - Calling getTimestamp() on a KV in cp.prePut() causes KV not to be flushed
 
 Improvement
 

From 5ae835ffe49523e850c936479ca3ca85f45df91d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 2 Jul 2012 17:57:17 +0000
Subject: [PATCH 0312/1540] HBASE-5955 Guava 11 drops MapEvictionListener and
 Hadoop 2.0.0-alpha requires it

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1356379 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       |  2 +-
 .../hbase/io/hfile/slab/SingleSizeCache.java  | 39 ++++++++++++-------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/pom.xml b/pom.xml
index d7ee84d57a12..6cf629fd2a87 100644
--- a/pom.xml
+++ b/pom.xml
@@ -977,7 +977,7 @@
     <commons-math.version>2.1</commons-math.version>
     <commons-configuration.version>1.6</commons-configuration.version>
     <metrics-core.version>2.1.2</metrics-core.version>
-    <guava.version>r09</guava.version>
+    <guava.version>11.0.2</guava.version>
     <jackson.version>1.8.8</jackson.version>
     <jasper.version>5.5.23</jasper.version>
     <jaxb-api.version>2.1</jaxb-api.version>
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
index 4d9f518e34bb..152f197dc3da 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
@@ -38,8 +38,9 @@
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.util.StringUtils;
 
-import com.google.common.collect.MapEvictionListener;
-import com.google.common.collect.MapMaker;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 
 /**
  * SingleSizeCache is a slab allocated cache that caches elements up to a single
@@ -91,18 +92,30 @@ public SingleSizeCache(int blockSize, int numBlocks,
     // This evictionListener is called whenever the cache automatically
     // evicts
     // something.
-    MapEvictionListener<BlockCacheKey, CacheablePair> listener = new MapEvictionListener<BlockCacheKey, CacheablePair>() {
-      @Override
-      public void onEviction(BlockCacheKey key, CacheablePair value) {
-        timeSinceLastAccess.set(System.nanoTime()
-            - value.recentlyAccessed.get());
-        stats.evict();
-        doEviction(key, value);
-      }
-    };
+    RemovalListener<BlockCacheKey, CacheablePair> listener =
+      new RemovalListener<BlockCacheKey, CacheablePair>() {
+        @Override
+        public void onRemoval(
+            RemovalNotification<BlockCacheKey, CacheablePair> notification) {
+          if (!notification.wasEvicted()) {
+            // Only process removals by eviction, not by replacement or
+            // explicit removal
+            return;
+          }
+          CacheablePair value = notification.getValue();
+          timeSinceLastAccess.set(System.nanoTime()
+              - value.recentlyAccessed.get());
+          stats.evict();
+          doEviction(notification.getKey(), value);
+        }
+      };
+
+    backingMap = CacheBuilder.newBuilder()
+        .maximumSize(numBlocks - 1)
+        .removalListener(listener)
+        .<BlockCacheKey, CacheablePair>build()
+        .asMap();
 
-    backingMap = new MapMaker().maximumSize(numBlocks - 1)
-        .evictionListener(listener).makeMap();
 
   }
 

From 681e475d6969e90852706481477a05daacf3c686 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 2 Jul 2012 18:35:20 +0000
Subject: [PATCH 0313/1540] HBASE-6281 Assignment need not be called for
 disabling table regions during clean cluster start up. (Rajesh)

Submitted by:Rajesh
Reviewed by:Ram, Stack, Ted


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1356396 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  9 ++-
 .../hbase/master/TestAssignmentManager.java   | 55 ++++++++++++++++++-
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 6a4e4316e651..0b5a65745bd7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2256,9 +2256,14 @@ public void assignAllUserRegions() throws IOException, InterruptedException {
     // If there are no servers we need not proceed with region assignment.
     if(servers.isEmpty()) return;
 
+    // Skip assignment for regions of tables in DISABLING state also because
+    // during clean cluster startup no RS is alive and regions map also doesn't
+    // have any information about the regions. See HBASE-6281.
+    Set<String> disablingAndDisabledTables = new HashSet<String>(this.disablingTables);
+    disablingAndDisabledTables.addAll(this.zkTable.getDisabledTables());
     // Scan META for all user regions, skipping any disabled tables
-    Map<HRegionInfo, ServerName> allRegions =
-      MetaReader.fullScan(catalogTracker, this.zkTable.getDisabledTables(), true);
+    Map<HRegionInfo, ServerName> allRegions = MetaReader.fullScan(catalogTracker,
+        disablingAndDisabledTables, true);
     if (allRegions == null || allRegions.isEmpty()) return;
 
     // Determine what type of assignment to do on startup
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 961500d23e16..cd3989a2e96c 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -772,8 +773,7 @@ private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final
     Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
       thenReturn(System.currentTimeMillis());
     // Return good result 'r' first and then return null to indicate end of scan
-    Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).
-      thenReturn(new Result[] { r }, (Result[]) null);
+    Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
     // If a get, return the above result too for REGIONINFO
     Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).
       thenReturn(r);
@@ -841,6 +841,49 @@ public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, K
         newRegionPlan.getDestination())));
   }
   
+  /**
+   * Test verifies whether assignment is skipped for regions of tables in DISABLING state during
+   * clean cluster startup. See HBASE-6281.
+   * 
+   * @throws KeeperException
+   * @throws IOException
+   * @throws Exception
+   */
+  @Test
+  public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
+      throws KeeperException, IOException, Exception {
+    this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        MockedLoadBalancer.class, LoadBalancer.class);
+    Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
+        new HashMap<ServerName, HServerLoad>(0));
+    List<ServerName> destServers = new ArrayList<ServerName>(1);
+    destServers.add(SERVERNAME_A);
+    Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
+    // To avoid cast exception in DisableTableHandler process.
+    //Server server = new HMaster(HTU.getConfiguration());
+    AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
+        this.serverManager);
+    AtomicBoolean gate = new AtomicBoolean(false);
+    if (balancer instanceof MockedLoadBalancer) {
+      ((MockedLoadBalancer) balancer).setGateVariable(gate);
+    }
+    try{
+      // set table in disabling state.
+      am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
+      am.joinCluster();
+      // should not call retainAssignment if we get empty regions in assignAllUserRegions.
+      assertFalse(
+          "Assign should not be invoked for disabling table regions during clean cluster startup.",
+          gate.get());
+      // need to change table state from disabling to disabled.
+      assertTrue("Table should be disabled.",
+          am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));      
+    } finally {
+      am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
+      am.shutdown();
+    }
+  }
+
   /**
    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
    * random assignment is called and the gate variable is set to true.
@@ -858,6 +901,14 @@ public ServerName randomAssignment(List<ServerName> servers) {
       this.gate.set(true);
       return randomServerName;
     }
+    
+    @Override
+    public Map<ServerName, List<HRegionInfo>> retainAssignment(
+        Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
+      this.gate.set(true);
+      return super.retainAssignment(regions, servers);
+    }
+
   }
 
   /**

From acadedcac82f29fd8b4ea31a886ac91e228eebe8 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 2 Jul 2012 23:27:46 +0000
Subject: [PATCH 0314/1540] HBASE-6303. HCD.setCompressionType should use Enum
 support for storing compression types as strings

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1356518 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HColumnDescriptor.java  | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 90701522267e..0094d368f005 100644
--- a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -567,14 +567,7 @@ public Compression.Algorithm getCompressionType() {
    * @return this (for chained invocation)
    */
   public HColumnDescriptor setCompressionType(Compression.Algorithm type) {
-    String compressionType;
-    switch (type) {
-      case LZO: compressionType = "LZO"; break;
-      case GZ: compressionType = "GZ"; break;
-      case SNAPPY: compressionType = "SNAPPY"; break;
-      default: compressionType = "NONE"; break;
-    }
-    return setValue(COMPRESSION, compressionType);
+    return setValue(COMPRESSION, type.getName().toUpperCase());
   }
 
   /** @return data block encoding algorithm used on disk */

From 1bce88e8996f197e5ed3312a4541d5a6753305a9 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 3 Jul 2012 04:36:22 +0000
Subject: [PATCH 0315/1540] Amend HBASE-6303. Likewise for
 HCD.setCompactionCompressionType

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1356569 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HColumnDescriptor.java  | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 0094d368f005..766381513d87 100644
--- a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -640,14 +640,7 @@ public Compression.Algorithm getCompactionCompressionType() {
    */
   public HColumnDescriptor setCompactionCompressionType(
       Compression.Algorithm type) {
-    String compressionType;
-    switch (type) {
-      case LZO: compressionType = "LZO"; break;
-      case GZ: compressionType = "GZ"; break;
-      case SNAPPY: compressionType = "SNAPPY"; break;
-      default: compressionType = "NONE"; break;
-    }
-    return setValue(COMPRESSION_COMPACT, compressionType);
+    return setValue(COMPRESSION_COMPACT, type.getName().toUpperCase());
   }
 
   /**

From d03811f9ea9259d9f6030726ecc86b8dfd64156a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 4 Jul 2012 15:04:07 +0000
Subject: [PATCH 0316/1540] HBASE-6322 Unnecessary creation of finalizers in
 HTablePool

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357290 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTablePool.java   |  3 +--
 .../hadoop/hbase/client/TestHTablePool.java      | 16 ----------------
 2 files changed, 1 insertion(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 9741dccc9348..fdf9aaae6679 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -320,12 +320,11 @@ int getCurrentPoolSize(String tableName) {
    * wrapped table back to the table pool
    * 
    */
-  class PooledHTable extends HTable {
+  class PooledHTable implements HTableInterface {
 
     private HTableInterface table; // actual table implementation
 
     public PooledHTable(HTableInterface table) throws IOException {
-      super(table.getConfiguration(), table.getTableName());
       this.table = table;
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
index 5ab918ea44fb..74f10e57195c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
@@ -183,22 +183,6 @@ public void testReturnDifferentTable() throws IOException {
         Assert.assertTrue("alien table rejected", true);
       }
     }
-
-    @Test
-    public void testClassCastException() {
-      //this test makes sure that client code that
-      //casts the table it got from pool to HTable won't break
-      HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(),
-        Integer.MAX_VALUE);
-      String tableName = Bytes.toString(TABLENAME);
-      try {
-        // get table and check if type is HTable
-        HTable table = (HTable) pool.getTable(tableName);
-        Assert.assertTrue("return type is HTable as expected", true);
-      } catch (ClassCastException e) {
-        Assert.fail("return type is not HTable");
-      }
-    }
   }
 
   @Category(MediumTests.class)

From 4caf79b6823a323ed369f7d5a705c36fab7e0775 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 4 Jul 2012 15:11:38 +0000
Subject: [PATCH 0317/1540] HBASE-6322 Unnecessary creation of finalizers in
 HTablePool; REVERT -- DOESN'T BELONG ON THIS BRANCH

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357299 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTablePool.java   |  3 ++-
 .../hadoop/hbase/client/TestHTablePool.java      | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index fdf9aaae6679..9741dccc9348 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -320,11 +320,12 @@ int getCurrentPoolSize(String tableName) {
    * wrapped table back to the table pool
    * 
    */
-  class PooledHTable implements HTableInterface {
+  class PooledHTable extends HTable {
 
     private HTableInterface table; // actual table implementation
 
     public PooledHTable(HTableInterface table) throws IOException {
+      super(table.getConfiguration(), table.getTableName());
       this.table = table;
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
index 74f10e57195c..5ab918ea44fb 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
@@ -183,6 +183,22 @@ public void testReturnDifferentTable() throws IOException {
         Assert.assertTrue("alien table rejected", true);
       }
     }
+
+    @Test
+    public void testClassCastException() {
+      //this test makes sure that client code that
+      //casts the table it got from pool to HTable won't break
+      HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(),
+        Integer.MAX_VALUE);
+      String tableName = Bytes.toString(TABLENAME);
+      try {
+        // get table and check if type is HTable
+        HTable table = (HTable) pool.getTable(tableName);
+        Assert.assertTrue("return type is HTable as expected", true);
+      } catch (ClassCastException e) {
+        Assert.fail("return type is not HTable");
+      }
+    }
   }
 
   @Category(MediumTests.class)

From e0f58508e2f5d79c5e98639344332ebb6458f84c Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 5 Jul 2012 07:22:25 +0000
Subject: [PATCH 0318/1540] HBASE-6330 TestImportExport has been failing
 against hadoop 0.23/2.0 profile [part2]

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357479 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 40 +++++++++++--------
 .../hbase/mapreduce/MapreduceTestingShim.java | 17 ++++++++
 .../hbase/mapreduce/TestImportExport.java     | 18 ++++-----
 3 files changed, 50 insertions(+), 25 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 19fca5803f59..e6f170300174 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -61,6 +61,7 @@
 import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -81,6 +82,7 @@
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
@@ -289,21 +291,6 @@ private void setupDataTestDir() {
     createSubDirAndSystemProperty(
       "hadoop.log.dir",
       testPath, "hadoop-log-dir");
-
-    // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
-    //  we want our own value to ensure uniqueness on the same machine
-    createSubDirAndSystemProperty(
-      "hadoop.tmp.dir",
-      testPath, "hadoop-tmp-dir");
-
-    // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
-    createSubDir(
-      "mapred.local.dir",
-      testPath, "mapred-local-dir");
-
-    createSubDirAndSystemProperty(
-      "mapred.working.dir",
-      testPath, "mapred-working-dir");
   }
 
   private void createSubDir(String propertyName, Path parent, String subDirName){
@@ -441,6 +428,24 @@ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
     return this.dfsCluster;
   }
 
+  /** This is used before starting HDFS and map-reduce mini-clusters */
+  private void createDirsAndSetProperties() {
+    setupClusterTestDir();
+    System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
+    createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
+    createDirAndSetProperty("mapred_output", MapreduceTestingShim.getMROutputDirProp());
+    createDirAndSetProperty("mapred_local", "mapred.local.dir");
+  }
+
+  private String createDirAndSetProperty(final String relPath, String property) {
+    String path = clusterTestDir.getPath() + "/" + relPath;
+    System.setProperty(property, path);
+    conf.set(property, path);
+    new File(path).mkdirs();
+    LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
+    return path;
+  }
+
   /**
    * Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
    * or does nothing.
@@ -1268,10 +1273,13 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
     conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
     mrCluster = new MiniMRCluster(servers,
       FileSystem.get(conf).getUri().toString(), 1);
+    JobConf jobConf = mrCluster.createJobConf();
     LOG.info("Mini mapreduce cluster started");
+
+    // This fixes TestImportTsv but breaks TestImportExport tests
     conf.set("mapred.job.tracker",
         mrCluster.createJobConf().get("mapred.job.tracker"));
-    /* this for mrv2 support */
+    // this for mrv2 support; mr1 ignores this 
     conf.set("mapreduce.framework.name", "yarn");
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
index 76de6a39c804..985097de328f 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
@@ -49,11 +49,17 @@ abstract public class MapreduceTestingShim {
   abstract public JobContext newJobContext(Configuration jobConf)
       throws IOException;
 
+  abstract public String obtainMROutputDirProp();
+  
   public static JobContext createJobContext(Configuration jobConf)
       throws IOException {
     return instance.newJobContext(jobConf);
   }
 
+  public static String getMROutputDirProp() {
+    return instance.obtainMROutputDirProp();
+  }
+  
   private static class MapreduceV1Shim extends MapreduceTestingShim {
     public JobContext newJobContext(Configuration jobConf) throws IOException {
       // Implementing:
@@ -68,6 +74,11 @@ public JobContext newJobContext(Configuration jobConf) throws IOException {
             "Failed to instantiate new JobContext(jobConf, new JobID())", e);
       }
     }
+
+    @Override
+    public String obtainMROutputDirProp() {
+      return "mapred.output.dir";
+    }
   };
 
   private static class MapreduceV2Shim extends MapreduceTestingShim {
@@ -83,6 +94,12 @@ public JobContext newJobContext(Configuration jobConf) {
             "Failed to return from Job.getInstance(jobConf)");
       }
     }
+
+    @Override
+    public String obtainMROutputDirProp() {
+      // o.a.h.mapreduce.lib.output FileOutputFormat.OUTDIR
+      return "mapreduce.output.fileoutputformat.outputdir";
+    }
   };
 
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index bc05754b64a1..902249240b83 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.hadoop.conf.Configuration;
@@ -27,7 +28,6 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -44,7 +45,6 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import static org.junit.Assert.assertEquals;
 
 @Category(MediumTests.class)
 public class TestImportExport {
@@ -58,12 +58,11 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
-  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    cluster = UTIL.startMiniCluster();
+    UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -105,16 +104,16 @@ public void testSimpleCase() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
     Job job = Export.createSubmittableJob(conf, args);
     job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
+    HBaseFsck.debugLsr(conf, new Path("."));
     assertTrue(job.isSuccessful());
 
-
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -123,13 +122,14 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
     job = Import.createSubmittableJob(conf, args);
     job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
+    HBaseFsck.debugLsr(conf, new Path("."));
     assertTrue(job.isSuccessful());
 
     Get g = new Get(ROW1);
@@ -174,7 +174,7 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -198,7 +198,7 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 

From b73717fccae1b83de9aad4c324be863b0cd89837 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Thu, 5 Jul 2012 10:36:58 +0000
Subject: [PATCH 0319/1540] HBASE-6328 FSHDFSUtils#recoverFileLease tries to
 rethrow InterruptedException but actually shallows it

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357541 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 3d7787985ce4..4186f0ad54c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -51,6 +51,7 @@ public class FSHDFSUtils extends FSUtils{
    */
   public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
 
+  @Override
   public void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
   throws IOException{
     if (!isAppendSupported(conf)) {
@@ -111,7 +112,9 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
       try {
         Thread.sleep(1000);
       } catch (InterruptedException ex) {
-        new InterruptedIOException().initCause(ex);
+        InterruptedIOException iioe = new InterruptedIOException();
+        iioe.initCause(ex);
+        throw iioe;
       }
     }
     LOG.info("Finished lease recover attempt for " + p);

From e36f0d2b80294a52a2cca1f5ba05218640f1a9d2 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 5 Jul 2012 17:23:07 +0000
Subject: [PATCH 0320/1540] HBASE-6314. Fast fail behavior for unauthenticated
 user (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357750 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/SecureClient.java | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index e85bf42220e1..720338f3d982 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 
 import javax.net.SocketFactory;
+import javax.security.sasl.SaslException;
 import java.io.*;
 import java.net.*;
 import java.security.PrivilegedExceptionAction;
@@ -185,6 +186,14 @@ private synchronized boolean setupSaslConnection(final InputStream in2,
      * again.
      * The other problem is to do with ticket expiry. To handle that,
      * a relogin is attempted.
+     * <p>
+     * The retry logic is governed by the {@link #shouldAuthenticateOverKrb}
+     * method. In case when the user doesn't have valid credentials, we don't
+     * need to retry (from cache or ticket). In such cases, it is prudent to
+     * throw a runtime exception when we receive a SaslException from the
+     * underlying authentication implementation, so there is no retry from 
+     * other high level (for eg, HCM or HBaseAdmin).
+     * </p>
      */
     private synchronized void handleSaslConnectionFailure(
         final int currRetries,
@@ -222,8 +231,16 @@ public Object run() throws IOException, InterruptedException {
             LOG.warn("Exception encountered while connecting to " +
                 "the server : " + ex);
           }
-          if (ex instanceof RemoteException)
+          if (ex instanceof RemoteException) {
             throw (RemoteException)ex;
+          }
+          if (ex instanceof SaslException) {
+            String msg = "SASL authentication failed." +
+              " The most likely cause is missing or invalid credentials." +
+              " Consider 'kinit'.";
+            LOG.fatal(msg, ex);
+            throw new RuntimeException(msg, ex);
+          }
           throw new IOException(ex);
         }
       });

From b2fa57f6d8b18ed61e9d9ac2f0869d5e7cc9820e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 5 Jul 2012 17:25:51 +0000
Subject: [PATCH 0321/1540] HBASE-6326 Avoid nested retry loops in
 HConnectionManager

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357752 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 16f0b5f79f0d..cdb55e0b9b6f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -807,17 +807,17 @@ public List<HRegionLocation> locateRegions(final byte [] tableName)
     public HRegionLocation locateRegion(final byte [] tableName,
         final byte [] row)
     throws IOException{
-      return locateRegion(tableName, row, true);
+      return locateRegion(tableName, row, true, true);
     }
 
     public HRegionLocation relocateRegion(final byte [] tableName,
         final byte [] row)
     throws IOException{
-      return locateRegion(tableName, row, false);
+      return locateRegion(tableName, row, false, true);
     }
 
     private HRegionLocation locateRegion(final byte [] tableName,
-      final byte [] row, boolean useCache)
+      final byte [] row, boolean useCache, boolean retry)
     throws IOException {
       if (this.closed) throw new IOException(toString() + " closed");
       if (tableName == null || tableName.length == 0) {
@@ -839,11 +839,11 @@ private HRegionLocation locateRegion(final byte [] tableName,
         }
       } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
         return locateRegionInMeta(HConstants.ROOT_TABLE_NAME, tableName, row,
-            useCache, metaRegionLock);
+            useCache, metaRegionLock, retry);
       } else {
         // Region not in the cache - have to go to the meta RS
         return locateRegionInMeta(HConstants.META_TABLE_NAME, tableName, row,
-            useCache, userRegionLock);
+            useCache, userRegionLock, retry);
       }
     }
 
@@ -913,7 +913,7 @@ public boolean processRow(Result result) throws IOException {
       */
     private HRegionLocation locateRegionInMeta(final byte [] parentTable,
       final byte [] tableName, final byte [] row, boolean useCache,
-      Object regionLockObject)
+      Object regionLockObject, boolean retry)
     throws IOException {
       HRegionLocation location;
       // If we are supposed to be using the cache, look in the cache to see if
@@ -925,13 +925,14 @@ private HRegionLocation locateRegionInMeta(final byte [] parentTable,
         }
       }
 
+      int localNumRetries = retry ? numRetries : 1;
       // build the key of the meta region we should be looking for.
       // the extra 9's on the end are necessary to allow "exact" matches
       // without knowing the precise region names.
       byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
         HConstants.NINES, false);
       for (int tries = 0; true; tries++) {
-        if (tries >= numRetries) {
+        if (tries >= localNumRetries) {
           throw new NoServerForRegionException("Unable to find region for "
             + Bytes.toStringBinary(row) + " after " + numRetries + " tries.");
         }
@@ -939,7 +940,7 @@ private HRegionLocation locateRegionInMeta(final byte [] parentTable,
         HRegionLocation metaLocation = null;
         try {
           // locate the root or meta region
-          metaLocation = locateRegion(parentTable, metaKey);
+          metaLocation = locateRegion(parentTable, metaKey, true, false);
           // If null still, go around again.
           if (metaLocation == null) continue;
           HRegionInterface server =
@@ -1488,7 +1489,7 @@ public <R> void processBatchCallback(
         for (int i = 0; i < workingList.size(); i++) {
           Row row = workingList.get(i);
           if (row != null) {
-            HRegionLocation loc = locateRegion(tableName, row.getRow(), true);
+            HRegionLocation loc = locateRegion(tableName, row.getRow());
             byte[] regionName = loc.getRegionInfo().getRegionName();
 
             MultiAction<R> actions = actionsByServer.get(loc);

From 1a02a526a1adbf85d043f94376f7bc7ce3f2e965 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 5 Jul 2012 17:39:01 +0000
Subject: [PATCH 0322/1540] HBASE-6293 HMaster does not go down while splitting
 logs even if explicit shutdown is called.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357759 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/MasterFileSystem.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 31ee9df64525..7de0136bcb05 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -189,6 +189,10 @@ void splitLogAfterStartup() {
         HLog.SPLIT_SKIP_ERRORS_DEFAULT);
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
     do {
+      if (master.isStopped()) {
+        LOG.warn("Master stopped while splitting logs");
+        break;
+      }
       List<ServerName> serverNames = new ArrayList<ServerName>();
       try {
         if (!this.fs.exists(logsDirPath)) return;

From 626363d32fe3a1ee3dbcf4820fb41045f72c62f7 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 5 Jul 2012 18:05:08 +0000
Subject: [PATCH 0323/1540] HBASE-6332. Improve POM for better integration with
 downstream ivy projects (Roman Shaposhnik)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357774 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pom.xml b/pom.xml
index 6cf629fd2a87..6ca62541c826 100644
--- a/pom.xml
+++ b/pom.xml
@@ -986,6 +986,7 @@
     <jersey.version>1.4</jersey.version>
     <jruby.version>1.6.5</jruby.version>
     <junit.version>4.10-HBASE-1</junit.version>
+    <slf4j.version>1.4.3</slf4j.version>
     <log4j.version>1.2.16</log4j.version>
     <mockito-all.version>1.8.5</mockito-all.version>
     <protobuf.version>2.4.0a</protobuf.version>
@@ -1298,6 +1299,8 @@
       <artifactId>junit</artifactId>
       <version>${junit.version}</version>
       <scope>test,runtime</scope>
+      <!-- FIXME: the following needs to go away once HBASE-4955 is fixed -->
+      <optional>true</optional>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>

From bb1ac541d697849631794b68ce5804ab3b49087c Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 5 Jul 2012 19:58:54 +0000
Subject: [PATCH 0324/1540] Revert HBASE-6330 - hadoop 2.0 was fixed but this
 broke hadoop 1.0 build

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357853 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 40 ++++++++-----------
 .../hbase/mapreduce/MapreduceTestingShim.java | 17 --------
 .../hbase/mapreduce/TestImportExport.java     | 18 ++++-----
 3 files changed, 25 insertions(+), 50 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index e6f170300174..19fca5803f59 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -61,7 +61,6 @@
 import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -82,7 +81,6 @@
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
@@ -291,6 +289,21 @@ private void setupDataTestDir() {
     createSubDirAndSystemProperty(
       "hadoop.log.dir",
       testPath, "hadoop-log-dir");
+
+    // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
+    //  we want our own value to ensure uniqueness on the same machine
+    createSubDirAndSystemProperty(
+      "hadoop.tmp.dir",
+      testPath, "hadoop-tmp-dir");
+
+    // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
+    createSubDir(
+      "mapred.local.dir",
+      testPath, "mapred-local-dir");
+
+    createSubDirAndSystemProperty(
+      "mapred.working.dir",
+      testPath, "mapred-working-dir");
   }
 
   private void createSubDir(String propertyName, Path parent, String subDirName){
@@ -428,24 +441,6 @@ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
     return this.dfsCluster;
   }
 
-  /** This is used before starting HDFS and map-reduce mini-clusters */
-  private void createDirsAndSetProperties() {
-    setupClusterTestDir();
-    System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
-    createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
-    createDirAndSetProperty("mapred_output", MapreduceTestingShim.getMROutputDirProp());
-    createDirAndSetProperty("mapred_local", "mapred.local.dir");
-  }
-
-  private String createDirAndSetProperty(final String relPath, String property) {
-    String path = clusterTestDir.getPath() + "/" + relPath;
-    System.setProperty(property, path);
-    conf.set(property, path);
-    new File(path).mkdirs();
-    LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
-    return path;
-  }
-
   /**
    * Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
    * or does nothing.
@@ -1273,13 +1268,10 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
     conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
     mrCluster = new MiniMRCluster(servers,
       FileSystem.get(conf).getUri().toString(), 1);
-    JobConf jobConf = mrCluster.createJobConf();
     LOG.info("Mini mapreduce cluster started");
-
-    // This fixes TestImportTsv but breaks TestImportExport tests
     conf.set("mapred.job.tracker",
         mrCluster.createJobConf().get("mapred.job.tracker"));
-    // this for mrv2 support; mr1 ignores this 
+    /* this for mrv2 support */
     conf.set("mapreduce.framework.name", "yarn");
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
index 985097de328f..76de6a39c804 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
@@ -49,17 +49,11 @@ abstract public class MapreduceTestingShim {
   abstract public JobContext newJobContext(Configuration jobConf)
       throws IOException;
 
-  abstract public String obtainMROutputDirProp();
-  
   public static JobContext createJobContext(Configuration jobConf)
       throws IOException {
     return instance.newJobContext(jobConf);
   }
 
-  public static String getMROutputDirProp() {
-    return instance.obtainMROutputDirProp();
-  }
-  
   private static class MapreduceV1Shim extends MapreduceTestingShim {
     public JobContext newJobContext(Configuration jobConf) throws IOException {
       // Implementing:
@@ -74,11 +68,6 @@ public JobContext newJobContext(Configuration jobConf) throws IOException {
             "Failed to instantiate new JobContext(jobConf, new JobID())", e);
       }
     }
-
-    @Override
-    public String obtainMROutputDirProp() {
-      return "mapred.output.dir";
-    }
   };
 
   private static class MapreduceV2Shim extends MapreduceTestingShim {
@@ -94,12 +83,6 @@ public JobContext newJobContext(Configuration jobConf) {
             "Failed to return from Job.getInstance(jobConf)");
       }
     }
-
-    @Override
-    public String obtainMROutputDirProp() {
-      // o.a.h.mapreduce.lib.output FileOutputFormat.OUTDIR
-      return "mapreduce.output.fileoutputformat.outputdir";
-    }
   };
 
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index 902249240b83..bc05754b64a1 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.hadoop.conf.Configuration;
@@ -28,6 +27,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
@@ -36,7 +36,6 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -45,6 +44,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import static org.junit.Assert.assertEquals;
 
 @Category(MediumTests.class)
 public class TestImportExport {
@@ -58,11 +58,12 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
+  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    UTIL.startMiniCluster();
+    cluster = UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -104,16 +105,16 @@ public void testSimpleCase() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
     Job job = Export.createSubmittableJob(conf, args);
     job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
-    HBaseFsck.debugLsr(conf, new Path("."));
     assertTrue(job.isSuccessful());
 
+
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -122,14 +123,13 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
     job = Import.createSubmittableJob(conf, args);
     job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
-    HBaseFsck.debugLsr(conf, new Path("."));
     assertTrue(job.isSuccessful());
 
     Get g = new Get(ROW1);
@@ -174,7 +174,7 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -198,7 +198,7 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 

From e852a79fe3354844e9366df917227c3101beb6c3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 5 Jul 2012 22:28:16 +0000
Subject: [PATCH 0325/1540] HBASE-5909 SlabStats should be a daemon thread

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1357974 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java    | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
index 6d343617a4ad..2557862d5a46 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
@@ -59,10 +59,8 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize {
   static final Log LOG = LogFactory.getLog(SlabCache.class);
   static final int STAT_THREAD_PERIOD_SECS = 60 * 5;
 
-  private final ScheduledExecutorService scheduleThreadPool = Executors
-      .newScheduledThreadPool(1,
-          new ThreadFactoryBuilder().setNameFormat("Slab Statistics #%d")
-              .build());
+  private final ScheduledExecutorService scheduleThreadPool = Executors.newScheduledThreadPool(1,
+      new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Slab Statistics #%d").build());
 
   long size;
   private final CacheStats stats;

From ad16cf5d1fc97580978b7ad901d7853fe3ad7f35 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 6 Jul 2012 03:02:05 +0000
Subject: [PATCH 0326/1540] HBASE-6253. Do not allow user to disable or drop
 ACL table (Gopinathan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358030 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/security/access/AccessController.java       |  4 ++++
 .../hbase/security/access/TestAccessController.java   | 11 +++++++++++
 2 files changed, 15 insertions(+)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index c84a93c3fb8b..2fe921badc57 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -616,6 +616,10 @@ public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName)
       throws IOException {
+    if (Bytes.equals(tableName, AccessControlLists.ACL_GLOBAL_NAME)) {
+      throw new AccessDeniedException("Not allowed to disable "
+          + AccessControlLists.ACL_TABLE_NAME_STR + " table.");
+    }
     requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 19d1a070389f..765f0af805cd 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -306,8 +306,19 @@ public Object run() throws Exception {
       }
     };
 
+    PrivilegedExceptionAction disableAclTable = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null),
+            AccessControlLists.ACL_TABLE_NAME);
+        return null;
+      }
+    };
+
     verifyAllowed(disableTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER);
     verifyDenied(disableTable, USER_RW, USER_RO, USER_NONE);
+    
+    // No user should be allowed to disable _acl_ table
+    verifyDenied(disableAclTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, USER_RW, USER_RO);
   }
 
   @Test

From 7498867506a04230afd6b3302101dda8461dab5b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 6 Jul 2012 18:24:39 +0000
Subject: [PATCH 0327/1540] HBASE-6311 Data error after majorCompaction caused
 by keeping MVCC for opened scanners (chunhui shen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358333 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanQueryMatcher.java  |  7 ++-
 .../hbase/regionserver/TestHRegion.java       | 50 +++++++++++++++++++
 2 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 95b7831c3846..3b5e21a6a98c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -291,10 +291,9 @@ public MatchCode match(KeyValue kv) throws IOException {
         }
         // Can't early out now, because DelFam come before any other keys
       }
-      if (retainDeletesInOutput ||
-          (!isUserScan &&
-              (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= 
-              timeToPurgeDeletes)) {
+      if (retainDeletesInOutput
+          || (!isUserScan && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes)
+          || kv.getMemstoreTS() > maxReadPointToTrackVersions) {
         // always include or it is not time yet to check whether it is OK
         // to purge deltes or not
         return MatchCode.INCLUDE;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 797c42be9e47..4bc9a5addaba 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -151,6 +151,56 @@ protected void tearDown() throws Exception {
   // /tmp/testtable
   //////////////////////////////////////////////////////////////////////////////
 
+  public void testCompactionAffectedByScanners() throws Exception {
+    String method = "testCompactionAffectedByScanners";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Bytes.toBytes("family");
+    Configuration conf = HBaseConfiguration.create();
+    this.region = initHRegion(tableName, method, conf, family);
+
+    Put put = new Put(Bytes.toBytes("r1"));
+    put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
+    region.put(put);
+    region.flushcache();
+
+
+    Scan scan = new Scan();
+    scan.setMaxVersions(3);
+    // open the first scanner
+    RegionScanner scanner1 = region.getScanner(scan);
+
+    Delete delete = new Delete(Bytes.toBytes("r1"));
+    region.delete(delete, null, false);
+    region.flushcache();
+
+    // open the second scanner
+    RegionScanner scanner2 = region.getScanner(scan);
+
+    List<KeyValue> results = new ArrayList<KeyValue>();
+
+    System.out.println("Smallest read point:" + region.getSmallestReadPoint());
+
+    // make a major compaction
+    region.compactStores(true);
+
+    // open the third scanner
+    RegionScanner scanner3 = region.getScanner(scan);
+
+    // get data from scanner 1, 2, 3 after major compaction
+    scanner1.next(results);
+    System.out.println(results);
+    assertEquals(1, results.size());
+
+    results.clear();
+    scanner2.next(results);
+    System.out.println(results);
+    assertEquals(0, results.size());
+
+    results.clear();
+    scanner3.next(results);
+    System.out.println(results);
+    assertEquals(0, results.size());
+  }
 
   public void testSkipRecoveredEditsReplay() throws Exception {
     String method = "testSkipRecoveredEditsReplay";

From 0bf93a36dc4d661656910f5d8729637763696d81 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 6 Jul 2012 21:37:45 +0000
Subject: [PATCH 0328/1540] HBASE-6341 Publicly expose HConnectionKey

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358436 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HConnectionManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index cdb55e0b9b6f..3edbc2eae5b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -381,7 +381,7 @@ public static <T> T execute(HConnectable<T> connectable) throws IOException {
    * added to the {@link HConnectionKey#properties} list.
    *
    */
-  static class HConnectionKey {
+  public static class HConnectionKey {
     public static String[] CONNECTION_PROPERTIES = new String[] {
         HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT,
         HConstants.ZOOKEEPER_CLIENT_PORT,

From ab105d78767a0b831a4a3791c98924be2f3fcb1e Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 6 Jul 2012 22:12:18 +0000
Subject: [PATCH 0329/1540] HBASE-6283 [region_mover.rb] Add option to exclude
 list of hosts on unload instead of just assuming the source node

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358449 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/region_mover.rb | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 94da5d90bd9b..f778a720344f 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -213,6 +213,16 @@ def stripServer(servers, hostname)
   return servername
 end
 
+# Returns a new serverlist that excludes the servername whose hostname portion
+# matches from the passed array of servers.
+def stripExcludes(servers, excludefile)
+  excludes = readExcludes(excludefile)
+  servers =  servers.find_all{|server| !excludes.contains(getHostnameFromServerName(server)) }
+  # return updated servers list
+  return servers
+end
+
+
 # Return servername that matches passed hostname
 def getServerName(servers, hostname)
   servername = nil
@@ -309,6 +319,10 @@ def unloadRegions(options, hostname)
   # Remove the server we are unloading from from list of servers.
   # Side-effect is the servername that matches this hostname 
   servername = stripServer(servers, hostname)
+
+  # Remove the servers in our exclude list from list of servers.
+  servers = stripExcludes(servers, options[:excludesFile])
+  puts "Valid region move targets: ", servers
   movedRegions = java.util.ArrayList.new()
   while true
     rs = getRegions(config, servername)
@@ -383,6 +397,29 @@ def loadRegions(options, hostname)
   end
 end
 
+# Returns an array of hosts to exclude as region move targets
+def readExcludes(filename)
+  if filename == nil
+    return java.util.ArrayList.new()
+  end 
+  if ! File.exist?(filename)  
+      puts "Error: Unable to read host exclude file: ", filename
+      raise RuntimeError
+  end 
+
+  f = File.new(filename, "r")
+  # Read excluded hosts list
+  excludes = java.util.ArrayList.new()
+  while (line = f.gets)
+    line.strip! # do an inplace drop of pre and post whitespaces
+    excludes.add(line) unless line.empty? # exclude empty lines
+  end
+  puts "Excluding hosts as region move targets: ", excludes
+  f.close
+  
+  return excludes
+end
+
 def getFilename(options, targetServer)
   filename = options[:file]
   if not filename
@@ -409,6 +446,9 @@ def getFilename(options, targetServer)
   opts.on('-d', '--debug', 'Display extra debug logging') do
     options[:debug] = true
   end
+  opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file|
+    options[:excludesFile] = file
+  end
 end
 optparse.parse!
 

From 68673c65657e7650d4a2ab755b1dba8cd7913b10 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 7 Jul 2012 02:11:43 +0000
Subject: [PATCH 0330/1540] HBASE-6313 Client hangs because the client is not
 notified (binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358489 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 0aed5bf6e26f..de1b85cf0731 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -68,8 +68,8 @@
  */
 public class HBaseClient {
 
-  private static final Log LOG =
-    LogFactory.getLog("org.apache.hadoop.ipc.HBaseClient");
+  private static final Log LOG = LogFactory
+      .getLog("org.apache.hadoop.ipc.HBaseClient");
   protected final PoolMap<ConnectionId, Connection> connections;
 
   protected final Class<? extends Writable> valueClass;   // class of call values
@@ -572,7 +572,7 @@ protected void receiveResponse() {
 
         if (LOG.isDebugEnabled())
           LOG.debug(getName() + " got value #" + id);
-        Call call = calls.remove(id);
+        Call call = calls.get(id);
 
         // Read the flag byte
         byte flag = in.readByte();
@@ -597,6 +597,7 @@ protected void receiveResponse() {
             call.setValue(value);
           }
         }
+        calls.remove(id);
       } catch (IOException e) {
         if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
           // Clean up open calls but don't treat this as a fatal condition,

From bb91baeff867d76cfd6701a14a1bae376258fada Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 8 Jul 2012 19:13:40 +0000
Subject: [PATCH 0331/1540] HBASE-6355 Allow HBase to compile against JDK7

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358817 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pom.xml b/pom.xml
index 6ca62541c826..fc6417a9714c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -355,6 +355,9 @@
             <target>${compileSource}</target>
             <showWarnings>true</showWarnings>
             <showDeprecation>false</showDeprecation>
+            <compilerArguments>
+              <Xlint:-options/>
+            </compilerArguments>
           </configuration>
         </plugin>
         <plugin>

From 36bf20e7f98922c37ba73e2f80de9087a00055f4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 8 Jul 2012 19:21:53 +0000
Subject: [PATCH 0332/1540] HBASE-6329 Stopping META regionserver when
 splitting region could cause daughter region to be assigned twice (chunhui
 shen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358821 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegionServer.java    |  9 ++++++---
 .../hadoop/hbase/regionserver/SplitRequest.java     |  7 +++++++
 .../hadoop/hbase/regionserver/SplitTransaction.java | 13 +++++--------
 .../regionserver/TestEndToEndSplitTransaction.java  |  3 ++-
 4 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 91c8aecdacfa..80416017e50f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -783,6 +783,11 @@ public void run() {
       this.hbaseMaster = null;
     }
     this.leases.close();
+
+    if (!killed) {
+      join();
+    }
+
     try {
       deleteMyEphemeralNode();
     } catch (KeeperException e) {
@@ -792,9 +797,6 @@ public void run() {
     LOG.info("stopping server " + this.serverNameFromMasterPOV +
       "; zookeeper connection closed.");
 
-    if (!killed) {
-      join();
-    }
     LOG.info(Thread.currentThread().getName() + " exiting");
   }
 
@@ -1616,6 +1618,7 @@ public void waitForServerOnline(){
   public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
       final boolean daughter)
   throws KeeperException, IOException {
+    checkOpen();
     LOG.info("Post open deploy tasks for region=" + r.getRegionNameAsString() +
       ", daughter=" + daughter);
     // Do checks to see if we need to compact (references or too many files)
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
index 6bfcc9067954..9ca424f7ce70 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
@@ -67,6 +67,13 @@ public void run() {
         st.execute(this.server, this.server);
         this.server.getMetrics().incrementSplitSuccessCount();
       } catch (Exception e) {
+        if (this.server.isStopping() || this.server.isStopped()) {
+          LOG.info(
+              "Skip rollback/cleanup of failed split of "
+                  + parent.getRegionNameAsString() + " because server is"
+                  + (this.server.isStopping() ? " stopping" : " stopped"), e);
+          return;
+        }
         try {
           LOG.info("Running rollback/cleanup of failed split of " +
             parent.getRegionNameAsString() + "; " + e.getMessage(), e);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index c1d6376c0856..36576b1aec84 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -341,11 +341,6 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
     boolean stopping = services != null && services.isStopping();
     // TODO: Is this check needed here?
     if (stopped || stopping) {
-      // add 2nd daughter first (see HBASE-4335)
-      MetaEditor.addDaughter(server.getCatalogTracker(),
-          b.getRegionInfo(), services.getServerName());
-      MetaEditor.addDaughter(server.getCatalogTracker(),
-          a.getRegionInfo(), services.getServerName());
       LOG.info("Not opening daughters " +
           b.getRegionInfo().getRegionNameAsString() +
           " and " +
@@ -396,7 +391,8 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
    * @param a second daughter region
    * @throws IOException If thrown, transaction failed. Call {@link #rollback(Server, RegionServerServices)}
    */
-  /* package */void transitionZKNode(final Server server, HRegion a, HRegion b)
+  /* package */void transitionZKNode(final Server server,
+      final RegionServerServices services, HRegion a, HRegion b)
       throws IOException {
     // Tell master about split by updating zk.  If we fail, abort.
     if (server != null && server.getZooKeeper() != null) {
@@ -420,7 +416,8 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
             parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo(),
             server.getServerName(), this.znodeVersion);
           spins++;
-        } while (this.znodeVersion != -1);
+        } while (this.znodeVersion != -1 && !server.isStopped()
+            && !services.isStopping());
       } catch (Exception e) {
         if (e instanceof InterruptedException) {
           Thread.currentThread().interrupt();
@@ -454,7 +451,7 @@ public PairOfSameType<HRegion> execute(final Server server,
   throws IOException {
     PairOfSameType<HRegion> regions = createDaughters(server, services);
     openDaughters(server, services, regions.getFirst(), regions.getSecond());
-    transitionZKNode(server, regions.getFirst(), regions.getSecond());
+    transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
     return regions;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 894b54e14e1a..33bbe9feba9d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -132,7 +132,8 @@ public void testMasterOpsWhileSplitting() throws Exception {
     assertTrue(test(con, tableName, lastRow, server));
 
     // 4. phase III
-    split.transitionZKNode(server, regions.getFirst(), regions.getSecond());
+    split.transitionZKNode(server, server, regions.getFirst(),
+        regions.getSecond());
     assertTrue(test(con, tableName, firstRow, server));
     assertTrue(test(con, tableName, lastRow, server));
   }

From da0e0a58427fdd84f127fc069b7e17a1bbea64bb Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 9 Jul 2012 07:45:54 +0000
Subject: [PATCH 0333/1540] HBASE-4379 [hbck] Does not complain about tables
 with no end region [Z,] (Anoop Sam John)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1358954 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 42 +++++++++++++-
 .../util/hbck/TableIntegrityErrorHandler.java |  9 +++
 .../hbck/TableIntegrityErrorHandlerImpl.java  |  7 +++
 .../hadoop/hbase/util/TestHBaseFsck.java      | 56 +++++++++++++++++++
 4 files changed, 113 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index cf356e43cff1..98fca20a12a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1700,6 +1700,13 @@ public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
             + " create a new region and regioninfo in HDFS to plug the hole.",
             getTableInfo(), hi);
       }
+      
+      @Override
+      public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
+        errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
+            "Last region should end with an empty key. You need to "
+                + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
+      }
 
       @Override
       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
@@ -1785,6 +1792,21 @@ public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
         fixes++;
       }
 
+      public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
+        errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
+            "Last region should end with an empty key. Creating a new "
+                + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
+        HTableDescriptor htd = getTableInfo().getHTD();
+        // from curEndKey to EMPTY_START_ROW
+        HRegionInfo newRegion = new HRegionInfo(htd.getName(), curEndKey,
+            HConstants.EMPTY_START_ROW);
+
+        HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
+        LOG.info("Table region end key was not empty. Created new empty region: " + newRegion
+            + " " + region);
+        fixes++;
+      }
+      
       /**
        * There is a hole in the hdfs regions that violates the table integrity
        * rules.  Create a new empty region that patches the hole.
@@ -1954,6 +1976,12 @@ void sidelineBigOverlaps(
      * @throws IOException
      */
     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
+      // When table is disabled no need to check for the region chain. Some of the regions
+      // accidently if deployed, this below code might report some issues like missing start
+      // or end regions or region hole in chain and may try to fix which is unwanted.
+      if (disabledTables.contains(this.tableName.getBytes())) {
+        return true;
+      }
       int originalErrorsCount = errors.getErrorList().size();
       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
       SortedSet<byte[]> splits = sc.getSplits();
@@ -2025,6 +2053,12 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc
         prevKey = key;
       }
 
+      // When the last region of a table is proper and having an empty end key, 'prevKey'
+      // will be null.
+      if (prevKey != null) {
+        handler.handleRegionEndKeyNotEmpty(prevKey);
+      }
+      
       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
         handler.handleOverlapGroup(overlap);
       }
@@ -2535,7 +2569,7 @@ public static enum ERROR_CODE {
       UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
-      FIRST_REGION_STARTKEY_NOT_EMPTY, DUPE_STARTKEYS,
+      FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT
     }
@@ -2543,6 +2577,7 @@ public static enum ERROR_CODE {
     public void report(String message);
     public void reportError(String message);
     public void reportError(ERROR_CODE errorCode, String message);
+    public void reportError(ERROR_CODE errorCode, String message, TableInfo table);
     public void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
     public void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info1, HbckInfo info2);
     public int summarize();
@@ -2578,6 +2613,11 @@ public synchronized void reportError(ERROR_CODE errorCode, String message) {
       showProgress = 0;
     }
 
+    public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
+      errorTables.add(table);
+      reportError(errorCode, message);
+    }
+    
     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
                                          HbckInfo info) {
       errorTables.add(table);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java
index 2d49d01af10b..4310bf866a35 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java
@@ -48,6 +48,15 @@ public interface TableIntegrityErrorHandler {
    *    has an empty start key.
    */
   void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException;
+  
+  /**
+   * Callback for handling case where a Table has a last region that does not
+   * have an empty end key.
+   *
+   * @param curEndKey The end key of the current last region. There should be a new region
+   *    with start key as this and an empty end key.
+   */
+  void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException;
 
   /**
    * Callback for handling a region that has the same start and end key.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java
index 2c34da806d9e..b142b0c45726 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java
@@ -54,6 +54,13 @@ public void setTableInfo(TableInfo ti2) {
   public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException {
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
+  }
+  
   /**
    * {@inheritDoc}
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 69be013c8fd3..91ea273d6c8b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1093,6 +1093,62 @@ public void testLingeringSplitParent() throws Exception {
     }
   }
 
+  /**
+   * This creates and fixes a bad table with a missing region which is the 1st region -- hole in
+   * meta and data missing in the fs.
+   */
+  @Test
+  public void testMissingFirstRegion() throws Exception {
+    String table = "testMissingFirstRegion";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by leaving a hole in the assignment, meta, and hdfs data
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
+          true, true);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
+      // fix hole
+      doFsck(conf, true);
+      // check that hole fixed
+      assertNoErrors(doFsck(conf, false));
+    } finally {
+      deleteTable(table);
+    }
+  }
+  
+  /**
+   * This creates and fixes a bad table with missing last region -- hole in meta and data missing in
+   * the fs.
+   */
+  @Test
+  public void testMissingLastRegion() throws Exception {
+    String table = "testMissingLastRegion";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by leaving a hole in the assignment, meta, and hdfs data
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
+          true, true);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
+      // fix hole
+      doFsck(conf, true);
+      // check that hole fixed
+      assertNoErrors(doFsck(conf, false));
+    } finally {
+      deleteTable(table);
+    }
+  }
+  
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 113459d2ba239dfe15fd39c7bd3055ec8cb301d8 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 10 Jul 2012 17:42:02 +0000
Subject: [PATCH 0334/1540] HBASE-6357 Failed distributed log splitting stuck
 on master web UI

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1359804 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                                 | 1 +
 .../org/apache/hadoop/hbase/master/SplitLogManager.java     | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index e2cf528da37a..57aa6f0d71fa 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -228,6 +228,7 @@ Bug
     [HBASE-5897] - prePut coprocessor hook causing substantial CPU usage
     [HBASE-5908] - TestHLogSplit.testTralingGarbageCorruptionFileSkipErrorsPasses should not use append to corrupt the HLog
     [HBASE-6265] - Calling getTimestamp() on a KV in cp.prePut() causes KV not to be flushed
+    [HBASE-6357] - Failed distributed log splitting stuck on master web UI
 
 Improvement
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 2a11756fd02e..18c53d7e6d31 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -268,8 +268,10 @@ public long splitLogDistributed(final List<Path> logDirs) throws IOException {
       tot_mgr_log_split_batch_err.incrementAndGet();
       LOG.warn("error while splitting logs in " + logDirs +
       " installed = " + batch.installed + " but only " + batch.done + " done");
-      throw new IOException("error or interrupt while splitting logs in "
-          + logDirs + " Task = " + batch);
+      String msg = "error or interrupted while splitting logs in "
+        + logDirs + " Task = " + batch;
+      status.abort(msg);
+      throw new IOException(msg);
     }
     for(Path logDir: logDirs){
       status.setStatus("Cleaning up log directory...");

From bdac60321bcdd1d94583c44b1239c1e997788540 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 11 Jul 2012 00:09:31 +0000
Subject: [PATCH 0335/1540] HBASE-6337 [MTTR] Remove renaming tmp log file in
 SplitLogManager (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1359959 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/SplitLogManager.java  |   4 +-
 .../hbase/regionserver/SplitLogWorker.java    |   4 +-
 .../hbase/regionserver/wal/HLogSplitter.java  | 126 +++++-------------
 .../hadoop/hbase/zookeeper/ZKSplitLog.java    |  26 +---
 .../hbase/regionserver/wal/TestHLogSplit.java |  35 +++--
 .../hbase/regionserver/wal/TestWALReplay.java |   8 +-
 6 files changed, 55 insertions(+), 148 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 18c53d7e6d31..dc1ff7082ed0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -133,10 +133,8 @@ public SplitLogManager(ZooKeeperWatcher zkw, final Configuration conf,
     this(zkw, conf, stopper, serverName, new TaskFinisher() {
       @Override
       public Status finish(String workerName, String logfile) {
-        String tmpname =
-          ZKSplitLog.getSplitLogDirTmpComponent(workerName, logfile);
         try {
-          HLogSplitter.moveRecoveredEditsFromTemp(tmpname, logfile, conf);
+          HLogSplitter.finishSplitLogFile(logfile, conf);
         } catch (IOException e) {
           LOG.warn("Could not finish splitting of log file " + logfile);
           return Status.ERR;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
index 89c8ab84da46..eaa21d658103 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -108,9 +108,7 @@ public Status exec(String filename, CancelableProgressable p) {
         // interrupted or has encountered a transient error and when it has
         // encountered a bad non-retry-able persistent error.
         try {
-          String tmpname =
-            ZKSplitLog.getSplitLogDirTmpComponent(serverName, filename);
-          if (HLogSplitter.splitLogFileToTemp(rootdir, tmpname,
+          if (HLogSplitter.splitLogFile(rootdir,
               fs.getFileStatus(new Path(filename)), fs, conf, p) == false) {
             return Status.PREEMPTED;
           }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 440bbb0383c8..cad382cfc875 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -326,16 +326,13 @@ private static long countTotalBytes(FileStatus[] logfiles) {
   }
 
   /**
-   * Splits a HLog file into a temporary staging area. tmpname is used to build
-   * the name of the staging area where the recovered-edits will be separated
-   * out by region and stored.
+   * Splits a HLog file into region's recovered-edits directory
    * <p>
    * If the log file has N regions then N recovered.edits files will be
    * produced. There is no buffering in this code. Instead it relies on the
    * buffering in the SequenceFileWriter.
    * <p>
    * @param rootDir
-   * @param tmpname
    * @param logfile
    * @param fs
    * @param conf
@@ -343,16 +340,16 @@ private static long countTotalBytes(FileStatus[] logfiles) {
    * @return false if it is interrupted by the progress-able.
    * @throws IOException
    */
-  static public boolean splitLogFileToTemp(Path rootDir, String tmpname,
-      FileStatus logfile, FileSystem fs,
-      Configuration conf, CancelableProgressable reporter) throws IOException {
+  static public boolean splitLogFile(Path rootDir, FileStatus logfile,
+      FileSystem fs, Configuration conf, CancelableProgressable reporter)
+      throws IOException {
     HLogSplitter s = new HLogSplitter(conf, rootDir, null, null /* oldLogDir */,
         fs);
-    return s.splitLogFileToTemp(logfile, tmpname, reporter);
+    return s.splitLogFile(logfile, reporter);
   }
 
-  public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
-      CancelableProgressable reporter)  throws IOException {	    
+  public boolean splitLogFile(FileStatus logfile,
+      CancelableProgressable reporter) throws IOException {
     final Map<byte[], Object> logWriters = Collections.
     synchronizedMap(new TreeMap<byte[], Object>(Bytes.BYTES_COMPARATOR));
     boolean isCorrupted = false;
@@ -384,7 +381,7 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
       in = getReader(fs, logfile, conf, skipErrors);
     } catch (CorruptedLogFileException e) {
       LOG.warn("Could not get reader, corrupted log file " + logPath, e);
-      ZKSplitLog.markCorrupted(rootDir, tmpname, fs);
+      ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);
       isCorrupted = true;
     }
     if (in == null) {
@@ -412,7 +409,7 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
         }
         WriterAndPath wap = (WriterAndPath)o;
         if (wap == null) {
-          wap = createWAP(region, entry, rootDir, tmpname, fs, conf);
+          wap = createWAP(region, entry, rootDir, fs, conf);
           numNewlyOpenedFiles++;
           if (wap == null) {
             // ignore edits from this region. It doesn't exist anymore.
@@ -447,7 +444,7 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
       }
     } catch (CorruptedLogFileException e) {
       LOG.warn("Could not parse, corrupted log file " + logPath, e);
-      ZKSplitLog.markCorrupted(rootDir, tmpname, fs);
+      ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);
       isCorrupted = true;
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
@@ -524,94 +521,40 @@ public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
   }
 
   /**
-   * Completes the work done by splitLogFileToTemp by moving the
-   * recovered.edits from the staging area to the respective region server's
-   * directories.
+   * Completes the work done by splitLogFile by archiving logs
    * <p>
    * It is invoked by SplitLogManager once it knows that one of the
-   * SplitLogWorkers have completed the splitLogFileToTemp() part. If the
-   * master crashes then this function might get called multiple times.
+   * SplitLogWorkers have completed the splitLogFile() part. If the master
+   * crashes then this function might get called multiple times.
    * <p>
-   * @param tmpname
+   * @param logfile
    * @param conf
    * @throws IOException
    */
-  public static void moveRecoveredEditsFromTemp(String tmpname,
-      String logfile, Configuration conf)
-  throws IOException{
+  public static void finishSplitLogFile(String logfile, Configuration conf)
+      throws IOException {
     Path rootdir = FSUtils.getRootDir(conf);
     Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
-    moveRecoveredEditsFromTemp(tmpname, rootdir, oldLogDir, logfile, conf);
+    finishSplitLogFile(rootdir, oldLogDir, logfile, conf);
   }
 
-  public static void moveRecoveredEditsFromTemp(String tmpname,
-      Path rootdir, Path oldLogDir,
-      String logfile, Configuration conf)
-  throws IOException{
+  public static void finishSplitLogFile(Path rootdir, Path oldLogDir,
+      String logfile, Configuration conf) throws IOException {
     List<Path> processedLogs = new ArrayList<Path>();
     List<Path> corruptedLogs = new ArrayList<Path>();
     FileSystem fs;
     fs = rootdir.getFileSystem(conf);
     Path logPath = new Path(logfile);
-    if (ZKSplitLog.isCorrupted(rootdir, tmpname, fs)) {
+    if (ZKSplitLog.isCorrupted(rootdir, logPath.getName(), fs)) {
       corruptedLogs.add(logPath);
     } else {
       processedLogs.add(logPath);
     }
-    Path stagingDir = ZKSplitLog.getSplitLogDir(rootdir, tmpname);
-    List<FileStatus> files = listAll(fs, stagingDir);
-    for (FileStatus f : files) {
-      Path src = f.getPath();
-      Path dst = ZKSplitLog.stripSplitLogTempDir(rootdir, src);
-      if (ZKSplitLog.isCorruptFlagFile(dst)) {
-        continue;
-      }
-      if (fs.exists(src)) {
-        if (fs.exists(dst)) {
-          fs.delete(dst, false);
-        } else {
-          Path regionDir = dst.getParent().getParent();
-          if (!fs.exists(regionDir)) {
-            // See HBASE-6050.
-            LOG.warn("Could not move recovered edits from " + src
-                + " to destination " + regionDir + " as it doesn't exist.");
-            continue;
-          }
-          Path dstdir = dst.getParent();
-          if (!fs.exists(dstdir)) {
-            if (!fs.mkdirs(dstdir)) LOG.warn("mkdir failed on " + dstdir);
-          }
-        }
-        fs.rename(src, dst);
-        LOG.debug(" moved " + src + " => " + dst);
-      } else {
-        LOG.debug("Could not move recovered edits from " + src +
-            " as it doesn't exist");
-      }
-    }
-    archiveLogs(null, corruptedLogs, processedLogs,
-        oldLogDir, fs, conf);
+    archiveLogs(null, corruptedLogs, processedLogs, oldLogDir, fs, conf);
+    Path stagingDir = ZKSplitLog.getSplitLogDir(rootdir, logPath.getName());
     fs.delete(stagingDir, true);
-    return;
   }
 
-  private static List<FileStatus> listAll(FileSystem fs, Path dir)
-  throws IOException {
-    List<FileStatus> fset = new ArrayList<FileStatus>(100);
-    FileStatus [] files = fs.exists(dir)? fs.listStatus(dir): null;
-    if (files != null) {
-      for (FileStatus f : files) {
-        if (f.isDir()) {
-          fset.addAll(listAll(fs, f.getPath()));
-        } else {
-          fset.add(f);
-        }
-      }
-    }
-    return fset;
-  }
-
-
   /**
    * Moves processed logs to a oldLogDir after successful processing Moves
    * corrupted logs (any log that couldn't be successfully parsed to corruptDir
@@ -1102,15 +1045,14 @@ void finish() {
     }
   }
 
-  private WriterAndPath createWAP(byte[] region, Entry entry,
-      Path rootdir, String tmpname, FileSystem fs, Configuration conf)
+  private WriterAndPath createWAP(byte[] region, Entry entry, Path rootdir,
+      FileSystem fs, Configuration conf)
   throws IOException {
-    Path regionedits = getRegionSplitEditsPath(fs, entry, rootdir,
-        tmpname==null);
+    Path regionedits = getRegionSplitEditsPath(fs, entry, rootdir, true);
     if (regionedits == null) {
       return null;
     }
-    if ((tmpname == null) && fs.exists(regionedits)) {
+    if (fs.exists(regionedits)) {
       LOG.warn("Found existing old edits file. It could be the "
           + "result of a previous failed split attempt. Deleting "
           + regionedits + ", length="
@@ -1119,18 +1061,10 @@ private WriterAndPath createWAP(byte[] region, Entry entry,
         LOG.warn("Failed delete of old " + regionedits);
       }
     }
-    Path editsfile;
-    if (tmpname != null) {
-      // During distributed log splitting the output by each
-      // SplitLogWorker is written to a temporary area.
-      editsfile = convertRegionEditsToTemp(rootdir, regionedits, tmpname);
-    } else {
-      editsfile = regionedits;
-    }
-    Writer w = createWriter(fs, editsfile, conf);
-    LOG.debug("Creating writer path=" + editsfile + " region="
+    Writer w = createWriter(fs, regionedits, conf);
+    LOG.debug("Creating writer path=" + regionedits + " region="
         + Bytes.toStringBinary(region));
-    return (new WriterAndPath(editsfile, w));
+    return (new WriterAndPath(regionedits, w));
   }
 
   Path convertRegionEditsToTemp(Path rootdir, Path edits, String tmpname) {
@@ -1318,7 +1252,7 @@ WriterAndPath getWriterAndPath(Entry entry) throws IOException {
       if (blacklistedRegions.contains(region)) {
         return null;
       }
-      ret = createWAP(region, entry, rootDir, null, fs, conf);
+      ret = createWAP(region, entry, rootDir, fs, conf);
       if (ret == null) {
         blacklistedRegions.add(region);
         return null;
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
index 02034dc86720..f02c17886f78 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
@@ -156,27 +156,13 @@ public static Path getSplitLogDir(Path rootdir, String tmpname) {
     return new Path(new Path(rootdir, HConstants.SPLIT_LOGDIR_NAME), tmpname);
   }
 
-  public static Path stripSplitLogTempDir(Path rootdir, Path file) {
-    int skipDepth = rootdir.depth() + 2;
-    List<String> components = new ArrayList<String>(10);
-    do {
-      components.add(file.getName());
-      file = file.getParent();
-    } while (file.depth() > skipDepth);
-    Path ret = rootdir;
-    for (int i = components.size() - 1; i >= 0; i--) {
-      ret = new Path(ret, components.get(i));
-    }
-    return ret;
-  }
-
   public static String getSplitLogDirTmpComponent(String worker, String file) {
     return (worker + "_" + ZKSplitLog.encode(file));
   }
 
-  public static void markCorrupted(Path rootdir, String tmpname,
+  public static void markCorrupted(Path rootdir, String logFileName,
       FileSystem fs) {
-    Path file = new Path(getSplitLogDir(rootdir, tmpname), "corrupt");
+    Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
     try {
       fs.createNewFile(file);
     } catch (IOException e) {
@@ -185,18 +171,14 @@ public static void markCorrupted(Path rootdir, String tmpname,
     }
   }
 
-  public static boolean isCorrupted(Path rootdir, String tmpname,
+  public static boolean isCorrupted(Path rootdir, String logFileName,
       FileSystem fs) throws IOException {
-    Path file = new Path(getSplitLogDir(rootdir, tmpname), "corrupt");
+    Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
     boolean isCorrupt;
     isCorrupt = fs.exists(file);
     return isCorrupt;
   }
 
-  public static boolean isCorruptFlagFile(Path file) {
-    return file.getName().equals("corrupt");
-  }
-
 
   public static class Counters {
     //SplitLogManager counters
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index e3cf968aaa03..69a4e226d947 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -1016,10 +1016,9 @@ public void testSplitLogFileWithOneRegion() throws IOException {
     generateHLogs(1, 10, -1);
     FileStatus logfile = fs.listStatus(hlogDir)[0];
     fs.initialize(fs.getUri(), conf);
-    HLogSplitter.splitLogFileToTemp(hbaseDir, "tmpdir", logfile, fs,
-        conf, reporter);
-    HLogSplitter.moveRecoveredEditsFromTemp("tmpdir", hbaseDir, oldLogDir,
-        logfile.getPath().toString(), conf);
+    HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter);
+    HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath()
+        .toString(), conf);
 
 
     Path originalLog = (fs.listStatus(oldLogDir))[0].getPath();
@@ -1046,10 +1045,9 @@ public void testSplitLogFileDeletedRegionDir()
     LOG.info("Region directory is" + regiondir);
     fs.delete(regiondir, true);
     
-    HLogSplitter.splitLogFileToTemp(hbaseDir, "tmpdir", logfile, fs,
-        conf, reporter);
-    HLogSplitter.moveRecoveredEditsFromTemp("tmpdir", hbaseDir, oldLogDir,
-        logfile.getPath().toString(), conf);
+    HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter);
+    HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath()
+        .toString(), conf);
     
     assertTrue(!fs.exists(regiondir));
     assertTrue(true);
@@ -1065,10 +1063,9 @@ public void testSplitLogFileEmpty() throws IOException {
 
     fs.initialize(fs.getUri(), conf);
 
-    HLogSplitter.splitLogFileToTemp(hbaseDir, "tmpdir", logfile, fs,
-        conf, reporter);
-    HLogSplitter.moveRecoveredEditsFromTemp("tmpdir", hbaseDir, oldLogDir,
-        logfile.getPath().toString(), conf);
+    HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter);
+    HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath()
+        .toString(), conf);
     Path tdir = HTableDescriptor.getTableDir(hbaseDir, TABLE_NAME);
     assertFalse(fs.exists(tdir));
 
@@ -1082,10 +1079,9 @@ public void testSplitLogFileMultipleRegions() throws IOException {
     FileStatus logfile = fs.listStatus(hlogDir)[0];
     fs.initialize(fs.getUri(), conf);
 
-    HLogSplitter.splitLogFileToTemp(hbaseDir, "tmpdir", logfile, fs,
-        conf, reporter);
-    HLogSplitter.moveRecoveredEditsFromTemp("tmpdir", hbaseDir, oldLogDir,
-        logfile.getPath().toString(), conf);
+    HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter);
+    HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath()
+        .toString(), conf);
     for (String region : regions) {
       Path recovered = getLogForRegion(hbaseDir, TABLE_NAME, region);
       assertEquals(10, countHLog(recovered, fs, conf));
@@ -1103,10 +1099,9 @@ public void testSplitLogFileFirstLineCorruptionLog()
         Corruptions.INSERT_GARBAGE_ON_FIRST_LINE, true, fs);
 
     fs.initialize(fs.getUri(), conf);
-    HLogSplitter.splitLogFileToTemp(hbaseDir, "tmpdir", logfile, fs,
-        conf, reporter);
-    HLogSplitter.moveRecoveredEditsFromTemp("tmpdir", hbaseDir, oldLogDir,
-        logfile.getPath().toString(), conf);
+    HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter);
+    HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath()
+        .toString(), conf);
 
     final Path corruptDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get(
         "hbase.regionserver.hlog.splitlog.corrupt.dir", ".corrupt"));
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index e9517dce4f42..6e6c1e113fe6 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -563,10 +563,10 @@ public void testSequentialEditLogSeqNum() throws IOException {
     wal.completeCacheFlush(hri.getEncodedNameAsBytes(), hri.getTableName(), sequenceNumber, false);
     wal.close();
     FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
-    HLogSplitter.splitLogFileToTemp(hbaseRootDir, hbaseRootDir + "/temp", listStatus[0], this.fs,
-        this.conf, null);
-    FileStatus[] listStatus1 = this.fs.listStatus(new Path(hbaseRootDir + "/temp/" + tableNameStr
-        + "/" + hri.getEncodedName() + "/recovered.edits"));
+    HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0], this.fs, this.conf,
+        null);
+    FileStatus[] listStatus1 = this.fs.listStatus(new Path(hbaseRootDir + "/"
+        + tableNameStr + "/" + hri.getEncodedName() + "/recovered.edits"));
     int editCount = 0;
     for (FileStatus fileStatus : listStatus1) {
       editCount = Integer.parseInt(fileStatus.getPath().getName());

From 84d44a169651b0e2be5a0237fdd0b9afd666b8d9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 11 Jul 2012 02:46:11 +0000
Subject: [PATCH 0336/1540] HBASE-6355 Addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1359998 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index fc6417a9714c..9fe2b3c60df3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -355,9 +355,7 @@
             <target>${compileSource}</target>
             <showWarnings>true</showWarnings>
             <showDeprecation>false</showDeprecation>
-            <compilerArguments>
-              <Xlint:-options/>
-            </compilerArguments>
+            <compilerArgument>-Xlint:-options</compilerArgument>
           </configuration>
         </plugin>
         <plugin>

From c2afb45db96d31a1f50ff9671547f85824b1cff2 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 11 Jul 2012 06:19:20 +0000
Subject: [PATCH 0337/1540] HBASE-6284 Introduce HRegion#doMiniBatchMutation()
 (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1360020 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 203 ++++++++++++------
 .../hbase/regionserver/HRegionServer.java     |  38 ++--
 .../metrics/OperationMetrics.java             |  11 +
 3 files changed, 163 insertions(+), 89 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 3f87e13f8823..0c570ab51ce2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1699,15 +1699,14 @@ void delete(Map<byte[], List<KeyValue>> familyMap, UUID clusterId,
   }
 
   /**
-   * Setup a Delete object with correct timestamps.
-   * Caller should the row and region locks.
-   * @param delete
+   * Setup correct timestamps in the KVs in Delete object.
+   * Caller should have the row and region locks.
+   * @param familyMap
    * @param now
    * @throws IOException
    */
-  private void prepareDeleteTimestamps(Delete delete, byte[] byteNow)
+  private void prepareDeleteTimestamps(Map<byte[], List<KeyValue>> familyMap, byte[] byteNow)
       throws IOException {
-    Map<byte[], List<KeyValue>> familyMap = delete.getFamilyMap();
     for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
 
       byte[] family = e.getKey();
@@ -1776,7 +1775,7 @@ private void internalDelete(Delete delete, UUID clusterId,
 
     updatesLock.readLock().lock();
     try {
-      prepareDeleteTimestamps(delete, byteNow);
+      prepareDeleteTimestamps(delete.getFamilyMap(), byteNow);
 
       if (writeToWAL) {
         // write/sync to WAL should happen before we touch memstore.
@@ -1903,31 +1902,47 @@ public boolean isDone() {
 
   /**
    * Perform a batch put with no pre-specified locks
-   * @see HRegion#put(Pair[])
+   * @see HRegion#batchMutate(Pair[])
    */
   public OperationStatus[] put(Put[] puts) throws IOException {
     @SuppressWarnings("unchecked")
-    Pair<Put, Integer> putsAndLocks[] = new Pair[puts.length];
+    Pair<Mutation, Integer> putsAndLocks[] = new Pair[puts.length];
 
     for (int i = 0; i < puts.length; i++) {
-      putsAndLocks[i] = new Pair<Put, Integer>(puts[i], null);
+      putsAndLocks[i] = new Pair<Mutation, Integer>(puts[i], null);
     }
-    return put(putsAndLocks);
+    return batchMutate(putsAndLocks);
   }
 
   /**
    * Perform a batch of puts.
-   *
    * @param putsAndLocks
    *          the list of puts paired with their requested lock IDs.
+   * @return an array of OperationStatus which internally contains the OperationStatusCode and the
+   *         exceptionMessage if any.
+   * @throws IOException
+   * @deprecated Instead use {@link HRegion#batchMutate(Pair[])}
+   */
+  @Deprecated
+  public OperationStatus[] put(Pair<Put, Integer>[] putsAndLocks) throws IOException {
+    Pair<Mutation, Integer>[] mutationsAndLocks = new Pair[putsAndLocks.length];
+    System.arraycopy(putsAndLocks, 0, mutationsAndLocks, 0, putsAndLocks.length);
+    return batchMutate(mutationsAndLocks);
+  }
+  
+  /**
+   * Perform a batch of mutations.
+   * It supports only Put and Delete mutations and will ignore other types passed.
+   * @param mutationsAndLocks
+   *          the list of mutations paired with their requested lock IDs.
    * @return an array of OperationStatus which internally contains the
    *         OperationStatusCode and the exceptionMessage if any.
    * @throws IOException
    */
-  public OperationStatus[] put(
-      Pair<Put, Integer>[] putsAndLocks) throws IOException {
-    BatchOperationInProgress<Pair<Put, Integer>> batchOp =
-      new BatchOperationInProgress<Pair<Put,Integer>>(putsAndLocks);
+  public OperationStatus[] batchMutate(
+      Pair<Mutation, Integer>[] mutationsAndLocks) throws IOException {
+    BatchOperationInProgress<Pair<Mutation, Integer>> batchOp =
+      new BatchOperationInProgress<Pair<Mutation,Integer>>(mutationsAndLocks);
 
     boolean initialized = false;
 
@@ -1941,10 +1956,10 @@ public OperationStatus[] put(
       try {
         if (!initialized) {
           this.writeRequestsCount.increment();
-          doPrePutHook(batchOp);
+          doPreMutationHook(batchOp);
           initialized = true;
         }
-        long addedSize = doMiniBatchPut(batchOp);
+        long addedSize = doMiniBatchMutation(batchOp);
         newSize = this.addAndGetGlobalMemstoreSize(addedSize);
       } finally {
         closeRegionOperation();
@@ -1956,18 +1971,32 @@ public OperationStatus[] put(
     return batchOp.retCodeDetails;
   }
 
-  private void doPrePutHook(BatchOperationInProgress<Pair<Put, Integer>> batchOp)
+  private void doPreMutationHook(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp)
       throws IOException {
     /* Run coprocessor pre hook outside of locks to avoid deadlock */
     WALEdit walEdit = new WALEdit();
     if (coprocessorHost != null) {
       for (int i = 0; i < batchOp.operations.length; i++) {
-        Pair<Put, Integer> nextPair = batchOp.operations[i];
-        Put put = nextPair.getFirst();
-        if (coprocessorHost.prePut(put, walEdit, put.getWriteToWAL())) {
-          // pre hook says skip this Put
-          // mark as success and skip in doMiniBatchPut
-          batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+        Pair<Mutation, Integer> nextPair = batchOp.operations[i];
+        Mutation m = nextPair.getFirst();
+        if (m instanceof Put) {
+          if (coprocessorHost.prePut((Put) m, walEdit, m.getWriteToWAL())) {
+            // pre hook says skip this Put
+            // mark as success and skip in doMiniBatchMutation
+            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+          }
+        } else if (m instanceof Delete) {
+          if (coprocessorHost.preDelete((Delete) m, walEdit, m.getWriteToWAL())) {
+            // pre hook says skip this Delete
+            // mark as success and skip in doMiniBatchMutation
+            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+          }
+        } else {
+          // In case of passing Append mutations along with the Puts and Deletes in batchMutate
+          // mark the operation return code as failure so that it will not be considered in
+          // the doMiniBatchMutation
+          batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE,
+              "Put/Delete mutations only supported in batchMutate() now");
         }
         if (!walEdit.isEmpty()) {
           batchOp.walEditsFromCoprocessors[i] = walEdit;
@@ -1977,14 +2006,19 @@ private void doPrePutHook(BatchOperationInProgress<Pair<Put, Integer>> batchOp)
     }
   }
 
+  // The mutation will be either a Put or Delete.
   @SuppressWarnings("unchecked")
-  private long doMiniBatchPut(
-      BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException {
+  private long doMiniBatchMutation(
+      BatchOperationInProgress<Pair<Mutation, Integer>> batchOp) throws IOException {
 
-    // The set of columnFamilies first seen.
-    Set<byte[]> cfSet = null;
+    // The set of columnFamilies first seen for Put.
+    Set<byte[]> putsCfSet = null;
     // variable to note if all Put items are for the same CF -- metrics related
-    boolean cfSetConsistent = true;
+    boolean putsCfSetConsistent = true;
+    // The set of columnFamilies first seen for Delete.
+    Set<byte[]> deletesCfSet = null;
+    // variable to note if all Delete items are for the same CF -- metrics related
+    boolean deletesCfSetConsistent = true;
     long startTimeMs = EnvironmentEdgeManager.currentTimeMillis();
 
     WALEdit walEdit = new WALEdit();
@@ -2002,6 +2036,7 @@ private long doMiniBatchPut(
     int firstIndex = batchOp.nextIndexToProcess;
     int lastIndexExclusive = firstIndex;
     boolean success = false;
+    int noOfPuts = 0, noOfDeletes = 0;
     try {
       // ------------------------------------
       // STEP 1. Try to acquire as many locks as we can, and ensure
@@ -2010,11 +2045,11 @@ private long doMiniBatchPut(
       int numReadyToWrite = 0;
       long now = EnvironmentEdgeManager.currentTimeMillis();
       while (lastIndexExclusive < batchOp.operations.length) {
-        Pair<Put, Integer> nextPair = batchOp.operations[lastIndexExclusive];
-        Put put = nextPair.getFirst();
+        Pair<Mutation, Integer> nextPair = batchOp.operations[lastIndexExclusive];
+        Mutation mutation = nextPair.getFirst();
         Integer providedLockId = nextPair.getSecond();
 
-        Map<byte[], List<KeyValue>> familyMap = put.getFamilyMap();
+        Map<byte[], List<KeyValue>> familyMap = mutation.getFamilyMap();
         // store the family map reference to allow for mutations
         familyMaps[lastIndexExclusive] = familyMap;
 
@@ -2025,22 +2060,24 @@ private long doMiniBatchPut(
           continue;
         }
 
-        // Check the families in the put. If bad, skip this one.
         try {
-          checkFamilies(familyMap.keySet());
-          checkTimestamps(put, now);
+          if (mutation instanceof Put) {
+            checkFamilies(familyMap.keySet());
+            checkTimestamps(mutation.getFamilyMap(), now);
+          } else {
+            prepareDelete((Delete) mutation);
+          }
         } catch (DoNotRetryIOException dnrioe) {
-          LOG.warn("No such column family in batch put", dnrioe);
+          LOG.warn("No such column family in batch mutation", dnrioe);
           batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
               OperationStatusCode.SANITY_CHECK_FAILURE, dnrioe.getMessage());
           lastIndexExclusive++;
           continue;
         }
-
         // If we haven't got any rows in our batch, we should block to
         // get the next one.
         boolean shouldBlock = numReadyToWrite == 0;
-        Integer acquiredLockId = getLock(providedLockId, put.getRow(), shouldBlock);
+        Integer acquiredLockId = getLock(providedLockId, mutation.getRow(), shouldBlock);
         if (acquiredLockId == null) {
           // We failed to grab another lock
           assert !shouldBlock : "Should never fail to get lock when blocking";
@@ -2052,25 +2089,35 @@ private long doMiniBatchPut(
         lastIndexExclusive++;
         numReadyToWrite++;
 
-        // If Column Families stay consistent through out all of the
-        // individual puts then metrics can be reported as a mutliput across
-        // column families in the first put.
-        if (cfSet == null) {
-          cfSet = put.getFamilyMap().keySet();
+        if (mutation instanceof Put) {
+          // If Column Families stay consistent through out all of the
+          // individual puts then metrics can be reported as a mutliput across
+          // column families in the first put.
+          if (putsCfSet == null) {
+            putsCfSet = mutation.getFamilyMap().keySet();
+          } else {
+            putsCfSetConsistent = putsCfSetConsistent
+                && mutation.getFamilyMap().keySet().equals(putsCfSet);
+          }
         } else {
-          cfSetConsistent = cfSetConsistent && put.getFamilyMap().keySet().equals(cfSet);
+          if (deletesCfSet == null) {
+            deletesCfSet = mutation.getFamilyMap().keySet();
+          } else {
+            deletesCfSetConsistent = deletesCfSetConsistent
+                && mutation.getFamilyMap().keySet().equals(deletesCfSet);
+          }
         }
       }
 
       // we should record the timestamp only after we have acquired the rowLock,
-      // otherwise, newer puts are not guaranteed to have a newer timestamp
+      // otherwise, newer puts/deletes are not guaranteed to have a newer timestamp
       now = EnvironmentEdgeManager.currentTimeMillis();
       byte[] byteNow = Bytes.toBytes(now);
 
-      // Nothing to put -- an exception in the above such as NoSuchColumnFamily?
+      // Nothing to put/delete -- an exception in the above such as NoSuchColumnFamily?
       if (numReadyToWrite <= 0) return 0L;
 
-      // We've now grabbed as many puts off the list as we can
+      // We've now grabbed as many mutations off the list as we can
 
       // ------------------------------------
       // STEP 2. Update any LATEST_TIMESTAMP timestamps
@@ -2079,10 +2126,14 @@ private long doMiniBatchPut(
         // skip invalid
         if (batchOp.retCodeDetails[i].getOperationStatusCode()
             != OperationStatusCode.NOT_RUN) continue;
-
-        updateKVTimestamps(
-            familyMaps[i].values(),
-            byteNow);
+        Mutation mutation = batchOp.operations[i].getFirst();
+        if (mutation instanceof Put) {
+          updateKVTimestamps(familyMaps[i].values(), byteNow);
+          noOfPuts++;
+        } else {
+          prepareDeleteTimestamps(familyMaps[i], byteNow);
+          noOfDeletes++;
+        }
       }
 
       this.updatesLock.readLock().lock();
@@ -2123,9 +2174,11 @@ private long doMiniBatchPut(
         }
         batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
 
-        Put p = batchOp.operations[i].getFirst();
-        if (!p.getWriteToWAL()) {
-          recordPutWithoutWal(p.getFamilyMap());
+        Mutation m = batchOp.operations[i].getFirst();
+        if (!m.getWriteToWAL()) {
+          if (m instanceof Put) {
+            recordPutWithoutWal(m.getFamilyMap());
+          }
           continue;
         }
         // Add WAL edits by CP
@@ -2141,7 +2194,7 @@ private long doMiniBatchPut(
       // -------------------------
       // STEP 5. Append the edit to WAL. Do not sync wal.
       // -------------------------
-      Put first = batchOp.operations[firstIndex].getFirst();
+      Mutation first = batchOp.operations[firstIndex].getFirst();
       txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
                walEdit, first.getClusterId(), now, this.htableDescriptor);
 
@@ -2177,7 +2230,7 @@ private long doMiniBatchPut(
 
       // ------------------------------------
       // STEP 9. Run coprocessor post hooks. This should be done after the wal is
-      // sycned so that the coprocessor contract is adhered to.
+      // synced so that the coprocessor contract is adhered to.
       // ------------------------------------
       if (coprocessorHost != null) {
         for (int i = firstIndex; i < lastIndexExclusive; i++) {
@@ -2186,11 +2239,14 @@ private long doMiniBatchPut(
               != OperationStatusCode.SUCCESS) {
             continue;
           }
-          Put p = batchOp.operations[i].getFirst();
-          coprocessorHost.postPut(p, walEdit, p.getWriteToWAL());
+          Mutation m = batchOp.operations[i].getFirst();
+          if (m instanceof Put) {
+            coprocessorHost.postPut((Put) m, walEdit, m.getWriteToWAL());
+          } else {
+            coprocessorHost.postDelete((Delete) m, walEdit, m.getWriteToWAL());
+          }
         }
       }
-
       success = true;
       return addedSize;
     } finally {
@@ -2212,14 +2268,26 @@ private long doMiniBatchPut(
       }
 
       // do after lock
-      final long endTimeMs = EnvironmentEdgeManager.currentTimeMillis();
+      final long netTimeMs = EnvironmentEdgeManager.currentTimeMillis()- startTimeMs;
             
       // See if the column families were consistent through the whole thing.
       // if they were then keep them. If they were not then pass a null.
       // null will be treated as unknown.
-      final Set<byte[]> keptCfs = cfSetConsistent ? cfSet : null;
-      this.opMetrics.updateMultiPutMetrics(keptCfs, endTimeMs - startTimeMs);
-
+      // Total time taken might be involving Puts and Deletes.
+      // Split the time for puts and deletes based on the total number of Puts and Deletes.
+      long timeTakenForPuts = 0;
+      if (noOfPuts > 0) {
+        // There were some Puts in the batch.
+        double noOfMutations = noOfPuts + noOfDeletes;
+        timeTakenForPuts = (long) (netTimeMs * (noOfPuts / noOfMutations));
+        final Set<byte[]> keptCfs = putsCfSetConsistent ? putsCfSet : null;
+        this.opMetrics.updateMultiPutMetrics(keptCfs, timeTakenForPuts);
+      }
+      if (noOfDeletes > 0) {
+        // There were some Deletes in the batch.
+        final Set<byte[]> keptCfs = deletesCfSetConsistent ? deletesCfSet : null;
+        this.opMetrics.updateMultiDeleteMetrics(keptCfs, netTimeMs - timeTakenForPuts);
+      }
       if (!success) {
         for (int i = firstIndex; i < lastIndexExclusive; i++) {
           if (batchOp.retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.NOT_RUN) {
@@ -2527,10 +2595,10 @@ private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
 
   /**
    * Remove all the keys listed in the map from the memstore. This method is
-   * called when a Put has updated memstore but subequently fails to update
+   * called when a Put/Delete has updated memstore but subequently fails to update
    * the wal. This method is then invoked to rollback the memstore.
    */
-  private void rollbackMemstore(BatchOperationInProgress<Pair<Put, Integer>> batchOp,
+  private void rollbackMemstore(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
                                 Map<byte[], List<KeyValue>>[] familyMaps,
                                 int start, int end) {
     int kvsRolledback = 0;
@@ -2571,9 +2639,6 @@ private void checkFamilies(Collection<byte[]> families)
       checkFamily(family);
     }
   }
-  private void checkTimestamps(Put p, long now) throws DoNotRetryIOException {
-    checkTimestamps(p.getFamilyMap(), now);
-  }
 
   private void checkTimestamps(final Map<byte[], List<KeyValue>> familyMap,
       long now) throws DoNotRetryIOException {
@@ -4208,7 +4273,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
           } else if (m instanceof Delete) {
             Delete d = (Delete) m;
             prepareDelete(d);
-            prepareDeleteTimestamps(d, byteNow);
+            prepareDeleteTimestamps(d.getFamilyMap(), byteNow);
           } else {
             throw new DoNotRetryIOException(
                 "Action must be Put or Delete. But was: "
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 80416017e50f..527a177fe608 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -92,6 +92,7 @@
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.MultiAction;
 import org.apache.hadoop.hbase.client.MultiResponse;
+import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Row;
@@ -2018,15 +2019,15 @@ public int put(final byte[] regionName, final List<Put> puts)
       }
 
       @SuppressWarnings("unchecked")
-      Pair<Put, Integer>[] putsWithLocks = new Pair[puts.size()];
+      Pair<Mutation, Integer>[] putsWithLocks = new Pair[puts.size()];
 
       for (Put p : puts) {
         Integer lock = getLockFromId(p.getLockId());
-        putsWithLocks[i++] = new Pair<Put, Integer>(p, lock);
+        putsWithLocks[i++] = new Pair<Mutation, Integer>(p, lock);
       }
 
       this.requestCount.addAndGet(puts.size());
-      OperationStatus codes[] = region.put(putsWithLocks);
+      OperationStatus codes[] = region.batchMutate(putsWithLocks);
       for (i = 0; i < codes.length; i++) {
         if (codes[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
           return i;
@@ -3376,20 +3377,17 @@ public <R> MultiResponse multi(MultiAction<R> multi) throws IOException {
       // actions in the list.
       Collections.sort(actionsForRegion);
       Row action;
-      List<Action<R>> puts = new ArrayList<Action<R>>();
+      List<Action<R>> mutations = new ArrayList<Action<R>>();
       for (Action<R> a : actionsForRegion) {
         action = a.getAction();
         int originalIndex = a.getOriginalIndex();
 
         try {
-          if (action instanceof Delete) {
-            delete(regionName, (Delete)action);
-            response.add(regionName, originalIndex, new Result());
+          if (action instanceof Delete || action instanceof Put) {
+            mutations.add(a); 
           } else if (action instanceof Get) {
             response.add(regionName, originalIndex,
                 get(regionName, (Get)action));
-          } else if (action instanceof Put) {
-            puts.add(a);  // wont throw.
           } else if (action instanceof Exec) {
             ExecResult result = execCoprocessor(regionName, (Exec)action);
             response.add(regionName, new Pair<Integer, Object>(
@@ -3418,7 +3416,7 @@ public <R> MultiResponse multi(MultiAction<R> multi) throws IOException {
       // We do the puts with result.put so we can get the batching efficiency
       // we so need. All this data munging doesn't seem great, but at least
       // we arent copying bytes or anything.
-      if (!puts.isEmpty()) {
+      if (!mutations.isEmpty()) {
         try {
           HRegion region = getRegion(regionName);
 
@@ -3426,30 +3424,30 @@ public <R> MultiResponse multi(MultiAction<R> multi) throws IOException {
             this.cacheFlusher.reclaimMemStoreMemory();
           }
 
-          List<Pair<Put,Integer>> putsWithLocks =
-              Lists.newArrayListWithCapacity(puts.size());
-          for (Action<R> a : puts) {
-            Put p = (Put) a.getAction();
+          List<Pair<Mutation,Integer>> mutationsWithLocks =
+              Lists.newArrayListWithCapacity(mutations.size());
+          for (Action<R> a : mutations) {
+            Mutation m = (Mutation) a.getAction();
 
             Integer lock;
             try {
-              lock = getLockFromId(p.getLockId());
+              lock = getLockFromId(m.getLockId());
             } catch (UnknownRowLockException ex) {
               response.add(regionName, a.getOriginalIndex(), ex);
               continue;
             }
-            putsWithLocks.add(new Pair<Put, Integer>(p, lock));
+            mutationsWithLocks.add(new Pair<Mutation, Integer>(m, lock));
           }
 
-          this.requestCount.addAndGet(puts.size());
+          this.requestCount.addAndGet(mutations.size());
 
           OperationStatus[] codes =
-              region.put(putsWithLocks.toArray(new Pair[]{}));
+              region.batchMutate(mutationsWithLocks.toArray(new Pair[]{}));
 
           for( int i = 0 ; i < codes.length ; i++) {
             OperationStatus code = codes[i];
 
-            Action<R> theAction = puts.get(i);
+            Action<R> theAction = mutations.get(i);
             Object result = null;
 
             if (code.getOperationStatusCode() == OperationStatusCode.SUCCESS) {
@@ -3464,7 +3462,7 @@ public <R> MultiResponse multi(MultiAction<R> multi) throws IOException {
           }
         } catch (IOException ioe) {
           // fail all the puts with the ioe in question.
-          for (Action<R> a: puts) {
+          for (Action<R> a: mutations) {
             response.add(regionName, a.getOriginalIndex(), ioe);
           }
         }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
index fa91293f067e..460afea2ec70 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
@@ -44,6 +44,7 @@ public class OperationMetrics {
   private static final String ICV_KEY = "incrementColumnValue_";
   private static final String INCREMENT_KEY = "increment_";
   private static final String MULTIPUT_KEY = "multiput_";
+  private static final String MULTIDELETE_KEY = "multidelete_";
   private static final String APPEND_KEY = "append_";
   
   /** Conf key controlling whether we should expose metrics.*/
@@ -99,6 +100,16 @@ public void updateMultiPutMetrics(Set<byte[]> columnFamilies, long value) {
     doUpdateTimeVarying(columnFamilies, MULTIPUT_KEY, value);
   }
 
+  /**
+   * Update the stats associated with {@link HTable#delete(java.util.List)}.
+   *
+   * @param columnFamilies Set of CF's this multidelete is associated with
+   * @param value the time
+   */
+  public void updateMultiDeleteMetrics(Set<byte[]> columnFamilies, long value) {
+    doUpdateTimeVarying(columnFamilies, MULTIDELETE_KEY, value);
+  }
+  
   /**
    * Update the metrics associated with a {@link Get}
    * 

From 2eb6eb7f65baf2d2868cd0c76ea5d7d08cc7136c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 11 Jul 2012 14:09:39 +0000
Subject: [PATCH 0338/1540] HBASE-2730 Expose RS work queue contents on web UI

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1360180 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/CompactSplitThread.java      | 36 +++++++++++++++++++
 .../hbase/regionserver/HRegionServer.java     |  4 +++
 .../hbase/regionserver/MemStoreFlusher.java   | 20 +++++++++++
 .../hbase/regionserver/RSDumpServlet.java     | 30 +++++++++++++++-
 4 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 429b98d0e047..072f07cd7743 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -21,6 +21,8 @@
 
 import java.io.IOException;
 import java.util.concurrent.Executors;
+import java.util.Iterator;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadFactory;
@@ -137,6 +139,40 @@ public String toString() {
             : largeCompactions.getQueue().size())
         + ", split_queue=" + splits.getQueue().size();
   }
+  
+  public String dumpQueue() {
+    StringBuffer queueLists = new StringBuffer();
+    queueLists.append("Compaction/Split Queue dump:\n");
+    queueLists.append("  LargeCompation Queue:\n");
+    BlockingQueue<Runnable> lq = largeCompactions.getQueue();
+    Iterator it = lq.iterator();
+    while(it.hasNext()){
+      queueLists.append("    "+it.next().toString());
+      queueLists.append("\n");
+    }
+    
+    if( smallCompactions != null ){
+      queueLists.append("\n");
+      queueLists.append("  SmallCompation Queue:\n");
+      lq = smallCompactions.getQueue();
+      it = lq.iterator();
+      while(it.hasNext()){
+        queueLists.append("    "+it.next().toString());
+        queueLists.append("\n");
+      }
+    }
+    
+    queueLists.append("\n");
+    queueLists.append("  Split Queue:\n");
+    lq = splits.getQueue();
+    it = lq.iterator();
+    while(it.hasNext()){
+      queueLists.append("    "+it.next().toString());
+      queueLists.append("\n");
+    }
+    
+    return queueLists.toString();
+  }
 
   public synchronized boolean requestSplit(final HRegion r) {
     // don't split regions that are blocking
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 527a177fe608..b1e723303ad4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -248,6 +248,9 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
 
   /** region server process name */
   public static final String REGIONSERVER = "regionserver";
+  
+  /** region server configuration name */
+  public static final String REGIONSERVER_CONF = "regionserver_conf";
 
   /*
    * Space is reserved in HRS constructor and then released when aborting to
@@ -1558,6 +1561,7 @@ private int putUpWebUI() throws IOException {
         this.infoServer.addServlet("status", "/rs-status", RSStatusServlet.class);
         this.infoServer.addServlet("dump", "/dump", RSDumpServlet.class);
         this.infoServer.setAttribute(REGIONSERVER, this);
+        this.infoServer.setAttribute(REGIONSERVER_CONF, conf);
         this.infoServer.start();
         break;
       } catch (BindException e) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index 221ca49cb523..df64dba9a936 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -472,6 +472,26 @@ public synchronized void reclaimMemStoreMemory() {
     }
   }
 
+  @Override
+  public String toString() {
+    return "flush_queue="
+        + flushQueue.size();
+  }
+  
+  public String dumpQueue() {
+    StringBuilder queueList = new StringBuilder();
+    queueList.append("Flush Queue Queue dump:\n");
+    queueList.append("  Flush Queue:\n");
+    java.util.Iterator<FlushQueueEntry> it = flushQueue.iterator();
+    
+    while(it.hasNext()){
+      queueList.append("    "+it.next().toString());
+      queueList.append("\n");
+    }
+    
+    return queueList.toString();
+  }
+  
   interface FlushQueueEntry extends Delayed {}
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
index df33d82070d5..a4ec6d389601 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
@@ -44,6 +44,10 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     HRegionServer hrs = (HRegionServer)getServletContext().getAttribute(
         HRegionServer.REGIONSERVER);
     assert hrs != null : "No RS in context!";
+    
+    Configuration hrsconf = (Configuration)getServletContext().getAttribute(
+        HRegionServer.REGIONSERVER_CONF);
+    assert hrsconf != null : "No RS conf in context";
 
     response.setContentType("text/plain");
     OutputStream os = response.getOutputStream();
@@ -80,6 +84,30 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     long tailKb = getTailKbParam(request);
     LogMonitoring.dumpTailOfLogs(out, tailKb);
     
+    out.println("\n\nRS Queue:");
+    out.println(LINE);
+    if(isShowQueueDump(hrsconf)) {
+      dumpQueue(hrs, out);
+    }
+    
     out.flush();
-  }  
+  }
+  
+  private boolean isShowQueueDump(Configuration conf){
+    return conf.getBoolean("hbase.regionserver.servlet.show.queuedump", true);
+  }
+  
+  private void dumpQueue(HRegionServer hrs, PrintWriter out)
+      throws IOException {
+    // 1. Print out Compaction/Split Queue
+    out.println("Compaction/Split Queue summary: " 
+        + hrs.compactSplitThread.toString() );
+    out.println(hrs.compactSplitThread.dumpQueue());
+
+    // 2. Print out flush Queue
+    out.println("\nFlush Queue summary: "
+        + hrs.cacheFlusher.toString());
+    out.println(hrs.cacheFlusher.dumpQueue());
+  }
+  
 }

From 9cc4d2f3e0cfef22271115978a2dae27f62fcb8a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 11 Jul 2012 18:34:38 +0000
Subject: [PATCH 0339/1540] HBASE-6369 HTable is not closed in
 AggregationClient (binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1360342 13f79535-47bb-0310-9956-ffa450edef68
---
 .../client/coprocessor/AggregationClient.java | 195 +++++++++++-------
 1 file changed, 125 insertions(+), 70 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
index 3a019d38981b..4974a298a7b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
@@ -90,8 +90,6 @@ public AggregationClient(Configuration cfg) {
   public <R, S> R max(final byte[] tableName, final ColumnInterpreter<R, S> ci,
       final Scan scan) throws Throwable {
     validateParameters(scan);
-    HTable table = new HTable(conf, tableName);
-
     class MaxCallBack implements Batch.Callback<R> {
       R max = null;
 
@@ -105,13 +103,21 @@ public synchronized void update(byte[] region, byte[] row, R result) {
       }
     }
     MaxCallBack aMaxCallBack = new MaxCallBack();
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(), new Batch.Call<AggregateProtocol, R>() {
-      @Override
-      public R call(AggregateProtocol instance) throws IOException {
-        return instance.getMax(ci, scan);
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(), new Batch.Call<AggregateProtocol, R>() {
+            @Override
+            public R call(AggregateProtocol instance) throws IOException {
+              return instance.getMax(ci, scan);
+            }
+          }, aMaxCallBack);
+    } finally {
+      if (table != null) {
+        table.close();
       }
-    }, aMaxCallBack);
+    }
     return aMaxCallBack.getMax();
   }
 
@@ -154,16 +160,23 @@ public synchronized void update(byte[] region, byte[] row, R result) {
         min = (min == null || (result != null && ci.compare(result, min) < 0)) ? result : min;
       }
     }
-    HTable table = new HTable(conf, tableName);
     MinCallBack minCallBack = new MinCallBack();
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(), new Batch.Call<AggregateProtocol, R>() {
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(), new Batch.Call<AggregateProtocol, R>() {
 
-      @Override
-      public R call(AggregateProtocol instance) throws IOException {
-        return instance.getMin(ci, scan);
+            @Override
+            public R call(AggregateProtocol instance) throws IOException {
+              return instance.getMin(ci, scan);
+            }
+          }, minCallBack);
+    } finally {
+      if (table != null) {
+        table.close();
       }
-    }, minCallBack);
+    }
     log.debug("Min fom all regions is: " + minCallBack.getMinimum());
     return minCallBack.getMinimum();
   }
@@ -197,14 +210,21 @@ public void update(byte[] region, byte[] row, Long result) {
       }
     }
     RowNumCallback rowNum = new RowNumCallback();
-    HTable table = new HTable(conf, tableName);
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(), new Batch.Call<AggregateProtocol, Long>() {
-      @Override
-      public Long call(AggregateProtocol instance) throws IOException {
-        return instance.getRowNum(ci, scan);
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(), new Batch.Call<AggregateProtocol, Long>() {
+            @Override
+            public Long call(AggregateProtocol instance) throws IOException {
+              return instance.getRowNum(ci, scan);
+            }
+          }, rowNum);
+    } finally {
+      if (table != null) {
+        table.close();
       }
-    }, rowNum);
+    }
     return rowNum.getRowNumCount();
   }
 
@@ -233,14 +253,21 @@ public synchronized void update(byte[] region, byte[] row, S result) {
       }
     }
     SumCallBack sumCallBack = new SumCallBack();
-    HTable table = new HTable(conf, tableName);
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(), new Batch.Call<AggregateProtocol, S>() {
-      @Override
-      public S call(AggregateProtocol instance) throws IOException {
-        return instance.getSum(ci, scan);
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(), new Batch.Call<AggregateProtocol, S>() {
+            @Override
+            public S call(AggregateProtocol instance) throws IOException {
+              return instance.getSum(ci, scan);
+            }
+          }, sumCallBack);
+    } finally {
+      if (table != null) {
+        table.close();
       }
-    }, sumCallBack);
+    }
     return sumCallBack.getSumResult();
   }
 
@@ -270,14 +297,23 @@ public synchronized void update(byte[] region, byte[] row, Pair<S, Long> result)
       }
     }
     AvgCallBack avgCallBack = new AvgCallBack();
-    HTable table = new HTable(conf, tableName);
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(), new Batch.Call<AggregateProtocol, Pair<S, Long>>() {
-      @Override
-      public Pair<S, Long> call(AggregateProtocol instance) throws IOException {
-        return instance.getAvg(ci, scan);
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(),
+          new Batch.Call<AggregateProtocol, Pair<S, Long>>() {
+            @Override
+            public Pair<S, Long> call(AggregateProtocol instance)
+                throws IOException {
+              return instance.getAvg(ci, scan);
+            }
+          }, avgCallBack);
+    } finally {
+      if (table != null) {
+        table.close();
       }
-    }, avgCallBack);
+    }
     return avgCallBack.getAvgArgs();
   }
 
@@ -333,17 +369,24 @@ public synchronized void update(byte[] region, byte[] row, Pair<List<S>, Long> r
       }
     }
     StdCallback stdCallback = new StdCallback();
-    HTable table = new HTable(conf, tableName);
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(),
-        new Batch.Call<AggregateProtocol, Pair<List<S>, Long>>() {
-          @Override
-          public Pair<List<S>, Long> call(AggregateProtocol instance)
-              throws IOException {
-            return instance.getStd(ci, scan);
-          }
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(),
+          new Batch.Call<AggregateProtocol, Pair<List<S>, Long>>() {
+            @Override
+            public Pair<List<S>, Long> call(AggregateProtocol instance)
+                throws IOException {
+              return instance.getStd(ci, scan);
+            }
 
-        }, stdCallback);
+          }, stdCallback);
+    } finally {
+      if (table != null) {
+        table.close();
+      }
+    }
     return stdCallback.getStdParams();
   }
 
@@ -408,17 +451,22 @@ public synchronized void update(byte[] region, byte[] row, List<S> result) {
       }
     }
     StdCallback stdCallback = new StdCallback();
-    HTable table = new HTable(conf, tableName);
-    table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), scan
-        .getStopRow(),
-        new Batch.Call<AggregateProtocol, List<S>>() {
-          @Override
-          public List<S> call(AggregateProtocol instance)
-              throws IOException {
-            return instance.getMedian(ci, scan);
-          }
+    HTable table = null;
+    try {
+      table = new HTable(conf, tableName);
+      table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(),
+          scan.getStopRow(), new Batch.Call<AggregateProtocol, List<S>>() {
+            @Override
+            public List<S> call(AggregateProtocol instance) throws IOException {
+              return instance.getMedian(ci, scan);
+            }
 
-        }, stdCallback);
+          }, stdCallback);
+    } finally {
+      if (table != null) {
+        table.close();
+      }
+    }
     return stdCallback.getMedianParams();
   }
 
@@ -460,20 +508,22 @@ public <R, S> R median(final byte[] tableName, ColumnInterpreter<R, S> ci,
     Scan scan2 = new Scan(scan);
     // inherit stop row from method parameter
     if (startRow != null) scan2.setStartRow(startRow);
-    HTable table = new HTable(conf, tableName);
-    int cacheSize = scan2.getCaching();
-    if (!scan2.getCacheBlocks() || scan2.getCaching() < 2) {
-      scan2.setCacheBlocks(true);
-      cacheSize = 5;
-      scan2.setCaching(cacheSize);
-    }
-    ResultScanner scanner = table.getScanner(scan2);
-    Result[] results = null;
-    byte[] qualifier = quals.pollFirst();
-    // qualifier for the weight column
-    byte[] weightQualifier = weighted ? quals.pollLast() : qualifier;
-    R value = null;
+    HTable table = null;
+    ResultScanner scanner = null;
     try {
+      table = new HTable(conf, tableName);
+      int cacheSize = scan2.getCaching();
+      if (!scan2.getCacheBlocks() || scan2.getCaching() < 2) {
+        scan2.setCacheBlocks(true);
+        cacheSize = 5;
+        scan2.setCaching(cacheSize);
+      }
+      scanner = table.getScanner(scan2);
+      Result[] results = null;
+      byte[] qualifier = quals.pollFirst();
+      // qualifier for the weight column
+      byte[] weightQualifier = weighted ? quals.pollLast() : qualifier;
+      R value = null;
       do {
         results = scanner.next(cacheSize);
         if (results != null && results.length > 0) {
@@ -495,7 +545,12 @@ public <R, S> R median(final byte[] tableName, ColumnInterpreter<R, S> ci,
         }
       } while (results != null && results.length > 0);
     } finally {
-      scanner.close();
+      if (scanner != null) {
+        scanner.close();
+      }
+      if (table != null) {
+        table.close();
+      }
     }
     return null;
   }

From 034f131523856c5447144c7dbc758a5504f18ca1 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 12 Jul 2012 20:56:29 +0000
Subject: [PATCH 0340/1540] HBASE-6334 TestImprovement for
 TestHRegion.testWritesWhileGetting

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1360937 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestHRegion.java       | 68 +++++++++++--------
 1 file changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 4bc9a5addaba..5a2685c5180c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MultithreadedTestUtil;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
@@ -3128,21 +3129,19 @@ public void run() {
 
   /**
    * Writes very wide records and gets the latest row every time..
-   * Flushes and compacts the region every now and then to keep things
-   * realistic.
+   * Flushes and compacts the region aggressivly to catch issues.
    *
    * @throws IOException          by flush / scan / compaction
    * @throws InterruptedException when joining threads
    */
   public void testWritesWhileGetting()
-    throws IOException, InterruptedException {
-    byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
+    throws Exception {
+    byte[] tableName = Bytes.toBytes("testWritesWhileGetting");
     int testCount = 100;
     int numRows = 1;
     int numFamilies = 10;
     int numQualifiers = 100;
-    int flushInterval = 10;
-    int compactInterval = 10 * flushInterval;
+    int compactInterval = 100;
     byte[][] families = new byte[numFamilies][];
     for (int i = 0; i < numFamilies; i++) {
       families[i] = Bytes.toBytes("family" + i);
@@ -3153,14 +3152,37 @@ public void testWritesWhileGetting()
     }
 
     String method = "testWritesWhileGetting";
-    this.region = initHRegion(tableName, method, families);
+    Configuration conf = HBaseConfiguration.create();
+    // This test flushes constantly and can cause many files to be created, possibly
+    // extending over the ulimit.  Make sure compactions are aggressive in reducing
+    // the number of HFiles created.
+    conf.setInt("hbase.hstore.compaction.min", 1);
+    conf.setInt("hbase.hstore.compaction.max", 1000);
+    this.region = initHRegion(tableName, method, conf, families);
+    PutThread putThread = null;
+    MultithreadedTestUtil.TestContext ctx =
+      new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
     try {
-      PutThread putThread = new PutThread(numRows, families, qualifiers);
+      putThread = new PutThread(numRows, families, qualifiers);
       putThread.start();
       putThread.waitForFirstPut();
 
-      FlushThread flushThread = new FlushThread();
-      flushThread.start();
+      // Add a thread that flushes as fast as possible
+      ctx.addThread(new RepeatingTestThread(ctx) {
+    	private int flushesSinceCompact = 0;
+    	private final int maxFlushesSinceCompact = 20;
+        public void doAnAction() throws Exception {
+          if (region.flushcache()) {
+            ++flushesSinceCompact;
+          }
+          // Compact regularly to avoid creating too many files and exceeding the ulimit.
+          if (flushesSinceCompact == maxFlushesSinceCompact) {
+            region.compactStores(false);
+            flushesSinceCompact = 0;
+          }
+        }
+      });
+      ctx.startThreads();
 
       Get get = new Get(Bytes.toBytes("row0"));
       Result result = null;
@@ -3170,15 +3192,6 @@ public void testWritesWhileGetting()
       long prevTimestamp = 0L;
       for (int i = 0; i < testCount; i++) {
 
-        if (i != 0 && i % compactInterval == 0) {
-          region.compactStores(true);
-        }
-
-        if (i != 0 && i % flushInterval == 0) {
-          //System.out.println("iteration = " + i);
-          flushThread.flush();
-        }
-
         boolean previousEmpty = result == null || result.isEmpty();
         result = region.get(get, null);
         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
@@ -3206,25 +3219,24 @@ public void testWritesWhileGetting()
                     ", New KV: " +
                     kv + "(memStoreTS:" + kv.getMemstoreTS() + ")"
                     );
-                assertEquals(previousKV.getValue(), thisValue);
+                assertEquals(0, Bytes.compareTo(previousKV.getValue(), thisValue));
               }
             }
             previousKV = kv;
           }
         }
       }
-
-      putThread.done();
+    } finally {
+      if (putThread != null) putThread.done();
 
       region.flushcache();
 
-      putThread.join();
-      putThread.checkNoError();
+      if (putThread != null) {
+        putThread.join();
+        putThread.checkNoError();
+      }
 
-      flushThread.done();
-      flushThread.join();
-      flushThread.checkNoError();
-    } finally {
+      ctx.stop();
       HRegion.closeHRegion(this.region);
       this.region = null;
     }

From 7db3bcc0dcea6d9ee3c08c8e28791f915df1d210 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 13 Jul 2012 00:14:26 +0000
Subject: [PATCH 0341/1540] CHANGES.txt for 0.94.1RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361012 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 149 insertions(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 57aa6f0d71fa..ffd18fc2f7c1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,154 @@
 HBase Change Log
 
+Release 0.94.1 - 7/12/2012
+Sub-task
+
+    [HBASE-5342] - Grant/Revoke global permissions
+    [HBASE-5372] - Table mutation operations should check table level rights, not global rights
+    [HBASE-5385] - Delete table/column should delete stored permissions on -acl- table
+    [HBASE-6061] - Fix ACL "Admin" Table inconsistent permission check
+    [HBASE-6062] - preCheckAndPut/Delete() checks for READ when also a WRITE is performed
+    [HBASE-6092] - Authorize flush, split, compact operations in AccessController
+    [HBASE-6157] - Revoke of Global permission is not taking effect without restart.
+    [HBASE-6181] - TestStoreFile fails with jdk1.7
+    [HBASE-6188] - Remove the concept of table owner
+    [HBASE-6209] - ACL Corrections for AccessControllerProtocol apis
+    [HBASE-6224] - add Pre and Post coprocessor hooks for BulkLoad
+    [HBASE-6238] - Grant on META not taking effect
+    [HBASE-6252] - TABLE ADMIN should be allowed to relocate regions
+    [HBASE-6253] - Do not allow user to disable or drop ACL table
+    [HBASE-6292] - Compact can skip the security access control
+    [HBASE-6355] - Allow HBase to compile against JDK7
+
+Bug
+
+    [HBASE-4379] - [hbck] Does not complain about tables with no end region [Z,]
+    [HBASE-4891] - HTable.ClientScanner needs to clone the Scan object
+    [HBASE-5546] - Master assigns region in the original region server when opening region failed
+    [HBASE-5722] - NPE in ZKUtil#getChildDataAndWatchForNewChildren when ZK not available or NW down.
+    [HBASE-5733] - AssignmentManager#processDeadServersAndRegionsInTransition can fail with NPE.
+    [HBASE-5741] - ImportTsv does not check for table existence
+    [HBASE-5757] - TableInputFormat should handle as many errors as possible
+    [HBASE-5806] - Handle split region related failures on master restart and RS restart
+    [HBASE-5840] - Open Region FAILED_OPEN doesn't clear the TaskMonitor Status, keeps showing the old status
+    [HBASE-5853] - java.lang.RuntimeException: readObject can't find class org.apache.hadoop.hdfs.protocol.HdfsFileStatus
+    [HBASE-5874] - When 'fs.default.name' not configured, the hbck tool and Merge tool throw IllegalArgumentException.
+    [HBASE-5875] - Process RIT and Master restart may remove an online server considering it as a dead server
+    [HBASE-5876] - TestImportExport has been failing against hadoop 0.23 profile
+    [HBASE-5894] - Table deletion failed but HBaseAdmin#deletetable reports it as success
+    [HBASE-5902] - Some scripts are not executable
+    [HBASE-5909] - SlabStats should be a daemon thread
+    [HBASE-5916] - RS restart just before master intialization we make the cluster non operative
+    [HBASE-5918] - Master will block forever at startup if root server dies between assigning root and assigning meta
+    [HBASE-5922] - HalfStoreFileReader seekBefore causes StackOverflowError
+    [HBASE-5927] - SSH and DisableTableHandler happening together does not clear the znode of the region and RIT map.
+    [HBASE-5928] - Hbck shouldn't npe when there are no tables.
+    [HBASE-5955] - Guava 11 drops MapEvictionListener and Hadoop 2.0.0-alpha requires it
+    [HBASE-5963] - ClassCastException: FileSystem$Cache$ClientFinalizer cannot be cast to Thread
+    [HBASE-5964] - HFileSystem: "No FileSystem for scheme: hdfs"
+    [HBASE-5975] - Failed suppression of fs shutdown hook with Hadoop 2.0.0
+    [HBASE-5986] - Clients can see holes in the META table when regions are being split
+    [HBASE-6002] - Possible chance of resource leak in HlogSplitter
+    [HBASE-6011] - Unable to start master in local mode
+    [HBASE-6016] - ServerShutdownHandler#processDeadRegion could return false for disabling table regions
+    [HBASE-6018] - hbck fails with a RejectedExecutionException when >50 regions present
+    [HBASE-6021] - NullPointerException when running LoadTestTool without specifying compression type
+    [HBASE-6029] - HBCK doesn't recover Balance switch if exception occurs in onlineHbck()
+    [HBASE-6046] - Master retry on ZK session expiry causes inconsistent region assignments.
+    [HBASE-6047] - Put.has() can't determine result correctly
+    [HBASE-6049] - Serializing "List" containing null elements will cause NullPointerException in HbaseObjectWritable.writeObject()
+    [HBASE-6050] - HLogSplitter renaming recovered.edits and CJ removing the parent directory race, making the HBCK think cluster is inconsistent.
+    [HBASE-6056] - Restore hbase-default version check
+    [HBASE-6065] - Log for flush would append a non-sequential edit in the hlog, leading to possible data loss
+    [HBASE-6068] - Secure HBase cluster : Client not able to call some admin APIs
+    [HBASE-6069] - TableInputFormatBase#createRecordReader() doesn't initialize TableRecordReader which causes NPE
+    [HBASE-6070] - AM.nodeDeleted and SSH races creating problems for regions under SPLIT
+    [HBASE-6088] - Region splitting not happened for long time due to ZK exception while creating RS_ZK_SPLITTING node
+    [HBASE-6089] - SSH and AM.joinCluster causes Concurrent Modification exception.
+    [HBASE-6095] - ActiveMasterManager NullPointerException
+    [HBASE-6115] - NullPointerException is thrown when root and meta table regions are assigning to another RS.
+    [HBASE-6122] - Backup master does not become Active master after ZK exception
+    [HBASE-6126] - Fix broke TestLocalHBaseCluster in 0.92/0.94
+    [HBASE-6133] - TestRestartCluster failing in 0.92
+    [HBASE-6141] - InterfaceAudience breaks 0.94 on older versions of hadoop
+    [HBASE-6146] - Disabling of Catalog tables should not be allowed
+    [HBASE-6158] - Data loss if the words 'merges' or 'splits' are used as Column Family name
+    [HBASE-6160] - META entries from daughters can be deleted before parent entries
+    [HBASE-6164] - Correct the bug in block encoding usage in bulkload
+    [HBASE-6185] - Update javadoc for ConstantSizeRegionSplitPolicy class
+    [HBASE-6195] - Increment data will be lost when the memstore is flushed
+    [HBASE-6200] - KeyComparator.compareWithoutRow can be wrong when families have the same prefix
+    [HBASE-6210] - Backport HBASE-6197 to 0.94
+    [HBASE-6227] - SSH and cluster startup causes data loss
+    [HBASE-6229] - AM.assign() should not set table state to ENABLED directly.
+    [HBASE-6236] - Offline meta repair fails if the HBase base mount point is on a different cluster/volume than its parent in a ViewFS or similar FS
+    [HBASE-6237] - Fix race on ACL table creation in TestTablePermissions
+    [HBASE-6240] - Race in HCM.getMaster stalls clients
+    [HBASE-6246] - Admin.move without specifying destination does not go through AccessController
+    [HBASE-6248] - Jetty init may fail if directory name contains "master"
+    [HBASE-6265] - Calling getTimestamp() on a KV in cp.prePut() causes KV not to be flushed
+    [HBASE-6269] - Lazyseek should use the maxSequenseId StoreFile's KeyValue as the latest KeyValue
+    [HBASE-6281] - Assignment need not be called for disabling table regions during clean cluster start up.
+    [HBASE-6284] - Introduce HRegion#doMiniBatchMutation()
+    [HBASE-6293] - HMaster does not go down while splitting logs even if explicit shutdown is called.
+    [HBASE-6303] - HCD.setCompressionType should use Enum support for storing compression types as strings
+    [HBASE-6311] - Data error after majorCompaction caused by keeping MVCC for opened scanners
+    [HBASE-6313] - Client hangs because the client is not notified
+    [HBASE-6326] - Avoid nested retry loops in HConnectionManager
+    [HBASE-6328] - FSHDFSUtils#recoverFileLease tries to rethrow InterruptedException but actually shallows it
+    [HBASE-6329] - Stopping META regionserver when splitting region could cause daughter region to be assigned twice
+    [HBASE-6337] - [MTTR] Remove renaming tmp log file in SplitLogManager
+    [HBASE-6369] - HTable is not closed in AggregationClient
+
+Improvement
+
+    [HBASE-4720] - Implement atomic update operations (checkAndPut, checkAndDelete) for REST client/server
+    [HBASE-5360] - [uberhbck] Add options for how to handle offline split parents.
+    [HBASE-5630] - hbck should disable the balancer using synchronousBalanceSwitch.
+    [HBASE-5802] - Change the default metrics class to NullContextWithUpdateThread
+    [HBASE-5838] - Add an LZ4 compression option to HFile
+    [HBASE-5887] - Make TestAcidGuarantees usable for system testing.
+    [HBASE-5892] - [hbck] Refactor parallel WorkItem* to Futures.
+    [HBASE-5913] - Speed up the full scan of META
+    [HBASE-5973] - Add ability for potentially long-running IPC calls to abort if client disconnects
+    [HBASE-6010] - Security audit logger configuration for log4j
+    [HBASE-6013] - Polish sharp edges from CopyTable
+    [HBASE-6022] - Include Junit in the libs when packaging so that TestAcidGaurntee can run
+    [HBASE-6023] - Normalize security audit logging level with Hadoop
+    [HBASE-6040] - Use block encoding and HBase handled checksum verification in bulk loading using HFileOutputFormat
+    [HBASE-6067] - HBase won't start when hbase.rootdir uses ViewFileSystem
+    [HBASE-6114] - CacheControl flags should be tunable per table schema per CF
+    [HBASE-6124] - Backport HBASE-6033 to 0.90, 0.92 and 0.94
+    [HBASE-6161] - Log Error when thrift server fails to start up.
+    [HBASE-6173] - hbck check specified tables only
+    [HBASE-6207] - Add jitter to client retry timer
+    [HBASE-6214] - Backport HBASE-5998 to 94.1
+    [HBASE-6244] - [REST] Result generators do not need to query table schema
+    [HBASE-6247] - [REST] HTablePool.putTable is deprecated
+    [HBASE-6267] - hbase.store.delete.expired.storefile should be true by default
+    [HBASE-6283] - [region_mover.rb] Add option to exclude list of hosts on unload instead of just assuming the source node.
+    [HBASE-6314] - Fast fail behavior for unauthenticated user
+    [HBASE-6332] - Improve POM for better integration with downstream ivy projects
+    [HBASE-6334] - TestImprovement for TestHRegion.testWritesWhileGetting
+    [HBASE-6341] - Publicly expose HConnectionKey
+
+New Feature
+
+    [HBASE-2214] - Do HBASE-1996 -- setting size to return in scan rather than count of rows -- properly
+    [HBASE-2730] - Expose RS work queue contents on web UI
+    [HBASE-5609] - Add the ability to pass additional information for slow query logging
+    [HBASE-5886] - Add new metric for possible data loss due to puts without WAL
+    [HBASE-6044] - copytable: remove rs.* parameters
+
+Task
+
+    [HBASE-6001] - Upgrade slf4j to 1.6.1
+    [HBASE-6034] - Upgrade Hadoop dependencies
+    [HBASE-6077] - Document the most common secure RPC troubleshooting resolutions
+    [HBASE-6129] - Backport of Add Increment Coalescing in thrift.
+    [HBASE-6131] - Add attribution for code added by HBASE-5533 metrics
+
+
 Release 0.94.0 - 5/1/2012
 Sub-task
 

From 31d4637c919e026d46e8dc5ca4554b3611af98e8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 13 Jul 2012 00:38:45 +0000
Subject: [PATCH 0342/1540] HBASE-6375 Master may be using a stale list of
 region servers for creating assignment plan during startup (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361018 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/AssignmentManager.java | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 0b5a65745bd7..287ec8d9b299 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2247,15 +2247,6 @@ private void setEnabledTable(HRegionInfo hri) {
    * @throws IOException
    */
   public void assignAllUserRegions() throws IOException, InterruptedException {
-    // Get all available servers
-    List<ServerName> servers = serverManager.getOnlineServersList();
-
-    // Remove the deadNotExpired servers from the server list.
-    removeDeadNotExpiredServers(servers);
-
-    // If there are no servers we need not proceed with region assignment.
-    if(servers.isEmpty()) return;
-
     // Skip assignment for regions of tables in DISABLING state also because
     // during clean cluster startup no RS is alive and regions map also doesn't
     // have any information about the regions. See HBASE-6281.
@@ -2266,6 +2257,15 @@ public void assignAllUserRegions() throws IOException, InterruptedException {
         disablingAndDisabledTables, true);
     if (allRegions == null || allRegions.isEmpty()) return;
 
+    // Get all available servers
+    List<ServerName> servers = serverManager.getOnlineServersList();
+
+    // Remove the deadNotExpired servers from the server list.
+    removeDeadNotExpiredServers(servers);
+
+    // If there are no servers we need not proceed with region assignment.
+    if(servers.isEmpty()) return;
+
     // Determine what type of assignment to do on startup
     boolean retainAssignment = master.getConfiguration().
       getBoolean("hbase.master.startup.retainassign", true);

From c3fcaa3560fcfd6bb1380bfd00bde1a698a5ad32 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 13 Jul 2012 01:04:23 +0000
Subject: [PATCH 0343/1540] HBASE-6377. HBASE-5533 metrics miss all operations
 submitted via MultiAction

Applied 6377-0.94-remove-get-put-delete-histograms.patch


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361028 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     | 20 --------------
 .../metrics/RegionServerMetrics.java          | 26 -------------------
 2 files changed, 46 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b1e723303ad4..230d6b1ea4cf 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1952,15 +1952,12 @@ public Result getClosestRowBefore(final byte[] regionName, final byte[] row,
   /** {@inheritDoc} */
   public Result get(byte[] regionName, Get get) throws IOException {
     checkOpen();
-    final long startTime = System.nanoTime();
     requestCount.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       return region.get(get, getLockFromId(get.getLockId()));
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
-    } finally {
-      this.metrics.getLatencies.update(System.nanoTime() - startTime);
     }
   }
 
@@ -1992,7 +1989,6 @@ public void put(final byte[] regionName, final Put put) throws IOException {
       throw new IllegalArgumentException("update has null row");
     }
 
-    final long startTime = System.nanoTime();
     checkOpen();
     this.requestCount.incrementAndGet();
     HRegion region = getRegion(regionName);
@@ -2004,8 +2000,6 @@ public void put(final byte[] regionName, final Put put) throws IOException {
       region.put(put, getLockFromId(put.getLockId()), writeToWAL);
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
-    } finally {
-      this.metrics.putLatencies.update(System.nanoTime() - startTime);
     }
   }
 
@@ -2015,7 +2009,6 @@ public int put(final byte[] regionName, final List<Put> puts)
     HRegion region = null;
     int i = 0;
 
-    final long startTime = System.nanoTime();
     try {
       region = getRegion(regionName);
       if (!region.getRegionInfo().isMetaTable()) {
@@ -2040,14 +2033,6 @@ public int put(final byte[] regionName, final List<Put> puts)
       return -1;
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
-    } finally {
-      // going to count this as puts.size() PUTs for latency calculations
-      final long totalTime = System.nanoTime() - startTime;
-      final long putCount = i;
-      final long perPutTime = totalTime / putCount;
-      for (int request = 0; request < putCount; request++) {
-        this.metrics.putLatencies.update(perPutTime);
-      }
     }
   }
 
@@ -2520,7 +2505,6 @@ public void leaseExpired() {
   public void delete(final byte[] regionName, final Delete delete)
       throws IOException {
     checkOpen();
-    final long startTime = System.nanoTime();
     try {
       boolean writeToWAL = delete.getWriteToWAL();
       this.requestCount.incrementAndGet();
@@ -2532,8 +2516,6 @@ public void delete(final byte[] regionName, final Delete delete)
       region.delete(delete, lid, writeToWAL);
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
-    } finally {
-      this.metrics.deleteLatencies.update(System.nanoTime() - startTime);
     }
   }
 
@@ -2551,12 +2533,10 @@ public int delete(final byte[] regionName, final List<Delete> deletes)
       int size = deletes.size();
       Integer[] locks = new Integer[size];
       for (Delete delete : deletes) {
-        final long startTime = System.nanoTime();
         this.requestCount.incrementAndGet();
         locks[i] = getLockFromId(delete.getLockId());
         region.delete(delete, locks[i], delete.getWriteToWAL());
         i++;
-        this.metrics.deleteLatencies.update(System.nanoTime() - startTime);
       }
     } catch (WrongRegionException ex) {
       LOG.debug("Batch deletes: " + i, ex);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index b00e4b8c44e6..9ce40b195326 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -130,25 +130,6 @@ public class RegionServerMetrics implements Updater {
   /** Block hit caching ratio for past N periods */
   public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
 
-  /**
-   * a latency histogram on 'get' requests
-   */
-  public final MetricsHistogram getLatencies = 
-      new MetricsHistogram("getRequestLatency", registry);
- 
-  /**
-   * a latency histogram on 'delete' requests
-   */
-  public final MetricsHistogram deleteLatencies = 
-      new MetricsHistogram("deleteRequestLatency", registry);
- 
-  /**
-   * a latency histogram on 'put' requests
-   */
-  public final MetricsHistogram putLatencies = 
-      new MetricsHistogram("putRequestLatency", registry);
- 
-  
   /*
    * Count of requests to the regionservers since last call to metrics update
    */
@@ -395,10 +376,6 @@ public void doUpdates(MetricsContext caller) {
       this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
       this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
 
-      this.putLatencies.pushMetric(this.metricsRecord);
-      this.deleteLatencies.pushMetric(this.metricsRecord);
-      this.getLatencies.pushMetric(this.metricsRecord);
-      
       // Mix in HFile and HLog metrics
       // Be careful. Here is code for MTVR from up in hadoop:
       // public synchronized void inc(final int numOps, final long time) {
@@ -586,9 +563,6 @@ public String toString() {
         Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
     sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
         Long.valueOf(this.slowHLogAppendCount.get()));
-    sb = appendHistogram(sb, this.deleteLatencies);
-    sb = appendHistogram(sb, this.getLatencies);
-    sb = appendHistogram(sb, this.putLatencies);
     sb = appendHistogram(sb, this.fsReadLatencyHistogram);
     sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
     sb = appendHistogram(sb, this.fsWriteLatencyHistogram);

From 43e9d64f2444c531cbc42f1d8589de00159c08c5 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 13 Jul 2012 03:18:29 +0000
Subject: [PATCH 0344/1540] HBASE-6384 hbck should group together those
 sidelined regions need to be bulk loaded later

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361036 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 98fca20a12a6..0aa1f36c04de 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -144,6 +144,7 @@ public class HBaseFsck {
   private static boolean rsSupportsOffline = true;
   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
+  private static final String TO_BE_LOADED = "to_be_loaded";
 
   /**********************
    * Internal resources
@@ -873,8 +874,20 @@ private Path getSidelineDir() throws IOException {
   /**
    * Sideline a region dir (instead of deleting it)
    */
-  Path sidelineRegionDir(FileSystem fs, HbckInfo hi)
-    throws IOException {
+  Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
+    return sidelineRegionDir(fs, null, hi);
+  }
+
+  /**
+   * Sideline a region dir (instead of deleting it)
+   *
+   * @param parentDir if specified, the region will be sidelined to
+   * folder like .../parentDir/<table name>/<region name>. The purpose
+   * is to group together similar regions sidelined, for example, those
+   * regions should be bulk loaded back later on. If null, it is ignored.
+   */
+  Path sidelineRegionDir(FileSystem fs,
+      String parentDir, HbckInfo hi) throws IOException {
     String tableName = Bytes.toString(hi.getTableName());
     Path regionDir = hi.getHdfsRegionDir();
 
@@ -883,7 +896,11 @@ Path sidelineRegionDir(FileSystem fs, HbckInfo hi)
       return null;
     }
 
-    Path sidelineTableDir= new Path(getSidelineDir(), tableName);
+    Path rootDir = getSidelineDir();
+    if (parentDir != null) {
+      rootDir = new Path(rootDir, parentDir);
+    }
+    Path sidelineTableDir= new Path(rootDir, tableName);
     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
     fs.mkdirs(sidelineRegionDir);
     boolean success = false;
@@ -1957,7 +1974,7 @@ void sidelineBigOverlaps(
           offline(regionToSideline.getRegionName());
 
           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
-          Path sidelineRegionDir = sidelineRegionDir(fs, regionToSideline);
+          Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
           if (sidelineRegionDir != null) {
             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
             LOG.info("After sidelined big overlapped region: "

From 89e6a42b924553300d27e1b97f8d5eadc23e51e9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 13 Jul 2012 13:27:44 +0000
Subject: [PATCH 0345/1540] HBASE-6380 bulkload should update the
 store.storeSize

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361204 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/Store.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index f7491fd28a8e..68219b90f653 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -554,8 +554,10 @@ void bulkLoadHFile(String srcPathStr) throws IOException {
         this.family.getBloomFilterType(), this.dataBlockEncoder);
     passSchemaMetricsTo(sf);
 
-    sf.createReader();
-
+    StoreFile.Reader r = sf.createReader();
+    this.storeSize += r.length();
+    this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+    
     LOG.info("Moved hfile " + srcPath + " into store directory " +
         homedir + " - updating store file list.");
 

From e0d9503dd7fa652e3208f105c850f3d01440eead Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 14 Jul 2012 00:47:14 +0000
Subject: [PATCH 0346/1540] HBASE-6389 Modify the conditions to ensure that
 Master waits for sufficient number of Region Servers before starting region
 assignments (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361458 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/ServerManager.java    | 22 ++++++++++++-------
 .../hadoop/hbase/HBaseTestingUtility.java     |  5 +++--
 .../TestRSKilledWhenMasterInitializing.java   |  7 ++++--
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 0ed32006ce14..28e868287366 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -557,12 +557,12 @@ private HRegionInterface getServerConnection(final ServerName sn)
    * Wait for the region servers to report in.
    * We will wait until one of this condition is met:
    *  - the master is stopped
-   *  - the 'hbase.master.wait.on.regionservers.timeout' is reached
    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
    *    region servers is reached
    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
    *   there have been no new region server in for
-   *      'hbase.master.wait.on.regionservers.interval' time
+   *      'hbase.master.wait.on.regionservers.interval' time AND
+   *   the 'hbase.master.wait.on.regionservers.timeout' is reached
    *
    * @throws InterruptedException
    */
@@ -571,11 +571,18 @@ public void waitForRegionServers(MonitoredTask status)
     final long interval = this.master.getConfiguration().
       getLong("hbase.master.wait.on.regionservers.interval", 1500);
     final long timeout = this.master.getConfiguration().
-    getLong("hbase.master.wait.on.regionservers.timeout", 4500);
+      getLong("hbase.master.wait.on.regionservers.timeout", 4500);
     final int minToStart = this.master.getConfiguration().
-    getInt("hbase.master.wait.on.regionservers.mintostart", 1);
-    final int maxToStart = this.master.getConfiguration().
-    getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
+      getInt("hbase.master.wait.on.regionservers.mintostart", 1);
+    int maxToStart = this.master.getConfiguration().
+      getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
+    if (maxToStart < minToStart) {
+        LOG.warn(String.format(
+            "The value of 'hbase.master.wait.on.regionservers.maxtostart' (%d)" +
+            " is set less than 'hbase.master.wait.on.regionservers.mintostart'" +
+            " (%d), ignoring.", maxToStart, minToStart));
+        maxToStart = Integer.MAX_VALUE;
+    }
 
     long now =  System.currentTimeMillis();
     final long startTime = now;
@@ -586,9 +593,8 @@ public void waitForRegionServers(MonitoredTask status)
     int oldCount = 0;
     while (
       !this.master.isStopped() &&
-        slept < timeout &&
         count < maxToStart &&
-        (lastCountChange+interval > now || count < minToStart)
+        (lastCountChange+interval > now || timeout > slept || count < minToStart)
       ){
 
       // Log some info at every interval time or if there is a change
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 19fca5803f59..4b8eafe2c370 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -624,8 +624,9 @@ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
 
     // These settings will make the server waits until this exact number of
     //  regions servers are connected.
-    conf.setInt("hbase.master.wait.on.regionservers.mintostart", numSlaves);
-    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", numSlaves);
+    String count = String.valueOf(numSlaves);
+    conf.setIfUnset("hbase.master.wait.on.regionservers.mintostart", count);
+    conf.setIfUnset("hbase.master.wait.on.regionservers.maxtostart", count);
 
     Configuration c = new Configuration(this.conf);
     this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index a0ed0bd83a75..431420c96156 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -66,8 +66,11 @@ public class TestRSKilledWhenMasterInitializing {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     // Set it so that this test runs with my custom master
-    TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL,
-        TestingMaster.class, HMaster.class);
+    Configuration conf = TESTUTIL.getConfiguration();      
+    conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
+    conf.setInt("hbase.master.wait.on.regionservers.mintostart", 3);
+    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 4);
+    
     // Start up the cluster.
     TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
   }

From f5a3da4d8330a15962729bb87c72b430c4d93bde Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 14 Jul 2012 00:52:49 +0000
Subject: [PATCH 0347/1540] CHANGES.txt/pom.xml for 0.94.1RC0 take two

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361460 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 8 +++++++-
 pom.xml     | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index ffd18fc2f7c1..24214f7341e9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.1 - 7/12/2012
+Release 0.94.1 - 7/13/2012
 Sub-task
 
     [HBASE-5342] - Grant/Revoke global permissions
@@ -98,7 +98,12 @@ Bug
     [HBASE-6328] - FSHDFSUtils#recoverFileLease tries to rethrow InterruptedException but actually shallows it
     [HBASE-6329] - Stopping META regionserver when splitting region could cause daughter region to be assigned twice
     [HBASE-6337] - [MTTR] Remove renaming tmp log file in SplitLogManager
+    [HBASE-6357] - Failed distributed log splitting stuck on master web UI
     [HBASE-6369] - HTable is not closed in AggregationClient
+    [HBASE-6375] - Master may be using a stale list of region servers for creating assignment plan during startup
+    [HBASE-6377] - HBASE-5533 metrics miss all operations submitted via MultiAction
+    [HBASE-6380] - bulkload should update the store.storeSize
+    [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments
 
 Improvement
 
@@ -131,6 +136,7 @@ Improvement
     [HBASE-6332] - Improve POM for better integration with downstream ivy projects
     [HBASE-6334] - TestImprovement for TestHRegion.testWritesWhileGetting
     [HBASE-6341] - Publicly expose HConnectionKey
+    [HBASE-6384] - hbck should group together those sidelined regions need to be bulk loaded later
 
 New Feature
 
diff --git a/pom.xml b/pom.xml
index 9fe2b3c60df3..f6f0c5a262bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.1-SNAPSHOT</version>
+  <version>0.94.1</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 2e1ae3fa39b30ec4f7db0414e3ee1de12df10be6 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 14 Jul 2012 02:24:43 +0000
Subject: [PATCH 0348/1540] HBASE-6394 verifyrep MR job map tasks throws
 NullPointerException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361470 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/replication/VerifyReplication.java      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index c37a4fbe607d..b7d540b0c4bb 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -35,7 +35,6 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
@@ -137,7 +136,10 @@ public Void connect(HConnection conn) throws IOException {
     }
 
     protected void cleanup(Context context) {
-      replicatedScanner.close();
+      if (replicatedScanner != null) {
+        replicatedScanner.close();
+        replicatedScanner = null;
+      }
     }
   }
 

From d25e1a993f734757875c0637566b569b2cceec39 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 14 Jul 2012 03:30:55 +0000
Subject: [PATCH 0349/1540] updated CHANGES.txt because of another checkin

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1361472 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 24214f7341e9..caeed1427fe0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -104,6 +104,7 @@ Bug
     [HBASE-6377] - HBASE-5533 metrics miss all operations submitted via MultiAction
     [HBASE-6380] - bulkload should update the store.storeSize
     [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments
+    [HBASE-6394] - verifyrep MR job map tasks throws NullPointerException
 
 Improvement
 

From d655f019bd546899c5a5176e9a1ecbe6333fca91 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 16 Jul 2012 20:51:51 +0000
Subject: [PATCH 0350/1540] HBASE-6397 [hbck] print out bulk load commands for
 sidelined regions if necessary

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1362247 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/HBaseFsck.java  | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 0aa1f36c04de..707c9471b404 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2134,9 +2134,14 @@ public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
   }
 
   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
-    for (Path k : regions.keySet()) {
-      System.out.println("To be bulk loaded sidelined region dir: "
-        + k.toString());
+    for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
+      String tableName = Bytes.toStringBinary(entry.getValue().getTableName());
+      Path path = entry.getKey();
+      System.out.println("This sidelined region dir should be bulk loaded: "
+        + path.toString());
+      System.out.println("Bulk load command looks like: "
+        + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
+        + path.toUri().getPath() + " "+ tableName);
     }
   }
 

From 4f2d7a233b6254a2815afc0b8e204ad79c0dffc0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 16 Jul 2012 23:10:53 +0000
Subject: [PATCH 0351/1540] Latest CHANGES.txt for 0.94.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1362307 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index caeed1427fe0..e4b07b1b5000 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.1 - 7/13/2012
+Release 0.94.1 - 7/16/2012
 Sub-task
 
     [HBASE-5342] - Grant/Revoke global permissions
@@ -105,6 +105,7 @@ Bug
     [HBASE-6380] - bulkload should update the store.storeSize
     [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments
     [HBASE-6394] - verifyrep MR job map tasks throws NullPointerException
+    [HBASE-6397] - [hbck] print out bulk load commands for sidelined regions if necessary
 
 Improvement
 

From 3b811687d45d712332a784ebcfcdd9798aa5f897 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 16 Jul 2012 23:14:08 +0000
Subject: [PATCH 0352/1540] HBASE-6382 Upgrade Jersey to 1.8 to match Hadoop 1
 and 2 (David S. Wang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1362308 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f6f0c5a262bf..ac0d75c15a92 100644
--- a/pom.xml
+++ b/pom.xml
@@ -984,7 +984,7 @@
     <jaxb-api.version>2.1</jaxb-api.version>
     <jetty.version>6.1.26</jetty.version>
     <jetty.jspapi.version>6.1.14</jetty.jspapi.version>
-    <jersey.version>1.4</jersey.version>
+    <jersey.version>1.8</jersey.version>
     <jruby.version>1.6.5</jruby.version>
     <junit.version>4.10-HBASE-1</junit.version>
     <slf4j.version>1.4.3</slf4j.version>

From d78c9ba823addaf7533bff3c303dd4cd273b626c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 16 Jul 2012 23:16:23 +0000
Subject: [PATCH 0353/1540] Latest CHANGES.txt for 0.94.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1362309 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index e4b07b1b5000..4bf00b2d4f90 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -138,6 +138,7 @@ Improvement
     [HBASE-6332] - Improve POM for better integration with downstream ivy projects
     [HBASE-6334] - TestImprovement for TestHRegion.testWritesWhileGetting
     [HBASE-6341] - Publicly expose HConnectionKey
+    [HBASE-6382] - Upgrade Jersey to 1.8 to match Hadoop 1 and 2
     [HBASE-6384] - hbck should group together those sidelined regions need to be bulk loaded later
 
 New Feature

From 983d08709e7182924b574f0c1bbef0ed132f3e59 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 19 Jul 2012 00:52:32 +0000
Subject: [PATCH 0354/1540] HBASE-6389 revert

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363193 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/ServerManager.java    | 22 +++++++------------
 .../hadoop/hbase/HBaseTestingUtility.java     |  5 ++---
 .../TestRSKilledWhenMasterInitializing.java   |  7 ++----
 3 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 28e868287366..0ed32006ce14 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -557,12 +557,12 @@ private HRegionInterface getServerConnection(final ServerName sn)
    * Wait for the region servers to report in.
    * We will wait until one of this condition is met:
    *  - the master is stopped
+   *  - the 'hbase.master.wait.on.regionservers.timeout' is reached
    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
    *    region servers is reached
    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
    *   there have been no new region server in for
-   *      'hbase.master.wait.on.regionservers.interval' time AND
-   *   the 'hbase.master.wait.on.regionservers.timeout' is reached
+   *      'hbase.master.wait.on.regionservers.interval' time
    *
    * @throws InterruptedException
    */
@@ -571,18 +571,11 @@ public void waitForRegionServers(MonitoredTask status)
     final long interval = this.master.getConfiguration().
       getLong("hbase.master.wait.on.regionservers.interval", 1500);
     final long timeout = this.master.getConfiguration().
-      getLong("hbase.master.wait.on.regionservers.timeout", 4500);
+    getLong("hbase.master.wait.on.regionservers.timeout", 4500);
     final int minToStart = this.master.getConfiguration().
-      getInt("hbase.master.wait.on.regionservers.mintostart", 1);
-    int maxToStart = this.master.getConfiguration().
-      getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
-    if (maxToStart < minToStart) {
-        LOG.warn(String.format(
-            "The value of 'hbase.master.wait.on.regionservers.maxtostart' (%d)" +
-            " is set less than 'hbase.master.wait.on.regionservers.mintostart'" +
-            " (%d), ignoring.", maxToStart, minToStart));
-        maxToStart = Integer.MAX_VALUE;
-    }
+    getInt("hbase.master.wait.on.regionservers.mintostart", 1);
+    final int maxToStart = this.master.getConfiguration().
+    getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
 
     long now =  System.currentTimeMillis();
     final long startTime = now;
@@ -593,8 +586,9 @@ public void waitForRegionServers(MonitoredTask status)
     int oldCount = 0;
     while (
       !this.master.isStopped() &&
+        slept < timeout &&
         count < maxToStart &&
-        (lastCountChange+interval > now || timeout > slept || count < minToStart)
+        (lastCountChange+interval > now || count < minToStart)
       ){
 
       // Log some info at every interval time or if there is a change
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 4b8eafe2c370..19fca5803f59 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -624,9 +624,8 @@ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
 
     // These settings will make the server waits until this exact number of
     //  regions servers are connected.
-    String count = String.valueOf(numSlaves);
-    conf.setIfUnset("hbase.master.wait.on.regionservers.mintostart", count);
-    conf.setIfUnset("hbase.master.wait.on.regionservers.maxtostart", count);
+    conf.setInt("hbase.master.wait.on.regionservers.mintostart", numSlaves);
+    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", numSlaves);
 
     Configuration c = new Configuration(this.conf);
     this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index 431420c96156..a0ed0bd83a75 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -66,11 +66,8 @@ public class TestRSKilledWhenMasterInitializing {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     // Set it so that this test runs with my custom master
-    Configuration conf = TESTUTIL.getConfiguration();      
-    conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
-    conf.setInt("hbase.master.wait.on.regionservers.mintostart", 3);
-    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 4);
-    
+    TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL,
+        TestingMaster.class, HMaster.class);
     // Start up the cluster.
     TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
   }

From 6d1e06c51b64e954d9ed54d37025bde1be4a6532 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 19 Jul 2012 01:46:01 +0000
Subject: [PATCH 0355/1540] HBASE-6392 UnknownRegionException blocks hbck from
 sideline big overlap regions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363202 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  34 +++---
 .../hadoop/hbase/util/TestHBaseFsck.java      | 101 +++++++++++++++++-
 2 files changed, 119 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 707c9471b404..41337fd89ac7 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1287,6 +1287,7 @@ private void undeployRegions(HbckInfo hi) throws IOException, InterruptedExcepti
    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
    * restart or failover may be required.
    */
+  @SuppressWarnings("deprecation")
   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
     if (hi.metaEntry == null && hi.hdfsEntry == null) {
       undeployRegions(hi);
@@ -1900,22 +1901,22 @@ void mergeOverlaps(Collection<HbckInfo> overlap)
           LOG.debug("Contained region dir before close");
           debugLsr(hi.getHdfsRegionDir());
           try {
+            LOG.info("Closing region: " + hi);
             closeRegion(hi);
           } catch (IOException ioe) {
-            // TODO exercise this
-            LOG.warn("Was unable to close region " + hi.getRegionNameAsString()
-                + ".  Just continuing... ");
+            LOG.warn("Was unable to close region " + hi
+              + ".  Just continuing... ", ioe);
           } catch (InterruptedException e) {
-            // TODO exercise this
-            LOG.warn("Was unable to close region " + hi.getRegionNameAsString()
-                + ".  Just continuing... ");
+            LOG.warn("Was unable to close region " + hi
+              + ".  Just continuing... ", e);
           }
 
           try {
             LOG.info("Offlining region: " + hi);
             offline(hi.getRegionName());
           } catch (IOException ioe) {
-            LOG.warn("Unable to offline region from master: " + hi, ioe);
+            LOG.warn("Unable to offline region from master: " + hi
+              + ".  Just continuing... ", ioe);
           }
         }
 
@@ -1964,14 +1965,21 @@ void sidelineBigOverlaps(
           try {
             LOG.info("Closing region: " + regionToSideline);
             closeRegion(regionToSideline);
-          } catch (InterruptedException ie) {
-            LOG.warn("Was unable to close region " + regionToSideline.getRegionNameAsString()
-              + ".  Interrupted.");
-            throw new IOException(ie);
+          } catch (IOException ioe) {
+            LOG.warn("Was unable to close region " + regionToSideline
+              + ".  Just continuing... ", ioe);
+          } catch (InterruptedException e) {
+            LOG.warn("Was unable to close region " + regionToSideline
+              + ".  Just continuing... ", e);
           }
 
-          LOG.info("Offlining region: " + regionToSideline);
-          offline(regionToSideline.getRegionName());
+          try {
+            LOG.info("Offlining region: " + regionToSideline);
+            offline(regionToSideline.getRegionName());
+          } catch (IOException ioe) {
+            LOG.warn("Unable to offline region from master: " + regionToSideline
+              + ".  Just continuing... ", ioe);
+          }
 
           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 91ea273d6c8b..46de3efa23f6 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -24,6 +24,7 @@
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -62,9 +63,11 @@
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -73,6 +76,8 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.collect.Multimap;
+
 /**
  * This tests HBaseFsck's ability to detect reasons for inconsistent tables.
  */
@@ -482,8 +487,7 @@ public void testDupeRegion() throws Exception {
       // differentiate on ts/regionId!  We actually need to recheck
       // deployments!
       HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
-      ServerName hsi;
-      while ( (hsi = findDeployedHSI(getDeployedHRIs(admin), hriDupe)) == null) {
+      while (findDeployedHSI(getDeployedHRIs(admin), hriDupe) == null) {
         Thread.sleep(250);
       }
 
@@ -546,7 +550,6 @@ public void testDegenerateRegions() throws Exception {
     }
   }
 
-
   /**
    * This creates and fixes a bad table where a region is completely contained
    * by another region.
@@ -584,6 +587,98 @@ public void testContainedRegionOverlap() throws Exception {
     }
   }
 
+  /**
+   * This creates and fixes a bad table where an overlap group of
+   * 3 regions. Set HBaseFsck.maxMerge to 2 to trigger sideline overlapped
+   * region. Mess around the meta data so that closeRegion/offlineRegion
+   * throws exceptions.
+   */
+  @Test
+  public void testSidelineOverlapRegion() throws Exception {
+    String table = "testSidelineOverlapRegion";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by creating an overlap
+      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+      HMaster master = cluster.getMaster();
+      HRegionInfo hriOverlap1 = createRegion(conf, tbl.getTableDescriptor(),
+        Bytes.toBytes("A"), Bytes.toBytes("AB"));
+      master.assignRegion(hriOverlap1);
+      master.getAssignmentManager().waitForAssignment(hriOverlap1);
+      HRegionInfo hriOverlap2 = createRegion(conf, tbl.getTableDescriptor(),
+        Bytes.toBytes("AB"), Bytes.toBytes("B"));
+      master.assignRegion(hriOverlap2);
+      master.getAssignmentManager().waitForAssignment(hriOverlap2);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
+        ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
+      assertEquals(3, hbck.getOverlapGroups(table).size());
+      assertEquals(ROWKEYS.length, countRows());
+
+      // mess around the overlapped regions, to trigger NotServingRegionException
+      Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
+      ServerName serverName = null;
+      byte[] regionName = null;
+      for (HbckInfo hbi: overlapGroups.values()) {
+        if ("A".equals(Bytes.toString(hbi.getStartKey()))
+            && "B".equals(Bytes.toString(hbi.getEndKey()))) {
+          regionName = hbi.getRegionName();
+
+          // get an RS not serving the region to force bad assignment info in to META.
+          int k = cluster.getServerWith(regionName);
+          for (int i = 0; i < 3; i++) {
+            if (i != k) {
+              HRegionServer rs = cluster.getRegionServer(i);
+              serverName = rs.getServerName();
+              break;
+            }
+          }
+
+          HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+          HBaseFsckRepair.closeRegionSilentlyAndWait(admin,
+            cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
+          admin.unassign(regionName, true);
+          break;
+        }
+      }
+
+      assertNotNull(regionName);
+      assertNotNull(serverName);
+      HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
+      Put put = new Put(regionName);
+      put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+        Bytes.toBytes(serverName.getHostAndPort()));
+      meta.put(put);
+
+      // fix the problem.
+      HBaseFsck fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setFixAssignments(true);
+      fsck.setFixMeta(true);
+      fsck.setFixHdfsHoles(true);
+      fsck.setFixHdfsOverlaps(true);
+      fsck.setFixHdfsOrphans(true);
+      fsck.setFixVersionFile(true);
+      fsck.setSidelineBigOverlaps(true);
+      fsck.setMaxMerge(2);
+      fsck.onlineHbck();
+
+      // verify that overlaps are fixed, and there are less rows
+      // since one region is sidelined.
+      HBaseFsck hbck2 = doFsck(conf,false);
+      assertNoErrors(hbck2);
+      assertEquals(0, hbck2.getOverlapGroups(table).size());
+      assertTrue(ROWKEYS.length > countRows());
+    } finally {
+       deleteTable(table);
+    }
+  }
+
   /**
    * This creates and fixes a bad table where a region is completely contained
    * by another region, and there is a hole (sort of like a bad split)

From 956e3258006a39f2d0304af73a4f995f556f89b3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 19 Jul 2012 02:45:17 +0000
Subject: [PATCH 0356/1540] HBASE-6426 Add Hadoop 2.0.x profile to 0.92+

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363211 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/pom.xml b/pom.xml
index ac0d75c15a92..76182d93a5a2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1994,6 +1994,88 @@
       </build>
     </profile>
 
+    <!--
+      profile for building against Hadoop 2.0.0-alpha. Activate using:
+       mvn -Dhadoop.profile=2.0
+    -->
+    <profile>
+      <id>hadoop-2.0</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>2.0</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>2.0.0-alpha</hadoop.version>
+        <slf4j.version>1.6.1</slf4j.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-annotations</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <!-- test deps for hadoop-2.0 profile -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>compile</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-resource</id>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>src/test/resources</directory>
+                      <includes>
+                        <include>hbase-site.xml</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>create-mrapp-generated-classpath</id>
+                <phase>generate-test-resources</phase>
+                <goals>
+                  <goal>build-classpath</goal>
+                </goals>
+                <configuration>
+                  <!-- needed to run the unit test for DS to generate
+                  the required classpath that is required in the env
+                  of the launch container in the mini mr/yarn cluster
+                  -->
+                  <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
 
     <!-- profiles for the tests
          See as well the properties of the project for the values

From 0080ea7d2653ee47e3e85fc322ff85aa415bcb9d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 19 Jul 2012 02:52:48 +0000
Subject: [PATCH 0357/1540] HBASE-6406 Remove TestReplicationPeer

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363213 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/TestReplicationPeer.java      | 82 -------------------
 1 file changed, 82 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeer.java

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeer.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeer.java
deleted file mode 100644
index 51bd6fa54cb9..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeer.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import junit.framework.Assert;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException.SessionExpiredException;
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-@Category(MediumTests.class)
-public class TestReplicationPeer {
-
-  private static final Log LOG = LogFactory.getLog(TestReplicationPeer.class);
-  private static HBaseTestingUtility utility;
-  private static Configuration conf;
-  private static ReplicationPeer rp;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    conf = HBaseConfiguration.create();
-    utility = new HBaseTestingUtility(conf);
-    conf = utility.getConfiguration();
-    utility.startMiniZKCluster();
-
-    rp = new ReplicationPeer(conf, "clusterKey", "clusterId");
-  }
-
-  @Test(timeout=300000)
-  public void testResetZooKeeperSession() throws Exception {
-    ZooKeeperWatcher zkw = rp.getZkw();
-    zkw.getRecoverableZooKeeper().exists("/1/2", false);
-
-    LOG.info("Expiring ReplicationPeer ZooKeeper session.");
-    utility.expireSession(zkw, null, false);
-
-    try {
-      LOG.info("Attempting to use expired ReplicationPeer ZooKeeper session.");
-      // Trying to use the expired session to assert that it is indeed closed
-      zkw.getRecoverableZooKeeper().exists("/1/2", false);
-    } catch (SessionExpiredException k) {
-      rp.reloadZkWatcher();
-
-      zkw = rp.getZkw();
-
-      // Try to use the connection again
-      LOG.info("Attempting to use refreshed "
-          + "ReplicationPeer ZooKeeper session.");
-      zkw.getRecoverableZooKeeper().exists("/1/2", false);
-
-      return;
-    }
-
-    Assert.fail("ReplicationPeer ZooKeeper session was not properly expired.");
-  }
-
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
-

From cf50bb00ed96561cc8960814c482bb9c9beb2d97 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 19 Jul 2012 16:36:49 +0000
Subject: [PATCH 0358/1540] HBASE-6420 Gracefully shutdown logsyncer

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363416 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLog.java   | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 72f35d7c3fdb..f308e6d230d0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -41,6 +41,7 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Lock;
@@ -980,7 +981,6 @@ public void closeAndDelete() throws IOException {
    */
   public void close() throws IOException {
     try {
-      logSyncerThread.interrupt();
       logSyncerThread.close();
       // Make sure we synced everything
       logSyncerThread.join(this.optionalFlushInterval*2);
@@ -1193,8 +1193,8 @@ public long append(HRegionInfo info, byte [] tableName, WALEdit edits,
    class LogSyncer extends HasThread {
 
     private final long optionalFlushInterval;
-    
-    private boolean closeLogSyncer = false;
+
+    private AtomicBoolean closeLogSyncer = new AtomicBoolean(false);
 
     // List of pending writes to the HLog. There corresponds to transactions
     // that have not yet returned to the client. We keep them cached here
@@ -1213,11 +1213,13 @@ public void run() {
       try {
         // awaiting with a timeout doesn't always
         // throw exceptions on interrupt
-        while(!this.isInterrupted() && !closeLogSyncer) {
+        while(!this.isInterrupted() && !closeLogSyncer.get()) {
 
           try {
             if (unflushedEntries.get() <= syncedTillHere) {
-              Thread.sleep(this.optionalFlushInterval);
+              synchronized (closeLogSyncer) {
+                closeLogSyncer.wait(this.optionalFlushInterval);
+              }
             }
             sync();
           } catch (IOException e) {
@@ -1256,9 +1258,12 @@ void hlogFlush(Writer writer, List<Entry> pending) throws IOException {
         writer.append(e);
       }
     }
-    
-    void close(){
-      closeLogSyncer = true;
+
+    void close() {
+      synchronized (closeLogSyncer) {
+        closeLogSyncer.set(true);
+        closeLogSyncer.notifyAll();
+      }
     }
   }
 

From a9a78c1afcba51f288794185f2e87c6acd73db15 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 19 Jul 2012 20:38:13 +0000
Subject: [PATCH 0359/1540] HBASE-4956 Control direct memory buffer consumption
 by HBaseClient (Bob Copeland)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/Result.java    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 57f5ecb599e0..d80b8c9bc0d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -462,10 +462,18 @@ public void readFields(final DataInput in)
       return;
     }
     byte [] raw = new byte[totalBuffer];
-    in.readFully(raw, 0, totalBuffer);
+    readChunked(in, raw, 0, totalBuffer);
     bytes = new ImmutableBytesWritable(raw, 0, totalBuffer);
   }
 
+  private void readChunked(final DataInput in, byte[] dest, int ofs, int len)
+  throws IOException {
+    int maxRead = 8192;
+
+    for (; ofs < len; ofs += maxRead)
+      in.readFully(dest, ofs, Math.min(len - ofs, maxRead));
+  }
+
   //Create KeyValue[] when needed
   private void readFields() {
     if (bytes == null) {

From df44f6731d413b7b7d08f19894b505a8e060e2d7 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 19 Jul 2012 21:57:19 +0000
Subject: [PATCH 0360/1540] HBASE-5985 TestMetaMigrationRemovingHTD failed with
 HADOOP 2.0.0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363561 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/TestMetaMigrationRemovingHTD.java    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
index d1c15afd5a1f..34e9956a373d 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
@@ -77,6 +77,11 @@ public static void setUpBeforeClass() throws Exception {
     FileSystem fs = FileSystem.get(conf);
     // find where hbase will root itself, so we can copy filesystem there
     Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
+    if (!fs.isDirectory(hbaseRootDir.getParent())) {
+      // mkdir at first
+      doFsCommand(shell,
+        new String [] {"-mkdir", hbaseRootDir.getParent().toString()});
+    }
     doFsCommand(shell,
       new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
     // See whats in minihdfs.

From c1d07d7fab02f5e0b8ea24567ea78546237fb77a Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 19 Jul 2012 22:19:49 +0000
Subject: [PATCH 0361/1540] HBASE-6319  ReplicationSource can call terminate on
 itself and deadlock HBASE-6325  [replication] Race in
 ReplicationSourceManager.init can initiate a failover even if the node is
 alive

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363570 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSource.java       |  5 +-
 .../ReplicationSourceManager.java             | 46 +++++++++++--------
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index a9c8ff7cdd58..e2257ea538ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -717,7 +717,10 @@ public void terminate(String reason, Exception cause) {
           + " because an error occurred: " + reason, cause);
     }
     this.running = false;
-    Threads.shutdown(this, this.sleepForRetries);
+    // Only wait for the thread to die if it's not us
+    if (!Thread.currentThread().equals(this)) {
+      Threads.shutdown(this, this.sleepForRetries);
+    }
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index e2dcf56bb32e..1d475d3427cd 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -79,7 +79,7 @@ public class ReplicationSourceManager {
   // The path to the latest log we saw, for new coming sources
   private Path latestPath;
   // List of all the other region servers in this cluster
-  private final List<String> otherRegionServers;
+  private final List<String> otherRegionServers = new ArrayList<String>();
   // Path to the hlogs directories
   private final Path logDir;
   // Path to the hlog archive
@@ -120,12 +120,9 @@ public ReplicationSourceManager(final ReplicationZookeeper zkHelper,
     this.sleepBeforeFailover = conf.getLong("replication.sleep.before.failover", 2000);
     this.zkHelper.registerRegionServerListener(
         new OtherRegionServerWatcher(this.zkHelper.getZookeeperWatcher()));
-    List<String> otherRSs =
-        this.zkHelper.getRegisteredRegionServers();
     this.zkHelper.registerRegionServerListener(
         new PeersWatcher(this.zkHelper.getZookeeperWatcher()));
     this.zkHelper.listPeersIdsAndWatch();
-    this.otherRegionServers = otherRSs == null ? new ArrayList<String>() : otherRSs;
     // It's preferable to failover 1 RS at a time, but with good zk servers
     // more could be processed at the same time.
     int nbWorkers = conf.getInt("replication.executor.workers", 1);
@@ -180,6 +177,7 @@ public void init() throws IOException {
       return;
     }
     synchronized (otherRegionServers) {
+      refreshOtherRegionServersList();
       LOG.info("Current list of replicators: " + currentReplicators
           + " other RSs: " + otherRegionServers);
     }
@@ -396,6 +394,27 @@ public void removePeer(String id) {
     this.zkHelper.deleteSource(id, true);
   }
 
+  /**
+   * Reads the list of region servers from ZK and atomically clears our
+   * local view of it and replaces it with the updated list.
+   * 
+   * @return true if the local list of the other region servers was updated
+   * with the ZK data (even if it was empty),
+   * false if the data was missing in ZK
+   */
+  private boolean refreshOtherRegionServersList() {
+    List<String> newRsList = zkHelper.getRegisteredRegionServers();
+    if (newRsList == null) {
+      return false;
+    } else {
+      synchronized (otherRegionServers) {
+        otherRegionServers.clear();
+        otherRegionServers.addAll(newRsList);
+      }
+    }
+    return true;
+  }
+
   /**
    * Watcher used to be notified of the other region server's death
    * in the local cluster. It initiates the process to transfer the queues
@@ -415,7 +434,7 @@ public OtherRegionServerWatcher(ZooKeeperWatcher watcher) {
      * @param path full path of the new node
      */
     public void nodeCreated(String path) {
-      refreshRegionServersList(path);
+      refreshListIfRightPath(path);
     }
 
     /**
@@ -426,7 +445,7 @@ public void nodeDeleted(String path) {
       if (stopper.isStopped()) {
         return;
       }
-      boolean cont = refreshRegionServersList(path);
+      boolean cont = refreshListIfRightPath(path);
       if (!cont) {
         return;
       }
@@ -442,23 +461,14 @@ public void nodeChildrenChanged(String path) {
       if (stopper.isStopped()) {
         return;
       }
-      refreshRegionServersList(path);
+      refreshListIfRightPath(path);
     }
 
-    private boolean refreshRegionServersList(String path) {
+    private boolean refreshListIfRightPath(String path) {
       if (!path.startsWith(zkHelper.getZookeeperWatcher().rsZNode)) {
         return false;
       }
-      List<String> newRsList = (zkHelper.getRegisteredRegionServers());
-      if (newRsList == null) {
-        return false;
-      } else {
-        synchronized (otherRegionServers) {
-          otherRegionServers.clear();
-          otherRegionServers.addAll(newRsList);
-        }
-      }
-      return true;
+      return refreshOtherRegionServersList();
     }
   }
 

From 7657d4d907cceab42fde089cf56dd7c6558ed785 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 19 Jul 2012 23:42:30 +0000
Subject: [PATCH 0362/1540] HBASE-5966 MapReduce based tests broken on Hadoop
 2.0.0-alpha (Gregory Chanan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1363586 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 19fca5803f59..3e9b24349636 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -81,6 +81,7 @@
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
@@ -1265,14 +1266,30 @@ public void startMiniMapReduceCluster() throws IOException {
   public void startMiniMapReduceCluster(final int servers) throws IOException {
     LOG.info("Starting mini mapreduce cluster...");
     // These are needed for the new and improved Map/Reduce framework
-    conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
+    Configuration c = getConfiguration();
+    String logDir = c.get("hadoop.log.dir");
+    String tmpDir = c.get("hadoop.tmp.dir");
+    if (logDir == null) {
+      logDir = tmpDir;
+    }
+    System.setProperty("hadoop.log.dir", logDir);
+    c.set("mapred.output.dir", tmpDir);
     mrCluster = new MiniMRCluster(servers,
       FileSystem.get(conf).getUri().toString(), 1);
     LOG.info("Mini mapreduce cluster started");
-    conf.set("mapred.job.tracker",
-        mrCluster.createJobConf().get("mapred.job.tracker"));
+    JobConf mrClusterJobConf = mrCluster.createJobConf();
+    c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
     /* this for mrv2 support */
     conf.set("mapreduce.framework.name", "yarn");
+    String rmAdress = mrClusterJobConf.get("yarn.resourcemanager.address");
+    if (rmAdress != null) {
+      conf.set("yarn.resourcemanager.address", rmAdress);
+    }
+    String schedulerAdress =
+      mrClusterJobConf.get("yarn.resourcemanager.scheduler.address");
+    if (schedulerAdress != null) {
+      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
+    }
   }
 
   /**

From 3f8317c039ff270c83b3aaa4d0bab17a61ee1c3a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 21 Jul 2012 17:25:33 +0000
Subject: [PATCH 0363/1540] HBASE-6363 HBaseConfiguration can carry a main
 method that dumps XML output for debug purposes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364131 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HBaseConfiguration.java  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 0477be8be1b5..5f4bb8916788 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -147,4 +147,12 @@ public static boolean isShowConfInServlet() {
     }
     return isShowConf;
   }
+
+  /** For debugging.  Dump configurations to system output as xml format.
+   * Master and RS configurations can also be dumped using
+   * http services. e.g. "curl http://master:60010/dump"
+   */
+  public static void main(String[] args) throws Exception {
+    HBaseConfiguration.create().writeXml(System.out);
+  }
 }

From d7462f879c197062d7b84593a2e3565b2264b9c3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 22 Jul 2012 01:36:54 +0000
Subject: [PATCH 0364/1540] HBASE-6406 Disable
 TestZooKeeper#testClientSessionExpired

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364207 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index fefa8b838fba..cd1623102d9d 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -101,7 +101,8 @@ public void setUp() throws Exception {
    * @throws IOException
    * @throws InterruptedException
    */
-  @Test
+  // fails frequently, disabled for now, see HBASE-6406
+  // @Test
   public void testClientSessionExpired()
   throws Exception {
     LOG.info("testClientSessionExpired");

From 8f31a06cbabed6440a59990be5b6c408f50d9d37 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 22 Jul 2012 19:12:54 +0000
Subject: [PATCH 0365/1540] HBASE-6433 Improve HBaseServer#getRemoteAddress by
 utilizing HBaseServer.Connection.hostAddress n(binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364397 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 3b1cb7975dfb..b2486bcaf59b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -174,8 +174,11 @@ public static InetAddress getRemoteIp() {
    *  @return String
    */
   public static String getRemoteAddress() {
-    InetAddress addr = getRemoteIp();
-    return (addr == null) ? null : addr.getHostAddress();
+    Call call = CurCall.get();
+    if (call != null) {
+      return call.connection.getHostAddress();
+    }
+    return null;
   }
 
   protected String bindAddress;

From 9730446e915914cda42c032dc81d195cc57d2e4a Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sun, 22 Jul 2012 21:40:03 +0000
Subject: [PATCH 0366/1540] HBASE-6440 SplitLogManager - log the exception when
 failed to finish split log file

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364436 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/SplitLogManager.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index dc1ff7082ed0..16d4f13352bb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -136,7 +136,7 @@ public Status finish(String workerName, String logfile) {
         try {
           HLogSplitter.finishSplitLogFile(logfile, conf);
         } catch (IOException e) {
-          LOG.warn("Could not finish splitting of log file " + logfile);
+          LOG.warn("Could not finish splitting of log file " + logfile, e);
           return Status.ERR;
         }
         return Status.DONE;

From 22672a48aa7d33c2c679a1d91c7a95b2f0a2d3d1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 23 Jul 2012 05:24:52 +0000
Subject: [PATCH 0367/1540] HBASE-5659
 TestAtomicOperation.testMultiRowMutationMultiThreads is still failing
 occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364502 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/StoreScanner.java  | 2 +-
 .../hadoop/hbase/regionserver/TestAtomicOperation.java      | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 44f6df13083c..1bc9d426a5b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -469,7 +469,7 @@ private boolean checkReseek() throws IOException {
     if (this.heap == null && this.lastTop != null) {
       resetScannerStack(this.lastTop);
       if (this.heap.peek() == null
-          || store.comparator.compare(this.lastTop, this.heap.peek()) != 0) {
+          || store.comparator.compareRows(this.lastTop, this.heap.peek()) != 0) {
         LOG.debug("Storescanner.peek() is changed where before = "
             + this.lastTop.toString() + ",and after = " + this.heap.peek());
         this.lastTop = null;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index d320d689f092..95d0248a659f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -255,7 +255,7 @@ public void testRowMutationMultiThreads() throws IOException {
     LOG.info("Starting test testRowMutationMultiThreads");
     initHRegion(tableName, getName(), fam1);
 
-    // create 100 threads, each will alternate between adding and
+    // create 10 threads, each will alternate between adding and
     // removing a column
     int numThreads = 10;
     int opsPerThread = 500;
@@ -339,10 +339,10 @@ public void testMultiRowMutationMultiThreads() throws IOException {
     LOG.info("Starting test testMultiRowMutationMultiThreads");
     initHRegion(tableName, getName(), fam1);
 
-    // create 100 threads, each will alternate between adding and
+    // create 10 threads, each will alternate between adding and
     // removing a column
     int numThreads = 10;
-    int opsPerThread = 1000;
+    int opsPerThread = 500;
     AtomicOperation[] all = new AtomicOperation[numThreads];
 
     AtomicLong timeStamps = new AtomicLong(0);

From eb7a2ffffedc29a2b91698e43d90d77990735ec3 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 23 Jul 2012 07:18:34 +0000
Subject: [PATCH 0368/1540] HBASE-4470 ServerNotRunningException coming out of
 assignRootAndMeta kills the Master (Gregory Chanan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/catalog/TestCatalogTracker.java     | 43 +++++++++++++++----
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
index 90fa45ac68f4..7a1bfae262ae 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
@@ -48,6 +48,7 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.util.Progressable;
 import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -239,14 +240,7 @@ public void run() {
     }
   }
 
-  /**
-   * Test we survive a connection refused {@link ConnectException}
-   * @throws IOException
-   * @throws InterruptedException
-   * @throws KeeperException
-   */
-  @Test
-  public void testGetMetaServerConnectionFails()
+  private void testVerifyMetaRegionLocationWithException(Exception ex)
   throws IOException, InterruptedException, KeeperException {
     // Mock an HRegionInterface.
     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
@@ -254,7 +248,7 @@ public void testGetMetaServerConnectionFails()
     try {
       // If a 'get' is called on mocked interface, throw connection refused.
       Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).
-        thenThrow(new ConnectException("Connection refused"));
+        thenThrow(ex);
       // Now start up the catalogtracker with our doctored Connection.
       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
       try {
@@ -273,6 +267,37 @@ public void testGetMetaServerConnectionFails()
     }
   }
 
+  /**
+   * Test we survive a connection refused {@link ConnectException}
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws KeeperException
+   */
+  @Test
+  public void testGetMetaServerConnectionFails()
+  throws IOException, InterruptedException, KeeperException {
+    testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
+  }
+
+  /**
+   * Test that verifyMetaRegionLocation properly handles getting a
+   * ServerNotRunningException. See HBASE-4470.
+   * Note this doesn't check the exact exception thrown in the
+   * HBASE-4470 as there it is thrown from getHConnection() and
+   * here it is thrown from get() -- but those are both called
+   * from the same function anyway, and this way is less invasive than
+   * throwing from getHConnection would be.
+   *
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws KeeperException
+   */
+  @Test
+  public void testVerifyMetaRegionServerNotRunning()
+  throws IOException, InterruptedException, KeeperException {
+    testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
+  }
+
   /**
    * Test get of root region fails properly if nothing to connect to.
    * @throws IOException

From a742ba973a99e25faebf015ee9fa3a2851976a24 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 24 Jul 2012 05:15:15 +0000
Subject: [PATCH 0369/1540] HBASE-5659 Addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364894 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/TestAtomicOperation.java      | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 95d0248a659f..6cbb2bc3f197 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -276,6 +276,9 @@ public void run() {
                 synchronized(region) {
                   LOG.debug("flushing");
                   region.flushcache();
+                  if (i%100==0) {
+                    region.compactStores();
+                  }
                 }
               }
               long ts = timeStamps.incrementAndGet();
@@ -361,6 +364,9 @@ public void run() {
                 synchronized(region) {
                   LOG.debug("flushing");
                   region.flushcache();
+                  if (i%100==0) {
+                    region.compactStores();
+                  }
                 }
               }
               long ts = timeStamps.incrementAndGet();

From 90424ffab66431f4fe8a27a4e523f8478347f453 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Jul 2012 08:55:07 +0000
Subject: [PATCH 0370/1540] HBASE-6445 rat check fails if hs_err_pid26514.log
 dropped in tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364935 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 76182d93a5a2..9f5700548074 100644
--- a/pom.xml
+++ b/pom.xml
@@ -513,6 +513,7 @@
           <version>0.8</version>
           <configuration>
             <excludes>
+              <exclude>**/.log</exclude>
               <exclude>**/.*</exclude>
               <exclude>**/*.tgz</exclude>
               <exclude>**/*.orig</exclude>

From fb48c35c4426c344577f0a03511c01634376ecbd Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 24 Jul 2012 08:57:24 +0000
Subject: [PATCH 0371/1540] HBASE-6445 rat check fails if hs_err_pid26514.log
 dropped in tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1364937 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 9f5700548074..fc8c7c6d5cc2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -513,7 +513,7 @@
           <version>0.8</version>
           <configuration>
             <excludes>
-              <exclude>**/.log</exclude>
+              <exclude>**/*.log</exclude>
               <exclude>**/.*</exclude>
               <exclude>**/*.tgz</exclude>
               <exclude>**/*.orig</exclude>

From 4da4ae61b7e9f69f06f54c70139fa7d2d191b4e5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 24 Jul 2012 18:22:50 +0000
Subject: [PATCH 0372/1540] HBASE-2214 Revert - breaks compatibility with
 0.92.x and 0.94.0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1365204 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ClientScanner.java    |  6 +---
 .../org/apache/hadoop/hbase/client/Scan.java  | 34 ++-----------------
 .../hadoop/hbase/regionserver/HRegion.java    |  7 ----
 .../hbase/regionserver/HRegionServer.java     | 10 ++----
 .../hbase/regionserver/RegionScanner.java     |  5 ---
 .../coprocessor/TestCoprocessorInterface.java |  5 ---
 6 files changed, 7 insertions(+), 60 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 166e33c39097..0fbabe5ded5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -95,13 +95,9 @@ public ClientScanner(final Configuration conf, final Scan scan,
       this.tableName = tableName;
       this.lastNext = System.currentTimeMillis();
       this.connection = connection;
-      if (scan.getMaxResultSize() > 0) {
-        this.maxScannerResultSize = scan.getMaxResultSize();
-      } else {
-        this.maxScannerResultSize = conf.getLong(
+      this.maxScannerResultSize = conf.getLong(
           HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
           HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
-      }
       this.scannerTimeout = (int) conf.getLong(
           HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
           HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 63cc975e06f7..093c93b68f6d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -51,11 +51,8 @@
  * To scan everything for each row, instantiate a Scan object.
  * <p>
  * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
- * If caching is NOT set, we will use the caching value of the hosting {@link HTable}.  See
- * {@link HTable#setScannerCaching(int)}. In addition to row caching, it is possible to specify a
- * maximum result size, using {@link #setMaxResultSize(long)}. When both are used,
- * single server requests are limited by either number of rows or maximum result size, whichever
- * limit comes first.
+ * If caching is NOT set, we will use the caching value of the hosting
+ * {@link HTable}.  See {@link HTable#setScannerCaching(int)}.
  * <p>
  * To further define the scope of what to get when scanning, perform additional
  * methods as outlined below.
@@ -87,7 +84,7 @@ public class Scan extends OperationWithAttributes implements Writable {
   private static final String RAW_ATTR = "_raw_";
   private static final String ISOLATION_LEVEL = "_isolationlevel_";
 
-  private static final byte SCAN_VERSION = (byte)3;
+  private static final byte SCAN_VERSION = (byte)2;
   private byte [] startRow = HConstants.EMPTY_START_ROW;
   private byte [] stopRow  = HConstants.EMPTY_END_ROW;
   private int maxVersions = 1;
@@ -103,7 +100,6 @@ public class Scan extends OperationWithAttributes implements Writable {
    * -1 means no caching
    */
   private int caching = -1;
-  private long maxResultSize = -1;
   private boolean cacheBlocks = true;
   private Filter filter = null;
   private TimeRange tr = new TimeRange();
@@ -153,7 +149,6 @@ public Scan(Scan scan) throws IOException {
     maxVersions = scan.getMaxVersions();
     batch = scan.getBatch();
     caching = scan.getCaching();
-    maxResultSize = scan.getMaxResultSize();
     cacheBlocks = scan.getCacheBlocks();
     filter = scan.getFilter(); // clone?
     TimeRange ctr = scan.getTimeRange();
@@ -327,24 +322,6 @@ public void setCaching(int caching) {
     this.caching = caching;
   }
 
-  /**
-   * @return the maximum result size in bytes. See {@link #setMaxResultSize(long)}
-   */
-  public long getMaxResultSize() {
-    return maxResultSize;
-  }
-
-  /**
-   * Set the maximum result size. The default is -1; this means that no specific
-   * maximum result size will be set for this scan, and the global configured
-   * value will be used instead. (Defaults to unlimited).
-   *
-   * @param maxResultSize The maximum result size in bytes.
-   */
-  public void setMaxResultSize(long maxResultSize) {
-    this.maxResultSize = maxResultSize;
-  }
-
   /**
    * Apply the specified server-side filter when performing the Scan.
    * @param filter filter to run on the server
@@ -523,7 +500,6 @@ public Map<String, Object> toMap(int maxCols) {
     map.put("maxVersions", this.maxVersions);
     map.put("batch", this.batch);
     map.put("caching", this.caching);
-    map.put("maxResultSize", this.maxResultSize);
     map.put("cacheBlocks", this.cacheBlocks);
     List<Long> timeRange = new ArrayList<Long>();
     timeRange.add(this.tr.getMin());
@@ -610,9 +586,6 @@ public void readFields(final DataInput in)
     if (version > 1) {
       readAttributes(in);
     }
-    if (version > 2) {
-      this.maxResultSize = in.readLong();
-    }
   }
 
   public void write(final DataOutput out)
@@ -646,7 +619,6 @@ public void write(final DataOutput out)
       }
     }
     writeAttributes(out);
-    out.writeLong(maxResultSize);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 0c570ab51ce2..024adb2b789a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3327,7 +3327,6 @@ class RegionScannerImpl implements RegionScanner {
     private int isScan;
     private boolean filterClosed = false;
     private long readPt;
-    private long maxResultSize;
 
     public HRegionInfo getRegionInfo() {
       return regionInfo;
@@ -3335,7 +3334,6 @@ public HRegionInfo getRegionInfo() {
     RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
       //DebugPrint.println("HRegionScanner.<init>");
 
-      this.maxResultSize = scan.getMaxResultSize();
       this.filter = scan.getFilter();
       this.batch = scan.getBatch();
       if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
@@ -3379,11 +3377,6 @@ public HRegionInfo getRegionInfo() {
       this(scan, null);
     }
 
-    @Override
-    public long getMaxResultSize() {
-      return maxResultSize;
-    }
-
     /**
      * Reset both the filter and the old filter.
      */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 230d6b1ea4cf..05a15ed7494f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2385,13 +2385,9 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
               : results.toArray(new Result[0]);
         }
       }
-      long maxResultSize;
-      if (s.getMaxResultSize() > 0) {
-        maxResultSize = s.getMaxResultSize();
-      } else {
-        maxResultSize = maxScannerResultSize;
-      }
-      for (int i = 0; i < nbRows && currentScanResultSize < maxResultSize; i++) {
+
+      for (int i = 0; i < nbRows
+          && currentScanResultSize < maxScannerResultSize; i++) {
         requestCount.incrementAndGet();
         // Collect values to be returned here
         boolean moreRows = s.next(values);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
index d87f70f7ba53..da95e901a8c9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -21,7 +21,6 @@
 
 import java.io.IOException;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.client.Scan;
 
 /**
  * RegionScanner describes iterators over rows in an HRegion.
@@ -50,8 +49,4 @@ public interface RegionScanner extends InternalScanner {
    */
   public boolean reseek(byte[] row) throws IOException;
 
-  /**
-   * @return The preferred max buffersize. See {@link Scan#setMaxResultSize(long)}
-   */
-  public long getMaxResultSize();
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index c30a528aaa12..99b2dd94661e 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -89,11 +89,6 @@ public boolean reseek(byte[] row) throws IOException {
       return false;
     }
 
-    @Override
-    public long getMaxResultSize() {
-      return delegate.getMaxResultSize();
-    }
-
   }
 
   public static class CoprocessorImpl extends BaseRegionObserver {

From 9168a913f747460c8e956a3886a92e07e9aa3006 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 24 Jul 2012 18:38:08 +0000
Subject: [PATCH 0373/1540] CHANGES.txt for 0.94.1RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1365210 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 4bf00b2d4f90..9f4e92a820db 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,11 +1,12 @@
 HBase Change Log
 
-Release 0.94.1 - 7/16/2012
+Release 0.94.1 - 7/24/2012
 Sub-task
 
     [HBASE-5342] - Grant/Revoke global permissions
     [HBASE-5372] - Table mutation operations should check table level rights, not global rights
     [HBASE-5385] - Delete table/column should delete stored permissions on -acl- table
+    [HBASE-5659] - TestAtomicOperation.testMultiRowMutationMultiThreads is still failing occasionally
     [HBASE-6061] - Fix ACL "Admin" Table inconsistent permission check
     [HBASE-6062] - preCheckAndPut/Delete() checks for READ when also a WRITE is performed
     [HBASE-6092] - Authorize flush, split, compact operations in AccessController
@@ -23,6 +24,7 @@ Sub-task
 Bug
 
     [HBASE-4379] - [hbck] Does not complain about tables with no end region [Z,]
+    [HBASE-4470] - ServerNotRunningException coming out of assignRootAndMeta kills the Master
     [HBASE-4891] - HTable.ClientScanner needs to clone the Scan object
     [HBASE-5546] - Master assigns region in the original region server when opening region failed
     [HBASE-5722] - NPE in ZKUtil#getChildDataAndWatchForNewChildren when ZK not available or NW down.
@@ -35,6 +37,7 @@ Bug
     [HBASE-5874] - When 'fs.default.name' not configured, the hbck tool and Merge tool throw IllegalArgumentException.
     [HBASE-5875] - Process RIT and Master restart may remove an online server considering it as a dead server
     [HBASE-5876] - TestImportExport has been failing against hadoop 0.23 profile
+    [HBASE-5883] - Backup master is going down due to connection refused exception
     [HBASE-5894] - Table deletion failed but HBaseAdmin#deletetable reports it as success
     [HBASE-5902] - Some scripts are not executable
     [HBASE-5909] - SlabStats should be a daemon thread
@@ -46,6 +49,7 @@ Bug
     [HBASE-5955] - Guava 11 drops MapEvictionListener and Hadoop 2.0.0-alpha requires it
     [HBASE-5963] - ClassCastException: FileSystem$Cache$ClientFinalizer cannot be cast to Thread
     [HBASE-5964] - HFileSystem: "No FileSystem for scheme: hdfs"
+    [HBASE-5966] - MapReduce based tests broken on Hadoop 2.0.0-alpha
     [HBASE-5975] - Failed suppression of fs shutdown hook with Hadoop 2.0.0
     [HBASE-5986] - Clients can see holes in the META table when regions are being split
     [HBASE-6002] - Possible chance of resource leak in HlogSplitter
@@ -94,6 +98,8 @@ Bug
     [HBASE-6303] - HCD.setCompressionType should use Enum support for storing compression types as strings
     [HBASE-6311] - Data error after majorCompaction caused by keeping MVCC for opened scanners
     [HBASE-6313] - Client hangs because the client is not notified
+    [HBASE-6319] - ReplicationSource can call terminate on itself and deadlock
+    [HBASE-6325] - [replication] Race in ReplicationSourceManager.init can initiate a failover even if the node is alive
     [HBASE-6326] - Avoid nested retry loops in HConnectionManager
     [HBASE-6328] - FSHDFSUtils#recoverFileLease tries to rethrow InterruptedException but actually shallows it
     [HBASE-6329] - Stopping META regionserver when splitting region could cause daughter region to be assigned twice
@@ -103,9 +109,15 @@ Bug
     [HBASE-6375] - Master may be using a stale list of region servers for creating assignment plan during startup
     [HBASE-6377] - HBASE-5533 metrics miss all operations submitted via MultiAction
     [HBASE-6380] - bulkload should update the store.storeSize
-    [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments
+    [HBASE-6392] - UnknownRegionException blocks hbck from sideline big overlap regions
     [HBASE-6394] - verifyrep MR job map tasks throws NullPointerException
     [HBASE-6397] - [hbck] print out bulk load commands for sidelined regions if necessary
+    [HBASE-6406] - TestReplicationPeer.testResetZooKeeperSession and TestZooKeeper.testClientSessionExpired fail frequently
+    [HBASE-6420] - Gracefully shutdown logsyncer
+    [HBASE-6426] - Add Hadoop 2.0.x profile to 0.92+
+    [HBASE-6440] - SplitLogManager - log the exception when failed to finish split log file
+    [HBASE-6443] - HLogSplitter should ignore 0 length files
+    [HBASE-6445] - rat check fails if hs_err_pid26514.log dropped in tests
 
 Improvement
 
@@ -138,13 +150,15 @@ Improvement
     [HBASE-6332] - Improve POM for better integration with downstream ivy projects
     [HBASE-6334] - TestImprovement for TestHRegion.testWritesWhileGetting
     [HBASE-6341] - Publicly expose HConnectionKey
+    [HBASE-6363] - HBaseConfiguration can carry a main method that dumps XML output for debug purposes
     [HBASE-6382] - Upgrade Jersey to 1.8 to match Hadoop 1 and 2
     [HBASE-6384] - hbck should group together those sidelined regions need to be bulk loaded later
+    [HBASE-6433] - Improve HBaseServer#getRemoteAddress by utilizing HBaseServer.Connection.hostAddress
 
 New Feature
 
-    [HBASE-2214] - Do HBASE-1996 -- setting size to return in scan rather than count of rows -- properly
     [HBASE-2730] - Expose RS work queue contents on web UI
+    [HBASE-4956] - Control direct memory buffer consumption by HBaseClient
     [HBASE-5609] - Add the ability to pass additional information for slow query logging
     [HBASE-5886] - Add new metric for possible data loss due to puts without WAL
     [HBASE-6044] - copytable: remove rs.* parameters
@@ -157,6 +171,10 @@ Task
     [HBASE-6129] - Backport of Add Increment Coalescing in thrift.
     [HBASE-6131] - Add attribution for code added by HBASE-5533 metrics
 
+Test
+
+    [HBASE-5985] - TestMetaMigrationRemovingHTD failed with HADOOP 2.0.0
+
 
 Release 0.94.0 - 5/1/2012
 Sub-task

From 22a391082b53d1dd8c812c04c6cd0a92008e22df Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 25 Jul 2012 09:30:41 +0000
Subject: [PATCH 0374/1540] HBASE-6447 Common TestZooKeeper failures on
 jenkins: testMasterSessionExpired and testCreateSilentIsReallySilent

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1365503 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/TestZooKeeper.java    | 47 ++++++++++++++++---
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index cd1623102d9d..b791b70dbb77 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -136,7 +137,9 @@ public void testRegionServerSessionExpired() throws Exception {
     testSanity();
   }
 
-  @Test
+  // @Test Disabled because seems to make no sense expiring master session
+  // and then trying to create table (down in testSanity); on master side
+  // it will fail because the master's session has expired -- St.Ack 07/24/2012
   public void testMasterSessionExpired() throws Exception {
     LOG.info("Starting testMasterSessionExpired");
     TEST_UTIL.expireMasterSession();
@@ -292,14 +295,46 @@ public void testCreateSilentIsReallySilent() throws InterruptedException,
     // Assumes the  root of the ZooKeeper space is writable as it creates a node
     // wherever the cluster home is defined.
     ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
-        "testMasterAddressManagerFromZK",
-        null);
+      "testMasterAddressManagerFromZK", null);
 
     // I set this acl after the attempted creation of the cluster home node.
-    zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
-    zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
-    zk.close();
+    // Add retries in case of retryable zk exceptions.
+    while (true) {
+      try {
+        zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
+        break;
+      } catch (KeeperException e) {
+        switch (e.code()) {
+          case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
+          case OPERATIONTIMEOUT:
+            LOG.warn("Possibly transient ZooKeeper exception: " + e);
+            Threads.sleep(100);
+            break;
+         default:
+            throw e;
+        }
+      }
+    }
 
+    while (true) {
+      try {
+        zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
+        break;
+      } catch (KeeperException e) {
+        switch (e.code()) {
+          case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
+          case OPERATIONTIMEOUT:
+            LOG.warn("Possibly transient ZooKeeper exception: " + e);
+            Threads.sleep(100);
+            break;
+         default:
+            throw e;
+        }
+      }
+    }
+    zk.close();
     ZKUtil.createAndFailSilent(zk2, aclZnode);
  }
   

From e1d19e07cfead3f64bf521b73618ff7e9bf9b8cf Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 25 Jul 2012 13:45:38 +0000
Subject: [PATCH 0375/1540] HBASE-6450 HBase startup should be with
 MALLOC_MAX_ARENA set

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1365583 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase-config.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 63b4f058eb68..1f25713a9436 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -81,6 +81,10 @@ if [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then
   . "${HBASE_CONF_DIR}/hbase-env.sh"
 fi
 
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
 if [ -z "$JAVA_HOME" ]; then
   for candidate in \
     /usr/lib/jvm/java-6-sun \

From 6bc36eabdd7d5aae9317bdc6ef3e87fa57465052 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 31 Jul 2012 04:00:08 +0000
Subject: [PATCH 0376/1540] HBASE-6427 Pluggable compaction and scan policies
 via coprocessors

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1367402 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/BaseRegionObserver.java |  36 ++-
 .../hbase/coprocessor/RegionObserver.java     |  94 +++++++
 .../hadoop/hbase/regionserver/HRegion.java    |   4 +-
 .../regionserver/RegionCoprocessorHost.java   | 125 ++++++++-
 .../hbase/regionserver/ScanQueryMatcher.java  |   6 +-
 .../hadoop/hbase/regionserver/ScanType.java   |  30 ++
 .../hadoop/hbase/regionserver/Store.java      |  91 ++++--
 .../hbase/regionserver/StoreScanner.java      |  32 +--
 .../hadoop/hbase/HBaseTestingUtility.java     |  16 ++
 .../apache/hadoop/hbase/MiniHBaseCluster.java |  28 ++
 .../hbase/client/TestFromClientSide.java      |   4 +-
 .../TestFromClientSideWithCoprocessor.java    |  42 +++
 .../coprocessor/SimpleRegionObserver.java     |  34 ++-
 .../regionserver/HFileReadWriteTest.java      |   3 +-
 .../regionserver/NoOpScanPolicyObserver.java  |  62 +++++
 .../TestCompactionWithCoprocessor.java        |  36 +++
 .../hbase/regionserver/TestMemStore.java      |   1 -
 .../hbase/regionserver/TestStoreScanner.java  |   3 +-
 .../hbase/util/TestCoprocessorScanPolicy.java | 262 ++++++++++++++++++
 19 files changed, 848 insertions(+), 61 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 93e53c308d15..5889f320ea7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -17,7 +17,7 @@
 package org.apache.hadoop.hbase.coprocessor;
 
 import java.util.List;
-import java.util.Map;
+import java.util.NavigableSet;
 
 import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -34,7 +34,9 @@
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
@@ -69,6 +71,13 @@ public void preClose(ObserverContext<RegionCoprocessorEnvironment> e,
   public void postClose(ObserverContext<RegionCoprocessorEnvironment> e,
       boolean abortRequested) { }
 
+  @Override
+  public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final KeyValueScanner memstoreScanner, final InternalScanner s)
+      throws IOException {
+    return null;
+  }
+
   @Override
   public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
   }
@@ -77,6 +86,17 @@ public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOE
   public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
   }
 
+  @Override
+  public InternalScanner preFlush(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
+      InternalScanner scanner) throws IOException {
+    return scanner;
+  }
+
+  @Override
+  public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
+      StoreFile resultFile) throws IOException {
+  }
+
   @Override
   public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
   }
@@ -100,6 +120,13 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
     return scanner;
   }
 
+  @Override
+  public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
+      final long earliestPutTs, final InternalScanner s) throws IOException {
+    return null;
+  }
+
   @Override
   public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
       final StoreFile resultFile) throws IOException {
@@ -236,6 +263,13 @@ public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvir
     return s;
   }
 
+  @Override
+  public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
+      final KeyValueScanner s) throws IOException {
+    return null;
+  }
+
   @Override
   public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e,
       final Scan scan, final RegionScanner s) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 889c4af0c81a..3dc2909bb6c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.NavigableSet;
 
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -33,9 +34,12 @@
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 
@@ -60,20 +64,63 @@ public interface RegionObserver extends Coprocessor {
    */
   void postOpen(final ObserverContext<RegionCoprocessorEnvironment> c);
 
+  /**
+   * Called before a memstore is flushed to disk and prior to creating the scanner to read from
+   * the memstore.  To override or modify how a memstore is flushed,
+   * implementing classes can return a new scanner to provide the KeyValues to be
+   * stored into the new {@code StoreFile} or null to perform the default processing.
+   * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * effect in this hook.
+   * @param c the environment provided by the region server
+   * @param store the store being flushed
+   * @param memstoreScanner the scanner for the memstore that is flushed
+   * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+   * @return the scanner to use during the flush.  {@code null} if the default implementation
+   * is to be used.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final KeyValueScanner memstoreScanner, final InternalScanner s)
+      throws IOException;
+
   /**
    * Called before the memstore is flushed to disk.
    * @param c the environment provided by the region server
    * @throws IOException if an error occurred on the coprocessor
+   * @deprecated use {@link #preFlush(ObserverContext, Store, InternalScanner)} instead
    */
   void preFlush(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
 
+  /**
+   * Called before a Store's memstore is flushed to disk.
+   * @param c the environment provided by the region server
+   * @param store the store where compaction is being requested
+   * @param scanner the scanner over existing data used in the store file
+   * @return the scanner to use during compaction.  Should not be {@code null}
+   * unless the implementation is writing new store files on its own.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preFlush(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      final InternalScanner scanner) throws IOException;
+
   /**
    * Called after the memstore is flushed to disk.
    * @param c the environment provided by the region server
    * @throws IOException if an error occurred on the coprocessor
+   * @deprecated use {@link #preFlush(ObserverContext, Store, InternalScanner)} instead.
    */
   void postFlush(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
 
+  /**
+   * Called after a Store's memstore is flushed to disk.
+   * @param c the environment provided by the region server
+   * @param store the store being flushed
+   * @param resultFile the new store file written out during compaction
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void postFlush(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      final StoreFile resultFile) throws IOException;
+
   /**
    * Called prior to selecting the {@link StoreFile}s to compact from the list
    * of available candidates.  To alter the files used for compaction, you may
@@ -123,6 +170,29 @@ void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
   InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final InternalScanner scanner) throws IOException;
 
+  /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into
+   * a new {@code StoreFile} and prior to creating the scanner used to read the
+   * input files.  To override or modify the compaction process,
+   * implementing classes can return a new scanner to provide the KeyValues to be
+   * stored into the new {@code StoreFile} or null to perform the default processing.
+   * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * effect in this hook.
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param scanners the list {@link StoreFileScanner}s to be read from
+   * @param scantype the {@link ScanType} indicating whether this is a major or minor compaction
+   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved
+   * store files
+   * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+   * @return the scanner to use during compaction.  {@code null} if the default implementation
+   * is to be used.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
+      final long earliestPutTs, final InternalScanner s) throws IOException;
+
   /**
    * Called after compaction has completed and the new store file has been
    * moved in to place.
@@ -545,6 +615,30 @@ RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment>
       final Scan scan, final RegionScanner s)
     throws IOException;
 
+  /**
+   * Called before a store opens a new scanner.
+   * This hook is called when a "user" scanner is opened.
+   * <p>
+   * See {@link #preFlushScannerOpen(ObserverContext, Store, KeyValueScanner, InternalScanner)}
+   * and {@link #preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner)}
+   * to override scanners created for flushes or compactions, resp.
+   * <p>
+   * Call CoprocessorEnvironment#complete to skip any subsequent chained
+   * coprocessors.
+   * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * effect in this hook.
+   * @param c the environment provided by the region server
+   * @param store the store being scanned
+   * @param scan the Scan specification
+   * @param targetCols columns to be used in the scanner
+   * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+   * @return a KeyValueScanner instance to use or {@code null} to use the default implementation
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
+      final KeyValueScanner s) throws IOException;
+
   /**
    * Called after the client opens a new scanner.
    * <p>
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 024adb2b789a..ac7ce418bc90 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1129,7 +1129,7 @@ void triggerMajorCompaction() {
    * @param majorCompaction True to force a major compaction regardless of thresholds
    * @throws IOException e
    */
-  void compactStores(final boolean majorCompaction)
+  public void compactStores(final boolean majorCompaction)
   throws IOException {
     if (majorCompaction) {
       this.triggerMajorCompaction();
@@ -3367,7 +3367,7 @@ public HRegionInfo getRegionInfo() {
       for (Map.Entry<byte[], NavigableSet<byte[]>> entry :
           scan.getFamilyMap().entrySet()) {
         Store store = stores.get(entry.getKey());
-        StoreScanner scanner = store.getScanner(scan, entry.getValue());
+        KeyValueScanner scanner = store.getScanner(scan, entry.getValue());
         scanners.add(scanner);
       }
       this.storeHeap = new KeyValueHeap(scanners, comparator);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 58afaf439b9f..f6efea5b1b23 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -303,6 +303,31 @@ public void postClose(boolean abortRequested) {
     }
   }
 
+  /**
+   * See
+   * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner)}
+   */
+  public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner> scanners,
+      ScanType scanType, long earliestPutTs) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    InternalScanner s = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver) env.getInstance()).preCompactScannerOpen(ctx, store, scanners,
+              scanType, earliestPutTs, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env,e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
   /**
    * Called prior to selecting the {@link StoreFile}s for compaction from
    * the list of currently available candidates.
@@ -389,7 +414,7 @@ public InternalScanner preCompact(Store store, InternalScanner scanner) throws I
    * Called after the store compaction has completed.
    * @param store the store being compacted
    * @param resultFile the new store file written during compaction
-   * @throws IOException 
+   * @throws IOException
    */
   public void postCompact(Store store, StoreFile resultFile) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
@@ -408,6 +433,31 @@ public void postCompact(Store store, StoreFile resultFile) throws IOException {
     }
   }
 
+  /**
+   * Invoked before a memstore flush
+   * @throws IOException
+   */
+  public InternalScanner preFlush(Store store, InternalScanner scanner) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    boolean bypass = false;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          scanner = ((RegionObserver)env.getInstance()).preFlush(
+              ctx, store, scanner);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env,e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? null : scanner;
+  }
+
   /**
    * Invoked before a memstore flush
    * @throws IOException 
@@ -429,9 +479,32 @@ public void preFlush() throws IOException {
     }
   }
 
+  /**
+   * See
+   * {@link RegionObserver#preFlush(ObserverContext, Store, KeyValueScanner)}
+   */
+  public InternalScanner preFlushScannerOpen(Store store, KeyValueScanner memstoreScanner) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    InternalScanner s = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver) env.getInstance()).preFlushScannerOpen(ctx, store, memstoreScanner, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
   /**
    * Invoked after a memstore flush
-   * @throws IOException 
+   * @throws IOException
    */
   public void postFlush() throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
@@ -450,9 +523,30 @@ public void postFlush() throws IOException {
     }
   }
 
+  /**
+   * Invoked after a memstore flush
+   * @throws IOException
+   */
+  public void postFlush(final Store store, final StoreFile storeFile) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postFlush(ctx, store, storeFile);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
   /**
    * Invoked just before a split
-   * @throws IOException 
+   * @throws IOException
    */
   public void preSplit() throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
@@ -1088,6 +1182,31 @@ public RegionScanner preScannerOpen(Scan scan) throws IOException {
     return bypass ? s : null;
   }
 
+  /**
+   * See
+   * {@link RegionObserver#preStoreScannerOpen(ObserverContext, Store, Scan, NavigableSet, KeyValueScanner)}
+   */
+  public KeyValueScanner preStoreScannerOpen(Store store, Scan scan,
+      final NavigableSet<byte[]> targetCols) throws IOException {
+    KeyValueScanner s = null;
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver) env.getInstance()).preStoreScannerOpen(ctx, store, scan,
+              targetCols, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
   /**
    * @param scan the Scan specification
    * @param s the scanner
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 3b5e21a6a98c..3435d62cd4e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -33,8 +33,6 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
-import org.apache.hadoop.hbase.regionserver.StoreScanner.ScanType;
-
 /**
  * A query matcher that is specifically designed for the scan case.
  */
@@ -136,7 +134,7 @@ public class ScanQueryMatcher {
    *  based on TTL
    */
   public ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
-      NavigableSet<byte[]> columns, StoreScanner.ScanType scanType,
+      NavigableSet<byte[]> columns, ScanType scanType,
       long readPointToUse, long earliestPutTs, long oldestUnexpiredTS) {
     this.tr = scan.getTimeRange();
     this.rowComparator = scanInfo.getComparator().getRawComparator();
@@ -183,7 +181,7 @@ public ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
    */
   ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
       NavigableSet<byte[]> columns, long oldestUnexpiredTS) {
-    this(scan, scanInfo, columns, StoreScanner.ScanType.USER_SCAN,
+    this(scan, scanInfo, columns, ScanType.USER_SCAN,
           Long.MAX_VALUE, /* max Readpoint to track versions */
         HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
new file mode 100644
index 000000000000..7b075120cbee
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Enum to distinguish general scan types.
+ */
+@InterfaceAudience.Private
+public enum ScanType {
+  MAJOR_COMPACT,
+  MINOR_COMPACT,
+  USER_SCAN
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 68219b90f653..aed956d88938 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -62,7 +62,6 @@
 import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.regionserver.StoreScanner.ScanType;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
@@ -225,9 +224,7 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
         Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
     LOG.info("time to purge deletes set to " + timeToPurgeDeletes +
         "ms in store " + this);
-    scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
-        family.getMaxVersions(), ttl, family.getKeepDeletedCells(),
-        timeToPurgeDeletes, this.comparator);
+    scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
     this.memstore = new MemStore(conf, this.comparator);
     this.storeNameStr = getColumnFamilyName();
 
@@ -698,15 +695,30 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
     if (set.size() == 0) {
       return null;
     }
-    Scan scan = new Scan();
-    scan.setMaxVersions(scanInfo.getMaxVersions());
     // Use a store scanner to find which rows to flush.
     // Note that we need to retain deletes, hence
     // treat this as a minor compaction.
-    InternalScanner scanner = new StoreScanner(this, scan, Collections
-        .singletonList(new CollectionBackedScanner(set, this.comparator)),
-        ScanType.MINOR_COMPACT, this.region.getSmallestReadPoint(),
-        HConstants.OLDEST_TIMESTAMP);
+    InternalScanner scanner = null;
+    KeyValueScanner memstoreScanner = new CollectionBackedScanner(set, this.comparator);
+    if (getHRegion().getCoprocessorHost() != null) {
+      scanner = getHRegion().getCoprocessorHost().preFlushScannerOpen(this, memstoreScanner);
+    }
+    if (scanner == null) {
+      Scan scan = new Scan();
+      scan.setMaxVersions(scanInfo.getMaxVersions());
+      scanner = new StoreScanner(this, scanInfo, scan, Collections.singletonList(new CollectionBackedScanner(
+          set, this.comparator)), ScanType.MINOR_COMPACT, this.region.getSmallestReadPoint(),
+          HConstants.OLDEST_TIMESTAMP);
+    }
+    if (getHRegion().getCoprocessorHost() != null) {
+      InternalScanner cpScanner =
+        getHRegion().getCoprocessorHost().preFlush(this, scanner);
+      // NULL scanner returned from coprocessor hooks means skip normal processing
+      if (cpScanner == null) {
+        return null;
+      }
+      scanner = cpScanner;
+    }
     try {
       // TODO:  We can fail in the below block before we complete adding this
       // flush to list of store files.  Add cleanup of anything put on filesystem
@@ -1543,20 +1555,27 @@ StoreFile.Writer compactStore(final Collection<StoreFile> filesToCompact,
     try {
       InternalScanner scanner = null;
       try {
-        Scan scan = new Scan();
-        scan.setMaxVersions(family.getMaxVersions());
-        /* include deletes, unless we are doing a major compaction */
-        scanner = new StoreScanner(this, scan, scanners,
-            majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
+        if (getHRegion().getCoprocessorHost() != null) {
+          scanner = getHRegion()
+              .getCoprocessorHost()
+              .preCompactScannerOpen(this, scanners,
+                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
+        }
+        if (scanner == null) {
+          Scan scan = new Scan();
+          scan.setMaxVersions(getFamily().getMaxVersions());
+          /* Include deletes, unless we are doing a major compaction */
+          scanner = new StoreScanner(this, getScanInfo(), scan, scanners,
+            majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
             smallestReadPoint, earliestPutTs);
-        if (region.getCoprocessorHost() != null) {
-          InternalScanner cpScanner = region.getCoprocessorHost().preCompact(
-              this, scanner);
+        }
+        if (getHRegion().getCoprocessorHost() != null) {
+          InternalScanner cpScanner =
+            getHRegion().getCoprocessorHost().preCompact(this, scanner);
           // NULL scanner returned from coprocessor hooks means skip normal processing
           if (cpScanner == null) {
             return null;
           }
-
           scanner = cpScanner;
         }
 
@@ -2036,11 +2055,18 @@ boolean getForceMajorCompaction() {
    * are not in a compaction.
    * @throws IOException
    */
-  public StoreScanner getScanner(Scan scan,
+  public KeyValueScanner getScanner(Scan scan,
       final NavigableSet<byte []> targetCols) throws IOException {
     lock.readLock().lock();
     try {
-      return new StoreScanner(this, scan, targetCols);
+      KeyValueScanner scanner = null;
+      if (getHRegion().getCoprocessorHost() != null) {
+        scanner = getHRegion().getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
+      }
+      if (scanner == null) {
+        scanner = new StoreScanner(this, getScanInfo(), scan, targetCols);
+      }
+      return scanner;
     } finally {
       lock.readLock().unlock();
     }
@@ -2152,7 +2178,7 @@ public int getCompactPriority(int priority) {
     }
   }
 
-  HRegion getHRegion() {
+  public HRegion getHRegion() {
     return this.region;
   }
 
@@ -2255,6 +2281,12 @@ public boolean commit(MonitoredTask status) throws IOException {
       }
       storeFile = Store.this.commitFile(storeFilePath, cacheFlushId,
                                snapshotTimeRangeTracker, flushedSize, status);
+      if (Store.this.getHRegion().getCoprocessorHost() != null) {
+        Store.this.getHRegion()
+            .getCoprocessorHost()
+            .postFlush(Store.this, storeFile);
+      }
+
       // Add new file to store files.  Clear snapshot too while we have
       // the Store write lock.
       return Store.this.updateStorefiles(storeFile, snapshot);
@@ -2297,6 +2329,10 @@ public KeyValue.KVComparator getComparator() {
     return comparator;
   }
 
+  public ScanInfo getScanInfo() {
+    return scanInfo;
+  }
+
   /**
    * Immutable information for scans over a store.
    */
@@ -2313,6 +2349,17 @@ public static class ScanInfo {
         + (2 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_INT)
         + Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN);
 
+    /**
+     * @param family {@link HColumnDescriptor} describing the column family
+     * @param ttl Store's TTL (in ms)
+     * @param timeToPurgeDeletes duration in ms after which a delete marker can
+     *        be purged during a major compaction.
+     * @param comparator The store's comparator
+     */
+    public ScanInfo(HColumnDescriptor family, long ttl, long timeToPurgeDeletes, KVComparator comparator) {
+      this(family.getName(), family.getMinVersions(), family.getMaxVersions(), ttl, family
+          .getKeepDeletedCells(), timeToPurgeDeletes, comparator);
+    }
     /**
      * @param family Name of this store's column family
      * @param minVersions Store's MIN_VERSIONS setting
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 1bc9d426a5b3..6204bcfc3ad9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
 import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -41,7 +42,7 @@
  * Scanner scans both the memstore and the HStore. Coalesce KeyValue stream
  * into List<KeyValue> for a single row.
  */
-class StoreScanner extends NonLazyKeyValueScanner
+public class StoreScanner extends NonLazyKeyValueScanner
     implements KeyValueScanner, InternalScanner, ChangedReadersObserver {
   static final Log LOG = LogFactory.getLog(StoreScanner.class);
   private Store store;
@@ -101,16 +102,16 @@ private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
    * @param columns which columns we are scanning
    * @throws IOException
    */
-  StoreScanner(Store store, Scan scan, final NavigableSet<byte[]> columns)
+  public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSet<byte[]> columns)
                               throws IOException {
-    this(store, scan.getCacheBlocks(), scan, columns, store.scanInfo.getTtl(),
-        store.scanInfo.getMinVersions());
+    this(store, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
+        scanInfo.getMinVersions());
     initializeMetricNames();
     if (columns != null && scan.isRaw()) {
       throw new DoNotRetryIOException(
           "Cannot specify any column for a raw scan");
     }
-    matcher = new ScanQueryMatcher(scan, store.scanInfo, columns,
+    matcher = new ScanQueryMatcher(scan, scanInfo, columns,
         ScanType.USER_SCAN, Long.MAX_VALUE, HConstants.LATEST_TIMESTAMP,
         oldestUnexpiredTS);
 
@@ -147,13 +148,13 @@ private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
    * @param smallestReadPoint the readPoint that we should use for tracking
    *          versions
    */
-  StoreScanner(Store store, Scan scan,
+  public StoreScanner(Store store, ScanInfo scanInfo, Scan scan,
       List<? extends KeyValueScanner> scanners, ScanType scanType,
       long smallestReadPoint, long earliestPutTs) throws IOException {
-    this(store, false, scan, null, store.scanInfo.getTtl(),
-        store.scanInfo.getMinVersions());
+    this(store, false, scan, null, scanInfo.getTtl(),
+        scanInfo.getMinVersions());
     initializeMetricNames();
-    matcher = new ScanQueryMatcher(scan, store.scanInfo, null, scanType,
+    matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType,
         smallestReadPoint, earliestPutTs, oldestUnexpiredTS);
 
     // Filter the list of scanners using Bloom filters, time range, TTL, etc.
@@ -170,7 +171,7 @@ private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
 
   /** Constructor for testing. */
   StoreScanner(final Scan scan, Store.ScanInfo scanInfo,
-      StoreScanner.ScanType scanType, final NavigableSet<byte[]> columns,
+      ScanType scanType, final NavigableSet<byte[]> columns,
       final List<KeyValueScanner> scanners) throws IOException {
     this(scan, scanInfo, scanType, columns, scanners,
         HConstants.LATEST_TIMESTAMP);
@@ -178,7 +179,7 @@ private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
 
   // Constructor for testing.
   StoreScanner(final Scan scan, Store.ScanInfo scanInfo,
-      StoreScanner.ScanType scanType, final NavigableSet<byte[]> columns,
+      ScanType scanType, final NavigableSet<byte[]> columns,
       final List<KeyValueScanner> scanners, long earliestPutTs)
           throws IOException {
     this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
@@ -544,14 +545,5 @@ List<KeyValueScanner> getAllScannersForTesting() {
   static void enableLazySeekGlobally(boolean enable) {
     lazySeekEnabledGlobally = enable;
   }
-
-  /**
-   * Enum to distinguish general scan types.
-   */
-  public static enum ScanType {
-    MAJOR_COMPACT,
-    MINOR_COMPACT,
-    USER_SCAN
-  }
 }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 3e9b24349636..6cfe5ea1fe56 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -758,6 +758,22 @@ public void flush(byte [] tableName) throws IOException {
     this.hbaseCluster.flushcache(tableName);
   }
 
+  /**
+   * Compact all regions in the mini hbase cluster
+   * @throws IOException
+   */
+  public void compact(boolean major) throws IOException {
+    this.hbaseCluster.compact(major);
+  }
+
+  /**
+   * Compact all of a table's reagion in the mini hbase cluster
+   * @throws IOException
+   */
+  public void compact(byte [] tableName, boolean major) throws IOException {
+    this.hbaseCluster.compact(tableName, major);
+  }
+
 
   /**
    * Create a table.
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index b36d12ca283f..6cfdee02fff6 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -446,6 +446,34 @@ public void flushcache(byte [] tableName) throws IOException {
     }
   }
 
+  /**
+   * Call flushCache on all regions on all participating regionservers.
+   * @throws IOException
+   */
+  public void compact(boolean major) throws IOException {
+    for (JVMClusterUtil.RegionServerThread t:
+        this.hbaseCluster.getRegionServers()) {
+      for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
+        r.compactStores(major);
+      }
+    }
+  }
+
+  /**
+   * Call flushCache on all regions of the specified table.
+   * @throws IOException
+   */
+  public void compact(byte [] tableName, boolean major) throws IOException {
+    for (JVMClusterUtil.RegionServerThread t:
+        this.hbaseCluster.getRegionServers()) {
+      for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
+        if(Bytes.equals(r.getTableDesc().getName(), tableName)) {
+          r.compactStores(major);
+        }
+      }
+    }
+  }
+
   /**
    * @return List of region server threads.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index d614508419de..256ddb8f9c79 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -103,12 +103,12 @@
 @Category(LargeTests.class)
 public class TestFromClientSide {
   final Log LOG = LogFactory.getLog(getClass());
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static byte [] ROW = Bytes.toBytes("testRow");
   private static byte [] FAMILY = Bytes.toBytes("testFamily");
   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
   private static byte [] VALUE = Bytes.toBytes("testValue");
-  private static int SLAVES = 3;
+  protected static int SLAVES = 3;
 
   /**
    * @throws java.lang.Exception
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java
new file mode 100644
index 000000000000..7b313dc67fac
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
+import org.apache.hadoop.hbase.regionserver.NoOpScanPolicyObserver;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test all client operations with a coprocessor that
+ * just implements the default flush/compact/scan policy
+ */
+@Category(LargeTests.class)
+public class TestFromClientSideWithCoprocessor extends TestFromClientSide {
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+        MultiRowMutationEndpoint.class.getName(), NoOpScanPolicyObserver.class.getName());
+    // We need more than one region server in this test
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index a691bacc4366..119a4878e985 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Arrays;
+import java.util.NavigableSet;
 
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.logging.Log;
@@ -42,7 +43,9 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
@@ -63,11 +66,13 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   boolean hadPreClose;
   boolean hadPostClose;
   boolean hadPreFlush;
+  boolean hadPreFlushScannerOpen;
   boolean hadPostFlush;
   boolean hadPreSplit;
   boolean hadPostSplit;
   boolean hadPreCompactSelect;
   boolean hadPostCompactSelect;
+  boolean hadPreCompactScanner;
   boolean hadPreCompact;
   boolean hadPostCompact;
   boolean hadPreGet = false;
@@ -87,6 +92,7 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   boolean hadPreScannerClose = false;
   boolean hadPostScannerClose = false;
   boolean hadPreScannerOpen = false;
+  boolean hadPreStoreScannerOpen = false;
   boolean hadPostScannerOpen = false;
   boolean hadPreBulkLoadHFile = false;
   boolean hadPostBulkLoadHFile = false;
@@ -120,12 +126,20 @@ public boolean wasClosed() {
   }
 
   @Override
-  public void preFlush(ObserverContext<RegionCoprocessorEnvironment> c) {
+  public InternalScanner preFlush(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner) {
     hadPreFlush = true;
+    return scanner;
   }
 
   @Override
-  public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c) {
+  public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
+    hadPreFlushScannerOpen = true;
+    return null;
+  }
+
+  @Override
+  public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile) {
     hadPostFlush = true;
   }
 
@@ -166,6 +180,14 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
     return scanner;
   }
 
+  @Override
+  public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
+      InternalScanner s) throws IOException {
+    hadPreCompactScanner = true;
+    return null;
+  }
+
   @Override
   public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
       Store store, StoreFile resultFile) {
@@ -184,6 +206,14 @@ public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvir
     return null;
   }
 
+  @Override
+  public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
+      final KeyValueScanner s) throws IOException {
+    hadPreStoreScannerOpen = true;
+    return null;
+  }
+
   @Override
   public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Scan scan, final RegionScanner s)
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java b/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
index ebc5373e2248..e6ff17305eca 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
@@ -61,7 +61,6 @@
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
-import org.apache.hadoop.hbase.regionserver.StoreScanner.ScanType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.LoadTestTool;
 import org.apache.hadoop.hbase.util.MD5Hash;
@@ -408,7 +407,7 @@ private void performMerge(List<StoreFileScanner> scanners, Store store,
       Scan scan = new Scan();
 
       // Include deletes
-      scanner = new StoreScanner(store, scan, scanners,
+      scanner = new StoreScanner(store, store.scanInfo, scan, scanners,
           ScanType.MAJOR_COMPACT, Long.MIN_VALUE, Long.MIN_VALUE);
 
       ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
new file mode 100644
index 000000000000..668c04372c20
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
@@ -0,0 +1,62 @@
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TestFromClientSideWithCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+
+/**
+ * RegionObserver that just reimplements the default behavior,
+ * in order to validate that all the necessary APIs for this are public
+ * This observer is also used in {@link TestFromClientSideWithCoprocessor} and
+ * {@link TestCompactionWithCoprocessor} to make sure that a wide range
+ * of functionality still behaves as expected.
+ */
+public class NoOpScanPolicyObserver extends BaseRegionObserver {
+  /**
+   * Reimplement the default behavior
+   */
+  @Override
+  public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
+    Store.ScanInfo oldSI = store.getScanInfo();
+    Store.ScanInfo scanInfo = new Store.ScanInfo(store.getFamily(), oldSI.getTtl(),
+        oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+    Scan scan = new Scan();
+    scan.setMaxVersions(oldSI.getMaxVersions());
+    return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
+        ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(),
+        HConstants.OLDEST_TIMESTAMP);
+  }
+
+  /**
+   * Reimplement the default behavior
+   */
+  @Override
+  public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
+      InternalScanner s) throws IOException {
+    // this demonstrates how to override the scanners default behavior
+    Store.ScanInfo oldSI = store.getScanInfo();
+    Store.ScanInfo scanInfo = new Store.ScanInfo(store.getFamily(), oldSI.getTtl(),
+        oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+    Scan scan = new Scan();
+    scan.setMaxVersions(oldSI.getMaxVersions());
+    return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion()
+        .getSmallestReadPoint(), earliestPutTs);
+  }
+
+  @Override
+  public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, final Scan scan, final NavigableSet<byte[]> targetCols, KeyValueScanner s)
+      throws IOException {
+    return new StoreScanner(store, store.getScanInfo(), scan, targetCols);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java
new file mode 100644
index 000000000000..ba30a9fdf388
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Make sure all compaction tests still pass with the preFlush and preCompact
+ * overridden to implement the default behavior
+ */
+@Category(MediumTests.class)
+public class TestCompactionWithCoprocessor extends TestCompaction {
+  /** constructor */
+  public TestCompactionWithCoprocessor() throws Exception {
+    super();
+    conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
+        NoOpScanPolicyObserver.class.getName());
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
index b1214f6c8540..083824fd0820 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
@@ -36,7 +36,6 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
-import org.apache.hadoop.hbase.regionserver.StoreScanner.ScanType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
index 3c582338e69b..01f0731549f2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
@@ -38,7 +38,6 @@
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
-import org.apache.hadoop.hbase.regionserver.StoreScanner.ScanType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdge;
@@ -559,7 +558,7 @@ public long currentTimeMillis() {
         KeyValue.COMPARATOR);
       StoreScanner scanner =
         new StoreScanner(scan, scanInfo,
-          StoreScanner.ScanType.MAJOR_COMPACT, null, scanners,
+          ScanType.MAJOR_COMPACT, null, scanners,
           HConstants.OLDEST_TIMESTAMP);
       List<KeyValue> results = new ArrayList<KeyValue>();
       results = new ArrayList<KeyValue>();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
new file mode 100644
index 000000000000..1915ca372762
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+// this is deliberately not in the o.a.h.h.regionserver package
+// in order to make sure all required classes/method are available
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.*;
+
+@Category(MediumTests.class)
+public class TestCoprocessorScanPolicy {
+  final Log LOG = LogFactory.getLog(getClass());
+  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final byte[] F = Bytes.toBytes("fam");
+  private static final byte[] Q = Bytes.toBytes("qual");
+  private static final byte[] R = Bytes.toBytes("row");
+
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+        ScanObserver.class.getName());
+    TEST_UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testBaseCases() throws Exception {
+    byte[] tableName = Bytes.toBytes("baseCases");
+    HTable t = TEST_UTIL.createTable(tableName, F, 1);
+    // set the version override to 2
+    Put p = new Put(R);
+    p.setAttribute("versions", new byte[]{});
+    p.add(F, tableName, Bytes.toBytes(2));
+    t.put(p);
+
+    // insert 2 versions
+    p = new Put(R);
+    p.add(F, Q, Q);
+    t.put(p);
+    p = new Put(R);
+    p.add(F, Q, Q);
+    t.put(p);
+    Get g = new Get(R);
+    g.setMaxVersions(10);
+    Result r = t.get(g);
+    assertEquals(2, r.size());
+
+    TEST_UTIL.flush(tableName);
+    TEST_UTIL.compact(tableName, true);
+
+    // both version are still visible even after a flush/compaction
+    g = new Get(R);
+    g.setMaxVersions(10);
+    r = t.get(g);
+    assertEquals(2, r.size());
+
+    // insert a 3rd version
+    p = new Put(R);
+    p.add(F, Q, Q);
+    t.put(p);
+    g = new Get(R);
+    g.setMaxVersions(10);
+    r = t.get(g);
+    // still only two version visible
+    assertEquals(2, r.size());
+
+    t.close();
+  }
+
+  @Test
+  public void testTTL() throws Exception {
+    byte[] tableName = Bytes.toBytes("testTTL");
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor(F)
+    .setMaxVersions(10)
+    .setTimeToLive(1);
+    desc.addFamily(hcd);
+    TEST_UTIL.getHBaseAdmin().createTable(desc);
+    HTable t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+    ManualEnvironmentEdge me = new ManualEnvironmentEdge();
+    me.setValue(now);
+    EnvironmentEdgeManagerTestHelper.injectEdge(me);
+    // 2s in the past
+    long ts = now - 2000;
+    // Set the TTL override to 3s
+    Put p = new Put(R);
+    p.setAttribute("ttl", new byte[]{});
+    p.add(F, tableName, Bytes.toBytes(3000L));
+    t.put(p);
+
+    p = new Put(R);
+    p.add(F, Q, ts, Q);
+    t.put(p);
+    p = new Put(R);
+    p.add(F, Q, ts+1, Q);
+    t.put(p);
+
+    // these two should be expired but for the override
+    // (their ts was 2s in the past)
+    Get g = new Get(R);
+    g.setMaxVersions(10);
+    Result r = t.get(g);
+    // still there?
+    assertEquals(2, r.size());
+
+    TEST_UTIL.flush(tableName);
+    TEST_UTIL.compact(tableName, true);
+
+    g = new Get(R);
+    g.setMaxVersions(10);
+    r = t.get(g);
+    // still there?
+    assertEquals(2, r.size());
+
+    // roll time forward 2s.
+    me.setValue(now + 2000);
+    // now verify that data eventually does expire
+    g = new Get(R);
+    g.setMaxVersions(10);
+    r = t.get(g);
+    // should be gone now
+    assertEquals(0, r.size());
+    t.close();
+  }
+
+  public static class ScanObserver extends BaseRegionObserver {
+    private Map<String, Long> ttls = new HashMap<String,Long>();
+    private Map<String, Integer> versions = new HashMap<String,Integer>();
+
+    // lame way to communicate with the coprocessor,
+    // since it is loaded by a different class loader
+    @Override
+    public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put,
+        final WALEdit edit, final boolean writeToWAL) throws IOException {
+      if (put.getAttribute("ttl") != null) {
+        KeyValue kv = put.getFamilyMap().values().iterator().next().get(0);
+        ttls.put(Bytes.toString(kv.getQualifier()), Bytes.toLong(kv.getValue()));
+        c.bypass();
+      } else if (put.getAttribute("versions") != null) {
+        KeyValue kv = put.getFamilyMap().values().iterator().next().get(0);
+        versions.put(Bytes.toString(kv.getQualifier()), Bytes.toInt(kv.getValue()));
+        c.bypass();
+      }
+    }
+
+    @Override
+    public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+        Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
+      Long newTtl = ttls.get(store.getTableName());
+      if (newTtl != null) {
+        System.out.println("PreFlush:" + newTtl);
+      }
+      Integer newVersions = versions.get(store.getTableName());
+      Store.ScanInfo oldSI = store.getScanInfo();
+      HColumnDescriptor family = store.getFamily();
+      Store.ScanInfo scanInfo = new Store.ScanInfo(family.getName(), family.getMinVersions(),
+          newVersions == null ? family.getMaxVersions() : newVersions,
+          newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
+          oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+      Scan scan = new Scan();
+      scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
+      return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
+          ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(),
+          HConstants.OLDEST_TIMESTAMP);
+    }
+
+    @Override
+    public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+        Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
+        long earliestPutTs, InternalScanner s) throws IOException {
+      Long newTtl = ttls.get(store.getTableName());
+      Integer newVersions = versions.get(store.getTableName());
+      Store.ScanInfo oldSI = store.getScanInfo();
+      HColumnDescriptor family = store.getFamily();
+      Store.ScanInfo scanInfo = new Store.ScanInfo(family.getName(), family.getMinVersions(),
+          newVersions == null ? family.getMaxVersions() : newVersions,
+          newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
+          oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+      Scan scan = new Scan();
+      scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
+      return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion()
+          .getSmallestReadPoint(), earliestPutTs);
+    }
+
+    @Override
+    public KeyValueScanner preStoreScannerOpen(
+        final ObserverContext<RegionCoprocessorEnvironment> c, Store store, final Scan scan,
+        final NavigableSet<byte[]> targetCols, KeyValueScanner s) throws IOException {
+      Long newTtl = ttls.get(store.getTableName());
+      Integer newVersions = versions.get(store.getTableName());
+      Store.ScanInfo oldSI = store.getScanInfo();
+      HColumnDescriptor family = store.getFamily();
+      Store.ScanInfo scanInfo = new Store.ScanInfo(family.getName(), family.getMinVersions(),
+          newVersions == null ? family.getMaxVersions() : newVersions,
+          newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
+          oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+      return new StoreScanner(store, scanInfo, scan, targetCols);
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+   new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 887284300aa89a83a2aed2697405b1c6a7f6fa49 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 31 Jul 2012 20:49:10 +0000
Subject: [PATCH 0377/1540] HBASE-6427 addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1367770 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/ScanType.java    | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
index 7b075120cbee..5f96c6a36259 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
@@ -17,12 +17,9 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-
 /**
  * Enum to distinguish general scan types.
  */
-@InterfaceAudience.Private
 public enum ScanType {
   MAJOR_COMPACT,
   MINOR_COMPACT,

From 936caf1b397cac46c16d1c7a00ea5d4722a7d07f Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 2 Aug 2012 20:24:20 +0000
Subject: [PATCH 0378/1540] HBASE-6460 hbck "-repairHoles" usage inconsistent
 with "-fixHdfsOrphans" (Jie Huang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1368689 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 41337fd89ac7..36939a62a974 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -3094,7 +3094,7 @@ protected static void printUsageAndExit() {
     System.err.println("");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
-    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
+    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
 
     Runtime.getRuntime().exit(-2);
   }

From 7ed6540ca69431da3885486239324671775707b4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 3 Aug 2012 14:05:35 +0000
Subject: [PATCH 0379/1540] HBASE-6359 KeyValue may return incorrect values
 after readFields() (Dave Revell)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1368967 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/KeyValue.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 59443835e992..8bf5d6a66391 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -2251,6 +2251,8 @@ public void readFields(int length, final DataInput in) throws IOException {
     this.rowCache = null;
     this.length = length;
     this.offset = 0;
+    this.timestampCache = -1;
+    this.keyLength = 0;
     this.bytes = new byte[this.length];
     in.readFully(this.bytes, 0, this.length);
   }

From fa687dbdd66dfd5ff1901397c711974b95bbaf39 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 4 Aug 2012 16:41:03 +0000
Subject: [PATCH 0380/1540] HBASE-6507 [hbck] TestHBaseFsck ran into
 TableNotEnabledException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1369364 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/TestHBaseFsck.java  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 46de3efa23f6..bc34d747e704 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -304,6 +304,18 @@ HTable setupTable(String tablename) throws Exception {
     }
     tbl.put(puts);
     tbl.flushCommits();
+    long endTime = System.currentTimeMillis() + 60000;
+    while (!TEST_UTIL.getHBaseAdmin().isTableEnabled(tablename)) {
+      try {
+        if (System.currentTimeMillis() > endTime) {
+          fail("Failed to enable table " + tablename + " after waiting for 60 sec");
+        }
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+        fail("Interrupted when waiting table " + tablename + " to be enabled");
+      }
+    }
     return tbl;
   }
 

From d7f43a6e7941037edbab1d06321081fb0a53719a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 5 Aug 2012 02:37:27 +0000
Subject: [PATCH 0381/1540] HBASE-6505 Allow shared RegionObserver state

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1369515 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/CoprocessorHost.java    |   1 +
 .../RegionCoprocessorEnvironment.java         |   5 +
 .../regionserver/RegionCoprocessorHost.java   |  61 +++++++--
 .../coprocessor/TestCoprocessorInterface.java | 125 ++++++++++++++++--
 4 files changed, 172 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 3e61aa23a2e3..36a1b3bc4429 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -209,6 +209,7 @@ public E load(Path path, String className, int priority,
           paths.add(file.toURL());
         }
       }
+      jarFile.close();
 
       StringTokenizer st = new StringTokenizer(cp, File.pathSeparator);
       while (st.hasMoreTokens()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
index 9395a3828a60..35a5c19f50e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import java.util.concurrent.ConcurrentMap;
+
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
@@ -31,4 +33,7 @@ public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment {
   /** @return reference to the region server services */
   public RegionServerServices getRegionServerServices();
 
+  /** @return shared data between all instances of this coprocessor */
+  public ConcurrentMap<String, Object> getSharedData();
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index f6efea5b1b23..07b46f903329 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -20,21 +20,39 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
-import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+
+import org.apache.commons.collections.map.AbstractReferenceMap;
+import org.apache.commons.collections.map.ReferenceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.coprocessor.*;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -45,9 +63,7 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.util.StringUtils;
 
-import java.io.IOException;
-import java.util.*;
-import java.util.regex.Matcher;
+import com.google.common.collect.ImmutableList;
 
 /**
  * Implements the coprocessor environment and runtime support for coprocessors
@@ -57,6 +73,9 @@ public class RegionCoprocessorHost
     extends CoprocessorHost<RegionCoprocessorHost.RegionEnvironment> {
 
   private static final Log LOG = LogFactory.getLog(RegionCoprocessorHost.class);
+  // The shared data map
+  private static ReferenceMap sharedDataMap =
+      new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK);
 
   /**
    * Encapsulation of the environment of each coprocessor
@@ -66,6 +85,7 @@ static class RegionEnvironment extends CoprocessorHost.Environment
 
     private HRegion region;
     private RegionServerServices rsServices;
+    ConcurrentMap<String, Object> sharedData;
 
     /**
      * Constructor
@@ -74,10 +94,11 @@ static class RegionEnvironment extends CoprocessorHost.Environment
      */
     public RegionEnvironment(final Coprocessor impl, final int priority,
         final int seq, final Configuration conf, final HRegion region,
-        final RegionServerServices services) {
+        final RegionServerServices services, final ConcurrentMap<String, Object> sharedData) {
       super(impl, priority, seq, conf);
       this.region = region;
       this.rsServices = services;
+      this.sharedData = sharedData;
     }
 
     /** @return the region */
@@ -95,6 +116,11 @@ public RegionServerServices getRegionServerServices() {
     public void shutdown() {
       super.shutdown();
     }
+
+    @Override
+    public ConcurrentMap<String, Object> getSharedData() {
+      return sharedData;
+    }
   }
 
   /** The region server services */
@@ -194,8 +220,19 @@ public RegionEnvironment createEnvironment(Class<?> implClass,
         break;
       }
     }
+    ConcurrentMap<String, Object> classData;
+    // make sure only one thread can add maps
+    synchronized (sharedDataMap) {
+      // as long as at least one RegionEnvironment holds on to its classData it will
+      // remain in this map
+      classData = (ConcurrentMap<String, Object>)sharedDataMap.get(implClass.getName());
+      if (classData == null) {
+        classData = new ConcurrentHashMap<String, Object>();
+        sharedDataMap.put(implClass.getName(), classData);
+      }
+    }
     return new RegionEnvironment(instance, priority, seq, conf, region,
-        rsServices);
+        rsServices, classData);
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 99b2dd94661e..fd1459046993 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -23,12 +23,15 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
@@ -105,14 +108,19 @@ public static class CoprocessorImpl extends BaseRegionObserver {
     private boolean postFlushCalled;
     private boolean preSplitCalled;
     private boolean postSplitCalled;
+    private ConcurrentMap<String, Object> sharedData;
 
     @Override
     public void start(CoprocessorEnvironment e) {
+      sharedData = ((RegionCoprocessorEnvironment)e).getSharedData();
+      // using new String here, so that there will be new object on each invocation
+      sharedData.putIfAbsent("test1", new Object());
       startCalled = true;
     }
 
     @Override
     public void stop(CoprocessorEnvironment e) {
+      sharedData = null;
       stopCalled = true;
     }
 
@@ -187,6 +195,104 @@ boolean wasCompacted() {
     boolean wasSplit() {
       return (preSplitCalled && postSplitCalled);
     }
+    Map<String, Object> getSharedData() {
+      return sharedData;
+    }
+  }
+
+  public static class CoprocessorII extends BaseRegionObserver {
+    private ConcurrentMap<String, Object> sharedData;
+    @Override
+    public void start(CoprocessorEnvironment e) {
+      sharedData = ((RegionCoprocessorEnvironment)e).getSharedData();
+      sharedData.putIfAbsent("test2", new Object());
+    }
+    @Override
+    public void stop(CoprocessorEnvironment e) {
+      sharedData = null;
+    }
+    @Override
+    public void preGet(final ObserverContext<RegionCoprocessorEnvironment> e,
+        final Get get, final List<KeyValue> results) throws IOException {
+      if (1/0 == 1) {
+        e.complete();
+      }
+    }
+
+    Map<String, Object> getSharedData() {
+      return sharedData;
+    }
+  }
+
+  public void testSharedData() throws IOException {
+    byte [] tableName = Bytes.toBytes("testtable");
+    byte [][] families = { fam1, fam2, fam3 };
+
+    Configuration hc = initSplit();
+    HRegion region = initHRegion(tableName, getName(), hc,
+      new Class<?>[]{}, families);
+
+    for (int i = 0; i < 3; i++) {
+      addContent(region, fam3);
+      region.flushcache();
+    }
+
+    region.compactStores();
+
+    byte [] splitRow = region.checkSplit();
+
+    assertNotNull(splitRow);
+    HRegion [] regions = split(region, splitRow);
+    for (int i = 0; i < regions.length; i++) {
+      regions[i] = reopenRegion(regions[i], CoprocessorImpl.class, CoprocessorII.class);
+    }
+    Coprocessor c = regions[0].getCoprocessorHost().
+        findCoprocessor(CoprocessorImpl.class.getName());
+    Coprocessor c2 = regions[0].getCoprocessorHost().
+        findCoprocessor(CoprocessorII.class.getName());
+    Object o = ((CoprocessorImpl)c).getSharedData().get("test1");
+    Object o2 = ((CoprocessorII)c2).getSharedData().get("test2");
+    assertNotNull(o);
+    assertNotNull(o2);
+    // to coprocessors get different sharedDatas
+    assertFalse(((CoprocessorImpl)c).getSharedData() == ((CoprocessorII)c2).getSharedData());
+    for (int i = 1; i < regions.length; i++) {
+      c = regions[i].getCoprocessorHost().
+          findCoprocessor(CoprocessorImpl.class.getName());
+      c2 = regions[i].getCoprocessorHost().
+          findCoprocessor(CoprocessorII.class.getName());
+      // make sure that all coprocessor of a class have identical sharedDatas
+      assertTrue(((CoprocessorImpl)c).getSharedData().get("test1") == o);
+      assertTrue(((CoprocessorII)c2).getSharedData().get("test2") == o2);
+    }
+    // now have all Environments fail
+    for (int i = 0; i < regions.length; i++) {
+      try {
+        Get g = new Get(regions[i].getStartKey());
+        regions[i].get(g, null);
+        fail();
+      } catch (DoNotRetryIOException xc) {
+      }
+      assertNull(regions[i].getCoprocessorHost().
+          findCoprocessor(CoprocessorII.class.getName()));
+    }
+    c = regions[0].getCoprocessorHost().
+        findCoprocessor(CoprocessorImpl.class.getName());
+    assertTrue(((CoprocessorImpl)c).getSharedData().get("test1") == o);
+    c = c2 = null;
+    // perform a GC
+    System.gc();
+    // reopen the region
+    region = reopenRegion(regions[0], CoprocessorImpl.class, CoprocessorII.class);
+    c = region.getCoprocessorHost().
+        findCoprocessor(CoprocessorImpl.class.getName());
+    // CPimpl is unaffected, still the same reference
+    assertTrue(((CoprocessorImpl)c).getSharedData().get("test1") == o);
+    c2 = region.getCoprocessorHost().
+        findCoprocessor(CoprocessorII.class.getName());
+    // new map and object created, hence the reference is different
+    // hence the old entry was indeed removed by the GC and new one has been created
+    assertFalse(((CoprocessorII)c2).getSharedData().get("test2") == o2);
   }
 
   public void testCoprocessorInterface() throws IOException {
@@ -195,7 +301,7 @@ public void testCoprocessorInterface() throws IOException {
 
     Configuration hc = initSplit();
     HRegion region = initHRegion(tableName, getName(), hc,
-      CoprocessorImpl.class, families);
+      new Class<?>[]{CoprocessorImpl.class}, families);
     for (int i = 0; i < 3; i++) {
       addContent(region, fam3);
       region.flushcache();
@@ -243,7 +349,7 @@ public void testCoprocessorInterface() throws IOException {
     }
   }
 
-  HRegion reopenRegion(final HRegion closedRegion, Class<?> implClass)
+  HRegion reopenRegion(final HRegion closedRegion, Class<?> ... implClasses)
       throws IOException {
     //HRegionInfo info = new HRegionInfo(tableName, null, null, false);
     HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
@@ -258,7 +364,9 @@ HRegion reopenRegion(final HRegion closedRegion, Class<?> implClass)
     RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
     r.setCoprocessorHost(host);
 
-    host.load(implClass, Coprocessor.PRIORITY_USER, conf);
+    for (Class<?> implClass : implClasses) {
+      host.load(implClass, Coprocessor.PRIORITY_USER, conf);
+    }
     // we need to manually call pre- and postOpen here since the
     // above load() is not the real case for CP loading. A CP is
     // expected to be loaded by default from 1) configuration; or 2)
@@ -271,7 +379,7 @@ HRegion reopenRegion(final HRegion closedRegion, Class<?> implClass)
   }
 
   HRegion initHRegion (byte [] tableName, String callingMethod,
-      Configuration conf, Class<?> implClass, byte [] ... families)
+      Configuration conf, Class<?> [] implClasses, byte [][] families)
       throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     for(byte [] family : families) {
@@ -285,10 +393,11 @@ HRegion initHRegion (byte [] tableName, String callingMethod,
     RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
     r.setCoprocessorHost(host);
 
-    host.load(implClass, Coprocessor.PRIORITY_USER, conf);
-
-    Coprocessor c = host.findCoprocessor(implClass.getName());
-    assertNotNull(c);
+    for (Class<?> implClass : implClasses) {
+      host.load(implClass, Coprocessor.PRIORITY_USER, conf);
+      Coprocessor c = host.findCoprocessor(implClass.getName());
+      assertNotNull(c);
+    }
 
     // Here we have to call pre and postOpen explicitly.
     host.preOpen();

From ca01bf36ab91534f3c91b0b64f527c44cf8e296f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 6 Aug 2012 20:48:19 +0000
Subject: [PATCH 0382/1540] HBASE-6373 Add more context information to audit
 log messages

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1370005 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/AccessController.java     | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 2fe921badc57..b82ad5dbecec 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -15,6 +15,7 @@
 package org.apache.hadoop.hbase.security.access;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
@@ -339,9 +340,15 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
 
   private void logResult(AuthResult result) {
     if (AUDITLOG.isTraceEnabled()) {
+      InetAddress remoteAddr = null;
+      RequestContext ctx = RequestContext.get();
+      if (ctx != null) {
+        remoteAddr = ctx.getRemoteAddress();
+      }
       AUDITLOG.trace("Access " + (result.isAllowed() ? "allowed" : "denied") +
           " for user " + (result.getUser() != null ? result.getUser().getShortName() : "UNKNOWN") +
           "; reason: " + result.getReason() +
+          "; remote address: " + (remoteAddr != null ? remoteAddr : "") +
           "; context: " + result.toContextString());
     }
   }

From 9636615783430bf9cfd0a80590f2227c89c0178d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 7 Aug 2012 20:15:44 +0000
Subject: [PATCH 0383/1540] HBASE-6523 HConnectionImplementation still does not
 recover from all ZK issues.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1370493 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/HConnectionManager.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 3edbc2eae5b3..7be6ccc3c7c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1663,8 +1663,9 @@ public void abort(final String msg, Throwable t) {
       if (t instanceof KeeperException) {
         LOG.info("This client just lost it's session with ZooKeeper, will"
             + " automatically reconnect when needed.");
-        if (t instanceof KeeperException.SessionExpiredException) {
-          LOG.info("ZK session expired. This disconnect could have been" +
+        if (t instanceof KeeperException.SessionExpiredException
+            || t instanceof KeeperException.ConnectionLossException) {
+          LOG.info("The ZK connection was or the session expired. This disconnect could have been" +
               " caused by a network partition or a long-running GC pause," +
               " either way it's recommended that you verify your environment.");
           resetZooKeeperTrackers();

From ba96408e6b5c92456f65f78a18f910f70ff004c4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 8 Aug 2012 02:05:56 +0000
Subject: [PATCH 0384/1540] HBASE-6523 Revert

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1370633 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/HConnectionManager.java   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 7be6ccc3c7c5..3edbc2eae5b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1663,9 +1663,8 @@ public void abort(final String msg, Throwable t) {
       if (t instanceof KeeperException) {
         LOG.info("This client just lost it's session with ZooKeeper, will"
             + " automatically reconnect when needed.");
-        if (t instanceof KeeperException.SessionExpiredException
-            || t instanceof KeeperException.ConnectionLossException) {
-          LOG.info("The ZK connection was or the session expired. This disconnect could have been" +
+        if (t instanceof KeeperException.SessionExpiredException) {
+          LOG.info("ZK session expired. This disconnect could have been" +
               " caused by a network partition or a long-running GC pause," +
               " either way it's recommended that you verify your environment.");
           resetZooKeeperTrackers();

From bc8981398e47d5ba85bc3d8e0a7afbf4adfc7360 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 8 Aug 2012 02:43:33 +0000
Subject: [PATCH 0385/1540] HBASE-6444 Expose the ability to set custom HTTP
 Request Headers for the REST client used by RemoteHTable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1370645 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/client/Client.java      | 47 ++++++++++++++++++-
 .../hbase/rest/client/RemoteHTable.java       |  2 +-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java
index 4ecbfa845a51..1e0dbbebd176 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java
@@ -21,6 +21,9 @@
 package org.apache.hadoop.hbase.rest.client;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
@@ -51,6 +54,8 @@ public class Client {
   private HttpClient httpClient;
   private Cluster cluster;
 
+  private Map<String, String> extraHeaders;
+
   /**
    * Default Constructor
    */
@@ -70,6 +75,7 @@ public Client(Cluster cluster) {
     managerParams.setConnectionTimeout(2000); // 2 s
     managerParams.setDefaultMaxConnectionsPerHost(10);
     managerParams.setMaxTotalConnections(100);
+    extraHeaders = new ConcurrentHashMap<String, String>();
     this.httpClient = new HttpClient(manager);
     HttpClientParams clientParams = httpClient.getParams();
     clientParams.setVersion(HttpVersion.HTTP_1_1);
@@ -84,6 +90,43 @@ public void shutdown() {
     manager.shutdown();
   }
 
+  /**
+   * @return the wrapped HttpClient
+   */
+  public HttpClient getHttpClient() {
+    return httpClient;
+  }
+
+  /**
+   * Add extra headers.  These extra headers will be applied to all http
+   * methods before they are removed. If any header is not used any more,
+   * client needs to remove it explicitly.
+   */
+  public void addExtraHeader(final String name, final String value) {
+    extraHeaders.put(name, value);
+  }
+
+  /**
+   * Get an extra header value.
+   */
+  public String getExtraHeader(final String name) {
+    return extraHeaders.get(name);
+  }
+
+  /**
+   * Get all extra headers (read-only).
+   */
+  public Map<String, String> getExtraHeaders() {
+    return Collections.unmodifiableMap(extraHeaders);
+  }
+
+  /**
+   * Remove an extra header.
+   */
+  public void removeExtraHeader(final String name) {
+    extraHeaders.remove(name);
+  }
+
   /**
    * Execute a transaction method given only the path. Will select at random
    * one of the members of the supplied cluster definition and iterate through
@@ -132,6 +175,9 @@ public int executePathOnly(Cluster cluster, HttpMethod method,
   public int executeURI(HttpMethod method, Header[] headers, String uri)
       throws IOException {
     method.setURI(new URI(uri, true));
+    for (Map.Entry<String, String> e: extraHeaders.entrySet()) {
+      method.addRequestHeader(e.getKey(), e.getValue());
+    }
     if (headers != null) {
       for (Header header: headers) {
         method.addRequestHeader(header);
@@ -452,5 +498,4 @@ public Response delete(Cluster cluster, String path) throws IOException {
       method.releaseConnection();
     }
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index 09744ef5f993..a5508c836074 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -76,7 +76,7 @@ public class RemoteHTable implements HTableInterface {
   final int maxRetries;
   final long sleepTime;
 
-  @SuppressWarnings("unchecked")
+  @SuppressWarnings("rawtypes")
   protected String buildRowSpec(final byte[] row, final Map familyMap, 
       final long startTime, final long endTime, final int maxVersions) {
     StringBuffer sb = new StringBuffer();

From fb6c00730fcc56416e3ccae14493b75b06dd71d5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 8 Aug 2012 05:53:17 +0000
Subject: [PATCH 0386/1540] Move the version on past the 0.94.1 release

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1370671 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index fc8c7c6d5cc2..07b48d09da86 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.1</version>
+  <version>0.94.2-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 10128518d0f22ce8b4aa8b88275a65e9d62a23cd Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 9 Aug 2012 01:07:44 +0000
Subject: [PATCH 0387/1540] HBASE-6522 Expose locks and leases to Coprocessors

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1371026 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegion.java  |  2 +-
 .../hadoop/hbase/regionserver/HRegionServer.java   |  6 ++----
 .../hbase/regionserver/RegionServerServices.java   |  5 +++++
 .../hbase/coprocessor/SimpleRegionObserver.java    | 14 ++++++++++++++
 .../hbase/util/MockRegionServerServices.java       |  6 ++++++
 5 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index ac7ce418bc90..bdf8b6576629 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3169,7 +3169,7 @@ boolean isRowLocked(final Integer lockId) {
    * simply return null if it could not acquire the lock.
    * @return lockid or null if waitForLock is false and the lock was unavailable.
    */
-  private Integer getLock(Integer lockid, byte [] row, boolean waitForLock)
+  public Integer getLock(Integer lockid, byte [] row, boolean waitForLock)
   throws IOException {
     Integer lid = null;
     if (lockid == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 05a15ed7494f..ebe8d6c3c7ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3182,10 +3182,8 @@ public long getProtocolVersion(final String protocol, final long clientVersion)
     throw new IOException("Unknown protocol: " + protocol);
   }
 
-  /**
-   * @return Return the leases.
-   */
-  protected Leases getLeases() {
+  @Override
+  public Leases getLeases() {
     return leases;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index d7ed4e82f33b..21b24ac9dcf7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -84,4 +84,9 @@ public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
    * @return Return the FileSystem object used by the regionserver
    */
   public FileSystem getFileSystem();
+
+  /**
+   * @return The RegionServer's "Leases" service
+   */
+  public Leases getLeases();
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index 119a4878e985..2a5baa593347 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -34,6 +34,7 @@
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
@@ -44,6 +45,7 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.Leases;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -97,6 +99,18 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   boolean hadPreBulkLoadHFile = false;
   boolean hadPostBulkLoadHFile = false;
 
+  @Override
+  public void start(CoprocessorEnvironment e) throws IOException {
+    // this only makes sure that leases and locks are available to coprocessors
+    // from external packages
+    RegionCoprocessorEnvironment re = (RegionCoprocessorEnvironment)e;
+    Leases leases = re.getRegionServerServices().getLeases();
+    leases.createLease("x", null);
+    leases.cancelLease("x");
+    Integer lid = re.getRegion().getLock(null, Bytes.toBytes("some row"), true);
+    re.getRegion().releaseRowLock(lid);
+  }
+
   @Override
   public void preOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
     hadPreOpen = true;
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
index 7d0275960cfc..6b25a5211b2a 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
 import org.apache.hadoop.hbase.regionserver.FlushRequester;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Leases;
 import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
@@ -156,4 +157,9 @@ public HFileSystem getFileSystem() {
   public void setFileSystem(FileSystem hfs) {
     this.hfs = (HFileSystem)hfs;
   }
+
+  @Override
+  public Leases getLeases() {
+    return null;
+  }
 }

From 56ff4309ca96dae6ae99fe55f8c652f9de9765ec Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 9 Aug 2012 06:21:45 +0000
Subject: [PATCH 0388/1540] HBASE-6520 MSLab May cause the Bytes.toLong not
 work correctly for increment (ShiXing)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1371045 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  2 +-
 .../hbase/regionserver/TestHRegion.java       | 32 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index bdf8b6576629..968bfb0d7dc0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4579,7 +4579,7 @@ public Result increment(Increment increment, Integer lockid,
             if (idx < results.size() &&
                 results.get(idx).matchingQualifier(column.getKey())) {
               KeyValue kv = results.get(idx);
-              amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset());
+              amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
               idx++;
             }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 5a2685c5180c..081d8680ce30 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -2602,6 +2602,38 @@ public void testIncrementColumnValue_WrongInitialSize() throws IOException {
     }
   }
 
+  public void testIncrement_WrongInitialSize() throws IOException {
+    this.region = initHRegion(tableName, getName(), fam1);
+    try {
+      byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
+      long row1Field1 = 0;
+      int row1Field2 = 1;
+      Put put1 = new Put(row1);
+      put1.add(fam1, qual1, Bytes.toBytes(row1Field1));
+      put1.add(fam1, qual2, Bytes.toBytes(row1Field2));
+      region.put(put1);
+      Increment increment = new Increment(row1);
+      increment.addColumn(fam1, qual1, 1);
+
+      //here we should be successful as normal
+      region.increment(increment, null, true);
+      assertICV(row1, fam1, qual1, row1Field1 + 1);
+
+      //failed to increment
+      increment = new Increment(row1);
+      increment.addColumn(fam1, qual2, 1);
+      try {
+        region.increment(increment, null, true);
+        fail("Expected to fail here");
+      } catch (Exception exception) {
+        // Expected.
+      }
+      assertICV(row1, fam1, qual2, row1Field2);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+  }
   private void assertICV(byte [] row,
                          byte [] familiy,
                          byte[] qualifier,

From 76ba1d996a8a9ab08faab740373e82f8c4b9369f Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 9 Aug 2012 21:52:21 +0000
Subject: [PATCH 0389/1540] HBASE-6552 TestAcidGuarantees system test should
 flush more aggressively (Gregory Chanan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1371502 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/TestAcidGuarantees.java      | 26 ++++++-------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
index 10039f59de1c..ad4f8e5596f4 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -241,15 +242,6 @@ public void runTestAtomicity(long millisToRun,
       int numGetters,
       int numScanners,
       int numUniqueRows) throws Exception {
-      runTestAtomicity(millisToRun, numWriters, numGetters, numScanners,
-		       numUniqueRows, true);
-  }
-
-  public void runTestAtomicity(long millisToRun,
-      int numWriters,
-      int numGetters,
-      int numScanners,
-      int numUniqueRows, boolean useFlusher) throws Exception {
     createTableIfMissing();
     TestContext ctx = new TestContext(util.getConfiguration());
     
@@ -266,13 +258,12 @@ public void runTestAtomicity(long millisToRun,
       ctx.addThread(writer);
     }
     // Add a flusher
-    if (useFlusher) {
-      ctx.addThread(new RepeatingTestThread(ctx) {
-        public void doAnAction() throws Exception {
-          util.flush();
-        }
-      });
-    }
+    ctx.addThread(new RepeatingTestThread(ctx) {
+      HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
+      public void doAnAction() throws Exception {
+        admin.flush(TABLE_NAME);
+      }
+    });
 
     List<AtomicGetReader> getters = Lists.newArrayList();
     for (int i = 0; i < numGetters; i++) {
@@ -360,8 +351,7 @@ public int run(String[] arg0) throws Exception {
     int numGetters = c.getInt("numGetters", 2);
     int numScanners = c.getInt("numScanners", 2);
     int numUniqueRows = c.getInt("numUniqueRows", 3);
-    // cannot run flusher in real cluster case.
-    runTestAtomicity(millis, numWriters, numGetters, numScanners, numUniqueRows, false);
+    runTestAtomicity(millis, numWriters, numGetters, numScanners, numUniqueRows);
     return 0;
   }
 

From 59607ba70ce92244f4b46a277714aec395defa6a Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 9 Aug 2012 22:22:06 +0000
Subject: [PATCH 0390/1540] HBASE-6525 bin/replication/copy_tables_desc.rb
 references non-existent class (David S. Wang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1371515 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/replication/copy_tables_desc.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/replication/copy_tables_desc.rb b/bin/replication/copy_tables_desc.rb
index ed85655933fe..13cfaffb8dfc 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -32,7 +32,6 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin
 import org.apache.hadoop.hbase.HTableDescriptor
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper
 
 # Name of this script
 NAME = "copy_tables_desc"

From 56068698a838eb8aac4c041b0329b1243dbc2c2e Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 9 Aug 2012 22:40:55 +0000
Subject: [PATCH 0391/1540] HBASE-6512 Incorrect OfflineMetaRepair log class
 name (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1371521 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index c586fd10d83c..5e91e2e39217 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -39,7 +39,7 @@
  * hbase org.apache.hadoop.hbase.util.hbck.OfflineMetaRepair ...
  */
 public class OfflineMetaRepair {
-  private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
+  private static final Log LOG = LogFactory.getLog(OfflineMetaRepair.class.getName());
 
   protected static void printUsageAndExit() {
     StringBuilder sb = new StringBuilder();

From 6149d7df18d19f09f76076ecab9fee53e76bd271 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 13 Aug 2012 19:32:19 +0000
Subject: [PATCH 0392/1540] HBASE-6308. Coprocessors should be loaded in a
 custom ClassLoader (James Baldassari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1372561 13f79535-47bb-0310-9956-ffa450edef68
---
 .../coprocessor/CoprocessorClassLoader.java   | 199 ++++++++++++++++++
 .../hbase/coprocessor/CoprocessorHost.java    |  15 +-
 .../hbase/coprocessor/TestClassLoading.java   |  76 +++++--
 3 files changed, 260 insertions(+), 30 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
new file mode 100644
index 000000000000..4d58b2c8948d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * ClassLoader used to load Coprocessor instances.
+ * 
+ * This ClassLoader always tries to load classes from the Coprocessor jar first 
+ * before delegating to the parent ClassLoader, thus avoiding dependency 
+ * conflicts between HBase's classpath and classes in the coprocessor's jar.  
+ * Certain classes are exempt from being loaded by this ClassLoader because it 
+ * would prevent them from being cast to the equivalent classes in the region 
+ * server.  For example, the Coprocessor interface needs to be loaded by the 
+ * region server's ClassLoader to prevent a ClassCastException when casting the 
+ * coprocessor implementation.
+ * 
+ * This ClassLoader also handles resource loading.  In most cases this 
+ * ClassLoader will attempt to load resources from the coprocessor jar first 
+ * before delegating to the parent.  However, like in class loading, 
+ * some resources need to be handled differently.  For all of the Hadoop 
+ * default configurations (e.g. hbase-default.xml) we will check the parent 
+ * ClassLoader first to prevent issues such as failing the HBase default 
+ * configuration version check.
+ */
+public class CoprocessorClassLoader extends URLClassLoader {
+  private static final Log LOG = 
+      LogFactory.getLog(CoprocessorClassLoader.class);
+  
+  /**
+   * If the class being loaded starts with any of these strings, we will skip
+   * trying to load it from the coprocessor jar and instead delegate 
+   * directly to the parent ClassLoader.
+   */
+  private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
+    // Java standard library:
+    "com.sun.",
+    "launcher.",
+    "java.",
+    "javax.",
+    "org.ietf",
+    "org.omg",
+    "org.w3c",
+    "org.xml",
+    "sunw.",
+    // Hadoop/HBase:
+    "org.apache.hadoop",
+  };
+  
+  /**
+   * If the resource being loaded matches any of these patterns, we will first 
+   * attempt to load the resource with the parent ClassLoader.  Only if the 
+   * resource is not found by the parent do we attempt to load it from the 
+   * coprocessor jar.
+   */
+  private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS = 
+      new Pattern[] {
+    Pattern.compile("^[^-]+-default\\.xml$")
+  };
+  
+  /**
+   * Creates a CoprocessorClassLoader that loads classes from the given paths.
+   * @param paths paths from which to load classes.
+   * @param parent the parent ClassLoader to set.
+   */
+  public CoprocessorClassLoader(List<URL> paths, ClassLoader parent) {
+    super(paths.toArray(new URL[]{}), parent);
+  }
+  
+  @Override
+  synchronized public Class<?> loadClass(String name) 
+      throws ClassNotFoundException {
+    // Delegate to the parent immediately if this class is exempt
+    if (isClassExempt(name)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Skipping exempt class " + name + 
+            " - delegating directly to parent");
+      }
+      return super.loadClass(name);
+    }
+    
+    // Check whether the class has already been loaded:
+    Class<?> clasz = findLoadedClass(name);
+    if (clasz != null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Class " + name + " already loaded");
+      }
+    }
+    else {
+      try {
+        // Try to find this class using the URLs passed to this ClassLoader, 
+        // which includes the coprocessor jar
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Finding class: " + name);
+        }
+        clasz = findClass(name);
+      } catch (ClassNotFoundException e) {
+        // Class not found using this ClassLoader, so delegate to parent
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Class " + name + " not found - delegating to parent");
+        }
+        try {
+          clasz = super.loadClass(name);
+        } catch (ClassNotFoundException e2) {
+          // Class not found in this ClassLoader or in the parent ClassLoader
+          // Log some debug output before rethrowing ClassNotFoundException
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Class " + name + " not found in parent loader");
+          }
+          throw e2;
+        }
+      }
+    }
+    
+    return clasz;
+  }
+  
+  @Override
+  synchronized public URL getResource(String name) {
+    URL resource = null;
+    boolean parentLoaded = false;
+    
+    // Delegate to the parent first if necessary
+    if (loadResourceUsingParentFirst(name)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Checking parent first for resource " + name);
+      }
+      resource = super.getResource(name);
+      parentLoaded = true;
+    }
+    
+    if (resource == null) {
+      // Try to find the resource in the coprocessor jar
+      resource = findResource(name);
+      if ((resource == null) && !parentLoaded) {
+        // Not found in the coprocessor jar and we haven't attempted to load 
+        // the resource in the parent yet; fall back to the parent
+        resource = super.getResource(name);
+      }
+    }
+
+    return resource;
+  }
+  
+  /**
+   * Determines whether the given class should be exempt from being loaded 
+   * by this ClassLoader.
+   * @param name the name of the class to test.
+   * @return true if the class should *not* be loaded by this ClassLoader; 
+   * false otherwise.
+   */
+  protected boolean isClassExempt(String name) {
+    for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
+      if (name.startsWith(exemptPrefix)) {
+        return true;
+      }
+    }
+    return false;
+  }
+  
+  /**
+   * Determines whether we should attempt to load the given resource using the  
+   * parent first before attempting to load the resource using this ClassLoader.
+   * @param name the name of the resource to test.
+   * @return true if we should attempt to load the resource using the parent 
+   * first; false if we should attempt to load the resource using this 
+   * ClassLoader first.
+   */
+  protected boolean loadResourceUsingParentFirst(String name) {
+    for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
+      if (resourcePattern.matcher(name).matches()) {
+        return true;
+      }
+    }
+    return false;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 36a1b3bc4429..68e241c2354b 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -43,7 +43,6 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.util.*;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
@@ -157,6 +156,8 @@ protected void loadSystemCoprocessors(Configuration conf, String confKey) {
   public E load(Path path, String className, int priority,
       Configuration conf) throws IOException {
     Class<?> implClass = null;
+    LOG.debug("Loading coprocessor class " + className + " with path " + 
+        path + " and priority " + priority);
 
     // Have we already loaded the class, perhaps from an earlier region open
     // for the same table?
@@ -164,12 +165,15 @@ public E load(Path path, String className, int priority,
       implClass = getClass().getClassLoader().loadClass(className);
     } catch (ClassNotFoundException e) {
       LOG.info("Class " + className + " needs to be loaded from a file - " +
-          path.toString() + ".");
+          path + ".");
       // go ahead to load from file system.
     }
 
     // If not, load
     if (implClass == null) {
+      if (path == null) {
+        throw new IOException("No jar path specified for " + className);
+      }
       // copy the jar to the local filesystem
       if (!path.toString().endsWith(".jar")) {
         throw new IOException(path.toString() + ": not a jar file?");
@@ -189,7 +193,6 @@ public E load(Path path, String className, int priority,
          aborts runaway user code */
 
       // load the jar and get the implementation main class
-      String cp = System.getProperty("java.class.path");
       // NOTE: Path.toURL is deprecated (toURI instead) but the URLClassLoader
       // unsurprisingly wants URLs, not URIs; so we will use the deprecated
       // method which returns URLs for as long as it is available
@@ -211,11 +214,7 @@ public E load(Path path, String className, int priority,
       }
       jarFile.close();
 
-      StringTokenizer st = new StringTokenizer(cp, File.pathSeparator);
-      while (st.hasMoreTokens()) {
-        paths.add((new File(st.nextToken())).getCanonicalFile().toURL());
-      }
-      ClassLoader cl = new URLClassLoader(paths.toArray(new URL[]{}),
+      ClassLoader cl = new CoprocessorClassLoader(paths,
         this.getClass().getClassLoader());
       Thread.currentThread().setContextClassLoader(cl);
       try {
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index dd14e2d3a990..bf4588711004 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -59,11 +59,12 @@ public class TestClassLoading {
   static final String cpName3 = "TestCP3";
   static final String cpName4 = "TestCP4";
   static final String cpName5 = "TestCP5";
+  static final String cpName6 = "TestCP6";
 
-  private static Class regionCoprocessor1 = ColumnAggregationEndpoint.class;
-  private static Class regionCoprocessor2 = GenericEndpoint.class;
-  private static Class regionServerCoprocessor = SampleRegionWALObserver.class;
-  private static Class masterCoprocessor = BaseMasterObserver.class;
+  private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
+  private static Class<?> regionCoprocessor2 = GenericEndpoint.class;
+  private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
+  private static Class<?> masterCoprocessor = BaseMasterObserver.class;
 
   private static final String[] regionServerSystemCoprocessors =
       new String[]{
@@ -290,6 +291,37 @@ public void testClassLoadingFromLocalFS() throws Exception {
     assertTrue("Class " + cpName3 + " was missing on a region", found);
   }
 
+  @Test
+  // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
+  public void testPrivateClassLoader() throws Exception {
+    File jarFile = buildCoprocessorJar(cpName4);
+
+    // create a table that references the jar
+    HTableDescriptor htd = new HTableDescriptor(cpName4);
+    htd.addFamily(new HColumnDescriptor("test"));
+    htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" +
+      Coprocessor.PRIORITY_USER);
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    admin.createTable(htd);
+    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
+
+    // verify that the coprocessor was loaded correctly
+    boolean found = false;
+    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
+    for (HRegion region:
+        hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
+      if (region.getRegionNameAsString().startsWith(cpName4)) {
+        Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
+        if (cp != null) {
+          found = true;
+          assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
+            cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
+        }
+      }
+    }
+    assertTrue("Class " + cpName4 + " was missing on a region", found);
+  }
+
   @Test
   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
   // less strict
@@ -298,8 +330,8 @@ public void testHBase3810() throws Exception {
 
     File jarFile1 = buildCoprocessorJar(cpName1);
     File jarFile2 = buildCoprocessorJar(cpName2);
-    File jarFile4 = buildCoprocessorJar(cpName4);
     File jarFile5 = buildCoprocessorJar(cpName5);
+    File jarFile6 = buildCoprocessorJar(cpName6);
 
     String cpKey1 = "COPROCESSOR$1";
     String cpKey2 = " Coprocessor$2 ";
@@ -322,13 +354,13 @@ public void testHBase3810() throws Exception {
     htd.setValue(cpKey3, cpValue3);
 
     // add 2 coprocessor by using new htd.addCoprocessor() api
-    htd.addCoprocessor(cpName4, new Path(jarFile4.getPath()),
+    htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()),
         Coprocessor.PRIORITY_USER, null);
     Map<String, String> kvs = new HashMap<String, String>();
     kvs.put("k1", "v1");
     kvs.put("k2", "v2");
     kvs.put("k3", "v3");
-    htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()),
+    htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()),
         Coprocessor.PRIORITY_USER, kvs);
 
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
@@ -341,9 +373,9 @@ public void testHBase3810() throws Exception {
 
     // verify that the coprocessor was loaded
     boolean found_2 = false, found_1 = false, found_3 = false,
-        found_4 = false, found_5 = false;
-    boolean found5_k1 = false, found5_k2 = false, found5_k3 = false,
-        found5_k4 = false;
+        found_5 = false, found_6 = false;
+    boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
+        found6_k4 = false;
 
     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
     for (HRegion region:
@@ -356,17 +388,17 @@ public void testHBase3810() throws Exception {
         found_3 = found_3 ||
             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
                 != null);
-        found_4 = found_4 ||
-            (region.getCoprocessorHost().findCoprocessor(cpName4) != null);
+        found_5 = found_5 ||
+            (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
 
         CoprocessorEnvironment env =
-            region.getCoprocessorHost().findCoprocessorEnvironment(cpName5);
+            region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
         if (env != null) {
-          found_5 = true;
+          found_6 = true;
           Configuration conf = env.getConfiguration();
-          found5_k1 = conf.get("k1") != null;
-          found5_k2 = conf.get("k2") != null;
-          found5_k3 = conf.get("k3") != null;
+          found6_k1 = conf.get("k1") != null;
+          found6_k2 = conf.get("k2") != null;
+          found6_k3 = conf.get("k3") != null;
         }
       }
     }
@@ -374,13 +406,13 @@ public void testHBase3810() throws Exception {
     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
-    assertTrue("Class " + cpName4 + " was missing on a region", found_4);
     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
+    assertTrue("Class " + cpName6 + " was missing on a region", found_6);
 
-    assertTrue("Configuration key 'k1' was missing on a region", found5_k1);
-    assertTrue("Configuration key 'k2' was missing on a region", found5_k2);
-    assertTrue("Configuration key 'k3' was missing on a region", found5_k3);
-    assertFalse("Configuration key 'k4' wasn't configured", found5_k4);
+    assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
+    assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
+    assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
+    assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
   }
 
   @Test

From 40bd364d1584ec6d0e197247b5bf86efa62561fc Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 15 Aug 2012 15:44:05 +0000
Subject: [PATCH 0393/1540] HBASE-5728 Methods Missing in HTableInterface

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373480 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HTableInterface.java  | 59 ++++++++++++++++++-
 .../hbase/coprocessor/CoprocessorHost.java    | 20 +++++++
 .../hbase/rest/client/RemoteHTable.java       | 20 +++++++
 3 files changed, 97 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
index e7d8cf889453..007f0a38e553 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
@@ -22,6 +22,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -29,8 +30,6 @@
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 
-import java.util.Map;
-
 /**
  * Used to communicate with a single HBase table.
  *
@@ -472,4 +471,60 @@ <T extends CoprocessorProtocol, R> void coprocessorExec(
       Class<T> protocol, byte[] startKey, byte[] endKey,
       Batch.Call<T,R> callable, Batch.Callback<R> callback)
       throws IOException, Throwable;
+
+  /**
+   * See {@link #setAutoFlush(boolean, boolean)}
+   *
+   * @param autoFlush
+   *          Whether or not to enable 'auto-flush'.
+   */
+  public void setAutoFlush(boolean autoFlush);
+
+  /**
+   * Turns 'auto-flush' on or off.
+   * <p>
+   * When enabled (default), {@link Put} operations don't get buffered/delayed
+   * and are immediately executed. Failed operations are not retried. This is
+   * slower but safer.
+   * <p>
+   * Turning off {@link #autoFlush} means that multiple {@link Put}s will be
+   * accepted before any RPC is actually sent to do the write operations. If the
+   * application dies before pending writes get flushed to HBase, data will be
+   * lost.
+   * <p>
+   * When you turn {@link #autoFlush} off, you should also consider the
+   * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put}
+   * requests will be retried on failure until successful. However, this can
+   * pollute the writeBuffer and slow down batching performance. Additionally,
+   * you may want to issue a number of Put requests and call
+   * {@link #flushCommits()} as a barrier. In both use cases, consider setting
+   * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()}
+   * has been called, regardless of success.
+   *
+   * @param autoFlush
+   *          Whether or not to enable 'auto-flush'.
+   * @param clearBufferOnFail
+   *          Whether to keep Put failures in the writeBuffer
+   * @see #flushCommits
+   */
+  public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail);
+
+  /**
+   * Returns the maximum size in bytes of the write buffer for this HTable.
+   * <p>
+   * The default value comes from the configuration parameter
+   * {@code hbase.client.write.buffer}.
+   * @return The size of the write buffer in bytes.
+   */
+  public long getWriteBufferSize();
+
+  /**
+   * Sets the size of the buffer in bytes.
+   * <p>
+   * If the new size is less than the current amount of data in the
+   * write buffer, the buffer gets flushed.
+   * @param writeBufferSize The new write buffer size, in bytes.
+   * @throws IOException if a remote or network exception occurs.
+   */
+  public void setWriteBufferSize(long writeBufferSize) throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 68e241c2354b..4fa1a14fd751 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -525,6 +525,26 @@ public <T extends CoprocessorProtocol> T coprocessorProxy(Class<T> protocol,
       public void mutateRow(RowMutations rm) throws IOException {
         table.mutateRow(rm);
       }
+
+      @Override
+      public void setAutoFlush(boolean autoFlush) {
+        table.setAutoFlush(autoFlush);
+      }
+
+      @Override
+      public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+        table.setAutoFlush(autoFlush, clearBufferOnFail);
+      }
+
+      @Override
+      public long getWriteBufferSize() {
+         return table.getWriteBufferSize();
+      }
+
+      @Override
+      public void setWriteBufferSize(long writeBufferSize) throws IOException {
+        table.setWriteBufferSize(writeBufferSize);
+      }
     }
 
     /** The coprocessor */
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index a5508c836074..ba638982411b 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -721,4 +721,24 @@ public <T extends CoprocessorProtocol, R> void coprocessorExec(
   public void mutateRow(RowMutations rm) throws IOException {
     throw new IOException("atomicMutation not supported");
   }
+
+  @Override
+  public void setAutoFlush(boolean autoFlush) {
+    throw new UnsupportedOperationException("setAutoFlush not implemented");
+  }
+
+  @Override
+  public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+    throw new UnsupportedOperationException("setAutoFlush not implemented");
+  }
+
+  @Override
+  public long getWriteBufferSize() {
+    throw new UnsupportedOperationException("getWriteBufferSize not implemented");
+  }
+
+  @Override
+  public void setWriteBufferSize(long writeBufferSize) throws IOException {
+    throw new IOException("setWriteBufferSize not supported");
+  }
 }

From 50d40c790816cb56bb4e82f807bc827a8ff67eec Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 15 Aug 2012 16:34:45 +0000
Subject: [PATCH 0394/1540] HBASE-6478.
 TestClassLoading.testClassLoadingFromLibDirInJar occasionally fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373521 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 17 +++++++++++++--
 .../hbase/coprocessor/TestClassLoading.java   | 21 ++++++++++++-------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 6cfe5ea1fe56..1c05fb8b9588 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1522,8 +1522,21 @@ public void waitTableAvailable(byte[] table, long timeoutMillis)
   throws InterruptedException, IOException {
     long startWait = System.currentTimeMillis();
     while (!getHBaseAdmin().isTableAvailable(table)) {
-      assertTrue("Timed out waiting for table " + Bytes.toStringBinary(table),
-          System.currentTimeMillis() - startWait < timeoutMillis);
+      assertTrue("Timed out waiting for table to become available " +
+        Bytes.toStringBinary(table),
+        System.currentTimeMillis() - startWait < timeoutMillis);
+      Thread.sleep(200);
+    }
+  }
+
+  public void waitTableEnabled(byte[] table, long timeoutMillis)
+  throws InterruptedException, IOException {
+    long startWait = System.currentTimeMillis();
+    while (!getHBaseAdmin().isTableAvailable(table) &&
+           !getHBaseAdmin().isTableEnabled(table)) {
+      assertTrue("Timed out waiting for table to become available and enabled " +
+         Bytes.toStringBinary(table),
+         System.currentTimeMillis() - startWait < timeoutMillis);
       Thread.sleep(200);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index bf4588711004..bea1fe1b4cb1 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -234,7 +234,7 @@ public void testClassLoadingFromHDFS() throws Exception {
       admin.deleteTable(tableName);
     }
     admin.createTable(htd);
-    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
+    waitForTable(htd.getName());
 
     // verify that the coprocessors were loaded
     boolean found1 = false, found2 = false, found2_k1 = false,
@@ -277,7 +277,7 @@ public void testClassLoadingFromLocalFS() throws Exception {
       Coprocessor.PRIORITY_USER);
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     admin.createTable(htd);
-    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
+    waitForTable(htd.getName());
 
     // verify that the coprocessor was loaded
     boolean found = false;
@@ -303,7 +303,7 @@ public void testPrivateClassLoader() throws Exception {
       Coprocessor.PRIORITY_USER);
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     admin.createTable(htd);
-    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
+    waitForTable(htd.getName());
 
     // verify that the coprocessor was loaded correctly
     boolean found = false;
@@ -369,7 +369,7 @@ public void testHBase3810() throws Exception {
       admin.deleteTable(tableName);
     }
     admin.createTable(htd);
-    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
+    waitForTable(htd.getName());
 
     // verify that the coprocessor was loaded
     boolean found_2 = false, found_1 = false, found_3 = false,
@@ -472,7 +472,7 @@ public void testClassLoadingFromLibDirInJar() throws Exception {
       admin.deleteTable(tableName);
     }
     admin.createTable(htd);
-    TEST_UTIL.waitTableAvailable(htd.getName(), 5000);
+    waitForTable(htd.getName());
 
     // verify that the coprocessors were loaded
     boolean found1 = false, found2 = false, found2_k1 = false,
@@ -544,7 +544,7 @@ public void testRegionServerCoprocessorsReported() throws Exception {
     String userTable1 = "userTable1";
     HTableDescriptor userTD1 = new HTableDescriptor(userTable1);
     admin.createTable(userTD1);
-    TEST_UTIL.waitTableAvailable(userTD1.getName(), 5000);
+    waitForTable(userTD1.getName());
 
     // table should be enabled now.
     assertTrue(admin.isTableEnabled(userTable1));
@@ -564,7 +564,7 @@ public void testRegionServerCoprocessorsReported() throws Exception {
     htd2.setValue("COPROCESSOR$1", jarFile1.toString() + "|" + userTableCP +
       "|" + Coprocessor.PRIORITY_USER);
     admin.createTable(htd2);
-    TEST_UTIL.waitTableAvailable(htd2.getName(), 5000);
+    waitForTable(htd2.getName());
     // table should be enabled now.
     assertTrue(admin.isTableEnabled(userTable2));
 
@@ -659,6 +659,13 @@ public void testMasterCoprocessorsReported() {
     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
   }
 
+  private void waitForTable(byte[] name) throws InterruptedException, IOException {
+    // First wait until all regions are online
+    TEST_UTIL.waitTableEnabled(name, 5000);
+    // Now wait a bit longer for the coprocessor hosts to load the CPs
+    Thread.sleep(1000);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From e1a1f798ddc002325e92fbfc4d99f17832e20570 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 15 Aug 2012 17:31:51 +0000
Subject: [PATCH 0395/1540] HBASE-6577 RegionScannerImpl.nextRow() should seek
 to next row

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 968bfb0d7dc0..dc66013c5d3c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3480,7 +3480,7 @@ private boolean nextInternal(int limit) throws IOException {
             // there're no left overs.
             // the reasons for calling this method are:
             // 1. reset the filters.
-            // 2. provide a hook to fast forward the row (used by subclasses)
+            // 2. fast forward the row
             nextRow(currentRow);
 
             // This row was totally filtered out, if this is NOT the last row,
@@ -3503,6 +3503,9 @@ private boolean filterRowKey(byte[] row) {
     }
 
     protected void nextRow(byte [] currentRow) throws IOException {
+      KeyValue kv = KeyValue.createLastOnRow(currentRow);
+      this.storeHeap.requestSeek(kv, true, true);
+
       while (Bytes.equals(currentRow, peekRow())) {
         this.storeHeap.next(MOCKED_LIST);
       }

From df66178ed3be8194991f43373d4b39a6065fd5e2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 15 Aug 2012 17:55:34 +0000
Subject: [PATCH 0396/1540] HBASE-6579 Unnecessary KV order check in
 StoreScanner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373540 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/StoreScanner.java | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 6204bcfc3ad9..26afb223fb3b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -342,11 +342,8 @@ public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOE
 
     LOOP: while((kv = this.heap.peek()) != null) {
       // Check that the heap gives us KVs in an increasing order.
-      if (prevKV != null && comparator != null
-          && comparator.compare(prevKV, kv) > 0) {
-        throw new IOException("Key " + prevKV + " followed by a " +
-            "smaller key " + kv + " in cf " + store);
-      }
+      assert prevKV == null || comparator == null || comparator.compare(prevKV, kv) <= 0 :
+        "Key " + prevKV + " followed by a " + "smaller key " + kv + " in cf " + store;
       prevKV = kv;
       ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
       switch(qcode) {

From 3b015347c17b41eb58958238915d9e174d158665 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 15 Aug 2012 19:28:30 +0000
Subject: [PATCH 0397/1540] HBASE-6263. Use default mode for Thrift gateway if
 not specified

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373593 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/thrift/ThriftServerRunner.java   | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index e03c7755fe17..36fd7fec8138 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -200,11 +200,14 @@ static void setServerImpl(CommandLine cmd, Configuration conf) {
           ++numChosen;
         }
       }
-      if (numChosen != 1) {
+      if (numChosen < 1) {
+        LOG.info("Using default thrift server type");
+        chosenType = DEFAULT;
+      } else if (numChosen > 1) {
         throw new AssertionError("Exactly one option out of " +
-            Arrays.toString(values()) + " has to be specified");
+          Arrays.toString(values()) + " has to be specified");
       }
-      LOG.info("Setting thrift server to " + chosenType.option);
+      LOG.info("Using thrift server type " + chosenType.option);
       conf.set(SERVER_TYPE_CONF_KEY, chosenType.option);
     }
 

From e69de35d90954f369f3ac7d768d5aa667e14b8cb Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 16 Aug 2012 00:23:44 +0000
Subject: [PATCH 0398/1540] HBASE-6562 Fake KVs are sometimes passed to filters

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373679 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     |  8 ++
 .../hadoop/hbase/regionserver/HRegion.java    |  5 +-
 .../hbase/client/TestFakeKeyInFilter.java     | 84 +++++++++++++++++++
 3 files changed, 95 insertions(+), 2 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 8bf5d6a66391..2f8c724ee209 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -954,6 +954,14 @@ public boolean isLatestTimestamp() {
       HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG);
   }
 
+  /**
+   * @return True if this is a "fake" KV created for internal seeking purposes,
+   * which should not be seen by user code
+   */
+  public boolean isInternal() {
+    byte type = getType();
+    return type == Type.Minimum.code || type == Type.Maximum.code;
+  }
   /**
    * @param now Time to set into <code>this</code> IFF timestamp ==
    * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop).
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index dc66013c5d3c..9b2996c0de5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3441,7 +3441,8 @@ private boolean nextInternal(int limit) throws IOException {
           rpcCall.throwExceptionIfCallerDisconnected();
         }
 
-        byte [] currentRow = peekRow();
+        KeyValue kv = this.storeHeap.peek();
+        byte [] currentRow = kv == null ? null : kv.getRow();
         if (isStopRow(currentRow)) {
           if (filter != null && filter.hasFilterRow()) {
             filter.filterRow(results);
@@ -3451,7 +3452,7 @@ private boolean nextInternal(int limit) throws IOException {
           }
 
           return false;
-        } else if (filterRowKey(currentRow)) {
+        } else if (kv != null && !kv.isInternal() && filterRowKey(currentRow)) {
           nextRow(currentRow);
         } else {
           byte [] nextRow;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java b/src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java
new file mode 100644
index 000000000000..d80748afbafa
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.filter.*;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Make sure the fake KVs created internally are never user visible
+ * (not even to filters)
+ */
+@Category(SmallTests.class)
+public class TestFakeKeyInFilter extends BinaryComparator {
+  protected static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  public TestFakeKeyInFilter() {
+    super(Bytes.toBytes("foo"));
+  }
+
+  @Override
+  public int compareTo(byte[] value, int offset, int length) {
+    if (value.length == 0) {
+      throw new RuntimeException("Found mysterious empty row");
+    }
+    return 0;
+  }
+
+  /**
+   * Simple way to verify the scenario.
+   * There are no KVs with an empty row key in the
+   * table, yet such a KV is presented to the filter.
+   */
+  @Test
+  public void testForEmptyRowKey() throws Exception {
+    byte[] table = Bytes.toBytes("testForEmptyRowKey");
+    byte[] row = Bytes.toBytes("myRow");
+    byte[] cf = Bytes.toBytes("myFamily");
+    byte[] cq = Bytes.toBytes("myColumn");
+    HTableDescriptor desc = new HTableDescriptor(table);
+    desc.addFamily(new HColumnDescriptor(cf));
+    HRegionInfo hri = new HRegionInfo(desc.getName(), null, null);
+    HRegion region = HRegion.createHRegion(hri, FSUtils.getRootDir(UTIL.getConfiguration()), UTIL.getConfiguration(), desc);
+    Put put = new Put(row);
+    put.add(cf, cq, cq);
+    region.put(put);
+    region.flushcache();
+    Scan scan = new Scan();
+    scan.addColumn(cf, cq);
+    WritableByteArrayComparable comparable = new TestFakeKeyInFilter();
+    Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, comparable);
+    scan.setFilter(filter);
+    RegionScanner scanner = region.getScanner(scan);
+    scanner.next(new ArrayList<KeyValue>());
+    scanner.close();
+    region.close();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From f733f7175806b5028e10b4754557eee8f8c70e26 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 16 Aug 2012 13:43:45 +0000
Subject: [PATCH 0399/1540] HBASE-6576 HBaseAdmin.createTable should wait until
 the table is enabled (Gregory)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373832 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 88 +++++++++++--------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index a112f0dac4f4..211a8ba3dd4d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotEnabledException;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.UnknownRegionException;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
@@ -398,57 +399,66 @@ public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
     }
     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
     int prevRegCount = 0;
+    boolean doneWithMetaScan = false;
     for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier;
       ++tries) {
-      // Wait for new table to come on-line
-      final AtomicInteger actualRegCount = new AtomicInteger(0);
-      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
-        @Override
-        public boolean processRow(Result rowResult) throws IOException {
-          HRegionInfo info = Writables.getHRegionInfoOrNull(
+      if (!doneWithMetaScan) {
+        // Wait for new table to come on-line
+        final AtomicInteger actualRegCount = new AtomicInteger(0);
+        MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
+          @Override
+          public boolean processRow(Result rowResult) throws IOException {
+            HRegionInfo info = Writables.getHRegionInfoOrNull(
               rowResult.getValue(HConstants.CATALOG_FAMILY,
-                  HConstants.REGIONINFO_QUALIFIER));
-          //If regioninfo is null, skip this row
-          if (null == info) {
+              HConstants.REGIONINFO_QUALIFIER));
+            //If regioninfo is null, skip this row
+            if (null == info) {
+              return true;
+            }
+            if (!(Bytes.equals(info.getTableName(), desc.getName()))) {
+              return false;
+            }
+            String hostAndPort = null;
+            byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
+              HConstants.SERVER_QUALIFIER);
+            // Make sure that regions are assigned to server
+            if (value != null && value.length > 0) {
+              hostAndPort = Bytes.toString(value);
+            }
+            if (!(info.isOffline() || info.isSplit()) && hostAndPort != null) {
+              actualRegCount.incrementAndGet();
+            }
             return true;
           }
-          if (!(Bytes.equals(info.getTableName(), desc.getName()))) {
-            return false;
+        };
+        MetaScanner.metaScan(conf, visitor, desc.getName());
+        if (actualRegCount.get() != numRegs) {
+          if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
+            throw new RegionOfflineException("Only " + actualRegCount.get() +
+              " of " + numRegs + " regions are online; retries exhausted.");
           }
-          String hostAndPort = null;
-          byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
-              HConstants.SERVER_QUALIFIER);
-          // Make sure that regions are assigned to server
-          if (value != null && value.length > 0) {
-            hostAndPort = Bytes.toString(value);
+          try { // Sleep
+            Thread.sleep(getPauseTime(tries));
+          } catch (InterruptedException e) {
+            throw new InterruptedIOException("Interrupted when opening" +
+              " regions; " + actualRegCount.get() + " of " + numRegs +
+              " regions processed so far");
           }
-          if (!(info.isOffline() || info.isSplit()) && hostAndPort != null) {
-            actualRegCount.incrementAndGet();
+          if (actualRegCount.get() > prevRegCount) { // Making progress
+            prevRegCount = actualRegCount.get();
+            tries = -1;
           }
-          return true;
-        }
-      };
-      MetaScanner.metaScan(conf, visitor, desc.getName());
-      if (actualRegCount.get() != numRegs) {
-        if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
-          throw new RegionOfflineException("Only " + actualRegCount.get() +
-            " of " + numRegs + " regions are online; retries exhausted.");
-        }
-        try { // Sleep
-          Thread.sleep(getPauseTime(tries));
-        } catch (InterruptedException e) {
-          throw new InterruptedIOException("Interrupted when opening" +
-              " regions; " + actualRegCount.get() + " of " + numRegs + 
-              " regions processed so far");
-        }
-        if (actualRegCount.get() > prevRegCount) { // Making progress
-          prevRegCount = actualRegCount.get();
-          tries = -1;
+        } else {
+          doneWithMetaScan = true;
         }
-      } else {
+      }
+      if (doneWithMetaScan && isTableEnabled(desc.getName())) {
         return;
       }
     }
+    throw new TableNotEnabledException(
+      "Retries exhausted while still waiting for table: "
+      + desc.getNameAsString() + " to be enabled");
   }
 
   /**

From dff315c1ac44d06980c25e843af219c59d0c0aad Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 16 Aug 2012 16:53:26 +0000
Subject: [PATCH 0400/1540] HBASE-6471 Performance regression caused by
 HBASE-4054

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373916 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HTablePool.java       | 45 +++++++++++++++----
 .../hadoop/hbase/rest/RegionsResource.java    | 30 ++++---------
 .../hadoop/hbase/client/TestHTablePool.java   | 16 -------
 3 files changed, 44 insertions(+), 47 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 9741dccc9348..398dc529fcf3 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.PoolMap;
@@ -173,11 +174,7 @@ public HTableInterface getTable(String tableName) {
     HTableInterface table = findOrCreateTable(tableName);
     // return a proxy table so when user closes the proxy, the actual table
     // will be returned to the pool
-    try {
-      return new PooledHTable(table);
-    } catch (IOException ioe) {
-      throw new RuntimeException(ioe);
-    }
+    return new PooledHTable(table);
   }
 
   /**
@@ -318,14 +315,13 @@ int getCurrentPoolSize(String tableName) {
   /**
    * A proxy class that implements HTableInterface.close method to return the
    * wrapped table back to the table pool
-   * 
+   *
    */
-  class PooledHTable extends HTable {
+  class PooledHTable implements HTableInterface {
 
     private HTableInterface table; // actual table implementation
 
-    public PooledHTable(HTableInterface table) throws IOException {
-      super(table.getConfiguration(), table.getTableName());
+    public PooledHTable(HTableInterface table) {
       this.table = table;
     }
 
@@ -372,6 +368,7 @@ public Result[] get(List<Get> gets) throws IOException {
     }
 
     @Override
+    @SuppressWarnings("deprecation")
     public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
       return table.getRowOrBefore(row, family);
     }
@@ -505,5 +502,35 @@ public String toString() {
     HTableInterface getWrappedTable() {
       return table;
     }
+
+    @Override
+    public void mutateRow(RowMutations rm) throws IOException {
+      table.mutateRow(rm);
+    }
+
+    @Override
+    public Result append(Append append) throws IOException {
+      return table.append(append);
+    }
+
+    @Override
+    public void setAutoFlush(boolean autoFlush) {
+      table.setAutoFlush(autoFlush);
+    }
+
+    @Override
+    public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+      table.setAutoFlush(autoFlush, clearBufferOnFail);
+    }
+
+    @Override
+    public long getWriteBufferSize() {
+      return table.getWriteBufferSize();
+    }
+
+    @Override
+    public void setWriteBufferSize(long writeBufferSize) throws IOException {
+      table.setWriteBufferSize(writeBufferSize);
+    }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index d527e3e8ca9c..6f5e0cc88480 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -21,7 +21,6 @@
 package org.apache.hadoop.hbase.rest;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.util.Map;
 
 import javax.ws.rs.GET;
@@ -37,13 +36,12 @@
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
+import org.apache.hadoop.hbase.client.MetaScanner;
 import org.apache.hadoop.hbase.rest.model.TableInfoModel;
 import org.apache.hadoop.hbase.rest.model.TableRegionModel;
+import org.apache.hadoop.hbase.util.Bytes;
 
 public class RegionsResource extends ResourceBase {
   private static final Log LOG = LogFactory.getLog(RegionsResource.class);
@@ -67,17 +65,6 @@ public RegionsResource(TableResource tableResource) throws IOException {
     this.tableResource = tableResource;
   }
 
-  private Map<HRegionInfo,HServerAddress> getTableRegions()
-      throws IOException {
-    HTablePool pool = servlet.getTablePool();
-    HTableInterface table = pool.getTable(tableResource.getName());
-    try {
-      return ((HTable)table).getRegionsInfo();
-    } finally {
-      table.close();
-    }
-  }
-
   @GET
   @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
   public Response get(final @Context UriInfo uriInfo) {
@@ -88,15 +75,14 @@ public Response get(final @Context UriInfo uriInfo) {
     try {
       String tableName = tableResource.getName();
       TableInfoModel model = new TableInfoModel(tableName);
-      Map<HRegionInfo,HServerAddress> regions = getTableRegions();
-      for (Map.Entry<HRegionInfo,HServerAddress> e: regions.entrySet()) {
+      Map<HRegionInfo,ServerName> regions = MetaScanner.allTableRegions(
+        servlet.getConfiguration(), Bytes.toBytes(tableName), false);
+      for (Map.Entry<HRegionInfo,ServerName> e: regions.entrySet()) {
         HRegionInfo hri = e.getKey();
-        HServerAddress addr = e.getValue();
-        InetSocketAddress sa = addr.getInetSocketAddress();
+        ServerName addr = e.getValue();
         model.add(
           new TableRegionModel(tableName, hri.getRegionId(),
-            hri.getStartKey(), hri.getEndKey(),
-            sa.getHostName() + ":" + Integer.valueOf(sa.getPort())));
+            hri.getStartKey(), hri.getEndKey(), addr.getHostAndPort()));
       }
       ResponseBuilder response = Response.ok(model);
       response.cacheControl(cacheControl);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
index 5ab918ea44fb..74f10e57195c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
@@ -183,22 +183,6 @@ public void testReturnDifferentTable() throws IOException {
         Assert.assertTrue("alien table rejected", true);
       }
     }
-
-    @Test
-    public void testClassCastException() {
-      //this test makes sure that client code that
-      //casts the table it got from pool to HTable won't break
-      HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(),
-        Integer.MAX_VALUE);
-      String tableName = Bytes.toString(TABLENAME);
-      try {
-        // get table and check if type is HTable
-        HTable table = (HTable) pool.getTable(tableName);
-        Assert.assertTrue("return type is HTable as expected", true);
-      } catch (ClassCastException e) {
-        Assert.fail("return type is not HTable");
-      }
-    }
   }
 
   @Category(MediumTests.class)

From 97fb637abc0bee23d48f59e102c42b4efd5daa68 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 16 Aug 2012 17:31:44 +0000
Subject: [PATCH 0401/1540] HBASE-6596 Revert HBASE-5022; it undoes HBC.create

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373935 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HBaseConfiguration.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 5f4bb8916788..5a211432b562 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -114,10 +114,13 @@ public static Configuration create() {
   /**
    * Creates a clone of passed configuration.
    * @param that Configuration to clone.
-   * @return a clone of passed configuration.
+   * @return a Configuration created with the hbase-*.xml files plus
+   * the given configuration.
    */
   public static Configuration create(final Configuration that) {
-    return new Configuration(that);
+    Configuration conf = create();
+    merge(conf, that);
+    return conf;
   }
 
   /**

From 4049ff816a03ab6bec7fb6c0ded56862da29533b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 16 Aug 2012 17:50:30 +0000
Subject: [PATCH 0402/1540] HBASE-6561 Gets/Puts with many columns send the
 RegionServer into an 'endless' loop

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373951 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  3 +
 .../hadoop/hbase/regionserver/MemStore.java   | 81 ++++++++++++-------
 .../hadoop/hbase/regionserver/Store.java      | 10 +--
 3 files changed, 61 insertions(+), 33 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 7f152b4bbb81..c2df587849d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -196,6 +196,9 @@ public enum OperationStatusCode {
   /** Parameter name for how often a region should should perform a major compaction */
   public static final String MAJOR_COMPACTION_PERIOD = "hbase.hregion.majorcompaction";
 
+  /** Parameter name for the maximum batch of KVs to be used in flushes and compactions */
+  public static final String COMPACTION_KV_MAX = "hbase.hstore.compaction.kv.max";
+
   /** Parameter name for HBase instance root directory */
   public static final String HBASE_DIR = "hbase.rootdir";
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 03ed7f10079c..9ee672044533 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -672,6 +672,10 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
     private KeyValue kvsetNextRow = null;
     private KeyValue snapshotNextRow = null;
 
+    // last iterated KVs for kvset and snapshot (to restore iterator state after reseek)
+    private KeyValue kvsetItRow = null;
+    private KeyValue snapshotItRow = null;
+    
     // iterator based scanning.
     private Iterator<KeyValue> kvsetIt;
     private Iterator<KeyValue> snapshotIt;
@@ -680,10 +684,6 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
     volatile KeyValueSkipListSet kvsetAtCreation;
     volatile KeyValueSkipListSet snapshotAtCreation;
 
-    // Sub lists on which we're iterating
-    private SortedSet<KeyValue> kvTail;
-    private SortedSet<KeyValue> snapshotTail;
-
     // the pre-calculated KeyValue to be returned by peek() or next()
     private KeyValue theNext;
 
@@ -715,17 +715,29 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
       snapshotAtCreation = snapshot;
     }
 
-    protected KeyValue getNext(Iterator<KeyValue> it) {
+    private KeyValue getNext(Iterator<KeyValue> it) {
       long readPoint = MultiVersionConsistencyControl.getThreadReadPoint();
 
-      while (it.hasNext()) {
-        KeyValue v = it.next();
-        if (v.getMemstoreTS() <= readPoint) {
-          return v;
+      KeyValue v = null;
+      try {
+        while (it.hasNext()) {
+          v = it.next();
+          if (v.getMemstoreTS() <= readPoint) {
+            return v;
+          }
         }
-      }
 
-      return null;
+        return null;
+      } finally {
+        if (v != null) {
+          // in all cases, remember the last KV iterated to
+          if (it == snapshotIt) {
+            snapshotItRow = v;
+          } else {
+            kvsetItRow = v;
+          }
+        }
+      }
     }
 
     /**
@@ -744,8 +756,10 @@ public synchronized boolean seek(KeyValue key) {
 
       // kvset and snapshot will never be null.
       // if tailSet can't find anything, SortedSet is empty (not null).
-      kvTail = kvsetAtCreation.tailSet(key);
-      snapshotTail = snapshotAtCreation.tailSet(key);
+      kvsetIt = kvsetAtCreation.tailSet(key).iterator();
+      snapshotIt = snapshotAtCreation.tailSet(key).iterator();
+      kvsetItRow = null;
+      snapshotItRow = null;
 
       return seekInSubLists(key);
     }
@@ -755,9 +769,6 @@ public synchronized boolean seek(KeyValue key) {
      * (Re)initialize the iterators after a seek or a reseek.
      */
     private synchronized boolean seekInSubLists(KeyValue key){
-      kvsetIt = kvTail.iterator();
-      snapshotIt = snapshotTail.iterator();
-
       kvsetNextRow = getNext(kvsetIt);
       snapshotNextRow = getNext(snapshotIt);
 
@@ -777,25 +788,20 @@ private synchronized boolean seekInSubLists(KeyValue key){
     @Override
     public synchronized boolean reseek(KeyValue key) {
       /*
-      See HBASE-4195 & HBASE-3855 for the background on this implementation.
+      See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation.
       This code is executed concurrently with flush and puts, without locks.
       Two points must be known when working on this code:
       1) It's not possible to use the 'kvTail' and 'snapshot'
        variables, as they are modified during a flush.
-      2) The ideal implementation for performances would use the sub skip list
+      2) The ideal implementation for performance would use the sub skip list
        implicitly pointed by the iterators 'kvsetIt' and
        'snapshotIt'. Unfortunately the Java API does not offer a method to
-       get it. So we're using the skip list that we kept when we created
-       the iterators. As these iterators could have been moved forward after
-       their creation, we're doing a kind of rewind here. It has a small
-       performance impact (we're using a wider list than necessary), and we
-       could see values that were not here when we read the list the first
-       time. We expect that the new values will be skipped by the test on
-       readpoint performed in the next() function.
+       get it. So we remember the last keys we iterated to and restore
+       the reseeked set to at least that point.
        */
 
-      kvTail = kvTail.tailSet(key);
-      snapshotTail = snapshotTail.tailSet(key);
+      kvsetIt = kvsetAtCreation.tailSet(getHighest(key, kvsetItRow)).iterator();
+      snapshotIt = snapshotAtCreation.tailSet(getHighest(key, snapshotItRow)).iterator();
 
       return seekInSubLists(key);
     }
@@ -836,7 +842,7 @@ public synchronized KeyValue next() {
      * This uses comparator.compare() to compare the KeyValue using the memstore
      * comparator.
      */
-    protected KeyValue getLowest(KeyValue first, KeyValue second) {
+    private KeyValue getLowest(KeyValue first, KeyValue second) {
       if (first == null && second == null) {
         return null;
       }
@@ -847,12 +853,31 @@ protected KeyValue getLowest(KeyValue first, KeyValue second) {
       return (first != null ? first : second);
     }
 
+    /*
+     * Returns the higher of the two key values, or null if they are both null.
+     * This uses comparator.compare() to compare the KeyValue using the memstore
+     * comparator.
+     */
+    private KeyValue getHighest(KeyValue first, KeyValue second) {
+      if (first == null && second == null) {
+        return null;
+      }
+      if (first != null && second != null) {
+        int compare = comparator.compare(first, second);
+        return (compare > 0 ? first : second);
+      }
+      return (first != null ? first : second);
+    }
+
     public synchronized void close() {
       this.kvsetNextRow = null;
       this.snapshotNextRow = null;
 
       this.kvsetIt = null;
       this.snapshotIt = null;
+
+      this.kvsetItRow = null;
+      this.snapshotItRow = null;
     }
 
     /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index aed956d88938..0c9ace525280 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -243,7 +243,7 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       this.region.memstoreFlushSize);
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
-    this.compactionKVMax = conf.getInt("hbase.hstore.compaction.kv.max", 10);
+    this.compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
 
     this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify",
         false);
@@ -706,9 +706,9 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
     if (scanner == null) {
       Scan scan = new Scan();
       scan.setMaxVersions(scanInfo.getMaxVersions());
-      scanner = new StoreScanner(this, scanInfo, scan, Collections.singletonList(new CollectionBackedScanner(
-          set, this.comparator)), ScanType.MINOR_COMPACT, this.region.getSmallestReadPoint(),
-          HConstants.OLDEST_TIMESTAMP);
+      scanner = new StoreScanner(this, scanInfo, scan,
+          Collections.singletonList(memstoreScanner), ScanType.MINOR_COMPACT,
+          this.region.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
     }
     if (getHRegion().getCoprocessorHost() != null) {
       InternalScanner cpScanner =
@@ -733,7 +733,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
           List<KeyValue> kvs = new ArrayList<KeyValue>();
           boolean hasMore;
           do {
-            hasMore = scanner.next(kvs);
+            hasMore = scanner.next(kvs, this.compactionKVMax);
             if (!kvs.isEmpty()) {
               for (KeyValue kv : kvs) {
                 // If we know that this KV is going to be included always, then let us

From f4f06e204d7fd173c0fbd0f1df4637c19d4a557a Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 16 Aug 2012 18:45:41 +0000
Subject: [PATCH 0403/1540] HBASE-6565. Coprocessor exec result Map is not
 thread safe (Yuan Kang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1373975 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HTable.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 24ef4e7597a5..f7927c00e66d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -28,6 +28,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Collections;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutorService;
@@ -1238,8 +1239,8 @@ public <T extends CoprocessorProtocol, R> Map<byte[],R> coprocessorExec(
       Batch.Call<T,R> callable)
       throws IOException, Throwable {
 
-    final Map<byte[],R> results = new TreeMap<byte[],R>(
-        Bytes.BYTES_COMPARATOR);
+    final Map<byte[],R> results =  Collections.synchronizedMap(new TreeMap<byte[],R>(
+        Bytes.BYTES_COMPARATOR));
     coprocessorExec(protocol, startKey, endKey, callable,
         new Batch.Callback<R>(){
       public void update(byte[] region, byte[] row, R value) {

From 912d6542a765cbdcd52bcde65e1632740e76cf97 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 16 Aug 2012 19:46:41 +0000
Subject: [PATCH 0404/1540] HBASE-6596 Revert HBASE-5022; it undoes HBC.create
 -- ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1374013 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 07b46f903329..0f61539b732d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -180,7 +180,7 @@ void loadTableCoprocessors(final Configuration conf) {
             }
             if (cfgSpec != null) {
               cfgSpec = cfgSpec.substring(cfgSpec.indexOf('|') + 1);
-              Configuration newConf = HBaseConfiguration.create(conf);
+              Configuration newConf = new Configuration(conf);
               Matcher m = HConstants.CP_HTD_ATTR_VALUE_PARAM_PATTERN.matcher(cfgSpec);
               while (m.find()) {
                 newConf.set(m.group(1), m.group(2));

From 1bfa7501eac3cd04c406689b0a77be7750680479 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 16 Aug 2012 21:10:19 +0000
Subject: [PATCH 0405/1540] HBASE-6593 TestAdmin times out sometimes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1374063 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/TestAdmin.java     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 37d9946dc590..d09a3f3f196f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -871,16 +871,19 @@ void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts,
       numVersions, blockSize);
     try {
       int rowCount = 0;
+      byte[] q = new byte[0];
 
       // insert rows into column families. The number of rows that have values
       // in a specific column family is decided by rowCounts[familyIndex]
       for (int index = 0; index < familyNames.length; index++) {
+        ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
         for (int i = 0; i < rowCounts[index]; i++) {
           byte[] k = Bytes.toBytes(i);
           Put put = new Put(k);
-          put.add(familyNames[index], new byte[0], k);
-          table.put(put);
+          put.add(familyNames[index], q, k);
+          puts.add(put);
         }
+        table.put(puts);
 
         if ( rowCount < rowCounts[index] ) {
           rowCount = rowCounts[index];

From 3c3c5369e95df00c261c52584e35a97790cf29f9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 16 Aug 2012 22:55:25 +0000
Subject: [PATCH 0406/1540] HBASE-6596 Revert HBASE-5022; it undoes HBC.create;
 ADDENDUM 2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1374098 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 5a211432b562..590e774f9c9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -112,7 +112,6 @@ public static Configuration create() {
   }
 
   /**
-   * Creates a clone of passed configuration.
    * @param that Configuration to clone.
    * @return a Configuration created with the hbase-*.xml files plus
    * the given configuration.

From 9a9525c0e5b0ebd83fde962e7680db72fa9bd4b9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 17 Aug 2012 17:27:12 +0000
Subject: [PATCH 0407/1540] HBASE-6538 Remove copy_table.rb script

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1374372 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/copy_table.rb | 166 ----------------------------------------------
 1 file changed, 166 deletions(-)
 delete mode 100644 bin/copy_table.rb

diff --git a/bin/copy_table.rb b/bin/copy_table.rb
deleted file mode 100644
index ad6e70321b82..000000000000
--- a/bin/copy_table.rb
+++ /dev/null
@@ -1,166 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Script that copies table in hbase.  As written, will not work for rare
-# case where there is more than one region in .META. table.  Does the
-# update of the hbase .META. and copies the directories in filesystem.  
-# HBase MUST be shutdown when you run this script.
-#
-# To see usage for this script, run: 
-#
-#  ${HBASE_HOME}/bin/hbase org.jruby.Main rename_table.rb
-#
-include Java
-import org.apache.hadoop.hbase.util.MetaUtils
-import org.apache.hadoop.hbase.util.FSUtils
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.hadoop.hbase.util.Writables
-import org.apache.hadoop.hbase.HConstants
-import org.apache.hadoop.hbase.HBaseConfiguration
-import org.apache.hadoop.hbase.HStoreKey
-import org.apache.hadoop.hbase.HRegionInfo
-import org.apache.hadoop.hbase.HTableDescriptor
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.hadoop.hbase.regionserver.HLogEdit
-import org.apache.hadoop.hbase.regionserver.HRegion
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.FileUtil
-import org.apache.commons.logging.Log
-import org.apache.commons.logging.LogFactory
-import java.util.TreeMap
-
-# Name of this script
-NAME = "copy_table"
-
-# Print usage for this script
-def usage
-  puts 'Usage: %s.rb <OLD_NAME> <NEW_NAME>' % NAME
-  exit!
-end
-
-# Passed 'dir' exists and is a directory else exception
-def isDirExists(fs, dir)
-  raise IOError.new("Does not exit: " + dir.toString()) unless fs.exists(dir)
-  raise IOError.new("Not a directory: " + dir.toString()) unless fs.isDirectory(dir)
-end
-
-# Returns true if the region belongs to passed table
-def isTableRegion(tableName, hri)
-  return Bytes.equals(hri.getTableDesc().getName(), tableName)
-end
-
-# Create new HRI based off passed 'oldHRI'
-def createHRI(tableName, oldHRI)
-  htd = oldHRI.getTableDesc()
-  newHtd = HTableDescriptor.new(tableName)
-  for family in htd.getFamilies()
-    newHtd.addFamily(family)
-  end
-  return HRegionInfo.new(newHtd, oldHRI.getStartKey(), oldHRI.getEndKey(),
-    oldHRI.isSplit())
-end
-
-# Check arguments
-if ARGV.size != 2
-  usage
-end
-
-# Check good table names were passed.
-oldTableName = HTableDescriptor.isLegalTableName(ARGV[0].to_java_bytes)
-newTableName = HTableDescriptor.isLegalTableName(ARGV[1].to_java_bytes)
-
-# Get configuration to use.
-c = HBaseConfiguration.new()
-
-# Set hadoop filesystem configuration using the hbase.rootdir.
-# Otherwise, we'll always use localhost though the hbase.rootdir
-# might be pointing at hdfs location.
-c.set("fs.default.name", c.get(HConstants::HBASE_DIR))
-fs = FileSystem.get(c)
-
-# If new table directory does not exit, create it.  Keep going if already
-# exists because maybe we are rerunning script because it failed first
-# time.
-rootdir = FSUtils.getRootDir(c)
-oldTableDir = Path.new(rootdir, Path.new(Bytes.toString(oldTableName)))
-isDirExists(fs, oldTableDir)
-newTableDir = Path.new(rootdir, Bytes.toString(newTableName))
-if !fs.exists(newTableDir)
-  fs.mkdirs(newTableDir)
-end
-
-# Get a logger and a metautils instance.
-LOG = LogFactory.getLog(NAME)
-utils = MetaUtils.new(c)
-
-# Start.  Get all meta rows.
-begin
-  # Get list of all .META. regions that contain old table name
-  metas = utils.getMETARows(oldTableName)
-  index = 0
-  for meta in metas
-    # For each row we find, move its region from old to new table.
-    # Need to update the encoded name in the hri as we move.
-    # After move, delete old entry and create a new.
-    LOG.info("Scanning " + meta.getRegionNameAsString())
-    metaRegion = utils.getMetaRegion(meta)
-    scanner = metaRegion.getScanner(HConstants::COL_REGIONINFO_ARRAY, oldTableName,
-      HConstants::LATEST_TIMESTAMP, nil) 
-    begin
-      key = HStoreKey.new()
-      value = TreeMap.new(Bytes.BYTES_COMPARATOR)
-      while scanner.next(key, value)
-        index = index + 1
-        keyStr = key.toString()
-        oldHRI = Writables.getHRegionInfo(value.get(HConstants::COL_REGIONINFO))
-        if !oldHRI
-          raise IOError.new(index.to_s + " HRegionInfo is null for " + keyStr)
-        end
-        unless isTableRegion(oldTableName, oldHRI)
-          # If here, we passed out the table.  Break.
-          break
-        end
-        oldRDir = Path.new(oldTableDir, Path.new(oldHRI.getEncodedName().to_s))
-        if !fs.exists(oldRDir)
-          LOG.warn(oldRDir.toString() + " does not exist -- region " +
-            oldHRI.getRegionNameAsString())
-        else
-           # Now make a new HRegionInfo to add to .META. for the new region.
-          newHRI = createHRI(newTableName, oldHRI)
-          newRDir = Path.new(newTableDir, Path.new(newHRI.getEncodedName().to_s))
-          # Move the region in filesystem
-          LOG.info("Copying " + oldRDir.toString() + " as " + newRDir.toString())
-          FileUtil.copy(fs, oldRDir, fs, newRDir, false, true, c)
-          # Create 'new' region
-          newR = HRegion.new(rootdir, utils.getLog(), fs, c, newHRI, nil)
-          # Add new row. NOTE: Presumption is that only one .META. region. If not,
-          # need to do the work to figure proper region to add this new region to.
-          LOG.info("Adding to meta: " + newR.toString())
-          HRegion.addRegionToMETA(metaRegion, newR)
-          LOG.info("Done copying: " + Bytes.toString(key.getRow()))
-        end
-        # Need to clear value else we keep appending values.
-        value.clear()
-      end
-    ensure
-      scanner.close()
-    end
-  end
-ensure
-  utils.shutdown()
-end

From 8d19299c1426ceea640d0bdb190bee7e35d41a01 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 17 Aug 2012 19:03:05 +0000
Subject: [PATCH 0408/1540] HBASE-6602 Region Server Dynamic Metrics can cause
 high cpu usage.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1374398 13f79535-47bb-0310-9956-ffa450edef68
---
 .../metrics/RegionServerDynamicMetrics.java   | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
index fd0efa4f2be6..44a6fadef60a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
@@ -62,6 +62,7 @@ public class RegionServerDynamicMetrics implements Updater {
     LogFactory.getLog(RegionServerDynamicStatistics.class);
   
   private boolean reflectionInitialized = false;
+  private boolean needsUpdateMessage = false;
   private Field recordMetricMapField;
   private Field registryMetricMapField;
 
@@ -100,14 +101,7 @@ public synchronized void setNumericMetric(String name, long amt) {
     MetricsLongValue m = (MetricsLongValue)registry.get(name);
     if (m == null) {
       m = new MetricsLongValue(name, this.registry);
-      try {
-        if (updateMbeanInfoIfMetricsListChanged != null) {
-          updateMbeanInfoIfMetricsListChanged.invoke(this.rsDynamicStatistics,
-              new Object[]{});
-        }
-      } catch (Exception e) {
-        LOG.error(e);
-      }
+      this.needsUpdateMessage = true;
     }
     m.set(amt);
   }
@@ -119,14 +113,7 @@ public synchronized void incrTimeVaryingMetric(
     MetricsTimeVaryingRate m = (MetricsTimeVaryingRate)registry.get(name);
     if (m == null) {
       m = new MetricsTimeVaryingRate(name, this.registry);
-      try {
-        if (updateMbeanInfoIfMetricsListChanged != null) {
-          updateMbeanInfoIfMetricsListChanged.invoke(this.rsDynamicStatistics,
-              new Object[]{});
-        }
-      } catch (Exception e) {
-        LOG.error(e);
-      }
+      this.needsUpdateMessage = true;
     }
     if (numOps > 0) {
       m.inc(numOps, amt);
@@ -139,7 +126,7 @@ public synchronized void incrTimeVaryingMetric(
    */
   @SuppressWarnings("rawtypes")
   public void clear() {
-    
+    this.needsUpdateMessage = true;
     // If this is the first clear use reflection to get the two maps that hold copies of our 
     // metrics on the hadoop metrics side. We have to use reflection because there is not 
     // remove metrics on the hadoop side. If we can't get them then clearing old metrics 
@@ -210,6 +197,21 @@ public void doUpdates(MetricsContext context) {
           value.getSecond().getAndSet(0));
     }
 
+    // If there are new metrics sending this message to jmx tells it to update everything.
+    // This is not ideal we should just move to metrics2 that has full support for dynamic metrics.
+    if (needsUpdateMessage) {
+      try {
+        if (updateMbeanInfoIfMetricsListChanged != null) {
+          updateMbeanInfoIfMetricsListChanged.invoke(this.rsDynamicStatistics,
+              new Object[]{});
+        }
+      } catch (Exception e) {
+        LOG.error(e);
+      }
+      needsUpdateMessage = false;
+    }
+
+
     synchronized (registry) {
       // Iterate through the registry to propagate the different rpc metrics.
       for (String metricName : registry.getKeyList() ) {

From c30a6a2aba7a877995a485faf04d06512bf475fe Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 20 Aug 2012 18:18:41 +0000
Subject: [PATCH 0409/1540] HBASE-6608 Fix for HBASE-6160, META entries from
 daughters can be deleted before parent entries, shouldn't compare
 HRegionInfo's (Enis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375158 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 10 ++++++----
 .../apache/hadoop/hbase/master/TestCatalogJanitor.java |  1 +
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index c1eb02a339ea..c3fcb99422eb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -139,14 +139,16 @@ int scan() throws IOException {
 
     // Now work on our list of found parents. See if any we can clean up.
     int cleaned = 0;
-    HashSet<HRegionInfo> parentNotCleaned = new HashSet<HRegionInfo>(); //regions whose parents are still around
+    HashSet<String> parentNotCleaned = new HashSet<String>(); //regions whose parents are still around
     for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
-      if (!parentNotCleaned.contains(e.getKey()) && cleanParent(e.getKey(), e.getValue())) {
+      if (!parentNotCleaned.contains(e.getKey().getEncodedName()) && cleanParent(e.getKey(), e.getValue())) {
         cleaned++;
       } else {
         // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160)
-        parentNotCleaned.add(getDaughterRegionInfo(e.getValue(), HConstants.SPLITA_QUALIFIER));
-        parentNotCleaned.add(getDaughterRegionInfo(e.getValue(), HConstants.SPLITB_QUALIFIER));
+        parentNotCleaned.add(getDaughterRegionInfo(
+              e.getValue(), HConstants.SPLITA_QUALIFIER).getEncodedName());
+        parentNotCleaned.add(getDaughterRegionInfo(
+              e.getValue(), HConstants.SPLITB_QUALIFIER).getEncodedName());
       }
     }
     if (cleaned != 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 5d76a99eecbf..47bc119ee651 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -519,6 +519,7 @@ public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
     final Map<HRegionInfo, Result> splitParents =
         new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
     splitParents.put(parent, makeResultFromHRegionInfo(parent, splita, splitb));
+    splita.setOffline(true);//simulate that splita goes offline when it is split
     splitParents.put(splita, makeResultFromHRegionInfo(splita, splitaa, splitab));
 
     CatalogJanitor janitor = spy(new CatalogJanitor(server, services));

From 0a693142a07a2284ca6f241df9d0f2f612b5f166 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 20 Aug 2012 20:23:26 +0000
Subject: [PATCH 0410/1540] HBASE-6378 the javadoc of setEnabledTable maybe not
 describe accurately

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375202 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index b925bb9086c9..a5924f7e6710 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -350,8 +350,8 @@ public void setDeletedTable(final String tableName)
   }
   
   /**
-   * Sets the ENABLED state in the cache and deletes the zookeeper node. Fails
-   * silently if the node is not in enabled in zookeeper
+   * Sets the ENABLED state in the cache and creates or force updates a node to
+   * ENABLED state for the specified table
    * 
    * @param tableName
    * @throws KeeperException

From f4fe457f116c68320439bd2386273f6c818f1148 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 20 Aug 2012 20:28:46 +0000
Subject: [PATCH 0411/1540] HBASE-6503 HBase Shell Documentation For DROP Is
 Outdated

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375206 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/shell/commands/drop.rb | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/main/ruby/shell/commands/drop.rb b/src/main/ruby/shell/commands/drop.rb
index 181b83500f64..40831d2fa06e 100644
--- a/src/main/ruby/shell/commands/drop.rb
+++ b/src/main/ruby/shell/commands/drop.rb
@@ -23,10 +23,7 @@ module Commands
     class Drop < Command
       def help
         return <<-EOF
-Drop the named table. Table must first be disabled. If table has
-more than one region, run a major compaction on .META.:
-
-  hbase> major_compact ".META."
+Drop the named table. Table must first be disabled: e.g. "hbase> drop 't1'"
 EOF
       end
 

From 9a3b6583316f29c47dc37176a81c3e187588cb8e Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 20 Aug 2012 21:30:40 +0000
Subject: [PATCH 0412/1540] HBASE-5714 Add write permissions check before any
 hbck run that modifies hdfs (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375228 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/FSUtils.java | 39 ++++++++++++++++
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 46 +++++++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index c7ca432386ef..088a4ba6c490 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
@@ -49,6 +50,8 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 
@@ -1027,6 +1030,42 @@ public static boolean delete(final FileSystem fs, final Path path, final boolean
     return fs.delete(path, recursive);
   }
 
+  /**
+   * Throw an exception if an action is not permitted by a user on a file.
+   * 
+   * @param ugi
+   *          the user
+   * @param file
+   *          the file
+   * @param action
+   *          the action
+   */
+  public static void checkAccess(UserGroupInformation ugi, FileStatus file,
+      FsAction action) throws AccessControlException {
+    if (ugi.getUserName().equals(file.getOwner())) {
+      if (file.getPermission().getUserAction().implies(action)) {
+        return;
+      }
+    } else if (contains(ugi.getGroupNames(), file.getGroup())) {
+      if (file.getPermission().getGroupAction().implies(action)) {
+        return;
+      }
+    } else if (file.getPermission().getOtherAction().implies(action)) {
+      return;
+    }
+    throw new AccessControlException("Permission denied:" + " action=" + action
+        + " path=" + file.getPath() + " user=" + ugi.getUserName());
+  }
+
+  private static boolean contains(String[] groups, String user) {
+    for (String group : groups) {
+      if (group.equals(user)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Calls fs.exists(). Checks if the specified path exists
    * 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 36939a62a974..ee2dfc555fb2 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -47,6 +47,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -78,12 +79,15 @@
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.base.Joiner;
@@ -182,6 +186,7 @@ public class HBaseFsck {
   private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
   private boolean checkMetaOnly = false;
+  private boolean ignorePreCheckPermission = false; // if pre-check permission
 
   /*********
    * State
@@ -1192,6 +1197,27 @@ private void checkAndFixConsistency()
     }
   }
 
+  private void preCheckPermission() throws IOException, AccessControlException {
+    if (shouldIgnorePreCheckPermission()) {
+      return;
+    }
+
+    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = hbaseDir.getFileSystem(conf);
+    UserGroupInformation ugi = User.getCurrent().getUGI();
+    FileStatus[] files = fs.listStatus(hbaseDir);
+    for (FileStatus file : files) {
+      try {
+        FSUtils.checkAccess(ugi, file, FsAction.WRITE);
+      } catch (AccessControlException ace) {
+        LOG.warn("Got AccessControlException when preCheckPermission ", ace);
+        System.err.println("Current user " + ugi.getUserName() + " does not have write perms to " + file.getPath()
+            + ". Please rerun hbck as hdfs user " + file.getOwner());
+        throw new AccessControlException(ace);
+      }
+    }
+  }
+
   /**
    * Deletes region from meta table
    */
@@ -3017,6 +3043,14 @@ boolean shouldFixSplitParents() {
     return fixSplitParents;
   }
 
+  public boolean shouldIgnorePreCheckPermission() {
+    return ignorePreCheckPermission;
+  }
+
+  public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
+    this.ignorePreCheckPermission = ignorePreCheckPermission;
+  }
+
   /**
    * @param mm maximum number of regions to merge into a single region.
    */
@@ -3091,6 +3125,7 @@ protected static void printUsageAndExit() {
     System.err.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
     System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
     System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
+    System.err.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
     System.err.println("");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
@@ -3174,6 +3209,8 @@ public static void main(String[] args) throws Exception {
         fsck.setSidelineBigOverlaps(true);
       } else if (cmd.equals("-fixSplitParents")) {
         fsck.setFixSplitParents(true);
+      } else if (cmd.equals("-ignorePreCheckPermission")) {
+        fsck.setIgnorePreCheckPermission(true);
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
@@ -3232,6 +3269,15 @@ public static void main(String[] args) throws Exception {
         System.out.println("Allow checking/fixes for table: " + cmd);
       }
     }
+
+    // pre-check current user has FS write permission or not
+    try {
+      fsck.preCheckPermission();
+    } catch (AccessControlException ace) {
+      Runtime.getRuntime().exit(-1);
+    } catch (IOException ioe) {
+      Runtime.getRuntime().exit(-1);
+    }
     // do the real work of fsck
     fsck.connect();
     int code = fsck.onlineHbck();

From 0346317253791b7c688bb9fa7932206ffbcbfc01 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 20 Aug 2012 23:38:50 +0000
Subject: [PATCH 0413/1540] HBASE-6603 RegionMetricsStorage.incrNumericMetric
 is called too often (M. Chen and Lars H)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375318 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/StoreScanner.java      | 139 +++++++++---------
 1 file changed, 72 insertions(+), 67 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 26afb223fb3b..a1cf76533705 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -340,82 +340,87 @@ public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOE
     KeyValue.KVComparator comparator =
         store != null ? store.getComparator() : null;
 
-    LOOP: while((kv = this.heap.peek()) != null) {
-      // Check that the heap gives us KVs in an increasing order.
-      assert prevKV == null || comparator == null || comparator.compare(prevKV, kv) <= 0 :
-        "Key " + prevKV + " followed by a " + "smaller key " + kv + " in cf " + store;
-      prevKV = kv;
-      ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
-      switch(qcode) {
-        case INCLUDE:
-        case INCLUDE_AND_SEEK_NEXT_ROW:
-        case INCLUDE_AND_SEEK_NEXT_COL:
-
-          Filter f = matcher.getFilter();
-          results.add(f == null ? kv : f.transform(kv));
-
-          if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) {
+    long cumulativeMetric = 0;
+    try {
+      LOOP: while((kv = this.heap.peek()) != null) {
+        // Check that the heap gives us KVs in an increasing order.
+        assert prevKV == null || comparator == null || comparator.compare(prevKV, kv) <= 0 :
+          "Key " + prevKV + " followed by a " + "smaller key " + kv + " in cf " + store;
+        prevKV = kv;
+        ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
+        switch(qcode) {
+          case INCLUDE:
+          case INCLUDE_AND_SEEK_NEXT_ROW:
+          case INCLUDE_AND_SEEK_NEXT_COL:
+
+            Filter f = matcher.getFilter();
+            results.add(f == null ? kv : f.transform(kv));
+
+            if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) {
+              if (!matcher.moreRowsMayExistAfter(kv)) {
+                outResult.addAll(results);
+                return false;
+              }
+              reseek(matcher.getKeyForNextRow(kv));
+            } else if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) {
+              reseek(matcher.getKeyForNextColumn(kv));
+            } else {
+              this.heap.next();
+            }
+
+            cumulativeMetric += kv.getLength();
+            if (limit > 0 && (results.size() == limit)) {
+              break LOOP;
+            }
+            continue;
+
+          case DONE:
+            // copy jazz
+            outResult.addAll(results);
+            return true;
+
+          case DONE_SCAN:
+            close();
+
+            // copy jazz
+            outResult.addAll(results);
+
+            return false;
+
+          case SEEK_NEXT_ROW:
+            // This is just a relatively simple end of scan fix, to short-cut end
+            // us if there is an endKey in the scan.
             if (!matcher.moreRowsMayExistAfter(kv)) {
               outResult.addAll(results);
               return false;
             }
-            reseek(matcher.getKeyForNextRow(kv));
-          } else if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) {
-            reseek(matcher.getKeyForNextColumn(kv));
-          } else {
-            this.heap.next();
-          }
 
-          RegionMetricsStorage.incrNumericMetric(metricNameGetSize, kv.getLength());
-          if (limit > 0 && (results.size() == limit)) {
-            break LOOP;
-          }
-          continue;
-
-        case DONE:
-          // copy jazz
-          outResult.addAll(results);
-          return true;
-
-        case DONE_SCAN:
-          close();
+            reseek(matcher.getKeyForNextRow(kv));
+            break;
 
-          // copy jazz
-          outResult.addAll(results);
+          case SEEK_NEXT_COL:
+            reseek(matcher.getKeyForNextColumn(kv));
+            break;
 
-          return false;
+          case SKIP:
+            this.heap.next();
+            break;
+
+          case SEEK_NEXT_USING_HINT:
+            KeyValue nextKV = matcher.getNextKeyHint(kv);
+            if (nextKV != null) {
+              reseek(nextKV);
+            } else {
+              heap.next();
+            }
+            break;
 
-        case SEEK_NEXT_ROW:
-          // This is just a relatively simple end of scan fix, to short-cut end
-          // us if there is an endKey in the scan.
-          if (!matcher.moreRowsMayExistAfter(kv)) {
-            outResult.addAll(results);
-            return false;
-          }
-
-          reseek(matcher.getKeyForNextRow(kv));
-          break;
-
-        case SEEK_NEXT_COL:
-          reseek(matcher.getKeyForNextColumn(kv));
-          break;
-
-        case SKIP:
-          this.heap.next();
-          break;
-
-        case SEEK_NEXT_USING_HINT:
-          KeyValue nextKV = matcher.getNextKeyHint(kv);
-          if (nextKV != null) {
-            reseek(nextKV);
-          } else {
-            heap.next();
-          }
-          break;
-
-        default:
-          throw new RuntimeException("UNEXPECTED");
+          default:
+            throw new RuntimeException("UNEXPECTED");
+        }
       }
+    } finally {
+      RegionMetricsStorage.incrNumericMetric(metricNameGetSize, cumulativeMetric);
     }
 
     if (!results.isEmpty()) {

From 8bb11c64d9796c1e2dd215e5046b97344295d867 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 21 Aug 2012 17:06:07 +0000
Subject: [PATCH 0414/1540] HBASE-6621 Reduce calls to Bytes.toInt

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375665 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/KeyValue.java    | 15 +++++++++++++++
 .../hadoop/hbase/io/hfile/HFileReaderV2.java      |  4 +++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 2f8c724ee209..bd87c34fb542 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -281,6 +281,21 @@ public KeyValue(final byte [] bytes, final int offset, final int length) {
     this.length = length;
   }
 
+  /**
+   * Creates a KeyValue from the specified byte array, starting at offset,
+   * for length <code>length</code>, and a known <code>keyLength</code>.
+   * @param bytes byte array
+   * @param offset offset to start of the KeyValue
+   * @param length length of the KeyValue
+   * @param keyLength length of the key portion of the KeyValue
+   */
+  public KeyValue(final byte [] bytes, final int offset, final int length, final int keyLength) {
+    this.bytes = bytes;
+    this.offset = offset;
+    this.length = length;
+    this.keyLength = keyLength;
+  }
+
   /** Constructors that build a new backing byte array from fields */
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 1f777cbcad64..3be58d326248 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -581,7 +581,9 @@ public KeyValue getKeyValue() {
         return null;
 
       KeyValue ret = new KeyValue(blockBuffer.array(),
-          blockBuffer.arrayOffset() + blockBuffer.position());
+          blockBuffer.arrayOffset() + blockBuffer.position(),
+          KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen,
+          currKeyLen);
       if (this.reader.shouldIncludeMemstoreTS()) {
         ret.setMemstoreTS(currMemstoreTS);
       }

From 8a723543110ae0007c2086ac034984d8918a0789 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 21 Aug 2012 23:01:24 +0000
Subject: [PATCH 0415/1540] HBASE-6631 TestHMasterRPCException in 0.92 failed
 twice on socket timeout

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375845 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/TestHMasterRPCException.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
index 9ff83c50e0f4..39b571f7fba6 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
@@ -49,7 +49,7 @@ public void testRPCException() throws Exception {
     InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort());
     try {
       HMasterInterface inf = (HMasterInterface) HBaseRPC.getProxy(
-        HMasterInterface.class,  HMasterInterface.VERSION, isa, conf, 100);
+        HMasterInterface.class,  HMasterInterface.VERSION, isa, conf, 100 x 10);
       inf.isMasterRunning();
       fail();
     } catch (RemoteException ex) {

From c665d4adbd6cec7dd9c9ab5ea6e53d4c938ea6f2 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 22 Aug 2012 01:44:03 +0000
Subject: [PATCH 0416/1540] HBASE-6631 TestHMasterRPCException in 0.92 failed
 twice on socket timeout; ADDENDUM -- FIX COMPILATION ERROR

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375871 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/TestHMasterRPCException.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
index 39b571f7fba6..c27cbf670fc6 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
@@ -49,7 +49,7 @@ public void testRPCException() throws Exception {
     InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort());
     try {
       HMasterInterface inf = (HMasterInterface) HBaseRPC.getProxy(
-        HMasterInterface.class,  HMasterInterface.VERSION, isa, conf, 100 x 10);
+        HMasterInterface.class,  HMasterInterface.VERSION, isa, conf, 100 * 10);
       inf.isMasterRunning();
       fail();
     } catch (RemoteException ex) {

From f996df7303d0f520b995275f0c4903fa2184392a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 22 Aug 2012 05:06:44 +0000
Subject: [PATCH 0417/1540] HBASE-6632 [0.92 UNIT TESTS]
 testCreateTableRPCTimeOut sets rpc timeout to 1500ms and leaves it
 (testHundredsOfTable fails w/ 1500ms timeout)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1375898 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/TestAdmin.java | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index d09a3f3f196f..53c8671f6b8e 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -1109,16 +1109,21 @@ public void testTableNameClash() throws Exception {
   @Test
   public void testCreateTableRPCTimeOut() throws Exception {
     String name = "testCreateTableRPCTimeOut";
+    int oldTimeout = TEST_UTIL.getConfiguration().
+      getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
-
-    int expectedRegions = 100;
-    // Use 80 bit numbers to make sure we aren't limited
-    byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
-    byte [] endKey =   { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
-    HBaseAdmin hbaseadmin = new HBaseAdmin(TEST_UTIL.getConfiguration());
-    hbaseadmin.createTable(new HTableDescriptor(name), startKey, endKey,
-      expectedRegions);
-    hbaseadmin.close();
+    try {
+      int expectedRegions = 100;
+      // Use 80 bit numbers to make sure we aren't limited
+      byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
+      byte [] endKey =   { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
+      HBaseAdmin hbaseadmin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+      hbaseadmin.createTable(new HTableDescriptor(name), startKey, endKey,
+        expectedRegions);
+      hbaseadmin.close();
+    } finally {
+      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, oldTimeout);
+    }
   }
 
   /**

From c65b7d321a0c658118865a54a9147d77d9238679 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Wed, 22 Aug 2012 12:31:48 +0000
Subject: [PATCH 0418/1540] HBASE-6364 Powering down the server host holding
 the .META. table causes HBase Client to take excessively long to recover and
 connect to reassigned .META. table

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1376013 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/HBaseClient.java  | 140 +++++++++++++++---
 .../hbase/util/EnvironmentEdgeManager.java    |   4 +-
 .../hbase/util/ManualEnvironmentEdge.java     |   4 +
 3 files changed, 128 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index de1b85cf0731..235c07892802 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -34,6 +34,7 @@
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -47,6 +48,8 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PoolMap;
 import org.apache.hadoop.hbase.util.PoolMap.PoolType;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -84,6 +87,7 @@ public class HBaseClient {
   protected final boolean tcpKeepAlive; // if T then use keepalives
   protected int pingInterval; // how often sends ping to the server in msecs
   protected int socketTimeout; // socket timeout
+  protected FailedServers failedServers;
 
   protected final SocketFactory socketFactory;           // how to create sockets
   private int refCount = 1;
@@ -95,6 +99,68 @@ public class HBaseClient {
   final static int DEFAULT_SOCKET_TIMEOUT = 20000; // 20 seconds
   final static int PING_CALL_ID = -1;
 
+  public final static String FAILED_SERVER_EXPIRY_KEY = "hbase.ipc.client.failed.servers.expiry";
+  public final static int FAILED_SERVER_EXPIRY_DEFAULT = 2000;
+
+  /**
+   * A class to manage a list of servers that failed recently.
+   */
+  static class FailedServers {
+    private final LinkedList<Pair<Long, String>> failedServers = new
+        LinkedList<Pair<Long, String>>();
+    private final int recheckServersTimeout;
+
+    FailedServers(Configuration conf) {
+      this.recheckServersTimeout = conf.getInt(
+          FAILED_SERVER_EXPIRY_KEY, FAILED_SERVER_EXPIRY_DEFAULT);
+    }
+
+    /**
+     * Add an address to the list of the failed servers list.
+     */
+    public synchronized void addToFailedServers(InetSocketAddress address) {
+      final long expiry = EnvironmentEdgeManager.currentTimeMillis() + recheckServersTimeout;
+      failedServers.addFirst(new Pair<Long, String>(expiry, address.toString()));
+    }
+
+    /**
+     * Check if the server should be considered as bad. Clean the old entries of the list.
+     *
+     * @return true if the server is in the failed servers list
+     */
+    public synchronized boolean isFailedServer(final InetSocketAddress address) {
+      if (failedServers.isEmpty()) {
+        return false;
+      }
+
+      final String lookup = address.toString();
+      final long now = EnvironmentEdgeManager.currentTimeMillis();
+
+      // iterate, looking for the search entry and cleaning expired entries
+      Iterator<Pair<Long, String>> it = failedServers.iterator();
+      while (it.hasNext()) {
+        Pair<Long, String> cur = it.next();
+        if (cur.getFirst() < now) {
+          it.remove();
+        } else {
+          if (lookup.equals(cur.getSecond())) {
+            return true;
+          }
+        }
+      }
+
+      return false;
+    }
+
+  }
+
+  public static class FailedServerException extends IOException {
+    public FailedServerException(String s) {
+      super(s);
+    }
+  }
+
+
   /**
    * set the ping interval value in configuration
    *
@@ -206,6 +272,15 @@ public long getStartTime() {
     }
   }
 
+  /**
+   * Creates a connection. Can be overridden by a subclass for testing.
+   *
+   * @param remoteId - the ConnectionId to use for the connection creation.
+   */
+  protected Connection createConnection(ConnectionId remoteId) throws IOException {
+    return new Connection(remoteId);
+  }
+
   /** Thread that reads responses and notifies callers.  Each connection owns a
    * socket connected to a remote address.  Calls are multiplexed through this
    * socket: responses may be delivered out of order. */
@@ -222,7 +297,7 @@ protected class Connection extends Thread {
     protected final AtomicBoolean shouldCloseConnection = new AtomicBoolean();  // indicate if the connection is closed
     protected IOException closeException; // close reason
 
-    public Connection(ConnectionId remoteId) throws IOException {
+    Connection(ConnectionId remoteId) throws IOException {
       if (remoteId.getAddress().isUnresolved()) {
         throw new UnknownHostException("unknown host: " +
                                        remoteId.getAddress().getHostName());
@@ -247,17 +322,30 @@ protected void touch() {
 
     /**
      * Add a call to this connection's call queue and notify
-     * a listener; synchronized.
-     * Returns false if called during shutdown.
+     * a listener; synchronized. If the connection is dead, the call is not added, and the
+     * caller is notified.
+     * This function can return a connection that is already marked as 'shouldCloseConnection'
+     *  It is up to the user code to check this status.
      * @param call to add
-     * @return true if the call was added.
      */
-    protected synchronized boolean addCall(Call call) {
-      if (shouldCloseConnection.get())
-        return false;
-      calls.put(call.id, call);
-      notify();
-      return true;
+    protected synchronized void addCall(Call call) {
+      // If the connection is about to close, we manage this as if the call was already added
+      //  to the connection calls list. If not, the connection creations are serialized, as
+      //  mentioned in HBASE-6364
+      if (this.shouldCloseConnection.get()) {
+        if (this.closeException == null) {
+          call.setException(new IOException(
+              "Call " + call.id + " not added as the connection " + remoteId + " is closing"));
+        } else {
+          call.setException(this.closeException);
+        }
+        synchronized (call) {
+          call.notifyAll();
+        }
+      } else {
+        calls.put(call.id, call);
+        notify();
+      }
     }
 
     /** This class sends a ping to the remote side when timeout on
@@ -355,6 +443,18 @@ protected synchronized void setupIOstreams()
         return;
       }
 
+      if (failedServers.isFailedServer(remoteId.getAddress())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Not trying to connect to " + remoteId.getAddress() +
+              " this server is in the failed servers list");
+        }
+        IOException e = new FailedServerException(
+            "This server is in the failed servers list: " + remoteId.getAddress());
+        markClosed(e);
+        close();
+        throw e;
+      }
+
       try {
         if (LOG.isDebugEnabled()) {
           LOG.debug("Connecting to "+remoteId);
@@ -372,6 +472,7 @@ protected synchronized void setupIOstreams()
         // start the receiver thread after the socket connection has been set up
         start();
       } catch (IOException e) {
+        failedServers.addToFailedServers(remoteId.address);
         markClosed(e);
         close();
 
@@ -789,6 +890,7 @@ public HBaseClient(Class<? extends Writable> valueClass, Configuration conf,
     this.clusterId = conf.get(HConstants.CLUSTER_ID, "default");
     this.connections = new PoolMap<ConnectionId, Connection>(
         getPoolType(conf), getPoolSize(conf));
+    this.failedServers = new FailedServers(conf);
   }
 
   /**
@@ -1029,20 +1131,22 @@ protected Connection getConnection(InetSocketAddress addr,
      * refs for keys in HashMap properly. For now its ok.
      */
     ConnectionId remoteId = new ConnectionId(addr, protocol, ticket, rpcTimeout);
-    do {
-      synchronized (connections) {
-        connection = connections.get(remoteId);
-        if (connection == null) {
-          connection = new Connection(remoteId);
-          connections.put(remoteId, connection);
-        }
+    synchronized (connections) {
+      connection = connections.get(remoteId);
+      if (connection == null) {
+        connection = createConnection(remoteId);
+        connections.put(remoteId, connection);
       }
-    } while (!connection.addCall(call));
+    }
+    connection.addCall(call);
 
     //we don't invoke the method below inside "synchronized (connections)"
     //block above. The reason for that is if the server happens to be slow,
     //it will take longer to establish a connection and that will slow the
     //entire system down.
+    //Moreover, if the connection is currently created, there will be many threads
+    // waiting here; as setupIOstreams is synchronized. If the connection fails with a
+    // timeout, they will all fail simultaneously. This is checked in setupIOstreams.
     connection.setupIOstreams();
     return connection;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
index 9984b4bd2f54..ce4b82b23a9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
@@ -45,7 +45,7 @@ public static EnvironmentEdge getDelegate() {
    * Resets the managed instance to the default instance: {@link
    * DefaultEnvironmentEdge}.
    */
-  static void reset() {
+  public static void reset() {
     injectEdge(new DefaultEnvironmentEdge());
   }
 
@@ -55,7 +55,7 @@ static void reset() {
    *
    * @param edge the new edge.
    */
-  static void injectEdge(EnvironmentEdge edge) {
+  public static void injectEdge(EnvironmentEdge edge) {
     if (edge == null) {
       reset();
     } else {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
index d698df17ea90..a56926aab1d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
@@ -32,6 +32,10 @@ public void setValue(long newValue) {
     value = newValue;
   }
 
+  public void incValue(long addedValue) {
+    value += addedValue;
+  }
+
   @Override
   public long currentTimeMillis() {
     return this.value;

From 98d7cd7473d37d694925249fdb8cc19cd8ac6205 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Wed, 22 Aug 2012 16:48:12 +0000
Subject: [PATCH 0419/1540] HBASE-6364 Powering down the server host holding
 the .META. table causes HBase Client to take excessively long to recover and
 connect to reassigned .META. table - security addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1376136 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/SecureClient.java | 40 ++++---------------
 1 file changed, 7 insertions(+), 33 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 720338f3d982..4c7bcbb85932 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -468,38 +468,12 @@ public SecureClient(Class<? extends Writable> valueClass, Configuration conf) {
     this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
   }
 
+  /**
+   * Creates a SecureConnection. Can be overridden by a subclass for testing.
+   * @param remoteId - the ConnectionId to use for the connection creation.
+   */
   @Override
-  protected SecureConnection getConnection(InetSocketAddress addr,
-                                   Class<? extends VersionedProtocol> protocol,
-                                   User ticket,
-                                   int rpcTimeout,
-                                   Call call)
-                                   throws IOException, InterruptedException {
-    if (!running.get()) {
-      // the client is stopped
-      throw new IOException("The client is stopped");
-    }
-    SecureConnection connection;
-    /* we could avoid this allocation for each RPC by having a
-     * connectionsId object and with set() method. We need to manage the
-     * refs for keys in HashMap properly. For now its ok.
-     */
-    ConnectionId remoteId = new ConnectionId(addr, protocol, ticket, rpcTimeout);
-    do {
-      synchronized (connections) {
-        connection = (SecureConnection)connections.get(remoteId);
-        if (connection == null) {
-          connection = new SecureConnection(remoteId);
-          connections.put(remoteId, connection);
-        }
-      }
-    } while (!connection.addCall(call));
-
-    //we don't invoke the method below inside "synchronized (connections)"
-    //block above. The reason for that is if the server happens to be slow,
-    //it will take longer to establish a connection and that will slow the
-    //entire system down.
-    connection.setupIOstreams();
-    return connection;
+  protected SecureConnection createConnection(ConnectionId remoteId) throws IOException {
+    return new SecureConnection(remoteId);
   }
-}
\ No newline at end of file
+}

From 43182aafcd1e88901408234353670f039c79bb5c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 23 Aug 2012 05:56:29 +0000
Subject: [PATCH 0420/1540] HBASE-5549 Master can fail if ZooKeeper session
 expires

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1376374 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/ActiveMasterManager.java     | 147 +++++++++---------
 .../apache/hadoop/hbase/master/HMaster.java   |   3 +-
 .../hbase/zookeeper/RecoverableZooKeeper.java |  33 +++-
 .../hbase/zookeeper/ZooKeeperWatcher.java     |   4 +
 .../hadoop/hbase/HBaseTestingUtility.java     |  79 ++++++++--
 .../apache/hadoop/hbase/TestZooKeeper.java    |  29 +++-
 .../master/TestDistributedLogSplitting.java   |   4 +-
 .../master/TestMasterZKSessionRecovery.java   |  34 ----
 .../TestSplitTransactionOnCluster.java        |   8 +-
 .../hbase/replication/TestReplication.java    |   6 +-
 10 files changed, 212 insertions(+), 135 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 484b7836a139..c1c4294b5cab 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -122,97 +122,94 @@ private void handleMasterNodeChange() {
    *
    * This also makes sure that we are watching the master znode so will be
    * notified if another master dies.
-   * @param startupStatus 
+   * @param startupStatus
    * @return True if no issue becoming active master else false if another
    * master was running or if some other problem (zookeeper, stop flag has been
    * set on this Master)
    */
   boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
-      ClusterStatusTracker clusterStatusTracker) {
-    startupStatus.setStatus("Trying to register in ZK as active master");
-    boolean cleanSetOfActiveMaster = true;
-    // Try to become the active master, watch if there is another master.
-    // Write out our ServerName as versioned bytes.
-    try {
-      String backupZNode = ZKUtil.joinZNode(
+    ClusterStatusTracker clusterStatusTracker) {
+    while (true) {
+      startupStatus.setStatus("Trying to register in ZK as active master");
+      // Try to become the active master, watch if there is another master.
+      // Write out our ServerName as versioned bytes.
+      try {
+        String backupZNode = ZKUtil.joinZNode(
           this.watcher.backupMasterAddressesZNode, this.sn.toString());
-      if (ZKUtil.createEphemeralNodeAndWatch(this.watcher,
+        if (ZKUtil.createEphemeralNodeAndWatch(this.watcher,
           this.watcher.masterAddressZNode, this.sn.getVersionedBytes())) {
-        // If we were a backup master before, delete our ZNode from the backup
-        // master directory since we are the active now
-        LOG.info("Deleting ZNode for " + backupZNode +
-                 " from backup master directory");
-        ZKUtil.deleteNodeFailSilent(this.watcher, backupZNode);
-
-        // We are the master, return
-        startupStatus.setStatus("Successfully registered as active master.");
+          // If we were a backup master before, delete our ZNode from the backup
+          // master directory since we are the active now
+          LOG.info("Deleting ZNode for " + backupZNode +
+            " from backup master directory");
+          ZKUtil.deleteNodeFailSilent(this.watcher, backupZNode);
+
+          // We are the master, return
+          startupStatus.setStatus("Successfully registered as active master.");
+          this.clusterHasActiveMaster.set(true);
+          LOG.info("Master=" + this.sn);
+          return true;
+        }
+
+        // There is another active master running elsewhere or this is a restart
+        // and the master ephemeral node has not expired yet.
         this.clusterHasActiveMaster.set(true);
-        LOG.info("Master=" + this.sn);
-        return cleanSetOfActiveMaster;
-      }
-      cleanSetOfActiveMaster = false;
-
-      // There is another active master running elsewhere or this is a restart
-      // and the master ephemeral node has not expired yet.
-      this.clusterHasActiveMaster.set(true);
-
-      /*
-       * Add a ZNode for ourselves in the backup master directory since we are
-       * not the active master.
-       *
-       * If we become the active master later, ActiveMasterManager will delete
-       * this node explicitly.  If we crash before then, ZooKeeper will delete
-       * this node for us since it is ephemeral.
-       */
-      LOG.info("Adding ZNode for " + backupZNode +
-               " in backup master directory");
-      ZKUtil.createEphemeralNodeAndWatch(this.watcher, backupZNode,
+
+        /*
+         * Add a ZNode for ourselves in the backup master directory since we are
+         * not the active master.
+         *
+         * If we become the active master later, ActiveMasterManager will delete
+         * this node explicitly.  If we crash before then, ZooKeeper will delete
+         * this node for us since it is ephemeral.
+         */
+        LOG.info("Adding ZNode for " + backupZNode +
+          " in backup master directory");
+        ZKUtil.createEphemeralNodeAndWatch(this.watcher, backupZNode,
           this.sn.getVersionedBytes());
 
-      String msg;
-      byte [] bytes =
-        ZKUtil.getDataAndWatch(this.watcher, this.watcher.masterAddressZNode);
-      if (bytes == null) {
-        msg = ("A master was detected, but went down before its address " +
-          "could be read.  Attempting to become the next active master");
-      } else {
-        ServerName currentMaster = ServerName.parseVersionedServerName(bytes);
-        if (ServerName.isSameHostnameAndPort(currentMaster, this.sn)) {
-          msg = ("Current master has this master's address, " +
-            currentMaster + "; master was restarted?  Waiting on znode " +
-            "to expire...");
-          // Hurry along the expiration of the znode.
-          ZKUtil.deleteNode(this.watcher, this.watcher.masterAddressZNode);
+        String msg;
+        byte [] bytes =
+          ZKUtil.getDataAndWatch(this.watcher, this.watcher.masterAddressZNode);
+        if (bytes == null) {
+          msg = ("A master was detected, but went down before its address " +
+            "could be read.  Attempting to become the next active master");
         } else {
-          msg = "Another master is the active master, " + currentMaster +
-          "; waiting to become the next active master";
+          ServerName currentMaster = ServerName.parseVersionedServerName(bytes);
+          if (ServerName.isSameHostnameAndPort(currentMaster, this.sn)) {
+            msg = ("Current master has this master's address, " +
+              currentMaster + "; master was restarted? Deleting node.");
+            // Hurry along the expiration of the znode.
+            ZKUtil.deleteNode(this.watcher, this.watcher.masterAddressZNode);
+          } else {
+            msg = "Another master is the active master, " + currentMaster +
+              "; waiting to become the next active master";
+          }
         }
+        LOG.info(msg);
+        startupStatus.setStatus(msg);
+      } catch (KeeperException ke) {
+        master.abort("Received an unexpected KeeperException, aborting", ke);
+        return false;
       }
-      LOG.info(msg);
-      startupStatus.setStatus(msg);
-    } catch (KeeperException ke) {
-      master.abort("Received an unexpected KeeperException, aborting", ke);
-      return false;
-    }
-    synchronized (this.clusterHasActiveMaster) {
-      while (this.clusterHasActiveMaster.get() && !this.master.isStopped()) {
-        try {
-          this.clusterHasActiveMaster.wait();
-        } catch (InterruptedException e) {
-          // We expect to be interrupted when a master dies, will fall out if so
-          LOG.debug("Interrupted waiting for master to die", e);
+      synchronized (this.clusterHasActiveMaster) {
+        while (this.clusterHasActiveMaster.get() && !this.master.isStopped()) {
+          try {
+            this.clusterHasActiveMaster.wait();
+          } catch (InterruptedException e) {
+            // We expect to be interrupted when a master dies, will fall out if so
+            LOG.debug("Interrupted waiting for master to die", e);
+          }
         }
+        if (!clusterStatusTracker.isClusterUp()) {
+          this.master.stop("Cluster went down before this master became active");
+        }
+        if (this.master.isStopped()) {
+          return false;
+        }
+        // Try to become active master again now that there is no active master
       }
-      if (!clusterStatusTracker.isClusterUp()) {
-        this.master.stop("Cluster went down before this master became active");
-      }
-      if (this.master.isStopped()) {
-        return cleanSetOfActiveMaster;
-      }
-      // Try to become active master again now that there is no active master
-      cleanSetOfActiveMaster = blockUntilBecomingActiveMaster(startupStatus,clusterStatusTracker);
     }
-    return cleanSetOfActiveMaster;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 00a94e5bc47c..29210c80d0a3 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1464,8 +1464,7 @@ public void abort(final String msg, final Throwable t) {
   private boolean tryRecoveringExpiredZKSession() throws InterruptedException,
       IOException, KeeperException, ExecutionException {
 
-    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":"
-      + this.serverName.getPort(), this, true);
+    this.zooKeeper.reconnectAfterExpiration();
 
     Callable<Boolean> callable = new Callable<Boolean> () {
       public Boolean call() throws InterruptedException,
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index b862aed06c77..52013e93f15b 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -69,7 +69,9 @@ public class RecoverableZooKeeper {
   // An identifier of this process in the cluster
   private final String identifier;
   private final byte[] id;
-  private int retryIntervalMillis;
+  private Watcher watcher;
+  private int sessionTimeout;
+  private String quorumServers;
 
   // The metadata attached to each piece of data has the
   // format:
@@ -84,18 +86,31 @@ public class RecoverableZooKeeper {
   private static final int ID_LENGTH_OFFSET = MAGIC_SIZE;
   private static final int ID_LENGTH_SIZE =  Bytes.SIZEOF_INT;
 
-  public RecoverableZooKeeper(String quorumServers, int seesionTimeout,
+  public RecoverableZooKeeper(String quorumServers, int sessionTimeout,
       Watcher watcher, int maxRetries, int retryIntervalMillis) 
   throws IOException {
-    this.zk = new ZooKeeper(quorumServers, seesionTimeout, watcher);
+    this.zk = new ZooKeeper(quorumServers, sessionTimeout, watcher);
     this.retryCounterFactory =
       new RetryCounterFactory(maxRetries, retryIntervalMillis);
-    this.retryIntervalMillis = retryIntervalMillis;
 
     // the identifier = processID@hostName
     this.identifier = ManagementFactory.getRuntimeMXBean().getName();
     LOG.info("The identifier of this process is " + identifier);
     this.id = Bytes.toBytes(identifier);
+    this.watcher = watcher;
+    this.sessionTimeout = sessionTimeout;
+    this.quorumServers = quorumServers;
+  }
+
+  public void reconnectAfterExpiration()
+        throws IOException, InterruptedException {
+    LOG.info("Closing dead ZooKeeper connection, session" +
+      " was: 0x"+Long.toHexString(zk.getSessionId()));
+    zk.close();
+    this.zk = new ZooKeeper(this.quorumServers,
+      this.sessionTimeout, this.watcher);
+    LOG.info("Recreated a ZooKeeper, session" +
+      " is: 0x"+Long.toHexString(zk.getSessionId()));
   }
 
   /**
@@ -124,6 +139,7 @@ public void delete(String path, int version)
             throw e;
 
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "delete");
             break;
@@ -151,6 +167,7 @@ public Stat exists(String path, Watcher watcher)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "exists");
             break;
@@ -177,6 +194,7 @@ public Stat exists(String path, boolean watch)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "exists");
             break;
@@ -213,6 +231,7 @@ public List<String> getChildren(String path, Watcher watcher)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "getChildren");
             break;
@@ -239,6 +258,7 @@ public List<String> getChildren(String path, boolean watch)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "getChildren");
             break;
@@ -266,6 +286,7 @@ public byte[] getData(String path, Watcher watcher, Stat stat)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "getData");
             break;
@@ -293,6 +314,7 @@ public byte[] getData(String path, boolean watch, Stat stat)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "getData");
             break;
@@ -322,6 +344,7 @@ public Stat setData(String path, byte[] data, int version)
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "setData");
             break;
@@ -418,6 +441,7 @@ private String createNonSequential(String path, byte[] data, List<ACL> acl,
             throw e;
 
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "create");
             break;
@@ -452,6 +476,7 @@ private String createSequential(String path, byte[] data,
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
           case OPERATIONTIMEOUT:
             retryOrThrow(retryCounter, e, "create");
             break;
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 7805c7e7c7b5..57e448413c22 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -243,6 +243,10 @@ public RecoverableZooKeeper getRecoverableZooKeeper() {
     return recoverableZooKeeper;
   }
 
+  public void reconnectAfterExpiration() throws IOException, InterruptedException {
+    recoverableZooKeeper.reconnectAfterExpiration();
+  }
+
   /**
    * Get the quorum address of this instance.
    * @return quorum string of this zookeeper connection instance
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 1c05fb8b9588..e15eb47ea26f 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -39,6 +39,7 @@
 import java.util.NavigableSet;
 import java.util.Random;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -71,6 +72,7 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.apache.hadoop.hbase.util.RegionSplitter;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
@@ -84,6 +86,7 @@
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.apache.zookeeper.ZooKeeper;
 
@@ -1342,7 +1345,7 @@ public void enableDebug(Class<?> clazz) {
    */
   public void expireMasterSession() throws Exception {
     HMaster master = hbaseCluster.getMaster();
-    expireSession(master.getZooKeeper(), master);
+    expireSession(master.getZooKeeper(), false);
   }
 
   /**
@@ -1352,16 +1355,22 @@ public void expireMasterSession() throws Exception {
    */
   public void expireRegionServerSession(int index) throws Exception {
     HRegionServer rs = hbaseCluster.getRegionServer(index);
-    expireSession(rs.getZooKeeper(), rs);
-  }
-
-  public void expireSession(ZooKeeperWatcher nodeZK, Server server)
+    expireSession(rs.getZooKeeper(), false);
+  }
+
+   /**
+    * Expire a ZooKeeper session as recommended in ZooKeeper documentation
+    * http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A4
+    * There are issues when doing this:
+    * [1] http://www.mail-archive.com/dev@zookeeper.apache.org/msg01942.html
+    * [2] https://issues.apache.org/jira/browse/ZOOKEEPER-1105
+    *
+    * @param nodeZK - the ZK to make expiry
+    * @param checkStatus - true to check if the we can create a HTable with the
+    *                    current configuration.
+    */
+  public void expireSession(ZooKeeperWatcher nodeZK, boolean checkStatus)
     throws Exception {
-    expireSession(nodeZK, server, false);
-  }
-
-  public void expireSession(ZooKeeperWatcher nodeZK, Server server,
-      boolean checkStatus) throws Exception {
     Configuration c = new Configuration(this.conf);
     String quorumServers = ZKConfig.getZKQuorumServersString(c);
     int sessionTimeout = 500;
@@ -1369,14 +1378,29 @@ public void expireSession(ZooKeeperWatcher nodeZK, Server server,
     byte[] password = zk.getSessionPasswd();
     long sessionID = zk.getSessionId();
 
+    // Expiry seems to be asynchronous (see comment from P. Hunt in [1]),
+    //  so we create a first watcher to be sure that the
+    //  event was sent. We expect that if our watcher receives the event
+    //  other watchers on the same machine will get is as well.
+    // When we ask to close the connection, ZK does not close it before
+    //  we receive all the events, so don't have to capture the event, just
+    //  closing the connection should be enough.
+    ZooKeeper monitor = new ZooKeeper(quorumServers,
+      1000, new org.apache.zookeeper.Watcher(){
+      @Override
+      public void process(WatchedEvent watchedEvent) {
+        LOG.info("Monitor ZKW received event="+watchedEvent);
+      }
+    } , sessionID, password);
+
+    // Making it expire
     ZooKeeper newZK = new ZooKeeper(quorumServers,
         sessionTimeout, EmptyWatcher.instance, sessionID, password);
     newZK.close();
-    final long sleep = 7000; // 7s seems enough to manage the timeout
-    LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID) +
-      "; sleeping=" + sleep);
+    LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID));
 
-    Thread.sleep(sleep);
+     // Now closing & waiting to be sure that the clients get it.
+     monitor.close();
 
     if (checkStatus) {
       new HTable(new Configuration(conf), HConstants.META_TABLE_NAME).close();
@@ -1545,7 +1569,7 @@ public void waitTableEnabled(byte[] table, long timeoutMillis)
    * Make sure that at least the specified number of region servers
    * are running
    * @param num minimum number of region servers that should be running
-   * @return True if we started some servers
+   * @return true if we started some servers
    * @throws IOException
    */
   public boolean ensureSomeRegionServersAvailable(final int num)
@@ -1561,6 +1585,31 @@ public boolean ensureSomeRegionServersAvailable(final int num)
   }
 
 
+  /**
+   * Make sure that at least the specified number of region servers
+   * are running. We don't count the ones that are currently stopping or are
+   * stopped.
+   * @param num minimum number of region servers that should be running
+   * @return true if we started some servers
+   * @throws IOException
+   */
+  public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
+    throws IOException {
+    boolean startedServer = ensureSomeRegionServersAvailable(num);
+
+    for (JVMClusterUtil.RegionServerThread rst :
+      hbaseCluster.getRegionServerThreads()) {
+
+      HRegionServer hrs = rst.getRegionServer();
+      if (hrs.isStopping() || hrs.isStopped()) {
+        LOG.info("A region server is stopped or stopping:"+hrs);
+        LOG.info("Started new server=" + hbaseCluster.startRegionServer());
+        startedServer = true;
+      }
+    }
+
+    return startedServer;
+  }
 
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index b791b70dbb77..fa537bc97d1c 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -26,6 +26,8 @@
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -40,6 +42,7 @@
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
@@ -97,6 +100,15 @@ public void setUp() throws Exception {
     TEST_UTIL.ensureSomeRegionServersAvailable(2);
   }
 
+  private ZooKeeperWatcher getZooKeeperWatcher(HConnection c) throws
+    NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+
+    Method getterZK = c.getClass().getMethod("getKeepAliveZooKeeperWatcher");
+    getterZK.setAccessible(true);
+
+    return (ZooKeeperWatcher) getterZK.invoke(c);
+  }
+
   /**
    * See HBASE-1232 and http://wiki.apache.org/hadoop/ZooKeeper/FAQ#4.
    * @throws IOException
@@ -111,7 +123,7 @@ public void testClientSessionExpired()
     new HTable(c, HConstants.META_TABLE_NAME).close();
     HConnection connection = HConnectionManager.getConnection(c);
     ZooKeeperWatcher connectionZK = connection.getZooKeeperWatcher();
-    TEST_UTIL.expireSession(connectionZK, null);
+    TEST_UTIL.expireSession(connectionZK, false);
 
     // provoke session expiration by doing something with ZK
     ZKUtil.dump(connectionZK);
@@ -350,6 +362,21 @@ public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
     ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
   }
 
+  /**
+   * Master recovery when the znode already exists. Internally, this
+   *  test differs from {@link #testMasterSessionExpired} because here
+   *  the master znode will exist in ZK.
+   */
+  @Test(timeout=20000)
+  public void testMasterZKSessionRecoveryFailure() throws Exception {
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    HMaster m = cluster.getMaster();
+    m.abort("Test recovery from zk session expired",
+      new KeeperException.SessionExpiredException());
+    assertFalse(m.isStopped());
+    testSanity();
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 3a0c6b07e89b..afd9f0e68c96 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -87,6 +87,8 @@ private void startCluster(int num_rs) throws Exception{
     LOG.info("Starting cluster");
     conf = HBaseConfiguration.create();
     conf.getLong("hbase.splitlog.max.resubmit", 0);
+    // Make the failure test faster
+    conf.setInt("zookeeper.recovery.retry", 0);
     TEST_UTIL = new HBaseTestingUtility(conf);
     TEST_UTIL.startMiniCluster(NUM_MASTERS, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
@@ -245,7 +247,7 @@ public void run() {
     slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch);
     //waitForCounter but for one of the 2 counters
     long curt = System.currentTimeMillis();
-    long waitTime = 30000;
+    long waitTime = 80000;
     long endt = curt + waitTime;
     while (curt < endt) {
       if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
index ef9d368aea8b..b732cb5f88ad 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
@@ -79,40 +79,6 @@ public void tearDown() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  /**
-   * Negative test of master recovery from zk session expiry.
-   * <p>
-   * Starts with one master. Fakes the master zk session expired.
-   * The master should be able to come up if he is able to create
-   * the node as active master.
-   * @throws Exception
-   */
-  @Test(timeout=10000)
-  public void testMasterZKSessionRecoveryFailure() throws Exception {
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-    HMaster m = cluster.getMaster();
-    m.abort("Test recovery from zk session expired",
-      new KeeperException.SessionExpiredException());
-    assertFalse(m.isStopped());
-  }
-
-  /**
-   * Positive test of master recovery from zk session expiry.
-   * <p>
-   * Starts with one master. Closes the master zk session.
-   * Ensures the master can recover the expired zk session.
-   * @throws Exception
-   */
-  @Test(timeout=60000)
-  public void testMasterZKSessionRecoverySuccess() throws Exception {
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-    HMaster m = cluster.getMaster();
-    m.getZooKeeperWatcher().close();
-    m.abort("Test recovery from zk session expired",
-      new KeeperException.SessionExpiredException());
-    assertFalse(m.isStopped());
-  }
-  
   /**
    * Tests that the master does not call retainAssignment after recovery from
    * expired zookeeper session. Without the HBASE-6046 fix master always tries
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 1c0806bd222c..fbd70090f367 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
@@ -83,7 +84,7 @@ public class TestSplitTransactionOnCluster {
   }
 
   @Before public void setup() throws IOException {
-    TESTING_UTIL.ensureSomeRegionServersAvailable(NB_SERVERS);
+    TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
     this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
     this.cluster = TESTING_UTIL.getMiniHBaseCluster();
   }
@@ -651,7 +652,10 @@ private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
     HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
     if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
       HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
-      LOG.info("Moving " + hri.getRegionNameAsString() + " to " +
+      assertNotNull(hrs);
+      assertNotNull(hri);
+      LOG.
+        info("Moving " + hri.getRegionNameAsString() + " to " +
         hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
       admin.move(hri.getEncodedNameAsBytes(),
         Bytes.toBytes(hrs.getServerName().toString()));
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index f61655a6be35..96eb211cccfb 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -100,6 +100,8 @@ public static void setUpBeforeClass() throws Exception {
     conf1.setLong("replication.source.sleepforretries", 100);
     conf1.setInt("hbase.regionserver.maxlogs", 10);
     conf1.setLong("hbase.master.logcleaner.ttl", 10);
+    conf1.setInt("zookeeper.recovery.retry", 1);
+    conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
     conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
     conf1.setBoolean("dfs.support.append", true);
     conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
@@ -757,9 +759,11 @@ public void queueFailover() throws Exception {
 
     int lastCount = 0;
 
+    final long start = System.currentTimeMillis();
     for (int i = 0; i < NB_RETRIES; i++) {
       if (i==NB_RETRIES-1) {
-        fail("Waited too much time for queueFailover replication");
+        fail("Waited too much time for queueFailover replication. " +
+          "Waited "+(System.currentTimeMillis() - start)+"ms.");
       }
       Scan scan2 = new Scan();
       ResultScanner scanner2 = htable2.getScanner(scan2);

From 782472a72c873a25dd41dfc954302dc4eb59cdea Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 23 Aug 2012 20:28:11 +0000
Subject: [PATCH 0421/1540] HBASE-6644 HBaseAdmin.createTable should wait more
 till table is enabled.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1376681 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/HBaseAdmin.java    | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 211a8ba3dd4d..28434728f806 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -450,10 +450,17 @@ public boolean processRow(Result rowResult) throws IOException {
           }
         } else {
           doneWithMetaScan = true;
+          tries = -1;
         }
-      }
-      if (doneWithMetaScan && isTableEnabled(desc.getName())) {
+      } else if (isTableEnabled(desc.getName())) {
         return;
+      } else {
+        try { // Sleep
+          Thread.sleep(getPauseTime(tries));
+        } catch (InterruptedException e) {
+          throw new InterruptedIOException("Interrupted when waiting" +
+            " for table to be enabled; meta scan was done");
+        }
       }
     }
     throw new TableNotEnabledException(

From e472b2f68a3ca8f836f13c21011e394c49bdaea6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 24 Aug 2012 01:20:19 +0000
Subject: [PATCH 0422/1540] HBASE-6577 REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1376804 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java   | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9b2996c0de5e..814e45a539f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3481,7 +3481,7 @@ private boolean nextInternal(int limit) throws IOException {
             // there're no left overs.
             // the reasons for calling this method are:
             // 1. reset the filters.
-            // 2. fast forward the row
+            // 2. provide a hook to fast forward the row (used by subclasses)
             nextRow(currentRow);
 
             // This row was totally filtered out, if this is NOT the last row,
@@ -3504,9 +3504,6 @@ private boolean filterRowKey(byte[] row) {
     }
 
     protected void nextRow(byte [] currentRow) throws IOException {
-      KeyValue kv = KeyValue.createLastOnRow(currentRow);
-      this.storeHeap.requestSeek(kv, true, true);
-
       while (Bytes.equals(currentRow, peekRow())) {
         this.storeHeap.next(MOCKED_LIST);
       }

From e616bb5fc503567d6c5924521a9b0f40d3399d43 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 24 Aug 2012 02:16:55 +0000
Subject: [PATCH 0423/1540] HBASE-3271 Allow .META. table to be exported (Liang
 Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1376807 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TableInputFormatBase.java | 14 +++++++++++-
 .../hbase/mapreduce/TestImportExport.java     | 22 ++++++++++++++++++-
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index a615710333f3..eb4954aaffb5 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -31,11 +31,13 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.mapreduce.InputFormat;
@@ -155,7 +157,17 @@ public List<InputSplit> getSplits(JobContext context) throws IOException {
     Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
     if (keys == null || keys.getFirst() == null ||
         keys.getFirst().length == 0) {
-      throw new IOException("Expecting at least one region.");
+      HRegionLocation regLoc = table.getRegionLocation(
+          HConstants.EMPTY_BYTE_ARRAY, false);
+      if (null == regLoc) {
+        throw new IOException("Expecting at least one region.");
+      }
+      List<InputSplit> splits = new ArrayList<InputSplit>(1);
+      InputSplit split = new TableSplit(table.getTableName(),
+          HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc
+              .getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0]);
+      splits.add(split);
+      return splits;
     }
     List<InputSplit> splits = new ArrayList<InputSplit>(keys.getFirst().length);
     for (int i = 0; i < keys.getFirst().length; i++) {
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index bc05754b64a1..a573cbc8d7de 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.hadoop.conf.Configuration;
@@ -44,7 +45,6 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import static org.junit.Assert.assertEquals;
 
 @Category(MediumTests.class)
 public class TestImportExport {
@@ -142,6 +142,26 @@ public void testSimpleCase() throws Exception {
     assertEquals(3, r.size());
   }
 
+  /**
+   * Test export .META. table
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testMetaExport() throws Exception {
+    String EXPORT_TABLE = ".META.";
+    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
+        cluster.getConfiguration()), args);
+    Configuration conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+  }
+
   @Test
   public void testWithDeletes() throws Exception {
     String EXPORT_TABLE = "exportWithDeletes";

From 48bfac8d1d858a2432374f51725a6fcf9a8de9c9 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Fri, 24 Aug 2012 18:32:52 +0000
Subject: [PATCH 0424/1540] HBASE-6437 Avoid admin.balance during master
 initialize

Submitted by:Rajesh
Reviewed by:Ted


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377023 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 29210c80d0a3..186c896875ae 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -970,6 +970,11 @@ private int getBalancerCutoffTime() {
 
   @Override
   public boolean balance() {
+    // if master not initialized, don't run balancer.
+    if (!this.initialized) {
+      LOG.debug("Master has not been initialized, don't run balancer.");
+      return false;
+    }
     // If balance not true, don't run balancer.
     if (!this.balanceSwitch) return false;
     // Do this call outside of synchronized block.

From 0c29ebd0e03a4b624dff24476735599e7e980222 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 24 Aug 2012 18:40:08 +0000
Subject: [PATCH 0425/1540] HBASE-6616 test failure in
 TestDelayedRpc#testTooManyDelayedRpcs

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377028 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
index 9046d99b2fa2..66bd1955c762 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
@@ -37,6 +37,7 @@
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Logger;
 import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.Level;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -126,6 +127,7 @@ public void testTooManyDelayedRpcs() throws Exception {
     ListAppender listAppender = new ListAppender();
     Logger log = Logger.getLogger("org.apache.hadoop.ipc.HBaseServer");
     log.addAppender(listAppender);
+    log.setLevel(Level.WARN);
 
     InetSocketAddress isa = new InetSocketAddress("localhost", 0);
     rpcServer = HBaseRPC.getServer(new TestRpcImpl(true),

From ee8b04bab9d9b5a7c2e011266f24956e5846fde3 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 24 Aug 2012 18:44:25 +0000
Subject: [PATCH 0426/1540] HBASE-6623 [replication] replication metrics value
 AgeOfLastShippedOp is not set correctly

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377036 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index e2257ea538ad..0866c61ab265 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -616,7 +616,7 @@ protected void shipEdits() {
         this.metrics.shippedOpsRate.inc(
             this.currentNbOperations);
         this.metrics.setAgeOfLastShippedOp(
-            this.entriesArray[this.entriesArray.length-1].getKey().getWriteTime());
+            this.entriesArray[currentNbEntries-1].getKey().getWriteTime());
         LOG.debug("Replicated in total: " + this.totalReplicatedEdits);
         break;
 

From b81d8f239a987027f8142a18f3dad3f2a881e435 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 24 Aug 2012 20:53:37 +0000
Subject: [PATCH 0427/1540] HBASE-6211 Put latencies in jmx

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377107 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/metrics/MetricsMBeanBase.java       | 74 +++++++++++++++++++
 .../metrics/histogram/MetricsHistogram.java   | 31 +++++---
 .../hbase/coprocessor/TestMasterObserver.java | 20 +++--
 .../hbase/metrics/TestMetricsMBeanBase.java   | 54 ++++++++++++++
 4 files changed, 161 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
index 37fdfc281ec4..ee6d8392ee34 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
@@ -29,7 +29,9 @@
 import javax.management.MBeanException;
 import javax.management.MBeanInfo;
 import javax.management.ReflectionException;
+import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
 
+import com.yammer.metrics.stats.Snapshot;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.metrics.util.MetricsBase;
@@ -104,6 +106,52 @@ protected void init() {
             "java.lang.String", metric.getDescription(), true, false, false) );
         extendedAttributes.put(metric.getName(), metric);
         LOG.info("MetricsString added: " + metric.getName());
+      } else if (metric instanceof MetricsHistogram) {
+
+        String metricName = metric.getName() + MetricsHistogram.NUM_OPS_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Long", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MIN_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Long", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MAX_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Long", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MEAN_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.STD_DEV_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MEDIAN_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
       }
       // else, its probably a hadoop metric already registered. Skip it.
     }
@@ -148,6 +196,32 @@ public Object getAttribute( String name )
           return ((MetricsRate) metric).getPreviousIntervalValue();
         } else if (metric instanceof MetricsString) {
           return ((MetricsString)metric).getValue();
+        } else if (metric instanceof MetricsHistogram)  {
+          MetricsHistogram hist = (MetricsHistogram) metric;
+          if (name.endsWith(MetricsHistogram.NUM_OPS_METRIC_NAME)) {
+            return hist.getCount();
+          } else if (name.endsWith(MetricsHistogram.MIN_METRIC_NAME)) {
+            return hist.getMin();
+          } else if (name.endsWith(MetricsHistogram.MAX_METRIC_NAME)) {
+            return hist.getMax();
+          } else if (name.endsWith(MetricsHistogram.MEAN_METRIC_NAME)) {
+            return (float) hist.getMean();
+          } else if (name.endsWith(MetricsHistogram.STD_DEV_METRIC_NAME)) {
+            return (float) hist.getStdDev();
+          } else if (name.endsWith(MetricsHistogram.MEDIAN_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.getMedian();
+          } else if (name.endsWith(MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.get75thPercentile();
+          } else if (name.endsWith(MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.get95thPercentile();
+          } else if (name.endsWith(MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.get99thPercentile();
+          }
+
         } else {
           LOG.warn( String.format("unknown metrics type %s for attribute %s",
                         metric.getClass().getName(), name) );
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
index 7b65a8fabda4..392cbf9876aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
@@ -39,6 +39,15 @@ public class MetricsHistogram extends MetricsBase {
   // the bias towards sampling from more recent data. 
   // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes
   private static final double DEFAULT_ALPHA = 0.015;
+  public static final String NUM_OPS_METRIC_NAME = "_num_ops";
+  public static final String MIN_METRIC_NAME = "_min";
+  public static final String MAX_METRIC_NAME = "_max";
+  public static final String MEAN_METRIC_NAME = "_mean";
+  public static final String STD_DEV_METRIC_NAME = "_std_dev";
+  public static final String MEDIAN_METRIC_NAME = "_median";
+  public static final String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile";
+  public static final String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile";
+  public static final String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile";
 
   /**
    * Constructor to create a new histogram metric
@@ -212,19 +221,19 @@ private double getVariance() {
   @Override
   public void pushMetric(MetricsRecord mr) {
     final Snapshot s = this.getSnapshot();
-    mr.setMetric(getName() + "_num_ops", this.getCount());
-    mr.setMetric(getName() + "_min", this.getMin());
-    mr.setMetric(getName() + "_max", this.getMax());
-    
-    mr.setMetric(getName() + "_mean", (float) this.getMean());
-    mr.setMetric(getName() + "_std_dev", (float) this.getStdDev());
-    
-    mr.setMetric(getName() + "_median", (float) s.getMedian());
-    mr.setMetric(getName() + "_75th_percentile", 
+    mr.setMetric(getName() + NUM_OPS_METRIC_NAME, this.getCount());
+    mr.setMetric(getName() + MIN_METRIC_NAME, this.getMin());
+    mr.setMetric(getName() + MAX_METRIC_NAME, this.getMax());
+
+    mr.setMetric(getName() + MEAN_METRIC_NAME, (float) this.getMean());
+    mr.setMetric(getName() + STD_DEV_METRIC_NAME, (float) this.getStdDev());
+
+    mr.setMetric(getName() + MEDIAN_METRIC_NAME, (float) s.getMedian());
+    mr.setMetric(getName() + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME,
         (float) s.get75thPercentile());
-    mr.setMetric(getName() + "_95th_percentile", 
+    mr.setMetric(getName() + NINETY_FIFTH_PERCENTILE_METRIC_NAME,
         (float) s.get95thPercentile());
-    mr.setMetric(getName() + "_99th_percentile", 
+    mr.setMetric(getName() + NINETY_NINETH_PERCENTILE_METRIC_NAME,
         (float) s.get99thPercentile());
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index b369c768c275..8910030a8555 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -31,6 +31,8 @@
 import java.util.Map;
 import java.util.NavigableMap;
 
+import junit.framework.Assert;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -684,6 +686,8 @@ public void testRegionTransitionOperations() throws Exception {
     // move half the open regions from RS 0 to RS 1
     HRegionServer rs = cluster.getRegionServer(0);
     byte[] destRS = Bytes.toBytes(cluster.getRegionServer(1).getServerName().toString());
+    //Make sure no regions are in transition now
+    waitForRITtoBeZero(master);
     List<HRegionInfo> openRegions = rs.getOnlineRegions();
     int moveCnt = openRegions.size()/2;
     for (int i=0; i<moveCnt; i++) {
@@ -692,7 +696,16 @@ public void testRegionTransitionOperations() throws Exception {
         master.move(openRegions.get(i).getEncodedNameAsBytes(), destRS);
       }
     }
+    //Make sure no regions are in transition now
+    waitForRITtoBeZero(master);
+    // now trigger a balance
+    master.balanceSwitch(true);
+    boolean balanceRun = master.balance();
+    assertTrue("Coprocessor should be called on region rebalancing",
+        cp.wasBalanceCalled());
+  }
 
+  private void waitForRITtoBeZero(HMaster master) throws IOException {
     // wait for assignments to finish
     AssignmentManager mgr = master.getAssignmentManager();
     Collection<AssignmentManager.RegionState> transRegions =
@@ -700,13 +713,6 @@ public void testRegionTransitionOperations() throws Exception {
     for (AssignmentManager.RegionState state : transRegions) {
       mgr.waitOnRegionToClearRegionsInTransition(state.getRegion());
     }
-
-    // now trigger a balance
-    master.balanceSwitch(true);
-    boolean balanceRun = master.balance();
-    assertTrue("Coprocessor should be called on region rebalancing",
-        cp.wasBalanceCalled());
-    table.close();
   }
 
   @org.junit.Rule
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java
index a6c771e9fd83..96e7772185db 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java
@@ -24,8 +24,13 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.management.AttributeNotFoundException;
 import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
 import javax.management.MBeanInfo;
+import javax.management.ReflectionException;
+import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+import com.yammer.metrics.stats.Snapshot;
 
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.metrics.MetricsContext;
@@ -35,6 +40,9 @@
 import org.apache.hadoop.metrics.util.MetricsRegistry;
 import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 import junit.framework.TestCase;
 import org.junit.experimental.categories.Category;
 
@@ -114,6 +122,52 @@ public void testGetMBeanInfo() {
         "varyRateNumOps", "java.lang.Integer", "test");
   }
 
+  public void testMetricsMBeanBaseHistogram()
+      throws ReflectionException, AttributeNotFoundException, MBeanException {
+    MetricsRegistry mr = new MetricsRegistry();
+    MetricsHistogram histo = mock(MetricsHistogram.class);
+    Snapshot snap = mock(Snapshot.class);
+
+    //Set up the mocks
+    String histoName = "MockHisto";
+    when(histo.getName()).thenReturn(histoName);
+    when(histo.getCount()).thenReturn(20l);
+    when(histo.getMin()).thenReturn(1l);
+    when(histo.getMax()).thenReturn(999l);
+    when(histo.getMean()).thenReturn(500.2);
+    when(histo.getStdDev()).thenReturn(1.2);
+    when(histo.getSnapshot()).thenReturn(snap);
+
+    when(snap.getMedian()).thenReturn(490.0);
+    when(snap.get75thPercentile()).thenReturn(550.0);
+    when(snap.get95thPercentile()).thenReturn(900.0);
+    when(snap.get99thPercentile()).thenReturn(990.0);
+
+    mr.add("myTestHisto", histo);
+
+    MetricsMBeanBase mBeanBase = new MetricsMBeanBase(mr, "test");
+
+    assertEquals(new Long(20), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.NUM_OPS_METRIC_NAME));
+    assertEquals(new Long(1), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.MIN_METRIC_NAME));
+    assertEquals(new Long(999), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.MAX_METRIC_NAME));
+    assertEquals(new Float(500.2), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.MEAN_METRIC_NAME));
+    assertEquals(new Float(1.2), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.STD_DEV_METRIC_NAME));
+
+    assertEquals(new Float(490.0), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.MEDIAN_METRIC_NAME));
+    assertEquals(new Float(550.0), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME));
+    assertEquals(new Float(900.0), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME));
+    assertEquals(new Float(990.0), mBeanBase
+        .getAttribute(histoName + MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME));
+  }
+
   protected void assertAttribute(MBeanAttributeInfo attr, String name,
       String type, String description) {
 

From 206530fdef97e90d5eeb49f5e93e3c6c64f5c56a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 24 Aug 2012 21:39:28 +0000
Subject: [PATCH 0428/1540] HBASE-6641 more message with DoNotRetryIOException
 in client

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377122 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/RetriesExhaustedWithDetailsException.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
index 4a98c5e17d60..ea14c068649b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
@@ -123,7 +123,7 @@ public static Map<String, Integer> classifyExs(List<Throwable> ths) {
     for (Throwable t : ths) {
       if (t == null) continue;
       String name = "";
-      if (t instanceof NoSuchColumnFamilyException) {
+      if (t instanceof DoNotRetryIOException) {
         name = t.getMessage();
       } else {
         name = t.getClass().getSimpleName();

From 81bf02da35eb9aef080afaa8dc12f60bd64931ef Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 24 Aug 2012 23:34:59 +0000
Subject: [PATCH 0429/1540] HBASE-6638

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377148 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTable.java    | 32 +-------------
 .../org/apache/hadoop/hbase/util/Threads.java | 44 ++++++++++++++++---
 .../hbase/client/TestFromClientSide.java      |  4 +-
 .../apache/hadoop/hbase/client/TestHCM.java   |  4 +-
 4 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index f7927c00e66d..9bea85f5faec 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Threads;
 
 /**
  * <p>Used to communicate with a single HBase table.
@@ -167,7 +168,7 @@ public HTable(Configuration conf, final byte [] tableName)
     this.pool = new ThreadPoolExecutor(1, maxThreads,
         keepAliveTime, TimeUnit.SECONDS,
         new SynchronousQueue<Runnable>(),
-        new DaemonThreadFactory());
+        Threads.newDaemonThreadFactory("hbase-table"));
     ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
 
     this.finishSetup();
@@ -1101,35 +1102,6 @@ ExecutorService getPool() {
     return this.pool;
   }
 
-  static class DaemonThreadFactory implements ThreadFactory {
-    static final AtomicInteger poolNumber = new AtomicInteger(1);
-        final ThreadGroup group;
-        final AtomicInteger threadNumber = new AtomicInteger(1);
-        final String namePrefix;
-
-        DaemonThreadFactory() {
-            SecurityManager s = System.getSecurityManager();
-            group = (s != null)? s.getThreadGroup() :
-                                 Thread.currentThread().getThreadGroup();
-            namePrefix = "hbase-table-pool" +
-                          poolNumber.getAndIncrement() +
-                         "-thread-";
-        }
-
-        public Thread newThread(Runnable r) {
-            Thread t = new Thread(group, r,
-                                  namePrefix + threadNumber.getAndIncrement(),
-                                  0);
-            if (!t.isDaemon()) {
-              t.setDaemon(true);
-            }
-            if (t.getPriority() != Thread.NORM_PRIORITY) {
-              t.setPriority(Thread.NORM_PRIORITY);
-            }
-            return t;
-        }
-  }
-
   /**
    * Enable or disable region cache prefetch for the table. It will be
    * applied for the given table's all HTable instances who share the same
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index c1c7ea4c84b3..2d581f395c38 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -36,6 +36,7 @@
  */
 public class Threads {
   protected static final Log LOG = LogFactory.getLog(Threads.class);
+  private static final AtomicInteger poolNumber = new AtomicInteger(1);
 
   /**
    * Utility method that sets name, daemon status and starts passed thread.
@@ -180,21 +181,50 @@ public static ThreadPoolExecutor getBoundedCachedThreadPool(
   
   
   /**
-   * Returns a {@link java.util.concurrent.ThreadFactory} that names each
-   * created thread uniquely, with a common prefix.
-   * 
-   * @param prefix  The prefix of every created Thread's name
+   * Returns a {@link java.util.concurrent.ThreadFactory} that names each created thread uniquely,
+   * with a common prefix.
+   * @param prefix The prefix of every created Thread's name
    * @return a {@link java.util.concurrent.ThreadFactory} that names threads
    */
   public static ThreadFactory getNamedThreadFactory(final String prefix) {
+    SecurityManager s = System.getSecurityManager();
+    final ThreadGroup threadGroup = (s != null) ? s.getThreadGroup() : Thread.currentThread()
+        .getThreadGroup();
+
     return new ThreadFactory() {
+      final AtomicInteger threadNumber = new AtomicInteger(1);
+      private final int poolNumber = Threads.poolNumber.getAndIncrement();
+      final ThreadGroup group = threadGroup;
+
+      @Override
+      public Thread newThread(Runnable r) {
+        final String name = prefix + "pool-" + poolNumber + "-thread-"
+            + threadNumber.getAndIncrement();
+        return new Thread(group, r, name);
+      }
+    };
+  }
 
-      private final AtomicInteger threadNumber = new AtomicInteger(1);
-      
+  /**
+   * Get a named {@link ThreadFactory} that just builds daemon threads
+   * @param prefix name prefix for all threads created from the factory
+   * @return a thread factory that creates named, daemon threads
+   */
+  public static ThreadFactory newDaemonThreadFactory(final String prefix) {
+    final ThreadFactory namedFactory = getNamedThreadFactory(prefix);
+    return new ThreadFactory() {
       @Override
       public Thread newThread(Runnable r) {
-        return new Thread(r, prefix + threadNumber.getAndIncrement());
+        Thread t = namedFactory.newThread(r);
+        if (!t.isDaemon()) {
+          t.setDaemon(true);
+        }
+        if (t.getPriority() != Thread.NORM_PRIORITY) {
+          t.setPriority(Thread.NORM_PRIORITY);
+        }
+        return t;
       }
+
     };
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 256ddb8f9c79..989c29e3f7b2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -63,7 +63,6 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HTable.DaemonThreadFactory;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
@@ -86,6 +85,7 @@
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -3880,7 +3880,7 @@ public void testUnmanagedHConnection() throws IOException {
     ExecutorService pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE,
         60, TimeUnit.SECONDS,
         new SynchronousQueue<Runnable>(),
-        new DaemonThreadFactory());
+        Threads.newDaemonThreadFactory("test-from-client-table"));
     ((ThreadPoolExecutor)pool).allowCoreThreadTimeOut(true);
     HTable t = new HTable(tableName, conn, pool);
     HBaseAdmin ha = new HBaseAdmin(conn);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 603a4fb8c245..023eaff0d8de 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -35,8 +35,8 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HTable.DaemonThreadFactory;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -148,7 +148,7 @@ public void testConnectionManagement() throws Exception{
     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
         60, TimeUnit.SECONDS,
         new SynchronousQueue<Runnable>(),
-        new DaemonThreadFactory());
+        Threads.newDaemonThreadFactory("test-hcm-table"));
 
     HTable table = new HTable(TABLE_NAME1, conn, pool);
     table.close();

From d61c17c4ccfb203551ff7f83651ffec3ed636cff Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 24 Aug 2012 23:40:13 +0000
Subject: [PATCH 0430/1540] HBASE-6529 With HFile v2, the region server will
 always perform an extra copy of source files

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377151 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 0c9ace525280..0d52a3e5fa89 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
@@ -534,7 +535,8 @@ void bulkLoadHFile(String srcPathStr) throws IOException {
 
     // Move the file if it's on another filesystem
     FileSystem srcFs = srcPath.getFileSystem(conf);
-    if (!srcFs.equals(fs)) {
+    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
+    if (!srcFs.equals(desFs)) {
       LOG.info("File " + srcPath + " on different filesystem than " +
           "destination store - moving to this filesystem.");
       Path tmpPath = getTmpPath();

From 8cefdd7aaba16e00082bd3d6b7803592a35d39fb Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 25 Aug 2012 00:01:26 +0000
Subject: [PATCH 0431/1540] HBASE-6496 Example ZK based scan policy

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377155 13f79535-47bb-0310-9956-ffa450edef68
---
 .../example/ZooKeeperScanPolicyObserver.java  | 230 ++++++++++++++++++
 .../TestZooKeeperScanPolicyObserver.java      | 123 ++++++++++
 2 files changed, 353 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
new file mode 100644
index 000000000000..f2bfe1157e08
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * This is an example showing how a RegionObserver could configured
+ * via ZooKeeper in order to control a Region compaction, flush, and scan policy.
+ *
+ * This also demonstrated the use of shared {@link RegionObserver} state.
+ * See {@link RegionCoprocessorEnvironment#getSharedData()}.
+ *
+ * This would be useful for an incremental backup tool, which would indicate the last
+ * time of a successful backup via ZK and instruct HBase to not delete data that was
+ * inserted since (based on wall clock time). 
+ *
+ * This implements org.apache.zookeeper.Watcher directly instead of using
+ * {@link ZooKeeperWatcher}, because RegionObservers come and go and currently
+ * listeners registered with ZooKeeperWatcher cannot be removed.
+ */
+public class ZooKeeperScanPolicyObserver extends BaseRegionObserver {
+  public static String node = "/backup/example/lastbackup";
+  public static String zkkey = "ZK";
+  private static final Log LOG = LogFactory.getLog(ZooKeeperScanPolicyObserver.class);
+
+  /**
+   * Internal watcher that keep "data" up to date asynchronously.
+   */
+  private static class ZKWatcher implements Watcher {
+    private byte[] data = null;
+    private ZooKeeper zk;
+    private volatile boolean needSetup = true;
+    private volatile long lastSetupTry = 0;
+
+    public ZKWatcher(ZooKeeper zk) {
+      this.zk = zk;
+      // trigger the listening
+      getData();
+    }
+
+    /**
+     * Get the maintained data. In case of any ZK exceptions this will retry
+     * establishing the connection (but not more than twice/minute).
+     *
+     * getData is on the critical path, so make sure it is fast unless there is
+     * a problem (network partion, ZK ensemble down, etc)
+     * Make sure at most one (unlucky) thread retries and other threads don't pile up
+     * while that threads tries to recreate the connection.
+     *
+     * @return the last know version of the data
+     */
+    public byte[] getData() {
+      // try at most twice/minute
+      if (needSetup && EnvironmentEdgeManager.currentTimeMillis() > lastSetupTry + 30000) {
+        synchronized (this) {
+          // make sure only one thread tries to reconnect
+          if (needSetup) {
+            needSetup = false;
+          } else {
+            return data;
+          }
+        }
+        // do this without the lock held to avoid threads piling up on this lock,
+        // as it can take a while
+        try {
+          LOG.debug("Connecting to ZK");
+          // record this attempt
+          lastSetupTry = EnvironmentEdgeManager.currentTimeMillis();
+          if (zk.exists(node, false) != null) {
+            data = zk.getData(node, this, null);
+            LOG.debug("Read synchronously: "+(data == null ? "null" : Bytes.toLong(data)));
+          } else {
+            zk.exists(node, this);
+          }
+        } catch (Exception x) {
+          // try again if this fails
+          needSetup = true;
+        }
+      }
+      return data;
+    }
+
+    @Override
+    public void process(WatchedEvent event) {
+      switch(event.getType()) {
+      case NodeDataChanged:
+      case NodeCreated:
+      try {
+        // get data and re-watch
+        data = zk.getData(node, this, null);
+        LOG.debug("Read asynchronously: "+(data == null ? "null" : Bytes.toLong(data)));
+      } catch (InterruptedException ix) {
+      } catch (KeeperException kx) {
+        needSetup = true;
+      }
+      break;
+
+      case NodeDeleted:
+      try {
+        // just re-watch
+        zk.exists(node, this);
+        data = null;
+      } catch (InterruptedException ix) {
+      } catch (KeeperException kx) {
+        needSetup = true;
+      }
+      break;
+
+      default:
+        // ignore
+      }
+    }
+  }
+
+  @Override
+  public void start(CoprocessorEnvironment e) throws IOException {
+    RegionCoprocessorEnvironment re = (RegionCoprocessorEnvironment) e;
+    if (!re.getSharedData().containsKey(zkkey)) {
+      // there is a short race here
+      // in the worst case we create a watcher that will be notified once
+      re.getSharedData().putIfAbsent(
+          zkkey,
+          new ZKWatcher(re.getRegionServerServices().getZooKeeper()
+              .getRecoverableZooKeeper().getZooKeeper()));
+    }
+  }
+
+  @Override
+  public void stop(CoprocessorEnvironment e) throws IOException {
+    // nothing to do here
+  }
+
+  protected ScanInfo getScanInfo(Store store, RegionCoprocessorEnvironment e) {
+    byte[] data = ((ZKWatcher)e.getSharedData().get(zkkey)).getData();
+    if (data == null) {
+      return null;
+    }
+    ScanInfo oldSI = store.getScanInfo();
+    if (oldSI.getTtl() == Long.MAX_VALUE) {
+      return null;
+    }
+    long ttl =  Math.max(EnvironmentEdgeManager.currentTimeMillis() - Bytes.toLong(data), oldSI.getTtl());    
+    return new ScanInfo(store.getFamily(), ttl,
+        oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+  }
+
+  @Override
+  public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
+    Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment());
+    if (scanInfo == null) {
+      // take default action
+      return null;
+    }
+    Scan scan = new Scan();
+    scan.setMaxVersions(scanInfo.getMaxVersions());
+    return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
+        ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(),
+        HConstants.OLDEST_TIMESTAMP);
+  }
+
+  @Override
+  public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
+      InternalScanner s) throws IOException {
+    Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment());
+    if (scanInfo == null) {
+      // take default action
+      return null;
+    }
+    Scan scan = new Scan();
+    scan.setMaxVersions(scanInfo.getMaxVersions());
+    return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion()
+        .getSmallestReadPoint(), earliestPutTs);
+  }
+
+  @Override
+  public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
+      final KeyValueScanner s) throws IOException {
+    Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment());
+    if (scanInfo == null) {
+      // take default action
+      return null;
+    }
+    return new StoreScanner(store, scanInfo, scan, targetCols);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
new file mode 100644
index 000000000000..bcf821fd7c46
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestZooKeeperScanPolicyObserver {
+  private static final Log LOG = LogFactory.getLog(TestZooKeeperScanPolicyObserver.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final byte[] F = Bytes.toBytes("fam");
+  private static final byte[] Q = Bytes.toBytes("qual");
+  private static final byte[] R = Bytes.toBytes("row");
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // Test we can first start the ZK cluster by itself
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+        ZooKeeperScanPolicyObserver.class.getName());
+    TEST_UTIL.startMiniZKCluster();
+    TEST_UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testScanPolicyObserver() throws Exception {
+    byte[] tableName = Bytes.toBytes("testScanPolicyObserver");
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor(F)
+    .setMaxVersions(10)
+    .setTimeToLive(1);
+    desc.addFamily(hcd);
+    TEST_UTIL.getHBaseAdmin().createTable(desc);
+    HTable t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+
+    ZooKeeperWatcher zkw = HConnectionManager.getConnection(TEST_UTIL.getConfiguration())
+        .getZooKeeperWatcher();
+    ZooKeeper zk = zkw.getRecoverableZooKeeper().getZooKeeper();
+    ZKUtil.createWithParents(zkw, ZooKeeperScanPolicyObserver.node);
+    // let's say test last backup was 1h ago
+    // using plain ZK here, because RecoverableZooKeeper add extra encoding to the data
+    zk.setData(ZooKeeperScanPolicyObserver.node, Bytes.toBytes(now - 3600*1000), -1);
+    
+    LOG.debug("Set time: "+Bytes.toLong(Bytes.toBytes(now - 3600*1000)));
+
+    long ts = now - 2000;
+    Put p = new Put(R);
+    p.add(F, Q, ts, Q);
+    t.put(p);
+    p = new Put(R);
+    p.add(F, Q, ts+1, Q);
+    t.put(p);
+
+    // these two should be expired but for the override
+    // (their ts was 2s in the past)
+    Get g = new Get(R);
+    g.setMaxVersions(10);
+    Result r = t.get(g);
+    // still there?
+    assertEquals(2, r.size());
+
+    TEST_UTIL.flush(tableName);
+    TEST_UTIL.compact(tableName, true);
+
+    g = new Get(R);
+    g.setMaxVersions(10);
+    r = t.get(g);
+    // still there?
+    assertEquals(2, r.size());
+    zk.setData(ZooKeeperScanPolicyObserver.node, Bytes.toBytes(now), -1);
+    LOG.debug("Set time: "+now);
+
+    TEST_UTIL.compact(tableName, true);
+
+    g = new Get(R);
+    g.setMaxVersions(10);
+    r = t.get(g);
+    // should be gone now
+    assertEquals(0, r.size());
+    t.close();
+  }
+}

From 5d2605adfb54aea157ef283c1f11f688c8f1fc87 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 25 Aug 2012 00:09:24 +0000
Subject: [PATCH 0432/1540] HBASE-6458 new comparator twice in checkAndPut,
 just reuse the first one (binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377158 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ebe8d6c3c7ef..1cfc200353b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2086,7 +2086,7 @@ public boolean checkAndPut(final byte[] regionName, final byte[] row,
       }
     }
     boolean result = checkAndMutate(regionName, row, family, qualifier,
-      CompareOp.EQUAL, new BinaryComparator(value), put,
+        CompareOp.EQUAL, comparator, put,
       lock);
     if (region.getCoprocessorHost() != null) {
       result = region.getCoprocessorHost().postCheckAndPut(row, family,

From 6a02df19f3bb1737b1a29c54462419836335d64c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 25 Aug 2012 01:41:29 +0000
Subject: [PATCH 0433/1540] HBASE-6496 Addendum - add test category

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377177 13f79535-47bb-0310-9956-ffa450edef68
---
 .../coprocessor/example/TestZooKeeperScanPolicyObserver.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
index bcf821fd7c46..91b4742082d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
@@ -39,7 +40,9 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(MediumTests.class)
 public class TestZooKeeperScanPolicyObserver {
   private static final Log LOG = LogFactory.getLog(TestZooKeeperScanPolicyObserver.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

From fd339e36269012411505b93443d9011f1472d93f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 25 Aug 2012 04:32:21 +0000
Subject: [PATCH 0434/1540] HBASE-5292 getsize per-CF metric incorrectly counts
 compaction related reads as well

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377205 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 31 +++++---
 .../hbase/regionserver/HRegionServer.java     |  4 +-
 .../hbase/regionserver/InternalScanner.java   | 20 +++++
 .../hbase/regionserver/KeyValueHeap.java      | 23 +++++-
 .../hbase/regionserver/StoreScanner.java      | 33 ++++++--
 .../regionserver/metrics/SchemaMetrics.java   |  3 +
 .../coprocessor/TestCoprocessorInterface.java | 12 +++
 .../TestRegionObserverInterface.java          | 17 +++-
 .../regionserver/TestRegionServerMetrics.java | 79 ++++++++++++++++++-
 9 files changed, 199 insertions(+), 23 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 814e45a539f7..704a66d100cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -44,7 +44,6 @@
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
@@ -199,7 +198,7 @@ public class HRegion implements HeapSize { // , Writable{
   // Registered region protocol handlers
   private ClassToInstanceMap<CoprocessorProtocol>
       protocolHandlers = MutableClassToInstanceMap.create();
-  
+
   private Map<String, Class<? extends CoprocessorProtocol>>
       protocolHandlerNames = Maps.newHashMap();
 
@@ -333,7 +332,6 @@ boolean isFlushRequested() {
   private RegionSplitPolicy splitPolicy;
   private final OperationMetrics opMetrics;
 
-
   /**
    * Should only be used for testing purposes
    */
@@ -914,7 +912,7 @@ private List<StoreFile> doClose(
         CompletionService<ImmutableList<StoreFile>> completionService =
           new ExecutorCompletionService<ImmutableList<StoreFile>>(
             storeCloserThreadPool);
-      
+
         // close each store in parallel
         for (final Store store : stores.values()) {
           completionService
@@ -2943,7 +2941,7 @@ private long replayRecoveredEdits(final Path edits,
       return currentEditSeqId;
     } finally {
       status.cleanup();
-      if (reader != null) {  
+      if (reader != null) {
          reader.close();
       }
     }
@@ -3389,6 +3387,12 @@ protected void resetFilters() {
     @Override
     public synchronized boolean next(List<KeyValue> outResults, int limit)
         throws IOException {
+      return next(outResults, limit, null);
+    }
+
+    @Override
+    public synchronized boolean next(List<KeyValue> outResults, int limit,
+        String metric) throws IOException {
       if (this.filterClosed) {
         throw new UnknownScannerException("Scanner was closed (timed out?) " +
             "after we renewed it. Could be caused by a very slow scanner " +
@@ -3403,7 +3407,7 @@ public synchronized boolean next(List<KeyValue> outResults, int limit)
 
         results.clear();
 
-        boolean returnResult = nextInternal(limit);
+        boolean returnResult = nextInternal(limit, metric);
 
         outResults.addAll(results);
         resetFilters();
@@ -3420,7 +3424,14 @@ public synchronized boolean next(List<KeyValue> outResults, int limit)
     public synchronized boolean next(List<KeyValue> outResults)
         throws IOException {
       // apply the batching limit by default
-      return next(outResults, batch);
+      return next(outResults, batch, null);
+    }
+
+    @Override
+    public synchronized boolean next(List<KeyValue> outResults, String metric)
+        throws IOException {
+      // apply the batching limit by default
+      return next(outResults, batch, metric);
     }
 
     /*
@@ -3430,7 +3441,7 @@ public synchronized boolean isFilterDone() {
       return this.filter != null && this.filter.filterAllRemaining();
     }
 
-    private boolean nextInternal(int limit) throws IOException {
+    private boolean nextInternal(int limit, String metric) throws IOException {
       RpcCallContext rpcCall = HBaseServer.getCurrentCall();
       while (true) {
         if (rpcCall != null) {
@@ -3457,7 +3468,7 @@ private boolean nextInternal(int limit) throws IOException {
         } else {
           byte [] nextRow;
           do {
-            this.storeHeap.next(results, limit - results.size());
+            this.storeHeap.next(results, limit - results.size(), metric);
             if (limit > 0 && results.size() == limit) {
               if (this.filter != null && filter.hasFilterRow()) {
                 throw new IncompatibleFilterException(
@@ -4169,7 +4180,7 @@ private List<KeyValue> get(Get get, boolean withCoprocessor)
     RegionScanner scanner = null;
     try {
       scanner = getScanner(scan);
-      scanner.next(results);
+      scanner.next(results, SchemaMetrics.METRIC_GETSIZE);
     } finally {
       if (scanner != null)
         scanner.close();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1cfc200353b1..b28ccb20e347 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -261,7 +261,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
 
   private RegionServerMetrics metrics;
 
-  @SuppressWarnings("unused")
   private RegionServerDynamicMetrics dynamicMetrics;
 
   // Compactions
@@ -308,7 +307,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
 
   // Instance of the hbase executor service.
   private ExecutorService service;
-  @SuppressWarnings("unused")
 
   // Replication services. If no replication, this handler will be null.
   private ReplicationSourceService replicationSourceHandler;
@@ -2390,7 +2388,7 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
           && currentScanResultSize < maxScannerResultSize; i++) {
         requestCount.incrementAndGet();
         // Collect values to be returned here
-        boolean moreRows = s.next(values);
+        boolean moreRows = s.next(values, SchemaMetrics.METRIC_NEXTSIZE);
         if (!values.isEmpty()) {
           for (KeyValue kv : values) {
             currentScanResultSize += kv.heapSize();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
index 0f5f36c0029a..6cbed5aeebf2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
@@ -47,6 +47,15 @@ public interface InternalScanner extends Closeable {
    * @throws IOException e
    */
   public boolean next(List<KeyValue> results) throws IOException;
+  
+  /**
+   * Grab the next row's worth of values.
+   * @param results return output array
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean next(List<KeyValue> results, String metric) throws IOException;
 
   /**
    * Grab the next row's worth of values with a limit on the number of values
@@ -57,6 +66,17 @@ public interface InternalScanner extends Closeable {
    * @throws IOException e
    */
   public boolean next(List<KeyValue> result, int limit) throws IOException;
+  
+  /**
+   * Grab the next row's worth of values with a limit on the number of values
+   * to return.
+   * @param result return output array
+   * @param limit limit on row count to get
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean next(List<KeyValue> result, int limit, String metric) throws IOException;
 
   /**
    * Closes the scanner and releases any resources it has allocated
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 659bbdd658ca..8613a872fcb5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -120,11 +120,27 @@ public KeyValue next()  throws IOException {
    * @return true if there are more keys, false if all scanners are done
    */
   public boolean next(List<KeyValue> result, int limit) throws IOException {
+    return next(result, limit, null);
+  }
+
+  /**
+   * Gets the next row of keys from the top-most scanner.
+   * <p>
+   * This method takes care of updating the heap.
+   * <p>
+   * This can ONLY be called when you are using Scanners that implement
+   * InternalScanner as well as KeyValueScanner (a {@link StoreScanner}).
+   * @param result output result list
+   * @param limit limit on row count to get
+   * @param metric the metric name
+   * @return true if there are more keys, false if all scanners are done
+   */
+  public boolean next(List<KeyValue> result, int limit, String metric) throws IOException {
     if (this.current == null) {
       return false;
     }
     InternalScanner currentAsInternal = (InternalScanner)this.current;
-    boolean mayContainMoreRows = currentAsInternal.next(result, limit);
+    boolean mayContainMoreRows = currentAsInternal.next(result, limit, metric);
     KeyValue pee = this.current.peek();
     /*
      * By definition, any InternalScanner must return false only when it has no
@@ -156,6 +172,11 @@ public boolean next(List<KeyValue> result) throws IOException {
     return next(result, -1);
   }
 
+  @Override
+  public boolean next(List<KeyValue> result, String metric) throws IOException {
+    return next(result, -1, metric);
+  }
+
   private static class KVScannerComparator implements Comparator<KeyValueScanner> {
     private KVComparator kvComparator;
     /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index a1cf76533705..8919a3869226 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -50,8 +50,8 @@ public class StoreScanner extends NonLazyKeyValueScanner
   private KeyValueHeap heap;
   private boolean cacheBlocks;
 
-  private String metricNameGetSize;
 
+  private String metricNamePrefix;
   // Used to indicate that the scanner has closed (see HBASE-1107)
   // Doesnt need to be volatile because it's always accessed via synchronized methods
   private boolean closing = false;
@@ -198,7 +198,7 @@ public StoreScanner(Store store, ScanInfo scanInfo, Scan scan,
   /**
    * Method used internally to initialize metric names throughout the
    * constructors.
-   * 
+   *
    * To be called after the store variable has been initialized!
    */
   private void initializeMetricNames() {
@@ -208,8 +208,8 @@ private void initializeMetricNames() {
       tableName = store.getTableName();
       family = Bytes.toString(store.getFamily().getName());
     }
-    metricNameGetSize = SchemaMetrics.generateSchemaMetricsPrefix(
-        tableName, family) + "getsize";
+    this.metricNamePrefix =
+        SchemaMetrics.generateSchemaMetricsPrefix(tableName, family);
   }
 
   /**
@@ -308,6 +308,18 @@ public synchronized boolean seek(KeyValue key) throws IOException {
    */
   @Override
   public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException {
+    return next(outResult, limit, null);
+  }
+
+  /**
+   * Get the next row of values from this Store.
+   * @param outResult
+   * @param limit
+   * @return true if there are more rows, false if scanner is done
+   */
+  @Override
+  public synchronized boolean next(List<KeyValue> outResult, int limit,
+      String metric) throws IOException {
 
     if (checkReseek()) {
       return true;
@@ -420,7 +432,10 @@ public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOE
         }
       }
     } finally {
-      RegionMetricsStorage.incrNumericMetric(metricNameGetSize, cumulativeMetric);
+      if (cumulativeMetric > 0 && metric != null) {
+        RegionMetricsStorage.incrNumericMetric(this.metricNamePrefix + metric,
+            cumulativeMetric);
+      }
     }
 
     if (!results.isEmpty()) {
@@ -436,7 +451,13 @@ public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOE
 
   @Override
   public synchronized boolean next(List<KeyValue> outResult) throws IOException {
-    return next(outResult, -1);
+    return next(outResult, -1, null);
+  }
+
+  @Override
+  public synchronized boolean next(List<KeyValue> outResult, String metric)
+      throws IOException {
+    return next(outResult, -1, metric);
   }
 
   // Implementation of ChangedReadersObserver
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 08afbbc039a1..e73513ceb953 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -176,6 +176,9 @@ public String toString() {
   /** Use for readability when obtaining non-compaction counters */
   public static final boolean NO_COMPACTION = false;
 
+  public static final String METRIC_GETSIZE = "getsize";
+  public static final String METRIC_NEXTSIZE = "nextsize";
+
   /**
    * A special schema metric value that means "all tables aggregated" or
    * "all column families aggregated" when used as a table name or a column
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index fd1459046993..1833aab9bd6b 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -66,11 +66,23 @@ public CustomScanner(RegionScanner delegate) {
     public boolean next(List<KeyValue> results) throws IOException {
       return delegate.next(results);
     }
+    
+    @Override
+    public boolean next(List<KeyValue> results, String metric) 
+        throws IOException {
+      return delegate.next(results, metric);
+    }
 
     @Override
     public boolean next(List<KeyValue> result, int limit) throws IOException {
       return delegate.next(result, limit);
     }
+    
+    @Override
+    public boolean next(List<KeyValue> result, int limit, String metric) 
+        throws IOException {
+      return delegate.next(result, limit, metric);
+    }
 
     @Override
     public void close() throws IOException {
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index d378ac98ccae..36318f20f63b 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -304,13 +304,26 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
         public boolean next(List<KeyValue> results) throws IOException {
           return next(results, -1);
         }
+        
+        @Override
+        public boolean next(List<KeyValue> results, String metric) 
+            throws IOException {
+          return next(results, -1, metric);
+        }
+
+        @Override
+        public boolean next(List<KeyValue> results, int limit) 
+            throws IOException{
+          return next(results, limit, null);
+        }
 
         @Override
-        public boolean next(List<KeyValue> results, int limit) throws IOException {
+        public boolean next(List<KeyValue> results, int limit, String metric) 
+            throws IOException {
           List<KeyValue> internalResults = new ArrayList<KeyValue>();
           boolean hasMore;
           do {
-            hasMore = scanner.next(internalResults, limit);
+            hasMore = scanner.next(internalResults, limit, metric);
             if (!internalResults.isEmpty()) {
               long row = Bytes.toLong(internalResults.get(0).getRow());
               if (row % 2 == 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 8d0dc8184c34..aab7787642f4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -29,6 +29,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.Append;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.
@@ -69,7 +71,7 @@ public class TestRegionServerMetrics {
   private static final SchemaMetrics ALL_METRICS =
       SchemaMetrics.ALL_SCHEMA_METRICS;
 
-  private static final HBaseTestingUtility TEST_UTIL =
+  private final HBaseTestingUtility TEST_UTIL =
       new HBaseTestingUtility();
 
   private Map<String, Long> startingMetrics;
@@ -270,5 +272,80 @@ public void testMultipleRegions() throws IOException, InterruptedException {
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+  private void assertSizeMetric(String table, String[] cfs, int[] metrics) {
+    // we have getsize & nextsize for each column family
+    assertEquals(cfs.length * 2, metrics.length);
+
+    for (int i =0; i < cfs.length; ++i) {
+      String prefix = SchemaMetrics.generateSchemaMetricsPrefix(table, cfs[i]);
+      String getMetric = prefix + SchemaMetrics.METRIC_GETSIZE;
+      String nextMetric = prefix + SchemaMetrics.METRIC_NEXTSIZE;
+
+      // verify getsize and nextsize matches
+      int getSize = RegionMetricsStorage.getNumericMetrics().containsKey(getMetric) ?
+          RegionMetricsStorage.getNumericMetrics().get(getMetric).intValue() : 0;
+      int nextSize = RegionMetricsStorage.getNumericMetrics().containsKey(nextMetric) ?
+          RegionMetricsStorage.getNumericMetrics().get(nextMetric).intValue() : 0;
+
+      assertEquals(metrics[i], getSize);
+      assertEquals(metrics[cfs.length + i], nextSize);
+    }
+  }
+
+  @Test
+  public void testGetNextSize() throws IOException, InterruptedException {
+    String rowName = "row1";
+    byte[] ROW = Bytes.toBytes(rowName);
+    String tableName = "SizeMetricTest";
+    byte[] TABLE = Bytes.toBytes(tableName);
+    String cf1Name = "cf1";
+    String cf2Name = "cf2";
+    String[] cfs = new String[] {cf1Name, cf2Name};
+    byte[] CF1 = Bytes.toBytes(cf1Name);
+    byte[] CF2 = Bytes.toBytes(cf2Name);
+
+    long ts = 1234;
+    HTable hTable = TEST_UTIL.createTable(TABLE, new byte[][]{CF1, CF2});
+    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+
+    Put p = new Put(ROW);
+    p.add(CF1, CF1, ts, CF1);
+    p.add(CF2, CF2, ts, CF2);
+    hTable.put(p);
+
+    KeyValue kv1 = new KeyValue(ROW, CF1, CF1, ts, CF1);
+    KeyValue kv2 = new KeyValue(ROW, CF2, CF2, ts, CF2);
+    int kvLength = kv1.getLength();
+    assertEquals(kvLength, kv2.getLength());
+
+    // only cf1.getsize is set on Get
+    hTable.get(new Get(ROW).addFamily(CF1));
+    assertSizeMetric(tableName, cfs, new int[] {kvLength, 0, 0, 0});
+
+    // only cf2.getsize is set on Get
+    hTable.get(new Get(ROW).addFamily(CF2));
+    assertSizeMetric(tableName, cfs, new int[] {kvLength, kvLength, 0, 0});
+
+    // only cf2.nextsize is set
+    for (Result res : hTable.getScanner(CF2)) {
+    }
+    assertSizeMetric(tableName, cfs,
+        new int[] {kvLength, kvLength, 0, kvLength});
+
+    // only cf2.nextsize is set
+    for (Result res : hTable.getScanner(CF1)) {
+    }
+    assertSizeMetric(tableName, cfs,
+        new int[] {kvLength, kvLength, kvLength, kvLength});
+
+    // getsize/nextsize should not be set on flush or compaction
+    for (HRegion hr : TEST_UTIL.getMiniHBaseCluster().getRegions(TABLE)) {
+      hr.flushcache();
+      hr.compactStores();
+    }
+    assertSizeMetric(tableName, cfs,
+        new int[] {kvLength, kvLength, kvLength, kvLength});
+  }
 }
 

From 412efdb40dd665c672e7d6f6ec0a233b19c847d6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 25 Aug 2012 04:38:27 +0000
Subject: [PATCH 0435/1540] HBASE-6514 unknown metrics type:
 org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram (Elliot Clark)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377207 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
index ee6d8392ee34..75971715ad89 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
@@ -73,7 +73,8 @@ protected MetricsMBeanBase( MetricsRegistry mr, String description ) {
   private static MetricsRegistry copyMinusHBaseMetrics(final MetricsRegistry mr) {
     MetricsRegistry copy = new MetricsRegistry();
     for (MetricsBase metric : mr.getMetricsList()) {
-      if (metric instanceof MetricsRate || metric instanceof MetricsString) {
+      if (metric instanceof MetricsRate || metric instanceof MetricsString ||
+          metric instanceof MetricsHistogram || metric instanceof ExactCounterMetric) {
         continue;
       }
       copy.add(metric.getName(), metric);

From 35bcce7054de40bde01e72692bdd6d84f420b254 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 25 Aug 2012 05:30:23 +0000
Subject: [PATCH 0436/1540] HBASE-6663 NPE race in HConnection if zookeeper is
 reset (Gregory Chanan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377215 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      |  12 +-
 .../hadoop/hbase/client/TestHConnection.java  | 163 ++++++++++++++++++
 2 files changed, 169 insertions(+), 6 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 3edbc2eae5b3..e7ebe0cfe0b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -778,13 +778,13 @@ private boolean testTableOnlineState(byte [] tableName, boolean online)
         // The root region is always enabled
         return online;
       }
-      getZooKeeperWatcher();
+      ZooKeeperWatcher zkw = getZooKeeperWatcher();
       String tableNameStr = Bytes.toString(tableName);
       try {
         if (online) {
-          return ZKTable.isEnabledTable(this.zooKeeper, tableNameStr);
+          return ZKTable.isEnabledTable(zkw, tableNameStr);
         }
-        return ZKTable.isDisabledTable(this.zooKeeper, tableNameStr);
+        return ZKTable.isDisabledTable(zkw, tableNameStr);
       } catch (KeeperException e) {
         throw new IOException("Enable/Disable failed", e);
       }
@@ -1689,11 +1689,11 @@ public boolean isAborted(){
 
     public int getCurrentNrHRS() throws IOException {
       try {
-        getZooKeeperWatcher();
+        ZooKeeperWatcher zkw = getZooKeeperWatcher();
         // We go to zk rather than to master to get count of regions to avoid
         // HTable having a Master dependency.  See HBase-2828
-        return ZKUtil.getNumberOfChildren(this.zooKeeper,
-            this.zooKeeper.rsZNode);
+        return ZKUtil.getNumberOfChildren(zkw,
+            zkw.rsZNode);
       } catch (KeeperException ke) {
         throw new IOException("Unexpected ZooKeeper exception", ke);
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java b/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java
new file mode 100644
index 000000000000..470616e231de
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java
@@ -0,0 +1,163 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MultithreadedTestUtil;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Various tests of using HConnection
+ */
+@Category(MediumTests.class)
+public class TestHConnection {
+  final Log LOG = LogFactory.getLog(getClass());
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final long MILLIS_TO_WAIT_FOR_RACE = 1000;
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Thread that periodically aborts the connection
+   */
+  class AbortThread extends RepeatingTestThread {
+    final HConnection connection;
+
+    public AbortThread(MultithreadedTestUtil.TestContext ctx, HConnection connection) {
+      super(ctx);
+      this.connection = connection;
+    }
+
+    @Override
+    public void doAnAction() throws IOException {
+      connection.abort("test session expired", new KeeperException.SessionExpiredException());
+    }
+  };
+
+  class HConnectionRaceTester extends HConnectionImplementation {
+    public HConnectionRaceTester(Configuration configuration, boolean managed) throws ZooKeeperConnectionException {
+      super(configuration, managed);
+    }
+
+    /**
+     * Sleep after calling getZookeeperWatcher to attempt to trigger a race condition.
+     * If the HConnection retrieves the ZooKeeperWatcher but does not cache the value,
+     * by the time the new watcher is used, it could be null if the connection was aborted.
+     * This sleep will increase the time between when the watcher is retrieved and when it is used.
+     */
+    public ZooKeeperWatcher getZooKeeperWatcher() throws ZooKeeperConnectionException {
+      ZooKeeperWatcher zkw = super.getZooKeeperWatcher();
+      try {
+        Thread.sleep(10);
+      } catch (InterruptedException ie) {
+        // Ignore
+      }
+      return zkw;
+    }
+  }
+
+  /**
+   * Test that a connection that is aborted while calling isTableDisabled doesn't NPE
+   */
+  @Test
+  public void testTableDisabledRace() throws Exception {
+    final HConnection connection = new HConnectionRaceTester(TEST_UTIL.getConfiguration(), true);
+    MultithreadedTestUtil.TestContext ctx =
+      new MultithreadedTestUtil.TestContext(TEST_UTIL.getConfiguration());
+    RepeatingTestThread disabledChecker = new RepeatingTestThread(ctx) {
+      @Override
+      public void doAnAction() throws IOException {
+        try {
+          connection.isTableDisabled(Bytes.toBytes("tableToCheck"));
+        } catch (IOException ioe) {
+          // Ignore.  ZK can legitimately fail, only care if we get a NullPointerException
+        }
+      }
+    };
+    AbortThread abortThread = new AbortThread(ctx, connection);
+
+    ctx.addThread(disabledChecker);
+    ctx.addThread(abortThread);
+    ctx.startThreads();
+    ctx.waitFor(MILLIS_TO_WAIT_FOR_RACE);
+    ctx.stop();
+  }
+
+  /**
+   * Test that a connection that is aborted while calling getCurrentNrNRS doesn't NPE
+   */
+  @Test
+  public void testGetCurrentNrHRSRace() throws Exception {
+    final HConnection connection = new HConnectionRaceTester(TEST_UTIL.getConfiguration(), true);
+    MultithreadedTestUtil.TestContext ctx =
+      new MultithreadedTestUtil.TestContext(TEST_UTIL.getConfiguration());
+    RepeatingTestThread getCurrentNrHRSCaller = new RepeatingTestThread(ctx) {
+      @Override
+      public void doAnAction() throws IOException {
+        try {
+          connection.getCurrentNrHRS();
+        } catch (IOException ioe) {
+          // Ignore.  ZK can legitimately fail, only care if we get a NullPointerException
+        }
+      }
+    };
+    AbortThread abortThread = new AbortThread(ctx, connection);
+
+    ctx.addThread(getCurrentNrHRSCaller);
+    ctx.addThread(abortThread);
+    ctx.startThreads();
+    ctx.waitFor(MILLIS_TO_WAIT_FOR_RACE);
+    ctx.stop();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From a74a5c284d1ea141f6b9b8bdd01db71957fef421 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 25 Aug 2012 05:32:05 +0000
Subject: [PATCH 0437/1540] HBASE-6662 Region server incorrectly reports its
 own address as master's address

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377217 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegionServer.java    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b28ccb20e347..5764997b29c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1780,6 +1780,7 @@ private ServerName getMaster() {
     ServerName masterServerName = null;
     long previousLogTime = 0;
     HMasterRegionInterface master = null;
+    InetSocketAddress masterIsa = null;
     while (keepLooping() && master == null) {
       masterServerName = this.masterAddressManager.getMasterAddress();
       if (masterServerName == null) {
@@ -1795,7 +1796,7 @@ private ServerName getMaster() {
         continue;
       }
 
-      InetSocketAddress isa =
+      masterIsa =
         new InetSocketAddress(masterServerName.getHostname(), masterServerName.getPort());
 
       LOG.info("Attempting connect to Master server at " +
@@ -1805,7 +1806,7 @@ private ServerName getMaster() {
         // should retry indefinitely.
         master = (HMasterRegionInterface) HBaseRPC.waitForProxy(
             HMasterRegionInterface.class, HMasterRegionInterface.VERSION,
-            isa, this.conf, -1,
+            masterIsa, this.conf, -1,
             this.rpcTimeout, this.rpcTimeout);
       } catch (IOException e) {
         e = e instanceof RemoteException ?
@@ -1827,7 +1828,7 @@ private ServerName getMaster() {
         }
       }
     }
-    LOG.info("Connected to master at " + isa);
+    LOG.info("Connected to master at " + masterIsa);
     this.hbaseMaster = master;
     return masterServerName;
   }

From 4b7c8c33946ed692e231b6a4e0c7dccd525dace9 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 27 Aug 2012 21:23:54 +0000
Subject: [PATCH 0438/1540] HBASE-6643 Accept encoded region name in
 compacting/spliting region from shell

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377877 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 108 +++++++++++-------
 .../apache/hadoop/hbase/client/TestAdmin.java |  26 ++++-
 2 files changed, 92 insertions(+), 42 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 28434728f806..abfe10d809b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -27,6 +27,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
@@ -61,7 +62,6 @@
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.util.StringUtils;
 
@@ -408,11 +408,13 @@ public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
           @Override
           public boolean processRow(Result rowResult) throws IOException {
-            HRegionInfo info = Writables.getHRegionInfoOrNull(
-              rowResult.getValue(HConstants.CATALOG_FAMILY,
-              HConstants.REGIONINFO_QUALIFIER));
-            //If regioninfo is null, skip this row
-            if (null == info) {
+            if (rowResult == null || rowResult.size() <= 0) {
+              return true;
+            }
+            HRegionInfo info = MetaReader.parseHRegionInfoFromCatalogResult(
+              rowResult, HConstants.REGIONINFO_QUALIFIER);
+            if (info == null) {
+              LOG.warn("No serialized HRegionInfo in " + rowResult);
               return true;
             }
             if (!(Bytes.equals(info.getTableName(), desc.getName()))) {
@@ -1179,16 +1181,15 @@ public void flush(final String tableNameOrRegionName)
   public void flush(final byte [] tableNameOrRegionName)
   throws IOException, InterruptedException {
     CatalogTracker ct = getCatalogTracker();
-    boolean isRegionName = isRegionName(tableNameOrRegionName, ct);
     try {
-      if (isRegionName) {
-        Pair<HRegionInfo, ServerName> pair =
-          MetaReader.getRegion(ct, tableNameOrRegionName);
-        if (pair == null || pair.getSecond() == null) {
+      Pair<HRegionInfo, ServerName> regionServerPair
+        = getRegion(tableNameOrRegionName, ct);
+      if (regionServerPair != null) {
+        if (regionServerPair.getSecond() == null) {
           LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
         } else {
-          flush(pair.getSecond(), pair.getFirst());
+          flush(regionServerPair.getSecond(), regionServerPair.getFirst());
         }
       } else {
         final String tableName = tableNameString(tableNameOrRegionName, ct);
@@ -1285,14 +1286,14 @@ private void compact(final byte [] tableNameOrRegionName, final boolean major)
   throws IOException, InterruptedException {
     CatalogTracker ct = getCatalogTracker();
     try {
-      if (isRegionName(tableNameOrRegionName, ct)) {
-        Pair<HRegionInfo, ServerName> pair =
-          MetaReader.getRegion(ct, tableNameOrRegionName);
-        if (pair == null || pair.getSecond() == null) {
+      Pair<HRegionInfo, ServerName> regionServerPair
+        = getRegion(tableNameOrRegionName, ct);
+      if (regionServerPair != null) {
+        if (regionServerPair.getSecond() == null) {
           LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
         } else {
-          compact(pair.getSecond(), pair.getFirst(), major);
+          compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major);
         }
       } else {
         final String tableName = tableNameString(tableNameOrRegionName, ct);
@@ -1458,15 +1459,14 @@ public void split(final byte [] tableNameOrRegionName,
       final byte [] splitPoint) throws IOException, InterruptedException {
     CatalogTracker ct = getCatalogTracker();
     try {
-      if (isRegionName(tableNameOrRegionName, ct)) {
-        // Its a possible region name.
-        Pair<HRegionInfo, ServerName> pair =
-          MetaReader.getRegion(ct, tableNameOrRegionName);
-        if (pair == null || pair.getSecond() == null) {
+      Pair<HRegionInfo, ServerName> regionServerPair
+        = getRegion(tableNameOrRegionName, ct);
+      if (regionServerPair != null) {
+        if (regionServerPair.getSecond() == null) {
           LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
         } else {
-          split(pair.getSecond(), pair.getFirst(), splitPoint);
+          split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
         }
       } else {
         final String tableName = tableNameString(tableNameOrRegionName, ct);
@@ -1520,20 +1520,46 @@ public void modifyTable(final byte [] tableName, HTableDescriptor htd)
 
   /**
    * @param tableNameOrRegionName Name of a table or name of a region.
-   * @param ct A {@link #CatalogTracker} instance (caller of this method usually has one).
-   * @return True if <code>tableNameOrRegionName</code> is a verified region
-   * name (we call {@link #MetaReader.getRegion(CatalogTracker catalogTracker,
-   * byte [] regionName)};) else false.
+   * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
+   * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
+   *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
+   *  else null.
    * Throw an exception if <code>tableNameOrRegionName</code> is null.
    * @throws IOException
    */
-  private boolean isRegionName(final byte[] tableNameOrRegionName,
-      CatalogTracker ct)
-  throws IOException {
+  Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName,
+      final CatalogTracker ct) throws IOException {
     if (tableNameOrRegionName == null) {
       throw new IllegalArgumentException("Pass a table name or region name");
     }
-    return (MetaReader.getRegion(ct, tableNameOrRegionName) != null);
+    Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
+    if (pair == null) {
+      final AtomicReference<Pair<HRegionInfo, ServerName>> result =
+        new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
+      final String encodedName = Bytes.toString(tableNameOrRegionName);
+      MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
+        @Override
+        public boolean processRow(Result data) throws IOException {
+          if (data == null || data.size() <= 0) {
+            return true;
+          }
+          HRegionInfo info = MetaReader.parseHRegionInfoFromCatalogResult(
+            data, HConstants.REGIONINFO_QUALIFIER);
+          if (info == null) {
+            LOG.warn("No serialized HRegionInfo in " + data);
+            return true;
+          }
+          if (!encodedName.equals(info.getEncodedName())) return true;
+          ServerName sn = MetaReader.getServerNameFromCatalogResult(data);
+          result.set(new Pair<HRegionInfo, ServerName>(info, sn));
+          return false; // found the region, stop
+        }
+      };
+
+      MetaScanner.metaScan(conf, visitor);
+      pair = result.get();
+    }
+    return pair;
   }
 
   /**
@@ -1731,18 +1757,18 @@ public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
     CompactionState state = CompactionState.NONE;
     CatalogTracker ct = getCatalogTracker();
     try {
-      if (isRegionName(tableNameOrRegionName, ct)) {
-        Pair<HRegionInfo, ServerName> pair =
-          MetaReader.getRegion(ct, tableNameOrRegionName);
-        if (pair == null || pair.getSecond() == null) {
+      Pair<HRegionInfo, ServerName> regionServerPair
+        = getRegion(tableNameOrRegionName, ct);
+      if (regionServerPair != null) {
+        if (regionServerPair.getSecond() == null) {
           LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
         } else {
-          ServerName sn = pair.getSecond();
+          ServerName sn = regionServerPair.getSecond();
           HRegionInterface rs =
             this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
           return CompactionState.valueOf(
-            rs.getCompactionState(pair.getFirst().getRegionName()));
+            rs.getCompactionState(regionServerPair.getFirst().getRegionName()));
         }
       } else {
         final String tableName = tableNameString(tableNameOrRegionName, ct);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 53c8671f6b8e..260b17b37062 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -40,6 +40,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.ExecutorService;
@@ -49,6 +50,7 @@
 import org.apache.hadoop.hbase.regionserver.wal.HLogUtilsForTests;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.*;
@@ -1551,7 +1553,29 @@ public void testDisableCatalogTable() throws Exception {
     htd.addFamily(hcd);
     TEST_UTIL.getHBaseAdmin().createTable(htd);
   }
-  
+
+  @Test
+  public void testGetRegion() throws Exception {
+    final String name = "testGetRegion";
+    LOG.info("Started " + name);
+    final byte [] nameBytes = Bytes.toBytes(name);
+    HTable t = TEST_UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
+    TEST_UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
+    CatalogTracker ct = new CatalogTracker(TEST_UTIL.getConfiguration());
+    ct.start();
+    try {
+      HRegionLocation regionLocation = t.getRegionLocation("mmm");
+      HRegionInfo region = regionLocation.getRegionInfo();
+      byte[] regionName = region.getRegionName();
+      Pair<HRegionInfo, ServerName> pair = admin.getRegion(regionName, ct);
+      assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName()));
+      pair = admin.getRegion(region.getEncodedNameAsBytes(), ct);
+      assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName()));
+    } finally {
+      ct.stop();
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From a6c692653387e000daad7a69e689df029340254d Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Tue, 28 Aug 2012 01:55:23 +0000
Subject: [PATCH 0439/1540] HBASE-6586 Quarantine Corrputed HFiles with hbck

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1377942 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/DoNotRetryIOException.java   |   3 +-
 .../apache/hadoop/hbase/HBaseIOException.java |  52 +++
 .../hbase/io/hfile/CorruptHFileException.java |  36 ++
 .../hbase/io/hfile/FixedFileTrailer.java      |  13 +-
 .../apache/hadoop/hbase/io/hfile/HFile.java   |  41 +-
 .../hbase/regionserver/wal/HLogSplitter.java  |   2 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java | 110 ++++++
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 229 ++++++++----
 .../util/hbck/HFileCorruptionChecker.java     | 349 ++++++++++++++++++
 .../hbase/io/hfile/TestFixedFileTrailer.java  |   2 +-
 .../hadoop/hbase/io/hfile/TestHFile.java      |  65 +++-
 .../hadoop/hbase/util/TestHBaseFsck.java      | 197 +++++++++-
 .../hbase/util/hbck/HbckTestingUtil.java      |  17 +
 13 files changed, 1023 insertions(+), 93 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/HBaseIOException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java

diff --git a/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java b/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java
index 98c5b9bb422b..5a0c6d2b6957 100644
--- a/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java
+++ b/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java
@@ -19,13 +19,12 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.io.IOException;
 
 /**
  * Subclass if exception is not meant to be retried: e.g.
  * {@link UnknownScannerException}
  */
-public class DoNotRetryIOException extends IOException {
+public class DoNotRetryIOException extends HBaseIOException {
 
   private static final long serialVersionUID = 1197446454511704139L;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java b/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java
new file mode 100644
index 000000000000..fb1a17df462a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+/**
+ * All hbase specific IOExceptions should be subclasses of HBaseIOException
+ */
+public class HBaseIOException extends IOException {
+
+  private static final long serialVersionUID = 1L;
+
+  public HBaseIOException() {
+    super();
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public HBaseIOException(String message) {
+    super(message);
+  }
+
+  /**
+   * {@inheritDoc}
+   **/
+  public HBaseIOException(String message, Throwable cause) {
+      super(message, cause);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public HBaseIOException(Throwable cause) {
+      super(cause);
+  }}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java
new file mode 100644
index 000000000000..03ca06cabecb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+/**
+ * This exception is thrown when attempts to read an HFile fail due to corruption or truncation
+ * issues.
+ */
+public class CorruptHFileException extends DoNotRetryIOException {
+  private static final long serialVersionUID = 1L;
+
+  CorruptHFileException(String m, Throwable t) {
+    super(m, t);
+  }
+
+  CorruptHFileException(String m) {
+    super(m);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
index 8e3c8219eb95..693643c74e70 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
@@ -19,6 +19,9 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
+import static org.apache.hadoop.hbase.io.hfile.HFile.MAX_FORMAT_VERSION;
+import static org.apache.hadoop.hbase.io.hfile.HFile.MIN_FORMAT_VERSION;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
@@ -33,9 +36,6 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
 
-import static org.apache.hadoop.hbase.io.hfile.HFile.MIN_FORMAT_VERSION;
-import static org.apache.hadoop.hbase.io.hfile.HFile.MAX_FORMAT_VERSION;
-
 import com.google.common.io.NullOutputStream;
 
 /**
@@ -320,12 +320,7 @@ public static FixedFileTrailer readFromStream(FSDataInputStream istream,
     int majorVersion = extractMajorVersion(version);
     int minorVersion = extractMinorVersion(version);
 
-    try {
-      HFile.checkFormatVersion(majorVersion);
-    } catch (IllegalArgumentException iae) {
-      // In this context, an invalid version might indicate a corrupt HFile.
-      throw new IOException(iae);
-    }
+    HFile.checkFormatVersion(majorVersion); // throws IAE if invalid
 
     int trailerSize = getTrailerSize(majorVersion);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 5139162ba34f..c46fcb215b2c 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -29,7 +29,6 @@
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -519,12 +518,32 @@ ByteBuffer getMetaBlock(String metaBlockName,
     DataBlockEncoding getEncodingOnDisk();
   }
 
+  /**
+   * Method returns the reader given the specified arguments.
+   * TODO This is a bad abstraction.  See HBASE-6635.
+   *
+   * @param path hfile's path
+   * @param fsdis an open checksummed stream of path's file
+   * @param fsdisNoFsChecksum an open unchecksummed stream of path's file
+   * @param size max size of the trailer.
+   * @param closeIStream boolean for closing file after the getting the reader version.
+   * @param cacheConf Cache configuation values, cannot be null.
+   * @param preferredEncodingInCache
+   * @param hfs
+   * @return an appropriate instance of HFileReader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
   private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
       FSDataInputStream fsdisNoFsChecksum,
       long size, boolean closeIStream, CacheConfig cacheConf,
       DataBlockEncoding preferredEncodingInCache, HFileSystem hfs)
       throws IOException {
-    FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, size);
+    FixedFileTrailer trailer = null;
+    try {
+      trailer = FixedFileTrailer.readFromStream(fsdis, size);
+    } catch (IllegalArgumentException iae) {
+      throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, iae);
+    }
     switch (trailer.getMajorVersion()) {
     case 1:
       return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
@@ -534,11 +553,18 @@ private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
           size, closeIStream,
           cacheConf, preferredEncodingInCache, hfs);
     default:
-      throw new IOException("Cannot instantiate reader for HFile version " +
-          trailer.getMajorVersion());
+      throw new CorruptHFileException("Invalid HFile version " + trailer.getMajorVersion());
     }
   }
 
+  /**
+   * @param fs A file system
+   * @param path Path to HFile
+   * @param cacheConf Cache configuration for hfile's contents
+   * @param preferredEncodingInCache Preferred in-cache data encoding algorithm.
+   * @return A version specific Hfile Reader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
   public static Reader createReaderWithEncoding(
       FileSystem fs, Path path, CacheConfig cacheConf,
       DataBlockEncoding preferredEncodingInCache) throws IOException {
@@ -565,6 +591,13 @@ public static Reader createReaderWithEncoding(
         preferredEncodingInCache, hfs);
   }
 
+  /**
+   * @param fs filesystem
+   * @param path Path to file to read
+   * @param cacheConf This must not be null.  @see {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}
+   * @return an active Reader instance
+   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
+   */
   public static Reader createReader(
       FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
     return createReaderWithEncoding(fs, path, cacheConf,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index cad382cfc875..ecac91d0b382 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -573,7 +573,7 @@ private static void archiveLogs(
       final List<Path> processedLogs, final Path oldLogDir,
       final FileSystem fs, final Configuration conf) throws IOException {
     final Path corruptDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get(
-        "hbase.regionserver.hlog.splitlog.corrupt.dir", ".corrupt"));
+        "hbase.regionserver.hlog.splitlog.corrupt.dir",  HConstants.CORRUPT_DIR_NAME));
 
     if (!fs.mkdirs(corruptDir)) {
       LOG.info("Unable to mkdir " + corruptDir);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 088a4ba6c490..22f536a5c0e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -29,6 +29,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -42,6 +43,7 @@
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -928,6 +930,114 @@ public static Path getTablePath(Path rootdir, final String tableName) {
     return new Path(rootdir, tableName);
   }
 
+  /**
+   * Filter for all dirs that don't start with '.'
+   */
+  public static class RegionDirFilter implements PathFilter {
+    // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
+    final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
+    final FileSystem fs;
+
+    public RegionDirFilter(FileSystem fs) {
+      this.fs = fs;
+    }
+
+    @Override
+    public boolean accept(Path rd) {
+      if (!regionDirPattern.matcher(rd.getName()).matches()) {
+        return false;
+      }
+
+      try {
+        return fs.getFileStatus(rd).isDir();
+      } catch (IOException ioe) {
+        // Maybe the file was moved or the fs was disconnected.
+        LOG.warn("Skipping file " + rd +" due to IOException", ioe);
+        return false;
+      }
+    }
+  }
+
+  /**
+   * Given a particular table dir, return all the regiondirs inside it, excluding files such as
+   * .tableinfo
+   * @param fs A file system for the Path
+   * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
+   * @return List of paths to valid region directories in table dir.
+   * @throws IOException
+   */
+  public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
+    // assumes we are in a table dir.
+    FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
+    List<Path> regionDirs = new ArrayList<Path>(rds.length);
+    for (FileStatus rdfs: rds) {
+      Path rdPath = rdfs.getPath();
+      regionDirs.add(rdPath);
+    }
+    return regionDirs;
+  }
+
+  /**
+   * Filter for all dirs that are legal column family names.  This is generally used for colfam
+   * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>.
+   */
+  public static class FamilyDirFilter implements PathFilter {
+    final FileSystem fs;
+
+    public FamilyDirFilter(FileSystem fs) {
+      this.fs = fs;
+    }
+
+    @Override
+    public boolean accept(Path rd) {
+      try {
+        // throws IAE if invalid
+        HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
+      } catch (IllegalArgumentException iae) {
+        // path name is an invalid family name and thus is excluded.
+        return false;
+      }
+
+      try {
+        return fs.getFileStatus(rd).isDir();
+      } catch (IOException ioe) {
+        // Maybe the file was moved or the fs was disconnected.
+        LOG.warn("Skipping file " + rd +" due to IOException", ioe);
+        return false;
+      }
+    }
+  }
+
+  /**
+   * Filter for HFiles that excludes reference files.
+   */
+  public static class HFileFilter implements PathFilter {
+    // This pattern will accept 0.90+ style hex hfies files but reject reference files
+    final public static Pattern hfilePattern = Pattern.compile("^([0-9a-f]+)$");
+
+    final FileSystem fs;
+
+    public HFileFilter(FileSystem fs) {
+      this.fs = fs;
+    }
+
+    @Override
+    public boolean accept(Path rd) {
+      if (!hfilePattern.matcher(rd.getName()).matches()) {
+        return false;
+      }
+
+      try {
+        // only files
+        return !fs.getFileStatus(rd).isDir();
+      } catch (IOException ioe) {
+        // Maybe the file was moved or the fs was disconnected.
+        LOG.warn("Skipping file " + rd +" due to IOException", ioe);
+        return false;
+      }
+    }
+  }
+
   /**
    * @param conf
    * @return Returns the filesystem of the hbase rootdir.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index ee2dfc555fb2..346cbecbc29b 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -36,6 +37,7 @@
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -81,6 +83,7 @@
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
@@ -159,8 +162,10 @@ public class HBaseFsck {
   private HConnection connection;
   private HBaseAdmin admin;
   private HTable meta;
-  private ScheduledThreadPoolExecutor executor; // threads to retrieve data from regionservers
+  protected ExecutorService executor; // threads to retrieve data from regionservers
   private long startMillis = System.currentTimeMillis();
+  private HFileCorruptionChecker hfcc;
+  private int retcode = 0;
 
   /***********
    * Options
@@ -237,6 +242,22 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
     executor = new ScheduledThreadPoolExecutor(numThreads);
   }
 
+  /**
+   * Constructor
+   *
+   * @param conf
+   *          Configuration object
+   * @throws MasterNotRunningException
+   *           if the master is not running
+   * @throws ZooKeeperConnectionException
+   *           if unable to connect to ZooKeeper
+   */
+  public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
+      ZooKeeperConnectionException, IOException {
+    this.conf = conf;
+    this.executor = exec;
+  }
+
   /**
    * To repair region consistency, one must call connect() in order to repair
    * online state.
@@ -3082,6 +3103,10 @@ public void includeTable(String table) {
     tablesIncluded.add(table);
   }
 
+  Set<String> getIncludedTables() {
+    return new HashSet<String>(tablesIncluded);
+  }
+
   /**
    * We are interested in only those tables that have not changed their state in
    * META during the last few seconds specified by hbase.admin.fsck.timelag
@@ -3099,7 +3124,27 @@ public void setSidelineDir(String sidelineDir) {
     this.sidelineDir = new Path(sidelineDir);
   }
 
-  protected static void printUsageAndExit() {
+  protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
+    return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles);
+  }
+
+  public HFileCorruptionChecker getHFilecorruptionChecker() {
+    return hfcc;
+  }
+
+  public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
+    this.hfcc = hfcc;
+  }
+
+  public void setRetCode(int code) {
+    this.retcode = code;
+  }
+
+  public int getRetCode() {
+    return retcode;
+  }
+
+  protected HBaseFsck printUsageAndExit() {
     System.err.println("Usage: fsck [opts] {only tables}");
     System.err.println(" where [opts] are:");
     System.err.println("   -help Display help options (this)");
@@ -3113,7 +3158,8 @@ protected static void printUsageAndExit() {
     System.err.println("   -metaonly Only check the state of ROOT and META tables.");
     System.err.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta and root.");
 
-    System.err.println("  Repair options: (expert features, use with caution!)");
+    System.err.println("");
+    System.err.println("  Metadata Repair options: (expert features, use with caution!)");
     System.err.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
     System.err.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
     System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
@@ -3126,16 +3172,25 @@ protected static void printUsageAndExit() {
     System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
     System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
     System.err.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
+
+    System.err.println("");
+    System.err.println("  Datafile Repair options: (expert features, use with caution!)");
+    System.err.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
+    System.err.println("   -sidelineCorruptHfiles  Quarantine corrupted HFiles.  implies -checkCorruptHfiles");
+
     System.err.println("");
+    System.err.println("  Metadata Repair shortcuts");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
     System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
 
-    Runtime.getRuntime().exit(-2);
+    setRetCode(-2);
+    return this;
   }
 
   /**
    * Main program
+   *
    * @param args
    * @throws Exception
    */
@@ -3147,162 +3202,204 @@ public static void main(String[] args) throws Exception {
     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
     conf.set("fs.defaultFS", defaultFs.toString());     // for hadoop 0.21+
     conf.set("fs.default.name", defaultFs.toString());  // for hadoop 0.20
-    HBaseFsck fsck = new HBaseFsck(conf);
+
+    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
+    ExecutorService exec = new ScheduledThreadPoolExecutor(numThreads);
+    HBaseFsck hbck = new HBaseFsck(conf, exec);
+    hbck.exec(exec, args);
+    int retcode = hbck.getRetCode();
+    Runtime.getRuntime().exit(retcode);
+  }
+
+  public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
+    InterruptedException {
     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
 
+    boolean checkCorruptHFiles = false;
+    boolean sidelineCorruptHFiles = false;
+
     // Process command-line args.
     for (int i = 0; i < args.length; i++) {
       String cmd = args[i];
       if (cmd.equals("-help") || cmd.equals("-h")) {
-        printUsageAndExit();
+        return printUsageAndExit();
       } else if (cmd.equals("-details")) {
-        fsck.setDisplayFullReport();
+        setDisplayFullReport();
       } else if (cmd.equals("-timelag")) {
         if (i == args.length - 1) {
           System.err.println("HBaseFsck: -timelag needs a value.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         try {
           long timelag = Long.parseLong(args[i+1]);
-          fsck.setTimeLag(timelag);
+          setTimeLag(timelag);
         } catch (NumberFormatException e) {
           System.err.println("-timelag needs a numeric value.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-sleepBeforeRerun")) {
         if (i == args.length - 1) {
           System.err.println("HBaseFsck: -sleepBeforeRerun needs a value.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         try {
           sleepBeforeRerun = Long.parseLong(args[i+1]);
         } catch (NumberFormatException e) {
           System.err.println("-sleepBeforeRerun needs a numeric value.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-sidelineDir")) {
         if (i == args.length - 1) {
           System.err.println("HBaseFsck: -sidelineDir needs a value.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         i++;
-        fsck.setSidelineDir(args[i]);
+        setSidelineDir(args[i]);
       } else if (cmd.equals("-fix")) {
         System.err.println("This option is deprecated, please use " +
           "-fixAssignments instead.");
-        fsck.setFixAssignments(true);
+        setFixAssignments(true);
       } else if (cmd.equals("-fixAssignments")) {
-        fsck.setFixAssignments(true);
+        setFixAssignments(true);
       } else if (cmd.equals("-fixMeta")) {
-        fsck.setFixMeta(true);
+        setFixMeta(true);
       } else if (cmd.equals("-fixHdfsHoles")) {
-        fsck.setFixHdfsHoles(true);
+        setFixHdfsHoles(true);
       } else if (cmd.equals("-fixHdfsOrphans")) {
-        fsck.setFixHdfsOrphans(true);
+        setFixHdfsOrphans(true);
       } else if (cmd.equals("-fixHdfsOverlaps")) {
-        fsck.setFixHdfsOverlaps(true);
+        setFixHdfsOverlaps(true);
       } else if (cmd.equals("-fixVersionFile")) {
-        fsck.setFixVersionFile(true);
+        setFixVersionFile(true);
       } else if (cmd.equals("-sidelineBigOverlaps")) {
-        fsck.setSidelineBigOverlaps(true);
+        setSidelineBigOverlaps(true);
       } else if (cmd.equals("-fixSplitParents")) {
-        fsck.setFixSplitParents(true);
+        setFixSplitParents(true);
       } else if (cmd.equals("-ignorePreCheckPermission")) {
-        fsck.setIgnorePreCheckPermission(true);
+        setIgnorePreCheckPermission(true);
+      } else if (cmd.equals("-checkCorruptHFiles")) {
+        checkCorruptHFiles = true;
+      } else if (cmd.equals("-sidelineCorruptHFiles")) {
+        sidelineCorruptHFiles = true;
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
-        fsck.setFixHdfsHoles(true);
-        fsck.setFixHdfsOrphans(true);
-        fsck.setFixMeta(true);
-        fsck.setFixAssignments(true);
-        fsck.setFixHdfsOverlaps(true);
-        fsck.setFixVersionFile(true);
-        fsck.setSidelineBigOverlaps(true);
-        fsck.setFixSplitParents(false);
+        setFixHdfsHoles(true);
+        setFixHdfsOrphans(true);
+        setFixMeta(true);
+        setFixAssignments(true);
+        setFixHdfsOverlaps(true);
+        setFixVersionFile(true);
+        setSidelineBigOverlaps(true);
+        setFixSplitParents(false);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
-        fsck.setFixHdfsHoles(true);
-        fsck.setFixHdfsOrphans(false);
-        fsck.setFixMeta(true);
-        fsck.setFixAssignments(true);
-        fsck.setFixHdfsOverlaps(false);
-        fsck.setSidelineBigOverlaps(false);
-        fsck.setFixSplitParents(false);
+        setFixHdfsHoles(true);
+        setFixHdfsOrphans(false);
+        setFixMeta(true);
+        setFixAssignments(true);
+        setFixHdfsOverlaps(false);
+        setSidelineBigOverlaps(false);
+        setFixSplitParents(false);
       } else if (cmd.equals("-maxOverlapsToSideline")) {
         if (i == args.length - 1) {
           System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         try {
           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
-          fsck.setMaxOverlapsToSideline(maxOverlapsToSideline);
+          setMaxOverlapsToSideline(maxOverlapsToSideline);
         } catch (NumberFormatException e) {
           System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-maxMerge")) {
         if (i == args.length - 1) {
           System.err.println("-maxMerge needs a numeric value argument.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         try {
           int maxMerge = Integer.parseInt(args[i+1]);
-          fsck.setMaxMerge(maxMerge);
+          setMaxMerge(maxMerge);
         } catch (NumberFormatException e) {
           System.err.println("-maxMerge needs a numeric value argument.");
-          printUsageAndExit();
+          return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-summary")) {
-        fsck.setSummary();
+        setSummary();
       } else if (cmd.equals("-metaonly")) {
-        fsck.setCheckMetaOnly();
+        setCheckMetaOnly();
       } else if (cmd.startsWith("-")) {
         System.err.println("Unrecognized option:" + cmd);
-        printUsageAndExit();
+        return printUsageAndExit();
       } else {
-        fsck.includeTable(cmd);
+        includeTable(cmd);
         System.out.println("Allow checking/fixes for table: " + cmd);
       }
     }
 
     // pre-check current user has FS write permission or not
     try {
-      fsck.preCheckPermission();
+      preCheckPermission();
     } catch (AccessControlException ace) {
       Runtime.getRuntime().exit(-1);
     } catch (IOException ioe) {
       Runtime.getRuntime().exit(-1);
     }
-    // do the real work of fsck
-    fsck.connect();
-    int code = fsck.onlineHbck();
-    // If we have changed the HBase state it is better to run fsck again
+
+    // do the real work of hbck
+    connect();
+
+    // if corrupt file mode is on, first fix them since they may be opened later
+    if (checkCorruptHFiles || sidelineCorruptHFiles) {
+      LOG.info("Checking all hfiles for corruption");
+      HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
+      setHFileCorruptionChecker(hfcc); // so we can get result
+      Collection<String> tables = getIncludedTables();
+      Collection<Path> tableDirs = new ArrayList<Path>();
+      Path rootdir = FSUtils.getRootDir(conf);
+      if (tables.size() > 0) {
+        for (String t : tables) {
+          tableDirs.add(FSUtils.getTablePath(rootdir, t));
+        }
+      } else {
+        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(conf), rootdir);
+      }
+      hfcc.checkTables(tableDirs);
+      PrintWriter out = new PrintWriter(System.out);
+      hfcc.report(out);
+      out.flush();
+    }
+
+    // check and fix table integrity, region consistency.
+    int code = onlineHbck();
+    setRetCode(code);
+    // If we have changed the HBase state it is better to run hbck again
     // to see if we haven't broken something else in the process.
     // We run it only once more because otherwise we can easily fall into
     // an infinite loop.
-    if (fsck.shouldRerun()) {
+    if (shouldRerun()) {
       try {
         LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
         Thread.sleep(sleepBeforeRerun);
       } catch (InterruptedException ie) {
-        Runtime.getRuntime().exit(code);
+        return this;
       }
       // Just report
-      fsck.setFixAssignments(false);
-      fsck.setFixMeta(false);
-      fsck.setFixHdfsHoles(false);
-      fsck.setFixHdfsOverlaps(false);
-      fsck.setFixVersionFile(false);
-      fsck.errors.resetErrors();
-      code = fsck.onlineHbck();
-    }
-
-    Runtime.getRuntime().exit(code);
+      setFixAssignments(false);
+      setFixMeta(false);
+      setFixHdfsHoles(false);
+      setFixHdfsOverlaps(false);
+      setFixVersionFile(false);
+      errors.resetErrors();
+      code = onlineHbck();
+      setRetCode(code);
+    }
+    return this;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
new file mode 100644
index 000000000000..7fb876d8e72a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util.hbck;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter;
+import org.apache.hadoop.hbase.util.FSUtils.HFileFilter;
+import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
+
+/**
+ * This class marches through all of the region's hfiles and verifies that
+ * they are all valid files. One just needs to instantiate the class, use
+ * checkTables(List<Path>) and then retrieve the corrupted hfiles (and
+ * quarantined files if in quarantining mode)
+ *
+ * The implementation currently parallelizes at the regionDir level.
+ */
+public class HFileCorruptionChecker {
+  private static final Log LOG = LogFactory.getLog(HFileCorruptionChecker.class);
+
+  final Configuration conf;
+  final FileSystem fs;
+  final CacheConfig cacheConf;
+  final ExecutorService executor;
+  final Set<Path> corrupted = new ConcurrentSkipListSet<Path>();
+  final Set<Path> failures = new ConcurrentSkipListSet<Path>();
+  final Set<Path> quarantined = new ConcurrentSkipListSet<Path>();
+  final Set<Path> missing = new ConcurrentSkipListSet<Path>();
+  final boolean inQuarantineMode;
+  final AtomicInteger hfilesChecked = new AtomicInteger();
+
+  public HFileCorruptionChecker(Configuration conf, ExecutorService executor,
+      boolean quarantine) throws IOException {
+    this.conf = conf;
+    this.fs = FileSystem.get(conf);
+    this.cacheConf = new CacheConfig(conf);
+    this.executor = executor;
+    this.inQuarantineMode = quarantine;
+  }
+
+  /**
+   * Checks a path to see if it is a valid hfile.
+   *
+   * @param p
+   *          full Path to an HFile
+   * @throws IOException
+   *           This is a connectivity related exception
+   */
+  protected void checkHFile(Path p) throws IOException {
+    HFile.Reader r = null;
+    try {
+      r = HFile.createReader(fs, p, cacheConf);
+    } catch (CorruptHFileException che) {
+      LOG.warn("Found corrupt HFile " + p, che);
+      corrupted.add(p);
+      if (inQuarantineMode) {
+        Path dest = createQuarantinePath(p);
+        LOG.warn("Quarantining corrupt HFile " + p + " into " + dest);
+        boolean success = fs.mkdirs(dest.getParent());
+        success = success ? fs.rename(p, dest): false;
+        if (!success) {
+          failures.add(p);
+        } else {
+          quarantined.add(dest);
+        }
+      }
+      return;
+    } catch (FileNotFoundException fnfe) {
+      LOG.warn("HFile " + p + " was missing.  Likely removed due to compaction/split?");
+      missing.add(p);
+    } finally {
+      hfilesChecked.addAndGet(1);
+      if (r != null) {
+        r.close(true);
+      }
+    }
+  }
+
+  /**
+   * Given a path, generates a new path to where we move a corrupted hfile (bad
+   * trailer, no trailer).
+   *
+   * @param hFile
+   *          Path to a corrupt hfile (assumes that it is HBASE_DIR/ table
+   *          /region/cf/file)
+   * @return path to where corrupted files are stored. This should be
+   *         HBASE_DIR/.corrupt/table/region/cf/file.
+   */
+  Path createQuarantinePath(Path hFile) {
+    // extract the normal dirs structure
+    Path cfDir = hFile.getParent();
+    Path regionDir = cfDir.getParent();
+    Path tableDir = regionDir.getParent();
+
+    // build up the corrupted dirs strcture
+    Path corruptBaseDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get(
+        "hbase.hfile.quarantine.dir", HConstants.CORRUPT_DIR_NAME));
+    Path corruptTableDir = new Path(corruptBaseDir, tableDir.getName());
+    Path corruptRegionDir = new Path(corruptTableDir, regionDir.getName());
+    Path corruptFamilyDir = new Path(corruptRegionDir, cfDir.getName());
+    Path corruptHfile = new Path(corruptFamilyDir, hFile.getName());
+    return corruptHfile;
+  }
+
+  /**
+   * Check all files in a column family dir.
+   *
+   * @param cfDir
+   *          column family directory
+   * @throws IOException
+   */
+  protected void checkColFamDir(Path cfDir) throws IOException {
+    FileStatus[] hfs = fs.listStatus(cfDir, new HFileFilter(fs)); // use same filter as scanner.
+    if (hfs.length == 0 && !fs.exists(cfDir)) {
+      // interestingly, listStatus does not throw an exception if the path does not exist.
+      LOG.warn("Colfam Directory " + cfDir +
+          " does not exist.  Likely due to concurrent split/compaction. Skipping.");
+      missing.add(cfDir);
+      return;
+    }
+    for (FileStatus hfFs : hfs) {
+      Path hf = hfFs.getPath();
+      checkHFile(hf);
+    }
+  }
+
+  /**
+   * Check all column families in a region dir.
+   *
+   * @param regionDir
+   *          region directory
+   * @throws IOException
+   */
+  protected void checkRegionDir(Path regionDir) throws IOException {
+    FileStatus[] cfs = fs.listStatus(regionDir, new FamilyDirFilter(fs));
+    if (cfs.length == 0 && !fs.exists(regionDir)) {
+      // interestingly, listStatus does not throw an exception if the path does not exist.
+      LOG.warn("Region Directory " + regionDir +
+          " does not exist.  Likely due to concurrent split/compaction. Skipping.");
+      missing.add(regionDir);
+      return;
+    }
+
+    for (FileStatus cfFs : cfs) {
+      Path cfDir = cfFs.getPath();
+      checkColFamDir(cfDir);
+    }
+  }
+
+  /**
+   * Check all the regiondirs in the specified tableDir
+   *
+   * @param tableDir
+   *          path to a table
+   * @throws IOException
+   */
+  void checkTableDir(Path tableDir) throws IOException {
+    FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
+    if (rds.length == 0 && !fs.exists(tableDir)) {
+      // interestingly listStatus does not throw an exception if the path does not exist.
+      LOG.warn("Table Directory " + tableDir +
+          " does not exist.  Likely due to concurrent delete. Skipping.");
+      missing.add(tableDir);
+      return;
+    }
+
+    // Parallelize check at the region dir level
+    List<RegionDirChecker> rdcs = new ArrayList<RegionDirChecker>();
+    List<Future<Void>> rdFutures;
+
+    for (FileStatus rdFs : rds) {
+      Path rdDir = rdFs.getPath();
+      RegionDirChecker work = new RegionDirChecker(rdDir);
+      rdcs.add(work);
+    }
+
+    // Submit and wait for completion
+    try {
+      rdFutures = executor.invokeAll(rdcs);
+    } catch (InterruptedException ie) {
+      Thread.currentThread().interrupt();
+      LOG.warn("Region dirs checking interrupted!", ie);
+      return;
+    }
+
+    for (int i = 0; i < rdFutures.size(); i++) {
+      Future<Void> f = rdFutures.get(i);
+      try {
+        f.get();
+      } catch (ExecutionException e) {
+        LOG.warn("Failed to quaratine an HFile in regiondir "
+            + rdcs.get(i).regionDir, e.getCause());
+        // rethrow IOExceptions
+        if (e.getCause() instanceof IOException) {
+          throw (IOException) e.getCause();
+        }
+
+        // rethrow RuntimeExceptions
+        if (e.getCause() instanceof RuntimeException) {
+          throw (RuntimeException) e.getCause();
+        }
+
+        // this should never happen
+        LOG.error("Unexpected exception encountered", e);
+        return; // bailing out.
+      } catch (InterruptedException ie) {
+        Thread.currentThread().interrupt();
+        LOG.warn("Region dirs check interrupted!", ie);
+        // bailing out
+        return;
+      }
+    }
+  }
+
+  /**
+   * An individual work item for parallelized regiondir processing. This is
+   * intentionally an inner class so it can use the shared error sets and fs.
+   */
+  private class RegionDirChecker implements Callable<Void> {
+    final Path regionDir;
+
+    RegionDirChecker(Path regionDir) {
+      this.regionDir = regionDir;
+    }
+
+    @Override
+    public Void call() throws IOException {
+      checkRegionDir(regionDir);
+      return null;
+    }
+  }
+
+  /**
+   * Check the specified table dirs for bad hfiles.
+   */
+  public void checkTables(Collection<Path> tables) throws IOException {
+    for (Path t : tables) {
+      checkTableDir(t);
+    }
+  }
+
+  /**
+   * @return the set of check failure file paths after checkTables is called.
+   */
+  public Collection<Path> getFailures() {
+    return new HashSet<Path>(failures);
+  }
+
+  /**
+   * @return the set of corrupted file paths after checkTables is called.
+   */
+  public Collection<Path> getCorrupted() {
+    return new HashSet<Path>(corrupted);
+  }
+
+  /**
+   * @return number of hfiles checked in the last HfileCorruptionChecker run
+   */
+  public int getHFilesChecked() {
+    return hfilesChecked.get();
+  }
+
+  /**
+   * @return the set of successfully quarantined paths after checkTables is called.
+   */
+  public Collection<Path> getQuarantined() {
+    return new HashSet<Path>(quarantined);
+  }
+
+  /**
+   * @return the set of paths that were missing.  Likely due to deletion/moves from
+   *  compaction or flushes.
+   */
+  public Collection<Path> getMissing() {
+    return new HashSet<Path>(missing);
+  }
+
+  /**
+   * Print a human readable summary of hfile quarantining operations.
+   * @param out
+   */
+  public void report(PrintWriter out) {
+    out.println("Checked " + hfilesChecked.get() + " hfile for corruption");
+    out.println("  HFiles corrupted:                  " + corrupted.size());
+    if (inQuarantineMode) {
+      out.println("    HFiles successfully quarantined: " + quarantined.size());
+      for (Path sq : quarantined) {
+        out.println("      " + sq);
+      }
+      out.println("    HFiles failed quarantine:        " + failures.size());
+      for (Path fq : failures) {
+        out.println("      " + fq);
+      }
+    }
+    out.println("    HFiles moved while checking:     " + missing.size());
+    for (Path mq : missing) {
+      out.println("      " + mq);
+    }
+
+    String initialState = (corrupted.size() == 0) ? "OK" : "CORRUPTED";
+    String fixedState = (corrupted.size() == quarantined.size()) ? "OK"
+        : "CORRUPTED";
+
+    if (inQuarantineMode) {
+      out.println("Summary: " + initialState + " => " + fixedState);
+    } else {
+      out.println("Summary: " + initialState);
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
index a9288ce979eb..7dd63d53591f 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
@@ -141,7 +141,7 @@ public void testTrailer() throws IOException {
         try {
           readTrailer(trailerPath);
           fail("Exception expected");
-        } catch (IOException ex) {
+        } catch (IllegalArgumentException ex) {
           // Make it easy to debug this.
           String msg = ex.getMessage();
           String cleanMsg = msg.replaceAll(
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
index ca2941613934..44dfb7388445 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
@@ -30,12 +30,15 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
-import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Writable;
 import org.junit.experimental.categories.Category;
@@ -56,11 +59,9 @@ public class TestHFile extends HBaseTestCase {
   private final int minBlockSize = 512;
   private static String localFormatter = "%010d";
   private static CacheConfig cacheConf = null;
-  private Map<String, Long> startingMetrics;
 
   @Override
   public void setUp() throws Exception {
-    startingMetrics = SchemaMetrics.getMetricsSnapshot();
     ROOT_DIR = this.getUnitTestdir("TestHFile").toString();
     super.setUp();
   }
@@ -68,7 +69,6 @@ public void setUp() throws Exception {
   @Override
   public void tearDown() throws Exception {
     super.tearDown();
-    SchemaMetrics.validateMetricChanges(startingMetrics);
   }
 
 
@@ -89,6 +89,61 @@ public void testEmptyHFile() throws IOException {
     assertNull(r.getLastKey());
   }
 
+  /**
+   * Create 0-length hfile and show that it fails
+   */
+  public void testCorrupt0LengthHFile() throws IOException {
+    if (cacheConf == null) cacheConf = new CacheConfig(conf);
+    Path f = new Path(ROOT_DIR, getName());
+    FSDataOutputStream fsos = fs.create(f);
+    fsos.close();
+
+    try {
+      Reader r = HFile.createReader(fs, f, cacheConf);
+    } catch (CorruptHFileException che) {
+      // Expected failure
+      return;
+    }
+    fail("Should have thrown exception");
+  }
+
+  public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
+    FileStatus fst = fs.getFileStatus(src);
+    long len = fst.getLen();
+    len = len / 2 ;
+
+    // create a truncated hfile
+    FSDataOutputStream fdos = fs.create(dst);
+    byte[] buf = new byte[(int)len];
+    FSDataInputStream fdis = fs.open(src);
+    fdis.read(buf);
+    fdos.write(buf);
+    fdis.close();
+    fdos.close();
+  }
+
+  /**
+   * Create a truncated hfile and verify that exception thrown.
+   */
+  public void testCorruptTruncatedHFile() throws IOException {
+    if (cacheConf == null) cacheConf = new CacheConfig(conf);
+    Path f = new Path(ROOT_DIR, getName());
+    Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
+    writeSomeRecords(w, 0, 100);
+    w.close();
+
+    Path trunc = new Path(f.getParent(), "trucated");
+    truncateFile(fs, w.getPath(), trunc);
+
+    try {
+      Reader r = HFile.createReader(fs, trunc, cacheConf);
+    } catch (CorruptHFileException che) {
+      // Expected failure
+      return;
+    }
+    fail("Should have thrown exception");
+  }
+
   // write some records into the tfile
   // write them twice
   private int writeSomeRecords(Writer writer, int start, int n)
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index bc34d747e704..e99e59f55d2b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -35,10 +35,13 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.ClusterStatus;
@@ -48,7 +51,7 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
@@ -62,12 +65,15 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.io.hfile.TestHFile;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
+import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
+import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -75,19 +81,21 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
 
 import com.google.common.collect.Multimap;
 
 /**
  * This tests HBaseFsck's ability to detect reasons for inconsistent tables.
  */
-@Category(MediumTests.class)
+@Category(LargeTests.class)
 public class TestHBaseFsck {
   final static Log LOG = LogFactory.getLog(TestHBaseFsck.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final static Configuration conf = TEST_UTIL.getConfiguration();
-  private final static byte[] FAM = Bytes.toBytes("fam");
-  private final static int REGION_ONLINE_TIMEOUT = 300;
+  private final static String FAM_STR = "fam";
+  private final static byte[] FAM = Bytes.toBytes(FAM_STR);
+  private final static int REGION_ONLINE_TIMEOUT = 800;
 
   // for the instance, reset every test run
   private HTable tbl;
@@ -1255,8 +1263,187 @@ public void testMissingLastRegion() throws Exception {
       deleteTable(table);
     }
   }
-  
+
+  /**
+   * We don't have an easy way to verify that a flush completed, so we loop until we find a
+   * legitimate hfile and return it.
+   * @param fs
+   * @param table
+   * @return Path of a flushed hfile.
+   * @throws IOException
+   */
+  Path getFlushedHFile(FileSystem fs, String table) throws IOException {
+    Path tableDir= FSUtils.getTablePath(FSUtils.getRootDir(conf), table);
+    Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
+    Path famDir = new Path(regionDir, FAM_STR);
+
+    // keep doing this until we get a legit hfile
+    while (true) {
+      FileStatus[] hfFss = fs.listStatus(famDir);
+      if (hfFss.length == 0) {
+        continue;
+      }
+      for (FileStatus hfs : hfFss) {
+        if (!hfs.isDir()) {
+          return hfs.getPath();
+        }
+      }
+    }
+  }
+
+  /**
+   * This creates a table and then corrupts an hfile.  Hbck should quarantine the file.
+   */
+  @Test(timeout=120000)
+  public void testQuarantineCorruptHFile() throws Exception {
+    String table = name.getMethodName();
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+      TEST_UTIL.getHBaseAdmin().flush(table); // flush is async.
+
+      FileSystem fs = FileSystem.get(conf);
+      Path hfile = getFlushedHFile(fs, table);
+
+      // Mess it up by leaving a hole in the assignment, meta, and hdfs data
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+
+      // create new corrupt file called deadbeef (valid hfile name)
+      Path corrupt = new Path(hfile.getParent(), "deadbeef");
+      TestHFile.truncateFile(fs, hfile, corrupt);
+      LOG.info("Created corrupted file " + corrupt);
+      HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
+
+      // we cannot enable here because enable never finished due to the corrupt region.
+      HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
+      assertEquals(res.getRetCode(), 0);
+      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
+      assertEquals(hfcc.getHFilesChecked(), 5);
+      assertEquals(hfcc.getCorrupted().size(), 1);
+      assertEquals(hfcc.getFailures().size(), 0);
+      assertEquals(hfcc.getQuarantined().size(), 1);
+      assertEquals(hfcc.getMissing().size(), 0);
+
+      // Its been fixed, verify that we can enable.
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  private void doQuarantineTest(String table, HBaseFsck hbck, int check, int corrupt, int fail,
+      int quar, int missing) throws Exception {
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+      TEST_UTIL.getHBaseAdmin().flush(table); // flush is async.
+
+      // Mess it up by leaving a hole in the assignment, meta, and hdfs data
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      
+      String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission", table};
+      ExecutorService exec = new ScheduledThreadPoolExecutor(10);
+      HBaseFsck res = hbck.exec(exec, args);
+
+      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
+      assertEquals(hfcc.getHFilesChecked(), check);
+      assertEquals(hfcc.getCorrupted().size(), corrupt);
+      assertEquals(hfcc.getFailures().size(), fail);
+      assertEquals(hfcc.getQuarantined().size(), quar);
+      assertEquals(hfcc.getMissing().size(), missing);
+
+      // its been fixed, verify that we can enable
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * This creates a table and simulates the race situation where a concurrent compaction or split
+   * has removed an hfile after the corruption checker learned about it.
+   */
+  @Test(timeout=120000)
+  public void testQuarantineMissingHFile() throws Exception {
+    String table = name.getMethodName();
+    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
+    // inject a fault in the hfcc created.
+    final FileSystem fs = FileSystem.get(conf);
+    HBaseFsck hbck = new HBaseFsck(conf, exec) {
+      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
+        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
+          boolean attemptedFirstHFile = false;
+          protected void checkHFile(Path p) throws IOException {
+            if (!attemptedFirstHFile) {
+              attemptedFirstHFile = true;
+              assertTrue(fs.delete(p, true)); // make sure delete happened.
+            }
+            super.checkHFile(p);
+          }
+        };
+      }
+    };
+    doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
+  }
+
+  /**
+   * This creates a table and simulates the race situation where a concurrent compaction or split
+   * has removed an colfam dir before the corruption checker got to it.
+   */
+  @Test(timeout=120000)
+  public void testQuarantineMissingFamdir() throws Exception {
+    String table = name.getMethodName();
+    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
+    // inject a fault in the hfcc created.
+    final FileSystem fs = FileSystem.get(conf);
+    HBaseFsck hbck = new HBaseFsck(conf, exec) {
+      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
+        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
+          boolean attemptedFirstFamDir = false;
+          protected void checkColFamDir(Path p) throws IOException {
+            if (!attemptedFirstFamDir) {
+              attemptedFirstFamDir = true;
+              assertTrue(fs.delete(p, true)); // make sure delete happened.
+            }
+            super.checkColFamDir(p);
+          }
+        };
+      }
+    };
+    doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
+  }
+
+  /**
+   * This creates a table and simulates the race situation where a concurrent compaction or split
+   * has removed a region dir before the corruption checker got to it.
+   */
+  @Test(timeout=120000)
+  public void testQuarantineMissingRegionDir() throws Exception {
+    String table = name.getMethodName();
+    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
+    // inject a fault in the hfcc created.
+    final FileSystem fs = FileSystem.get(conf);
+    HBaseFsck hbck = new HBaseFsck(conf, exec) {
+      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
+        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
+          boolean attemptedFirstRegionDir = false;
+          protected void checkRegionDir(Path p) throws IOException {
+            if (!attemptedFirstRegionDir) {
+              attemptedFirstRegionDir = true;
+              assertTrue(fs.delete(p, true)); // make sure delete happened.
+            }
+            super.checkRegionDir(p);
+          }
+        };
+      }
+    };
+    doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+  @org.junit.Rule
+  public TestName name = new TestName();
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index f4c598378966..b25c4cb60343 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -22,6 +22,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.util.HBaseFsck;
@@ -59,6 +61,21 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
     return fsck;
   }
 
+  /**
+   * Runs hbck with the -sidelineCorruptHFiles option
+   * @param conf
+   * @param table table constraint
+   * @return <returncode, hbckInstance>
+   * @throws Exception
+   */
+  public static HBaseFsck doHFileQuarantine(Configuration conf, String table) throws Exception {
+    String[] args = {"-sidelineCorruptHFiles", "-ignorePreCheckPermission", table};
+    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
+    HBaseFsck hbck = new HBaseFsck(conf, exec);
+    hbck.exec(exec, args);
+    return hbck;
+  }
+
   public static void assertNoErrors(HBaseFsck fsck) throws Exception {
     List<ERROR_CODE> errs = fsck.getErrors().getErrorList();
     assertEquals(new ArrayList<ERROR_CODE>(), errs);

From 0a301d0d7082f1a0b490928bded50a765016d020 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 28 Aug 2012 18:54:58 +0000
Subject: [PATCH 0440/1540] HBASE-6671 Kerberos authenticated super user should
 be able to retrieve proxied delegation tokens (Francis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1378268 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       |  2 +-
 .../hbase/security/token/TokenProvider.java   | 21 +++++++++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index 07b48d09da86..23b3fd2d8cf1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1844,7 +1844,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>0.23.2-SNAPSHOT</hadoop.version>
+        <hadoop.version>0.23.3-SNAPSHOT</hadoop.version>
         <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencies>
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java
index 0e7e87267b53..0a3a3a67ab18 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.Token;
 
@@ -80,8 +81,7 @@ public Token<AuthenticationTokenIdentifier> getAuthenticationToken()
     }
     if (currentUser == null) {
       throw new AccessDeniedException("No authenticated user for request!");
-    } else if (ugi.getAuthenticationMethod() !=
-        UserGroupInformation.AuthenticationMethod.KERBEROS) {
+    } else if (!isAllowedDelegationTokenOp(ugi)) {
       LOG.warn("Token generation denied for user="+currentUser.getName()
           +", authMethod="+ugi.getAuthenticationMethod());
       throw new AccessDeniedException(
@@ -91,6 +91,23 @@ public Token<AuthenticationTokenIdentifier> getAuthenticationToken()
     return secretManager.generateToken(currentUser.getName());
   }
 
+  /**
+   * @param ugi
+   * @return true if delegation token operation is allowed
+   */
+  private boolean isAllowedDelegationTokenOp(UserGroupInformation ugi) throws IOException {
+    AuthenticationMethod authMethod = ugi.getAuthenticationMethod();
+    if (authMethod == AuthenticationMethod.PROXY) {
+      authMethod = ugi.getRealUser().getAuthenticationMethod();
+    }
+    if (authMethod != AuthenticationMethod.KERBEROS
+        && authMethod != AuthenticationMethod.KERBEROS_SSL
+        && authMethod != AuthenticationMethod.CERTIFICATE) {
+      return false;
+    }
+    return true;
+  }
+
   @Override
   public String whoami() {
     return RequestContext.getRequestUserName();

From f33dbf55587e8acea678f92a79a37dab2d77b217 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 29 Aug 2012 22:57:11 +0000
Subject: [PATCH 0441/1540] HBASE-6686 HFile Quarantine fails with missing dirs
 in hadoop 2.0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1378763 13f79535-47bb-0310-9956-ffa450edef68
---
 .../util/hbck/HFileCorruptionChecker.java     | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
index 7fb876d8e72a..f96024cd2834 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
@@ -149,9 +149,19 @@ Path createQuarantinePath(Path hFile) {
    * @throws IOException
    */
   protected void checkColFamDir(Path cfDir) throws IOException {
-    FileStatus[] hfs = fs.listStatus(cfDir, new HFileFilter(fs)); // use same filter as scanner.
+    FileStatus[] hfs = null;
+    try {
+      hfs = fs.listStatus(cfDir, new HFileFilter(fs)); // use same filter as scanner.
+    } catch (FileNotFoundException fnfe) {
+      // Hadoop 0.23+ listStatus semantics throws an exception if the path does not exist.
+      LOG.warn("Colfam Directory " + cfDir +
+          " does not exist.  Likely due to concurrent split/compaction. Skipping.");
+      missing.add(cfDir);
+      return;
+    }
+
+    // Hadoop 1.0 listStatus does not throw an exception if the path does not exist.
     if (hfs.length == 0 && !fs.exists(cfDir)) {
-      // interestingly, listStatus does not throw an exception if the path does not exist.
       LOG.warn("Colfam Directory " + cfDir +
           " does not exist.  Likely due to concurrent split/compaction. Skipping.");
       missing.add(cfDir);
@@ -171,9 +181,19 @@ protected void checkColFamDir(Path cfDir) throws IOException {
    * @throws IOException
    */
   protected void checkRegionDir(Path regionDir) throws IOException {
-    FileStatus[] cfs = fs.listStatus(regionDir, new FamilyDirFilter(fs));
+    FileStatus[] cfs = null;
+    try {
+      cfs = fs.listStatus(regionDir, new FamilyDirFilter(fs));
+    } catch (FileNotFoundException fnfe) {
+      // Hadoop 0.23+ listStatus semantics throws an exception if the path does not exist.
+      LOG.warn("Region Directory " + regionDir +
+          " does not exist.  Likely due to concurrent split/compaction. Skipping.");
+      missing.add(regionDir);
+      return;
+    }
+
+    // Hadoop 1.0 listStatus does not throw an exception if the path does not exist.
     if (cfs.length == 0 && !fs.exists(regionDir)) {
-      // interestingly, listStatus does not throw an exception if the path does not exist.
       LOG.warn("Region Directory " + regionDir +
           " does not exist.  Likely due to concurrent split/compaction. Skipping.");
       missing.add(regionDir);

From 29a60730c86c7bed80c7811845080ec25f8fd360 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 30 Aug 2012 05:11:00 +0000
Subject: [PATCH 0442/1540] HBASE-662 [shell] Cannot describe '.META.' or
 '-ROOT-' tables

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1378802 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HBaseAdmin.java   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index abfe10d809b5..b87ad7e3f6e5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -897,7 +897,9 @@ public boolean isTableEnabled(String tableName) throws IOException {
    * @throws IOException if a remote or network exception occurs
    */
   public boolean isTableEnabled(byte[] tableName) throws IOException {
-    HTableDescriptor.isLegalTableName(tableName);
+    if (!HTableDescriptor.isMetaTable(tableName)) {
+      HTableDescriptor.isLegalTableName(tableName);
+    }
     return connection.isTableEnabled(tableName);
   }
 
@@ -916,7 +918,9 @@ public boolean isTableDisabled(final String tableName) throws IOException {
    * @throws IOException if a remote or network exception occurs
    */
   public boolean isTableDisabled(byte[] tableName) throws IOException {
-    HTableDescriptor.isLegalTableName(tableName);
+    if (!HTableDescriptor.isMetaTable(tableName)) {
+      HTableDescriptor.isLegalTableName(tableName);
+    }
     return connection.isTableDisabled(tableName);
   }
 

From 6b039024c6a771a360b1e77946e72423e1d6cfd2 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 30 Aug 2012 05:29:39 +0000
Subject: [PATCH 0443/1540] HBASE-5042 TestReadWriteConsistencyControl should
 be renamed

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1378805 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestReadWriteConsistencyControl.java      | 136 ------------------
 1 file changed, 136 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java
index fd4583360daf..e69de29bb2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java
@@ -1,136 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import junit.framework.TestCase;
-import org.apache.hadoop.hbase.SmallTests;
-import org.junit.experimental.categories.Category;
-
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-@Category(SmallTests.class)
-public class TestReadWriteConsistencyControl extends TestCase {
-  static class Writer implements Runnable {
-    final AtomicBoolean finished;
-    final MultiVersionConsistencyControl mvcc;
-    final AtomicBoolean status;
-
-    Writer(AtomicBoolean finished, MultiVersionConsistencyControl mvcc, AtomicBoolean status) {
-      this.finished = finished;
-      this.mvcc = mvcc;
-      this.status = status;
-    }
-    private Random rnd = new Random();
-    public boolean failed = false;
-
-    public void run() {
-      while (!finished.get()) {
-        MultiVersionConsistencyControl.WriteEntry e = mvcc.beginMemstoreInsert();
-//        System.out.println("Begin write: " + e.getWriteNumber());
-        // 10 usec - 500usec (including 0)
-        int sleepTime = rnd.nextInt(500);
-        // 500 * 1000 = 500,000ns = 500 usec
-        // 1 * 100 = 100ns = 1usec
-        try {
-          if (sleepTime > 0)
-            Thread.sleep(0, sleepTime * 1000);
-        } catch (InterruptedException e1) {
-        }
-        try {
-          mvcc.completeMemstoreInsert(e);
-        } catch (RuntimeException ex) {
-          // got failure
-          System.out.println(ex.toString());
-          ex.printStackTrace();
-          status.set(false);
-          return;
-          // Report failure if possible.
-        }
-      }
-    }
-  }
-
-  public void testParallelism() throws Exception {
-    final MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
-
-    final AtomicBoolean finished = new AtomicBoolean(false);
-
-    // fail flag for the reader thread
-    final AtomicBoolean readerFailed = new AtomicBoolean(false);
-    final AtomicLong failedAt = new AtomicLong();
-    Runnable reader = new Runnable() {
-      public void run() {
-        long prev = mvcc.memstoreReadPoint();
-        while (!finished.get()) {
-          long newPrev = mvcc.memstoreReadPoint();
-          if (newPrev < prev) {
-            // serious problem.
-            System.out.println("Reader got out of order, prev: " +
-            prev + " next was: " + newPrev);
-            readerFailed.set(true);
-            // might as well give up
-            failedAt.set(newPrev);
-            return;
-          }
-        }
-      }
-    };
-
-    // writer thread parallelism.
-    int n = 20;
-    Thread [] writers = new Thread[n];
-    AtomicBoolean [] statuses = new AtomicBoolean[n];
-    Thread readThread = new Thread(reader);
-
-    for (int i = 0 ; i < n ; ++i ) {
-      statuses[i] = new AtomicBoolean(true);
-      writers[i] = new Thread(new Writer(finished, mvcc, statuses[i]));
-      writers[i].start();
-    }
-    readThread.start();
-
-    try {
-      Thread.sleep(10 * 1000);
-    } catch (InterruptedException ex) {
-    }
-
-    finished.set(true);
-
-    readThread.join();
-    for (int i = 0; i < n; ++i) {
-      writers[i].join();
-    }
-
-    // check failure.
-    assertFalse(readerFailed.get());
-    for (int i = 0; i < n; ++i) {
-      assertTrue(statuses[i].get());
-    }
-
-
-  }
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
-

From c5afcd9e35655537a9b9af0dc8ec8a2801ec21ce Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 30 Aug 2012 05:30:23 +0000
Subject: [PATCH 0444/1540] HBASE-5042 TestReadWriteConsistencyControl should
 be renamed

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1378806 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestMultiVersionConsistencyControl.java   | 134 ++++++++++++++++++
 .../TestReadWriteConsistencyControl.java      |   0
 2 files changed, 134 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java
new file mode 100644
index 000000000000..7908b3001e13
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.experimental.categories.Category;
+
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This is a hammer test that verifies MultiVersionConsistencyControl in a
+ * multiple writer single reader scenario.
+ */
+@Category(SmallTests.class)
+public class TestMultiVersionConsistencyControl extends TestCase {
+  static class Writer implements Runnable {
+    final AtomicBoolean finished;
+    final MultiVersionConsistencyControl mvcc;
+    final AtomicBoolean status;
+
+    Writer(AtomicBoolean finished, MultiVersionConsistencyControl mvcc, AtomicBoolean status) {
+      this.finished = finished;
+      this.mvcc = mvcc;
+      this.status = status;
+    }
+
+    private Random rnd = new Random();
+    public boolean failed = false;
+
+    public void run() {
+      while (!finished.get()) {
+        MultiVersionConsistencyControl.WriteEntry e = mvcc.beginMemstoreInsert();
+        // System.out.println("Begin write: " + e.getWriteNumber());
+        // 10 usec - 500usec (including 0)
+        int sleepTime = rnd.nextInt(500);
+        // 500 * 1000 = 500,000ns = 500 usec
+        // 1 * 100 = 100ns = 1usec
+        try {
+          if (sleepTime > 0) Thread.sleep(0, sleepTime * 1000);
+        } catch (InterruptedException e1) {
+        }
+        try {
+          mvcc.completeMemstoreInsert(e);
+        } catch (RuntimeException ex) {
+          // got failure
+          System.out.println(ex.toString());
+          ex.printStackTrace();
+          status.set(false);
+          return;
+          // Report failure if possible.
+        }
+      }
+    }
+  }
+
+  public void testParallelism() throws Exception {
+    final MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
+
+    final AtomicBoolean finished = new AtomicBoolean(false);
+
+    // fail flag for the reader thread
+    final AtomicBoolean readerFailed = new AtomicBoolean(false);
+    final AtomicLong failedAt = new AtomicLong();
+    Runnable reader = new Runnable() {
+      public void run() {
+        long prev = mvcc.memstoreReadPoint();
+        while (!finished.get()) {
+          long newPrev = mvcc.memstoreReadPoint();
+          if (newPrev < prev) {
+            // serious problem.
+            System.out.println("Reader got out of order, prev: " + prev + " next was: " + newPrev);
+            readerFailed.set(true);
+            // might as well give up
+            failedAt.set(newPrev);
+            return;
+          }
+        }
+      }
+    };
+
+    // writer thread parallelism.
+    int n = 20;
+    Thread[] writers = new Thread[n];
+    AtomicBoolean[] statuses = new AtomicBoolean[n];
+    Thread readThread = new Thread(reader);
+
+    for (int i = 0; i < n; ++i) {
+      statuses[i] = new AtomicBoolean(true);
+      writers[i] = new Thread(new Writer(finished, mvcc, statuses[i]));
+      writers[i].start();
+    }
+    readThread.start();
+
+    try {
+      Thread.sleep(10 * 1000);
+    } catch (InterruptedException ex) {
+    }
+
+    finished.set(true);
+
+    readThread.join();
+    for (int i = 0; i < n; ++i) {
+      writers[i].join();
+    }
+
+    // check failure.
+    assertFalse(readerFailed.get());
+    for (int i = 0; i < n; ++i) {
+      assertTrue(statuses[i].get());
+    }
+
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestReadWriteConsistencyControl.java
deleted file mode 100644
index e69de29bb2d1..000000000000

From d74390b232ea6cbf694231822b68a077e56cf046 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 30 Aug 2012 16:37:41 +0000
Subject: [PATCH 0445/1540] HBASE-6685 Thrift DemoClient.pl got
 NullPointerException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379010 13f79535-47bb-0310-9956-ffa450edef68
---
 src/examples/thrift/DemoClient.pl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/examples/thrift/DemoClient.pl b/src/examples/thrift/DemoClient.pl
index 6020f01aa01b..17d92fad2f0a 100644
--- a/src/examples/thrift/DemoClient.pl
+++ b/src/examples/thrift/DemoClient.pl
@@ -141,7 +141,6 @@ ($)
 
 # this row name is valid utf8
 $key = "foo";
-# $mutations = [ Hbase::Mutation->new ( column => "entry:$key", value => $valid ) ];
 # This is another way to use the Mutation class
 my $mutation = Hbase::Mutation->new ();
 $mutation->{column} = "entry:$key";
@@ -151,7 +150,7 @@ ($)
 
 # non-utf8 is not allowed in row names
 eval {
-	$mutations = [ Hbase::Mutation->new ( column => "entry:$key", value => $invalid ) ];
+	$mutations = [ Hbase::Mutation->new ( { column => "entry:$key", value => $invalid } ) ];
 	# this can throw a TApplicationException (HASH) error
 	$client->mutateRow ($demo_table, $key, $mutations);
 	die ("shouldn't get here!");

From fcabac94b0ba7b6658e0e712c5bc2a183b2ebe85 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 30 Aug 2012 19:22:37 +0000
Subject: [PATCH 0446/1540] HBASE-6291 Don't retry increments on an invalid
 cell

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379076 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 18 ++++++++----
 .../hbase/client/TestFromClientSide.java      | 29 ++++++++++++++++++-
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 704a66d100cb..2af26e79fcd2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4591,7 +4591,13 @@ public Result increment(Increment increment, Integer lockid,
             if (idx < results.size() &&
                 results.get(idx).matchingQualifier(column.getKey())) {
               KeyValue kv = results.get(idx);
-              amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
+              if(kv.getValueLength() == Bytes.SIZEOF_LONG) {
+                amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), Bytes.SIZEOF_LONG);
+              } else {
+                // throw DoNotRetryIOException instead of IllegalArgumentException
+                throw new DoNotRetryIOException(
+                    "Attempted to increment field that isn't 64 bits wide");
+              }
               idx++;
             }
 
@@ -4640,11 +4646,10 @@ public Result increment(Increment increment, Integer lockid,
       }
     } finally {
       closeRegionOperation();
+      long after = EnvironmentEdgeManager.currentTimeMillis();
+      this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
     }
     
-    long after = EnvironmentEdgeManager.currentTimeMillis();
-    this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
-
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -4692,7 +4697,7 @@ public long incrementColumnValue(byte [] row, byte [] family,
 
         if (!results.isEmpty()) {
           KeyValue kv = results.get(0);
-          if(kv.getValueLength() == 8){
+          if(kv.getValueLength() == Bytes.SIZEOF_LONG){
             byte [] buffer = kv.getBuffer();
             int valueOffset = kv.getValueOffset();
             result += Bytes.toLong(buffer, valueOffset, Bytes.SIZEOF_LONG);
@@ -4748,7 +4753,8 @@ public long incrementColumnValue(byte [] row, byte [] family,
       requestFlush();
     }
     if(wrongLength){
-    	throw new IOException("Attempted to increment field that isn't 64 bits wide");
+      throw new DoNotRetryIOException(
+          "Attempted to increment field that isn't 64 bits wide");
     }
     return result;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 989c29e3f7b2..b52c0b32f6de 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -53,6 +53,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -4117,7 +4118,7 @@ public void testAppend() throws Exception {
  
   @Test
   public void testIncrementWithDeletes() throws Exception {
-    LOG.info("Starting testIncrement");
+    LOG.info("Starting testIncrementWithDeletes");
     final byte [] TABLENAME = Bytes.toBytes("testIncrementWithDeletes");
     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
     final byte[] COLUMN = Bytes.toBytes("column");
@@ -4136,6 +4137,32 @@ public void testIncrementWithDeletes() throws Exception {
     assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
   }
 
+  @Test
+  public void testIncrementingInvalidValue() throws Exception {
+    LOG.info("Starting testIncrementingInvalidValue");
+    final byte [] TABLENAME = Bytes.toBytes("testIncrementingInvalidValue");
+    HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
+    final byte[] COLUMN = Bytes.toBytes("column");
+    Put p = new Put(ROW);
+    // write an integer here (not a Long)
+    p.add(FAMILY, COLUMN, Bytes.toBytes(5));
+    ht.put(p);
+    try {
+      ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
+      fail("Should have thrown DoNotRetryIOException");
+    } catch (DoNotRetryIOException iox) {
+      // success
+    }
+    Increment inc = new Increment(ROW);
+    inc.addColumn(FAMILY, COLUMN, 5);
+    try {
+      ht.increment(inc);
+      fail("Should have thrown DoNotRetryIOException");
+    } catch (DoNotRetryIOException iox) {
+      // success
+    }
+  }
+
   @Test
   public void testIncrement() throws Exception {
     LOG.info("Starting testIncrement");

From dc0b293ac0c8d633f9fcaffd4cef8f6f070d5037 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 30 Aug 2012 23:22:11 +0000
Subject: [PATCH 0447/1540] HBASE-6257 Avoid unnecessary flush & compact on
 Meta in admin.rb

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379221 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/admin.rb | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index a70e05e18053..b6042da24c69 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -141,8 +141,6 @@ def drop(table_name)
       raise ArgumentError, "Table #{table_name} is enabled. Disable it first.'" if enabled?(table_name)
 
       @admin.deleteTable(table_name)
-      flush(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME)
-      major_compact(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME)
     end
 
     #----------------------------------------------------------------------------------------------
@@ -150,8 +148,6 @@ def drop(table_name)
     def drop_all(regex)
       regex = regex.to_s
       failed  = @admin.deleteTables(regex).map { |t| t.getNameAsString }
-      flush(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME)
-      major_compact(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME)
       return failed
     end
 

From 1727945618bb14354d30ebfda4d60f8517837df9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 30 Aug 2012 23:39:15 +0000
Subject: [PATCH 0448/1540] HBASE-6550 Refactoring ReplicationSink to make it
 more responsive of cluster health (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379229 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/Replication.java |  1 +
 .../regionserver/ReplicationSink.java         | 59 ++++++++++++++++---
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 464de865b08b..eb288f321b70 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -126,6 +126,7 @@ public void stopReplicationService() {
   public void join() {
     if (this.replication) {
       this.replicationManager.join();
+      this.replicationSink.stopReplicationSinkServices();
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 9c3f387f3644..c8bca24315c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -22,15 +22,20 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Row;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.Stoppable;
 
 import java.io.IOException;
@@ -38,6 +43,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 /**
  * This class is responsible for replicating the edits coming
@@ -59,8 +68,8 @@ public class ReplicationSink {
   // Name of the HDFS directory that contains the temporary rep logs
   public static final String REPLICATION_LOG_DIR = ".replogs";
   private final Configuration conf;
-  // Pool used to replicated
-  private final HTablePool pool;
+  private final ExecutorService sharedThreadPool;
+  private final HConnection sharedHtableCon;
   private final ReplicationSinkMetrics metrics;
 
   /**
@@ -72,12 +81,28 @@ public class ReplicationSink {
    */
   public ReplicationSink(Configuration conf, Stoppable stopper)
       throws IOException {
-    this.conf = conf;
-    this.pool = new HTablePool(this.conf,
-        conf.getInt("replication.sink.htablepool.capacity", 10));
+    this.conf = HBaseConfiguration.create(conf);
+    decorateConf();
+    this.sharedHtableCon = HConnectionManager.createConnection(this.conf);
+    this.sharedThreadPool = new ThreadPoolExecutor(1, 
+        conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE), 
+        conf.getLong("hbase.htable.threads.keepalivetime", 60), TimeUnit.SECONDS,
+        new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("hbase-repl"));
+    ((ThreadPoolExecutor)this.sharedThreadPool).allowCoreThreadTimeOut(true);
     this.metrics = new ReplicationSinkMetrics();
   }
 
+  /**
+   * decorate the Configuration object to make replication more receptive to
+   * delays: lessen the timeout and numTries.
+   */
+  private void decorateConf() {
+    this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+        this.conf.getInt("replication.sink.client.retries.number", 1));
+    this.conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
+        this.conf.getInt("replication.sink.client.ops.timeout", 20));
+  }
+
   /**
    * Replicate this array of entries directly into the local cluster
    * using the native client.
@@ -156,6 +181,26 @@ private <K, V> List<V> addToMultiMap(Map<K, List<V>> map, K key, V value) {
     return values;
   }
 
+  /**
+   * stop the thread pool executor. It is called when the regionserver is stopped.
+   */
+  public void stopReplicationSinkServices() {
+    try {
+      this.sharedThreadPool.shutdown();
+      if (!this.sharedThreadPool.awaitTermination(60000, TimeUnit.MILLISECONDS)) {
+        this.sharedThreadPool.shutdownNow();
+      }
+    } catch (InterruptedException e) {
+      LOG.warn("Interrupted while closing the table pool", e); // ignoring it as we are closing.
+      Thread.currentThread().interrupt();
+    }
+    try {
+      this.sharedHtableCon.close();
+    } catch (IOException e) {
+      LOG.warn("IOException while closing the connection", e); // ignoring as we are closing.
+    }
+  }  
+
   /**
    * Do the changes and handle the pool
    * @param tableName table to insert into
@@ -168,7 +213,7 @@ private void batch(byte[] tableName, List<Row> rows) throws IOException {
     }
     HTableInterface table = null;
     try {
-      table = this.pool.getTable(tableName);
+      table = new HTable(tableName, this.sharedHtableCon, this.sharedThreadPool);
       table.batch(rows);
       this.metrics.appliedOpsRate.inc(rows.size());
     } catch (InterruptedException ix) {

From 697f1e9e772aeb2d76ed1f911ea205267a46e630 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 31 Aug 2012 00:08:00 +0000
Subject: [PATCH 0449/1540] HBASE-6165 Replication can overrun .META. scans on
 cluster re-start (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379236 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   | 11 +++
 .../hadoop/hbase/ipc/HBaseRpcMetrics.java     |  2 +
 .../apache/hadoop/hbase/ipc/HBaseServer.java  | 69 +++++++++++--------
 .../hbase/regionserver/HRegionServer.java     | 53 +++++++-------
 4 files changed, 80 insertions(+), 55 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index c2df587849d4..25f5e15d876c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -650,6 +650,17 @@ public static enum Modify {
   public static final String ENABLE_WAL_COMPRESSION =
     "hbase.regionserver.wal.enablecompression";
 
+  /**
+   * QOS attributes: these attributes are used to demarcate RPC call processing
+   * by different set of handlers. For example, HIGH_QOS tagged methods are
+   * handled by high priority handlers.
+   */
+  public static final int NORMAL_QOS = 0;
+  public static final int QOS_THRESHOLD = 10;
+  public static final int HIGH_QOS = 100;
+  public static final int REPLICATION_QOS = 5; // normal_QOS < replication_QOS < high_QOS
+
+  
   private HConstants() {
     // Can't be instantiated with this ctor.
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
index 828a56e5dd39..bc897d970123 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
@@ -98,6 +98,8 @@ public HBaseRpcMetrics(String hostName, String port) {
          new MetricsTimeVaryingInt("rpcAuthorizationSuccesses", registry);
   public MetricsTimeVaryingRate rpcSlowResponseTime =
       new MetricsTimeVaryingRate("RpcSlowResponse", registry);
+  public final MetricsIntValue replicationCallQueueLen =
+    new MetricsIntValue("replicationCallQueueLen", registry);
 
   private void initMethods(Class<? extends VersionedProtocol> protocol) {
     for (Method m : protocol.getDeclaredMethods()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index b2486bcaf59b..e89be5d215cc 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -225,6 +225,10 @@ public static String getRemoteAddress() {
   protected int numConnections = 0;
   private Handler[] handlers = null;
   private Handler[] priorityHandlers = null;
+  /** replication related queue; */
+  private BlockingQueue<Call> replicationQueue;
+  private int numOfReplicationHandlers = 0;
+  private Handler[] replicationHandlers = null;
   protected HBaseRPCErrorHandler errorHandler = null;
 
   /**
@@ -1299,8 +1303,11 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
       if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
         priorityCallQueue.put(call);
         updateCallQueueLenMetrics(priorityCallQueue);
+      } else if (replicationQueue != null && getQosLevel(param) == HConstants.REPLICATION_QOS) {
+        replicationQueue.put(call);
+        updateCallQueueLenMetrics(replicationQueue);
       } else {
-        callQueue.put(call);              // queue the call; maybe blocked here
+        callQueue.put(call); // queue the call; maybe blocked here
         updateCallQueueLenMetrics(callQueue);
       }
     }
@@ -1327,6 +1334,8 @@ private void updateCallQueueLenMetrics(BlockingQueue<Call> queue) {
       rpcMetrics.callQueueLen.set(callQueue.size());
     } else if (queue == priorityCallQueue) {
       rpcMetrics.priorityCallQueueLen.set(priorityCallQueue.size());
+    } else if (queue == replicationQueue) {
+      rpcMetrics.replicationCallQueueLen.set(replicationQueue.size());
     } else {
       LOG.warn("Unknown call queue");
     }
@@ -1345,6 +1354,8 @@ public Handler(final BlockingQueue<Call> cq, int instanceNumber) {
       if (cq == priorityCallQueue) {
         // this is just an amazing hack, but it works.
         threadName = "PRI " + threadName;
+      } else if (cq == replicationQueue) {
+        threadName = "REPL " + threadName;
       }
       this.setName(threadName);
       this.status = TaskMonitor.get().createRPCStatus(threadName);
@@ -1513,7 +1524,11 @@ protected HBaseServer(String bindAddress, int port,
     this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
     this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout",
                                      2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
-
+    this.numOfReplicationHandlers = 
+      conf.getInt("hbase.regionserver.replication.handler.count", 3);
+    if (numOfReplicationHandlers > 0) {
+      this.replicationQueue = new LinkedBlockingQueue<Call>(maxQueueSize);
+    }
     // Start the listener here and let it bind to the port
     listener = new Listener();
     this.port = listener.getAddress().getPort();
@@ -1613,20 +1628,21 @@ public void openServer() {
   public synchronized void startThreads() {
     responder.start();
     listener.start();
-    handlers = new Handler[handlerCount];
-
-    for (int i = 0; i < handlerCount; i++) {
-      handlers[i] = new Handler(callQueue, i);
-      handlers[i].start();
+    handlers = startHandlers(callQueue, handlerCount);
+    priorityHandlers = startHandlers(priorityCallQueue, priorityHandlerCount);
+    replicationHandlers = startHandlers(replicationQueue, numOfReplicationHandlers);
     }
 
-    if (priorityHandlerCount > 0) {
-      priorityHandlers = new Handler[priorityHandlerCount];
-      for (int i = 0 ; i < priorityHandlerCount; i++) {
-        priorityHandlers[i] = new Handler(priorityCallQueue, i);
-        priorityHandlers[i].start();
-      }
+  private Handler[] startHandlers(BlockingQueue<Call> queue, int numOfHandlers) {
+    if (numOfHandlers <= 0) {
+      return null;
+    }
+    Handler[] handlers = new Handler[numOfHandlers];
+    for (int i = 0; i < numOfHandlers; i++) {
+      handlers[i] = new Handler(queue, i);
+      handlers[i].start();
     }
+    return handlers;
   }
 
   /** Stops the service.  No new calls will be handled after this is called. */
@@ -1634,20 +1650,9 @@ public synchronized void startThreads() {
   public synchronized void stop() {
     LOG.info("Stopping server on " + port);
     running = false;
-    if (handlers != null) {
-      for (Handler handler : handlers) {
-        if (handler != null) {
-          handler.interrupt();
-        }
-      }
-    }
-    if (priorityHandlers != null) {
-      for (Handler handler : priorityHandlers) {
-        if (handler != null) {
-          handler.interrupt();
-        }
-      }
-    }
+    stopHandlers(handlers);
+    stopHandlers(priorityHandlers);
+    stopHandlers(replicationHandlers);
     listener.interrupt();
     listener.doStop();
     responder.interrupt();
@@ -1657,6 +1662,16 @@ public synchronized void stop() {
     }
   }
 
+  private void stopHandlers(Handler[] handlers) {
+    if (handlers != null) {
+      for (Handler handler : handlers) {
+        if (handler != null) {
+          handler.interrupt();
+        }
+      }
+    }
+  }
+
   /** Wait for the server to be stopped.
    * Does not wait for all subthreads to finish.
    *  See {@link #stop()}.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5764997b29c5..e0a7c24784e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -415,7 +415,7 @@ public HRegionServer(Configuration conf)
         conf.getInt("hbase.regionserver.handler.count", 10),
         conf.getInt("hbase.regionserver.metahandler.count", 10),
         conf.getBoolean("hbase.rpc.verbose", false),
-        conf, QOS_THRESHOLD);
+        conf, HConstants.QOS_THRESHOLD);
     // Set our address.
     this.isa = this.rpcServer.getListenerAddress();
 
@@ -447,9 +447,6 @@ private static void checkCodecs(final Configuration c) throws IOException {
     }
   }
 
-  private static final int NORMAL_QOS = 0;
-  private static final int QOS_THRESHOLD = 10;  // the line between low and high qos
-  private static final int HIGH_QOS = 100;
 
   @Retention(RetentionPolicy.RUNTIME)
   private @interface QosPriority {
@@ -487,7 +484,7 @@ public boolean isMetaRegion(byte[] regionName) {
 
     @Override
     public Integer apply(Writable from) {
-      if (!(from instanceof Invocation)) return NORMAL_QOS;
+      if (!(from instanceof Invocation)) return HConstants.NORMAL_QOS;
 
       Invocation inv = (Invocation) from;
       String methodName = inv.getMethodName();
@@ -505,13 +502,13 @@ public Integer apply(Writable from) {
           scannerId = (Long) inv.getParameters()[0];
         } catch (ClassCastException ignored) {
           // LOG.debug("Low priority: " + from);
-          return NORMAL_QOS; // doh.
+          return HConstants.NORMAL_QOS;
         }
         String scannerIdString = Long.toString(scannerId);
         RegionScanner scanner = scanners.get(scannerIdString);
         if (scanner != null && scanner.getRegionInfo().isMetaRegion()) {
           // LOG.debug("High priority scanner request: " + scannerId);
-          return HIGH_QOS;
+          return HConstants.HIGH_QOS;
         }
       } else if (inv.getParameterClasses().length == 0) {
        // Just let it through.  This is getOnlineRegions, etc.
@@ -521,7 +518,7 @@ public Integer apply(Writable from) {
           // LOG.debug("High priority with method: " + methodName +
           // " and region: "
           // + Bytes.toString((byte[]) inv.getParameters()[0]));
-          return HIGH_QOS;
+          return HConstants.HIGH_QOS;
         }
       } else if (inv.getParameterClasses()[0] == MultiAction.class) {
         MultiAction<?> ma = (MultiAction<?>) inv.getParameters()[0];
@@ -537,12 +534,12 @@ public Integer apply(Writable from) {
           if (isMetaRegion(region)) {
             // LOG.debug("High priority multi with region: " +
             // Bytes.toString(region));
-            return HIGH_QOS; // short circuit for the win.
+            return HConstants.HIGH_QOS; // short circuit for the win.
           }
         }
       }
       // LOG.debug("Low priority: " + from.toString());
-      return NORMAL_QOS;
+      return HConstants.NORMAL_QOS;
     }
   }
 
@@ -1924,7 +1921,7 @@ void closeUserRegions(final boolean abort) {
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public HRegionInfo getRegionInfo(final byte[] regionName)
   throws NotServingRegionException, IOException {
     checkOpen();
@@ -2604,7 +2601,7 @@ Integer getLockFromId(long lockId) throws IOException {
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public void unlockRow(byte[] regionName, long lockId) throws IOException {
     checkOpen();
     NullPointerException npe = null;
@@ -2686,14 +2683,14 @@ public void leaseExpired() {
   // Region open/close direct RPCs
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public RegionOpeningState openRegion(HRegionInfo region)
   throws IOException {
     return openRegion(region, -1);
   }
 
   @Override
-  @QosPriority(priority = HIGH_QOS)
+  @QosPriority(priority = HConstants.HIGH_QOS)
   public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode)
       throws IOException {
     return openRegion(region, versionOfOfflineNode, null);
@@ -2761,7 +2758,7 @@ private void checkIfRegionInTransition(HRegionInfo region,
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public void openRegions(List<HRegionInfo> regions)
   throws IOException {
     checkOpen();
@@ -2771,14 +2768,14 @@ public void openRegions(List<HRegionInfo> regions)
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public boolean closeRegion(HRegionInfo region)
   throws IOException {
     return closeRegion(region, true, -1);
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public boolean closeRegion(final HRegionInfo region,
     final int versionOfClosingNode)
   throws IOException {
@@ -2786,13 +2783,13 @@ public boolean closeRegion(final HRegionInfo region,
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public boolean closeRegion(HRegionInfo region, final boolean zk)
   throws IOException {
     return closeRegion(region, zk, -1);
   }
 
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   protected boolean closeRegion(HRegionInfo region, final boolean zk,
     final int versionOfClosingNode)
   throws IOException {
@@ -2811,7 +2808,7 @@ protected boolean closeRegion(HRegionInfo region, final boolean zk,
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public boolean closeRegion(byte[] encodedRegionName, boolean zk)
     throws IOException {
     return closeRegion(encodedRegionName, false, zk);
@@ -2891,7 +2888,7 @@ protected boolean closeRegion(byte[] encodedRegionName, final boolean abort,
   // Manual remote region administration RPCs
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public void flushRegion(HRegionInfo regionInfo)
       throws NotServingRegionException, IOException {
     checkOpen();
@@ -2901,7 +2898,7 @@ public void flushRegion(HRegionInfo regionInfo)
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public void splitRegion(HRegionInfo regionInfo)
       throws NotServingRegionException, IOException {
     splitRegion(regionInfo, null);
@@ -2918,7 +2915,7 @@ public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public void compactRegion(HRegionInfo regionInfo, boolean major)
       throws NotServingRegionException, IOException {
     checkOpen();
@@ -2964,7 +2961,7 @@ ReentrantReadWriteLock.WriteLock getWriteLock() {
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public List<HRegionInfo> getOnlineRegions() throws IOException {
     checkOpen();
     List<HRegionInfo> list = new ArrayList<HRegionInfo>(onlineRegions.size());
@@ -3161,7 +3158,7 @@ protected void checkOpen() throws IOException {
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public ProtocolSignature getProtocolSignature(
       String protocol, long version, int clientMethodsHashCode)
   throws IOException {
@@ -3172,7 +3169,7 @@ public ProtocolSignature getProtocolSignature(
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public long getProtocolVersion(final String protocol, final long clientVersion)
   throws IOException {
     if (protocol.equals(HRegionInterface.class.getName())) {
@@ -3334,7 +3331,7 @@ public long incrementColumnValue(byte[] regionName, byte[] row,
    * @deprecated Use {@link #getServerName()} instead.
    */
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.HIGH_QOS)
   public HServerInfo getHServerInfo() throws IOException {
     checkOpen();
     return new HServerInfo(new HServerAddress(this.isa),
@@ -3632,7 +3629,7 @@ public static HRegionServer constructRegionServer(
   }
 
   @Override
-  @QosPriority(priority=HIGH_QOS)
+  @QosPriority(priority=HConstants.REPLICATION_QOS)
   public void replicateLogEntries(final HLog.Entry[] entries)
   throws IOException {
     checkOpen();

From cdedacdce2c73be28d25a7b8ca4fddf72f0ee00a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 31 Aug 2012 01:01:04 +0000
Subject: [PATCH 0450/1540] HBASE-6587 Region would be assigned twice in the
 case of all RS offline

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379242 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/AssignmentManager.java  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 287ec8d9b299..eb82e4e604a5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2911,9 +2911,13 @@ protected void chore() {
            //decide on action upon timeout
             actOnTimeOut(regionState);
           } else if (this.allRegionServersOffline && !allRSsOffline) {
-            // if some RSs just came back online, we can start the
-            // the assignment right away
-            actOnTimeOut(regionState);
+            RegionPlan existingPlan = regionPlans.get(regionState.getRegion().getEncodedName());
+            if (existingPlan == null
+                || !this.serverManager.isServerOnline(existingPlan.getDestination())) {
+              // if some RSs just came back online, we can start the
+              // the assignment right away
+              actOnTimeOut(regionState);
+            }
           }
         }
       }

From b608eaf6ef040b3009c66016b608b1da3a997689 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 31 Aug 2012 03:53:49 +0000
Subject: [PATCH 0451/1540] HBASE-6537 Race between balancer and disable table
 can lead to inconsistent cluster (Zhou wenjian)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379277 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/AssignmentManager.java  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index eb82e4e604a5..61fa7ff0c3f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1257,14 +1257,15 @@ private void updateTimers(final ServerName sn) {
    * @param regionInfo
    */
   public void regionOffline(final HRegionInfo regionInfo) {
+    // remove the region plan as well just in case.
+    clearRegionPlan(regionInfo);
+    setOffline(regionInfo);
+
     synchronized(this.regionsInTransition) {
       if (this.regionsInTransition.remove(regionInfo.getEncodedName()) != null) {
         this.regionsInTransition.notifyAll();
       }
     }
-    // remove the region plan as well just in case.
-    clearRegionPlan(regionInfo);
-    setOffline(regionInfo);
   }
 
   /**
@@ -2065,7 +2066,8 @@ public void unassign(HRegionInfo region, boolean force) {
       if (t instanceof RemoteException) {
         t = ((RemoteException)t).unwrapRemoteException();
         if (t instanceof NotServingRegionException) {
-          if (checkIfRegionBelongsToDisabling(region)) {
+          if (checkIfRegionBelongsToDisabling(region)
+              || checkIfRegionBelongsToDisabled(region)) {
             // Remove from the regionsinTransition map
             LOG.info("While trying to recover the table "
                 + region.getTableNameAsString()

From 7a143edfd311bd8e2d6443e94620831a75158610 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Fri, 31 Aug 2012 05:01:00 +0000
Subject: [PATCH 0452/1540] HBASE-6321  ReplicationSource dies reading the
 peer's id HBASE-6647  [performance regression] appendNoSync/HBASE-4528
 doesn't             take deferred log flush into account

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379290 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 31 ++++++----
 .../hadoop/hbase/regionserver/wal/HLog.java   | 13 +++-
 .../replication/ReplicationZookeeper.java     | 61 +++++++++++++++----
 .../regionserver/ReplicationSource.java       | 20 +++---
 4 files changed, 90 insertions(+), 35 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 2af26e79fcd2..e8d2c2684c9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2212,10 +2212,8 @@ private long doMiniBatchMutation(
       // -------------------------
       // STEP 7. Sync wal.
       // -------------------------
-      if (walEdit.size() > 0 &&
-          (this.regionInfo.isMetaRegion() ||
-           !this.htableDescriptor.isDeferredLogFlush())) {
-        this.log.sync(txid);
+      if (walEdit.size() > 0) {
+        syncOrDefer(txid);
       }
       walSyncSuccessful = true;
       // ------------------------------------------------------------------
@@ -4314,10 +4312,8 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
         }
 
         // 9. sync WAL if required
-        if (walEdit.size() > 0 &&
-            (this.regionInfo.isMetaRegion() ||
-             !this.htableDescriptor.isDeferredLogFlush())) {
-          this.log.sync(txid);
+        if (walEdit.size() > 0) {
+          syncOrDefer(txid);
         }
         walSyncSuccessful = true;
 
@@ -4515,7 +4511,7 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
         releaseRowLock(lid);
       }
       if (writeToWAL) {
-        this.log.sync(txid); // sync the transaction log outside the rowlock
+        syncOrDefer(txid); // sync the transaction log outside the rowlock
       }
     } finally {
       closeRegionOperation();
@@ -4642,7 +4638,7 @@ public Result increment(Increment increment, Integer lockid,
         releaseRowLock(lid);
       }
       if (writeToWAL) {
-        this.log.sync(txid); // sync the transaction log outside the rowlock
+        syncOrDefer(txid); // sync the transaction log outside the rowlock
       }
     } finally {
       closeRegionOperation();
@@ -4738,7 +4734,7 @@ public long incrementColumnValue(byte [] row, byte [] family,
         releaseRowLock(lid);
       }
       if (writeToWAL) {
-        this.log.sync(txid); // sync the transaction log outside the rowlock
+        syncOrDefer(txid); // sync the transaction log outside the rowlock
       }
     } finally {
       closeRegionOperation();
@@ -5151,6 +5147,19 @@ private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
     dataInMemoryWithoutWAL.addAndGet(putSize);
   }
 
+  /**
+   * Calls sync with the given transaction ID if the region's table is not
+   * deferring it.
+   * @param txid should sync up to which transaction
+   * @throws IOException If anything goes wrong with DFS
+   */
+  private void syncOrDefer(long txid) throws IOException {
+    if (this.regionInfo.isMetaRegion() ||
+      !this.htableDescriptor.isDeferredLogFlush()) {
+      this.log.sync(txid);
+    }
+  }
+
   /**
    * A mocked list implementaion - discards all updates.
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index f308e6d230d0..531d04999e18 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1187,10 +1187,14 @@ public long append(HRegionInfo info, byte [] tableName, WALEdit edits,
   }
 
   /**
-   * This thread is responsible to call syncFs and buffer up the writers while
-   * it happens.
+   * This class is responsible to hold the HLog's appended Entry list
+   * and to sync them according to a configurable interval.
+   *
+   * Deferred log flushing works first by piggy backing on this process by
+   * simply not sync'ing the appended Entry. It can also be sync'd by other
+   * non-deferred log flushed entries outside of this thread.
    */
-   class LogSyncer extends HasThread {
+  class LogSyncer extends HasThread {
 
     private final long optionalFlushInterval;
 
@@ -1221,6 +1225,9 @@ public void run() {
                 closeLogSyncer.wait(this.optionalFlushInterval);
               }
             }
+            // Calling sync since we waited or had unflushed entries.
+            // Entries appended but not sync'd are taken care of here AKA
+            // deferred log flush
             sync();
           } catch (IOException e) {
             LOG.error("Error while syncing, requesting close of hlog ", e);
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 61740589ee5f..70c2ead8b9ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -29,6 +29,7 @@
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -38,7 +39,9 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.regionserver.Replication;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
@@ -238,19 +241,7 @@ public List<ServerName> getSlavesAddresses(String peerClusterId) {
     try {
       addresses = fetchSlavesAddresses(peer.getZkw());
     } catch (KeeperException ke) {
-      if (ke instanceof ConnectionLossException
-          || ke instanceof SessionExpiredException) {
-        LOG.warn(
-            "Lost the ZooKeeper connection for peer " + peer.getClusterKey(),
-            ke);
-        try {
-          peer.reloadZkWatcher();
-        } catch(IOException io) {
-          LOG.warn(
-              "Creation of ZookeeperWatcher failed for peer "
-                  + peer.getClusterKey(), io);
-        }
-      }
+      reconnectPeer(ke, peer);
       addresses = Collections.emptyList();
     }
     peer.setRegionServers(addresses);
@@ -798,6 +789,50 @@ public long getHLogRepPosition(String peerId, String hlog)
     return data == null || data.length() == 0 ? 0 : Long.parseLong(data);
   }
 
+  /**
+   * Returns the UUID of the provided peer id. Should a connection loss or session
+   * expiration happen, the ZK handler will be reopened once and if it still doesn't
+   * work then it will bail and return null.
+   * @param peerId the peer's ID that will be converted into a UUID
+   * @return a UUID or null if there's a ZK connection issue
+   */
+  public UUID getPeerUUID(String peerId) {
+    ReplicationPeer peer = getPeerClusters().get(peerId);
+    UUID peerUUID = null;
+    try {
+      peerUUID = getUUIDForCluster(peer.getZkw());
+    } catch (KeeperException ke) {
+      reconnectPeer(ke, peer);
+    }
+    return peerUUID;
+  }
+
+  /**
+   * Get the UUID for the provided ZK watcher. Doesn't handle any ZK exceptions
+   * @param zkw watcher connected to an ensemble
+   * @return the UUID read from zookeeper
+   * @throws KeeperException
+   */
+  public UUID getUUIDForCluster(ZooKeeperWatcher zkw) throws KeeperException {
+    return UUID.fromString(ClusterId.readClusterIdZNode(zkw));
+  }
+
+  private void reconnectPeer(KeeperException ke, ReplicationPeer peer) {
+    if (ke instanceof ConnectionLossException
+      || ke instanceof SessionExpiredException) {
+      LOG.warn(
+        "Lost the ZooKeeper connection for peer " + peer.getClusterKey(),
+        ke);
+      try {
+        peer.reloadZkWatcher();
+      } catch(IOException io) {
+        LOG.warn(
+          "Creation of ZookeeperWatcher failed for peer "
+            + peer.getClusterKey(), io);
+      }
+    }
+  }
+
   public void registerRegionServerListener(ZooKeeperListener listener) {
     this.zookeeper.registerListener(listener);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 0866c61ab265..89d3bdb6391a 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -185,8 +185,7 @@ public void init(final Configuration conf,
     this.metrics = new ReplicationSourceMetrics(peerClusterZnode);
 
     try {
-      this.clusterId = UUID.fromString(ClusterId.readClusterIdZNode(zkHelper
-          .getZookeeperWatcher()));
+      this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher());
     } catch (KeeperException ke) {
       throw new IOException("Could not read cluster id", ke);
     }
@@ -245,13 +244,19 @@ public void run() {
     if (!this.isActive()) {
       return;
     }
+    int sleepMultiplier = 1;
     // delay this until we are in an asynchronous thread
-    try {
-      this.peerClusterId = UUID.fromString(ClusterId
-          .readClusterIdZNode(zkHelper.getPeerClusters().get(peerId).getZkw()));
-    } catch (KeeperException ke) {
-      this.terminate("Could not read peer's cluster id", ke);
+    while (this.peerClusterId == null) {
+      this.peerClusterId = zkHelper.getPeerUUID(this.peerId);
+      if (this.peerClusterId == null) {
+        if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) {
+          sleepMultiplier++;
+        }
+      }
     }
+    // resetting to 1 to reuse later
+    sleepMultiplier = 1;
+
     LOG.info("Replicating "+clusterId + " -> " + peerClusterId);
 
     // If this is recovered, the queue is already full and the first log
@@ -265,7 +270,6 @@ public void run() {
             peerClusterZnode, e);
       }
     }
-    int sleepMultiplier = 1;
     // Loop until we close down
     while (isActive()) {
       // Sleep until replication is enabled again

From fdd36eb5da58681eb6b4cc07a7921b9d92a2d1b9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 1 Sep 2012 04:35:29 +0000
Subject: [PATCH 0453/1540] HBASE-6268 Can't enable a table on a 0.94 cluster
 from a 0.92 client

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1379688 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index a5924f7e6710..16c0a7a3c4e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -280,7 +280,8 @@ public boolean isEnabledTable(String tableName) {
   public static boolean isEnabledTable(final ZooKeeperWatcher zkw,
       final String tableName)
   throws KeeperException {
-    return getTableState(zkw, tableName) == TableState.ENABLED;
+    TableState state = getTableState(zkw, tableName);
+    return state == null || state == TableState.ENABLED;
   }
 
   public boolean isDisablingOrDisabledTable(final String tableName) {

From ec3d7e911495a9785901ab62fed92e702e828324 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Tue, 4 Sep 2012 17:52:52 +0000
Subject: [PATCH 0454/1540] HBASE-6516 hbck cannot detect any IOException while
 ".tableinfo" file is missing (Jie Huang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1380759 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/TableInfoMissingException.java      | 47 +++++++++++++++++++
 .../hadoop/hbase/util/FSTableDescriptors.java | 46 ++++++++++++------
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  7 +--
 .../hadoop/hbase/util/TestHBaseFsck.java      | 21 +++++++++
 4 files changed, 104 insertions(+), 17 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java

diff --git a/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java b/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java
new file mode 100644
index 000000000000..bc8bc7fc1948
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * 
+ * Failed to find .tableinfo file under table dir
+ *
+ */
+@InterfaceAudience.Private
+@SuppressWarnings("serial")
+public class TableInfoMissingException extends HBaseIOException {
+
+  public TableInfoMissingException() {
+    super();
+  }
+
+  public TableInfoMissingException( String message ) {
+    super(message);
+  }
+
+  public TableInfoMissingException( String message, Throwable t ) {
+    super(message, t);
+  }
+
+  public TableInfoMissingException( Throwable t ) {
+    super(t);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index a59acde01b46..4913a19bb5f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -41,7 +41,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableDescriptors;
-import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableInfoMissingException;
 
 
 /**
@@ -158,7 +158,18 @@ public HTableDescriptor get(final String tablename)
         return tdm.getTableDescriptor();
       }
     }
-    HTableDescriptor htd = getTableDescriptor(this.fs, this.rootdir, tablename);
+    
+    HTableDescriptor htd = null;
+    try {
+       htd = getTableDescriptor(this.fs, this.rootdir, tablename);
+    } catch (NullPointerException e) {
+      LOG.debug("Exception during readTableDecriptor. Current table name = "
+          + tablename, e);
+    } catch (IOException ioe) {
+      LOG.debug("Exception during readTableDecriptor. Current table name = "
+          + tablename, ioe);
+    }
+    
     if (htd == null) {
       LOG.warn("The following folder is in HBase's root directory and " +
         "doesn't contain a table descriptor, " +
@@ -253,7 +264,7 @@ private static FileStatus getTableInfoPath(final FileSystem fs,
    * @return The 'current' tableinfo file.
    * @throws IOException
    */
-  private static FileStatus getTableInfoPath(final FileSystem fs,
+  public static FileStatus getTableInfoPath(final FileSystem fs,
       final Path tabledir)
   throws IOException {
     FileStatus [] status = FSUtils.listStatus(fs, tabledir, new PathFilter() {
@@ -370,21 +381,25 @@ static long getTableInfoModtime(final FileSystem fs, final Path rootdir,
   public static HTableDescriptor getTableDescriptor(FileSystem fs,
       Path hbaseRootDir, byte[] tableName)
   throws IOException {
-     return getTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
+     HTableDescriptor htd = null;
+     try {
+       htd = getTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
+     } catch (NullPointerException e) {
+       LOG.debug("Exception during readTableDecriptor. Current table name = "
+           + Bytes.toString(tableName), e);
+     }
+     return htd;
   }
 
   static HTableDescriptor getTableDescriptor(FileSystem fs,
-      Path hbaseRootDir, String tableName) {
+      Path hbaseRootDir, String tableName) throws NullPointerException, IOException{
     HTableDescriptor htd = null;
-    try {
-      htd = getTableDescriptor(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
-    } catch (NullPointerException e) {
-      LOG.debug("Exception during readTableDecriptor. Current table name = " +
-        tableName , e);
-    } catch (IOException ioe) {
-      LOG.debug("Exception during readTableDecriptor. Current table name = " +
-        tableName , ioe);
+    // ignore both -ROOT- and .META. tables
+    if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0
+        || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) {
+      return null;
     }
+    htd = getTableDescriptor(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
     return htd;
   }
 
@@ -392,7 +407,10 @@ public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
   throws IOException, NullPointerException {
     if (tableDir == null) throw new NullPointerException();
     FileStatus status = getTableInfoPath(fs, tableDir);
-    if (status == null) return null;
+    if (status == null) {
+      throw new TableInfoMissingException("No .tableinfo file under "
+          + tableDir.toUri());
+    }
     FSDataInputStream fsDataInputStream = fs.open(status.getPath());
     HTableDescriptor hTableDescriptor = null;
     try {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 346cbecbc29b..99c55840ce29 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -714,8 +714,9 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
               hbaseRoot, tableName);
           modTInfo.htds.add(htd);
         } catch (IOException ioe) {
-          LOG.error("Unable to read .tableinfo from " + hbaseRoot, ioe);
-          throw ioe;
+          LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
+          errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, 
+              "Unable to read .tableinfo from " + hbaseRoot);
         }
       }
       modTInfo.addRegionInfo(hbi);
@@ -2648,7 +2649,7 @@ public static enum ERROR_CODE {
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
-      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT
+      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE
     }
     public void clear();
     public void report(String message);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index e99e59f55d2b..696e1221b969 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -407,6 +407,27 @@ public void testHbckThreadpooling() throws Exception {
     }    
   }
 
+  @Test
+  public void testHbckMissingTableinfo() throws Exception {
+    String table = "tableInfo";
+    FileSystem fs = null;
+    Path tableinfo = null;
+    try {
+      setupTable(table);
+      Path hbaseTableDir = new Path(conf.get(HConstants.HBASE_DIR) + "/" + table );
+      fs = hbaseTableDir.getFileSystem(conf);
+      FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
+      tableinfo = status.getPath();
+      fs.rename(tableinfo, new Path("/.tableinfo"));
+      
+      HBaseFsck hbck = doFsck(conf, false); 
+      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
+    } finally {
+      fs.rename(new Path("/.tableinfo"), tableinfo);
+      deleteTable(table);
+    }
+  }
+   
   /**
    * This create and fixes a bad table with regions that have a duplicate
    * start key

From dc0b0dde007b446656c1c9064afd8d288577291e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 4 Sep 2012 20:34:15 +0000
Subject: [PATCH 0455/1540] HBASE-6711 Avoid local results copy in StoreScanner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1380867 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/StoreScanner.java       | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 8919a3869226..c255d5690dd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -346,13 +346,13 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
 
     KeyValue kv;
     KeyValue prevKV = null;
-    List<KeyValue> results = new ArrayList<KeyValue>();
 
     // Only do a sanity-check if store and comparator are available.
     KeyValue.KVComparator comparator =
         store != null ? store.getComparator() : null;
 
     long cumulativeMetric = 0;
+    int count = 0;
     try {
       LOOP: while((kv = this.heap.peek()) != null) {
         // Check that the heap gives us KVs in an increasing order.
@@ -366,11 +366,11 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
           case INCLUDE_AND_SEEK_NEXT_COL:
 
             Filter f = matcher.getFilter();
-            results.add(f == null ? kv : f.transform(kv));
+            outResult.add(f == null ? kv : f.transform(kv));
+            count++;
 
             if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) {
               if (!matcher.moreRowsMayExistAfter(kv)) {
-                outResult.addAll(results);
                 return false;
               }
               reseek(matcher.getKeyForNextRow(kv));
@@ -381,29 +381,23 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
             }
 
             cumulativeMetric += kv.getLength();
-            if (limit > 0 && (results.size() == limit)) {
+            if (limit > 0 && (count == limit)) {
               break LOOP;
             }
             continue;
 
           case DONE:
-            // copy jazz
-            outResult.addAll(results);
             return true;
 
           case DONE_SCAN:
             close();
 
-            // copy jazz
-            outResult.addAll(results);
-
             return false;
 
           case SEEK_NEXT_ROW:
             // This is just a relatively simple end of scan fix, to short-cut end
             // us if there is an endKey in the scan.
             if (!matcher.moreRowsMayExistAfter(kv)) {
-              outResult.addAll(results);
               return false;
             }
 
@@ -438,9 +432,7 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
       }
     }
 
-    if (!results.isEmpty()) {
-      // copy jazz
-      outResult.addAll(results);
+    if (count > 0) {
       return true;
     }
 

From cb6471df228bcb3072be91347f497965303c3b88 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 5 Sep 2012 15:13:55 +0000
Subject: [PATCH 0456/1540] HBASE-6288 In hbase-daemons.sh, description of the
 default backup-master file path is wrong

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1381219 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/master-backup.sh | 2 +-
 conf/hbase-env.sh    | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/master-backup.sh b/bin/master-backup.sh
index d20f5793e094..114757c7633b 100755
--- a/bin/master-backup.sh
+++ b/bin/master-backup.sh
@@ -25,7 +25,7 @@
 # Environment Variables
 #
 #   HBASE_BACKUP_MASTERS File naming remote hosts.
-#     Default is ${HADOOP_CONF_DIR}/backup-masters
+#     Default is ${HBASE_CONF_DIR}/backup-masters
 #   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
 #   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
 #   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index bc293b4b52f0..8e74de662779 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -61,6 +61,9 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
 # File naming hosts on which HRegionServers will run.  $HBASE_HOME/conf/regionservers by default.
 # export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
 
+# File naming hosts on which backup HMaster will run.  $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
 # Extra ssh options.  Empty by default.
 # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
 

From 22b6f4ed7884a6e1ea7e91a56e4c4cb9d77ff729 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 5 Sep 2012 18:47:17 +0000
Subject: [PATCH 0457/1540] HBASE-6649 [0.92 UNIT TESTS]
 TestReplication.queueFailover occasionally fails [Part-1]

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1381289 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 89d3bdb6391a..b3451235bdbb 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -425,7 +425,12 @@ protected boolean readAllEntriesToReplicateOrNextFile() throws IOException{
           currentNbEntries >= this.replicationQueueNbCapacity) {
         break;
       }
-      entry = this.reader.next(entriesArray[currentNbEntries]);
+      try {
+        entry = this.reader.next(entriesArray[currentNbEntries]);
+      } catch (IOException ie) {
+        LOG.debug("Break on IOE: " + ie.getMessage());
+        break;
+      }
     }
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +

From c2e966c6fb18eb5aa942eb59c24f68a4b697f3c0 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 6 Sep 2012 16:44:44 +0000
Subject: [PATCH 0458/1540] HBASE-6715
 TestFromClientSide.testCacheOnWriteEvictOnClose is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1381678 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestFromClientSide.java      | 30 +++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index b52c0b32f6de..6ce3fa3a61ce 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -48,6 +48,7 @@
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -102,6 +103,7 @@
  * Each creates a table named for the method and does its stuff against that.
  */
 @Category(LargeTests.class)
+@SuppressWarnings ("deprecation")
 public class TestFromClientSide {
   final Log LOG = LogFactory.getLog(getClass());
   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -4366,8 +4368,8 @@ public void testClientPoolThreadLocal() throws IOException {
     }
 
     final Object waitLock = new Object();
-
     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
+    final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
     for (int versions = numVersions; versions < numVersions * 2; versions++) {
       final int versionsCopy = versions;
       executorService.submit(new Callable<Void>() {
@@ -4392,6 +4394,11 @@ public Void call() {
               waitLock.wait();
             }
           } catch (Exception e) {
+          } catch (AssertionError e) {
+            // the error happens in a thread, it won't fail the test,
+            // need to pass it to the caller for proper handling.
+            error.set(e);
+            LOG.error(e);
           }
 
           return null;
@@ -4402,9 +4409,9 @@ public Void call() {
       waitLock.notifyAll();
     }
     executorService.shutdownNow();
+    assertNull(error.get());
   }
 
-
   @Test
   public void testCheckAndPut() throws IOException {
     final byte [] anotherrow = Bytes.toBytes("anotherrow");
@@ -4450,6 +4457,7 @@ public void testCheckAndPut() throws IOException {
   * @throws Exception
   */
   @Test
+  @SuppressWarnings ("unused")
   public void testScanMetrics() throws Exception {
     byte [] TABLENAME = Bytes.toBytes("testScanMetrics");
 
@@ -4518,9 +4526,8 @@ public void testScanMetrics() throws Exception {
     ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
     assertEquals("Did not access all the regions in the table", numOfRegions,
         scanMetricsWithClose.countOfRegions.getCurrentIntervalValue());
-
   }
-  
+
   private ScanMetrics getScanMetrics(Scan scan) throws Exception {
     byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
     assertTrue("Serialized metrics were not found.", serializedMetrics != null);
@@ -4559,6 +4566,20 @@ public void testCacheOnWriteEvictOnClose() throws Exception {
     long startBlockCount = cache.getBlockCount();
     long startBlockHits = cache.getStats().getHitCount();
     long startBlockMiss = cache.getStats().getMissCount();
+
+    // wait till baseline is stable, (minimal 500 ms)
+    for (int i = 0; i < 5; i++) {
+      Thread.sleep(100);
+      if (startBlockCount != cache.getBlockCount()
+          || startBlockHits != cache.getStats().getHitCount()
+          || startBlockMiss != cache.getStats().getMissCount()) {
+        startBlockCount = cache.getBlockCount();
+        startBlockHits = cache.getStats().getHitCount();
+        startBlockMiss = cache.getStats().getMissCount();
+        i = -1;
+      }
+    }
+
     // insert data
     Put put = new Put(ROW);
     put.add(FAMILY, QUALIFIER, data);
@@ -4701,7 +4722,6 @@ public void testGetRegionsInRange() throws Exception {
     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
     int numOfRegions = TEST_UTIL.createMultiRegions(table, FAMILY);
     assertEquals(25, numOfRegions);
-    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
 
     // Get the regions in this range
     List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,

From ab5ab9aa1455052a28b52a7852742b675aa52a9f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 7 Sep 2012 04:04:42 +0000
Subject: [PATCH 0459/1540] HBASE-6286 Upgrade maven-compiler-plugin to 2.5.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1381861 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 23b3fd2d8cf1..8d72ae44800f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -350,6 +350,7 @@
         </plugin>
         <plugin>
           <artifactId>maven-compiler-plugin</artifactId>
+          <version>2.5.1</version>
           <configuration>
             <source>${compileSource}</source>
             <target>${compileSource}</target>

From 27b0d68de371f70b124e4d7a64cf1b762efb1487 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 7 Sep 2012 06:52:09 +0000
Subject: [PATCH 0460/1540] HBASE-6432 HRegionServer doesn't properly set
 clusterId in conf

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1381907 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegionServer.java     | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e0a7c24784e8..9630d05905d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -150,6 +150,7 @@
 import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
@@ -353,6 +354,11 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
    */
   private ObjectName mxBean = null;
 
+  /**
+   * ClusterId
+   */
+  private ClusterId clusterId = null;
+
   /**
    * Starts a HRegionServer at the default location
    *
@@ -552,6 +558,12 @@ public Integer apply(Writable from) {
   private void preRegistrationInitialization(){
     try {
       initializeZooKeeper();
+
+      clusterId = new ClusterId(zooKeeper, this);
+      if(clusterId.hasId()) {
+        conf.set(HConstants.CLUSTER_ID, clusterId.getId());
+      }
+
       initializeThreads();
       int nbBlocks = conf.getInt("hbase.regionserver.nbreservationblocks", 4);
       for (int i = 0; i < nbBlocks; i++) {

From 005cbcf93f54c877306dfd0df9902b592da21e55 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 7 Sep 2012 20:34:59 +0000
Subject: [PATCH 0461/1540] HBASE-6713 Stopping META/ROOT RS may take 50mins
 when some region is splitting (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1382163 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     | 34 ++++++++++++++++---
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9630d05905d7..761c98237e38 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -766,17 +766,29 @@ public void run() {
       // Just skip out w/o closing regions.  Used when testing.
     } else if (abortRequested) {
       if (this.fsOk) {
-        closeAllRegions(abortRequested); // Don't leave any open file handles
+        closeUserRegions(abortRequested); // Don't leave any open file handles
       }
       LOG.info("aborting server " + this.serverNameFromMasterPOV);
     } else {
-      closeAllRegions(abortRequested);
+      closeUserRegions(abortRequested);
       closeAllScanners();
       LOG.info("stopping server " + this.serverNameFromMasterPOV);
     }
     // Interrupt catalog tracker here in case any regions being opened out in
     // handlers are stuck waiting on meta or root.
     if (this.catalogTracker != null) this.catalogTracker.stop();
+
+    // Closing the compactSplit thread before closing meta regions
+    if (!this.killed && containsMetaTableRegions()) {
+      if (!abortRequested || this.fsOk) {
+        if (this.compactSplitThread != null) {
+          this.compactSplitThread.join();
+          this.compactSplitThread = null;
+        }
+        closeMetaTableRegions(abortRequested);
+      }
+    }
+
     if (!this.killed && this.fsOk) {
       waitOnAllRegionsToClose(abortRequested);
       LOG.info("stopping server " + this.serverNameFromMasterPOV +
@@ -811,11 +823,16 @@ public void run() {
     LOG.info(Thread.currentThread().getName() + " exiting");
   }
 
+  private boolean containsMetaTableRegions() {
+    return onlineRegions.containsKey(HRegionInfo.ROOT_REGIONINFO.getEncodedName())
+        || onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
+  }
+
   private boolean areAllUserRegionsOffline() {
     if (getNumberOfOnlineRegions() > 2) return false;
     boolean allUserRegionsOffline = true;
     for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
-      if (!e.getValue().getRegionInfo().isMetaRegion()) {
+      if (!e.getValue().getRegionInfo().isMetaTable()) {
         allUserRegionsOffline = false;
         break;
       }
@@ -1890,7 +1907,14 @@ private MapWritable reportForDuty() throws IOException {
    */
   protected void closeAllRegions(final boolean abort) {
     closeUserRegions(abort);
-    // Only root and meta should remain.  Are we carrying root or meta?
+    closeMetaTableRegions(abort);
+  }
+
+  /**
+   * Close root and meta regions if we carry them
+   * @param abort Whether we're running an abort.
+   */
+  void closeMetaTableRegions(final boolean abort) {
     HRegion meta = null;
     HRegion root = null;
     this.lock.writeLock().lock();
@@ -1922,7 +1946,7 @@ void closeUserRegions(final boolean abort) {
     try {
       for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
         HRegion r = e.getValue();
-        if (!r.getRegionInfo().isMetaRegion() && r.isAvailable()) {
+        if (!r.getRegionInfo().isMetaTable() && r.isAvailable()) {
           // Don't update zk with this close transition; pass false.
           closeRegion(r.getRegionInfo(), abort, false);
         }

From 6be9826a22fbe1c18df1b0834e20af04ab1df69f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 7 Sep 2012 23:20:07 +0000
Subject: [PATCH 0462/1540] HBASE-6340 HBase RPC should allow protocol
 extension with common interfaces.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1382206 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/coprocessor/Exec.java   |  2 +-
 .../org/apache/hadoop/hbase/ipc/Invocation.java | 17 +++++++++--------
 .../hadoop/hbase/ipc/WritableRpcEngine.java     |  4 ++--
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
index 75535b5efc42..da1489d3b4b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
@@ -64,7 +64,7 @@ public Exec(Configuration configuration,
       byte[] row,
       Class<? extends CoprocessorProtocol> protocol,
       Method method, Object[] parameters) {
-    super(method, parameters);
+    super(method, protocol, parameters);
     this.conf = configuration;
     this.referenceRow = row;
     this.protocol = protocol;
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
index 055787e560d4..10065709b762 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
@@ -45,26 +45,27 @@ public class Invocation extends VersionedWritable implements Configurable {
 
   public Invocation() {}
 
-  public Invocation(Method method, Object[] parameters) {
+  public Invocation(Method method,
+      Class<? extends VersionedProtocol> declaringClass, Object[] parameters) {
     this.methodName = method.getName();
     this.parameterClasses = method.getParameterTypes();
     this.parameters = parameters;
-    if (method.getDeclaringClass().equals(VersionedProtocol.class)) {
+    if (declaringClass.equals(VersionedProtocol.class)) {
       //VersionedProtocol is exempted from version check.
       clientVersion = 0;
       clientMethodsHash = 0;
     } else {
       try {
-        Field versionField = method.getDeclaringClass().getField("VERSION");
+        Field versionField = declaringClass.getField("VERSION");
         versionField.setAccessible(true);
-        this.clientVersion = versionField.getLong(method.getDeclaringClass());
+        this.clientVersion = versionField.getLong(declaringClass);
       } catch (NoSuchFieldException ex) {
-        throw new RuntimeException("The " + method.getDeclaringClass(), ex);
+        throw new RuntimeException("The " + declaringClass, ex);
       } catch (IllegalAccessException ex) {
         throw new RuntimeException(ex);
       }
-      this.clientMethodsHash = ProtocolSignature.getFingerprint(method
-          .getDeclaringClass().getMethods());
+      this.clientMethodsHash = ProtocolSignature.getFingerprint(
+          declaringClass.getMethods());
     }
   }
 
@@ -169,4 +170,4 @@ public Configuration getConf() {
   public byte getVersion() {
     return RPC_VERSION;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index ccc2809c5ec7..386111745042 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -148,7 +148,7 @@ public Object invoke(Object proxy, Method method, Object[] args)
       }
 
       HbaseObjectWritable value = (HbaseObjectWritable)
-        client.call(new Invocation(method, args), address,
+        client.call(new Invocation(method, protocol, args), address,
                     protocol, ticket, rpcTimeout);
       if (logDebug) {
         // FIGURE HOW TO TURN THIS OFF!
@@ -210,7 +210,7 @@ public Object[] call(Method method, Object[][] params,
 
     Invocation[] invocations = new Invocation[params.length];
     for (int i = 0; i < params.length; i++)
-      invocations[i] = new Invocation(method, params[i]);
+      invocations[i] = new Invocation(method, protocol, params[i]);
     HBaseClient client = CLIENTS.getClient(conf);
     try {
     Writable[] wrappedValues =

From a7dde5bb90376e4ff7ee9d1f2016383907135eb4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 7 Sep 2012 23:20:20 +0000
Subject: [PATCH 0463/1540] HBASE-6340 HBase RPC should allow protocol
 extension with common interfaces.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1382207 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/ipc/TestProtocolExtension.java      | 103 ++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java

diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
new file mode 100644
index 000000000000..fdc642513442
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/** Unit test for Protocol extending common interface. */
+@Category(SmallTests.class)
+public class TestProtocolExtension {
+  private static final String ADDRESS = "0.0.0.0";
+
+  public static final Log LOG =
+    LogFactory.getLog(TestProtocolExtension.class);
+  
+  private static Configuration conf = new Configuration();
+
+  public interface ProtocolExtention {
+    void logClassName();
+  }
+
+  public interface TestProtocol extends VersionedProtocol, ProtocolExtention {
+    public static final long VERSION = 7L;
+
+    void ping() throws IOException;
+
+    // @Override  // Uncomment to make the test pass
+    // public void logClassName();
+}
+
+  public static class TestImpl implements TestProtocol {
+    public long getProtocolVersion(String protocol, long clientVersion) {
+      return TestProtocol.VERSION;
+    }
+
+    @Override
+    public void ping() {}
+
+    @Override
+    public void logClassName() {
+      LOG.info(this.getClass().getName());
+    }
+
+    @Override
+    public ProtocolSignature getProtocolSignature(String protocol,
+        long clientVersion, int clientMethodsHash) throws IOException {
+      return new ProtocolSignature(VERSION, null);
+    }
+  }
+
+  @Test
+  public void testCalls() throws Exception {
+    RpcServer server = HBaseRPC.getServer(TestProtocol.class,
+                                  new TestImpl(),
+                                  new Class<?>[]{ProtocolExtention.class},
+                                  ADDRESS,
+                                  6016,
+                                  10, 10, false,
+                                  conf, 10);
+    TestProtocol proxy = null;
+    try {
+      server.start();
+
+      InetSocketAddress addr = server.getListenerAddress();
+      proxy = (TestProtocol)HBaseRPC.getProxy(
+          TestProtocol.class, TestProtocol.VERSION, addr, conf, 10000);
+
+      proxy.ping();
+
+      proxy.logClassName();
+    } finally {
+      server.stop();
+      if(proxy!=null) HBaseRPC.stopProxy(proxy);
+    }
+  }
+  
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 31bf1ba2a0888e4de9668501b1bf4ee8fd644efb Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 8 Sep 2012 20:19:33 +0000
Subject: [PATCH 0464/1540] HBASE-6734 Code duplication in
 LoadIncrementalHFiles (Richard Ding)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1382354 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java       | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 922e62165143..dea6aaa14e83 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -212,12 +212,6 @@ public void doBulkLoad(Path hfofDir, final HTable table)
         return;
       }
 
-      if (queue.isEmpty()) {
-        LOG.warn("Bulk load operation did not find any files to load in " +
-        "directory " + hfofDir.toUri() + ".  Does it contain files in " +
-        "subdirectories that correspond to column family names?");
-      }
-
       // Assumes that region splits can happen while this occurs.
       while (!queue.isEmpty()) {
         // need to reload split keys each iteration.

From f7087c5ae39ccfb924d77203ac9da81c1348b2ba Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Sun, 9 Sep 2012 16:21:30 +0000
Subject: [PATCH 0465/1540] HBASE-5631 hbck should handle case where .tableinfo
 file is missing (Jie Huang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1382530 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 137 +++++++++++++++++-
 .../hadoop/hbase/util/TestHBaseFsck.java      |  33 ++++-
 .../hbase/util/hbck/HbckTestingUtil.java      |   5 +-
 3 files changed, 163 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 99c55840ce29..c12977abf2b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -26,6 +26,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -53,6 +54,7 @@
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
@@ -177,6 +179,7 @@ public class HBaseFsck {
   private boolean fixHdfsHoles = false; // fix fs holes?
   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
+  private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
   private boolean fixSplitParents = false; // fix lingering split parents
 
@@ -226,6 +229,8 @@ public class HBaseFsck {
    * When initially looking at HDFS, we attempt to find any orphaned data.
    */
   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
+  
+  private Map<String, Set<String>> orphanTableDirs = new HashMap<String, Set<String>>();
 
   /**
    * Constructor
@@ -328,7 +333,8 @@ private void clearState() {
    */
   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
     // Initial pass to fix orphans.
-    if (shouldFixHdfsOrphans() || shouldFixHdfsHoles() || shouldFixHdfsOverlaps()) {
+    if (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
+        || shouldFixHdfsOverlaps() || shouldFixTableOrphans()) {
       LOG.info("Loading regioninfos HDFS");
       // if nothing is happening this should always complete in two iterations.
       int maxIterations = conf.getInt("hbase.hbck.integrityrepair.iterations.max", 3);
@@ -380,7 +386,7 @@ public int onlineConsistencyRepair() throws IOException, KeeperException,
     if (!checkMetaOnly) {
       reportTablesInFlux();
     }
-
+    
     // get regions according to what is online on each RegionServer
     loadDeployedRegions();
 
@@ -394,6 +400,9 @@ public int onlineConsistencyRepair() throws IOException, KeeperException,
     // Get disabled tables from ZooKeeper
     loadDisabledTables();
 
+    // fix the orphan tables
+    fixOrphanTables();
+
     // Check and fix consistency
     checkAndFixConsistency();
 
@@ -706,7 +715,7 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
       if (modTInfo == null) {
         // only executed once per table.
         modTInfo = new TableInfo(tableName);
-        Path hbaseRoot = new Path(conf.get(HConstants.HBASE_DIR));
+        Path hbaseRoot = FSUtils.getRootDir(conf);
         tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
@@ -714,9 +723,14 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
               hbaseRoot, tableName);
           modTInfo.htds.add(htd);
         } catch (IOException ioe) {
-          LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
-          errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, 
-              "Unable to read .tableinfo from " + hbaseRoot);
+          if (!orphanTableDirs.containsKey(tableName)) {
+            LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
+            //should only report once for each table
+            errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, 
+                "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
+            Set<String> columns = new HashSet<String>();
+            orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
+          }
         }
       }
       modTInfo.addRegionInfo(hbi);
@@ -724,6 +738,103 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
 
     return tablesInfo;
   }
+  
+  /**
+   * To get the column family list according to the column family dirs
+   * @param columns
+   * @param hbi
+   * @return
+   * @throws IOException
+   */
+  private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
+    Path regionDir = hbi.getHdfsRegionDir();
+    FileSystem fs = regionDir.getFileSystem(conf);
+    FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
+    for (FileStatus subdir : subDirs) {
+      String columnfamily = subdir.getPath().getName();
+      columns.add(columnfamily);
+    }
+    return columns;
+  }
+  
+  /**
+   * To fabricate a .tableinfo file with following contents<br>
+   * 1. the correct tablename <br>
+   * 2. the correct colfamily list<br>
+   * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
+   * @param tableName
+   * @throws IOException
+   */
+  private boolean fabricateTableInfo(String tableName, Set<String> columns) throws IOException {
+    if (columns ==null || columns.isEmpty()) return false;
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (String columnfamimly : columns) {
+      htd.addFamily(new HColumnDescriptor(columnfamimly));
+    }
+    FSTableDescriptors.createTableDescriptor(htd, conf, true);
+    return true;
+  }
+  
+  /**
+   * To fix orphan table by creating a .tableinfo file under tableDir <br>
+   * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
+   * 2. else create a default .tableinfo file with following items<br>
+   * &nbsp;2.1 the correct tablename <br>
+   * &nbsp;2.2 the correct colfamily list<br>
+   * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
+   * @throws IOException
+   */
+  public void fixOrphanTables() throws IOException {
+    if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
+
+      Path hbaseRoot = FSUtils.getRootDir(conf);
+      List<String> tmpList = new ArrayList<String>();
+      tmpList.addAll(orphanTableDirs.keySet());
+      HTableDescriptor[] htds = getHTableDescriptors(tmpList);
+      Iterator iter = orphanTableDirs.entrySet().iterator();
+      int j = 0; 
+      int numFailedCase = 0;
+      while (iter.hasNext()) {
+        Entry<String, Set<String>> entry = (Entry<String, Set<String>>) iter.next();
+        String tableName = entry.getKey();
+        LOG.info("Trying to fix orphan table error: " + tableName);
+        if (j < htds.length) {
+          if (tableName.equals(Bytes.toString(htds[j].getName()))) {
+            HTableDescriptor htd = htds[j];
+            LOG.info("fixing orphan table: " + tableName + " from cache");
+            FSTableDescriptors.createTableDescriptor(
+                hbaseRoot.getFileSystem(conf), hbaseRoot, htd, true);
+            j++;
+            iter.remove();
+          }
+        } else {
+          if (fabricateTableInfo(tableName, entry.getValue())) {
+            LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
+            LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
+            iter.remove();
+          } else {
+            LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
+            numFailedCase++;
+          }
+        }
+        fixes++;
+      }
+
+      if (orphanTableDirs.isEmpty()) {
+        // all orphanTableDirs are luckily recovered
+        // re-run doFsck after recovering the .tableinfo file
+        setShouldRerun();
+        LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
+      } else if (numFailedCase > 0) {
+        LOG.error("Failed to fix " + numFailedCase
+            + " OrphanTables with default .tableinfo files");
+      }
+
+    }
+    //cleanup the list
+    orphanTableDirs.clear();
+
+  }
 
   /**
    * This borrows code from MasterFileSystem.bootstrap()
@@ -3024,7 +3135,15 @@ public void setFixHdfsHoles(boolean shouldFix) {
   boolean shouldFixHdfsHoles() {
     return fixHdfsHoles;
   }
-
+  
+  public void setFixTableOrphans(boolean shouldFix) {
+    fixTableOrphans = shouldFix;
+  }
+   
+  boolean shouldFixTableOrphans() {
+    return fixTableOrphans;
+  }
+  
   public void setFixHdfsOverlaps(boolean shouldFix) {
     fixHdfsOverlaps = shouldFix;
   }
@@ -3166,6 +3285,7 @@ protected HBaseFsck printUsageAndExit() {
     System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
     System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
     System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
+    System.err.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
     System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
     System.err.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
     System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
@@ -3270,6 +3390,8 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setFixHdfsHoles(true);
       } else if (cmd.equals("-fixHdfsOrphans")) {
         setFixHdfsOrphans(true);
+      } else if (cmd.equals("-fixTableOrphans")) {
+        setFixTableOrphans(true);
       } else if (cmd.equals("-fixHdfsOverlaps")) {
         setFixHdfsOverlaps(true);
       } else if (cmd.equals("-fixVersionFile")) {
@@ -3396,6 +3518,7 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
       setFixHdfsHoles(false);
       setFixHdfsOverlaps(false);
       setFixVersionFile(false);
+      setFixTableOrphans(false);
       errors.resetErrors();
       code = onlineHbck();
       setRetCode(code);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 696e1221b969..6f6a9434b483 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -406,28 +406,55 @@ public void testHbckThreadpooling() throws Exception {
       deleteTable(table);
     }    
   }
-
+  
   @Test
-  public void testHbckMissingTableinfo() throws Exception {
+  public void testHbckFixOrphanTable() throws Exception {
     String table = "tableInfo";
     FileSystem fs = null;
     Path tableinfo = null;
     try {
       setupTable(table);
+      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+      
       Path hbaseTableDir = new Path(conf.get(HConstants.HBASE_DIR) + "/" + table );
       fs = hbaseTableDir.getFileSystem(conf);
       FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
       tableinfo = status.getPath();
       fs.rename(tableinfo, new Path("/.tableinfo"));
       
+      //to report error if .tableinfo is missing.
       HBaseFsck hbck = doFsck(conf, false); 
       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
+      
+      // fix OrphanTable with default .tableinfo
+      hbck = doFsck(conf, true);
+      assertNoErrors(hbck);
+      status = null;
+      status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
+      assertNotNull(status);
+      
+      HTableDescriptor htd = admin.getTableDescriptor(table.getBytes());
+      htd.setValue("NOT_DEFAULT", "true");
+      admin.disableTable(table);
+      admin.modifyTable(table.getBytes(), htd);
+      admin.enableTable(table);
+      fs.delete(status.getPath(), true);
+      
+      // fix OrphanTable with cache
+      htd = admin.getTableDescriptor(table.getBytes());
+      hbck = doFsck(conf, true);
+      assertNoErrors(hbck);
+      status = null;
+      status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
+      assertNotNull(status);
+      htd = admin.getTableDescriptor(table.getBytes());
+      assertEquals(htd.getValue("NOT_DEFAULT"), "true");
     } finally {
       fs.rename(new Path("/.tableinfo"), tableinfo);
       deleteTable(table);
     }
   }
-   
+
   /**
    * This create and fixes a bad table with regions that have a duplicate
    * start key
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index b25c4cb60343..299ae6e64d69 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -37,12 +37,12 @@ public static HBaseFsck doFsck(
 
   public static HBaseFsck doFsck(
       Configuration conf, boolean fix, String table) throws Exception {
-    return doFsck(conf, fix, fix, fix, fix,fix, fix, table);
+    return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, table);
   }
 
   public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
       boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
-      boolean fixHdfsOrphans, boolean fixVersionFile,
+      boolean fixHdfsOrphans, boolean fixTableOrphans, boolean fixVersionFile,
       String table) throws Exception {
     HBaseFsck fsck = new HBaseFsck(conf);
     fsck.connect();
@@ -53,6 +53,7 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
     fsck.setFixHdfsHoles(fixHdfsHoles);
     fsck.setFixHdfsOverlaps(fixHdfsOverlaps);
     fsck.setFixHdfsOrphans(fixHdfsOrphans);
+    fsck.setFixTableOrphans(fixTableOrphans);
     fsck.setFixVersionFile(fixVersionFile);
     if (table != null) {
       fsck.includeTable(table);

From b40dd44273b4964fa81ea70bc10680a218fdddfe Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 10 Sep 2012 03:05:06 +0000
Subject: [PATCH 0466/1540] HBASE-5631 ADDENDUM (extra comments)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1382628 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 6f6a9434b483..1066414a9a32 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -426,7 +426,7 @@ public void testHbckFixOrphanTable() throws Exception {
       HBaseFsck hbck = doFsck(conf, false); 
       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
       
-      // fix OrphanTable with default .tableinfo
+      // fix OrphanTable with default .tableinfo (htd not yet cached on master)
       hbck = doFsck(conf, true);
       assertNoErrors(hbck);
       status = null;
@@ -441,7 +441,7 @@ public void testHbckFixOrphanTable() throws Exception {
       fs.delete(status.getPath(), true);
       
       // fix OrphanTable with cache
-      htd = admin.getTableDescriptor(table.getBytes());
+      htd = admin.getTableDescriptor(table.getBytes()); // warms up cached htd on master
       hbck = doFsck(conf, true);
       assertNoErrors(hbck);
       status = null;

From 124d48e61ad112ffe48c721d8af42e3c2da0fe8a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 11 Sep 2012 18:53:14 +0000
Subject: [PATCH 0467/1540] HBASE-6340 HBase RPC should allow protocol
 extension with common interfaces.: REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1383537 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/coprocessor/Exec.java |   2 +-
 .../apache/hadoop/hbase/ipc/Invocation.java   |  17 ++-
 .../hadoop/hbase/ipc/WritableRpcEngine.java   |   4 +-
 .../hbase/ipc/TestProtocolExtension.java      | 103 ------------------
 4 files changed, 11 insertions(+), 115 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
index da1489d3b4b5..75535b5efc42 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
@@ -64,7 +64,7 @@ public Exec(Configuration configuration,
       byte[] row,
       Class<? extends CoprocessorProtocol> protocol,
       Method method, Object[] parameters) {
-    super(method, protocol, parameters);
+    super(method, parameters);
     this.conf = configuration;
     this.referenceRow = row;
     this.protocol = protocol;
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
index 10065709b762..055787e560d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
@@ -45,27 +45,26 @@ public class Invocation extends VersionedWritable implements Configurable {
 
   public Invocation() {}
 
-  public Invocation(Method method,
-      Class<? extends VersionedProtocol> declaringClass, Object[] parameters) {
+  public Invocation(Method method, Object[] parameters) {
     this.methodName = method.getName();
     this.parameterClasses = method.getParameterTypes();
     this.parameters = parameters;
-    if (declaringClass.equals(VersionedProtocol.class)) {
+    if (method.getDeclaringClass().equals(VersionedProtocol.class)) {
       //VersionedProtocol is exempted from version check.
       clientVersion = 0;
       clientMethodsHash = 0;
     } else {
       try {
-        Field versionField = declaringClass.getField("VERSION");
+        Field versionField = method.getDeclaringClass().getField("VERSION");
         versionField.setAccessible(true);
-        this.clientVersion = versionField.getLong(declaringClass);
+        this.clientVersion = versionField.getLong(method.getDeclaringClass());
       } catch (NoSuchFieldException ex) {
-        throw new RuntimeException("The " + declaringClass, ex);
+        throw new RuntimeException("The " + method.getDeclaringClass(), ex);
       } catch (IllegalAccessException ex) {
         throw new RuntimeException(ex);
       }
-      this.clientMethodsHash = ProtocolSignature.getFingerprint(
-          declaringClass.getMethods());
+      this.clientMethodsHash = ProtocolSignature.getFingerprint(method
+          .getDeclaringClass().getMethods());
     }
   }
 
@@ -170,4 +169,4 @@ public Configuration getConf() {
   public byte getVersion() {
     return RPC_VERSION;
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index 386111745042..ccc2809c5ec7 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -148,7 +148,7 @@ public Object invoke(Object proxy, Method method, Object[] args)
       }
 
       HbaseObjectWritable value = (HbaseObjectWritable)
-        client.call(new Invocation(method, protocol, args), address,
+        client.call(new Invocation(method, args), address,
                     protocol, ticket, rpcTimeout);
       if (logDebug) {
         // FIGURE HOW TO TURN THIS OFF!
@@ -210,7 +210,7 @@ public Object[] call(Method method, Object[][] params,
 
     Invocation[] invocations = new Invocation[params.length];
     for (int i = 0; i < params.length; i++)
-      invocations[i] = new Invocation(method, protocol, params[i]);
+      invocations[i] = new Invocation(method, params[i]);
     HBaseClient client = CLIENTS.getClient(conf);
     try {
     Writable[] wrappedValues =
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
deleted file mode 100644
index fdc642513442..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.ipc;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.SmallTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/** Unit test for Protocol extending common interface. */
-@Category(SmallTests.class)
-public class TestProtocolExtension {
-  private static final String ADDRESS = "0.0.0.0";
-
-  public static final Log LOG =
-    LogFactory.getLog(TestProtocolExtension.class);
-  
-  private static Configuration conf = new Configuration();
-
-  public interface ProtocolExtention {
-    void logClassName();
-  }
-
-  public interface TestProtocol extends VersionedProtocol, ProtocolExtention {
-    public static final long VERSION = 7L;
-
-    void ping() throws IOException;
-
-    // @Override  // Uncomment to make the test pass
-    // public void logClassName();
-}
-
-  public static class TestImpl implements TestProtocol {
-    public long getProtocolVersion(String protocol, long clientVersion) {
-      return TestProtocol.VERSION;
-    }
-
-    @Override
-    public void ping() {}
-
-    @Override
-    public void logClassName() {
-      LOG.info(this.getClass().getName());
-    }
-
-    @Override
-    public ProtocolSignature getProtocolSignature(String protocol,
-        long clientVersion, int clientMethodsHash) throws IOException {
-      return new ProtocolSignature(VERSION, null);
-    }
-  }
-
-  @Test
-  public void testCalls() throws Exception {
-    RpcServer server = HBaseRPC.getServer(TestProtocol.class,
-                                  new TestImpl(),
-                                  new Class<?>[]{ProtocolExtention.class},
-                                  ADDRESS,
-                                  6016,
-                                  10, 10, false,
-                                  conf, 10);
-    TestProtocol proxy = null;
-    try {
-      server.start();
-
-      InetSocketAddress addr = server.getListenerAddress();
-      proxy = (TestProtocol)HBaseRPC.getProxy(
-          TestProtocol.class, TestProtocol.VERSION, addr, conf, 10000);
-
-      proxy.ping();
-
-      proxy.logClassName();
-    } finally {
-      server.stop();
-      if(proxy!=null) HBaseRPC.stopProxy(proxy);
-    }
-  }
-  
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}

From 4d559b1459c732d540258b7453268af364a09232 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 12 Sep 2012 04:05:21 +0000
Subject: [PATCH 0468/1540] HBASE-6757 Very inefficient behaviour of scan using
 FilterList

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1383749 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/filter/FilterList.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index fee1a41df624..c3064e9b59c7 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -203,9 +203,6 @@ public ReturnCode filterKeyValue(KeyValue v) {
         switch (code) {
         case INCLUDE:
           continue;
-        case NEXT_ROW:
-        case SKIP:
-          return ReturnCode.SKIP;
         default:
           return code;
         }

From 7538a94704bdf13e4f0a69921b776e13439c36f0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 12 Sep 2012 04:11:15 +0000
Subject: [PATCH 0469/1540] HBASE-6340 Reapply with fix for SecureRpcEngine

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1383754 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/ipc/SecureRpcEngine.java     |   4 +-
 .../hadoop/hbase/client/coprocessor/Exec.java |   2 +-
 .../apache/hadoop/hbase/ipc/Invocation.java   |  17 +--
 .../hadoop/hbase/ipc/WritableRpcEngine.java   |   4 +-
 .../hbase/ipc/TestProtocolExtension.java      | 103 ++++++++++++++++++
 5 files changed, 117 insertions(+), 13 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
index 8219bea79930..87cbb5117123 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
@@ -162,7 +162,7 @@ public Object invoke(Object proxy, Method method, Object[] args)
         startTime = System.currentTimeMillis();
       }
       HbaseObjectWritable value = (HbaseObjectWritable)
-        client.call(new Invocation(method, args), address,
+        client.call(new Invocation(method, protocol, args), address,
                     protocol, ticket, rpcTimeout);
       if (logDebug) {
         long callTime = System.currentTimeMillis() - startTime;
@@ -234,7 +234,7 @@ public Object[] call(Method method, Object[][] params,
 
     Invocation[] invocations = new Invocation[params.length];
     for (int i = 0; i < params.length; i++)
-      invocations[i] = new Invocation(method, params[i]);
+      invocations[i] = new Invocation(method, protocol, params[i]);
     SecureClient client = CLIENTS.getClient(conf);
     try {
       Writable[] wrappedValues =
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
index 75535b5efc42..da1489d3b4b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
@@ -64,7 +64,7 @@ public Exec(Configuration configuration,
       byte[] row,
       Class<? extends CoprocessorProtocol> protocol,
       Method method, Object[] parameters) {
-    super(method, parameters);
+    super(method, protocol, parameters);
     this.conf = configuration;
     this.referenceRow = row;
     this.protocol = protocol;
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
index 055787e560d4..10065709b762 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
@@ -45,26 +45,27 @@ public class Invocation extends VersionedWritable implements Configurable {
 
   public Invocation() {}
 
-  public Invocation(Method method, Object[] parameters) {
+  public Invocation(Method method,
+      Class<? extends VersionedProtocol> declaringClass, Object[] parameters) {
     this.methodName = method.getName();
     this.parameterClasses = method.getParameterTypes();
     this.parameters = parameters;
-    if (method.getDeclaringClass().equals(VersionedProtocol.class)) {
+    if (declaringClass.equals(VersionedProtocol.class)) {
       //VersionedProtocol is exempted from version check.
       clientVersion = 0;
       clientMethodsHash = 0;
     } else {
       try {
-        Field versionField = method.getDeclaringClass().getField("VERSION");
+        Field versionField = declaringClass.getField("VERSION");
         versionField.setAccessible(true);
-        this.clientVersion = versionField.getLong(method.getDeclaringClass());
+        this.clientVersion = versionField.getLong(declaringClass);
       } catch (NoSuchFieldException ex) {
-        throw new RuntimeException("The " + method.getDeclaringClass(), ex);
+        throw new RuntimeException("The " + declaringClass, ex);
       } catch (IllegalAccessException ex) {
         throw new RuntimeException(ex);
       }
-      this.clientMethodsHash = ProtocolSignature.getFingerprint(method
-          .getDeclaringClass().getMethods());
+      this.clientMethodsHash = ProtocolSignature.getFingerprint(
+          declaringClass.getMethods());
     }
   }
 
@@ -169,4 +170,4 @@ public Configuration getConf() {
   public byte getVersion() {
     return RPC_VERSION;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index ccc2809c5ec7..386111745042 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -148,7 +148,7 @@ public Object invoke(Object proxy, Method method, Object[] args)
       }
 
       HbaseObjectWritable value = (HbaseObjectWritable)
-        client.call(new Invocation(method, args), address,
+        client.call(new Invocation(method, protocol, args), address,
                     protocol, ticket, rpcTimeout);
       if (logDebug) {
         // FIGURE HOW TO TURN THIS OFF!
@@ -210,7 +210,7 @@ public Object[] call(Method method, Object[][] params,
 
     Invocation[] invocations = new Invocation[params.length];
     for (int i = 0; i < params.length; i++)
-      invocations[i] = new Invocation(method, params[i]);
+      invocations[i] = new Invocation(method, protocol, params[i]);
     HBaseClient client = CLIENTS.getClient(conf);
     try {
     Writable[] wrappedValues =
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
new file mode 100644
index 000000000000..fdc642513442
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/** Unit test for Protocol extending common interface. */
+@Category(SmallTests.class)
+public class TestProtocolExtension {
+  private static final String ADDRESS = "0.0.0.0";
+
+  public static final Log LOG =
+    LogFactory.getLog(TestProtocolExtension.class);
+  
+  private static Configuration conf = new Configuration();
+
+  public interface ProtocolExtention {
+    void logClassName();
+  }
+
+  public interface TestProtocol extends VersionedProtocol, ProtocolExtention {
+    public static final long VERSION = 7L;
+
+    void ping() throws IOException;
+
+    // @Override  // Uncomment to make the test pass
+    // public void logClassName();
+}
+
+  public static class TestImpl implements TestProtocol {
+    public long getProtocolVersion(String protocol, long clientVersion) {
+      return TestProtocol.VERSION;
+    }
+
+    @Override
+    public void ping() {}
+
+    @Override
+    public void logClassName() {
+      LOG.info(this.getClass().getName());
+    }
+
+    @Override
+    public ProtocolSignature getProtocolSignature(String protocol,
+        long clientVersion, int clientMethodsHash) throws IOException {
+      return new ProtocolSignature(VERSION, null);
+    }
+  }
+
+  @Test
+  public void testCalls() throws Exception {
+    RpcServer server = HBaseRPC.getServer(TestProtocol.class,
+                                  new TestImpl(),
+                                  new Class<?>[]{ProtocolExtention.class},
+                                  ADDRESS,
+                                  6016,
+                                  10, 10, false,
+                                  conf, 10);
+    TestProtocol proxy = null;
+    try {
+      server.start();
+
+      InetSocketAddress addr = server.getListenerAddress();
+      proxy = (TestProtocol)HBaseRPC.getProxy(
+          TestProtocol.class, TestProtocol.VERSION, addr, conf, 10000);
+
+      proxy.ping();
+
+      proxy.logClassName();
+    } finally {
+      server.stop();
+      if(proxy!=null) HBaseRPC.stopProxy(proxy);
+    }
+  }
+  
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From e3c11db9ce901d001e0b0ddb474b2b87d1c98f0f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 12 Sep 2012 06:16:50 +0000
Subject: [PATCH 0470/1540] HBASE-5997 Fix concerns raised in HBASE-5922
 related to HalfStoreFileReader

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1383792 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/HalfStoreFileReader.java  | 27 +++++++++++++++++--
 .../hadoop/hbase/regionserver/Store.java      |  3 ++-
 .../hbase/io/TestHalfStoreFileReader.java     |  3 +++
 3 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index b53fa0bca5bb..da8f3f088498 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -52,6 +52,10 @@ public class HalfStoreFileReader extends StoreFile.Reader {
   // This is the key we split around.  Its the first possible entry on a row:
   // i.e. empty column and a timestamp of LATEST_TIMESTAMP.
   protected final byte [] splitkey;
+  
+  private byte[] firstKey = null;
+  
+  private boolean firstKeySeeked = false;
 
   /**
    * @param fs
@@ -141,8 +145,11 @@ public boolean seekBefore(byte[] key) throws IOException {
       public boolean seekBefore(byte [] key, int offset, int length)
       throws IOException {
         if (top) {
-          if (getComparator().compare(key, offset, length, splitkey, 0,
-              splitkey.length) < 0) {
+          byte[] fk = getFirstKey();
+          // This will be null when the file is empty in which we can not seekBefore to any key
+          if (fk == null) return false;
+          if (getComparator().compare(key, offset, length, fk, 0,
+              fk.length) <= 0) {
             return false;
           }
         } else {
@@ -271,4 +278,20 @@ public byte[] midkey() throws IOException {
     // Returns null to indicate file is not splitable.
     return null;
   }
+  
+  @Override
+  public byte[] getFirstKey() {
+    if (!firstKeySeeked) {
+      HFileScanner scanner = getScanner(true, true, false);
+      try {
+        if (scanner.seekTo()) {
+          this.firstKey = Bytes.toBytes(scanner.getKey());
+        }
+        firstKeySeeked = true;
+      } catch (IOException e) {
+        LOG.warn("Failed seekTo first KV in the file", e);
+      }
+    }
+    return this.firstKey;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 0d52a3e5fa89..43ff0360c07a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1852,6 +1852,7 @@ private void rowAtOrBeforeFromStoreFile(final StoreFile f,
     }
     // TODO: Cache these keys rather than make each time?
     byte [] fk = r.getFirstKey();
+    if (fk == null) return;
     KeyValue firstKV = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
     byte [] lk = r.getLastKey();
     KeyValue lastKV = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
@@ -1865,7 +1866,7 @@ private void rowAtOrBeforeFromStoreFile(final StoreFile f,
       firstOnRow = new KeyValue(lastKV.getRow(), HConstants.LATEST_TIMESTAMP);
     }
     // Get a scanner that caches blocks and that uses pread.
-    HFileScanner scanner = r.getHFileReader().getScanner(true, true, false);
+    HFileScanner scanner = r.getScanner(true, true, false);
     // Seek scanner.  If can't seek it, return.
     if (!seekToScanner(scanner, firstOnRow, firstKV)) return;
     // If we found candidate on firstOnRow, just return. THIS WILL NEVER HAPPEN!
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
index 3f695b191a93..7d8eaa75aa7c 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
@@ -195,6 +195,9 @@ public void testHalfScanner() throws IOException {
       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
       assertEquals(items.get(0), foundKeyValue);
 
+      // Try to seek before the splitKey in the top file
+      foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
+      assertNull(foundKeyValue);
     }
 
   private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,

From 6c4e39dcae76daf369729ae8b5f1bcb32c8cb994 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Thu, 13 Sep 2012 01:32:58 +0000
Subject: [PATCH 0471/1540] HBASE-6710 0.92/0.94 compatibility issues due to
 HBASE-5206

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1384181 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      |   6 +-
 .../apache/hadoop/hbase/util/HBaseFsck.java   |   4 +-
 .../hadoop/hbase/zookeeper/ZKTable.java       | 144 +++----------
 .../hbase/zookeeper/ZKTableReadOnly.java      | 153 ++++++++++++++
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java |   3 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java     |  21 +-
 .../apache/hadoop/hbase/client/TestAdmin.java |   4 +-
 .../hadoop/hbase/zookeeper/TestZKTable.java   | 193 ++++++++++++++++--
 .../hbase/zookeeper/TestZKTableReadOnly.java  | 103 ++++++++++
 9 files changed, 493 insertions(+), 138 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index e7ebe0cfe0b5..ba2dad3b9cac 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -76,7 +76,7 @@
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
-import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.ipc.RemoteException;
@@ -782,9 +782,9 @@ private boolean testTableOnlineState(byte [] tableName, boolean online)
       String tableNameStr = Bytes.toString(tableName);
       try {
         if (online) {
-          return ZKTable.isEnabledTable(zkw, tableNameStr);
+          return ZKTableReadOnly.isEnabledTable(zkw, tableNameStr);
         }
-        return ZKTable.isDisabledTable(zkw, tableNameStr);
+        return ZKTableReadOnly.isDisabledTable(zkw, tableNameStr);
       } catch (KeeperException e) {
         throw new IOException("Enable/Disable failed", e);
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index c12977abf2b9..1d7f9dfec52c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -89,7 +89,7 @@
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
-import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -1156,7 +1156,7 @@ private void loadDisabledTables()
       public Void connect(HConnection connection) throws IOException {
         ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
         try {
-          for (String tableName : ZKTable.getDisabledOrDisablingTables(zkw)) {
+          for (String tableName : ZKTableReadOnly.getDisabledOrDisablingTables(zkw)) {
             disabledTables.add(Bytes.toBytes(tableName));
           }
         } catch (KeeperException ke) {
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 16c0a7a3c4e8..2ac04e9ebcc6 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -36,9 +36,8 @@
  * Reads, caches and sets state up in zookeeper.  If multiple read/write
  * clients, will make for confusion.  Read-only clients other than
  * AssignmentManager interested in learning table state can use the
- * read-only utility methods {@link #isEnabledTable(ZooKeeperWatcher, String)}
- * and {@link #isDisabledTable(ZooKeeperWatcher, String)}.
- * 
+ * read-only utility methods in {@link ZKTableReadOnly}.
+ *
  * <p>To save on trips to the zookeeper ensemble, internally we cache table
  * state.
  */
@@ -63,8 +62,8 @@ public class ZKTable {
   // Have watcher on table znode so all are notified of state or schema change.
   /**
    * States a Table can be in.
-   * {@link TableState#ENABLED} is not used currently; its the absence of state
-   * in zookeeper that indicates an enabled table currently.
+   * Compatibility note: ENABLED does not exist in 0.92 releases.  In 0.92, the absence of
+   * the znode indicates the table is enabled.
    */
   public static enum TableState {
     ENABLED,
@@ -87,7 +86,7 @@ private void populateTableStates()
   throws KeeperException {
     synchronized (this.cache) {
       List<String> children =
-        ZKUtil.listChildrenNoWatch(this.watcher, this.watcher.tableZNode);
+        ZKUtil.listChildrenNoWatch(this.watcher, this.watcher.masterTableZNode);
       if (children == null) return;
       for (String child: children) {
         TableState state = getTableState(this.watcher, child);
@@ -105,18 +104,7 @@ private void populateTableStates()
   private static TableState getTableState(final ZooKeeperWatcher zkw,
       final String child)
   throws KeeperException {
-    String znode = ZKUtil.joinZNode(zkw.tableZNode, child);
-    byte [] data = ZKUtil.getData(zkw, znode);
-    if (data == null || data.length <= 0) {
-      // Null if table is enabled.
-      return null;
-    }
-    String str = Bytes.toString(data);
-    try {
-      return TableState.valueOf(str);
-    } catch (IllegalArgumentException e) {
-      throw new IllegalArgumentException(str);
-    }
+    return ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode, child);
   }
 
   /**
@@ -226,11 +214,29 @@ public boolean checkEnabledAndSetDisablingTable(final String tableName)
 
   private void setTableState(final String tableName, final TableState state)
   throws KeeperException {
-    String znode = ZKUtil.joinZNode(this.watcher.tableZNode, tableName);
+    String znode = ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName);
     if (ZKUtil.checkExists(this.watcher, znode) == -1) {
       ZKUtil.createAndFailSilent(this.watcher, znode);
     }
+    String znode92 = ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName);
+    boolean settingToEnabled = (state == TableState.ENABLED);
+    // 0.92 format znode differs in that it is deleted to represent ENABLED,
+    // so only create if we are not setting to enabled.
+    if (!settingToEnabled) {
+      if (ZKUtil.checkExists(this.watcher, znode92) == -1) {
+        ZKUtil.createAndFailSilent(this.watcher, znode92);
+      }
+    }
     synchronized (this.cache) {
+      if (settingToEnabled) {
+        ZKUtil.deleteNodeFailSilent(this.watcher, znode92);
+      }
+      else {
+        ZKUtil.setData(this.watcher, znode92, Bytes.toBytes(state.toString()));
+      }
+      // Set the current format znode after the 0.92 format znode.
+      // This is so in the case of failure, the AssignmentManager is guaranteed to
+      // see the state was not applied, since it uses the current format znode internally.
       ZKUtil.setData(this.watcher, znode, Bytes.toBytes(state.toString()));
       this.cache.put(tableName, state);
     }
@@ -240,22 +246,6 @@ public boolean isDisabledTable(final String tableName) {
     return isTableState(tableName, TableState.DISABLED);
   }
 
-  /**
-   * Go to zookeeper and see if state of table is {@link TableState#DISABLED}.
-   * This method does not use cache as {@link #isDisabledTable(String)} does.
-   * This method is for clients other than {@link AssignmentManager}
-   * @param zkw
-   * @param tableName
-   * @return True if table is enabled.
-   * @throws KeeperException
-   */
-  public static boolean isDisabledTable(final ZooKeeperWatcher zkw,
-      final String tableName)
-  throws KeeperException {
-    TableState state = getTableState(zkw, tableName);
-    return isTableState(TableState.DISABLED, state);
-  }
-
   public boolean isDisablingTable(final String tableName) {
     return isTableState(tableName, TableState.DISABLING);
   }
@@ -268,46 +258,12 @@ public boolean isEnabledTable(String tableName) {
     return isTableState(tableName, TableState.ENABLED);
   }
 
-  /**
-   * Go to zookeeper and see if state of table is {@link TableState#ENABLED}.
-   * This method does not use cache as {@link #isEnabledTable(String)} does.
-   * This method is for clients other than {@link AssignmentManager}
-   * @param zkw
-   * @param tableName
-   * @return True if table is enabled.
-   * @throws KeeperException
-   */
-  public static boolean isEnabledTable(final ZooKeeperWatcher zkw,
-      final String tableName)
-  throws KeeperException {
-    TableState state = getTableState(zkw, tableName);
-    return state == null || state == TableState.ENABLED;
-  }
-
   public boolean isDisablingOrDisabledTable(final String tableName) {
     synchronized (this.cache) {
       return isDisablingTable(tableName) || isDisabledTable(tableName);
     }
   }
 
-  /**
-   * Go to zookeeper and see if state of table is {@link TableState#DISABLING}
-   * of {@link TableState#DISABLED}.
-   * This method does not use cache as {@link #isEnabledTable(String)} does.
-   * This method is for clients other than {@link AssignmentManager}.
-   * @param zkw
-   * @param tableName
-   * @return True if table is enabled.
-   * @throws KeeperException
-   */
-  public static boolean isDisablingOrDisabledTable(final ZooKeeperWatcher zkw,
-      final String tableName)
-  throws KeeperException {
-    TableState state = getTableState(zkw, tableName);
-    return isTableState(TableState.DISABLING, state) ||
-      isTableState(TableState.DISABLED, state);
-  }
-
   public boolean isEnabledOrDisablingTable(final String tableName) {
     synchronized (this.cache) {
       return isEnabledTable(tableName) || isDisablingTable(tableName);
@@ -323,15 +279,10 @@ public boolean isDisabledOrEnablingTable(final String tableName) {
   private boolean isTableState(final String tableName, final TableState state) {
     synchronized (this.cache) {
       TableState currentState = this.cache.get(tableName);
-      return isTableState(currentState, state);
+      return ZKTableReadOnly.isTableState(currentState, state);
     }
   }
 
-  private static boolean isTableState(final TableState expectedState,
-      final TableState currentState) {
-    return currentState != null && currentState.equals(expectedState);
-  }
-
   /**
    * Deletes the table in zookeeper.  Fails silently if the
    * table is not currently disabled in zookeeper.  Sets no watches.
@@ -341,12 +292,17 @@ private static boolean isTableState(final TableState expectedState,
   public void setDeletedTable(final String tableName)
   throws KeeperException {
     synchronized (this.cache) {
+      ZKUtil.deleteNodeFailSilent(this.watcher,
+        ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName));
+      // Delete the current format znode after the 0.92 format znode.
+      // This is so in the case of failure, the AssignmentManager is guaranteed to
+      // see the table was not deleted, since it uses the current format znode internally.
+      ZKUtil.deleteNodeFailSilent(this.watcher,
+        ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName));
       if (this.cache.remove(tableName) == null) {
         LOG.warn("Moving table " + tableName + " state to deleted but was " +
           "already deleted");
       }
-      ZKUtil.deleteNodeFailSilent(this.watcher,
-        ZKUtil.joinZNode(this.watcher.tableZNode, tableName));
     }
   }
   
@@ -389,38 +345,4 @@ public Set<String> getDisabledTables() {
     return disabledTables;
   }
 
-  /**
-   * Gets a list of all the tables set as disabled in zookeeper.
-   * @return Set of disabled tables, empty Set if none
-   * @throws KeeperException 
-   */
-  public static Set<String> getDisabledTables(ZooKeeperWatcher zkw)
-  throws KeeperException {
-    Set<String> disabledTables = new HashSet<String>();
-    List<String> children =
-      ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode);
-    for (String child: children) {
-      TableState state = getTableState(zkw, child);
-      if (state == TableState.DISABLED) disabledTables.add(child);
-    }
-    return disabledTables;
-  }
-
-  /**
-   * Gets a list of all the tables set as disabled in zookeeper.
-   * @return Set of disabled tables, empty Set if none
-   * @throws KeeperException 
-   */
-  public static Set<String> getDisabledOrDisablingTables(ZooKeeperWatcher zkw)
-  throws KeeperException {
-    Set<String> disabledTables = new HashSet<String>();
-    List<String> children =
-      ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode);
-    for (String child: children) {
-      TableState state = getTableState(zkw, child);
-      if (state == TableState.DISABLED || state == TableState.DISABLING)
-        disabledTables.add(child);
-    }
-    return disabledTables;
-  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
new file mode 100644
index 000000000000..1784f794374b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
@@ -0,0 +1,153 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Non-instantiable class that provides helper functions for
+ * clients other than {@link AssignmentManager} for reading the
+ * state of a table in ZK.
+ *
+ * <p>Does not cache state like {@link ZKTable}, actually reads from ZK each call.
+ */
+public class ZKTableReadOnly {
+
+  private ZKTableReadOnly() {}
+
+  /**
+   * Go to zookeeper and see if state of table is {@link TableState#DISABLED}.
+   * This method does not use cache as {@link #isDisabledTable(String)} does.
+   * This method is for clients other than {@link AssignmentManager}
+   * @param zkw
+   * @param tableName
+   * @return True if table is enabled.
+   * @throws KeeperException
+   */
+  public static boolean isDisabledTable(final ZooKeeperWatcher zkw,
+      final String tableName)
+  throws KeeperException {
+    TableState state = getTableState(zkw, tableName);
+    return isTableState(TableState.DISABLED, state);
+  }
+
+  /**
+   * Go to zookeeper and see if state of table is {@link TableState#ENABLED}.
+   * @param zkw
+   * @param tableName
+   * @return True if table is enabled.
+   * @throws KeeperException
+   */
+  public static boolean isEnabledTable(final ZooKeeperWatcher zkw,
+      final String tableName) throws KeeperException {
+    TableState state = getTableState(zkw, tableName);
+    return state == null || state == TableState.ENABLED;
+  }
+
+  /**
+   * Go to zookeeper and see if state of table is {@link TableState#DISABLING}
+   * of {@link TableState#DISABLED}.
+   * @param zkw
+   * @param tableName
+   * @return True if table is enabled.
+   * @throws KeeperException
+   */
+  public static boolean isDisablingOrDisabledTable(final ZooKeeperWatcher zkw,
+      final String tableName) throws KeeperException {
+    TableState state = getTableState(zkw, tableName);
+    return isTableState(TableState.DISABLING, state) ||
+      isTableState(TableState.DISABLED, state);
+  }
+
+  /**
+   * Gets a list of all the tables set as disabled in zookeeper.
+   * @return Set of disabled tables, empty Set if none
+   * @throws KeeperException
+   */
+  public static Set<String> getDisabledTables(ZooKeeperWatcher zkw)
+  throws KeeperException {
+    Set<String> disabledTables = new HashSet<String>();
+    List<String> children =
+      ZKUtil.listChildrenNoWatch(zkw, zkw.clientTableZNode);
+    for (String child: children) {
+      TableState state = getTableState(zkw, child);
+      if (state == TableState.DISABLED) disabledTables.add(child);
+    }
+    return disabledTables;
+  }
+
+  /**
+   * Gets a list of all the tables set as disabled in zookeeper.
+   * @return Set of disabled tables, empty Set if none
+   * @throws KeeperException
+   */
+  public static Set<String> getDisabledOrDisablingTables(ZooKeeperWatcher zkw)
+  throws KeeperException {
+    Set<String> disabledTables = new HashSet<String>();
+    List<String> children =
+      ZKUtil.listChildrenNoWatch(zkw, zkw.clientTableZNode);
+    for (String child: children) {
+      TableState state = getTableState(zkw, child);
+      if (state == TableState.DISABLED || state == TableState.DISABLING)
+        disabledTables.add(child);
+    }
+    return disabledTables;
+  }
+
+  static boolean isTableState(final TableState expectedState,
+    final TableState currentState) {
+    return currentState != null && currentState.equals(expectedState);
+  }
+
+  /**
+   * Read the TableState from ZooKeeper
+   * @throws KeeperException
+   */
+  static TableState getTableState(final ZooKeeperWatcher zkw,
+    final String child) throws KeeperException {
+    return getTableState(zkw, zkw.clientTableZNode, child);
+  }
+
+  /**
+   * @deprecated Only for 0.92/0.94 compatibility.  Use getTableState(zkw, child) instead.
+   */
+  static TableState getTableState(final ZooKeeperWatcher zkw,
+    final String parent, final String child) throws KeeperException {
+    String znode = ZKUtil.joinZNode(parent, child);
+    byte [] data = ZKUtil.getData(zkw, znode);
+    if (data == null || data.length <= 0) {
+      return null;
+    }
+    String str = Bytes.toString(data);
+    try {
+      return TableState.valueOf(str);
+    } catch (IllegalArgumentException e) {
+      throw new IllegalArgumentException(str);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 56d8cb404813..b95dd3a2dddf 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -725,7 +725,8 @@ private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
           (node.equals(zkw.clusterIdZNode) == true) ||
           (node.equals(zkw.rsZNode) == true) ||
           (node.equals(zkw.backupMasterAddressesZNode) == true) ||
-          (node.startsWith(zkw.tableZNode) == true)) {
+          (node.startsWith(zkw.masterTableZNode) == true) ||
+          (node.startsWith(zkw.masterTableZNode92) == true)) {
         return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE;
       }
       return Ids.CREATOR_ALL_ACL;
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 57e448413c22..b720d785e1a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -96,8 +96,13 @@ public class ZooKeeperWatcher implements Watcher, Abortable {
   public String clusterStateZNode;
   // znode used for region transitioning and assignment
   public String assignmentZNode;
-  // znode used for table disabling/enabling
-  public String tableZNode;
+  // znode that the master uses for reading/writing the table disabling/enabling states
+  public String masterTableZNode;
+  // znode where the client reads table enabling/disabling states.
+  public String clientTableZNode;
+  // znode where the master writes table disabling/enabling states in the format expected
+  // by 0.92.0/0.92.1 clients for backwards compatibility.  See HBASE-6710 for details.
+  public String masterTableZNode92;
   // znode containing the unique cluster ID
   public String clusterIdZNode;
   // znode used for log splitting work assignment
@@ -162,7 +167,8 @@ private void createBaseZNodes() throws ZooKeeperConnectionException {
       ZKUtil.createAndFailSilent(this, assignmentZNode);
       ZKUtil.createAndFailSilent(this, rsZNode);
       ZKUtil.createAndFailSilent(this, drainingZNode);
-      ZKUtil.createAndFailSilent(this, tableZNode);
+      ZKUtil.createAndFailSilent(this, masterTableZNode);
+      ZKUtil.createAndFailSilent(this, masterTableZNode92);
       ZKUtil.createAndFailSilent(this, splitLogZNode);
       ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
     } catch (KeeperException e) {
@@ -210,8 +216,13 @@ private void setNodeNames(Configuration conf) {
         conf.get("zookeeper.znode.state", "shutdown"));
     assignmentZNode = ZKUtil.joinZNode(baseZNode,
         conf.get("zookeeper.znode.unassigned", "unassigned"));
-    tableZNode = ZKUtil.joinZNode(baseZNode,
-        conf.get("zookeeper.znode.tableEnableDisable", "table"));
+    String tableZNodeDefault = "table";
+    masterTableZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.masterTableEnableDisable", tableZNodeDefault));
+    clientTableZNode = ZKUtil.joinZNode(baseZNode,
+            conf.get("zookeeper.znode.clientTableEnableDisable", tableZNodeDefault));
+    masterTableZNode92 = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.masterTableEnableDisable92", "table92"));
     clusterIdZNode = ZKUtil.joinZNode(baseZNode,
         conf.get("zookeeper.znode.clusterId", "hbaseid"));
     splitLogZNode = ZKUtil.joinZNode(baseZNode,
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 260b17b37062..eab7ccef19d7 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -51,7 +51,7 @@
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.*;
 import org.junit.experimental.categories.Category;
@@ -1005,7 +1005,7 @@ public void testEnableDisableAddColumnDeleteColumn() throws Exception {
     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
     byte [] tableName = Bytes.toBytes("testMasterAdmin");
     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
-    while (!ZKTable.isEnabledTable(zkw, "testMasterAdmin")) {
+    while (!ZKTableReadOnly.isEnabledTable(zkw, "testMasterAdmin")) {
       Thread.sleep(10);
     }
     this.admin.disableTable(tableName);
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
index 0a78f4039f32..4748b95ceb4d 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
@@ -28,15 +28,18 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 @Category(MediumTests.class)
 public class TestZKTable {
-  private static final Log LOG = LogFactory.getLog(TestZooKeeperNodeTracker.class);
+  private static final Log LOG = LogFactory.getLog(TestZKTable.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
   @BeforeClass
@@ -49,22 +52,23 @@ public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniZKCluster();
   }
 
+  Abortable abortable = new Abortable() {
+    @Override
+    public void abort(String why, Throwable e) {
+      LOG.info(why, e);
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+  };
+
   @Test
   public void testTableStates()
   throws ZooKeeperConnectionException, IOException, KeeperException {
     final String name = "testDisabled";
-    Abortable abortable = new Abortable() {
-      @Override
-      public void abort(String why, Throwable e) {
-        LOG.info(why, e);
-      }
-      
-      @Override
-      public boolean isAborted() {
-        return false;
-      }
-      
-    };
+
     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
       name, abortable, true);
     ZKTable zkt = new ZKTable(zkw);
@@ -106,8 +110,169 @@ public boolean isAborted() {
     assertFalse(zkt.isTablePresent(name));
   }
 
+  /**
+   * Test that ZK table writes table state in formats expected by 0.92 and 0.94 clients
+   */
+  @Test
+  public void test9294Compatibility() throws Exception {
+    final String tableName = "test9294Compatibility";
+
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+      tableName, abortable, true);
+    ZKTable zkt = new ZKTable(zkw);
+    zkt.setEnabledTable(tableName);
+    // check that current/0.94 format table has proper ENABLED format
+    assertTrue(
+      ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode,  tableName) == TableState.ENABLED);
+    // check that 0.92 format table is null, as expected by 0.92.0/0.92.1 clients
+    assertTrue(ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode92, tableName) == null);
+  }
+
+  /**
+   * RecoverableZookeeper that throws a KeeperException after throwExceptionInNumOperations
+   */
+  class ThrowingRecoverableZookeeper extends RecoverableZooKeeper {
+    private ZooKeeperWatcher zkw;
+    private int throwExceptionInNumOperations;
+
+    public ThrowingRecoverableZookeeper(ZooKeeperWatcher zkw) throws Exception {
+      super(ZKConfig.getZKQuorumServersString(TEST_UTIL.getConfiguration()),
+        HConstants.DEFAULT_ZK_SESSION_TIMEOUT, zkw, 3, 1000);
+      this.zkw = zkw;
+      this.throwExceptionInNumOperations = 0; // indicate not to throw an exception
+    }
+
+    public void setThrowExceptionInNumOperations(int throwExceptionInNumOperations) {
+      this.throwExceptionInNumOperations = throwExceptionInNumOperations;
+    }
+
+    private void checkThrowKeeperException() throws KeeperException {
+      if (throwExceptionInNumOperations == 1) {
+        throwExceptionInNumOperations = 0;
+        throw new KeeperException.DataInconsistencyException();
+      }
+      if(throwExceptionInNumOperations > 0) throwExceptionInNumOperations--;
+    }
+
+    public Stat setData(String path, byte[] data, int version)
+    throws KeeperException, InterruptedException {
+      checkThrowKeeperException();
+      return zkw.getRecoverableZooKeeper().setData(path, data, version);
+    }
+
+    public void delete(String path, int version)
+    throws InterruptedException, KeeperException {
+      checkThrowKeeperException();
+      zkw.getRecoverableZooKeeper().delete(path, version);
+    }
+  }
+  /**
+   * Because two ZooKeeper nodes are written for each table state transition
+   * {@link ZooKeeperWatcher#masterTableZNode} and {@link ZooKeeperWatcher#masterTableZNode92}
+   * it is possible that we fail in between the two operations and are left with
+   * inconsistent state.  Check that we can get back to a consistent state by
+   * retrying the operation.
+   */
+  @Test
+  public void testDisableTableRetry() throws Exception {
+    final String tableName = "testDisableTableRetry";
+
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+      tableName, abortable, true);
+    ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw);
+    ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw);
+    Mockito.doReturn(throwing).when(spyZookeeperWatcher).getRecoverableZooKeeper();
+
+    ZKTable zkt = new ZKTable(spyZookeeperWatcher);
+    zkt.setEnabledTable(tableName);
+    assertTrue(zkt.isEnabledOrDisablingTable(tableName));
+    boolean caughtExpectedException = false;
+    try {
+      // throw an exception on the second ZK operation, which means the first will succeed.
+      throwing.setThrowExceptionInNumOperations(2);
+      zkt.setDisabledTable(tableName);
+    } catch (KeeperException ke) {
+      caughtExpectedException = true;
+    }
+    assertTrue(caughtExpectedException);
+    assertFalse(zkt.isDisabledTable(tableName));
+    // try again, ensure table is disabled
+    zkt.setDisabledTable(tableName);
+    // ensure disabled from master perspective
+    assertTrue(zkt.isDisabledTable(tableName));
+    // ensure disabled from client perspective
+    assertTrue(ZKTableReadOnly.isDisabledTable(zkw, tableName));
+  }
+
+  /**
+   * Same as above, but with enableTable
+   */
+  @Test
+  public void testEnableTableRetry() throws Exception {
+    final String tableName = "testEnableTableRetry";
+
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+      tableName, abortable, true);
+    ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw);
+    ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw);
+    Mockito.doReturn(throwing).when(spyZookeeperWatcher).getRecoverableZooKeeper();
+
+    ZKTable zkt = new ZKTable(spyZookeeperWatcher);
+    zkt.setDisabledTable(tableName);
+    assertTrue(zkt.isDisabledTable(tableName));
+    boolean caughtExpectedException = false;
+    try {
+      // throw an exception on the second ZK operation, which means the first will succeed.
+      throwing.throwExceptionInNumOperations = 2;
+      zkt.setEnabledTable(tableName);
+    } catch (KeeperException ke) {
+      caughtExpectedException = true;
+    }
+    assertTrue(caughtExpectedException);
+    assertFalse(zkt.isEnabledTable(tableName));
+    // try again, ensure table is enabled
+    zkt.setEnabledTable(tableName);
+    // ensure enabled from master perspective
+    assertTrue(zkt.isEnabledTable(tableName));
+    // ensure enabled from client perspective
+    assertTrue(ZKTableReadOnly.isEnabledTable(zkw, tableName));
+  }
+
+  /**
+   * Same as above, but with deleteTable
+   */
+  @Test
+  public void testDeleteTableRetry() throws Exception {
+    final String tableName = "testEnableTableRetry";
+
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+      tableName, abortable, true);
+    ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw);
+    ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw);
+    Mockito.doReturn(throwing).when(spyZookeeperWatcher).getRecoverableZooKeeper();
+
+    ZKTable zkt = new ZKTable(spyZookeeperWatcher);
+    zkt.setDisabledTable(tableName);
+    assertTrue(zkt.isDisabledTable(tableName));
+    boolean caughtExpectedException = false;
+    try {
+      // throw an exception on the second ZK operation, which means the first will succeed.
+      throwing.setThrowExceptionInNumOperations(2);
+      zkt.setDeletedTable(tableName);
+    } catch (KeeperException ke) {
+      caughtExpectedException = true;
+    }
+    assertTrue(caughtExpectedException);
+    assertTrue(zkt.isTablePresent(tableName));
+    // try again, ensure table is deleted
+    zkt.setDeletedTable(tableName);
+    // ensure deleted from master perspective
+    assertFalse(zkt.isTablePresent(tableName));
+    // ensure deleted from client perspective
+    assertFalse(ZKTableReadOnly.getDisabledTables(zkw).contains(tableName));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java
new file mode 100644
index 000000000000..12b398ad5e51
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java
@@ -0,0 +1,103 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestZKTableReadOnly {
+  private static final Log LOG = LogFactory.getLog(TestZooKeeperNodeTracker.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  Abortable abortable = new Abortable() {
+    @Override
+    public void abort(String why, Throwable e) {
+      LOG.info(why, e);
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+  };
+
+  private boolean enableAndCheckEnabled(ZooKeeperWatcher zkw, String tableName) throws Exception {
+    // set the table to enabled, as that is the only state that differs
+    // between the two formats
+    ZKTable zkt = new ZKTable(zkw);
+    zkt.setEnabledTable(tableName);
+    return ZKTableReadOnly.isEnabledTable(zkw, tableName);
+  }
+
+  /**
+   * Test that client ZK reader can handle the 0.92 table znode format.
+   */
+  @Test
+  public void testClientCompatibilityWith92ZNode() throws Exception {
+    final String tableName = "testClientCompatibilityWith92ZNode";
+
+    // Set the client to read from the 0.92 table znode format
+    Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+    String znode92 = conf.get("zookeeper.znode.masterTableEnableDisable92", "table92");
+    conf.set("zookeeper.znode.clientTableEnableDisable", znode92);
+
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
+      tableName, abortable, true);
+    assertTrue(enableAndCheckEnabled(zkw, tableName));
+  }
+
+  /**
+   * Test that client ZK reader can handle the current (0.94) table format znode
+   */
+  @Test
+  public void testClientCompatibilityWith94ZNode() throws Exception {
+    final String tableName = "testClientCompatibilityWith94ZNode";
+
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+      tableName, abortable, true);
+    assertTrue(enableAndCheckEnabled(zkw, tableName));
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 7b140df6e0229da4b414a113e3009ec38db328d8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 13 Sep 2012 15:45:54 +0000
Subject: [PATCH 0472/1540] HBASE-6769 HRS.multi eats
 NoSuchColumnFamilyException (Elliott Clark)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1384378 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  1 +
 .../hadoop/hbase/regionserver/HRegion.java    | 15 ++++--
 .../hbase/regionserver/HRegionServer.java     |  5 ++
 .../hbase/client/TestFromClientSide.java      | 22 ++++++++
 .../hbase/regionserver/TestHRegion.java       | 51 +++++++++++++++++--
 5 files changed, 87 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 25f5e15d876c..5f4a6c381dd8 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -38,6 +38,7 @@ public final class HConstants {
   public enum OperationStatusCode {
     NOT_RUN,
     SUCCESS,
+    BAD_FAMILY,
     SANITY_CHECK_FAILURE,
     FAILURE;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e8d2c2684c9a..d27bff54458a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2065,10 +2065,19 @@ private long doMiniBatchMutation(
           } else {
             prepareDelete((Delete) mutation);
           }
-        } catch (DoNotRetryIOException dnrioe) {
-          LOG.warn("No such column family in batch mutation", dnrioe);
+        } catch (NoSuchColumnFamilyException nscf) {
+          LOG.warn("No such column family in batch mutation", nscf);
           batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
-              OperationStatusCode.SANITY_CHECK_FAILURE, dnrioe.getMessage());
+              OperationStatusCode.BAD_FAMILY, nscf.getMessage());
+          lastIndexExclusive++;
+          continue;
+        } catch (DoNotRetryIOException fsce) {
+          // The only thing that throws a generic DoNotRetryIOException in the above code is
+          // checkTimestamps so that DoNotRetryIOException means that timestamps were invalid.
+          // If more checks are added, be sure to revisit this assumption.
+          LOG.warn("Batch Mutation did not pass sanity check", fsce);
+          batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
+              OperationStatusCode.SANITY_CHECK_FAILURE, fsce.getMessage());
           lastIndexExclusive++;
           continue;
         }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 761c98237e38..1f8326311098 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3464,7 +3464,12 @@ public <R> MultiResponse multi(MultiAction<R> multi) throws IOException {
               result = new Result();
             } else if (code.getOperationStatusCode()
                 == OperationStatusCode.SANITY_CHECK_FAILURE) {
+              // Don't send a FailedSanityCheckException as older clients will not know about
+              // that class being a subclass of DoNotRetryIOException
+              // and will retry mutations that will never succeed.
               result = new DoNotRetryIOException(code.getExceptionMsg());
+            } else if (code.getOperationStatusCode() == OperationStatusCode.BAD_FAMILY) {
+              result = new NoSuchColumnFamilyException(code.getExceptionMsg());
             }
             // FAILURE && NOT_RUN becomes null, aka: need to run again.
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 6ce3fa3a61ce..d3cab721800f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -85,6 +85,7 @@
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
@@ -3614,6 +3615,25 @@ public void testPut() throws IOException {
     }
   }
 
+  @Test
+  public void testPutNoCF() throws IOException {
+    final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
+    final byte[] VAL = Bytes.toBytes(100);
+    HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), new byte[][]{FAMILY});
+
+    boolean caughtNSCFE = false;
+
+    try {
+      Put p = new Put(ROW);
+      p.add(BAD_FAM, QUALIFIER, VAL);
+      table.put(p);
+    } catch (RetriesExhaustedWithDetailsException e) {
+      caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
+    }
+    assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
+
+  }
+
   @Test
   public void testRowsPut() throws IOException {
     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
@@ -4165,6 +4185,8 @@ public void testIncrementingInvalidValue() throws Exception {
     }
   }
 
+
+
   @Test
   public void testIncrement() throws Exception {
     LOG.info("Starting testIncrement");
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 081d8680ce30..21f3ded987de 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -558,7 +558,7 @@ public void testFamilyWithAndWithoutColon() throws Exception {
       boolean exception = false;
       try {
         this.region.put(p);
-      } catch (DoNotRetryIOException e) {
+      } catch (NoSuchColumnFamilyException e) {
         exception = true;
       }
       assertTrue(exception);
@@ -599,7 +599,7 @@ public void testBatchPut() throws Exception {
       codes = this.region.put(puts);
       assertEquals(10, codes.length);
       for (int i = 0; i < 10; i++) {
-        assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
+        assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
           OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
       }
       assertEquals(1, HLog.getSyncTime().count);
@@ -637,7 +637,7 @@ public void doWork() throws IOException {
       assertEquals(1, HLog.getSyncTime().count);
       codes = retFromThread.get();
       for (int i = 0; i < 10; i++) {
-        assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
+        assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
           OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
       }
   
@@ -654,7 +654,7 @@ public void doWork() throws IOException {
       codes = region.put(putsAndLocks.toArray(new Pair[0]));
       LOG.info("...performed put");
       for (int i = 0; i < 10; i++) {
-        assertEquals((i == 5) ? OperationStatusCode.SANITY_CHECK_FAILURE :
+        assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
           OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
       }
       // Make sure we didn't do an extra batch
@@ -670,6 +670,45 @@ public void doWork() throws IOException {
     }
   }
 
+  public void testBatchPutWithTsSlop() throws Exception {
+    byte[] b = Bytes.toBytes(getName());
+    byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
+    byte[] qual = Bytes.toBytes("qual");
+    byte[] val = Bytes.toBytes("val");
+
+    HBaseConfiguration conf = new HBaseConfiguration();
+
+
+    // add data with a timestamp that is too recent for range. Ensure assert
+    conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
+    this.region = initHRegion(b, getName(), conf, cf);
+
+    try{
+      HLog.getSyncTime(); // clear counter from prior tests
+      assertEquals(0, HLog.getSyncTime().count);
+
+      final Put[] puts = new Put[10];
+      for (int i = 0; i < 10; i++) {
+        puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
+        puts[i].add(cf, qual, val);
+      }
+
+      OperationStatus[] codes = this.region.put(puts);
+      assertEquals(10, codes.length);
+      for (int i = 0; i < 10; i++) {
+        assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i]
+            .getOperationStatusCode());
+      }
+      assertEquals(0, HLog.getSyncTime().count);
+
+
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   // checkAndMutate tests
   //////////////////////////////////////////////////////////////////////////////
@@ -1202,6 +1241,7 @@ public void testPutWithLatestTS() throws IOException {
 
   }
 
+
   /**
    * Tests that there is server-side filtering for invalid timestamp upper
    * bound. Note that the timestamp lower bound is automatically handled for us
@@ -1217,6 +1257,7 @@ public void testPutWithTsSlop() throws IOException {
     // add data with a timestamp that is too recent for range. Ensure assert
     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
     this.region = initHRegion(tableName, method, conf, families);
+    boolean caughtExcep = false;
     try {
       try {
         // no TS specified == use latest. should not error
@@ -1229,7 +1270,9 @@ public void testPutWithTsSlop() throws IOException {
         fail("Expected IOE for TS out of configured timerange");
       } catch (DoNotRetryIOException ioe) {
         LOG.debug("Received expected exception", ioe);
+        caughtExcep = true;
       }
+      assertTrue("Should catch FailedSanityCheckException", caughtExcep);
     } finally {
       HRegion.closeHRegion(this.region);
       this.region = null;

From d00d8060203963626ad48d3dafabe22f64fb7faf Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 13 Sep 2012 16:40:36 +0000
Subject: [PATCH 0473/1540] CHANGES.txt and pom.xml in preparation for 0.94.2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1384401 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |   2 +-
 2 files changed, 106 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 9f4e92a820db..a47382fef20f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,110 @@
 HBase Change Log
 
+Release 0.94.2 - 9/13/2012
+Sub-task
+
+    [HBASE-6257] - Avoid unnecessary flush & compact on Meta in admin.rb.
+    [HBASE-6496] - Example ZK based scan policy
+
+Bug
+
+    [HBASE-5292] - getsize per-CF metric incorrectly counts compaction related reads as well
+    [HBASE-5549] - Master can fail if ZooKeeper session expires
+    [HBASE-5997] - Fix concerns raised in HBASE-5922 related to HalfStoreFileReader
+    [HBASE-6165] - Replication can overrun .META. scans on cluster re-start
+    [HBASE-6211] - Put latencies in jmx
+    [HBASE-6263] - Use default mode for HBase Thrift gateway if not specified
+    [HBASE-6268] - Can't enable a table on a 0.94 cluster from a 0.92 client
+    [HBASE-6321] - ReplicationSource dies reading the peer's id
+    [HBASE-6340] - HBase RPC should allow protocol extension with common interfaces.
+    [HBASE-6359] - KeyValue may return incorrect values after readFields()
+    [HBASE-6364] - Powering down the server host holding the .META. table causes HBase Client to take excessively long to recover and connect to reassigned .META. table
+    [HBASE-6378] - the javadoc of setEnabledTable maybe not describe accurately
+    [HBASE-6432] - HRegionServer doesn't properly set clusterId in conf
+    [HBASE-6437] - Avoid admin.balance during master initialize
+    [HBASE-6447] - Common TestZooKeeper failures on jenkins: testMasterSessionExpired and testCreateSilentIsReallySilent
+    [HBASE-6450] - HBase startup should be with MALLOC_MAX_ARENA set
+    [HBASE-6460] - hbck "-repairHoles" usage inconsistent with "-fixHdfsOrphans"
+    [HBASE-6471] - Performance regression caused by HBASE-4054
+    [HBASE-6478] - TestClassLoading.testClassLoadingFromLibDirInJar occasionally fails
+    [HBASE-6503] - HBase Shell Documentation For DROP Is Outdated
+    [HBASE-6512] - Incorrect OfflineMetaRepair log class name
+    [HBASE-6514] - unknown metrics type: org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram
+    [HBASE-6516] - hbck cannot detect any IOException while ".tableinfo" file is missing
+    [HBASE-6520] - MSLab May cause the Bytes.toLong not work correctly for increment
+    [HBASE-6525] - bin/replication/copy_tables_desc.rb references non-existent class
+    [HBASE-6529] - With HFile v2, the region server will always perform an extra copy of source files
+    [HBASE-6537] - Race between balancer and disable table can lead to inconsistent cluster
+    [HBASE-6552] - TestAcidGuarantees system test should flush more aggressively
+    [HBASE-6561] - Gets/Puts with many columns send the RegionServer into an "endless" loop
+    [HBASE-6562] - Fake KVs are sometimes passed to filters
+    [HBASE-6565] - Coprocessor exec result Map is not thread safe
+    [HBASE-6576] - HBaseAdmin.createTable should wait until the table is enabled
+    [HBASE-6579] - Unnecessary KV order check in StoreScanner
+    [HBASE-6587] - Region would be assigned twice in the case of all RS offline
+    [HBASE-6596] - Revert HBASE-5022; it undoes HBC.create
+    [HBASE-6602] - Region Server Dynamic Metrics can cause high cpu usage.
+    [HBASE-6603] - RegionMetricsStorage.incrNumericMetric is called too often
+    [HBASE-6608] - Fix for HBASE-6160, META entries from daughters can be deleted before parent entries, shouldn't compare HRegionInfo's
+    [HBASE-6615] - hbase.rs.evictblocksonclose seems to be ineffective
+    [HBASE-6616] - test failure in TestDelayedRpc#testTooManyDelayedRpcs
+    [HBASE-6621] - Reduce calls to Bytes.toInt
+    [HBASE-6623] - [replication] replication metrics value AgeOfLastShippedOp is not set correctly
+    [HBASE-6631] - TestHMasterRPCException in 0.92 failed twice on socket timeout
+    [HBASE-6632] - [0.92 UNIT TESTS] testCreateTableRPCTimeOut sets rpc timeout to 1500ms and leaves it (testHundredsOfTable fails w/ 1500ms timeout)
+    [HBASE-6638] - Move DaemonThreadFactory into Threads (0.94)
+    [HBASE-6641] - more message with DoNotRetryIOException in client
+    [HBASE-6647] - [performance regression] appendNoSync/HBASE-4528 doesn't take deferred log flush into account
+    [HBASE-6648] - [0.92 UNIT TESTS] TestMasterObserver.testRegionTransitionOperations fails occasionally
+    [HBASE-6649] - [0.92 UNIT TESTS] TestReplication.queueFailover occasionally fails [Part-1]
+    [HBASE-6662] - Region server incorrectly reports its own address as master's address
+    [HBASE-6663] - NPE race in HConnection if zookeeper is reset
+    [HBASE-6671] - Kerberos authenticated super user should be able to retrieve proxied delegation tokens
+    [HBASE-6685] - Thrift DemoClient.pl got NullPointerException
+    [HBASE-6686] - HFile Quarantine fails with missing dirs in hadoop 2.0
+    [HBASE-6710] - 0.92/0.94 compatibility issues due to HBASE-5206
+    [HBASE-6711] - Avoid local results copy in StoreScanner
+    [HBASE-6713] - Stopping META/ROOT RS may take 50mins when some region is splitting
+    [HBASE-6734] - Code duplication in LoadIncrementalHFiles
+    [HBASE-6757] - Very inefficient behaviour of scan using FilterList
+    [HBASE-6762] - HBASE-6340 broke SecureRPCEngine
+    [HBASE-6769] - HRS.multi eats NoSuchColumnFamilyException since HBASE-5021
+
+Improvement
+
+    [HBASE-3271] - Allow .META. table to be exported
+    [HBASE-5631] - hbck should handle case where .tableinfo file is missing.
+    [HBASE-5714] - Add write permissions check before any hbck run that modifies hdfs.
+    [HBASE-5728] - Methods Missing in HTableInterface
+    [HBASE-6286] - Upgrade maven-compiler-plugin to 2.5.1
+    [HBASE-6291] - Don't retry increments on an invalid cell
+    [HBASE-6308] - Coprocessors should be loaded in a custom ClassLoader to prevent dependency conflicts with HBase
+    [HBASE-6373] - Add more context information to audit log messages
+    [HBASE-6444] - Expose the ability to set custom HTTP Request Headers for the REST client used by RemoteHTable
+    [HBASE-6458] - new comparator twice in checkAndPut, just reuse the first one
+    [HBASE-6522] - Expose locks and leases to Coprocessors
+    [HBASE-6586] - Quarantine Corrupted HFiles with hbck
+    [HBASE-6643] - Accept encoded region name in compacting/spliting region from shell
+    [HBASE-6644] - HBaseAdmin.createTable should wait more till table is enabled.
+
+New Feature
+
+    [HBASE-6427] - Pluggable compaction and scan policies via coprocessors
+    [HBASE-6505] - Allow shared RegionObserver state
+    [HBASE-6550] - Refactoring ReplicationSink to make it more responsive of cluster health
+
+Task
+
+    [HBASE-5042] - TestReadWriteConsistencyControl should be renamed
+    [HBASE-6288] - In hbase-daemons.sh, description of the default backup-master file path is wrong
+    [HBASE-6538] - Remove copy_table.rb script
+
+Test
+
+    [HBASE-6507] - [hbck] TestHBaseFsck ran into TableNotEnabledException
+    [HBASE-6593] - TestAdmin times out sometimes
+
+
 Release 0.94.1 - 7/24/2012
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 8d72ae44800f..66b3aaa0510e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.2-SNAPSHOT</version>
+  <version>0.94.2</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From d8c4960bcc6d9028d9012aa606fcd8c2ea31a4ba Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Fri, 14 Sep 2012 22:07:45 +0000
Subject: [PATCH 0474/1540] HBASE-6792 Remove interface audience annotations in
 0.94/0.92 introduced by HBASE-6516

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1384947 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/TableInfoMissingException.java     | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java b/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java
index bc8bc7fc1948..2faf198e4c2e 100644
--- a/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java
+++ b/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java
@@ -17,14 +17,11 @@
  */
 package org.apache.hadoop.hbase;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-
 /**
  * 
  * Failed to find .tableinfo file under table dir
  *
  */
-@InterfaceAudience.Private
 @SuppressWarnings("serial")
 public class TableInfoMissingException extends HBaseIOException {
 

From d453c33581611eaf33caa01fcaef3d05c23e931d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 15 Sep 2012 07:33:26 +0000
Subject: [PATCH 0475/1540] HBASE-6504 Adding GC details prevents HBase from
 starting in non-distributed mode

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1385027 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/rolling-restart.sh | 3 ++-
 bin/start-hbase.sh     | 3 ++-
 bin/stop-hbase.sh      | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index ec4edb5b1fa2..72111eddb88b 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -76,7 +76,8 @@ for x in "$@" ; do
 done
 
 # quick function to get a value from the HBase config file
-distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
+# HBASE-6504 - only take the first line of the output in case verbose gc is on
+distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1`
 if [ "$distMode" == 'false' ]; then
   if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then
     echo Cant do selective rolling restart if not running distributed
diff --git a/bin/start-hbase.sh b/bin/start-hbase.sh
index 6240ee649abf..6f3cd9085d94 100755
--- a/bin/start-hbase.sh
+++ b/bin/start-hbase.sh
@@ -38,7 +38,8 @@ then
   exit $errCode
 fi
 
-distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
+# HBASE-6504 - only take the first line of the output in case verbose gc is on
+distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1`
 
 
 if [ "$distMode" == 'false' ] 
diff --git a/bin/stop-hbase.sh b/bin/stop-hbase.sh
index b3828345d51d..e9749fee2889 100755
--- a/bin/stop-hbase.sh
+++ b/bin/stop-hbase.sh
@@ -60,7 +60,8 @@ done
 echo
 
 # distributed == false means that the HMaster will kill ZK when it exits
-distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
+# HBASE-6504 - only take the first line of the output in case verbose gc is on
+distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1`
 if [ "$distMode" == 'true' ] 
 then
   # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message

From 79f78b760491724058977d6367a2f121dbcaaf6f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 16 Sep 2012 02:52:42 +0000
Subject: [PATCH 0476/1540] HBASE-6438 RegionAlreadyInTransitionException needs
 to give more info to avoid assignment inconsistencies (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1385209 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 53 +++++++++++++------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 61fa7ff0c3f0..7e732e5d8274 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1585,13 +1585,14 @@ private RegionState forceRegionStateToOffline(final HRegionInfo region,
   private void assign(final HRegionInfo region, final RegionState state,
       final boolean setOfflineInZK, final boolean forceNewPlan,
       boolean hijack) {
+    boolean regionAlreadyInTransitionException = false;
     for (int i = 0; i < this.maximumAssignmentAttempts; i++) {
       int versionOfOfflineNode = -1;
       if (setOfflineInZK) {
         // get the version of the znode after setting it to OFFLINE.
         // versionOfOfflineNode will be -1 if the znode was not set to OFFLINE
-        versionOfOfflineNode = setOfflineInZooKeeper(state,
-            hijack);
+        versionOfOfflineNode = setOfflineInZooKeeper(state, hijack,
+            regionAlreadyInTransitionException);
         if(versionOfOfflineNode != -1){
           if (isDisabledorDisablingRegionInRIT(region)) {
             return;
@@ -1666,22 +1667,34 @@ private void assign(final HRegionInfo region, final RegionState state,
         if (t instanceof RemoteException) {
           t = ((RemoteException) t).unwrapRemoteException();
           if (t instanceof RegionAlreadyInTransitionException) {
-            String errorMsg = "Failed assignment in: " + plan.getDestination()
-                + " due to " + t.getMessage();
-            LOG.error(errorMsg, t);
-            return;
+            regionAlreadyInTransitionException = true;
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Failed assignment in: " + plan.getDestination() + " due to "
+                  + t.getMessage());
+            }
           }
         }
-        LOG.warn("Failed assignment of " +
-          state.getRegion().getRegionNameAsString() + " to " +
-          plan.getDestination() + ", trying to assign elsewhere instead; " +
-          "retry=" + i, t);
+        LOG.warn("Failed assignment of "
+            + state.getRegion().getRegionNameAsString()
+            + " to "
+            + plan.getDestination()
+            + ", trying to assign "
+            + (regionAlreadyInTransitionException ? "to the same region server"
+                + " because of RegionAlreadyInTransitionException;" : "elsewhere instead; ")
+            + "retry=" + i, t);
         // Clean out plan we failed execute and one that doesn't look like it'll
         // succeed anyways; we need a new plan!
         // Transition back to OFFLINE
         state.update(RegionState.State.OFFLINE);
-        // Force a new plan and reassign.  Will return null if no servers.
-        if (getRegionPlan(state, plan.getDestination(), true) == null) {
+        // If region opened on destination of present plan, reassigning to new
+        // RS may cause double assignments. In case of RegionAlreadyInTransitionException
+        // reassigning to same RS.
+        RegionPlan newPlan = plan;
+        if (!regionAlreadyInTransitionException) {
+          // Force a new plan and reassign. Will return null if no servers.
+          newPlan = getRegionPlan(state, plan.getDestination(), true);
+        }
+        if (newPlan == null) {
           this.timeoutMonitor.setAllRegionServersOffline(true);
           LOG.warn("Unable to find a viable location to assign region " +
             state.getRegion().getRegionNameAsString());
@@ -1709,17 +1722,23 @@ private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) {
    * @param state
    * @param hijack
    *          - true if needs to be hijacked and reassigned, false otherwise.
+   * @param regionAlreadyInTransitionException  
+   *          - true if we need to retry assignment because of RegionAlreadyInTransitionException.       
    * @return the version of the offline node if setting of the OFFLINE node was
    *         successful, -1 otherwise.
    */
-  int setOfflineInZooKeeper(final RegionState state,
-      boolean hijack) {
+  int setOfflineInZooKeeper(final RegionState state, boolean hijack,
+      boolean regionAlreadyInTransitionException) {
     // In case of reassignment the current state in memory need not be
     // OFFLINE. 
     if (!hijack && !state.isClosed() && !state.isOffline()) {
-      String msg = "Unexpected state : " + state + " .. Cannot transit it to OFFLINE.";
-      this.master.abort(msg, new IllegalStateException(msg));
-      return -1;
+      if (!regionAlreadyInTransitionException ) {
+        String msg = "Unexpected state : " + state + " .. Cannot transit it to OFFLINE.";
+        this.master.abort(msg, new IllegalStateException(msg));
+        return -1;
+      } 
+      LOG.debug("Unexpected state : " + state
+          + " but retrying to assign because RegionAlreadyInTransitionException.");
     }
     boolean allowZNodeCreation = false;
     // Under reassignment if the current state is PENDING_OPEN

From 1e4eb797abd9593c25d0047aa9a3be64d9765f99 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 18 Sep 2012 14:59:36 +0000
Subject: [PATCH 0477/1540] HBASE-6438 Addendum checks
 regionAlreadyInTransitionException when generating region plan (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1387209 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/AssignmentManager.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 7e732e5d8274..c0e4e3519bb9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1619,7 +1619,7 @@ private void assign(final HRegionInfo region, final RegionState state,
         LOG.debug("Server stopped; skipping assign of " + state);
         return;
       }
-      RegionPlan plan = getRegionPlan(state, forceNewPlan);
+      RegionPlan plan = getRegionPlan(state, !regionAlreadyInTransitionException && forceNewPlan);
       if (plan == null) {
         LOG.debug("Unable to determine a plan to assign " + state);
         this.timeoutMonitor.setAllRegionServersOffline(true);

From 7b5d05963e7c7c0a6647a58b2f64bcf7c6ff2228 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 18 Sep 2012 17:04:49 +0000
Subject: [PATCH 0478/1540] HBASE-6803 script hbase should add
 JAVA_LIBRARY_PATH to LD_LIBRARY_PATH

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1387260 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/hbase b/bin/hbase
index 3661ba8f8312..62702057be9e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -323,6 +323,7 @@ HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}"
 if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH"
 fi
 
 # Enable security logging on the master and regionserver only

From 9b0e243f8d5bde1f73a713ee49f7ce27e1c1a560 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 20 Sep 2012 05:01:52 +0000
Subject: [PATCH 0479/1540] HBASE-6844 upgrade 0.23 version dependency in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1387856 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 66b3aaa0510e..97afd3e86e20 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1845,7 +1845,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>0.23.3-SNAPSHOT</hadoop.version>
+        <hadoop.version>0.23.3</hadoop.version>
         <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencies>

From a352090eea643ae17ccd27857aca74f9cef1c2ca Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 20 Sep 2012 05:41:16 +0000
Subject: [PATCH 0480/1540] HBASE-6842 the jar used in coprocessor is not
 deleted in local which will exhaust the space of /tmp

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1387862 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 4fa1a14fd751..2fdaf6fd1443 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -183,7 +183,8 @@ public E load(Path path, String className, int priority,
           java.io.File.separator +"." + pathPrefix +
           "." + className + "." + System.currentTimeMillis() + ".jar");
       fs.copyToLocalFile(path, dst);
-      fs.deleteOnExit(dst);
+      File tmpLocal = new File(dst.toString());
+      tmpLocal.deleteOnExit();
 
       // TODO: code weaving goes here
 

From 07486bb50ed89ed8a967816d69963dc13a02edc7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 20 Sep 2012 05:47:16 +0000
Subject: [PATCH 0481/1540] HBASE-6839 Operations may be executed without
 rowLock

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1387864 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegion.java    | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index d27bff54458a..74e02d619ff9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2084,7 +2084,14 @@ private long doMiniBatchMutation(
         // If we haven't got any rows in our batch, we should block to
         // get the next one.
         boolean shouldBlock = numReadyToWrite == 0;
-        Integer acquiredLockId = getLock(providedLockId, mutation.getRow(), shouldBlock);
+        Integer acquiredLockId = null;
+        try {
+          acquiredLockId = getLock(providedLockId, mutation.getRow(),
+              shouldBlock);
+        } catch (IOException ioe) {
+          LOG.warn("Failed getting lock in batch put, row="
+                  + Bytes.toStringBinary(mutation.getRow()), ioe);
+        }
         if (acquiredLockId == null) {
           // We failed to grab another lock
           assert !shouldBlock : "Should never fail to get lock when blocking";
@@ -3103,7 +3110,8 @@ private Integer internalObtainRowLock(final byte[] row, boolean waitForLock)
           try {
             if (!existingLatch.await(this.rowLockWaitDuration,
                             TimeUnit.MILLISECONDS)) {
-                return null;
+              throw new IOException("Timed out on getting lock for row="
+                  + Bytes.toStringBinary(row));
             }
           } catch (InterruptedException ie) {
             // Empty

From 536e1c9d27bf4346220382ed6e7a9f40975d3c56 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 20 Sep 2012 18:59:05 +0000
Subject: [PATCH 0482/1540] HBASE-6847  HBASE-6649 broke replication (Devaraj
 Das via JD)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388160 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSource.java       | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index b3451235bdbb..7710b47c71a3 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -338,10 +338,6 @@ public void run() {
         }
       } finally {
         try {
-          // if current path is null, it means we processEndOfFile hence
-          if (this.currentPath != null && !gotIOE) {
-            this.position = this.reader.getPosition();
-          }
           if (this.reader != null) {
             this.reader.close();
           }
@@ -391,7 +387,8 @@ protected boolean readAllEntriesToReplicateOrNextFile() throws IOException{
     if (this.position != 0) {
       this.reader.seek(this.position);
     }
-    HLog.Entry entry = this.reader.next(this.entriesArray[currentNbEntries]);
+    long startPosition = this.position;
+    HLog.Entry entry = readNextAndSetPosition();
     while (entry != null) {
       WALEdit edit = entry.getEdit();
       this.metrics.logEditsReadRate.inc(1);
@@ -420,13 +417,13 @@ protected boolean readAllEntriesToReplicateOrNextFile() throws IOException{
         }
       }
       // Stop if too many entries or too big
-      if ((this.reader.getPosition() - this.position)
+      if ((this.reader.getPosition() - startPosition)
           >= this.replicationQueueSizeCapacity ||
           currentNbEntries >= this.replicationQueueNbCapacity) {
         break;
       }
       try {
-        entry = this.reader.next(entriesArray[currentNbEntries]);
+        entry = readNextAndSetPosition();
       } catch (IOException ie) {
         LOG.debug("Break on IOE: " + ie.getMessage());
         break;
@@ -434,12 +431,22 @@ protected boolean readAllEntriesToReplicateOrNextFile() throws IOException{
     }
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +
-        " and size: " + (this.reader.getPosition() - this.position));
+        " and size: " + (this.reader.getPosition() - startPosition));
     // If we didn't get anything and the queue has an object, it means we
     // hit the end of the file for sure
     return seenEntries == 0 && processEndOfFile();
   }
 
+  private HLog.Entry readNextAndSetPosition() throws IOException {
+    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
+    // Store the position so that in the future the reader can start
+    // reading from here. If the above call to next() throws an
+    // exception, the position won't be changed and retry will happen
+    // from the last known good position
+    this.position = this.reader.getPosition();
+    return entry;
+  } 
+
   private void connectToPeers() {
     // Connect to peer cluster first, unless we have to stop
     while (this.isActive() && this.currentPeers.size() == 0) {

From 1e20730cac8b0ed1232909b8cfaf3189850f7a84 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 21 Sep 2012 17:52:40 +0000
Subject: [PATCH 0483/1540] HBASE-6488 HBase wont run on IPv6 on OSes that use
 zone-indexes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388587 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/executor/ExecutorService.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
index 7fb4266ac6e1..012dc0cbc95e 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
@@ -104,7 +104,7 @@ public enum ExecutorType {
      * @return Conflation of the executor type and the passed servername.
      */
     String getExecutorName(String serverName) {
-      return this.toString() + "-" + serverName;
+      return this.toString() + "-" + serverName.replace("%", "%%");
     }
   }
 

From 45d038dfa369fd30ec5a130a7e98138b9bf1f95f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 21 Sep 2012 18:56:33 +0000
Subject: [PATCH 0484/1540] CHANGES.txt for 0.94.2RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388618 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index a47382fef20f..72e140049731 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,10 +1,11 @@
 HBase Change Log
 
-Release 0.94.2 - 9/13/2012
+Release 0.94.2 - 9/21/2012
 Sub-task
 
     [HBASE-6257] - Avoid unnecessary flush & compact on Meta in admin.rb.
     [HBASE-6496] - Example ZK based scan policy
+    [HBASE-6792] - Remove interface audience annotations in 0.94/0.92 introduced by HBASE-6516
 
 Bug
 
@@ -22,12 +23,15 @@ Bug
     [HBASE-6378] - the javadoc of setEnabledTable maybe not describe accurately
     [HBASE-6432] - HRegionServer doesn't properly set clusterId in conf
     [HBASE-6437] - Avoid admin.balance during master initialize
+    [HBASE-6438] - RegionAlreadyInTransitionException needs to give more info to avoid assignment inconsistencies
     [HBASE-6447] - Common TestZooKeeper failures on jenkins: testMasterSessionExpired and testCreateSilentIsReallySilent
     [HBASE-6450] - HBase startup should be with MALLOC_MAX_ARENA set
     [HBASE-6460] - hbck "-repairHoles" usage inconsistent with "-fixHdfsOrphans"
     [HBASE-6471] - Performance regression caused by HBASE-4054
     [HBASE-6478] - TestClassLoading.testClassLoadingFromLibDirInJar occasionally fails
+    [HBASE-6488] - HBase wont run on IPv6 on OSes that use zone-indexes
     [HBASE-6503] - HBase Shell Documentation For DROP Is Outdated
+    [HBASE-6504] - Adding GC details prevents HBase from starting in non-distributed mode
     [HBASE-6512] - Incorrect OfflineMetaRepair log class name
     [HBASE-6514] - unknown metrics type: org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram
     [HBASE-6516] - hbck cannot detect any IOException while ".tableinfo" file is missing
@@ -69,6 +73,11 @@ Bug
     [HBASE-6757] - Very inefficient behaviour of scan using FilterList
     [HBASE-6762] - HBASE-6340 broke SecureRPCEngine
     [HBASE-6769] - HRS.multi eats NoSuchColumnFamilyException since HBASE-5021
+    [HBASE-6803] - script hbase should add JAVA_LIBRARY_PATH to LD_LIBRARY_PATH
+    [HBASE-6839] - Operations may be executed without holding rowLock
+    [HBASE-6842] - the jar used in coprocessor is not deleted in local which will exhaust the space of /tmp
+    [HBASE-6844] - upgrade 0.23 version dependency in 0.94
+    [HBASE-6847] - HBASE-6649 broke replication
 
 Improvement
 

From d42f76ea757882e2c09cd1d72ac5da1e9bed1131 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 21 Sep 2012 19:01:30 +0000
Subject: [PATCH 0485/1540] HBASE-6714
 TestMultiSlaveReplication#testMultiSlaveReplication may fail

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388621 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestMultiSlaveReplication.java            | 29 +++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
index 147a3d25cd3f..f7fea5f2d963 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
@@ -173,7 +173,10 @@ public void testMultiSlaveReplication() throws Exception {
     deleteAndWait(row2, htable1, htable2, htable3);
     // Even if the log was rolled in the middle of the replication
     // "row" is still replication.
-    checkRow(row, 1, htable2, htable3);
+    checkRow(row, 1, htable2);
+    // Replication thread of cluster 2 may be sleeping, and since row2 is not there in it, 
+    // we should wait before checking.
+    checkWithWait(row, 1, htable3);
 
     // cleanup the rest
     deleteAndWait(row, htable1, htable2, htable3);
@@ -183,7 +186,29 @@ public void testMultiSlaveReplication() throws Exception {
     utility2.shutdownMiniCluster();
     utility1.shutdownMiniCluster();
   }
-
+ 
+  private void checkWithWait(byte[] row, int count, HTable table) throws Exception {
+    Get get = new Get(row);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      if (i == NB_RETRIES - 1) {
+        fail("Waited too much time while getting the row.");
+      }
+      boolean rowReplicated = false;
+      Result res = table.get(get);
+      if (res.size() >= 1) {
+        LOG.info("Row is replicated");
+        rowReplicated = true;
+        assertEquals(count, res.size());
+        break;
+      }
+      if (rowReplicated) {
+        break;
+      } else {
+        Thread.sleep(SLEEP_TIME);
+      }
+    }
+  }
+  
   private void checkRow(byte[] row, int count, HTable... tables) throws IOException {
     Get get = new Get(row);
     for (HTable table : tables) {

From 896344b0d48803073b6835adca6459917499740e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 21 Sep 2012 21:31:02 +0000
Subject: [PATCH 0486/1540] CHANGES.txt for 0.94.2RC1, accomodate new commits

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388691 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 72e140049731..f2be55bc8b43 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -69,6 +69,7 @@ Bug
     [HBASE-6710] - 0.92/0.94 compatibility issues due to HBASE-5206
     [HBASE-6711] - Avoid local results copy in StoreScanner
     [HBASE-6713] - Stopping META/ROOT RS may take 50mins when some region is splitting
+    [HBASE-6714] - TestMultiSlaveReplication#testMultiSlaveReplication may fail
     [HBASE-6734] - Code duplication in LoadIncrementalHFiles
     [HBASE-6757] - Very inefficient behaviour of scan using FilterList
     [HBASE-6762] - HBASE-6340 broke SecureRPCEngine

From 35aba50be85e3a77ebd248f429d2077af9867fea Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Fri, 21 Sep 2012 21:42:37 +0000
Subject: [PATCH 0487/1540] HBASE-6860  [replication] HBASE-6550 is too
 aggressive, DDOSes .META.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388695 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSink.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index c8bca24315c5..41b19ff7fa5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -98,9 +98,9 @@ public ReplicationSink(Configuration conf, Stoppable stopper)
    */
   private void decorateConf() {
     this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
-        this.conf.getInt("replication.sink.client.retries.number", 1));
+        this.conf.getInt("replication.sink.client.retries.number", 4));
     this.conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
-        this.conf.getInt("replication.sink.client.ops.timeout", 20));
+        this.conf.getInt("replication.sink.client.ops.timeout", 10000));
   }
 
   /**

From 6bee1177a20368384370f81b4e909f645a5712a3 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 21 Sep 2012 21:52:42 +0000
Subject: [PATCH 0488/1540] HBASE-6858 Fix the incorrect BADVERSION checking in
 the recoverable zookeeper

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388700 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/zookeeper/RecoverableZooKeeper.java     | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 52013e93f15b..c59b59765f04 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -353,12 +353,9 @@ public Stat setData(String path, byte[] data, int version)
             try{
               Stat stat = new Stat();
               byte[] revData = zk.getData(path, false, stat);
-              int idLength = Bytes.toInt(revData, ID_LENGTH_SIZE);
-              int dataLength = revData.length-ID_LENGTH_SIZE-idLength;
-              int dataOffset = ID_LENGTH_SIZE+idLength;
-              
-              if(Bytes.compareTo(revData, ID_LENGTH_SIZE, id.length, 
-                  revData, dataOffset, dataLength) == 0) {
+              int idLength = Bytes.toInt(revData, MAGIC_OFFSET);
+              if (Bytes.compareTo(revData, MAGIC_OFFSET + ID_OFFSET, idLength,
+                  id, 0, id.length) == 0) {
                 // the bad version is caused by previous successful setData
                 return stat;
               }

From a442f99aa9ac748353901c82092bca0e34449822 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 21 Sep 2012 21:57:26 +0000
Subject: [PATCH 0489/1540] CHANGES.txt for 0.94.2RC1, accomodate yet more
 commits

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388701 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index f2be55bc8b43..5e3f8e68335d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -79,6 +79,7 @@ Bug
     [HBASE-6842] - the jar used in coprocessor is not deleted in local which will exhaust the space of /tmp
     [HBASE-6844] - upgrade 0.23 version dependency in 0.94
     [HBASE-6847] - HBASE-6649 broke replication
+    [HBASE-6858] - Fix the incorrect BADVERSION checking in the recoverable zookeeper
 
 Improvement
 
@@ -96,6 +97,7 @@ Improvement
     [HBASE-6586] - Quarantine Corrupted HFiles with hbck
     [HBASE-6643] - Accept encoded region name in compacting/spliting region from shell
     [HBASE-6644] - HBaseAdmin.createTable should wait more till table is enabled.
+    [HBASE-6860] - [replication] HBASE-6550 is too aggressive, DDOSes .META.
 
 New Feature
 

From 9526aeb2a8d51c0ac8a1f83326a04e30eed78dad Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 21 Sep 2012 22:08:11 +0000
Subject: [PATCH 0490/1540] HBASE-6858 Fix the incorrect BADVERSION checking in
 the recoverable zookeeper; REVERT.... BROKE

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388704 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/zookeeper/RecoverableZooKeeper.java     | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index c59b59765f04..52013e93f15b 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -353,9 +353,12 @@ public Stat setData(String path, byte[] data, int version)
             try{
               Stat stat = new Stat();
               byte[] revData = zk.getData(path, false, stat);
-              int idLength = Bytes.toInt(revData, MAGIC_OFFSET);
-              if (Bytes.compareTo(revData, MAGIC_OFFSET + ID_OFFSET, idLength,
-                  id, 0, id.length) == 0) {
+              int idLength = Bytes.toInt(revData, ID_LENGTH_SIZE);
+              int dataLength = revData.length-ID_LENGTH_SIZE-idLength;
+              int dataOffset = ID_LENGTH_SIZE+idLength;
+              
+              if(Bytes.compareTo(revData, ID_LENGTH_SIZE, id.length, 
+                  revData, dataOffset, dataLength) == 0) {
                 // the bad version is caused by previous successful setData
                 return stat;
               }

From 99776e693d1a386cfc8598a86b56944fa3fc9e97 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 21 Sep 2012 22:37:47 +0000
Subject: [PATCH 0491/1540] CHANGES.txt for 0.94.2RC1, last try hopefully

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388711 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 5e3f8e68335d..3a73801b95d6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -79,7 +79,6 @@ Bug
     [HBASE-6842] - the jar used in coprocessor is not deleted in local which will exhaust the space of /tmp
     [HBASE-6844] - upgrade 0.23 version dependency in 0.94
     [HBASE-6847] - HBASE-6649 broke replication
-    [HBASE-6858] - Fix the incorrect BADVERSION checking in the recoverable zookeeper
 
 Improvement
 

From e1d3e4a7a372dc26334efcab9764b04c2f344122 Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Sat, 22 Sep 2012 21:12:17 +0000
Subject: [PATCH 0492/1540] HBASE-6851  Fix race condition in
 TableAuthManager.updateGlobalCache()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1388898 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/TableAuthManager.java     | 265 ++++++++++--------
 .../security/access/TestTablePermissions.java |  20 ++
 2 files changed, 162 insertions(+), 123 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
index a40a57685094..9c09b95fecbc 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
@@ -40,20 +40,59 @@
  * Performs authorization checks for a given user's assigned permissions
  */
 public class TableAuthManager {
+  private static class PermissionCache<T extends Permission> {
+    /** Cache of user permissions */
+    private ListMultimap<String,T> userCache = ArrayListMultimap.create();
+    /** Cache of group permissions */
+    private ListMultimap<String,T> groupCache = ArrayListMultimap.create();
+
+    public List<T> getUser(String user) {
+      return userCache.get(user);
+    }
+
+    public void putUser(String user, T perm) {
+      userCache.put(user, perm);
+    }
+
+    public List<T> replaceUser(String user, Iterable<? extends T> perms) {
+      return userCache.replaceValues(user, perms);
+    }
+
+    public List<T> getGroup(String group) {
+      return groupCache.get(group);
+    }
+
+    public void putGroup(String group, T perm) {
+      groupCache.put(group, perm);
+    }
+
+    public List<T> replaceGroup(String group, Iterable<? extends T> perms) {
+      return groupCache.replaceValues(group, perms);
+    }
+
+    /**
+     * Returns a combined map of user and group permissions, with group names prefixed by
+     * {@link AccessControlLists#GROUP_PREFIX}.
+     */
+    public ListMultimap<String,T> getAllPermissions() {
+      ListMultimap<String,T> tmp = ArrayListMultimap.create();
+      tmp.putAll(userCache);
+      for (String group : groupCache.keySet()) {
+        tmp.putAll(AccessControlLists.GROUP_PREFIX + group, groupCache.get(group));
+      }
+      return tmp;
+    }
+  }
+
   private static Log LOG = LogFactory.getLog(TableAuthManager.class);
 
   private static TableAuthManager instance;
 
-  /** Cache of global user permissions */
-  private ListMultimap<String,Permission> USER_CACHE = ArrayListMultimap.create();
-  /** Cache of global group permissions */
-  private ListMultimap<String,Permission> GROUP_CACHE = ArrayListMultimap.create();
-
-  private ConcurrentSkipListMap<byte[], ListMultimap<String,TablePermission>> TABLE_USER_CACHE =
-      new ConcurrentSkipListMap<byte[], ListMultimap<String,TablePermission>>(Bytes.BYTES_COMPARATOR);
+  /** Cache of global permissions */
+  private volatile PermissionCache<Permission> globalCache;
 
-  private ConcurrentSkipListMap<byte[], ListMultimap<String,TablePermission>> TABLE_GROUP_CACHE =
-      new ConcurrentSkipListMap<byte[], ListMultimap<String,TablePermission>>(Bytes.BYTES_COMPARATOR);
+  private ConcurrentSkipListMap<byte[], PermissionCache<TablePermission>> tableCache =
+      new ConcurrentSkipListMap<byte[], PermissionCache<TablePermission>>(Bytes.BYTES_COMPARATOR);
 
   private Configuration conf;
   private ZKPermissionWatcher zkperms;
@@ -69,15 +108,20 @@ private TableAuthManager(ZooKeeperWatcher watcher, Configuration conf)
     }
 
     // initialize global permissions based on configuration
-    initGlobal(conf);
+    globalCache = initGlobal(conf);
   }
 
-  private void initGlobal(Configuration conf) throws IOException {
+  /**
+   * Returns a new {@code PermissionCache} initialized with permission assignments
+   * from the {@code hbase.superuser} configuration key.
+   */
+  private PermissionCache<Permission> initGlobal(Configuration conf) throws IOException {
     User user = User.getCurrent();
     if (user == null) {
       throw new IOException("Unable to obtain the current user, " +
           "authorization checks for internal operations will not work correctly!");
     }
+    PermissionCache<Permission> newCache = new PermissionCache<Permission>();
     String currentUser = user.getShortName();
 
     // the system user is always included
@@ -86,13 +130,14 @@ private void initGlobal(Configuration conf) throws IOException {
     if (superusers != null) {
       for (String name : superusers) {
         if (AccessControlLists.isGroupPrincipal(name)) {
-          GROUP_CACHE.put(AccessControlLists.getGroupName(name),
+          newCache.putGroup(AccessControlLists.getGroupName(name),
               new Permission(Permission.Action.values()));
         } else {
-          USER_CACHE.put(name, new Permission(Permission.Action.values()));
+          newCache.putUser(name, new Permission(Permission.Action.values()));
         }
       }
     }
+    return newCache;
   }
 
   public ZKPermissionWatcher getZKPermissionWatcher() {
@@ -121,21 +166,21 @@ public void refreshCacheFromWritable(byte[] table, byte[] data) throws IOExcepti
    * @param userPerms
    */
   private void updateGlobalCache(ListMultimap<String,TablePermission> userPerms) {
-    USER_CACHE.clear();
-    GROUP_CACHE.clear();
+    PermissionCache<Permission> newCache = null;
     try {
-      initGlobal(conf);
+      newCache = initGlobal(conf);
+      for (Map.Entry<String,TablePermission> entry : userPerms.entries()) {
+        if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
+          newCache.putGroup(AccessControlLists.getGroupName(entry.getKey()),
+              new Permission(entry.getValue().getActions()));
+        } else {
+          newCache.putUser(entry.getKey(), new Permission(entry.getValue().getActions()));
+        }
+      }
+      globalCache = newCache;
     } catch (IOException e) {
       // Never happens
-      LOG.error("Error occured while updating the user cache", e);
-    }
-    for (Map.Entry<String,TablePermission> entry : userPerms.entries()) {
-      if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
-        GROUP_CACHE.put(AccessControlLists.getGroupName(entry.getKey()),
-                        new Permission(entry.getValue().getActions()));
-      } else {
-        USER_CACHE.put(entry.getKey(), new Permission(entry.getValue().getActions()));
-      }
+      LOG.error("Error occured while updating the global cache", e);
     }
   }
 
@@ -148,39 +193,24 @@ private void updateGlobalCache(ListMultimap<String,TablePermission> userPerms) {
    * @param tablePerms
    */
   private void updateTableCache(byte[] table, ListMultimap<String,TablePermission> tablePerms) {
-    // split user from group assignments so we don't have to prepend the group
-    // prefix every time we query for groups
-    ListMultimap<String,TablePermission> userPerms = ArrayListMultimap.create();
-    ListMultimap<String,TablePermission> groupPerms = ArrayListMultimap.create();
+    PermissionCache<TablePermission> newTablePerms = new PermissionCache<TablePermission>();
 
     for (Map.Entry<String,TablePermission> entry : tablePerms.entries()) {
       if (AccessControlLists.isGroupPrincipal(entry.getKey())) {
-        groupPerms.put(AccessControlLists.getGroupName(entry.getKey()), entry.getValue());
+        newTablePerms.putGroup(AccessControlLists.getGroupName(entry.getKey()), entry.getValue());
       } else {
-        userPerms.put(entry.getKey(), entry.getValue());
+        newTablePerms.putUser(entry.getKey(), entry.getValue());
       }
     }
 
-    TABLE_GROUP_CACHE.put(table, groupPerms);
-    TABLE_USER_CACHE.put(table, userPerms);
-  }
-
-  private List<TablePermission> getUserPermissions(String username, byte[] table) {
-    ListMultimap<String, TablePermission> tablePerms = TABLE_USER_CACHE.get(table);
-    if (tablePerms != null) {
-      return tablePerms.get(username);
-    }
-
-    return null;
+    tableCache.put(table, newTablePerms);
   }
 
-  private List<TablePermission> getGroupPermissions(String groupName, byte[] table) {
-    ListMultimap<String, TablePermission> tablePerms = TABLE_GROUP_CACHE.get(table);
-    if (tablePerms != null) {
-      return tablePerms.get(groupName);
+  private PermissionCache<TablePermission> getTablePermissions(byte[] table) {
+    if (!tableCache.containsKey(table)) {
+      tableCache.putIfAbsent(table, new PermissionCache<TablePermission>());
     }
-
-    return null;
+    return tableCache.get(table);
   }
 
   /**
@@ -215,14 +245,14 @@ public boolean authorize(User user, Permission.Action action) {
       return false;
     }
 
-    if (authorize(USER_CACHE.get(user.getShortName()), action)) {
+    if (authorize(globalCache.getUser(user.getShortName()), action)) {
       return true;
     }
 
     String[] groups = user.getGroupNames();
     if (groups != null) {
       for (String group : groups) {
-        if (authorize(GROUP_CACHE.get(group), action)) {
+        if (authorize(globalCache.getGroup(group), action)) {
           return true;
         }
       }
@@ -251,18 +281,20 @@ private boolean authorize(List<TablePermission> perms, byte[] table, byte[] fami
 
   public boolean authorize(User user, byte[] table, KeyValue kv,
       TablePermission.Action action) {
-    List<TablePermission> userPerms = getUserPermissions(
-        user.getShortName(), table);
-    if (authorize(userPerms, table, kv, action)) {
-      return true;
-    }
+    PermissionCache<TablePermission> tablePerms = tableCache.get(table);
+    if (tablePerms != null) {
+      List<TablePermission> userPerms = tablePerms.getUser(user.getShortName());
+      if (authorize(userPerms, table, kv, action)) {
+        return true;
+      }
 
-    String[] groupNames = user.getGroupNames();
-    if (groupNames != null) {
-      for (String group : groupNames) {
-        List<TablePermission> groupPerms = getGroupPermissions(group, table);
-        if (authorize(groupPerms, table, kv, action)) {
-          return true;
+      String[] groupNames = user.getGroupNames();
+      if (groupNames != null) {
+        for (String group : groupNames) {
+          List<TablePermission> groupPerms = tablePerms.getGroup(group);
+          if (authorize(groupPerms, table, kv, action)) {
+            return true;
+          }
         }
       }
     }
@@ -291,7 +323,7 @@ private boolean authorize(List<TablePermission> perms, byte[] table, KeyValue kv
    * stored user permissions.
    */
   public boolean authorizeUser(String username, Permission.Action action) {
-    return authorize(USER_CACHE.get(username), action);
+    return authorize(globalCache.getUser(username), action);
   }
 
   /**
@@ -315,7 +347,7 @@ public boolean authorizeUser(String username, byte[] table, byte[] family,
     if (authorizeUser(username, action)) {
       return true;
     }
-    return authorize(getUserPermissions(username, table), table, family,
+    return authorize(getTablePermissions(table).getUser(username), table, family,
         qualifier, action);
   }
 
@@ -325,7 +357,7 @@ public boolean authorizeUser(String username, byte[] table, byte[] family,
    * permissions.
    */
   public boolean authorizeGroup(String groupName, Permission.Action action) {
-    return authorize(GROUP_CACHE.get(groupName), action);
+    return authorize(globalCache.getGroup(groupName), action);
   }
 
   /**
@@ -343,7 +375,7 @@ public boolean authorizeGroup(String groupName, byte[] table, byte[] family,
     if (authorizeGroup(groupName, action)) {
       return true;
     }
-    return authorize(getGroupPermissions(groupName, table), table, family, action);
+    return authorize(getTablePermissions(table).getGroup(groupName), table, family, action);
   }
 
   public boolean authorize(User user, byte[] table, byte[] family,
@@ -376,24 +408,26 @@ public boolean authorize(User user, byte[] table, byte[] family,
    */
   public boolean matchPermission(User user,
       byte[] table, byte[] family, TablePermission.Action action) {
-    List<TablePermission> userPerms = getUserPermissions(
-        user.getShortName(), table);
-    if (userPerms != null) {
-      for (TablePermission p : userPerms) {
-        if (p.matchesFamily(table, family, action)) {
-          return true;
+    PermissionCache<TablePermission> tablePerms = tableCache.get(table);
+    if (tablePerms != null) {
+      List<TablePermission> userPerms = tablePerms.getUser(user.getShortName());
+      if (userPerms != null) {
+        for (TablePermission p : userPerms) {
+          if (p.matchesFamily(table, family, action)) {
+            return true;
+          }
         }
       }
-    }
 
-    String[] groups = user.getGroupNames();
-    if (groups != null) {
-      for (String group : groups) {
-        List<TablePermission> groupPerms = getGroupPermissions(group, table);
-        if (groupPerms != null) {
-          for (TablePermission p : groupPerms) {
-            if (p.matchesFamily(table, family, action)) {
-              return true;
+      String[] groups = user.getGroupNames();
+      if (groups != null) {
+        for (String group : groups) {
+          List<TablePermission> groupPerms = tablePerms.getGroup(group);
+          if (groupPerms != null) {
+            for (TablePermission p : groupPerms) {
+              if (p.matchesFamily(table, family, action)) {
+                return true;
+              }
             }
           }
         }
@@ -406,24 +440,26 @@ public boolean matchPermission(User user,
   public boolean matchPermission(User user,
       byte[] table, byte[] family, byte[] qualifier,
       TablePermission.Action action) {
-    List<TablePermission> userPerms = getUserPermissions(
-        user.getShortName(), table);
-    if (userPerms != null) {
-      for (TablePermission p : userPerms) {
-        if (p.matchesFamilyQualifier(table, family, qualifier, action)) {
-          return true;
+    PermissionCache<TablePermission> tablePerms = tableCache.get(table);
+    if (tablePerms != null) {
+      List<TablePermission> userPerms = tablePerms.getUser(user.getShortName());
+      if (userPerms != null) {
+        for (TablePermission p : userPerms) {
+          if (p.matchesFamilyQualifier(table, family, qualifier, action)) {
+            return true;
+          }
         }
       }
-    }
 
-    String[] groups = user.getGroupNames();
-    if (groups != null) {
-      for (String group : groups) {
-        List<TablePermission> groupPerms = getGroupPermissions(group, table);
-        if (groupPerms != null) {
-          for (TablePermission p : groupPerms) {
-            if (p.matchesFamilyQualifier(table, family, qualifier, action)) {
-              return true;
+      String[] groups = user.getGroupNames();
+      if (groups != null) {
+        for (String group : groups) {
+          List<TablePermission> groupPerms = tablePerms.getGroup(group);
+          if (groupPerms != null) {
+            for (TablePermission p : groupPerms) {
+              if (p.matchesFamilyQualifier(table, family, qualifier, action)) {
+                return true;
+              }
             }
           }
         }
@@ -434,8 +470,7 @@ public boolean matchPermission(User user,
   }
 
   public void remove(byte[] table) {
-    TABLE_USER_CACHE.remove(table);
-    TABLE_GROUP_CACHE.remove(table);
+    tableCache.remove(table);
   }
 
   /**
@@ -447,13 +482,9 @@ public void remove(byte[] table) {
    */
   public void setUserPermissions(String username, byte[] table,
       List<TablePermission> perms) {
-    ListMultimap<String,TablePermission> tablePerms = TABLE_USER_CACHE.get(table);
-    if (tablePerms == null) {
-      tablePerms = ArrayListMultimap.create();
-      TABLE_USER_CACHE.put(table, tablePerms);
-    }
-    tablePerms.replaceValues(username, perms);
-    writeToZooKeeper(table, tablePerms, TABLE_GROUP_CACHE.get(table));
+    PermissionCache<TablePermission> tablePerms = getTablePermissions(table);
+    tablePerms.replaceUser(username, perms);
+    writeToZooKeeper(table, tablePerms);
   }
 
   /**
@@ -465,29 +496,17 @@ public void setUserPermissions(String username, byte[] table,
    */
   public void setGroupPermissions(String group, byte[] table,
       List<TablePermission> perms) {
-    ListMultimap<String,TablePermission> tablePerms = TABLE_GROUP_CACHE.get(table);
-    if (tablePerms == null) {
-      tablePerms = ArrayListMultimap.create();
-      TABLE_GROUP_CACHE.put(table, tablePerms);
-    }
-    tablePerms.replaceValues(group, perms);
-    writeToZooKeeper(table, TABLE_USER_CACHE.get(table), tablePerms);
+    PermissionCache<TablePermission> tablePerms = getTablePermissions(table);
+    tablePerms.replaceGroup(group, perms);
+    writeToZooKeeper(table, tablePerms);
   }
 
   public void writeToZooKeeper(byte[] table,
-      ListMultimap<String,TablePermission> userPerms,
-      ListMultimap<String,TablePermission> groupPerms) {
-    ListMultimap<String,TablePermission> tmp = ArrayListMultimap.create();
-    if (userPerms != null) {
-      tmp.putAll(userPerms);
-    }
-    if (groupPerms != null) {
-      for (String group : groupPerms.keySet()) {
-        tmp.putAll(AccessControlLists.GROUP_PREFIX + group,
-            groupPerms.get(group));
-      }
+      PermissionCache<TablePermission> tablePerms) {
+    byte[] serialized = new byte[0];
+    if (tablePerms != null) {
+      serialized = AccessControlLists.writePermissionsAsBytes(tablePerms.getAllPermissions(), conf);
     }
-    byte[] serialized = AccessControlLists.writePermissionsAsBytes(tmp, conf);
     zkperms.writeToZookeeper(table, serialized);
   }
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
index 6407b166c7d6..9c04d59d3422 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.AfterClass;
@@ -360,4 +361,23 @@ public void testGlobalPermission() throws Exception {
                  },
                  user3Perms.get(0).getActions());
   }
+
+  @Test
+  public void testAuthManager() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+    /* test a race condition causing TableAuthManager to sometimes fail global permissions checks
+     * when the global cache is being updated
+     */
+    TableAuthManager authManager = TableAuthManager.get(ZKW, conf);
+    // currently running user is the system user and should have global admin perms
+    User currentUser = User.getCurrent();
+    assertTrue(authManager.authorize(currentUser, Permission.Action.ADMIN));
+    for (int i=1; i<=50; i++) {
+      AccessControlLists.addUserPermission(conf, new UserPermission(Bytes.toBytes("testauth"+i),
+          Permission.Action.ADMIN, Permission.Action.READ, Permission.Action.WRITE));
+      // make sure the system user still shows as authorized
+      assertTrue("Failed current user auth check on iter "+i,
+          authManager.authorize(currentUser, Permission.Action.ADMIN));
+    }
+  }
 }

From 79e899de25abbfbf5c272d2bcdc200eb8292ab05 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Sep 2012 22:05:01 +0000
Subject: [PATCH 0493/1540] HBASE-6784 TestCoprocessorScanPolicy is sometimes
 flaky when run locally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1389619 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/TestCoprocessorScanPolicy.java      | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
index 1915ca372762..15533a6367db 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
@@ -90,12 +90,14 @@ public void testBaseCases() throws Exception {
     p.add(F, tableName, Bytes.toBytes(2));
     t.put(p);
 
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+
     // insert 2 versions
     p = new Put(R);
-    p.add(F, Q, Q);
+    p.add(F, Q, now, Q);
     t.put(p);
     p = new Put(R);
-    p.add(F, Q, Q);
+    p.add(F, Q, now+1, Q);
     t.put(p);
     Get g = new Get(R);
     g.setMaxVersions(10);
@@ -113,7 +115,7 @@ public void testBaseCases() throws Exception {
 
     // insert a 3rd version
     p = new Put(R);
-    p.add(F, Q, Q);
+    p.add(F, Q, now+2, Q);
     t.put(p);
     g = new Get(R);
     g.setMaxVersions(10);

From dcab2e2f891e8ca682cb2402e932c2e28a9523d7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 25 Sep 2012 17:37:34 +0000
Subject: [PATCH 0494/1540] HBASE-6868 Skip checksum is broke; are we
 double-checksumming by default?

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1390012 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java       | 2 ++
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java     | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
index fe72375ad9d2..ef1cda02c6d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -73,6 +73,8 @@ public HFileSystem(Configuration conf, boolean useHBaseChecksum)
     // This manifests itself in that incorrect data is read and HFileBlocks won't be able to read
     // their header magic numbers. See HBASE-5885
     if (useHBaseChecksum && !(fs instanceof LocalFileSystem)) {
+      conf = new Configuration(conf);
+      conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
       this.noChecksumFs = newInstanceFileSystem(conf);
       this.noChecksumFs.setVerifyChecksum(false);
     } else {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1f8326311098..8f9cf0c02d71 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -378,7 +378,7 @@ public HRegionServer(Configuration conf)
     // do we use checksum verfication in the hbase? If hbase checksum verification
     // is enabled, then we automatically switch off hdfs checksum verification.
     this.useHBaseChecksum = conf.getBoolean(
-      HConstants.HBASE_CHECKSUM_VERIFICATION, true);
+      HConstants.HBASE_CHECKSUM_VERIFICATION, false);
 
     // Config'ed params
     this.numRetries = conf.getInt("hbase.client.retries.number", 10);

From 98aef1a9b022b8077bf920dd97a332c6a9b055d8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 25 Sep 2012 17:42:11 +0000
Subject: [PATCH 0495/1540] Updated CHANGES.txt for 0.94.2RC2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1390015 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 3a73801b95d6..d5889973bde2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.2 - 9/21/2012
+Release 0.94.2 - 9/25/2012
 Sub-task
 
     [HBASE-6257] - Avoid unnecessary flush & compact on Meta in admin.rb.
@@ -74,11 +74,14 @@ Bug
     [HBASE-6757] - Very inefficient behaviour of scan using FilterList
     [HBASE-6762] - HBASE-6340 broke SecureRPCEngine
     [HBASE-6769] - HRS.multi eats NoSuchColumnFamilyException since HBASE-5021
+    [HBASE-6784] - TestCoprocessorScanPolicy is sometimes flaky when run locally
     [HBASE-6803] - script hbase should add JAVA_LIBRARY_PATH to LD_LIBRARY_PATH
     [HBASE-6839] - Operations may be executed without holding rowLock
     [HBASE-6842] - the jar used in coprocessor is not deleted in local which will exhaust the space of /tmp
     [HBASE-6844] - upgrade 0.23 version dependency in 0.94
     [HBASE-6847] - HBASE-6649 broke replication
+    [HBASE-6851] - Race condition in TableAuthManager.updateGlobalCache()
+    [HBASE-6868] - Skip checksum is broke; are we double-checksumming by default?
 
 Improvement
 

From 16745f7b9e815686830c9e1168dbbd4b4e6af48c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 27 Sep 2012 00:21:39 +0000
Subject: [PATCH 0496/1540] HBASE-6679 RegionServer aborts due to race between
 compaction and split

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1390783 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 43ff0360c07a..2fd20552e258 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -148,7 +148,7 @@ public class Store extends SchemaConfigured implements HeapSize {
    * List of store files inside this store. This is an immutable list that
    * is atomically replaced when its contents change.
    */
-  private ImmutableList<StoreFile> storefiles = null;
+  private volatile ImmutableList<StoreFile> storefiles = null;
 
   List<StoreFile> filesCompacting = Lists.newArrayList();
 

From 8eff37fa6c92219eac3ad13a9900fd1878d56125 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 27 Sep 2012 16:04:12 +0000
Subject: [PATCH 0497/1540] HBASE-6854 Deletion of SPLITTING node on split
 rollback should clear the region from RIT (Ram)

Submitted by:Ram
Reviewed by:Bijieshan, Stack


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391074 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  8 ++-
 .../hbase/regionserver/SplitTransaction.java  |  2 +-
 .../TestSplitTransactionOnCluster.java        | 56 +++++++++++++++++++
 3 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index c0e4e3519bb9..2cbc8fc87a5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1125,9 +1125,13 @@ public void nodeDeleted(final String path) {
       if (rs != null) {
         HRegionInfo regionInfo = rs.getRegion();
         if (rs.isSplit()) {
-          LOG.debug("Ephemeral node deleted, regionserver crashed?, " +
-            "clearing from RIT; rs=" + rs);
+          LOG.debug("Ephemeral node deleted, regionserver crashed?, offlining the region"
+              + rs.getRegion() + " clearing from RIT;");
           regionOffline(rs.getRegion());
+        } else if (rs.isSplitting()) {
+          LOG.debug("Ephemeral node deleted.  Found in SPLITTING state. " + "Removing from RIT "
+              + rs.getRegion());
+          this.regionsInTransition.remove(regionName);
         } else {
           LOG.debug("The znode of region " + regionInfo.getRegionNameAsString()
               + " has been deleted.");
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 36576b1aec84..7c1e23c5eb97 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -539,7 +539,7 @@ private static Path getSplitDir(final HRegion r) {
    * to create it.
    * @see #cleanupSplitDir(FileSystem, Path)
    */
-  private static void createSplitDir(final FileSystem fs, final Path splitdir)
+  void createSplitDir(final FileSystem fs, final Path splitdir)
   throws IOException {
     if (fs.exists(splitdir)) {
       LOG.info("The " + splitdir
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index fbd70090f367..01ff8887e28a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -31,12 +31,16 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.handler.SplitRegionHandler;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -587,6 +591,58 @@ int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName ser
     }
   }
   
+  @Test
+  public void testShouldClearRITWhenNodeFoundInSplittingState() throws Exception {
+    final byte[] tableName = Bytes.toBytes("testShouldClearRITWhenNodeFoundInSplittingState");
+    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+    try {
+      // Create table then get the single region for our new table.
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor("cf"));
+      admin.createTable(htd);
+
+      List<HRegion> regions = cluster.getRegions(tableName);
+      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+      SplitTransaction st = null;
+
+      st = new MockedSplitTransaction(regions.get(0), null) {
+        @Override
+        void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
+          throw new IOException("");
+        }
+      };
+
+      try {
+        st.execute(regionServer, regionServer);
+      } catch (IOException e) {
+        String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
+            .getRegionInfo().getEncodedName());
+
+        assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+        AssignmentManager am = cluster.getMaster().getAssignmentManager();
+        while (!am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName())) {
+          Thread.sleep(200);
+        }
+        RegionState regionState = am.getRegionsInTransition().get(regions.get(0).getRegionInfo()
+            .getEncodedName());
+        assertTrue(regionState.getState() == RegionState.State.SPLITTING);
+        assertTrue(st.rollback(regionServer, regionServer));
+        assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+        while (am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName())) {
+          // Just in case the nodeDeleted event did not get executed.
+          Thread.sleep(200);
+        }
+      }
+    } finally {
+      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+        admin.disableTable(tableName);
+        admin.deleteTable(tableName);
+        admin.close();
+      }
+    }
+  }
+  
   public static class MockedSplitTransaction extends SplitTransaction {
 
     public MockedSplitTransaction(HRegion r, byte[] splitrow) {

From 0907b8bf50f4e257cf8ea77a301a4a341cabf213 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 27 Sep 2012 20:56:47 +0000
Subject: [PATCH 0498/1540] HBASE-6888 HBase scripts ignore any HBASE_OPTS set
 in the environment

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391214 13f79535-47bb-0310-9956-ffa450edef68
---
 conf/hbase-env.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index 8e74de662779..5b5ba96045ac 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -34,7 +34,7 @@
 # Below are what we set by default.  May only work with SUN JVM.
 # For more on why as well as other possible settings,
 # see http://wiki.apache.org/hadoop/PerformanceTuning
-export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC"
 
 # Uncomment below to enable java garbage collection logging in the .out file.
 # export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" 

From dc63cafb5c9aaef1480a748a38a6951b760f92a0 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 28 Sep 2012 02:23:08 +0000
Subject: [PATCH 0499/1540] HBASE-6688 folder referred by thrift demo app
 instructions is outdated

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391284 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/thrift2/package.html  | 67 +++++++++++++------
 .../apache/hadoop/hbase/thrift/package.html   | 29 ++++----
 2 files changed, 60 insertions(+), 36 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/package.html b/src/main/java/org/apache/hadoop/hbase/thrift2/package.html
index 8751a8021a72..3d25e4d382ca 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/package.html
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/package.html
@@ -26,23 +26,24 @@
 This package contains a Thrift interface definition file for an HBase RPC
 service and a Java server implementation.
 
-There is currently 2 thrift server implementations in HBase, the packages:
+There are currently 2 thrift server implementations in HBase, the packages:
 
 <ul>
-  <li>org.apache.hadoop.hbase.thrift - This may one day be marked as depreceated.</li>
-  <li>org.apache.hadoop.hbase.thrift2 - This is intended to closely match to the HTable interface.</li>  
+  <li>org.apache.hadoop.hbase.thrift: This may one day be marked as depreceated.</li>
+  <li>org.apache.hadoop.hbase.thrift2: i.e. this package. This is intended to closely match to the HTable interface and
+  to one day supercede the older thrift (the old thrift mimics an API HBase no longer has).</li>  
 </ul>
 
 <h2><a name="whatisthrift">What is Thrift?</a></h2>
 
-<p>From http://thrift.apache.org/</p> 
   
 <p>"Thrift is a software framework for scalable cross-language services 
 development. It combines a software stack with a code generation engine to
 build services that work efficiently and seamlessly between C++, Java, Python,
 PHP, Ruby, Erlang, Perl, Haskell, C#, Cocoa, JavaScript, Node.js, Smalltalk,
 and OCaml. Originally developed at Facebook, Thrift was open sourced in April
-2007 and entered the Apache Incubator in May, 2008"</p>
+2007 and entered the Apache Incubator in May, 2008".
+From http://thrift.apache.org/</p> 
 
 <h2><a name="description">Description</a></h2>
 
@@ -52,33 +53,55 @@ <h2><a name="description">Description</a></h2>
 server boiler plate in <code>org.apache.hadoop.hbase.thrift2.ThriftServer</code>.
 The generated interfaces, types, and RPC utility files are checked into SVN under the
 <code>org.apache.hadoop.hbase.thrift2.generated</code> directory.
-
 </p>
+<p>To stop, use:
+<pre>
+  ./bin/hbase-daemon.sh stop thrift
+</pre>
+
+These are the command line arguments the Thrift server understands in addition to <code>start</code> and <code>stop</code>:
+<dl>
+    <dt><code>-b, --bind</code></dt>
+    <dd>Address to bind the Thrift server to. Not supported by the Nonblocking and HsHa server [default: <code>0.0.0.0</code>]</dd>
+
+    <dt><code>-p, --port</code></dt>
+    <dd>Port to bind to [default: <code>9090</code>]</dd>
+
+    <dt><code>-f, --framed</code></dt>
+    <dd>Use framed transport (implied when using one of the non-blocking servers)</dd>
+
+    <dt><code>-c, --compact</code></dt>
+    <dd>Use the compact protocol [default: binary protocol]</dd>
+
+    <dt><code>-h, --help</code></dt>
+    <dd>Displays usage information for the Thrift server</dd>
+
+    <dt><code>-threadpool</code></dt>
+    <dd>Use the TThreadPoolServer. This is the default.</dd>
+
+    <dt><code>-hsha</code></dt>
+    <dd>Use the THsHaServer. This implies the framed transport.</dd>
 
-<p>The files were generated by running the commands:
+    <dt><code>-nonblocking</code></dt>
+    <dd>Use the TNonblockingServer. This implies the framed transport.</dd>
+</dl>
+
+<h3><a name="details">Details</a></h3>
+
+<p>HBase currently uses version 0.8.0 of Apache Thrift.</p>
+<p>The files were generated by running the commands under the hbase checkout dir:
 <pre>
-  thrift -strict --gen java src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
-  mv gen-java/org/apache/hadoop/hbase/thrift2/generated src/main/java/org/apache/hadoop/hbase/thrift2/generated
+  thrift -strict --gen java:hashcode ./src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+  # Move the generated files into place their expected location under hbase
+  mv gen-java/org/apache/hadoop/hbase/thrift2/generated/* src/main/java/org/apache/hadoop/hbase/thrift2/generated/
+  # Remove the gen-java file made by thrift
   rm -rf gen-java
 </pre>
-</p>
 
 <p>The 'thrift' binary is the Thrift compiler, and it is distributed separately from HBase
 in a Thrift release. Additionally, specific language runtime libraries are a
 part of a Thrift release. A version of the Java runtime is included in HBase via maven.
 </p>
 
-<p>To start ThriftServer, use:
-<pre>
-  ./bin/hbase-daemon.sh start thrift2 [--port=PORT]
-</pre>
-The default port is 9090.
-</p>
-
-<p>To stop, use:
-<pre>
-  ./bin/hbase-daemon.sh stop thrift
-</pre>
-</p>
 </body>
 </html>
diff --git a/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html b/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html
index f41a9ab477be..b82a98a6736e 100644
--- a/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html
+++ b/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html
@@ -23,9 +23,10 @@
 Provides an HBase <a href="http://incubator.apache.org/thrift/">Thrift</a>
 service.
 
-This directory contains a Thrift interface definition file for an Hbase RPC
+This directory contains a Thrift interface definition file for an HBase RPC
 service and a Java server implementation.
 
+
 <h2><a name="whatisthrift">What is Thrift?</a></h2> 
 <p><blockquote>"Thrift is a software framework for scalable cross-language services development.
 It combines a software stack with a code generation engine to build services
@@ -34,9 +35,11 @@ <h2><a name="whatisthrift">What is Thrift?</a></h2>
 
 <h2><a name="description">Description</a></h2>
 
-<p><i>Important note:</i> This Thrift interface is <i>deprecated</i> and scheduled for removal in HBase 0.22.
-A new version that matches the client API that was introduced in HBase 0.21 can be found
-in the <code>contrib</code> directory.
+<p><i>Important note:</i> We tried to deprecate this Thrift interface and replace it
+with the Interface defined over in the thrift2 package only this package will not die.
+Folks keep adding to it and fixing it up so its around for another while until someone
+takes command and drives this package out of existence replacing it w/ an Interface that
+better matches the hbase API (this package was modelled on old HBase API long since dropped).
 </p>
 
 <p>The {@link org.apache.hadoop.hbase.thrift.generated.Hbase.Iface HBase API} is defined in the
@@ -84,25 +87,23 @@ <h2><a name="description">Description</a></h2>
     <dd>Use the TNonblockingServer. This implies the framed transport.</dd>
 </dl>
 
-<p></p><i>Important note:</i> The <code>bind</code> option only works with the default ThreadPoolServer.
-This will be fixed in the next Thrift version. See <a href="https://issues.apache.org/jira/browse/HBASE-2155">HBASE-2155</a>
-for more details on this issue.
-
 <h3><a name="details">Details</a></h3>
 
-<p>HBase currently uses version 0.2.0 of Apache Thrift.</p>
+<p>HBase currently uses version 0.8.0 of Apache Thrift.</p>
 
-<p>The files were generated by running the commands:
+<p>The files were generated by running the commands under the hbase checkout dir:
 <pre>
-  thrift -strict --gen java:hashcode Hbase.thrift
-  mv gen-java/org/apache/hadoop/hbase/thrift/generated .
+  thrift -strict --gen java:hashcode ./src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
+  # Move the generated files into place their expected location under hbase
+  mv gen-java/org/apache/hadoop/hbase/thrift/generated/* src/main/java/org/apache/hadoop/hbase/thrift/generated/
+  # Remove the gen-java file made by thrift
   rm -rf gen-java
 </pre>
 
 <p>The 'thrift' binary is the Thrift compiler, and it is distributed as a part
 of the Thrift package. Additionally, specific language runtime libraries are a
-part of the Thrift package. A version of the Java runtime is checked into SVN
-under the <code>hbase/lib</code> directory.</p>
+part of the Thrift package. A version of the Java runtime is included in HBase
+via Maven.</p>
 
 </body>
 </html>

From d02e466bc2ce14737fb12c44b91a8111cf62ade9 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 29 Sep 2012 04:01:57 +0000
Subject: [PATCH 0500/1540] HBASE-6871 HFileBlockIndex Write Error in HFile V2
 due to incorrect split into intermediate index blocks

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391742 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/hfile/HFileBlockIndex.java       | 23 +++--
 .../TestHFileInlineToRootChunkConversion.java | 84 +++++++++++++++++++
 2 files changed, 100 insertions(+), 7 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index 576c988f6163..c4f671405eea 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -745,7 +745,7 @@ public void setMaxChunkSize(int maxChunkSize) {
      * @throws IOException
      */
     public long writeIndexBlocks(FSDataOutputStream out) throws IOException {
-      if (curInlineChunk.getNumEntries() != 0) {
+      if (curInlineChunk != null && curInlineChunk.getNumEntries() != 0) {
         throw new IOException("Trying to write a multi-level block index, " +
             "but are " + curInlineChunk.getNumEntries() + " entries in the " +
             "last inline chunk.");
@@ -756,9 +756,11 @@ public long writeIndexBlocks(FSDataOutputStream out) throws IOException {
       byte[] midKeyMetadata = numLevels > 1 ? rootChunk.getMidKeyMetadata()
           : null;
 
-      while (rootChunk.getRootSize() > maxChunkSize) {
-        rootChunk = writeIntermediateLevel(out, rootChunk);
-        numLevels += 1;
+      if (curInlineChunk != null) {
+        while (rootChunk.getRootSize() > maxChunkSize) {
+          rootChunk = writeIntermediateLevel(out, rootChunk);
+          numLevels += 1;
+        }
       }
 
       // write the root level
@@ -920,11 +922,18 @@ private void expectNumLevels(int expectedNumLevels) {
      */
     @Override
     public boolean shouldWriteBlock(boolean closing) {
-      if (singleLevelOnly)
+      if (singleLevelOnly) {
         throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);
+      }
 
-      if (curInlineChunk.getNumEntries() == 0)
+      if (curInlineChunk == null) {
+        throw new IllegalStateException("curInlineChunk is null; has shouldWriteBlock been " +
+            "called with closing=true and then called again?");
+      }
+
+      if (curInlineChunk.getNumEntries() == 0) {
         return false;
+      }
 
       // We do have some entries in the current inline chunk.
       if (closing) {
@@ -934,7 +943,7 @@ public boolean shouldWriteBlock(boolean closing) {
 
           expectNumLevels(1);
           rootChunk = curInlineChunk;
-          curInlineChunk = new BlockIndexChunk();
+          curInlineChunk = null;  // Disallow adding any more index entries.
           return false;
         }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
new file mode 100644
index 000000000000..2b672849022b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+/**
+ * Test a case when an inline index chunk is converted to a root one. This reproduces the bug in
+ * HBASE-6871. We write a carefully selected number of relatively large keys so that we accumulate
+ * a leaf index chunk that only goes over the configured index chunk size after adding the last
+ * key/value. The bug is in that when we close the file, we convert that inline (leaf-level) chunk
+ * into a root chunk, but then look at the size of that root chunk, find that it is greater than
+ * the configured chunk size, and split it into a number of intermediate index blocks that should
+ * really be leaf-level blocks. If more keys were added, we would flush the leaf-level block, add
+ * another entry to the root-level block, and that would prevent us from upgrading the leaf-level
+ * chunk to the root chunk, thus not triggering the bug. 
+ */
+public class TestHFileInlineToRootChunkConversion {
+  private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+  private final Configuration conf = testUtil.getConfiguration();
+  
+  @Test
+  public void testWriteHFile() throws Exception {
+    Path hfPath = new Path(testUtil.getDataTestDir(),
+        TestHFileInlineToRootChunkConversion.class.getSimpleName() + ".hfile");
+    int maxChunkSize = 1024;
+    FileSystem fs = FileSystem.get(conf);
+    CacheConfig cacheConf = new CacheConfig(conf);
+    conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize); 
+    HFileWriterV2 hfw =
+        (HFileWriterV2) new HFileWriterV2.WriterFactoryV2(conf, cacheConf)
+            .withBlockSize(16)
+            .withPath(fs, hfPath).create();
+    List<byte[]> keys = new ArrayList<byte[]>();
+    StringBuilder sb = new StringBuilder();
+
+    for (int i = 0; i < 4; ++i) {
+      sb.append("key" + String.format("%05d", i));
+      sb.append("_");
+      for (int j = 0; j < 100; ++j) {
+        sb.append('0' + j);
+      }
+      String keyStr = sb.toString();
+      sb.setLength(0);
+
+      byte[] k = Bytes.toBytes(keyStr);
+      System.out.println("Key: " + Bytes.toString(k));
+      keys.add(k);
+      byte[] v = Bytes.toBytes("value" + i);
+      hfw.append(k, v);
+    }
+    hfw.close();
+
+    HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs, hfPath, cacheConf);
+    HFileScanner scanner = reader.getScanner(true, true);
+    for (int i = 0; i < keys.size(); ++i) {
+      scanner.seekTo(keys.get(i));
+    }
+    reader.close();
+  }
+}

From a5aa31318c48529420de2b5df6b7d7c2f7cc3f08 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Sat, 29 Sep 2012 06:43:56 +0000
Subject: [PATCH 0501/1540] HBASE-6853 IllegalArgument Exception is thrown when
 an empty region is spliitted(Priya)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391760 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/SplitTransaction.java  |  7 ++-
 .../TestSplitTransactionOnCluster.java        | 47 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 7c1e23c5eb97..85f585d79b12 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -275,7 +275,12 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
       if (exceptionToThrow instanceof IOException) throw (IOException)exceptionToThrow;
       throw new IOException(exceptionToThrow);
     }
-
+            
+    if (hstoreFilesToSplit.size() == 0) {
+      String errorMsg = "No store files to split for the region " + this.parent.getRegionInfo();
+      LOG.error(errorMsg);
+      throw new IOException(errorMsg);
+    }
     if (!testing) {
       services.removeFromOnlineRegions(this.parent.getRegionInfo().getEncodedName());
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 01ff8887e28a..070e6d47469a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -643,6 +643,53 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
     }
   }
   
+  @Test
+  public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfullyExecuteRollback()
+      throws Exception {
+    final byte[] tableName = Bytes
+        .toBytes("testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfullyExecuteRollback");
+    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+    try {
+      // Create table then get the single region for our new table.
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor("cf"));
+      admin.createTable(htd);
+      List<HRegion> regions = cluster.getRegions(tableName);
+      HRegionInfo hri = getAndCheckSingleTableRegion(regions);
+      int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
+      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+      // Turn off balancer so it doesn't cut in and mess up our placements.
+      this.admin.setBalancerRunning(false, false);
+      // Turn off the meta scanner so it don't remove parent on us.
+      cluster.getMaster().setCatalogJanitorEnabled(false);
+      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
+      printOutRegions(server, "Initial regions: ");
+      // Now split.
+      SplitTransaction st = null;
+      st = new MockedSplitTransaction(regions.get(0), null);
+      try {
+        st.execute(regionServer, regionServer);
+      } catch (IOException e) {
+        List<HRegion> daughters = cluster.getRegions(tableName);
+        assertTrue(daughters.size() == 1);
+
+        String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
+            .getRegionInfo().getEncodedName());
+        assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+        assertTrue(st.rollback(regionServer, regionServer));
+        assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+      }
+    } finally {
+      admin.setBalancerRunning(true, false);
+      cluster.getMaster().setCatalogJanitorEnabled(true);
+      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+        admin.disableTable(tableName);
+        admin.deleteTable(tableName);
+        admin.close();
+      }
+    }
+  }
   public static class MockedSplitTransaction extends SplitTransaction {
 
     public MockedSplitTransaction(HRegion r, byte[] splitrow) {

From 215d2162756b15eabea743298eca0121be3ae3ed Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 29 Sep 2012 20:40:27 +0000
Subject: [PATCH 0502/1540] HBASE-6871 HFileBlockIndex Write Error in HFile V2
 due to incorrect split into intermediate index blocks; ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391869 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/hfile/TestHFileInlineToRootChunkConversion.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
index 2b672849022b..6491f3c605e0 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
@@ -38,6 +38,7 @@
  * another entry to the root-level block, and that would prevent us from upgrading the leaf-level
  * chunk to the root chunk, thus not triggering the bug. 
  */
+@Category(SmallTests.class)
 public class TestHFileInlineToRootChunkConversion {
   private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
   private final Configuration conf = testUtil.getConfiguration();

From 267b941c4c6bd0fdb48c86dd651a1ad03e29ba9a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 29 Sep 2012 21:00:20 +0000
Subject: [PATCH 0503/1540] HBASE-6871 HFileBlockIndex Write Error in HFile V2
 due to incorrect split into intermediate index blocks; ADDENDUM2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391877 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       | 39 +++++++++++++++----
 .../TestHFileInlineToRootChunkConversion.java |  1 +
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/pom.xml b/pom.xml
index 97afd3e86e20..f0daf8f45b65 100644
--- a/pom.xml
+++ b/pom.xml
@@ -871,8 +871,14 @@
 
                 <!-- Using Unix cp to preserve symlinks, using script to handle wildcards -->
                 <echo file="${project.build.directory}/copynativelibs.sh">
-                    if [ `ls ${project.build.directory}/nativelib | wc -l` -ne 0 ]; then
-                      cp -PR ${project.build.directory}/nativelib/lib* ${project.build.directory}/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform}
+					which cygpath 2&gt; /dev/null
+					if [ $? = 1 ]; then
+						BUILD_DIR="${project.build.directory}"
+					else
+						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+					fi
+                    if [ `ls $BUILD_DIR/nativelib | wc -l` -ne 0 ]; then
+                      cp -PR $BUILD_DIR/nativelib/lib* $BUILD_DIR/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform}
                     fi
                 </echo>
                 <exec executable="sh" dir="${project.build.directory}" failonerror="true">
@@ -880,11 +886,19 @@
                 </exec>
 
                 <!-- Using Unix tar to preserve symlinks -->
-                <exec executable="tar" failonerror="yes"
-                  dir="${project.build.directory}/${project.build.finalName}">
-                    <arg value="czf"/>
-                    <arg value="${project.build.directory}/${project.build.finalName}.tar.gz"/>
-                    <arg value="${project.build.finalName}/" />
+				<echo file="${project.build.directory}/maketar.sh">
+					which cygpath 2&gt; /dev/null
+					if [ $? = 1 ]; then
+						BUILD_DIR="${project.build.directory}"
+					else
+						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+					fi
+					
+					cd $BUILD_DIR/${project.build.finalName}
+					tar czf $BUILD_DIR/${project.build.finalName}.tar.gz ${project.build.finalName}
+				</echo>
+                <exec executable="sh" dir="${project.build.directory}" failonerror="yes">
+                    <arg line="./maketar.sh"/>
                 </exec>
 
               </target>
@@ -1448,6 +1462,17 @@
             <build.platform>Mac_OS_X-${sun.arch.data.model}</build.platform>
         </properties>
     </profile>
+	<profile>
+		<id>os.windows</id>
+		<activation>
+			<os>
+				<family>Windows</family>
+			</os>
+		</activation>
+		<properties>
+			<build.platform>cygwin</build.platform>
+		</properties>
+	</profile>
 
     <!-- this profile should be activated for release builds -->
     <profile>
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
index 6491f3c605e0..419fc0b39cb2 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.junit.experimental.categories.Category;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;

From 061a2457b184c8559453bddf20cb9042380b6c93 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 29 Sep 2012 21:02:58 +0000
Subject: [PATCH 0504/1540] HBASE-6871 HFileBlockIndex Write Error in HFile V2
 due to incorrect split into intermediate index blocks; ADDENDUM2 OVERCOMMIT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391878 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       | 39 ++++---------------
 .../TestHFileInlineToRootChunkConversion.java |  1 -
 2 files changed, 7 insertions(+), 33 deletions(-)

diff --git a/pom.xml b/pom.xml
index f0daf8f45b65..97afd3e86e20 100644
--- a/pom.xml
+++ b/pom.xml
@@ -871,14 +871,8 @@
 
                 <!-- Using Unix cp to preserve symlinks, using script to handle wildcards -->
                 <echo file="${project.build.directory}/copynativelibs.sh">
-					which cygpath 2&gt; /dev/null
-					if [ $? = 1 ]; then
-						BUILD_DIR="${project.build.directory}"
-					else
-						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
-					fi
-                    if [ `ls $BUILD_DIR/nativelib | wc -l` -ne 0 ]; then
-                      cp -PR $BUILD_DIR/nativelib/lib* $BUILD_DIR/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform}
+                    if [ `ls ${project.build.directory}/nativelib | wc -l` -ne 0 ]; then
+                      cp -PR ${project.build.directory}/nativelib/lib* ${project.build.directory}/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform}
                     fi
                 </echo>
                 <exec executable="sh" dir="${project.build.directory}" failonerror="true">
@@ -886,19 +880,11 @@
                 </exec>
 
                 <!-- Using Unix tar to preserve symlinks -->
-				<echo file="${project.build.directory}/maketar.sh">
-					which cygpath 2&gt; /dev/null
-					if [ $? = 1 ]; then
-						BUILD_DIR="${project.build.directory}"
-					else
-						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
-					fi
-					
-					cd $BUILD_DIR/${project.build.finalName}
-					tar czf $BUILD_DIR/${project.build.finalName}.tar.gz ${project.build.finalName}
-				</echo>
-                <exec executable="sh" dir="${project.build.directory}" failonerror="yes">
-                    <arg line="./maketar.sh"/>
+                <exec executable="tar" failonerror="yes"
+                  dir="${project.build.directory}/${project.build.finalName}">
+                    <arg value="czf"/>
+                    <arg value="${project.build.directory}/${project.build.finalName}.tar.gz"/>
+                    <arg value="${project.build.finalName}/" />
                 </exec>
 
               </target>
@@ -1462,17 +1448,6 @@
             <build.platform>Mac_OS_X-${sun.arch.data.model}</build.platform>
         </properties>
     </profile>
-	<profile>
-		<id>os.windows</id>
-		<activation>
-			<os>
-				<family>Windows</family>
-			</os>
-		</activation>
-		<properties>
-			<build.platform>cygwin</build.platform>
-		</properties>
-	</profile>
 
     <!-- this profile should be activated for release builds -->
     <profile>
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
index 419fc0b39cb2..6491f3c605e0 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
@@ -23,7 +23,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.junit.experimental.categories.Category;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;

From 6abd8dec5e7d183cd920ff223d21d255667b8a3b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 29 Sep 2012 21:03:22 +0000
Subject: [PATCH 0505/1540] HBASE-6871 HFileBlockIndex Write Error in HFile V2
 due to incorrect split into intermediate index blocks; ADDENDUM2
 REAPPLICATION

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391879 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/hfile/TestHFileInlineToRootChunkConversion.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
index 6491f3c605e0..419fc0b39cb2 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.junit.experimental.categories.Category;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;

From 095a5b695e5e8793a30f981382e45ef3e976612e Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 29 Sep 2012 21:04:08 +0000
Subject: [PATCH 0506/1540] HBASE-4565 Maven HBase build broken on cygwin with
 copynativelib.sh call

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1391880 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/pom.xml b/pom.xml
index 97afd3e86e20..f0daf8f45b65 100644
--- a/pom.xml
+++ b/pom.xml
@@ -871,8 +871,14 @@
 
                 <!-- Using Unix cp to preserve symlinks, using script to handle wildcards -->
                 <echo file="${project.build.directory}/copynativelibs.sh">
-                    if [ `ls ${project.build.directory}/nativelib | wc -l` -ne 0 ]; then
-                      cp -PR ${project.build.directory}/nativelib/lib* ${project.build.directory}/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform}
+					which cygpath 2&gt; /dev/null
+					if [ $? = 1 ]; then
+						BUILD_DIR="${project.build.directory}"
+					else
+						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+					fi
+                    if [ `ls $BUILD_DIR/nativelib | wc -l` -ne 0 ]; then
+                      cp -PR $BUILD_DIR/nativelib/lib* $BUILD_DIR/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform}
                     fi
                 </echo>
                 <exec executable="sh" dir="${project.build.directory}" failonerror="true">
@@ -880,11 +886,19 @@
                 </exec>
 
                 <!-- Using Unix tar to preserve symlinks -->
-                <exec executable="tar" failonerror="yes"
-                  dir="${project.build.directory}/${project.build.finalName}">
-                    <arg value="czf"/>
-                    <arg value="${project.build.directory}/${project.build.finalName}.tar.gz"/>
-                    <arg value="${project.build.finalName}/" />
+				<echo file="${project.build.directory}/maketar.sh">
+					which cygpath 2&gt; /dev/null
+					if [ $? = 1 ]; then
+						BUILD_DIR="${project.build.directory}"
+					else
+						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+					fi
+					
+					cd $BUILD_DIR/${project.build.finalName}
+					tar czf $BUILD_DIR/${project.build.finalName}.tar.gz ${project.build.finalName}
+				</echo>
+                <exec executable="sh" dir="${project.build.directory}" failonerror="yes">
+                    <arg line="./maketar.sh"/>
                 </exec>
 
               </target>
@@ -1448,6 +1462,17 @@
             <build.platform>Mac_OS_X-${sun.arch.data.model}</build.platform>
         </properties>
     </profile>
+	<profile>
+		<id>os.windows</id>
+		<activation>
+			<os>
+				<family>Windows</family>
+			</os>
+		</activation>
+		<properties>
+			<build.platform>cygwin</build.platform>
+		</properties>
+	</profile>
 
     <!-- this profile should be activated for release builds -->
     <profile>

From 63f92e7ed199e77bf2e33bc129bdcecf1ed30a4d Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 1 Oct 2012 17:33:31 +0000
Subject: [PATCH 0507/1540] HBASE-6906 TestHBaseFsck#testQuarantine* tests are
 flakey due to TestNotEnabledException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1392456 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/TestHBaseFsck.java    | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 1066414a9a32..ef664afdc604 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1379,6 +1379,9 @@ public void testQuarantineCorruptHFile() throws Exception {
     }
   }
 
+  /**
+  * Test that use this should have a timeout, because this method could potentially wait forever.
+  */
   private void doQuarantineTest(String table, HBaseFsck hbck, int check, int corrupt, int fail,
       int quar, int missing) throws Exception {
     try {
@@ -1401,7 +1404,16 @@ private void doQuarantineTest(String table, HBaseFsck hbck, int check, int corru
       assertEquals(hfcc.getMissing().size(), missing);
 
       // its been fixed, verify that we can enable
-      TEST_UTIL.getHBaseAdmin().enableTable(table);
+      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+      admin.enableTableAsync(table);
+      while (!admin.isTableEnabled(table)) {
+        try {
+          Thread.sleep(250);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+          fail("Interrupted when trying to enable table " + table);
+        }
+      }
     } finally {
       deleteTable(table);
     }

From 835cdef163cf807fac2aff22233728e0893256e4 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 1 Oct 2012 17:35:17 +0000
Subject: [PATCH 0508/1540] HBASE-6901 Store file compactSelection throws
 ArrayIndexOutOfBoundsException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1392459 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Store.java      | 18 ++---
 .../mapreduce/TestHFileOutputFormat.java      | 75 ++++++++++++++++++-
 2 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 2fd20552e258..20164ea41ad3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1402,6 +1402,15 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
       int start = 0;
       double r = compactSelection.getCompactSelectionRatio();
 
+      // remove bulk import files that request to be excluded from minors
+      compactSelection.getFilesToCompact().removeAll(Collections2.filter(
+          compactSelection.getFilesToCompact(),
+          new Predicate<StoreFile>() {
+            public boolean apply(StoreFile input) {
+              return input.excludeFromMinorCompaction();
+            }
+          }));
+
       // skip selection algorithm if we don't have enough files
       if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
         if(LOG.isDebugEnabled()) {
@@ -1413,15 +1422,6 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
         return compactSelection;
       }
 
-      // remove bulk import files that request to be excluded from minors
-      compactSelection.getFilesToCompact().removeAll(Collections2.filter(
-          compactSelection.getFilesToCompact(),
-          new Predicate<StoreFile>() {
-            public boolean apply(StoreFile input) {
-              return input.excludeFromMinorCompaction();
-            }
-          }));
-
       /* TODO: add sorting + unit test back in when HBASE-2856 is fixed
       // Sort files by size to correct when normal skew is altered by bulk load.
       Collections.sort(filesToCompact, StoreFile.Comparators.FILE_SIZE);
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index a6bcbac44954..c21cd056dec9 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -686,12 +686,85 @@ private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue>
     }
   }
 
+  /**
+   * This test is to test the scenario happened in HBASE-6901.
+   * All files are bulk loaded and excluded from minor compaction.
+   * Without the fix of HBASE-6901, an ArrayIndexOutOfBoundsException
+   * will be thrown.
+   */
+  @Test
+  public void testExcludeAllFromMinorCompaction() throws Exception {
+    Configuration conf = util.getConfiguration();
+    conf.setInt("hbase.hstore.compaction.min", 2);
+    generateRandomStartKeys(5);
+
+    try {
+      util.startMiniCluster();
+      final FileSystem fs = util.getDFSCluster().getFileSystem();
+      HBaseAdmin admin = new HBaseAdmin(conf);
+      HTable table = util.createTable(TABLE_NAME, FAMILIES);
+      assertEquals("Should start with empty table", 0, util.countRows(table));
+
+      // deep inspection: get the StoreFile dir
+      final Path storePath = Store.getStoreHomedir(
+          HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), TABLE_NAME),
+          admin.getTableRegions(TABLE_NAME).get(0).getEncodedName(),
+          FAMILIES[0]);
+      assertEquals(0, fs.listStatus(storePath).length);
+
+      // Generate two bulk load files
+      conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
+          true);
+      util.startMiniMapReduceCluster();
+
+      for (int i = 0; i < 2; i++) {
+        Path testDir = util.getDataTestDir("testExcludeAllFromMinorCompaction_" + i);
+        runIncrementalPELoad(conf, table, testDir);
+        // Perform the actual load
+        new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table);
+      }
+
+      // Ensure data shows up
+      int expectedRows = 2 * NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT;
+      assertEquals("LoadIncrementalHFiles should put expected data in table",
+          expectedRows, util.countRows(table));
+
+      // should have a second StoreFile now
+      assertEquals(2, fs.listStatus(storePath).length);
+
+      // minor compactions shouldn't get rid of the file
+      admin.compact(TABLE_NAME);
+      try {
+        quickPoll(new Callable<Boolean>() {
+          public Boolean call() throws Exception {
+            return fs.listStatus(storePath).length == 1;
+          }
+        }, 5000);
+        throw new IOException("SF# = " + fs.listStatus(storePath).length);
+      } catch (AssertionError ae) {
+        // this is expected behavior
+      }
+
+      // a major compaction should work though
+      admin.majorCompact(TABLE_NAME);
+      quickPoll(new Callable<Boolean>() {
+        public Boolean call() throws Exception {
+          return fs.listStatus(storePath).length == 1;
+        }
+      }, 5000);
+
+    } finally {
+      util.shutdownMiniMapReduceCluster();
+      util.shutdownMiniCluster();
+    }
+  }
+
   @Test
   public void testExcludeMinorCompaction() throws Exception {
     Configuration conf = util.getConfiguration();
     conf.setInt("hbase.hstore.compaction.min", 2);
     Path testDir = util.getDataTestDir("testExcludeMinorCompaction");
-    byte[][] startKeys = generateRandomStartKeys(5);
+    generateRandomStartKeys(5);
 
     try {
       util.startMiniCluster();

From 1c35d88766829664a0d84cec3d4a8df9c974311d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 1 Oct 2012 21:23:00 +0000
Subject: [PATCH 0509/1540] HBASE-6299 RS starts region open while fails ack to
 HMaster.sendRegionOpen() causes inconsistency in HMaster's region state and a
 series of successive problems

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1392620 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/AssignmentManager.java    | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 2cbc8fc87a5b..e275b81acffb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1678,6 +1678,15 @@ private void assign(final HRegionInfo region, final RegionState state,
             }
           }
         }
+        if (t instanceof java.net.SocketTimeoutException 
+            && this.serverManager.isServerOnline(plan.getDestination())) {
+          LOG.warn("Call openRegion() to " + plan.getDestination()
+              + " has timed out when trying to assign "
+              + region.getRegionNameAsString()
+              + ", but the region might already be opened on "
+              + plan.getDestination() + ".", t);
+          return;
+        }
         LOG.warn("Failed assignment of "
             + state.getRegion().getRegionNameAsString()
             + " to "

From 69e2def126d6472d6d516fe8792aac8ca3be296e Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Tue, 2 Oct 2012 20:39:14 +0000
Subject: [PATCH 0510/1540] HBASE-6914 Scans/Gets/Mutations don't give a good
 error if the table is disabled.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1393164 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java       |  8 ++++++++
 .../apache/hadoop/hbase/client/TestAdmin.java  | 18 ++++++++----------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index ba2dad3b9cac..219a0bd7a83e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -813,6 +813,14 @@ public HRegionLocation locateRegion(final byte [] tableName,
     public HRegionLocation relocateRegion(final byte [] tableName,
         final byte [] row)
     throws IOException{
+
+      // Since this is an explicit request not to use any caching, finding
+      // disabled tables should not be desirable.  This will ensure that an exception is thrown when
+      // the first time a disabled table is interacted with.
+      if (isTableDisabled(tableName)) {
+        throw new DoNotRetryIOException(Bytes.toString(tableName) + " is disabled.");
+      }
+
       return locateRegion(tableName, row, false, true);
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index eab7ccef19d7..3eb511c5cf60 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -236,9 +236,7 @@ public void testDisableAndEnableTable() throws IOException {
     boolean ok = false;
     try {
       ht.get(get);
-    } catch (NotServingRegionException e) {
-      ok = true;
-    } catch (RetriesExhaustedException e) {
+    } catch (DoNotRetryIOException e) {
       ok = true;
     }
     assertTrue(ok);
@@ -284,23 +282,22 @@ public void testDisableAndEnableTables() throws IOException {
     try {
       ht1.get(get);
       ht2.get(get);
-    } catch (NotServingRegionException e) {
-      ok = true;
-    } catch (RetriesExhaustedException e) {
+    } catch (DoNotRetryIOException e) {
       ok = true;
     }
+
     assertTrue(ok);
     this.admin.enableTables("testDisableAndEnableTable.*");
 
     // Test that tables are enabled
     try {
       ht1.get(get);
-    } catch (RetriesExhaustedException e) {
+    } catch (IOException e) {
       ok = false;
     }
     try {
       ht2.get(get);
-    } catch (RetriesExhaustedException e) {
+    } catch (IOException e) {
       ok = false;
     }
     assertTrue(ok);
@@ -1011,9 +1008,10 @@ public void testEnableDisableAddColumnDeleteColumn() throws Exception {
     this.admin.disableTable(tableName);
     try {
       new HTable(TEST_UTIL.getConfiguration(), tableName);
-    } catch (org.apache.hadoop.hbase.client.RegionOfflineException e) {
-      // Expected
+    } catch (DoNotRetryIOException e) {
+      //expected
     }
+
     this.admin.addColumn(tableName, new HColumnDescriptor("col2"));
     this.admin.enableTable(tableName);
     try {

From 21285b64688122b4a8abee77b7cc9ae9d10464e5 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 2 Oct 2012 23:15:51 +0000
Subject: [PATCH 0511/1540] HBASE-6927 WrongFS using HRegionInfo.getTableDesc()
 and different fs for hbase.root and fs.defaultFS

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1393235 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/region_mover.rb                                    | 2 +-
 src/main/java/org/apache/hadoop/hbase/HRegionInfo.java | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index f778a720344f..5c6bea0ae1b2 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -129,7 +129,7 @@ def isSuccessfulScan(admin, r)
   scan.setBatch(1)
   scan.setCaching(1)
   scan.setFilter(FirstKeyOnlyFilter.new()) 
-  table = getTable(admin.getConfiguration(), r.getTableDesc().getName()) 
+  table = getTable(admin.getConfiguration(), r.getTableName()) 
   scanner = table.getScanner(scan)
   begin
     results = scanner.next() 
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 97ad5fdb3709..0ee74e77e58d 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -554,6 +554,8 @@ public boolean containsRow(byte[] row) {
    @Deprecated
   public HTableDescriptor getTableDesc() {
     Configuration c = HBaseConfiguration.create();
+    c.set("fs.defaultFS", c.get(HConstants.HBASE_DIR));
+    c.set("fs.default.name", c.get(HConstants.HBASE_DIR));
     FileSystem fs;
     try {
       fs = FileSystem.get(c);

From 732b54c8c3420a8023cfd12eff2d110acdcebc7e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Oct 2012 00:40:11 +0000
Subject: [PATCH 0512/1540] HBASE-6912 Filters are not properly applied in
 certain cases (revert HBASE-6562)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1393854 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  5 +-
 .../hbase/client/TestFakeKeyInFilter.java     | 84 -------------------
 .../hbase/client/TestFromClientSide.java      | 27 ++++++
 3 files changed, 29 insertions(+), 87 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 74e02d619ff9..7204a8b5619a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3467,8 +3467,7 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           rpcCall.throwExceptionIfCallerDisconnected();
         }
 
-        KeyValue kv = this.storeHeap.peek();
-        byte [] currentRow = kv == null ? null : kv.getRow();
+        byte [] currentRow = peekRow();
         if (isStopRow(currentRow)) {
           if (filter != null && filter.hasFilterRow()) {
             filter.filterRow(results);
@@ -3478,7 +3477,7 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           }
 
           return false;
-        } else if (kv != null && !kv.isInternal() && filterRowKey(currentRow)) {
+        } else if (filterRowKey(currentRow)) {
           nextRow(currentRow);
         } else {
           byte [] nextRow;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java b/src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java
deleted file mode 100644
index d80748afbafa..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFakeKeyInFilter.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import java.util.ArrayList;
-
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.filter.*;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-/**
- * Make sure the fake KVs created internally are never user visible
- * (not even to filters)
- */
-@Category(SmallTests.class)
-public class TestFakeKeyInFilter extends BinaryComparator {
-  protected static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
-  public TestFakeKeyInFilter() {
-    super(Bytes.toBytes("foo"));
-  }
-
-  @Override
-  public int compareTo(byte[] value, int offset, int length) {
-    if (value.length == 0) {
-      throw new RuntimeException("Found mysterious empty row");
-    }
-    return 0;
-  }
-
-  /**
-   * Simple way to verify the scenario.
-   * There are no KVs with an empty row key in the
-   * table, yet such a KV is presented to the filter.
-   */
-  @Test
-  public void testForEmptyRowKey() throws Exception {
-    byte[] table = Bytes.toBytes("testForEmptyRowKey");
-    byte[] row = Bytes.toBytes("myRow");
-    byte[] cf = Bytes.toBytes("myFamily");
-    byte[] cq = Bytes.toBytes("myColumn");
-    HTableDescriptor desc = new HTableDescriptor(table);
-    desc.addFamily(new HColumnDescriptor(cf));
-    HRegionInfo hri = new HRegionInfo(desc.getName(), null, null);
-    HRegion region = HRegion.createHRegion(hri, FSUtils.getRootDir(UTIL.getConfiguration()), UTIL.getConfiguration(), desc);
-    Put put = new Put(row);
-    put.add(cf, cq, cq);
-    region.put(put);
-    region.flushcache();
-    Scan scan = new Scan();
-    scan.addColumn(cf, cq);
-    WritableByteArrayComparable comparable = new TestFakeKeyInFilter();
-    Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, comparable);
-    scan.setFilter(filter);
-    RegionScanner scanner = region.getScanner(scan);
-    scanner.next(new ArrayList<KeyValue>());
-    scanner.close();
-    region.close();
-  }
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index d3cab721800f..3e62a6bc6dc2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4788,6 +4788,33 @@ public void testGetRegionsInRange() throws Exception {
     regionsList = table.getRegionsInRange(startKey, endKey);
     assertEquals(1, regionsList.size());
   }
+
+  @Test
+  public void testJira6912() throws Exception {
+    byte [] TABLE = Bytes.toBytes("testJira6912");
+    HTable foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
+
+    List<Put> puts = new ArrayList<Put>();
+    for (int i=0;i !=100; i++){
+      Put put = new Put(Bytes.toBytes(i));
+      put.add(FAMILY, FAMILY, Bytes.toBytes(i));
+      puts.add(put);
+    }
+    foo.put(puts);
+    // If i comment this out it works
+    TEST_UTIL.flush();
+
+    Scan scan = new Scan();
+    scan.setStartRow(Bytes.toBytes(1));
+    scan.setStopRow(Bytes.toBytes(3));
+    scan.addColumn(FAMILY, FAMILY);
+    scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));
+
+    ResultScanner scanner = foo.getScanner(scan);
+    Result[] bar = scanner.next(100);
+    assertEquals(1, bar.length);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 41af4834632db26c907d5e6ef94f1eac0de4d220 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 4 Oct 2012 21:17:32 +0000
Subject: [PATCH 0513/1540] HBASE-6916  HBA logs at info level errors that
 won't show in the shell

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394272 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 22 ++++++++-----------
 src/test/ruby/hbase/admin_test.rb             | 11 ----------
 2 files changed, 9 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index b87ad7e3f6e5..71711c134fa5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1088,16 +1088,16 @@ public void closeRegion(final byte [] regionname, final String serverName)
       if (serverName != null) {
         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
         if (pair == null || pair.getFirst() == null) {
-          LOG.info("No region in .META. for " +
-            Bytes.toStringBinary(regionname) + "; pair=" + pair);
+          throw new UnknownRegionException(Bytes.toStringBinary(regionname));
         } else {
           closeRegion(new ServerName(serverName), pair.getFirst());
         }
       } else {
         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
-        if (pair == null || pair.getSecond() == null) {
-          LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(regionname) + "; pair=" + pair);
+        if (pair == null) {
+          throw new UnknownRegionException(Bytes.toStringBinary(regionname));
+        } else if (pair.getSecond() == null) {
+          throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
         } else {
           closeRegion(pair.getSecond(), pair.getFirst());
         }
@@ -1190,8 +1190,7 @@ public void flush(final byte [] tableNameOrRegionName)
         = getRegion(tableNameOrRegionName, ct);
       if (regionServerPair != null) {
         if (regionServerPair.getSecond() == null) {
-          LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
+          throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
         } else {
           flush(regionServerPair.getSecond(), regionServerPair.getFirst());
         }
@@ -1294,8 +1293,7 @@ private void compact(final byte [] tableNameOrRegionName, final boolean major)
         = getRegion(tableNameOrRegionName, ct);
       if (regionServerPair != null) {
         if (regionServerPair.getSecond() == null) {
-          LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
+          throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
         } else {
           compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major);
         }
@@ -1467,8 +1465,7 @@ public void split(final byte [] tableNameOrRegionName,
         = getRegion(tableNameOrRegionName, ct);
       if (regionServerPair != null) {
         if (regionServerPair.getSecond() == null) {
-          LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
+            throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
         } else {
           split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
         }
@@ -1765,8 +1762,7 @@ public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
         = getRegion(tableNameOrRegionName, ct);
       if (regionServerPair != null) {
         if (regionServerPair.getSecond() == null) {
-          LOG.info("No server in .META. for " +
-            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + regionServerPair);
+          throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
         } else {
           ServerName sn = regionServerPair.getSecond();
           HRegionInterface rs =
diff --git a/src/test/ruby/hbase/admin_test.rb b/src/test/ruby/hbase/admin_test.rb
index 0c2672b89f53..97bf4433a2d2 100644
--- a/src/test/ruby/hbase/admin_test.rb
+++ b/src/test/ruby/hbase/admin_test.rb
@@ -160,17 +160,6 @@ def setup
 
     #-------------------------------------------------------------------------------
 
-    define_test "close should work without region server name" do
-      if admin.exists?(@create_test_name)
-        admin.disable(@create_test_name)
-        admin.drop(@create_test_name)
-      end
-      admin.create(@create_test_name, 'foo')
-      admin.close_region(@create_test_name + ',,0', nil)
-    end
-
-    #-------------------------------------------------------------------------------
-
     define_test "describe should fail for non-existent tables" do
       assert_raise(ArgumentError) do
         admin.describe('.NOT.EXISTS.')

From 8fc355c74c2a8ebb5b64afb1ef2ae91526fda6b1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Oct 2012 22:46:53 +0000
Subject: [PATCH 0514/1540] HBASE-6946 JavaDoc missing from release tarballs

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394310 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/pom.xml b/pom.xml
index f0daf8f45b65..ac9c7fba66bb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1491,6 +1491,21 @@
               </execution>
             </executions>
           </plugin>
+          <plugin>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <version>2.6.1</version>
+            <configuration>
+              <docfilessubdirs>true</docfilessubdirs>
+            </configuration>
+            <executions>
+              <execution>
+                <phase>prepare-package</phase>
+                <goals>
+                  <goal>javadoc</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
         </plugins>
       </build>
     </profile>

From 41518b434a1ad2a5f0ffead7c171ccfb2c6b3b91 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 5 Oct 2012 06:32:08 +0000
Subject: [PATCH 0515/1540] HBASE-6900 RegionScanner.reseek() creates NPE when
 a flush or compaction happens before the reseek.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394378 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/StoreScanner.java      |  6 ++--
 .../hbase/regionserver/TestHRegion.java       | 35 +++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index c255d5690dd3..a87d326fd61f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -529,8 +529,10 @@ private void resetScannerStack(KeyValue lastTopKey) throws IOException {
 
   @Override
   public synchronized boolean reseek(KeyValue kv) throws IOException {
-    //Heap cannot be null, because this is only called from next() which
-    //guarantees that heap will never be null before this call.
+    //Heap will not be null, if this is called from next() which.
+    //If called from RegionScanner.reseek(...) make sure the scanner
+    //stack is reset if needed.
+    checkReseek();
     if (explicitColumnQuery && lazySeekEnabledGlobally) {
       return heap.requestSeek(kv, true, useRowColBloom);
     } else {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 21f3ded987de..0ce2e7e49c8c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -90,6 +90,7 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.apache.hadoop.hbase.util.Threads;
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
@@ -202,6 +203,40 @@ public void testCompactionAffectedByScanners() throws Exception {
     System.out.println(results);
     assertEquals(0, results.size());
   }
+  
+  @Test
+  public void testToShowNPEOnRegionScannerReseek() throws Exception{
+    String method = "testToShowNPEOnRegionScannerReseek";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Bytes.toBytes("family");
+    Configuration conf = HBaseConfiguration.create();
+    this.region = initHRegion(tableName, method, conf, family);
+
+    Put put = new Put(Bytes.toBytes("r1"));
+    put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
+    region.put(put);
+    put = new Put(Bytes.toBytes("r2"));
+    put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
+    region.put(put);
+    region.flushcache();
+
+
+    Scan scan = new Scan();
+    scan.setMaxVersions(3);
+    // open the first scanner
+    RegionScanner scanner1 = region.getScanner(scan);
+
+    System.out.println("Smallest read point:" + region.getSmallestReadPoint());
+    
+    region.compactStores(true);
+
+    scanner1.reseek(Bytes.toBytes("r2"));
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    scanner1.next(results);
+    KeyValue keyValue = results.get(0);
+    Assert.assertTrue(Bytes.compareTo(keyValue.getRow(), Bytes.toBytes("r2")) == 0);
+    scanner1.close();
+  }
 
   public void testSkipRecoveredEditsReplay() throws Exception {
     String method = "testSkipRecoveredEditsReplay";

From ba89ec749db70e763d7439d4aa9fd4c150c7f854 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 5 Oct 2012 19:07:54 +0000
Subject: [PATCH 0516/1540] HBASE-6889 Ignore source control files with
 apache-rat (Jesse Yates)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394736 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pom.xml b/pom.xml
index ac9c7fba66bb..2f98d04add88 100644
--- a/pom.xml
+++ b/pom.xml
@@ -533,6 +533,9 @@
 	      <exclude>docs/*</exclude> <!-- auto-gen docs -->
               <!--It don't like freebsd license-->
               <exclude>**/src/site/resources/css/freebsd_docbook.css</exclude>
+              <!--  exclude source control files -->
+              <exclude>.git/**</exclude>
+              <exclude>.svn/**</exclude>
             </excludes>
           </configuration>
 	</plugin>

From b80722ac067383fa837f46693ac9b92faa9755f6 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Fri, 5 Oct 2012 19:55:54 +0000
Subject: [PATCH 0517/1540] HBASE-5582  "No HServerInfo found for" should be a
 WARNING message (Kevin Odell via JD)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394767 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
index 225df66f2a9f..b3b21303f64e 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
@@ -94,7 +94,8 @@ public void nodeDeleted(String path) {
         serverName + "]");
       ServerName sn = ServerName.parseServerName(serverName);
       if (!serverManager.isServerOnline(sn)) {
-        LOG.info(serverName.toString() + " is not online");
+        LOG.warn(serverName.toString() + " is not online or isn't known to the master."+
+         "The latter could be caused by a DNS misconfiguration.");
         return;
       }
       remove(sn);

From 205c9b83acb8c30772c4e0035f8f8ccbd65ab2a9 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Fri, 5 Oct 2012 23:16:35 +0000
Subject: [PATCH 0518/1540] HBASE-6920 On timeout connecting to master, client
 can get stuck and never make progress

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394857 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 29 ++++--
 .../org/apache/hadoop/hbase/ipc/HBaseRPC.java |  2 +-
 .../apache/hadoop/hbase/ipc/RpcEngine.java    |  8 ++
 .../hadoop/hbase/ipc/WritableRpcEngine.java   | 15 +++-
 .../hbase/client/TestClientTimeouts.java      | 89 +++++++++++++++++++
 .../hbase/ipc/RandomTimeoutRpcEngine.java     | 77 ++++++++++++++++
 6 files changed, 206 insertions(+), 14 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 219a0bd7a83e..37f4c1bc761e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -616,6 +616,23 @@ public Configuration getConfiguration() {
       return this.conf;
     }
 
+    /**
+     * Log failure of getMaster attempt
+     * @return true if should retry
+     */
+    private boolean shouldRetryGetMaster(int tries, Exception e) {
+      if (tries == numRetries - 1) {
+        // This was our last chance - don't bother sleeping
+        LOG.info("getMaster attempt " + tries + " of " + numRetries +
+          " failed; no more retrying.", e);
+        return false;
+      }
+      LOG.info("getMaster attempt " + tries + " of " + numRetries +
+        " failed; retrying after sleep of " +
+        ConnectionUtils.getPauseTime(this.pause, tries), e);
+      return true;
+    }
+
     public HMasterInterface getMaster()
     throws MasterNotRunningException, ZooKeeperConnectionException {
       // TODO: REMOVE.  MOVE TO HBaseAdmin and redo as a Callable!!!
@@ -673,15 +690,9 @@ public HMasterInterface getMaster()
             }
 
           } catch (IOException e) {
-            if (tries == numRetries - 1) {
-              // This was our last chance - don't bother sleeping
-              LOG.info("getMaster attempt " + tries + " of " + numRetries +
-                " failed; no more retrying.", e);
-              break;
-            }
-            LOG.info("getMaster attempt " + tries + " of " + numRetries +
-              " failed; retrying after sleep of " +
-              ConnectionUtils.getPauseTime(this.pause, tries), e);
+            if (!shouldRetryGetMaster(tries, e)) break;
+          } catch (UndeclaredThrowableException ute) {
+            if (!shouldRetryGetMaster(tries, ute)) break;
           }
 
           // Cannot connect to master or it is not running. Sleep & retry
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
index 1c45c0a2fea1..64a8671c1e25 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -129,7 +129,7 @@ private static synchronized RpcEngine getProtocolEngine(Class protocol,
   }
 
   // return the RpcEngine that handles a proxy object
-  private static synchronized RpcEngine getProxyEngine(Object proxy) {
+  static synchronized RpcEngine getProxyEngine(Object proxy) {
     return PROXY_ENGINES.get(proxy.getClass());
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
index d48aeaef2c4d..5c49177e4c3b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
+import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -37,6 +38,13 @@ VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
                   User ticket, Configuration conf,
                   SocketFactory factory, int rpcTimeout) throws IOException;
 
+  /** Construct a client-side proxy object, specifying an InvocationHandler for testing purposes */
+  VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+                  long clientVersion, InetSocketAddress addr,
+                  User ticket, Configuration conf,
+                  SocketFactory factory, int rpcTimeout, InvocationHandler handler)
+  throws IOException;
+
   /** Stop this proxy. */
   void stopProxy(VersionedProtocol proxy);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index 386111745042..9b24d27fb8fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -121,7 +121,7 @@ protected void stopClient(HBaseClient client) {
 
   protected final static ClientCache CLIENTS = new ClientCache();
 
-  private static class Invoker implements InvocationHandler {
+  static class Invoker implements InvocationHandler {
     private Class<? extends VersionedProtocol> protocol;
     private InetSocketAddress address;
     private User ticket;
@@ -172,13 +172,12 @@ synchronized protected void close() {
   public VersionedProtocol getProxy(
       Class<? extends VersionedProtocol> protocol, long clientVersion,
       InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout)
+      Configuration conf, SocketFactory factory, int rpcTimeout, InvocationHandler handler)
     throws IOException {
 
       VersionedProtocol proxy =
           (VersionedProtocol) Proxy.newProxyInstance(
-              protocol.getClassLoader(), new Class[] { protocol },
-              new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+              protocol.getClassLoader(), new Class[] { protocol }, handler);
     if (proxy instanceof VersionedProtocol) {
       long serverVersion = ((VersionedProtocol)proxy)
         .getProtocolVersion(protocol.getName(), clientVersion);
@@ -190,6 +189,14 @@ public VersionedProtocol getProxy(
     return proxy;
   }
 
+  public VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+      long clientVersion, InetSocketAddress addr,
+      User ticket, Configuration conf,
+      SocketFactory factory, int rpcTimeout) throws IOException {
+    return getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
+      new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+  }
+
   /**
    * Stop this proxy and release its invoker's resource
    * @param proxy the proxy to be stopped
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java b/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
new file mode 100644
index 000000000000..95114d0e6704
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
@@ -0,0 +1,89 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.RandomTimeoutRpcEngine;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestClientTimeouts {
+  final Log LOG = LogFactory.getLog(getClass());
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  protected static int SLAVES = 1;
+
+ /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    RandomTimeoutRpcEngine.setProtocolEngine(conf, HMasterInterface.class);
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Test that a client that fails an RPC to the master retries properly and
+   * doesn't throw any unexpected exceptions.
+   * @throws Exception
+   */
+  @Test
+  public void testClientGetMasterFailure() throws Exception {
+    long lastLimit = HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT;
+    HConnection lastConnection = null;
+    for (int i = 0; i < 5; ++i) {
+      // Ensure the HBaseAdmin uses a new connection by changing Configuration.
+      Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+      conf.setLong(HConstants.HBASE_CLIENT_PREFETCH_LIMIT, ++lastLimit);
+      try {
+        HBaseAdmin admin = new HBaseAdmin(conf);
+        HConnection connection = admin.getConnection();
+        assertFalse(connection == lastConnection);
+        // Ensure the RandomTimeoutRpcEngine is actually being used.
+        assertTrue(RandomTimeoutRpcEngine.isProxyForObject((admin.getMaster())));
+      } catch (MasterNotRunningException ex) {
+        // Since we are randomly throwing SocketTimeoutExceptions, it is possible to get
+        // a MasterNotRunningException.  We only care about other exceptions -- i.e.
+        // UndeclaredThrowableException.
+      }
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java b/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
new file mode 100644
index 000000000000..ca83e0de5b90
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
@@ -0,0 +1,77 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.Random;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import javax.net.SocketFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.security.User;
+
+/**
+ * RpcEngine that random throws a SocketTimeoutEngine for testing.
+ * Make sure to call setProtocolEngine to have the client actually use the RpcEngine
+ * for a specific protocol
+ */
+public class RandomTimeoutRpcEngine extends WritableRpcEngine {
+
+  private static final Random RANDOM = new Random(System.currentTimeMillis());
+  public static double chanceOfTimeout = 0.3;
+
+  public VersionedProtocol getProxy(
+      Class<? extends VersionedProtocol> protocol, long clientVersion,
+      InetSocketAddress addr, User ticket,
+      Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException {
+    RandomTimeoutInvoker handler =
+      new RandomTimeoutInvoker(protocol, addr, ticket, conf, factory, rpcTimeout);
+    return super.getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout, handler);
+  }
+
+  // Call this in order to set this class to run as the RpcEngine for the given protocol
+  public static void setProtocolEngine(Configuration conf, Class protocol) {
+    HBaseRPC.setProtocolEngine(conf, protocol, RandomTimeoutRpcEngine.class);
+  }
+
+  public static boolean isProxyForObject(Object proxy) {
+    return HBaseRPC.getProxyEngine(proxy).getClass().equals(RandomTimeoutRpcEngine.class);
+  }
+
+  static class RandomTimeoutInvoker extends Invoker {
+
+    public RandomTimeoutInvoker(Class<? extends VersionedProtocol> protocol,
+        InetSocketAddress address, User ticket,
+        Configuration conf, SocketFactory factory, int rpcTimeout) {
+      super(protocol, address, ticket, conf, factory, rpcTimeout);
+    }
+
+    public Object invoke(Object proxy, Method method, Object[] args)
+    throws Throwable {
+      if (RANDOM.nextFloat() < chanceOfTimeout) {
+        throw new SocketTimeoutException("fake timeout");
+      }
+      return super.invoke(proxy, method, args);
+    }
+  }
+}

From 5e55369121b1a262b832c00fd5f44d5542e67d40 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Oct 2012 02:32:43 +0000
Subject: [PATCH 0519/1540] New CHANGES.txt for 0.94.2RC3

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394901 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index d5889973bde2..e86827ab43b6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.2 - 9/25/2012
+Release 0.94.2 - 10/05/2012
 Sub-task
 
     [HBASE-6257] - Avoid unnecessary flush & compact on Meta in admin.rb.
@@ -9,6 +9,7 @@ Sub-task
 
 Bug
 
+    [HBASE-4565] - Maven HBase build broken on cygwin with copynativelib.sh call.
     [HBASE-5292] - getsize per-CF metric incorrectly counts compaction related reads as well
     [HBASE-5549] - Master can fail if ZooKeeper session expires
     [HBASE-5997] - Fix concerns raised in HBASE-5922 related to HalfStoreFileReader
@@ -16,6 +17,7 @@ Bug
     [HBASE-6211] - Put latencies in jmx
     [HBASE-6263] - Use default mode for HBase Thrift gateway if not specified
     [HBASE-6268] - Can't enable a table on a 0.94 cluster from a 0.92 client
+    [HBASE-6299] - RS starting region open while failing ack to HMaster.sendRegionOpen() causes inconsistency in HMaster's region state and a series of successive problems
     [HBASE-6321] - ReplicationSource dies reading the peer's id
     [HBASE-6340] - HBase RPC should allow protocol extension with common interfaces.
     [HBASE-6359] - KeyValue may return incorrect values after readFields()
@@ -41,7 +43,6 @@ Bug
     [HBASE-6537] - Race between balancer and disable table can lead to inconsistent cluster
     [HBASE-6552] - TestAcidGuarantees system test should flush more aggressively
     [HBASE-6561] - Gets/Puts with many columns send the RegionServer into an "endless" loop
-    [HBASE-6562] - Fake KVs are sometimes passed to filters
     [HBASE-6565] - Coprocessor exec result Map is not thread safe
     [HBASE-6576] - HBaseAdmin.createTable should wait until the table is enabled
     [HBASE-6579] - Unnecessary KV order check in StoreScanner
@@ -64,8 +65,10 @@ Bug
     [HBASE-6662] - Region server incorrectly reports its own address as master's address
     [HBASE-6663] - NPE race in HConnection if zookeeper is reset
     [HBASE-6671] - Kerberos authenticated super user should be able to retrieve proxied delegation tokens
+    [HBASE-6679] - RegionServer aborts due to race between compaction and split
     [HBASE-6685] - Thrift DemoClient.pl got NullPointerException
     [HBASE-6686] - HFile Quarantine fails with missing dirs in hadoop 2.0
+    [HBASE-6688] - folder referred by thrift demo app instructions is outdated
     [HBASE-6710] - 0.92/0.94 compatibility issues due to HBASE-5206
     [HBASE-6711] - Avoid local results copy in StoreScanner
     [HBASE-6713] - Stopping META/ROOT RS may take 50mins when some region is splitting
@@ -81,11 +84,24 @@ Bug
     [HBASE-6844] - upgrade 0.23 version dependency in 0.94
     [HBASE-6847] - HBASE-6649 broke replication
     [HBASE-6851] - Race condition in TableAuthManager.updateGlobalCache()
+    [HBASE-6854] - Deletion of SPLITTING node on split rollback should clear the region from RIT
     [HBASE-6868] - Skip checksum is broke; are we double-checksumming by default?
+    [HBASE-6871] - HFileBlockIndex Write Error in HFile V2 due to incorrect split into intermediate index blocks
+    [HBASE-6888] - HBase scripts ignore any HBASE_OPTS set in the environment
+    [HBASE-6889] - Ignore source control files with apache-rat
+    [HBASE-6900] - RegionScanner.reseek() creates NPE when a flush or compaction happens before the reseek.
+    [HBASE-6901] - Store file compactSelection throws ArrayIndexOutOfBoundsException
+    [HBASE-6906] - TestHBaseFsck#testQuarantine* tests are flakey due to TableNotEnabledException
+    [HBASE-6912] - Filters are not properly applied in certain cases
+    [HBASE-6916] - HBA logs at info level errors that won't show in the shell
+    [HBASE-6920] - On timeout connecting to master, client can get stuck and never make progress
+    [HBASE-6927] - WrongFS using HRegionInfo.getTableDesc() and different fs for hbase.root and fs.defaultFS
+    [HBASE-6946] - JavaDoc missing from release tarballs
 
 Improvement
 
     [HBASE-3271] - Allow .META. table to be exported
+    [HBASE-5582] - "No HServerInfo found for" should be a WARNING message
     [HBASE-5631] - hbck should handle case where .tableinfo file is missing.
     [HBASE-5714] - Add write permissions check before any hbck run that modifies hdfs.
     [HBASE-5728] - Methods Missing in HTableInterface
@@ -100,6 +116,7 @@ Improvement
     [HBASE-6643] - Accept encoded region name in compacting/spliting region from shell
     [HBASE-6644] - HBaseAdmin.createTable should wait more till table is enabled.
     [HBASE-6860] - [replication] HBASE-6550 is too aggressive, DDOSes .META.
+    [HBASE-6914] - Scans/Gets/Mutations don't give a good error if the table is disabled.
 
 New Feature
 

From bb54590ab53df252beffec5cfd8e54942429964f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Oct 2012 02:59:37 +0000
Subject: [PATCH 0520/1540] HBASE-6920 Addendum - fix SecureRpcEngine

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1394908 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/SecureRpcEngine.java | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
index 87cbb5117123..58334c310388 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
@@ -196,7 +196,7 @@ synchronized protected void close() {
   public VersionedProtocol getProxy(
       Class<? extends VersionedProtocol> protocol, long clientVersion,
       InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout)
+      Configuration conf, SocketFactory factory, int rpcTimeout, InvocationHandler handler)
   throws IOException {
     if (User.isSecurityEnabled()) {
       HBaseSaslRpcServer.init(conf);
@@ -204,7 +204,7 @@ public VersionedProtocol getProxy(
     VersionedProtocol proxy =
         (VersionedProtocol) Proxy.newProxyInstance(
             protocol.getClassLoader(), new Class[] { protocol },
-            new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+            handler);
     long serverVersion = proxy.getProtocolVersion(protocol.getName(),
                                                   clientVersion);
     if (serverVersion != clientVersion) {
@@ -214,6 +214,14 @@ public VersionedProtocol getProxy(
     return proxy;
   }
 
+  public VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+      long clientVersion, InetSocketAddress addr,
+      User ticket, Configuration conf,
+      SocketFactory factory, int rpcTimeout) throws IOException {
+    return getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
+        new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+  }
+
   /**
    * Stop this proxy and release its invoker's resource
    * @param proxy the proxy to be stopped

From 7bfd09e973627b1dbcd017734f8bc364b6072586 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Oct 2012 15:13:19 +0000
Subject: [PATCH 0521/1540] HBASE-6920 Addendum2. Remove test code which does
 not work for the Secure build

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1395093 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/ipc/SecureRpcEngine.java     | 12 +--
 .../org/apache/hadoop/hbase/ipc/HBaseRPC.java |  2 +-
 .../apache/hadoop/hbase/ipc/RpcEngine.java    |  8 --
 .../hadoop/hbase/ipc/WritableRpcEngine.java   | 15 +---
 .../hbase/client/TestClientTimeouts.java      | 89 -------------------
 .../hbase/ipc/RandomTimeoutRpcEngine.java     | 77 ----------------
 6 files changed, 7 insertions(+), 196 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
index 58334c310388..87cbb5117123 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
@@ -196,7 +196,7 @@ synchronized protected void close() {
   public VersionedProtocol getProxy(
       Class<? extends VersionedProtocol> protocol, long clientVersion,
       InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout, InvocationHandler handler)
+      Configuration conf, SocketFactory factory, int rpcTimeout)
   throws IOException {
     if (User.isSecurityEnabled()) {
       HBaseSaslRpcServer.init(conf);
@@ -204,7 +204,7 @@ public VersionedProtocol getProxy(
     VersionedProtocol proxy =
         (VersionedProtocol) Proxy.newProxyInstance(
             protocol.getClassLoader(), new Class[] { protocol },
-            handler);
+            new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
     long serverVersion = proxy.getProtocolVersion(protocol.getName(),
                                                   clientVersion);
     if (serverVersion != clientVersion) {
@@ -214,14 +214,6 @@ public VersionedProtocol getProxy(
     return proxy;
   }
 
-  public VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
-      long clientVersion, InetSocketAddress addr,
-      User ticket, Configuration conf,
-      SocketFactory factory, int rpcTimeout) throws IOException {
-    return getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
-        new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
-  }
-
   /**
    * Stop this proxy and release its invoker's resource
    * @param proxy the proxy to be stopped
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
index 64a8671c1e25..1c45c0a2fea1 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -129,7 +129,7 @@ private static synchronized RpcEngine getProtocolEngine(Class protocol,
   }
 
   // return the RpcEngine that handles a proxy object
-  static synchronized RpcEngine getProxyEngine(Object proxy) {
+  private static synchronized RpcEngine getProxyEngine(Object proxy) {
     return PROXY_ENGINES.get(proxy.getClass());
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
index 5c49177e4c3b..d48aeaef2c4d 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
@@ -19,7 +19,6 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
-import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -38,13 +37,6 @@ VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
                   User ticket, Configuration conf,
                   SocketFactory factory, int rpcTimeout) throws IOException;
 
-  /** Construct a client-side proxy object, specifying an InvocationHandler for testing purposes */
-  VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
-                  long clientVersion, InetSocketAddress addr,
-                  User ticket, Configuration conf,
-                  SocketFactory factory, int rpcTimeout, InvocationHandler handler)
-  throws IOException;
-
   /** Stop this proxy. */
   void stopProxy(VersionedProtocol proxy);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index 9b24d27fb8fa..386111745042 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -121,7 +121,7 @@ protected void stopClient(HBaseClient client) {
 
   protected final static ClientCache CLIENTS = new ClientCache();
 
-  static class Invoker implements InvocationHandler {
+  private static class Invoker implements InvocationHandler {
     private Class<? extends VersionedProtocol> protocol;
     private InetSocketAddress address;
     private User ticket;
@@ -172,12 +172,13 @@ synchronized protected void close() {
   public VersionedProtocol getProxy(
       Class<? extends VersionedProtocol> protocol, long clientVersion,
       InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout, InvocationHandler handler)
+      Configuration conf, SocketFactory factory, int rpcTimeout)
     throws IOException {
 
       VersionedProtocol proxy =
           (VersionedProtocol) Proxy.newProxyInstance(
-              protocol.getClassLoader(), new Class[] { protocol }, handler);
+              protocol.getClassLoader(), new Class[] { protocol },
+              new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
     if (proxy instanceof VersionedProtocol) {
       long serverVersion = ((VersionedProtocol)proxy)
         .getProtocolVersion(protocol.getName(), clientVersion);
@@ -189,14 +190,6 @@ public VersionedProtocol getProxy(
     return proxy;
   }
 
-  public VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
-      long clientVersion, InetSocketAddress addr,
-      User ticket, Configuration conf,
-      SocketFactory factory, int rpcTimeout) throws IOException {
-    return getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
-      new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
-  }
-
   /**
    * Stop this proxy and release its invoker's resource
    * @param proxy the proxy to be stopped
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java b/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
deleted file mode 100644
index 95114d0e6704..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
-import org.apache.hadoop.hbase.ipc.RandomTimeoutRpcEngine;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(MediumTests.class)
-public class TestClientTimeouts {
-  final Log LOG = LogFactory.getLog(getClass());
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  protected static int SLAVES = 1;
-
- /**
-   * @throws java.lang.Exception
-   */
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    Configuration conf = TEST_UTIL.getConfiguration();
-    RandomTimeoutRpcEngine.setProtocolEngine(conf, HMasterInterface.class);
-    TEST_UTIL.startMiniCluster(SLAVES);
-  }
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  /**
-   * Test that a client that fails an RPC to the master retries properly and
-   * doesn't throw any unexpected exceptions.
-   * @throws Exception
-   */
-  @Test
-  public void testClientGetMasterFailure() throws Exception {
-    long lastLimit = HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT;
-    HConnection lastConnection = null;
-    for (int i = 0; i < 5; ++i) {
-      // Ensure the HBaseAdmin uses a new connection by changing Configuration.
-      Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
-      conf.setLong(HConstants.HBASE_CLIENT_PREFETCH_LIMIT, ++lastLimit);
-      try {
-        HBaseAdmin admin = new HBaseAdmin(conf);
-        HConnection connection = admin.getConnection();
-        assertFalse(connection == lastConnection);
-        // Ensure the RandomTimeoutRpcEngine is actually being used.
-        assertTrue(RandomTimeoutRpcEngine.isProxyForObject((admin.getMaster())));
-      } catch (MasterNotRunningException ex) {
-        // Since we are randomly throwing SocketTimeoutExceptions, it is possible to get
-        // a MasterNotRunningException.  We only care about other exceptions -- i.e.
-        // UndeclaredThrowableException.
-      }
-    }
-  }
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java b/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
deleted file mode 100644
index ca83e0de5b90..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.ipc;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.net.SocketTimeoutException;
-import java.util.Random;
-import org.apache.hadoop.hbase.ipc.VersionedProtocol;
-import javax.net.SocketFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.security.User;
-
-/**
- * RpcEngine that random throws a SocketTimeoutEngine for testing.
- * Make sure to call setProtocolEngine to have the client actually use the RpcEngine
- * for a specific protocol
- */
-public class RandomTimeoutRpcEngine extends WritableRpcEngine {
-
-  private static final Random RANDOM = new Random(System.currentTimeMillis());
-  public static double chanceOfTimeout = 0.3;
-
-  public VersionedProtocol getProxy(
-      Class<? extends VersionedProtocol> protocol, long clientVersion,
-      InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException {
-    RandomTimeoutInvoker handler =
-      new RandomTimeoutInvoker(protocol, addr, ticket, conf, factory, rpcTimeout);
-    return super.getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout, handler);
-  }
-
-  // Call this in order to set this class to run as the RpcEngine for the given protocol
-  public static void setProtocolEngine(Configuration conf, Class protocol) {
-    HBaseRPC.setProtocolEngine(conf, protocol, RandomTimeoutRpcEngine.class);
-  }
-
-  public static boolean isProxyForObject(Object proxy) {
-    return HBaseRPC.getProxyEngine(proxy).getClass().equals(RandomTimeoutRpcEngine.class);
-  }
-
-  static class RandomTimeoutInvoker extends Invoker {
-
-    public RandomTimeoutInvoker(Class<? extends VersionedProtocol> protocol,
-        InetSocketAddress address, User ticket,
-        Configuration conf, SocketFactory factory, int rpcTimeout) {
-      super(protocol, address, ticket, conf, factory, rpcTimeout);
-    }
-
-    public Object invoke(Object proxy, Method method, Object[] args)
-    throws Throwable {
-      if (RANDOM.nextFloat() < chanceOfTimeout) {
-        throw new SocketTimeoutException("fake timeout");
-      }
-      return super.invoke(proxy, method, args);
-    }
-  }
-}

From 170a710d4c16a03687fe03096437af9bad886439 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 7 Oct 2012 19:08:52 +0000
Subject: [PATCH 0522/1540] New CHANGES.txt for 0.94.2RC3 - add jira that
 committed but not tagged as such

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1395367 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index e86827ab43b6..2c08801097fe 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.2 - 10/05/2012
+Release 0.94.2 - 10/08/2012
 Sub-task
 
     [HBASE-6257] - Avoid unnecessary flush & compact on Meta in admin.rb.
@@ -84,6 +84,7 @@ Bug
     [HBASE-6844] - upgrade 0.23 version dependency in 0.94
     [HBASE-6847] - HBASE-6649 broke replication
     [HBASE-6851] - Race condition in TableAuthManager.updateGlobalCache()
+    [HBASE-6853] - IllegalArgument Exception is thrown when an empty region is spliitted.
     [HBASE-6854] - Deletion of SPLITTING node on split rollback should clear the region from RIT
     [HBASE-6868] - Skip checksum is broke; are we double-checksumming by default?
     [HBASE-6871] - HFileBlockIndex Write Error in HFile V2 due to incorrect split into intermediate index blocks

From 5d12d184024bfa77fd979848a50fa5309beee041 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 10 Oct 2012 17:52:09 +0000
Subject: [PATCH 0523/1540] HBASE-6853 IllegalArgument Exception is thrown when
 an empty region is spliitted(Ram) : Addendum for testcase failure

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1396708 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestSplitTransactionOnCluster.java    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 070e6d47469a..fc124222b341 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -643,7 +643,7 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
     }
   }
   
-  @Test
+  @Test(timeout = 15000)
   public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfullyExecuteRollback()
       throws Exception {
     final byte[] tableName = Bytes
@@ -654,6 +654,9 @@ public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfull
       HTableDescriptor htd = new HTableDescriptor(tableName);
       htd.addFamily(new HColumnDescriptor("cf"));
       admin.createTable(htd);
+      while (!(cluster.getRegions(tableName).size() == 1)) {
+        Thread.sleep(100);
+      }
       List<HRegion> regions = cluster.getRegions(tableName);
       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
       int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);

From 5f4bc1d05c37bfac8ece4d883ee84a8e58b567f6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 12 Oct 2012 05:17:53 +0000
Subject: [PATCH 0524/1540] HBASE-6978 Minor typo in ReplicationSource
 SocketTimeoutException error handling

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1397442 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 7710b47c71a3..b1c54c3d90d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -647,7 +647,7 @@ protected void shipEdits() {
             // This exception means we waited for more than 60s and nothing
             // happened, the cluster is alive and calling it right away
             // even for a test just makes things worse.
-            sleepForRetries("Encountered a SocketTimeoutException. Since the" +
+            sleepForRetries("Encountered a SocketTimeoutException. Since the " +
               "call to the remote cluster timed out, which is usually " +
               "caused by a machine failure or a massive slowdown",
               this.socketTimeoutMultiplier);

From f84d63bf308d66a45cb2fa312fbb884a5690c080 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 13 Oct 2012 04:38:34 +0000
Subject: [PATCH 0525/1540] Move the version on past 0.94.2 not its released to
 0.94.3-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1397768 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/pom.xml b/pom.xml
index 2f98d04add88..ae9c2ee99766 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.2</version>
+  <version>0.94.3-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need
@@ -331,7 +331,7 @@
               profile on the command line as follows:
 
               $ ~/bin/mvn/bin/mvn  -Papache-release release:perform
-              
+
               I've also been kiling the release:prepare step mid-way to check the
               release.properties it generates at the top-level.  Sometimes it refers
               to HEAD rather than to the svn branch.
@@ -343,7 +343,7 @@
              -->
             <releaseProfiles>apache-release</releaseProfiles>
             <!--This stops our running tests for each stage of maven release.
-                But it builds the test jar.  From SUREFIRE-172.      
+                But it builds the test jar.  From SUREFIRE-172.
               -->
             <arguments>-Dmaven.test.skip.exec</arguments>
           </configuration>
@@ -453,9 +453,9 @@
           <artifactId>build-helper-maven-plugin</artifactId>
           <version>1.5</version>
         </plugin>
-        <!--This plugin's configuration is used to store Eclipse m2e settings only. It 
-          has no influence on the Maven build itself. m2e does not provide any safeguards against 
-          rogue maven plugins that leak classloaders, modify random files inside workspace 
+        <!--This plugin's configuration is used to store Eclipse m2e settings only. It
+          has no influence on the Maven build itself. m2e does not provide any safeguards against
+          rogue maven plugins that leak classloaders, modify random files inside workspace
           or throw nasty exceptions to fail the build -->
         <plugin>
           <groupId>org.eclipse.m2e</groupId>
@@ -477,7 +477,7 @@
                   <action>
                     <execute></execute>
                   </action>
-                </pluginExecution> 
+                </pluginExecution>
                 <pluginExecution>
                   <pluginExecutionFilter>
                     <groupId>org.apache.maven.plugins</groupId>
@@ -510,7 +510,7 @@
         </plugin>
 	<plugin> <!-- in plugin management section so excludes can be inherited -->
 	  <groupId>org.apache.rat</groupId>
-          <artifactId>apache-rat-plugin</artifactId>	  
+          <artifactId>apache-rat-plugin</artifactId>
           <version>0.8</version>
           <configuration>
             <excludes>
@@ -526,7 +526,7 @@
               <exclude>**/generated/**</exclude>
               <exclude>**/conf/*</exclude>
               <exclude>**/*.avpr</exclude>
-              <exclude>**/*.svg</exclude> <!-- vector graphics --> 
+              <exclude>**/*.svg</exclude> <!-- vector graphics -->
               <exclude>**/*.vm</exclude> <!-- apache doxia generated -->
               <exclude>**/control</exclude>
               <exclude>**/conffile</exclude>
@@ -896,7 +896,7 @@
 					else
 						BUILD_DIR=`cygpath --unix '${project.build.directory}'`
 					fi
-					
+
 					cd $BUILD_DIR/${project.build.finalName}
 					tar czf $BUILD_DIR/${project.build.finalName}.tar.gz ${project.build.finalName}
 				</echo>

From 629c6426e421ac70fb1addce38c6dbd7852c4718 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 16 Oct 2012 05:12:12 +0000
Subject: [PATCH 0526/1540] HBASE-6996 HRegion.mutateRowsWithLocks should call
 checkResources/checkReadOnl

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1398645 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 7204a8b5619a..5523634bc073 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4230,6 +4230,9 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
       Collection<byte[]> rowsToLock) throws IOException {
     boolean flush = false;
 
+    checkReadOnly();
+    checkResources();
+
     startRegionOperation();
     List<Integer> acquiredLocks = null;
     try {

From c84c2eea946dae919668b0d9040e80e4683a80c1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Oct 2012 04:16:56 +0000
Subject: [PATCH 0527/1540] HBASE-6032 Port HFileBlockIndex improvement from
 HBASE-5987 (Liyin, Ted, Stack)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1399513 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  21 ++--
 .../hbase/io/hfile/HFileBlockIndex.java       | 119 +++++++++++++++---
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  58 ++++++---
 .../apache/hadoop/hbase/HBaseTestCase.java    |  33 ++++-
 .../hbase/io/hfile/TestHFileBlockIndex.java   |   4 +-
 .../hadoop/hbase/io/hfile/TestReseekTo.java   |   4 +-
 .../hadoop/hbase/io/hfile/TestSeekTo.java     |  83 ++++++++++++
 7 files changed, 273 insertions(+), 49 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 5f4a6c381dd8..9e7001b5bfbc 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -79,7 +79,7 @@ public enum OperationStatusCode {
   /** Cluster is fully-distributed */
   public static final boolean CLUSTER_IS_DISTRIBUTED = true;
 
-  /** Default value for cluster distributed mode */  
+  /** Default value for cluster distributed mode */
   public static final boolean DEFAULT_CLUSTER_DISTRIBUTED = CLUSTER_IS_LOCAL;
 
   /** default host address */
@@ -187,13 +187,13 @@ public enum OperationStatusCode {
 
   /** Default value for thread wake frequency */
   public static final int DEFAULT_THREAD_WAKE_FREQUENCY = 10 * 1000;
-  
+
   /** Parameter name for how often we should try to write a version file, before failing */
   public static final String VERSION_FILE_WRITE_ATTEMPTS = "hbase.server.versionfile.writeattempts";
 
   /** Parameter name for how often we should try to write a version file, before failing */
   public static final int DEFAULT_VERSION_FILE_WRITE_ATTEMPTS = 3;
-  
+
   /** Parameter name for how often a region should should perform a major compaction */
   public static final String MAJOR_COMPACTION_PERIOD = "hbase.hregion.majorcompaction";
 
@@ -618,10 +618,10 @@ public static enum Modify {
   /** File permission umask to use when creating hbase data files */
   public static final String DATA_FILE_UMASK_KEY = "hbase.data.umask";
 
-  /** 
+  /**
    * If this parameter is set to true, then hbase will read
-   * data and then verify checksums. Checksum verification 
-   * inside hdfs will be switched off.  However, if the hbase-checksum 
+   * data and then verify checksums. Checksum verification
+   * inside hdfs will be switched off.  However, if the hbase-checksum
    * verification fails, then it will switch back to using
    * hdfs checksums for verifiying data that is being read from storage.
    *
@@ -629,7 +629,7 @@ public static enum Modify {
    * verify any checksums, instead it will depend on checksum verification
    * being done in the hdfs client.
    */
-  public static final String HBASE_CHECKSUM_VERIFICATION = 
+  public static final String HBASE_CHECKSUM_VERIFICATION =
       "hbase.regionserver.checksum.verify";
 
   /**
@@ -661,7 +661,12 @@ public static enum Modify {
   public static final int HIGH_QOS = 100;
   public static final int REPLICATION_QOS = 5; // normal_QOS < replication_QOS < high_QOS
 
-  
+  /**
+   * The byte array represents for NO_NEXT_INDEXED_KEY;
+   * The actual value is irrelevant because this is always compared by reference.
+   */
+  public static final byte [] NO_NEXT_INDEXED_KEY = Bytes.toBytes("NO_NEXT_INDEXED_KEY");
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index c4f671405eea..d655d3d8e700 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -36,6 +36,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -176,16 +177,56 @@ public HFileBlock seekToDataBlock(final byte[] key, int keyOffset,
         int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
         boolean pread, boolean isCompaction)
         throws IOException {
+      BlockWithScanInfo blockWithScanInfo = loadDataBlockWithScanInfo(key, keyOffset, keyLength,
+          currentBlock, cacheBlocks, pread, isCompaction);
+      if (blockWithScanInfo == null) {
+        return null;
+      } else {
+        return blockWithScanInfo.getHFileBlock();
+      }
+    }
+
+    /**
+     * Return the BlockWithScanInfo which contains the DataBlock with other scan info
+     * such as nextIndexedKey.
+     * This function will only be called when the HFile version is larger than 1.
+     *
+     * @param key the key we are looking for
+     * @param keyOffset the offset of the key in its byte array
+     * @param keyLength the length of the key
+     * @param currentBlock the current block, to avoid re-reading the same
+     *          block
+     * @param cacheBlocks
+     * @param pread
+     * @param isCompaction
+     * @return the BlockWithScanInfo which contains the DataBlock with other scan info
+     *         such as nextIndexedKey.
+     * @throws IOException
+     */
+    public BlockWithScanInfo loadDataBlockWithScanInfo(final byte[] key, int keyOffset,
+        int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
+        boolean pread, boolean isCompaction)
+        throws IOException {
       int rootLevelIndex = rootBlockContainingKey(key, keyOffset, keyLength);
       if (rootLevelIndex < 0 || rootLevelIndex >= blockOffsets.length) {
         return null;
       }
 
+      // the next indexed key
+      byte[] nextIndexedKey = null;
+
       // Read the next-level (intermediate or leaf) index block.
       long currentOffset = blockOffsets[rootLevelIndex];
       int currentOnDiskSize = blockDataSizes[rootLevelIndex];
 
+      if (rootLevelIndex < blockKeys.length - 1) {
+        nextIndexedKey = blockKeys[rootLevelIndex + 1];
+      } else {
+        nextIndexedKey = HConstants.NO_NEXT_INDEXED_KEY;
+      }
+
       int lookupLevel = 1; // How many levels deep we are in our lookup.
+      int index = -1;
 
       HFileBlock block;
       while (true) {
@@ -236,8 +277,8 @@ public HFileBlock seekToDataBlock(final byte[] key, int keyOffset,
         // Locate the entry corresponding to the given key in the non-root
         // (leaf or intermediate-level) index block.
         ByteBuffer buffer = block.getBufferWithoutHeader();
-        if (!locateNonRootIndexEntry(buffer, key, keyOffset, keyLength,
-            comparator)) {
+        index = locateNonRootIndexEntry(buffer, key, keyOffset, keyLength, comparator);
+        if (index == -1) {
           throw new IOException("The key "
               + Bytes.toStringBinary(key, keyOffset, keyLength)
               + " is before the" + " first key of the non-root index block "
@@ -246,6 +287,12 @@ public HFileBlock seekToDataBlock(final byte[] key, int keyOffset,
 
         currentOffset = buffer.getLong();
         currentOnDiskSize = buffer.getInt();
+
+        // Only update next indexed key if there is a next indexed key in the current level
+        byte[] tmpNextIndexedKey = getNonRootIndexedKey(buffer, index + 1);
+        if (tmpNextIndexedKey != null) {
+          nextIndexedKey = tmpNextIndexedKey;
+        }
       }
 
       if (lookupLevel != searchTreeLevel) {
@@ -253,7 +300,9 @@ public HFileBlock seekToDataBlock(final byte[] key, int keyOffset,
             " but the number of levels is " + searchTreeLevel);
       }
 
-      return block;
+      // set the next indexed key for the current block.
+      BlockWithScanInfo blockWithScanInfo = new BlockWithScanInfo(block, nextIndexedKey);
+      return blockWithScanInfo;
     }
 
     /**
@@ -378,6 +427,41 @@ private void add(final byte[] key, final long offset, final int dataSize) {
       rootByteSize += SECONDARY_INDEX_ENTRY_OVERHEAD + key.length;
     }
 
+    /**
+     * The indexed key at the ith position in the nonRootIndex. The position starts at 0.
+     * @param nonRootIndex
+     * @param i the ith position
+     * @return The indexed key at the ith position in the nonRootIndex.
+     */
+    private byte[] getNonRootIndexedKey(ByteBuffer nonRootIndex, int i) {
+      int numEntries = nonRootIndex.getInt(0);
+      if (i < 0 || i >= numEntries) {
+        return null;
+      }
+
+      // Entries start after the number of entries and the secondary index.
+      // The secondary index takes numEntries + 1 ints.
+      int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+      // Targetkey's offset relative to the end of secondary index
+      int targetKeyRelOffset = nonRootIndex.getInt(
+          Bytes.SIZEOF_INT * (i + 1));
+
+      // The offset of the target key in the blockIndex buffer
+      int targetKeyOffset = entriesOffset     // Skip secondary index
+          + targetKeyRelOffset               // Skip all entries until mid
+          + SECONDARY_INDEX_ENTRY_OVERHEAD;  // Skip offset and on-disk-size
+
+      // We subtract the two consecutive secondary index elements, which
+      // gives us the size of the whole (offset, onDiskSize, key) tuple. We
+      // then need to subtract the overhead of offset and onDiskSize.
+      int targetKeyLength = nonRootIndex.getInt(Bytes.SIZEOF_INT * (i + 2)) -
+        targetKeyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD;
+
+      int from = nonRootIndex.arrayOffset() + targetKeyOffset;
+      int to = from + targetKeyLength;
+      return Arrays.copyOfRange(nonRootIndex.array(), from, to);
+    }
+
     /**
      * Performs a binary search over a non-root level index block. Utilizes the
      * secondary index, which records the offsets of (offset, onDiskSize,
@@ -478,31 +562,30 @@ else if (cmp < 0)
      * @param key the byte array containing the key
      * @param keyOffset the offset of the key in its byte array
      * @param keyLength the length of the key
-     * @return true in the case the index entry containing the given key was
-     *         found, false in the case the given key is before the first key
+     * @return the index position where the given key was found,
+     *         otherwise return -1 in the case the given key is before the first key.
      *
      */
-    static boolean locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key,
+    static int locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key,
         int keyOffset, int keyLength, RawComparator<byte[]> comparator) {
       int entryIndex = binarySearchNonRootIndex(key, keyOffset, keyLength,
           nonRootBlock, comparator);
 
-      if (entryIndex == -1) {
-        return false;
-      }
+      if (entryIndex != -1) {
+        int numEntries = nonRootBlock.getInt(0);
 
-      int numEntries = nonRootBlock.getInt(0);
+        // The end of secondary index and the beginning of entries themselves.
+        int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
 
-      // The end of secondary index and the beginning of entries themselves.
-      int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+        // The offset of the entry we are interested in relative to the end of
+        // the secondary index.
+        int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT
+            * (1 + entryIndex));
 
-      // The offset of the entry we are interested in relative to the end of
-      // the secondary index.
-      int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT
-          * (1 + entryIndex));
+        nonRootBlock.position(entriesOffset + entryRelOffset);
+      }
 
-      nonRootBlock.position(entriesOffset + entryRelOffset);
-      return true;
+      return entryIndex;
     }
 
     /**
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 3be58d326248..757990f845f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -29,6 +29,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
@@ -429,6 +430,15 @@ protected abstract static class AbstractScannerV2
       extends AbstractHFileReader.Scanner {
     protected HFileBlock block;
 
+    /**
+     * The next indexed key is to keep track of the indexed key of the next data block.
+     * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
+     * current data block is the last data block.
+     *
+     * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
+     */
+    protected byte[] nextIndexedKey;
+
     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
         final boolean pread, final boolean isCompaction) {
       super(r, cacheBlocks, pread, isCompaction);
@@ -452,19 +462,20 @@ protected int seekTo(byte[] key, int offset, int length, boolean rewind)
         throws IOException {
       HFileBlockIndex.BlockIndexReader indexReader =
           reader.getDataBlockIndexReader();
-      HFileBlock seekToBlock = indexReader.seekToDataBlock(key, offset, length,
-          block, cacheBlocks, pread, isCompaction);
-      if (seekToBlock == null) {
+      BlockWithScanInfo blockWithScanInfo =
+        indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
+            cacheBlocks, pread, isCompaction);
+      if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
         // This happens if the key e.g. falls before the beginning of the file.
         return -1;
       }
-      return loadBlockAndSeekToKey(seekToBlock, rewind, key, offset, length,
-          false);
+      return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
+          blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
     }
 
     protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
 
-    protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock,
+    protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
         boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
         throws IOException;
 
@@ -477,17 +488,28 @@ public int seekTo(byte[] key, int offset, int length) throws IOException {
 
     @Override
     public int reseekTo(byte[] key, int offset, int length) throws IOException {
+      int compared;
       if (isSeeked()) {
         ByteBuffer bb = getKey();
-        int compared = reader.getComparator().compare(key, offset,
+        compared = reader.getComparator().compare(key, offset,
             length, bb.array(), bb.arrayOffset(), bb.limit());
         if (compared < 1) {
           // If the required key is less than or equal to current key, then
           // don't do anything.
           return compared;
+        } else {
+          if (this.nextIndexedKey != null &&
+              (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
+               reader.getComparator().compare(key, offset, length,
+                   nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
+            // The reader shall continue to scan the current data block instead of querying the
+            // block index as long as it knows the target key is strictly smaller than
+            // the next indexed key or the current data block is the last data block.
+            return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
+                false, key, offset, length, false);
+          }
         }
       }
-
       // Don't rewind on a reseek operation, because reseek implies that we are
       // always going forward in the file.
       return seekTo(key, offset, length, false);
@@ -503,6 +525,7 @@ public boolean seekBefore(byte[] key, int offset, int length)
         return false;
       }
       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
+
       if (reader.getComparator().compare(firstKey.array(),
           firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0)
       {
@@ -519,11 +542,11 @@ public boolean seekBefore(byte[] key, int offset, int length)
         seekToBlock = reader.readBlock(previousBlockOffset,
             seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
             pread, isCompaction, BlockType.DATA);
-
         // TODO shortcut: seek forward in this block to the last key of the
         // block.
       }
-      loadBlockAndSeekToKey(seekToBlock, true, key, offset, length, true);
+      byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
+      loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
       return true;
     }
 
@@ -701,14 +724,17 @@ public boolean seekTo() throws IOException {
     }
 
     @Override
-    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
-        byte[] key, int offset, int length, boolean seekBefore)
+    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+        boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
         throws IOException {
       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
         updateCurrBlock(seekToBlock);
       } else if (rewind) {
         blockBuffer.rewind();
       }
+
+      // Update the nextIndexedKey
+      this.nextIndexedKey = nextIndexedKey;
       return blockSeek(key, offset, length, seekBefore);
     }
 
@@ -733,6 +759,9 @@ private void updateCurrBlock(HFileBlock newBlock) {
       blockBuffer = block.getBufferWithoutHeader();
       readKeyValueLen();
       blockFetches++;
+
+      // Reset the next indexed key
+      this.nextIndexedKey = null;
     }
 
     private final void readKeyValueLen() {
@@ -1018,14 +1047,15 @@ protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
     }
 
     @Override
-    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
-        byte[] key, int offset, int length, boolean seekBefore)
+    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+        boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
         throws IOException  {
       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
         updateCurrentBlock(seekToBlock);
       } else if (rewind) {
         seeker.rewind();
       }
+      this.nextIndexedKey = nextIndexedKey;
       return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index 4a57e4da8255..b4b5728a598b 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -71,7 +71,7 @@ public abstract class HBaseTestCase extends TestCase {
   protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
 
   private boolean localfs = false;
-  protected Path testDir = null;
+  protected static Path testDir = null;
   protected FileSystem fs = null;
   protected HRegion root = null;
   protected HRegion meta = null;
@@ -180,8 +180,14 @@ protected Path getUnitTestdir(String testName) {
    * @return An {@link HRegion}
    * @throws IOException
    */
-  protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
+  public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
       byte [] endKey)
+  throws IOException {
+    return createNewHRegion(desc, startKey, endKey, this.conf);
+  }
+
+  public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
+      byte [] endKey, Configuration conf)
   throws IOException {
     FileSystem filesystem = FileSystem.get(conf);
     HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
@@ -248,10 +254,11 @@ protected HTableDescriptor createTableDescriptor(final String name,
    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
    * @param r
    * @param columnFamily
+   * @param column
    * @throws IOException
    * @return count of what we added.
    */
-  protected static long addContent(final HRegion r, final byte [] columnFamily)
+  public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
   throws IOException {
     byte [] startKey = r.getRegionInfo().getStartKey();
     byte [] endKey = r.getRegionInfo().getEndKey();
@@ -259,10 +266,24 @@ protected static long addContent(final HRegion r, final byte [] columnFamily)
     if (startKeyBytes == null || startKeyBytes.length == 0) {
       startKeyBytes = START_KEY_BYTES;
     }
-    return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
+    return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
       startKeyBytes, endKey, -1);
   }
 
+  /**
+   * Add content to region <code>r</code> on the passed column
+   * <code>column</code>.
+   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
+   * @param r
+   * @param columnFamily
+   * @throws IOException
+   * @return count of what we added.
+   */
+  protected static long addContent(final HRegion r, final byte [] columnFamily)
+  throws IOException {
+    return addContent(r, columnFamily, null);
+  }
+
   /**
    * Add content to region <code>r</code> on the passed column
    * <code>column</code>.
@@ -273,12 +294,12 @@ protected static long addContent(final HRegion r, final byte [] columnFamily)
    * @return count of what we added.
    */
   protected static long addContent(final Incommon updater,
-                                   final String columnFamily) throws IOException {
+      final String columnFamily) throws IOException {
     return addContent(updater, columnFamily, START_KEY_BYTES, null);
   }
 
   protected static long addContent(final Incommon updater, final String family,
-                                   final String column) throws IOException {
+      final String column) throws IOException {
     return addContent(updater, family, column, START_KEY_BYTES, null);
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 4f007553859e..8cbbc2397493 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -380,8 +380,8 @@ public void testSecondaryIndexBinarySearch() throws IOException {
       // Now test we can get the offset and the on-disk-size using a
       // higher-level API function.s
       boolean locateBlockResult =
-        BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey,
-            searchKey.length / 2, searchKey.length, Bytes.BYTES_RAWCOMPARATOR);
+        (BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey,
+            searchKey.length / 2, searchKey.length, Bytes.BYTES_RAWCOMPARATOR) != -1);
 
       if (i == 0) {
         assertFalse(locateBlockResult);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
index 6430f328c98a..6c859fdb1dc4 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
@@ -22,6 +22,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import junit.framework.Assert;
+
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -87,7 +89,7 @@ public void testReseekTo() throws Exception {
       String value = valueList.get(i);
       long start = System.nanoTime();
       scanner.reseekTo(Bytes.toBytes(key));
-      assertEquals(value, scanner.getValueString());
+      assertEquals("i is " + i, value, scanner.getValueString());
     }
 
     reader.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
index bb7d75a8f016..1a2ce15d9ee6 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
@@ -98,6 +98,89 @@ public void testSeekBefore() throws Exception {
     reader.close();
   }
 
+  public void testSeekBeforeWithReSeekTo() throws Exception {
+    Path p = makeNewFile();
+    HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf));
+    reader.loadFileInfo();
+    HFileScanner scanner = reader.getScanner(false, true);
+    assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
+    assertEquals(false, scanner.seekBefore(toKV("b").getKey()));
+    assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
+
+    // seekBefore d, so the scanner points to c
+    assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
+    assertEquals("c", toRowStr(scanner.getKeyValue()));
+    // reseekTo e and g
+    assertEquals(0, scanner.reseekTo(toKV("c").getKey()));
+    assertEquals("c", toRowStr(scanner.getKeyValue()));
+    assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore e, so the scanner points to c
+    assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
+    assertEquals("c", toRowStr(scanner.getKeyValue()));
+    // reseekTo e and g
+    assertEquals(0, scanner.reseekTo(toKV("e").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+    assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore f, so the scanner points to e
+    assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+    // reseekTo e and g
+    assertEquals(0, scanner.reseekTo(toKV("e").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+    assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore g, so the scanner points to e
+    assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+    // reseekTo e and g again
+    assertEquals(0, scanner.reseekTo(toKV("e").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+    assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore h, so the scanner points to g
+    assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+    // reseekTo g
+    assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore i, so the scanner points to g
+    assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+    // reseekTo g
+    assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore j, so the scanner points to i
+    assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
+    assertEquals("i", toRowStr(scanner.getKeyValue()));
+    // reseekTo i
+    assertEquals(0, scanner.reseekTo(toKV("i").getKey()));
+    assertEquals("i", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore k, so the scanner points to i
+    assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
+    assertEquals("i", toRowStr(scanner.getKeyValue()));
+    // reseekTo i and k
+    assertEquals(0, scanner.reseekTo(toKV("i").getKey()));
+    assertEquals("i", toRowStr(scanner.getKeyValue()));
+    assertEquals(0, scanner.reseekTo(toKV("k").getKey()));
+    assertEquals("k", toRowStr(scanner.getKeyValue()));
+
+    // seekBefore l, so the scanner points to k
+    assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
+    assertEquals("k", toRowStr(scanner.getKeyValue()));
+    // reseekTo k
+    assertEquals(0, scanner.reseekTo(toKV("k").getKey()));
+    assertEquals("k", toRowStr(scanner.getKeyValue()));
+  }
+
   public void testSeekTo() throws Exception {
     Path p = makeNewFile();
     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf));

From 931c42789e0d761d33416a2d72db73686758d68b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Oct 2012 05:36:10 +0000
Subject: [PATCH 0528/1540] HBASE-6032 Addendum, forgot to add two files

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1399521 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/hfile/BlockWithScanInfo.java     |  44 +++++++
 .../hbase/regionserver/TestBlocksScanned.java | 117 ++++++++++++++++++
 2 files changed, 161 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
new file mode 100644
index 000000000000..ceb05e359368
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+/**
+ * BlockWithScanInfo is wrapper class for HFileBlock with other attributes. These attributes are
+ * supposed to be much cheaper to be maintained in each caller thread than in HFileBlock itself.
+ */
+public class BlockWithScanInfo {
+  private final HFileBlock hFileBlock;
+  /**
+   * The first key in the next block following this one in the HFile.
+   * If this key is unknown, this is reference-equal with HConstants.NO_NEXT_INDEXED_KEY
+   */
+  private final byte[] nextIndexedKey;
+
+  public BlockWithScanInfo(HFileBlock hFileBlock, byte[] nextIndexedKey) {
+    this.hFileBlock = hFileBlock;
+    this.nextIndexedKey = nextIndexedKey;
+  }
+
+  public HFileBlock getHFileBlock() {
+    return hFileBlock;
+  }
+
+  public byte[] getNextIndexedKey() {
+    return nextIndexedKey;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
new file mode 100644
index 000000000000..19c690f6c560
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@SuppressWarnings("deprecation")
+@Category(SmallTests.class)
+public class TestBlocksScanned extends HBaseTestCase {
+  private static byte [] TABLE = Bytes.toBytes("TestBlocksScanned");
+  private static byte [] FAMILY = Bytes.toBytes("family");
+  private static byte [] COL = Bytes.toBytes("col");
+  private static byte [] START_KEY = Bytes.toBytes("aaa");
+  private static byte [] END_KEY = Bytes.toBytes("zzz");
+  private static int BLOCK_SIZE = 70;
+
+  private static HBaseTestingUtility TEST_UTIL = null;
+  private static HTableDescriptor TESTTABLEDESC = null;
+
+   @Override
+   public void setUp() throws Exception {
+     super.setUp();
+     SchemaMetrics.setUseTableNameInTest(true);
+     TEST_UTIL = new HBaseTestingUtility();
+     TESTTABLEDESC = new HTableDescriptor(TABLE);
+
+     TESTTABLEDESC.addFamily(
+         new HColumnDescriptor(FAMILY)
+         .setMaxVersions(10)
+         .setBlockCacheEnabled(true)
+         .setBlocksize(BLOCK_SIZE)
+         .setCompressionType(Compression.Algorithm.NONE)
+     );
+   }
+
+   @Test
+  public void testBlocksScanned() throws Exception {
+    HRegion r = createNewHRegion(TESTTABLEDESC, START_KEY, END_KEY,
+        TEST_UTIL.getConfiguration());
+    addContent(r, FAMILY, COL);
+    r.flushcache();
+
+    // Get the per-cf metrics
+    SchemaMetrics schemaMetrics =
+      SchemaMetrics.getInstance(Bytes.toString(TABLE), Bytes.toString(FAMILY));
+    Map<String, Long> schemaMetricSnapshot = SchemaMetrics.getMetricsSnapshot();
+
+    // Do simple test of getting one row only first.
+    Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz"));
+    scan.addColumn(FAMILY, COL);
+    scan.setMaxVersions(1);
+
+    InternalScanner s = r.getScanner(scan);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    while (s.next(results));
+    s.close();
+
+    int expectResultSize = 'z' - 'a';
+    Assert.assertEquals(expectResultSize, results.size());
+
+    int kvPerBlock = (int) Math.ceil(BLOCK_SIZE / (double) results.get(0).getLength());
+    Assert.assertEquals(2, kvPerBlock);
+
+    long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock);
+    long expectIndexBlockRead = expectDataBlockRead;
+
+    verifyDataAndIndexBlockRead(schemaMetricSnapshot, schemaMetrics,
+        expectDataBlockRead, expectIndexBlockRead);
+  }
+
+  private void verifyDataAndIndexBlockRead(Map<String, Long> previousMetricSnapshot,
+      SchemaMetrics schemaMetrics, long expectDataBlockRead, long expectedIndexBlockRead){
+    Map<String, Long> currentMetricsSnapshot = SchemaMetrics.getMetricsSnapshot();
+    Map<String, Long> diffs =
+      SchemaMetrics.diffMetrics(previousMetricSnapshot, currentMetricsSnapshot);
+
+    long dataBlockRead = SchemaMetrics.getLong(diffs,
+        schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT));
+    long indexBlockRead = SchemaMetrics.getLong(diffs,
+        schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, BlockMetricType.READ_COUNT));
+
+    Assert.assertEquals(expectDataBlockRead, dataBlockRead);
+    Assert.assertEquals(expectedIndexBlockRead, indexBlockRead);
+  }
+}

From 0bb16f7a68232836df954f642e024c36c57fb6fc Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Oct 2012 21:59:07 +0000
Subject: [PATCH 0529/1540] HBASE-6974 Metric for blocked updates (Michael
 Drzal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1399881 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 12 +++++++--
 .../hbase/regionserver/HRegionServer.java     |  7 +++++
 .../hbase/regionserver/MemStoreFlusher.java   | 27 ++++++++++++++++++-
 .../metrics/RegionServerMetrics.java          | 14 ++++++++++
 4 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 5523634bc073..1bced40b56dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -221,6 +221,7 @@ public class HRegion implements HeapSize { // , Writable{
 
   final Counter readRequestsCount = new Counter();
   final Counter writeRequestsCount = new Counter();
+  final Counter updatesBlockedMs = new Counter();
 
   /**
    * The directory for the table this region is part of.
@@ -2448,9 +2449,11 @@ private void checkResources() {
     if (this.getRegionInfo().isMetaRegion()) return;
 
     boolean blocked = false;
+    long startTime = 0;
     while (this.memstoreSize.get() > this.blockingMemStoreSize) {
       requestFlush();
       if (!blocked) {
+        startTime = EnvironmentEdgeManager.currentTimeMillis();
         LOG.info("Blocking updates for '" + Thread.currentThread().getName() +
           "' on region " + Bytes.toStringBinary(getRegionName()) +
           ": memstore size " +
@@ -2463,11 +2466,16 @@ private void checkResources() {
         try {
           wait(threadWakeFrequency);
         } catch (InterruptedException e) {
-          // continue;
+          Thread.currentThread().interrupt();
         }
       }
     }
     if (blocked) {
+      // Add in the blocked time if appropriate
+      final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+      if(totalTime > 0 ){
+        this.updatesBlockedMs.add(totalTime);
+      }
       LOG.info("Unblocking updates for region " + this + " '"
           + Thread.currentThread().getName() + "'");
     }
@@ -4791,7 +4799,7 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      34 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      35 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
       (5 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 8f9cf0c02d71..cb15bb6d95f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1358,6 +1358,7 @@ protected void metrics() {
     long totalStaticBloomSize = 0;
     long numPutsWithoutWAL = 0;
     long dataInMemoryWithoutWAL = 0;
+    long updatesBlockedMs = 0;
 
     // Note that this is a map of Doubles instead of Longs. This is because we
     // do effective integer division, which would perhaps truncate more than it
@@ -1374,6 +1375,7 @@ protected void metrics() {
       dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
       readRequestsCount += r.readRequestsCount.get();
       writeRequestsCount += r.writeRequestsCount.get();
+      updatesBlockedMs += r.updatesBlockedMs.get();
       synchronized (r.stores) {
         stores += r.stores.size();
         for (Map.Entry<byte[], Store> ee : r.stores.entrySet()) {
@@ -1451,6 +1453,11 @@ protected void metrics() {
         .getCompactionQueueSize());
     this.metrics.flushQueueSize.set(cacheFlusher
         .getFlushQueueSize());
+    this.metrics.updatesBlockedSeconds.update(updatesBlockedMs > 0 ? 
+        updatesBlockedMs/1000: 0);
+    final long updatesBlockedMsHigherWater = cacheFlusher.getUpdatesBlockedMsHighWater().get();
+    this.metrics.updatesBlockedSecondsHighWater.update(updatesBlockedMsHigherWater > 0 ? 
+        updatesBlockedMsHigherWater/1000: 0);
 
     BlockCache blockCache = cacheConfig.getBlockCache();
     if (blockCache != null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index df64dba9a936..c91404ddb05a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -42,8 +42,10 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.util.StringUtils;
+import org.cliffc.high_scale_lib.Counter;
 
 import com.google.common.base.Preconditions;
 
@@ -82,6 +84,7 @@ class MemStoreFlusher extends HasThread implements FlushRequester {
     "hbase.regionserver.global.memstore.lowerLimit";
   private long blockingStoreFilesNumber;
   private long blockingWaitTime;
+  private final Counter updatesBlockedMsHighWater = new Counter();
 
   /**
    * @param conf
@@ -144,6 +147,10 @@ static long getMemStoreLimit(final long max, final float limit,
     return (long)(max * effectiveLimit);
   }
 
+  public Counter getUpdatesBlockedMsHighWater() {
+    return this.updatesBlockedMsHighWater;
+  }
+
   /**
    * The memstore across all regions has exceeded the low water mark. Pick
    * one region to flush and flush it synchronously (this is called from the
@@ -450,11 +457,22 @@ private boolean isTooManyStoreFiles(HRegion region) {
    * to the lower limit. This method blocks callers until we're down to a safe
    * amount of memstore consumption.
    */
-  public synchronized void reclaimMemStoreMemory() {
+  public void reclaimMemStoreMemory() {
     if (isAboveHighWaterMark()) {
       lock.lock();
       try {
+        boolean blocked = false;
+        long startTime = 0;
         while (isAboveHighWaterMark() && !server.isStopped()) {
+          if(!blocked){
+            startTime = EnvironmentEdgeManager.currentTimeMillis();
+            LOG.info("Blocking updates on " + server.toString() +
+            ": the global memstore size " +
+            StringUtils.humanReadableInt(server.getRegionServerAccounting().getGlobalMemstoreSize()) +
+            " is >= than blocking " +
+            StringUtils.humanReadableInt(globalMemStoreLimit) + " size");
+          }
+          blocked = true;
           wakeupFlushThread();
           try {
             // we should be able to wait forever, but we've seen a bug where
@@ -464,6 +482,13 @@ public synchronized void reclaimMemStoreMemory() {
             Thread.currentThread().interrupt();
           }
         }
+        if(blocked){
+          final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+          if(totalTime > 0){
+            this.updatesBlockedMsHighWater.add(totalTime);
+          }
+          LOG.info("Unblocking updates for server " + server.toString());
+        }
       } finally {
         lock.unlock();
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index 9ce40b195326..e6590f5ba3d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -297,6 +297,18 @@ public class RegionServerMetrics implements Updater {
   public final MetricsLongValue checksumFailuresCount =
     new MetricsLongValue("checksumFailuresCount", registry);
 
+  /**
+   * time blocked on lack of resources
+   */
+  public final MetricsHistogram updatesBlockedSeconds = new MetricsHistogram(
+      "updatesBlockedSeconds", registry);
+
+  /**
+   * time blocked on memstoreHW
+   */
+  public final MetricsHistogram updatesBlockedSecondsHighWater = new MetricsHistogram(
+      "updatesBlockedSecondsHighWater",registry);
+
   public RegionServerMetrics() {
     MetricsContext context = MetricsUtil.getContext("hbase");
     metricsRecord = MetricsUtil.createRecord(context, "regionserver");
@@ -436,6 +448,8 @@ public void doUpdates(MetricsContext caller) {
       this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
       this.regionSplitFailureCount.pushMetric(this.metricsRecord);
       this.checksumFailuresCount.pushMetric(this.metricsRecord);
+      this.updatesBlockedSeconds.pushMetric(this.metricsRecord);
+      this.updatesBlockedSecondsHighWater.pushMetric(this.metricsRecord);
     }
     this.metricsRecord.update();
   }

From c61000d91df6af36b7efd0c579051c753281915b Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 19 Oct 2012 19:13:23 +0000
Subject: [PATCH 0530/1540] Update affiliation

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1400237 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index ae9c2ee99766..a9607fed9e63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,8 +113,8 @@
       <name>Andrew Purtell</name>
       <email>apurtell@apache.org</email>
       <timezone>-8</timezone>
-      <organization>Trend Micro</organization>
-      <organizationUrl>http://www.trendmicro.com</organizationUrl>
+      <organization>Intel</organization>
+      <organizationUrl>http://www.intel.com</organizationUrl>
     </developer>
   <developer>
       <id>dmeil</id>

From 114d45939ae6231c3bd85365424dd0dec31abd19 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 20 Oct 2012 03:59:44 +0000
Subject: [PATCH 0531/1540] HBASE-7020 Backport HBASE-6336 Split point should
 not be equal to start row or end row (Sergey Shelukhin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1400359 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 20164ea41ad3..5da808e0f397 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -2011,9 +2011,9 @@ public byte[] getSplitPoint() {
         KeyValue firstKey = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
         byte [] lk = r.getLastKey();
         KeyValue lastKey = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
-        // if the midkey is the same as the first and last keys, then we cannot
+        // if the midkey is the same as the first or last keys, then we cannot
         // (ever) split this region.
-        if (this.comparator.compareRows(mk, firstKey) == 0 &&
+        if (this.comparator.compareRows(mk, firstKey) == 0 ||
             this.comparator.compareRows(mk, lastKey) == 0) {
           if (LOG.isDebugEnabled()) {
             LOG.debug("cannot split because midkey is the same as first or " +

From d41cc2e327a0827b7222df29a7755e623c3beb69 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 20 Oct 2012 04:51:36 +0000
Subject: [PATCH 0532/1540] HBASE-7021 Default to Hadoop 1.0.4 in 0.94 and add
 Hadoop 1.1 profile

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1400366 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 83 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index a9607fed9e63..a1ff8ca70c33 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1570,7 +1570,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>1.0.3</hadoop.version>
+        <hadoop.version>1.0.4</hadoop.version>
         <slf4j.version>1.4.3</slf4j.version>
       </properties>
       <dependencies>
@@ -1642,7 +1642,7 @@
     <profile>
       <id>security</id>
       <properties>
-        <hadoop.version>1.0.3</hadoop.version>
+        <hadoop.version>1.0.4</hadoop.version>
       </properties>
       <build>
         <finalName>${project.artifactId}-${project.version}-security</finalName>
@@ -1696,6 +1696,87 @@
     </profile>
 
 
+    <!--
+      profile for building against Hadoop 1.1. Activate using:
+       mvn -Dhadoop.profile=1.1
+    -->
+    <profile>
+      <id>hadoop-1.1</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>1.1</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>1.1.0</hadoop.version>
+        <slf4j.version>1.4.3</slf4j.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-core</artifactId>
+          <version>${hadoop.version}</version>
+          <optional>true</optional>
+          <exclusions>
+            <exclusion>
+              <groupId>hsqldb</groupId>
+              <artifactId>hsqldb</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>net.sf.kosmosfs</groupId>
+              <artifactId>kfs</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.eclipse.jdt</groupId>
+              <artifactId>core</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>net.java.dev.jets3t</groupId>
+              <artifactId>jets3t</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>oro</groupId>
+              <artifactId>oro</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-test</artifactId>
+          <version>${hadoop.version}</version>
+          <optional>true</optional>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-resource</id>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>src/test/resources</directory>
+                      <includes>
+                        <include>hbase-site.xml</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!--
       profile for building against Hadoop 0.22.0. Activate using:
        mvn -Dhadoop.profile=22

From 5b48ec4eab7a5615b4343681c718abf906beacde Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Mon, 22 Oct 2012 17:10:08 +0000
Subject: [PATCH 0533/1540] HBASE-6951 Allow the master info server to be
 started in a read only mode.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1400957 13f79535-47bb-0310-9956-ffa450edef68
---
 .../resources/hbase-webapps/master/table.jsp  |  7 ++-
 .../apache/hadoop/hbase/TestInfoServers.java  | 55 +++++++++++++++----
 2 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index b7b0f333dd3a..3dd7fdca4741 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -45,6 +45,7 @@
   String tableHeader = "<h2>Table Regions</h2><table><tr><th>Name</th><th>Region Server</th><th>Start Key</th><th>End Key</th><th>Requests</th></tr>";
   ServerName rl = master.getCatalogTracker().getRootLocation();
   boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false);
+  boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false);
   Map<String, Integer> frags = null;
   if (showFragmentation) {
       frags = FSUtils.getTableFragmentation(master);
@@ -65,7 +66,7 @@
 <%
   String action = request.getParameter("action");
   String key = request.getParameter("key");
-  if ( action != null ) {
+  if ( !readOnly && action != null ) {
 %>
 <head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
 <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
@@ -237,6 +238,8 @@
 HConnectionManager.deleteConnection(hbadmin.getConfiguration(), false);
 %>
 
+
+<% if (!readOnly) { %>
 <p><hr><p>
 Actions:
 <p>
@@ -273,7 +276,7 @@ Actions:
 </table>
 </center>
 <p>
-
+<% } %>
 <%
 }
 %>
diff --git a/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java b/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
index adb5365cbeb3..f7c90d4993be 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
@@ -26,6 +26,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -48,6 +49,7 @@ public static void beforeClass() throws Exception {
     // Set them to ephemeral ports so they will start
     UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
     UTIL.getConfiguration().setInt("hbase.regionserver.info.port", 0);
+    UTIL.getConfiguration().setBoolean("hbase.master.ui.readonly", true);
     UTIL.startMiniCluster();
   }
 
@@ -64,12 +66,12 @@ public void testInfoServersRedirect() throws Exception {
     // give the cluster time to start up
     new HTable(UTIL.getConfiguration(), ".META.").close();
     int port = UTIL.getHBaseCluster().getMaster().getInfoServer().getPort();
-    assertHasExpectedContent(new URL("http://localhost:" + port +
-      "/index.html"), "master-status");
+    assertContainsContent(new URL("http://localhost:" + port +
+        "/index.html"), "master-status");
     port = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer().
       getInfoServer().getPort();
-    assertHasExpectedContent(new URL("http://localhost:" + port +
-      "/index.html"), "rs-status");
+    assertContainsContent(new URL("http://localhost:" + port +
+        "/index.html"), "rs-status");
   }
 
   /**
@@ -84,17 +86,49 @@ public void testInfoServersStatusPages() throws Exception {
     // give the cluster time to start up
     new HTable(UTIL.getConfiguration(), ".META.").close();
     int port = UTIL.getHBaseCluster().getMaster().getInfoServer().getPort();
-    assertHasExpectedContent(new URL("http://localhost:" + port +
-      "/master-status"), "META");
+    assertContainsContent(new URL("http://localhost:" + port +
+        "/master-status"), "META");
     port = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer().
       getInfoServer().getPort();
-    assertHasExpectedContent(new URL("http://localhost:" + port +
-      "/rs-status"), "META");
+    assertContainsContent(new URL("http://localhost:" + port +
+        "/rs-status"), "META");
   }
 
-  private void assertHasExpectedContent(final URL u, final String expected)
+  @Test
+  public void testMasterServerReadOnly() throws Exception {
+    String sTableName = "testMasterServerReadOnly";
+    byte[] tableName = Bytes.toBytes(sTableName);
+    byte[] cf = Bytes.toBytes("d");
+    UTIL.createTable(tableName, cf);
+    new HTable(UTIL.getConfiguration(), tableName).close();
+    int port = UTIL.getHBaseCluster().getMaster().getInfoServer().getPort();
+    assertDoesNotContainContent(
+      new URL("http://localhost:" + port + "/table.jsp?name=" + sTableName + "&action=split&key="),
+      "Table action request accepted");
+    assertDoesNotContainContent(
+      new URL("http://localhost:" + port + "/table.jsp?name=" + sTableName),
+      "Actions:");
+  }
+
+  private void assertContainsContent(final URL u, final String expected)
   throws IOException {
     LOG.info("Testing " + u.toString() + " has " + expected);
+    String content = getUrlContent(u);
+    assertTrue("expected=" + expected + ", content=" + content,
+      content.contains(expected));
+  }
+
+
+
+  private void assertDoesNotContainContent(final URL u, final String expected)
+      throws IOException {
+    LOG.info("Testing " + u.toString() + " has " + expected);
+    String content = getUrlContent(u);
+    assertTrue("Does Not Contain =" + expected + ", content=" + content,
+        !content.contains(expected));
+  }
+
+  private String getUrlContent(URL u) throws IOException {
     java.net.URLConnection c = u.openConnection();
     c.connect();
     StringBuilder sb = new StringBuilder();
@@ -105,8 +139,7 @@ private void assertHasExpectedContent(final URL u, final String expected)
     }
     bis.close();
     String content = sb.toString();
-    assertTrue("expected=" + expected + ", content=" + content,
-      content.contains(expected));
+    return content;
   }
 
   @org.junit.Rule

From 510cb9f71d5d7fab6f4eb0c88e83d105111be1fd Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 22 Oct 2012 22:58:02 +0000
Subject: [PATCH 0534/1540] HBASE-6583. Enhance LoadTestTool to automatically
 create column families if not present (Sergey Shelukhin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401116 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/LoadTestTool.java     | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index 9fa7acd4bf9f..5b372ae0a439 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -155,6 +155,10 @@ private void applyColumnFamilyOptions(byte[] tableName,
     admin.disableTable(tableName);
     for (byte[] cf : columnFamilies) {
       HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
+      boolean isNewCf = columnDesc == null;
+      if (isNewCf) {
+        columnDesc = new HColumnDescriptor(cf);
+      }
       if (bloomType != null) {
         columnDesc.setBloomFilterType(bloomType);
       }
@@ -165,7 +169,11 @@ private void applyColumnFamilyOptions(byte[] tableName,
         columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
         columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
       }
-      admin.modifyColumn(tableName, columnDesc);
+      if (isNewCf) {
+        admin.addColumn(tableName, columnDesc);
+      } else {
+        admin.modifyColumn(tableName, columnDesc);
+      }
     }
     LOG.info("Enabling table " + Bytes.toString(tableName));
     admin.enableTable(tableName);

From e2feae6730775795be5696fc7e25139ffd99db13 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 22 Oct 2012 23:00:59 +0000
Subject: [PATCH 0535/1540] HBASE-5314 Gracefully rolling restart region
 servers in rolling-restart.sh (Yifeng)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401117 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/rolling-restart.sh | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 72111eddb88b..07a78c71edec 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -34,7 +34,7 @@
 #
 # Modelled after $HADOOP_HOME/bin/slaves.sh.
 
-usage="Usage: $0 [--config <hbase-confdir>] [--rs-only] [--master-only]"
+usage="Usage: $0 [--config <hbase-confdir>] [--rs-only] [--master-only] [--graceful]"
 
 bin=`dirname "$0"`
 bin=`cd "$bin">/dev/null; pwd`
@@ -56,16 +56,24 @@ function usage() {
 
 RR_RS=1
 RR_MASTER=1
+RR_GRACEFUL=0
 
 for x in "$@" ; do
   case "$x" in
     --rs-only|-r)
       RR_RS=1
       RR_MASTER=0
+      RR_GRACEFUL=0
       ;;
     --master-only)
       RR_RS=0
       RR_MASTER=1
+      RR_GRACEFUL=0
+      ;;
+    --graceful)
+      RR_RS=0
+      RR_MASTER=0
+      RR_GRACEFUL=1
       ;;
     *)
       echo Bad argument: $x
@@ -85,6 +93,9 @@ if [ "$distMode" == 'false' ]; then
   fi
   "$bin"/hbase-daemon.sh restart master
 else 
+  zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent`
+  if [ "$zparent" == "null" ]; then zparent="/hbase"; fi
+
   if [ $RR_MASTER -eq 1 ]; then
     # stop all masters before re-start to avoid races for master znode
     "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master 
@@ -92,8 +103,6 @@ else
       --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
 
     # make sure the master znode has been deleted before continuing
-    zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent`
-    if [ "$zparent" == "null" ]; then zparent="/hbase"; fi
     zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master`
     if [ "$zmaster" == "null" ]; then zmaster="master"; fi
     zmaster=$zparent/$zmaster
@@ -137,4 +146,17 @@ else
     "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
       --hosts "${HBASE_REGIONSERVERS}" restart regionserver
   fi
+
+  if [ $RR_GRACEFUL -eq 1 ]; then
+    # gracefully restart all online regionservers
+    online_regionservers=`$bin/hbase zkcli ls $zparent/rs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"`
+    for rs in $online_regionservers
+    do
+        rs_parts=(${rs//,/ })
+        hostname=${rs_parts[0]}
+        echo "Gracefully restarting: $hostname"
+        "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname"
+        sleep 1
+    done
+  fi
 fi

From cc468d66c3d91b8057c2409134750a788265ffa7 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Tue, 23 Oct 2012 00:00:54 +0000
Subject: [PATCH 0536/1540] HBASE-7016 port HBASE-6518 'Bytes.toBytesBinary()
 incorrect trailing backslash escape' to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401127 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/Bytes.java | 11 ++---------
 .../java/org/apache/hadoop/hbase/util/TestBytes.java  |  7 +++++++
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index ece0aae01a43..e10619c89c58 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -378,19 +378,12 @@ public static byte toBinaryFromHex(byte ch) {
   }
 
   public static byte [] toBytesBinary(String in) {
-    // this may be bigger than we need, but lets be safe.
+    // this may be bigger than we need, but let's be safe.
     byte [] b = new byte[in.length()];
     int size = 0;
     for (int i = 0; i < in.length(); ++i) {
       char ch = in.charAt(i);
-      if (ch == '\\') {
-        // begin hex escape:
-        char next = in.charAt(i+1);
-        if (next != 'x') {
-          // invalid escape sequence, ignore this one.
-          b[size++] = (byte)ch;
-          continue;
-        }
+      if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
         // ok, take next 2 hex digits.
         char hd1 = in.charAt(i+2);
         char hd2 = in.charAt(i+3);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
index e7fea6c465f7..87f60f5ac5a6 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
@@ -290,6 +290,13 @@ public void testFixedSizeString() throws IOException {
     assertEquals("", Bytes.readStringFixedSize(dis, 9));
   }
 
+  public void testToBytesBinaryTrailingBackslashes() throws Exception {
+    try {
+      Bytes.toBytesBinary("abc\\x00\\x01\\");
+    } catch (StringIndexOutOfBoundsException ex) {
+      fail("Illegal string access: " + ex.getMessage());
+    }
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From d49083345e4e31972fff0bdb849221c0c45986b6 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Tue, 23 Oct 2012 00:23:02 +0000
Subject: [PATCH 0537/1540] HBASE-6733 TestReplication.queueFailover
 occasionally fails [Part-2]

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401130 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/ReplicationSource.java      | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index b1c54c3d90d2..fca909d7356c 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -279,8 +279,15 @@ public void run() {
         }
         continue;
       }
+      Path oldPath = getCurrentPath(); //note that in the current scenario,
+                                       //oldPath will be null when a log roll
+                                       //happens.
       // Get a new path
-      if (!getNextPath()) {
+      boolean hasCurrentPath = getNextPath();
+      if (getCurrentPath() != null && oldPath == null) {
+        sleepMultiplier = 1; //reset the sleepMultiplier on a path change
+      }
+      if (!hasCurrentPath) {
         if (sleepForRetries("No log to process", sleepMultiplier)) {
           sleepMultiplier++;
         }

From 4ae1fa6c33224441bbb67e2a18c8d1ddbfc1b621 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 23 Oct 2012 16:04:46 +0000
Subject: [PATCH 0538/1540] HBASE-6942 Endpoint implementation for bulk delete
 rows (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401332 13f79535-47bb-0310-9956-ffa450edef68
---
 .../example/BulkDeleteEndpoint.java           | 210 +++++++++
 .../example/BulkDeleteProtocol.java           |  83 ++++
 .../example/BulkDeleteResponse.java           |  62 +++
 .../example/TestBulkDeleteProtocol.java       | 407 ++++++++++++++++++
 4 files changed, 762 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java
new file mode 100644
index 000000000000..d259f9ff6986
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.OperationStatus;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class BulkDeleteEndpoint extends BaseEndpointCoprocessor implements BulkDeleteProtocol {
+  private static final String NO_OF_VERSIONS_TO_DELETE = "noOfVersionsToDelete";
+  private static final Log LOG = LogFactory.getLog(BulkDeleteEndpoint.class);
+  
+  @Override
+  public BulkDeleteResponse delete(Scan scan, byte deleteType, Long timestamp,
+      int rowBatchSize) {
+    long totalRowsDeleted = 0L;
+    long totalVersionsDeleted = 0L;
+    BulkDeleteResponse response = new BulkDeleteResponse();
+    HRegion region = ((RegionCoprocessorEnvironment) getEnvironment()).getRegion();
+    boolean hasMore = true;
+    RegionScanner scanner = null;
+    if (scan.getFilter() == null && deleteType == DeleteType.ROW) {
+      // What we need is just the rowkeys. So only 1st KV from any row is enough.
+      // Only when it is a row delete, we can apply this filter
+      // In other types we rely on the scan to know which all columns to be deleted.
+      scan.setFilter(new FirstKeyOnlyFilter());
+    }
+    // When the delete is based on some conditions so that Filters are available in the scan,
+    // we assume that the scan is perfect having necessary column(s) only.
+    try {
+      scanner = region.getScanner(scan);
+      while (hasMore) {
+        List<List<KeyValue>> deleteRows = new ArrayList<List<KeyValue>>(rowBatchSize);
+        for (int i = 0; i < rowBatchSize; i++) {
+          List<KeyValue> results = new ArrayList<KeyValue>();
+          hasMore = scanner.next(results);
+          if (results.size() > 0) {
+            deleteRows.add(results);
+          }
+          if (!hasMore) {
+            // There are no more rows.
+            break;
+          }
+        }
+        if (deleteRows.size() > 0) {
+          Pair<Mutation, Integer>[] deleteWithLockArr = new Pair[deleteRows.size()];
+          int i = 0;
+          for (List<KeyValue> deleteRow : deleteRows) {
+            Delete delete = createDeleteMutation(deleteRow, deleteType, timestamp);
+            deleteWithLockArr[i++] = new Pair<Mutation, Integer>(delete, null);
+          }
+          OperationStatus[] opStatus = region.batchMutate(deleteWithLockArr);
+          for (i = 0; i < opStatus.length; i++) {
+            if (opStatus[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
+              break;
+            }
+            totalRowsDeleted++;
+            if (deleteType == DeleteType.VERSION) {
+              byte[] versionsDeleted = deleteWithLockArr[i].getFirst().getAttribute(
+                  NO_OF_VERSIONS_TO_DELETE);
+              if (versionsDeleted != null) {
+                totalVersionsDeleted += Bytes.toInt(versionsDeleted);
+              }
+            }
+          }
+        } 
+      }
+    } catch (IOException ioe) {
+      LOG.error(ioe);
+      response.setIoException(ioe);
+    } finally {
+      if (scanner != null) {
+        try {
+          scanner.close();
+        } catch (IOException ioe) {
+          LOG.error(ioe);
+        }
+      }
+    }
+    response.setRowsDeleted(totalRowsDeleted);
+    response.setVersionsDeleted(totalVersionsDeleted);
+    return response;
+  }
+
+  private Delete createDeleteMutation(List<KeyValue> deleteRow, byte deleteType, Long timestamp) {
+    long ts;
+    if (timestamp == null) {
+      ts = HConstants.LATEST_TIMESTAMP;
+    } else {
+      ts = timestamp;
+    }
+    // We just need the rowkey. Get it from 1st KV.
+    byte[] row = deleteRow.get(0).getRow();
+    Delete delete = new Delete(row, ts, null);
+    if (deleteType != DeleteType.ROW) {
+      switch (deleteType) {
+      case DeleteType.FAMILY:
+        Set<byte[]> families = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+        for (KeyValue kv : deleteRow) {
+          if (families.add(kv.getFamily())) {
+            delete.deleteFamily(kv.getFamily(), ts);
+          }
+        }
+        break;
+
+      case DeleteType.COLUMN:
+        Set<Column> columns = new HashSet<Column>();
+        for (KeyValue kv : deleteRow) {
+          Column column = new Column(kv.getFamily(), kv.getQualifier());
+          if (columns.add(column)) {
+            // Making deleteColumns() calls more than once for the same cf:qualifier is not correct
+            // Every call to deleteColumns() will add a new KV to the familymap which will finally
+            // get written to the memstore as part of delete().
+            delete.deleteColumns(column.family, column.qualifier, ts);
+          }
+        }
+        break;
+
+      case DeleteType.VERSION:
+        // When some timestamp was passed to the delete() call only one version of the column (with
+        // given timestamp) will be deleted. If no timestamp passed, it will delete N versions.
+        // How many versions will get deleted depends on the Scan being passed. All the KVs that
+        // the scan fetched will get deleted.
+        int noOfVersionsToDelete = 0;
+        if (timestamp == null) {
+          for (KeyValue kv : deleteRow) {
+            delete.deleteColumn(kv.getFamily(), kv.getQualifier(), kv.getTimestamp());
+            noOfVersionsToDelete++;
+          }
+        } else {
+          columns = new HashSet<Column>();
+          for (KeyValue kv : deleteRow) {
+            Column column = new Column(kv.getFamily(), kv.getQualifier());
+            // Only one version of particular column getting deleted.
+            if (columns.add(column)) {
+              delete.deleteColumn(column.family, column.qualifier, ts);
+              noOfVersionsToDelete++;
+            }
+          }
+        }
+        delete.setAttribute(NO_OF_VERSIONS_TO_DELETE, Bytes.toBytes(noOfVersionsToDelete));
+      }
+    }
+    return delete;
+  }
+  
+  private static class Column {
+    private byte[] family;
+    private byte[] qualifier;
+
+    public Column(byte[] family, byte[] qualifier) {
+      this.family = family;
+      this.qualifier = qualifier;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      if (!(other instanceof Column)) {
+        return false;
+      }
+      Column column = (Column) other;
+      return Bytes.equals(this.family, column.family)
+          && Bytes.equals(this.qualifier, column.qualifier);
+    }
+
+    @Override
+    public int hashCode() {
+      int h = 31;
+      h = h + 13 * Bytes.hashCode(this.family);
+      h = h + 13 * Bytes.hashCode(this.qualifier);
+      return h;
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java
new file mode 100644
index 000000000000..0a3d2cafc3f6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+
+/**
+ * Defines a protocol to delete data in bulk based on a scan. The scan can be range scan or with
+ * conditions(filters) etc.
+ * </br> Example: <code><pre>
+ * Scan scan = new Scan();
+ * // set scan properties(rowkey range, filters, timerange etc).
+ * HTable ht = ...;
+ * long noOfDeletedRows = 0L;
+ * Batch.Call&lt;BulkDeleteProtocol, BulkDeleteResponse&gt; callable = 
+ *     new Batch.Call&lt;BulkDeleteProtocol, BulkDeleteResponse&gt;() {
+ *   public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
+ *     return instance.deleteRows(scan, BulkDeleteProtocol.DeleteType, timestamp, rowBatchSize);
+ *   }
+ * };
+ * Map&lt;byte[], BulkDeleteResponse&gt; result = ht.coprocessorExec(BulkDeleteProtocol.class,
+ *      scan.getStartRow(), scan.getStopRow(), callable);
+ *  for (BulkDeleteResponse response : result.values()) {
+ *    noOfDeletedRows = response.getRowsDeleted();
+ *  }
+ * </pre></code>
+ */
+public interface BulkDeleteProtocol extends CoprocessorProtocol {
+  
+  public interface DeleteType {
+    /** 
+     * Delete full row
+     */
+    byte ROW = 0;
+    /**
+     * Delete full family(s).
+     * Which family(s) to be deleted will be determined by the Scan.
+     * Scan need to select all the families which need to be deleted.
+     */
+    byte FAMILY = 1;
+    /**
+     * Delete full column(s).
+     * Which column(s) to be deleted will be determined by the Scan.
+     * Scan need to select all the qualifiers which need to be deleted.
+     */
+    byte COLUMN = 2;
+    /**
+     * Delete one or more version(s) of column(s).
+     * Which column(s) and version(s) to be deleted will be determined by the Scan.
+     * Scan need to select all the qualifiers and its versions which need to be deleted.
+     * When a timestamp is passed only one version at that timestamp will be deleted(even if scan
+     * fetches many versions)
+     */
+    byte VERSION = 3;
+  }
+  
+  /**
+   * 
+   * @param scan
+   * @param deleteType
+   * @param timestamp
+   * @param rowBatchSize
+   *          The number of rows which need to be accumulated by scan and delete as one batch
+   * @return
+   */
+  BulkDeleteResponse delete(Scan scan, byte deleteType, Long timestamp, int rowBatchSize);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java
new file mode 100644
index 000000000000..0ccabb8a2ffc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+/**
+ * Wrapper class which returns the result of the bulk deletion operation happened at the server for
+ * a region. This includes the total number of rows deleted and/or any {@link IOException} which is
+ * happened while doing the operation. It will also include total number of versions deleted, when
+ * the delete type is VERSION.
+ */
+public class BulkDeleteResponse implements Serializable {
+  private static final long serialVersionUID = -8192337710525997237L;
+  private long rowsDeleted;
+  private IOException ioException;
+  private long versionsDeleted;
+
+  public BulkDeleteResponse() {
+
+  }
+
+  public void setRowsDeleted(long rowsDeleted) {
+    this.rowsDeleted = rowsDeleted;
+  }
+
+  public long getRowsDeleted() {
+    return rowsDeleted;
+  }
+
+  public void setIoException(IOException ioException) {
+    this.ioException = ioException;
+  }
+
+  public IOException getIoException() {
+    return ioException;
+  }
+
+  public long getVersionsDeleted() {
+    return versionsDeleted;
+  }
+
+  public void setVersionsDeleted(long versionsDeleted) {
+    this.versionsDeleted = versionsDeleted;
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java
new file mode 100644
index 000000000000..8b4d51348361
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.example.BulkDeleteProtocol.DeleteType;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestBulkDeleteProtocol {
+  private static final byte[] FAMILY1 = Bytes.toBytes("cf1");
+  private static final byte[] FAMILY2 = Bytes.toBytes("cf2");
+  private static final byte[] QUALIFIER1 = Bytes.toBytes("c1");
+  private static final byte[] QUALIFIER2 = Bytes.toBytes("c2");
+  private static final byte[] QUALIFIER3 = Bytes.toBytes("c3");
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
+        BulkDeleteEndpoint.class.getName());
+    TEST_UTIL.startMiniCluster(2);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testBulkDeleteEndpoint() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteEndpoint");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      byte[] rowkey = Bytes.toBytes(j);
+      puts.add(createPut(rowkey, "v1"));
+    }
+    ht.put(puts);
+    // Deleting all the rows.
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 500, DeleteType.ROW,
+        null);
+    assertEquals(100, noOfRowsDeleted);
+
+    int rows = 0;
+    for (Result result : ht.getScanner(new Scan())) {
+      rows++;
+    }
+    assertEquals(0, rows);
+  }
+
+  @Test
+  public void testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion()
+      throws Throwable {
+    byte[] tableName = Bytes
+        .toBytes("testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      byte[] rowkey = Bytes.toBytes(j);
+      puts.add(createPut(rowkey, "v1"));
+    }
+    ht.put(puts);
+    // Deleting all the rows.
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 10, DeleteType.ROW, null);
+    assertEquals(100, noOfRowsDeleted);
+
+    int rows = 0;
+    for (Result result : ht.getScanner(new Scan())) {
+      rows++;
+    }
+    assertEquals(0, rows);
+  }
+  
+  private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize,
+      final byte deleteType, final Long timeStamp) throws Throwable {
+    HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    long noOfDeletedRows = 0L;
+    Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable = 
+        new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
+      public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
+        return instance.delete(scan, deleteType, timeStamp, rowBatchSize);
+      }
+    };
+    Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
+        scan.getStartRow(), scan.getStopRow(), callable);
+    for (BulkDeleteResponse response : result.values()) {
+      noOfDeletedRows += response.getRowsDeleted();
+    }
+    return noOfDeletedRows;
+  }
+
+  @Test
+  public void testBulkDeleteWithConditionBasedDelete() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteWithConditionBasedDelete");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      byte[] rowkey = Bytes.toBytes(j);
+      String value = (j % 10 == 0) ? "v1" : "v2";
+      puts.add(createPut(rowkey, value));
+    }
+    ht.put(puts);
+    Scan scan = new Scan();
+    FilterList fl = new FilterList(Operator.MUST_PASS_ALL);
+    SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILY1, QUALIFIER3,
+        CompareOp.EQUAL, Bytes.toBytes("v1"));
+    //fl.addFilter(new FirstKeyOnlyFilter());
+    fl.addFilter(scvf);
+    scan.setFilter(fl);
+    // Deleting all the rows where cf1:c1=v1
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.ROW, null);
+    assertEquals(10, noOfRowsDeleted);
+
+    int rows = 0;
+    for (Result result : ht.getScanner(new Scan())) {
+      rows++;
+    }
+    assertEquals(90, rows);
+  }
+
+  @Test
+  public void testBulkDeleteColumn() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteColumn");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      byte[] rowkey = Bytes.toBytes(j);
+      String value = (j % 10 == 0) ? "v1" : "v2";
+      puts.add(createPut(rowkey, value));
+    }
+    ht.put(puts);
+    Scan scan = new Scan ();
+    scan.addColumn(FAMILY1, QUALIFIER2);
+    // Delete the column cf1:col2
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.COLUMN, null);
+    assertEquals(100, noOfRowsDeleted);
+
+    int rows = 0;
+    for (Result result : ht.getScanner(new Scan())) {
+      assertEquals(2, result.getFamilyMap(FAMILY1).size());
+      assertTrue(result.getColumn(FAMILY1, QUALIFIER2).isEmpty());
+      assertEquals(1, result.getColumn(FAMILY1, QUALIFIER1).size());
+      assertEquals(1, result.getColumn(FAMILY1, QUALIFIER3).size());
+      rows++;
+    }
+    assertEquals(100, rows);
+  }
+  
+  @Test
+  public void testBulkDeleteFamily() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteFamily");
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(FAMILY1));
+    htd.addFamily(new HColumnDescriptor(FAMILY2));
+    TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
+    HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      Put put = new Put(Bytes.toBytes(j));
+      put.add(FAMILY1, QUALIFIER1, "v1".getBytes());
+      put.add(FAMILY2, QUALIFIER2, "v2".getBytes());
+      puts.add(put);
+    }
+    ht.put(puts);
+    Scan scan = new Scan ();
+    scan.addFamily(FAMILY1);
+    // Delete the column family cf1
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null);
+    assertEquals(100, noOfRowsDeleted);
+    int rows = 0;
+    for (Result result : ht.getScanner(new Scan())) {
+      assertTrue(result.getFamilyMap(FAMILY1).isEmpty());
+      assertEquals(1, result.getColumn(FAMILY2, QUALIFIER2).size());
+      rows++;
+    }
+    assertEquals(100, rows);
+  }
+  
+  @Test
+  public void testBulkDeleteColumnVersion() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersion");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      Put put = new Put(Bytes.toBytes(j));
+      byte[] value = "v1".getBytes();
+      put.add(FAMILY1, QUALIFIER1, 1234L, value);
+      put.add(FAMILY1, QUALIFIER2, 1234L, value);
+      put.add(FAMILY1, QUALIFIER3, 1234L, value);
+      // Latest version values
+      value = "v2".getBytes();
+      put.add(FAMILY1, QUALIFIER1, value);
+      put.add(FAMILY1, QUALIFIER2, value);
+      put.add(FAMILY1, QUALIFIER3, value);
+      put.add(FAMILY1, null, value);
+      puts.add(put);
+    }
+    ht.put(puts);
+    Scan scan = new Scan ();
+    scan.addFamily(FAMILY1);
+    // Delete the latest version values of all the columns in family cf1.
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION,
+        HConstants.LATEST_TIMESTAMP);
+    assertEquals(100, noOfRowsDeleted);
+    int rows = 0;
+    scan = new Scan ();
+    scan.setMaxVersions();
+    for (Result result : ht.getScanner(scan)) {
+      assertEquals(3, result.getFamilyMap(FAMILY1).size());
+      List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER1);
+      assertEquals(1, column.size());
+      assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
+      
+      column = result.getColumn(FAMILY1, QUALIFIER2);
+      assertEquals(1, column.size());
+      assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
+      
+      column = result.getColumn(FAMILY1, QUALIFIER3);
+      assertEquals(1, column.size());
+      assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
+      rows++;
+    }
+    assertEquals(100, rows);
+  }
+  
+  @Test
+  public void testBulkDeleteColumnVersionBasedOnTS() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersionBasedOnTS");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      Put put = new Put(Bytes.toBytes(j));
+      // TS = 1000L
+      byte[] value = "v1".getBytes();
+      put.add(FAMILY1, QUALIFIER1, 1000L, value);
+      put.add(FAMILY1, QUALIFIER2, 1000L, value);
+      put.add(FAMILY1, QUALIFIER3, 1000L, value);
+      // TS = 1234L
+      value = "v2".getBytes();
+      put.add(FAMILY1, QUALIFIER1, 1234L, value);
+      put.add(FAMILY1, QUALIFIER2, 1234L, value);
+      put.add(FAMILY1, QUALIFIER3, 1234L, value);
+      // Latest version values
+      value = "v3".getBytes();
+      put.add(FAMILY1, QUALIFIER1, value);
+      put.add(FAMILY1, QUALIFIER2, value);
+      put.add(FAMILY1, QUALIFIER3, value);
+      puts.add(put);
+    }
+    ht.put(puts);
+    Scan scan = new Scan ();
+    scan.addColumn(FAMILY1, QUALIFIER3);
+    // Delete the column cf1:c3's one version at TS=1234 
+    long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, 1234L);
+    assertEquals(100, noOfRowsDeleted);
+    int rows = 0;
+    scan = new Scan ();
+    scan.setMaxVersions();
+    for (Result result : ht.getScanner(scan)) {
+      assertEquals(3, result.getFamilyMap(FAMILY1).size());
+      assertEquals(3, result.getColumn(FAMILY1, QUALIFIER1).size());
+      assertEquals(3, result.getColumn(FAMILY1, QUALIFIER2).size());
+      List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER3);
+      assertEquals(2, column.size());
+      assertTrue(Bytes.equals("v3".getBytes(), column.get(0).getValue()));
+      assertTrue(Bytes.equals("v1".getBytes(), column.get(1).getValue()));
+      rows++;
+    }
+    assertEquals(100, rows);
+  }
+  
+  @Test
+  public void testBulkDeleteWithNumberOfVersions() throws Throwable {
+    byte[] tableName = Bytes.toBytes("testBulkDeleteWithNumberOfVersions");
+    HTable ht = createTable(tableName);
+    List<Put> puts = new ArrayList<Put>(100);
+    for (int j = 0; j < 100; j++) {
+      Put put = new Put(Bytes.toBytes(j));
+      // TS = 1000L
+      byte[] value = "v1".getBytes();
+      put.add(FAMILY1, QUALIFIER1, 1000L, value);
+      put.add(FAMILY1, QUALIFIER2, 1000L, value);
+      put.add(FAMILY1, QUALIFIER3, 1000L, value);
+      // TS = 1234L
+      value = "v2".getBytes();
+      put.add(FAMILY1, QUALIFIER1, 1234L, value);
+      put.add(FAMILY1, QUALIFIER2, 1234L, value);
+      put.add(FAMILY1, QUALIFIER3, 1234L, value);
+      // TS = 2000L
+      value = "v3".getBytes();
+      put.add(FAMILY1, QUALIFIER1, 2000L, value);
+      put.add(FAMILY1, QUALIFIER2, 2000L, value);
+      put.add(FAMILY1, QUALIFIER3, 2000L, value);
+      // Latest version values
+      value = "v4".getBytes();
+      put.add(FAMILY1, QUALIFIER1, value);
+      put.add(FAMILY1, QUALIFIER2, value);
+      put.add(FAMILY1, QUALIFIER3, value);
+      puts.add(put);
+    }
+    ht.put(puts);
+    
+    // Delete all the versions of columns cf1:c1 and cf1:c2 falling with the time range
+    // [1000,2000)
+    final Scan scan = new Scan();
+    scan.addColumn(FAMILY1, QUALIFIER1);
+    scan.addColumn(FAMILY1, QUALIFIER2);
+    scan.setTimeRange(1000L, 2000L);
+    scan.setMaxVersions();
+    
+    long noOfDeletedRows = 0L;
+    long noOfVersionsDeleted = 0L;
+    Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable = 
+        new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
+      public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
+        return instance.delete(scan, DeleteType.VERSION, null, 500);
+      }
+    };
+    Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
+        scan.getStartRow(), scan.getStopRow(), callable);
+    for (BulkDeleteResponse response : result.values()) {
+      noOfDeletedRows += response.getRowsDeleted();
+      noOfVersionsDeleted += response.getVersionsDeleted();
+    }
+    assertEquals(100, noOfDeletedRows);
+    assertEquals(400, noOfVersionsDeleted);
+    
+    int rows = 0;
+    Scan scan1 = new Scan ();
+    scan1.setMaxVersions();
+    for (Result res : ht.getScanner(scan1)) {
+      assertEquals(3, res.getFamilyMap(FAMILY1).size());
+      List<KeyValue> column = res.getColumn(FAMILY1, QUALIFIER1);
+      assertEquals(2, column.size());
+      assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
+      assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
+      column = res.getColumn(FAMILY1, QUALIFIER2);
+      assertEquals(2, column.size());
+      assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
+      assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
+      assertEquals(4, res.getColumn(FAMILY1, QUALIFIER3).size());
+      rows++;
+    }
+    assertEquals(100, rows);
+  }
+  
+  private HTable createTable(byte[] tableName) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY1);
+    hcd.setMaxVersions(10);// Just setting 10 as I am not testing with more than 10 versions here
+    htd.addFamily(hcd);
+    TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
+    HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    return ht;
+  }
+
+  private Put createPut(byte[] rowkey, String value) throws IOException {
+    Put put = new Put(rowkey);
+    put.add(FAMILY1, QUALIFIER1, value.getBytes());
+    put.add(FAMILY1, QUALIFIER2, value.getBytes());
+    put.add(FAMILY1, QUALIFIER3, value.getBytes());
+    return put;
+  }
+}
\ No newline at end of file

From 770605db8c7f6e565f700eb07c8adf27cfb7ea8e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 23 Oct 2012 18:53:08 +0000
Subject: [PATCH 0539/1540] HBASE-6728 prevent OOM possibility due to per
 connection responseQueue being unbounded

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401382 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/HBaseServer.java  |  67 +++++++--
 .../hbase/regionserver/HRegionServer.java     |  14 +-
 .../metrics/RegionServerDynamicMetrics.java   |  19 ++-
 .../hadoop/hbase/util/SizeBasedThrottler.java | 131 +++++++++++++++++
 .../hbase/util/TestSizeBasedThrottler.java    | 135 ++++++++++++++++++
 5 files changed, 347 insertions(+), 19 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index e89be5d215cc..2a8efe2312e7 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -25,6 +25,7 @@
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.net.BindException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -48,12 +49,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -66,17 +67,17 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.ByteBufferOutputStream;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.SizeBasedThrottler;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.ipc.RPC.VersionMismatch;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.cliffc.high_scale_lib.Counter;
 
 import com.google.common.base.Function;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
-import org.cliffc.high_scale_lib.Counter;
-
 /** An abstract IPC service.  IPC calls take a single {@link Writable} as a
  * parameter, and return a {@link Writable} as their value.  A service runs on
  * a port and is defined by a parameter class and a value class.
@@ -209,6 +210,14 @@ public static String getRemoteAddress() {
   protected final boolean tcpKeepAlive; // if T then use keepalives
   protected final long purgeTimeout;    // in milliseconds
 
+  // responseQueuesSizeThrottler is shared among all responseQueues,
+  // it bounds memory occupied by responses in all responseQueues
+  final SizeBasedThrottler responseQueuesSizeThrottler;
+
+  // RESPONSE_QUEUE_MAX_SIZE limits total size of responses in every response queue
+  private static final long DEFAULT_RESPONSE_QUEUES_MAX_SIZE = 1024 * 1024 * 1024; // 1G
+  private static final String RESPONSE_QUEUES_MAX_SIZE = "ipc.server.response.queue.maxsize";
+
   volatile protected boolean running = true;         // true while server runs
   protected BlockingQueue<Call> callQueue; // queued calls
   protected final Counter callQueueSize = new Counter();
@@ -946,7 +955,7 @@ private boolean processResponse(final LinkedList<Call> responseQueue,
           //
           // Extract the first call
           //
-          call = responseQueue.removeFirst();
+          call = responseQueue.peek();
           SocketChannel channel = call.connection.channel;
           if (LOG.isDebugEnabled()) {
             LOG.debug(getName() + ": responding to #" + call.id + " from " +
@@ -957,9 +966,13 @@ private boolean processResponse(final LinkedList<Call> responseQueue,
           //
           int numBytes = channelWrite(channel, call.response);
           if (numBytes < 0) {
+            // Error flag is set, so returning here closes connection and
+            // clears responseQueue.                   
             return true;
           }
           if (!call.response.hasRemaining()) {
+            responseQueue.poll();
+            responseQueuesSizeThrottler.decrease(call.response.limit());    
             call.connection.decRpcCount();
             //noinspection RedundantIfStatement
             if (numElements == 1) {    // last call fully processes.
@@ -972,12 +985,6 @@ private boolean processResponse(final LinkedList<Call> responseQueue,
                         call.connection + " Wrote " + numBytes + " bytes.");
             }
           } else {
-            //
-            // If we were unable to write the entire response out, then
-            // insert in Selector queue.
-            //
-            call.connection.responseQueue.addFirst(call);
-
             if (inHandler) {
               // set the serve time when the response has to be sent later
               call.timestamp = System.currentTimeMillis();
@@ -1031,15 +1038,31 @@ void doRespond(Call call) throws IOException {
       call.timestamp = System.currentTimeMillis();
 
       boolean doRegister = false;
+      boolean closed;
+      try {
+        responseQueuesSizeThrottler.increase(call.response.remaining());
+      } catch (InterruptedException ie) {
+        throw new InterruptedIOException(ie.getMessage());
+      }
       synchronized (call.connection.responseQueue) {
-        call.connection.responseQueue.addLast(call);
-        if (call.connection.responseQueue.size() == 1) {
-          doRegister = !processResponse(call.connection.responseQueue, false);
+        closed = call.connection.closed;
+        if (!closed) {
+          call.connection.responseQueue.addLast(call);
+
+          if (call.connection.responseQueue.size() == 1) {
+            doRegister = !processResponse(call.connection.responseQueue, false);
+          }
         }
       }
       if (doRegister) {
         enqueueInSelector(call);
       }
+      if (closed) {
+        // Connection was closed when we tried to submit response, but we
+        // increased responseQueues size already. It shoud be
+        // decreased here.
+        responseQueuesSizeThrottler.decrease(call.response.remaining());
+      }      
     }
 
     private synchronized void incPending() {   // call waiting to be enqueued.
@@ -1064,6 +1087,8 @@ protected class Connection {
                                          //version are read
     private boolean headerRead = false;  //if the connection header that
                                          //follows version is read.
+
+    protected volatile boolean closed = false;    // indicates if connection was closed
     protected SocketChannel channel;
     private ByteBuffer data;
     private ByteBuffer dataLengthBuffer;
@@ -1313,6 +1338,7 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
     }
 
     protected synchronized void close() {
+      closed = true;
       data = null;
       dataLengthBuffer = null;
       if (!channel.isOpen())
@@ -1542,6 +1568,9 @@ protected HBaseServer(String bindAddress, int port,
     this.delayedCalls = new AtomicInteger(0);
 
 
+    this.responseQueuesSizeThrottler = new SizeBasedThrottler(
+        conf.getLong(RESPONSE_QUEUES_MAX_SIZE, DEFAULT_RESPONSE_QUEUES_MAX_SIZE));
+
     // Create the responder here
     responder = new Responder();
   }
@@ -1596,6 +1625,14 @@ protected void closeConnection(Connection connection) {
       }
     }
     connection.close();
+    long bytes = 0;
+    synchronized (connection.responseQueue) {
+      for (Call c : connection.responseQueue) {
+        bytes += c.response.limit();
+      }
+      connection.responseQueue.clear();
+    }
+    responseQueuesSizeThrottler.decrease(bytes);    
     rpcMetrics.numOpenConnections.set(numConnections);
   }
 
@@ -1813,4 +1850,8 @@ private static int channelIO(ReadableByteChannel readCh,
   public static RpcCallContext getCurrentCall() {
     return CurCall.get();
   }
+
+  public long getResponseQueueSize(){
+    return responseQueuesSizeThrottler.getCurrentValue();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index cb15bb6d95f7..7d9872f42b00 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -115,6 +115,7 @@
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
 import org.apache.hadoop.hbase.ipc.HBaseRpcMetrics;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
 import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.ipc.Invocation;
@@ -232,6 +233,9 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   // Server to handle client requests. Default access so can be accessed by
   // unit tests.
   RpcServer rpcServer;
+  
+  // Server to handle client requests.
+  private HBaseServer server;  
 
   private final InetSocketAddress isa;
 
@@ -422,6 +426,7 @@ public HRegionServer(Configuration conf)
         conf.getInt("hbase.regionserver.metahandler.count", 10),
         conf.getBoolean("hbase.rpc.verbose", false),
         conf, HConstants.QOS_THRESHOLD);
+    if (rpcServer instanceof HBaseServer) server = (HBaseServer) rpcServer;
     // Set our address.
     this.isa = this.rpcServer.getListenerAddress();
 
@@ -1007,7 +1012,7 @@ protected void handleReportForDutyResponse(final MapWritable c)
       this.hlog = setupWALAndReplication();
       // Init in here rather than in constructor after thread name has been set
       this.metrics = new RegionServerMetrics();
-      this.dynamicMetrics = RegionServerDynamicMetrics.newInstance();
+      this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(this);
       startServiceThreads();
       LOG.info("Serving as " + this.serverNameFromMasterPOV +
         ", RPC listening on " + this.isa +
@@ -3764,4 +3769,11 @@ public String getCompactionState(final byte[] regionName) throws IOException {
       HRegionInfo info = region.getRegionInfo();
       return CompactionRequest.getCompactionState(info.getRegionId()).name();
   }
+
+  public long getResponseQueueSize(){
+    if (server != null) {
+      return server.getResponseQueueSize();
+    }
+    return 0;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
index 44a6fadef60a..cd76211419d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
@@ -24,12 +24,12 @@
 import java.lang.reflect.Method;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.metrics.MetricsContext;
 import org.apache.hadoop.metrics.MetricsRecord;
@@ -58,6 +58,7 @@ public class RegionServerDynamicMetrics implements Updater {
   private MetricsContext context;
   private final RegionServerDynamicStatistics rsDynamicStatistics;
   private Method updateMbeanInfoIfMetricsListChanged = null;
+  private HRegionServer regionServer;
   private static final Log LOG =
     LogFactory.getLog(RegionServerDynamicStatistics.class);
   
@@ -73,13 +74,14 @@ public class RegionServerDynamicMetrics implements Updater {
    */
   public final MetricsRegistry registry = new MetricsRegistry();
 
-  private RegionServerDynamicMetrics() {
+  private RegionServerDynamicMetrics(HRegionServer regionServer) {
     this.context = MetricsUtil.getContext("hbase");
     this.metricsRecord = MetricsUtil.createRecord(
                             this.context,
                             "RegionServerDynamicStatistics");
     context.registerUpdater(this);
     this.rsDynamicStatistics = new RegionServerDynamicStatistics(this.registry);
+    this.regionServer = regionServer;
     try {
       updateMbeanInfoIfMetricsListChanged =
         this.rsDynamicStatistics.getClass().getSuperclass()
@@ -91,9 +93,9 @@ private RegionServerDynamicMetrics() {
     }
   }
 
-  public static RegionServerDynamicMetrics newInstance() {
+  public static RegionServerDynamicMetrics newInstance(HRegionServer regionServer) {
     RegionServerDynamicMetrics metrics =
-      new RegionServerDynamicMetrics();
+      new RegionServerDynamicMetrics(regionServer);
     return metrics;
   }
 
@@ -183,6 +185,13 @@ public void doUpdates(MetricsContext context) {
     for (Entry<String, AtomicLong> entry : RegionMetricsStorage.getNumericMetrics().entrySet()) {
       this.setNumericMetric(entry.getKey(), entry.getValue().getAndSet(0));
     }
+
+    /* export estimated size of all response queues */
+    if (regionServer != null) {
+      long responseQueueSize = regionServer.getResponseQueueSize();
+      this.setNumericMetric("responseQueuesSize", responseQueueSize);
+    }
+
     /* get dynamically created numeric metrics, and push the metrics.
      * These ones aren't to be reset; they are cumulative. */
     for (Entry<String, AtomicLong> entry : RegionMetricsStorage.getNumericPersistentMetrics().entrySet()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java b/src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java
new file mode 100644
index 000000000000..af578878ed87
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Utility class that can be used to implement
+ * queues with limited capacity (in terms of memory).
+ * It maintains internal counter and provides
+ * two operations: increase and decrease.
+ * Increase blocks until internal counter is lower than
+ * given threshold and then increases internal counter.
+ * Decrease decreases internal counter and wakes up
+ * waiting threads if counter is lower than threshold.
+ *
+ * This implementation allows you to set the value of internal
+ * counter to be greater than threshold. It happens
+ * when internal counter is lower than threshold and
+ * increase method is called with parameter 'delta' big enough
+ * so that sum of delta and internal counter is greater than
+ * threshold. This is not a bug, this is a feature.
+ * It solves some problems:
+ *   - thread calling increase with big parameter will not be
+ *     starved by other threads calling increase with small
+ *     arguments.
+ *   - thread calling increase with argument greater than
+ *     threshold won't deadlock. This is useful when throttling
+ *     queues - you can submit object that is bigger than limit.
+ *
+ * This implementation introduces small costs in terms of
+ * synchronization (no synchronization in most cases at all), but is
+ * vulnerable to races. For details see documentation of
+ * increase method.
+ */
+public class SizeBasedThrottler {
+
+  private final long threshold;
+  private final AtomicLong currentSize;
+
+  /**
+   * Creates SizeBoundary with provided threshold
+   *
+   * @param threshold threshold used by instance
+   */
+  public SizeBasedThrottler(long threshold) {
+    if (threshold <= 0) {
+      throw new IllegalArgumentException("Treshold must be greater than 0");
+    }
+    this.threshold = threshold;
+    this.currentSize = new AtomicLong(0);
+  }
+
+  /**
+   * Blocks until internal counter is lower than threshold
+   * and then increases value of internal counter.
+   *
+   * THIS METHOD IS VULNERABLE TO RACES.
+   * It may happen that increment operation will
+   * succeed immediately, even if it should block. This happens when
+   * at least two threads call increase at the some moment. The decision
+   * whether to block is made at the beginning, without synchronization.
+   * If value of currentSize is lower than threshold at that time, call
+   * will succeed immediately. It is possible, that 2 threads will make
+   * decision not to block, even if one of them should block.
+   *
+   * @param delta increase internal counter by this value
+   * @return new value of internal counter
+   * @throws InterruptedException when interrupted during waiting
+   */
+  public synchronized long increase(long delta) throws InterruptedException{
+    if (currentSize.get() >= threshold) {
+      synchronized (this) {
+        while (currentSize.get() >= threshold) {
+          wait();
+        }
+      }
+    }
+
+    return currentSize.addAndGet(delta);
+  }
+
+
+  /**
+   * Decreases value of internal counter. Wakes up waiting threads if required.
+   *
+   * @param delta decrease internal counter by this value
+   * @return new value of internal counter
+   */
+  public synchronized long decrease(long delta) {
+    final long newSize = currentSize.addAndGet(-delta);
+
+    if (newSize < threshold && newSize + delta >= threshold) {
+      synchronized (this) {
+        notifyAll();
+      }
+    }
+
+    return newSize;
+  }
+
+  /**
+   *
+   * @return current value of internal counter
+   */
+  public synchronized long getCurrentValue(){
+    return currentSize.get();
+  }
+
+  /**
+   * @return threshold
+   */
+  public long getThreshold(){
+    return threshold;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
new file mode 100644
index 000000000000..971c0247a6b2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * This tests some race conditions that can happen
+ * occasionally, but not every time.
+ */
+@Category(MediumTests.class)
+public class TestSizeBasedThrottler {
+
+  private static final int REPEATS = 100;
+
+  private Thread makeThread(final SizeBasedThrottler throttler,
+      final AtomicBoolean failed, final int delta,
+      final int limit, final CountDownLatch latch) {
+
+    Thread ret = new Thread(new Runnable() {
+
+      @Override
+      public void run() {
+        try {
+          latch.await();
+          if (throttler.increase(delta) > limit) {
+            failed.set(true);
+          }
+          throttler.decrease(delta);
+        } catch (Exception e) {
+          failed.set(true);
+        }
+      }
+    });
+
+    ret.start();
+    return ret;
+  }
+
+  private void runGenericTest(int threshold, int delta, int maxValueAllowed,
+      int numberOfThreads, long timeout) {
+    SizeBasedThrottler throttler = new SizeBasedThrottler(threshold);
+    AtomicBoolean failed = new AtomicBoolean(false);
+
+    ArrayList<Thread> threads = new ArrayList<Thread>(numberOfThreads);
+    CountDownLatch latch = new CountDownLatch(1);
+    long timeElapsed = 0;
+
+    for (int i = 0; i < numberOfThreads; ++i) {
+      threads.add(makeThread(throttler, failed, delta, maxValueAllowed, latch));
+    }
+
+    latch.countDown();
+    for (Thread t : threads) {
+      try {
+        long beforeJoin = System.currentTimeMillis();
+        t.join(timeout - timeElapsed);
+        timeElapsed += System.currentTimeMillis() - beforeJoin;
+        if (t.isAlive() || timeElapsed >= timeout) {
+          fail("Timeout reached.");
+        }
+      } catch (InterruptedException e) {
+        fail("Got InterruptedException");
+      }
+    }
+
+    assertFalse(failed.get());
+  }
+
+  @Test
+  public void testSmallIncreases(){
+    for (int i = 0; i < REPEATS; ++i) {
+      runGenericTest(
+          10, // threshold
+          1,  // delta
+          15, // fail if throttler's value
+              // exceeds 15
+          1000, // use 1000 threads
+          200 // wait for 200ms
+          );
+    }
+  }
+
+  @Test
+  public void testBigIncreases() {
+    for (int i = 0; i < REPEATS; ++i) {
+      runGenericTest(
+          1, // threshold
+          2, // delta
+          4, // fail if throttler's value
+             // exceeds 4
+          1000, // use 1000 threads
+          200 // wait for 200ms
+          );
+    }
+  }
+
+  @Test
+  public void testIncreasesEqualToThreshold(){
+    for (int i = 0; i < REPEATS; ++i) {
+      runGenericTest(
+          1, // threshold
+          1, // delta
+          2, // fail if throttler's value
+             // exceeds 2
+          1000, // use 1000 threads
+          200 // wait for 200ms
+          );
+    }
+  }
+}

From 863212972a9d9a8a8bbe1fc60a924ae3071af771 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Wed, 24 Oct 2012 01:10:48 +0000
Subject: [PATCH 0540/1540] HBASE-7018 Fix and Improve TableDescriptor caching
 for bulk assignment

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401526 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/TableDescriptors.java |  9 +---
 .../hbase/regionserver/HRegionServer.java     |  2 +-
 .../hadoop/hbase/util/FSTableDescriptors.java | 54 +++++++++++--------
 .../hbase/master/TestCatalogJanitor.java      |  5 +-
 4 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index 145589504b5e..58c211bd8821 100644
--- a/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Map;
 
@@ -29,22 +28,18 @@ public interface TableDescriptors {
   /**
    * @param tablename
    * @return HTableDescriptor for tablename
-   * @throws TableExistsException
-   * @throws FileNotFoundException
    * @throws IOException
    */
   public HTableDescriptor get(final String tablename)
-  throws FileNotFoundException, IOException;
+  throws IOException;
 
   /**
    * @param tablename
    * @return HTableDescriptor for tablename
-   * @throws TableExistsException
-   * @throws FileNotFoundException
    * @throws IOException
    */
   public HTableDescriptor get(final byte[] tablename)
-  throws FileNotFoundException, IOException;
+  throws IOException;
 
   /**
    * Get Map of all HTableDescriptors. Populates the descriptor cache as a
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7d9872f42b00..44cc5c123cce 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2773,7 +2773,7 @@ private RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNo
       htd = htds.get(region.getTableNameAsString());
       if (htd == null) {
         htd = this.tableDescriptors.get(region.getTableName());
-        htds.put(region.getRegionNameAsString(), htd);
+        htds.put(region.getTableNameAsString(), htd);
       }
     }
     this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(),
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 4913a19bb5f4..51411219a313 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -122,7 +122,7 @@ public FSTableDescriptors(final FileSystem fs, final Path rootdir,
    */
   @Override
   public HTableDescriptor get(final byte [] tablename)
-  throws FileNotFoundException, IOException {
+  throws IOException {
     return get(Bytes.toString(tablename));
   }
 
@@ -131,7 +131,7 @@ public HTableDescriptor get(final byte [] tablename)
    */
   @Override
   public HTableDescriptor get(final String tablename)
-  throws FileNotFoundException, IOException {
+  throws IOException {
     invocations++;
     if (HTableDescriptor.ROOT_TABLEDESC.getNameAsString().equals(tablename)) {
       cachehits++;
@@ -148,20 +148,19 @@ public HTableDescriptor get(final String tablename)
     }
 
     // Look in cache of descriptors.
-    TableDescriptorModtime tdm = this.cache.get(tablename);
+    TableDescriptorModtime cachedtdm = this.cache.get(tablename);
 
-    // Check mod time has not changed (this is trip to NN).
-    long modtime = getTableInfoModtime(this.fs, this.rootdir, tablename);
-    if (tdm != null) {
-      if (modtime <= tdm.getModtime()) {
+    if (cachedtdm != null) {
+      // Check mod time has not changed (this is trip to NN).
+      if (getTableInfoModtime(this.fs, this.rootdir, tablename) <= cachedtdm.getModtime()) {
         cachehits++;
-        return tdm.getTableDescriptor();
+        return cachedtdm.getTableDescriptor();
       }
     }
     
-    HTableDescriptor htd = null;
+    TableDescriptorModtime tdmt = null;
     try {
-       htd = getTableDescriptor(this.fs, this.rootdir, tablename);
+      tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename);
     } catch (NullPointerException e) {
       LOG.debug("Exception during readTableDecriptor. Current table name = "
           + tablename, e);
@@ -170,14 +169,14 @@ public HTableDescriptor get(final String tablename)
           + tablename, ioe);
     }
     
-    if (htd == null) {
+    if (tdmt == null) {
       LOG.warn("The following folder is in HBase's root directory and " +
         "doesn't contain a table descriptor, " +
         "do consider deleting it: " + tablename);
     } else {
-      this.cache.put(tablename, new TableDescriptorModtime(modtime, htd));
+      this.cache.put(tablename, tdmt);
     }
-    return htd;
+    return tdmt == null ? null : tdmt.getTableDescriptor();
   }
 
   /* (non-Javadoc)
@@ -231,7 +230,7 @@ public HTableDescriptor remove(final String tablename)
       }
     }
     TableDescriptorModtime tdm = this.cache.remove(tablename);
-    return tdm == null? null: tdm.getTableDescriptor();
+    return tdm == null ? null : tdm.getTableDescriptor();
   }
 
   /**
@@ -383,7 +382,9 @@ public static HTableDescriptor getTableDescriptor(FileSystem fs,
   throws IOException {
      HTableDescriptor htd = null;
      try {
-       htd = getTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
+       TableDescriptorModtime tdmt =
+         getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName));
+       htd = tdmt == null ? null : tdmt.getTableDescriptor();
      } catch (NullPointerException e) {
        LOG.debug("Exception during readTableDecriptor. Current table name = "
            + Bytes.toString(tableName), e);
@@ -392,19 +393,23 @@ public static HTableDescriptor getTableDescriptor(FileSystem fs,
   }
 
   static HTableDescriptor getTableDescriptor(FileSystem fs,
+      Path hbaseRootDir, String tableName) throws NullPointerException, IOException {
+    TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, hbaseRootDir, tableName);
+    return tdmt == null ? null : tdmt.getTableDescriptor();
+  }
+
+  static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs,
       Path hbaseRootDir, String tableName) throws NullPointerException, IOException{
-    HTableDescriptor htd = null;
     // ignore both -ROOT- and .META. tables
     if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0
         || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) {
       return null;
     }
-    htd = getTableDescriptor(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
-    return htd;
+    return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
   }
 
-  public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
-  throws IOException, NullPointerException {
+  static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir)
+  throws NullPointerException, IOException {
     if (tableDir == null) throw new NullPointerException();
     FileStatus status = getTableInfoPath(fs, tableDir);
     if (status == null) {
@@ -419,8 +424,15 @@ public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
     } finally {
       fsDataInputStream.close();
     }
-    return hTableDescriptor;
+    return new TableDescriptorModtime(status.getModificationTime(), hTableDescriptor);
+  }
+
+  public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
+  throws IOException, NullPointerException {
+    TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, tableDir);
+    return tdmt == null? null: tdmt.getTableDescriptor();
   }
+ 
 
   /**
    * Update table descriptor
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 47bc119ee651..b4afa3bebbdf 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -27,7 +27,6 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -247,13 +246,13 @@ public Map<String, HTableDescriptor> getAll() throws IOException {
         
         @Override
         public HTableDescriptor get(byte[] tablename)
-        throws FileNotFoundException, IOException {
+        throws IOException {
           return get(Bytes.toString(tablename));
         }
         
         @Override
         public HTableDescriptor get(String tablename)
-        throws FileNotFoundException, IOException {
+        throws IOException {
           return createHTableDescriptor();
         }
         

From 8c3657702c79f8c72145c128df91f6a899c959ab Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 24 Oct 2012 04:47:20 +0000
Subject: [PATCH 0541/1540] HBASE-6843 loading lzo error when using coprocessor
 (Andy)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401550 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
index 4d58b2c8948d..7d0ba922d3c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
@@ -67,6 +67,7 @@ public class CoprocessorClassLoader extends URLClassLoader {
     "sunw.",
     // Hadoop/HBase:
     "org.apache.hadoop",
+    "com.hadoop",
   };
   
   /**

From b6dea5fdf8501b7cbf60ea9e80af7aefbe9084ea Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 24 Oct 2012 06:10:36 +0000
Subject: [PATCH 0542/1540] HBASE-7037 ReplicationPeer logs at WARN level
 aborting server instead of at FATAL

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1401564 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/replication/ReplicationPeer.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 0fec27f427be..c972a5d92835 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -154,7 +154,7 @@ public Configuration getConfiguration() {
 
   @Override
   public void abort(String why, Throwable e) {
-    LOG.warn("The ReplicationPeer coresponding to peer " + clusterKey
+    LOG.fatal("The ReplicationPeer coresponding to peer " + clusterKey
         + " was aborted for the following reason(s):" + why, e);
   }
 

From ecb50e02fe29b1b9ea22ad76af70a3476c66cf7f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 25 Oct 2012 16:05:46 +0000
Subject: [PATCH 0543/1540] HBASE-7048 Regionsplitter requires the hadoop
 config path to be in hbase classpath (Ming Ma)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402198 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/RegionSplitter.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index 02718832b7f4..619a715196c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -631,7 +631,7 @@ static LinkedList<Pair<byte[], byte[]>> splitScan(
     Path hbDir = new Path(table.getConfiguration().get(HConstants.HBASE_DIR));
     Path tableDir = HTableDescriptor.getTableDir(hbDir, table.getTableName());
     Path splitFile = new Path(tableDir, "_balancedSplit");
-    FileSystem fs = FileSystem.get(table.getConfiguration());
+    FileSystem fs = tableDir.getFileSystem(table.getConfiguration());
 
     // clear the cache to forcibly refresh region information
     table.clearRegionCache();
@@ -712,7 +712,7 @@ static LinkedList<Pair<byte[], byte[]>> getSplits(HTable table,
     Path hbDir = new Path(table.getConfiguration().get(HConstants.HBASE_DIR));
     Path tableDir = HTableDescriptor.getTableDir(hbDir, table.getTableName());
     Path splitFile = new Path(tableDir, "_balancedSplit");
-    FileSystem fs = FileSystem.get(table.getConfiguration());
+    FileSystem fs = tableDir.getFileSystem(table.getConfiguration());
 
     // using strings because (new byte[]{0}).equals(new byte[]{0}) == false
     Set<Pair<String, String>> daughterRegions = Sets.newHashSet();

From 431218cc764364080fc9511c5be528c3ae09f653 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 25 Oct 2012 16:32:52 +0000
Subject: [PATCH 0544/1540] HBASE-5257 Allow INCLUDE_AND_NEXT_COL in filters
 and use it in ColumnPaginationFilter (Varun)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402210 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/filter/ColumnCountGetFilter.java    |  3 +-
 .../hbase/filter/ColumnPaginationFilter.java  |  6 +-
 .../apache/hadoop/hbase/filter/Filter.java    |  4 ++
 .../hadoop/hbase/filter/FilterList.java       | 10 ++-
 .../hbase/regionserver/ScanQueryMatcher.java  |  6 +-
 .../filter/TestColumnPaginationFilter.java    |  2 +-
 .../hadoop/hbase/filter/TestFilter.java       |  8 ++-
 .../hadoop/hbase/filter/TestFilterList.java   | 71 +++++++++++++++++++
 8 files changed, 103 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
index c6946bb5bcb0..2bca1c4be021 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
@@ -64,7 +64,8 @@ public boolean filterAllRemaining() {
   @Override
   public ReturnCode filterKeyValue(KeyValue v) {
     this.count++;
-    return filterAllRemaining() ? ReturnCode.SKIP: ReturnCode.INCLUDE;
+    return filterAllRemaining() ? ReturnCode.NEXT_COL:
+                                  ReturnCode.INCLUDE_AND_NEXT_COL;
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
index d6b128018363..1214a1afbd5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
@@ -30,7 +30,8 @@
 /**
  * A filter, based on the ColumnCountGetFilter, takes two arguments: limit and offset.
  * This filter can be used for row-based indexing, where references to other tables are stored across many columns,
- * in order to efficient lookups and paginated results for end users.
+ * in order to efficient lookups and paginated results for end users. Only most recent versions are considered
+ * for pagination.
  */
 public class ColumnPaginationFilter extends FilterBase
 {
@@ -76,7 +77,8 @@ public ReturnCode filterKeyValue(KeyValue v)
       return ReturnCode.NEXT_ROW;
     }
 
-    ReturnCode code = count < offset ? ReturnCode.SKIP : ReturnCode.INCLUDE;
+    ReturnCode code = count < offset ? ReturnCode.NEXT_COL :
+                                       ReturnCode.INCLUDE_AND_NEXT_COL;
     count++;
     return code;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
index 02ea5f5c97c0..ac1079f82bf4 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
@@ -112,6 +112,10 @@ public enum ReturnCode {
      * Include the KeyValue
      */
     INCLUDE,
+    /**
+     * Include the KeyValue and seek to the next column skipping older versions.
+     */
+    INCLUDE_AND_NEXT_COL,
     /**
      * Skip this KeyValue
      */
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index c3064e9b59c7..8e12d3933ea2 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -201,6 +201,9 @@ public ReturnCode filterKeyValue(KeyValue v) {
         }
         ReturnCode code = filter.filterKeyValue(v);
         switch (code) {
+        // Override INCLUDE and continue to evaluate.
+        case INCLUDE_AND_NEXT_COL:
+          rc = ReturnCode.INCLUDE_AND_NEXT_COL;
         case INCLUDE:
           continue;
         default:
@@ -213,7 +216,12 @@ public ReturnCode filterKeyValue(KeyValue v) {
 
         switch (filter.filterKeyValue(v)) {
         case INCLUDE:
-          rc = ReturnCode.INCLUDE;
+          if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) {
+            rc = ReturnCode.INCLUDE;
+          }
+          break;
+        case INCLUDE_AND_NEXT_COL:
+          rc = ReturnCode.INCLUDE_AND_NEXT_COL;
           // must continue here to evaluate all filters
         case NEXT_ROW:
         case SKIP:
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 3435d62cd4e6..d1cab8ef6167 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -337,8 +337,9 @@ public MatchCode match(KeyValue kv) throws IOException {
      * counter for even that KV which may be discarded later on by Filter. This
      * would lead to incorrect results in certain cases.
      */
+    ReturnCode filterResponse = ReturnCode.SKIP;
     if (filter != null) {
-      ReturnCode filterResponse = filter.filterKeyValue(kv);
+      filterResponse = filter.filterKeyValue(kv);
       if (filterResponse == ReturnCode.SKIP) {
         return MatchCode.SKIP;
       } else if (filterResponse == ReturnCode.NEXT_COL) {
@@ -360,6 +361,9 @@ public MatchCode match(KeyValue kv) throws IOException {
      */
     if (colChecker == MatchCode.SEEK_NEXT_ROW) {
       stickyNextRow = true;
+    } else if (filter != null && colChecker == MatchCode.INCLUDE &&
+               filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL) {
+      return MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
     }
     return colChecker;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java
index e26ce7e383ae..720d882d0fba 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java
@@ -81,7 +81,7 @@ private Filter serializationTest(Filter filter) throws Exception {
     private void basicFilterTests(ColumnPaginationFilter filter) throws Exception
     {
       KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1);
-      assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
+      assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE_AND_NEXT_COL);
     }
 
     /**
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index 130d84dd9819..d5837006ca48 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -1471,8 +1471,14 @@ private void verifyScanFullNoValues(Scan s, KeyValue [] kvs, boolean useLen)
 
 
   public void testColumnPaginationFilter() throws Exception {
+      // Test that the filter skips multiple column versions.
+      Put p = new Put(ROWS_ONE[0]);
+      p.setWriteToWAL(false);
+      p.add(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]);
+      this.region.put(p);
+      this.region.flushcache();
 
-     // Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row
+      // Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row
       KeyValue [] expectedKVs = {
         // testRowOne-0
         new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 2b69a6fb0fd6..05c0269e3d65 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -233,6 +233,77 @@ public void testSerialization() throws Exception {
     // TODO: Run TESTS!!!
   }
 
+  /**
+   * Test filterKeyValue logic.
+   * @throws Exception
+   */
+  public void testFilterKeyValue() throws Exception {
+    Filter includeFilter = new FilterBase() {
+      @Override
+      public Filter.ReturnCode filterKeyValue(KeyValue v) {
+        return Filter.ReturnCode.INCLUDE;
+      }
+
+      @Override
+      public void readFields(DataInput arg0) throws IOException {}
+
+      @Override
+      public void write(DataOutput arg0) throws IOException {}
+    };
+
+    Filter alternateFilter = new FilterBase() {
+      boolean returnInclude = true;
+
+      @Override
+      public Filter.ReturnCode filterKeyValue(KeyValue v) {
+        Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE :
+                                                       Filter.ReturnCode.SKIP;
+        returnInclude = !returnInclude;
+        return returnCode;
+      }
+
+      @Override
+      public void readFields(DataInput arg0) throws IOException {}
+
+      @Override
+      public void write(DataOutput arg0) throws IOException {}
+    };
+
+    Filter alternateIncludeFilter = new FilterBase() {
+      boolean returnIncludeOnly = false;
+
+      @Override
+      public Filter.ReturnCode filterKeyValue(KeyValue v) {
+        Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE :
+                                                           Filter.ReturnCode.INCLUDE_AND_NEXT_COL;
+        returnIncludeOnly = !returnIncludeOnly;
+        return returnCode;
+      }
+
+      @Override
+      public void readFields(DataInput arg0) throws IOException {}
+
+      @Override
+      public void write(DataOutput arg0) throws IOException {}
+    };
+
+    // Check must pass one filter.
+    FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE,
+        Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
+    // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL.
+    assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null));
+    // INCLUDE, SKIP, INCLUDE. 
+    assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
+
+    // Check must pass all filter.
+    FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL,
+        Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
+    // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL.
+    assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null));
+    // INCLUDE, SKIP, INCLUDE. 
+    assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null));
+  }
+
   /**
    * Test pass-thru of hints.
    */

From f959fa36c00a5548e19aaf60b735c6eab19138b1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 25 Oct 2012 16:44:37 +0000
Subject: [PATCH 0545/1540] HBASE-7040 Port HBASE-5867 Improve Compaction
 Throttle Default to 0.94 (Sergey Shelukhin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402215 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/CompactSplitThread.java      | 76 ++++++-------------
 .../hadoop/hbase/regionserver/Store.java      |  7 ++
 2 files changed, 30 insertions(+), 53 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 072f07cd7743..add8f75cec1b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -50,7 +50,6 @@ public class CompactSplitThread implements CompactionRequestor {
   private final ThreadPoolExecutor largeCompactions;
   private final ThreadPoolExecutor smallCompactions;
   private final ThreadPoolExecutor splits;
-  private final long throttleSize;
 
   /**
    * Splitting should not take place if the total number of regions exceed this.
@@ -71,22 +70,11 @@ public class CompactSplitThread implements CompactionRequestor {
         "hbase.regionserver.thread.compaction.large", 1));
     int smallThreads = conf.getInt(
         "hbase.regionserver.thread.compaction.small", 1);
-    if (conf.get("hbase.regionserver.thread.compaction.throttle") != null) {
-      throttleSize = conf.getLong(
-          "hbase.regionserver.thread.compaction.throttle", 0);
-    } else {
-      // we have a complicated default. see HBASE-3877
-      long flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
-          HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
-      long splitSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
-          HConstants.DEFAULT_MAX_FILE_SIZE);
-      throttleSize = Math.min(flushSize * 2, splitSize / 2);
-    }
 
     int splitThreads = conf.getInt("hbase.regionserver.thread.split", 1);
 
     // if we have throttle threads, make sure the user also specified size
-    Preconditions.checkArgument(smallThreads == 0 || throttleSize > 0);
+    Preconditions.checkArgument(largeThreads > 0 && smallThreads > 0);
 
     final String n = Thread.currentThread().getName();
 
@@ -102,22 +90,18 @@ public Thread newThread(Runnable r) {
       });
     this.largeCompactions
         .setRejectedExecutionHandler(new CompactionRequest.Rejection());
-    if (smallThreads <= 0) {
-      this.smallCompactions = null;
-    } else {
-      this.smallCompactions = new ThreadPoolExecutor(smallThreads, smallThreads,
-          60, TimeUnit.SECONDS, new PriorityBlockingQueue<Runnable>(),
-          new ThreadFactory() {
-            @Override
-            public Thread newThread(Runnable r) {
-              Thread t = new Thread(r);
-              t.setName(n + "-smallCompactions-" + System.currentTimeMillis());
-              return t;
-            }
-        });
-      this.smallCompactions
-          .setRejectedExecutionHandler(new CompactionRequest.Rejection());
-    }
+    this.smallCompactions = new ThreadPoolExecutor(smallThreads, smallThreads,
+        60, TimeUnit.SECONDS, new PriorityBlockingQueue<Runnable>(),
+        new ThreadFactory() {
+          @Override
+          public Thread newThread(Runnable r) {
+            Thread t = new Thread(r);
+            t.setName(n + "-smallCompactions-" + System.currentTimeMillis());
+            return t;
+          }
+      });
+    this.smallCompactions
+        .setRejectedExecutionHandler(new CompactionRequest.Rejection());
     this.splits = (ThreadPoolExecutor)
         Executors.newFixedThreadPool(splitThreads,
             new ThreadFactory() {
@@ -132,14 +116,12 @@ public Thread newThread(Runnable r) {
 
   @Override
   public String toString() {
-    return "compaction_queue="
-        + (smallCompactions != null ? "("
-            + largeCompactions.getQueue().size() + ":"
-            + smallCompactions.getQueue().size() + ")"
-            : largeCompactions.getQueue().size())
+    return "compaction_queue=("
+        + largeCompactions.getQueue().size() + ":"
+        + smallCompactions.getQueue().size() + ")"
         + ", split_queue=" + splits.getQueue().size();
   }
-  
+
   public String dumpQueue() {
     StringBuffer queueLists = new StringBuffer();
     queueLists.append("Compaction/Split Queue dump:\n");
@@ -238,17 +220,11 @@ public synchronized void requestCompaction(final HRegion r, final Store s,
       if (priority != Store.NO_PRIORITY) {
         cr.setPriority(priority);
       }
-      ThreadPoolExecutor pool = largeCompactions;
-      if (smallCompactions != null && throttleSize > cr.getSize()) {
-        // smallCompactions is like the 10 items or less line at Walmart
-        pool = smallCompactions;
-      }
+      ThreadPoolExecutor pool = s.throttleCompaction(cr.getSize())
+          ? largeCompactions : smallCompactions;
       pool.execute(cr);
       if (LOG.isDebugEnabled()) {
-        String type = "";
-        if (smallCompactions != null) {
-          type = (pool == smallCompactions) ? "Small " : "Large ";
-        }
+        String type = (pool == smallCompactions) ? "Small " : "Large ";
         LOG.debug(type + "Compaction requested: " + cr
             + (why != null && !why.isEmpty() ? "; Because: " + why : "")
             + "; " + this);
@@ -267,8 +243,7 @@ public synchronized void requestCompaction(final HRegion r, final Store s,
   void interruptIfNecessary() {
     splits.shutdown();
     largeCompactions.shutdown();
-    if (smallCompactions != null)
-      smallCompactions.shutdown();
+    smallCompactions.shutdown();
   }
 
   private void waitFor(ThreadPoolExecutor t, String name) {
@@ -286,9 +261,7 @@ private void waitFor(ThreadPoolExecutor t, String name) {
   void join() {
     waitFor(splits, "Split Thread");
     waitFor(largeCompactions, "Large Compaction Thread");
-    if (smallCompactions != null) {
-      waitFor(smallCompactions, "Small Compaction Thread");
-    }
+    waitFor(smallCompactions, "Small Compaction Thread");
   }
 
   /**
@@ -298,10 +271,7 @@ void join() {
    * @return The current size of the regions queue.
    */
   public int getCompactionQueueSize() {
-    int size = largeCompactions.getQueue().size();
-    if (smallCompactions != null)
-      size += smallCompactions.getQueue().size();
-    return size;
+    return largeCompactions.getQueue().size() + smallCompactions.getQueue().size();
   }
 
   private boolean shouldSplitRegion() {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 5da808e0f397..6ef155e5c159 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -2181,6 +2181,13 @@ public int getCompactPriority(int priority) {
     }
   }
 
+  boolean throttleCompaction(long compactionSize) {
+    long throttlePoint = conf.getLong(
+        "hbase.regionserver.thread.compaction.throttle",  
+        2 * this.minFilesToCompact * this.region.memstoreFlushSize);  
+    return compactionSize > throttlePoint;  
+  } 
+
   public HRegion getHRegion() {
     return this.region;
   }

From ced2ecb322e306ebe621b3067a52a483d11ab933 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 26 Oct 2012 04:08:50 +0000
Subject: [PATCH 0546/1540] HBASE-7053 port blockcache configurability (part of
 HBASE-6312, and HBASE-7033) to 0.94 (Sergey Shelukhin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402385 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/CacheConfig.java    |  2 +-
 .../hbase/io/hfile/DoubleBlockCache.java      |  2 +-
 .../hadoop/hbase/io/hfile/LruBlockCache.java  | 20 +++++++++++++------
 .../io/hfile/TestHFileDataBlockEncoder.java   |  2 +-
 .../hbase/io/hfile/TestLruBlockCache.java     | 12 +++++++----
 5 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 70dbbbaf9734..13585b9f8827 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -349,7 +349,7 @@ private static synchronized BlockCache instantiateBlockCache(
       StringUtils.humanReadableInt(cacheSize));
     if (offHeapCacheSize <= 0) {
       globalBlockCache = new LruBlockCache(cacheSize,
-          StoreFile.DEFAULT_BLOCKSIZE_SMALL);
+          StoreFile.DEFAULT_BLOCKSIZE_SMALL, conf);
     } else {
       globalBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
           StoreFile.DEFAULT_BLOCKSIZE_SMALL, blockSize, conf);
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
index 1b721df8913b..ac36da4f3c4b 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
@@ -65,7 +65,7 @@ public DoubleBlockCache(long onHeapSize, long offHeapSize,
         + StringUtils.humanReadableInt(onHeapSize)
         + "bytes with an average block size of "
         + StringUtils.humanReadableInt(onHeapBlockSize) + " bytes.");
-    onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize);
+    onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize, conf);
 
     LOG.info("Creating off-heap cache of size "
         + StringUtils.humanReadableInt(offHeapSize)
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 7299e0392a7d..ad0878b58c86 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -96,6 +96,9 @@ public class LruBlockCache implements BlockCache, HeapSize {
 
   static final Log LOG = LogFactory.getLog(LruBlockCache.class);
 
+  static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
+  static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
+
   /** Default Configuration Parameters*/
 
   /** Backing Concurrent Map Configuration */
@@ -178,23 +181,28 @@ public class LruBlockCache implements BlockCache, HeapSize {
    * this class.
    * @param maxSize maximum size of cache, in bytes
    * @param blockSize approximate size of each block, in bytes
+   * @param conf configuration
    */
-  public LruBlockCache(long maxSize, long blockSize) {
-    this(maxSize, blockSize, true);
+  public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
+    this(maxSize, blockSize, true, conf);
   }
 
   /**
    * Constructor used for testing.  Allows disabling of the eviction thread.
    */
-  public LruBlockCache(long maxSize, long blockSize, boolean evictionThread) {
+  public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
     this(maxSize, blockSize, evictionThread,
         (int)Math.ceil(1.2*maxSize/blockSize),
-        DEFAULT_LOAD_FACTOR, DEFAULT_CONCURRENCY_LEVEL,
-        DEFAULT_MIN_FACTOR, DEFAULT_ACCEPTABLE_FACTOR,
-        DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR,
+        DEFAULT_LOAD_FACTOR, 
+        DEFAULT_CONCURRENCY_LEVEL,
+        conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR), 
+        conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR), 
+        DEFAULT_SINGLE_FACTOR, 
+        DEFAULT_MULTI_FACTOR,
         DEFAULT_MEMORY_FACTOR);
   }
 
+
   /**
    * Configurable constructor.  Use this constructor if not using defaults.
    * @param maxSize maximum size of this cache, in bytes
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
index 613ad7da816a..16323a646e49 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
@@ -93,7 +93,7 @@ public void tearDown() throws IOException {
   public void testEncodingWithCache() {
     HFileBlock block = getSampleHFileBlock();
     LruBlockCache blockCache =
-        new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
+        new LruBlockCache(8 * 1024 * 1024, 32 * 1024, TEST_UTIL.getConfiguration());
     HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(block, false);
     BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
     blockCache.cacheBlock(cacheKey, cacheBlock);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index d980948fe250..d1b3c417ec76 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -24,6 +24,8 @@
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
@@ -51,6 +53,8 @@
 public class TestLruBlockCache {
 
   private Map<String, Long> startingMetrics;
+  private final HBaseTestingUtility TEST_UTIL =
+      new HBaseTestingUtility();
 
   public TestLruBlockCache(boolean useTableName) {
     SchemaMetrics.setUseTableNameInTest(useTableName);
@@ -77,7 +81,7 @@ public void testBackgroundEvictionThread() throws Exception {
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 9); // room for 9, will evict
 
-    LruBlockCache cache = new LruBlockCache(maxSize,blockSize);
+    LruBlockCache cache = new LruBlockCache(maxSize, blockSize, TEST_UTIL.getConfiguration());
 
     CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
 
@@ -104,7 +108,7 @@ public void testCacheSimple() throws Exception {
     long maxSize = 1000000;
     long blockSize = calculateBlockSizeDefault(maxSize, 101);
 
-    LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
+    LruBlockCache cache = new LruBlockCache(maxSize, blockSize, TEST_UTIL.getConfiguration());
 
     CachedItem [] blocks = generateRandomBlocks(100, blockSize);
 
@@ -164,7 +168,7 @@ public void testCacheEvictionSimple() throws Exception {
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 10);
 
-    LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
+    LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, TEST_UTIL.getConfiguration());
 
     CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
 
@@ -205,7 +209,7 @@ public void testCacheEvictionTwoPriorities() throws Exception {
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 10);
 
-    LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
+    LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, TEST_UTIL.getConfiguration());
 
     CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single");
     CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi");

From 5be3f349811511c51cc8cbe5194eda32b1e62a65 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 26 Oct 2012 05:16:39 +0000
Subject: [PATCH 0547/1540] HBASE-6852 SchemaMetrics.updateOnCacheHit costs too
 much while full scanning a table with all of its fields (Cheng Hao and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402392 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  2 +
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  2 +
 .../regionserver/metrics/SchemaMetrics.java   | 79 +++++++++++++++++--
 .../metrics/TestSchemaMetrics.java            |  2 +
 4 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index b93c5c483104..961daddafcd7 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -391,6 +391,8 @@ public void close(boolean evictOnClose) throws IOException {
       this.istream.close();
       this.istream = null;
     }
+    
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV1
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 757990f845f4..ccee0d02a804 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -424,6 +424,8 @@ public void close(boolean evictOnClose) throws IOException {
         istream = null;
       }
     }
+    
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV2
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index e73513ceb953..61d2ef969c90 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -29,6 +29,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -38,7 +39,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -236,6 +236,9 @@ public String toString() {
   public static final SchemaMetrics ALL_SCHEMA_METRICS =
     getInstance(TOTAL_KEY, TOTAL_KEY);
 
+  /** Threshold for flush the metrics, currently used only for "on cache hit" */
+  private static final long THRESHOLD_METRICS_FLUSH = 100l;
+
   /**
    * Whether to include table name in metric names. If this is null, it has not
    * been initialized. This is a global instance, but we also have a copy of it
@@ -254,6 +257,8 @@ public String toString() {
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
   private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
+  private final AtomicLongArray onHitCacheMetrics= 
+      new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length);
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -261,6 +266,9 @@ private SchemaMetrics(final String tableName, final String cfName) {
 
     for (BlockCategory blockCategory : BlockCategory.values()) {
       for (boolean isCompaction : BOOL_VALUES) {
+        // initialize the cache metrics
+        onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0);
+        
         for (BlockMetricType metricType : BlockMetricType.values()) {
           if (!metricType.compactionAware && isCompaction) {
             continue;
@@ -336,6 +344,11 @@ public static SchemaMetrics getInstance(String tableName, String cfName) {
     return existingMetrics != null ? existingMetrics : schemaMetrics;
   }
 
+  private static final int getCacheHitMetricIndex (BlockCategory blockCategory,
+      boolean isCompaction) {
+    return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0);
+  }
+  
   private static final int getBlockMetricIndex(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
     int i = 0;
@@ -365,11 +378,20 @@ public String getBloomMetricName(boolean isInBloom) {
    */
   private void incrNumericMetric(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
+    incrNumericMetric (blockCategory, isCompaction, metricType, 1);
+  }
+  
+  /**
+   * Increments the given metric, both per-CF and aggregate, for both the given
+   * category and all categories in aggregate (four counters total).
+   */
+  private void incrNumericMetric(BlockCategory blockCategory,
+      boolean isCompaction, BlockMetricType metricType, long amount) {
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
     RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
-        isCompaction, metricType), 1);
+        isCompaction, metricType), amount);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
       incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction,
@@ -445,14 +467,59 @@ public void updatePersistentStoreMetric(StoreMetricType storeMetricType,
    */
   public void updateOnCacheHit(BlockCategory blockCategory,
       boolean isCompaction) {
+    updateOnCacheHit (blockCategory, isCompaction, 1);
+  }
+  
+  /**
+   * Updates the number of hits and the total number of block reads on a block
+   * cache hit.
+   */
+  public void updateOnCacheHit(BlockCategory blockCategory,
+      boolean isCompaction, long count) {
     blockCategory.expectSpecific();
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT);
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT);
-    if (this != ALL_SCHEMA_METRICS) {
-      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction);
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    
+    if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) {
+      flushCertainOnCacheHitMetrics(blockCategory, isCompaction);
+    }
+  }
+  
+  private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) {
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0);
+    
+    if (tempCount > 0) {
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount);
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount);
+      if (this != ALL_SCHEMA_METRICS) {
+        ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, tempCount);
+      }
+    }
+  }
+  
+  /**
+   * Flush the on cache hit metrics;
+   */
+  private void flushOnCacheHitMetrics() {
+    for (BlockCategory blockCategory : BlockCategory.values()) {
+      for (boolean isCompaction : BOOL_VALUES) {
+        flushCertainOnCacheHitMetrics (blockCategory, isCompaction);
+      }
     }
   }
 
+  /**
+   * Notify the SchemaMetrics to flush all of the the metrics
+   */
+  public void flushMetrics() {
+    // currently only for "on cache hit metrics"
+    flushOnCacheHitMetrics();
+    
+    if (this != ALL_SCHEMA_METRICS) {
+      ALL_SCHEMA_METRICS.flushMetrics();
+    }
+  }
+  
   /**
    * Updates read time, the number of misses, and the total number of block
    * reads on a block cache miss.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
index dbe425f1ccb7..a6f022313c36 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
@@ -201,8 +201,10 @@ public void testIncrements() {
 
           for (boolean isCompaction : BOOL_VALUES) {
             sm.updateOnCacheHit(blockCat, isCompaction);
+            sm.flushMetrics();
             checkMetrics();
             sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt());
+            sm.flushMetrics();
             checkMetrics();
           }
 

From b0fe19c464f570eaf2bf4835c1f9d98a0eb3daef Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 26 Oct 2012 17:29:12 +0000
Subject: [PATCH 0548/1540] HBASE-6852 REVERT due to test failures.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402588 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  2 -
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  2 -
 .../regionserver/metrics/SchemaMetrics.java   | 79 ++-----------------
 .../metrics/TestSchemaMetrics.java            |  2 -
 4 files changed, 6 insertions(+), 79 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index 961daddafcd7..b93c5c483104 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -391,8 +391,6 @@ public void close(boolean evictOnClose) throws IOException {
       this.istream.close();
       this.istream = null;
     }
-    
-    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV1
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index ccee0d02a804..757990f845f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -424,8 +424,6 @@ public void close(boolean evictOnClose) throws IOException {
         istream = null;
       }
     }
-    
-    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV2
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 61d2ef969c90..e73513ceb953 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -29,7 +29,6 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -39,6 +38,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -236,9 +236,6 @@ public String toString() {
   public static final SchemaMetrics ALL_SCHEMA_METRICS =
     getInstance(TOTAL_KEY, TOTAL_KEY);
 
-  /** Threshold for flush the metrics, currently used only for "on cache hit" */
-  private static final long THRESHOLD_METRICS_FLUSH = 100l;
-
   /**
    * Whether to include table name in metric names. If this is null, it has not
    * been initialized. This is a global instance, but we also have a copy of it
@@ -257,8 +254,6 @@ public String toString() {
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
   private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
-  private final AtomicLongArray onHitCacheMetrics= 
-      new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length);
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -266,9 +261,6 @@ private SchemaMetrics(final String tableName, final String cfName) {
 
     for (BlockCategory blockCategory : BlockCategory.values()) {
       for (boolean isCompaction : BOOL_VALUES) {
-        // initialize the cache metrics
-        onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0);
-        
         for (BlockMetricType metricType : BlockMetricType.values()) {
           if (!metricType.compactionAware && isCompaction) {
             continue;
@@ -344,11 +336,6 @@ public static SchemaMetrics getInstance(String tableName, String cfName) {
     return existingMetrics != null ? existingMetrics : schemaMetrics;
   }
 
-  private static final int getCacheHitMetricIndex (BlockCategory blockCategory,
-      boolean isCompaction) {
-    return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0);
-  }
-  
   private static final int getBlockMetricIndex(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
     int i = 0;
@@ -378,20 +365,11 @@ public String getBloomMetricName(boolean isInBloom) {
    */
   private void incrNumericMetric(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
-    incrNumericMetric (blockCategory, isCompaction, metricType, 1);
-  }
-  
-  /**
-   * Increments the given metric, both per-CF and aggregate, for both the given
-   * category and all categories in aggregate (four counters total).
-   */
-  private void incrNumericMetric(BlockCategory blockCategory,
-      boolean isCompaction, BlockMetricType metricType, long amount) {
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
     RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
-        isCompaction, metricType), amount);
+        isCompaction, metricType), 1);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
       incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction,
@@ -467,59 +445,14 @@ public void updatePersistentStoreMetric(StoreMetricType storeMetricType,
    */
   public void updateOnCacheHit(BlockCategory blockCategory,
       boolean isCompaction) {
-    updateOnCacheHit (blockCategory, isCompaction, 1);
-  }
-  
-  /**
-   * Updates the number of hits and the total number of block reads on a block
-   * cache hit.
-   */
-  public void updateOnCacheHit(BlockCategory blockCategory,
-      boolean isCompaction, long count) {
     blockCategory.expectSpecific();
-    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
-    
-    if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) {
-      flushCertainOnCacheHitMetrics(blockCategory, isCompaction);
-    }
-  }
-  
-  private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) {
-    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
-    long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0);
-    
-    if (tempCount > 0) {
-      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount);
-      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount);
-      if (this != ALL_SCHEMA_METRICS) {
-        ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, tempCount);
-      }
-    }
-  }
-  
-  /**
-   * Flush the on cache hit metrics;
-   */
-  private void flushOnCacheHitMetrics() {
-    for (BlockCategory blockCategory : BlockCategory.values()) {
-      for (boolean isCompaction : BOOL_VALUES) {
-        flushCertainOnCacheHitMetrics (blockCategory, isCompaction);
-      }
-    }
-  }
-
-  /**
-   * Notify the SchemaMetrics to flush all of the the metrics
-   */
-  public void flushMetrics() {
-    // currently only for "on cache hit metrics"
-    flushOnCacheHitMetrics();
-    
+    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT);
+    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT);
     if (this != ALL_SCHEMA_METRICS) {
-      ALL_SCHEMA_METRICS.flushMetrics();
+      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction);
     }
   }
-  
+
   /**
    * Updates read time, the number of misses, and the total number of block
    * reads on a block cache miss.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
index a6f022313c36..dbe425f1ccb7 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
@@ -201,10 +201,8 @@ public void testIncrements() {
 
           for (boolean isCompaction : BOOL_VALUES) {
             sm.updateOnCacheHit(blockCat, isCompaction);
-            sm.flushMetrics();
             checkMetrics();
             sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt());
-            sm.flushMetrics();
             checkMetrics();
           }
 

From 3ab7fe3e6eb060602a4ee00f25a254eaf911f00e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 27 Oct 2012 21:28:33 +0000
Subject: [PATCH 0549/1540] HBASE-6846 BitComparator bug -
 ArrayIndexOutOfBoundsException (Lucian George Iordache)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1402888 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/BitComparator.java    |  2 +-
 .../hbase/filter/TestBitComparator.java       | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java
index bc29a42a69fe..088a0997e10d 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java
@@ -80,7 +80,7 @@ public int compareTo(byte[] value, int offset, int length) {
     }
     int b = 0;
     //Iterating backwards is faster because we can quit after one non-zero byte.
-    for (int i = value.length - 1; i >= 0 && b == 0; i--) {
+    for (int i = length - 1; i >= 0 && b == 0; i--) {
       switch (bitOperator) {
         case AND:
           b = (this.value[i] & value[i+offset]) & 0xff;
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java b/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
index 9007f074aa3e..50888af7625f 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
@@ -32,6 +32,11 @@ public class TestBitComparator extends TestCase {
   private static byte[] data1 = new byte[]{15, 0, 0, 0, 0, 0};
   private static byte[] data2 = new byte[]{0, 0, 0, 0, 0, 15};
   private static byte[] data3 = new byte[]{15, 15, 15, 15, 15};
+  
+  // data for testing compareTo method with offset and length parameters
+  private static byte[] data1_2 = new byte[]{15, 15, 0, 0, 0, 0, 0, 15};
+  private static byte[] data2_2 = new byte[]{15, 0, 0, 0, 0, 0, 15, 15};
+  
   private final int Equal = 0;
   private final int NotEqual = 1;
 
@@ -65,6 +70,26 @@ private void testOperation(byte[] data, byte[] comparatorBytes, BitComparator.Bi
     assertEquals(comparator.compareTo(data), expected);
   }
 
+  public void testANDOperationWithOffset() {
+    testOperationWithOffset(data1_2, ones, BitComparator.BitwiseOp.AND, Equal);
+    testOperationWithOffset(data1_2, data0, BitComparator.BitwiseOp.AND, NotEqual);
+    testOperationWithOffset(data2_2, data1, BitComparator.BitwiseOp.AND, NotEqual);
+  }
+
+  public void testOROperationWithOffset() {
+    testOperationWithOffset(data1_2, zeros, BitComparator.BitwiseOp.OR, Equal);
+    testOperationWithOffset(data2_2, data1, BitComparator.BitwiseOp.OR, Equal);
+  }
+
+  public void testXOROperationWithOffset() {
+    testOperationWithOffset(data2_2, data1, BitComparator.BitwiseOp.XOR, Equal);
+  }
+
+  private void testOperationWithOffset(byte[] data, byte[] comparatorBytes, BitComparator.BitwiseOp operator, int expected) {
+    BitComparator comparator = new BitComparator(comparatorBytes, operator);
+    assertEquals(comparator.compareTo(data, 1, comparatorBytes.length), expected);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From dbf918f05a39997c1d6f2c1777421d806035e3bc Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Mon, 29 Oct 2012 23:45:11 +0000
Subject: [PATCH 0550/1540] HBASE-6700  [replication] empty znodes created
 during queue failovers aren't             deleted (Terry Zhang via JD)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1403582 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/replication/ReplicationZookeeper.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 70c2ead8b9ff..bc20acd9b976 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -683,14 +683,14 @@ public SortedMap<String, SortedSet<String>> copyQueuesFromRS(String znode) {
         // number-startcode-number-otherstartcode-number-anotherstartcode-etc
         String newCluster = cluster+"-"+znode;
         String newClusterZnode = ZKUtil.joinZNode(rsServerNameZnode, newCluster);
-        ZKUtil.createNodeIfNotExistsAndWatch(this.zookeeper, newClusterZnode,
-          HConstants.EMPTY_BYTE_ARRAY);
         String clusterPath = ZKUtil.joinZNode(nodePath, cluster);
         List<String> hlogs = ZKUtil.listChildrenNoWatch(this.zookeeper, clusterPath);
         // That region server didn't have anything to replicate for this cluster
         if (hlogs == null || hlogs.size() == 0) {
           continue;
         }
+        ZKUtil.createNodeIfNotExistsAndWatch(this.zookeeper, newClusterZnode,
+            HConstants.EMPTY_BYTE_ARRAY);
         SortedSet<String> logQueue = new TreeSet<String>();
         queues.put(newCluster, logQueue);
         for (String hlog : hlogs) {

From 3a7c9f7abd9d20d3096cfe9628d7226fa2f7df28 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 30 Oct 2012 17:20:49 +0000
Subject: [PATCH 0551/1540] HBASE-7038 Port HBASE-5970 Improve the
 AssignmentManager#updateTimer and speed up handling opened event to 0.94
 (Sergey Shelukhin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1403787 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 56 ++++++++++++++++++-
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index e275b81acffb..503061d52dc5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -109,6 +109,8 @@ public class AssignmentManager extends ZooKeeperListener {
 
   private TimeoutMonitor timeoutMonitor;
 
+  private TimerUpdater timerUpdater;
+
   private LoadBalancer balancer;
 
   /**
@@ -153,6 +155,13 @@ public class AssignmentManager extends ZooKeeperListener {
   private final NavigableMap<ServerName, Set<HRegionInfo>> servers =
     new TreeMap<ServerName, Set<HRegionInfo>>();
 
+  /**
+   * Contains the server which need to update timer, these servers will be
+   * handled by {@link TimerUpdater}
+   */
+  private final ConcurrentSkipListSet<ServerName> serversInUpdatingTimer = 
+    new ConcurrentSkipListSet<ServerName>();
+
   /**
    * Region to server assignment map.
    * Contains the server a given region is currently assigned to.
@@ -207,6 +216,10 @@ public AssignmentManager(Server master, ServerManager serverManager,
       conf.getInt("hbase.master.assignment.timeoutmonitor.period", 10000),
       master, serverManager,
       conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 1800000));
+    this.timerUpdater = new TimerUpdater(conf.getInt(
+        "hbase.master.assignment.timerupdater.period", 10000), master);
+    Threads.setDaemonThreadRunning(timerUpdater.getThread(),
+        master.getServerName() + ".timerUpdater");
     this.zkTable = new ZKTable(this.master.getZooKeeper());
     this.maximumAssignmentAttempts =
       this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
@@ -1215,8 +1228,17 @@ void regionOnline(HRegionInfo regionInfo, ServerName sn) {
     }
     // Remove plan if one.
     clearRegionPlan(regionInfo);
-    // Update timers for all regions in transition going against this server.
-    updateTimers(sn);
+    // Add the server to serversInUpdatingTimer
+    addToServersInUpdatingTimer(sn);
+  }
+
+  /**
+   * Add the server to the set serversInUpdatingTimer, then {@link TimerUpdater}
+   * will update timers for this server in background
+   * @param sn
+   */
+  private void addToServersInUpdatingTimer(final ServerName sn) {
+    this.serversInUpdatingTimer.add(sn);
   }
 
   /**
@@ -2888,6 +2910,35 @@ public List<HRegionInfo> getRegionsOfTable(byte[] tableName) {
     return tableRegions;
   }
 
+  /**
+   * Update timers for all regions in transition going against the server in the
+   * serversInUpdatingTimer.
+   */
+  public class TimerUpdater extends Chore {
+
+    public TimerUpdater(final int period, final Stoppable stopper) {
+      super("AssignmentTimerUpdater", period, stopper);
+    }
+
+    @Override
+    protected void chore() {
+      ServerName serverToUpdateTimer = null;
+      while (!serversInUpdatingTimer.isEmpty() && !stopper.isStopped()) {
+        if (serverToUpdateTimer == null) {
+          serverToUpdateTimer = serversInUpdatingTimer.first();
+        } else {
+          serverToUpdateTimer = serversInUpdatingTimer
+              .higher(serverToUpdateTimer);
+        }
+        if (serverToUpdateTimer == null) {
+          break;
+        }
+        updateTimers(serverToUpdateTimer);
+        serversInUpdatingTimer.remove(serverToUpdateTimer);
+      }
+    }
+  }
+
   /**
    * Monitor to check for time outs on region transition operations
    */
@@ -3428,6 +3479,7 @@ public void write(DataOutput out) throws IOException {
 
   public void stop() {
     this.timeoutMonitor.interrupt();
+    this.timerUpdater.interrupt();
   }
   
   /**

From 02cc5b24b1ba8d5531bd6c978d60167381f35bd8 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 30 Oct 2012 17:44:53 +0000
Subject: [PATCH 0552/1540] HBASE-7060 Region load balancing by table does not
 handle the case where a table's region count is lower than the number of the
 RS in the cluster (Ted Yu and Tianying)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1403801 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/DefaultLoadBalancer.java       | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
index 9b132c0286a1..82a33e0a3a3c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
@@ -229,12 +229,14 @@ public List<RegionPlan> balanceCluster(
     NavigableMap<ServerAndLoad, List<HRegionInfo>> serversByLoad =
       new TreeMap<ServerAndLoad, List<HRegionInfo>>();
     int numRegions = 0;
+    int maxRegionCountPerServer = 0;
     // Iterate so we can count regions as we build the map
     for (Map.Entry<ServerName, List<HRegionInfo>> server: clusterState.entrySet()) {
       List<HRegionInfo> regions = server.getValue();
       int sz = regions.size();
       if (sz == 0) emptyRegionServerPresent = true;
       numRegions += sz;
+      if (maxRegionCountPerServer < sz) maxRegionCountPerServer = sz;
       serversByLoad.put(new ServerAndLoad(server.getKey(), sz), regions);
     }
     // Check if we even need to do any load balancing
@@ -254,6 +256,7 @@ public List<RegionPlan> balanceCluster(
     }
     int min = numRegions / numServers;
     int max = numRegions % numServers == 0 ? min : min + 1;
+    if (maxRegionCountPerServer == 1) return null; // table is balanced
 
     // Using to check balance result.
     StringBuilder strBalanceParam = new StringBuilder();
@@ -315,13 +318,21 @@ public List<RegionPlan> balanceCluster(
     fetchFromTail = false;
 
     Map<ServerName, Integer> underloadedServers = new HashMap<ServerName, Integer>();
+    int maxToTake = numRegions - (int)average;
     for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server:
         serversByLoad.entrySet()) {
+      if (maxToTake == 0) break; // no more to take
       int regionCount = server.getKey().getLoad();
-      if (regionCount >= min) {
-        break;
+      if (regionCount >= min && regionCount > 0) {
+        continue; // look for other servers which haven't reached min
+      }
+      int regionsToPut = min - regionCount;
+      if (regionsToPut == 0)
+      {
+        regionsToPut = 1;
+        maxToTake--;
       }
-      underloadedServers.put(server.getKey().getServerName(), min - regionCount);
+      underloadedServers.put(server.getKey().getServerName(), regionsToPut);
     }
     // number of servers that get new regions
     int serversUnderloaded = underloadedServers.size();

From c3711ed5692d5f0e1f9b5dd4636e8abdb926d9ee Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 30 Oct 2012 18:09:41 +0000
Subject: [PATCH 0553/1540]     HBASE-7069

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1403808 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HTable.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 9bea85f5faec..bc5f6e3caa9b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -688,7 +688,7 @@ public Result[] get(List<Get> gets) throws IOException {
    * {@inheritDoc}
    */
   @Override
-  public synchronized void batch(final List<?extends Row> actions, final Object[] results)
+  public void batch(final List<?extends Row> actions, final Object[] results)
       throws InterruptedException, IOException {
     connection.processBatch(actions, tableName, pool, results);
   }
@@ -697,7 +697,7 @@ public synchronized void batch(final List<?extends Row> actions, final Object[]
    * {@inheritDoc}
    */
   @Override
-  public synchronized Object[] batch(final List<? extends Row> actions) throws InterruptedException, IOException {
+  public Object[] batch(final List<? extends Row> actions) throws InterruptedException, IOException {
     Object[] results = new Object[actions.size()];
     connection.processBatch(actions, tableName, pool, results);
     return results;

From b9202d4d25c0bca73595ffea59d53652d8ca5768 Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Tue, 30 Oct 2012 20:58:45 +0000
Subject: [PATCH 0554/1540] HBASE-7073 OperationMetrics needs to cache the
 value of hbase.metrics.exposeOperationTimes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1403865 13f79535-47bb-0310-9956-ffa450edef68
---
 .../metrics/OperationMetrics.java             | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
index 460afea2ec70..eacb702d9f08 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
@@ -51,11 +51,12 @@ public class OperationMetrics {
   private static final String CONF_KEY =
       "hbase.metrics.exposeOperationTimes";
 
-  private String tableName = null;
-  private String regionName = null;
-  private String regionMetrixPrefix = null;
-  private Configuration conf = null;
-  
+  private final String tableName;
+  private final String regionName;
+  private final String regionMetrixPrefix;
+  private final Configuration conf;
+  private final boolean exposeTimes;
+
 
   /**
    * Create a new OperationMetrics
@@ -78,6 +79,14 @@ public OperationMetrics(Configuration conf, HRegionInfo regionInfo) {
       }
       this.regionMetrixPrefix =
           SchemaMetrics.generateRegionMetricsPrefix(this.tableName, this.regionName);
+      this.exposeTimes = this.conf.getBoolean(CONF_KEY, true);
+    } else {
+      //Make all the final values happy.
+      this.conf = null;
+      this.tableName = null;
+      this.regionName = null;
+      this.regionMetrixPrefix = null;
+      this.exposeTimes = false;
     }
   }
   
@@ -212,7 +221,7 @@ private void doUpdateTimeVarying(Set<byte[]> columnFamilies, String key, long va
   }
 
   private void doSafeIncTimeVarying(String prefix, String key, long value) {
-    if (conf.getBoolean(CONF_KEY, true)) {
+    if (exposeTimes) {
       if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) {
         String m = prefix + key;
         RegionMetricsStorage.incrTimeVaryingMetric(m, value);

From 9cf93e02a79c68eb906daef905c95e0fd289dcae Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 31 Oct 2012 22:04:34 +0000
Subject: [PATCH 0555/1540] HBASE-7051 CheckAndPut should properly read MVCC

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404379 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 1bced40b56dc..cd8a18d27bd9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2354,6 +2354,8 @@ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
 
       // Lock row
       Integer lid = getLock(lockId, get.getRow(), true);
+      // wait for all previous transactions to complete (with lock held)
+      mvcc.completeMemstoreInsert(mvcc.beginMemstoreInsert());
       List<KeyValue> result = new ArrayList<KeyValue>();
       try {
         result = get(get, false);

From 153bcf7ccb9b53e0c0afc27132110688590931b5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 31 Oct 2012 22:12:28 +0000
Subject: [PATCH 0556/1540] HBASE-7077 Test for: CheckAndPut should properly
 read MVCC (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404383 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestHBase7051.java     | 171 ++++++++++++++++++
 .../hbase/regionserver/TestHRegion.java       |   2 +-
 2 files changed, 172 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
new file mode 100644
index 000000000000..aa0bcc5261b4
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
@@ -0,0 +1,171 @@
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class TestHBase7051 {
+
+  private static volatile boolean putCompleted = false;
+  private static CountDownLatch latch = new CountDownLatch(1);
+  private boolean checkAndPutCompleted = false;
+  private static int count = 0;
+
+  @Test
+  public void testPutAndCheckAndPutInParallel() throws Exception {
+
+    final String tableName = "testPutAndCheckAndPut";
+    final String family = "f1";
+    Configuration conf = HBaseConfiguration.create();
+    conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
+    final MockHRegion region = (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName),
+        tableName, conf, Bytes.toBytes(family));
+
+    List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
+    Put[] puts = new Put[1];
+    Put put = new Put(Bytes.toBytes("r1"));
+    put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10"));
+    puts[0] = put;
+    Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
+
+    putsAndLocks.add(pair);
+
+    count++;
+    region.batchMutate(putsAndLocks.toArray(new Pair[0]));
+    makeCheckAndPut(family, region);
+
+    makePut(family, region);
+    while (!checkAndPutCompleted) {
+      Thread.sleep(100);
+    }
+    Scan s = new Scan();
+    RegionScanner scanner = region.getScanner(s);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    scanner.next(results, 2);
+    for (KeyValue keyValue : results) {
+      assertEquals("50",Bytes.toString(keyValue.getValue()));
+    }
+
+  }
+
+  private void makePut(final String family, final MockHRegion region) {
+    new Thread() {
+      public void run() {
+        List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
+        Put[] puts = new Put[1];
+        Put put = new Put(Bytes.toBytes("r1"));
+        put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50"));
+        puts[0] = put;
+        try {
+          Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
+          putsAndLocks.add(pair);
+          count++;
+          region.batchMutate(putsAndLocks.toArray(new Pair[0]));
+        } catch (IOException e) {
+          // TODO Auto-generated catch block
+          e.printStackTrace();
+        }
+      }
+    }.start();
+  }
+
+  private void makeCheckAndPut(final String family, final MockHRegion region) {
+    new Thread() {
+
+      public void run() {
+        Put[] puts = new Put[1];
+        Put put = new Put(Bytes.toBytes("r1"));
+        put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11"));
+        puts[0] = put;
+        try {
+          while (putCompleted == false) {
+            try {
+              Thread.sleep(100);
+            } catch (InterruptedException e) {
+              // TODO Auto-generated catch block
+              e.printStackTrace();
+            }
+          }
+          count++;
+          region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"),
+              CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, null, true);
+          checkAndPutCompleted = true;
+        } catch (IOException e) {
+          // TODO Auto-generated catch block
+          e.printStackTrace();
+        }
+      }
+    }.start();
+  }
+
+  public static class MockHRegion extends HRegion {
+
+    public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
+        final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) {
+      super(tableDir, log, fs, conf, regionInfo, htd, rsServices);
+    }
+
+    @Override
+    public void releaseRowLock(Integer lockId) {
+      if (count == 1) {
+        super.releaseRowLock(lockId);
+        return;
+      }
+
+      if (count == 2) {
+        try {
+          putCompleted = true;
+          super.releaseRowLock(lockId);
+          latch.await();
+        } catch (InterruptedException e) {
+          // TODO Auto-generated catch block
+          e.printStackTrace();
+        }
+      }
+      if (count == 3) {
+        super.releaseRowLock(lockId);
+        try {
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {
+          // TODO Auto-generated catch block
+          e.printStackTrace();
+        }
+        latch.countDown();
+      }
+    }
+
+    @Override
+    public Integer getLock(Integer lockid, byte[] row, boolean waitForLock) throws IOException {
+      if (count == 3) {
+        latch.countDown();
+      }
+      return super.getLock(lockid, row, waitForLock);
+    }
+
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 0ce2e7e49c8c..6a91edf89587 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -4046,7 +4046,7 @@ private static HRegion initHRegion (byte [] tableName, String callingMethod,
    * @throws IOException
    * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
    */
-  private static HRegion initHRegion (byte [] tableName, String callingMethod,
+  public static HRegion initHRegion (byte [] tableName, String callingMethod,
       Configuration conf, byte [] ... families)
     throws IOException{
     return initHRegion(tableName, null, null, callingMethod, conf, families);

From 989db9031bcab3cb9662d0941999eaaeece9c495 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Wed, 31 Oct 2012 22:18:50 +0000
Subject: [PATCH 0557/1540] HBASE-6904 In the HBase shell, an error is thrown
 that states replication-related znodes already exist

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404385 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 52013e93f15b..57ffe86a2c9e 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -436,7 +436,7 @@ private String createNonSequential(String path, byte[] data, List<ACL> acl,
                   Bytes.toStringBinary(data));
               throw e;
             }
-            LOG.error("Node " + path + " already exists and this is not a " +
+            LOG.info("Node " + path + " already exists and this is not a " +
                 "retry");
             throw e;
 

From b77292d94dab15fabe9f9f23501da3e801db33fd Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 1 Nov 2012 04:58:53 +0000
Subject: [PATCH 0558/1540] HBASE-6852 SchemaMetrics.updateOnCacheHit costs too
 much while full scanning a table with all of its fields (Cheng Hao and LarsH)
 - REAPPLY

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404464 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  2 +
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  2 +
 .../regionserver/metrics/SchemaMetrics.java   | 83 +++++++++++++++++--
 .../metrics/TestSchemaMetrics.java            |  2 +
 4 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index b93c5c483104..961daddafcd7 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -391,6 +391,8 @@ public void close(boolean evictOnClose) throws IOException {
       this.istream.close();
       this.istream = null;
     }
+    
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV1
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 757990f845f4..ccee0d02a804 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -424,6 +424,8 @@ public void close(boolean evictOnClose) throws IOException {
         istream = null;
       }
     }
+    
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV2
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index e73513ceb953..8d6a24b43b97 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -29,6 +29,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -38,7 +39,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -236,6 +236,9 @@ public String toString() {
   public static final SchemaMetrics ALL_SCHEMA_METRICS =
     getInstance(TOTAL_KEY, TOTAL_KEY);
 
+  /** Threshold for flush the metrics, currently used only for "on cache hit" */
+  private static final long THRESHOLD_METRICS_FLUSH = 100l;
+
   /**
    * Whether to include table name in metric names. If this is null, it has not
    * been initialized. This is a global instance, but we also have a copy of it
@@ -254,6 +257,8 @@ public String toString() {
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
   private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
+  private final AtomicLongArray onHitCacheMetrics= 
+      new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length);
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -261,6 +266,9 @@ private SchemaMetrics(final String tableName, final String cfName) {
 
     for (BlockCategory blockCategory : BlockCategory.values()) {
       for (boolean isCompaction : BOOL_VALUES) {
+        // initialize the cache metrics
+        onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0);
+        
         for (BlockMetricType metricType : BlockMetricType.values()) {
           if (!metricType.compactionAware && isCompaction) {
             continue;
@@ -336,6 +344,11 @@ public static SchemaMetrics getInstance(String tableName, String cfName) {
     return existingMetrics != null ? existingMetrics : schemaMetrics;
   }
 
+  private static final int getCacheHitMetricIndex (BlockCategory blockCategory,
+      boolean isCompaction) {
+    return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0);
+  }
+  
   private static final int getBlockMetricIndex(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
     int i = 0;
@@ -365,15 +378,24 @@ public String getBloomMetricName(boolean isInBloom) {
    */
   private void incrNumericMetric(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
+    incrNumericMetric (blockCategory, isCompaction, metricType, 1);
+  }
+  
+  /**
+   * Increments the given metric, both per-CF and aggregate, for both the given
+   * category and all categories in aggregate (four counters total).
+   */
+  private void incrNumericMetric(BlockCategory blockCategory,
+      boolean isCompaction, BlockMetricType metricType, long amount) {
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
     RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
-        isCompaction, metricType), 1);
+        isCompaction, metricType), amount);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
       incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction,
-          metricType);
+          metricType, amount);
     }
   }
 
@@ -445,14 +467,59 @@ public void updatePersistentStoreMetric(StoreMetricType storeMetricType,
    */
   public void updateOnCacheHit(BlockCategory blockCategory,
       boolean isCompaction) {
+    updateOnCacheHit(blockCategory, isCompaction, 1);
+  }
+  
+  /**
+   * Updates the number of hits and the total number of block reads on a block
+   * cache hit.
+   */
+  public void updateOnCacheHit(BlockCategory blockCategory,
+      boolean isCompaction, long count) {
     blockCategory.expectSpecific();
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT);
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT);
-    if (this != ALL_SCHEMA_METRICS) {
-      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction);
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    
+    if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) {
+      flushCertainOnCacheHitMetrics(blockCategory, isCompaction);
+    }
+  }
+  
+  private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) {
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0);
+    
+    if (tempCount > 0) {
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount);
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount);
+      if (this != ALL_SCHEMA_METRICS) {
+        ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, tempCount);
+      }
+    }
+  }
+  
+  /**
+   * Flush the on cache hit metrics;
+   */
+  private void flushOnCacheHitMetrics() {
+    for (BlockCategory blockCategory : BlockCategory.values()) {
+      for (boolean isCompaction : BOOL_VALUES) {
+        flushCertainOnCacheHitMetrics (blockCategory, isCompaction);
+      }
     }
   }
 
+  /**
+   * Notify the SchemaMetrics to flush all of the the metrics
+   */
+  public void flushMetrics() {
+    // currently only for "on cache hit metrics"
+    flushOnCacheHitMetrics();
+    
+    if (this != ALL_SCHEMA_METRICS) {
+      ALL_SCHEMA_METRICS.flushMetrics();
+    }
+  }
+  
   /**
    * Updates read time, the number of misses, and the total number of block
    * reads on a block cache miss.
@@ -746,6 +813,7 @@ private static final String stripTimeVaryingSuffix(String metricKey) {
   public static Map<String, Long> getMetricsSnapshot() {
     Map<String, Long> metricsSnapshot = new TreeMap<String, Long>();
     for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) {
+      cfm.flushMetrics();
       for (String metricName : cfm.getAllMetricNames()) {
         long metricValue;
         if (isTimeVaryingKey(metricName)) {
@@ -796,6 +864,7 @@ public static Map<String, Long> diffMetrics(Map<String, Long> a,
   }
 
   public static void validateMetricChanges(Map<String, Long> oldMetrics) {
+    
     final Map<String, Long> newMetrics = getMetricsSnapshot();
     final Map<String, Long> allCfDeltas = new TreeMap<String, Long>();
     final Map<String, Long> allBlockCategoryDeltas =
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
index dbe425f1ccb7..a6f022313c36 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
@@ -201,8 +201,10 @@ public void testIncrements() {
 
           for (boolean isCompaction : BOOL_VALUES) {
             sm.updateOnCacheHit(blockCat, isCompaction);
+            sm.flushMetrics();
             checkMetrics();
             sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt());
+            sm.flushMetrics();
             checkMetrics();
           }
 

From c34779813edd433e0eaa858a7a65503bf30106ee Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 1 Nov 2012 15:24:55 +0000
Subject: [PATCH 0559/1540] HBASE-7077 ADDENDUM, add TestCategory

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404641 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/TestHBase7051.java    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
index aa0bcc5261b4..df6c293e3a95 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
@@ -15,6 +15,7 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
@@ -25,9 +26,11 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 import com.google.common.collect.Lists;
 
+@Category(SmallTests.class)
 public class TestHBase7051 {
 
   private static volatile boolean putCompleted = false;

From a30182efa24dd86bb4d0bd6601fc3b456e405fcf Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 1 Nov 2012 17:44:53 +0000
Subject: [PATCH 0560/1540] HBASE-6852, REVERT again, due to unexplained test
 failures that only occur on the jenkins machines

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404691 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  2 -
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  2 -
 .../regionserver/metrics/SchemaMetrics.java   | 83 ++-----------------
 .../metrics/TestSchemaMetrics.java            |  2 -
 4 files changed, 7 insertions(+), 82 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index 961daddafcd7..b93c5c483104 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -391,8 +391,6 @@ public void close(boolean evictOnClose) throws IOException {
       this.istream.close();
       this.istream = null;
     }
-    
-    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV1
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index ccee0d02a804..757990f845f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -424,8 +424,6 @@ public void close(boolean evictOnClose) throws IOException {
         istream = null;
       }
     }
-    
-    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV2
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 8d6a24b43b97..e73513ceb953 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -29,7 +29,6 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -39,6 +38,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -236,9 +236,6 @@ public String toString() {
   public static final SchemaMetrics ALL_SCHEMA_METRICS =
     getInstance(TOTAL_KEY, TOTAL_KEY);
 
-  /** Threshold for flush the metrics, currently used only for "on cache hit" */
-  private static final long THRESHOLD_METRICS_FLUSH = 100l;
-
   /**
    * Whether to include table name in metric names. If this is null, it has not
    * been initialized. This is a global instance, but we also have a copy of it
@@ -257,8 +254,6 @@ public String toString() {
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
   private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
-  private final AtomicLongArray onHitCacheMetrics= 
-      new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length);
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -266,9 +261,6 @@ private SchemaMetrics(final String tableName, final String cfName) {
 
     for (BlockCategory blockCategory : BlockCategory.values()) {
       for (boolean isCompaction : BOOL_VALUES) {
-        // initialize the cache metrics
-        onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0);
-        
         for (BlockMetricType metricType : BlockMetricType.values()) {
           if (!metricType.compactionAware && isCompaction) {
             continue;
@@ -344,11 +336,6 @@ public static SchemaMetrics getInstance(String tableName, String cfName) {
     return existingMetrics != null ? existingMetrics : schemaMetrics;
   }
 
-  private static final int getCacheHitMetricIndex (BlockCategory blockCategory,
-      boolean isCompaction) {
-    return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0);
-  }
-  
   private static final int getBlockMetricIndex(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
     int i = 0;
@@ -378,24 +365,15 @@ public String getBloomMetricName(boolean isInBloom) {
    */
   private void incrNumericMetric(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
-    incrNumericMetric (blockCategory, isCompaction, metricType, 1);
-  }
-  
-  /**
-   * Increments the given metric, both per-CF and aggregate, for both the given
-   * category and all categories in aggregate (four counters total).
-   */
-  private void incrNumericMetric(BlockCategory blockCategory,
-      boolean isCompaction, BlockMetricType metricType, long amount) {
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
     RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
-        isCompaction, metricType), amount);
+        isCompaction, metricType), 1);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
       incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction,
-          metricType, amount);
+          metricType);
     }
   }
 
@@ -467,59 +445,14 @@ public void updatePersistentStoreMetric(StoreMetricType storeMetricType,
    */
   public void updateOnCacheHit(BlockCategory blockCategory,
       boolean isCompaction) {
-    updateOnCacheHit(blockCategory, isCompaction, 1);
-  }
-  
-  /**
-   * Updates the number of hits and the total number of block reads on a block
-   * cache hit.
-   */
-  public void updateOnCacheHit(BlockCategory blockCategory,
-      boolean isCompaction, long count) {
     blockCategory.expectSpecific();
-    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
-    
-    if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) {
-      flushCertainOnCacheHitMetrics(blockCategory, isCompaction);
-    }
-  }
-  
-  private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) {
-    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
-    long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0);
-    
-    if (tempCount > 0) {
-      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount);
-      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount);
-      if (this != ALL_SCHEMA_METRICS) {
-        ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, tempCount);
-      }
-    }
-  }
-  
-  /**
-   * Flush the on cache hit metrics;
-   */
-  private void flushOnCacheHitMetrics() {
-    for (BlockCategory blockCategory : BlockCategory.values()) {
-      for (boolean isCompaction : BOOL_VALUES) {
-        flushCertainOnCacheHitMetrics (blockCategory, isCompaction);
-      }
-    }
-  }
-
-  /**
-   * Notify the SchemaMetrics to flush all of the the metrics
-   */
-  public void flushMetrics() {
-    // currently only for "on cache hit metrics"
-    flushOnCacheHitMetrics();
-    
+    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT);
+    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT);
     if (this != ALL_SCHEMA_METRICS) {
-      ALL_SCHEMA_METRICS.flushMetrics();
+      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction);
     }
   }
-  
+
   /**
    * Updates read time, the number of misses, and the total number of block
    * reads on a block cache miss.
@@ -813,7 +746,6 @@ private static final String stripTimeVaryingSuffix(String metricKey) {
   public static Map<String, Long> getMetricsSnapshot() {
     Map<String, Long> metricsSnapshot = new TreeMap<String, Long>();
     for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) {
-      cfm.flushMetrics();
       for (String metricName : cfm.getAllMetricNames()) {
         long metricValue;
         if (isTimeVaryingKey(metricName)) {
@@ -864,7 +796,6 @@ public static Map<String, Long> diffMetrics(Map<String, Long> a,
   }
 
   public static void validateMetricChanges(Map<String, Long> oldMetrics) {
-    
     final Map<String, Long> newMetrics = getMetricsSnapshot();
     final Map<String, Long> allCfDeltas = new TreeMap<String, Long>();
     final Map<String, Long> allBlockCategoryDeltas =
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
index a6f022313c36..dbe425f1ccb7 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
@@ -201,10 +201,8 @@ public void testIncrements() {
 
           for (boolean isCompaction : BOOL_VALUES) {
             sm.updateOnCacheHit(blockCat, isCompaction);
-            sm.flushMetrics();
             checkMetrics();
             sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt());
-            sm.flushMetrics();
             checkMetrics();
           }
 

From f10a4a5a0b2a09df6a0c9fc154a946356917efe8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 1 Nov 2012 20:54:05 +0000
Subject: [PATCH 0561/1540] HBASE-6796 Backport HBASE-5547, Don't delete HFiles
 in backup mode. (Jesse Yates)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404762 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/BaseConfigurable.java |  42 ++
 .../java/org/apache/hadoop/hbase/Chore.java   |   8 +
 .../org/apache/hadoop/hbase/HConstants.java   |  14 +-
 .../hadoop/hbase/backup/HFileArchiver.java    | 625 ++++++++++++++++++
 .../hadoop/hbase/master/CatalogJanitor.java   |   9 +-
 .../apache/hadoop/hbase/master/HMaster.java   |  26 +-
 .../hadoop/hbase/master/LogCleaner.java       | 165 -----
 .../hadoop/hbase/master/MasterFileSystem.java |   3 +-
 .../cleaner/BaseHFileCleanerDelegate.java     |  53 ++
 .../BaseLogCleanerDelegate.java}              |  46 +-
 .../hbase/master/cleaner/CleanerChore.java    | 222 +++++++
 .../master/cleaner/FileCleanerDelegate.java   |  39 ++
 .../hbase/master/cleaner/HFileCleaner.java    |  51 ++
 .../hbase/master/cleaner/LogCleaner.java      |  56 ++
 .../cleaner/TimeToLiveHFileCleaner.java       |  91 +++
 .../{ => cleaner}/TimeToLiveLogCleaner.java   |  19 +-
 .../hadoop/hbase/regionserver/HRegion.java    |  41 +-
 .../hadoop/hbase/regionserver/Store.java      |  13 +-
 .../hadoop/hbase/regionserver/StoreFile.java  |  17 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |   5 +
 .../master/ReplicationLogCleaner.java         |  25 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |  36 +
 .../hadoop/hbase/util/HFileArchiveUtil.java   | 111 ++++
 src/main/resources/hbase-default.xml          |  22 +-
 .../hadoop/hbase/HBaseTestingUtility.java     |   6 +-
 .../hbase/backup/TestHFileArchiving.java      | 293 ++++++++
 .../hbase/master/TestCatalogJanitor.java      | 163 ++++-
 .../master/cleaner/TestCleanerChore.java      | 272 ++++++++
 .../master/cleaner/TestHFileCleaner.java      | 241 +++++++
 .../master/{ => cleaner}/TestLogsCleaner.java |   3 +-
 .../CheckedArchivingHFileCleaner.java         |  46 ++
 .../hbase/util/HFileArchiveTestingUtil.java   | 239 +++++++
 .../hbase/util/TestFSTableDescriptors.java    |  15 +-
 .../hbase/util/TestHFileArchiveUtil.java      |  77 +++
 34 files changed, 2840 insertions(+), 254 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
 rename src/main/java/org/apache/hadoop/hbase/master/{LogCleanerDelegate.java => cleaner/BaseLogCleanerDelegate.java} (52%)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
 rename src/main/java/org/apache/hadoop/hbase/master/{ => cleaner}/TimeToLiveLogCleaner.java (86%)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
 rename src/test/java/org/apache/hadoop/hbase/master/{ => cleaner}/TestLogsCleaner.java (98%)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java

diff --git a/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java b/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java
new file mode 100644
index 000000000000..36efb5031e4d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * HBase version of Hadoop's Configured class that doesn't initialize the
+ * configuration via {@link #setConf(Configuration)} in the constructor, but
+ * only sets the configuration through the {@link #setConf(Configuration)}
+ * method
+ */
+public class BaseConfigurable implements Configurable {
+
+  private Configuration conf;
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/Chore.java b/src/main/java/org/apache/hadoop/hbase/Chore.java
index 38f476bbfb8a..547eeec71087 100644
--- a/src/main/java/org/apache/hadoop/hbase/Chore.java
+++ b/src/main/java/org/apache/hadoop/hbase/Chore.java
@@ -78,6 +78,7 @@ public void run() {
       LOG.fatal(getName() + "error", t);
     } finally {
       LOG.info(getName() + " exiting");
+      cleanup();
     }
   }
 
@@ -110,4 +111,11 @@ protected boolean initialChore() {
   protected void sleep() {
     this.sleeper.sleep();
   }
+
+  /**
+   * Called when the chore has completed, allowing subclasses to cleanup any
+   * extra overhead
+   */
+  protected void cleanup() {
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 9e7001b5bfbc..ba657e038c79 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -590,12 +590,6 @@ public static enum Modify {
     */
   public static final float HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD = 0.2f;
 
-  public static final List<String> HBASE_NON_USER_TABLE_DIRS = new ArrayList<String>(
-      Arrays.asList(new String[]{ HREGION_LOGDIR_NAME, HREGION_OLDLOGDIR_NAME,
-          CORRUPT_DIR_NAME, Bytes.toString(META_TABLE_NAME),
-          Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME,
-          HBCK_SIDELINEDIR_NAME }));
-
   public static final Pattern CP_HTD_ATTR_KEY_PATTERN = Pattern.compile
       ("^coprocessor\\$([0-9]+)$", Pattern.CASE_INSENSITIVE);
   public static final Pattern CP_HTD_ATTR_VALUE_PATTERN =
@@ -666,6 +660,14 @@ public static enum Modify {
    * The actual value is irrelevant because this is always compared by reference.
    */
   public static final byte [] NO_NEXT_INDEXED_KEY = Bytes.toBytes("NO_NEXT_INDEXED_KEY");
+  
+  /** Directory under /hbase where archived hfiles are stored */
+  public static final String HFILE_ARCHIVE_DIRECTORY = ".archive";
+
+  public static final List<String> HBASE_NON_USER_TABLE_DIRS = new ArrayList<String>(
+      Arrays.asList(new String[] { HREGION_LOGDIR_NAME, HREGION_OLDLOGDIR_NAME, CORRUPT_DIR_NAME,
+          Bytes.toString(META_TABLE_NAME), Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME,
+          HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY }));
 
   private HConstants() {
     // Can't be instantiated with this ctor.
diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
new file mode 100644
index 000000000000..ed787e42d0f1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -0,0 +1,625 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.io.MultipleIOException;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+
+/**
+ * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
+ * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on
+ * the state of the system.
+ */
+public class HFileArchiver {
+  private static final Log LOG = LogFactory.getLog(HFileArchiver.class);
+  private static final String SEPARATOR = ".";
+
+  private HFileArchiver() {
+    // hidden ctor since this is just a util
+  }
+
+  /**
+   * Cleans up all the files for a HRegion by archiving the HFiles to the
+   * archive directory
+   * @param fs the file system object
+   * @param info HRegionInfo for region to be deleted
+   * @throws IOException
+   */
+  public static void archiveRegion(FileSystem fs, HRegionInfo info)
+      throws IOException {
+    Path rootDir = FSUtils.getRootDir(fs.getConf());
+    archiveRegion(fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()),
+      HRegion.getRegionDir(rootDir, info));
+  }
+
+
+  /**
+   * Remove an entire region from the table directory via archiving the region's hfiles.
+   * @param fs {@link FileSystem} from which to remove the region
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+   * @param regionDir {@link Path} to where a region is being stored (for building the archive path)
+   * @return <tt>true</tt> if the region was sucessfully deleted. <tt>false</tt> if the filesystem
+   *         operations could not complete.
+   * @throws IOException if the request cannot be completed
+   */
+  public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("ARCHIVING region " + regionDir.toString());
+    }
+
+    // otherwise, we archive the files
+    // make sure we can archive
+    if (tableDir == null || regionDir == null) {
+      LOG.error("No archive directory could be found because tabledir (" + tableDir
+          + ") or regiondir (" + regionDir + "was null. Deleting files instead.");
+      deleteRegionWithoutArchiving(fs, regionDir);
+      // we should have archived, but failed to. Doesn't matter if we deleted
+      // the archived files correctly or not.
+      return false;
+    }
+
+    // make sure the regiondir lives under the tabledir
+    Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
+    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(fs.getConf(), tableDir, regionDir);
+
+    LOG.debug("Have an archive directory, preparing to move files");
+    FileStatusConverter getAsFile = new FileStatusConverter(fs);
+    // otherwise, we attempt to archive the store files
+
+    // build collection of just the store directories to archive
+    Collection<File> toArchive = new ArrayList<File>();
+    final PathFilter dirFilter = new FSUtils.DirFilter(fs);
+    PathFilter nonHidden = new PathFilter() {
+      @Override
+      public boolean accept(Path file) {
+        return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
+      }
+    };
+    FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
+    // if there no files, we can just delete the directory and return;
+    if (storeDirs == null) {
+      LOG.debug("Region directory (" + regionDir + ") was empty, just deleting and returning!");
+      return deleteRegionWithoutArchiving(fs, regionDir);
+    }
+
+    // convert the files in the region to a File
+    toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile));
+    LOG.debug("Archiving:" + toArchive);
+    boolean success = false;
+    try {
+      success = resolveAndArchive(fs, regionArchiveDir, toArchive);
+    } catch (IOException e) {
+      success = false;
+    }
+
+    // if that was successful, then we delete the region
+    if (success) {
+      LOG.debug("Successfully resolved and archived, now can just delete region.");
+      return deleteRegionWithoutArchiving(fs, regionDir);
+    }
+
+    throw new IOException("Received error when attempting to archive files (" + toArchive
+        + "), cannot delete region directory.");
+  }
+
+  /**
+   * Remove the store files, either by archiving them or outright deletion
+   * @param fs the filesystem where the store files live
+   * @param parent Parent region hosting the store files
+   * @param conf {@link Configuration} to examine to determine the archive directory
+   * @param family the family hosting the store files
+   * @param compactedFiles files to be disposed of. No further reading of these files should be
+   *          attempted; otherwise likely to cause an {@link IOException}
+   * @throws IOException if the files could not be correctly disposed.
+   */
+  public static void archiveStoreFiles(FileSystem fs, HRegion parent,
+      Configuration conf, byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
+
+    // sometimes in testing, we don't have rss, so we need to check for that
+    if (fs == null) {
+      LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:"
+          + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family));
+      deleteStoreFilesWithoutArchiving(compactedFiles);
+      return;
+    }
+
+    // short circuit if we don't have any files to delete
+    if (compactedFiles.size() == 0) {
+      LOG.debug("No store files to dispose, done!");
+      return;
+    }
+
+    // build the archive path
+    if (parent == null || family == null) throw new IOException(
+        "Need to have a parent region and a family to archive from.");
+
+    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family);
+
+    // make sure we don't archive if we can't and that the archive dir exists
+    if (!fs.mkdirs(storeArchiveDir)) {
+      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
+          + Bytes.toString(family) + ", deleting compacted files instead.");
+    }
+
+    // otherwise we attempt to archive the store files
+    LOG.debug("Archiving compacted store files.");
+
+    // wrap the storefile into a File
+    StoreToFile getStorePath = new StoreToFile(fs);
+    Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath);
+
+    // do the actual archive
+    if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) {
+      throw new IOException("Failed to archive/delete all the files for region:"
+          + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
+          + " into " + storeArchiveDir + "Something is probably arwy on the filesystem.");
+    }
+  }
+
+  /**
+   * Archive the given files and resolve any conflicts with existing files via appending the time
+   * archiving started (so all conflicts in the same group have the same timestamp appended).
+   * <p>
+   * If any of the passed files to archive are directories, archives the all files under that
+   * directory. Archive directory structure for children is the base archive directory name + the
+   * parent directory and is built recursively is passed files are directories themselves.
+   * @param fs {@link FileSystem} on which to archive the files
+   * @param baseArchiveDir base archive directory to archive the given files
+   * @param toArchive files to be archived
+   * @return <tt>true</tt> on success, <tt>false</tt> otherwise
+   * @throws IOException on unexpected failure
+   */
+  private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir,
+      Collection<File> toArchive) throws IOException {
+    LOG.debug("Starting to archive files:" + toArchive);
+    long start = EnvironmentEdgeManager.currentTimeMillis();
+    List<File> failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start);
+
+    // clean out the failures by just deleting them
+    if (failures.size() > 0) {
+      try {
+        LOG.error("Failed to complete archive, deleting extra store files.");
+        deleteFilesWithoutArchiving(failures);
+      } catch (IOException e) {
+        LOG.warn("Failed to delete store file(s) when archiving failed", e);
+      }
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Resolve any conflict with an existing archive file via timestamp-append
+   * renaming of the existing file and then archive the passed in files.
+   * @param fs {@link FileSystem} on which to archive the files
+   * @param baseArchiveDir base archive directory to store the files. If any of
+   *          the files to archive are directories, will append the name of the
+   *          directory to the base archive directory name, creating a parallel
+   *          structure.
+   * @param toArchive files/directories that need to be archvied
+   * @param start time the archiving started - used for resolving archive
+   *          conflicts.
+   * @return the list of failed to archive files.
+   * @throws IOException if an unexpected file operation exception occured
+   */
+  private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
+      Collection<File> toArchive, long start) throws IOException {
+    // short circuit if no files to move
+    if (toArchive.size() == 0) return Collections.emptyList();
+
+    LOG.debug("moving files to the archive directory: " + baseArchiveDir);
+
+    // make sure the archive directory exists
+    if (!fs.exists(baseArchiveDir)) {
+      if (!fs.mkdirs(baseArchiveDir)) {
+        throw new IOException("Failed to create the archive directory:" + baseArchiveDir
+            + ", quitting archive attempt.");
+      }
+      LOG.debug("Created archive directory:" + baseArchiveDir);
+    }
+
+    List<File> failures = new ArrayList<File>();
+    String startTime = Long.toString(start);
+    for (File file : toArchive) {
+      // if its a file archive it
+      try {
+        LOG.debug("Archiving:" + file);
+        if (file.isFile()) {
+          // attempt to archive the file
+          if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) {
+            LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir);
+            failures.add(file);
+          }
+        } else {
+          // otherwise its a directory and we need to archive all files
+          LOG.debug(file + " is a directory, archiving children files");
+          // so we add the directory name to the one base archive
+          Path parentArchiveDir = new Path(baseArchiveDir, file.getName());
+          // and then get all the files from that directory and attempt to
+          // archive those too
+          Collection<File> children = file.getChildren();
+          failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start));
+        }
+      } catch (IOException e) {
+        LOG.warn("Failed to archive file: " + file, e);
+        failures.add(file);
+      }
+    }
+    return failures;
+  }
+
+  /**
+   * Attempt to archive the passed in file to the archive directory.
+   * <p>
+   * If the same file already exists in the archive, it is moved to a timestamped directory under
+   * the archive directory and the new file is put in its place.
+   * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles
+   * @param currentFile {@link Path} to the original HFile that will be archived
+   * @param archiveStartTime time the archiving started, to resolve naming conflicts
+   * @return <tt>true</tt> if the file is successfully archived. <tt>false</tt> if there was a
+   *         problem, but the operation still completed.
+   * @throws IOException on failure to complete {@link FileSystem} operations.
+   */
+  private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
+      String archiveStartTime) throws IOException {
+    // build path as it should be in the archive
+    String filename = currentFile.getName();
+    Path archiveFile = new Path(archiveDir, filename);
+    FileSystem fs = currentFile.getFileSystem();
+
+    // if the file already exists in the archive, move that one to a timestamped backup. This is a
+    // really, really unlikely situtation, where we get the same name for the existing file, but
+    // is included just for that 1 in trillion chance.
+    if (fs.exists(archiveFile)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("File:" + archiveFile + " already exists in archive, moving to "
+            + "timestamped backup and overwriting current.");
+      }
+
+      // move the archive file to the stamped backup
+      Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
+      if (!fs.rename(archiveFile, backedupArchiveFile)) {
+        LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
+            + ", deleting existing file in favor of newer.");
+        // try to delete the exisiting file, if we can't rename it
+        if (!fs.delete(archiveFile, false)) {
+          throw new IOException("Couldn't delete existing archive file (" + archiveFile
+              + ") or rename it to the backup file (" + backedupArchiveFile
+              + ")to make room for similarly named file.");
+        }
+      }
+      LOG.debug("Backed up archive file from: " + archiveFile);
+    }
+
+    LOG.debug("No existing file in archive for:" + archiveFile + ", free to archive original file.");
+
+    // at this point, we should have a free spot for the archive file
+    if (currentFile.moveAndClose(archiveFile)) {
+      LOG.error("Failed to archive file:" + currentFile);
+      return false;
+    } else if (LOG.isDebugEnabled()) {
+      LOG.debug("Finished archiving file from: " + currentFile + ", to: " + archiveFile);
+    }
+    return true;
+  }
+
+  /**
+   * Simple delete of regular files from the {@link FileSystem}.
+   * <p>
+   * This method is a more generic implementation that the other deleteXXX
+   * methods in this class, allowing more code reuse at the cost of a couple
+   * more, short-lived objects (which should have minimum impact on the jvm).
+   * @param fs {@link FileSystem} where the files live
+   * @param files {@link Collection} of files to be deleted
+   * @throws IOException if a file cannot be deleted. All files will be
+   *           attempted to deleted before throwing the exception, rather than
+   *           failing at the first file.
+   */
+  private static void deleteFilesWithoutArchiving(Collection<File> files) throws IOException {
+    List<IOException> errors = new ArrayList<IOException>(0);
+    for (File file : files) {
+      try {
+        LOG.debug("Deleting region file:" + file);
+        file.delete();
+      } catch (IOException e) {
+        LOG.error("Failed to delete file:" + file);
+        errors.add(e);
+      }
+    }
+    if (errors.size() > 0) {
+      throw MultipleIOException.createIOException(errors);
+    }
+  }
+
+  /**
+   * Without regard for backup, delete a region. Should be used with caution.
+   * @param regionDir {@link Path} to the region to be deleted.
+   * @param fs FileSystem from which to delete the region
+   * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise
+   * @throws IOException on filesystem operation failure
+   */
+  private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
+      throws IOException {
+    if (fs.delete(regionDir, true)) {
+      LOG.debug("Deleted all region files in: " + regionDir);
+      return true;
+    }
+    LOG.debug("Failed to delete region directory:" + regionDir);
+    return false;
+  }
+
+  /**
+   * Just do a simple delete of the given store files
+   * <p>
+   * A best effort is made to delete each of the files, rather than bailing on the first failure.
+   * <p>
+   * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes
+   * less resources, but is limited in terms of usefulness
+   * @param compactedFiles store files to delete from the file system.
+   * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
+   *           throwing the exception, rather than failing at the first file.
+   */
+  private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles)
+      throws IOException {
+    LOG.debug("Deleting store files without archiving.");
+    List<IOException> errors = new ArrayList<IOException>(0);
+    for (StoreFile hsf : compactedFiles) {
+      try {
+        hsf.deleteReader();
+      } catch (IOException e) {
+        LOG.error("Failed to delete store file:" + hsf.getPath());
+        errors.add(e);
+      }
+    }
+    if (errors.size() > 0) {
+      throw MultipleIOException.createIOException(errors);
+    }
+  }
+
+  /**
+   * Adapt a type to match the {@link File} interface, which is used internally for handling
+   * archival/removal of files
+   * @param <T> type to adapt to the {@link File} interface
+   */
+  private static abstract class FileConverter<T> implements Function<T, File> {
+    protected final FileSystem fs;
+
+    public FileConverter(FileSystem fs) {
+      this.fs = fs;
+    }
+  }
+
+  /**
+   * Convert a FileStatus to something we can manage in the archiving
+   */
+  private static class FileStatusConverter extends FileConverter<FileStatus> {
+    public FileStatusConverter(FileSystem fs) {
+      super(fs);
+    }
+
+    @Override
+    public File apply(FileStatus input) {
+      return new FileablePath(fs, input.getPath());
+    }
+  }
+
+  /**
+   * Convert the {@link StoreFile} into something we can manage in the archive
+   * methods
+   */
+  private static class StoreToFile extends FileConverter<StoreFile> {
+    public StoreToFile(FileSystem fs) {
+      super(fs);
+    }
+
+    @Override
+    public File apply(StoreFile input) {
+      return new FileableStoreFile(fs, input);
+    }
+  }
+
+  /**
+   * Wrapper to handle file operations uniformly
+   */
+  private static abstract class File {
+    protected final FileSystem fs;
+
+    public File(FileSystem fs) {
+      this.fs = fs;
+    }
+
+    /**
+     * Delete the file
+     * @throws IOException on failure
+     */
+    abstract void delete() throws IOException;
+
+    /**
+     * Check to see if this is a file or a directory
+     * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise
+     * @throws IOException on {@link FileSystem} connection error
+     */
+    abstract boolean isFile() throws IOException;
+
+    /**
+     * @return if this is a directory, returns all the children in the
+     *         directory, otherwise returns an empty list
+     * @throws IOException
+     */
+    abstract Collection<File> getChildren() throws IOException;
+
+    /**
+     * close any outside readers of the file
+     * @throws IOException
+     */
+    abstract void close() throws IOException;
+
+    /**
+     * @return the name of the file (not the full fs path, just the individual
+     *         file name)
+     */
+    abstract String getName();
+
+    /**
+     * @return the path to this file
+     */
+    abstract Path getPath();
+
+    /**
+     * Move the file to the given destination
+     * @param dest
+     * @return <tt>true</tt> on success
+     * @throws IOException
+     */
+    public boolean moveAndClose(Path dest) throws IOException {
+      this.close();
+      Path p = this.getPath();
+      return !fs.rename(p, dest);
+    }
+
+    /**
+     * @return the {@link FileSystem} on which this file resides
+     */
+    public FileSystem getFileSystem() {
+      return this.fs;
+    }
+
+    @Override
+    public String toString() {
+      return this.getClass() + ", file:" + getPath().toString();
+    }
+  }
+
+  /**
+   * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}.
+   */
+  private static class FileablePath extends File {
+    private final Path file;
+    private final FileStatusConverter getAsFile;
+
+    public FileablePath(FileSystem fs, Path file) {
+      super(fs);
+      this.file = file;
+      this.getAsFile = new FileStatusConverter(fs);
+    }
+
+    @Override
+    public void delete() throws IOException {
+      if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file);
+    }
+
+    @Override
+    public String getName() {
+      return file.getName();
+    }
+
+    @Override
+    public Collection<File> getChildren() throws IOException {
+      if (fs.isFile(file)) return Collections.emptyList();
+      return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile);
+    }
+
+    @Override
+    public boolean isFile() throws IOException {
+      return fs.isFile(file);
+    }
+
+    @Override
+    public void close() throws IOException {
+      // NOOP - files are implicitly closed on removal
+    }
+
+    @Override
+    Path getPath() {
+      return file;
+    }
+  }
+
+  /**
+   * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem}
+   * .
+   */
+  private static class FileableStoreFile extends File {
+    StoreFile file;
+
+    public FileableStoreFile(FileSystem fs, StoreFile store) {
+      super(fs);
+      this.file = store;
+    }
+
+    @Override
+    public void delete() throws IOException {
+      file.deleteReader();
+    }
+
+    @Override
+    public String getName() {
+      return file.getPath().getName();
+    }
+
+    @Override
+    public boolean isFile() {
+      return true;
+    }
+
+    @Override
+    public Collection<File> getChildren() throws IOException {
+      // storefiles don't have children
+      return Collections.emptyList();
+    }
+
+    @Override
+    public void close() throws IOException {
+      file.closeReader(true);
+    }
+
+    @Override
+    Path getPath() {
+      return file.getPath();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index c3fcb99422eb..53a43589ad73 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -39,11 +39,10 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -51,7 +50,6 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
 
-
 /**
  * A janitor for the catalog tables.  Scans the <code>.META.</code> catalog
  * table on a period looking for unused regions to garbage collect.
@@ -237,7 +235,7 @@ boolean cleanParent(final HRegionInfo parent, Result rowContent)
     if (hasNoReferences(a) && hasNoReferences(b)) {
       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
         " because daughter splits no longer hold references");
-	  // wipe out daughter references from parent region
+      // wipe out daughter references from parent region in meta
       removeDaughtersFromParent(parent);
 
       // This latter regionOffline should not be necessary but is done for now
@@ -248,8 +246,7 @@ boolean cleanParent(final HRegionInfo parent, Result rowContent)
         this.services.getAssignmentManager().regionOffline(parent);
       }
       FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
-      Path rootdir = this.services.getMasterFileSystem().getRootDir();
-      HRegion.deleteRegion(fs, rootdir, parent);
+      HFileArchiver.archiveRegion(fs, parent);
       MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
       result = true;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 186c896875ae..b476638dbcb0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -44,6 +44,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -75,6 +76,8 @@
 import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
 import org.apache.hadoop.hbase.ipc.ProtocolSignature;
 import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
 import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
@@ -93,6 +96,7 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Pair;
@@ -203,6 +207,7 @@ public class HMaster extends HasThread
 
   private CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
+  private HFileCleaner hfileCleaner;
 
   private MasterCoprocessorHost cpHost;
   private final ServerName serverName;
@@ -834,12 +839,19 @@ private void startServiceThreads() throws IOException{
 
    // Start log cleaner thread
    String n = Thread.currentThread().getName();
+   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
    this.logCleaner =
-      new LogCleaner(conf.getInt("hbase.master.cleaner.interval", 60 * 1000),
+      new LogCleaner(cleanerInterval,
          this, conf, getMasterFileSystem().getFileSystem(),
          getMasterFileSystem().getOldLogDir());
          Threads.setDaemonThreadRunning(logCleaner.getThread(), n + ".oldLogCleaner");
 
+   //start the hfile archive cleaner thread
+    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
+    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
+        .getFileSystem(), archiveDir);
+    Threads.setDaemonThreadRunning(hfileCleaner.getThread(), n + ".archivedHFileCleaner");
+
    // Put up info server.
    int port = this.conf.getInt("hbase.master.info.port", 60010);
    if (port >= 0) {
@@ -866,6 +878,8 @@ private void stopServiceThreads() {
     if (this.rpcServer != null) this.rpcServer.stop();
     // Clean up and close up shop
     if (this.logCleaner!= null) this.logCleaner.interrupt();
+    if (this.hfileCleaner != null) this.hfileCleaner.interrupt();
+
     if (this.infoServer != null) {
       LOG.info("Stopping infoServer");
       try {
@@ -1815,4 +1829,12 @@ void registerMBean() {
     MBeanUtil.registerMBean("Master", "Master", mxBeanInfo);
     LOG.info("Registered HMaster MXBean");
   }
-}
+
+  /**
+   * Exposed for Testing!
+   * @return the current hfile cleaner
+   */
+  public HFileCleaner getHFileCleaner() {
+    return this.hfileCleaner;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
deleted file mode 100644
index b5fc66501939..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
-
-/**
- * This Chore, everytime it runs, will clear the HLogs in the old logs folder
- * that are deletable for each log cleaner in the chain.
- */
-public class LogCleaner extends Chore {
-  static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
-
-  private final FileSystem fs;
-  private final Path oldLogDir;
-  private List<LogCleanerDelegate> logCleanersChain;
-  private final Configuration conf;
-
-  /**
-   *
-   * @param p the period of time to sleep between each run
-   * @param s the stopper
-   * @param conf configuration to use
-   * @param fs handle to the FS
-   * @param oldLogDir the path to the archived logs
-   */
-  public LogCleaner(final int p, final Stoppable s,
-                        Configuration conf, FileSystem fs,
-                        Path oldLogDir) {
-    super("LogsCleaner", p, s);
-    this.fs = fs;
-    this.oldLogDir = oldLogDir;
-    this.conf = conf;
-    this.logCleanersChain = new LinkedList<LogCleanerDelegate>();
-
-    initLogCleanersChain();
-  }
-
-  /*
-   * Initialize the chain of log cleaners from the configuration. The default
-   * in this chain are: TimeToLiveLogCleaner and ReplicationLogCleaner.
-   */
-  private void initLogCleanersChain() {
-    String[] logCleaners = conf.getStrings(HBASE_MASTER_LOGCLEANER_PLUGINS);
-    if (logCleaners != null) {
-      for (String className : logCleaners) {
-        LogCleanerDelegate logCleaner = newLogCleaner(className, conf);
-        addLogCleaner(logCleaner);
-      }
-    }
-  }
-
-  /**
-   * A utility method to create new instances of LogCleanerDelegate based
-   * on the class name of the LogCleanerDelegate.
-   * @param className fully qualified class name of the LogCleanerDelegate
-   * @param conf
-   * @return the new instance
-   */
-  public static LogCleanerDelegate newLogCleaner(String className, Configuration conf) {
-    try {
-      Class c = Class.forName(className);
-      LogCleanerDelegate cleaner = (LogCleanerDelegate) c.newInstance();
-      cleaner.setConf(conf);
-      return cleaner;
-    } catch(Exception e) {
-      LOG.warn("Can NOT create LogCleanerDelegate: " + className, e);
-      // skipping if can't instantiate
-      return null;
-    }
-  }
-
-  /**
-   * Add a LogCleanerDelegate to the log cleaner chain. A log file is deletable
-   * if it is deletable for each LogCleanerDelegate in the chain.
-   * @param logCleaner
-   */
-  public void addLogCleaner(LogCleanerDelegate logCleaner) {
-    if (logCleaner != null && !logCleanersChain.contains(logCleaner)) {
-      logCleanersChain.add(logCleaner);
-      LOG.debug("Add log cleaner in chain: " + logCleaner.getClass().getName());
-    }
-  }
-
-  @Override
-  protected void chore() {
-    try {
-      FileStatus [] files = FSUtils.listStatus(this.fs, this.oldLogDir, null);
-      if (files == null) return;
-      FILE: for (FileStatus file : files) {
-        Path filePath = file.getPath();
-        if (HLog.validateHLogFilename(filePath.getName())) {
-          for (LogCleanerDelegate logCleaner : logCleanersChain) {
-            if (logCleaner.isStopped()) {
-              LOG.warn("A log cleaner is stopped, won't delete any log.");
-              return;
-            }
-
-            if (!logCleaner.isLogDeletable(filePath) ) {
-              // this log is not deletable, continue to process next log file
-              continue FILE;
-            }
-          }
-          // delete this log file if it passes all the log cleaners
-          this.fs.delete(filePath, true);
-        } else {
-          LOG.warn("Found a wrongly formated file: "
-              + file.getPath().getName());
-          this.fs.delete(filePath, true);
-        }
-      }
-    } catch (IOException e) {
-      e = RemoteExceptionHandler.checkIOException(e);
-      LOG.warn("Error while cleaning the logs", e);
-    }
-  }
-
-  @Override
-  public void run() {
-    try {
-      super.run();
-    } finally {
-      for (LogCleanerDelegate lc: this.logCleanersChain) {
-        try {
-          lc.stop("Exiting");
-        } catch (Throwable t) {
-          LOG.warn("Stopping", t);
-        }
-      }
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 7de0136bcb05..3552c22066f1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
@@ -440,7 +441,7 @@ public static void setInfoFamilyCachingForMeta(final boolean b) {
 
 
   public void deleteRegion(HRegionInfo region) throws IOException {
-    fs.delete(HRegion.getRegionDir(rootdir, region), true);
+    HFileArchiver.archiveRegion(fs, region);
   }
 
   public void deleteTable(byte[] tableName) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
new file mode 100644
index 000000000000..ed890ac98928
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.BaseConfigurable;
+
+/**
+ * Base class for the hfile cleaning function inside the master. By default, only the
+ * {@link TimeToLiveHFileCleaner} is called.
+ * <p>
+ * If other effects are needed, implement your own LogCleanerDelegate and add it to the
+ * configuration "hbase.master.hfilecleaner.plugins", which is a comma-separated list of fully
+ * qualified class names. The <code>HFileCleaner<code> will build the cleaner chain in 
+ * order the order specified by the configuration.
+ * <p>
+ * For subclasses, setConf will be called exactly <i>once</i> before using the cleaner.
+ * <p>
+ * Since {@link BaseHFileCleanerDelegate HFileCleanerDelegates} are created in
+ * HFileCleaner by reflection, classes that implements this interface <b>must</b>
+ * provide a default constructor.
+ */
+@InterfaceAudience.Private
+public abstract class BaseHFileCleanerDelegate extends BaseConfigurable implements
+    FileCleanerDelegate {
+
+  private boolean stopped = false;
+
+  @Override
+  public void stop(String why) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
similarity index 52%
rename from src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
rename to src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
index 3c8eb0a1c2c1..d62d347d4868 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
@@ -1,6 +1,4 @@
-/*
- * Copyright 2010 The Apache Software Foundation
- *
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,32 +15,42 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase.master;
+package org.apache.hadoop.hbase.master.cleaner;
 
-import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.BaseConfigurable;
 
 /**
- * Interface for the log cleaning function inside the master. By default, two 
- * cleaners: <code>TimeToLiveLogCleaner</code> and 
- * <code>ReplicationLogCleaner</code> are called in order. So if other 
- * effects are needed, implement your own LogCleanerDelegate and add it to the
+ * Base class for the log cleaning function inside the master. By default, two
+ * cleaners: <code>TimeToLiveLogCleaner</code> and
+ * <code>ReplicationLogCleaner</code> are called in order. So if other effects
+ * are needed, implement your own LogCleanerDelegate and add it to the
  * configuration "hbase.master.logcleaner.plugins", which is a comma-separated
  * list of fully qualified class names. LogsCleaner will add it to the chain.
- *
- * <p>HBase ships with LogsCleaner as the default implementation.
- *
- * <p>This interface extends Configurable, so setConf needs to be called once
- * before using the cleaner.
- * Since LogCleanerDelegates are created in LogsCleaner by reflection. Classes
- * that implements this interface should provide a default constructor.
+ * <p>
+ * HBase ships with LogsCleaner as the default implementation.
+ * <p>
+ * This interface extends Configurable, so setConf needs to be called once
+ * before using the cleaner. Since LogCleanerDelegates are created in
+ * LogsCleaner by reflection. Classes that implements this interface should
+ * provide a default constructor.
  */
-public interface LogCleanerDelegate extends Configurable, Stoppable {
+@InterfaceAudience.Private
+public abstract class BaseLogCleanerDelegate extends BaseConfigurable implements FileCleanerDelegate {
+
+  @Override
+  public boolean isFileDeletable(Path file) {
+    return isLogDeletable(file);
+  }
+
   /**
    * Should the master delete the log or keep it?
+   * <p>
+   * Implementing classes should override {@link #isFileDeletable(Path)} instead.
    * @param filePath full path to log.
    * @return true if the log is deletable, false if not
    */
-  public boolean isLogDeletable(Path filePath);
+  @Deprecated
+  public abstract boolean isLogDeletable(Path filePath);
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
new file mode 100644
index 000000000000..c5be87f989cc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Abstract Cleaner that uses a chain of delegates to clean a directory of files
+ * @param <T> Cleaner delegate class that is dynamically loaded from configuration
+ */
+public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
+
+  private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
+
+  private final FileSystem fs;
+  private final Path oldFileDir;
+  private final Configuration conf;
+  List<T> cleanersChain;
+
+  /**
+   * @param name name of the chore being run
+   * @param sleepPeriod the period of time to sleep between each run
+   * @param s the stopper
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param oldFileDir the path to the archived files
+   * @param confKey configuration key for the classes to instantiate
+   */
+  public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
+      FileSystem fs, Path oldFileDir, String confKey) {
+    super(name, sleepPeriod, s);
+    this.fs = fs;
+    this.oldFileDir = oldFileDir;
+    this.conf = conf;
+
+    initCleanerChain(confKey);
+  }
+
+  /**
+   * Validate the file to see if it even belongs in the directory. If it is valid, then the file
+   * will go through the cleaner delegates, but otherwise the file is just deleted.
+   * @param file full {@link Path} of the file to be checked
+   * @return <tt>true</tt> if the file is valid, <tt>false</tt> otherwise
+   */
+  protected abstract boolean validate(Path file);
+
+  /**
+   * Instanitate and initialize all the file cleaners set in the configuration
+   * @param confKey key to get the file cleaner classes from the configuration
+   */
+  private void initCleanerChain(String confKey) {
+    this.cleanersChain = new LinkedList<T>();
+    String[] logCleaners = conf.getStrings(confKey);
+    if (logCleaners != null) {
+      for (String className : logCleaners) {
+        T logCleaner = newFileCleaner(className, conf);
+        if (logCleaner != null) this.cleanersChain.add(logCleaner);
+      }
+    }
+  }
+
+  /**
+   * A utility method to create new instances of LogCleanerDelegate based on the class name of the
+   * LogCleanerDelegate.
+   * @param className fully qualified class name of the LogCleanerDelegate
+   * @param conf
+   * @return the new instance
+   */
+  public T newFileCleaner(String className, Configuration conf) {
+    try {
+      Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
+        FileCleanerDelegate.class);
+      @SuppressWarnings("unchecked")
+      T cleaner = (T) c.newInstance();
+      cleaner.setConf(conf);
+      return cleaner;
+    } catch (Exception e) {
+      LOG.warn("Can NOT create CleanerDelegate: " + className, e);
+      // skipping if can't instantiate
+      return null;
+    }
+  }
+
+  @Override
+  protected void chore() {
+    try {
+      FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
+      // if the path (file or directory) doesn't exist, then we can just return
+      if (files == null) return;
+      // loop over the found files and see if they should be deleted
+      for (FileStatus file : files) {
+        try {
+          if (file.isDir()) checkAndDeleteDirectory(file.getPath());
+          else checkAndDelete(file.getPath());
+        } catch (IOException e) {
+          e = RemoteExceptionHandler.checkIOException(e);
+          LOG.warn("Error while cleaning the logs", e);
+        }
+      }
+    } catch (IOException e) {
+      LOG.warn("Failed to get status of:" + oldFileDir);
+    }
+
+  }
+
+  /**
+   * Attempt to delete a directory and all files under that directory. Each child file is passed
+   * through the delegates to see if it can be deleted. If the directory has not children when the
+   * cleaners have finished it is deleted.
+   * <p>
+   * If new children files are added between checks of the directory, the directory will <b>not</b>
+   * be deleted.
+   * @param toCheck directory to check
+   * @return <tt>true</tt> if the directory was deleted, <tt>false</tt> otherwise.
+   * @throws IOException if there is an unexpected filesystem error
+   */
+  private boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
+    LOG.debug("Checking directory: " + toCheck);
+    FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
+    // if the directory doesn't exist, then we are done
+    if (children == null) return true;
+
+    boolean canDeleteThis = true;
+    for (FileStatus child : children) {
+      Path path = child.getPath();
+      // attempt to delete all the files under the directory
+      if (child.isDir()) {
+        if (!checkAndDeleteDirectory(path)) {
+          canDeleteThis = false;
+        }
+      }
+      // otherwise we can just check the file
+      else if (!checkAndDelete(path)) {
+        canDeleteThis = false;
+      }
+    }
+
+    // if all the children have been deleted, then we should try to delete this directory. However,
+    // don't do so recursively so we don't delete files that have been added since we checked.
+    return canDeleteThis ? fs.delete(toCheck, false) : false;
+  }
+
+  /**
+   * Run the given file through each of the cleaners to see if it should be deleted, deleting it if
+   * necessary.
+   * @param filePath path of the file to check (and possibly delete)
+   * @throws IOException if cann't delete a file because of a filesystem issue
+   * @throws IllegalArgumentException if the file is a directory and has children
+   */
+  private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
+    // first check to see if the path is valid
+    if (!validate(filePath)) {
+      LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
+      boolean success = this.fs.delete(filePath, true);
+      if (!success) LOG.warn("Attempted to delete:" + filePath
+          + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
+
+      return success;
+    }
+    // check each of the cleaners for the file
+    for (T cleaner : cleanersChain) {
+      if (cleaner.isStopped() || this.stopper.isStopped()) {
+        LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
+            + this.oldFileDir);
+        return false;
+      }
+
+      if (!cleaner.isFileDeletable(filePath)) {
+        // this file is not deletable, then we are done
+        LOG.debug(filePath + " is not deletable according to:" + cleaner);
+        return false;
+      }
+    }
+    // delete this file if it passes all the cleaners
+    LOG.debug("Removing:" + filePath + " from archive");
+    boolean success = this.fs.delete(filePath, false);
+    if (!success) {
+      LOG.warn("Attempted to delete:" + filePath
+          + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
+    }
+    return success;
+  }
+
+  @Override
+  public void cleanup() {
+    for (T lc : this.cleanersChain) {
+      try {
+        lc.stop("Exiting");
+      } catch (Throwable t) {
+        LOG.warn("Stopping", t);
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
new file mode 100644
index 000000000000..4cf503881c79
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+
+/**
+ * General interface for cleaning files from a folder (generally an archive or
+ * backup folder). These are chained via the {@link CleanerChore} to determine
+ * if a given file should be deleted.
+ */
+@InterfaceAudience.Private
+public interface FileCleanerDelegate extends Configurable, Stoppable {
+
+  /**
+   * Should the master delete the file or keep it?
+   * @param file full path to the file to check
+   * @return <tt>true</tt> if the file is deletable, <tt>false</tt> if not
+   */
+  public boolean isFileDeletable(Path file);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
new file mode 100644
index 000000000000..06a612d5ba52
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+/**
+ * This Chore, every time it runs, will clear the HFiles in the hfile archive
+ * folder that are deletable for each HFile cleaner in the chain.
+ */
+@InterfaceAudience.Private
+public class HFileCleaner extends CleanerChore<BaseHFileCleanerDelegate> {
+
+  public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins";
+
+  /**
+   * @param period the period of time to sleep between each run
+   * @param stopper the stopper
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param directory directory to be cleaned
+   */
+  public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs,
+      Path directory) {
+    super("HFileCleaner", period, stopper, conf, fs, directory, MASTER_HFILE_CLEANER_PLUGINS);
+  }
+
+  @Override
+  protected boolean validate(Path file) {
+    return StoreFile.validateStoreFileName(file.getName());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
new file mode 100644
index 000000000000..0e5bd3d86740
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+/**
+ * This Chore, every time it runs, will attempt to delete the HLogs in the old logs folder. The HLog
+ * is only deleted if none of the cleaner delegates says otherwise.
+ * @see BaseLogCleanerDelegate
+ */
+@InterfaceAudience.Private
+public class LogCleaner extends CleanerChore<BaseLogCleanerDelegate> {
+  static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
+
+  /**
+   * @param p the period of time to sleep between each run
+   * @param s the stopper
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param oldLogDir the path to the archived logs
+   */
+  public LogCleaner(final int p, final Stoppable s, Configuration conf, FileSystem fs,
+      Path oldLogDir) {
+    super("LogsCleaner", p, s, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS);
+  }
+
+  @Override
+  protected boolean validate(Path file) {
+    return HLog.validateHLogFilename(file.getName());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
new file mode 100644
index 000000000000..40877b75c24f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * HFile cleaner that uses the timestamp of the hfile to determine if it should be deleted. By
+ * default they are allowed to live for {@value TimeToLiveHFileCleaner#DEFAULT_TTL}
+ */
+@InterfaceAudience.Private
+public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate {
+
+  public static final Log LOG = LogFactory.getLog(TimeToLiveHFileCleaner.class.getName());
+  public static final String TTL_CONF_KEY = "hbase.master.hfilecleaner.ttl";
+  // default ttl = 5 minutes
+  private static final long DEFAULT_TTL = 60000 * 5;
+  // Configured time a hfile can be kept after it was moved to the archive
+  private long ttl;
+  private FileSystem fs;
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.ttl = conf.getLong(TTL_CONF_KEY, DEFAULT_TTL);
+    super.setConf(conf);
+  }
+
+  @Override
+  public boolean isFileDeletable(Path filePath) {
+    if (!instantiateFS()) {
+      return false;
+    }
+    long time = 0;
+    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+    try {
+      FileStatus fStat = fs.getFileStatus(filePath);
+      time = fStat.getModificationTime();
+    } catch (IOException e) {
+      LOG.error("Unable to get modification time of file " + filePath.getName()
+          + ", not deleting it.", e);
+      return false;
+    }
+    long life = currentTime - time;
+    LOG.debug("Life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " + time);
+    if (life < 0) {
+      LOG.warn("Found a log (" + filePath + ") newer than current time (" + currentTime + " < "
+          + time + "), probably a clock skew");
+      return false;
+    }
+    return life > ttl;
+  }
+
+  /**
+   * setup the filesystem, if it hasn't been already
+   */
+  private synchronized boolean instantiateFS() {
+    if (this.fs == null) {
+      try {
+        this.fs = FileSystem.get(this.getConf());
+      } catch (IOException e) {
+        LOG.error("Couldn't instantiate the file system, not deleting file, just incase");
+        return false;
+      }
+    }
+    return true;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
similarity index 86%
rename from src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
rename to src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
index dde820700d2f..6043a8a2847d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
@@ -1,6 +1,4 @@
-/*
- * Copyright 2010 The Apache Software Foundation
- *
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,12 +15,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase.master;
+package org.apache.hadoop.hbase.master.cleaner;
 
 import java.io.IOException;
 
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,9 +30,9 @@
  * Log cleaner that uses the timestamp of the hlog to determine if it should
  * be deleted. By default they are allowed to live for 10 minutes.
  */
-public class TimeToLiveLogCleaner implements LogCleanerDelegate {
+@InterfaceAudience.Private
+public class TimeToLiveLogCleaner extends BaseLogCleanerDelegate {
   static final Log LOG = LogFactory.getLog(TimeToLiveLogCleaner.class.getName());
-  private Configuration conf;
   // Configured time a log can be kept after it was closed
   private long ttl;
   private boolean stopped = false;
@@ -43,7 +42,7 @@ public boolean isLogDeletable(Path filePath) {
     long time = 0;
     long currentTime = System.currentTimeMillis();
     try {
-      FileStatus fStat = filePath.getFileSystem(conf).getFileStatus(filePath);
+      FileStatus fStat = filePath.getFileSystem(this.getConf()).getFileStatus(filePath);
       time = fStat.getModificationTime();
     } catch (IOException e) {
       LOG.error("Unable to get modification time of file " + filePath.getName() +
@@ -61,14 +60,10 @@ public boolean isLogDeletable(Path filePath) {
 
   @Override
   public void setConf(Configuration conf) {
-    this.conf = conf;
+    super.setConf(conf);
     this.ttl = conf.getLong("hbase.master.logcleaner.ttl", 600000);
   }
 
-  @Override
-  public Configuration getConf() {
-    return conf;
-  }
 
   @Override
   public void stop(String why) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index cd8a18d27bd9..fd1b825a67a0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -77,6 +77,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Delete;
@@ -869,16 +870,7 @@ private List<StoreFile> doClose(
       writestate.writesEnabled = false;
       wasFlushing = writestate.flushing;
       LOG.debug("Closing " + this + ": disabling compactions & flushes");
-      while (writestate.compacting > 0 || writestate.flushing) {
-        LOG.debug("waiting for " + writestate.compacting + " compactions" +
-            (writestate.flushing ? " & cache flush" : "") +
-            " to complete for region " + this);
-        try {
-          writestate.wait();
-        } catch (InterruptedException iex) {
-          // continue
-        }
-      }
+      waitForFlushesAndCompactions();
     }
     // If we were not just flushing, is it worth doing a preflush...one
     // that will clear out of the bulk of the memstore before we put up
@@ -953,6 +945,26 @@ public ImmutableList<StoreFile> call() throws IOException {
     }
   }
 
+  /**
+   * Wait for all current flushes and compactions of the region to complete.
+   * <p>
+   * Exposed for TESTING.
+   */
+  public void waitForFlushesAndCompactions() {
+    synchronized (writestate) {
+      while (writestate.compacting > 0 || writestate.flushing) {
+        LOG.debug("waiting for " + writestate.compacting + " compactions"
+            + (writestate.flushing ? " & cache flush" : "") + " to complete for region " + this);
+        try {
+          writestate.wait();
+        } catch (InterruptedException iex) {
+          // essentially ignore and propagate the interrupt back up
+          Thread.currentThread().interrupt();
+        }
+      }
+    }
+  }
+
   protected ThreadPoolExecutor getStoreOpenAndCloseThreadPool(
       final String threadNamePrefix) {
     int numStores = Math.max(1, this.htableDescriptor.getFamilies().size());
@@ -4088,8 +4100,13 @@ public static HRegion merge(HRegion a, HRegion b)
       LOG.debug("Files for new region");
       listPaths(fs, dstRegion.getRegionDir());
     }
-    deleteRegion(fs, a.getRegionDir());
-    deleteRegion(fs, b.getRegionDir());
+
+    // delete out the 'A' region
+    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(a.getConf()), a.getTableDir(),
+      a.getRegionDir());
+    // delete out the 'B' region
+    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(b.getConf()), b.getTableDir(),
+      b.getRegionDir());
 
     LOG.info("merge completed. New region is " + dstRegion);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 6ef155e5c159..f9e1103b0c10 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.HeapSize;
@@ -457,7 +458,7 @@ protected void rollback(final KeyValue kv) {
   /**
    * @return All store files.
    */
-  List<StoreFile> getStorefiles() {
+  public List<StoreFile> getStorefiles() {
     return this.storefiles;
   }
 
@@ -1732,10 +1733,12 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
 
       // Tell observers that list of StoreFiles has changed.
       notifyChangedReadersObservers();
-      // Finally, delete old store files.
-      for (StoreFile hsf: compactedFiles) {
-        hsf.deleteReader();
-      }
+
+      // let the archive util decide if we should archive or delete the files
+      LOG.debug("Removing store files after compaction...");
+      HFileArchiver.archiveStoreFiles(this.fs, this.region, this.conf, this.family.getName(),
+        compactedFiles);
+
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
       LOG.error("Failed replacing compacted files in " + this.storeNameStr +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 76566e565cff..af3ab839a2b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -861,13 +861,20 @@ static Path getRandomFilename(final FileSystem fs,
   }
 
   /**
-   * Write out a split reference.
-   *
-   * Package local so it doesnt leak out of regionserver.
-   *
+   * Validate the store file name.
+   * @param fileName name of the file to validate
+   * @return <tt>true</tt> if the file could be a valid store file, <tt>false</tt> otherwise
+   */
+  public static boolean validateStoreFileName(String fileName) {
+    return !fileName.contains("-");
+  }
+
+  /**
+   * Write out a split reference. Package local so it doesnt leak out of
+   * regionserver.
    * @param fs
    * @param splitDir Presumes path format is actually
-   * <code>SOME_DIRECTORY/REGIONNAME/FAMILY</code>.
+   *          <code>SOME_DIRECTORY/REGIONNAME/FAMILY</code>.
    * @param f File to split.
    * @param splitRow
    * @param range
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 531d04999e18..56659512001e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1768,6 +1768,11 @@ protected Path getDir() {
     return dir;
   }
   
+  /**
+   * @param filename name of the file to validate
+   * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
+   *         otherwise
+   */
   public static boolean validateHLogFilename(String filename) {
     return pattern.matcher(filename).matches();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 05d03104ff29..21a58db58f4d 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.master.LogCleanerDelegate;
+import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -40,9 +40,8 @@
  * Implementation of a log cleaner that checks if a log is still scheduled for
  * replication before deleting it when its TTL is over.
  */
-public class ReplicationLogCleaner implements LogCleanerDelegate, Abortable {
+public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abortable {
   private static final Log LOG = LogFactory.getLog(ReplicationLogCleaner.class);
-  private Configuration conf;
   private ReplicationZookeeper zkHelper;
   private Set<String> hlogs = new HashSet<String>();
   private boolean stopped = false;
@@ -67,7 +66,7 @@ public boolean isLogDeletable(Path filePath) {
 
     // all members of this class are null if replication is disabled, and we
     // return true since false would render the LogsCleaner useless
-    if (this.conf == null) {
+    if (this.getConf() == null) {
       return true;
     }
     String log = filePath.getName();
@@ -122,18 +121,18 @@ private boolean refreshHLogsAndSearch(String searchedLog) {
   }
 
   @Override
-  public void setConf(Configuration conf) {
+  public void setConf(Configuration config) {
     // If replication is disabled, keep all members null
-    if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
+    if (!config.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
       return;
     }
     // Make my own Configuration.  Then I'll have my own connection to zk that
     // I can close myself when comes time.
-    this.conf = new Configuration(conf);
+    Configuration conf = new Configuration(config);
+    super.setConf(conf);
     try {
-      ZooKeeperWatcher zkw =
-          new ZooKeeperWatcher(this.conf, "replicationLogCleaner", null);
-      this.zkHelper = new ReplicationZookeeper(this, this.conf, zkw);
+      ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "replicationLogCleaner", null);
+      this.zkHelper = new ReplicationZookeeper(this, conf, zkw);
     } catch (KeeperException e) {
       LOG.error("Error while configuring " + this.getClass().getName(), e);
     } catch (IOException e) {
@@ -142,10 +141,6 @@ public void setConf(Configuration conf) {
     refreshHLogsAndSearch(null);
   }
 
-  @Override
-  public Configuration getConf() {
-    return conf;
-  }
 
   @Override
   public void stop(String why) {
@@ -156,7 +151,7 @@ public void stop(String why) {
       this.zkHelper.getZookeeperWatcher().close();
     }
     // Not sure why we're deleting a connection that we never acquired or used
-    HConnectionManager.deleteConnection(this.conf, true);
+    HConnectionManager.deleteConnection(this.getConf(), true);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 22f536a5c0e6..27b61a25e634 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -562,6 +562,10 @@ public static Path getRootDir(final Configuration c) throws IOException {
     return p.makeQualified(fs);
   }
 
+  public static void setRootDir(final Configuration c, final Path root) throws IOException {
+    c.set(HConstants.HBASE_DIR, root.toString());
+  }
+
   /**
    * Checks if root region exists
    *
@@ -1187,4 +1191,36 @@ private static boolean contains(String[] groups, String user) {
   public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
     return fs.exists(path);
   }
+
+  /**
+   * Log the current state of the filesystem from a certain root directory
+   * @param fs filesystem to investigate
+   * @param root root file/directory to start logging from
+   * @param LOG log to output information
+   * @throws IOException if an unexpected exception occurs
+   */
+  public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
+      throws IOException {
+    LOG.debug("Current file system:");
+    logFSTree(LOG, fs, root, "|-");
+  }
+
+  /**
+   * Recursive helper to log the state of the FS
+   * @see #logFileSystemState(FileSystem, Path, Log)
+   */
+  private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
+      throws IOException {
+    FileStatus[] files = FSUtils.listStatus(fs, root, null);
+    if (files == null) return;
+
+    for (FileStatus file : files) {
+      if (file.isDir()) {
+        LOG.debug(prefix + file.getPath().getName() + "/");
+        logFSTree(LOG, fs, file.getPath(), prefix + "---");
+      } else {
+        LOG.debug(prefix + file.getPath().getName());
+      }
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
new file mode 100644
index 000000000000..266fd1a00137
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Store;
+
+/**
+ * Helper class for all utilities related to archival/retrieval of HFiles
+ */
+public class HFileArchiveUtil {
+
+  private HFileArchiveUtil() {
+    // non-external instantiation - util class
+  }
+
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param region parent region information under which the store currently
+   *          lives
+   * @param family name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or
+   *         <tt>null</tt> if it should not be archived
+   */
+  public static Path getStoreArchivePath(Configuration conf, HRegion region, byte [] family){
+    return getStoreArchivePath(conf, region.getRegionInfo(), region.getTableDir(), family);
+  }
+
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name. Can be null.
+   * @param region parent region information under which the store currently lives
+   * @param tabledir directory for the table under which the store currently lives
+   * @param family name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or <tt>null</tt> if it should
+   *         not be archived
+   */
+  public static Path getStoreArchivePath(Configuration conf, HRegionInfo region, Path tabledir,
+      byte[] family) {
+    Path tableArchiveDir = getTableArchivePath(tabledir);
+    return Store.getStoreHomedir(tableArchiveDir,
+      HRegionInfo.encodeRegionName(region.getRegionName()), family);
+  }
+
+  /**
+   * Get the archive directory for a given region under the specified table
+   * @param conf {@link Configuration} to read the archive directory from. Can be null
+   * @param tabledir the original table directory. Cannot be null.
+   * @param regiondir the path to the region directory. Cannot be null.
+   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
+   *         should not be archived
+   */
+  public static Path getRegionArchiveDir(Configuration conf, Path tabledir, Path regiondir) {
+    // get the archive directory for a table
+    Path archiveDir = getTableArchivePath(tabledir);
+
+    // then add on the region path under the archive
+    String encodedRegionName = regiondir.getName();
+    return HRegion.getRegionDir(archiveDir, encodedRegionName);
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Get the path to the table's archive directory.
+   * <p>
+   * Generally of the form: /hbase/.archive/[tablename]
+   * @param tabledir directory of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(Path tabledir) {
+    Path root = tabledir.getParent();
+    return new Path(new Path(root,HConstants.HFILE_ARCHIVE_DIRECTORY), tabledir.getName());
+  }
+
+  /**
+   * Get the full path to the archive directory on the configured {@link FileSystem}
+   * @param conf to look for archive directory name and root directory. Cannot be null. Notes for
+   *          testing: requires a FileSystem root directory to be specified.
+   * @return the full {@link Path} to the archive directory, as defined by the configuration
+   * @throws IOException if an unexpected error occurs
+   */
+  public static Path getArchivePath(Configuration conf) throws IOException {
+    return new Path(FSUtils.getRootDir(conf), HConstants.HFILE_ARCHIVE_DIRECTORY);
+  }
+}
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 44ee6895ad73..f73c47704bd7 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -299,7 +299,7 @@
   </property>
   <property>
     <name>hbase.master.logcleaner.plugins</name>
-    <value>org.apache.hadoop.hbase.master.TimeToLiveLogCleaner</value>
+    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner</value>
     <description>A comma-separated list of LogCleanerDelegate invoked by
     the LogsCleaner service. These WAL/HLog cleaners are called in order,
     so put the HLog cleaner that prunes the most HLog files in front. To
@@ -869,5 +869,23 @@
 	In both cases, the aggregated metric M across tables and cfs will be reported.
     </description>
   </property>
-
+  <property>
+    <name>hbase.table.archive.directory</name>
+    <value>.archive</value>
+    <description>Per-table directory name under which to backup files for a
+      table. Files are moved to the same directories as they would be under the
+      table directory, but instead are just one level lower (under
+      table/.archive/... rather than table/...). Currently only applies to HFiles.</description>
+  </property>
+  <property>
+    <name>hbase.master.hfilecleaner.plugins</name>
+    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner</value>
+    <description>A comma-separated list of HFileCleanerDelegate invoked by
+    the HFileCleaner service. These HFiles cleaners are called in order,
+    so put the cleaner that prunes the most files in front. To
+    implement your own HFileCleanerDelegate, just put it in HBase's classpath
+    and add the fully qualified class name here. Always add the above
+    default log cleaners in the list as they will be overwritten in hbase-site.xml.
+    </description>
+  </property>
 </configuration>
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index e15eb47ea26f..887765399b1d 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -940,7 +940,11 @@ public HTable createTable(byte[] tableName, byte[][] families,
    * @param tableName existing table
    */
   public void deleteTable(byte[] tableName) throws IOException {
-    getHBaseAdmin().disableTable(tableName);
+    try {
+      getHBaseAdmin().disableTable(tableName);
+    } catch (TableNotEnabledException e) {
+      LOG.debug("Table: " + Bytes.toString(tableName) + " already disabled, so just deleting it.");
+    }
     getHBaseAdmin().deleteTable(tableName);
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
new file mode 100644
index 000000000000..f6142f24fda9
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -0,0 +1,293 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
+ * a region
+ */
+@Category(LargeTests.class)
+public class TestHFileArchiving {
+
+  private static final String STRING_TABLE_NAME = "test_table";
+
+  private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
+  private static final byte[] TEST_FAM = Bytes.toBytes("fam");
+
+  /**
+   * Setup the config for the cluster
+   */
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster();
+  }
+
+  private static void setupConf(Configuration conf) {
+    // disable the ui
+    conf.setInt("hbase.regionsever.info.port", -1);
+    // drop the memstore size so we get flushes
+    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+    // disable major compactions
+    conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
+  }
+
+  @Before
+  public void setup() throws Exception {
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    // cleanup the cluster if its up still
+    if (UTIL.getHBaseAdmin().tableExists(STRING_TABLE_NAME)) {
+      UTIL.deleteTable(TABLE_NAME);
+    }
+    // and cleanup the archive directory
+    try {
+      clearArchiveDirectory();
+    } catch (IOException e) {
+      Assert.fail("Failure to delete archive directory:" + e.getMessage());
+    }
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    try {
+      UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      // NOOP;
+    }
+  }
+
+  @Test
+  public void testRemovesRegionDirOnArchive() throws Exception {
+    final HBaseAdmin admin = UTIL.getHBaseAdmin();
+
+    // get the current store files for the region
+    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
+    // make sure we only have 1 region serving this table
+    assertEquals(1, servingRegions.size());
+    HRegion region = servingRegions.get(0);
+
+    // and load the table
+    UTIL.loadRegion(region, TEST_FAM);
+
+    // shutdown the table so we can manipulate the files
+    admin.disableTable(STRING_TABLE_NAME);
+
+    FileSystem fs = UTIL.getTestFileSystem();
+
+    // now attempt to depose the region
+    Path regionDir = HRegion.getRegionDir(region.getTableDir().getParent(), region.getRegionInfo());
+
+    HFileArchiver.archiveRegion(fs, region.getRegionInfo());
+
+    // check for the existence of the archive directory and some files in it
+    Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
+    assertTrue(fs.exists(archiveDir));
+
+    // check to make sure the store directory was copied
+    FileStatus[] stores = fs.listStatus(archiveDir);
+    assertTrue(stores.length == 1);
+
+    // make sure we archived the store files
+    FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
+    assertTrue(storeFiles.length > 0);
+
+    // then ensure the region's directory isn't present
+    assertFalse(fs.exists(regionDir));
+  }
+
+  /**
+   * Test that the region directory is removed when we archive a region without store files, but
+   * still has hidden files.
+   * @throws Exception
+   */
+  @Test
+  public void testDeleteRegionWithNoStoreFiles() throws Exception {
+    // get the current store files for the region
+    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
+    // make sure we only have 1 region serving this table
+    assertEquals(1, servingRegions.size());
+    HRegion region = servingRegions.get(0);
+
+    FileSystem fs = region.getFilesystem();
+
+    // make sure there are some files in the regiondir
+    Path rootDir = FSUtils.getRootDir(fs.getConf());
+    Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
+    FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
+    Assert.assertNotNull("No files in the region directory", regionFiles);
+    if (LOG.isDebugEnabled()) {
+      List<Path> files = new ArrayList<Path>();
+      for (FileStatus file : regionFiles) {
+        files.add(file.getPath());
+      }
+      LOG.debug("Current files:" + files);
+    }
+    // delete the visible folders so we just have hidden files/folders
+    final PathFilter dirFilter = new FSUtils.DirFilter(fs);
+    PathFilter nonHidden = new PathFilter() {
+      @Override
+      public boolean accept(Path file) {
+        return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
+      }
+    };
+    FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
+    for (FileStatus store : storeDirs) {
+      LOG.debug("Deleting store for test");
+      fs.delete(store.getPath(), true);
+    }
+
+    // then archive the region
+    HFileArchiver.archiveRegion(fs, region.getRegionInfo());
+
+    // and check to make sure the region directoy got deleted
+    assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
+  }
+
+  @Test
+  public void testArchiveOnTableDelete() throws Exception {
+    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
+    // make sure we only have 1 region serving this table
+    assertEquals(1, servingRegions.size());
+    HRegion region = servingRegions.get(0);
+
+    // get the parent RS and monitor
+    HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
+    FileSystem fs = hrs.getFileSystem();
+
+    // put some data on the region
+    LOG.debug("-------Loading table");
+    UTIL.loadRegion(region, TEST_FAM);
+
+    // get the hfiles in the region
+    List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
+    assertEquals("More that 1 region for test table.", 1, regions.size());
+
+    region = regions.get(0);
+    // wait for all the compactions to complete
+    region.waitForFlushesAndCompactions();
+
+    // disable table to prevent new updates
+    UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
+    LOG.debug("Disabled table");
+
+    // remove all the files from the archive to get a fair comparison
+    clearArchiveDirectory();
+
+    // then get the current store files
+    Path regionDir = region.getRegionDir();
+    List<String> storeFiles = getAllFileNames(fs, regionDir);
+    // remove all the non-storefile named files for the region
+    for (int i = 0; i < storeFiles.size(); i++) {
+      String file = storeFiles.get(i);
+      if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
+        storeFiles.remove(i--);
+      }
+    }
+    storeFiles.remove(HRegion.REGIONINFO_FILE);
+
+    // then delete the table so the hfiles get archived
+    UTIL.deleteTable(TABLE_NAME);
+
+    // then get the files in the archive directory.
+    Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
+    List<String> archivedFiles = getAllFileNames(fs, archiveDir);
+    Collections.sort(storeFiles);
+    Collections.sort(archivedFiles);
+
+    LOG.debug("Store files:");
+    for (int i = 0; i < storeFiles.size(); i++) {
+      LOG.debug(i + " - " + storeFiles.get(i));
+    }
+    LOG.debug("Archive files:");
+    for (int i = 0; i < archivedFiles.size(); i++) {
+      LOG.debug(i + " - " + archivedFiles.get(i));
+    }
+
+    assertTrue("Archived files are missing some of the store files!",
+      archivedFiles.containsAll(storeFiles));
+  }
+
+  private void clearArchiveDirectory() throws IOException {
+    UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
+  }
+
+  /**
+   * Get the names of all the files below the given directory
+   * @param fs
+   * @param archiveDir
+   * @return
+   * @throws IOException
+   */
+  private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
+    FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
+    return recurseOnFiles(fs, files, new ArrayList<String>());
+  }
+
+  /** Recursively lookup all the file names under the file[] array **/
+  private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
+      throws IOException {
+    if (files == null || files.length == 0) return fileNames;
+
+    for (FileStatus file : files) {
+      if (file.isDir()) {
+        recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
+      } else fileNames.add(file.getPath().getName());
+    }
+    return fileNames;
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index b4afa3bebbdf..f48e7a5daf6e 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -19,11 +19,12 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 
@@ -35,6 +36,8 @@
 import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -59,9 +62,12 @@
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
@@ -543,6 +549,161 @@ public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
     janitor.join();
   }
 
+  @Test
+  public void testArchiveOldRegion() throws Exception {
+    String table = "table";
+    HBaseTestingUtility htu = new HBaseTestingUtility();
+    setRootDirAndCleanIt(htu, "testCleanParent");
+    Server server = new MockServer(htu);
+    MasterServices services = new MockMasterServices(server);
+
+    // create the janitor
+    CatalogJanitor janitor = new CatalogJanitor(server, services);
+
+    // Create regions.
+    HTableDescriptor htd = new HTableDescriptor(table);
+    htd.addFamily(new HColumnDescriptor("f"));
+    HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
+    HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
+    HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
+    // Test that when both daughter regions are in place, that we do not
+    // remove the parent.
+    List<KeyValue> kvs = new ArrayList<KeyValue>();
+    kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
+        HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
+    kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
+        HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
+    Result r = new Result(kvs);
+
+    FileSystem fs = FileSystem.get(htu.getConfiguration());
+    Path rootdir = services.getMasterFileSystem().getRootDir();
+    // have to set the root directory since we use it in HFileDisposer to figure out to get to the
+    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
+    // the single test passes, but when the full suite is run, things get borked).
+    FSUtils.setRootDir(fs.getConf(), rootdir);
+    Path tabledir = HTableDescriptor.getTableDir(rootdir, htd.getName());
+    Path storedir = Store.getStoreHomedir(tabledir, parent.getEncodedName(),
+      htd.getColumnFamilies()[0].getName());
+
+    // delete the file and ensure that the files have been archived
+    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
+      tabledir, htd.getColumnFamilies()[0].getName());
+
+    // enable archiving, make sure that files get archived
+    addMockStoreFiles(2, services, storedir);
+    // get the current store files for comparison
+    FileStatus[] storeFiles = fs.listStatus(storedir);
+    for (FileStatus file : storeFiles) {
+      System.out.println("Have store file:" + file.getPath());
+    }
+
+    // do the cleaning of the parent
+    assertTrue(janitor.cleanParent(parent, r));
+
+    // and now check to make sure that the files have actually been archived
+    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
+    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
+
+    // cleanup
+    services.stop("Test finished");
+    server.stop("shutdown");
+    janitor.join();
+  }
+
+  /**
+   * Test that if a store file with the same name is present as those already backed up cause the
+   * already archived files to be timestamped backup
+   */
+  @Test
+  public void testDuplicateHFileResolution() throws Exception {
+    String table = "table";
+    HBaseTestingUtility htu = new HBaseTestingUtility();
+    setRootDirAndCleanIt(htu, "testCleanParent");
+    Server server = new MockServer(htu);
+    MasterServices services = new MockMasterServices(server);
+
+    // create the janitor
+    CatalogJanitor janitor = new CatalogJanitor(server, services);
+
+    // Create regions.
+    HTableDescriptor htd = new HTableDescriptor(table);
+    htd.addFamily(new HColumnDescriptor("f"));
+    HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
+    HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
+    HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
+    // Test that when both daughter regions are in place, that we do not
+    // remove the parent.
+    List<KeyValue> kvs = new ArrayList<KeyValue>();
+    kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
+        HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
+    kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
+        HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
+    Result r = new Result(kvs);
+
+    FileSystem fs = FileSystem.get(htu.getConfiguration());
+
+    Path rootdir = services.getMasterFileSystem().getRootDir();
+    // have to set the root directory since we use it in HFileDisposer to figure out to get to the
+    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
+    // the single test passes, but when the full suite is run, things get borked).
+    FSUtils.setRootDir(fs.getConf(), rootdir);
+    Path tabledir = HTableDescriptor.getTableDir(rootdir, parent.getTableName());
+    Path storedir = Store.getStoreHomedir(tabledir, parent.getEncodedName(),
+      htd.getColumnFamilies()[0].getName());
+    System.out.println("Old root:" + rootdir);
+    System.out.println("Old table:" + tabledir);
+    System.out.println("Old store:" + storedir);
+
+    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
+      tabledir, htd.getColumnFamilies()[0].getName());
+    System.out.println("Old archive:" + storeArchive);
+
+    // enable archiving, make sure that files get archived
+    addMockStoreFiles(2, services, storedir);
+    // get the current store files for comparison
+    FileStatus[] storeFiles = fs.listStatus(storedir);
+
+    // do the cleaning of the parent
+    assertTrue(janitor.cleanParent(parent, r));
+
+    // and now check to make sure that the files have actually been archived
+    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
+    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
+
+    // now add store files with the same names as before to check backup
+    // enable archiving, make sure that files get archived
+    addMockStoreFiles(2, services, storedir);
+
+    // do the cleaning of the parent
+    assertTrue(janitor.cleanParent(parent, r));
+
+    // and now check to make sure that the files have actually been archived
+    archivedStoreFiles = fs.listStatus(storeArchive);
+    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
+
+    // cleanup
+    services.stop("Test finished");
+    server.stop("shutdown");
+    janitor.join();
+  }
+
+  private void addMockStoreFiles(int count, MasterServices services, Path storedir)
+      throws IOException {
+    // get the existing store files
+    FileSystem fs = services.getMasterFileSystem().getFileSystem();
+    fs.mkdirs(storedir);
+    // create the store files in the parent
+    for (int i = 0; i < count; i++) {
+      Path storeFile = new Path(storedir, "_store" + i);
+      FSDataOutputStream dos = fs.create(storeFile, true);
+      dos.writeBytes("Some data: " + i);
+      dos.close();
+    }
+    // make sure the mock store files are there
+    FileStatus[] storeFiles = fs.listStatus(storedir);
+    assertEquals(count, storeFiles.length);
+  }
+
   private Result makeResultFromHRegionInfo(HRegionInfo region, HRegionInfo splita,
       HRegionInfo splitb) throws IOException {
     List<KeyValue> kvs = new ArrayList<KeyValue>();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
new file mode 100644
index 000000000000..f860aeccd9c8
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+@Category(SmallTests.class)
+public class TestCleanerChore {
+
+  private static final Log LOG = LogFactory.getLog(TestCleanerChore.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @After
+  public void cleanup() throws Exception {
+    // delete and recreate the test directory, ensuring a clean test dir between tests
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    fs.delete(testDir, true);
+    fs.mkdirs(testDir);
+  }
+
+  @Test
+  public void testSavesFilesOnRequest() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, NeverDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
+
+    // create the directory layout in the directory to clean
+    Path parent = new Path(testDir, "parent");
+    Path file = new Path(parent, "someFile");
+    fs.mkdirs(parent);
+    // touch a new file
+    fs.create(file).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+
+    // run the chore
+    chore.chore();
+
+    // verify all the files got deleted
+    assertTrue("File didn't get deleted", fs.exists(file));
+    assertTrue("Empty directory didn't get deleted", fs.exists(parent));
+  }
+
+  @Test
+  public void testDeletesEmptyDirectories() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
+
+    // create the directory layout in the directory to clean
+    Path parent = new Path(testDir, "parent");
+    Path child = new Path(parent, "child");
+    Path file = new Path(child, "someFile");
+    fs.mkdirs(child);
+    // touch a new file
+    fs.create(file).close();
+    // also create a file in the top level directory
+    Path topFile = new Path(testDir, "topFile");
+    fs.create(topFile).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+    assertTrue("Test file didn't get created.", fs.exists(topFile));
+
+    // run the chore
+    chore.chore();
+
+    // verify all the files got deleted
+    assertFalse("File didn't get deleted", fs.exists(topFile));
+    assertFalse("File didn't get deleted", fs.exists(file));
+    assertFalse("Empty directory didn't get deleted", fs.exists(child));
+    assertFalse("Empty directory didn't get deleted", fs.exists(parent));
+  }
+
+  /**
+   * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a
+   * directory.
+   * @throws Exception on failure
+   */
+  @Test
+  public void testDoesNotCheckDirectories() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
+    // spy on the delegate to ensure that we don't check for directories
+    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
+    AlwaysDelete spy = Mockito.spy(delegate);
+    chore.cleanersChain.set(0, spy);
+
+    // create the directory layout in the directory to clean
+    Path parent = new Path(testDir, "parent");
+    Path file = new Path(parent, "someFile");
+    fs.mkdirs(parent);
+    // touch a new file
+    fs.create(file).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+
+    chore.chore();
+    // make sure we never checked the directory
+    Mockito.verify(spy, Mockito.never()).isFileDeletable(parent);
+    Mockito.reset(spy);
+  }
+
+  @Test
+  public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
+
+    // also create a file in the top level directory
+    Path topFile = new Path(testDir, "topFile");
+    fs.create(topFile).close();
+    assertTrue("Test file didn't get created.", fs.exists(topFile));
+
+    // stop the chore
+    stop.stop("testing stop");
+
+    // run the chore
+    chore.chore();
+
+    // test that the file still exists
+    assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
+  }
+
+  /**
+   * While cleaning a directory, all the files in the directory may be deleted, but there may be
+   * another file added, in which case the directory shouldn't be deleted.
+   * @throws IOException on failure
+   */
+  @Test
+  public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    final Path testDir = UTIL.getDataTestDir();
+    final FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
+    // spy on the delegate to ensure that we don't check for directories
+    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
+    AlwaysDelete spy = Mockito.spy(delegate);
+    chore.cleanersChain.set(0, spy);
+
+    // create the directory layout in the directory to clean
+    final Path parent = new Path(testDir, "parent");
+    Path file = new Path(parent, "someFile");
+    fs.mkdirs(parent);
+    // touch a new file
+    fs.create(file).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+    final Path addedFile = new Path(parent, "addedFile");
+
+    // when we attempt to delete the original file, add another file in the same directory
+    Mockito.doAnswer(new Answer<Boolean>() {
+      @Override
+      public Boolean answer(InvocationOnMock invocation) throws Throwable {
+        fs.create(addedFile).close();
+        FSUtils.logFileSystemState(fs, testDir, LOG);
+        return (Boolean) invocation.callRealMethod();
+      }
+    }).when(spy).isFileDeletable(Mockito.any(Path.class));
+
+    // run the chore
+    chore.chore();
+
+    // make sure all the directories + added file exist, but the original file is deleted
+    assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
+    assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
+    assertFalse("Original file unexpectedly retained", fs.exists(file));
+    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
+    Mockito.reset(spy);
+  }
+
+  private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
+
+    public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,
+        Path oldFileDir, String confkey) {
+      super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey);
+    }
+
+    // all paths are valid
+    @Override
+    protected boolean validate(Path file) {
+      return true;
+    }
+  };
+
+  public static class AlwaysDelete extends BaseHFileCleanerDelegate {
+    @Override
+    public boolean isFileDeletable(Path file) {
+      return true;
+    }
+  }
+
+  public static class NeverDelete extends BaseHFileCleanerDelegate {
+    @Override
+    public boolean isFileDeletable(Path file) {
+      return false;
+    }
+  }
+
+  /**
+   * Simple helper class that just keeps track of whether or not its stopped.
+   */
+  private static class StoppableImplementation implements Stoppable {
+    private volatile boolean stop;
+
+    @Override
+    public void stop(String why) {
+      this.stop = true;
+    }
+
+    @Override
+    public boolean isStopped() {
+      return this.stop;
+    }
+
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
new file mode 100644
index 000000000000..ed80c3acbcfe
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.util.EnvironmentEdge;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestHFileCleaner {
+  private static final Log LOG = LogFactory.getLog(TestHFileCleaner.class);
+
+  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    // have to use a minidfs cluster because the localfs doesn't modify file times correctly
+    UTIL.startMiniDFSCluster(1);
+  }
+
+  @AfterClass
+  public static void shutdownCluster() throws Exception {
+    UTIL.shutdownMiniDFSCluster();
+  }
+
+  @Test
+  public void testTTLCleaner() throws IOException, InterruptedException {
+    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
+    Path root = UTIL.getDataTestDir();
+    Path file = new Path(root, "file");
+    fs.createNewFile(file);
+    long createTime = System.currentTimeMillis();
+    assertTrue("Test file not created!", fs.exists(file));
+    TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner();
+    // update the time info for the file, so the cleaner removes it
+    fs.setTimes(file, createTime - 100, -1);
+    Configuration conf = UTIL.getConfiguration();
+    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100);
+    cleaner.setConf(conf);
+    assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs)
+        + " with create time:" + createTime, cleaner.isFileDeletable(file));
+  }
+
+  /**
+   * @param file to check
+   * @return loggable information about the file
+   */
+  private String getFileStats(Path file, FileSystem fs) throws IOException {
+    FileStatus status = fs.getFileStatus(file);
+    return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:"
+        + status.getAccessTime();
+  }
+
+  @Test(timeout = 2000)
+  public void testHFileCleaning() throws Exception {
+    final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
+    String prefix = "someHFileThatWouldBeAUUID";
+    Configuration conf = UTIL.getConfiguration();
+    // set TTL
+    long ttl = 2000;
+    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+      "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner");
+    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
+    Server server = new DummyServer();
+    Path archivedHfileDir = new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
+    FileSystem fs = FileSystem.get(conf);
+    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
+
+    // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
+    final long createTime = System.currentTimeMillis();
+    fs.delete(archivedHfileDir, true);
+    fs.mkdirs(archivedHfileDir);
+    // Case 1: 1 invalid file, which should be deleted directly
+    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
+    // Case 2: 1 "recent" file, not even deletable for the first log cleaner
+    // (TimeToLiveLogCleaner), so we are not going down the chain
+    LOG.debug("Now is: " + createTime);
+    for (int i = 1; i < 32; i++) {
+      // Case 3: old files which would be deletable for the first log cleaner
+      // (TimeToLiveHFileCleaner),
+      Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i)));
+      fs.createNewFile(fileName);
+      // set the creation time past ttl to ensure that it gets removed
+      fs.setTimes(fileName, createTime - ttl - 1, -1);
+      LOG.debug("Creating " + getFileStats(fileName, fs));
+    }
+
+    // Case 2: 1 newer file, not even deletable for the first log cleaner
+    // (TimeToLiveLogCleaner), so we are not going down the chain
+    Path saved = new Path(archivedHfileDir, "thisFileShouldBeSaved.00000000000");
+    fs.createNewFile(saved);
+    // set creation time in the future, so definitely within TTL
+    fs.setTimes(saved, createTime + (ttl * 2), -1);
+    LOG.debug("Creating " + getFileStats(saved, fs));
+
+    assertEquals(33, fs.listStatus(archivedHfileDir).length);
+
+    // set a custom edge manager to handle time checking
+    EnvironmentEdge setTime = new EnvironmentEdge() {
+      @Override
+      public long currentTimeMillis() {
+        return createTime;
+      }
+    };
+    EnvironmentEdgeManager.injectEdge(setTime);
+
+    // run the chore
+    cleaner.chore();
+
+    for (FileStatus file : fs.listStatus(archivedHfileDir)) {
+      LOG.debug("Kept hfile: " + file.getPath());
+    }
+
+    // ensure we only end up with the saved file
+    assertEquals("Didn't dev expected number of files in the archive!", 1,
+      fs.listStatus(archivedHfileDir).length);
+
+    cleaner.interrupt();
+    // reset the edge back to the original edge
+    EnvironmentEdgeManager.injectEdge(originalEdge);
+  }
+
+  @Test
+  public void testRemovesEmptyDirectories() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+    // no cleaner policies = delete all files
+    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
+    Server server = new DummyServer();
+    Path archivedHfileDir = new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+    // setup the cleaner
+    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
+    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
+
+    // make all the directories for archiving files
+    Path table = new Path(archivedHfileDir, "table");
+    Path region = new Path(table, "regionsomthing");
+    Path family = new Path(region, "fam");
+    Path file = new Path(family, "file12345");
+    fs.mkdirs(family);
+    if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
+    fs.create(file).close();
+    if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);
+
+    // run the chore to cleanup the files (and the directories above it)
+    cleaner.chore();
+
+    // make sure all the parent directories get removed
+    assertFalse("family directory not removed for empty directory", fs.exists(family));
+    assertFalse("region directory not removed for empty directory", fs.exists(region));
+    assertFalse("table directory not removed for empty directory", fs.exists(table));
+    assertTrue("archive directory", fs.exists(archivedHfileDir));
+  }
+
+  static class DummyServer implements Server {
+
+    @Override
+    public Configuration getConfiguration() {
+      return UTIL.getConfiguration();
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      try {
+        return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+      return null;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+
+    @Override
+    public ServerName getServerName() {
+      return new ServerName("regionserver,60020,000000");
+    }
+
+    @Override
+    public void abort(String why, Throwable e) {
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {}
+
+    @Override
+    public boolean isStopped() {
+      return false;
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
similarity index 98%
rename from src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
rename to src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index ffda68d0f17c..e38309c27870 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -17,7 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase.master;
+package org.apache.hadoop.hbase.master.cleaner;
 
 import static org.junit.Assert.assertEquals;
 
@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
new file mode 100644
index 000000000000..efc9cb654d02
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+
+/**
+ * HFile archive cleaner that just tells you if it has been run already or not (and allows resets) -
+ * always attempts to delete the passed file.
+ * <p>
+ * Just a helper class for testing to make sure the cleaner has been run.
+ */
+public class CheckedArchivingHFileCleaner extends BaseHFileCleanerDelegate {
+
+  private static boolean checked;
+
+  @Override
+  public boolean isFileDeletable(Path file) {
+    checked = true;
+    return true;
+  }
+
+  public static boolean getChecked() {
+    return checked;
+  }
+
+  public static void resetCheck() {
+    checked = false;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
new file mode 100644
index 000000000000..7609e0ee5405
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Store;
+
+/**
+ * Test helper for testing archiving of HFiles
+ */
+public class HFileArchiveTestingUtil {
+
+  private static final Log LOG = LogFactory.getLog(HFileArchiveTestingUtil.class);
+
+  private HFileArchiveTestingUtil() {
+    // NOOP private ctor since this is just a utility class
+  }
+
+  public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
+      FileSystem fs, boolean hasTimedBackup) {
+
+    List<List<String>> lists = getFileLists(previous, archived);
+    List<String> original = lists.get(0);
+    Collections.sort(original);
+
+    List<String> currentFiles = lists.get(1);
+    Collections.sort(currentFiles);
+
+    List<String> backedup = lists.get(2);
+    Collections.sort(backedup);
+
+    // check the backed up files versus the current (should match up, less the
+    // backup time in the name)
+    if (!hasTimedBackup == (backedup.size() > 0)) {
+      LOG.debug("backedup files doesn't match expected.");
+      return false;
+    }
+    String msg = null;
+    if (hasTimedBackup) {
+      msg = assertArchiveEquality(original, backedup);
+      if (msg != null) {
+        LOG.debug(msg);
+        return false;
+      }
+    }
+    msg = assertArchiveEquality(original, currentFiles);
+    if (msg != null) {
+      LOG.debug(msg);
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Compare the archived files to the files in the original directory
+   * @param previous original files that should have been archived
+   * @param archived files that were archived
+   * @param fs filessystem on which the archiving took place
+   * @throws IOException
+   */
+  public static void assertArchiveEqualToOriginal(FileStatus[] previous, FileStatus[] archived,
+      FileSystem fs) throws IOException {
+    assertArchiveEqualToOriginal(previous, archived, fs, false);
+  }
+
+  /**
+   * Compare the archived files to the files in the original directory
+   * @param previous original files that should have been archived
+   * @param archived files that were archived
+   * @param fs {@link FileSystem} on which the archiving took place
+   * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
+   *          copy of the files in the archive directory (and the original files).
+   * @throws IOException
+   */
+  public static void assertArchiveEqualToOriginal(FileStatus[] previous, FileStatus[] archived,
+      FileSystem fs, boolean hasTimedBackup) throws IOException {
+
+    List<List<String>> lists = getFileLists(previous, archived);
+    List<String> original = lists.get(0);
+    Collections.sort(original);
+
+    List<String> currentFiles = lists.get(1);
+    Collections.sort(currentFiles);
+
+    List<String> backedup = lists.get(2);
+    Collections.sort(backedup);
+
+    // check the backed up files versus the current (should match up, less the
+    // backup time in the name)
+    assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
+      backedup.size() > 0);
+    String msg = null;
+    if (hasTimedBackup) {
+      assertArchiveEquality(original, backedup);
+      assertNull(msg, msg);
+    }
+
+    // do the rest of the comparison
+    msg = assertArchiveEquality(original, currentFiles);
+    assertNull(msg, msg);
+  }
+
+  private static String assertArchiveEquality(List<String> expected, List<String> archived) {
+    String compare = compareFileLists(expected, archived);
+    if (!(expected.size() == archived.size())) return "Not the same number of current files\n"
+        + compare;
+    if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
+    return null;
+  }
+
+  /**
+   * @return <expected, gotten, backup>, where each is sorted
+   */
+  private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
+    List<List<String>> files = new ArrayList<List<String>>();
+
+    // copy over the original files
+    List<String> originalFileNames = convertToString(previous);
+    files.add(originalFileNames);
+
+    List<String> currentFiles = new ArrayList<String>(previous.length);
+    List<FileStatus> backedupFiles = new ArrayList<FileStatus>(previous.length);
+    for (FileStatus f : archived) {
+      String name = f.getPath().getName();
+      // if the file has been backed up
+      if (name.contains(".")) {
+        Path parent = f.getPath().getParent();
+        String shortName = name.split("[.]")[0];
+        Path modPath = new Path(parent, shortName);
+        FileStatus file = new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
+            f.getBlockSize(), f.getModificationTime(), modPath);
+        backedupFiles.add(file);
+      } else {
+        // otherwise, add it to the list to compare to the original store files
+        currentFiles.add(name);
+      }
+    }
+
+    files.add(currentFiles);
+    files.add(convertToString(backedupFiles));
+    return files;
+  }
+
+  private static List<String> convertToString(FileStatus[] files) {
+    return convertToString(Arrays.asList(files));
+  }
+
+  private static List<String> convertToString(List<FileStatus> files) {
+    List<String> originalFileNames = new ArrayList<String>(files.size());
+    for (FileStatus f : files) {
+      originalFileNames.add(f.getPath().getName());
+    }
+    return originalFileNames;
+  }
+
+  /* Get a pretty representation of the differences */
+  private static String compareFileLists(List<String> expected, List<String> gotten) {
+    StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten ("
+        + gotten.size() + "):\n");
+    List<String> notFound = new ArrayList<String>();
+    for (String s : expected) {
+      if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
+      else notFound.add(s);
+    }
+    sb.append("Not Found:\n");
+    for (String s : notFound) {
+      sb.append(s + "\n");
+    }
+    sb.append("\nExtra:\n");
+    for (String s : gotten) {
+      if (!expected.contains(s)) sb.append(s + "\n");
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Helper method to get the archive directory for the specified region
+   * @param conf {@link Configuration} to check for the name of the archive directory
+   * @param region region that is being archived
+   * @return {@link Path} to the archive directory for the given region
+   */
+  public static Path getRegionArchiveDir(Configuration conf, HRegion region) {
+    return HFileArchiveUtil.getRegionArchiveDir(conf, region.getTableDir(), region.getRegionDir());
+  }
+
+  /**
+   * Helper method to get the store archive directory for the specified region
+   * @param conf {@link Configuration} to check for the name of the archive directory
+   * @param region region that is being archived
+   * @param store store that is archiving files
+   * @return {@link Path} to the store archive directory for the given region
+   */
+  public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store) {
+    return HFileArchiveUtil.getStoreArchivePath(conf, region, store.getFamily().getName());
+  }
+
+  public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
+      byte[] storeName) throws IOException {
+    byte[] table = Bytes.toBytes(tableName);
+    // get the RS and region serving our table
+    List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
+    HRegion region = servingRegions.get(0);
+
+    // check that we actually have some store files that were archived
+    Store store = region.getStore(storeName);
+    return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
index 0db4d425f73e..c5dda2f216e1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
@@ -54,7 +54,7 @@ public void testRegexAgainstOldStyleTableInfo() {
 
   @Test
   public void testCreateAndUpdate() throws IOException {
-    Path testdir = UTIL.getDataTestDir();
+    Path testdir = UTIL.getDataTestDir("testCreate");
     HTableDescriptor htd = new HTableDescriptor("testCreate");
     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
     assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd));
@@ -252,6 +252,19 @@ public void testTableInfoFileStatusComparator() {
     }
   }
 
+  @Test
+  public void testReadingArchiveDirectoryFromFS() throws IOException {
+    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
+    try {
+      new FSTableDescriptors(fs, FSUtils.getRootDir(UTIL.getConfiguration()))
+          .get(HConstants.HFILE_ARCHIVE_DIRECTORY);
+      fail("Shouldn't be able to read a table descriptor for the archive directory.");
+    } catch (IOException e) {
+      LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
+          + e.getMessage());
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java b/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java
new file mode 100644
index 000000000000..9175bc27d046
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test that the utility works as expected
+ */
+@Category(SmallTests.class)
+public class TestHFileArchiveUtil {
+
+  @Test
+  public void testGetTableArchivePath() {
+    assertNotNull(HFileArchiveUtil.getTableArchivePath(new Path("table")));
+    assertNotNull(HFileArchiveUtil.getTableArchivePath(new Path("root", new Path("table"))));
+  }
+
+  @Test
+  public void testGetArchivePath() throws Exception {
+    Configuration conf = new Configuration();
+    FSUtils.setRootDir(conf, new Path("root"));
+    assertNotNull(HFileArchiveUtil.getArchivePath(conf));
+  }
+  
+  @Test
+  public void testRegionArchiveDir() {
+    Path tableDir = new Path("table");
+    Path regionDir = new Path("region");
+    assertNotNull(HFileArchiveUtil.getRegionArchiveDir(null, tableDir, regionDir));
+  }
+  
+  @Test
+  public void testGetStoreArchivePath(){
+      byte[] family = Bytes.toBytes("Family");
+    Path tabledir = new Path("table");
+    HRegionInfo region = new HRegionInfo(Bytes.toBytes("table"));
+    Configuration conf = null;
+    assertNotNull(HFileArchiveUtil.getStoreArchivePath(conf, region, tabledir, family));
+    conf = new Configuration();
+    assertNotNull(HFileArchiveUtil.getStoreArchivePath(conf, region, tabledir, family));
+
+    // do a little mocking of a region to get the same results
+    HRegion mockRegion = Mockito.mock(HRegion.class);
+    Mockito.when(mockRegion.getRegionInfo()).thenReturn(region);
+    Mockito.when(mockRegion.getTableDir()).thenReturn(tabledir);
+
+    assertNotNull(HFileArchiveUtil.getStoreArchivePath(null, mockRegion, family));
+    conf = new Configuration();
+    assertNotNull(HFileArchiveUtil.getStoreArchivePath(conf, mockRegion, family));
+  }
+}

From 80be523fcbbfebe1f1ecef1817fad23b696e6e5f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 1 Nov 2012 21:00:32 +0000
Subject: [PATCH 0562/1540] HBASE-7017 Backport '[replication] The
 replication-executor should make sure the file that it is replicating is
 closed before declaring success on that file' to 0.94 (Devaraj Das)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404764 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/HLog.java   | 13 ++++----
 .../replication/regionserver/Replication.java |  4 +--
 .../regionserver/ReplicationSource.java       | 31 +++++++++++++++----
 .../ReplicationSourceManager.java             | 17 ++++++++--
 .../TestReplicationSourceManager.java         |  2 +-
 5 files changed, 50 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 56659512001e..2bb5ea192fa1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -634,12 +634,6 @@ OutputStream getOutputStream() {
       if (nextWriter instanceof SequenceFileLogWriter) {
         nextHdfsOut = ((SequenceFileLogWriter)nextWriter).getWriterFSDataOutputStream();
       }
-      // Tell our listeners that a new log was created
-      if (!this.listeners.isEmpty()) {
-        for (WALActionsListener i : this.listeners) {
-          i.postLogRoll(oldPath, newPath);
-        }
-      }
 
       synchronized (updateLock) {
         // Clean up current writer.
@@ -655,6 +649,13 @@ OutputStream getOutputStream() {
           " for " + FSUtils.getPath(newPath));
         this.numEntries.set(0);
       }
+      // Tell our listeners that a new log was created
+      if (!this.listeners.isEmpty()) {
+        for (WALActionsListener i : this.listeners) {
+          i.postLogRoll(oldPath, newPath);
+        }
+      }
+
       // Can we delete any of the old log files?
       if (this.outputfiles.size() > 0) {
         if (this.lastSeqWritten.isEmpty()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index eb288f321b70..128199349af4 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -188,12 +188,12 @@ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey,
 
   @Override
   public void preLogRoll(Path oldPath, Path newPath) throws IOException {
-    // Not interested
+    getReplicationManager().preLogRoll(newPath);
   }
 
   @Override
   public void postLogRoll(Path oldPath, Path newPath) throws IOException {
-    getReplicationManager().logRolled(newPath);
+    getReplicationManager().postLogRoll(newPath);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index fca909d7356c..09b8797b7348 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -293,6 +293,18 @@ public void run() {
         }
         continue;
       }
+      boolean currentWALisBeingWrittenTo = false;
+      //For WAL files we own (rather than recovered), take a snapshot of whether the
+      //current WAL file (this.currentPath) is in use (for writing) NOW!
+      //Since the new WAL paths are enqueued only after the prev WAL file
+      //is 'closed', presence of an element in the queue means that
+      //the previous WAL file was closed, else the file is in use (currentPath)
+      //We take the snapshot now so that we are protected against races
+      //where a new file gets enqueued while the current file is being processed
+      //(and where we just finished reading the current file).
+      if (!this.queueRecovered && queue.size() == 0) {
+        currentWALisBeingWrittenTo = true;
+      }
       // Open a reader on it
       if (!openReader(sleepMultiplier)) {
         // Reset the sleep multiplier, else it'd be reused for the next file
@@ -311,7 +323,7 @@ public void run() {
       boolean gotIOE = false;
       currentNbEntries = 0;
       try {
-        if(readAllEntriesToReplicateOrNextFile()) {
+        if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) {
           continue;
         }
       } catch (IOException ioe) {
@@ -360,7 +372,7 @@ public void run() {
       if (this.isActive() && (gotIOE || currentNbEntries == 0)) {
         if (this.lastLoggedPosition != this.position) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.position, queueRecovered);
+              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
           this.lastLoggedPosition = this.position;
         }
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
@@ -369,7 +381,7 @@ public void run() {
         continue;
       }
       sleepMultiplier = 1;
-      shipEdits();
+      shipEdits(currentWALisBeingWrittenTo);
 
     }
     if (this.conn != null) {
@@ -385,11 +397,13 @@ public void run() {
   /**
    * Read all the entries from the current log files and retain those
    * that need to be replicated. Else, process the end of the current file.
+   * @param currentWALisBeingWrittenTo is the current WAL being written to
    * @return true if we got nothing and went to the next file, false if we got
    * entries
    * @throws IOException
    */
-  protected boolean readAllEntriesToReplicateOrNextFile() throws IOException{
+  protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo)
+      throws IOException{
     long seenEntries = 0;
     if (this.position != 0) {
       this.reader.seek(this.position);
@@ -439,6 +453,9 @@ protected boolean readAllEntriesToReplicateOrNextFile() throws IOException{
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +
         " and size: " + (this.reader.getPosition() - startPosition));
+    if (currentWALisBeingWrittenTo) {
+      return false;
+    }
     // If we didn't get anything and the queue has an object, it means we
     // hit the end of the file for sure
     return seenEntries == 0 && processEndOfFile();
@@ -611,8 +628,10 @@ private int countDistinctRowKeys(WALEdit edit) {
 
   /**
    * Do the shipping logic
+   * @param currentWALisBeingWrittenTo was the current WAL being (seemingly) 
+   * written to when this method was called
    */
-  protected void shipEdits() {
+  protected void shipEdits(boolean currentWALisBeingWrittenTo) {
     int sleepMultiplier = 1;
     if (this.currentNbEntries == 0) {
       LOG.warn("Was given 0 edits to ship");
@@ -631,7 +650,7 @@ protected void shipEdits() {
         rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries));
         if (this.lastLoggedPosition != this.position) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.position, queueRecovered);
+              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
           this.lastLoggedPosition = this.position;
         }
         this.totalReplicatedEdits += currentNbEntries;
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 1d475d3427cd..b67661ff02c1 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -145,11 +145,16 @@ public ReplicationSourceManager(final ReplicationZookeeper zkHelper,
    * @param id id of the peer cluster
    * @param position current location in the log
    * @param queueRecovered indicates if this queue comes from another region server
+   * @param holdLogInZK if true then the log is retained in ZK
    */
-  public void logPositionAndCleanOldLogs(Path log, String id, long position, boolean queueRecovered) {
+  public void logPositionAndCleanOldLogs(Path log, String id, long position, 
+      boolean queueRecovered, boolean holdLogInZK) {
     String key = log.getName();
     LOG.info("Going to report log #" + key + " for position " + position + " in " + log);
     this.zkHelper.writeReplicationStatus(key, id, position);
+    if (holdLogInZK) {
+     return;
+    }
     synchronized (this.hlogsById) {
       SortedSet<String> hlogs = this.hlogsById.get(id);
       if (!queueRecovered && hlogs.first() != key) {
@@ -251,7 +256,7 @@ public List<ReplicationSourceInterface> getSources() {
     return this.sources;
   }
 
-  void logRolled(Path newLog) throws IOException {
+  void preLogRoll(Path newLog) throws IOException {
     if (!this.replicating.get()) {
       LOG.warn("Replication stopped, won't add new log");
       return;
@@ -277,6 +282,14 @@ void logRolled(Path newLog) throws IOException {
     }
 
     this.latestPath = newLog;
+  }
+
+  void postLogRoll(Path newLog) throws IOException {
+    if (!this.replicating.get()) {
+      LOG.warn("Replication stopped, won't add new log");
+      return;
+    }
+
     // This only updates the sources we own, not the recovered ones
     for (ReplicationSourceInterface source : this.sources) {
       source.enqueueLog(newLog);    
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 5828154d08b0..33ee4d0942c2 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -202,7 +202,7 @@ public void testLogRoll() throws Exception {
     hlog.rollWriter();
 
     manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
-        "1", 0, false);
+        "1", 0, false, false);
 
     HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
         System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);

From deb55159c5ba9d060898f2cd5131122d5f362980 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 1 Nov 2012 21:40:47 +0000
Subject: [PATCH 0563/1540] HBASE-6925 Change socket write size from 8K to 64K
 for HBaseServer (Karthik)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1404776 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 2a8efe2312e7..1b1bb7737c32 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -1751,7 +1751,7 @@ public HBaseRpcMetrics getRpcMetrics() {
    * done in chunks of this size. Most RPC requests and responses would be
    * be smaller.
    */
-  private static int NIO_BUFFER_LIMIT = 8*1024; //should not be more than 64KB.
+  private static int NIO_BUFFER_LIMIT = 64 * 1024; //should not be more than 64KB.
 
   /**
    * This is a wrapper around {@link java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer)}.

From ab74cc3ec3595c97ae37ed2db85c4189592ec913 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 2 Nov 2012 17:55:00 +0000
Subject: [PATCH 0564/1540] HBASE-7087 Add to NOTICE.txt a note on jamon being
 MPL

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405074 13f79535-47bb-0310-9956-ffa450edef68
---
 NOTICE.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/NOTICE.txt b/NOTICE.txt
index 3ae710800cd6..337c93beafcd 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -3,6 +3,10 @@ Foundation (http://www.apache.org/).
 
 In addition, this product includes software developed by:
 
+Jamon (http://www.jamon.org/) is a text template engine for Java used by our
+UI.  It uses the Mozilla Public License (http://www.mozilla.org/MPL/)
+See the tail of http://www.jamon.org/About.html
+
 JUnit (http://www.junit.org/) included under the Common Public License v1.0.  See
 the full text here: http://junit.sourceforge.net/cpl-v10.html
 

From f0d26bde21588cdcefcc6715cbd6cd0813bbc775 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 2 Nov 2012 18:11:13 +0000
Subject: [PATCH 0565/1540] HBASE-7086 Enhance ResourceChecker to log stack
 trace for potentially hanging threads

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405081 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ResourceChecker.java  | 49 ++++++++++++++-----
 .../hbase/ResourceCheckerJUnitRule.java       |  2 +-
 2 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
index 8b9b1bb5a159..aa1213622da6 100644
--- a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
+++ b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
@@ -38,6 +38,11 @@
 public class ResourceChecker {
   private static final Log LOG = LogFactory.getLog(ResourceChecker.class);
 
+  enum Phase {
+    INITIAL, INTERMEDIATE, END
+  }
+  private static Set<String> initialThreadNames = new HashSet<String>();
+
   /**
    * On unix, we know how to get the number of open file descriptor
    */
@@ -45,8 +50,14 @@ private static class ResourceAnalyzer {
     private static final OperatingSystemMXBean osStats;
     private static final UnixOperatingSystemMXBean unixOsStats;
 
-    public long getThreadsCount() {
-      return Thread.getAllStackTraces().size();
+    public long getThreadsCount(Phase phase) {
+      Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
+      if (phase == Phase.INITIAL) {
+        for (Thread t : stackTraces.keySet()) {
+          initialThreadNames.add(t.getName());
+        }
+      }
+      return stackTraces.size();
     }
 
     public long getOpenFileDescriptorCount() {
@@ -102,14 +113,14 @@ public long getConnectionCount(){
 
   public boolean checkThreads(String tagLine) {
     boolean isOk = true;
+    long threadCount = rc.getThreadsCount(Phase.INTERMEDIATE);
 
-    if (rc.getThreadsCount() > MAX_THREADS_COUNT) {
+    if (threadCount > MAX_THREADS_COUNT) {
       LOG.error(
         tagLine + ": too many threads used. We use " +
-          rc.getThreadsCount() + " our max is " + MAX_THREADS_COUNT);
+          threadCount + " our max is " + MAX_THREADS_COUNT);
       isOk = false;
     }
-
     return isOk;
   }
 
@@ -132,19 +143,20 @@ public final void init(String tagLine) {
           rc.getMaxFileDescriptorCount() + " our is " + MAX_FILE_HANDLES_COUNT);
     }
 
-    logInfo(tagLine);
+    logInfo(Phase.INITIAL, tagLine);
 
-    initialThreadsCount = rc.getThreadsCount();
+    initialThreadsCount = rc.getThreadsCount(Phase.INITIAL);
     initialFileHandlesCount = rc.getOpenFileDescriptorCount();
     initialConnectionCount= rc.getConnectionCount();
 
     check(tagLine);
   }
 
-  public void logInfo(String tagLine) {
+  public void logInfo(Phase phase, String tagLine) {
+    long threadCount = rc.getThreadsCount(phase);
     LOG.info(
-      tagLine + ": " +
-        rc.getThreadsCount() + " threads" +
+        tagLine + ": " +
+        threadCount + " threads" +
         (initialThreadsCount > 0 ?
           " (was " + initialThreadsCount + "), " : ", ") +
         rc.getOpenFileDescriptorCount() + " file descriptors" +
@@ -153,7 +165,7 @@ public void logInfo(String tagLine) {
         rc.getConnectionCount() + " connections" +
         (initialConnectionCount > 0 ?
           " (was " + initialConnectionCount + "), " : ", ") +
-        (initialThreadsCount > 0 && rc.getThreadsCount() > initialThreadsCount ?
+        (initialThreadsCount > 0 && threadCount > initialThreadsCount ?
           " -thread leak?- " : "") +
         (initialFileHandlesCount > 0 &&
           rc.getOpenFileDescriptorCount() > initialFileHandlesCount ?
@@ -162,6 +174,21 @@ public void logInfo(String tagLine) {
           rc.getConnectionCount() > initialConnectionCount ?
           " -connection leak?- " : "" )
     );
+    if (phase == Phase.END) {
+      Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
+      if (stackTraces.size() > initialThreadNames.size()) {
+        LOG.info(tagLine + ": potentially hanging thread");
+
+        for (Thread t : stackTraces.keySet()) {
+          if (!initialThreadNames.contains(t.getName())) {
+            StackTraceElement[] stackElements = stackTraces.get(t);
+            for (StackTraceElement ele : stackElements) {
+              LOG.info("\t" + ele);
+            }
+          }
+        }
+      }
+    }
   }
 
 
diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java b/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
index 6d1ab258c70c..56638c95e892 100644
--- a/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
+++ b/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
@@ -48,7 +48,7 @@ private void start(String testName) {
   private void end(String testName) {
     if (!endDone) {
       endDone = true;
-      cu.logInfo("after " + testName);
+      cu.logInfo(ResourceChecker.Phase.END, "after " + testName);
       cu.check("after "+testName);
     }
   }

From 253844894c5c4398240fd5fd89609629e6354e1a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 2 Nov 2012 18:13:07 +0000
Subject: [PATCH 0566/1540] HBASE-6852 RE-REAPPLY, Cheng worked tirelessly to
 fix the issues.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405083 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  2 +
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  2 +
 .../regionserver/metrics/SchemaMetrics.java   | 81 +++++++++++++++++--
 3 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index b93c5c483104..4dafb311ca4e 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -391,6 +391,8 @@ public void close(boolean evictOnClose) throws IOException {
       this.istream.close();
       this.istream = null;
     }
+
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV1
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 757990f845f4..b54175ad9951 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -424,6 +424,8 @@ public void close(boolean evictOnClose) throws IOException {
         istream = null;
       }
     }
+
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV2
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index e73513ceb953..9330834cc015 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -29,6 +29,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -38,7 +39,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -236,6 +236,9 @@ public String toString() {
   public static final SchemaMetrics ALL_SCHEMA_METRICS =
     getInstance(TOTAL_KEY, TOTAL_KEY);
 
+  /** Threshold for flush the metrics, currently used only for "on cache hit" */
+  private static final long THRESHOLD_METRICS_FLUSH = 100l;
+
   /**
    * Whether to include table name in metric names. If this is null, it has not
    * been initialized. This is a global instance, but we also have a copy of it
@@ -254,6 +257,8 @@ public String toString() {
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
   private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
+  private final AtomicLongArray onHitCacheMetrics= 
+      new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length);
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -261,6 +266,9 @@ private SchemaMetrics(final String tableName, final String cfName) {
 
     for (BlockCategory blockCategory : BlockCategory.values()) {
       for (boolean isCompaction : BOOL_VALUES) {
+        // initialize the cache metrics
+        onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0);
+        
         for (BlockMetricType metricType : BlockMetricType.values()) {
           if (!metricType.compactionAware && isCompaction) {
             continue;
@@ -336,6 +344,11 @@ public static SchemaMetrics getInstance(String tableName, String cfName) {
     return existingMetrics != null ? existingMetrics : schemaMetrics;
   }
 
+  private static final int getCacheHitMetricIndex (BlockCategory blockCategory,
+      boolean isCompaction) {
+    return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0);
+  }
+  
   private static final int getBlockMetricIndex(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
     int i = 0;
@@ -365,15 +378,24 @@ public String getBloomMetricName(boolean isInBloom) {
    */
   private void incrNumericMetric(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
+    incrNumericMetric (blockCategory, isCompaction, metricType, 1);
+  }
+  
+  /**
+   * Increments the given metric, both per-CF and aggregate, for both the given
+   * category and all categories in aggregate (four counters total).
+   */
+  private void incrNumericMetric(BlockCategory blockCategory,
+      boolean isCompaction, BlockMetricType metricType, long amount) {
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
     RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
-        isCompaction, metricType), 1);
+        isCompaction, metricType), amount);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
       incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction,
-          metricType);
+          metricType, amount);
     }
   }
 
@@ -445,14 +467,60 @@ public void updatePersistentStoreMetric(StoreMetricType storeMetricType,
    */
   public void updateOnCacheHit(BlockCategory blockCategory,
       boolean isCompaction) {
+    updateOnCacheHit(blockCategory, isCompaction, 1);
+  }
+  
+  /**
+   * Updates the number of hits and the total number of block reads on a block
+   * cache hit.
+   */
+  public void updateOnCacheHit(BlockCategory blockCategory,
+      boolean isCompaction, long count) {
     blockCategory.expectSpecific();
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT);
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT);
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    
+    if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) {
+      flushCertainOnCacheHitMetrics(blockCategory, isCompaction);
+    }
+    
+    if (this != ALL_SCHEMA_METRICS) {
+      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, count);
+    }
+  }
+  
+  private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) {
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0);
+    
+    if (tempCount > 0) {
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount);
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount);
+    }
+  }
+  
+  /**
+   * Flush the on cache hit metrics;
+   */
+  private void flushOnCacheHitMetrics() {
+    for (BlockCategory blockCategory : BlockCategory.values()) {
+      for (boolean isCompaction : BOOL_VALUES) {
+        flushCertainOnCacheHitMetrics (blockCategory, isCompaction);
+      }
+    }
+    
     if (this != ALL_SCHEMA_METRICS) {
-      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction);
+      ALL_SCHEMA_METRICS.flushOnCacheHitMetrics();
     }
   }
 
+  /**
+   * Notify the SchemaMetrics to flush all of the the metrics
+   */
+  public void flushMetrics() {
+    // currently only for "on cache hit metrics"
+    flushOnCacheHitMetrics();
+  }
+  
   /**
    * Updates read time, the number of misses, and the total number of block
    * reads on a block cache miss.
@@ -746,6 +814,7 @@ private static final String stripTimeVaryingSuffix(String metricKey) {
   public static Map<String, Long> getMetricsSnapshot() {
     Map<String, Long> metricsSnapshot = new TreeMap<String, Long>();
     for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) {
+      cfm.flushMetrics();
       for (String metricName : cfm.getAllMetricNames()) {
         long metricValue;
         if (isTimeVaryingKey(metricName)) {

From 760a387110296d2799f59c2026ef495823a592b1 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 2 Nov 2012 23:02:54 +0000
Subject: [PATCH 0567/1540] HBASE-7086 Enhance ResourceChecker to log stack
 trace for potentially hanging threads, addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405207 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/ResourceChecker.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
index aa1213622da6..62a51445ab23 100644
--- a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
+++ b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
@@ -177,10 +177,9 @@ public void logInfo(Phase phase, String tagLine) {
     if (phase == Phase.END) {
       Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
       if (stackTraces.size() > initialThreadNames.size()) {
-        LOG.info(tagLine + ": potentially hanging thread");
-
         for (Thread t : stackTraces.keySet()) {
           if (!initialThreadNames.contains(t.getName())) {
+            LOG.info(tagLine + ": potentially hanging thread - " + t.getName());
             StackTraceElement[] stackElements = stackTraces.get(t);
             for (StackTraceElement ele : stackElements) {
               LOG.info("\t" + ele);

From 99916698e0a124e29fc99b17d2db5c5334bc188a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 3 Nov 2012 03:55:41 +0000
Subject: [PATCH 0568/1540] HBASE-6796 ADDENDUM, remove spurious time limit
 from testHFileCleaning

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405275 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
index ed80c3acbcfe..e3ad07c73fd5 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
@@ -88,7 +88,7 @@ private String getFileStats(Path file, FileSystem fs) throws IOException {
         + status.getAccessTime();
   }
 
-  @Test(timeout = 2000)
+  @Test
   public void testHFileCleaning() throws Exception {
     final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
     String prefix = "someHFileThatWouldBeAUUID";

From 9642e6bdceea5b103abb8dd839ec30403f561939 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 3 Nov 2012 06:22:19 +0000
Subject: [PATCH 0569/1540] HBASE-6305 TestLocalHBaseCluster hangs with hadoop
 2.0/0.23 builds. (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405286 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
index f4ef1cd97050..8b1c01848df3 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
@@ -44,7 +44,9 @@ public class TestLocalHBaseCluster {
   @Test
   public void testLocalHBaseCluster() throws Exception {
     Configuration conf = TEST_UTIL.getConfiguration();
-    conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir("hbase.rootdir").toString());
+    conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir("hbase.rootdir").
+      makeQualified(TEST_UTIL.getTestFileSystem().getUri(), 
+        TEST_UTIL.getTestFileSystem().getWorkingDirectory()).toString());
     MiniZooKeeperCluster zkCluster = TEST_UTIL.startMiniZKCluster();
     conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkCluster.getClientPort()));
     LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1, MyHMaster.class,

From 8738056386f751f0b39ebb27bb303418e5092fb6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 3 Nov 2012 22:55:05 +0000
Subject: [PATCH 0570/1540] HBASE-6389 Modify the conditions to ensure that
 Master waits for sufficient number of Region Servers before starting region
 assignments (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405440 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/ServerManager.java    | 44 ++++++++++++++-----
 .../hadoop/hbase/HBaseTestingUtility.java     | 36 +++++++++++++--
 .../hbase/master/TestMasterFailover.java      |  8 ++--
 .../TestRSKilledWhenMasterInitializing.java   |  7 ++-
 4 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 0ed32006ce14..41118cca9344 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -66,6 +66,18 @@
  * server from the original instance.
  */
 public class ServerManager {
+  public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
+      "hbase.master.wait.on.regionservers.maxtostart";
+
+  public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
+      "hbase.master.wait.on.regionservers.mintostart";
+
+  public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
+      "hbase.master.wait.on.regionservers.timeout";
+
+  public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
+      "hbase.master.wait.on.regionservers.interval";
+
   private static final Log LOG = LogFactory.getLog(ServerManager.class);
 
   // Set if we are to shutdown the cluster.
@@ -557,25 +569,38 @@ private HRegionInterface getServerConnection(final ServerName sn)
    * Wait for the region servers to report in.
    * We will wait until one of this condition is met:
    *  - the master is stopped
-   *  - the 'hbase.master.wait.on.regionservers.timeout' is reached
    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
    *    region servers is reached
    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
    *   there have been no new region server in for
-   *      'hbase.master.wait.on.regionservers.interval' time
+   *      'hbase.master.wait.on.regionservers.interval' time AND
+   *   the 'hbase.master.wait.on.regionservers.timeout' is reached
    *
    * @throws InterruptedException
    */
   public void waitForRegionServers(MonitoredTask status)
   throws InterruptedException {
     final long interval = this.master.getConfiguration().
-      getLong("hbase.master.wait.on.regionservers.interval", 1500);
+      getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
     final long timeout = this.master.getConfiguration().
-    getLong("hbase.master.wait.on.regionservers.timeout", 4500);
-    final int minToStart = this.master.getConfiguration().
-    getInt("hbase.master.wait.on.regionservers.mintostart", 1);
-    final int maxToStart = this.master.getConfiguration().
-    getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
+      getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
+    int minToStart = this.master.getConfiguration().
+      getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
+    if (minToStart < 1) {
+      LOG.warn(String.format(
+        "The value of '%s' (%d) can not be less than 1, ignoring.",
+        WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
+      minToStart = 1;
+    }
+    int maxToStart = this.master.getConfiguration().
+      getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
+    if (maxToStart < minToStart) {
+        LOG.warn(String.format(
+            "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
+            WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
+            WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
+        maxToStart = Integer.MAX_VALUE;
+    }
 
     long now =  System.currentTimeMillis();
     final long startTime = now;
@@ -586,9 +611,8 @@ public void waitForRegionServers(MonitoredTask status)
     int oldCount = 0;
     while (
       !this.master.isStopped() &&
-        slept < timeout &&
         count < maxToStart &&
-        (lastCountChange+interval > now || count < minToStart)
+        (lastCountChange+interval > now || timeout > slept || count < minToStart)
       ){
 
       // Log some info at every interval time or if there is a change
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 887765399b1d..d13188f7205c 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -63,6 +63,7 @@
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -73,6 +74,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.RegionSplitter;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
@@ -627,9 +629,13 @@ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
     createRootDir();
 
     // These settings will make the server waits until this exact number of
-    //  regions servers are connected.
-    conf.setInt("hbase.master.wait.on.regionservers.mintostart", numSlaves);
-    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", numSlaves);
+    // regions servers are connected.
+    if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) {
+      conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves);
+    }
+    if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) {
+      conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves);
+    }
 
     Configuration c = new Configuration(this.conf);
     this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves);
@@ -709,6 +715,9 @@ public void shutdownMiniHBaseCluster() throws IOException {
       hbaseAdmin.close();
       hbaseAdmin = null;
     }
+    // unset the configuration for MIN and MAX RS to start
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1);
     if (this.hbaseCluster != null) {
       this.hbaseCluster.shutdown();
       // Wait till hbase is down before going on to shutdown zk.
@@ -1360,6 +1369,27 @@ public void expireMasterSession() throws Exception {
   public void expireRegionServerSession(int index) throws Exception {
     HRegionServer rs = hbaseCluster.getRegionServer(index);
     expireSession(rs.getZooKeeper(), false);
+    decrementMinRegionServerCount();
+  }
+
+  private void decrementMinRegionServerCount() {
+    // decrement the count for this.conf, for newly spwaned master
+    // this.hbaseCluster shares this configuration too
+    decrementMinRegionServerCount(getConfiguration());
+
+    // each master thread keeps a copy of configuration
+    for (MasterThread master : getHBaseCluster().getMasterThreads()) {
+      decrementMinRegionServerCount(master.getMaster().getConfiguration());
+    }
+  }
+
+  private void decrementMinRegionServerCount(Configuration conf) {
+    int currentCount = conf.getInt(
+        ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
+    if (currentCount != -1) {
+      conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
+          Math.max(currentCount - 1, 1));
+    }
   }
 
    /**
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 14cdb90a7b34..108c2820542d 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -308,8 +308,8 @@ public void testMasterFailoverWithMockedRIT() throws Exception {
     // Need to drop the timeout much lower
     conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
     conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
-    conf.setInt("hbase.master.wait.on.regionservers.mintostart", 3);
-    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 3);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 3);
 
     // Start the cluster
     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
@@ -608,8 +608,8 @@ public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
     // Need to drop the timeout much lower
     conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
     conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
-    conf.setInt("hbase.master.wait.on.regionservers.mintostart", 1);
-    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 2);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
 
     // Create and start the cluster
     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index a0ed0bd83a75..e4157cddb706 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -66,8 +66,11 @@ public class TestRSKilledWhenMasterInitializing {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     // Set it so that this test runs with my custom master
-    TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL,
-        TestingMaster.class, HMaster.class);
+    Configuration conf = TESTUTIL.getConfiguration();
+    conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 4);
+
     // Start up the cluster.
     TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
   }

From 1e93e06fcb069b4e76bfdbc719e5f023d3b0d17a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 5 Nov 2012 00:55:21 +0000
Subject: [PATCH 0571/1540] HBASE-7095 Cannot set 'lenAsVal' for KeyOnlyFilter
 from shell (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405684 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/filter/KeyOnlyFilter.java  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
index b2eb3a5fffb0..5cf659ce4d26 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
@@ -48,9 +48,13 @@ public KeyValue transform(KeyValue kv) {
   }
 
   public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments) {
-    Preconditions.checkArgument(filterArguments.size() == 0,
-                                "Expected: 0 but got: %s", filterArguments.size());
-    return new KeyOnlyFilter();
+    Preconditions.checkArgument((filterArguments.size() == 0 || filterArguments.size() == 1),
+                                "Expected: 0 or 1 but got: %s", filterArguments.size());
+    KeyOnlyFilter filter = new KeyOnlyFilter();
+    if (filterArguments.size() == 1) {
+      filter.lenAsVal = ParseFilter.convertByteArrayToBoolean(filterArguments.get(0));
+    }
+    return filter;
   }
 
   public void write(DataOutput out) throws IOException {

From 834e13372794f69bca9af034df4ccd9448c859b3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 5 Nov 2012 05:26:52 +0000
Subject: [PATCH 0572/1540] HBASE-7089 Allow filter to be specified for Get
 from HBase shell (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405697 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/table.rb        |  7 ++++++
 src/main/ruby/shell/commands/get.rb |  1 +
 src/test/ruby/hbase/table_test.rb   | 35 +++++++++++++++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index 5bcbb4dc323e..34b33c883305 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -118,6 +118,7 @@ def get(row, *args)
 
       # Get maxlength parameter if passed
       maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH]
+      filter = args.delete(FILTER) if args[FILTER]
 
       unless args.empty?
         columns = args[COLUMN] || args[COLUMNS]
@@ -162,6 +163,12 @@ def get(row, *args)
         end
       end
 
+      unless filter.class == String
+        get.setFilter(filter)
+      else
+        get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter))
+      end
+
       # Call hbase for the results
       result = @table.get(get)
       return nil if result.isEmpty
diff --git a/src/main/ruby/shell/commands/get.rb b/src/main/ruby/shell/commands/get.rb
index 754c3d6b08f3..8456d58a5cb6 100644
--- a/src/main/ruby/shell/commands/get.rb
+++ b/src/main/ruby/shell/commands/get.rb
@@ -33,6 +33,7 @@ def help
   hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1}
   hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4}
   hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4}
+  hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"}
   hbase> get 't1', 'r1', 'c1'
   hbase> get 't1', 'r1', 'c1', 'c2'
   hbase> get 't1', 'r1', ['c1', 'c2']
diff --git a/src/test/ruby/hbase/table_test.rb b/src/test/ruby/hbase/table_test.rb
index 5d56e18518ea..c87921dba992 100644
--- a/src/test/ruby/hbase/table_test.rb
+++ b/src/test/ruby/hbase/table_test.rb
@@ -313,6 +313,22 @@ def setup
       assert_equal(res.keys.sort, [ 'x:a', 'x:b' ])
     end
 
+    define_test "get should support FILTER" do
+      @test_table.put(1, "x:v", "thisvalue")
+      begin
+        res = @test_table.get('1', FILTER => "ValueFilter(=, 'binary:thisvalue')")
+        assert_not_nil(res)
+        assert_kind_of(Hash, res)
+        assert_not_nil(res['x:v'])
+        assert_nil(res['x:a'])
+        res = @test_table.get('1', FILTER => "ValueFilter(=, 'binary:thatvalue')")
+        assert_nil(res)
+      ensure
+        # clean up newly added columns for this test only.
+        @test_table.delete(1, "x:v")
+      end
+    end
+
     #-------------------------------------------------------------------------------
 
     define_test "scan should work w/o any params" do
@@ -418,5 +434,24 @@ def setup
       res = @test_table.scan { |row, cells| rows[row] = cells }
       assert_equal(rows.keys.size, res)
     end
+
+    define_test "scan should support FILTER" do
+      @test_table.put(1, "x:v", "thisvalue")
+      begin
+        res = @test_table.scan FILTER => "ValueFilter(=, 'binary:thisvalue')"
+        assert_not_equal(res, {}, "Result is empty")
+        assert_kind_of(Hash, res)
+        assert_not_nil(res['1'])
+        assert_not_nil(res['1']['x:v'])
+        assert_nil(res['1']['x:a'])
+        assert_nil(res['2'])
+        res = @test_table.scan FILTER => "ValueFilter(=, 'binary:thatvalue')"
+        assert_equal(res, {}, "Result is not empty")
+      ensure
+        # clean up newly added columns for this test only.
+        @test_table.delete(1, "x:v")
+      end
+    end
+
   end
 end

From e5e3c980e7095d98a8f544736226f752f934ef3e Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 5 Nov 2012 18:35:01 +0000
Subject: [PATCH 0573/1540] HBASE-7097. Log message in SecureServer.class uses
 wrong class name (Y. Sreenivasulu Reddy)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1405906 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/ipc/SecureServer.java     | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index c72281987f83..e5f6c4a52985 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -87,9 +87,10 @@ public abstract class SecureServer extends HBaseServer {
   public static final byte CURRENT_VERSION = 4;
   public static final Set<Byte> INSECURE_VERSIONS = ImmutableSet.of((byte) 3);
 
-  public static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.SecureServer");
-  private static final Log AUDITLOG =
-    LogFactory.getLog("SecurityLogger.org.apache.hadoop.ipc.SecureServer");
+  public static final Log LOG = LogFactory.getLog(SecureServer.class);
+  private static final Log AUDITLOG = LogFactory.getLog("SecurityLogger." +
+    SecureServer.class.getName());
+
   private static final String AUTH_FAILED_FOR = "Auth failed for ";
   private static final String AUTH_SUCCESSFUL_FOR = "Auth successful for ";
 

From cf7a593935632f7a749e0ecad89a5d5adc51688c Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 7 Nov 2012 02:24:37 +0000
Subject: [PATCH 0574/1540] Amend HBASE-7097. Change per-request logging in
 SecureServer to TRACE level

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1406420 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/SecureServer.java | 53 +++++++++++--------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index e5f6c4a52985..f9497a174d99 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -172,9 +172,10 @@ private void wrapWithSasl(ByteBufferOutputStream response)
           token = ((SecureConnection)connection).saslServer.wrap(buf.array(),
               buf.arrayOffset(), buf.remaining());
         }
-        if (LOG.isDebugEnabled())
-          LOG.debug("Adding saslServer wrapped token of size " + token.length
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Adding saslServer wrapped token of size " + token.length
               + " as call response.");
+        }
         buf.clear();
         DataOutputStream saslOut = new DataOutputStream(response);
         saslOut.writeInt(token.length);
@@ -277,8 +278,9 @@ HBaseSaslRpcServer.SASL_PROPS, new SaslDigestCallbackHandler(
               UserGroupInformation current = UserGroupInformation
                   .getCurrentUser();
               String fullName = current.getUserName();
-              if (LOG.isDebugEnabled())
-                LOG.debug("Kerberos principal name is " + fullName);
+              if (LOG.isTraceEnabled()) {
+                LOG.trace("Kerberos principal name is " + fullName);
+              }
               final String names[] = HBaseSaslRpcServer.splitKerberosName(fullName);
               if (names.length != 3) {
                 throw new AccessControlException(
@@ -299,13 +301,15 @@ public Object run() throws SaslException {
               throw new AccessControlException(
                   "Unable to find SASL server implementation for "
                       + authMethod.getMechanismName());
-            if (LOG.isDebugEnabled())
-              LOG.debug("Created SASL server with mechanism = "
+            if (LOG.isTraceEnabled()) {
+              LOG.trace("Created SASL server with mechanism = "
                   + authMethod.getMechanismName());
+            }
           }
-          if (LOG.isDebugEnabled())
-            LOG.debug("Have read input token of size " + saslToken.length
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("Have read input token of size " + saslToken.length
                 + " for processing by saslServer.evaluateResponse()");
+          }
           replyToken = saslServer.evaluateResponse(saslToken);
         } catch (IOException e) {
           IOException sendToClient = e;
@@ -326,28 +330,33 @@ public Object run() throws SaslException {
           throw e;
         }
         if (replyToken != null) {
-          if (LOG.isDebugEnabled())
-            LOG.debug("Will send token of size " + replyToken.length
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("Will send token of size " + replyToken.length
                 + " from saslServer.");
+          }
           doSaslReply(SaslStatus.SUCCESS, new BytesWritable(replyToken), null,
               null);
         }
         if (saslServer.isComplete()) {
-          LOG.debug("SASL server context established. Negotiated QoP is "
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("SASL server context established. Negotiated QoP is "
               + saslServer.getNegotiatedProperty(Sasl.QOP));
+          }
           String qop = (String) saslServer.getNegotiatedProperty(Sasl.QOP);
           useWrap = qop != null && !"auth".equalsIgnoreCase(qop);
           ticket = getAuthorizedUgi(saslServer.getAuthorizationID());
-          LOG.debug("SASL server successfully authenticated client: " + ticket);
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("SASL server successfully authenticated client: " + ticket);
+          }
           rpcMetrics.authenticationSuccesses.inc();
           AUDITLOG.info(AUTH_SUCCESSFUL_FOR + ticket);
           saslContextEstablished = true;
         }
       } else {
-        if (LOG.isDebugEnabled())
-          LOG.debug("Have read input token of size " + saslToken.length
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Have read input token of size " + saslToken.length
               + " for processing by saslServer.unwrap()");
-
+        }
         if (!useWrap) {
           processOneRpc(saslToken);
         } else {
@@ -558,8 +567,9 @@ private void processUnwrappedData(byte[] inBuf) throws IOException,
           int unwrappedDataLength = unwrappedDataLengthBuffer.getInt();
 
           if (unwrappedDataLength == HBaseClient.PING_CALL_ID) {
-            if (LOG.isDebugEnabled())
-              LOG.debug("Received ping message");
+            if (LOG.isTraceEnabled()) {
+              LOG.trace("Received ping message");
+            }
             unwrappedDataLengthBuffer.clear();
             continue; // ping message
           }
@@ -599,8 +609,8 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
         new DataInputStream(new ByteArrayInputStream(buf));
       int id = dis.readInt();                    // try to read an id
 
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(" got #" + id);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace(" got #" + id);
       }
 
       Writable param = ReflectionUtils.newInstance(paramClass, conf);           // read param
@@ -631,7 +641,9 @@ private boolean authorizeConnection() throws IOException {
         }
         rpcMetrics.authorizationSuccesses.inc();
       } catch (AuthorizationException ae) {
-        LOG.debug("Connection authorization failed: "+ae.getMessage(), ae);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Connection authorization failed: "+ae.getMessage(), ae);
+        }
         rpcMetrics.authorizationFailures.inc();
         SecureCall failedCall = new SecureCall(AUTHORIZATION_FAILED_CALLID, null, this,
             null, 0);
@@ -673,7 +685,6 @@ protected SecureServer(String bindAddress, int port,
     this.authorize =
       conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false);
     this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
-    LOG.debug("security enabled="+isSecurityEnabled);
 
     if (isSecurityEnabled) {
       HBaseSaslRpcServer.init(conf);

From 7c332d98d9ea70d583092ea32c4e09b0b07a55dd Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 7 Nov 2012 16:22:28 +0000
Subject: [PATCH 0575/1540] HBASE-6958 TestAssignmentManager sometimes fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1406700 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/TestAssignmentManager.java   | 83 ++++++++++---------
 1 file changed, 45 insertions(+), 38 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index cd3989a2e96c..6e921497a42e 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -20,7 +20,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -802,43 +801,49 @@ public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, K
         MockedLoadBalancer.class, LoadBalancer.class);
     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
         this.serverManager);
-    // Boolean variable used for waiting until randomAssignment is called and new
-    // plan is generated.
-    AtomicBoolean gate = new AtomicBoolean(false);
-    if (balancer instanceof MockedLoadBalancer) {
-      ((MockedLoadBalancer) balancer).setGateVariable(gate);
-    }
-    ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
-    int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
-    ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
-        EventType.RS_ZK_REGION_FAILED_OPEN, v);
-    String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
-    RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
-        SERVERNAME_A);
-    am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
-    // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed
-    am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
-    RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
-    List<ServerName> serverList = new ArrayList<ServerName>(2);
-    serverList.add(SERVERNAME_B);
-    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
-    am.nodeDataChanged(path);
-    // here we are waiting until the random assignment in the load balancer is called.
-    while (!gate.get()) {
-      Thread.sleep(10);
-    }
-    // new region plan may take some time to get updated after random assignment is called and 
-    // gate is set to true.
-    RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
-    while (newRegionPlan == null) {
-      Thread.sleep(10);
-      newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
-    }
-    // the new region plan created may contain the same RS as destination but it should
-    // be new plan.
-    assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
-    assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
+    try {
+      // Boolean variable used for waiting until randomAssignment is called and new
+      // plan is generated.
+      AtomicBoolean gate = new AtomicBoolean(false);
+      if (balancer instanceof MockedLoadBalancer) {
+        ((MockedLoadBalancer) balancer).setGateVariable(gate);
+      }
+      ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
+      int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
+      ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
+          EventType.RS_ZK_REGION_FAILED_OPEN, v);
+      String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
+      RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
+          SERVERNAME_A);
+      am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
+      // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed
+      am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
+      RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
+      List<ServerName> serverList = new ArrayList<ServerName>(2);
+      serverList.add(SERVERNAME_B);
+      Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
+      am.nodeDataChanged(path);
+      // here we are waiting until the random assignment in the load balancer is called.
+      while (!gate.get()) {
+        Thread.sleep(10);
+      }
+      // new region plan may take some time to get updated after random assignment is called and
+      // gate is set to true.
+      RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
+      while (newRegionPlan == null) {
+        Thread.sleep(10);
+        newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
+      }
+      // the new region plan created may contain the same RS as destination but it should
+      // be new plan.
+      assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
+      assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
         newRegionPlan.getDestination())));
+    } finally {
+      this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        DefaultLoadBalancer.class, LoadBalancer.class);
+      am.shutdown();
+    }
   }
   
   /**
@@ -877,8 +882,10 @@ public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
           gate.get());
       // need to change table state from disabling to disabled.
       assertTrue("Table should be disabled.",
-          am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));      
+          am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
     } finally {
+      this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        DefaultLoadBalancer.class, LoadBalancer.class);
       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
       am.shutdown();
     }

From 9d0da10bb4cd0a8dfa36f3528cc0b9f7ddc875e3 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Sat, 10 Nov 2012 07:56:09 +0000
Subject: [PATCH 0576/1540] HBASE-6665 ROOT region should not be splitted even
 with META row as explicit split key (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1407727 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegion.java  |  6 +++---
 .../org/apache/hadoop/hbase/client/TestAdmin.java  | 14 +++++++++++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index fd1b825a67a0..06f93e16c2f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5053,10 +5053,10 @@ protected void prepareToSplit() {
    * is based on the size of the store.
    */
   public byte[] checkSplit() {
-    // Can't split META
-    if (getRegionInfo().isMetaRegion()) {
+    // Can't split ROOT/META
+    if (this.regionInfo.isMetaTable()) {
       if (shouldForceSplit()) {
-        LOG.warn("Cannot split meta regions in HBase 0.20 and above");
+        LOG.warn("Cannot split root/meta regions in HBase 0.20 and above");
       }
       return null;
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 3eb511c5cf60..60adb5ce12ec 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -1573,7 +1573,19 @@ public void testGetRegion() throws Exception {
       ct.stop();
     }
   }
-
+  
+  @Test
+  public void testRootTableSplit() throws Exception {
+    Scan s = new Scan();
+    HTable rootTable = new HTable(TEST_UTIL.getConfiguration(), HConstants.ROOT_TABLE_NAME);
+    ResultScanner scanner = rootTable.getScanner(s);
+    Result metaEntry = scanner.next();
+    this.admin.split(HConstants.ROOT_TABLE_NAME, metaEntry.getRow());
+    Thread.sleep(1000);
+    List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(HConstants.ROOT_TABLE_NAME);
+    assertEquals("ROOT region should not be splitted.",1, regions.size());
+  }
+  
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 81ccb90c9241001c25c4dfcbd7b6e1cdcaa0c2b6 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 11 Nov 2012 14:37:24 +0000
Subject: [PATCH 0577/1540] HBASE-7143 TestMetaMigrationRemovingHTD fails when
 used with Hadoop 0.23/2.x (Andrey Klochlov)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408012 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/TestMetaMigrationRemovingHTD.java      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
index 34e9956a373d..6eff27c66545 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
@@ -79,8 +79,7 @@ public static void setUpBeforeClass() throws Exception {
     Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
     if (!fs.isDirectory(hbaseRootDir.getParent())) {
       // mkdir at first
-      doFsCommand(shell,
-        new String [] {"-mkdir", hbaseRootDir.getParent().toString()});
+      fs.mkdirs(hbaseRootDir.getParent());
     }
     doFsCommand(shell,
       new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});

From dadd236f99dc500a53488814865a11b655fce082 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 11 Nov 2012 21:04:24 +0000
Subject: [PATCH 0578/1540] HBASE-7142 TestSplitLogManager#testDeadWorker may
 fail because of hard limit on the TimeoutMonitor's timeout period (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408119 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/TestSplitLogManager.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 6602134aacee..13bace757655 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -432,8 +432,8 @@ public void testDeadWorker() throws Exception {
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1"));
     waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
     slm.handleDeadWorker("worker1");
-    waitForCounter(tot_mgr_resubmit, 0, 1, 1000);
-    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, 1000);
+    waitForCounter(tot_mgr_resubmit, 0, 1, 3000);
+    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, 3000);
 
     int version1 = ZKUtil.checkExists(zkw, tasknode);
     assertTrue(version1 > version);

From 8dc9cbefa1595397c35d5d846391307241da42d2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 12 Nov 2012 19:46:01 +0000
Subject: [PATCH 0579/1540] HBASE-7103 Need to fail split if SPLIT znode is
 deleted even before the split is completed. (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408421 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/SplitTransaction.java  |  42 +++---
 .../TestSplitTransactionOnCluster.java        | 140 +++++++++++++++++-
 2 files changed, 156 insertions(+), 26 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 85f585d79b12..044c8f096c88 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -107,10 +107,6 @@ public class SplitTransaction {
    * we need to rollback.
    */
   enum JournalEntry {
-    /**
-     * Before creating node in splitting state.
-     */
-    STARTED_SPLITTING,
     /**
      * Set region as in transition, set it into SPLITTING state.
      */
@@ -236,20 +232,31 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
       server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout",
           this.fileSplitTimeout);
 
-    this.journal.add(JournalEntry.STARTED_SPLITTING);
     // Set ephemeral SPLITTING znode up in zk.  Mocked servers sometimes don't
     // have zookeeper so don't do zk stuff if server or zookeeper is null
     if (server != null && server.getZooKeeper() != null) {
       try {
-        this.znodeVersion = createNodeSplitting(server.getZooKeeper(),
+        createNodeSplitting(server.getZooKeeper(),
           this.parent.getRegionInfo(), server.getServerName());
       } catch (KeeperException e) {
-        throw new IOException("Failed setting SPLITTING znode on " +
+        throw new IOException("Failed creating SPLITTING znode on " +
           this.parent.getRegionNameAsString(), e);
       }
     }
     this.journal.add(JournalEntry.SET_SPLITTING_IN_ZK);
-
+    if (server != null && server.getZooKeeper() != null) {
+      try {
+        // Transition node from SPLITTING to SPLITTING after creating the split node.
+        // Master will get the callback for node change only if the transition is successful.
+        // Note that if the transition fails then the rollback will delete the created znode
+        // TODO : May be we can add some new state to znode and handle the new state incase of success/failure
+        this.znodeVersion = transitionNodeSplitting(server.getZooKeeper(),
+            this.parent.getRegionInfo(), server.getServerName(), -1);
+      } catch (KeeperException e) {
+        throw new IOException("Failed setting SPLITTING znode on "
+            + this.parent.getRegionNameAsString(), e);
+      }
+    }
     createSplitDir(this.parent.getFilesystem(), this.splitdir);
     this.journal.add(JournalEntry.CREATE_SPLIT_DIR);
  
@@ -740,15 +747,9 @@ public boolean rollback(final Server server, final RegionServerServices services
       JournalEntry je = iterator.previous();
       switch(je) {
       
-      case STARTED_SPLITTING:
-        if (server != null && server.getZooKeeper() != null) {
-          cleanZK(server, this.parent.getRegionInfo(), false);
-        }
-        break;
-      
       case SET_SPLITTING_IN_ZK:
         if (server != null && server.getZooKeeper() != null) {
-          cleanZK(server, this.parent.getRegionInfo(), true);
+          cleanZK(server, this.parent.getRegionInfo());
         }
         break;
 
@@ -841,15 +842,11 @@ static void cleanupAnySplitDetritus(final HRegion r) throws IOException {
     LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
   }
 
-  private static void cleanZK(final Server server, final HRegionInfo hri, boolean abort) {
+  private static void cleanZK(final Server server, final HRegionInfo hri) {
     try {
       // Only delete if its in expected state; could have been hijacked.
       ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
         EventType.RS_ZK_REGION_SPLITTING);
-    } catch (KeeperException.NoNodeException nn) {
-      if (abort) {
-        server.abort("Failed cleanup of " + hri.getRegionNameAsString(), nn);
-      }
     } catch (KeeperException e) {
       server.abort("Failed cleanup of " + hri.getRegionNameAsString(), e);
     }
@@ -869,7 +866,7 @@ private static void cleanZK(final Server server, final HRegionInfo hri, boolean
    * @throws KeeperException 
    * @throws IOException 
    */
-  int createNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo region,
+  void createNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo region,
       final ServerName serverName) throws KeeperException, IOException {
     LOG.debug(zkw.prefix("Creating ephemeral node for " +
       region.getEncodedName() + " in SPLITTING state"));
@@ -881,9 +878,6 @@ int createNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo region,
     if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
       throw new IOException("Failed create of ephemeral " + node);
     }
-    // Transition node from SPLITTING to SPLITTING and pick up version so we
-    // can be sure this znode is ours; version is needed deleting.
-    return transitionNodeSplitting(zkw, region, serverName, -1);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index fc124222b341..4ed8beafdf5c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -27,6 +27,8 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.NavigableMap;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -37,6 +39,7 @@
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -70,7 +73,11 @@ public class TestSplitTransactionOnCluster {
   private HBaseAdmin admin = null;
   private MiniHBaseCluster cluster = null;
   private static final int NB_SERVERS = 2;
-
+  private static CountDownLatch latch = new CountDownLatch(1);
+  private static boolean secondSplit = false;
+  private static boolean callRollBack = false;
+  private static boolean firstSplitCompleted = false;
+  
   private static final HBaseTestingUtility TESTING_UTIL =
     new HBaseTestingUtility();
 
@@ -564,7 +571,7 @@ int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent,
       } else {
         st = new MockedSplitTransaction(regions.get(0), null) {
           @Override
-          int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
+          void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
               throws KeeperException, IOException {
             throw new IOException();
           }
@@ -643,6 +650,100 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
     }
   }
   
+  @Test(timeout = 2000000)
+  public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
+    final byte[] tableName = Bytes
+        .toBytes("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
+    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+    try {
+      // Create table then get the single region for our new table.
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor("cf"));
+      admin.createTable(htd);
+      HTable t = new HTable(cluster.getConfiguration(), tableName);
+      while (!(cluster.getRegions(tableName).size() == 1)) {
+        Thread.sleep(100);
+      }
+      final List<HRegion> regions = cluster.getRegions(tableName);
+      HRegionInfo hri = getAndCheckSingleTableRegion(regions);
+      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+      final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+      insertData(tableName, admin, t);
+      // Turn off balancer so it doesn't cut in and mess up our placements.
+      this.admin.setBalancerRunning(false, false);
+      // Turn off the meta scanner so it don't remove parent on us.
+      cluster.getMaster().setCatalogJanitorEnabled(false);
+
+      new Thread() {
+        public void run() {
+          SplitTransaction st = null;
+          st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
+          try {
+            st.prepare();
+            st.execute(regionServer, regionServer);
+          } catch (IOException e) {
+
+          }
+        }
+      }.start();
+      while (!callRollBack) {
+        Thread.sleep(100);
+      }
+      SplitTransaction st = null;
+      st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
+      try {
+        secondSplit = true;
+        st.prepare();
+        st.execute(regionServer, regionServer);
+      } catch (IOException e) {
+        LOG.debug("Rollback started :"+ e.getMessage());
+        st.rollback(regionServer, regionServer);
+      }
+      while (!firstSplitCompleted) {
+        Thread.sleep(100);
+      }
+      NavigableMap<String, RegionState> rit = cluster.getMaster().getAssignmentManager()
+          .getRegionsInTransition();
+      while (rit.containsKey(hri.getTableNameAsString())) {
+        Thread.sleep(100);
+      }
+      List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
+      // Region server side split is successful.
+      assertEquals("The parent region should be splitted", 2, onlineRegions.size());
+      //Should be present in RIT
+      List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager().getRegionsOfTable(tableName);
+      // Master side should also reflect the same
+      assertEquals("No of regions in master", 2, regionsOfTable.size());
+    } finally {
+      admin.setBalancerRunning(true, false);
+      secondSplit = false;
+      firstSplitCompleted = false;
+      callRollBack = false;
+      cluster.getMaster().setCatalogJanitorEnabled(true);
+      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+        admin.disableTable(tableName);
+        admin.deleteTable(tableName);
+        admin.close();
+      }
+    }
+  }
+
+  private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
+      InterruptedException {
+    Put p = new Put(Bytes.toBytes("row1"));
+    p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
+    t.put(p);
+    p = new Put(Bytes.toBytes("row2"));
+    p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
+    t.put(p);
+    p = new Put(Bytes.toBytes("row3"));
+    p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
+    t.put(p);
+    p = new Put(Bytes.toBytes("row4"));
+    p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
+    t.put(p);
+    admin.flush(tableName);
+  }
   @Test(timeout = 15000)
   public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfullyExecuteRollback()
       throws Exception {
@@ -695,8 +796,43 @@ public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfull
   }
   public static class MockedSplitTransaction extends SplitTransaction {
 
+    private HRegion currentRegion;
     public MockedSplitTransaction(HRegion r, byte[] splitrow) {
       super(r, splitrow);
+      this.currentRegion = r;
+    }
+    
+    @Override
+    void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
+        throws IOException {
+      if (this.currentRegion.getRegionInfo().getTableNameAsString()
+          .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
+        try {
+          if (!secondSplit){
+            callRollBack = true;
+            latch.await();
+          }
+        } catch (InterruptedException e) {
+        }
+       
+      }
+      super.transitionZKNode(server, services, a, b);
+      if (this.currentRegion.getRegionInfo().getTableNameAsString()
+          .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
+        firstSplitCompleted = true;
+      }
+    }
+    @Override
+    public boolean rollback(Server server, RegionServerServices services) throws IOException {
+      if (this.currentRegion.getRegionInfo().getTableNameAsString()
+          .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
+        if(secondSplit){
+          super.rollback(server, services);
+          latch.countDown();
+          return true;
+        }
+      }
+      return super.rollback(server, services);
     }
 
   }

From 4afeaccc0ceaac9a8e7a3ea22877539ec7340151 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Mon, 12 Nov 2012 20:07:13 +0000
Subject: [PATCH 0580/1540] HBASE-4913 Per-CF compaction Via the Shell

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408435 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       |  88 +++++++++++++--
 .../hadoop/hbase/ipc/HRegionInterface.java    |  14 +++
 .../hbase/regionserver/HRegionServer.java     |  31 +++++-
 src/main/ruby/hbase/admin.rb                  |  22 +++-
 src/main/ruby/shell/commands/compact.rb       |  20 +++-
 src/main/ruby/shell/commands/major_compact.rb |  21 +++-
 .../regionserver/TestCompactionState.java     | 103 +++++++++++++++---
 7 files changed, 257 insertions(+), 42 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 71711c134fa5..f1b804bfde08 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1247,7 +1247,35 @@ public void compact(final String tableNameOrRegionName)
    */
   public void compact(final byte [] tableNameOrRegionName)
   throws IOException, InterruptedException {
-    compact(tableNameOrRegionName, false);
+    compact(tableNameOrRegionName, null, false);
+  }
+  
+  /**
+   * Compact a column family within a table or region.
+   * Asynchronous operation.
+   *
+   * @param tableOrRegionName table or region to compact
+   * @param columnFamily column family within a table or region
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   */
+  public void compact(String tableOrRegionName, String columnFamily)
+    throws IOException,  InterruptedException {
+    compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
+  }
+
+  /**
+   * Compact a column family within a table or region.
+   * Asynchronous operation.
+   *
+   * @param tableNameOrRegionName table or region to compact
+   * @param columnFamily column family within a table or region
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   */
+  public void compact(final byte [] tableNameOrRegionName, final byte[] columnFamily)
+  throws IOException, InterruptedException {
+    compact(tableNameOrRegionName, columnFamily, false);
   }
 
   /**
@@ -1273,7 +1301,36 @@ public void majorCompact(final String tableNameOrRegionName)
    */
   public void majorCompact(final byte [] tableNameOrRegionName)
   throws IOException, InterruptedException {
-    compact(tableNameOrRegionName, true);
+    compact(tableNameOrRegionName, null, true);
+  }
+  
+  /**
+   * Major compact a column family within a table or region.
+   * Asynchronous operation.
+   *
+   * @param tableNameOrRegionName table or region to major compact
+   * @param columnFamily column family within a table or region
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   */
+  public void majorCompact(final String tableNameOrRegionName,
+    final String columnFamily) throws IOException, InterruptedException {
+    majorCompact(Bytes.toBytes(tableNameOrRegionName),
+      Bytes.toBytes(columnFamily));
+  }
+
+  /**
+   * Major compact a column family within a table or region.
+   * Asynchronous operation.
+   *
+   * @param tableNameOrRegionName table or region to major compact
+   * @param columnFamily column family within a table or region
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   */
+  public void majorCompact(final byte [] tableNameOrRegionName,
+    final byte[] columnFamily) throws IOException, InterruptedException {
+    compact(tableNameOrRegionName, columnFamily, true);
   }
 
   /**
@@ -1281,11 +1338,13 @@ public void majorCompact(final byte [] tableNameOrRegionName)
    * Asynchronous operation.
    *
    * @param tableNameOrRegionName table or region to compact
+   * @param columnFamily column family within a table or region
    * @param major True if we are to do a major compaction.
    * @throws IOException if a remote or network exception occurs
    * @throws InterruptedException
    */
-  private void compact(final byte [] tableNameOrRegionName, final boolean major)
+  private void compact(final byte [] tableNameOrRegionName,
+    final byte[] columnFamily, final boolean major)
   throws IOException, InterruptedException {
     CatalogTracker ct = getCatalogTracker();
     try {
@@ -1295,7 +1354,7 @@ private void compact(final byte [] tableNameOrRegionName, final boolean major)
         if (regionServerPair.getSecond() == null) {
           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
         } else {
-          compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major);
+          compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
         }
       } else {
         final String tableName = tableNameString(tableNameOrRegionName, ct);
@@ -1306,7 +1365,7 @@ private void compact(final byte [] tableNameOrRegionName, final boolean major)
           if (pair.getFirst().isOffline()) continue;
           if (pair.getSecond() == null) continue;
           try {
-            compact(pair.getSecond(), pair.getFirst(), major);
+            compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
           } catch (NotServingRegionException e) {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
@@ -1322,11 +1381,26 @@ private void compact(final byte [] tableNameOrRegionName, final boolean major)
   }
 
   private void compact(final ServerName sn, final HRegionInfo hri,
-      final boolean major)
+      final boolean major, final byte [] family)
   throws IOException {
     HRegionInterface rs =
       this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
-    rs.compactRegion(hri, major);
+    if (family != null) {
+      try {
+        rs.compactRegion(hri, major, family);
+      } catch (IOException ioe) {
+        String notFoundMsg = "java.lang.NoSuchMethodException: org.apache.hadoop.hbase.ipc.HRegionInterface."
+          + "compactRegion(org.apache.hadoop.hbase.HRegionInfo, boolean, [B)";
+        if (ioe.getMessage().contains(notFoundMsg)) {
+          throw new IOException("per-column family compaction not supported on this version "
+            + "of the HBase server.  You may still compact at the table or region level by "
+          	+ "omitting the column family name.  Alternatively, you can upgrade the HBase server");
+        }
+        throw ioe;
+      }
+    } else {
+      rs.compactRegion(hri, major);
+    }
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index d5e329ecf6c3..9886b3a0dc09 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -532,6 +532,20 @@ void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
   void compactRegion(HRegionInfo regionInfo, boolean major)
   throws NotServingRegionException, IOException;
 
+  /**
+   * Compacts a column-family within a specified region.
+   * Performs a major compaction if specified.
+   * <p>
+   * This method is asynchronous.
+   * @param regionInfo region to compact
+   * @param major true to force major compaction
+   * @param columnFamily column family within a region to compact
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void compactRegion(HRegionInfo regionInfo, boolean major, byte[] columnFamily)
+  throws NotServingRegionException, IOException;
+   
   /**
    * Replicates the given entries. The guarantee is that the given entries
    * will be durable on the slave cluster if this method returns without
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 44cc5c123cce..1ab2d387dc61 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2966,16 +2966,41 @@ public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
   @QosPriority(priority=HConstants.HIGH_QOS)
   public void compactRegion(HRegionInfo regionInfo, boolean major)
       throws NotServingRegionException, IOException {
+    compactRegion(regionInfo, major, null);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void compactRegion(HRegionInfo regionInfo, boolean major,  byte[] family)
+      throws NotServingRegionException, IOException {
     checkOpen();
     HRegion region = getRegion(regionInfo.getRegionName());
+    Store store = null;
+    if (family != null) {
+      store = region.getStore(family);
+      if (store == null) {
+        throw new IOException("column family " + Bytes.toString(family) +
+          " does not exist in region " + new String(region.getRegionNameAsString()));
+      }
+    }
+
     if (major) {
-      region.triggerMajorCompaction();
+      if (family != null) {
+        store.triggerMajorCompaction();
+      } else {
+        region.triggerMajorCompaction();
+      }
     }
     LOG.trace("User-triggered compaction requested for region " +
       region.getRegionNameAsString());
-    compactSplitThread.requestCompaction(region, "User-triggered "
-        + (major ? "major " : "") + "compaction",
+    String log = "User-triggered " + (major ? "major " : "") + "compaction";
+    if (family != null) {
+      compactSplitThread.requestCompaction(region, store, log,
         Store.PRIORITY_USER);
+    } else {
+      compactSplitThread.requestCompaction(region, log,
+        Store.PRIORITY_USER);
+    }
   }
 
   /** @return the info server */
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index b6042da24c69..4cf696377c8f 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -50,15 +50,25 @@ def flush(table_or_region_name)
     end
 
     #----------------------------------------------------------------------------------------------
-    # Requests a table or region compaction
-    def compact(table_or_region_name)
-      @admin.compact(table_or_region_name)
+    # Requests a table or region or column family compaction
+    def compact(table_or_region_name, family)
+      if family == nil
+        @admin.compact(table_or_region_name)
+      else
+        # We are compacting a column family within a region.
+        @admin.compact(table_or_region_name, family)
+      end
     end
 
     #----------------------------------------------------------------------------------------------
-    # Requests a table or region major compaction
-    def major_compact(table_or_region_name)
-      @admin.majorCompact(table_or_region_name)
+    # Requests a table or region or column family major compaction
+    def major_compact(table_or_region_name, family)
+      if family == nil
+        @admin.majorCompact(table_or_region_name)
+      else
+        # We are major compacting a column family within a region or table.
+        @admin.majorCompact(table_or_region_name, family)
+      end
     end
 
     #----------------------------------------------------------------------------------------------
diff --git a/src/main/ruby/shell/commands/compact.rb b/src/main/ruby/shell/commands/compact.rb
index d8f71de4904f..fa0b82eef8cc 100644
--- a/src/main/ruby/shell/commands/compact.rb
+++ b/src/main/ruby/shell/commands/compact.rb
@@ -23,14 +23,24 @@ module Commands
     class Compact < Command
       def help
         return <<-EOF
-Compact all regions in passed table or pass a region row
-to compact an individual region
-EOF
+          Compact all regions in passed table or pass a region row
+          to compact an individual region. You can also compact a single column
+          family within a region.
+          Examples:
+          Compact all regions in a table:
+          hbase> compact 't1'
+          Compact an entire region:
+          hbase> compact 'r1'
+          Compact only a column family within a region:
+          hbase> compact 'r1', 'c1'
+          Compact a column family within a table:
+          hbase> compact 't1', 'c1'
+        EOF
       end
 
-      def command(table_or_region_name)
+      def command(table_or_region_name, family = nil)
         format_simple_command do
-          admin.compact(table_or_region_name)
+          admin.compact(table_or_region_name, family)
         end
       end
     end
diff --git a/src/main/ruby/shell/commands/major_compact.rb b/src/main/ruby/shell/commands/major_compact.rb
index 56b081e3f968..325344b5265a 100644
--- a/src/main/ruby/shell/commands/major_compact.rb
+++ b/src/main/ruby/shell/commands/major_compact.rb
@@ -23,14 +23,25 @@ module Commands
     class MajorCompact < Command
       def help
         return <<-EOF
-Run major compaction on passed table or pass a region row
-to major compact an individual region
-EOF
+          Run major compaction on passed table or pass a region row
+          to major compact an individual region. To compact a single
+          column family within a region specify the region name
+          followed by the column family name.
+          Examples:
+          Compact all regions in a table:
+          hbase> major_compact 't1'
+          Compact an entire region:
+          hbase> major_compact 'r1'
+          Compact a single column family within a region:
+          hbase> major_compact 'r1', 'c1'
+          Compact a single column family within a table:
+          hbase> major_compact 't1', 'c1'
+        EOF
       end
 
-      def command(table_or_region_name)
+      def command(table_or_region_name, family = nil)
         format_simple_command do
-          admin.major_compact(table_or_region_name)
+          admin.major_compact(table_or_region_name, family)
         end
       end
     end
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
index 8fc2f593df97..79594d730c65 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
@@ -60,12 +60,52 @@ public static void tearDownAfterClass() throws Exception {
 
   @Test(timeout=60000)
   public void testMajorCompaction() throws IOException, InterruptedException {
-    compaction("testMajorCompaction", 8, CompactionState.MAJOR);
+    compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
   }
 
   @Test(timeout=60000)
   public void testMinorCompaction() throws IOException, InterruptedException {
-    compaction("testMinorCompaction", 15, CompactionState.MINOR);
+    compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
+  }
+
+  @Test(timeout=60000)
+  public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
+    compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
+  }
+
+  @Test(timeout=60000)
+  public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
+    compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
+  }
+
+  @Test
+  public void testInvalidColumnFamily() throws IOException, InterruptedException {
+    byte [] table = Bytes.toBytes("testInvalidColumnFamily");
+    byte [] family = Bytes.toBytes("family");
+    byte [] fakecf = Bytes.toBytes("fakecf");
+    boolean caughtMinorCompact = false;
+    boolean caughtMajorCompact = false;
+    HTable ht = null;
+    try {
+      ht = TEST_UTIL.createTable(table, family);
+      HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+      try {
+        admin.compact(table, fakecf);
+      } catch (IOException ioe) {
+        caughtMinorCompact = true;
+      }
+      try {
+        admin.majorCompact(table, fakecf);
+      } catch (IOException ioe) {
+        caughtMajorCompact = true;
+      }
+    } finally {
+      if (ht != null) {
+        TEST_UTIL.deleteTable(table);
+      }
+      assertTrue(caughtMinorCompact);
+      assertTrue(caughtMajorCompact);
+    }
   }
 
   /**
@@ -75,27 +115,40 @@ public void testMinorCompaction() throws IOException, InterruptedException {
    * @param tableName
    * @param flushes
    * @param expectedState
+   * @param singleFamily otherwise, run compaction on all cfs
    * @throws IOException
    * @throws InterruptedException
    */
   private void compaction(final String tableName, final int flushes,
-      final CompactionState expectedState) throws IOException, InterruptedException {
+      final CompactionState expectedState, boolean singleFamily)
+      throws IOException, InterruptedException {
     // Create a table with regions
     byte [] table = Bytes.toBytes(tableName);
     byte [] family = Bytes.toBytes("family");
+    byte [][] families =
+      {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
     HTable ht = null;
     try {
-      ht = TEST_UTIL.createTable(table, family);
-      loadData(ht, family, 3000, flushes);
+      ht = TEST_UTIL.createTable(table, families);
+      loadData(ht, families, 3000, flushes);
       HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
       List<HRegion> regions = rs.getOnlineRegions(table);
-      int countBefore = countStoreFiles(regions, family);
+      int countBefore = countStoreFilesInFamilies(regions, families);
+      int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
       assertTrue(countBefore > 0); // there should be some data files
       HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
       if (expectedState == CompactionState.MINOR) {
-        admin.compact(tableName);
+        if (singleFamily) {
+          admin.compact(table, family);
+        } else {
+          admin.compact(table);
+        }
       } else {
-        admin.majorCompact(table);
+        if (singleFamily) {
+          admin.majorCompact(table, family);
+        } else {
+          admin.majorCompact(table);
+        }
       }
       long curt = System.currentTimeMillis();
       long waitTime = 5000;
@@ -126,10 +179,22 @@ private void compaction(final String tableName, final int flushes,
         // Now, compaction should be done.
         assertEquals(CompactionState.NONE, state);
       }
-      int countAfter = countStoreFiles(regions, family);
+      int countAfter = countStoreFilesInFamilies(regions, families);
+      int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
       assertTrue(countAfter < countBefore);
-      if (expectedState == CompactionState.MAJOR) assertTrue(1 == countAfter);
-      else assertTrue(1 < countAfter);
+      if (!singleFamily) {
+        if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
+        else assertTrue(families.length < countAfter);
+      } else {
+        int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
+        // assert only change was to single column family
+        assertTrue(singleFamDiff == (countBefore - countAfter));
+        if (expectedState == CompactionState.MAJOR) {
+          assertTrue(1 == countAfterSingleFamily);
+        } else {
+          assertTrue(1 < countAfterSingleFamily);
+        }
+      }
     } finally {
       if (ht != null) {
         TEST_UTIL.deleteTable(table);
@@ -137,16 +202,20 @@ private void compaction(final String tableName, final int flushes,
     }
   }
 
-  private static int countStoreFiles(
+  private static int countStoreFilesInFamily(
       List<HRegion> regions, final byte[] family) {
+    return countStoreFilesInFamilies(regions, new byte[][]{family});
+  }
+
+  private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[][] families) {
     int count = 0;
     for (HRegion region: regions) {
-      count += region.getStoreFileList(new byte[][]{family}).size();
+      count += region.getStoreFileList(families).size();
     }
     return count;
   }
-
-  private static void loadData(final HTable ht, final byte[] family,
+  
+  private static void loadData(final HTable ht, final byte[][] families,
       final int rows, final int flushes) throws IOException {
     List<Put> puts = new ArrayList<Put>(rows);
     byte[] qualifier = Bytes.toBytes("val");
@@ -154,7 +223,9 @@ private static void loadData(final HTable ht, final byte[] family,
       for (int k = 0; k < rows; k++) {
         byte[] row = Bytes.toBytes(random.nextLong());
         Put p = new Put(row);
-        p.add(family, qualifier, row);
+        for (int j = 0; j < families.length; ++j) {
+          p.add(families[ j ], qualifier, row);
+        }
         puts.add(p);
       }
       ht.put(puts);

From 8e03629b705c99f0c90b05318d43bb60c43c0b7e Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Mon, 12 Nov 2012 23:17:29 +0000
Subject: [PATCH 0581/1540] HBASE-7151 Better log message for Per-CF
 compactions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408501 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1ab2d387dc61..a4b4b818173c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2991,9 +2991,10 @@ public void compactRegion(HRegionInfo regionInfo, boolean major,  byte[] family)
         region.triggerMajorCompaction();
       }
     }
+    String familyLogMsg = (family != null)?" for column family: " + Bytes.toString(family):"";
     LOG.trace("User-triggered compaction requested for region " +
-      region.getRegionNameAsString());
-    String log = "User-triggered " + (major ? "major " : "") + "compaction";
+      region.getRegionNameAsString() + familyLogMsg);
+    String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
     if (family != null) {
       compactSplitThread.requestCompaction(region, store, log,
         Store.PRIORITY_USER);

From 00984895734947d6aebf61dfe8c1a80fc39a398e Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 13 Nov 2012 04:17:15 +0000
Subject: [PATCH 0582/1540] HBASE-5984
 TestLogRolling.testLogRollOnPipelineRestart failed with HADOOP 2.0.0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408574 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/wal/TestLogRolling.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
index cb7efc3d0918..a42c801e30bc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
@@ -394,7 +394,7 @@ public void testLogRollOnDatanodeDeath() throws Exception {
    * restarted.
    * @throws Exception
    */
-  @Test
+  //DISABLED BECAUSE FLAKEY @Test
   public void testLogRollOnPipelineRestart() throws Exception {
     LOG.info("Starting testLogRollOnPipelineRestart");
     assertTrue("This test requires HLog file replication.",

From 7ce05fceb19b0f9d15624f49fe35a026cddbb31b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Nov 2012 06:54:16 +0000
Subject: [PATCH 0583/1540] HBASE-5898 Consider double-checked locking for
 block cache lock (Todd, Elliot, LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408621 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/hfile/AbstractHFileReader.java   |   5 -
 .../hadoop/hbase/io/hfile/BlockCache.java     |   5 +-
 .../hbase/io/hfile/DoubleBlockCache.java      |   8 +-
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  10 +-
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  | 121 ++++++++++--------
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   7 +-
 .../hbase/io/hfile/SimpleBlockCache.java      |   2 +-
 .../hbase/io/hfile/slab/SingleSizeCache.java  |   4 +-
 .../hadoop/hbase/io/hfile/slab/SlabCache.java |  13 +-
 .../hadoop/hbase/io/hfile/CacheTestUtils.java |  14 +-
 .../hbase/io/hfile/TestCacheOnWrite.java      |   2 +-
 .../io/hfile/TestHFileDataBlockEncoder.java   |   2 +-
 .../hbase/io/hfile/TestLruBlockCache.java     |  76 +++++------
 .../TestCacheOnWriteInSchema.java             |   2 +-
 14 files changed, 140 insertions(+), 131 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
index f418f905c98b..5573b661e60b 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
@@ -21,7 +21,6 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
@@ -90,10 +89,6 @@ public abstract class AbstractHFileReader extends SchemaConfigured
   /** Block cache configuration. */
   protected final CacheConfig cacheConf;
 
-  protected AtomicLong cacheHits = new AtomicLong();
-  protected AtomicLong blockLoads = new AtomicLong();
-  protected AtomicLong metaLoads = new AtomicLong();
-
   /** Path of file */
   protected final Path path;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
index 6f0e33e71628..e3908b321318 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -48,9 +48,12 @@ public interface BlockCache {
    * Fetch block from cache.
    * @param cacheKey Block to fetch.
    * @param caching Whether this request has caching enabled (used for stats)
+   * @param repeat Whether this is a repeat lookup for the same block
+   *        (used to avoid double counting cache misses when doing double-check locking)
+   *        {@see HFileReaderV2#readBlock(long, long, boolean, boolean, boolean, BlockType)}
    * @return Block or null if block is not in 2 cache.
    */
-  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching);
+  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat);
 
   /**
    * Evict block from cache.
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
index ac36da4f3c4b..0a6c062dffcc 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
@@ -90,14 +90,14 @@ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
   }
 
   @Override
-  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
     Cacheable cachedBlock;
 
-    if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching)) != null) {
+    if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching, repeat)) != null) {
       stats.hit(caching);
       return cachedBlock;
 
-    } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching)) != null) {
+    } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching, repeat)) != null) {
       if (caching) {
         onHeapCache.cacheBlock(cacheKey, cachedBlock);
       }
@@ -105,7 +105,7 @@ public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
       return cachedBlock;
     }
 
-    stats.miss(caching);
+    if (!repeat) stats.miss(caching);
     return null;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index 4dafb311ca4e..f642e3dbc222 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -225,15 +225,12 @@ public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
 
     // Per meta key from any given file, synchronize reads for said block
     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
-      metaLoads.incrementAndGet();
-
       // Check cache for block.  If found return.
       if (cacheConf.isBlockCacheEnabled()) {
         HFileBlock cachedBlock =
           (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
-              cacheConf.shouldCacheBlockOnRead(effectiveCategory));
+              cacheConf.shouldCacheBlockOnRead(effectiveCategory), false);
         if (cachedBlock != null) {
-          cacheHits.incrementAndGet();
           getSchemaMetrics().updateOnCacheHit(effectiveCategory,
               SchemaMetrics.NO_COMPACTION);
           return cachedBlock.getBufferWithoutHeader();
@@ -290,15 +287,12 @@ ByteBuffer readBlockBuffer(int block, boolean cacheBlock,
     // the other choice is to duplicate work (which the cache would prevent you
     // from doing).
     synchronized (dataBlockIndexReader.getRootBlockKey(block)) {
-      blockLoads.incrementAndGet();
-
       // Check cache for block.  If found return.
       if (cacheConf.isBlockCacheEnabled()) {
         HFileBlock cachedBlock =
           (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
-              cacheConf.shouldCacheDataOnRead());
+              cacheConf.shouldCacheDataOnRead(), false);
         if (cachedBlock != null) {
-          cacheHits.incrementAndGet();
           getSchemaMetrics().updateOnCacheHit(
               cachedBlock.getBlockType().getCategory(), isCompaction);
           return cachedBlock.getBufferWithoutHeader();
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index b54175ad9951..093d8567e062 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -210,8 +210,6 @@ public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
     // is OK to do for meta blocks because the meta block index is always
     // single-level.
     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
-      metaLoads.incrementAndGet();
-
       // Check cache for block. If found return.
       long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
       BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
@@ -220,11 +218,10 @@ public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
       cacheBlock &= cacheConf.shouldCacheDataOnRead();
       if (cacheConf.isBlockCacheEnabled()) {
         HFileBlock cachedBlock =
-          (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock);
+          (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false);
         if (cachedBlock != null) {
           // Return a distinct 'shallow copy' of the block,
           // so pos does not get messed by the scanner
-          cacheHits.incrementAndGet();
           getSchemaMetrics().updateOnCacheHit(BlockCategory.META, false);
           return cachedBlock.getBufferWithoutHeader();
         }
@@ -288,69 +285,85 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
         new BlockCacheKey(name, dataBlockOffset,
             dataBlockEncoder.getEffectiveEncodingInCache(isCompaction),
             expectedBlockType);
-    IdLock.Entry lockEntry = offsetLock.getLockEntry(dataBlockOffset);
+
+    boolean useLock = false;
+    IdLock.Entry lockEntry = null;
+
     try {
-      blockLoads.incrementAndGet();
+      while (true) {
 
-      // Check cache for block. If found return.
-      if (cacheConf.isBlockCacheEnabled()) {
-        HFileBlock cachedBlock = (HFileBlock)
-            cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock);
-        if (cachedBlock != null) {
-          BlockCategory blockCategory =
-              cachedBlock.getBlockType().getCategory();
-          cacheHits.incrementAndGet();
+        if (useLock) {
+          lockEntry = offsetLock.getLockEntry(dataBlockOffset);
+        }
 
-          getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction);
+        // Check cache for block. If found return.
+        if (cacheConf.isBlockCacheEnabled()) {
+          // Try and get the block from the block cache.  If the useLock variable is true then this
+          // is the second time through the loop and it should not be counted as a block cache miss.
+          HFileBlock cachedBlock = (HFileBlock)
+              cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, useLock);
+          if (cachedBlock != null) {
+            BlockCategory blockCategory =
+                cachedBlock.getBlockType().getCategory();
 
-          if (cachedBlock.getBlockType() == BlockType.DATA) {
-            HFile.dataBlockReadCnt.incrementAndGet();
-          }
+            getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction);
 
-          validateBlockType(cachedBlock, expectedBlockType);
+            if (cachedBlock.getBlockType() == BlockType.DATA) {
+              HFile.dataBlockReadCnt.incrementAndGet();
+            }
 
-          // Validate encoding type for encoded blocks. We include encoding
-          // type in the cache key, and we expect it to match on a cache hit.
-          if (cachedBlock.getBlockType() == BlockType.ENCODED_DATA &&
-              cachedBlock.getDataBlockEncoding() !=
-              dataBlockEncoder.getEncodingInCache()) {
-            throw new IOException("Cached block under key " + cacheKey + " " +
-                "has wrong encoding: " + cachedBlock.getDataBlockEncoding() +
-                " (expected: " + dataBlockEncoder.getEncodingInCache() + ")");
+            validateBlockType(cachedBlock, expectedBlockType);
+
+            // Validate encoding type for encoded blocks. We include encoding
+            // type in the cache key, and we expect it to match on a cache hit.
+            if (cachedBlock.getBlockType() == BlockType.ENCODED_DATA &&
+                cachedBlock.getDataBlockEncoding() !=
+                    dataBlockEncoder.getEncodingInCache()) {
+              throw new IOException("Cached block under key " + cacheKey + " " +
+                  "has wrong encoding: " + cachedBlock.getDataBlockEncoding() +
+                  " (expected: " + dataBlockEncoder.getEncodingInCache() + ")");
+            }
+            return cachedBlock;
           }
-          return cachedBlock;
+          // Carry on, please load.
+        }
+        if (!useLock) {
+          // check cache again with lock
+          useLock = true;
+          continue;
         }
-        // Carry on, please load.
-      }
-
-      // Load block from filesystem.
-      long startTimeNs = System.nanoTime();
-      HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset,
-          onDiskBlockSize, -1, pread);
-      hfileBlock = dataBlockEncoder.diskToCacheFormat(hfileBlock,
-          isCompaction);
-      validateBlockType(hfileBlock, expectedBlockType);
-      passSchemaMetricsTo(hfileBlock);
-      BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
 
-      final long delta = System.nanoTime() - startTimeNs;
-      HFile.offerReadLatency(delta, pread);
-      getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta);
+        // Load block from filesystem.
+        long startTimeNs = System.nanoTime();
+        HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset,
+            onDiskBlockSize, -1, pread);
+        hfileBlock = dataBlockEncoder.diskToCacheFormat(hfileBlock,
+            isCompaction);
+        validateBlockType(hfileBlock, expectedBlockType);
+        passSchemaMetricsTo(hfileBlock);
+        BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
+
+        final long delta = System.nanoTime() - startTimeNs;
+        HFile.offerReadLatency(delta, pread);
+        getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta);
+
+        // Cache the block if necessary
+        if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
+            hfileBlock.getBlockType().getCategory())) {
+          cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
+              cacheConf.isInMemory());
+        }
 
-      // Cache the block if necessary
-      if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
-              hfileBlock.getBlockType().getCategory())) {
-        cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
-            cacheConf.isInMemory());
-      }
+        if (hfileBlock.getBlockType() == BlockType.DATA) {
+          HFile.dataBlockReadCnt.incrementAndGet();
+        }
 
-      if (hfileBlock.getBlockType() == BlockType.DATA) {
-        HFile.dataBlockReadCnt.incrementAndGet();
+        return hfileBlock;
       }
-
-      return hfileBlock;
     } finally {
-      offsetLock.releaseLockEntry(lockEntry);
+      if (lockEntry != null) {
+        offsetLock.releaseLockEntry(lockEntry);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index ad0878b58c86..e6987ff71853 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -327,13 +327,16 @@ protected long updateSizeMetrics(CachedBlock cb, boolean evict) {
    * Get the buffer of the block with the specified name.
    * @param cacheKey block's cache key
    * @param caching true if the caller caches blocks on cache misses
+   * @param repeat Whether this is a repeat lookup for the same block
+   *        (used to avoid double counting cache misses when doing double-check locking)
+   *        {@see HFileReaderV2#readBlock(long, long, boolean, boolean, boolean, BlockType)}
    * @return buffer of specified cache key, or null if not in cache
    */
   @Override
-  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
     CachedBlock cb = map.get(cacheKey);
     if(cb == null) {
-      stats.miss(caching);
+      if (!repeat) stats.miss(caching);
       return null;
     }
     stats.hit(caching);
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
index 1d234301d773..a14909be5fca 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
@@ -68,7 +68,7 @@ public synchronized long size() {
     return cache.size();
   }
 
-  public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) {
+  public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
     processQueue(); // clear out some crap.
     Ref ref = cache.get(cacheKey);
     if (ref == null)
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
index 152f197dc3da..d68c2d5f434f 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
@@ -152,10 +152,10 @@ public void cacheBlock(BlockCacheKey blockName, Cacheable toBeCached) {
   }
 
   @Override
-  public Cacheable getBlock(BlockCacheKey key, boolean caching) {
+  public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) {
     CacheablePair contentBlock = backingMap.get(key);
     if (contentBlock == null) {
-      stats.miss(caching);
+      if (!repeat) stats.miss(caching);
       return null;
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
index 2557862d5a46..9fc60433d9e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
@@ -228,24 +228,25 @@ public CacheStats getStats() {
 
   /**
    * Get the buffer of the block with the specified name.
-   *
-   * @param key
    * @param caching
+   * @param key
+   * @param repeat
+   *
    * @return buffer of specified block name, or null if not in cache
    */
-  public Cacheable getBlock(BlockCacheKey key, boolean caching) {
+  public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) {
     SingleSizeCache cachedBlock = backingStore.get(key);
     if (cachedBlock == null) {
-      stats.miss(caching);
+      if (!repeat) stats.miss(caching);
       return null;
     }
 
-    Cacheable contentBlock = cachedBlock.getBlock(key, caching);
+    Cacheable contentBlock = cachedBlock.getBlock(key, caching, false);
 
     if (contentBlock != null) {
       stats.hit(caching);
     } else {
-      stats.miss(caching);
+      if (!repeat) stats.miss(caching);
     }
     return contentBlock;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
index 106056ff0b03..609c69c4c990 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
@@ -92,12 +92,12 @@ public void doAnAction() throws Exception {
             }
             toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block);
             Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName,
-                false);
+                false, false);
             if (retrievedBlock != null) {
               assertEquals(ourBlock.block, retrievedBlock);
               toBeTested.evictBlock(ourBlock.blockName);
               hits.incrementAndGet();
-              assertNull(toBeTested.getBlock(ourBlock.blockName, false));
+              assertNull(toBeTested.getBlock(ourBlock.blockName, false, false));
             } else {
               miss.incrementAndGet();
             }
@@ -125,7 +125,7 @@ public static void testCacheSimple(BlockCache toBeTested, int blockSize,
     HFileBlockPair[] blocks = generateHFileBlocks(numBlocks, blockSize);
     // Confirm empty
     for (HFileBlockPair block : blocks) {
-      assertNull(toBeTested.getBlock(block.blockName, true));
+      assertNull(toBeTested.getBlock(block.blockName, true, false));
     }
 
     // Add blocks
@@ -138,7 +138,7 @@ public static void testCacheSimple(BlockCache toBeTested, int blockSize,
     // MapMaker makes no guarantees when it will evict, so neither can we.
 
     for (HFileBlockPair block : blocks) {
-      HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true);
+      HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true, false);
       if (buf != null) {
         assertEquals(block.block, buf);
       }
@@ -149,7 +149,7 @@ public static void testCacheSimple(BlockCache toBeTested, int blockSize,
 
     for (HFileBlockPair block : blocks) {
       try {
-        if (toBeTested.getBlock(block.blockName, true) != null) {
+        if (toBeTested.getBlock(block.blockName, true, false) != null) {
           toBeTested.cacheBlock(block.blockName, block.block);
           fail("Cache should not allow re-caching a block");
         }
@@ -179,7 +179,7 @@ public static void hammerSingleKey(final BlockCache toBeTested,
         @Override
         public void doAnAction() throws Exception {
           ByteArrayCacheable returned = (ByteArrayCacheable) toBeTested
-              .getBlock(key, false);
+              .getBlock(key, false, false);
           assertArrayEquals(buf, returned.buf);
           totalQueries.incrementAndGet();
         }
@@ -218,7 +218,7 @@ public void doAnAction() throws Exception {
             final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
 
             ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested
-                .getBlock(key, true);
+                .getBlock(key, true, false);
             if (gotBack != null) {
               assertArrayEquals(gotBack.buf, bac.buf);
             } else {
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index c8163f3aa755..aa20de3dce5d 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -240,7 +240,7 @@ private void readStoreFile() throws IOException {
           false, null);
       BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
           offset, encodingInCache, block.getBlockType());
-      boolean isCached = blockCache.getBlock(blockCacheKey, true) != null;
+      boolean isCached = blockCache.getBlock(blockCacheKey, true, false) != null;
       boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
       if (shouldBeCached != isCached) {
         throw new AssertionError(
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
index 16323a646e49..4eea2ff40add 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
@@ -98,7 +98,7 @@ public void testEncodingWithCache() {
     BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
     blockCache.cacheBlock(cacheKey, cacheBlock);
 
-    HeapSize heapSize = blockCache.getBlock(cacheKey, false);
+    HeapSize heapSize = blockCache.getBlock(cacheKey, false, false);
     assertTrue(heapSize instanceof HFileBlock);
 
     HFileBlock returnedBlock = (HFileBlock) heapSize;;
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index d1b3c417ec76..b6aef4e47540 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -116,7 +116,7 @@ public void testCacheSimple() throws Exception {
 
     // Confirm empty
     for (CachedItem block : blocks) {
-      assertTrue(cache.getBlock(block.cacheKey, true) == null);
+      assertTrue(cache.getBlock(block.cacheKey, true, false) == null);
     }
 
     // Add blocks
@@ -130,7 +130,7 @@ public void testCacheSimple() throws Exception {
 
     // Check if all blocks are properly cached and retrieved
     for (CachedItem block : blocks) {
-      HeapSize buf = cache.getBlock(block.cacheKey, true);
+      HeapSize buf = cache.getBlock(block.cacheKey, true, false);
       assertTrue(buf != null);
       assertEquals(buf.heapSize(), block.heapSize());
     }
@@ -150,7 +150,7 @@ public void testCacheSimple() throws Exception {
 
     // Check if all blocks are properly cached and retrieved
     for (CachedItem block : blocks) {
-      HeapSize buf = cache.getBlock(block.cacheKey, true);
+      HeapSize buf = cache.getBlock(block.cacheKey, true, false);
       assertTrue(buf != null);
       assertEquals(buf.heapSize(), block.heapSize());
     }
@@ -195,10 +195,10 @@ public void testCacheEvictionSimple() throws Exception {
         (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
 
     // All blocks except block 0 and 1 should be in the cache
-    assertTrue(cache.getBlock(blocks[0].cacheKey, true) == null);
-    assertTrue(cache.getBlock(blocks[1].cacheKey, true) == null);
+    assertTrue(cache.getBlock(blocks[0].cacheKey, true, false) == null);
+    assertTrue(cache.getBlock(blocks[1].cacheKey, true, false) == null);
     for(int i=2;i<blocks.length;i++) {
-      assertEquals(cache.getBlock(blocks[i].cacheKey, true),
+      assertEquals(cache.getBlock(blocks[i].cacheKey, true, false),
           blocks[i]);
     }
   }
@@ -220,7 +220,7 @@ public void testCacheEvictionTwoPriorities() throws Exception {
     for (CachedItem block : multiBlocks) {
       cache.cacheBlock(block.cacheKey, block);
       expectedCacheSize += block.cacheBlockHeapSize();
-      assertEquals(cache.getBlock(block.cacheKey, true), block);
+      assertEquals(cache.getBlock(block.cacheKey, true, false), block);
     }
 
     // Add the single blocks (no get)
@@ -250,14 +250,14 @@ public void testCacheEvictionTwoPriorities() throws Exception {
     // This test makes multi go barely over its limit, in-memory
     // empty, and the rest in single.  Two single evictions and
     // one multi eviction expected.
-    assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true) == null);
-    assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true) == null);
+    assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false) == null);
+    assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false) == null);
 
     // And all others to be cached
     for(int i=1;i<4;i++) {
-      assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true),
+      assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false),
           singleBlocks[i]);
-      assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true),
+      assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false),
           multiBlocks[i]);
     }
   }
@@ -295,7 +295,7 @@ public void testCacheEvictionThreePriorities() throws Exception {
       // Add and get multi blocks
       cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
       expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
-      cache.getBlock(multiBlocks[i].cacheKey, true);
+      cache.getBlock(multiBlocks[i].cacheKey, true, false);
 
       // Add memory blocks as such
       cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
@@ -317,10 +317,10 @@ public void testCacheEvictionThreePriorities() throws Exception {
     assertEquals(1, cache.getEvictedCount());
 
     // Verify oldest single block is the one evicted
-    assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true));
+    assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false));
 
     // Change the oldest remaining single block to a multi
-    cache.getBlock(singleBlocks[1].cacheKey, true);
+    cache.getBlock(singleBlocks[1].cacheKey, true, false);
 
     // Insert another single block
     cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
@@ -330,7 +330,7 @@ public void testCacheEvictionThreePriorities() throws Exception {
     assertEquals(2, cache.getEvictedCount());
 
     // Oldest multi block should be evicted now
-    assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true));
+    assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false));
 
     // Insert another memory block
     cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
@@ -340,7 +340,7 @@ public void testCacheEvictionThreePriorities() throws Exception {
     assertEquals(3, cache.getEvictedCount());
 
     // Oldest memory block should be evicted now
-    assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true));
+    assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false));
 
     // Add a block that is twice as big (should force two evictions)
     CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
@@ -351,12 +351,12 @@ public void testCacheEvictionThreePriorities() throws Exception {
     assertEquals(6, cache.getEvictedCount());
 
     // Expect three remaining singles to be evicted
-    assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true));
-    assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true));
-    assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true));
+    assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false));
 
     // Make the big block a multi block
-    cache.getBlock(bigBlocks[0].cacheKey, true);
+    cache.getBlock(bigBlocks[0].cacheKey, true, false);
 
     // Cache another single big block
     cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
@@ -366,9 +366,9 @@ public void testCacheEvictionThreePriorities() throws Exception {
     assertEquals(9, cache.getEvictedCount());
 
     // Expect three remaining multis to be evicted
-    assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true));
-    assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true));
-    assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true));
+    assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false));
 
     // Cache a big memory block
     cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
@@ -378,9 +378,9 @@ public void testCacheEvictionThreePriorities() throws Exception {
     assertEquals(12, cache.getEvictedCount());
 
     // Expect three remaining in-memory to be evicted
-    assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true));
-    assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true));
-    assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true));
+    assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false));
 
 
   }
@@ -408,7 +408,7 @@ public void testScanResistance() throws Exception {
     // Add 5 multi blocks
     for (CachedItem block : multiBlocks) {
       cache.cacheBlock(block.cacheKey, block);
-      cache.getBlock(block.cacheKey, true);
+      cache.getBlock(block.cacheKey, true, false);
     }
 
     // Add 5 single blocks
@@ -423,10 +423,10 @@ public void testScanResistance() throws Exception {
     assertEquals(4, cache.getEvictedCount());
 
     // Should have been taken off equally from single and multi
-    assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true));
-    assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true));
-    assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true));
-    assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true));
+    assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false));
+    assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false));
 
     // Let's keep "scanning" by adding single blocks.  From here on we only
     // expect evictions from the single bucket.
@@ -477,7 +477,7 @@ public void testResizeBlockCache() throws Exception {
 
       // Add and get multi blocks
       cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
-      cache.getBlock(multiBlocks[i].cacheKey, true);
+      cache.getBlock(multiBlocks[i].cacheKey, true, false);
 
       // Add memory blocks as such
       cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
@@ -497,16 +497,16 @@ public void testResizeBlockCache() throws Exception {
 
     // And the oldest 5 blocks from each category should be gone
     for(int i=0;i<5;i++) {
-      assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true));
-      assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true));
-      assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true));
+      assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false));
+      assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false));
+      assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false));
     }
 
     // And the newest 5 blocks should still be accessible
     for(int i=5;i<10;i++) {
-      assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true));
-      assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true));
-      assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true));
+      assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false));
+      assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false));
+      assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false));
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
index 4e5aff6fc51b..f230b1ea96bb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
@@ -213,7 +213,7 @@ private void readStoreFile(Path path) throws IOException {
           false, null);
         BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
           offset);
-        boolean isCached = cache.getBlock(blockCacheKey, true) != null;
+        boolean isCached = cache.getBlock(blockCacheKey, true, false) != null;
         boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
         if (shouldBeCached != isCached) {
           throw new AssertionError(

From 2e0c3e92d617ccdd77d9ad39938361d600f67505 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Nov 2012 07:00:20 +0000
Subject: [PATCH 0584/1540] CHANGES.txt, pom.xml in preparation for 0.94.3rc0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408622 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 2c08801097fe..fed92a2f5c8d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,73 @@
 HBase Change Log
 
+Release 0.94.3 - 11/12/2012
+Sub-task
+
+    [HBASE-4913] - Per-CF compaction Via the Shell
+    [HBASE-6305] - TestLocalHBaseCluster hangs with hadoop 2.0/0.23 builds.
+    [HBASE-6925] - Change socket write size from 8K to 64K for HBaseServer
+    [HBASE-6996] - HRegion.mutateRowsWithLocks should call checkResources/checkReadOnly
+    [HBASE-7076] - Add test that increment/append properly integrate with MVCC
+    [HBASE-7077] - Test for: CheckAndPut should properly read MVCC
+    [HBASE-7078] - Add a test that append is atomic
+
+Bug
+
+    [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments
+    [HBASE-6583] - Enhance Hbase load test tool to automatically create column families if not present
+    [HBASE-6665] - ROOT region should not be splitted even with META row as explicit split key
+    [HBASE-6700] - [replication] empty znodes created during queue failovers aren't deleted
+    [HBASE-6728] - [89-fb] prevent OOM possibility due to per connection responseQueue being unbounded
+    [HBASE-6733] - [0.92 UNIT TESTS] TestReplication.queueFailover occasionally fails [Part-2]
+    [HBASE-6796] - Backport HBASE-5547, Don't delete HFiles in backup mode.
+    [HBASE-6843] - loading lzo error when using coprocessor
+    [HBASE-6846] - BitComparator bug - ArrayIndexOutOfBoundsException
+    [HBASE-6904] - In the HBase shell, an error is thrown that states replication-related znodes already exist
+    [HBASE-6958] - TestAssignmentManager sometimes fails
+    [HBASE-6974] - Metric for blocked updates
+    [HBASE-6978] - Minor typo in ReplicationSource SocketTimeoutException error handling
+    [HBASE-7017] - Backport "[replication] The replication-executor should make sure the file that it is replicating is closed before declaring success on that file" to 0.94
+    [HBASE-7018] - Fix and Improve TableDescriptor caching for bulk assignment
+    [HBASE-7021] - Default to Hadoop 1.0.4 in 0.94 and add Hadoop 1.1 profile
+    [HBASE-7037] - ReplicationPeer logs at WARN level aborting server instead of at FATAL
+    [HBASE-7048] - Regionsplitter requires the hadoop config path to be in hbase classpath
+    [HBASE-7051] - CheckAndPut should properly read MVCC
+    [HBASE-7060] - Region load balancing by table does not handle the case where a table's region count is lower than the number of the RS in the cluster
+    [HBASE-7069] - HTable.batch does not have to be synchronized
+    [HBASE-7086] - Enhance ResourceChecker to log stack trace for potentially hanging threads
+    [HBASE-7095] - Cannot set 'lenAsVal' for KeyOnlyFilter from shell
+    [HBASE-7103] - Need to fail split if SPLIT znode is deleted even before the split is completed.
+    [HBASE-7143] - TestMetaMigrationRemovingHTD fails when used with Hadoop 0.23/2.x
+
+Improvement
+
+    [HBASE-5257] - Allow INCLUDE_AND_NEXT_COL in filters and use it in ColumnPaginationFilter
+    [HBASE-5314] - Gracefully rolling restart region servers in rolling-restart.sh
+    [HBASE-5898] - Consider double-checked locking for block cache lock
+    [HBASE-6852] - SchemaMetrics.updateOnCacheHit costs too much while full scanning a table with all of its fields
+    [HBASE-6942] - Endpoint implementation for bulk deletion of data
+    [HBASE-6951] - Allow the master info server to be started in a read only mode.
+    [HBASE-7073] - OperationMetrics needs to cache the value of hbase.metrics.exposeOperationTimes
+    [HBASE-7089] - Allow filter to be specified for Get from HBase shell
+    [HBASE-7097] - Log message in SecureServer.class uses wrong class name
+    [HBASE-7151] - Better log message for Per-CF compactions
+
+Task
+
+    [HBASE-6032] - Port HFileBlockIndex improvement from HBASE-5987
+    [HBASE-7016] - port HBASE-6518 'Bytes.toBytesBinary() incorrect trailing backslash escape' to 0.94
+    [HBASE-7020] - Backport HBASE-6336 Split point should not be equal to start row or end row
+    [HBASE-7038] - Port HBASE-5970 Improve the AssignmentManager#updateTimer and speed up handling opened event to 0.94
+    [HBASE-7040] - Port HBASE-5867 Improve Compaction Throttle Default to 0.94
+    [HBASE-7053] - port blockcache configurability (part of HBASE-6312, and HBASE-7033) to 0.94
+    [HBASE-7087] - Add to NOTICE.txt a note on jamon being MPL
+
+Test
+
+    [HBASE-5984] - TestLogRolling.testLogRollOnPipelineRestart failed with HADOOP 2.0.0
+    [HBASE-7142] - TestSplitLogManager#testDeadWorker may fail because of hard limit on the TimeoutMonitor's timeout period
+
+
 Release 0.94.2 - 10/08/2012
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index a1ff8ca70c33..8e139caef685 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.3-SNAPSHOT</version>
+  <version>0.94.3</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 5df1cd5efd2c54fe328abf15b3b24bcc89e13caa Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Nov 2012 19:25:31 +0000
Subject: [PATCH 0585/1540] HBASE-4913 Addendum: Fix TestShell

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1408904 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/admin.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 4cf696377c8f..85367bfec6c7 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -51,7 +51,7 @@ def flush(table_or_region_name)
 
     #----------------------------------------------------------------------------------------------
     # Requests a table or region or column family compaction
-    def compact(table_or_region_name, family)
+    def compact(table_or_region_name, family = nil)
       if family == nil
         @admin.compact(table_or_region_name)
       else
@@ -62,7 +62,7 @@ def compact(table_or_region_name, family)
 
     #----------------------------------------------------------------------------------------------
     # Requests a table or region or column family major compaction
-    def major_compact(table_or_region_name, family)
+    def major_compact(table_or_region_name, family = nil)
       if family == nil
         @admin.majorCompact(table_or_region_name)
       else

From 89a1731f12c4e70049527db739d67d605730edf4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 14 Nov 2012 21:37:09 +0000
Subject: [PATCH 0586/1540] HBASE-7042 Master Coprocessor Endpoint (Francis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1409431 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 19 ++++
 .../hadoop/hbase/client/coprocessor/Exec.java |  8 ++
 .../hbase/client/coprocessor/ExecResult.java  |  4 +
 .../hadoop/hbase/ipc/HMasterInterface.java    | 20 ++++
 .../hbase/ipc/MasterExecRPCInvoker.java       | 65 +++++++++++++
 .../apache/hadoop/hbase/master/HMaster.java   | 93 ++++++++++++++++++-
 .../hbase/master/MasterCoprocessorHost.java   |  7 ++
 .../hadoop/hbase/master/MasterServices.java   | 27 ++++++
 .../coprocessor/TestCoprocessorEndpoint.java  | 31 +++++++
 .../hbase/master/TestCatalogJanitor.java      | 11 +++
 10 files changed, 281 insertions(+), 4 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index f1b804bfde08..ba3be8e996a7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -22,6 +22,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.lang.reflect.Proxy;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.LinkedList;
@@ -55,8 +56,10 @@
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.ipc.MasterExecRPCInvoker;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.util.Addressing;
@@ -1888,4 +1891,20 @@ public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
     }
     return state;
   }
+
+  /**
+   * Creates and returns a proxy to the CoprocessorProtocol instance running in the
+   * master.
+   *
+   * @param protocol The class or interface defining the remote protocol
+   * @return A CoprocessorProtocol instance
+   */
+  public <T extends CoprocessorProtocol> T coprocessorProxy(
+      Class<T> protocol) {
+    return (T) Proxy.newProxyInstance(this.getClass().getClassLoader(),
+        new Class[]{protocol},
+        new MasterExecRPCInvoker(conf,
+            connection,
+            protocol));
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
index da1489d3b4b5..cafa8b050404 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
@@ -21,6 +21,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Row;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
@@ -60,6 +61,13 @@ public class Exec extends Invocation implements Row {
   public Exec() {
   }
 
+  public Exec(Configuration configuration,
+      Class<? extends CoprocessorProtocol> protocol,
+      Method method, Object[] parameters) {
+    this(configuration, HConstants.EMPTY_BYTE_ARRAY,
+        protocol, method, parameters);
+  }
+
   public Exec(Configuration configuration,
       byte[] row,
       Class<? extends CoprocessorProtocol> protocol,
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java
index 065b3f224635..e4812aff2d9e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.client.coprocessor;
 
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Classes;
@@ -53,6 +54,9 @@ public class ExecResult implements Writable {
   public ExecResult() {
   }
 
+  public ExecResult(Object value) {
+    this(HConstants.EMPTY_BYTE_ARRAY, value);
+  }
   public ExecResult(byte[] region, Object value) {
     this.regionName = region;
     this.value = value;
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index 645b30643c14..c17da9754ab4 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -26,6 +26,8 @@
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.security.TokenInfo;
 import org.apache.hadoop.hbase.security.KerberosInfo;
 import org.apache.hadoop.hbase.util.Pair;
@@ -266,4 +268,22 @@ public void unassign(final byte [] regionName, final boolean force)
    * @return array of HTableDescriptor
    */
   public HTableDescriptor[] getHTableDescriptors(List<String> tableNames);
+
+  /**
+   * Executes a single {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}
+   * method using the registered protocol handlers.
+   * {@link CoprocessorProtocol} implementations must be registered via the
+   * {@link org.apache.hadoop.hbase.master.MasterServices#registerProtocol(Class, CoprocessorProtocol)}
+   * method before they are available.
+   *
+   * @param call an {@code Exec} instance identifying the protocol, method name,
+   *     and parameters for the method invocation
+   * @return an {@code ExecResult} instance containing the region name of the
+   *     invocation and the return value
+   * @throws IOException if no registered protocol handler is found or an error
+   *     occurs during the invocation
+   * @see org.apache.hadoop.hbase.master.MasterServices#registerProtocol(Class, CoprocessorProtocol)
+   */
+  public ExecResult execCoprocessor(Exec call)
+      throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java b/src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java
new file mode 100644
index 000000000000..caaa430a825c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.ServerCallable;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+
+/**
+ * Backs a {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol} subclass proxy and forwards method
+ * invocations for server execution.  Note that internally this will issue a
+ * separate RPC call for each method invocation.
+ */
+public class MasterExecRPCInvoker implements InvocationHandler {
+  // LOG is NOT in hbase subpackage intentionally so that the default HBase
+  // DEBUG log level does NOT emit RPC-level logging.
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.MasterExecRPCInvoker");
+
+  private Configuration conf;
+  private final HConnection connection;
+  private Class<? extends CoprocessorProtocol> protocol;
+
+  public MasterExecRPCInvoker(Configuration conf,
+                              HConnection connection,
+                              Class<? extends CoprocessorProtocol> protocol) {
+    this.conf = conf;
+    this.connection = connection;
+    this.protocol = protocol;
+  }
+
+  @Override
+  public Object invoke(Object instance, final Method method, final Object[] args)
+      throws Throwable {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Call: "+method.getName()+", "+(args != null ? args.length : 0));
+    }
+    Exec exec = new Exec(conf, protocol, method, args);
+    ExecResult result = connection.getMaster().execCoprocessor(exec);
+    LOG.debug("Master Result is value="+result.getValue());
+    return result.getValue();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b476638dbcb0..0cf1eba2a84f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -22,13 +22,13 @@
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -41,6 +41,9 @@
 
 import javax.management.ObjectName;
 
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -67,9 +70,12 @@
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseServer;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
@@ -88,7 +94,6 @@
 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
-import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -223,6 +228,13 @@ public class HMaster extends HasThread
    */
   private ObjectName mxBean = null;
 
+  // Registered master protocol handlers
+  private ClassToInstanceMap<CoprocessorProtocol>
+      protocolHandlers = MutableClassToInstanceMap.create();
+
+  private Map<String, Class<? extends CoprocessorProtocol>>
+      protocolHandlerNames = Maps.newHashMap();
+
   /**
    * Initializes the HMaster. The steps are as follows:
    * <p>
@@ -1291,7 +1303,7 @@ public void disableTable(final byte [] tableName) throws IOException {
       cpHost.preDisableTable(tableName);
     }
     this.executorService.submit(new DisableTableHandler(this, tableName,
-      catalogTracker, assignmentManager, false));
+        catalogTracker, assignmentManager, false));
 
     if (cpHost != null) {
       cpHost.postDisableTable(tableName);
@@ -1552,6 +1564,7 @@ public ZooKeeperWatcher getZooKeeper() {
     return zooKeeper;
   }
 
+  @Override
   public MasterCoprocessorHost getCoprocessorHost() {
     return cpHost;
   }
@@ -1750,6 +1763,78 @@ public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
     return list.toArray(new HTableDescriptor [] {});
   }
 
+  @Override
+  public <T extends CoprocessorProtocol> boolean registerProtocol(
+      Class<T> protocol, T handler) {
+
+    /* No stacking of protocol handlers is currently allowed.  The
+     * first to claim wins!
+     */
+    if (protocolHandlers.containsKey(protocol)) {
+      LOG.error("Protocol "+protocol.getName()+
+          " already registered, rejecting request from "+
+          handler
+      );
+      return false;
+    }
+
+    protocolHandlers.putInstance(protocol, handler);
+    protocolHandlerNames.put(protocol.getName(), protocol);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Registered master protocol handler: protocol="+protocol.getName());
+    }
+    return true;
+  }
+
+  @Override
+  public ExecResult execCoprocessor(Exec call) throws IOException {
+    Class<? extends CoprocessorProtocol> protocol = call.getProtocol();
+    if (protocol == null) {
+      String protocolName = call.getProtocolName();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Received dynamic protocol exec call with protocolName " + protocolName);
+      }
+      // detect the actual protocol class
+      protocol  = protocolHandlerNames.get(protocolName);
+      if (protocol == null) {
+        throw new HBaseRPC.UnknownProtocolException(protocol,
+            "No matching handler for master protocol "+protocolName);
+      }
+    }
+    if (!protocolHandlers.containsKey(protocol)) {
+      throw new HBaseRPC.UnknownProtocolException(protocol,
+          "No matching handler for protocol ");
+    }
+
+    CoprocessorProtocol handler = protocolHandlers.getInstance(protocol);
+    Object value;
+
+    try {
+      Method method = protocol.getMethod(
+          call.getMethodName(), call.getParameterClasses());
+      method.setAccessible(true);
+
+      value = method.invoke(handler, call.getParameters());
+    } catch (InvocationTargetException e) {
+      Throwable target = e.getTargetException();
+      if (target instanceof IOException) {
+        throw (IOException)target;
+      }
+      IOException ioe = new IOException(target.toString());
+      ioe.setStackTrace(target.getStackTrace());
+      throw ioe;
+    } catch (Throwable e) {
+      if (!(e instanceof IOException)) {
+        LOG.error("Unexpected throwable object ", e);
+      }
+      IOException ioe = new IOException(e.toString());
+      ioe.setStackTrace(e.getStackTrace());
+      throw ioe;
+    }
+
+    return new ExecResult(value);
+  }
+
   /**
    * Get all table descriptors
    * @return All descriptors or null if none.
@@ -1837,4 +1922,4 @@ void registerMBean() {
   public HFileCleaner getHFileCleaner() {
     return this.hfileCleaner;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 4beafb21509e..68ab56644f6c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.coprocessor.*;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 
 import java.io.IOException;
 
@@ -69,6 +70,12 @@ public MasterServices getMasterServices() {
   public MasterEnvironment createEnvironment(final Class<?> implClass,
       final Coprocessor instance, final int priority, final int seq,
       final Configuration conf) {
+    for (Class c : implClass.getInterfaces()) {
+      if (CoprocessorProtocol.class.isAssignableFrom(c)) {
+        masterServices.registerProtocol(c, (CoprocessorProtocol)instance);
+        break;
+      }
+    }
     return new MasterEnvironment(implClass, instance, priority, seq, conf,
         masterServices);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 6d6d00914c2f..8be3c75ac20e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 
 /**
@@ -78,4 +79,30 @@ public void createTable(HTableDescriptor desc, byte [][] splitKeys)
    * @return true if master enables ServerShutdownHandler;
    */
   public boolean isServerShutdownHandlerEnabled();
+
+  /**
+   * @return returns the master coprocessor host
+   */
+  public MasterCoprocessorHost getCoprocessorHost();
+
+  /**
+   * Registers a new CoprocessorProtocol subclass and instance to
+   * be available for handling
+   * {@link HMaster#execCoprocessor(org.apache.hadoop.hbase.client.coprocessor.Exec)} calls.
+   *
+   * <p>
+   * Only a single protocol type/handler combination may be registered.
+   *
+   * After the first registration, subsequent calls with the same protocol type
+   * will fail with a return value of {@code false}.
+   * </p>
+   * @param protocol a {@code CoprocessorProtocol} subinterface defining the
+   * protocol methods
+   * @param handler an instance implementing the interface
+   * @param <T> the protocol type
+   * @return {@code true} if the registration was successful, {@code false}
+   * otherwise
+   */
+  public <T extends CoprocessorProtocol> boolean registerProtocol(
+      Class<T> protocol, T handler);
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
index e38cdcd4d7c0..1c596d5831c2 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
@@ -76,6 +77,8 @@ public static void setupBeforeClass() throws Exception {
     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
         "org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint",
         "org.apache.hadoop.hbase.coprocessor.GenericEndpoint");
+    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+        "org.apache.hadoop.hbase.coprocessor.GenericEndpoint");
 
     util.startMiniCluster(2);
     cluster = util.getMiniHBaseCluster();
@@ -136,6 +139,34 @@ public void testGeneric() throws Throwable {
     table.close();
   }
 
+  @Test
+  public void testMasterGeneric() throws Throwable {
+    HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
+    GenericProtocol protocol = admin.coprocessorProxy(GenericProtocol.class);
+    String workResult1 = protocol.doWork("foo");
+    assertEquals("foo", workResult1);
+    byte[] workResult2 = protocol.doWork(new byte[]{1});
+    assertArrayEquals(new byte[]{1}, workResult2);
+    byte workResult3 = protocol.doWork((byte)1);
+    assertEquals((byte)1, workResult3);
+    char workResult4 = protocol.doWork('c');
+    assertEquals('c', workResult4);
+    boolean workResult5 = protocol.doWork(true);
+    assertEquals(true, workResult5);
+    short workResult6 = protocol.doWork((short)1);
+    assertEquals((short)1, workResult6);
+    int workResult7 = protocol.doWork(5);
+    assertEquals(5, workResult7);
+    long workResult8 = protocol.doWork(5l);
+    assertEquals(5l, workResult8);
+    double workResult9 = protocol.doWork(6d);
+    assertEquals(6d, workResult9, 0.01);
+    float workResult10 = protocol.doWork(6f);
+    assertEquals(6f, workResult10, 0.01);
+    Text workResult11 = protocol.doWork(new Text("foo"));
+    assertEquals(new Text("foo"), workResult11);
+  }
+
   @Ignore @Test
   public void testAggregation() throws Throwable {
     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index f48e7a5daf6e..c83d4baa9693 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -58,6 +58,7 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -274,6 +275,16 @@ public void add(HTableDescriptor htd) throws IOException {
     public boolean isServerShutdownHandlerEnabled() {
       return true;
     }
+
+    @Override
+    public MasterCoprocessorHost getCoprocessorHost() {
+      return null;
+    }
+
+    @Override
+    public <T extends CoprocessorProtocol> boolean registerProtocol(Class<T> protocol, T handler) {
+      return false;
+    }
   }
 
   @Test

From 4aa7c7921e71affed9fe5e19988fa8b332258760 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 15 Nov 2012 05:29:34 +0000
Subject: [PATCH 0587/1540] HBASE-7165
 TestSplitLogManager.testUnassignedTimeout is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1409650 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/TestSplitLogManager.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 13bace757655..582a802a4ca7 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -386,7 +386,7 @@ public void testUnassignedTimeout() throws Exception {
         TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-    int to = 1000;
+    int to = 4000;
     conf.setInt("hbase.splitlog.manager.timeout", to);
     conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to);
     conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
@@ -411,10 +411,10 @@ public void testUnassignedTimeout() throws Exception {
     // since we have stopped heartbeating the owned node therefore it should
     // get resubmitted
     LOG.info("waiting for manager to resubmit the orphan task");
-    waitForCounter(tot_mgr_resubmit, 0, 1, to + 500);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
 
     // now all the nodes are unassigned. manager should post another rescan
-    waitForCounter(tot_mgr_resubmit_unassigned, 0, 1, 2 * to + 500);
+    waitForCounter(tot_mgr_resubmit_unassigned, 0, 1, 2 * to + to/2);
   }
 
   @Test

From 39900b9568e367d3c65dfac75b472b9fe8effb5b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 15 Nov 2012 22:23:13 +0000
Subject: [PATCH 0588/1540] HBASE-7166 TestSplitTransactionOnCluster tests are
 flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1410064 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestSplitTransactionOnCluster.java        | 40 ++++++++++++-------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 4ed8beafdf5c..e777081607d5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -145,8 +145,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       // Now try splitting and it should work.
       split(hri, server, regionCount);
       // Get daughters
-      List<HRegion> daughters = cluster.getRegions(tableName);
-      assertTrue(daughters.size() >= 2);
+      List<HRegion> daughters = checkAndGetDaughters(tableName);
       // Assert the ephemeral node is up in zk.
       String path = ZKAssign.getNodeName(t.getConnection().getZooKeeperWatcher(),
         hri.getEncodedName());
@@ -174,7 +173,12 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
         assertTrue(daughters.contains(r));
       }
       // Finally assert that the ephemeral SPLIT znode was cleaned up.
-      stats = t.getConnection().getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
+      for (int i=0; i<100; i++) {
+        // wait a bit (10s max) for the node to disappear
+        stats = t.getConnection().getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
+        if (stats == null) break;
+        Thread.sleep(100);
+      }
       LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
       assertTrue(stats == null);
     } finally {
@@ -228,8 +232,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       // Now try splitting and it should work.
       split(hri, server, regionCount);
       // Get daughters
-      List<HRegion> daughters = cluster.getRegions(tableName);
-      assertTrue(daughters.size() >= 2);
+      checkAndGetDaughters(tableName);
       // OK, so split happened after we cleared the blocking node.
     } finally {
       admin.setBalancerRunning(true, false);
@@ -271,8 +274,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       // Now split.
       split(hri, server, regionCount);
       // Get daughters
-      List<HRegion> daughters = cluster.getRegions(tableName);
-      assertTrue(daughters.size() >= 2);
+      List<HRegion> daughters = checkAndGetDaughters(tableName);
       // Remove one of the daughters from .META. to simulate failed insert of
       // daughter region up into .META.
       removeDaughterFromMeta(daughters.get(0).getRegionName());
@@ -328,8 +330,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       // Now split.
       split(hri, server, regionCount);
       // Get daughters
-      List<HRegion> daughters = cluster.getRegions(tableName);
-      assertTrue(daughters.size() >= 2);
+      List<HRegion> daughters = checkAndGetDaughters(tableName);
       // Now split one of the daughters.
       regionCount = server.getOnlineRegions().size();
       HRegionInfo daughter = daughters.get(0).getRegionInfo();
@@ -411,8 +412,7 @@ public void testMasterRestartWhenSplittingIsPartial()
       // Now try splitting and it should work.
       split(hri, server, regionCount);
       // Get daughters
-      List<HRegion> daughters = cluster.getRegions(tableName);
-      assertTrue(daughters.size() >= 2);
+      checkAndGetDaughters(tableName);
       // Assert the ephemeral node is up in zk.
       String path = ZKAssign.getNodeName(t.getConnection()
           .getZooKeeperWatcher(), hri.getEncodedName());
@@ -489,8 +489,7 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
       
       split(hri, server, regionCount);
       // Get daughters
-      List<HRegion> daughters = cluster.getRegions(tableName);
-      assertTrue(daughters.size() >= 2);
+      checkAndGetDaughters(tableName);
       // Assert the ephemeral node is up in zk.
       String path = ZKAssign.getNodeName(t.getConnection()
           .getZooKeeperWatcher(), hri.getEncodedName());
@@ -836,7 +835,20 @@ public boolean rollback(Server server, RegionServerServices services) throws IOE
     }
 
   }
-  
+
+  private List<HRegion> checkAndGetDaughters(byte[] tableName)
+      throws InterruptedException {    
+    List<HRegion> daughters = null;
+    // try up to 10s
+    for (int i=0; i<100; i++) {
+      daughters = cluster.getRegions(tableName);
+      if (daughters.size() >= 2) break;
+      Thread.sleep(100);
+    }
+    assertTrue(daughters.size() >= 2);
+    return daughters;
+  }
+
   private MockMasterWithoutCatalogJanitor abortAndWaitForMaster() 
   throws IOException, InterruptedException {
     cluster.abortMaster(0);

From 4e2de65accbcac1d18fe76522680dc0b73ae296d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 16 Nov 2012 05:25:11 +0000
Subject: [PATCH 0589/1540] HBASE-7177
 TestZooKeeperScanPolicyObserver.testScanPolicyObserver is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1410200 13f79535-47bb-0310-9956-ffa450edef68
---
 .../example/TestZooKeeperScanPolicyObserver.java            | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
index 91b4742082d1..ee05cb7da111 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
@@ -84,9 +84,13 @@ public void testScanPolicyObserver() throws Exception {
     // let's say test last backup was 1h ago
     // using plain ZK here, because RecoverableZooKeeper add extra encoding to the data
     zk.setData(ZooKeeperScanPolicyObserver.node, Bytes.toBytes(now - 3600*1000), -1);
-    
+
     LOG.debug("Set time: "+Bytes.toLong(Bytes.toBytes(now - 3600*1000)));
 
+    // sleep for 1s to give the ZK change a chance to reach the watcher in the observer.
+    // TODO: Better to wait for the data to be propagated
+    Thread.sleep(1000);
+
     long ts = now - 2000;
     Put p = new Put(R);
     p.add(F, Q, ts, Q);

From f646d081b5b3ba76b85b0155aaf266cd064f4442 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Fri, 16 Nov 2012 23:26:08 +0000
Subject: [PATCH 0590/1540] HBASE-7170 Allow HConnectionImplementation to
 reconnect to master multiple times

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1410622 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 11 +---
 .../hbase/client/TestFromClientSide.java      | 60 ++++++++++++++++---
 2 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 37f4c1bc761e..e3f483ae0600 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -495,7 +495,6 @@ static class HConnectionImplementation implements HConnection, Closeable {
     private volatile boolean closed;
     private volatile boolean aborted;
     private volatile HMasterInterface master;
-    private volatile boolean masterChecked;
     // ZooKeeper reference
     private volatile ZooKeeperWatcher zooKeeper;
     // ZooKeeper-based master address tracker
@@ -574,7 +573,6 @@ public HConnectionImplementation(Configuration conf, boolean managed)
           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
 
       this.master = null;
-      this.masterChecked = false;
     }
 
     private synchronized void ensureZookeeperTrackers()
@@ -662,9 +660,7 @@ public HMasterInterface getMaster()
         this.master = null;
 
         for (int tries = 0;
-          !this.closed &&
-          !this.masterChecked && this.master == null &&
-          tries < numRetries;
+          !this.closed && this.master == null && tries < numRetries;
         tries++) {
 
           try {
@@ -703,10 +699,6 @@ public HMasterInterface getMaster()
             throw new RuntimeException("Thread was interrupted while trying to connect to master.");
           }
         }
-        // Avoid re-checking in the future if this is a managed HConnection,
-        // even if we failed to acquire a master.
-        // (this is to retain the existing behavior before HBASE-5058)
-        this.masterChecked = managed;
 
         if (this.master == null) {
           if (sn == null) {
@@ -1756,7 +1748,6 @@ void close(boolean stopProxy) {
           HBaseRPC.stopProxy(master);
         }
         master = null;
-        masterChecked = false;
       }
       if (stopProxy) {
         for (HRegionInterface i : servers.values()) {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 3e62a6bc6dc2..6ab59d9d8aa7 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@@ -3888,6 +3889,24 @@ public void testListTables() throws IOException, InterruptedException {
     }
   }
 
+  /**
+   * creates an HTable for tableName using an unmanaged HConnection.
+   *
+   * @param tableName - table to create
+   * @return the created HTable object
+   * @throws IOException
+   */
+  HTable createUnmangedHConnectionHTable(final byte [] tableName) throws IOException {
+    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
+    HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
+    ExecutorService pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE,
+      60, TimeUnit.SECONDS,
+      new SynchronousQueue<Runnable>(),
+      Threads.newDaemonThreadFactory("test-from-client-table"));
+    ((ThreadPoolExecutor)pool).allowCoreThreadTimeOut(true);
+    return new HTable(tableName, conn, pool);
+  }
+
   /**
    * simple test that just executes parts of the client
    * API that accept a pre-created HConnction instance
@@ -3897,18 +3916,41 @@ public void testListTables() throws IOException, InterruptedException {
   @Test
   public void testUnmanagedHConnection() throws IOException {
     final byte[] tableName = Bytes.toBytes("testUnmanagedHConnection");
-    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-    HConnection conn = HConnectionManager.createConnection(TEST_UTIL
-        .getConfiguration());
-    ExecutorService pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE,
-        60, TimeUnit.SECONDS,
-        new SynchronousQueue<Runnable>(),
-        Threads.newDaemonThreadFactory("test-from-client-table"));
-    ((ThreadPoolExecutor)pool).allowCoreThreadTimeOut(true);
-    HTable t = new HTable(tableName, conn, pool);
+    HTable t = createUnmangedHConnectionHTable(tableName);
+    HBaseAdmin ha = new HBaseAdmin(t.getConnection());
+    assertTrue(ha.tableExists(tableName));
+    assertTrue(t.get(new Get(ROW)).isEmpty());
+  }
+
+  /**
+   * test of that unmanaged HConnections are able to reconnect
+   * properly (see HBASE-5058)
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testUnmanagedHConnectionReconnect() throws Exception {
+    final byte[] tableName = Bytes.toBytes("testUnmanagedHConnectionReconnect");
+    HTable t = createUnmangedHConnectionHTable(tableName);
+    HConnection conn = t.getConnection();
     HBaseAdmin ha = new HBaseAdmin(conn);
     assertTrue(ha.tableExists(tableName));
     assertTrue(t.get(new Get(ROW)).isEmpty());
+
+    // stop the master
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    cluster.stopMaster(0, false);
+    cluster.waitOnMaster(0);
+
+    // start up a new master
+    cluster.startMaster();
+    assertTrue(cluster.waitForActiveAndReadyMaster());
+
+    // test that the same unmanaged connection works with a new
+    // HBaseAdmin and can connect to the new master;
+    HBaseAdmin newAdmin = new HBaseAdmin(conn);
+    assertTrue(newAdmin.tableExists(tableName));
+    assert(newAdmin.getClusterStatus().getServersSize() == SLAVES);
   }
 
   @Test

From 610111330863bebdd9220a25b065dc20beb0855a Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 21 Nov 2012 07:35:55 +0000
Subject: [PATCH 0591/1540] HBASE-7009 Port HBaseCluster interface/tests to
 0.94 (HBASE-6241)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1411998 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       |   7 +-
 .../hadoop/hbase/util/AbstractHBaseTool.java  |   7 +-
 .../org/apache/hadoop/hbase/util/Threads.java |  22 +-
 .../apache/hadoop/hbase/ClusterManager.java   | 132 ++++
 .../hadoop/hbase/DistributedHBaseCluster.java | 270 +++++++++
 .../org/apache/hadoop/hbase/HBaseCluster.java | 264 ++++++++
 .../hadoop/hbase/HBaseClusterManager.java     | 212 +++++++
 .../hadoop/hbase/HBaseTestingUtility.java     |  76 ++-
 ...egrationTestDataIngestWithChaosMonkey.java | 133 +++++
 .../hbase/IntegrationTestingUtility.java      | 130 ++++
 .../apache/hadoop/hbase/IntegrationTests.java |  39 ++
 .../hadoop/hbase/IntegrationTestsDriver.java  |  73 +++
 .../org/apache/hadoop/hbase/LargeTests.java   |   1 +
 .../org/apache/hadoop/hbase/MediumTests.java  |   1 +
 .../apache/hadoop/hbase/MiniHBaseCluster.java | 143 ++++-
 .../org/apache/hadoop/hbase/SmallTests.java   |   1 +
 .../hadoop/hbase/TestCheckTestClasses.java    | 186 ++++++
 .../hadoop/hbase/ipc/TestPBOnWritableRpc.java |   3 +
 .../hadoop/hbase/master/TestMXBean.java       |   3 +
 .../hbase/master/TestMasterFailover.java      |  23 +-
 .../hbase/metrics/TestExactCounterMetric.java |   3 +
 .../TestExponentiallyDecayingSample.java      |   5 +
 .../hbase/metrics/TestMetricsHistogram.java   |   3 +
 .../TestEndToEndSplitTransaction.java         |  17 +-
 .../hadoop/hbase/regionserver/TestMXBean.java |   3 +
 .../TestRSKilledWhenMasterInitializing.java   |   3 +
 .../apache/hadoop/hbase/util/ChaosMonkey.java | 563 ++++++++++++++++++
 .../hadoop/hbase/util/LoadTestTool.java       |  12 +-
 .../hadoop/hbase/util/RestartMetaTest.java    |   3 +-
 .../hbase/util/StoppableImplementation.java   |  40 ++
 30 files changed, 2307 insertions(+), 71 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/ClusterManager.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/HBaseCluster.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTests.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java

diff --git a/pom.xml b/pom.xml
index 8e139caef685..b81823527d21 100644
--- a/pom.xml
+++ b/pom.xml
@@ -403,7 +403,6 @@
             <excludes>
               <exlude>${unittest.include}</exlude>
               <exclude>**/*$*</exclude>
-              <exclude>${test.exclude.pattern}</exclude>
             </excludes>
             <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
             <environmentVariables>
@@ -764,6 +763,12 @@
           <configuration>
             <skip>false</skip>
             <forkMode>always</forkMode>
+            <!-- TODO: failsafe does timeout, but verify does not fail the build because of the timeout.
+                 I believe it is a failsafe bug, we may consider using surefire -->
+            <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
+            <argLine>-enableassertions -Xmx1900m
+              -Djava.security.egd=file:/dev/./urandom</argLine>
+            <testFailureIgnore>false</testFailureIgnore>
           </configuration>
       </plugin>
       <plugin>
diff --git a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
index 779cdc1fd547..814dd740ac50 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -63,7 +63,7 @@ public abstract class AbstractHBaseTool implements Tool {
   protected abstract void processOptions(CommandLine cmd);
 
   /** The "main function" of the tool */
-  protected abstract void doWork() throws Exception;
+  protected abstract int doWork() throws Exception;
 
   @Override
   public Configuration getConf() {
@@ -99,13 +99,14 @@ public final int run(String[] args) throws Exception {
 
     processOptions(cmd);
 
+    int ret = EXIT_FAILURE;
     try {
-      doWork();
+      ret = doWork();
     } catch (Exception e) {
       LOG.error("Error running command-line tool", e);
       return EXIT_FAILURE;
     }
-    return EXIT_SUCCESS;
+    return ret;
   }
 
   private boolean sanityCheckOptions(CommandLine cmd) {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index 2d581f395c38..a5a4d1311916 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -126,7 +126,7 @@ public static void threadDumpingIsAlive(final Thread t)
   /**
    * @param millis How long to sleep for in milliseconds.
    */
-  public static void sleep(int millis) {
+  public static void sleep(long millis) {
     try {
       Thread.sleep(millis);
     } catch (InterruptedException e) {
@@ -158,15 +158,15 @@ public static void sleepWithoutInterrupt(final long msToWait) {
   }
 
   /**
-   * Create a new CachedThreadPool with a bounded number as the maximum 
+   * Create a new CachedThreadPool with a bounded number as the maximum
    * thread size in the pool.
-   * 
+   *
    * @param maxCachedThread the maximum thread could be created in the pool
    * @param timeout the maximum time to wait
    * @param unit the time unit of the timeout argument
    * @param threadFactory the factory to use when creating new threads
-   * @return threadPoolExecutor the cachedThreadPool with a bounded number 
-   * as the maximum thread size in the pool. 
+   * @return threadPoolExecutor the cachedThreadPool with a bounded number
+   * as the maximum thread size in the pool.
    */
   public static ThreadPoolExecutor getBoundedCachedThreadPool(
       int maxCachedThread, long timeout, TimeUnit unit,
@@ -178,12 +178,13 @@ public static ThreadPoolExecutor getBoundedCachedThreadPool(
     boundedCachedThreadPool.allowCoreThreadTimeOut(true);
     return boundedCachedThreadPool;
   }
-  
-  
+
+
   /**
-   * Returns a {@link java.util.concurrent.ThreadFactory} that names each created thread uniquely,
-   * with a common prefix.
-   * @param prefix The prefix of every created Thread's name
+   * Returns a {@link java.util.concurrent.ThreadFactory} that names each
+   * created thread uniquely, with a common prefix.
+   *
+   * @param prefix  The prefix of every created Thread's name
    * @return a {@link java.util.concurrent.ThreadFactory} that names threads
    */
   public static ThreadFactory getNamedThreadFactory(final String prefix) {
@@ -205,6 +206,7 @@ public Thread newThread(Runnable r) {
     };
   }
 
+
   /**
    * Get a named {@link ThreadFactory} that just builds daemon threads
    * @param prefix name prefix for all threads created from the factory
diff --git a/src/test/java/org/apache/hadoop/hbase/ClusterManager.java b/src/test/java/org/apache/hadoop/hbase/ClusterManager.java
new file mode 100644
index 000000000000..5a5b010959b2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/ClusterManager.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configured;
+
+
+/**
+ * ClusterManager is an api to manage servers in a distributed environment. It provides services
+ * for starting / stopping / killing Hadoop/HBase daemons. Concrete implementations provide actual
+ * functionality for carrying out deployment-specific tasks.
+ */
+@InterfaceAudience.Private
+public abstract class ClusterManager extends Configured {
+  protected static final Log LOG = LogFactory.getLog(ClusterManager.class);
+
+  private static final String SIGKILL = "SIGKILL";
+  private static final String SIGSTOP = "SIGSTOP";
+  private static final String SIGCONT = "SIGCONT";
+
+  public ClusterManager() {
+  }
+
+  /**
+   * Type of the service daemon
+   */
+  public static enum ServiceType {
+    HADOOP_NAMENODE("namenode"),
+    HADOOP_DATANODE("datanode"),
+    HADOOP_JOBTRACKER("jobtracker"),
+    HADOOP_TASKTRACKER("tasktracker"),
+    HBASE_MASTER("master"),
+    HBASE_REGIONSERVER("regionserver");
+
+    private String name;
+
+    ServiceType(String name) {
+      this.name = name;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    @Override
+    public String toString() {
+      return getName();
+    }
+  }
+
+  /**
+   * Start the service on the given host
+   */
+  public abstract void start(ServiceType service, String hostname) throws IOException;
+
+  /**
+   * Stop the service on the given host
+   */
+  public abstract void stop(ServiceType service, String hostname) throws IOException;
+
+  /**
+   * Restart the service on the given host
+   */
+  public abstract void restart(ServiceType service, String hostname) throws IOException;
+
+  /**
+   * Send the given posix signal to the service
+   */
+  public abstract void signal(ServiceType service, String signal,
+      String hostname) throws IOException;
+
+  /**
+   * Kill the service running on given host
+   */
+  public void kill(ServiceType service, String hostname) throws IOException {
+    signal(service, SIGKILL, hostname);
+  }
+
+  /**
+   * Suspend the service running on given host
+   */
+  public void suspend(ServiceType service, String hostname) throws IOException {
+    signal(service, SIGSTOP, hostname);
+  }
+
+  /**
+   * Resume the services running on given hosts
+   */
+  public void resume(ServiceType service, String hostname) throws IOException {
+    signal(service, SIGCONT, hostname);
+  }
+
+  /**
+   * Returns whether the service is running on the remote host. This only checks whether the
+   * service still has a pid.
+   */
+  public abstract boolean isRunning(ServiceType service, String hostname) throws IOException;
+
+  /* TODO: further API ideas:
+   *
+   * //return services running on host:
+   * ServiceType[] getRunningServicesOnHost(String hostname);
+   *
+   * //return which services can be run on host (for example, to query whether hmaster can run on this host)
+   * ServiceType[] getRunnableServicesOnHost(String hostname);
+   *
+   * //return which hosts can run this service
+   * String[] getRunnableHostsForService(ServiceType service);
+   */
+
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
new file mode 100644
index 000000000000..53d4ce3e3f0b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterManager.ServiceType;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.util.Threads;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Manages the interactions with an already deployed distributed cluster (as opposed to
+ * a pseudo-distributed, or mini/local cluster). This is used by integration and system tests.
+ */
+@InterfaceAudience.Private
+public class DistributedHBaseCluster extends HBaseCluster {
+
+  private HBaseAdmin admin;
+
+  private ClusterManager clusterManager;
+
+  public DistributedHBaseCluster(Configuration conf, ClusterManager clusterManager)
+      throws IOException {
+    super(conf);
+    this.clusterManager = clusterManager;
+    this.admin = new HBaseAdmin(conf);
+    this.initialClusterStatus = getClusterStatus();
+  }
+
+  public void setClusterManager(ClusterManager clusterManager) {
+    this.clusterManager = clusterManager;
+  }
+
+  public ClusterManager getClusterManager() {
+    return clusterManager;
+  }
+
+  /**
+   * Returns a ClusterStatus for this HBase cluster
+   * @throws IOException
+   */
+  @Override
+  public ClusterStatus getClusterStatus() throws IOException {
+    return admin.getClusterStatus();
+  }
+
+  @Override
+  public ClusterStatus getInitialClusterStatus() throws IOException {
+    return initialClusterStatus;
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (this.admin != null) {
+      admin.close();
+    }
+  }
+
+  @Override
+  public void startRegionServer(String hostname) throws IOException {
+    LOG.info("Starting RS on: " + hostname);
+    clusterManager.start(ServiceType.HBASE_REGIONSERVER, hostname);
+  }
+
+  @Override
+  public void killRegionServer(ServerName serverName) throws IOException {
+    LOG.info("Aborting RS: " + serverName.getServerName());
+    clusterManager.kill(ServiceType.HBASE_REGIONSERVER, serverName.getHostname());
+  }
+
+  @Override
+  public void stopRegionServer(ServerName serverName) throws IOException {
+    LOG.info("Stopping RS: " + serverName.getServerName());
+    clusterManager.stop(ServiceType.HBASE_REGIONSERVER, serverName.getHostname());
+  }
+
+  @Override
+  public void waitForRegionServerToStop(ServerName serverName, long timeout) throws IOException {
+    waitForServiceToStop(ServiceType.HBASE_REGIONSERVER, serverName, timeout);
+  }
+
+  private void waitForServiceToStop(ServiceType service, ServerName serverName, long timeout)
+    throws IOException {
+    LOG.info("Waiting service:" + service + " to stop: " + serverName.getServerName());
+    long start = System.currentTimeMillis();
+
+    while ((System.currentTimeMillis() - start) < timeout) {
+      if (!clusterManager.isRunning(service, serverName.getHostname())) {
+        return;
+      }
+      Threads.sleep(1000);
+    }
+    throw new IOException("did timeout waiting for service to stop:" + serverName);
+  }
+
+  @Override
+  public HMasterInterface getMasterAdmin() throws IOException {
+    HConnection conn = HConnectionManager.getConnection(conf);
+    return conn.getMaster();
+  }
+
+  @Override
+  public void startMaster(String hostname) throws IOException {
+    LOG.info("Starting Master on: " + hostname);
+    clusterManager.start(ServiceType.HBASE_MASTER, hostname);
+  }
+
+  @Override
+  public void killMaster(ServerName serverName) throws IOException {
+    LOG.info("Aborting Master: " + serverName.getServerName());
+    clusterManager.kill(ServiceType.HBASE_MASTER, serverName.getHostname());
+  }
+
+  @Override
+  public void stopMaster(ServerName serverName) throws IOException {
+    LOG.info("Stopping Master: " + serverName.getServerName());
+    clusterManager.stop(ServiceType.HBASE_MASTER, serverName.getHostname());
+  }
+
+  @Override
+  public void waitForMasterToStop(ServerName serverName, long timeout) throws IOException {
+    waitForServiceToStop(ServiceType.HBASE_MASTER, serverName, timeout);
+  }
+
+  @Override
+  public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
+    long start = System.currentTimeMillis();
+    while (System.currentTimeMillis() - start < timeout) {
+      try {
+        getMasterAdmin();
+        return true;
+      } catch (MasterNotRunningException m) {
+        LOG.warn("Master not started yet " + m);
+      } catch (ZooKeeperConnectionException e) {
+        LOG.warn("Failed to connect to ZK " + e);
+      }
+      Threads.sleep(1000);
+    }
+    return false;
+  }
+
+  @Override
+  public ServerName getServerHoldingRegion(byte[] regionName) throws IOException {
+    HConnection connection = admin.getConnection();
+    HRegionLocation regionLoc = connection.locateRegion(regionName);
+    if (regionLoc == null) {
+      return null;
+    }
+
+    org.apache.hadoop.hbase.HServerInfo sn
+		= connection.getHRegionConnection(regionLoc.getHostname(), regionLoc.getPort()).getHServerInfo();
+
+    return new ServerName(sn.getServerAddress().getHostname(), sn.getServerAddress().getPort(), sn.getStartCode());
+  }
+
+  @Override
+  public void waitUntilShutDown() {
+    //Simply wait for a few seconds for now (after issuing serverManager.kill
+    throw new RuntimeException("Not implemented yet");
+  }
+
+  @Override
+  public void shutdown() throws IOException {
+    //not sure we want this
+    throw new RuntimeException("Not implemented yet");
+  }
+
+  @Override
+  public boolean isDistributedCluster() {
+    return true;
+  }
+
+  @Override
+  public void restoreClusterStatus(ClusterStatus initial) throws IOException {
+    //TODO: caution: not tested throughly
+    ClusterStatus current = getClusterStatus();
+
+    //restore masters
+
+    //check whether current master has changed
+    if (!ServerName.isSameHostnameAndPort(initial.getMaster(), current.getMaster())) {
+      //master has changed, we would like to undo this.
+      //1. Kill the current backups
+      //2. Stop current master
+      //3. Start a master at the initial hostname (if not already running as backup)
+      //4. Start backup masters
+      boolean foundOldMaster = false;
+      for (ServerName currentBackup : current.getBackupMasters()) {
+        if (!ServerName.isSameHostnameAndPort(currentBackup, initial.getMaster())) {
+          stopMaster(currentBackup);
+        } else {
+          foundOldMaster = true;
+        }
+      }
+      stopMaster(current.getMaster());
+      if (foundOldMaster) { //if initial master is not running as a backup
+        startMaster(initial.getMaster().getHostname());
+      }
+      waitForActiveAndReadyMaster(); //wait so that active master takes over
+
+      //start backup masters
+      for (ServerName backup : initial.getBackupMasters()) {
+        //these are not started in backup mode, but we should already have an active master
+        startMaster(backup.getHostname());
+      }
+    } else {
+      //current master has not changed, match up backup masters
+      HashMap<String, ServerName> initialBackups = new HashMap<String, ServerName>();
+      HashMap<String, ServerName> currentBackups = new HashMap<String, ServerName>();
+
+      for (ServerName server : initial.getBackupMasters()) {
+        initialBackups.put(server.getHostname(), server);
+      }
+      for (ServerName server : current.getBackupMasters()) {
+        currentBackups.put(server.getHostname(), server);
+      }
+
+      for (String hostname : Sets.difference(initialBackups.keySet(), currentBackups.keySet())) {
+        startMaster(hostname);
+      }
+
+      for (String hostname : Sets.difference(currentBackups.keySet(), initialBackups.keySet())) {
+        stopMaster(currentBackups.get(hostname));
+      }
+    }
+
+    //restore region servers
+    HashMap<String, ServerName> initialServers = new HashMap<String, ServerName>();
+    HashMap<String, ServerName> currentServers = new HashMap<String, ServerName>();
+
+    for (ServerName server : initial.getServers()) {
+      initialServers.put(server.getHostname(), server);
+    }
+    for (ServerName server : current.getServers()) {
+      currentServers.put(server.getHostname(), server);
+    }
+
+    for (String hostname : Sets.difference(initialServers.keySet(), currentServers.keySet())) {
+      startRegionServer(hostname);
+    }
+
+    for (String hostname : Sets.difference(currentServers.keySet(), initialServers.keySet())) {
+      stopRegionServer(currentServers.get(hostname));
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java
new file mode 100644
index 000000000000..4a5e39420b05
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+
+/**
+ * This class defines methods that can help with managing HBase clusters
+ * from unit tests and system tests. There are 3 types of cluster deployments:
+ * <ul>
+ * <li><b>MiniHBaseCluster:</b> each server is run in the same JVM in separate threads,
+ * used by unit tests</li>
+ * <li><b>DistributedHBaseCluster:</b> the cluster is pre-deployed, system and integration tests can
+ * interact with the cluster. </li>
+ * <li><b>ProcessBasedLocalHBaseCluster:</b> each server is deployed locally but in separate
+ * JVMs. </li>
+ * </ul>
+ * <p>
+ * HBaseCluster unifies the way tests interact with the cluster, so that the same test can
+ * be run against a mini-cluster during unit test execution, or a distributed cluster having
+ * tens/hundreds of nodes during execution of integration tests.
+ *
+ * <p>
+ * HBaseCluster exposes client-side public interfaces to tests, so that tests does not assume
+ * running in a particular mode. Not all the tests are suitable to be run on an actual cluster,
+ * and some tests will still need to mock stuff and introspect internal state. For those use
+ * cases from unit tests, or if more control is needed, you can use the subclasses directly.
+ * In that sense, this class does not abstract away <strong>every</strong> interface that
+ * MiniHBaseCluster or DistributedHBaseCluster provide.
+ */
+@InterfaceAudience.Private
+public abstract class HBaseCluster implements Closeable, Configurable {
+  static final Log LOG = LogFactory.getLog(HBaseCluster.class.getName());
+  protected Configuration conf;
+
+  /** the status of the cluster before we begin */
+  protected ClusterStatus initialClusterStatus;
+
+  /**
+   * Construct an HBaseCluster
+   * @param conf Configuration to be used for cluster
+   */
+  public HBaseCluster(Configuration conf) {
+    setConf(conf);
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Returns a ClusterStatus for this HBase cluster.
+   * @see #getInitialClusterStatus()
+   */
+  public abstract ClusterStatus getClusterStatus() throws IOException;
+
+  /**
+   * Returns a ClusterStatus for this HBase cluster as observed at the
+   * starting of the HBaseCluster
+   */
+  public ClusterStatus getInitialClusterStatus() throws IOException {
+    return initialClusterStatus;
+  }
+
+  /**
+   * Returns an {@link HmasterInterface} to the active master
+   */
+  public abstract HMasterInterface getMasterAdmin()
+      throws IOException;
+
+  /**
+   * Starts a new region server on the given hostname or if this is a mini/local cluster,
+   * starts a region server locally.
+   * @param hostname the hostname to start the regionserver on
+   * @throws IOException if something goes wrong
+   */
+  public abstract void startRegionServer(String hostname) throws IOException;
+
+  /**
+   * Kills the region server process if this is a distributed cluster, otherwise
+   * this causes the region server to exit doing basic clean up only.
+   * @throws IOException if something goes wrong
+   */
+  public abstract void killRegionServer(ServerName serverName) throws IOException;
+
+  /**
+   * Stops the given region server, by attempting a gradual stop.
+   * @return whether the operation finished with success
+   * @throws IOException if something goes wrong
+   */
+  public abstract void stopRegionServer(ServerName serverName) throws IOException;
+
+  /**
+   * Wait for the specified region server to join the cluster
+   * @return whether the operation finished with success
+   * @throws IOException if something goes wrong or timeout occurs
+   */
+  public void waitForRegionServerToStart(String hostname, long timeout)
+      throws IOException {
+    long start = System.currentTimeMillis();
+    while ((System.currentTimeMillis() - start) < timeout) {
+      for (ServerName server : getClusterStatus().getServers()) {
+        if (server.getHostname().equals(hostname)) {
+          return;
+        }
+      }
+      Threads.sleep(100);
+    }
+    throw new IOException("did timeout waiting for region server to start:" + hostname);
+  }
+
+  /**
+   * Wait for the specified region server to stop the thread / process.
+   * @return whether the operation finished with success
+   * @throws IOException if something goes wrong or timeout occurs
+   */
+  public abstract void waitForRegionServerToStop(ServerName serverName, long timeout)
+      throws IOException;
+
+  /**
+   * Starts a new master on the given hostname or if this is a mini/local cluster,
+   * starts a master locally.
+   * @param hostname the hostname to start the master on
+   * @return whether the operation finished with success
+   * @throws IOException if something goes wrong
+   */
+  public abstract void startMaster(String hostname) throws IOException;
+
+  /**
+   * Kills the master process if this is a distributed cluster, otherwise,
+   * this causes master to exit doing basic clean up only.
+   * @throws IOException if something goes wrong
+   */
+  public abstract void killMaster(ServerName serverName) throws IOException;
+
+  /**
+   * Stops the given master, by attempting a gradual stop.
+   * @throws IOException if something goes wrong
+   */
+  public abstract void stopMaster(ServerName serverName) throws IOException;
+
+  /**
+   * Wait for the specified master to stop the thread / process.
+   * @throws IOException if something goes wrong or timeout occurs
+   */
+  public abstract void waitForMasterToStop(ServerName serverName, long timeout)
+      throws IOException;
+
+  /**
+   * Blocks until there is an active master and that master has completed
+   * initialization.
+   *
+   * @return true if an active master becomes available.  false if there are no
+   *         masters left.
+   * @throws IOException if something goes wrong or timeout occurs
+   */
+  public boolean waitForActiveAndReadyMaster()
+      throws IOException {
+    return waitForActiveAndReadyMaster(Long.MAX_VALUE);
+  }
+
+  /**
+   * Blocks until there is an active master and that master has completed
+   * initialization.
+   * @param timeout the timeout limit in ms
+   * @return true if an active master becomes available.  false if there are no
+   *         masters left.
+   */
+  public abstract boolean waitForActiveAndReadyMaster(long timeout)
+      throws IOException;
+
+  /**
+   * Wait for HBase Cluster to shut down.
+   */
+  public abstract void waitUntilShutDown() throws IOException;
+
+  /**
+   * Shut down the HBase cluster
+   */
+  public abstract void shutdown() throws IOException;
+
+  /**
+   * Restores the cluster to it's initial state if this is a real cluster,
+   * otherwise does nothing.
+   */
+  public void restoreInitialStatus() throws IOException {
+    restoreClusterStatus(getInitialClusterStatus());
+  }
+
+  /**
+   * Restores the cluster to given state if this is a real cluster,
+   * otherwise does nothing.
+   */
+  public void restoreClusterStatus(ClusterStatus desiredStatus) throws IOException {
+  }
+
+  /**
+   * Get the ServerName of region server serving ROOT region
+   */
+  public ServerName getServerHoldingRoot() throws IOException {
+    return getServerHoldingRegion(HRegionInfo.ROOT_REGIONINFO.getRegionName());
+  }
+
+  /**
+   * Get the ServerName of region server serving the first META region
+   */
+  public ServerName getServerHoldingMeta() throws IOException {
+    return getServerHoldingRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
+  }
+
+  /**
+   * Get the ServerName of region server serving the specified region
+   * @param regionName Name of the region in bytes
+   * @return ServerName that hosts the region or null
+   */
+  public abstract ServerName getServerHoldingRegion(byte[] regionName) throws IOException;
+
+  /**
+   * @return whether we are interacting with a distributed cluster as opposed to an
+   * in-process mini/local cluster.
+   */
+  public boolean isDistributedCluster() {
+    return false;
+  }
+
+  /**
+   * Closes all the resources held open for this cluster. Note that this call does not shutdown
+   * the cluster.
+   * @see #shutdown()
+   */
+  @Override
+  public abstract void close() throws IOException;
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
new file mode 100644
index 000000000000..b432e72a9128
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HBaseClusterManager.CommandProvider.Operation;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.util.Shell;
+
+/**
+ * A default cluster manager for HBase. Uses SSH, and hbase shell scripts
+ * to manage the cluster. Assumes Unix-like commands are available like 'ps',
+ * 'kill', etc. Also assumes the user running the test has enough "power" to start & stop
+ * servers on the remote machines (for example, the test user could be the same user as the
+ * user the daemon isrunning as)
+ */
+@InterfaceAudience.Private
+public class HBaseClusterManager extends ClusterManager {
+
+  /**
+   * Executes commands over SSH
+   */
+  static class RemoteShell extends Shell.ShellCommandExecutor {
+
+    private String hostname;
+
+    private String sshCmd = "/usr/bin/ssh";
+    private String sshOptions = System.getenv("HBASE_SSH_OPTS"); //from conf/hbase-env.sh
+
+    public RemoteShell(String hostname, String[] execString, File dir, Map<String, String> env,
+        long timeout) {
+      super(execString, dir, env, timeout);
+      this.hostname = hostname;
+    }
+
+    public RemoteShell(String hostname, String[] execString, File dir, Map<String, String> env) {
+      super(execString, dir, env);
+      this.hostname = hostname;
+    }
+
+    public RemoteShell(String hostname, String[] execString, File dir) {
+      super(execString, dir);
+      this.hostname = hostname;
+    }
+
+    public RemoteShell(String hostname, String[] execString) {
+      super(execString);
+      this.hostname = hostname;
+    }
+
+    public String[] getExecString() {
+      return new String[] {
+          "bash", "-c",
+          StringUtils.join(new String[] { sshCmd,
+              sshOptions == null ? "" : sshOptions,
+              hostname,
+              "\"" + StringUtils.join(super.getExecString(), " ") + "\""
+          }, " ")};
+    }
+
+    @Override
+    public void execute() throws IOException {
+      super.execute();
+    }
+
+    public void setSshCmd(String sshCmd) {
+      this.sshCmd = sshCmd;
+    }
+
+    public void setSshOptions(String sshOptions) {
+      this.sshOptions = sshOptions;
+    }
+
+    public String getSshCmd() {
+      return sshCmd;
+    }
+
+    public String getSshOptions() {
+      return sshOptions;
+    }
+  }
+
+  /**
+   * Provides command strings for services to be executed by Shell. CommandProviders are
+   * pluggable, and different deployments(windows, bigtop, etc) can be managed by
+   * plugging-in custom CommandProvider's or ClusterManager's.
+   */
+  static abstract class CommandProvider {
+
+    enum Operation {
+      START, STOP, RESTART
+    }
+
+    public abstract String getCommand(ServiceType service, Operation op);
+
+    public String isRunningCommand(ServiceType service) {
+      return findPidCommand(service);
+    }
+
+    protected String findPidCommand(ServiceType service) {
+      return String.format("ps aux | grep %s | grep -v grep | tr -s ' ' | cut -d ' ' -f2",
+          service);
+    }
+
+    public String signalCommand(ServiceType service, String signal) {
+      return String.format("%s | xargs kill -s %s", findPidCommand(service), signal);
+    }
+  }
+
+  /**
+   * CommandProvider to manage the service using bin/hbase-* scripts
+   */
+  static class HBaseShellCommandProvider extends CommandProvider {
+    private String getHBaseHome() {
+      return System.getenv("HBASE_HOME");
+    }
+
+    private String getConfig() {
+      String confDir = System.getenv("HBASE_CONF_DIR");
+      if (confDir != null) {
+        return String.format("--config %s", confDir);
+      }
+      return "";
+    }
+
+    @Override
+    public String getCommand(ServiceType service, Operation op) {
+      return String.format("%s/bin/hbase-daemon.sh %s %s %s", getHBaseHome(), getConfig(),
+          op.toString().toLowerCase(), service);
+    }
+  }
+
+  public HBaseClusterManager() {
+    super();
+  }
+
+  protected CommandProvider getCommandProvider(ServiceType service) {
+    //TODO: make it pluggable, or auto-detect the best command provider, should work with
+    //hadoop daemons as well
+    return new HBaseShellCommandProvider();
+  }
+
+  /**
+   * Execute the given command on the host using SSH
+   * @return pair of exit code and command output
+   * @throws IOException if something goes wrong.
+   */
+  private Pair<Integer, String> exec(String hostname, String... cmd) throws IOException {
+    LOG.info("Executing remote command: " + StringUtils.join(cmd, " ") + " , hostname:" + hostname);
+
+    RemoteShell shell = new RemoteShell(hostname, cmd);
+    shell.execute();
+
+    LOG.info("Executed remote command, exit code:" + shell.getExitCode()
+        + " , output:" + shell.getOutput());
+
+    return new Pair<Integer, String>(shell.getExitCode(), shell.getOutput());
+  }
+
+  private void exec(String hostname, ServiceType service, Operation op) throws IOException {
+    exec(hostname, getCommandProvider(service).getCommand(service, op));
+  }
+
+  @Override
+  public void start(ServiceType service, String hostname) throws IOException {
+    exec(hostname, service, Operation.START);
+  }
+
+  @Override
+  public void stop(ServiceType service, String hostname) throws IOException {
+    exec(hostname, service, Operation.STOP);
+  }
+
+  @Override
+  public void restart(ServiceType service, String hostname) throws IOException {
+    exec(hostname, service, Operation.RESTART);
+  }
+
+  @Override
+  public void signal(ServiceType service, String signal, String hostname) throws IOException {
+    exec(hostname, getCommandProvider(service).signalCommand(service, signal));
+  }
+
+  @Override
+  public boolean isRunning(ServiceType service, String hostname) throws IOException {
+    String ret = exec(hostname, getCommandProvider(service).isRunningCommand(service))
+        .getSecond();
+    return ret.length() > 0;
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index d13188f7205c..998ad1552f52 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -97,7 +97,9 @@
  * old HBaseTestCase and HBaseClusterTestCase functionality.
  * Create an instance and keep it around testing HBase.  This class is
  * meant to be your one-stop shop for anything you might need testing.  Manages
- * one cluster at a time only.
+ * one cluster at a time only. Managed cluster can be an in-process
+ * {@link MiniHBaseCluster}, or a deployed cluster of type {@link DistributedHBaseCluster}.
+ * Not all methods work with the real cluster.
  * Depends on log4j being on classpath and
  * hbase-site.xml for logging and test-run configuration.  It does not set
  * logging levels nor make changes to configuration parameters.
@@ -120,7 +122,7 @@ public class HBaseTestingUtility {
   private boolean passedZkCluster = false;
   private MiniDFSCluster dfsCluster = null;
 
-  private MiniHBaseCluster hbaseCluster = null;
+  private HBaseCluster hbaseCluster = null;
   private MiniMRCluster mrCluster = null;
 
   // Directory where we put the data for this instance of HBaseTestingUtility
@@ -213,6 +215,10 @@ public Configuration getConfiguration() {
     return this.conf;
   }
 
+  public void setHBaseCluster(HBaseCluster hbaseCluster) {
+    this.hbaseCluster = hbaseCluster;
+  }
+
   /**
    * @return Where to write test data on local filesystem; usually
    * {@link #DEFAULT_BASE_TEST_DIRECTORY}
@@ -650,7 +656,7 @@ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
 
     getHBaseAdmin(); // create immediately the hbaseAdmin
     LOG.info("Minicluster is up");
-    return this.hbaseCluster;
+    return (MiniHBaseCluster)this.hbaseCluster;
   }
 
   /**
@@ -678,7 +684,11 @@ public void restartHBaseCluster(int servers) throws IOException, InterruptedExce
    * @see #startMiniCluster()
    */
   public MiniHBaseCluster getMiniHBaseCluster() {
-    return this.hbaseCluster;
+    if (this.hbaseCluster instanceof MiniHBaseCluster) {
+      return (MiniHBaseCluster)this.hbaseCluster;
+    }
+    throw new RuntimeException(hbaseCluster + " not an instance of " +
+                               MiniHBaseCluster.class.getName());
   }
 
   /**
@@ -721,7 +731,7 @@ public void shutdownMiniHBaseCluster() throws IOException {
     if (this.hbaseCluster != null) {
       this.hbaseCluster.shutdown();
       // Wait till hbase is down before going on to shutdown zk.
-      this.hbaseCluster.join();
+      this.hbaseCluster.waitUntilShutDown();
       this.hbaseCluster = null;
     }
   }
@@ -759,7 +769,7 @@ public Path createRootDir() throws IOException {
    * @throws IOException
    */
   public void flush() throws IOException {
-    this.hbaseCluster.flushcache();
+    getMiniHBaseCluster().flushcache();
   }
 
   /**
@@ -767,7 +777,7 @@ public void flush() throws IOException {
    * @throws IOException
    */
   public void flush(byte [] tableName) throws IOException {
-    this.hbaseCluster.flushcache(tableName);
+    getMiniHBaseCluster().flushcache(tableName);
   }
 
   /**
@@ -775,7 +785,7 @@ public void flush(byte [] tableName) throws IOException {
    * @throws IOException
    */
   public void compact(boolean major) throws IOException {
-    this.hbaseCluster.compact(major);
+    getMiniHBaseCluster().compact(major);
   }
 
   /**
@@ -783,7 +793,7 @@ public void compact(boolean major) throws IOException {
    * @throws IOException
    */
   public void compact(byte [] tableName, boolean major) throws IOException {
-    this.hbaseCluster.compact(tableName, major);
+    getMiniHBaseCluster().compact(tableName, major);
   }
 
 
@@ -1003,6 +1013,7 @@ public int loadTable(final HTable t, final byte[] f) throws IOException {
     t.flushCommits();
     return rowCount;
   }
+
   /**
    * Load region with rows from 'aaa' to 'zzz'.
    * @param r Region
@@ -1162,9 +1173,10 @@ public int createMultiRegions(final Configuration c, final HTable table,
     HConnection conn = table.getConnection();
     conn.clearRegionCache();
     // assign all the new regions IF table is enabled.
-    if (getHBaseAdmin().isTableEnabled(table.getTableName())) {
+    HBaseAdmin admin = getHBaseAdmin();
+    if (admin.isTableEnabled(table.getTableName())) {
       for(HRegionInfo hri : newRegions) {
-        hbaseCluster.getMaster().assignRegion(hri);
+        admin.assign(hri.getRegionName());
       }
     }
 
@@ -1275,8 +1287,8 @@ public HRegionServer getRSForFirstRegionInTable(byte[] tableName)
       Bytes.toString(tableName));
     byte [] firstrow = metaRows.get(0);
     LOG.debug("FirstRow=" + Bytes.toString(firstrow));
-    int index = hbaseCluster.getServerWith(firstrow);
-    return hbaseCluster.getRegionServerThreads().get(index).getRegionServer();
+    int index = getMiniHBaseCluster().getServerWith(firstrow);
+    return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer();
   }
 
   /**
@@ -1357,7 +1369,7 @@ public void enableDebug(Class<?> clazz) {
    * @throws Exception
    */
   public void expireMasterSession() throws Exception {
-    HMaster master = hbaseCluster.getMaster();
+    HMaster master = getMiniHBaseCluster().getMaster();
     expireSession(master.getZooKeeper(), false);
   }
 
@@ -1367,7 +1379,7 @@ public void expireMasterSession() throws Exception {
    * @throws Exception
    */
   public void expireRegionServerSession(int index) throws Exception {
-    HRegionServer rs = hbaseCluster.getRegionServer(index);
+    HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
     expireSession(rs.getZooKeeper(), false);
     decrementMinRegionServerCount();
   }
@@ -1441,13 +1453,27 @@ public void process(WatchedEvent watchedEvent) {
     }
   }
 
-
   /**
-   * Get the HBase cluster.
+   * Get the Mini HBase cluster.
    *
    * @return hbase cluster
+   * @see #getHBaseClusterInterface()
    */
   public MiniHBaseCluster getHBaseCluster() {
+    return getMiniHBaseCluster();
+  }
+
+  /**
+   * Returns the HBaseCluster instance.
+   * <p>Returned object can be any of the subclasses of HBaseCluster, and the
+   * tests referring this should not assume that the cluster is a mini cluster or a
+   * distributed one. If the test only works on a mini cluster, then specific
+   * method {@link #getMiniHBaseCluster()} can be used instead w/o the
+   * need to type-cast.
+   */
+  public HBaseCluster getHBaseClusterInterface() {
+    //implementation note: we should rename this method as #getHBaseCluster(),
+    //but this would require refactoring 90+ calls.
     return hbaseCluster;
   }
 
@@ -1609,8 +1635,8 @@ public void waitTableEnabled(byte[] table, long timeoutMillis)
   public boolean ensureSomeRegionServersAvailable(final int num)
       throws IOException {
     boolean startedServer = false;
-
-    for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i){
+    MiniHBaseCluster hbaseCluster = getMiniHBaseCluster();
+    for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i) {
       LOG.info("Started new server=" + hbaseCluster.startRegionServer());
       startedServer = true;
     }
@@ -1632,12 +1658,12 @@ public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
     boolean startedServer = ensureSomeRegionServersAvailable(num);
 
     for (JVMClusterUtil.RegionServerThread rst :
-      hbaseCluster.getRegionServerThreads()) {
+      getMiniHBaseCluster().getRegionServerThreads()) {
 
       HRegionServer hrs = rst.getRegionServer();
       if (hrs.isStopping() || hrs.isStopped()) {
         LOG.info("A region server is stopped or stopping:"+hrs);
-        LOG.info("Started new server=" + hbaseCluster.startRegionServer());
+        LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
         startedServer = true;
       }
     }
@@ -1907,7 +1933,9 @@ public HTable createRandomTable(String tableName,
         Bytes.toBytes(String.format(keyFormat, splitStartKey)),
         Bytes.toBytes(String.format(keyFormat, splitEndKey)),
         numRegions);
-    hbaseCluster.flushcache(HConstants.META_TABLE_NAME);
+    if (hbaseCluster != null) {
+      getMiniHBaseCluster().flushcache(HConstants.META_TABLE_NAME);
+    }
 
     for (int iFlush = 0; iFlush < numFlushes; ++iFlush) {
       for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) {
@@ -1942,7 +1970,9 @@ public HTable createRandomTable(String tableName,
       }
       LOG.info("Initiating flush #" + iFlush + " for table " + tableName);
       table.flushCommits();
-      hbaseCluster.flushcache(tableNameBytes);
+      if (hbaseCluster != null) {
+        getMiniHBaseCluster().flushcache(tableNameBytes);
+      }
     }
 
     return table;
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
new file mode 100644
index 000000000000..72f76b0dcfea
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChaosMonkey;
+import org.apache.hadoop.hbase.util.LoadTestTool;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * A system test which does large data ingestion and verify using {@link LoadTestTool},
+ * while killing the region servers and the master(s) randomly. You can configure how long
+ * should the load test run by using "hbase.IntegrationTestDataIngestWithChaosMonkey.runtime"
+ * configuration parameter.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestDataIngestWithChaosMonkey {
+
+  private static final String TABLE_NAME = "TestDataIngestWithChaosMonkey";
+  private static final int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
+
+  /** A soft limit on how long we should run */
+  private static final String RUN_TIME_KEY = "hbase.IntegrationTestDataIngestWithChaosMonkey.runtime";
+
+  //run for 5 min by default
+  private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000;
+
+  private static final Log LOG = LogFactory.getLog(IntegrationTestDataIngestWithChaosMonkey.class);
+  private IntegrationTestingUtility util;
+  private HBaseCluster cluster;
+  private ChaosMonkey monkey;
+
+  @Before
+  public void setUp() throws Exception {
+    util = new IntegrationTestingUtility();
+
+    util.initializeCluster(NUM_SLAVES_BASE);
+
+    cluster = util.getHBaseClusterInterface();
+    deleteTableIfNecessary();
+
+    monkey = new ChaosMonkey(util, ChaosMonkey.EVERY_MINUTE_RANDOM_ACTION_POLICY);
+    monkey.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    monkey.stop("test has finished, that's why");
+    monkey.waitForStop();
+    util.restoreCluster();
+  }
+
+  private void deleteTableIfNecessary() throws IOException {
+    if (util.getHBaseAdmin().tableExists(TABLE_NAME)) {
+      util.deleteTable(Bytes.toBytes(TABLE_NAME));
+    }
+  }
+
+  @Test
+  public void testDataIngest() throws Exception {
+    LOG.info("Running testDataIngest");
+    LOG.info("Cluster size:" + util.getHBaseClusterInterface().getClusterStatus().getServersSize());
+
+    LoadTestTool loadTool = new LoadTestTool();
+    loadTool.setConf(util.getConfiguration());
+
+    long start = System.currentTimeMillis();
+    long runtime = util.getConfiguration().getLong(RUN_TIME_KEY, DEFAULT_RUN_TIME);
+    long startKey = 0;
+
+    long numKeys = estimateDataSize();
+    while (System.currentTimeMillis() - start < 0.9 * runtime) {
+      LOG.info("Intended run time: " + (runtime/60000) + " min, left:" +
+          ((runtime - (System.currentTimeMillis() - start))/60000) + " min");
+
+      int ret = loadTool.run(new String[] {
+          "-tn", TABLE_NAME,
+          "-write", "10:100:20",
+          "-start_key", String.valueOf(startKey),
+          "-num_keys", String.valueOf(numKeys)
+      });
+
+      //assert that load was successful
+      Assert.assertEquals(0, ret);
+
+      ret = loadTool.run(new String[] {
+          "-tn", TABLE_NAME,
+          "-read", "100:20",
+          "-start_key", String.valueOf(startKey),
+          "-num_keys", String.valueOf(numKeys)
+      });
+
+      //assert that verify was successful
+      Assert.assertEquals(0, ret);
+      startKey += numKeys;
+    }
+  }
+
+  /** Estimates a data size based on the cluster size */
+  protected long estimateDataSize() throws IOException {
+    //base is a 4 slave node cluster.
+    ClusterStatus status = cluster.getClusterStatus();
+    int numRegionServers = status.getServersSize();
+    int multiplication = Math.max(1, numRegionServers / NUM_SLAVES_BASE);
+
+    return 10000 * multiplication;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java
new file mode 100644
index 000000000000..d5a6feeaf124
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Facility for <strong>integration/system</strong> tests. This extends {@link HBaseTestingUtility}
+ * and adds-in the functionality needed by integration and system tests. This class understands
+ * distributed and pseudo-distributed/local cluster deployments, and abstracts those from the tests
+ * in this module.
+ * <p>
+ * IntegrationTestingUtility is constructed and used by the integration tests, but the tests
+ * themselves should not assume a particular deployment. They can rely on the methods in this
+ * class and HBaseCluster. Before the testing begins, the test should initialize the cluster by
+ * calling {@link #initializeCluster(int)}.
+ * <p>
+ * The cluster that is used defaults to a mini cluster, but it can be forced to use a distributed
+ * cluster by calling {@link #setUseDistributedCluster(Configuration)}. This method is invoked by
+ * test drivers (maven, IntegrationTestsDriver, etc) before initializing the cluster
+ * via {@link #initializeCluster(int)}. Individual tests should not directly call
+ * {@link #setUseDistributedCluster(Configuration)}.
+ */
+public class IntegrationTestingUtility extends HBaseTestingUtility {
+
+  public IntegrationTestingUtility() {
+    this(HBaseConfiguration.create());
+  }
+
+  public IntegrationTestingUtility(Configuration conf) {
+    super(conf);
+  }
+
+  /**
+   * Configuration that controls whether this utility assumes a running/deployed cluster.
+   * This is different than "hbase.cluster.distributed" since that parameter indicates whether the
+   * cluster is in an actual distributed environment, while this shows that there is a
+   * deployed (distributed or pseudo-distributed) cluster running, and we do not need to
+   * start a mini-cluster for tests.
+   */
+  public static final String IS_DISTRIBUTED_CLUSTER = "hbase.test.cluster.distributed";
+
+  /**
+   * Initializes the state of the cluster. It starts a new in-process mini cluster, OR
+   * if we are given an already deployed distributed cluster it initializes the state.
+   * @param numSlaves Number of slaves to start up if we are booting a mini cluster. Otherwise
+   * we check whether this many nodes are available and throw an exception if not.
+   */
+  public void initializeCluster(int numSlaves) throws Exception {
+    if (isDistributedCluster()) {
+      createDistributedHBaseCluster();
+      checkNodeCount(numSlaves);
+    } else {
+      startMiniCluster(numSlaves);
+    }
+  }
+
+  /**
+   * Checks whether we have more than numSlaves nodes. Throws an
+   * exception otherwise.
+   */
+  public void checkNodeCount(int numSlaves) throws Exception {
+    HBaseCluster cluster = getHBaseClusterInterface();
+    if (cluster.getClusterStatus().getServers().size() < numSlaves) {
+      throw new Exception("Cluster does not have enough nodes:" + numSlaves);
+    }
+  }
+
+  /**
+   * Restores the cluster to the initial state if it is a distributed cluster, otherwise, shutdowns the
+   * mini cluster.
+   */
+  public void restoreCluster() throws IOException {
+    if (isDistributedCluster()) {
+      getHBaseClusterInterface().restoreInitialStatus();
+    } else {
+      getMiniHBaseCluster().shutdown();
+    }
+  }
+
+  /**
+   * Sets the configuration property to use a distributed cluster for the integration tests. Test drivers
+   * should use this to enforce cluster deployment.
+   */
+  public static void setUseDistributedCluster(Configuration conf) {
+    conf.setBoolean(IS_DISTRIBUTED_CLUSTER, true);
+    System.setProperty(IS_DISTRIBUTED_CLUSTER, "true");
+  }
+
+  /**
+   * @return whether we are interacting with a distributed cluster as opposed to and in-process mini
+   * cluster or a local cluster.
+   * @see IntegrationTestingUtility#setUseDistributedCluster(Configuration)
+   */
+  private boolean isDistributedCluster() {
+    Configuration conf = getConfiguration();
+    boolean isDistributedCluster = false;
+    isDistributedCluster = Boolean.parseBoolean(System.getProperty(IS_DISTRIBUTED_CLUSTER, "false"));
+    if (!isDistributedCluster) {
+      isDistributedCluster = conf.getBoolean(IS_DISTRIBUTED_CLUSTER, false);
+    }
+    return isDistributedCluster;
+  }
+
+  private void createDistributedHBaseCluster() throws IOException {
+    Configuration conf = getConfiguration();
+    ClusterManager clusterManager = new HBaseClusterManager();
+    setHBaseCluster(new DistributedHBaseCluster(conf, clusterManager));
+    getHBaseAdmin();
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTests.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTests.java
new file mode 100644
index 000000000000..d429e2405101
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTests.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+/**
+ * Tag a test as 'integration/system' test, meaning that the test class has the following
+ * characteristics: <ul>
+ *  <li> Possibly takes hours to complete</li>
+ *  <li> Can be run on a mini cluster or an actual cluster</li>
+ *  <li> Can make changes to the given cluster (starting stopping daemons, etc)</li>
+ *  <li> Should not be run in parallel of other integration tests</li>
+ * </ul>
+ *
+ * Integration / System tests should have a class name starting with "IntegrationTest", and
+ * should be annotated with @Category(IntegrationTests.class). Integration tests can be run
+ * using the IntegrationTestsDriver class or from mvn verify.
+ *
+ * @see SmallTests
+ * @see MediumTests
+ * @see LargeTests
+ */
+public interface IntegrationTests {
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
new file mode 100644
index 000000000000..fd6053f41b4c
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.hbase.util.AbstractHBaseTool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.internal.TextListener;
+import org.junit.runner.JUnitCore;
+import org.junit.runner.Result;
+
+/**
+ * This class drives the Integration test suite execution. Executes all
+ * tests having @Category(IntegrationTests.class) annotation against an
+ * already deployed distributed cluster.
+ */
+public class IntegrationTestsDriver extends AbstractHBaseTool {
+
+  public static void main(String[] args) throws Exception {
+    int ret = ToolRunner.run(new IntegrationTestsDriver(), args);
+    System.exit(ret);
+  }
+
+  @Override
+  protected void addOptions() {
+  }
+
+  @Override
+  protected void processOptions(CommandLine cmd) {
+  }
+
+  /**
+   * Returns test classes annotated with @Category(IntegrationTests.class)
+   */
+  private Class<?>[] findIntegrationTestClasses() throws ClassNotFoundException, IOException {
+    TestCheckTestClasses util = new TestCheckTestClasses();
+    List<Class<?>> classes = util.findTestClasses(IntegrationTests.class);
+    return classes.toArray(new Class<?>[classes.size()]);
+  }
+
+  @Override
+  protected int doWork() throws Exception {
+
+    //this is called from the command line, so we should set to use the distributed cluster
+    IntegrationTestingUtility.setUseDistributedCluster(conf);
+
+    JUnitCore junit = new JUnitCore();
+    junit.addListener(new TextListener(System.out));
+    Result result = junit.run(findIntegrationTestClasses());
+
+    return result.wasSuccessful() ? 0 : 1;
+  }
+
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/LargeTests.java b/src/test/java/org/apache/hadoop/hbase/LargeTests.java
index f1b46fa92ea7..da9abdafc7b8 100644
--- a/src/test/java/org/apache/hadoop/hbase/LargeTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/LargeTests.java
@@ -33,6 +33,7 @@
  *
  * @see SmallTests
  * @see MediumTests
+ * @see IntegrationTests
  */
 public interface LargeTests {
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/MediumTests.java b/src/test/java/org/apache/hadoop/hbase/MediumTests.java
index bbbde7cdcca0..24a44b9523e6 100644
--- a/src/test/java/org/apache/hadoop/hbase/MediumTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/MediumTests.java
@@ -32,6 +32,7 @@
  *
  * @see SmallTests
  * @see LargeTests
+ * @see IntegrationTests
  */
 public interface MediumTests {
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 6cfdee02fff6..f110b589dc54 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -28,6 +28,8 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -35,6 +37,8 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.io.MapWritable;
 
@@ -44,9 +48,8 @@
  * if we are running on DistributedFilesystem, create a FileSystem instance
  * each and will close down their instance on the way out.
  */
-public class MiniHBaseCluster {
+public class MiniHBaseCluster extends HBaseCluster {
   static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName());
-  private Configuration conf;
   public LocalHBaseCluster hbaseCluster;
   private static int index;
 
@@ -69,11 +72,19 @@ public MiniHBaseCluster(Configuration conf, int numRegionServers)
    * @throws IOException
    */
   public MiniHBaseCluster(Configuration conf, int numMasters,
-      int numRegionServers)
-  throws IOException, InterruptedException {
-    this.conf = conf;
+                             int numRegionServers)
+      throws IOException, InterruptedException {
+    this(conf, numMasters, numRegionServers, null, null);
+  }
+
+  public MiniHBaseCluster(Configuration conf, int numMasters, int numRegionServers,
+         Class<? extends HMaster> masterClass,
+         Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+      throws IOException, InterruptedException {
+    super(conf);
     conf.set(HConstants.MASTER_PORT, "0");
-    init(numMasters, numRegionServers);
+    init(numMasters, numRegionServers, masterClass, regionserverClass);
+    this.initialClusterStatus = getClusterStatus();
   }
 
   public Configuration getConfiguration() {
@@ -178,12 +189,21 @@ public void run() {
     }
   }
 
-  private void init(final int nMasterNodes, final int nRegionNodes)
+  private void init(final int nMasterNodes, final int nRegionNodes,
+          Class<? extends HMaster> masterClass,
+          Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
   throws IOException, InterruptedException {
     try {
+      if (masterClass == null){
+       masterClass =  HMaster.class;
+      }
+      if (regionserverClass == null){
+       regionserverClass = MiniHBaseCluster.MiniHBaseClusterRegionServer.class;
+      }
+
       // start up a LocalHBaseCluster
       hbaseCluster = new LocalHBaseCluster(conf, nMasterNodes, 0,
-        HMaster.class, MiniHBaseCluster.MiniHBaseClusterRegionServer.class);
+         masterClass, regionserverClass);
 
       // manually add the regionservers as other users
       for (int i=0; i<nRegionNodes; i++) {
@@ -204,6 +224,54 @@ private void init(final int nMasterNodes, final int nRegionNodes)
     }
   }
 
+  @Override
+  public void startRegionServer(String hostname) throws IOException {
+    this.startRegionServer();
+  }
+
+  @Override
+  public void killRegionServer(ServerName serverName) throws IOException {
+    HRegionServer server = getRegionServer(getRegionServerIndex(serverName));
+    if (server instanceof MiniHBaseClusterRegionServer) {
+      LOG.info("Killing " + server.toString());
+      ((MiniHBaseClusterRegionServer) server).kill();
+    } else {
+      abortRegionServer(getRegionServerIndex(serverName));
+    }
+  }
+
+  @Override
+  public void stopRegionServer(ServerName serverName) throws IOException {
+    stopRegionServer(getRegionServerIndex(serverName));
+  }
+
+  @Override
+  public void waitForRegionServerToStop(ServerName serverName, long timeout) throws IOException {
+    //ignore timeout for now
+    waitOnRegionServer(getRegionServerIndex(serverName));
+  }
+
+  @Override
+  public void startMaster(String hostname) throws IOException {
+    this.startMaster();
+  }
+
+  @Override
+  public void killMaster(ServerName serverName) throws IOException {
+    abortMaster(getMasterIndex(serverName));
+  }
+
+  @Override
+  public void stopMaster(ServerName serverName) throws IOException {
+    stopMaster(getMasterIndex(serverName));
+  }
+
+  @Override
+  public void waitForMasterToStop(ServerName serverName, long timeout) throws IOException {
+    //ignore timeout for now
+    waitOnMaster(getMasterIndex(serverName));
+  }
+
   /**
    * Starts a region server thread running
    *
@@ -299,6 +367,11 @@ public JVMClusterUtil.MasterThread startMaster() throws IOException {
     return t;
   }
 
+  @Override
+  public HMasterInterface getMasterAdmin() {
+    return this.hbaseCluster.getActiveMaster();
+  }
+
   /**
    * Returns the current active master, if available.
    * @return the active HMaster, null if none is active.
@@ -373,15 +446,18 @@ public String waitOnMaster(final int serverNumber) {
    *         masters left.
    * @throws InterruptedException
    */
-  public boolean waitForActiveAndReadyMaster() throws InterruptedException {
+  public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
     List<JVMClusterUtil.MasterThread> mts;
-    while (!(mts = getMasterThreads()).isEmpty()) {
+    long start = System.currentTimeMillis();
+    while (!(mts = getMasterThreads()).isEmpty()
+        && (System.currentTimeMillis() - start) < timeout) {
       for (JVMClusterUtil.MasterThread mt : mts) {
         if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) {
           return true;
         }
       }
-      Thread.sleep(100);
+
+      Threads.sleep(100);
     }
     return false;
   }
@@ -418,6 +494,16 @@ public void shutdown() throws IOException {
     HConnectionManager.deleteAllConnections(false);
   }
 
+  @Override
+  public void close() throws IOException {
+  }
+
+  @Override
+  public ClusterStatus getClusterStatus() throws IOException {
+    HMaster master = getMaster();
+    return master == null ? null : master.getClusterStatus();
+  }
+
   /**
    * Call flushCache on all regions on all participating regionservers.
    * @throws IOException
@@ -540,6 +626,15 @@ public int getServerWith(byte[] regionName) {
     return index;
   }
 
+  @Override
+  public ServerName getServerHoldingRegion(byte[] regionName) throws IOException {
+    int index = getServerWith(regionName);
+    if (index < 0) {
+      return null;
+    }
+    return getRegionServer(index).getServerName();
+  }
+
   /**
    * Counts the total numbers of regions being served by the currently online
    * region servers by asking each how many regions they have.  Does not look
@@ -553,4 +648,30 @@ public long countServedRegions() {
     }
     return count;
   }
+
+  @Override
+  public void waitUntilShutDown() {
+    this.hbaseCluster.join();
+  }
+
+  protected int getRegionServerIndex(ServerName serverName) {
+    //we have a small number of region servers, this should be fine for now.
+    List<RegionServerThread> servers = getRegionServerThreads();
+    for (int i=0; i < servers.size(); i++) {
+      if (servers.get(i).getRegionServer().getServerName().equals(serverName)) {
+        return i;
+      }
+    }
+    return -1;
+  }
+
+  protected int getMasterIndex(ServerName serverName) {
+    List<MasterThread> masters = getMasterThreads();
+    for (int i = 0; i < masters.size(); i++) {
+      if (masters.get(i).getMaster().getServerName().equals(serverName)) {
+        return i;
+      }
+    }
+    return -1;
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/SmallTests.java b/src/test/java/org/apache/hadoop/hbase/SmallTests.java
index c702f5a1fd5e..eba6a948f0b7 100644
--- a/src/test/java/org/apache/hadoop/hbase/SmallTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/SmallTests.java
@@ -29,6 +29,7 @@
  *
  * @see MediumTests
  * @see LargeTests
+ * @see IntegrationTests
  */
 public interface SmallTests {
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java b/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
new file mode 100644
index 000000000000..a5ac6f4cd712
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import static junit.framework.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Suite;
+
+
+/**
+ * Checks tests are categorized.
+ */
+@Category(SmallTests.class)
+public class TestCheckTestClasses {
+
+  private FileFilter TEST_CLASS_FILE_FILTER = new FileFilter() {
+    @Override
+    public boolean accept(File file) {
+      return file.isDirectory() || isTestClassFile(file);
+
+    }
+    private boolean isTestClassFile(File file) {
+      String fileName = file.getName();
+      return fileName.endsWith(".class")
+          && (fileName.startsWith("Test") || fileName.startsWith("IntegrationTest"));
+    }
+  };
+
+  /**
+   * Throws an assertion if we find a test class without category (small/medium/large/integration).
+   * List all the test classes without category in the assertion message.
+   */
+  @Test
+  public void checkClasses() throws Exception {
+    List<Class<?>> badClasses = new java.util.ArrayList<Class<?>>();
+
+    for (Class<?> c : findTestClasses()) {
+      if (!existCategoryAnnotation(c, null)) {
+        badClasses.add(c);
+      }
+    }
+
+    assertTrue("There are " + badClasses.size() + " test classes without category: "
+      + badClasses, badClasses.isEmpty());
+  }
+
+  /** Returns whether the class has @Category annotation having the xface value.
+   */
+  private boolean existCategoryAnnotation(Class<?> c, Class<?> xface) {
+    Category category = c.getAnnotation(Category.class);
+
+    if (category != null) {
+      if (xface == null) {
+        return true;
+      }
+      for (Class<?> cc : category.value()) {
+        if (cc.equals(xface)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  /*
+  * A class is considered as a test class if:
+   *  - it's not Abstract AND
+   *  - one or more of its methods is annotated with org.junit.Test OR
+   *  - the class is annotated with Suite.SuiteClasses
+  * */
+  private boolean isTestClass(Class<?> c) {
+    if (Modifier.isAbstract(c.getModifiers())) {
+      return false;
+    }
+
+    if (c.getAnnotation(Suite.SuiteClasses.class) != null) {
+      return true;
+    }
+
+    for (Method met : c.getMethods()) {
+      if (met.getAnnotation(Test.class) != null) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Finds test classes which are annotated with @Category having xface value
+   * @param xface the @Category value
+   */
+  public List<Class<?>> findTestClasses(Class<?> xface) throws ClassNotFoundException, IOException {
+    List<Class<?>> classes = new ArrayList<Class<?>>();
+    for (Class<?> c : findTestClasses()) {
+      if (existCategoryAnnotation(c, xface)) {
+        classes.add(c);
+      }
+    }
+    return classes;
+  }
+
+  private List<Class<?>> findTestClasses() throws ClassNotFoundException, IOException {
+    final String packageName = "org.apache.hadoop.hbase";
+    final String path = packageName.replace('.', '/');
+
+    Enumeration<URL> resources = this.getClass().getClassLoader().getResources(path);
+    List<File> dirs = new ArrayList<File>();
+
+    while (resources.hasMoreElements()) {
+      URL resource = resources.nextElement();
+      dirs.add(new File(resource.getFile()));
+    }
+
+    List<Class<?>> classes = new ArrayList<Class<?>>();
+    for (File directory : dirs) {
+      classes.addAll(findTestClasses(directory, packageName));
+    }
+
+    return classes;
+  }
+
+
+  private List<Class<?>> findTestClasses(File baseDirectory, String packageName)
+    throws ClassNotFoundException {
+    List<Class<?>> classes = new ArrayList<Class<?>>();
+    if (!baseDirectory.exists()) {
+      return classes;
+    }
+
+    File[] files = baseDirectory.listFiles(TEST_CLASS_FILE_FILTER);
+    assertNotNull(files);
+    Pattern p = Pattern.compile("hbase-hadoop\\d?-compat");
+    for (File file : files) {
+      final String fileName = file.getName();
+      if (p.matcher(file.getAbsolutePath()).find()) {
+        continue;
+      }
+
+      if (file.isDirectory()) {
+        classes.addAll(findTestClasses(file, packageName + "." + fileName));
+      } else {
+        Class<?> c = Class.forName(
+          packageName + '.' + fileName.substring(0, fileName.length() - 6),
+          false,
+          this.getClass().getClassLoader());
+
+        if (isTestClass(c)) {
+          classes.add(c);
+        }
+      }
+    }
+
+    return classes;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
index d5a906850d29..aa6eb9ad447b 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
@@ -25,14 +25,17 @@
 import java.net.InetSocketAddress;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 import com.google.protobuf.DescriptorProtos;
 import com.google.protobuf.DescriptorProtos.EnumDescriptorProto;
 
 /** Unit tests to test PB-based types on WritableRpcEngine. */
+@Category(MediumTests.class)
 public class TestPBOnWritableRpc {
 
   private static Configuration conf = new Configuration();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java b/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java
index 379f70c0ca89..ec63d81b1d38 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java
@@ -25,11 +25,14 @@
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(MediumTests.class)
 public class TestMXBean {
 
   private static final HBaseTestingUtility TEST_UTIL =
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 108c2820542d..479d9048fbcd 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -35,7 +35,18 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
@@ -44,9 +55,9 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -382,7 +393,7 @@ public void testMasterFailoverWithMockedRIT() throws Exception {
     enabledAndAssignedRegions.add(enabledRegions.remove(0));
     enabledAndAssignedRegions.add(enabledRegions.remove(0));
     enabledAndAssignedRegions.add(closingRegion);
-    
+
     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
     disabledAndAssignedRegions.add(disabledRegions.remove(0));
     disabledAndAssignedRegions.add(disabledRegions.remove(0));
@@ -620,18 +631,18 @@ public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
     // Create a ZKW to use in the test
     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
         "unittest", new Abortable() {
-          
+
           @Override
           public void abort(String why, Throwable e) {
             LOG.error("Fatal ZK Error: " + why, e);
             org.junit.Assert.assertFalse("Fatal ZK error", true);
           }
-          
+
           @Override
           public boolean isAborted() {
             return false;
           }
-          
+
     });
 
     // get all the master threads
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java
index c9e5b6e3af09..a0ba248289c3 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java
@@ -22,9 +22,12 @@
 
 import junit.framework.Assert;
 
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.util.Pair;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(SmallTests.class)
 public class TestExactCounterMetric {
 
   @Test
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
index b34ec88f032f..9d09486ee118 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java
@@ -22,8 +22,13 @@
 
 import com.yammer.metrics.stats.ExponentiallyDecayingSample;
 import com.yammer.metrics.stats.Snapshot;
+
+import org.apache.hadoop.hbase.SmallTests;
+
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(SmallTests.class)
 public class TestExponentiallyDecayingSample {
   
   @Test
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
index 57778eee5230..da0e87edfdfd 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
@@ -22,10 +22,13 @@
 import java.util.Random;
 
 import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+import org.apache.hadoop.hbase.SmallTests;
 import com.yammer.metrics.stats.Snapshot;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(SmallTests.class)
 public class TestMetricsHistogram {
 
   @Test
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 33bbe9feba9d..34e97010e5b3 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.StoppableImplementation;
 import org.apache.hadoop.hbase.util.Threads;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -170,7 +171,7 @@ public void testFromClientSideWhileSplitting() throws Throwable {
     //for daughters.
     HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
 
-    Stoppable stopper = new SimpleStoppable();
+    Stoppable stopper = new StoppableImplementation();
     RegionSplitter regionSplitter = new RegionSplitter(table);
     RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
 
@@ -193,20 +194,6 @@ public void testFromClientSideWhileSplitting() throws Throwable {
     regionChecker.verify();
   }
 
-  private static class SimpleStoppable implements Stoppable {
-    volatile boolean stopped = false;
-
-    @Override
-    public void stop(String why) {
-      this.stopped = true;
-    }
-
-    @Override
-    public boolean isStopped() {
-      return stopped;
-    }
-  }
-
   static class RegionSplitter extends Thread {
     Throwable ex;
     HTable table;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java
index 83e02c7a214e..cfa79057e720 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java
@@ -21,10 +21,13 @@
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.MediumTests;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(MediumTests.class)
 public class TestMXBean {
 
   private static final HBaseTestingUtility TEST_UTIL =
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index e4157cddb706..a7287a1b84a2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -55,7 +56,9 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
+@Category(LargeTests.class)
 public class TestRSKilledWhenMasterInitializing {
   private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
new file mode 100644
index 000000000000..2d67a74c90cc
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -0,0 +1,563 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Random;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseCluster;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.IntegrationTestDataIngestWithChaosMonkey;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * A utility to injects faults in a running cluster.
+ * <p>
+ * ChaosMonkey defines Action's and Policy's. Actions are sequences of events, like
+ *  - Select a random server to kill
+ *  - Sleep for 5 sec
+ *  - Start the server on the same host
+ * Actions can also be complex events, like rolling restart of all of the servers.
+ * <p>
+ * Policies on the other hand are responsible for executing the actions based on a strategy.
+ * The default policy is to execute a random action every minute based on predefined action
+ * weights. ChaosMonkey executes predefined named policies until it is stopped. More than one
+ * policy can be active at any time.
+ * <p>
+ * Chaos monkey can be run from the command line, or can be invoked from integration tests.
+ * See {@link IntegrationTestDataIngestWithChaosMonkey} or other integration tests that use
+ * chaos monkey for code examples.
+ * <p>
+ * ChaosMonkey class is indeed inspired by the Netflix's same-named tool:
+ * http://techblog.netflix.com/2012/07/chaos-monkey-released-into-wild.html
+ */
+public class ChaosMonkey extends AbstractHBaseTool implements Stoppable {
+
+  private static final Log LOG = LogFactory.getLog(ChaosMonkey.class);
+
+  private static final long ONE_SEC = 1000;
+  private static final long FIVE_SEC = 5 * ONE_SEC;
+  private static final long ONE_MIN = 60 * ONE_SEC;
+  private static final long TIMEOUT = ONE_MIN;
+
+  final IntegrationTestingUtility util;
+
+  /**
+   * Construct a new ChaosMonkey
+   * @param util the HBaseIntegrationTestingUtility already configured
+   * @param policies names of pre-defined policies to use
+   */
+  public ChaosMonkey(IntegrationTestingUtility util, String... policies) {
+    this.util = util;
+    setPoliciesByName(policies);
+  }
+
+  private void setPoliciesByName(String... policies) {
+    this.policies = new Policy[policies.length];
+    for (int i=0; i < policies.length; i++) {
+      this.policies[i] = NAMED_POLICIES.get(policies[i]);
+    }
+  }
+
+  /**
+   * Context for Action's
+   */
+  private static class ActionContext {
+    private IntegrationTestingUtility util;
+
+    ActionContext(IntegrationTestingUtility util) {
+      this.util = util;
+    }
+
+    IntegrationTestingUtility getHaseIntegrationTestingUtility() {
+      return util;
+    }
+
+    HBaseCluster getHBaseCluster() {
+      return util.getHBaseClusterInterface();
+    }
+  }
+
+  /**
+   * A (possibly mischievous) action that the ChaosMonkey can perform.
+   */
+  private static class Action {
+    long sleepTime; //how long should we sleep
+    ActionContext context;
+    HBaseCluster cluster;
+    ClusterStatus initialStatus;
+    ServerName[] initialServers;
+
+    public Action(long sleepTime) {
+      this.sleepTime = sleepTime;
+    }
+
+    void init(ActionContext context) throws Exception {
+      this.context = context;
+      cluster = context.getHBaseCluster();
+      initialStatus = cluster.getInitialClusterStatus();
+      Collection<ServerName> regionServers = initialStatus.getServers();
+      initialServers = regionServers.toArray(new ServerName[regionServers.size()]);
+    }
+
+    void perform() throws Exception { };
+
+    /** Returns current region servers */
+    ServerName[] getCurrentServers() throws IOException {
+      Collection<ServerName> regionServers = cluster.getClusterStatus().getServers();
+      return regionServers.toArray(new ServerName[regionServers.size()]);
+    }
+
+    void killMaster(ServerName server) throws IOException {
+      LOG.info("Killing master:" + server);
+      cluster.killMaster(server);
+      cluster.waitForMasterToStop(server, TIMEOUT);
+      LOG.info("Killed master server:" + server);
+    }
+
+    void startMaster(ServerName server) throws IOException {
+      LOG.info("Starting master:" + server.getHostname());
+      cluster.startMaster(server.getHostname());
+      cluster.waitForActiveAndReadyMaster(TIMEOUT);
+      LOG.info("Started master: " + server);
+    }
+
+    void restartMaster(ServerName server, long sleepTime) throws IOException {
+      killMaster(server);
+      sleep(sleepTime);
+      startMaster(server);
+    }
+
+    void killRs(ServerName server) throws IOException {
+      LOG.info("Killing region server:" + server);
+      cluster.killRegionServer(server);
+      cluster.waitForRegionServerToStop(server, TIMEOUT);
+      LOG.info("Killed region server:" + server + ". Reported num of rs:"
+          + cluster.getClusterStatus().getServersSize());
+    }
+
+    void startRs(ServerName server) throws IOException {
+      LOG.info("Starting region server:" + server.getHostname());
+      cluster.startRegionServer(server.getHostname());
+      cluster.waitForRegionServerToStart(server.getHostname(), TIMEOUT);
+      LOG.info("Started region server:" + server + ". Reported num of rs:"
+          + cluster.getClusterStatus().getServersSize());
+    }
+
+    void sleep(long sleepTime) {
+      LOG.info("Sleeping for:" + sleepTime);
+      Threads.sleep(sleepTime);
+    }
+
+    void restartRs(ServerName server, long sleepTime) throws IOException {
+      killRs(server);
+      sleep(sleepTime);
+      startRs(server);
+    }
+  }
+
+  private static class RestartActiveMaster extends Action {
+    public RestartActiveMaster(long sleepTime) {
+      super(sleepTime);
+    }
+    @Override
+    void perform() throws Exception {
+      LOG.info("Performing action: Restart active master");
+
+      ServerName master = cluster.getClusterStatus().getMaster();
+      restartMaster(master, sleepTime);
+    }
+  }
+
+  private static class RestartRandomRs extends Action {
+    public RestartRandomRs(long sleepTime) {
+      super(sleepTime);
+    }
+
+    @Override
+    void init(ActionContext context) throws Exception {
+      super.init(context);
+    }
+
+    @Override
+    void perform() throws Exception {
+      LOG.info("Performing action: Restart random region server");
+      ServerName server = selectRandomItem(getCurrentServers());
+
+      restartRs(server, sleepTime);
+    }
+  }
+
+  private static class RestartRsHoldingMeta extends RestartRandomRs {
+    public RestartRsHoldingMeta(long sleepTime) {
+      super(sleepTime);
+    }
+    @Override
+    void perform() throws Exception {
+      LOG.info("Performing action: Restart region server holding META");
+      ServerName server = cluster.getServerHoldingMeta();
+      if (server == null) {
+        LOG.warn("No server is holding .META. right now.");
+        return;
+      }
+      restartRs(server, sleepTime);
+    }
+  }
+
+  private static class RestartRsHoldingRoot extends RestartRandomRs {
+    public RestartRsHoldingRoot(long sleepTime) {
+      super(sleepTime);
+    }
+    @Override
+    void perform() throws Exception {
+      LOG.info("Performing action: Restart region server holding ROOT");
+      ServerName server = cluster.getServerHoldingMeta();
+      if (server == null) {
+        LOG.warn("No server is holding -ROOT- right now.");
+        return;
+      }
+      restartRs(server, sleepTime);
+    }
+  }
+
+  /**
+   * Restarts a ratio of the running regionservers at the same time
+   */
+  private static class BatchRestartRs extends Action {
+    float ratio; //ratio of regionservers to restart
+
+    public BatchRestartRs(long sleepTime, float ratio) {
+      super(sleepTime);
+      this.ratio = ratio;
+    }
+
+    @Override
+    void init(ActionContext context) throws Exception {
+      super.init(context);
+    }
+
+    @Override
+    void perform() throws Exception {
+      LOG.info(String.format("Performing action: Batch restarting %d%% of region servers",
+          (int)(ratio * 100)));
+      List<ServerName> selectedServers = selectRandomItems(getCurrentServers(), ratio);
+
+      for (ServerName server : selectedServers) {
+        LOG.info("Killing region server:" + server);
+        cluster.killRegionServer(server);
+      }
+
+      for (ServerName server : selectedServers) {
+        cluster.waitForRegionServerToStop(server, TIMEOUT);
+      }
+
+      LOG.info("Killed " + selectedServers.size() + " region servers. Reported num of rs:"
+          + cluster.getClusterStatus().getServersSize());
+
+      sleep(sleepTime);
+
+      for (ServerName server : selectedServers) {
+        LOG.info("Starting region server:" + server.getHostname());
+        cluster.startRegionServer(server.getHostname());
+
+      }
+      for (ServerName server : selectedServers) {
+        cluster.waitForRegionServerToStart(server.getHostname(), TIMEOUT);
+      }
+      LOG.info("Started " + selectedServers.size() +" region servers. Reported num of rs:"
+          + cluster.getClusterStatus().getServersSize());
+    }
+  }
+
+  /**
+   * Restarts a ratio of the regionservers in a rolling fashion. At each step, either kills a
+   * server, or starts one, sleeping randomly (0-sleepTime) in between steps.
+   */
+  private static class RollingBatchRestartRs extends BatchRestartRs {
+    public RollingBatchRestartRs(long sleepTime, float ratio) {
+      super(sleepTime, ratio);
+    }
+
+    @Override
+    void perform() throws Exception {
+      LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers",
+          (int)(ratio * 100)));
+      Random random = new Random();
+      List<ServerName> selectedServers = selectRandomItems(getCurrentServers(), ratio);
+
+      Queue<ServerName> serversToBeKilled = new LinkedList<ServerName>(selectedServers);
+      Queue<ServerName> deadServers = new LinkedList<ServerName>();
+
+      //
+      while (!serversToBeKilled.isEmpty() || !deadServers.isEmpty()) {
+        boolean action = true; //action true = kill server, false = start server
+
+        if (serversToBeKilled.isEmpty() || deadServers.isEmpty()) {
+          action = deadServers.isEmpty();
+        } else {
+          action = random.nextBoolean();
+        }
+
+        if (action) {
+          ServerName server = serversToBeKilled.remove();
+          killRs(server);
+          deadServers.add(server);
+        } else {
+          ServerName server = deadServers.remove();
+          startRs(server);
+        }
+
+        sleep(random.nextInt((int)sleepTime));
+      }
+    }
+  }
+
+  /**
+   * A context for a Policy
+   */
+  private static class PolicyContext extends ActionContext {
+    PolicyContext(IntegrationTestingUtility util) {
+      super(util);
+    }
+  }
+
+  /**
+   * A policy to introduce chaos to the cluster
+   */
+  private static abstract class Policy extends StoppableImplementation implements Runnable {
+    PolicyContext context;
+    public void init(PolicyContext context) throws Exception {
+      this.context = context;
+    }
+  }
+
+  /**
+   * A policy, which picks a random action according to the given weights,
+   * and performs it every configurable period.
+   */
+  private static class PeriodicRandomActionPolicy extends Policy {
+    private long period;
+    private List<Pair<Action, Integer>> actions;
+
+    PeriodicRandomActionPolicy(long period, List<Pair<Action, Integer>> actions) {
+      this.period = period;
+      this.actions = actions;
+    }
+
+    @Override
+    public void run() {
+      //add some jitter
+      int jitter = new Random().nextInt((int)period);
+      LOG.info("Sleeping for " + jitter + " to add jitter");
+      Threads.sleep(jitter);
+
+      while (!isStopped()) {
+        long start = System.currentTimeMillis();
+        Action action = selectWeightedRandomItem(actions);
+
+        try {
+          action.perform();
+        } catch (Exception ex) {
+          LOG.warn("Exception occured during performing action: "
+              + StringUtils.stringifyException(ex));
+        }
+
+        long sleepTime = period - (System.currentTimeMillis() - start);
+        if (sleepTime > 0) {
+          LOG.info("Sleeping for:" + sleepTime);
+          Threads.sleep(sleepTime);
+        }
+      }
+    }
+
+    @Override
+    public void init(PolicyContext context) throws Exception {
+      super.init(context);
+      LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period:" + period);
+      for (Pair<Action, Integer> action : actions) {
+        action.getFirst().init(this.context);
+      }
+    }
+  }
+
+  /** Selects a random item from the given items */
+  static <T> T selectRandomItem(T[] items) {
+    Random random = new Random();
+    return items[random.nextInt(items.length)];
+  }
+
+  /** Selects a random item from the given items with weights*/
+  static <T> T selectWeightedRandomItem(List<Pair<T, Integer>> items) {
+    Random random = new Random();
+    int totalWeight = 0;
+    for (Pair<T, Integer> pair : items) {
+      totalWeight += pair.getSecond();
+    }
+
+    int cutoff = random.nextInt(totalWeight);
+    int cummulative = 0;
+    T item = null;
+
+    //warn: O(n)
+    for (int i=0; i<items.size(); i++) {
+      int curWeight = items.get(i).getSecond();
+      if ( cutoff < cummulative + curWeight) {
+        item = items.get(i).getFirst();
+        break;
+      }
+      cummulative += curWeight;
+    }
+
+    return item;
+  }
+
+  /** Selects and returns ceil(ratio * items.length) random items from the given array */
+  static <T> List<T> selectRandomItems(T[] items, float ratio) {
+    Random random = new Random();
+    int remaining = (int)Math.ceil(items.length * ratio);
+
+    List<T> selectedItems = new ArrayList<T>(remaining);
+
+    for (int i=0; i<items.length && remaining > 0; i++) {
+      if (random.nextFloat() < ((float)remaining/(items.length-i))) {
+        selectedItems.add(items[i]);
+        remaining--;
+      }
+    }
+
+    return selectedItems;
+  }
+
+  /**
+   * All actions that deal with RS's with the following weights (relative probabilities):
+   *  - Restart active master (sleep 5 sec)                    : 2
+   *  - Restart random regionserver (sleep 5 sec)              : 2
+   *  - Restart random regionserver (sleep 60 sec)             : 2
+   *  - Restart META regionserver (sleep 5 sec)                : 1
+   *  - Restart ROOT regionserver (sleep 5 sec)                : 1
+   *  - Batch restart of 50% of regionservers (sleep 5 sec)    : 2
+   *  - Rolling restart of 100% of regionservers (sleep 5 sec) : 2
+   */
+  @SuppressWarnings("unchecked")
+  private static final List<Pair<Action, Integer>> ALL_ACTIONS = Lists.newArrayList(
+      new Pair<Action,Integer>(new RestartActiveMaster(FIVE_SEC), 2),
+      new Pair<Action,Integer>(new RestartRandomRs(FIVE_SEC), 2),
+      new Pair<Action,Integer>(new RestartRandomRs(ONE_MIN), 2),
+      new Pair<Action,Integer>(new RestartRsHoldingMeta(FIVE_SEC), 1),
+      new Pair<Action,Integer>(new RestartRsHoldingRoot(FIVE_SEC), 1),
+      new Pair<Action,Integer>(new BatchRestartRs(FIVE_SEC, 0.5f), 2),
+      new Pair<Action,Integer>(new RollingBatchRestartRs(FIVE_SEC, 1.0f), 2)
+  );
+
+  public static final String EVERY_MINUTE_RANDOM_ACTION_POLICY = "EVERY_MINUTE_RANDOM_ACTION_POLICY";
+
+  private Policy[] policies;
+  private Thread[] monkeyThreads;
+
+  public void start() throws Exception {
+    monkeyThreads = new Thread[policies.length];
+
+    for (int i=0; i<policies.length; i++) {
+      policies[i].init(new PolicyContext(this.util));
+      Thread monkeyThread = new Thread(policies[i]);
+      monkeyThread.start();
+      monkeyThreads[i] = monkeyThread;
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    for (Policy policy : policies) {
+      policy.stop(why);
+    }
+  }
+
+  @Override
+  public boolean isStopped() {
+    return policies[0].isStopped();
+  }
+
+  /**
+   * Wait for ChaosMonkey to stop.
+   * @throws InterruptedException
+   */
+  public void waitForStop() throws InterruptedException {
+    for (Thread monkeyThread : monkeyThreads) {
+      monkeyThread.join();
+    }
+  }
+
+  private static final Map<String, Policy> NAMED_POLICIES = Maps.newHashMap();
+  static {
+    NAMED_POLICIES.put(EVERY_MINUTE_RANDOM_ACTION_POLICY,
+        new PeriodicRandomActionPolicy(ONE_MIN, ALL_ACTIONS));
+  }
+
+  @Override
+  protected void addOptions() {
+    addOptWithArg("policy", "a named policy defined in ChaosMonkey.java. Possible values: "
+        + NAMED_POLICIES.keySet());
+    //we can add more options, and make policies more configurable
+  }
+
+  @Override
+  protected void processOptions(CommandLine cmd) {
+    String[] policies = cmd.getOptionValues("policy");
+    if (policies != null) {
+      setPoliciesByName(policies);
+    }
+  }
+
+  @Override
+  protected int doWork() throws Exception {
+    start();
+    waitForStop();
+    return 0;
+  }
+
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    IntegrationTestingUtility.setUseDistributedCluster(conf);
+    IntegrationTestingUtility util = new IntegrationTestingUtility(conf);
+    util.initializeCluster(1);
+
+    ChaosMonkey monkey = new ChaosMonkey(util, EVERY_MINUTE_RANDOM_ACTION_POLICY);
+    int ret = ToolRunner.run(conf, monkey, args);
+    System.exit(ret);
+  }
+
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index 5b372ae0a439..b2d1e8f99d34 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -297,7 +297,7 @@ private void parseColumnFamilyOptions(CommandLine cmd) {
   }
 
   @Override
-  protected void doWork() throws IOException {
+  protected int doWork() throws IOException {
     if (cmd.hasOption(OPT_ZK_QUORUM)) {
       conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
     }
@@ -343,6 +343,16 @@ protected void doWork() throws IOException {
     if (isRead) {
       readerThreads.waitForFinish();
     }
+
+    boolean success = true;
+    if (isWrite) {
+      success = success && writerThreads.getNumWriteFailures() == 0;
+    }
+    if (isRead) {
+      success = success && readerThreads.getNumReadErrors() == 0
+          && readerThreads.getNumReadFailures() == 0;
+    }
+    return success ? 0 : 1;
   }
 
   public static void main(String[] args) {
diff --git a/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java b/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
index 7b9f8b031d64..b49aaee3785c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
@@ -89,7 +89,7 @@ private void loadData() throws IOException {
   }
 
   @Override
-  protected void doWork() throws IOException {
+  protected int doWork() throws Exception {
     ProcessBasedLocalHBaseCluster hbaseCluster =
         new ProcessBasedLocalHBaseCluster(conf, hbaseHome, numRegionServers);
 
@@ -130,6 +130,7 @@ protected void doWork() throws IOException {
           + Bytes.toStringBinary(result.getFamilyMap(HConstants.CATALOG_FAMILY)
               .get(HConstants.SERVER_QUALIFIER)));
     }
+    return 0;
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java b/src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java
new file mode 100644
index 000000000000..51a22f3d2406
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.Stoppable;
+
+/**
+ * A base implementation for a Stoppable service
+ */
+@InterfaceAudience.Private
+public class StoppableImplementation implements Stoppable {
+  volatile boolean stopped = false;
+
+  @Override
+  public void stop(String why) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isStopped() {
+    return stopped;
+  }
+}
\ No newline at end of file

From ef960718f36df557c8fc8b723660435187e3039d Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 26 Nov 2012 19:40:31 +0000
Subject: [PATCH 0592/1540] HBASE-7190 Add an option to hbck to check only meta
 and assignment

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1413800 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  42 +++--
 .../hadoop/hbase/util/TestHBaseFsck.java      | 154 ++++++++++++++++++
 2 files changed, 186 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 1d7f9dfec52c..4cdb11320f8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -176,6 +176,7 @@ public class HBaseFsck {
   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
   private boolean fixAssignments = false; // fix assignment errors?
   private boolean fixMeta = false; // fix meta errors?
+  private boolean checkHdfs = true; // load and check fs consistency?
   private boolean fixHdfsHoles = false; // fix fs holes?
   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
@@ -333,8 +334,8 @@ private void clearState() {
    */
   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
     // Initial pass to fix orphans.
-    if (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
-        || shouldFixHdfsOverlaps() || shouldFixTableOrphans()) {
+    if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
+        || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
       LOG.info("Loading regioninfos HDFS");
       // if nothing is happening this should always complete in two iterations.
       int maxIterations = conf.getInt("hbase.hbck.integrityrepair.iterations.max", 3);
@@ -391,8 +392,10 @@ public int onlineConsistencyRepair() throws IOException, KeeperException,
     loadDeployedRegions();
 
     // load regiondirs and regioninfos from HDFS
-    loadHdfsRegionDirs();
-    loadHdfsRegionInfos();
+    if (shouldCheckHdfs()) {
+      loadHdfsRegionDirs();
+      loadHdfsRegionInfos();
+    }
 
     // Empty cells in .META.?
     reportEmptyMetaCells();
@@ -791,7 +794,7 @@ public void fixOrphanTables() throws IOException {
       List<String> tmpList = new ArrayList<String>();
       tmpList.addAll(orphanTableDirs.keySet());
       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
-      Iterator iter = orphanTableDirs.entrySet().iterator();
+      Iterator<Entry<String, Set<String>>> iter = orphanTableDirs.entrySet().iterator();
       int j = 0; 
       int numFailedCase = 0;
       while (iter.hasNext()) {
@@ -1492,8 +1495,12 @@ private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
       errors.print(msg);
       undeployRegions(hbi);
       setShouldRerun();
-      HBaseFsckRepair.fixUnassigned(admin, hbi.getHdfsHRI());
-      HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
+      HRegionInfo hri = hbi.getHdfsHRI();
+      if (hri == null) {
+        hri = hbi.metaEntry;
+      }
+      HBaseFsckRepair.fixUnassigned(admin, hri);
+      HBaseFsckRepair.waitUntilAssigned(admin, hri);
     }
   }
 
@@ -1505,7 +1512,8 @@ private void checkRegionConsistency(final String key, final HbckInfo hbi)
     String descriptiveName = hbi.toString();
 
     boolean inMeta = hbi.metaEntry != null;
-    boolean inHdfs = hbi.getHdfsRegionDir()!= null;
+    // In case not checking HDFS, assume the region is on HDFS
+    boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
     boolean isDeployed = !hbi.deployedOn.isEmpty();
     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
@@ -1515,7 +1523,7 @@ private void checkRegionConsistency(final String key, final HbckInfo hbi)
     boolean splitParent =
       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
-    boolean recentlyModified = hbi.getHdfsRegionDir() != null &&
+    boolean recentlyModified = inHdfs &&
       hbi.getModTime() + timelag > System.currentTimeMillis();
 
     // ========== First the healthy cases =============
@@ -3128,6 +3136,14 @@ boolean shouldFixMeta() {
     return fixMeta;
   }
 
+  public void setCheckHdfs(boolean checking) {
+    checkHdfs = checking;
+  }
+
+  boolean shouldCheckHdfs() {
+    return checkHdfs;
+  }
+
   public void setFixHdfsHoles(boolean shouldFix) {
     fixHdfsHoles = shouldFix;
   }
@@ -3283,6 +3299,8 @@ protected HBaseFsck printUsageAndExit() {
     System.err.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
     System.err.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
     System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
+    System.err.println("   -noHdfsChecking   Don't load/check region info from HDFS."
+        + " Assumes META region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
     System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
     System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
     System.err.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
@@ -3386,6 +3404,8 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setFixAssignments(true);
       } else if (cmd.equals("-fixMeta")) {
         setFixMeta(true);
+      } else if (cmd.equals("-noHdfsChecking")) {
+        setCheckHdfs(false);
       } else if (cmd.equals("-fixHdfsHoles")) {
         setFixHdfsHoles(true);
       } else if (cmd.equals("-fixHdfsOrphans")) {
@@ -3417,6 +3437,7 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setFixVersionFile(true);
         setSidelineBigOverlaps(true);
         setFixSplitParents(false);
+        setCheckHdfs(true);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
         setFixHdfsHoles(true);
@@ -3426,6 +3447,7 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setFixHdfsOverlaps(false);
         setSidelineBigOverlaps(false);
         setFixSplitParents(false);
+        setCheckHdfs(true);
       } else if (cmd.equals("-maxOverlapsToSideline")) {
         if (i == args.length - 1) {
           System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
@@ -3537,7 +3559,7 @@ void debugLsr(Path p) throws IOException {
    * ls -r for debugging purposes
    */
   public static void debugLsr(Configuration conf, Path p) throws IOException {
-    if (!LOG.isDebugEnabled()) {
+    if (!LOG.isDebugEnabled() || p == null) {
       return;
     }
     FileSystem fs = p.getFileSystem(conf);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index ef664afdc604..6498ae409af3 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1312,6 +1312,160 @@ public void testMissingLastRegion() throws Exception {
     }
   }
 
+  /**
+   * Test -noHdfsChecking option can detect and fix assignments issue.
+   */
+  @Test
+  public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
+    String table = "testFixAssignmentsAndNoHdfsChecking";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by closing a region
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
+        Bytes.toBytes("B"), true, false, false, false);
+
+      // verify there is no other errors
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // verify that noHdfsChecking report the same errors
+      HBaseFsck fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setCheckHdfs(false);
+      fsck.onlineHbck();
+      assertErrors(fsck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // verify that fixAssignments works fine with noHdfsChecking
+      fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setCheckHdfs(false);
+      fsck.setFixAssignments(true);
+      fsck.onlineHbck();
+      assertTrue(fsck.shouldRerun());
+      fsck.onlineHbck();
+      assertNoErrors(fsck);
+
+      assertEquals(ROWKEYS.length, countRows());
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * Test -noHdfsChecking option can detect region is not in meta but deployed.
+   * However, it can not fix it without checking Hdfs because we need to get
+   * the region info from Hdfs in this case, then to patch the meta.
+   */
+  @Test
+  public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
+    String table = "testFixMetaNotWorkingWithNoHdfsChecking";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by deleting a region from the metadata
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
+        Bytes.toBytes("B"), false, true, false, false);
+
+      // verify there is no other errors
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // verify that noHdfsChecking report the same errors
+      HBaseFsck fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setCheckHdfs(false);
+      fsck.onlineHbck();
+      assertErrors(fsck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // verify that fixMeta doesn't work with noHdfsChecking
+      fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setCheckHdfs(false);
+      fsck.setFixAssignments(true);
+      fsck.setFixMeta(true);
+      fsck.onlineHbck();
+      assertFalse(fsck.shouldRerun());
+      assertErrors(fsck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+    } finally {
+      deleteTable(table);
+    }
+  }
+
+  /**
+   * Test -fixHdfsHoles doesn't work with -noHdfsChecking option,
+   * and -noHdfsChecking can't detect orphan Hdfs region.
+   */
+  @Test
+  public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
+    String table = "testFixHdfsHolesNotWorkingWithNoHdfsChecking";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by creating an overlap in the metadata
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
+        Bytes.toBytes("B"), true, true, false, true);
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
+
+      HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
+        Bytes.toBytes("A2"), Bytes.toBytes("B"));
+      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
+      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+        .waitForAssignment(hriOverlap);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
+        ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // verify that noHdfsChecking can't detect ORPHAN_HDFS_REGION
+      HBaseFsck fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setCheckHdfs(false);
+      fsck.onlineHbck();
+      assertErrors(fsck, new ERROR_CODE[] {
+        ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // verify that fixHdfsHoles doesn't work with noHdfsChecking
+      fsck = new HBaseFsck(conf);
+      fsck.connect();
+      fsck.setDisplayFullReport(); // i.e. -details
+      fsck.setTimeLag(0);
+      fsck.setCheckHdfs(false);
+      fsck.setFixHdfsHoles(true);
+      fsck.setFixHdfsOverlaps(true);
+      fsck.setFixHdfsOrphans(true);
+      fsck.onlineHbck();
+      assertFalse(fsck.shouldRerun());
+      assertErrors(fsck, new ERROR_CODE[] {
+        ERROR_CODE.HOLE_IN_REGION_CHAIN});
+    } finally {
+      if (TEST_UTIL.getHBaseAdmin().isTableDisabled(table)) {
+        TEST_UTIL.getHBaseAdmin().enableTable(table);
+      }
+      deleteTable(table);
+    }
+  }
+
   /**
    * We don't have an easy way to verify that a flush completed, so we loop until we find a
    * legitimate hfile and return it.

From fba12fd0d7be162e34216eb3b8b7df6ea4f657dc Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Tue, 27 Nov 2012 16:12:35 +0000
Subject: [PATCH 0593/1540] HBASE-7214: CleanerChore logs too much, so much so
 it obscures all else that is going on

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1414244 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/cleaner/CleanerChore.java    | 12 +++++++++---
 .../hbase/master/cleaner/TimeToLiveHFileCleaner.java |  5 ++++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index c5be87f989cc..9c350c942dd4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -143,7 +143,9 @@ protected void chore() {
    * @throws IOException if there is an unexpected filesystem error
    */
   private boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
-    LOG.debug("Checking directory: " + toCheck);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Checking directory: " + toCheck);
+    }
     FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
     // if the directory doesn't exist, then we are done
     if (children == null) return true;
@@ -195,12 +197,16 @@ private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumen
 
       if (!cleaner.isFileDeletable(filePath)) {
         // this file is not deletable, then we are done
-        LOG.debug(filePath + " is not deletable according to:" + cleaner);
+        if (LOG.isTraceEnabled()) {
+          LOG.trace(filePath + " is not deletable according to:" + cleaner);
+        }
         return false;
       }
     }
     // delete this file if it passes all the cleaners
-    LOG.debug("Removing:" + filePath + " from archive");
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Removing:" + filePath + " from archive");
+    }
     boolean success = this.fs.delete(filePath, false);
     if (!success) {
       LOG.warn("Attempted to delete:" + filePath
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
index 40877b75c24f..82231afbe4f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
@@ -65,7 +65,10 @@ public boolean isFileDeletable(Path filePath) {
       return false;
     }
     long life = currentTime - time;
-    LOG.debug("Life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " + time);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: "
+          + time);
+    }
     if (life < 0) {
       LOG.warn("Found a log (" + filePath + ") newer than current time (" + currentTime + " < "
           + time + "), probably a clock skew");

From 7626e9b3b7e6a895b43d61b36c166bf6ec2b6123 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 28 Nov 2012 20:08:39 +0000
Subject: [PATCH 0594/1540] HBASE-7231 port HBASE-7200 create integration test
 for balancing regions and killing region servers to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1414890 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/IngestIntegrationTestBase.java      | 123 ++++++++++++
 ...egrationTestDataIngestWithChaosMonkey.java |  81 +-------
 ...ntegrationTestRebalanceAndKillServers.java | 100 ++++++++++
 .../apache/hadoop/hbase/util/ChaosMonkey.java | 184 +++++++++++++-----
 .../hadoop/hbase/util/LoadTestTool.java       |  59 ++++--
 5 files changed, 417 insertions(+), 130 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java

diff --git a/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java b/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java
new file mode 100644
index 000000000000..9c603ae09ea9
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.LoadTestTool;
+
+/**
+ * A base class for tests that do something with the cluster while running
+ * {@link LoadTestTool} to write and verify some data.
+ */
+public abstract class IngestIntegrationTestBase {
+  private static String tableName = null;
+
+  /** A soft limit on how long we should run */
+  private static final String RUN_TIME_KEY = "hbase.%s.runtime";
+
+  protected static final Log LOG = LogFactory.getLog(IngestIntegrationTestBase.class);
+  protected IntegrationTestingUtility util;
+  protected HBaseCluster cluster;
+  private LoadTestTool loadTool;
+
+  protected void setUp(int numSlavesBase) throws Exception {
+    tableName = this.getClass().getSimpleName();
+    util = new IntegrationTestingUtility();
+    LOG.info("Initializing cluster with " + numSlavesBase + " servers");
+    util.initializeCluster(numSlavesBase);
+    LOG.info("Done initializing cluster");
+    cluster = util.getHBaseClusterInterface();
+    deleteTableIfNecessary();
+    loadTool = new LoadTestTool();
+    loadTool.setConf(util.getConfiguration());
+    // Initialize load test tool before we start breaking things;
+    // LoadTestTool init, even when it is a no-op, is very fragile.
+    int ret = loadTool.run(new String[] { "-tn", tableName, "-init_only" });
+    Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret);
+  }
+
+  protected void tearDown() throws Exception {
+    LOG.info("Restoring the cluster");
+    util.restoreCluster();
+    LOG.info("Done restoring the cluster");
+  }
+
+  private void deleteTableIfNecessary() throws IOException {
+    if (util.getHBaseAdmin().tableExists(tableName)) {
+      util.deleteTable(Bytes.toBytes(tableName));
+    }
+  }
+
+  protected void runIngestTest(long defaultRunTime, int keysPerServerPerIter,
+      int colsPerKey, int recordSize, int writeThreads) throws Exception {
+    LOG.info("Running ingest");
+    LOG.info("Cluster size:" + util.getHBaseClusterInterface().getClusterStatus().getServersSize());
+
+    long start = System.currentTimeMillis();
+    String runtimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName());
+    long runtime = util.getConfiguration().getLong(runtimeKey, defaultRunTime);
+    long startKey = 0;
+
+    long numKeys = getNumKeys(keysPerServerPerIter);
+    while (System.currentTimeMillis() - start < 0.9 * runtime) {
+      LOG.info("Intended run time: " + (runtime/60000) + " min, left:" +
+          ((runtime - (System.currentTimeMillis() - start))/60000) + " min");
+
+      int ret = loadTool.run(new String[] {
+          "-tn", tableName,
+          "-write", String.format("%d:%d:%d", colsPerKey, recordSize, writeThreads),
+          "-start_key", String.valueOf(startKey),
+          "-num_keys", String.valueOf(numKeys),
+          "-skip_init"
+      });
+      if (0 != ret) {
+        String errorMsg = "Load failed with error code " + ret;
+        LOG.error(errorMsg);
+        Assert.fail(errorMsg);
+      }
+
+      ret = loadTool.run(new String[] {
+          "-tn", tableName,
+          "-read", "100:20",
+          "-start_key", String.valueOf(startKey),
+          "-num_keys", String.valueOf(numKeys),
+          "-skip_init"
+      });
+      if (0 != ret) {
+        String errorMsg = "Verification failed with error code " + ret;
+        LOG.error(errorMsg);
+        Assert.fail(errorMsg);
+      }
+      startKey += numKeys;
+    }
+  }
+
+  /** Estimates a data size based on the cluster size */
+  private long getNumKeys(int keysPerServer)
+      throws IOException {
+    int numRegionServers = cluster.getClusterStatus().getServersSize();
+    return keysPerServer * numRegionServers;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
index 72f76b0dcfea..842ecf96bc1e 100644
--- a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
@@ -22,11 +22,8 @@
 
 import junit.framework.Assert;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChaosMonkey;
-import org.apache.hadoop.hbase.util.LoadTestTool;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -39,95 +36,33 @@
  * configuration parameter.
  */
 @Category(IntegrationTests.class)
-public class IntegrationTestDataIngestWithChaosMonkey {
+public class IntegrationTestDataIngestWithChaosMonkey extends IngestIntegrationTestBase {
 
-  private static final String TABLE_NAME = "TestDataIngestWithChaosMonkey";
   private static final int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
 
-  /** A soft limit on how long we should run */
-  private static final String RUN_TIME_KEY = "hbase.IntegrationTestDataIngestWithChaosMonkey.runtime";
-
-  //run for 5 min by default
+  // run for 5 min by default
   private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000;
 
-  private static final Log LOG = LogFactory.getLog(IntegrationTestDataIngestWithChaosMonkey.class);
-  private IntegrationTestingUtility util;
-  private HBaseCluster cluster;
   private ChaosMonkey monkey;
 
   @Before
   public void setUp() throws Exception {
-    util = new IntegrationTestingUtility();
-
-    util.initializeCluster(NUM_SLAVES_BASE);
-
-    cluster = util.getHBaseClusterInterface();
-    deleteTableIfNecessary();
-
+    super.setUp(NUM_SLAVES_BASE);
     monkey = new ChaosMonkey(util, ChaosMonkey.EVERY_MINUTE_RANDOM_ACTION_POLICY);
     monkey.start();
   }
 
   @After
   public void tearDown() throws Exception {
-    monkey.stop("test has finished, that's why");
-    monkey.waitForStop();
-    util.restoreCluster();
-  }
-
-  private void deleteTableIfNecessary() throws IOException {
-    if (util.getHBaseAdmin().tableExists(TABLE_NAME)) {
-      util.deleteTable(Bytes.toBytes(TABLE_NAME));
+    if (monkey != null) {
+      monkey.stop("test has finished, that's why");
+      monkey.waitForStop();
     }
+    super.tearDown();
   }
 
   @Test
   public void testDataIngest() throws Exception {
-    LOG.info("Running testDataIngest");
-    LOG.info("Cluster size:" + util.getHBaseClusterInterface().getClusterStatus().getServersSize());
-
-    LoadTestTool loadTool = new LoadTestTool();
-    loadTool.setConf(util.getConfiguration());
-
-    long start = System.currentTimeMillis();
-    long runtime = util.getConfiguration().getLong(RUN_TIME_KEY, DEFAULT_RUN_TIME);
-    long startKey = 0;
-
-    long numKeys = estimateDataSize();
-    while (System.currentTimeMillis() - start < 0.9 * runtime) {
-      LOG.info("Intended run time: " + (runtime/60000) + " min, left:" +
-          ((runtime - (System.currentTimeMillis() - start))/60000) + " min");
-
-      int ret = loadTool.run(new String[] {
-          "-tn", TABLE_NAME,
-          "-write", "10:100:20",
-          "-start_key", String.valueOf(startKey),
-          "-num_keys", String.valueOf(numKeys)
-      });
-
-      //assert that load was successful
-      Assert.assertEquals(0, ret);
-
-      ret = loadTool.run(new String[] {
-          "-tn", TABLE_NAME,
-          "-read", "100:20",
-          "-start_key", String.valueOf(startKey),
-          "-num_keys", String.valueOf(numKeys)
-      });
-
-      //assert that verify was successful
-      Assert.assertEquals(0, ret);
-      startKey += numKeys;
-    }
-  }
-
-  /** Estimates a data size based on the cluster size */
-  protected long estimateDataSize() throws IOException {
-    //base is a 4 slave node cluster.
-    ClusterStatus status = cluster.getClusterStatus();
-    int numRegionServers = status.getServersSize();
-    int multiplication = Math.max(1, numRegionServers / NUM_SLAVES_BASE);
-
-    return 10000 * multiplication;
+    runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 100, 20);
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java
new file mode 100644
index 000000000000..c6d761ec8df1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChaosMonkey;
+import org.apache.hadoop.hbase.util.LoadTestTool;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.ChaosMonkey.Action;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartActiveMaster;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRandomRs;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import com.google.common.collect.Lists;
+
+/**
+ * A system test which does large data ingestion and verify using {@link LoadTestTool},
+ * while killing the region servers and the master(s) randomly. You can configure how long
+ * should the load test run by using "hbase.IntegrationTestRebalanceAndKillServers  s.runtime"
+ * configuration parameter.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestRebalanceAndKillServers extends IngestIntegrationTestBase {
+  private static final int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
+  private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000; // run for 5 min by default
+
+  private static final long KILL_SERVICE_EVERY_MS = 45 * 1000;
+  private static final int SERVER_PER_MASTER_KILL = 3;
+  private static final long KILL_SERVER_FOR_MS = 5 * 1000;
+  private static final long KILL_MASTER_FOR_MS = 100;
+
+  private static final long UNBALANCE_REGIONS_EVERY_MS = 30 * 1000;
+  /** @see ChaosMonkey.UnbalanceRegionsAction#UnbalanceRegionsAction(double, double) */
+  private static final double UNBALANCE_TO_FRC_OF_SERVERS = 0.5;
+  /** @see ChaosMonkey.UnbalanceRegionsAction#UnbalanceRegionsAction(double, double) */
+  private static final double UNBALANCE_FRC_OF_REGIONS = 0.5;
+
+  private static final long BALANCE_REGIONS_EVERY_MS = 10 * 1000;
+
+  private ChaosMonkey monkey;
+
+  @Before
+  @SuppressWarnings("unchecked")
+  public void setUp() throws Exception {
+    super.setUp(NUM_SLAVES_BASE);
+
+    ChaosMonkey.Policy killPolicy = new ChaosMonkey.PeriodicRandomActionPolicy(
+      KILL_SERVICE_EVERY_MS,
+      new Pair<Action,Integer>(new ChaosMonkey.RestartActiveMaster(KILL_MASTER_FOR_MS), 1),
+      new Pair<Action,Integer>(new ChaosMonkey.RestartRandomRs(KILL_SERVER_FOR_MS), SERVER_PER_MASTER_KILL));
+
+    ChaosMonkey.Policy unbalancePolicy = new ChaosMonkey.PeriodicRandomActionPolicy(
+      UNBALANCE_REGIONS_EVERY_MS,
+      new ChaosMonkey.UnbalanceRegionsAction(UNBALANCE_FRC_OF_REGIONS, UNBALANCE_TO_FRC_OF_SERVERS));
+
+    ChaosMonkey.Policy balancePolicy = new ChaosMonkey.PeriodicRandomActionPolicy(
+      BALANCE_REGIONS_EVERY_MS, new ChaosMonkey.ForceBalancerAction());
+
+    monkey = new ChaosMonkey(util, killPolicy, unbalancePolicy, balancePolicy);
+    monkey.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (monkey != null) {
+      monkey.stop("tearDown");
+      monkey.waitForStop();
+    }
+    super.tearDown();
+  }
+
+  @Test
+  public void testDataIngest() throws Exception {
+    runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 100, 20);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index 2d67a74c90cc..ad63e6e956f1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -20,12 +20,15 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Random;
+import java.util.Set;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.logging.Log;
@@ -34,15 +37,19 @@
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseCluster;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.IntegrationTestingUtility;
 import org.apache.hadoop.hbase.IntegrationTestDataIngestWithChaosMonkey;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.protobuf.ServiceException;
 
 /**
  * A utility to injects faults in a running cluster.
@@ -86,6 +93,16 @@ public ChaosMonkey(IntegrationTestingUtility util, String... policies) {
     setPoliciesByName(policies);
   }
 
+  /**
+   * Construct a new ChaosMonkey
+   * @param util the HBaseIntegrationTestingUtility already configured
+   * @param policies custom policies to use
+   */
+  public ChaosMonkey(IntegrationTestingUtility util, Policy... policies) {
+    this.util = util;
+    this.policies = policies;
+  }
+
   private void setPoliciesByName(String... policies) {
     this.policies = new Policy[policies.length];
     for (int i=0; i < policies.length; i++) {
@@ -115,16 +132,16 @@ HBaseCluster getHBaseCluster() {
   /**
    * A (possibly mischievous) action that the ChaosMonkey can perform.
    */
-  private static class Action {
-    long sleepTime; //how long should we sleep
-    ActionContext context;
-    HBaseCluster cluster;
-    ClusterStatus initialStatus;
-    ServerName[] initialServers;
-
-    public Action(long sleepTime) {
-      this.sleepTime = sleepTime;
-    }
+  public static class Action {
+    // TODO: interesting question - should actions be implemented inside
+    //       ChaosMonkey, or outside? If they are inside (initial), the class becomes
+    //       huge and all-encompassing; if they are outside ChaosMonkey becomes just
+    //       a random task scheduler. For now, keep inside.
+
+    protected ActionContext context;
+    protected HBaseCluster cluster;
+    protected ClusterStatus initialStatus;
+    protected ServerName[] initialServers;
 
     void init(ActionContext context) throws Exception {
       this.context = context;
@@ -136,33 +153,28 @@ void init(ActionContext context) throws Exception {
 
     void perform() throws Exception { };
 
+    // TODO: perhaps these methods should be elsewhere?
     /** Returns current region servers */
-    ServerName[] getCurrentServers() throws IOException {
+    protected ServerName[] getCurrentServers() throws IOException {
       Collection<ServerName> regionServers = cluster.getClusterStatus().getServers();
       return regionServers.toArray(new ServerName[regionServers.size()]);
     }
 
-    void killMaster(ServerName server) throws IOException {
+    protected void killMaster(ServerName server) throws IOException {
       LOG.info("Killing master:" + server);
       cluster.killMaster(server);
       cluster.waitForMasterToStop(server, TIMEOUT);
       LOG.info("Killed master server:" + server);
     }
 
-    void startMaster(ServerName server) throws IOException {
+    protected void startMaster(ServerName server) throws IOException {
       LOG.info("Starting master:" + server.getHostname());
       cluster.startMaster(server.getHostname());
       cluster.waitForActiveAndReadyMaster(TIMEOUT);
       LOG.info("Started master: " + server);
     }
 
-    void restartMaster(ServerName server, long sleepTime) throws IOException {
-      killMaster(server);
-      sleep(sleepTime);
-      startMaster(server);
-    }
-
-    void killRs(ServerName server) throws IOException {
+    protected void killRs(ServerName server) throws IOException {
       LOG.info("Killing region server:" + server);
       cluster.killRegionServer(server);
       cluster.waitForRegionServerToStop(server, TIMEOUT);
@@ -170,19 +182,33 @@ void killRs(ServerName server) throws IOException {
           + cluster.getClusterStatus().getServersSize());
     }
 
-    void startRs(ServerName server) throws IOException {
+    protected void startRs(ServerName server) throws IOException {
       LOG.info("Starting region server:" + server.getHostname());
       cluster.startRegionServer(server.getHostname());
       cluster.waitForRegionServerToStart(server.getHostname(), TIMEOUT);
       LOG.info("Started region server:" + server + ". Reported num of rs:"
           + cluster.getClusterStatus().getServersSize());
     }
+  }
+
+  private static class RestartActionBase extends Action {
+    long sleepTime; // how long should we sleep
+
+    public RestartActionBase(long sleepTime) {
+      this.sleepTime = sleepTime;
+    }
 
     void sleep(long sleepTime) {
       LOG.info("Sleeping for:" + sleepTime);
       Threads.sleep(sleepTime);
     }
 
+    void restartMaster(ServerName server, long sleepTime) throws IOException {
+      killMaster(server);
+      sleep(sleepTime);
+      startMaster(server);
+    }
+
     void restartRs(ServerName server, long sleepTime) throws IOException {
       killRs(server);
       sleep(sleepTime);
@@ -190,7 +216,7 @@ void restartRs(ServerName server, long sleepTime) throws IOException {
     }
   }
 
-  private static class RestartActiveMaster extends Action {
+  public static class RestartActiveMaster extends RestartActionBase {
     public RestartActiveMaster(long sleepTime) {
       super(sleepTime);
     }
@@ -203,16 +229,11 @@ void perform() throws Exception {
     }
   }
 
-  private static class RestartRandomRs extends Action {
+  public static class RestartRandomRs extends RestartActionBase {
     public RestartRandomRs(long sleepTime) {
       super(sleepTime);
     }
 
-    @Override
-    void init(ActionContext context) throws Exception {
-      super.init(context);
-    }
-
     @Override
     void perform() throws Exception {
       LOG.info("Performing action: Restart random region server");
@@ -222,7 +243,7 @@ void perform() throws Exception {
     }
   }
 
-  private static class RestartRsHoldingMeta extends RestartRandomRs {
+  public static class RestartRsHoldingMeta extends RestartRandomRs {
     public RestartRsHoldingMeta(long sleepTime) {
       super(sleepTime);
     }
@@ -238,7 +259,7 @@ void perform() throws Exception {
     }
   }
 
-  private static class RestartRsHoldingRoot extends RestartRandomRs {
+  public static class RestartRsHoldingRoot extends RestartRandomRs {
     public RestartRsHoldingRoot(long sleepTime) {
       super(sleepTime);
     }
@@ -257,7 +278,7 @@ void perform() throws Exception {
   /**
    * Restarts a ratio of the running regionservers at the same time
    */
-  private static class BatchRestartRs extends Action {
+  public static class BatchRestartRs extends RestartActionBase {
     float ratio; //ratio of regionservers to restart
 
     public BatchRestartRs(long sleepTime, float ratio) {
@@ -265,11 +286,6 @@ public BatchRestartRs(long sleepTime, float ratio) {
       this.ratio = ratio;
     }
 
-    @Override
-    void init(ActionContext context) throws Exception {
-      super.init(context);
-    }
-
     @Override
     void perform() throws Exception {
       LOG.info(String.format("Performing action: Batch restarting %d%% of region servers",
@@ -307,7 +323,7 @@ void perform() throws Exception {
    * Restarts a ratio of the regionservers in a rolling fashion. At each step, either kills a
    * server, or starts one, sleeping randomly (0-sleepTime) in between steps.
    */
-  private static class RollingBatchRestartRs extends BatchRestartRs {
+  public static class RollingBatchRestartRs extends BatchRestartRs {
     public RollingBatchRestartRs(long sleepTime, float ratio) {
       super(sleepTime, ratio);
     }
@@ -346,6 +362,71 @@ void perform() throws Exception {
     }
   }
 
+  public static class UnbalanceRegionsAction extends Action {
+    private double fractionOfRegions;
+    private double fractionOfServers;
+    private Random random = new Random();
+
+    /**
+     * Unbalances the regions on the cluster by choosing "target" servers, and moving
+     * some regions from each of the non-target servers to random target servers.
+     * @param fractionOfRegions Fraction of regions to move from each server.
+     * @param fractionOfServers Fraction of servers to be chosen as targets.
+     */
+    public UnbalanceRegionsAction(double fractionOfRegions, double fractionOfServers) {
+      this.fractionOfRegions = fractionOfRegions;
+      this.fractionOfServers = fractionOfServers;
+    }
+
+    @Override
+    void perform() throws Exception {
+      LOG.info("Unbalancing regions");
+      ClusterStatus status = this.cluster.getClusterStatus();
+      List<ServerName> victimServers = new LinkedList<ServerName>(status.getServers());
+      int targetServerCount = (int)Math.ceil(fractionOfServers * victimServers.size());
+      List<byte[]> targetServers = new ArrayList<byte[]>(targetServerCount);
+      for (int i = 0; i < targetServerCount; ++i) {
+        int victimIx = random.nextInt(victimServers.size());
+        String serverName = victimServers.remove(victimIx).getServerName();
+        targetServers.add(Bytes.toBytes(serverName));
+      }
+
+      List<byte[]> victimRegions = new LinkedList<byte[]>();
+      for (ServerName server : victimServers) {
+        HServerLoad serverLoad = status.getLoad(server);
+        // Ugh.
+        List<byte[]> regions = new LinkedList<byte[]>(serverLoad.getRegionsLoad().keySet());
+        int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size());
+        LOG.debug("Removing " + victimRegionCount + " regions from " + server.getServerName());
+        for (int i = 0; i < victimRegionCount; ++i) {
+          int victimIx = random.nextInt(regions.size());
+          String regionId = HRegionInfo.encodeRegionName(regions.remove(victimIx));
+          victimRegions.add(Bytes.toBytes(regionId));
+        }
+      }
+
+      LOG.info("Moving " + victimRegions.size() + " regions from " + victimServers.size()
+          + " servers to " + targetServers.size() + " different servers");
+      HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
+      for (byte[] victimRegion : victimRegions) {
+        int targetIx = random.nextInt(targetServers.size());
+        admin.move(victimRegion, targetServers.get(targetIx));
+      }
+    }
+  }
+
+  public static class ForceBalancerAction extends Action {
+    @Override
+    void perform() throws Exception {
+      LOG.info("Balancing regions");
+      HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
+      boolean result = admin.balancer();
+      if (!result) {
+        LOG.error("Balancer didn't succeed");
+      }
+    }
+  }
+
   /**
    * A context for a Policy
    */
@@ -358,7 +439,7 @@ private static class PolicyContext extends ActionContext {
   /**
    * A policy to introduce chaos to the cluster
    */
-  private static abstract class Policy extends StoppableImplementation implements Runnable {
+  public static abstract class Policy extends StoppableImplementation implements Runnable {
     PolicyContext context;
     public void init(PolicyContext context) throws Exception {
       this.context = context;
@@ -369,19 +450,32 @@ public void init(PolicyContext context) throws Exception {
    * A policy, which picks a random action according to the given weights,
    * and performs it every configurable period.
    */
-  private static class PeriodicRandomActionPolicy extends Policy {
-    private long period;
+  public static class PeriodicRandomActionPolicy extends Policy {
+    private long periodMs;
     private List<Pair<Action, Integer>> actions;
 
-    PeriodicRandomActionPolicy(long period, List<Pair<Action, Integer>> actions) {
-      this.period = period;
+    public PeriodicRandomActionPolicy(long periodMs, List<Pair<Action, Integer>> actions) {
+      this.periodMs = periodMs;
       this.actions = actions;
     }
 
+    public PeriodicRandomActionPolicy(long periodMs, Pair<Action, Integer>... actions) {
+      // We don't expect it to be modified.
+      this(periodMs, Arrays.asList(actions));
+    }
+
+    public PeriodicRandomActionPolicy(long periodMs, Action... actions) {
+      this.periodMs = periodMs;
+      this.actions = new ArrayList<Pair<Action, Integer>>(actions.length);
+      for (Action action : actions) {
+        this.actions.add(new Pair<Action, Integer>(action, 1));
+      }
+    }
+
     @Override
     public void run() {
       //add some jitter
-      int jitter = new Random().nextInt((int)period);
+      int jitter = new Random().nextInt((int)periodMs);
       LOG.info("Sleeping for " + jitter + " to add jitter");
       Threads.sleep(jitter);
 
@@ -396,7 +490,7 @@ public void run() {
               + StringUtils.stringifyException(ex));
         }
 
-        long sleepTime = period - (System.currentTimeMillis() - start);
+        long sleepTime = periodMs - (System.currentTimeMillis() - start);
         if (sleepTime > 0) {
           LOG.info("Sleeping for:" + sleepTime);
           Threads.sleep(sleepTime);
@@ -407,7 +501,7 @@ public void run() {
     @Override
     public void init(PolicyContext context) throws Exception {
       super.init(context);
-      LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period:" + period);
+      LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period:" + periodMs);
       for (Pair<Action, Integer> action : actions) {
         action.getFirst().init(this.context);
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index b2d1e8f99d34..4d73ba8be107 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -95,6 +95,8 @@ public class LoadTestTool extends AbstractHBaseTool {
   private static final String OPT_START_KEY = "start_key";
   private static final String OPT_TABLE_NAME = "tn";
   private static final String OPT_ZK_QUORUM = "zk";
+  private static final String OPT_SKIP_INIT = "skip_init";
+  private static final String OPT_INIT_ONLY = "init_only";
 
   private static final long DEFAULT_START_KEY = 0;
 
@@ -126,6 +128,11 @@ public class LoadTestTool extends AbstractHBaseTool {
   private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
   private int verifyPercent;
 
+  // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
+  //       console tool itself should only be used from console.
+  private boolean isSkipInit = false;
+  private boolean isInitOnly = false;
+
   private String[] splitColonSeparated(String option,
       int minNumCols, int maxNumCols) {
     String optVal = cmd.getOptionValue(option);
@@ -186,6 +193,7 @@ protected void addOptions() {
     addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write");
     addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD);
     addOptWithArg(OPT_READ, OPT_USAGE_READ);
+    addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading");
     addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM);
     addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION);
     addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE);
@@ -200,10 +208,12 @@ protected void addOptions() {
         "separate puts for every column in a row");
     addOptNoArg(OPT_ENCODE_IN_CACHE_ONLY, OPT_ENCODE_IN_CACHE_ONLY_USAGE);
 
-    addRequiredOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
+    addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
     addOptWithArg(OPT_START_KEY, "The first key to read/write " +
         "(a 0-based index). The default value is " +
         DEFAULT_START_KEY + ".");
+    addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table "
+        + "already exists");
   }
 
   @Override
@@ -212,20 +222,35 @@ protected void processOptions(CommandLine cmd) {
 
     tableName = Bytes.toBytes(cmd.getOptionValue(OPT_TABLE_NAME,
         DEFAULT_TABLE_NAME));
-    startKey = parseLong(cmd.getOptionValue(OPT_START_KEY,
-        String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE);
-    long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1,
-        Long.MAX_VALUE - startKey);
-    endKey = startKey + numKeys;
 
     isWrite = cmd.hasOption(OPT_WRITE);
     isRead = cmd.hasOption(OPT_READ);
+    isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
 
-    if (!isWrite && !isRead) {
+    if (!isWrite && !isRead && !isInitOnly) {
       throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " +
           "-" + OPT_READ + " has to be specified");
     }
 
+    if (isInitOnly && (isRead || isWrite)) {
+      throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with"
+          + " either -" + OPT_WRITE + " or -" + OPT_READ);
+    }
+
+    if (!isInitOnly) {
+      if (!cmd.hasOption(OPT_NUM_KEYS)) {
+        throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in "
+            + "read or write mode");
+      }
+      startKey = parseLong(cmd.getOptionValue(OPT_START_KEY,
+          String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE);
+      long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1,
+          Long.MAX_VALUE - startKey);
+      endKey = startKey + numKeys;
+      isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
+      System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
+    }
+
     encodeInCacheOnly = cmd.hasOption(OPT_ENCODE_IN_CACHE_ONLY);
     parseColumnFamilyOptions(cmd);
 
@@ -274,8 +299,6 @@ protected void processOptions(CommandLine cmd) {
       System.out.println("Percent of keys to verify: " + verifyPercent);
       System.out.println("Reader threads: " + numReaderThreads);
     }
-
-    System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
   }
 
   private void parseColumnFamilyOptions(CommandLine cmd) {
@@ -296,15 +319,27 @@ private void parseColumnFamilyOptions(CommandLine cmd) {
         StoreFile.BloomType.valueOf(bloomStr);
   }
 
+  public void initTestTable() throws IOException {
+    HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName,
+        COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo);
+    applyColumnFamilyOptions(tableName, COLUMN_FAMILIES);
+  }
+
   @Override
   protected int doWork() throws IOException {
     if (cmd.hasOption(OPT_ZK_QUORUM)) {
       conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
     }
 
-    HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName,
-        COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo);
-    applyColumnFamilyOptions(tableName, COLUMN_FAMILIES);
+    if (isInitOnly) {
+      LOG.info("Initializing only; no reads or writes");
+      initTestTable();
+      return 0;
+    }
+
+    if (!isSkipInit) {
+      initTestTable();
+    }
 
     if (isWrite) {
       writerThreads = new MultiThreadedWriter(conf, tableName, COLUMN_FAMILY);

From 03d93a1cdc99420dae9ad65561db9579f4b9fb81 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 28 Nov 2012 22:40:18 +0000
Subject: [PATCH 0595/1540] HBASE-7172
 TestSplitLogManager.testVanishingTaskZNode() fails when run individually and
 is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1414975 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/SplitLogManager.java  |  4 +-
 .../hbase/master/TestSplitLogManager.java     | 67 ++++++++-----------
 2 files changed, 30 insertions(+), 41 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 16d4f13352bb..58d537131298 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -338,7 +338,9 @@ private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {
             LOG.warn("No more task remaining (ZK or task map), splitting "
               + "should have completed. Remaining tasks in ZK " + remainingInZK
               + ", active tasks in map " + actual);
-            return;
+            if (remainingInZK == 0 && actual == 0) {
+              return;
+            }
           }
           batch.wait(100);
           if (stopper.isStopped()) {
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 582a802a4ca7..649c26e0f67b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -65,6 +65,7 @@ public class TestSplitLogManager {
   private static boolean stopped = false;
   private SplitLogManager slm;
   private Configuration conf;
+  private int to;
 
   private final static HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
@@ -105,6 +106,11 @@ public void setup() throws Exception {
 
     stopped = false;
     resetCounters();
+    to = 4000;
+    conf.setInt("hbase.splitlog.manager.timeout", to);
+    conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to);
+    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
+    to = to + 4 * 100;
   }
 
   @After
@@ -194,26 +200,20 @@ public void testOrphanTaskAcquisition() throws Exception {
         TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-    int to = 1000;
-    conf.setInt("hbase.splitlog.manager.timeout", to);
-    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
-    to = to + 2 * 100;
-
-
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
     waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, 100);
     Task task = slm.findOrCreateOrphanTask(tasknode);
     assertTrue(task.isOrphan());
-    waitForCounter(tot_mgr_heartbeat, 0, 1, 100);
+    waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
     assertFalse(task.isUnassigned());
     long curt = System.currentTimeMillis();
     assertTrue((task.last_update <= curt) &&
         (task.last_update > (curt - 1000)));
     LOG.info("waiting for manager to resubmit the orphan task");
-    waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
     assertTrue(task.isUnassigned());
-    waitForCounter(tot_mgr_rescan, 0, 1, to + 100);
+    waitForCounter(tot_mgr_rescan, 0, 1, to + to/2);
   }
 
   @Test
@@ -229,12 +229,12 @@ public void testUnassignedOrphan() throws Exception {
 
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
-    waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, 100);
+    waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
     Task task = slm.findOrCreateOrphanTask(tasknode);
     assertTrue(task.isOrphan());
     assertTrue(task.isUnassigned());
     // wait for RESCAN node to be created
-    waitForCounter(tot_mgr_rescan, 0, 1, 500);
+    waitForCounter(tot_mgr_rescan, 0, 1, to/2);
     Task task2 = slm.findOrCreateOrphanTask(tasknode);
     assertTrue(task == task2);
     LOG.debug("task = " + task);
@@ -250,11 +250,6 @@ public void testUnassignedOrphan() throws Exception {
   public void testMultipleResubmits() throws Exception {
     LOG.info("TestMultipleResbmits - no indefinite resubmissions");
 
-    int to = 1000;
-    conf.setInt("hbase.splitlog.manager.timeout", to);
-    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
-    to = to + 2 * 100;
-
     conf.setInt("hbase.splitlog.max.resubmit", 2);
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
@@ -264,19 +259,19 @@ public void testMultipleResubmits() throws Exception {
     int version = ZKUtil.checkExists(zkw, tasknode);
 
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1"));
-    waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
-    waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
+    waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
     int version1 = ZKUtil.checkExists(zkw, tasknode);
     assertTrue(version1 > version);
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker2"));
-    waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
-    waitForCounter(tot_mgr_resubmit, 1, 2, to + 100);
+    waitForCounter(tot_mgr_heartbeat, 1, 2, to/2);
+    waitForCounter(tot_mgr_resubmit, 1, 2, to + to/2);
     int version2 = ZKUtil.checkExists(zkw, tasknode);
     assertTrue(version2 > version1);
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker3"));
-    waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
-    waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + 100);
-    Thread.sleep(to + 100);
+    waitForCounter(tot_mgr_heartbeat, 1, 2, to/2);
+    waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + to/2);
+    Thread.sleep(to + to/2);
     assertEquals(2L, tot_mgr_resubmit.get());
   }
 
@@ -284,8 +279,6 @@ public void testMultipleResubmits() throws Exception {
   public void testRescanCleanup() throws Exception {
     LOG.info("TestRescanCleanup - ensure RESCAN nodes are cleaned up");
 
-    conf.setInt("hbase.splitlog.manager.timeout", 1000);
-    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
@@ -294,7 +287,7 @@ public void testRescanCleanup() throws Exception {
     int version = ZKUtil.checkExists(zkw, tasknode);
 
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1"));
-    waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
+    waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
     waitForCounter(new Expr() {
       @Override
       public long eval() {
@@ -308,7 +301,7 @@ public long eval() {
       assertTrue(Arrays.equals(TaskState.TASK_UNASSIGNED.get("dummy-master"),
           taskstate));
       
-      waitForCounter(tot_mgr_rescan_deleted, 0, 1, 1000);
+      waitForCounter(tot_mgr_rescan_deleted, 0, 1, to/2);
     } else {
       LOG.warn("Could not run test. Lost ZK connection?");
     }
@@ -330,7 +323,7 @@ public void testTaskDone() throws Exception {
         batch.wait();
       }
     }
-    waitForCounter(tot_mgr_task_deleted, 0, 1, 1000);
+    waitForCounter(tot_mgr_task_deleted, 0, 1, to/2);
     assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1);
   }
 
@@ -350,7 +343,7 @@ public void testTaskErr() throws Exception {
         batch.wait();
       }
     }
-    waitForCounter(tot_mgr_task_deleted, 0, 1, 1000);
+    waitForCounter(tot_mgr_task_deleted, 0, 1, to/2);
     assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1);
     conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLog.DEFAULT_MAX_RESUBMIT);
   }
@@ -366,7 +359,7 @@ public void testTaskResigned() throws Exception {
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_RESIGNED.get("worker"));
     int version = ZKUtil.checkExists(zkw, tasknode);
 
-    waitForCounter(tot_mgr_resubmit, 0, 1, 1000);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
     int version1 = ZKUtil.checkExists(zkw, tasknode);
     assertTrue(version1 > version);
 
@@ -386,15 +379,9 @@ public void testUnassignedTimeout() throws Exception {
         TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-    int to = 4000;
-    conf.setInt("hbase.splitlog.manager.timeout", to);
-    conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to);
-    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
-
-
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
-    waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, 100);
+    waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
 
 
     // submit another task which will stay in unassigned mode
@@ -430,10 +417,10 @@ public void testDeadWorker() throws Exception {
     int version = ZKUtil.checkExists(zkw, tasknode);
 
     ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1"));
-    waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
+    waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
     slm.handleDeadWorker("worker1");
-    waitForCounter(tot_mgr_resubmit, 0, 1, 3000);
-    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, 3000);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
+    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, to + to/2);
 
     int version1 = ZKUtil.checkExists(zkw, tasknode);
     assertTrue(version1 > version);

From c502cdfcd98ab80e2fa33e9038e463dfae4906a3 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 29 Nov 2012 00:15:39 +0000
Subject: [PATCH 0596/1540] HBASE-7235 TestMasterObserver is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1415006 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/TestMasterObserver.java  | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 8910030a8555..4a68a94ab8c2 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -541,7 +542,7 @@ public void testTableOperations() throws Exception {
 
     // modify table
     htd.setMaxFileSize(512 * 1024 * 1024);
-    admin.modifyTable(TEST_TABLE, htd);
+    modifyTableSync(admin, TEST_TABLE, htd);
     // preModifyTable can't bypass default action.
     assertTrue("Test table should have been modified",
       cp.wasModifyTableCalled());
@@ -584,7 +585,7 @@ public void testTableOperations() throws Exception {
 
     // modify table
     htd.setMaxFileSize(512 * 1024 * 1024);
-    admin.modifyTable(TEST_TABLE, htd);
+    modifyTableSync(admin, TEST_TABLE, htd);
     assertTrue("Test table should have been modified",
         cp.wasModifyTableCalled());
 
@@ -629,6 +630,19 @@ public void testTableOperations() throws Exception {
         cp.wasDeleteTableCalled());
   }
 
+  private void modifyTableSync(HBaseAdmin admin, byte[] tableName, HTableDescriptor htd)
+      throws IOException {
+    admin.modifyTable(tableName, htd);
+    //wait until modify table finishes
+    for (int t = 0; t < 100; t++) { //10 sec timeout
+      HTableDescriptor td = admin.getTableDescriptor(htd.getName());
+      if (td.equals(htd)) {
+        break;
+      }
+      Threads.sleep(100);
+    }
+  }
+
   @Test
   public void testRegionTransitionOperations() throws Exception {
     MiniHBaseCluster cluster = UTIL.getHBaseCluster();

From 036856d7ec500489e21527e158337354974260e6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 29 Nov 2012 06:20:14 +0000
Subject: [PATCH 0597/1540] HBASE-7230 port HBASE-7109 integration tests on
 cluster are not getting picked up from distribution to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1415054 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ClassFinder.java  | 232 ++++++++++++
 .../apache/hadoop/hbase/ClassTestFinder.java  | 116 ++++++
 .../hadoop/hbase/IntegrationTestsDriver.java  |  22 +-
 .../hadoop/hbase/TestCheckTestClasses.java    | 131 +------
 .../apache/hadoop/hbase/TestClassFinder.java  | 345 ++++++++++++++++++
 5 files changed, 711 insertions(+), 135 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/ClassFinder.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestClassFinder.java

diff --git a/src/test/java/org/apache/hadoop/hbase/ClassFinder.java b/src/test/java/org/apache/hadoop/hbase/ClassFinder.java
new file mode 100644
index 000000000000..2461534d65b8
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/ClassFinder.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.jar.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A class that finds a set of classes that are locally accessible
+ * (from .class or .jar files), and satisfy the conditions that are
+ * imposed by name and class filters provided by the user.
+ */
+public class ClassFinder {
+  private static final Log LOG = LogFactory.getLog(ClassFinder.class);
+  private static String CLASS_EXT = ".class";
+
+  private FileNameFilter fileNameFilter;
+  private ClassFilter classFilter;
+  private FileFilter fileFilter;
+
+  public static interface FileNameFilter {
+    public boolean isCandidateFile(String fileName, String absFilePath);
+  };
+
+  public static interface ClassFilter {
+    public boolean isCandidateClass(Class<?> c);
+  };
+
+  public ClassFinder(FileNameFilter fileNameFilter, ClassFilter classFilter) {
+    this.classFilter = classFilter;
+    this.fileNameFilter = fileNameFilter;
+    this.fileFilter = new FileFilterWithName(fileNameFilter);
+  }
+
+  /**
+   * Finds the classes in current package (of ClassFinder) and nested packages.
+   * @param proceedOnExceptions whether to ignore exceptions encountered for
+   *        individual jars/files/classes, and proceed looking for others.
+   */
+  public Set<Class<?>> findClasses(boolean proceedOnExceptions)
+    throws ClassNotFoundException, IOException, LinkageError {
+    return findClasses(this.getClass().getPackage().getName(), proceedOnExceptions);
+  }
+
+  /**
+   * Finds the classes in a package and nested packages.
+   * @param packageName package names
+   * @param proceedOnExceptions whether to ignore exceptions encountered for
+   *        individual jars/files/classes, and proceed looking for others.
+   */
+  public Set<Class<?>> findClasses(String packageName, boolean proceedOnExceptions)
+    throws ClassNotFoundException, IOException, LinkageError {
+    final String path = packageName.replace('.', '/');
+    final Pattern jarResourceRe = Pattern.compile("^file:(.+\\.jar)!/" + path + "$");
+
+    Enumeration<URL> resources = ClassLoader.getSystemClassLoader().getResources(path);
+    List<File> dirs = new ArrayList<File>();
+    List<String> jars = new ArrayList<String>();
+
+    while (resources.hasMoreElements()) {
+      URL resource = resources.nextElement();
+      String resourcePath = resource.getFile();
+      Matcher matcher = jarResourceRe.matcher(resourcePath);
+      if (matcher.find()) {
+        jars.add(matcher.group(1));
+      } else {
+        dirs.add(new File(resource.getFile()));
+      }
+    }
+
+    Set<Class<?>> classes = new HashSet<Class<?>>();
+    for (File directory : dirs) {
+      classes.addAll(findClassesFromFiles(directory, packageName, proceedOnExceptions));
+    }
+    for (String jarFileName : jars) {
+      classes.addAll(findClassesFromJar(jarFileName, packageName, proceedOnExceptions));
+    }
+    return classes;
+  }
+
+  private Set<Class<?>> findClassesFromJar(String jarFileName,
+      String packageName, boolean proceedOnExceptions)
+    throws IOException, ClassNotFoundException, LinkageError {
+    JarInputStream jarFile = null;
+    try {
+      jarFile = new JarInputStream(new FileInputStream(jarFileName));
+    } catch (IOException ioEx) {
+      if (!proceedOnExceptions) {
+        throw ioEx;
+      }
+      LOG.error("Failed to look for classes in " + jarFileName + ": " + ioEx);
+    }
+
+    Set<Class<?>> classes = new HashSet<Class<?>>();
+    JarEntry entry = null;
+    while (true) {
+      try {
+        entry = jarFile.getNextJarEntry();
+      } catch (IOException ioEx) {
+        if (!proceedOnExceptions) {
+          throw ioEx;
+        }
+        LOG.error("Failed to get next entry from " + jarFileName + ": " + ioEx);
+        break;
+      }
+      if (entry == null) {
+        break; // loop termination condition
+      }
+
+      String className = entry.getName();
+      if (!className.endsWith(CLASS_EXT)) {
+        continue;
+      }
+      int ix = className.lastIndexOf('/');
+      String fileName = (ix >= 0) ? className.substring(ix + 1) : className;
+      if (!this.fileNameFilter.isCandidateFile(fileName, className)) {
+        continue;
+      }
+      className = className
+          .substring(0, className.length() - CLASS_EXT.length()).replace('/', '.');
+      if (!className.startsWith(packageName)) {
+        continue;
+      }
+      Class<?> c = makeClass(className, proceedOnExceptions);
+      if (c != null) {
+        if (!classes.add(c)) {
+          LOG.error("Ignoring duplicate class " + className);
+        }
+      }
+    }
+    return classes;
+  }
+
+  private Set<Class<?>> findClassesFromFiles(File baseDirectory, String packageName,
+      boolean proceedOnExceptions) throws ClassNotFoundException, LinkageError {
+    Set<Class<?>> classes = new HashSet<Class<?>>();
+    if (!baseDirectory.exists()) {
+      LOG.error("Failed to find " + baseDirectory.getAbsolutePath());
+      return classes;
+    }
+
+    File[] files = baseDirectory.listFiles(this.fileFilter);
+    if (files == null) {
+      LOG.error("Failed to get files from " + baseDirectory.getAbsolutePath());
+      return classes;
+    }
+
+    for (File file : files) {
+      final String fileName = file.getName();
+      if (file.isDirectory()) {
+        classes.addAll(findClassesFromFiles(file, packageName + "." + fileName,
+            proceedOnExceptions));
+      } else {
+        String className = packageName + '.'
+            + fileName.substring(0, fileName.length() - CLASS_EXT.length());
+        Class<?> c = makeClass(className, proceedOnExceptions);
+        if (c != null) {
+          if (!classes.add(c)) {
+            LOG.error("Ignoring duplicate class " + className);
+          }
+        }
+      }
+    }
+    return classes;
+  }
+
+  private Class<?> makeClass(String className, boolean proceedOnExceptions)
+    throws ClassNotFoundException, LinkageError {
+    try {
+      Class<?> c = Class.forName(className, false, this.getClass().getClassLoader());
+      return classFilter.isCandidateClass(c) ? c : null;
+    } catch (ClassNotFoundException classNotFoundEx) {
+      if (!proceedOnExceptions) {
+        throw classNotFoundEx;
+      }
+      LOG.error("Failed to instantiate or check " + className + ": " + classNotFoundEx);
+    } catch (LinkageError linkageEx) {
+      if (!proceedOnExceptions) {
+        throw linkageEx;
+      }
+      LOG.error("Failed to instantiate or check " + className + ": " + linkageEx);
+    }
+    return null;
+  }
+
+  private class FileFilterWithName implements FileFilter {
+    private FileNameFilter nameFilter;
+
+    public FileFilterWithName(FileNameFilter nameFilter) {
+      this.nameFilter = nameFilter;
+    }
+
+    @Override
+    public boolean accept(File file) {
+      return file.isDirectory()
+          || (file.getName().endsWith(CLASS_EXT)
+              && nameFilter.isCandidateFile(file.getName(), file.getAbsolutePath()));
+    }
+  };
+};
diff --git a/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java b/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java
new file mode 100644
index 000000000000..0fefcd57313d
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hbase.ClassFinder.ClassFilter;
+import org.apache.hadoop.hbase.ClassFinder.FileNameFilter;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Suite;
+
+/**
+ * ClassFinder that is pre-configured with filters that will only allow test classes.
+ * The name is strange because a logical name would start with "Test" and be confusing.
+ */
+public class ClassTestFinder extends ClassFinder {
+
+  public ClassTestFinder() {
+    super(new TestFileNameFilter(), new TestClassFilter());
+  }
+
+  public ClassTestFinder(Class<?> category) {
+    super(new TestFileNameFilter(), new TestClassFilter(category));
+  }
+
+  public static Class<?>[] getCategoryAnnotations(Class<?> c) {
+    Category category = c.getAnnotation(Category.class);
+    if (category != null) {
+      return category.value();
+    }
+    return new Class<?>[0];
+  }
+
+  private static class TestFileNameFilter implements FileNameFilter {
+    private static final Pattern hadoopCompactRe =
+        Pattern.compile("hbase-hadoop\\d?-compat");
+
+    @Override
+    public boolean isCandidateFile(String fileName, String absFilePath) {
+      boolean isTestFile = fileName.startsWith("Test")
+          || fileName.startsWith("IntegrationTest");
+      return isTestFile && !hadoopCompactRe.matcher(absFilePath).find();
+    }
+  };
+
+  /*
+  * A class is considered as a test class if:
+   *  - it's not Abstract AND
+   *  - one or more of its methods is annotated with org.junit.Test OR
+   *  - the class is annotated with Suite.SuiteClasses
+  * */
+  private static class TestClassFilter implements ClassFilter {
+    private Class<?> categoryAnnotation = null;
+    public TestClassFilter(Class<?> categoryAnnotation) {
+      this.categoryAnnotation = categoryAnnotation;
+    }
+
+    public TestClassFilter() {
+      this(null);
+    }
+
+    @Override
+    public boolean isCandidateClass(Class<?> c) {
+      return isTestClass(c) && isCategorizedClass(c);
+    }
+
+    private boolean isTestClass(Class<?> c) {
+      if (Modifier.isAbstract(c.getModifiers())) {
+        return false;
+      }
+
+      if (c.getAnnotation(Suite.SuiteClasses.class) != null) {
+        return true;
+      }
+
+      for (Method met : c.getMethods()) {
+        if (met.getAnnotation(Test.class) != null) {
+          return true;
+        }
+      }
+
+      return false;
+    }
+
+    private boolean isCategorizedClass(Class<?> c) {
+      if (this.categoryAnnotation == null) {
+        return true;
+      }
+      for (Class<?> cc : getCategoryAnnotations(c)) {
+        if (cc.equals(this.categoryAnnotation)) {
+          return true;
+        }
+      }
+      return false;
+    }
+  };
+};
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
index fd6053f41b4c..3b233ce40e41 100644
--- a/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.hadoop.hbase.util.AbstractHBaseTool;
@@ -28,12 +29,16 @@
 import org.junit.runner.JUnitCore;
 import org.junit.runner.Result;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 /**
  * This class drives the Integration test suite execution. Executes all
  * tests having @Category(IntegrationTests.class) annotation against an
  * already deployed distributed cluster.
  */
 public class IntegrationTestsDriver extends AbstractHBaseTool {
+  private static final Log LOG = LogFactory.getLog(IntegrationTestsDriver.class);
 
   public static void main(String[] args) throws Exception {
     int ret = ToolRunner.run(new IntegrationTestsDriver(), args);
@@ -51,21 +56,24 @@ protected void processOptions(CommandLine cmd) {
   /**
    * Returns test classes annotated with @Category(IntegrationTests.class)
    */
-  private Class<?>[] findIntegrationTestClasses() throws ClassNotFoundException, IOException {
-    TestCheckTestClasses util = new TestCheckTestClasses();
-    List<Class<?>> classes = util.findTestClasses(IntegrationTests.class);
-    return classes.toArray(new Class<?>[classes.size()]);
-  }
+  private Class<?>[] findIntegrationTestClasses()
+    throws ClassNotFoundException, LinkageError, IOException {
+     ClassTestFinder classFinder = new ClassTestFinder(IntegrationTests.class);
+     Set<Class<?>> classes = classFinder.findClasses(true);
+     return classes.toArray(new Class<?>[classes.size()]);
+   }
+
 
   @Override
   protected int doWork() throws Exception {
-
     //this is called from the command line, so we should set to use the distributed cluster
     IntegrationTestingUtility.setUseDistributedCluster(conf);
+    Class<?>[] classes = findIntegrationTestClasses();
+    LOG.info("Found " + classes.length + " integration tests to run");
 
     JUnitCore junit = new JUnitCore();
     junit.addListener(new TextListener(System.out));
-    Result result = junit.run(findIntegrationTestClasses());
+    Result result = junit.run(classes);
 
     return result.wasSuccessful() ? 0 : 1;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java b/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
index a5ac6f4cd712..b5413bc95a19 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
@@ -42,20 +42,6 @@
  */
 @Category(SmallTests.class)
 public class TestCheckTestClasses {
-
-  private FileFilter TEST_CLASS_FILE_FILTER = new FileFilter() {
-    @Override
-    public boolean accept(File file) {
-      return file.isDirectory() || isTestClassFile(file);
-
-    }
-    private boolean isTestClassFile(File file) {
-      String fileName = file.getName();
-      return fileName.endsWith(".class")
-          && (fileName.startsWith("Test") || fileName.startsWith("IntegrationTest"));
-    }
-  };
-
   /**
    * Throws an assertion if we find a test class without category (small/medium/large/integration).
    * List all the test classes without category in the assertion message.
@@ -63,124 +49,13 @@ private boolean isTestClassFile(File file) {
   @Test
   public void checkClasses() throws Exception {
     List<Class<?>> badClasses = new java.util.ArrayList<Class<?>>();
-
-    for (Class<?> c : findTestClasses()) {
-      if (!existCategoryAnnotation(c, null)) {
+    ClassTestFinder classFinder = new ClassTestFinder();
+    for (Class<?> c : classFinder.findClasses(false)) {
+      if (ClassTestFinder.getCategoryAnnotations(c).length == 0) {
         badClasses.add(c);
       }
     }
-
     assertTrue("There are " + badClasses.size() + " test classes without category: "
       + badClasses, badClasses.isEmpty());
   }
-
-  /** Returns whether the class has @Category annotation having the xface value.
-   */
-  private boolean existCategoryAnnotation(Class<?> c, Class<?> xface) {
-    Category category = c.getAnnotation(Category.class);
-
-    if (category != null) {
-      if (xface == null) {
-        return true;
-      }
-      for (Class<?> cc : category.value()) {
-        if (cc.equals(xface)) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  /*
-  * A class is considered as a test class if:
-   *  - it's not Abstract AND
-   *  - one or more of its methods is annotated with org.junit.Test OR
-   *  - the class is annotated with Suite.SuiteClasses
-  * */
-  private boolean isTestClass(Class<?> c) {
-    if (Modifier.isAbstract(c.getModifiers())) {
-      return false;
-    }
-
-    if (c.getAnnotation(Suite.SuiteClasses.class) != null) {
-      return true;
-    }
-
-    for (Method met : c.getMethods()) {
-      if (met.getAnnotation(Test.class) != null) {
-        return true;
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Finds test classes which are annotated with @Category having xface value
-   * @param xface the @Category value
-   */
-  public List<Class<?>> findTestClasses(Class<?> xface) throws ClassNotFoundException, IOException {
-    List<Class<?>> classes = new ArrayList<Class<?>>();
-    for (Class<?> c : findTestClasses()) {
-      if (existCategoryAnnotation(c, xface)) {
-        classes.add(c);
-      }
-    }
-    return classes;
-  }
-
-  private List<Class<?>> findTestClasses() throws ClassNotFoundException, IOException {
-    final String packageName = "org.apache.hadoop.hbase";
-    final String path = packageName.replace('.', '/');
-
-    Enumeration<URL> resources = this.getClass().getClassLoader().getResources(path);
-    List<File> dirs = new ArrayList<File>();
-
-    while (resources.hasMoreElements()) {
-      URL resource = resources.nextElement();
-      dirs.add(new File(resource.getFile()));
-    }
-
-    List<Class<?>> classes = new ArrayList<Class<?>>();
-    for (File directory : dirs) {
-      classes.addAll(findTestClasses(directory, packageName));
-    }
-
-    return classes;
-  }
-
-
-  private List<Class<?>> findTestClasses(File baseDirectory, String packageName)
-    throws ClassNotFoundException {
-    List<Class<?>> classes = new ArrayList<Class<?>>();
-    if (!baseDirectory.exists()) {
-      return classes;
-    }
-
-    File[] files = baseDirectory.listFiles(TEST_CLASS_FILE_FILTER);
-    assertNotNull(files);
-    Pattern p = Pattern.compile("hbase-hadoop\\d?-compat");
-    for (File file : files) {
-      final String fileName = file.getName();
-      if (p.matcher(file.getAbsolutePath()).find()) {
-        continue;
-      }
-
-      if (file.isDirectory()) {
-        classes.addAll(findTestClasses(file, packageName + "." + fileName));
-      } else {
-        Class<?> c = Class.forName(
-          packageName + '.' + fileName.substring(0, fileName.length() - 6),
-          false,
-          this.getClass().getClassLoader());
-
-        if (isTestClass(c)) {
-          classes.add(c);
-        }
-      }
-    }
-
-    return classes;
-  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestClassFinder.java b/src/test/java/org/apache/hadoop/hbase/TestClassFinder.java
new file mode 100644
index 000000000000..3bd8e655eda8
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestClassFinder.java
@@ -0,0 +1,345 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.jar.*;
+import javax.tools.*;
+
+import org.apache.hadoop.hbase.SmallTests;
+
+import org.junit.experimental.categories.Category;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.commons.io.FileUtils;
+
+@Category(SmallTests.class)
+public class TestClassFinder {
+  private static final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+  private static final String BASEPKG = "tfcpkg";
+  // Use unique jar/class/package names in each test case with the help
+  // of these global counters; we are mucking with ClassLoader in this test
+  // and we don't want individual test cases to conflict via it.
+  private static AtomicLong testCounter = new AtomicLong(0);
+  private static AtomicLong jarCounter = new AtomicLong(0);
+
+  private static String basePath = null;
+
+  // Default name/class filters for testing.
+  private static final ClassFinder.FileNameFilter trueNameFilter =
+      new ClassFinder.FileNameFilter() {
+    @Override
+    public boolean isCandidateFile(String fileName, String absFilePath) {
+      return true;
+    }
+  };
+  private static final ClassFinder.ClassFilter trueClassFilter =
+      new ClassFinder.ClassFilter() {
+    @Override
+    public boolean isCandidateClass(Class<?> c) {
+      return true;
+    }
+  };
+
+  @BeforeClass
+  public static void createTestDir() throws IOException {
+    basePath = testUtil.getDataTestDir(TestClassFinder.class.getSimpleName()).toString();
+    if (!basePath.endsWith("/")) {
+      basePath += "/";
+    }
+    // Make sure we get a brand new directory.
+    File testDir = new File(basePath);
+    if (testDir.exists()) {
+      deleteTestDir();
+    }
+    assertTrue(testDir.mkdirs());
+  }
+
+  @AfterClass
+  public static void deleteTestDir() throws IOException {
+    testUtil.cleanupTestDir(TestClassFinder.class.getSimpleName());
+  }
+
+  @Test
+  public void testClassFinderCanFindClassesInJars() throws Exception {
+    long counter = testCounter.incrementAndGet();
+    FileAndPath c1 = compileTestClass(counter, "", "c1");
+    FileAndPath c2 = compileTestClass(counter, ".nested", "c2");
+    FileAndPath c3 = compileTestClass(counter, "", "c3");
+    packageAndLoadJar(c1, c3);
+    packageAndLoadJar(c2);
+
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> allClasses = allClassesFinder.findClasses(
+        makePackageName("", counter), false);
+    assertEquals(3, allClasses.size());
+  }
+
+  @Test
+  public void testClassFinderHandlesConflicts() throws Exception {
+    long counter = testCounter.incrementAndGet();
+    FileAndPath c1 = compileTestClass(counter, "", "c1");
+    FileAndPath c2 = compileTestClass(counter, "", "c2");
+    packageAndLoadJar(c1, c2);
+    packageAndLoadJar(c1);
+
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> allClasses = allClassesFinder.findClasses(
+        makePackageName("", counter), false);
+    assertEquals(2, allClasses.size());
+  }
+
+  @Test
+  public void testClassFinderHandlesNestedPackages() throws Exception {
+    final String NESTED = ".nested";
+    final String CLASSNAME1 = "c2";
+    final String CLASSNAME2 = "c3";
+    long counter = testCounter.incrementAndGet();
+    FileAndPath c1 = compileTestClass(counter, "", "c1");
+    FileAndPath c2 = compileTestClass(counter, NESTED, CLASSNAME1);
+    FileAndPath c3 = compileTestClass(counter, NESTED, CLASSNAME2);
+    packageAndLoadJar(c1, c2);
+    packageAndLoadJar(c3);
+
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> nestedClasses = allClassesFinder.findClasses(
+        makePackageName(NESTED, counter), false);
+    assertEquals(2, nestedClasses.size());
+    Class<?> nestedClass1 = makeClass(NESTED, CLASSNAME1, counter);
+    assertTrue(nestedClasses.contains(nestedClass1));
+    Class<?> nestedClass2 = makeClass(NESTED, CLASSNAME2, counter);
+    assertTrue(nestedClasses.contains(nestedClass2));
+  }
+
+  @Test
+  public void testClassFinderFiltersByNameInJar() throws Exception {
+    final String CLASSNAME = "c1";
+    final String CLASSNAMEEXCPREFIX = "c2";
+    long counter = testCounter.incrementAndGet();
+    FileAndPath c1 = compileTestClass(counter, "", CLASSNAME);
+    FileAndPath c2 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "1");
+    FileAndPath c3 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "2");
+    packageAndLoadJar(c1, c2, c3);
+
+    ClassFinder.FileNameFilter notExcNameFilter = new ClassFinder.FileNameFilter() {
+      @Override
+      public boolean isCandidateFile(String fileName, String absFilePath) {
+        return !fileName.startsWith(CLASSNAMEEXCPREFIX);
+      }
+    };
+    ClassFinder incClassesFinder = new ClassFinder(notExcNameFilter, trueClassFilter);
+    Set<Class<?>> incClasses = incClassesFinder.findClasses(
+        makePackageName("", counter), false);
+    assertEquals(1, incClasses.size());
+    Class<?> incClass = makeClass("", CLASSNAME, counter);
+    assertTrue(incClasses.contains(incClass));
+  }
+
+  @Test
+  public void testClassFinderFiltersByClassInJar() throws Exception {
+    final String CLASSNAME = "c1";
+    final String CLASSNAMEEXCPREFIX = "c2";
+    long counter = testCounter.incrementAndGet();
+    FileAndPath c1 = compileTestClass(counter, "", CLASSNAME);
+    FileAndPath c2 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "1");
+    FileAndPath c3 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "2");
+    packageAndLoadJar(c1, c2, c3);
+
+    final ClassFinder.ClassFilter notExcClassFilter = new ClassFinder.ClassFilter() {
+      @Override
+      public boolean isCandidateClass(Class<?> c) {
+        return !c.getSimpleName().startsWith(CLASSNAMEEXCPREFIX);
+      }
+    };
+    ClassFinder incClassesFinder = new ClassFinder(trueNameFilter, notExcClassFilter);
+    Set<Class<?>> incClasses = incClassesFinder.findClasses(
+        makePackageName("", counter), false);
+    assertEquals(1, incClasses.size());
+    Class<?> incClass = makeClass("", CLASSNAME, counter);
+    assertTrue(incClasses.contains(incClass));
+  }
+
+  @Test
+  public void testClassFinderCanFindClassesInDirs() throws Exception {
+    // Well, technically, we are not guaranteed that the classes will
+    // be in dirs, but during normal build they would be.
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> allClasses = allClassesFinder.findClasses(
+        this.getClass().getPackage().getName(), false);
+    assertTrue(allClasses.contains(this.getClass()));
+    assertTrue(allClasses.contains(ClassFinder.class));
+  }
+
+  @Test
+  public void testClassFinderFiltersByNameInDirs() throws Exception {
+    final String thisName = this.getClass().getSimpleName();
+    ClassFinder.FileNameFilter notThisFilter = new ClassFinder.FileNameFilter() {
+      @Override
+      public boolean isCandidateFile(String fileName, String absFilePath) {
+        return !fileName.equals(thisName + ".class");
+      }
+    };
+    String thisPackage = this.getClass().getPackage().getName();
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> allClasses = allClassesFinder.findClasses(thisPackage, false);
+    ClassFinder notThisClassFinder = new ClassFinder(notThisFilter, trueClassFilter);
+    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(thisPackage, false);
+    assertFalse(notAllClasses.contains(this.getClass()));
+    assertEquals(allClasses.size() - 1, notAllClasses.size());
+  }
+
+  @Test
+  public void testClassFinderFiltersByClassInDirs() throws Exception {
+    ClassFinder.ClassFilter notThisFilter = new ClassFinder.ClassFilter() {
+      @Override
+      public boolean isCandidateClass(Class<?> c) {
+        return c != TestClassFinder.class;
+      }
+    };
+    String thisPackage = this.getClass().getPackage().getName();
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> allClasses = allClassesFinder.findClasses(thisPackage, false);
+    ClassFinder notThisClassFinder = new ClassFinder(trueNameFilter, notThisFilter);
+    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(thisPackage, false);
+    assertFalse(notAllClasses.contains(this.getClass()));
+    assertEquals(allClasses.size() - 1, notAllClasses.size());
+  }
+
+  @Test
+  public void testClassFinderDefaultsToOwnPackage() throws Exception {
+    // Correct handling of nested packages is tested elsewhere, so here we just assume
+    // pkgClasses is the correct answer that we don't have to check.
+    ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter);
+    Set<Class<?>> pkgClasses = allClassesFinder.findClasses(
+        ClassFinder.class.getPackage().getName(), false);
+    Set<Class<?>> defaultClasses = allClassesFinder.findClasses(false);
+    assertArrayEquals(pkgClasses.toArray(), defaultClasses.toArray());
+  }
+
+  private static class FileAndPath {
+    String path;
+    File file;
+    public FileAndPath(String path, File file) {
+      this.file = file;
+      this.path = path;
+    }
+  }
+
+  private static Class<?> makeClass(String nestedPkgSuffix,
+      String className, long counter) throws ClassNotFoundException {
+    return Class.forName(
+        makePackageName(nestedPkgSuffix, counter) + "." + className + counter);
+  }
+
+  private static String makePackageName(String nestedSuffix, long counter) {
+    return BASEPKG + counter + nestedSuffix;
+  }
+
+  /**
+   * Compiles the test class with bogus code into a .class file.
+   * Unfortunately it's very tedious.
+   * @param counter Unique test counter.
+   * @param packageNameSuffix Package name suffix (e.g. ".suffix") for nesting, or "".
+   * @return The resulting .class file and the location in jar it is supposed to go to.
+   */
+  private static FileAndPath compileTestClass(long counter,
+      String packageNameSuffix, String classNamePrefix) throws Exception {
+    classNamePrefix = classNamePrefix + counter;
+    String packageName = makePackageName(packageNameSuffix, counter);
+    String javaPath = basePath + classNamePrefix + ".java";
+    String classPath = basePath + classNamePrefix + ".class";
+    PrintStream source = new PrintStream(javaPath);
+    source.println("package " + packageName + ";");
+    source.println("public class " + classNamePrefix
+        + " { public static void main(String[] args) { } };");
+    source.close();
+    JavaCompiler jc = ToolProvider.getSystemJavaCompiler();
+    int result = jc.run(null, null, null, javaPath);
+    assertEquals(0, result);
+    File classFile = new File(classPath);
+    assertTrue(classFile.exists());
+    return new FileAndPath(packageName.replace('.', '/') + '/', classFile);
+  }
+
+  /**
+   * Makes a jar out of some class files. Unfortunately it's very tedious.
+   * @param filesInJar Files created via compileTestClass.
+   */
+  private static void packageAndLoadJar(FileAndPath... filesInJar) throws Exception {
+    // First, write the bogus jar file.
+    String path = basePath + "jar" + jarCounter.incrementAndGet() + ".jar";
+    Manifest manifest = new Manifest();
+    manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+    FileOutputStream fos = new FileOutputStream(path);
+    JarOutputStream jarOutputStream = new JarOutputStream(fos, manifest);
+    // Directory entries for all packages have to be added explicitly for
+    // resources to be findable via ClassLoader. Directory entries must end
+    // with "/"; the initial one is expected to, also.
+    Set<String> pathsInJar = new HashSet<String>();
+    for (FileAndPath fileAndPath : filesInJar) {
+      String pathToAdd = fileAndPath.path;
+      while (pathsInJar.add(pathToAdd)) {
+        int ix = pathToAdd.lastIndexOf('/', pathToAdd.length() - 2);
+        if (ix < 0) {
+          break;
+        }
+        pathToAdd = pathToAdd.substring(0, ix);
+      }
+    }
+    for (String pathInJar : pathsInJar) {
+      jarOutputStream.putNextEntry(new JarEntry(pathInJar));
+      jarOutputStream.closeEntry();
+    }
+    for (FileAndPath fileAndPath : filesInJar) {
+      File file = fileAndPath.file;
+      jarOutputStream.putNextEntry(
+          new JarEntry(fileAndPath.path + file.getName()));
+      byte[] allBytes = new byte[(int)file.length()];
+      FileInputStream fis = new FileInputStream(file);
+      fis.read(allBytes);
+      fis.close();
+      jarOutputStream.write(allBytes);
+      jarOutputStream.closeEntry();
+    }
+    jarOutputStream.close();
+    fos.close();
+
+    // Add the file to classpath.
+    File jarFile = new File(path);
+    assertTrue(jarFile.exists());
+    URLClassLoader urlClassLoader = (URLClassLoader)ClassLoader.getSystemClassLoader();
+    Method method = URLClassLoader.class
+        .getDeclaredMethod("addURL", new Class[] { URL.class });
+    method.setAccessible(true);
+    method.invoke(urlClassLoader, new Object[] { jarFile.toURI().toURL() });
+  }
+};

From 48313c07b871db01f4c423cd3a1bf9bd75f8e322 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Fri, 30 Nov 2012 17:54:45 +0000
Subject: [PATCH 0598/1540] HBASE-7124: typo in pom.xml with "exlude", no
 definition of "test.exclude.pattern" (Li Ping Zhang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1415768 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index b81823527d21..d2c42e7e9726 100644
--- a/pom.xml
+++ b/pom.xml
@@ -401,7 +401,7 @@
               <include>${integrationtest.include}</include>
             </includes>
             <excludes>
-              <exlude>${unittest.include}</exlude>
+              <exclude>${unittest.include}</exclude>
               <exclude>**/*$*</exclude>
             </excludes>
             <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>

From d965c348c2f627bc745ab47336d059a06a04713c Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 30 Nov 2012 22:25:14 +0000
Subject: [PATCH 0599/1540] HBASE-7204 Make hbck ErrorReporter pluggable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1415876 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 114 ++++++++++--------
 .../hadoop/hbase/util/TestHBaseFsck.java      |  86 +++++++++++++
 2 files changed, 149 insertions(+), 51 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 4cdb11320f8e..80aea36fd28d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -46,6 +46,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -93,6 +94,9 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.base.Joiner;
@@ -146,7 +150,7 @@
  * can be used to limit the kinds of repairs hbck will do.  See the code in
  * {@link #printUsageAndExit()} for more details.
  */
-public class HBaseFsck {
+public class HBaseFsck extends Configured implements Tool {
   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
@@ -159,7 +163,6 @@ public class HBaseFsck {
    * Internal resources
    **********************/
   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
-  private Configuration conf;
   private ClusterStatus status;
   private HConnection connection;
   private HBaseAdmin admin;
@@ -200,7 +203,7 @@ public class HBaseFsck {
   /*********
    * State
    *********/
-  private ErrorReporter errors = new PrintingErrorReporter();
+  final private ErrorReporter errors;
   int fixes = 0;
 
   /**
@@ -241,8 +244,9 @@ public class HBaseFsck {
    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
    */
   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
-      ZooKeeperConnectionException, IOException {
-    this.conf = conf;
+      ZooKeeperConnectionException, IOException, ClassNotFoundException {
+    super(conf);
+    errors = getErrorReporter(conf);
 
     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
     executor = new ScheduledThreadPoolExecutor(numThreads);
@@ -259,8 +263,9 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
    *           if unable to connect to ZooKeeper
    */
   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
-      ZooKeeperConnectionException, IOException {
-    this.conf = conf;
+      ZooKeeperConnectionException, IOException, ClassNotFoundException {
+    super(conf);
+    errors = getErrorReporter(getConf());
     this.executor = exec;
   }
 
@@ -269,8 +274,8 @@ public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunni
    * online state.
    */
   public void connect() throws IOException {
-    admin = new HBaseAdmin(conf);
-    meta = new HTable(conf, HConstants.META_TABLE_NAME);
+    admin = new HBaseAdmin(getConf());
+    meta = new HTable(getConf(), HConstants.META_TABLE_NAME);
     status = admin.getMaster().getClusterStatus();
     connection = admin.getConnection();
   }
@@ -338,7 +343,7 @@ public void offlineHdfsIntegrityRepair() throws IOException, InterruptedExceptio
         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
       LOG.info("Loading regioninfos HDFS");
       // if nothing is happening this should always complete in two iterations.
-      int maxIterations = conf.getInt("hbase.hbck.integrityrepair.iterations.max", 3);
+      int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
       int curIter = 0;
       do {
         clearState(); // clears hbck state and reset fixes to 0 and.
@@ -458,7 +463,7 @@ private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOExce
    */
   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
     Path p = hi.getHdfsRegionDir();
-    FileSystem fs = p.getFileSystem(conf);
+    FileSystem fs = p.getFileSystem(getConf());
     FileStatus[] dirs = fs.listStatus(p);
     if (dirs == null) {
       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
@@ -483,7 +488,7 @@ private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
         byte[] start, end;
         HFile.Reader hf = null;
         try {
-          CacheConfig cacheConf = new CacheConfig(conf);
+          CacheConfig cacheConf = new CacheConfig(getConf());
           hf = HFile.createReader(fs, hfile.getPath(), cacheConf);
           hf.loadFileInfo();
           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
@@ -531,7 +536,7 @@ private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
     // create new region on hdfs.  move data into place.
     HRegionInfo hri = new HRegionInfo(template.getName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
     LOG.info("Creating new region : " + hri);
-    HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, hri, template);
+    HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
     Path target = region.getRegionDir();
 
     // rename all the data to new region
@@ -643,7 +648,7 @@ private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
     }
 
     Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
-    FileSystem fs = regioninfo.getFileSystem(conf);
+    FileSystem fs = regioninfo.getFileSystem(getConf());
 
     FSDataInputStream in = fs.open(regioninfo);
     HRegionInfo hri = new HRegionInfo();
@@ -718,11 +723,11 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
       if (modTInfo == null) {
         // only executed once per table.
         modTInfo = new TableInfo(tableName);
-        Path hbaseRoot = FSUtils.getRootDir(conf);
+        Path hbaseRoot = FSUtils.getRootDir(getConf());
         tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
-              FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
+              FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(getConf()),
               hbaseRoot, tableName);
           modTInfo.htds.add(htd);
         } catch (IOException ioe) {
@@ -751,7 +756,7 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
    */
   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
     Path regionDir = hbi.getHdfsRegionDir();
-    FileSystem fs = regionDir.getFileSystem(conf);
+    FileSystem fs = regionDir.getFileSystem(getConf());
     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
     for (FileStatus subdir : subDirs) {
       String columnfamily = subdir.getPath().getName();
@@ -774,7 +779,7 @@ private boolean fabricateTableInfo(String tableName, Set<String> columns) throws
     for (String columnfamimly : columns) {
       htd.addFamily(new HColumnDescriptor(columnfamimly));
     }
-    FSTableDescriptors.createTableDescriptor(htd, conf, true);
+    FSTableDescriptors.createTableDescriptor(htd, getConf(), true);
     return true;
   }
   
@@ -790,12 +795,12 @@ private boolean fabricateTableInfo(String tableName, Set<String> columns) throws
   public void fixOrphanTables() throws IOException {
     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
 
-      Path hbaseRoot = FSUtils.getRootDir(conf);
+      Path hbaseRoot = FSUtils.getRootDir(getConf());
       List<String> tmpList = new ArrayList<String>();
       tmpList.addAll(orphanTableDirs.keySet());
       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
       Iterator<Entry<String, Set<String>>> iter = orphanTableDirs.entrySet().iterator();
-      int j = 0; 
+      int j = 0;
       int numFailedCase = 0;
       while (iter.hasNext()) {
         Entry<String, Set<String>> entry = (Entry<String, Set<String>>) iter.next();
@@ -806,7 +811,7 @@ public void fixOrphanTables() throws IOException {
             HTableDescriptor htd = htds[j];
             LOG.info("fixing orphan table: " + tableName + " from cache");
             FSTableDescriptors.createTableDescriptor(
-                hbaseRoot.getFileSystem(conf), hbaseRoot, htd, true);
+                hbaseRoot.getFileSystem(getConf()), hbaseRoot, htd, true);
             j++;
             iter.remove();
           }
@@ -845,8 +850,8 @@ public void fixOrphanTables() throws IOException {
    * @return an open .META. HRegion
    */
   private HRegion createNewRootAndMeta() throws IOException {
-    Path rootdir = new Path(conf.get(HConstants.HBASE_DIR));
-    Configuration c = conf;
+    Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR));
+    Configuration c = getConf();
     HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
     MasterFileSystem.setInfoFamilyCachingForRoot(false);
     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
@@ -989,7 +994,7 @@ private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
     for (TableInfo tInfo : tablesInfo.values()) {
       TableIntegrityErrorHandler handler;
       if (fixHoles || fixOverlaps) {
-        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
+        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
           fixHoles, fixOverlaps);
       } else {
         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
@@ -1004,7 +1009,7 @@ private SortedMap<String, TableInfo> checkHdfsIntegrity(boolean fixHoles,
 
   private Path getSidelineDir() throws IOException {
     if (sidelineDir == null) {
-      Path hbaseDir = FSUtils.getRootDir(conf);
+      Path hbaseDir = FSUtils.getRootDir(getConf());
       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
           + startMillis);
@@ -1121,8 +1126,8 @@ void sidelineTable(FileSystem fs, byte[] table, Path hbaseDir,
    */
   Path sidelineOldRootAndMeta() throws IOException {
     // put current -ROOT- and .META. aside.
-    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
-    FileSystem fs = hbaseDir.getFileSystem(conf);
+    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
+    FileSystem fs = hbaseDir.getFileSystem(getConf());
     Path backupDir = getSidelineDir();
     fs.mkdirs(backupDir);
 
@@ -1154,7 +1159,7 @@ Path sidelineOldRootAndMeta() throws IOException {
    */
   private void loadDisabledTables()
   throws ZooKeeperConnectionException, IOException {
-    HConnectionManager.execute(new HConnectable<Void>(conf) {
+    HConnectionManager.execute(new HConnectable<Void>(getConf()) {
       @Override
       public Void connect(HConnection connection) throws IOException {
         ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
@@ -1182,8 +1187,8 @@ private boolean isTableDisabled(HRegionInfo regionInfo) {
    * regionInfoMap
    */
   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
-    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
-    FileSystem fs = rootDir.getFileSystem(conf);
+    Path rootDir = new Path(getConf().get(HConstants.HBASE_DIR));
+    FileSystem fs = rootDir.getFileSystem(getConf());
 
     // list all tables from HDFS
     List<FileStatus> tableDirs = Lists.newArrayList();
@@ -1211,8 +1216,8 @@ public void loadHdfsRegionDirs() throws IOException, InterruptedException {
         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
             + " file.");
         setShouldRerun();
-        FSUtils.setVersion(fs, rootDir, conf.getInt(
-            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), conf.getInt(
+        FSUtils.setVersion(fs, rootDir, getConf().getInt(
+            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
       }
@@ -1338,8 +1343,8 @@ private void preCheckPermission() throws IOException, AccessControlException {
       return;
     }
 
-    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
-    FileSystem fs = hbaseDir.getFileSystem(conf);
+    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
+    FileSystem fs = hbaseDir.getFileSystem(getConf());
     UserGroupInformation ugi = User.getCurrent().getUGI();
     FileStatus[] files = fs.listStatus(hbaseDir);
     for (FileStatus file : files) {
@@ -1566,7 +1571,7 @@ else if (!inMeta && !inHdfs && !isDeployed) {
         }
 
         LOG.info("Patching .META. with .regioninfo: " + hbi.getHdfsHRI());
-        HBaseFsckRepair.fixMetaHoleOnline(conf, hbi.getHdfsHRI());
+        HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
 
         tryAssignmentRepair(hbi, "Trying to reassign region...");
       }
@@ -1582,7 +1587,7 @@ else if (!inMeta && !inHdfs && !isDeployed) {
         }
 
         LOG.info("Patching .META. with with .regioninfo: " + hbi.getHdfsHRI());
-        HBaseFsckRepair.fixMetaHoleOnline(conf, hbi.getHdfsHRI());
+        HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
 
         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
       }
@@ -1744,7 +1749,7 @@ public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOEx
     debugLsr(contained.getHdfsRegionDir());
 
     // rename the contained into the container.
-    FileSystem fs = targetRegionDir.getFileSystem(conf);
+    FileSystem fs = targetRegionDir.getFileSystem(getConf());
     FileStatus[] dirs = fs.listStatus(contained.getHdfsRegionDir());
 
     if (dirs == null) {
@@ -2359,7 +2364,7 @@ HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
     HTableDescriptor[] htd = new HTableDescriptor[0];
      try {
        LOG.info("getHTableDescriptors == tableNames => " + tableNames);
-       htd = new HBaseAdmin(conf).getTableDescriptors(tableNames);
+       htd = new HBaseAdmin(getConf()).getTableDescriptors(tableNames);
      } catch (IOException e) {
        LOG.debug("Exception getting table descriptors", e);
      }
@@ -2502,12 +2507,12 @@ public boolean processRow(Result result) throws IOException {
     };
 
     // Scan -ROOT- to pick up META regions
-    MetaScanner.metaScan(conf, visitor, null, null,
+    MetaScanner.metaScan(getConf(), visitor, null, null,
       Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME);
 
     if (!checkMetaOnly) {
       // Scan .META. to pick up user regions
-      MetaScanner.metaScan(conf, visitor);
+      MetaScanner.metaScan(getConf(), visitor);
     }
 
     errors.print("");
@@ -2761,6 +2766,12 @@ private void printTableSummary(SortedMap<String, TableInfo> tablesInfo) {
     }
   }
 
+  private static ErrorReporter getErrorReporter(
+      final Configuration conf) throws ClassNotFoundException {
+    Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
+    return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
+  }
+
   public interface ErrorReporter {
     public static enum ERROR_CODE {
       UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
@@ -3261,7 +3272,7 @@ public void setSidelineDir(String sidelineDir) {
   }
 
   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
-    return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles);
+    return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
   }
 
   public HFileCorruptionChecker getHFilecorruptionChecker() {
@@ -3334,7 +3345,6 @@ protected HBaseFsck printUsageAndExit() {
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
-
     // create a fsck object
     Configuration conf = HBaseConfiguration.create();
     Path hbasedir = new Path(conf.get(HConstants.HBASE_DIR));
@@ -3342,12 +3352,14 @@ public static void main(String[] args) throws Exception {
     conf.set("fs.defaultFS", defaultFs.toString());     // for hadoop 0.21+
     conf.set("fs.default.name", defaultFs.toString());  // for hadoop 0.20
 
-    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
-    ExecutorService exec = new ScheduledThreadPoolExecutor(numThreads);
-    HBaseFsck hbck = new HBaseFsck(conf, exec);
-    hbck.exec(exec, args);
-    int retcode = hbck.getRetCode();
-    Runtime.getRuntime().exit(retcode);
+    int ret = ToolRunner.run(new HBaseFsck(conf), args);
+    System.exit(ret);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    exec(executor, args);
+    return getRetCode();
   }
 
   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
@@ -3506,13 +3518,13 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
       setHFileCorruptionChecker(hfcc); // so we can get result
       Collection<String> tables = getIncludedTables();
       Collection<Path> tableDirs = new ArrayList<Path>();
-      Path rootdir = FSUtils.getRootDir(conf);
+      Path rootdir = FSUtils.getRootDir(getConf());
       if (tables.size() > 0) {
         for (String t : tables) {
           tableDirs.add(FSUtils.getTablePath(rootdir, t));
         }
       } else {
-        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(conf), rootdir);
+        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
       }
       hfcc.checkTables(tableDirs);
       PrintWriter out = new PrintWriter(System.out);
@@ -3552,7 +3564,7 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
    * ls -r for debugging purposes
    */
   void debugLsr(Path p) throws IOException {
-    debugLsr(conf, p);
+    debugLsr(getConf(), p);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 6498ae409af3..bade8005c5d3 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -70,6 +70,8 @@
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
+import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
@@ -1654,6 +1656,90 @@ protected void checkRegionDir(Path p) throws IOException {
     doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
   }
 
+  /**
+   * Test pluggable error reporter. It can be plugged in
+   * from system property or configuration.
+   */
+  @Test
+  public void testErrorReporter() throws Exception {
+    try {
+      MockErrorReporter.calledCount = 0;
+      doFsck(conf, false);
+      assertEquals(MockErrorReporter.calledCount, 0);
+
+      conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
+      doFsck(conf, false);
+      assertTrue(MockErrorReporter.calledCount > 20);
+    } finally {
+      conf.set("hbasefsck.errorreporter", "");
+      MockErrorReporter.calledCount = 0;
+    }
+  }
+
+  static class MockErrorReporter implements ErrorReporter {
+    static int calledCount = 0;
+
+    public void clear() {
+      calledCount++;
+    }
+
+    public void report(String message) {
+      calledCount++;
+    }
+
+    public void reportError(String message) {
+      calledCount++;
+    }
+
+    public void reportError(ERROR_CODE errorCode, String message) {
+      calledCount++;
+    }
+
+    public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
+      calledCount++;
+    }
+
+    public void reportError(ERROR_CODE errorCode,
+        String message, TableInfo table, HbckInfo info) {
+      calledCount++;
+    }
+
+    public void reportError(ERROR_CODE errorCode, String message,
+        TableInfo table, HbckInfo info1, HbckInfo info2) {
+      calledCount++;
+    }
+
+    public int summarize() {
+      return ++calledCount;
+    }
+
+    public void detail(String details) {
+      calledCount++;
+    }
+
+    public ArrayList<ERROR_CODE> getErrorList() {
+      calledCount++;
+      return new ArrayList<ERROR_CODE>();
+    }
+
+    public void progress() {
+      calledCount++;
+    }
+
+    public void print(String message) {
+      calledCount++;
+    }
+
+    public void resetErrors() {
+      calledCount++;
+    }
+
+    public boolean tableHasErrors(TableInfo table) {
+      calledCount++;
+      return false;
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From d82a4430201c438cbc656b6a22287b5a8bd6180d Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 1 Dec 2012 00:41:24 +0000
Subject: [PATCH 0600/1540] HBASE-7204 Make hbck ErrorReporter pluggable,
 ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1415898 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java     | 2 +-
 src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 80aea36fd28d..7fe81ca6f82c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2797,7 +2797,7 @@ public static enum ERROR_CODE {
     public boolean tableHasErrors(TableInfo table);
   }
 
-  private static class PrintingErrorReporter implements ErrorReporter {
+  static class PrintingErrorReporter implements ErrorReporter {
     public int errorCount = 0;
     private int showProgress;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index bade8005c5d3..e0d3c1959a1b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -71,6 +71,7 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
+import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
@@ -1671,7 +1672,8 @@ public void testErrorReporter() throws Exception {
       doFsck(conf, false);
       assertTrue(MockErrorReporter.calledCount > 20);
     } finally {
-      conf.set("hbasefsck.errorreporter", "");
+      conf.set("hbasefsck.errorreporter",
+        PrintingErrorReporter.class.getName());
       MockErrorReporter.calledCount = 0;
     }
   }

From a279fdd9a7ee25799fba91e2dafa39af25141c58 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 1 Dec 2012 22:52:44 +0000
Subject: [PATCH 0601/1540] HBASE-7252 TestSizeBasedThrottler fails
 occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1416073 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/TestSizeBasedThrottler.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
index 971c0247a6b2..e974b9a330c6 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
@@ -100,7 +100,7 @@ public void testSmallIncreases(){
           15, // fail if throttler's value
               // exceeds 15
           1000, // use 1000 threads
-          200 // wait for 200ms
+          500 // wait for 500ms
           );
     }
   }
@@ -114,7 +114,7 @@ public void testBigIncreases() {
           4, // fail if throttler's value
              // exceeds 4
           1000, // use 1000 threads
-          200 // wait for 200ms
+          500 // wait for 500ms
           );
     }
   }
@@ -128,7 +128,7 @@ public void testIncreasesEqualToThreshold(){
           2, // fail if throttler's value
              // exceeds 2
           1000, // use 1000 threads
-          200 // wait for 200ms
+          500 // wait for 500ms
           );
     }
   }

From 7596585e11137972d00d86ea33a0e2c61b244499 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 4 Dec 2012 18:16:21 +0000
Subject: [PATCH 0602/1540] HBASE-6423 Writes should not block reads on
 blocking updates to memstores

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417079 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/RegionTooBusyException.java  |  47 ++++++
 .../hadoop/hbase/regionserver/HRegion.java    | 128 +++++++++++++---
 .../hbase/regionserver/TestHRegion.java       | 144 ++++++++----------
 .../regionserver/TestHRegionBusyWait.java     |  90 +++++++++++
 4 files changed, 310 insertions(+), 99 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java

diff --git a/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java b/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
new file mode 100644
index 000000000000..f5217bcdcf32
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown by a region server if it will block and wait to serve a request.
+ * For example, the client wants to insert something to a region while the
+ * region is compacting.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RegionTooBusyException extends IOException {
+  private static final long serialVersionUID = 1728345723728342L;
+
+  /** default constructor */
+  public RegionTooBusyException() {
+    super();
+  }
+
+  /**
+   * Constructor
+   * @param msg message
+   */
+  public RegionTooBusyException(final String msg) {
+    super(msg);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 06f93e16c2f5..82bd7d2de613 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -55,6 +55,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.commons.logging.Log;
@@ -76,6 +77,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RegionTooBusyException;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.Append;
@@ -235,6 +237,25 @@ public class HRegion implements HeapSize { // , Writable{
   final Configuration conf;
   final int rowLockWaitDuration;
   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
+
+  // The internal wait duration to acquire a lock before read/update
+  // from the region. It is not per row. The purpose of this wait time
+  // is to avoid waiting a long time while the region is busy, so that
+  // we can release the IPC handler soon enough to improve the
+  // availability of the region server. It can be adjusted by
+  // tuning configuration "hbase.busy.wait.duration".
+  final long busyWaitDuration;
+  static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+
+  // If updating multiple rows in one call, wait longer,
+  // i.e. waiting for busyWaitDuration * # of rows. However,
+  // we can limit the max multiplier.
+  final int maxBusyWaitMultiplier;
+
+  // Max busy wait duration. There is no point to wait longer than the RPC
+  // purge timeout, when a RPC call will be terminated by the RPC engine.
+  final long maxBusyWaitDuration;
+
   final HRegionInfo regionInfo;
   final Path regiondir;
   KeyValue.KVComparator comparator;
@@ -354,6 +375,10 @@ public HRegion(){
     this.coprocessorHost = null;
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
     this.opMetrics = new OperationMetrics();
+
+    this.maxBusyWaitDuration = 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+    this.busyWaitDuration = DEFAULT_BUSY_WAIT_DURATION;
+    this.maxBusyWaitMultiplier = 2;
   }
 
   /**
@@ -400,6 +425,17 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
     this.opMetrics = new OperationMetrics(conf, this.regionInfo);
 
+    this.busyWaitDuration = conf.getLong(
+      "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);
+    this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);
+    if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {
+      throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("
+        + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("
+        + maxBusyWaitMultiplier + "). Their product should be positive");
+    }
+    this.maxBusyWaitDuration = conf.getLong("ipc.client.call.purge.timeout",
+      2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+
     /*
      * timestamp.slop provides a server-side constraint on the timestamp. This
      * assumes that you base your TS around currentTimeMillis(). In this case,
@@ -883,6 +919,7 @@ private List<StoreFile> doClose(
 
     this.closing.set(true);
     status.setStatus("Disabling writes for close");
+    // block waiting for the lock for closing
     lock.writeLock().lock();
     try {
       if (this.isClosed()) {
@@ -1192,6 +1229,7 @@ public boolean compact(CompactionRequest cr)
       return false;
     }
     Preconditions.checkArgument(cr.getHRegion().equals(this));
+    // block waiting for the lock for compaction
     lock.readLock().lock();
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Compacting " + cr.getStore() + " in " + this);
@@ -1271,7 +1309,7 @@ public boolean flushcache() throws IOException {
     }
     MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
     status.setStatus("Acquiring readlock on region");
-    lock.readLock().lock();
+    lock(lock.readLock());
     try {
       if (this.closed.get()) {
         LOG.debug("Skipping flush on " + this + " because closed");
@@ -1406,6 +1444,7 @@ protected boolean internalFlushcache(
     // end up in both snapshot and memstore (makes it difficult to do atomic
     // rows then)
     status.setStatus("Obtaining lock to block concurrent updates");
+    // block waiting for the lock for internal flush
     this.updatesLock.writeLock().lock();
     long flushsize = this.memstoreSize.get();
     status.setStatus("Preparing to flush by snapshotting stores");
@@ -1784,7 +1823,7 @@ private void internalDelete(Delete delete, UUID clusterId,
     byte [] byteNow = Bytes.toBytes(now);
     boolean flush = false;
 
-    updatesLock.readLock().lock();
+    lock(updatesLock.readLock());
     try {
       prepareDeleteTimestamps(delete.getFamilyMap(), byteNow);
 
@@ -2163,7 +2202,7 @@ private long doMiniBatchMutation(
         }
       }
 
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock(), numReadyToWrite);
       locked = true;
 
       //
@@ -2457,7 +2496,8 @@ private void updateKVTimestamps(
    * this and the synchronize on 'this' inside in internalFlushCache to send
    * the notify.
    */
-  private void checkResources() {
+  private void checkResources()
+      throws RegionTooBusyException, InterruptedIOException {
 
     // If catalog region, do not impose resource constraints or block updates.
     if (this.getRegionInfo().isMetaRegion()) return;
@@ -2475,12 +2515,30 @@ private void checkResources() {
           " is >= than blocking " +
           StringUtils.humanReadableInt(this.blockingMemStoreSize) + " size");
       }
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      long timeToWait = startTime + busyWaitDuration - now;
+      if (timeToWait <= 0L) {
+        final long totalTime = now - startTime;
+        this.updatesBlockedMs.add(totalTime);
+        LOG.info("Failed to unblock updates for region " + this + " '"
+          + Thread.currentThread().getName() + "' in " + totalTime
+          + "ms. The region is still busy.");
+        throw new RegionTooBusyException("region is flushing");
+      }
       blocked = true;
       synchronized(this) {
         try {
-          wait(threadWakeFrequency);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
+          wait(Math.min(timeToWait, threadWakeFrequency));
+        } catch (InterruptedException ie) {
+          final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+          if (totalTime > 0) {
+            this.updatesBlockedMs.add(totalTime);
+          }
+          LOG.info("Interrupted while waiting to unblock updates for region "
+            + this + " '" + Thread.currentThread().getName() + "'");
+          InterruptedIOException iie = new InterruptedIOException();
+          iie.initCause(ie);
+          throw iie;
         }
       }
     }
@@ -2547,7 +2605,7 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
     byte[] byteNow = Bytes.toBytes(now);
     boolean flush = false;
 
-    this.updatesLock.readLock().lock();
+    lock(this.updatesLock.readLock());
     try {
       checkFamilies(familyMap.keySet());
       checkTimestamps(familyMap, now);
@@ -3172,6 +3230,7 @@ byte[] getRowFromLock(final Integer lockid) {
    * @param lockId  The lock ID to release.
    */
   public void releaseRowLock(final Integer lockId) {
+    if (lockId == null) return; // null lock id, do nothing
     HashedBytes rowKey = lockIds.remove(lockId);
     if (rowKey == null) {
       LOG.warn("Release unknown lockId: " + lockId);
@@ -4303,7 +4362,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
       }
 
       // 3. acquire the region lock
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock(), acquiredLocks.size());
       locked = true;
 
       // 4. Get a mvcc write number
@@ -4456,7 +4515,7 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
     this.writeRequestsCount.increment();
     try {
       Integer lid = getLock(lockid, row, true);
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock());
       try {
         long now = EnvironmentEdgeManager.currentTimeMillis();
         // Process each family
@@ -4607,7 +4666,7 @@ public Result increment(Increment increment, Integer lockid,
     this.writeRequestsCount.increment();
     try {
       Integer lid = getLock(lockid, row, true);
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock());
       try {
         long now = EnvironmentEdgeManager.currentTimeMillis();
         // Process each family
@@ -4725,7 +4784,7 @@ public long incrementColumnValue(byte [] row, byte [] family,
     this.writeRequestsCount.increment();
     try {
       Integer lid = obtainRowLock(row);
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock());
       try {
         Store store = stores.get(family);
 
@@ -5119,13 +5178,16 @@ public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {
    * #closeRegionOperation needs to be called in the try's finally block
    * Acquires a read lock and checks if the region is closing or closed.
    * @throws NotServingRegionException when the region is closing or closed
+   * @throws RegionTooBusyException if failed to get the lock in time
+   * @throws InterruptedIOException if interrupted while waiting for a lock
    */
-  private void startRegionOperation() throws NotServingRegionException {
+  private void startRegionOperation()
+      throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
     if (this.closing.get()) {
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
           " is closing");
     }
-    lock.readLock().lock();
+    lock(lock.readLock());
     if (this.closed.get()) {
       lock.readLock().unlock();
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
@@ -5147,15 +5209,17 @@ private void closeRegionOperation(){
    * #closeBulkRegionOperation needs to be called in the try's finally block
    * Acquires a writelock and checks if the region is closing or closed.
    * @throws NotServingRegionException when the region is closing or closed
+   * @throws RegionTooBusyException if failed to get the lock in time
+   * @throws InterruptedIOException if interrupted while waiting for a lock
    */
   private void startBulkRegionOperation(boolean writeLockNeeded)
-  throws NotServingRegionException {
+      throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
     if (this.closing.get()) {
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
           " is closing");
     }
-    if (writeLockNeeded) lock.writeLock().lock();
-    else lock.readLock().lock();
+    if (writeLockNeeded) lock(lock.writeLock());
+    else lock(lock.readLock());
     if (this.closed.get()) {
       if (writeLockNeeded) lock.writeLock().unlock();
       else lock.readLock().unlock();
@@ -5168,7 +5232,7 @@ private void startBulkRegionOperation(boolean writeLockNeeded)
    * Closes the lock. This needs to be called in the finally block corresponding
    * to the try block of #startRegionOperation
    */
-  private void closeBulkRegionOperation(){
+  private void closeBulkRegionOperation() {
     if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();
     else lock.readLock().unlock();
   }
@@ -5193,6 +5257,33 @@ private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
     dataInMemoryWithoutWAL.addAndGet(putSize);
   }
 
+  private void lock(final Lock lock)
+      throws RegionTooBusyException, InterruptedIOException {
+    lock(lock, 1);
+  }
+
+  /**
+   * Try to acquire a lock.  Throw RegionTooBusyException
+   * if failed to get the lock in time. Throw InterruptedIOException
+   * if interrupted while waiting for the lock.
+   */
+  private void lock(final Lock lock, final int multiplier)
+      throws RegionTooBusyException, InterruptedIOException {
+    try {
+      final long waitTime = Math.min(maxBusyWaitDuration,
+        busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
+      if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
+        throw new RegionTooBusyException(
+          "failed to get a lock in " + waitTime + "ms");
+      }
+    } catch (InterruptedException ie) {
+      LOG.info("Interrupted while waiting for a lock");
+      InterruptedIOException iie = new InterruptedIOException();
+      iie.initCause(ie);
+      throw iie;
+    }
+  }
+
   /**
    * Calls sync with the given transaction ID if the region's table is not
    * deferring it.
@@ -5232,7 +5323,6 @@ public int size() {
     }
   };
 
-
   /**
    * Facility for dumping and compacting catalog tables.
    * Only does catalog tables since these are only tables we for sure know
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 6a91edf89587..392ae58a3ea9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -21,6 +21,7 @@
 
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -51,17 +52,15 @@
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MultithreadedTestUtil;
-import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -71,8 +70,6 @@
 import org.apache.hadoop.hbase.filter.NullComparator;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -105,6 +102,7 @@
  * HRegions or in the HBaseMaster, so only basic testing is possible.
  */
 @Category(MediumTests.class)
+@SuppressWarnings("deprecation")
 public class TestHRegion extends HBaseTestCase {
   // Do not spin up clusters in here.  If you need to spin up a cluster, do it
   // over in TestHRegionOnCluster.
@@ -157,7 +155,6 @@ public void testCompactionAffectedByScanners() throws Exception {
     String method = "testCompactionAffectedByScanners";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    Configuration conf = HBaseConfiguration.create();
     this.region = initHRegion(tableName, method, conf, family);
 
     Put put = new Put(Bytes.toBytes("r1"));
@@ -209,7 +206,6 @@ public void testToShowNPEOnRegionScannerReseek() throws Exception{
     String method = "testToShowNPEOnRegionScannerReseek";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    Configuration conf = HBaseConfiguration.create();
     this.region = initHRegion(tableName, method, conf, family);
 
     Put put = new Put(Bytes.toBytes("r1"));
@@ -242,7 +238,6 @@ public void testSkipRecoveredEditsReplay() throws Exception {
     String method = "testSkipRecoveredEditsReplay";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    Configuration conf = HBaseConfiguration.create();
     this.region = initHRegion(tableName, method, conf, family);
     try {
       Path regiondir = region.getRegionDir();
@@ -288,7 +283,7 @@ public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
     String method = "testSkipRecoveredEditsReplaySomeIgnored";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    this.region = initHRegion(tableName, method, conf, family);
     try {
       Path regiondir = region.getRegionDir();
       FileSystem fs = region.getFilesystem();
@@ -338,7 +333,7 @@ public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
     String method = "testSkipRecoveredEditsReplayAllIgnored";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    this.region = initHRegion(tableName, method, conf, family);
     try {
       Path regiondir = region.getRegionDir();
       FileSystem fs = region.getFilesystem();
@@ -464,7 +459,7 @@ public void testWeirdCacheBehaviour() throws Exception {
     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
-    this.region = initHRegion(TABLE, getName(), FAMILIES);
+    this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
     try {
       String value = "this is the value";
       String value2 = "this is some other value";
@@ -585,7 +580,7 @@ private void putRows(HRegion r, int numRows, String value, String key)
   public void testFamilyWithAndWithoutColon() throws Exception {
     byte [] b = Bytes.toBytes(getName());
     byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
-    this.region = initHRegion(b, getName(), cf);
+    this.region = initHRegion(b, getName(), conf, cf);
     try {
       Put p = new Put(b);
       byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
@@ -609,7 +604,7 @@ public void testBatchPut() throws Exception {
     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
     byte[] qual = Bytes.toBytes("qual");
     byte[] val = Bytes.toBytes("val");
-    this.region = initHRegion(b, getName(), cf);
+    this.region = initHRegion(b, getName(), conf, cf);
     try {
       HLog.getSyncTime(); // clear counter from prior tests
       assertEquals(0, HLog.getSyncTime().count);
@@ -643,7 +638,7 @@ public void testBatchPut() throws Exception {
       Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
 
       MultithreadedTestUtil.TestContext ctx =
-        new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+        new MultithreadedTestUtil.TestContext(conf);
       final AtomicReference<OperationStatus[]> retFromThread =
         new AtomicReference<OperationStatus[]>();
       TestThread putter = new TestThread(ctx) {
@@ -710,9 +705,7 @@ public void testBatchPutWithTsSlop() throws Exception {
     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
     byte[] qual = Bytes.toBytes("qual");
     byte[] val = Bytes.toBytes("val");
-
-    HBaseConfiguration conf = new HBaseConfiguration();
-
+    Configuration conf = HBaseConfiguration.create(this.conf);
 
     // add data with a timestamp that is too recent for range. Ensure assert
     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -759,7 +752,7 @@ public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting empty data in key
       Put put = new Put(row1);
@@ -834,7 +827,7 @@ public void testCheckAndMutate_WithWrongValue() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in key
       Put put = new Put(row1);
@@ -868,7 +861,7 @@ public void testCheckAndMutate_WithCorrectValue() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in key
       Put put = new Put(row1);
@@ -906,7 +899,7 @@ public void testCheckAndPut_ThatPutWasWritten() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in the key to check
       Put put = new Put(row1);
@@ -945,7 +938,7 @@ public void testCheckAndPut_ThatPutWasWritten() throws IOException{
   }
 
   public void testCheckAndPut_wrongRowInPut() throws IOException {
-    this.region = initHRegion(tableName, this.getName(), COLUMNS);
+    this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
     try {
       Put put = new Put(row2);
       put.add(fam1, qual1, value1);
@@ -980,7 +973,7 @@ public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Put content
       Put put = new Put(row1);
@@ -1055,7 +1048,7 @@ public void testDelete_multiDeleteColumn() throws IOException {
     put.add(fam1, qual, 2, value);
 
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       region.put(put);
 
@@ -1085,7 +1078,7 @@ public void testDelete_CheckFamily() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1, fam2, fam3);
+    this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
     try {
       List<KeyValue> kvs  = new ArrayList<KeyValue>();
       kvs.add(new KeyValue(row1, fam4, null, null));
@@ -1123,7 +1116,7 @@ public void testDelete_mixed() throws IOException, InterruptedException {
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
 
@@ -1191,7 +1184,7 @@ public void testDeleteRowWithFutureTs() throws IOException {
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       byte [] row = Bytes.toBytes("table_name");
       // column names
@@ -1234,7 +1227,7 @@ public void testPutWithLatestTS() throws IOException {
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       byte [] row = Bytes.toBytes("row1");
       // column names
@@ -1287,7 +1280,7 @@ public void testPutWithTsSlop() throws IOException {
     byte[] fam = Bytes.toBytes("info");
     byte[][] families = { fam };
     String method = this.getName();
-    HBaseConfiguration conf = new HBaseConfiguration();
+    Configuration conf = HBaseConfiguration.create(this.conf);
 
     // add data with a timestamp that is too recent for range. Ensure assert
     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -1318,7 +1311,7 @@ public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
     byte [] tableName = Bytes.toBytes("test_table");
     byte [] fam1 = Bytes.toBytes("columnA");
     byte [] fam2 = Bytes.toBytes("columnB");
-    this.region = initHRegion(tableName, getName(), fam1, fam2);
+    this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
     try {
       byte [] rowA = Bytes.toBytes("rowA");
       byte [] rowB = Bytes.toBytes("rowB");
@@ -1371,7 +1364,7 @@ public void testDeleteFamily_PostInsert() throws IOException, InterruptedExcepti
 
   public void doTestDelete_AndPostInsert(Delete delete)
       throws IOException, InterruptedException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
       Put put = new Put(row);
@@ -1424,7 +1417,7 @@ public void testDelete_CheckTimestampUpdated()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Building checkerList
       List<KeyValue> kvs  = new ArrayList<KeyValue>();
@@ -1464,7 +1457,7 @@ public void testGet_FamilyChecker() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       Get get = new Get(row1);
       get.addColumn(fam2, col1);
@@ -1495,7 +1488,7 @@ public void testGet_Basic() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Add to memstore
       Put put = new Put(row1);
@@ -1545,7 +1538,7 @@ public void testGet_Empty() throws IOException {
     byte [] fam = Bytes.toBytes("fam");
 
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam);
+    this.region = initHRegion(tableName, method, conf, fam);
     try {
       Get get = new Get(row);
       get.addFamily(fam);
@@ -1565,7 +1558,8 @@ public void testGet_Empty() throws IOException {
   public void stestGet_Root() throws IOException {
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
+    this.region = initHRegion(HConstants.ROOT_TABLE_NAME,
+      method, conf, HConstants.CATALOG_FAMILY);
     try {
       //Add to memstore
       Put put = new Put(HConstants.EMPTY_START_ROW);
@@ -1797,7 +1791,7 @@ public void testGetScanner_WithOkFamilies() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       Scan scan = new Scan();
       scan.addFamily(fam1);
@@ -1822,7 +1816,7 @@ public void testGetScanner_WithNotOkFamilies() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       Scan scan = new Scan();
       scan.addFamily(fam2);
@@ -1851,7 +1845,7 @@ public void testGetScanner_WithNoFamilies() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
 
       //Putting data in Region
@@ -1899,7 +1893,7 @@ public void testGetScanner_WithRegionClosed() throws IOException {
     //Setting up region
     String method = this.getName();
     try {
-      this.region = initHRegion(tableName, method, families);
+      this.region = initHRegion(tableName, method, conf, families);
     } catch (IOException e) {
       e.printStackTrace();
       fail("Got IOException during initHRegion, " + e.getMessage());
@@ -1935,7 +1929,7 @@ public void testRegionScanner_Next() throws IOException {
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2002,7 +1996,7 @@ public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2062,7 +2056,7 @@ public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2127,7 +2121,7 @@ public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2209,7 +2203,7 @@ public void testScanner_Wildcard_FromMemStore_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2270,7 +2264,7 @@ public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in Region
       Put put = null;
@@ -2321,7 +2315,7 @@ public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
   public void testScanner_StopRow1542() throws IOException {
     byte [] tableName = Bytes.toBytes("test_table");
     byte [] family = Bytes.toBytes("testFamily");
-    this.region = initHRegion(tableName, getName(), family);
+    this.region = initHRegion(tableName, getName(), conf, family);
     try {
       byte [] row1 = Bytes.toBytes("row111");
       byte [] row2 = Bytes.toBytes("row222");
@@ -2368,7 +2362,7 @@ public void testScanner_StopRow1542() throws IOException {
   }
 
   public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2396,7 +2390,7 @@ public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
   public void testIncrementColumnValue_BumpSnapshot() throws IOException {
     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 42L;
       long incr = 44L;
@@ -2435,7 +2429,7 @@ public void testIncrementColumnValue_BumpSnapshot() throws IOException {
   }
 
   public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2469,7 +2463,7 @@ public void run() {
   public void testIncrementColumnValue_heapSize() throws IOException {
     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
 
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long byAmount = 1L;
       long size;
@@ -2488,7 +2482,7 @@ public void testIncrementColumnValue_heapSize() throws IOException {
 
   public void testIncrementColumnValue_UpdatingInPlace_Negative()
     throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 3L;
       long amount = -1L;
@@ -2509,7 +2503,7 @@ public void testIncrementColumnValue_UpdatingInPlace_Negative()
 
   public void testIncrementColumnValue_AddingNew()
     throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2538,7 +2532,7 @@ public void testIncrementColumnValue_AddingNew()
   }
 
   public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2566,7 +2560,7 @@ public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
 
   public void testIncrementColumnValue_AddingNewAfterSFCheck()
     throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2605,7 +2599,7 @@ public void testIncrementColumnValue_AddingNewAfterSFCheck()
    * @throws IOException
    */
   public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2653,7 +2647,7 @@ public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws I
   }
 
   public void testIncrementColumnValue_WrongInitialSize() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
       int row1Field1 = 0;
@@ -2681,7 +2675,7 @@ public void testIncrementColumnValue_WrongInitialSize() throws IOException {
   }
 
   public void testIncrement_WrongInitialSize() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
       long row1Field1 = 0;
@@ -2757,7 +2751,7 @@ public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in Region
       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2972,7 +2966,7 @@ public void testFlushCacheWhileScanning() throws IOException, InterruptedExcepti
     int compactInterval = 10 * flushAndScanInterval;
 
     String method = "testFlushCacheWhileScanning";
-    this.region = initHRegion(tableName,method, family);
+    this.region = initHRegion(tableName,method, conf, family);
     try {
       FlushThread flushThread = new FlushThread();
       flushThread.start();
@@ -3103,7 +3097,7 @@ public void testWritesWhileScanning()
     }
 
     String method = "testWritesWhileScanning";
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       PutThread putThread = new PutThread(numRows, families, qualifiers);
       putThread.start();
@@ -3225,6 +3219,8 @@ public void run() {
             }
             numPutsFinished++;
           }
+        } catch (InterruptedIOException e) {
+          // This is fine. It means we are done, or didn't get the lock on time
         } catch (IOException e) {
           LOG.error("error while putting records", e);
           error = e;
@@ -3261,8 +3257,9 @@ public void testWritesWhileGetting()
       qualifiers[i] = Bytes.toBytes("qual" + i);
     }
 
+    Configuration conf = HBaseConfiguration.create(this.conf);
+
     String method = "testWritesWhileGetting";
-    Configuration conf = HBaseConfiguration.create();
     // This test flushes constantly and can cause many files to be created, possibly
     // extending over the ulimit.  Make sure compactions are aggressive in reducing
     // the number of HFiles created.
@@ -3271,7 +3268,7 @@ public void testWritesWhileGetting()
     this.region = initHRegion(tableName, method, conf, families);
     PutThread putThread = null;
     MultithreadedTestUtil.TestContext ctx =
-      new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+      new MultithreadedTestUtil.TestContext(conf);
     try {
       putThread = new PutThread(numRows, families, qualifiers);
       putThread.start();
@@ -3357,7 +3354,7 @@ public void testHolesInMeta() throws Exception {
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
-        HBaseConfiguration.create(), family);
+        conf, family);
     try {
       byte[] rowNotServed = Bytes.toBytes("a");
       Get g = new Get(rowNotServed);
@@ -3421,7 +3418,7 @@ public void testIndexesScanWithOneDeletedRow() throws IOException {
 
     //Setting up region
     String method = "testIndexesScanWithOneDeletedRow";
-    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    this.region = initHRegion(tableName, method, conf, family);
     try {
       Put put = new Put(Bytes.toBytes(1L));
       put.add(family, qual1, 1L, Bytes.toBytes(1L));
@@ -3874,7 +3871,6 @@ public void run() {
    */
   @Test
   public void testParallelIncrementWithMemStoreFlush() throws Exception {
-    Configuration conf = HBaseConfiguration.create();
     String method = "testParallelIncrementWithMemStoreFlush";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Incrementer.family;
@@ -4007,7 +4003,8 @@ private void assertScan(final HRegion r, final byte [] fs,
   }
 
   private Configuration initSplit() {
-    Configuration conf = HBaseConfiguration.create();
+    Configuration conf = HBaseConfiguration.create(this.conf);
+
     // Always compact if there is more than one store file.
     conf.setInt("hbase.hstore.compactionThreshold", 2);
 
@@ -4025,19 +4022,6 @@ private Configuration initSplit() {
     return conf;
   }
 
-  /**
-   * @param tableName
-   * @param callingMethod
-   * @param families
-   * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
-   * @throws IOException
-   */
-  private static HRegion initHRegion (byte [] tableName, String callingMethod,
-    byte[] ... families)
-  throws IOException {
-    return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
-  }
-
   /**
    * @param tableName
    * @param callingMethod
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
new file mode 100644
index 000000000000..10a93702e903
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
@@ -0,0 +1,90 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.RegionTooBusyException;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * TestHRegion with hbase.busy.wait.duration set to 1000 (1 second).
+ * We can't use parameterized test since TestHRegion is old fashion.
+ */
+@Category(MediumTests.class)
+@SuppressWarnings("deprecation")
+public class TestHRegionBusyWait extends TestHRegion {
+  public TestHRegionBusyWait() {
+    conf.set("hbase.busy.wait.duration", "1000");
+  }
+
+  /**
+   * Test RegionTooBusyException thrown when region is busy
+   */
+  @Test (timeout=2000)
+  public void testRegionTooBusy() throws IOException {
+    String method = "testRegionTooBusy";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Bytes.toBytes("family");
+    region = initHRegion(tableName, method, conf, family);
+    final AtomicBoolean stopped = new AtomicBoolean(true);
+    Thread t = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          region.lock.writeLock().lock();
+          stopped.set(false);
+          while (!stopped.get()) {
+            Thread.sleep(100);
+          }
+        } catch (InterruptedException ie) {
+        } finally {
+          region.lock.writeLock().unlock();
+        }
+      }
+    });
+    t.start();
+    Get get = new Get(row);
+    try {
+      while (stopped.get()) {
+        Thread.sleep(100);
+      }
+      region.get(get, null);
+      fail("Should throw RegionTooBusyException");
+    } catch (InterruptedException ie) {
+      fail("test interrupted");
+    } catch (RegionTooBusyException e) {
+      // Good, expected
+    } finally {
+      stopped.set(true);
+      try {
+        t.join();
+      } catch (Throwable e) {
+      }
+
+      HRegion.closeHRegion(region);
+      region = null;
+    }
+  }
+}

From 07b3d7bb6e42aad828d2d7a0522e1dd7f0884485 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 4 Dec 2012 18:41:03 +0000
Subject: [PATCH 0603/1540] HBASE-7260 Upgrade hadoop 1 dependency to hadoop
 1.1.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417107 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d2c42e7e9726..d93cbbada0e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1714,7 +1714,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>1.1.0</hadoop.version>
+        <hadoop.version>1.1.1</hadoop.version>
         <slf4j.version>1.4.3</slf4j.version>
       </properties>
       <dependencies>

From a1c73dcb846c29bb9e52c7ae5395ab2dc12a396c Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Tue, 4 Dec 2012 22:58:05 +0000
Subject: [PATCH 0604/1540] HBASE-5888: Clover profile in build (Andrey
 Klochkov)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417231 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/pom.xml b/pom.xml
index d93cbbada0e4..f02bf298361c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1019,6 +1019,7 @@
     <thrift.version>0.8.0</thrift.version>
     <zookeeper.version>3.4.3</zookeeper.version>
     <hadoop-snappy.version>0.0.1-SNAPSHOT</hadoop-snappy.version>
+    <clover.version>2.6.3</clover.version>
 
     <package.prefix>/usr</package.prefix>
     <package.conf.dir>/etc/hbase</package.conf.dir>
@@ -2357,6 +2358,61 @@
         <surefire.firstPartGroups></surefire.firstPartGroups>
       </properties>
     </profile>
+
+    <!-- Profile for running clover. You need to have a clover license under ~/.clover.license for ${clover.version}
+or you can provide the license with -Dmaven.clover.licenseLocation=/path/to/license. Committers can find
+the license under https://svn.apache.org/repos/private/committers/donated-licenses/clover/
+Note that clover 2.6.3 does not run with maven 3, so you have to use maven2. The report will be generated
+under target/site/clover/index.html when you run
+MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
+    <profile>
+      <id>clover</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+        <property>
+          <name>clover</name>
+        </property>
+      </activation>
+      <properties>
+        <maven.clover.licenseLocation>${user.home}/.clover.license</maven.clover.licenseLocation>
+        <clover.version>2.6.3</clover.version>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.atlassian.maven.plugins</groupId>
+            <artifactId>maven-clover2-plugin</artifactId>
+            <version>${clover.version}</version>
+            <configuration>
+              <includesAllSourceRoots>true</includesAllSourceRoots>
+              <includesTestSourceRoots>true</includesTestSourceRoots>
+              <targetPercentage>50%</targetPercentage>
+              <generateHtml>true</generateHtml>
+              <generateXml>true</generateXml>
+              <excludes>
+                <exclude>**/generated/**</exclude>
+              </excludes>
+            </configuration>
+            <executions>
+              <execution>
+                <id>clover-setup</id>
+                <phase>process-sources</phase>
+                <goals>
+                  <goal>setup</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>clover</id>
+                <phase>site</phase>
+                <goals>
+                  <goal>clover</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <!-- See http://jira.codehaus.org/browse/MSITE-443 why the settings need to be here and not in pluginManagement. -->

From bebc9885bf1533410e30e8eab07d3d90c579337a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 4 Dec 2012 23:20:06 +0000
Subject: [PATCH 0605/1540] HBASE-7249 add test name filter to
 IntegrationTestsDriver

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417236 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ClassTestFinder.java  |  4 +-
 .../hadoop/hbase/IntegrationTestsDriver.java  | 38 ++++++++++++++++---
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java b/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java
index 0fefcd57313d..f40aa7984b5b 100644
--- a/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java
+++ b/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java
@@ -50,7 +50,7 @@ public static Class<?>[] getCategoryAnnotations(Class<?> c) {
     return new Class<?>[0];
   }
 
-  private static class TestFileNameFilter implements FileNameFilter {
+  public static class TestFileNameFilter implements FileNameFilter {
     private static final Pattern hadoopCompactRe =
         Pattern.compile("hbase-hadoop\\d?-compat");
 
@@ -68,7 +68,7 @@ public boolean isCandidateFile(String fileName, String absFilePath) {
    *  - one or more of its methods is annotated with org.junit.Test OR
    *  - the class is annotated with Suite.SuiteClasses
   * */
-  private static class TestClassFilter implements ClassFilter {
+  public static class TestClassFilter implements ClassFilter {
     private Class<?> categoryAnnotation = null;
     public TestClassFilter(Class<?> categoryAnnotation) {
       this.categoryAnnotation = categoryAnnotation;
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
index 3b233ce40e41..a6f3760bc44e 100644
--- a/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.hadoop.hbase.util.AbstractHBaseTool;
@@ -38,30 +39,55 @@
  * already deployed distributed cluster.
  */
 public class IntegrationTestsDriver extends AbstractHBaseTool {
+  private static final String TESTS_ARG = "test";
   private static final Log LOG = LogFactory.getLog(IntegrationTestsDriver.class);
+  private IntegrationTestFilter intTestFilter = new IntegrationTestFilter();
 
   public static void main(String[] args) throws Exception {
     int ret = ToolRunner.run(new IntegrationTestsDriver(), args);
     System.exit(ret);
   }
 
+  private class IntegrationTestFilter extends ClassTestFinder.TestClassFilter {
+    private Pattern testFilterRe = Pattern.compile(".*");
+    public IntegrationTestFilter() {
+      super(IntegrationTests.class);
+    }
+
+    public void setPattern(String pattern) {
+      testFilterRe = Pattern.compile(pattern);
+    }
+
+    @Override
+    public boolean isCandidateClass(Class<?> c) {
+      return super.isCandidateClass(c) && testFilterRe.matcher(c.getName()).find();
+    }
+  }
+
   @Override
   protected void addOptions() {
+    addOptWithArg(TESTS_ARG, "a Java regular expression to filter tests on");
   }
 
   @Override
   protected void processOptions(CommandLine cmd) {
+    String testFilterString = cmd.getOptionValue(TESTS_ARG, null);
+    if (testFilterString != null) {
+      intTestFilter.setPattern(testFilterString);
+    }
   }
 
   /**
-   * Returns test classes annotated with @Category(IntegrationTests.class)
+   * Returns test classes annotated with @Category(IntegrationTests.class),
+   * according to the filter specific on the command line (if any).
    */
   private Class<?>[] findIntegrationTestClasses()
     throws ClassNotFoundException, LinkageError, IOException {
-     ClassTestFinder classFinder = new ClassTestFinder(IntegrationTests.class);
-     Set<Class<?>> classes = classFinder.findClasses(true);
-     return classes.toArray(new Class<?>[classes.size()]);
-   }
+    ClassTestFinder.TestFileNameFilter nameFilter = new ClassTestFinder.TestFileNameFilter();
+    ClassFinder classFinder = new ClassFinder(nameFilter, intTestFilter);
+    Set<Class<?>> classes = classFinder.findClasses(true);
+    return classes.toArray(new Class<?>[classes.size()]);
+  }
 
 
   @Override
@@ -78,4 +104,4 @@ protected int doWork() throws Exception {
     return result.wasSuccessful() ? 0 : 1;
   }
 
-}
\ No newline at end of file
+}

From f6234d6d90476155d9e763a4626ce3c892685201 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 5 Dec 2012 00:05:39 +0000
Subject: [PATCH 0606/1540] HBASE-6423 Writes should not block reads on
 blocking updates to memstores - Addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417243 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 82bd7d2de613..fe351d28ba82 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4878,7 +4878,7 @@ private void checkFamily(final byte [] family)
       ClassSize.OBJECT +
       ClassSize.ARRAY +
       35 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
-      (5 * Bytes.SIZEOF_LONG) +
+      (6 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +

From e06140fce16aaa202e6d5c8b2310348528406c99 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 5 Dec 2012 00:17:18 +0000
Subject: [PATCH 0607/1540] HBASE-6423 Writes should not block reads on
 blocking updates to memstores - Addendum 2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417247 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index fe351d28ba82..6371820ed281 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4877,8 +4877,8 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      35 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
-      (6 * Bytes.SIZEOF_LONG) +
+      35 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+      (7 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +

From b20e4815cd8b4fa001df836dcd2a5218dfe5d1aa Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 5 Dec 2012 00:32:27 +0000
Subject: [PATCH 0608/1540] HBASE-6206 Large tests fail with jdk1.7

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417249 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/Get.java  | 4 ++++
 src/main/java/org/apache/hadoop/hbase/client/Scan.java | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index 3f35caecd999..dc83d1d729f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.client;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.io.TimeRange;
@@ -131,6 +132,9 @@ public Get addColumn(byte [] family, byte [] qualifier) {
     if(set == null) {
       set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
     }
+    if (qualifier == null) {
+      qualifier = HConstants.EMPTY_BYTE_ARRAY;
+    }
     set.add(qualifier);
     familyMap.put(family, set);
     return this;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 093c93b68f6d..024059b2f25b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -215,6 +215,9 @@ public Scan addColumn(byte [] family, byte [] qualifier) {
     if(set == null) {
       set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
     }
+    if (qualifier == null) {
+      qualifier = HConstants.EMPTY_BYTE_ARRAY;
+    }
     set.add(qualifier);
     familyMap.put(family, set);
 

From 7a2c4c361e3d13e2fdbcb4da939adf043c443e0f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 5 Dec 2012 00:58:06 +0000
Subject: [PATCH 0609/1540] HBASE-7273 Upgrade zookeeper dependency to 3.4.5
 for 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417255 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f02bf298361c..28463c18a5af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1017,7 +1017,7 @@
     <protobuf.version>2.4.0a</protobuf.version>
     <stax-api.version>1.0.1</stax-api.version>
     <thrift.version>0.8.0</thrift.version>
-    <zookeeper.version>3.4.3</zookeeper.version>
+    <zookeeper.version>3.4.5</zookeeper.version>
     <hadoop-snappy.version>0.0.1-SNAPSHOT</hadoop-snappy.version>
     <clover.version>2.6.3</clover.version>
 

From 65d09081ab3642dac8b37c293782f9dfc2572576 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 5 Dec 2012 17:54:58 +0000
Subject: [PATCH 0610/1540] HBASE-7253 Backport Compaction Tool to 0.94;
 includes HBASE-5616, HBASE-5693, HBASE-6327, HBASE-7253

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417559 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   6 +-
 .../master/handler/CreateTableHandler.java    |   7 +-
 .../hbase/regionserver/CompactionTool.java    | 468 ++++++++++++++++++
 .../hadoop/hbase/regionserver/Compactor.java  | 205 ++++++++
 .../hadoop/hbase/regionserver/HRegion.java    | 129 +++--
 .../hadoop/hbase/regionserver/Store.java      | 296 ++++-------
 .../compactions/CompactionProgress.java       |   1 -
 .../hadoop/hbase/util/ChecksumType.java       |   9 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |  21 +-
 .../hbase/regionserver/TestCompaction.java    |   4 +-
 10 files changed, 890 insertions(+), 256 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index e6987ff71853..5f6f652eec14 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -645,7 +645,7 @@ public void logStats() {
     // Log size
     long totalSize = heapSize();
     long freeSize = maxSize - totalSize;
-    LruBlockCache.LOG.debug("LRU Stats: " +
+    LruBlockCache.LOG.debug("Stats: " +
         "total=" + StringUtils.byteDesc(totalSize) + ", " +
         "free=" + StringUtils.byteDesc(freeSize) + ", " +
         "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
@@ -653,11 +653,11 @@ public void logStats() {
         "accesses=" + stats.getRequestCount() + ", " +
         "hits=" + stats.getHitCount() + ", " +
         "hitRatio=" +
-          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) +
+          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
         "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
         "cachingHits=" + stats.getHitCachingCount() + ", " +
         "cachingHitsRatio=" +
-          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) +
+          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) + ", " +
         "evictions=" + stats.getEvictionCount() + ", " +
         "evicted=" + stats.getEvictedCount() + ", " +
         "evictedPerRun=" + stats.evictedPerEviction());
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 95750606df78..af25defc0d1f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -142,16 +142,12 @@ private void handleCreateTable() throws IOException, KeeperException {
     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
     final int batchSize =
       this.conf.getInt("hbase.master.createtable.batchsize", 100);
-    HLog hlog = null;
     for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
       HRegionInfo newRegion = this.newRegions[regionIdx];
       // 1. Create HRegion
       HRegion region = HRegion.createHRegion(newRegion,
         this.fileSystemManager.getRootDir(), this.conf,
-        this.hTableDescriptor, hlog);
-      if (hlog == null) {
-        hlog = region.getLog();
-      }
+        this.hTableDescriptor, null, false, true);
 
       regionInfos.add(region.getRegionInfo());
       if (regionIdx % batchSize == 0) {
@@ -163,7 +159,6 @@ private void handleCreateTable() throws IOException, KeeperException {
       // 3. Close the new region to flush to disk.  Close log file too.
       region.close();
     }
-    hlog.closeAndDelete();
     if (regionInfos.size() > 0) {
       MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
new file mode 100644
index 000000000000..e123c22761fe
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -0,0 +1,468 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/*
+ * The CompactionTool allows to execute a compaction specifying a:
+ * <ul>
+ *  <li>table folder (all regions and families will be compacted)
+ *  <li>region folder (all families in the region will be compacted)
+ *  <li>family folder (the store files will be compacted)
+ * </ul>
+ */
+@InterfaceAudience.Public
+public class CompactionTool extends Configured implements Tool {
+  private static final Log LOG = LogFactory.getLog(CompactionTool.class);
+
+  private final static String CONF_TMP_DIR = "hbase.tmp.dir";
+  private final static String CONF_COMPACT_ONCE = "hbase.compactiontool.compact.once";
+  private final static String CONF_DELETE_COMPACTED = "hbase.compactiontool.delete";
+  private final static String CONF_COMPLETE_COMPACTION = "hbase.hstore.compaction.complete";
+
+  /**
+   * Class responsible to execute the Compaction on the specified path.
+   * The path can be a table, region or family directory.
+   */
+  private static class CompactionWorker {
+    private final boolean keepCompactedFiles;
+    private final boolean deleteCompacted;
+    private final Configuration conf;
+    private final FileSystem fs;
+    private final Path tmpDir;
+
+    public CompactionWorker(final FileSystem fs, final Configuration conf) {
+      this.conf = conf;
+      this.keepCompactedFiles = !conf.getBoolean(CONF_COMPLETE_COMPACTION, true);
+      this.deleteCompacted = conf.getBoolean(CONF_DELETE_COMPACTED, false);
+      this.tmpDir = new Path(conf.get(CONF_TMP_DIR));
+      this.fs = fs;
+    }
+
+    /**
+     * Execute the compaction on the specified path.
+     *
+     * @param path Directory path on which run a
+     * @param compactOnce Execute just a single step of compaction.
+     */
+    public void compact(final Path path, final boolean compactOnce) throws IOException {
+      if (isFamilyDir(fs, path)) {
+        Path regionDir = path.getParent();
+        Path tableDir = regionDir.getParent();
+        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+        HRegion region = loadRegion(fs, conf, htd, regionDir);
+        compactStoreFiles(region, path, compactOnce);
+      } else if (isRegionDir(fs, path)) {
+        Path tableDir = path.getParent();
+        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+        compactRegion(htd, path, compactOnce);
+      } else if (isTableDir(fs, path)) {
+        compactTable(path, compactOnce);
+      } else {
+        throw new IOException(
+          "Specified path is not a table, region or family directory. path=" + path);
+      }
+    }
+
+    private void compactTable(final Path tableDir, final boolean compactOnce)
+        throws IOException {
+      HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+      LOG.info("Compact table=" + htd.getNameAsString());
+      for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
+        compactRegion(htd, regionDir, compactOnce);
+      }
+    }
+
+    private void compactRegion(final HTableDescriptor htd, final Path regionDir,
+        final boolean compactOnce) throws IOException {
+      HRegion region = loadRegion(fs, conf, htd, regionDir);
+      LOG.info("Compact table=" + htd.getNameAsString() +
+        " region=" + region.getRegionNameAsString());
+      for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+        compactStoreFiles(region, familyDir, compactOnce);
+      }
+    }
+
+    /**
+     * Execute the actual compaction job.
+     * If the compact once flag is not specified, execute the compaction until
+     * no more compactions are needed. Uses the Configuration settings provided.
+     */
+    private void compactStoreFiles(final HRegion region, final Path familyDir,
+        final boolean compactOnce) throws IOException {
+      LOG.info("Compact table=" + region.getTableDesc().getNameAsString() +
+        " region=" + region.getRegionNameAsString() +
+        " family=" + familyDir.getName());
+      Store store = getStore(region, familyDir);
+      do {
+        CompactionRequest cr = store.requestCompaction();
+        StoreFile storeFile = store.compact(cr);
+        if (storeFile != null) {
+          if (keepCompactedFiles && deleteCompacted) {
+            fs.delete(storeFile.getPath(), false);
+          }
+        }
+      } while (store.needsCompaction() && !compactOnce);
+    }
+
+    /**
+     * Create a "mock" HStore that uses the tmpDir specified by the user and
+     * the store dir to compact as source.
+     */
+    private Store getStore(final HRegion region, final Path storeDir) throws IOException {
+      byte[] familyName = Bytes.toBytes(storeDir.getName());
+      HColumnDescriptor hcd = region.getTableDesc().getFamily(familyName);
+      // Create a Store w/ check of hbase.rootdir blanked out and return our
+      // list of files instead of have Store search its home dir.
+      return new Store(tmpDir, region, hcd, fs, conf) {
+        @Override
+        public FileStatus[] getStoreFiles() throws IOException {
+          return this.fs.listStatus(getHomedir());
+        }
+
+        @Override
+        Path createStoreHomeDir(FileSystem fs, Path homedir) throws IOException {
+          return storeDir;
+        }
+      };
+    }
+
+    private static HRegion loadRegion(final FileSystem fs, final Configuration conf,
+        final HTableDescriptor htd, final Path regionDir) throws IOException {
+      Path rootDir = regionDir.getParent().getParent();
+      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+      return HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
+    }
+  }
+
+  private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
+    Path regionInfo = new Path(path, HRegion.REGIONINFO_FILE);
+    return fs.exists(regionInfo);
+  }
+
+  private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
+    return FSTableDescriptors.getTableInfoPath(fs, path) != null;
+  }
+
+  private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException {
+    return isRegionDir(fs, path.getParent());
+  }
+
+  private static class CompactionMapper
+      extends Mapper<LongWritable, Text, NullWritable, NullWritable> {
+    private CompactionWorker compactor = null;
+    private boolean compactOnce = false;
+
+    @Override
+    public void setup(Context context) {
+      Configuration conf = context.getConfiguration();
+      compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false);
+
+      try {
+        FileSystem fs = FileSystem.get(conf);
+        this.compactor = new CompactionWorker(fs, conf);
+      } catch (IOException e) {
+        throw new RuntimeException("Could not get the input FileSystem", e);
+      }
+    }
+
+    @Override
+    public void map(LongWritable key, Text value, Context context)
+        throws InterruptedException, IOException {
+      Path path = new Path(value.toString());
+      this.compactor.compact(path, compactOnce);
+    }
+  }
+
+  /**
+   * Input format that uses store files block location as input split locality.
+   */
+  private static class CompactionInputFormat extends TextInputFormat {
+    @Override
+    protected boolean isSplitable(JobContext context, Path file) {
+      return true;
+    }
+
+    /**
+     * Returns a split for each store files directory using the block location
+     * of each file as locality reference.
+     */
+    @Override
+    public List<InputSplit> getSplits(JobContext job) throws IOException {
+      List<InputSplit> splits = new ArrayList<InputSplit>();
+      List<FileStatus> files = listStatus(job);
+
+      Text key = new Text();
+      for (FileStatus file: files) {
+        Path path = file.getPath();
+        FileSystem fs = path.getFileSystem(job.getConfiguration());
+        LineReader reader = new LineReader(fs.open(path));
+        long pos = 0;
+        int n;
+        try {
+          while ((n = reader.readLine(key)) > 0) {
+            String[] hosts = getStoreDirHosts(fs, path);
+            splits.add(new FileSplit(path, pos, n, hosts));
+            pos += n;
+          }
+        } finally {
+          reader.close();
+        }
+      }
+
+      return splits;
+    }
+
+    /**
+     * return the top hosts of the store files, used by the Split
+     */
+    private static String[] getStoreDirHosts(final FileSystem fs, final Path path)
+        throws IOException {
+      FileStatus[] files = FSUtils.listStatus(fs, path, null);
+      if (files == null) {
+        return new String[] {};
+      }
+
+      HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
+      for (FileStatus hfileStatus: files) {
+        HDFSBlocksDistribution storeFileBlocksDistribution =
+          FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen());
+        hdfsBlocksDistribution.add(storeFileBlocksDistribution);
+      }
+
+      List<String> hosts = hdfsBlocksDistribution.getTopHosts();
+      return hosts.toArray(new String[hosts.size()]);
+    }
+
+    /**
+     * Create the input file for the given directories to compact.
+     * The file is a TextFile with each line corrisponding to a
+     * store files directory to compact.
+     */
+    public static void createInputFile(final FileSystem fs, final Path path,
+        final Set<Path> toCompactDirs) throws IOException {
+      // Extract the list of store dirs
+      List<Path> storeDirs = new LinkedList<Path>();
+      for (Path compactDir: toCompactDirs) {
+        if (isFamilyDir(fs, compactDir)) {
+          storeDirs.add(compactDir);
+        } else if (isRegionDir(fs, compactDir)) {
+          for (Path familyDir: FSUtils.getFamilyDirs(fs, compactDir)) {
+            storeDirs.add(familyDir);
+          }
+        } else if (isTableDir(fs, compactDir)) {
+          // Lookup regions
+          for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) {
+            for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+              storeDirs.add(familyDir);
+            }
+          }
+        } else {
+          throw new IOException(
+            "Specified path is not a table, region or family directory. path=" + compactDir);
+        }
+      }
+
+      // Write Input File
+      FSDataOutputStream stream = fs.create(path);
+      LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact.");
+      try {
+        final byte[] newLine = Bytes.toBytes("\n");
+        for (Path storeDir: storeDirs) {
+          stream.write(Bytes.toBytes(storeDir.toString()));
+          stream.write(newLine);
+        }
+      } finally {
+        stream.close();
+      }
+    }
+  }
+
+  /**
+   * Execute compaction, using a Map-Reduce job.
+   */
+  private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
+      final boolean compactOnce) throws Exception {
+    Configuration conf = getConf();
+    conf.setBoolean(CONF_COMPACT_ONCE, compactOnce);
+
+    Job job = new Job(conf);
+    job.setJobName("CompactionTool");
+    job.setJarByClass(CompactionTool.class);
+    job.setMapperClass(CompactionMapper.class);
+    job.setInputFormatClass(CompactionInputFormat.class);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    job.setMapSpeculativeExecution(false);
+    job.setNumReduceTasks(0);
+
+    String stagingName = "compact-" + EnvironmentEdgeManager.currentTimeMillis();
+    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
+    fs.mkdirs(stagingDir);
+    try {
+      // Create input file with the store dirs
+      Path inputPath = new Path(stagingDir, stagingName);
+      CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs);
+      CompactionInputFormat.addInputPath(job, inputPath);
+
+      // Initialize credential for secure cluster
+      TableMapReduceUtil.initCredentials(job);
+
+      // Start the MR Job and wait
+      return job.waitForCompletion(true) ? 0 : 1;
+    } finally {
+      fs.delete(stagingDir, true);
+    }
+  }
+
+  /**
+   * Execute compaction, from this client, one path at the time.
+   */
+  private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
+      final boolean compactOnce) throws IOException {
+    CompactionWorker worker = new CompactionWorker(fs, getConf());
+    for (Path path: toCompactDirs) {
+      worker.compact(path, compactOnce);
+    }
+    return 0;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    Set<Path> toCompactDirs = new HashSet<Path>();
+    boolean compactOnce = false;
+    boolean mapred = false;
+
+    Configuration conf = getConf();
+    FileSystem fs = FileSystem.get(conf);
+
+    try {
+      for (int i = 0; i < args.length; ++i) {
+        String opt = args[i];
+        if (opt.equals("-compactOnce")) {
+          compactOnce = true;
+        } else if (opt.equals("-mapred")) {
+          mapred = true;
+        } else if (!opt.startsWith("-")) {
+          Path path = new Path(opt);
+          FileStatus status = fs.getFileStatus(path);
+          if (!status.isDir()) {
+            printUsage("Specified path is not a directory. path=" + path);
+            return 1;
+          }
+          toCompactDirs.add(path);
+        } else {
+          printUsage();
+        }
+      }
+    } catch (Exception e) {
+      printUsage(e.getMessage());
+      return 1;
+    }
+
+    if (toCompactDirs.size() == 0) {
+      printUsage("No directories to compact specified.");
+      return 1;
+    }
+
+    // Execute compaction!
+    if (mapred) {
+      return doMapReduce(fs, toCompactDirs, compactOnce);
+    } else {
+      return doClient(fs, toCompactDirs, compactOnce);
+    }
+  }
+
+  private void printUsage() {
+    printUsage(null);
+  }
+
+  private void printUsage(final String message) {
+    if (message != null && message.length() > 0) {
+      System.err.println(message);
+    }
+    System.err.println("Usage: java " + this.getClass().getName() + " \\");
+    System.err.println("  [-compactOnce] [-mapred] [-D<property=value>]* files...");
+    System.err.println();
+    System.err.println("Options:");
+    System.err.println(" mapred         Use MapReduce to run compaction.");
+    System.err.println(" compactOnce    Execute just one compaction step. (default: while needed)");
+    System.err.println();
+    System.err.println("Note: -D properties will be applied to the conf used. ");
+    System.err.println("For example: ");
+    System.err.println(" To preserve input files, pass -D"+CONF_COMPLETE_COMPACTION+"=false");
+    System.err.println(" To stop delete of compacted file, pass -D"+CONF_DELETE_COMPACTED+"=false");
+    System.err.println(" To set tmp dir, pass -D"+CONF_TMP_DIR+"=ALTERNATE_DIR");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println(" To compact the full 'TestTable' using MapReduce:");
+    System.err.println(" $ bin/hbase " + this.getClass().getName() + " -mapred hdfs:///hbase/TestTable");
+    System.err.println();
+    System.err.println(" To compact column family 'x' of the table 'TestTable' region 'abc':");
+    System.err.println(" $ bin/hbase " + this.getClass().getName() + " hdfs:///hbase/TestTable/abc/x");
+  }
+
+  public static void main(String[] args) throws Exception {
+    System.exit(ToolRunner.run(HBaseConfiguration.create(), new CompactionTool(), args));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
new file mode 100644
index 000000000000..bba7a9a5306e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Compact passed set of files.
+ * Create an instance and then call {@ink #compact(Store, Collection, boolean, long)}.
+ */
+@InterfaceAudience.Private
+class Compactor extends Configured {
+  private static final Log LOG = LogFactory.getLog(Compactor.class);
+  private CompactionProgress progress;
+
+  Compactor(final Configuration c) {
+    super(c);
+  }
+
+  /**
+   * Do a minor/major compaction on an explicit set of storefiles from a Store.
+   *
+   * @param store Store the files belong to
+   * @param filesToCompact which files to compact
+   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
+   * @param maxId Readers maximum sequence id.
+   * @return Product of compaction or null if all cells expired or deleted and
+   * nothing made it through the compaction.
+   * @throws IOException
+   */
+  StoreFile.Writer compact(final Store store,
+      final Collection<StoreFile> filesToCompact,
+      final boolean majorCompaction, final long maxId)
+  throws IOException {
+    // Calculate maximum key count after compaction (for blooms)
+    // Also calculate earliest put timestamp if major compaction
+    int maxKeyCount = 0;
+    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+    for (StoreFile file: filesToCompact) {
+      StoreFile.Reader r = file.getReader();
+      if (r == null) {
+        LOG.warn("Null reader for " + file.getPath());
+        continue;
+      }
+      // NOTE: getFilterEntries could cause under-sized blooms if the user
+      // switches bloom type (e.g. from ROW to ROWCOL)
+      long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType())?
+        r.getFilterEntries() : r.getEntries();
+      maxKeyCount += keyCount;
+      // For major compactions calculate the earliest put timestamp of all
+      // involved storefiles. This is used to remove family delete marker during
+      // compaction.
+      if (majorCompaction) {
+        byte [] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
+        if (tmp == null) {
+          // There's a file with no information, must be an old one
+          // assume we have very old puts
+          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
+        } else {
+          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
+        }
+      }
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Compacting " + file +
+          ", keycount=" + keyCount +
+          ", bloomtype=" + r.getBloomFilterType().toString() +
+          ", size=" + StringUtils.humanReadableInt(r.length()) +
+          ", encoding=" + r.getHFileReader().getEncodingOnDisk() +
+          (majorCompaction? ", earliestPutTs=" + earliestPutTs: ""));
+      }
+    }
+
+    // keep track of compaction progress
+    this.progress = new CompactionProgress(maxKeyCount);
+
+    // For each file, obtain a scanner:
+    List<StoreFileScanner> scanners = StoreFileScanner
+      .getScannersForStoreFiles(filesToCompact, false, false, true);
+
+    // Get some configs
+    int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
+    Compression.Algorithm compression = store.getFamily().getCompression();
+    // Avoid overriding compression setting for major compactions if the user
+    // has not specified it separately
+    Compression.Algorithm compactionCompression =
+      (store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE) ?
+      store.getFamily().getCompactionCompression(): compression;
+    // Make the instantiation lazy in case compaction produces no product; i.e.
+    // where all source cells are expired or deleted.
+    StoreFile.Writer writer = null;
+    // Find the smallest read point across all the Scanners.
+    long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
+    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
+    try {
+      InternalScanner scanner = null;
+      try {
+        Scan scan = new Scan();
+        scan.setMaxVersions(store.getFamily().getMaxVersions());
+        /* Include deletes, unless we are doing a major compaction */
+        scanner = new StoreScanner(store, store.getScanInfo(), scan, scanners,
+          majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
+          smallestReadPoint, earliestPutTs);
+        if (store.getHRegion().getCoprocessorHost() != null) {
+          InternalScanner cpScanner =
+            store.getHRegion().getCoprocessorHost().preCompact(store, scanner);
+          // NULL scanner returned from coprocessor hooks means skip normal processing
+          if (cpScanner == null) {
+            return null;
+          }
+          scanner = cpScanner;
+        }
+
+        int bytesWritten = 0;
+        // Since scanner.next() can return 'false' but still be delivering data,
+        // we have to use a do/while loop.
+        List<KeyValue> kvs = new ArrayList<KeyValue>();
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        boolean hasMore;
+        do {
+          hasMore = scanner.next(kvs, compactionKVMax);
+          if (writer == null && !kvs.isEmpty()) {
+            writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true);
+          }
+          if (writer != null) {
+            // output to writer:
+            for (KeyValue kv : kvs) {
+              if (kv.getMemstoreTS() <= smallestReadPoint) {
+                kv.setMemstoreTS(0);
+              }
+              writer.append(kv);
+              // update progress per key
+              ++progress.currentCompactedKVs;
+
+              // check periodically to see if a system stop is requested
+              if (Store.closeCheckInterval > 0) {
+                bytesWritten += kv.getLength();
+                if (bytesWritten > Store.closeCheckInterval) {
+                  bytesWritten = 0;
+                  isInterrupted(store, writer);
+                }
+              }
+            }
+          }
+          kvs.clear();
+        } while (hasMore);
+      } finally {
+        if (scanner != null) {
+          scanner.close();
+        }
+      }
+    } finally {
+      if (writer != null) {
+        writer.appendMetadata(maxId, majorCompaction);
+        writer.close();
+      }
+    }
+    return writer;
+  }
+
+  void isInterrupted(final Store store, final StoreFile.Writer writer)
+  throws IOException {
+    if (store.getHRegion().areWritesEnabled()) return;
+    // Else cleanup.
+    writer.close();
+    store.getFileSystem().delete(writer.getPath(), false);
+    throw new InterruptedIOException( "Aborting compaction of store " + store +
+      " in region " + store.getHRegion() + " because user requested stop.");
+  }
+
+  CompactionProgress getProgress() {
+    return this.progress;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 6371820ed281..442ff1823641 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.UnsupportedEncodingException;
@@ -62,6 +63,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -230,12 +232,12 @@ public class HRegion implements HeapSize { // , Writable{
    * The directory for the table this region is part of.
    * This directory contains the directory for this region.
    */
-  final Path tableDir;
+  private final Path tableDir;
 
-  final HLog log;
-  final FileSystem fs;
-  final Configuration conf;
-  final int rowLockWaitDuration;
+  private final HLog log;
+  private final FileSystem fs;
+  private final Configuration conf;
+  private final int rowLockWaitDuration;
   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
 
   // The internal wait duration to acquire a lock before read/update
@@ -256,8 +258,8 @@ public class HRegion implements HeapSize { // , Writable{
   // purge timeout, when a RPC call will be terminated by the RPC engine.
   final long maxBusyWaitDuration;
 
-  final HRegionInfo regionInfo;
-  final Path regiondir;
+  private final HRegionInfo regionInfo;
+  private final Path regiondir;
   KeyValue.KVComparator comparator;
 
   private ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;
@@ -724,7 +726,7 @@ public AtomicLong getMemstoreSize() {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     if (this.rsAccounting != null) {
       rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-    }  
+    }
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -750,7 +752,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
 
     // and then create the file
     Path tmpPath = new Path(getTmpDir(), REGIONINFO_FILE);
-    
+
     // if datanode crashes or if the RS goes down just before the close is called while trying to
     // close the created regioninfo file in the .tmp directory then on next
     // creation we will be getting AlreadyCreatedException.
@@ -758,7 +760,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     if (FSUtils.isExists(fs, tmpPath)) {
       FSUtils.delete(fs, tmpPath, true);
     }
-    
+
     FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
 
     try {
@@ -775,6 +777,26 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     }
   }
 
+  /**
+   * @param fs
+   * @param dir
+   * @return An HRegionInfo instance gotten from the <code>.regioninfo</code> file under region dir
+   * @throws IOException
+   */
+  public static HRegionInfo loadDotRegionInfoFileContent(final FileSystem fs, final Path dir)
+  throws IOException {
+    Path regioninfo = new Path(dir, HRegion.REGIONINFO_FILE);
+    if (!fs.exists(regioninfo)) throw new FileNotFoundException(regioninfo.toString());
+    FSDataInputStream in = fs.open(regioninfo);
+    try {
+      HRegionInfo hri = new HRegionInfo();
+      hri.readFields(in);
+      return hri;
+    } finally {
+      in.close();
+    }
+  }
+
   /** @return a HRegionInfo object for this region */
   public HRegionInfo getRegionInfo() {
     return this.regionInfo;
@@ -1021,19 +1043,16 @@ protected ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(
     return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
   }
 
-  private ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
+  static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
       final String threadNamePrefix) {
-    ThreadPoolExecutor openAndCloseThreadPool = Threads
-        .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
-            new ThreadFactory() {
-              private int count = 1;
-
-              public Thread newThread(Runnable r) {
-                Thread t = new Thread(r, threadNamePrefix + "-" + count++);
-                return t;
-              }
-            });
-    return openAndCloseThreadPool;
+    return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+      new ThreadFactory() {
+        private int count = 1;
+
+        public Thread newThread(Runnable r) {
+          return new Thread(r, threadNamePrefix + "-" + count++);
+        }
+      });
   }
 
    /**
@@ -1979,7 +1998,7 @@ public OperationStatus[] put(Pair<Put, Integer>[] putsAndLocks) throws IOExcepti
     System.arraycopy(putsAndLocks, 0, mutationsAndLocks, 0, putsAndLocks.length);
     return batchMutate(mutationsAndLocks);
   }
-  
+
   /**
    * Perform a batch of mutations.
    * It supports only Put and Delete mutations and will ignore other types passed.
@@ -2333,7 +2352,7 @@ private long doMiniBatchMutation(
 
       // do after lock
       final long netTimeMs = EnvironmentEdgeManager.currentTimeMillis()- startTimeMs;
-            
+
       // See if the column families were consistent through the whole thing.
       // if they were then keep them. If they were not then pass a null.
       // null will be treated as unknown.
@@ -2636,7 +2655,7 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updatePutMetrics(familyMap.keySet(), after - now);
-    
+
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -3754,6 +3773,7 @@ public static void closeHRegion(final HRegion r) throws IOException {
    * @param conf
    * @param hTableDescriptor
    * @param hlog shared HLog
+   * @param boolean initialize - true to initialize the region
    * @return new HRegion
    *
    * @throws IOException
@@ -3761,7 +3781,36 @@ public static void closeHRegion(final HRegion r) throws IOException {
   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                       final Configuration conf,
                                       final HTableDescriptor hTableDescriptor,
-                                      final HLog hlog)
+                                      final HLog hlog,
+                                      final boolean initialize)
+      throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor,
+        hlog, initialize, false);
+  }
+
+  /**
+   * Convenience method creating new HRegions. Used by createTable.
+   * The {@link HLog} for the created region needs to be closed
+   * explicitly, if it is not null.
+   * Use {@link HRegion#getLog()} to get access.
+   *
+   * @param info Info for region to create.
+   * @param rootDir Root directory for HBase instance
+   * @param conf
+   * @param hTableDescriptor
+   * @param hlog shared HLog
+   * @param boolean initialize - true to initialize the region
+   * @param boolean ignoreHLog
+      - true to skip generate new hlog if it is null, mostly for createTable
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog,
+                                      final boolean initialize, final boolean ignoreHLog)
       throws IOException {
     LOG.info("creating HRegion " + info.getTableNameAsString()
         + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
@@ -3773,16 +3822,26 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
     FileSystem fs = FileSystem.get(conf);
     fs.mkdirs(regionDir);
     HLog effectiveHLog = hlog;
-    if (hlog == null) {
+    if (hlog == null && !ignoreHLog) {
       effectiveHLog = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
           new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf);
     }
     HRegion region = HRegion.newHRegion(tableDir,
         effectiveHLog, fs, conf, info, hTableDescriptor, null);
-    region.initialize();
+    if (initialize) {
+      region.initialize();
+    }
     return region;
   }
 
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog)
+    throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor, hlog, true);
+  }
+
   /**
    * Open a Region.
    * @param info Info for region to be opened.
@@ -4294,7 +4353,7 @@ private List<KeyValue> get(Get get, boolean withCoprocessor)
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updateGetMetrics(get.familySet(), after - now);
-    
+
     return results;
   }
 
@@ -4622,10 +4681,10 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
       closeRegionOperation();
     }
 
-    
+
     long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - before);
-    
+
     if (flush) {
       // Request a cache flush. Do it outside update lock.
       requestFlush();
@@ -4750,7 +4809,7 @@ public Result increment(Increment increment, Integer lockid,
       long after = EnvironmentEdgeManager.currentTimeMillis();
       this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
     }
-    
+
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -5243,7 +5302,7 @@ private void closeBulkRegionOperation() {
    */
   private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
     if (numPutsWithoutWAL.getAndIncrement() == 0) {
-      LOG.info("writing data to region " + this + 
+      LOG.info("writing data to region " + this +
                " with WAL disabled. Data may be lost in the event of a crash.");
     }
 
@@ -5355,11 +5414,11 @@ public static void main(String[] args) throws IOException {
     final HLog log = new HLog(fs, logdir, oldLogDir, c);
     try {
       processTable(fs, tableDir, log, c, majorCompact);
-     } finally {
+    } finally {
        log.close();
        // TODO: is this still right?
        BlockCache bc = new CacheConfig(c).getBlockCache();
        if (bc != null) bc.shutdown();
-     }
+    }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index f9e1103b0c10..9a02c16c77c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
-import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -132,9 +131,6 @@ public class Store extends SchemaConfigured implements HeapSize {
   private volatile long totalUncompressedBytes = 0L;
   private final Object flushLock = new Object();
   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final String storeNameStr;
-  private CompactionProgress progress;
-  private final int compactionKVMax;
   private final boolean verifyBulkLoads;
 
   /* The default priority for user-specified compaction requests.
@@ -158,10 +154,6 @@ public class Store extends SchemaConfigured implements HeapSize {
     new CopyOnWriteArraySet<ChangedReadersObserver>();
 
   private final int blocksize;
-  /** Compression algorithm for flush files and minor compaction */
-  private final Compression.Algorithm compression;
-  /** Compression algorithm for major compaction */
-  private final Compression.Algorithm compactionCompression;
   private HFileDataBlockEncoder dataBlockEncoder;
 
   /** Checksum configuration */
@@ -171,6 +163,8 @@ public class Store extends SchemaConfigured implements HeapSize {
   // Comparing KeyValues
   final KeyValue.KVComparator comparator;
 
+  private final Compactor compactor;
+
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -185,25 +179,16 @@ public class Store extends SchemaConfigured implements HeapSize {
   protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     FileSystem fs, Configuration conf)
   throws IOException {
-    super(conf, region.getTableDesc().getNameAsString(),
+    super(conf, region.getRegionInfo().getTableNameAsString(),
         Bytes.toString(family.getName()));
-    HRegionInfo info = region.regionInfo;
+    HRegionInfo info = region.getRegionInfo();
     this.fs = fs;
-    this.homedir = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
-    if (!this.fs.exists(this.homedir)) {
-      if (!this.fs.mkdirs(this.homedir))
-        throw new IOException("Failed create of: " + this.homedir.toString());
-    }
+    Path p = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
+    this.homedir = createStoreHomeDir(this.fs, p);
     this.region = region;
     this.family = family;
     this.conf = conf;
     this.blocksize = family.getBlocksize();
-    this.compression = family.getCompression();
-    // avoid overriding compression setting for major compactions if the user
-    // has not specified it separately
-    this.compactionCompression =
-      (family.getCompactionCompression() != Compression.Algorithm.NONE) ?
-        family.getCompactionCompression() : this.compression;
 
     this.dataBlockEncoder =
         new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(),
@@ -228,7 +213,6 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
         "ms in store " + this);
     scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
     this.memstore = new MemStore(conf, this.comparator);
-    this.storeNameStr = getColumnFamilyName();
 
     // By default, compact if storefile.count >= minFilesToCompact
     this.minFilesToCompact = Math.max(2,
@@ -245,10 +229,8 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       this.region.memstoreFlushSize);
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
-    this.compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
 
-    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify",
-        false);
+    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
 
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
@@ -260,6 +242,47 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.checksumType = getChecksumType(conf);
     // initilize bytes per checksum
     this.bytesPerChecksum = getBytesPerChecksum(conf);
+    // Create a compaction tool instance
+    this.compactor = new Compactor(this.conf);
+  }
+
+  /**
+   * @param family
+   * @return
+   */
+  long getTTL(final HColumnDescriptor family) {
+    // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
+    long ttl = family.getTimeToLive();
+    if (ttl == HConstants.FOREVER) {
+      // Default is unlimited ttl.
+      ttl = Long.MAX_VALUE;
+    } else if (ttl == -1) {
+      ttl = Long.MAX_VALUE;
+    } else {
+      // Second -> ms adjust for user data
+      ttl *= 1000;
+    }
+    return ttl;
+  }
+
+  /**
+   * Create this store's homedir
+   * @param fs
+   * @param homedir
+   * @return Return <code>homedir</code>
+   * @throws IOException
+   */
+  Path createStoreHomeDir(final FileSystem fs,
+      final Path homedir) throws IOException {
+    if (!fs.exists(homedir)) {
+      if (!fs.mkdirs(homedir))
+        throw new IOException("Failed create of: " + homedir.toString());
+    }
+    return homedir;
+  }
+
+  FileSystem getFileSystem() {
+    return this.fs;
   }
 
   /**
@@ -320,7 +343,7 @@ public static Path getStoreHomedir(final Path tabledir,
    * Return the directory in which this store stores its
    * StoreFiles
    */
-  public Path getHomedir() {
+  Path getHomedir() {
     return homedir;
   }
 
@@ -339,6 +362,10 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
     this.dataBlockEncoder = blockEncoder;
   }
 
+  FileStatus [] getStoreFiles() throws IOException {
+    return FSUtils.listStatus(this.fs, this.homedir, null);
+  }
+
   /**
    * Creates an unsorted list of StoreFile loaded in parallel
    * from the given directory.
@@ -346,7 +373,7 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
    */
   private List<StoreFile> loadStoreFiles() throws IOException {
     ArrayList<StoreFile> results = new ArrayList<StoreFile>();
-    FileStatus files[] = FSUtils.listStatus(this.fs, this.homedir, null);
+    FileStatus files[] = getStoreFiles();
 
     if (files == null || files.length == 0) {
       return results;
@@ -637,7 +664,7 @@ public Void call() throws IOException {
           storeFileCloserThreadPool.shutdownNow();
         }
       }
-      LOG.debug("closed " + this.storeNameStr);
+      LOG.info("Closed " + this);
       return result;
     } finally {
       this.lock.writeLock().unlock();
@@ -723,6 +750,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
       scanner = cpScanner;
     }
     try {
+      int compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
       // TODO:  We can fail in the below block before we complete adding this
       // flush to list of store files.  Add cleanup of anything put on filesystem
       // if we fail.
@@ -736,7 +764,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
           List<KeyValue> kvs = new ArrayList<KeyValue>();
           boolean hasMore;
           do {
-            hasMore = scanner.next(kvs, this.compactionKVMax);
+            hasMore = scanner.next(kvs, compactionKVMax);
             if (!kvs.isEmpty()) {
               for (KeyValue kv : kvs) {
                 // If we know that this KV is going to be included always, then let us
@@ -828,7 +856,7 @@ private StoreFile commitFile(final Path path,
    */
   private StoreFile.Writer createWriterInTmp(int maxKeyCount)
   throws IOException {
-    return createWriterInTmp(maxKeyCount, this.compression, false);
+    return createWriterInTmp(maxKeyCount, this.family.getCompression(), false);
   }
 
   /*
@@ -981,16 +1009,12 @@ void deleteChangedReaderObserver(ChangedReadersObserver o) {
    * @param cr
    *          compaction details obtained from requestCompaction()
    * @throws IOException
+   * @return Storefile we compacted into or null if we failed or opted out early.
    */
-  void compact(CompactionRequest cr) throws IOException {
-    if (cr == null || cr.getFiles().isEmpty()) {
-      return;
-    }
-    Preconditions.checkArgument(cr.getStore().toString()
-        .equals(this.toString()));
-
+  StoreFile compact(CompactionRequest cr) throws IOException {
+    if (cr == null || cr.getFiles().isEmpty()) return null;
+    Preconditions.checkArgument(cr.getStore().toString().equals(this.toString()));
     List<StoreFile> filesToCompact = cr.getFiles();
-
     synchronized (filesCompacting) {
       // sanity check: we're compacting files that this store knows about
       // TODO: change this to LOG.error() after more debugging
@@ -1002,19 +1026,26 @@ void compact(CompactionRequest cr) throws IOException {
 
     // Ready to go. Have list of files to compact.
     LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
-        + this.storeNameStr + " of "
+        + this + " of "
         + this.region.getRegionInfo().getRegionNameAsString()
         + " into tmpdir=" + region.getTmpDir() + ", seqid=" + maxId + ", totalSize="
         + StringUtils.humanReadableInt(cr.getSize()));
 
     StoreFile sf = null;
     try {
-      StoreFile.Writer writer = compactStore(filesToCompact, cr.isMajor(),
-          maxId);
+      StoreFile.Writer writer =
+        this.compactor.compact(this, filesToCompact, cr.isMajor(), maxId);
       // Move the compaction into place.
-      sf = completeCompaction(filesToCompact, writer);
-      if (region.getCoprocessorHost() != null) {
-        region.getCoprocessorHost().postCompact(this, sf);
+      if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
+        sf = completeCompaction(filesToCompact, writer);
+        if (region.getCoprocessorHost() != null) {
+          region.getCoprocessorHost().postCompact(this, sf);
+        }
+      } else {
+        // Create storefile around what we wrote with a reader on it.
+        sf = new StoreFile(this.fs, writer.getPath(), this.conf, this.cacheConf,
+          this.family.getBloomFilterType(), this.dataBlockEncoder);
+        sf.createReader();
       }
     } finally {
       synchronized (filesCompacting) {
@@ -1023,7 +1054,7 @@ void compact(CompactionRequest cr) throws IOException {
     }
 
     LOG.info("Completed" + (cr.isMajor() ? " major " : " ") + "compaction of "
-        + filesToCompact.size() + " file(s) in " + this.storeNameStr + " of "
+        + filesToCompact.size() + " file(s) in " + this + " of "
         + this.region.getRegionInfo().getRegionNameAsString()
         + " into " +
         (sf == null ? "none" : sf.getPath().getName()) +
@@ -1031,6 +1062,7 @@ void compact(CompactionRequest cr) throws IOException {
           StringUtils.humanReadableInt(sf.getReader().length()))
         + "; total size for store is "
         + StringUtils.humanReadableInt(storeSize));
+    return sf;
   }
 
   /**
@@ -1070,7 +1102,8 @@ public void compactRecentForTesting(int N) throws IOException {
 
     try {
       // Ready to go. Have list of files to compact.
-      StoreFile.Writer writer = compactStore(filesToCompact, isMajor, maxId);
+      StoreFile.Writer writer =
+        this.compactor.compact(this, filesToCompact, isMajor, maxId);
       // Move the compaction into place.
       StoreFile sf = completeCompaction(filesToCompact, writer);
       if (region.getCoprocessorHost() != null) {
@@ -1119,10 +1152,10 @@ public static long getLowestTimestamp(final List<StoreFile> candidates)
   }
 
   /** getter for CompactionProgress object
-   * @return CompactionProgress object
+   * @return CompactionProgress object; can be null
    */
   public CompactionProgress getCompactionProgress() {
-    return this.progress;
+    return this.compactor.getProgress();
   }
 
   /*
@@ -1174,19 +1207,19 @@ private boolean isMajorCompaction(final List<StoreFile> filesToCompact) throws I
         if (sf.isMajorCompaction() &&
             (this.ttl == HConstants.FOREVER || oldest < this.ttl)) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Skipping major compaction of " + this.storeNameStr +
+            LOG.debug("Skipping major compaction of " + this +
                 " because one (major) compacted file only and oldestTime " +
                 oldest + "ms is < ttl=" + this.ttl);
           }
         } else if (this.ttl != HConstants.FOREVER && oldest > this.ttl) {
-          LOG.debug("Major compaction triggered on store " + this.storeNameStr +
+          LOG.debug("Major compaction triggered on store " + this +
             ", because keyvalues outdated; time since last major compaction " +
             (now - lowTimestamp) + "ms");
           result = true;
         }
       } else {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Major compaction triggered on store " + this.storeNameStr +
+          LOG.debug("Major compaction triggered on store " + this +
               "; time since last major compaction " + (now - lowTimestamp) + "ms");
         }
         result = true;
@@ -1376,12 +1409,12 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
              compactSelection.getFilesToCompact().get(pos).getReader().length()
                > maxCompactSize &&
              !compactSelection.getFilesToCompact().get(pos).isReference()) ++pos;
-      compactSelection.clearSubList(0, pos);
+      if (pos != 0) compactSelection.clearSubList(0, pos);
     }
 
     if (compactSelection.getFilesToCompact().isEmpty()) {
       LOG.debug(this.getHRegionInfo().getEncodedName() + " - " +
-        this.storeNameStr + ": no store files to compact");
+        this + ": no store files to compact");
       compactSelection.emptyFileList();
       return compactSelection;
     }
@@ -1468,7 +1501,7 @@ public boolean apply(StoreFile input) {
       // if we don't have enough files to compact, just wait
       if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Skipped compaction of " + this.storeNameStr
+          LOG.debug("Skipped compaction of " + this
             + ".  Only " + (end - start) + " file(s) of size "
             + StringUtils.humanReadableInt(totalSize)
             + " have met compaction criteria.");
@@ -1494,149 +1527,6 @@ public boolean apply(StoreFile input) {
     return compactSelection;
   }
 
-  /**
-   * Do a minor/major compaction on an explicit set of storefiles in a Store.
-   * Uses the scan infrastructure to make it easy.
-   *
-   * @param filesToCompact which files to compact
-   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
-   * @param maxId Readers maximum sequence id.
-   * @return Product of compaction or null if all cells expired or deleted and
-   * nothing made it through the compaction.
-   * @throws IOException
-   */
-  StoreFile.Writer compactStore(final Collection<StoreFile> filesToCompact,
-                               final boolean majorCompaction, final long maxId)
-      throws IOException {
-    // calculate maximum key count after compaction (for blooms)
-    int maxKeyCount = 0;
-    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
-    for (StoreFile file : filesToCompact) {
-      StoreFile.Reader r = file.getReader();
-      if (r != null) {
-        // NOTE: getFilterEntries could cause under-sized blooms if the user
-        //       switches bloom type (e.g. from ROW to ROWCOL)
-        long keyCount = (r.getBloomFilterType() == family.getBloomFilterType())
-          ? r.getFilterEntries() : r.getEntries();
-        maxKeyCount += keyCount;
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Compacting " + file +
-            ", keycount=" + keyCount +
-            ", bloomtype=" + r.getBloomFilterType().toString() +
-            ", size=" + StringUtils.humanReadableInt(r.length()) +
-            ", encoding=" + r.getHFileReader().getEncodingOnDisk());
-        }
-      }
-      // For major compactions calculate the earliest put timestamp
-      // of all involved storefiles. This is used to remove 
-      // family delete marker during the compaction.
-      if (majorCompaction) {
-        byte[] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
-        if (tmp == null) {
-          // there's a file with no information, must be an old one
-          // assume we have very old puts
-          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
-        } else {
-          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
-        }
-      }
-    }
-
-    // keep track of compaction progress
-    progress = new CompactionProgress(maxKeyCount);
-
-    // For each file, obtain a scanner:
-    List<StoreFileScanner> scanners = StoreFileScanner
-      .getScannersForStoreFiles(filesToCompact, false, false, true);
-
-    // Make the instantiation lazy in case compaction produces no product; i.e.
-    // where all source cells are expired or deleted.
-    StoreFile.Writer writer = null;
-    // Find the smallest read point across all the Scanners.
-    long smallestReadPoint = region.getSmallestReadPoint();
-    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
-    try {
-      InternalScanner scanner = null;
-      try {
-        if (getHRegion().getCoprocessorHost() != null) {
-          scanner = getHRegion()
-              .getCoprocessorHost()
-              .preCompactScannerOpen(this, scanners,
-                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
-        }
-        if (scanner == null) {
-          Scan scan = new Scan();
-          scan.setMaxVersions(getFamily().getMaxVersions());
-          /* Include deletes, unless we are doing a major compaction */
-          scanner = new StoreScanner(this, getScanInfo(), scan, scanners,
-            majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
-            smallestReadPoint, earliestPutTs);
-        }
-        if (getHRegion().getCoprocessorHost() != null) {
-          InternalScanner cpScanner =
-            getHRegion().getCoprocessorHost().preCompact(this, scanner);
-          // NULL scanner returned from coprocessor hooks means skip normal processing
-          if (cpScanner == null) {
-            return null;
-          }
-          scanner = cpScanner;
-        }
-
-        int bytesWritten = 0;
-        // since scanner.next() can return 'false' but still be delivering data,
-        // we have to use a do/while loop.
-        ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
-        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
-        boolean hasMore;
-        do {
-          hasMore = scanner.next(kvs, this.compactionKVMax);
-          if (writer == null && !kvs.isEmpty()) {
-            writer = createWriterInTmp(maxKeyCount, this.compactionCompression,
-                true);
-          }
-          if (writer != null) {
-            // output to writer:
-            for (KeyValue kv : kvs) {
-              if (kv.getMemstoreTS() <= smallestReadPoint) {
-                kv.setMemstoreTS(0);
-              }
-              writer.append(kv);
-              // update progress per key
-              ++progress.currentCompactedKVs;
-
-              // check periodically to see if a system stop is requested
-              if (Store.closeCheckInterval > 0) {
-                bytesWritten += kv.getLength();
-                if (bytesWritten > Store.closeCheckInterval) {
-                  bytesWritten = 0;
-                  if (!this.region.areWritesEnabled()) {
-                    writer.close();
-                    fs.delete(writer.getPath(), false);
-                    throw new InterruptedIOException(
-                        "Aborting compaction of store " + this +
-                        " in region " + this.region +
-                        " because user requested stop.");
-                  }
-                }
-              }
-            }
-          }
-          kvs.clear();
-        } while (hasMore);
-      } finally {
-        if (scanner != null) {
-          scanner.close();
-        }
-      }
-    } finally {
-      if (writer != null) {
-        writer.appendMetadata(maxId, majorCompaction);
-        writer.close();
-      }
-    }
-    return writer;
-  }
-
   /**
    * Validates a store file by opening and closing it. In HFileV2 this should
    * not be an expensive operation.
@@ -1741,7 +1631,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
 
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
-      LOG.error("Failed replacing compacted files in " + this.storeNameStr +
+      LOG.error("Failed replacing compacted files in " + this +
         ". Compacted file is " + (result == null? "none": result.toString()) +
         ".  Files replaced " + compactedFiles.toString() +
         " some of which may have been already removed", e);
@@ -2027,7 +1917,7 @@ public byte[] getSplitPoint() {
         return mk.getRow();
       }
     } catch(IOException e) {
-      LOG.warn("Failed getting store size for " + this.storeNameStr, e);
+      LOG.warn("Failed getting store size for " + this, e);
     } finally {
       this.lock.readLock().unlock();
     }
@@ -2080,7 +1970,7 @@ public KeyValueScanner getScanner(Scan scan,
 
   @Override
   public String toString() {
-    return this.storeNameStr;
+    return getColumnFamilyName();
   }
 
   /**
@@ -2196,7 +2086,7 @@ public HRegion getHRegion() {
   }
 
   HRegionInfo getHRegionInfo() {
-    return this.region.regionInfo;
+    return this.region.getRegionInfo();
   }
 
   /**
@@ -2324,8 +2214,8 @@ public CacheConfig getCacheConfig() {
 
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
-          + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+          + (17 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
index 9bc66e1413fd..f91b7822c4ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
@@ -49,5 +49,4 @@ public CompactionProgress(long totalCompactingKVs) {
   public float getProgressPct() {
     return currentCompactedKVs / totalCompactingKVs;
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
index d2329e10c588..885625b044c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
@@ -25,9 +25,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.util.ChecksumFactory;
-
 /**
  * Checksum types. The Checksum type is a one byte number
  * that stores a representation of the checksum algorithm
@@ -70,7 +67,7 @@ public void initialize() {
         ctor = ChecksumFactory.newConstructor(PURECRC32);
         LOG.info("Checksum using " + PURECRC32);
       } catch (Exception e) {
-        LOG.info(PURECRC32 + " not available.");
+        LOG.trace(PURECRC32 + " not available.");
       }
       try {
         // The default checksum class name is java.util.zip.CRC32. 
@@ -80,7 +77,7 @@ public void initialize() {
           LOG.info("Checksum can use " + JDKCRC);
         }
       } catch (Exception e) {
-        LOG.warn(JDKCRC + " not available. ",  e);
+        LOG.trace(JDKCRC + " not available.");
       }
     }
 
@@ -113,7 +110,7 @@ public void initialize() {
         ctor = ChecksumFactory.newConstructor(PURECRC32C);
         LOG.info("Checksum can use " + PURECRC32C);
       } catch (Exception e) {
-        LOG.info(PURECRC32C + " not available. ");
+        LOG.trace(PURECRC32C + " not available.");
       }
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 27b61a25e634..18074efa6042 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -151,7 +151,7 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
    */
   public static FSDataOutputStream create(FileSystem fs, Path path,
       FsPermission perm, boolean overwrite) throws IOException {
-    LOG.debug("Creating file:" + path + "with permission:" + perm);
+    LOG.debug("Creating file=" + path + " with permission=" + perm);
 
     return fs.create(path, perm, overwrite,
         fs.getConf().getInt("io.file.buffer.size", 4096),
@@ -1012,6 +1012,25 @@ public boolean accept(Path rd) {
     }
   }
 
+  /**
+   * Given a particular region dir, return all the familydirs inside it
+   *
+   * @param fs A file system for the Path
+   * @param regionDir Path to a specific region directory
+   * @return List of paths to valid family directories in region dir.
+   * @throws IOException
+   */
+  public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
+    // assumes we are in a region dir.
+    FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
+    List<Path> familyDirs = new ArrayList<Path>(fds.length);
+    for (FileStatus fdfs: fds) {
+      Path fdPath = fdfs.getPath();
+      familyDirs.add(fdPath);
+    }
+    return familyDirs;
+  }
+
   /**
    * Filter for HFiles that excludes reference files.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 8134f4a93ecb..93908e7c8670 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -587,8 +587,10 @@ public void testCompactionWithCorruptResult() throws Exception {
 
     List<StoreFile> storeFiles = store.getStorefiles();
     long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
+    Compactor tool = new Compactor(this.conf);
 
-    StoreFile.Writer compactedFile = store.compactStore(storeFiles, false, maxId);
+    StoreFile.Writer compactedFile =
+      tool.compact(store, storeFiles, false, maxId);
 
     // Now lets corrupt the compacted file.
     FileSystem fs = FileSystem.get(conf);

From e38e3bd945deee8f562f2673dca64bd3ba3f4dc4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 6 Dec 2012 00:29:20 +0000
Subject: [PATCH 0611/1540] HBASE-7279 Avoid copying the rowkey in
 RegionScanner, StoreScanner, and ScanQueryMatcher

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417716 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     | 35 ++++-----------
 .../apache/hadoop/hbase/client/Result.java    |  2 +-
 .../hadoop/hbase/regionserver/HRegion.java    | 45 ++++++++++---------
 .../hbase/regionserver/ScanQueryMatcher.java  |  8 +++-
 .../hbase/regionserver/StoreScanner.java      | 14 ++++--
 .../hbase/regionserver/TestQueryMatcher.java  | 12 +++--
 6 files changed, 59 insertions(+), 57 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index bd87c34fb542..c9c962ec9d0f 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -216,9 +216,6 @@ public static Type codeToType(final byte b) {
   private int offset = 0;
   private int length = 0;
 
-  // the row cached
-  private volatile byte [] rowCache = null;
-
   /**
    * @return True if a delete type, a {@link KeyValue.Type#Delete} or
    * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn}
@@ -987,7 +984,6 @@ public boolean updateLatestStamp(final byte [] now) {
       int tsOffset = getTimestampOffset();
       System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
       // clear cache or else getTimestamp() possibly returns an old value
-      timestampCache = -1L;
       return true;
     }
     return false;
@@ -1037,28 +1033,19 @@ public boolean updateLatestStamp(final byte [] now) {
    * @return Row in a new byte array.
    */
   public byte [] getRow() {
-    if (rowCache == null) {
-      int o = getRowOffset();
-      short l = getRowLength();
-      // initialize and copy the data into a local variable
-      // in case multiple threads race here.
-      byte local[] = new byte[l];
-      System.arraycopy(getBuffer(), o, local, 0, l);
-      rowCache = local; // volatile assign
-    }
-    return rowCache;
+    int o = getRowOffset();
+    short l = getRowLength();
+    byte result[] = new byte[l];
+    System.arraycopy(getBuffer(), o, result, 0, l);
+    return result;
   }
 
   /**
    *
    * @return Timestamp
    */
-  private long timestampCache = -1;
   public long getTimestamp() {
-    if (timestampCache == -1) {
-      timestampCache = getTimestamp(getKeyLength());
-    }
-    return timestampCache;
+    return getTimestamp(getKeyLength());
   }
 
   /**
@@ -2260,21 +2247,17 @@ int compareTimestamps(final long ltimestamp, final long rtimestamp) {
 
   // HeapSize
   public long heapSize() {
-    return ClassSize.align(ClassSize.OBJECT + (2 * ClassSize.REFERENCE) +
-        ClassSize.align(ClassSize.ARRAY) + ClassSize.align(length) +
-        (3 * Bytes.SIZEOF_INT) +
-        ClassSize.align(ClassSize.ARRAY) +
-        (2 * Bytes.SIZEOF_LONG));
+    return ClassSize.align(ClassSize.OBJECT + ClassSize.REFERENCE
+        + ClassSize.align(ClassSize.ARRAY) + ClassSize.align(length)
+        + (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_LONG);
   }
 
   // this overload assumes that the length bytes have already been read,
   // and it expects the length of the KeyValue to be explicitly passed
   // to it.
   public void readFields(int length, final DataInput in) throws IOException {
-    this.rowCache = null;
     this.length = length;
     this.offset = 0;
-    this.timestampCache = -1;
     this.keyLength = 0;
     this.bytes = new byte[this.length];
     in.readFully(this.bytes, 0, this.length);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java
index d80b8c9bc0d4..532359652505 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -96,7 +96,7 @@ public Result(KeyValue [] kvs) {
    * @param kvs List of KeyValues
    */
   public Result(List<KeyValue> kvs) {
-    this(kvs.toArray(new KeyValue[0]));
+    this(kvs.toArray(new KeyValue[kvs.size()]));
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 442ff1823641..87edae3ec72a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3567,8 +3567,16 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           rpcCall.throwExceptionIfCallerDisconnected();
         }
 
-        byte [] currentRow = peekRow();
-        if (isStopRow(currentRow)) {
+        KeyValue current = this.storeHeap.peek();
+        byte[] currentRow = null;
+        int offset = 0;
+        short length = 0;
+        if (current != null) {
+          currentRow = current.getBuffer();
+          offset = current.getRowOffset();
+          length = current.getRowLength();
+        }
+        if (isStopRow(currentRow, offset, length)) {
           if (filter != null && filter.hasFilterRow()) {
             filter.filterRow(results);
           }
@@ -3577,10 +3585,10 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           }
 
           return false;
-        } else if (filterRowKey(currentRow)) {
-          nextRow(currentRow);
+        } else if (filterRowKey(currentRow, offset, length)) {
+          nextRow(currentRow, offset, length);
         } else {
-          byte [] nextRow;
+          KeyValue nextKv;
           do {
             this.storeHeap.next(results, limit - results.size(), metric);
             if (limit > 0 && results.size() == limit) {
@@ -3590,9 +3598,10 @@ private boolean nextInternal(int limit, String metric) throws IOException {
               }
               return true; // we are expecting more yes, but also limited to how many we can return.
             }
-          } while (Bytes.equals(currentRow, nextRow = peekRow()));
+            nextKv = this.storeHeap.peek();
+          } while (nextKv != null && nextKv.matchingRow(currentRow, offset, length));
 
-          final boolean stopRow = isStopRow(nextRow);
+          final boolean stopRow = nextKv == null || isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
 
           // now that we have an entire row, lets process with a filters:
 
@@ -3607,7 +3616,7 @@ private boolean nextInternal(int limit, String metric) throws IOException {
             // the reasons for calling this method are:
             // 1. reset the filters.
             // 2. provide a hook to fast forward the row (used by subclasses)
-            nextRow(currentRow);
+            nextRow(currentRow, offset, length);
 
             // This row was totally filtered out, if this is NOT the last row,
             // we should continue on.
@@ -3623,29 +3632,25 @@ private boolean filterRow() {
       return filter != null
           && filter.filterRow();
     }
-    private boolean filterRowKey(byte[] row) {
+    private boolean filterRowKey(byte[] row, int offset, short length) {
       return filter != null
-          && filter.filterRowKey(row, 0, row.length);
+          && filter.filterRowKey(row, offset, length);
     }
 
-    protected void nextRow(byte [] currentRow) throws IOException {
-      while (Bytes.equals(currentRow, peekRow())) {
-        this.storeHeap.next(MOCKED_LIST);
+    protected void nextRow(byte [] currentRow, int offset, short length) throws IOException {
+      KeyValue next;
+      while((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
+        this.storeHeap.next(MOCKED_LIST);       
       }
       results.clear();
       resetFilters();
     }
 
-    private byte[] peekRow() {
-      KeyValue kv = this.storeHeap.peek();
-      return kv == null ? null : kv.getRow();
-    }
-
-    private boolean isStopRow(byte [] currentRow) {
+    private boolean isStopRow(byte [] currentRow, int offset, short length) {
       return currentRow == null ||
           (stopRow != null &&
           comparator.compareRows(stopRow, 0, stopRow.length,
-              currentRow, 0, currentRow.length) <= isScan);
+              currentRow, offset, length) <= isScan);
     }
 
     @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index d1cab8ef6167..4b52159b59b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -82,6 +82,8 @@ public class ScanQueryMatcher {
   /* row is not private for tests */
   /** Row the query is on */
   byte [] row;
+  int rowOffset;
+  short rowLength;
   
   /**
    * Oldest put in any of the involved store files
@@ -222,7 +224,7 @@ public MatchCode match(KeyValue kv) throws IOException {
     short rowLength = Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT);
     offset += Bytes.SIZEOF_SHORT;
 
-    int ret = this.rowComparator.compareRows(row, 0, row.length,
+    int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,
         bytes, offset, rowLength);
     if (ret <= -1) {
       return MatchCode.DONE;
@@ -385,8 +387,10 @@ public boolean moreRowsMayExistAfter(KeyValue kv) {
    * Set current row
    * @param row
    */
-  public void setRow(byte [] row) {
+  public void setRow(byte [] row, int offset, short length) {
     this.row = row;
+    this.rowOffset = offset;
+    this.rowLength = length;
     reset();
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index a87d326fd61f..bed4ad2cfacb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -340,8 +340,11 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
 
     // only call setRow if the row changes; avoids confusing the query matcher
     // if scanning intra-row
-    if ((matcher.row == null) || !peeked.matchingRow(matcher.row)) {
-      matcher.setRow(peeked.getRow());
+    byte[] row = peeked.getBuffer();
+    int offset = peeked.getRowOffset();
+    short length = peeked.getRowLength();
+    if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
+      matcher.setRow(row, offset, length);
     }
 
     KeyValue kv;
@@ -521,9 +524,12 @@ private void resetScannerStack(KeyValue lastTopKey) throws IOException {
     if (kv == null) {
       kv = lastTopKey;
     }
-    if ((matcher.row == null) || !kv.matchingRow(matcher.row)) {
+    byte[] row = kv.getBuffer();
+    int offset = kv.getRowOffset();
+    short length = kv.getRowLength();
+    if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
       matcher.reset();
-      matcher.setRow(kv.getRow());
+      matcher.setRow(row, offset, length);
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index 79db715467e4..956572c115c9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -113,7 +113,8 @@ public void testMatch_ExplicitColumns()
     memstore.add(new KeyValue(row2, fam1, col1, data));
 
     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
-    qm.setRow(memstore.get(0).getRow());
+    KeyValue k = memstore.get(0);
+    qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
 
     for (KeyValue kv : memstore){
       actual.add(qm.match(kv));
@@ -158,7 +159,8 @@ public void testMatch_Wildcard()
 
     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
 
-    qm.setRow(memstore.get(0).getRow());
+    KeyValue k = memstore.get(0);
+    qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
 
     for(KeyValue kv : memstore) {
       actual.add(qm.match(kv));
@@ -210,7 +212,8 @@ public void testMatch_ExpiredExplicit()
         new KeyValue(row2, fam1, col1, now-10, data)
     };
 
-    qm.setRow(kvs[0].getRow());
+    KeyValue k = kvs[0];
+    qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
 
     List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
     for (KeyValue kv : kvs) {
@@ -262,7 +265,8 @@ public void testMatch_ExpiredWildcard()
         new KeyValue(row1, fam2, col5, now-10000, data),
         new KeyValue(row2, fam1, col1, now-10, data)
     };
-    qm.setRow(kvs[0].getRow());
+    KeyValue k = kvs[0];
+    qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
 
     List<ScanQueryMatcher.MatchCode> actual =
         new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);

From b5f12b168e67eb1d6ef93fd83f70a7da1610a66a Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 6 Dec 2012 05:59:46 +0000
Subject: [PATCH 0612/1540] HBASE-7253 Backport Compaction Tool to 0.94; REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1417740 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   6 +-
 .../master/handler/CreateTableHandler.java    |   7 +-
 .../hbase/regionserver/CompactionTool.java    | 468 ------------------
 .../hadoop/hbase/regionserver/Compactor.java  | 205 --------
 .../hadoop/hbase/regionserver/HRegion.java    | 129 ++---
 .../hadoop/hbase/regionserver/Store.java      | 296 +++++++----
 .../compactions/CompactionProgress.java       |   1 +
 .../hadoop/hbase/util/ChecksumType.java       |   9 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |  21 +-
 .../hbase/regionserver/TestCompaction.java    |   4 +-
 10 files changed, 256 insertions(+), 890 deletions(-)
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 5f6f652eec14..e6987ff71853 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -645,7 +645,7 @@ public void logStats() {
     // Log size
     long totalSize = heapSize();
     long freeSize = maxSize - totalSize;
-    LruBlockCache.LOG.debug("Stats: " +
+    LruBlockCache.LOG.debug("LRU Stats: " +
         "total=" + StringUtils.byteDesc(totalSize) + ", " +
         "free=" + StringUtils.byteDesc(freeSize) + ", " +
         "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
@@ -653,11 +653,11 @@ public void logStats() {
         "accesses=" + stats.getRequestCount() + ", " +
         "hits=" + stats.getHitCount() + ", " +
         "hitRatio=" +
-          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
+          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) +
         "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
         "cachingHits=" + stats.getHitCachingCount() + ", " +
         "cachingHitsRatio=" +
-          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) + ", " +
+          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) +
         "evictions=" + stats.getEvictionCount() + ", " +
         "evicted=" + stats.getEvictedCount() + ", " +
         "evictedPerRun=" + stats.evictedPerEviction());
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index af25defc0d1f..95750606df78 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -142,12 +142,16 @@ private void handleCreateTable() throws IOException, KeeperException {
     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
     final int batchSize =
       this.conf.getInt("hbase.master.createtable.batchsize", 100);
+    HLog hlog = null;
     for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
       HRegionInfo newRegion = this.newRegions[regionIdx];
       // 1. Create HRegion
       HRegion region = HRegion.createHRegion(newRegion,
         this.fileSystemManager.getRootDir(), this.conf,
-        this.hTableDescriptor, null, false, true);
+        this.hTableDescriptor, hlog);
+      if (hlog == null) {
+        hlog = region.getLog();
+      }
 
       regionInfos.add(region.getRegionInfo());
       if (regionIdx % batchSize == 0) {
@@ -159,6 +163,7 @@ private void handleCreateTable() throws IOException, KeeperException {
       // 3. Close the new region to flush to disk.  Close log file too.
       region.close();
     }
+    hlog.closeAndDelete();
     if (regionInfos.size() > 0) {
       MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
deleted file mode 100644
index e123c22761fe..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ /dev/null
@@ -1,468 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.LineReader;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HDFSBlocksDistribution;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-/*
- * The CompactionTool allows to execute a compaction specifying a:
- * <ul>
- *  <li>table folder (all regions and families will be compacted)
- *  <li>region folder (all families in the region will be compacted)
- *  <li>family folder (the store files will be compacted)
- * </ul>
- */
-@InterfaceAudience.Public
-public class CompactionTool extends Configured implements Tool {
-  private static final Log LOG = LogFactory.getLog(CompactionTool.class);
-
-  private final static String CONF_TMP_DIR = "hbase.tmp.dir";
-  private final static String CONF_COMPACT_ONCE = "hbase.compactiontool.compact.once";
-  private final static String CONF_DELETE_COMPACTED = "hbase.compactiontool.delete";
-  private final static String CONF_COMPLETE_COMPACTION = "hbase.hstore.compaction.complete";
-
-  /**
-   * Class responsible to execute the Compaction on the specified path.
-   * The path can be a table, region or family directory.
-   */
-  private static class CompactionWorker {
-    private final boolean keepCompactedFiles;
-    private final boolean deleteCompacted;
-    private final Configuration conf;
-    private final FileSystem fs;
-    private final Path tmpDir;
-
-    public CompactionWorker(final FileSystem fs, final Configuration conf) {
-      this.conf = conf;
-      this.keepCompactedFiles = !conf.getBoolean(CONF_COMPLETE_COMPACTION, true);
-      this.deleteCompacted = conf.getBoolean(CONF_DELETE_COMPACTED, false);
-      this.tmpDir = new Path(conf.get(CONF_TMP_DIR));
-      this.fs = fs;
-    }
-
-    /**
-     * Execute the compaction on the specified path.
-     *
-     * @param path Directory path on which run a
-     * @param compactOnce Execute just a single step of compaction.
-     */
-    public void compact(final Path path, final boolean compactOnce) throws IOException {
-      if (isFamilyDir(fs, path)) {
-        Path regionDir = path.getParent();
-        Path tableDir = regionDir.getParent();
-        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
-        HRegion region = loadRegion(fs, conf, htd, regionDir);
-        compactStoreFiles(region, path, compactOnce);
-      } else if (isRegionDir(fs, path)) {
-        Path tableDir = path.getParent();
-        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
-        compactRegion(htd, path, compactOnce);
-      } else if (isTableDir(fs, path)) {
-        compactTable(path, compactOnce);
-      } else {
-        throw new IOException(
-          "Specified path is not a table, region or family directory. path=" + path);
-      }
-    }
-
-    private void compactTable(final Path tableDir, final boolean compactOnce)
-        throws IOException {
-      HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
-      LOG.info("Compact table=" + htd.getNameAsString());
-      for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
-        compactRegion(htd, regionDir, compactOnce);
-      }
-    }
-
-    private void compactRegion(final HTableDescriptor htd, final Path regionDir,
-        final boolean compactOnce) throws IOException {
-      HRegion region = loadRegion(fs, conf, htd, regionDir);
-      LOG.info("Compact table=" + htd.getNameAsString() +
-        " region=" + region.getRegionNameAsString());
-      for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
-        compactStoreFiles(region, familyDir, compactOnce);
-      }
-    }
-
-    /**
-     * Execute the actual compaction job.
-     * If the compact once flag is not specified, execute the compaction until
-     * no more compactions are needed. Uses the Configuration settings provided.
-     */
-    private void compactStoreFiles(final HRegion region, final Path familyDir,
-        final boolean compactOnce) throws IOException {
-      LOG.info("Compact table=" + region.getTableDesc().getNameAsString() +
-        " region=" + region.getRegionNameAsString() +
-        " family=" + familyDir.getName());
-      Store store = getStore(region, familyDir);
-      do {
-        CompactionRequest cr = store.requestCompaction();
-        StoreFile storeFile = store.compact(cr);
-        if (storeFile != null) {
-          if (keepCompactedFiles && deleteCompacted) {
-            fs.delete(storeFile.getPath(), false);
-          }
-        }
-      } while (store.needsCompaction() && !compactOnce);
-    }
-
-    /**
-     * Create a "mock" HStore that uses the tmpDir specified by the user and
-     * the store dir to compact as source.
-     */
-    private Store getStore(final HRegion region, final Path storeDir) throws IOException {
-      byte[] familyName = Bytes.toBytes(storeDir.getName());
-      HColumnDescriptor hcd = region.getTableDesc().getFamily(familyName);
-      // Create a Store w/ check of hbase.rootdir blanked out and return our
-      // list of files instead of have Store search its home dir.
-      return new Store(tmpDir, region, hcd, fs, conf) {
-        @Override
-        public FileStatus[] getStoreFiles() throws IOException {
-          return this.fs.listStatus(getHomedir());
-        }
-
-        @Override
-        Path createStoreHomeDir(FileSystem fs, Path homedir) throws IOException {
-          return storeDir;
-        }
-      };
-    }
-
-    private static HRegion loadRegion(final FileSystem fs, final Configuration conf,
-        final HTableDescriptor htd, final Path regionDir) throws IOException {
-      Path rootDir = regionDir.getParent().getParent();
-      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
-      return HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
-    }
-  }
-
-  private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
-    Path regionInfo = new Path(path, HRegion.REGIONINFO_FILE);
-    return fs.exists(regionInfo);
-  }
-
-  private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
-    return FSTableDescriptors.getTableInfoPath(fs, path) != null;
-  }
-
-  private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException {
-    return isRegionDir(fs, path.getParent());
-  }
-
-  private static class CompactionMapper
-      extends Mapper<LongWritable, Text, NullWritable, NullWritable> {
-    private CompactionWorker compactor = null;
-    private boolean compactOnce = false;
-
-    @Override
-    public void setup(Context context) {
-      Configuration conf = context.getConfiguration();
-      compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false);
-
-      try {
-        FileSystem fs = FileSystem.get(conf);
-        this.compactor = new CompactionWorker(fs, conf);
-      } catch (IOException e) {
-        throw new RuntimeException("Could not get the input FileSystem", e);
-      }
-    }
-
-    @Override
-    public void map(LongWritable key, Text value, Context context)
-        throws InterruptedException, IOException {
-      Path path = new Path(value.toString());
-      this.compactor.compact(path, compactOnce);
-    }
-  }
-
-  /**
-   * Input format that uses store files block location as input split locality.
-   */
-  private static class CompactionInputFormat extends TextInputFormat {
-    @Override
-    protected boolean isSplitable(JobContext context, Path file) {
-      return true;
-    }
-
-    /**
-     * Returns a split for each store files directory using the block location
-     * of each file as locality reference.
-     */
-    @Override
-    public List<InputSplit> getSplits(JobContext job) throws IOException {
-      List<InputSplit> splits = new ArrayList<InputSplit>();
-      List<FileStatus> files = listStatus(job);
-
-      Text key = new Text();
-      for (FileStatus file: files) {
-        Path path = file.getPath();
-        FileSystem fs = path.getFileSystem(job.getConfiguration());
-        LineReader reader = new LineReader(fs.open(path));
-        long pos = 0;
-        int n;
-        try {
-          while ((n = reader.readLine(key)) > 0) {
-            String[] hosts = getStoreDirHosts(fs, path);
-            splits.add(new FileSplit(path, pos, n, hosts));
-            pos += n;
-          }
-        } finally {
-          reader.close();
-        }
-      }
-
-      return splits;
-    }
-
-    /**
-     * return the top hosts of the store files, used by the Split
-     */
-    private static String[] getStoreDirHosts(final FileSystem fs, final Path path)
-        throws IOException {
-      FileStatus[] files = FSUtils.listStatus(fs, path, null);
-      if (files == null) {
-        return new String[] {};
-      }
-
-      HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
-      for (FileStatus hfileStatus: files) {
-        HDFSBlocksDistribution storeFileBlocksDistribution =
-          FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen());
-        hdfsBlocksDistribution.add(storeFileBlocksDistribution);
-      }
-
-      List<String> hosts = hdfsBlocksDistribution.getTopHosts();
-      return hosts.toArray(new String[hosts.size()]);
-    }
-
-    /**
-     * Create the input file for the given directories to compact.
-     * The file is a TextFile with each line corrisponding to a
-     * store files directory to compact.
-     */
-    public static void createInputFile(final FileSystem fs, final Path path,
-        final Set<Path> toCompactDirs) throws IOException {
-      // Extract the list of store dirs
-      List<Path> storeDirs = new LinkedList<Path>();
-      for (Path compactDir: toCompactDirs) {
-        if (isFamilyDir(fs, compactDir)) {
-          storeDirs.add(compactDir);
-        } else if (isRegionDir(fs, compactDir)) {
-          for (Path familyDir: FSUtils.getFamilyDirs(fs, compactDir)) {
-            storeDirs.add(familyDir);
-          }
-        } else if (isTableDir(fs, compactDir)) {
-          // Lookup regions
-          for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) {
-            for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
-              storeDirs.add(familyDir);
-            }
-          }
-        } else {
-          throw new IOException(
-            "Specified path is not a table, region or family directory. path=" + compactDir);
-        }
-      }
-
-      // Write Input File
-      FSDataOutputStream stream = fs.create(path);
-      LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact.");
-      try {
-        final byte[] newLine = Bytes.toBytes("\n");
-        for (Path storeDir: storeDirs) {
-          stream.write(Bytes.toBytes(storeDir.toString()));
-          stream.write(newLine);
-        }
-      } finally {
-        stream.close();
-      }
-    }
-  }
-
-  /**
-   * Execute compaction, using a Map-Reduce job.
-   */
-  private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
-      final boolean compactOnce) throws Exception {
-    Configuration conf = getConf();
-    conf.setBoolean(CONF_COMPACT_ONCE, compactOnce);
-
-    Job job = new Job(conf);
-    job.setJobName("CompactionTool");
-    job.setJarByClass(CompactionTool.class);
-    job.setMapperClass(CompactionMapper.class);
-    job.setInputFormatClass(CompactionInputFormat.class);
-    job.setOutputFormatClass(NullOutputFormat.class);
-    job.setMapSpeculativeExecution(false);
-    job.setNumReduceTasks(0);
-
-    String stagingName = "compact-" + EnvironmentEdgeManager.currentTimeMillis();
-    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
-    fs.mkdirs(stagingDir);
-    try {
-      // Create input file with the store dirs
-      Path inputPath = new Path(stagingDir, stagingName);
-      CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs);
-      CompactionInputFormat.addInputPath(job, inputPath);
-
-      // Initialize credential for secure cluster
-      TableMapReduceUtil.initCredentials(job);
-
-      // Start the MR Job and wait
-      return job.waitForCompletion(true) ? 0 : 1;
-    } finally {
-      fs.delete(stagingDir, true);
-    }
-  }
-
-  /**
-   * Execute compaction, from this client, one path at the time.
-   */
-  private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
-      final boolean compactOnce) throws IOException {
-    CompactionWorker worker = new CompactionWorker(fs, getConf());
-    for (Path path: toCompactDirs) {
-      worker.compact(path, compactOnce);
-    }
-    return 0;
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    Set<Path> toCompactDirs = new HashSet<Path>();
-    boolean compactOnce = false;
-    boolean mapred = false;
-
-    Configuration conf = getConf();
-    FileSystem fs = FileSystem.get(conf);
-
-    try {
-      for (int i = 0; i < args.length; ++i) {
-        String opt = args[i];
-        if (opt.equals("-compactOnce")) {
-          compactOnce = true;
-        } else if (opt.equals("-mapred")) {
-          mapred = true;
-        } else if (!opt.startsWith("-")) {
-          Path path = new Path(opt);
-          FileStatus status = fs.getFileStatus(path);
-          if (!status.isDir()) {
-            printUsage("Specified path is not a directory. path=" + path);
-            return 1;
-          }
-          toCompactDirs.add(path);
-        } else {
-          printUsage();
-        }
-      }
-    } catch (Exception e) {
-      printUsage(e.getMessage());
-      return 1;
-    }
-
-    if (toCompactDirs.size() == 0) {
-      printUsage("No directories to compact specified.");
-      return 1;
-    }
-
-    // Execute compaction!
-    if (mapred) {
-      return doMapReduce(fs, toCompactDirs, compactOnce);
-    } else {
-      return doClient(fs, toCompactDirs, compactOnce);
-    }
-  }
-
-  private void printUsage() {
-    printUsage(null);
-  }
-
-  private void printUsage(final String message) {
-    if (message != null && message.length() > 0) {
-      System.err.println(message);
-    }
-    System.err.println("Usage: java " + this.getClass().getName() + " \\");
-    System.err.println("  [-compactOnce] [-mapred] [-D<property=value>]* files...");
-    System.err.println();
-    System.err.println("Options:");
-    System.err.println(" mapred         Use MapReduce to run compaction.");
-    System.err.println(" compactOnce    Execute just one compaction step. (default: while needed)");
-    System.err.println();
-    System.err.println("Note: -D properties will be applied to the conf used. ");
-    System.err.println("For example: ");
-    System.err.println(" To preserve input files, pass -D"+CONF_COMPLETE_COMPACTION+"=false");
-    System.err.println(" To stop delete of compacted file, pass -D"+CONF_DELETE_COMPACTED+"=false");
-    System.err.println(" To set tmp dir, pass -D"+CONF_TMP_DIR+"=ALTERNATE_DIR");
-    System.err.println();
-    System.err.println("Examples:");
-    System.err.println(" To compact the full 'TestTable' using MapReduce:");
-    System.err.println(" $ bin/hbase " + this.getClass().getName() + " -mapred hdfs:///hbase/TestTable");
-    System.err.println();
-    System.err.println(" To compact column family 'x' of the table 'TestTable' region 'abc':");
-    System.err.println(" $ bin/hbase " + this.getClass().getName() + " hdfs:///hbase/TestTable/abc/x");
-  }
-
-  public static void main(String[] args) throws Exception {
-    System.exit(ToolRunner.run(HBaseConfiguration.create(), new CompactionTool(), args));
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
deleted file mode 100644
index bba7a9a5306e..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Compact passed set of files.
- * Create an instance and then call {@ink #compact(Store, Collection, boolean, long)}.
- */
-@InterfaceAudience.Private
-class Compactor extends Configured {
-  private static final Log LOG = LogFactory.getLog(Compactor.class);
-  private CompactionProgress progress;
-
-  Compactor(final Configuration c) {
-    super(c);
-  }
-
-  /**
-   * Do a minor/major compaction on an explicit set of storefiles from a Store.
-   *
-   * @param store Store the files belong to
-   * @param filesToCompact which files to compact
-   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
-   * @param maxId Readers maximum sequence id.
-   * @return Product of compaction or null if all cells expired or deleted and
-   * nothing made it through the compaction.
-   * @throws IOException
-   */
-  StoreFile.Writer compact(final Store store,
-      final Collection<StoreFile> filesToCompact,
-      final boolean majorCompaction, final long maxId)
-  throws IOException {
-    // Calculate maximum key count after compaction (for blooms)
-    // Also calculate earliest put timestamp if major compaction
-    int maxKeyCount = 0;
-    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
-    for (StoreFile file: filesToCompact) {
-      StoreFile.Reader r = file.getReader();
-      if (r == null) {
-        LOG.warn("Null reader for " + file.getPath());
-        continue;
-      }
-      // NOTE: getFilterEntries could cause under-sized blooms if the user
-      // switches bloom type (e.g. from ROW to ROWCOL)
-      long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType())?
-        r.getFilterEntries() : r.getEntries();
-      maxKeyCount += keyCount;
-      // For major compactions calculate the earliest put timestamp of all
-      // involved storefiles. This is used to remove family delete marker during
-      // compaction.
-      if (majorCompaction) {
-        byte [] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
-        if (tmp == null) {
-          // There's a file with no information, must be an old one
-          // assume we have very old puts
-          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
-        } else {
-          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
-        }
-      }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Compacting " + file +
-          ", keycount=" + keyCount +
-          ", bloomtype=" + r.getBloomFilterType().toString() +
-          ", size=" + StringUtils.humanReadableInt(r.length()) +
-          ", encoding=" + r.getHFileReader().getEncodingOnDisk() +
-          (majorCompaction? ", earliestPutTs=" + earliestPutTs: ""));
-      }
-    }
-
-    // keep track of compaction progress
-    this.progress = new CompactionProgress(maxKeyCount);
-
-    // For each file, obtain a scanner:
-    List<StoreFileScanner> scanners = StoreFileScanner
-      .getScannersForStoreFiles(filesToCompact, false, false, true);
-
-    // Get some configs
-    int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
-    Compression.Algorithm compression = store.getFamily().getCompression();
-    // Avoid overriding compression setting for major compactions if the user
-    // has not specified it separately
-    Compression.Algorithm compactionCompression =
-      (store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE) ?
-      store.getFamily().getCompactionCompression(): compression;
-    // Make the instantiation lazy in case compaction produces no product; i.e.
-    // where all source cells are expired or deleted.
-    StoreFile.Writer writer = null;
-    // Find the smallest read point across all the Scanners.
-    long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
-    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
-    try {
-      InternalScanner scanner = null;
-      try {
-        Scan scan = new Scan();
-        scan.setMaxVersions(store.getFamily().getMaxVersions());
-        /* Include deletes, unless we are doing a major compaction */
-        scanner = new StoreScanner(store, store.getScanInfo(), scan, scanners,
-          majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
-          smallestReadPoint, earliestPutTs);
-        if (store.getHRegion().getCoprocessorHost() != null) {
-          InternalScanner cpScanner =
-            store.getHRegion().getCoprocessorHost().preCompact(store, scanner);
-          // NULL scanner returned from coprocessor hooks means skip normal processing
-          if (cpScanner == null) {
-            return null;
-          }
-          scanner = cpScanner;
-        }
-
-        int bytesWritten = 0;
-        // Since scanner.next() can return 'false' but still be delivering data,
-        // we have to use a do/while loop.
-        List<KeyValue> kvs = new ArrayList<KeyValue>();
-        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
-        boolean hasMore;
-        do {
-          hasMore = scanner.next(kvs, compactionKVMax);
-          if (writer == null && !kvs.isEmpty()) {
-            writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true);
-          }
-          if (writer != null) {
-            // output to writer:
-            for (KeyValue kv : kvs) {
-              if (kv.getMemstoreTS() <= smallestReadPoint) {
-                kv.setMemstoreTS(0);
-              }
-              writer.append(kv);
-              // update progress per key
-              ++progress.currentCompactedKVs;
-
-              // check periodically to see if a system stop is requested
-              if (Store.closeCheckInterval > 0) {
-                bytesWritten += kv.getLength();
-                if (bytesWritten > Store.closeCheckInterval) {
-                  bytesWritten = 0;
-                  isInterrupted(store, writer);
-                }
-              }
-            }
-          }
-          kvs.clear();
-        } while (hasMore);
-      } finally {
-        if (scanner != null) {
-          scanner.close();
-        }
-      }
-    } finally {
-      if (writer != null) {
-        writer.appendMetadata(maxId, majorCompaction);
-        writer.close();
-      }
-    }
-    return writer;
-  }
-
-  void isInterrupted(final Store store, final StoreFile.Writer writer)
-  throws IOException {
-    if (store.getHRegion().areWritesEnabled()) return;
-    // Else cleanup.
-    writer.close();
-    store.getFileSystem().delete(writer.getPath(), false);
-    throw new InterruptedIOException( "Aborting compaction of store " + store +
-      " in region " + store.getHRegion() + " because user requested stop.");
-  }
-
-  CompactionProgress getProgress() {
-    return this.progress;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 87edae3ec72a..f98d32d298d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.EOFException;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.UnsupportedEncodingException;
@@ -63,7 +62,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -232,12 +230,12 @@ public class HRegion implements HeapSize { // , Writable{
    * The directory for the table this region is part of.
    * This directory contains the directory for this region.
    */
-  private final Path tableDir;
+  final Path tableDir;
 
-  private final HLog log;
-  private final FileSystem fs;
-  private final Configuration conf;
-  private final int rowLockWaitDuration;
+  final HLog log;
+  final FileSystem fs;
+  final Configuration conf;
+  final int rowLockWaitDuration;
   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
 
   // The internal wait duration to acquire a lock before read/update
@@ -258,8 +256,8 @@ public class HRegion implements HeapSize { // , Writable{
   // purge timeout, when a RPC call will be terminated by the RPC engine.
   final long maxBusyWaitDuration;
 
-  private final HRegionInfo regionInfo;
-  private final Path regiondir;
+  final HRegionInfo regionInfo;
+  final Path regiondir;
   KeyValue.KVComparator comparator;
 
   private ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;
@@ -726,7 +724,7 @@ public AtomicLong getMemstoreSize() {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     if (this.rsAccounting != null) {
       rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-    }
+    }  
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -752,7 +750,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
 
     // and then create the file
     Path tmpPath = new Path(getTmpDir(), REGIONINFO_FILE);
-
+    
     // if datanode crashes or if the RS goes down just before the close is called while trying to
     // close the created regioninfo file in the .tmp directory then on next
     // creation we will be getting AlreadyCreatedException.
@@ -760,7 +758,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     if (FSUtils.isExists(fs, tmpPath)) {
       FSUtils.delete(fs, tmpPath, true);
     }
-
+    
     FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
 
     try {
@@ -777,26 +775,6 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     }
   }
 
-  /**
-   * @param fs
-   * @param dir
-   * @return An HRegionInfo instance gotten from the <code>.regioninfo</code> file under region dir
-   * @throws IOException
-   */
-  public static HRegionInfo loadDotRegionInfoFileContent(final FileSystem fs, final Path dir)
-  throws IOException {
-    Path regioninfo = new Path(dir, HRegion.REGIONINFO_FILE);
-    if (!fs.exists(regioninfo)) throw new FileNotFoundException(regioninfo.toString());
-    FSDataInputStream in = fs.open(regioninfo);
-    try {
-      HRegionInfo hri = new HRegionInfo();
-      hri.readFields(in);
-      return hri;
-    } finally {
-      in.close();
-    }
-  }
-
   /** @return a HRegionInfo object for this region */
   public HRegionInfo getRegionInfo() {
     return this.regionInfo;
@@ -1043,16 +1021,19 @@ protected ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(
     return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
   }
 
-  static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
+  private ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
       final String threadNamePrefix) {
-    return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
-      new ThreadFactory() {
-        private int count = 1;
-
-        public Thread newThread(Runnable r) {
-          return new Thread(r, threadNamePrefix + "-" + count++);
-        }
-      });
+    ThreadPoolExecutor openAndCloseThreadPool = Threads
+        .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+            new ThreadFactory() {
+              private int count = 1;
+
+              public Thread newThread(Runnable r) {
+                Thread t = new Thread(r, threadNamePrefix + "-" + count++);
+                return t;
+              }
+            });
+    return openAndCloseThreadPool;
   }
 
    /**
@@ -1998,7 +1979,7 @@ public OperationStatus[] put(Pair<Put, Integer>[] putsAndLocks) throws IOExcepti
     System.arraycopy(putsAndLocks, 0, mutationsAndLocks, 0, putsAndLocks.length);
     return batchMutate(mutationsAndLocks);
   }
-
+  
   /**
    * Perform a batch of mutations.
    * It supports only Put and Delete mutations and will ignore other types passed.
@@ -2352,7 +2333,7 @@ private long doMiniBatchMutation(
 
       // do after lock
       final long netTimeMs = EnvironmentEdgeManager.currentTimeMillis()- startTimeMs;
-
+            
       // See if the column families were consistent through the whole thing.
       // if they were then keep them. If they were not then pass a null.
       // null will be treated as unknown.
@@ -2655,7 +2636,7 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updatePutMetrics(familyMap.keySet(), after - now);
-
+    
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -3778,7 +3759,6 @@ public static void closeHRegion(final HRegion r) throws IOException {
    * @param conf
    * @param hTableDescriptor
    * @param hlog shared HLog
-   * @param boolean initialize - true to initialize the region
    * @return new HRegion
    *
    * @throws IOException
@@ -3786,36 +3766,7 @@ public static void closeHRegion(final HRegion r) throws IOException {
   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                       final Configuration conf,
                                       final HTableDescriptor hTableDescriptor,
-                                      final HLog hlog,
-                                      final boolean initialize)
-      throws IOException {
-    return createHRegion(info, rootDir, conf, hTableDescriptor,
-        hlog, initialize, false);
-  }
-
-  /**
-   * Convenience method creating new HRegions. Used by createTable.
-   * The {@link HLog} for the created region needs to be closed
-   * explicitly, if it is not null.
-   * Use {@link HRegion#getLog()} to get access.
-   *
-   * @param info Info for region to create.
-   * @param rootDir Root directory for HBase instance
-   * @param conf
-   * @param hTableDescriptor
-   * @param hlog shared HLog
-   * @param boolean initialize - true to initialize the region
-   * @param boolean ignoreHLog
-      - true to skip generate new hlog if it is null, mostly for createTable
-   * @return new HRegion
-   *
-   * @throws IOException
-   */
-  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
-                                      final Configuration conf,
-                                      final HTableDescriptor hTableDescriptor,
-                                      final HLog hlog,
-                                      final boolean initialize, final boolean ignoreHLog)
+                                      final HLog hlog)
       throws IOException {
     LOG.info("creating HRegion " + info.getTableNameAsString()
         + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
@@ -3827,26 +3778,16 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
     FileSystem fs = FileSystem.get(conf);
     fs.mkdirs(regionDir);
     HLog effectiveHLog = hlog;
-    if (hlog == null && !ignoreHLog) {
+    if (hlog == null) {
       effectiveHLog = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
           new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf);
     }
     HRegion region = HRegion.newHRegion(tableDir,
         effectiveHLog, fs, conf, info, hTableDescriptor, null);
-    if (initialize) {
-      region.initialize();
-    }
+    region.initialize();
     return region;
   }
 
-  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
-                                      final Configuration conf,
-                                      final HTableDescriptor hTableDescriptor,
-                                      final HLog hlog)
-    throws IOException {
-    return createHRegion(info, rootDir, conf, hTableDescriptor, hlog, true);
-  }
-
   /**
    * Open a Region.
    * @param info Info for region to be opened.
@@ -4358,7 +4299,7 @@ private List<KeyValue> get(Get get, boolean withCoprocessor)
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updateGetMetrics(get.familySet(), after - now);
-
+    
     return results;
   }
 
@@ -4686,10 +4627,10 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
       closeRegionOperation();
     }
 
-
+    
     long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - before);
-
+    
     if (flush) {
       // Request a cache flush. Do it outside update lock.
       requestFlush();
@@ -4814,7 +4755,7 @@ public Result increment(Increment increment, Integer lockid,
       long after = EnvironmentEdgeManager.currentTimeMillis();
       this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
     }
-
+    
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -5307,7 +5248,7 @@ private void closeBulkRegionOperation() {
    */
   private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
     if (numPutsWithoutWAL.getAndIncrement() == 0) {
-      LOG.info("writing data to region " + this +
+      LOG.info("writing data to region " + this + 
                " with WAL disabled. Data may be lost in the event of a crash.");
     }
 
@@ -5419,11 +5360,11 @@ public static void main(String[] args) throws IOException {
     final HLog log = new HLog(fs, logdir, oldLogDir, c);
     try {
       processTable(fs, tableDir, log, c, majorCompact);
-    } finally {
+     } finally {
        log.close();
        // TODO: is this still right?
        BlockCache bc = new CacheConfig(c).getBlockCache();
        if (bc != null) bc.shutdown();
-    }
+     }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 9a02c16c77c8..f9e1103b0c10 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -131,6 +132,9 @@ public class Store extends SchemaConfigured implements HeapSize {
   private volatile long totalUncompressedBytes = 0L;
   private final Object flushLock = new Object();
   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+  private final String storeNameStr;
+  private CompactionProgress progress;
+  private final int compactionKVMax;
   private final boolean verifyBulkLoads;
 
   /* The default priority for user-specified compaction requests.
@@ -154,6 +158,10 @@ public class Store extends SchemaConfigured implements HeapSize {
     new CopyOnWriteArraySet<ChangedReadersObserver>();
 
   private final int blocksize;
+  /** Compression algorithm for flush files and minor compaction */
+  private final Compression.Algorithm compression;
+  /** Compression algorithm for major compaction */
+  private final Compression.Algorithm compactionCompression;
   private HFileDataBlockEncoder dataBlockEncoder;
 
   /** Checksum configuration */
@@ -163,8 +171,6 @@ public class Store extends SchemaConfigured implements HeapSize {
   // Comparing KeyValues
   final KeyValue.KVComparator comparator;
 
-  private final Compactor compactor;
-
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -179,16 +185,25 @@ public class Store extends SchemaConfigured implements HeapSize {
   protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     FileSystem fs, Configuration conf)
   throws IOException {
-    super(conf, region.getRegionInfo().getTableNameAsString(),
+    super(conf, region.getTableDesc().getNameAsString(),
         Bytes.toString(family.getName()));
-    HRegionInfo info = region.getRegionInfo();
+    HRegionInfo info = region.regionInfo;
     this.fs = fs;
-    Path p = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
-    this.homedir = createStoreHomeDir(this.fs, p);
+    this.homedir = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
+    if (!this.fs.exists(this.homedir)) {
+      if (!this.fs.mkdirs(this.homedir))
+        throw new IOException("Failed create of: " + this.homedir.toString());
+    }
     this.region = region;
     this.family = family;
     this.conf = conf;
     this.blocksize = family.getBlocksize();
+    this.compression = family.getCompression();
+    // avoid overriding compression setting for major compactions if the user
+    // has not specified it separately
+    this.compactionCompression =
+      (family.getCompactionCompression() != Compression.Algorithm.NONE) ?
+        family.getCompactionCompression() : this.compression;
 
     this.dataBlockEncoder =
         new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(),
@@ -213,6 +228,7 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
         "ms in store " + this);
     scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
     this.memstore = new MemStore(conf, this.comparator);
+    this.storeNameStr = getColumnFamilyName();
 
     // By default, compact if storefile.count >= minFilesToCompact
     this.minFilesToCompact = Math.max(2,
@@ -229,8 +245,10 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       this.region.memstoreFlushSize);
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
+    this.compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
 
-    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
+    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify",
+        false);
 
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
@@ -242,47 +260,6 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.checksumType = getChecksumType(conf);
     // initilize bytes per checksum
     this.bytesPerChecksum = getBytesPerChecksum(conf);
-    // Create a compaction tool instance
-    this.compactor = new Compactor(this.conf);
-  }
-
-  /**
-   * @param family
-   * @return
-   */
-  long getTTL(final HColumnDescriptor family) {
-    // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
-    long ttl = family.getTimeToLive();
-    if (ttl == HConstants.FOREVER) {
-      // Default is unlimited ttl.
-      ttl = Long.MAX_VALUE;
-    } else if (ttl == -1) {
-      ttl = Long.MAX_VALUE;
-    } else {
-      // Second -> ms adjust for user data
-      ttl *= 1000;
-    }
-    return ttl;
-  }
-
-  /**
-   * Create this store's homedir
-   * @param fs
-   * @param homedir
-   * @return Return <code>homedir</code>
-   * @throws IOException
-   */
-  Path createStoreHomeDir(final FileSystem fs,
-      final Path homedir) throws IOException {
-    if (!fs.exists(homedir)) {
-      if (!fs.mkdirs(homedir))
-        throw new IOException("Failed create of: " + homedir.toString());
-    }
-    return homedir;
-  }
-
-  FileSystem getFileSystem() {
-    return this.fs;
   }
 
   /**
@@ -343,7 +320,7 @@ public static Path getStoreHomedir(final Path tabledir,
    * Return the directory in which this store stores its
    * StoreFiles
    */
-  Path getHomedir() {
+  public Path getHomedir() {
     return homedir;
   }
 
@@ -362,10 +339,6 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
     this.dataBlockEncoder = blockEncoder;
   }
 
-  FileStatus [] getStoreFiles() throws IOException {
-    return FSUtils.listStatus(this.fs, this.homedir, null);
-  }
-
   /**
    * Creates an unsorted list of StoreFile loaded in parallel
    * from the given directory.
@@ -373,7 +346,7 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
    */
   private List<StoreFile> loadStoreFiles() throws IOException {
     ArrayList<StoreFile> results = new ArrayList<StoreFile>();
-    FileStatus files[] = getStoreFiles();
+    FileStatus files[] = FSUtils.listStatus(this.fs, this.homedir, null);
 
     if (files == null || files.length == 0) {
       return results;
@@ -664,7 +637,7 @@ public Void call() throws IOException {
           storeFileCloserThreadPool.shutdownNow();
         }
       }
-      LOG.info("Closed " + this);
+      LOG.debug("closed " + this.storeNameStr);
       return result;
     } finally {
       this.lock.writeLock().unlock();
@@ -750,7 +723,6 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
       scanner = cpScanner;
     }
     try {
-      int compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
       // TODO:  We can fail in the below block before we complete adding this
       // flush to list of store files.  Add cleanup of anything put on filesystem
       // if we fail.
@@ -764,7 +736,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
           List<KeyValue> kvs = new ArrayList<KeyValue>();
           boolean hasMore;
           do {
-            hasMore = scanner.next(kvs, compactionKVMax);
+            hasMore = scanner.next(kvs, this.compactionKVMax);
             if (!kvs.isEmpty()) {
               for (KeyValue kv : kvs) {
                 // If we know that this KV is going to be included always, then let us
@@ -856,7 +828,7 @@ private StoreFile commitFile(final Path path,
    */
   private StoreFile.Writer createWriterInTmp(int maxKeyCount)
   throws IOException {
-    return createWriterInTmp(maxKeyCount, this.family.getCompression(), false);
+    return createWriterInTmp(maxKeyCount, this.compression, false);
   }
 
   /*
@@ -1009,12 +981,16 @@ void deleteChangedReaderObserver(ChangedReadersObserver o) {
    * @param cr
    *          compaction details obtained from requestCompaction()
    * @throws IOException
-   * @return Storefile we compacted into or null if we failed or opted out early.
    */
-  StoreFile compact(CompactionRequest cr) throws IOException {
-    if (cr == null || cr.getFiles().isEmpty()) return null;
-    Preconditions.checkArgument(cr.getStore().toString().equals(this.toString()));
+  void compact(CompactionRequest cr) throws IOException {
+    if (cr == null || cr.getFiles().isEmpty()) {
+      return;
+    }
+    Preconditions.checkArgument(cr.getStore().toString()
+        .equals(this.toString()));
+
     List<StoreFile> filesToCompact = cr.getFiles();
+
     synchronized (filesCompacting) {
       // sanity check: we're compacting files that this store knows about
       // TODO: change this to LOG.error() after more debugging
@@ -1026,26 +1002,19 @@ StoreFile compact(CompactionRequest cr) throws IOException {
 
     // Ready to go. Have list of files to compact.
     LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
-        + this + " of "
+        + this.storeNameStr + " of "
         + this.region.getRegionInfo().getRegionNameAsString()
         + " into tmpdir=" + region.getTmpDir() + ", seqid=" + maxId + ", totalSize="
         + StringUtils.humanReadableInt(cr.getSize()));
 
     StoreFile sf = null;
     try {
-      StoreFile.Writer writer =
-        this.compactor.compact(this, filesToCompact, cr.isMajor(), maxId);
+      StoreFile.Writer writer = compactStore(filesToCompact, cr.isMajor(),
+          maxId);
       // Move the compaction into place.
-      if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
-        sf = completeCompaction(filesToCompact, writer);
-        if (region.getCoprocessorHost() != null) {
-          region.getCoprocessorHost().postCompact(this, sf);
-        }
-      } else {
-        // Create storefile around what we wrote with a reader on it.
-        sf = new StoreFile(this.fs, writer.getPath(), this.conf, this.cacheConf,
-          this.family.getBloomFilterType(), this.dataBlockEncoder);
-        sf.createReader();
+      sf = completeCompaction(filesToCompact, writer);
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().postCompact(this, sf);
       }
     } finally {
       synchronized (filesCompacting) {
@@ -1054,7 +1023,7 @@ StoreFile compact(CompactionRequest cr) throws IOException {
     }
 
     LOG.info("Completed" + (cr.isMajor() ? " major " : " ") + "compaction of "
-        + filesToCompact.size() + " file(s) in " + this + " of "
+        + filesToCompact.size() + " file(s) in " + this.storeNameStr + " of "
         + this.region.getRegionInfo().getRegionNameAsString()
         + " into " +
         (sf == null ? "none" : sf.getPath().getName()) +
@@ -1062,7 +1031,6 @@ StoreFile compact(CompactionRequest cr) throws IOException {
           StringUtils.humanReadableInt(sf.getReader().length()))
         + "; total size for store is "
         + StringUtils.humanReadableInt(storeSize));
-    return sf;
   }
 
   /**
@@ -1102,8 +1070,7 @@ public void compactRecentForTesting(int N) throws IOException {
 
     try {
       // Ready to go. Have list of files to compact.
-      StoreFile.Writer writer =
-        this.compactor.compact(this, filesToCompact, isMajor, maxId);
+      StoreFile.Writer writer = compactStore(filesToCompact, isMajor, maxId);
       // Move the compaction into place.
       StoreFile sf = completeCompaction(filesToCompact, writer);
       if (region.getCoprocessorHost() != null) {
@@ -1152,10 +1119,10 @@ public static long getLowestTimestamp(final List<StoreFile> candidates)
   }
 
   /** getter for CompactionProgress object
-   * @return CompactionProgress object; can be null
+   * @return CompactionProgress object
    */
   public CompactionProgress getCompactionProgress() {
-    return this.compactor.getProgress();
+    return this.progress;
   }
 
   /*
@@ -1207,19 +1174,19 @@ private boolean isMajorCompaction(final List<StoreFile> filesToCompact) throws I
         if (sf.isMajorCompaction() &&
             (this.ttl == HConstants.FOREVER || oldest < this.ttl)) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Skipping major compaction of " + this +
+            LOG.debug("Skipping major compaction of " + this.storeNameStr +
                 " because one (major) compacted file only and oldestTime " +
                 oldest + "ms is < ttl=" + this.ttl);
           }
         } else if (this.ttl != HConstants.FOREVER && oldest > this.ttl) {
-          LOG.debug("Major compaction triggered on store " + this +
+          LOG.debug("Major compaction triggered on store " + this.storeNameStr +
             ", because keyvalues outdated; time since last major compaction " +
             (now - lowTimestamp) + "ms");
           result = true;
         }
       } else {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Major compaction triggered on store " + this +
+          LOG.debug("Major compaction triggered on store " + this.storeNameStr +
               "; time since last major compaction " + (now - lowTimestamp) + "ms");
         }
         result = true;
@@ -1409,12 +1376,12 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
              compactSelection.getFilesToCompact().get(pos).getReader().length()
                > maxCompactSize &&
              !compactSelection.getFilesToCompact().get(pos).isReference()) ++pos;
-      if (pos != 0) compactSelection.clearSubList(0, pos);
+      compactSelection.clearSubList(0, pos);
     }
 
     if (compactSelection.getFilesToCompact().isEmpty()) {
       LOG.debug(this.getHRegionInfo().getEncodedName() + " - " +
-        this + ": no store files to compact");
+        this.storeNameStr + ": no store files to compact");
       compactSelection.emptyFileList();
       return compactSelection;
     }
@@ -1501,7 +1468,7 @@ public boolean apply(StoreFile input) {
       // if we don't have enough files to compact, just wait
       if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Skipped compaction of " + this
+          LOG.debug("Skipped compaction of " + this.storeNameStr
             + ".  Only " + (end - start) + " file(s) of size "
             + StringUtils.humanReadableInt(totalSize)
             + " have met compaction criteria.");
@@ -1527,6 +1494,149 @@ public boolean apply(StoreFile input) {
     return compactSelection;
   }
 
+  /**
+   * Do a minor/major compaction on an explicit set of storefiles in a Store.
+   * Uses the scan infrastructure to make it easy.
+   *
+   * @param filesToCompact which files to compact
+   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
+   * @param maxId Readers maximum sequence id.
+   * @return Product of compaction or null if all cells expired or deleted and
+   * nothing made it through the compaction.
+   * @throws IOException
+   */
+  StoreFile.Writer compactStore(final Collection<StoreFile> filesToCompact,
+                               final boolean majorCompaction, final long maxId)
+      throws IOException {
+    // calculate maximum key count after compaction (for blooms)
+    int maxKeyCount = 0;
+    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+    for (StoreFile file : filesToCompact) {
+      StoreFile.Reader r = file.getReader();
+      if (r != null) {
+        // NOTE: getFilterEntries could cause under-sized blooms if the user
+        //       switches bloom type (e.g. from ROW to ROWCOL)
+        long keyCount = (r.getBloomFilterType() == family.getBloomFilterType())
+          ? r.getFilterEntries() : r.getEntries();
+        maxKeyCount += keyCount;
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Compacting " + file +
+            ", keycount=" + keyCount +
+            ", bloomtype=" + r.getBloomFilterType().toString() +
+            ", size=" + StringUtils.humanReadableInt(r.length()) +
+            ", encoding=" + r.getHFileReader().getEncodingOnDisk());
+        }
+      }
+      // For major compactions calculate the earliest put timestamp
+      // of all involved storefiles. This is used to remove 
+      // family delete marker during the compaction.
+      if (majorCompaction) {
+        byte[] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
+        if (tmp == null) {
+          // there's a file with no information, must be an old one
+          // assume we have very old puts
+          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
+        } else {
+          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
+        }
+      }
+    }
+
+    // keep track of compaction progress
+    progress = new CompactionProgress(maxKeyCount);
+
+    // For each file, obtain a scanner:
+    List<StoreFileScanner> scanners = StoreFileScanner
+      .getScannersForStoreFiles(filesToCompact, false, false, true);
+
+    // Make the instantiation lazy in case compaction produces no product; i.e.
+    // where all source cells are expired or deleted.
+    StoreFile.Writer writer = null;
+    // Find the smallest read point across all the Scanners.
+    long smallestReadPoint = region.getSmallestReadPoint();
+    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
+    try {
+      InternalScanner scanner = null;
+      try {
+        if (getHRegion().getCoprocessorHost() != null) {
+          scanner = getHRegion()
+              .getCoprocessorHost()
+              .preCompactScannerOpen(this, scanners,
+                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
+        }
+        if (scanner == null) {
+          Scan scan = new Scan();
+          scan.setMaxVersions(getFamily().getMaxVersions());
+          /* Include deletes, unless we are doing a major compaction */
+          scanner = new StoreScanner(this, getScanInfo(), scan, scanners,
+            majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
+            smallestReadPoint, earliestPutTs);
+        }
+        if (getHRegion().getCoprocessorHost() != null) {
+          InternalScanner cpScanner =
+            getHRegion().getCoprocessorHost().preCompact(this, scanner);
+          // NULL scanner returned from coprocessor hooks means skip normal processing
+          if (cpScanner == null) {
+            return null;
+          }
+          scanner = cpScanner;
+        }
+
+        int bytesWritten = 0;
+        // since scanner.next() can return 'false' but still be delivering data,
+        // we have to use a do/while loop.
+        ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        boolean hasMore;
+        do {
+          hasMore = scanner.next(kvs, this.compactionKVMax);
+          if (writer == null && !kvs.isEmpty()) {
+            writer = createWriterInTmp(maxKeyCount, this.compactionCompression,
+                true);
+          }
+          if (writer != null) {
+            // output to writer:
+            for (KeyValue kv : kvs) {
+              if (kv.getMemstoreTS() <= smallestReadPoint) {
+                kv.setMemstoreTS(0);
+              }
+              writer.append(kv);
+              // update progress per key
+              ++progress.currentCompactedKVs;
+
+              // check periodically to see if a system stop is requested
+              if (Store.closeCheckInterval > 0) {
+                bytesWritten += kv.getLength();
+                if (bytesWritten > Store.closeCheckInterval) {
+                  bytesWritten = 0;
+                  if (!this.region.areWritesEnabled()) {
+                    writer.close();
+                    fs.delete(writer.getPath(), false);
+                    throw new InterruptedIOException(
+                        "Aborting compaction of store " + this +
+                        " in region " + this.region +
+                        " because user requested stop.");
+                  }
+                }
+              }
+            }
+          }
+          kvs.clear();
+        } while (hasMore);
+      } finally {
+        if (scanner != null) {
+          scanner.close();
+        }
+      }
+    } finally {
+      if (writer != null) {
+        writer.appendMetadata(maxId, majorCompaction);
+        writer.close();
+      }
+    }
+    return writer;
+  }
+
   /**
    * Validates a store file by opening and closing it. In HFileV2 this should
    * not be an expensive operation.
@@ -1631,7 +1741,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
 
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
-      LOG.error("Failed replacing compacted files in " + this +
+      LOG.error("Failed replacing compacted files in " + this.storeNameStr +
         ". Compacted file is " + (result == null? "none": result.toString()) +
         ".  Files replaced " + compactedFiles.toString() +
         " some of which may have been already removed", e);
@@ -1917,7 +2027,7 @@ public byte[] getSplitPoint() {
         return mk.getRow();
       }
     } catch(IOException e) {
-      LOG.warn("Failed getting store size for " + this, e);
+      LOG.warn("Failed getting store size for " + this.storeNameStr, e);
     } finally {
       this.lock.readLock().unlock();
     }
@@ -1970,7 +2080,7 @@ public KeyValueScanner getScanner(Scan scan,
 
   @Override
   public String toString() {
-    return getColumnFamilyName();
+    return this.storeNameStr;
   }
 
   /**
@@ -2086,7 +2196,7 @@ public HRegion getHRegion() {
   }
 
   HRegionInfo getHRegionInfo() {
-    return this.region.getRegionInfo();
+    return this.region.regionInfo;
   }
 
   /**
@@ -2214,8 +2324,8 @@ public CacheConfig getCacheConfig() {
 
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (17 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
-          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+          + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
index f91b7822c4ad..9bc66e1413fd 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
@@ -49,4 +49,5 @@ public CompactionProgress(long totalCompactingKVs) {
   public float getProgressPct() {
     return currentCompactedKVs / totalCompactingKVs;
   }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
index 885625b044c0..d2329e10c588 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
@@ -25,6 +25,9 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+
 /**
  * Checksum types. The Checksum type is a one byte number
  * that stores a representation of the checksum algorithm
@@ -67,7 +70,7 @@ public void initialize() {
         ctor = ChecksumFactory.newConstructor(PURECRC32);
         LOG.info("Checksum using " + PURECRC32);
       } catch (Exception e) {
-        LOG.trace(PURECRC32 + " not available.");
+        LOG.info(PURECRC32 + " not available.");
       }
       try {
         // The default checksum class name is java.util.zip.CRC32. 
@@ -77,7 +80,7 @@ public void initialize() {
           LOG.info("Checksum can use " + JDKCRC);
         }
       } catch (Exception e) {
-        LOG.trace(JDKCRC + " not available.");
+        LOG.warn(JDKCRC + " not available. ",  e);
       }
     }
 
@@ -110,7 +113,7 @@ public void initialize() {
         ctor = ChecksumFactory.newConstructor(PURECRC32C);
         LOG.info("Checksum can use " + PURECRC32C);
       } catch (Exception e) {
-        LOG.trace(PURECRC32C + " not available.");
+        LOG.info(PURECRC32C + " not available. ");
       }
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 18074efa6042..27b61a25e634 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -151,7 +151,7 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
    */
   public static FSDataOutputStream create(FileSystem fs, Path path,
       FsPermission perm, boolean overwrite) throws IOException {
-    LOG.debug("Creating file=" + path + " with permission=" + perm);
+    LOG.debug("Creating file:" + path + "with permission:" + perm);
 
     return fs.create(path, perm, overwrite,
         fs.getConf().getInt("io.file.buffer.size", 4096),
@@ -1012,25 +1012,6 @@ public boolean accept(Path rd) {
     }
   }
 
-  /**
-   * Given a particular region dir, return all the familydirs inside it
-   *
-   * @param fs A file system for the Path
-   * @param regionDir Path to a specific region directory
-   * @return List of paths to valid family directories in region dir.
-   * @throws IOException
-   */
-  public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
-    // assumes we are in a region dir.
-    FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
-    List<Path> familyDirs = new ArrayList<Path>(fds.length);
-    for (FileStatus fdfs: fds) {
-      Path fdPath = fdfs.getPath();
-      familyDirs.add(fdPath);
-    }
-    return familyDirs;
-  }
-
   /**
    * Filter for HFiles that excludes reference files.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 93908e7c8670..8134f4a93ecb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -587,10 +587,8 @@ public void testCompactionWithCorruptResult() throws Exception {
 
     List<StoreFile> storeFiles = store.getStorefiles();
     long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
-    Compactor tool = new Compactor(this.conf);
 
-    StoreFile.Writer compactedFile =
-      tool.compact(store, storeFiles, false, maxId);
+    StoreFile.Writer compactedFile = store.compactStore(storeFiles, false, maxId);
 
     // Now lets corrupt the compacted file.
     FileSystem fs = FileSystem.get(conf);

From 6d1fdc4aea0c76e88504171094a0d78084a55dae Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 6 Dec 2012 23:36:29 +0000
Subject: [PATCH 0613/1540] HBASE-7282 Backport Compaction Tool to 0.94; RETRY

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1418131 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   6 +-
 .../master/handler/CreateTableHandler.java    |   7 +-
 .../hbase/regionserver/CompactionTool.java    | 468 ++++++++++++++++++
 .../hadoop/hbase/regionserver/Compactor.java  | 213 ++++++++
 .../hadoop/hbase/regionserver/HRegion.java    | 129 +++--
 .../hadoop/hbase/regionserver/Store.java      | 296 ++++-------
 .../compactions/CompactionProgress.java       |   1 -
 .../hadoop/hbase/util/ChecksumType.java       |   9 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |  21 +-
 .../hbase/regionserver/TestCompaction.java    |   4 +-
 10 files changed, 898 insertions(+), 256 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index e6987ff71853..5f6f652eec14 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -645,7 +645,7 @@ public void logStats() {
     // Log size
     long totalSize = heapSize();
     long freeSize = maxSize - totalSize;
-    LruBlockCache.LOG.debug("LRU Stats: " +
+    LruBlockCache.LOG.debug("Stats: " +
         "total=" + StringUtils.byteDesc(totalSize) + ", " +
         "free=" + StringUtils.byteDesc(freeSize) + ", " +
         "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
@@ -653,11 +653,11 @@ public void logStats() {
         "accesses=" + stats.getRequestCount() + ", " +
         "hits=" + stats.getHitCount() + ", " +
         "hitRatio=" +
-          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) +
+          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
         "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
         "cachingHits=" + stats.getHitCachingCount() + ", " +
         "cachingHitsRatio=" +
-          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) +
+          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) + ", " +
         "evictions=" + stats.getEvictionCount() + ", " +
         "evicted=" + stats.getEvictedCount() + ", " +
         "evictedPerRun=" + stats.evictedPerEviction());
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 95750606df78..af25defc0d1f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -142,16 +142,12 @@ private void handleCreateTable() throws IOException, KeeperException {
     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
     final int batchSize =
       this.conf.getInt("hbase.master.createtable.batchsize", 100);
-    HLog hlog = null;
     for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
       HRegionInfo newRegion = this.newRegions[regionIdx];
       // 1. Create HRegion
       HRegion region = HRegion.createHRegion(newRegion,
         this.fileSystemManager.getRootDir(), this.conf,
-        this.hTableDescriptor, hlog);
-      if (hlog == null) {
-        hlog = region.getLog();
-      }
+        this.hTableDescriptor, null, false, true);
 
       regionInfos.add(region.getRegionInfo());
       if (regionIdx % batchSize == 0) {
@@ -163,7 +159,6 @@ private void handleCreateTable() throws IOException, KeeperException {
       // 3. Close the new region to flush to disk.  Close log file too.
       region.close();
     }
-    hlog.closeAndDelete();
     if (regionInfos.size() > 0) {
       MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
new file mode 100644
index 000000000000..e123c22761fe
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -0,0 +1,468 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/*
+ * The CompactionTool allows to execute a compaction specifying a:
+ * <ul>
+ *  <li>table folder (all regions and families will be compacted)
+ *  <li>region folder (all families in the region will be compacted)
+ *  <li>family folder (the store files will be compacted)
+ * </ul>
+ */
+@InterfaceAudience.Public
+public class CompactionTool extends Configured implements Tool {
+  private static final Log LOG = LogFactory.getLog(CompactionTool.class);
+
+  private final static String CONF_TMP_DIR = "hbase.tmp.dir";
+  private final static String CONF_COMPACT_ONCE = "hbase.compactiontool.compact.once";
+  private final static String CONF_DELETE_COMPACTED = "hbase.compactiontool.delete";
+  private final static String CONF_COMPLETE_COMPACTION = "hbase.hstore.compaction.complete";
+
+  /**
+   * Class responsible to execute the Compaction on the specified path.
+   * The path can be a table, region or family directory.
+   */
+  private static class CompactionWorker {
+    private final boolean keepCompactedFiles;
+    private final boolean deleteCompacted;
+    private final Configuration conf;
+    private final FileSystem fs;
+    private final Path tmpDir;
+
+    public CompactionWorker(final FileSystem fs, final Configuration conf) {
+      this.conf = conf;
+      this.keepCompactedFiles = !conf.getBoolean(CONF_COMPLETE_COMPACTION, true);
+      this.deleteCompacted = conf.getBoolean(CONF_DELETE_COMPACTED, false);
+      this.tmpDir = new Path(conf.get(CONF_TMP_DIR));
+      this.fs = fs;
+    }
+
+    /**
+     * Execute the compaction on the specified path.
+     *
+     * @param path Directory path on which run a
+     * @param compactOnce Execute just a single step of compaction.
+     */
+    public void compact(final Path path, final boolean compactOnce) throws IOException {
+      if (isFamilyDir(fs, path)) {
+        Path regionDir = path.getParent();
+        Path tableDir = regionDir.getParent();
+        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+        HRegion region = loadRegion(fs, conf, htd, regionDir);
+        compactStoreFiles(region, path, compactOnce);
+      } else if (isRegionDir(fs, path)) {
+        Path tableDir = path.getParent();
+        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+        compactRegion(htd, path, compactOnce);
+      } else if (isTableDir(fs, path)) {
+        compactTable(path, compactOnce);
+      } else {
+        throw new IOException(
+          "Specified path is not a table, region or family directory. path=" + path);
+      }
+    }
+
+    private void compactTable(final Path tableDir, final boolean compactOnce)
+        throws IOException {
+      HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+      LOG.info("Compact table=" + htd.getNameAsString());
+      for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
+        compactRegion(htd, regionDir, compactOnce);
+      }
+    }
+
+    private void compactRegion(final HTableDescriptor htd, final Path regionDir,
+        final boolean compactOnce) throws IOException {
+      HRegion region = loadRegion(fs, conf, htd, regionDir);
+      LOG.info("Compact table=" + htd.getNameAsString() +
+        " region=" + region.getRegionNameAsString());
+      for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+        compactStoreFiles(region, familyDir, compactOnce);
+      }
+    }
+
+    /**
+     * Execute the actual compaction job.
+     * If the compact once flag is not specified, execute the compaction until
+     * no more compactions are needed. Uses the Configuration settings provided.
+     */
+    private void compactStoreFiles(final HRegion region, final Path familyDir,
+        final boolean compactOnce) throws IOException {
+      LOG.info("Compact table=" + region.getTableDesc().getNameAsString() +
+        " region=" + region.getRegionNameAsString() +
+        " family=" + familyDir.getName());
+      Store store = getStore(region, familyDir);
+      do {
+        CompactionRequest cr = store.requestCompaction();
+        StoreFile storeFile = store.compact(cr);
+        if (storeFile != null) {
+          if (keepCompactedFiles && deleteCompacted) {
+            fs.delete(storeFile.getPath(), false);
+          }
+        }
+      } while (store.needsCompaction() && !compactOnce);
+    }
+
+    /**
+     * Create a "mock" HStore that uses the tmpDir specified by the user and
+     * the store dir to compact as source.
+     */
+    private Store getStore(final HRegion region, final Path storeDir) throws IOException {
+      byte[] familyName = Bytes.toBytes(storeDir.getName());
+      HColumnDescriptor hcd = region.getTableDesc().getFamily(familyName);
+      // Create a Store w/ check of hbase.rootdir blanked out and return our
+      // list of files instead of have Store search its home dir.
+      return new Store(tmpDir, region, hcd, fs, conf) {
+        @Override
+        public FileStatus[] getStoreFiles() throws IOException {
+          return this.fs.listStatus(getHomedir());
+        }
+
+        @Override
+        Path createStoreHomeDir(FileSystem fs, Path homedir) throws IOException {
+          return storeDir;
+        }
+      };
+    }
+
+    private static HRegion loadRegion(final FileSystem fs, final Configuration conf,
+        final HTableDescriptor htd, final Path regionDir) throws IOException {
+      Path rootDir = regionDir.getParent().getParent();
+      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+      return HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
+    }
+  }
+
+  private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
+    Path regionInfo = new Path(path, HRegion.REGIONINFO_FILE);
+    return fs.exists(regionInfo);
+  }
+
+  private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
+    return FSTableDescriptors.getTableInfoPath(fs, path) != null;
+  }
+
+  private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException {
+    return isRegionDir(fs, path.getParent());
+  }
+
+  private static class CompactionMapper
+      extends Mapper<LongWritable, Text, NullWritable, NullWritable> {
+    private CompactionWorker compactor = null;
+    private boolean compactOnce = false;
+
+    @Override
+    public void setup(Context context) {
+      Configuration conf = context.getConfiguration();
+      compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false);
+
+      try {
+        FileSystem fs = FileSystem.get(conf);
+        this.compactor = new CompactionWorker(fs, conf);
+      } catch (IOException e) {
+        throw new RuntimeException("Could not get the input FileSystem", e);
+      }
+    }
+
+    @Override
+    public void map(LongWritable key, Text value, Context context)
+        throws InterruptedException, IOException {
+      Path path = new Path(value.toString());
+      this.compactor.compact(path, compactOnce);
+    }
+  }
+
+  /**
+   * Input format that uses store files block location as input split locality.
+   */
+  private static class CompactionInputFormat extends TextInputFormat {
+    @Override
+    protected boolean isSplitable(JobContext context, Path file) {
+      return true;
+    }
+
+    /**
+     * Returns a split for each store files directory using the block location
+     * of each file as locality reference.
+     */
+    @Override
+    public List<InputSplit> getSplits(JobContext job) throws IOException {
+      List<InputSplit> splits = new ArrayList<InputSplit>();
+      List<FileStatus> files = listStatus(job);
+
+      Text key = new Text();
+      for (FileStatus file: files) {
+        Path path = file.getPath();
+        FileSystem fs = path.getFileSystem(job.getConfiguration());
+        LineReader reader = new LineReader(fs.open(path));
+        long pos = 0;
+        int n;
+        try {
+          while ((n = reader.readLine(key)) > 0) {
+            String[] hosts = getStoreDirHosts(fs, path);
+            splits.add(new FileSplit(path, pos, n, hosts));
+            pos += n;
+          }
+        } finally {
+          reader.close();
+        }
+      }
+
+      return splits;
+    }
+
+    /**
+     * return the top hosts of the store files, used by the Split
+     */
+    private static String[] getStoreDirHosts(final FileSystem fs, final Path path)
+        throws IOException {
+      FileStatus[] files = FSUtils.listStatus(fs, path, null);
+      if (files == null) {
+        return new String[] {};
+      }
+
+      HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
+      for (FileStatus hfileStatus: files) {
+        HDFSBlocksDistribution storeFileBlocksDistribution =
+          FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen());
+        hdfsBlocksDistribution.add(storeFileBlocksDistribution);
+      }
+
+      List<String> hosts = hdfsBlocksDistribution.getTopHosts();
+      return hosts.toArray(new String[hosts.size()]);
+    }
+
+    /**
+     * Create the input file for the given directories to compact.
+     * The file is a TextFile with each line corrisponding to a
+     * store files directory to compact.
+     */
+    public static void createInputFile(final FileSystem fs, final Path path,
+        final Set<Path> toCompactDirs) throws IOException {
+      // Extract the list of store dirs
+      List<Path> storeDirs = new LinkedList<Path>();
+      for (Path compactDir: toCompactDirs) {
+        if (isFamilyDir(fs, compactDir)) {
+          storeDirs.add(compactDir);
+        } else if (isRegionDir(fs, compactDir)) {
+          for (Path familyDir: FSUtils.getFamilyDirs(fs, compactDir)) {
+            storeDirs.add(familyDir);
+          }
+        } else if (isTableDir(fs, compactDir)) {
+          // Lookup regions
+          for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) {
+            for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+              storeDirs.add(familyDir);
+            }
+          }
+        } else {
+          throw new IOException(
+            "Specified path is not a table, region or family directory. path=" + compactDir);
+        }
+      }
+
+      // Write Input File
+      FSDataOutputStream stream = fs.create(path);
+      LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact.");
+      try {
+        final byte[] newLine = Bytes.toBytes("\n");
+        for (Path storeDir: storeDirs) {
+          stream.write(Bytes.toBytes(storeDir.toString()));
+          stream.write(newLine);
+        }
+      } finally {
+        stream.close();
+      }
+    }
+  }
+
+  /**
+   * Execute compaction, using a Map-Reduce job.
+   */
+  private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
+      final boolean compactOnce) throws Exception {
+    Configuration conf = getConf();
+    conf.setBoolean(CONF_COMPACT_ONCE, compactOnce);
+
+    Job job = new Job(conf);
+    job.setJobName("CompactionTool");
+    job.setJarByClass(CompactionTool.class);
+    job.setMapperClass(CompactionMapper.class);
+    job.setInputFormatClass(CompactionInputFormat.class);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    job.setMapSpeculativeExecution(false);
+    job.setNumReduceTasks(0);
+
+    String stagingName = "compact-" + EnvironmentEdgeManager.currentTimeMillis();
+    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
+    fs.mkdirs(stagingDir);
+    try {
+      // Create input file with the store dirs
+      Path inputPath = new Path(stagingDir, stagingName);
+      CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs);
+      CompactionInputFormat.addInputPath(job, inputPath);
+
+      // Initialize credential for secure cluster
+      TableMapReduceUtil.initCredentials(job);
+
+      // Start the MR Job and wait
+      return job.waitForCompletion(true) ? 0 : 1;
+    } finally {
+      fs.delete(stagingDir, true);
+    }
+  }
+
+  /**
+   * Execute compaction, from this client, one path at the time.
+   */
+  private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
+      final boolean compactOnce) throws IOException {
+    CompactionWorker worker = new CompactionWorker(fs, getConf());
+    for (Path path: toCompactDirs) {
+      worker.compact(path, compactOnce);
+    }
+    return 0;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    Set<Path> toCompactDirs = new HashSet<Path>();
+    boolean compactOnce = false;
+    boolean mapred = false;
+
+    Configuration conf = getConf();
+    FileSystem fs = FileSystem.get(conf);
+
+    try {
+      for (int i = 0; i < args.length; ++i) {
+        String opt = args[i];
+        if (opt.equals("-compactOnce")) {
+          compactOnce = true;
+        } else if (opt.equals("-mapred")) {
+          mapred = true;
+        } else if (!opt.startsWith("-")) {
+          Path path = new Path(opt);
+          FileStatus status = fs.getFileStatus(path);
+          if (!status.isDir()) {
+            printUsage("Specified path is not a directory. path=" + path);
+            return 1;
+          }
+          toCompactDirs.add(path);
+        } else {
+          printUsage();
+        }
+      }
+    } catch (Exception e) {
+      printUsage(e.getMessage());
+      return 1;
+    }
+
+    if (toCompactDirs.size() == 0) {
+      printUsage("No directories to compact specified.");
+      return 1;
+    }
+
+    // Execute compaction!
+    if (mapred) {
+      return doMapReduce(fs, toCompactDirs, compactOnce);
+    } else {
+      return doClient(fs, toCompactDirs, compactOnce);
+    }
+  }
+
+  private void printUsage() {
+    printUsage(null);
+  }
+
+  private void printUsage(final String message) {
+    if (message != null && message.length() > 0) {
+      System.err.println(message);
+    }
+    System.err.println("Usage: java " + this.getClass().getName() + " \\");
+    System.err.println("  [-compactOnce] [-mapred] [-D<property=value>]* files...");
+    System.err.println();
+    System.err.println("Options:");
+    System.err.println(" mapred         Use MapReduce to run compaction.");
+    System.err.println(" compactOnce    Execute just one compaction step. (default: while needed)");
+    System.err.println();
+    System.err.println("Note: -D properties will be applied to the conf used. ");
+    System.err.println("For example: ");
+    System.err.println(" To preserve input files, pass -D"+CONF_COMPLETE_COMPACTION+"=false");
+    System.err.println(" To stop delete of compacted file, pass -D"+CONF_DELETE_COMPACTED+"=false");
+    System.err.println(" To set tmp dir, pass -D"+CONF_TMP_DIR+"=ALTERNATE_DIR");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println(" To compact the full 'TestTable' using MapReduce:");
+    System.err.println(" $ bin/hbase " + this.getClass().getName() + " -mapred hdfs:///hbase/TestTable");
+    System.err.println();
+    System.err.println(" To compact column family 'x' of the table 'TestTable' region 'abc':");
+    System.err.println(" $ bin/hbase " + this.getClass().getName() + " hdfs:///hbase/TestTable/abc/x");
+  }
+
+  public static void main(String[] args) throws Exception {
+    System.exit(ToolRunner.run(HBaseConfiguration.create(), new CompactionTool(), args));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
new file mode 100644
index 000000000000..ad13248695a3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Compact passed set of files.
+ * Create an instance and then call {@ink #compact(Store, Collection, boolean, long)}.
+ */
+@InterfaceAudience.Private
+class Compactor extends Configured {
+  private static final Log LOG = LogFactory.getLog(Compactor.class);
+  private CompactionProgress progress;
+
+  Compactor(final Configuration c) {
+    super(c);
+  }
+
+  /**
+   * Do a minor/major compaction on an explicit set of storefiles from a Store.
+   *
+   * @param store Store the files belong to
+   * @param filesToCompact which files to compact
+   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
+   * @param maxId Readers maximum sequence id.
+   * @return Product of compaction or null if all cells expired or deleted and
+   * nothing made it through the compaction.
+   * @throws IOException
+   */
+  StoreFile.Writer compact(final Store store,
+      final Collection<StoreFile> filesToCompact,
+      final boolean majorCompaction, final long maxId)
+  throws IOException {
+    // Calculate maximum key count after compaction (for blooms)
+    // Also calculate earliest put timestamp if major compaction
+    int maxKeyCount = 0;
+    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+    for (StoreFile file : filesToCompact) {
+      StoreFile.Reader r = file.getReader();
+      if (r == null) {
+        LOG.warn("Null reader for " + file.getPath());
+        continue;
+      }
+      // NOTE: getFilterEntries could cause under-sized blooms if the user
+      //       switches bloom type (e.g. from ROW to ROWCOL)
+      long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType()) ?
+          r.getFilterEntries() : r.getEntries();
+      maxKeyCount += keyCount;
+      // For major compactions calculate the earliest put timestamp
+      // of all involved storefiles. This is used to remove 
+      // family delete marker during the compaction.
+      if (majorCompaction) {
+        byte[] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
+        if (tmp == null) {
+          // There's a file with no information, must be an old one
+          // assume we have very old puts
+          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
+        } else {
+          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
+        }
+      }
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Compacting " + file +
+          ", keycount=" + keyCount +
+          ", bloomtype=" + r.getBloomFilterType().toString() +
+          ", size=" + StringUtils.humanReadableInt(r.length()) +
+          ", encoding=" + r.getHFileReader().getEncodingOnDisk() +
+          (majorCompaction? ", earliestPutTs=" + earliestPutTs: ""));
+      }
+    }
+
+    // keep track of compaction progress
+    this.progress = new CompactionProgress(maxKeyCount);
+    // Get some configs
+    int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
+    Compression.Algorithm compression = store.getFamily().getCompression();
+    // Avoid overriding compression setting for major compactions if the user
+    // has not specified it separately
+    Compression.Algorithm compactionCompression =
+      (store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE) ?
+      store.getFamily().getCompactionCompression(): compression;
+
+    // For each file, obtain a scanner:
+    List<StoreFileScanner> scanners = StoreFileScanner
+      .getScannersForStoreFiles(filesToCompact, false, false, true);
+
+    // Make the instantiation lazy in case compaction produces no product; i.e.
+    // where all source cells are expired or deleted.
+    StoreFile.Writer writer = null;
+    // Find the smallest read point across all the Scanners.
+    long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
+    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
+    try {
+      InternalScanner scanner = null;
+      try {
+        if (store.getHRegion().getCoprocessorHost() != null) {
+          scanner = store.getHRegion()
+              .getCoprocessorHost()
+              .preCompactScannerOpen(store, scanners,
+                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
+        }
+        if (scanner == null) {
+          Scan scan = new Scan();
+          scan.setMaxVersions(store.getFamily().getMaxVersions());
+          /* Include deletes, unless we are doing a major compaction */
+          scanner = new StoreScanner(store, store.getScanInfo(), scan, scanners,
+            majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
+            smallestReadPoint, earliestPutTs);
+        }
+        if (store.getHRegion().getCoprocessorHost() != null) {
+          InternalScanner cpScanner =
+            store.getHRegion().getCoprocessorHost().preCompact(store, scanner);
+          // NULL scanner returned from coprocessor hooks means skip normal processing
+          if (cpScanner == null) {
+            return null;
+          }
+          scanner = cpScanner;
+        }
+
+        int bytesWritten = 0;
+        // since scanner.next() can return 'false' but still be delivering data,
+        // we have to use a do/while loop.
+        List<KeyValue> kvs = new ArrayList<KeyValue>();
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        boolean hasMore;
+        do {
+          hasMore = scanner.next(kvs, compactionKVMax);
+          if (writer == null && !kvs.isEmpty()) {
+            writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true);
+          }
+          if (writer != null) {
+            // output to writer:
+            for (KeyValue kv : kvs) {
+              if (kv.getMemstoreTS() <= smallestReadPoint) {
+                kv.setMemstoreTS(0);
+              }
+              writer.append(kv);
+              // update progress per key
+              ++progress.currentCompactedKVs;
+
+              // check periodically to see if a system stop is requested
+              if (Store.closeCheckInterval > 0) {
+                bytesWritten += kv.getLength();
+                if (bytesWritten > Store.closeCheckInterval) {
+                  bytesWritten = 0;
+                  isInterrupted(store, writer);
+                }
+              }
+            }
+          }
+          kvs.clear();
+        } while (hasMore);
+      } finally {
+        if (scanner != null) {
+          scanner.close();
+        }
+      }
+    } finally {
+      if (writer != null) {
+        writer.appendMetadata(maxId, majorCompaction);
+        writer.close();
+      }
+    }
+    return writer;
+  }
+
+  void isInterrupted(final Store store, final StoreFile.Writer writer)
+  throws IOException {
+    if (store.getHRegion().areWritesEnabled()) return;
+    // Else cleanup.
+    writer.close();
+    store.getFileSystem().delete(writer.getPath(), false);
+    throw new InterruptedIOException( "Aborting compaction of store " + store +
+      " in region " + store.getHRegion() + " because user requested stop.");
+  }
+
+  CompactionProgress getProgress() {
+    return this.progress;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f98d32d298d5..87edae3ec72a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.UnsupportedEncodingException;
@@ -62,6 +63,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -230,12 +232,12 @@ public class HRegion implements HeapSize { // , Writable{
    * The directory for the table this region is part of.
    * This directory contains the directory for this region.
    */
-  final Path tableDir;
+  private final Path tableDir;
 
-  final HLog log;
-  final FileSystem fs;
-  final Configuration conf;
-  final int rowLockWaitDuration;
+  private final HLog log;
+  private final FileSystem fs;
+  private final Configuration conf;
+  private final int rowLockWaitDuration;
   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
 
   // The internal wait duration to acquire a lock before read/update
@@ -256,8 +258,8 @@ public class HRegion implements HeapSize { // , Writable{
   // purge timeout, when a RPC call will be terminated by the RPC engine.
   final long maxBusyWaitDuration;
 
-  final HRegionInfo regionInfo;
-  final Path regiondir;
+  private final HRegionInfo regionInfo;
+  private final Path regiondir;
   KeyValue.KVComparator comparator;
 
   private ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;
@@ -724,7 +726,7 @@ public AtomicLong getMemstoreSize() {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     if (this.rsAccounting != null) {
       rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-    }  
+    }
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -750,7 +752,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
 
     // and then create the file
     Path tmpPath = new Path(getTmpDir(), REGIONINFO_FILE);
-    
+
     // if datanode crashes or if the RS goes down just before the close is called while trying to
     // close the created regioninfo file in the .tmp directory then on next
     // creation we will be getting AlreadyCreatedException.
@@ -758,7 +760,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     if (FSUtils.isExists(fs, tmpPath)) {
       FSUtils.delete(fs, tmpPath, true);
     }
-    
+
     FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
 
     try {
@@ -775,6 +777,26 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     }
   }
 
+  /**
+   * @param fs
+   * @param dir
+   * @return An HRegionInfo instance gotten from the <code>.regioninfo</code> file under region dir
+   * @throws IOException
+   */
+  public static HRegionInfo loadDotRegionInfoFileContent(final FileSystem fs, final Path dir)
+  throws IOException {
+    Path regioninfo = new Path(dir, HRegion.REGIONINFO_FILE);
+    if (!fs.exists(regioninfo)) throw new FileNotFoundException(regioninfo.toString());
+    FSDataInputStream in = fs.open(regioninfo);
+    try {
+      HRegionInfo hri = new HRegionInfo();
+      hri.readFields(in);
+      return hri;
+    } finally {
+      in.close();
+    }
+  }
+
   /** @return a HRegionInfo object for this region */
   public HRegionInfo getRegionInfo() {
     return this.regionInfo;
@@ -1021,19 +1043,16 @@ protected ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(
     return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
   }
 
-  private ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
+  static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
       final String threadNamePrefix) {
-    ThreadPoolExecutor openAndCloseThreadPool = Threads
-        .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
-            new ThreadFactory() {
-              private int count = 1;
-
-              public Thread newThread(Runnable r) {
-                Thread t = new Thread(r, threadNamePrefix + "-" + count++);
-                return t;
-              }
-            });
-    return openAndCloseThreadPool;
+    return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+      new ThreadFactory() {
+        private int count = 1;
+
+        public Thread newThread(Runnable r) {
+          return new Thread(r, threadNamePrefix + "-" + count++);
+        }
+      });
   }
 
    /**
@@ -1979,7 +1998,7 @@ public OperationStatus[] put(Pair<Put, Integer>[] putsAndLocks) throws IOExcepti
     System.arraycopy(putsAndLocks, 0, mutationsAndLocks, 0, putsAndLocks.length);
     return batchMutate(mutationsAndLocks);
   }
-  
+
   /**
    * Perform a batch of mutations.
    * It supports only Put and Delete mutations and will ignore other types passed.
@@ -2333,7 +2352,7 @@ private long doMiniBatchMutation(
 
       // do after lock
       final long netTimeMs = EnvironmentEdgeManager.currentTimeMillis()- startTimeMs;
-            
+
       // See if the column families were consistent through the whole thing.
       // if they were then keep them. If they were not then pass a null.
       // null will be treated as unknown.
@@ -2636,7 +2655,7 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updatePutMetrics(familyMap.keySet(), after - now);
-    
+
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -3759,6 +3778,7 @@ public static void closeHRegion(final HRegion r) throws IOException {
    * @param conf
    * @param hTableDescriptor
    * @param hlog shared HLog
+   * @param boolean initialize - true to initialize the region
    * @return new HRegion
    *
    * @throws IOException
@@ -3766,7 +3786,36 @@ public static void closeHRegion(final HRegion r) throws IOException {
   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                       final Configuration conf,
                                       final HTableDescriptor hTableDescriptor,
-                                      final HLog hlog)
+                                      final HLog hlog,
+                                      final boolean initialize)
+      throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor,
+        hlog, initialize, false);
+  }
+
+  /**
+   * Convenience method creating new HRegions. Used by createTable.
+   * The {@link HLog} for the created region needs to be closed
+   * explicitly, if it is not null.
+   * Use {@link HRegion#getLog()} to get access.
+   *
+   * @param info Info for region to create.
+   * @param rootDir Root directory for HBase instance
+   * @param conf
+   * @param hTableDescriptor
+   * @param hlog shared HLog
+   * @param boolean initialize - true to initialize the region
+   * @param boolean ignoreHLog
+      - true to skip generate new hlog if it is null, mostly for createTable
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog,
+                                      final boolean initialize, final boolean ignoreHLog)
       throws IOException {
     LOG.info("creating HRegion " + info.getTableNameAsString()
         + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
@@ -3778,16 +3827,26 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
     FileSystem fs = FileSystem.get(conf);
     fs.mkdirs(regionDir);
     HLog effectiveHLog = hlog;
-    if (hlog == null) {
+    if (hlog == null && !ignoreHLog) {
       effectiveHLog = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
           new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf);
     }
     HRegion region = HRegion.newHRegion(tableDir,
         effectiveHLog, fs, conf, info, hTableDescriptor, null);
-    region.initialize();
+    if (initialize) {
+      region.initialize();
+    }
     return region;
   }
 
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog)
+    throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor, hlog, true);
+  }
+
   /**
    * Open a Region.
    * @param info Info for region to be opened.
@@ -4299,7 +4358,7 @@ private List<KeyValue> get(Get get, boolean withCoprocessor)
     // do after lock
     final long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updateGetMetrics(get.familySet(), after - now);
-    
+
     return results;
   }
 
@@ -4627,10 +4686,10 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
       closeRegionOperation();
     }
 
-    
+
     long after = EnvironmentEdgeManager.currentTimeMillis();
     this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - before);
-    
+
     if (flush) {
       // Request a cache flush. Do it outside update lock.
       requestFlush();
@@ -4755,7 +4814,7 @@ public Result increment(Increment increment, Integer lockid,
       long after = EnvironmentEdgeManager.currentTimeMillis();
       this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
     }
-    
+
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
       requestFlush();
@@ -5248,7 +5307,7 @@ private void closeBulkRegionOperation() {
    */
   private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
     if (numPutsWithoutWAL.getAndIncrement() == 0) {
-      LOG.info("writing data to region " + this + 
+      LOG.info("writing data to region " + this +
                " with WAL disabled. Data may be lost in the event of a crash.");
     }
 
@@ -5360,11 +5419,11 @@ public static void main(String[] args) throws IOException {
     final HLog log = new HLog(fs, logdir, oldLogDir, c);
     try {
       processTable(fs, tableDir, log, c, majorCompact);
-     } finally {
+    } finally {
        log.close();
        // TODO: is this still right?
        BlockCache bc = new CacheConfig(c).getBlockCache();
        if (bc != null) bc.shutdown();
-     }
+    }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index f9e1103b0c10..9a02c16c77c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
-import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -132,9 +131,6 @@ public class Store extends SchemaConfigured implements HeapSize {
   private volatile long totalUncompressedBytes = 0L;
   private final Object flushLock = new Object();
   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final String storeNameStr;
-  private CompactionProgress progress;
-  private final int compactionKVMax;
   private final boolean verifyBulkLoads;
 
   /* The default priority for user-specified compaction requests.
@@ -158,10 +154,6 @@ public class Store extends SchemaConfigured implements HeapSize {
     new CopyOnWriteArraySet<ChangedReadersObserver>();
 
   private final int blocksize;
-  /** Compression algorithm for flush files and minor compaction */
-  private final Compression.Algorithm compression;
-  /** Compression algorithm for major compaction */
-  private final Compression.Algorithm compactionCompression;
   private HFileDataBlockEncoder dataBlockEncoder;
 
   /** Checksum configuration */
@@ -171,6 +163,8 @@ public class Store extends SchemaConfigured implements HeapSize {
   // Comparing KeyValues
   final KeyValue.KVComparator comparator;
 
+  private final Compactor compactor;
+
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -185,25 +179,16 @@ public class Store extends SchemaConfigured implements HeapSize {
   protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     FileSystem fs, Configuration conf)
   throws IOException {
-    super(conf, region.getTableDesc().getNameAsString(),
+    super(conf, region.getRegionInfo().getTableNameAsString(),
         Bytes.toString(family.getName()));
-    HRegionInfo info = region.regionInfo;
+    HRegionInfo info = region.getRegionInfo();
     this.fs = fs;
-    this.homedir = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
-    if (!this.fs.exists(this.homedir)) {
-      if (!this.fs.mkdirs(this.homedir))
-        throw new IOException("Failed create of: " + this.homedir.toString());
-    }
+    Path p = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
+    this.homedir = createStoreHomeDir(this.fs, p);
     this.region = region;
     this.family = family;
     this.conf = conf;
     this.blocksize = family.getBlocksize();
-    this.compression = family.getCompression();
-    // avoid overriding compression setting for major compactions if the user
-    // has not specified it separately
-    this.compactionCompression =
-      (family.getCompactionCompression() != Compression.Algorithm.NONE) ?
-        family.getCompactionCompression() : this.compression;
 
     this.dataBlockEncoder =
         new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(),
@@ -228,7 +213,6 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
         "ms in store " + this);
     scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
     this.memstore = new MemStore(conf, this.comparator);
-    this.storeNameStr = getColumnFamilyName();
 
     // By default, compact if storefile.count >= minFilesToCompact
     this.minFilesToCompact = Math.max(2,
@@ -245,10 +229,8 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       this.region.memstoreFlushSize);
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
-    this.compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
 
-    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify",
-        false);
+    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
 
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
@@ -260,6 +242,47 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.checksumType = getChecksumType(conf);
     // initilize bytes per checksum
     this.bytesPerChecksum = getBytesPerChecksum(conf);
+    // Create a compaction tool instance
+    this.compactor = new Compactor(this.conf);
+  }
+
+  /**
+   * @param family
+   * @return
+   */
+  long getTTL(final HColumnDescriptor family) {
+    // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
+    long ttl = family.getTimeToLive();
+    if (ttl == HConstants.FOREVER) {
+      // Default is unlimited ttl.
+      ttl = Long.MAX_VALUE;
+    } else if (ttl == -1) {
+      ttl = Long.MAX_VALUE;
+    } else {
+      // Second -> ms adjust for user data
+      ttl *= 1000;
+    }
+    return ttl;
+  }
+
+  /**
+   * Create this store's homedir
+   * @param fs
+   * @param homedir
+   * @return Return <code>homedir</code>
+   * @throws IOException
+   */
+  Path createStoreHomeDir(final FileSystem fs,
+      final Path homedir) throws IOException {
+    if (!fs.exists(homedir)) {
+      if (!fs.mkdirs(homedir))
+        throw new IOException("Failed create of: " + homedir.toString());
+    }
+    return homedir;
+  }
+
+  FileSystem getFileSystem() {
+    return this.fs;
   }
 
   /**
@@ -320,7 +343,7 @@ public static Path getStoreHomedir(final Path tabledir,
    * Return the directory in which this store stores its
    * StoreFiles
    */
-  public Path getHomedir() {
+  Path getHomedir() {
     return homedir;
   }
 
@@ -339,6 +362,10 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
     this.dataBlockEncoder = blockEncoder;
   }
 
+  FileStatus [] getStoreFiles() throws IOException {
+    return FSUtils.listStatus(this.fs, this.homedir, null);
+  }
+
   /**
    * Creates an unsorted list of StoreFile loaded in parallel
    * from the given directory.
@@ -346,7 +373,7 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
    */
   private List<StoreFile> loadStoreFiles() throws IOException {
     ArrayList<StoreFile> results = new ArrayList<StoreFile>();
-    FileStatus files[] = FSUtils.listStatus(this.fs, this.homedir, null);
+    FileStatus files[] = getStoreFiles();
 
     if (files == null || files.length == 0) {
       return results;
@@ -637,7 +664,7 @@ public Void call() throws IOException {
           storeFileCloserThreadPool.shutdownNow();
         }
       }
-      LOG.debug("closed " + this.storeNameStr);
+      LOG.info("Closed " + this);
       return result;
     } finally {
       this.lock.writeLock().unlock();
@@ -723,6 +750,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
       scanner = cpScanner;
     }
     try {
+      int compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
       // TODO:  We can fail in the below block before we complete adding this
       // flush to list of store files.  Add cleanup of anything put on filesystem
       // if we fail.
@@ -736,7 +764,7 @@ private Path internalFlushCache(final SortedSet<KeyValue> set,
           List<KeyValue> kvs = new ArrayList<KeyValue>();
           boolean hasMore;
           do {
-            hasMore = scanner.next(kvs, this.compactionKVMax);
+            hasMore = scanner.next(kvs, compactionKVMax);
             if (!kvs.isEmpty()) {
               for (KeyValue kv : kvs) {
                 // If we know that this KV is going to be included always, then let us
@@ -828,7 +856,7 @@ private StoreFile commitFile(final Path path,
    */
   private StoreFile.Writer createWriterInTmp(int maxKeyCount)
   throws IOException {
-    return createWriterInTmp(maxKeyCount, this.compression, false);
+    return createWriterInTmp(maxKeyCount, this.family.getCompression(), false);
   }
 
   /*
@@ -981,16 +1009,12 @@ void deleteChangedReaderObserver(ChangedReadersObserver o) {
    * @param cr
    *          compaction details obtained from requestCompaction()
    * @throws IOException
+   * @return Storefile we compacted into or null if we failed or opted out early.
    */
-  void compact(CompactionRequest cr) throws IOException {
-    if (cr == null || cr.getFiles().isEmpty()) {
-      return;
-    }
-    Preconditions.checkArgument(cr.getStore().toString()
-        .equals(this.toString()));
-
+  StoreFile compact(CompactionRequest cr) throws IOException {
+    if (cr == null || cr.getFiles().isEmpty()) return null;
+    Preconditions.checkArgument(cr.getStore().toString().equals(this.toString()));
     List<StoreFile> filesToCompact = cr.getFiles();
-
     synchronized (filesCompacting) {
       // sanity check: we're compacting files that this store knows about
       // TODO: change this to LOG.error() after more debugging
@@ -1002,19 +1026,26 @@ void compact(CompactionRequest cr) throws IOException {
 
     // Ready to go. Have list of files to compact.
     LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
-        + this.storeNameStr + " of "
+        + this + " of "
         + this.region.getRegionInfo().getRegionNameAsString()
         + " into tmpdir=" + region.getTmpDir() + ", seqid=" + maxId + ", totalSize="
         + StringUtils.humanReadableInt(cr.getSize()));
 
     StoreFile sf = null;
     try {
-      StoreFile.Writer writer = compactStore(filesToCompact, cr.isMajor(),
-          maxId);
+      StoreFile.Writer writer =
+        this.compactor.compact(this, filesToCompact, cr.isMajor(), maxId);
       // Move the compaction into place.
-      sf = completeCompaction(filesToCompact, writer);
-      if (region.getCoprocessorHost() != null) {
-        region.getCoprocessorHost().postCompact(this, sf);
+      if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
+        sf = completeCompaction(filesToCompact, writer);
+        if (region.getCoprocessorHost() != null) {
+          region.getCoprocessorHost().postCompact(this, sf);
+        }
+      } else {
+        // Create storefile around what we wrote with a reader on it.
+        sf = new StoreFile(this.fs, writer.getPath(), this.conf, this.cacheConf,
+          this.family.getBloomFilterType(), this.dataBlockEncoder);
+        sf.createReader();
       }
     } finally {
       synchronized (filesCompacting) {
@@ -1023,7 +1054,7 @@ void compact(CompactionRequest cr) throws IOException {
     }
 
     LOG.info("Completed" + (cr.isMajor() ? " major " : " ") + "compaction of "
-        + filesToCompact.size() + " file(s) in " + this.storeNameStr + " of "
+        + filesToCompact.size() + " file(s) in " + this + " of "
         + this.region.getRegionInfo().getRegionNameAsString()
         + " into " +
         (sf == null ? "none" : sf.getPath().getName()) +
@@ -1031,6 +1062,7 @@ void compact(CompactionRequest cr) throws IOException {
           StringUtils.humanReadableInt(sf.getReader().length()))
         + "; total size for store is "
         + StringUtils.humanReadableInt(storeSize));
+    return sf;
   }
 
   /**
@@ -1070,7 +1102,8 @@ public void compactRecentForTesting(int N) throws IOException {
 
     try {
       // Ready to go. Have list of files to compact.
-      StoreFile.Writer writer = compactStore(filesToCompact, isMajor, maxId);
+      StoreFile.Writer writer =
+        this.compactor.compact(this, filesToCompact, isMajor, maxId);
       // Move the compaction into place.
       StoreFile sf = completeCompaction(filesToCompact, writer);
       if (region.getCoprocessorHost() != null) {
@@ -1119,10 +1152,10 @@ public static long getLowestTimestamp(final List<StoreFile> candidates)
   }
 
   /** getter for CompactionProgress object
-   * @return CompactionProgress object
+   * @return CompactionProgress object; can be null
    */
   public CompactionProgress getCompactionProgress() {
-    return this.progress;
+    return this.compactor.getProgress();
   }
 
   /*
@@ -1174,19 +1207,19 @@ private boolean isMajorCompaction(final List<StoreFile> filesToCompact) throws I
         if (sf.isMajorCompaction() &&
             (this.ttl == HConstants.FOREVER || oldest < this.ttl)) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Skipping major compaction of " + this.storeNameStr +
+            LOG.debug("Skipping major compaction of " + this +
                 " because one (major) compacted file only and oldestTime " +
                 oldest + "ms is < ttl=" + this.ttl);
           }
         } else if (this.ttl != HConstants.FOREVER && oldest > this.ttl) {
-          LOG.debug("Major compaction triggered on store " + this.storeNameStr +
+          LOG.debug("Major compaction triggered on store " + this +
             ", because keyvalues outdated; time since last major compaction " +
             (now - lowTimestamp) + "ms");
           result = true;
         }
       } else {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Major compaction triggered on store " + this.storeNameStr +
+          LOG.debug("Major compaction triggered on store " + this +
               "; time since last major compaction " + (now - lowTimestamp) + "ms");
         }
         result = true;
@@ -1376,12 +1409,12 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
              compactSelection.getFilesToCompact().get(pos).getReader().length()
                > maxCompactSize &&
              !compactSelection.getFilesToCompact().get(pos).isReference()) ++pos;
-      compactSelection.clearSubList(0, pos);
+      if (pos != 0) compactSelection.clearSubList(0, pos);
     }
 
     if (compactSelection.getFilesToCompact().isEmpty()) {
       LOG.debug(this.getHRegionInfo().getEncodedName() + " - " +
-        this.storeNameStr + ": no store files to compact");
+        this + ": no store files to compact");
       compactSelection.emptyFileList();
       return compactSelection;
     }
@@ -1468,7 +1501,7 @@ public boolean apply(StoreFile input) {
       // if we don't have enough files to compact, just wait
       if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Skipped compaction of " + this.storeNameStr
+          LOG.debug("Skipped compaction of " + this
             + ".  Only " + (end - start) + " file(s) of size "
             + StringUtils.humanReadableInt(totalSize)
             + " have met compaction criteria.");
@@ -1494,149 +1527,6 @@ public boolean apply(StoreFile input) {
     return compactSelection;
   }
 
-  /**
-   * Do a minor/major compaction on an explicit set of storefiles in a Store.
-   * Uses the scan infrastructure to make it easy.
-   *
-   * @param filesToCompact which files to compact
-   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
-   * @param maxId Readers maximum sequence id.
-   * @return Product of compaction or null if all cells expired or deleted and
-   * nothing made it through the compaction.
-   * @throws IOException
-   */
-  StoreFile.Writer compactStore(final Collection<StoreFile> filesToCompact,
-                               final boolean majorCompaction, final long maxId)
-      throws IOException {
-    // calculate maximum key count after compaction (for blooms)
-    int maxKeyCount = 0;
-    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
-    for (StoreFile file : filesToCompact) {
-      StoreFile.Reader r = file.getReader();
-      if (r != null) {
-        // NOTE: getFilterEntries could cause under-sized blooms if the user
-        //       switches bloom type (e.g. from ROW to ROWCOL)
-        long keyCount = (r.getBloomFilterType() == family.getBloomFilterType())
-          ? r.getFilterEntries() : r.getEntries();
-        maxKeyCount += keyCount;
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Compacting " + file +
-            ", keycount=" + keyCount +
-            ", bloomtype=" + r.getBloomFilterType().toString() +
-            ", size=" + StringUtils.humanReadableInt(r.length()) +
-            ", encoding=" + r.getHFileReader().getEncodingOnDisk());
-        }
-      }
-      // For major compactions calculate the earliest put timestamp
-      // of all involved storefiles. This is used to remove 
-      // family delete marker during the compaction.
-      if (majorCompaction) {
-        byte[] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
-        if (tmp == null) {
-          // there's a file with no information, must be an old one
-          // assume we have very old puts
-          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
-        } else {
-          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
-        }
-      }
-    }
-
-    // keep track of compaction progress
-    progress = new CompactionProgress(maxKeyCount);
-
-    // For each file, obtain a scanner:
-    List<StoreFileScanner> scanners = StoreFileScanner
-      .getScannersForStoreFiles(filesToCompact, false, false, true);
-
-    // Make the instantiation lazy in case compaction produces no product; i.e.
-    // where all source cells are expired or deleted.
-    StoreFile.Writer writer = null;
-    // Find the smallest read point across all the Scanners.
-    long smallestReadPoint = region.getSmallestReadPoint();
-    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
-    try {
-      InternalScanner scanner = null;
-      try {
-        if (getHRegion().getCoprocessorHost() != null) {
-          scanner = getHRegion()
-              .getCoprocessorHost()
-              .preCompactScannerOpen(this, scanners,
-                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
-        }
-        if (scanner == null) {
-          Scan scan = new Scan();
-          scan.setMaxVersions(getFamily().getMaxVersions());
-          /* Include deletes, unless we are doing a major compaction */
-          scanner = new StoreScanner(this, getScanInfo(), scan, scanners,
-            majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
-            smallestReadPoint, earliestPutTs);
-        }
-        if (getHRegion().getCoprocessorHost() != null) {
-          InternalScanner cpScanner =
-            getHRegion().getCoprocessorHost().preCompact(this, scanner);
-          // NULL scanner returned from coprocessor hooks means skip normal processing
-          if (cpScanner == null) {
-            return null;
-          }
-          scanner = cpScanner;
-        }
-
-        int bytesWritten = 0;
-        // since scanner.next() can return 'false' but still be delivering data,
-        // we have to use a do/while loop.
-        ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
-        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
-        boolean hasMore;
-        do {
-          hasMore = scanner.next(kvs, this.compactionKVMax);
-          if (writer == null && !kvs.isEmpty()) {
-            writer = createWriterInTmp(maxKeyCount, this.compactionCompression,
-                true);
-          }
-          if (writer != null) {
-            // output to writer:
-            for (KeyValue kv : kvs) {
-              if (kv.getMemstoreTS() <= smallestReadPoint) {
-                kv.setMemstoreTS(0);
-              }
-              writer.append(kv);
-              // update progress per key
-              ++progress.currentCompactedKVs;
-
-              // check periodically to see if a system stop is requested
-              if (Store.closeCheckInterval > 0) {
-                bytesWritten += kv.getLength();
-                if (bytesWritten > Store.closeCheckInterval) {
-                  bytesWritten = 0;
-                  if (!this.region.areWritesEnabled()) {
-                    writer.close();
-                    fs.delete(writer.getPath(), false);
-                    throw new InterruptedIOException(
-                        "Aborting compaction of store " + this +
-                        " in region " + this.region +
-                        " because user requested stop.");
-                  }
-                }
-              }
-            }
-          }
-          kvs.clear();
-        } while (hasMore);
-      } finally {
-        if (scanner != null) {
-          scanner.close();
-        }
-      }
-    } finally {
-      if (writer != null) {
-        writer.appendMetadata(maxId, majorCompaction);
-        writer.close();
-      }
-    }
-    return writer;
-  }
-
   /**
    * Validates a store file by opening and closing it. In HFileV2 this should
    * not be an expensive operation.
@@ -1741,7 +1631,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
 
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
-      LOG.error("Failed replacing compacted files in " + this.storeNameStr +
+      LOG.error("Failed replacing compacted files in " + this +
         ". Compacted file is " + (result == null? "none": result.toString()) +
         ".  Files replaced " + compactedFiles.toString() +
         " some of which may have been already removed", e);
@@ -2027,7 +1917,7 @@ public byte[] getSplitPoint() {
         return mk.getRow();
       }
     } catch(IOException e) {
-      LOG.warn("Failed getting store size for " + this.storeNameStr, e);
+      LOG.warn("Failed getting store size for " + this, e);
     } finally {
       this.lock.readLock().unlock();
     }
@@ -2080,7 +1970,7 @@ public KeyValueScanner getScanner(Scan scan,
 
   @Override
   public String toString() {
-    return this.storeNameStr;
+    return getColumnFamilyName();
   }
 
   /**
@@ -2196,7 +2086,7 @@ public HRegion getHRegion() {
   }
 
   HRegionInfo getHRegionInfo() {
-    return this.region.regionInfo;
+    return this.region.getRegionInfo();
   }
 
   /**
@@ -2324,8 +2214,8 @@ public CacheConfig getCacheConfig() {
 
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
-          + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+          + (17 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
index 9bc66e1413fd..f91b7822c4ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
@@ -49,5 +49,4 @@ public CompactionProgress(long totalCompactingKVs) {
   public float getProgressPct() {
     return currentCompactedKVs / totalCompactingKVs;
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
index d2329e10c588..885625b044c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
@@ -25,9 +25,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.util.ChecksumFactory;
-
 /**
  * Checksum types. The Checksum type is a one byte number
  * that stores a representation of the checksum algorithm
@@ -70,7 +67,7 @@ public void initialize() {
         ctor = ChecksumFactory.newConstructor(PURECRC32);
         LOG.info("Checksum using " + PURECRC32);
       } catch (Exception e) {
-        LOG.info(PURECRC32 + " not available.");
+        LOG.trace(PURECRC32 + " not available.");
       }
       try {
         // The default checksum class name is java.util.zip.CRC32. 
@@ -80,7 +77,7 @@ public void initialize() {
           LOG.info("Checksum can use " + JDKCRC);
         }
       } catch (Exception e) {
-        LOG.warn(JDKCRC + " not available. ",  e);
+        LOG.trace(JDKCRC + " not available.");
       }
     }
 
@@ -113,7 +110,7 @@ public void initialize() {
         ctor = ChecksumFactory.newConstructor(PURECRC32C);
         LOG.info("Checksum can use " + PURECRC32C);
       } catch (Exception e) {
-        LOG.info(PURECRC32C + " not available. ");
+        LOG.trace(PURECRC32C + " not available.");
       }
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 27b61a25e634..18074efa6042 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -151,7 +151,7 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
    */
   public static FSDataOutputStream create(FileSystem fs, Path path,
       FsPermission perm, boolean overwrite) throws IOException {
-    LOG.debug("Creating file:" + path + "with permission:" + perm);
+    LOG.debug("Creating file=" + path + " with permission=" + perm);
 
     return fs.create(path, perm, overwrite,
         fs.getConf().getInt("io.file.buffer.size", 4096),
@@ -1012,6 +1012,25 @@ public boolean accept(Path rd) {
     }
   }
 
+  /**
+   * Given a particular region dir, return all the familydirs inside it
+   *
+   * @param fs A file system for the Path
+   * @param regionDir Path to a specific region directory
+   * @return List of paths to valid family directories in region dir.
+   * @throws IOException
+   */
+  public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
+    // assumes we are in a region dir.
+    FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
+    List<Path> familyDirs = new ArrayList<Path>(fds.length);
+    for (FileStatus fdfs: fds) {
+      Path fdPath = fdfs.getPath();
+      familyDirs.add(fdPath);
+    }
+    return familyDirs;
+  }
+
   /**
    * Filter for HFiles that excludes reference files.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 8134f4a93ecb..93908e7c8670 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -587,8 +587,10 @@ public void testCompactionWithCorruptResult() throws Exception {
 
     List<StoreFile> storeFiles = store.getStorefiles();
     long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
+    Compactor tool = new Compactor(this.conf);
 
-    StoreFile.Writer compactedFile = store.compactStore(storeFiles, false, maxId);
+    StoreFile.Writer compactedFile =
+      tool.compact(store, storeFiles, false, maxId);
 
     // Now lets corrupt the compacted file.
     FileSystem fs = FileSystem.get(conf);

From 60bb406359beb2347a1a505ff3b0f98f720db266 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Sat, 8 Dec 2012 00:46:44 +0000
Subject: [PATCH 0614/1540] HBASE-7300  HbckTestingUtil needs to keep a static
 executor to lower the number of threads used

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1418570 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 299ae6e64d69..343402196f89 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 
 public class HbckTestingUtil {
+  private static ExecutorService exec = new ScheduledThreadPoolExecutor(10);
   public static HBaseFsck doFsck(
       Configuration conf, boolean fix) throws Exception {
     return doFsck(conf, fix, null);
@@ -44,7 +45,7 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
       boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
       boolean fixHdfsOrphans, boolean fixTableOrphans, boolean fixVersionFile,
       String table) throws Exception {
-    HBaseFsck fsck = new HBaseFsck(conf);
+    HBaseFsck fsck = new HBaseFsck(conf, exec);
     fsck.connect();
     fsck.setDisplayFullReport(); // i.e. -details
     fsck.setTimeLag(0);
@@ -71,7 +72,6 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
    */
   public static HBaseFsck doHFileQuarantine(Configuration conf, String table) throws Exception {
     String[] args = {"-sidelineCorruptHFiles", "-ignorePreCheckPermission", table};
-    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
     HBaseFsck hbck = new HBaseFsck(conf, exec);
     hbck.exec(exec, args);
     return hbck;

From 0b82fd5e9e2a5a82ca747f981db1999197faec4c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 8 Dec 2012 21:41:02 +0000
Subject: [PATCH 0615/1540] HBASE-7301 Force ipv4 for unit tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1418758 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 28463c18a5af..ae22b125df67 100644
--- a/pom.xml
+++ b/pom.xml
@@ -767,7 +767,7 @@
                  I believe it is a failsafe bug, we may consider using surefire -->
             <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
             <argLine>-enableassertions -Xmx1900m
-              -Djava.security.egd=file:/dev/./urandom</argLine>
+              -Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true</argLine>
             <testFailureIgnore>false</testFailureIgnore>
           </configuration>
       </plugin>

From 2c13d32b5691ec9688203db5fbe2753283a493f0 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 9 Dec 2012 22:12:53 +0000
Subject: [PATCH 0616/1540] HBASE-6317 Master clean start up and Partially
 enabled tables make region assignment inconsistent (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1419174 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 83 ++++++++++++------
 .../master/handler/EnableTableHandler.java    | 71 +++++++++++-----
 .../hbase/master/TestAssignmentManager.java   | 84 +++++++++++++++++--
 3 files changed, 187 insertions(+), 51 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 503061d52dc5..e0598a8a887a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -142,9 +142,10 @@ public class AssignmentManager extends ZooKeeperListener {
 
   // store all the table names in disabling state
   Set<String> disablingTables = new HashSet<String>(1);
-  // store all the enabling state tablenames.
-  Set<String> enablingTables = new HashSet<String>(1);
-
+  // store all the enabling state table names and corresponding online servers' regions.
+  // This may be needed to avoid calling assign twice for the regions of the ENABLING table
+  // that could have been assigned through processRIT.
+  Map<String, List<HRegionInfo>> enablingTables = new HashMap<String, List<HRegionInfo>>(1);
   /**
    * Server to regions assignment map.
    * Contains the set of regions currently assigned to a given server.
@@ -273,6 +274,16 @@ public ServerName getRegionServerOfRegion(HRegionInfo hri) {
     }
   }
 
+  /**
+   * Gives enabling table regions.
+   * 
+   * @param tableName
+   * @return list of regionInfos
+   */
+  public List<HRegionInfo> getEnablingTableRegions(String tableName){
+    return this.enablingTables.get(tableName);
+  }
+
   /**
    * Add a regionPlan for the specified region.
    * @param encodedName 
@@ -364,7 +375,9 @@ void joinCluster() throws IOException,
     // Recover the tables that were not fully moved to DISABLED state.
     // These tables are in DISABLING state when the master restarted/switched.
     boolean isWatcherCreated = recoverTableInDisablingState(this.disablingTables);
-    recoverTableInEnablingState(this.enablingTables, isWatcherCreated);
+    recoverTableInEnablingState(this.enablingTables.keySet(), isWatcherCreated);
+    this.enablingTables.clear();
+    this.disablingTables.clear();
   }
 
   /**
@@ -509,6 +522,10 @@ void processRegionsInTransition(final RegionTransitionData data,
     String encodedRegionName = regionInfo.getEncodedName();
     LOG.info("Processing region " + regionInfo.getRegionNameAsString() +
       " in state " + data.getEventType());
+    List<HRegionInfo> hris = this.enablingTables.get(regionInfo.getTableNameAsString());
+    if (hris != null && !hris.isEmpty()) {
+      hris.remove(regionInfo);
+    }
     synchronized (regionsInTransition) {
       RegionState regionState = regionsInTransition.get(encodedRegionName);
       if (regionState != null ||
@@ -2306,11 +2323,12 @@ public void assignAllUserRegions() throws IOException, InterruptedException {
     // Skip assignment for regions of tables in DISABLING state also because
     // during clean cluster startup no RS is alive and regions map also doesn't
     // have any information about the regions. See HBASE-6281.
-    Set<String> disablingAndDisabledTables = new HashSet<String>(this.disablingTables);
-    disablingAndDisabledTables.addAll(this.zkTable.getDisabledTables());
+    Set<String> disablingDisabledAndEnablingTables = new HashSet<String>(this.disablingTables);
+    disablingDisabledAndEnablingTables.addAll(this.zkTable.getDisabledTables());
+    disablingDisabledAndEnablingTables.addAll(this.enablingTables.keySet());
     // Scan META for all user regions, skipping any disabled tables
     Map<HRegionInfo, ServerName> allRegions = MetaReader.fullScan(catalogTracker,
-        disablingAndDisabledTables, true);
+        disablingDisabledAndEnablingTables, true);
     if (allRegions == null || allRegions.isEmpty()) return;
 
     // Get all available servers
@@ -2558,13 +2576,14 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOE
         // from ENABLED state when application calls disableTable.
         // It can't be in DISABLED state, because DISABLED states transitions
         // from DISABLING state.
-        if (false == checkIfRegionsBelongsToEnabling(regionInfo)) {
-          LOG.warn("Region " + regionInfo.getEncodedName() +
-            " has null regionLocation." + " But its table " + tableName +
-            " isn't in ENABLING state.");
+        boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+        addTheTablesInPartialState(regionInfo);
+        if (enabling) {
+          addToEnablingTableRegions(regionInfo);
+        } else {
+          LOG.warn("Region " + regionInfo.getEncodedName() + " has null regionLocation."
+              + " But its table " + tableName + " isn't in ENABLING state.");
         }
-        addTheTablesInPartialState(this.disablingTables, this.enablingTables, regionInfo,
-            tableName);
       } else if (!onlineServers.contains(regionLocation)) {
         // Region is located on a server that isn't online
         List<Pair<HRegionInfo, Result>> offlineRegions =
@@ -2575,8 +2594,7 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOE
         }
         offlineRegions.add(new Pair<HRegionInfo,Result>(regionInfo, result));
         disabled = checkIfRegionBelongsToDisabled(regionInfo);
-        disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
-            this.enablingTables, regionInfo, tableName);
+        disablingOrEnabling = addTheTablesInPartialState(regionInfo);
         // need to enable the table if not disabled or disabling or enabling
         // this will be used in rolling restarts
         enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
@@ -2597,16 +2615,18 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOE
         }
         // Region is being served and on an active server
         // add only if region not in disabled and enabling table
-        if (false == checkIfRegionBelongsToDisabled(regionInfo)
-            && false == checkIfRegionsBelongsToEnabling(regionInfo)) {
+        boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+        disabled = checkIfRegionBelongsToDisabled(regionInfo);
+        if (!enabling && !disabled) {
           synchronized (this.regions) {
             regions.put(regionInfo, regionLocation);
             addToServers(regionLocation, regionInfo);
           }
         }
-        disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
-            this.enablingTables, regionInfo, tableName);
-        disabled = checkIfRegionBelongsToDisabled(regionInfo);
+        disablingOrEnabling = addTheTablesInPartialState(regionInfo);
+        if (enabling) {
+          addToEnablingTableRegions(regionInfo);
+        }
         // need to enable the table if not disabled or disabling or enabling
         // this will be used in rolling restarts
         enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
@@ -2616,6 +2636,18 @@ Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOE
     return offlineServers;
   }
 
+  private void addToEnablingTableRegions(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
+    List<HRegionInfo> hris = this.enablingTables.get(tableName);
+    if (!hris.contains(regionInfo)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adding region" + regionInfo.getRegionNameAsString()
+            + " to enabling table " + tableName + ".");
+      }
+      hris.add(regionInfo);
+    }
+  }
+  
   private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
       boolean disablingOrEnabling, String tableName) {
     if (!disabled && !disablingOrEnabling
@@ -2624,14 +2656,15 @@ private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
     }
   }
 
-  private Boolean addTheTablesInPartialState(Set<String> disablingTables,
-      Set<String> enablingTables, HRegionInfo regionInfo,
-      String disablingTableName) {
+  private Boolean addTheTablesInPartialState(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
     if (checkIfRegionBelongsToDisabling(regionInfo)) {
-      disablingTables.add(disablingTableName);
+      this.disablingTables.add(tableName);
       return true;
     } else if (checkIfRegionsBelongsToEnabling(regionInfo)) {
-      enablingTables.add(disablingTableName);
+      if (!this.enablingTables.containsKey(tableName)) {
+        this.enablingTables.put(tableName, new ArrayList<HRegionInfo>());
+      } 
       return true;
     } 
     return false;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
index 00f8e72738ca..aa147b532399 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 
@@ -27,6 +28,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
@@ -34,7 +36,11 @@
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.BulkAssigner;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -46,6 +52,7 @@ public class EnableTableHandler extends EventHandler {
   private final String tableNameStr;
   private final AssignmentManager assignmentManager;
   private final CatalogTracker ct;
+  private boolean retainAssignment = false;
 
   public EnableTableHandler(Server server, byte [] tableName,
       CatalogTracker catalogTracker, AssignmentManager assignmentManager,
@@ -56,6 +63,7 @@ public EnableTableHandler(Server server, byte [] tableName,
     this.tableNameStr = Bytes.toString(tableName);
     this.ct = catalogTracker;
     this.assignmentManager = assignmentManager;
+    this.retainAssignment = skipTableStateCheck;
     // Check if table exists
     if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
       throw new TableNotFoundException(Bytes.toString(tableName));
@@ -99,10 +107,12 @@ public void process() {
       LOG.error("Error trying to enable the table " + this.tableNameStr, e);
     } catch (KeeperException e) {
       LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+    } catch (InterruptedException e) {
+      LOG.error("Error trying to enable the table " + this.tableNameStr, e);
     }
   }
 
-  private void handleEnableTable() throws IOException, KeeperException {
+  private void handleEnableTable() throws IOException, KeeperException, InterruptedException {
     // I could check table is disabling and if so, not enable but require
     // that user first finish disabling but that might be obnoxious.
 
@@ -111,18 +121,18 @@ private void handleEnableTable() throws IOException, KeeperException {
     boolean done = false;
     // Get the regions of this table. We're done when all listed
     // tables are onlined.
-    List<HRegionInfo> regionsInMeta;
-    regionsInMeta = MetaReader.getTableRegions(this.ct, tableName, true);
-    int countOfRegionsInTable = regionsInMeta.size();
-    List<HRegionInfo> regions = regionsToAssign(regionsInMeta);
+    List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations = MetaReader
+        .getTableRegionsAndLocations(this.ct, tableName, true);
+    int countOfRegionsInTable = tableRegionsAndLocations.size();
+    List<HRegionInfo> regions = regionsToAssignWithServerName(tableRegionsAndLocations);
     int regionsCount = regions.size();
     if (regionsCount == 0) {
       done = true;
     }
     LOG.info("Table has " + countOfRegionsInTable + " regions of which " +
       regionsCount + " are offline.");
-    BulkEnabler bd = new BulkEnabler(this.server, regions,
-      countOfRegionsInTable);
+    BulkEnabler bd = new BulkEnabler(this.server, regions, countOfRegionsInTable,
+        this.retainAssignment);
     try {
       if (bd.bulkAssign()) {
         done = true;
@@ -140,17 +150,34 @@ private void handleEnableTable() throws IOException, KeeperException {
 
   /**
    * @param regionsInMeta This datastructure is edited by this method.
-   * @return The <code>regionsInMeta</code> list minus the regions that have
-   * been onlined; i.e. List of regions that need onlining.
+   * @return List of regions neither in transition nor assigned.
    * @throws IOException
    */
-  private List<HRegionInfo> regionsToAssign(
-    final List<HRegionInfo> regionsInMeta)
-  throws IOException {
-    final List<HRegionInfo> onlineRegions =
-      this.assignmentManager.getRegionsOfTable(tableName);
-    regionsInMeta.removeAll(onlineRegions);
-    return regionsInMeta;
+  private List<HRegionInfo> regionsToAssignWithServerName(
+      final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException {
+    ServerManager serverManager = ((HMaster) this.server).getServerManager();
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+    List<HRegionInfo> enablingTableRegions = this.assignmentManager
+        .getEnablingTableRegions(this.tableNameStr);
+    final List<HRegionInfo> onlineRegions = this.assignmentManager.getRegionsOfTable(tableName);
+    for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
+      HRegionInfo hri = regionLocation.getFirst();
+      ServerName sn = regionLocation.getSecond();
+      if (this.retainAssignment) {
+        // Region may be available in enablingTableRegions during master startup only.
+        if (enablingTableRegions != null && enablingTableRegions.contains(hri)) {
+          regions.add(hri);
+          if (sn != null && serverManager.isServerOnline(sn)) {
+            this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn));
+          }
+        }
+      } else if (onlineRegions.contains(hri)) {
+        continue;
+      } else {
+        regions.add(hri);
+      }
+    }
+    return regions;
   }
 
   /**
@@ -160,12 +187,14 @@ class BulkEnabler extends BulkAssigner {
     private final List<HRegionInfo> regions;
     // Count of regions in table at time this assign was launched.
     private final int countOfRegionsInTable;
+    private final boolean retainAssignment;
 
     BulkEnabler(final Server server, final List<HRegionInfo> regions,
-        final int countOfRegionsInTable) {
+        final int countOfRegionsInTable,final boolean retainAssignment) {
       super(server);
       this.regions = regions;
       this.countOfRegionsInTable = countOfRegionsInTable;
+      this.retainAssignment = retainAssignment;
     }
 
     @Override
@@ -173,7 +202,7 @@ protected void populatePool(ExecutorService pool) throws IOException {
       boolean roundRobinAssignment = this.server.getConfiguration().getBoolean(
           "hbase.master.enabletable.roundrobin", false);
 
-      if (!roundRobinAssignment) {
+      if (retainAssignment || !roundRobinAssignment) {
         for (HRegionInfo region : regions) {
           if (assignmentManager.isRegionInTransition(region) != null) {
             continue;
@@ -181,7 +210,11 @@ protected void populatePool(ExecutorService pool) throws IOException {
           final HRegionInfo hri = region;
           pool.execute(new Runnable() {
             public void run() {
-              assignmentManager.assign(hri, true);
+              if (retainAssignment) {
+                assignmentManager.assign(hri, true, false, false);
+              } else {
+                assignmentManager.assign(hri, true);
+              }
             }
           });
         }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 6e921497a42e..ba4606964f73 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -74,6 +74,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
 
 import com.google.protobuf.ServiceException;
 
@@ -91,6 +92,10 @@ public class TestAssignmentManager {
   private static final HRegionInfo REGIONINFO =
     new HRegionInfo(Bytes.toBytes("t"),
       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
+  private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
+      Bytes.toBytes("a"),Bytes.toBytes( "b"));
+  private static int assignmentCount;
+  private static boolean enabling = false;  
 
   // Mocked objects or; get redone for each test.
   private Server server;
@@ -768,14 +773,27 @@ private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final
     // with an encoded name by doing a Get on .META.
     HRegionInterface ri = Mockito.mock(HRegionInterface.class);
     // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
+    Result[] result = null;
+    if (enabling) {
+      result = new Result[2];
+      result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+      result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
+    }
     Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
     Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
       thenReturn(System.currentTimeMillis());
-    // Return good result 'r' first and then return null to indicate end of scan
-    Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
-    // If a get, return the above result too for REGIONINFO
-    Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).
-      thenReturn(r);
+   if (enabling) {
+      Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
+          (Result[]) null);
+      // If a get, return the above result too for REGIONINFO_2
+      Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
+          getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
+    } else {
+      // Return good result 'r' first and then return null to indicate end of scan
+      Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
+      // If a get, return the above result too for REGIONINFO
+      Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
+    }
     // Get a connection w/ mocked up common methods.
     HConnection connection = HConnectionTestingUtility.
       getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
@@ -891,6 +909,53 @@ public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
     }
   }
 
+  /**
+   * Test verifies whether all the enabling table regions assigned only once during master startup.
+   * 
+   * @throws KeeperException
+   * @throws IOException
+   * @throws Exception
+   */
+  @Test
+  public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
+    enabling = true;
+    this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        DefaultLoadBalancer.class, LoadBalancer.class);
+    Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
+    serverAndLoad.put(SERVERNAME_A, null);
+    Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
+    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
+    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
+    HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
+    Server server = new HMaster(HTU.getConfiguration());
+    Whitebox.setInternalState(server, "serverManager", this.serverManager);
+    assignmentCount = 0;
+    AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
+        this.serverManager);
+    am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
+        HConstants.EMPTY_END_ROW), SERVERNAME_A);
+    am.gate.set(false);
+    try {
+      // set table in enabling state.
+      am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
+      ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
+
+      am.joinCluster();
+      while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
+        Thread.sleep(10);
+      }
+      assertEquals("Number of assignments should be equal.", 2, assignmentCount);
+      assertTrue("Table should be enabled.",
+          am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
+    } finally {
+      enabling = false;
+      am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
+      am.shutdown();
+      ZKAssign.deleteAllNodes(this.watcher);
+      assignmentCount = 0;
+    }
+  }
+
   /**
    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
    * random assignment is called and the gate variable is set to true.
@@ -960,8 +1025,13 @@ void processRegionsInTransition(final RegionTransitionData data,
     @Override
     public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
         boolean hijack) {
-      assignInvoked = true;
-      super.assign(region, setOfflineInZK, forceNewPlan, hijack);
+      if (enabling) {
+        assignmentCount++;
+        this.regionOnline(region, SERVERNAME_A);
+      } else {
+        assignInvoked = true;
+        super.assign(region, setOfflineInZK, forceNewPlan, hijack);
+      }
     }
     
     @Override

From c4ec5a54d319c1a3219bd38326d5a07f4ef7b52d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 11 Dec 2012 05:46:45 +0000
Subject: [PATCH 0617/1540] HBASE-7180 RegionScannerImpl.next() is inefficient.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1420005 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 46 +++++++++++------
 .../hbase/regionserver/HRegionServer.java     | 38 ++++++++------
 .../hbase/regionserver/RegionScanner.java     | 49 +++++++++++++++++++
 .../coprocessor/TestCoprocessorInterface.java | 16 ++++++
 4 files changed, 120 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 87edae3ec72a..2ad3f0435c07 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3490,6 +3490,10 @@ public HRegionInfo getRegionInfo() {
       this(scan, null);
     }
 
+    @Override
+    public long getMvccReadPoint() {
+      return this.readPt;
+    }
     /**
      * Reset both the filter and the old filter.
      */
@@ -3500,7 +3504,7 @@ protected void resetFilters() {
     }
 
     @Override
-    public synchronized boolean next(List<KeyValue> outResults, int limit)
+    public boolean next(List<KeyValue> outResults, int limit)
         throws IOException {
       return next(outResults, limit, null);
     }
@@ -3520,30 +3524,42 @@ public synchronized boolean next(List<KeyValue> outResults, int limit,
         // This could be a new thread from the last time we called next().
         MultiVersionConsistencyControl.setThreadReadPoint(this.readPt);
 
-        results.clear();
-
-        boolean returnResult = nextInternal(limit, metric);
-
-        outResults.addAll(results);
-        resetFilters();
-        if (isFilterDone()) {
-          return false;
-        }
-        return returnResult;
+        return nextRaw(outResults, limit, metric);
       } finally {
         closeRegionOperation();
       }
     }
 
     @Override
-    public synchronized boolean next(List<KeyValue> outResults)
+    public boolean nextRaw(List<KeyValue> outResults, String metric)
+        throws IOException {
+      return nextRaw(outResults, batch, metric);
+    }
+
+    @Override
+    public boolean nextRaw(List<KeyValue> outResults, int limit,
+        String metric) throws IOException {
+      results.clear();
+
+      boolean returnResult = nextInternal(limit, metric);
+
+      outResults.addAll(results);
+      resetFilters();
+      if (isFilterDone()) {
+        return false;
+      }
+      return returnResult;
+    }
+
+    @Override
+    public boolean next(List<KeyValue> outResults)
         throws IOException {
       // apply the batching limit by default
       return next(outResults, batch, null);
     }
 
     @Override
-    public synchronized boolean next(List<KeyValue> outResults, String metric)
+    public boolean next(List<KeyValue> outResults, String metric)
         throws IOException {
       // apply the batching limit by default
       return next(outResults, batch, metric);
@@ -5245,7 +5261,7 @@ public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {
    * @throws RegionTooBusyException if failed to get the lock in time
    * @throws InterruptedIOException if interrupted while waiting for a lock
    */
-  private void startRegionOperation()
+  public void startRegionOperation()
       throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
     if (this.closing.get()) {
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
@@ -5263,7 +5279,7 @@ private void startRegionOperation()
    * Closes the lock. This needs to be called in the finally block corresponding
    * to the try block of #startRegionOperation
    */
-  private void closeRegionOperation(){
+  public void closeRegionOperation(){
     lock.readLock().unlock();
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a4b4b818173c..b3181d61d129 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -165,6 +165,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 import org.codehaus.jackson.map.ObjectMapper;
+import org.joda.time.field.MillisDurationField;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
@@ -2430,23 +2431,32 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
         }
       }
 
-      for (int i = 0; i < nbRows
-          && currentScanResultSize < maxScannerResultSize; i++) {
-        requestCount.incrementAndGet();
-        // Collect values to be returned here
-        boolean moreRows = s.next(values, SchemaMetrics.METRIC_NEXTSIZE);
-        if (!values.isEmpty()) {
-          for (KeyValue kv : values) {
-            currentScanResultSize += kv.heapSize();
+      MultiVersionConsistencyControl.setThreadReadPoint(s.getMvccReadPoint());
+      region.startRegionOperation();
+      try {
+        int i = 0;
+        synchronized(s) {
+          for (; i < nbRows
+              && currentScanResultSize < maxScannerResultSize; i++) {
+            // Collect values to be returned here
+            boolean moreRows = s.nextRaw(values, SchemaMetrics.METRIC_NEXTSIZE);
+            if (!values.isEmpty()) {
+              for (KeyValue kv : values) {
+                currentScanResultSize += kv.heapSize();
+              }
+              results.add(new Result(values));
+            }
+            if (!moreRows) {
+              break;
+            }
+            values.clear();
           }
-          results.add(new Result(values));
         }
-        if (!moreRows) {
-          break;
-        }
-        values.clear();
+        requestCount.addAndGet(i);
+        region.readRequestsCount.add(i);
+      } finally {
+        region.closeRegionOperation();
       }
-
       // coprocessor postNext hook
       if (region != null && region.getCoprocessorHost() != null) {
         region.getCoprocessorHost().postScannerNext(s, results, nbRows, true);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
index da95e901a8c9..7b6762cbd399 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -20,7 +20,10 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.util.List;
+
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
 
 /**
  * RegionScanner describes iterators over rows in an HRegion.
@@ -49,4 +52,50 @@ public interface RegionScanner extends InternalScanner {
    */
   public boolean reseek(byte[] row) throws IOException;
 
+  /**
+   * @return The Scanner's MVCC readPt see {@link MultiVersionConsistencyControl}
+   */
+  public long getMvccReadPoint();
+
+  /**
+   * Grab the next row's worth of values with the default limit on the number of values
+   * to return.
+   * This is a special internal method to be called from coprocessor hooks to avoid expensive setup.
+   * Caller must set the thread's readpoint, start and close a region operation, an synchronize on the scanner object.
+   * See {@link #nextRaw(List, int, String)}
+   * @param result return output array
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean nextRaw(List<KeyValue> result, String metric) throws IOException;
+
+  /**
+   * Grab the next row's worth of values with a limit on the number of values
+   * to return.
+   * This is a special internal method to be called from coprocessor hooks to avoid expensive setup.
+   * Caller must set the thread's readpoint, start and close a region operation, an synchronize on the scanner object.
+   * Example:
+   * <code><pre>
+   * HRegion region = ...;
+   * RegionScanner scanner = ...
+   * MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint());
+   * region.startRegionOperation();
+   * try {
+   *   synchronized(scanner) {
+   *     ...
+   *     boolean moreRows = scanner.nextRaw(values);
+   *     ...
+   *   }
+   * } finally {
+   *   region.closeRegionOperation();
+   * }
+   * </pre></code>
+   * @param result return output array
+   * @param limit limit on row count to get
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean nextRaw(List<KeyValue> result, int limit, String metric) throws IOException;
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 1833aab9bd6b..3dcd8f12c77d 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -84,6 +84,18 @@ public boolean next(List<KeyValue> result, int limit, String metric)
       return delegate.next(result, limit, metric);
     }
 
+    @Override
+    public boolean nextRaw(List<KeyValue> result, int limit, String metric) 
+        throws IOException {
+      return delegate.nextRaw(result, limit, metric);
+    }
+
+    @Override
+    public boolean nextRaw(List<KeyValue> result, String metric) 
+        throws IOException {
+      return delegate.nextRaw(result, metric);
+    }
+
     @Override
     public void close() throws IOException {
       delegate.close();
@@ -104,6 +116,10 @@ public boolean reseek(byte[] row) throws IOException {
       return false;
     }
 
+    @Override
+    public long getMvccReadPoint() {
+      return delegate.getMvccReadPoint();
+    }
   }
 
   public static class CoprocessorImpl extends BaseRegionObserver {

From 490bdbe32fce4c9afee5b1104cf1f657d8579754 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 11 Dec 2012 18:46:59 +0000
Subject: [PATCH 0618/1540] HBASE-7307 MetaReader.tableExists should not return
 false if the specified table regions has been split (Rajesh)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1420332 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/MetaReader.java      |  1 -
 .../TestSplitTransactionOnCluster.java        | 39 +++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index 1da3b27724a4..780ed0af1d48 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -432,7 +432,6 @@ public boolean visit(Result r) throws IOException {
           return true;
         }
         if (!isInsideTable(this.current, tableNameBytes)) return false;
-        if (this.current.isSplitParent()) return true;
         // Else call super and add this Result to the collection.
         super.visit(r);
         // Stop collecting regions from table after we get one.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index e777081607d5..1d3d1db9e33e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
@@ -727,6 +728,44 @@ public void run() {
     }
   }
 
+  @Test(timeout = 20000)
+  public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
+    final byte[] tableName = 
+        Bytes.toBytes("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
+    HRegionServer regionServer = null;
+    List<HRegion> regions = null;
+    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+    try {
+      // Create table then get the single region for our new table.
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor("cf"));
+      admin.createTable(htd);
+      HTable t = new HTable(cluster.getConfiguration(), tableName);
+      regions = cluster.getRegions(tableName);
+      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+      regionServer = cluster.getRegionServer(regionServerIndex);
+      insertData(tableName, admin, t);
+      // Turn off balancer so it doesn't cut in and mess up our placements.
+      cluster.getMaster().setCatalogJanitorEnabled(false);
+      boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
+          Bytes.toString(tableName));
+      assertEquals("The specified table should present.", true, tableExists);
+      SplitTransaction st = new SplitTransaction(regions.get(0), Bytes.toBytes("row2"));
+      try {
+        st.prepare();
+        st.createDaughters(regionServer, regionServer);
+      } catch (IOException e) {
+
+      }
+      tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
+          Bytes.toString(tableName));
+      assertEquals("The specified table should present.", true, tableExists);
+    } finally {
+      cluster.getMaster().setCatalogJanitorEnabled(true);
+      admin.close();
+    }
+  }
+  
   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
       InterruptedException {
     Put p = new Put(Bytes.toBytes("row1"));

From e4dbda711692cfdd96b0cb6b8af925f55758ec3a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 11 Dec 2012 23:26:49 +0000
Subject: [PATCH 0619/1540] HBASE-5258 Move coprocessors set out of RegionLoad
 (Sergey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1420482 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HServerLoad.java  | 58 +++---------
 .../hbase/regionserver/HRegionServer.java     | 13 ++-
 .../hbase/coprocessor/TestClassLoading.java   | 90 ++-----------------
 3 files changed, 28 insertions(+), 133 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
index f683ebed80e8..ffdbc6bf6515 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
@@ -60,30 +60,14 @@ public class HServerLoad extends VersionedWritable
   private int maxHeapMB = 0;
 
   // Regionserver-level coprocessors, e.g., WALObserver implementations.
-  // Region-level coprocessors, on the other hand, are stored inside RegionLoad
-  // objects.
-  private Set<String> coprocessors =
-      new TreeSet<String>();
+  private Set<String> coprocessors = new TreeSet<String>();
 
   /**
    * HBASE-4070: Improve region server metrics to report loaded coprocessors.
-   *
-   * @return Returns the set of all coprocessors on this
-   * regionserver, where this set is the union of the
-   * regionserver-level coprocessors on one hand, and all of the region-level
-   * coprocessors, on the other.
-   *
-   * We must iterate through all regions loaded on this regionserver to
-   * obtain all of the region-level coprocessors.
+   * @return the set of all the server-wide coprocessors on this regionserver
    */
-  public String[] getCoprocessors() {
-    TreeSet<String> returnValue = new TreeSet<String>(coprocessors);
-    for (Map.Entry<byte[], RegionLoad> rls: getRegionsLoad().entrySet()) {
-      for (String coprocessor: rls.getValue().getCoprocessors()) {
-        returnValue.add(coprocessor);
-      }
-    }
-    return returnValue.toArray(new String[0]);
+  public String[] getRsCoprocessors() {
+    return coprocessors.toArray(new String[0]);
   }
 
   /** per-region load metrics */
@@ -145,10 +129,6 @@ public byte getVersion() {
      */
     private int totalStaticBloomSizeKB;
 
-    // Region-level coprocessors.
-    Set<String> coprocessors =
-        new TreeSet<String>();
-
     /**
      * Constructor, for Writable
      */
@@ -168,7 +148,6 @@ public RegionLoad() {
      * @param writeRequestsCount
      * @param totalCompactingKVs
      * @param currentCompactedKVs
-     * @param coprocessors
      */
     public RegionLoad(final byte[] name, final int stores,
         final int storefiles, final int storeUncompressedSizeMB,
@@ -177,8 +156,7 @@ public RegionLoad(final byte[] name, final int stores,
         final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
         final int totalStaticBloomSizeKB,
         final long readRequestsCount, final long writeRequestsCount,
-        final long totalCompactingKVs, final long currentCompactedKVs,
-        final Set<String> coprocessors) {
+        final long totalCompactingKVs, final long currentCompactedKVs) {
       this.name = name;
       this.stores = stores;
       this.storefiles = storefiles;
@@ -193,12 +171,6 @@ public RegionLoad(final byte[] name, final int stores,
       this.writeRequestsCount = writeRequestsCount;
       this.totalCompactingKVs = totalCompactingKVs;
       this.currentCompactedKVs = currentCompactedKVs;
-      this.coprocessors = coprocessors;
-    }
-
-    // Getters
-    private String[] getCoprocessors() {
-      return coprocessors.toArray(new String[0]);
     }
 
     /**
@@ -400,9 +372,9 @@ private void readFields92(DataInput in) throws IOException {
       this.totalCompactingKVs = in.readLong();
       this.currentCompactedKVs = in.readLong();
       int coprocessorsSize = in.readInt();
-      coprocessors = new TreeSet<String>();
+      // Backward compatibility - there may be coprocessors in the region load, ignore them.
       for (int i = 0; i < coprocessorsSize; i++) {
-        coprocessors.add(in.readUTF());
+        in.readUTF();
       }
     }
     
@@ -431,9 +403,9 @@ public void readFields(DataInput in) throws IOException {
       this.totalCompactingKVs = WritableUtils.readVLong(in);
       this.currentCompactedKVs = WritableUtils.readVLong(in);
       int coprocessorsSize = WritableUtils.readVInt(in);
-      coprocessors = new TreeSet<String>();
+      // Backward compatibility - there may be coprocessors in the region load, ignore them.
       for (int i = 0; i < coprocessorsSize; i++) {
-        coprocessors.add(in.readUTF());
+        in.readUTF();
       }
     }
 
@@ -454,10 +426,9 @@ public void write(DataOutput out) throws IOException {
       WritableUtils.writeVInt(out, totalStaticBloomSizeKB);
       WritableUtils.writeVLong(out, totalCompactingKVs);
       WritableUtils.writeVLong(out, currentCompactedKVs);
-      WritableUtils.writeVInt(out, coprocessors.size());
-      for (String coprocessor: coprocessors) {
-        out.writeUTF(coprocessor);
-      }
+      // Backward compatibility - write out 0 as coprocessor count,
+      // we don't report region-level coprocessors anymore.
+      WritableUtils.writeVInt(out, 0);
     }
 
     /**
@@ -503,11 +474,6 @@ public String toString() {
       }
       sb = Strings.appendKeyValue(sb, "compactionProgressPct",
           compactionProgressPct);
-      String coprocessors = Arrays.toString(getCoprocessors());
-      if (coprocessors != null) {
-        sb = Strings.appendKeyValue(sb, "coprocessors",
-            Arrays.toString(getCoprocessors()));
-      }
       return sb.toString();
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b3181d61d129..5b17faa2e408 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -44,6 +44,7 @@
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -1096,8 +1097,7 @@ private HServerLoad.RegionLoad createRegionLoad(final HRegion r) {
         storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, rootIndexSizeKB,
         totalStaticIndexSizeKB, totalStaticBloomSizeKB,
         (int) r.readRequestsCount.get(), (int) r.writeRequestsCount.get(),
-        totalCompactingKVs, currentCompactedKVs,
-        r.getCoprocessorHost().getCoprocessors());
+        totalCompactingKVs, currentCompactedKVs);
   }
 
   /**
@@ -3776,8 +3776,13 @@ public List<HRegion> getOnlineRegions(byte[] tableName) {
 
   // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
   public String[] getCoprocessors() {
-    HServerLoad hsl = buildServerLoad();
-    return hsl == null? null: hsl.getCoprocessors();
+    TreeSet<String> coprocessors = new TreeSet<String>(
+        this.hlog.getCoprocessorHost().getCoprocessors());
+    Collection<HRegion> regions = getOnlineRegionsLocalContext();
+    for (HRegion region: regions) {
+      coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
+    }
+    return coprocessors.toArray(new String[0]);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index bea1fe1b4cb1..45da51ae2870 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -68,14 +68,6 @@ public class TestClassLoading {
 
   private static final String[] regionServerSystemCoprocessors =
       new String[]{
-      regionCoprocessor1.getSimpleName(),
-      regionServerCoprocessor.getSimpleName()
-  };
-
-  private static final String[] regionServerSystemAndUserCoprocessors =
-      new String[] {
-      regionCoprocessor1.getSimpleName(),
-      regionCoprocessor2.getSimpleName(),
       regionServerCoprocessor.getSimpleName()
   };
 
@@ -424,6 +416,8 @@ public void testClassLoadingFromLibDirInJar() throws Exception {
     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
 
     byte buffer[] = new byte[BUFFER_SIZE];
+    // TODO: code here and elsewhere in this file is duplicated w/TestClassFinder.
+    //       Some refactoring may be in order...
     // Open archive file
     FileOutputStream stream = new FileOutputStream(outerJarFile);
     JarOutputStream out = new JarOutputStream(stream, new Manifest());
@@ -433,7 +427,7 @@ public void testClassLoadingFromLibDirInJar() throws Exception {
       JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName());
       jarAdd.setTime(jarFile.lastModified());
       out.putNextEntry(jarAdd);
-  
+
       // Write file to archive
       FileInputStream in = new FileInputStream(jarFile);
       while (true) {
@@ -505,82 +499,12 @@ public void testClassLoadingFromLibDirInJar() throws Exception {
 
   @Test
   public void testRegionServerCoprocessorsReported() throws Exception {
-    // HBASE 4070: Improve region server metrics to report loaded coprocessors
-    // to master: verify that each regionserver is reporting the correct set of
-    // loaded coprocessors.
-
-    // We rely on the fact that getCoprocessors() will return a sorted
-    // display of the coprocessors' names, so for example, regionCoprocessor1's
-    // name "ColumnAggregationEndpoint" will appear before regionCoprocessor2's
-    // name "GenericEndpoint" because "C" is before "G" lexicographically.
+    // This was a test for HBASE-4070.
+    // We are removing coprocessors from region load in HBASE-5258.
+    // Therefore, this test now only checks system coprocessors.
 
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
-
-    // disable all user tables, if any are loaded.
-    for (HTableDescriptor htd: admin.listTables()) {
-      if (!htd.isMetaTable()) {
-        String tableName = htd.getNameAsString();
-        if (admin.isTableEnabled(tableName)) {
-          try {
-            admin.disableTable(htd.getNameAsString());
-          } catch (TableNotEnabledException e) {
-            // ignoring this exception for now : not sure why it's happening.
-          }
-        }
-      }
-    }
-
-    // should only be system coprocessors loaded at this point.
-    assertAllRegionServers(regionServerSystemCoprocessors,null);
-
-    // The next two tests enable and disable user tables to see if coprocessor
-    // load reporting changes as coprocessors are loaded and unloaded.
-    //
-
-    // Create a table.
-    // should cause regionCoprocessor2 to be loaded, since we've specified it
-    // for loading on any user table with USER_REGION_COPROCESSOR_CONF_KEY
-    // in setUpBeforeClass().
-    String userTable1 = "userTable1";
-    HTableDescriptor userTD1 = new HTableDescriptor(userTable1);
-    admin.createTable(userTD1);
-    waitForTable(userTD1.getName());
-
-    // table should be enabled now.
-    assertTrue(admin.isTableEnabled(userTable1));
-    assertAllRegionServers(regionServerSystemAndUserCoprocessors, userTable1);
-
-    // unload and make sure we're back to only system coprocessors again.
-    admin.disableTable(userTable1);
     assertAllRegionServers(regionServerSystemCoprocessors,null);
-
-    // create another table, with its own specified coprocessor.
-    String userTable2 = "userTable2";
-    HTableDescriptor htd2 = new HTableDescriptor(userTable2);
-
-    String userTableCP = "userTableCP";
-    File jarFile1 = buildCoprocessorJar(userTableCP);
-    htd2.addFamily(new HColumnDescriptor("myfamily"));
-    htd2.setValue("COPROCESSOR$1", jarFile1.toString() + "|" + userTableCP +
-      "|" + Coprocessor.PRIORITY_USER);
-    admin.createTable(htd2);
-    waitForTable(htd2.getName());
-    // table should be enabled now.
-    assertTrue(admin.isTableEnabled(userTable2));
-
-    ArrayList<String> existingCPsPlusNew =
-        new ArrayList<String>(Arrays.asList(regionServerSystemAndUserCoprocessors));
-    existingCPsPlusNew.add(userTableCP);
-    String[] existingCPsPlusNewArray = new String[existingCPsPlusNew.size()];
-    assertAllRegionServers(existingCPsPlusNew.toArray(existingCPsPlusNewArray),
-        userTable2);
-
-    admin.disableTable(userTable2);
-    assertTrue(admin.isTableDisabled(userTable2));
-
-    // we should be back to only system coprocessors again.
-    assertAllRegionServers(regionServerSystemCoprocessors, null);
-
   }
 
   /**
@@ -627,7 +551,7 @@ void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
       }
       boolean any_failed = false;
       for(Map.Entry<ServerName,HServerLoad> server: servers.entrySet()) {
-        actualCoprocessors = server.getValue().getCoprocessors();
+        actualCoprocessors = server.getValue().getRsCoprocessors();
         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
           LOG.debug("failed comparison: actual: " +
               Arrays.toString(actualCoprocessors) +

From 2214d47b43cd31013bf5e2a76448e8fc6d5f876c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 12 Dec 2012 07:00:04 +0000
Subject: [PATCH 0620/1540] HBASE-7328
 IntegrationTestRebalanceAndKillServersTargeted supercedes
 IntegrationTestRebalanceAndKillServers, remove

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1420545 13f79535-47bb-0310-9956-ffa450edef68
---
 ...ntegrationTestRebalanceAndKillServers.java | 100 ------------------
 1 file changed, 100 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java

diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java
deleted file mode 100644
index c6d761ec8df1..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/IntegrationTestRebalanceAndKillServers.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase;
-
-import java.io.IOException;
-
-import junit.framework.Assert;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.ChaosMonkey;
-import org.apache.hadoop.hbase.util.LoadTestTool;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.ChaosMonkey.Action;
-import org.apache.hadoop.hbase.util.ChaosMonkey.RestartActiveMaster;
-import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRandomRs;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import com.google.common.collect.Lists;
-
-/**
- * A system test which does large data ingestion and verify using {@link LoadTestTool},
- * while killing the region servers and the master(s) randomly. You can configure how long
- * should the load test run by using "hbase.IntegrationTestRebalanceAndKillServers  s.runtime"
- * configuration parameter.
- */
-@Category(IntegrationTests.class)
-public class IntegrationTestRebalanceAndKillServers extends IngestIntegrationTestBase {
-  private static final int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
-  private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000; // run for 5 min by default
-
-  private static final long KILL_SERVICE_EVERY_MS = 45 * 1000;
-  private static final int SERVER_PER_MASTER_KILL = 3;
-  private static final long KILL_SERVER_FOR_MS = 5 * 1000;
-  private static final long KILL_MASTER_FOR_MS = 100;
-
-  private static final long UNBALANCE_REGIONS_EVERY_MS = 30 * 1000;
-  /** @see ChaosMonkey.UnbalanceRegionsAction#UnbalanceRegionsAction(double, double) */
-  private static final double UNBALANCE_TO_FRC_OF_SERVERS = 0.5;
-  /** @see ChaosMonkey.UnbalanceRegionsAction#UnbalanceRegionsAction(double, double) */
-  private static final double UNBALANCE_FRC_OF_REGIONS = 0.5;
-
-  private static final long BALANCE_REGIONS_EVERY_MS = 10 * 1000;
-
-  private ChaosMonkey monkey;
-
-  @Before
-  @SuppressWarnings("unchecked")
-  public void setUp() throws Exception {
-    super.setUp(NUM_SLAVES_BASE);
-
-    ChaosMonkey.Policy killPolicy = new ChaosMonkey.PeriodicRandomActionPolicy(
-      KILL_SERVICE_EVERY_MS,
-      new Pair<Action,Integer>(new ChaosMonkey.RestartActiveMaster(KILL_MASTER_FOR_MS), 1),
-      new Pair<Action,Integer>(new ChaosMonkey.RestartRandomRs(KILL_SERVER_FOR_MS), SERVER_PER_MASTER_KILL));
-
-    ChaosMonkey.Policy unbalancePolicy = new ChaosMonkey.PeriodicRandomActionPolicy(
-      UNBALANCE_REGIONS_EVERY_MS,
-      new ChaosMonkey.UnbalanceRegionsAction(UNBALANCE_FRC_OF_REGIONS, UNBALANCE_TO_FRC_OF_SERVERS));
-
-    ChaosMonkey.Policy balancePolicy = new ChaosMonkey.PeriodicRandomActionPolicy(
-      BALANCE_REGIONS_EVERY_MS, new ChaosMonkey.ForceBalancerAction());
-
-    monkey = new ChaosMonkey(util, killPolicy, unbalancePolicy, balancePolicy);
-    monkey.start();
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    if (monkey != null) {
-      monkey.stop("tearDown");
-      monkey.waitForStop();
-    }
-    super.tearDown();
-  }
-
-  @Test
-  public void testDataIngest() throws Exception {
-    runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 100, 20);
-  }
-}

From d187757a876ad333709d3bc06031a8c7e1e290f6 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 13 Dec 2012 04:45:05 +0000
Subject: [PATCH 0621/1540] HBASE-7205 Coprocessor classloader is replicated
 for all regions in the HRegionServer (Ted Yu and Adrian Muraru)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1421076 13f79535-47bb-0310-9956-ffa450edef68
---
 .../coprocessor/CoprocessorClassLoader.java   | 27 ++++--
 .../hbase/coprocessor/CoprocessorHost.java    | 88 ++++++++++++++++---
 .../hbase/coprocessor/TestClassLoading.java   | 64 +++++++++++---
 3 files changed, 146 insertions(+), 33 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
index 7d0ba922d3c6..42d7dfe24d34 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
@@ -65,9 +65,13 @@ public class CoprocessorClassLoader extends URLClassLoader {
     "org.w3c",
     "org.xml",
     "sunw.",
-    // Hadoop/HBase:
-    "org.apache.hadoop",
+    // logging
+    "org.apache.commons.logging",
+    "org.apache.log4j",
     "com.hadoop",
+    // Hadoop/HBase/ZK:
+    "org.apache.hadoop",
+    "org.apache.zookeeper",
   };
   
   /**
@@ -80,7 +84,12 @@ public class CoprocessorClassLoader extends URLClassLoader {
       new Pattern[] {
     Pattern.compile("^[^-]+-default\\.xml$")
   };
-  
+
+  /**
+   * Parent classloader used to load any class not matching the exemption list.
+   */
+  private final ClassLoader parent;
+
   /**
    * Creates a CoprocessorClassLoader that loads classes from the given paths.
    * @param paths paths from which to load classes.
@@ -88,8 +97,12 @@ public class CoprocessorClassLoader extends URLClassLoader {
    */
   public CoprocessorClassLoader(List<URL> paths, ClassLoader parent) {
     super(paths.toArray(new URL[]{}), parent);
+    this.parent = parent;
+    if (parent == null) {
+      throw new IllegalArgumentException("No parent classloader!");
+    }
   }
-  
+
   @Override
   synchronized public Class<?> loadClass(String name) 
       throws ClassNotFoundException {
@@ -99,9 +112,9 @@ synchronized public Class<?> loadClass(String name)
         LOG.debug("Skipping exempt class " + name + 
             " - delegating directly to parent");
       }
-      return super.loadClass(name);
+      return parent.loadClass(name);
     }
-    
+
     // Check whether the class has already been loaded:
     Class<?> clasz = findLoadedClass(name);
     if (clasz != null) {
@@ -123,7 +136,7 @@ synchronized public Class<?> loadClass(String name)
           LOG.debug("Class " + name + " not found - delegating to parent");
         }
         try {
-          clasz = super.loadClass(name);
+          clasz = parent.loadClass(name);
         } catch (ClassNotFoundException e2) {
           // Class not found in this ClassLoader or in the parent ClassLoader
           // Log some debug output before rethrowing ClassNotFoundException
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 2fdaf6fd1443..b9f413a8ba95 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import com.google.common.collect.MapMaker;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -44,6 +46,7 @@
 import java.io.IOException;
 import java.net.URL;
 import java.util.*;
+import java.util.concurrent.ConcurrentMap;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 
@@ -72,6 +75,15 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
   protected String pathPrefix;
   protected volatile int loadSequence;
 
+  /*
+   * External classloaders cache keyed by external jar path.
+   * ClassLoader instance is stored as a weak-reference
+   * to allow GC'ing when no CoprocessorHost is using it
+   * (@see HBASE-7205)
+   */
+  static ConcurrentMap<Path, ClassLoader> classLoadersCache =
+      new MapMaker().concurrencyLevel(3).weakValues().makeMap();
+
   public CoprocessorHost() {
     pathPrefix = UUID.randomUUID().toString();
   }
@@ -159,14 +171,27 @@ public E load(Path path, String className, int priority,
     LOG.debug("Loading coprocessor class " + className + " with path " + 
         path + " and priority " + priority);
 
-    // Have we already loaded the class, perhaps from an earlier region open
-    // for the same table?
-    try {
-      implClass = getClass().getClassLoader().loadClass(className);
-    } catch (ClassNotFoundException e) {
-      LOG.info("Class " + className + " needs to be loaded from a file - " +
-          path + ".");
-      // go ahead to load from file system.
+    ClassLoader cl = null;
+    if (path == null) {
+      try {
+        implClass = getClass().getClassLoader().loadClass(className);
+      } catch (ClassNotFoundException e) {
+        throw new IOException("No jar path specified for " + className);
+      }
+    } else {
+      // Have we already loaded the class, perhaps from an earlier region open
+      // for the same table?
+      cl = classLoadersCache.get(path);
+      if (cl != null){
+        LOG.debug("Found classloader "+ cl + "for "+path.toString());
+        try {
+          implClass = cl.loadClass(className);
+        } catch (ClassNotFoundException e) {
+          LOG.info("Class " + className + " needs to be loaded from a file - " +
+              path + ".");
+          // go ahead to load from file system.
+        }
+      }
     }
 
     // If not, load
@@ -198,7 +223,8 @@ public E load(Path path, String className, int priority,
       // unsurprisingly wants URLs, not URIs; so we will use the deprecated
       // method which returns URLs for as long as it is available
       List<URL> paths = new ArrayList<URL>();
-      paths.add(new File(dst.toString()).getCanonicalFile().toURL());
+      URL url = new File(dst.toString()).getCanonicalFile().toURL();
+      paths.add(url);
 
       JarFile jarFile = new JarFile(dst.toString());
       Enumeration<JarEntry> entries = jarFile.entries();
@@ -215,17 +241,33 @@ public E load(Path path, String className, int priority,
       }
       jarFile.close();
 
-      ClassLoader cl = new CoprocessorClassLoader(paths,
-        this.getClass().getClassLoader());
-      Thread.currentThread().setContextClassLoader(cl);
+      cl = new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
+      // cache cp classloader as a weak value, will be GC'ed when no reference left
+      ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
+      if (prev != null) {
+        //lost update race, use already added classloader
+        cl = prev;
+      }
+
       try {
         implClass = cl.loadClass(className);
       } catch (ClassNotFoundException e) {
-        throw new IOException(e);
+        throw new IOException("Cannot load external coprocessor class " + className, e);
       }
     }
 
-    return loadInstance(implClass, priority, conf);
+    //load custom code for coprocessor
+    Thread currentThread = Thread.currentThread();
+    ClassLoader hostClassLoader = currentThread.getContextClassLoader();
+    try{
+      // switch temporarily to the thread classloader for custom CP
+      currentThread.setContextClassLoader(cl);
+      E cpInstance = loadInstance(implClass, priority, conf);
+      return cpInstance;
+    } finally {
+      // restore the fresh (host) classloader
+      currentThread.setContextClassLoader(hostClassLoader);
+    }
   }
 
   /**
@@ -301,6 +343,24 @@ public Coprocessor findCoprocessor(String className) {
     return null;
   }
 
+  /**
+   * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
+   * jar files.
+   * @return A set of ClassLoader instances
+   */
+  Set<ClassLoader> getExternalClassLoaders() {
+    Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
+    final ClassLoader systemClassLoader = this.getClass().getClassLoader();
+    for (E env : coprocessors) {
+      ClassLoader cl = env.getInstance().getClass().getClassLoader();
+      if (cl != systemClassLoader ){
+        //do not include system classloader
+        externalClassLoaders.add(cl);
+      }
+    }
+    return externalClassLoaders;
+  }
+
   /**
    * Find a coprocessor environment by class name
    * @param className the class name
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index 45da51ae2870..b89ef183121c 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -60,6 +60,7 @@ public class TestClassLoading {
   static final String cpName4 = "TestCP4";
   static final String cpName5 = "TestCP5";
   static final String cpName6 = "TestCP6";
+  static final String cpNameInvalid = "TestCPInvalid";
 
   private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
   private static Class<?> regionCoprocessor2 = GenericEndpoint.class;
@@ -199,16 +200,18 @@ public void testClassLoadingFromHDFS() throws Exception {
       new Path(fs.getUri().toString() + Path.SEPARATOR));
     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
       jarFile1.getName();
+    Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
     assertTrue("Copy jar file to HDFS failed.",
-      fs.exists(new Path(jarFileOnHDFS1)));
+      fs.exists(pathOnHDFS1));
     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
 
     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
         new Path(fs.getUri().toString() + Path.SEPARATOR));
     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
       jarFile2.getName();
+    Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
     assertTrue("Copy jar file to HDFS failed.",
-      fs.exists(new Path(jarFileOnHDFS2)));
+      fs.exists(pathOnHDFS2));
     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
 
     // create a table that references the coprocessors
@@ -220,41 +223,78 @@ public void testClassLoadingFromHDFS() throws Exception {
       // with configuration values
     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
+    // same jar but invalid class name (should fail to load this class)
+    htd.setValue("COPROCESSOR$3", jarFileOnHDFS2.toString() + "|" + cpNameInvalid +
+      "|" + Coprocessor.PRIORITY_USER);
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     if (admin.tableExists(tableName)) {
       admin.disableTable(tableName);
       admin.deleteTable(tableName);
     }
-    admin.createTable(htd);
+    CoprocessorHost.classLoadersCache.clear();
+    byte[] startKey = {10, 63};
+    byte[] endKey = {12, 43};
+    admin.createTable(htd, startKey, endKey, 4);
     waitForTable(htd.getName());
 
     // verify that the coprocessors were loaded
-    boolean found1 = false, found2 = false, found2_k1 = false,
-        found2_k2 = false, found2_k3 = false;
+    boolean foundTableRegion=false;
+    boolean found_invalid = true, found1 = true, found2 = true, found2_k1 = true,
+        found2_k2 = true, found2_k3 = true;
+    Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
+        new HashMap<HRegion, Set<ClassLoader>>();
     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
     for (HRegion region:
         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
       if (region.getRegionNameAsString().startsWith(tableName)) {
+        foundTableRegion = true;
         CoprocessorEnvironment env;
         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
-        if (env != null) {
-          found1 = true;
-        }
+        found1 = found1 && (env != null);
         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
+        found2 = found2 && (env != null);
         if (env != null) {
-          found2 = true;
           Configuration conf = env.getConfiguration();
-          found2_k1 = conf.get("k1") != null;
-          found2_k2 = conf.get("k2") != null;
-          found2_k3 = conf.get("k3") != null;
+          found2_k1 = found2_k1 && (conf.get("k1") != null);
+          found2_k2 = found2_k2 && (conf.get("k2") != null);
+          found2_k3 = found2_k3 && (conf.get("k3") != null);
+        } else {
+          found2_k1 = found2_k2 = found2_k3 = false;
         }
+        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpNameInvalid);
+        found_invalid = found_invalid && (env != null);
+
+        regionsActiveClassLoaders
+            .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
       }
     }
+
+    assertTrue("No region was found for table " + tableName, foundTableRegion);
     assertTrue("Class " + cpName1 + " was missing on a region", found1);
     assertTrue("Class " + cpName2 + " was missing on a region", found2);
+    //an invalid CP class name is defined for this table, validate that it is not loaded
+    assertFalse("Class " + cpNameInvalid + " was found on a region", found_invalid);
     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
+    // check if CP classloaders are cached
+    assertTrue(jarFileOnHDFS1 + " was not cached",
+      CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS1));
+    assertTrue(jarFileOnHDFS2 + " was not cached",
+      CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS2));
+    //two external jar used, should be one classloader per jar
+    assertEquals("The number of cached classloaders should be equal to the number" +
+      " of external jar files",
+      2, CoprocessorHost.classLoadersCache.size());
+    //check if region active classloaders are shared across all RS regions
+    Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
+        CoprocessorHost.classLoadersCache.values());
+    for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
+      assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
+            + " ClassLoader Cache:" + externalClassLoaders
+            + " Region ClassLoaders:" + regionCP.getValue(),
+            externalClassLoaders.containsAll(regionCP.getValue()));
+    }
   }
 
   @Test

From d7e711d77c6f3f504e0915a5d3b93b10c27bce7a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 13 Dec 2012 18:53:08 +0000
Subject: [PATCH 0622/1540] roll version to 0.94.4-snapshot

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1421423 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index ae22b125df67..c67fe64cefc8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.3</version>
+  <version>0.94.4-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 32007aba011b5222cd1c05a986cd44dd7cd4f949 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 13 Dec 2012 19:32:13 +0000
Subject: [PATCH 0623/1540] HBASE-7336 HFileBlock.readAtOffset does not work
 well with multiple threads

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1421439 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileBlock.java     | 38 +++++++++++--------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index d0904aaf8fb7..edc633f4adc6 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -29,6 +29,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1255,6 +1257,8 @@ private abstract static class AbstractFSReader implements FSReader {
     /** The path (if any) where this data is coming from */
     protected Path path;
 
+    private final Lock streamLock = new ReentrantLock();
+
     /** The default buffer size for our buffered streams */
     public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
 
@@ -1329,23 +1333,9 @@ protected int readAtOffset(FSDataInputStream istream,
             "-byte array at offset " + destOffset);
       }
 
-      if (pread) {
-        // Positional read. Better for random reads.
-        int extraSize = peekIntoNextBlock ? hdrSize : 0;
-
-        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
-        if (ret < size) {
-          throw new IOException("Positional read of " + size + " bytes " +
-              "failed at offset " + fileOffset + " (returned " + ret + ")");
-        }
-
-        if (ret == size || ret < size + extraSize) {
-          // Could not read the next block's header, or did not try.
-          return -1;
-        }
-      } else {
+      if (!pread && streamLock.tryLock()) {
         // Seek + read. Better for scanning.
-        synchronized (istream) {
+        try {
           istream.seek(fileOffset);
 
           long realOffset = istream.getPos();
@@ -1363,6 +1353,22 @@ protected int readAtOffset(FSDataInputStream istream,
           // Try to read the next block header.
           if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
             return -1;
+        } finally {
+          streamLock.unlock();
+        }
+      } else {
+        // Positional read. Better for random reads; or when the streamLock is already locked.
+        int extraSize = peekIntoNextBlock ? hdrSize : 0;
+
+        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
+        if (ret < size) {
+          throw new IOException("Positional read of " + size + " bytes " +
+              "failed at offset " + fileOffset + " (returned " + ret + ")");
+        }
+
+        if (ret == size || ret < size + extraSize) {
+          // Could not read the next block's header, or did not try.
+          return -1;
         }
       }
 

From 5059453ef6077ff788231e5083c0fe2f7fbd7db6 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Thu, 13 Dec 2012 19:45:42 +0000
Subject: [PATCH 0624/1540] HBASE-7341 Deprecate RowLocks in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1421447 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/UnknownRowLockException.java |  1 +
 .../apache/hadoop/hbase/client/Delete.java    | 20 +++++
 .../org/apache/hadoop/hbase/client/Get.java   |  1 +
 .../hadoop/hbase/client/HTableInterface.java  |  2 +
 .../hadoop/hbase/client/HTablePool.java       |  6 ++
 .../apache/hadoop/hbase/client/Increment.java |  4 +
 .../apache/hadoop/hbase/client/Mutation.java  |  2 +
 .../org/apache/hadoop/hbase/client/Put.java   |  3 +
 .../apache/hadoop/hbase/client/RowLock.java   |  1 +
 .../hbase/coprocessor/CoprocessorHost.java    |  6 ++
 .../hadoop/hbase/regionserver/HRegion.java    | 79 +++++++++++++++++++
 .../hbase/regionserver/HRegionServer.java     |  7 ++
 .../hbase/rest/client/RemoteHTable.java       |  6 ++
 13 files changed, 138 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
index 8ca50a9d8079..675e6e2b5cdc 100644
--- a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
+++ b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
@@ -22,6 +22,7 @@
 
 /**
  * Thrown if a region server is passed an unknown row lock id
+ * @deprecated row locks are deprecated (and thus so our associated exceptions).
  */
 public class UnknownRowLockException extends DoNotRetryIOException {
   private static final long serialVersionUID = 993179627856392526L;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index a806f8a29d07..26986e7f048e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -85,6 +85,25 @@ public Delete(byte [] row) {
     this(row, HConstants.LATEST_TIMESTAMP, null);
   }
 
+  /**
+   * Create a Delete operation for the specified row and timestamp.<p>
+   *
+   * If no further operations are done, this will delete all columns in all
+   * families of the specified row with a timestamp less than or equal to the
+   * specified timestamp.<p>
+   *
+   * This timestamp is ONLY used for a delete row operation.  If specifying
+   * families or columns, you must specify each timestamp individually.
+   * @param row row key
+   * @param timestamp maximum version timestamp (only for delete row)
+   * @param rowLock previously acquired row lock, or null
+   * @deprecated {@link RowLock} is deprecated, use #de
+   */
+  public Delete(byte [] row, long timestamp) {
+    this.row = row;
+    this.ts = timestamp;
+  }
+
   /**
    * Create a Delete operation for the specified row and timestamp, using
    * an optional row lock.<p>
@@ -98,6 +117,7 @@ public Delete(byte [] row) {
    * @param row row key
    * @param timestamp maximum version timestamp (only for delete row)
    * @param rowLock previously acquired row lock, or null
+   * @deprecated {@link RowLock} is deprecated, use {@link #Delete(byte[], long)}.
    */
   public Delete(byte [] row, long timestamp, RowLock rowLock) {
     this.row = row;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index dc83d1d729f7..0fb68385cac7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -98,6 +98,7 @@ public Get(byte [] row) {
    * all columns in all families of the specified row.
    * @param row row key
    * @param rowLock previously acquired row lock, or null
+   * @deprecated {@link RowLock} is deprecated, use {@link #Get(byte[])}.
    */
   public Get(byte [] row, RowLock rowLock) {
     this.row = row;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
index 007f0a38e553..404567b8c931 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
@@ -380,6 +380,7 @@ long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
    * @throws IOException if a remote or network exception occurs.
    * @see RowLock
    * @see #unlockRow
+   * @deprecated {@link RowLock} and associated operations are deprecated
    */
   RowLock lockRow(byte[] row) throws IOException;
 
@@ -390,6 +391,7 @@ long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
    * @throws IOException if a remote or network exception occurs.
    * @see RowLock
    * @see #unlockRow
+   * @deprecated {@link RowLock} and associated operations are deprecated
    */
   void unlockRow(RowLock rl) throws IOException;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 398dc529fcf3..698241bc1f22 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -458,11 +458,17 @@ public void close() throws IOException {
       returnTable(table);
     }
 
+    /**
+     * @deprecated {@link RowLock} and associated operations are deprecated
+     */
     @Override
     public RowLock lockRow(byte[] row) throws IOException {
       return table.lockRow(row);
     }
 
+    /**
+     * @deprecated {@link RowLock} and associated operations are deprecated
+     */
     @Override
     public void unlockRow(RowLock rl) throws IOException {
       table.unlockRow(rl);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Increment.java b/src/main/java/org/apache/hadoop/hbase/client/Increment.java
index ef0b65458fd4..52fac0479ed8 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Increment.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Increment.java
@@ -73,6 +73,8 @@ public Increment(byte [] row) {
    * At least one column must be incremented.
    * @param row row key
    * @param rowLock previously acquired row lock, or null
+   * @deprecated {@link RowLock} and associated operations are deprecated,
+   * use {@link #Increment(byte[])}
    */
   public Increment(byte [] row, RowLock rowLock) {
     this.row = row;
@@ -114,6 +116,7 @@ public Increment addColumn(byte [] family, byte [] qualifier, long amount) {
   /**
    * Method for retrieving the increment's RowLock
    * @return RowLock
+   * @deprecated {@link RowLock} and associated operations are deprecated
    */
   public RowLock getRowLock() {
     return new RowLock(this.row, this.lockId);
@@ -122,6 +125,7 @@ public RowLock getRowLock() {
   /**
    * Method for retrieving the increment's lockId
    * @return lockId
+   * @deprecated {@link RowLock} and associated operations are deprecated
    */
   public long getLockId() {
     return this.lockId;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 79a6d1fd7b2f..c25fffc26af9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -164,6 +164,7 @@ public int compareTo(final Row d) {
   /**
    * Method for retrieving the delete's RowLock
    * @return RowLock
+   * @deprecated {@link RowLock} and associated operations are deprecated
    */
   public RowLock getRowLock() {
     return new RowLock(this.row, this.lockId);
@@ -173,6 +174,7 @@ public RowLock getRowLock() {
    * Method for retrieving the delete's lock ID.
    *
    * @return The lock ID.
+   * @deprecated {@link RowLock} and associated operations are deprecated
    */
   public long getLockId() {
   return this.lockId;
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index 1ec81ff1110a..4596e57584b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -67,6 +67,7 @@ public Put(byte [] row) {
    * Create a Put operation for the specified row, using an existing row lock.
    * @param row row key
    * @param rowLock previously acquired row lock, or null
+   * @deprecated {@link RowLock} and associated operations are deprecated, use {@link #Put(byte[])}
    */
   public Put(byte [] row, RowLock rowLock) {
       this(row, HConstants.LATEST_TIMESTAMP, rowLock);
@@ -87,6 +88,8 @@ public Put(byte[] row, long ts) {
    * @param row row key
    * @param ts timestamp
    * @param rowLock previously acquired row lock, or null
+   * @deprecated {@link RowLock} and associated operations are deprecated,
+   * use {@link #Put(byte[], long)}
    */
   public Put(byte [] row, long ts, RowLock rowLock) {
     if(row == null || row.length > HConstants.MAX_ROW_LENGTH) {
diff --git a/src/main/java/org/apache/hadoop/hbase/client/RowLock.java b/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
index 56b0787b24d6..5888ec844096 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
@@ -21,6 +21,7 @@
 
 /**
  * Holds row name and lock id.
+ * @deprecated {@link RowLock} and associated operations are deprecated.
  */
 public class RowLock {
   private byte [] row = null;
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index b9f413a8ba95..44c497b5c204 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -535,11 +535,17 @@ public byte[] getTableName() {
         return tableName;
       }
 
+      /**
+       * @deprecated {@link RowLock} and associated operations are deprecated.
+       */
       public RowLock lockRow(byte[] row) throws IOException {
         throw new RuntimeException(
           "row locking is not allowed within the coprocessor environment");
       }
 
+      /**
+       * @deprecated {@link RowLock} and associated operations are deprecated.
+       */
       public void unlockRow(RowLock rl) throws IOException {
         throw new RuntimeException(
           "row locking is not allowed within the coprocessor environment");
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 2ad3f0435c07..f87432f4df1e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1722,11 +1722,23 @@ private void prepareDelete(Delete delete) throws IOException {
   //////////////////////////////////////////////////////////////////////////////
   // set() methods for client use.
   //////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * @param delete delete object
+   * @param writeToWAL append to the write ahead lock or not
+   * @throws IOException read exceptions
+   */
+  public void delete(Delete delete, boolean writeToWAL)
+  throws IOException {
+    delete(delete, null, writeToWAL);
+  }
+
   /**
    * @param delete delete object
    * @param lockid existing lock id, or null for grab a lock
    * @param writeToWAL append to the write ahead lock or not
    * @throws IOException read exceptions
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
    */
   public void delete(Delete delete, Integer lockid, boolean writeToWAL)
   throws IOException {
@@ -1901,6 +1913,7 @@ public void put(Put put, boolean writeToWAL) throws IOException {
    * @param put
    * @param lockid
    * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
    */
   public void put(Put put, Integer lockid) throws IOException {
     this.put(put, lockid, put.getWriteToWAL());
@@ -1913,6 +1926,7 @@ public void put(Put put, Integer lockid) throws IOException {
    * @param lockid
    * @param writeToWAL
    * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
    */
   public void put(Put put, Integer lockid, boolean writeToWAL)
   throws IOException {
@@ -2386,6 +2400,24 @@ private long doMiniBatchMutation(
   //the getting of the lock happens before, so that you would just pass it into
   //the methods. So in the case of checkAndMutate you could just do lockRow,
   //get, put, unlockRow or something
+ /**
+  *
+  * @param row
+  * @param family
+  * @param qualifier
+  * @param compareOp
+  * @param comparator
+  * @param writeToWAL
+  * @throws IOException
+  * @return true if the new put was execute, false otherwise
+  */
+ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
+     CompareOp compareOp, WritableByteArrayComparable comparator, Writable w,
+     boolean writeToWAL)
+ throws IOException {
+   return checkAndMutate(row, family, qualifier, compareOp, comparator, w, null, writeToWAL);
+ }
+  
   /**
    *
    * @param row
@@ -2397,6 +2429,7 @@ private long doMiniBatchMutation(
    * @param writeToWAL
    * @throws IOException
    * @return true if the new put was execute, false otherwise
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
    */
   public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
       CompareOp compareOp, WritableByteArrayComparable comparator, Writable w,
@@ -4315,11 +4348,21 @@ private static void listPaths(FileSystem fs, Path dir) throws IOException {
   //
   // HBASE-880
   //
+  /**
+   * @param get get object
+   * @return result
+   * @throws IOException read exceptions
+   */
+  public Result get(final Get get) throws IOException {
+    return get(get, null);
+  }
+
   /**
    * @param get get object
    * @param lockid existing lock id, or null for no previous lock
    * @return result
    * @throws IOException read exceptions
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
    */
   public Result get(final Get get, final Integer lockid) throws IOException {
     checkRow(get.getRow(), "Get");
@@ -4564,6 +4607,23 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
 
   // TODO: There's a lot of boiler plate code identical
   // to increment... See how to better unify that.
+
+  /**
+  *
+  * Perform one or more append operations on a row.
+  * <p>
+  * Appends performed are done under row lock but reads do not take locks out
+  * so this can be seen partially complete by gets and scans.
+  *
+  * @param append
+  * @param writeToWAL
+  * @return new keyvalues after increment
+  * @throws IOException
+  */
+ public Result append(Append append, boolean writeToWAL)
+     throws IOException {
+   return append(append, null, writeToWAL);
+ }
   /**
    *
    * Perform one or more append operations on a row.
@@ -4576,6 +4636,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
    * @param writeToWAL
    * @return new keyvalues after increment
    * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
    */
   public Result append(Append append, Integer lockid, boolean writeToWAL)
       throws IOException {
@@ -4714,6 +4775,22 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
     return append.isReturnResults() ? new Result(allKVs) : null;
   }
 
+  /**
+  *
+  * Perform one or more increment operations on a row.
+  * <p>
+  * Increments performed are done under row lock but reads do not take locks
+  * out so this can be seen partially complete by gets and scans.
+  * @param increment
+  * @param writeToWAL
+  * @return new keyvalues after increment
+  * @throws IOException
+  */
+  public Result increment(Increment increment, boolean writeToWAL)
+  throws IOException {
+    return increment(increment, null, writeToWAL);
+  }
+
   /**
    *
    * Perform one or more increment operations on a row.
@@ -4725,6 +4802,8 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
    * @param writeToWAL
    * @return new keyvalues after increment
    * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+
    */
   public Result increment(Increment increment, Integer lockid,
       boolean writeToWAL)
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5b17faa2e408..6f527582548e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -97,6 +97,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowLock;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.coprocessor.Exec;
@@ -2599,6 +2600,9 @@ public int delete(final byte[] regionName, final List<Delete> deletes)
     return -1;
   }
 
+  /**
+   * @deprecated {@link RowLock} and associated operations are deprecated.
+   */
   public long lockRow(byte[] regionName, byte[] row) throws IOException {
     checkOpen();
     NullPointerException npe = null;
@@ -2658,6 +2662,9 @@ Integer getLockFromId(long lockId) throws IOException {
     return rl;
   }
 
+  /**
+   * @deprecated {@link RowLock} and associated operations are deprecated.
+   */
   @Override
   @QosPriority(priority=HConstants.HIGH_QOS)
   public void unlockRow(byte[] regionName, long lockId) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index ba638982411b..52e57bad3cd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -575,10 +575,16 @@ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
     throw new IOException("getRowOrBefore not supported");
   }
 
+  /**
+   * @deprecated {@link RowLock} and associated operations are deprecated
+   */
   public RowLock lockRow(byte[] row) throws IOException {
     throw new IOException("lockRow not implemented");
   }
 
+  /**
+   * @deprecated {@link RowLock} and associated operations are deprecated
+   */
   public void unlockRow(RowLock rl) throws IOException {
     throw new IOException("unlockRow not implemented");
   }

From 99b9f38fc4a00538964cb8a0c263eaf59020c759 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 13 Dec 2012 19:54:28 +0000
Subject: [PATCH 0625/1540] HBASE-7343 Fix flaky condition for
 TestDrainingServer (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1421455 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/TestDrainingServer.java      | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java b/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
index a1992c3d5347..4679cf27e351 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
@@ -54,13 +54,14 @@ public class TestDrainingServer {
   private static final byte [] TABLENAME = Bytes.toBytes("t");
   private static final byte [] FAMILY = Bytes.toBytes("f");
   private static final int COUNT_OF_REGIONS = HBaseTestingUtility.KEYS.length;
+  private static final int NB_SLAVES = 5;
 
   /**
    * Spin up a cluster with a bunch of regions on it.
    */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.startMiniCluster(5);
+    TEST_UTIL.startMiniCluster(NB_SLAVES);
     TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
     HTableDescriptor htd = new HTableDescriptor(TABLENAME);
@@ -73,14 +74,25 @@ public static void setUpBeforeClass() throws Exception {
       createTableDescriptor(fs, FSUtils.getRootDir(TEST_UTIL.getConfiguration()), htd);
     // Assign out the regions we just created.
     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+    MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
     admin.disableTable(TABLENAME);
     admin.enableTable(TABLENAME);
-    ZKAssign.blockUntilNoRIT(zkw);
-    // Assert that every regionserver has some regions on it.
-    MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
-    for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
-      HRegionServer hrs = cluster.getRegionServer(i);
-      Assert.assertFalse(hrs.getOnlineRegions().isEmpty());
+    boolean ready = false;
+    while (!ready) {
+      ZKAssign.blockUntilNoRIT(zkw);
+      // Assert that every regionserver has some regions on it, else invoke the balancer.
+      ready = true;
+      for (int i = 0; i < NB_SLAVES; i++) {
+        HRegionServer hrs = cluster.getRegionServer(i);
+        if (hrs.getOnlineRegions().isEmpty()) {
+          ready = false;
+          break;
+        }
+      }
+      if (!ready) {
+        admin.balancer();
+        Thread.sleep(100);
+      }
     }
   }
 

From 04ad41169ff8777e6c18d0577b1d6c01fbbe250b Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 15 Dec 2012 17:34:32 +0000
Subject: [PATCH 0626/1540] HBASE-7197. Add multi get to RemoteHTable (Elliott
 Clark)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1422297 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/rest/client/RemoteHTable.java       | 104 ++++++++++++++----
 .../hbase/rest/client/TestRemoteTable.java    |  39 +++++++
 2 files changed, 119 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index 52e57bad3cd3..45c41bd2d676 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -142,6 +142,29 @@ protected String buildRowSpec(final byte[] row, final Map familyMap,
     return sb.toString();
   }
 
+  protected String buildMultiRowSpec(final byte[][] rows, int maxVersions) {
+    StringBuilder sb = new StringBuilder();
+    sb.append('/');
+    sb.append(Bytes.toStringBinary(name));
+    sb.append("/multiget/");
+    if (rows == null || rows.length == 0) {
+      return sb.toString();
+    }
+    sb.append("?");
+    for(int i=0; i<rows.length; i++) {
+      byte[] rk = rows[i];
+      if (i != 0) {
+        sb.append('&');
+      }
+      sb.append("row=");
+      sb.append(Bytes.toStringBinary(rk));
+    }
+    sb.append("&v=");
+    sb.append(maxVersions);
+
+    return sb.toString();
+  }
+
   protected Result[] buildResultFromModel(final CellSetModel model) {
     List<Result> results = new ArrayList<Result>();
     for (RowModel row: model.getRows()) {
@@ -267,30 +290,68 @@ public Result get(Get get) throws IOException {
     if (get.getFilter() != null) {
       LOG.warn("filters not supported on gets");
     }
+    Result[] results = getResults(spec);
+    if (results.length > 0) {
+      if (results.length > 1) {
+        LOG.warn("too many results for get (" + results.length + ")");
+      }
+      return results[0];
+    } else {
+      return new Result();
+    }
+  }
+
+  public Result[] get(List<Get> gets) throws IOException {
+    byte[][] rows = new byte[gets.size()][];
+    int maxVersions = 1;
+    int count = 0;
+
+    for (Get g : gets) {
+
+      if (count == 0) {
+        maxVersions = g.getMaxVersions();
+      } else if (g.getMaxVersions() != maxVersions) {
+        LOG.warn("MaxVersions on Gets do not match, using the first in the list ("
+            + maxVersions +")");
+      }
+
+      if (g.getFilter() != null) {
+        LOG.warn("filters not supported on gets");
+      }
+
+      rows[count] = g.getRow();
+      count++;
+    }
+
+    String spec = buildMultiRowSpec(rows, maxVersions);
+
+    return getResults(spec);
+  }
+
+  private Result[] getResults(String spec) throws IOException {
     for (int i = 0; i < maxRetries; i++) {
       Response response = client.get(spec, Constants.MIMETYPE_PROTOBUF);
       int code = response.getCode();
       switch (code) {
-      case 200:
-        CellSetModel model = new CellSetModel();
-        model.getObjectFromMessage(response.getBody());
-        Result[] results = buildResultFromModel(model);
-        if (results.length > 0) {
-          if (results.length > 1) {
-            LOG.warn("too many results for get (" + results.length + ")");
+        case 200:
+          CellSetModel model = new CellSetModel();
+          model.getObjectFromMessage(response.getBody());
+          Result[] results = buildResultFromModel(model);
+          if (results.length > 0) {
+            return results;
           }
-          return results[0];
-        }
-        // fall through
-      case 404:
-        return new Result();
-      case 509:
-        try {
-          Thread.sleep(sleepTime);
-        } catch (InterruptedException e) { }
-        break;
-      default:
-        throw new IOException("get request returned " + code);
+          // fall through
+        case 404:
+          return new Result[0];
+
+        case 509:
+          try {
+            Thread.sleep(sleepTime);
+          } catch (InterruptedException e) {
+          }
+          break;
+        default:
+          throw new IOException("get request returned " + code);
       }
     }
     throw new IOException("get request timed out");
@@ -695,11 +756,6 @@ public Object[] batch(List<? extends Row> actions) throws IOException {
     throw new IOException("batch not supported");
   }
 
-  @Override
-  public Result[] get(List<Get> gets) throws IOException {
-    throw new IOException("get(List<Get>) not supported");
-  }
-
   @Override
   public <T extends CoprocessorProtocol> T coprocessorProxy(Class<T> protocol,
       byte[] row) {
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index 7b7a67063f4b..f6265ac598a1 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -221,6 +221,45 @@ public void testGet() throws IOException {
     assertEquals(2, count);
   }
 
+  @Test
+  public void testMultiGet() throws Exception {
+    ArrayList<Get> gets = new ArrayList<Get>();
+    gets.add(new Get(ROW_1));
+    gets.add(new Get(ROW_2));
+    Result[] results = remoteTable.get(gets);
+    assertNotNull(results);
+    assertEquals(2, results.length);
+    assertEquals(1, results[0].size());
+    assertEquals(2, results[1].size());
+
+    //Test Versions
+    gets = new ArrayList<Get>();
+    Get g = new Get(ROW_1);
+    g.setMaxVersions(3);
+    gets.add(g);
+    gets.add(new Get(ROW_2));
+    results = remoteTable.get(gets);
+    assertNotNull(results);
+    assertEquals(2, results.length);
+    assertEquals(1, results[0].size());
+    assertEquals(3, results[1].size());
+
+    //404
+    gets = new ArrayList<Get>();
+    gets.add(new Get(Bytes.toBytes("RESALLYREALLYNOTTHERE")));
+    results = remoteTable.get(gets);
+    assertNotNull(results);
+    assertEquals(0, results.length);
+
+    gets = new ArrayList<Get>();
+    gets.add(new Get(Bytes.toBytes("RESALLYREALLYNOTTHERE")));
+    gets.add(new Get(ROW_1));
+    gets.add(new Get(ROW_2));
+    results = remoteTable.get(gets);
+    assertNotNull(results);
+    assertEquals(0, results.length);
+  }
+
   @Test
   public void testPut() throws IOException {
     Put put = new Put(ROW_3);

From c7c0b10b227d0248345b2bd41d384e4501dd1406 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 15 Dec 2012 18:40:52 +0000
Subject: [PATCH 0627/1540] HBASE-7359. [REST] 'accessToken' in RemoteHTable is
 vestigial

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1422310 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/rest/client/RemoteHTable.java       | 59 +++++++++----------
 .../hbase/rest/PerformanceEvaluation.java     | 14 +----
 .../hbase/rest/client/TestRemoteTable.java    |  2 +-
 3 files changed, 32 insertions(+), 43 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index 45c41bd2d676..e3be8e1a8970 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -72,7 +72,6 @@ public class RemoteHTable implements HTableInterface {
   final Client client;
   final Configuration conf;
   final byte[] name;
-  final String accessToken;
   final int maxRetries;
   final long sleepTime;
 
@@ -81,10 +80,6 @@ protected String buildRowSpec(final byte[] row, final Map familyMap,
       final long startTime, final long endTime, final int maxVersions) {
     StringBuffer sb = new StringBuffer();
     sb.append('/');
-    if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
-    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(row));
@@ -210,18 +205,32 @@ public RemoteHTable(Client client, String name) {
    * @param client
    * @param name
    * @param accessToken
+   * @deprecated accessToken is not used and will be removed
    */
+  @Deprecated
   public RemoteHTable(Client client, String name, String accessToken) {
     this(client, HBaseConfiguration.create(), Bytes.toBytes(name), accessToken);
   }
 
+  /**
+   * Constructor
+   * @param client
+   * @param conf
+   * @param name
+   */
+  public RemoteHTable(Client client, Configuration conf, String name) {
+    this(client, conf, Bytes.toBytes(name), null);
+  }
+
   /**
    * Constructor
    * @param client
    * @param conf
    * @param name
    * @param accessToken
+   * @deprecated accessToken is not used and will be removed
    */
+  @Deprecated
   public RemoteHTable(Client client, Configuration conf, String name,
       String accessToken) {
     this(client, conf, Bytes.toBytes(name), accessToken);
@@ -229,14 +238,28 @@ public RemoteHTable(Client client, Configuration conf, String name,
 
   /**
    * Constructor
+   * @param client
    * @param conf
+   * @param name
    */
+  public RemoteHTable(Client client, Configuration conf, byte[] name) {
+    this(client, conf, name, null);
+  }
+
+  /**
+   * Constructor
+   * @param client
+   * @param conf
+   * @param name
+   * @param accessToken
+   * @deprecated accessToken is not used and will be removed
+   */
+  @Deprecated
   public RemoteHTable(Client client, Configuration conf, byte[] name,
       String accessToken) {
     this.client = client;
     this.conf = conf;
     this.name = name;
-    this.accessToken = accessToken;
     this.maxRetries = conf.getInt("hbase.rest.client.max.retries", 10);
     this.sleepTime = conf.getLong("hbase.rest.client.sleep", 1000);
   }
@@ -252,10 +275,6 @@ public Configuration getConfiguration() {
   public HTableDescriptor getTableDescriptor() throws IOException {
     StringBuilder sb = new StringBuilder();
     sb.append('/');
-    if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
-    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append("schema");
@@ -367,10 +386,6 @@ public void put(Put put) throws IOException {
     CellSetModel model = buildModelFromPut(put);
     StringBuilder sb = new StringBuilder();
     sb.append('/');
-    if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');      
-    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(put.getRow()));
@@ -425,10 +440,6 @@ public void put(List<Put> puts) throws IOException {
     // build path for multiput
     StringBuilder sb = new StringBuilder();
     sb.append('/');
-    if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');      
-    }
     sb.append(Bytes.toStringBinary(name));
     sb.append("/$multiput"); // can be any nonexistent row
     for (int i = 0; i < maxRetries; i++) {
@@ -494,10 +505,6 @@ public Scanner(Scan scan) throws IOException {
       }
       StringBuffer sb = new StringBuffer();
       sb.append('/');
-      if (accessToken != null) {
-        sb.append(accessToken);
-        sb.append('/');
-      }
       sb.append(Bytes.toStringBinary(name));
       sb.append('/');
       sb.append("scanner");
@@ -658,10 +665,6 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
     CellSetModel model = buildModelFromPut(put);
     StringBuilder sb = new StringBuilder();
     sb.append('/');
-    if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
-    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(put.getRow()));
@@ -697,10 +700,6 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
     CellSetModel model = buildModelFromPut(put);
     StringBuilder sb = new StringBuilder();
     sb.append('/');
-    if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
-    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(row));
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java
index 23673c71198b..41893a4d40fd 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java
@@ -114,7 +114,6 @@ public class PerformanceEvaluation  {
 
   protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
   protected static Cluster cluster = new Cluster();
-  protected static String accessToken = null;
 
   volatile Configuration conf;
   private boolean nomapred = false;
@@ -449,8 +448,7 @@ public void setStatus(String msg) {
    */
   private boolean checkTable() throws IOException {
     HTableDescriptor tableDescriptor = getTableDescriptor();
-    RemoteAdmin admin =
-      new RemoteAdmin(new Client(cluster), conf, accessToken);
+    RemoteAdmin admin = new RemoteAdmin(new Client(cluster), conf);
     if (!admin.isTableAvailable(tableDescriptor.getName())) {
       admin.createTable(tableDescriptor);
       return true;
@@ -714,8 +712,7 @@ protected int getReportingPeriod() {
     }
     
     void testSetup() throws IOException {
-      this.table = new RemoteHTable(new Client(cluster), conf, tableName,
-        accessToken);
+      this.table = new RemoteHTable(new Client(cluster), conf, tableName);
     }
 
     void testTakedown()  throws IOException {
@@ -1133,7 +1130,6 @@ protected void printUsage(final String message) {
     System.err.println();
     System.err.println("Options:");
     System.err.println(" host          String. Specify Stargate endpoint.");
-    System.err.println(" token         String. API access token.");
     System.err.println(" rows          Integer. Rows each client runs. Default: One million");
     System.err.println(" rowsPerPut    Integer. Rows each Stargate (multi)Put. Default: 100");
     System.err.println(" nomapred      (Flag) Run multiple clients using threads " +
@@ -1208,12 +1204,6 @@ public int doCommandLine(final String[] args) {
           continue;
         }
 
-        final String token = "--token=";
-        if (cmd.startsWith(token)) {
-          accessToken = cmd.substring(token.length());
-          continue;
-        }
-
         Class<? extends Test> cmdClass = determineCommandClass(cmd);
         if (cmdClass != null) {
           getArgs(i + 1, args);
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index f6265ac598a1..a3ab9ec19ef0 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -101,7 +101,7 @@ public static void setUpBeforeClass() throws Exception {
     remoteTable = new RemoteHTable(
       new Client(new Cluster().add("localhost", 
           REST_TEST_UTIL.getServletPort())),
-        TEST_UTIL.getConfiguration(), TABLE, null);
+        TEST_UTIL.getConfiguration(), TABLE);
   }
 
   @AfterClass

From fac3d3b7f490aa6edbc3ebac10318f37ad92c59f Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Sun, 16 Dec 2012 10:25:38 +0000
Subject: [PATCH 0628/1540] HBASE-6175 TestFSUtils flaky on hdfs getFileStatus
 method

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1422503 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/TestFSUtils.java | 65 ++++++++++++-------
 1 file changed, 41 insertions(+), 24 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
index 339a1207ee9c..21ac5298c002 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -90,20 +90,26 @@ private void WriteDataToHDFS(FileSystem fs, Path file, int dataSize)
       // given the default replication factor is 3, the same as the number of
       // datanodes; the locality index for each host should be 100%,
       // or getWeight for each host should be the same as getUniqueBlocksWeights
-      FileStatus status = fs.getFileStatus(testFile);
-      HDFSBlocksDistribution blocksDistribution =
-        FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
-      long uniqueBlocksTotalWeight =
-        blocksDistribution.getUniqueBlocksTotalWeight();
-      for (String host : hosts) {
-        long weight = blocksDistribution.getWeight(host);
-        assertTrue(uniqueBlocksTotalWeight == weight);
-      }
-    } finally {
+      final long maxTime = System.currentTimeMillis() + 2000;
+      boolean ok;
+      do {
+        ok = true;
+        FileStatus status = fs.getFileStatus(testFile);
+        HDFSBlocksDistribution blocksDistribution =
+          FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
+        long uniqueBlocksTotalWeight =
+          blocksDistribution.getUniqueBlocksTotalWeight();
+        for (String host : hosts) {
+          long weight = blocksDistribution.getWeight(host);
+          ok = (ok && uniqueBlocksTotalWeight == weight);
+        }
+      } while (!ok && System.currentTimeMillis() < maxTime);
+      assertTrue(ok);
+      } finally {
       htu.shutdownMiniDFSCluster();
     }
 
-    
+
     try {
       // set up a cluster with 4 nodes
       String hosts[] = new String[] { "host1", "host2", "host3", "host4" };
@@ -118,16 +124,22 @@ private void WriteDataToHDFS(FileSystem fs, Path file, int dataSize)
       // given the default replication factor is 3, we will have total of 9
       // replica of blocks; thus the host with the highest weight should have
       // weight == 3 * DEFAULT_BLOCK_SIZE
-      FileStatus status = fs.getFileStatus(testFile);
-      HDFSBlocksDistribution blocksDistribution =
-        FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
-      long uniqueBlocksTotalWeight =
-        blocksDistribution.getUniqueBlocksTotalWeight();
-      
-      String tophost = blocksDistribution.getTopHosts().get(0);
-      long weight = blocksDistribution.getWeight(tophost);
+      final long maxTime = System.currentTimeMillis() + 2000;
+      long weight;
+      long uniqueBlocksTotalWeight;
+      do {
+        FileStatus status = fs.getFileStatus(testFile);
+        HDFSBlocksDistribution blocksDistribution =
+          FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
+        uniqueBlocksTotalWeight = blocksDistribution.getUniqueBlocksTotalWeight();
+
+        String tophost = blocksDistribution.getTopHosts().get(0);
+        weight = blocksDistribution.getWeight(tophost);
+
+        // NameNode is informed asynchronously, so we may have a delay. See HBASE-6175
+      } while (uniqueBlocksTotalWeight != weight && System.currentTimeMillis() < maxTime);
       assertTrue(uniqueBlocksTotalWeight == weight);
-      
+
     } finally {
       htu.shutdownMiniDFSCluster();
     }
@@ -146,11 +158,16 @@ private void WriteDataToHDFS(FileSystem fs, Path file, int dataSize)
       
       // given the default replication factor is 3, we will have total of 3
       // replica of blocks; thus there is one host without weight
-      FileStatus status = fs.getFileStatus(testFile);
-      HDFSBlocksDistribution blocksDistribution =
-        FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
+      final long maxTime = System.currentTimeMillis() + 2000;
+      HDFSBlocksDistribution blocksDistribution;
+      do {
+        FileStatus status = fs.getFileStatus(testFile);
+        blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
+        // NameNode is informed asynchronously, so we may have a delay. See HBASE-6175
+      }
+      while (blocksDistribution.getTopHosts().size() != 3 && System.currentTimeMillis() < maxTime);
       assertEquals("Wrong number of hosts distributing blocks.", 3,
-          blocksDistribution.getTopHosts().size());
+        blocksDistribution.getTopHosts().size());
     } finally {
       htu.shutdownMiniDFSCluster();
     }

From 921ce8c5802ee3f7919849c5fc4a10540bf3725a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 17 Dec 2012 06:42:18 +0000
Subject: [PATCH 0629/1540] HBASE-7336 Revert due to OOMs on TestHFileBlock
 potentially caused by this.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1422767 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileBlock.java     | 38 ++++++++-----------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index edc633f4adc6..d0904aaf8fb7 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -29,8 +29,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1257,8 +1255,6 @@ private abstract static class AbstractFSReader implements FSReader {
     /** The path (if any) where this data is coming from */
     protected Path path;
 
-    private final Lock streamLock = new ReentrantLock();
-
     /** The default buffer size for our buffered streams */
     public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
 
@@ -1333,9 +1329,23 @@ protected int readAtOffset(FSDataInputStream istream,
             "-byte array at offset " + destOffset);
       }
 
-      if (!pread && streamLock.tryLock()) {
+      if (pread) {
+        // Positional read. Better for random reads.
+        int extraSize = peekIntoNextBlock ? hdrSize : 0;
+
+        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
+        if (ret < size) {
+          throw new IOException("Positional read of " + size + " bytes " +
+              "failed at offset " + fileOffset + " (returned " + ret + ")");
+        }
+
+        if (ret == size || ret < size + extraSize) {
+          // Could not read the next block's header, or did not try.
+          return -1;
+        }
+      } else {
         // Seek + read. Better for scanning.
-        try {
+        synchronized (istream) {
           istream.seek(fileOffset);
 
           long realOffset = istream.getPos();
@@ -1353,22 +1363,6 @@ protected int readAtOffset(FSDataInputStream istream,
           // Try to read the next block header.
           if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
             return -1;
-        } finally {
-          streamLock.unlock();
-        }
-      } else {
-        // Positional read. Better for random reads; or when the streamLock is already locked.
-        int extraSize = peekIntoNextBlock ? hdrSize : 0;
-
-        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
-        if (ret < size) {
-          throw new IOException("Positional read of " + size + " bytes " +
-              "failed at offset " + fileOffset + " (returned " + ret + ")");
-        }
-
-        if (ret == size || ret < size + extraSize) {
-          // Could not read the next block's header, or did not try.
-          return -1;
         }
       }
 

From 4ac8a085d57fb51c58c5d4c640395f6ccf7cc0ce Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 17 Dec 2012 18:34:45 +0000
Subject: [PATCH 0630/1540] HBASE-7336 Reapply, the OOMs were not caused by
 this.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423084 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileBlock.java     | 38 +++++++++++--------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index d0904aaf8fb7..edc633f4adc6 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -29,6 +29,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1255,6 +1257,8 @@ private abstract static class AbstractFSReader implements FSReader {
     /** The path (if any) where this data is coming from */
     protected Path path;
 
+    private final Lock streamLock = new ReentrantLock();
+
     /** The default buffer size for our buffered streams */
     public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
 
@@ -1329,23 +1333,9 @@ protected int readAtOffset(FSDataInputStream istream,
             "-byte array at offset " + destOffset);
       }
 
-      if (pread) {
-        // Positional read. Better for random reads.
-        int extraSize = peekIntoNextBlock ? hdrSize : 0;
-
-        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
-        if (ret < size) {
-          throw new IOException("Positional read of " + size + " bytes " +
-              "failed at offset " + fileOffset + " (returned " + ret + ")");
-        }
-
-        if (ret == size || ret < size + extraSize) {
-          // Could not read the next block's header, or did not try.
-          return -1;
-        }
-      } else {
+      if (!pread && streamLock.tryLock()) {
         // Seek + read. Better for scanning.
-        synchronized (istream) {
+        try {
           istream.seek(fileOffset);
 
           long realOffset = istream.getPos();
@@ -1363,6 +1353,22 @@ protected int readAtOffset(FSDataInputStream istream,
           // Try to read the next block header.
           if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
             return -1;
+        } finally {
+          streamLock.unlock();
+        }
+      } else {
+        // Positional read. Better for random reads; or when the streamLock is already locked.
+        int extraSize = peekIntoNextBlock ? hdrSize : 0;
+
+        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
+        if (ret < size) {
+          throw new IOException("Positional read of " + size + " bytes " +
+              "failed at offset " + fileOffset + " (returned " + ret + ")");
+        }
+
+        if (ret == size || ret < size + extraSize) {
+          // Could not read the next block's header, or did not try.
+          return -1;
         }
       }
 

From 8d80ab904e514224869c381a5bee1eccc6226a81 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 17 Dec 2012 19:42:57 +0000
Subject: [PATCH 0631/1540] HBASE-7342 Split operation without split key
 incorrectly finds the middle key in off-by-one error (Aleksandr Shulman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423112 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index d655d3d8e700..8611b11ac725 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -342,7 +342,7 @@ public byte[] midkey() throws IOException {
         midKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + keyLen);
       } else {
         // The middle of the root-level index.
-        midKey = blockKeys[(rootCount - 1) / 2];
+        midKey = blockKeys[rootCount / 2];
       }
 
       this.midKey.set(midKey);
@@ -1429,5 +1429,4 @@ public long getCumulativeNumKV(int i) {
   public static int getMaxChunkSize(Configuration conf) {
     return conf.getInt(MAX_CHUNK_SIZE_KEY, DEFAULT_MAX_CHUNK_SIZE);
   }
-
 }

From 5729d778950049e5c02789d368303790ab27fbf5 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 17 Dec 2012 20:07:33 +0000
Subject: [PATCH 0632/1540] HBASE-7338 Fix flaky condition for
 org.apache.hadoop.hbase.TestRegionRebalancing.testRebalanceOnRegionServerNumberChange

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423115 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/TestRegionRebalancing.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
index 5574b7f97ab7..ec04f4d8a651 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
@@ -218,9 +218,10 @@ private List<HRegionServer> getOnlineRegionServers() {
    * Wait until all the regions are assigned.
    */
   private void waitForAllRegionsAssigned() throws IOException {
-    while (getRegionCount() < 22) {
+    int totalRegions = HBaseTestingUtility.KEYS.length+2;
+    while (getRegionCount() < totalRegions) {
     // while (!cluster.getMaster().allRegionsAssigned()) {
-      LOG.debug("Waiting for there to be 22 regions, but there are " + getRegionCount() + " right now.");
+      LOG.debug("Waiting for there to be "+ totalRegions +" regions, but there are " + getRegionCount() + " right now.");
       try {
         Thread.sleep(200);
       } catch (InterruptedException e) {}

From 09ebf9f84fef6af31b4e38c89f5ab597a10466b1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 18 Dec 2012 06:03:31 +0000
Subject: [PATCH 0633/1540] HBASE-7371 Blocksize in TestHFileBlock is
 unintentionally small

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423284 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index 6456ccbee3db..bf6f064f7711 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -55,7 +55,6 @@
 import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.apache.hadoop.io.compress.Compressor;
 
 import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
@@ -719,7 +718,8 @@ private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
       }
       BlockType bt = BlockType.values()[blockTypeOrdinal];
       DataOutputStream dos = hbw.startWriting(bt);
-      for (int j = 0; j < rand.nextInt(500); ++j) {
+      int size = rand.nextInt(500);
+      for (int j = 0; j < size; ++j) {
         // This might compress well.
         dos.writeShort(i + 1);
         dos.writeInt(j + 1);

From 7cad84a970831178bd3082a08a54f2c73111ce7e Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 18 Dec 2012 17:55:55 +0000
Subject: [PATCH 0634/1540] HBASE-7376 Acquiring readLock does not apply
 timeout in HRegion#flushcache (binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423553 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f87432f4df1e..8aed783440f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1328,7 +1328,8 @@ public boolean flushcache() throws IOException {
     }
     MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
     status.setStatus("Acquiring readlock on region");
-    lock(lock.readLock());
+    // block waiting for the lock for flushing cache
+    lock.readLock().lock();
     try {
       if (this.closed.get()) {
         LOG.debug("Skipping flush on " + this + " because closed");

From 58f3f30cb362424ac43ac8b39c90a5bb53812f9b Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Tue, 18 Dec 2012 20:13:53 +0000
Subject: [PATCH 0635/1540] HBASE-6775 Use ZK.multi when available for
 HBASE-6710 0.92/0.94 compatibility fix

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423616 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |   3 +
 .../hbase/zookeeper/RecoverableZooKeeper.java |  61 +++++
 .../hadoop/hbase/zookeeper/ZKTable.java       |  30 ++-
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 244 +++++++++++++++++-
 src/main/resources/hbase-default.xml          |  11 +
 .../hadoop/hbase/zookeeper/TestZKTable.java   |  47 +++-
 .../hbase/zookeeper/TestZKTableReadOnly.java  |  38 ++-
 7 files changed, 393 insertions(+), 41 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index ba657e038c79..3c276de014a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -157,6 +157,9 @@ public enum OperationStatusCode {
   /** Default value for ZooKeeper session timeout */
   public static final int DEFAULT_ZK_SESSION_TIMEOUT = 180 * 1000;
 
+  /** Configuration key for whether to use ZK.multi */
+  public static final String ZOOKEEPER_USEMULTI = "hbase.zookeeper.useMulti";
+
   /** Parameter name for port region server listens on. */
   public static final String REGIONSERVER_PORT = "hbase.regionserver.port";
 
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 57ffe86a2c9e..3ca866ccc347 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -32,11 +33,16 @@
 import org.apache.zookeeper.AsyncCallback;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Op;
+import org.apache.zookeeper.OpResult;
 import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.proto.CreateRequest;
+import org.apache.zookeeper.proto.SetDataRequest;
 
 /**
  * A zookeeper that can handle 'recoverable' errors.
@@ -490,6 +496,61 @@ private String createSequential(String path, byte[] data,
     }
   }
 
+  /**
+   * Convert Iterable of {@link ZKOp} we got into the ZooKeeper.Op
+   * instances to actually pass to multi (need to do this in order to appendMetaData).
+   */
+  private Iterable<Op> prepareZKMulti(Iterable<Op> ops)
+  throws UnsupportedOperationException {
+    if(ops == null) return null;
+
+    List<Op> preparedOps = new LinkedList<Op>();
+    for (Op op : ops) {
+      if (op.getType() == ZooDefs.OpCode.create) {
+        CreateRequest create = (CreateRequest)op.toRequestRecord();
+        preparedOps.add(Op.create(create.getPath(), appendMetaData(create.getData()),
+          create.getAcl(), create.getFlags()));
+      } else if (op.getType() == ZooDefs.OpCode.delete) {
+        // no need to appendMetaData for delete
+        preparedOps.add(op);
+      } else if (op.getType() == ZooDefs.OpCode.setData) {
+        SetDataRequest setData = (SetDataRequest)op.toRequestRecord();
+        preparedOps.add(Op.setData(setData.getPath(), appendMetaData(setData.getData()),
+          setData.getVersion()));
+      } else {
+        throw new UnsupportedOperationException("Unexpected ZKOp type: " + op.getClass().getName());
+      }
+    }
+    return preparedOps;
+  }
+
+  /**
+   * Run multiple operations in a transactional manner. Retry before throwing exception
+   */
+  public List<OpResult> multi(Iterable<Op> ops)
+  throws KeeperException, InterruptedException {
+    RetryCounter retryCounter = retryCounterFactory.create();
+    Iterable<Op> multiOps = prepareZKMulti(ops);
+    while (true) {
+      try {
+        return zk.multi(multiOps);
+      } catch (KeeperException e) {
+        switch (e.code()) {
+          case CONNECTIONLOSS:
+          case SESSIONEXPIRED:
+          case OPERATIONTIMEOUT:
+            retryOrThrow(retryCounter, e, "multi");
+            break;
+
+          default:
+            throw e;
+        }
+      }
+      retryCounter.sleepUntilNextRetry();
+      retryCounter.useRetry();
+    }
+  }
+
   private String findPreviousSequentialNode(String path)
     throws KeeperException, InterruptedException {
     int lastSlashIdx = path.lastIndexOf('/');
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 2ac04e9ebcc6..41cd25fd7352 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -21,6 +21,7 @@
 
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -29,6 +30,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -228,16 +230,19 @@ private void setTableState(final String tableName, final TableState state)
       }
     }
     synchronized (this.cache) {
+      List<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
       if (settingToEnabled) {
-        ZKUtil.deleteNodeFailSilent(this.watcher, znode92);
+        ops.add(ZKUtilOp.deleteNodeFailSilent(znode92));
       }
       else {
-        ZKUtil.setData(this.watcher, znode92, Bytes.toBytes(state.toString()));
+        ops.add(ZKUtilOp.setData(znode92, Bytes.toBytes(state.toString())));
       }
-      // Set the current format znode after the 0.92 format znode.
+      // If not running multi-update either because of configuration or failure,
+      // set the current format znode after the 0.92 format znode.
       // This is so in the case of failure, the AssignmentManager is guaranteed to
       // see the state was not applied, since it uses the current format znode internally.
-      ZKUtil.setData(this.watcher, znode, Bytes.toBytes(state.toString()));
+      ops.add(ZKUtilOp.setData(znode, Bytes.toBytes(state.toString())));
+      ZKUtil.multiOrSequential(this.watcher, ops, true);
       this.cache.put(tableName, state);
     }
   }
@@ -292,13 +297,16 @@ private boolean isTableState(final String tableName, final TableState state) {
   public void setDeletedTable(final String tableName)
   throws KeeperException {
     synchronized (this.cache) {
-      ZKUtil.deleteNodeFailSilent(this.watcher,
-        ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName));
-      // Delete the current format znode after the 0.92 format znode.
-      // This is so in the case of failure, the AssignmentManager is guaranteed to
-      // see the table was not deleted, since it uses the current format znode internally.
-      ZKUtil.deleteNodeFailSilent(this.watcher,
-        ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName));
+      List<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
+      ops.add(ZKUtilOp.deleteNodeFailSilent(
+        ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName)));
+      // If not running multi-update either because of configuration or failure,
+      // delete the current format znode after the 0.92 format znode.  This is so in the case of
+      // failure, the AssignmentManager is guaranteed to see the table was not deleted, since it
+      // uses the current format znode internally.
+      ops.add(ZKUtilOp.deleteNodeFailSilent(
+        ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName)));
+      ZKUtil.multiOrSequential(this.watcher, ops, true);
       if (this.cache.remove(tableName) == null) {
         LOG.warn("Moving table " + tableName + " state to deleted but was " +
           "already deleted");
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index b95dd3a2dddf..0e9755be62dd 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -26,6 +26,8 @@
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
 
@@ -39,15 +41,22 @@
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
 import org.apache.zookeeper.AsyncCallback;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.Op;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.proto.CreateRequest;
+import org.apache.zookeeper.proto.DeleteRequest;
+import org.apache.zookeeper.proto.SetDataRequest;
 
 /**
  * Internal HBase utility class for ZooKeeper.
@@ -249,9 +258,6 @@ public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
   /**
    * Check if the specified node exists.  Sets no watches.
    *
-   * Returns true if node exists, false if not.  Returns an exception if there
-   * is an unexpected zookeeper exception.
-   *
    * @param zkw zk reference
    * @param znode path of node to watch
    * @return version of the node if it exists, -1 if does not exist
@@ -700,7 +706,13 @@ public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
    */
   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
   throws KeeperException, KeeperException.NoNodeException {
-    setData(zkw, znode, data, -1);
+    setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
+  }
+
+  private static void setData(ZooKeeperWatcher zkw, SetData setData)
+  throws KeeperException, KeeperException.NoNodeException {
+    SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
+    setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
   }
 
   public static boolean isSecureZooKeeper(Configuration conf) {
@@ -896,14 +908,20 @@ public static void asyncCreate(ZooKeeperWatcher zkw,
    * @throws KeeperException if unexpected zookeeper exception
    */
   public static void createAndFailSilent(ZooKeeperWatcher zkw,
-      String znode)
+      String znode) throws KeeperException {
+    createAndFailSilent(zkw,
+      (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, new byte[0]));
+  }
+
+  private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
   throws KeeperException {
+    CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
+    String znode = create.getPath();
     try {
       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
       waitForZKConnectionIfAuthenticating(zkw);
       if (zk.exists(znode, false) == null) {
-        zk.create(znode, new byte[0], createACL(zkw,znode),
-            CreateMode.PERSISTENT);
+        zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
       }
     } catch(KeeperException.NodeExistsException nee) {
     } catch(KeeperException.NoAuthException nee){
@@ -989,8 +1007,15 @@ public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
    */
   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
   throws KeeperException {
+    deleteNodeFailSilent(zkw,
+      (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
+  }
+
+  private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
+      DeleteNodeFailSilent dnfs) throws KeeperException {
+    DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
     try {
-      zkw.getRecoverableZooKeeper().delete(node, -1);
+      zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
     } catch(KeeperException.NoNodeException nne) {
     } catch(InterruptedException ie) {
       zkw.interruptedException(ie);
@@ -1038,6 +1063,209 @@ public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
     }
   }
 
+  /**
+   * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
+   * These actions are higher-level than {@link ZKOp} actions, which represent
+   * individual actions in the ZooKeeper API, like create.
+   */
+  public abstract static class ZKUtilOp {
+    private String path;
+
+    private ZKUtilOp(String path) {
+      this.path = path;
+    }
+
+    /**
+     * @return a createAndFailSilent ZKUtilOp
+     */
+    public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
+      return new CreateAndFailSilent(path, data);
+    }
+
+    /**
+     * @return a deleteNodeFailSilent ZKUtilOP
+     */
+    public static ZKUtilOp deleteNodeFailSilent(String path) {
+      return new DeleteNodeFailSilent(path);
+    }
+
+    /**
+     * @return a setData ZKUtilOp
+     */
+    public static ZKUtilOp setData(String path, byte [] data) {
+      return new SetData(path, data);
+    }
+
+    /**
+     * @return path to znode where the ZKOp will occur
+     */
+    public String getPath() {
+      return path;
+    }
+
+    /**
+     * ZKUtilOp representing createAndFailSilent in ZooKeeper
+     * (attempt to create node, ignore error if already exists)
+     */
+    public static class CreateAndFailSilent extends ZKUtilOp {
+      private byte [] data;
+
+      private CreateAndFailSilent(String path, byte [] data) {
+        super(path);
+        this.data = data;
+      }
+
+      public byte[] getData() {
+        return data;
+      }
+
+      @Override
+      public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof CreateAndFailSilent)) return false;
+
+        CreateAndFailSilent op = (CreateAndFailSilent) o;
+        return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
+      }
+    }
+
+    /**
+     * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
+     * (attempt to delete node, ignore error if node doesn't exist)
+     */
+    public static class DeleteNodeFailSilent extends ZKUtilOp {
+      private DeleteNodeFailSilent(String path) {
+        super(path);
+      }
+
+      @Override
+      public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof DeleteNodeFailSilent)) return false;
+
+        return super.equals(o);
+      }
+    }
+
+    /**
+     * @return ZKUtilOp representing setData in ZooKeeper
+     */
+    public static class SetData extends ZKUtilOp {
+      private byte [] data;
+
+      private SetData(String path, byte [] data) {
+        super(path);
+        this.data = data;
+      }
+
+      public byte[] getData() {
+        return data;
+      }
+
+      @Override
+      public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof SetData)) return false;
+
+        SetData op = (SetData) o;
+        return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
+      }
+    }
+  }
+
+  /**
+   * Convert from ZKUtilOp to ZKOp
+   */
+  private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
+  throws UnsupportedOperationException {
+    if(op == null) return null;
+
+    if (op instanceof CreateAndFailSilent) {
+      CreateAndFailSilent cafs = (CreateAndFailSilent)op;
+      return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
+        CreateMode.PERSISTENT);
+    } else if (op instanceof DeleteNodeFailSilent) {
+      DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
+      return Op.delete(dnfs.getPath(), -1);
+    } else if (op instanceof SetData) {
+      SetData sd = (SetData)op;
+      return Op.setData(sd.getPath(), sd.getData(), -1);
+    } else {
+      throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
+        + op.getClass().getName());
+    }
+  }
+
+  /**
+   * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
+   * Otherwise, run the list of operations sequentially.
+   *
+   * If all of the following are true:
+   * - runSequentialOnMultiFailure is true
+   * - hbase.zookeeper.useMulti is true
+   * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
+   * Then:
+   * - we retry the operations one-by-one (sequentially)
+   *
+   * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
+   * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
+   * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
+   * This will cause all operations in the multi to fail, however, because
+   * the NodeExistsException that zk.create throws will fail the multi transaction.
+   * In this case, if the previous conditions hold, the commands are run sequentially, which should
+   * result in the correct final state, but means that the operations will not run atomically.
+   *
+   * @throws KeeperException
+   */
+  public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
+      boolean runSequentialOnMultiFailure) throws KeeperException {
+    if (ops == null) return;
+    boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
+
+    if (useMulti) {
+      List<Op> zkOps = new LinkedList<Op>();
+      for (ZKUtilOp op : ops) {
+        zkOps.add(toZooKeeperOp(zkw, op));
+      }
+      try {
+        zkw.getRecoverableZooKeeper().multi(zkOps);
+      } catch (KeeperException ke) {
+       switch (ke.code()) {
+         case NODEEXISTS:
+         case NONODE:
+         case BADVERSION:
+         case NOAUTH:
+           // if we get an exception that could be solved by running sequentially
+           // (and the client asked us to), then break out and run sequentially
+           if (runSequentialOnMultiFailure) {
+             LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
+               + "  Attempting to run operations sequentially because"
+               + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
+             break;
+           }
+          default:
+            throw ke;
+        }
+      } catch (InterruptedException ie) {
+        zkw.interruptedException(ie);
+      }
+    }
+
+    // run sequentially
+    for (ZKUtilOp op : ops) {
+      if (op instanceof CreateAndFailSilent) {
+        createAndFailSilent(zkw, (CreateAndFailSilent)op);
+      } else if (op instanceof DeleteNodeFailSilent) {
+        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent)op);
+      } else if (op instanceof SetData) {
+        setData(zkw, (SetData)op);
+      } else {
+        throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
+          + op.getClass().getName());
+      }
+    }
+  }
+
   //
   // ZooKeeper cluster information
   //
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index f73c47704bd7..3efa8517bbb0 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -690,6 +690,17 @@
     for more information.
     </description>
   </property>
+  <property>
+    <name>hbase.zookeeper.useMulti</name>
+    <value>false</value>
+    <description>Instructs HBase to make use of ZooKeeper's multi-update functionality.
+    This allows certain ZooKeeper operations to complete more quickly and prevents some issues
+    with rare ZooKeeper failure scenarios (see the release note of HBASE-6710 for an example).
+    IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+
+    and will not be downgraded.  ZooKeeper versions before 3.4 do not support multi-update and will
+    not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495).
+    </description>
+  </property>
   <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
 
   <!--
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
index 4748b95ceb4d..643c583e60f7 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
@@ -27,6 +27,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
 import org.apache.zookeeper.KeeperException;
@@ -110,14 +111,9 @@ public void testTableStates()
     assertFalse(zkt.isTablePresent(name));
   }
 
-  /**
-   * Test that ZK table writes table state in formats expected by 0.92 and 0.94 clients
-   */
-  @Test
-  public void test9294Compatibility() throws Exception {
-    final String tableName = "test9294Compatibility";
-
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+  private void runTest9294CompatibilityTest(String tableName, Configuration conf)
+  throws Exception {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
       tableName, abortable, true);
     ZKTable zkt = new ZKTable(zkw);
     zkt.setEnabledTable(tableName);
@@ -128,6 +124,22 @@ public void test9294Compatibility() throws Exception {
     assertTrue(ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode92, tableName) == null);
   }
 
+  /**
+   * Test that ZK table writes table state in formats expected by 0.92 and 0.94 clients
+   */
+  @Test
+  public void test9294Compatibility() throws Exception {
+    // without useMulti
+    String tableName = "test9294Compatibility";
+    runTest9294CompatibilityTest(tableName, TEST_UTIL.getConfiguration());
+
+    // with useMulti
+    tableName = "test9294CompatibilityWithMulti";
+    Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
+    runTest9294CompatibilityTest(tableName, conf);
+  }
+
   /**
    * RecoverableZookeeper that throws a KeeperException after throwExceptionInNumOperations
    */
@@ -170,14 +182,17 @@ public void delete(String path, int version)
    * Because two ZooKeeper nodes are written for each table state transition
    * {@link ZooKeeperWatcher#masterTableZNode} and {@link ZooKeeperWatcher#masterTableZNode92}
    * it is possible that we fail in between the two operations and are left with
-   * inconsistent state.  Check that we can get back to a consistent state by
-   * retrying the operation.
+   * inconsistent state (when hbase.zookeeper.useMulti is false).
+   * Check that we can get back to a consistent state by retrying the operation.
    */
   @Test
   public void testDisableTableRetry() throws Exception {
     final String tableName = "testDisableTableRetry";
 
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+    Configuration conf = TEST_UTIL.getConfiguration();
+    // test only relevant if useMulti is false
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
       tableName, abortable, true);
     ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw);
     ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw);
@@ -211,7 +226,10 @@ public void testDisableTableRetry() throws Exception {
   public void testEnableTableRetry() throws Exception {
     final String tableName = "testEnableTableRetry";
 
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+    Configuration conf = TEST_UTIL.getConfiguration();
+    // test only relevant if useMulti is false
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
       tableName, abortable, true);
     ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw);
     ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw);
@@ -245,7 +263,10 @@ public void testEnableTableRetry() throws Exception {
   public void testDeleteTableRetry() throws Exception {
     final String tableName = "testEnableTableRetry";
 
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+    Configuration conf = TEST_UTIL.getConfiguration();
+    // test only relevant if useMulti is false
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
       tableName, abortable, true);
     ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw);
     ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw);
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java
index 12b398ad5e51..36baf6c71250 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java
@@ -68,33 +68,53 @@ private boolean enableAndCheckEnabled(ZooKeeperWatcher zkw, String tableName) th
     return ZKTableReadOnly.isEnabledTable(zkw, tableName);
   }
 
+  private void runClientCompatiblityWith92ZNodeTest(String tableName, Configuration conf)
+  throws Exception {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
+      tableName, abortable, true);
+    assertTrue(enableAndCheckEnabled(zkw, tableName));
+  }
   /**
-   * Test that client ZK reader can handle the 0.92 table znode format.
+   * Test that client ZK reader can handle the 0.92 table format znode.
    */
   @Test
   public void testClientCompatibilityWith92ZNode() throws Exception {
-    final String tableName = "testClientCompatibilityWith92ZNode";
-
+    // test without useMulti
+    String tableName = "testClientCompatibilityWith92ZNode";
     // Set the client to read from the 0.92 table znode format
     Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
     String znode92 = conf.get("zookeeper.znode.masterTableEnableDisable92", "table92");
     conf.set("zookeeper.znode.clientTableEnableDisable", znode92);
+    runClientCompatiblityWith92ZNodeTest(tableName, conf);
 
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
+    // test with useMulti
+    tableName = "testClientCompatibilityWith92ZNodeUseMulti";
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
+    runClientCompatiblityWith92ZNodeTest(tableName, conf);
+  }
+
+  private void runClientCompatibilityWith94ZNodeTest(String tableName, Configuration conf)
+  throws Exception {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
       tableName, abortable, true);
     assertTrue(enableAndCheckEnabled(zkw, tableName));
   }
 
   /**
-   * Test that client ZK reader can handle the current (0.94) table format znode
+   * Test that client ZK reader can handle the current (0.94) table format znode.
    */
   @Test
   public void testClientCompatibilityWith94ZNode() throws Exception {
-    final String tableName = "testClientCompatibilityWith94ZNode";
+    String tableName = "testClientCompatibilityWith94ZNode";
 
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
-      tableName, abortable, true);
-    assertTrue(enableAndCheckEnabled(zkw, tableName));
+    // without useMulti
+    runClientCompatibilityWith94ZNodeTest(tableName, TEST_UTIL.getConfiguration());
+
+    // with useMulti
+    tableName = "testClientCompatiblityWith94ZNodeUseMulti";
+    Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
+    runClientCompatibilityWith94ZNodeTest(tableName, conf);
   }
 
   @org.junit.Rule

From c36b94e9891b4b7ab946b011de061ca7223ca370 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 18 Dec 2012 20:26:51 +0000
Subject: [PATCH 0636/1540] HBASE-4791 Allow Secure Zookeeper JAAS
 configuration to be programmatically set (rather than only by reading JAAS
 configuration file)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423620 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   |   4 +
 .../hbase/master/HMasterCommandLine.java      |   6 +
 .../hbase/regionserver/HRegionServer.java     |   4 +
 .../hadoop/hbase/zookeeper/HQuorumPeer.java   |   6 +
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 197 +++++++++++++++++-
 5 files changed, 208 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0cf1eba2a84f..1b6129963aa5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -281,6 +281,10 @@ public HMaster(final Configuration conf)
     this.rsFatals = new MemoryBoundedLogMessageBuffer(
         conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
 
+    // login the zookeeper client principal (if using security)
+    ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
+      "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
+
     // initialize server principal (if using secure Hadoop)
     User.login(conf, "hbase.master.keytab.file",
       "hbase.master.kerberos.principal", this.isa.getHostName());
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index ae9faf9a3488..e6fcce54cc5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.apache.hadoop.hbase.util.ServerCommandLine;
 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.zookeeper.KeeperException;
 
 public class HMasterCommandLine extends ServerCommandLine {
@@ -124,6 +125,11 @@ private int startMaster() {
               + HConstants.ZOOKEEPER_CLIENT_PORT);
         }
         zooKeeperCluster.setDefaultClientPort(zkClientPort);
+
+        // login the zookeeper server principal (if using security)
+        ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
+          "hbase.zookeeper.server.kerberos.principal", null);
+
         int clientPort = zooKeeperCluster.startup(zkDataPath);
         if (clientPort != zkClientPort) {
           String errorMsg = "Could not start ZK at requested port of " +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 6f527582548e..5296e6ac5a59 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -437,6 +437,10 @@ public HRegionServer(Configuration conf)
     this.rpcServer.setQosFunction(new QosFunction());
     this.startcode = System.currentTimeMillis();
 
+    // login the zookeeper client principal (if using security)
+    ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
+      "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
+
     // login the server principal (if using secure Hadoop)
     User.login(this.conf, "hbase.regionserver.keytab.file",
       "hbase.regionserver.kerberos.principal", this.isa.getHostName());
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
index 84b79a6a927d..43a9e974cb0f 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
@@ -61,6 +61,12 @@ public static void main(String[] args) {
       writeMyID(zkProperties);
       QuorumPeerConfig zkConfig = new QuorumPeerConfig();
       zkConfig.parseProperties(zkProperties);
+
+      // login the zookeeper server principal (if using security)
+      ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
+        "hbase.zookeeper.server.kerberos.principal",
+        zkConfig.getClientPortAddress().getHostName());
+
       runZKServer(zkConfig);
     } catch (Exception e) {
       e.printStackTrace();
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 0e9755be62dd..4de15b09737c 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -24,12 +24,21 @@
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.net.InetSocketAddress;
+import java.net.InetAddress;
 import java.net.Socket;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.security.auth.login.LoginException;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -54,6 +63,8 @@
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
 import org.apache.zookeeper.proto.CreateRequest;
 import org.apache.zookeeper.proto.DeleteRequest;
 import org.apache.zookeeper.proto.SetDataRequest;
@@ -117,6 +128,170 @@ public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
         retry, retryIntervalMillis);
   }
 
+  /**
+   * Log in the current zookeeper server process using the given configuration
+   * keys for the credential file and login principal.
+   *
+   * <p><strong>This is only applicable when running on secure hbase</strong>
+   * On regular HBase (without security features), this will safely be ignored.
+   * </p>
+   *
+   * @param conf The configuration data to use
+   * @param keytabFileKey Property key used to configure the path to the credential file
+   * @param userNameKey Property key used to configure the login principal
+   * @param hostname Current hostname to use in any credentials
+   * @throws IOException underlying exception from SecurityUtil.login() call
+   */
+  public static void loginServer(Configuration conf, String keytabFileKey,
+      String userNameKey, String hostname) throws IOException {
+    login(conf, keytabFileKey, userNameKey, hostname,
+          ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
+          JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
+  }
+
+  /**
+   * Log in the current zookeeper client using the given configuration
+   * keys for the credential file and login principal.
+   *
+   * <p><strong>This is only applicable when running on secure hbase</strong>
+   * On regular HBase (without security features), this will safely be ignored.
+   * </p>
+   *
+   * @param conf The configuration data to use
+   * @param keytabFileKey Property key used to configure the path to the credential file
+   * @param userNameKey Property key used to configure the login principal
+   * @param hostname Current hostname to use in any credentials
+   * @throws IOException underlying exception from SecurityUtil.login() call
+   */
+  public static void loginClient(Configuration conf, String keytabFileKey,
+      String userNameKey, String hostname) throws IOException {
+    login(conf, keytabFileKey, userNameKey, hostname,
+          ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+          JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
+  }
+
+  /**
+   * Log in the current process using the given configuration keys for the
+   * credential file and login principal.
+   *
+   * <p><strong>This is only applicable when running on secure hbase</strong>
+   * On regular HBase (without security features), this will safely be ignored.
+   * </p>
+   *
+   * @param conf The configuration data to use
+   * @param keytabFileKey Property key used to configure the path to the credential file
+   * @param userNameKey Property key used to configure the login principal
+   * @param hostname Current hostname to use in any credentials
+   * @param loginContextProperty property name to expose the entry name
+   * @param loginContextName jaas entry name
+   * @throws IOException underlying exception from SecurityUtil.login() call
+   */
+  private static void login(Configuration conf, String keytabFileKey,
+      String userNameKey, String hostname,
+      String loginContextProperty, String loginContextName)
+      throws IOException {
+    if (!isSecureZooKeeper(conf))
+      return;
+
+    // User has specified a jaas.conf, keep this one as the good one.
+    // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
+    if (System.getProperty("java.security.auth.login.config") != null)
+      return;
+
+    String keytabFilename = conf.get(keytabFileKey);
+    String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
+    String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
+
+    // Initialize the "jaas.conf" for keyTab/principal,
+    // If keyTab is not specified use the Ticket Cache.
+    // and set the zookeeper login context name.
+    JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
+      keytabFilename, principalName);
+    javax.security.auth.login.Configuration.setConfiguration(jaasConf);
+    System.setProperty(loginContextProperty, loginContextName);
+  }
+
+  /**
+   * A JAAS configuration that defines the login modules that we want to use for login.
+   */
+  private static class JaasConfiguration extends javax.security.auth.login.Configuration {
+    private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
+      "zookeeper-server-keytab-kerberos";
+    private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
+      "zookeeper-client-keytab-kerberos";
+
+    private static final Map<String, String> BASIC_JAAS_OPTIONS =
+      new HashMap<String,String>();
+    static {
+      String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
+      if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
+        BASIC_JAAS_OPTIONS.put("debug", "true");
+      }
+    }
+
+    private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
+      new HashMap<String,String>();
+    static {
+      KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
+      KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
+      KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
+      KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
+    }
+
+    private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
+      new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
+                                LoginModuleControlFlag.REQUIRED,
+                                KEYTAB_KERBEROS_OPTIONS);
+
+    private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
+      new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
+
+    private javax.security.auth.login.Configuration baseConfig;
+    private final String loginContextName;
+    private final boolean useTicketCache;
+    private final String keytabFile;
+    private final String principal;
+
+    public JaasConfiguration(String loginContextName, String principal) {
+      this(loginContextName, principal, null, true);
+    }
+
+    public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
+      this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
+    }
+
+    private JaasConfiguration(String loginContextName, String principal,
+                             String keytabFile, boolean useTicketCache) {
+      try {
+        this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
+      } catch (SecurityException e) {
+        this.baseConfig = null;
+      }
+      this.loginContextName = loginContextName;
+      this.useTicketCache = useTicketCache;
+      this.keytabFile = keytabFile;
+      this.principal = principal;
+      LOG.info("JaasConfiguration loginContextName=" + loginContextName +
+               " principal=" + principal + " useTicketCache=" + useTicketCache +
+               " keytabFile=" + keytabFile);
+    }
+
+    @Override
+    public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
+      if (loginContextName.equals(appName)) {
+        if (!useTicketCache) {
+          KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
+          KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
+        }
+        KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
+        KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
+        return KEYTAB_KERBEROS_CONF;
+      }
+      if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
+      return(null);
+    }
+  }
+
   //
   // Helper methods
   //
@@ -715,16 +890,20 @@ private static void setData(ZooKeeperWatcher zkw, SetData setData)
     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
   }
 
+  /**
+   * Returns whether or not secure authentication is enabled
+   * (whether <code>hbase.security.authentication</code> is set to
+   * <code>kerberos</code>.
+   */
   public static boolean isSecureZooKeeper(Configuration conf) {
-    // TODO: We need a better check for security enabled ZooKeeper. Currently
-    // the secure ZooKeeper client is set up using a supplied JaaS
-    // configuration file. But if the system property for the JaaS
-    // configuration file is set, this may not be an exclusive indication
-    // that HBase should set ACLs on znodes. As an alternative, we could do
-    // this more like Hadoop and build a JaaS configuration programmatically
-    // based on a site conf setting. The scope of such a change will be
-    // addressed in HBASE-4791.
-    return (System.getProperty("java.security.auth.login.config") != null);
+    // hbase shell need to use:
+    //    -Djava.security.auth.login.config=user-jaas.conf
+    // since each user has a different jaas.conf
+    if (System.getProperty("java.security.auth.login.config") != null)
+      return true;
+
+    // Master & RSs uses hbase.zookeeper.client.*
+    return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
   }
 
   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {

From 18eaece879144f9de9ce860d97978f07128df3d9 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Tue, 18 Dec 2012 20:43:07 +0000
Subject: [PATCH 0637/1540] HBASE-6775 Use ZK.multi when available (Addendum:
 add test file)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423635 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/zookeeper/TestZKMulti.java   | 291 ++++++++++++++++++
 1 file changed, 291 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java

diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java
new file mode 100644
index 000000000000..dd00372a94c5
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java
@@ -0,0 +1,291 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.LinkedList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test ZooKeeper multi-update functionality
+ */
+@Category(MediumTests.class)
+public class TestZKMulti {
+  private static final Log LOG = LogFactory.getLog(TestZKMulti.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static ZooKeeperWatcher zkw = null;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setBoolean("hbase.zookeeper.useMulti", true);
+    Abortable abortable = new Abortable() {
+      @Override
+      public void abort(String why, Throwable e) {
+        LOG.info(why, e);
+      }
+
+      @Override
+      public boolean isAborted() {
+        return false;
+      }
+    };
+    zkw = new ZooKeeperWatcher(conf,
+      "TestZKMulti", abortable, true);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  @Test
+  public void testSimpleMulti() throws Exception {
+    // null multi
+    ZKUtil.multiOrSequential(zkw, null, false);
+
+    // empty multi
+    ZKUtil.multiOrSequential(zkw, new LinkedList<ZKUtilOp>(), false);
+
+    // single create
+    String path = ZKUtil.joinZNode(zkw.baseZNode, "testSimpleMulti");
+    LinkedList<ZKUtilOp> singleCreate = new LinkedList<ZKUtilOp>();
+    singleCreate.add(ZKUtilOp.createAndFailSilent(path, new byte[0]));
+    ZKUtil.multiOrSequential(zkw, singleCreate, false);
+    assertTrue(ZKUtil.checkExists(zkw, path) != -1);
+
+    // single setdata
+    LinkedList<ZKUtilOp> singleSetData = new LinkedList<ZKUtilOp>();
+    byte [] data = Bytes.toBytes("foobar");
+    singleSetData.add(ZKUtilOp.setData(path, data));
+    ZKUtil.multiOrSequential(zkw, singleSetData, false);
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path), data));
+
+    // single delete
+    LinkedList<ZKUtilOp> singleDelete = new LinkedList<ZKUtilOp>();
+    singleDelete.add(ZKUtilOp.deleteNodeFailSilent(path));
+    ZKUtil.multiOrSequential(zkw, singleDelete, false);
+    assertTrue(ZKUtil.checkExists(zkw, path) == -1);
+  }
+
+  @Test
+  public void testComplexMulti() throws Exception {
+    String path1 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti1");
+    String path2 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti2");
+    String path3 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti3");
+    String path4 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti4");
+    String path5 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti5");
+    String path6 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti6");
+    // create 4 nodes that we'll setData on or delete later
+    LinkedList<ZKUtilOp> create4Nodes = new LinkedList<ZKUtilOp>();
+    create4Nodes.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1)));
+    create4Nodes.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2)));
+    create4Nodes.add(ZKUtilOp.createAndFailSilent(path3, Bytes.toBytes(path3)));
+    create4Nodes.add(ZKUtilOp.createAndFailSilent(path4, Bytes.toBytes(path4)));
+    ZKUtil.multiOrSequential(zkw, create4Nodes, false);
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.toBytes(path1)));
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2), Bytes.toBytes(path2)));
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path3), Bytes.toBytes(path3)));
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path4), Bytes.toBytes(path4)));
+
+    // do multiple of each operation (setData, delete, create)
+    LinkedList<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
+    // setData
+    ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
+    ops.add(ZKUtilOp.setData(path2, Bytes.add(Bytes.toBytes(path2), Bytes.toBytes(path2))));
+    // delete
+    ops.add(ZKUtilOp.deleteNodeFailSilent(path3));
+    ops.add(ZKUtilOp.deleteNodeFailSilent(path4));
+    // create
+    ops.add(ZKUtilOp.createAndFailSilent(path5, Bytes.toBytes(path5)));
+    ops.add(ZKUtilOp.createAndFailSilent(path6, Bytes.toBytes(path6)));
+    ZKUtil.multiOrSequential(zkw, ops, false);
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1),
+      Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2),
+      Bytes.add(Bytes.toBytes(path2), Bytes.toBytes(path2))));
+    assertTrue(ZKUtil.checkExists(zkw, path3) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, path4) == -1);
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path5), Bytes.toBytes(path5)));
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path6), Bytes.toBytes(path6)));
+  }
+
+  @Test
+  public void testSingleFailure() throws Exception {
+    // try to delete a node that doesn't exist
+    boolean caughtNoNode = false;
+    String path = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureZ");
+    LinkedList<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.deleteNodeFailSilent(path));
+    try {
+      ZKUtil.multiOrSequential(zkw, ops, false);
+    } catch (KeeperException.NoNodeException nne) {
+      caughtNoNode = true;
+    }
+    assertTrue(caughtNoNode);
+
+    // try to setData on a node that doesn't exist
+    caughtNoNode = false;
+    ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.setData(path, Bytes.toBytes(path)));
+    try {
+      ZKUtil.multiOrSequential(zkw, ops, false);
+    } catch (KeeperException.NoNodeException nne) {
+      caughtNoNode = true;
+    }
+    assertTrue(caughtNoNode);
+
+    // try to create on a node that already exists
+    boolean caughtNodeExists = false;
+    ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.createAndFailSilent(path, Bytes.toBytes(path)));
+    ZKUtil.multiOrSequential(zkw, ops, false);
+    try {
+      ZKUtil.multiOrSequential(zkw, ops, false);
+    } catch (KeeperException.NodeExistsException nee) {
+      caughtNodeExists = true;
+    }
+    assertTrue(caughtNodeExists);
+  }
+
+  @Test
+  public void testSingleFailureInMulti() throws Exception {
+    // try a multi where all but one operation succeeds
+    String pathA = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureInMultiA");
+    String pathB = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureInMultiB");
+    String pathC = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureInMultiC");
+    LinkedList<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA)));
+    ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB)));
+    ops.add(ZKUtilOp.deleteNodeFailSilent(pathC));
+    boolean caughtNoNode = false;
+    try {
+      ZKUtil.multiOrSequential(zkw, ops, false);
+    } catch (KeeperException.NoNodeException nne) {
+      caughtNoNode = true;
+    }
+    assertTrue(caughtNoNode);
+    // assert that none of the operations succeeded
+    assertTrue(ZKUtil.checkExists(zkw, pathA) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathB) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathC) == -1);
+  }
+
+  @Test
+  public void testMultiFailure() throws Exception {
+    String pathX = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureX");
+    String pathY = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureY");
+    String pathZ = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureZ");
+    // create X that we will use to fail create later
+    LinkedList<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX)));
+    ZKUtil.multiOrSequential(zkw, ops, false);
+
+    // fail one of each create ,setData, delete
+    String pathV = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureV");
+    String pathW = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureW");
+    ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail  -- already exists
+    ops.add(ZKUtilOp.setData(pathY, Bytes.toBytes(pathY))); // fail -- doesn't exist
+    ops.add(ZKUtilOp.deleteNodeFailSilent(pathZ)); // fail -- doesn't exist
+    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathV))); // pass
+    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathW))); // pass
+    boolean caughtNodeExists = false;
+    try {
+      ZKUtil.multiOrSequential(zkw, ops, false);
+    } catch (KeeperException.NodeExistsException nee) {
+      // check first operation that fails throws exception
+      caughtNodeExists = true;
+    }
+    assertTrue(caughtNodeExists);
+    // check that no modifications were made
+    assertFalse(ZKUtil.checkExists(zkw, pathX) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathY) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathZ) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathW) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathV) == -1);
+
+    // test that with multiple failures, throws an exception corresponding to first failure in list
+    ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.setData(pathY, Bytes.toBytes(pathY))); // fail -- doesn't exist
+    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- exists
+    boolean caughtNoNode = false;
+    try {
+      ZKUtil.multiOrSequential(zkw, ops, false);
+    } catch (KeeperException.NoNodeException nne) {
+      // check first operation that fails throws exception
+      caughtNoNode = true;
+    }
+    assertTrue(caughtNoNode);
+    // check that no modifications were made
+    assertFalse(ZKUtil.checkExists(zkw, pathX) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathY) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathZ) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathW) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, pathV) == -1);
+  }
+
+  @Test
+  public void testRunSequentialOnMultiFailure() throws Exception {
+    String path1 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential1");
+    String path2 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential2");
+    String path3 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential3");
+    String path4 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential4");
+
+    // create some nodes that we will use later
+    LinkedList<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1)));
+    ops.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2)));
+    ZKUtil.multiOrSequential(zkw, ops, false);
+
+    // test that, even with operations that fail, the ones that would pass will pass
+    // with runSequentialOnMultiFailure
+    ops = new LinkedList<ZKUtilOp>();
+    ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); // pass
+    ops.add(ZKUtilOp.deleteNodeFailSilent(path2)); // pass
+    ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); // fail -- node doesn't exist
+    ops.add(ZKUtilOp.createAndFailSilent(path4,
+      Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4)))); // pass
+    ZKUtil.multiOrSequential(zkw, ops, true);
+    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1),
+      Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
+    assertTrue(ZKUtil.checkExists(zkw, path2) == -1);
+    assertTrue(ZKUtil.checkExists(zkw, path3) == -1);
+    assertFalse(ZKUtil.checkExists(zkw, path4) == -1);
+  }
+}

From 8cecc62def83382a4adfb722c09ddfd4dfc64a43 Mon Sep 17 00:00:00 2001
From: gchanan <gchanan@unknown>
Date: Tue, 18 Dec 2012 21:27:53 +0000
Subject: [PATCH 0638/1540] HBASE-7377 Clean up TestHBase7051

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423671 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestHBase7051.java     | 151 ++++++++++--------
 1 file changed, 82 insertions(+), 69 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
index df6c293e3a95..a80ea5c864af 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
@@ -16,6 +16,9 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.MultithreadedTestUtil;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
@@ -30,19 +33,32 @@
 
 import com.google.common.collect.Lists;
 
+/**
+ * Test of HBASE-7051; that checkAndPuts and puts behave atomically with respect to each other.
+ * Rather than perform a bunch of trials to verify atomicity, this test recreates a race condition
+ * that causes the test to fail if checkAndPut doesn't wait for outstanding put transactions
+ * to complete.  It does this by invasively overriding HRegion function to affect the timing of
+ * the operations.
+ */
 @Category(SmallTests.class)
 public class TestHBase7051 {
 
-  private static volatile boolean putCompleted = false;
   private static CountDownLatch latch = new CountDownLatch(1);
-  private boolean checkAndPutCompleted = false;
-  private static int count = 0;
-
+  private enum TestStep {
+    INIT,                  // initial put of 10 to set value of the cell
+    PUT_STARTED,           // began doing a put of 50 to cell
+    PUT_COMPLETED,         // put complete (released RowLock, but may not have advanced MVCC).
+    CHECKANDPUT_STARTED,   // began checkAndPut: if 10 -> 11
+    CHECKANDPUT_COMPLETED  // completed checkAndPut
+    // NOTE: at the end of these steps, the value of the cell should be 50, not 11!
+  }
+  private static volatile TestStep testStep = TestStep.INIT;
+  private final String family = "f1";
+  	 
   @Test
   public void testPutAndCheckAndPutInParallel() throws Exception {
 
     final String tableName = "testPutAndCheckAndPut";
-    final String family = "f1";
     Configuration conf = HBaseConfiguration.create();
     conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
     final MockHRegion region = (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName),
@@ -57,14 +73,16 @@ public void testPutAndCheckAndPutInParallel() throws Exception {
 
     putsAndLocks.add(pair);
 
-    count++;
     region.batchMutate(putsAndLocks.toArray(new Pair[0]));
-    makeCheckAndPut(family, region);
-
-    makePut(family, region);
-    while (!checkAndPutCompleted) {
+    MultithreadedTestUtil.TestContext ctx =
+      new MultithreadedTestUtil.TestContext(conf);
+    ctx.addThread(new PutThread(ctx, region));
+    ctx.addThread(new CheckAndPutThread(ctx, region));
+    ctx.startThreads();
+    while (testStep != TestStep.CHECKANDPUT_COMPLETED) {
       Thread.sleep(100);
     }
+    ctx.stop();
     Scan s = new Scan();
     RegionScanner scanner = region.getScanner(s);
     List<KeyValue> results = new ArrayList<KeyValue>();
@@ -75,54 +93,46 @@ public void testPutAndCheckAndPutInParallel() throws Exception {
 
   }
 
-  private void makePut(final String family, final MockHRegion region) {
-    new Thread() {
-      public void run() {
-        List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
-        Put[] puts = new Put[1];
-        Put put = new Put(Bytes.toBytes("r1"));
-        put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50"));
-        puts[0] = put;
-        try {
-          Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
-          putsAndLocks.add(pair);
-          count++;
-          region.batchMutate(putsAndLocks.toArray(new Pair[0]));
-        } catch (IOException e) {
-          // TODO Auto-generated catch block
-          e.printStackTrace();
-        }
-      }
-    }.start();
-  }
+  private class PutThread extends TestThread {
+    private MockHRegion region;
+    PutThread(TestContext ctx, MockHRegion region) {
+      super(ctx);
+      this.region = region;
+    }
 
-  private void makeCheckAndPut(final String family, final MockHRegion region) {
-    new Thread() {
+    public void doWork() throws Exception {
+      List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
+      Put[] puts = new Put[1];
+      Put put = new Put(Bytes.toBytes("r1"));
+      put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50"));
+      puts[0] = put;
+      Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
+      putsAndLocks.add(pair);
+      testStep = TestStep.PUT_STARTED;
+      region.batchMutate(putsAndLocks.toArray(new Pair[0]));
+    }
+  }
 
-      public void run() {
-        Put[] puts = new Put[1];
-        Put put = new Put(Bytes.toBytes("r1"));
-        put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11"));
-        puts[0] = put;
-        try {
-          while (putCompleted == false) {
-            try {
-              Thread.sleep(100);
-            } catch (InterruptedException e) {
-              // TODO Auto-generated catch block
-              e.printStackTrace();
-            }
-          }
-          count++;
-          region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"),
-              CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, null, true);
-          checkAndPutCompleted = true;
-        } catch (IOException e) {
-          // TODO Auto-generated catch block
-          e.printStackTrace();
-        }
+  private class CheckAndPutThread extends TestThread {
+    private MockHRegion region;
+    CheckAndPutThread(TestContext ctx, MockHRegion region) {
+      super(ctx);
+      this.region = region;
+   }
+
+    public void doWork() throws Exception {
+      Put[] puts = new Put[1];
+      Put put = new Put(Bytes.toBytes("r1"));
+      put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11"));
+      puts[0] = put;
+      while (testStep != TestStep.PUT_COMPLETED) {
+        Thread.sleep(100);
       }
-    }.start();
+      testStep = TestStep.CHECKANDPUT_STARTED;
+      region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"),
+        CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, null, true);
+      testStep = TestStep.CHECKANDPUT_COMPLETED;
+    }
   }
 
   public static class MockHRegion extends HRegion {
@@ -134,36 +144,39 @@ public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
 
     @Override
     public void releaseRowLock(Integer lockId) {
-      if (count == 1) {
+      if (testStep == TestStep.INIT) {
         super.releaseRowLock(lockId);
         return;
       }
 
-      if (count == 2) {
+      if (testStep == TestStep.PUT_STARTED) {
         try {
-          putCompleted = true;
+          testStep = TestStep.PUT_COMPLETED;
           super.releaseRowLock(lockId);
+          // put has been written to the memstore and the row lock has been released, but the
+          // MVCC has not been advanced.  Prior to fixing HBASE-7051, the following order of
+          // operations would cause the non-atomicity to show up:
+          // 1) Put releases row lock (where we are now)
+          // 2) CheckAndPut grabs row lock and reads the value prior to the put (10)
+          //    because the MVCC has not advanced
+          // 3) Put advances MVCC
+          // So, in order to recreate this order, we wait for the checkAndPut to grab the rowLock
+          // (see below), and then wait some more to give the checkAndPut time to read the old
+          // value.
           latch.await();
+          Thread.sleep(1000);
         } catch (InterruptedException e) {
-          // TODO Auto-generated catch block
-          e.printStackTrace();
+          Thread.currentThread().interrupt();
         }
       }
-      if (count == 3) {
+      else if (testStep == TestStep.CHECKANDPUT_STARTED) {
         super.releaseRowLock(lockId);
-        try {
-          Thread.sleep(1000);
-        } catch (InterruptedException e) {
-          // TODO Auto-generated catch block
-          e.printStackTrace();
-        }
-        latch.countDown();
       }
     }
 
     @Override
     public Integer getLock(Integer lockid, byte[] row, boolean waitForLock) throws IOException {
-      if (count == 3) {
+      if (testStep == TestStep.CHECKANDPUT_STARTED) {
         latch.countDown();
       }
       return super.getLock(lockid, row, waitForLock);

From 63fa829a6c2df2bf18bf45710e151dd0e22f938b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 19 Dec 2012 06:54:19 +0000
Subject: [PATCH 0639/1540] HBASE-7381 Lightweight data transfer for Class
 Result (Cheng Hao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423771 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/Result.java   | 11 +++++++++++
 .../hadoop/hbase/mapred/TableRecordReaderImpl.java    |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 532359652505..bf8bda31ec5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -649,4 +649,15 @@ public static void compareResults(Result res1, Result res2)
       }
     }
   }
+  
+  /**
+   * Copy another Result into this one. Needed for the old Mapred framework
+   * @param other
+   */
+  public void copyFrom(Result other) {
+    this.row = other.row;
+    this.bytes = other.bytes;
+    this.familyMap = other.familyMap;
+    this.kvs = other.kvs;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
index 5212ccdd85b4..72e5774afcdd 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -227,7 +227,7 @@ public boolean next(ImmutableBytesWritable key, Result value)
       if (result != null && result.size() > 0) {
         key.set(result.getRow());
         lastSuccessfulRow = key.get();
-        Writables.copyWritable(result, value);
+        value.copyFrom(result);
         return true;
       }
       return false;

From 4e10b36ea57659fedfaabd77bcfa3f358853a384 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 19 Dec 2012 07:35:38 +0000
Subject: [PATCH 0640/1540] HBASE-7283 Backport HBASE-6564 + HBASE-7202 to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423774 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/backup/HFileArchiver.java    |  34 +++-
 .../hadoop/hbase/master/MasterFileSystem.java |  17 ++
 .../handler/TableDeleteFamilyHandler.java     |   7 +
 .../hadoop/hbase/HBaseTestingUtility.java     |  38 +++++
 .../hbase/backup/TestHFileArchiving.java      |  88 ++++++++--
 .../handler/TestTableDeleteFamilyHandler.java | 159 ++++++++++++++++++
 6 files changed, 331 insertions(+), 12 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java

diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index ed787e42d0f1..0da7c69b725e 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -146,6 +146,38 @@ public boolean accept(Path file) {
         + "), cannot delete region directory.");
   }
 
+  /**
+   * Remove from the specified region the store files of the specified column family,
+   * either by archiving them or outright deletion
+   * @param fs the filesystem where the store files live
+   * @param conf {@link Configuration} to examine to determine the archive directory
+   * @param parent Parent region hosting the store files
+   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+   * @param family the family hosting the store files
+   * @throws IOException if the files could not be correctly disposed.
+   */
+  public static void archiveFamily(FileSystem fs, Configuration conf,
+      HRegionInfo parent, Path tableDir, byte[] family) throws IOException {
+    Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family)));
+    FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, null);
+    if (storeFiles == null) {
+      LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() +
+          ", family=" + Bytes.toString(family));
+      return;
+    }
+
+    FileStatusConverter getAsFile = new FileStatusConverter(fs);
+    Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile);
+    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family);
+
+    // do the actual archive
+    if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) {
+      throw new IOException("Failed to archive/delete all the files for region:"
+          + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
+          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
+    }
+  }
+
   /**
    * Remove the store files, either by archiving them or outright deletion
    * @param fs the filesystem where the store files live
@@ -196,7 +228,7 @@ public static void archiveStoreFiles(FileSystem fs, HRegion parent,
     if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) {
       throw new IOException("Failed to archive/delete all the files for region:"
           + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + "Something is probably arwy on the filesystem.");
+          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 3552c22066f1..612852e095ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -454,6 +454,23 @@ public void updateRegionInfo(HRegionInfo region) {
     //      @see HRegion.checkRegioninfoOnFilesystem()
   }
 
+  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
+      throws IOException {
+    // archive family store files
+    Path tableDir = new Path(rootdir, region.getTableNameAsString());
+    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
+
+    // delete the family folder
+    Path familyDir = new Path(tableDir,
+      new Path(region.getEncodedName(), Bytes.toString(familyName)));
+    if (fs.delete(familyDir, true) == false) {
+      throw new IOException("Could not delete family "
+          + Bytes.toString(familyName) + " from FileSystem for region "
+          + region.getRegionNameAsString() + "(" + region.getEncodedName()
+          + ")");
+    }
+  }
+
   public void stop() {
     if (splitLogManager != null) {
       this.splitLogManager.stop();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
index 89889316744a..9322ba6479b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -49,6 +50,12 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
       this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName);
     // Update in-memory descriptor cache
     this.masterServices.getTableDescriptors().add(htd);
+    // Remove the column family from the file system
+    MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+    for (HRegionInfo hri : hris) {
+      // Delete the family directory in FS for all the regions one by one
+      mfs.deleteFamilyFromFS(hri, familyName);
+    }
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 998ad1552f52..033d4b78d16c 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1014,6 +1014,36 @@ public int loadTable(final HTable t, final byte[] f) throws IOException {
     return rowCount;
   }
 
+  /**
+   * Load table of multiple column families with rows from 'aaa' to 'zzz'.
+   * @param t Table
+   * @param f Array of Families to load
+   * @return Count of rows loaded.
+   * @throws IOException
+   */
+  public int loadTable(final HTable t, final byte[][] f) throws IOException {
+    t.setAutoFlush(false);
+    byte[] k = new byte[3];
+    int rowCount = 0;
+    for (byte b1 = 'a'; b1 <= 'z'; b1++) {
+      for (byte b2 = 'a'; b2 <= 'z'; b2++) {
+        for (byte b3 = 'a'; b3 <= 'z'; b3++) {
+          k[0] = b1;
+          k[1] = b2;
+          k[2] = b3;
+          Put put = new Put(k);
+          for (int i = 0; i < f.length; i++) {
+            put.add(f[i], null, k);
+          }
+          t.put(put);
+          rowCount++;
+        }
+      }
+    }
+    t.flushCommits();
+    return rowCount;
+  }
+
   /**
    * Load region with rows from 'aaa' to 'zzz'.
    * @param r Region
@@ -1148,6 +1178,9 @@ public int createMultiRegions(final Configuration c, final HTable table,
     // and end key. Adding the custom regions below adds those blindly,
     // including the new start region from empty to "bbb". lg
     List<byte[]> rows = getMetaTableRows(htd.getName());
+    String regionToDeleteInFS = table
+        .getRegionsInRange(Bytes.toBytes(""), Bytes.toBytes("")).get(0)
+        .getRegionInfo().getEncodedName();
     List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
     // add custom ones
     int count = 0;
@@ -1169,6 +1202,11 @@ public int createMultiRegions(final Configuration c, final HTable table,
         Bytes.toStringBinary(row));
       meta.delete(new Delete(row));
     }
+    // remove the "old" region from FS
+    Path tableDir = new Path(getDefaultRootDirPath().toString()
+        + System.getProperty("file.separator") + htd.getNameAsString()
+        + System.getProperty("file.separator") + regionToDeleteInFS);
+    getDFSCluster().getFileSystem().delete(tableDir);
     // flush cache of regions
     HConnection conn = table.getConnection();
     conn.clearRegionCache();
diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index f6142f24fda9..e1dd0c5ddb4c 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -55,7 +55,7 @@
  * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
  * a region
  */
-@Category(LargeTests.class)
+@Category(MediumTests.class)
 public class TestHFileArchiving {
 
   private static final String STRING_TABLE_NAME = "test_table";
@@ -230,18 +230,70 @@ public void testArchiveOnTableDelete() throws Exception {
 
     // then get the current store files
     Path regionDir = region.getRegionDir();
-    List<String> storeFiles = getAllFileNames(fs, regionDir);
-    // remove all the non-storefile named files for the region
+    List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
+
+    // then delete the table so the hfiles get archived
+    UTIL.deleteTable(TABLE_NAME);
+
+    // then get the files in the archive directory.
+    Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
+    List<String> archivedFiles = getAllFileNames(fs, archiveDir);
+    Collections.sort(storeFiles);
+    Collections.sort(archivedFiles);
+
+    LOG.debug("Store files:");
     for (int i = 0; i < storeFiles.size(); i++) {
-      String file = storeFiles.get(i);
-      if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
-        storeFiles.remove(i--);
-      }
+      LOG.debug(i + " - " + storeFiles.get(i));
     }
-    storeFiles.remove(HRegion.REGIONINFO_FILE);
+    LOG.debug("Archive files:");
+    for (int i = 0; i < archivedFiles.size(); i++) {
+      LOG.debug(i + " - " + archivedFiles.get(i));
+    }
+
+    assertTrue("Archived files are missing some of the store files!",
+      archivedFiles.containsAll(storeFiles));
+  }
+
+  /**
+   * Test that the store files are archived when a column family is removed.
+   * @throws Exception
+   */
+  @Test
+  public void testArchiveOnTableFamilyDelete() throws Exception {
+    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
+    // make sure we only have 1 region serving this table
+    assertEquals(1, servingRegions.size());
+    HRegion region = servingRegions.get(0);
+
+    // get the parent RS and monitor
+    HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
+    FileSystem fs = hrs.getFileSystem();
+
+    // put some data on the region
+    LOG.debug("-------Loading table");
+    UTIL.loadRegion(region, TEST_FAM);
+
+    // get the hfiles in the region
+    List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
+    assertEquals("More that 1 region for test table.", 1, regions.size());
+
+    region = regions.get(0);
+    // wait for all the compactions to complete
+    region.waitForFlushesAndCompactions();
+
+    // disable table to prevent new updates
+    UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
+    LOG.debug("Disabled table");
+
+    // remove all the files from the archive to get a fair comparison
+    clearArchiveDirectory();
+
+    // then get the current store files
+    Path regionDir = region.getRegionDir();
+    List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
 
     // then delete the table so the hfiles get archived
-    UTIL.deleteTable(TABLE_NAME);
+    UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
 
     // then get the files in the archive directory.
     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
@@ -290,4 +342,18 @@ private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<Stri
     }
     return fileNames;
   }
-}
\ No newline at end of file
+
+  private List<String> getRegionStoreFiles(final FileSystem fs, final Path regionDir) 
+      throws IOException {
+    List<String> storeFiles = getAllFileNames(fs, regionDir);
+    // remove all the non-storefile named files for the region
+    for (int i = 0; i < storeFiles.size(); i++) {
+      String file = storeFiles.get(i);
+      if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
+        storeFiles.remove(i--);
+      }
+    }
+    storeFiles.remove(HRegion.REGIONINFO_FILE);
+    return storeFiles;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java
new file mode 100644
index 000000000000..ad56bd4c1d77
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java
@@ -0,0 +1,159 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestTableDeleteFamilyHandler {
+
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final String TABLENAME = "column_family_handlers";
+  private static final byte[][] FAMILIES = new byte[][] { Bytes.toBytes("cf1"),
+      Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
+
+  /**
+   * Start up a mini cluster and put a small table of empty regions into it.
+   * 
+   * @throws Exception
+   */
+  @BeforeClass
+  public static void beforeAllTests() throws Exception {
+
+    TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
+    TEST_UTIL.startMiniCluster(2);
+
+    // Create a table of three families. This will assign a region.
+    TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
+    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
+
+    // Create multiple regions in all the three column families
+    TEST_UTIL.createMultiRegions(t, FAMILIES[0]);
+
+    // Load the table with data for all families
+    TEST_UTIL.loadTable(t, FAMILIES);
+
+    TEST_UTIL.flush();
+
+    t.close();
+  }
+
+  @AfterClass
+  public static void afterAllTests() throws Exception {
+    TEST_UTIL.deleteTable(Bytes.toBytes(TABLENAME));
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void setup() throws IOException, InterruptedException {
+    TEST_UTIL.ensureSomeRegionServersAvailable(2);
+  }
+
+  @Test
+  public void deleteColumnFamilyWithMultipleRegions() throws Exception {
+
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    HTableDescriptor beforehtd = admin.getTableDescriptor(Bytes
+        .toBytes(TABLENAME));
+
+    FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
+
+    // 1 - Check if table exists in descriptor
+    assertTrue(admin.isTableAvailable(TABLENAME));
+
+    // 2 - Check if all three families exist in descriptor
+    assertEquals(3, beforehtd.getColumnFamilies().length);
+    HColumnDescriptor[] families = beforehtd.getColumnFamilies();
+    for (int i = 0; i < families.length; i++) {
+
+      assertTrue(families[i].getNameAsString().equals("cf" + (i + 1)));
+    }
+
+    // 3 - Check if table exists in FS
+    Path tableDir = new Path(TEST_UTIL.getDefaultRootDirPath().toString() + "/"
+        + TABLENAME);
+    assertTrue(fs.exists(tableDir));
+
+    // 4 - Check if all the 3 column families exist in FS
+    FileStatus[] fileStatus = fs.listStatus(tableDir);
+    for (int i = 0; i < fileStatus.length; i++) {
+      if (fileStatus[i].isDir() == true) {
+        FileStatus[] cf = fs.listStatus(fileStatus[i].getPath());
+        int k = 1;
+        for (int j = 0; j < cf.length; j++) {
+          if (cf[j].isDir() == true
+              && cf[j].getPath().getName().startsWith(".") == false) {
+            assertTrue(cf[j].getPath().getName().equals("cf" + k));
+            k++;
+          }
+        }
+      }
+    }
+
+    // TEST - Disable and delete the column family
+    admin.disableTable(TABLENAME);
+    admin.deleteColumn(TABLENAME, "cf2");
+
+    // 5 - Check if only 2 column families exist in the descriptor
+    HTableDescriptor afterhtd = admin.getTableDescriptor(Bytes
+        .toBytes(TABLENAME));
+    assertEquals(2, afterhtd.getColumnFamilies().length);
+    HColumnDescriptor[] newFamilies = afterhtd.getColumnFamilies();
+    assertTrue(newFamilies[0].getNameAsString().equals("cf1"));
+    assertTrue(newFamilies[1].getNameAsString().equals("cf3"));
+
+    // 6 - Check if the second column family is gone from the FS
+    fileStatus = fs.listStatus(tableDir);
+    for (int i = 0; i < fileStatus.length; i++) {
+      if (fileStatus[i].isDir() == true) {
+        FileStatus[] cf = fs.listStatus(fileStatus[i].getPath());
+        for (int j = 0; j < cf.length; j++) {
+          if (cf[j].isDir() == true) {
+            assertFalse(cf[j].getPath().getName().equals("cf2"));
+          }
+        }
+      }
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = 
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 9e8666f638ac468526dcd15dc290dccd09c32e64 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 19 Dec 2012 16:33:21 +0000
Subject: [PATCH 0641/1540] HBASE-6585 Audit log messages should contain info
 about the higher level operation being executed (Matteo Bertozzi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423927 13f79535-47bb-0310-9956-ffa450edef68
---
 conf/log4j.properties                         |   1 +
 .../security/access/AccessController.java     | 144 +++++++++---------
 2 files changed, 77 insertions(+), 68 deletions(-)

diff --git a/conf/log4j.properties b/conf/log4j.properties
index f6c33051780d..24d7b2b798c1 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -39,6 +39,7 @@ log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
 log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 log4j.category.SecurityLogger=${hbase.security.logger}
 log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
 
 #
 # Null Appender
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index b82ad5dbecec..891c1e58b6b5 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -112,12 +112,14 @@ private static class AuthResult {
     private final byte[] family;
     private final byte[] qualifier;
     private final Permission.Action action;
+    private final String request;
     private final String reason;
     private final User user;
 
-    public AuthResult(boolean allowed, String reason,  User user,
+    public AuthResult(boolean allowed, String request, String reason,  User user,
         Permission.Action action, byte[] table, byte[] family, byte[] qualifier) {
       this.allowed = allowed;
+      this.request = request;
       this.reason = reason;
       this.user = user;
       this.table = table;
@@ -132,6 +134,8 @@ public AuthResult(boolean allowed, String reason,  User user,
 
     public String getReason() { return reason; }
 
+    public String getRequest() { return request; }
+
     public String toContextString() {
       return "(user=" + (user != null ? user.getName() : "UNKNOWN") + ", " +
           "scope=" + (table == null ? "GLOBAL" : Bytes.toString(table)) + ", " +
@@ -145,23 +149,23 @@ public String toString() {
           .append(toContextString()).toString();
     }
 
-    public static AuthResult allow(String reason, User user, Permission.Action action,
+    public static AuthResult allow(String request, String reason, User user, Permission.Action action,
         byte[] table, byte[] family, byte[] qualifier) {
-      return new AuthResult(true, reason, user, action, table, family, qualifier);
+      return new AuthResult(true, request, reason, user, action, table, family, qualifier);
     }
 
-    public static AuthResult allow(String reason, User user, Permission.Action action, byte[] table) {
-      return new AuthResult(true, reason, user, action, table, null, null);
+    public static AuthResult allow(String request, String reason, User user, Permission.Action action, byte[] table) {
+      return new AuthResult(true, request, reason, user, action, table, null, null);
     }
 
-    public static AuthResult deny(String reason, User user,
+    public static AuthResult deny(String request, String reason, User user,
         Permission.Action action, byte[] table) {
-      return new AuthResult(false, reason, user, action, table, null, null);
+      return new AuthResult(false, request, reason, user, action, table, null, null);
     }
 
-    public static AuthResult deny(String reason, User user,
+    public static AuthResult deny(String request, String reason, User user,
         Permission.Action action, byte[] table, byte[] family, byte[] qualifier) {
-      return new AuthResult(false, reason, user, action, table, family, qualifier);
+      return new AuthResult(false, request, reason, user, action, table, family, qualifier);
     }
   }
 
@@ -252,7 +256,7 @@ void updateACL(RegionCoprocessorEnvironment e,
    * the request
    * @return
    */
-  AuthResult permissionGranted(User user, TablePermission.Action permRequest,
+  AuthResult permissionGranted(String request, User user, TablePermission.Action permRequest,
       RegionCoprocessorEnvironment e,
       Map<byte [], ? extends Collection<?>> families) {
     HRegionInfo hri = e.getRegion().getRegionInfo();
@@ -262,12 +266,12 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
     // this is a very common operation, so deal with it quickly.
     if (hri.isRootRegion() || hri.isMetaRegion()) {
       if (permRequest == TablePermission.Action.READ) {
-        return AuthResult.allow("All users allowed", user, permRequest, tableName);
+        return AuthResult.allow(request, "All users allowed", user, permRequest, tableName);
       }
     }
 
     if (user == null) {
-      return AuthResult.deny("No user associated with request!", null, permRequest, tableName);
+      return AuthResult.deny(request, "No user associated with request!", null, permRequest, tableName);
     }
 
     // Users with CREATE/ADMIN rights need to modify .META. and _acl_ table
@@ -281,12 +285,12 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
        (authManager.authorize(user, Permission.Action.CREATE) ||
         authManager.authorize(user, Permission.Action.ADMIN)))
     {
-       return AuthResult.allow("Table permission granted", user, permRequest, tableName);
+       return AuthResult.allow(request, "Table permission granted", user, permRequest, tableName);
     }
 
     // 2. check for the table-level, if successful we can short-circuit
     if (authManager.authorize(user, tableName, (byte[])null, permRequest)) {
-      return AuthResult.allow("Table permission granted", user, permRequest, tableName);
+      return AuthResult.allow(request, "Table permission granted", user, permRequest, tableName);
     }
 
     // 3. check permissions against the requested families
@@ -307,7 +311,7 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
             for (byte[] qualifier : familySet) {
               if (!authManager.authorize(user, tableName, family.getKey(),
                                          qualifier, permRequest)) {
-                return AuthResult.deny("Failed qualifier check", user,
+                return AuthResult.deny(request, "Failed qualifier check", user,
                     permRequest, tableName, family.getKey(), qualifier);
               }
             }
@@ -316,25 +320,25 @@ AuthResult permissionGranted(User user, TablePermission.Action permRequest,
             for (KeyValue kv : kvList) {
               if (!authManager.authorize(user, tableName, family.getKey(),
                       kv.getQualifier(), permRequest)) {
-                return AuthResult.deny("Failed qualifier check", user,
+                return AuthResult.deny(request, "Failed qualifier check", user,
                     permRequest, tableName, family.getKey(), kv.getQualifier());
               }
             }
           }
         } else {
           // no qualifiers and family-level check already failed
-          return AuthResult.deny("Failed family check", user, permRequest,
+          return AuthResult.deny(request, "Failed family check", user, permRequest,
               tableName, family.getKey(), null);
         }
       }
 
       // all family checks passed
-      return AuthResult.allow("All family checks passed", user, permRequest,
+      return AuthResult.allow(request, "All family checks passed", user, permRequest,
           tableName);
     }
 
     // 4. no families to check and table level access failed
-    return AuthResult.deny("No families to check and table permission failed",
+    return AuthResult.deny(request, "No families to check and table permission failed",
         user, permRequest, tableName);
   }
 
@@ -349,6 +353,7 @@ private void logResult(AuthResult result) {
           " for user " + (result.getUser() != null ? result.getUser().getShortName() : "UNKNOWN") +
           "; reason: " + result.getReason() +
           "; remote address: " + (remoteAddr != null ? remoteAddr : "") +
+          "; request: " + result.getRequest() +
           "; context: " + result.toContextString());
     }
   }
@@ -377,18 +382,20 @@ private User getActiveUser() throws IOException {
    * @throws IOException if obtaining the current user fails
    * @throws AccessDeniedException if user has no authorization
    */
-  private void requirePermission(byte[] tableName, byte[] family, byte[] qualifier,
+  private void requirePermission(String request, byte[] tableName, byte[] family, byte[] qualifier,
       Action... permissions) throws IOException {
     User user = getActiveUser();
     AuthResult result = null;
 
     for (Action permission : permissions) {
       if (authManager.authorize(user, tableName, family, qualifier, permission)) {
-        result = AuthResult.allow("Table permission granted", user, permission, tableName, family, qualifier);
+        result = AuthResult.allow(request, "Table permission granted", user,
+                                  permission, tableName, family, qualifier);
         break;
       } else {
         // rest of the world
-        result = AuthResult.deny("Insufficient permissions", user, permission, tableName, family, qualifier);
+        result = AuthResult.deny(request, "Insufficient permissions", user,
+                                 permission, tableName, family, qualifier);
       }
     }
     logResult(result);
@@ -403,12 +410,12 @@ private void requirePermission(byte[] tableName, byte[] family, byte[] qualifier
    * @throws IOException if obtaining the current user fails
    * @throws AccessDeniedException if authorization is denied
    */
-  private void requirePermission(Permission.Action perm) throws IOException {
+  private void requirePermission(String request, Permission.Action perm) throws IOException {
     User user = getActiveUser();
     if (authManager.authorize(user, perm)) {
-      logResult(AuthResult.allow("Global check allowed", user, perm, null));
+      logResult(AuthResult.allow(request, "Global check allowed", user, perm, null));
     } else {
-      logResult(AuthResult.deny("Global check failed", user, perm, null));
+      logResult(AuthResult.deny(request, "Global check failed", user, perm, null));
       throw new AccessDeniedException("Insufficient permissions for user '" +
           (user != null ? user.getShortName() : "null") +"' (global, action=" +
           perm.toString() + ")");
@@ -423,7 +430,7 @@ private void requirePermission(Permission.Action perm) throws IOException {
    * @param families The set of column families present/required in the request
    * @throws AccessDeniedException if the authorization check failed
    */
-  private void requirePermission(Permission.Action perm,
+  private void requirePermission(String request, Permission.Action perm,
         RegionCoprocessorEnvironment env, Collection<byte[]> families)
       throws IOException {
     // create a map of family-qualifier
@@ -431,7 +438,7 @@ private void requirePermission(Permission.Action perm,
     for (byte[] family : families) {
       familyMap.put(family, null);
     }
-    requirePermission(perm, env, familyMap);
+    requirePermission(request, perm, env, familyMap);
   }
 
   /**
@@ -442,12 +449,12 @@ private void requirePermission(Permission.Action perm,
    * @param families The map of column families-qualifiers.
    * @throws AccessDeniedException if the authorization check failed
    */
-  private void requirePermission(Permission.Action perm,
+  private void requirePermission(String request, Permission.Action perm,
         RegionCoprocessorEnvironment env,
         Map<byte[], ? extends Collection<?>> families)
       throws IOException {
     User user = getActiveUser();
-    AuthResult result = permissionGranted(user, perm, env, families);
+    AuthResult result = permissionGranted(request, user, perm, env, families);
     logResult(result);
 
     if (!result.isAllowed()) {
@@ -532,7 +539,7 @@ public void stop(CoprocessorEnvironment env) {
   @Override
   public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
       HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
-    requirePermission(Permission.Action.CREATE);
+    requirePermission("createTable", Permission.Action.CREATE);
   }
 
   @Override
@@ -551,7 +558,7 @@ public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName)
       throws IOException {
-   requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+   requirePermission("deleteTable", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -563,7 +570,7 @@ public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
       HTableDescriptor htd) throws IOException {
-    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+    requirePermission("modifyTable", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -580,7 +587,7 @@ public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
       HColumnDescriptor column) throws IOException {
-    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+    requirePermission("addColumn", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -590,7 +597,7 @@ public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
       HColumnDescriptor descriptor) throws IOException {
-    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+    requirePermission("modifyColumn", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -600,7 +607,7 @@ public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName,
       byte[] col) throws IOException {
-    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+    requirePermission("deleteColumn", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -613,7 +620,7 @@ public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c, byte[] tableName)
       throws IOException {
-    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+    requirePermission("enableTable", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -627,7 +634,7 @@ public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c, byt
       throw new AccessDeniedException("Not allowed to disable "
           + AccessControlLists.ACL_TABLE_NAME_STR + " table.");
     }
-    requirePermission(tableName, null, null, Action.ADMIN, Action.CREATE);
+    requirePermission("disableTable", tableName, null, null, Action.ADMIN, Action.CREATE);
   }
 
   @Override
@@ -637,7 +644,7 @@ public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preMove(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo region,
       ServerName srcServer, ServerName destServer) throws IOException {
-    requirePermission(region.getTableName(), null, null, Action.ADMIN);
+    requirePermission("move", region.getTableName(), null, null, Action.ADMIN);
   }
 
   @Override
@@ -648,7 +655,7 @@ public void postMove(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preAssign(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo)
       throws IOException {
-    requirePermission(regionInfo.getTableName(), null, null, Action.ADMIN);
+    requirePermission("assign", regionInfo.getTableName(), null, null, Action.ADMIN);
   }
 
   @Override
@@ -658,7 +665,7 @@ public void postAssign(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo,
       boolean force) throws IOException {
-    requirePermission(regionInfo.getTableName(), null, null, Action.ADMIN);
+    requirePermission("unassign", regionInfo.getTableName(), null, null, Action.ADMIN);
   }
 
   @Override
@@ -668,7 +675,7 @@ public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preBalance(ObserverContext<MasterCoprocessorEnvironment> c)
       throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+    requirePermission("balance", Permission.Action.ADMIN);
   }
   @Override
   public void postBalance(ObserverContext<MasterCoprocessorEnvironment> c)
@@ -677,7 +684,7 @@ public void postBalance(ObserverContext<MasterCoprocessorEnvironment> c)
   @Override
   public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> c,
       boolean newValue) throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+    requirePermission("balanceSwitch", Permission.Action.ADMIN);
     return newValue;
   }
   @Override
@@ -687,13 +694,13 @@ public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> c,
   @Override
   public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> c)
       throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+    requirePermission("shutdown", Permission.Action.ADMIN);
   }
 
   @Override
   public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> c)
       throws IOException {
-    requirePermission(Permission.Action.ADMIN);
+    requirePermission("stopMaster", Permission.Action.ADMIN);
   }
 
   @Override
@@ -738,32 +745,32 @@ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
 
   @Override
   public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+    requirePermission("flush", getTableName(e.getEnvironment()), null, null, Action.ADMIN);
   }
 
   @Override
   public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+    requirePermission("split", getTableName(e.getEnvironment()), null, null, Action.ADMIN);
   }
 
   @Override
   public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
       final Store store, final InternalScanner scanner) throws IOException {
-    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+    requirePermission("compact", getTableName(e.getEnvironment()), null, null, Action.ADMIN);
     return scanner;
   }
 
   @Override
   public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> e,
       final Store store, final List<StoreFile> candidates) throws IOException {
-    requirePermission(getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+    requirePermission("compactSelection", getTableName(e.getEnvironment()), null, null, Action.ADMIN);
   }
 
   @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> c,
       final byte [] row, final byte [] family, final Result result)
       throws IOException {
-    requirePermission(TablePermission.Action.READ, c.getEnvironment(),
+    requirePermission("getClosestRowBefore", TablePermission.Action.READ, c.getEnvironment(),
         (family != null ? Lists.newArrayList(family) : null));
   }
 
@@ -776,7 +783,7 @@ public void preGet(final ObserverContext<RegionCoprocessorEnvironment> c,
       */
     RegionCoprocessorEnvironment e = c.getEnvironment();
     User requestUser = getActiveUser();
-    AuthResult authResult = permissionGranted(requestUser,
+    AuthResult authResult = permissionGranted("get", requestUser,
         TablePermission.Action.READ, e, get.getFamilyMap());
     if (!authResult.isAllowed()) {
       if (hasFamilyQualifierPermission(requestUser,
@@ -793,7 +800,7 @@ public void preGet(final ObserverContext<RegionCoprocessorEnvironment> c,
         } else {
           get.setFilter(filter);
         }
-        logResult(AuthResult.allow("Access allowed with filter", requestUser,
+        logResult(AuthResult.allow("get", "Access allowed with filter", requestUser,
             TablePermission.Action.READ, authResult.table));
       } else {
         logResult(authResult);
@@ -809,7 +816,7 @@ public void preGet(final ObserverContext<RegionCoprocessorEnvironment> c,
   @Override
   public boolean preExists(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Get get, final boolean exists) throws IOException {
-    requirePermission(TablePermission.Action.READ, c.getEnvironment(),
+    requirePermission("exists", TablePermission.Action.READ, c.getEnvironment(),
         get.familySet());
     return exists;
   }
@@ -818,7 +825,7 @@ public boolean preExists(final ObserverContext<RegionCoprocessorEnvironment> c,
   public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Put put, final WALEdit edit, final boolean writeToWAL)
       throws IOException {
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(),
+    requirePermission("put", TablePermission.Action.WRITE, c.getEnvironment(),
         put.getFamilyMap());
   }
 
@@ -834,7 +841,7 @@ public void postPut(final ObserverContext<RegionCoprocessorEnvironment> c,
   public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Delete delete, final WALEdit edit, final boolean writeToWAL)
       throws IOException {
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(),
+    requirePermission("delete", TablePermission.Action.WRITE, c.getEnvironment(),
         delete.getFamilyMap());
   }
 
@@ -854,8 +861,8 @@ public boolean preCheckAndPut(final ObserverContext<RegionCoprocessorEnvironment
       final WritableByteArrayComparable comparator, final Put put,
       final boolean result) throws IOException {
     Collection<byte[]> familyMap = Arrays.asList(new byte[][]{family});
-    requirePermission(TablePermission.Action.READ, c.getEnvironment(), familyMap);
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(), familyMap);
+    requirePermission("checkAndPut", TablePermission.Action.READ, c.getEnvironment(), familyMap);
+    requirePermission("checkAndPut", TablePermission.Action.WRITE, c.getEnvironment(), familyMap);
     return result;
   }
 
@@ -866,8 +873,8 @@ public boolean preCheckAndDelete(final ObserverContext<RegionCoprocessorEnvironm
       final WritableByteArrayComparable comparator, final Delete delete,
       final boolean result) throws IOException {
     Collection<byte[]> familyMap = Arrays.asList(new byte[][]{family});
-    requirePermission(TablePermission.Action.READ, c.getEnvironment(), familyMap);
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(), familyMap);
+    requirePermission("checkAndDelete", TablePermission.Action.READ, c.getEnvironment(), familyMap);
+    requirePermission("checkAndDelete", TablePermission.Action.WRITE, c.getEnvironment(), familyMap);
     return result;
   }
 
@@ -876,7 +883,7 @@ public long preIncrementColumnValue(final ObserverContext<RegionCoprocessorEnvir
       final byte [] row, final byte [] family, final byte [] qualifier,
       final long amount, final boolean writeToWAL)
       throws IOException {
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(),
+    requirePermission("incrementColumnValue", TablePermission.Action.WRITE, c.getEnvironment(),
         Arrays.asList(new byte[][]{family}));
     return -1;
   }
@@ -884,7 +891,7 @@ public long preIncrementColumnValue(final ObserverContext<RegionCoprocessorEnvir
   @Override
   public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> c, Append append)
       throws IOException {
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(), append.getFamilyMap());
+    requirePermission("append", TablePermission.Action.WRITE, c.getEnvironment(), append.getFamilyMap());
     return null;
   }
 
@@ -892,7 +899,7 @@ public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> c, Append
   public Result preIncrement(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Increment increment)
       throws IOException {
-    requirePermission(TablePermission.Action.WRITE, c.getEnvironment(),
+    requirePermission("increment", TablePermission.Action.WRITE, c.getEnvironment(),
         increment.getFamilyMap().keySet());
     return null;
   }
@@ -906,7 +913,7 @@ public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvir
       */
     RegionCoprocessorEnvironment e = c.getEnvironment();
     User user = getActiveUser();
-    AuthResult authResult = permissionGranted(user, TablePermission.Action.READ, e,
+    AuthResult authResult = permissionGranted("scannerOpen", user, TablePermission.Action.READ, e,
         scan.getFamilyMap());
     if (!authResult.isAllowed()) {
       if (hasFamilyQualifierPermission(user, TablePermission.Action.READ, e,
@@ -923,7 +930,7 @@ public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvir
         } else {
           scan.setFilter(filter);
         }
-        logResult(AuthResult.allow("Access allowed with filter", user,
+        logResult(AuthResult.allow("scannerOpen", "Access allowed with filter", user,
             TablePermission.Action.READ, authResult.table));
       } else {
         // no table/family level perms and no qualifier level perms, reject
@@ -999,7 +1006,7 @@ public void grant(UserPermission perm) throws IOException {
         LOG.debug("Received request to grant access permission " + perm.toString());
       }
 
-      requirePermission(perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN);
+      requirePermission("grant", perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN);
 
       AccessControlLists.addUserPermission(regionEnv.getConfiguration(), perm);
       if (AUDITLOG.isTraceEnabled()) {
@@ -1029,7 +1036,8 @@ public void revoke(UserPermission perm) throws IOException {
         LOG.debug("Received request to revoke access permission " + perm.toString());
       }
 
-      requirePermission(perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN);
+      requirePermission("revoke", perm.getTable(), perm.getFamily(),
+                        perm.getQualifier(), Action.ADMIN);
 
       AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), perm);
       if (AUDITLOG.isTraceEnabled()) {
@@ -1055,7 +1063,7 @@ public void revoke(byte[] user, TablePermission permission)
   public List<UserPermission> getUserPermissions(final byte[] tableName) throws IOException {
     // only allowed to be called on _acl_ region
     if (aclRegion) {
-      requirePermission(tableName, null, null, Action.ADMIN);
+      requirePermission("userPermissions", tableName, null, null, Action.ADMIN);
 
       List<UserPermission> perms = AccessControlLists.getUserPermissions(
         regionEnv.getConfiguration(), tableName);
@@ -1089,12 +1097,12 @@ public void checkPermissions(Permission[] permissions) throws IOException {
             }
           }
 
-          requirePermission(action, regionEnv, familyMap);
+          requirePermission("checkPermissions", action, regionEnv, familyMap);
         }
 
       } else {
         for (Permission.Action action : permission.getActions()) {
-          requirePermission(action);
+          requirePermission("checkPermissions", action);
         }
       }
     }

From caea1eef7c7d211c6ecc269e024beda46d9ec4bf Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 19 Dec 2012 18:01:35 +0000
Subject: [PATCH 0642/1540] HBASE-7199 hbck should check lingering reference
 hfile and have option to sideline them automatically

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423987 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |  2 +-
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 81 ++++++++++++++++++-
 .../hadoop/hbase/util/TestHBaseFsck.java      | 29 +++++++
 .../hbase/util/hbck/HbckTestingUtil.java      |  5 +-
 4 files changed, 112 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index af3ab839a2b4..40df89926c5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -320,7 +320,7 @@ public static boolean isReference(final Path p, final Matcher m) {
    * @return Calculated path to parent region file.
    * @throws IOException
    */
-  static Path getReferredToFile(final Path p) {
+  public static Path getReferredToFile(final Path p) {
     Matcher m = REF_NAME_PARSER.matcher(p.getName());
     if (m == null || !m.matches()) {
       LOG.warn("Failed match of store file name " + p.toString());
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 7fe81ca6f82c..8ef8be2e55ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -83,6 +83,7 @@
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
@@ -186,6 +187,7 @@ public class HBaseFsck extends Configured implements Tool {
   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
   private boolean fixSplitParents = false; // fix lingering split parents
+  private boolean fixReferenceFiles = false; // fix lingering reference store file
 
   // limit checking/fixes to listed tables, if empty attempt to check/fix all
   // -ROOT- and .META. are always checked
@@ -437,6 +439,8 @@ public int onlineHbck() throws IOException, KeeperException, InterruptedExceptio
       admin.setBalancerRunning(oldBalancer, false);
     }
 
+    offlineReferenceFileRepair();
+
     // Print table summary
     printTableSummary(tablesInfo);
     return errors.summarize();
@@ -592,6 +596,67 @@ private int restoreHdfsIntegrity() throws IOException, InterruptedException {
     return errors.getErrorList().size();
   }
 
+  /**
+   * Scan all the store file names to find any lingering reference files,
+   * which refer to some none-exiting files. If "fix" option is enabled,
+   * any lingering reference file will be sidelined if found.
+   * <p>
+   * Lingering reference file prevents a region from opening. It has to
+   * be fixed before a cluster can start properly.
+   */
+  private void offlineReferenceFileRepair() throws IOException {
+    Configuration conf = getConf();
+    Path hbaseRoot = FSUtils.getRootDir(conf);
+    FileSystem fs = hbaseRoot.getFileSystem(conf);
+    Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
+    for (Path path: allFiles.values()) {
+      boolean isReference = false;
+      try {
+        isReference = StoreFile.isReference(path);
+      } catch (Throwable t) {
+        // Ignore. Some files may not be store files at all.
+        // For example, files under .oldlogs folder in .META.
+        // Warning message is already logged by
+        // StoreFile#isReference.
+      }
+      if (!isReference) continue;
+
+      Path referredToFile = StoreFile.getReferredToFile(path);
+      if (fs.exists(referredToFile)) continue;  // good, expected
+
+      // Found a lingering reference file
+      errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
+        "Found lingering reference file " + path);
+      if (!shouldFixReferenceFiles()) continue;
+
+      // Now, trying to fix it since requested
+      boolean success = false;
+      String pathStr = path.toString();
+
+      // A reference file path should be like
+      // ${hbase.rootdir}/table_name/region_id/family_name/referred_file.region_name
+      // Up 3 directories to get the table folder.
+      // So the file will be sidelined to a similar folder structure.
+      int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
+      for (int i = 0; index > 0 && i < 3; i++) {
+        index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index);
+      }
+      if (index > 0) {
+        Path rootDir = getSidelineDir();
+        Path dst = new Path(rootDir, pathStr.substring(index));
+        fs.mkdirs(dst.getParent());
+        LOG.info("Trying to sildeline reference file"
+          + path + " to " + dst);
+        setShouldRerun();
+
+        success = fs.rename(path, dst);
+      }
+      if (!success) {
+        LOG.error("Failed to sideline reference file " + path);
+      }
+    }
+  }
+
   /**
    * TODO -- need to add tests for this.
    */
@@ -2779,7 +2844,7 @@ public static enum ERROR_CODE {
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
-      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE
+      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE
     }
     public void clear();
     public void report(String message);
@@ -3211,6 +3276,14 @@ boolean shouldFixSplitParents() {
     return fixSplitParents;
   }
 
+  public void setFixReferenceFiles(boolean shouldFix) {
+    fixReferenceFiles = shouldFix;
+  }
+
+  boolean shouldFixReferenceFiles() {
+    return fixReferenceFiles;
+  }
+
   public boolean shouldIgnorePreCheckPermission() {
     return ignorePreCheckPermission;
   }
@@ -3322,6 +3395,7 @@ protected HBaseFsck printUsageAndExit() {
     System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
     System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
     System.err.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
+    System.err.println("   -fixReferenceFiles  Try to offline lingering reference store files");
 
     System.err.println("");
     System.err.println("  Datafile Repair options: (expert features, use with caution!)");
@@ -3331,7 +3405,7 @@ protected HBaseFsck printUsageAndExit() {
     System.err.println("");
     System.err.println("  Metadata Repair shortcuts");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
-        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
+        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles");
     System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
 
     setRetCode(-2);
@@ -3438,6 +3512,8 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         checkCorruptHFiles = true;
       } else if (cmd.equals("-sidelineCorruptHFiles")) {
         sidelineCorruptHFiles = true;
+      } else if (cmd.equals("-fixReferenceFiles")) {
+        setFixReferenceFiles(true);
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
@@ -3450,6 +3526,7 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setSidelineBigOverlaps(true);
         setFixSplitParents(false);
         setCheckHdfs(true);
+        setFixReferenceFiles(true);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
         setFixHdfsHoles(true);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index e0d3c1959a1b..3771d9a6bd0a 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1657,6 +1657,35 @@ protected void checkRegionDir(Path p) throws IOException {
     doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
   }
 
+  /**
+   * Test fixing lingering reference file.
+   */
+  @Test
+  public void testLingeringReferenceFile() throws Exception {
+    String table = "testLingeringReferenceFile";
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // Mess it up by creating a fake reference file
+      FileSystem fs = FileSystem.get(conf);
+      Path tableDir= FSUtils.getTablePath(FSUtils.getRootDir(conf), table);
+      Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
+      Path famDir = new Path(regionDir, FAM_STR);
+      Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
+      fs.create(fakeReferenceFile);
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
+      // fix reference file
+      doFsck(conf, true);
+      // check that reference file fixed
+      assertNoErrors(doFsck(conf, false));
+    } finally {
+      deleteTable(table);
+    }
+  }
+
   /**
    * Test pluggable error reporter. It can be plugged in
    * from system property or configuration.
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 343402196f89..99f4f9b8a1c8 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -38,13 +38,13 @@ public static HBaseFsck doFsck(
 
   public static HBaseFsck doFsck(
       Configuration conf, boolean fix, String table) throws Exception {
-    return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, table);
+    return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, fix, table);
   }
 
   public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
       boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
       boolean fixHdfsOrphans, boolean fixTableOrphans, boolean fixVersionFile,
-      String table) throws Exception {
+      boolean fixReferenceFiles, String table) throws Exception {
     HBaseFsck fsck = new HBaseFsck(conf, exec);
     fsck.connect();
     fsck.setDisplayFullReport(); // i.e. -details
@@ -56,6 +56,7 @@ public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
     fsck.setFixHdfsOrphans(fixHdfsOrphans);
     fsck.setFixTableOrphans(fixTableOrphans);
     fsck.setFixVersionFile(fixVersionFile);
+    fsck.setFixReferenceFiles(fixReferenceFiles);
     if (table != null) {
       fsck.includeTable(table);
     }

From 81d81f8aa28638060b91f0bc75dd2a5b59325828 Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Wed, 19 Dec 2012 18:18:42 +0000
Subject: [PATCH 0643/1540] HBASE-7357  Use hbase.security.authentication for
 client / server security negotiation

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1423997 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/SecureClient.java    |  2 +-
 .../org/apache/hadoop/hbase/ipc/SecureServer.java    |  2 +-
 .../java/org/apache/hadoop/hbase/security/User.java  | 12 ++++++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 4c7bcbb85932..c0d10e019572 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -98,7 +98,7 @@ public SecureConnection(ConnectionId remoteId) throws IOException {
 
       User ticket = remoteId.getTicket();
       Class<?> protocol = remoteId.getProtocol();
-      this.useSasl = User.isSecurityEnabled();
+      this.useSasl = User.isHBaseSecurityEnabled(conf);
       if (useSasl && protocol != null) {
         TokenInfo tokenInfo = protocol.getAnnotation(TokenInfo.class);
         if (tokenInfo != null) {
diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index f9497a174d99..d1b89610e009 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -684,7 +684,7 @@ protected SecureServer(String bindAddress, int port,
         conf, serverName, highPriorityLevel);
     this.authorize =
       conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false);
-    this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+    this.isSecurityEnabled = User.isHBaseSecurityEnabled(this.conf);
 
     if (isSecurityEnabled) {
       HBaseSaslRpcServer.init(conf);
diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index eb516a4444a7..f15c470012cc 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -22,6 +22,7 @@
 
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.util.Methods;
 import org.apache.hadoop.mapred.JobConf;
@@ -216,12 +217,15 @@ public static boolean isSecurityEnabled() {
   }
 
   /**
-   * Returns whether or not secure authentication is enabled for HBase
-   * (whether <code>hbase.security.authentication</code> is set to
-   * <code>kerberos</code>.
+   * Returns whether or not secure authentication is enabled for HBase.  Note that
+   * HBase security requires HDFS security to provide any guarantees, so this requires that
+   * both <code>hbase.security.authentication</code> and <code>hadoop.security.authentication</code>
+   * are set to <code>kerberos</code>.
    */
   public static boolean isHBaseSecurityEnabled(Configuration conf) {
-    return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY));
+    return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY)) &&
+        "kerberos".equalsIgnoreCase(
+            conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION));
   }
 
   /* Concrete implementations */

From d9624e3e12f4728a6acf389451f7261414c663ba Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 19 Dec 2012 18:44:38 +0000
Subject: [PATCH 0644/1540] HBASE-7374. Expose table operations for
 coprocessors by way of MasterServices

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424017 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/MasterServices.java   | 60 ++++++++++++++++++-
 .../hbase/master/TestCatalogJanitor.java      | 32 ++++++++--
 2 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 8be3c75ac20e..eb0e5400acd0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -21,13 +21,12 @@
 
 import java.io.IOException;
 
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableDescriptors;
-import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
-import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 
 /**
  * Services Master supplies
@@ -70,6 +69,63 @@ public interface MasterServices extends Server {
   public void createTable(HTableDescriptor desc, byte [][] splitKeys)
       throws IOException;
 
+  /**
+   * Delete a table
+   * @param tableName The table name
+   * @throws IOException
+   */
+  public void deleteTable(final byte[] tableName) throws IOException;
+
+  /**
+   * Modify the descriptor of an existing table
+   * @param tableName The table name
+   * @param descriptor The updated table descriptor
+   * @throws IOException
+   */
+  public void modifyTable(final byte[] tableName, final HTableDescriptor descriptor)
+      throws IOException;
+
+  /**
+   * Enable an existing table
+   * @param tableName The table name
+   * @throws IOException
+   */
+  public void enableTable(final byte[] tableName) throws IOException;
+
+  /**
+   * Disable an existing table
+   * @param tableName The table name
+   * @throws IOException
+   */
+  public void disableTable(final byte[] tableName) throws IOException;
+
+  /**
+   * Add a new column to an existing table
+   * @param tableName The table name
+   * @param column The column definition
+   * @throws IOException
+   */
+  public void addColumn(final byte[] tableName, final HColumnDescriptor column)
+      throws IOException;
+
+  /**
+   * Modify the column descriptor of an existing column in an existing table
+   * @param tableName The table name
+   * @param descriptor The updated column definition
+   * @throws IOException
+   */
+  public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor)
+      throws IOException;
+
+  /**
+   * Delete a column from an existing table
+   * @param tableName The table name
+   * @param columnName The column name
+   * @throws IOException
+   */
+  public void deleteColumn(final byte[] tableName, final byte[] columnName)
+      throws IOException;
+
   /**
    * @return Return table descriptors implementation.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index c83d4baa9693..1f75842d5c08 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -241,13 +241,11 @@ public TableDescriptors getTableDescriptors() {
       return new TableDescriptors() {
         @Override
         public HTableDescriptor remove(String tablename) throws IOException {
-          // TODO Auto-generated method stub
           return null;
         }
         
         @Override
         public Map<String, HTableDescriptor> getAll() throws IOException {
-          // TODO Auto-generated method stub
           return null;
         }
         
@@ -265,8 +263,6 @@ public HTableDescriptor get(String tablename)
         
         @Override
         public void add(HTableDescriptor htd) throws IOException {
-          // TODO Auto-generated method stub
-          
         }
       };
     }
@@ -285,6 +281,34 @@ public MasterCoprocessorHost getCoprocessorHost() {
     public <T extends CoprocessorProtocol> boolean registerProtocol(Class<T> protocol, T handler) {
       return false;
     }
+
+    @Override
+    public void deleteTable(byte[] tableName) throws IOException {
+    }
+
+    @Override
+    public void modifyTable(byte[] tableName, HTableDescriptor descriptor) throws IOException {
+    }
+
+    @Override
+    public void enableTable(byte[] tableName) throws IOException {
+    }
+
+    @Override
+    public void disableTable(byte[] tableName) throws IOException {
+    }
+
+    @Override
+    public void addColumn(byte[] tableName, HColumnDescriptor column) throws IOException {
+    }
+
+    @Override
+    public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor) throws IOException {
+    }
+
+    @Override
+    public void deleteColumn(byte[] tableName, byte[] columnName) throws IOException {
+    }
   }
 
   @Test

From f429a5b112833550dd3fa54989f1fc920368ebb6 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 19 Dec 2012 20:29:25 +0000
Subject: [PATCH 0645/1540] HBASE-7331. Add access control for region open and
 close, and stopping the regionserver (Vandana Ayyalasomayajula)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424085 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 120 ++++++++++++++----
 .../hbase/security/access/SecureTestUtil.java |   6 +-
 .../security/access/TestAccessController.java |  82 ++++++++++++
 .../hbase/coprocessor/BaseRegionObserver.java |  22 +++-
 .../hbase/coprocessor/CoprocessorHost.java    |   8 +-
 .../hbase/coprocessor/RegionObserver.java     |  57 ++++++++-
 .../RegionServerCoprocessorEnvironment.java   |  29 +++++
 .../coprocessor/RegionServerObserver.java     |  35 +++++
 .../hbase/regionserver/HRegionServer.java     |  46 ++++++-
 .../regionserver/RegionCoprocessorHost.java   |  37 +++++-
 .../RegionServerCoprocessorHost.java          | 104 +++++++++++++++
 11 files changed, 503 insertions(+), 43 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 891c1e58b6b5..c9e4dee12b1d 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -46,6 +46,8 @@
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
@@ -62,6 +64,7 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.security.access.Permission.Action;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
@@ -101,7 +104,7 @@
  * </p>
  */
 public class AccessController extends BaseRegionObserver
-    implements MasterObserver, AccessControllerProtocol {
+    implements MasterObserver, RegionServerObserver, AccessControllerProtocol {
   /**
    * Represents the result of an authorization check for logging and error
    * reporting.
@@ -518,17 +521,31 @@ private boolean hasFamilyQualifierPermission(User user,
 
   /* ---- MasterObserver implementation ---- */
   public void start(CoprocessorEnvironment env) throws IOException {
-    // if running on HMaster
+
+    ZooKeeperWatcher zk = null;
     if (env instanceof MasterCoprocessorEnvironment) {
-      MasterCoprocessorEnvironment e = (MasterCoprocessorEnvironment)env;
-      this.authManager = TableAuthManager.get(
-          e.getMasterServices().getZooKeeper(),
-          e.getConfiguration());
+      // if running on HMaster
+      MasterCoprocessorEnvironment mEnv = (MasterCoprocessorEnvironment) env;
+      zk = mEnv.getMasterServices().getZooKeeper();      
+    } else if (env instanceof RegionServerCoprocessorEnvironment) {      
+      RegionServerCoprocessorEnvironment rsEnv = (RegionServerCoprocessorEnvironment) env;
+      zk = rsEnv.getRegionServerServices().getZooKeeper();      
+    } else if (env instanceof RegionCoprocessorEnvironment) {
+      // if running at region
+      regionEnv = (RegionCoprocessorEnvironment) env;
+      zk = regionEnv.getRegionServerServices().getZooKeeper();
     }
 
-    // if running at region
-    if (env instanceof RegionCoprocessorEnvironment) {
-      regionEnv = (RegionCoprocessorEnvironment)env;
+    // If zk is null or IOException while obtaining auth manager,
+    // throw RuntimeException so that the coprocessor is unloaded.
+    if (zk != null) {
+      try {
+        this.authManager = TableAuthManager.get(zk, env.getConfiguration());
+      } catch (IOException ioe) {
+        throw new RuntimeException("Error obtaining TableAuthManager", ioe);
+      }
+    } else {
+      throw new RuntimeException("Error obtaining TableAuthManager, zk found null.");
     }
   }
 
@@ -714,27 +731,34 @@ public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
   /* ---- RegionObserver implementation ---- */
 
   @Override
-  public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
-    RegionCoprocessorEnvironment e = c.getEnvironment();
-    final HRegion region = e.getRegion();
+  public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+    RegionCoprocessorEnvironment env = e.getEnvironment();
+    final HRegion region = env.getRegion();
     if (region == null) {
-      LOG.error("NULL region from RegionCoprocessorEnvironment in postOpen()");
+      LOG.error("NULL region from RegionCoprocessorEnvironment in preOpen()");
       return;
+    } else {
+      HRegionInfo regionInfo = region.getRegionInfo();
+      if (isSpecialTable(regionInfo)) {
+        isSystemOrSuperUser(regionEnv.getConfiguration());
+      } else {
+        requirePermission("open", Action.ADMIN);
+      }
     }
+  }
 
-    try {
-      this.authManager = TableAuthManager.get(
-          e.getRegionServerServices().getZooKeeper(),
-          regionEnv.getConfiguration());
-    } catch (IOException ioe) {
-      // pass along as a RuntimeException, so that the coprocessor is unloaded
-      throw new RuntimeException("Error obtaining TableAuthManager", ioe);
+  @Override
+  public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
+    RegionCoprocessorEnvironment env = c.getEnvironment();
+    final HRegion region = env.getRegion();
+    if (region == null) {
+      LOG.error("NULL region from RegionCoprocessorEnvironment in postOpen()");
+      return;
     }
-
     if (AccessControlLists.isAclRegion(region)) {
       aclRegion = true;
       try {
-        initialize(e);
+        initialize(env);
       } catch (IOException ex) {
         // if we can't obtain permissions, it's better to fail
         // than perform checks incorrectly
@@ -1135,4 +1159,56 @@ private byte[] getTableName(RegionCoprocessorEnvironment e) {
     }
     return tableName;
   }
+
+
+  @Override
+  public void preClose(ObserverContext<RegionCoprocessorEnvironment> e, boolean abortRequested)
+      throws IOException {
+    requirePermission("close", Permission.Action.ADMIN);
+  }
+
+  @Override
+  public void preLockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+      byte[] row) throws IOException {
+    requirePermission("lockRow", getTableName(ctx.getEnvironment()), null, null,
+      Permission.Action.WRITE, Permission.Action.CREATE);
+  }
+
+  @Override
+  public void preUnlockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+      long lockId) throws IOException {
+    requirePermission("unlockRow", getTableName(ctx.getEnvironment()), null, null,
+      Permission.Action.WRITE, Permission.Action.CREATE);
+  }
+
+  private void isSystemOrSuperUser(Configuration conf) throws IOException {
+    User user = User.getCurrent();
+    if (user == null) {
+      throw new IOException("Unable to obtain the current user, "
+          + "authorization checks for internal operations will not work correctly!");
+    }
+
+    String currentUser = user.getShortName();
+    List<String> superusers = Lists.asList(currentUser,
+      conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]));
+
+    User activeUser = getActiveUser();
+    if (!(superusers.contains(activeUser.getShortName()))) {
+      throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null")
+          + "is not system or super user.");
+    }
+  }
+
+  private boolean isSpecialTable(HRegionInfo regionInfo) {
+    byte[] tableName = regionInfo.getTableName();
+    return tableName.equals(AccessControlLists.ACL_TABLE_NAME)
+      || tableName.equals(Bytes.toBytes("-ROOT-"))
+      || tableName.equals(Bytes.toBytes(".META."));
+  }
+
+  @Override
+  public void preStopRegionServer(ObserverContext<RegionServerCoprocessorEnvironment> env)
+      throws IOException {
+    requirePermission("stop", Permission.Action.ADMIN);
+  }
 }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
index 1a087b6a9268..4aa0e18f401f 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.ipc.SecureRpcEngine;
 import org.apache.hadoop.hbase.security.User;
 
@@ -32,8 +33,9 @@ public static void enableSecurity(Configuration conf) throws IOException {
     conf.set("hadoop.security.authorization", "false");
     conf.set("hadoop.security.authentication", "simple");
     conf.set("hbase.rpc.engine", SecureRpcEngine.class.getName());
-    conf.set("hbase.coprocessor.master.classes", AccessController.class.getName());
-    conf.set("hbase.coprocessor.region.classes", AccessController.class.getName());
+    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, AccessController.class.getName());
+    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName());
+    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, AccessController.class.getName());
     // add the process running user to superusers
     String currentUser = User.getCurrent().getName();
     conf.set("hbase.superuser", "admin,"+currentUser);
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 765f0af805cd..ad6dda4ccf32 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.UnknownRowLockException;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
@@ -51,9 +52,11 @@
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.security.access.Permission.Action;
@@ -79,6 +82,8 @@ public class TestAccessController {
   private static User USER_ADMIN;
   // user with rw permissions
   private static User USER_RW;
+  // user with rw permissions on table.
+  private static User USER_RW_ON_TABLE;
   // user with read-only permissions
   private static User USER_RO;
   // user is table owner. will have all permissions on table
@@ -93,6 +98,7 @@ public class TestAccessController {
 
   private static MasterCoprocessorEnvironment CP_ENV;
   private static RegionCoprocessorEnvironment RCP_ENV;
+  private static RegionServerCoprocessorEnvironment RSCP_ENV;
   private static AccessController ACCESS_CONTROLLER;
 
   @BeforeClass
@@ -107,6 +113,10 @@ public static void setupBeforeClass() throws Exception {
     ACCESS_CONTROLLER = (AccessController) cpHost.findCoprocessor(AccessController.class.getName());
     CP_ENV = cpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER,
       Coprocessor.PRIORITY_HIGHEST, 1, conf);
+    RegionServerCoprocessorHost rsHost = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0)
+        .getCoprocessorHost();
+    RSCP_ENV = rsHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER, 
+      Coprocessor.PRIORITY_HIGHEST, 1, conf);
 
     // Wait for the ACL table to become available
     TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
@@ -116,6 +126,7 @@ public static void setupBeforeClass() throws Exception {
     USER_ADMIN = User.createUserForTesting(conf, "admin2", new String[0]);
     USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
     USER_RO = User.createUserForTesting(conf, "rouser", new String[0]);
+    USER_RW_ON_TABLE = User.createUserForTesting(conf, "rwuser_1", new String[0]);
     USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
     USER_CREATE = User.createUserForTesting(conf, "tbl_create", new String[0]);
     USER_NONE = User.createUserForTesting(conf, "nouser", new String[0]);
@@ -148,6 +159,9 @@ public static void setupBeforeClass() throws Exception {
 
     protocol.grant(new UserPermission(Bytes.toBytes(USER_CREATE.getShortName()), TEST_TABLE, null,
         Permission.Action.CREATE));
+    
+    protocol.grant(new UserPermission(Bytes.toBytes(USER_RW_ON_TABLE.getShortName()), TEST_TABLE,
+      null, Permission.Action.READ, Permission.Action.WRITE));
   }
 
   @AfterClass
@@ -161,6 +175,8 @@ public void verifyAllowed(User user, PrivilegedExceptionAction... actions) throw
         user.runAs(action);
       } catch (AccessDeniedException ade) {
         fail("Expected action to pass for user '" + user.getShortName() + "' but was denied");
+      } catch (UnknownRowLockException exp){
+        //expected
       }
     }
   }
@@ -1271,4 +1287,70 @@ public Void run() throws Exception {
     }
 
   }
+
+  @Test
+  public void testLockAction() throws Exception {
+    PrivilegedExceptionAction lockAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preLockRow(ObserverContext.createAndPrepare(RCP_ENV, null), null,
+          Bytes.toBytes("random_row"));
+        return null;
+      }
+    };
+    verifyAllowed(lockAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE, USER_RW_ON_TABLE);
+    verifyDenied(lockAction, USER_RO, USER_RW, USER_NONE);
+  }
+
+  @Test
+  public void testUnLockAction() throws Exception {
+    PrivilegedExceptionAction unLockAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preUnlockRow(ObserverContext.createAndPrepare(RCP_ENV, null), null,
+          123456);
+        return null;
+      }
+    };
+    verifyAllowed(unLockAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW_ON_TABLE);
+    verifyDenied(unLockAction, USER_NONE, USER_RO, USER_RW);
+  }
+
+  @Test
+  public void testStopRegionServer() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preStopRegionServer(ObserverContext.createAndPrepare(RSCP_ENV, null));
+        return null;
+      }
+    };
+
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE);
+  }
+
+  @Test
+  public void testOpenRegion() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preOpen(ObserverContext.createAndPrepare(RCP_ENV, null));
+        return null;
+      }
+    };
+
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
+  }
+
+  @Test
+  public void testCloseRegion() throws Exception {
+    PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preClose(ObserverContext.createAndPrepare(RCP_ENV, null), false);
+        return null;
+      }
+    };
+
+    verifyAllowed(action, SUPERUSER, USER_ADMIN);
+    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
+  }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 5889f320ea7d..abc972ce7113 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -58,14 +58,14 @@ public void start(CoprocessorEnvironment e) throws IOException { }
   public void stop(CoprocessorEnvironment e) throws IOException { }
 
   @Override
-  public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) { }
+  public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException { }
 
   @Override
   public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) { }
 
   @Override
-  public void preClose(ObserverContext<RegionCoprocessorEnvironment> e,
-      boolean abortRequested) { }
+  public void preClose(ObserverContext<RegionCoprocessorEnvironment> c, boolean abortRequested)
+      throws IOException { }
 
   @Override
   public void postClose(ObserverContext<RegionCoprocessorEnvironment> e,
@@ -320,4 +320,20 @@ public boolean postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> c
     List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException {
     return hasLoaded;
   }
+
+  @Override
+  public void preLockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+      byte[] row) throws IOException { }
+
+  @Override
+  public void preUnlockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+      long lockId) throws IOException { }
+
+  @Override
+  public void postLockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+      byte[] row) throws IOException { }
+
+  @Override
+  public void postUnlockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+      long lockId) throws IOException { }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 44c497b5c204..bc2927d7fc71 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -58,11 +58,13 @@
  */
 public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
   public static final String REGION_COPROCESSOR_CONF_KEY =
-      "hbase.coprocessor.region.classes";
+    "hbase.coprocessor.region.classes";
+  public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
+    "hbase.coprocessor.regionserver.classes";
   public static final String USER_REGION_COPROCESSOR_CONF_KEY =
-      "hbase.coprocessor.user.region.classes";
+    "hbase.coprocessor.user.region.classes";
   public static final String MASTER_COPROCESSOR_CONF_KEY =
-      "hbase.coprocessor.master.classes";
+    "hbase.coprocessor.master.classes";
   public static final String WAL_COPROCESSOR_CONF_KEY =
     "hbase.coprocessor.wal.classes";
 
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 3dc2909bb6c0..fa65fed45862 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -55,8 +55,9 @@ public interface RegionObserver extends Coprocessor {
   /**
    * Called before the region is reported as open to the master.
    * @param c the environment provided by the region server
+   * @throws IOException if an error occurred on the coprocessor
    */
-  void preOpen(final ObserverContext<RegionCoprocessorEnvironment> c);
+  void preOpen(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
 
   /**
    * Called after the region is reported as open to the master.
@@ -227,9 +228,10 @@ void postSplit(final ObserverContext<RegionCoprocessorEnvironment> c, final HReg
    * Called before the region is reported as closed to the master.
    * @param c the environment provided by the region server
    * @param abortRequested true if the region server is aborting
+   * @throws IOException 
    */
   void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
-      boolean abortRequested);
+      boolean abortRequested) throws IOException;
 
   /**
    * Called after the region is reported as closed to the master.
@@ -771,4 +773,55 @@ void preBulkLoadHFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
    */
   boolean postBulkLoadHFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
     List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException;
+  
+  /**
+   * Called before locking a row. 
+   *
+   * @param ctx 
+   * @param regionName 
+   * @param row 
+   * @throws IOException Signals that an I/O exception has occurred.
+   * @deprecated Will be removed in 0.96
+   */
+  @Deprecated
+  void preLockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    final byte[] regionName, final byte[] row) throws IOException; 
+  
+  /**
+   * Called after locking a row.
+   *
+   * @param ctx 
+   * @param regionName the region name
+   * @param row 
+   * @throws IOException Signals that an I/O exception has occurred.
+   * @deprecated Will be removed in 0.96
+   */
+  @Deprecated
+  void postLockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    final byte[] regionName, final byte[] row) throws IOException;
+  
+  /**
+   * Called before unlocking a row.
+   *
+   * @param ctx 
+   * @param regionName 
+   * @param lockId the lock id
+   * @throws IOException Signals that an I/O exception has occurred.
+   * @deprecated Will be removed in 0.96
+   */
+  @Deprecated
+  void preUnlockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    final byte[] regionName, final long lockId) throws IOException; 
+  
+  /**
+   * Called after unlocking a row.
+   * @param ctx 
+   * @param regionName the region name
+   * @param lockId the lock id
+   * @throws IOException Signals that an I/O exception has occurred.
+   * @deprecated Will be removed in 0.96
+   */
+  @Deprecated
+  void postUnlockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+    final byte[] regionName, final long lockId) throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
new file mode 100644
index 000000000000..6a07b8b4dc29
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironment {
+
+  /** @return reference to the HMaster services */
+  RegionServerServices getRegionServerServices();
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
new file mode 100644
index 000000000000..044d086587d3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.Coprocessor;
+
+public interface RegionServerObserver extends Coprocessor {
+
+  /**
+   * Called before stopping region server.
+   * @param env An instance of RegionServerCoprocessorEnvironment
+   * @throws IOException Signals that an I/O exception has occurred.
+   */
+  void preStopRegionServer(final ObserverContext<RegionServerCoprocessorEnvironment> env)
+      throws IOException;
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5296e6ac5a59..b86d05872aa2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -236,7 +236,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   // Server to handle client requests. Default access so can be accessed by
   // unit tests.
   RpcServer rpcServer;
-  
+
   // Server to handle client requests.
   private HBaseServer server;  
 
@@ -366,6 +366,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
    */
   private ClusterId clusterId = null;
 
+  private RegionServerCoprocessorHost rsHost;
+
   /**
    * Starts a HRegionServer at the default location
    *
@@ -1020,6 +1022,7 @@ protected void handleReportForDutyResponse(final MapWritable c)
       // Init in here rather than in constructor after thread name has been set
       this.metrics = new RegionServerMetrics();
       this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(this);
+      this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
       startServiceThreads();
       LOG.info("Serving as " + this.serverNameFromMasterPOV +
         ", RPC listening on " + this.isa +
@@ -1027,6 +1030,7 @@ protected void handleReportForDutyResponse(final MapWritable c)
         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
       isOnline = true;
     } catch (Throwable e) {
+      LOG.warn("Exception in region server : ", e);
       this.isOnline = false;
       stop("Failed initialization");
       throw convertThrowableToIOE(cleanup(e, "Failed init"),
@@ -1582,6 +1586,7 @@ public void uncaughtException(Thread t, Throwable e) {
     this.splitLogWorker = new SplitLogWorker(this.zooKeeper,
         this.getConfiguration(), this.getServerName().toString());
     splitLogWorker.start();
+    
   }
 
   /**
@@ -1649,10 +1654,15 @@ public CatalogTracker getCatalogTracker() {
 
   @Override
   public void stop(final String msg) {
-    this.stopped = true;
-    LOG.info("STOPPED: " + msg);
-    // Wakes run() if it is sleeping
-    sleeper.skipSleepCycle();
+    try {
+      this.rsHost.preStop(msg);
+      this.stopped = true;
+      LOG.info("STOPPED: " + msg);
+      // Wakes run() if it is sleeping
+      sleeper.skipSleepCycle();
+    } catch (IOException exp) {
+      LOG.warn("The region server did not stop", exp);
+    }
   }
 
   public void waitForServerOnline(){
@@ -2623,6 +2633,9 @@ public long lockRow(byte[] regionName, byte[] row) throws IOException {
     requestCount.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().preLockRow(regionName, row);
+      }
       Integer r = region.obtainRowLock(row);
       long lockId = addRowLock(r, region);
       LOG.debug("Row lock " + lockId + " explicitly acquired by client");
@@ -2687,6 +2700,9 @@ public void unlockRow(byte[] regionName, long lockId) throws IOException {
     requestCount.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().preUnLockRow(regionName, lockId);
+      }
       String lockName = String.valueOf(lockId);
       Integer r = rowlocks.remove(lockName);
       if (r == null) {
@@ -2863,6 +2879,11 @@ protected boolean closeRegion(HRegionInfo region, final boolean zk,
     final int versionOfClosingNode)
   throws IOException {
     checkOpen();
+    //Check for permissions to close.
+    HRegion actualRegion = this.getFromOnlineRegions(region.getEncodedName());
+    if (actualRegion.getCoprocessorHost() != null) {
+      actualRegion.getCoprocessorHost().preClose(false);
+    }
     LOG.info("Received close region: " + region.getRegionNameAsString() +
       ". Version of ZK closing node:" + versionOfClosingNode);
     boolean hasit = this.onlineRegions.containsKey(region.getEncodedName());
@@ -2910,6 +2931,17 @@ protected boolean closeRegion(HRegionInfo region, final boolean abort,
    */
   protected boolean closeRegion(HRegionInfo region, final boolean abort,
       final boolean zk, final int versionOfClosingNode) {
+    
+    HRegion actualRegion = this.getFromOnlineRegions(region.getEncodedName());
+    if ((actualRegion != null) && (actualRegion.getCoprocessorHost() !=null)){
+      try {
+        actualRegion.getCoprocessorHost().preClose(abort);
+      } catch (IOException e) {
+        LOG.warn(e);
+        return false;
+      }
+    }
+    
     if (this.regionsInTransitionInRS.containsKey(region.getEncodedNameAsBytes())) {
       LOG.warn("Received close for region we are already opening or closing; " +
           region.getEncodedName());
@@ -3610,6 +3642,10 @@ public ZooKeeperWatcher getZooKeeperWatcher() {
     return this.zooKeeper;
   }
 
+  public RegionServerCoprocessorHost getCoprocessorHost(){
+    return this.rsHost;
+  }
+
 
   public ConcurrentSkipListMap<byte[], Boolean> getRegionsInTransitionInRS() {
     return this.regionsInTransitionInRS;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 0f61539b732d..27e194f7c744 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -265,7 +264,7 @@ private void handleCoprocessorThrowableNoRethrow(
   /**
    * Invoked before a region open
    */
-  public void preOpen() {
+  public void preOpen(){
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -285,7 +284,7 @@ public void preOpen() {
   /**
    * Invoked after a region open
    */
-  public void postOpen() {
+  public void postOpen(){
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -306,7 +305,7 @@ public void postOpen() {
    * Invoked before a region is closed
    * @param abortRequested true if the server is aborting
    */
-  public void preClose(boolean abortRequested) {
+  public void preClose(boolean abortRequested) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -314,7 +313,7 @@ public void preClose(boolean abortRequested) {
         try {
           ((RegionObserver)env.getInstance()).preClose(ctx, abortRequested);
         } catch (Throwable e) {
-          handleCoprocessorThrowableNoRethrow(env, e);
+          handleCoprocessorThrowable(env, e);
         }
       }
     }
@@ -324,7 +323,7 @@ public void preClose(boolean abortRequested) {
    * Invoked after a region is closed
    * @param abortRequested true if the server is aborting
    */
-  public void postClose(boolean abortRequested) {
+  public void postClose(boolean abortRequested){
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
@@ -1483,5 +1482,31 @@ public boolean postBulkLoadHFile(List<Pair<byte[], String>> familyPaths, boolean
 
     return hasLoaded;
   }
+  
+  public void preLockRow(byte[] regionName, byte[] row) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        ((RegionObserver) env.getInstance()).preLockRow(ctx, regionName, row);
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preUnLockRow(byte[] regionName, long lockId) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        ((RegionObserver) env.getInstance()).preUnlockRow(ctx, regionName, lockId);
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
 
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
new file mode 100644
index 000000000000..2302f9e9698b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
+
+public class RegionServerCoprocessorHost extends
+    CoprocessorHost<RegionServerCoprocessorHost.RegionServerEnvironment> {
+
+  private RegionServerServices rsServices;
+
+  public RegionServerCoprocessorHost(RegionServerServices rsServices, Configuration conf) {
+    this.rsServices = rsServices;
+    this.conf = conf;
+    // load system default cp's from configuration.
+    loadSystemCoprocessors(conf, REGIONSERVER_COPROCESSOR_CONF_KEY);
+  }
+
+  @Override
+  public RegionServerEnvironment createEnvironment(Class<?> implClass, Coprocessor instance,
+      int priority, int sequence, Configuration conf) {
+    return new RegionServerEnvironment(implClass, instance, priority, sequence, conf,
+        this.rsServices);
+  }
+
+  public void preStop(String message) throws IOException {
+    ObserverContext<RegionServerCoprocessorEnvironment> ctx = null;
+    for (RegionServerEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionServerObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        ((RegionServerObserver) env.getInstance()).preStopRegionServer(ctx);
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Coprocessor environment extension providing access to region server related services.
+   */
+  static class RegionServerEnvironment extends CoprocessorHost.Environment implements
+      RegionServerCoprocessorEnvironment {
+
+    private RegionServerServices regionServerServices;
+
+    public RegionServerEnvironment(final Class<?> implClass, final Coprocessor impl,
+        final int priority, final int seq, final Configuration conf,
+        final RegionServerServices services) {
+      super(impl, priority, seq, conf);
+      this.regionServerServices = services;
+    }
+
+    @Override
+    public RegionServerServices getRegionServerServices() {
+      return regionServerServices;
+    }
+  }
+
+  /**
+   * Environment priority comparator. Coprocessors are chained in sorted order.
+   */
+  static class EnvironmentPriorityComparator implements Comparator<CoprocessorEnvironment> {
+    public int compare(final CoprocessorEnvironment env1, final CoprocessorEnvironment env2) {
+      if (env1.getPriority() < env2.getPriority()) {
+        return -1;
+      } else if (env1.getPriority() > env2.getPriority()) {
+        return 1;
+      }
+      if (env1.getLoadSequence() < env2.getLoadSequence()) {
+        return -1;
+      } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
+        return 1;
+      }
+      return 0;
+    }
+  }
+
+}

From 64a8524458b32a09e814324d51a5314fdfbd7251 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Wed, 19 Dec 2012 22:22:58 +0000
Subject: [PATCH 0646/1540] HBASE-5778  Fix HLog compression's
 incompatibilities

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424172 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/Compressor.java    |   7 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |  10 +-
 .../wal/SequenceFileLogReader.java            |  23 ++-
 .../ReplicationHLogReaderManager.java         | 136 ++++++++++++++++++
 .../regionserver/ReplicationSource.java       |  64 ++++-----
 .../wal/FaultySequenceFileLogReader.java      |   3 +
 .../hbase/regionserver/wal/TestHLog.java      |  10 +-
 .../hbase/regionserver/wal/TestHLogSplit.java |   2 +-
 .../wal/TestHLogSplitCompressed.java          |  35 +++++
 .../hbase/replication/TestReplication.java    |  12 +-
 .../TestReplicationWithCompression.java       |  40 ++++++
 11 files changed, 288 insertions(+), 54 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
index 8193e1f91161..c19d5b3fc684 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
@@ -143,7 +143,12 @@ static int uncompressIntoArray(byte[] to, int offset, DataInput in,
       // the status byte also acts as the higher order byte of the dictionary
       // entry
       short dictIdx = toShort(status, in.readByte());
-      byte[] entry = dict.getEntry(dictIdx);
+      byte[] entry;
+      try {
+        entry = dict.getEntry(dictIdx);
+      } catch (Exception ex) {
+        throw new IOException("Unable to uncompress the log entry", ex);
+      }
       if (entry == null) {
         throw new IOException("Missing dictionary entry for index "
             + dictIdx);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 2bb5ea192fa1..cbe3e251be0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -167,6 +167,7 @@ public interface Reader {
     Entry next(Entry reuse) throws IOException;
     void seek(long pos) throws IOException;
     long getPosition() throws IOException;
+    void reset() throws IOException;
   }
 
   public interface Writer {
@@ -695,15 +696,18 @@ protected Writer createWriterInstance(final FileSystem fs, final Path path,
 
   /**
    * Get a reader for the WAL.
+   * The proper way to tail a log that can be under construction is to first use this method
+   * to get a reader then call {@link HLog.Reader#reset()} to see the new data. It will also
+   * take care of keeping implementation-specific context (like compression).
    * @param fs
    * @param path
    * @param conf
    * @return A WAL reader.  Close when done with it.
    * @throws IOException
    */
-  public static Reader getReader(final FileSystem fs,
-    final Path path, Configuration conf)
-  throws IOException {
+  public static Reader getReader(final FileSystem fs, final Path path,
+                                 Configuration conf)
+      throws IOException {
     try {
 
       if (logReaderClass == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index f5fb00f1218d..9d7928b230a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -139,15 +139,17 @@ public long getPos() throws IOException {
 
   Configuration conf;
   WALReader reader;
+  FileSystem fs;
 
   // Needed logging exceptions
   Path path;
   int edit = 0;
   long entryStart = 0;
+  boolean emptyCompressionContext = true;
   /**
    * Compression context to use reading.  Can be null if no compression.
    */
-  private CompressionContext compressionContext = null;
+  protected CompressionContext compressionContext = null;
 
   protected Class<? extends HLogKey> keyClass;
 
@@ -173,6 +175,7 @@ public void init(FileSystem fs, Path path, Configuration conf)
     this.conf = conf;
     this.path = path;
     reader = new WALReader(fs, path, conf);
+    this.fs = fs;
 
     // If compression is enabled, new dictionaries are created here.
     boolean compression = reader.isWALCompressionEnabled();
@@ -237,11 +240,22 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
       throw addFileInfoToException(ioe);
     }
     edit++;
+    if (compressionContext != null && emptyCompressionContext) {
+      emptyCompressionContext = false;
+    }
     return b? e: null;
   }
 
   @Override
   public void seek(long pos) throws IOException {
+    if (compressionContext != null && emptyCompressionContext) {
+      while (next() != null) {
+        if (getPosition() == pos) {
+          emptyCompressionContext = false;
+          break;
+        }
+      }
+    }
     try {
       reader.seek(pos);
     } catch (IOException ioe) {
@@ -286,4 +300,11 @@ protected IOException addFileInfoToException(final IOException ioe)
 
     return ioe;
   }
+
+  @Override
+  public void reset() throws IOException {
+    // Resetting the reader lets us see newly added data if the file is being written to
+    // We also keep the same compressionContext which was previously populated for this file
+    reader = new WALReader(fs, path, conf);
+  }
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
new file mode 100644
index 000000000000..ce8585cb0d87
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+import java.io.IOException;
+
+/**
+ * Wrapper class around HLog to help manage the implementation details
+ * such as compression.
+ */
+@InterfaceAudience.Private
+public class ReplicationHLogReaderManager {
+
+  private static final Log LOG = LogFactory.getLog(ReplicationHLogReaderManager.class);
+  private final FileSystem fs;
+  private final Configuration conf;
+  private long position = 0;
+  private HLog.Reader reader;
+  private Path lastPath;
+
+  /**
+   * Creates the helper but doesn't open any file
+   * Use setInitialPosition after using the constructor if some content needs to be skipped
+   * @param fs
+   * @param conf
+   */
+  public ReplicationHLogReaderManager(FileSystem fs, Configuration conf) {
+    this.fs = fs;
+    this.conf = conf;
+  }
+
+  /**
+   * Opens the file at the current position
+   * @param path
+   * @return
+   * @throws IOException
+   */
+  public HLog.Reader openReader(Path path) throws IOException {
+    // Detect if this is a new file, if so get a new reader else
+    // reset the current reader so that we see the new data
+    if (this.reader == null || !this.lastPath.equals(path)) {
+      this.reader = HLog.getReader(this.fs, path, this.conf);
+      this.lastPath = path;
+    } else {
+      this.reader.reset();
+    }
+    return this.reader;
+  }
+
+  /**
+   * Get the next entry, returned and also added in the array
+   * @param entriesArray
+   * @param currentNbEntries
+   * @return a new entry or null
+   * @throws IOException
+   */
+  public HLog.Entry readNextAndSetPosition(HLog.Entry[] entriesArray,
+                                           int currentNbEntries) throws IOException {
+    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
+    // Store the position so that in the future the reader can start
+    // reading from here. If the above call to next() throws an
+    // exception, the position won't be changed and retry will happen
+    // from the last known good position
+    this.position = this.reader.getPosition();
+    // We need to set the CC to null else it will be compressed when sent to the sink
+    if (entry != null) {
+      entry.setCompressionContext(null);
+    }
+    return entry;
+  }
+
+  /**
+   * Advance the reader to the current position
+   * @throws IOException
+   */
+  public void seek() throws IOException {
+    if (this.position != 0) {
+      this.reader.seek(this.position);
+    }
+  }
+
+  /**
+   * Get the position that we stopped reading at
+   * @return current position, cannot be negative
+   */
+  public long getPosition() {
+    return this.position;
+  }
+
+  public void setPosition(long pos) {
+    this.position = pos;
+  }
+
+  /**
+   * Close the current reader
+   * @throws IOException
+   */
+  public void closeReader() throws IOException {
+    if (this.reader != null) {
+      this.reader.close();
+    }
+  }
+
+  /**
+   * Tell the helper to reset internal state
+   */
+  public void finishCurrentFile() {
+    this.position = 0;
+    this.reader = null;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 09b8797b7348..9f26e501ca93 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -105,8 +105,6 @@ public class ReplicationSource extends Thread
   private int replicationQueueNbCapacity;
   // Our reader for the current log
   private HLog.Reader reader;
-  // Current position in the log
-  private long position = 0;
   // Last position in the log that we sent to ZooKeeper
   private long lastLoggedPosition = -1;
   // Path of the current log
@@ -132,10 +130,15 @@ public class ReplicationSource extends Thread
   private int currentNbEntries = 0;
   // Current number of operations (Put/Delete) that we need to replicate
   private int currentNbOperations = 0;
+  // Current size of data we need to replicate
+  private int currentSize = 0;
   // Indicates if this particular source is running
   private volatile boolean running = true;
   // Metrics for this source
   private ReplicationSourceMetrics metrics;
+  // Handle on the log reader helper
+  private ReplicationHLogReaderManager repLogReader;
+
 
   /**
    * Instantiation method used by region servers
@@ -183,7 +186,7 @@ public void init(final Configuration conf,
         this.conf.getLong("replication.source.sleepforretries", 1000);
     this.fs = fs;
     this.metrics = new ReplicationSourceMetrics(peerClusterZnode);
-
+    this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf);
     try {
       this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher());
     } catch (KeeperException ke) {
@@ -263,8 +266,8 @@ public void run() {
     // normally has a position (unless the RS failed between 2 logs)
     if (this.queueRecovered) {
       try {
-        this.position = this.zkHelper.getHLogRepPosition(
-            this.peerClusterZnode, this.queue.peek().getName());
+        this.repLogReader.setPosition(this.zkHelper.getHLogRepPosition(
+            this.peerClusterZnode, this.queue.peek().getName()));
       } catch (KeeperException e) {
         this.terminate("Couldn't get the position of this recovered queue " +
             peerClusterZnode, e);
@@ -322,6 +325,7 @@ public void run() {
 
       boolean gotIOE = false;
       currentNbEntries = 0;
+      currentSize = 0;
       try {
         if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) {
           continue;
@@ -357,9 +361,7 @@ public void run() {
         }
       } finally {
         try {
-          if (this.reader != null) {
-            this.reader.close();
-          }
+          this.repLogReader.closeReader();
         } catch (IOException e) {
           gotIOE = true;
           LOG.warn("Unable to finalize the tailing of a file", e);
@@ -370,10 +372,10 @@ public void run() {
       // wait a bit and retry.
       // But if we need to stop, don't bother sleeping
       if (this.isActive() && (gotIOE || currentNbEntries == 0)) {
-        if (this.lastLoggedPosition != this.position) {
+        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.position;
+              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.repLogReader.getPosition();
         }
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
           sleepMultiplier++;
@@ -405,11 +407,9 @@ public void run() {
   protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo)
       throws IOException{
     long seenEntries = 0;
-    if (this.position != 0) {
-      this.reader.seek(this.position);
-    }
-    long startPosition = this.position;
-    HLog.Entry entry = readNextAndSetPosition();
+    this.repLogReader.seek();
+    HLog.Entry entry =
+        this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
     while (entry != null) {
       WALEdit edit = entry.getEdit();
       this.metrics.logEditsReadRate.inc(1);
@@ -433,18 +433,18 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
           }
           currentNbOperations += countDistinctRowKeys(edit);
           currentNbEntries++;
+          currentSize += entry.getEdit().heapSize();
         } else {
           this.metrics.logEditsFilteredRate.inc(1);
         }
       }
       // Stop if too many entries or too big
-      if ((this.reader.getPosition() - startPosition)
-          >= this.replicationQueueSizeCapacity ||
+      if (currentSize >= this.replicationQueueSizeCapacity ||
           currentNbEntries >= this.replicationQueueNbCapacity) {
         break;
       }
       try {
-        entry = readNextAndSetPosition();
+        entry = this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
       } catch (IOException ie) {
         LOG.debug("Break on IOE: " + ie.getMessage());
         break;
@@ -452,7 +452,7 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
     }
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +
-        " and size: " + (this.reader.getPosition() - startPosition));
+        " and size: " + this.currentSize);
     if (currentWALisBeingWrittenTo) {
       return false;
     }
@@ -461,16 +461,6 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
     return seenEntries == 0 && processEndOfFile();
   }
 
-  private HLog.Entry readNextAndSetPosition() throws IOException {
-    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
-    // Store the position so that in the future the reader can start
-    // reading from here. If the above call to next() throws an
-    // exception, the position won't be changed and retry will happen
-    // from the last known good position
-    this.position = this.reader.getPosition();
-    return entry;
-  } 
-
   private void connectToPeers() {
     // Connect to peer cluster first, unless we have to stop
     while (this.isActive() && this.currentPeers.size() == 0) {
@@ -509,10 +499,9 @@ protected boolean getNextPath() {
   protected boolean openReader(int sleepMultiplier) {
     try {
       LOG.debug("Opening log for replication " + this.currentPath.getName() +
-          " at " + this.position);
+          " at " + this.repLogReader.getPosition());
       try {
-       this.reader = null;
-       this.reader = HLog.getReader(this.fs, this.currentPath, this.conf);
+        this.reader = repLogReader.openReader(this.currentPath);
       } catch (FileNotFoundException fnfe) {
         if (this.queueRecovered) {
           // We didn't find the log in the archive directory, look if it still
@@ -648,10 +637,10 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo) {
         HRegionInterface rrs = getRS();
         LOG.debug("Replicating " + currentNbEntries);
         rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries));
-        if (this.lastLoggedPosition != this.position) {
+        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.position;
+              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.repLogReader.getPosition();
         }
         this.totalReplicatedEdits += currentNbEntries;
         this.metrics.shippedBatchesRate.inc(1);
@@ -721,7 +710,8 @@ protected boolean isPeerEnabled() {
   protected boolean processEndOfFile() {
     if (this.queue.size() != 0) {
       this.currentPath = null;
-      this.position = 0;
+      this.repLogReader.finishCurrentFile();
+      this.reader = null;
       return true;
     } else if (this.queueRecovered) {
       this.manager.closeRecoveredQueue(this);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
index 16db1675ae1f..45c47270647c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
@@ -49,6 +49,9 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
         HLogKey key = HLog.newKey(conf);
         WALEdit val = new WALEdit();
         HLog.Entry e = new HLog.Entry(key, val);
+        if (compressionContext != null) {
+          e.setCompressionContext(compressionContext);
+        }
         b = this.reader.next(e.getKey(), e.getEdit());
         nextQueue.offer(e);
         numberOfFileEntries++;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 9fa4480af770..8feb5d6db2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -475,15 +475,13 @@ public void run() {
       throw t.exception;
 
     // Make sure you can read all the content
-    SequenceFile.Reader reader
-      = new SequenceFile.Reader(this.fs, walPath, this.conf);
+    HLog.Reader reader = HLog.getReader(this.fs, walPath, this.conf);
     int count = 0;
-    HLogKey key = HLog.newKey(conf);
-    WALEdit val = new WALEdit();
-    while (reader.next(key, val)) {
+    HLog.Entry entry = new HLog.Entry();
+    while (reader.next(entry) != null) {
       count++;
       assertTrue("Should be one KeyValue per WALEdit",
-                 val.getKeyValues().size() == 1);
+                 entry.getEdit().getKeyValues().size() == 1);
     }
     assertEquals(total, count);
     reader.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 69a4e226d947..37366280e821 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -86,7 +86,7 @@ public class TestHLogSplit {
   private Configuration conf;
   private FileSystem fs;
 
-  private final static HBaseTestingUtility
+  protected final static HBaseTestingUtility
           TEST_UTIL = new HBaseTestingUtility();
 
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
new file mode 100644
index 000000000000..101678a9a21a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestHLogSplitCompressed extends TestHLogSplit {
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TestHLogSplit.setUpBeforeClass();
+    TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index 96eb211cccfb..45e89979efd3 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -61,7 +61,7 @@ public class TestReplication {
 
   private static final Log LOG = LogFactory.getLog(TestReplication.class);
 
-  private static Configuration conf1;
+  protected static Configuration conf1 = HBaseConfiguration.create();
   private static Configuration conf2;
   private static Configuration CONF_WITH_LOCALFS;
 
@@ -91,7 +91,6 @@ public class TestReplication {
    */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    conf1 = HBaseConfiguration.create();
     conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
     // smaller block size and capacity to trigger more operations
     // and test them
@@ -520,7 +519,7 @@ public void testDisableInactivePeer() throws Exception {
 
     // disable and start the peer
     admin.disablePeer("2");
-    utility2.startMiniHBaseCluster(1, 1);
+    utility2.startMiniHBaseCluster(1, 2);
     Get get = new Get(rowkey);
     for (int i = 0; i < NB_RETRIES; i++) {
       Result res = htable2.get(get);
@@ -760,7 +759,8 @@ public void queueFailover() throws Exception {
     int lastCount = 0;
 
     final long start = System.currentTimeMillis();
-    for (int i = 0; i < NB_RETRIES; i++) {
+    int i = 0;
+    while (true) {
       if (i==NB_RETRIES-1) {
         fail("Waited too much time for queueFailover replication. " +
           "Waited "+(System.currentTimeMillis() - start)+"ms.");
@@ -772,6 +772,8 @@ public void queueFailover() throws Exception {
       if (res2.length < initialCount) {
         if (lastCount < res2.length) {
           i--; // Don't increment timeout if we make progress
+        } else {
+          i++;
         }
         lastCount = res2.length;
         LOG.info("Only got " + lastCount + " rows instead of " +
@@ -791,7 +793,7 @@ public void run() {
           Thread.sleep(timeout);
           utility.expireRegionServerSession(rs);
         } catch (Exception e) {
-          LOG.error(e);
+          LOG.error("Couldn't kill a region server", e);
         }
       }
     };
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
new file mode 100644
index 000000000000..2c87a7bf524a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Run the same test as TestReplication but with HLog compression enabled
+ */
+@Category(LargeTests.class)
+public class TestReplicationWithCompression extends TestReplication {
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+    TestReplication.setUpBeforeClass();
+  }
+}

From 8bdde5d36882ceec57f7982a0457d2223ea7e5b5 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Wed, 19 Dec 2012 22:45:41 +0000
Subject: [PATCH 0647/1540] HBASE-7158  Allow CopyTable to identify the source
 cluster (for replication scenarios)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424192 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/Mutation.java  |  1 +
 .../apache/hadoop/hbase/mapreduce/Import.java | 27 ++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index c25fffc26af9..812b560d5df4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -193,6 +193,7 @@ public long getTimeStamp() {
    * @param clusterId
    */
   public void setClusterId(UUID clusterId) {
+    if (clusterId == null) return;
     byte[] val = new byte[2*Bytes.SIZEOF_LONG];
     Bytes.putLong(val, 0, clusterId.getMostSignificantBits());
     Bytes.putLong(val, Bytes.SIZEOF_LONG, clusterId.getLeastSignificantBits());
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 03d3f8d35962..651810444a08 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -22,23 +22,30 @@
 import java.io.IOException;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.UUID;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * Import data written by {@link Export}.
@@ -47,6 +54,7 @@ public class Import {
   final static String NAME = "import";
   final static String CF_RENAME_PROP = "HBASE_IMPORTER_RENAME_CFS";
   final static String BULK_OUTPUT_CONF_KEY = "import.bulk.output";
+  private static final Log LOG = LogFactory.getLog(Import.class);
 
   /**
    * A mapper that just writes out KeyValues.
@@ -88,6 +96,7 @@ public void setup(Context context) {
   static class Importer
   extends TableMapper<ImmutableBytesWritable, Mutation> {
     private Map<byte[], byte[]> cfRenameMap;
+    private UUID clusterId;
       
     /**
      * @param row  The current table row key.
@@ -128,16 +137,32 @@ private void writeResult(ImmutableBytesWritable key, Result result, Context cont
         }
       }
       if (put != null) {
+        put.setClusterId(clusterId);
         context.write(key, put);
       }
       if (delete != null) {
+        delete.setClusterId(clusterId);
         context.write(key, delete);
       }
     }
 
     @Override
     public void setup(Context context) {
-      cfRenameMap = createCfRenameMap(context.getConfiguration());
+      Configuration conf = context.getConfiguration();
+      cfRenameMap = createCfRenameMap(conf);
+      try {
+        HConnection connection = HConnectionManager.getConnection(conf);
+        ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
+        ReplicationZookeeper zkHelper = new ReplicationZookeeper(connection, conf, zkw);
+        clusterId = zkHelper.getUUIDForCluster(zkw);
+      } catch (ZooKeeperConnectionException e) {
+        LOG.error("Problem connecting to ZooKeper during task setup", e);
+      } catch (KeeperException e) {
+        LOG.error("Problem reading ZooKeeper data during task setup", e);
+      } catch (IOException e) {
+        LOG.error("Problem setting up task", e);
+      }
+
     }
   }
 

From 090fe6013562916f57c2dd19ab4af6404d121368 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Wed, 19 Dec 2012 23:44:17 +0000
Subject: [PATCH 0648/1540] Missing imports for HBASE-7158

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424229 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 651810444a08..31238de4e636 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -36,6 +36,8 @@
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -317,4 +319,4 @@ public static void main(String[] args) throws Exception {
     Job job = createSubmittableJob(conf, otherArgs);
     System.exit(job.waitForCompletion(true) ? 0 : 1);
   }
-}
\ No newline at end of file
+}

From df43831ef783846780424cf25d984c929beade10 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 19 Dec 2012 23:48:40 +0000
Subject: [PATCH 0649/1540] HBASE-7259 Deadlock in HBaseClient when
 KeeperException occured (liwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424233 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HConnectionManager.java   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index e3f483ae0600..08fd2a9c23ee 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -494,6 +494,7 @@ static class HConnectionImplementation implements HConnection, Closeable {
     private final Object masterLock = new Object();
     private volatile boolean closed;
     private volatile boolean aborted;
+    private volatile boolean resetting;
     private volatile HMasterInterface master;
     // ZooKeeper reference
     private volatile ZooKeeperWatcher zooKeeper;
@@ -505,6 +506,8 @@ static class HConnectionImplementation implements HConnection, Closeable {
     private final Object metaRegionLock = new Object();
 
     private final Object userRegionLock = new Object();
+	
+    private final Object resetLock = new Object();
 
     private final Configuration conf;
     // Known region HServerAddress.toString() -> HRegionInterface
@@ -573,6 +576,7 @@ public HConnectionImplementation(Configuration conf, boolean managed)
           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
 
       this.master = null;
+      this.resetting = false;
     }
 
     private synchronized void ensureZookeeperTrackers()
@@ -1678,7 +1682,12 @@ public void abort(final String msg, Throwable t) {
           LOG.info("ZK session expired. This disconnect could have been" +
               " caused by a network partition or a long-running GC pause," +
               " either way it's recommended that you verify your environment.");
+          synchronized (resetLock) {
+            if (resetting) return;
+            this.resetting = true;
+          }
           resetZooKeeperTrackers();
+          this.resetting = false;
         }
         return;
       }

From 52b9ec67dad86e3f0733000f94dfc9fcf4e8132f Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 20 Dec 2012 03:23:01 +0000
Subject: [PATCH 0650/1540] Trying to see if this fixes Jenkins (TestRep sleep
 .5 => 1.5s)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424276 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/replication/TestReplication.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index 45e89979efd3..e49ddc901b0d 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -78,7 +78,7 @@ public class TestReplication {
   private static final int NB_ROWS_IN_BATCH = 100;
   private static final int NB_ROWS_IN_BIG_BATCH =
       NB_ROWS_IN_BATCH * 10;
-  private static final long SLEEP_TIME = 500;
+  private static final long SLEEP_TIME = 1500;
   private static final int NB_RETRIES = 10;
 
   private static final byte[] tableName = Bytes.toBytes("test");

From bf8bbcf94a4f81f00a0901fbb2a5b0b2cfe751ff Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 20 Dec 2012 19:18:59 +0000
Subject: [PATCH 0651/1540] HBASE-7091: Support custom GC options in
 hbase-env.sh

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424646 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase         | 15 +++++++++++++++
 conf/hbase-env.sh | 20 ++++++++++++++++----
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/bin/hbase b/bin/hbase
index 62702057be9e..f23160580f6e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -248,6 +248,21 @@ fi
 # restore ordinary behaviour
 unset IFS
 
+#Set the right GC options based on the what we are running
+declare -a client_cmds=("shell" "hbck" "hlog" "hfile" "zkcli")
+for cmd in $client_cmds; do
+	if [[ $cmd == $COMMAND ]]; then
+		client=true
+		break
+	fi
+done
+
+if [[ $client ]]; then
+	HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS"
+else
+	HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS"
+fi
+
 # figure out which class to run
 if [ "$COMMAND" = "shell" ] ; then
   # eg export JRUBY_HOME=/usr/local/share/jruby
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index 5b5ba96045ac..da53a27f7f76 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -21,6 +21,10 @@
 
 # Set environment variables here.
 
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
 # The java implementation to use.  Java 1.6 required.
 # export JAVA_HOME=/usr/java/jdk1.6.0/
 
@@ -34,12 +38,20 @@
 # Below are what we set by default.  May only work with SUN JVM.
 # For more on why as well as other possible settings,
 # see http://wiki.apache.org/hadoop/PerformanceTuning
-export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC"
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+
+# Uncomment below to enable java garbage collection logging for the server-side processes
+# this enables basic gc logging for the server processes to the .out file
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
 
-# Uncomment below to enable java garbage collection logging in the .out file.
-# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" 
+# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
 
-# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS)
+# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
+# This applies to both the server and client GC options above
 # export HBASE_USE_GC_LOGFILE=true
 
 

From a132fd0e9f9d2c950076016e552a4d5d35fa28df Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 20 Dec 2012 20:59:58 +0000
Subject: [PATCH 0652/1540] HBASE-5778 Revert, to check on test failures
 potentially caused by this.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424702 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/Compressor.java    |   7 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |  10 +-
 .../wal/SequenceFileLogReader.java            |  23 +--
 .../ReplicationHLogReaderManager.java         | 136 ------------------
 .../regionserver/ReplicationSource.java       |  64 +++++----
 .../wal/FaultySequenceFileLogReader.java      |   3 -
 .../hbase/regionserver/wal/TestHLog.java      |  10 +-
 .../hbase/regionserver/wal/TestHLogSplit.java |   2 +-
 .../wal/TestHLogSplitCompressed.java          |  35 -----
 .../hbase/replication/TestReplication.java    |  12 +-
 .../TestReplicationWithCompression.java       |  40 ------
 11 files changed, 54 insertions(+), 288 deletions(-)
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
index c19d5b3fc684..8193e1f91161 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
@@ -143,12 +143,7 @@ static int uncompressIntoArray(byte[] to, int offset, DataInput in,
       // the status byte also acts as the higher order byte of the dictionary
       // entry
       short dictIdx = toShort(status, in.readByte());
-      byte[] entry;
-      try {
-        entry = dict.getEntry(dictIdx);
-      } catch (Exception ex) {
-        throw new IOException("Unable to uncompress the log entry", ex);
-      }
+      byte[] entry = dict.getEntry(dictIdx);
       if (entry == null) {
         throw new IOException("Missing dictionary entry for index "
             + dictIdx);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index cbe3e251be0c..2bb5ea192fa1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -167,7 +167,6 @@ public interface Reader {
     Entry next(Entry reuse) throws IOException;
     void seek(long pos) throws IOException;
     long getPosition() throws IOException;
-    void reset() throws IOException;
   }
 
   public interface Writer {
@@ -696,18 +695,15 @@ protected Writer createWriterInstance(final FileSystem fs, final Path path,
 
   /**
    * Get a reader for the WAL.
-   * The proper way to tail a log that can be under construction is to first use this method
-   * to get a reader then call {@link HLog.Reader#reset()} to see the new data. It will also
-   * take care of keeping implementation-specific context (like compression).
    * @param fs
    * @param path
    * @param conf
    * @return A WAL reader.  Close when done with it.
    * @throws IOException
    */
-  public static Reader getReader(final FileSystem fs, final Path path,
-                                 Configuration conf)
-      throws IOException {
+  public static Reader getReader(final FileSystem fs,
+    final Path path, Configuration conf)
+  throws IOException {
     try {
 
       if (logReaderClass == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index 9d7928b230a8..f5fb00f1218d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -139,17 +139,15 @@ public long getPos() throws IOException {
 
   Configuration conf;
   WALReader reader;
-  FileSystem fs;
 
   // Needed logging exceptions
   Path path;
   int edit = 0;
   long entryStart = 0;
-  boolean emptyCompressionContext = true;
   /**
    * Compression context to use reading.  Can be null if no compression.
    */
-  protected CompressionContext compressionContext = null;
+  private CompressionContext compressionContext = null;
 
   protected Class<? extends HLogKey> keyClass;
 
@@ -175,7 +173,6 @@ public void init(FileSystem fs, Path path, Configuration conf)
     this.conf = conf;
     this.path = path;
     reader = new WALReader(fs, path, conf);
-    this.fs = fs;
 
     // If compression is enabled, new dictionaries are created here.
     boolean compression = reader.isWALCompressionEnabled();
@@ -240,22 +237,11 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
       throw addFileInfoToException(ioe);
     }
     edit++;
-    if (compressionContext != null && emptyCompressionContext) {
-      emptyCompressionContext = false;
-    }
     return b? e: null;
   }
 
   @Override
   public void seek(long pos) throws IOException {
-    if (compressionContext != null && emptyCompressionContext) {
-      while (next() != null) {
-        if (getPosition() == pos) {
-          emptyCompressionContext = false;
-          break;
-        }
-      }
-    }
     try {
       reader.seek(pos);
     } catch (IOException ioe) {
@@ -300,11 +286,4 @@ protected IOException addFileInfoToException(final IOException ioe)
 
     return ioe;
   }
-
-  @Override
-  public void reset() throws IOException {
-    // Resetting the reader lets us see newly added data if the file is being written to
-    // We also keep the same compressionContext which was previously populated for this file
-    reader = new WALReader(fs, path, conf);
-  }
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
deleted file mode 100644
index ce8585cb0d87..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.replication.regionserver;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-
-import java.io.IOException;
-
-/**
- * Wrapper class around HLog to help manage the implementation details
- * such as compression.
- */
-@InterfaceAudience.Private
-public class ReplicationHLogReaderManager {
-
-  private static final Log LOG = LogFactory.getLog(ReplicationHLogReaderManager.class);
-  private final FileSystem fs;
-  private final Configuration conf;
-  private long position = 0;
-  private HLog.Reader reader;
-  private Path lastPath;
-
-  /**
-   * Creates the helper but doesn't open any file
-   * Use setInitialPosition after using the constructor if some content needs to be skipped
-   * @param fs
-   * @param conf
-   */
-  public ReplicationHLogReaderManager(FileSystem fs, Configuration conf) {
-    this.fs = fs;
-    this.conf = conf;
-  }
-
-  /**
-   * Opens the file at the current position
-   * @param path
-   * @return
-   * @throws IOException
-   */
-  public HLog.Reader openReader(Path path) throws IOException {
-    // Detect if this is a new file, if so get a new reader else
-    // reset the current reader so that we see the new data
-    if (this.reader == null || !this.lastPath.equals(path)) {
-      this.reader = HLog.getReader(this.fs, path, this.conf);
-      this.lastPath = path;
-    } else {
-      this.reader.reset();
-    }
-    return this.reader;
-  }
-
-  /**
-   * Get the next entry, returned and also added in the array
-   * @param entriesArray
-   * @param currentNbEntries
-   * @return a new entry or null
-   * @throws IOException
-   */
-  public HLog.Entry readNextAndSetPosition(HLog.Entry[] entriesArray,
-                                           int currentNbEntries) throws IOException {
-    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
-    // Store the position so that in the future the reader can start
-    // reading from here. If the above call to next() throws an
-    // exception, the position won't be changed and retry will happen
-    // from the last known good position
-    this.position = this.reader.getPosition();
-    // We need to set the CC to null else it will be compressed when sent to the sink
-    if (entry != null) {
-      entry.setCompressionContext(null);
-    }
-    return entry;
-  }
-
-  /**
-   * Advance the reader to the current position
-   * @throws IOException
-   */
-  public void seek() throws IOException {
-    if (this.position != 0) {
-      this.reader.seek(this.position);
-    }
-  }
-
-  /**
-   * Get the position that we stopped reading at
-   * @return current position, cannot be negative
-   */
-  public long getPosition() {
-    return this.position;
-  }
-
-  public void setPosition(long pos) {
-    this.position = pos;
-  }
-
-  /**
-   * Close the current reader
-   * @throws IOException
-   */
-  public void closeReader() throws IOException {
-    if (this.reader != null) {
-      this.reader.close();
-    }
-  }
-
-  /**
-   * Tell the helper to reset internal state
-   */
-  public void finishCurrentFile() {
-    this.position = 0;
-    this.reader = null;
-  }
-
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 9f26e501ca93..09b8797b7348 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -105,6 +105,8 @@ public class ReplicationSource extends Thread
   private int replicationQueueNbCapacity;
   // Our reader for the current log
   private HLog.Reader reader;
+  // Current position in the log
+  private long position = 0;
   // Last position in the log that we sent to ZooKeeper
   private long lastLoggedPosition = -1;
   // Path of the current log
@@ -130,15 +132,10 @@ public class ReplicationSource extends Thread
   private int currentNbEntries = 0;
   // Current number of operations (Put/Delete) that we need to replicate
   private int currentNbOperations = 0;
-  // Current size of data we need to replicate
-  private int currentSize = 0;
   // Indicates if this particular source is running
   private volatile boolean running = true;
   // Metrics for this source
   private ReplicationSourceMetrics metrics;
-  // Handle on the log reader helper
-  private ReplicationHLogReaderManager repLogReader;
-
 
   /**
    * Instantiation method used by region servers
@@ -186,7 +183,7 @@ public void init(final Configuration conf,
         this.conf.getLong("replication.source.sleepforretries", 1000);
     this.fs = fs;
     this.metrics = new ReplicationSourceMetrics(peerClusterZnode);
-    this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf);
+
     try {
       this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher());
     } catch (KeeperException ke) {
@@ -266,8 +263,8 @@ public void run() {
     // normally has a position (unless the RS failed between 2 logs)
     if (this.queueRecovered) {
       try {
-        this.repLogReader.setPosition(this.zkHelper.getHLogRepPosition(
-            this.peerClusterZnode, this.queue.peek().getName()));
+        this.position = this.zkHelper.getHLogRepPosition(
+            this.peerClusterZnode, this.queue.peek().getName());
       } catch (KeeperException e) {
         this.terminate("Couldn't get the position of this recovered queue " +
             peerClusterZnode, e);
@@ -325,7 +322,6 @@ public void run() {
 
       boolean gotIOE = false;
       currentNbEntries = 0;
-      currentSize = 0;
       try {
         if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) {
           continue;
@@ -361,7 +357,9 @@ public void run() {
         }
       } finally {
         try {
-          this.repLogReader.closeReader();
+          if (this.reader != null) {
+            this.reader.close();
+          }
         } catch (IOException e) {
           gotIOE = true;
           LOG.warn("Unable to finalize the tailing of a file", e);
@@ -372,10 +370,10 @@ public void run() {
       // wait a bit and retry.
       // But if we need to stop, don't bother sleeping
       if (this.isActive() && (gotIOE || currentNbEntries == 0)) {
-        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
+        if (this.lastLoggedPosition != this.position) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.repLogReader.getPosition();
+              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.position;
         }
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
           sleepMultiplier++;
@@ -407,9 +405,11 @@ public void run() {
   protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo)
       throws IOException{
     long seenEntries = 0;
-    this.repLogReader.seek();
-    HLog.Entry entry =
-        this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
+    if (this.position != 0) {
+      this.reader.seek(this.position);
+    }
+    long startPosition = this.position;
+    HLog.Entry entry = readNextAndSetPosition();
     while (entry != null) {
       WALEdit edit = entry.getEdit();
       this.metrics.logEditsReadRate.inc(1);
@@ -433,18 +433,18 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
           }
           currentNbOperations += countDistinctRowKeys(edit);
           currentNbEntries++;
-          currentSize += entry.getEdit().heapSize();
         } else {
           this.metrics.logEditsFilteredRate.inc(1);
         }
       }
       // Stop if too many entries or too big
-      if (currentSize >= this.replicationQueueSizeCapacity ||
+      if ((this.reader.getPosition() - startPosition)
+          >= this.replicationQueueSizeCapacity ||
           currentNbEntries >= this.replicationQueueNbCapacity) {
         break;
       }
       try {
-        entry = this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
+        entry = readNextAndSetPosition();
       } catch (IOException ie) {
         LOG.debug("Break on IOE: " + ie.getMessage());
         break;
@@ -452,7 +452,7 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
     }
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +
-        " and size: " + this.currentSize);
+        " and size: " + (this.reader.getPosition() - startPosition));
     if (currentWALisBeingWrittenTo) {
       return false;
     }
@@ -461,6 +461,16 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
     return seenEntries == 0 && processEndOfFile();
   }
 
+  private HLog.Entry readNextAndSetPosition() throws IOException {
+    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
+    // Store the position so that in the future the reader can start
+    // reading from here. If the above call to next() throws an
+    // exception, the position won't be changed and retry will happen
+    // from the last known good position
+    this.position = this.reader.getPosition();
+    return entry;
+  } 
+
   private void connectToPeers() {
     // Connect to peer cluster first, unless we have to stop
     while (this.isActive() && this.currentPeers.size() == 0) {
@@ -499,9 +509,10 @@ protected boolean getNextPath() {
   protected boolean openReader(int sleepMultiplier) {
     try {
       LOG.debug("Opening log for replication " + this.currentPath.getName() +
-          " at " + this.repLogReader.getPosition());
+          " at " + this.position);
       try {
-        this.reader = repLogReader.openReader(this.currentPath);
+       this.reader = null;
+       this.reader = HLog.getReader(this.fs, this.currentPath, this.conf);
       } catch (FileNotFoundException fnfe) {
         if (this.queueRecovered) {
           // We didn't find the log in the archive directory, look if it still
@@ -637,10 +648,10 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo) {
         HRegionInterface rrs = getRS();
         LOG.debug("Replicating " + currentNbEntries);
         rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries));
-        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
+        if (this.lastLoggedPosition != this.position) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.repLogReader.getPosition();
+              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.position;
         }
         this.totalReplicatedEdits += currentNbEntries;
         this.metrics.shippedBatchesRate.inc(1);
@@ -710,8 +721,7 @@ protected boolean isPeerEnabled() {
   protected boolean processEndOfFile() {
     if (this.queue.size() != 0) {
       this.currentPath = null;
-      this.repLogReader.finishCurrentFile();
-      this.reader = null;
+      this.position = 0;
       return true;
     } else if (this.queueRecovered) {
       this.manager.closeRecoveredQueue(this);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
index 45c47270647c..16db1675ae1f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
@@ -49,9 +49,6 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
         HLogKey key = HLog.newKey(conf);
         WALEdit val = new WALEdit();
         HLog.Entry e = new HLog.Entry(key, val);
-        if (compressionContext != null) {
-          e.setCompressionContext(compressionContext);
-        }
         b = this.reader.next(e.getKey(), e.getEdit());
         nextQueue.offer(e);
         numberOfFileEntries++;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 8feb5d6db2d1..9fa4480af770 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -475,13 +475,15 @@ public void run() {
       throw t.exception;
 
     // Make sure you can read all the content
-    HLog.Reader reader = HLog.getReader(this.fs, walPath, this.conf);
+    SequenceFile.Reader reader
+      = new SequenceFile.Reader(this.fs, walPath, this.conf);
     int count = 0;
-    HLog.Entry entry = new HLog.Entry();
-    while (reader.next(entry) != null) {
+    HLogKey key = HLog.newKey(conf);
+    WALEdit val = new WALEdit();
+    while (reader.next(key, val)) {
       count++;
       assertTrue("Should be one KeyValue per WALEdit",
-                 entry.getEdit().getKeyValues().size() == 1);
+                 val.getKeyValues().size() == 1);
     }
     assertEquals(total, count);
     reader.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 37366280e821..69a4e226d947 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -86,7 +86,7 @@ public class TestHLogSplit {
   private Configuration conf;
   private FileSystem fs;
 
-  protected final static HBaseTestingUtility
+  private final static HBaseTestingUtility
           TEST_UTIL = new HBaseTestingUtility();
 
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
deleted file mode 100644
index 101678a9a21a..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.regionserver.wal;
-
-
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.LargeTests;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestHLogSplitCompressed extends TestHLogSplit {
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TestHLogSplit.setUpBeforeClass();
-    TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
-  }
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index e49ddc901b0d..19703a25793f 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -61,7 +61,7 @@ public class TestReplication {
 
   private static final Log LOG = LogFactory.getLog(TestReplication.class);
 
-  protected static Configuration conf1 = HBaseConfiguration.create();
+  private static Configuration conf1;
   private static Configuration conf2;
   private static Configuration CONF_WITH_LOCALFS;
 
@@ -91,6 +91,7 @@ public class TestReplication {
    */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
+    conf1 = HBaseConfiguration.create();
     conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
     // smaller block size and capacity to trigger more operations
     // and test them
@@ -519,7 +520,7 @@ public void testDisableInactivePeer() throws Exception {
 
     // disable and start the peer
     admin.disablePeer("2");
-    utility2.startMiniHBaseCluster(1, 2);
+    utility2.startMiniHBaseCluster(1, 1);
     Get get = new Get(rowkey);
     for (int i = 0; i < NB_RETRIES; i++) {
       Result res = htable2.get(get);
@@ -759,8 +760,7 @@ public void queueFailover() throws Exception {
     int lastCount = 0;
 
     final long start = System.currentTimeMillis();
-    int i = 0;
-    while (true) {
+    for (int i = 0; i < NB_RETRIES; i++) {
       if (i==NB_RETRIES-1) {
         fail("Waited too much time for queueFailover replication. " +
           "Waited "+(System.currentTimeMillis() - start)+"ms.");
@@ -772,8 +772,6 @@ public void queueFailover() throws Exception {
       if (res2.length < initialCount) {
         if (lastCount < res2.length) {
           i--; // Don't increment timeout if we make progress
-        } else {
-          i++;
         }
         lastCount = res2.length;
         LOG.info("Only got " + lastCount + " rows instead of " +
@@ -793,7 +791,7 @@ public void run() {
           Thread.sleep(timeout);
           utility.expireRegionServerSession(rs);
         } catch (Exception e) {
-          LOG.error("Couldn't kill a region server", e);
+          LOG.error(e);
         }
       }
     };
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
deleted file mode 100644
index 2c87a7bf524a..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.LargeTests;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-/**
- * Run the same test as TestReplication but with HLog compression enabled
- */
-@Category(LargeTests.class)
-public class TestReplicationWithCompression extends TestReplication {
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
-    TestReplication.setUpBeforeClass();
-  }
-}

From 28e6408e095a9ae514de01491f58f490fce8360b Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 20 Dec 2012 21:46:49 +0000
Subject: [PATCH 0653/1540] HBASE-7398 [0.94 UNIT TESTS] TestAssignmentManager
 fails frequently on CentOS 5

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424718 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/Mocking.java   | 46 +++++++++++
 .../hbase/master/TestAssignmentManager.java   | 78 +++++++++----------
 2 files changed, 82 insertions(+), 42 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/Mocking.java

diff --git a/src/test/java/org/apache/hadoop/hbase/master/Mocking.java b/src/test/java/org/apache/hadoop/hbase/master/Mocking.java
new file mode 100644
index 000000000000..187e07df806a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/Mocking.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+
+/**
+ * Package scoped mocking utility.
+ */
+public class Mocking {
+
+  static void waitForRegionPendingOpenInRIT(AssignmentManager am, String encodedName)
+    throws InterruptedException {
+    // We used to do a check like this:
+    //!Mocking.verifyRegionState(this.watcher, REGIONINFO, EventType.M_ZK_REGION_OFFLINE)) {
+    // There is a race condition with this: because we may do the transition to
+    // RS_ZK_REGION_OPENING before the RIT is internally updated. We need to wait for the
+    // RIT to be as we need it to be instead. This cannot happen in a real cluster as we
+    // update the RIT before sending the openRegion request.
+
+    boolean wait = true;
+    while (wait) {
+      RegionState state = am.getRegionsInTransition().get(encodedName);
+      if (state != null && state.isPendingOpen()){
+        wait = false;
+      } else {
+        Thread.sleep(1);
+      }
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index ba4606964f73..aee85370b602 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -36,9 +36,9 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.client.Get;
@@ -61,12 +61,12 @@
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.Watcher;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -82,7 +82,7 @@
 /**
  * Test {@link AssignmentManager}
  */
-@Category(SmallTests.class)
+@Category(MediumTests.class)
 public class TestAssignmentManager {
   private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
   private static final ServerName SERVERNAME_A =
@@ -162,7 +162,7 @@ public void after() throws KeeperException {
 
   /**
    * Test a balance going on at same time as a master failover
-   * 
+   *
    * @throws IOException
    * @throws KeeperException
    * @throws InterruptedException
@@ -184,10 +184,8 @@ public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
       int versionid =
         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
       assertNotSame(versionid, -1);
-      while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO,
-          EventType.M_ZK_REGION_OFFLINE)) {
-        Threads.sleep(1);
-      }
+      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+
       // Get current versionid else will fail on transition from OFFLINE to
       // OPENING below
       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
@@ -228,10 +226,8 @@ public void testBalanceOnMasterFailoverScenarioWithClosedNode()
         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
       assertNotSame(versionid, -1);
       am.gate.set(false);
-      while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO,
-          EventType.M_ZK_REGION_OFFLINE)) {
-        Threads.sleep(1);
-      }
+      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+
       // Get current versionid else will fail on transition from OFFLINE to
       // OPENING below
       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
@@ -271,10 +267,8 @@ public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
       int versionid =
         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
       assertNotSame(versionid, -1);
-      while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO,
-          EventType.M_ZK_REGION_OFFLINE)) {
-        Threads.sleep(1);
-      }
+      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+
       am.gate.set(false);
       // Get current versionid else will fail on transition from OFFLINE to
       // OPENING below
@@ -313,10 +307,11 @@ private void createRegionPlanAndBalance(final AssignmentManager am,
    * from one server to another mocking regionserver responding over zk.
    * @throws IOException
    * @throws KeeperException
+   * @throws InterruptedException
    */
-  @Test
+  @Test(timeout = 10000)
   public void testBalance()
-  throws IOException, KeeperException {
+  throws IOException, KeeperException, InterruptedException {
     // Create and startup an executor.  This is used by AssignmentManager
     // handling zk callbacks.
     ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
@@ -350,11 +345,9 @@ public void testBalance()
       // AM is going to notice above CLOSED and queue up a new assign.  The
       // assign will go to open the region in the new location set by the
       // balancer.  The zk node will be OFFLINE waiting for regionserver to
-      // transition it through OPENING, OPENED.  Wait till we see the OFFLINE
-      // zk node before we proceed.
-      while (!ZKAssign.verifyRegionState(this.watcher, REGIONINFO, EventType.M_ZK_REGION_OFFLINE)) {
-        Threads.sleep(1);
-      }
+      // transition it through OPENING, OPENED.  Wait till we see the RIT
+      // before we proceed.
+      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
       // Get current versionid else will fail on transition from OFFLINE to OPENING below
       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
       assertNotSame(-1, versionid);
@@ -407,7 +400,7 @@ public void testShutdownHandler() throws KeeperException, IOException {
 
   /**
    * To test closed region handler to remove rit and delete corresponding znode if region in pending
-   * close or closing while processing shutdown of a region server.(HBASE-5927). 
+   * close or closing while processing shutdown of a region server.(HBASE-5927).
    * @throws KeeperException
    * @throws IOException
    */
@@ -417,7 +410,7 @@ public void testSSHWhenDisableTableInProgress()
     testCaseWithPartiallyDisabledState(TableState.DISABLING);
     testCaseWithPartiallyDisabledState(TableState.DISABLED);
   }
-  
+
   /**
    * To test if the split region is removed from RIT if the region was in SPLITTING state
    * but the RS has actually completed the splitting in META but went down. See HBASE-6070
@@ -451,7 +444,7 @@ private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws Keep
     am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
         State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
-    
+
     RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
         REGIONINFO.getRegionName(), SERVERNAME_A);
     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
@@ -459,11 +452,11 @@ private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws Keep
     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
 
     try {
-      
+
       processServerShutdownHandler(ct, am, regionSplitDone);
       // check znode deleted or not.
       // In both cases the znode should be deleted.
-      
+
       if(regionSplitDone){
         assertTrue("Region state of region in SPLITTING should be removed from rit.",
             am.regionsInTransition.isEmpty());
@@ -506,7 +499,7 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     } else {
       am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
     }
-    
+
     RegionTransitionData data = new RegionTransitionData(EventType.M_ZK_REGION_CLOSING,
         REGIONINFO.getRegionName(), SERVERNAME_A);
     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
@@ -581,7 +574,7 @@ private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager a
    * @param hri Region to serialize into HRegionInfo
    * @return A mocked up Result that fakes a Get on a row in the
    * <code>.META.</code> table.
-   * @throws IOException 
+   * @throws IOException
    */
   private Result getMetaTableRowResult(final HRegionInfo hri,
       final ServerName sn)
@@ -600,13 +593,13 @@ private Result getMetaTableRowResult(final HRegionInfo hri,
       Bytes.toBytes(sn.getStartcode())));
     return new Result(kvs);
   }
-  
+
   /**
    * @param sn ServerName to use making startcode and server in meta
    * @param hri Region to serialize into HRegionInfo
    * @return A mocked up Result that fakes a Get on a row in the
    * <code>.META.</code> table.
-   * @throws IOException 
+   * @throws IOException
    */
   private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
       throws IOException {
@@ -668,12 +661,12 @@ public void testUnassignWithSplitAtSameTime() throws KeeperException, IOExceptio
       am.shutdown();
     }
   }
-  
+
   /**
    * Tests the processDeadServersAndRegionsInTransition should not fail with NPE
    * when it failed to get the children. Let's abort the system in this
    * situation
-   * @throws ServiceException 
+   * @throws ServiceException
    */
   @Test(timeout = 5000)
   public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
@@ -713,8 +706,8 @@ public RecoverableZooKeeper getRecoverableZooKeeper() {
    * @param region region to be created as offline
    * @param serverName server event originates from
    * @return Version of znode created.
-   * @throws KeeperException 
-   * @throws IOException 
+   * @throws KeeperException
+   * @throws IOException
    */
   // Copied from SplitTransaction rather than open the method over there in
   // the regionserver package.
@@ -807,9 +800,9 @@ private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final
         server, manager, ct, balancer, executor);
     return am;
   }
-  
+
   /**
-   * TestCase verifies that the regionPlan is updated whenever a region fails to open 
+   * TestCase verifies that the regionPlan is updated whenever a region fails to open
    * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546).
    */
   @Test
@@ -857,17 +850,18 @@ public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, K
       assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
       assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
         newRegionPlan.getDestination())));
+      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
     } finally {
       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
         DefaultLoadBalancer.class, LoadBalancer.class);
       am.shutdown();
     }
   }
-  
+
   /**
    * Test verifies whether assignment is skipped for regions of tables in DISABLING state during
    * clean cluster startup. See HBASE-6281.
-   * 
+   *
    * @throws KeeperException
    * @throws IOException
    * @throws Exception
@@ -973,7 +967,7 @@ public ServerName randomAssignment(List<ServerName> servers) {
       this.gate.set(true);
       return randomServerName;
     }
-    
+
     @Override
     public Map<ServerName, List<HRegionInfo>> retainAssignment(
         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {

From bc0371e478153da28a67b3563446994c85c338a9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 21 Dec 2012 05:59:06 +0000
Subject: [PATCH 0654/1540] HBASE-5778 Reapply, Test failures not caused by
 this. Sorry for the noise.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1424810 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/Compressor.java    |   7 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |  10 +-
 .../wal/SequenceFileLogReader.java            |  23 ++-
 .../ReplicationHLogReaderManager.java         | 136 ++++++++++++++++++
 .../regionserver/ReplicationSource.java       |  64 ++++-----
 .../wal/FaultySequenceFileLogReader.java      |   3 +
 .../hbase/regionserver/wal/TestHLog.java      |  10 +-
 .../hbase/regionserver/wal/TestHLogSplit.java |   2 +-
 .../wal/TestHLogSplitCompressed.java          |  35 +++++
 .../hbase/replication/TestReplication.java    |  12 +-
 .../TestReplicationWithCompression.java       |  40 ++++++
 11 files changed, 288 insertions(+), 54 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
index 8193e1f91161..c19d5b3fc684 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java
@@ -143,7 +143,12 @@ static int uncompressIntoArray(byte[] to, int offset, DataInput in,
       // the status byte also acts as the higher order byte of the dictionary
       // entry
       short dictIdx = toShort(status, in.readByte());
-      byte[] entry = dict.getEntry(dictIdx);
+      byte[] entry;
+      try {
+        entry = dict.getEntry(dictIdx);
+      } catch (Exception ex) {
+        throw new IOException("Unable to uncompress the log entry", ex);
+      }
       if (entry == null) {
         throw new IOException("Missing dictionary entry for index "
             + dictIdx);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 2bb5ea192fa1..cbe3e251be0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -167,6 +167,7 @@ public interface Reader {
     Entry next(Entry reuse) throws IOException;
     void seek(long pos) throws IOException;
     long getPosition() throws IOException;
+    void reset() throws IOException;
   }
 
   public interface Writer {
@@ -695,15 +696,18 @@ protected Writer createWriterInstance(final FileSystem fs, final Path path,
 
   /**
    * Get a reader for the WAL.
+   * The proper way to tail a log that can be under construction is to first use this method
+   * to get a reader then call {@link HLog.Reader#reset()} to see the new data. It will also
+   * take care of keeping implementation-specific context (like compression).
    * @param fs
    * @param path
    * @param conf
    * @return A WAL reader.  Close when done with it.
    * @throws IOException
    */
-  public static Reader getReader(final FileSystem fs,
-    final Path path, Configuration conf)
-  throws IOException {
+  public static Reader getReader(final FileSystem fs, final Path path,
+                                 Configuration conf)
+      throws IOException {
     try {
 
       if (logReaderClass == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index f5fb00f1218d..9d7928b230a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -139,15 +139,17 @@ public long getPos() throws IOException {
 
   Configuration conf;
   WALReader reader;
+  FileSystem fs;
 
   // Needed logging exceptions
   Path path;
   int edit = 0;
   long entryStart = 0;
+  boolean emptyCompressionContext = true;
   /**
    * Compression context to use reading.  Can be null if no compression.
    */
-  private CompressionContext compressionContext = null;
+  protected CompressionContext compressionContext = null;
 
   protected Class<? extends HLogKey> keyClass;
 
@@ -173,6 +175,7 @@ public void init(FileSystem fs, Path path, Configuration conf)
     this.conf = conf;
     this.path = path;
     reader = new WALReader(fs, path, conf);
+    this.fs = fs;
 
     // If compression is enabled, new dictionaries are created here.
     boolean compression = reader.isWALCompressionEnabled();
@@ -237,11 +240,22 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
       throw addFileInfoToException(ioe);
     }
     edit++;
+    if (compressionContext != null && emptyCompressionContext) {
+      emptyCompressionContext = false;
+    }
     return b? e: null;
   }
 
   @Override
   public void seek(long pos) throws IOException {
+    if (compressionContext != null && emptyCompressionContext) {
+      while (next() != null) {
+        if (getPosition() == pos) {
+          emptyCompressionContext = false;
+          break;
+        }
+      }
+    }
     try {
       reader.seek(pos);
     } catch (IOException ioe) {
@@ -286,4 +300,11 @@ protected IOException addFileInfoToException(final IOException ioe)
 
     return ioe;
   }
+
+  @Override
+  public void reset() throws IOException {
+    // Resetting the reader lets us see newly added data if the file is being written to
+    // We also keep the same compressionContext which was previously populated for this file
+    reader = new WALReader(fs, path, conf);
+  }
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
new file mode 100644
index 000000000000..ce8585cb0d87
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+import java.io.IOException;
+
+/**
+ * Wrapper class around HLog to help manage the implementation details
+ * such as compression.
+ */
+@InterfaceAudience.Private
+public class ReplicationHLogReaderManager {
+
+  private static final Log LOG = LogFactory.getLog(ReplicationHLogReaderManager.class);
+  private final FileSystem fs;
+  private final Configuration conf;
+  private long position = 0;
+  private HLog.Reader reader;
+  private Path lastPath;
+
+  /**
+   * Creates the helper but doesn't open any file
+   * Use setInitialPosition after using the constructor if some content needs to be skipped
+   * @param fs
+   * @param conf
+   */
+  public ReplicationHLogReaderManager(FileSystem fs, Configuration conf) {
+    this.fs = fs;
+    this.conf = conf;
+  }
+
+  /**
+   * Opens the file at the current position
+   * @param path
+   * @return
+   * @throws IOException
+   */
+  public HLog.Reader openReader(Path path) throws IOException {
+    // Detect if this is a new file, if so get a new reader else
+    // reset the current reader so that we see the new data
+    if (this.reader == null || !this.lastPath.equals(path)) {
+      this.reader = HLog.getReader(this.fs, path, this.conf);
+      this.lastPath = path;
+    } else {
+      this.reader.reset();
+    }
+    return this.reader;
+  }
+
+  /**
+   * Get the next entry, returned and also added in the array
+   * @param entriesArray
+   * @param currentNbEntries
+   * @return a new entry or null
+   * @throws IOException
+   */
+  public HLog.Entry readNextAndSetPosition(HLog.Entry[] entriesArray,
+                                           int currentNbEntries) throws IOException {
+    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
+    // Store the position so that in the future the reader can start
+    // reading from here. If the above call to next() throws an
+    // exception, the position won't be changed and retry will happen
+    // from the last known good position
+    this.position = this.reader.getPosition();
+    // We need to set the CC to null else it will be compressed when sent to the sink
+    if (entry != null) {
+      entry.setCompressionContext(null);
+    }
+    return entry;
+  }
+
+  /**
+   * Advance the reader to the current position
+   * @throws IOException
+   */
+  public void seek() throws IOException {
+    if (this.position != 0) {
+      this.reader.seek(this.position);
+    }
+  }
+
+  /**
+   * Get the position that we stopped reading at
+   * @return current position, cannot be negative
+   */
+  public long getPosition() {
+    return this.position;
+  }
+
+  public void setPosition(long pos) {
+    this.position = pos;
+  }
+
+  /**
+   * Close the current reader
+   * @throws IOException
+   */
+  public void closeReader() throws IOException {
+    if (this.reader != null) {
+      this.reader.close();
+    }
+  }
+
+  /**
+   * Tell the helper to reset internal state
+   */
+  public void finishCurrentFile() {
+    this.position = 0;
+    this.reader = null;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 09b8797b7348..9f26e501ca93 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -105,8 +105,6 @@ public class ReplicationSource extends Thread
   private int replicationQueueNbCapacity;
   // Our reader for the current log
   private HLog.Reader reader;
-  // Current position in the log
-  private long position = 0;
   // Last position in the log that we sent to ZooKeeper
   private long lastLoggedPosition = -1;
   // Path of the current log
@@ -132,10 +130,15 @@ public class ReplicationSource extends Thread
   private int currentNbEntries = 0;
   // Current number of operations (Put/Delete) that we need to replicate
   private int currentNbOperations = 0;
+  // Current size of data we need to replicate
+  private int currentSize = 0;
   // Indicates if this particular source is running
   private volatile boolean running = true;
   // Metrics for this source
   private ReplicationSourceMetrics metrics;
+  // Handle on the log reader helper
+  private ReplicationHLogReaderManager repLogReader;
+
 
   /**
    * Instantiation method used by region servers
@@ -183,7 +186,7 @@ public void init(final Configuration conf,
         this.conf.getLong("replication.source.sleepforretries", 1000);
     this.fs = fs;
     this.metrics = new ReplicationSourceMetrics(peerClusterZnode);
-
+    this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf);
     try {
       this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher());
     } catch (KeeperException ke) {
@@ -263,8 +266,8 @@ public void run() {
     // normally has a position (unless the RS failed between 2 logs)
     if (this.queueRecovered) {
       try {
-        this.position = this.zkHelper.getHLogRepPosition(
-            this.peerClusterZnode, this.queue.peek().getName());
+        this.repLogReader.setPosition(this.zkHelper.getHLogRepPosition(
+            this.peerClusterZnode, this.queue.peek().getName()));
       } catch (KeeperException e) {
         this.terminate("Couldn't get the position of this recovered queue " +
             peerClusterZnode, e);
@@ -322,6 +325,7 @@ public void run() {
 
       boolean gotIOE = false;
       currentNbEntries = 0;
+      currentSize = 0;
       try {
         if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) {
           continue;
@@ -357,9 +361,7 @@ public void run() {
         }
       } finally {
         try {
-          if (this.reader != null) {
-            this.reader.close();
-          }
+          this.repLogReader.closeReader();
         } catch (IOException e) {
           gotIOE = true;
           LOG.warn("Unable to finalize the tailing of a file", e);
@@ -370,10 +372,10 @@ public void run() {
       // wait a bit and retry.
       // But if we need to stop, don't bother sleeping
       if (this.isActive() && (gotIOE || currentNbEntries == 0)) {
-        if (this.lastLoggedPosition != this.position) {
+        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.position;
+              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.repLogReader.getPosition();
         }
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
           sleepMultiplier++;
@@ -405,11 +407,9 @@ public void run() {
   protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo)
       throws IOException{
     long seenEntries = 0;
-    if (this.position != 0) {
-      this.reader.seek(this.position);
-    }
-    long startPosition = this.position;
-    HLog.Entry entry = readNextAndSetPosition();
+    this.repLogReader.seek();
+    HLog.Entry entry =
+        this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
     while (entry != null) {
       WALEdit edit = entry.getEdit();
       this.metrics.logEditsReadRate.inc(1);
@@ -433,18 +433,18 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
           }
           currentNbOperations += countDistinctRowKeys(edit);
           currentNbEntries++;
+          currentSize += entry.getEdit().heapSize();
         } else {
           this.metrics.logEditsFilteredRate.inc(1);
         }
       }
       // Stop if too many entries or too big
-      if ((this.reader.getPosition() - startPosition)
-          >= this.replicationQueueSizeCapacity ||
+      if (currentSize >= this.replicationQueueSizeCapacity ||
           currentNbEntries >= this.replicationQueueNbCapacity) {
         break;
       }
       try {
-        entry = readNextAndSetPosition();
+        entry = this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
       } catch (IOException ie) {
         LOG.debug("Break on IOE: " + ie.getMessage());
         break;
@@ -452,7 +452,7 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
     }
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +
-        " and size: " + (this.reader.getPosition() - startPosition));
+        " and size: " + this.currentSize);
     if (currentWALisBeingWrittenTo) {
       return false;
     }
@@ -461,16 +461,6 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
     return seenEntries == 0 && processEndOfFile();
   }
 
-  private HLog.Entry readNextAndSetPosition() throws IOException {
-    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
-    // Store the position so that in the future the reader can start
-    // reading from here. If the above call to next() throws an
-    // exception, the position won't be changed and retry will happen
-    // from the last known good position
-    this.position = this.reader.getPosition();
-    return entry;
-  } 
-
   private void connectToPeers() {
     // Connect to peer cluster first, unless we have to stop
     while (this.isActive() && this.currentPeers.size() == 0) {
@@ -509,10 +499,9 @@ protected boolean getNextPath() {
   protected boolean openReader(int sleepMultiplier) {
     try {
       LOG.debug("Opening log for replication " + this.currentPath.getName() +
-          " at " + this.position);
+          " at " + this.repLogReader.getPosition());
       try {
-       this.reader = null;
-       this.reader = HLog.getReader(this.fs, this.currentPath, this.conf);
+        this.reader = repLogReader.openReader(this.currentPath);
       } catch (FileNotFoundException fnfe) {
         if (this.queueRecovered) {
           // We didn't find the log in the archive directory, look if it still
@@ -648,10 +637,10 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo) {
         HRegionInterface rrs = getRS();
         LOG.debug("Replicating " + currentNbEntries);
         rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries));
-        if (this.lastLoggedPosition != this.position) {
+        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.position;
+              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.repLogReader.getPosition();
         }
         this.totalReplicatedEdits += currentNbEntries;
         this.metrics.shippedBatchesRate.inc(1);
@@ -721,7 +710,8 @@ protected boolean isPeerEnabled() {
   protected boolean processEndOfFile() {
     if (this.queue.size() != 0) {
       this.currentPath = null;
-      this.position = 0;
+      this.repLogReader.finishCurrentFile();
+      this.reader = null;
       return true;
     } else if (this.queueRecovered) {
       this.manager.closeRecoveredQueue(this);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
index 16db1675ae1f..45c47270647c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
@@ -49,6 +49,9 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
         HLogKey key = HLog.newKey(conf);
         WALEdit val = new WALEdit();
         HLog.Entry e = new HLog.Entry(key, val);
+        if (compressionContext != null) {
+          e.setCompressionContext(compressionContext);
+        }
         b = this.reader.next(e.getKey(), e.getEdit());
         nextQueue.offer(e);
         numberOfFileEntries++;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 9fa4480af770..8feb5d6db2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -475,15 +475,13 @@ public void run() {
       throw t.exception;
 
     // Make sure you can read all the content
-    SequenceFile.Reader reader
-      = new SequenceFile.Reader(this.fs, walPath, this.conf);
+    HLog.Reader reader = HLog.getReader(this.fs, walPath, this.conf);
     int count = 0;
-    HLogKey key = HLog.newKey(conf);
-    WALEdit val = new WALEdit();
-    while (reader.next(key, val)) {
+    HLog.Entry entry = new HLog.Entry();
+    while (reader.next(entry) != null) {
       count++;
       assertTrue("Should be one KeyValue per WALEdit",
-                 val.getKeyValues().size() == 1);
+                 entry.getEdit().getKeyValues().size() == 1);
     }
     assertEquals(total, count);
     reader.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 69a4e226d947..37366280e821 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -86,7 +86,7 @@ public class TestHLogSplit {
   private Configuration conf;
   private FileSystem fs;
 
-  private final static HBaseTestingUtility
+  protected final static HBaseTestingUtility
           TEST_UTIL = new HBaseTestingUtility();
 
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
new file mode 100644
index 000000000000..101678a9a21a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestHLogSplitCompressed extends TestHLogSplit {
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TestHLogSplit.setUpBeforeClass();
+    TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index 19703a25793f..e49ddc901b0d 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -61,7 +61,7 @@ public class TestReplication {
 
   private static final Log LOG = LogFactory.getLog(TestReplication.class);
 
-  private static Configuration conf1;
+  protected static Configuration conf1 = HBaseConfiguration.create();
   private static Configuration conf2;
   private static Configuration CONF_WITH_LOCALFS;
 
@@ -91,7 +91,6 @@ public class TestReplication {
    */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    conf1 = HBaseConfiguration.create();
     conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
     // smaller block size and capacity to trigger more operations
     // and test them
@@ -520,7 +519,7 @@ public void testDisableInactivePeer() throws Exception {
 
     // disable and start the peer
     admin.disablePeer("2");
-    utility2.startMiniHBaseCluster(1, 1);
+    utility2.startMiniHBaseCluster(1, 2);
     Get get = new Get(rowkey);
     for (int i = 0; i < NB_RETRIES; i++) {
       Result res = htable2.get(get);
@@ -760,7 +759,8 @@ public void queueFailover() throws Exception {
     int lastCount = 0;
 
     final long start = System.currentTimeMillis();
-    for (int i = 0; i < NB_RETRIES; i++) {
+    int i = 0;
+    while (true) {
       if (i==NB_RETRIES-1) {
         fail("Waited too much time for queueFailover replication. " +
           "Waited "+(System.currentTimeMillis() - start)+"ms.");
@@ -772,6 +772,8 @@ public void queueFailover() throws Exception {
       if (res2.length < initialCount) {
         if (lastCount < res2.length) {
           i--; // Don't increment timeout if we make progress
+        } else {
+          i++;
         }
         lastCount = res2.length;
         LOG.info("Only got " + lastCount + " rows instead of " +
@@ -791,7 +793,7 @@ public void run() {
           Thread.sleep(timeout);
           utility.expireRegionServerSession(rs);
         } catch (Exception e) {
-          LOG.error(e);
+          LOG.error("Couldn't kill a region server", e);
         }
       }
     };
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
new file mode 100644
index 000000000000..2c87a7bf524a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Run the same test as TestReplication but with HLog compression enabled
+ */
+@Category(LargeTests.class)
+public class TestReplicationWithCompression extends TestReplication {
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+    TestReplication.setUpBeforeClass();
+  }
+}

From fddb4e7805cff5c6f8b10568647fa528cd9b17e6 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 21 Dec 2012 19:43:35 +0000
Subject: [PATCH 0655/1540] HBASE-7421 TestHFileCleaner->testHFileCleaning has
 an aggressive timeout

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1425110 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
index e3ad07c73fd5..23920997d92f 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
@@ -88,7 +88,7 @@ private String getFileStats(Path file, FileSystem fs) throws IOException {
         + status.getAccessTime();
   }
 
-  @Test
+  @Test(timeout = 60 *1000)
   public void testHFileCleaning() throws Exception {
     final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
     String prefix = "someHFileThatWouldBeAUUID";

From faf20ab9043d4f353ceecc03d6eb1b222a0db667 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 21 Dec 2012 21:32:15 +0000
Subject: [PATCH 0656/1540] HBASE 7422 MasterFS doesn't set configuration for
 internal FileSystem

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1425152 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/MasterFileSystem.java |  2 +
 .../hbase/master/TestMasterFileSystem.java    | 66 +++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 612852e095ed..5999a53f120d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -99,6 +99,8 @@ public MasterFileSystem(Server master, MasterServices services,
     String fsUri = this.fs.getUri().toString();
     conf.set("fs.default.name", fsUri);
     conf.set("fs.defaultFS", fsUri);
+    // make sure the fs has the same conf
+    fs.setConf(conf);
     this.distributedLogSplitting =
       conf.getBoolean("hbase.master.distributed.log.splitting", true);
     if (this.distributedLogSplitting) {
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
new file mode 100644
index 000000000000..0781d1190dab
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the master filesystem in a local cluster
+ */
+@Category(MediumTests.class)
+public class TestMasterFileSystem {
+
+  private static final Log LOG = LogFactory.getLog(TestMasterFileSystem.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setupTest() throws Exception {
+    UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void teardownTest() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testFsUriSetProperly() throws Exception {
+    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+    MasterFileSystem fs = master.getMasterFileSystem();
+    Path masterRoot = FSUtils.getRootDir(fs.conf);
+    Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf());
+    // make sure the fs and the found root dir have the same scheme
+    LOG.debug("from fs uri:" + FileSystem.getDefaultUri(fs.getFileSystem().getConf()));
+    LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.conf));
+    // make sure the set uri matches by forcing it.
+    assertEquals(masterRoot, rootDir);
+  }
+
+}

From 85d9f6b709ccfa67e7ee60e11bad54bfeab456cd Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 22 Dec 2012 07:11:47 +0000
Subject: [PATCH 0657/1540] HBASE-7417 Test patch, hopefully fixes
 TestReplication

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1425251 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 31 ++++++++++++-------
 .../hbase/replication/TestReplication.java    |  4 +--
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 033d4b78d16c..8ecdeb23ad8c 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -39,7 +39,6 @@
 import java.util.NavigableSet;
 import java.util.Random;
 import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -1110,7 +1109,7 @@ public String checksumRows(final HTable table) throws Exception {
    */
   public int createMultiRegions(HTable table, byte[] columnFamily)
   throws IOException {
-    return createMultiRegions(getConfiguration(), table, columnFamily);
+    return createMultiRegions(table, columnFamily, true);
   }
 
   public static final byte[][] KEYS = {
@@ -1127,16 +1126,16 @@ public int createMultiRegions(HTable table, byte[] columnFamily)
 
   /**
    * Creates many regions names "aaa" to "zzz".
-   * @param c Configuration to use.
+   *
    * @param table  The table to use for the data.
    * @param columnFamily  The family to insert the data into.
+   * @param cleanupFS  True if a previous region should be remove from the FS  
    * @return count of regions created.
    * @throws IOException When creating the regions fails.
    */
-  public int createMultiRegions(final Configuration c, final HTable table,
-      final byte[] columnFamily)
+  public int createMultiRegions(HTable table, byte[] columnFamily, boolean cleanupFS)
   throws IOException {
-    return createMultiRegions(c, table, columnFamily, KEYS);
+    return createMultiRegions(getConfiguration(), table, columnFamily, KEYS, cleanupFS);
   }
 
   /**
@@ -1164,7 +1163,12 @@ public int createMultiRegions(final Configuration c, final HTable table,
   }
 
   public int createMultiRegions(final Configuration c, final HTable table,
-      final byte[] columnFamily, byte [][] startKeys)
+      final byte[] columnFamily, byte [][] startKeys) throws IOException {
+    return createMultiRegions(c, table, columnFamily, startKeys, true);
+  }
+  
+  public int createMultiRegions(final Configuration c, final HTable table,
+          final byte[] columnFamily, byte [][] startKeys, boolean cleanupFS)
   throws IOException {
     Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
     HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
@@ -1202,11 +1206,14 @@ public int createMultiRegions(final Configuration c, final HTable table,
         Bytes.toStringBinary(row));
       meta.delete(new Delete(row));
     }
-    // remove the "old" region from FS
-    Path tableDir = new Path(getDefaultRootDirPath().toString()
-        + System.getProperty("file.separator") + htd.getNameAsString()
-        + System.getProperty("file.separator") + regionToDeleteInFS);
-    getDFSCluster().getFileSystem().delete(tableDir);
+    if (cleanupFS) {
+      // see HBASE-7417 - this confused TestReplication
+      // remove the "old" region from FS
+      Path tableDir = new Path(getDefaultRootDirPath().toString()
+          + System.getProperty("file.separator") + htd.getNameAsString()
+          + System.getProperty("file.separator") + regionToDeleteInFS);
+      getDFSCluster().getFileSystem().delete(tableDir);
+    }
     // flush cache of regions
     HConnection conn = table.getConnection();
     conn.clearRegionCache();
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index e49ddc901b0d..bc2c16f89c6a 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -79,7 +79,7 @@ public class TestReplication {
   private static final int NB_ROWS_IN_BIG_BATCH =
       NB_ROWS_IN_BATCH * 10;
   private static final long SLEEP_TIME = 1500;
-  private static final int NB_RETRIES = 10;
+  private static final int NB_RETRIES = 15;
 
   private static final byte[] tableName = Bytes.toBytes("test");
   private static final byte[] famName = Bytes.toBytes("f");
@@ -716,7 +716,7 @@ public void testVerifyRepJob() throws Exception {
    */
   @Test(timeout=300000)
   public void queueFailover() throws Exception {
-    utility1.createMultiRegions(htable1, famName);
+    utility1.createMultiRegions(htable1, famName, false);
 
     // killing the RS with .META. can result into failed puts until we solve
     // IO fencing

From e1fba0e20eb7cb6adf317c06695b9f96e7c99c5b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Dec 2012 02:10:15 +0000
Subject: [PATCH 0658/1540] HBASE-7431 TestSplitTransactionOnCluster tests
 still flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1425573 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HBaseTestingUtility.java   | 10 +++++++---
 .../TestSplitTransactionOnCluster.java             | 14 ++++++++++----
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 8ecdeb23ad8c..91dc7827433e 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1702,17 +1702,21 @@ public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
     throws IOException {
     boolean startedServer = ensureSomeRegionServersAvailable(num);
 
+    int nonStoppedServers = 0;
     for (JVMClusterUtil.RegionServerThread rst :
       getMiniHBaseCluster().getRegionServerThreads()) {
 
       HRegionServer hrs = rst.getRegionServer();
       if (hrs.isStopping() || hrs.isStopped()) {
         LOG.info("A region server is stopped or stopping:"+hrs);
-        LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
-        startedServer = true;
+      } else {
+        nonStoppedServers++;
       }
     }
-
+    for (int i=nonStoppedServers; i<num; ++i) {
+      LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
+      startedServer = true;
+    }
     return startedServer;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 1d3d1db9e33e..a2fbbfc3886d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -539,14 +539,13 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
    * @throws KeeperException
    */
   @Test
-  public void testSplitBeforeSettingSplittingInZK() throws IOException,
+  public void testSplitBeforeSettingSplittingInZK() throws Exception,
       InterruptedException, KeeperException {
     testSplitBeforeSettingSplittingInZK(true);
     testSplitBeforeSettingSplittingInZK(false);
   }
 
-  private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws IOException,
-      KeeperException {
+  private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws Exception {
     final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
     
     HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
@@ -556,7 +555,12 @@ private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws IOE
       htd.addFamily(new HColumnDescriptor("cf"));
       admin.createTable(htd);
 
-      List<HRegion> regions = cluster.getRegions(tableName);
+      List<HRegion> regions = null;
+      for (int i=0; i<100; i++) {
+        regions = cluster.getRegions(tableName);
+        if (regions.size() > 0) break;
+        Thread.sleep(100);
+      }
       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
       SplitTransaction st = null;
@@ -582,6 +586,8 @@ void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName se
       } catch (IOException e) {
         String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
             .getRegionInfo().getEncodedName());
+        // make sure the client is uptodate
+        regionServer.getZooKeeper().sync(node);
         if (nodeCreated) {
           assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
         } else {

From 70a6b1f909d1c2c17d90f9f19b5cfd9ba3da382f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 25 Dec 2012 05:03:27 +0000
Subject: [PATCH 0659/1540] HBASE-7432 TestHBaseFsck prevents testsuite from
 finishing

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1425719 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b86d05872aa2..6f09507d5cff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2881,7 +2881,7 @@ protected boolean closeRegion(HRegionInfo region, final boolean zk,
     checkOpen();
     //Check for permissions to close.
     HRegion actualRegion = this.getFromOnlineRegions(region.getEncodedName());
-    if (actualRegion.getCoprocessorHost() != null) {
+    if (actualRegion != null && actualRegion.getCoprocessorHost() != null) {
       actualRegion.getCoprocessorHost().preClose(false);
     }
     LOG.info("Received close region: " + region.getRegionNameAsString() +
@@ -2933,7 +2933,7 @@ protected boolean closeRegion(HRegionInfo region, final boolean abort,
       final boolean zk, final int versionOfClosingNode) {
     
     HRegion actualRegion = this.getFromOnlineRegions(region.getEncodedName());
-    if ((actualRegion != null) && (actualRegion.getCoprocessorHost() !=null)){
+    if (actualRegion != null && actualRegion.getCoprocessorHost() != null) {
       try {
         actualRegion.getCoprocessorHost().preClose(abort);
       } catch (IOException e) {

From f356d580c5f9bd2ca7f4db5b44af05ed10b652df Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 25 Dec 2012 16:58:50 +0000
Subject: [PATCH 0660/1540] HBASE-7435 BuiltInGzipDecompressor is only released
 during full GC

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1425756 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/io/hfile/Compression.java    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
index 45f8c89793e4..0498ee6aaf38 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.DoNotPool;
 import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -308,6 +309,9 @@ public Decompressor getDecompressor() {
     public void returnDecompressor(Decompressor decompressor) {
       if (decompressor != null) {
         CodecPool.returnDecompressor(decompressor);
+        if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
+          decompressor.end();
+        }
       }
     }
 

From 16ba7941aec71878b7b2002acb3fd57a26f570ee Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 26 Dec 2012 22:31:54 +0000
Subject: [PATCH 0661/1540] HBASE-7438 TestSplitTransactionOnCluster has too
 many infinite loops

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426067 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestSplitTransactionOnCluster.java        | 77 ++++++++++++-------
 1 file changed, 49 insertions(+), 28 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index a2fbbfc3886d..7519c86cf4b4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -21,9 +21,11 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.util.List;
@@ -162,12 +164,8 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       // Now crash the server
       cluster.abortRegionServer(tableRegionIndex);
       waitUntilRegionServerDead();
+      awaitDaughters(tableName, daughters.size());
 
-      // Wait till regions are back on line again.
-      while(cluster.getRegions(tableName).size() < daughters.size()) {
-        LOG.info("Waiting for repair to happen");
-        Thread.sleep(1000);
-      }
       // Assert daughters are online.
       regions = cluster.getRegions(tableName);
       for (HRegion r: regions) {
@@ -282,11 +280,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       // Now crash the server
       cluster.abortRegionServer(tableRegionIndex);
       waitUntilRegionServerDead();
-      // Wait till regions are back on line again.
-      while(cluster.getRegions(tableName).size() < daughters.size()) {
-        LOG.info("Waiting for repair to happen");
-        Thread.sleep(1000);
-      }
+      awaitDaughters(tableName, daughters.size());
       // Assert daughters are online.
       regions = cluster.getRegions(tableName);
       for (HRegion r: regions) {
@@ -344,21 +338,18 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
         if (r.getRegionInfo().equals(daughter)) daughterRegion = r;
       }
       assertTrue(daughterRegion != null);
-      while (true) {
+      for (int i=0; i<100; i++) {
         if (!daughterRegion.hasReferences()) break;
         Threads.sleep(100);
       }
+      assertFalse("Waiting for refereces to be compacted", daughterRegion.hasReferences());
       split(daughter, server, regionCount);
       // Get list of daughters
       daughters = cluster.getRegions(tableName);
       // Now crash the server
       cluster.abortRegionServer(tableRegionIndex);
       waitUntilRegionServerDead();
-      // Wait till regions are back on line again.
-      while(cluster.getRegions(tableName).size() < daughters.size()) {
-        LOG.info("Waiting for repair to happen");
-        Thread.sleep(1000);
-      }
+      awaitDaughters(tableName, daughters.size());
       // Assert daughters are online and ONLY the original daughters -- that
       // fixup didn't insert one during server shutdown recover.
       regions = cluster.getRegions(tableName);
@@ -504,12 +495,13 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
       byte[] data = ZKUtil.getDataNoWatch(t.getConnection()
           .getZooKeeperWatcher(), node, stat);
       // ZKUtil.create
-      while (data != null) {
+      for (int i=0; data != null && i<60; i++) {
         Thread.sleep(1000);
         data = ZKUtil.getDataNoWatch(t.getConnection().getZooKeeperWatcher(),
             node, stat);
 
       }
+      assertNull("Waited too long for ZK node to be removed: "+node, data);
       
       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
 
@@ -634,18 +626,24 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
 
         assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
         AssignmentManager am = cluster.getMaster().getAssignmentManager();
-        while (!am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName())) {
+        for (int i = 0; !am.getRegionsInTransition().containsKey(
+            regions.get(0).getRegionInfo().getEncodedName())
+            && i < 10; i++) {
           Thread.sleep(200);
         }
+        assertTrue("region is not in transition",
+            am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName()));
         RegionState regionState = am.getRegionsInTransition().get(regions.get(0).getRegionInfo()
             .getEncodedName());
         assertTrue(regionState.getState() == RegionState.State.SPLITTING);
         assertTrue(st.rollback(regionServer, regionServer));
         assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-        while (am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName())) {
+        for (int i=0; am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName()) && i<10; i++) {
           // Just in case the nodeDeleted event did not get executed.
           Thread.sleep(200);
         }
+        assertFalse("region is still in transition",
+            am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName()));
       }
     } finally {
       if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
@@ -667,9 +665,12 @@ public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exc
       htd.addFamily(new HColumnDescriptor("cf"));
       admin.createTable(htd);
       HTable t = new HTable(cluster.getConfiguration(), tableName);
-      while (!(cluster.getRegions(tableName).size() == 1)) {
+      // wait for up to 10s
+      for (int i=0; cluster.getRegions(tableName).size() != 1 && i<100; i++) {
         Thread.sleep(100);
       }
+      assertTrue("waited too long for table to get online",
+          cluster.getRegions(tableName).size() == 1);
       final List<HRegion> regions = cluster.getRegions(tableName);
       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
@@ -692,9 +693,10 @@ public void run() {
           }
         }
       }.start();
-      while (!callRollBack) {
+      for (int i=0; !callRollBack && i<100; i++) {
         Thread.sleep(100);
       }
+      assertTrue("Waited too long for rollback", callRollBack);
       SplitTransaction st = null;
       st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
       try {
@@ -705,14 +707,16 @@ public void run() {
         LOG.debug("Rollback started :"+ e.getMessage());
         st.rollback(regionServer, regionServer);
       }
-      while (!firstSplitCompleted) {
+      for (int i=0; !firstSplitCompleted && i<100; i++) {
         Thread.sleep(100);
       }
+      assertTrue("fist split did not complete", firstSplitCompleted);
       NavigableMap<String, RegionState> rit = cluster.getMaster().getAssignmentManager()
           .getRegionsInTransition();
-      while (rit.containsKey(hri.getTableNameAsString())) {
+      for (int i=0; rit.containsKey(hri.getTableNameAsString()) && i<100; i++) {
         Thread.sleep(100);
       }
+      assertFalse("region still in transition", rit.containsKey(hri.getTableNameAsString()));
       List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
       // Region server side split is successful.
       assertEquals("The parent region should be splitted", 2, onlineRegions.size());
@@ -799,9 +803,10 @@ public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfull
       HTableDescriptor htd = new HTableDescriptor(tableName);
       htd.addFamily(new HColumnDescriptor("cf"));
       admin.createTable(htd);
-      while (!(cluster.getRegions(tableName).size() == 1)) {
+      for (int i=0; cluster.getRegions(tableName).size() != 1 && i<100; i++) {
         Thread.sleep(100);
       }
+      assertTrue("Table not online", cluster.getRegions(tableName).size() == 1);
       List<HRegion> regions = cluster.getRegions(tableName);
       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
       int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
@@ -910,10 +915,11 @@ private void split(final HRegionInfo hri, final HRegionServer server,
       final int regionCount)
   throws IOException, InterruptedException {
     this.admin.split(hri.getRegionNameAsString());
-    while (server.getOnlineRegions().size() <= regionCount) {
+    for (int i=0; server.getOnlineRegions().size() <= regionCount && i<100; i++) {
       LOG.debug("Waiting on region to split");
       Thread.sleep(100);
     }
+    assertFalse("Waited too long for split", server.getOnlineRegions().size() <= regionCount);
   }
 
   private void removeDaughterFromMeta(final byte [] regionName) throws IOException {
@@ -960,13 +966,15 @@ private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
         Bytes.toBytes(hrs.getServerName().toString()));
     }
     // Wait till table region is up on the server that is NOT carrying .META..
-    while (true) {
+    for (int i=0; i<100; i++) {
       tableRegionIndex = cluster.getServerWith(hri.getRegionName());
       if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
       LOG.debug("Waiting on region move off the .META. server; current index " +
         tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
       Thread.sleep(100);
     }
+    assertTrue("Region not moved off .META. server", tableRegionIndex != -1
+        && tableRegionIndex != metaServerIndex);
     // Verify for sure table region is not on same server as .META.
     tableRegionIndex = cluster.getServerWith(hri.getRegionName());
     assertTrue(tableRegionIndex != -1);
@@ -1004,11 +1012,24 @@ private void printOutRegions(final HRegionServer hrs, final String prefix)
 
   private void waitUntilRegionServerDead() throws InterruptedException {
     // Wait until the master processes the RS shutdown
-    while (cluster.getMaster().getClusterStatus().
-        getServers().size() == NB_SERVERS) {
+    for (int i=0; cluster.getMaster().getClusterStatus().
+        getServers().size() == NB_SERVERS && i<100; i++) {
       LOG.info("Waiting on server to go down");
       Thread.sleep(100);
     }
+    assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
+    getServers().size() == NB_SERVERS);
+  }
+
+  private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
+    // Wait till regions are back on line again.
+    for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
+      LOG.info("Waiting for repair to happen");
+      Thread.sleep(1000);
+    }
+    if (cluster.getRegions(tableName).size() < numDaughters) {
+      fail("Waiting too long for daughter regions");
+    }
   }
 
   @org.junit.Rule

From 06b743303ed9bca522797786f5f3daaafa85931d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 28 Dec 2012 06:31:55 +0000
Subject: [PATCH 0662/1540] HBASE-3776 Add Bloom Filter Support to
 HFileOutputFormat (Anoop Sam John)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426415 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/HFileOutputFormat.java    | 101 +++++++++++++-----
 1 file changed, 77 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
index d2962ee6b9e6..acb7d73f5631 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
@@ -47,6 +47,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -55,11 +56,10 @@
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Partitioner;
 import org.apache.hadoop.mapreduce.RecordWriter;
@@ -79,7 +79,7 @@
 public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable, KeyValue> {
   static Log LOG = LogFactory.getLog(HFileOutputFormat.class);
   static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression";
-  TimeRangeTracker trt = new TimeRangeTracker();
+  private static final String BLOOM_TYPE_CONF_KEY = "hbase.hfileoutputformat.families.bloomtype";
   private static final String DATABLOCK_ENCODING_CONF_KEY = 
      "hbase.mapreduce.hfileoutputformat.datablock.encoding";
 
@@ -103,6 +103,7 @@ public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final Task
 
     // create a map from column family to the compression algorithm
     final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf);
+    final Map<byte[], String> bloomTypeMap = createFamilyBloomMap(conf);
     
     String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
     final HFileDataBlockEncoder encoder;
@@ -164,7 +165,6 @@ public void write(ImmutableBytesWritable row, KeyValue kv)
 
         // we now have the proper HLog writer. full steam ahead
         kv.updateLatestStamp(this.now);
-        trt.includeTimestamp(kv);
         wl.writer.append(kv);
         wl.written += length;
 
@@ -185,9 +185,9 @@ private void rollWriters() throws IOException {
         this.rollRequested = false;
       }
 
-      /* Create a new HFile.Writer.
+      /* Create a new StoreFile.Writer.
        * @param family
-       * @return A WriterLength, containing a new HFile.Writer.
+       * @return A WriterLength, containing a new StoreFile.Writer.
        * @throws IOException
        */
       private WriterLength getNewWriter(byte[] family, Configuration conf)
@@ -196,20 +196,28 @@ private WriterLength getNewWriter(byte[] family, Configuration conf)
         Path familydir = new Path(outputdir, Bytes.toString(family));
         String compression = compressionMap.get(family);
         compression = compression == null ? defaultCompression : compression;
-        wl.writer = HFile.getWriterFactoryNoCache(conf)
-            .withPath(fs, StoreFile.getUniqueFile(fs, familydir))
-            .withBlockSize(blocksize)
-            .withCompression(compression)
-            .withComparator(KeyValue.KEY_COMPARATOR)
+        String bloomTypeStr = bloomTypeMap.get(family);
+        BloomType bloomType = BloomType.NONE;
+        if (bloomTypeStr != null) {
+          bloomType = BloomType.valueOf(bloomTypeStr);
+        }
+        Configuration tempConf = new Configuration(conf);
+        tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
+        wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blocksize)
+            .withOutputDir(familydir)
+            .withCompression(AbstractHFileWriter.compressionByName(compression))
+            .withBloomType(bloomType)
+            .withComparator(KeyValue.COMPARATOR)
             .withDataBlockEncoder(encoder)
             .withChecksumType(Store.getChecksumType(conf))
             .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
-            .create();
+            .build();
+
         this.writers.put(family, wl);
         return wl;
       }
 
-      private void close(final HFile.Writer w) throws IOException {
+      private void close(final StoreFile.Writer w) throws IOException {
         if (w != null) {
           w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
               Bytes.toBytes(System.currentTimeMillis()));
@@ -219,8 +227,7 @@ private void close(final HFile.Writer w) throws IOException {
               Bytes.toBytes(true));
           w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY,
               Bytes.toBytes(compactionExclude));
-          w.appendFileInfo(StoreFile.TIMERANGE_KEY,
-              WritableUtils.toByteArray(trt));
+          w.appendTrackedTimestampsToMetadata();
           w.close();
         }
       }
@@ -239,7 +246,7 @@ public void close(TaskAttemptContext c)
    */
   static class WriterLength {
     long written = 0;
-    HFile.Writer writer = null;
+    StoreFile.Writer writer = null;
   }
 
   /**
@@ -366,7 +373,8 @@ public static void configureIncrementalLoad(Job job, HTable table)
 
     // Set compression algorithms based on column families
     configureCompression(table, conf);
-
+    configureBloomType(table, conf);
+    
     TableMapReduceUtil.addDependencyJars(job);
     LOG.info("Incremental table output configured.");
   }
@@ -403,25 +411,39 @@ private static Class<? extends Partitioner> getTotalOrderPartitionerClass()
    *         algorithm
    */
   static Map<byte[], String> createFamilyCompressionMap(Configuration conf) {
-    Map<byte[], String> compressionMap = new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR);
-    String compressionConf = conf.get(COMPRESSION_CONF_KEY, "");
-    for (String familyConf : compressionConf.split("&")) {
+    return createFamilyConfValueMap(conf, COMPRESSION_CONF_KEY);
+  }
+
+  private static Map<byte[], String> createFamilyBloomMap(Configuration conf) {
+    return createFamilyConfValueMap(conf, BLOOM_TYPE_CONF_KEY);
+  }
+  
+  /**
+   * Run inside the task to deserialize column family to given conf value map.
+   * 
+   * @param conf
+   * @param confName
+   * @return a map of column family to the given configuration value
+   */
+  private static Map<byte[], String> createFamilyConfValueMap(Configuration conf, String confName) {
+    Map<byte[], String> confValMap = new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR);
+    String confVal = conf.get(confName, "");
+    for (String familyConf : confVal.split("&")) {
       String[] familySplit = familyConf.split("=");
       if (familySplit.length != 2) {
         continue;
       }
-
       try {
-        compressionMap.put(URLDecoder.decode(familySplit[0], "UTF-8").getBytes(),
+        confValMap.put(URLDecoder.decode(familySplit[0], "UTF-8").getBytes(),
             URLDecoder.decode(familySplit[1], "UTF-8"));
       } catch (UnsupportedEncodingException e) {
         // will not happen with UTF-8 encoding
         throw new AssertionError(e);
       }
     }
-    return compressionMap;
+    return confValMap;
   }
-
+  
   /**
    * Serialize column family to compression algorithm map to configuration.
    * Invoked while configuring the MR job for incremental load.
@@ -451,4 +473,35 @@ static void configureCompression(HTable table, Configuration conf) throws IOExce
     // Get rid of the last ampersand
     conf.set(COMPRESSION_CONF_KEY, compressionConfigValue.toString());
   }
+  
+  /**
+   * Serialize column family to bloom type map to configuration.
+   * Invoked while configuring the MR job for incremental load.
+   *
+   * @throws IOException
+   *           on failure to read column family descriptors
+   */
+  static void configureBloomType(HTable table, Configuration conf) throws IOException {
+    HTableDescriptor tableDescriptor = table.getTableDescriptor();
+    if (tableDescriptor == null) {
+      // could happen with mock table instance
+      return;
+    }
+    StringBuilder bloomTypeConfigValue = new StringBuilder();
+    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
+    int i = 0;
+    for (HColumnDescriptor familyDescriptor : families) {
+      if (i++ > 0) {
+        bloomTypeConfigValue.append('&');
+      }
+      bloomTypeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
+      bloomTypeConfigValue.append('=');
+      String bloomType = familyDescriptor.getBloomFilterType().toString();
+      if (bloomType == null) {
+        bloomType = HColumnDescriptor.DEFAULT_BLOOMFILTER;
+      }
+      bloomTypeConfigValue.append(URLEncoder.encode(bloomType, "UTF-8"));
+    }
+    conf.set(BLOOM_TYPE_CONF_KEY, bloomTypeConfigValue.toString());
+  }
 }

From ea66addec8efdb1680cef5d5c0e92a6df1016b78 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 28 Dec 2012 23:14:31 +0000
Subject: [PATCH 0663/1540] HBASE-7412 Fix how HTableDescriptor handles default
 max file size and flush size

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426658 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HTableDescriptor.java | 33 ++++++++++---------
 .../ConstantSizeRegionSplitPolicy.java        | 15 +++++----
 .../hadoop/hbase/regionserver/HRegion.java    |  4 +--
 ...creasingToUpperBoundRegionSplitPolicy.java | 12 +++++--
 .../hadoop/hbase/TestHTableDescriptor.java    | 21 ++++++++++++
 .../apache/hadoop/hbase/client/TestAdmin.java |  6 ++++
 6 files changed, 64 insertions(+), 27 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index df63478a7382..c17765e3535a 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -31,10 +31,9 @@
 import java.util.TreeMap;
 import java.util.regex.Matcher;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.WritableComparable;
@@ -589,20 +588,21 @@ public void setName(byte[] name) {
 
   /** 
    * Returns the maximum size upto which a region can grow to after which a region
-   * split is triggered. The region size is represented by the size of the biggest 
+   * split is triggered. The region size is represented by the size of the biggest
    * store file in that region.
-   * 
-   * @return max hregion size for table
-   * 
+   *
+   * @return max hregion size for table, -1 if not set.
+   *
    * @see #setMaxFileSize(long)
    */
   public long getMaxFileSize() {
     byte [] value = getValue(MAX_FILESIZE_KEY);
-    if (value != null)
-      return Long.valueOf(Bytes.toString(value)).longValue();
-    return HConstants.DEFAULT_MAX_FILE_SIZE;
+    if (value != null) {
+      return Long.parseLong(Bytes.toString(value));
+    }
+    return -1;
   }
-  
+
   /**
    * Sets the maximum size upto which a region can grow to after which a region
    * split is triggered. The region size is represented by the size of the biggest 
@@ -624,16 +624,17 @@ public void setMaxFileSize(long maxFileSize) {
 
   /**
    * Returns the size of the memstore after which a flush to filesystem is triggered.
-   * 
-   * @return memory cache flush size for each hregion
-   * 
+   *
+   * @return memory cache flush size for each hregion, -1 if not set.
+   *
    * @see #setMemStoreFlushSize(long)
    */
   public long getMemStoreFlushSize() {
     byte [] value = getValue(MEMSTORE_FLUSHSIZE_KEY);
-    if (value != null)
-      return Long.valueOf(Bytes.toString(value)).longValue();
-    return DEFAULT_MEMSTORE_FLUSH_SIZE;
+    if (value != null) {
+      return Long.parseLong(Bytes.toString(value));
+    }
+    return -1;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 2098c55cc6dc..90ba28138ad6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -17,7 +17,9 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
 
 /**
  * A {@link RegionSplitPolicy} implementation which splits a region
@@ -34,14 +36,15 @@ public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
   @Override
   protected void configureForRegion(HRegion region) {
     super.configureForRegion(region);
-    long maxFileSize = region.getTableDesc().getMaxFileSize();
-
-    // By default we split region if a file > HConstants.DEFAULT_MAX_FILE_SIZE.
-    if (maxFileSize == HConstants.DEFAULT_MAX_FILE_SIZE) {
-      maxFileSize = getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+    Configuration conf = getConf();
+    HTableDescriptor desc = region.getTableDesc();
+    if (desc != null) {
+      this.desiredMaxFileSize = desc.getMaxFileSize();
+    }
+    if (this.desiredMaxFileSize <= 0) {
+      this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
         HConstants.DEFAULT_MAX_FILE_SIZE);
     }
-    this.desiredMaxFileSize = maxFileSize;
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 8aed783440f3..e2e6fab65bf6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -465,9 +465,9 @@ void setHTableSpecificConf() {
     LOG.info("Setting up tabledescriptor config now ...");
     long flushSize = this.htableDescriptor.getMemStoreFlushSize();
 
-    if (flushSize == HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE) {
+    if (flushSize <= 0) {
       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
-         HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
+        HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
     }
     this.memstoreFlushSize = flushSize;
     this.blockingMemStoreSize = this.memstoreFlushSize *
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
index bb9c52e0ca18..0c9f15195ffa 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
@@ -22,6 +22,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -45,10 +46,15 @@ public class IncreasingToUpperBoundRegionSplitPolicy
   @Override
   protected void configureForRegion(HRegion region) {
     super.configureForRegion(region);
-    this.flushSize = region.getTableDesc() != null?
-      region.getTableDesc().getMemStoreFlushSize():
-      getConf().getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
+    Configuration conf = getConf();
+    HTableDescriptor desc = region.getTableDesc();
+    if (desc != null) {
+      this.flushSize = desc.getMemStoreFlushSize();
+    }
+    if (this.flushSize <= 0) {
+      this.flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
         HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
+    }
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
index f7c0ccaec0a2..15e9d936c3e5 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
@@ -63,4 +63,25 @@ public void testRemoveString() throws Exception {
     assertEquals(null, desc.getValue(key));
   }
 
+  /**
+   * Test default value handling for maxFileSize
+   */
+  @Test
+  public void testGetMaxFileSize() {
+    HTableDescriptor desc = new HTableDescriptor("table");
+    assertEquals(-1, desc.getMaxFileSize());
+    desc.setMaxFileSize(1111L);
+    assertEquals(1111L, desc.getMaxFileSize());
+  }
+
+  /**
+   * Test default value handling for memStoreFlushSize
+   */
+  @Test
+  public void testGetMemStoreFlushSize() {
+    HTableDescriptor desc = new HTableDescriptor("table");
+    assertEquals(-1, desc.getMemStoreFlushSize());
+    desc.setMemStoreFlushSize(1111L);
+    assertEquals(1111L, desc.getMemStoreFlushSize());
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 60adb5ce12ec..db12f999924c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -368,6 +368,9 @@ public void testOnlineChangeTableSchema() throws IOException, InterruptedExcepti
     assertTrue(htd.equals(copy));
     // Now amend the copy. Introduce differences.
     long newFlushSize = htd.getMemStoreFlushSize() / 2;
+    if (newFlushSize <=0) {
+      newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
+    }
     copy.setMemStoreFlushSize(newFlushSize);
     final String key = "anyoldkey";
     assertTrue(htd.getValue(key) == null);
@@ -456,6 +459,9 @@ public void testShouldFailOnlineSchemaUpdateIfOnlineSchemaIsNotEnabled()
     assertTrue(htd.equals(copy));
     // Now amend the copy. Introduce differences.
     long newFlushSize = htd.getMemStoreFlushSize() / 2;
+    if (newFlushSize <=0) {
+      newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
+    }
     copy.setMemStoreFlushSize(newFlushSize);
     final String key = "anyoldkey";
     assertTrue(htd.getValue(key) == null);

From 922ce40a536004fcc5a8268405f0af26b56f424b Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 29 Dec 2012 01:16:51 +0000
Subject: [PATCH 0664/1540] HBASE-7412 Fix how HTableDescriptor handles default
 max file size and flush size - ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426682 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/backup/TestHFileArchiving.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index e1dd0c5ddb4c..0b4f6dfbbd6a 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -81,6 +82,10 @@ private static void setupConf(Configuration conf) {
     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
     // disable major compactions
     conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
+
+    // prevent aggressive region split
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      ConstantSizeRegionSplitPolicy.class.getName());
   }
 
   @Before

From ec8f84a8c598b32688c741de5ceaf8a020d76453 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 29 Dec 2012 02:32:25 +0000
Subject: [PATCH 0665/1540] pom.xml, changes.txt for 0.94.4

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426687 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index fed92a2f5c8d..dcfe597074b2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,87 @@
 HBase Change Log
 
+Release 0.94.4 - 12/28/2012
+Sub-task
+
+    [HBASE-3776] - Add Bloom Filter Support to HFileOutputFormat
+    [HBASE-6206] - Large tests fail with jdk1.7
+    [HBASE-7009] - Port HBaseCluster interface/tests to 0.94
+    [HBASE-7042] - Master Coprocessor Endpoint
+    [HBASE-7282] - Backport Compaction Tool to 0.94
+    [HBASE-7331] - Add access control for region open and close, row locking, and stopping the regionserver
+    [HBASE-7336] - HFileBlock.readAtOffset does not work well with multiple threads
+    [HBASE-7371] - Blocksize in TestHFileBlock is unintentionally small
+    [HBASE-7431] - TestSplitTransactionOnCluster tests still flaky
+    [HBASE-7438] - TestSplitTransactionOnCluster has too many infinite loops
+
+Bug
+
+    [HBASE-6175] - TestFSUtils flaky on hdfs getFileStatus method
+    [HBASE-6317] - Master clean start up and Partially enabled tables make region assignment inconsistent.
+    [HBASE-6327] - HLog can be null when create table
+    [HBASE-6423] - Writes should not block reads on blocking updates to memstores
+    [HBASE-7091] - support custom GC options in hbase-env.sh
+    [HBASE-7158] - Allow CopyTable to identify the source cluster (for replication scenarios)
+    [HBASE-7165] - TestSplitLogManager.testUnassignedTimeout is flaky
+    [HBASE-7166] - TestSplitTransactionOnCluster tests are flaky
+    [HBASE-7172] - TestSplitLogManager.testVanishingTaskZNode() fails when run individually and is flaky
+    [HBASE-7177] - TestZooKeeperScanPolicyObserver.testScanPolicyObserver is flaky
+    [HBASE-7180] - RegionScannerImpl.next() is inefficient.
+    [HBASE-7205] - Coprocessor classloader is replicated for all regions in the HRegionServer
+    [HBASE-7214] - CleanerChore logs too much, so much so it obscures all else that is going on
+    [HBASE-7230] - port HBASE-7109 integration tests on cluster are not getting picked up from distribution to 0.94
+    [HBASE-7235] - TestMasterObserver is flaky
+    [HBASE-7251] - Avoid flood logs during client disconnect during batch get operation
+    [HBASE-7252] - TestSizeBasedThrottler fails occasionally
+    [HBASE-7259] - Deadlock in HBaseClient when KeeperException occured
+    [HBASE-7260] - Upgrade hadoop 1 dependency to hadoop 1.1.1
+    [HBASE-7273] - Upgrade zookeeper dependency to 3.4.5 for 0.94
+    [HBASE-7279] - Avoid copying the rowkey in RegionScanner, StoreScanner, and ScanQueryMatcher
+    [HBASE-7300] - HbckTestingUtil needs to keep a static executor to lower the number of threads used
+    [HBASE-7301] - Force ipv4 for unit tests
+    [HBASE-7307] - MetaReader.tableExists should not return false if the specified table regions has been split
+    [HBASE-7338] - Fix flaky condition for org.apache.hadoop.hbase.TestRegionRebalancing.testRebalanceOnRegionServerNumberChange
+    [HBASE-7342] - Split operation without split key incorrectly finds the middle key in off-by-one error
+    [HBASE-7343] - Fix flaky condition for TestDrainingServer
+    [HBASE-7357] - HBaseClient and HBaseServer should use hbase.security.authentication when negotiating authentication
+    [HBASE-7376] - Acquiring readLock does not apply timeout in HRegion#flushcache
+    [HBASE-7398] - [0.94 UNIT TESTS] TestAssignmentManager fails frequently on CentOS 5
+    [HBASE-7412] - Fix how HTableDescriptor handles default max file size and flush size
+    [HBASE-7417] - TestReplication is flaky
+    [HBASE-7421] - TestHFileCleaner->testHFileCleaning has an aggressive timeout
+    [HBASE-7422] - MasterFS doesn't set configuration for internal FileSystem
+    [HBASE-7432] - TestHBaseFsck prevents testsuite from finishing
+    [HBASE-7435] - BuiltInGzipDecompressor is only released during full GC
+
+Improvement
+
+    [HBASE-4791] - Allow Secure Zookeeper JAAS configuration to be programmatically set (rather than only by reading JAAS configuration file)
+    [HBASE-5616] - Make compaction code standalone
+    [HBASE-5693] - When creating a region, the master initializes it and creates a memstore within the master server
+    [HBASE-5778] - Fix HLog compression's incompatibilities
+    [HBASE-5888] - Clover profile in build
+    [HBASE-6585] - Audit log messages should contain info about the higher level operation being executed
+    [HBASE-6775] - Use ZK.multi when available for HBASE-6710 0.92/0.94 compatibility fix
+    [HBASE-7190] - Add an option to hbck to check only meta and assignment
+    [HBASE-7197] - Add multi get to RemoteHTable
+    [HBASE-7199] - hbck should check lingering reference hfile and have option to sideline them automatically
+    [HBASE-7204] - Make hbck ErrorReporter pluggable
+    [HBASE-7231] - port HBASE-7200 create integration test for balancing regions and killing region servers to 0.94
+    [HBASE-7249] - add test name filter to IntegrationTestsDriver
+    [HBASE-7328] - IntegrationTestRebalanceAndKillServersTargeted supercedes IntegrationTestRebalanceAndKillServers, remove
+    [HBASE-7359] - [REST] 'accessToken' in RemoteHTable is vestigial
+    [HBASE-7374] - Expose master table operations for coprocessors by way of MasterServices
+    [HBASE-7377] - Clean up TestHBase7051
+    [HBASE-7381] - Lightweight data transfer for Class Result
+
+Task
+
+    [HBASE-5258] - Move coprocessors set out of RegionLoad
+    [HBASE-7170] - [0.94 branch] Allow HConnectionImplementation to reconnect to master multiple times
+    [HBASE-7283] - Backport HBASE-6564 + HBASE-7202 to 0.94
+    [HBASE-7341] - Deprecate RowLocks in 0.94
+
+
 Release 0.94.3 - 11/12/2012
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index c67fe64cefc8..90f9011914af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.4-SNAPSHOT</version>
+  <version>0.94.4</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 3e3b4b6fd4ebd7390f55bd1a62f47ebdd774334f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 29 Dec 2012 04:18:49 +0000
Subject: [PATCH 0666/1540] HBASE-7440 ReplicationZookeeper#addPeer is racy
 (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426704 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                           |  1 +
 .../hadoop/hbase/replication/ReplicationPeer.java     |  7 +++++--
 .../hbase/replication/ReplicationZookeeper.java       | 11 +++++++----
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index dcfe597074b2..c87308c795ce 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -52,6 +52,7 @@ Bug
     [HBASE-7422] - MasterFS doesn't set configuration for internal FileSystem
     [HBASE-7432] - TestHBaseFsck prevents testsuite from finishing
     [HBASE-7435] - BuiltInGzipDecompressor is only released during full GC
+    [HBASE-7440] - ReplicationZookeeper#addPeer is racy
 
 Improvement
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index c972a5d92835..d977830efc29 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -79,8 +79,11 @@ public ReplicationPeer(Configuration conf, String key,
   public void startStateTracker(ZooKeeperWatcher zookeeper, String peerStateNode)
       throws KeeperException {
     if (ZKUtil.checkExists(zookeeper, peerStateNode) == -1) {
-      ZKUtil.createAndWatch(zookeeper, peerStateNode,
-          Bytes.toBytes(PeerState.ENABLED.name())); // enabled by default
+      // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
+      // peer-state znode. This happens while adding a peer.
+      // The peer state data is set as "ENABLED" by default.
+      ZKUtil.createNodeIfNotExistsAndWatch(zookeeper, peerStateNode,
+        Bytes.toBytes(PeerState.ENABLED.name()));
     }
     this.peerStateTracker = new PeerStateTracker(peerStateNode, zookeeper,
         this);
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index bc20acd9b976..0e73431e35c9 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -389,10 +389,13 @@ public void addPeer(String id, String clusterKey) throws IOException {
         throw new IllegalArgumentException("Cannot add existing peer");
       }
       ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
-      ZKUtil.createAndWatch(this.zookeeper,
-          ZKUtil.joinZNode(this.peersZNode, id), Bytes.toBytes(clusterKey));
-      ZKUtil.createAndWatch(this.zookeeper, getPeerStateNode(id),
-          Bytes.toBytes(PeerState.ENABLED.name())); // enabled by default
+      ZKUtil.createAndWatch(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id),
+        Bytes.toBytes(clusterKey));
+      // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
+      // peer-state znode. This happens while adding a peer.
+      // The peer state data is set as "ENABLED" by default.
+      ZKUtil.createNodeIfNotExistsAndWatch(this.zookeeper, getPeerStateNode(id),
+        Bytes.toBytes(PeerState.ENABLED.name()));
     } catch (KeeperException e) {
       throw new IOException("Unable to add peer", e);
     }

From 5552e068754184ef2797d9c2d30b1119ce108d78 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 29 Dec 2012 06:23:37 +0000
Subject: [PATCH 0667/1540] HBASE-7455 Increase timeouts in TestReplication and
 TestSplitLogWorker

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1426711 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                                   | 1 +
 .../apache/hadoop/hbase/regionserver/TestSplitLogWorker.java  | 2 +-
 .../org/apache/hadoop/hbase/replication/TestReplication.java  | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index c87308c795ce..01d2b5c6194f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -53,6 +53,7 @@ Bug
     [HBASE-7432] - TestHBaseFsck prevents testsuite from finishing
     [HBASE-7435] - BuiltInGzipDecompressor is only released during full GC
     [HBASE-7440] - ReplicationZookeeper#addPeer is racy
+    [HBASE-7455] - Increase timeouts in TestReplication and TestSplitLogWorker
 
 Improvement
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index 26b9865fd0be..2611ca0ba66e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -139,7 +139,7 @@ public void testAcquireTaskAtStartup() throws Exception {
       "rs", neverEndingTask);
     slw.start();
     try {
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 100);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tatas")), "rs"));
     } finally {
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index bc2c16f89c6a..c99092e949d2 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -78,8 +78,8 @@ public class TestReplication {
   private static final int NB_ROWS_IN_BATCH = 100;
   private static final int NB_ROWS_IN_BIG_BATCH =
       NB_ROWS_IN_BATCH * 10;
-  private static final long SLEEP_TIME = 1500;
-  private static final int NB_RETRIES = 15;
+  private static final long SLEEP_TIME = 2000;
+  private static final int NB_RETRIES = 20;
 
   private static final byte[] tableName = Bytes.toBytes("test");
   private static final byte[] famName = Bytes.toBytes("f");

From 6bd81fa9cfd0b23d8ed10f3b7a47700b1f6fe04a Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Mon, 31 Dec 2012 00:44:45 +0000
Subject: [PATCH 0668/1540] HBASE-7466: Fix junit dependency type in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1427028 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 90f9011914af..503c7aaa13fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1324,7 +1324,7 @@
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <version>${junit.version}</version>
-      <scope>test,runtime</scope>
+      <scope>test</scope>
       <!-- FIXME: the following needs to go away once HBASE-4955 is fixed -->
       <optional>true</optional>
     </dependency>

From 7739796d3befdd7c56bd4a49020993323faf48c5 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 31 Dec 2012 19:13:20 +0000
Subject: [PATCH 0669/1540] HBASE-7469. [REST] Share a HBaseAdmin instance

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1427227 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/rest/RESTServlet.java   | 7 +++++++
 .../java/org/apache/hadoop/hbase/rest/RootResource.java  | 4 +---
 .../org/apache/hadoop/hbase/rest/SchemaResource.java     | 4 ++--
 .../hadoop/hbase/rest/StorageClusterStatusResource.java  | 4 +---
 .../hadoop/hbase/rest/StorageClusterVersionResource.java | 4 +---
 .../java/org/apache/hadoop/hbase/rest/TableResource.java | 9 +--------
 6 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
index 1b83f474eef3..89c32a3c9ea9 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
@@ -23,6 +23,7 @@
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTablePool;
 import org.apache.hadoop.hbase.rest.metrics.RESTMetrics;
 
@@ -34,6 +35,7 @@ public class RESTServlet implements Constants {
   private final Configuration conf;
   private final HTablePool pool;
   private final RESTMetrics metrics = new RESTMetrics();
+  private final HBaseAdmin admin;
 
   /**
    * @return the RESTServlet singleton instance
@@ -69,6 +71,11 @@ public synchronized static void stop() {
   RESTServlet(Configuration conf) throws IOException {
     this.conf = conf;
     this.pool = new HTablePool(conf, 10);
+    this.admin = new HBaseAdmin(conf);
+  }
+
+  HBaseAdmin getAdmin() {
+    return admin;
   }
 
   HTablePool getTablePool() {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
index 1931318ce43d..b7c7078bfd96 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
@@ -37,7 +37,6 @@
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.rest.model.TableListModel;
 import org.apache.hadoop.hbase.rest.model.TableModel;
 
@@ -62,8 +61,7 @@ public RootResource() throws IOException {
 
   private final TableListModel getTableList() throws IOException {
     TableListModel tableList = new TableListModel();
-    HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
-    HTableDescriptor[] list = admin.listTables();
+    HTableDescriptor[] list = servlet.getAdmin().listTables();
     for (HTableDescriptor htd: list) {
       tableList.add(new TableModel(htd.getNameAsString()));
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index ddc10d1dc950..51626ab0ef19 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -182,7 +182,7 @@ private Response update(final TableSchemaModel model, final boolean replace,
       final UriInfo uriInfo) {
     try {
       byte[] name = Bytes.toBytes(tableResource.getName());
-      HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
+      HBaseAdmin admin = servlet.getAdmin();
       if (replace || !admin.tableExists(name)) {
         return replace(name, model, uriInfo, admin);
       } else {
@@ -224,7 +224,7 @@ public Response delete(final @Context UriInfo uriInfo) {
     }
     servlet.getMetrics().incrementRequests(1);
     try {
-      HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
+      HBaseAdmin admin = servlet.getAdmin();
       boolean success = false;
       for (int i = 0; i < 10; i++) try {
         admin.disableTable(tableResource.getName());
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
index bba0ae187435..ccaf4f0723c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel;
 
 public class StorageClusterStatusResource extends ResourceBase {
@@ -67,8 +66,7 @@ public Response get(final @Context UriInfo uriInfo) {
     }
     servlet.getMetrics().incrementRequests(1);
     try {
-      HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
-      ClusterStatus status = admin.getClusterStatus();
+      ClusterStatus status = servlet.getAdmin().getClusterStatus();
       StorageClusterStatusModel model = new StorageClusterStatusModel();
       model.setRegions(status.getRegionsCount());
       model.setRequests(status.getRequestsCount());
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
index 6768f7d43328..0cf7ac7623ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
@@ -34,7 +34,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel;
 
 public class StorageClusterVersionResource extends ResourceBase {
@@ -64,9 +63,8 @@ public Response get(final @Context UriInfo uriInfo) {
     }
     servlet.getMetrics().incrementRequests(1);
     try {
-      HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
       StorageClusterVersionModel model = new StorageClusterVersionModel();
-      model.setVersion(admin.getClusterStatus().getHBaseVersion());
+      model.setVersion(servlet.getAdmin().getClusterStatus().getHBaseVersion());
       ResponseBuilder response = Response.ok(model);
       response.cacheControl(cacheControl);
       servlet.getMetrics().incrementSucessfulGetRequests(1);
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index f8e0d1827346..88cc68f02f29 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -27,8 +27,6 @@
 import javax.ws.rs.PathParam;
 import javax.ws.rs.QueryParam;
 
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-
 public class TableResource extends ResourceBase {
 
   String table;
@@ -53,12 +51,7 @@ String getName() {
    * @throws IOException
    */
   boolean exists() throws IOException {
-    HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
-    try {
-      return admin.tableExists(table);
-    } finally {
-      admin.close();
-    }
+    return servlet.getAdmin().tableExists(table);
   }
 
   @Path("exists")

From e77ef762343f1238ca2fbc2f08a55159f170ca15 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 1 Jan 2013 21:53:10 +0000
Subject: [PATCH 0670/1540] HBASE-7464. [REST] Sending HTML for errors is
 unhelpful

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1427568 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/rest/Constants.java   |   2 +
 .../hadoop/hbase/rest/ExistsResource.java     |   9 +-
 .../hadoop/hbase/rest/MultiRowResource.java   |  13 +-
 .../hadoop/hbase/rest/RegionsResource.java    |  10 +-
 .../hadoop/hbase/rest/RootResource.java       |   6 +-
 .../apache/hadoop/hbase/rest/RowResource.java | 117 +++++++++++++-----
 .../hbase/rest/ScannerInstanceResource.java   |  31 ++++-
 .../hadoop/hbase/rest/ScannerResource.java    |  25 ++--
 .../hadoop/hbase/rest/SchemaResource.java     |  51 +++++---
 .../rest/StorageClusterStatusResource.java    |   6 +-
 .../rest/StorageClusterVersionResource.java   |   6 +-
 11 files changed, 188 insertions(+), 88 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
index 55ca1c6bbb5d..2180469ca278 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
@@ -36,4 +36,6 @@ public interface Constants {
   public static final String MIMETYPE_BINARY = "application/octet-stream";
   public static final String MIMETYPE_PROTOBUF = "application/x-protobuf";
   public static final String MIMETYPE_JSON = "application/json";
+
+  public static final String CRLF = "\r\n";
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
index 435c82b54ff7..82d6d8c7f202 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
@@ -24,7 +24,6 @@
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -58,10 +57,14 @@ public ExistsResource(TableResource tableResource) throws IOException {
   public Response get(final @Context UriInfo uriInfo) {
     try {
       if (!tableResource.exists()) {
-        throw new WebApplicationException(Response.Status.NOT_FOUND);
+        return Response.status(Response.Status.NOT_FOUND)
+          .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+          .build();
       }
     } catch (IOException e) {
-      throw new WebApplicationException(Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
     ResponseBuilder response = Response.ok();
     response.cacheControl(cacheControl);
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index 6adfc63f9faa..b04a89aeb278 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -29,7 +29,6 @@
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MultivaluedMap;
 import javax.ws.rs.core.Response;
@@ -74,9 +73,12 @@ public Response get(final @Context UriInfo uriInfo) {
           rowSpec.setMaxVersions(this.versions);
         }
 
-        ResultGenerator generator = ResultGenerator.fromRowSpec(this.tableResource.getName(), rowSpec, null);
+        ResultGenerator generator =
+          ResultGenerator.fromRowSpec(this.tableResource.getName(), rowSpec, null);
         if (!generator.hasNext()) {
-          throw new WebApplicationException(Response.Status.NOT_FOUND);
+          return Response.status(Response.Status.NOT_FOUND)
+            .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+            .build();
         }
 
         KeyValue value = null;
@@ -93,8 +95,9 @@ public Response get(final @Context UriInfo uriInfo) {
       return Response.ok(model).build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e,
-              Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
 
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index 6f5e0cc88480..db0fbe7b53a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -25,7 +25,6 @@
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -90,11 +89,14 @@ public Response get(final @Context UriInfo uriInfo) {
       return response.build();
     } catch (TableNotFoundException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(Response.Status.NOT_FOUND);
+      return Response.status(Response.Status.NOT_FOUND)
+        .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+        .build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e,
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
index b7c7078bfd96..8f188782fa58 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
@@ -26,7 +26,6 @@
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -82,8 +81,9 @@ public Response get(final @Context UriInfo uriInfo) {
       return response.build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e, 
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index 97c109360e2b..066cb2138800 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -30,7 +30,6 @@
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Response;
@@ -41,6 +40,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.HTablePool;
@@ -89,7 +89,9 @@ public Response get(final @Context UriInfo uriInfo) {
       ResultGenerator generator =
         ResultGenerator.fromRowSpec(tableResource.getName(), rowspec, null);
       if (!generator.hasNext()) {
-        throw new WebApplicationException(Response.Status.NOT_FOUND);
+        return Response.status(Response.Status.NOT_FOUND)
+          .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+          .build();
       }
       int count = 0;
       CellSetModel model = new CellSetModel();
@@ -112,10 +114,21 @@ public Response get(final @Context UriInfo uriInfo) {
       model.addRow(rowModel);
       servlet.getMetrics().incrementSucessfulGetRequests(1);
       return Response.ok(model).build();
-    } catch (IOException e) {
-      servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e,
-                  Response.Status.SERVICE_UNAVAILABLE);
+    } catch (RuntimeException e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
+      if (e.getCause() instanceof TableNotFoundException) {
+        return Response.status(Response.Status.NOT_FOUND)
+          .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+          .build();
+      }
+      return Response.status(Response.Status.BAD_REQUEST)
+        .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+        .build();
+    } catch (Exception e) {
+      servlet.getMetrics().incrementFailedPutRequests(1);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
@@ -129,13 +142,17 @@ public Response getBinary(final @Context UriInfo uriInfo) {
     // doesn't make sense to use a non specific coordinate as this can only
     // return a single cell
     if (!rowspec.hasColumns() || rowspec.getColumns().length > 1) {
-      throw new WebApplicationException(Response.Status.BAD_REQUEST);
+      return Response.status(Response.Status.BAD_REQUEST)
+        .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+        .build();
     }
     try {
       ResultGenerator generator =
         ResultGenerator.fromRowSpec(tableResource.getName(), rowspec, null);
       if (!generator.hasNext()) {
-        throw new WebApplicationException(Response.Status.NOT_FOUND);
+        return Response.status(Response.Status.NOT_FOUND)
+          .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+          .build();
       }
       KeyValue value = generator.next();
       ResponseBuilder response = Response.ok(value.getValue());
@@ -144,15 +161,18 @@ public Response getBinary(final @Context UriInfo uriInfo) {
       return response.build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e,
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
   Response update(final CellSetModel model, final boolean replace) {
     servlet.getMetrics().incrementRequests(1);
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
 
     if (CHECK_PUT.equalsIgnoreCase(check)) {
@@ -174,7 +194,9 @@ Response update(final CellSetModel model, final boolean replace) {
           key = rowspec.getRow();
         }
         if (key == null) {
-          throw new WebApplicationException(Response.Status.BAD_REQUEST);
+          return Response.status(Response.Status.BAD_REQUEST)
+            .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+            .build();
         }
         Put put = new Put(key);
         int i = 0;
@@ -186,7 +208,9 @@ Response update(final CellSetModel model, final boolean replace) {
             col = null;
           }
           if (col == null) {
-            throw new WebApplicationException(Response.Status.BAD_REQUEST);
+            return Response.status(Response.Status.BAD_REQUEST)
+              .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+              .build();
           }
           byte [][] parts = KeyValue.parseColumn(col);
           if (parts.length == 2 && parts[1].length > 0) {
@@ -208,8 +232,9 @@ Response update(final CellSetModel model, final boolean replace) {
       return response.build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedPutRequests(1);
-      throw new WebApplicationException(e,
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     } finally {
       if (table != null) try {
         table.close();
@@ -222,7 +247,9 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
       final boolean replace) {
     servlet.getMetrics().incrementRequests(1);
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
     HTablePool pool = servlet.getTablePool();
     HTableInterface table = null;
@@ -247,7 +274,9 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
         timestamp = Long.valueOf(vals.get(0));
       }
       if (column == null) {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
       Put put = new Put(row);
       byte parts[][] = KeyValue.parseColumn(column);
@@ -265,8 +294,9 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
       return Response.ok().build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedPutRequests(1);
-      throw new WebApplicationException(e,
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     } finally {
       if (table != null) try {
         table.close();
@@ -323,7 +353,9 @@ public Response delete(final @Context UriInfo uriInfo) {
     }
     servlet.getMetrics().incrementRequests(1);
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
     Delete delete = null;
     if (rowspec.hasTimestamp())
@@ -358,8 +390,9 @@ public Response delete(final @Context UriInfo uriInfo) {
       }
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedDeleteRequests(1);
-      throw new WebApplicationException(e, 
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     } finally {
       if (table != null) try {
         table.close();
@@ -380,7 +413,9 @@ Response checkAndPut(final CellSetModel model) {
     HTableInterface table = null;
     try {
       if (model.getRows().size() != 1) {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
 
       RowModel rowModel = model.getRows().get(0);
@@ -392,7 +427,9 @@ Response checkAndPut(final CellSetModel model) {
       List<CellModel> cellModels = rowModel.getCells();
       int cellModelCount = cellModels.size();
       if (key == null || cellModelCount <= 1) {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
 
       Put put = new Put(key);
@@ -412,10 +449,14 @@ Response checkAndPut(final CellSetModel model) {
           put.add(valueToPutParts[0], valueToPutParts[1], valueToPutCell
             .getTimestamp(), valueToPutCell.getValue());
         } else {
-          throw new WebApplicationException(Response.Status.BAD_REQUEST);
+          return Response.status(Response.Status.BAD_REQUEST)
+            .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+            .build();
         }
       } else {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
 
       table = pool.getTable(this.tableResource.getName());
@@ -431,7 +472,9 @@ Response checkAndPut(final CellSetModel model) {
       }
       return response.build();
     } catch (IOException e) {
-      throw new WebApplicationException(e, Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     } finally {
       if (table != null) try {
         table.close();
@@ -452,7 +495,9 @@ Response checkAndDelete(final CellSetModel model) {
     Delete delete = null;
     try {
       if (model.getRows().size() != 1) {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
       RowModel rowModel = model.getRows().get(0);
       byte[] key = rowModel.getKey();
@@ -460,7 +505,9 @@ Response checkAndDelete(final CellSetModel model) {
         key = rowspec.getRow();
       }
       if (key == null) {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
 
       delete = new Delete(key);
@@ -470,14 +517,18 @@ Response checkAndDelete(final CellSetModel model) {
         try {
           valueToDeleteColumn = rowspec.getColumns()[0];
         } catch (final ArrayIndexOutOfBoundsException e) {
-          throw new WebApplicationException(Response.Status.BAD_REQUEST);
+          return Response.status(Response.Status.BAD_REQUEST)
+            .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+            .build();
         }
       }
       byte[][] parts = KeyValue.parseColumn(valueToDeleteColumn);
       if (parts.length == 2 && parts[1].length > 0) {
         delete.deleteColumns(parts[0], parts[1]);
       } else {
-        throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        return Response.status(Response.Status.BAD_REQUEST)
+          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+          .build();
       }
 
       table = pool.getTable(tableResource.getName());
@@ -494,7 +545,9 @@ Response checkAndDelete(final CellSetModel model) {
       }
       return response.build();
     } catch (IOException e) {
-      throw new WebApplicationException(e, Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     } finally {
       if (table != null) try {
         table.close();
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
index fbd6cfaea667..3bb9129c4759 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
@@ -26,7 +26,6 @@
 import javax.ws.rs.GET;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -54,10 +53,12 @@ public class ScannerInstanceResource extends ResourceBase {
     cacheControl.setNoTransform(false);
   }
 
-  ResultGenerator generator;
-  String id;
+  ResultGenerator generator = null;
+  String id = null;
   int batch = 1;
 
+  public ScannerInstanceResource() throws IOException { }
+
   public ScannerInstanceResource(String table, String id, 
       ResultGenerator generator, int batch) throws IOException {
     this.id = id;
@@ -73,6 +74,12 @@ public Response get(final @Context UriInfo uriInfo,
       LOG.debug("GET " + uriInfo.getAbsolutePath());
     }
     servlet.getMetrics().incrementRequests(1);
+    if (generator == null) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
+      return Response.status(Response.Status.NOT_FOUND)
+        .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+        .build();
+    }
     CellSetModel model = new CellSetModel();
     RowModel rowModel = null;
     byte[] rowKey = null;
@@ -92,7 +99,9 @@ public Response get(final @Context UriInfo uriInfo,
           servlet.getMetrics().incrementFailedDeleteRequests(1);
         }
         servlet.getMetrics().incrementFailedGetRequests(1);
-        throw new WebApplicationException(Response.Status.GONE);
+        return Response.status(Response.Status.GONE)
+          .type(MIMETYPE_TEXT).entity("Gone" + CRLF)
+          .build();
       }
       if (value == null) {
         LOG.info("generator exhausted");
@@ -139,6 +148,12 @@ public Response getBinary(final @Context UriInfo uriInfo) {
         MIMETYPE_BINARY);
     }
     servlet.getMetrics().incrementRequests(1);
+    if (generator == null) {
+      servlet.getMetrics().incrementFailedGetRequests(1);
+      return Response.status(Response.Status.NOT_FOUND)
+        .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+        .build();
+    }
     try {
       KeyValue value = generator.next();
       if (value == null) {
@@ -161,7 +176,9 @@ public Response getBinary(final @Context UriInfo uriInfo) {
         servlet.getMetrics().incrementFailedDeleteRequests(1);
       }
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(Response.Status.GONE);
+      return Response.status(Response.Status.GONE)
+        .type(MIMETYPE_TEXT).entity("Gone" + CRLF)
+        .build();
     }
   }
 
@@ -172,7 +189,9 @@ public Response delete(final @Context UriInfo uriInfo) {
     }
     servlet.getMetrics().incrementRequests(1);
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
     if (ScannerResource.delete(id)) {
       servlet.getMetrics().incrementSucessfulDeleteRequests(1);
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
index be19cb91f533..22122a30a534 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
@@ -31,7 +31,6 @@
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
@@ -77,7 +76,9 @@ Response update(final ScannerModel model, final boolean replace,
       final UriInfo uriInfo) {
     servlet.getMetrics().incrementRequests(1);
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
     byte[] endRow = model.hasEndRow() ? model.getEndRow() : null;
     RowSpec spec = new RowSpec(model.getStartRow(), endRow,
@@ -99,19 +100,21 @@ Response update(final ScannerModel model, final boolean replace,
       URI uri = builder.path(id).build();
       servlet.getMetrics().incrementSucessfulPutRequests(1);
       return Response.created(uri).build();
-    } catch (IOException e) {
-      servlet.getMetrics().incrementFailedPutRequests(1);
-      throw new WebApplicationException(e,
-              Response.Status.SERVICE_UNAVAILABLE);
     } catch (RuntimeException e) {
       servlet.getMetrics().incrementFailedPutRequests(1);
       if (e.getCause() instanceof TableNotFoundException) {
-        throw new WebApplicationException(e, Response.Status.NOT_FOUND);
+        return Response.status(Response.Status.NOT_FOUND)
+          .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+          .build();
       }
-      throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
+      return Response.status(Response.Status.BAD_REQUEST)
+        .type(MIMETYPE_TEXT).entity("Bad request" + CRLF)
+        .build();
     } catch (Exception e) {
       servlet.getMetrics().incrementFailedPutRequests(1);
-      throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
@@ -137,11 +140,11 @@ public Response post(final ScannerModel model,
 
   @Path("{scanner: .+}")
   public ScannerInstanceResource getScannerInstanceResource(
-      final @PathParam("scanner") String id) {
+      final @PathParam("scanner") String id) throws IOException {
     ScannerInstanceResource instance = scanners.get(id);
     if (instance == null) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(Response.Status.NOT_FOUND);
+      return new ScannerInstanceResource();
     } else {
       servlet.getMetrics().incrementSucessfulGetRequests(1);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index 51626ab0ef19..6926f52d1065 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -29,7 +29,6 @@
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -100,18 +99,23 @@ public Response get(final @Context UriInfo uriInfo) {
       return response.build();
     } catch (TableNotFoundException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(Response.Status.NOT_FOUND);
+      return Response.status(Response.Status.NOT_FOUND)
+        .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+        .build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e,
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
   private Response replace(final byte[] name, final TableSchemaModel model,
       final UriInfo uriInfo, final HBaseAdmin admin) {
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
     try {
       HTableDescriptor htd = new HTableDescriptor(name);
@@ -135,19 +139,24 @@ private Response replace(final byte[] name, final TableSchemaModel model,
         servlet.getMetrics().incrementSucessfulPutRequests(1);
       } catch (TableExistsException e) {
         // race, someone else created a table with the same name
-        throw new WebApplicationException(e, Response.Status.NOT_MODIFIED);
+        return Response.status(Response.Status.NOT_MODIFIED)
+          .type(MIMETYPE_TEXT).entity("Not modified" + CRLF)
+          .build();
       }
       return Response.created(uriInfo.getAbsolutePath()).build();
     } catch (IOException e) {
-      throw new WebApplicationException(e,
-            Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
   private Response update(final byte[] name, final TableSchemaModel model,
       final UriInfo uriInfo, final HBaseAdmin admin) {
     if (servlet.isReadOnly()) {
-      throw new WebApplicationException(Response.Status.FORBIDDEN);
+      return Response.status(Response.Status.FORBIDDEN)
+        .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF)
+        .build();
     }
     try {
       HTableDescriptor htd = admin.getTableDescriptor(name);
@@ -165,16 +174,18 @@ private Response update(final byte[] name, final TableSchemaModel model,
           }
         }
       } catch (IOException e) {
-        throw new WebApplicationException(e,
-            Response.Status.INTERNAL_SERVER_ERROR);
+        return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+          .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+          .build();
       } finally {
         admin.enableTable(tableResource.getName());
       }
       servlet.getMetrics().incrementSucessfulPutRequests(1);
       return Response.ok().build();
     } catch (IOException e) {
-      throw new WebApplicationException(e,
-          Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
@@ -190,8 +201,9 @@ private Response update(final TableSchemaModel model, final boolean replace,
       }
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedPutRequests(1);
-      throw new WebApplicationException(e, 
-            Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 
@@ -240,11 +252,14 @@ public Response delete(final @Context UriInfo uriInfo) {
       return Response.ok().build();
     } catch (TableNotFoundException e) {
       servlet.getMetrics().incrementFailedDeleteRequests(1);
-      throw new WebApplicationException(Response.Status.NOT_FOUND);
+      return Response.status(Response.Status.NOT_FOUND)
+        .type(MIMETYPE_TEXT).entity("Not found" + CRLF)
+        .build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedDeleteRequests(1);
-      throw new WebApplicationException(e, 
-            Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
index ccaf4f0723c4..9da5252f3941 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
@@ -24,7 +24,6 @@
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -99,8 +98,9 @@ public Response get(final @Context UriInfo uriInfo) {
       return response.build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e, 
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
index 0cf7ac7623ac..0ea30041ed77 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
@@ -24,7 +24,6 @@
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
@@ -71,8 +70,9 @@ public Response get(final @Context UriInfo uriInfo) {
       return response.build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new WebApplicationException(e, 
-                  Response.Status.SERVICE_UNAVAILABLE);
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE)
+        .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
+        .build();
     }
   }
 }

From a1438cdbe74c1454f94531d1ddc25744dcd7a791 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 2 Jan 2013 16:18:42 +0000
Subject: [PATCH 0671/1540] HBASE-7472. [REST] Support MIME type
 application/protobuf

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1427845 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/rest/Constants.java   |  1 +
 .../hadoop/hbase/rest/ExistsResource.java     |  2 +-
 .../hadoop/hbase/rest/MultiRowResource.java   |  3 +-
 .../hadoop/hbase/rest/RegionsResource.java    |  3 +-
 .../hadoop/hbase/rest/RootResource.java       |  3 +-
 .../apache/hadoop/hbase/rest/RowResource.java |  9 +++--
 .../hbase/rest/ScannerInstanceResource.java   |  3 +-
 .../hadoop/hbase/rest/ScannerResource.java    |  6 ++--
 .../hadoop/hbase/rest/SchemaResource.java     |  9 +++--
 .../rest/StorageClusterStatusResource.java    |  3 +-
 .../hadoop/hbase/rest/VersionResource.java    |  3 +-
 .../consumer/ProtobufMessageBodyConsumer.java |  2 +-
 .../producer/ProtobufMessageBodyProducer.java |  2 +-
 .../hbase/rest/TestMultiRowResource.java      |  2 ++
 .../hadoop/hbase/rest/TestRowResource.java    |  5 +++
 .../hbase/rest/TestScannerResource.java       |  4 +++
 .../hbase/rest/TestScannersWithFilters.java   |  4 +++
 .../hadoop/hbase/rest/TestSchemaResource.java | 11 +++++++
 .../hadoop/hbase/rest/TestStatusResource.java | 13 ++++++--
 .../hadoop/hbase/rest/TestTableResource.java  | 33 ++++++++++++++-----
 .../hbase/rest/TestVersionResource.java       | 24 +++++++++-----
 21 files changed, 108 insertions(+), 37 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
index 2180469ca278..21d76feafd7e 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
@@ -35,6 +35,7 @@ public interface Constants {
   public static final String MIMETYPE_XML = "text/xml";
   public static final String MIMETYPE_BINARY = "application/octet-stream";
   public static final String MIMETYPE_PROTOBUF = "application/x-protobuf";
+  public static final String MIMETYPE_PROTOBUF_IETF = "application/protobuf";
   public static final String MIMETYPE_JSON = "application/json";
 
   public static final String CRLF = "\r\n";
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
index 82d6d8c7f202..62a1b0e29cfc 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
@@ -53,7 +53,7 @@ public ExistsResource(TableResource tableResource) throws IOException {
 
   @GET
   @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
-    MIMETYPE_BINARY})
+    MIMETYPE_PROTOBUF_IETF, MIMETYPE_BINARY})
   public Response get(final @Context UriInfo uriInfo) {
     try {
       if (!tableResource.exists()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index b04a89aeb278..887e8a1a523d 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -59,7 +59,8 @@ public MultiRowResource(TableResource tableResource, String versions) throws IOE
   }
 
   @GET
-  @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo) {
     MultivaluedMap<String, String> params = uriInfo.getQueryParameters();
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index db0fbe7b53a8..8d4b621f4013 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -65,7 +65,8 @@ public RegionsResource(TableResource tableResource) throws IOException {
   }
 
   @GET
-  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("GET " + uriInfo.getAbsolutePath());
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
index 8f188782fa58..e1e6d72203f8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
@@ -68,7 +68,8 @@ private final TableListModel getTableList() throws IOException {
   }
 
   @GET
-  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("GET " + uriInfo.getAbsolutePath());
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index 066cb2138800..d933bcfd75ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -79,7 +79,8 @@ public RowResource(TableResource tableResource, String rowspec,
   }
 
   @GET
-  @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("GET " + uriInfo.getAbsolutePath());
@@ -305,7 +306,8 @@ Response updateBinary(final byte[] message, final HttpHeaders headers,
   }
 
   @PUT
-  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response put(final CellSetModel model,
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
@@ -326,7 +328,8 @@ public Response putBinary(final byte[] message,
   }
 
   @POST
-  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response post(final CellSetModel model,
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
index 3bb9129c4759..7f58e96c4d7a 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
@@ -67,7 +67,8 @@ public ScannerInstanceResource(String table, String id,
   }
 
   @GET
-  @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo, 
       @QueryParam("n") int maxRows, final @QueryParam("c") int maxValues) {
     if (LOG.isDebugEnabled()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
index 22122a30a534..8ebda4f4aea6 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
@@ -119,7 +119,8 @@ Response update(final ScannerModel model, final boolean replace,
   }
 
   @PUT
-  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response put(final ScannerModel model, 
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
@@ -129,7 +130,8 @@ public Response put(final ScannerModel model,
   }
 
   @POST
-  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response post(final ScannerModel model,
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index 6926f52d1065..864b2a617ec0 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -85,7 +85,8 @@ private HTableDescriptor getTableSchema() throws IOException,
   }
 
   @GET
-  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("GET " + uriInfo.getAbsolutePath());
@@ -208,7 +209,8 @@ private Response update(final TableSchemaModel model, final boolean replace,
   }
 
   @PUT
-  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response put(final TableSchemaModel model, 
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
@@ -219,7 +221,8 @@ public Response put(final TableSchemaModel model,
   }
 
   @POST
-  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response post(final TableSchemaModel model, 
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
index 9da5252f3941..16af87b2ac46 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
@@ -58,7 +58,8 @@ public StorageClusterStatusResource() throws IOException {
   }
 
   @GET
-  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("GET " + uriInfo.getAbsolutePath());
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
index 5217ee2c9612..cdb55dc9bd36 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
@@ -70,7 +70,8 @@ public VersionResource() throws IOException {
    * @return a response for a version request 
    */
   @GET
-  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
+  @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
+    MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context ServletContext context, 
       final @Context UriInfo uriInfo) {
     if (LOG.isDebugEnabled()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java
index 6fe2dd08de9e..daccb8d1d6be 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java
@@ -43,7 +43,7 @@
  * ProtobufMessageHandler interface capable handlers for decoding protobuf input.
  */
 @Provider
-@Consumes(Constants.MIMETYPE_PROTOBUF)
+@Consumes({Constants.MIMETYPE_PROTOBUF, Constants.MIMETYPE_PROTOBUF_IETF})
 public class ProtobufMessageBodyConsumer 
     implements MessageBodyReader<ProtobufMessageHandler> {
   private static final Log LOG =
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
index a1b4b70831ad..0117604264c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
@@ -43,7 +43,7 @@
  * sent, then writeTo to perform the actual I/O.
  */
 @Provider
-@Produces(Constants.MIMETYPE_PROTOBUF)
+@Produces({Constants.MIMETYPE_PROTOBUF, Constants.MIMETYPE_PROTOBUF_IETF})
 public class ProtobufMessageBodyProducer
   implements MessageBodyWriter<ProtobufMessageHandler> {
 
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
index c5b714370d47..e45ed2adb258 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
@@ -117,6 +117,7 @@ public void testMultiCellGetJSON() throws IOException, JAXBException {
 
     Response response = client.get(path.toString(), Constants.MIMETYPE_JSON);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
 
     client.delete(row_5_url);
     client.delete(row_6_url);
@@ -143,6 +144,7 @@ public void testMultiCellGetXML() throws IOException, JAXBException {
 
     Response response = client.get(path.toString(), Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
 
     client.delete(row_5_url);
     client.delete(row_6_url);
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
index 1b67046c4f05..2a0284e00f43 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
@@ -208,6 +208,7 @@ private static void checkValueXML(String table, String row, String column,
       String value) throws IOException, JAXBException {
     Response response = getValueXML(table, row, column);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     CellSetModel cellSet = (CellSetModel)
       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
     RowModel rowModel = cellSet.getRows().get(0);
@@ -220,6 +221,7 @@ private static void checkValueXML(String url, String table, String row,
       String column, String value) throws IOException, JAXBException {
     Response response = getValueXML(url);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     CellSetModel cellSet = (CellSetModel)
       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
     RowModel rowModel = cellSet.getRows().get(0);
@@ -257,6 +259,7 @@ private static void checkValuePB(String table, String row, String column,
       String value) throws IOException {
     Response response = getValuePB(table, row, column);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
     CellSetModel cellSet = new CellSetModel();
     cellSet.getObjectFromMessage(response.getBody());
     RowModel rowModel = cellSet.getRows().get(0);
@@ -499,6 +502,7 @@ public void testSingleCellGetPutBinary() throws IOException {
 
     response = client.get(path, Constants.MIMETYPE_BINARY);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
     assertTrue(Bytes.equals(response.getBody(), body));
     boolean foundTimestampHeader = false;
     for (Header header: response.getHeaders()) {
@@ -522,6 +526,7 @@ public void testSingleCellGetJSON() throws IOException, JAXBException {
     Thread.yield();
     response = client.get(path, Constants.MIMETYPE_JSON);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
     response = deleteRow(TABLE, ROW_4);
     assertEquals(response.getCode(), 200);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
index 6e5849b30386..3c09c383cf57 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
@@ -125,6 +125,7 @@ private static int fullTableScan(ScannerModel model) throws IOException {
       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
       assertTrue(response.getCode() == 200 || response.getCode() == 204);
       if (response.getCode() == 200) {
+        assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
         CellSetModel cellSet = new CellSetModel();
         cellSet.getObjectFromMessage(response.getBody());
         Iterator<RowModel> rows = cellSet.getRows().iterator();
@@ -208,6 +209,7 @@ public void testSimpleScannerXML() throws IOException, JAXBException {
     // get a cell set
     response = client.get(scannerURI, Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     CellSetModel cellSet = (CellSetModel)
       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
     // confirm batch size conformance
@@ -251,6 +253,7 @@ public void testSimpleScannerPB() throws IOException {
     // get a cell set
     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
     CellSetModel cellSet = new CellSetModel();
     cellSet.getObjectFromMessage(response.getBody());
     // confirm batch size conformance
@@ -293,6 +296,7 @@ public void testSimpleScannerBinary() throws IOException {
     // get a cell
     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
     // verify that data was returned
     assertTrue(response.getBody().length > 0);
     // verify that the expected X-headers are present
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java b/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
index 53f388c0007b..d83e18627b2e 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 
 import static org.junit.Assert.*;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -226,6 +227,7 @@ private static void verifyScan(Scan s, long expectedRows, long expectedKeys)
     // get a cell set
     response = client.get(scannerURI, Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     CellSetModel cells = (CellSetModel)
       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
 
@@ -260,6 +262,7 @@ private static void verifyScanFull(Scan s, KeyValue [] kvs)
     // get a cell set
     response = client.get(scannerURI, Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     CellSetModel cellSet = (CellSetModel)
       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
 
@@ -313,6 +316,7 @@ private static void verifyScanNoEarlyOut(Scan s, long expectedRows,
     // get a cell set
     response = client.get(scannerURI, Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     CellSetModel cellSet = (CellSetModel)
       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
 
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
index c967baba6031..1d2ed5dca491 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 
 import static org.junit.Assert.*;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -110,6 +111,7 @@ public void testTableCreateAndDeleteXML() throws IOException, JAXBException {
     // retrieve the schema and validate it
     response = client.get(schemaPath, Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     model = fromXML(response.getBody());
     TestTableSchemaModel.checkModel(model, TABLE1);
 
@@ -148,6 +150,15 @@ public void testTableCreateAndDeletePB() throws IOException, JAXBException {
     // retrieve the schema and validate it
     response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
+    model = new TableSchemaModel();
+    model.getObjectFromMessage(response.getBody());
+    TestTableSchemaModel.checkModel(model, TABLE2);
+
+    // retrieve the schema and validate it with alternate pbuf type
+    response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF);
+    assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
     model = new TableSchemaModel();
     model.getObjectFromMessage(response.getBody());
     TestTableSchemaModel.checkModel(model, TABLE2);
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
index cffdcb67af93..ea61dc46277d 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 
 import static org.junit.Assert.*;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -95,6 +96,7 @@ public static void tearDownAfterClass() throws Exception {
   public void testGetClusterStatusXML() throws IOException, JAXBException {
     Response response = client.get("/status/cluster", Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     StorageClusterStatusModel model = (StorageClusterStatusModel)
       context.createUnmarshaller().unmarshal(
         new ByteArrayInputStream(response.getBody()));
@@ -103,16 +105,21 @@ public void testGetClusterStatusXML() throws IOException, JAXBException {
 
   @Test
   public void testGetClusterStatusPB() throws IOException {
-    Response response = client.get("/status/cluster", 
-      Constants.MIMETYPE_PROTOBUF);
+    Response response = client.get("/status/cluster", Constants.MIMETYPE_PROTOBUF);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
     StorageClusterStatusModel model = new StorageClusterStatusModel();
     model.getObjectFromMessage(response.getBody());
     validate(model);
+    response = client.get("/status/cluster", Constants.MIMETYPE_PROTOBUF_IETF);
+    assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
+    model = new StorageClusterStatusModel();
+    model.getObjectFromMessage(response.getBody());
+    validate(model);
   }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
index 514db41ab907..4cf4636328e7 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.util.StringUtils;
 
 import static org.junit.Assert.*;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -179,12 +180,14 @@ void checkTableInfo(TableInfoModel model) {
   public void testTableListText() throws IOException {
     Response response = client.get("/", Constants.MIMETYPE_TEXT);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type"));
   }
 
   @Test
   public void testTableListXML() throws IOException, JAXBException {
     Response response = client.get("/", Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     TableListModel model = (TableListModel)
       context.createUnmarshaller()
         .unmarshal(new ByteArrayInputStream(response.getBody()));
@@ -195,29 +198,37 @@ public void testTableListXML() throws IOException, JAXBException {
   public void testTableListJSON() throws IOException {
     Response response = client.get("/", Constants.MIMETYPE_JSON);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
   }
 
   @Test
   public void testTableListPB() throws IOException, JAXBException {
     Response response = client.get("/", Constants.MIMETYPE_PROTOBUF);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
     TableListModel model = new TableListModel();
     model.getObjectFromMessage(response.getBody());
     checkTableList(model);
+    response = client.get("/", Constants.MIMETYPE_PROTOBUF_IETF);
+    assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
+    model = new TableListModel();
+    model.getObjectFromMessage(response.getBody());
+    checkTableList(model);
   }
 
   @Test
   public void testTableInfoText() throws IOException {
-    Response response = client.get("/" + TABLE + "/regions",
-      Constants.MIMETYPE_TEXT);
+    Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_TEXT);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type"));
   }
 
   @Test
   public void testTableInfoXML() throws IOException, JAXBException {
-    Response response = client.get("/" + TABLE + "/regions", 
-      Constants.MIMETYPE_XML);
+    Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_XML);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     TableInfoModel model = (TableInfoModel)
       context.createUnmarshaller()
         .unmarshal(new ByteArrayInputStream(response.getBody()));
@@ -226,19 +237,25 @@ public void testTableInfoXML() throws IOException, JAXBException {
 
   @Test
   public void testTableInfoJSON() throws IOException {
-    Response response = client.get("/" + TABLE + "/regions", 
-      Constants.MIMETYPE_JSON);
+    Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_JSON);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
   }
 
   @Test
   public void testTableInfoPB() throws IOException, JAXBException {
-    Response response = client.get("/" + TABLE + "/regions",
-      Constants.MIMETYPE_PROTOBUF);
+    Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_PROTOBUF);
     assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
     TableInfoModel model = new TableInfoModel();
     model.getObjectFromMessage(response.getBody());
     checkTableInfo(model);
+    response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_PROTOBUF_IETF);
+    assertEquals(response.getCode(), 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
+    model = new TableInfoModel();
+    model.getObjectFromMessage(response.getBody());
+    checkTableInfo(model);
   }
 
   @org.junit.Rule
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
index a416499caa38..5e388a9ca953 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
@@ -26,8 +26,6 @@
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.rest.client.Client;
@@ -38,6 +36,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 
 import static org.junit.Assert.*;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -47,8 +46,6 @@
 
 @Category(MediumTests.class)
 public class TestVersionResource {
-  private static final Log LOG = LogFactory.getLog(TestVersionResource.class);
-
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
     new HBaseRESTTestingUtility();
@@ -97,6 +94,7 @@ private static void validate(VersionModel model) {
   public void testGetStargateVersionText() throws IOException {
     Response response = client.get("/version", Constants.MIMETYPE_TEXT);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type"));
     String body = Bytes.toString(response.getBody());
     assertTrue(body.length() > 0);
     assertTrue(body.contains(RESTServlet.VERSION_STRING));
@@ -114,34 +112,41 @@ public void testGetStargateVersionText() throws IOException {
   public void testGetStargateVersionXML() throws IOException, JAXBException {
     Response response = client.get("/version", Constants.MIMETYPE_XML);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     VersionModel model = (VersionModel)
       context.createUnmarshaller().unmarshal(
         new ByteArrayInputStream(response.getBody()));
     validate(model);
-    LOG.info("success retrieving Stargate version as XML");
   }
 
   @Test
   public void testGetStargateVersionJSON() throws IOException {
     Response response = client.get("/version", Constants.MIMETYPE_JSON);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
   }
 
   @Test
   public void testGetStargateVersionPB() throws IOException {
     Response response = client.get("/version", Constants.MIMETYPE_PROTOBUF);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
     VersionModel model = new VersionModel();
     model.getObjectFromMessage(response.getBody());
     validate(model);
-    LOG.info("success retrieving Stargate version as protobuf");
+    response = client.get("/version", Constants.MIMETYPE_PROTOBUF_IETF);
+    assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
+    model = new VersionModel();
+    model.getObjectFromMessage(response.getBody());
+    validate(model);
   }
 
   @Test
   public void testGetStorageClusterVersionText() throws IOException {
-    Response response = client.get("/version/cluster", 
-      Constants.MIMETYPE_TEXT);
+    Response response = client.get("/version/cluster", Constants.MIMETYPE_TEXT);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type"));
   }
 
   @Test
@@ -149,19 +154,20 @@ public void testGetStorageClusterVersionXML() throws IOException,
       JAXBException {
     Response response = client.get("/version/cluster",Constants.MIMETYPE_XML);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
     StorageClusterVersionModel clusterVersionModel = 
       (StorageClusterVersionModel)
         context.createUnmarshaller().unmarshal(
           new ByteArrayInputStream(response.getBody()));
     assertNotNull(clusterVersionModel);
     assertNotNull(clusterVersionModel.getVersion());
-    LOG.info("success retrieving storage cluster version as XML");
   }
 
   @Test
   public void doTestGetStorageClusterVersionJSON() throws IOException {
     Response response = client.get("/version/cluster", Constants.MIMETYPE_JSON);
     assertTrue(response.getCode() == 200);
+    assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
   }
 
   @org.junit.Rule

From 40c8526cf6cc291a41c13a4f137a74c0c7bd86ce Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 2 Jan 2013 23:35:59 +0000
Subject: [PATCH 0672/1540] HBASE-7466: Fix junit dependency typo in 0.94
 (Follow Up)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428112 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 503c7aaa13fb..e2238c8a41db 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1324,7 +1324,7 @@
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <version>${junit.version}</version>
-      <scope>test</scope>
+      <scope>runtime</scope>
       <!-- FIXME: the following needs to go away once HBASE-4955 is fixed -->
       <optional>true</optional>
     </dependency>

From ca545465929c5c1a7e26a2584ef2e9d99eee27fa Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 2 Jan 2013 23:42:49 +0000
Subject: [PATCH 0673/1540] HBASE-7467 CleanerChore checkAndDeleteDirectory not
 deleting empty directories (Jean-Marc)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428114 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/cleaner/CleanerChore.java    | 35 ++++++++--
 .../master/cleaner/TestCleanerChore.java      | 69 +++++++++++++++++--
 2 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 9c350c942dd4..b815b3b985f8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -142,13 +142,23 @@ protected void chore() {
    * @return <tt>true</tt> if the directory was deleted, <tt>false</tt> otherwise.
    * @throws IOException if there is an unexpected filesystem error
    */
-  private boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
+  public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
     if (LOG.isTraceEnabled()) {
       LOG.trace("Checking directory: " + toCheck);
     }
     FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
     // if the directory doesn't exist, then we are done
-    if (children == null) return true;
+    if (children == null) {
+      try {
+        return fs.delete(toCheck, false);
+      } catch (IOException e) {
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Couldn't delete directory: " + toCheck, e);
+        }
+      }
+      // couldn't delete w/o exception, so we can't return success.
+      return false;
+    }     
 
     boolean canDeleteThis = true;
     for (FileStatus child : children) {
@@ -165,9 +175,22 @@ else if (!checkAndDelete(path)) {
       }
     }
 
-    // if all the children have been deleted, then we should try to delete this directory. However,
-    // don't do so recursively so we don't delete files that have been added since we checked.
-    return canDeleteThis ? fs.delete(toCheck, false) : false;
+    // if the directory has children, we can't delete it, so we are done
+    if (!canDeleteThis) return false;
+
+    // otherwise, all the children (that we know about) have been deleted, so we should try to
+    // delete this directory. However, don't do so recursively so we don't delete files that have
+    // been added since we last checked.
+    try {
+      return fs.delete(toCheck, false);
+    } catch (IOException e) {
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Couldn't delete directory: " + toCheck, e);
+      }
+    }
+
+    // couldn't delete w/o exception, so we can't return success.
+    return false;
   }
 
   /**
@@ -225,4 +248,4 @@ public void cleanup() {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
index f860aeccd9c8..3ce4de300415 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
@@ -47,10 +47,7 @@ public class TestCleanerChore {
   @After
   public void cleanup() throws Exception {
     // delete and recreate the test directory, ensuring a clean test dir between tests
-    Path testDir = UTIL.getDataTestDir();
-    FileSystem fs = UTIL.getTestFileSystem();
-    fs.delete(testDir, true);
-    fs.mkdirs(testDir);
+    UTIL.cleanupTestDir();
   }
 
   @Test
@@ -94,8 +91,10 @@ public void testDeletesEmptyDirectories() throws Exception {
     // create the directory layout in the directory to clean
     Path parent = new Path(testDir, "parent");
     Path child = new Path(parent, "child");
+    Path emptyChild = new Path(parent, "emptyChild");
     Path file = new Path(child, "someFile");
     fs.mkdirs(child);
+    fs.mkdirs(emptyChild);
     // touch a new file
     fs.create(file).close();
     // also create a file in the top level directory
@@ -223,6 +222,66 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable {
     Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
     Mockito.reset(spy);
   }
+  
+  /**
+   * The cleaner runs in a loop, where it first checks to see all the files under a directory can be
+   * deleted. If they all can, then we try to delete the directory. However, a file may be added
+   * that directory to after the original check. This ensures that we don't accidentally delete that
+   * directory on and don't get spurious IOExceptions.
+   * <p>
+   * This was from HBASE-7465.
+   * @throws Exception on failure
+   */
+  @Test
+  public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    // need to use a localutil to not break the rest of the test that runs on the local FS, which
+    // gets hosed when we start to use a minicluster.
+    HBaseTestingUtility localUtil = new HBaseTestingUtility();
+    Configuration conf = localUtil.getConfiguration();
+    final Path testDir = UTIL.getDataTestDir();
+    final FileSystem fs = UTIL.getTestFileSystem();
+    LOG.debug("Writing test data to: " + testDir);
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
+    // spy on the delegate to ensure that we don't check for directories
+    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
+    AlwaysDelete spy = Mockito.spy(delegate);
+    chore.cleanersChain.set(0, spy);
+
+    // create the directory layout in the directory to clean
+    final Path parent = new Path(testDir, "parent");
+    Path file = new Path(parent, "someFile");
+    fs.mkdirs(parent);
+    // touch a new file
+    fs.create(file).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+    final Path racyFile = new Path(parent, "addedFile");
+
+    // when we attempt to delete the original file, add another file in the same directory
+    Mockito.doAnswer(new Answer<Boolean>() {
+      @Override
+      public Boolean answer(InvocationOnMock invocation) throws Throwable {
+        fs.create(racyFile).close();
+        FSUtils.logFileSystemState(fs, testDir, LOG);
+        return (Boolean) invocation.callRealMethod();
+      }
+    }).when(spy).isFileDeletable(Mockito.any(Path.class));
+
+    // attempt to delete the directory, which
+    if (chore.checkAndDeleteDirectory(parent)) {
+      throw new Exception(
+          "Reported success deleting directory, should have failed when adding file mid-iteration");
+    }
+
+    // make sure all the directories + added file exist, but the original file is deleted
+    assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
+    assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
+    assertFalse("Original file unexpectedly retained", fs.exists(file));
+    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
+  }
 
   private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
 
@@ -269,4 +328,4 @@ public boolean isStopped() {
     }
 
   }
-}
\ No newline at end of file
+}

From a044a8265074ac8c29f84dfac510373fc8bc009a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 3 Jan 2013 00:14:41 +0000
Subject: [PATCH 0674/1540] HBASE-7442 HBase remote CopyTable not working when
 security enabled (James Kinley)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428119 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/ipc/SecureRpcEngine.java     | 36 +++++++++----------
 .../apache/hadoop/hbase/ipc/HBaseClient.java  |  7 ++++
 .../hbase/mapreduce/TableMapReduceUtil.java   | 13 +++++++
 3 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
index 87cbb5117123..7d76d053fe49 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
@@ -22,6 +22,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
@@ -72,33 +73,31 @@ private SecureRpcEngine() {
     super();
   }                                  // no public ctor
 
-  /* Cache a client using its socket factory as the hash key */
+  /* Cache a client using the configured clusterId */
   static private class ClientCache {
-    private Map<SocketFactory, SecureClient> clients =
-      new HashMap<SocketFactory, SecureClient>();
+    private Map<String, SecureClient> clients =
+      new HashMap<String, SecureClient>();
 
     protected ClientCache() {}
 
     /**
-     * Construct & cache an IPC client with the user-provided SocketFactory
-     * if no cached client exists.
+     * Construct & cache an IPC client with the configured
+     * {@link HConstants#CLUSTER_ID} if no cached client exists.
      *
-     * @param conf Configuration
-     * @param factory socket factory
+     * @param conf
+     *          Configuration
+     * @param factory
+     *          socket factory
      * @return an IPC client
      */
     protected synchronized SecureClient getClient(Configuration conf,
         SocketFactory factory) {
-      // Construct & cache client.  The configuration is only used for timeout,
-      // and Clients have connection pools.  So we can either (a) lose some
-      // connection pooling and leak sockets, or (b) use the same timeout for all
-      // configurations.  Since the IPC is usually intended globally, not
-      // per-job, we choose (a).
-      SecureClient client = clients.get(factory);
+      String clusterId = conf.get(HConstants.CLUSTER_ID, "default");
+      SecureClient client = clients.get(clusterId);
       if (client == null) {
         // Make an hbase client instead of hadoop Client.
         client = new SecureClient(HbaseObjectWritable.class, conf, factory);
-        clients.put(factory, client);
+        clients.put(clusterId, client);
       } else {
         client.incCount();
       }
@@ -106,10 +105,11 @@ protected synchronized SecureClient getClient(Configuration conf,
     }
 
     /**
-     * Construct & cache an IPC client with the default SocketFactory
-     * if no cached client exists.
+     * Construct & cache an IPC client with the configured
+     * {@link HConstants#CLUSTER_ID} if no cached client exists.
      *
-     * @param conf Configuration
+     * @param conf
+     *          Configuration
      * @return an IPC client
      */
     protected synchronized SecureClient getClient(Configuration conf) {
@@ -125,7 +125,7 @@ protected void stopClient(SecureClient client) {
       synchronized (this) {
         client.decCount();
         if (client.isZeroReference()) {
-          clients.remove(client.getSocketFactory());
+          clients.remove(client.getClusterId());
         }
       }
       if (client.isZeroReference()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 235c07892802..5a2100261da4 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -1202,4 +1202,11 @@ public int hashCode() {
              (ticket == null ? 0 : ticket.hashCode()) )) ^ rpcTimeout;
     }
   }
+
+  /**
+   * @return the clusterId
+   */
+  public String getClusterId() {
+    return clusterId;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 8d96b1d4cef9..3d06e07f8f27 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -219,6 +219,19 @@ public static void initTableMapperJob(String table, Scan scan,
   public static void initCredentials(Job job) throws IOException {
     if (User.isHBaseSecurityEnabled(job.getConfiguration())) {
       try {
+        // init credentials for remote cluster
+        String quorumAddress = job.getConfiguration().get(
+            TableOutputFormat.QUORUM_ADDRESS);
+        if (quorumAddress != null) {
+          String[] parts = ZKUtil.transformClusterKey(quorumAddress);
+          Configuration peerConf = HBaseConfiguration.create(job
+              .getConfiguration());
+          peerConf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
+          peerConf.set("hbase.zookeeper.client.port", parts[1]);
+          peerConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
+          User.getCurrent().obtainAuthTokenForJob(peerConf, job);
+        }
+        
         User.getCurrent().obtainAuthTokenForJob(job.getConfiguration(), job);
       } catch (InterruptedException ie) {
         LOG.info("Interrupted obtaining user authentication token");

From bd195adcfd6607e9a4064495ad31eee75bdb1240 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 3 Jan 2013 02:12:57 +0000
Subject: [PATCH 0675/1540] HBASE-7351, HBASE-7399, HBASE-7406. Periodic health
 check chore (Vandana Ayyalasomayajula)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428142 13f79535-47bb-0310-9956-ffa450edef68
---
 src/examples/healthcheck/healthcheck.sh       |  84 +++++++++++
 .../org/apache/hadoop/hbase/HConstants.java   |  13 ++
 .../apache/hadoop/hbase/HealthCheckChore.java |  95 ++++++++++++
 .../apache/hadoop/hbase/HealthChecker.java    | 127 ++++++++++++++++
 .../org/apache/hadoop/hbase/HealthReport.java |  89 ++++++++++++
 .../apache/hadoop/hbase/master/HMaster.java   |  26 +++-
 .../hbase/regionserver/HRegionServer.java     |  26 ++++
 .../hbase/TestNodeHealthCheckChore.java       | 137 ++++++++++++++++++
 8 files changed, 596 insertions(+), 1 deletion(-)
 create mode 100644 src/examples/healthcheck/healthcheck.sh
 create mode 100644 src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/HealthChecker.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/HealthReport.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java

diff --git a/src/examples/healthcheck/healthcheck.sh b/src/examples/healthcheck/healthcheck.sh
new file mode 100644
index 000000000000..584636054ddc
--- /dev/null
+++ b/src/examples/healthcheck/healthcheck.sh
@@ -0,0 +1,84 @@
+#!/bin/bash 
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #     http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ # This is an example script for checking health of a node ( master or region server). 
+ # The health chore script should essentially output an message containing "ERROR" at an undesirable
+ # outcome of the checks in the script. 
+
+err=0;
+
+function check_disks {
+
+for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do
+    fsdev=""
+    fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`;
+    if [ -z "$fsdev" ] ; then
+      msg_="$msg_ $m(u)"
+    else
+      msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`"
+    fi
+  done
+
+  if [ -z "$msg_" ] ; then
+    echo "disks ok" ; exit 0
+  else
+    echo "$msg_" ; exit 2
+  fi
+
+}
+
+function check_link {
+  /usr/bin/snmpwalk -t 5 -Oe  -Oq  -Os -v 1 -c public localhost if | \
+        awk ' { 
+          split($1,a,".") ;
+          if ( a[1] == "ifIndex" ) { ifIndex[a[2]] = $2 }
+          if ( a[1] == "ifDescr" ) { ifDescr[a[2]] = $2 }
+          if ( a[1] == "ifType" ) { ifType[a[2]] = $2 }
+          if ( a[1] == "ifSpeed" ) { ifSpeed[a[2]] = $2 }
+          if ( a[1] == "ifAdminStatus" ) { ifAdminStatus[a[2]] = $2 }
+          if ( a[1] == "ifOperStatus" ) { ifOperStatus[a[2]] = $2 }
+        }
+        END {
+        up=0;
+        for (i in ifIndex ) {
+          if ( ifType[i] == 6 && ifAdminStatus[i] == 1 && ifOperStatus[i] == 1 && ifSpeed[i] == 1000000000 ) {
+            up=i;
+          }
+        }
+        if ( up == 0 ) { print "check link" ; exit 2 }
+        else { print ifDescr[up],"ok" }
+        }'
+  exit $? ;
+}
+
+for check in disks link ; do
+  msg=`check_${check}` ;
+  if [ $? -eq 0 ] ; then
+    ok_msg="$ok_msg$msg,"
+  else
+    err_msg="$err_msg$msg,"
+  fi
+done
+
+if [ ! -z "$err_msg" ] ; then
+  echo -n "ERROR $err_msg " 
+fi
+if [ ! -z "$ok_msg" ] ; then
+  echo -n "OK: $ok_msg" 
+fi
+echo
+exit 0
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 3c276de014a8..401fd8736825 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -672,6 +672,19 @@ public static enum Modify {
           Bytes.toString(META_TABLE_NAME), Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME,
           HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY }));
 
+  /** Health script related settings. */
+  public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location";
+  public static final String HEALTH_SCRIPT_TIMEOUT = "hbase.node.health.script.timeout";
+  public static final String HEALTH_CHORE_WAKE_FREQ =
+      "hbase.node.health.script.frequency";
+  public static final long DEFAULT_HEALTH_SCRIPT_TIMEOUT = 60000;
+  /**
+   * The maximum number of health check failures a server can encounter consecutively.
+   */
+  public static final String HEALTH_FAILURE_THRESHOLD =
+      "hbase.node.health.failure.threshold";
+  public static final int DEFAULT_HEALTH_FAILURE_THRESHOLD = 3;
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java b/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java
new file mode 100644
index 000000000000..bae41f6a0048
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java
@@ -0,0 +1,95 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HealthChecker.HealthCheckerExitStatus;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * The Class HealthCheckChore for running health checker regularly.
+ */
+ public class HealthCheckChore extends Chore {
+  private static Log LOG = LogFactory.getLog(HealthCheckChore.class);
+  private HealthChecker healthChecker;
+  private Configuration config;
+  private int threshold;
+  private int numTimesUnhealthy = 0;
+  private long failureWindow;
+  private long startWindow;
+
+  public HealthCheckChore(int sleepTime, Stoppable stopper, Configuration conf) {
+    super("HealthChecker", sleepTime, stopper);
+    LOG.info("Health Check Chore runs every " + StringUtils.formatTime(sleepTime));
+    this.config = conf;
+    String healthCheckScript = this.config.get(HConstants.HEALTH_SCRIPT_LOC);
+    long scriptTimeout = this.config.getLong(HConstants.HEALTH_SCRIPT_TIMEOUT,
+      HConstants.DEFAULT_HEALTH_SCRIPT_TIMEOUT);
+    healthChecker = new HealthChecker();
+    healthChecker.init(healthCheckScript, scriptTimeout);
+    this.threshold = config.getInt(HConstants.HEALTH_FAILURE_THRESHOLD,
+      HConstants.DEFAULT_HEALTH_FAILURE_THRESHOLD);
+    this.failureWindow = this.threshold * sleepTime;
+  }
+
+  @Override
+  protected void chore() {
+    HealthReport report = healthChecker.checkHealth();
+    boolean isHealthy = (report.getStatus() == HealthCheckerExitStatus.SUCCESS);
+    if (!isHealthy) {
+      boolean needToStop = decideToStop();
+      if (needToStop) {
+        this.stopper.stop("The region server reported unhealthy " + threshold
+            + " number of times consecutively.");
+      }
+      // Always log health report.
+      LOG.info("Health status at " + StringUtils.formatTime(System.currentTimeMillis()) + " : "
+          + report.getHealthReport());
+    }
+  }
+
+  private boolean decideToStop() {
+    boolean stop = false;
+    if (numTimesUnhealthy == 0) {
+      // First time we are seeing a failure. No need to stop, just
+      // record the time.
+      numTimesUnhealthy++;
+      startWindow = System.currentTimeMillis();
+    } else {
+      if ((System.currentTimeMillis() - startWindow) < failureWindow) {
+        numTimesUnhealthy++;
+        if (numTimesUnhealthy == threshold) {
+          stop = true;
+        } else {
+          stop = false;
+        }
+      } else {
+        // Outside of failure window, so we reset to 1.
+        numTimesUnhealthy = 1;
+        startWindow = System.currentTimeMillis();
+        stop = false;
+      }
+    }
+    return stop;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/HealthChecker.java b/src/main/java/org/apache/hadoop/hbase/HealthChecker.java
new file mode 100644
index 000000000000..e77317c81db5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/HealthChecker.java
@@ -0,0 +1,127 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell.ExitCodeException;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+
+/**
+ * A utility for executing an external script that checks the health of
+ * the node. An example script can be found at
+ * <tt>src/examples/healthcheck/healthcheck.sh</tt>
+ */
+class HealthChecker {
+
+  private static Log LOG = LogFactory.getLog(HealthChecker.class);
+  private ShellCommandExecutor shexec = null;
+  private String exceptionStackTrace;
+
+  /** Pattern used for searching in the output of the node health script */
+  static private final String ERROR_PATTERN = "ERROR";
+
+  private String healthCheckScript;
+  private long scriptTimeout;
+
+  enum HealthCheckerExitStatus {
+    SUCCESS,
+    TIMED_OUT,
+    FAILED_WITH_EXIT_CODE,
+    FAILED_WITH_EXCEPTION,
+    FAILED
+  }
+
+  /**
+   * Initialize.
+   *
+   * @param configuration
+   */
+  public void init(String location, long timeout) {
+    this.healthCheckScript = location;
+    this.scriptTimeout = timeout;
+    ArrayList<String> execScript = new ArrayList<String>();
+    execScript.add(healthCheckScript);
+    this.shexec = new ShellCommandExecutor(execScript.toArray(new String[execScript.size()]), null,
+        null, scriptTimeout);
+    LOG.info("HealthChecker initialized.");
+  }
+
+  public HealthReport checkHealth() {
+    HealthCheckerExitStatus status = HealthCheckerExitStatus.SUCCESS;
+    try {
+      shexec.execute();
+    } catch (ExitCodeException e) {
+      // ignore the exit code of the script
+      LOG.warn("Caught exception : " + e);
+      status = HealthCheckerExitStatus.FAILED_WITH_EXIT_CODE;
+    } catch (IOException e) {
+      LOG.warn("Caught exception : " + e);
+      if (!shexec.isTimedOut()) {
+        status = HealthCheckerExitStatus.FAILED_WITH_EXCEPTION;
+        exceptionStackTrace = org.apache.hadoop.util.StringUtils.stringifyException(e);
+      } else {
+        status = HealthCheckerExitStatus.TIMED_OUT;
+      }
+    } finally {
+      if (status == HealthCheckerExitStatus.SUCCESS) {
+        if (hasErrors(shexec.getOutput())) {
+          status = HealthCheckerExitStatus.FAILED;
+        }
+      }
+    }
+    return new HealthReport(status, getHealthReport(status));
+  }
+
+  private boolean hasErrors(String output) {
+    String[] splits = output.split("\n");
+    for (String split : splits) {
+      if (split.startsWith(ERROR_PATTERN)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private String getHealthReport(HealthCheckerExitStatus status){
+    String healthReport = null;
+    switch (status) {
+    case SUCCESS:
+      healthReport = "Server is healthy.";
+      break;
+    case TIMED_OUT:
+      healthReport = "Health script timed out";
+      break;
+    case FAILED_WITH_EXCEPTION:
+      healthReport = exceptionStackTrace;
+      break;
+    case FAILED_WITH_EXIT_CODE:
+      healthReport = "Health script failed with exit code.";
+      break;
+    case FAILED:
+      healthReport = shexec.getOutput();
+      break;
+    }
+    return healthReport;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/HealthReport.java b/src/main/java/org/apache/hadoop/hbase/HealthReport.java
new file mode 100644
index 000000000000..5292929b7f64
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/HealthReport.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.HealthChecker.HealthCheckerExitStatus;
+
+/**
+ * The Class RegionServerHealthReport containing information about
+ * health of the region server.
+ */
+class HealthReport {
+
+  private HealthCheckerExitStatus status;
+  private String healthReport;
+
+  HealthReport(HealthCheckerExitStatus status, String healthReport) {
+    super();
+    this.status = status;
+    this.healthReport = healthReport;
+  }
+
+  /**
+   * Gets the status of the region server.
+   *
+   * @return HealthCheckerExitStatus
+   */
+  HealthCheckerExitStatus getStatus() {
+    return status;
+  }
+ 
+  /**
+   * Gets the health report of the region server.
+   *
+   * @return String
+   */
+  String getHealthReport() {
+    return healthReport;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((healthReport == null) ? 0 : healthReport.hashCode());
+    result = prime * result + ((status == null) ? 0 : status.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof HealthReport)) {
+      return false;
+    }
+    HealthReport other = (HealthReport) obj;
+    if (healthReport == null) {
+      if (other.healthReport != null) {
+        return false;
+      }
+    } else if (!healthReport.equals(other.healthReport)) {
+      return false;
+    }
+    if (status != other.status) {
+      return false;
+    }
+    return true;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 1b6129963aa5..d439b2687261 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.Server;
@@ -235,6 +236,9 @@ public class HMaster extends HasThread
   private Map<String, Class<? extends CoprocessorProtocol>>
       protocolHandlerNames = Maps.newHashMap();
 
+  /** The health check chore. */
+  private HealthCheckChore healthCheckChore;
+
   /**
    * Initializes the HMaster. The steps are as follows:
    * <p>
@@ -303,6 +307,13 @@ public HMaster(final Configuration conf)
     this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);
     this.rpcServer.startThreads();
     this.metrics = new MasterMetrics(getServerName().toString());
+
+    // Health checker thread.
+    int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
+      HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
+    if (isHealthCheckerConfigured()) {
+      healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
+    }
   }
 
   /**
@@ -878,7 +889,12 @@ this, conf, getMasterFileSystem().getFileSystem(),
      this.infoServer.setAttribute(MASTER, this);
      this.infoServer.start();
     }
-   
+
+   // Start the health checker
+   if (this.healthCheckChore != null) {
+     Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker");
+   }
+
     // Start allowing requests to happen.
     this.rpcServer.openServer();
     if (LOG.isDebugEnabled()) {
@@ -905,6 +921,9 @@ private void stopServiceThreads() {
       }
     }
     if (this.executorService != null) this.executorService.shutdown();
+    if (this.healthCheckChore != null) {
+      this.healthCheckChore.interrupt();
+    }
   }
 
   private static Thread getAndStartBalancerChore(final HMaster master) {
@@ -1926,4 +1945,9 @@ void registerMBean() {
   public HFileCleaner getHFileCleaner() {
     return this.hfileCleaner;
   }
+
+  private boolean isHealthCheckerConfigured() {
+    String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
+    return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 6f09507d5cff..d1a33d2bad56 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,6 +66,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
@@ -368,6 +369,9 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
 
   private RegionServerCoprocessorHost rsHost;
 
+  /** The health check chore. */
+  private HealthCheckChore healthCheckChore;
+
   /**
    * Starts a HRegionServer at the default location
    *
@@ -660,6 +664,13 @@ private void initializeThreads() throws IOException {
     this.compactionChecker = new CompactionChecker(this,
       this.threadWakeFrequency * multiplier, this);
 
+    // Health checker thread.
+    int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
+      HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
+    if (isHealthCheckerConfigured()) {
+      healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
+    }
+
     this.leases = new Leases((int) conf.getLong(
         HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
         HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD),
@@ -775,6 +786,9 @@ public void run() {
     if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();
     if (this.compactionChecker != null)
       this.compactionChecker.interrupt();
+    if (this.healthCheckChore != null) {
+      this.healthCheckChore.interrupt();
+    }
 
     if (this.killed) {
       // Just skip out w/o closing regions.  Used when testing.
@@ -1559,6 +1573,10 @@ public void uncaughtException(Thread t, Throwable e) {
       handler);
     Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), n +
       ".compactionChecker", handler);
+    if (this.healthCheckChore != null) {
+      Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker",
+        handler);
+    }
 
     // Leases is not a Thread. Internally it runs a daemon thread. If it gets
     // an unhandled exception, it will just exit.
@@ -1789,6 +1807,9 @@ protected void kill() {
   protected void join() {
     Threads.shutdown(this.compactionChecker.getThread());
     Threads.shutdown(this.cacheFlusher.getThread());
+    if (this.healthCheckChore != null) {
+      Threads.shutdown(this.healthCheckChore.getThread());
+    }
     if (this.hlogRoller != null) {
       Threads.shutdown(this.hlogRoller.getThread());
     }
@@ -3864,4 +3885,9 @@ public long getResponseQueueSize(){
     }
     return 0;
   }
+
+  private boolean isHealthCheckerConfigured() {
+    String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
+    return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java b/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java
new file mode 100644
index 000000000000..05f699dfb0d2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.HealthChecker.HealthCheckerExitStatus;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestNodeHealthCheckChore {
+
+  private static final Log LOG = LogFactory.getLog(TestNodeHealthCheckChore.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private File healthScriptFile;
+
+
+  @After
+  public void cleanUp() throws IOException {
+    UTIL.cleanupTestDir();
+  }
+
+  @Test
+  public void testHealthChecker() throws Exception {
+    Configuration config = getConfForNodeHealthScript();
+    config.addResource(healthScriptFile.getName());
+    String location = healthScriptFile.getAbsolutePath();
+    long timeout = config.getLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 100);
+
+    String normalScript = "echo \"I am all fine\"";
+    createScript(normalScript, true);
+    HealthChecker checker = new HealthChecker();
+    checker.init(location, timeout);
+    HealthReport report = checker.checkHealth();
+    assertTrue(report.getStatus() == HealthCheckerExitStatus.SUCCESS);
+    LOG.info("Health Status:" + checker);
+
+    String errorScript = "echo ERROR\n echo \"Node not healthy\"";
+    createScript(errorScript, true);
+    report = checker.checkHealth();
+    assertTrue(report.getStatus() == HealthCheckerExitStatus.FAILED);
+    LOG.info("Health Status:" + report.getHealthReport());
+
+    String timeOutScript = "sleep 4\n echo\"I am fine\"";
+    createScript(timeOutScript, true);
+    report = checker.checkHealth();
+    assertTrue(report.getStatus() == HealthCheckerExitStatus.TIMED_OUT);
+    LOG.info("Health Status:" + report.getHealthReport());
+
+    healthScriptFile.delete();
+  }
+
+  @Test
+  public void testNodeHealthChore() throws Exception{
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = getConfForNodeHealthScript();
+    String errorScript = "echo ERROR\n echo \"Node not healthy\"";
+    createScript(errorScript, true);
+    HealthCheckChore rsChore = new HealthCheckChore(100, stop, conf);
+    //Default threshold is three.
+    rsChore.chore();
+    rsChore.chore();
+    assertFalse("Stoppable must not be stopped.", stop.isStopped());
+    rsChore.chore();
+    assertTrue("Stoppable must have been stopped.", stop.isStopped());
+  }
+
+  private void createScript(String scriptStr, boolean setExecutable)
+      throws Exception {
+    healthScriptFile.createNewFile();
+    PrintWriter pw = new PrintWriter(new FileOutputStream(healthScriptFile));
+    pw.println(scriptStr);
+    pw.flush();
+    pw.close();
+    healthScriptFile.setExecutable(setExecutable);
+  }
+
+  private Configuration getConfForNodeHealthScript() {
+    Configuration conf = UTIL.getConfiguration();
+    File tempDir = new File(UTIL.getDataTestDir().toString());
+    tempDir.mkdirs();
+    healthScriptFile = new File(tempDir.getAbsolutePath(), "HealthScript.sh");
+    conf.set(HConstants.HEALTH_SCRIPT_LOC,
+      healthScriptFile.getAbsolutePath());
+    conf.setLong(HConstants.HEALTH_FAILURE_THRESHOLD, 3);
+    conf.setLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 100);
+    return conf;
+  }
+
+  /**
+   * Simple helper class that just keeps track of whether or not its stopped.
+   */
+  private static class StoppableImplementation implements Stoppable {
+    private volatile boolean stop = false;
+
+    @Override
+    public void stop(String why) {
+      this.stop = true;
+    }
+
+    @Override
+    public boolean isStopped() {
+      return this.stop;
+    }
+
+  }
+}

From 26ad48d5babcea138634c9659a90994e810f5273 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 3 Jan 2013 05:32:41 +0000
Subject: [PATCH 0676/1540] HBASE-7485 TestSplitLogManager is still flaky on
 windows (Enis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428169 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/TestSplitLogManager.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 649c26e0f67b..749ccd7d70ed 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -202,7 +202,7 @@ public void testOrphanTaskAcquisition() throws Exception {
 
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
-    waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, 100);
+    waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
     Task task = slm.findOrCreateOrphanTask(tasknode);
     assertTrue(task.isOrphan());
     waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
@@ -447,6 +447,7 @@ public void testEmptyLogDir() throws Exception {
   public void testVanishingTaskZNode() throws Exception {
     LOG.info("testVanishingTaskZNode");
     conf.setInt("hbase.splitlog.manager.unassigned.timeout", 0);
+    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 1000);
     slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
     slm.finishInitialization();
     FileSystem fs = TEST_UTIL.getTestFileSystem();
@@ -475,7 +476,7 @@ public void run() {
       // remove the task znode, to finish the distributed log splitting
       ZKUtil.deleteNode(zkw, znode);
       waitForCounter(tot_mgr_get_data_nonode, 0, 1, 30000);
-      waitForCounter(tot_mgr_log_split_batch_success, 0, 1, 1000);
+      waitForCounter(tot_mgr_log_split_batch_success, 0, 1, to/2);
       assertTrue(fs.exists(logFile));
     } finally {
       if (thread != null) {

From 2862d4337924239874db15575392df7d61529e7f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 3 Jan 2013 05:38:02 +0000
Subject: [PATCH 0677/1540] HBASE-7483 TestHRegionOnCluster and
 TestSplitTransactionOnCluster are racy with HBaseAdmin.move()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428171 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/TestHRegionOnCluster.java      | 5 +++--
 .../hbase/regionserver/TestSplitTransactionOnCluster.java    | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
index 681634c61a2a..7c7aaf7da47c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
@@ -84,8 +84,9 @@ public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
       hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
           Bytes.toBytes(targetServer.getServerName().getServerName()));
       do {
-        Thread.sleep(1);
-      } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
+        Thread.sleep(10);
+      } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum ||
+          cluster.getMaster().getAssignmentManager().getRegionServerOfRegion(regionInfo) == null);
 
       // Put data: r2->v2
       putDataAndVerify(table, "r2", FAMILY, "v2", 2);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 7519c86cf4b4..b0e0c9d642d6 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -962,6 +962,11 @@ private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
       LOG.
         info("Moving " + hri.getRegionNameAsString() + " to " +
         hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
+      for (int i = 0; cluster.getMaster().getAssignmentManager()
+          .getRegionServerOfRegion(hri) == null
+          && i < 100; i++) {
+        Thread.sleep(10);
+      }
       admin.move(hri.getEncodedNameAsBytes(),
         Bytes.toBytes(hrs.getServerName().toString()));
     }

From 4f2cd435ca5285089b8e150591ba5dc2b8695f3d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 3 Jan 2013 05:44:28 +0000
Subject: [PATCH 0678/1540] CHANGES.txt for 0.94.4RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428173 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 01d2b5c6194f..9a4be492e2f3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.4 - 12/28/2012
+Release 0.94.4 - 1/2/2012
 Sub-task
 
     [HBASE-3776] - Add Bloom Filter Support to HFileOutputFormat
@@ -11,6 +11,8 @@ Sub-task
     [HBASE-7331] - Add access control for region open and close, row locking, and stopping the regionserver
     [HBASE-7336] - HFileBlock.readAtOffset does not work well with multiple threads
     [HBASE-7371] - Blocksize in TestHFileBlock is unintentionally small
+    [HBASE-7399] - Health check chore for HMaster
+    [HBASE-7406] - Example health checker script
     [HBASE-7431] - TestSplitTransactionOnCluster tests still flaky
     [HBASE-7438] - TestSplitTransactionOnCluster has too many infinite loops
 
@@ -53,7 +55,13 @@ Bug
     [HBASE-7432] - TestHBaseFsck prevents testsuite from finishing
     [HBASE-7435] - BuiltInGzipDecompressor is only released during full GC
     [HBASE-7440] - ReplicationZookeeper#addPeer is racy
+    [HBASE-7442] - HBase remote CopyTable not working when security enabled
     [HBASE-7455] - Increase timeouts in TestReplication and TestSplitLogWorker
+    [HBASE-7464] - [REST] Sending HTML for errors is unhelpful
+    [HBASE-7466] - Fix junit dependency typo in 0.94
+    [HBASE-7467] - CleanerChore checkAndDeleteDirectory not deleting empty directories
+    [HBASE-7483] - TestHRegionOnCluster and TestSplitTransactionOnCluster are racy with HBaseAdmin.move()
+    [HBASE-7485] - TestSplitLogManager is still flaky on windows
 
 Improvement
 
@@ -71,10 +79,13 @@ Improvement
     [HBASE-7231] - port HBASE-7200 create integration test for balancing regions and killing region servers to 0.94
     [HBASE-7249] - add test name filter to IntegrationTestsDriver
     [HBASE-7328] - IntegrationTestRebalanceAndKillServersTargeted supercedes IntegrationTestRebalanceAndKillServers, remove
+    [HBASE-7351] - Periodic health check chore
     [HBASE-7359] - [REST] 'accessToken' in RemoteHTable is vestigial
     [HBASE-7374] - Expose master table operations for coprocessors by way of MasterServices
     [HBASE-7377] - Clean up TestHBase7051
     [HBASE-7381] - Lightweight data transfer for Class Result
+    [HBASE-7469] - [REST] Share a HBaseAdmin instance
+    [HBASE-7472] - [REST] Support MIME type application/protobuf
 
 Task
 

From e62d7d0a339f913b9e6555946d845c519a552429 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 4 Jan 2013 00:37:16 +0000
Subject: [PATCH 0679/1540] HBASE-7423. HFileArchiver should not use the
 configuration from the Filesystem

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1428684 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/backup/HFileArchiver.java    | 19 +++++++++++--------
 .../hadoop/hbase/master/CatalogJanitor.java   |  6 +++---
 .../hadoop/hbase/master/MasterFileSystem.java | 16 ++++++++--------
 .../hadoop/hbase/regionserver/HRegion.java    |  4 ++--
 .../hadoop/hbase/regionserver/Store.java      | 12 ++++++------
 .../hbase/backup/TestHFileArchiving.java      |  4 ++--
 .../hbase/master/TestCatalogJanitor.java      |  7 +++----
 7 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index 0da7c69b725e..a2f7f1247c43 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -62,20 +62,22 @@ private HFileArchiver() {
   /**
    * Cleans up all the files for a HRegion by archiving the HFiles to the
    * archive directory
+   * @param conf the configuration to use
    * @param fs the file system object
    * @param info HRegionInfo for region to be deleted
    * @throws IOException
    */
-  public static void archiveRegion(FileSystem fs, HRegionInfo info)
+  public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo info)
       throws IOException {
-    Path rootDir = FSUtils.getRootDir(fs.getConf());
-    archiveRegion(fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()),
+    Path rootDir = FSUtils.getRootDir(conf);
+    archiveRegion(conf, fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()),
       HRegion.getRegionDir(rootDir, info));
   }
 
 
   /**
    * Remove an entire region from the table directory via archiving the region's hfiles.
+   * @param conf the configuration to use
    * @param fs {@link FileSystem} from which to remove the region
    * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
    *          the archive path)
@@ -85,7 +87,8 @@ public static void archiveRegion(FileSystem fs, HRegionInfo info)
    *         operations could not complete.
    * @throws IOException if the request cannot be completed
    */
-  public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
+  public static boolean archiveRegion(Configuration conf, FileSystem fs, Path rootdir,
+      Path tableDir, Path regionDir)
       throws IOException {
     if (LOG.isDebugEnabled()) {
       LOG.debug("ARCHIVING region " + regionDir.toString());
@@ -104,7 +107,7 @@ public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir,
 
     // make sure the regiondir lives under the tabledir
     Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
-    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(fs.getConf(), tableDir, regionDir);
+    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(conf, tableDir, regionDir);
 
     LOG.debug("Have an archive directory, preparing to move files");
     FileStatusConverter getAsFile = new FileStatusConverter(fs);
@@ -180,16 +183,16 @@ public static void archiveFamily(FileSystem fs, Configuration conf,
 
   /**
    * Remove the store files, either by archiving them or outright deletion
+   * @param conf {@link Configuration} to examine to determine the archive directory
    * @param fs the filesystem where the store files live
    * @param parent Parent region hosting the store files
-   * @param conf {@link Configuration} to examine to determine the archive directory
    * @param family the family hosting the store files
    * @param compactedFiles files to be disposed of. No further reading of these files should be
    *          attempted; otherwise likely to cause an {@link IOException}
    * @throws IOException if the files could not be correctly disposed.
    */
-  public static void archiveStoreFiles(FileSystem fs, HRegion parent,
-      Configuration conf, byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
+  public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegion parent,
+      byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
 
     // sometimes in testing, we don't have rss, so we need to check for that
     if (fs == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 53a43589ad73..94ff5bc3f8b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -214,7 +214,7 @@ static HRegionInfo getHRegionInfo(final Result result)
 
   /**
    * If daughters no longer hold reference to the parents, delete the parent.
-   * @param server HRegionInterface of meta server to talk to 
+   * @param server HRegionInterface of meta server to talk to
    * @param parent HRegionInfo of split offlined parent
    * @param rowContent Content of <code>parent</code> row in
    * <code>metaRegionName</code>
@@ -246,7 +246,7 @@ boolean cleanParent(final HRegionInfo parent, Result rowContent)
         this.services.getAssignmentManager().regionOffline(parent);
       }
       FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
-      HFileArchiver.archiveRegion(fs, parent);
+      HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
       MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
       result = true;
     }
@@ -291,7 +291,7 @@ private void removeDaughtersFromParent(final HRegionInfo parent)
   /**
    * Checks if a daughter region -- either splitA or splitB -- still holds
    * references to parent.
-   * @param parent Parent region name. 
+   * @param parent Parent region name.
    * @param split Which column family.
    * @param qualifier Which of the daughters to look at, splitA or splitB.
    * @return A pair where the first boolean says whether or not the daughter
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 5999a53f120d..694ba0d20b8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -247,13 +247,13 @@ void splitLogAfterStartup() {
       }
     } while (retrySplitting);
   }
-  
+
   public void splitLog(final ServerName serverName) throws IOException {
     List<ServerName> serverNames = new ArrayList<ServerName>();
     serverNames.add(serverName);
     splitLog(serverNames);
   }
-  
+
   public void splitLog(final List<ServerName> serverNames) throws IOException {
     long splitTime = 0, splitLogSize = 0;
     List<Path> logDirs = new ArrayList<Path>();
@@ -279,7 +279,7 @@ public void splitLog(final List<ServerName> serverNames) throws IOException {
       LOG.info("No logs to split");
       return;
     }
-      
+
     if (distributedLogSplitting) {
       splitLogManager.handleDeadWorkers(serverNames);
       splitTime = EnvironmentEdgeManager.currentTimeMillis();
@@ -290,7 +290,7 @@ public void splitLog(final List<ServerName> serverNames) throws IOException {
         // splitLogLock ensures that dead region servers' logs are processed
         // one at a time
         this.splitLogLock.lock();
-        try {              
+        try {
           HLogSplitter splitter = HLogSplitter.createLogSplitter(
             conf, rootdir, logDir, oldLogDir, this.fs);
           try {
@@ -344,7 +344,7 @@ private Path checkRootDir(final Path rd, final Configuration c,
         // there is one datanode it will succeed. Permission problems should have
         // already been caught by mkdirs above.
         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, 
+          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
         		  HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
       } else {
         if (!fs.isDirectory(rd)) {
@@ -352,7 +352,7 @@ private Path checkRootDir(final Path rd, final Configuration c,
         }
         // as above
         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, 
+          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
         		  HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
       }
     } catch (IllegalArgumentException iae) {
@@ -443,7 +443,7 @@ public static void setInfoFamilyCachingForMeta(final boolean b) {
 
 
   public void deleteRegion(HRegionInfo region) throws IOException {
-    HFileArchiver.archiveRegion(fs, region);
+    HFileArchiver.archiveRegion(conf, fs, region);
   }
 
   public void deleteTable(byte[] tableName) throws IOException {
@@ -481,7 +481,7 @@ public void stop() {
 
   /**
    * Create new HTableDescriptor in HDFS.
-   * 
+   *
    * @param htableDescriptor
    */
   public void createTableDescriptor(HTableDescriptor htableDescriptor)
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e2e6fab65bf6..757ab2c70b7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4275,10 +4275,10 @@ public static HRegion merge(HRegion a, HRegion b)
     }
 
     // delete out the 'A' region
-    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(a.getConf()), a.getTableDir(),
+    HFileArchiver.archiveRegion(a.getConf(), fs, FSUtils.getRootDir(a.getConf()), a.getTableDir(),
       a.getRegionDir());
     // delete out the 'B' region
-    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(b.getConf()), b.getTableDir(),
+    HFileArchiver.archiveRegion(b.getConf(), fs, FSUtils.getRootDir(b.getConf()), b.getTableDir(),
       b.getRegionDir());
 
     LOG.info("merge completed. New region is " + dstRegion);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 9a02c16c77c8..076bf9d79e20 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -584,7 +584,7 @@ void bulkLoadHFile(String srcPathStr) throws IOException {
     StoreFile.Reader r = sf.createReader();
     this.storeSize += r.length();
     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
-    
+
     LOG.info("Moved hfile " + srcPath + " into store directory " +
         homedir + " - updating store file list.");
 
@@ -1626,7 +1626,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
 
       // let the archive util decide if we should archive or delete the files
       LOG.debug("Removing store files after compaction...");
-      HFileArchiver.archiveStoreFiles(this.fs, this.region, this.conf, this.family.getName(),
+      HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.region, this.family.getName(),
         compactedFiles);
 
     } catch (IOException e) {
@@ -2076,10 +2076,10 @@ public int getCompactPriority(int priority) {
 
   boolean throttleCompaction(long compactionSize) {
     long throttlePoint = conf.getLong(
-        "hbase.regionserver.thread.compaction.throttle",  
-        2 * this.minFilesToCompact * this.region.memstoreFlushSize);  
-    return compactionSize > throttlePoint;  
-  } 
+        "hbase.regionserver.thread.compaction.throttle",
+        2 * this.minFilesToCompact * this.region.memstoreFlushSize);
+    return compactionSize > throttlePoint;
+  }
 
   public HRegion getHRegion() {
     return this.region;
diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index 0b4f6dfbbd6a..eb17ac7e7f32 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -137,7 +137,7 @@ public void testRemovesRegionDirOnArchive() throws Exception {
     // now attempt to depose the region
     Path regionDir = HRegion.getRegionDir(region.getTableDir().getParent(), region.getRegionInfo());
 
-    HFileArchiver.archiveRegion(fs, region.getRegionInfo());
+    HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
 
     // check for the existence of the archive directory and some files in it
     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
@@ -197,7 +197,7 @@ public boolean accept(Path file) {
     }
 
     // then archive the region
-    HFileArchiver.archiveRegion(fs, region.getRegionInfo());
+    HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
 
     // and check to make sure the region directoy got deleted
     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 1f75842d5c08..45f106ce63e5 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -68,7 +68,6 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
@@ -129,7 +128,7 @@ public ZooKeeperWatcher getZooKeeper() {
     public void abort(String why, Throwable e) {
       //no-op
     }
-    
+
     @Override
     public boolean isAborted() {
       return false;
@@ -206,7 +205,7 @@ public CatalogTracker getCatalogTracker() {
 
     @Override
     public Configuration getConfiguration() {
-      return null;
+      return mfs.conf;
     }
 
     @Override
@@ -218,7 +217,7 @@ public ServerName getServerName() {
     public void abort(String why, Throwable e) {
       //no-op
     }
-    
+
     @Override
     public boolean isAborted() {
       return false;

From e34c5553f5cc64ff69e8e59a04dc23b269dde755 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 5 Jan 2013 05:09:24 +0000
Subject: [PATCH 0680/1540] HBASE-7497
 TestDistributedLogSplitting.testDelayedDeleteOnFailure times out occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1429202 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/TestDistributedLogSplitting.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index afd9f0e68c96..823d7da933de 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -269,7 +269,7 @@ public void run() {
         "tot_wkr_preempt_task");
   }
 
-  @Test(timeout=25000)
+  @Test(timeout=30000)
   public void testDelayedDeleteOnFailure() throws Exception {
     LOG.info("testDelayedDeleteOnFailure");
     startCluster(1);

From 092f4d9464ed63719d0a51cfd317afad9f91bc62 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 5 Jan 2013 19:32:38 +0000
Subject: [PATCH 0681/1540] HBASE-7498 Make REST server thread pool size
 configurable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1429365 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/Main.java    | 12 ++++++++++
 src/main/resources/hbase-default.xml          | 23 +++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index 7e2681bb9331..2bde21820454 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -44,6 +44,7 @@
 import org.mortbay.jetty.nio.SelectChannelConnector;
 import org.mortbay.jetty.servlet.Context;
 import org.mortbay.jetty.servlet.ServletHolder;
+import org.mortbay.thread.QueuedThreadPool;
 
 import com.sun.jersey.spi.container.servlet.ServletContainer;
 
@@ -139,6 +140,17 @@ public static void main(String[] args) throws Exception {
 
     server.addConnector(connector);
 
+    // Set the default max thread number to 100 to limit
+    // the number of concurrent requests so that REST server doesn't OOM easily.
+    // Jetty set the default max thread number to 250, if we don't set it.
+    //
+    // Our default min thread number 2 is the same as that used by Jetty.
+    int maxThreads = servlet.getConfiguration().getInt("hbase.rest.threads.max", 100);
+    int minThreads = servlet.getConfiguration().getInt("hbase.rest.threads.min", 2);
+    QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads);
+    threadPool.setMinThreads(minThreads);
+    server.setThreadPool(threadPool);
+
     server.setSendServerVersion(false);
     server.setSendDateHeader(false);
     server.setStopAtShutdown(true);
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 3efa8517bbb0..f8e9664a12ea 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -899,4 +899,27 @@
     default log cleaners in the list as they will be overwritten in hbase-site.xml.
     </description>
   </property>
+  <property>
+    <name>hbase.rest.threads.max</name>
+    <value>100</value>
+    <description>
+        The maximum number of threads of the REST server thread pool.
+        Threads in the pool are reused to process REST requests. This
+        controls the maximum number of requests processed concurrently.
+        It may help to control the memory used by the REST server to
+        avoid OOM issues. If the thread pool is full, incoming requests
+        will be queued up and wait for some free threads. The default
+        is 100.
+    </description>
+  </property>
+  <property>
+    <name>hbase.rest.threads.min</name>
+    <value>2</value>
+    <description>
+        The minimum number of threads of the REST server thread pool.
+        The thread pool always has at least these number of threads so
+        the REST server is ready to serve incoming requests. The default
+        is 2.
+    </description>
+  </property>
 </configuration>

From 79bad4526cfc15152603ab5c6d017a741faa7513 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 5 Jan 2013 23:13:29 +0000
Subject: [PATCH 0682/1540] HBASE-7483 Addendum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1429421 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
index 7c7aaf7da47c..9b10426309fd 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
@@ -86,7 +86,8 @@ public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
       do {
         Thread.sleep(10);
       } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum ||
-          cluster.getMaster().getAssignmentManager().getRegionServerOfRegion(regionInfo) == null);
+          !targetServer.getServerName().equals(
+              cluster.getMaster().getAssignmentManager().getRegionServerOfRegion(regionInfo)));
 
       // Put data: r2->v2
       putDataAndVerify(table, "r2", FAMILY, "v2", 2);

From 0c254b9a9fc132a0490a965028c297e84acb7d6d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 6 Jan 2013 03:11:54 +0000
Subject: [PATCH 0683/1540] HBASE-7499 TestScannerTimeout timeout is too
 aggressive.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1429458 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/TestScannerTimeout.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
index 1f9358324973..01e2a542b3f8 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
@@ -50,7 +50,7 @@ public class TestScannerTimeout {
   private final static int NB_ROWS = 10;
   // Be careful w/ what you set this timer too... it can get in the way of
   // the mini cluster coming up -- the verification in particular.
-  private final static int SCANNER_TIMEOUT = 10000;
+  private final static int SCANNER_TIMEOUT = 15000;
   private final static int SCANNER_CACHING = 5;
 
    /**

From a42c88b9acbb9b7a9904e7d0ef984f35fe8b2bc0 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 7 Jan 2013 21:00:05 +0000
Subject: [PATCH 0684/1540] HBASE-7476 HBase shell count command doesn't escape
 binary output

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1430004 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/table.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index 34b33c883305..ed4113228565 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -89,7 +89,8 @@ def count(interval = 1000, caching_rows = 10)
         count += 1
         next unless (block_given? && count % interval == 0)
         # Allow command modules to visualize counting process
-        yield(count, String.from_java_bytes(row.getRow))
+        yield(count, 
+              org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow))
       end
 
       # Return the counter

From c25e5c222b568e593d424f8266f10daa60f9846e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 8 Jan 2013 21:04:11 +0000
Subject: [PATCH 0685/1540] HBASE-7369 HConnectionManager should remove aborted
 connections (Bryan Baugher)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1430533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 10 +++++-
 .../apache/hadoop/hbase/client/TestHCM.java   | 35 +++++++++++++++++--
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 08fd2a9c23ee..fe775cd44616 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -182,6 +182,10 @@ public static HConnection getConnection(Configuration conf)
       if (connection == null) {
         connection = new HConnectionImplementation(conf, true);
         HBASE_INSTANCES.put(connectionKey, connection);
+      } else if (connection.isClosed()) {
+        HConnectionManager.deleteConnection(connectionKey, true, true);
+        connection = new HConnectionImplementation(conf, true);
+        HBASE_INSTANCES.put(connectionKey, connection);
       }
       connection.incCount();
       return connection;
@@ -1775,7 +1779,11 @@ void close(boolean stopProxy) {
 
     public void close() {
       if (managed) {
-        HConnectionManager.deleteConnection((HConnection)this, stopProxy, false);
+        if (aborted) {
+          HConnectionManager.deleteStaleConnection(this);
+        } else {
+          HConnectionManager.deleteConnection(this, stopProxy, false);
+        }
       } else {
         close(true);
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 023eaff0d8de..66548a1d7f92 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -19,13 +19,15 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -34,7 +36,14 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
 import org.junit.AfterClass;
@@ -115,6 +124,28 @@ public static void createNewConfigurations() throws SecurityException,
   private static int getHConnectionManagerCacheSize(){
     return HConnectionTestingUtility.getConnectionCount();
   }
+  
+  @Test
+  public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
+    // Save off current HConnections
+    Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances = 
+        new HashMap<HConnectionKey, HConnectionImplementation>();
+    oldHBaseInstances.putAll(HConnectionManager.HBASE_INSTANCES);
+    
+    HConnectionManager.HBASE_INSTANCES.clear();
+
+    try {
+      HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
+      connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
+          "test abortingHConnectionRemovesItselfFromHCM"));
+      Assert.assertNotSame(connection,
+        HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
+    } finally {
+      // Put original HConnections back
+      HConnectionManager.HBASE_INSTANCES.clear();
+      HConnectionManager.HBASE_INSTANCES.putAll(oldHBaseInstances);
+    }
+  }
 
   /**
    * Test that when we delete a location using the first row of a region

From 70836393f35ab7b3b61404834791d912997f7ecc Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Tue, 8 Jan 2013 22:36:29 +0000
Subject: [PATCH 0686/1540] HBASE-7513 HDFSBlocksDistribution shouldn't send
 NPEs when something goes wrong

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1430580 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HDFSBlocksDistribution.java  | 11 ++-
 .../hbase/TestHDFSBlocksDistribution.java     | 69 +++++++++++++++++++
 2 files changed, 77 insertions(+), 3 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
index 29c59afb40ae..6682d172a53c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
+++ b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
@@ -29,7 +29,9 @@
 
 
 /**
- * Data structure to describe the distribution of HDFS blocks amount hosts
+ * Data structure to describe the distribution of HDFS blocks amount hosts.
+ *
+ * Adding erroneous data will be ignored silently.
  */
 public class HDFSBlocksDistribution {
   private Map<String,HostAndWeight> hostAndWeights = null;
@@ -120,8 +122,10 @@ public synchronized String toString() {
    */
   public void addHostsAndBlockWeight(String[] hosts, long weight) {
     if (hosts == null || hosts.length == 0) {
-      throw new NullPointerException("empty hosts");
+      // erroneous data
+      return;
     }
+
     addUniqueWeight(weight);
     for (String hostname : hosts) {
       addHostAndBlockWeight(hostname, weight);
@@ -144,7 +148,8 @@ private void addUniqueWeight(long weight) {
    */
   private void addHostAndBlockWeight(String host, long weight) {
     if (host == null) {
-      throw new NullPointerException("Passed hostname is null");
+      // erroneous data
+      return;
     }
 
     HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java b/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java
new file mode 100644
index 000000000000..ea694067c782
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java
@@ -0,0 +1,69 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static junit.framework.Assert.assertEquals;
+
+@Category(SmallTests.class)
+public class TestHDFSBlocksDistribution {
+  @Test
+  public void testAddHostsAndBlockWeight() throws Exception {
+    HDFSBlocksDistribution distribution = new HDFSBlocksDistribution();
+    distribution.addHostsAndBlockWeight(null, 100);
+    assertEquals("Expecting no hosts weights", 0, distribution.getHostAndWeights().size());
+    distribution.addHostsAndBlockWeight(new String[0], 100);
+    assertEquals("Expecting no hosts weights", 0, distribution.getHostAndWeights().size());
+    distribution.addHostsAndBlockWeight(new String[] {"test"}, 101);
+    assertEquals("Should be one host", 1, distribution.getHostAndWeights().size());
+    distribution.addHostsAndBlockWeight(new String[] {"test"}, 202);
+    assertEquals("Should be one host", 1, distribution.getHostAndWeights().size());
+    assertEquals("test host should have weight 303", 303,
+        distribution.getHostAndWeights().get("test").getWeight());
+    distribution.addHostsAndBlockWeight(new String[] {"testTwo"}, 222);
+    assertEquals("Should be two hosts", 2, distribution.getHostAndWeights().size());
+    assertEquals("Total weight should be 525", 525, distribution.getUniqueBlocksTotalWeight());
+  }
+
+  public class MockHDFSBlocksDistribution extends HDFSBlocksDistribution {
+    public Map<String,HostAndWeight> getHostAndWeights() {
+      HashMap<String, HostAndWeight> map = new HashMap<String, HostAndWeight>();
+      map.put("test", new HostAndWeight(null, 100));
+      return map;
+    }
+
+  }
+
+  @Test
+  public void testAdd() throws Exception {
+    HDFSBlocksDistribution distribution = new HDFSBlocksDistribution();
+    distribution.add(new MockHDFSBlocksDistribution());
+    assertEquals("Expecting no hosts weights", 0, distribution.getHostAndWeights().size());
+    distribution.addHostsAndBlockWeight(new String[]{"test"}, 10);
+    assertEquals("Should be one host", 1, distribution.getHostAndWeights().size());
+    distribution.add(new MockHDFSBlocksDistribution());
+    assertEquals("Should be one host", 1, distribution.getHostAndWeights().size());
+    assertEquals("Total weight should be 10", 10, distribution.getUniqueBlocksTotalWeight());
+  }
+}

From cf4d76aac305f484e57bdb7b2e2aaaca5b0a04ad Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Wed, 9 Jan 2013 09:50:38 +0000
Subject: [PATCH 0687/1540] HBASE-7505 Server will hang when stopping cluster,
 caused by waiting for split threads(Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1430751 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/CompactSplitThread.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index add8f75cec1b..28662689b0d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -252,6 +252,9 @@ private void waitFor(ThreadPoolExecutor t, String name) {
       try {
         done = t.awaitTermination(60, TimeUnit.SECONDS);
         LOG.debug("Waiting for " + name + " to finish...");
+        if (!done) {
+          t.shutdownNow();
+        }
       } catch (InterruptedException ie) {
         LOG.debug("Interrupted waiting for " + name + " to finish...");
       }

From f151f8cf5b07121a82248b83942b285bffbd507f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 9 Jan 2013 20:33:45 +0000
Subject: [PATCH 0688/1540] HBASE-7515 Store.loadStoreFiles should close opened
 files if there's an exception (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1431046 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Store.java      | 62 ++++++++++++-------
 .../hadoop/hbase/regionserver/StoreFile.java  | 11 +++-
 2 files changed, 51 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 076bf9d79e20..0839e3c59d43 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -412,26 +413,38 @@ public StoreFile call() throws IOException {
       totalValidStoreFile++;
     }
 
+    IOException ioe = null;
     try {
       for (int i = 0; i < totalValidStoreFile; i++) {
-        Future<StoreFile> future = completionService.take();
-        StoreFile storeFile = future.get();
-        long length = storeFile.getReader().length();
-        this.storeSize += length;
-        this.totalUncompressedBytes +=
-          storeFile.getReader().getTotalUncompressedBytes();
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("loaded " + storeFile.toStringDetailed());
-        }
-        results.add(storeFile);
-      }
-    } catch (InterruptedException e) {
-      throw new IOException(e);
-    } catch (ExecutionException e) {
-      throw new IOException(e.getCause());
+        try {
+          Future<StoreFile> future = completionService.take();
+          StoreFile storeFile = future.get();
+          long length = storeFile.getReader().length();
+          this.storeSize += length;
+          this.totalUncompressedBytes +=
+              storeFile.getReader().getTotalUncompressedBytes();
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("loaded " + storeFile.toStringDetailed());
+          }
+          results.add(storeFile);
+        } catch (InterruptedException e) {
+          if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
+        } catch (ExecutionException e) {
+          if (ioe == null) ioe = new IOException(e.getCause());
+        } 
+      } 
     } finally {
       storeFileOpenerThreadPool.shutdownNow();
     }
+    if (ioe != null) {
+      // close StoreFile readers
+      try {
+        for (StoreFile file : results) {
+          if (file != null) file.closeReader(true);
+        }
+      } catch (IOException e) { }
+      throw ioe;
+    }
 
     return results;
   }
@@ -651,18 +664,25 @@ public Void call() throws IOException {
           });
         }
 
+        IOException ioe = null;
         try {
           for (int i = 0; i < result.size(); i++) {
-            Future<Void> future = completionService.take();
-            future.get();
+            try {
+              Future<Void> future = completionService.take();
+              future.get();
+            } catch (InterruptedException e) {
+              if (ioe == null) {
+                ioe = new InterruptedIOException();
+                ioe.initCause(e);
+              }
+            } catch (ExecutionException e) {
+              if (ioe == null) ioe = new IOException(e.getCause());
+            }
           }
-        } catch (InterruptedException e) {
-          throw new IOException(e);
-        } catch (ExecutionException e) {
-          throw new IOException(e.getCause());
         } finally {
           storeFileCloserThreadPool.shutdownNow();
         }
+        if (ioe != null) throw ioe;
       }
       LOG.info("Closed " + this);
       return result;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 40df89926c5b..bf31c32cf449 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -599,7 +599,16 @@ private Reader open() throws IOException {
    */
   public Reader createReader() throws IOException {
     if (this.reader == null) {
-      this.reader = open();
+      try {
+        this.reader = open();
+      } catch (IOException e) {
+        try {
+          this.closeReader(true);
+        } catch (IOException ee) {              
+        }
+        throw e;
+      }
+
     }
     return this.reader;
   }

From d2d2334cc72dd3e2bedfb3e89522587e548d962d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 9 Jan 2013 23:15:12 +0000
Subject: [PATCH 0689/1540] HBASE-7524 hbase-policy.xml is improperly set thus
 all rules in it can be by-passed (Kai Zheng)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1431142 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/security/HBasePolicyProvider.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java b/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java
index 0c4b4cbbaee0..cf1d3f1bf27a 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java
@@ -44,7 +44,7 @@ public Service[] getServices() {
   public static void init(Configuration conf,
       ServiceAuthorizationManager authManager) {
     // set service-level authorization security policy
-    conf.set("hadoop.policy.file", "hbase-policy.xml");
+    System.setProperty("hadoop.policy.file", "hbase-policy.xml");
     if (conf.getBoolean(
           ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false)) {
       authManager.refresh(conf, new HBasePolicyProvider());

From 0085d099aa86b909ee70fd622b5c388928b90787 Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Thu, 10 Jan 2013 08:29:03 +0000
Subject: [PATCH 0690/1540] HBASE-7504 -ROOT- may be offline forever after
 FullGC of RS (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1431204 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/handler/ServerShutdownHandler.java  | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 8fad16359eb8..5d7ea853519d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -102,6 +102,12 @@ private void verifyAndAssignRoot()
       getLong("hbase.catalog.verification.timeout", 1000);
     if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
       this.services.getAssignmentManager().assignRoot();
+    } else if (serverName.equals(server.getCatalogTracker().getRootLocation())) {
+      throw new IOException("-ROOT- is onlined on the dead server "
+          + serverName);
+    } else {
+      LOG.info("Skip assigning -ROOT-, because it is online on the "
+          + server.getCatalogTracker().getRootLocation());
     }
   }
 

From 5aa38214ae7244c13cf81d215d5af1b7341d0454 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Fri, 11 Jan 2013 00:53:49 +0000
Subject: [PATCH 0691/1540] HBASE-7530  [replication] Work around HDFS-4380
 else we get NPEs HBASE-7531  [replication] NPE in SequenceFileLogReader
 because             ReplicationSource doesn't nullify the reader HBASE-7534 
 [replication] TestReplication.queueFailover can fail             because
 HBaseTestingUtility.createMultiRegions is dangerous

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1431769 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/ReplicationSource.java |  1 +
 .../apache/hadoop/hbase/HBaseTestingUtility.java    | 13 +++++++++++++
 .../hadoop/hbase/replication/TestReplication.java   |  9 +++------
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 9f26e501ca93..866d92b203e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -554,6 +554,7 @@ protected boolean openReader(int sleepMultiplier) {
       }
     } catch (IOException ioe) {
       LOG.warn(peerClusterZnode + " Got: ", ioe);
+      this.reader = null;
       // TODO Need a better way to determinate if a file is really gone but
       // TODO without scanning all logs dir
       if (sleepMultiplier == this.maxRetriesMultiplier) {
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 91dc7827433e..0279f06ab78f 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1124,6 +1124,19 @@ public int createMultiRegions(HTable table, byte[] columnFamily)
     Bytes.toBytes("xxx"), Bytes.toBytes("yyy")
   };
 
+  public static final byte[][] KEYS_FOR_HBA_CREATE_TABLE = {
+      Bytes.toBytes("bbb"),
+      Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
+      Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
+      Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
+      Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
+      Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
+      Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
+      Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
+      Bytes.toBytes("xxx"), Bytes.toBytes("yyy"), Bytes.toBytes("zzz")
+  };
+
+
   /**
    * Creates many regions names "aaa" to "zzz".
    *
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index c99092e949d2..deea43cd76d7 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -92,9 +92,8 @@ public class TestReplication {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
-    // smaller block size and capacity to trigger more operations
-    // and test them
-    conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
+    // smaller log roll size to trigger more events
+    conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
     conf1.setInt("replication.source.size.capacity", 1024);
     conf1.setLong("replication.source.sleepforretries", 100);
     conf1.setInt("hbase.regionserver.maxlogs", 10);
@@ -142,7 +141,7 @@ public static void setUpBeforeClass() throws Exception {
     table.addFamily(fam);
     HBaseAdmin admin1 = new HBaseAdmin(conf1);
     HBaseAdmin admin2 = new HBaseAdmin(conf2);
-    admin1.createTable(table);
+    admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
     admin2.createTable(table);
     htable1 = new HTable(conf1, tableName);
     htable1.setWriteBufferSize(1024);
@@ -716,8 +715,6 @@ public void testVerifyRepJob() throws Exception {
    */
   @Test(timeout=300000)
   public void queueFailover() throws Exception {
-    utility1.createMultiRegions(htable1, famName, false);
-
     // killing the RS with .META. can result into failed puts until we solve
     // IO fencing
     int rsToKill1 =

From abeddb4eeeff4f2def11782e901690baafdd6aa4 Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Fri, 11 Jan 2013 05:48:58 +0000
Subject: [PATCH 0692/1540] HBASE-7506 Judgment of carrying ROOT/META will
 become wrong when expiring server (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1431903 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/handler/ServerShutdownHandler.java | 32 +++++++++++++------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 5d7ea853519d..a61e825d0f4b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -191,20 +191,32 @@ public void process() throws IOException {
 
       // Assign root and meta if we were carrying them.
       if (isCarryingRoot()) { // -ROOT-
-        LOG.info("Server " + serverName +
-            " was carrying ROOT. Trying to assign.");
-        this.services.getAssignmentManager().
-          regionOffline(HRegionInfo.ROOT_REGIONINFO);
-        verifyAndAssignRootWithRetries();
+        // Check again: region may be assigned to other where because of RIT
+        // timeout
+        if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
+          LOG.info("Server " + serverName
+              + " was carrying ROOT. Trying to assign.");
+          this.services.getAssignmentManager().regionOffline(
+              HRegionInfo.ROOT_REGIONINFO);
+          verifyAndAssignRootWithRetries();
+        } else {
+          LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
+        }
       }
 
       // Carrying meta?
       if (isCarryingMeta()) {
-        LOG.info("Server " + serverName +
-          " was carrying META. Trying to assign.");
-        this.services.getAssignmentManager().
-          regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
-        this.services.getAssignmentManager().assignMeta();
+        // Check again: region may be assigned to other where because of RIT
+        // timeout
+        if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
+          LOG.info("Server " + serverName
+              + " was carrying META. Trying to assign.");
+          this.services.getAssignmentManager().regionOffline(
+              HRegionInfo.FIRST_META_REGIONINFO);
+          this.services.getAssignmentManager().assignMeta();
+        } else {
+          LOG.info("META has been assigned to otherwhere, skip assigning.");
+        }
       }
 
       // We don't want worker thread in the MetaServerShutdownHandler

From efd78cd94f9ef0a0ebf82ec1dbca67d7082b9794 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 11 Jan 2013 19:54:08 +0000
Subject: [PATCH 0693/1540] HBASE-7441 Make ClusterManager in
 IntegrationTestingUtility pluggable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1432251 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HConstants.java |  5 ++++-
 .../hadoop/hbase/IntegrationTestingUtility.java       | 11 ++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 401fd8736825..ea4815c96ecb 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -72,7 +72,10 @@ public enum OperationStatusCode {
 
   /** Config for pluggable load balancers */
   public static final String HBASE_MASTER_LOADBALANCER_CLASS = "hbase.master.loadbalancer.class";
-
+  
+  /** Config for pluggable hbase cluster manager */
+  public static final String HBASE_CLUSTER_MANAGER_CLASS = "hbase.it.clustermanager.class";
+  
   /** Cluster is standalone or pseudo-distributed */
   public static final boolean CLUSTER_IS_LOCAL = false;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java
index d5a6feeaf124..45cbc01e7097 100644
--- a/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java
@@ -18,9 +18,10 @@
 
 package org.apache.hadoop.hbase;
 
-import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.IOException;
 
 /**
  * Facility for <strong>integration/system</strong> tests. This extends {@link HBaseTestingUtility}
@@ -122,7 +123,11 @@ private boolean isDistributedCluster() {
 
   private void createDistributedHBaseCluster() throws IOException {
     Configuration conf = getConfiguration();
-    ClusterManager clusterManager = new HBaseClusterManager();
+    Class<? extends ClusterManager> clusterManagerClass = conf.getClass(
+      HConstants.HBASE_CLUSTER_MANAGER_CLASS, HBaseClusterManager.class,
+      ClusterManager.class);
+    ClusterManager clusterManager = ReflectionUtils.newInstance(
+      clusterManagerClass, conf);
     setHBaseCluster(new DistributedHBaseCluster(conf, clusterManager));
     getHBaseAdmin();
   }

From 87c189ba6e452f5489be7c0e0a38b2f400f4f5e4 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 11 Jan 2013 20:51:29 +0000
Subject: [PATCH 0694/1540] HBASE-6824. Introduce ${hbase.local.dir} and save
 coprocessor jars there

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1432282 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/CoprocessorHost.java    | 21 +++++++++++--------
 .../hbase/master/MasterCoprocessorHost.java   |  1 +
 .../regionserver/RegionCoprocessorHost.java   |  1 +
 src/main/resources/hbase-default.xml          |  9 +++++++-
 .../hadoop/hbase/HBaseTestingUtility.java     |  4 ++++
 .../hbase/coprocessor/TestClassLoading.java   | 18 +++++++++-------
 6 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index bc2927d7fc71..0241e11be20f 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -68,6 +68,10 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
   public static final String WAL_COPROCESSOR_CONF_KEY =
     "hbase.coprocessor.wal.classes";
 
+  //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
+  private static final String COPROCESSOR_JARS_DIR = File.separator
+      + "coprocessor" + File.separator + "jars" + File.separator;
+
   private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
   /** Ordered set of loaded coprocessors with lock */
   protected SortedSet<E> coprocessors =
@@ -205,13 +209,13 @@ public E load(Path path, String className, int priority,
       if (!path.toString().endsWith(".jar")) {
         throw new IOException(path.toString() + ": not a jar file?");
       }
-      FileSystem fs = path.getFileSystem(HBaseConfiguration.create());
-      Path dst = new Path(System.getProperty("java.io.tmpdir") +
-          java.io.File.separator +"." + pathPrefix +
+      FileSystem fs = path.getFileSystem(this.conf);
+      File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
+      parentDir.mkdirs();
+      File dst = new File(parentDir, "." + pathPrefix +
           "." + className + "." + System.currentTimeMillis() + ".jar");
-      fs.copyToLocalFile(path, dst);
-      File tmpLocal = new File(dst.toString());
-      tmpLocal.deleteOnExit();
+      fs.copyToLocalFile(path, new Path(dst.toString()));
+      dst.deleteOnExit();
 
       // TODO: code weaving goes here
 
@@ -225,7 +229,7 @@ public E load(Path path, String className, int priority,
       // unsurprisingly wants URLs, not URIs; so we will use the deprecated
       // method which returns URLs for as long as it is available
       List<URL> paths = new ArrayList<URL>();
-      URL url = new File(dst.toString()).getCanonicalFile().toURL();
+      URL url = dst.getCanonicalFile().toURL();
       paths.add(url);
 
       JarFile jarFile = new JarFile(dst.toString());
@@ -233,8 +237,7 @@ public E load(Path path, String className, int priority,
       while (entries.hasMoreElements()) {
         JarEntry entry = entries.nextElement();
         if (entry.getName().matches("/lib/[^/]+\\.jar")) {
-          File file = new File(System.getProperty("java.io.tmpdir") +
-              java.io.File.separator +"." + pathPrefix +
+          File file = new File(parentDir, "." + pathPrefix +
               "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
           IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
           file.deleteOnExit();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 68ab56644f6c..5f21a9c606b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -62,6 +62,7 @@ public MasterServices getMasterServices() {
   private MasterServices masterServices;
 
   MasterCoprocessorHost(final MasterServices services, final Configuration conf) {
+    this.conf = conf;
     this.masterServices = services;
     loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 27e194f7c744..e9a4084bb63d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -135,6 +135,7 @@ public ConcurrentMap<String, Object> getSharedData() {
    */
   public RegionCoprocessorHost(final HRegion region,
       final RegionServerServices rsServices, final Configuration conf) {
+    this.conf = conf;
     this.rsServices = rsServices;
     this.region = region;
     this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode());
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index f8e9664a12ea..999ae5f1b7c5 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -49,13 +49,20 @@
   </property>
   <property>
     <name>hbase.tmp.dir</name>
-    <value>/tmp/hbase-${user.name}</value>
+    <value>${java.io.tmpdir}/hbase-${user.name}</value>
     <description>Temporary directory on the local filesystem.
     Change this setting to point to a location more permanent
     than '/tmp' (The '/tmp' directory is often cleared on
     machine restart).
     </description>
   </property>
+  <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local/</value>
+    <description>Directory on the local filesystem to be used 
+    as a local storage.
+    </description>
+  </property>
   <property>
     <name>hbase.master.info.port</name>
     <value>60010</value>
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 0279f06ab78f..dc0883bc86b1 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -315,6 +315,10 @@ private void setupDataTestDir() {
     createSubDirAndSystemProperty(
       "mapred.working.dir",
       testPath, "mapred-working-dir");
+
+    createSubDir(
+      "hbase.local.dir",
+      testPath, "hbase-local-dir");
   }
 
   private void createSubDir(String propertyName, Path parent, String subDirName){
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index b89ef183121c..ba03e068cde9 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -166,7 +166,7 @@ private File buildCoprocessorJar(String className) throws Exception {
     // the classpath is {hbaseSrc}/target/classes.
     String currentDir = new File(".").getAbsolutePath();
     String classpath =
-        currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" +
+        currentDir + File.separator + "target"+ File.separator + "classes" +
         System.getProperty("path.separator") +
         System.getProperty("surefire.test.class.path");
     options.add(classpath);
@@ -297,6 +297,10 @@ public void testClassLoadingFromHDFS() throws Exception {
     }
   }
 
+  private String getLocalPath(File file) {
+    return new Path(file.toURI()).toString();
+  }
+
   @Test
   // HBASE-3516: Test CP Class loading from local file system
   public void testClassLoadingFromLocalFS() throws Exception {
@@ -305,7 +309,7 @@ public void testClassLoadingFromLocalFS() throws Exception {
     // create a table that references the jar
     HTableDescriptor htd = new HTableDescriptor(cpName3);
     htd.addFamily(new HColumnDescriptor("test"));
-    htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" +
+    htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
       Coprocessor.PRIORITY_USER);
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     admin.createTable(htd);
@@ -331,7 +335,7 @@ public void testPrivateClassLoader() throws Exception {
     // create a table that references the jar
     HTableDescriptor htd = new HTableDescriptor(cpName4);
     htd.addFamily(new HColumnDescriptor("test"));
-    htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" +
+    htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
       Coprocessor.PRIORITY_USER);
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
     admin.createTable(htd);
@@ -369,9 +373,9 @@ public void testHBase3810() throws Exception {
     String cpKey2 = " Coprocessor$2 ";
     String cpKey3 = " coprocessor$03 ";
 
-    String cpValue1 = jarFile1.toString() + "|" + cpName1 + "|" +
+    String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
         Coprocessor.PRIORITY_USER;
-    String cpValue2 = jarFile2.toString() + " | " + cpName2 + " | ";
+    String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
     // load from default class loader
     String cpValue3 =
         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
@@ -386,13 +390,13 @@ public void testHBase3810() throws Exception {
     htd.setValue(cpKey3, cpValue3);
 
     // add 2 coprocessor by using new htd.addCoprocessor() api
-    htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()),
+    htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
         Coprocessor.PRIORITY_USER, null);
     Map<String, String> kvs = new HashMap<String, String>();
     kvs.put("k1", "v1");
     kvs.put("k2", "v2");
     kvs.put("k3", "v3");
-    htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()),
+    htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
         Coprocessor.PRIORITY_USER, kvs);
 
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();

From 6f93996629e9504305cbe5750f65f26ffc856f12 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 11 Jan 2013 22:22:18 +0000
Subject: [PATCH 0695/1540] HBASE-7540 Make znode dump to print a dump of
 replication znodes (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1432319 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 4de15b09737c..10d583ad0676 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1467,6 +1467,11 @@ public static String dump(ZooKeeperWatcher zkw) {
       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
         sb.append("\n ").append(child);
       }
+      try {
+        getReplicationZnodesDump(zkw, sb);
+      } catch (KeeperException ke) {
+        LOG.warn("Couldn't get the replication znode dump." + ke.getStackTrace());
+      }
       sb.append("\nQuorum Server Statistics:");
       String[] servers = zkw.getQuorum().split(",");
       for (String server : servers) {
@@ -1493,6 +1498,25 @@ public static String dump(ZooKeeperWatcher zkw) {
     return sb.toString();
   }
 
+  private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
+      throws KeeperException {
+    String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
+      "replication");
+    String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
+    if (ZKUtil.checkExists(zkw, replicationZnode) == -1)
+      return;
+    // do a ls -r on this znode
+    List<String> stack = new LinkedList<String>();
+    stack.add(replicationZnode);
+    do {
+      String znodeToProcess = stack.remove(stack.size() - 1);
+      sb.append("\n").append(znodeToProcess).append(": ")
+          .append(Bytes.toString(ZKUtil.getData(zkw, znodeToProcess)));
+      for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
+        stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
+      }
+    } while (stack.size() > 0);
+  }
   /**
    * Gets the statistics from the given server.
    *

From 736622d1a6aa2dbbf1cdf052460b099cc43d844b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 11 Jan 2013 22:25:44 +0000
Subject: [PATCH 0696/1540] HBASE-7502 TestScannerTimeout fails on snapshot
 branch (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1432320 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/TestScannerTimeout.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
index 01e2a542b3f8..d4de8970b47f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
@@ -84,7 +84,7 @@ public static void tearDownAfterClass() throws Exception {
    */
   @Before
   public void setUp() throws Exception {
-    TEST_UTIL.ensureSomeRegionServersAvailable(2);
+    TEST_UTIL.ensureSomeNonStoppedRegionServersAvailable(2);
   }
 
   /**

From 3421bff987a92e89c64daaa5079ab9ed4bddfa68 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 14 Jan 2013 05:31:39 +0000
Subject: [PATCH 0697/1540] HBASE-7550 Synchronization problem in
 AssignmentManager

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1432806 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/AssignmentManager.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index e0598a8a887a..e118f485b1de 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1161,7 +1161,9 @@ public void nodeDeleted(final String path) {
         } else if (rs.isSplitting()) {
           LOG.debug("Ephemeral node deleted.  Found in SPLITTING state. " + "Removing from RIT "
               + rs.getRegion());
-          this.regionsInTransition.remove(regionName);
+          synchronized(this.regionsInTransition) {
+            this.regionsInTransition.remove(regionName);
+          }
         } else {
           LOG.debug("The znode of region " + regionInfo.getRegionNameAsString()
               + " has been deleted.");

From 93359573f1f6cc6740ce1671cf441b9279119fbc Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 14 Jan 2013 06:05:15 +0000
Subject: [PATCH 0698/1540] HBASE-7468 TestSplitTransactionOnCluster hangs
 frequently

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1432808 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestSplitTransactionOnCluster.java        | 207 +++++++++---------
 1 file changed, 102 insertions(+), 105 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index b0e0c9d642d6..8a4c7ffbdf0b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -541,58 +541,55 @@ private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws Exc
     final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
     
     HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-    try {
-      // Create table then get the single region for our new table.
-      HTableDescriptor htd = new HTableDescriptor(tableName);
-      htd.addFamily(new HColumnDescriptor("cf"));
-      admin.createTable(htd);
+    // Create table then get the single region for our new table.
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor("cf"));
+    admin.createTable(htd);
 
-      List<HRegion> regions = null;
-      for (int i=0; i<100; i++) {
-        regions = cluster.getRegions(tableName);
-        if (regions.size() > 0) break;
-        Thread.sleep(100);
-      }
-      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
-      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
-      SplitTransaction st = null;
+    List<HRegion> regions = null;
+    for (int i=0; i<100; i++) {
+      regions = cluster.getRegions(tableName);
+      if (regions.size() > 0) break;
+      Thread.sleep(100);
+    }
+    int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+    SplitTransaction st = null;
+    if (nodeCreated) {
+      st = new MockedSplitTransaction(regions.get(0), null) {
+        @Override
+        int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent,
+            ServerName serverName, int version) throws KeeperException, IOException {
+          throw new IOException();
+        }
+      };
+    } else {
+      st = new MockedSplitTransaction(regions.get(0), null) {
+        @Override
+        void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
+            throws KeeperException, IOException {
+          throw new IOException();
+        }
+      };
+    }
+    try {
+      st.execute(regionServer, regionServer);
+    } catch (IOException e) {
+      String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
+          .getRegionInfo().getEncodedName());
+      // make sure the client is uptodate
+      regionServer.getZooKeeper().sync(node);
       if (nodeCreated) {
-        st = new MockedSplitTransaction(regions.get(0), null) {
-          @Override
-          int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent,
-              ServerName serverName, int version) throws KeeperException, IOException {
-            throw new IOException();
-          }
-        };
+        assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
       } else {
-        st = new MockedSplitTransaction(regions.get(0), null) {
-          @Override
-          void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
-              throws KeeperException, IOException {
-            throw new IOException();
-          }
-        };
-      }
-      try {
-        st.execute(regionServer, regionServer);
-      } catch (IOException e) {
-        String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
-            .getRegionInfo().getEncodedName());
-        // make sure the client is uptodate
-        regionServer.getZooKeeper().sync(node);
-        if (nodeCreated) {
-          assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-        } else {
-          assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-        }
-        assertTrue(st.rollback(regionServer, regionServer));
         assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
       }
-    } finally {
-      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
-        admin.disableTable(tableName);
-        admin.deleteTable(tableName);
-      }
+      assertTrue(st.rollback(regionServer, regionServer));
+      assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+    }
+    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
     }
   }
   
@@ -600,57 +597,54 @@ void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName se
   public void testShouldClearRITWhenNodeFoundInSplittingState() throws Exception {
     final byte[] tableName = Bytes.toBytes("testShouldClearRITWhenNodeFoundInSplittingState");
     HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-    try {
-      // Create table then get the single region for our new table.
-      HTableDescriptor htd = new HTableDescriptor(tableName);
-      htd.addFamily(new HColumnDescriptor("cf"));
-      admin.createTable(htd);
-
-      List<HRegion> regions = cluster.getRegions(tableName);
-      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
-      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
-      SplitTransaction st = null;
-
-      st = new MockedSplitTransaction(regions.get(0), null) {
-        @Override
-        void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
-          throw new IOException("");
-        }
-      };
-
-      try {
-        st.execute(regionServer, regionServer);
-      } catch (IOException e) {
-        String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
-            .getRegionInfo().getEncodedName());
+    // Create table then get the single region for our new table.
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor("cf"));
+    admin.createTable(htd);
+
+    HRegion region = cluster.getRegions(tableName).get(0);
+    int regionServerIndex = cluster.getServerWith(region.getRegionName());
+    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+    SplitTransaction st = null;
+
+    st = new MockedSplitTransaction(region, null) {
+      @Override
+      void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
+        throw new IOException("");
+      }
+    };
 
-        assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-        AssignmentManager am = cluster.getMaster().getAssignmentManager();
-        for (int i = 0; !am.getRegionsInTransition().containsKey(
-            regions.get(0).getRegionInfo().getEncodedName())
-            && i < 10; i++) {
-          Thread.sleep(200);
-        }
-        assertTrue("region is not in transition",
-            am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName()));
-        RegionState regionState = am.getRegionsInTransition().get(regions.get(0).getRegionInfo()
-            .getEncodedName());
-        assertTrue(regionState.getState() == RegionState.State.SPLITTING);
-        assertTrue(st.rollback(regionServer, regionServer));
-        assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-        for (int i=0; am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName()) && i<10; i++) {
-          // Just in case the nodeDeleted event did not get executed.
-          Thread.sleep(200);
-        }
-        assertFalse("region is still in transition",
-            am.getRegionsInTransition().containsKey(regions.get(0).getRegionInfo().getEncodedName()));
+    try {
+      st.execute(regionServer, regionServer);
+    } catch (IOException e) {
+      String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), region
+          .getRegionInfo().getEncodedName());
+
+      assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+      AssignmentManager am = cluster.getMaster().getAssignmentManager();
+      for (int i = 0; !am.getRegionsInTransition().containsKey(
+          region.getRegionInfo().getEncodedName())
+          && i < 100; i++) {
+        Thread.sleep(200);
       }
-    } finally {
-      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
-        admin.disableTable(tableName);
-        admin.deleteTable(tableName);
-        admin.close();
+      assertTrue("region is not in transition "+region,
+          am.getRegionsInTransition().containsKey(region.getRegionInfo().getEncodedName()));
+      RegionState regionState = am.getRegionsInTransition().get(region.getRegionInfo()
+          .getEncodedName());
+      assertTrue(regionState.getState() == RegionState.State.SPLITTING);
+      assertTrue(st.rollback(regionServer, regionServer));
+      assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
+      for (int i=0; am.getRegionsInTransition().containsKey(region.getRegionInfo().getEncodedName()) && i<100; i++) {
+        // Just in case the nodeDeleted event did not get executed.
+        Thread.sleep(200);
       }
+      assertFalse("region is still in transition",
+          am.getRegionsInTransition().containsKey(region.getRegionInfo().getEncodedName()));
+    }
+    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+      admin.close();
     }
   }
   
@@ -730,11 +724,11 @@ public void run() {
       firstSplitCompleted = false;
       callRollBack = false;
       cluster.getMaster().setCatalogJanitorEnabled(true);
-      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
-        admin.disableTable(tableName);
-        admin.deleteTable(tableName);
-        admin.close();
-      }
+    }
+    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+      admin.close();
     }
   }
 
@@ -831,16 +825,19 @@ public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfull
             .getRegionInfo().getEncodedName());
         assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
         assertTrue(st.rollback(regionServer, regionServer));
+        for(int i=0; ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1 && i<100; i++) {
+          Thread.sleep(100);
+        }
         assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
       }
     } finally {
       admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
-      if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
-        admin.disableTable(tableName);
-        admin.deleteTable(tableName);
-        admin.close();
-      }
+    }
+    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+      admin.close();
     }
   }
   public static class MockedSplitTransaction extends SplitTransaction {

From 450d5027fb7742304b97f1a2e6771ed493eea685 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 14 Jan 2013 23:02:08 +0000
Subject: [PATCH 0699/1540] HBASE-5416 Improve performance of scans with some
 kind of filters. (Sergey Shelukhin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433195 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     |  13 ++
 .../org/apache/hadoop/hbase/client/Scan.java  |  31 ++-
 .../apache/hadoop/hbase/filter/Filter.java    |   8 +
 .../hadoop/hbase/filter/FilterBase.java       |  10 +
 .../hadoop/hbase/filter/FilterList.java       |  10 +
 .../SingleColumnValueExcludeFilter.java       |  20 +-
 .../hbase/filter/SingleColumnValueFilter.java |   9 +
 .../hadoop/hbase/filter/SkipFilter.java       |   4 +
 .../hadoop/hbase/filter/WhileMatchFilter.java |   4 +
 .../hadoop/hbase/regionserver/HRegion.java    | 192 +++++++++++++++---
 .../hbase/regionserver/HRegionServer.java     |   2 +
 .../TestSingleColumnValueExcludeFilter.java   |  25 ++-
 .../hbase/regionserver/TestHRegion.java       | 173 +++++++++++++++-
 13 files changed, 451 insertions(+), 50 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index c9c962ec9d0f..39d1f095b7f1 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -1750,6 +1750,19 @@ public static KeyValue createFirstOnRow(final byte [] row) {
     return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
   }
 
+  /**
+   * Create a KeyValue that is smaller than all other possible KeyValues
+   * for the given row. That is any (valid) KeyValue on 'row' would sort
+   * _after_ the result.
+   *
+   * @param row - row key (arbitrary byte array)
+   * @return First possible KeyValue on passed <code>row</code>
+   */
+  public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
+    return new KeyValue(row, roffset, rlength,
+        null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
+  }
+
   /**
    * Creates a KeyValue that is smaller than all other KeyValues that
    * are older than the passed timestamp.
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 024059b2f25b..458f6b807912 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -82,6 +82,7 @@
  */
 public class Scan extends OperationWithAttributes implements Writable {
   private static final String RAW_ATTR = "_raw_";
+  private static final String ONDEMAND_ATTR = "_ondemand_";
   private static final String ISOLATION_LEVEL = "_isolationlevel_";
 
   private static final byte SCAN_VERSION = (byte)2;
@@ -459,6 +460,34 @@ public boolean getCacheBlocks() {
     return cacheBlocks;
   }
 
+  /**
+   * Set the value indicating whether loading CFs on demand should be allowed (cluster
+   * default is false). On-demand CF loading doesn't load column families until necessary, e.g.
+   * if you filter on one column, the other column family data will be loaded only for the rows
+   * that are included in result, not all rows like in normal case.
+   * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true,
+   * this can deliver huge perf gains when there's a cf with lots of data; however, it can
+   * also lead to some inconsistent results, as follows:
+   * - if someone does a concurrent update to both column families in question you may get a row
+   *   that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } }
+   *   someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan
+   *   filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 },
+   *   { video => "my dog" } }.
+   * - if there's a concurrent split and you have more than 2 column families, some rows may be
+   *   missing some column families.
+   */
+  public void setLoadColumnFamiliesOnDemand(boolean value) {
+    setAttribute(ONDEMAND_ATTR, Bytes.toBytes(value));
+  }
+
+  /**
+   * Get the logical value indicating whether on-demand CF loading should be allowed.
+   */
+  public boolean doLoadColumnFamiliesOnDemand() {
+    byte[] attr = getAttribute(ONDEMAND_ATTR);
+    return attr == null ? false : Bytes.toBoolean(attr);
+  }
+
   /**
    * Compile the table and column family (i.e. schema) information
    * into a String. Useful for parsing and aggregation by debugging,
@@ -488,7 +517,7 @@ public Map<String, Object> getFingerprint() {
    * Useful for debugging, logging, and administration tools.
    * @param maxCols a limit on the number of columns output prior to truncation
    * @return Map
-   */ 
+   */
   @Override
   public Map<String, Object> toMap(int maxCols) {
     // start with the fingerpring map and build on top of it
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
index ac1079f82bf4..eb8aac1cf3c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
@@ -167,4 +167,12 @@ public enum ReturnCode {
    * not sure which key to seek to next.
    */
   public KeyValue getNextKeyHint(final KeyValue currentKV);
+
+  /**
+   * Check that given column family is essential for filter to check row.  Most
+   * filters always return true here. But some could have more sophisticated
+   * logic which could significantly reduce scanning process by not even
+   * touching columns until we are 100% sure that it's data is needed in result.
+   */
+  public boolean isFamilyEssential(byte[] name);
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
index 48e26c5ebab7..271116448107 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
@@ -130,6 +130,16 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
     return null;
   }
 
+  /**
+   * By default, we require all scan's column families to be present. Our
+   * subclasses may be more precise.
+   *
+   * @inheritDoc
+   */
+  public boolean isFamilyEssential(byte[] name) {
+    return true;
+  }
+
   /**
    * Given the filter's arguments it constructs the filter
    * <p>
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 8e12d3933ea2..9521c7444a74 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -317,6 +317,16 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
     return keyHint;
   }
 
+  @Override
+  public boolean isFamilyEssential(byte[] name) {
+    for (Filter filter : filters) {
+      if (filter.isFamilyEssential(name)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   @Override
   public String toString() {
     return toString(MAX_LOG_FILTERS);
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java
index 7c7607f13da2..638629cea638 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java
@@ -24,6 +24,8 @@
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 
 import java.util.ArrayList;
+import java.util.List;
+import java.util.Iterator;
 
 /**
  * A {@link Filter} that checks a single column value, but does not emit the
@@ -76,16 +78,22 @@ public SingleColumnValueExcludeFilter(byte[] family, byte[] qualifier,
     super(family, qualifier, compareOp, comparator);
   }
 
-  public ReturnCode filterKeyValue(KeyValue keyValue) {
-    ReturnCode superRetCode = super.filterKeyValue(keyValue);
-    if (superRetCode == ReturnCode.INCLUDE) {
+  // We cleaned result row in FilterRow to be consistent with scanning process.
+  public boolean hasFilterRow() {
+   return true;
+  }
+
+  // Here we remove from row all key values from testing column
+  public void filterRow(List<KeyValue> kvs) {
+    Iterator it = kvs.iterator();
+    while (it.hasNext()) {
+      KeyValue kv = (KeyValue)it.next();
       // If the current column is actually the tested column,
       // we will skip it instead.
-      if (keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) {
-        return ReturnCode.SKIP;
+      if (kv.matchingColumn(this.columnFamily, this.columnQualifier)) {
+        it.remove();
       }
     }
-    return superRetCode;
   }
 
   public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments) {
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
index f6e0cb7ed13f..4d2d504489b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
@@ -307,6 +307,15 @@ public void write(final DataOutput out) throws IOException {
     out.writeBoolean(latestVersionOnly);
   }
 
+  /**
+   * The only CF this filter needs is given column family. So, it's the only essential
+   * column in whole scan. If filterIfMissing == false, all families are essential,
+   * because of possibility of skipping the rows without any data in filtered CF.
+   */
+  public boolean isFamilyEssential(byte[] name) {
+    return !this.filterIfMissing || Bytes.equals(name, this.columnFamily);
+  }
+
   @Override
   public String toString() {
     return String.format("%s (%s, %s, %s, %s)",
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
index 57fa99157944..8894a116acbf 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
@@ -104,6 +104,10 @@ public void readFields(DataInput in) throws IOException {
     }
   }
 
+  public boolean isFamilyEssential(byte[] name) {
+    return filter.isFamilyEssential(name);
+  }
+
   @Override
   public String toString() {
     return this.getClass().getSimpleName() + " " + this.filter.toString();
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
index 242e0bdb7215..e822ef4b6acb 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
@@ -105,6 +105,10 @@ public void readFields(DataInput in) throws IOException {
     }
   }
 
+  public boolean isFamilyEssential(byte[] name) {
+    return filter.isFamilyEssential(name);
+  }
+
   @Override
   public String toString() {
     return this.getClass().getSimpleName() + " " + this.filter.toString();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 757ab2c70b7d..c19ea0a4de89 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -178,6 +178,8 @@ public class HRegion implements HeapSize { // , Writable{
   public static final Log LOG = LogFactory.getLog(HRegion.class);
   private static final String MERGEDIR = ".merges";
 
+  public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY = "hbase.hregion.scan.loadColumnFamiliesOnDemand";
+
   final AtomicBoolean closed = new AtomicBoolean(false);
   /* Closing can take some time; use the closing flag if there is stuff we don't
    * want to do while in closing state; e.g. like offer this region up to the
@@ -263,8 +265,13 @@ public class HRegion implements HeapSize { // , Writable{
   KeyValue.KVComparator comparator;
 
   private ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;
+  /**
+   * The default setting for whether to enable on-demand CF loading for
+   * scan requests to this region. Requests can override it.
+   */
+  private boolean isLoadingCfsOnDemandDefault = false;
 
-  /*
+  /**
    * @return The smallest mvcc readPoint across all the scanners in this
    * region. Writes older than this readPoint, are included  in every
    * read operation.
@@ -416,6 +423,8 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
     this.conf = conf;
     this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
                     DEFAULT_ROWLOCK_WAIT_DURATION);
+
+    this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, false);
     this.regionInfo = regionInfo;
     this.htableDescriptor = htd;
     this.rsServices = rsServices;
@@ -857,6 +866,10 @@ public MultiVersionConsistencyControl getMVCC() {
      return mvcc;
    }
 
+   public boolean isLoadingCfsOnDemandDefault() {
+     return this.isLoadingCfsOnDemandDefault;
+   }
+
   /**
    * Close down this HRegion.  Flush the cache, shut down each HStore, don't
    * service any more calls.
@@ -3467,6 +3480,15 @@ public Path getTableDir() {
   class RegionScannerImpl implements RegionScanner {
     // Package local for testability
     KeyValueHeap storeHeap = null;
+    /** Heap of key-values that are not essential for the provided filters and are thus read
+     * on demand, if on-demand column family loading is enabled.*/
+    KeyValueHeap joinedHeap = null;
+    /**
+     * If the joined heap data gathering is interrupted due to scan limits, this will
+     * contain the row for which we are populating the values.*/
+    private KeyValue joinedContinuationRow = null;
+    // KeyValue indicating that limit is reached when scanning
+    private final KeyValue KV_LIMIT = new KeyValue();
     private final byte [] stopRow;
     private Filter filter;
     private List<KeyValue> results = new ArrayList<KeyValue>();
@@ -3506,7 +3528,10 @@ public HRegionInfo getRegionInfo() {
         scannerReadPoints.put(this, this.readPt);
       }
 
+      // Here we separate all scanners into two lists - scanner that provide data required
+      // by the filter to operate (scanners list) and all others (joinedScanners list).
       List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
+      List<KeyValueScanner> joinedScanners = new ArrayList<KeyValueScanner>();
       if (additionalScanners != null) {
         scanners.addAll(additionalScanners);
       }
@@ -3515,9 +3540,17 @@ public HRegionInfo getRegionInfo() {
           scan.getFamilyMap().entrySet()) {
         Store store = stores.get(entry.getKey());
         KeyValueScanner scanner = store.getScanner(scan, entry.getValue());
-        scanners.add(scanner);
+        if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand()
+          || this.filter.isFamilyEssential(entry.getKey())) {
+          scanners.add(scanner);
+        } else {
+          joinedScanners.add(scanner);
+        }
       }
       this.storeHeap = new KeyValueHeap(scanners, comparator);
+      if (!joinedScanners.isEmpty()) {
+        this.joinedHeap = new KeyValueHeap(joinedScanners, comparator);
+      }
     }
 
     RegionScannerImpl(Scan scan) throws IOException {
@@ -3599,6 +3632,42 @@ public boolean next(List<KeyValue> outResults, String metric)
       return next(outResults, batch, metric);
     }
 
+    private void populateFromJoinedHeap(int limit, String metric) throws IOException {
+      assert joinedContinuationRow != null;
+      KeyValue kv = populateResult(this.joinedHeap, limit, joinedContinuationRow.getBuffer(),
+        joinedContinuationRow.getRowOffset(), joinedContinuationRow.getRowLength(), metric);
+      if (kv != KV_LIMIT) {
+        // We are done with this row, reset the continuation.
+        joinedContinuationRow = null;
+      }
+      // As the data is obtained from two independent heaps, we need to
+      // ensure that result list is sorted, because Result relies on that.
+      Collections.sort(results, comparator);
+    }
+
+    /**
+     * Fetches records with this row into result list, until next row or limit (if not -1).
+     * @param heap KeyValueHeap to fetch data from. It must be positioned on correct row before call.
+     * @param limit Max amount of KVs to place in result list, -1 means no limit.
+     * @param currentRow Byte array with key we are fetching.
+     * @param offset offset for currentRow
+     * @param length length for currentRow
+     * @param metric Metric key to be passed into KeyValueHeap::next().
+     * @return true if limit reached, false otherwise.
+     */
+    private KeyValue populateResult(KeyValueHeap heap, int limit, byte[] currentRow, int offset,
+        short length, String metric) throws IOException {
+      KeyValue nextKv;
+      do {
+        heap.next(results, limit - results.size(), metric);
+        if (limit > 0 && results.size() == limit) {
+          return KV_LIMIT;
+        }
+        nextKv = heap.peek();
+      } while (nextKv != null && nextKv.matchingRow(currentRow, offset, length));
+      return nextKv;
+    }
+
     /*
      * @return True if a filter rules the scanner is over, done.
      */
@@ -3608,6 +3677,11 @@ public synchronized boolean isFilterDone() {
 
     private boolean nextInternal(int limit, String metric) throws IOException {
       RpcCallContext rpcCall = HBaseServer.getCurrentCall();
+      // The loop here is used only when at some point during the next we determine
+      // that due to effects of filters or otherwise, we have an empty row in the result.
+      // Then we loop and try again. Otherwise, we must get out on the first iteration via return,
+      // "true" if there's more data to read, "false" if there isn't (storeHeap is at a stop row,
+      // and joinedHeap has no more data to read for the last row (if set, joinedContinuationRow).
       while (true) {
         if (rpcCall != null) {
           // If a user specifies a too-restrictive or too-slow scanner, the
@@ -3617,7 +3691,9 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           rpcCall.throwExceptionIfCallerDisconnected();
         }
 
+        // Let's see what we have in the storeHeap.
         KeyValue current = this.storeHeap.peek();
+
         byte[] currentRow = null;
         int offset = 0;
         short length = 0;
@@ -3626,41 +3702,48 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           offset = current.getRowOffset();
           length = current.getRowLength();
         }
-        if (isStopRow(currentRow, offset, length)) {
-          if (filter != null && filter.hasFilterRow()) {
-            filter.filterRow(results);
+        boolean stopRow = isStopRow(currentRow, offset, length);
+        // Check if we were getting data from the joinedHeap abd hit the limit.
+        // If not, then it's main path - getting results from storeHeap.
+        if (joinedContinuationRow == null) {
+          // First, check if we are at a stop row. If so, there are no more results.
+          if (stopRow) {
+            if (filter != null && filter.hasFilterRow()) {
+              filter.filterRow(results);
+            }
+            if (filter != null && filter.filterRow()) {
+              results.clear();
+            }
+            return false;
           }
-          if (filter != null && filter.filterRow()) {
-            results.clear();
+
+          // Check if rowkey filter wants to exclude this row. If so, loop to next.
+          // Techically, if we hit limits before on this row, we don't need this call.
+          if (filterRowKey(currentRow, offset, length)) {
+            nextRow(currentRow, offset, length);
+            continue;
           }
 
-          return false;
-        } else if (filterRowKey(currentRow, offset, length)) {
-          nextRow(currentRow, offset, length);
-        } else {
-          KeyValue nextKv;
-          do {
-            this.storeHeap.next(results, limit - results.size(), metric);
-            if (limit > 0 && results.size() == limit) {
-              if (this.filter != null && filter.hasFilterRow()) {
-                throw new IncompatibleFilterException(
-                  "Filter with filterRow(List<KeyValue>) incompatible with scan with limit!");
-              }
-              return true; // we are expecting more yes, but also limited to how many we can return.
+          // Ok, we are good, let's try to get some results from the main heap.
+          KeyValue nextKv = populateResult(this.storeHeap, limit, currentRow, offset, length, metric);
+          if (nextKv == KV_LIMIT) {
+            if (this.filter != null && filter.hasFilterRow()) {
+              throw new IncompatibleFilterException(
+                "Filter whose hasFilterRow() returns true is incompatible with scan with limit!");
             }
-            nextKv = this.storeHeap.peek();
-          } while (nextKv != null && nextKv.matchingRow(currentRow, offset, length));
-
-          final boolean stopRow = nextKv == null || isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
-
-          // now that we have an entire row, lets process with a filters:
+            return true; // We hit the limit.
+          }
+          stopRow = nextKv == null || isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
+          // save that the row was empty before filters applied to it.
+          final boolean isEmptyRow = results.isEmpty();
 
-          // first filter with the filterRow(List)
+          // We have the part of the row necessary for filtering (all of it, usually).
+          // First filter with the filterRow(List).            
           if (filter != null && filter.hasFilterRow()) {
             filter.filterRow(results);
           }
 
-          if (results.isEmpty() || filterRow()) {
+          if (isEmptyRow || filterRow()) {
             // this seems like a redundant step - we already consumed the row
             // there're no left overs.
             // the reasons for calling this method are:
@@ -3669,12 +3752,48 @@ private boolean nextInternal(int limit, String metric) throws IOException {
             nextRow(currentRow, offset, length);
 
             // This row was totally filtered out, if this is NOT the last row,
-            // we should continue on.
-
+            // we should continue on. Otherwise, nothing else to do.
             if (!stopRow) continue;
+            return false;
+          }
+
+          // Ok, we are done with storeHeap for this row.
+          // Now we may need to fetch additional, non-essential data into row.
+          // These values are not needed for filter to work, so we postpone their
+          // fetch to (possibly) reduce amount of data loads from disk.
+          if (this.joinedHeap != null) {
+            KeyValue nextJoinedKv = joinedHeap.peek();
+            // If joinedHeap is pointing to some other row, try to seek to a correct one.
+            // We don't need to recheck that row here - populateResult will take care of that.
+            boolean mayHaveData =
+              (nextJoinedKv != null && nextJoinedKv.matchingRow(currentRow, offset, length))
+              || this.joinedHeap.seek(KeyValue.createFirstOnRow(currentRow, offset, length));
+            if (mayHaveData) {
+              joinedContinuationRow = current;
+              populateFromJoinedHeap(limit, metric);
+            }
           }
-          return !stopRow;
+        } else {
+          // Populating from the joined map was stopped by limits, populate some more.
+          populateFromJoinedHeap(limit, metric);
         }
+
+        // We may have just called populateFromJoinedMap and hit the limits. If that is
+        // the case, we need to call it again on the next next() invocation.
+        if (joinedContinuationRow != null) {
+          return true;
+        }
+
+        // Finally, we are done with both joinedHeap and storeHeap.
+        // Double check to prevent empty rows from appearing in result. It could be
+        // the case when SingleValueExcludeFilter is used.
+        if (results.isEmpty()) {
+          nextRow(currentRow, offset, length);
+          if (!stopRow) continue;
+        }
+
+        // We are done. Return the result.
+        return !stopRow;
       }
     }
 
@@ -3709,6 +3828,10 @@ public synchronized void close() {
         storeHeap.close();
         storeHeap = null;
       }
+      if (joinedHeap != null) {
+        joinedHeap.close();
+        joinedHeap = null;
+      }
       // no need to sychronize here.
       scannerReadPoints.remove(this);
       this.filterClosed = true;
@@ -3723,16 +3846,21 @@ public synchronized boolean reseek(byte[] row) throws IOException {
       if (row == null) {
         throw new IllegalArgumentException("Row cannot be null.");
       }
+      boolean result = false;
       startRegionOperation();
       try {
         // This could be a new thread from the last time we called next().
         MultiVersionConsistencyControl.setThreadReadPoint(this.readPt);
         KeyValue kv = KeyValue.createFirstOnRow(row);
         // use request seek to make use of the lazy seek option. See HBASE-5520
-        return this.storeHeap.requestSeek(kv, true, true);
+        result = this.storeHeap.requestSeek(kv, true, true);
+        if (this.joinedHeap != null) {
+          result = this.joinedHeap.requestSeek(kv, true, true) || result;
+        }
       } finally {
         closeRegionOperation();
       }
+      return result;
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index d1a33d2bad56..e6be923edd60 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2382,6 +2382,8 @@ public long openScanner(byte[] regionName, Scan scan) throws IOException {
     try {
       HRegion r = getRegion(regionName);
       r.checkRow(scan.getStartRow(), "Scan");
+      scan.setLoadColumnFamiliesOnDemand(r.isLoadingCfsOnDemandDefault()
+          || scan.doLoadColumnFamiliesOnDemand());
       r.prepareScanner(scan);
       RegionScanner s = null;
       if (r.getCoprocessorHost() != null) {
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
index b7cb6b5211b7..31f5db453321 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
@@ -27,6 +27,9 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
 
+import java.util.List;
+import java.util.ArrayList;
+
 /**
  * Tests for {@link SingleColumnValueExcludeFilter}. Because this filter
  * extends {@link SingleColumnValueFilter}, only the added functionality is
@@ -53,16 +56,18 @@ public void testFilterKeyValue() throws Exception {
         CompareOp.EQUAL, VAL_1);
 
     // A 'match' situation
-    KeyValue kv;
-    kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1);
-    // INCLUDE expected because test column has not yet passed
-    assertTrue("otherColumn", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
-    kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1);
-    // Test column will pass (will match), will SKIP because test columns are excluded
-    assertTrue("testedMatch", filter.filterKeyValue(kv) == Filter.ReturnCode.SKIP);
-    // Test column has already passed and matched, all subsequent columns are INCLUDE
-    kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1);
-    assertTrue("otherColumn", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
+    List<KeyValue> kvs = new ArrayList<KeyValue>();
+    KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1);
+
+    kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1));
+    kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1));
+    kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1));
+
+    filter.filterRow(kvs);
+
+    assertEquals("resultSize", kvs.size(), 2);
+    assertTrue("leftKV1", KeyValue.COMPARATOR.compare(kvs.get(0), kv) == 0);
+    assertTrue("leftKV2", KeyValue.COMPARATOR.compare(kvs.get(1), kv) == 0);
     assertFalse("allRemainingWhenMatch", filter.filterAllRemaining());
 
     // A 'mismatch' situation
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 392ae58a3ea9..9b3141f87861 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -20,6 +20,8 @@
 package org.apache.hadoop.hbase.regionserver;
 
 
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
@@ -66,9 +68,11 @@
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.NullComparator;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -200,7 +204,7 @@ public void testCompactionAffectedByScanners() throws Exception {
     System.out.println(results);
     assertEquals(0, results.size());
   }
-  
+
   @Test
   public void testToShowNPEOnRegionScannerReseek() throws Exception{
     String method = "testToShowNPEOnRegionScannerReseek";
@@ -2816,6 +2820,173 @@ public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
     }
   }
 
+  /**
+   * Added for HBASE-5416
+   *
+   * Here we test scan optimization when only subset of CFs are used in filter
+   * conditions.
+   */
+  public void testScanner_JoinedScanners() throws IOException {
+    byte [] tableName = Bytes.toBytes("testTable");
+    byte [] cf_essential = Bytes.toBytes("essential");
+    byte [] cf_joined = Bytes.toBytes("joined");
+    byte [] cf_alpha = Bytes.toBytes("alpha");
+    this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
+    try {
+      byte [] row1 = Bytes.toBytes("row1");
+      byte [] row2 = Bytes.toBytes("row2");
+      byte [] row3 = Bytes.toBytes("row3");
+
+      byte [] col_normal = Bytes.toBytes("d");
+      byte [] col_alpha = Bytes.toBytes("a");
+
+      byte [] filtered_val = Bytes.toBytes(3);
+
+      Put put = new Put(row1);
+      put.add(cf_essential, col_normal, Bytes.toBytes(1));
+      put.add(cf_joined, col_alpha, Bytes.toBytes(1));
+      region.put(put);
+
+      put = new Put(row2);
+      put.add(cf_essential, col_alpha, Bytes.toBytes(2));
+      put.add(cf_joined, col_normal, Bytes.toBytes(2));
+      put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
+      region.put(put);
+
+      put = new Put(row3);
+      put.add(cf_essential, col_normal, filtered_val);
+      put.add(cf_joined, col_normal, filtered_val);
+      region.put(put);
+
+      // Check two things:
+      // 1. result list contains expected values
+      // 2. result list is sorted properly
+
+      Scan scan = new Scan();
+      Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
+                                                         CompareOp.NOT_EQUAL, filtered_val);
+      scan.setFilter(filter);
+      scan.setLoadColumnFamiliesOnDemand(true);
+      InternalScanner s = region.getScanner(scan);
+
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      assertTrue(s.next(results));
+      assertEquals(results.size(), 1);
+      results.clear();
+
+      assertTrue(s.next(results));
+      assertEquals(results.size(), 3);
+      assertTrue("orderCheck", results.get(0).matchingFamily(cf_alpha));
+      assertTrue("orderCheck", results.get(1).matchingFamily(cf_essential));
+      assertTrue("orderCheck", results.get(2).matchingFamily(cf_joined));
+      results.clear();
+
+      assertFalse(s.next(results));
+      assertEquals(results.size(), 0);
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+  }
+
+  /**
+   * HBASE-5416
+   *
+   * Test case when scan limits amount of KVs returned on each next() call.
+   */
+  public void testScanner_JoinedScannersWithLimits() throws IOException {
+    final byte [] tableName = Bytes.toBytes("testTable");
+    final byte [] cf_first = Bytes.toBytes("first");
+    final byte [] cf_second = Bytes.toBytes("second");
+
+    this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
+    try {
+      final byte [] col_a = Bytes.toBytes("a");
+      final byte [] col_b = Bytes.toBytes("b");
+
+      Put put;
+
+      for (int i = 0; i < 10; i++) {
+        put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
+        put.add(cf_first, col_a, Bytes.toBytes(i));
+        if (i < 5) {
+          put.add(cf_first, col_b, Bytes.toBytes(i));
+          put.add(cf_second, col_a, Bytes.toBytes(i));
+          put.add(cf_second, col_b, Bytes.toBytes(i));
+        }
+        region.put(put);
+      }
+
+      Scan scan = new Scan();
+      scan.setLoadColumnFamiliesOnDemand(true);
+      Filter bogusFilter = new FilterBase() {
+        @Override
+        public boolean isFamilyEssential(byte[] name) {
+          return Bytes.equals(name, cf_first);
+        }
+        @Override
+        public void readFields(DataInput arg0) throws IOException {
+        }
+
+        @Override
+        public void write(DataOutput arg0) throws IOException {
+        }
+      };
+
+      scan.setFilter(bogusFilter);
+      InternalScanner s = region.getScanner(scan);
+
+      // Our data looks like this:
+      // r0: first:a, first:b, second:a, second:b
+      // r1: first:a, first:b, second:a, second:b
+      // r2: first:a, first:b, second:a, second:b
+      // r3: first:a, first:b, second:a, second:b
+      // r4: first:a, first:b, second:a, second:b
+      // r5: first:a
+      // r6: first:a
+      // r7: first:a
+      // r8: first:a
+      // r9: first:a
+
+      // But due to next's limit set to 3, we should get this:
+      // r0: first:a, first:b, second:a
+      // r0: second:b
+      // r1: first:a, first:b, second:a
+      // r1: second:b
+      // r2: first:a, first:b, second:a
+      // r2: second:b
+      // r3: first:a, first:b, second:a
+      // r3: second:b
+      // r4: first:a, first:b, second:a
+      // r4: second:b
+      // r5: first:a
+      // r6: first:a
+      // r7: first:a
+      // r8: first:a
+      // r9: first:a
+
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      int index = 0;
+      while (true) {
+        boolean more = s.next(results, 3);
+        if ((index >> 1) < 5) {
+          if (index % 2 == 0)
+            assertEquals(results.size(), 3);
+          else
+            assertEquals(results.size(), 1);
+        }
+        else
+          assertEquals(results.size(), 1);
+        results.clear();
+        index++;
+        if (!more) break;
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   // Split test
   //////////////////////////////////////////////////////////////////////////////

From 2768d0c731de10fbd6ed983226d6edc9cc954437 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Jan 2013 03:21:54 +0000
Subject: [PATCH 0700/1540] HBASE-7562 ZKUtil: missing "else condition" in
 multi processing (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433274 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 10d583ad0676..83fc5cff970c 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1428,19 +1428,19 @@ public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
       } catch (InterruptedException ie) {
         zkw.interruptedException(ie);
       }
-    }
-
-    // run sequentially
-    for (ZKUtilOp op : ops) {
-      if (op instanceof CreateAndFailSilent) {
-        createAndFailSilent(zkw, (CreateAndFailSilent)op);
-      } else if (op instanceof DeleteNodeFailSilent) {
-        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent)op);
-      } else if (op instanceof SetData) {
-        setData(zkw, (SetData)op);
-      } else {
-        throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
-          + op.getClass().getName());
+    } else {
+      // run sequentially
+      for (ZKUtilOp op : ops) {
+        if (op instanceof CreateAndFailSilent) {
+          createAndFailSilent(zkw, (CreateAndFailSilent) op);
+        } else if (op instanceof DeleteNodeFailSilent) {
+          deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
+        } else if (op instanceof SetData) {
+          setData(zkw, (SetData) op);
+        } else {
+          throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
+              + op.getClass().getName());
+        }
       }
     }
   }

From c2faadd2a114fefe31ce265be891a5d3e6acb679 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Jan 2013 04:25:56 +0000
Subject: [PATCH 0701/1540] HBASE-7562 revert due to TestZKMulti failure

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433283 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 83fc5cff970c..10d583ad0676 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1428,19 +1428,19 @@ public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
       } catch (InterruptedException ie) {
         zkw.interruptedException(ie);
       }
-    } else {
-      // run sequentially
-      for (ZKUtilOp op : ops) {
-        if (op instanceof CreateAndFailSilent) {
-          createAndFailSilent(zkw, (CreateAndFailSilent) op);
-        } else if (op instanceof DeleteNodeFailSilent) {
-          deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
-        } else if (op instanceof SetData) {
-          setData(zkw, (SetData) op);
-        } else {
-          throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
-              + op.getClass().getName());
-        }
+    }
+
+    // run sequentially
+    for (ZKUtilOp op : ops) {
+      if (op instanceof CreateAndFailSilent) {
+        createAndFailSilent(zkw, (CreateAndFailSilent)op);
+      } else if (op instanceof DeleteNodeFailSilent) {
+        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent)op);
+      } else if (op instanceof SetData) {
+        setData(zkw, (SetData)op);
+      } else {
+        throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
+          + op.getClass().getName());
       }
     }
   }

From 22a5cb1e38483a245f4e07b60a2acdecc6670c57 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Jan 2013 17:15:16 +0000
Subject: [PATCH 0702/1540] HBASE-5498 Secure Bulk Load (Francis Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433532 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     |  71 +++-
 .../access/SecureBulkLoadEndpoint.java        | 317 ++++++++++++++++++
 .../access/SecureBulkLoadProtocol.java        |  67 ++++
 .../TestSecureLoadIncrementalHFiles.java      |  56 ++++
 ...ureLoadIncrementalHFilesSplitRecovery.java |  66 ++++
 .../hbase/security/access/SecureTestUtil.java |   3 +-
 .../security/access/TestAccessController.java | 139 +++++++-
 .../coprocessor/SecureBulkLoadClient.java     |  92 +++++
 .../mapreduce/LoadIncrementalHFiles.java      |  86 ++++-
 .../hadoop/hbase/regionserver/HRegion.java    |  69 +++-
 .../hadoop/hbase/regionserver/Store.java      |   6 +-
 .../mapreduce/TestLoadIncrementalHFiles.java  |   7 +-
 ...estLoadIncrementalHFilesSplitRecovery.java |  15 +-
 13 files changed, 963 insertions(+), 31 deletions(-)
 create mode 100644 security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
 create mode 100644 security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
 create mode 100644 security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
 create mode 100644 security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index c9e4dee12b1d..136f025891bc 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -18,7 +18,9 @@
 import java.net.InetAddress;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -65,6 +67,7 @@
 import org.apache.hadoop.hbase.security.access.Permission.Action;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.util.Pair;
 
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
@@ -452,7 +455,7 @@ private void requirePermission(String request, Permission.Action perm,
    * @param families The map of column families-qualifiers.
    * @throws AccessDeniedException if the authorization check failed
    */
-  private void requirePermission(String request, Permission.Action perm,
+  public void requirePermission(String request, Permission.Action perm,
         RegionCoprocessorEnvironment env,
         Map<byte[], ? extends Collection<?>> families)
       throws IOException {
@@ -1017,6 +1020,72 @@ private void requireScannerOwner(InternalScanner s)
     }
   }
 
+  /**
+   * Verifies user has WRITE privileges on
+   * the Column Families involved in the bulkLoadHFile
+   * request. Specific Column Write privileges are presently
+   * ignored.
+   */
+  @Override
+  public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
+      List<Pair<byte[], String>> familyPaths) throws IOException {
+    List<byte[]> cfs = new LinkedList<byte[]>();
+    for(Pair<byte[],String> el : familyPaths) {
+      cfs.add(el.getFirst());
+    }
+    requirePermission("bulkLoadHFile", Permission.Action.WRITE, ctx.getEnvironment(), cfs);
+  }
+
+  private AuthResult hasSomeAccess(RegionCoprocessorEnvironment e, String request, Action action) throws IOException {
+    User requestUser = getActiveUser();
+    byte[] tableName = e.getRegion().getTableDesc().getName();
+    AuthResult authResult = permissionGranted(request, requestUser,
+        action, e, Collections.EMPTY_MAP);
+    if (!authResult.isAllowed()) {
+      for(UserPermission userPerm:
+          AccessControlLists.getUserPermissions(regionEnv.getConfiguration(), tableName)) {
+        for(Permission.Action userAction: userPerm.getActions()) {
+          if(userAction.equals(action)) {
+            return AuthResult.allow(request, "Access allowed", requestUser,
+                action, tableName);
+          }
+        }
+      }
+    }
+    return authResult;
+  }
+
+  /**
+   * Authorization check for
+   * SecureBulkLoadProtocol.prepareBulkLoad()
+   * @param e
+   * @throws IOException
+   */
+  public void prePrepareBulkLoad(RegionCoprocessorEnvironment e) throws IOException {
+    AuthResult authResult = hasSomeAccess(e, "prepareBulkLoad", Action.WRITE);
+    logResult(authResult);
+    if (!authResult.isAllowed()) {
+      throw new AccessDeniedException("Insufficient permissions (table=" +
+        e.getRegion().getTableDesc().getNameAsString() + ", action=WRITE)");
+    }
+  }
+
+  /**
+   * Authorization security check for
+   * SecureBulkLoadProtocol.cleanupBulkLoad()
+   * @param e
+   * @throws IOException
+   */
+  //TODO this should end up as a coprocessor hook
+  public void preCleanupBulkLoad(RegionCoprocessorEnvironment e) throws IOException {
+    AuthResult authResult = hasSomeAccess(e, "cleanupBulkLoad", Action.WRITE);
+    logResult(authResult);
+    if (!authResult.isAllowed()) {
+      throw new AccessDeniedException("Insufficient permissions (table=" +
+        e.getRegion().getTableDesc().getNameAsString() + ", action=WRITE)");
+    }
+  }
+
   /* ---- AccessControllerProtocol implementation ---- */
   /*
    * These methods are only allowed to be called against the _acl_ region(s).
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
new file mode 100644
index 000000000000..e422710b72fd
--- /dev/null
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
@@ -0,0 +1,317 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.ipc.RequestContext;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Methods;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.PrivilegedAction;
+import java.security.SecureRandom;
+import java.util.List;
+
+/**
+ * Coprocessor service for bulk loads in secure mode.
+ * This coprocessor has to be installed as part of enabling
+ * security in HBase.
+ *
+ * This service addresses two issues:
+ *
+ * 1. Moving files in a secure filesystem wherein the HBase Client
+ * and HBase Server are different filesystem users.
+ * 2. Does moving in a secure manner. Assuming that the filesystem
+ * is POSIX compliant.
+ *
+ * The algorithm is as follows:
+ *
+ * 1. Create an hbase owned staging directory which is
+ * world traversable (711): /hbase/staging
+ * 2. A user writes out data to his secure output directory: /user/foo/data
+ * 3. A call is made to hbase to create a secret staging directory
+ * which globally rwx (777): /user/staging/averylongandrandomdirectoryname
+ * 4. The user makes the data world readable and writable, then moves it
+ * into the random staging directory, then calls bulkLoadHFiles()
+ *
+ * Like delegation tokens the strength of the security lies in the length
+ * and randomness of the secret directory.
+ *
+ */
+@InterfaceAudience.Private
+public class SecureBulkLoadEndpoint extends BaseEndpointCoprocessor
+    implements SecureBulkLoadProtocol {
+
+  public static final long VERSION = 0L;
+
+  //Random number is 320 bits wide
+  private static final int RANDOM_WIDTH = 320;
+  //We picked 32 as the radix, so the character set
+  //will only contain alpha numeric values
+  //320/5 = 64 characters
+  private static final int RANDOM_RADIX = 32;
+
+  private static Log LOG = LogFactory.getLog(SecureBulkLoadEndpoint.class);
+
+  private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");
+  private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");
+  private final static String BULKLOAD_STAGING_DIR = "hbase.bulkload.staging.dir";
+
+  private SecureRandom random;
+  private FileSystem fs;
+  private Configuration conf;
+
+  //two levels so it doesn't get deleted accidentally
+  //no sticky bit in Hadoop 1.0
+  private Path baseStagingDir;
+
+  private RegionCoprocessorEnvironment env;
+
+
+  @Override
+  public void start(CoprocessorEnvironment env) {
+    super.start(env);
+
+    this.env = (RegionCoprocessorEnvironment)env;
+    random = new SecureRandom();
+    conf = env.getConfiguration();
+    baseStagingDir = getBaseStagingDir(conf);
+
+    try {
+      fs = FileSystem.get(conf);
+      fs.mkdirs(baseStagingDir, PERM_HIDDEN);
+      fs.setPermission(baseStagingDir, PERM_HIDDEN);
+      //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased
+      fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN);
+      FileStatus status = fs.getFileStatus(baseStagingDir);
+      if(status == null) {
+        throw new IllegalStateException("Failed to create staging directory");
+      }
+      if(!status.getPermission().equals(PERM_HIDDEN)) {
+        throw new IllegalStateException("Directory already exists but permissions aren't set to '-rwx--x--x' ");
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException("Failed to get FileSystem instance",e);
+    }
+  }
+
+  @Override
+  public String prepareBulkLoad(byte[] tableName) throws IOException {
+    getAccessController().prePrepareBulkLoad(env);
+    return createStagingDir(baseStagingDir, getActiveUser(), tableName).toString();
+  }
+
+  @Override
+  public void cleanupBulkLoad(String bulkToken) throws IOException {
+    getAccessController().preCleanupBulkLoad(env);
+    fs.delete(createStagingDir(baseStagingDir,
+        getActiveUser(),
+        env.getRegion().getTableDesc().getName(),
+        new Path(bulkToken).getName()),
+        true);
+  }
+
+  @Override
+  public boolean bulkLoadHFiles(final List<Pair<byte[], String>> familyPaths,
+                                final Token<?> userToken, final String bulkToken) throws IOException {
+    User user = getActiveUser();
+    final UserGroupInformation ugi = user.getUGI();
+    if(userToken != null) {
+      ugi.addToken(userToken);
+    } else if(User.isSecurityEnabled()) {
+      //we allow this to pass through in "simple" security mode
+      //for mini cluster testing
+      throw new DoNotRetryIOException("User token cannot be null");
+    }
+
+    HRegion region = env.getRegion();
+    boolean bypass = false;
+    if (region.getCoprocessorHost() != null) {
+      bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
+    }
+    boolean loaded = false;
+    if (!bypass) {
+      loaded = ugi.doAs(new PrivilegedAction<Boolean>() {
+        @Override
+        public Boolean run() {
+          FileSystem fs = null;
+          try {
+            Configuration conf = env.getConfiguration();
+            fs = FileSystem.get(conf);
+            for(Pair<byte[], String> el: familyPaths) {
+              Path p = new Path(el.getSecond());
+              LOG.debug("Setting permission for: " + p);
+              fs.setPermission(p, PERM_ALL_ACCESS);
+              Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));
+              if(!fs.exists(stageFamily)) {
+                fs.mkdirs(stageFamily);
+                fs.setPermission(stageFamily, PERM_ALL_ACCESS);
+              }
+            }
+            //We call bulkLoadHFiles as requesting user
+            //To enable access prior to staging
+            return env.getRegion().bulkLoadHFiles(familyPaths,
+                new SecureBulkLoadListener(fs, bulkToken));
+          } catch (Exception e) {
+            LOG.error("Failed to complete bulk load", e);
+          }
+          return false;
+        }
+      });
+    }
+    if (region.getCoprocessorHost() != null) {
+      loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
+    }
+    return loaded;
+  }
+
+  @Override
+  public long getProtocolVersion(String protocol, long clientVersion)
+      throws IOException {
+    if (SecureBulkLoadProtocol.class.getName().equals(protocol)) {
+      return SecureBulkLoadEndpoint.VERSION;
+    }
+    LOG.warn("Unknown protocol requested: " + protocol);
+    return -1;
+  }
+
+  private AccessController getAccessController() {
+    return (AccessController) this.env.getRegion()
+        .getCoprocessorHost().findCoprocessor(AccessController.class.getName());
+  }
+
+  private Path createStagingDir(Path baseDir, User user, byte[] tableName) throws IOException {
+    String randomDir = user.getShortName()+"__"+Bytes.toString(tableName)+"__"+
+        (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));
+    return createStagingDir(baseDir, user, tableName, randomDir);
+  }
+
+  private Path createStagingDir(Path baseDir,
+                                User user,
+                                byte[] tableName,
+                                String randomDir) throws IOException {
+    Path p = new Path(baseDir, randomDir);
+    fs.mkdirs(p, PERM_ALL_ACCESS);
+    fs.setPermission(p, PERM_ALL_ACCESS);
+    return p;
+  }
+
+  private User getActiveUser() throws IOException {
+    User user = RequestContext.getRequestUser();
+    if (!RequestContext.isInRequestContext()) {
+      throw new DoNotRetryIOException("Failed to get requesting user");
+    }
+
+    //this is for testing
+    if("simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {
+      return User.createUserForTesting(conf, user.getShortName(), new String[]{});
+    }
+
+    return user;
+  }
+
+  /**
+   * This returns the staging path for a given column family.
+   * This is needed for clean recovery and called reflectively in LoadIncrementalHFiles
+   */
+  public static Path getStagingPath(Configuration conf, String bulkToken, byte[] family) {
+    Path stageP = new Path(getBaseStagingDir(conf), bulkToken);
+    return new Path(stageP, Bytes.toString(family));
+  }
+
+  private static Path getBaseStagingDir(Configuration conf) {
+    return new Path(conf.get(BULKLOAD_STAGING_DIR, "/tmp/hbase-staging"));
+  }
+
+  private static class SecureBulkLoadListener implements HRegion.BulkLoadListener {
+    private FileSystem fs;
+    private String stagingDir;
+
+    public SecureBulkLoadListener(FileSystem fs, String stagingDir) {
+      this.fs = fs;
+      this.stagingDir = stagingDir;
+    }
+
+    @Override
+    public String prepareBulkLoad(final byte[] family, final String srcPath) throws IOException {
+      Path p = new Path(srcPath);
+      Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));
+
+      if(!isFile(p)) {
+        throw new IOException("Path does not reference a file: " + p);
+      }
+
+      LOG.debug("Moving " + p + " to " + stageP);
+      if(!fs.rename(p, stageP)) {
+        throw new IOException("Failed to move HFile: " + p + " to " + stageP);
+      }
+      return stageP.toString();
+    }
+
+    @Override
+    public void doneBulkLoad(byte[] family, String srcPath) throws IOException {
+      LOG.debug("Bulk Load done for: " + srcPath);
+    }
+
+    @Override
+    public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {
+      Path p = new Path(srcPath);
+      Path stageP = new Path(stagingDir,
+          new Path(Bytes.toString(family), p.getName()));
+      LOG.debug("Moving " + stageP + " back to " + p);
+      if(!fs.rename(stageP, p))
+        throw new IOException("Failed to move HFile: " + stageP + " to " + p);
+    }
+
+    /**
+     * Check if the path is referencing a file.
+     * This is mainly needed to avoid symlinks.
+     * @param p
+     * @return true if the p is a file
+     * @throws IOException
+     */
+    private boolean isFile(Path p) throws IOException {
+      FileStatus status = fs.getFileStatus(p);
+      boolean isFile = !status.isDir();
+      try {
+        isFile = isFile && !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);
+      } catch (Exception e) {
+      }
+      return isFile;
+    }
+  }
+}
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
new file mode 100644
index 000000000000..e381d3c0472c
--- /dev/null
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.security.TokenInfo;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.token.Token;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Provides a secure way to bulk load data onto HBase
+ * These are internal API. Bulk load should be initiated
+ * via {@link org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles}
+ * with security enabled.
+ */
+@TokenInfo("HBASE_AUTH_TOKEN")
+public interface SecureBulkLoadProtocol extends CoprocessorProtocol {
+
+  /**
+   * Prepare for bulk load.
+   * Will be called before bulkLoadHFiles()
+   * @param tableName
+   * @return a bulkToken which uniquely identifies the bulk session
+   * @throws IOException
+   */
+  String prepareBulkLoad(byte[] tableName) throws IOException;
+
+  /**
+   * Cleanup after bulk load.
+   * Will be called after bulkLoadHFiles().
+   * @param bulkToken
+   * @throws IOException
+   */
+  void cleanupBulkLoad(String bulkToken) throws IOException;
+
+  /**
+   * Secure version of HRegionServer.bulkLoadHFiles().
+   * @param familyPaths column family to HFile path pairs
+   * @param userToken requesting user's HDFS delegation token
+   * @param bulkToken
+   * @return
+   * @throws IOException
+   */
+  boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
+                         Token<?> userToken, String bulkToken) throws IOException;
+
+}
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
new file mode 100644
index 000000000000..03ec4c571833
--- /dev/null
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.security.access.AccessControlLists;
+import org.apache.hadoop.hbase.security.access.SecureTestUtil;
+
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Reruns TestLoadIncrementalHFiles using LoadIncrementalHFiles using secure mode.
+ * This suite is unable to verify the security handoff/turnover
+ * as miniCluster is running as system user thus has root privileges
+ * and delegation tokens don't seem to work on miniDFS.
+ *
+ * Thus SecureBulkload can only be completely verified by running
+ * integration tests against a secure cluster. This suite is still
+ * invaluable as it verifies the other mechanisms that need to be
+ * supported as part of a LoadIncrementalFiles call.
+ */
+@Category(LargeTests.class)
+public class TestSecureLoadIncrementalHFiles extends  TestLoadIncrementalHFiles{
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    useSecure = true;
+    // setup configuration
+    SecureTestUtil.enableSecurity(util.getConfiguration());
+
+    util.startMiniCluster();
+
+    // Wait for the ACL table to become available
+    util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+  }
+
+}
+
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
new file mode 100644
index 000000000000..e8593b2cdce2
--- /dev/null
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.security.access.AccessControlLists;
+import org.apache.hadoop.hbase.security.access.SecureTestUtil;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+
+/**
+ * Reruns TestSecureLoadIncrementalHFilesSplitRecovery
+ * using LoadIncrementalHFiles in secure mode.
+ * This suite is unable to verify the security handoff/turnover
+ * as miniCluster is running as system user thus has root privileges
+ * and delegation tokens don't seem to work on miniDFS.
+ *
+ * Thus SecureBulkload can only be completely verified by running
+ * integration tests against a secure cluster. This suite is still
+ * invaluable as it verifies the other mechanisms that need to be
+ * supported as part of a LoadIncrementalFiles call.
+ */
+@Category(LargeTests.class)
+public class TestSecureLoadIncrementalHFilesSplitRecovery extends TestLoadIncrementalHFilesSplitRecovery {
+
+  //This "overrides" the parent static method
+  //make sure they are in sync
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    useSecure = true;
+    util = new HBaseTestingUtility();
+    // setup configuration
+    SecureTestUtil.enableSecurity(util.getConfiguration());
+
+    util.startMiniCluster();
+
+    // Wait for the ACL table to become available
+    util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+  }
+
+  //Disabling this test as it does not work in secure mode
+  @Test
+  @Override
+  public void testBulkLoadPhaseFailure() {
+  }
+}
+
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
index 4aa0e18f401f..5d55760a5a7c 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
@@ -34,7 +34,8 @@ public static void enableSecurity(Configuration conf) throws IOException {
     conf.set("hadoop.security.authentication", "simple");
     conf.set("hbase.rpc.engine", SecureRpcEngine.class.getName());
     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, AccessController.class.getName());
-    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName());
+    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName()+
+            ","+SecureBulkLoadEndpoint.class.getName());
     conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, AccessController.class.getName());
     // add the process running user to superusers
     String currentUser = User.getCurrent().getName();
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index ad6dda4ccf32..7b00cf088d9d 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -28,12 +28,17 @@
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.UnknownRowLockException;
@@ -53,6 +58,9 @@
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
@@ -192,22 +200,35 @@ public void verifyDenied(User user, PrivilegedExceptionAction... actions) throws
       try {
         user.runAs(action);
         fail("Expected AccessDeniedException for user '" + user.getShortName() + "'");
-      } catch (RetriesExhaustedWithDetailsException e) {
-        // in case of batch operations, and put, the client assembles a
-        // RetriesExhaustedWithDetailsException instead of throwing an
-        // AccessDeniedException
+      } catch (AccessDeniedException ade) {
+        // expected result
+      } catch (IOException e) {
         boolean isAccessDeniedException = false;
-        for (Throwable ex : e.getCauses()) {
-          if (ex instanceof AccessDeniedException) {
-            isAccessDeniedException = true;
-            break;
+        if(e instanceof RetriesExhaustedWithDetailsException) {
+          // in case of batch operations, and put, the client assembles a
+          // RetriesExhaustedWithDetailsException instead of throwing an
+          // AccessDeniedException
+          for(Throwable ex : ((RetriesExhaustedWithDetailsException) e).getCauses()) {
+            if (ex instanceof AccessDeniedException) {
+              isAccessDeniedException = true;
+              break;
+            }
           }
         }
+        else {
+          // For doBulkLoad calls AccessDeniedException
+          // is buried in the stack trace
+          Throwable ex = e;
+          do {
+            if (ex instanceof AccessDeniedException) {
+              isAccessDeniedException = true;
+              break;
+            }
+          } while((ex = ex.getCause()) != null);
+        }
         if (!isAccessDeniedException) {
           fail("Not receiving AccessDeniedException for user '" + user.getShortName() + "'");
         }
-      } catch (AccessDeniedException ade) {
-        // expected result
       }
     }
   }
@@ -631,6 +652,104 @@ public Object run() throws Exception {
     verifyReadWrite(checkAndPut);
   }
 
+  @Test
+  public void testBulkLoad() throws Exception {
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    final Path dir = TEST_UTIL.getDataTestDir("testBulkLoad");
+    fs.mkdirs(dir);
+    //need to make it globally writable
+    //so users creating HFiles have write permissions
+    fs.setPermission(dir, FsPermission.valueOf("-rwxrwxrwx"));
+
+    PrivilegedExceptionAction bulkLoadAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        int numRows = 3;
+
+        //Making the assumption that the test table won't split between the range
+        byte[][][] hfileRanges = {{{(byte)0}, {(byte)9}}};
+
+        Path bulkLoadBasePath = new Path(dir, new Path(User.getCurrent().getName()));
+        new BulkLoadHelper(bulkLoadBasePath)
+            .bulkLoadHFile(TEST_TABLE, TEST_FAMILY, Bytes.toBytes("q"), hfileRanges, numRows);
+
+        return null;
+      }
+    };
+    verifyWrite(bulkLoadAction);
+  }
+
+  public class BulkLoadHelper {
+    private final FileSystem fs;
+    private final Path loadPath;
+    private final Configuration conf;
+
+    public BulkLoadHelper(Path loadPath) throws IOException {
+      fs = TEST_UTIL.getTestFileSystem();
+      conf = TEST_UTIL.getConfiguration();
+      loadPath = loadPath.makeQualified(fs);
+      this.loadPath = loadPath;
+    }
+
+    private void createHFile(Path path,
+        byte[] family, byte[] qualifier,
+        byte[] startKey, byte[] endKey, int numRows) throws IOException {
+
+      HFile.Writer writer = null;
+      long now = System.currentTimeMillis();
+      try {
+        writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
+            .withPath(fs, path)
+            .withComparator(KeyValue.KEY_COMPARATOR)
+            .create();
+        // subtract 2 since numRows doesn't include boundary keys
+        for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, true, numRows-2)) {
+          KeyValue kv = new KeyValue(key, family, qualifier, now, key);
+          writer.append(kv);
+        }
+      } finally {
+        if(writer != null)
+          writer.close();
+      }
+    }
+
+    private void bulkLoadHFile(
+        byte[] tableName,
+        byte[] family,
+        byte[] qualifier,
+        byte[][][] hfileRanges,
+        int numRowsPerRange) throws Exception {
+
+      Path familyDir = new Path(loadPath, Bytes.toString(family));
+      fs.mkdirs(familyDir);
+      int hfileIdx = 0;
+      for (byte[][] range : hfileRanges) {
+        byte[] from = range[0];
+        byte[] to = range[1];
+        createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)),
+            family, qualifier, from, to, numRowsPerRange);
+      }
+      //set global read so RegionServer can move it
+      setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));
+
+      HTable table = new HTable(conf, tableName);
+      TEST_UTIL.waitTableAvailable(tableName, 30000);
+      LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
+      loader.doBulkLoad(loadPath, table);
+    }
+
+    public void setPermission(Path dir, FsPermission perm) throws IOException {
+      if(!fs.getFileStatus(dir).isDir()) {
+        fs.setPermission(dir,perm);
+      }
+      else {
+        for(FileStatus el : fs.listStatus(dir)) {
+          fs.setPermission(el.getPath(), perm);
+          setPermission(el.getPath() , perm);
+        }
+      }
+    }
+  }
+
   @Test
   public void testAppend() throws Exception {
 
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
new file mode 100644
index 000000000000..a5218331676c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Methods;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.token.Token;
+
+import java.io.IOException;
+import java.util.List;
+
+public class SecureBulkLoadClient {
+  private static Class protocolClazz;
+  private static Class endpointClazz;
+  private Object proxy;
+  private HTable table;
+
+  public SecureBulkLoadClient(HTable table) throws IOException {
+    this(table, HConstants.EMPTY_START_ROW);
+  }
+
+  public SecureBulkLoadClient(HTable table, byte[] startRow) throws IOException {
+    try {
+      protocolClazz = protocolClazz!=null?protocolClazz:
+          Class.forName("org.apache.hadoop.hbase.security.access.SecureBulkLoadProtocol");
+      endpointClazz = endpointClazz!=null?endpointClazz:
+          Class.forName("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
+      proxy = table.coprocessorProxy(protocolClazz, startRow);
+      this.table = table;
+    } catch (ClassNotFoundException e) {
+      throw new IOException("Failed to initialize SecureBulkLoad", e);
+    }
+  }
+
+  public String prepareBulkLoad(byte[] tableName) throws IOException {
+    try {
+      String bulkToken = (String) Methods.call(protocolClazz, proxy,
+          "prepareBulkLoad", new Class[]{byte[].class}, new Object[]{tableName});
+      return bulkToken;
+    } catch (Exception e) {
+      throw new IOException("Failed to prepareBulkLoad", e);
+    }
+  }
+
+  public void cleanupBulkLoad(String bulkToken) throws IOException {
+    try {
+      Methods.call(protocolClazz, proxy,
+          "cleanupBulkLoad", new Class[]{String.class},new Object[]{bulkToken});
+    } catch (Exception e) {
+      throw new IOException("Failed to prepareBulkLoad", e);
+    }
+  }
+
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
+                         Token<?> userToken, String bulkToken) throws IOException {
+    try {
+      return (Boolean)Methods.call(protocolClazz, proxy, "bulkLoadHFiles",
+          new Class[]{List.class, Token.class, String.class},new Object[]{familyPaths, userToken, bulkToken});
+    } catch (Exception e) {
+      throw new IOException("Failed to bulkLoadHFiles", e);
+    }
+  }
+
+  public Path getStagingPath(String bulkToken, byte[] family) throws IOException {
+    try {
+      return (Path)Methods.call(endpointClazz, null, "getStagingPath",
+          new Class[]{Configuration.class, String.class, byte[].class},
+          new Object[]{table.getConfiguration(), bulkToken, family});
+    } catch (Exception e) {
+      throw new IOException("Failed to getStagingPath", e);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index dea6aaa14e83..448eca1ea02f 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -60,6 +60,7 @@
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.ServerCallable;
+import org.apache.hadoop.hbase.coprocessor.SecureBulkLoadClient;
 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.io.Reference.Range;
@@ -73,8 +74,10 @@
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -98,10 +101,21 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
 
   public static String NAME = "completebulkload";
 
-  public LoadIncrementalHFiles(Configuration conf) throws Exception {
+  private boolean useSecure;
+  private Token<?> userToken;
+  private String bulkToken;
+
+  //package private for testing
+  LoadIncrementalHFiles(Configuration conf, Boolean useSecure) throws Exception {
     super(conf);
     this.cfg = conf;
     this.hbAdmin = new HBaseAdmin(conf);
+    //added simple for testing
+    this.useSecure = useSecure != null ? useSecure : User.isHBaseSecurityEnabled(conf);
+  }
+
+  public LoadIncrementalHFiles(Configuration conf) throws Exception {
+    this(conf, null);
   }
 
   private void usage() {
@@ -212,6 +226,18 @@ public void doBulkLoad(Path hfofDir, final HTable table)
         return;
       }
 
+      //If using secure bulk load
+      //prepare staging directory and token
+      if(useSecure) {
+        //This condition is here for unit testing
+        //Since delegation token doesn't work in mini cluster
+        if(User.isSecurityEnabled()) {
+          FileSystem fs = FileSystem.get(cfg);
+          userToken = fs.getDelegationToken("renewer");
+        }
+        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getTableName());
+      }
+
       // Assumes that region splits can happen while this occurs.
       while (!queue.isEmpty()) {
         // need to reload split keys each iteration.
@@ -240,6 +266,18 @@ public void doBulkLoad(Path hfofDir, final HTable table)
       }
 
     } finally {
+      if(useSecure) {
+        if(userToken != null) {
+          try {
+            userToken.cancel(cfg);
+          } catch (Exception e) {
+            LOG.warn("Failed to cancel HDFS delegation token.", e);
+          }
+        }
+        if(bulkToken != null) {
+          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
+        }
+      }
       pool.shutdown();
       if (queue != null && !queue.isEmpty()) {
         StringBuilder err = new StringBuilder();
@@ -473,11 +511,49 @@ protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
         tableName, first) {
       @Override
       public Boolean call() throws Exception {
-        LOG.debug("Going to connect to server " + location + " for row "
-            + Bytes.toStringBinary(row));
-        byte[] regionName = location.getRegionInfo().getRegionName();
-        return server.bulkLoadHFiles(famPaths, regionName);
+        SecureBulkLoadClient secureClient = null;
+        boolean success = false;
+
+        try {
+          LOG.debug("Going to connect to server " + location + " for row "
+              + Bytes.toStringBinary(row));
+          byte[] regionName = location.getRegionInfo().getRegionName();
+          if(!useSecure) {
+            success = server.bulkLoadHFiles(famPaths, regionName);
+          } else {
+            HTable table = new HTable(conn.getConfiguration(), tableName);
+            secureClient = new SecureBulkLoadClient(table, location.getRegionInfo().getStartKey());
+            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken);
+          }
+          return success;
+        } finally {
+          //Best effort copying of files that might not have been imported
+          //from the staging directory back to original location
+          //in user directory
+          if(secureClient != null && !success) {
+            FileSystem fs = FileSystem.get(cfg);
+            for(Pair<byte[], String> el : famPaths) {
+              Path hfileStagingPath = null;
+              Path hfileOrigPath = new Path(el.getSecond());
+              try {
+                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
+                    hfileOrigPath.getName());
+                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
+                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
+                      hfileStagingPath);
+                } else if(fs.exists(hfileStagingPath)){
+                  LOG.debug("Unable to move back file " + hfileOrigPath + " from " +
+                      hfileStagingPath);
+                }
+              } catch(Exception ex) {
+                LOG.debug("Unable to move back file " + hfileOrigPath + " from " +
+                    hfileStagingPath, ex);
+              }
+            }
+          }
+        }
       }
+
     };
 
     try {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c19ea0a4de89..c79f9a9e0c84 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3373,8 +3373,22 @@ private static boolean hasMultipleColumnFamilies(
    * @return true if successful, false if failed recoverably
    * @throws IOException if failed unrecoverably.
    */
-  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths)
-  throws IOException {
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths) throws IOException {
+    return bulkLoadHFiles(familyPaths, null);
+  }
+
+  /**
+   * Attempts to atomically load a group of hfiles.  This is critical for loading
+   * rows with multiple column families atomically.
+   *
+   * @param familyPaths List of Pair<byte[] column family, String hfilePath>
+   * @param bulkLoadListener Internal hooks enabling massaging/preparation of a
+   * file about to be bulk loaded
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if failed unrecoverably.
+   */
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
+      BulkLoadListener bulkLoadListener) throws IOException {
     Preconditions.checkNotNull(familyPaths);
     // we need writeLock for multi-family bulk load
     startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));
@@ -3434,7 +3448,14 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths)
         String path = p.getSecond();
         Store store = getStore(familyName);
         try {
-          store.bulkLoadHFile(path);
+          String finalPath = path;
+          if(bulkLoadListener != null) {
+            finalPath = bulkLoadListener.prepareBulkLoad(familyName, path);
+          }
+          store.bulkLoadHFile(finalPath);
+          if(bulkLoadListener != null) {
+            bulkLoadListener.doneBulkLoad(familyName, path);
+          }
         } catch (IOException ioe) {
           // a failure here causes an atomicity violation that we currently
           // cannot recover from since it is likely a failed hdfs operation.
@@ -3442,6 +3463,14 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths)
           // TODO Need a better story for reverting partial failures due to HDFS.
           LOG.error("There was a partial failure due to IO when attempting to" +
               " load " + Bytes.toString(p.getFirst()) + " : "+ p.getSecond());
+          if(bulkLoadListener != null) {
+            try {
+              bulkLoadListener.failedBulkLoad(familyName, path);
+            } catch (Exception ex) {
+              LOG.error("Error while calling failedBulkLoad for family "+
+                  Bytes.toString(familyName)+" with path "+path, ex);
+            }
+          }
           throw ioe;
         }
       }
@@ -5650,4 +5679,38 @@ public static void main(String[] args) throws IOException {
        if (bc != null) bc.shutdown();
     }
   }
+
+  /**
+   * Listener class to enable callers of
+   * bulkLoadHFile() to perform any necessary
+   * pre/post processing of a given bulkload call
+   */
+  public static interface BulkLoadListener {
+
+    /**
+     * Called before an HFile is actually loaded
+     * @param family family being loaded to
+     * @param srcPath path of HFile
+     * @return final path to be used for actual loading
+     * @throws IOException
+     */
+    String prepareBulkLoad(byte[] family, String srcPath) throws IOException;
+
+    /**
+     * Called after a successful HFile load
+     * @param family family being loaded to
+     * @param srcPath path of HFile
+     * @throws IOException
+     */
+    void doneBulkLoad(byte[] family, String srcPath) throws IOException;
+
+    /**
+     * Called after a failed HFile load
+     * @param family family being loaded to
+     * @param srcPath path of HFile
+     * @throws IOException
+     */
+    void failedBulkLoad(byte[] family, String srcPath) throws IOException;
+
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 0839e3c59d43..b895688b6e16 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -577,7 +577,11 @@ void bulkLoadHFile(String srcPathStr) throws IOException {
     // Move the file if it's on another filesystem
     FileSystem srcFs = srcPath.getFileSystem(conf);
     FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
-    if (!srcFs.equals(desFs)) {
+    //We can't compare FileSystem instances as
+    //equals() includes UGI instance as part of the comparison
+    //and won't work when doing SecureBulkLoad
+    //TODO deal with viewFS
+    if (!srcFs.getUri().equals(desFs.getUri())) {
       LOG.info("File " + srcPath + " on different filesystem than " +
           "destination store - moving to this filesystem.");
       Path tmpPath = getTmpPath();
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index d0f9ef72e665..11df9c214278 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -60,7 +60,9 @@ public class TestLoadIncrementalHFiles {
   public static String COMPRESSION =
     Compression.Algorithm.NONE.getName();
 
-  private static HBaseTestingUtility util = new HBaseTestingUtility();
+  static HBaseTestingUtility util = new HBaseTestingUtility();
+  //used by secure subclass
+  static boolean useSecure = false;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
@@ -149,8 +151,7 @@ private void runTest(String testName, BloomType bloomType,
 
     HTable table = new HTable(util.getConfiguration(), TABLE);
     util.waitTableAvailable(TABLE, 30000);
-    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
-      util.getConfiguration());
+    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), useSecure);
     loader.doBulkLoad(dir, table);
 
     assertEquals(expectedRows, util.countRows(table));
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
index 301ee27f1a92..0ac498071ff5 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
@@ -68,7 +68,9 @@
 public class TestLoadIncrementalHFilesSplitRecovery {
   final static Log LOG = LogFactory.getLog(TestHRegionServerBulkLoad.class);
 
-  private static HBaseTestingUtility util;
+  static HBaseTestingUtility util;
+  //used by secure subclass
+  static boolean useSecure = false;
 
   final static int NUM_CFS = 10;
   final static byte[] QUAL = Bytes.toBytes("qual");
@@ -135,8 +137,7 @@ private Path buildBulkFiles(String table, int value) throws Exception {
    */
   private void populateTable(String table, int value) throws Exception {
     // create HFiles for different column families
-    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration());
+    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration(), useSecure);
     Path bulk1 = buildBulkFiles(table, value);
     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
     lih.doBulkLoad(bulk1, t);
@@ -228,7 +229,7 @@ public void testBulkLoadPhaseFailure() throws Exception {
     final AtomicInteger attmptedCalls = new AtomicInteger();
     final AtomicInteger failedCalls = new AtomicInteger();
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+        util.getConfiguration(), useSecure) {
 
       protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
           byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis)
@@ -296,7 +297,7 @@ public void testSplitWhileBulkLoadPhase() throws Exception {
     // files to fail when attempt to atomically import.  This is recoverable.
     final AtomicInteger attemptedCalls = new AtomicInteger();
     LoadIncrementalHFiles lih2 = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+        util.getConfiguration(), useSecure) {
 
       protected void bulkLoadPhase(final HTable htable, final HConnection conn,
           ExecutorService pool, Deque<LoadQueueItem> queue,
@@ -337,7 +338,7 @@ public void testGroupOrSplitPresplit() throws Exception {
 
     final AtomicInteger countedLqis= new AtomicInteger();
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+        util.getConfiguration(), useSecure) {
       protected List<LoadQueueItem> groupOrSplit(
           Multimap<ByteBuffer, LoadQueueItem> regionGroups,
           final LoadQueueItem item, final HTable htable,
@@ -369,7 +370,7 @@ public void testGroupOrSplitFailure() throws Exception {
     setupTable(table, 10);
 
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+        util.getConfiguration(), useSecure) {
       int i = 0;
 
       protected List<LoadQueueItem> groupOrSplit(

From e0a16e42057d06cfdfabdf9f5d66f8f416721b7e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Jan 2013 19:28:08 +0000
Subject: [PATCH 0703/1540] HBASE-7562 ZKUtil: missing "else condition" in
 multi processing (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433593 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java    | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 10d583ad0676..13dd29ff3caa 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1420,6 +1420,7 @@ public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
                + "  Attempting to run operations sequentially because"
                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
+             processSequentially(zkw, ops);
              break;
            }
           default:
@@ -1428,19 +1429,24 @@ public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
       } catch (InterruptedException ie) {
         zkw.interruptedException(ie);
       }
+    } else {
+      // run sequentially
+      processSequentially(zkw, ops);
     }
+  }
 
-    // run sequentially
+  private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
+      throws KeeperException, NoNodeException {
     for (ZKUtilOp op : ops) {
       if (op instanceof CreateAndFailSilent) {
-        createAndFailSilent(zkw, (CreateAndFailSilent)op);
+        createAndFailSilent(zkw, (CreateAndFailSilent) op);
       } else if (op instanceof DeleteNodeFailSilent) {
-        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent)op);
+        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
       } else if (op instanceof SetData) {
-        setData(zkw, (SetData)op);
+        setData(zkw, (SetData) op);
       } else {
         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
-          + op.getClass().getName());
+            + op.getClass().getName());
       }
     }
   }

From 47afc1136943c5e6b89d4cc3e7da6ef3ae991a52 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 15 Jan 2013 22:17:00 +0000
Subject: [PATCH 0704/1540] HBASE-7551 nodeChildrenChange event may happen
 after the transition to RS_ZK_REGION_SPLITTING in SplitTransaction causing
 the SPLIT event to be missed in the master side. (Ram, Ted, and Lars H)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433700 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/AssignmentManager.java    | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index e118f485b1de..1245c6ff3c88 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1205,9 +1205,20 @@ private void makeRegionOnline(RegionState rs, HRegionInfo regionInfo) {
   public void nodeChildrenChanged(String path) {
     if(path.equals(watcher.assignmentZNode)) {
       try {
-        // Just make sure we see the changes for the new znodes
-        ZKUtil.listChildrenAndWatchThem(watcher,
+        List<String> children = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
             watcher.assignmentZNode);
+        if (children != null) {
+          Stat stat = new Stat();
+          for (String child : children) {
+            stat.setVersion(0);
+            RegionTransitionData data = ZKAssign.getDataAndWatch(watcher,
+                ZKUtil.joinZNode(watcher.assignmentZNode, child), stat);
+            // See HBASE-7551, handle splitting here as well, in case we miss the node change event
+            if (stat.getVersion() > 0 && data.getEventType() == EventType.RS_ZK_REGION_SPLITTING) {
+              handleRegion(data, stat.getVersion());
+            }
+          }
+        }
       } catch(KeeperException e) {
         master.abort("Unexpected ZK exception reading unassigned children", e);
       }

From 4583d83eb6268759876f3f5cc40c50245ef55676 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 16 Jan 2013 02:39:30 +0000
Subject: [PATCH 0705/1540] HBASE-7549 Make HTableInterface#batch() javadoc
 proper (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1433806 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/HTableInterface.java   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
index 404567b8c931..e7e296c53879 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
@@ -75,12 +75,12 @@ public interface HTableInterface extends Closeable {
   boolean exists(Get get) throws IOException;
 
   /**
-   * Method that does a batch call on Deletes, Gets and Puts. The ordering of
-   * execution of the actions is not defined. Meaning if you do a Put and a
+   * Method that does a batch call on Deletes, Gets, Puts, Increments, Appends and RowMutations.
+   * The execution ordering of the actions is not defined. Meaning if you do a Put and a
    * Get in the same {@link #batch} call, you will not necessarily be
    * guaranteed that the Get returns what the Put had put.
    *
-   * @param actions list of Get, Put, Delete objects
+   * @param actions list of Get, Put, Delete, Increment, Append, RowMutations objects
    * @param results Empty Object[], same size as actions. Provides access to partial
    *                results, in case an exception is thrown. A null in the result array means that
    *                the call for that action failed, even after retries
@@ -93,7 +93,7 @@ public interface HTableInterface extends Closeable {
    * Same as {@link #batch(List, Object[])}, but returns an array of
    * results instead of using a results parameter reference.
    *
-   * @param actions list of Get, Put, Delete objects
+   * @param actions list of Get, Put, Delete, Increment, Append, RowMutations objects
    * @return the results from the actions. A null in the return array means that
    *         the call for that action failed, even after retries
    * @throws IOException

From 969342185e36f15fe29c06944f00f27c4601b0d5 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 16 Jan 2013 16:26:11 +0000
Subject: [PATCH 0706/1540] HBASE-7575 FSUtils#getTableStoreFilePathMap should
 all ignore non-table folders

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434021 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java    | 18 +++++++++++++-----
 .../org/apache/hadoop/hbase/util/FSUtils.java  |  6 +++---
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index ea4815c96ecb..e70bd3be708c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -19,12 +19,13 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 import java.util.regex.Pattern;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -670,10 +671,17 @@ public static enum Modify {
   /** Directory under /hbase where archived hfiles are stored */
   public static final String HFILE_ARCHIVE_DIRECTORY = ".archive";
 
-  public static final List<String> HBASE_NON_USER_TABLE_DIRS = new ArrayList<String>(
-      Arrays.asList(new String[] { HREGION_LOGDIR_NAME, HREGION_OLDLOGDIR_NAME, CORRUPT_DIR_NAME,
-          Bytes.toString(META_TABLE_NAME), Bytes.toString(ROOT_TABLE_NAME), SPLIT_LOGDIR_NAME,
-          HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY }));
+  /** Directories that are not HBase table directories */
+  public static final List<String> HBASE_NON_TABLE_DIRS =
+    Collections.unmodifiableList(Arrays.asList(new String[] { HREGION_LOGDIR_NAME,
+      HREGION_OLDLOGDIR_NAME, CORRUPT_DIR_NAME, SPLIT_LOGDIR_NAME,
+      HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY }));
+
+  /** Directories that are not HBase user table directories */
+  public static final List<String> HBASE_NON_USER_TABLE_DIRS =
+    Collections.unmodifiableList(Arrays.asList((String[])ArrayUtils.addAll(
+      new String[] { Bytes.toString(META_TABLE_NAME), Bytes.toString(ROOT_TABLE_NAME) },
+      HBASE_NON_TABLE_DIRS.toArray())));
 
   /** Health script related settings. */
   public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location";
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 18074efa6042..179b86d88bec 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1096,11 +1096,11 @@ public static Map<String, Path> getTableStoreFilePathMap(
     // presumes any directory under hbase.rootdir is a table
     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
     for (FileStatus tableDir : tableDirs) {
-      // Skip the .log directory.  All others should be tables.  Inside a table,
-      // there are compaction.dir directories to skip.  Otherwise, all else
+      // Skip the .log and other non-table directories.  All others should be tables.
+      // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
       // should be regions. 
       Path d = tableDir.getPath();
-      if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
+      if (HConstants.HBASE_NON_TABLE_DIRS.contains(d.getName())) {
         continue;
       }
       FileStatus[] regionDirs = fs.listStatus(d, df);

From 282f80e2829f619e0d756e16ff855f154679abd5 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 16 Jan 2013 18:36:45 +0000
Subject: [PATCH 0707/1540] HBASE-7581. TestAccessController depends on the
 execution order (nkeywal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434098 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/TestAccessController.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 7b00cf088d9d..fab6c8ef5897 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -640,10 +640,9 @@ public Object run() throws Exception {
     // action for checkAndPut()
     PrivilegedExceptionAction checkAndPut = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
+        HTable t = new HTable(conf, TEST_TABLE);
         Put p = new Put(Bytes.toBytes("random_row"));
         p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1));
-
-        HTable t = new HTable(conf, TEST_TABLE);
         t.checkAndPut(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
           Bytes.toBytes("test_value"), p);
         return null;
@@ -676,6 +675,10 @@ public Object run() throws Exception {
       }
     };
     verifyWrite(bulkLoadAction);
+
+    // Reinit after the bulk upload
+    TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE);
+    TEST_UTIL.getHBaseAdmin().enableTable(TEST_TABLE);
   }
 
   public class BulkLoadHelper {

From 42d4f640e84f12191114a83e80e663f2c5861077 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 16 Jan 2013 18:39:31 +0000
Subject: [PATCH 0708/1540] HBASE-7584. Improve TestAccessController.testAppend

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434099 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/security/access/TestAccessController.java  | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index fab6c8ef5897..3425cd8a6192 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -758,9 +758,15 @@ public void testAppend() throws Exception {
 
     PrivilegedExceptionAction appendAction = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        Append append = new Append(TEST_TABLE);
-        append.add(TEST_FAMILY, Bytes.toBytes("qualifier"), Bytes.toBytes("value"));
-        ACCESS_CONTROLLER.preAppend(ObserverContext.createAndPrepare(RCP_ENV, null), append);
+        byte[] row = Bytes.toBytes("random_row");
+        byte[] qualifier = Bytes.toBytes("q");
+        HTable t = new HTable(conf, TEST_TABLE);
+        Put put = new Put(row);
+        put.add(TEST_FAMILY, qualifier, Bytes.toBytes(1));
+        t.put(put);
+        Append append = new Append(row);
+        append.add(TEST_FAMILY, qualifier, Bytes.toBytes(2));
+        t.append(append);
         return null;
       }
     };

From 05934d740531323ee07a954358fa07e34d3852af Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 16 Jan 2013 20:44:20 +0000
Subject: [PATCH 0709/1540] HBASE-7587. Fix two findbugs warnings in
 RowResource (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434378 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/rest/RowResource.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index d933bcfd75ad..c348f365db06 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -239,7 +239,9 @@ Response update(final CellSetModel model, final boolean replace) {
     } finally {
       if (table != null) try {
         table.close();
-      } catch (IOException ioe) { }
+      } catch (IOException ioe) {
+        LOG.debug("Exception received while closing the table", ioe);
+      }
     }
   }
 
@@ -554,7 +556,9 @@ Response checkAndDelete(final CellSetModel model) {
     } finally {
       if (table != null) try {
         table.close();
-      } catch (IOException ioe) { }
+      } catch (IOException ioe) {
+        LOG.debug("Exception received while closing the table", ioe);
+      }
     }
   }
 }

From 991e6bc7391ba3222e651431d91ce68c997275e7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 16 Jan 2013 22:11:03 +0000
Subject: [PATCH 0710/1540] HBASE-7578 TestCatalogTracker hangs occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434437 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  | 73 ++-----------------
 .../apache/hadoop/hbase/master/HMaster.java   |  3 +-
 .../hbase/regionserver/HRegionServer.java     |  3 +-
 .../hbase/catalog/TestCatalogTracker.java     |  8 +-
 .../TestMetaReaderEditorNoCluster.java        |  2 +-
 .../hbase/master/TestCatalogJanitor.java      |  2 +-
 6 files changed, 12 insertions(+), 79 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 331642af7243..5f2f1482c7f2 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -118,11 +118,6 @@ public class CatalogTracker {
    */
   private ServerName metaLocation;
 
-  /*
-   * Timeout waiting on root or meta to be set.
-   */
-  private final int defaultTimeout;
-
   private boolean stopped = false;
 
   static final byte [] ROOT_REGION_NAME =
@@ -158,33 +153,13 @@ public CatalogTracker(final Configuration conf) throws IOException {
    * @throws IOException 
    */
   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
-      final Abortable abortable)
+      Abortable abortable)
   throws IOException {
-    this(zk, conf, abortable,
-      conf.getInt("hbase.catalogtracker.default.timeout", 1000));
-  }
-
-  /**
-   * Constructs the catalog tracker.  Find current state of catalog tables.
-   * Begin active tracking by executing {@link #start()} post construction.
-   * @param zk If zk is null, we'll create an instance (and shut it down
-   * when {@link #stop()} is called) else we'll use what is passed.
-   * @param conf
-   * @param abortable If fatal exception we'll call abort on this.  May be null.
-   * If it is we'll use the Connection associated with the passed
-   * {@link Configuration} as our Abortable.
-   * @param defaultTimeout Timeout to use.  Pass zero for no timeout
-   * ({@link Object#wait(long)} when passed a <code>0</code> waits for ever).
-   * @throws IOException
-   */
-  public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
-      Abortable abortable, final int defaultTimeout)
-  throws IOException {
-    this(zk, conf, HConnectionManager.getConnection(conf), abortable, defaultTimeout);
+    this(zk, conf, HConnectionManager.getConnection(conf), abortable);
   }
 
   CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
-      HConnection connection, Abortable abortable, final int defaultTimeout)
+      HConnection connection, Abortable abortable)
   throws IOException {
     this.connection = connection;
     if (abortable == null) {
@@ -222,7 +197,6 @@ public void nodeDeleted(String path) {
         ct.resetMetaLocation();
       }
     };
-    this.defaultTimeout = defaultTimeout;
   }
 
   /**
@@ -369,24 +343,6 @@ HRegionInterface getRootServerConnection(long timeout)
     return getCachedConnection(waitForRoot(timeout));
   }
 
-  /**
-   * Gets a connection to the server hosting root, as reported by ZooKeeper,
-   * waiting for the default timeout specified on instantiation.
-   * @see #waitForRoot(long) for additional information
-   * @return connection to server hosting root
-   * @throws NotAllMetaRegionsOnlineException if timed out waiting
-   * @throws IOException
-   * @deprecated Use #getRootServerConnection(long)
-   */
-  public HRegionInterface waitForRootServerConnectionDefault()
-  throws NotAllMetaRegionsOnlineException, IOException {
-    try {
-      return getRootServerConnection(this.defaultTimeout);
-    } catch (InterruptedException e) {
-      throw new NotAllMetaRegionsOnlineException("Interrupted");
-    }
-  }
-
   /**
    * Gets a connection to the server currently hosting <code>.META.</code> or
    * null if location is not currently available.
@@ -476,10 +432,10 @@ public void waitForMeta() throws InterruptedException {
    */
   public ServerName waitForMeta(long timeout)
   throws InterruptedException, IOException, NotAllMetaRegionsOnlineException {
-    long stop = System.currentTimeMillis() + timeout;
+    long stop = timeout == 0 ? Long.MAX_VALUE : System.currentTimeMillis() + timeout;
     long waitTime = Math.min(50, timeout);
     synchronized (metaAvailable) {
-      while(!stopped && (timeout == 0 || System.currentTimeMillis() < stop)) {
+      while(!stopped && System.currentTimeMillis() < stop) {
         if (getMetaServerConnection() != null) {
           return metaLocation;
         }
@@ -508,25 +464,6 @@ public HRegionInterface waitForMetaServerConnection(long timeout)
     return getCachedConnection(waitForMeta(timeout));
   }
 
-  /**
-   * Gets a connection to the server hosting meta, as reported by ZooKeeper,
-   * waiting up to the specified timeout for availability.
-   * Used in tests.
-   * @see #waitForMeta(long) for additional information
-   * @return connection to server hosting meta
-   * @throws NotAllMetaRegionsOnlineException if timed out or interrupted
-   * @throws IOException
-   * @deprecated Does not retry; use an HTable instance instead.
-   */
-  public HRegionInterface waitForMetaServerConnectionDefault()
-  throws NotAllMetaRegionsOnlineException, IOException {
-    try {
-      return getCachedConnection(waitForMeta(defaultTimeout));
-    } catch (InterruptedException e) {
-      throw new NotAllMetaRegionsOnlineException("Interrupted");
-    }
-  }
-
   /**
    * Called when we figure current meta is off (called from zk callback).
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index d439b2687261..c68063001d16 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -441,8 +441,7 @@ private boolean becomeActiveMaster(MonitoredTask startupStatus)
    */
   private void initializeZKBasedSystemTrackers() throws IOException,
       InterruptedException, KeeperException {
-    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,
-        this, conf.getInt("hbase.master.catalog.timeout", Integer.MAX_VALUE));
+    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
     this.catalogTracker.start();
 
     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e6be923edd60..5424e067a483 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -622,8 +622,7 @@ private void initializeZooKeeper() throws IOException, InterruptedException {
     blockAndCheckIfStopped(this.clusterStatusTracker);
 
     // Create the catalog tracker and start it;
-    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,
-      this, this.conf.getInt("hbase.regionserver.catalog.timeout", Integer.MAX_VALUE));
+    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
     catalogTracker.start();
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
index 7a1bfae262ae..5e6fbd7d1566 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
@@ -103,7 +103,7 @@ public boolean isAborted()  {
   private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
   throws IOException, InterruptedException {
     CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(),
-      c, this.abortable, 0);
+      c, this.abortable);
     ct.start();
     return ct;
   }
@@ -213,10 +213,8 @@ public void testServerNotRunningIOException()
           @Override
           public void run() {
             try {
-              metaSet.set(ct.waitForMetaServerConnectionDefault() !=  null);
-            } catch (NotAllMetaRegionsOnlineException e) {
-              throw new RuntimeException(e);
-            } catch (IOException e) {
+              metaSet.set(ct.waitForMetaServerConnection(100000) !=  null);
+            } catch (Exception e) {
               throw new RuntimeException(e);
             }
           }
diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
index 00424683f932..2003c6278c1c 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
@@ -153,7 +153,7 @@ public void testRideOverServerNotRunning() throws IOException, InterruptedExcept
         when(connection).getHRegionConnection(Mockito.anyString(), Mockito.anyInt());
 
       // Now start up the catalogtracker with our doctored Connection.
-      ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0);
+      ct = new CatalogTracker(zkw, null, connection, ABORTABLE);
       ct.start();
       // Scan meta for user tables and verify we got back expected answer.
       NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn);
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 45f106ce63e5..7a51f6c354d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -101,7 +101,7 @@ class MockServer implements Server {
       this.ct = Mockito.mock(CatalogTracker.class);
       HRegionInterface hri = Mockito.mock(HRegionInterface.class);
       Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
-      Mockito.when(ct.waitForMetaServerConnectionDefault()).thenReturn(hri);
+      Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
     }
 
     @Override

From 10969196f9b4c53e231c638adb82710c4bf4e013 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 17 Jan 2013 01:41:47 +0000
Subject: [PATCH 0711/1540] HBASE-7592 HConnectionManager.getHTableDescriptor()
 compares too much

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434532 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HConnectionManager.java | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index fe775cd44616..d12c008ccf49 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1839,20 +1839,16 @@ public HTableDescriptor getHTableDescriptor(final byte[] tableName)
       if (this.master == null) {
         this.master = getMaster();
       }
-      HTableDescriptor hTableDescriptor = null;
+
       HTableDescriptor[] htds = master.getHTableDescriptors();
       if (htds != null && htds.length > 0) {
         for (HTableDescriptor htd: htds) {
           if (Bytes.equals(tableName, htd.getName())) {
-            hTableDescriptor = htd;
+            return htd;
           }
         }
       }
-      //HTableDescriptor htd = master.getHTableDescriptor(tableName);
-      if (hTableDescriptor == null) {
-        throw new TableNotFoundException(Bytes.toString(tableName));
-      }
-      return hTableDescriptor;
+      throw new TableNotFoundException(Bytes.toString(tableName));
     }
   }
 

From 0784efa9b79f7dad6cf86553d7b796bc93ae19d3 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 17 Jan 2013 17:33:25 +0000
Subject: [PATCH 0712/1540] HBASE-7034 Bad version, failed OPENING to OPENED
 but master thinks it is open anyways (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434797 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/zookeeper/RecoverableZooKeeper.java |   7 +-
 .../zookeeper/TestRecoverableZooKeeper.java   | 123 ++++++++++++++++++
 2 files changed, 124 insertions(+), 6 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 3ca866ccc347..851d650edf43 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -359,12 +359,7 @@ public Stat setData(String path, byte[] data, int version)
             try{
               Stat stat = new Stat();
               byte[] revData = zk.getData(path, false, stat);
-              int idLength = Bytes.toInt(revData, ID_LENGTH_SIZE);
-              int dataLength = revData.length-ID_LENGTH_SIZE-idLength;
-              int dataOffset = ID_LENGTH_SIZE+idLength;
-              
-              if(Bytes.compareTo(revData, ID_LENGTH_SIZE, id.length, 
-                  revData, dataOffset, dataLength) == 0) {
+              if (Bytes.equals(revData, newData)) {
                 // the bad version is caused by previous successful setData
                 return stat;
               }
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
new file mode 100644
index 000000000000..a7b0c4b34c72
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestRecoverableZooKeeper {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  Abortable abortable = new Abortable() {
+    @Override
+    public void abort(String why, Throwable e) {
+
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+  };
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  @Test
+  public void testSetDataVersionMismatchInLoop() throws Exception {
+    String znode = "/hbase/unassigned/9af7cfc9b15910a0b3d714bf40a3248f";
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Properties properties = ZKConfig.makeZKProps(conf);
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testSetDataVersionMismatchInLoop",
+        abortable, true);
+    String ensemble = ZKConfig.getZKQuorumServersString(properties);
+    RecoverableZooKeeper rzk = ZKUtil.connect(conf, ensemble, zkw);
+    rzk.create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+    rzk.setData(znode, "OPENING".getBytes(), 0);
+    Field zkField = RecoverableZooKeeper.class.getDeclaredField("zk");
+    zkField.setAccessible(true);
+    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
+    ZookeeperStub zkStub = new ZookeeperStub(ensemble, timeout, zkw);
+    zkStub.setThrowExceptionInNumOperations(1);
+    zkField.set(rzk, zkStub);
+    byte[] opened = "OPENED".getBytes();
+    rzk.setData(znode, opened, 1);
+    byte[] data = rzk.getData(znode, false, new Stat());
+    assertTrue(Bytes.equals(opened, data));
+  }
+
+  class ZookeeperStub extends ZooKeeper {
+
+    private int throwExceptionInNumOperations;
+
+    public ZookeeperStub(String connectString, int sessionTimeout, Watcher watcher)
+        throws IOException {
+      super(connectString, sessionTimeout, watcher);
+    }
+
+    public void setThrowExceptionInNumOperations(int throwExceptionInNumOperations) {
+      this.throwExceptionInNumOperations = throwExceptionInNumOperations;
+    }
+
+    private void checkThrowKeeperException() throws KeeperException {
+      if (throwExceptionInNumOperations == 1) {
+        throwExceptionInNumOperations = 0;
+        throw new KeeperException.ConnectionLossException();
+      }
+      if (throwExceptionInNumOperations > 0)
+        throwExceptionInNumOperations--;
+    }
+
+    @Override
+    public Stat setData(String path, byte[] data, int version) throws KeeperException,
+        InterruptedException {
+      Stat stat = super.setData(path, data, version);
+      checkThrowKeeperException();
+      return stat;
+    }
+  }
+}

From 12b511c513da9ec9fe4f17847b1c0ad27a7a633a Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 17 Jan 2013 18:22:54 +0000
Subject: [PATCH 0713/1540] HBASE-7585. TestAccessController tests should close
 HTables

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434847 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/TestAccessController.java | 480 +++++++++++++-----
 1 file changed, 360 insertions(+), 120 deletions(-)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 3425cd8a6192..a2420d1a9f29 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -145,31 +145,35 @@ public static void setupBeforeClass() throws Exception {
     htd.setOwner(USER_OWNER);
     admin.createTable(htd);
 
-    // initilize access control
-    HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol = meta.coprocessorProxy(AccessControllerProtocol.class,
-      TEST_TABLE);
-
     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0);
     RegionCoprocessorHost rcpHost = region.getCoprocessorHost();
     RCP_ENV = rcpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER,
       Coprocessor.PRIORITY_HIGHEST, 1, conf);
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()),
+    // initilize access control
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        TEST_TABLE);
+
+     protocol.grant(new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()),
         Permission.Action.ADMIN, Permission.Action.CREATE, Permission.Action.READ,
         Permission.Action.WRITE));
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_RW.getShortName()), TEST_TABLE,
+      protocol.grant(new UserPermission(Bytes.toBytes(USER_RW.getShortName()), TEST_TABLE,
         TEST_FAMILY, Permission.Action.READ, Permission.Action.WRITE));
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
+      protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
         TEST_FAMILY, Permission.Action.READ));
 
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_CREATE.getShortName()), TEST_TABLE, null,
+      protocol.grant(new UserPermission(Bytes.toBytes(USER_CREATE.getShortName()), TEST_TABLE, null,
         Permission.Action.CREATE));
     
-    protocol.grant(new UserPermission(Bytes.toBytes(USER_RW_ON_TABLE.getShortName()), TEST_TABLE,
-      null, Permission.Action.READ, Permission.Action.WRITE));
+      protocol.grant(new UserPermission(Bytes.toBytes(USER_RW_ON_TABLE.getShortName()), TEST_TABLE,
+        null, Permission.Action.READ, Permission.Action.WRITE));
+    } finally {
+      acl.close();
+    }
   }
 
   @AfterClass
@@ -374,8 +378,13 @@ public Object run() throws Exception {
 
   @Test
   public void testMove() throws Exception {
+    Map<HRegionInfo, HServerAddress> regions;
     HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE);
-    Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+    try {
+      regions = table.getRegionsInfo();
+    } finally {
+      table.close();
+    }
     final Map.Entry<HRegionInfo, HServerAddress> firstRegion = regions.entrySet().iterator().next();
     final ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
@@ -392,8 +401,13 @@ public Object run() throws Exception {
 
   @Test
   public void testAssign() throws Exception {
+    Map<HRegionInfo, HServerAddress> regions;
     HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE);
-    Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+    try {
+      regions = table.getRegionsInfo();
+    } finally {
+      table.close();
+    }
     final Map.Entry<HRegionInfo, HServerAddress> firstRegion = regions.entrySet().iterator().next();
 
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
@@ -410,8 +424,13 @@ public Object run() throws Exception {
 
   @Test
   public void testUnassign() throws Exception {
+    Map<HRegionInfo, HServerAddress> regions;
     HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE);
-    Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+    try {
+      regions = table.getRegionsInfo();
+    } finally {
+      table.close();
+    }
     final Map.Entry<HRegionInfo, HServerAddress> firstRegion = regions.entrySet().iterator().next();
 
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
@@ -553,7 +572,11 @@ public Object run() throws Exception {
         Get g = new Get(Bytes.toBytes("random_row"));
         g.addFamily(TEST_FAMILY);
         HTable t = new HTable(conf, TEST_TABLE);
-        t.get(g);
+        try {
+          t.get(g);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -566,14 +589,18 @@ public Object run() throws Exception {
         s.addFamily(TEST_FAMILY);
 
         HTable table = new HTable(conf, TEST_TABLE);
-        ResultScanner scanner = table.getScanner(s);
         try {
-          for (Result r = scanner.next(); r != null; r = scanner.next()) {
-            // do nothing
+          ResultScanner scanner = table.getScanner(s);
+          try {
+            for (Result r = scanner.next(); r != null; r = scanner.next()) {
+              // do nothing
+            }
+          } catch (IOException e) {
+          } finally {
+            scanner.close();
           }
-        } catch (IOException e) {
         } finally {
-          scanner.close();
+          table.close();
         }
         return null;
       }
@@ -590,7 +617,11 @@ public Object run() throws Exception {
         Put p = new Put(Bytes.toBytes("random_row"));
         p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1));
         HTable t = new HTable(conf, TEST_TABLE);
-        t.put(p);
+        try {
+          t.put(p);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -602,7 +633,11 @@ public Object run() throws Exception {
         Delete d = new Delete(Bytes.toBytes("random_row"));
         d.deleteFamily(TEST_FAMILY);
         HTable t = new HTable(conf, TEST_TABLE);
-        t.delete(d);
+        try {
+          t.delete(d);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -614,7 +649,11 @@ public Object run() throws Exception {
         Increment inc = new Increment(Bytes.toBytes("random_row"));
         inc.addColumn(TEST_FAMILY, Bytes.toBytes("Qualifier"), 1);
         HTable t = new HTable(conf, TEST_TABLE);
-        t.increment(inc);
+        try {
+          t.increment(inc);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -628,10 +667,13 @@ public void testReadWrite() throws Exception {
       public Object run() throws Exception {
         Delete d = new Delete(Bytes.toBytes("random_row"));
         d.deleteFamily(TEST_FAMILY);
-
         HTable t = new HTable(conf, TEST_TABLE);
-        t.checkAndDelete(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
-          Bytes.toBytes("test_value"), d);
+        try {
+          t.checkAndDelete(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
+            Bytes.toBytes("test_value"), d);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -640,11 +682,15 @@ public Object run() throws Exception {
     // action for checkAndPut()
     PrivilegedExceptionAction checkAndPut = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
-        HTable t = new HTable(conf, TEST_TABLE);
         Put p = new Put(Bytes.toBytes("random_row"));
         p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1));
-        t.checkAndPut(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
-          Bytes.toBytes("test_value"), p);
+        HTable t = new HTable(conf, TEST_TABLE);
+        try {
+          t.checkAndPut(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"),
+           Bytes.toBytes("test_value"), p);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -735,9 +781,13 @@ private void bulkLoadHFile(
       setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));
 
       HTable table = new HTable(conf, tableName);
-      TEST_UTIL.waitTableAvailable(tableName, 30000);
-      LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
-      loader.doBulkLoad(loadPath, table);
+      try {
+        TEST_UTIL.waitTableAvailable(tableName, 30000);
+        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
+        loader.doBulkLoad(loadPath, table);
+      } finally {
+        table.close();
+      }
     }
 
     public void setPermission(Path dir, FsPermission perm) throws IOException {
@@ -760,13 +810,17 @@ public void testAppend() throws Exception {
       public Object run() throws Exception {
         byte[] row = Bytes.toBytes("random_row");
         byte[] qualifier = Bytes.toBytes("q");
-        HTable t = new HTable(conf, TEST_TABLE);
         Put put = new Put(row);
         put.add(TEST_FAMILY, qualifier, Bytes.toBytes(1));
-        t.put(put);
         Append append = new Append(row);
         append.add(TEST_FAMILY, qualifier, Bytes.toBytes(2));
-        t.append(append);
+        HTable t = new HTable(conf, TEST_TABLE);
+        try {
+          t.put(put);
+          t.append(append);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -781,10 +835,14 @@ public void testGrantRevoke() throws Exception {
     PrivilegedExceptionAction grantAction = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-        AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-          TEST_TABLE);
-        protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
+        try {
+          AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+            TEST_TABLE);
+          protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
             TEST_FAMILY, (byte[]) null, Action.READ));
+        } finally {
+          acl.close();
+        }
         return null;
       }
     };
@@ -792,10 +850,14 @@ public Object run() throws Exception {
     PrivilegedExceptionAction revokeAction = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-        AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-          TEST_TABLE);
-        protocol.revoke(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
+        try {
+          AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+            TEST_TABLE);
+          protocol.revoke(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE,
             TEST_FAMILY, (byte[]) null, Action.READ));
+        } finally {
+          acl.close();
+        }
         return null;
       }
     };
@@ -803,9 +865,13 @@ public Object run() throws Exception {
     PrivilegedExceptionAction getPermissionsAction = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-        AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-          TEST_TABLE);
-        protocol.getUserPermissions(TEST_TABLE);
+        try {
+          AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+            TEST_TABLE);
+          protocol.getUserPermissions(TEST_TABLE);
+        } finally {
+          acl.close();
+        }
         return null;
       }
     };
@@ -844,11 +910,6 @@ public void testPostGrantRevoke() throws Exception {
     User gblUser = User
         .createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]);
 
-    // perms only stored against the first region
-    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-      tableName);
-
     // prepare actions:
     PrivilegedExceptionAction putActionAll = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
@@ -856,7 +917,11 @@ public Object run() throws Exception {
         p.add(family1, qualifier, Bytes.toBytes("v1"));
         p.add(family2, qualifier, Bytes.toBytes("v2"));
         HTable t = new HTable(conf, tableName);
-        t.put(p);
+        try {
+          t.put(p);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -865,7 +930,11 @@ public Object run() throws Exception {
         Put p = new Put(Bytes.toBytes("a"));
         p.add(family1, qualifier, Bytes.toBytes("v1"));
         HTable t = new HTable(conf, tableName);
-        t.put(p);
+        try {
+          t.put(p);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -874,7 +943,11 @@ public Object run() throws Exception {
         Put p = new Put(Bytes.toBytes("a"));
         p.add(family2, qualifier, Bytes.toBytes("v2"));
         HTable t = new HTable(conf, tableName);
-        t.put(p);
+        try {
+          t.put(p);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -884,7 +957,11 @@ public Object run() throws Exception {
         g.addFamily(family1);
         g.addFamily(family2);
         HTable t = new HTable(conf, tableName);
-        t.get(g);
+        try {
+          t.get(g);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -893,7 +970,11 @@ public Object run() throws Exception {
         Get g = new Get(Bytes.toBytes("random_row"));
         g.addFamily(family1);
         HTable t = new HTable(conf, tableName);
-        t.get(g);
+        try {
+          t.get(g);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -902,7 +983,11 @@ public Object run() throws Exception {
         Get g = new Get(Bytes.toBytes("random_row"));
         g.addFamily(family2);
         HTable t = new HTable(conf, tableName);
-        t.get(g);
+        try {
+          t.get(g);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -912,7 +997,11 @@ public Object run() throws Exception {
         d.deleteFamily(family1);
         d.deleteFamily(family2);
         HTable t = new HTable(conf, tableName);
-        t.delete(d);
+        try {
+          t.delete(d);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -921,7 +1010,11 @@ public Object run() throws Exception {
         Delete d = new Delete(Bytes.toBytes("random_row"));
         d.deleteFamily(family1);
         HTable t = new HTable(conf, tableName);
-        t.delete(d);
+        try {
+          t.delete(d);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -930,7 +1023,11 @@ public Object run() throws Exception {
         Delete d = new Delete(Bytes.toBytes("random_row"));
         d.deleteFamily(family2);
         HTable t = new HTable(conf, tableName);
-        t.delete(d);
+        try {
+          t.delete(d);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -945,12 +1042,20 @@ public Object run() throws Exception {
     verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant table read permission
-    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+        Permission.Action.READ));
+      protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
         Permission.Action.READ));
-    protocol
-        .grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
+    } finally {
+      acl.close();
+    }
 
     Thread.sleep(100);
+
     // check
     verifyAllowed(tblUser, getActionAll, getAction1, getAction2);
     verifyDenied(tblUser, putActionAll, putAction1, putAction2);
@@ -961,10 +1066,18 @@ public Object run() throws Exception {
     verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant table write permission
-    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
         Permission.Action.WRITE));
-    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
+      protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
         Permission.Action.WRITE));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     verifyDenied(tblUser, getActionAll, getAction1, getAction2);
@@ -976,10 +1089,18 @@ public Object run() throws Exception {
     verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // revoke table permission
-    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null,
         Permission.Action.READ, Permission.Action.WRITE));
-    protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null));
-    protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName())));
+      protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null));
+      protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName())));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     verifyDenied(tblUser, getActionAll, getAction1, getAction2);
@@ -991,10 +1112,17 @@ public Object run() throws Exception {
     verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant column family read permission
-    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family1,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family1,
+        Permission.Action.READ));
+      protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
         Permission.Action.READ));
-    protocol
-        .grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), Permission.Action.READ));
+    } finally {
+      acl.close();
+    }
 
     Thread.sleep(100);
 
@@ -1009,10 +1137,18 @@ public Object run() throws Exception {
     verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // grant column family write permission
-    protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2,
         Permission.Action.WRITE));
-    protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
+      protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()),
         Permission.Action.WRITE));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     // READ from family1, WRITE to family2 are allowed
@@ -1027,8 +1163,15 @@ public Object run() throws Exception {
     verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2);
 
     // revoke column family permission
-    protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2));
-    protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName())));
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2));
+      protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName())));
+    } finally {
+      acl.close();
+    }
 
     Thread.sleep(100);
 
@@ -1061,7 +1204,6 @@ public void testPostGrantRevokeAtQualifierLevel() throws Exception {
 
     // create table
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
-
     if (admin.tableExists(tableName)) {
       admin.disableTable(tableName);
       admin.deleteTable(tableName);
@@ -1074,16 +1216,16 @@ public void testPostGrantRevokeAtQualifierLevel() throws Exception {
     // create temp users
     User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]);
 
-    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-      tableName);
-
     PrivilegedExceptionAction getQualifierAction = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         Get g = new Get(Bytes.toBytes("random_row"));
         g.addColumn(family1, qualifier);
         HTable t = new HTable(conf, tableName);
-        t.get(g);
+        try {
+          t.get(g);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -1092,7 +1234,11 @@ public Object run() throws Exception {
         Put p = new Put(Bytes.toBytes("random_row"));
         p.add(family1, qualifier, Bytes.toBytes("v1"));
         HTable t = new HTable(conf, tableName);
-        t.put(p);
+        try {
+          t.put(p);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
@@ -1102,18 +1248,38 @@ public Object run() throws Exception {
         d.deleteColumn(family1, qualifier);
         // d.deleteFamily(family1);
         HTable t = new HTable(conf, tableName);
-        t.delete(d);
+        try {
+          t.delete(d);
+        } finally {
+          t.close();
+        }
         return null;
       }
     };
 
-    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1));
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1));
+    } finally {
+      acl.close();
+    }
+
     verifyDenied(user, getQualifierAction);
     verifyDenied(user, putQualifierAction);
     verifyDenied(user, deleteQualifierAction);
 
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
         qualifier, Permission.Action.READ));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     verifyAllowed(user, getQualifierAction);
@@ -1122,8 +1288,16 @@ public Object run() throws Exception {
 
     // only grant write permission
     // TODO: comment this portion after HBASE-3583
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
         qualifier, Permission.Action.WRITE));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     verifyDenied(user, getQualifierAction);
@@ -1131,8 +1305,16 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteQualifierAction);
 
     // grant both read and write permission.
-    protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
         qualifier, Permission.Action.READ, Permission.Action.WRITE));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     verifyAllowed(user, getQualifierAction);
@@ -1140,8 +1322,16 @@ public Object run() throws Exception {
     verifyAllowed(user, deleteQualifierAction);
 
     // revoke family level permission won't impact column level.
-    protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1,
         qualifier));
+    } finally {
+      acl.close();
+    }
+
     Thread.sleep(100);
 
     verifyDenied(user, getQualifierAction);
@@ -1173,11 +1363,15 @@ public void testPermissionList() throws Exception {
     htd.setOwner(USER_OWNER);
     admin.createTable(htd);
 
+    List<UserPermission> perms;
     HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-      tableName);
-
-    List<UserPermission> perms = protocol.getUserPermissions(tableName);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      perms = protocol.getUserPermissions(tableName);
+    } finally {
+      acl.close();
+    }
 
     UserPermission ownerperm = new UserPermission(Bytes.toBytes(USER_OWNER.getName()), tableName,
         null, Action.values());
@@ -1192,8 +1386,16 @@ public void testPermissionList() throws Exception {
     // grant read permission
     UserPermission upToSet = new UserPermission(user, tableName, family1, qualifier,
         Permission.Action.READ);
-    protocol.grant(upToSet);
-    perms = protocol.getUserPermissions(tableName);
+
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(upToSet);
+      perms = protocol.getUserPermissions(tableName);
+    } finally {
+      acl.close();
+    }
 
     UserPermission upToVerify = new UserPermission(user, tableName, family1, qualifier,
         Permission.Action.READ);
@@ -1207,16 +1409,31 @@ public void testPermissionList() throws Exception {
     // grant read+write
     upToSet = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE,
         Permission.Action.READ);
-    protocol.grant(upToSet);
-    perms = protocol.getUserPermissions(tableName);
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.grant(upToSet);
+      perms = protocol.getUserPermissions(tableName);
+    } finally {
+      acl.close();
+    }
 
     upToVerify = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE,
         Permission.Action.READ);
     assertTrue("User should be granted permission: " + upToVerify.toString(),
       hasFoundUserPermission(upToVerify, perms));
 
-    protocol.revoke(upToSet);
-    perms = protocol.getUserPermissions(tableName);
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      protocol.revoke(upToSet);
+      perms = protocol.getUserPermissions(tableName);
+    } finally {
+      acl.close();
+    }
+
     assertFalse("User should not be granted permission: " + upToVerify.toString(),
       hasFoundUserPermission(upToVerify, perms));
 
@@ -1226,7 +1443,16 @@ public void testPermissionList() throws Exception {
     User newOwner = User.createUserForTesting(conf, "new_owner", new String[] {});
     htd.setOwner(newOwner);
     admin.modifyTable(tableName, htd);
-    perms = protocol.getUserPermissions(tableName);
+
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        tableName);
+      perms = protocol.getUserPermissions(tableName);
+    } finally {
+      acl.close();
+    }
+
     UserPermission newOwnerperm = new UserPermission(Bytes.toBytes(newOwner.getName()), tableName,
         null, Action.values());
     assertTrue("New owner should have all permissions on table",
@@ -1244,16 +1470,18 @@ private void verifyGlobal(PrivilegedExceptionAction<?> action) throws Exception
   }
 
   public void checkGlobalPerms(Permission.Action... actions) throws IOException {
-    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-      new byte[0]);
-
     Permission[] perms = new Permission[actions.length];
     for (int i = 0; i < actions.length; i++) {
       perms[i] = new Permission(actions[i]);
     }
-
-    protocol.checkPermissions(perms);
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        new byte[0]);
+      protocol.checkPermissions(perms);
+    } finally {
+      acl.close();
+    }
   }
 
   public void checkTablePerms(byte[] table, byte[] family, byte[] column,
@@ -1268,10 +1496,13 @@ public void checkTablePerms(byte[] table, byte[] family, byte[] column,
 
   public void checkTablePerms(byte[] table, Permission... perms) throws IOException {
     HTable acl = new HTable(conf, table);
-    AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-      new byte[0]);
-
-    protocol.checkPermissions(perms);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        new byte[0]);
+      protocol.checkPermissions(perms);
+    } finally {
+      acl.close();
+    }
   }
 
   public void grant(AccessControllerProtocol protocol, User user, byte[] t, byte[] f, byte[] q,
@@ -1281,10 +1512,6 @@ public void grant(AccessControllerProtocol protocol, User user, byte[] t, byte[]
 
   @Test
   public void testCheckPermissions() throws Exception {
-    final HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
-    final AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
-      TEST_TABLE);
-
     // --------------------------------------
     // test global permissions
     PrivilegedExceptionAction<Void> globalAdmin = new PrivilegedExceptionAction<Void>() {
@@ -1318,9 +1545,16 @@ public Void run() throws Exception {
     User userColumn = User.createUserForTesting(conf, "user_check_perms_family", new String[0]);
     User userQualifier = User.createUserForTesting(conf, "user_check_perms_q", new String[0]);
 
-    grant(protocol, userTable, TEST_TABLE, null, null, Permission.Action.READ);
-    grant(protocol, userColumn, TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ);
-    grant(protocol, userQualifier, TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ);
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        TEST_TABLE);
+      grant(protocol, userTable, TEST_TABLE, null, null, Permission.Action.READ);
+      grant(protocol, userColumn, TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ);
+      grant(protocol, userQualifier, TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ);
+    } finally {
+      acl.close();
+    }
 
     PrivilegedExceptionAction<Void> tableRead = new PrivilegedExceptionAction<Void>() {
       @Override
@@ -1405,15 +1639,21 @@ public Void run() throws Exception {
 
     // --------------------------------------
     // check for wrong table region
+    acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
     try {
-      // but ask for TablePermissions for TEST_TABLE
-      protocol.checkPermissions(new Permission[] { (Permission) new TablePermission(TEST_TABLE,
+      AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class,
+        TEST_TABLE);
+      try {
+        // but ask for TablePermissions for TEST_TABLE
+        protocol.checkPermissions(new Permission[] { (Permission) new TablePermission(TEST_TABLE,
           null, (byte[]) null, Permission.Action.CREATE) });
-      fail("this should have thrown CoprocessorException");
-    } catch (CoprocessorException ex) {
-      // expected
+        fail("this should have thrown CoprocessorException");
+      } catch (CoprocessorException ex) {
+        // expected
+      }
+    } finally {
+      acl.close();
     }
-
   }
 
   @Test

From 995a59332edb3f3b5ab3b1058746384434cb8df5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 17 Jan 2013 18:43:09 +0000
Subject: [PATCH 0714/1540] HBASE-7602 TestFromClientSide.testPoolBehavior is
 incorrect

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434856 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestFromClientSide.java      | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 6ab59d9d8aa7..aa0a7f2a1ffb 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4322,9 +4322,9 @@ public void testPoolBehavior() throws IOException, InterruptedException {
 
     // Build a SynchronousQueue that we use for thread coordination
     final SynchronousQueue<Object> queue = new SynchronousQueue<Object>();
-    List<Thread> threads = new ArrayList<Thread>(5);
+    List<Runnable> tasks = new ArrayList<Runnable>(5);
     for (int i = 0; i < 5; i++) {
-      threads.add(new Thread() {
+      tasks.add(new Runnable() {
         public void run() {
           try {
             // The thread blocks here until we decide to let it go
@@ -4333,25 +4333,28 @@ public void run() {
         }
       });
     }
-    // First, add two threads and make sure the pool size follows
-    pool.submit(threads.get(0));
+    // First, add two tasks and make sure the pool size follows
+    pool.submit(tasks.get(0));
     assertEquals(1, pool.getPoolSize());
-    pool.submit(threads.get(1));
+    pool.submit(tasks.get(1));
     assertEquals(2, pool.getPoolSize());
 
-    // Next, terminate those threads and then make sure the pool is still the
+    // Next, terminate those tasks and then make sure the pool is still the
     // same size
     queue.put(new Object());
-    threads.get(0).join();
     queue.put(new Object());
-    threads.get(1).join();
     assertEquals(2, pool.getPoolSize());
 
+    //ensure that ThreadPoolExecutor knows that tasks are finished.
+    while (pool.getCompletedTaskCount() < 2) {
+      Threads.sleep(1);
+    }
+
     // Now let's simulate adding a RS meaning that we'll go up to three
     // concurrent threads. The pool should not grow larger than three.
-    pool.submit(threads.get(2));
-    pool.submit(threads.get(3));
-    pool.submit(threads.get(4));
+    pool.submit(tasks.get(2));
+    pool.submit(tasks.get(3));
+    pool.submit(tasks.get(4));
     assertEquals(3, pool.getPoolSize());
     queue.put(new Object());
     queue.put(new Object());

From b94a818be07567917ba0aa1e4aeb15f99106844c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 17 Jan 2013 22:21:53 +0000
Subject: [PATCH 0715/1540] HBASE-7617
 TestHRegionOnCluster.testDataCorrectnessReplayingRecoveredEdits still fails
 occasionally.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1434937 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/TestHRegionOnCluster.java    | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
index 9b10426309fd..e0cf4155865d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java
@@ -81,8 +81,15 @@ public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
       int targetServerNum = (originServerNum + 1) % NUM_RS;
       HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
       assertFalse(originServer.equals(targetServer));
+
+      do {
+        Thread.sleep(10);
+      } while (!originServer.getServerName().equals(
+              cluster.getMaster().getAssignmentManager().getRegionServerOfRegion(regionInfo)));
+
       hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
           Bytes.toBytes(targetServer.getServerName().getServerName()));
+
       do {
         Thread.sleep(10);
       } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum ||

From 63c2ac7ad5b46032969c6e22e4b80b55b6b09fa9 Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Fri, 18 Jan 2013 19:35:35 +0000
Subject: [PATCH 0716/1540] HBASE-5458 Thread safety issues with
 Compression.Algorithm.GZ and CompressionTest

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1435317 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/Compression.java    | 108 ++++++++++++------
 1 file changed, 74 insertions(+), 34 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
index 0498ee6aaf38..3c06af19d294 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
@@ -97,35 +97,55 @@ private static ClassLoader getClassLoaderForCodec() {
   public static enum Algorithm {
     LZO("lzo") {
       // Use base type to avoid compile-time dependencies.
-      private transient CompressionCodec lzoCodec;
+      private volatile transient CompressionCodec lzoCodec;
+      private transient Object lock = new Object();
 
       @Override
       CompressionCodec getCodec(Configuration conf) {
         if (lzoCodec == null) {
-          try {
-            Class<?> externalCodec =
-                getClassLoaderForCodec().loadClass("com.hadoop.compression.lzo.LzoCodec");
-            lzoCodec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
-                new Configuration(conf));
-          } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
+          synchronized (lock) {
+            if (lzoCodec == null) {
+              lzoCodec = buildCodec(conf);
+            }
           }
         }
         return lzoCodec;
       }
+
+      private CompressionCodec buildCodec(Configuration conf) {
+        try {
+          Class<?> externalCodec =
+              ClassLoader.getSystemClassLoader()
+                  .loadClass("com.hadoop.compression.lzo.LzoCodec");
+          return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+              new Configuration(conf));
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
+      }
     },
     GZ("gz") {
-      private transient GzipCodec codec;
+      private volatile transient GzipCodec codec;
+      private transient Object lock = new Object();
 
       @Override
       DefaultCodec getCodec(Configuration conf) {
         if (codec == null) {
-          codec = new ReusableStreamGzipCodec();
-          codec.setConf(new Configuration(conf));
+          synchronized (lock) {
+            if (codec == null) {
+              codec = buildCodec(conf);
+            }
+          }
         }
 
         return codec;
       }
+
+      private GzipCodec buildCodec(Configuration conf) {
+        GzipCodec gzcodec = new ReusableStreamGzipCodec();
+        gzcodec.setConf(new Configuration(conf));
+        return gzcodec;
+      }
     },
 
     NONE("none") {
@@ -161,43 +181,63 @@ public synchronized OutputStream createCompressionStream(
       }
     },
     SNAPPY("snappy") {
-        // Use base type to avoid compile-time dependencies.
-        private transient CompressionCodec snappyCodec;
-
-        @Override
-        CompressionCodec getCodec(Configuration conf) {
-          if (snappyCodec == null) {
-            try {
-              Class<?> externalCodec =
-                  getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.SnappyCodec");
-              snappyCodec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
-                  conf);
-            } catch (ClassNotFoundException e) {
-              throw new RuntimeException(e);
+      // Use base type to avoid compile-time dependencies.
+      private volatile transient CompressionCodec snappyCodec;
+      private transient Object lock = new Object();
+
+      @Override
+      CompressionCodec getCodec(Configuration conf) {
+        if (snappyCodec == null) {
+          synchronized (lock) {
+            if (snappyCodec == null) {
+              snappyCodec = buildCodec(conf);
             }
           }
-          return snappyCodec;
         }
+        return snappyCodec;
+      }
+
+      private CompressionCodec buildCodec(Configuration conf) {
+        try {
+          Class<?> externalCodec =
+              ClassLoader.getSystemClassLoader()
+                  .loadClass("org.apache.hadoop.io.compress.SnappyCodec");
+          return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+              conf);
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
+      }
     },
     LZ4("lz4") {
       // Use base type to avoid compile-time dependencies.
-      private transient CompressionCodec lz4Codec;
+      private volatile transient CompressionCodec lz4Codec;
+      private transient Object lock = new Object();
 
       @Override
       CompressionCodec getCodec(Configuration conf) {
         if (lz4Codec == null) {
-          try {
-            Class<?> externalCodec =
-                getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.Lz4Codec");
-            lz4Codec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
-                conf);
-          } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
+          synchronized (lock) {
+            if (lz4Codec == null) {
+              lz4Codec = buildCodec(conf);
+            }
           }
+          buildCodec(conf);
         }
         return lz4Codec;
       }
-  };
+
+      private CompressionCodec buildCodec(Configuration conf) {
+        try {
+          Class<?> externalCodec =
+              getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.Lz4Codec");
+          return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+              conf);
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    };
 
     private final Configuration conf;
     private final String compressName;

From e30557a7b23f3e66259bce6bec129ac3bceb7e4b Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Mon, 21 Jan 2013 02:22:32 +0000
Subject: [PATCH 0717/1540] HBASE-7507 Make memstore flush be able to retry
 after exception (Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1436110 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   | 11 ++++
 .../hadoop/hbase/regionserver/Store.java      | 55 ++++++++++++++++++-
 2 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e70bd3be708c..14a66421c76c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -477,6 +477,17 @@ public static enum Modify {
    */
   public static long DEFAULT_HBASE_CLIENT_PAUSE = 1000;
 
+  /**
+   * Parameter name for server pause value, used mostly as value to wait before
+   * running a retry of a failed operation.
+   */
+  public static String HBASE_SERVER_PAUSE = "hbase.server.pause";
+
+  /**
+   * Default value of {@link #HBASE_SERVER_PAUSE}.
+   */
+  public static int DEFAULT_HBASE_SERVER_PAUSE = 1000;
+
   /**
    * Parameter name for maximum retries, used as maximum for all retryable
    * operations such as fetching of the root region from root region server,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index b895688b6e16..6a11a24040cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -166,6 +166,10 @@ public class Store extends SchemaConfigured implements HeapSize {
 
   private final Compactor compactor;
 
+  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
+  private static int flush_retries_number;
+  private static int pauseTime;
+
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -245,6 +249,17 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.bytesPerChecksum = getBytesPerChecksum(conf);
     // Create a compaction tool instance
     this.compactor = new Compactor(this.conf);
+    if (Store.flush_retries_number == 0) {
+      Store.flush_retries_number = conf.getInt(
+          "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
+      Store.pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE,
+          HConstants.DEFAULT_HBASE_SERVER_PAUSE);
+      if (Store.flush_retries_number <= 0) {
+        throw new IllegalArgumentException(
+            "hbase.hstore.flush.retries.number must be > 0, not "
+                + Store.flush_retries_number);
+      }
+    }
   }
 
   /**
@@ -722,8 +737,43 @@ private Path flushCache(final long logCacheFlushId,
     // If an exception happens flushing, we let it out without clearing
     // the memstore snapshot.  The old snapshot will be returned when we say
     // 'snapshot', the next time flush comes around.
-    return internalFlushCache(
-        snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);
+    // Retry after catching exception when flushing, otherwise server will abort
+    // itself
+    IOException lastException = null;
+    for (int i = 0; i < Store.flush_retries_number; i++) {
+      try {
+        Path pathName = internalFlushCache(snapshot, logCacheFlushId,
+            snapshotTimeRangeTracker, flushedSize, status);
+        try {
+          // Path name is null if there is no entry to flush
+          if (pathName != null) {
+            validateStoreFile(pathName);
+          }
+          return pathName;
+        } catch (Exception e) {
+          LOG.warn("Failed validating store file " + pathName
+              + ", retring num=" + i, e);
+          if (e instanceof IOException) {
+            lastException = (IOException) e;
+          } else {
+            lastException = new IOException(e);
+          }
+        }
+      } catch (IOException e) {
+        LOG.warn("Failed flushing store file, retring num=" + i, e);
+        lastException = e;
+      }
+      if (lastException != null) {
+        try {
+          Thread.sleep(pauseTime);
+        } catch (InterruptedException e) {
+          IOException iie = new InterruptedIOException();
+          iie.initCause(e);
+          throw iie;
+        }
+      }
+    }
+    throw lastException;
   }
 
   /*
@@ -842,7 +892,6 @@ private StoreFile commitFile(final Path path,
     // Write-out finished successfully, move into the right spot
     String fileName = path.getName();
     Path dstPath = new Path(homedir, fileName);
-    validateStoreFile(path);
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);

From daee9715f2bce619f99611c7f7bdbd6892d818a8 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 22 Jan 2013 02:07:20 +0000
Subject: [PATCH 0718/1540] HBASE-6669 Add BigDecimalColumnInterpreter for
 doing aggregations using AggregationClient (Anil Gupta)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1436726 13f79535-47bb-0310-9956-ffa450edef68
---
 .../BigDecimalColumnInterpreter.java          | 104 +++
 .../TestBigDecimalColumnInterpreter.java      | 670 ++++++++++++++++++
 2 files changed, 774 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
new file mode 100644
index 000000000000..d1594428041b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
@@ -0,0 +1,104 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client.coprocessor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * ColumnInterpreter for doing Aggregation's with BigDecimal columns. 
+ * This class is required at the RegionServer also.
+ *
+ */
+public class BigDecimalColumnInterpreter implements ColumnInterpreter<BigDecimal, BigDecimal> {
+  private static final Log log = LogFactory.getLog(BigDecimalColumnInterpreter.class);
+
+  @Override
+  public void readFields(DataInput arg0) throws IOException {
+  }
+
+  @Override
+  public void write(DataOutput arg0) throws IOException {
+  }
+
+  @Override
+  public BigDecimal getValue(byte[] family, byte[] qualifier, KeyValue kv)
+      throws IOException {
+    if ((kv == null || kv.getValue() == null)) return null;
+    return Bytes.toBigDecimal(kv.getValue()).setScale(2, RoundingMode.HALF_EVEN);
+  }
+
+  @Override
+  public BigDecimal add(BigDecimal val1, BigDecimal val2) {
+    if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) {
+      return ((val1 == null) ? val2 : val1);
+    }
+    if (val1 == null) return null;
+    return val1.add(val2).setScale(2, RoundingMode.HALF_EVEN);
+  }
+
+  @Override
+  public BigDecimal getMaxValue() {
+    return BigDecimal.valueOf(Double.MAX_VALUE);
+  }
+
+  @Override
+  public BigDecimal getMinValue() {
+    return BigDecimal.valueOf(Double.MIN_VALUE);
+  }
+
+  @Override
+  public BigDecimal multiply(BigDecimal val1, BigDecimal val2) {
+    return (((val1 == null) || (val2 == null)) ? null : val1.multiply(val2).setScale(2,
+      RoundingMode.HALF_EVEN));
+  }
+
+  @Override
+  public BigDecimal increment(BigDecimal val) {
+    return ((val == null) ? null : val.add(BigDecimal.ONE));
+  }
+
+  @Override
+  public BigDecimal castToReturnType(BigDecimal val) {
+    return val;
+  }
+
+  @Override
+  public int compare(BigDecimal val1, BigDecimal val2) {
+    if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) {
+      return ((val1 == null) ? -1 : 1);
+    }
+    if (val1 == null) return 0;
+    return val1.compareTo(val2);
+  }
+
+  @Override
+  public double divideForAvg(BigDecimal val1, Long paramLong) {
+    return (((paramLong == null) || (val1 == null)) ? (Double.NaN) :
+      val1.doubleValue() / paramLong.doubleValue());
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java
new file mode 100644
index 000000000000..62a0d00e83e2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java
@@ -0,0 +1,670 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import static org.junit.Assert.assertEquals;
+import java.math.BigDecimal;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
+import org.apache.hadoop.hbase.client.coprocessor.BigDecimalColumnInterpreter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * A test class to test BigDecimalColumnInterpreter for AggregationsProtocol
+ */
+@Category(MediumTests.class)
+public class TestBigDecimalColumnInterpreter {
+  protected static Log myLog = LogFactory.getLog(TestBigDecimalColumnInterpreter.class);
+
+  /**
+   * Creating the test infrastructure.
+   */
+  private static final byte[] TEST_TABLE = Bytes.toBytes("TestTable");
+  private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
+  private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
+  private static final byte[] TEST_MULTI_CQ = Bytes.toBytes("TestMultiCQ");
+
+  private static byte[] ROW = Bytes.toBytes("testRow");
+  private static final int ROWSIZE = 20;
+  private static final int rowSeperator1 = 5;
+  private static final int rowSeperator2 = 12;
+  private static byte[][] ROWS = makeN(ROW, ROWSIZE);
+
+  private static HBaseTestingUtility util = new HBaseTestingUtility();
+  private static Configuration conf = util.getConfiguration();
+
+  /**
+   * A set up method to start the test cluster. AggregateProtocolImpl is registered and will be
+   * loaded during region startup.
+   * @throws Exception
+   */
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+
+    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+      "org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
+
+    util.startMiniCluster(2);
+    HTable table = util.createTable(TEST_TABLE, TEST_FAMILY);
+    util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY, new byte[][] {
+        HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1], ROWS[rowSeperator2] });
+    /**
+     * The testtable has one CQ which is always populated and one variable CQ for each row rowkey1:
+     * CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2
+     */
+    for (int i = 0; i < ROWSIZE; i++) {
+      Put put = new Put(ROWS[i]);
+      put.setWriteToWAL(false);
+      BigDecimal bd = new BigDecimal(i);
+      put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(bd));
+      table.put(put);
+      Put p2 = new Put(ROWS[i]);
+      put.setWriteToWAL(false);
+      p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(bd)),
+        Bytes.toBytes(bd.multiply(new BigDecimal("0.10"))));
+      table.put(p2);
+    }
+    table.close();
+  }
+
+  /**
+   * Shutting down the cluster
+   * @throws Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    util.shutdownMiniCluster();
+  }
+
+  /**
+   * an infrastructure method to prepare rows for the testtable.
+   * @param base
+   * @param n
+   * @return
+   */
+  private static byte[][] makeN(byte[] base, int n) {
+    byte[][] ret = new byte[n][];
+    for (int i = 0; i < n; i++) {
+      ret[i] = Bytes.add(base, Bytes.toBytes(i));
+    }
+    return ret;
+  }
+
+  /**
+   * ****************** Test cases for Median **********************
+   */
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testMedianWithValidRange() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal median = aClient.median(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("8.00"), median);
+  }
+
+  /**
+   * ***************Test cases for Maximum *******************
+   */
+
+  /**
+   * give max for the entire table.
+   * @throws Throwable
+   */
+  @Test
+  public void testMax() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal maximum = aClient.max(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("19.00"), maximum);
+  }
+
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testMaxWithValidRange2() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[15]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal max = aClient.max(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("14.00"), max);
+  }
+
+  @Test
+  public void testMaxWithValidRangeWithoutCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal maximum = aClient.max(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("19.00"), maximum);
+  }
+
+  @Test
+  public void testMaxWithValidRange2WithoutCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[7]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal max = aClient.max(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("6.00"), max);
+  }
+
+  @Test
+  public void testMaxWithValidRangeWithNullCF() {
+    AggregationClient aClient = new AggregationClient(conf);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Scan scan = new Scan();
+    BigDecimal max = null;
+    try {
+      max = aClient.max(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+      max = null;
+    }
+    assertEquals(null, max);// CP will throw an IOException about the
+    // null column family, and max will be set to 0
+  }
+
+  @Test
+  public void testMaxWithInvalidRange() {
+    AggregationClient aClient = new AggregationClient(conf);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Scan scan = new Scan();
+    scan.setStartRow(ROWS[4]);
+    scan.setStopRow(ROWS[2]);
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    BigDecimal max = new BigDecimal(Long.MIN_VALUE);
+    ;
+    try {
+      max = aClient.max(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+      max = BigDecimal.ZERO;
+    }
+    assertEquals(BigDecimal.ZERO, max);// control should go to the catch block
+  }
+
+  @Test
+  public void testMaxWithInvalidRange2() throws Throwable {
+    BigDecimal max = new BigDecimal(Long.MIN_VALUE);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[4]);
+    scan.setStopRow(ROWS[4]);
+    try {
+      AggregationClient aClient = new AggregationClient(conf);
+      final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+      max = aClient.max(TEST_TABLE, ci, scan);
+    } catch (Exception e) {
+      max = BigDecimal.ZERO;
+    }
+    assertEquals(BigDecimal.ZERO, max);// control should go to the catch block
+  }
+
+  @Test
+  public void testMaxWithFilter() throws Throwable {
+    BigDecimal max = BigDecimal.ZERO;
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
+    scan.setFilter(f);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    max = aClient.max(TEST_TABLE, ci, scan);
+    assertEquals(null, max);
+  }
+
+  /**
+   * **************************Test cases for Minimum ***********************
+   */
+
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testMinWithValidRange() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(HConstants.EMPTY_START_ROW);
+    scan.setStopRow(HConstants.EMPTY_END_ROW);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = aClient.min(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("0.00"), min);
+  }
+
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testMinWithValidRange2() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[15]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = aClient.min(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("5.00"), min);
+  }
+
+  @Test
+  public void testMinWithValidRangeWithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(HConstants.EMPTY_START_ROW);
+    scan.setStopRow(HConstants.EMPTY_END_ROW);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = aClient.min(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("0.00"), min);
+  }
+
+  @Test
+  public void testMinWithValidRange2WithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[7]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = aClient.min(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("0.60"), min);
+  }
+
+  @Test
+  public void testMinWithValidRangeWithNullCF() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[15]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = null;
+    try {
+      min = aClient.min(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, min);// CP will throw an IOException about the
+    // null column family, and max will be set to 0
+  }
+
+  @Test
+  public void testMinWithInvalidRange() {
+    AggregationClient aClient = new AggregationClient(conf);
+    BigDecimal min = null;
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[4]);
+    scan.setStopRow(ROWS[2]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    try {
+      min = aClient.min(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, min);// control should go to the catch block
+  }
+
+  @Test
+  public void testMinWithInvalidRange2() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[6]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = null;
+    try {
+      min = aClient.min(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, min);// control should go to the catch block
+  }
+
+  @Test
+  public void testMinWithFilter() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
+    scan.setFilter(f);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal min = null;
+    min = aClient.min(TEST_TABLE, ci, scan);
+    assertEquals(null, min);
+  }
+
+  /**
+   * *************** Test cases for Sum *********************
+   */
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testSumWithValidRange() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("190.00"), sum);
+  }
+
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testSumWithValidRange2() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[15]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("95.00"), sum);
+  }
+
+  @Test
+  public void testSumWithValidRangeWithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("209.00"), sum); // 190 + 19
+  }
+
+  @Test
+  public void testSumWithValidRange2WithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[7]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan);
+    assertEquals(new BigDecimal("6.60"), sum); // 6 + 60
+  }
+
+  @Test
+  public void testSumWithValidRangeWithNullCF() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[7]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = null;
+    try {
+      sum = aClient.sum(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, sum);// CP will throw an IOException about the
+    // null column family, and max will be set to 0
+  }
+
+  @Test
+  public void testSumWithInvalidRange() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[2]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = null;
+    try {
+      sum = aClient.sum(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, sum);// control should go to the catch block
+  }
+
+  @Test
+  public void testSumWithFilter() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setFilter(f);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    BigDecimal sum = null;
+    sum = aClient.sum(TEST_TABLE, ci, scan);
+    assertEquals(null, sum);
+  }
+
+  /**
+   * ****************************** Test Cases for Avg **************
+   */
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testAvgWithValidRange() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double avg = aClient.avg(TEST_TABLE, ci, scan);
+    assertEquals(9.5, avg, 0);
+  }
+
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testAvgWithValidRange2() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[15]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double avg = aClient.avg(TEST_TABLE, ci, scan);
+    assertEquals(9.5, avg, 0);
+  }
+
+  @Test
+  public void testAvgWithValidRangeWithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double avg = aClient.avg(TEST_TABLE, ci, scan);
+    assertEquals(10.45, avg, 0.01);
+  }
+
+  @Test
+  public void testAvgWithValidRange2WithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[7]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double avg = aClient.avg(TEST_TABLE, ci, scan);
+    assertEquals(6 + 0.60, avg, 0);
+  }
+
+  @Test
+  public void testAvgWithValidRangeWithNullCF() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Double avg = null;
+    try {
+      avg = aClient.avg(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, avg);// CP will throw an IOException about the
+    // null column family, and max will be set to 0
+  }
+
+  @Test
+  public void testAvgWithInvalidRange() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[1]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Double avg = null;
+    try {
+      avg = aClient.avg(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, avg);// control should go to the catch block
+  }
+
+  @Test
+  public void testAvgWithFilter() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
+    scan.setFilter(f);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Double avg = null;
+    avg = aClient.avg(TEST_TABLE, ci, scan);
+    assertEquals(Double.NaN, avg, 0);
+  }
+
+  /**
+   * ****************** Test cases for STD **********************
+   */
+  /**
+   * @throws Throwable
+   */
+  @Test
+  public void testStdWithValidRange() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double std = aClient.std(TEST_TABLE, ci, scan);
+    assertEquals(5.766, std, 0.05d);
+  }
+
+  /**
+   * need to change this
+   * @throws Throwable
+   */
+  @Test
+  public void testStdWithValidRange2() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    scan.setStartRow(ROWS[5]);
+    scan.setStopRow(ROWS[15]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double std = aClient.std(TEST_TABLE, ci, scan);
+    assertEquals(2.87, std, 0.05d);
+  }
+
+  /**
+   * need to change this
+   * @throws Throwable
+   */
+  @Test
+  public void testStdWithValidRangeWithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double std = aClient.std(TEST_TABLE, ci, scan);
+    assertEquals(6.342, std, 0.05d);
+  }
+
+  @Test
+  public void testStdWithValidRange2WithNoCQ() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[7]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    double std = aClient.std(TEST_TABLE, ci, scan);
+    System.out.println("std is:" + std);
+    assertEquals(0, std, 0.05d);
+  }
+
+  @Test
+  public void testStdWithValidRangeWithNullCF() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[17]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Double std = null;
+    try {
+      std = aClient.std(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, std);// CP will throw an IOException about the
+    // null column family, and max will be set to 0
+  }
+
+  @Test
+  public void testStdWithInvalidRange() {
+    AggregationClient aClient = new AggregationClient(conf);
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setStartRow(ROWS[6]);
+    scan.setStopRow(ROWS[1]);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Double std = null;
+    try {
+      std = aClient.std(TEST_TABLE, ci, scan);
+    } catch (Throwable e) {
+    }
+    assertEquals(null, std);// control should go to the catch block
+  }
+
+  @Test
+  public void testStdWithFilter() throws Throwable {
+    AggregationClient aClient = new AggregationClient(conf);
+    Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
+    Scan scan = new Scan();
+    scan.addFamily(TEST_FAMILY);
+    scan.setFilter(f);
+    final ColumnInterpreter<BigDecimal, BigDecimal> ci = new BigDecimalColumnInterpreter();
+    Double std = null;
+    std = aClient.std(TEST_TABLE, ci, scan);
+    assertEquals(Double.NaN, std, 0);
+  }
+}

From 9b490477a67679df82445dd01c6f3f6fc9ac5011 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 22 Jan 2013 18:44:07 +0000
Subject: [PATCH 0719/1540] HBASE-7644 Port HBASE-4802 'Disable show table
 metrics in bulk loader' to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437091 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/metrics/SchemaConfigured.java   | 10 ++++------
 .../hbase/regionserver/metrics/SchemaMetrics.java      | 10 +++++++---
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
index 4821b53b2338..74939518fb90 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
@@ -74,12 +74,10 @@ public class SchemaConfigured implements HeapSize, SchemaAware {
 
   /** A helper constructor that configures the "use table name" flag. */
   private SchemaConfigured(Configuration conf) {
-    if (conf != null) {
-      SchemaMetrics.configureGlobally(conf);
-      // Even though we now know if table-level metrics are used, we can't
-      // initialize schemaMetrics yet, because CF and table name are only known
-      // to the calling constructor.
-    }
+    SchemaMetrics.configureGlobally(conf);
+    // Even though we now know if table-level metrics are used, we can't
+    // initialize schemaMetrics yet, because CF and table name are only known
+    // to the calling constructor.
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 9330834cc015..62d3f8e6db9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -584,9 +584,13 @@ public void updateBloomMetrics(boolean isInBloom) {
    * instantiating HFile readers/writers.
    */
   public static void configureGlobally(Configuration conf) {
-    final boolean useTableNameNew =
-        conf.getBoolean(SHOW_TABLE_NAME_CONF_KEY, false);
-    setUseTableName(useTableNameNew);
+    if (conf != null) {
+      final boolean useTableNameNew =
+          conf.getBoolean(SHOW_TABLE_NAME_CONF_KEY, false);
+      setUseTableName(useTableNameNew);
+    } else {
+      setUseTableName(false);
+    }
   }
 
   /**

From 8c94563b40dbfeb10465e555e15276a61ec3fb6a Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 22 Jan 2013 20:04:08 +0000
Subject: [PATCH 0720/1540] HBASE-7646 Make forkedProcessTimeoutInSeconds
 configurable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437132 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index e2238c8a41db..9a17f7e6112c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -375,7 +375,7 @@
           </dependencies>
 
           <configuration>
-            <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
+            <forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
             <argLine>-enableassertions -Xmx1900m -Djava.security.egd=file:/dev/./urandom</argLine>
             <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
           </configuration>
@@ -1049,6 +1049,7 @@
     <surefire.firstPartGroups>org.apache.hadoop.hbase.SmallTests</surefire.firstPartGroups>
     <surefire.secondPartGroups>org.apache.hadoop.hbase.MediumTests</surefire.secondPartGroups>
     <test.output.tofile>true</test.output.tofile>
+    <surefire.timeout>900</surefire.timeout>
   </properties>
 
   <!-- Sorted by groups of dependencies then groupId and artifactId -->

From d873d2897f50d72d0f440d0e972665ad6e549b6e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 22 Jan 2013 23:48:46 +0000
Subject: [PATCH 0721/1540] HBASE-7599 Port HBASE-6066 (low hanging read path
 improvements) to 0.94 (Devaraj Das)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437237 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 51 +++++++++++--------
 .../hbase/regionserver/HRegionServer.java     | 12 +++--
 2 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c79f9a9e0c84..f20275bfa5ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3520,7 +3520,6 @@ class RegionScannerImpl implements RegionScanner {
     private final KeyValue KV_LIMIT = new KeyValue();
     private final byte [] stopRow;
     private Filter filter;
-    private List<KeyValue> results = new ArrayList<KeyValue>();
     private int batch;
     private int isScan;
     private boolean filterClosed = false;
@@ -3635,11 +3634,16 @@ public boolean nextRaw(List<KeyValue> outResults, String metric)
     @Override
     public boolean nextRaw(List<KeyValue> outResults, int limit,
         String metric) throws IOException {
-      results.clear();
-
-      boolean returnResult = nextInternal(limit, metric);
-
-      outResults.addAll(results);
+      boolean returnResult;
+      if (outResults.isEmpty()) {
+        // Usually outResults is empty. This is true when next is called
+        // to handle scan or get operation.
+        returnResult = nextInternal(outResults, limit, metric);
+      } else {
+        List<KeyValue> tmpList = new ArrayList<KeyValue>();
+        returnResult = nextInternal(tmpList, limit, metric);
+        outResults.addAll(tmpList);
+      }
       resetFilters();
       if (isFilterDone()) {
         return false;
@@ -3661,10 +3665,12 @@ public boolean next(List<KeyValue> outResults, String metric)
       return next(outResults, batch, metric);
     }
 
-    private void populateFromJoinedHeap(int limit, String metric) throws IOException {
+    private void populateFromJoinedHeap(List<KeyValue> results, int limit, String metric)
+        throws IOException {
       assert joinedContinuationRow != null;
-      KeyValue kv = populateResult(this.joinedHeap, limit, joinedContinuationRow.getBuffer(),
-        joinedContinuationRow.getRowOffset(), joinedContinuationRow.getRowLength(), metric);
+      KeyValue kv = populateResult(results, this.joinedHeap, limit,
+        joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(),
+        joinedContinuationRow.getRowLength(), metric);
       if (kv != KV_LIMIT) {
         // We are done with this row, reset the continuation.
         joinedContinuationRow = null;
@@ -3676,6 +3682,7 @@ private void populateFromJoinedHeap(int limit, String metric) throws IOException
 
     /**
      * Fetches records with this row into result list, until next row or limit (if not -1).
+     * @param results
      * @param heap KeyValueHeap to fetch data from. It must be positioned on correct row before call.
      * @param limit Max amount of KVs to place in result list, -1 means no limit.
      * @param currentRow Byte array with key we are fetching.
@@ -3684,8 +3691,8 @@ private void populateFromJoinedHeap(int limit, String metric) throws IOException
      * @param metric Metric key to be passed into KeyValueHeap::next().
      * @return true if limit reached, false otherwise.
      */
-    private KeyValue populateResult(KeyValueHeap heap, int limit, byte[] currentRow, int offset,
-        short length, String metric) throws IOException {
+    private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int limit,
+        byte[] currentRow, int offset, short length, String metric) throws IOException {
       KeyValue nextKv;
       do {
         heap.next(results, limit - results.size(), metric);
@@ -3704,7 +3711,11 @@ public synchronized boolean isFilterDone() {
       return this.filter != null && this.filter.filterAllRemaining();
     }
 
-    private boolean nextInternal(int limit, String metric) throws IOException {
+    private boolean nextInternal(List<KeyValue> results, int limit, String metric)
+    throws IOException {
+      if (!results.isEmpty()) {
+        throw new IllegalArgumentException("First parameter should be an empty list");
+      }
       RpcCallContext rpcCall = HBaseServer.getCurrentCall();
       // The loop here is used only when at some point during the next we determine
       // that due to effects of filters or otherwise, we have an empty row in the result.
@@ -3750,11 +3761,13 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           // Techically, if we hit limits before on this row, we don't need this call.
           if (filterRowKey(currentRow, offset, length)) {
             nextRow(currentRow, offset, length);
+            results.clear();
             continue;
           }
 
           // Ok, we are good, let's try to get some results from the main heap.
-          KeyValue nextKv = populateResult(this.storeHeap, limit, currentRow, offset, length, metric);
+          KeyValue nextKv = populateResult(results, this.storeHeap, limit, currentRow, 
+              offset, length, metric);
           if (nextKv == KV_LIMIT) {
             if (this.filter != null && filter.hasFilterRow()) {
               throw new IncompatibleFilterException(
@@ -3773,13 +3786,8 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           }
 
           if (isEmptyRow || filterRow()) {
-            // this seems like a redundant step - we already consumed the row
-            // there're no left overs.
-            // the reasons for calling this method are:
-            // 1. reset the filters.
-            // 2. provide a hook to fast forward the row (used by subclasses)
             nextRow(currentRow, offset, length);
-
+            results.clear();
             // This row was totally filtered out, if this is NOT the last row,
             // we should continue on. Otherwise, nothing else to do.
             if (!stopRow) continue;
@@ -3799,12 +3807,12 @@ private boolean nextInternal(int limit, String metric) throws IOException {
               || this.joinedHeap.seek(KeyValue.createFirstOnRow(currentRow, offset, length));
             if (mayHaveData) {
               joinedContinuationRow = current;
-              populateFromJoinedHeap(limit, metric);
+              populateFromJoinedHeap(results, limit, metric);
             }
           }
         } else {
           // Populating from the joined map was stopped by limits, populate some more.
-          populateFromJoinedHeap(limit, metric);
+          populateFromJoinedHeap(results, limit, metric);
         }
 
         // We may have just called populateFromJoinedMap and hit the limits. If that is
@@ -3840,7 +3848,6 @@ protected void nextRow(byte [] currentRow, int offset, short length) throws IOEx
       while((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
         this.storeHeap.next(MOCKED_LIST);       
       }
-      results.clear();
       resetFilters();
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5424e067a483..f459c54211ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2457,8 +2457,10 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
             results, nbRows);
         if (!results.isEmpty()) {
           for (Result r : results) {
-            for (KeyValue kv : r.raw()) {
-              currentScanResultSize += kv.heapSize();
+            if (maxScannerResultSize < Long.MAX_VALUE){
+              for (KeyValue kv : r.raw()) {
+                currentScanResultSize += kv.heapSize();
+              }
             }
           }
         }
@@ -2478,8 +2480,10 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
             // Collect values to be returned here
             boolean moreRows = s.nextRaw(values, SchemaMetrics.METRIC_NEXTSIZE);
             if (!values.isEmpty()) {
-              for (KeyValue kv : values) {
-                currentScanResultSize += kv.heapSize();
+              if (maxScannerResultSize < Long.MAX_VALUE){
+                for (KeyValue kv : values) {
+                  currentScanResultSize += kv.heapSize();
+                }
               }
               results.add(new Result(values));
             }

From dea551999573828e626cd9d8a58852076b23780b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 22 Jan 2013 23:55:56 +0000
Subject: [PATCH 0722/1540] HBASE-7293 [replication] Remove dead sinks from
 ReplicationSource.currentPeers and pick new ones

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437238 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 866d92b203e4..304c5a2db687 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -22,6 +22,7 @@
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.ConnectException;
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -667,6 +668,9 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo) {
               "call to the remote cluster timed out, which is usually " +
               "caused by a machine failure or a massive slowdown",
               this.socketTimeoutMultiplier);
+          } else if (ioe instanceof ConnectException) {
+            LOG.warn("Peer is unavailable, rechecking all sinks: ", ioe);
+            chooseSinks();
           } else {
             LOG.warn("Can't replicate because of a local or network error: ", ioe);
           }

From 275f2f436d9cb1157b187bc9659b916ea0c28ead Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 23 Jan 2013 02:34:06 +0000
Subject: [PATCH 0723/1540] HBASE-7638 [0.94] region cache entry should only be
 removed on error if the error is from the server currently in cache (Sergey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437254 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 32 ++++++++++++-------
 .../apache/hadoop/hbase/client/TestHCM.java   | 28 +++++++++++++++-
 2 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index d12c008ccf49..6677eeed452f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -987,7 +987,7 @@ private HRegionLocation locateRegionInMeta(final byte [] parentTable,
                 return location;
               }
             } else {
-              deleteCachedLocation(tableName, row);
+              deleteCachedLocation(tableName, row, null);
             }
 
             // Query the root or meta region for the location of the meta region
@@ -1134,25 +1134,35 @@ HRegionLocation getCachedLocation(final byte [] tableName,
      * @param tableName tableName
      * @param row
      */
-    void deleteCachedLocation(final byte [] tableName, final byte [] row) {
+    void deleteCachedLocation(final byte[] tableName, final byte[] row, HRegionLocation source) {
+      boolean isStaleDelete = false;
+      HRegionLocation oldLocation = null;
       synchronized (this.cachedRegionLocations) {
         Map<byte[], HRegionLocation> tableLocations =
             getTableLocations(tableName);
         // start to examine the cache. we can only do cache actions
         // if there's something in the cache for this table.
         if (!tableLocations.isEmpty()) {
-          HRegionLocation rl = getCachedLocation(tableName, row);
-          if (rl != null) {
-            tableLocations.remove(rl.getRegionInfo().getStartKey());
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Removed " +
-                rl.getRegionInfo().getRegionNameAsString() +
-                " for tableName=" + Bytes.toString(tableName) +
-                " from cache " + "because of " + Bytes.toStringBinary(row));
+          oldLocation = getCachedLocation(tableName, row);
+          if (oldLocation != null) {
+            isStaleDelete = (source != null) && !oldLocation.equals(source);
+            if (!isStaleDelete) {
+              tableLocations.remove(oldLocation.getRegionInfo().getStartKey());
             }
           }
         }
       }
+      if (LOG.isDebugEnabled()) {
+        if (isStaleDelete) {
+          LOG.debug("Received an error from " + source.getHostnamePort() + " for ["
+            + oldLocation.getRegionInfo().getRegionNameAsString()  + "]; not removing "
+            + oldLocation.getHostnamePort() + " from cache.");
+        } else if (oldLocation != null) {
+          LOG.debug("Removed [" + oldLocation.getRegionInfo().getRegionNameAsString()
+            + "] for tableName=" + Bytes.toString(tableName) + " from cache because of ["
+            + Bytes.toStringBinary(row) + "]");
+        }
+      }
     }
 
     @Override
@@ -1593,7 +1603,7 @@ public <R> void processBatchCallback(
             actionCount++;
             Row row = list.get(i);
             workingList.add(row);
-            deleteCachedLocation(tableName, row.getRow());
+            deleteCachedLocation(tableName, row.getRow(), lastServers[i]);
           } else {
             if (results[i] != null && results[i] instanceof Throwable) {
               actionCount++;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 66548a1d7f92..369b54b69be2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -61,6 +61,7 @@ public class TestHCM {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final byte[] TABLE_NAME = Bytes.toBytes("test");
   private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
+  private static final byte[] TABLE_NAME2 = Bytes.toBytes("test2");
   private static final byte[] FAM_NAM = Bytes.toBytes("f");
   private static final byte[] ROW = Bytes.toBytes("bbb");
 
@@ -162,7 +163,7 @@ public void testRegionCaching() throws Exception{
     HConnectionManager.HConnectionImplementation conn =
         (HConnectionManager.HConnectionImplementation)table.getConnection();
     assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
-    conn.deleteCachedLocation(TABLE_NAME, ROW);
+    conn.deleteCachedLocation(TABLE_NAME, ROW, null);
     HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
     assertNull("What is this location?? " + rl, rl);
     table.close();
@@ -326,6 +327,31 @@ public void testCreateConnection() throws Exception {
     assertTrue(c2 != c3);
   }
 
+  @Test(timeout = 60000)
+  public void testDeleteFromCacheOnlyWorksFromTheSameSource() throws Exception{
+    HTable table = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
+    TEST_UTIL.createMultiRegions(table, FAM_NAM);
+    Put put = new Put(ROW);
+    put.add(FAM_NAM, ROW, ROW);
+    table.put(put);
+    HConnectionManager.HConnectionImplementation conn =
+        (HConnectionManager.HConnectionImplementation)table.getConnection();
+
+    HRegionLocation location = conn.getCachedLocation(TABLE_NAME2, ROW);
+    assertNotNull(location);
+    HRegionLocation anySource = new HRegionLocation(location.getRegionInfo(),
+        location.getHostname(), location.getPort() - 1);
+    conn.deleteCachedLocation(TABLE_NAME2, ROW, anySource);
+    location = conn.getCachedLocation(TABLE_NAME2, ROW);
+    assertNotNull(location); // different server, no delete.
+
+    conn.deleteCachedLocation(TABLE_NAME2, ROW, location);
+    location = conn.getCachedLocation(TABLE_NAME2, ROW);
+    assertNull(location); // same server, delete.
+
+    TEST_UTIL.deleteTable(TABLE_NAME2);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From a4b13ea4b4653be3666e017418f499dabb01edef Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 23 Jan 2013 16:22:58 +0000
Subject: [PATCH 0724/1540] HBASE-7648 TestAcidGuarantees.testMixedAtomicity
 hangs sometimes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437539 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
index ad4f8e5596f4..227d24387452 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -82,6 +83,9 @@ public TestAcidGuarantees() {
     // Set small flush size for minicluster so we exercise reseeking scanners
     Configuration conf = HBaseConfiguration.create();
     conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128*1024));
+    // prevent aggressive region split
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      ConstantSizeRegionSplitPolicy.class.getName());
     util = new HBaseTestingUtility(conf);
   }
   

From 9dd61e62390508f541c66e9c3938706f5d582608 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 24 Jan 2013 06:23:33 +0000
Subject: [PATCH 0725/1540] HBASE-7638 Revert, due to concerns of failing
 tests.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1437865 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 32 +++++++------------
 .../apache/hadoop/hbase/client/TestHCM.java   | 28 +---------------
 2 files changed, 12 insertions(+), 48 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 6677eeed452f..d12c008ccf49 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -987,7 +987,7 @@ private HRegionLocation locateRegionInMeta(final byte [] parentTable,
                 return location;
               }
             } else {
-              deleteCachedLocation(tableName, row, null);
+              deleteCachedLocation(tableName, row);
             }
 
             // Query the root or meta region for the location of the meta region
@@ -1134,35 +1134,25 @@ HRegionLocation getCachedLocation(final byte [] tableName,
      * @param tableName tableName
      * @param row
      */
-    void deleteCachedLocation(final byte[] tableName, final byte[] row, HRegionLocation source) {
-      boolean isStaleDelete = false;
-      HRegionLocation oldLocation = null;
+    void deleteCachedLocation(final byte [] tableName, final byte [] row) {
       synchronized (this.cachedRegionLocations) {
         Map<byte[], HRegionLocation> tableLocations =
             getTableLocations(tableName);
         // start to examine the cache. we can only do cache actions
         // if there's something in the cache for this table.
         if (!tableLocations.isEmpty()) {
-          oldLocation = getCachedLocation(tableName, row);
-          if (oldLocation != null) {
-            isStaleDelete = (source != null) && !oldLocation.equals(source);
-            if (!isStaleDelete) {
-              tableLocations.remove(oldLocation.getRegionInfo().getStartKey());
+          HRegionLocation rl = getCachedLocation(tableName, row);
+          if (rl != null) {
+            tableLocations.remove(rl.getRegionInfo().getStartKey());
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Removed " +
+                rl.getRegionInfo().getRegionNameAsString() +
+                " for tableName=" + Bytes.toString(tableName) +
+                " from cache " + "because of " + Bytes.toStringBinary(row));
             }
           }
         }
       }
-      if (LOG.isDebugEnabled()) {
-        if (isStaleDelete) {
-          LOG.debug("Received an error from " + source.getHostnamePort() + " for ["
-            + oldLocation.getRegionInfo().getRegionNameAsString()  + "]; not removing "
-            + oldLocation.getHostnamePort() + " from cache.");
-        } else if (oldLocation != null) {
-          LOG.debug("Removed [" + oldLocation.getRegionInfo().getRegionNameAsString()
-            + "] for tableName=" + Bytes.toString(tableName) + " from cache because of ["
-            + Bytes.toStringBinary(row) + "]");
-        }
-      }
     }
 
     @Override
@@ -1603,7 +1593,7 @@ public <R> void processBatchCallback(
             actionCount++;
             Row row = list.get(i);
             workingList.add(row);
-            deleteCachedLocation(tableName, row.getRow(), lastServers[i]);
+            deleteCachedLocation(tableName, row.getRow());
           } else {
             if (results[i] != null && results[i] instanceof Throwable) {
               actionCount++;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 369b54b69be2..66548a1d7f92 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -61,7 +61,6 @@ public class TestHCM {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final byte[] TABLE_NAME = Bytes.toBytes("test");
   private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
-  private static final byte[] TABLE_NAME2 = Bytes.toBytes("test2");
   private static final byte[] FAM_NAM = Bytes.toBytes("f");
   private static final byte[] ROW = Bytes.toBytes("bbb");
 
@@ -163,7 +162,7 @@ public void testRegionCaching() throws Exception{
     HConnectionManager.HConnectionImplementation conn =
         (HConnectionManager.HConnectionImplementation)table.getConnection();
     assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
-    conn.deleteCachedLocation(TABLE_NAME, ROW, null);
+    conn.deleteCachedLocation(TABLE_NAME, ROW);
     HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
     assertNull("What is this location?? " + rl, rl);
     table.close();
@@ -327,31 +326,6 @@ public void testCreateConnection() throws Exception {
     assertTrue(c2 != c3);
   }
 
-  @Test(timeout = 60000)
-  public void testDeleteFromCacheOnlyWorksFromTheSameSource() throws Exception{
-    HTable table = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
-    TEST_UTIL.createMultiRegions(table, FAM_NAM);
-    Put put = new Put(ROW);
-    put.add(FAM_NAM, ROW, ROW);
-    table.put(put);
-    HConnectionManager.HConnectionImplementation conn =
-        (HConnectionManager.HConnectionImplementation)table.getConnection();
-
-    HRegionLocation location = conn.getCachedLocation(TABLE_NAME2, ROW);
-    assertNotNull(location);
-    HRegionLocation anySource = new HRegionLocation(location.getRegionInfo(),
-        location.getHostname(), location.getPort() - 1);
-    conn.deleteCachedLocation(TABLE_NAME2, ROW, anySource);
-    location = conn.getCachedLocation(TABLE_NAME2, ROW);
-    assertNotNull(location); // different server, no delete.
-
-    conn.deleteCachedLocation(TABLE_NAME2, ROW, location);
-    location = conn.getCachedLocation(TABLE_NAME2, ROW);
-    assertNull(location); // same server, delete.
-
-    TEST_UTIL.deleteTable(TABLE_NAME2);
-  }
-
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 0232b466578824e798b96c7d713aeb8ada25c101 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 24 Jan 2013 18:13:55 +0000
Subject: [PATCH 0726/1540] HBASE-7628 Port HBASE-6509 fast-forwarding
 FuzzyRowFilter to 0.94 (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1438114 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/FuzzyRowFilter.java   | 289 ++++++++++++++++++
 .../hadoop/hbase/io/HbaseObjectWritable.java  |   3 +
 .../hbase/filter/TestFuzzyRowFilter.java      | 204 +++++++++++++
 .../hbase/io/TestHbaseObjectWritable.java     |   2 +-
 4 files changed, 497 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
new file mode 100644
index 000000000000..ef8909a0021d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * Filters data based on fuzzy row key. Performs fast-forwards during scanning.
+ * It takes pairs (row key, fuzzy info) to match row keys. Where fuzzy info is
+ * a byte array with 0 or 1 as its values:
+ * <ul>
+ *   <li>
+ *     0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position
+ *         must match
+ *   </li>
+ *   <li>
+ *     1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this
+ *         position can be different from the one in provided row key
+ *   </li>
+ * </ul>
+ *
+ *
+ * Example:
+ * Let's assume row key format is userId_actionId_year_month. Length of userId is fixed
+ * and is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively.
+ *
+ * Let's assume that we need to fetch all users that performed certain action (encoded as "99")
+ * in Jan of any year. Then the pair (row key, fuzzy info) would be the following:
+ * row key = "????_99_????_01" (one can use any value instead of "?")
+ * fuzzy info = "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00"
+ *
+ * I.e. fuzzy info tells the matching mask is "????_99_????_01", where at ? can be any value.
+ *
+ */
+public class FuzzyRowFilter extends FilterBase {
+  private List<Pair<byte[], byte[]>> fuzzyKeysData;
+  private boolean done = false;
+
+  /**
+   * Used internally for reflection, do NOT use it directly
+   */
+  public FuzzyRowFilter() {
+  }
+
+  public FuzzyRowFilter(List<Pair<byte[], byte[]>> fuzzyKeysData) {
+    this.fuzzyKeysData = fuzzyKeysData;
+  }
+
+  // TODO: possible improvement: save which fuzzy row key to use when providing a hint
+  @Override
+  public ReturnCode filterKeyValue(KeyValue kv) {
+    byte[] rowKey = kv.getRow();
+    // assigning "worst" result first and looking for better options
+    SatisfiesCode bestOption = SatisfiesCode.NO_NEXT;
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      SatisfiesCode satisfiesCode =
+              satisfies(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond());
+      if (satisfiesCode == SatisfiesCode.YES) {
+        return ReturnCode.INCLUDE;
+      }
+
+      if (satisfiesCode == SatisfiesCode.NEXT_EXISTS) {
+        bestOption = SatisfiesCode.NEXT_EXISTS;
+      }
+    }
+
+    if (bestOption == SatisfiesCode.NEXT_EXISTS) {
+      return ReturnCode.SEEK_NEXT_USING_HINT;
+    }
+
+    // the only unhandled SatisfiesCode is NO_NEXT, i.e. we are done
+    done = true;
+    return ReturnCode.NEXT_ROW;
+  }
+
+  @Override
+  public KeyValue getNextKeyHint(KeyValue currentKV) {
+    byte[] rowKey = currentKV.getRow();
+    byte[] nextRowKey = null;
+    // Searching for the "smallest" row key that satisfies at least one fuzzy row key
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      byte[] nextRowKeyCandidate = getNextForFuzzyRule(rowKey,
+              fuzzyData.getFirst(), fuzzyData.getSecond());
+      if (nextRowKeyCandidate == null) {
+        continue;
+      }
+      if (nextRowKey == null || Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0) {
+        nextRowKey = nextRowKeyCandidate;
+      }
+    }
+
+    if (nextRowKey == null) {
+      // SHOULD NEVER happen
+      // TODO: is there a better way than throw exception? (stop the scanner?)
+      throw new IllegalStateException("No next row key that satisfies fuzzy exists when" +
+                                         " getNextKeyHint() is invoked." +
+                                         " Filter: " + this.toString() +
+                                         " currentKV: " + currentKV.toString());
+    }
+
+    return KeyValue.createFirstOnRow(nextRowKey);
+  }
+
+  @Override
+  public boolean filterAllRemaining() {
+    return done;
+  }
+
+  @Override
+  public void write(DataOutput dataOutput) throws IOException {
+    dataOutput.writeInt(this.fuzzyKeysData.size());
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      Bytes.writeByteArray(dataOutput, fuzzyData.getFirst());
+      Bytes.writeByteArray(dataOutput, fuzzyData.getSecond());
+    }
+  }
+
+  @Override
+  public void readFields(DataInput dataInput) throws IOException {
+    int count = dataInput.readInt();
+    this.fuzzyKeysData = new ArrayList<Pair<byte[], byte[]>>(count);
+    for (int i = 0; i < count; i++) {
+      byte[] keyBytes = Bytes.readByteArray(dataInput);
+      byte[] keyMeta = Bytes.readByteArray(dataInput);
+      this.fuzzyKeysData.add(new Pair<byte[], byte[]>(keyBytes, keyMeta));
+    }
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder();
+    sb.append("FuzzyRowFilter");
+    sb.append("{fuzzyKeysData=");
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      sb.append('{').append(Bytes.toStringBinary(fuzzyData.getFirst())).append(":");
+      sb.append(Bytes.toStringBinary(fuzzyData.getSecond())).append('}');
+    }
+    sb.append("}, ");
+    return sb.toString();
+  }
+
+  // Utility methods
+
+  static enum SatisfiesCode {
+    // row satisfies fuzzy rule
+    YES,
+    // row doesn't satisfy fuzzy rule, but there's possible greater row that does
+    NEXT_EXISTS,
+    // row doesn't satisfy fuzzy rule and there's no greater row that does
+    NO_NEXT
+  }
+
+  static SatisfiesCode satisfies(byte[] row,
+                                         byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    return satisfies(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
+  }
+
+  private static SatisfiesCode satisfies(byte[] row, int offset, int length,
+                                         byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    if (row == null) {
+      // do nothing, let scan to proceed
+      return SatisfiesCode.YES;
+    }
+
+    boolean nextRowKeyCandidateExists = false;
+
+    for (int i = 0; i < fuzzyKeyMeta.length && i < length; i++) {
+      // First, checking if this position is fixed and not equals the given one
+      boolean byteAtPositionFixed = fuzzyKeyMeta[i] == 0;
+      boolean fixedByteIncorrect = byteAtPositionFixed && fuzzyKeyBytes[i] != row[i + offset];
+      if (fixedByteIncorrect) {
+        // in this case there's another row that satisfies fuzzy rule and bigger than this row
+        if (nextRowKeyCandidateExists) {
+          return SatisfiesCode.NEXT_EXISTS;
+        }
+
+        // If this row byte is less than fixed then there's a byte array bigger than
+        // this row and which satisfies the fuzzy rule. Otherwise there's no such byte array:
+        // this row is simply bigger than any byte array that satisfies the fuzzy rule
+        boolean rowByteLessThanFixed = (row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF);
+        return  rowByteLessThanFixed ? SatisfiesCode.NEXT_EXISTS : SatisfiesCode.NO_NEXT;
+      }
+
+      // Second, checking if this position is not fixed and byte value is not the biggest. In this
+      // case there's a byte array bigger than this row and which satisfies the fuzzy rule. To get
+      // bigger byte array that satisfies the rule we need to just increase this byte
+      // (see the code of getNextForFuzzyRule below) by one.
+      // Note: if non-fixed byte is already at biggest value, this doesn't allow us to say there's
+      //       bigger one that satisfies the rule as it can't be increased.
+      if (fuzzyKeyMeta[i] == 1 && !isMax(fuzzyKeyBytes[i])) {
+        nextRowKeyCandidateExists = true;
+      }
+    }
+
+    return SatisfiesCode.YES;
+  }
+
+  private static boolean isMax(byte fuzzyKeyByte) {
+    return (fuzzyKeyByte & 0xFF) == 255;
+  }
+
+  static byte[] getNextForFuzzyRule(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    return getNextForFuzzyRule(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
+  }
+
+  /**
+   * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists,
+   *         null otherwise
+   */
+  private static byte[] getNextForFuzzyRule(byte[] row, int offset, int length,
+                                            byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    // To find out the next "smallest" byte array that satisfies fuzzy rule and "greater" than
+    // the given one we do the following:
+    // 1. setting values on all "fixed" positions to the values from fuzzyKeyBytes
+    // 2. if during the first step given row did not increase, then we increase the value at
+    //    the first "non-fixed" position (where it is not maximum already)
+
+    // It is easier to perform this by using fuzzyKeyBytes copy and setting "non-fixed" position
+    // values than otherwise.
+    byte[] result = Arrays.copyOf(fuzzyKeyBytes,
+                                  length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length);
+    int toInc = -1;
+
+    boolean increased = false;
+    for (int i = 0; i < result.length; i++) {
+      if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) {
+        result[i] = row[offset + i];
+        if (!isMax(row[i])) {
+          // this is "non-fixed" position and is not at max value, hence we can increase it
+          toInc = i;
+        }
+      } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == 0) {
+        if ((row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF)) {
+          // if setting value for any fixed position increased the original array,
+          // we are OK
+          increased = true;
+          break;
+        }
+        if ((row[i + offset] & 0xFF) > (fuzzyKeyBytes[i] & 0xFF)) {
+          // if setting value for any fixed position makes array "smaller", then just stop:
+          // in case we found some non-fixed position to increase we will do it, otherwise
+          // there's no "next" row key that satisfies fuzzy rule and "greater" than given row
+          break;
+        }
+      }
+    }
+
+    if (!increased) {
+      if (toInc < 0) {
+        return null;
+      }
+      result[toInc]++;
+
+      // Setting all "non-fixed" positions to zeroes to the right of the one we increased so
+      // that found "next" row key is the smallest possible
+      for (int i = toInc + 1; i < result.length; i++) {
+        if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) {
+          result[i] = 0;
+        }
+      }
+    }
+
+    return result;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
index d1d199030105..98a65959288f 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -72,6 +72,7 @@
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.DependentColumnFilter;
 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
 import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
 import org.apache.hadoop.hbase.filter.PageFilter;
@@ -265,6 +266,8 @@ public class HbaseObjectWritable implements Writable, WritableWithSize, Configur
     //java.lang.reflect.Array is a placeholder for arrays not defined above
     GENERIC_ARRAY_CODE = code++;
     addToMap(Array.class, GENERIC_ARRAY_CODE);
+    
+    addToMap(FuzzyRowFilter.class, code++);
 
     // make sure that this is the last statement in this static block
     NEXT_CLASS_CODE = code;
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
new file mode 100644
index 000000000000..0b930657e955
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestFuzzyRowFilter {
+  @Test
+  public void testSatisfies() {
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 0, 0, 1}, // row to check
+                                     new byte[]{1, 0, 1}, // fuzzy row
+                                     new byte[]{0, 1, 0})); // mask
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 1, 0, 1},
+                                     new byte[]{1, 0, 1},
+                                     new byte[]{0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 2, 0, 1},
+                                     new byte[]{1, 0, 1},
+                                     new byte[]{0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{2, 3, 1, 1, 1},
+                                     new byte[]{1, 0, 1},
+                                     new byte[]{0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES,
+            FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 3, 3},
+                                     new byte[]{1, 2, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0}, // row to check
+                                     new byte[]{1, 2, 0, 3}, // fuzzy row
+                                     new byte[]{0, 0, 1, 0})); // mask
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0},
+                                     new byte[]{1, (byte) 245, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) 245, 1, 3, 0},
+                                     new byte[]{1, 1, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{1, 3, 1, 3, 0},
+                                     new byte[]{1, 2, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{2, 1, 1, 1, 0},
+                                     new byte[]{1, 2, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 0, 1},
+                                     new byte[]{0, 1, 2},
+                                     new byte[]{1, 0, 0}));
+  }
+
+  @Test
+  public void testGetNextForFuzzyRule() {
+    assertNext(
+            new byte[]{0, 1, 2}, // fuzzy row
+            new byte[]{1, 0, 0}, // mask
+            new byte[]{1, 2, 1, 0, 1}, // current
+            new byte[]{2, 1, 2, 0, 0}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 2}, // fuzzy row
+            new byte[]{1, 0, 0}, // mask
+            new byte[]{1, 1, 2, 0, 1}, // current
+            new byte[]{1, 1, 2, 0, 2}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 0, 2, 0}, // fuzzy row
+            new byte[]{1, 0, 1, 0, 1}, // mask
+            new byte[]{1, 0, 2, 0, 1}, // current
+            new byte[]{1, 1, 0, 2, 0}); // expected next
+
+    assertNext(
+            new byte[]{1, 0, 1},
+            new byte[]{0, 1, 0},
+            new byte[]{1, (byte) 128, 2, 0, 1},
+            new byte[]{1, (byte) 129, 1, 0, 0});
+
+    assertNext(
+            new byte[]{0, 1, 0, 1},
+            new byte[]{1, 0, 1, 0},
+            new byte[]{5, 1, 0, 1},
+            new byte[]{5, 1, 1, 1});
+
+    assertNext(
+            new byte[]{0, 1, 0, 1},
+            new byte[]{1, 0, 1, 0},
+            new byte[]{5, 1, 0, 1, 1},
+            new byte[]{5, 1, 0, 1, 2});
+
+    assertNext(
+            new byte[]{0, 1, 0, 0}, // fuzzy row
+            new byte[]{1, 0, 1, 1}, // mask
+            new byte[]{5, 1, (byte) 255, 1}, // current
+            new byte[]{5, 1, (byte) 255, 2}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 0, 1}, // fuzzy row
+            new byte[]{1, 0, 1, 0}, // mask
+            new byte[]{5, 1, (byte) 255, 1}, // current
+            new byte[]{6, 1, 0, 1}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 0, 1}, // fuzzy row
+            new byte[]{1, 0, 1, 0}, // mask
+            new byte[]{5, 1, (byte) 255, 0}, // current
+            new byte[]{5, 1, (byte) 255, 1}); // expected next
+
+    assertNext(
+            new byte[]{5, 1, 1, 0},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{5, 1, (byte) 255, 1},
+            new byte[]{5, 1, (byte) 255, 2});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{1, 1, 2, 2},
+            new byte[]{1, 1, 2, 3});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{1, 1, 3, 2},
+            new byte[]{1, 1, 3, 3});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 2, 3},
+            new byte[]{1, 1, 2, 4});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 3, 2},
+            new byte[]{1, 1, 3, 3});
+
+    assertNext(
+            new byte[]{1, 1, 0, 0},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{0, 1, 3, 2},
+            new byte[]{1, 1, 0, 0});
+
+    // No next for this one
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{2, 3, 1, 1, 1}, // row to check
+            new byte[]{1, 0, 1}, // fuzzy row
+            new byte[]{0, 1, 0})); // mask
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{1, (byte) 245, 1, 3, 0},
+            new byte[]{1, 1, 0, 3},
+            new byte[]{0, 0, 1, 0}));
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{1, 3, 1, 3, 0},
+            new byte[]{1, 2, 0, 3},
+            new byte[]{0, 0, 1, 0}));
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{2, 1, 1, 1, 0},
+            new byte[]{1, 2, 0, 3},
+            new byte[]{0, 0, 1, 0}));
+  }
+
+  private void assertNext(byte[] fuzzyRow, byte[] mask, byte[] current, byte[] expected) {
+    byte[] nextForFuzzyRule = FuzzyRowFilter.getNextForFuzzyRule(current, fuzzyRow, mask);
+    Assert.assertArrayEquals(expected, nextForFuzzyRule);
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+          new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
index 0e467ab2bebb..892dda1f94f5 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
@@ -539,7 +539,7 @@ public void testGetClassCode() throws IOException{
    * note on the test above. 
    */
   public void testGetNextObjectCode(){
-    assertEquals(82,HbaseObjectWritable.getNextClassCode());
+    assertEquals(83,HbaseObjectWritable.getNextClassCode());
   }
 
   @org.junit.Rule

From 2b3540643804a8e92eaec39c0ba67ebdc83b1905 Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Thu, 24 Jan 2013 20:45:56 +0000
Subject: [PATCH 0727/1540] HBASE-7647 0.94 hfiles v2.1 are not backwards
 compatible with HFilev2.0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1438175 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileBlock.java     | 150 ++++++++++++------
 .../hadoop/hbase/io/hfile/HFileWriterV2.java  |  12 +-
 src/main/resources/hbase-default.xml          |   8 +
 .../hadoop/hbase/io/hfile/CacheTestUtils.java |   2 +-
 .../hadoop/hbase/io/hfile/TestChecksum.java   |  21 +--
 .../hadoop/hbase/io/hfile/TestHFileBlock.java |  28 ++--
 .../io/hfile/TestHFileBlockCompatibility.java |  16 +-
 .../hbase/io/hfile/TestHFileBlockIndex.java   |   4 +-
 .../io/hfile/TestHFileDataBlockEncoder.java   |   8 +-
 .../hbase/io/hfile/TestHFileWriterV2.java     |  21 ++-
 10 files changed, 173 insertions(+), 97 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index edc633f4adc6..6bf74372b710 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -112,18 +112,18 @@ public class HFileBlock extends SchemaConfigured implements Cacheable {
    * There is a 1 byte checksum type, followed by a 4 byte bytesPerChecksum
    * followed by another 4 byte value to store sizeofDataOnDisk.
    */
-  static final int HEADER_SIZE = HEADER_SIZE_NO_CHECKSUM + Bytes.SIZEOF_BYTE +
+  static final int HEADER_SIZE_WITH_CHECKSUMS = HEADER_SIZE_NO_CHECKSUM + Bytes.SIZEOF_BYTE +
                                  2 * Bytes.SIZEOF_INT;
 
   /**
    * The size of block header when blockType is {@link BlockType#ENCODED_DATA}.
    * This extends normal header by adding the id of encoder.
    */
-  public static final int ENCODED_HEADER_SIZE = HEADER_SIZE
+  public static final int ENCODED_HEADER_SIZE = HEADER_SIZE_WITH_CHECKSUMS
       + DataBlockEncoding.ID_SIZE;
 
   /** Just an array of bytes of the right size. */
-  static final byte[] DUMMY_HEADER = new byte[HEADER_SIZE];
+  static final byte[] DUMMY_HEADER_WITH_CHECKSUM = new byte[HEADER_SIZE_WITH_CHECKSUMS];
   static final byte[] DUMMY_HEADER_NO_CHECKSUM = 
      new byte[HEADER_SIZE_NO_CHECKSUM];
 
@@ -194,7 +194,7 @@ public HFileBlock deserialize(ByteBuffer buf) throws IOException{
 
   /**
    * The on-disk size of the next block, including the header, obtained by
-   * peeking into the first {@link HEADER_SIZE} bytes of the next block's
+   * peeking into the first {@link HFileBlock#headerSize(int)} bytes of the next block's
    * header, or -1 if unknown.
    */
   private int nextBlockOnDiskSizeWithHeader = -1;
@@ -212,9 +212,9 @@ public HFileBlock deserialize(ByteBuffer buf) throws IOException{
    *          compression is disabled.
    * @param prevBlockOffset the offset of the previous block in the
    *          {@link HFile}
-   * @param buf block header ({@link #HEADER_SIZE} bytes) followed by
+   * @param buf block header {@link HFileBlock#headerSize(int)} bytes) followed by
    *          uncompressed data. This
-   * @param fillHeader true to fill in the first {@link #HEADER_SIZE} bytes of
+   * @param fillHeader true to fill in the first {@link HFileBlock#headerSize(int)} bytes of
    *          the buffer based on the header fields provided
    * @param offset the file offset the block was read from
    * @param minorVersion the minor version of this block
@@ -322,7 +322,7 @@ public long getPrevBlockOffset() {
   }
 
   /**
-   * Writes header fields into the first {@link HEADER_SIZE} bytes of the
+   * Writes header fields into the first {@link ©HEADER_SIZE_WITH_CHECKSUMS} bytes of the
    * buffer. Resets the buffer position to the end of header as side effect.
    */
   private void overwriteHeader() {
@@ -395,7 +395,7 @@ private void sanityCheckAssertion(long valueFromBuf, long valueFromField,
 
   /**
    * Checks if the block is internally consistent, i.e. the first
-   * {@link #HEADER_SIZE} bytes of the buffer contain a valid header consistent
+   * {@link HFileBlock#headerSize(int)} bytes of the buffer contain a valid header consistent
    * with the fields. This function is primary for testing and debugging, and
    * is not thread-safe, because it alters the internal buffer pointer.
    */
@@ -433,7 +433,7 @@ void sanityCheck() throws IOException {
           + ", got " + buf.limit());
     }
 
-    // We might optionally allocate HEADER_SIZE more bytes to read the next
+    // We might optionally allocate HEADER_SIZE_WITH_CHECKSUMS more bytes to read the next
     // block's, header, so there are two sensible values for buffer capacity.
     int size = uncompressedSizeWithoutHeader + hdrSize + cksumBytes;
     if (buf.capacity() != size &&
@@ -645,7 +645,7 @@ private enum State {
     /**
      * The stream we use to accumulate data in uncompressed format for each
      * block. We reset this stream at the end of each block and reuse it. The
-     * header is written as the first {@link #HEADER_SIZE} bytes into this
+     * header is written as the first {@link HFileBlock#headerSize(int)}  bytes into this
      * stream.
      */
     private ByteArrayOutputStream baosInMemory;
@@ -696,7 +696,7 @@ private enum State {
     /**
      * Valid in the READY state. Contains the header and the uncompressed (but
      * potentially encoded, if this is a data block) bytes, so the length is
-     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE}.
+     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#headerSize(int)}.
      * Does not store checksums.
      */
     private byte[] uncompressedBytesWithHeader;
@@ -723,6 +723,8 @@ private enum State {
     private ChecksumType checksumType;
     private int bytesPerChecksum;
 
+    private final int minorVersion;
+
     /**
      * @param compressionAlgorithm compression algorithm to use
      * @param dataBlockEncoderAlgo data block encoding algorithm to use
@@ -731,7 +733,9 @@ private enum State {
      */
     public Writer(Compression.Algorithm compressionAlgorithm,
           HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS,
+          int minorVersion,
           ChecksumType checksumType, int bytesPerChecksum) {
+      this.minorVersion = minorVersion;
       compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
       this.dataBlockEncoder = dataBlockEncoder != null
           ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
@@ -749,9 +753,10 @@ public Writer(Compression.Algorithm compressionAlgorithm,
               "for algorithm " + compressionAlgorithm, e);
         }
       }
-      if (bytesPerChecksum < HEADER_SIZE) {
+      if (minorVersion > MINOR_VERSION_NO_CHECKSUM
+          && bytesPerChecksum < HEADER_SIZE_WITH_CHECKSUMS) {
         throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
-            " Minimum is " + HEADER_SIZE + " but the configured value is " +
+            " Minimum is " + HEADER_SIZE_WITH_CHECKSUMS + " but the configured value is " +
             bytesPerChecksum);
       }
       
@@ -782,7 +787,7 @@ public DataOutputStream startWriting(BlockType newBlockType)
       blockType = newBlockType;
 
       baosInMemory.reset();
-      baosInMemory.write(DUMMY_HEADER);
+      baosInMemory.write(getDummyHeaderForVersion(this.minorVersion));
 
       state = State.WRITING;
 
@@ -849,15 +854,62 @@ private void finishBlock() throws IOException {
      * outputbyte stream 'baos'.
      */
     private void doCompressionAndChecksumming() throws IOException {
+      if ( minorVersion <= MINOR_VERSION_NO_CHECKSUM) {
+        version20compression();
+      } else {
+        version21ChecksumAndCompression();
+      }
+    }
+
+    private void version20compression() throws IOException {
+      onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
+
+      if (compressAlgo != NONE) {
+        compressedByteStream.reset();
+        compressedByteStream.write(DUMMY_HEADER_NO_CHECKSUM);
+
+        compressionStream.resetState();
+
+        compressionStream.write(uncompressedBytesWithHeader, headerSize(this.minorVersion),
+            uncompressedBytesWithHeader.length - headerSize(this.minorVersion));
+
+
+        compressionStream.flush();
+        compressionStream.finish();
+        onDiskDataSizeWithHeader = compressedByteStream.size(); // data size
+        onDiskBytesWithHeader = compressedByteStream.toByteArray();
+
+        put20Header(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length);
+
+
+        //set the header for the uncompressed bytes (for cache-on-write)
+        put20Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length + onDiskChecksum.length,
+            uncompressedBytesWithHeader.length);
+
+      } else {
+        onDiskBytesWithHeader = uncompressedBytesWithHeader;
+
+        onDiskDataSizeWithHeader = onDiskBytesWithHeader.length;
+
+        //set the header for the uncompressed bytes
+        put20Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length);
+      }
+    }
+
+    private void version21ChecksumAndCompression() throws IOException {
       // do the compression
       if (compressAlgo != NONE) {
         compressedByteStream.reset();
-        compressedByteStream.write(DUMMY_HEADER);
+        compressedByteStream.write(DUMMY_HEADER_WITH_CHECKSUM);
 
         compressionStream.resetState();
 
-        compressionStream.write(uncompressedBytesWithHeader, HEADER_SIZE,
-            uncompressedBytesWithHeader.length - HEADER_SIZE);
+        compressionStream.write(uncompressedBytesWithHeader, headerSize(this.minorVersion),
+            uncompressedBytesWithHeader.length - headerSize(this.minorVersion));
 
         compressionStream.flush();
         compressionStream.finish();
@@ -871,7 +923,7 @@ private void doCompressionAndChecksumming() throws IOException {
 
 
         onDiskBytesWithHeader = compressedByteStream.toByteArray();
-        putHeader(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+        put21Header(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
             uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
 
        // generate checksums for header and data. The checksums are
@@ -885,9 +937,9 @@ private void doCompressionAndChecksumming() throws IOException {
         onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
 
         //set the header for the uncompressed bytes (for cache-on-write)
-        putHeader(uncompressedBytesWithHeader, 0,
-          onDiskBytesWithHeader.length + onDiskChecksum.length,
-          uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+        put21Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length + onDiskChecksum.length,
+            uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
 
       } else {
         // If we are not using any compression, then the
@@ -901,9 +953,9 @@ private void doCompressionAndChecksumming() throws IOException {
         onDiskChecksum = new byte[numBytes];
 
         //set the header for the uncompressed bytes
-        putHeader(uncompressedBytesWithHeader, 0,
-          onDiskBytesWithHeader.length + onDiskChecksum.length,
-          uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+        put21Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length + onDiskChecksum.length,
+            uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
 
         ChecksumUtil.generateChecksums(
           uncompressedBytesWithHeader, 0, uncompressedBytesWithHeader.length,
@@ -923,11 +975,11 @@ private void encodeDataBlockForDisk() throws IOException {
 
       // do data block encoding, if data block encoder is set
       ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader,
-          HEADER_SIZE, uncompressedBytesWithHeader.length -
-          HEADER_SIZE).slice();
+          headerSize(this.minorVersion), uncompressedBytesWithHeader.length -
+          headerSize(this.minorVersion)).slice();
       Pair<ByteBuffer, BlockType> encodingResult =
           dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
-              includesMemstoreTS, DUMMY_HEADER);
+              includesMemstoreTS, getDummyHeaderForVersion(this.minorVersion));
 
       BlockType encodedBlockType = encodingResult.getSecond();
       if (encodedBlockType == BlockType.ENCODED_DATA) {
@@ -940,10 +992,10 @@ private void encodeDataBlockForDisk() throws IOException {
               "block encoder: " + encodedBlockType);
         }
         if (userDataStream.size() !=
-            uncompressedBytesWithHeader.length - HEADER_SIZE) {
+            uncompressedBytesWithHeader.length - headerSize(this.minorVersion)) {
           throw new IOException("Uncompressed size mismatch: "
               + userDataStream.size() + " vs. "
-              + (uncompressedBytesWithHeader.length - HEADER_SIZE));
+              + (uncompressedBytesWithHeader.length - headerSize(this.minorVersion)));
         }
       }
     }
@@ -956,17 +1008,25 @@ private void encodeDataBlockForDisk() throws IOException {
      * @param onDiskDataSize size of the block on disk with header
      *        and data but not including the checksums
      */
-    private void putHeader(byte[] dest, int offset, int onDiskSize,
-        int uncompressedSize, int onDiskDataSize) {
+    private void put21Header(byte[] dest, int offset, int onDiskSize,
+                             int uncompressedSize, int onDiskDataSize) {
       offset = blockType.put(dest, offset);
-      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
-      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
+      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_WITH_CHECKSUMS);
+      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_WITH_CHECKSUMS);
       offset = Bytes.putLong(dest, offset, prevOffset);
       offset = Bytes.putByte(dest, offset, checksumType.getCode());
       offset = Bytes.putInt(dest, offset, bytesPerChecksum);
       offset = Bytes.putInt(dest, offset, onDiskDataSizeWithHeader);
     }
 
+
+    private void put20Header(byte[] dest, int offset, int onDiskSize,
+                             int uncompressedSize) {
+      offset = blockType.put(dest, offset);
+      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_NO_CHECKSUM);
+      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_NO_CHECKSUM);
+      Bytes.putLong(dest, offset, prevOffset);
+    }
     /**
      * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
      * the offset of this block so that it can be referenced in the next block
@@ -999,7 +1059,7 @@ public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
     private void writeHeaderAndData(DataOutputStream out) throws IOException {
       ensureBlockReady();
       out.write(onDiskBytesWithHeader);
-      if (compressAlgo == NONE) {
+      if (compressAlgo == NONE && minorVersion > MINOR_VERSION_NO_CHECKSUM) {
         if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
           throw new IOException("A " + blockType 
               + " without compression should have checksums " 
@@ -1062,7 +1122,7 @@ public void releaseCompressor() {
      */
     int getOnDiskSizeWithoutHeader() {
       expectState(State.BLOCK_READY);
-      return onDiskBytesWithHeader.length + onDiskChecksum.length - HEADER_SIZE;
+      return onDiskBytesWithHeader.length + onDiskChecksum.length - headerSize(this.minorVersion);
     }
 
     /**
@@ -1082,7 +1142,7 @@ int getOnDiskSizeWithHeader() {
      */
     int getUncompressedSizeWithoutHeader() {
       expectState(State.BLOCK_READY);
-      return uncompressedBytesWithHeader.length - HEADER_SIZE;
+      return uncompressedBytesWithHeader.length - headerSize(this.minorVersion);
     }
 
     /**
@@ -1158,7 +1218,7 @@ public HFileBlock getBlockForCaching() {
       return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
           getUncompressedSizeWithoutHeader(), prevOffset,
           getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
-          includesMemstoreTS, MINOR_VERSION_WITH_CHECKSUM,
+          includesMemstoreTS, this.minorVersion,
           0, ChecksumType.NULL.getCode(),  // no checksums in cached data
           onDiskBytesWithHeader.length + onDiskChecksum.length);
     }
@@ -1458,7 +1518,7 @@ public FSReaderV1(FSDataInputStream istream, Algorithm compressAlgo,
      * coming to end of the compressed section.
      *
      * The block returned is still a version 2 block, and in particular, its
-     * first {@link #HEADER_SIZE} bytes contain a valid version 2 header.
+     * first {@link #HEADER_SIZE_WITH_CHECKSUMS} bytes contain a valid version 2 header.
      *
      * @param offset the offset of the block to read in the file
      * @param onDiskSizeWithMagic the on-disk size of the version 1 block,
@@ -1540,8 +1600,8 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
    */
   private static class PrefetchedHeader {
     long offset = -1;
-    byte[] header = new byte[HEADER_SIZE];
-    ByteBuffer buf = ByteBuffer.wrap(header, 0, HEADER_SIZE);
+    byte[] header = new byte[HEADER_SIZE_WITH_CHECKSUMS];
+    ByteBuffer buf = ByteBuffer.wrap(header, 0, HEADER_SIZE_WITH_CHECKSUMS);
   }
 
   /** Reads version 2 blocks from the filesystem. */
@@ -1607,7 +1667,7 @@ public FSReaderV2(FSDataInputStream istream,
      */
     FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
         long fileSize) throws IOException {
-      this(istream, istream, compressAlgo, fileSize, 
+      this(istream, istream, compressAlgo, fileSize,
            HFileReaderV2.MAX_MINOR_VERSION, null, null);
     }
 
@@ -2074,24 +2134,24 @@ static private int headerSize(int minorVersion) {
     if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
       return HEADER_SIZE_NO_CHECKSUM;
     }
-    return HEADER_SIZE;
+    return HEADER_SIZE_WITH_CHECKSUMS;
   }
 
   /**
-   * Return the appropriate DUMMY_HEADER for the minor version
+   * Return the appropriate DUMMY_HEADER_WITH_CHECKSUM for the minor version
    */
   public byte[] getDummyHeaderForVersion() {
     return getDummyHeaderForVersion(minorVersion);
   }
 
   /**
-   * Return the appropriate DUMMY_HEADER for the minor version
+   * Return the appropriate DUMMY_HEADER_WITH_CHECKSUM for the minor version
    */
   static private byte[] getDummyHeaderForVersion(int minorVersion) {
     if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
       return DUMMY_HEADER_NO_CHECKSUM;
     }
-    return DUMMY_HEADER;
+    return DUMMY_HEADER_WITH_CHECKSUM;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index ae9513b3c200..a299b3cde4f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -32,8 +32,10 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
 import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
@@ -87,6 +89,8 @@ public class HFileWriterV2 extends AbstractHFileWriter {
   private final boolean includeMemstoreTS = true;
   private long maxMemstoreTS = 0;
 
+  private int minorVersion = HFileReaderV2.MAX_MINOR_VERSION;
+
   static class WriterFactoryV2 extends HFile.WriterFactory {
     WriterFactoryV2(Configuration conf, CacheConfig cacheConf) {
       super(conf, cacheConf);
@@ -115,6 +119,9 @@ public HFileWriterV2(Configuration conf, CacheConfig cacheConf,
     SchemaMetrics.configureGlobally(conf);
     this.checksumType = checksumType;
     this.bytesPerChecksum = bytesPerChecksum;
+    if (!conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, false)) {
+      this.minorVersion = 0;
+    }
     finishInit(conf);
   }
 
@@ -125,7 +132,7 @@ private void finishInit(final Configuration conf) {
 
     // HFile filesystem-level (non-caching) block writer
     fsBlockWriter = new HFileBlock.Writer(compressAlgo, blockEncoder,
-        includeMemstoreTS, checksumType, bytesPerChecksum);
+        includeMemstoreTS, minorVersion, checksumType, bytesPerChecksum);
 
     // Data block index writer
     boolean cacheIndexesOnWrite = cacheConf.shouldCacheIndexesOnWrite();
@@ -364,8 +371,7 @@ public void close() throws IOException {
     finishBlock();
     writeInlineBlocks(true);
 
-    FixedFileTrailer trailer = new FixedFileTrailer(2, 
-                                 HFileReaderV2.MAX_MINOR_VERSION);
+    FixedFileTrailer trailer = new FixedFileTrailer(2, minorVersion);
 
     // Write out the metadata blocks if any.
     if (!metaNames.isEmpty()) {
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 999ae5f1b7c5..31a7b802d639 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -485,6 +485,14 @@
           cache at the time the index is being written.
       </description>
   </property>
+  <property>
+      <name>hbase.regionserver.checksum.verify</name>
+      <value>false</value>
+      <description>
+         Allow hbase to do checksums rather than using hdfs checksums. This is a backwards
+         incompatible change.
+      </description>
+    </property>
   <property>
       <name>hfile.index.block.max.size</name>
       <value>131072</value>
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
index 609c69c4c990..88b8708376f2 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
@@ -326,7 +326,7 @@ private static HFileBlockPair[] generateHFileBlocks(int blockSize,
           prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
           blockSize, includesMemstoreTS, HFileBlock.MINOR_VERSION_NO_CHECKSUM,
           0, ChecksumType.NULL.getCode(),
-          onDiskSizeWithoutHeader + HFileBlock.HEADER_SIZE);
+          onDiskSizeWithoutHeader + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS);
 
       String strKey;
       /* No conflicting keys */
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
index 4179725350e1..168fc67204fa 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
@@ -21,20 +21,11 @@
 
 import static org.junit.Assert.*;
 
-import java.io.ByteArrayOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.DataOutputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.zip.Checksum;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -44,13 +35,9 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChecksumType;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.io.compress.Compressor;
 
 import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
 import org.junit.Before;
@@ -96,7 +83,7 @@ public void testChecksumCorruption() throws IOException {
             + algo);
         FSDataOutputStream os = fs.create(path);
         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-            true, HFile.DEFAULT_CHECKSUM_TYPE,
+            true, 1, HFile.DEFAULT_CHECKSUM_TYPE,
             HFile.DEFAULT_BYTES_PER_CHECKSUM);
         long totalSize = 0;
         for (int blockId = 0; blockId < 2; ++blockId) {
@@ -189,7 +176,7 @@ public void testChecksumChunks() throws IOException {
                              algo + bytesPerChecksum);
         FSDataOutputStream os = fs.create(path);
         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-          true, HFile.DEFAULT_CHECKSUM_TYPE, bytesPerChecksum);
+          true, 1,HFile.DEFAULT_CHECKSUM_TYPE, bytesPerChecksum);
 
         // write one block. The block has data
         // that is at least 6 times more than the checksum chunk size
@@ -206,7 +193,7 @@ public void testChecksumChunks() throws IOException {
         os.close();
 
         long expectedChunks = ChecksumUtil.numChunks(
-                               dataSize + HFileBlock.HEADER_SIZE,
+                               dataSize + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS,
                                bytesPerChecksum);
         LOG.info("testChecksumChunks: pread=" + pread +
                    ", bytesPerChecksum=" + bytesPerChecksum +
@@ -228,7 +215,7 @@ public void testChecksumChunks() throws IOException {
         assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());
 
         // verify that we have the expected number of checksum chunks
-        assertEquals(totalSize, HFileBlock.HEADER_SIZE + dataSize + 
+        assertEquals(totalSize, HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + dataSize +
                      expectedChunks * HFileBlock.CHECKSUM_SIZE);
 
         // assert that we did not encounter hbase checksum verification failures
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index bf6f064f7711..489a1d2b55de 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.apache.hadoop.io.compress.Compressor;
 
 import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
@@ -196,7 +197,8 @@ static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
       boolean includesMemstoreTS) throws IOException {
     final BlockType blockType = BlockType.DATA;
     HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        includesMemstoreTS, HFileReaderV2.MAX_MINOR_VERSION,
+        HFile.DEFAULT_CHECKSUM_TYPE,
         HFile.DEFAULT_BYTES_PER_CHECKSUM);
     DataOutputStream dos = hbw.startWriting(blockType);
     writeTestBlockContents(dos);
@@ -210,7 +212,7 @@ public String createTestBlockStr(Compression.Algorithm algo,
       int correctLength) throws IOException {
     HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
     byte[] testV2Block = hbw.getHeaderAndDataForTest();
-    int osOffset = HFileBlock.HEADER_SIZE + 9;
+    int osOffset = HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + 9;
     if (testV2Block.length == correctLength) {
       // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
       // variations across operating systems.
@@ -297,7 +299,9 @@ public void testReaderV2() throws IOException {
             + algo);
         FSDataOutputStream os = fs.create(path);
         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-            includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+            includesMemstoreTS,
+            HFileReaderV2.MAX_MINOR_VERSION,
+            HFile.DEFAULT_CHECKSUM_TYPE,
             HFile.DEFAULT_BYTES_PER_CHECKSUM);
         long totalSize = 0;
         for (int blockId = 0; blockId < 2; ++blockId) {
@@ -325,13 +329,13 @@ public void testReaderV2() throws IOException {
         if (algo == GZ) {
           is = fs.open(path);
           hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
-          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE +
+          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS +
                                 b.totalChecksumBytes(), -1, pread);
           assertEquals(blockStr, b.toString());
           int wrongCompressedSize = 2172;
           try {
             b = hbr.readBlockData(0, wrongCompressedSize
-                + HFileBlock.HEADER_SIZE, -1, pread);
+                + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS, -1, pread);
             fail("Exception expected");
           } catch (IOException ex) {
             String expectedPrefix = "On-disk size without header provided is "
@@ -363,7 +367,9 @@ public void testDataBlockEncoding() throws IOException {
           HFileDataBlockEncoder dataBlockEncoder =
               new HFileDataBlockEncoderImpl(encoding);
           HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
-              includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+              includesMemstoreTS,
+              HFileReaderV2.MAX_MINOR_VERSION,
+              HFile.DEFAULT_CHECKSUM_TYPE,
               HFile.DEFAULT_BYTES_PER_CHECKSUM);
           long totalSize = 0;
           final List<Integer> encodedSizes = new ArrayList<Integer>();
@@ -505,7 +511,7 @@ public void testPreviousOffset() throws IOException {
           for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
             if (!pread) {
               assertEquals(is.getPos(), curOffset + (i == 0 ? 0 :
-                  HFileBlock.HEADER_SIZE));
+                  HFileBlock.HEADER_SIZE_WITH_CHECKSUMS));
             }
 
             assertEquals(expectedOffsets.get(i).longValue(), curOffset);
@@ -706,7 +712,9 @@ private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
     boolean cacheOnWrite = expectedContents != null;
     FSDataOutputStream os = fs.create(path);
     HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
-        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        includesMemstoreTS,
+        HFileReaderV2.MAX_MINOR_VERSION,
+        HFile.DEFAULT_CHECKSUM_TYPE,
         HFile.DEFAULT_BYTES_PER_CHECKSUM);
     Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
     long totalSize = 0;
@@ -764,7 +772,7 @@ public void testBlockHeapSize() {
     }
 
     for (int size : new int[] { 100, 256, 12345 }) {
-      byte[] byteArr = new byte[HFileBlock.HEADER_SIZE + size];
+      byte[] byteArr = new byte[HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size];
       ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
       HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
           HFileBlock.FILL_HEADER, -1, includesMemstoreTS, 
@@ -772,7 +780,7 @@ public void testBlockHeapSize() {
           0);
       long byteBufferExpectedSize =
           ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
-              + HFileBlock.HEADER_SIZE + size);
+              + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size);
       long hfileBlockExpectedSize =
           ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true));
       long expected = hfileBlockExpectedSize + byteBufferExpectedSize;
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
index 4d9b1580d4cd..d7b44f4cd365 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
@@ -28,32 +28,18 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.io.DoubleOutputStream;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.ClassSize;
-import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
@@ -417,7 +403,7 @@ private enum State {
     /**
      * Valid in the READY state. Contains the header and the uncompressed (but
      * potentially encoded, if this is a data block) bytes, so the length is
-     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE}.
+     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE_WITH_CHECKSUMS}.
      */
     private byte[] uncompressedBytesWithHeader;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 8cbbc2397493..6280c21836e6 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -216,7 +216,9 @@ public void readIndex() throws IOException {
   private void writeWholeIndex() throws IOException {
     assertEquals(0, keys.size());
     HFileBlock.Writer hbw = new HFileBlock.Writer(compr, null,
-        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        includesMemstoreTS,
+        1,
+        HFile.DEFAULT_CHECKSUM_TYPE,
         HFile.DEFAULT_BYTES_PER_CHECKSUM);
     FSDataOutputStream outputStream = fs.create(path);
     HFileBlockIndex.BlockIndexWriter biw =
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
index 4eea2ff40add..0c076f267ed6 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
@@ -124,9 +124,9 @@ public void testEncodingWritePath() {
     HFileBlock block = getSampleHFileBlock();
     Pair<ByteBuffer, BlockType> result =
         blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
-            includesMemstoreTS, HFileBlock.DUMMY_HEADER);
+            includesMemstoreTS, HFileBlock.DUMMY_HEADER_WITH_CHECKSUM);
 
-    int size = result.getFirst().limit() - HFileBlock.HEADER_SIZE;
+    int size = result.getFirst().limit() - HFileBlock.HEADER_SIZE_WITH_CHECKSUMS;
     HFileBlock blockOnDisk = new HFileBlock(result.getSecond(),
         size, size, -1, result.getFirst(), HFileBlock.FILL_HEADER, 0,
         includesMemstoreTS, block.getMinorVersion(),
@@ -156,8 +156,8 @@ private HFileBlock getSampleHFileBlock() {
     ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
         generator.generateTestKeyValues(60), includesMemstoreTS);
     int size = keyValues.limit();
-    ByteBuffer buf = ByteBuffer.allocate(size + HFileBlock.HEADER_SIZE);
-    buf.position(HFileBlock.HEADER_SIZE);
+    ByteBuffer buf = ByteBuffer.allocate(size + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS);
+    buf.position(HFileBlock.HEADER_SIZE_WITH_CHECKSUMS);
     keyValues.rewind();
     buf.put(keyValues);
     HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
index cd0852b3ee27..1ddafa68cd25 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Random;
 
@@ -46,14 +47,24 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /**
  * Testing writing a version 2 {@link HFile}. This is a low-level test written
  * during the development of {@link HFileWriterV2}.
  */
 @Category(SmallTests.class)
+@RunWith(Parameterized.class)
 public class TestHFileWriterV2 {
 
+  private final boolean useChecksums;
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> parameters() {
+    return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
+  }
+
   private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class);
 
   private static final HBaseTestingUtility TEST_UTIL =
@@ -62,9 +73,14 @@ public class TestHFileWriterV2 {
   private Configuration conf;
   private FileSystem fs;
 
+  public TestHFileWriterV2(boolean useChecksums) {
+    this.useChecksums = useChecksums;
+  }
+
   @Before
   public void setUp() throws IOException {
     conf = TEST_UTIL.getConfiguration();
+    conf.setBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, useChecksums);
     fs = FileSystem.get(conf);
   }
 
@@ -139,10 +155,13 @@ private void writeDataAndReadFromHFile(Path hfilePath,
         FixedFileTrailer.readFromStream(fsdis, fileSize);
 
     assertEquals(2, trailer.getMajorVersion());
+    assertEquals(useChecksums?1:0, trailer.getMinorVersion());
     assertEquals(entryCount, trailer.getEntryCount());
 
     HFileBlock.FSReader blockReader =
-        new HFileBlock.FSReaderV2(fsdis, compressAlgo, fileSize);
+        new HFileBlock.FSReaderV2(fsdis,fsdis, compressAlgo, fileSize,
+            this.useChecksums?HFileReaderV2.MAX_MINOR_VERSION:HFileReaderV2.MIN_MINOR_VERSION,
+            null, null);
     // Comparator class name is stored in the trailer in version 2.
     RawComparator<byte []> comparator = trailer.createComparator();
     HFileBlockIndex.BlockIndexReader dataBlockIndexReader =

From 2d562cefcb28362e0a4d0db701714bf9d157f6cb Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 25 Jan 2013 06:15:45 +0000
Subject: [PATCH 0728/1540] move version forward

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1438334 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 9a17f7e6112c..7f75032c8791 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.4</version>
+  <version>0.94.5-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From ead44ba4754d170373ff800e7863bd3815368f3e Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 26 Jan 2013 00:21:01 +0000
Subject: [PATCH 0729/1540] HBASE-7654. Add List<String> getCoprocessors() to
 HTableDescriptor (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1438790 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HTableDescriptor.java | 30 ++++++++++++++-
 .../hadoop/hbase/TestHTableDescriptor.java    | 38 +++++++++++++++++++
 2 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index c17765e3535a..934587d5b257 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -22,10 +22,12 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -1033,10 +1035,10 @@ public void addCoprocessor(String className, Path jarFilePath,
     setValue(key, value);
   }
 
-  
+
   /**
    * Check if the table has an attached co-processor represented by the name className
-   * 
+   *
    * @param className - Class name of the co-processor
    * @return true of the table has a co-processor className
    */
@@ -1066,6 +1068,30 @@ public boolean hasCoprocessor(String className) {
     return false;
   }
 
+  /**
+   * Return the list of attached co-processor represented by their name className
+   *
+   * @return The list of co-processors classNames
+   */
+  public List<String> getCoprocessors() {
+    List<String> result = new ArrayList<String>();
+    Matcher keyMatcher;
+    Matcher valueMatcher;
+    for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e : this.values.entrySet()) {
+      keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(Bytes.toString(e.getKey().get()));
+      if (!keyMatcher.matches()) {
+        continue;
+      }
+      valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(Bytes
+          .toString(e.getValue().get()));
+      if (!valueMatcher.matches()) {
+        continue;
+      }
+      result.add(valueMatcher.group(2).trim()); // classname is the 2nd field
+    }
+    return result;
+  }
+
   /**
    * Remove a coprocessor from those set on the table
    * @param className Class name of the co-processor
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
index 15e9d936c3e5..186d5d29faad 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
@@ -22,6 +22,7 @@
 import static org.junit.Assert.assertTrue;
 
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -48,6 +49,43 @@ public void testGetSetRemoveCP() throws Exception {
     assertFalse(desc.hasCoprocessor(className));
   }
 
+  /**
+   * Test cps in the table description
+   * @throws Exception
+   */
+  @Test
+  public void testSetListRemoveCP() throws Exception {
+    HTableDescriptor desc = new HTableDescriptor("testGetSetRemoveCP");
+    // simple CP
+    String className1 = BaseRegionObserver.class.getName();
+    String className2 = SampleRegionWALObserver.class.getName();
+    // Check that any coprocessor is present.
+    assertTrue(desc.getCoprocessors().size() == 0);
+
+    // Add the 1 coprocessor and check if present.
+    desc.addCoprocessor(className1);
+    assertTrue(desc.getCoprocessors().size() == 1);
+    assertTrue(desc.getCoprocessors().contains(className1));
+
+    // Add the 2nd coprocessor and check if present.
+    // remove it and check that it is gone
+    desc.addCoprocessor(className2);
+    assertTrue(desc.getCoprocessors().size() == 2);
+    assertTrue(desc.getCoprocessors().contains(className2));
+
+    // Remove one and check
+    desc.removeCoprocessor(className1);
+    assertTrue(desc.getCoprocessors().size() == 1);
+    assertFalse(desc.getCoprocessors().contains(className1));
+    assertTrue(desc.getCoprocessors().contains(className2));
+
+    // Remove the last and check
+    desc.removeCoprocessor(className2);
+    assertTrue(desc.getCoprocessors().size() == 0);
+    assertFalse(desc.getCoprocessors().contains(className1));
+    assertFalse(desc.getCoprocessors().contains(className2));
+  }
+
   /**
    * Test that we add and remove strings from settings properly.
    * @throws Exception

From 898f6616630a98a80bce32f67f46fb2701068109 Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Sat, 26 Jan 2013 03:44:11 +0000
Subject: [PATCH 0730/1540] HBASE-7669 ROOT region wouldn't be handled by
 PRI-IPC-Handler(Chunhui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1438832 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegionServer.java       | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index f459c54211ad..d561d3ded465 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -496,14 +496,14 @@ public QosFunction() {
       annotatedQos = qosMap;
     }
 
-    public boolean isMetaRegion(byte[] regionName) {
+    public boolean isMetaTable(byte[] regionName) {
       HRegion region;
       try {
         region = getRegion(regionName);
       } catch (NotServingRegionException ignored) {
         return false;
       }
-      return region.getRegionInfo().isMetaRegion();
+      return region.getRegionInfo().isMetaTable();
     }
 
     @Override
@@ -530,7 +530,7 @@ public Integer apply(Writable from) {
         }
         String scannerIdString = Long.toString(scannerId);
         RegionScanner scanner = scanners.get(scannerIdString);
-        if (scanner != null && scanner.getRegionInfo().isMetaRegion()) {
+        if (scanner != null && scanner.getRegionInfo().isMetaTable()) {
           // LOG.debug("High priority scanner request: " + scannerId);
           return HConstants.HIGH_QOS;
         }
@@ -538,7 +538,7 @@ public Integer apply(Writable from) {
        // Just let it through.  This is getOnlineRegions, etc.
       } else if (inv.getParameterClasses()[0] == byte[].class) {
         // first arg is byte array, so assume this is a regionname:
-        if (isMetaRegion((byte[]) inv.getParameters()[0])) {
+        if (isMetaTable((byte[]) inv.getParameters()[0])) {
           // LOG.debug("High priority with method: " + methodName +
           // " and region: "
           // + Bytes.toString((byte[]) inv.getParameters()[0]));
@@ -555,7 +555,7 @@ public Integer apply(Writable from) {
         // AND this
         // regionserver hosts META/-ROOT-
         for (byte[] region : regions) {
-          if (isMetaRegion(region)) {
+          if (isMetaTable(region)) {
             // LOG.debug("High priority multi with region: " +
             // Bytes.toString(region));
             return HConstants.HIGH_QOS; // short circuit for the win.

From 3dc27466df9aab33ac020b5905dd4b1a3413f3cf Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sat, 26 Jan 2013 21:59:54 +0000
Subject: [PATCH 0731/1540] HBASE-7643 HFileArchiver.resolveAndArchive() race
 condition may lead to snapshot data loss

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1438973 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/backup/HFileArchiver.java    | 83 ++++++++++---------
 .../hbase/backup/TestHFileArchiving.java      | 68 +++++++++++++++
 2 files changed, 111 insertions(+), 40 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index a2f7f1247c43..42415a9fdce6 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -55,6 +55,9 @@ public class HFileArchiver {
   private static final Log LOG = LogFactory.getLog(HFileArchiver.class);
   private static final String SEPARATOR = ".";
 
+  /** Number of retries in case of fs operation failure */
+  private static final int DEFAULT_RETRIES_NUMBER = 3;
+
   private HFileArchiver() {
     // hidden ctor since this is just a util
   }
@@ -136,6 +139,7 @@ public boolean accept(Path file) {
     try {
       success = resolveAndArchive(fs, regionArchiveDir, toArchive);
     } catch (IOException e) {
+      LOG.error("Failed to archive: " + toArchive, e);
       success = false;
     }
 
@@ -146,7 +150,7 @@ public boolean accept(Path file) {
     }
 
     throw new IOException("Received error when attempting to archive files (" + toArchive
-        + "), cannot delete region directory.");
+        + "), cannot delete region directory. ");
   }
 
   /**
@@ -254,14 +258,12 @@ private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir,
     long start = EnvironmentEdgeManager.currentTimeMillis();
     List<File> failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start);
 
-    // clean out the failures by just deleting them
+    // notify that some files were not archived.
+    // We can't delete the files otherwise snapshots or other backup system
+    // that relies on the archiver end up with data loss.
     if (failures.size() > 0) {
-      try {
-        LOG.error("Failed to complete archive, deleting extra store files.");
-        deleteFilesWithoutArchiving(failures);
-      } catch (IOException e) {
-        LOG.warn("Failed to delete store file(s) when archiving failed", e);
-      }
+      LOG.warn("Failed to complete archive of: " + failures +
+        ". Those files are still in the original location, and they may slow down reads.");
       return false;
     }
     return true;
@@ -364,50 +366,51 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
         if (!fs.delete(archiveFile, false)) {
           throw new IOException("Couldn't delete existing archive file (" + archiveFile
               + ") or rename it to the backup file (" + backedupArchiveFile
-              + ")to make room for similarly named file.");
+              + ") to make room for similarly named file.");
         }
       }
       LOG.debug("Backed up archive file from: " + archiveFile);
     }
 
-    LOG.debug("No existing file in archive for:" + archiveFile + ", free to archive original file.");
+    LOG.debug("No existing file in archive for:" + archiveFile +
+        ", free to archive original file.");
 
     // at this point, we should have a free spot for the archive file
-    if (currentFile.moveAndClose(archiveFile)) {
-      LOG.error("Failed to archive file:" + currentFile);
-      return false;
-    } else if (LOG.isDebugEnabled()) {
-      LOG.debug("Finished archiving file from: " + currentFile + ", to: " + archiveFile);
-    }
-    return true;
-  }
+    boolean success = false;
+    for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) {
+      if (i > 0) {
+        // Ensure that the archive directory exists.
+        // The previous "move to archive" operation has failed probably because
+        // the cleaner has removed our archive directory (HBASE-7643).
+        // (we're in a retry loop, so don't worry too much about the exception)
+        try {
+          if (!fs.exists(archiveDir)) {
+            if (fs.mkdirs(archiveDir)) {
+              LOG.debug("Created archive directory:" + archiveDir);
+            }
+          }
+        } catch (IOException e) {
+          LOG.warn("Failed to create the archive directory: " + archiveDir, e);
+        }
+      }
 
-  /**
-   * Simple delete of regular files from the {@link FileSystem}.
-   * <p>
-   * This method is a more generic implementation that the other deleteXXX
-   * methods in this class, allowing more code reuse at the cost of a couple
-   * more, short-lived objects (which should have minimum impact on the jvm).
-   * @param fs {@link FileSystem} where the files live
-   * @param files {@link Collection} of files to be deleted
-   * @throws IOException if a file cannot be deleted. All files will be
-   *           attempted to deleted before throwing the exception, rather than
-   *           failing at the first file.
-   */
-  private static void deleteFilesWithoutArchiving(Collection<File> files) throws IOException {
-    List<IOException> errors = new ArrayList<IOException>(0);
-    for (File file : files) {
       try {
-        LOG.debug("Deleting region file:" + file);
-        file.delete();
+        success = currentFile.moveAndClose(archiveFile);
       } catch (IOException e) {
-        LOG.error("Failed to delete file:" + file);
-        errors.add(e);
+        LOG.warn("Failed to archive file: " + currentFile + " on try #" + i, e);
+        success = false;
       }
     }
-    if (errors.size() > 0) {
-      throw MultipleIOException.createIOException(errors);
+
+    if (!success) {
+      LOG.error("Failed to archive file:" + currentFile);
+      return false;
     }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Finished archiving file from: " + currentFile + ", to: " + archiveFile);
+    }
+    return true;
   }
 
   /**
@@ -553,7 +556,7 @@ public File(FileSystem fs) {
     public boolean moveAndClose(Path dest) throws IOException {
       this.close();
       Path p = this.getPath();
-      return !fs.rename(p, dest);
+      return fs.rename(p, dest);
     }
 
     /**
diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index eb17ac7e7f32..2d177f8c10c1 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -36,7 +36,11 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -44,6 +48,7 @@
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.StoppableImplementation;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -319,6 +324,69 @@ public void testArchiveOnTableFamilyDelete() throws Exception {
       archivedFiles.containsAll(storeFiles));
   }
 
+  /**
+   * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
+   */
+  @Test
+  public void testCleaningRace() throws Exception {
+    final long TEST_TIME = 20 * 1000;
+
+    Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
+    Path rootDir = UTIL.getDataTestDir("testCleaningRace");
+    FileSystem fs = UTIL.getTestFileSystem();
+
+    Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+    Path regionDir = new Path("table", "abcdef");
+    Path familyDir = new Path(regionDir, "cf");
+
+    Path sourceRegionDir = new Path(rootDir, regionDir);
+    fs.mkdirs(sourceRegionDir);
+
+    Stoppable stoppable = new StoppableImplementation();
+
+    // The cleaner should be looping without long pauses to reproduce the race condition.
+    HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
+    try {
+      cleaner.start();
+
+      // Keep creating/archiving new files while the cleaner is running in the other thread
+      long startTime = System.currentTimeMillis();
+      for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
+        Path file = new Path(familyDir,  String.valueOf(fid));
+        Path sourceFile = new Path(rootDir, file);
+        Path archiveFile = new Path(archiveDir, file);
+
+        fs.createNewFile(sourceFile);
+
+        try {
+          // Try to archive the file
+          HFileArchiver.archiveRegion(conf, fs, rootDir,
+              sourceRegionDir.getParent(), sourceRegionDir);
+
+          // The archiver succeded, the file is no longer in the original location
+          // but it's in the archive location.
+          LOG.debug("hfile=" + fid + " should be in the archive");
+          assertTrue(fs.exists(archiveFile));
+          assertFalse(fs.exists(sourceFile));
+        } catch (IOException e) {
+          // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
+          // in this case, the file should not be archived, and we should have the file
+          // in the original location.
+          LOG.debug("hfile=" + fid + " should be in the source location");
+          assertFalse(fs.exists(archiveFile));
+          assertTrue(fs.exists(sourceFile));
+
+          // Avoid to have this file in the next run
+          fs.delete(sourceFile, false);
+        }
+      }
+    } finally {
+      stoppable.stop("test end");
+      cleaner.join();
+      fs.delete(rootDir, true);
+    }
+  }
+
   private void clearArchiveDirectory() throws IOException {
     UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
   }

From 9f5a11e62659e094117aaaf758a7ebd16cd95b51 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 27 Jan 2013 00:18:10 +0000
Subject: [PATCH 0732/1540] HBASE-7681 Address some recent random test failures

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439004 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/TestNodeHealthCheckChore.java       |  17 +-
 .../apache/hadoop/hbase/client/TestAdmin.java | 201 +++++++++---------
 .../coprocessor/SimpleRegionObserver.java     |   2 -
 .../hbase/io/hfile/TestLruBlockCache.java     |   2 +-
 .../hadoop/hbase/mapreduce/TestImportTsv.java |  14 +-
 .../hbase/master/TestSplitLogManager.java     |   2 +-
 .../regionserver/TestSplitLogWorker.java      |  28 +--
 .../hbase/thrift/TestThriftServerCmdLine.java |   2 +-
 .../util/TestMiniClusterLoadSequential.java   |   2 +-
 9 files changed, 136 insertions(+), 134 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java b/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java
index 05f699dfb0d2..9ae974e33fd4 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java
@@ -19,6 +19,7 @@
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -55,26 +56,28 @@ public void testHealthChecker() throws Exception {
     Configuration config = getConfForNodeHealthScript();
     config.addResource(healthScriptFile.getName());
     String location = healthScriptFile.getAbsolutePath();
-    long timeout = config.getLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 100);
+    long timeout = config.getLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 200);
 
-    String normalScript = "echo \"I am all fine\"";
-    createScript(normalScript, true);
     HealthChecker checker = new HealthChecker();
     checker.init(location, timeout);
+
+    String normalScript = "echo \"I am all fine\"";
+    createScript(normalScript, true);
     HealthReport report = checker.checkHealth();
-    assertTrue(report.getStatus() == HealthCheckerExitStatus.SUCCESS);
+    assertEquals(HealthCheckerExitStatus.SUCCESS, report.getStatus());
+
     LOG.info("Health Status:" + checker);
 
     String errorScript = "echo ERROR\n echo \"Node not healthy\"";
     createScript(errorScript, true);
     report = checker.checkHealth();
-    assertTrue(report.getStatus() == HealthCheckerExitStatus.FAILED);
+    assertEquals(HealthCheckerExitStatus.FAILED, report.getStatus());
     LOG.info("Health Status:" + report.getHealthReport());
 
     String timeOutScript = "sleep 4\n echo\"I am fine\"";
     createScript(timeOutScript, true);
     report = checker.checkHealth();
-    assertTrue(report.getStatus() == HealthCheckerExitStatus.TIMED_OUT);
+    assertEquals(HealthCheckerExitStatus.TIMED_OUT, report.getStatus());
     LOG.info("Health Status:" + report.getHealthReport());
 
     healthScriptFile.delete();
@@ -113,7 +116,7 @@ private Configuration getConfForNodeHealthScript() {
     conf.set(HConstants.HEALTH_SCRIPT_LOC,
       healthScriptFile.getAbsolutePath());
     conf.setLong(HConstants.HEALTH_FAILURE_THRESHOLD, 3);
-    conf.setLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 100);
+    conf.setLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 200);
     return conf;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index db12f999924c..66b5dd786d36 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -874,119 +874,116 @@ void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts,
     assertFalse(admin.tableExists(tableName));
     final HTable table = TEST_UTIL.createTable(tableName, familyNames,
       numVersions, blockSize);
-    try {
-      int rowCount = 0;
-      byte[] q = new byte[0];
-
-      // insert rows into column families. The number of rows that have values
-      // in a specific column family is decided by rowCounts[familyIndex]
-      for (int index = 0; index < familyNames.length; index++) {
-        ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
-        for (int i = 0; i < rowCounts[index]; i++) {
-          byte[] k = Bytes.toBytes(i);
-          Put put = new Put(k);
-          put.add(familyNames[index], q, k);
-          puts.add(put);
-        }
-        table.put(puts);
-
-        if ( rowCount < rowCounts[index] ) {
-          rowCount = rowCounts[index];
-        }
+    int rowCount = 0;
+    byte[] q = new byte[0];
+
+    // insert rows into column families. The number of rows that have values
+    // in a specific column family is decided by rowCounts[familyIndex]
+    for (int index = 0; index < familyNames.length; index++) {
+      ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
+      for (int i = 0; i < rowCounts[index]; i++) {
+        byte[] k = Bytes.toBytes(i);
+        Put put = new Put(k);
+        put.add(familyNames[index], q, k);
+        puts.add(put);
       }
+      table.put(puts);
 
-      // get the initial layout (should just be one region)
-      Map<HRegionInfo,HServerAddress> m = table.getRegionsInfo();
-      System.out.println("Initial regions (" + m.size() + "): " + m);
-      assertTrue(m.size() == 1);
-
-      // Verify row count
-      Scan scan = new Scan();
-      ResultScanner scanner = table.getScanner(scan);
-      int rows = 0;
-      for(@SuppressWarnings("unused") Result result : scanner) {
-        rows++;
+      if ( rowCount < rowCounts[index] ) {
+        rowCount = rowCounts[index];
       }
-      scanner.close();
-      assertEquals(rowCount, rows);
-
-      // Have an outstanding scan going on to make sure we can scan over splits.
-      scan = new Scan();
-      scanner = table.getScanner(scan);
-      // Scan first row so we are into first region before split happens.
-      scanner.next();
+    }
 
-      final AtomicInteger count = new AtomicInteger(0);
-      Thread t = new Thread("CheckForSplit") {
-        public void run() {
-          for (int i = 0; i < 20; i++) {
-            try {
-              sleep(1000);
-            } catch (InterruptedException e) {
-              continue;
-            }
-            // check again    table = new HTable(conf, tableName);
-            Map<HRegionInfo, HServerAddress> regions = null;
-            try {
-              regions = table.getRegionsInfo();
-            } catch (IOException e) {
-              e.printStackTrace();
-            }
-            if (regions == null) continue;
-            count.set(regions.size());
-            if (count.get() >= 2) break;
-            LOG.debug("Cycle waiting on split");
+    // get the initial layout (should just be one region)
+    Map<HRegionInfo,HServerAddress> m = table.getRegionsInfo();
+    System.out.println("Initial regions (" + m.size() + "): " + m);
+    assertTrue(m.size() == 1);
+
+    // Verify row count
+    Scan scan = new Scan();
+    ResultScanner scanner = table.getScanner(scan);
+    int rows = 0;
+    for(@SuppressWarnings("unused") Result result : scanner) {
+      rows++;
+    }
+    scanner.close();
+    assertEquals(rowCount, rows);
+
+    // Have an outstanding scan going on to make sure we can scan over splits.
+    scan = new Scan();
+    scanner = table.getScanner(scan);
+    // Scan first row so we are into first region before split happens.
+    scanner.next();
+
+    final AtomicInteger count = new AtomicInteger(0);
+    Thread t = new Thread("CheckForSplit") {
+      public void run() {
+        for (int i = 0; i < 20; i++) {
+          try {
+            sleep(1000);
+          } catch (InterruptedException e) {
+            continue;
           }
-        }
-      };
-      t.start();
-      // Split the table
-      this.admin.split(tableName, splitPoint);
-      t.join();
-
-      // Verify row count
-      rows = 1; // We counted one row above.
-      for (@SuppressWarnings("unused") Result result : scanner) {
-        rows++;
-        if (rows > rowCount) {
-          scanner.close();
-          assertTrue("Scanned more than expected (" + rowCount + ")", false);
+          // check again    table = new HTable(conf, tableName);
+          Map<HRegionInfo, HServerAddress> regions = null;
+          try {
+            regions = table.getRegionsInfo();
+          } catch (IOException e) {
+            e.printStackTrace();
+          }
+          if (regions == null) continue;
+          count.set(regions.size());
+          if (count.get() >= 2) break;
+          LOG.debug("Cycle waiting on split");
         }
       }
-      scanner.close();
-      assertEquals(rowCount, rows);
-
-      Map<HRegionInfo, HServerAddress> regions = null;
-      try {
-        regions = table.getRegionsInfo();
-      } catch (IOException e) {
-        e.printStackTrace();
+    };
+    t.start();
+    // Split the table
+    this.admin.split(tableName, splitPoint);
+    t.join();
+
+    // Verify row count
+    rows = 1; // We counted one row above.
+    for (@SuppressWarnings("unused") Result result : scanner) {
+      rows++;
+      if (rows > rowCount) {
+        scanner.close();
+        assertTrue("Scanned more than expected (" + rowCount + ")", false);
       }
-      assertEquals(2, regions.size());
-      HRegionInfo[] r = regions.keySet().toArray(new HRegionInfo[0]);
-      if (splitPoint != null) {
-        // make sure the split point matches our explicit configuration
-        assertEquals(Bytes.toString(splitPoint),
-            Bytes.toString(r[0].getEndKey()));
-        assertEquals(Bytes.toString(splitPoint),
-            Bytes.toString(r[1].getStartKey()));
-        LOG.debug("Properly split on " + Bytes.toString(splitPoint));
-      } else {
-        if (familyNames.length > 1) {
-          int splitKey = Bytes.toInt(r[0].getEndKey());
-          // check if splitKey is based on the largest column family
-          // in terms of it store size
-          int deltaForLargestFamily = Math.abs(rowCount/2 - splitKey);
-          for (int index = 0; index < familyNames.length; index++) {
-            int delta = Math.abs(rowCounts[index]/2 - splitKey);
-            assertTrue(delta >= deltaForLargestFamily);
-          }
+    }
+    scanner.close();
+    assertEquals(rowCount, rows);
+
+    Map<HRegionInfo, HServerAddress> regions = null;
+    try {
+      regions = table.getRegionsInfo();
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+    assertEquals(2, regions.size());
+    HRegionInfo[] r = regions.keySet().toArray(new HRegionInfo[0]);
+    if (splitPoint != null) {
+      // make sure the split point matches our explicit configuration
+      assertEquals(Bytes.toString(splitPoint),
+          Bytes.toString(r[0].getEndKey()));
+      assertEquals(Bytes.toString(splitPoint),
+          Bytes.toString(r[1].getStartKey()));
+      LOG.debug("Properly split on " + Bytes.toString(splitPoint));
+    } else {
+      if (familyNames.length > 1) {
+        int splitKey = Bytes.toInt(r[0].getEndKey());
+        // check if splitKey is based on the largest column family
+        // in terms of it store size
+        int deltaForLargestFamily = Math.abs(rowCount/2 - splitKey);
+        for (int index = 0; index < familyNames.length; index++) {
+          int delta = Math.abs(rowCounts[index]/2 - splitKey);
+          assertTrue(delta >= deltaForLargestFamily);
         }
       }
-    } finally {
-      TEST_UTIL.deleteTable(tableName);
-      table.close();
     }
+    TEST_UTIL.deleteTable(tableName);
+    table.close();
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index 2a5baa593347..70b628c13c99 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -107,8 +107,6 @@ public void start(CoprocessorEnvironment e) throws IOException {
     Leases leases = re.getRegionServerServices().getLeases();
     leases.createLease("x", null);
     leases.cancelLease("x");
-    Integer lid = re.getRegion().getLock(null, Bytes.toBytes("some row"), true);
-    re.getRegion().releaseRowLock(lid);
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index b6aef4e47540..4554a0e84a3f 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -94,7 +94,7 @@ public void testBackgroundEvictionThread() throws Exception {
     int n = 0;
     while(cache.getEvictionCount() == 0) {
       Thread.sleep(200);
-      assertTrue(n++ < 10);
+      assertTrue(n++ < 20);
     }
     System.out.println("Background Evictions run: " + cache.getEvictionCount());
 
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index ff1ede3ec3d5..a21b99b013b9 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -23,6 +23,8 @@
 import java.util.List;
 import java.util.ArrayList;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -52,6 +54,7 @@
 
 @Category(MediumTests.class)
 public class TestImportTsv {
+  private static final Log LOG = LogFactory.getLog(TestImportTsv.class);
 
   @Test
   public void testTsvParserSpecParsing() {
@@ -185,7 +188,7 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
     // Cluster
     HBaseTestingUtility htu1 = new HBaseTestingUtility();
 
-    MiniHBaseCluster cluster = htu1.startMiniCluster();
+    htu1.startMiniCluster();
     htu1.startMiniMapReduceCluster();
 
     GenericOptionsParser opts = new GenericOptionsParser(htu1.getConfiguration(), args);
@@ -193,7 +196,6 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
     args = opts.getRemainingArgs();
 
     try {
-
       FileSystem fs = FileSystem.get(conf);
       FSDataOutputStream op = fs.create(new Path(inputFile), true);
       String line = "KEY\u001bVALUE1\u001bVALUE2\n";
@@ -202,13 +204,14 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
 
       final byte[] FAM = Bytes.toBytes(family);
       final byte[] TAB = Bytes.toBytes(tableName);
-      final byte[] QA = Bytes.toBytes("A");
-      final byte[] QB = Bytes.toBytes("B");
       if (conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY) == null) {
         HTableDescriptor desc = new HTableDescriptor(TAB);
         desc.addFamily(new HColumnDescriptor(FAM));
-        new HBaseAdmin(conf).createTable(desc);
+        HBaseAdmin admin = new HBaseAdmin(conf);
+        admin.createTable(desc);
+        admin.close();
       } else { // set the hbaseAdmin as we are not going through main()
+        LOG.info("set the hbaseAdmin");
         ImportTsv.createHbaseAdmin(conf);
       }
       Job job = ImportTsv.createSubmittableJob(conf, args);
@@ -250,6 +253,7 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
           // continue
         }
       }
+      table.close();
       assertTrue(verified);
     } finally {
       htu1.shutdownMiniMapReduceCluster();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 749ccd7d70ed..d76601861eb6 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -106,7 +106,7 @@ public void setup() throws Exception {
 
     stopped = false;
     resetCounters();
-    to = 4000;
+    to = 6000;
     conf.setInt("hbase.splitlog.manager.timeout", to);
     conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to);
     conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index 2611ca0ba66e..6fa56dbf78c1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -139,7 +139,7 @@ public void testAcquireTaskAtStartup() throws Exception {
       "rs", neverEndingTask);
     slw.start();
     try {
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tatas")), "rs"));
     } finally {
@@ -174,10 +174,10 @@ public void testRaceForTask() throws Exception {
     slw1.start();
     slw2.start();
     try {
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
       // Assert that either the tot_wkr_failed_to_grab_task_owned count was set of if
       // not it, that we fell through to the next counter in line and it was set.
-      assertTrue(waitForCounterBoolean(tot_wkr_failed_to_grab_task_owned, 0, 1, 1000) ||
+      assertTrue(waitForCounterBoolean(tot_wkr_failed_to_grab_task_owned, 0, 1, 1500) ||
         tot_wkr_failed_to_grab_task_lost_race.get() == 1);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "trft")), "svr1") ||
@@ -206,14 +206,14 @@ public void testPreemptTask() throws Exception {
         TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
       assertEquals(1, slw.taskReadySeq);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tpt_task")), "tpt_svr"));
 
       ZKUtil.setData(zkw, ZKSplitLog.getEncodedNodeName(zkw, "tpt_task"),
         TaskState.TASK_UNASSIGNED.get("manager"));
-      waitForCounter(tot_wkr_preempt_task, 0, 1, 1000);
+      waitForCounter(tot_wkr_preempt_task, 0, 1, 1500);
     } finally {
       stopSplitLogWorker(slw);
     }
@@ -234,7 +234,7 @@ public void testMultipleTasks() throws Exception {
         TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
       // now the worker is busy doing the above task
 
       // create another task
@@ -245,9 +245,9 @@ public void testMultipleTasks() throws Exception {
       // preempt the first task, have it owned by another worker
       ZKUtil.setData(zkw, ZKSplitLog.getEncodedNodeName(zkw, "tmt_task"),
         TaskState.TASK_OWNED.get("another-worker"));
-      waitForCounter(tot_wkr_preempt_task, 0, 1, 1000);
+      waitForCounter(tot_wkr_preempt_task, 0, 1, 1500);
 
-      waitForCounter(tot_wkr_task_acquired, 1, 2, 1000);
+      waitForCounter(tot_wkr_task_acquired, 1, 2, 1500);
       assertEquals(2, slw.taskReadySeq);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tmt_task_2")), "tmt_svr"));
@@ -264,19 +264,19 @@ public void testRescan() throws Exception {
         "svr", neverEndingTask);
     slw.start();
     Thread.yield(); // let the worker start
-    Thread.sleep(100);
+    Thread.sleep(200);
 
     String task = ZKSplitLog.getEncodedNodeName(zkw, "task");
     zkw.getRecoverableZooKeeper().create(task,
       TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
       CreateMode.PERSISTENT);
 
-    waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
+    waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
     // now the worker is busy doing the above task
 
     // preempt the task, have it owned by another worker
     ZKUtil.setData(zkw, task, TaskState.TASK_UNASSIGNED.get("manager"));
-    waitForCounter(tot_wkr_preempt_task, 0, 1, 1000);
+    waitForCounter(tot_wkr_preempt_task, 0, 1, 1500);
 
     // create a RESCAN node
     String rescan = ZKSplitLog.getEncodedNodeName(zkw, "RESCAN");
@@ -284,13 +284,13 @@ public void testRescan() throws Exception {
       TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
       CreateMode.PERSISTENT_SEQUENTIAL);
 
-    waitForCounter(tot_wkr_task_acquired, 1, 2, 1000);
+    waitForCounter(tot_wkr_task_acquired, 1, 2, 1500);
     // RESCAN node might not have been processed if the worker became busy
     // with the above task. preempt the task again so that now the RESCAN
     // node is processed
     ZKUtil.setData(zkw, task, TaskState.TASK_UNASSIGNED.get("manager"));
-    waitForCounter(tot_wkr_preempt_task, 1, 2, 1000);
-    waitForCounter(tot_wkr_task_acquired_rescan, 0, 1, 1000);
+    waitForCounter(tot_wkr_preempt_task, 1, 2, 1500);
+    waitForCounter(tot_wkr_task_acquired_rescan, 0, 1, 1500);
 
     List<String> nodes = ZKUtil.listChildrenNoWatch(zkw, zkw.splitLogZNode);
     LOG.debug(nodes);
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index f3df9ed62c26..516f3215a635 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -139,7 +139,7 @@ public void run() {
     cmdLineThread.start();
   }
 
-  @Test(timeout=60 * 1000)
+  @Test(timeout=120 * 1000)
   public void testRunThriftServer() throws Exception {
     List<String> args = new ArrayList<String>();
     if (implType != null) {
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
index d24648c014d6..90f7c6cd0de1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
@@ -57,7 +57,7 @@ public class TestMiniClusterLoadSequential {
   protected static final byte[] CF = Bytes.toBytes("load_test_cf");
   protected static final int NUM_THREADS = 8;
   protected static final int NUM_RS = 2;
-  protected static final int TIMEOUT_MS = 120000;
+  protected static final int TIMEOUT_MS = 180000;
   protected static final HBaseTestingUtility TEST_UTIL =
       new HBaseTestingUtility();
 

From 7bf2286b6b0305adf7f41031f6b7ccda2741c90d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 27 Jan 2013 05:40:30 +0000
Subject: [PATCH 0733/1540] HBASE-7681 Addendum, close tables in
 TestRegionServerMetrics

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439025 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/TestRegionServerMetrics.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index aab7787642f4..9b82cf655ceb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -201,6 +201,7 @@ public void testOperationMetrics() throws IOException {
       // Three gets. one for gets. One for append. One for increment.
       assertTimeVaryingMetricCount(4, TABLE_NAME, cf, regionName, "get_");
   
+      hTable.close();
     }
   
   @Test
@@ -227,6 +228,8 @@ public void testRemoveRegionMetrics() throws IOException, InterruptedException {
     admin.deleteTable(TABLE_NAME.getBytes());
 
     assertTimeVaryingMetricCount(0, TABLE_NAME, cf, regionName, "get_");
+
+    hTable.close();
   }
   
   @Test
@@ -346,6 +349,8 @@ public void testGetNextSize() throws IOException, InterruptedException {
     }
     assertSizeMetric(tableName, cfs,
         new int[] {kvLength, kvLength, kvLength, kvLength});
+
+    hTable.close();
   }
 }
 

From 0b12c1d26b72884fa1e578a15408d3972614d1be Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 27 Jan 2013 21:06:32 +0000
Subject: [PATCH 0734/1540] HBASE-7687
 TestCatalogTracker.testServerNotRunningIOException fails occasionally (Ted
 Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439179 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/catalog/TestCatalogTracker.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
index 5e6fbd7d1566..c22123d44092 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
@@ -25,6 +25,7 @@
 import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -209,18 +210,20 @@ public void testServerNotRunningIOException()
         // So, do this in a thread and then reset meta location to break it out
         // of its wait after a bit of time.
         final AtomicBoolean metaSet = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(1);
         Thread t = new Thread() {
           @Override
           public void run() {
             try {
-              metaSet.set(ct.waitForMetaServerConnection(100000) !=  null);
+              latch.countDown();
+              metaSet.set(ct.waitForMeta(100000) !=  null);
             } catch (Exception e) {
               throw new RuntimeException(e);
             }
           }
         };
         t.start();
-        while(!t.isAlive()) Threads.sleep(1);
+        latch.await();
         Threads.sleep(1);
         // Now reset the meta as though it were redeployed.
         ct.setMetaLocation(SN);

From 7a96a1a9fad5d762c3b7f1713ba1a507d3dd98be Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 28 Jan 2013 18:02:39 +0000
Subject: [PATCH 0735/1540] HBASE-7507 Revert, due to 0.94 test stability
 issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439534 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   | 11 ----
 .../hadoop/hbase/regionserver/Store.java      | 55 +------------------
 2 files changed, 3 insertions(+), 63 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 14a66421c76c..e70bd3be708c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -477,17 +477,6 @@ public static enum Modify {
    */
   public static long DEFAULT_HBASE_CLIENT_PAUSE = 1000;
 
-  /**
-   * Parameter name for server pause value, used mostly as value to wait before
-   * running a retry of a failed operation.
-   */
-  public static String HBASE_SERVER_PAUSE = "hbase.server.pause";
-
-  /**
-   * Default value of {@link #HBASE_SERVER_PAUSE}.
-   */
-  public static int DEFAULT_HBASE_SERVER_PAUSE = 1000;
-
   /**
    * Parameter name for maximum retries, used as maximum for all retryable
    * operations such as fetching of the root region from root region server,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 6a11a24040cb..b895688b6e16 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -166,10 +166,6 @@ public class Store extends SchemaConfigured implements HeapSize {
 
   private final Compactor compactor;
 
-  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
-  private static int flush_retries_number;
-  private static int pauseTime;
-
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -249,17 +245,6 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.bytesPerChecksum = getBytesPerChecksum(conf);
     // Create a compaction tool instance
     this.compactor = new Compactor(this.conf);
-    if (Store.flush_retries_number == 0) {
-      Store.flush_retries_number = conf.getInt(
-          "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
-      Store.pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE,
-          HConstants.DEFAULT_HBASE_SERVER_PAUSE);
-      if (Store.flush_retries_number <= 0) {
-        throw new IllegalArgumentException(
-            "hbase.hstore.flush.retries.number must be > 0, not "
-                + Store.flush_retries_number);
-      }
-    }
   }
 
   /**
@@ -737,43 +722,8 @@ private Path flushCache(final long logCacheFlushId,
     // If an exception happens flushing, we let it out without clearing
     // the memstore snapshot.  The old snapshot will be returned when we say
     // 'snapshot', the next time flush comes around.
-    // Retry after catching exception when flushing, otherwise server will abort
-    // itself
-    IOException lastException = null;
-    for (int i = 0; i < Store.flush_retries_number; i++) {
-      try {
-        Path pathName = internalFlushCache(snapshot, logCacheFlushId,
-            snapshotTimeRangeTracker, flushedSize, status);
-        try {
-          // Path name is null if there is no entry to flush
-          if (pathName != null) {
-            validateStoreFile(pathName);
-          }
-          return pathName;
-        } catch (Exception e) {
-          LOG.warn("Failed validating store file " + pathName
-              + ", retring num=" + i, e);
-          if (e instanceof IOException) {
-            lastException = (IOException) e;
-          } else {
-            lastException = new IOException(e);
-          }
-        }
-      } catch (IOException e) {
-        LOG.warn("Failed flushing store file, retring num=" + i, e);
-        lastException = e;
-      }
-      if (lastException != null) {
-        try {
-          Thread.sleep(pauseTime);
-        } catch (InterruptedException e) {
-          IOException iie = new InterruptedIOException();
-          iie.initCause(e);
-          throw iie;
-        }
-      }
-    }
-    throw lastException;
+    return internalFlushCache(
+        snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);
   }
 
   /*
@@ -892,6 +842,7 @@ private StoreFile commitFile(final Path path,
     // Write-out finished successfully, move into the right spot
     String fileName = path.getName();
     Path dstPath = new Path(homedir, fileName);
+    validateStoreFile(path);
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);

From de4d2d16cbb4690a0de8c544871b174243ebeb44 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 28 Jan 2013 18:04:40 +0000
Subject: [PATCH 0736/1540] HBASE-7599 Revert, due to 0.94 test stability
 issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439538 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 51 ++++++++-----------
 .../hbase/regionserver/HRegionServer.java     | 12 ++---
 2 files changed, 26 insertions(+), 37 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f20275bfa5ef..c79f9a9e0c84 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3520,6 +3520,7 @@ class RegionScannerImpl implements RegionScanner {
     private final KeyValue KV_LIMIT = new KeyValue();
     private final byte [] stopRow;
     private Filter filter;
+    private List<KeyValue> results = new ArrayList<KeyValue>();
     private int batch;
     private int isScan;
     private boolean filterClosed = false;
@@ -3634,16 +3635,11 @@ public boolean nextRaw(List<KeyValue> outResults, String metric)
     @Override
     public boolean nextRaw(List<KeyValue> outResults, int limit,
         String metric) throws IOException {
-      boolean returnResult;
-      if (outResults.isEmpty()) {
-        // Usually outResults is empty. This is true when next is called
-        // to handle scan or get operation.
-        returnResult = nextInternal(outResults, limit, metric);
-      } else {
-        List<KeyValue> tmpList = new ArrayList<KeyValue>();
-        returnResult = nextInternal(tmpList, limit, metric);
-        outResults.addAll(tmpList);
-      }
+      results.clear();
+
+      boolean returnResult = nextInternal(limit, metric);
+
+      outResults.addAll(results);
       resetFilters();
       if (isFilterDone()) {
         return false;
@@ -3665,12 +3661,10 @@ public boolean next(List<KeyValue> outResults, String metric)
       return next(outResults, batch, metric);
     }
 
-    private void populateFromJoinedHeap(List<KeyValue> results, int limit, String metric)
-        throws IOException {
+    private void populateFromJoinedHeap(int limit, String metric) throws IOException {
       assert joinedContinuationRow != null;
-      KeyValue kv = populateResult(results, this.joinedHeap, limit,
-        joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(),
-        joinedContinuationRow.getRowLength(), metric);
+      KeyValue kv = populateResult(this.joinedHeap, limit, joinedContinuationRow.getBuffer(),
+        joinedContinuationRow.getRowOffset(), joinedContinuationRow.getRowLength(), metric);
       if (kv != KV_LIMIT) {
         // We are done with this row, reset the continuation.
         joinedContinuationRow = null;
@@ -3682,7 +3676,6 @@ private void populateFromJoinedHeap(List<KeyValue> results, int limit, String me
 
     /**
      * Fetches records with this row into result list, until next row or limit (if not -1).
-     * @param results
      * @param heap KeyValueHeap to fetch data from. It must be positioned on correct row before call.
      * @param limit Max amount of KVs to place in result list, -1 means no limit.
      * @param currentRow Byte array with key we are fetching.
@@ -3691,8 +3684,8 @@ private void populateFromJoinedHeap(List<KeyValue> results, int limit, String me
      * @param metric Metric key to be passed into KeyValueHeap::next().
      * @return true if limit reached, false otherwise.
      */
-    private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int limit,
-        byte[] currentRow, int offset, short length, String metric) throws IOException {
+    private KeyValue populateResult(KeyValueHeap heap, int limit, byte[] currentRow, int offset,
+        short length, String metric) throws IOException {
       KeyValue nextKv;
       do {
         heap.next(results, limit - results.size(), metric);
@@ -3711,11 +3704,7 @@ public synchronized boolean isFilterDone() {
       return this.filter != null && this.filter.filterAllRemaining();
     }
 
-    private boolean nextInternal(List<KeyValue> results, int limit, String metric)
-    throws IOException {
-      if (!results.isEmpty()) {
-        throw new IllegalArgumentException("First parameter should be an empty list");
-      }
+    private boolean nextInternal(int limit, String metric) throws IOException {
       RpcCallContext rpcCall = HBaseServer.getCurrentCall();
       // The loop here is used only when at some point during the next we determine
       // that due to effects of filters or otherwise, we have an empty row in the result.
@@ -3761,13 +3750,11 @@ private boolean nextInternal(List<KeyValue> results, int limit, String metric)
           // Techically, if we hit limits before on this row, we don't need this call.
           if (filterRowKey(currentRow, offset, length)) {
             nextRow(currentRow, offset, length);
-            results.clear();
             continue;
           }
 
           // Ok, we are good, let's try to get some results from the main heap.
-          KeyValue nextKv = populateResult(results, this.storeHeap, limit, currentRow, 
-              offset, length, metric);
+          KeyValue nextKv = populateResult(this.storeHeap, limit, currentRow, offset, length, metric);
           if (nextKv == KV_LIMIT) {
             if (this.filter != null && filter.hasFilterRow()) {
               throw new IncompatibleFilterException(
@@ -3786,8 +3773,13 @@ private boolean nextInternal(List<KeyValue> results, int limit, String metric)
           }
 
           if (isEmptyRow || filterRow()) {
+            // this seems like a redundant step - we already consumed the row
+            // there're no left overs.
+            // the reasons for calling this method are:
+            // 1. reset the filters.
+            // 2. provide a hook to fast forward the row (used by subclasses)
             nextRow(currentRow, offset, length);
-            results.clear();
+
             // This row was totally filtered out, if this is NOT the last row,
             // we should continue on. Otherwise, nothing else to do.
             if (!stopRow) continue;
@@ -3807,12 +3799,12 @@ private boolean nextInternal(List<KeyValue> results, int limit, String metric)
               || this.joinedHeap.seek(KeyValue.createFirstOnRow(currentRow, offset, length));
             if (mayHaveData) {
               joinedContinuationRow = current;
-              populateFromJoinedHeap(results, limit, metric);
+              populateFromJoinedHeap(limit, metric);
             }
           }
         } else {
           // Populating from the joined map was stopped by limits, populate some more.
-          populateFromJoinedHeap(results, limit, metric);
+          populateFromJoinedHeap(limit, metric);
         }
 
         // We may have just called populateFromJoinedMap and hit the limits. If that is
@@ -3848,6 +3840,7 @@ protected void nextRow(byte [] currentRow, int offset, short length) throws IOEx
       while((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
         this.storeHeap.next(MOCKED_LIST);       
       }
+      results.clear();
       resetFilters();
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index d561d3ded465..7c4041d3498f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2457,10 +2457,8 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
             results, nbRows);
         if (!results.isEmpty()) {
           for (Result r : results) {
-            if (maxScannerResultSize < Long.MAX_VALUE){
-              for (KeyValue kv : r.raw()) {
-                currentScanResultSize += kv.heapSize();
-              }
+            for (KeyValue kv : r.raw()) {
+              currentScanResultSize += kv.heapSize();
             }
           }
         }
@@ -2480,10 +2478,8 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
             // Collect values to be returned here
             boolean moreRows = s.nextRaw(values, SchemaMetrics.METRIC_NEXTSIZE);
             if (!values.isEmpty()) {
-              if (maxScannerResultSize < Long.MAX_VALUE){
-                for (KeyValue kv : values) {
-                  currentScanResultSize += kv.heapSize();
-                }
+              for (KeyValue kv : values) {
+                currentScanResultSize += kv.heapSize();
               }
               results.add(new Result(values));
             }

From fbc3613987f089f02f360c933a0dbba55abe7f53 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 28 Jan 2013 18:43:26 +0000
Subject: [PATCH 0737/1540] HBASE-7693 Hostname returned by
 TableInputFormatBase.reverseDNS contains trailing period (Robert Dyer)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439558 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/TableInputFormatBase.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index eb4954aaffb5..ad855e215b56 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.JobContext;
@@ -215,7 +216,7 @@ public List<InputSplit> getSplits(JobContext context) throws IOException {
   private String reverseDNS(InetAddress ipAddress) throws NamingException {
     String hostName = this.reverseDNSCacheMap.get(ipAddress);
     if (hostName == null) {
-      hostName = DNS.reverseDns(ipAddress, this.nameServer);
+      hostName = Strings.domainNamePointerToHostName(DNS.reverseDns(ipAddress, this.nameServer));
       this.reverseDNSCacheMap.put(ipAddress, hostName);
     }
     return hostName;

From 044b39d2afd45dff892134b5b4079bc2b1e8980a Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Tue, 29 Jan 2013 00:50:02 +0000
Subject: [PATCH 0738/1540] HBASE-7626 Backport client connection cleanup from
 HBASE-7460

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439723 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/ipc/SecureRpcEngine.java     | 188 ++++++------------
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  |   2 +-
 .../hbase/client/HConnectionManager.java      |  97 +++++----
 .../apache/hadoop/hbase/ipc/HBaseClient.java  |  26 ---
 .../org/apache/hadoop/hbase/ipc/HBaseRPC.java | 159 +++------------
 .../apache/hadoop/hbase/ipc/RpcEngine.java    |  20 +-
 .../hadoop/hbase/ipc/WritableRpcEngine.java   | 170 ++++++----------
 .../hbase/regionserver/HRegionServer.java     |  14 +-
 .../master/ReplicationLogCleaner.java         |   2 +-
 .../resources/hbase-webapps/master/table.jsp  |   2 +-
 .../apache/hadoop/hbase/MiniHBaseCluster.java |   2 +-
 .../hbase/catalog/TestCatalogTracker.java     |  14 +-
 .../TestMetaReaderEditorNoCluster.java        |   2 +-
 .../apache/hadoop/hbase/client/TestHCM.java   |   4 +-
 .../hadoop/hbase/ipc/TestDelayedRpc.java      | 146 ++++++++------
 .../hadoop/hbase/ipc/TestPBOnWritableRpc.java |  14 +-
 .../hbase/ipc/TestProtocolExtension.java      |   9 +-
 .../hbase/master/TestCatalogJanitor.java      |   2 +-
 .../hbase/master/TestHMasterRPCException.java |  12 +-
 .../util/hbck/OfflineMetaRebuildTestCore.java |   2 +-
 .../util/hbck/TestOfflineMetaRebuildBase.java |   2 +-
 21 files changed, 357 insertions(+), 532 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
index 7d76d053fe49..fd505d71b598 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
@@ -20,11 +20,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.security.HBasePolicyProvider;
@@ -33,20 +29,11 @@
 import org.apache.hadoop.hbase.security.token.AuthenticationTokenSecretManager;
 import org.apache.hadoop.hbase.util.Objects;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
-import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
 
-import javax.net.SocketFactory;
-import java.io.DataInput;
-import java.io.DataOutput;
 import java.io.IOException;
 import java.lang.reflect.*;
-import java.net.ConnectException;
 import java.net.InetSocketAddress;
-import java.net.SocketTimeoutException;
-import java.util.HashMap;
-import java.util.Map;
 
 /**
  * A loadable RPC engine supporting SASL authentication of connections, using
@@ -64,93 +51,46 @@
  */
 public class SecureRpcEngine implements RpcEngine {
   // Leave this out in the hadoop ipc package but keep class name.  Do this
-  // so that we dont' get the logging of this class's invocations by doing our
+  // so that we do not get the logging of this class' invocations by doing our
   // blanket enabling DEBUG on the o.a.h.h. package.
   protected static final Log LOG =
     LogFactory.getLog("org.apache.hadoop.ipc.SecureRpcEngine");
 
-  private SecureRpcEngine() {
-    super();
-  }                                  // no public ctor
-
-  /* Cache a client using the configured clusterId */
-  static private class ClientCache {
-    private Map<String, SecureClient> clients =
-      new HashMap<String, SecureClient>();
-
-    protected ClientCache() {}
-
-    /**
-     * Construct & cache an IPC client with the configured
-     * {@link HConstants#CLUSTER_ID} if no cached client exists.
-     *
-     * @param conf
-     *          Configuration
-     * @param factory
-     *          socket factory
-     * @return an IPC client
-     */
-    protected synchronized SecureClient getClient(Configuration conf,
-        SocketFactory factory) {
-      String clusterId = conf.get(HConstants.CLUSTER_ID, "default");
-      SecureClient client = clients.get(clusterId);
-      if (client == null) {
-        // Make an hbase client instead of hadoop Client.
-        client = new SecureClient(HbaseObjectWritable.class, conf, factory);
-        clients.put(clusterId, client);
-      } else {
-        client.incCount();
-      }
-      return client;
-    }
+  private Configuration conf;
+  private SecureClient client;
 
-    /**
-     * Construct & cache an IPC client with the configured
-     * {@link HConstants#CLUSTER_ID} if no cached client exists.
-     *
-     * @param conf
-     *          Configuration
-     * @return an IPC client
-     */
-    protected synchronized SecureClient getClient(Configuration conf) {
-      return getClient(conf, SocketFactory.getDefault());
+  @Override
+  public void setConf(Configuration config) {
+    this.conf = config;
+    if (User.isHBaseSecurityEnabled(conf)) {
+      HBaseSaslRpcServer.init(conf);
     }
-
-    /**
-     * Stop a RPC client connection
-     * A RPC client is closed only when its reference count becomes zero.
-     * @param client client to stop
-     */
-    protected void stopClient(SecureClient client) {
-      synchronized (this) {
-        client.decCount();
-        if (client.isZeroReference()) {
-          clients.remove(client.getClusterId());
-        }
-      }
-      if (client.isZeroReference()) {
-        client.stop();
-      }
+    // check for an already created client
+    if (this.client != null) {
+      this.client.stop();
     }
+    this.client = new SecureClient(HbaseObjectWritable.class, conf);
   }
 
-  protected final static ClientCache CLIENTS = new ClientCache();
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
 
   private static class Invoker implements InvocationHandler {
     private Class<? extends VersionedProtocol> protocol;
     private InetSocketAddress address;
     private User ticket;
     private SecureClient client;
-    private boolean isClosed = false;
     final private int rpcTimeout;
 
-    public Invoker(Class<? extends VersionedProtocol> protocol,
-        InetSocketAddress address, User ticket,
-        Configuration conf, SocketFactory factory, int rpcTimeout) {
+    public Invoker(SecureClient client,
+        Class<? extends VersionedProtocol> protocol,
+        InetSocketAddress address, User ticket, int rpcTimeout) {
       this.protocol = protocol;
       this.address = address;
       this.ticket = ticket;
-      this.client = CLIENTS.getClient(conf, factory);
+      this.client = client;
       this.rpcTimeout = rpcTimeout;
     }
 
@@ -170,14 +110,6 @@ public Object invoke(Object proxy, Method method, Object[] args)
       }
       return value.get();
     }
-
-    /* close the IPC client that's responsible for this invoker's RPCs */
-    synchronized protected void close() {
-      if (!isClosed) {
-        isClosed = true;
-        CLIENTS.stopClient(client);
-      }
-    }
   }
 
   /**
@@ -187,24 +119,30 @@ synchronized protected void close() {
    * @param protocol interface
    * @param clientVersion version we are expecting
    * @param addr remote address
-   * @param ticket ticket
    * @param conf configuration
-   * @param factory socket factory
    * @return proxy
    * @throws java.io.IOException e
    */
-  public VersionedProtocol getProxy(
-      Class<? extends VersionedProtocol> protocol, long clientVersion,
-      InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout)
+  @Override
+  public <T extends VersionedProtocol> T getProxy(
+      Class<T> protocol, long clientVersion,
+      InetSocketAddress addr,
+      Configuration conf, int rpcTimeout)
   throws IOException {
-    if (User.isSecurityEnabled()) {
-      HBaseSaslRpcServer.init(conf);
+    if (this.client == null) {
+      throw new IOException("Client must be initialized by calling setConf(Configuration)");
     }
-    VersionedProtocol proxy =
-        (VersionedProtocol) Proxy.newProxyInstance(
+
+    T proxy =
+        (T) Proxy.newProxyInstance(
             protocol.getClassLoader(), new Class[] { protocol },
-            new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+            new Invoker(this.client, protocol, addr, User.getCurrent(),
+                HBaseRPC.getRpcTimeout(rpcTimeout)));
+    /*
+     * TODO: checking protocol version only needs to be done once when we setup a new
+     * SecureClient.Connection.  Doing it every time we retrieve a proxy instance is resulting
+     * in unnecessary RPC traffic.
+     */
     long serverVersion = proxy.getProtocolVersion(protocol.getName(),
                                                   clientVersion);
     if (serverVersion != clientVersion) {
@@ -214,50 +152,48 @@ public VersionedProtocol getProxy(
     return proxy;
   }
 
-  /**
-   * Stop this proxy and release its invoker's resource
-   * @param proxy the proxy to be stopped
-   */
-  public void stopProxy(VersionedProtocol proxy) {
-    if (proxy!=null) {
-      ((Invoker)Proxy.getInvocationHandler(proxy)).close();
-    }
-  }
-
-
   /** Expert: Make multiple, parallel calls to a set of servers. */
+  @Override
   public Object[] call(Method method, Object[][] params,
                        InetSocketAddress[] addrs,
                        Class<? extends VersionedProtocol> protocol,
                        User ticket, Configuration conf)
     throws IOException, InterruptedException {
+    if (this.client == null) {
+      throw new IOException("Client must be initialized by calling setConf(Configuration)");
+    }
 
     Invocation[] invocations = new Invocation[params.length];
-    for (int i = 0; i < params.length; i++)
+    for (int i = 0; i < params.length; i++) {
       invocations[i] = new Invocation(method, protocol, params[i]);
-    SecureClient client = CLIENTS.getClient(conf);
-    try {
-      Writable[] wrappedValues =
-        client.call(invocations, addrs, protocol, ticket);
+    }
 
-      if (method.getReturnType() == Void.TYPE) {
-        return null;
-      }
+    Writable[] wrappedValues =
+      client.call(invocations, addrs, protocol, ticket);
 
-      Object[] values =
-          (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
-      for (int i = 0; i < values.length; i++)
-        if (wrappedValues[i] != null)
-          values[i] = ((HbaseObjectWritable)wrappedValues[i]).get();
+    if (method.getReturnType() == Void.TYPE) {
+      return null;
+    }
+
+    Object[] values =
+        (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
+    for (int i = 0; i < values.length; i++)
+      if (wrappedValues[i] != null)
+        values[i] = ((HbaseObjectWritable)wrappedValues[i]).get();
+
+    return values;
+  }
 
-      return values;
-    } finally {
-      CLIENTS.stopClient(client);
+  @Override
+  public void close() {
+    if (this.client != null) {
+      this.client.stop();
     }
   }
 
   /** Construct a server for a protocol implementation instance listening on a
    * port and address, with a secret manager. */
+  @Override
   public Server getServer(Class<? extends VersionedProtocol> protocol,
       final Object instance,
       Class<?>[] ifaces,
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 69434f7dd867..c78d70963734 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -169,7 +169,7 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 <%def userTables>
 <%java>
    HTableDescriptor[] tables = admin.listTables();
-   HConnectionManager.deleteConnection(admin.getConfiguration(), false);
+   HConnectionManager.deleteConnection(admin.getConfiguration());
 </%java>
 <%if (tables != null && tables.length > 0)%>
 <table>
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index d12c008ccf49..fd7a9470274a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -68,6 +68,7 @@
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.ipc.RpcEngine;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -102,7 +103,7 @@
  * <p>But sharing connections
  * makes clean up of {@link HConnection} instances a little awkward.  Currently,
  * clients cleanup by calling
- * {@link #deleteConnection(Configuration, boolean)}.  This will shutdown the
+ * {@link #deleteConnection(Configuration)}.  This will shutdown the
  * zookeeper connection the HConnection was using and clean up all
  * HConnection resources as well as stopping proxies to servers out on the
  * cluster. Not running the cleanup will not end the world; it'll
@@ -123,7 +124,7 @@
  * }
  * </pre>
  * <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd
- * register a shutdown hook that called {@link #deleteAllConnections(boolean)}
+ * register a shutdown hook that called {@link #deleteAllConnections()}
  * on its way out but the order in which shutdown hooks run is not defined so
  * were problematic for clients of HConnection that wanted to register their
  * own shutdown hooks so we removed ours though this shifts the onus for
@@ -183,7 +184,7 @@ public static HConnection getConnection(Configuration conf)
         connection = new HConnectionImplementation(conf, true);
         HBASE_INSTANCES.put(connectionKey, connection);
       } else if (connection.isClosed()) {
-        HConnectionManager.deleteConnection(connectionKey, true, true);
+        HConnectionManager.deleteConnection(connectionKey, true);
         connection = new HConnectionImplementation(conf, true);
         HBASE_INSTANCES.put(connectionKey, connection);
       }
@@ -216,13 +217,25 @@ public static HConnection createConnection(Configuration conf)
    *          configuration whose identity is used to find {@link HConnection}
    *          instance.
    * @param stopProxy
-   *          Shuts down all the proxy's put up to cluster members including to
-   *          cluster HMaster. Calls
-   *          {@link HBaseRPC#stopProxy(org.apache.hadoop.hbase.ipc.VersionedProtocol)}
-   *          .
+   *          No longer used.  This parameter is ignored.
+   * @deprecated use {@link #createConnection(org.apache.hadoop.conf.Configuration)} instead
    */
+  @Deprecated
   public static void deleteConnection(Configuration conf, boolean stopProxy) {
-    deleteConnection(new HConnectionKey(conf), stopProxy, false);
+    deleteConnection(conf);
+  }
+
+  /**
+   * Delete connection information for the instance specified by configuration.
+   * If there are no more references to it, this will then close connection to
+   * the zookeeper ensemble and let go of all resources.
+   *
+   * @param conf
+   *          configuration whose identity is used to find {@link HConnection}
+   *          instance.
+   */
+  public static void deleteConnection(Configuration conf) {
+    deleteConnection(new HConnectionKey(conf), false);
   }
 
   /**
@@ -233,32 +246,40 @@ public static void deleteConnection(Configuration conf, boolean stopProxy) {
    * @param connection
    */
   public static void deleteStaleConnection(HConnection connection) {
-    deleteConnection(connection, true, true);
+    deleteConnection(connection, true);
   }
 
   /**
    * Delete information for all connections.
-   * @param stopProxy stop the proxy as well
-   * @throws IOException
+   * @param stopProxy No longer used.  This parameter is ignored.
+   * @deprecated use {@link #deleteAllConnections()} instead
    */
+  @Deprecated
   public static void deleteAllConnections(boolean stopProxy) {
+    deleteAllConnections();
+  }
+
+  /**
+   * Delete information for all connections.
+   * @throws IOException
+   */
+  public static void deleteAllConnections() {
     synchronized (HBASE_INSTANCES) {
       Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
       connectionKeys.addAll(HBASE_INSTANCES.keySet());
       for (HConnectionKey connectionKey : connectionKeys) {
-        deleteConnection(connectionKey, stopProxy, false);
+        deleteConnection(connectionKey, false);
       }
       HBASE_INSTANCES.clear();
     }
   }
 
-  private static void deleteConnection(HConnection connection, boolean stopProxy,
-      boolean staleConnection) {
+  private static void deleteConnection(HConnection connection, boolean staleConnection) {
     synchronized (HBASE_INSTANCES) {
       for (Entry<HConnectionKey, HConnectionImplementation> connectionEntry : HBASE_INSTANCES
           .entrySet()) {
         if (connectionEntry.getValue() == connection) {
-          deleteConnection(connectionEntry.getKey(), stopProxy, staleConnection);
+          deleteConnection(connectionEntry.getKey(), staleConnection);
           break;
         }
       }
@@ -266,7 +287,7 @@ private static void deleteConnection(HConnection connection, boolean stopProxy,
   }
 
   private static void deleteConnection(HConnectionKey connectionKey,
-      boolean stopProxy, boolean staleConnection) {
+      boolean staleConnection) {
     synchronized (HBASE_INSTANCES) {
       HConnectionImplementation connection = HBASE_INSTANCES
           .get(connectionKey);
@@ -274,9 +295,7 @@ private static void deleteConnection(HConnectionKey connectionKey,
         connection.decCount();
         if (connection.isZeroReference() || staleConnection) {
           HBASE_INSTANCES.remove(connectionKey);
-          connection.close(stopProxy);
-        } else if (stopProxy) {
-          connection.stopProxyOnClose(stopProxy);
+          connection.internalClose();
         }
       }else {
         LOG.error("Connection not found in the list, can't delete it "+
@@ -514,6 +533,9 @@ static class HConnectionImplementation implements HConnection, Closeable {
     private final Object resetLock = new Object();
 
     private final Configuration conf;
+
+    private RpcEngine rpcEngine;
+
     // Known region HServerAddress.toString() -> HRegionInterface
 
     private final Map<String, HRegionInterface> servers =
@@ -541,7 +563,6 @@ static class HConnectionImplementation implements HConnection, Closeable {
     private final Set<Integer> regionCachePrefetchDisabledTables =
       new CopyOnWriteArraySet<Integer>();
 
-    private boolean stopProxy;
     private int refCount;
 
     // indicates whether this connection's life cycle is managed
@@ -579,6 +600,7 @@ public HConnectionImplementation(Configuration conf, boolean managed)
           HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
 
+      this.rpcEngine = HBaseRPC.getProtocolEngine(conf);
       this.master = null;
       this.resetting = false;
     }
@@ -683,7 +705,7 @@ public HMasterInterface getMaster()
             }
             InetSocketAddress isa =
               new InetSocketAddress(sn.getHostname(), sn.getPort());
-            HMasterInterface tryMaster = (HMasterInterface)HBaseRPC.getProxy(
+            HMasterInterface tryMaster = rpcEngine.getProxy(
                 HMasterInterface.class, HMasterInterface.VERSION, isa, this.conf,
                 this.rpcTimeout);
 
@@ -1310,7 +1332,7 @@ HRegionInterface getHRegionConnection(final String hostname, final int port,
               InetSocketAddress address = isa != null? isa:
                 new InetSocketAddress(hostname, port);
               // definitely a cache miss. establish an RPC for this RS
-              server = (HRegionInterface) HBaseRPC.waitForProxy(
+              server = HBaseRPC.waitForProxy(this.rpcEngine,
                   serverInterfaceClass, HRegionInterface.VERSION,
                   address, this.conf,
                   this.maxRPCAttempts, this.rpcTimeout, this.rpcTimeout);
@@ -1723,10 +1745,6 @@ public int getCurrentNrHRS() throws IOException {
       }
     }
 
-    public void stopProxyOnClose(boolean stopProxy) {
-      this.stopProxy = stopProxy;
-    }
-
     /**
      * Increment this client's reference count.
      */
@@ -1752,22 +1770,17 @@ boolean isZeroReference() {
       return refCount == 0;
     }
 
-    void close(boolean stopProxy) {
+    void internalClose() {
       if (this.closed) {
         return;
       }
-      if (master != null) {
-        if (stopProxy) {
-          HBaseRPC.stopProxy(master);
-        }
-        master = null;
-      }
-      if (stopProxy) {
-        for (HRegionInterface i : servers.values()) {
-          HBaseRPC.stopProxy(i);
-        }
-      }
+      master = null;
+
       this.servers.clear();
+      if (this.rpcEngine != null) {
+        this.rpcEngine.close();
+      }
+
       if (this.zooKeeper != null) {
         LOG.info("Closed zookeeper sessionid=0x" +
           Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
@@ -1782,21 +1795,21 @@ public void close() {
         if (aborted) {
           HConnectionManager.deleteStaleConnection(this);
         } else {
-          HConnectionManager.deleteConnection(this, stopProxy, false);
+          HConnectionManager.deleteConnection(this, false);
         }
       } else {
-        close(true);
+        internalClose();
       }
       if (LOG.isTraceEnabled()) LOG.debug("" + this.zooKeeper + " closed.");
     }
 
     /**
      * Close the connection for good, regardless of what the current value of
-     * {@link #refCount} is. Ideally, {@link refCount} should be zero at this
+     * {@link #refCount} is. Ideally, {@link #refCount} should be zero at this
      * point, which would be the case if all of its consumers close the
      * connection. However, on the off chance that someone is unable to close
      * the connection, perhaps because it bailed out prematurely, the method
-     * below will ensure that this {@link Connection} instance is cleaned up.
+     * below will ensure that this {@link HConnection} instance is cleaned up.
      * Caveat: The JVM may take an unknown amount of time to call finalize on an
      * unreachable object, so our hope is that every consumer cleans up after
      * itself, like any good citizen.
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 5a2100261da4..c5b76c95e375 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -90,7 +90,6 @@ public class HBaseClient {
   protected FailedServers failedServers;
 
   protected final SocketFactory socketFactory;           // how to create sockets
-  private int refCount = 1;
   protected String clusterId;
 
   final private static String PING_INTERVAL_NAME = "ipc.ping.interval";
@@ -198,31 +197,6 @@ static int getSocketTimeout(Configuration conf) {
     return conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT);
   }
 
-  /**
-   * Increment this client's reference count
-   *
-   */
-  synchronized void incCount() {
-    refCount++;
-  }
-
-  /**
-   * Decrement this client's reference count
-   *
-   */
-  synchronized void decCount() {
-    refCount--;
-  }
-
-  /**
-   * Return if this client has no reference
-   *
-   * @return true if this client has no reference; false otherwise
-   */
-  synchronized boolean isZeroReference() {
-    return refCount==0;
-  }
-
   /** A call waiting for a value. */
   protected class Call {
     final int id;                                       // call id
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
index 1c45c0a2fea1..27884f9d0fd9 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -83,14 +83,6 @@ private HBaseRPC() {
    */
   public static final String RPC_ENGINE_PROP = "hbase.rpc.engine";
 
-  // cache of RpcEngines by protocol
-  private static final Map<Class,RpcEngine> PROTOCOL_ENGINES
-    = new HashMap<Class,RpcEngine>();
-
-  // track what RpcEngine is used by a proxy class, for stopProxy()
-  private static final Map<Class,RpcEngine> PROXY_ENGINES
-    = new HashMap<Class,RpcEngine>();
-
   // thread-specific RPC timeout, which may override that of RpcEngine
   private static ThreadLocal<Integer> rpcTimeout = new ThreadLocal<Integer>() {
     @Override
@@ -99,40 +91,19 @@ protected Integer initialValue() {
       }
     };
 
-  // set a protocol to use a non-default RpcEngine
-  static void setProtocolEngine(Configuration conf,
-                                Class protocol, Class engine) {
-    conf.setClass(RPC_ENGINE_PROP+"."+protocol.getName(), engine, RpcEngine.class);
-  }
-
-  // return the RpcEngine configured to handle a protocol
-  private static synchronized RpcEngine getProtocolEngine(Class protocol,
-                                                          Configuration conf) {
-    RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
-    if (engine == null) {
-      // check for a configured default engine
-      Class<?> defaultEngine =
-          conf.getClass(RPC_ENGINE_PROP, WritableRpcEngine.class);
+  /**
+   * Returns a new instance of the configured {@link RpcEngine} implementation.
+   */
+  public static synchronized RpcEngine getProtocolEngine(Configuration conf) {
+    // check for a configured default engine
+    Class<?> impl =
+        conf.getClass(RPC_ENGINE_PROP, WritableRpcEngine.class);
 
-      // check for a per interface override
-      Class<?> impl = conf.getClass(RPC_ENGINE_PROP+"."+protocol.getName(),
-                                    defaultEngine);
-      LOG.debug("Using "+impl.getName()+" for "+protocol.getName());
-      engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
-      if (protocol.isInterface())
-        PROXY_ENGINES.put(Proxy.getProxyClass(protocol.getClassLoader(),
-                                              protocol),
-                          engine);
-      PROTOCOL_ENGINES.put(protocol, engine);
-    }
+    LOG.debug("Using RpcEngine: "+impl.getName());
+    RpcEngine engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
     return engine;
   }
 
-  // return the RpcEngine that handles a proxy object
-  private static synchronized RpcEngine getProxyEngine(Object proxy) {
-    return PROXY_ENGINES.get(proxy.getClass());
-  }
-
   /**
    * A version mismatch for the RPC protocol.
    */
@@ -219,21 +190,22 @@ public Class getProtocol() {
    * @throws IOException e
    */
   @SuppressWarnings("unchecked")
-  public static VersionedProtocol waitForProxy(Class protocol,
+  public static <T extends VersionedProtocol> T waitForProxy(RpcEngine rpcClient,
+                                               Class<T> protocol,
                                                long clientVersion,
                                                InetSocketAddress addr,
                                                Configuration conf,
                                                int maxAttempts,
                                                int rpcTimeout,
                                                long timeout
-                                               ) throws IOException {
+  ) throws IOException {
     // HBase does limited number of reconnects which is different from hadoop.
     long startTime = System.currentTimeMillis();
     IOException ioe;
     int reconnectAttempts = 0;
     while (true) {
       try {
-        return getProxy(protocol, clientVersion, addr, conf, rpcTimeout);
+        return rpcClient.getProxy(protocol, clientVersion, addr, conf, rpcTimeout);
       } catch(SocketTimeoutException te) {  // namenode is busy
         LOG.info("Problem connecting to server: " + addr);
         ioe = te;
@@ -293,88 +265,6 @@ private static void handleConnectionException(int retries, int maxAttmpts,
     }
   }
   
-  /**
-   * Construct a client-side proxy object that implements the named protocol,
-   * talking to a server at the named address.
-   *
-   * @param protocol interface
-   * @param clientVersion version we are expecting
-   * @param addr remote address
-   * @param conf configuration
-   * @param factory socket factory
-   * @param rpcTimeout timeout for each RPC
-   * @return proxy
-   * @throws IOException e
-   */
-  public static VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
-      long clientVersion, InetSocketAddress addr, Configuration conf,
-      SocketFactory factory, int rpcTimeout) throws IOException {
-    return getProxy(protocol, clientVersion, addr,
-        User.getCurrent(), conf, factory, rpcTimeout);
-  }
-
-  /**
-   * Construct a client-side proxy object that implements the named protocol,
-   * talking to a server at the named address.
-   *
-   * @param protocol interface
-   * @param clientVersion version we are expecting
-   * @param addr remote address
-   * @param ticket ticket
-   * @param conf configuration
-   * @param factory socket factory
-   * @param rpcTimeout timeout for each RPC
-   * @return proxy
-   * @throws IOException e
-   */
-  public static VersionedProtocol getProxy(
-      Class<? extends VersionedProtocol> protocol,
-      long clientVersion, InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout)
-  throws IOException {
-    VersionedProtocol proxy =
-        getProtocolEngine(protocol,conf)
-            .getProxy(protocol, clientVersion, addr, ticket, conf, factory, Math.min(rpcTimeout, HBaseRPC.getRpcTimeout()));
-    long serverVersion = proxy.getProtocolVersion(protocol.getName(),
-                                                  clientVersion);
-    if (serverVersion == clientVersion) {
-      return proxy;
-    }
-    throw new VersionMismatch(protocol.getName(), clientVersion,
-                              serverVersion);
-  }
-
-  /**
-   * Construct a client-side proxy object with the default SocketFactory
-   *
-   * @param protocol interface
-   * @param clientVersion version we are expecting
-   * @param addr remote address
-   * @param conf configuration
-   * @param rpcTimeout timeout for each RPC
-   * @return a proxy instance
-   * @throws IOException e
-   */
-  public static VersionedProtocol getProxy(
-      Class<? extends VersionedProtocol> protocol,
-      long clientVersion, InetSocketAddress addr, Configuration conf,
-      int rpcTimeout)
-      throws IOException {
-
-    return getProxy(protocol, clientVersion, addr, conf, NetUtils
-        .getDefaultSocketFactory(conf), rpcTimeout);
-  }
-
-  /**
-   * Stop this proxy and release its invoker's resource
-   * @param proxy the proxy to be stopped
-   */
-  public static void stopProxy(VersionedProtocol proxy) {
-    if (proxy!=null) {
-      getProxyEngine(proxy).stopProxy(proxy);
-    }
-  }
-
   /**
    * Expert: Make multiple, parallel calls to a set of servers.
    *
@@ -385,7 +275,7 @@ public static void stopProxy(VersionedProtocol proxy) {
    * @return values
    * @throws IOException e
    * @deprecated Instead of calling statically, use
-   *     {@link HBaseRPC#getProtocolEngine(Class, org.apache.hadoop.conf.Configuration)}
+   *     {@link HBaseRPC#getProtocolEngine(org.apache.hadoop.conf.Configuration)}
    *     to obtain an {@link RpcEngine} instance and then use
    *     {@link RpcEngine#call(java.lang.reflect.Method, Object[][], java.net.InetSocketAddress[], Class, org.apache.hadoop.hbase.security.User, org.apache.hadoop.conf.Configuration)}
    */
@@ -396,8 +286,15 @@ public static Object[] call(Method method, Object[][] params,
       User ticket,
       Configuration conf)
     throws IOException, InterruptedException {
-    return getProtocolEngine(protocol, conf)
-      .call(method, params, addrs, protocol, ticket, conf);
+    Object[] result = null;
+    RpcEngine engine = null;
+    try {
+      engine = getProtocolEngine(conf);
+      result = engine.call(method, params, addrs, protocol, ticket, conf);
+    } finally {
+      engine.close();
+    }
+    return result;
   }
 
   /**
@@ -430,7 +327,7 @@ public static RpcServer getServer(Class protocol,
                                  final int numHandlers,
                                  int metaHandlerCount, final boolean verbose, Configuration conf, int highPriorityLevel)
     throws IOException {
-    return getProtocolEngine(protocol, conf)
+    return getProtocolEngine(conf)
         .getServer(protocol, instance, ifaces, bindAddress, port, numHandlers, metaHandlerCount, verbose, conf, highPriorityLevel);
   }
 
@@ -445,4 +342,12 @@ public static int getRpcTimeout() {
   public static void resetRpcTimeout() {
     HBaseRPC.rpcTimeout.remove();
   }
+
+  /**
+   * Returns the lower of the thread-local RPC time from {@link #setRpcTimeout(int)} and the given
+   * default timeout.
+   */
+  public static int getRpcTimeout(int defaultTimeout) {
+    return Math.min(defaultTimeout, HBaseRPC.rpcTimeout.get());
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
index d48aeaef2c4d..faf725c43ed2 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
@@ -22,23 +22,24 @@
 import java.lang.reflect.Method;
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import javax.net.SocketFactory;
 
-import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.conf.Configuration;
 
 /** An RPC implementation. */
-interface RpcEngine {
+@InterfaceAudience.Private
+public interface RpcEngine extends Configurable {
 
+  /* Client-related methods */
   /** Construct a client-side proxy object. */
-  VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
-                  long clientVersion, InetSocketAddress addr,
-                  User ticket, Configuration conf,
-                  SocketFactory factory, int rpcTimeout) throws IOException;
+  <T extends VersionedProtocol> T getProxy(Class<T> protocol,
+                                           long clientVersion, InetSocketAddress addr,
+                                           Configuration conf, int rpcTimeout) throws IOException;
 
-  /** Stop this proxy. */
-  void stopProxy(VersionedProtocol proxy);
+  /** Shutdown this instance */
+  void close();
 
   /** Expert: Make multiple, parallel calls to a set of servers. */
   Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs,
@@ -46,6 +47,7 @@ Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs,
                 User ticket, Configuration conf)
     throws IOException, InterruptedException;
 
+  /* Server-related methods */
   /** Construct a server for a protocol implementation instance. */
   RpcServer getServer(Class<? extends VersionedProtocol> protocol, Object instance,
                        Class<?>[] ifaces, String bindAddress,
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index 386111745042..a93fa4997d8d 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -57,85 +57,21 @@ class WritableRpcEngine implements RpcEngine {
   // DEBUG log level does NOT emit RPC-level logging. 
   private static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.RPCEngine");
 
-  /* Cache a client using its socket factory as the hash key */
-  static private class ClientCache {
-    private Map<SocketFactory, HBaseClient> clients =
-      new HashMap<SocketFactory, HBaseClient>();
-
-    protected ClientCache() {}
-
-    /**
-     * Construct & cache an IPC client with the user-provided SocketFactory
-     * if no cached client exists.
-     *
-     * @param conf Configuration
-     * @param factory socket factory
-     * @return an IPC client
-     */
-    protected synchronized HBaseClient getClient(Configuration conf,
-        SocketFactory factory) {
-      // Construct & cache client.  The configuration is only used for timeout,
-      // and Clients have connection pools.  So we can either (a) lose some
-      // connection pooling and leak sockets, or (b) use the same timeout for
-      // all configurations.  Since the IPC is usually intended globally, not
-      // per-job, we choose (a).
-      HBaseClient client = clients.get(factory);
-      if (client == null) {
-        // Make an hbase client instead of hadoop Client.
-        client = new HBaseClient(HbaseObjectWritable.class, conf, factory);
-        clients.put(factory, client);
-      } else {
-        client.incCount();
-      }
-      return client;
-    }
-
-    /**
-     * Construct & cache an IPC client with the default SocketFactory
-     * if no cached client exists.
-     *
-     * @param conf Configuration
-     * @return an IPC client
-     */
-    protected synchronized HBaseClient getClient(Configuration conf) {
-      return getClient(conf, SocketFactory.getDefault());
-    }
-
-    /**
-     * Stop a RPC client connection
-     * A RPC client is closed only when its reference count becomes zero.
-     * @param client client to stop
-     */
-    protected void stopClient(HBaseClient client) {
-      synchronized (this) {
-        client.decCount();
-        if (client.isZeroReference()) {
-          clients.remove(client.getSocketFactory());
-        }
-      }
-      if (client.isZeroReference()) {
-        client.stop();
-      }
-    }
-  }
-
-  protected final static ClientCache CLIENTS = new ClientCache();
-
   private static class Invoker implements InvocationHandler {
     private Class<? extends VersionedProtocol> protocol;
     private InetSocketAddress address;
     private User ticket;
     private HBaseClient client;
-    private boolean isClosed = false;
     final private int rpcTimeout;
 
-    public Invoker(Class<? extends VersionedProtocol> protocol,
+    public Invoker(HBaseClient client,
+                   Class<? extends VersionedProtocol> protocol,
                    InetSocketAddress address, User ticket,
-                   Configuration conf, SocketFactory factory, int rpcTimeout) {
+                   Configuration conf, int rpcTimeout) {
       this.protocol = protocol;
       this.address = address;
       this.ticket = ticket;
-      this.client = CLIENTS.getClient(conf, factory);
+      this.client = client;
       this.rpcTimeout = rpcTimeout;
     }
 
@@ -157,78 +93,98 @@ public Object invoke(Object proxy, Method method, Object[] args)
       }
       return value.get();
     }
+  }
 
-    /* close the IPC client that's responsible for this invoker's RPCs */
-    synchronized protected void close() {
-      if (!isClosed) {
-        isClosed = true;
-        CLIENTS.stopClient(client);
-      }
+  private Configuration conf;
+  private HBaseClient client;
+
+  @Override
+  public void setConf(Configuration config) {
+    this.conf = config;
+    // check for an already created client
+    if (this.client != null) {
+      this.client.stop();
     }
+    this.client = new HBaseClient(HbaseObjectWritable.class, conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
   }
 
   /** Construct a client-side proxy object that implements the named protocol,
    * talking to a server at the named address. */
-  public VersionedProtocol getProxy(
-      Class<? extends VersionedProtocol> protocol, long clientVersion,
-      InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout)
+  @Override
+  public <T extends VersionedProtocol> T getProxy(
+      Class<T> protocol, long clientVersion,
+      InetSocketAddress addr, Configuration conf, int rpcTimeout)
     throws IOException {
+    if (this.client == null) {
+      throw new IOException("Client must be initialized by calling setConf(Configuration)");
+    }
 
-      VersionedProtocol proxy =
-          (VersionedProtocol) Proxy.newProxyInstance(
+    T proxy =
+          (T) Proxy.newProxyInstance(
               protocol.getClassLoader(), new Class[] { protocol },
-              new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
-    if (proxy instanceof VersionedProtocol) {
-      long serverVersion = ((VersionedProtocol)proxy)
-        .getProtocolVersion(protocol.getName(), clientVersion);
-      if (serverVersion != clientVersion) {
-        throw new HBaseRPC.VersionMismatch(protocol.getName(), clientVersion,
-                                      serverVersion);
-      }
+              new Invoker(client, protocol, addr, User.getCurrent(), conf,
+                  HBaseRPC.getRpcTimeout(rpcTimeout)));
+
+    /*
+     * TODO: checking protocol version only needs to be done once when we setup a new
+     * HBaseClient.Connection.  Doing it every time we retrieve a proxy instance is resulting
+     * in unnecessary RPC traffic.
+     */
+    long serverVersion = ((VersionedProtocol)proxy)
+      .getProtocolVersion(protocol.getName(), clientVersion);
+    if (serverVersion != clientVersion) {
+      throw new HBaseRPC.VersionMismatch(protocol.getName(), clientVersion,
+                                    serverVersion);
     }
+
     return proxy;
   }
 
-  /**
-   * Stop this proxy and release its invoker's resource
-   * @param proxy the proxy to be stopped
-   */
-  public void stopProxy(VersionedProtocol proxy) {
-    if (proxy!=null) {
-      ((Invoker)Proxy.getInvocationHandler(proxy)).close();
-    }
-  }
 
 
   /** Expert: Make multiple, parallel calls to a set of servers. */
+  @Override
   public Object[] call(Method method, Object[][] params,
                        InetSocketAddress[] addrs,
                        Class<? extends VersionedProtocol> protocol,
                        User ticket, Configuration conf)
     throws IOException, InterruptedException {
+    if (this.client == null) {
+      throw new IOException("Client must be initialized by calling setConf(Configuration)");
+    }
 
     Invocation[] invocations = new Invocation[params.length];
-    for (int i = 0; i < params.length; i++)
+    for (int i = 0; i < params.length; i++) {
       invocations[i] = new Invocation(method, protocol, params[i]);
-    HBaseClient client = CLIENTS.getClient(conf);
-    try {
+    }
+
     Writable[] wrappedValues =
-      client.call(invocations, addrs, protocol, ticket);
+        client.call(invocations, addrs, protocol, ticket);
 
     if (method.getReturnType() == Void.TYPE) {
       return null;
     }
 
     Object[] values =
-      (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
-    for (int i = 0; i < values.length; i++)
-      if (wrappedValues[i] != null)
+        (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
+    for (int i = 0; i < values.length; i++) {
+      if (wrappedValues[i] != null) {
         values[i] = ((HbaseObjectWritable)wrappedValues[i]).get();
+      }
+    }
 
     return values;
-    } finally {
-      CLIENTS.stopClient(client);
+  }
+
+  @Override
+  public void close() {
+    if (this.client != null) {
+      this.client.stop();
     }
   }
 
@@ -428,7 +384,7 @@ public Writable call(Class<? extends VersionedProtocol> protocol,
      * client Operations.
      * @param call The call to log.
      * @param tag  The tag that will be used to indicate this event in the log.
-     * @param client          The address of the client who made this call.
+     * @param clientAddress   The address of the client who made this call.
      * @param startTime       The time that the call was initiated, in ms.
      * @param processingTime  The duration that the call took to run, in ms.
      * @param qTime           The duration that the call spent on the queue 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7c4041d3498f..86d1337e5eff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -123,6 +123,7 @@
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.ipc.Invocation;
 import org.apache.hadoop.hbase.ipc.ProtocolSignature;
+import org.apache.hadoop.hbase.ipc.RpcEngine;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
@@ -234,6 +235,9 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   // Remote HMaster
   private HMasterRegionInterface hbaseMaster;
 
+  // RPC Engine for master connection
+  private RpcEngine rpcEngine;
+
   // Server to handle client requests. Default access so can be accessed by
   // unit tests.
   RpcServer rpcServer;
@@ -587,6 +591,8 @@ private void preRegistrationInitialization(){
       for (int i = 0; i < nbBlocks; i++) {
         reservedSpace.add(new byte[HConstants.DEFAULT_SIZE_RESERVATION_BLOCK]);
       }
+
+      this.rpcEngine = HBaseRPC.getProtocolEngine(conf);
     } catch (Throwable t) {
       // Call stop if error or process will stick around for ever since server
       // puts up non-daemon threads.
@@ -828,10 +834,8 @@ public void run() {
     }
 
     // Make sure the proxy is down.
-    if (this.hbaseMaster != null) {
-      HBaseRPC.stopProxy(this.hbaseMaster);
-      this.hbaseMaster = null;
-    }
+    this.hbaseMaster = null;
+    this.rpcEngine.close();
     this.leases.close();
 
     if (!killed) {
@@ -1878,7 +1882,7 @@ private ServerName getMaster() {
       try {
         // Do initial RPC setup. The final argument indicates that the RPC
         // should retry indefinitely.
-        master = (HMasterRegionInterface) HBaseRPC.waitForProxy(
+        master = HBaseRPC.waitForProxy(this.rpcEngine,
             HMasterRegionInterface.class, HMasterRegionInterface.VERSION,
             masterIsa, this.conf, -1,
             this.rpcTimeout, this.rpcTimeout);
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 21a58db58f4d..ee8941bed885 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -151,7 +151,7 @@ public void stop(String why) {
       this.zkHelper.getZookeeperWatcher().close();
     }
     // Not sure why we're deleting a connection that we never acquired or used
-    HConnectionManager.deleteConnection(this.getConf(), true);
+    HConnectionManager.deleteConnection(this.getConf());
   }
 
   @Override
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 3dd7fdca4741..69a2e34bc402 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -235,7 +235,7 @@
 }
 } // end else
 
-HConnectionManager.deleteConnection(hbadmin.getConfiguration(), false);
+HConnectionManager.deleteConnection(hbadmin.getConfiguration());
 %>
 
 
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index f110b589dc54..288bd6ccbd41 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -491,7 +491,7 @@ public void shutdown() throws IOException {
     if (this.hbaseCluster != null) {
       this.hbaseCluster.shutdown();
     }
-    HConnectionManager.deleteAllConnections(false);
+    HConnectionManager.deleteAllConnections();
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
index c22123d44092..5071a89237f3 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
@@ -163,7 +163,7 @@ public void run() {
       // Join the thread... should exit shortly.
       t.join();
     } finally {
-      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+      HConnectionManager.deleteConnection(UTIL.getConfiguration());
     }
   }
 
@@ -237,7 +237,7 @@ public void run() {
       }
     } finally {
       // Clear out our doctored connection or could mess up subsequent tests.
-      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+      HConnectionManager.deleteConnection(UTIL.getConfiguration());
     }
   }
 
@@ -264,7 +264,7 @@ private void testVerifyMetaRegionLocationWithException(Exception ex)
       }
     } finally {
       // Clear out our doctored connection or could mess up subsequent tests.
-      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+      HConnectionManager.deleteConnection(UTIL.getConfiguration());
     }
   }
 
@@ -347,7 +347,7 @@ public void testTimeoutWaitForMeta()
       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
       ct.waitForMeta(100);
     } finally {
-      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+      HConnectionManager.deleteConnection(UTIL.getConfiguration());
     }
   }
 
@@ -437,20 +437,20 @@ void doWaiting() throws InterruptedException {
       // Now meta is available.
       Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
     } finally {
-      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+      HConnectionManager.deleteConnection(UTIL.getConfiguration());
     }
   }
 
   /**
    * @param implementation An {@link HRegionInterface} instance; you'll likely
    * want to pass a mocked HRS; can be null.
-   * @return Mock up a connection that returns a {@link Configuration} when
+   * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
    * {@link HConnection#getConfiguration()} is called, a 'location' when
    * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
    * and that returns the passed {@link HRegionInterface} instance when
    * {@link HConnection#getHRegionConnection(String, int)}
    * is called (Be sure call
-   * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration, boolean)}
+   * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
    * when done with this mocked Connection.
    * @throws IOException
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
index 2003c6278c1c..222b847b951c 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
@@ -166,7 +166,7 @@ public void testRideOverServerNotRunning() throws IOException, InterruptedExcept
         openScanner((byte [])Mockito.any(), (Scan)Mockito.any());
     } finally {
       if (ct != null) ct.stop();
-      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+      HConnectionManager.deleteConnection(UTIL.getConfiguration());
       zkw.close();
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 66548a1d7f92..d95ddcd35021 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -193,7 +193,7 @@ public void testConnectionManagement() throws Exception{
   }
 
   /**
-   * Make sure that {@link HConfiguration} instances that are essentially the
+   * Make sure that {@link Configuration} instances that are essentially the
    * same map to the same {@link HConnection} instance.
    */
   @Test
@@ -267,7 +267,7 @@ public void testConnectionUniqueness() throws Exception {
     } finally {
       for (HConnection c: connections) {
         // Clean up connections made so we don't interfere w/ subsequent tests.
-        HConnectionManager.deleteConnection(c.getConfiguration(), true);
+        HConnectionManager.deleteConnection(c.getConfiguration());
       }
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
index 66bd1955c762..ee390cd14e54 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
@@ -70,30 +70,38 @@ private void testDelayedRpc(boolean delayReturnValue) throws Exception {
     rpcServer = HBaseRPC.getServer(new TestRpcImpl(delayReturnValue),
         new Class<?>[]{ TestRpcImpl.class },
         isa.getHostName(), isa.getPort(), 1, 0, true, conf, 0);
-    rpcServer.start();
+    RpcEngine rpcEngine = null;
+    try {
+      rpcServer.start();
+      rpcEngine = HBaseRPC.getProtocolEngine(conf);
 
-    TestRpc client = (TestRpc) HBaseRPC.getProxy(TestRpc.class, 0,
+      TestRpc client = rpcEngine.getProxy(TestRpc.class, 0,
         rpcServer.getListenerAddress(), conf, 1000);
 
-    List<Integer> results = new ArrayList<Integer>();
-
-    TestThread th1 = new TestThread(client, true, results);
-    TestThread th2 = new TestThread(client, false, results);
-    TestThread th3 = new TestThread(client, false, results);
-    th1.start();
-    Thread.sleep(100);
-    th2.start();
-    Thread.sleep(200);
-    th3.start();
-
-    th1.join();
-    th2.join();
-    th3.join();
-
-    assertEquals(UNDELAYED, results.get(0).intValue());
-    assertEquals(UNDELAYED, results.get(1).intValue());
-    assertEquals(results.get(2).intValue(), delayReturnValue ? DELAYED :
-        0xDEADBEEF);
+      List<Integer> results = new ArrayList<Integer>();
+
+      TestThread th1 = new TestThread(client, true, results);
+      TestThread th2 = new TestThread(client, false, results);
+      TestThread th3 = new TestThread(client, false, results);
+      th1.start();
+      Thread.sleep(100);
+      th2.start();
+      Thread.sleep(200);
+      th3.start();
+
+      th1.join();
+      th2.join();
+      th3.join();
+
+      assertEquals(UNDELAYED, results.get(0).intValue());
+      assertEquals(UNDELAYED, results.get(1).intValue());
+      assertEquals(results.get(2).intValue(), delayReturnValue ? DELAYED :
+          0xDEADBEEF);
+    } finally {
+      if (rpcEngine != null) {
+        rpcEngine.close();
+      }
+    }
   }
 
   private static class ListAppender extends AppenderSkeleton {
@@ -133,33 +141,42 @@ public void testTooManyDelayedRpcs() throws Exception {
     rpcServer = HBaseRPC.getServer(new TestRpcImpl(true),
         new Class<?>[]{ TestRpcImpl.class },
         isa.getHostName(), isa.getPort(), 1, 0, true, conf, 0);
-    rpcServer.start();
-    TestRpc client = (TestRpc) HBaseRPC.getProxy(TestRpc.class, 0,
-        rpcServer.getListenerAddress(), conf, 1000);
+    RpcEngine rpcEngine = null;
+    try {
+      rpcServer.start();
+      rpcEngine = HBaseRPC.getProtocolEngine(conf);
 
-    Thread threads[] = new Thread[MAX_DELAYED_RPC + 1];
+      TestRpc client = rpcEngine.getProxy(TestRpc.class, 0,
+          rpcServer.getListenerAddress(), conf, 1000);
 
-    for (int i = 0; i < MAX_DELAYED_RPC; i++) {
-      threads[i] = new TestThread(client, true, null);
-      threads[i].start();
-    }
+      Thread threads[] = new Thread[MAX_DELAYED_RPC + 1];
 
-    /* No warnings till here. */
-    assertTrue(listAppender.getMessages().isEmpty());
+      for (int i = 0; i < MAX_DELAYED_RPC; i++) {
+        threads[i] = new TestThread(client, true, null);
+        threads[i].start();
+      }
 
-    /* This should give a warning. */
-    threads[MAX_DELAYED_RPC] = new TestThread(client, true, null);
-    threads[MAX_DELAYED_RPC].start();
+      /* No warnings till here. */
+      assertTrue(listAppender.getMessages().isEmpty());
 
-    for (int i = 0; i < MAX_DELAYED_RPC; i++) {
-      threads[i].join();
-    }
+      /* This should give a warning. */
+      threads[MAX_DELAYED_RPC] = new TestThread(client, true, null);
+      threads[MAX_DELAYED_RPC].start();
+
+      for (int i = 0; i < MAX_DELAYED_RPC; i++) {
+        threads[i].join();
+      }
 
-    assertFalse(listAppender.getMessages().isEmpty());
-    assertTrue(listAppender.getMessages().get(0).startsWith(
-        "Too many delayed calls"));
+      assertFalse(listAppender.getMessages().isEmpty());
+      assertTrue(listAppender.getMessages().get(0).startsWith(
+          "Too many delayed calls"));
 
-    log.removeAppender(listAppender);
+      log.removeAppender(listAppender);
+    } finally {
+      if (rpcEngine != null) {
+        rpcEngine.close();
+      }
+    }
   }
 
   public interface TestRpc extends VersionedProtocol {
@@ -177,7 +194,6 @@ private static class TestRpcImpl implements TestRpc {
     /**
      * @param delayReturnValue Should the response to the delayed call be set
      * at the start or the end of the delay.
-     * @param delay Amount of milliseconds to delay the call by
      */
     public TestRpcImpl(boolean delayReturnValue) {
       this.delayReturnValue = delayReturnValue;
@@ -256,30 +272,38 @@ public void testEndDelayThrowing() throws IOException {
     rpcServer = HBaseRPC.getServer(new FaultyTestRpc(),
         new Class<?>[]{ TestRpcImpl.class },
         isa.getHostName(), isa.getPort(), 1, 0, true, conf, 0);
-    rpcServer.start();
+    RpcEngine rpcEngine = null;
+    try {
+      rpcServer.start();
+      rpcEngine = HBaseRPC.getProtocolEngine(conf);
 
-    TestRpc client = (TestRpc) HBaseRPC.getProxy(TestRpc.class, 0,
-        rpcServer.getListenerAddress(), conf, 1000);
+      TestRpc client = rpcEngine.getProxy(TestRpc.class, 0,
+          rpcServer.getListenerAddress(), conf, 1000);
 
-    int result = 0xDEADBEEF;
+      int result = 0xDEADBEEF;
 
-    try {
-      result = client.test(false);
-    } catch (Exception e) {
-      fail("No exception should have been thrown.");
-    }
-    assertEquals(result, UNDELAYED);
+      try {
+        result = client.test(false);
+      } catch (Exception e) {
+        fail("No exception should have been thrown.");
+      }
+      assertEquals(result, UNDELAYED);
 
-    boolean caughtException = false;
-    try {
-      result = client.test(true);
-    } catch(Exception e) {
-      // Exception thrown by server is enclosed in a RemoteException.
-      if (e.getCause().getMessage().startsWith(
-          "java.lang.Exception: Something went wrong"))
-        caughtException = true;
+      boolean caughtException = false;
+      try {
+        result = client.test(true);
+      } catch(Exception e) {
+        // Exception thrown by server is enclosed in a RemoteException.
+        if (e.getCause().getMessage().startsWith(
+            "java.lang.Exception: Something went wrong"))
+          caughtException = true;
+      }
+      assertTrue(caughtException);
+    } finally {
+      if (rpcEngine != null) {
+        rpcEngine.close();
+      }
     }
-    assertTrue(caughtException);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
index aa6eb9ad447b..797a26d948ad 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
@@ -83,18 +83,18 @@ private void testCallsInternal(Configuration conf) throws Exception {
     RpcServer rpcServer = HBaseRPC.getServer(new TestImpl(),
       new Class<?>[] {TestProtocol.class},
         "localhost", // BindAddress is IP we got for this server.
-        9999, // port number
+        0, // port number
         2, // number of handlers
         0, // we dont use high priority handlers in master
         conf.getBoolean("hbase.rpc.verbose", false), conf,
         0);
-    TestProtocol proxy = null;
+    RpcEngine rpcEngine = null;
     try {
       rpcServer.start();
+      rpcEngine = HBaseRPC.getProtocolEngine(conf);
 
-      InetSocketAddress isa =
-        new InetSocketAddress("localhost", 9999);
-      proxy = (TestProtocol) HBaseRPC.waitForProxy(
+      InetSocketAddress isa = rpcServer.getListenerAddress();
+      TestProtocol proxy = HBaseRPC.waitForProxy(rpcEngine,
         TestProtocol.class, TestProtocol.VERSION,
         isa, conf, -1, 8000, 8000);
 
@@ -118,8 +118,8 @@ private void testCallsInternal(Configuration conf) throws Exception {
       assertNotSame(sendProto, retProto);
     } finally {
       rpcServer.stop();
-      if(proxy != null) {
-        HBaseRPC.stopProxy(proxy);
+      if (rpcEngine != null) {
+        rpcEngine.close();
       }
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
index fdc642513442..a8a0c6cc3877 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java
@@ -80,12 +80,13 @@ public void testCalls() throws Exception {
                                   6016,
                                   10, 10, false,
                                   conf, 10);
-    TestProtocol proxy = null;
+    RpcEngine rpcEngine = null;
     try {
       server.start();
+      rpcEngine = HBaseRPC.getProtocolEngine(conf);
 
       InetSocketAddress addr = server.getListenerAddress();
-      proxy = (TestProtocol)HBaseRPC.getProxy(
+      TestProtocol proxy = rpcEngine.getProxy(
           TestProtocol.class, TestProtocol.VERSION, addr, conf, 10000);
 
       proxy.ping();
@@ -93,7 +94,9 @@ public void testCalls() throws Exception {
       proxy.logClassName();
     } finally {
       server.stop();
-      if(proxy!=null) HBaseRPC.stopProxy(proxy);
+      if (rpcEngine != null) {
+        rpcEngine.close();
+      }
     }
   }
   
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 7a51f6c354d1..6b17c762864b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -145,7 +145,7 @@ public void stop(String why) {
         this.ct.stop();
       }
       if (this.connection != null) {
-        HConnectionManager.deleteConnection(this.connection.getConfiguration(), true);
+        HConnectionManager.deleteConnection(this.connection.getConfiguration());
       }
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
index c27cbf670fc6..8c4de71bf114 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
@@ -20,6 +20,7 @@
 
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -29,6 +30,7 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.RpcEngine;
 import org.apache.hadoop.ipc.RemoteException;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -47,9 +49,11 @@ public void testRPCException() throws Exception {
 
     ServerName sm = hm.getServerName();
     InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort());
+    RpcEngine rpcEngine = null;
     try {
-      HMasterInterface inf = (HMasterInterface) HBaseRPC.getProxy(
-        HMasterInterface.class,  HMasterInterface.VERSION, isa, conf, 100 * 10);
+      rpcEngine = HBaseRPC.getProtocolEngine(conf);
+      HMasterInterface inf = rpcEngine.getProxy(
+          HMasterInterface.class,  HMasterInterface.VERSION, isa, conf, 100 * 10);
       inf.isMasterRunning();
       fail();
     } catch (RemoteException ex) {
@@ -57,6 +61,10 @@ public void testRPCException() throws Exception {
           "org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet"));
     } catch (Throwable t) {
       fail("Unexpected throwable: " + t);
+    } finally {
+      if (rpcEngine != null) {
+        rpcEngine.close();
+      }
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
index 201d38cbf6df..078f1f68aa5c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
@@ -102,7 +102,7 @@ public void setUpBefore() throws Exception {
   @After
   public void tearDownAfter() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
-    HConnectionManager.deleteConnection(conf, true);
+    HConnectionManager.deleteConnection(conf);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
index bb8b0e7b3ae8..4bdc6454f58e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
@@ -57,7 +57,7 @@ public void testMetaRebuild() throws Exception {
     // shutdown the minicluster
     TEST_UTIL.shutdownMiniHBaseCluster();
     TEST_UTIL.shutdownMiniZKCluster();
-    HConnectionManager.deleteConnection(conf, false);
+    HConnectionManager.deleteConnection(conf);
 
     // rebuild meta table from scratch
     HBaseFsck fsck = new HBaseFsck(conf);

From c50ebbd0cd47d9e2af6773ed340094749cf65526 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 29 Jan 2013 06:23:23 +0000
Subject: [PATCH 0739/1540] HBASE-7694 Secure HBase should use repliation call
 queue (Jieshan Bean)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1439754 13f79535-47bb-0310-9956-ffa450edef68
---
 .../main/java/org/apache/hadoop/hbase/ipc/SecureServer.java | 6 ++++++
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java  | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index d1b89610e009..3448d997f790 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -21,6 +21,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.io.WritableWithSize;
 import org.apache.hadoop.hbase.security.HBaseSaslRpcServer;
@@ -620,8 +621,13 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
 
       if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
         priorityCallQueue.put(call);
+        updateCallQueueLenMetrics(priorityCallQueue);
+      } else if (replicationQueue != null && getQosLevel(param) == HConstants.REPLICATION_QOS) {
+        replicationQueue.put(call);
+        updateCallQueueLenMetrics(replicationQueue);
       } else {
         callQueue.put(call);              // queue the call; maybe blocked here
+        updateCallQueueLenMetrics(callQueue);
       }
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 1b1bb7737c32..8813a009944e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -235,7 +235,7 @@ public static String getRemoteAddress() {
   private Handler[] handlers = null;
   private Handler[] priorityHandlers = null;
   /** replication related queue; */
-  private BlockingQueue<Call> replicationQueue;
+  protected BlockingQueue<Call> replicationQueue;
   private int numOfReplicationHandlers = 0;
   private Handler[] replicationHandlers = null;
   protected HBaseRPCErrorHandler errorHandler = null;
@@ -1355,7 +1355,7 @@ protected synchronized void close() {
    * Reports length of the call queue to HBaseRpcMetrics.
    * @param queue Which queue to report
    */
-  private void updateCallQueueLenMetrics(BlockingQueue<Call> queue) {
+  protected void updateCallQueueLenMetrics(BlockingQueue<Call> queue) {
     if (queue == callQueue) {
       rpcMetrics.callQueueLen.set(callQueue.size());
     } else if (queue == priorityCallQueue) {

From 6dc291513d61c190c9ca62af33b13c151e8f06d2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 29 Jan 2013 18:48:58 +0000
Subject: [PATCH 0740/1540] HBASE-2611 Handle RS that fails while processing
 the failure of another one (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440054 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/ReplicationZookeeper.java     | 53 +++++++++++++++++++
 .../ReplicationSourceManager.java             | 23 +++++---
 2 files changed, 69 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 0e73431e35c9..f23c31c62965 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.ConnectionLossException;
 import org.apache.zookeeper.KeeperException.SessionExpiredException;
@@ -658,6 +659,58 @@ public boolean lockOtherRS(String znode) {
     return true;
   }
 
+  /**
+   * It "atomically" copies all the hlogs queues from another region server and returns them all
+   * sorted per peer cluster (appended with the dead server's znode).
+   * @param znode
+   * @return HLog queues sorted per peer cluster
+   */
+  public SortedMap<String, SortedSet<String>> copyQueuesFromRSUsingMulti(String znode) {
+    SortedMap<String, SortedSet<String>> queues = new TreeMap<String, SortedSet<String>>();
+    String deadRSZnodePath = ZKUtil.joinZNode(rsZNode, znode);// hbase/replication/rs/deadrs
+    List<String> peerIdsToProcess = null;
+    List<ZKUtilOp> listOfOps = new ArrayList<ZKUtil.ZKUtilOp>();
+    try {
+      peerIdsToProcess = ZKUtil.listChildrenNoWatch(this.zookeeper, deadRSZnodePath);
+      if (peerIdsToProcess == null) return null; // node already processed
+      for (String peerId : peerIdsToProcess) {
+        String newPeerId = peerId + "-" + znode;
+        String newPeerZnode = ZKUtil.joinZNode(this.rsServerNameZnode, newPeerId);
+        // check the logs queue for the old peer cluster
+        String oldClusterZnode = ZKUtil.joinZNode(deadRSZnodePath, peerId);
+        List<String> hlogs = ZKUtil.listChildrenNoWatch(this.zookeeper, oldClusterZnode);
+        if (hlogs == null || hlogs.size() == 0) continue; // empty log queue.
+        // create the new cluster znode
+        SortedSet<String> logQueue = new TreeSet<String>();
+        queues.put(newPeerId, logQueue);
+        ZKUtilOp op = ZKUtilOp.createAndFailSilent(newPeerZnode, HConstants.EMPTY_BYTE_ARRAY);
+        listOfOps.add(op);
+        // get the offset of the logs and set it to new znodes
+        for (String hlog : hlogs) {
+          String oldHlogZnode = ZKUtil.joinZNode(oldClusterZnode, hlog);
+          byte[] logOffset = ZKUtil.getData(this.zookeeper, oldHlogZnode);
+          LOG.debug("Creating " + hlog + " with data " + Bytes.toString(logOffset));
+          String newLogZnode = ZKUtil.joinZNode(newPeerZnode, hlog);
+          listOfOps.add(ZKUtilOp.createAndFailSilent(newLogZnode, logOffset));
+          // add ops for deleting
+          listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldHlogZnode));
+          logQueue.add(hlog);
+        }
+        // add delete op for peer
+        listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
+      }
+      // add delete op for dead rs
+      listOfOps.add(ZKUtilOp.deleteNodeFailSilent(deadRSZnodePath));
+      LOG.debug(" The multi list size is: " + listOfOps.size());
+      ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
+      LOG.info("Atomically moved the dead regionserver logs. ");
+    } catch (KeeperException e) {
+      // Multi call failed; it looks like some other regionserver took away the logs.
+      LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
+    }
+    return queues;
+  }
+
   /**
    * This methods copies all the hlogs queues from another region server
    * and returns them all sorted per peer cluster (appended with the dead
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index b67661ff02c1..4c6c897ca682 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
@@ -581,14 +582,22 @@ public void run() {
         LOG.info("Not transferring queue since we are shutting down");
         return;
       }
-      if (!zkHelper.lockOtherRS(rsZnode)) {
-        return;
+      SortedMap<String, SortedSet<String>> newQueues = null;
+
+      // check whether there is multi support. If yes, use it.
+      if (conf.getBoolean(HConstants.ZOOKEEPER_USEMULTI, true)) {
+        LOG.info("Atomically moving " + rsZnode + "'s hlogs to my queue");
+        newQueues = zkHelper.copyQueuesFromRSUsingMulti(rsZnode);
+      } else {
+        LOG.info("Moving " + rsZnode + "'s hlogs to my queue");
+        if (!zkHelper.lockOtherRS(rsZnode)) {
+          return;
+        }
+        newQueues = zkHelper.copyQueuesFromRS(rsZnode);
+        zkHelper.deleteRsQueues(rsZnode);
       }
-      LOG.info("Moving " + rsZnode + "'s hlogs to my queue");
-      SortedMap<String, SortedSet<String>> newQueues =
-          zkHelper.copyQueuesFromRS(rsZnode);
-      zkHelper.deleteRsQueues(rsZnode);
-      if (newQueues == null || newQueues.size() == 0) {
+      // process of copying over the failed queue is completed.
+      if (newQueues.size() == 0) {
         return;
       }
 

From 8aa5ccf6b606fb76c37c898cb7b4d52fe1f4560d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 29 Jan 2013 21:31:30 +0000
Subject: [PATCH 0741/1540] HBASE-7545 [replication] Break out TestReplication
 into manageable classes (Jean-Daniel Cryans)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440150 13f79535-47bb-0310-9956-ffa450edef68
---
 .../wal/SequenceFileLogReader.java            |   2 +-
 .../replication/TestMasterReplication.java    |   2 +-
 .../TestMultiSlaveReplication.java            |   2 +-
 .../replication/TestReplicationBase.java      | 153 +++++++++
 .../TestReplicationDisableInactivePeer.java   |  92 +++++
 .../TestReplicationQueueFailover.java         | 133 ++++++++
 ...stReplicationQueueFailoverCompressed.java} |   4 +-
 ...on.java => TestReplicationSmallTests.java} | 323 +-----------------
 8 files changed, 398 insertions(+), 313 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java
 rename src/test/java/org/apache/hadoop/hbase/replication/{TestReplicationWithCompression.java => TestReplicationQueueFailoverCompressed.java} (90%)
 rename src/test/java/org/apache/hadoop/hbase/replication/{TestReplication.java => TestReplicationSmallTests.java} (57%)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index 9d7928b230a8..a56be623d004 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -265,7 +265,7 @@ public void seek(long pos) throws IOException {
 
   @Override
   public long getPosition() throws IOException {
-    return reader.getPosition();
+    return reader != null ? reader.getPosition() : 0;
   }
 
   protected IOException addFileInfoToException(final IOException ioe)
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
index 9d7c9c186c12..6a0c2ccbbb51 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
@@ -53,7 +53,7 @@
 @Category(LargeTests.class)
 public class TestMasterReplication {
 
-  private static final Log LOG = LogFactory.getLog(TestReplication.class);
+  private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
 
   private Configuration conf1;
   private Configuration conf2;
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
index f7fea5f2d963..1672d978e308 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
@@ -47,7 +47,7 @@
 @Category(LargeTests.class)
 public class TestMultiSlaveReplication {
 
-  private static final Log LOG = LogFactory.getLog(TestReplication.class);
+  private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
 
   private static Configuration conf1;
   private static Configuration conf2;
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
new file mode 100644
index 000000000000..c57449c226e6
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -0,0 +1,153 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * This class is only a base for other integration-level replication tests.
+ * Do not add tests here.
+ * TestReplicationSmallTests is where tests that don't require bring machines up/down should go
+ * All other tests should have their own classes and extend this one
+ */
+public class TestReplicationBase {
+
+  private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
+
+  protected static Configuration conf1 = HBaseConfiguration.create();
+  protected static Configuration conf2;
+  protected static Configuration CONF_WITH_LOCALFS;
+
+  protected static ZooKeeperWatcher zkw1;
+  protected static ZooKeeperWatcher zkw2;
+
+  protected static ReplicationAdmin admin;
+
+  protected static HTable htable1;
+  protected static HTable htable2;
+
+  protected static HBaseTestingUtility utility1;
+  protected static HBaseTestingUtility utility2;
+  protected static final int NB_ROWS_IN_BATCH = 100;
+  protected static final int NB_ROWS_IN_BIG_BATCH =
+      NB_ROWS_IN_BATCH * 10;
+  protected static final long SLEEP_TIME = 1000;
+  protected static final int NB_RETRIES = 10;
+
+  protected static final byte[] tableName = Bytes.toBytes("test");
+  protected static final byte[] famName = Bytes.toBytes("f");
+  protected static final byte[] row = Bytes.toBytes("row");
+  protected static final byte[] noRepfamName = Bytes.toBytes("norep");
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
+    // smaller log roll size to trigger more events
+    conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
+    conf1.setInt("replication.source.size.capacity", 1024);
+    conf1.setLong("replication.source.sleepforretries", 100);
+    conf1.setInt("hbase.regionserver.maxlogs", 10);
+    conf1.setLong("hbase.master.logcleaner.ttl", 10);
+    conf1.setInt("zookeeper.recovery.retry", 1);
+    conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
+    conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
+    conf1.setBoolean("dfs.support.append", true);
+    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
+    conf1.setInt("replication.stats.thread.period.seconds", 5);
+
+    utility1 = new HBaseTestingUtility(conf1);
+    utility1.startMiniZKCluster();
+    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
+    // Have to reget conf1 in case zk cluster location different
+    // than default
+    conf1 = utility1.getConfiguration();  
+    zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
+    admin = new ReplicationAdmin(conf1);
+    LOG.info("Setup first Zk");
+
+    // Base conf2 on conf1 so it gets the right zk cluster.
+    conf2 = HBaseConfiguration.create(conf1);
+    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
+    conf2.setInt("hbase.client.retries.number", 6);
+    conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
+    conf2.setBoolean("dfs.support.append", true);
+
+    utility2 = new HBaseTestingUtility(conf2);
+    utility2.setZkCluster(miniZK);
+    zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
+
+    admin.addPeer("2", utility2.getClusterKey());
+    setIsReplication(true);
+
+    LOG.info("Setup second Zk");
+    CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
+    utility1.startMiniCluster(2);
+    utility2.startMiniCluster(2);
+
+    HTableDescriptor table = new HTableDescriptor(tableName);
+    HColumnDescriptor fam = new HColumnDescriptor(famName);
+    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+    table.addFamily(fam);
+    fam = new HColumnDescriptor(noRepfamName);
+    table.addFamily(fam);
+    HBaseAdmin admin1 = new HBaseAdmin(conf1);
+    HBaseAdmin admin2 = new HBaseAdmin(conf2);
+    admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+    admin2.createTable(table);
+    htable1 = new HTable(conf1, tableName);
+    htable1.setWriteBufferSize(1024);
+    htable2 = new HTable(conf2, tableName);
+  }
+
+  protected static void setIsReplication(boolean rep) throws Exception {
+    LOG.info("Set rep " + rep);
+    admin.setReplicating(rep);
+    Thread.sleep(SLEEP_TIME);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    utility2.shutdownMiniCluster();
+    utility1.shutdownMiniCluster();
+  }
+
+
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
new file mode 100644
index 000000000000..b089fbe8f298
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
@@ -0,0 +1,92 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.fail;
+
+@Category(LargeTests.class)
+public class TestReplicationDisableInactivePeer extends TestReplicationBase {
+
+  private static final Log LOG = LogFactory.getLog(TestReplicationDisableInactivePeer.class);
+
+  /**
+   * Test disabling an inactive peer. Add a peer which is inactive, trying to
+   * insert, disable the peer, then activate the peer and make sure nothing is
+   * replicated. In Addition, enable the peer and check the updates are
+   * replicated.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 600000)
+  public void testDisableInactivePeer() throws Exception {
+
+    // enabling and shutdown the peer
+    admin.enablePeer("2");
+    utility2.shutdownMiniHBaseCluster();
+
+    byte[] rowkey = Bytes.toBytes("disable inactive peer");
+    Put put = new Put(rowkey);
+    put.add(famName, row, row);
+    htable1.put(put);
+
+    // wait for the sleep interval of the master cluster to become long
+    Thread.sleep(SLEEP_TIME * NB_RETRIES);
+
+    // disable and start the peer
+    admin.disablePeer("2");
+    utility2.startMiniHBaseCluster(1, 2);
+    Get get = new Get(rowkey);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      Result res = htable2.get(get);
+      if (res.size() >= 1) {
+        fail("Replication wasn't disabled");
+      } else {
+        LOG.info("Row not replicated, let's wait a bit more...");
+        Thread.sleep(SLEEP_TIME);
+      }
+    }
+
+    // Test enable replication
+    admin.enablePeer("2");
+    // wait since the sleep interval would be long
+    Thread.sleep(SLEEP_TIME * NB_RETRIES);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      Result res = htable2.get(get);
+      if (res.size() == 0) {
+        LOG.info("Row not available");
+        Thread.sleep(SLEEP_TIME * NB_RETRIES);
+      } else {
+        assertArrayEquals(res.value(), row);
+        return;
+      }
+    }
+    fail("Waited too much time for put replication");
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java
new file mode 100644
index 000000000000..9e7e0f555eac
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java
@@ -0,0 +1,133 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.fail;
+
+@Category(LargeTests.class)
+public class TestReplicationQueueFailover extends TestReplicationBase {
+
+  private static final Log LOG = LogFactory.getLog(TestReplicationQueueFailover.class);
+
+  /**
+   * Load up multiple tables over 2 region servers and kill a source during
+   * the upload. The failover happens internally.
+   *
+   * WARNING this test sometimes fails because of HBASE-3515
+   *
+   * @throws Exception
+   */
+  @Test(timeout=300000)
+  public void queueFailover() throws Exception {
+    // killing the RS with .META. can result into failed puts until we solve
+    // IO fencing
+    int rsToKill1 =
+        utility1.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0;
+    int rsToKill2 =
+        utility2.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0;
+
+    // Takes about 20 secs to run the full loading, kill around the middle
+    Thread killer1 = killARegionServer(utility1, 7500, rsToKill1);
+    Thread killer2 = killARegionServer(utility2, 10000, rsToKill2);
+
+    LOG.info("Start loading table");
+    int initialCount = utility1.loadTable(htable1, famName);
+    LOG.info("Done loading table");
+    killer1.join(5000);
+    killer2.join(5000);
+    LOG.info("Done waiting for threads");
+
+    Result[] res;
+    while (true) {
+      try {
+        Scan scan = new Scan();
+        ResultScanner scanner = htable1.getScanner(scan);
+        res = scanner.next(initialCount);
+        scanner.close();
+        break;
+      } catch (UnknownScannerException ex) {
+        LOG.info("Cluster wasn't ready yet, restarting scanner");
+      }
+    }
+    // Test we actually have all the rows, we may miss some because we
+    // don't have IO fencing.
+    if (res.length != initialCount) {
+      LOG.warn("We lost some rows on the master cluster!");
+      // We don't really expect the other cluster to have more rows
+      initialCount = res.length;
+    }
+
+    int lastCount = 0;
+
+    final long start = System.currentTimeMillis();
+    int i = 0;
+    while (true) {
+      if (i==NB_RETRIES-1) {
+        fail("Waited too much time for queueFailover replication. " +
+            "Waited "+(System.currentTimeMillis() - start)+"ms.");
+      }
+      Scan scan2 = new Scan();
+      ResultScanner scanner2 = htable2.getScanner(scan2);
+      Result[] res2 = scanner2.next(initialCount * 2);
+      scanner2.close();
+      if (res2.length < initialCount) {
+        if (lastCount < res2.length) {
+          i--; // Don't increment timeout if we make progress
+        } else {
+          i++;
+        }
+        lastCount = res2.length;
+        LOG.info("Only got " + lastCount + " rows instead of " +
+            initialCount + " current i=" + i);
+        Thread.sleep(SLEEP_TIME*2);
+      } else {
+        break;
+      }
+    }
+  }
+
+  private static Thread killARegionServer(final HBaseTestingUtility utility,
+                                          final long timeout, final int rs) {
+    Thread killer = new Thread() {
+      public void run() {
+        try {
+          Thread.sleep(timeout);
+          utility.expireRegionServerSession(rs);
+        } catch (Exception e) {
+          LOG.error("Couldn't kill a region server", e);
+        }
+      }
+    };
+    killer.setDaemon(true);
+    killer.start();
+    return killer;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java
similarity index 90%
rename from src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
rename to src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java
index 2c87a7bf524a..35c0715cb1e1 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithCompression.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java
@@ -27,7 +27,7 @@
  * Run the same test as TestReplication but with HLog compression enabled
  */
 @Category(LargeTests.class)
-public class TestReplicationWithCompression extends TestReplication {
+public class TestReplicationQueueFailoverCompressed extends TestReplicationQueueFailover {
 
   /**
    * @throws java.lang.Exception
@@ -35,6 +35,6 @@ public class TestReplicationWithCompression extends TestReplication {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
-    TestReplication.setUpBeforeClass();
+    TestReplicationBase.setUpBeforeClass();
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
similarity index 57%
rename from src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
rename to src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index deea43cd76d7..374ea6848de8 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -1,165 +1,35 @@
-/*
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package org.apache.hadoop.hbase.replication;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.UnknownScannerException;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
+import org.apache.hadoop.hbase.client.*;
 import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.mapreduce.Job;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-@Category(LargeTests.class)
-public class TestReplication {
-
-  private static final Log LOG = LogFactory.getLog(TestReplication.class);
-
-  protected static Configuration conf1 = HBaseConfiguration.create();
-  private static Configuration conf2;
-  private static Configuration CONF_WITH_LOCALFS;
-
-  private static ZooKeeperWatcher zkw1;
-  private static ZooKeeperWatcher zkw2;
-
-  private static ReplicationAdmin admin;
-
-  private static HTable htable1;
-  private static HTable htable2;
-
-  private static HBaseTestingUtility utility1;
-  private static HBaseTestingUtility utility2;
-  private static final int NB_ROWS_IN_BATCH = 100;
-  private static final int NB_ROWS_IN_BIG_BATCH =
-      NB_ROWS_IN_BATCH * 10;
-  private static final long SLEEP_TIME = 2000;
-  private static final int NB_RETRIES = 20;
-
-  private static final byte[] tableName = Bytes.toBytes("test");
-  private static final byte[] famName = Bytes.toBytes("f");
-  private static final byte[] row = Bytes.toBytes("row");
-  private static final byte[] noRepfamName = Bytes.toBytes("norep");
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
-    // smaller log roll size to trigger more events
-    conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
-    conf1.setInt("replication.source.size.capacity", 1024);
-    conf1.setLong("replication.source.sleepforretries", 100);
-    conf1.setInt("hbase.regionserver.maxlogs", 10);
-    conf1.setLong("hbase.master.logcleaner.ttl", 10);
-    conf1.setInt("zookeeper.recovery.retry", 1);
-    conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
-    conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
-    conf1.setBoolean("dfs.support.append", true);
-    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
-
-    utility1 = new HBaseTestingUtility(conf1);
-    utility1.startMiniZKCluster();
-    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
-    // Have to reget conf1 in case zk cluster location different
-    // than default
-    conf1 = utility1.getConfiguration();  
-    zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
-    admin = new ReplicationAdmin(conf1);
-    LOG.info("Setup first Zk");
-
-    // Base conf2 on conf1 so it gets the right zk cluster.
-    conf2 = HBaseConfiguration.create(conf1);
-    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
-    conf2.setInt("hbase.client.retries.number", 6);
-    conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
-    conf2.setBoolean("dfs.support.append", true);
-
-    utility2 = new HBaseTestingUtility(conf2);
-    utility2.setZkCluster(miniZK);
-    zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
-
-    admin.addPeer("2", utility2.getClusterKey());
-    setIsReplication(true);
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
-    LOG.info("Setup second Zk");
-    CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
-    utility1.startMiniCluster(2);
-    utility2.startMiniCluster(2);
-
-    HTableDescriptor table = new HTableDescriptor(tableName);
-    HColumnDescriptor fam = new HColumnDescriptor(famName);
-    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-    table.addFamily(fam);
-    fam = new HColumnDescriptor(noRepfamName);
-    table.addFamily(fam);
-    HBaseAdmin admin1 = new HBaseAdmin(conf1);
-    HBaseAdmin admin2 = new HBaseAdmin(conf2);
-    admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
-    admin2.createTable(table);
-    htable1 = new HTable(conf1, tableName);
-    htable1.setWriteBufferSize(1024);
-    htable2 = new HTable(conf2, tableName);
-  }
+@Category(LargeTests.class)
+public class TestReplicationSmallTests extends TestReplicationBase {
 
-  private static void setIsReplication(boolean rep) throws Exception {
-    LOG.info("Set rep " + rep);
-    admin.setReplicating(rep);
-    Thread.sleep(SLEEP_TIME);
-  }
+  private static final Log LOG = LogFactory.getLog(TestReplicationSmallTests.class);
 
   /**
    * @throws java.lang.Exception
    */
   @Before
   public void setUp() throws Exception {
-
+    htable1.setAutoFlush(true);
     // Starting and stopping replication can make us miss new logs,
     // rolling like this makes sure the most recent one gets added to the queue
     for ( JVMClusterUtil.RegionServerThread r :
@@ -182,7 +52,7 @@ public void setUp() throws Exception {
       Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH);
       scanner.close();
       if (res.length != 0) {
-       if (res.length < lastCount) {
+        if (res.length < lastCount) {
           i--; // Don't increment timeout if we make progress
         }
         lastCount = res.length;
@@ -194,15 +64,6 @@ public void setUp() throws Exception {
     }
   }
 
-  /**
-   * @throws java.lang.Exception
-   */
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    utility2.shutdownMiniCluster();
-    utility1.shutdownMiniCluster();
-  }
-
   /**
    * Verify that version and column delete marker types are replicated
    * correctly.
@@ -225,11 +86,11 @@ public void testDeleteTypes() throws Exception {
     put = new Put(row);
     put.add(famName, row, t+1, v2);
     htable1.put(put);
-    
+
     put = new Put(row);
     put.add(famName, row, t+2, v3);
     htable1.put(put);
-    
+
     Get get = new Get(row);
     get.setMaxVersions();
     for (int i = 0; i < NB_RETRIES; i++) {
@@ -375,9 +236,6 @@ public void testSmallBatch() throws Exception {
         break;
       }
     }
-
-    htable1.setAutoFlush(true);
-
   }
 
   /**
@@ -493,60 +351,6 @@ public void testDisableEnable() throws Exception {
     fail("Waited too much time for put replication");
   }
 
-  /**
-   * Test disabling an inactive peer. Add a peer which is inactive, trying to
-   * insert, disable the peer, then activate the peer and make sure nothing is
-   * replicated. In Addition, enable the peer and check the updates are
-   * replicated.
-   *
-   * @throws Exception
-   */
-  @Test(timeout = 600000)
-  public void testDisableInactivePeer() throws Exception {
-
-    // enabling and shutdown the peer
-    admin.enablePeer("2");
-    utility2.shutdownMiniHBaseCluster();
-
-    byte[] rowkey = Bytes.toBytes("disable inactive peer");
-    Put put = new Put(rowkey);
-    put.add(famName, row, row);
-    htable1.put(put);
-
-    // wait for the sleep interval of the master cluster to become long
-    Thread.sleep(SLEEP_TIME * NB_RETRIES);
-
-    // disable and start the peer
-    admin.disablePeer("2");
-    utility2.startMiniHBaseCluster(1, 2);
-    Get get = new Get(rowkey);
-    for (int i = 0; i < NB_RETRIES; i++) {
-      Result res = htable2.get(get);
-      if (res.size() >= 1) {
-        fail("Replication wasn't disabled");
-      } else {
-        LOG.info("Row not replicated, let's wait a bit more...");
-        Thread.sleep(SLEEP_TIME);
-      }
-    }
-
-    // Test enable replication
-    admin.enablePeer("2");
-    // wait since the sleep interval would be long
-    Thread.sleep(SLEEP_TIME * NB_RETRIES);
-    for (int i = 0; i < NB_RETRIES; i++) {
-      Result res = htable2.get(get);
-      if (res.size() == 0) {
-        LOG.info("Row not available");
-        Thread.sleep(SLEEP_TIME * NB_RETRIES);
-      } else {
-        assertArrayEquals(res.value(), row);
-        return;
-      }
-    }
-    fail("Waited too much time for put replication");
-  }
-
   /**
    * Integration test for TestReplicationAdmin, removes and re-add a peer
    * cluster
@@ -601,6 +405,7 @@ public void testAddAndRemoveClusters() throws Exception {
     }
   }
 
+
   /**
    * Do a more intense version testSmallBatch, one  that will trigger
    * hlog rolling and other non-trivial code paths
@@ -700,107 +505,9 @@ public void testVerifyRepJob() throws Exception {
       fail("Job failed, see the log");
     }
     assertEquals(0, job.getCounters().
-            findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
-        assertEquals(NB_ROWS_IN_BATCH, job.getCounters().
-            findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
-  }
-
-  /**
-   * Load up multiple tables over 2 region servers and kill a source during
-   * the upload. The failover happens internally.
-   *
-   * WARNING this test sometimes fails because of HBASE-3515
-   *
-   * @throws Exception
-   */
-  @Test(timeout=300000)
-  public void queueFailover() throws Exception {
-    // killing the RS with .META. can result into failed puts until we solve
-    // IO fencing
-    int rsToKill1 =
-        utility1.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0;
-    int rsToKill2 =
-        utility2.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0;
-
-    // Takes about 20 secs to run the full loading, kill around the middle
-    Thread killer1 = killARegionServer(utility1, 7500, rsToKill1);
-    Thread killer2 = killARegionServer(utility2, 10000, rsToKill2);
-
-    LOG.info("Start loading table");
-    int initialCount = utility1.loadTable(htable1, famName);
-    LOG.info("Done loading table");
-    killer1.join(5000);
-    killer2.join(5000);
-    LOG.info("Done waiting for threads");
-
-    Result[] res;
-    while (true) {
-      try {
-        Scan scan = new Scan();
-        ResultScanner scanner = htable1.getScanner(scan);
-        res = scanner.next(initialCount);
-        scanner.close();
-        break;
-      } catch (UnknownScannerException ex) {
-        LOG.info("Cluster wasn't ready yet, restarting scanner");
-      }
-    }
-    // Test we actually have all the rows, we may miss some because we
-    // don't have IO fencing.
-    if (res.length != initialCount) {
-      LOG.warn("We lost some rows on the master cluster!");
-      // We don't really expect the other cluster to have more rows
-      initialCount = res.length;
-    }
-
-    int lastCount = 0;
-
-    final long start = System.currentTimeMillis();
-    int i = 0;
-    while (true) {
-      if (i==NB_RETRIES-1) {
-        fail("Waited too much time for queueFailover replication. " +
-          "Waited "+(System.currentTimeMillis() - start)+"ms.");
-      }
-      Scan scan2 = new Scan();
-      ResultScanner scanner2 = htable2.getScanner(scan2);
-      Result[] res2 = scanner2.next(initialCount * 2);
-      scanner2.close();
-      if (res2.length < initialCount) {
-        if (lastCount < res2.length) {
-          i--; // Don't increment timeout if we make progress
-        } else {
-          i++;
-        }
-        lastCount = res2.length;
-        LOG.info("Only got " + lastCount + " rows instead of " +
-            initialCount + " current i=" + i);
-        Thread.sleep(SLEEP_TIME*2);
-      } else {
-        break;
-      }
-    }
-  }
-
-  private static Thread killARegionServer(final HBaseTestingUtility utility,
-                                   final long timeout, final int rs) {
-    Thread killer = new Thread() {
-      public void run() {
-        try {
-          Thread.sleep(timeout);
-          utility.expireRegionServerSession(rs);
-        } catch (Exception e) {
-          LOG.error("Couldn't kill a region server", e);
-        }
-      }
-    };
-    killer.setDaemon(true);
-    killer.start();
-    return killer;
+        findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
+    assertEquals(NB_ROWS_IN_BATCH, job.getCounters().
+        findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
   }
 
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-

From 976d38702e074b88c4f19da33e545b08e392c3a1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Jan 2013 05:43:11 +0000
Subject: [PATCH 0742/1540] HBASE-7684 NullPointerException in SecureClient
 when Call is cleaned up due to RPC timeout (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440282 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/SecureClient.java    | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index c0d10e019572..26258bbd4a6d 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -375,10 +375,16 @@ protected void receiveResponse() {
           if (LOG.isDebugEnabled()) {
             LOG.debug("call #"+id+", response is:\n"+value.toString());
           }
-          call.setValue(value);
+          // it's possible that this call may have been cleaned up due to a RPC
+          // timeout, so check if it still exists before setting the value.
+          if (call != null) {
+            call.setValue(value);
+          }
         } else if (state == Status.ERROR.state) {
-          call.setException(new RemoteException(WritableUtils.readString(in),
-                                                WritableUtils.readString(in)));
+          if (call != null) {
+            call.setException(new RemoteException(WritableUtils.readString(in), WritableUtils
+                .readString(in)));
+          }
         } else if (state == Status.FATAL.state) {
           // Close the connection
           markClosed(new RemoteException(WritableUtils.readString(in),

From a7b0e4092f1c7df9c0b3b7fdd8d6b920ca2e046b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Jan 2013 05:47:55 +0000
Subject: [PATCH 0743/1540] HBASE-7705 Make the method getCurrentPoolSize of
 HTablePool public (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440283 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HTablePool.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 698241bc1f22..d9615acfdb5b 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -308,7 +308,7 @@ public void close() throws IOException {
     this.tables.clear();
   }
 
-  int getCurrentPoolSize(String tableName) {
+  public int getCurrentPoolSize(String tableName) {
     return tables.size(tableName);
   }
 

From 842949b0371ce1091d33b2ec7ba56cf4a0c310f3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Jan 2013 05:51:00 +0000
Subject: [PATCH 0744/1540] HBASE-7685 Closing socket connection can't be
 removed from SecureClient (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440285 13f79535-47bb-0310-9956-ffa450edef68
---
 .../main/java/org/apache/hadoop/hbase/ipc/SecureClient.java   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 26258bbd4a6d..d11592874a78 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -418,9 +418,7 @@ protected synchronized void close() {
       // release the resources
       // first thing to do;take the connection out of the connection list
       synchronized (connections) {
-        if (connections.get(remoteId) == this) {
-          connections.remove(remoteId);
-        }
+        connections.remove(remoteId, this);
       }
 
       // close the streams and therefore the socket

From cde3adc9b720616f68553108b949737e718f884e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Jan 2013 17:21:10 +0000
Subject: [PATCH 0745/1540] HBASE-7545 addendum, increase timeouts for
 replication tests due to recent failures.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/replication/TestReplicationBase.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index c57449c226e6..82b8df2080d2 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -63,7 +63,7 @@ public class TestReplicationBase {
   protected static final int NB_ROWS_IN_BIG_BATCH =
       NB_ROWS_IN_BATCH * 10;
   protected static final long SLEEP_TIME = 1000;
-  protected static final int NB_RETRIES = 10;
+  protected static final int NB_RETRIES = 15;
 
   protected static final byte[] tableName = Bytes.toBytes("test");
   protected static final byte[] famName = Bytes.toBytes("f");

From 715b6f8bbadae5a9ad5a34b9259c33786357d950 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Jan 2013 19:38:23 +0000
Subject: [PATCH 0746/1540] HBASE-7545 trying with increase timeout for
 TestReplicationSmallTests.loadTesting

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440604 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/replication/TestReplicationBase.java  | 1 +
 .../hadoop/hbase/replication/TestReplicationSmallTests.java   | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index 82b8df2080d2..938109f43f6c 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -64,6 +64,7 @@ public class TestReplicationBase {
       NB_ROWS_IN_BATCH * 10;
   protected static final long SLEEP_TIME = 1000;
   protected static final int NB_RETRIES = 15;
+  protected static final int NB_RETRIES_FOR_BIG_BATCH = 30;
 
   protected static final byte[] tableName = Bytes.toBytes("test");
   protected static final byte[] famName = Bytes.toBytes("f");
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 374ea6848de8..d8a0f5dcd8f9 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -432,13 +432,13 @@ public void loadTesting() throws Exception {
 
     scan = new Scan();
 
-    for (int i = 0; i < NB_RETRIES; i++) {
+    for (int i = 0; i < NB_RETRIES_FOR_BIG_BATCH; i++) {
 
       scanner = htable2.getScanner(scan);
       res = scanner.next(NB_ROWS_IN_BIG_BATCH);
       scanner.close();
       if (res.length != NB_ROWS_IN_BIG_BATCH) {
-        if (i == NB_RETRIES-1) {
+        if (i == NB_RETRIES_FOR_BIG_BATCH-1) {
           int lastRow = -1;
           for (Result result : res) {
             int currentRow = Bytes.toInt(result.getRow());

From d48d8d566ad7f2cbc5dd7345c4157fbeee9ffbba Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 30 Jan 2013 22:24:14 +0000
Subject: [PATCH 0747/1540] HBASE-7702: Adding filtering to Import jobs

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440713 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/Import.java | 117 +++++++++++++++++-
 .../hbase/mapreduce/TestImportExport.java     | 101 +++++++++++++++
 2 files changed, 215 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 31238de4e636..361d30f0a46e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -20,6 +20,9 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
@@ -32,12 +35,13 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -53,10 +57,15 @@
  * Import data written by {@link Export}.
  */
 public class Import {
+  private static final Log LOG = LogFactory.getLog(Import.class);
   final static String NAME = "import";
   final static String CF_RENAME_PROP = "HBASE_IMPORTER_RENAME_CFS";
   final static String BULK_OUTPUT_CONF_KEY = "import.bulk.output";
-  private static final Log LOG = LogFactory.getLog(Import.class);
+  final static String FILTER_CLASS_CONF_KEY = "import.filter.class";
+  final static String FILTER_ARGS_CONF_KEY = "import.filter.args";
+
+  // Optional filter to use for mappers
+  private static Filter filter;
 
   /**
    * A mapper that just writes out KeyValues.
@@ -79,6 +88,10 @@ public void map(ImmutableBytesWritable row, Result value,
     throws IOException {
       try {
         for (KeyValue kv : value.raw()) {
+          kv = filterKv(kv);
+          // skip if we filtered it out
+          if (kv == null) continue;
+
           context.write(row, convertKv(kv, cfRenameMap));
         }
       } catch (InterruptedException e) {
@@ -89,6 +102,7 @@ public void map(ImmutableBytesWritable row, Result value,
     @Override
     public void setup(Context context) {
       cfRenameMap = createCfRenameMap(context.getConfiguration());
+      filter = instantiateFilter(context.getConfiguration());
     }
   }
 
@@ -124,6 +138,10 @@ private void writeResult(ImmutableBytesWritable key, Result result, Context cont
       Put put = null;
       Delete delete = null;
       for (KeyValue kv : result.raw()) {
+        kv = filterKv(kv);
+        // skip if we filter it out
+        if (kv == null) continue;
+
         kv = convertKv(kv, cfRenameMap);
         // Deletes and Puts are gathered and written when finished
         if (kv.isDelete()) {
@@ -152,6 +170,8 @@ private void writeResult(ImmutableBytesWritable key, Result result, Context cont
     public void setup(Context context) {
       Configuration conf = context.getConfiguration();
       cfRenameMap = createCfRenameMap(conf);
+      filter = instantiateFilter(conf);
+
       try {
         HConnection connection = HConnectionManager.getConnection(conf);
         ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
@@ -168,6 +188,77 @@ public void setup(Context context) {
     }
   }
 
+  /**
+   * Create a {@link Filter} to apply to all incoming keys ({@link KeyValue KeyValues}) to
+   * optionally not include in the job output
+   * @param conf {@link Configuration} from which to load the filter
+   * @return the filter to use for the task, or <tt>null</tt> if no filter to should be used
+   * @throws IllegalArgumentException if the filter is misconfigured
+   */
+  private static Filter instantiateFilter(Configuration conf) {
+    // get the filter, if it was configured
+    Class<? extends Filter> filterClass = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
+    if (filterClass == null) {
+      LOG.debug("No configured filter class, accepting all keyvalues.");
+      return null;
+    }
+    LOG.debug("Attempting to create filter:" + filterClass);
+
+    try {
+      Method m = filterClass.getMethod("createFilterFromArguments", ArrayList.class);
+      return (Filter) m.invoke(null, getFilterArgs(conf));
+    } catch (IllegalAccessException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (SecurityException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (NoSuchMethodException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (IllegalArgumentException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (InvocationTargetException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static ArrayList<byte[]> getFilterArgs(Configuration conf) {
+    ArrayList<byte[]> args = new ArrayList<byte[]>();
+    String[] sargs = conf.getStrings(FILTER_ARGS_CONF_KEY);
+    for (String arg : sargs) {
+      // all the filters' instantiation methods expected quoted args since they are coming from
+      // the shell, so add them here, though its shouldn't really be needed :-/
+      args.add(Bytes.toBytes("'" + arg + "'"));
+    }
+    return args;
+  }
+
+  /**
+   * Attempt to filter out the keyvalue
+   * @param kv {@link KeyValue} on which to apply the filter
+   * @return <tt>null</tt> if the key should not be written, otherwise returns the original
+   *         {@link KeyValue}
+   */
+  private static KeyValue filterKv(KeyValue kv) {
+    // apply the filter and skip this kv if the filter doesn't apply
+    if (filter != null) {
+      Filter.ReturnCode code = filter.filterKeyValue(kv);
+      System.out.println("Filter returned:" + code);
+      // if its not an accept type, then skip this kv
+      if (!(code.equals(Filter.ReturnCode.INCLUDE) || code
+          .equals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL))) {
+        if (LOG.isDebugEnabled()) {
+          System.out.println("Skipping key: " + kv + " from filter decision: " + code);
+        }
+        return null;
+      }
+    }
+    return kv;
+  }
+
   // helper: create a new KeyValue based on CF rename map
   private static KeyValue convertKv(KeyValue kv, Map<byte[], byte[]> cfRenameMap) {
     if(cfRenameMap != null) {
@@ -266,6 +357,17 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
     FileInputFormat.setInputPaths(job, inputDir);
     job.setInputFormatClass(SequenceFileInputFormat.class);
     String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
+
+    // make sure we get the filter in the jars
+    try {
+      Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
+      if (filter != null) {
+        TableMapReduceUtil.addDependencyJars(conf, filter);
+      }
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+
     if (hfileOutPath != null) {
       job.setMapperClass(KeyValueImporter.class);
       HTable table = new HTable(conf, tableName);
@@ -298,6 +400,15 @@ private static void usage(final String errorMsg) {
     System.err.println("By default Import will load data directly into HBase. To instead generate");
     System.err.println("HFiles of data to prepare for a bulk data load, pass the option:");
     System.err.println("  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+    System.err
+        .println(" To apply a generic org.apache.hadoop.hbase.filter.Filter to the input, use");
+    System.err.println("  -D" + FILTER_CLASS_CONF_KEY + "=<name of filter class>");
+    System.err.println("  -D" + FILTER_ARGS_CONF_KEY + "=<comma separated list of args for filter");
+    System.err.println(" NOTE: The filter will be applied BEFORE doing key renames via the "
+        + CF_RENAME_PROP + " property. Futher, filters will only use the"
+        + "Filter#filterKeyValue(KeyValue) method to determine if the KeyValue should be added;"
+        + " Filter.ReturnCode#INCLUDE and #INCLUDE_AND_NEXT_COL will be considered as including "
+        + "the KeyValue.");
     System.err.println("For performance consider the following options:\n"
         + "  -Dmapred.map.tasks.speculative.execution=false\n"
         + "  -Dmapred.reduce.tasks.speculative.execution=false");
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index a573cbc8d7de..c0cd2f156933 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -18,8 +18,11 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -36,11 +39,14 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -242,4 +248,99 @@ public void testWithDeletes() throws Exception {
     assertEquals(now, res[6].getTimestamp());
     t.close();
   }
+
+  @Test
+  public void testWithFilter() throws Exception {
+    String EXPORT_TABLE = "exportSimpleCase_ImportWithFilter";
+    HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
+    UTIL.getHBaseAdmin().createTable(desc);
+    HTable exportTable = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
+
+    Put p = new Put(ROW1);
+    p.add(FAMILYA, QUAL, now, QUAL);
+    p.add(FAMILYA, QUAL, now + 1, QUAL);
+    p.add(FAMILYA, QUAL, now + 2, QUAL);
+    p.add(FAMILYA, QUAL, now + 3, QUAL);
+    p.add(FAMILYA, QUAL, now + 4, QUAL);
+    exportTable.put(p);
+
+    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
+
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
+        cluster.getConfiguration()), args);
+    Configuration conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+
+    String IMPORT_TABLE = "importWithFilter";
+    desc = new HTableDescriptor(IMPORT_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
+    UTIL.getHBaseAdmin().createTable(desc);
+
+    HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
+    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
+        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
+        "1000" };
+
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+
+    // get the count of the source table for that time range
+    PrefixFilter filter = new PrefixFilter(ROW1);
+    int count = getCount(exportTable, filter);
+
+    Assert.assertEquals("Unexpected row count between export and import tables", count,
+      getCount(importTable, null));
+
+    // and then test that a broken command doesn't bork everything - easier here because we don't
+    // need to re-run the export job
+
+    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
+        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
+        OUTPUT_DIR, "1000" };
+
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
+
+    // cleanup
+    exportTable.close();
+    importTable.close();
+  }
+
+  /**
+   * Count the number of keyvalues in the specified table for the given timerange
+   * @param start
+   * @param end
+   * @param table
+   * @return
+   * @throws IOException
+   */
+  private int getCount(HTable table, Filter filter) throws IOException {
+    Scan scan = new Scan();
+    scan.setFilter(filter);
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      count += res.size();
+    }
+    results.close();
+    return count;
+  }
 }

From 939e5a9b713cb0dc7cae2133b3ef32dc76eb9018 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 30 Jan 2013 23:46:10 +0000
Subject: [PATCH 0748/1540] HBASE-7702: Adding filtering to Import jobs -
 Forgot this bit...

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440736 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/Import.java | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 361d30f0a46e..e5a797f81f93 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -23,6 +23,7 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
@@ -339,6 +340,27 @@ static public void configureCfRenaming(Configuration conf,
     conf.set(CF_RENAME_PROP, sb.toString());
   }
   
+  /**
+   * Add a Filter to be instantiated on import
+   * @param conf Configuration to update (will be passed to the job)
+   * @param clazz {@link Filter} subclass to instantiate on the server.
+   * @param args List of arguments to pass to the filter on instantiation
+   */
+  public static void addFilterAndArguments(Configuration conf, Class<? extends Filter> clazz,
+      List<String> args) {
+    conf.set(Import.FILTER_CLASS_CONF_KEY, clazz.getName());
+
+    // build the param string for the key
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < args.size(); i++) {
+      String arg = args.get(i);
+      builder.append(arg);
+      if (i != args.size() - 1) {
+        builder.append(",");
+      }
+    }
+    conf.set(Import.FILTER_ARGS_CONF_KEY, builder.toString());
+  }
 
   /**
    * Sets up the actual job.

From 80d9a75c0f074b0689d0cb0ffae67b96e03e759a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 31 Jan 2013 01:19:43 +0000
Subject: [PATCH 0749/1540] HBASE-7715 FSUtils#waitOnSafeMode can incorrectly
 loop on standby NN (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440779 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/FSUtils.java | 31 +++++++++++++++++--
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 179b86d88bec..d5923d8a2074 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -23,6 +23,7 @@
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -227,6 +228,31 @@ public static void checkFileSystemAvailable(final FileSystem fs)
     throw io;
   }
 
+  /**
+   * We use reflection because {@link DistributedFileSystem#setSafeMode(
+   * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1
+   * 
+   * @param dfs
+   * @return whether we're in safe mode
+   * @throws IOException
+   */
+  private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
+    boolean inSafeMode = false;
+    try {
+      Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
+          org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
+      inSafeMode = (Boolean) m.invoke(dfs,
+        org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
+    } catch (Exception e) {
+      if (e instanceof IOException) throw (IOException) e;
+      
+      // Check whether dfs is on safemode.
+      inSafeMode = dfs.setSafeMode(
+        org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);      
+    }
+    return inSafeMode;    
+  }
+  
   /**
    * Check whether dfs is in safemode. 
    * @param conf
@@ -238,8 +264,7 @@ public static void checkDfsSafeMode(final Configuration conf)
     FileSystem fs = FileSystem.get(conf);
     if (fs instanceof DistributedFileSystem) {
       DistributedFileSystem dfs = (DistributedFileSystem)fs;
-      // Check whether dfs is on safemode.
-      isInSafeMode = dfs.setSafeMode(org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);
+      isInSafeMode = isInSafeMode(dfs);
     }
     if (isInSafeMode) {
       throw new IOException("File system is in safemode, it can't be written now");
@@ -526,7 +551,7 @@ public static void waitOnSafeMode(final Configuration conf,
     if (!(fs instanceof DistributedFileSystem)) return;
     DistributedFileSystem dfs = (DistributedFileSystem)fs;
     // Make sure dfs is not in safe mode
-    while (dfs.setSafeMode(org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET)) {
+    while (isInSafeMode(dfs)) {
       LOG.info("Waiting for dfs to exit safe mode...");
       try {
         Thread.sleep(wait);

From 9ffd863bfcb4089c6078ffd0a256b54eb211cb45 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 31 Jan 2013 02:25:57 +0000
Subject: [PATCH 0750/1540] HBASE-7717 Wait until regions are assigned in
 TestSplitTransactionOnCluster (Lars H and Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1440801 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestSplitTransactionOnCluster.java        | 192 +++++++++---------
 1 file changed, 94 insertions(+), 98 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 8a4c7ffbdf0b..e51f9979c914 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -77,9 +77,9 @@ public class TestSplitTransactionOnCluster {
   private MiniHBaseCluster cluster = null;
   private static final int NB_SERVERS = 2;
   private static CountDownLatch latch = new CountDownLatch(1);
-  private static boolean secondSplit = false;
-  private static boolean callRollBack = false;
-  private static boolean firstSplitCompleted = false;
+  private static volatile boolean secondSplit = false;
+  private static volatile boolean callRollBack = false;
+  private static volatile boolean firstSplitCompleted = false;
   
   private static final HBaseTestingUtility TESTING_UTIL =
     new HBaseTestingUtility();
@@ -108,6 +108,82 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     return regions.get(0).getRegionInfo();
   }
 
+  @Test(timeout = 2000000)
+  public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
+    final byte[] tableName = Bytes
+        .toBytes("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
+    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
+    try {
+      // Create table then get the single region for our new table.
+      HTable t = TESTING_UTIL.createTable(tableName, Bytes.toBytes("cf"));
+
+      final List<HRegion> regions = cluster.getRegions(tableName);
+      HRegionInfo hri = getAndCheckSingleTableRegion(regions);
+      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+      final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+      insertData(tableName, admin, t);
+      // Turn off balancer so it doesn't cut in and mess up our placements.
+      this.admin.setBalancerRunning(false, false);
+      // Turn off the meta scanner so it don't remove parent on us.
+      cluster.getMaster().setCatalogJanitorEnabled(false);
+
+      new Thread() {
+        public void run() {
+          SplitTransaction st = null;
+          st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
+          try {
+            st.prepare();
+            st.execute(regionServer, regionServer);
+          } catch (IOException e) {
+
+          }
+        }
+      }.start();
+      for (int i = 0; !callRollBack && i < 100; i++) {
+        Thread.sleep(100);
+      }
+      assertTrue("Waited too long for rollback", callRollBack);
+      SplitTransaction st = null;
+      st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
+      try {
+        secondSplit = true;
+        st.prepare();
+        st.execute(regionServer, regionServer);
+      } catch (IOException e) {
+        LOG.debug("Rollback started :"+ e.getMessage());
+        st.rollback(regionServer, regionServer);
+      }
+      for (int i=0; !firstSplitCompleted && i<100; i++) {
+        Thread.sleep(100);
+      }
+      assertTrue("fist split did not complete", firstSplitCompleted);
+      NavigableMap<String, RegionState> rit = cluster.getMaster().getAssignmentManager()
+          .getRegionsInTransition();
+      for (int i=0; rit.containsKey(hri.getTableNameAsString()) && i<100; i++) {
+        Thread.sleep(100);
+      }
+      assertFalse("region still in transition", rit.containsKey(hri.getTableNameAsString()));
+      List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
+      // Region server side split is successful.
+      assertEquals("The parent region should be splitted", 2, onlineRegions.size());
+      //Should be present in RIT
+      List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager().getRegionsOfTable(tableName);
+      // Master side should also reflect the same
+      assertEquals("No of regions in master", 2, regionsOfTable.size());
+    } finally {
+      admin.setBalancerRunning(true, false);
+      secondSplit = false;
+      firstSplitCompleted = false;
+      callRollBack = false;
+      cluster.getMaster().setCatalogJanitorEnabled(true);
+    }
+    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+      admin.close();
+    }
+  }
+
   /**
    * A test that intentionally has master fail the processing of the split message.
    * Tests that the regionserver split ephemeral node gets cleaned up if it
@@ -185,6 +261,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
       SplitRegionHandler.TEST_SKIP = false;
       admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
+      t.close();
     }
   }
 
@@ -236,6 +313,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     } finally {
       admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
+      t.close();
     }
   }
 
@@ -289,6 +367,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     } finally {
       admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
+      t.close();
     }
   }
 
@@ -360,6 +439,7 @@ private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
     } finally {
       admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
+      t.close();
     }
   }
   
@@ -436,6 +516,7 @@ public void testMasterRestartWhenSplittingIsPartial()
       SplitRegionHandler.TEST_SKIP = false;
       admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
+      t.close();
     }
   }
 
@@ -456,11 +537,7 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
 
     // Create table then get the single region for our new table.
     this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor(HConstants.CATALOG_FAMILY);
-    htd.addFamily(hcd);
-    this.admin.createTable(htd);
-    HTable t = new HTable(TESTING_UTIL.getConfiguration(), tableName);
+    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
@@ -517,6 +594,7 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
       SplitRegionHandler.TEST_SKIP = false;
       this.admin.setBalancerRunning(true, false);
       cluster.getMaster().setCatalogJanitorEnabled(true);
+      t.close();
     }
   }
 
@@ -601,6 +679,10 @@ public void testShouldClearRITWhenNodeFoundInSplittingState() throws Exception {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.addFamily(new HColumnDescriptor("cf"));
     admin.createTable(htd);
+    for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
+      Thread.sleep(100);
+    }
+    assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
 
     HRegion region = cluster.getRegions(tableName).get(0);
     int regionServerIndex = cluster.getServerWith(region.getRegionName());
@@ -648,90 +730,6 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
     }
   }
   
-  @Test(timeout = 2000000)
-  public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
-    final byte[] tableName = Bytes
-        .toBytes("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
-    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-    try {
-      // Create table then get the single region for our new table.
-      HTableDescriptor htd = new HTableDescriptor(tableName);
-      htd.addFamily(new HColumnDescriptor("cf"));
-      admin.createTable(htd);
-      HTable t = new HTable(cluster.getConfiguration(), tableName);
-      // wait for up to 10s
-      for (int i=0; cluster.getRegions(tableName).size() != 1 && i<100; i++) {
-        Thread.sleep(100);
-      }
-      assertTrue("waited too long for table to get online",
-          cluster.getRegions(tableName).size() == 1);
-      final List<HRegion> regions = cluster.getRegions(tableName);
-      HRegionInfo hri = getAndCheckSingleTableRegion(regions);
-      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
-      final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
-      insertData(tableName, admin, t);
-      // Turn off balancer so it doesn't cut in and mess up our placements.
-      this.admin.setBalancerRunning(false, false);
-      // Turn off the meta scanner so it don't remove parent on us.
-      cluster.getMaster().setCatalogJanitorEnabled(false);
-
-      new Thread() {
-        public void run() {
-          SplitTransaction st = null;
-          st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
-          try {
-            st.prepare();
-            st.execute(regionServer, regionServer);
-          } catch (IOException e) {
-
-          }
-        }
-      }.start();
-      for (int i=0; !callRollBack && i<100; i++) {
-        Thread.sleep(100);
-      }
-      assertTrue("Waited too long for rollback", callRollBack);
-      SplitTransaction st = null;
-      st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2"));
-      try {
-        secondSplit = true;
-        st.prepare();
-        st.execute(regionServer, regionServer);
-      } catch (IOException e) {
-        LOG.debug("Rollback started :"+ e.getMessage());
-        st.rollback(regionServer, regionServer);
-      }
-      for (int i=0; !firstSplitCompleted && i<100; i++) {
-        Thread.sleep(100);
-      }
-      assertTrue("fist split did not complete", firstSplitCompleted);
-      NavigableMap<String, RegionState> rit = cluster.getMaster().getAssignmentManager()
-          .getRegionsInTransition();
-      for (int i=0; rit.containsKey(hri.getTableNameAsString()) && i<100; i++) {
-        Thread.sleep(100);
-      }
-      assertFalse("region still in transition", rit.containsKey(hri.getTableNameAsString()));
-      List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
-      // Region server side split is successful.
-      assertEquals("The parent region should be splitted", 2, onlineRegions.size());
-      //Should be present in RIT
-      List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager().getRegionsOfTable(tableName);
-      // Master side should also reflect the same
-      assertEquals("No of regions in master", 2, regionsOfTable.size());
-    } finally {
-      admin.setBalancerRunning(true, false);
-      secondSplit = false;
-      firstSplitCompleted = false;
-      callRollBack = false;
-      cluster.getMaster().setCatalogJanitorEnabled(true);
-    }
-    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
-      admin.disableTable(tableName);
-      admin.deleteTable(tableName);
-      admin.close();
-    }
-  }
-
   @Test(timeout = 20000)
   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
     final byte[] tableName = 
@@ -741,10 +739,8 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
     HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
     try {
       // Create table then get the single region for our new table.
-      HTableDescriptor htd = new HTableDescriptor(tableName);
-      htd.addFamily(new HColumnDescriptor("cf"));
-      admin.createTable(htd);
-      HTable t = new HTable(cluster.getConfiguration(), tableName);
+      HTable t = TESTING_UTIL.createTable(tableName, Bytes.toBytes("cf"));
+
       regions = cluster.getRegions(tableName);
       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
       regionServer = cluster.getRegionServer(regionServerIndex);
@@ -797,10 +793,10 @@ public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfull
       HTableDescriptor htd = new HTableDescriptor(tableName);
       htd.addFamily(new HColumnDescriptor("cf"));
       admin.createTable(htd);
-      for (int i=0; cluster.getRegions(tableName).size() != 1 && i<100; i++) {
+      for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
         Thread.sleep(100);
       }
-      assertTrue("Table not online", cluster.getRegions(tableName).size() == 1);
+      assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
       List<HRegion> regions = cluster.getRegions(tableName);
       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
       int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);

From d91aef7e0c16c61c8da6d9e38263bb2b1b5b97e7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 31 Jan 2013 19:25:03 +0000
Subject: [PATCH 0751/1540] HBASE-7717 addendum, really wait for all tables in
 TestSplitTransactionOnCluster.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441151 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestSplitTransactionOnCluster.java        | 34 ++++++++++---------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index e51f9979c914..94303b7df141 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -115,8 +115,7 @@ public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exc
     HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
     try {
       // Create table then get the single region for our new table.
-      HTable t = TESTING_UTIL.createTable(tableName, Bytes.toBytes("cf"));
-
+      HTable t = createTableAndWait(tableName, Bytes.toBytes("cf"));
       final List<HRegion> regions = cluster.getRegions(tableName);
       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
@@ -200,8 +199,7 @@ public void run() {
       Bytes.toBytes("ephemeral");
 
     // Create table then get the single region for our new table.
-    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-
+    HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
 
@@ -271,8 +269,7 @@ public void run() {
       Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
 
     // Create table then get the single region for our new table.
-    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-
+    HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
 
@@ -330,8 +327,7 @@ public void run() {
     final byte [] tableName = Bytes.toBytes("testShutdownSimpleFixup");
 
     // Create table then get the single region for our new table.
-    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-
+    HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
 
@@ -383,8 +379,7 @@ public void run() {
       Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
 
     // Create table then get the single region for our new table.
-    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-
+    HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
 
@@ -459,8 +454,7 @@ public void testMasterRestartWhenSplittingIsPartial()
     final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
 
     // Create table then get the single region for our new table.
-    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-
+    HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
 
@@ -537,8 +531,7 @@ public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
 
     // Create table then get the single region for our new table.
     this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-    HTable t = TESTING_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
-
+    HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
 
@@ -739,8 +732,7 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
     HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
     try {
       // Create table then get the single region for our new table.
-      HTable t = TESTING_UTIL.createTable(tableName, Bytes.toBytes("cf"));
-
+      HTable t = createTableAndWait(tableName, Bytes.toBytes("cf"));
       regions = cluster.getRegions(tableName);
       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
       regionServer = cluster.getRegionServer(regionServerIndex);
@@ -1030,6 +1022,16 @@ private void awaitDaughters(byte[] tableName, int numDaughters) throws Interrupt
     }
   }
 
+  private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
+      InterruptedException {
+    HTable t = TESTING_UTIL.createTable(tableName, cf);
+    for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
+      Thread.sleep(100);
+    }
+    assertTrue("Table not online: "+Bytes.toString(tableName), cluster.getRegions(tableName).size() != 0);
+    return t;
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From a9dd10389cab6d666ff61c993df2301e1a330129 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 31 Jan 2013 22:43:30 +0000
Subject: [PATCH 0752/1540] HBASE-7729
 TestCatalogTrackerOnCluster.testbadOriginalRootLocation fails occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441237 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java
index fe371565a0f8..5b53c47e1095 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java
@@ -46,6 +46,9 @@ public class TestCatalogTrackerOnCluster {
     UTIL.startMiniCluster();
     // Shutdown hbase.
     UTIL.shutdownMiniHBaseCluster();
+    // Give the various ZKWatchers some time to settle their affairs.
+    Thread.sleep(1000);
+
     // Mess with the root location in the running zk.  Set it to be nonsense.
     ZooKeeperWatcher zookeeper = new ZooKeeperWatcher(UTIL.getConfiguration(),
       "Bad Root Location Writer", new Abortable() {

From bc3fc8de5d90a598494597b0c8156710b95146fc Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 31 Jan 2013 23:11:01 +0000
Subject: [PATCH 0753/1540] HBASE-7730 HBaseAdmin#synchronousBalanceSwitch is
 not compatible with 0.92

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441252 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 22 +++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index ba3be8e996a7..fd311865037a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -23,6 +23,7 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.LinkedList;
@@ -89,7 +90,9 @@ public class HBaseAdmin implements Abortable, Closeable {
   // want to wait a long time.
   private final int retryLongerMultiplier;
   private boolean aborted;
-  
+
+  private static volatile boolean synchronousBalanceSwitchSupported = true;
+
   /**
    * Constructor
    *
@@ -1477,10 +1480,21 @@ public boolean balanceSwitch(final boolean b)
    */
   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
   throws MasterNotRunningException, ZooKeeperConnectionException {
-    if (synchronous == false) {
-      return balanceSwitch(on);
+    if (synchronous && synchronousBalanceSwitchSupported) {
+      try {
+        return getMaster().synchronousBalanceSwitch(on);
+      } catch (UndeclaredThrowableException ute) {
+        String error = ute.getCause().getMessage();
+        if (error != null && error.matches(
+            "(?s).+NoSuchMethodException:.+synchronousBalanceSwitch.+")) {
+          LOG.info("HMaster doesn't support synchronousBalanceSwitch");
+          synchronousBalanceSwitchSupported = false;
+        } else {
+          throw ute;
+        }
+      }
     }
-    return getMaster().synchronousBalanceSwitch(on);
+    return balanceSwitch(on);
   }
 
   /**

From 24cd23c30f8b5f9ac32ee693f84cc2a15db58f38 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 1 Feb 2013 21:24:56 +0000
Subject: [PATCH 0754/1540] HBASE-7728 deadlock occurs between hlog roller and
 hlog syncer (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441637 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/wal/HLog.java | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index cbe3e251be0c..8496295fa144 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1303,16 +1303,24 @@ private void syncer(long txid) throws IOException {
       // issue the sync to HDFS. If sync is successful, then update
       // syncedTillHere to indicate that transactions till this
       // number has been successfully synced.
+      IOException ioe = null;
+      List<Entry> pending = null;
       synchronized (flushLock) {
         if (txid <= this.syncedTillHere) {
           return;
         }
         doneUpto = this.unflushedEntries.get();
-        List<Entry> pending = logSyncerThread.getPendingWrites();
+        pending = logSyncerThread.getPendingWrites();
         try {
           logSyncerThread.hlogFlush(tempWriter, pending);
         } catch(IOException io) {
-          synchronized (this.updateLock) {
+          ioe = io;
+          LOG.error("syncer encountered error, will retry. txid=" + txid, ioe);
+        }
+      }
+      if (ioe != null && pending != null) {
+        synchronized (this.updateLock) {
+          synchronized (flushLock) {
             // HBASE-4387, HBASE-5623, retry with updateLock held
             tempWriter = this.writer;
             logSyncerThread.hlogFlush(tempWriter, pending);
@@ -1325,7 +1333,7 @@ private void syncer(long txid) throws IOException {
       }
       try {
         tempWriter.sync();
-      } catch(IOException io) {
+      } catch (IOException io) {
         synchronized (this.updateLock) {
           // HBASE-4387, HBASE-5623, retry with updateLock held
           tempWriter = this.writer;

From df7216baa621a51002c83043868c5c6066af5479 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 1 Feb 2013 22:15:15 +0000
Subject: [PATCH 0755/1540] HBASE-7731 Append/Increment methods in HRegion
 don't check whether the table is readonly or not (Devaraj)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441648 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  2 +
 .../hbase/regionserver/TestHRegion.java       | 58 ++++++++++++++++++-
 2 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c79f9a9e0c84..0580e3875f84 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4809,6 +4809,7 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
     long size = 0;
     long txid = 0;
 
+    checkReadOnly();
     // Lock row
     startRegionOperation();
     this.writeRequestsCount.increment();
@@ -4978,6 +4979,7 @@ public Result increment(Increment increment, Integer lockid,
     long size = 0;
     long txid = 0;
 
+    checkReadOnly();
     // Lock row
     startRegionOperation();
     this.writeRequestsCount.increment();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 9b3141f87861..dec7fc557ed2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -502,6 +502,41 @@ public void testWeirdCacheBehaviour() throws Exception {
     }
   }
 
+  public void testAppendWithReadOnlyTable() throws Exception {
+    byte[] TABLE = Bytes.toBytes("readOnlyTable");
+    this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
+    boolean exceptionCaught = false;
+    Append append = new Append(Bytes.toBytes("somerow"));
+    append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 
+        Bytes.toBytes("somevalue"));
+    try {
+      region.append(append, false);
+    } catch (IOException e) {
+      exceptionCaught = true;
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+    assertTrue(exceptionCaught == true);
+  }
+
+  public void testIncrWithReadOnlyTable() throws Exception {
+    byte[] TABLE = Bytes.toBytes("readOnlyTable");
+    this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
+    boolean exceptionCaught = false;    
+    Increment inc = new Increment(Bytes.toBytes("somerow"));
+    inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
+    try {
+      region.increment(inc, false);
+    } catch (IOException e) {
+      exceptionCaught = true;
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+    assertTrue(exceptionCaught == true);
+  }
+
   private void deleteColumns(HRegion r, String value, String keyPrefix)
   throws IOException {
     InternalScanner scanner = buildScanner(keyPrefix, value, r);
@@ -3525,7 +3560,7 @@ public void testHolesInMeta() throws Exception {
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
-        conf, family);
+        conf, false, family);
     try {
       byte[] rowNotServed = Bytes.toBytes("a");
       Get g = new Get(rowNotServed);
@@ -4204,7 +4239,22 @@ private Configuration initSplit() {
   public static HRegion initHRegion (byte [] tableName, String callingMethod,
       Configuration conf, byte [] ... families)
     throws IOException{
-    return initHRegion(tableName, null, null, callingMethod, conf, families);
+    return initHRegion(tableName, null, null, callingMethod, conf, false, families);
+  }
+
+  /**
+   * @param tableName
+   * @param callingMethod
+   * @param conf
+   * @param isReadOnly
+   * @param families
+   * @throws IOException
+   * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
+   */
+  public static HRegion initHRegion (byte [] tableName, String callingMethod,
+      Configuration conf, boolean isReadOnly, byte [] ... families)
+    throws IOException{
+    return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
   }
 
   /**
@@ -4213,14 +4263,16 @@ public static HRegion initHRegion (byte [] tableName, String callingMethod,
    * @param stopKey
    * @param callingMethod
    * @param conf
+   * @param isReadOnly
    * @param families
    * @throws IOException
    * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
    */
   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
-      String callingMethod, Configuration conf, byte[]... families)
+      String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
       throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.setReadOnly(isReadOnly);
     for(byte [] family : families) {
       htd.addFamily(new HColumnDescriptor(family));
     }

From d73cde21d95d5a850110901bab85a03668eda38e Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 2 Feb 2013 00:31:40 +0000
Subject: [PATCH 0756/1540] HBASE-7738 REST server should publish metrics that
 are available via HTTP

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441667 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/Main.java    | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index 2bde21820454..2f5639f8498a 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -34,6 +34,8 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.jmx.JMXJsonServlet;
+import org.apache.hadoop.metrics.MetricsServlet;
 import org.apache.hadoop.net.DNS;
 
 import java.util.List;
@@ -57,6 +59,7 @@
  * <li>-ro --readonly : server mode</li>
  * </ul>
  */
+@SuppressWarnings("deprecation")
 public class Main implements Constants {
 
   private static void printUsageAndExit(Options options, int exitCode) {
@@ -154,12 +157,27 @@ public static void main(String[] args) throws Exception {
     server.setSendServerVersion(false);
     server.setSendDateHeader(false);
     server.setStopAtShutdown(true);
+
+    // set up the JMX servlet container for Jetty
+    ServletHolder jmx = new ServletHolder(JMXJsonServlet.class);
+    // set up the metrics servlet container for Jetty
+    ServletHolder metrics = new ServletHolder(MetricsServlet.class);
+
+    String metricsPath =
+      servlet.getConfiguration().get("hbase.rest.path.metrics", "/metrics");
+    String jmxPath =
+      servlet.getConfiguration().get("hbase.rest.path.jmx", "/jmx");
+
       // set up context
     Context context = new Context(server, "/", Context.SESSIONS);
+    context.addServlet(metrics, metricsPath);
+    context.addServlet(jmx, jmxPath);
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
 
-    // login the server principal (if using secure Hadoop)   
+    context.getServletContext().setAttribute("hadoop.conf", conf);
+
+    // login the server principal (if using secure Hadoop)
     if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
       String machineName = Strings.domainNamePointerToHostName(
         DNS.getDefaultHost(conf.get("hbase.rest.dns.interface", "default"),

From eee15e0be4b8d5981ad3c4df64a6e975387691ed Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 2 Feb 2013 02:22:58 +0000
Subject: [PATCH 0757/1540] HBASE-7740 Recheck matching row for joined scanners
 (Sergey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441686 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 0580e3875f84..0bed5816bea6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3793,10 +3793,11 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           if (this.joinedHeap != null) {
             KeyValue nextJoinedKv = joinedHeap.peek();
             // If joinedHeap is pointing to some other row, try to seek to a correct one.
-            // We don't need to recheck that row here - populateResult will take care of that.
             boolean mayHaveData =
               (nextJoinedKv != null && nextJoinedKv.matchingRow(currentRow, offset, length))
-              || this.joinedHeap.seek(KeyValue.createFirstOnRow(currentRow, offset, length));
+              || (this.joinedHeap.seek(KeyValue.createFirstOnRow(currentRow, offset, length))
+                  && joinedHeap.peek() != null
+                  && joinedHeap.peek().matchingRow(currentRow, offset, length));
             if (mayHaveData) {
               joinedContinuationRow = current;
               populateFromJoinedHeap(limit, metric);

From 60efed290475a2d7923a8b352074f1696a4f8234 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 2 Feb 2013 04:46:48 +0000
Subject: [PATCH 0758/1540] HBASE-5664 CP hooks in Scan flow for fast forward
 when filter filters out a row (Anoop Sam John)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441701 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/BaseRegionObserver.java |  6 ++++
 .../hbase/coprocessor/RegionObserver.java     | 21 +++++++++++++
 .../hadoop/hbase/regionserver/HRegion.java    | 31 +++++++++++++------
 .../regionserver/RegionCoprocessorHost.java   | 29 +++++++++++++++++
 4 files changed, 77 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index abc972ce7113..0c8034e09ea7 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -290,6 +290,12 @@ public boolean postScannerNext(final ObserverContext<RegionCoprocessorEnvironmen
     return hasMore;
   }
 
+  @Override
+  public boolean postScannerFilterRow(final ObserverContext<RegionCoprocessorEnvironment> e,
+      final InternalScanner s, final byte[] currentRow, final boolean hasMore) throws IOException {
+    return hasMore;
+  }
+  
   @Override
   public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> e,
       final InternalScanner s) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index fa65fed45862..57f4c8fc38d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -696,6 +696,27 @@ boolean postScannerNext(final ObserverContext<RegionCoprocessorEnvironment> c,
       final boolean hasNext)
     throws IOException;
 
+  /**
+   * This will be called by the scan flow when the current scanned row is being filtered out by the
+   * filter. The filter may be filtering out the row via any of the below scenarios
+   * <ol>
+   * <li>
+   * <code>boolean filterRowKey(byte [] buffer, int offset, int length)</code> returning true</li>
+   * <li>
+   * <code>boolean filterRow()</code> returning true</li>
+   * <li>
+   * <code>void filterRow(List<KeyValue> kvs)</code> removing all the kvs from the passed List</li>
+   * </ol>
+   * @param c the environment provided by the region server
+   * @param s the scanner
+   * @param currentRow The current rowkey which got filtered out
+   * @param hasMore the 'has more' indication
+   * @return whether more rows are available for the scanner or not
+   * @throws IOException
+   */
+  boolean postScannerFilterRow(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final InternalScanner s, final byte[] currentRow, final boolean hasMore) throws IOException;
+  
   /**
    * Called before the client closes a scanner.
    * <p>
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 0bed5816bea6..18e9822f8f88 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1710,7 +1710,7 @@ protected RegionScanner getScanner(Scan scan,
 
   protected RegionScanner instantiateRegionScanner(Scan scan,
       List<KeyValueScanner> additionalScanners) throws IOException {
-    return new RegionScannerImpl(scan, additionalScanners);
+    return new RegionScannerImpl(scan, additionalScanners, this);
   }
 
   /*
@@ -3525,13 +3525,16 @@ class RegionScannerImpl implements RegionScanner {
     private int isScan;
     private boolean filterClosed = false;
     private long readPt;
+    private HRegion region;
 
     public HRegionInfo getRegionInfo() {
       return regionInfo;
     }
-    RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
-      //DebugPrint.println("HRegionScanner.<init>");
-
+    
+    RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region)
+        throws IOException {
+      // DebugPrint.println("HRegionScanner.<init>");
+      this.region = region;
       this.filter = scan.getFilter();
       this.batch = scan.getBatch();
       if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
@@ -3582,8 +3585,8 @@ public HRegionInfo getRegionInfo() {
       }
     }
 
-    RegionScannerImpl(Scan scan) throws IOException {
-      this(scan, null);
+    RegionScannerImpl(Scan scan, HRegion region) throws IOException {
+      this(scan, null, region);
     }
 
     @Override
@@ -3749,7 +3752,8 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           // Check if rowkey filter wants to exclude this row. If so, loop to next.
           // Techically, if we hit limits before on this row, we don't need this call.
           if (filterRowKey(currentRow, offset, length)) {
-            nextRow(currentRow, offset, length);
+            boolean moreRows = nextRow(currentRow, offset, length);
+            if (!moreRows) return false;
             continue;
           }
 
@@ -3778,7 +3782,8 @@ private boolean nextInternal(int limit, String metric) throws IOException {
             // the reasons for calling this method are:
             // 1. reset the filters.
             // 2. provide a hook to fast forward the row (used by subclasses)
-            nextRow(currentRow, offset, length);
+            boolean moreRows = nextRow(currentRow, offset, length);
+            if (!moreRows) return false;
 
             // This row was totally filtered out, if this is NOT the last row,
             // we should continue on. Otherwise, nothing else to do.
@@ -3818,7 +3823,8 @@ private boolean nextInternal(int limit, String metric) throws IOException {
         // Double check to prevent empty rows from appearing in result. It could be
         // the case when SingleValueExcludeFilter is used.
         if (results.isEmpty()) {
-          nextRow(currentRow, offset, length);
+          boolean moreRows = nextRow(currentRow, offset, length);
+          if (!moreRows) return false;
           if (!stopRow) continue;
         }
 
@@ -3836,13 +3842,18 @@ private boolean filterRowKey(byte[] row, int offset, short length) {
           && filter.filterRowKey(row, offset, length);
     }
 
-    protected void nextRow(byte [] currentRow, int offset, short length) throws IOException {
+    protected boolean nextRow(byte [] currentRow, int offset, short length) throws IOException {
       KeyValue next;
       while((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
         this.storeHeap.next(MOCKED_LIST);       
       }
       results.clear();
       resetFilters();
+      // Calling the hook in CP which allows it to do a fast forward
+      if (this.region.getCoprocessorHost() != null) {
+        return this.region.getCoprocessorHost().postScannerFilterRow(this, currentRow);
+      }
+      return true;
     }
 
     private boolean isStopRow(byte [] currentRow, int offset, short length) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index e9a4084bb63d..7934a932fc9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1329,6 +1329,35 @@ public boolean postScannerNext(final InternalScanner s,
     return hasMore;
   }
 
+  /**
+   * This will be called by the scan flow when the current scanned row is being filtered out by the
+   * filter.
+   * @param s the scanner
+   * @param currentRow The current rowkey which got filtered out
+   * @return whether more rows are available for the scanner or not
+   * @throws IOException
+   */
+  public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow)
+      throws IOException {
+    boolean hasMore = true; // By default assume more rows there.
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          hasMore = ((RegionObserver) env.getInstance()).postScannerFilterRow(ctx, s, currentRow,
+              hasMore);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return hasMore;
+  }
+ 
   /**
    * @param s the scanner
    * @return true if default behavior should be bypassed, false otherwise

From c5351dae1d1e920096f348f5aab10a4708d8d355 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sat, 2 Feb 2013 05:59:39 +0000
Subject: [PATCH 0759/1540] HBASE-7738 REST server should publish metrics that
 are available via HTTP - ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441703 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/Main.java    | 33 +++++++++++--------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index 2f5639f8498a..412d73f7ebc9 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -158,24 +158,31 @@ public static void main(String[] args) throws Exception {
     server.setSendDateHeader(false);
     server.setStopAtShutdown(true);
 
-    // set up the JMX servlet container for Jetty
-    ServletHolder jmx = new ServletHolder(JMXJsonServlet.class);
-    // set up the metrics servlet container for Jetty
-    ServletHolder metrics = new ServletHolder(MetricsServlet.class);
-
-    String metricsPath =
-      servlet.getConfiguration().get("hbase.rest.path.metrics", "/metrics");
-    String jmxPath =
-      servlet.getConfiguration().get("hbase.rest.path.jmx", "/jmx");
-
       // set up context
     Context context = new Context(server, "/", Context.SESSIONS);
-    context.addServlet(metrics, metricsPath);
-    context.addServlet(jmx, jmxPath);
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
 
-    context.getServletContext().setAttribute("hadoop.conf", conf);
+    // Disable the JMX and metrics servlet by default so that
+    // not to confuse existing applications use jmx/metrics as table name
+    if (servlet.getConfiguration().getBoolean(
+        "hbase.rest.enable.jmx_metrics", false)) {
+
+      // set up the JMX servlet container for Jetty
+      ServletHolder jmx = new ServletHolder(JMXJsonServlet.class);
+      // set up the metrics servlet container for Jetty
+      ServletHolder metrics = new ServletHolder(MetricsServlet.class);
+
+      String metricsPath =
+        servlet.getConfiguration().get("hbase.rest.path.metrics", "/metrics");
+      String jmxPath =
+        servlet.getConfiguration().get("hbase.rest.path.jmx", "/jmx");
+
+      context.addServlet(metrics, metricsPath);
+      context.addServlet(jmx, jmxPath);
+
+      context.getServletContext().setAttribute("hadoop.conf", conf);
+    }
 
     // login the server principal (if using secure Hadoop)
     if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {

From d5c2a4692556c6130c865485dcdc1950af09a98d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 2 Feb 2013 07:33:38 +0000
Subject: [PATCH 0760/1540] HBASE-3996 Support multiple tables and scanners as
 input to the mapper in map/reduce jobs (Eran Kutner, Bryan Baugher)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441709 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/Scan.java  |  11 +-
 .../mapreduce/MultiTableInputFormat.java      | 106 ++++++++
 .../mapreduce/MultiTableInputFormatBase.java  | 216 +++++++++++++++
 .../hbase/mapreduce/TableMapReduceUtil.java   |  64 ++++-
 .../hadoop/hbase/mapreduce/TableSplit.java    | 110 +++++++-
 .../mapreduce/TestMultiTableInputFormat.java  | 254 ++++++++++++++++++
 6 files changed, 749 insertions(+), 12 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 458f6b807912..0518719826f8 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -92,10 +92,13 @@ public class Scan extends OperationWithAttributes implements Writable {
   private int batch = -1;
   // If application wants to collect scan metrics, it needs to
   // call scan.setAttribute(SCAN_ATTRIBUTES_ENABLE, Bytes.toBytes(Boolean.TRUE))
-  static public String SCAN_ATTRIBUTES_METRICS_ENABLE =
-    "scan.attributes.metrics.enable";
-  static public String SCAN_ATTRIBUTES_METRICS_DATA =
-    "scan.attributes.metrics.data";
+  static public String SCAN_ATTRIBUTES_METRICS_ENABLE = "scan.attributes.metrics.enable";
+  static public String SCAN_ATTRIBUTES_METRICS_DATA = "scan.attributes.metrics.data";
+  
+  // If an application wants to use multiple scans over different tables each scan must
+  // define this attribute with the appropriate table name by calling
+  // scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName))
+  static public final String SCAN_ATTRIBUTES_TABLE_NAME = "scan.attributes.table.name";
 
   /*
    * -1 means no caching
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
new file mode 100644
index 000000000000..3a4ceab73a19
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * Convert HBase tabular data from multiple scanners into a format that 
+ * is consumable by Map/Reduce.
+ *
+ * <p>
+ * Usage example
+ * </p>
+ *
+ * <pre>
+ * List<Scan> scans = new ArrayList<Scan>();
+ * 
+ * Scan scan1 = new Scan();
+ * scan1.setStartRow(firstRow1);
+ * scan1.setStopRow(lastRow1);
+ * scan1.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, table1);
+ * scans.add(scan1);
+ *
+ * Scan scan2 = new Scan();
+ * scan2.setStartRow(firstRow2);
+ * scan2.setStopRow(lastRow2);
+ * scan1.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, table2);
+ * scans.add(scan2);
+ *
+ * TableMapReduceUtil.initTableMapperJob(scans, TableMapper.class, Text.class,
+ *     IntWritable.class, job);
+ * </pre>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class MultiTableInputFormat extends MultiTableInputFormatBase implements
+    Configurable {
+
+  /** Job parameter that specifies the scan list. */
+  public static final String SCANS = "hbase.mapreduce.scans";
+
+  /** The configuration. */
+  private Configuration conf = null;
+
+  /**
+   * Returns the current configuration.
+   *
+   * @return The current configuration.
+   * @see org.apache.hadoop.conf.Configurable#getConf()
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Sets the configuration. This is used to set the details for the tables to
+   *  be scanned.
+   *
+   * @param configuration The configuration to set.
+   * @see org.apache.hadoop.conf.Configurable#setConf(
+   *        org.apache.hadoop.conf.Configuration)
+   */
+  @Override
+  public void setConf(Configuration configuration) {
+    this.conf = configuration;
+    String[] rawScans = conf.getStrings(SCANS);
+    if (rawScans.length <= 0) {
+      throw new IllegalArgumentException("There must be at least 1 scan configuration set to : "
+          + SCANS);
+    }
+    List<Scan> scans = new ArrayList<Scan>();
+
+    for (int i = 0; i < rawScans.length; i++) {
+      try {
+        scans.add(TableMapReduceUtil.convertStringToScan(rawScans[i]));
+      } catch (IOException e) {
+        throw new RuntimeException("Failed to convert Scan : " + rawScans[i] + " to string", e);
+      }
+    }
+    this.setScans(scans);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
new file mode 100644
index 000000000000..76a1632bc192
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * A base for {@link MultiTableInputFormat}s. Receives a list of
+ * {@link Scan} instances that define the input tables and
+ * filters etc. Subclasses may use other TableRecordReader implementations.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class MultiTableInputFormatBase extends
+    InputFormat<ImmutableBytesWritable, Result> {
+
+  final Log LOG = LogFactory.getLog(MultiTableInputFormatBase.class);
+
+  /** Holds the set of scans used to define the input. */
+  private List<Scan> scans;
+
+  /** The reader scanning the table, can be a custom one. */
+  private TableRecordReader tableRecordReader = null;
+
+  /**
+   * Builds a TableRecordReader. If no TableRecordReader was provided, uses the
+   * default.
+   *
+   * @param split The split to work with.
+   * @param context The current context.
+   * @return The newly created record reader.
+   * @throws IOException When creating the reader fails.
+   * @throws InterruptedException when record reader initialization fails
+   * @see org.apache.hadoop.mapreduce.InputFormat#createRecordReader(
+   *      org.apache.hadoop.mapreduce.InputSplit,
+   *      org.apache.hadoop.mapreduce.TaskAttemptContext)
+   */
+  @Override
+  public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
+      InputSplit split, TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    TableSplit tSplit = (TableSplit) split;
+
+    if (tSplit.getTableName() == null) {
+      throw new IOException("Cannot create a record reader because of a"
+          + " previous error. Please look at the previous logs lines from"
+          + " the task's full log for more details.");
+    }
+    HTable table =
+        new HTable(context.getConfiguration(), tSplit.getTableName());
+
+    TableRecordReader trr = this.tableRecordReader;
+    // if no table record reader was provided use default
+    if (trr == null) {
+      trr = new TableRecordReader();
+    }
+    Scan sc = tSplit.getScan();
+    sc.setStartRow(tSplit.getStartRow());
+    sc.setStopRow(tSplit.getEndRow());
+    trr.setScan(sc);
+    trr.setHTable(table);
+    trr.initialize(split, context);
+    return trr;
+  }
+
+  /**
+   * Calculates the splits that will serve as input for the map tasks. The
+   * number of splits matches the number of regions in a table.
+   *
+   * @param context The current job context.
+   * @return The list of input splits.
+   * @throws IOException When creating the list of splits fails.
+   * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)
+   */
+  @Override
+  public List<InputSplit> getSplits(JobContext context) throws IOException {
+    if (scans.isEmpty()) {
+      throw new IOException("No scans were provided.");
+    }
+    List<InputSplit> splits = new ArrayList<InputSplit>();
+
+    for (Scan scan : scans) {
+      byte[] tableName = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
+      if (tableName == null) 
+        throw new IOException("A scan object did not have a table name");
+      HTable table = new HTable(context.getConfiguration(), tableName);
+      Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
+      if (keys == null || keys.getFirst() == null ||
+          keys.getFirst().length == 0) {
+        throw new IOException("Expecting at least one region for table : "
+            + Bytes.toString(tableName));
+      }
+      int count = 0;
+      
+      byte[] startRow = scan.getStartRow();
+      byte[] stopRow = scan.getStopRow();
+
+      for (int i = 0; i < keys.getFirst().length; i++) {
+        if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+          continue;
+        }
+        String regionLocation =
+            table.getRegionLocation(keys.getFirst()[i], false).getHostname();
+        
+        // determine if the given start and stop keys fall into the range
+        if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+            Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+            (stopRow.length == 0 ||
+                Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
+          byte[] splitStart =
+              startRow.length == 0 ||
+                  Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
+                  .getFirst()[i] : startRow;
+          byte[] splitStop =
+              (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
+                  stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
+                  .getSecond()[i] : stopRow;
+          InputSplit split =
+              new TableSplit(tableName, scan, splitStart,
+                  splitStop, regionLocation);
+          splits.add(split);
+          if (LOG.isDebugEnabled())
+            LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+        }
+      }
+      table.close();
+    }
+    return splits;
+  }
+
+  /**
+   * Test if the given region is to be included in the InputSplit while
+   * splitting the regions of a table.
+   * <p>
+   * This optimization is effective when there is a specific reasoning to
+   * exclude an entire region from the M-R job, (and hence, not contributing to
+   * the InputSplit), given the start and end keys of the same. <br>
+   * Useful when we need to remember the last-processed top record and revisit
+   * the [last, current) interval for M-R processing, continuously. In addition
+   * to reducing InputSplits, reduces the load on the region server as well, due
+   * to the ordering of the keys. <br>
+   * <br>
+   * Note: It is possible that <code>endKey.length() == 0 </code> , for the last
+   * (recent) region. <br>
+   * Override this method, if you want to bulk exclude regions altogether from
+   * M-R. By default, no region is excluded( i.e. all regions are included).
+   *
+   * @param startKey Start key of the region
+   * @param endKey End key of the region
+   * @return true, if this region needs to be included as part of the input
+   *         (default).
+   */
+  protected boolean includeRegionInSplit(final byte[] startKey,
+      final byte[] endKey) {
+    return true;
+  }
+
+  /**
+   * Allows subclasses to get the list of {@link Scan} objects.
+   */
+  protected List<Scan> getScans() {
+    return this.scans;
+  }
+
+  /**
+   * Allows subclasses to set the list of {@link Scan} objects.
+   *
+   * @param scans The list of {@link Scan} used to define the input
+   */
+  protected void setScans(List<Scan> scans) {
+    this.scans = scans;
+  }
+
+  /**
+   * Allows subclasses to set the {@link TableRecordReader}.
+   *
+   * @param tableRecordReader A different {@link TableRecordReader}
+   *          implementation.
+   */
+  protected void setTableRecordReader(TableRecordReader tableRecordReader) {
+    this.tableRecordReader = tableRecordReader;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 3d06e07f8f27..29ebff5fb82d 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -28,8 +28,10 @@
 import java.lang.reflect.InvocationTargetException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
@@ -46,7 +48,6 @@
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.InputFormat;
@@ -215,6 +216,67 @@ public static void initTableMapperJob(String table, Scan scan,
       initTableMapperJob(table, scan, mapper, outputKeyClass,
               outputValueClass, job, addDependencyJars, TableInputFormat.class);
   }
+  
+  /**
+   * Use this before submitting a Multi TableMap job. It will appropriately set
+   * up the job.
+   *
+   * @param scans The list of {@link Scan} objects to read from.
+   * @param mapper The mapper class to use.
+   * @param outputKeyClass The class of the output key.
+   * @param outputValueClass The class of the output value.
+   * @param job The current job to adjust. Make sure the passed job is carrying
+   *          all necessary HBase configuration.
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(List<Scan> scans,
+      Class<? extends TableMapper> mapper,
+      Class<? extends WritableComparable> outputKeyClass,
+      Class<? extends Writable> outputValueClass, Job job) throws IOException {
+    initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
+        true);
+  }
+
+  /**
+   * Use this before submitting a Multi TableMap job. It will appropriately set
+   * up the job.
+   *
+   * @param scans The list of {@link Scan} objects to read from.
+   * @param mapper The mapper class to use.
+   * @param outputKeyClass The class of the output key.
+   * @param outputValueClass The class of the output value.
+   * @param job The current job to adjust. Make sure the passed job is carrying
+   *          all necessary HBase configuration.
+   * @param addDependencyJars upload HBase jars and jars for any of the
+   *          configured job classes via the distributed cache (tmpjars).
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(List<Scan> scans,
+      Class<? extends TableMapper> mapper,
+      Class<? extends WritableComparable> outputKeyClass,
+      Class<? extends Writable> outputValueClass, Job job,
+      boolean addDependencyJars) throws IOException {
+    job.setInputFormatClass(MultiTableInputFormat.class);
+    if (outputValueClass != null) {
+      job.setMapOutputValueClass(outputValueClass);
+    }
+    if (outputKeyClass != null) {
+      job.setMapOutputKeyClass(outputKeyClass);
+    }
+    job.setMapperClass(mapper);
+    HBaseConfiguration.addHbaseResources(job.getConfiguration());
+    List<String> scanStrings = new ArrayList<String>();
+
+    for (Scan scan : scans) {
+      scanStrings.add(convertScanToString(scan));
+    }
+    job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
+      scanStrings.toArray(new String[scanStrings.size()]));
+
+    if (addDependencyJars) {
+      addDependencyJars(job);
+    }
+  }
 
   public static void initCredentials(Job job) throws IOException {
     if (User.isHBaseSecurityEnabled(job.getConfiguration())) {
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
index cfc232f6615b..bb9fb46cf382 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
@@ -24,26 +24,65 @@
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapreduce.InputSplit;
 
 /**
- * A table split corresponds to a key range (low, high). All references to row
- * below refer to the key of the row.
+ * A table split corresponds to a key range (low, high) and an optional scanner.
+ * All references to row below refer to the key of the row.
  */
 public class TableSplit extends InputSplit
-implements Writable, Comparable<TableSplit> {
+implements Writable, Comparable<TableSplit> { 
+  public static final Log LOG = LogFactory.getLog(TableSplit.class);
+
+  // should be < 0 (@see #readFields(DataInput))
+  // version 1 supports Scan data member
+  enum Version {
+    UNVERSIONED(0),
+    // Initial number we put on TableSplit when we introduced versioning.
+    INITIAL(-1);
+
+    final int code;
+    static final Version[] byCode;
+    static {
+      byCode = Version.values();
+      for (int i = 0; i < byCode.length; i++) {
+        if (byCode[i].code != -1 * i) {
+          throw new AssertionError("Values in this enum should be descending by one");
+        }
+      }
+    }
+
+    Version(int code) {
+      this.code = code;
+    }
+
+    boolean atLeast(Version other) {
+      return code <= other.code;
+    }
+
+    static Version fromCode(int code) {
+      return byCode[code * -1];
+    }
+  }
+
+  private static final Version VERSION = Version.INITIAL;
 
   private byte [] tableName;
   private byte [] startRow;
   private byte [] endRow;
   private String regionLocation;
+  private String scan = ""; // stores the serialized form of the Scan
 
   /** Default constructor. */
   public TableSplit() {
-    this(HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
+    this(HConstants.EMPTY_BYTE_ARRAY, null, HConstants.EMPTY_BYTE_ARRAY,
       HConstants.EMPTY_BYTE_ARRAY, "");
   }
 
@@ -51,17 +90,47 @@ public TableSplit() {
    * Creates a new instance while assigning all variables.
    *
    * @param tableName  The name of the current table.
+   * @param scan The scan associated with this split.
    * @param startRow  The start row of the split.
    * @param endRow  The end row of the split.
    * @param location  The location of the region.
    */
-  public TableSplit(byte [] tableName, byte [] startRow, byte [] endRow,
+  public TableSplit(byte [] tableName, Scan scan, byte [] startRow, byte [] endRow,
       final String location) {
     this.tableName = tableName;
+    try {
+      this.scan =
+        (null == scan) ? "" : TableMapReduceUtil.convertScanToString(scan);
+    } catch (IOException e) {
+      LOG.warn("Failed to convert Scan to String", e);
+    }
     this.startRow = startRow;
     this.endRow = endRow;
     this.regionLocation = location;
   }
+  
+  /**
+   * Creates a new instance without a scanner.
+   *
+   * @param tableName The name of the current table.
+   * @param startRow The start row of the split.
+   * @param endRow The end row of the split.
+   * @param location The location of the region.
+   */
+  public TableSplit(byte[] tableName, byte[] startRow, byte[] endRow,
+      final String location) {
+    this(tableName, null, startRow, endRow, location);
+  }
+
+  /**
+   * Returns a Scan object from the stored string representation.
+   *
+   * @return Returns a Scan object based on the stored scanner.
+   * @throws IOException
+   */
+  public Scan getScan() throws IOException {
+    return TableMapReduceUtil.convertStringToScan(this.scan);
+  }
 
   /**
    * Returns the table name.
@@ -130,10 +199,29 @@ public long getLength() {
    */
   @Override
   public void readFields(DataInput in) throws IOException {
-    tableName = Bytes.readByteArray(in);
+    Version version = Version.UNVERSIONED;
+    // TableSplit was not versioned in the beginning.
+    // In order to introduce it now, we make use of the fact
+    // that tableName was written with Bytes.writeByteArray,
+    // which encodes the array length as a vint which is >= 0.
+    // Hence if the vint is >= 0 we have an old version and the vint
+    // encodes the length of tableName.
+    // If < 0 we just read the version and the next vint is the length.
+    // @see Bytes#readByteArray(DataInput)
+    int len = WritableUtils.readVInt(in);
+    if (len < 0) {
+      // what we just read was the version
+      version = Version.fromCode(len);
+      len = WritableUtils.readVInt(in);
+    }
+    tableName = new byte[len];
+    in.readFully(tableName);
     startRow = Bytes.readByteArray(in);
     endRow = Bytes.readByteArray(in);
     regionLocation = Bytes.toString(Bytes.readByteArray(in));
+    if (version.atLeast(Version.INITIAL)) {
+      scan = Bytes.toString(Bytes.readByteArray(in));
+    }
   }
 
   /**
@@ -144,10 +232,12 @@ public void readFields(DataInput in) throws IOException {
    */
   @Override
   public void write(DataOutput out) throws IOException {
+    WritableUtils.writeVInt(out, VERSION.code);
     Bytes.writeByteArray(out, tableName);
     Bytes.writeByteArray(out, startRow);
     Bytes.writeByteArray(out, endRow);
     Bytes.writeByteArray(out, Bytes.toBytes(regionLocation));
+    Bytes.writeByteArray(out, Bytes.toBytes(scan));
   }
 
   /**
@@ -171,7 +261,12 @@ public String toString() {
    */
   @Override
   public int compareTo(TableSplit split) {
-    return Bytes.compareTo(getStartRow(), split.getStartRow());
+    // If The table name of the two splits is the same then compare start row
+    // otherwise compare based on table names
+    int tableNameComparison =
+        Bytes.compareTo(getTableName(), split.getTableName());
+    return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo(
+        getStartRow(), split.getStartRow());
   }
 
   @Override
@@ -188,6 +283,7 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
         int result = tableName != null ? Arrays.hashCode(tableName) : 0;
+        result = 31 * result + (scan != null ? scan.hashCode() : 0);
         result = 31 * result + (startRow != null ? Arrays.hashCode(startRow) : 0);
         result = 31 * result + (endRow != null ? Arrays.hashCode(endRow) : 0);
         result = 31 * result + (regionLocation != null ? regionLocation.hashCode() : 0);
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java
new file mode 100644
index 000000000000..89802f249ad3
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests various scan start and stop row scenarios. This is set in a scan and
+ * tested in a MapReduce job to see if that is handed over and done properly
+ * too.
+ */
+@Category(LargeTests.class)
+public class TestMultiTableInputFormat {
+
+  static final Log LOG = LogFactory.getLog(TestMultiTableInputFormat.class);
+  static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  static final String TABLE_NAME = "scantest";
+  static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
+  static final String KEY_STARTROW = "startRow";
+  static final String KEY_LASTROW = "stpRow";
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // switch TIF to log at DEBUG level
+    TEST_UTIL.enableDebug(MultiTableInputFormat.class);
+    TEST_UTIL.enableDebug(MultiTableInputFormatBase.class);
+    // start mini hbase cluster
+    TEST_UTIL.startMiniCluster(3);
+    // create and fill table
+    for (int i = 0; i < 3; i++) {
+      HTable table =
+          TEST_UTIL.createTable(Bytes.toBytes(TABLE_NAME + String.valueOf(i)),
+              INPUT_FAMILY);
+      TEST_UTIL.createMultiRegions(table, INPUT_FAMILY);
+      TEST_UTIL.loadTable(table, INPUT_FAMILY);
+    }
+    // start MR cluster
+    TEST_UTIL.startMiniMapReduceCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
+    TEST_UTIL.shutdownMiniCluster();
+  }
+  
+  @After
+  public void tearDown() throws Exception {
+    Configuration c = TEST_UTIL.getConfiguration();
+    FileUtil.fullyDelete(new File(c.get("hadoop.tmp.dir")));
+  }
+
+  /**
+   * Pass the key and value to reducer.
+   */
+  public static class ScanMapper extends
+      TableMapper<ImmutableBytesWritable, ImmutableBytesWritable> {
+    /**
+     * Pass the key and value to reduce.
+     *
+     * @param key The key, here "aaa", "aab" etc.
+     * @param value The value is the same as the key.
+     * @param context The task context.
+     * @throws IOException When reading the rows fails.
+     */
+    @Override
+    public void map(ImmutableBytesWritable key, Result value, Context context)
+        throws IOException, InterruptedException {
+      if (value.size() != 1) {
+        throw new IOException("There should only be one input column");
+      }
+      Map<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> cf =
+          value.getMap();
+      if (!cf.containsKey(INPUT_FAMILY)) {
+        throw new IOException("Wrong input columns. Missing: '" +
+            Bytes.toString(INPUT_FAMILY) + "'.");
+      }
+      String val = Bytes.toStringBinary(value.getValue(INPUT_FAMILY, null));
+      LOG.debug("map: key -> " + Bytes.toStringBinary(key.get()) +
+          ", value -> " + val);
+      context.write(key, key);
+    }
+  }
+
+  /**
+   * Checks the last and first keys seen against the scanner boundaries.
+   */
+  public static class ScanReducer
+      extends
+      Reducer<ImmutableBytesWritable, ImmutableBytesWritable,
+      NullWritable, NullWritable> {
+    private String first = null;
+    private String last = null;
+
+    protected void reduce(ImmutableBytesWritable key,
+        Iterable<ImmutableBytesWritable> values, Context context)
+        throws IOException, InterruptedException {
+      int count = 0;
+      for (ImmutableBytesWritable value : values) {
+        String val = Bytes.toStringBinary(value.get());
+        LOG.debug("reduce: key[" + count + "] -> " +
+            Bytes.toStringBinary(key.get()) + ", value -> " + val);
+        if (first == null) first = val;
+        last = val;
+        count++;
+      }
+      assertEquals(3, count);
+    }
+
+    protected void cleanup(Context context) throws IOException,
+        InterruptedException {
+      Configuration c = context.getConfiguration();
+      String startRow = c.get(KEY_STARTROW);
+      String lastRow = c.get(KEY_LASTROW);
+      LOG.info("cleanup: first -> \"" + first + "\", start row -> \"" +
+          startRow + "\"");
+      LOG.info("cleanup: last -> \"" + last + "\", last row -> \"" + lastRow +
+          "\"");
+      if (startRow != null && startRow.length() > 0) {
+        assertEquals(startRow, first);
+      }
+      if (lastRow != null && lastRow.length() > 0) {
+        assertEquals(lastRow, last);
+      }
+    }
+  }
+
+  @Test
+  public void testScanEmptyToEmpty() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, null, null);
+  }
+  
+  @Test
+  public void testScanEmptyToAPP() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, "app", "apo");
+  }
+
+  @Test
+  public void testScanOBBToOPP() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan("obb", "opp", "opo");
+  }
+
+  @Test
+  public void testScanOPPToEmpty() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan("opp", null, "zzz");
+  }
+
+  @Test
+  public void testScanYZYToEmpty() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan("yzy", null, "zzz");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  private void testScan(String start, String stop, String last)
+      throws IOException, InterruptedException, ClassNotFoundException {
+    String jobName =
+        "Scan" + (start != null ? start.toUpperCase() : "Empty") + "To" +
+            (stop != null ? stop.toUpperCase() : "Empty");
+    LOG.info("Before map/reduce startup - job " + jobName);
+    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
+    
+    c.set(KEY_STARTROW, start != null ? start : "");
+    c.set(KEY_LASTROW, last != null ? last : "");
+    
+    List<Scan> scans = new ArrayList<Scan>();
+    
+    for(int i=0; i<3; i++){
+      Scan scan = new Scan();
+      
+      scan.addFamily(INPUT_FAMILY);
+      scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(TABLE_NAME + i));
+      
+      if (start != null) {
+        scan.setStartRow(Bytes.toBytes(start));
+      }
+      if (stop != null) {
+        scan.setStopRow(Bytes.toBytes(stop));
+      }
+      
+      scans.add(scan);
+      
+      LOG.info("scan before: " + scan);
+    }
+    
+    Job job = new Job(c, jobName);
+
+    TableMapReduceUtil.initTableMapperJob(scans, ScanMapper.class,
+        ImmutableBytesWritable.class, ImmutableBytesWritable.class, job);
+    job.setReducerClass(ScanReducer.class);
+    job.setNumReduceTasks(1); // one to get final "first" and "last" key
+    FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
+    LOG.info("Started " + job.getJobName());
+    job.waitForCompletion(true);
+    assertTrue(job.isSuccessful());
+    LOG.info("After map/reduce completion - job " + jobName);
+  }
+}

From 9ca91974750cf111b1098002aebbb559071dbada Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 2 Feb 2013 07:38:07 +0000
Subject: [PATCH 0761/1540] CHANGES.txt and pom.xml for 0.94.5RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1441710 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 9a4be492e2f3..214c0cc6261d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,88 @@
 HBase Change Log
 
+Release 0.94.5 - 2/2/2012
+Sub-task
+
+    [HBASE-2611] - Handle RS that fails while processing the failure of another one
+    [HBASE-7626] - Backport portions of HBASE-7460 to 0.94
+    [HBASE-7687] - TestCatalogTracker.testServerNotRunningIOException fails occasionally
+    [HBASE-7738] - REST server should publish metrics that are available via HTTP
+
+Bug
+
+    [HBASE-5458] - Thread safety issues with Compression.Algorithm.GZ and CompressionTest
+    [HBASE-6824] - Introduce ${hbase.local.dir} and save coprocessor jars there
+    [HBASE-7034] - Bad version, failed OPENING to OPENED but master thinks it is open anyways
+    [HBASE-7293] - [replication] Remove dead sinks from ReplicationSource.currentPeers and pick new ones
+    [HBASE-7423] - HFileArchiver should not use the configuration from the Filesystem
+    [HBASE-7468] - TestSplitTransactionOnCluster hangs frequently
+    [HBASE-7476] - HBase shell count command doesn't escape binary output
+    [HBASE-7497] - TestDistributedLogSplitting.testDelayedDeleteOnFailure times out occasionally
+    [HBASE-7498] - Make REST server thread pool size configurable
+    [HBASE-7499] - TestScannerTimeout timeout is too aggressive.
+    [HBASE-7502] - TestScannerTimeout fails on snapshot branch
+    [HBASE-7504] - -ROOT- may be offline forever after FullGC of RS
+    [HBASE-7505] - Server will hang when stopping cluster, caused by waiting for split threads
+    [HBASE-7506] - Judgment of carrying ROOT/META will become wrong when expiring server
+    [HBASE-7513] - HDFSBlocksDistribution shouldn't send NPEs when something goes wrong
+    [HBASE-7515] - Store.loadStoreFiles should close opened files if there's an exception
+    [HBASE-7524] - hbase-policy.xml is improperly set thus all rules in it can be by-passed
+    [HBASE-7530] - [replication] Work around HDFS-4380 else we get NPEs
+    [HBASE-7531] - [replication] NPE in SequenceFileLogReader because ReplicationSource doesn't nullify the reader
+    [HBASE-7534] - [replication] TestReplication.queueFailover can fail because HBaseTestingUtility.createMultiRegions is dangerous
+    [HBASE-7545] - [replication] Break out TestReplication into manageable classes
+    [HBASE-7549] - Make HTableInterface#batch() javadoc proper
+    [HBASE-7550] - Synchronization problem in AssignmentManager
+    [HBASE-7551] - nodeChildrenChange event may happen after the transition to RS_ZK_REGION_SPLITTING in SplitTransaction causing the SPLIT event to be missed in the master side.
+    [HBASE-7562] - ZKUtil: missing "else condition" in multi processing
+    [HBASE-7575] - FSUtils#getTableStoreFilePathMap should all ignore non-table folders
+    [HBASE-7578] - TestCatalogTracker hangs occasionally
+    [HBASE-7581] - TestAccessController depends on the execution order
+    [HBASE-7584] - Improve TestAccessController.testAppend
+    [HBASE-7587] - Fix two findbugs warning in RowResource
+    [HBASE-7592] - HConnectionManager.getHTableDescriptor() compares too much
+    [HBASE-7602] - TestFromClientSide.testPoolBehavior is incorrect
+    [HBASE-7617] - TestHRegionOnCluster.testDataCorrectnessReplayingRecoveredEdits still fails occasionally.
+    [HBASE-7628] - Port HBASE-6509 fast-forwarding FuzzyRowFilter to 0.94
+    [HBASE-7643] - HFileArchiver.resolveAndArchive() race condition may lead to snapshot data loss
+    [HBASE-7644] - Port HBASE-4802 'Disable show table metrics in bulk loader' to 0.94
+    [HBASE-7646] - Make forkedProcessTimeoutInSeconds configurable
+    [HBASE-7647] - 0.94 hfiles v2.1 are not backwards compatible with HFilev2.0
+    [HBASE-7648] - TestAcidGuarantees.testMixedAtomicity hangs sometimes
+    [HBASE-7654] - Add List<String> getCoprocessors() to HTableDescriptor
+    [HBASE-7669] - ROOT region wouldn't be handled by PRI-IPC-Handler
+    [HBASE-7681] - Address some recent random test failures
+    [HBASE-7684] - NullPointerException in SecureClient when Call is cleaned up due to RPC timeout
+    [HBASE-7685] - Closing socket connection can't be removed from SecureClient
+    [HBASE-7693] - Hostname returned by TableInputFormatBase.reverseDNS contains trailing period
+    [HBASE-7694] - Secure HBase should use replication call queue
+    [HBASE-7702] - Adding filtering to Import jobs
+    [HBASE-7715] - FSUtils#waitOnSafeMode can incorrectly loop on standby NN
+    [HBASE-7717] - Wait until regions are assigned in TestSplitTransactionOnCluster
+    [HBASE-7728] - deadlock occurs between hlog roller and hlog syncer
+    [HBASE-7729] - TestCatalogTrackerOnCluster.testbadOriginalRootLocation fails occasionally
+    [HBASE-7730] - HBaseAdmin#synchronousBalanceSwitch is not compatible with 0.92
+    [HBASE-7731] - Append/Increment methods in HRegion don't check whether the table is readonly or not
+    [HBASE-7740] - Recheck matching row for joined scanners
+
+Improvement
+
+    [HBASE-3996] - Support multiple tables and scanners as input to the mapper in map/reduce jobs
+    [HBASE-5416] - Improve performance of scans with some kind of filters.
+    [HBASE-5498] - Secure Bulk Load
+    [HBASE-5664] - CP hooks in Scan flow for fast forward when filter filters out a row
+    [HBASE-7441] - Make ClusterManager in IntegrationTestingUtility pluggable
+    [HBASE-7540] - Make znode dump to print a dump of replication znodes
+
+New Feature
+
+    [HBASE-6669] - Add BigDecimalColumnInterpreter for doing aggregations using AggregationClient
+
+Wish
+
+    [HBASE-7705] - Make the method getCurrentPoolSize of HTablePool public
+
+
 Release 0.94.4 - 1/2/2012
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 7f75032c8791..3487e49317c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.5-SNAPSHOT</version>
+  <version>0.94.5</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 340e61fee069224746aab77dee29b2f3ca5830f0 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 6 Feb 2013 00:00:42 +0000
Subject: [PATCH 0762/1540] HBASE-7748. Add DelimitedKeyPrefixRegionSplitPolicy

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1442803 13f79535-47bb-0310-9956-ffa450edef68
---
 .../DelimitedKeyPrefixRegionSplitPolicy.java  | 88 +++++++++++++++++++
 .../KeyPrefixRegionSplitPolicy.java           | 20 +++--
 .../regionserver/TestRegionSplitPolicy.java   | 38 +++++++-
 3 files changed, 138 insertions(+), 8 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java
new file mode 100644
index 000000000000..c0940edb356b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * A custom RegionSplitPolicy implementing a SplitPolicy that groups
+ * rows by a prefix of the row-key with a delimiter. Only the first delimiter
+ * for the row key will define the prefix of the row key that is used for grouping.
+ *
+ * This ensures that a region is not split "inside" a prefix of a row key.
+ * I.e. rows can be co-located in a region by their prefix.
+ *
+ * As an example, if you have row keys delimited with <code>_</code>, like
+ * <code>userid_eventtype_eventid</code>, and use prefix delimiter _, this split policy
+ * ensures that all rows starting with the same userid, belongs to the same region.
+ * @see KeyPrefixRegionSplitPolicy
+ */
+@InterfaceAudience.Private
+public class DelimitedKeyPrefixRegionSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
+
+  private static final Log LOG = LogFactory
+      .getLog(DelimitedKeyPrefixRegionSplitPolicy.class);
+  public static final String DELIMITER_KEY = "DelimitedKeyPrefixRegionSplitPolicy.delimiter";
+
+  private byte[] delimiter = null;
+
+  @Override
+  protected void configureForRegion(HRegion region) {
+    super.configureForRegion(region);
+    if (region != null) {
+
+      // read the prefix length from the table descriptor
+      String delimiterString = region.getTableDesc().getValue(
+          DELIMITER_KEY);
+      if (delimiterString == null || delimiterString.length() == 0) {
+        LOG.error(DELIMITER_KEY + " not specified for table "
+            + region.getTableDesc().getNameAsString()
+            + ". Using default RegionSplitPolicy");
+        return;
+      }
+
+      delimiter = Bytes.toBytes(delimiterString);
+    }
+  }
+
+  @Override
+  protected byte[] getSplitPoint() {
+    byte[] splitPoint = super.getSplitPoint();
+    if (delimiter != null) {
+
+      //find the first occurrence of delimiter in split point
+      int index = com.google.common.primitives.Bytes.indexOf(splitPoint, delimiter);
+      if (index < 0) {
+        LOG.warn("Delimiter " + Bytes.toString(delimiter) + "  not found for split key "
+            + Bytes.toString(splitPoint));
+        return splitPoint;
+      }
+
+      // group split keys by a prefix
+      return Arrays.copyOf(splitPoint, Math.min(index, splitPoint.length));
+    } else {
+      return splitPoint;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java
index bfd895429960..85c3dd9da6d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java
@@ -27,12 +27,14 @@
  * rows by a prefix of the row-key
  *
  * This ensures that a region is not split "inside" a prefix of a row key.
- * I.e. rows can be co-located in a regionb by their prefix.
+ * I.e. rows can be co-located in a region by their prefix.
  */
 public class KeyPrefixRegionSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
   private static final Log LOG = LogFactory
       .getLog(KeyPrefixRegionSplitPolicy.class);
-  public static String PREFIX_LENGTH_KEY = "prefix_split_key_policy.prefix_length";
+  @Deprecated
+  public static final String PREFIX_LENGTH_KEY_DEPRECATED = "prefix_split_key_policy.prefix_length";
+  public static final String PREFIX_LENGTH_KEY = "KeyPrefixRegionSplitPolicy.prefix_length";
 
   private int prefixLength = 0;
 
@@ -46,10 +48,14 @@ protected void configureForRegion(HRegion region) {
       String prefixLengthString = region.getTableDesc().getValue(
           PREFIX_LENGTH_KEY);
       if (prefixLengthString == null) {
-        LOG.error(PREFIX_LENGTH_KEY + " not specified for table "
-            + region.getTableDesc().getNameAsString()
-            + ". Using default RegionSplitPolicy");
-        return;
+        //read the deprecated value
+        prefixLengthString = region.getTableDesc().getValue(PREFIX_LENGTH_KEY_DEPRECATED);
+        if (prefixLengthString == null) {
+          LOG.error(PREFIX_LENGTH_KEY + " not specified for table "
+              + region.getTableDesc().getNameAsString()
+              + ". Using default RegionSplitPolicy");
+          return;
+        }
       }
       try {
         prefixLength = Integer.parseInt(prefixLengthString);
@@ -75,4 +81,4 @@ protected byte[] getSplitPoint() {
       return splitPoint;
     }
   }
-}
\ No newline at end of file
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
index 300155b20a26..69e6704cfecc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
@@ -250,7 +250,43 @@ public void testGetSplitPoint() throws IOException {
         Bytes.toString(policy.getSplitPoint()));
   }
 
+  @Test
+  public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
+    HTableDescriptor myHtd = new HTableDescriptor();
+    myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
+        DelimitedKeyPrefixRegionSplitPolicy.class.getName());
+    myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
+
+    HRegion myMockRegion = Mockito.mock(HRegion.class);
+    Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
+    Mockito.doReturn(stores).when(myMockRegion).getStores();
+
+    Store mockStore = Mockito.mock(Store.class);
+    Mockito.doReturn(2000L).when(mockStore).getSize();
+    Mockito.doReturn(true).when(mockStore).canSplit();
+    Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint();
+    stores.put(new byte[] { 1 }, mockStore);
+
+    DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
+        .create(myMockRegion, conf);
+
+    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
+
+    Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
+    Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
+        .getExplicitSplitPoint();
+
+    policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
+        .create(myMockRegion, conf);
+
+    assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
+
+    Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
+    .getExplicitSplitPoint();
+    assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
\ No newline at end of file
+}

From 11de780ee8ee1d7eddebd33220527d863d95bf17 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 6 Feb 2013 04:52:12 +0000
Subject: [PATCH 0763/1540] HBASE-7771 Secure HBase Client in MR job causes
 tasks to wait forever (Francis and Matteo)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1442843 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 32 ++++++-------------
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |  6 ----
 .../hbase/zookeeper/ZooKeeperWatcher.java     | 26 ---------------
 .../hbase/zookeeper/TestZooKeeperACL.java     |  1 -
 4 files changed, 10 insertions(+), 55 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 13dd29ff3caa..1147ad744f1b 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -198,7 +198,13 @@ private static void login(Configuration conf, String keytabFileKey,
     if (System.getProperty("java.security.auth.login.config") != null)
       return;
 
+    // No keytab specified, no auth
     String keytabFilename = conf.get(keytabFileKey);
+    if (keytabFilename == null) {
+      LOG.warn("no keytab specified for: " + keytabFileKey);
+      return;
+    }
+
     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
 
@@ -903,7 +909,8 @@ public static boolean isSecureZooKeeper(Configuration conf) {
       return true;
 
     // Master & RSs uses hbase.zookeeper.client.*
-    return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
+    return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) &&
+         conf.get("hbase.zookeeper.client.keytab.file") != null);
   }
 
   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
@@ -926,15 +933,6 @@ private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
     }
   }
 
-  public static void waitForZKConnectionIfAuthenticating(ZooKeeperWatcher zkw)
-      throws InterruptedException {
-    if (isSecureZooKeeper(zkw.getConfiguration())) {
-       LOG.debug("Waiting for ZooKeeperWatcher to authenticate");
-       zkw.saslLatch.await();
-       LOG.debug("Done waiting.");
-    }
-  }
-
   //
   // Node creation
   //
@@ -961,7 +959,6 @@ public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
       String znode, byte [] data)
   throws KeeperException {
     try {
-      waitForZKConnectionIfAuthenticating(zkw);
       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
           CreateMode.EPHEMERAL);
     } catch (KeeperException.NodeExistsException nee) {
@@ -1001,7 +998,6 @@ public static boolean createNodeIfNotExistsAndWatch(
       ZooKeeperWatcher zkw, String znode, byte [] data)
   throws KeeperException {
     try {
-      waitForZKConnectionIfAuthenticating(zkw);
       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
           CreateMode.PERSISTENT);
     } catch (KeeperException.NodeExistsException nee) {
@@ -1039,7 +1035,6 @@ public static int createAndWatch(ZooKeeperWatcher zkw,
       String znode, byte [] data)
   throws KeeperException, KeeperException.NodeExistsException {
     try {
-      waitForZKConnectionIfAuthenticating(zkw);
       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
           CreateMode.PERSISTENT);
       return zkw.getRecoverableZooKeeper().exists(znode, zkw).getVersion();
@@ -1067,13 +1062,8 @@ public static int createAndWatch(ZooKeeperWatcher zkw,
   public static void asyncCreate(ZooKeeperWatcher zkw,
       String znode, byte [] data, final AsyncCallback.StringCallback cb,
       final Object ctx) {
-    try {
-      waitForZKConnectionIfAuthenticating(zkw);
-      zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
-          createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
-    } catch (InterruptedException e) {
-      zkw.interruptedException(e);
-    }
+    zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
+        createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
   }
 
   /**
@@ -1098,7 +1088,6 @@ private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilen
     String znode = create.getPath();
     try {
       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
-      waitForZKConnectionIfAuthenticating(zkw);
       if (zk.exists(znode, false) == null) {
         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
       }
@@ -1135,7 +1124,6 @@ public static void createWithParents(ZooKeeperWatcher zkw, String znode)
       if(znode == null) {
         return;
       }
-      waitForZKConnectionIfAuthenticating(zkw);
       zkw.getRecoverableZooKeeper().create(znode, new byte[0], createACL(zkw, znode),
           CreateMode.PERSISTENT);
     } catch(KeeperException.NodeExistsException nee) {
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index ea8799097510..c6e607ef1334 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -71,12 +71,6 @@ public ZooKeeperNodeTracker(ZooKeeperWatcher watcher, String node,
    * or {@link #getData(boolean)} to get the data of the node if it is available.
    */
   public synchronized void start() {
-    try {
-      ZKUtil.waitForZKConnectionIfAuthenticating(watcher);
-    } catch (InterruptedException e) {
-      throw new IllegalStateException("ZookeeperNodeTracker on " + this.node 
-          + " interuppted while waiting for SASL Authentication", e);
-    }
     this.watcher.registerListener(this);
     try {
       if(ZKUtil.watchAndCheckExists(watcher, node)) {
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index b720d785e1a8..0274db97311b 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -71,10 +71,6 @@ public class ZooKeeperWatcher implements Watcher, Abortable {
   private final List<ZooKeeperListener> listeners =
     new CopyOnWriteArrayList<ZooKeeperListener>();
 
-  // Used by ZKUtil:waitForZKConnectionIfAuthenticating to wait for SASL
-  // negotiation to complete
-  public CountDownLatch saslLatch = new CountDownLatch(1);
-
   // set of unassigned nodes watched
   private Set<String> unassignedNodes = new HashSet<String>();
 
@@ -354,34 +350,12 @@ private void connectionEvent(WatchedEvent event) {
         LOG.debug(this.identifier + " connected");
         break;
 
-      case SaslAuthenticated:
-        if (ZKUtil.isSecureZooKeeper(this.conf)) {
-          // We are authenticated, clients can proceed.
-          saslLatch.countDown();
-        }
-        break;
-
-      case AuthFailed:
-        if (ZKUtil.isSecureZooKeeper(this.conf)) {
-          // We could not be authenticated, but clients should proceed anyway.
-          // Only access to znodes that require SASL authentication will be
-          // denied. The client may never need to access them.
-          saslLatch.countDown();
-        }
-        break;
-
       // Abort the server if Disconnected or Expired
       case Disconnected:
         LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
         break;
 
       case Expired:
-        if (ZKUtil.isSecureZooKeeper(this.conf)) {
-          // We consider Expired equivalent to AuthFailed for this
-          // connection. Authentication is never going to complete. The
-          // client should proceed to do cleanup.
-          saslLatch.countDown();
-        }
         String msg = prefix(this.identifier + " received expired from " +
           "ZooKeeper, aborting");
         // TODO: One thought is to add call to ZooKeeperListener so say,
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
index 8ab6011e6135..fba1cafa65f9 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
@@ -86,7 +86,6 @@ public static void setUpBeforeClass() throws Exception {
     zkw = new ZooKeeperWatcher(
       new Configuration(TEST_UTIL.getConfiguration()),
         TestZooKeeper.class.getName(), null);
-    ZKUtil.waitForZKConnectionIfAuthenticating(zkw);
   }
 
   /**

From 97063f811635b3677caac47372c7108da7d88355 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Wed, 6 Feb 2013 14:57:45 +0000
Subject: [PATCH 0764/1540] HBASE-6513 Test errors when building on MacOS (Lars
 Francke)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1442994 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/util/TestRegionSplitCalculator.java | 59 ++++++++++---------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
index d0e65d952b8d..a2419f997c06 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
@@ -27,6 +27,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.SortedSet;
+import java.util.UUID;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -40,21 +41,21 @@
 
 @Category(SmallTests.class)
 public class TestRegionSplitCalculator {
-  final static Log LOG = LogFactory.getLog(TestRegionSplitCalculator.class);
+  private static final Log LOG = LogFactory.getLog(TestRegionSplitCalculator.class);
 
   /**
    * This is range uses a user specified start and end keys. It also has an
-   * extra time based tiebreaker so that different ranges with the same
-   * start/end key pair count as different regions.
+   * extra tiebreaker so that different ranges with the same start/end key pair
+   * count as different regions.
    */
   static class SimpleRange implements KeyRange {
     byte[] start, end;
-    long tiebreaker;
+    UUID tiebreaker;
 
     SimpleRange(byte[] start, byte[] end) {
       this.start = start;
       this.end = end;
-      this.tiebreaker = System.nanoTime();
+      this.tiebreaker = UUID.randomUUID();
     }
 
     @Override
@@ -109,9 +110,9 @@ String dump(SortedSet<byte[]> splits, Multimap<byte[], SimpleRange> regions) {
     // we display this way because the last end key should be displayed as well.
     StringBuilder sb = new StringBuilder();
     for (byte[] k : splits) {
-      sb.append(Bytes.toString(k) + ":\t");
+      sb.append(Bytes.toString(k)).append(":\t");
       for (SimpleRange r : regions.get(k)) {
-        sb.append(r.toString() + "\t");
+        sb.append(r.toString()).append("\t");
       }
       sb.append("\n");
     }
@@ -148,7 +149,7 @@ public void testSplitCalculatorNoEdge() {
     LOG.info("Empty");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions);
-    assertEquals(res, "");
+    assertEquals("", res);
   }
 
   @Test
@@ -162,7 +163,7 @@ public void testSplitCalculatorSingleEdge() {
     LOG.info("Single edge");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1, 0);
-    assertEquals(res, "A:\t[A, B]\t\n" + "B:\t\n");
+    assertEquals("A:\t[A, B]\t\n" + "B:\t\n", res);
   }
 
   @Test
@@ -176,7 +177,7 @@ public void testSplitCalculatorDegenerateEdge() {
     LOG.info("Single empty edge");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1);
-    assertEquals(res, "A:\t[A, A]\t\n");
+    assertEquals("A:\t[A, A]\t\n", res);
   }
 
   @Test
@@ -194,8 +195,8 @@ public void testSplitCalculatorCoverSplit() {
     LOG.info("AC covers AB, BC");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 2, 2, 0);
-    assertEquals(res, "A:\t[A, B]\t[A, C]\t\n" + "B:\t[A, C]\t[B, C]\t\n"
-        + "C:\t\n");
+    assertEquals("A:\t[A, B]\t[A, C]\t\n" + "B:\t[A, C]\t[B, C]\t\n"
+        + "C:\t\n", res);
   }
 
   @Test
@@ -213,8 +214,8 @@ public void testSplitCalculatorOverEndpoint() {
     LOG.info("AB, BD covers BC");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1, 2, 1, 0);
-    assertEquals(res, "A:\t[A, B]\t\n" + "B:\t[B, C]\t[B, D]\t\n"
-        + "C:\t[B, D]\t\n" + "D:\t\n");
+    assertEquals("A:\t[A, B]\t\n" + "B:\t[B, C]\t[B, D]\t\n"
+        + "C:\t[B, D]\t\n" + "D:\t\n", res);
   }
 
   @Test
@@ -232,8 +233,8 @@ public void testSplitCalculatorHoles() {
     LOG.info("Hole between C and E");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1, 1, 0, 1, 0);
-    assertEquals(res, "A:\t[A, B]\t\n" + "B:\t[B, C]\t\n" + "C:\t\n"
-        + "E:\t[E, F]\t\n" + "F:\t\n");
+    assertEquals("A:\t[A, B]\t\n" + "B:\t[B, C]\t\n" + "C:\t\n"
+        + "E:\t[E, F]\t\n" + "F:\t\n", res);
   }
 
   @Test
@@ -249,8 +250,8 @@ public void testSplitCalculatorOverreach() {
     LOG.info("AC and BD overlap but share no start/end keys");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1, 2, 1, 0);
-    assertEquals(res, "A:\t[A, C]\t\n" + "B:\t[A, C]\t[B, D]\t\n"
-        + "C:\t[B, D]\t\n" + "D:\t\n");
+    assertEquals("A:\t[A, C]\t\n" + "B:\t[A, C]\t[B, D]\t\n"
+        + "C:\t[B, D]\t\n" + "D:\t\n", res);
   }
 
   @Test
@@ -266,7 +267,7 @@ public void testSplitCalculatorFloor() {
     LOG.info("AC and AB overlap in the beginning");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 2, 1, 0);
-    assertEquals(res, "A:\t[A, B]\t[A, C]\t\n" + "B:\t[A, C]\t\n" + "C:\t\n");
+    assertEquals("A:\t[A, B]\t[A, C]\t\n" + "B:\t[A, C]\t\n" + "C:\t\n", res);
   }
 
   @Test
@@ -282,13 +283,15 @@ public void testSplitCalculatorCeil() {
     LOG.info("AC and BC overlap in the end");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1, 2, 0);
-    assertEquals(res, "A:\t[A, C]\t\n" + "B:\t[A, C]\t[B, C]\t\n" + "C:\t\n");
+    assertEquals("A:\t[A, C]\t\n" + "B:\t[A, C]\t[B, C]\t\n" + "C:\t\n", res);
   }
 
   @Test
   public void testSplitCalculatorEq() {
     SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"));
     SimpleRange b = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"));
+
+    LOG.info(a.tiebreaker + " - " + b.tiebreaker);
     RegionSplitCalculator<SimpleRange> sc = new RegionSplitCalculator<SimpleRange>(
         cmp);
     sc.add(a);
@@ -298,7 +301,7 @@ public void testSplitCalculatorEq() {
     LOG.info("AC and AC overlap completely");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 2, 0);
-    assertEquals(res, "A:\t[A, C]\t[A, C]\t\n" + "C:\t\n");
+    assertEquals("A:\t[A, C]\t[A, C]\t\n" + "C:\t\n", res);
   }
 
   @Test
@@ -312,7 +315,7 @@ public void testSplitCalculatorBackwards() {
     LOG.info("CA is backwards");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions); // expect nothing
-    assertEquals(res, "");
+    assertEquals("", res);
   }
 
   @Test
@@ -332,11 +335,11 @@ public void testComplex() {
     LOG.info("Something fairly complex");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 3, 3, 3, 1, 2, 0, 1, 0, 1, 0);
-    assertEquals(res, "A:\t[A, Am]\t[A, B]\t[A, C]\t\n"
+    assertEquals("A:\t[A, Am]\t[A, B]\t[A, C]\t\n"
         + "Am:\t[A, B]\t[A, C]\t[Am, C]\t\n"
         + "B:\t[A, C]\t[Am, C]\t[B, E]\t\n" + "C:\t[B, E]\t\n"
         + "D:\t[B, E]\t[D, E]\t\n" + "E:\t\n" + "F:\t[F, G]\t\n" + "G:\t\n"
-        + "H:\t[H, I]\t\n" + "I:\t\n");
+        + "H:\t[H, I]\t\n" + "I:\t\n", res);
   }
 
   @Test
@@ -351,8 +354,8 @@ public void testBeginEndMarker() {
     LOG.info("Special cases -- empty");
     String res = dump(sc.getSplits(), regions);
     checkDepths(sc.getSplits(), regions, 1, 1, 1, 0);
-    assertEquals(res, ":\t[, A]\t\n" + "A:\t[A, B]\t\n" + "B:\t[B, ]\t\n"
-        + "null:\t\n");
+    assertEquals(":\t[, A]\t\n" + "A:\t[A, B]\t\n" + "B:\t[B, ]\t\n"
+        + "null:\t\n", res);
   }
 
   @Test
@@ -383,8 +386,8 @@ public void testBigRanges() {
 
     SimpleRange r1 = bigRanges.get(1);
     SimpleRange r2 = bigRanges.get(2);
-    assertEquals(Bytes.toString(r1.start), "A");
-    assertEquals(Bytes.toString(r2.start), "A");
+    assertEquals("A", Bytes.toString(r1.start));
+    assertEquals("A", Bytes.toString(r2.start));
     String r1e = Bytes.toString(r1.end);
     String r2e = Bytes.toString(r2.end);
     assertTrue((r1e.equals("C") && r2e.equals("E"))

From dfcdfaa6b177170ff9b19d20c4882f81fa4cfe4e Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 6 Feb 2013 16:54:05 +0000
Subject: [PATCH 0765/1540] HBASE-7757 Add web UI to REST server and Thrift
 server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443064 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       | 12 +++
 .../org/apache/hadoop/hbase/rest/Main.java    | 48 +++++------
 .../hadoop/hbase/thrift/ThriftServer.java     | 34 ++++++++
 .../hadoop/hbase/thrift2/ThriftServer.java    | 41 +++++++++-
 .../resources/hbase-webapps/rest/index.html   | 20 +++++
 .../resources/hbase-webapps/rest/rest.jsp     | 74 +++++++++++++++++
 .../resources/hbase-webapps/thrift/index.html | 20 +++++
 .../resources/hbase-webapps/thrift/thrift.jsp | 80 +++++++++++++++++++
 8 files changed, 302 insertions(+), 27 deletions(-)
 create mode 100644 src/main/resources/hbase-webapps/rest/index.html
 create mode 100644 src/main/resources/hbase-webapps/rest/rest.jsp
 create mode 100644 src/main/resources/hbase-webapps/thrift/index.html
 create mode 100644 src/main/resources/hbase-webapps/thrift/thrift.jsp

diff --git a/pom.xml b/pom.xml
index 3487e49317c5..af285c0df335 100644
--- a/pom.xml
+++ b/pom.xml
@@ -842,6 +842,18 @@
                              package="org.apache.hadoop.hbase.generated.regionserver"
                              webxml="${build.webapps}/regionserver/WEB-INF/web.xml"/>
 
+                <mkdir dir="${build.webapps}/rest/WEB-INF"/>
+                <jspcompiler uriroot="${src.webapps}/rest"
+                             outputdir="${generated.sources}/java"
+                             package="org.apache.hadoop.hbase.generated.rest"
+                             webxml="${build.webapps}/rest/WEB-INF/web.xml"/>
+
+                <mkdir dir="${build.webapps}/thrift/WEB-INF"/>
+                <jspcompiler uriroot="${src.webapps}/thrift"
+                             outputdir="${generated.sources}/java"
+                             package="org.apache.hadoop.hbase.generated.thrift"
+                             webxml="${build.webapps}/thrift/WEB-INF/web.xml"/>
+
                 <exec executable="sh">
                   <arg line="${basedir}/src/saveVersion.sh ${project.version} ${generated.sources}/java"/>
                 </exec>
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index 412d73f7ebc9..2a8f95686bd8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -32,10 +32,9 @@
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.rest.filter.GzipFilter;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.jmx.JMXJsonServlet;
-import org.apache.hadoop.metrics.MetricsServlet;
 import org.apache.hadoop.net.DNS;
 
 import java.util.List;
@@ -59,14 +58,13 @@
  * <li>-ro --readonly : server mode</li>
  * </ul>
  */
-@SuppressWarnings("deprecation")
 public class Main implements Constants {
 
   private static void printUsageAndExit(Options options, int exitCode) {
     HelpFormatter formatter = new HelpFormatter();
     formatter.printHelp("bin/hbase rest start", "", options,
       "\nTo run the REST server as a daemon, execute " +
-      "bin/hbase-daemon.sh start|stop rest [-p <port>] [-ro]\n", true);
+      "bin/hbase-daemon.sh start|stop rest [--infoport <port>] [-p <port>] [-ro]\n", true);
     System.exit(exitCode);
   }
 
@@ -86,6 +84,7 @@ public static void main(String[] args) throws Exception {
     options.addOption("p", "port", true, "Port to bind to [default: 8080]");
     options.addOption("ro", "readonly", false, "Respond only to GET HTTP " +
       "method requests [default: false]");
+    options.addOption(null, "infoport", true, "Port for web UI");
 
     CommandLine commandLine = null;
     try {
@@ -109,6 +108,14 @@ public static void main(String[] args) throws Exception {
       LOG.debug("readonly set to true");
     }
 
+    // check for user-defined info server port setting, if so override the conf
+    if (commandLine != null && commandLine.hasOption("infoport")) {
+      String val = commandLine.getOptionValue("infoport");
+      servlet.getConfiguration()
+          .setInt("hbase.rest.info.port", Integer.valueOf(val));
+      LOG.debug("Web UI port set to " + val);
+    }
+
     @SuppressWarnings("unchecked")
     List<String> remainingArgs = commandLine != null ?
         commandLine.getArgList() : new ArrayList<String>();
@@ -158,32 +165,11 @@ public static void main(String[] args) throws Exception {
     server.setSendDateHeader(false);
     server.setStopAtShutdown(true);
 
-      // set up context
+    // set up context
     Context context = new Context(server, "/", Context.SESSIONS);
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
 
-    // Disable the JMX and metrics servlet by default so that
-    // not to confuse existing applications use jmx/metrics as table name
-    if (servlet.getConfiguration().getBoolean(
-        "hbase.rest.enable.jmx_metrics", false)) {
-
-      // set up the JMX servlet container for Jetty
-      ServletHolder jmx = new ServletHolder(JMXJsonServlet.class);
-      // set up the metrics servlet container for Jetty
-      ServletHolder metrics = new ServletHolder(MetricsServlet.class);
-
-      String metricsPath =
-        servlet.getConfiguration().get("hbase.rest.path.metrics", "/metrics");
-      String jmxPath =
-        servlet.getConfiguration().get("hbase.rest.path.jmx", "/jmx");
-
-      context.addServlet(metrics, metricsPath);
-      context.addServlet(jmx, jmxPath);
-
-      context.getServletContext().setAttribute("hadoop.conf", conf);
-    }
-
     // login the server principal (if using secure Hadoop)
     if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
       String machineName = Strings.domainNamePointerToHostName(
@@ -193,6 +179,16 @@ public static void main(String[] args) throws Exception {
         machineName);
     }
 
+    // Put up info server.
+    int port = conf.getInt("hbase.rest.info.port", 8085);
+    if (port >= 0) {
+      conf.setLong("startcode", System.currentTimeMillis());
+      String a = conf.get("hbase.rest.info.bindAddress", "0.0.0.0");
+      InfoServer infoServer = new InfoServer("rest", a, port, false, conf);
+      infoServer.setAttribute("hbase.conf", conf);
+      infoServer.start();
+    }
+
     // start server
     server.start();
     server.join();
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
index 5812a7a5157a..1737c0552c01 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.thrift.ThriftServerRunner.ImplType;
+import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 
@@ -58,6 +59,8 @@ public class ThriftServer {
   private Configuration conf;
   ThriftServerRunner serverRunner;
 
+  private InfoServer infoServer;
+
   //
   // Main program and support routines
   //
@@ -84,6 +87,16 @@ private static void printUsageAndExit(Options options, int exitCode)
    void doMain(final String[] args) throws Exception {
      processOptions(args);
      serverRunner = new ThriftServerRunner(conf);
+
+     // Put up info server.
+     int port = conf.getInt("hbase.thrift.info.port", 9095);
+     if (port >= 0) {
+       conf.setLong("startcode", System.currentTimeMillis());
+       String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0");
+       infoServer = new InfoServer("thrift", a, port, false, conf);
+       infoServer.setAttribute("hbase.conf", conf);
+       infoServer.start();
+     }
      serverRunner.run();
   }
 
@@ -100,6 +113,7 @@ private void processOptions(final String[] args) throws Exception {
     options.addOption("f", FRAMED_OPTION, false, "Use framed transport");
     options.addOption("c", COMPACT_OPTION, false, "Use the compact protocol");
     options.addOption("h", "help", false, "Print help information");
+    options.addOption(null, "infoport", true, "Port for web UI");
 
     options.addOption("m", MIN_WORKERS_OPTION, true,
         "The minimum number of worker threads for " +
@@ -146,6 +160,18 @@ private void processOptions(final String[] args) throws Exception {
       printUsageAndExit(options, -1);
     }
 
+    // check for user-defined info server port setting, if so override the conf
+    try {
+      if (cmd.hasOption("infoport")) {
+        String val = cmd.getOptionValue("infoport");
+        conf.setInt("hbase.thrift.info.port", Integer.valueOf(val));
+        LOG.debug("Web UI port set to " + val);
+      }
+    } catch (NumberFormatException e) {
+      LOG.error("Could not parse the value provided for the infoport option", e);
+      printUsageAndExit(options, -1);
+    }
+
     // Make optional changes to the configuration based on command-line options
     optionToConf(cmd, MIN_WORKERS_OPTION,
         conf, TBoundedThreadPoolServer.MIN_WORKER_THREADS_CONF_KEY);
@@ -170,6 +196,14 @@ private void processOptions(final String[] args) throws Exception {
   }
 
   public void stop() {
+    if (this.infoServer != null) {
+      LOG.info("Stopping infoServer");
+      try {
+        this.infoServer.stop();
+      } catch (Exception ex) {
+        ex.printStackTrace();
+      }
+    }
     serverRunner.shutdown();
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index 721b663aacf8..1b97c6399ae4 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.thrift.CallQueue.Call;
 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
 import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
+import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TCompactProtocol;
 import org.apache.thrift.protocol.TProtocolFactory;
@@ -65,6 +66,7 @@
  * ThriftServer - this class starts up a Thrift server which implements the HBase API specified in the
  * HbaseClient.thrift IDL file.
  */
+@SuppressWarnings({ "rawtypes", "unchecked" })
 public class ThriftServer {
   private static final Log log = LogFactory.getLog(ThriftServer.class);
 
@@ -90,6 +92,7 @@ private static Options getOptions() {
     options.addOption("f", "framed", false, "Use framed transport");
     options.addOption("c", "compact", false, "Use the compact protocol");
     options.addOption("h", "help", false, "Print help information");
+    options.addOption(null, "infoport", true, "Port for web UI");
 
     OptionGroup servers = new OptionGroup();
     servers.addOption(
@@ -225,14 +228,27 @@ public static void main(String[] args) throws Exception {
       ThriftMetrics metrics = new ThriftMetrics(
           listenPort, conf, THBaseService.Iface.class);
 
+      String implType = "threadpool";
+      if (nonblocking) {
+        implType = "nonblocking";
+      } else if (hsha) {
+        implType = "hsha";
+      }
+
+      conf.set("hbase.regionserver.thrift.server.type", implType);
+      conf.setInt("hbase.regionserver.thrift.port", listenPort);
+
       // Construct correct ProtocolFactory
-      TProtocolFactory protocolFactory = getTProtocolFactory(cmd.hasOption("compact"));
+      boolean compact = cmd.hasOption("compact");
+      TProtocolFactory protocolFactory = getTProtocolFactory(compact);
       THBaseService.Iface handler =
           ThriftHBaseServiceHandler.newInstance(conf, metrics);
       THBaseService.Processor processor = new THBaseService.Processor(handler);
+      conf.setBoolean("hbase.regionserver.thrift.compact", compact);
 
       boolean framed = cmd.hasOption("framed") || nonblocking || hsha;
       TTransportFactory transportFactory = getTTransportFactory(framed);
+      conf.setBoolean("hbase.regionserver.thrift.framed", framed);
 
       // TODO: Remove once HBASE-2155 is resolved
       if (cmd.hasOption("bind") && (nonblocking || hsha)) {
@@ -242,6 +258,29 @@ public static void main(String[] args) throws Exception {
         System.exit(1);
       }
 
+      // check for user-defined info server port setting, if so override the conf
+      try {
+        if (cmd.hasOption("infoport")) {
+          String val = cmd.getOptionValue("infoport");
+          conf.setInt("hbase.thrift.info.port", Integer.valueOf(val));
+          log.debug("Web UI port set to " + val);
+        }
+      } catch (NumberFormatException e) {
+        log.error("Could not parse the value provided for the infoport option", e);
+        printUsage();
+        System.exit(1);
+      }
+
+      // Put up info server.
+      int port = conf.getInt("hbase.thrift.info.port", 9095);
+      if (port >= 0) {
+        conf.setLong("startcode", System.currentTimeMillis());
+        String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0");
+        InfoServer infoServer = new InfoServer("thrift", a, port, false, conf);
+        infoServer.setAttribute("hbase.conf", conf);
+        infoServer.start();
+      }
+
       InetSocketAddress inetSocketAddress = bindToPort(cmd.getOptionValue("bind"), listenPort);
 
       if (nonblocking) {
diff --git a/src/main/resources/hbase-webapps/rest/index.html b/src/main/resources/hbase-webapps/rest/index.html
new file mode 100644
index 000000000000..e4084b7c4887
--- /dev/null
+++ b/src/main/resources/hbase-webapps/rest/index.html
@@ -0,0 +1,20 @@
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<meta HTTP-EQUIV="REFRESH" content="0;url=/rest.jsp"/>
diff --git a/src/main/resources/hbase-webapps/rest/rest.jsp b/src/main/resources/hbase-webapps/rest/rest.jsp
new file mode 100644
index 000000000000..ba9856c13a8f
--- /dev/null
+++ b/src/main/resources/hbase-webapps/rest/rest.jsp
@@ -0,0 +1,74 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<%@ page contentType="text/html;charset=UTF-8"
+  import="org.apache.hadoop.conf.Configuration"
+  import="org.apache.hadoop.hbase.HBaseConfiguration"
+  import="org.apache.hadoop.hbase.util.VersionInfo"
+  import="java.util.Date"
+%>
+
+<%
+Configuration conf = (Configuration)getServletContext().getAttribute("hbase.conf");
+long startcode = conf.getLong("startcode", System.currentTimeMillis());
+String listenPort = conf.get("hbase.rest.port", "8080");
+String serverInfo = listenPort + "," + String.valueOf(startcode);
+%>
+
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Commenting out DOCTYPE so our blue outline shows on hadoop 0.20.205.0, etc.
+     See tail of HBASE-2110 for explaination.
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+-->
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
+<title>HBase REST Server</title>
+<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
+</head>
+
+<body>
+<a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo.png" alt="HBase Logo" title="HBase Logo" /></a>
+<h1 id="page_title">RESTServer: <%= serverInfo %></h1>
+<p id="links_menu">
+  <a href="/logs/">Local logs</a>,
+  <a href="/stacks">Thread Dump</a>,
+  <a href="/logLevel">Log Level</a>,
+<% if (HBaseConfiguration.isShowConfInServlet()) { %>
+  <a href="/conf">HBase Configuration</a>
+<% } %>
+</p>
+<hr id="head_rule" />
+
+<h2>Attributes</h2>
+<table id="attributes_table">
+<col style="width: 10%;"/>
+<col />
+<col style="width: 20%;"/>
+<tr><th>Attribute Name</th><th>Value</th><th>Description</th></tr>
+<tr><td>HBase Version</td><td><%= VersionInfo.getVersion() %>, r<%= VersionInfo.getRevision() %></td><td>HBase version and revision</td></tr>
+<tr><td>HBase Compiled</td><td><%= VersionInfo.getDate() %>, <%= VersionInfo.getUser() %></td><td>When HBase version was compiled and by whom</td></tr>
+<tr><td>REST Server Start Time</td><td><%= new Date(startcode) %></td><td>Date stamp of when this REST server was started</td></tr>
+</table>
+
+<hr id="foot_rule" />
+<a href="http://wiki.apache.org/hadoop/Hbase/Stargate">Apache HBase Wiki on REST</a>
+
+</body>
+</html>
diff --git a/src/main/resources/hbase-webapps/thrift/index.html b/src/main/resources/hbase-webapps/thrift/index.html
new file mode 100644
index 000000000000..9925269e8959
--- /dev/null
+++ b/src/main/resources/hbase-webapps/thrift/index.html
@@ -0,0 +1,20 @@
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<meta HTTP-EQUIV="REFRESH" content="0;url=/thrift.jsp"/>
diff --git a/src/main/resources/hbase-webapps/thrift/thrift.jsp b/src/main/resources/hbase-webapps/thrift/thrift.jsp
new file mode 100644
index 000000000000..eee99406b7db
--- /dev/null
+++ b/src/main/resources/hbase-webapps/thrift/thrift.jsp
@@ -0,0 +1,80 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<%@ page contentType="text/html;charset=UTF-8"
+  import="org.apache.hadoop.conf.Configuration"
+  import="org.apache.hadoop.hbase.HBaseConfiguration"
+  import="org.apache.hadoop.hbase.util.VersionInfo"
+  import="java.util.Date"
+%>
+
+<%
+Configuration conf = (Configuration)getServletContext().getAttribute("hbase.conf");
+long startcode = conf.getLong("startcode", System.currentTimeMillis());
+String listenPort = conf.get("hbase.regionserver.thrift.port", "9090");
+String serverInfo = listenPort + "," + String.valueOf(startcode);
+String implType = conf.get("hbase.regionserver.thrift.server.type", "threadpool");
+String compact = conf.get("hbase.regionserver.thrift.compact", "false");
+String framed = conf.get("hbase.regionserver.thrift.framed", "false");
+%>
+
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Commenting out DOCTYPE so our blue outline shows on hadoop 0.20.205.0, etc.
+     See tail of HBASE-2110 for explaination.
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+-->
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
+<title>HBase Thrift Server</title>
+<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
+</head>
+
+<body>
+<a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo.png" alt="HBase Logo" title="HBase Logo" /></a>
+<h1 id="page_title">ThriftServer: <%= serverInfo %></h1>
+<p id="links_menu">
+  <a href="/logs/">Local logs</a>,
+  <a href="/stacks">Thread Dump</a>,
+  <a href="/logLevel">Log Level</a>,
+<% if (HBaseConfiguration.isShowConfInServlet()) { %>
+  <a href="/conf">HBase Configuration</a>
+<% } %>
+</p>
+<hr id="head_rule" />
+
+<h2>Attributes</h2>
+<table id="attributes_table">
+<col style="width: 10%;"/>
+<col />
+<col style="width: 20%;"/>
+<tr><th>Attribute Name</th><th>Value</th><th>Description</th></tr>
+<tr><td>HBase Version</td><td><%= VersionInfo.getVersion() %>, r<%= VersionInfo.getRevision() %></td><td>HBase version and revision</td></tr>
+<tr><td>HBase Compiled</td><td><%= VersionInfo.getDate() %>, <%= VersionInfo.getUser() %></td><td>When HBase version was compiled and by whom</td></tr>
+<tr><td>Thrift Server Start Time</td><td><%= new Date(startcode) %></td><td>Date stamp of when this Thrift server was started</td></tr>
+<tr><td>Thrift Impl Type</td><td><%= implType %></td><td>Thrift RPC engine implementation type chosen by this Thrift server</td></tr>
+<tr><td>Compact Protocol</td><td><%= compact %></td><td>Thrift RPC engine uses compact protocol</td></tr>
+<tr><td>Framed Transport</td><td><%= framed %></td><td>Thrift RPC engine uses framed transport</td></tr>
+</table>
+
+<hr id="foot_rule" />
+<a href="http://wiki.apache.org/hadoop/Hbase/ThriftApi">Apache HBase Wiki on Thrift</a>
+
+</body>
+</html>

From 74a84ea1d287ae3e474c1dda15d3e61526727317 Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Wed, 6 Feb 2013 20:14:25 +0000
Subject: [PATCH 0766/1540] HBASE-7772 Cluster ID is not initialized correctly
 in RPC client

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443181 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HConnectionManager.java    | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index fd7a9470274a..a988fb64b9f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -600,7 +600,6 @@ public HConnectionImplementation(Configuration conf, boolean managed)
           HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
 
-      this.rpcEngine = HBaseRPC.getProtocolEngine(conf);
       this.master = null;
       this.resetting = false;
     }
@@ -613,6 +612,9 @@ private synchronized void ensureZookeeperTrackers()
       }
       if (clusterId == null) {
         clusterId = new ClusterId(zooKeeper, this);
+        if (clusterId.hasId()) {
+          conf.set(HConstants.CLUSTER_ID, clusterId.getId());
+        }
       }
       if (masterAddressTracker == null) {
         masterAddressTracker = new MasterAddressTracker(zooKeeper, this);
@@ -622,6 +624,10 @@ private synchronized void ensureZookeeperTrackers()
         rootRegionTracker = new RootRegionTracker(zooKeeper, this);
         rootRegionTracker.start();
       }
+      // RpcEngine needs access to zookeeper data, like cluster ID
+      if (rpcEngine == null) {
+        this.rpcEngine = HBaseRPC.getProtocolEngine(conf);
+      }
     }
 
     private synchronized void resetZooKeeperTrackers() {
@@ -700,9 +706,6 @@ public HMasterInterface getMaster()
               throw new MasterNotRunningException();
             }
 
-            if (clusterId.hasId()) {
-              conf.set(HConstants.CLUSTER_ID, clusterId.getId());
-            }
             InetSocketAddress isa =
               new InetSocketAddress(sn.getHostname(), sn.getPort());
             HMasterInterface tryMaster = rpcEngine.getProxy(
@@ -1325,9 +1328,6 @@ HRegionInterface getHRegionConnection(final String hostname, final int port,
           server = this.servers.get(rsName);
           if (server == null) {
             try {
-              if (clusterId.hasId()) {
-                conf.set(HConstants.CLUSTER_ID, clusterId.getId());
-              }
               // Only create isa when we need to.
               InetSocketAddress address = isa != null? isa:
                 new InetSocketAddress(hostname, port);

From 4f239e7cc80c59fc82bbdd345b7e71e65e2f59a0 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 7 Feb 2013 18:11:35 +0000
Subject: [PATCH 0767/1540] HBASE-7698 race between RS shutdown thread and
 openregionhandler causes region to get stuck (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443638 13f79535-47bb-0310-9956-ffa450edef68
---
 .../handler/OpenRegionHandler.java            | 11 +++++--
 .../handler/TestOpenRegionHandler.java        | 29 +++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 0a95e2a2447e..6b559097d034 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -84,6 +84,8 @@ public HRegionInfo getRegionInfo() {
 
   @Override
   public void process() throws IOException {
+    boolean transitionToFailedOpen = false;
+    boolean openSuccessful = false;
     try {
       final String name = regionInfo.getRegionNameAsString();
       if (this.server.isStopped() || this.rsServices.isStopping()) {
@@ -120,6 +122,7 @@ public void process() throws IOException {
           this.rsServices.isStopping()) {
         cleanupFailedOpen(region);
         tryTransitionToFailedOpen(regionInfo);
+        transitionToFailedOpen = true;
         return;
       }
 
@@ -131,17 +134,21 @@ public void process() throws IOException {
         // In case (a), the Master will process us as a dead server. In case
         // (b) the region is already being handled elsewhere anyway.
         cleanupFailedOpen(region);
+        transitionToFailedOpen = true;
         return;
       }
       // Successful region open, and add it to OnlineRegions
       this.rsServices.addToOnlineRegions(region);
-
+      openSuccessful = true;
       // Done!  Successful region open
       LOG.debug("Opened " + name + " on server:" +
         this.server.getServerName());
     } finally {
       this.rsServices.getRegionsInTransitionInRS().
           remove(this.regionInfo.getEncodedNameAsBytes());
+      if (!openSuccessful && !transitionToFailedOpen) {
+        tryTransitionToFailedOpen(regionInfo);
+      }
     }
   }
 
@@ -359,7 +366,7 @@ public boolean progress() {
     return region;
   }
 
-  private void cleanupFailedOpen(final HRegion region) throws IOException {
+  void cleanupFailedOpen(final HRegion region) throws IOException {
     if (region != null) region.close();
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
index 55a8c4a54a77..8958de53699f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
@@ -178,6 +178,35 @@ boolean updateMeta(final HRegion r) {
     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, data.getEventType());
   }
   
+  @Test
+  public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception {
+    Server server = new MockServer(HTU);
+    RegionServerServices rsServices = new MockRegionServerServices();
+    // Create it OFFLINE, which is what it expects
+    ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
+    // Create the handler
+    OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD) {
+      @Override
+      boolean updateMeta(HRegion r) {
+        return false;
+      };
+
+      @Override
+      void cleanupFailedOpen(HRegion region) throws IOException {
+        throw new IOException("FileSystem got closed.");
+      }
+    };
+    rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
+    try {
+      handler.process();
+    } catch (Exception e) {
+      // Ignore the IOException that we have thrown from cleanupFailedOpen
+    }
+    RegionTransitionData data =
+        ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName());
+    assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, data.getEventType());
+  }
+  
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 48ffe86978753904db66ddde4463c50846a43664 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 7 Feb 2013 18:58:45 +0000
Subject: [PATCH 0768/1540] HBASE-7785 rolling-restart.sh script unable to
 check expiration of master znode (Samir Ahmic)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443660 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/rolling-restart.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 07a78c71edec..e7329feb3c31 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -107,7 +107,7 @@ else
     if [ "$zmaster" == "null" ]; then zmaster="master"; fi
     zmaster=$zparent/$zmaster
     echo -n "Waiting for Master ZNode ${zmaster} to expire"
-    while ! bin/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do
+    while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do
       echo -n "."
       sleep 1
     done

From 310b82f3ecec1a0cfd1f6fce03166fec099f72c0 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 7 Feb 2013 23:02:13 +0000
Subject: [PATCH 0769/1540] HBASE-7561 Display the total number of regions for
 a given table on the master webUI (Michael Weng)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443768 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index c78d70963734..2d3427d81a4d 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -178,6 +178,7 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 <%if (frags != null) %>
         <th title="Fragmentation - Will be 0% after a major compaction and fluctuate during normal usage.">Frag.</th>
 </%if>
+    <th>Online Regions</th>
     <th>Description</th>
 </tr>
 <%for HTableDescriptor htDesc : tables%>
@@ -186,6 +187,7 @@ org.apache.hadoop.hbase.HBaseConfiguration;
     <%if (frags != null) %>
         <td align="center"><% frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %></td>
     </%if>
+    <td><% master.getAssignmentManager().getRegionsOfTable(htDesc.getName()).size() %>
     <td><% htDesc.toStringCustomizedValues() %></td>
 </tr>
 </%for>

From 9536e6e004d613ed54ada875b9bb6de60f2950d2 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 7 Feb 2013 23:04:12 +0000
Subject: [PATCH 0770/1540] HBASE-7776 Use ErrorReporter/Log instead of
 System.out in hbck

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443769 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 191 ++++++++++--------
 .../util/hbck/HFileCorruptionChecker.java     |  24 +--
 2 files changed, 122 insertions(+), 93 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8ef8be2e55ab..674ac1a08243 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -1417,8 +1418,9 @@ private void preCheckPermission() throws IOException, AccessControlException {
         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
       } catch (AccessControlException ace) {
         LOG.warn("Got AccessControlException when preCheckPermission ", ace);
-        System.err.println("Current user " + ugi.getUserName() + " does not have write perms to " + file.getPath()
-            + ". Please rerun hbck as hdfs user " + file.getOwner());
+        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
+          + " does not have write perms to " + file.getPath()
+          + ". Please rerun hbck as hdfs user " + file.getOwner());
         throw new AccessControlException(ace);
       }
     }
@@ -2327,19 +2329,19 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc
 
       if (details) {
         // do full region split map dump
-        System.out.println("---- Table '"  +  this.tableName
+        errors.print("---- Table '"  +  this.tableName
             + "': region split map");
         dump(splits, regions);
-        System.out.println("---- Table '"  +  this.tableName
+        errors.print("---- Table '"  +  this.tableName
             + "': overlap groups");
         dumpOverlapProblems(overlapGroups);
-        System.out.println("There are " + overlapGroups.keySet().size()
+        errors.print("There are " + overlapGroups.keySet().size()
             + " overlap groups with " + overlapGroups.size()
             + " overlapping regions");
       }
       if (!sidelinedRegions.isEmpty()) {
         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
-        System.out.println("---- Table '"  +  this.tableName
+        errors.print("---- Table '"  +  this.tableName
             + "': sidelined big overlapped regions");
         dumpSidelinedRegions(sidelinedRegions);
       }
@@ -2354,13 +2356,15 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc
      */
     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
       // we display this way because the last end key should be displayed as well.
+      StringBuilder sb = new StringBuilder();
       for (byte[] k : splits) {
-        System.out.print(Bytes.toStringBinary(k) + ":\t");
+        sb.setLength(0); // clear out existing buffer, if any.
+        sb.append(Bytes.toStringBinary(k) + ":\t");
         for (HbckInfo r : regions.get(k)) {
-          System.out.print("[ "+ r.toString() + ", "
+          sb.append("[ "+ r.toString() + ", "
               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
         }
-        System.out.println();
+        errors.print(sb.toString());
       }
     }
   }
@@ -2369,12 +2373,12 @@ public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
     // we display this way because the last end key should be displayed as
     // well.
     for (byte[] k : regions.keySet()) {
-      System.out.print(Bytes.toStringBinary(k) + ":\n");
+      errors.print(Bytes.toStringBinary(k) + ":");
       for (HbckInfo r : regions.get(k)) {
-        System.out.print("[ " + r.toString() + ", "
-            + Bytes.toStringBinary(r.getEndKey()) + "]\n");
+        errors.print("[ " + r.toString() + ", "
+            + Bytes.toStringBinary(r.getEndKey()) + "]");
       }
-      System.out.println("----");
+      errors.print("----");
     }
   }
 
@@ -2382,9 +2386,9 @@ public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
       String tableName = Bytes.toStringBinary(entry.getValue().getTableName());
       Path path = entry.getKey();
-      System.out.println("This sidelined region dir should be bulk loaded: "
+      errors.print("This sidelined region dir should be bulk loaded: "
         + path.toString());
-      System.out.println("Bulk load command looks like: "
+      errors.print("Bulk load command looks like: "
         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
         + path.toUri().getPath() + " "+ tableName);
     }
@@ -2815,23 +2819,25 @@ public int compare(HbckInfo l, HbckInfo r) {
    * Prints summary of all tables found on the system.
    */
   private void printTableSummary(SortedMap<String, TableInfo> tablesInfo) {
-    System.out.println("Summary:");
+    StringBuilder sb = new StringBuilder();
+    errors.print("Summary:");
     for (TableInfo tInfo : tablesInfo.values()) {
       if (errors.tableHasErrors(tInfo)) {
-        System.out.println("Table " + tInfo.getName() + " is inconsistent.");
+        errors.print("Table " + tInfo.getName() + " is inconsistent.");
       } else {
-        System.out.println("  " + tInfo.getName() + " is okay.");
+        errors.print("  " + tInfo.getName() + " is okay.");
       }
-      System.out.println("    Number of regions: " + tInfo.getNumRegions());
-      System.out.print("    Deployed on: ");
+      errors.print("    Number of regions: " + tInfo.getNumRegions());
+      sb.setLength(0); // clear out existing buffer, if any.
+      sb.append("    Deployed on: ");
       for (ServerName server : tInfo.deployedOn) {
-        System.out.print(" " + server.toString());
+        sb.append(" " + server.toString());
       }
-      System.out.println();
+      errors.print(sb.toString());
     }
   }
 
-  private static ErrorReporter getErrorReporter(
+  static ErrorReporter getErrorReporter(
       final Configuration conf) throws ClassNotFoundException {
     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
@@ -2844,7 +2850,8 @@ public static enum ERROR_CODE {
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
-      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE
+      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
+      WRONG_USAGE
     }
     public void clear();
     public void report(String message);
@@ -2878,6 +2885,11 @@ public void clear() {
     }
 
     public synchronized void reportError(ERROR_CODE errorCode, String message) {
+      if (errorCode == ERROR_CODE.WRONG_USAGE) {
+        System.err.println(message);
+        return;
+      }
+
       errorList.add(errorCode);
       if (!summary) {
         System.out.println("ERROR: " + message);
@@ -3365,48 +3377,53 @@ public int getRetCode() {
   }
 
   protected HBaseFsck printUsageAndExit() {
-    System.err.println("Usage: fsck [opts] {only tables}");
-    System.err.println(" where [opts] are:");
-    System.err.println("   -help Display help options (this)");
-    System.err.println("   -details Display full report of all regions.");
-    System.err.println("   -timelag <timeInSeconds>  Process only regions that " +
+    StringWriter sw = new StringWriter(2048);
+    PrintWriter out = new PrintWriter(sw);
+    out.println("Usage: fsck [opts] {only tables}");
+    out.println(" where [opts] are:");
+    out.println("   -help Display help options (this)");
+    out.println("   -details Display full report of all regions.");
+    out.println("   -timelag <timeInSeconds>  Process only regions that " +
                        " have not experienced any metadata updates in the last " +
                        " <timeInSeconds> seconds.");
-    System.err.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
+    out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
         " before checking if the fix worked if run with -fix");
-    System.err.println("   -summary Print only summary of the tables and status.");
-    System.err.println("   -metaonly Only check the state of ROOT and META tables.");
-    System.err.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta and root.");
-
-    System.err.println("");
-    System.err.println("  Metadata Repair options: (expert features, use with caution!)");
-    System.err.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
-    System.err.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
-    System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
-    System.err.println("   -noHdfsChecking   Don't load/check region info from HDFS."
+    out.println("   -summary Print only summary of the tables and status.");
+    out.println("   -metaonly Only check the state of ROOT and META tables.");
+    out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta and root.");
+
+    out.println("");
+    out.println("  Metadata Repair options: (expert features, use with caution!)");
+    out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
+    out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
+    out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
+    out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
         + " Assumes META region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
-    System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
-    System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
-    System.err.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
-    System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
-    System.err.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
-    System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
-    System.err.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
-    System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
-    System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
-    System.err.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
-    System.err.println("   -fixReferenceFiles  Try to offline lingering reference store files");
-
-    System.err.println("");
-    System.err.println("  Datafile Repair options: (expert features, use with caution!)");
-    System.err.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
-    System.err.println("   -sidelineCorruptHfiles  Quarantine corrupted HFiles.  implies -checkCorruptHfiles");
-
-    System.err.println("");
-    System.err.println("  Metadata Repair shortcuts");
-    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
+    out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
+    out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
+    out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
+    out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
+    out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
+    out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
+    out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
+    out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
+    out.println("   -fixSplitParents  Try to force offline split parents to be online.");
+    out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
+    out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
+
+    out.println("");
+    out.println("  Datafile Repair options: (expert features, use with caution!)");
+    out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
+    out.println("   -sidelineCorruptHfiles  Quarantine corrupted HFiles.  implies -checkCorruptHfiles");
+
+    out.println("");
+    out.println("  Metadata Repair shortcuts");
+    out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles");
-    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
+    out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
+
+    out.flush();
+    errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
 
     setRetCode(-2);
     return this;
@@ -3452,39 +3469,40 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setDisplayFullReport();
       } else if (cmd.equals("-timelag")) {
         if (i == args.length - 1) {
-          System.err.println("HBaseFsck: -timelag needs a value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
           return printUsageAndExit();
         }
         try {
           long timelag = Long.parseLong(args[i+1]);
           setTimeLag(timelag);
         } catch (NumberFormatException e) {
-          System.err.println("-timelag needs a numeric value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
           return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-sleepBeforeRerun")) {
         if (i == args.length - 1) {
-          System.err.println("HBaseFsck: -sleepBeforeRerun needs a value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "HBaseFsck: -sleepBeforeRerun needs a value.");
           return printUsageAndExit();
         }
         try {
           sleepBeforeRerun = Long.parseLong(args[i+1]);
         } catch (NumberFormatException e) {
-          System.err.println("-sleepBeforeRerun needs a numeric value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
           return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-sidelineDir")) {
         if (i == args.length - 1) {
-          System.err.println("HBaseFsck: -sidelineDir needs a value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
           return printUsageAndExit();
         }
         i++;
         setSidelineDir(args[i]);
       } else if (cmd.equals("-fix")) {
-        System.err.println("This option is deprecated, please use " +
-          "-fixAssignments instead.");
+        errors.reportError(ERROR_CODE.WRONG_USAGE,
+          "This option is deprecated, please use  -fixAssignments instead.");
         setFixAssignments(true);
       } else if (cmd.equals("-fixAssignments")) {
         setFixAssignments(true);
@@ -3539,27 +3557,31 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setCheckHdfs(true);
       } else if (cmd.equals("-maxOverlapsToSideline")) {
         if (i == args.length - 1) {
-          System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxOverlapsToSideline needs a numeric value argument.");
           return printUsageAndExit();
         }
         try {
           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
           setMaxOverlapsToSideline(maxOverlapsToSideline);
         } catch (NumberFormatException e) {
-          System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxOverlapsToSideline needs a numeric value argument.");
           return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-maxMerge")) {
         if (i == args.length - 1) {
-          System.err.println("-maxMerge needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxMerge needs a numeric value argument.");
           return printUsageAndExit();
         }
         try {
           int maxMerge = Integer.parseInt(args[i+1]);
           setMaxMerge(maxMerge);
         } catch (NumberFormatException e) {
-          System.err.println("-maxMerge needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxMerge needs a numeric value argument.");
           return printUsageAndExit();
         }
         i++;
@@ -3568,11 +3590,11 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
       } else if (cmd.equals("-metaonly")) {
         setCheckMetaOnly();
       } else if (cmd.startsWith("-")) {
-        System.err.println("Unrecognized option:" + cmd);
+        errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
         return printUsageAndExit();
       } else {
         includeTable(cmd);
-        System.out.println("Allow checking/fixes for table: " + cmd);
+        errors.print("Allow checking/fixes for table: " + cmd);
       }
     }
 
@@ -3604,9 +3626,7 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
       }
       hfcc.checkTables(tableDirs);
-      PrintWriter out = new PrintWriter(System.out);
-      hfcc.report(out);
-      out.flush();
+      hfcc.report(errors);
     }
 
     // check and fix table integrity, region consistency.
@@ -3641,13 +3661,22 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
    * ls -r for debugging purposes
    */
   void debugLsr(Path p) throws IOException {
-    debugLsr(getConf(), p);
+    debugLsr(getConf(), p, errors);
+  }
+
+  /**
+   * ls -r for debugging purposes
+   */
+  public static void debugLsr(Configuration conf,
+      Path p) throws IOException {
+    debugLsr(conf, p, new PrintingErrorReporter());
   }
 
   /**
    * ls -r for debugging purposes
    */
-  public static void debugLsr(Configuration conf, Path p) throws IOException {
+  public static void debugLsr(Configuration conf,
+      Path p, ErrorReporter errors) throws IOException {
     if (!LOG.isDebugEnabled() || p == null) {
       return;
     }
@@ -3657,7 +3686,7 @@ public static void debugLsr(Configuration conf, Path p) throws IOException {
       // nothing
       return;
     }
-    System.out.println(p);
+    errors.print(p.toString());
 
     if (fs.isFile(p)) {
       return;
@@ -3666,7 +3695,7 @@ public static void debugLsr(Configuration conf, Path p) throws IOException {
     if (fs.getFileStatus(p).isDir()) {
       FileStatus[] fss= fs.listStatus(p);
       for (FileStatus status : fss) {
-        debugLsr(conf, status.getPath());
+        debugLsr(conf, status.getPath(), errors);
       }
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
index f96024cd2834..0ac44d839e7b 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
@@ -19,7 +19,6 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -45,6 +44,7 @@
 import org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter;
 import org.apache.hadoop.hbase.util.FSUtils.HFileFilter;
 import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
+import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
 
 /**
  * This class marches through all of the region's hfiles and verifies that
@@ -338,22 +338,22 @@ public Collection<Path> getMissing() {
    * Print a human readable summary of hfile quarantining operations.
    * @param out
    */
-  public void report(PrintWriter out) {
-    out.println("Checked " + hfilesChecked.get() + " hfile for corruption");
-    out.println("  HFiles corrupted:                  " + corrupted.size());
+  public void report(ErrorReporter out) {
+    out.print("Checked " + hfilesChecked.get() + " hfile for corruption");
+    out.print("  HFiles corrupted:                  " + corrupted.size());
     if (inQuarantineMode) {
-      out.println("    HFiles successfully quarantined: " + quarantined.size());
+      out.print("    HFiles successfully quarantined: " + quarantined.size());
       for (Path sq : quarantined) {
-        out.println("      " + sq);
+        out.print("      " + sq);
       }
-      out.println("    HFiles failed quarantine:        " + failures.size());
+      out.print("    HFiles failed quarantine:        " + failures.size());
       for (Path fq : failures) {
-        out.println("      " + fq);
+        out.print("      " + fq);
       }
     }
-    out.println("    HFiles moved while checking:     " + missing.size());
+    out.print("    HFiles moved while checking:     " + missing.size());
     for (Path mq : missing) {
-      out.println("      " + mq);
+      out.print("      " + mq);
     }
 
     String initialState = (corrupted.size() == 0) ? "OK" : "CORRUPTED";
@@ -361,9 +361,9 @@ public void report(PrintWriter out) {
         : "CORRUPTED";
 
     if (inQuarantineMode) {
-      out.println("Summary: " + initialState + " => " + fixedState);
+      out.print("Summary: " + initialState + " => " + fixedState);
     } else {
-      out.println("Summary: " + initialState);
+      out.print("Summary: " + initialState);
     }
   }
 }

From 6bd4ed7ab28e77fdfc8b81bd32795b0ff082a129 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 8 Feb 2013 05:26:24 +0000
Subject: [PATCH 0771/1540] HBASE-7793 Port HBASE-5564 Bulkload is discarding
 duplicate records to 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443842 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/ImportTsv.java     | 89 +++++++++++++++--
 .../hbase/mapreduce/TsvImporterMapper.java    | 11 ++-
 .../hadoop/hbase/mapreduce/TestImportTsv.java | 95 +++++++++++++++++--
 3 files changed, 174 insertions(+), 21 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index 7924cb958411..4a61da77eee9 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -79,7 +79,16 @@ static class TsvParser {
 
     private int rowKeyColumnIndex;
 
-    public static String ROWKEY_COLUMN_SPEC="HBASE_ROW_KEY";
+    private int maxColumnCount;
+
+    // Default value must be negative
+    public static final int DEFAULT_TIMESTAMP_COLUMN_INDEX = -1;
+
+    private int timestampKeyColumnIndex = DEFAULT_TIMESTAMP_COLUMN_INDEX;
+
+    public static String ROWKEY_COLUMN_SPEC = "HBASE_ROW_KEY";
+
+    public static String TIMESTAMPKEY_COLUMN_SPEC = "HBASE_TS_KEY";
 
     /**
      * @param columnsSpecification the list of columns to parser out, comma separated.
@@ -96,8 +105,9 @@ public TsvParser(String columnsSpecification, String separatorStr) {
       ArrayList<String> columnStrings = Lists.newArrayList(
         Splitter.on(',').trimResults().split(columnsSpecification));
 
-      families = new byte[columnStrings.size()][];
-      qualifiers = new byte[columnStrings.size()][];
+      maxColumnCount = columnStrings.size();
+      families = new byte[maxColumnCount][];
+      qualifiers = new byte[maxColumnCount][];
 
       for (int i = 0; i < columnStrings.size(); i++) {
         String str = columnStrings.get(i);
@@ -105,6 +115,12 @@ public TsvParser(String columnsSpecification, String separatorStr) {
           rowKeyColumnIndex = i;
           continue;
         }
+        
+        if (TIMESTAMPKEY_COLUMN_SPEC.equals(str)) {
+          timestampKeyColumnIndex = i;
+          continue;
+        }
+        
         String[] parts = str.split(":", 2);
         if (parts.length == 1) {
           families[i] = str.getBytes();
@@ -116,6 +132,14 @@ public TsvParser(String columnsSpecification, String separatorStr) {
       }
     }
 
+    public boolean hasTimestamp() {
+      return timestampKeyColumnIndex != DEFAULT_TIMESTAMP_COLUMN_INDEX;
+    }
+
+    public int getTimestampKeyColumnIndex() {
+      return timestampKeyColumnIndex;
+    }
+
     public int getRowKeyColumnIndex() {
       return rowKeyColumnIndex;
     }
@@ -129,7 +153,7 @@ public byte[] getQualifier(int idx) {
     public ParsedLine parse(byte[] lineBytes, int length)
     throws BadTsvLineException {
       // Enumerate separator offsets
-      ArrayList<Integer> tabOffsets = new ArrayList<Integer>(families.length);
+      ArrayList<Integer> tabOffsets = new ArrayList<Integer>(maxColumnCount);
       for (int i = 0; i < length; i++) {
         if (lineBytes[i] == separatorByte) {
           tabOffsets.add(i);
@@ -141,10 +165,13 @@ public ParsedLine parse(byte[] lineBytes, int length)
 
       tabOffsets.add(length);
 
-      if (tabOffsets.size() > families.length) {
+      if (tabOffsets.size() > maxColumnCount) {
         throw new BadTsvLineException("Excessive columns");
       } else if (tabOffsets.size() <= getRowKeyColumnIndex()) {
         throw new BadTsvLineException("No row key");
+      } else if (hasTimestamp()
+          && tabOffsets.size() <= getTimestampKeyColumnIndex()) {
+        throw new BadTsvLineException("No timestamp");
       }
       return new ParsedLine(tabOffsets, lineBytes);
     }
@@ -164,6 +191,24 @@ public int getRowKeyOffset() {
       public int getRowKeyLength() {
         return getColumnLength(rowKeyColumnIndex);
       }
+      
+      public long getTimestamp(long ts) throws BadTsvLineException {
+        // Return ts if HBASE_TS_KEY is not configured in column spec
+        if (!hasTimestamp()) {
+          return ts;
+        }
+
+        String timeStampStr = Bytes.toString(lineBytes,
+            getColumnOffset(timestampKeyColumnIndex),
+            getColumnLength(timestampKeyColumnIndex));
+        try {
+          return Long.parseLong(timeStampStr);
+        } catch (NumberFormatException nfe) {
+          // treat this record as bad record
+          throw new BadTsvLineException("Invalid timestamp " + timeStampStr);
+        }
+      }
+      
       public int getColumnOffset(int idx) {
         if (idx > 0)
           return tabOffsets.get(idx - 1) + 1;
@@ -285,7 +330,11 @@ private static void usage(final String errorMsg) {
       "column name HBASE_ROW_KEY is used to designate that this column should be used\n" +
       "as the row key for each imported record. You must specify exactly one column\n" +
       "to be the row key, and you must specify a column name for every column that exists in the\n" +
-      "input data.\n" +
+      "input data. Another special column HBASE_TS_KEY designates that this column should be\n" +
+      "used as timestamp for each record. Unlike HBASE_ROW_KEY, HBASE_TS_KEY is optional.\n" +
+      "You must specify atmost one column as timestamp key for each imported record.\n" +
+      "Record with invalid timestamps (blank, non-numeric) will be treated as bad record.\n" +
+      "Note: if you use this option, then 'importtsv.timestamp' option will be ignored.\n" +
       "\n" +
       "By default importtsv will load data directly into HBase. To instead generate\n" +
       "HFiles of data to prepare for a bulk data load, pass the option:\n" +
@@ -344,11 +393,33 @@ public static void main(String[] args) throws Exception {
       System.exit(-1);
     }
 
-    // Make sure one or more columns are specified
-    if (columns.length < 2) {
-      usage("One or more columns in addition to the row key are required");
+    // Make sure we have at most one column as the timestamp key
+    int tskeysFound = 0;
+    for (String col : columns) {
+      if (col.equals(TsvParser.TIMESTAMPKEY_COLUMN_SPEC))
+        tskeysFound++;
+    }
+    if (tskeysFound > 1) {
+      usage("Must specify at most one column as "
+          + TsvParser.TIMESTAMPKEY_COLUMN_SPEC);
+      System.exit(-1);
+    }
+    
+    // Make sure one or more columns are specified excluding rowkey and
+    // timestamp key
+    if (columns.length - (rowkeysFound + tskeysFound) < 1) {
+      usage("One or more columns in addition to the row key and timestamp(optional) are required");
       System.exit(-1);
     }
+
+    // If timestamp option is not specified, use current system time.
+    long timstamp = conf
+        .getLong(TIMESTAMP_CONF_KEY, System.currentTimeMillis());
+
+    // Set it back to replace invalid timestamp (non-numeric) with current
+    // system time
+    conf.setLong(TIMESTAMP_CONF_KEY, timstamp); 
+    
     hbaseAdmin = new HBaseAdmin(conf);
     Job job = createSubmittableJob(conf, otherArgs);
     System.exit(job.waitForCompletion(true) ? 0 : 1);
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
index 406dc9cabdff..398c708266e2 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
@@ -101,7 +101,9 @@ protected void doSetup(Context context) {
       separator = new String(Base64.decode(separator));
     }
 
-    ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, System.currentTimeMillis());
+    // Should never get 0 as we are setting this to a valid value in job
+    // configuration.
+    ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, 0);
 
     skipBadLines = context.getConfiguration().getBoolean(
         ImportTsv.SKIP_LINES_CONF_KEY, true);
@@ -124,10 +126,15 @@ public void map(LongWritable offset, Text value,
         new ImmutableBytesWritable(lineBytes,
             parsed.getRowKeyOffset(),
             parsed.getRowKeyLength());
+      // Retrieve timestamp if exists
+      ts = parsed.getTimestamp(ts);
 
       Put put = new Put(rowKey.copyBytes());
       for (int i = 0; i < parsed.getColumnCount(); i++) {
-        if (i == parser.getRowKeyColumnIndex()) continue;
+        if (i == parser.getRowKeyColumnIndex()
+            || i == parser.getTimestampKeyColumnIndex()) {
+          continue;
+        }
         KeyValue kv = new KeyValue(
             lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(),
             parser.getFamily(i), 0, parser.getFamily(i).length,
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index a21b99b013b9..77d044b944ca 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -64,6 +64,7 @@ public void testTsvParserSpecParsing() {
     assertNull(parser.getFamily(0));
     assertNull(parser.getQualifier(0));
     assertEquals(0, parser.getRowKeyColumnIndex());
+    assertFalse(parser.hasTimestamp());
 
     parser = new TsvParser("HBASE_ROW_KEY,col1:scol1", "\t");
     assertNull(parser.getFamily(0));
@@ -71,6 +72,7 @@ public void testTsvParserSpecParsing() {
     assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1));
     assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1));
     assertEquals(0, parser.getRowKeyColumnIndex());
+    assertFalse(parser.hasTimestamp());
 
     parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,col1:scol2", "\t");
     assertNull(parser.getFamily(0));
@@ -80,6 +82,19 @@ public void testTsvParserSpecParsing() {
     assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(2));
     assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(2));
     assertEquals(0, parser.getRowKeyColumnIndex());
+    assertFalse(parser.hasTimestamp());
+    
+    parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,HBASE_TS_KEY,col1:scol2",
+        "\t");
+    assertNull(parser.getFamily(0));
+    assertNull(parser.getQualifier(0));
+    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1));
+    assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1));
+    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(3));
+    assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(3));
+    assertEquals(0, parser.getRowKeyColumnIndex());
+    assertTrue(parser.hasTimestamp());
+    assertEquals(2, parser.getTimestampKeyColumnIndex());
   }
 
   @Test
@@ -93,10 +108,32 @@ public void testTsvParser() throws BadTsvLineException {
     assertNull(parser.getQualifier(2));
     assertEquals(2, parser.getRowKeyColumnIndex());
     
+    assertEquals(TsvParser.DEFAULT_TIMESTAMP_COLUMN_INDEX, parser
+        .getTimestampKeyColumnIndex());
+    
     byte[] line = Bytes.toBytes("val_a\tval_b\tval_c\tval_d");
     ParsedLine parsed = parser.parse(line, line.length);
     checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
   }
+  
+  
+  @Test
+  public void testTsvParserWithTimestamp() throws BadTsvLineException {
+    TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t");
+    assertNull(parser.getFamily(0));
+    assertNull(parser.getQualifier(0));
+    assertNull(parser.getFamily(1));
+    assertNull(parser.getQualifier(1));
+    assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(2));
+    assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(2));
+    assertEquals(0, parser.getRowKeyColumnIndex());
+    assertEquals(1, parser.getTimestampKeyColumnIndex());
+
+    byte[] line = Bytes.toBytes("rowkey\t1234\tval_a");
+    ParsedLine parsed = parser.parse(line, line.length);
+    assertEquals(1234l, parsed.getTimestamp(-1));
+    checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
+  }
 
   private void checkParsing(ParsedLine parsed, Iterable<String> expected) {
     ArrayList<String> parsedCols = new ArrayList<String>();
@@ -123,29 +160,48 @@ private void assertBytesEquals(byte[] a, byte[] b) {
   public void testTsvParserBadTsvLineExcessiveColumns() throws BadTsvLineException {
     TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t");
     byte[] line = Bytes.toBytes("val_a\tval_b\tval_c");
-    ParsedLine parsed = parser.parse(line, line.length);
+    parser.parse(line, line.length);
   }
 
   @Test(expected=BadTsvLineException.class)
   public void testTsvParserBadTsvLineZeroColumn() throws BadTsvLineException {
     TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t");
     byte[] line = Bytes.toBytes("");
-    ParsedLine parsed = parser.parse(line, line.length);
+    parser.parse(line, line.length);
   }
 
   @Test(expected=BadTsvLineException.class)
   public void testTsvParserBadTsvLineOnlyKey() throws BadTsvLineException {
     TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t");
     byte[] line = Bytes.toBytes("key_only");
-    ParsedLine parsed = parser.parse(line, line.length);
+    parser.parse(line, line.length);
   }
 
   @Test(expected=BadTsvLineException.class)
   public void testTsvParserBadTsvLineNoRowKey() throws BadTsvLineException {
     TsvParser parser = new TsvParser("col_a,HBASE_ROW_KEY", "\t");
     byte[] line = Bytes.toBytes("only_cola_data_and_no_row_key");
+    parser.parse(line, line.length);
+  }
+  
+  @Test(expected = BadTsvLineException.class)
+  public void testTsvParserInvalidTimestamp() throws BadTsvLineException {
+    TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t");
+    assertEquals(1, parser.getTimestampKeyColumnIndex());
+    byte[] line = Bytes.toBytes("rowkey\ttimestamp\tval_a");
     ParsedLine parsed = parser.parse(line, line.length);
+    assertEquals(-1, parsed.getTimestamp(-1));
+    checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
   }
+  
+  @Test(expected = BadTsvLineException.class)
+  public void testTsvParserNoTimestampValue() throws BadTsvLineException {
+    TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY", "\t");
+    assertEquals(2, parser.getTimestampKeyColumnIndex());
+    byte[] line = Bytes.toBytes("rowkey\tval_a");
+    parser.parse(line, line.length);
+  }
+  
 
   @Test
   public void testMROnTable()
@@ -162,8 +218,25 @@ public void testMROnTable()
         INPUT_FILE
     };
 
-    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, args, 1);
+    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 1);
+  }
+  
+  @Test
+  public void testMROnTableWithTimestamp() throws Exception {
+    String TABLE_NAME = "TestTable";
+    String FAMILY = "FAM";
+    String INPUT_FILE = "InputFile1.csv";
+
+    // Prepare the arguments required for the test.
+    String[] args = new String[] {
+        "-D" + ImportTsv.COLUMNS_CONF_KEY
+            + "=HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B",
+        "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=,", TABLE_NAME, INPUT_FILE };
+
+    String data = "KEY,1234,VALUE1,VALUE2\n";
+    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, data, args, 1);
   }
+  
 
   @Test
   public void testMROnTableWithCustomMapper()
@@ -179,11 +252,11 @@ public void testMROnTableWithCustomMapper()
         INPUT_FILE
     };
 
-    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, args, 3);
+    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3);
   }
 
   private void doMROnTableTest(String inputFile, String family, String tableName,
-                               String[] args, int valueMultiplier) throws Exception {
+                               String data, String[] args, int valueMultiplier) throws Exception {
 
     // Cluster
     HBaseTestingUtility htu1 = new HBaseTestingUtility();
@@ -198,8 +271,10 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
     try {
       FileSystem fs = FileSystem.get(conf);
       FSDataOutputStream op = fs.create(new Path(inputFile), true);
-      String line = "KEY\u001bVALUE1\u001bVALUE2\n";
-      op.write(line.getBytes(HConstants.UTF8_ENCODING));
+      if (data == null) {
+        data = "KEY\u001bVALUE1\u001bVALUE2\n";
+      }
+      op.write(Bytes.toBytes(data));
       op.close();
 
       final byte[] FAM = Bytes.toBytes(family);
@@ -273,11 +348,11 @@ public void testBulkOutputWithoutAnExistingTable() throws Exception {
         "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b",
         "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=output", TABLE_NAME,
         INPUT_FILE };
-    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, args, 3);
+    doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3);
   }
 
   public static String toU8Str(byte[] bytes) throws UnsupportedEncodingException {
-    return new String(bytes, HConstants.UTF8_ENCODING);
+    return new String(bytes);
   }
 
   @org.junit.Rule

From 732e5b8b6191da1ac0a0667c1514c2f78b5da2b8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 8 Feb 2013 05:48:22 +0000
Subject: [PATCH 0772/1540] CHANGES.txt for 0.94.5RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1443843 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 214c0cc6261d..43da66851228 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.5 - 2/2/2012
+Release 0.94.5 - 2/7/2012
 Sub-task
 
     [HBASE-2611] - Handle RS that fails while processing the failure of another one
@@ -11,6 +11,7 @@ Sub-task
 Bug
 
     [HBASE-5458] - Thread safety issues with Compression.Algorithm.GZ and CompressionTest
+    [HBASE-6513] - Test errors when building on MacOS
     [HBASE-6824] - Introduce ${hbase.local.dir} and save coprocessor jars there
     [HBASE-7034] - Bad version, failed OPENING to OPENED but master thinks it is open anyways
     [HBASE-7293] - [replication] Remove dead sinks from ReplicationSource.currentPeers and pick new ones
@@ -56,6 +57,7 @@ Bug
     [HBASE-7685] - Closing socket connection can't be removed from SecureClient
     [HBASE-7693] - Hostname returned by TableInputFormatBase.reverseDNS contains trailing period
     [HBASE-7694] - Secure HBase should use replication call queue
+    [HBASE-7698] - race between RS shutdown thread and openregionhandler causes region to get stuck
     [HBASE-7702] - Adding filtering to Import jobs
     [HBASE-7715] - FSUtils#waitOnSafeMode can incorrectly loop on standby NN
     [HBASE-7717] - Wait until regions are assigned in TestSplitTransactionOnCluster
@@ -64,6 +66,11 @@ Bug
     [HBASE-7730] - HBaseAdmin#synchronousBalanceSwitch is not compatible with 0.92
     [HBASE-7731] - Append/Increment methods in HRegion don't check whether the table is readonly or not
     [HBASE-7740] - Recheck matching row for joined scanners
+    [HBASE-7771] - Secure HBase Client in MR job causes tasks to wait forever
+    [HBASE-7772] - clusterId is not set in conf properly if only TableMapReduceUtil.initCredentials() is called
+    [HBASE-7776] - Use ErrorReporter/Log instead of System.out in hbck
+    [HBASE-7785] - rolling-restart.sh script unable to check expiration of master znode
+    [HBASE-7793] - Port HBASE-5564 Bulkload is discarding duplicate records to 0.94
 
 Improvement
 
@@ -73,10 +80,13 @@ Improvement
     [HBASE-5664] - CP hooks in Scan flow for fast forward when filter filters out a row
     [HBASE-7441] - Make ClusterManager in IntegrationTestingUtility pluggable
     [HBASE-7540] - Make znode dump to print a dump of replication znodes
+    [HBASE-7561] - Display the total number of regions for a given table on the master webUI
+    [HBASE-7757] - Add web UI to REST server and Thrift server
 
 New Feature
 
     [HBASE-6669] - Add BigDecimalColumnInterpreter for doing aggregations using AggregationClient
+    [HBASE-7748] - Add DelimitedKeyPrefixRegionSplitPolicy
 
 Wish
 

From 5b8fd6cd3872784d4901a0bcbd63c703187b7e13 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 11 Feb 2013 20:33:33 +0000
Subject: [PATCH 0773/1540] HBASE-7761 MemStore.USEMSLAB_DEFAULT is false,
 hbase-default.xml says it's true (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1444953 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/MemStore.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 9ee672044533..6e96df900c53 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -60,7 +60,7 @@ public class MemStore implements HeapSize {
 
   static final String USEMSLAB_KEY =
     "hbase.hregion.memstore.mslab.enabled";
-  private static final boolean USEMSLAB_DEFAULT = false;
+  private static final boolean USEMSLAB_DEFAULT = true;
 
   private Configuration conf;
 

From 1e260da24ee9c91c4a03e340ed28b10dd6f6b0ce Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 11 Feb 2013 21:34:00 +0000
Subject: [PATCH 0774/1540] HBASE-6132 ColumnCountGetFilter & PageFilter not
 working with FilterList (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1444972 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/FilterList.java       |   5 +-
 .../filter/TestColumnCountGetFilter.java      | 149 ++++++++++++++++++
 .../hadoop/hbase/filter/TestFilterList.java   |   4 -
 3 files changed, 151 insertions(+), 7 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 9521c7444a74..39465a569b6e 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -253,12 +253,11 @@ public boolean hasFilterRow() {
   public boolean filterRow() {
     for (Filter filter : filters) {
       if (operator == Operator.MUST_PASS_ALL) {
-        if (filter.filterAllRemaining() || filter.filterRow()) {
+        if (filter.filterRow()) {
           return true;
         }
       } else if (operator == Operator.MUST_PASS_ONE) {
-        if (!filter.filterAllRemaining()
-            && !filter.filterRow()) {
+        if (!filter.filterRow()) {
           return false;
         }
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java
new file mode 100644
index 000000000000..f725e618f600
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestColumnCountGetFilter {
+
+  private final static HBaseTestingUtility TEST_UTIL = new
+      HBaseTestingUtility();
+
+  @Test
+  public void testColumnCountGetFilter() throws IOException {
+    String family = "Family";
+    HTableDescriptor htd = new HTableDescriptor("testColumnCountGetFilter");
+    htd.addFamily(new HColumnDescriptor(family));
+    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+    HRegion region = HRegion.createHRegion(info, TEST_UTIL.
+      getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+    try {
+      String valueString = "ValueString";
+      String row = "row-1";
+      List<String> columns = generateRandomWords(10000, "column");
+      Put p = new Put(Bytes.toBytes(row));
+      p.setWriteToWAL(false);
+      for (String column : columns) {
+        KeyValue kv = KeyValueTestUtil.create(row, family, column, 0, valueString);
+        p.add(kv);
+      }
+      region.put(p);
+
+      Get get = new Get(row.getBytes());
+      Filter filter = new ColumnCountGetFilter(100);
+      get.setFilter(filter);
+      Scan scan = new Scan(get);
+      InternalScanner scanner = region.getScanner(scan);
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      scanner.next(results);
+      assertEquals(100, results.size());
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
+
+    region.close();
+    region.getLog().closeAndDelete();
+  }
+
+  @Test
+  public void testColumnCountGetFilterWithFilterList() throws IOException {
+    String family = "Family";
+    HTableDescriptor htd = new HTableDescriptor("testColumnCountGetFilter");
+    htd.addFamily(new HColumnDescriptor(family));
+    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+    HRegion region = HRegion.createHRegion(info, TEST_UTIL.
+      getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+    try {
+      String valueString = "ValueString";
+      String row = "row-1";
+      List<String> columns = generateRandomWords(10000, "column");
+      Put p = new Put(Bytes.toBytes(row));
+      p.setWriteToWAL(false);
+      for (String column : columns) {
+        KeyValue kv = KeyValueTestUtil.create(row, family, column, 0, valueString);
+        p.add(kv);
+      }
+      region.put(p);
+
+      Get get = new Get(row.getBytes());
+      FilterList filterLst = new FilterList ();
+      filterLst.addFilter( new ColumnCountGetFilter(100));
+      get.setFilter(filterLst);
+      Scan scan = new Scan(get);
+      InternalScanner scanner = region.getScanner(scan);
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      scanner.next(results);
+      assertEquals(100, results.size());
+    } finally {
+      region.close();
+      region.getLog().closeAndDelete();
+    }
+
+    region.close();
+    region.getLog().closeAndDelete();
+  }
+
+  List<String> generateRandomWords(int numberOfWords, String suffix) {
+    Set<String> wordSet = new HashSet<String>();
+    for (int i = 0; i < numberOfWords; i++) {
+      int lengthOfWords = (int) (Math.random()*2) + 1;
+      char[] wordChar = new char[lengthOfWords];
+      for (int j = 0; j < wordChar.length; j++) {
+        wordChar[j] = (char) (Math.random() * 26 + 97);
+      }
+      String word;
+      if (suffix == null) {
+        word = new String(wordChar);
+      } else {
+        word = new String(wordChar) + suffix;
+      }
+      wordSet.add(word);
+    }
+    List<String> wordList = new ArrayList<String>(wordSet);
+    return wordList;
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 05c0269e3d65..b3a1596954f6 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -104,7 +104,6 @@ public void testMPONE() throws Exception {
     /* We should filter any row */
     rowkey = Bytes.toBytes("z");
     assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
-    assertTrue(filterMPONE.filterRow());
     assertTrue(filterMPONE.filterAllRemaining());
 
   }
@@ -146,9 +145,6 @@ public void testMPALL() throws Exception {
     // Should fail here; row should be filtered out.
     KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey);
     assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterKeyValue(kv));
-
-    // Both filters in Set should be satisfied by now
-    assertTrue(filterMPALL.filterRow());
   }
 
   /**

From 1378b8c0be5d6f2c5dec5a8af44b1925290c7c9a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 11 Feb 2013 21:56:08 +0000
Subject: [PATCH 0775/1540] HBASE-7814 Port HBASE-6963 'unable to run hbck on a
 secure cluster' to 0.94 (Sergey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1444978 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/FSUtils.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index d5923d8a2074..f1cc677f91d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1200,7 +1200,7 @@ public static boolean delete(final FileSystem fs, final Path path, final boolean
    */
   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
       FsAction action) throws AccessControlException {
-    if (ugi.getUserName().equals(file.getOwner())) {
+    if (ugi.getShortUserName().equals(file.getOwner())) {
       if (file.getPermission().getUserAction().implies(action)) {
         return;
       }
@@ -1212,7 +1212,7 @@ public static void checkAccess(UserGroupInformation ugi, FileStatus file,
       return;
     }
     throw new AccessControlException("Permission denied:" + " action=" + action
-        + " path=" + file.getPath() + " user=" + ugi.getUserName());
+        + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
   }
 
   private static boolean contains(String[] groups, String user) {

From 93ec77ccb44bf6d0121d6acc609ef09077693e5c Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 12 Feb 2013 00:37:04 +0000
Subject: [PATCH 0776/1540] HBASE-7814 revert due to hadoop 0.20 compatibility
 concern

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1445004 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/FSUtils.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index f1cc677f91d9..d5923d8a2074 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1200,7 +1200,7 @@ public static boolean delete(final FileSystem fs, final Path path, final boolean
    */
   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
       FsAction action) throws AccessControlException {
-    if (ugi.getShortUserName().equals(file.getOwner())) {
+    if (ugi.getUserName().equals(file.getOwner())) {
       if (file.getPermission().getUserAction().implies(action)) {
         return;
       }
@@ -1212,7 +1212,7 @@ public static void checkAccess(UserGroupInformation ugi, FileStatus file,
       return;
     }
     throw new AccessControlException("Permission denied:" + " action=" + action
-        + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
+        + " path=" + file.getPath() + " user=" + ugi.getUserName());
   }
 
   private static boolean contains(String[] groups, String user) {

From d738315143ec02d4c27104248daf26ff5628f439 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 12 Feb 2013 15:47:11 +0000
Subject: [PATCH 0777/1540] HBASE-7829 zookeeper kerberos conf keytab and
 principal parameters interchanged (Francis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1445211 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 1147ad744f1b..d3206c6b3218 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -212,7 +212,7 @@ private static void login(Configuration conf, String keytabFileKey,
     // If keyTab is not specified use the Ticket Cache.
     // and set the zookeeper login context name.
     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
-      keytabFilename, principalName);
+        principalName, keytabFilename);
     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
     System.setProperty(loginContextProperty, loginContextName);
   }

From 883ec673361e3ab5700d326e5f321f803d4d9370 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 12 Feb 2013 20:45:29 +0000
Subject: [PATCH 0778/1540] HBASE-7521 fix HBASE-6060 (regions stuck in opening
 state) in 0.94 (Sergey and Rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1445350 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  96 +++++----
 .../master/handler/ServerShutdownHandler.java | 192 +++++++++++-------
 .../hbase/regionserver/HRegionServer.java     | 178 +++++++++++-----
 .../regionserver/RegionServerServices.java    |  15 +-
 .../handler/CloseRegionHandler.java           |   3 +-
 .../handler/OpenRegionHandler.java            |  50 +----
 .../apache/hadoop/hbase/master/Mocking.java   |  13 ++
 .../hbase/master/TestAssignmentManager.java   | 133 ++++++++++--
 .../master/TestZKBasedOpenCloseRegion.java    |  29 ++-
 .../handler/TestCloseRegionHandler.java       |  24 +--
 .../handler/TestOpenRegionHandler.java        |  52 ++++-
 .../hbase/util/MockRegionServerServices.java  |  18 +-
 12 files changed, 539 insertions(+), 264 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 1245c6ff3c88..03bd313df7de 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -274,6 +274,17 @@ public ServerName getRegionServerOfRegion(HRegionInfo hri) {
     }
   }
 
+  /**
+   * Checks whether the region is assigned.
+   * @param hri HRegion for which this function returns the result
+   * @return True iff assigned.
+   */
+  public boolean isRegionAssigned(HRegionInfo hri) {
+    synchronized (this.regions ) {
+      return regions.containsKey(hri);
+    }
+  }
+
   /**
    * Gives enabling table regions.
    * 
@@ -849,10 +860,10 @@ private void handleRegion(final RegionTransitionData data, int expectedVersion)
             break;
           }
           if (regionState == null ||
-              (!regionState.isPendingOpen() && !regionState.isOpening())) {
+              (!regionState.isOffline() && !regionState.isPendingOpen() && !regionState.isOpening())) {
             LOG.warn("Received FAILED_OPEN for region " + prettyPrintedRegionName +
                 " from server " + data.getOrigin() + " but region was in " +
-                " the state " + regionState + " and not in PENDING_OPEN or OPENING");
+                " the state " + regionState + " and not in OFFLINE, PENDING_OPEN or OPENING");
             return;
           }
           // Handle this the same as if it were opened and then closed.
@@ -874,16 +885,13 @@ private void handleRegion(final RegionTransitionData data, int expectedVersion)
             failoverProcessedRegions.put(encodedName, hri);
             break;
           }
-          // Should see OPENING after we have asked it to OPEN or additional
-          // times after already being in state of OPENING
           if (regionState == null ||
-              (!regionState.isPendingOpen() && !regionState.isOpening())) {
-            LOG.warn("Received OPENING for region " +
-                prettyPrintedRegionName +
-                " from server " + data.getOrigin() + " but region was in " +
-                " the state " + regionState + " and not " +
-                "in expected PENDING_OPEN or OPENING states");
-            return;
+              (!regionState.isOffline() && !regionState.isPendingOpen() &&
+                  !regionState.isOpening())) {
+              LOG.warn("Received OPENING for region " + prettyPrintedRegionName +  " from server " +
+                sn + " but region was in " + " the state " + regionState + " and not " +
+                "in expected OFFLINE, PENDING_OPEN or OPENING states");
+              return;
           }
           // Transition to OPENING (or update stamp if already OPENING)
           regionState.update(RegionState.State.OPENING,
@@ -903,12 +911,12 @@ private void handleRegion(final RegionTransitionData data, int expectedVersion)
           }
           // Should see OPENED after OPENING but possible after PENDING_OPEN
           if (regionState == null ||
-              (!regionState.isPendingOpen() && !regionState.isOpening())) {
+              (!regionState.isOffline() && !regionState.isPendingOpen() && !regionState.isOpening())) {
             LOG.warn("Received OPENED for region " +
                 prettyPrintedRegionName +
                 " from server " + data.getOrigin() + " but region was in " +
                 " the state " + regionState + " and not " +
-                "in expected PENDING_OPEN or OPENING states");
+                "in expected OFFLINE, PENDING_OPEN or OPENING states");
             return;
           }
           // Handle OPENED by removing from transition and deleted zk node
@@ -1525,7 +1533,7 @@ static class CreateUnassignedAsyncCallback implements AsyncCallback.StringCallba
     @Override
     public void processResult(int rc, String path, Object ctx, String name) {
       if (rc != 0) {
-        // Thisis resultcode.  If non-zero, need to resubmit.
+     // This is resultcode.  If non-zero, need to resubmit.
         LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
           "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
         this.zkw.abort("Connectionloss writing unassigned at " + path +
@@ -1684,14 +1692,17 @@ private void assign(final HRegionInfo region, final RegionState state,
       try {
         LOG.debug("Assigning region " + state.getRegion().getRegionNameAsString() +
           " to " + plan.getDestination().toString());
-        // Transition RegionState to PENDING_OPEN
-        state.update(RegionState.State.PENDING_OPEN, System.currentTimeMillis(),
-            plan.getDestination());
-        // Send OPEN RPC. This can fail if the server on other end is is not up.
-        // Pass the version that was obtained while setting the node to OFFLINE.
-        RegionOpeningState regionOpenState = serverManager.sendRegionOpen(plan
-            .getDestination(), state.getRegion(), versionOfOfflineNode);
-        if (regionOpenState == RegionOpeningState.ALREADY_OPENED) {
+        RegionOpeningState regionOpenState = serverManager.sendRegionOpen(plan.getDestination(),
+            state.getRegion(), versionOfOfflineNode);
+        if (regionOpenState == RegionOpeningState.OPENED) {
+          // Transition RegionState to PENDING_OPEN
+          if (state.isOffline() && !state.isOpening()) {
+            state.update(RegionState.State.PENDING_OPEN,
+                System.currentTimeMillis(), plan.getDestination());
+          }
+          if (state.isOpening()) return;
+          if (state.isOpened()) return;
+        } else if (regionOpenState == RegionOpeningState.ALREADY_OPENED) {
           // Remove region from in-memory transition and unassigned node from ZK
           // While trying to enable the table the regions of the table were
           // already enabled.
@@ -3191,12 +3202,14 @@ public boolean isCarryingRegion(ServerName serverName, HRegionInfo hri) {
 
     return matchAM;
   }
+
   /**
-   * Process shutdown server removing any assignments.
+   * Start processing of shutdown server.
    * @param sn Server that went down.
-   * @return list of regions in transition on this server
+   * @return Pair that has a set of regions in transition TO the dead server and
+   * a list of regions that were in transition, and also ON this server.
    */
-  public List<RegionState> processServerShutdown(final ServerName sn) {
+  public Pair<Set<HRegionInfo>, List<RegionState>> processServerShutdown(final ServerName sn) {
     // Clean out any existing assignment plans for this server
     synchronized (this.regionPlans) {
       for (Iterator <Map.Entry<String, RegionPlan>> i =
@@ -3213,30 +3226,36 @@ public List<RegionState> processServerShutdown(final ServerName sn) {
     // TODO: Do we want to sync on RIT here?
     // Remove this server from map of servers to regions, and remove all regions
     // of this server from online map of regions.
-    Set<HRegionInfo> deadRegions = null;
-    List<RegionState> rits = new ArrayList<RegionState>();
+    Set<HRegionInfo> deadRegions = new TreeSet<HRegionInfo>();
     synchronized (this.regions) {
       Set<HRegionInfo> assignedRegions = this.servers.remove(sn);
-      if (assignedRegions == null || assignedRegions.isEmpty()) {
-        // No regions on this server, we are done, return empty list of RITs
-        return rits;
-      }
-      deadRegions = new TreeSet<HRegionInfo>(assignedRegions);
-      for (HRegionInfo region : deadRegions) {
-        this.regions.remove(region);
+      if (assignedRegions != null && !assignedRegions.isEmpty()) {
+        deadRegions.addAll(assignedRegions);
+        for (HRegionInfo region : deadRegions) {
+          this.regions.remove(region);
+        }
       }
     }
     // See if any of the regions that were online on this server were in RIT
     // If they are, normal timeouts will deal with them appropriately so
     // let's skip a manual re-assignment.
+    Set<HRegionInfo> ritsGoingToServer = new ConcurrentSkipListSet<HRegionInfo>();
+    List<RegionState> ritsOnServer = new ArrayList<RegionState>();
     synchronized (regionsInTransition) {
-      for (RegionState region : this.regionsInTransition.values()) {
-        if (deadRegions.remove(region.getRegion())) {
-          rits.add(region);
+      for (RegionState state : this.regionsInTransition.values()) {
+        // If destination server in RegionState is same as dead server then add to regions to assign
+        // Skip the region in OFFLINE state because destionation server in RegionState is master
+        // server name. Skip the region if the destionation server in RegionState is other than dead
+        // server.
+        if ((state.getServerName() != null) && state.getServerName().equals(sn)) {
+          ritsGoingToServer.add(state.getRegion());
+        }
+        if (deadRegions.contains(state.getRegion())) {
+          ritsOnServer.add(state);
         }
       }
     }
-    return rits;
+    return new Pair<Set<HRegionInfo>, List<RegionState>>(ritsGoingToServer, ritsOnServer);
   }
 
   /**
@@ -3556,4 +3575,5 @@ protected void setEnabledTable(String tableName) {
       this.master.abort(errorMsg, e);
     }
   }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index a61e825d0f4b..8b889f7db921 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -20,9 +20,11 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -41,6 +43,8 @@
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -243,13 +247,6 @@ public void process() throws IOException {
         return;
       }
 
-      // Clean out anything in regions in transition.  Being conservative and
-      // doing after log splitting.  Could do some states before -- OPENING?
-      // OFFLINE? -- and then others after like CLOSING that depend on log
-      // splitting.
-      List<RegionState> regionsInTransition =
-        this.services.getAssignmentManager().
-          processServerShutdown(this.serverName);
 
       // Wait on meta to come online; we need it to progress.
       // TODO: Best way to hold strictly here?  We should build this retry logic
@@ -282,77 +279,134 @@ public void process() throws IOException {
         }
       }
 
-      // Skip regions that were in transition unless CLOSING or PENDING_CLOSE
-      for (RegionState rit : regionsInTransition) {
-        if (!rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()) {
-          LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
-          " from list of regions to assign because in RIT; region state: " +
-          rit.getState());
-          if (hris != null) hris.remove(rit.getRegion());
-        }
-      }
-
-      assert regionsInTransition != null;
-      LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) +
-        " region(s) that " + (serverName == null? "null": serverName)  +
-        " was carrying (skipping " +
-        regionsInTransition.size() +
-        " regions(s) that are already in transition)");
-
-      // Iterate regions that were on this server and assign them
-      if (hris != null) {
-        for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
-          RegionState rit = this.services.getAssignmentManager().isRegionInTransition(e.getKey());
-          if (processDeadRegion(e.getKey(), e.getValue(),
-              this.services.getAssignmentManager(),
-              this.server.getCatalogTracker())) {
-            ServerName addressFromAM = this.services.getAssignmentManager()
-                .getRegionServerOfRegion(e.getKey());
-            if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()) {
-              // Skip regions that were in transition unless CLOSING or
-              // PENDING_CLOSE
-              LOG.info("Skip assigning region " + rit.toString());
-            } else if (addressFromAM != null
-                && !addressFromAM.equals(this.serverName)) {
-              LOG.debug("Skip assigning region "
-                    + e.getKey().getRegionNameAsString()
-                    + " because it has been opened in "
-                    + addressFromAM.getServerName());
-              } else {
-                this.services.getAssignmentManager().assign(e.getKey(), true);
-              }
-          } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
-            // This will happen when the RS went down and the call back for the SPLIITING or SPLIT
-            // has not yet happened for node Deleted event.  In that case if the region was actually split
-            // but the RS had gone down before completing the split process then will not try to
-            // assign the parent region again. In that case we should make the region offline and
-            // also delete the region from RIT.
-            HRegionInfo region = rit.getRegion();
-            AssignmentManager am = this.services.getAssignmentManager();
-            am.regionOffline(region);
-          }
-          // If the table was partially disabled and the RS went down, we should clear the RIT
-          // and remove the node for the region.
-          // The rit that we use may be stale in case the table was in DISABLING state
-          // but though we did assign we will not be clearing the znode in CLOSING state.
-          // Doing this will have no harm. See HBASE-5927
-          if (rit != null
-              && (rit.isClosing() || rit.isPendingClose())
-              && this.services.getAssignmentManager().getZKTable()
-                  .isDisablingOrDisabledTable(rit.getRegion().getTableNameAsString())) {
-            HRegionInfo hri = rit.getRegion();
-            AssignmentManager am = this.services.getAssignmentManager();
-            am.deleteClosingOrClosedNode(hri);
-            am.regionOffline(hri);
+      // Returns set of regions that had regionplans against the downed server and a list of
+      // the intersection of regions-in-transition and regions that were on the server that died.
+      Pair<Set<HRegionInfo>, List<RegionState>> p = this.services.getAssignmentManager()
+          .processServerShutdown(this.serverName);
+      Set<HRegionInfo> ritsGoingToServer = p.getFirst();
+      List<RegionState> ritsOnServer = p.getSecond();
+
+      List<HRegionInfo> regionsToAssign = getRegionsToAssign(hris, ritsOnServer, ritsGoingToServer);
+      for (HRegionInfo hri : ritsGoingToServer) {
+        if (!this.services.getAssignmentManager().isRegionAssigned(hri)) {
+          if (!regionsToAssign.contains(hri)) {
+            regionsToAssign.add(hri);
           }
         }
       }
+      for (HRegionInfo hri : regionsToAssign) {
+        this.services.getAssignmentManager().assign(hri, true);
+      }
+      LOG.info(regionsToAssign.size() + " regions which were planned to open on " + this.serverName
+          + " have been re-assigned.");
     } finally {
       this.deadServers.finish(serverName);
     }
     LOG.info("Finished processing of shutdown of " + serverName);
   }
 
+  /**
+   * Figure what to assign from the dead server considering state of RIT and whats up in .META.
+   * @param metaHRIs Regions that .META. says were assigned to the dead server
+   * @param ritsOnServer Regions that were in transition, and on the dead server.
+   * @param ritsGoingToServer Regions that were in transition to the dead server.
+   * @return List of regions to assign or null if aborting.
+   * @throws IOException
+   */
+  private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Result> metaHRIs,
+      final List<RegionState> ritsOnServer, Set<HRegionInfo> ritsGoingToServer) throws IOException {
+    List<HRegionInfo> toAssign = new ArrayList<HRegionInfo>();
+    // If no regions on the server, then nothing to assign (Regions that were currently being
+    // assigned will be retried over in the AM#assign method).
+    if (metaHRIs == null || metaHRIs.isEmpty()) return toAssign;
+    // Remove regions that we do not want to reassign such as regions that are
+    // OFFLINE. If region is OFFLINE against this server, its probably being assigned over
+    // in the single region assign method in AM; do not assign it here too. TODO: VERIFY!!!
+    // TODO: Currently OFFLINE is too messy. Its done on single assign but bulk done when bulk
+    // assigning and then there is special handling when master joins a cluster.
+    //
+    // If split, the zk callback will have offlined. Daughters will be in the
+    // list of hris we got from scanning the .META. These should be reassigned. Not the parent.
+    for (RegionState rs : ritsOnServer) {
+      if (!rs.isClosing() && !rs.isPendingClose() && !rs.isSplitting()) {
+        LOG.debug("Removed " + rs.getRegion().getRegionNameAsString()
+            + " from list of regions to assign because region state: " + rs.getState());
+        metaHRIs.remove(rs.getRegion());
+      }
+    }
+
+    for (Map.Entry<HRegionInfo, Result> e : metaHRIs.entrySet()) {
+      RegionState rit = services.getAssignmentManager().getRegionsInTransition().get(
+          e.getKey().getEncodedName());
+      AssignmentManager assignmentManager = this.services.getAssignmentManager();
+      if (processDeadRegion(e.getKey(), e.getValue(), assignmentManager,
+          this.server.getCatalogTracker())) {
+        ServerName addressFromAM = assignmentManager.getRegionServerOfRegion(e.getKey());
+        if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()
+            && !ritsGoingToServer.contains(e.getKey())) {
+          // Skip regions that were in transition unless CLOSING or
+          // PENDING_CLOSE
+          LOG.info("Skip assigning region " + rit.toString());
+        } else if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
+          LOG.debug("Skip assigning region " + e.getKey().getRegionNameAsString()
+              + " because it has been opened in " + addressFromAM.getServerName());
+          ritsGoingToServer.remove(e.getKey());
+        } else {
+          if (rit != null) {
+            // clean zk node
+            try {
+              LOG.info("Reassigning region with rs =" + rit + " and deleting zk node if exists");
+              ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), e.getKey());
+            } catch (KeeperException ke) {
+              this.server.abort("Unexpected ZK exception deleting unassigned node " + e.getKey(),
+                  ke);
+              return null;
+            }
+          }
+          toAssign.add(e.getKey());
+        }
+      } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
+        // This will happen when the RS went down and the call back for the SPLIITING or SPLIT
+        // has not yet happened for node Deleted event. In that case if the region was actually
+        // split but the RS had gone down before completing the split process then will not try
+        // to assign the parent region again. In that case we should make the region offline
+        // and also delete the region from RIT.
+        HRegionInfo region = rit.getRegion();
+        AssignmentManager am = assignmentManager;
+        am.regionOffline(region);
+        ritsGoingToServer.remove(region);
+      }
+      // If the table was partially disabled and the RS went down, we should clear the RIT
+      // and remove the node for the region. The rit that we use may be stale in case the table
+      // was in DISABLING state but though we did assign we will not be clearing the znode in
+      // CLOSING state. Doing this will have no harm. See HBASE-5927
+      toAssign = checkForDisablingOrDisabledTables(ritsGoingToServer, toAssign, rit, assignmentManager);
+    }
+    return toAssign;
+  }
+
+  private List<HRegionInfo> checkForDisablingOrDisabledTables(Set<HRegionInfo> regionsFromRIT,
+      List<HRegionInfo> toAssign, RegionState rit, AssignmentManager assignmentManager) {
+    if (rit == null) {
+      return toAssign;
+    }
+    if (!rit.isClosing() && !rit.isPendingClose()) {
+      return toAssign;
+    }
+    if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(
+        rit.getRegion().getTableNameAsString())) {
+      return toAssign;
+    }
+    HRegionInfo hri = rit.getRegion();
+    AssignmentManager am = assignmentManager;
+    am.deleteClosingOrClosedNode(hri);
+    am.regionOffline(hri);
+    // To avoid region assignment if table is in disabling or disabled state.
+    toAssign.remove(hri);
+    regionsFromRIT.remove(hri);
+    return toAssign;
+  }
+
   /**
    * Process a dead region from a dead RS. Checks if the region is disabled or
    * disabling or if the region has a partially completed split.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 86d1337e5eff..8b13135d0b21 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -104,6 +104,7 @@
 import org.apache.hadoop.hbase.client.coprocessor.Exec;
 import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
@@ -157,6 +158,7 @@
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -2810,7 +2812,6 @@ public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNod
   private RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode,
       Map<String, HTableDescriptor> htds) throws IOException {
     checkOpen();
-    checkIfRegionInTransition(region, OPEN);
     HRegion onlineRegion = this.getFromOnlineRegions(region.getEncodedName());
     if (null != onlineRegion) {
       // See HBASE-5094. Cross check with META if still this RS is owning the
@@ -2827,46 +2828,103 @@ private RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNo
         this.removeFromOnlineRegions(region.getEncodedName());
       }
     }
-    LOG.info("Received request to open region: " +
-      region.getRegionNameAsString());
-    HTableDescriptor htd = null;
-    if (htds == null) {
-      htd = this.tableDescriptors.get(region.getTableName());
-    } else {
-      htd = htds.get(region.getTableNameAsString());
-      if (htd == null) {
+    // Added to in-memory RS RIT that we are trying to open this region.
+    // Clear it if we fail queuing an open executor.
+    addRegionsInTransition(region, OPEN);
+    try {
+      LOG.info("Received request to open region: " +
+        region.getRegionNameAsString());
+      HTableDescriptor htd = null;
+      if (htds == null) {
         htd = this.tableDescriptors.get(region.getTableName());
-        htds.put(region.getTableNameAsString(), htd);
-      }
-    }
-    this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(),
-        true);
-    // Need to pass the expected version in the constructor.
-    if (region.isRootRegion()) {
-      this.service.submit(new OpenRootHandler(this, this, region, htd,
-          versionOfOfflineNode));
-    } else if (region.isMetaRegion()) {
-      this.service.submit(new OpenMetaHandler(this, this, region, htd,
-          versionOfOfflineNode));
-    } else {
-      this.service.submit(new OpenRegionHandler(this, this, region, htd,
-          versionOfOfflineNode));
+      } else {
+        htd = htds.get(region.getTableNameAsString());
+        if (htd == null) {
+          htd = this.tableDescriptors.get(region.getTableName());
+          htds.put(region.getTableNameAsString(), htd);
+        }
+      }
+
+      // Mark the region as OPENING up in zk.  This is how we tell the master control of the
+      // region has passed to this regionserver.
+      int version = transitionZookeeperOfflineToOpening(region, versionOfOfflineNode);
+      // Need to pass the expected version in the constructor.
+      if (region.isRootRegion()) {
+        this.service.submit(new OpenRootHandler(this, this, region, htd, version));
+      } else if (region.isMetaRegion()) {
+        this.service.submit(new OpenMetaHandler(this, this, region, htd, version));
+      } else {
+        this.service.submit(new OpenRegionHandler(this, this, region, htd, version));
+      }
+    } catch (IOException ie) {
+      // Clear from this server's RIT list else will stick around for ever.
+      removeFromRegionsInTransition(region);
+      throw ie;
     }
     return RegionOpeningState.OPENED;
   }
 
-  private void checkIfRegionInTransition(HRegionInfo region,
-      String currentAction) throws RegionAlreadyInTransitionException {
-    byte[] encodedName = region.getEncodedNameAsBytes();
-    if (this.regionsInTransitionInRS.containsKey(encodedName)) {
-      boolean openAction = this.regionsInTransitionInRS.get(encodedName);
-      // The below exception message will be used in master.
-      throw new RegionAlreadyInTransitionException("Received:" + currentAction +
-        " for the region:" + region.getRegionNameAsString() +
-        " ,which we are already trying to " +
-        (openAction ? OPEN : CLOSE)+ ".");
-    }
-  }
+  /**
+   * Transition ZK node from OFFLINE to OPENING. The master will get a callback
+   * and will know that the region is now ours.
+   *
+   * @param hri
+   *          HRegionInfo whose znode we are updating
+   * @param versionOfOfflineNode
+   *          Version Of OfflineNode that needs to be compared before changing
+   *          the node's state from OFFLINE
+   * @throws IOException
+   */
+  int transitionZookeeperOfflineToOpening(final HRegionInfo hri, int versionOfOfflineNode)
+      throws IOException {
+    // TODO: should also handle transition from CLOSED?
+    int version = -1;
+    try {
+      // Initialize the znode version.
+      version = ZKAssign.transitionNode(this.zooKeeper, hri, this.getServerName(),
+          EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, versionOfOfflineNode);
+    } catch (KeeperException e) {
+      LOG.error("Error transition from OFFLINE to OPENING for region=" + hri.getEncodedName(), e);
+    }
+    if (version == -1) {
+      // TODO: Fix this sloppyness. The exception should be coming off zk
+      // directly, not an
+      // intepretation at this high-level (-1 when we call transitionNode can
+      // mean many things).
+      throw new IOException("Failed transition from OFFLINE to OPENING for region="
+          + hri.getEncodedName());
+    }
+    return version;
+  }
+
+   /**
+    * String currentAction) throws RegionAlreadyInTransitionException { Add
+    * region to this regionservers list of in transitions regions ONLY if its not
+    * already byte[] encodedName = region.getEncodedNameAsBytes(); in transition.
+    * If a region already in RIT, we throw
+    * {@link RegionAlreadyInTransitionException}. if
+    * (this.regionsInTransitionInRS.containsKey(encodedName)) { Callers need to
+    * call {@link #removeFromRegionsInTransition(HRegionInfo)} when done or if
+    * boolean openAction = this.regionsInTransitionInRS.get(encodedName); error
+    * processing.
+    *
+    * @param region
+    *          Region to add
+    * @param currentAction
+    *          Whether OPEN or CLOSE.
+    * @throws RegionAlreadyInTransitionException
+    */
+   protected void addRegionsInTransition(final HRegionInfo region, final String currentAction)
+       throws RegionAlreadyInTransitionException {
+     Boolean action = this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(),
+         currentAction.equals(OPEN));
+     if (action != null) {
+       // The below exception message will be used in master.
+       throw new RegionAlreadyInTransitionException("Received:" + currentAction + " for the region:"
+           + region.getRegionNameAsString() + " for the region:" + region.getRegionNameAsString()
+           + ", which we are already trying to " + (action ? OPEN : CLOSE) + ".");
+     }
+   }
 
   @Override
   @QosPriority(priority=HConstants.HIGH_QOS)
@@ -2919,7 +2977,6 @@ protected boolean closeRegion(HRegionInfo region, final boolean zk,
       throw new NotServingRegionException("Received close for "
         + region.getRegionNameAsString() + " but we are not serving it");
     }
-    checkIfRegionInTransition(region, CLOSE);
     return closeRegion(region, false, zk, versionOfClosingNode);
   }
 
@@ -2944,7 +3001,7 @@ protected boolean closeRegion(HRegionInfo region, final boolean abort,
   }
 
 
-    /**
+  /**
    * @param region Region to close
    * @param abort True if we are aborting
    * @param zk True if we are to update zk about the region close; if the close
@@ -2967,25 +3024,29 @@ protected boolean closeRegion(HRegionInfo region, final boolean abort,
         return false;
       }
     }
-    
-    if (this.regionsInTransitionInRS.containsKey(region.getEncodedNameAsBytes())) {
-      LOG.warn("Received close for region we are already opening or closing; " +
-          region.getEncodedName());
+    try {
+      addRegionsInTransition(region, CLOSE);
+    } catch (RegionAlreadyInTransitionException rate) {
+      LOG.warn("Received close for region we are already opening or closing; "
+          + region.getEncodedName());
       return false;
     }
-    this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(), false);
-    CloseRegionHandler crh = null;
-    if (region.isRootRegion()) {
-      crh = new CloseRootHandler(this, this, region, abort, zk,
-        versionOfClosingNode);
-    } else if (region.isMetaRegion()) {
-      crh = new CloseMetaHandler(this, this, region, abort, zk,
-        versionOfClosingNode);
-    } else {
-      crh = new CloseRegionHandler(this, this, region, abort, zk,
-        versionOfClosingNode);
+    boolean success = false;
+    try {
+      CloseRegionHandler crh = null;
+      if (region.isRootRegion()) {
+        crh = new CloseRootHandler(this, this, region, abort, zk, versionOfClosingNode);
+      } else if (region.isMetaRegion()) {
+        crh = new CloseMetaHandler(this, this, region, abort, zk, versionOfClosingNode);
+      } else {
+        crh = new CloseRegionHandler(this, this, region, abort, zk, versionOfClosingNode);
+      }
+      this.service.submit(crh);
+      success = true;
+    } finally {
+      // Remove from this server's RIT.
+      if (!success) removeFromRegionsInTransition(region);
     }
-    this.service.submit(crh);
     return true;
   }
 
@@ -3672,9 +3733,14 @@ public RegionServerCoprocessorHost getCoprocessorHost(){
     return this.rsHost;
   }
 
+  @Override
+  public boolean removeFromRegionsInTransition(final HRegionInfo hri) {
+    return this.regionsInTransitionInRS.remove(hri.getEncodedNameAsBytes());
+  }
 
-  public ConcurrentSkipListMap<byte[], Boolean> getRegionsInTransitionInRS() {
-    return this.regionsInTransitionInRS;
+  @Override
+  public boolean containsKeyInRegionsInTransition(final HRegionInfo hri) {
+    return this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes());
   }
 
   public ExecutorService getExecutorService() {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 21b24ac9dcf7..1c667048b560 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
@@ -75,10 +76,18 @@ public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
   public RpcServer getRpcServer();
 
   /**
-   * Get the regions that are currently being opened or closed in the RS
-   * @return map of regions in transition in this RS
+   * Remove passed <code>hri</code> from the internal list of regions in transition on this
+   * regionserver.
+   * @param hri Region to remove.
+   * @return True if removed
    */
-  public Map<byte[], Boolean> getRegionsInTransitionInRS();
+  public boolean removeFromRegionsInTransition(HRegionInfo hri);
+  /**
+   * @param hri
+   * @return True if the internal list of regions in transition includes the
+   *         passed <code>hri</code>.
+   */
+  public boolean containsKeyInRegionsInTransition(HRegionInfo hri);
 
   /**
    * @return Return the FileSystem object used by the regionserver
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
index 78eb4e619128..286a6dae1ef0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
@@ -148,8 +148,7 @@ public void process() {
       // Done!  Region is closed on this RS
       LOG.debug("Closed region " + region.getRegionNameAsString());
     } finally {
-      this.rsServices.getRegionsInTransitionInRS().
-          remove(this.regionInfo.getEncodedNameAsBytes());
+      this.rsServices.removeFromRegionsInTransition(this.regionInfo);
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 6b559097d034..241ad0ba154e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -52,8 +52,7 @@ public class OpenRegionHandler extends EventHandler {
   // the total open. We'll fail the open if someone hijacks our znode; we can
   // tell this has happened if version is not as expected.
   private volatile int version = -1;
-  //version of the offline node that was set by the master
-  private volatile int versionOfOfflineNode = -1;
+
 
   public OpenRegionHandler(final Server server,
       final RegionServerServices rsServices, HRegionInfo regionInfo,
@@ -62,20 +61,20 @@ public OpenRegionHandler(final Server server,
   }
   public OpenRegionHandler(final Server server,
       final RegionServerServices rsServices, HRegionInfo regionInfo,
-      HTableDescriptor htd, int versionOfOfflineNode) {
+      HTableDescriptor htd, int version) {
     this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION,
-        versionOfOfflineNode);
+        version);
   }
 
   protected OpenRegionHandler(final Server server,
       final RegionServerServices rsServices, final HRegionInfo regionInfo,
       final HTableDescriptor htd, EventType eventType,
-      final int versionOfOfflineNode) {
+      final int version) {
     super(server, eventType);
     this.rsServices = rsServices;
     this.regionInfo = regionInfo;
     this.htd = htd;
-    this.versionOfOfflineNode = versionOfOfflineNode;
+    this.version = version;
   }
 
   public HRegionInfo getRegionInfo() {
@@ -96,15 +95,6 @@ public void process() throws IOException {
       // Check that this region is not already online
       HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
 
-      // If fails, just return.  Someone stole the region from under us.
-      // Calling transitionZookeeperOfflineToOpening initalizes this.version.
-      if (!transitionZookeeperOfflineToOpening(encodedName,
-          versionOfOfflineNode)) {
-        LOG.warn("Region was hijacked? It no longer exists, encodedName=" +
-          encodedName);
-        return;
-      }
-
       // Open region.  After a successful open, failures in subsequent
       // processing needs to do a close as part of cleanup.
       region = openRegion();
@@ -144,8 +134,7 @@ public void process() throws IOException {
       LOG.debug("Opened " + name + " on server:" +
         this.server.getServerName());
     } finally {
-      this.rsServices.getRegionsInTransitionInRS().
-          remove(this.regionInfo.getEncodedNameAsBytes());
+      this.rsServices.removeFromRegionsInTransition(this.regionInfo);
       if (!openSuccessful && !transitionToFailedOpen) {
         tryTransitionToFailedOpen(regionInfo);
       }
@@ -370,33 +359,6 @@ void cleanupFailedOpen(final HRegion region) throws IOException {
     if (region != null) region.close();
   }
 
-  /**
-   * Transition ZK node from OFFLINE to OPENING.
-   * @param encodedName Name of the znode file (Region encodedName is the znode
-   * name).
-   * @param versionOfOfflineNode - version Of OfflineNode that needs to be compared
-   * before changing the node's state from OFFLINE 
-   * @return True if successful transition.
-   */
-  boolean transitionZookeeperOfflineToOpening(final String encodedName,
-      int versionOfOfflineNode) {
-    // TODO: should also handle transition from CLOSED?
-    try {
-      // Initialize the znode version.
-      this.version = ZKAssign.transitionNode(server.getZooKeeper(), regionInfo,
-          server.getServerName(), EventType.M_ZK_REGION_OFFLINE,
-          EventType.RS_ZK_REGION_OPENING, versionOfOfflineNode);
-    } catch (KeeperException e) {
-      LOG.error("Error transition from OFFLINE to OPENING for region=" +
-        encodedName, e);
-    }
-    boolean b = isGoodVersion();
-    if (!b) {
-      LOG.warn("Failed transition from OFFLINE to OPENING for region=" +
-        encodedName);
-    }
-    return b;
-  }
 
   /**
    * Update our OPENING state in zookeeper.
diff --git a/src/test/java/org/apache/hadoop/hbase/master/Mocking.java b/src/test/java/org/apache/hadoop/hbase/master/Mocking.java
index 187e07df806a..6dd379f38320 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/Mocking.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/Mocking.java
@@ -43,4 +43,17 @@ static void waitForRegionPendingOpenInRIT(AssignmentManager am, String encodedNa
       }
     }
   }
+
+  static void waitForRegionOfflineInRIT(AssignmentManager am, String encodedName)
+      throws InterruptedException {
+    boolean wait = true;
+    while (wait) {
+      RegionState state = am.getRegionsInTransition().get(encodedName);
+      if (state != null && state.isOffline()) {
+        wait = false;
+      } else {
+        Thread.sleep(1);
+      }
+    }
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index aee85370b602..5295dc25df8d 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -184,11 +184,17 @@ public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
       int versionid =
         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
       assertNotSame(versionid, -1);
-      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+      Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
 
-      // Get current versionid else will fail on transition from OFFLINE to
+      // Get the OFFLINE version id.  May have to wait some for it to happen.
       // OPENING below
-      versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+      while (true) {
+        int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+        if (vid != versionid) {
+          versionid = vid;
+          break;
+        }
+      }
       assertNotSame(-1, versionid);
       // This uglyness below is what the openregionhandler on RS side does.
       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
@@ -226,11 +232,17 @@ public void testBalanceOnMasterFailoverScenarioWithClosedNode()
         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
       assertNotSame(versionid, -1);
       am.gate.set(false);
-      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+      Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
 
       // Get current versionid else will fail on transition from OFFLINE to
       // OPENING below
-      versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+      while (true) {
+        int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+        if (vid != versionid) {
+          versionid = vid;
+          break;
+        }
+      }
       assertNotSame(-1, versionid);
       // This uglyness below is what the openregionhandler on RS side does.
       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
@@ -267,12 +279,18 @@ public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
       int versionid =
         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
       assertNotSame(versionid, -1);
-      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+      Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
 
       am.gate.set(false);
       // Get current versionid else will fail on transition from OFFLINE to
       // OPENING below
-      versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+      while (true) {
+        int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+        if (vid != versionid) {
+          versionid = vid;
+          break;
+        }
+      }
       assertNotSame(-1, versionid);
       // This uglyness below is what the openregionhandler on RS side does.
       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
@@ -347,9 +365,15 @@ public void testBalance()
       // balancer.  The zk node will be OFFLINE waiting for regionserver to
       // transition it through OPENING, OPENED.  Wait till we see the RIT
       // before we proceed.
-      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+      Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
       // Get current versionid else will fail on transition from OFFLINE to OPENING below
-      versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+      while (true) {
+        int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
+        if (vid != versionid) {
+          versionid = vid;
+          break;
+        }
+      }
       assertNotSame(-1, versionid);
       // This uglyness below is what the openregionhandler on RS side does.
       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
@@ -389,7 +413,7 @@ public void testShutdownHandler() throws KeeperException, IOException {
     AssignmentManager am =
       new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
     try {
-      processServerShutdownHandler(ct, am, false);
+      processServerShutdownHandler(ct, am, false, null);
     } finally {
       executor.shutdown();
       am.shutdown();
@@ -452,8 +476,7 @@ private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws Keep
     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
 
     try {
-
-      processServerShutdownHandler(ct, am, regionSplitDone);
+      processServerShutdownHandler(ct, am, regionSplitDone, null);
       // check znode deleted or not.
       // In both cases the znode should be deleted.
 
@@ -492,7 +515,7 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     am.regionOnline(REGIONINFO, SERVERNAME_A);
     // adding region in pending close.
     am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
-        State.PENDING_CLOSE));
+        State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
 
     if (state == TableState.DISABLING) {
       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
@@ -507,7 +530,7 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
 
     try {
-      processServerShutdownHandler(ct, am, false);
+      processServerShutdownHandler(ct, am, false, null);
       // check znode deleted or not.
       // In both cases the znode should be deleted.
       assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
@@ -525,7 +548,8 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     }
   }
 
-  private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion)
+  private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
+    boolean splitRegion, ServerName sn)
       throws IOException {
     // Make sure our new AM gets callbacks; once registered, can't unregister.
     // Thats ok because we make a new zk watcher for each test.
@@ -534,12 +558,22 @@ private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager a
     // Make an RS Interface implementation.  Make it so a scanner can go against it.
     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
     // Get a meta row result that has region up on SERVERNAME_A
+
     Result r = null;
-    if (splitRegion) {
-      r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
+    if (sn == null) {
+      if (splitRegion) {
+        r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
+      } else {
+        r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+      }
     } else {
-      r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+      if (sn.equals(SERVERNAME_A)) {
+        r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+      } else if (sn.equals(SERVERNAME_B)) {
+        r = new Result(new KeyValue[0]);
+      }
     }
+
     Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
       thenReturn(System.currentTimeMillis());
     // Return a good result first and then return null to indicate end of scan
@@ -563,8 +597,13 @@ private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager a
     // I need a services instance that will return the AM
     MasterServices services = Mockito.mock(MasterServices.class);
     Mockito.when(services.getAssignmentManager()).thenReturn(am);
-    ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
-      services, deadServers, SERVERNAME_A, false);
+    Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
+    ServerShutdownHandler handler = null;
+    if (sn != null) {
+      handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
+    } else {
+      handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
+    }
     handler.process();
     // The region in r will have been assigned.  It'll be up in zk as unassigned.
   }
@@ -850,7 +889,7 @@ public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, K
       assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
       assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
         newRegionPlan.getDestination())));
-      Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
+      Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
     } finally {
       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
         DefaultLoadBalancer.class, LoadBalancer.class);
@@ -950,6 +989,58 @@ public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOExc
     }
   }
 
+
+
+  /**
+   * When region in transition if region server opening the region gone down then region assignment
+   * taking long time(Waiting for timeout monitor to trigger assign). HBASE-5396(HBASE-6060) fixes this
+   * scenario. This test case verifies whether SSH calling assign for the region in transition or not.
+   *
+   * @throws KeeperException
+   * @throws IOException
+   * @throws ServiceException
+   */
+  @Test
+  public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
+      ServiceException {
+    testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
+    testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
+  }
+
+  private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
+      throws IOException, KeeperException, ServiceException {
+    // We need a mocked catalog tracker.
+    CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+    // Create an AM.
+    AssignmentManagerWithExtrasForTesting am =
+        setUpMockedAssignmentManager(this.server, this.serverManager);
+    // adding region in pending open.
+    if (regionInOffline) {
+      ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
+      am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
+          State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
+    } else {
+      am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
+          State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
+    }
+    // adding region plan
+    am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
+    am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
+
+    try {
+      processServerShutdownHandler(ct, am, false, SERVERNAME_A);
+      processServerShutdownHandler(ct, am, false, SERVERNAME_B);
+      if(regionInOffline){
+        assertFalse("Assign should not be invoked.", am.assignInvoked);
+      } else {
+        assertTrue("Assign should be invoked.", am.assignInvoked);
+      }
+    } finally {
+      am.regionsInTransition.remove(REGIONINFO.getEncodedName());
+      am.regionPlans.remove(REGIONINFO.getEncodedName());
+    }
+  }
+
   /**
    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
    * random assignment is called and the gate variable is set to true.
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
index 8c3f67e1e490..02555a8d98b1 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
@@ -69,6 +70,8 @@ public class TestZKBasedOpenCloseRegion {
 
   @BeforeClass public static void beforeAllTests() throws Exception {
     Configuration c = TEST_UTIL.getConfiguration();
+    c.setClass(HConstants.REGION_SERVER_IMPL, TestZKBasedOpenCloseRegionRegionServer.class,
+              HRegionServer.class);
     c.setBoolean("dfs.support.append", true);
     c.setInt("hbase.regionserver.info.port", 0);
     TEST_UTIL.startMiniCluster(2);
@@ -94,6 +97,22 @@ public class TestZKBasedOpenCloseRegion {
     waitUntilAllRegionsAssigned();
   }
 
+  /**
+   * Special HRegionServer used in these tests that allows access to
+   * {@link #addRegionsInTransition(HRegionInfo, String)}.
+   */
+  public static class TestZKBasedOpenCloseRegionRegionServer extends HRegionServer {
+    public TestZKBasedOpenCloseRegionRegionServer(Configuration conf)
+        throws IOException, InterruptedException {
+      super(conf);
+    }
+    @Override
+    public void addRegionsInTransition(HRegionInfo region,
+        String currentAction) throws RegionAlreadyInTransitionException {
+      super.addRegionsInTransition(region, currentAction);
+    }
+  }
+
   /**
    * Test we reopen a region once closed.
    * @throws Exception
@@ -244,8 +263,10 @@ public void testRSAlreadyProcessingRegion() throws Exception {
         cluster.getLiveRegionServerThreads().get(1).getRegionServer();
     HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
 
-    // fake that hr1 is processing the region
-    hr1.getRegionsInTransitionInRS().putIfAbsent(hri.getEncodedNameAsBytes(), true);
+    // Fake that hr1 is processing the region. At top of this test we made a
+    // regionserver that gave access addRegionsInTransition. Need to cast as
+    // TestZKBasedOpenCloseRegionRegionServer.
+    ((TestZKBasedOpenCloseRegionRegionServer) hr1).addRegionsInTransition(hri, "OPEN");
 
     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
     EventHandlerListener openListener =
@@ -262,7 +283,7 @@ public void testRSAlreadyProcessingRegion() throws Exception {
     assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null);
 
     // remove the block and reset the boolean
-    hr1.getRegionsInTransitionInRS().remove(hri.getEncodedNameAsBytes());
+    hr1.removeFromRegionsInTransition(hri);
     reopenEventProcessed.set(false);
     
     // now try moving a region when there is no region in transition.
@@ -333,7 +354,7 @@ public void testRegionOpenFailsDueToIOException() throws Exception {
     }
     Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState);
     assertFalse("Region should not be in RIT",
-        regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes()));
+        regionServer.containsKeyInRegionsInTransition(REGIONINFO));
   }
   
   private static void waitUntilAllRegionsAssigned()
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
index 07f8fc4d7696..10ccbacd5e22 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
@@ -204,18 +204,18 @@ private void OpenRegion(Server server, RegionServerServices rss,
            HTableDescriptor htd, HRegionInfo hri)
            throws IOException, NodeExistsException, KeeperException {
            // Create it OFFLINE node, which is what Master set before sending OPEN RPC
-           ZKAssign.createNodeOffline(server.getZooKeeper(), hri,
-             server.getServerName());
-           OpenRegionHandler openHandler = new OpenRegionHandler(server, rss, hri,
-             htd);
-           openHandler.process();
-           RegionTransitionData data =
-             ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName());
-       
-           // delete the node, which is what Master do after the region is opened
-           ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
-             EventType.RS_ZK_REGION_OPENED);
-         }  
+
+
+       ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
+       int version = ZKAssign.transitionNodeOpening(server.getZooKeeper(), hri, server.getServerName());
+       OpenRegionHandler openHandler = new OpenRegionHandler(server, rss, hri, htd, version);
+       openHandler.process();
+       RegionTransitionData data = ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName());
+
+       // delete the node, which is what Master do after the region is opened
+       ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
+         EventType.RS_ZK_REGION_OPENED);
+     }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
index 8958de53699f..3a8f1321fc98 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
@@ -19,16 +19,26 @@
  */
 package org.apache.hadoop.hbase.regionserver.handler;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.MockRegionServerServices;
@@ -57,13 +67,20 @@ public class TestOpenRegionHandler {
   private int testIndex = 0;
 
   @BeforeClass public static void before() throws Exception {
-    HTU.startMiniZKCluster();
+    Configuration c = HTU.getConfiguration();
+    c.setClass(HConstants.REGION_SERVER_IMPL, TestOpenRegionHandlerRegionServer.class,
+              HRegionServer.class);    
+    HTU.startMiniCluster();
     TEST_HTD = new HTableDescriptor("TestOpenRegionHandler.java");
   }
 
   @AfterClass public static void after() throws IOException {
     TEST_HTD = null;
-    HTU.shutdownMiniZKCluster();
+    try {
+      HTU.shutdownMiniCluster();
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
   }
 
   /**
@@ -135,6 +152,7 @@ public void testFailedOpenRegion() throws Exception {
 
     // Create it OFFLINE, which is what it expects
     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
+    ZKAssign.transitionNodeOpening(server.getZooKeeper(), TEST_HRI, server.getServerName());
 
     // Create the handler
     OpenRegionHandler handler =
@@ -160,7 +178,7 @@ public void testFailedUpdateMeta() throws Exception {
 
     // Create it OFFLINE, which is what it expects
     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
-
+    ZKAssign.transitionNodeOpening(server.getZooKeeper(), TEST_HRI, server.getServerName());
     // Create the handler
     OpenRegionHandler handler =
       new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD) {
@@ -178,14 +196,28 @@ boolean updateMeta(final HRegion r) {
     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, data.getEventType());
   }
   
+  public static class TestOpenRegionHandlerRegionServer extends HRegionServer {
+    public TestOpenRegionHandlerRegionServer(Configuration conf)
+        throws IOException, InterruptedException {
+      super(conf);
+    }
+    @Override
+    public void addRegionsInTransition(HRegionInfo region,
+        String currentAction) throws RegionAlreadyInTransitionException {
+      super.addRegionsInTransition(region, currentAction);
+    }
+  }
+  
   @Test
   public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception {
-    Server server = new MockServer(HTU);
-    RegionServerServices rsServices = new MockRegionServerServices();
+    MiniHBaseCluster cluster = HTU.getHBaseCluster();
+    HRegionServer server =
+        cluster.getLiveRegionServerThreads().get(0).getRegionServer();
     // Create it OFFLINE, which is what it expects
     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
+    ZKAssign.transitionNodeOpening(server.getZooKeeper(), TEST_HRI, server.getServerName());
     // Create the handler
-    OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD) {
+    OpenRegionHandler handler = new OpenRegionHandler(server, server, TEST_HRI, TEST_HTD) {
       @Override
       boolean updateMeta(HRegion r) {
         return false;
@@ -196,7 +228,7 @@ void cleanupFailedOpen(HRegion region) throws IOException {
         throw new IOException("FileSystem got closed.");
       }
     };
-    rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
+    ((TestOpenRegionHandlerRegionServer)server).addRegionsInTransition(TEST_HRI, "OPEN");
     try {
       handler.process();
     } catch (Exception e) {
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
index 6b25a5211b2a..f115fb29e016 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
@@ -25,6 +25,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.fs.HFileSystem;
@@ -90,11 +91,6 @@ public RpcServer getRpcServer() {
     return null;
   }
 
-  @Override
-  public ConcurrentSkipListMap<byte[], Boolean> getRegionsInTransitionInRS() {
-    return rit;
-  }
-
   @Override
   public FlushRequester getFlushRequester() {
     return null;
@@ -162,4 +158,16 @@ public void setFileSystem(FileSystem hfs) {
   public Leases getLeases() {
     return null;
   }
+
+  @Override
+  public boolean removeFromRegionsInTransition(HRegionInfo hri) {
+    // TODO Auto-generated method stub
+    return false;
+  }
+
+  @Override
+  public boolean containsKeyInRegionsInTransition(HRegionInfo hri) {
+    // TODO Auto-generated method stub
+    return false;
+  }
 }

From 8637dce8fe759d4cfea148c4a70d6fbda516720d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 12 Feb 2013 21:49:06 +0000
Subject: [PATCH 0779/1540] HBASE-7814 Port HBASE-6963 'unable to run hbck on a
 secure cluster' to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1445405 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/FSUtils.java   | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index d5923d8a2074..1dce78f24067 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -23,6 +23,7 @@
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -1200,7 +1201,19 @@ public static boolean delete(final FileSystem fs, final Path path, final boolean
    */
   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
       FsAction action) throws AccessControlException {
-    if (ugi.getUserName().equals(file.getOwner())) {
+    // See HBASE-7814. UserGroupInformation from hadoop 0.20.x may not support getShortUserName().
+    String username;
+    try {
+      Method m = UserGroupInformation.class.getMethod("getShortUserName", new Class<?>[]{});
+      username = (String) m.invoke(ugi);
+    } catch (NoSuchMethodException e) {
+      username = ugi.getUserName();
+    } catch (InvocationTargetException e) {
+      username = ugi.getUserName();      
+    } catch (IllegalAccessException iae) {
+      username = ugi.getUserName();      
+    }
+    if (username.equals(file.getOwner())) {
       if (file.getPermission().getUserAction().implies(action)) {
         return;
       }
@@ -1212,7 +1225,7 @@ public static void checkAccess(UserGroupInformation ugi, FileStatus file,
       return;
     }
     throw new AccessControlException("Permission denied:" + " action=" + action
-        + " path=" + file.getPath() + " user=" + ugi.getUserName());
+        + " path=" + file.getPath() + " user=" + username);
   }
 
   private static boolean contains(String[] groups, String user) {

From de4a68fb185d7dab31966a7ff2a15d927b3e8348 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 14 Feb 2013 03:08:16 +0000
Subject: [PATCH 0780/1540] HBASE-7832. Use User.getShortName() in FSUtils

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1446036 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/FSUtils.java | 22 +++++--------------
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  6 ++---
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 1dce78f24067..62f7ed5cd604 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -23,7 +23,6 @@
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -52,6 +51,7 @@
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.security.AccessControlException;
@@ -1192,32 +1192,22 @@ public static boolean delete(final FileSystem fs, final Path path, final boolean
   /**
    * Throw an exception if an action is not permitted by a user on a file.
    * 
-   * @param ugi
+   * @param user
    *          the user
    * @param file
    *          the file
    * @param action
    *          the action
    */
-  public static void checkAccess(UserGroupInformation ugi, FileStatus file,
+  public static void checkAccess(User user, FileStatus file,
       FsAction action) throws AccessControlException {
-    // See HBASE-7814. UserGroupInformation from hadoop 0.20.x may not support getShortUserName().
-    String username;
-    try {
-      Method m = UserGroupInformation.class.getMethod("getShortUserName", new Class<?>[]{});
-      username = (String) m.invoke(ugi);
-    } catch (NoSuchMethodException e) {
-      username = ugi.getUserName();
-    } catch (InvocationTargetException e) {
-      username = ugi.getUserName();      
-    } catch (IllegalAccessException iae) {
-      username = ugi.getUserName();      
-    }
+    // See HBASE-7814. UserGroupInformation from hadoop 0.20.x may not support getShortName().
+    String username = user.getShortName();
     if (username.equals(file.getOwner())) {
       if (file.getPermission().getUserAction().implies(action)) {
         return;
       }
-    } else if (contains(ugi.getGroupNames(), file.getGroup())) {
+    } else if (contains(user.getGroupNames(), file.getGroup())) {
       if (file.getPermission().getGroupAction().implies(action)) {
         return;
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 674ac1a08243..070a4f9e36d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1411,14 +1411,14 @@ private void preCheckPermission() throws IOException, AccessControlException {
 
     Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
     FileSystem fs = hbaseDir.getFileSystem(getConf());
-    UserGroupInformation ugi = User.getCurrent().getUGI();
+    User user = User.getCurrent();
     FileStatus[] files = fs.listStatus(hbaseDir);
     for (FileStatus file : files) {
       try {
-        FSUtils.checkAccess(ugi, file, FsAction.WRITE);
+        FSUtils.checkAccess(user, file, FsAction.WRITE);
       } catch (AccessControlException ace) {
         LOG.warn("Got AccessControlException when preCheckPermission ", ace);
-        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
+        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + user.getShortName()
           + " does not have write perms to " + file.getPath()
           + ". Please rerun hbck as hdfs user " + file.getOwner());
         throw new AccessControlException(ace);

From 9baf667686df69cffff410bca0d3ec5bd55dec50 Mon Sep 17 00:00:00 2001
From: Devaraj Das <ddas@apache.org>
Date: Fri, 15 Feb 2013 07:31:55 +0000
Subject: [PATCH 0781/1540] HBASE-7851. Include the guava classes as a
 dependency for jobs using mapreduce.TableMapReduceUtil (ddas).

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1446468 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                                      | 1 +
 .../org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 43da66851228..103b2918407c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -71,6 +71,7 @@ Bug
     [HBASE-7776] - Use ErrorReporter/Log instead of System.out in hbck
     [HBASE-7785] - rolling-restart.sh script unable to check expiration of master znode
     [HBASE-7793] - Port HBASE-5564 Bulkload is discarding duplicate records to 0.94
+    [HBASE-7851] - Include the guava classes as a dependency for jobs using mapreduce.TableMapReduceUtil
 
 Improvement
 
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 29ebff5fb82d..e49525920687 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -513,6 +513,7 @@ public static void addDependencyJars(Job job) throws IOException {
       addDependencyJars(job.getConfiguration(),
           org.apache.zookeeper.ZooKeeper.class,
           com.google.protobuf.Message.class,
+          com.google.common.collect.ImmutableSet.class,
           job.getMapOutputKeyClass(),
           job.getMapOutputValueClass(),
           job.getInputFormatClass(),

From 195cc8391a09a504deadd84e2b22b3eaea9436d8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 16 Feb 2013 17:45:58 +0000
Subject: [PATCH 0782/1540] Fixing CHANGES.txt and setting release to
 0.94.6-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1446918 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 -
 pom.xml     | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 103b2918407c..43da66851228 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -71,7 +71,6 @@ Bug
     [HBASE-7776] - Use ErrorReporter/Log instead of System.out in hbck
     [HBASE-7785] - rolling-restart.sh script unable to check expiration of master znode
     [HBASE-7793] - Port HBASE-5564 Bulkload is discarding duplicate records to 0.94
-    [HBASE-7851] - Include the guava classes as a dependency for jobs using mapreduce.TableMapReduceUtil
 
 Improvement
 
diff --git a/pom.xml b/pom.xml
index af285c0df335..451f9fb98093 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.5</version>
+  <version>0.94.6-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 20942d6d782b2b9352f9dc330df1d76015f6751d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 17 Feb 2013 23:52:20 +0000
Subject: [PATCH 0783/1540] HBASE-7867 setPreallocSize is different with
 COMMENT in setupTestEnv in MiniZooKeeperCluster.java (DaeMyung Kang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1447105 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
index 786ae2102182..2551e6868b26 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
@@ -121,7 +121,7 @@ private static void setupTestEnv() {
     // resulting in test failure (client timeout on first session).
     // set env and directly in order to handle static init/gc issues
     System.setProperty("zookeeper.preAllocSize", "100");
-    FileTxnLog.setPreallocSize(100);
+    FileTxnLog.setPreallocSize(100 * 1024);
   }
 
   public int startup(File baseDir) throws IOException, InterruptedException {

From 309f3990f731e670daf81830fb69d0f0b61d80cf Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Wed, 20 Feb 2013 14:15:21 +0000
Subject: [PATCH 0784/1540] HBASE-7870 Delete(byte [] row, long timestamp)
 constructor is deprecate and should not. (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1448176 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/Delete.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 26986e7f048e..4472dcb41b69 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -82,7 +82,7 @@ public Delete() {
    * @param row row key
    */
   public Delete(byte [] row) {
-    this(row, HConstants.LATEST_TIMESTAMP, null);
+    this(row, HConstants.LATEST_TIMESTAMP);
   }
 
   /**
@@ -96,8 +96,6 @@ public Delete(byte [] row) {
    * families or columns, you must specify each timestamp individually.
    * @param row row key
    * @param timestamp maximum version timestamp (only for delete row)
-   * @param rowLock previously acquired row lock, or null
-   * @deprecated {@link RowLock} is deprecated, use #de
    */
   public Delete(byte [] row, long timestamp) {
     this.row = row;

From ef6996ae4421824f8b57c06bdd46af9fc977b2d6 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Fri, 22 Feb 2013 16:29:46 +0000
Subject: [PATCH 0785/1540] 
 HBASE-7866-TestSplitTransactionOnCluster.testSplitBeforeSettingSplittingInZK
 failed 3 times in a row (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449106 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestSplitTransactionOnCluster.java        | 39 +++++++++++++++++--
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 94303b7df141..5918af811106 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -631,7 +631,7 @@ private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws Exc
         @Override
         int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent,
             ServerName serverName, int version) throws KeeperException, IOException {
-          throw new IOException();
+          throw new TransitionToSplittingFailedException();
         }
       };
     } else {
@@ -639,14 +639,35 @@ int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent,
         @Override
         void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
             throws KeeperException, IOException {
-          throw new IOException();
+          throw new SplittingNodeCreationFailedException ();
         }
       };
     }
+    String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
+        .getRegionInfo().getEncodedName());
+    // make sure the client is uptodate
+    regionServer.getZooKeeper().sync(node);
+    for (int i = 0; i < 100; i++) {
+      // We expect the znode to be deleted by this time. Here the znode could be in OPENED state and the 
+      // master has not yet deleted the znode.
+      if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
+        Thread.sleep(100);
+      }
+    }
+    
     try {
       st.execute(regionServer, regionServer);
     } catch (IOException e) {
-      String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
+      // check for the specific instance in case the Split failed due to the existence of the znode in OPENED state.
+      // This will at least make the test to fail;
+      if (nodeCreated) {
+        assertTrue("Should be instance of TransitionToSplittingFailedException",
+            e instanceof TransitionToSplittingFailedException);
+      } else {
+        assertTrue("Should be instance of CreateSplittingNodeFailedException",
+            e instanceof SplittingNodeCreationFailedException );
+      }
+      node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
           .getRegionInfo().getEncodedName());
       // make sure the client is uptodate
       regionServer.getZooKeeper().sync(node);
@@ -1047,5 +1068,17 @@ protected void startCatalogJanitorChore() {
       LOG.debug("Customised master executed.");
     }
   }
+  
+  private static class TransitionToSplittingFailedException extends IOException {
+    public TransitionToSplittingFailedException() {
+      super();
+    }
+  }
+
+  private static class SplittingNodeCreationFailedException  extends IOException {
+    public SplittingNodeCreationFailedException () {
+      super();
+    }
+  }
 }
 

From 8d292d32a26e66324e87eeafb189df99d9ad93d5 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 23 Feb 2013 02:25:52 +0000
Subject: [PATCH 0786/1540] HBASE-7913 Secure Rest server should login before
 getting an instance of Rest servlet (Arpit)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449263 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/Main.java    | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index 2a8f95686bd8..cb3134146499 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -76,8 +76,17 @@ private static void printUsageAndExit(Options options, int exitCode) {
   public static void main(String[] args) throws Exception {
     Log LOG = LogFactory.getLog("RESTServer");
 
-	VersionInfo.logVersion();
+    VersionInfo.logVersion();
     Configuration conf = HBaseConfiguration.create();
+    // login the server principal (if using secure Hadoop)
+    if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
+      String machineName = Strings.domainNamePointerToHostName(
+        DNS.getDefaultHost(conf.get("hbase.rest.dns.interface", "default"),
+          conf.get("hbase.rest.dns.nameserver", "default")));
+      User.login(conf, "hbase.rest.keytab.file", "hbase.rest.kerberos.principal",
+        machineName);
+    }
+
     RESTServlet servlet = RESTServlet.getInstance(conf);
 
     Options options = new Options();
@@ -170,15 +179,6 @@ public static void main(String[] args) throws Exception {
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
 
-    // login the server principal (if using secure Hadoop)
-    if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
-      String machineName = Strings.domainNamePointerToHostName(
-        DNS.getDefaultHost(conf.get("hbase.rest.dns.interface", "default"),
-          conf.get("hbase.rest.dns.nameserver", "default")));
-      User.login(conf, "hbase.rest.keytab.file", "hbase.rest.kerberos.principal",
-        machineName);
-    }
-
     // Put up info server.
     int port = conf.getInt("hbase.rest.info.port", 8085);
     if (port >= 0) {
@@ -193,4 +193,4 @@ public static void main(String[] args) throws Exception {
     server.start();
     server.join();
   }
-}
+}
\ No newline at end of file

From 04d1af89700f73cd877a3cef9fddac595ca451f5 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 23 Feb 2013 17:09:21 +0000
Subject: [PATCH 0787/1540] HBASE-7919 Wrong key is used in
 ServerManager#getServerConnection() to retrieve from Map serverConnections
 (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449372 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/ServerManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 41118cca9344..039702007613 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -556,7 +556,7 @@ public boolean sendRegionClose(ServerName server, HRegionInfo region,
    */
   private HRegionInterface getServerConnection(final ServerName sn)
   throws IOException {
-    HRegionInterface hri = this.serverConnections.get(sn.toString());
+    HRegionInterface hri = this.serverConnections.get(sn);
     if (hri == null) {
       LOG.debug("New connection to " + sn.toString());
       hri = this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());

From c253eac25ae2758945f6be1fffd5ac637ad14fcb Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 24 Feb 2013 16:02:17 +0000
Subject: [PATCH 0788/1540] HBASE-7916 HMaster uses wrong InetSocketAddress
 parameter to throw exception (Jean-Marc)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449500 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c68063001d16..e15b6e6bd0d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -266,7 +266,7 @@ public HMaster(final Configuration conf)
     // Creation of a HSA will force a resolve.
     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
     if (initialIsa.getAddress() == null) {
-      throw new IllegalArgumentException("Failed resolve of " + this.isa);
+      throw new IllegalArgumentException("Failed resolve of " + initialIsa);
     }
     int numHandlers = conf.getInt("hbase.master.handler.count",
       conf.getInt("hbase.regionserver.handler.count", 25));

From 99a9328680dfd91d953d8b86d1a1addf82eb033f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 25 Feb 2013 05:22:02 +0000
Subject: [PATCH 0789/1540] HBASE-7920 Move isFamilyEssential(byte[] name) out
 of Filter interface in 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449598 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/filter/Filter.java    |  8 --------
 .../hadoop/hbase/filter/FilterBase.java       | 19 +++++++++++++++++--
 .../hadoop/hbase/filter/FilterList.java       |  3 +--
 .../hadoop/hbase/filter/SkipFilter.java       |  2 +-
 .../hadoop/hbase/filter/WhileMatchFilter.java |  2 +-
 .../hadoop/hbase/regionserver/HRegion.java    |  3 ++-
 6 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
index eb8aac1cf3c8..ac1079f82bf4 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
@@ -167,12 +167,4 @@ public enum ReturnCode {
    * not sure which key to seek to next.
    */
   public KeyValue getNextKeyHint(final KeyValue currentKV);
-
-  /**
-   * Check that given column family is essential for filter to check row.  Most
-   * filters always return true here. But some could have more sophisticated
-   * logic which could significantly reduce scanning process by not even
-   * touching columns until we are 100% sure that it's data is needed in result.
-   */
-  public boolean isFamilyEssential(byte[] name);
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
index 271116448107..5d087f045c50 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
@@ -131,14 +131,29 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
   }
 
   /**
+   * Check that given column family is essential for filter to check row.  Most
+   * filters always return true here. But some could have more sophisticated
+   * logic which could significantly reduce scanning process by not even
+   * touching columns until we are 100% sure that it's data is needed in result.
+   *
    * By default, we require all scan's column families to be present. Our
    * subclasses may be more precise.
-   *
-   * @inheritDoc
    */
   public boolean isFamilyEssential(byte[] name) {
     return true;
   }
+  
+  /**
+   * Check that given column family is essential for filter to check row.
+   * This accommodates Filter implementation which didn't have this capability
+   * 
+   * @param filter
+   * @param name column family name
+   * @return whether column family is essential
+   */
+  public static boolean isFamilyEssential(Filter filter, byte[] name) {
+    return !(filter instanceof FilterBase) || ((FilterBase) filter).isFamilyEssential(name);
+  }
 
   /**
    * Given the filter's arguments it constructs the filter
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 39465a569b6e..f65065ce8dbb 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -316,10 +316,9 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
     return keyHint;
   }
 
-  @Override
   public boolean isFamilyEssential(byte[] name) {
     for (Filter filter : filters) {
-      if (filter.isFamilyEssential(name)) {
+      if (FilterBase.isFamilyEssential(filter, name)) {
         return true;
       }
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
index 8894a116acbf..c27ac175d303 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
@@ -105,7 +105,7 @@ public void readFields(DataInput in) throws IOException {
   }
 
   public boolean isFamilyEssential(byte[] name) {
-    return filter.isFamilyEssential(name);
+    return FilterBase.isFamilyEssential(this.filter, name);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
index e822ef4b6acb..591a247c1e1e 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
@@ -106,7 +106,7 @@ public void readFields(DataInput in) throws IOException {
   }
 
   public boolean isFamilyEssential(byte[] name) {
-    return filter.isFamilyEssential(name);
+    return FilterBase.isFamilyEssential(this.filter, name);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 18e9822f8f88..c20a0cc44f12 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -98,6 +98,7 @@
 import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.io.HeapSize;
@@ -3573,7 +3574,7 @@ public HRegionInfo getRegionInfo() {
         Store store = stores.get(entry.getKey());
         KeyValueScanner scanner = store.getScanner(scan, entry.getValue());
         if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand()
-          || this.filter.isFamilyEssential(entry.getKey())) {
+          || FilterBase.isFamilyEssential(this.filter, entry.getKey())) {
           scanners.add(scanner);
         } else {
           joinedScanners.add(scanner);

From 611a62ccbb20aa701e26a21919c762e1ec10e97d Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Mon, 25 Feb 2013 15:56:04 +0000
Subject: [PATCH 0790/1540] Updated pom.xml

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449756 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 451f9fb98093..14e54e0c77b4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -241,8 +241,8 @@
       <name>Ramkrishna S Vasudevan</name>
       <email>ramkrishna@apache.org</email>
       <timezone>+5</timezone>
-      <organization>Huawei</organization>
-      <organizationUrl>http://www.huawei.com</organizationUrl>
+      <organization>Intel</organization>
+      <organizationUrl>http://www.intel.in</organizationUrl>
     </developer>
   </developers>
 

From cf4fd267b68c34ec862d392bf333b3a1b6613306 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 25 Feb 2013 17:08:57 +0000
Subject: [PATCH 0791/1540] HBASE-7929 Reapply hbase-7507 "Make memstore flush
 be able to retry after exception" to 0.94 branch

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449779 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   | 11 ++++
 .../hadoop/hbase/regionserver/Store.java      | 55 ++++++++++++++++++-
 2 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e70bd3be708c..14a66421c76c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -477,6 +477,17 @@ public static enum Modify {
    */
   public static long DEFAULT_HBASE_CLIENT_PAUSE = 1000;
 
+  /**
+   * Parameter name for server pause value, used mostly as value to wait before
+   * running a retry of a failed operation.
+   */
+  public static String HBASE_SERVER_PAUSE = "hbase.server.pause";
+
+  /**
+   * Default value of {@link #HBASE_SERVER_PAUSE}.
+   */
+  public static int DEFAULT_HBASE_SERVER_PAUSE = 1000;
+
   /**
    * Parameter name for maximum retries, used as maximum for all retryable
    * operations such as fetching of the root region from root region server,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index b895688b6e16..6a11a24040cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -166,6 +166,10 @@ public class Store extends SchemaConfigured implements HeapSize {
 
   private final Compactor compactor;
 
+  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
+  private static int flush_retries_number;
+  private static int pauseTime;
+
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -245,6 +249,17 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.bytesPerChecksum = getBytesPerChecksum(conf);
     // Create a compaction tool instance
     this.compactor = new Compactor(this.conf);
+    if (Store.flush_retries_number == 0) {
+      Store.flush_retries_number = conf.getInt(
+          "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
+      Store.pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE,
+          HConstants.DEFAULT_HBASE_SERVER_PAUSE);
+      if (Store.flush_retries_number <= 0) {
+        throw new IllegalArgumentException(
+            "hbase.hstore.flush.retries.number must be > 0, not "
+                + Store.flush_retries_number);
+      }
+    }
   }
 
   /**
@@ -722,8 +737,43 @@ private Path flushCache(final long logCacheFlushId,
     // If an exception happens flushing, we let it out without clearing
     // the memstore snapshot.  The old snapshot will be returned when we say
     // 'snapshot', the next time flush comes around.
-    return internalFlushCache(
-        snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);
+    // Retry after catching exception when flushing, otherwise server will abort
+    // itself
+    IOException lastException = null;
+    for (int i = 0; i < Store.flush_retries_number; i++) {
+      try {
+        Path pathName = internalFlushCache(snapshot, logCacheFlushId,
+            snapshotTimeRangeTracker, flushedSize, status);
+        try {
+          // Path name is null if there is no entry to flush
+          if (pathName != null) {
+            validateStoreFile(pathName);
+          }
+          return pathName;
+        } catch (Exception e) {
+          LOG.warn("Failed validating store file " + pathName
+              + ", retring num=" + i, e);
+          if (e instanceof IOException) {
+            lastException = (IOException) e;
+          } else {
+            lastException = new IOException(e);
+          }
+        }
+      } catch (IOException e) {
+        LOG.warn("Failed flushing store file, retring num=" + i, e);
+        lastException = e;
+      }
+      if (lastException != null) {
+        try {
+          Thread.sleep(pauseTime);
+        } catch (InterruptedException e) {
+          IOException iie = new InterruptedIOException();
+          iie.initCause(e);
+          throw iie;
+        }
+      }
+    }
+    throw lastException;
   }
 
   /*
@@ -842,7 +892,6 @@ private StoreFile commitFile(final Path path,
     // Write-out finished successfully, move into the right spot
     String fileName = path.getName();
     Path dstPath = new Path(homedir, fileName);
-    validateStoreFile(path);
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);

From bbc76c8947590bbaf45a4fa62a0ad5273345cb91 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 25 Feb 2013 19:21:22 +0000
Subject: [PATCH 0792/1540] HBASE-7915 Secure ThriftServer needs to login
 before calling HBaseHandler (Arpit)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449819 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/thrift/ThriftServer.java | 11 +++++++++++
 .../hadoop/hbase/thrift/ThriftServerRunner.java      | 12 ------------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
index 1737c0552c01..43da992e72b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
@@ -30,9 +30,12 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.thrift.ThriftServerRunner.ImplType;
 import org.apache.hadoop.hbase.util.InfoServer;
+import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 
 /**
@@ -86,6 +89,14 @@ private static void printUsageAndExit(Options options, int exitCode)
    */
    void doMain(final String[] args) throws Exception {
      processOptions(args);
+     // login the server principal (if using secure Hadoop)
+     if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
+       String machineName = Strings.domainNamePointerToHostName(
+         DNS.getDefaultHost(conf.get("hbase.thrift.dns.interface", "default"),
+           conf.get("hbase.thrift.dns.nameserver", "default")));
+       User.login(conf, "hbase.thrift.keytab.file",
+           "hbase.thrift.kerberos.principal", machineName);
+     }
      serverRunner = new ThriftServerRunner(conf);
 
      // Put up info server.
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 36fd7fec8138..98ee569410ae 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -66,7 +66,6 @@
 import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.thrift.CallQueue.Call;
 import org.apache.hadoop.hbase.thrift.generated.AlreadyExists;
 import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
@@ -82,9 +81,7 @@
 import org.apache.hadoop.hbase.thrift.generated.TScan;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.net.DNS;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TCompactProtocol;
@@ -366,15 +363,6 @@ private void setupServer() throws Exception {
           tserver.getClass().getName());
     }
 
-    // login the server principal (if using secure Hadoop)
-    if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
-      String machineName = Strings.domainNamePointerToHostName(
-        DNS.getDefaultHost(conf.get("hbase.thrift.dns.interface", "default"),
-          conf.get("hbase.thrift.dns.nameserver", "default")));
-      User.login(conf, "hbase.thrift.keytab.file",
-          "hbase.thrift.kerberos.principal", machineName);
-    }
-
     registerFilters(conf);
   }
 

From e197cb509e0dc9cc994f292a1130b13bf9a55df7 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 25 Feb 2013 22:01:57 +0000
Subject: [PATCH 0793/1540] HBASE-7824 Improve master start up time when there
 is log splitting work (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449920 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  |   2 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 110 +++++++----
 .../hadoop/hbase/master/MasterFileSystem.java |  33 ++--
 .../hadoop/hbase/master/ServerManager.java    |  10 +
 .../master/handler/ServerShutdownHandler.java |   4 +
 .../hbase/regionserver/HRegionServer.java     |   7 +-
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |   2 +-
 .../master/TestDistributedLogSplitting.java   | 178 ++++++++++++++++--
 .../TestRSKilledWhenMasterInitializing.java   |  25 +--
 9 files changed, 275 insertions(+), 96 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 5f2f1482c7f2..58c61ee1703d 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -118,7 +118,7 @@ public class CatalogTracker {
    */
   private ServerName metaLocation;
 
-  private boolean stopped = false;
+  private volatile boolean stopped = false;
 
   static final byte [] ROOT_REGION_NAME =
     HRegionInfo.ROOT_REGIONINFO.getRegionName();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index e15b6e6bd0d1..035775961ef0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -41,9 +41,6 @@
 
 import javax.management.ObjectName;
 
-import com.google.common.collect.ClassToInstanceMap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MutableClassToInstanceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -123,6 +120,10 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
+
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
  * master.  If many masters are started, all compete.  Whichever wins goes on to
@@ -552,14 +553,37 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     if (!masterRecovery) {
       this.assignmentManager.startTimeOutMonitor();
     }
-    // TODO: Should do this in background rather than block master startup
-    status.setStatus("Splitting logs after master startup");
-    splitLogAfterStartup(this.fileSystemManager);
 
-    // Make sure root and meta assigned before proceeding.
-    assignRootAndMeta(status);
+    // get a list for previously failed RS which need recovery work
+    Set<ServerName> failedServers = this.fileSystemManager.getFailedServersFromLogFolders();
+    ServerName preRootServer = this.catalogTracker.getRootLocation();
+    if (preRootServer != null && failedServers.contains(preRootServer)) {
+      // create recovered edits file for _ROOT_ server
+      this.fileSystemManager.splitLog(preRootServer);
+      failedServers.remove(preRootServer);
+    }
+
+    // Make sure root assigned before proceeding.
+    assignRoot(status);
+
+    // log splitting for .META. server
+    ServerName preMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+    if (preMetaServer != null && failedServers.contains(preMetaServer)) {
+      // create recovered edits file for .META. server
+      this.fileSystemManager.splitLog(preMetaServer);
+      failedServers.remove(preMetaServer);
+    }
+    // Make sure meta assigned before proceeding.
+    assignMeta(status, preRootServer);
+
     enableServerShutdownHandler();
 
+    // handle other dead servers in SSH
+    status.setStatus("Submit log splitting work of non-meta region servers");
+    for (ServerName curServer : failedServers) {
+      this.serverManager.processDeadServer(curServer);
+    }
+
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
     // before we assign all user regions or else the assignment will fail.
@@ -631,22 +655,13 @@ protected void startCatalogJanitorChore() {
   }
 
   /**
-   * Override to change master's splitLogAfterStartup. Used testing
-   * @param mfs
-   */
-  protected void splitLogAfterStartup(final MasterFileSystem mfs) {
-    mfs.splitLogAfterStartup();
-  }
-
-  /**
-   * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,
-   * assign them.
+   * Check <code>-ROOT-</code> is assigned. If not, assign it.
+   * @param status MonitoredTask
    * @throws InterruptedException
    * @throws IOException
    * @throws KeeperException
-   * @return Count of regions we assigned.
    */
-  int assignRootAndMeta(MonitoredTask status)
+  private void assignRoot(MonitoredTask status)
   throws InterruptedException, IOException, KeeperException {
     int assigned = 0;
     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
@@ -677,16 +692,32 @@ int assignRootAndMeta(MonitoredTask status)
     LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getRootLocation());
 
-    // Work on meta region
+    status.setStatus("ROOT assigned.");
+  }
+
+  /**
+   * Check <code>.META.</code> is assigned. If not, assign it.
+   * @param status MonitoredTask
+   * @param previousRootServer ServerName of previous root region server before current start up
+   * @return
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private void assignMeta(MonitoredTask status, ServerName previousRootServer)
+      throws InterruptedException,
+      IOException, KeeperException {
+    int assigned = 0;
+    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
+
     status.setStatus("Assigning META region");
-    rit = this.assignmentManager.
-      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
+    boolean rit =
+        this.assignmentManager
+            .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
     boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
     if (!rit && !metaRegionLocation) {
-      ServerName currentMetaServer =
-        this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-      if (currentMetaServer != null
-          && !currentMetaServer.equals(currentRootServer)) {
+      ServerName currentMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+      if (currentMetaServer != null && !currentMetaServer.equals(previousRootServer)) {
         splitLogAndExpireIfOnline(currentMetaServer);
       }
       assignmentManager.assignMeta();
@@ -696,15 +727,14 @@ int assignRootAndMeta(MonitoredTask status)
       enableSSHandWaitForMeta();
       assigned++;
     } else {
-      // Region already assigned.  We didnt' assign it.  Add to in-memory state.
+      // Region already assigned. We didnt' assign it. Add to in-memory state.
       this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
         this.catalogTracker.getMetaLocation());
     }
     enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
-    LOG.info(".META. assigned=" + assigned + ", rit=" + rit +
-      ", location=" + catalogTracker.getMetaLocation());
-    status.setStatus("META and ROOT assigned.");
-    return assigned;
+    LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
+        + catalogTracker.getMetaLocation());
+    status.setStatus("META assigned.");
   }
 
   private void enableSSHandWaitForMeta() throws IOException,
@@ -763,8 +793,7 @@ public boolean visit(Result r) throws IOException {
   }
 
   /**
-   * Split a server's log and expire it if we find it is one of the online
-   * servers.
+   * Expire a server if we find it is one of the online servers.
    * @param sn ServerName to check.
    * @throws IOException
    */
@@ -1626,12 +1655,23 @@ public void shutdown() {
     }
     if (this.assignmentManager != null) this.assignmentManager.shutdown();
     if (this.serverManager != null) this.serverManager.shutdownCluster();
+
     try {
       if (this.clusterStatusTracker != null){
         this.clusterStatusTracker.setClusterDown();
       }
     } catch (KeeperException e) {
-      LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      if (e instanceof KeeperException.SessionExpiredException) {
+        LOG.warn("ZK session expired. Retry a new connection...");
+        try {
+          this.zooKeeper.reconnectAfterExpiration();
+          this.clusterStatusTracker.setClusterDown();
+        } catch (Exception ex) {
+          LOG.warn("Retry setClusterDown failed", ex);
+        }
+      } else {
+        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 694ba0d20b8e..5c7b8aa1e778 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
@@ -184,30 +185,31 @@ public String getClusterId() {
   }
 
   /**
-   * Inspect the log directory to recover any log file without
-   * an active region server.
+   * Inspect the log directory to find dead servers which need log splitting
    */
-  void splitLogAfterStartup() {
+  Set<ServerName> getFailedServersFromLogFolders() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
-        HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+      HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+    
+    Set<ServerName> serverNames = new HashSet<ServerName>();
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
+
     do {
       if (master.isStopped()) {
-        LOG.warn("Master stopped while splitting logs");
+        LOG.warn("Master stopped while trying to get failed servers.");
         break;
       }
-      List<ServerName> serverNames = new ArrayList<ServerName>();
       try {
-        if (!this.fs.exists(logsDirPath)) return;
+        if (!this.fs.exists(logsDirPath)) return serverNames;
         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
         // Get online servers after getting log folders to avoid log folder deletion of newly
         // checked in region servers . see HBASE-5916
-        Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
-            .keySet();
+        Set<ServerName> onlineServers =
+            ((HMaster) master).getServerManager().getOnlineServers().keySet();
 
         if (logFolders == null || logFolders.length == 0) {
           LOG.debug("No log files to split, proceeding...");
-          return;
+          return serverNames;
         }
         for (FileStatus status : logFolders) {
           String sn = status.getPath().getName();
@@ -221,22 +223,19 @@ void splitLogAfterStartup() {
                 + "to a known region server, splitting");
             serverNames.add(serverName);
           } else {
-            LOG.info("Log folder " + status.getPath()
-                + " belongs to an existing region server");
+            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
           }
         }
-        splitLog(serverNames);
         retrySplitting = false;
       } catch (IOException ioe) {
-        LOG.warn("Failed splitting of " + serverNames, ioe);
+        LOG.warn("Failed getting failed servers to be recovered.", ioe);
         if (!checkFileSystem()) {
           LOG.warn("Bad Filesystem, exiting");
           Runtime.getRuntime().halt(1);
         }
         try {
           if (retrySplitting) {
-            Thread.sleep(conf.getInt(
-              "hbase.hlog.split.failure.retry.interval", 30 * 1000));
+            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
           }
         } catch (InterruptedException e) {
           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
@@ -246,6 +245,8 @@ void splitLogAfterStartup() {
         }
       }
     } while (retrySplitting);
+
+    return serverNames;
   }
 
   public void splitLog(final ServerName serverName) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 039702007613..6d5c45a3cdc7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -424,6 +424,16 @@ public synchronized void expireServer(final ServerName serverName) {
         carryingRoot + ", meta=" + carryingMeta);
   }
 
+  /**
+   * The function is to allow master to submit known dead servers into SSH
+   * @param serverName
+   */
+  void processDeadServer(final ServerName serverName) {
+    this.deadservers.add(serverName);
+    this.services.getExecutorService().submit(
+      new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName, true));
+  }
+
   /**
    * Expire the servers which died during master's initialization. It will be
    * called after HMaster#assignRootAndMeta.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 8b889f7db921..b545891c1d2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -179,6 +179,10 @@ public String toString() {
   public void process() throws IOException {
     final ServerName serverName = this.serverName;
     try {
+      if (this.server.isStopped()) {
+        throw new IOException("Server is stopped");
+      }
+
       try {
         if (this.shouldSplitHlog) {
           LOG.info("Splitting logs for " + serverName);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 8b13135d0b21..b780b8e04296 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,12 +66,12 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
-import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterAddressTracker;
 import org.apache.hadoop.hbase.NotServingRegionException;
@@ -171,7 +171,6 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.joda.time.field.MillisDurationField;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
@@ -1678,7 +1677,9 @@ public CatalogTracker getCatalogTracker() {
   @Override
   public void stop(final String msg) {
     try {
-      this.rsHost.preStop(msg);
+      if (this.rsHost != null) {
+        this.rsHost.preStop(msg);
+      }
       this.stopped = true;
       LOG.info("STOPPED: " + msg);
       // Wakes run() if it is sleeping
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index c6e607ef1334..4365f78e7172 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -45,7 +45,7 @@ public abstract class ZooKeeperNodeTracker extends ZooKeeperListener {
   /** Used to abort if a fatal error occurs */
   protected final Abortable abortable;
 
-  private boolean stopped = false;
+  private volatile boolean stopped = false;
 
   /**
    * Constructs a new ZK node tracker.
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 823d7da933de..36a3c547e137 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -19,22 +19,29 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_mgr_wait_for_zk_delete;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_final_transistion_failed;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_done;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_err;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_resigned;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,7 +50,13 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -54,8 +67,9 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -82,15 +96,21 @@ public class TestDistributedLogSplitting {
   Configuration conf;
   HBaseTestingUtility TEST_UTIL;
 
+
   private void startCluster(int num_rs) throws Exception{
+    conf = HBaseConfiguration.create();
+    startCluster(NUM_MASTERS, num_rs, conf);
+  }
+
+  private void startCluster(int num_master, int num_rs, Configuration inConf) throws Exception {
     ZKSplitLog.Counters.resetCounters();
     LOG.info("Starting cluster");
-    conf = HBaseConfiguration.create();
+    this.conf = inConf;
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
     TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(NUM_MASTERS, num_rs);
+    TEST_UTIL.startMiniCluster(num_master, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
@@ -102,6 +122,10 @@ private void startCluster(int num_rs) throws Exception{
 
   @After
   public void after() throws Exception {
+    for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
+      mt.getMaster().abort("closing...", new Exception("Trace info"));
+    }
+
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -205,6 +229,89 @@ public void testRecoveredEdits() throws Exception {
     assertEquals(NUM_LOG_LINES, count);
   }
 
+  @Test(timeout = 300000)
+  public void testMasterStartsUpWithLogSplittingWork() throws Exception {
+    LOG.info("testMasterStartsUpWithLogSplittingWork");
+    Configuration curConf = HBaseConfiguration.create();
+    curConf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
+    startCluster(2, NUM_RS, curConf);
+
+    final int NUM_REGIONS_TO_CREATE = 40;
+    final int NUM_LOG_LINES = 1000;
+    // turn off load balancing to prevent regions from moving around otherwise
+    // they will consume recovered.edits
+    master.balanceSwitch(false);
+
+    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
+    HTable ht = installTable(zkw, "table", "f", NUM_REGIONS_TO_CREATE);
+
+    List<HRegionInfo> regions = null;
+    HRegionServer hrs = null;
+    for (int i = 0; i < NUM_RS; i++) {
+      boolean isCarryingMeta = false;
+      hrs = rsts.get(i).getRegionServer();
+      regions = hrs.getOnlineRegions();
+      for (HRegionInfo region : regions) {
+        if (region.isRootRegion() || region.isMetaRegion()) {
+          isCarryingMeta = true;
+          break;
+        }
+      }
+      if (isCarryingMeta) {
+        continue;
+      }
+      break;
+    }
+
+    LOG.info("#regions = " + regions.size());
+    Iterator<HRegionInfo> it = regions.iterator();
+    while (it.hasNext()) {
+      HRegionInfo region = it.next();
+      if (region.isMetaTable()) {
+        it.remove();
+      }
+    }
+    makeHLog(hrs.getWAL(), regions, "table", NUM_LOG_LINES, 100);
+
+    // abort master
+    abortMaster(cluster);
+
+    // abort RS
+    int numRS = cluster.getLiveRegionServerThreads().size();
+    LOG.info("Aborting region server: " + hrs.getServerName());
+    hrs.abort("testing");
+
+    // wait for the RS dies
+    long start = EnvironmentEdgeManager.currentTimeMillis();
+    while (cluster.getLiveRegionServerThreads().size() > (numRS - 1)) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue(false);
+      }
+      Thread.sleep(200);
+    }
+
+    Thread.sleep(2000);
+    LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
+    
+    startMasterAndWaitUntilLogSplit(cluster);
+    
+    start = EnvironmentEdgeManager.currentTimeMillis();
+    while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue("Timedout", false);
+      }
+      Thread.sleep(200);
+    }
+
+    LOG.info("Current Open Regions After Master Node Starts Up:"
+        + getAllOnlineRegions(cluster).size());
+
+    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+
+    ht.close();
+  }
+
   /**
    * The original intention of this test was to force an abort of a region
    * server and to make sure that the failure path in the region servers is
@@ -393,33 +500,40 @@ public void makeHLog(HLog log,
       List<HRegionInfo> hris, String tname,
       int num_edits, int edit_size) throws IOException {
 
+    // remove root and meta region
+    hris.remove(HRegionInfo.ROOT_REGIONINFO);
+    hris.remove(HRegionInfo.FIRST_META_REGIONINFO);
     byte[] table = Bytes.toBytes(tname);
     HTableDescriptor htd = new HTableDescriptor(tname);
     byte[] value = new byte[edit_size];
     for (int i = 0; i < edit_size; i++) {
-      value[i] = (byte)('a' + (i % 26));
+      value[i] = (byte) ('a' + (i % 26));
     }
     int n = hris.size();
     int[] counts = new int[n];
-    int j = 0;
     if (n > 0) {
       for (int i = 0; i < num_edits; i += 1) {
         WALEdit e = new WALEdit();
-        byte [] row = Bytes.toBytes("r" + Integer.toString(i));
-        byte [] family = Bytes.toBytes("f");
-        byte [] qualifier = Bytes.toBytes("c" + Integer.toString(i));
-        e.add(new KeyValue(row, family, qualifier,
-            System.currentTimeMillis(), value));
-        j++;
-        log.append(hris.get(j % n), table, e, System.currentTimeMillis(), htd);
-        counts[j % n] += 1;
+        HRegionInfo curRegionInfo = hris.get(i % n);
+        byte[] startRow = curRegionInfo.getStartKey();
+        if (startRow == null || startRow.length == 0) {
+          startRow = new byte[] { 0, 0, 0, 0, 1 };
+        }
+        byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
+        row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because
+                                             // HBaseTestingUtility.createMultiRegions use 5 bytes
+                                             // key
+        byte[] family = Bytes.toBytes("f");
+        byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
+        e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
+        log.append(curRegionInfo, table, e, System.currentTimeMillis(), htd);
+        counts[i % n] += 1;
       }
     }
     log.sync();
     log.close();
     for (int i = 0; i < n; i++) {
-      LOG.info("region " + hris.get(i).getRegionNameAsString() +
-          " has " + counts[i] + " edits");
+      LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
     }
     return;
   }
@@ -479,6 +593,30 @@ private void waitForCounter(AtomicLong ctr, long oldval, long newval,
     assertTrue(false);
   }
 
+  private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException {
+    for (MasterThread mt : cluster.getLiveMasterThreads()) {
+      if (mt.getMaster().isActiveMaster()) {
+        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
+        mt.join();
+        break;
+      }
+    }
+    LOG.debug("Master is aborted");
+  }
+
+  private void startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
+      throws IOException, InterruptedException {
+    cluster.startMaster();
+    HMaster master = cluster.getMaster();
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    ServerManager serverManager = master.getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
+      Thread.sleep(100);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index a7287a1b84a2..eb2f42dccc87 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -25,7 +25,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -37,19 +36,16 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.TestMasterFailover;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -102,19 +98,6 @@ public TestingMaster(Configuration conf) throws IOException,
       super(conf);
     }
 
-    @Override
-    protected void splitLogAfterStartup(MasterFileSystem mfs) {
-      super.splitLogAfterStartup(mfs);
-      logSplit = true;
-      // If "TestingMaster.sleep" is set, sleep after log split.
-      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
-        int duration = getConfiguration().getInt(
-            "TestingMaster.sleep.duration", 0);
-        Threads.sleep(duration);
-      }
-    }
-
-
     public boolean isLogSplitAfterStartup() {
       return logSplit;
     }
@@ -249,11 +232,13 @@ private void abortMaster(MiniHBaseCluster cluster)
   private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
       throws IOException, InterruptedException {
     TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
-    while (!master.isLogSplitAfterStartup()) {
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    ServerManager serverManager = cluster.getMaster().getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
       Thread.sleep(100);
     }
-    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
-        + master.isInitialized());
     return master;
   }
 

From ad78c4e8c844e23da6635bbc5f338db7f3c9f97a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 26 Feb 2013 01:56:38 +0000
Subject: [PATCH 0794/1540] HBASE-7884 ByteBloomFilter's performance can be
 improved by avoiding multiplication when generating hash (clockfly)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1449996 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/ByteBloomFilter.java    | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
index e9c50332a678..7f666ac87193 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
@@ -418,24 +418,27 @@ public static boolean contains(byte[] buf, int offset, int length,
 
     int hash1 = hash.hash(buf, offset, length, 0);
     int hash2 = hash.hash(buf, offset, length, hash1);
-    int bloomBitSize = bloomSize * 8;
-
+    int bloomBitSize = bloomSize << 3;
+    
     if (randomGeneratorForTest == null) {
       // Production mode.
+      int compositeHash = hash1;
       for (int i = 0; i < hashCount; i++) {
-        long hashLoc = Math.abs((hash1 + i * hash2) % bloomBitSize);
-        if (!get(hashLoc, bloomArray, bloomOffset))
+        int hashLoc = Math.abs(compositeHash % bloomBitSize);
+        compositeHash += hash2;
+        if (!get(hashLoc, bloomArray, bloomOffset)) {
           return false;
+        }
       }
     } else {
       // Test mode with "fake lookups" to estimate "ideal false positive rate".
       for (int i = 0; i < hashCount; i++) {
-        long hashLoc = randomGeneratorForTest.nextInt(bloomBitSize);
-        if (!get(hashLoc, bloomArray, bloomOffset))
+        int hashLoc = randomGeneratorForTest.nextInt(bloomBitSize);
+        if (!get(hashLoc, bloomArray, bloomOffset)){
           return false;
+        }
       }
     }
-
     return true;
   }
 
@@ -461,9 +464,9 @@ void set(long pos) {
    * @param pos index of bit
    * @return true if bit at specified index is 1, false if 0.
    */
-  static boolean get(long pos, byte[] bloomArray, int bloomOffset) {
-    int bytePos = (int)(pos / 8);
-    int bitPos = (int)(pos % 8);
+  static boolean get(int pos, byte[] bloomArray, int bloomOffset) {
+    int bytePos = pos >> 3; //pos / 8
+    int bitPos = pos & 0x7; //pos % 8
     byte curByte = bloomArray[bloomOffset + bytePos];
     curByte &= bitvals[bitPos];
     return (curByte != 0);

From 195e23fe01466e2ff4bd5445e760555bfcf5b860 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 26 Feb 2013 02:20:08 +0000
Subject: [PATCH 0795/1540] HBASE-7914 Port the fix of HBASE-6748 into 0.94
 branch (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450000 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/SplitLogManager.java  | 58 ++++++++++++++++---
 .../hbase/zookeeper/RecoverableZooKeeper.java |  2 +-
 2 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 58d537131298..02c96629ce18 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -23,10 +23,11 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
-import java.util.Set;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -44,11 +45,11 @@
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.SplitLogWorker;
 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
-import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog.TaskState;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -59,7 +60,6 @@
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
-import org.apache.hadoop.hbase.zookeeper.ZKSplitLog.TaskState;
 
 import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.*;
 import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.*;
@@ -116,6 +116,8 @@ public class SplitLogManager extends ZooKeeperListener {
   private Set<String> deadWorkers = null;
   private Object deadWorkersLock = new Object();
 
+  private Set<String> failedDeletions = null;
+
   /**
    * Wrapper around {@link #SplitLogManager(ZooKeeperWatcher, Configuration,
    * Stoppable, String, TaskFinisher)} that provides a task finisher for
@@ -178,6 +180,8 @@ public SplitLogManager(ZooKeeperWatcher zkw, Configuration conf,
         conf.getInt("hbase.splitlog.manager.timeoutmonitor.period",
             1000),
         stopper);
+
+    this.failedDeletions = Collections.synchronizedSet(new HashSet<String>());
   }
 
   public void finishInitialization(boolean masterRecovery) {
@@ -416,11 +420,12 @@ private void setDone(String path, TerminationStatus status) {
         }
       }
     }
-    // delete the task node in zk. Keep trying indefinitely - its an async
+    // delete the task node in zk. It's an async
     // call and no one is blocked waiting for this node to be deleted. All
     // task names are unique (log.<timestamp>) there is no risk of deleting
     // a future task.
-    deleteNode(path, Long.MAX_VALUE);
+    // if a deletion fails, TimeoutMonitor will retry the same deletion later
+    deleteNode(path, zkretries);
     return;
   }
 
@@ -532,6 +537,21 @@ private void handleUnassignedTask(String path) {
     }
   }
 
+  /**
+   * Helper function to check whether to abandon retries in ZooKeeper AsyncCallback functions
+   * @param statusCode integer value of a ZooKeeper exception code
+   * @param action description message about the retried action
+   * @return true when need to abandon retries, otherwise false
+   */
+  private boolean shouldAbandonRetries(int statusCode, String action) {
+    if (statusCode == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+      LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries for "
+          + "action=" + action);
+      return true;
+    }
+    return false;
+  }
+
   private void heartbeat(String path, int new_version,
       String workerName) {
     Task task = findOrCreateOrphanTask(path);
@@ -653,8 +673,7 @@ private void deleteNodeSuccess(String path) {
   }
 
   private void deleteNodeFailure(String path) {
-    LOG.fatal("logic failure, failing to delete a node should never happen " +
-        "because delete has infinite retries");
+    LOG.info("Failed to delete node " + path + " and will retry soon.");
     return;
   }
 
@@ -1000,6 +1019,16 @@ protected void chore() {
         tot_mgr_resubmit_unassigned.incrementAndGet();
         LOG.debug("resubmitting unassigned task(s) after timeout");
       }
+
+      // Retry previously failed deletes
+      if (failedDeletions.size() > 0) {
+        List<String> tmpPaths = new ArrayList<String>(failedDeletions);
+        failedDeletions.removeAll(tmpPaths);
+        for (String tmpPath : tmpPaths) {
+          // deleteNode is an async call
+          deleteNode(tmpPath, zkretries);
+        }
+      }
     }
   }
 
@@ -1014,6 +1043,10 @@ class CreateAsyncCallback implements AsyncCallback.StringCallback {
     public void processResult(int rc, String path, Object ctx, String name) {
       tot_mgr_node_create_result.incrementAndGet();
       if (rc != 0) {
+        if (shouldAbandonRetries(rc, "Create znode " + path)) {
+          createNodeFailure(path);
+          return;
+        }
         if (rc == KeeperException.Code.NODEEXISTS.intValue()) {
           // What if there is a delete pending against this pre-existing
           // znode? Then this soon-to-be-deleted task znode must be in TASK_DONE
@@ -1053,6 +1086,9 @@ public void processResult(int rc, String path, Object ctx, byte[] data,
         Stat stat) {
       tot_mgr_get_data_result.incrementAndGet();
       if (rc != 0) {
+        if (shouldAbandonRetries(rc, "GetData from znode " + path)) {
+          return;
+        }
         if (rc == KeeperException.Code.NONODE.intValue()) {
           tot_mgr_get_data_nonode.incrementAndGet();
           // The task znode has been deleted. Must be some pending delete
@@ -1096,6 +1132,10 @@ class DeleteAsyncCallback implements AsyncCallback.VoidCallback {
     public void processResult(int rc, String path, Object ctx) {
       tot_mgr_node_delete_result.incrementAndGet();
       if (rc != 0) {
+        if (shouldAbandonRetries(rc, "Delete znode " + path)) {
+          failedDeletions.add(path);
+          return;
+        }
         if (rc != KeeperException.Code.NONODE.intValue()) {
           tot_mgr_node_delete_err.incrementAndGet();
           Long retry_count = (Long) ctx;
@@ -1103,6 +1143,7 @@ public void processResult(int rc, String path, Object ctx) {
               path + " remaining retries=" + retry_count);
           if (retry_count == 0) {
             LOG.warn("delete failed " + path);
+            failedDeletions.add(path);
             deleteNodeFailure(path);
           } else {
             deleteNode(path, retry_count - 1);
@@ -1134,6 +1175,9 @@ class CreateRescanAsyncCallback implements AsyncCallback.StringCallback {
     @Override
     public void processResult(int rc, String path, Object ctx, String name) {
       if (rc != 0) {
+        if (shouldAbandonRetries(rc, "CreateRescan znode " + path)) {
+          return;
+        }
         Long retry_count = (Long)ctx;
         LOG.warn("rc=" + KeeperException.Code.get(rc) + " for "+ path +
             " remaining retries=" + retry_count);
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 851d650edf43..ba5db2d76fb8 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -70,7 +70,7 @@
 public class RecoverableZooKeeper {
   private static final Log LOG = LogFactory.getLog(RecoverableZooKeeper.class);
   // the actual ZooKeeper client instance
-  private ZooKeeper zk;
+  private volatile ZooKeeper zk;
   private final RetryCounterFactory retryCounterFactory;
   // An identifier of this process in the cluster
   private final String identifier;

From f919834ffb1accab2ce85161e685316d23434491 Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Tue, 26 Feb 2013 02:38:57 +0000
Subject: [PATCH 0796/1540] HBASE-7671 Flushing memstore again after last
 failure could cause data loss

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450001 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |   4 +
 .../hadoop/hbase/regionserver/Store.java      |   2 +-
 .../hbase/regionserver/wal/TestWALReplay.java | 141 +++++++++++++++++-
 3 files changed, 144 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c20a0cc44f12..ea92a5bc2a77 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1448,6 +1448,10 @@ protected boolean internalFlushcache(MonitoredTask status)
   protected boolean internalFlushcache(
       final HLog wal, final long myseqid, MonitoredTask status)
   throws IOException {
+    if (this.rsServices != null && this.rsServices.isAborted()) {
+      // Don't flush when server aborting, it's unsafe
+      throw new IOException("Aborting flush because server is abortted...");
+    }
     final long startTime = EnvironmentEdgeManager.currentTimeMillis();
     // Clear flush flag.
     // Record latest flush time
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 6a11a24040cb..545495373631 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -729,7 +729,7 @@ void snapshot() {
    * @return Path The path name of the tmp file to which the store was flushed
    * @throws IOException
    */
-  private Path flushCache(final long logCacheFlushId,
+  protected Path flushCache(final long logCacheFlushId,
       SortedSet<KeyValue> snapshot,
       TimeRangeTracker snapshotTimeRangeTracker,
       AtomicLong flushedSize,
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 6e6c1e113fe6..9d4e7a7e10f7 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -21,12 +21,16 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.SortedSet;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -34,21 +38,31 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.regionserver.FlushRequester;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdge;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.Strings;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -427,6 +441,129 @@ public void testReplayEditsAfterPartialFlush()
     assertEquals(result.size(), result1b.size());
   }
 
+  /**
+   * Test that we could recover the data correctly after aborting flush. In the
+   * test, first we abort flush after writing some data, then writing more data
+   * and flush again, at last verify the data.
+   * @throws IOException
+   */
+  @Test
+  public void testReplayEditsAfterAbortingFlush() throws IOException {
+    final String tableNameStr = "testReplayEditsAfterAbortingFlush";
+    final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
+    final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
+    deleteDir(basedir);
+    final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
+    HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
+    region3.close();
+    region3.getLog().closeAndDelete();
+    // Write countPerFamily edits into the three families. Do a flush on one
+    // of the families during the load of edits so its seqid is not same as
+    // others to test we do right thing when different seqids.
+    HLog wal = createWAL(this.conf);
+    final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
+    RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
+    Mockito.doReturn(false).when(rsServices).isAborted();
+    HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd,
+        rsServices) {
+      @Override
+      protected Store instantiateHStore(Path tableDir, HColumnDescriptor c)
+          throws IOException {
+        return new Store(tableDir, this, c, fs, conf) {
+          @Override
+          protected Path flushCache(final long logCacheFlushId,
+              SortedSet<KeyValue> snapshot,
+              TimeRangeTracker snapshotTimeRangeTracker,
+              AtomicLong flushedSize, MonitoredTask status) throws IOException {
+            if (throwExceptionWhenFlushing.get()) {
+              throw new IOException("Simulated exception by tests");
+            }
+            return super.flushCache(logCacheFlushId, snapshot,
+                snapshotTimeRangeTracker, flushedSize, status);
+          }
+        };
+      }
+    };
+    long seqid = region.initialize();
+    // HRegionServer usually does this. It knows the largest seqid across all
+    // regions.
+    wal.setSequenceNumber(seqid);
+
+    int writtenRowCount = 10;
+    List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
+        htd.getFamilies());
+    for (int i = 0; i < writtenRowCount; i++) {
+      Put put = new Put(Bytes.toBytes(tableNameStr + Integer.toString(i)));
+      put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
+          Bytes.toBytes("val"));
+      region.put(put);
+    }
+
+    // Now assert edits made it in.
+    RegionScanner scanner = region.getScanner(new Scan());
+    assertEquals(writtenRowCount, getScannedCount(scanner));
+
+    // Let us flush the region
+    throwExceptionWhenFlushing.set(true);
+    try {
+      region.flushcache();
+      fail("Injected exception hasn't been thrown");
+    } catch (Throwable t) {
+      LOG.info("Expected simulated exception when flushing region,"
+          + t.getMessage());
+      // simulated to abort server
+      Mockito.doReturn(true).when(rsServices).isAborted();
+    }
+    // writing more data
+    int moreRow = 10;
+    for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
+      Put put = new Put(Bytes.toBytes(tableNameStr + Integer.toString(i)));
+      put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
+          Bytes.toBytes("val"));
+      region.put(put);
+    }
+    writtenRowCount += moreRow;
+    // call flush again
+    throwExceptionWhenFlushing.set(false);
+    try {
+      region.flushcache();
+    } catch (IOException t) {
+      LOG.info("Expected exception when flushing region because server is stopped,"
+          + t.getMessage());
+    }
+
+    region.close(true);
+    wal.close();
+
+    // Let us try to split and recover
+    runWALSplit(this.conf);
+    HLog wal2 = createWAL(this.conf);
+    Mockito.doReturn(false).when(rsServices).isAborted();
+    HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd,
+        rsServices);
+    long seqid2 = region2.initialize();
+    // HRegionServer usually does this. It knows the largest seqid across all
+    // regions.
+    wal2.setSequenceNumber(seqid2);
+
+    scanner = region2.getScanner(new Scan());
+    assertEquals(writtenRowCount, getScannedCount(scanner));
+  }
+
+  private int getScannedCount(RegionScanner scanner) throws IOException {
+    int scannedCount = 0;
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    while (true) {
+      boolean existMore = scanner.next(results);
+      if (!results.isEmpty())
+        scannedCount++;
+      if (!existMore)
+        break;
+      results.clear();
+    }
+    return scannedCount;
+  }
+
   /**
    * Create an HRegion with the result of a HLog split and test we only see the
    * good edits

From 4f64ada6379b53a9758901df6222d27b46aca4f4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 26 Feb 2013 04:45:38 +0000
Subject: [PATCH 0797/1540] HBASE-7723 Remove NameNode URI from ZK splitlogs
 (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450018 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/SplitLogWorker.java    | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
index eaa21d658103..0dbc4f9e2a35 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -107,9 +107,10 @@ public Status exec(String filename, CancelableProgressable p) {
         // TODO have to correctly figure out when log splitting has been
         // interrupted or has encountered a transient error and when it has
         // encountered a bad non-retry-able persistent error.
-        try {
+        try {          
+          String relativeLogPath = getRelativeLogPath(filename);
           if (HLogSplitter.splitLogFile(rootdir,
-              fs.getFileStatus(new Path(filename)), fs, conf, p) == false) {
+              fs.getFileStatus(new Path(rootdir, relativeLogPath)), fs, conf, p) == false) {
             return Status.PREEMPTED;
           }
         } catch (InterruptedIOException iioe) {
@@ -129,6 +130,21 @@ public Status exec(String filename, CancelableProgressable p) {
         }
         return Status.DONE;
       }
+
+      private String getRelativeLogPath(String logPath) {
+        StringBuilder sb = new StringBuilder();
+        String znodeDelimiter = Character.toString(Path.SEPARATOR_CHAR);
+        String[] filenameSplits = logPath.split(znodeDelimiter);
+        int len = filenameSplits.length;
+        String relativeLogPath = logPath;
+        if (len > 3) {
+          // the last three terms are .logs/server/log-file
+          relativeLogPath = sb.append(filenameSplits[len - 3]).append(znodeDelimiter)
+            .append(filenameSplits[len - 2]).append(znodeDelimiter)
+            .append(filenameSplits[len - 1]).toString();
+        }
+        return relativeLogPath;
+      }
     });
   }
 

From 85053bc8f756b3ec38585b8d6940e868ca04fd44 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 26 Feb 2013 08:51:52 +0000
Subject: [PATCH 0798/1540] HBASE-7883 Update memstore size when removing the
 entries in append operation (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450084 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/MemStore.java   |  4 +++-
 .../hbase/regionserver/TestMemStore.java      | 22 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 6e96df900c53..2f543739b9ae 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -584,7 +584,9 @@ private long upsert(KeyValue kv) {
         if (kv.getType() == KeyValue.Type.Put.getCode() &&
             kv.getMemstoreTS() == 0) {
           // false means there was a change, so give us the size.
-          addedSize -= heapSizeChange(kv, true);
+          long delta = heapSizeChange(cur, true);
+          addedSize -= delta;
+          this.size.addAndGet(-delta);
           it.remove();
         }
       } else {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
index 083824fd0820..62245f0c01de 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
@@ -25,6 +25,7 @@
 import java.rmi.UnexpectedException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -850,7 +851,26 @@ public void testUpsertMSLAB() throws Exception {
   }
 
   /**
-   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
+   * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
+   * as older keyvalues are deleted from the memstore.
+   * @throws Exception
+   */
+  public void testUpsertMemstoreSize() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    memstore = new MemStore(conf, KeyValue.COMPARATOR);
+    long oldSize = memstore.size.get();
+    
+    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
+    this.memstore.upsert(Collections.singletonList(kv1));
+    long newSize = this.memstore.size.get();
+    assert(newSize > oldSize);
+    
+    KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
+    this.memstore.upsert(Collections.singletonList(kv2));
+    assertEquals(newSize, this.memstore.size.get());
+  }
+
+  /**   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
    * @param hmc Instance to add rows to.
    * @return How many rows we added.
    * @throws IOException

From e14208caee4fae0ce6c875da161215682e1952bd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 26 Feb 2013 21:54:34 +0000
Subject: [PATCH 0799/1540] HBASE-4210 Allow coprocessor to interact with
 batches per region sent from a client (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450465 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/BaseRegionObserver.java |  12 ++
 .../hbase/coprocessor/RegionObserver.java     |  25 +++++
 .../hadoop/hbase/regionserver/HRegion.java    |  16 +++
 .../MiniBatchOperationInProgress.java         | 102 +++++++++++++++++
 .../regionserver/RegionCoprocessorHost.java   |  49 ++++++++
 .../coprocessor/SimpleRegionObserver.java     |  35 +++++-
 .../TestRegionObserverInterface.java          |   8 +-
 .../TestMiniBatchOperationInProgress.java     | 105 ++++++++++++++++++
 8 files changed, 347 insertions(+), 5 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 0c8034e09ea7..891cba9ee38f 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -35,6 +36,7 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -185,6 +187,16 @@ public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> e,
   public void postDelete(final ObserverContext<RegionCoprocessorEnvironment> e,
       final Delete delete, final WALEdit edit, final boolean writeToWAL) throws IOException {
   }
+  
+  @Override
+  public void preBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
+  }
+
+  @Override
+  public void postBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
+  }
 
   @Override
   public boolean preCheckAndPut(final ObserverContext<RegionCoprocessorEnvironment> e,
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 57f4c8fc38d4..31fd9387ac6e 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -35,6 +36,7 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -403,6 +405,29 @@ void postDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Delete delete, final WALEdit edit, final boolean writeToWAL)
     throws IOException;
 
+  /**
+   * This will be called for every batch mutation operation happening at the server. This will be
+   * called after acquiring the locks on the mutating rows and after applying the proper timestamp
+   * for each Mutation at the server. The batch may contain Put/Delete. By setting OperationStatus
+   * of Mutations ({@link MiniBatchOperationInProgress#setOperationStatus(int, OperationStatus)}),
+   * {@link RegionObserver} can make HRegion to skip these Mutations.
+   * @param c the environment provided by the region server
+   * @param miniBatchOp batch of Mutations getting applied to region.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void preBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException;
+
+  /**
+   * This will be called after applying a batch of Mutations on a region. The Mutations are added to
+   * memstore and WAL.
+   * @param c the environment provided by the region server
+   * @param miniBatchOp batch of Mutations applied to region.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void postBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException;
+  
   /**
    * Called before checkAndPut
    * <p>
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index ea92a5bc2a77..1c4038425014 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2263,6 +2263,14 @@ private long doMiniBatchMutation(
       // ----------------------------------
       w = mvcc.beginMemstoreInsert();
 
+      // calling the pre CP hook for batch mutation
+      if (coprocessorHost != null) {
+        MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp = 
+          new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(batchOp.operations, 
+          batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);
+        if (coprocessorHost.preBatchMutate(miniBatchOp)) return 0L;
+      }
+
       // ------------------------------------
       // STEP 3. Write back to memstore
       // Write to memstore. It is ok to write to memstore
@@ -2336,6 +2344,14 @@ private long doMiniBatchMutation(
         syncOrDefer(txid);
       }
       walSyncSuccessful = true;
+      // calling the post CP hook for batch mutation
+      if (coprocessorHost != null) {
+        MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp = 
+          new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(batchOp.operations, 
+          batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);
+        coprocessorHost.postBatchMutate(miniBatchOp);
+      }
+      
       // ------------------------------------------------------------------
       // STEP 8. Advance mvcc. This will make this put visible to scanners and getters.
       // ------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
new file mode 100644
index 000000000000..4a109c3700f8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+/**
+ * Wraps together the mutations which are applied as a batch to the region and their operation
+ * status and WALEdits. 
+ * @see RegionObserver#preBatchMutate(ObserverContext, MiniBatchOperationInProgress)
+ * @see RegionObserver#postBatchMutate(ObserverContext, MiniBatchOperationInProgress)
+ * @param <T> Pair<Mutation, Integer> pair of Mutations and associated rowlock ids.
+ */
+public class MiniBatchOperationInProgress<T> {
+  private final T[] operations;
+  private final OperationStatus[] retCodeDetails;
+  private final WALEdit[] walEditsFromCoprocessors;
+  private final int firstIndex;
+  private final int lastIndexExclusive;
+
+  public MiniBatchOperationInProgress(T[] operations, OperationStatus[] retCodeDetails,
+      WALEdit[] walEditsFromCoprocessors, int firstIndex, int lastIndexExclusive) {
+    this.operations = operations;
+    this.retCodeDetails = retCodeDetails;
+    this.walEditsFromCoprocessors = walEditsFromCoprocessors;
+    this.firstIndex = firstIndex;
+    this.lastIndexExclusive = lastIndexExclusive;
+  }
+
+  /**
+   * @return The number of operations(Mutations) involved in this batch.
+   */
+  public int size() {
+    return this.lastIndexExclusive - this.firstIndex;
+  }
+
+  /**
+   * @param index
+   * @return The operation(Mutation) at the specified position.
+   */
+  public T getOperation(int index) {
+    return operations[getAbsoluteIndex(index)];
+  }
+
+  /**
+   * Sets the status code for the operation(Mutation) at the specified position.
+   * By setting this status, {@link RegionObserver} can make HRegion to skip Mutations.
+   * @param index
+   * @param opStatus
+   */
+  public void setOperationStatus(int index, OperationStatus opStatus) {
+    this.retCodeDetails[getAbsoluteIndex(index)] = opStatus;
+  }
+
+  /**
+   * @param index
+   * @return Gets the status code for the operation(Mutation) at the specified position.
+   */
+  public OperationStatus getOperationStatus(int index) {
+    return this.retCodeDetails[getAbsoluteIndex(index)];
+  }
+
+  /**
+   * Sets the walEdit for the operation(Mutation) at the specified position.
+   * @param index
+   * @param walEdit
+   */
+  public void setWalEdit(int index, WALEdit walEdit) {
+    this.walEditsFromCoprocessors[getAbsoluteIndex(index)] = walEdit;
+  }
+
+  /**
+   * @param index
+   * @return Gets the walEdit for the operation(Mutation) at the specified position.
+   */
+  public WALEdit getWalEdit(int index) {
+    return this.walEditsFromCoprocessors[getAbsoluteIndex(index)];
+  }
+
+  private int getAbsoluteIndex(int index) {
+    if (index < 0 || this.firstIndex + index >= this.lastIndexExclusive) {
+      throw new ArrayIndexOutOfBoundsException(index);
+    }
+    return this.firstIndex + index;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 7934a932fc9c..4e97dc32cf0a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -885,6 +886,54 @@ public void postDelete(Delete delete, WALEdit edit,
       }
     }
   }
+  
+  /**
+   * @param miniBatchOp
+   * @return true if default processing should be bypassed
+   * @throws IOException
+   */
+  public boolean preBatchMutate(
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
+    boolean bypass = false;
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).preBatchMutate(ctx, miniBatchOp);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param miniBatchOp
+   * @throws IOException
+   */
+  public void postBatchMutate(
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
+    ObserverContext<RegionCoprocessorEnvironment> ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).postBatchMutate(ctx, miniBatchOp);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
 
   /**
    * @param row row to check
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index 70b628c13c99..f587c2e3cd97 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Increment;
@@ -46,6 +47,7 @@
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -98,7 +100,9 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   boolean hadPostScannerOpen = false;
   boolean hadPreBulkLoadHFile = false;
   boolean hadPostBulkLoadHFile = false;
-
+  boolean hadPreBatchMutate = false;
+  boolean hadPostBatchMutate = false;
+  
   @Override
   public void start(CoprocessorEnvironment e) throws IOException {
     // this only makes sure that leases and locks are available to coprocessors
@@ -391,6 +395,26 @@ public void postDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
     hadPostDeleted = true;
   }
 
+  @Override
+  public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
+      MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
+    RegionCoprocessorEnvironment e = c.getEnvironment();
+    assertNotNull(e);
+    assertNotNull(e.getRegion());
+    assertNotNull(miniBatchOp);
+    hadPreBatchMutate = true;
+  }
+
+  @Override
+  public void postBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
+    RegionCoprocessorEnvironment e = c.getEnvironment();
+    assertNotNull(e);
+    assertNotNull(e.getRegion());
+    assertNotNull(miniBatchOp);
+    hadPostBatchMutate = true;
+  }
+  
   @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> c,
       final byte[] row, final byte[] family, final Result result)
@@ -483,6 +507,15 @@ public boolean hadPrePut() {
   public boolean hadPostPut() {
     return hadPostPut;
   }
+  
+  public boolean hadPreBatchMutate() {
+    return hadPreBatchMutate;
+  }
+
+  public boolean hadPostBatchMutate() {
+    return hadPostBatchMutate;
+  }
+  
   public boolean hadDelete() {
     return !beforeDelete;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index 36318f20f63b..bba0365b64f8 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -102,9 +102,9 @@ public void testRegionObserver() throws IOException {
 
     verifyMethodResult(SimpleRegionObserver.class,
         new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
-            "hadDelete"},
+            "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
         TEST_TABLE,
-        new Boolean[] {false, false, true, true, false}
+        new Boolean[] {false, false, true, true, true, true, false}
     );
 
     Get get = new Get(ROW);
@@ -128,9 +128,9 @@ public void testRegionObserver() throws IOException {
 
     verifyMethodResult(SimpleRegionObserver.class,
         new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
-            "hadDelete"},
+             "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
         TEST_TABLE,
-        new Boolean[] {true, true, true, true, true}
+        new Boolean[] {true, true, true, true, true, true, true}
     );
     util.deleteTable(tableName);
     table.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java
new file mode 100644
index 000000000000..a83beb14e6b2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestMiniBatchOperationInProgress {
+
+  @Test
+  public void testMiniBatchOperationInProgressMethods() {
+    Pair<Mutation, Integer>[] operations = new Pair[10];
+    OperationStatus[] retCodeDetails = new OperationStatus[10];
+    WALEdit[] walEditsFromCoprocessors = new WALEdit[10];
+    for (int i = 0; i < 10; i++) {
+      operations[i] = new Pair<Mutation, Integer>(new Put(Bytes.toBytes(i)), null);
+    }
+    MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatch = 
+      new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(operations, retCodeDetails, 
+      walEditsFromCoprocessors, 0, 5);
+
+    assertEquals(5, miniBatch.size());
+    assertTrue(Bytes.equals(Bytes.toBytes(0), miniBatch.getOperation(0).getFirst().getRow()));
+    assertTrue(Bytes.equals(Bytes.toBytes(2), miniBatch.getOperation(2).getFirst().getRow()));
+    assertTrue(Bytes.equals(Bytes.toBytes(4), miniBatch.getOperation(4).getFirst().getRow()));
+    try {
+      miniBatch.getOperation(5);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    miniBatch.setOperationStatus(1, OperationStatus.FAILURE);
+    assertEquals(OperationStatus.FAILURE, retCodeDetails[1]);
+    try {
+      miniBatch.setOperationStatus(6, OperationStatus.FAILURE);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.setWalEdit(5, new WALEdit());
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+
+    miniBatch = new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(operations,
+        retCodeDetails, walEditsFromCoprocessors, 7, 10);
+    try {
+      miniBatch.setWalEdit(-1, new WALEdit());
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.getOperation(-1);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.getOperation(3);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.getOperationStatus(9);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.setOperationStatus(3, OperationStatus.FAILURE);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    assertTrue(Bytes.equals(Bytes.toBytes(7), miniBatch.getOperation(0).getFirst().getRow()));
+    assertTrue(Bytes.equals(Bytes.toBytes(9), miniBatch.getOperation(2).getFirst().getRow()));
+    miniBatch.setOperationStatus(1, OperationStatus.SUCCESS);
+    assertEquals(OperationStatus.SUCCESS, retCodeDetails[8]);
+    WALEdit wal = new WALEdit();
+    miniBatch.setWalEdit(0, wal);
+    assertEquals(wal, walEditsFromCoprocessors[7]);
+  }
+}

From fba9da7ce5891a4c818f20d334956c22a4222dbf Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 27 Feb 2013 00:08:47 +0000
Subject: [PATCH 0800/1540] HBASE-7945 Remove flaky TestCatalogTrackerOnCluster

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450544 13f79535-47bb-0310-9956-ffa450edef68
---
 .../catalog/TestCatalogTrackerOnCluster.java  | 84 -------------------
 1 file changed, 84 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java

diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java
deleted file mode 100644
index 5b53c47e1095..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.catalog;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Do {@link CatalogTracker} tests on running cluster.
- */
-@Category(LargeTests.class)
-public class TestCatalogTrackerOnCluster {
-  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-  private static final Log LOG =
-    LogFactory.getLog(TestCatalogTrackerOnCluster.class);
-
-  /**
-   * @throws Exception 
-   * @see {https://issues.apache.org/jira/browse/HBASE-3445}
-   */
-  @Test public void testBadOriginalRootLocation() throws Exception {
-    UTIL.getConfiguration().setInt("ipc.socket.timeout", 3000);
-    // Launch cluster so it does bootstrapping.
-    UTIL.startMiniCluster();
-    // Shutdown hbase.
-    UTIL.shutdownMiniHBaseCluster();
-    // Give the various ZKWatchers some time to settle their affairs.
-    Thread.sleep(1000);
-
-    // Mess with the root location in the running zk.  Set it to be nonsense.
-    ZooKeeperWatcher zookeeper = new ZooKeeperWatcher(UTIL.getConfiguration(),
-      "Bad Root Location Writer", new Abortable() {
-        @Override
-        public void abort(String why, Throwable e) {
-          LOG.error("Abort was called on 'bad root location writer'", e);
-        }
-        
-        @Override
-        public boolean isAborted() {
-          return false;
-        }
-    });
-    ServerName nonsense =
-      new ServerName("example.org", 1234, System.currentTimeMillis());
-    RootLocationEditor.setRootLocation(zookeeper, nonsense);
-
-    // Bring back up the hbase cluster.  See if it can deal with nonsense root
-    // location. The cluster should start and be fully available.
-    UTIL.startMiniHBaseCluster(1, 1);
-
-    // if we can create a table, it's a good sign that it's working
-    UTIL.createTable(
-      getClass().getSimpleName().getBytes(), "family".getBytes());
-
-    UTIL.shutdownMiniCluster();
-  }
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
-

From 1f8d3e982e5fd672733ce9d364a593b149cc3b4a Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 27 Feb 2013 04:09:04 +0000
Subject: [PATCH 0801/1540] HBASE-4210 Allow coprocessor to interact with
 batches per region sent from a client (Anoop). ADDENDUM patch for fixing line
 endings

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450593 13f79535-47bb-0310-9956-ffa450edef68
---
 .../MiniBatchOperationInProgress.java         | 204 ++++++++---------
 .../TestMiniBatchOperationInProgress.java     | 210 +++++++++---------
 2 files changed, 207 insertions(+), 207 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
index 4a109c3700f8..ce96aa75bffd 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
@@ -1,102 +1,102 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.hadoop.hbase.coprocessor.RegionObserver;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-
-/**
- * Wraps together the mutations which are applied as a batch to the region and their operation
- * status and WALEdits. 
- * @see RegionObserver#preBatchMutate(ObserverContext, MiniBatchOperationInProgress)
- * @see RegionObserver#postBatchMutate(ObserverContext, MiniBatchOperationInProgress)
- * @param <T> Pair<Mutation, Integer> pair of Mutations and associated rowlock ids.
- */
-public class MiniBatchOperationInProgress<T> {
-  private final T[] operations;
-  private final OperationStatus[] retCodeDetails;
-  private final WALEdit[] walEditsFromCoprocessors;
-  private final int firstIndex;
-  private final int lastIndexExclusive;
-
-  public MiniBatchOperationInProgress(T[] operations, OperationStatus[] retCodeDetails,
-      WALEdit[] walEditsFromCoprocessors, int firstIndex, int lastIndexExclusive) {
-    this.operations = operations;
-    this.retCodeDetails = retCodeDetails;
-    this.walEditsFromCoprocessors = walEditsFromCoprocessors;
-    this.firstIndex = firstIndex;
-    this.lastIndexExclusive = lastIndexExclusive;
-  }
-
-  /**
-   * @return The number of operations(Mutations) involved in this batch.
-   */
-  public int size() {
-    return this.lastIndexExclusive - this.firstIndex;
-  }
-
-  /**
-   * @param index
-   * @return The operation(Mutation) at the specified position.
-   */
-  public T getOperation(int index) {
-    return operations[getAbsoluteIndex(index)];
-  }
-
-  /**
-   * Sets the status code for the operation(Mutation) at the specified position.
-   * By setting this status, {@link RegionObserver} can make HRegion to skip Mutations.
-   * @param index
-   * @param opStatus
-   */
-  public void setOperationStatus(int index, OperationStatus opStatus) {
-    this.retCodeDetails[getAbsoluteIndex(index)] = opStatus;
-  }
-
-  /**
-   * @param index
-   * @return Gets the status code for the operation(Mutation) at the specified position.
-   */
-  public OperationStatus getOperationStatus(int index) {
-    return this.retCodeDetails[getAbsoluteIndex(index)];
-  }
-
-  /**
-   * Sets the walEdit for the operation(Mutation) at the specified position.
-   * @param index
-   * @param walEdit
-   */
-  public void setWalEdit(int index, WALEdit walEdit) {
-    this.walEditsFromCoprocessors[getAbsoluteIndex(index)] = walEdit;
-  }
-
-  /**
-   * @param index
-   * @return Gets the walEdit for the operation(Mutation) at the specified position.
-   */
-  public WALEdit getWalEdit(int index) {
-    return this.walEditsFromCoprocessors[getAbsoluteIndex(index)];
-  }
-
-  private int getAbsoluteIndex(int index) {
-    if (index < 0 || this.firstIndex + index >= this.lastIndexExclusive) {
-      throw new ArrayIndexOutOfBoundsException(index);
-    }
-    return this.firstIndex + index;
-  }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+/**
+ * Wraps together the mutations which are applied as a batch to the region and their operation
+ * status and WALEdits. 
+ * @see RegionObserver#preBatchMutate(ObserverContext, MiniBatchOperationInProgress)
+ * @see RegionObserver#postBatchMutate(ObserverContext, MiniBatchOperationInProgress)
+ * @param <T> Pair<Mutation, Integer> pair of Mutations and associated rowlock ids.
+ */
+public class MiniBatchOperationInProgress<T> {
+  private final T[] operations;
+  private final OperationStatus[] retCodeDetails;
+  private final WALEdit[] walEditsFromCoprocessors;
+  private final int firstIndex;
+  private final int lastIndexExclusive;
+
+  public MiniBatchOperationInProgress(T[] operations, OperationStatus[] retCodeDetails,
+      WALEdit[] walEditsFromCoprocessors, int firstIndex, int lastIndexExclusive) {
+    this.operations = operations;
+    this.retCodeDetails = retCodeDetails;
+    this.walEditsFromCoprocessors = walEditsFromCoprocessors;
+    this.firstIndex = firstIndex;
+    this.lastIndexExclusive = lastIndexExclusive;
+  }
+
+  /**
+   * @return The number of operations(Mutations) involved in this batch.
+   */
+  public int size() {
+    return this.lastIndexExclusive - this.firstIndex;
+  }
+
+  /**
+   * @param index
+   * @return The operation(Mutation) at the specified position.
+   */
+  public T getOperation(int index) {
+    return operations[getAbsoluteIndex(index)];
+  }
+
+  /**
+   * Sets the status code for the operation(Mutation) at the specified position.
+   * By setting this status, {@link RegionObserver} can make HRegion to skip Mutations.
+   * @param index
+   * @param opStatus
+   */
+  public void setOperationStatus(int index, OperationStatus opStatus) {
+    this.retCodeDetails[getAbsoluteIndex(index)] = opStatus;
+  }
+
+  /**
+   * @param index
+   * @return Gets the status code for the operation(Mutation) at the specified position.
+   */
+  public OperationStatus getOperationStatus(int index) {
+    return this.retCodeDetails[getAbsoluteIndex(index)];
+  }
+
+  /**
+   * Sets the walEdit for the operation(Mutation) at the specified position.
+   * @param index
+   * @param walEdit
+   */
+  public void setWalEdit(int index, WALEdit walEdit) {
+    this.walEditsFromCoprocessors[getAbsoluteIndex(index)] = walEdit;
+  }
+
+  /**
+   * @param index
+   * @return Gets the walEdit for the operation(Mutation) at the specified position.
+   */
+  public WALEdit getWalEdit(int index) {
+    return this.walEditsFromCoprocessors[getAbsoluteIndex(index)];
+  }
+
+  private int getAbsoluteIndex(int index) {
+    if (index < 0 || this.firstIndex + index >= this.lastIndexExclusive) {
+      throw new ArrayIndexOutOfBoundsException(index);
+    }
+    return this.firstIndex + index;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java
index a83beb14e6b2..5ad95a0b21f9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMiniBatchOperationInProgress.java
@@ -1,105 +1,105 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(SmallTests.class)
-public class TestMiniBatchOperationInProgress {
-
-  @Test
-  public void testMiniBatchOperationInProgressMethods() {
-    Pair<Mutation, Integer>[] operations = new Pair[10];
-    OperationStatus[] retCodeDetails = new OperationStatus[10];
-    WALEdit[] walEditsFromCoprocessors = new WALEdit[10];
-    for (int i = 0; i < 10; i++) {
-      operations[i] = new Pair<Mutation, Integer>(new Put(Bytes.toBytes(i)), null);
-    }
-    MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatch = 
-      new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(operations, retCodeDetails, 
-      walEditsFromCoprocessors, 0, 5);
-
-    assertEquals(5, miniBatch.size());
-    assertTrue(Bytes.equals(Bytes.toBytes(0), miniBatch.getOperation(0).getFirst().getRow()));
-    assertTrue(Bytes.equals(Bytes.toBytes(2), miniBatch.getOperation(2).getFirst().getRow()));
-    assertTrue(Bytes.equals(Bytes.toBytes(4), miniBatch.getOperation(4).getFirst().getRow()));
-    try {
-      miniBatch.getOperation(5);
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    miniBatch.setOperationStatus(1, OperationStatus.FAILURE);
-    assertEquals(OperationStatus.FAILURE, retCodeDetails[1]);
-    try {
-      miniBatch.setOperationStatus(6, OperationStatus.FAILURE);
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    try {
-      miniBatch.setWalEdit(5, new WALEdit());
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-
-    miniBatch = new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(operations,
-        retCodeDetails, walEditsFromCoprocessors, 7, 10);
-    try {
-      miniBatch.setWalEdit(-1, new WALEdit());
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    try {
-      miniBatch.getOperation(-1);
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    try {
-      miniBatch.getOperation(3);
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    try {
-      miniBatch.getOperationStatus(9);
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    try {
-      miniBatch.setOperationStatus(3, OperationStatus.FAILURE);
-      fail("Should throw Exception while accessing out of range");
-    } catch (ArrayIndexOutOfBoundsException e) {
-    }
-    assertTrue(Bytes.equals(Bytes.toBytes(7), miniBatch.getOperation(0).getFirst().getRow()));
-    assertTrue(Bytes.equals(Bytes.toBytes(9), miniBatch.getOperation(2).getFirst().getRow()));
-    miniBatch.setOperationStatus(1, OperationStatus.SUCCESS);
-    assertEquals(OperationStatus.SUCCESS, retCodeDetails[8]);
-    WALEdit wal = new WALEdit();
-    miniBatch.setWalEdit(0, wal);
-    assertEquals(wal, walEditsFromCoprocessors[7]);
-  }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestMiniBatchOperationInProgress {
+
+  @Test
+  public void testMiniBatchOperationInProgressMethods() {
+    Pair<Mutation, Integer>[] operations = new Pair[10];
+    OperationStatus[] retCodeDetails = new OperationStatus[10];
+    WALEdit[] walEditsFromCoprocessors = new WALEdit[10];
+    for (int i = 0; i < 10; i++) {
+      operations[i] = new Pair<Mutation, Integer>(new Put(Bytes.toBytes(i)), null);
+    }
+    MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatch = 
+      new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(operations, retCodeDetails, 
+      walEditsFromCoprocessors, 0, 5);
+
+    assertEquals(5, miniBatch.size());
+    assertTrue(Bytes.equals(Bytes.toBytes(0), miniBatch.getOperation(0).getFirst().getRow()));
+    assertTrue(Bytes.equals(Bytes.toBytes(2), miniBatch.getOperation(2).getFirst().getRow()));
+    assertTrue(Bytes.equals(Bytes.toBytes(4), miniBatch.getOperation(4).getFirst().getRow()));
+    try {
+      miniBatch.getOperation(5);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    miniBatch.setOperationStatus(1, OperationStatus.FAILURE);
+    assertEquals(OperationStatus.FAILURE, retCodeDetails[1]);
+    try {
+      miniBatch.setOperationStatus(6, OperationStatus.FAILURE);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.setWalEdit(5, new WALEdit());
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+
+    miniBatch = new MiniBatchOperationInProgress<Pair<Mutation, Integer>>(operations,
+        retCodeDetails, walEditsFromCoprocessors, 7, 10);
+    try {
+      miniBatch.setWalEdit(-1, new WALEdit());
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.getOperation(-1);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.getOperation(3);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.getOperationStatus(9);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    try {
+      miniBatch.setOperationStatus(3, OperationStatus.FAILURE);
+      fail("Should throw Exception while accessing out of range");
+    } catch (ArrayIndexOutOfBoundsException e) {
+    }
+    assertTrue(Bytes.equals(Bytes.toBytes(7), miniBatch.getOperation(0).getFirst().getRow()));
+    assertTrue(Bytes.equals(Bytes.toBytes(9), miniBatch.getOperation(2).getFirst().getRow()));
+    miniBatch.setOperationStatus(1, OperationStatus.SUCCESS);
+    assertEquals(OperationStatus.SUCCESS, retCodeDetails[8]);
+    WALEdit wal = new WALEdit();
+    miniBatch.setWalEdit(0, wal);
+    assertEquals(wal, walEditsFromCoprocessors[7]);
+  }
+}

From 3b923a92c58f4e4829cb8523bb7e7583383438d4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 27 Feb 2013 15:00:59 +0000
Subject: [PATCH 0802/1540] HBASE-7878 recoverFileLease does not check return
 value of recoverLease (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450793 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/FSHDFSUtils.java | 34 +++++++++++++++----
 .../hbase/regionserver/wal/TestHLog.java      | 30 +++++++++++-----
 2 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 4186f0ad54c0..87730d766c8c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -50,6 +50,8 @@ public class FSHDFSUtils extends FSUtils{
    * in o.a.h.hdfs.protocol.HdfsConstants cause of HDFS-1620.
    */
   public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
+  
+  public static final String TEST_TRIGGER_DFS_APPEND = "hbase.test.trigger.dfs.append";
 
   @Override
   public void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
@@ -68,13 +70,23 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
 
     // Trying recovery
     boolean recovered = false;
+    long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 300000);
+    // conf parameter passed from unit test, indicating whether fs.append() should be triggered
+    boolean triggerAppend = conf.getBoolean(TEST_TRIGGER_DFS_APPEND, false);
+    Exception ex = null;    
     while (!recovered) {
       try {
         try {
           if (fs instanceof DistributedFileSystem) {
             DistributedFileSystem dfs = (DistributedFileSystem)fs;
-            DistributedFileSystem.class.getMethod("recoverLease",
-              new Class[] {Path.class}).invoke(dfs, p);
+            if (triggerAppend) throw new IOException("recoverFileLease");
+            try {
+              recovered = (Boolean) DistributedFileSystem.class.getMethod(
+                "recoverLease", new Class[] { Path.class }).invoke(dfs, p);
+            } catch (InvocationTargetException ite) {
+              // function was properly called, but threw it's own exception
+              throw (IOException) ite.getCause();
+            }
           } else {
             throw new Exception("Not a DistributedFileSystem");
           }
@@ -82,12 +94,20 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
           // function was properly called, but threw it's own exception
           throw (IOException) ite.getCause();
         } catch (Exception e) {
-          LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
-            ", trying fs.append instead");
+          // hdfs 2.0 may throw RecoveryInProgressException
+          if (!e.getClass().getName().contains("RecoveryInProgressException")) {
+            LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
+                ", trying fs.append instead");
+            ex = e;
+          }
+        }
+        if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) {
+          ex = null; // assume the following append() call would succeed
           FSDataOutputStream out = fs.append(p);
           out.close();
+          recovered = true;
         }
-        recovered = true;
+        if (recovered) break;
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
         if (e instanceof AlreadyBeingCreatedException) {
@@ -111,9 +131,9 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
       }
       try {
         Thread.sleep(1000);
-      } catch (InterruptedException ex) {
+      } catch (InterruptedException ie) {
         InterruptedIOException iioe = new InterruptedIOException();
-        iioe.initCause(ex);
+        iioe.initCause(ie);
         throw iioe;
       }
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 8feb5d6db2d1..06768f9a7ec3 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@@ -52,7 +53,6 @@
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -81,7 +81,7 @@ public class TestHLog  {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static Path hbaseDir;
   private static Path oldLogDir;
-
+  
   @Before
   public void setUp() throws Exception {
 
@@ -101,6 +101,7 @@ public static void setUpBeforeClass() throws Exception {
     // Make block sizes small.
     TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
     // needed for testAppendClose()
+    TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true);
     TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
     // quicker heartbeat interval for faster DN death notification
     TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000);
@@ -370,17 +371,29 @@ private void verifySplits(List<Path> splits, final int howmany)
     }
   }
   
-  // For this test to pass, requires:
-  // 1. HDFS-200 (append support)
-  // 2. HDFS-988 (SafeMode should freeze file operations
-  //              [FSNamesystem.nextGenerationStampForBlock])
-  // 3. HDFS-142 (on restart, maintain pendingCreates)
+  /*
+   * We pass different values to recoverFileLease() so that different code paths are covered
+   * 
+   * For this test to pass, requires:
+   * 1. HDFS-200 (append support)
+   * 2. HDFS-988 (SafeMode should freeze file operations
+   *              [FSNamesystem.nextGenerationStampForBlock])
+   * 3. HDFS-142 (on restart, maintain pendingCreates)
+   */
   @Test
   public void testAppendClose() throws Exception {
+    testAppendClose(true);
+    testAppendClose(false);
+  }
+
+  /*
+   * @param triggerDirectAppend whether to trigger direct call of fs.append()
+   */
+  public void testAppendClose(final boolean triggerDirectAppend) throws Exception {
     byte [] tableName = Bytes.toBytes(getName());
     HRegionInfo regioninfo = new HRegionInfo(tableName,
              HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
-    Path subdir = new Path(dir, "hlogdir");
+    Path subdir = new Path(dir, "hlogdir-" + triggerDirectAppend);
     Path archdir = new Path(dir, "hlogdir_archive");
     HLog wal = new HLog(fs, subdir, archdir, conf);
     final int total = 20;
@@ -454,6 +467,7 @@ class RecoverLogThread extends Thread {
       public Exception exception = null;
       public void run() {
           try {
+            rlConf.setBoolean(FSHDFSUtils.TEST_TRIGGER_DFS_APPEND, triggerDirectAppend);
             FSUtils.getInstance(fs, rlConf)
               .recoverFileLease(recoveredFs, walPath, rlConf);
           } catch (IOException e) {

From 539ff195e8e3d4f8a9e48f33140bfa752d1b7a22 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 27 Feb 2013 17:53:46 +0000
Subject: [PATCH 0803/1540] HBASE-7878 add debug log for recoverLease timeout

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450883 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 87730d766c8c..04dbfb9ea84f 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -103,6 +103,7 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
         }
         if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) {
           ex = null; // assume the following append() call would succeed
+          LOG.debug("recoverLease times out after " + recoveryTimeout + " ms. Calling append");
           FSDataOutputStream out = fs.append(p);
           out.close();
           recovered = true;

From 50a47a9023c1a8f44039f46cdb3d8ecfd84d36d3 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 27 Feb 2013 19:09:57 +0000
Subject: [PATCH 0804/1540] HBASE-7878 Addendum for trying fs.append in case
 recoverLease throws exception

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1450924 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/FSHDFSUtils.java   | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 04dbfb9ea84f..e59e46d32746 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -94,12 +94,9 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
           // function was properly called, but threw it's own exception
           throw (IOException) ite.getCause();
         } catch (Exception e) {
-          // hdfs 2.0 may throw RecoveryInProgressException
-          if (!e.getClass().getName().contains("RecoveryInProgressException")) {
-            LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
-                ", trying fs.append instead");
-            ex = e;
-          }
+          LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
+              ", trying fs.append instead");
+          ex = e;
         }
         if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) {
           ex = null; // assume the following append() call would succeed

From 1b9883488c18d946be393f070a97d32093947c59 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 27 Feb 2013 23:19:48 +0000
Subject: [PATCH 0805/1540] HBASE-7777 HBCK check for lingering split parents
 should check for child regions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451028 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/MetaReader.java      |  18 ++
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  58 ++++-
 .../TestEndToEndSplitTransaction.java         | 227 +++++++++---------
 .../hadoop/hbase/util/TestHBaseFsck.java      | 137 ++++++++++-
 4 files changed, 309 insertions(+), 131 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index 780ed0af1d48..cbb6d483bec9 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.ipc.RemoteException;
 
@@ -402,6 +403,8 @@ public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r,
     return Writables.getHRegionInfoOrNull(bytes);
   }
 
+
+
   /**
    * Checks if the specified table exists.  Looks at the META table hosted on
    * the specified server.
@@ -449,6 +452,21 @@ void add(Result r) {
     return visitor.getResults().size() >= 1;
   }
 
+  /**
+   * Returns the daughter regions by reading the corresponding columns of the catalog table
+   * Result.
+   * @param data a Result object from the catalog table scan
+   * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
+   * parent
+   */
+  public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
+    HRegionInfo splitA = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY,
+      HConstants.SPLITA_QUALIFIER));
+    HRegionInfo splitB = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY,
+      HConstants.SPLITB_QUALIFIER));
+    return new PairOfSameType<HRegionInfo>(splitA, splitB);
+  }
+
   /**
    * Gets all of the regions of the specified table.
    * @param catalogTracker
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 070a4f9e36d3..052ae95e774d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -236,7 +236,7 @@ public class HBaseFsck extends Configured implements Tool {
    * When initially looking at HDFS, we attempt to find any orphaned data.
    */
   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
-  
+
   private Map<String, Set<String>> orphanTableDirs = new HashMap<String, Set<String>>();
 
   /**
@@ -395,7 +395,7 @@ public int onlineConsistencyRepair() throws IOException, KeeperException,
     if (!checkMetaOnly) {
       reportTablesInFlux();
     }
-    
+
     // get regions according to what is online on each RegionServer
     loadDeployedRegions();
 
@@ -800,19 +800,21 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
           if (!orphanTableDirs.containsKey(tableName)) {
             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
             //should only report once for each table
-            errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, 
+            errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
             Set<String> columns = new HashSet<String>();
             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
           }
         }
       }
-      modTInfo.addRegionInfo(hbi);
+      if (!hbi.isSkipChecks()) {
+        modTInfo.addRegionInfo(hbi);
+      }
     }
 
     return tablesInfo;
   }
-  
+
   /**
    * To get the column family list according to the column family dirs
    * @param columns
@@ -830,7 +832,7 @@ private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throw
     }
     return columns;
   }
-  
+
   /**
    * To fabricate a .tableinfo file with following contents<br>
    * 1. the correct tablename <br>
@@ -848,7 +850,7 @@ private boolean fabricateTableInfo(String tableName, Set<String> columns) throws
     FSTableDescriptors.createTableDescriptor(htd, getConf(), true);
     return true;
   }
-  
+
   /**
    * To fix orphan table by creating a .tableinfo file under tableDir <br>
    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
@@ -1661,6 +1663,18 @@ else if (!inMeta && !inHdfs && !isDeployed) {
 
     // ========== Cases where the region is in META =============
     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
+      // check whether this is an actual error, or just transient state where parent
+      // is not cleaned
+      if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
+        // check that split daughters are there
+        HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
+        HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
+        if (infoA != null && infoB != null) {
+          // we already processed or will process daughters. Move on, nothing to see here.
+          hbi.setSkipChecks(true);
+          return;
+        }
+      }
       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
           + descriptiveName + " is a split parent in META, in HDFS, "
           + "and not deployed on any region server. This could be transient.");
@@ -1791,7 +1805,9 @@ SortedMap<String, TableInfo> checkIntegrity() throws IOException {
         modTInfo.addServer(server);
       }
 
-      modTInfo.addRegionInfo(hbi);
+      if (!hbi.isSkipChecks()) {
+        modTInfo.addRegionInfo(hbi);
+      }
 
       tablesInfo.put(tableName, modTInfo);
     }
@@ -2555,7 +2571,8 @@ public boolean processRow(Result result) throws IOException {
               || hri.isMetaRegion() || hri.isRootRegion())) {
             return true;
           }
-          MetaEntry m = new MetaEntry(hri, sn, ts);
+          PairOfSameType<HRegionInfo> daughters = MetaReader.getDaughterRegions(result);
+          MetaEntry m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
           HbckInfo hbInfo = new HbckInfo(m);
           HbckInfo previous = regionInfoMap.put(hri.getEncodedName(), hbInfo);
           if (previous != null) {
@@ -2594,11 +2611,19 @@ public boolean processRow(Result result) throws IOException {
   static class MetaEntry extends HRegionInfo {
     ServerName regionServer;   // server hosting this region
     long modTime;          // timestamp of most recent modification metadata
+    HRegionInfo splitA, splitB; //split daughters
 
     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
+      this(rinfo, regionServer, modTime, null, null);
+    }
+
+    public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
+        HRegionInfo splitA, HRegionInfo splitB) {
       super(rinfo);
       this.regionServer = regionServer;
       this.modTime = modTime;
+      this.splitA = splitA;
+      this.splitB = splitB;
     }
 
     public boolean equals(Object o) {
@@ -2647,6 +2672,7 @@ public static class HbckInfo implements KeyRange {
     private HdfsEntry hdfsEntry = null; // info in HDFS
     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
+    private boolean skipChecks = false; // whether to skip further checks to this region info.
 
     HbckInfo(MetaEntry metaEntry) {
       this.metaEntry = metaEntry;
@@ -2764,6 +2790,14 @@ HRegionInfo getHdfsHRI() {
       }
       return hdfsEntry.hri;
     }
+
+    public void setSkipChecks(boolean skipChecks) {
+      this.skipChecks = skipChecks;
+    }
+
+    public boolean isSkipChecks() {
+      return skipChecks;
+    }
   }
 
   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
@@ -3239,15 +3273,15 @@ public void setFixHdfsHoles(boolean shouldFix) {
   boolean shouldFixHdfsHoles() {
     return fixHdfsHoles;
   }
-  
+
   public void setFixTableOrphans(boolean shouldFix) {
     fixTableOrphans = shouldFix;
   }
-   
+
   boolean shouldFixTableOrphans() {
     return fixTableOrphans;
   }
-  
+
   public void setFixHdfsOverlaps(boolean shouldFix) {
     fixHdfsOverlaps = shouldFix;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 34e97010e5b3..4e62a1fa6bf8 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -43,6 +43,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
@@ -199,7 +200,6 @@ static class RegionSplitter extends Thread {
     HTable table;
     byte[] tableName, family;
     HBaseAdmin admin;
-    HTable metaTable;
     HRegionServer rs;
 
     RegionSplitter(HTable table) throws IOException {
@@ -208,7 +208,6 @@ static class RegionSplitter extends Thread {
       this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
       admin = TEST_UTIL.getHBaseAdmin();
       rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
-      metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
     }
 
     public void run() {
@@ -239,14 +238,14 @@ public void run() {
           addData(start);
           addData(mid);
 
-          flushAndBlockUntilDone(region.getRegionName());
-          compactAndBlockUntilDone(region.getRegionName());
+          flushAndBlockUntilDone(admin, rs, region.getRegionName());
+          compactAndBlockUntilDone(admin, rs, region.getRegionName());
 
           log("Initiating region split for:" + region.getRegionNameAsString());
           try {
             admin.split(region.getRegionName(), splitPoint);
             //wait until the split is complete
-            blockUntilRegionSplit(50000, region.getRegionName(), true);
+            blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
 
           } catch (NotServingRegionException ex) {
             //ignore
@@ -254,10 +253,6 @@ public void run() {
         }
       } catch (Throwable ex) {
         this.ex = ex;
-      } finally {
-        if (metaTable != null) {
-          IOUtils.closeQuietly(metaTable);
-        }
       }
     }
 
@@ -270,106 +265,6 @@ void addData(int start) throws IOException {
       }
       table.flushCommits();
     }
-
-    void flushAndBlockUntilDone(byte[] regionName) throws IOException, InterruptedException {
-      log("flushing region: " + Bytes.toStringBinary(regionName));
-      admin.flush(regionName);
-      log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
-      Threads.sleepWithoutInterrupt(500);
-      while (rs.cacheFlusher.getFlushQueueSize() > 0) {
-        Threads.sleep(50);
-      }
-    }
-
-    void compactAndBlockUntilDone(byte[] regionName) throws IOException,
-      InterruptedException {
-      log("Compacting region: " + Bytes.toStringBinary(regionName));
-      admin.majorCompact(regionName);
-      log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
-      Threads.sleepWithoutInterrupt(500);
-      while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
-        Threads.sleep(50);
-      }
-    }
-
-    /** bloks until the region split is complete in META and region server opens the daughters */
-    void blockUntilRegionSplit(long timeout, final byte[] regionName, boolean waitForDaughters)
-        throws IOException, InterruptedException {
-      long start = System.currentTimeMillis();
-      log("blocking until region is split:" +  Bytes.toStringBinary(regionName));
-      HRegionInfo daughterA = null, daughterB = null;
-
-      while (System.currentTimeMillis() - start < timeout) {
-        Result result = getRegionRow(regionName);
-        if (result == null) {
-          break;
-        }
-
-        HRegionInfo region = MetaEditor.getHRegionInfo(result);
-        if(region.isSplitParent()) {
-          log("found parent region: " + region.toString());
-          PairOfSameType<HRegionInfo> pair = MetaEditor.getDaughterRegions(result);
-          daughterA = pair.getFirst();
-          daughterB = pair.getSecond();
-          break;
-        }
-        sleep(100);
-      }
-
-      //if we are here, this means the region split is complete or timed out
-      if (waitForDaughters) {
-        long rem = timeout - (System.currentTimeMillis() - start);
-        blockUntilRegionIsInMeta(rem, daughterA.getRegionName());
-
-        rem = timeout - (System.currentTimeMillis() - start);
-        blockUntilRegionIsInMeta(rem, daughterB.getRegionName());
-
-        rem = timeout - (System.currentTimeMillis() - start);
-        blockUntilRegionIsOpenedByRS(rem, daughterA.getRegionName());
-
-        rem = timeout - (System.currentTimeMillis() - start);
-        blockUntilRegionIsOpenedByRS(rem, daughterB.getRegionName());
-      }
-    }
-
-    Result getRegionRow(byte[] regionName) throws IOException {
-      Get get = new Get(regionName);
-      return metaTable.get(get);
-    }
-
-    void blockUntilRegionIsInMeta(long timeout, byte[] regionName)
-        throws IOException, InterruptedException {
-      log("blocking until region is in META: " + Bytes.toStringBinary(regionName));
-      long start = System.currentTimeMillis();
-      while (System.currentTimeMillis() - start < timeout) {
-        Result result = getRegionRow(regionName);
-        if (result != null) {
-          HRegionInfo info = MetaEditor.getHRegionInfo(result);
-          if (info != null && !info.isOffline()) {
-            log("found region in META: " + Bytes.toStringBinary(regionName));
-            break;
-          }
-        }
-        sleep(10);
-      }
-    }
-
-    void blockUntilRegionIsOpenedByRS(long timeout, byte[] regionName)
-      throws IOException, InterruptedException {
-      log("blocking until region is opened by region server: " + Bytes.toStringBinary(regionName));
-      long start = System.currentTimeMillis();
-      while (System.currentTimeMillis() - start < timeout) {
-        List<HRegion> regions = rs.getOnlineRegions(tableName);
-        for (HRegion region : regions) {
-          if (Bytes.compareTo(region.getRegionName(), regionName) == 0) {
-            log("found region open in RS: " + Bytes.toStringBinary(regionName));
-            return;
-          }
-        }
-        sleep(10);
-      }
-    }
-
   }
 
   /**
@@ -476,6 +371,120 @@ public static void log(String msg) {
     LOG.info(msg);
   }
 
+  /* some utility methods for split tests */
+
+  public static void flushAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
+      throws IOException, InterruptedException {
+    log("flushing region: " + Bytes.toStringBinary(regionName));
+    admin.flush(regionName);
+    log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
+    Threads.sleepWithoutInterrupt(500);
+    while (rs.cacheFlusher.getFlushQueueSize() > 0) {
+      Threads.sleep(50);
+    }
+  }
+
+  public static void compactAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
+      throws IOException, InterruptedException {
+    log("Compacting region: " + Bytes.toStringBinary(regionName));
+    admin.majorCompact(regionName);
+    log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
+    Threads.sleepWithoutInterrupt(500);
+    while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
+      Threads.sleep(50);
+    }
+  }
+
+  /** Blocks until the region split is complete in META and region server opens the daughters */
+  public static void blockUntilRegionSplit(Configuration conf, long timeout,
+      final byte[] regionName, boolean waitForDaughters)
+      throws IOException, InterruptedException {
+    long start = System.currentTimeMillis();
+    log("blocking until region is split:" +  Bytes.toStringBinary(regionName));
+    HRegionInfo daughterA = null, daughterB = null;
+    HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
+
+    try {
+      while (System.currentTimeMillis() - start < timeout) {
+        Result result = getRegionRow(metaTable, regionName);
+        if (result == null) {
+          break;
+        }
+
+        HRegionInfo region = MetaReader.parseCatalogResult(result).getFirst();
+        if(region.isSplitParent()) {
+          log("found parent region: " + region.toString());
+          PairOfSameType<HRegionInfo> pair = MetaReader.getDaughterRegions(result);
+          daughterA = pair.getFirst();
+          daughterB = pair.getSecond();
+          break;
+        }
+        Threads.sleep(100);
+      }
+
+      //if we are here, this means the region split is complete or timed out
+      if (waitForDaughters) {
+        long rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsInMeta(metaTable, rem, daughterA);
+
+        rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsInMeta(metaTable, rem, daughterB);
+
+        rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsOpened(conf, rem, daughterA);
+
+        rem = timeout - (System.currentTimeMillis() - start);
+        blockUntilRegionIsOpened(conf, rem, daughterB);
+      }
+    } finally {
+      IOUtils.closeQuietly(metaTable);
+    }
+  }
+
+  public static Result getRegionRow(HTable metaTable, byte[] regionName) throws IOException {
+    Get get = new Get(regionName);
+    return metaTable.get(get);
+  }
+
+  public static void blockUntilRegionIsInMeta(HTable metaTable, long timeout, HRegionInfo hri)
+      throws IOException, InterruptedException {
+    log("blocking until region is in META: " + hri.getRegionNameAsString());
+    long start = System.currentTimeMillis();
+    while (System.currentTimeMillis() - start < timeout) {
+      Result result = getRegionRow(metaTable, hri.getRegionName());
+      if (result != null) {
+        HRegionInfo info = MetaReader.parseCatalogResult(result).getFirst();
+        if (info != null && !info.isOffline()) {
+          log("found region in META: " + hri.getRegionNameAsString());
+          break;
+        }
+      }
+      Threads.sleep(10);
+    }
+  }
+
+  public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
+      throws IOException, InterruptedException {
+    log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
+    long start = System.currentTimeMillis();
+    HTable table = new HTable(conf, hri.getTableName());
+
+    try {
+      Get get = new Get(hri.getStartKey());
+      while (System.currentTimeMillis() - start < timeout) {
+        try {
+          table.get(get);
+          break;
+        } catch(IOException ex) {
+          //wait some more
+        }
+        Threads.sleep(10);
+      }
+    } finally {
+      IOUtils.closeQuietly(table);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 3771d9a6bd0a..84d8db89f160 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -38,6 +38,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -54,6 +55,7 @@
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -70,11 +72,12 @@
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
-import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
-import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
+import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
+import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -409,7 +412,7 @@ public void testHbckThreadpooling() throws Exception {
       deleteTable(table);
     }    
   }
-  
+
   @Test
   public void testHbckFixOrphanTable() throws Exception {
     String table = "tableInfo";
@@ -418,31 +421,31 @@ public void testHbckFixOrphanTable() throws Exception {
     try {
       setupTable(table);
       HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
-      
+
       Path hbaseTableDir = new Path(conf.get(HConstants.HBASE_DIR) + "/" + table );
       fs = hbaseTableDir.getFileSystem(conf);
       FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
       tableinfo = status.getPath();
       fs.rename(tableinfo, new Path("/.tableinfo"));
-      
+
       //to report error if .tableinfo is missing.
-      HBaseFsck hbck = doFsck(conf, false); 
+      HBaseFsck hbck = doFsck(conf, false);
       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
-      
+
       // fix OrphanTable with default .tableinfo (htd not yet cached on master)
       hbck = doFsck(conf, true);
       assertNoErrors(hbck);
       status = null;
       status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
       assertNotNull(status);
-      
+
       HTableDescriptor htd = admin.getTableDescriptor(table.getBytes());
       htd.setValue("NOT_DEFAULT", "true");
       admin.disableTable(table);
       admin.modifyTable(table.getBytes(), htd);
       admin.enableTable(table);
       fs.delete(status.getPath(), true);
-      
+
       // fix OrphanTable with cache
       htd = admin.getTableDescriptor(table.getBytes()); // warms up cached htd on master
       hbck = doFsck(conf, true);
@@ -1185,6 +1188,7 @@ public void testFixByTable() throws Exception {
   @Test
   public void testLingeringSplitParent() throws Exception {
     String table = "testLingeringSplitParent";
+    HTable meta = null;
     try {
       setupTable(table);
       assertEquals(ROWKEYS.length, countRows());
@@ -1198,7 +1202,7 @@ public void testLingeringSplitParent() throws Exception {
         Bytes.toBytes("C"), true, true, false);
 
       // Create a new meta entry to fake it as a split parent.
-      HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
+      meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
       HRegionInfo hri = location.getRegionInfo();
 
       HRegionInfo a = new HRegionInfo(tbl.getTableName(),
@@ -1256,6 +1260,119 @@ public void testLingeringSplitParent() throws Exception {
       assertEquals(ROWKEYS.length, countRows());
     } finally {
       deleteTable(table);
+      IOUtils.closeQuietly(meta);
+    }
+  }
+
+  /**
+   * Tests that LINGERING_SPLIT_PARENT is not erroneously reported for
+   * valid cases where the daughters are there.
+   */
+  @Test
+  public void testValidLingeringSplitParent() throws Exception {
+    String table = "testLingeringSplitParent";
+    HTable meta = null;
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table);
+      HRegionLocation location = tbl.getRegionLocation("B");
+
+      meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
+      HRegionInfo hri = location.getRegionInfo();
+
+      // do a regular split
+      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+      byte[] regionName = location.getRegionInfo().getRegionName();
+      admin.split(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
+      TestEndToEndSplitTransaction.blockUntilRegionSplit(
+          TEST_UTIL.getConfiguration(), 60000, regionName, true);
+
+      // TODO: fixHdfsHoles does not work against splits, since the parent dir lingers on
+      // for some time until children references are deleted. HBCK erroneously sees this as
+      // overlapping regions
+      HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, null);
+      assertErrors(hbck, new ERROR_CODE[] {}); //no LINGERING_SPLIT_PARENT reported
+
+      // assert that the split META entry is still there.
+      Get get = new Get(hri.getRegionName());
+      Result result = meta.get(get);
+      assertNotNull(result);
+      assertNotNull(MetaReader.parseCatalogResult(result).getFirst());
+
+      assertEquals(ROWKEYS.length, countRows());
+
+      // assert that we still have the split regions
+      assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1); //SPLITS + 1 is # regions pre-split.
+      assertNoErrors(doFsck(conf, false));
+    } finally {
+      deleteTable(table);
+      IOUtils.closeQuietly(meta);
+    }
+  }
+
+  /**
+   * Split crashed after write to META finished for the parent region, but
+   * failed to write daughters (pre HBASE-7721 codebase)
+   */
+  @Test
+  public void testSplitDaughtersNotInMeta() throws Exception {
+    String table = "testSplitdaughtersNotInMeta";
+    HTable meta = null;
+    try {
+      setupTable(table);
+      assertEquals(ROWKEYS.length, countRows());
+
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table);
+      HRegionLocation location = tbl.getRegionLocation("B");
+
+      meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
+      HRegionInfo hri = location.getRegionInfo();
+
+      // do a regular split
+      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+      byte[] regionName = location.getRegionInfo().getRegionName();
+      admin.split(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
+      TestEndToEndSplitTransaction.blockUntilRegionSplit(
+          TEST_UTIL.getConfiguration(), 60000, regionName, true);
+
+      PairOfSameType<HRegionInfo> daughters = MetaReader.getDaughterRegions(meta.get(new Get(regionName)));
+
+      // Delete daughter regions from meta, but not hdfs, unassign it.
+      Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
+      undeployRegion(admin, hris.get(daughters.getFirst()), daughters.getFirst());
+      undeployRegion(admin, hris.get(daughters.getSecond()), daughters.getSecond());
+
+      meta.delete(new Delete(daughters.getFirst().getRegionName()));
+      meta.delete(new Delete(daughters.getSecond().getRegionName()));
+      meta.flushCommits();
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); //no LINGERING_SPLIT_PARENT
+
+      // now fix it. The fix should not revert the region split, but add daughters to META
+      hbck = doFsck(conf, true, true, false, false, false, false, false, false, null);
+      assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
+          ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // assert that the split META entry is still there.
+      Get get = new Get(hri.getRegionName());
+      Result result = meta.get(get);
+      assertNotNull(result);
+      assertNotNull(MetaReader.parseCatalogResult(result).getFirst());
+
+      assertEquals(ROWKEYS.length, countRows());
+
+      // assert that we still have the split regions
+      assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1); //SPLITS + 1 is # regions pre-split.
+      assertNoErrors(doFsck(conf, false)); //should be fixed by now
+    } finally {
+      deleteTable(table);
+      IOUtils.closeQuietly(meta);
     }
   }
 

From 43edd93495d68e254aea15e78fb73a54c21584b7 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 28 Feb 2013 00:33:06 +0000
Subject: [PATCH 0806/1540] HBASE-7725: Add ability to create custom compaction
 request

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451057 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/BaseRegionObserver.java |  36 +++++
 .../hbase/coprocessor/RegionObserver.java     | 128 +++++++++++++++---
 .../regionserver/CompactSplitThread.java      |  61 ++++++---
 .../regionserver/CompactionRequestor.java     |  42 +++++-
 .../hadoop/hbase/regionserver/Compactor.java  |  52 +++++--
 .../hbase/regionserver/HRegionServer.java     |  31 +++--
 .../regionserver/RegionCoprocessorHost.java   |  50 +++----
 .../hadoop/hbase/regionserver/Store.java      |  39 +++---
 .../compactions/CompactionRequest.java        |  99 ++++++++++----
 .../hbase/regionserver/TestCompaction.java    | 102 +++++++++++++-
 10 files changed, 498 insertions(+), 142 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 891cba9ee38f..81c590f380e7 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -41,6 +41,8 @@
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Pair;
@@ -108,10 +110,23 @@ public void postSplit(ObserverContext<RegionCoprocessorEnvironment> e, HRegion l
       throws IOException {
   }
 
+  @Override
+  public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final List<StoreFile> candidates, final CompactionRequest request)
+      throws IOException {
+    preCompactSelection(c, store, candidates);
+  }
+
   @Override
   public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final List<StoreFile> candidates) throws IOException { }
 
+  @Override
+  public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final ImmutableList<StoreFile> selected, CompactionRequest request) {
+    postCompactSelection(c, store, selected);
+  }
+
   @Override
   public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final ImmutableList<StoreFile> selected) { }
@@ -122,6 +137,13 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
     return scanner;
   }
 
+  @Override
+  public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
+      final Store store, final InternalScanner scanner, CompactionRequest request)
+      throws IOException {
+    return preCompact(e, store, scanner);
+  }
+
   @Override
   public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
@@ -129,11 +151,25 @@ public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoproce
     return null;
   }
 
+  @Override
+  public InternalScanner preCompactScannerOpen(
+      final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      List<? extends KeyValueScanner> scanners, final ScanType scanType, final long earliestPutTs,
+      final InternalScanner s, CompactionRequest request) throws IOException {
+    return preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s);
+  }
+
   @Override
   public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
       final StoreFile resultFile) throws IOException {
   }
 
+  @Override
+  public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
+      final StoreFile resultFile, CompactionRequest request) throws IOException {
+    postCompact(e, store, resultFile);
+  }
+
   @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> e,
       final byte [] row, final byte [] family, final Result result)
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 31fd9387ac6e..93ff2528ba94 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 
@@ -125,9 +126,23 @@ void postFlush(final ObserverContext<RegionCoprocessorEnvironment> c, final Stor
       final StoreFile resultFile) throws IOException;
 
   /**
-   * Called prior to selecting the {@link StoreFile}s to compact from the list
-   * of available candidates.  To alter the files used for compaction, you may
-   * mutate the passed in list of candidates.
+   * Called prior to selecting the {@link StoreFile StoreFiles} to compact from the list of
+   * available candidates. To alter the files used for compaction, you may mutate the passed in list
+   * of candidates.
+   * @param c the environment provided by the region server
+   * @param store the store where compaction is being requested
+   * @param candidates the store files currently available for compaction
+   * @param request custom compaction request
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final List<StoreFile> candidates, final CompactionRequest request)
+      throws IOException;
+
+  /**
+   * Called prior to selecting the {@link StoreFile}s to compact from the list of available
+   * candidates. To alter the files used for compaction, you may mutate the passed in list of
+   * candidates.
    * @param c the environment provided by the region server
    * @param store the store where compaction is being requested
    * @param candidates the store files currently available for compaction
@@ -147,9 +162,20 @@ void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final ImmutableList<StoreFile> selected);
 
   /**
-   * Called prior to writing the {@link StoreFile}s selected for compaction into
-   * a new {@code StoreFile}.  To override or modify the compaction process,
-   * implementing classes have two options:
+   * Called after the {@link StoreFile}s to compact have been selected from the available
+   * candidates.
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param selected the store files selected to compact
+   * @param request custom compaction request
+   */
+  void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final ImmutableList<StoreFile> selected, CompactionRequest request);
+
+  /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile}. To override or modify the compaction process, implementing classes have two
+   * options:
    * <ul>
    *   <li>Wrap the provided {@link InternalScanner} with a custom
    *   implementation that is returned from this method.  The custom scanner
@@ -164,38 +190,85 @@ void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
    * </ul>
    * @param c the environment provided by the region server
    * @param store the store being compacted
-   * @param scanner the scanner over existing data used in the store file
-   * rewriting
-   * @return the scanner to use during compaction.  Should not be {@code null}
-   * unless the implementation is writing new store files on its own.
+   * @param scanner the scanner over existing data used in the store file rewriting
+   * @return the scanner to use during compaction. Should not be {@code null} unless the
+   *         implementation is writing new store files on its own.
    * @throws IOException if an error occurred on the coprocessor
    */
   InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final InternalScanner scanner) throws IOException;
 
   /**
-   * Called prior to writing the {@link StoreFile}s selected for compaction into
-   * a new {@code StoreFile} and prior to creating the scanner used to read the
-   * input files.  To override or modify the compaction process,
-   * implementing classes can return a new scanner to provide the KeyValues to be
-   * stored into the new {@code StoreFile} or null to perform the default processing.
-   * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile}. To override or modify the compaction process, implementing classes have two
+   * options:
+   * <ul>
+   * <li>Wrap the provided {@link InternalScanner} with a custom implementation that is returned
+   * from this method. The custom scanner can then inspect {@link KeyValue}s from the wrapped
+   * scanner, applying its own policy to what gets written.</li>
+   * <li>Call {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} and provide a
+   * custom implementation for writing of new {@link StoreFile}s. <strong>Note: any implementations
+   * bypassing core compaction using this approach must write out new store files themselves or the
+   * existing data will no longer be available after compaction.</strong></li>
+   * </ul>
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param scanner the scanner over existing data used in the store file rewriting
+   * @param request the requested compaction
+   * @return the scanner to use during compaction. Should not be {@code null} unless the
+   *         implementation is writing new store files on its own.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final InternalScanner scanner, CompactionRequest request)
+      throws IOException;
+
+  /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile} and prior to creating the scanner used to read the input files. To override
+   * or modify the compaction process, implementing classes can return a new scanner to provide the
+   * KeyValues to be stored into the new {@code StoreFile} or null to perform the default
+   * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
    * effect in this hook.
    * @param c the environment provided by the region server
    * @param store the store being compacted
    * @param scanners the list {@link StoreFileScanner}s to be read from
    * @param scantype the {@link ScanType} indicating whether this is a major or minor compaction
-   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved
-   * store files
+   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store
+   *          files
    * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
-   * @return the scanner to use during compaction.  {@code null} if the default implementation
-   * is to be used.
+   * @return the scanner to use during compaction. {@code null} if the default implementation is to
+   *         be used.
    * @throws IOException if an error occurred on the coprocessor
    */
   InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
       final long earliestPutTs, final InternalScanner s) throws IOException;
 
+  /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile} and prior to creating the scanner used to read the input files. To override
+   * or modify the compaction process, implementing classes can return a new scanner to provide the
+   * KeyValues to be stored into the new {@code StoreFile} or null to perform the default
+   * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * effect in this hook.
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param scanners the list {@link StoreFileScanner}s to be read from
+   * @param scanType the {@link ScanType} indicating whether this is a major or minor compaction
+   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store
+   *          files
+   * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+   * @param request the requested compaction
+   * @return the scanner to use during compaction. {@code null} if the default implementation is to
+   *         be used.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
+      final long earliestPutTs, final InternalScanner s, CompactionRequest request)
+      throws IOException;
+
   /**
    * Called after compaction has completed and the new store file has been
    * moved in to place.
@@ -208,9 +281,20 @@ void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c, final St
       StoreFile resultFile) throws IOException;
 
   /**
-   * Called before the region is split.
+   * Called after compaction has completed and the new store file has been moved in to place.
    * @param c the environment provided by the region server
-   * (e.getRegion() returns the parent region)
+   * @param store the store being compacted
+   * @param resultFile the new store file written out during compaction
+   * @param request the requested compaction
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      StoreFile resultFile, CompactionRequest request) throws IOException;
+
+  /**
+   * Called before the region is split.
+   * @param c the environment provided by the region server (e.getRegion() returns the parent
+   *          region)
    * @throws IOException if an error occurred on the coprocessor
    */
   void preSplit(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 28662689b0d5..e1ebe6d8c84a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -21,7 +21,9 @@
 
 import java.io.IOException;
 import java.util.concurrent.Executors;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
@@ -184,37 +186,51 @@ public synchronized void requestSplit(final HRegion r, byte[] midKey) {
     }
   }
 
-  public synchronized void requestCompaction(final HRegion r,
-      final String why) throws IOException {
-    for(Store s : r.getStores().values()) {
-      requestCompaction(r, s, why, Store.NO_PRIORITY);
-    }
+  @Override
+  public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why)
+      throws IOException {
+    return requestCompaction(r, why, null);
   }
 
-  public synchronized void requestCompaction(final HRegion r, final Store s,
-      final String why) throws IOException {
-    requestCompaction(r, s, why, Store.NO_PRIORITY);
+  @Override
+  public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+      List<CompactionRequest> requests) throws IOException {
+    return requestCompaction(r, why, Store.NO_PRIORITY, requests);
   }
 
-  public synchronized void requestCompaction(final HRegion r, final String why,
-      int p) throws IOException {
-    for(Store s : r.getStores().values()) {
-      requestCompaction(r, s, why, p);
+  @Override
+  public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+      final String why,
+      CompactionRequest request) throws IOException {
+    return requestCompaction(r, s, why, Store.NO_PRIORITY, request);
+  }
+
+  @Override
+  public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+      int pri, final List<CompactionRequest> requests) throws IOException {
+    List<CompactionRequest> ret;
+    // not a special compaction request, so make out own list
+    if (requests == null) {
+      ret = new ArrayList<CompactionRequest>(r.getStores().size());
+      for (Store s : r.getStores().values()) {
+        ret.add(requestCompaction(r, s, why, pri, null));
+      }
+    } else {
+      ret = new ArrayList<CompactionRequest>(requests.size());
+      for (CompactionRequest request : requests) {
+        ret.add(requestCompaction(r, request.getStore(), why, pri, request));
+      }
     }
+    return ret;
   }
 
-  /**
-   * @param r HRegion store belongs to
-   * @param s Store to request compaction on
-   * @param why Why compaction requested -- used in debug messages
-   * @param priority override the default priority (NO_PRIORITY == decide)
-   */
-  public synchronized void requestCompaction(final HRegion r, final Store s,
-      final String why, int priority) throws IOException {
+  @Override
+  public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+      final String why, int priority, CompactionRequest request) throws IOException {
     if (this.server.isStopped()) {
-      return;
+      return null;
     }
-    CompactionRequest cr = s.requestCompaction(priority);
+    CompactionRequest cr = s.requestCompaction(priority, request);
     if (cr != null) {
       cr.setServer(server);
       if (priority != Store.NO_PRIORITY) {
@@ -235,6 +251,7 @@ public synchronized void requestCompaction(final HRegion r, final Store s,
             " because compaction request was cancelled");
       }
     }
+    return cr;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
index 255c4ecb458b..978854fedc7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
@@ -20,39 +20,71 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 
 public interface CompactionRequestor {
   /**
    * @param r Region to compact
    * @param why Why compaction was requested -- used in debug messages
+   * @return the created {@link CompactionRequest CompactionRequets} or, if no compactions were
+   *         started, an empty list
+   * @throws IOException
+   */
+  public List<CompactionRequest> requestCompaction(final HRegion r, final String why)
+      throws IOException;
+
+  /**
+   * @param r Region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @param requests custom compaction requests. Each compaction must specify the store on which it
+   *          is acting. Can be <tt>null</tt> in which case a compaction will be attempted on all
+   *          stores for the region.
+   * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+   *         compactions were started
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final String why) throws IOException;
+  public List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+      List<CompactionRequest> requests) throws IOException;
 
   /**
    * @param r Region to compact
    * @param s Store within region to compact
    * @param why Why compaction was requested -- used in debug messages
+   * @param request custom compaction request for the {@link HRegion} and {@link Store}. Custom
+   *          request must be <tt>null</tt> or be constructed with matching region and store.
+   * @return The created {@link CompactionRequest} or <tt>null</tt> if no compaction was started.
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final Store s, final String why) throws IOException;
+  public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+      CompactionRequest request) throws IOException;
 
   /**
    * @param r Region to compact
    * @param why Why compaction was requested -- used in debug messages
    * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @param requests custom compaction requests. Each compaction must specify the store on which it
+   *          is acting. Can be <tt>null</tt> in which case a compaction will be attempted on all
+   *          stores for the region.
+   * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+   *         compactions were started.
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final String why, int pri) throws IOException;
+  public List<CompactionRequest> requestCompaction(final HRegion r, final String why, int pri,
+      List<CompactionRequest> requests) throws IOException;
 
   /**
    * @param r Region to compact
    * @param s Store within region to compact
    * @param why Why compaction was requested -- used in debug messages
    * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @param request request custom compaction request to run. {@link Store} and {@link HRegion} for
+   *          the request must match the region and store specified here.
+   * @return The created {@link CompactionRequest} or <tt>null</tt> if no compaction was started
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final Store s,
-      final String why, int pri) throws IOException;
+  public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+      int pri, CompactionRequest request) throws IOException;
 
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
index ad13248695a3..900a137a6aa8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.util.StringUtils;
 
@@ -50,24 +51,45 @@ class Compactor extends Configured {
   }
 
   /**
-   * Do a minor/major compaction on an explicit set of storefiles from a Store.
-   *
-   * @param store Store the files belong to
-   * @param filesToCompact which files to compact
-   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
-   * @param maxId Readers maximum sequence id.
-   * @return Product of compaction or null if all cells expired or deleted and
-   * nothing made it through the compaction.
+   * Compact a list of files for testing. Creates a fake {@link CompactionRequest} to pass to the
+   * actual compaction method
+   * @param store store which should be compacted
+   * @param conf configuration to use when generating the compaction selection
+   * @param filesToCompact the files to compact. They are used a the compaction selection for the
+   *          generated {@link CompactionRequest}
+   * @param isMajor <tt>true</tt> to initiate a major compaction (prune all deletes, max versions,
+   *          etc)
+   * @param maxId maximum sequenceID == the last key of all files in the compaction
+   * @return product of the compaction or null if all cells expired or deleted and nothing made it
+   *         through the compaction.
    * @throws IOException
    */
-  StoreFile.Writer compact(final Store store,
+  public StoreFile.Writer compactForTesting(final Store store, Configuration conf,
       final Collection<StoreFile> filesToCompact,
-      final boolean majorCompaction, final long maxId)
-  throws IOException {
+      boolean isMajor, long maxId) throws IOException {
+    return compact(CompactionRequest.getRequestForTesting(store, conf, filesToCompact, isMajor),
+      maxId);
+  }
+
+  /**
+   * Do a minor/major compaction on an explicit set of storefiles from a Store.
+   * @param request the requested compaction that contains all necessary information to complete the
+   *          compaction (i.e. the store, the files, etc.)
+   * @return Product of compaction or null if all cells expired or deleted and nothing made it
+   *         through the compaction.
+   * @throws IOException
+   */
+  StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
     // Calculate maximum key count after compaction (for blooms)
     // Also calculate earliest put timestamp if major compaction
     int maxKeyCount = 0;
     long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+
+    // pull out the interesting things from the CR for ease later
+    final Store store = request.getStore();
+    final boolean majorCompaction = request.isMajor();
+    final List<StoreFile> filesToCompact = request.getFiles();
+
     for (StoreFile file : filesToCompact) {
       StoreFile.Reader r = file.getReader();
       if (r == null) {
@@ -76,7 +98,8 @@ StoreFile.Writer compact(final Store store,
       }
       // NOTE: getFilterEntries could cause under-sized blooms if the user
       //       switches bloom type (e.g. from ROW to ROWCOL)
-      long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType()) ?
+      long keyCount = (r.getBloomFilterType() == store.getFamily()
+          .getBloomFilterType()) ?
           r.getFilterEntries() : r.getEntries();
       maxKeyCount += keyCount;
       // For major compactions calculate the earliest put timestamp
@@ -130,7 +153,8 @@ StoreFile.Writer compact(final Store store,
           scanner = store.getHRegion()
               .getCoprocessorHost()
               .preCompactScannerOpen(store, scanners,
-                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
+                majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs,
+                request);
         }
         if (scanner == null) {
           Scan scan = new Scan();
@@ -142,7 +166,7 @@ StoreFile.Writer compact(final Store store,
         }
         if (store.getHRegion().getCoprocessorHost() != null) {
           InternalScanner cpScanner =
-            store.getHRegion().getCoprocessorHost().preCompact(store, scanner);
+            store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
           // NULL scanner returned from coprocessor hooks means skip normal processing
           if (cpScanner == null) {
             return null;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b780b8e04296..158cd7cd3084 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1272,17 +1272,17 @@ protected void chore() {
           try {
             if (s.needsCompaction()) {
               // Queue a compaction. Will recognize if major is needed.
-              this.instance.compactSplitThread.requestCompaction(r, s,
-                getName() + " requests compaction");
+              this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                  + " requests compaction", null);
             } else if (s.isMajorCompaction()) {
-              if (majorCompactPriority == DEFAULT_PRIORITY ||
-                  majorCompactPriority > r.getCompactPriority()) {
-                this.instance.compactSplitThread.requestCompaction(r, s,
-                    getName() + " requests major compaction; use default priority");
+              if (majorCompactPriority == DEFAULT_PRIORITY
+                  || majorCompactPriority > r.getCompactPriority()) {
+                this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                    + " requests major compaction; use default priority", null);
               } else {
-               this.instance.compactSplitThread.requestCompaction(r, s,
-                  getName() + " requests major compaction; use configured priority",
-                  this.majorCompactPriority);
+                this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                    + " requests major compaction; use configured priority",
+                  this.majorCompactPriority, null);
               }
             }
           } catch (IOException e) {
@@ -1705,7 +1705,7 @@ public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
     // Do checks to see if we need to compact (references or too many files)
     for (Store s : r.getStores().values()) {
       if (s.hasReferences() || s.needsCompaction()) {
-        getCompactionRequester().requestCompaction(r, s, "Opening Region");
+        getCompactionRequester().requestCompaction(r, s, "Opening Region", null);
       }
     }
     // Update ZK, ROOT or META
@@ -3138,10 +3138,10 @@ public void compactRegion(HRegionInfo regionInfo, boolean major,  byte[] family)
     String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
     if (family != null) {
       compactSplitThread.requestCompaction(region, store, log,
-        Store.PRIORITY_USER);
+        Store.PRIORITY_USER, null);
     } else {
       compactSplitThread.requestCompaction(region, log,
-        Store.PRIORITY_USER);
+        Store.PRIORITY_USER, null);
     }
   }
 
@@ -3962,4 +3962,11 @@ private boolean isHealthCheckerConfigured() {
     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
   }
+
+  /**
+   * @return the underlying {@link CompactSplitThread} for the servers
+   */
+  public CompactSplitThread getCompactSplitThread() {
+    return this.compactSplitThread;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 4e97dc32cf0a..0da7b2d0fb0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -57,6 +57,7 @@
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -343,10 +344,10 @@ public void postClose(boolean abortRequested){
 
   /**
    * See
-   * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner)}
+   * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner, CompactionRequest)}
    */
   public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner> scanners,
-      ScanType scanType, long earliestPutTs) throws IOException {
+      ScanType scanType, long earliestPutTs, CompactionRequest request) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     InternalScanner s = null;
     for (RegionEnvironment env: coprocessors) {
@@ -354,7 +355,7 @@ public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner>
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
           s = ((RegionObserver) env.getInstance()).preCompactScannerOpen(ctx, store, scanners,
-              scanType, earliestPutTs, s);
+            scanType, earliestPutTs, s, request);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env,e);
         }
@@ -367,25 +368,25 @@ public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner>
   }
 
   /**
-   * Called prior to selecting the {@link StoreFile}s for compaction from
-   * the list of currently available candidates.
+   * Called prior to selecting the {@link StoreFile}s for compaction from the list of currently
+   * available candidates.
    * @param store The store where compaction is being requested
    * @param candidates The currently available store files
+   * @param request custom compaction request
    * @return If {@code true}, skip the normal selection process and use the current list
    * @throws IOException
    */
-  public boolean preCompactSelection(Store store, List<StoreFile> candidates) throws IOException {
+  public boolean preCompactSelection(Store store, List<StoreFile> candidates,
+      CompactionRequest request) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     boolean bypass = false;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          ((RegionObserver)env.getInstance()).preCompactSelection(
-              ctx, store, candidates);
+          ((RegionObserver) env.getInstance()).preCompactSelection(ctx, store, candidates, request);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env,e);
-
         }
         bypass |= ctx.shouldBypass();
         if (ctx.shouldComplete()) {
@@ -397,20 +398,20 @@ public boolean preCompactSelection(Store store, List<StoreFile> candidates) thro
   }
 
   /**
-   * Called after the {@link StoreFile}s to be compacted have been selected
-   * from the available candidates.
+   * Called after the {@link StoreFile}s to be compacted have been selected from the available
+   * candidates.
    * @param store The store where compaction is being requested
    * @param selected The store files selected to compact
+   * @param request custom compaction
    */
-  public void postCompactSelection(Store store,
-      ImmutableList<StoreFile> selected) {
+  public void postCompactSelection(Store store, ImmutableList<StoreFile> selected,
+      CompactionRequest request) {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          ((RegionObserver)env.getInstance()).postCompactSelection(
-              ctx, store, selected);
+          ((RegionObserver) env.getInstance()).postCompactSelection(ctx, store, selected, request);
         } catch (Throwable e) {
           handleCoprocessorThrowableNoRethrow(env,e);
         }
@@ -425,19 +426,20 @@ public void postCompactSelection(Store store,
    * Called prior to rewriting the store files selected for compaction
    * @param store the store being compacted
    * @param scanner the scanner used to read store data during compaction
-   * @throws IOException 
+   * @param request the compaction that will be executed
+   * @throws IOException
    */
-  public InternalScanner preCompact(Store store, InternalScanner scanner) throws IOException {
+  public InternalScanner preCompact(Store store, InternalScanner scanner, 
+      CompactionRequest request) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     boolean bypass = false;
-    for (RegionEnvironment env: coprocessors) {
+    for (RegionEnvironment env : coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          scanner = ((RegionObserver)env.getInstance()).preCompact(
-              ctx, store, scanner);
+          scanner = ((RegionObserver) env.getInstance()).preCompact(ctx, store, scanner, request);
         } catch (Throwable e) {
-          handleCoprocessorThrowable(env,e);
+          handleCoprocessorThrowable(env, e);
         }
         bypass |= ctx.shouldBypass();
         if (ctx.shouldComplete()) {
@@ -452,15 +454,17 @@ public InternalScanner preCompact(Store store, InternalScanner scanner) throws I
    * Called after the store compaction has completed.
    * @param store the store being compacted
    * @param resultFile the new store file written during compaction
+   * @param request the compaction that is being executed
    * @throws IOException
    */
-  public void postCompact(Store store, StoreFile resultFile) throws IOException {
+  public void postCompact(Store store, StoreFile resultFile, CompactionRequest request)
+      throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          ((RegionObserver)env.getInstance()).postCompact(ctx, store, resultFile);
+          ((RegionObserver) env.getInstance()).postCompact(ctx, store, resultFile, request);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env, e);
         }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 545495373631..c9432009847b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1106,13 +1106,12 @@ StoreFile compact(CompactionRequest cr) throws IOException {
 
     StoreFile sf = null;
     try {
-      StoreFile.Writer writer =
-        this.compactor.compact(this, filesToCompact, cr.isMajor(), maxId);
+      StoreFile.Writer writer = this.compactor.compact(cr, maxId);
       // Move the compaction into place.
       if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
         sf = completeCompaction(filesToCompact, writer);
         if (region.getCoprocessorHost() != null) {
-          region.getCoprocessorHost().postCompact(this, sf);
+          region.getCoprocessorHost().postCompact(this, sf, cr);
         }
       } else {
         // Create storefile around what we wrote with a reader on it.
@@ -1175,12 +1174,12 @@ public void compactRecentForTesting(int N) throws IOException {
 
     try {
       // Ready to go. Have list of files to compact.
-      StoreFile.Writer writer =
-        this.compactor.compact(this, filesToCompact, isMajor, maxId);
+      StoreFile.Writer writer = this.compactor.compactForTesting(this, conf, filesToCompact,
+        isMajor, maxId);
       // Move the compaction into place.
       StoreFile sf = completeCompaction(filesToCompact, writer);
       if (region.getCoprocessorHost() != null) {
-        region.getCoprocessorHost().postCompact(this, sf);
+        region.getCoprocessorHost().postCompact(this, sf, null);
       }
     } finally {
       synchronized (filesCompacting) {
@@ -1331,16 +1330,16 @@ long getNextMajorCompactTime() {
   }
 
   public CompactionRequest requestCompaction() throws IOException {
-    return requestCompaction(NO_PRIORITY);
+    return requestCompaction(NO_PRIORITY, null);
   }
 
-  public CompactionRequest requestCompaction(int priority) throws IOException {
+  public CompactionRequest requestCompaction(int priority, CompactionRequest request)
+      throws IOException {
     // don't even select for compaction if writes are disabled
     if (!this.region.areWritesEnabled()) {
       return null;
     }
 
-    CompactionRequest ret = null;
     this.lock.readLock().lock();
     try {
       synchronized (filesCompacting) {
@@ -1357,8 +1356,7 @@ public CompactionRequest requestCompaction(int priority) throws IOException {
 
         boolean override = false;
         if (region.getCoprocessorHost() != null) {
-          override = region.getCoprocessorHost().preCompactSelection(
-              this, candidates);
+          override = region.getCoprocessorHost().preCompactSelection(this, candidates, request);
         }
         CompactSelection filesToCompact;
         if (override) {
@@ -1370,7 +1368,7 @@ public CompactionRequest requestCompaction(int priority) throws IOException {
 
         if (region.getCoprocessorHost() != null) {
           region.getCoprocessorHost().postCompactSelection(this,
-              ImmutableList.copyOf(filesToCompact.getFilesToCompact()));
+            ImmutableList.copyOf(filesToCompact.getFilesToCompact()), request);
         }
 
         // no files to compact
@@ -1396,15 +1394,24 @@ public CompactionRequest requestCompaction(int priority) throws IOException {
 
         // everything went better than expected. create a compaction request
         int pri = getCompactPriority(priority);
-        ret = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+        //not a special compaction request, so we need to make one
+        if(request == null){
+          request = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+        } else {
+          // update the request with what the system thinks the request should be
+          // its up to the request if it wants to listen
+          request.setSelection(filesToCompact);
+          request.setIsMajor(isMajor);
+          request.setPriority(pri);
+        }
       }
     } finally {
       this.lock.readLock().unlock();
     }
-    if (ret != null) {
-      CompactionRequest.preRequest(ret);
+    if (request != null) {
+      CompactionRequest.preRequest(request);
     }
-    return ret;
+    return request;
   }
 
   public void finishRequest(CompactionRequest cr) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
index 0a931bb6b051..65f14beb9782 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
@@ -20,6 +20,8 @@
 package org.apache.hadoop.hbase.regionserver.compactions;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.RejectedExecutionHandler;
@@ -28,6 +30,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -51,9 +54,9 @@ public class CompactionRequest implements Comparable<CompactionRequest>,
     static final Log LOG = LogFactory.getLog(CompactionRequest.class);
     private final HRegion r;
     private final Store s;
-    private final CompactSelection compactSelection;
-    private final long totalSize;
-    private final boolean isMajor;
+    private CompactSelection compactSelection;
+    private long totalSize;
+    private boolean isMajor;
     private int p;
     private final Long timeInNanos;
     private HRegionServer server = null;
@@ -66,23 +69,51 @@ public class CompactionRequest implements Comparable<CompactionRequest>,
     private static final ConcurrentHashMap<Long, AtomicInteger>
       minorCompactions = new ConcurrentHashMap<Long, AtomicInteger>();
 
-    public CompactionRequest(HRegion r, Store s,
-        CompactSelection files, boolean isMajor, int p) {
-      Preconditions.checkNotNull(r);
-      Preconditions.checkNotNull(files);
+  /**
+   * Create a simple compaction request just for testing - this lets you specify everything you
+   * would need in the general case of testing compactions from an external perspective (e.g.
+   * requesting a compaction through the HRegion).
+   * @param store
+   * @param conf
+   * @param selection
+   * @param isMajor
+   * @return a request that is useful in requesting compactions for testing
+   */
+  public static CompactionRequest getRequestForTesting(Store store, Configuration conf,
+      Collection<StoreFile> selection, boolean isMajor) {
+    return new CompactionRequest(store.getHRegion(), store, new CompactSelection(conf,
+        new ArrayList<StoreFile>(
+        selection)), isMajor, 0, System.nanoTime());
+  }
 
-      this.r = r;
-      this.s = s;
-      this.compactSelection = files;
-      long sz = 0;
-      for (StoreFile sf : files.getFilesToCompact()) {
-        sz += sf.getReader().length();
-      }
-      this.totalSize = sz;
-      this.isMajor = isMajor;
-      this.p = p;
-      this.timeInNanos = System.nanoTime();
+  /**
+   * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+   * compaction before being used. Uses the current system time on creation as the start time.
+   * @param region region that is being compacted
+   * @param store store which is being compacted
+   * @param priority specified priority with which this compaction should enter the queue.
+   */
+  public CompactionRequest(HRegion region, Store store, int priority) {
+    this(region, store, null, false, priority, System.nanoTime());
+  }
+
+  public CompactionRequest(HRegion r, Store s, CompactSelection files, boolean isMajor, int p) {
+    // delegate to the internal constructor after checking basic preconditions
+    this(Preconditions.checkNotNull(r), s, Preconditions.checkNotNull(files), isMajor, p, System
+        .nanoTime());
+  }
+
+  private CompactionRequest(HRegion region, Store store, CompactSelection files, boolean isMajor,
+      int priority, long startTime) {
+    this.r = region;
+    this.s = store;
+    this.isMajor = isMajor;
+    this.p = priority;
+    this.timeInNanos = startTime;
+    if (files != null) {
+      this.setSelection(files);
     }
+  }
 
     /**
      * Find out if a given region in compaction now.
@@ -216,6 +247,28 @@ public void setServer(HRegionServer hrs) {
       this.server = hrs;
     }
 
+    /**
+     * Set the files (and, implicitly, the size of the compaction based on those files)
+     * @param files files that should be included in the compaction
+     */
+    public void setSelection(CompactSelection files) {
+      long sz = 0;
+      for (StoreFile sf : files.getFilesToCompact()) {
+        sz += sf.getReader().length();
+      }
+      this.totalSize = sz;
+      this.compactSelection = files;
+    }
+
+    /**
+     * Specify if this compaction should be a major compaction based on the state of the store
+     * @param isMajor <tt>true</tt> if the system determines that this compaction should be a major
+     *          compaction
+     */
+    public void setIsMajor(boolean isMajor) {
+      this.isMajor = isMajor;
+    }
+
     @Override
     public String toString() {
       String fsList = Joiner.on(", ").join(
@@ -255,11 +308,11 @@ public void run() {
           server.getMetrics().addCompaction(now - start, this.totalSize);
           // degenerate case: blocked regions require recursive enqueues
           if (s.getCompactPriority() <= 0) {
-            server.compactSplitThread
-              .requestCompaction(r, s, "Recursive enqueue");
+            server.getCompactSplitThread()
+              .requestCompaction(r, s, "Recursive enqueue", null);
           } else {
             // see if the compaction has caused us to exceed max region size
-            server.compactSplitThread.requestSplit(r);
+            server.getCompactSplitThread().requestSplit(r);
           }
         }
       } catch (IOException ex) {
@@ -271,7 +324,7 @@ public void run() {
         server.checkFileSystem();
       } finally {
         s.finishRequest(this);
-        LOG.debug("CompactSplitThread status: " + server.compactSplitThread);
+      LOG.debug("CompactSplitThread status: " + server.getCompactSplitThread());
       }
     }
 
@@ -299,4 +352,4 @@ public void rejectedExecution(Runnable request, ThreadPoolExecutor pool) {
         }
       }
     }
-  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 93908e7c8670..4971a86852cc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -24,10 +24,12 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -52,9 +54,11 @@
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
@@ -566,8 +570,12 @@ private int count() throws IOException {
   }
 
   private void createStoreFile(final HRegion region) throws IOException {
+    createStoreFile(region, Bytes.toString(COLUMN_FAMILY));
+  }
+
+  private void createStoreFile(final HRegion region, String family) throws IOException {
     HRegionIncommon loader = new HRegionIncommon(region);
-    addContent(loader, Bytes.toString(COLUMN_FAMILY));
+    addContent(loader, family);
     loader.flushcache();
   }
 
@@ -589,8 +597,8 @@ public void testCompactionWithCorruptResult() throws Exception {
     long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
     Compactor tool = new Compactor(this.conf);
 
-    StoreFile.Writer compactedFile =
-      tool.compact(store, storeFiles, false, maxId);
+    StoreFile.Writer compactedFile = tool.compactForTesting(store, this.conf, storeFiles, false,
+      maxId);
 
     // Now lets corrupt the compacted file.
     FileSystem fs = FileSystem.get(conf);
@@ -628,7 +636,7 @@ public void testNonUserMajorCompactionRequest() throws Exception {
     }
     store.triggerMajorCompaction();
 
-    CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY);
+    CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY, null);
     assertNotNull("Expected to receive a compaction request", request);
     assertEquals(
       "System-requested major compaction should not occur if there are too many store files",
@@ -646,7 +654,7 @@ public void testUserMajorCompactionRequest() throws IOException{
       createStoreFile(r);
     }
     store.triggerMajorCompaction();
-    CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER);
+    CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER, null);
     assertNotNull("Expected to receive a compaction request", request);
     assertEquals(
       "User-requested major compaction should always occur, even if there are too many store files",
@@ -654,6 +662,90 @@ public void testUserMajorCompactionRequest() throws IOException{
       request.isMajor());
   }
 
+  /**
+   * Create a custom compaction request and be sure that we can track it through the queue, knowing
+   * when the compaction is completed.
+   */
+  public void testTrackingCompactionRequest() throws Exception {
+    // setup a compact/split thread on a mock server
+    HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+    Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf());
+    CompactSplitThread thread = new CompactSplitThread(mockServer);
+    Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+    // simple stop for the metrics - we ignore any updates in the test
+    RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class);
+    Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics);
+
+    // setup a region/store with some files
+    Store store = r.getStore(COLUMN_FAMILY);
+    createStoreFile(r);
+    for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
+      createStoreFile(r);
+    }
+
+    CountDownLatch latch = new CountDownLatch(1);
+    TrackableCompactionRequest request = new TrackableCompactionRequest(r, store, latch);
+    thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, request);
+    // wait for the latch to complete.
+    latch.await();
+
+    thread.interruptIfNecessary();
+  }
+
+  public void testMultipleCustomCompactionRequests() throws Exception {
+    // setup a compact/split thread on a mock server
+    HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+    Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf());
+    CompactSplitThread thread = new CompactSplitThread(mockServer);
+    Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+    // simple stop for the metrics - we ignore any updates in the test
+    RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class);
+    Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics);
+
+    // setup a region/store with some files
+    int numStores = r.getStores().size();
+    List<CompactionRequest> requests = new ArrayList<CompactionRequest>(numStores);
+    CountDownLatch latch = new CountDownLatch(numStores);
+    // create some store files and setup requests for each store on which we want to do a
+    // compaction
+    for (Store store : r.getStores().values()) {
+      createStoreFile(r, store.getColumnFamilyName());
+      createStoreFile(r, store.getColumnFamilyName());
+      createStoreFile(r, store.getColumnFamilyName());
+      requests.add(new TrackableCompactionRequest(r, store, latch));
+    }
+
+    thread.requestCompaction(r, "test mulitple custom comapctions", Store.PRIORITY_USER,
+      Collections.unmodifiableList(requests));
+
+    // wait for the latch to complete.
+    latch.await();
+
+    thread.interruptIfNecessary();
+  }
+
+  /**
+   * Simple {@link CompactionRequest} on which you can wait until the requested compaction finishes.
+   */
+  public static class TrackableCompactionRequest extends CompactionRequest {
+    private CountDownLatch done;
+
+    /**
+     * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+     * compaction before being used.
+     */
+    public TrackableCompactionRequest(HRegion region, Store store, CountDownLatch finished) {
+      super(region, store, Store.PRIORITY_USER);
+      this.done = finished;
+    }
+
+    @Override
+    public void run() {
+      super.run();
+      this.done.countDown();
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 46db70556e495ffab2dde4bfc23fc33d0569e20b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Feb 2013 02:41:59 +0000
Subject: [PATCH 0807/1540] HBASE-7878 revert due to TestHLogSplit test failure

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451077 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/FSHDFSUtils.java | 30 ++++---------------
 .../hbase/regionserver/wal/TestHLog.java      | 30 +++++--------------
 2 files changed, 14 insertions(+), 46 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index e59e46d32746..4186f0ad54c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -50,8 +50,6 @@ public class FSHDFSUtils extends FSUtils{
    * in o.a.h.hdfs.protocol.HdfsConstants cause of HDFS-1620.
    */
   public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
-  
-  public static final String TEST_TRIGGER_DFS_APPEND = "hbase.test.trigger.dfs.append";
 
   @Override
   public void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
@@ -70,23 +68,13 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
 
     // Trying recovery
     boolean recovered = false;
-    long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 300000);
-    // conf parameter passed from unit test, indicating whether fs.append() should be triggered
-    boolean triggerAppend = conf.getBoolean(TEST_TRIGGER_DFS_APPEND, false);
-    Exception ex = null;    
     while (!recovered) {
       try {
         try {
           if (fs instanceof DistributedFileSystem) {
             DistributedFileSystem dfs = (DistributedFileSystem)fs;
-            if (triggerAppend) throw new IOException("recoverFileLease");
-            try {
-              recovered = (Boolean) DistributedFileSystem.class.getMethod(
-                "recoverLease", new Class[] { Path.class }).invoke(dfs, p);
-            } catch (InvocationTargetException ite) {
-              // function was properly called, but threw it's own exception
-              throw (IOException) ite.getCause();
-            }
+            DistributedFileSystem.class.getMethod("recoverLease",
+              new Class[] {Path.class}).invoke(dfs, p);
           } else {
             throw new Exception("Not a DistributedFileSystem");
           }
@@ -95,17 +83,11 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
           throw (IOException) ite.getCause();
         } catch (Exception e) {
           LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
-              ", trying fs.append instead");
-          ex = e;
-        }
-        if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) {
-          ex = null; // assume the following append() call would succeed
-          LOG.debug("recoverLease times out after " + recoveryTimeout + " ms. Calling append");
+            ", trying fs.append instead");
           FSDataOutputStream out = fs.append(p);
           out.close();
-          recovered = true;
         }
-        if (recovered) break;
+        recovered = true;
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
         if (e instanceof AlreadyBeingCreatedException) {
@@ -129,9 +111,9 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
       }
       try {
         Thread.sleep(1000);
-      } catch (InterruptedException ie) {
+      } catch (InterruptedException ex) {
         InterruptedIOException iioe = new InterruptedIOException();
-        iioe.initCause(ie);
+        iioe.initCause(ex);
         throw iioe;
       }
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 06768f9a7ec3..8feb5d6db2d1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -42,7 +42,6 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@@ -53,6 +52,7 @@
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
+import org.apache.hadoop.io.SequenceFile;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -81,7 +81,7 @@ public class TestHLog  {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static Path hbaseDir;
   private static Path oldLogDir;
-  
+
   @Before
   public void setUp() throws Exception {
 
@@ -101,7 +101,6 @@ public static void setUpBeforeClass() throws Exception {
     // Make block sizes small.
     TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
     // needed for testAppendClose()
-    TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true);
     TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
     // quicker heartbeat interval for faster DN death notification
     TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000);
@@ -371,29 +370,17 @@ private void verifySplits(List<Path> splits, final int howmany)
     }
   }
   
-  /*
-   * We pass different values to recoverFileLease() so that different code paths are covered
-   * 
-   * For this test to pass, requires:
-   * 1. HDFS-200 (append support)
-   * 2. HDFS-988 (SafeMode should freeze file operations
-   *              [FSNamesystem.nextGenerationStampForBlock])
-   * 3. HDFS-142 (on restart, maintain pendingCreates)
-   */
+  // For this test to pass, requires:
+  // 1. HDFS-200 (append support)
+  // 2. HDFS-988 (SafeMode should freeze file operations
+  //              [FSNamesystem.nextGenerationStampForBlock])
+  // 3. HDFS-142 (on restart, maintain pendingCreates)
   @Test
   public void testAppendClose() throws Exception {
-    testAppendClose(true);
-    testAppendClose(false);
-  }
-
-  /*
-   * @param triggerDirectAppend whether to trigger direct call of fs.append()
-   */
-  public void testAppendClose(final boolean triggerDirectAppend) throws Exception {
     byte [] tableName = Bytes.toBytes(getName());
     HRegionInfo regioninfo = new HRegionInfo(tableName,
              HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
-    Path subdir = new Path(dir, "hlogdir-" + triggerDirectAppend);
+    Path subdir = new Path(dir, "hlogdir");
     Path archdir = new Path(dir, "hlogdir_archive");
     HLog wal = new HLog(fs, subdir, archdir, conf);
     final int total = 20;
@@ -467,7 +454,6 @@ class RecoverLogThread extends Thread {
       public Exception exception = null;
       public void run() {
           try {
-            rlConf.setBoolean(FSHDFSUtils.TEST_TRIGGER_DFS_APPEND, triggerDirectAppend);
             FSUtils.getInstance(fs, rlConf)
               .recoverFileLease(recoveredFs, walPath, rlConf);
           } catch (IOException e) {

From 631686446926366b18081556f46b058b893fde66 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 28 Feb 2013 03:54:27 +0000
Subject: [PATCH 0808/1540] HBASE-7929 Reapply hbase-7507 "Make memstore flush
 be able to retry after exception" to 0.94 branch.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451087 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   | 11 ----
 .../hadoop/hbase/regionserver/Store.java      | 55 +------------------
 2 files changed, 3 insertions(+), 63 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 14a66421c76c..e70bd3be708c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -477,17 +477,6 @@ public static enum Modify {
    */
   public static long DEFAULT_HBASE_CLIENT_PAUSE = 1000;
 
-  /**
-   * Parameter name for server pause value, used mostly as value to wait before
-   * running a retry of a failed operation.
-   */
-  public static String HBASE_SERVER_PAUSE = "hbase.server.pause";
-
-  /**
-   * Default value of {@link #HBASE_SERVER_PAUSE}.
-   */
-  public static int DEFAULT_HBASE_SERVER_PAUSE = 1000;
-
   /**
    * Parameter name for maximum retries, used as maximum for all retryable
    * operations such as fetching of the root region from root region server,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index c9432009847b..442dbdf3e66c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -166,10 +166,6 @@ public class Store extends SchemaConfigured implements HeapSize {
 
   private final Compactor compactor;
 
-  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
-  private static int flush_retries_number;
-  private static int pauseTime;
-
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -249,17 +245,6 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.bytesPerChecksum = getBytesPerChecksum(conf);
     // Create a compaction tool instance
     this.compactor = new Compactor(this.conf);
-    if (Store.flush_retries_number == 0) {
-      Store.flush_retries_number = conf.getInt(
-          "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
-      Store.pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE,
-          HConstants.DEFAULT_HBASE_SERVER_PAUSE);
-      if (Store.flush_retries_number <= 0) {
-        throw new IllegalArgumentException(
-            "hbase.hstore.flush.retries.number must be > 0, not "
-                + Store.flush_retries_number);
-      }
-    }
   }
 
   /**
@@ -737,43 +722,8 @@ protected Path flushCache(final long logCacheFlushId,
     // If an exception happens flushing, we let it out without clearing
     // the memstore snapshot.  The old snapshot will be returned when we say
     // 'snapshot', the next time flush comes around.
-    // Retry after catching exception when flushing, otherwise server will abort
-    // itself
-    IOException lastException = null;
-    for (int i = 0; i < Store.flush_retries_number; i++) {
-      try {
-        Path pathName = internalFlushCache(snapshot, logCacheFlushId,
-            snapshotTimeRangeTracker, flushedSize, status);
-        try {
-          // Path name is null if there is no entry to flush
-          if (pathName != null) {
-            validateStoreFile(pathName);
-          }
-          return pathName;
-        } catch (Exception e) {
-          LOG.warn("Failed validating store file " + pathName
-              + ", retring num=" + i, e);
-          if (e instanceof IOException) {
-            lastException = (IOException) e;
-          } else {
-            lastException = new IOException(e);
-          }
-        }
-      } catch (IOException e) {
-        LOG.warn("Failed flushing store file, retring num=" + i, e);
-        lastException = e;
-      }
-      if (lastException != null) {
-        try {
-          Thread.sleep(pauseTime);
-        } catch (InterruptedException e) {
-          IOException iie = new InterruptedIOException();
-          iie.initCause(e);
-          throw iie;
-        }
-      }
-    }
-    throw lastException;
+    return internalFlushCache(
+        snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);
   }
 
   /*
@@ -892,6 +842,7 @@ private StoreFile commitFile(final Path path,
     // Write-out finished successfully, move into the right spot
     String fileName = path.getName();
     Path dstPath = new Path(homedir, fileName);
+    validateStoreFile(path);
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);

From 157c6ea8d94c771771da60b3696180e8b351eba0 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Feb 2013 05:32:17 +0000
Subject: [PATCH 0809/1540] HBASE-7944 Replication leaks file reader resource &
 not reset currentNbOperations (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451100 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationHLogReaderManager.java         | 7 ++++++-
 .../hbase/replication/regionserver/ReplicationSource.java  | 1 +
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
index ce8585cb0d87..ae5c3bdc839e 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -122,6 +122,7 @@ public void setPosition(long pos) {
   public void closeReader() throws IOException {
     if (this.reader != null) {
       this.reader.close();
+      this.reader = null;
     }
   }
 
@@ -130,7 +131,11 @@ public void closeReader() throws IOException {
    */
   public void finishCurrentFile() {
     this.position = 0;
-    this.reader = null;
+    try {
+      this.closeReader();
+    } catch (IOException e) {
+      LOG.warn("Unable to close reader", e);
+    }
   }
 
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 304c5a2db687..138dfd260bf5 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -325,6 +325,7 @@ public void run() {
       }
 
       boolean gotIOE = false;
+      currentNbOperations = 0;
       currentNbEntries = 0;
       currentSize = 0;
       try {

From 3c44b28c94ae753daf511fa00a4cec3c203aa80d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Feb 2013 18:48:11 +0000
Subject: [PATCH 0810/1540] HBASE-7920 Addendum fixes checking for FilterList

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451287 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
index 5d087f045c50..291b5dd9cf4e 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
@@ -152,7 +152,8 @@ public boolean isFamilyEssential(byte[] name) {
    * @return whether column family is essential
    */
   public static boolean isFamilyEssential(Filter filter, byte[] name) {
-    return !(filter instanceof FilterBase) || ((FilterBase) filter).isFamilyEssential(name);
+    return (!(filter instanceof FilterBase) || ((FilterBase) filter).isFamilyEssential(name)) &&
+           (!(filter instanceof FilterList) || ((FilterList) filter).isFamilyEssential(name));
   }
 
   /**

From 1320d9dcbc64c4b7924c7acfb7e02d6fe927bf78 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 28 Feb 2013 23:05:11 +0000
Subject: [PATCH 0811/1540] HBASE-7700 TestColumnSeeking is mathematically
 bound to fail (J-D)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451418 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
index ab952efb037e..e865a8a66205 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
@@ -132,6 +132,7 @@ public void testDuplicateVersions() throws IOException {
         Scan scan = new Scan();
         scan.setMaxVersions();
         if (i < numberOfTests) {
+          if (columnLists[i].size() == 0) continue; // HBASE-7700
           kvSet = kvMaps[i].values();
           for (String column : columnLists[i]) {
             scan.addColumn(familyBytes, Bytes.toBytes(column));
@@ -244,6 +245,7 @@ public void testReseeking() throws IOException {
       Scan scan = new Scan();
       scan.setMaxVersions();
       if (i < numberOfTests) {
+        if (columnLists[i].size() == 0) continue; // HBASE-7700
         kvSet = kvMaps[i].values();
         for (String column : columnLists[i]) {
           scan.addColumn(familyBytes, Bytes.toBytes(column));

From ed3123299514ec747fcbbecc0e6589087648c347 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 28 Feb 2013 23:08:09 +0000
Subject: [PATCH 0812/1540] HBASE-7869 Provide way to not start LogSyncer
 thread (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451420 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  9 ++++-
 .../hadoop/hbase/regionserver/wal/HLog.java   | 40 ++++++++++++-------
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 1c4038425014..db72c084bbca 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -364,6 +364,7 @@ boolean isFlushRequested() {
   private HTableDescriptor htableDescriptor = null;
   private RegionSplitPolicy splitPolicy;
   private final OperationMetrics opMetrics;
+  private final boolean deferredLogSyncDisabled;
 
   /**
    * Should only be used for testing purposes
@@ -389,6 +390,7 @@ public HRegion(){
     this.maxBusyWaitDuration = 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
     this.busyWaitDuration = DEFAULT_BUSY_WAIT_DURATION;
     this.maxBusyWaitMultiplier = 2;
+    this.deferredLogSyncDisabled = false;
   }
 
   /**
@@ -457,7 +459,10 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
     this.timestampSlop = conf.getLong(
         "hbase.hregion.keyvalue.timestamp.slop.millisecs",
         HConstants.LATEST_TIMESTAMP);
-
+    // When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled.
+    this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval",
+        1 * 1000) <= 0;
+    
     if (rsServices != null) {
       this.rsAccounting = this.rsServices.getRegionServerAccounting();
       // don't initialize coprocessors if not running within a regionserver
@@ -5644,7 +5649,7 @@ private void lock(final Lock lock, final int multiplier)
    */
   private void syncOrDefer(long txid) throws IOException {
     if (this.regionInfo.isMetaRegion() ||
-      !this.htableDescriptor.isDeferredLogFlush()) {
+      !this.htableDescriptor.isDeferredLogFlush() || this.deferredLogSyncDisabled) {
       this.log.sync(txid);
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 8496295fa144..33cb56228a00 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -244,7 +244,7 @@ public interface Writer {
   /**
    * Thread that handles optional sync'ing
    */
-  private final LogSyncer logSyncerThread;
+  private final LogSyncer logSyncer;
 
   /** Number of log close errors tolerated before we abort */
   private final int closeErrorsTolerated;
@@ -438,9 +438,16 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     // handle the reflection necessary to call getNumCurrentReplicas()
     this.getNumCurrentReplicas = getGetNumCurrentReplicas(this.hdfs_out);
 
-    logSyncerThread = new LogSyncer(this.optionalFlushInterval);
-    Threads.setDaemonThreadRunning(logSyncerThread.getThread(),
-        Thread.currentThread().getName() + ".logSyncer");
+    logSyncer = new LogSyncer(this.optionalFlushInterval);
+    // When optionalFlushInterval is set as 0, don't start a thread for deferred log sync.
+    if (this.optionalFlushInterval > 0) {
+      Threads.setDaemonThreadRunning(logSyncer.getThread(), Thread.currentThread().getName()
+          + ".logSyncer");
+    } else {
+      LOG.info("hbase.regionserver.optionallogflushinterval is set as "
+          + this.optionalFlushInterval + ". Deferred log syncing won't work. "
+          + "Any Mutation, marked to be deferred synced, will be flushed immediately.");
+    }
     coprocessorHost = new WALCoprocessorHost(this, conf);
   }
   
@@ -985,12 +992,15 @@ public void closeAndDelete() throws IOException {
    * @throws IOException
    */
   public void close() throws IOException {
-    try {
-      logSyncerThread.close();
-      // Make sure we synced everything
-      logSyncerThread.join(this.optionalFlushInterval*2);
-    } catch (InterruptedException e) {
-      LOG.error("Exception while waiting for syncer thread to die", e);
+    // When optionalFlushInterval is 0, the logSyncer is not started as a Thread.
+    if (this.optionalFlushInterval > 0) {
+      try {
+        logSyncer.close();
+        // Make sure we synced everything
+        logSyncer.join(this.optionalFlushInterval * 2);
+      } catch (InterruptedException e) {
+        LOG.error("Exception while waiting for syncer thread to die", e);
+      }
     }
 
     cacheFlushLock.lock();
@@ -1310,9 +1320,9 @@ private void syncer(long txid) throws IOException {
           return;
         }
         doneUpto = this.unflushedEntries.get();
-        pending = logSyncerThread.getPendingWrites();
+        pending = logSyncer.getPendingWrites();
         try {
-          logSyncerThread.hlogFlush(tempWriter, pending);
+          logSyncer.hlogFlush(tempWriter, pending);
         } catch(IOException io) {
           ioe = io;
           LOG.error("syncer encountered error, will retry. txid=" + txid, ioe);
@@ -1323,7 +1333,7 @@ private void syncer(long txid) throws IOException {
           synchronized (flushLock) {
             // HBASE-4387, HBASE-5623, retry with updateLock held
             tempWriter = this.writer;
-            logSyncerThread.hlogFlush(tempWriter, pending);
+            logSyncer.hlogFlush(tempWriter, pending);
           }
         }
       }
@@ -1473,7 +1483,7 @@ protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
       // coprocessor hook:
       if (!coprocessorHost.preWALWrite(info, logKey, logEdit)) {
         // write to our buffer for the Hlog file.
-        logSyncerThread.append(new HLog.Entry(logKey, logEdit));
+        logSyncer.append(new HLog.Entry(logKey, logEdit));
       }
       long took = System.currentTimeMillis() - now;
       coprocessorHost.postWALWrite(info, logKey, logEdit);
@@ -1602,7 +1612,7 @@ public void completeCacheFlush(final byte [] encodedRegionName,
         WALEdit edit = completeCacheFlushLogEdit();
         HLogKey key = makeKey(encodedRegionName, tableName, logSeqId,
             System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
-        logSyncerThread.append(new Entry(key, edit));
+        logSyncer.append(new Entry(key, edit));
         txid = this.unflushedEntries.incrementAndGet();
         writeTime.inc(System.currentTimeMillis() - now);
         long len = 0;

From 84bc6a46924daae48c9cb1b2e447c71705c7bd48 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Fri, 1 Mar 2013 16:40:39 +0000
Subject: [PATCH 0813/1540] HBASE-7928  Scanning .META. with startRow and/or
 stopRow is not giving proper results (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451645 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  2 ++
 .../apache/hadoop/hbase/client/HTable.java    | 28 ++++++++++++++--
 .../apache/hadoop/hbase/client/TestAdmin.java | 33 +++++++++++++++++++
 3 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e70bd3be708c..f23c30154c69 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -305,6 +305,8 @@ public enum OperationStatusCode {
 
   /** delimiter used between portions of a region name */
   public static final int META_ROW_DELIMITER = ',';
+  
+  public static final byte[] META_ROW_DELIMITER_BYTES = Bytes.toBytes(",,");
 
   /** The catalog family as a string*/
   public static final String CATALOG_FAMILY_STR = "info";
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index bc5f6e3caa9b..0f641df1f462 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -625,11 +625,33 @@ public ResultScanner getScanner(final Scan scan) throws IOException {
     if (scan.getCaching() <= 0) {
       scan.setCaching(getScannerCaching());
     }
-    return new ClientScanner(getConfiguration(), scan, getTableName(),
-        this.connection);
+    if (Bytes.equals(this.getTableName(), HConstants.META_TABLE_NAME)) {
+      changeStartAndStopRowIfMeta(scan);
+    }
+
+    return new ClientScanner(getConfiguration(), scan, getTableName(), this.connection);
   }
 
-  /**
+  private void changeStartAndStopRowIfMeta(final Scan scan) {
+    if (scan.getStartRow() != null && scan.getStartRow().length != 0
+        && !isValidMetaTableRow(scan.getStartRow())) {
+      scan.setStartRow(Bytes.add(scan.getStartRow(), HConstants.META_ROW_DELIMITER_BYTES,
+          Bytes.toBytes(HConstants.ZEROES)));
+    }
+    if (scan.getStopRow() != null && scan.getStopRow().length != 0
+        && !isValidMetaTableRow(scan.getStopRow())) {
+      scan.setStopRow(Bytes.add(scan.getStopRow(), HConstants.META_ROW_DELIMITER_BYTES,
+          Bytes.toBytes(HConstants.NINES)));
+    }
+  }
+
+  private boolean isValidMetaTableRow(byte[] metaRow) {
+    return (com.google.common.primitives.Bytes
+        .indexOf(metaRow, HConstants.META_ROW_DELIMITER_BYTES) != -1);
+  }
+
+
+/**
    * {@inheritDoc}
    */
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 66b5dd786d36..4d5faedf858a 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -345,6 +345,39 @@ public void testHColumnValidName() {
            assertTrue(exceptionThrown);
        }
    }
+
+  @Test
+  public void testHBASE7928() throws Exception {
+    final byte[] tableName = Bytes.toBytes("a");
+    final byte[] tableName1 = Bytes.toBytes("b");
+    final byte[] tableName2 = Bytes.toBytes("c");
+    try {
+
+      TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
+
+      TEST_UTIL.createTable(tableName1, HConstants.CATALOG_FAMILY).close();
+
+      TEST_UTIL.createTable(tableName2, HConstants.CATALOG_FAMILY).close();
+      while (!admin.isTableAvailable(tableName2)) {
+        Thread.sleep(1);
+      }
+      Scan s = new Scan();
+      s.setStartRow(Bytes.toBytes("a1"));
+      s.setStopRow(Bytes.toBytes("b1"));
+      HTable table = new HTable(admin.getConfiguration(), HConstants.META_TABLE_NAME);
+      ResultScanner scanner = table.getScanner(s);
+      Result[] result = scanner.next(5);
+      assertEquals("Only one row should be selected", 1, result.length);
+    } finally {
+      admin.disableTable(tableName);
+      admin.deleteTable(tableName);
+      admin.disableTable(tableName1);
+      admin.deleteTable(tableName1);
+      admin.disableTable(tableName2);
+      admin.deleteTable(tableName2);
+    }
+  }
+  
   /**
    * Verify schema modification takes.
    * @throws IOException

From 56ebd45941d6f1b2db79cf46e75c89209c913c47 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Sat, 2 Mar 2013 07:56:48 +0000
Subject: [PATCH 0814/1540] HBASE-7928-Scanning .META. with startRow and/or
 stopRow is not giving proper results - Revert(Ram).  Affects
 TestMetaMigrationRemovingHTD

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1451846 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  2 --
 .../apache/hadoop/hbase/client/HTable.java    | 28 ++--------------
 .../apache/hadoop/hbase/client/TestAdmin.java | 33 -------------------
 3 files changed, 3 insertions(+), 60 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index f23c30154c69..e70bd3be708c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -305,8 +305,6 @@ public enum OperationStatusCode {
 
   /** delimiter used between portions of a region name */
   public static final int META_ROW_DELIMITER = ',';
-  
-  public static final byte[] META_ROW_DELIMITER_BYTES = Bytes.toBytes(",,");
 
   /** The catalog family as a string*/
   public static final String CATALOG_FAMILY_STR = "info";
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 0f641df1f462..bc5f6e3caa9b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -625,33 +625,11 @@ public ResultScanner getScanner(final Scan scan) throws IOException {
     if (scan.getCaching() <= 0) {
       scan.setCaching(getScannerCaching());
     }
-    if (Bytes.equals(this.getTableName(), HConstants.META_TABLE_NAME)) {
-      changeStartAndStopRowIfMeta(scan);
-    }
-
-    return new ClientScanner(getConfiguration(), scan, getTableName(), this.connection);
-  }
-
-  private void changeStartAndStopRowIfMeta(final Scan scan) {
-    if (scan.getStartRow() != null && scan.getStartRow().length != 0
-        && !isValidMetaTableRow(scan.getStartRow())) {
-      scan.setStartRow(Bytes.add(scan.getStartRow(), HConstants.META_ROW_DELIMITER_BYTES,
-          Bytes.toBytes(HConstants.ZEROES)));
-    }
-    if (scan.getStopRow() != null && scan.getStopRow().length != 0
-        && !isValidMetaTableRow(scan.getStopRow())) {
-      scan.setStopRow(Bytes.add(scan.getStopRow(), HConstants.META_ROW_DELIMITER_BYTES,
-          Bytes.toBytes(HConstants.NINES)));
-    }
+    return new ClientScanner(getConfiguration(), scan, getTableName(),
+        this.connection);
   }
 
-  private boolean isValidMetaTableRow(byte[] metaRow) {
-    return (com.google.common.primitives.Bytes
-        .indexOf(metaRow, HConstants.META_ROW_DELIMITER_BYTES) != -1);
-  }
-
-
-/**
+  /**
    * {@inheritDoc}
    */
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 4d5faedf858a..66b5dd786d36 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -345,39 +345,6 @@ public void testHColumnValidName() {
            assertTrue(exceptionThrown);
        }
    }
-
-  @Test
-  public void testHBASE7928() throws Exception {
-    final byte[] tableName = Bytes.toBytes("a");
-    final byte[] tableName1 = Bytes.toBytes("b");
-    final byte[] tableName2 = Bytes.toBytes("c");
-    try {
-
-      TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
-
-      TEST_UTIL.createTable(tableName1, HConstants.CATALOG_FAMILY).close();
-
-      TEST_UTIL.createTable(tableName2, HConstants.CATALOG_FAMILY).close();
-      while (!admin.isTableAvailable(tableName2)) {
-        Thread.sleep(1);
-      }
-      Scan s = new Scan();
-      s.setStartRow(Bytes.toBytes("a1"));
-      s.setStopRow(Bytes.toBytes("b1"));
-      HTable table = new HTable(admin.getConfiguration(), HConstants.META_TABLE_NAME);
-      ResultScanner scanner = table.getScanner(s);
-      Result[] result = scanner.next(5);
-      assertEquals("Only one row should be selected", 1, result.length);
-    } finally {
-      admin.disableTable(tableName);
-      admin.deleteTable(tableName);
-      admin.disableTable(tableName1);
-      admin.deleteTable(tableName1);
-      admin.disableTable(tableName2);
-      admin.deleteTable(tableName2);
-    }
-  }
-  
   /**
    * Verify schema modification takes.
    * @throws IOException

From 9ac6bc5fcc2df930278a348959c7c60c873be02b Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 4 Mar 2013 11:24:50 +0000
Subject: [PATCH 0815/1540] HBASE-7360 Backport Snapshots to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452257 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     |   50 +
 .../security/access/TestAccessController.java |   52 +
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  |   33 +-
 .../java/org/apache/hadoop/hbase/Chore.java   |    8 +
 .../hadoop/hbase/DaemonThreadFactory.java     |   49 +
 .../hadoop/hbase/HColumnDescriptor.java       |    8 +-
 .../org/apache/hadoop/hbase/HConstants.java   |   12 +-
 .../org/apache/hadoop/hbase/HRegionInfo.java  |    3 +
 .../apache/hadoop/hbase/HTableDescriptor.java |    3 +
 .../hadoop/hbase/backup/HFileArchiver.java    |   70 +-
 .../hadoop/hbase/catalog/MetaEditor.java      |   92 +-
 .../hadoop/hbase/client/HBaseAdmin.java       |  413 +++-
 .../hbase/coprocessor/BaseMasterObserver.java |   47 +
 .../hbase/coprocessor/MasterObserver.java     |   97 +
 .../hbase/errorhandling/ForeignException.java |  194 ++
 .../ForeignExceptionDispatcher.java           |  119 +
 .../ForeignExceptionListener.java             |   40 +
 .../errorhandling/ForeignExceptionSnare.java  |   67 +
 .../hbase/errorhandling/TimeoutException.java |   67 +
 .../TimeoutExceptionInjector.java             |  130 +
 .../hadoop/hbase/executor/EventHandler.java   |    2 +
 .../hbase/executor/ExecutorService.java       |    2 +
 .../org/apache/hadoop/hbase/io/FileLink.java  |  455 ++++
 .../org/apache/hadoop/hbase/io/HFileLink.java |  371 +++
 .../org/apache/hadoop/hbase/io/HLogLink.java  |   69 +
 .../hadoop/hbase/io/HalfStoreFileReader.java  |   32 +-
 .../hadoop/hbase/io/HbaseObjectWritable.java  |    5 +
 .../apache/hadoop/hbase/io/hfile/HFile.java   |   59 +-
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   16 +-
 .../hadoop/hbase/ipc/HMasterInterface.java    |   19 +
 .../apache/hadoop/hbase/master/HMaster.java   |  134 +
 .../hbase/master/MasterCoprocessorHost.java   |  143 ++
 .../hadoop/hbase/master/MasterFileSystem.java |   70 +
 .../hadoop/hbase/master/SnapshotSentinel.java |   57 +
 .../hbase/master/cleaner/CleanerChore.java    |    5 +-
 .../hbase/master/cleaner/HFileCleaner.java    |    4 +
 .../master/cleaner/HFileLinkCleaner.java      |   91 +
 .../master/handler/CreateTableHandler.java    |  153 +-
 .../master/handler/DeleteTableHandler.java    |   47 +-
 .../master/handler/DisableTableHandler.java   |    1 +
 .../master/handler/TableEventHandler.java     |    4 +-
 .../master/snapshot/CloneSnapshotHandler.java |  150 ++
 .../DisabledTableSnapshotHandler.java         |  131 +
 .../snapshot/EnabledTableSnapshotHandler.java |   97 +
 .../snapshot/MasterSnapshotVerifier.java      |  249 ++
 .../snapshot/RestoreSnapshotHandler.java      |  155 ++
 .../master/snapshot/SnapshotFileCache.java    |  308 +++
 .../master/snapshot/SnapshotHFileCleaner.java |  104 +
 .../master/snapshot/SnapshotLogCleaner.java   |  102 +
 .../master/snapshot/SnapshotManager.java      |  916 +++++++
 .../master/snapshot/TakeSnapshotHandler.java  |  237 ++
 .../hadoop/hbase/procedure/Procedure.java     |  377 +++
 .../hbase/procedure/ProcedureCoordinator.java |  268 ++
 .../procedure/ProcedureCoordinatorRpcs.java   |   85 +
 .../hbase/procedure/ProcedureMember.java      |  232 ++
 .../hbase/procedure/ProcedureMemberRpcs.java  |   73 +
 .../hadoop/hbase/procedure/Subprocedure.java  |  331 +++
 .../hbase/procedure/SubprocedureFactory.java  |   40 +
 .../procedure/ZKProcedureCoordinatorRpcs.java |  267 ++
 .../procedure/ZKProcedureMemberRpcs.java      |  350 +++
 .../hbase/procedure/ZKProcedureUtil.java      |  286 +++
 .../hadoop/hbase/protobuf/ProtobufUtil.java   |   67 +
 .../generated/ErrorHandlingProtos.java        | 2185 +++++++++++++++++
 .../hbase/protobuf/generated/HBaseProtos.java |  851 +++++++
 .../hadoop/hbase/regionserver/HRegion.java    |  121 +-
 .../hbase/regionserver/HRegionServer.java     |   27 +
 .../hadoop/hbase/regionserver/Store.java      |   29 +-
 .../hadoop/hbase/regionserver/StoreFile.java  |  169 +-
 .../snapshot/FlushSnapshotSubprocedure.java   |  161 ++
 .../snapshot/RegionServerSnapshotManager.java |  377 +++
 .../hadoop/hbase/regionserver/wal/HLog.java   |    4 +-
 .../snapshot/CopyRecoveredEditsTask.java      |   90 +
 .../snapshot/CorruptedSnapshotException.java  |   56 +
 .../hadoop/hbase/snapshot/ExportSnapshot.java |  701 ++++++
 .../snapshot/ExportSnapshotException.java     |   43 +
 .../snapshot/HBaseSnapshotException.java      |   77 +
 .../hbase/snapshot/HSnapshotDescription.java  |  124 +
 .../snapshot/ReferenceRegionHFilesTask.java   |  127 +
 .../snapshot/ReferenceServerWALsTask.java     |  108 +
 .../snapshot/RestoreSnapshotException.java    |   43 +
 .../hbase/snapshot/RestoreSnapshotHelper.java |  588 +++++
 .../snapshot/SnapshotCreationException.java   |   54 +
 .../snapshot/SnapshotDescriptionUtils.java    |  360 +++
 .../SnapshotDoesNotExistException.java        |   45 +
 .../snapshot/SnapshotExistsException.java     |   40 +
 .../hadoop/hbase/snapshot/SnapshotInfo.java   |  303 +++
 .../hbase/snapshot/SnapshotLogSplitter.java   |  196 ++
 .../hbase/snapshot/SnapshotReferenceUtil.java |  251 ++
 .../hadoop/hbase/snapshot/SnapshotTask.java   |   67 +
 .../hbase/snapshot/TableInfoCopyTask.java     |   73 +
 .../snapshot/TablePartiallyOpenException.java |   51 +
 .../hbase/snapshot/TakeSnapshotUtils.java     |  323 +++
 .../snapshot/UnknownSnapshotException.java    |   42 +
 .../hadoop/hbase/util/FSTableDescriptors.java |   24 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |   94 +-
 .../apache/hadoop/hbase/util/FSVisitor.java   |  194 ++
 .../hadoop/hbase/util/HFileArchiveUtil.java   |   75 +-
 .../hadoop/hbase/util/ModifyRegionUtils.java  |  176 ++
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java |   79 +-
 src/main/protobuf/ErrorHandling.proto         |   58 +
 src/main/protobuf/hbase.proto                 |   39 +
 src/main/ruby/hbase/admin.rb                  |   30 +
 src/main/ruby/shell.rb                        |   12 +
 .../ruby/shell/commands/clone_snapshot.rb     |   40 +
 .../ruby/shell/commands/delete_snapshot.rb    |   37 +
 .../ruby/shell/commands/list_snapshots.rb     |   52 +
 .../ruby/shell/commands/restore_snapshot.rb   |   41 +
 src/main/ruby/shell/commands/snapshot.rb      |   37 +
 .../hadoop/hbase/HBaseTestingUtility.java     |   14 +
 .../hadoop/hbase/TestHTableDescriptor.java    |   45 +
 .../hbase/backup/TestHFileArchiving.java      |    2 +-
 .../client/TestRestoreSnapshotFromClient.java |  391 +++
 .../hbase/client/TestSnapshotFromAdmin.java   |  151 ++
 .../hbase/client/TestSnapshotFromClient.java  |  231 ++
 .../hbase/client/TestSnapshotsFromAdmin.java  |  136 +
 .../hbase/coprocessor/TestMasterObserver.java |  149 ++
 .../TestForeignExceptionDispatcher.java       |  123 +
 .../TestForeignExceptionSerialization.java    |   82 +
 .../TestTimeoutExceptionInjector.java         |  103 +
 .../apache/hadoop/hbase/io/TestFileLink.java  |  244 ++
 .../hbase/io/hfile/TestLruBlockCache.java     |   17 +-
 .../master/cleaner/TestHFileLinkCleaner.java  |  174 ++
 .../cleaner/TestSnapshotFromMaster.java       |  382 +++
 .../snapshot/TestSnapshotFileCache.java       |  230 ++
 .../snapshot/TestSnapshotHFileCleaner.java    |   89 +
 .../snapshot/TestSnapshotLogCleaner.java      |   85 +
 .../master/snapshot/TestSnapshotManager.java  |  155 ++
 .../hadoop/hbase/procedure/TestProcedure.java |  234 ++
 .../procedure/TestProcedureCoordinator.java   |  349 +++
 .../hbase/procedure/TestProcedureMember.java  |  444 ++++
 .../hbase/procedure/TestZKProcedure.java      |  405 +++
 .../procedure/TestZKProcedureControllers.java |  429 ++++
 .../hbase/regionserver/TestStoreFile.java     |  181 +-
 .../hbase/snapshot/SnapshotTestingUtils.java  |  255 ++
 .../snapshot/TestCopyRecoveredEditsTask.java  |  126 +
 .../hbase/snapshot/TestExportSnapshot.java    |  257 ++
 .../snapshot/TestFlushSnapshotFromClient.java |  369 +++
 .../TestReferenceRegionHFilesTask.java        |   92 +
 .../TestRestoreFlushSnapshotFromClient.java   |  254 ++
 .../snapshot/TestRestoreSnapshotHelper.java   |  200 ++
 .../TestSnapshotDescriptionUtils.java         |  104 +
 .../snapshot/TestSnapshotLogSplitter.java     |  176 ++
 .../hbase/snapshot/TestSnapshotTask.java      |   58 +
 .../hbase/snapshot/TestWALReferenceTask.java  |  103 +
 .../hadoop/hbase/util/TestFSVisitor.java      |  225 ++
 .../hbase/util/TestHFileArchiveUtil.java      |    3 +-
 146 files changed, 23605 insertions(+), 254 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/io/FileLink.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/io/HLogLink.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
 create mode 100644 src/main/protobuf/ErrorHandling.proto
 create mode 100644 src/main/protobuf/hbase.proto
 create mode 100644 src/main/ruby/shell/commands/clone_snapshot.rb
 create mode 100644 src/main/ruby/shell/commands/delete_snapshot.rb
 create mode 100644 src/main/ruby/shell/commands/list_snapshots.rb
 create mode 100644 src/main/ruby/shell/commands/restore_snapshot.rb
 create mode 100644 src/main/ruby/shell/commands/snapshot.rb
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 136f025891bc..28dea867eee2 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.ProtocolSignature;
 import org.apache.hadoop.hbase.ipc.RequestContext;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -730,6 +731,55 @@ public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
     AccessControlLists.init(ctx.getEnvironment().getMasterServices());
   }
 
+  @Override
+  public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+    requirePermission("snapshot", Permission.Action.ADMIN);
+  }
+
+  @Override
+  public void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+    requirePermission("cloneSnapshot", Permission.Action.ADMIN);
+  }
+
+  @Override
+  public void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+    requirePermission("restoreSnapshot", Permission.Action.ADMIN);
+  }
+
+  @Override
+  public void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot) throws IOException {
+    requirePermission("deleteSnapshot", Permission.Action.ADMIN);
+  }
+
+  @Override
+  public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot) throws IOException {
+  }
 
   /* ---- RegionObserver implementation ---- */
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index a2420d1a9f29..f57357826cec 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -113,6 +113,11 @@ public class TestAccessController {
   public static void setupBeforeClass() throws Exception {
     // setup configuration
     conf = TEST_UTIL.getConfiguration();
+    conf.set("hbase.master.hfilecleaner.plugins",
+      "org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner," +
+      "org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner");
+    conf.set("hbase.master.logcleaner.plugins",
+      "org.apache.hadoop.hbase.master.snapshot.SnapshotLogCleaner");
     SecureTestUtil.enableSecurity(conf);
 
     TEST_UTIL.startMiniCluster();
@@ -1721,4 +1726,51 @@ public Object run() throws Exception {
     verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
   }
 
+
+  @Test
+  public void testSnapshot() throws Exception {
+    PrivilegedExceptionAction snapshotAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preSnapshot(ObserverContext.createAndPrepare(CP_ENV, null),
+          null, null);
+        return null;
+      }
+    };
+
+    PrivilegedExceptionAction deleteAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preDeleteSnapshot(ObserverContext.createAndPrepare(CP_ENV, null),
+          null);
+        return null;
+      }
+    };
+
+    PrivilegedExceptionAction restoreAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preRestoreSnapshot(ObserverContext.createAndPrepare(CP_ENV, null),
+          null, null);
+        return null;
+      }
+    };
+
+    PrivilegedExceptionAction cloneAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        ACCESS_CONTROLLER.preCloneSnapshot(ObserverContext.createAndPrepare(CP_ENV, null),
+          null, null);
+        return null;
+      }
+    };
+
+    verifyAllowed(snapshotAction, SUPERUSER, USER_ADMIN);
+    verifyDenied(snapshotAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
+
+    verifyAllowed(cloneAction, SUPERUSER, USER_ADMIN);
+    verifyDenied(deleteAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
+
+    verifyAllowed(restoreAction, SUPERUSER, USER_ADMIN);
+    verifyDenied(restoreAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
+
+    verifyAllowed(deleteAction, SUPERUSER, USER_ADMIN);
+    verifyDenied(cloneAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
+  }
 }
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 2d3427d81a4d..f7f7a12086db 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -43,6 +43,7 @@ org.apache.hadoop.hbase.client.HBaseAdmin;
 org.apache.hadoop.hbase.client.HConnectionManager;
 org.apache.hadoop.hbase.HTableDescriptor;
 org.apache.hadoop.hbase.HBaseConfiguration;
+org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 </%import>
 <%if format.equals("json") %>
   <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &>
@@ -123,6 +124,7 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 <%if (metaLocation != null) %>
 <& userTables &>
 </%if>
+<& userSnapshots &>
 <%if (servers != null) %>
 <& regionServers &>
 </%if>
@@ -169,7 +171,6 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 <%def userTables>
 <%java>
    HTableDescriptor[] tables = admin.listTables();
-   HConnectionManager.deleteConnection(admin.getConfiguration());
 </%java>
 <%if (tables != null && tables.length > 0)%>
 <table>
@@ -197,6 +198,32 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 </%if>
 </%def>
 
+<%def userSnapshots>
+<%java>
+   List<SnapshotDescription> snapshots = admin.listSnapshots();
+</%java>
+<%if (snapshots != null && snapshots.size() > 0)%>
+<table>
+<tr>
+    <th>Snapshot</th>
+    <th>Table</th>
+    <th>Creation Time</th>
+    <th>Type</th>
+</tr>
+<%for SnapshotDescription snapshotDesc : snapshots%>
+<tr>
+    <td><% snapshotDesc.getName() %></td>
+    <td><a href="table.jsp?name=<% snapshotDesc.getTable() %>"><% snapshotDesc.getTable() %></a></td>
+    <td><% new Date(snapshotDesc.getCreationTime()) %></td>
+    <td><% snapshotDesc.getType() %></td>
+</tr>
+</%for>
+
+<p> <% snapshots.size() %> snapshot(s) in set.</p>
+</table>
+</%if>
+</%def>
+
 <%def regionServers>
 <h2>Region Servers</h2>
 <%if (servers != null && servers.size() > 0)%>
@@ -254,3 +281,7 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 </table>
 </%if>
 </%def>
+
+<%java>
+   HConnectionManager.deleteConnection(admin.getConfiguration(), false);
+</%java>
diff --git a/src/main/java/org/apache/hadoop/hbase/Chore.java b/src/main/java/org/apache/hadoop/hbase/Chore.java
index 547eeec71087..1581b0c04d2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/Chore.java
+++ b/src/main/java/org/apache/hadoop/hbase/Chore.java
@@ -91,6 +91,14 @@ public void triggerNow() {
     this.sleeper.skipSleepCycle();
   }
 
+  /*
+   * Exposed for TESTING!
+   * calls directly the chore method, from the current thread.
+   */
+  public void choreForTesting() {
+    chore();
+  }
+
   /**
    * Override to run a task before we start looping.
    * @return true if initial chore was successful
diff --git a/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java b/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java
new file mode 100644
index 000000000000..d621cbf2a930
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Thread factory that creates daemon threads
+ */
+public class DaemonThreadFactory implements ThreadFactory {
+  static final AtomicInteger poolNumber = new AtomicInteger(1);
+  final ThreadGroup group;
+  final AtomicInteger threadNumber = new AtomicInteger(1);
+  final String namePrefix;
+
+  public DaemonThreadFactory(String name) {
+    SecurityManager s = System.getSecurityManager();
+    group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
+    namePrefix = name + poolNumber.getAndIncrement() + "-thread-";
+  }
+
+  @Override
+  public Thread newThread(Runnable r) {
+    Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
+    if (!t.isDaemon()) {
+      t.setDaemon(true);
+    }
+    if (t.getPriority() != Thread.NORM_PRIORITY) {
+      t.setPriority(Thread.NORM_PRIORITY);
+    }
+    return t;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 766381513d87..b1cb9c317115 100644
--- a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableComparable;
@@ -403,7 +404,7 @@ public HColumnDescriptor(final byte[] familyName, final int minVersions,
    * @throws IllegalArgumentException If not null and not a legitimate family
    * name: i.e. 'printable' and ends in a ':' (Null passes are allowed because
    * <code>b</code> can be null when deserializing).  Cannot start with a '.'
-   * either.
+   * either. Also Family can not be an empty value or equal "recovered.edits".
    */
   public static byte [] isLegalFamilyName(final byte [] b) {
     if (b == null) {
@@ -420,6 +421,11 @@ public HColumnDescriptor(final byte[] familyName, final int minVersions,
           Bytes.toString(b));
       }
     }
+    byte[] recoveredEdit = Bytes.toBytes(HLog.RECOVERED_EDITS_DIR);
+    if (Bytes.equals(recoveredEdit, b)) {
+      throw new IllegalArgumentException("Family name cannot be: " +
+          HLog.RECOVERED_EDITS_DIR);
+    }
     return b;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e70bd3be708c..485006b54a97 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -671,11 +671,21 @@ public static enum Modify {
   /** Directory under /hbase where archived hfiles are stored */
   public static final String HFILE_ARCHIVE_DIRECTORY = ".archive";
 
+  /**
+   * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for
+   * remaining snapshot constants; this is here to keep HConstants dependencies at a minimum and
+   * uni-directional.
+   */
+  public static final String SNAPSHOT_DIR_NAME = ".snapshot";
+
+  /** Temporary directory used for table creation and deletion */
+  public static final String HBASE_TEMP_DIRECTORY = ".tmp";
+
   /** Directories that are not HBase table directories */
   public static final List<String> HBASE_NON_TABLE_DIRS =
     Collections.unmodifiableList(Arrays.asList(new String[] { HREGION_LOGDIR_NAME,
       HREGION_OLDLOGDIR_NAME, CORRUPT_DIR_NAME, SPLIT_LOGDIR_NAME,
-      HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY }));
+      HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY, SNAPSHOT_DIR_NAME, HBASE_TEMP_DIRECTORY }));
 
   /** Directories that are not HBase user table directories */
   public static final List<String> HBASE_NON_USER_TABLE_DIRS =
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 0ee74e77e58d..942050522588 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -80,6 +80,9 @@ public class HRegionInfo extends VersionedWritable
   private static final int ENC_SEPARATOR = '.';
   public  static final int MD5_HEX_LENGTH   = 32;
 
+  /** A non-capture group so that this can be embedded. */
+  public static final String ENCODED_REGION_NAME_REGEX = "(?:[a-f0-9]+)";
+
   /**
    * Does region name contain its encoded name?
    * @param regionName region name
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 934587d5b257..a20c383fe6d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -358,6 +358,9 @@ public static boolean isMetaTable(final byte [] tableName) {
       Bytes.equals(tableName, HConstants.META_TABLE_NAME);
   }
 
+  // A non-capture group so that this can be embedded.
+  public static final String VALID_USER_TABLE_REGEX = "(?:[a-zA-Z_0-9][a-zA-Z_0-9.-]*)";
+
   /**
    * Check passed byte buffer, "tableName", is legal user-space table name.
    * @return Returns passed <code>tableName</code> param
diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index 42415a9fdce6..6083328fddaa 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -56,7 +56,7 @@ public class HFileArchiver {
   private static final String SEPARATOR = ".";
 
   /** Number of retries in case of fs operation failure */
-  private static final int DEFAULT_RETRIES_NUMBER = 3;
+  private static final int DEFAULT_RETRIES_NUMBER = 6;
 
   private HFileArchiver() {
     // hidden ctor since this is just a util
@@ -73,14 +73,12 @@ private HFileArchiver() {
   public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo info)
       throws IOException {
     Path rootDir = FSUtils.getRootDir(conf);
-    archiveRegion(conf, fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()),
+    archiveRegion(fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()),
       HRegion.getRegionDir(rootDir, info));
   }
 
-
   /**
    * Remove an entire region from the table directory via archiving the region's hfiles.
-   * @param conf the configuration to use
    * @param fs {@link FileSystem} from which to remove the region
    * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
    *          the archive path)
@@ -90,8 +88,7 @@ public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo
    *         operations could not complete.
    * @throws IOException if the request cannot be completed
    */
-  public static boolean archiveRegion(Configuration conf, FileSystem fs, Path rootdir,
-      Path tableDir, Path regionDir)
+  public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
       throws IOException {
     if (LOG.isDebugEnabled()) {
       LOG.debug("ARCHIVING region " + regionDir.toString());
@@ -110,7 +107,7 @@ public static boolean archiveRegion(Configuration conf, FileSystem fs, Path root
 
     // make sure the regiondir lives under the tabledir
     Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
-    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(conf, tableDir, regionDir);
+    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, tableDir, regionDir);
 
     LOG.debug("Have an archive directory, preparing to move files");
     FileStatusConverter getAsFile = new FileStatusConverter(fs);
@@ -150,7 +147,7 @@ public boolean accept(Path file) {
     }
 
     throw new IOException("Received error when attempting to archive files (" + toArchive
-        + "), cannot delete region directory. ");
+        + "), cannot delete region directory.");
   }
 
   /**
@@ -239,6 +236,35 @@ public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegion
     }
   }
 
+  /**
+   * Archive the store file
+   * @param fs the filesystem where the store files live
+   * @param regionInfo region hosting the store files
+   * @param conf {@link Configuration} to examine to determine the archive directory
+   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+   * @param family the family hosting the store files
+   * @param storeFile file to be archived
+   * @throws IOException if the files could not be correctly disposed.
+   */
+  public static void archiveStoreFile(FileSystem fs, HRegionInfo regionInfo,
+      Configuration conf, Path tableDir, byte[] family, Path storeFile) throws IOException {
+    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
+    // make sure we don't archive if we can't and that the archive dir exists
+    if (!fs.mkdirs(storeArchiveDir)) {
+      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
+          + Bytes.toString(family) + ", deleting compacted files instead.");
+    }
+
+    // do the actual archive
+    long start = EnvironmentEdgeManager.currentTimeMillis();
+    File file = new FileablePath(fs, storeFile);
+    if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) {
+      throw new IOException("Failed to archive/delete the file for region:"
+          + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family)
+          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
+    }
+  }
+
   /**
    * Archive the given files and resolve any conflicts with existing files via appending the time
    * archiving started (so all conflicts in the same group have the same timestamp appended).
@@ -413,6 +439,34 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
     return true;
   }
 
+  /**
+   * Simple delete of regular files from the {@link FileSystem}.
+   * <p>
+   * This method is a more generic implementation that the other deleteXXX
+   * methods in this class, allowing more code reuse at the cost of a couple
+   * more, short-lived objects (which should have minimum impact on the jvm).
+   * @param fs {@link FileSystem} where the files live
+   * @param files {@link Collection} of files to be deleted
+   * @throws IOException if a file cannot be deleted. All files will be
+   *           attempted to deleted before throwing the exception, rather than
+   *           failing at the first file.
+   */
+  private static void deleteFilesWithoutArchiving(Collection<File> files) throws IOException {
+    List<IOException> errors = new ArrayList<IOException>(0);
+    for (File file : files) {
+      try {
+        LOG.debug("Deleting region file:" + file);
+        file.delete();
+      } catch (IOException e) {
+        LOG.error("Failed to delete file:" + file);
+        errors.add(e);
+      }
+    }
+    if (errors.size() > 0) {
+      throw MultipleIOException.createIOException(errors);
+    }
+  }
+
   /**
    * Without regard for backup, delete a region. Should be used with caution.
    * @param regionDir {@link Path} to the region to be deleted.
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
index 9537a60032ba..c7ce68433f31 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -18,8 +18,10 @@
 package org.apache.hadoop.hbase.catalog;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.net.ConnectException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -30,6 +32,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -124,11 +127,44 @@ static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
    * @param d Delete to add to .META.
    * @throws IOException
    */
-  static void deleteMetaTable(final CatalogTracker ct, final Delete d)
-  throws IOException {
+  static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
+      throws IOException {
+    List<Delete> dels = new ArrayList<Delete>(1);
+    dels.add(d);
+    deleteFromMetaTable(ct, dels);
+  }
+
+  /**
+   * Delete the passed <code>deletes</code> from the <code>.META.</code> table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param deletes Deletes to add to .META.  This list should support #remove.
+   * @throws IOException
+   */
+  public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
+      throws IOException {
+    HTable t = MetaReader.getMetaHTable(ct);
+    try {
+      t.delete(deletes);
+    } finally {
+      t.close();
+    }
+  }
+
+  /**
+   * Execute the passed <code>mutations</code> against <code>.META.</code> table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param mutations Puts and Deletes to execute on .META.
+   * @throws IOException
+   */
+  static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
+      throws IOException {
     HTable t = MetaReader.getMetaHTable(ct);
     try {
-      t.delete(d);
+      t.batch(mutations);
+    } catch (InterruptedException e) {
+      InterruptedIOException ie = new InterruptedIOException(e.getMessage());
+      ie.initCause(e);
+      throw ie;
     } finally {
       t.close();
     }
@@ -272,10 +308,56 @@ public static void deleteRegion(CatalogTracker catalogTracker,
       HRegionInfo regionInfo)
   throws IOException {
     Delete delete = new Delete(regionInfo.getRegionName());
-    deleteMetaTable(catalogTracker, delete);
+    deleteFromMetaTable(catalogTracker, delete);
     LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META");
   }
 
+  /**
+   * Deletes the specified regions from META.
+   * @param catalogTracker
+   * @param regionsInfo list of regions to be deleted from META
+   * @throws IOException
+   */
+  public static void deleteRegions(CatalogTracker catalogTracker,
+      List<HRegionInfo> regionsInfo) throws IOException {
+    List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
+    for (HRegionInfo hri: regionsInfo) {
+      deletes.add(new Delete(hri.getRegionName()));
+    }
+    deleteFromMetaTable(catalogTracker, deletes);
+    LOG.info("Deleted from META, regions: " + regionsInfo);
+  }
+
+  /**
+   * Adds and Removes the specified regions from .META.
+   * @param catalogTracker
+   * @param regionsToRemove list of regions to be deleted from META
+   * @param regionsToAdd list of regions to be added to META
+   * @throws IOException
+   */
+  public static void mutateRegions(CatalogTracker catalogTracker,
+      final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
+      throws IOException {
+    List<Mutation> mutation = new ArrayList<Mutation>();
+    if (regionsToRemove != null) {
+      for (HRegionInfo hri: regionsToRemove) {
+        mutation.add(new Delete(hri.getRegionName()));
+      }
+    }
+    if (regionsToAdd != null) {
+      for (HRegionInfo hri: regionsToAdd) {
+        mutation.add(makePutFromRegionInfo(hri));
+      }
+    }
+    mutateMetaTable(catalogTracker, mutation);
+    if (regionsToRemove != null && regionsToRemove.size() > 0) {
+      LOG.debug("Deleted from META, regions: " + regionsToRemove);
+    }
+    if (regionsToAdd != null && regionsToAdd.size() > 0) {
+      LOG.debug("Add to META, regions: " + regionsToAdd);
+    }
+  }
+
   /**
    * Deletes daughters references in offlined split parent.
    * @param catalogTracker
@@ -289,7 +371,7 @@ public static void deleteDaughtersReferencesInParent(CatalogTracker catalogTrack
     Delete delete = new Delete(parent.getRegionName());
     delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
     delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
-    deleteMetaTable(catalogTracker, delete);
+    deleteFromMetaTable(catalogTracker, delete);
     LOG.info("Deleted daughters references, qualifier=" + Bytes.toStringBinary(HConstants.SPLITA_QUALIFIER) +
       " and qualifier=" + Bytes.toStringBinary(HConstants.SPLITB_QUALIFIER) +
       ", from parent " + parent.getRegionNameAsString());
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index fd311865037a..d69007afc752 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -61,14 +61,24 @@
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.ipc.MasterExecRPCInvoker;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.util.StringUtils;
 
+import com.google.protobuf.ServiceException;
+
 /**
  * Provides an interface to manage HBase database table metadata + general
  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
@@ -194,7 +204,7 @@ public void abort(String why, Throwable e) {
     this.aborted = true;
     throw new RuntimeException(why, e);
   }
-  
+
   @Override
   public boolean isAborted(){
     return this.aborted;
@@ -601,7 +611,7 @@ public void deleteTable(final byte [] tableName) throws IOException {
         // continue
       }
     }
-    
+
     if (tableExists) {
       throw new IOException("Retries exhausted, it took too long to wait"+
         " for the table " + Bytes.toString(tableName) + " to be deleted.");
@@ -676,9 +686,23 @@ public void enableTable(final byte [] tableName)
     enableTableAsync(tableName);
 
     // Wait until all regions are enabled
+    waitUntilTableIsEnabled(tableName);
+
+    LOG.info("Enabled table " + Bytes.toString(tableName));
+  }
+
+  /**
+   * Wait for the table to be enabled and available
+   * If enabling the table exceeds the retry period, an exception is thrown.
+   * @param tableName name of the table
+   * @throws IOException if a remote or network exception occurs or
+   *    table is not enabled after the retries period.
+   */
+  private void waitUntilTableIsEnabled(final byte[] tableName) throws IOException {
     boolean enabled = false;
+    long start = EnvironmentEdgeManager.currentTimeMillis();
     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
-      enabled = isTableEnabled(tableName);
+      enabled = isTableEnabled(tableName) && isTableAvailable(tableName);
       if (enabled) {
         break;
       }
@@ -697,10 +721,10 @@ public void enableTable(final byte [] tableName)
       }
     }
     if (!enabled) {
-      throw new IOException("Unable to enable table " +
-        Bytes.toString(tableName));
+      long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
+      throw new IOException("Table '" + Bytes.toString(tableName) +
+        "' not yet enabled, after " + msec + "ms.");
     }
-    LOG.info("Enabled table " + Bytes.toString(tableName));
   }
 
   public void enableTableAsync(final String tableName)
@@ -1119,7 +1143,7 @@ public void closeRegion(final byte [] regionname, final String serverName)
    * servername is provided then based on the online regions in the specified
    * regionserver the specified region will be closed. The master will not be
    * informed of the close. Note that the regionname is the encoded regionname.
-   * 
+   *
    * @param encodedRegionName
    *          The encoded region name; i.e. the hash that makes up the region
    *          name suffix: e.g. if regionname is
@@ -1255,7 +1279,7 @@ public void compact(final byte [] tableNameOrRegionName)
   throws IOException, InterruptedException {
     compact(tableNameOrRegionName, null, false);
   }
-  
+
   /**
    * Compact a column family within a table or region.
    * Asynchronous operation.
@@ -1309,7 +1333,7 @@ public void majorCompact(final byte [] tableNameOrRegionName)
   throws IOException, InterruptedException {
     compact(tableNameOrRegionName, null, true);
   }
-  
+
   /**
    * Major compact a column family within a table or region.
    * Asynchronous operation.
@@ -1761,7 +1785,7 @@ public static void checkHBaseAvailable(Configuration conf)
    * @param tableName the name of the table
    * @return Ordered list of {@link HRegionInfo}.
    * @throws IOException
-   */  
+   */
   public List<HRegionInfo> getTableRegions(final byte[] tableName)
   throws IOException {
     CatalogTracker ct = getCatalogTracker();
@@ -1773,7 +1797,7 @@ public List<HRegionInfo> getTableRegions(final byte[] tableName)
     }
     return Regions;
   }
-  
+
   public void close() throws IOException {
     if (this.connection != null) {
       this.connection.close();
@@ -1793,14 +1817,14 @@ public HTableDescriptor[] getTableDescriptors(List<String> tableNames)
 
   /**
    * Roll the log writer. That is, start writing log messages to a new file.
-   * 
+   *
    * @param serverName
    *          The servername of the regionserver. A server name is made of host,
    *          port and startcode. This is mandatory. Here is an example:
    *          <code> host187.example.com,60020,1289493121758</code>
    * @return If lots of logs, flush the returned regions so next time through
    * we can clean logs. Returns null if nothing to flush.  Names are actual
-   * region names as returned by {@link HRegionInfo#getEncodedName()}  
+   * region names as returned by {@link HRegionInfo#getEncodedName()}
    * @throws IOException if a remote or network exception occurs
    * @throws FailedLogCloseException
    */
@@ -1921,4 +1945,367 @@ public <T extends CoprocessorProtocol> T coprocessorProxy(
             connection,
             protocol));
   }
+
+
+  /**
+   * Create a timestamp consistent snapshot for the given table.
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will fail with
+   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link HTableDescriptor#isLegalTableName(byte[])}.
+   * @param snapshotName name of the snapshot to be created
+   * @param tableName name of the table for which snapshot is created
+   * @throws IOException if a remote or network exception occurs
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+  public void snapshot(final String snapshotName, final String tableName) throws IOException,
+      SnapshotCreationException, IllegalArgumentException {
+    snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
+  }
+
+  /**
+   * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
+   * taken. If the table is disabled, an offline snapshot is taken.
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will fail with
+   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link HTableDescriptor#isLegalTableName(byte[])}.
+   * @param snapshotName name of the snapshot to be created
+   * @param tableName name of the table for which snapshot is created
+   * @throws IOException if a remote or network exception occurs
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+  public void snapshot(final byte[] snapshotName, final byte[] tableName) throws IOException,
+      SnapshotCreationException, IllegalArgumentException {
+    snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName));
+  }
+
+  /**
+   * Create typed snapshot of the table.
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will fail with
+   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link HTableDescriptor#isLegalTableName(byte[])}.
+   * <p>
+   * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
+   * snapshots stored on the cluster
+   * @param tableName name of the table to snapshot
+   * @param type type of snapshot to take
+   * @throws IOException we fail to reach the master
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+  public void snapshot(final String snapshotName, final String tableName,
+      SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
+      IllegalArgumentException {
+    SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+    builder.setTable(tableName);
+    builder.setName(snapshotName);
+    builder.setType(type);
+    snapshot(builder.build());
+  }
+
+  /**
+   * Take a snapshot and wait for the server to complete that snapshot (blocking).
+   * <p>
+   * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
+   * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
+   * time for a single cluster).
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will fail with
+   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link HTableDescriptor#isLegalTableName(byte[])}.
+   * <p>
+   * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
+   * unless you are sure about the type of snapshot that you want to take.
+   * @param snapshot snapshot to take
+   * @throws IOException or we lose contact with the master.
+   * @throws SnapshotCreationException if snapshot failed to be taken
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+  public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
+      IllegalArgumentException {
+    HSnapshotDescription snapshotWritable = new HSnapshotDescription(snapshot);
+
+    try {
+      // actually take the snapshot
+      long max = takeSnapshotAsync(snapshot);
+      long start = EnvironmentEdgeManager.currentTimeMillis();
+      long maxPauseTime = max / this.numRetries;
+      boolean done = false;
+      int tries = 0;
+      LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
+          SnapshotDescriptionUtils.toString(snapshot) + "' to complete. (max " +
+          maxPauseTime + " ms per retry)");
+      while (tries == 0 || (EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done) {
+        try {
+          // sleep a backoff <= pauseTime amount
+          long sleep = getPauseTime(tries++);
+          sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
+          LOG.debug("(#" + tries + ") Sleeping: " + sleep +
+            "ms while waiting for snapshot completion.");
+          Thread.sleep(sleep);
+
+        } catch (InterruptedException e) {
+          LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete");
+          Thread.currentThread().interrupt();
+        }
+        LOG.debug("Getting current status of snapshot from master...");
+        done = getMaster().isSnapshotDone(snapshotWritable);
+      }
+
+      if (!done) {
+        throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
+            + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
+      }
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
+    }
+  }
+
+  /**
+   * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
+   * <p>
+   * Only a single snapshot should be taken at a time, or results may be undefined.
+   * @param snapshot snapshot to take
+   * @return the max time in millis to wait for the snapshot
+   * @throws IOException if the snapshot did not succeed or we lose contact with the master.
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+  public long takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
+      SnapshotCreationException {
+    SnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
+    HSnapshotDescription snapshotWritable = new HSnapshotDescription(snapshot);
+    return getMaster().snapshot(snapshotWritable);
+  }
+
+  /**
+   * Check the current state of the passed snapshot.
+   * <p>
+   * There are three possible states:
+   * <ol>
+   * <li>running - returns <tt>false</tt></li>
+   * <li>finished - returns <tt>true</tt></li>
+   * <li>finished with error - throws the exception that caused the snapshot to fail</li>
+   * </ol>
+   * <p>
+   * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
+   * run/started since the snapshot your are checking, you will recieve an
+   * {@link UnknownSnapshotException}.
+   * @param snapshot description of the snapshot to check
+   * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
+   * running
+   * @throws IOException if we have a network issue
+   * @throws HBaseSnapshotException if the snapshot failed
+   * @throws UnknownSnapshotException if the requested snapshot is unknown
+   */
+  public boolean isSnapshotFinished(final SnapshotDescription snapshot)
+      throws IOException, HBaseSnapshotException, UnknownSnapshotException {
+    try {
+      return getMaster().isSnapshotDone(new HSnapshotDescription(snapshot));
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
+    }
+  }
+
+  /**
+   * Restore the specified snapshot on the original table. (The table must be disabled)
+   * Before restoring the table, a new snapshot with the current table state is created.
+   * In case of failure, the table will be rolled back to the its original state.
+   *
+   * @param snapshotName name of the snapshot to restore
+   * @throws IOException if a remote or network exception occurs
+   * @throws RestoreSnapshotException if snapshot failed to be restored
+   * @throws IllegalArgumentException if the restore request is formatted incorrectly
+   */
+  public void restoreSnapshot(final byte[] snapshotName)
+      throws IOException, RestoreSnapshotException {
+    restoreSnapshot(Bytes.toString(snapshotName));
+  }
+
+  /**
+   * Restore the specified snapshot on the original table. (The table must be disabled)
+   * Before restoring the table, a new snapshot with the current table state is created.
+   * In case of failure, the table will be rolled back to its original state.
+   *
+   * @param snapshotName name of the snapshot to restore
+   * @throws IOException if a remote or network exception occurs
+   * @throws RestoreSnapshotException if snapshot failed to be restored
+   * @throws IllegalArgumentException if the restore request is formatted incorrectly
+   */
+  public void restoreSnapshot(final String snapshotName)
+      throws IOException, RestoreSnapshotException {
+    String rollbackSnapshot = snapshotName + "-" + EnvironmentEdgeManager.currentTimeMillis();
+
+    String tableName = null;
+    for (SnapshotDescription snapshotInfo: listSnapshots()) {
+      if (snapshotInfo.getName().equals(snapshotName)) {
+        tableName = snapshotInfo.getTable();
+        break;
+      }
+    }
+
+    if (tableName == null) {
+      throw new RestoreSnapshotException(
+        "Unable to find the table name for snapshot=" + snapshotName);
+    }
+
+    // Take a snapshot of the current state
+    snapshot(rollbackSnapshot, tableName);
+
+    // Restore snapshot
+    try {
+      internalRestoreSnapshot(snapshotName, tableName);
+    } catch (IOException e) {
+      // Try to rollback
+      try {
+        String msg = "Restore snapshot=" + snapshotName +
+          " failed. Rollback to snapshot=" + rollbackSnapshot + " succeeded.";
+        LOG.error(msg, e);
+        internalRestoreSnapshot(rollbackSnapshot, tableName);
+        throw new RestoreSnapshotException(msg, e);
+      } catch (IOException ex) {
+        String msg = "Failed to restore and rollback to snapshot=" + rollbackSnapshot;
+        LOG.error(msg, ex);
+        throw new RestoreSnapshotException(msg, ex);
+      }
+    }
+  }
+
+  /**
+   * Create a new table by cloning the snapshot content.
+   *
+   * @param snapshotName name of the snapshot to be cloned
+   * @param tableName name of the table where the snapshot will be restored
+   * @throws IOException if a remote or network exception occurs
+   * @throws TableExistsException if table to be created already exists
+   * @throws RestoreSnapshotException if snapshot failed to be cloned
+   * @throws IllegalArgumentException if the specified table has not a valid name
+   */
+  public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
+      throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
+    cloneSnapshot(Bytes.toString(snapshotName), Bytes.toString(tableName));
+  }
+
+  /**
+   * Create a new table by cloning the snapshot content.
+   *
+   * @param snapshotName name of the snapshot to be cloned
+   * @param tableName name of the table where the snapshot will be restored
+   * @throws IOException if a remote or network exception occurs
+   * @throws TableExistsException if table to be created already exists
+   * @throws RestoreSnapshotException if snapshot failed to be cloned
+   * @throws IllegalArgumentException if the specified table has not a valid name
+   */
+  public void cloneSnapshot(final String snapshotName, final String tableName)
+      throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
+    if (tableExists(tableName)) {
+      throw new TableExistsException("Table '" + tableName + " already exists");
+    }
+    internalRestoreSnapshot(snapshotName, tableName);
+    waitUntilTableIsEnabled(Bytes.toBytes(tableName));
+  }
+
+  /**
+   * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
+   * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
+   * create an HTable instance to this table before it is available.
+   * @param snapshot snapshot to restore
+   * @param tableName table name to restore the snapshot on
+   * @throws IOException if a remote or network exception occurs
+   * @throws RestoreSnapshotException if snapshot failed to be restored
+   * @throws IllegalArgumentException if the restore request is formatted incorrectly
+   */
+  private void internalRestoreSnapshot(final String snapshotName, final String tableName)
+      throws IOException, RestoreSnapshotException {
+    HSnapshotDescription snapshot = new HSnapshotDescription(
+      SnapshotDescription.newBuilder().setName(snapshotName).setTable(tableName).build());
+
+    try {
+      // actually restore the snapshot
+      getMaster().restoreSnapshot(snapshot);
+
+      final long maxPauseTime = 5000;
+      boolean done = false;
+      int tries = 0;
+      while (!done) {
+        try {
+          // sleep a backoff <= pauseTime amount
+          long sleep = getPauseTime(tries++);
+          sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
+          LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
+          Thread.sleep(sleep);
+        } catch (InterruptedException e) {
+          LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
+          Thread.currentThread().interrupt();
+        }
+        LOG.debug("Getting current status of snapshot restore from master...");
+        done = getMaster().isRestoreSnapshotDone(snapshot);
+      }
+      if (!done) {
+        throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
+      }
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
+    }
+  }
+
+  /**
+   * List completed snapshots.
+   * @return a list of snapshot descriptors for completed snapshots
+   * @throws IOException if a network error occurs
+   */
+  public List<SnapshotDescription> listSnapshots() throws IOException {
+    List<SnapshotDescription> snapshots = new LinkedList<SnapshotDescription>();
+    try {
+      for (HSnapshotDescription snapshot: getMaster().getCompletedSnapshots()) {
+        snapshots.add(snapshot.getProto());
+      }
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
+    }
+    return snapshots;
+  }
+
+  /**
+   * Delete an existing snapshot.
+   * @param snapshotName name of the snapshot
+   * @throws IOException if a remote or network exception occurs
+   */
+  public void deleteSnapshot(final byte[] snapshotName) throws IOException {
+    // make sure the snapshot is possibly valid
+    HTableDescriptor.isLegalTableName(snapshotName);
+    // do the delete
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder()
+      .setName(Bytes.toString(snapshotName)).build();
+    try {
+      getMaster().deleteSnapshot(new HSnapshotDescription(snapshot));
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
+    }
+  }
+
+  /**
+   * Delete an existing snapshot.
+   * @param snapshotName name of the snapshot
+   * @throws IOException if a remote or network exception occurs
+   */
+  public void deleteSnapshot(final String snapshotName) throws IOException {
+    deleteSnapshot(Bytes.toBytes(snapshotName));
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
index 428feb14b9ec..3b71ff879d18 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
 
@@ -185,4 +186,50 @@ public void postMove(ObserverContext<MasterCoprocessorEnvironment> ctx,
       HRegionInfo region, ServerName srcServer, ServerName destServer)
   throws IOException {
   }
+
+  @Override
+  public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException {
+  }
+
+  @Override
+  public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot) throws IOException {
+  }
+
+  @Override
+  public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot) throws IOException {
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index a36eb1e6bc4d..084279e829a0 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -21,6 +21,7 @@
 package org.apache.hadoop.hbase.coprocessor;
 
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
 
@@ -289,4 +290,100 @@ void preStopMaster(final ObserverContext<MasterCoprocessorEnvironment> ctx)
    */
   void postStartMaster(final ObserverContext<MasterCoprocessorEnvironment> ctx)
       throws IOException;
+
+  /**
+   * Called before a new snapshot is taken.
+   * Called as part of snapshot RPC call.
+   * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor for the snapshot
+   * @param hTableDescriptor the hTableDescriptor of the table to snapshot
+   * @throws IOException
+   */
+  void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException;
+
+  /**
+   * Called after the snapshot operation has been requested.
+   * Called as part of snapshot RPC call.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor for the snapshot
+   * @param hTableDescriptor the hTableDescriptor of the table to snapshot
+   * @throws IOException
+   */
+  void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException;
+
+  /**
+   * Called before a snapshot is cloned.
+   * Called as part of restoreSnapshot RPC call.
+   * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor for the snapshot
+   * @param hTableDescriptor the hTableDescriptor of the table to create
+   * @throws IOException
+   */
+  void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException;
+
+  /**
+   * Called after a snapshot clone operation has been requested.
+   * Called as part of restoreSnapshot RPC call.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor for the snapshot
+   * @param hTableDescriptor the hTableDescriptor of the table to create
+   * @throws IOException
+   */
+  void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException;
+
+  /**
+   * Called before a snapshot is restored.
+   * Called as part of restoreSnapshot RPC call.
+   * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor for the snapshot
+   * @param hTableDescriptor the hTableDescriptor of the table to restore
+   * @throws IOException
+   */
+  void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException;
+
+  /**
+   * Called after a snapshot restore operation has been requested.
+   * Called as part of restoreSnapshot RPC call.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor for the snapshot
+   * @param hTableDescriptor the hTableDescriptor of the table to restore
+   * @throws IOException
+   */
+  void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws IOException;
+
+  /**
+   * Called before a snapshot is deleted.
+   * Called as part of deleteSnapshot RPC call.
+   * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor of the snapshot to delete
+   * @throws IOException
+   */
+  void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot) throws IOException;
+
+  /**
+   * Called after the delete snapshot operation has been requested.
+   * Called as part of deleteSnapshot RPC call.
+   * @param ctx the environment to interact with the framework and master
+   * @param snapshot the SnapshotDescriptor of the snapshot to delete
+   * @throws IOException
+   */
+  void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+      final SnapshotDescription snapshot) throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java
new file mode 100644
index 000000000000..ee340cb20209
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * A ForeignException is an exception from another thread or process.
+ * <p>
+ * ForeignExceptions are sent to 'remote' peers to signal an abort in the face of failures.
+ * When serialized for transmission we encode using Protobufs to ensure version compatibility.
+ * <p>
+ * Foreign exceptions contain a Throwable as its cause.  This can be a "regular" exception
+ * generated locally or a ProxyThrowable that is a representation of the original exception
+ * created on original 'remote' source.  These ProxyThrowables have their their stacks traces and
+ * messages overridden to reflect the original 'remote' exception.  The only way these
+ * ProxyThrowables are generated are by this class's {@link #deserialize(byte[])} method.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public class ForeignException extends IOException {
+
+  /**
+   * Name of the throwable's source such as a host or thread name.  Must be non-null.
+   */
+  private final String source;
+
+  /**
+   * Create a new ForeignException that can be serialized.  It is assumed that this came form a
+   * local source.
+   * @param source
+   * @param cause
+   */
+  public ForeignException(String source, Throwable cause) {
+    super(cause);
+    assert source != null;
+    assert cause != null;
+    this.source = source;
+  }
+
+  /**
+   * Create a new ForeignException that can be serialized.  It is assumed that this is locally
+   * generated.
+   * @param source
+   * @param msg
+   */
+  public ForeignException(String source, String msg) {
+    super(new IllegalArgumentException(msg));
+    this.source = source;
+  }
+
+  public String getSource() {
+    return source;
+  }
+
+  /**
+   * The cause of a ForeignException can be an exception that was generated on a local in process
+   * thread, or a thread from a 'remote' separate process.
+   *
+   * If the cause is a ProxyThrowable, we know it came from deserialization which usually means
+   * it came from not only another thread, but also from a remote thread.
+   *
+   * @return true if went through deserialization, false if locally generated
+   */
+  public boolean isRemote() {
+    return getCause() instanceof ProxyThrowable;
+  }
+
+  @Override
+  public String toString() {
+    String className = getCause().getClass().getName()  ;
+    return className + " via " + getSource() + ":" + getLocalizedMessage();
+  }
+
+  /**
+   * Convert a stack trace to list of {@link StackTraceElement}.
+   * @param trace the stack trace to convert to protobuf message
+   * @return <tt>null</tt> if the passed stack is <tt>null</tt>.
+   */
+  private static List<StackTraceElementMessage> toStackTraceElementMessages(
+      StackTraceElement[] trace) {
+    // if there is no stack trace, ignore it and just return the message
+    if (trace == null) return null;
+    // build the stack trace for the message
+    List<StackTraceElementMessage> pbTrace =
+        new ArrayList<StackTraceElementMessage>(trace.length);
+    for (StackTraceElement elem : trace) {
+      StackTraceElementMessage.Builder stackBuilder = StackTraceElementMessage.newBuilder();
+      stackBuilder.setDeclaringClass(elem.getClassName());
+      stackBuilder.setFileName(elem.getFileName());
+      stackBuilder.setLineNumber(elem.getLineNumber());
+      stackBuilder.setMethodName(elem.getMethodName());
+      pbTrace.add(stackBuilder.build());
+    }
+    return pbTrace;
+  }
+
+  /**
+   * This is a Proxy Throwable that contains the information of the original remote exception
+   */
+  private static class ProxyThrowable extends Throwable {
+    ProxyThrowable(String msg, StackTraceElement[] trace) {
+      super(msg);
+      this.setStackTrace(trace);
+    }
+  }
+
+  /**
+   * Converts a ForeignException to an array of bytes.
+   * @param source the name of the external exception source
+   * @param t the "local" external exception (local)
+   * @return protobuf serialized version of ForeignException
+   */
+  public static byte[] serialize(String source, Throwable t) {
+    GenericExceptionMessage.Builder gemBuilder = GenericExceptionMessage.newBuilder();
+    gemBuilder.setClassName(t.getClass().getName());
+    if (t.getMessage() != null) {
+      gemBuilder.setMessage(t.getMessage());
+    }
+    // set the stack trace, if there is one
+    List<StackTraceElementMessage> stack =
+        ForeignException.toStackTraceElementMessages(t.getStackTrace());
+    if (stack != null) {
+      gemBuilder.addAllTrace(stack);
+    }
+    GenericExceptionMessage payload = gemBuilder.build();
+    ForeignExceptionMessage.Builder exception = ForeignExceptionMessage.newBuilder();
+    exception.setGenericException(payload).setSource(source);
+    ForeignExceptionMessage eem = exception.build();
+    return eem.toByteArray();
+  }
+
+  /**
+   * Takes a series of bytes and tries to generate an ForeignException instance for it.
+   * @param bytes
+   * @return the ForeignExcpetion instance
+   * @throws InvalidProtocolBufferException if there was deserialization problem this is thrown.
+   */
+  public static ForeignException deserialize(byte[] bytes) throws InvalidProtocolBufferException {
+    // figure out the data we need to pass
+    ForeignExceptionMessage eem = ForeignExceptionMessage.parseFrom(bytes);
+    GenericExceptionMessage gem = eem.getGenericException();
+    StackTraceElement [] trace = ForeignException.toStackTrace(gem.getTraceList());
+    ProxyThrowable dfe = new ProxyThrowable(gem.getMessage(), trace);
+    ForeignException e = new ForeignException(eem.getSource(), dfe);
+    return e;
+  }
+
+  /**
+   * Unwind a serialized array of {@link StackTraceElementMessage}s to a
+   * {@link StackTraceElement}s.
+   * @param traceList list that was serialized
+   * @return the deserialized list or <tt>null</tt> if it couldn't be unwound (e.g. wasn't set on
+   *         the sender).
+   */
+  private static StackTraceElement[] toStackTrace(List<StackTraceElementMessage> traceList) {
+    if (traceList == null || traceList.size() == 0) {
+      return new StackTraceElement[0]; // empty array
+    }
+    StackTraceElement[] trace = new StackTraceElement[traceList.size()];
+    for (int i = 0; i < traceList.size(); i++) {
+      StackTraceElementMessage elem = traceList.get(i);
+      trace[i] = new StackTraceElement(
+          elem.getDeclaringClass(), elem.getMethodName(), elem.getFileName(), elem.getLineNumber());
+    }
+    return trace;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java
new file mode 100644
index 000000000000..ceb3b840a7b4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The dispatcher acts as the state holding entity for foreign error handling.  The first
+ * exception received by the dispatcher get passed directly to the listeners.  Subsequent
+ * exceptions are dropped.
+ * <p>
+ * If there are multiple dispatchers that are all in the same foreign exception monitoring group,
+ * ideally all these monitors are "peers" -- any error on one dispatcher should get propagated to
+ * all others (via rpc, or some other mechanism).  Due to racing error conditions the exact reason
+ * for failure may be different on different peers, but the fact that they are in error state
+ * should eventually hold on all.
+ * <p>
+ * This is thread-safe and must be because this is expected to be used to propagate exceptions
+ * from foreign threads.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ForeignExceptionDispatcher implements ForeignExceptionListener, ForeignExceptionSnare {
+  public static final Log LOG = LogFactory.getLog(ForeignExceptionDispatcher.class);
+  protected final String name;
+  protected final List<ForeignExceptionListener> listeners =
+      new ArrayList<ForeignExceptionListener>();
+  private ForeignException exception;
+
+  public ForeignExceptionDispatcher(String name) {
+    this.name = name;
+  }
+
+  public ForeignExceptionDispatcher() {
+    this("");
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public synchronized void receive(ForeignException e) {
+    // if we already have an exception, then ignore it
+    if (exception != null) return;
+
+    LOG.debug(name + " accepting received exception" , e);
+    // mark that we got the error
+    if (e != null) {
+      exception = e;
+    } else {
+      exception = new ForeignException(name, "");
+    }
+
+    // notify all the listeners
+    dispatch(e);
+  }
+
+  @Override
+  public synchronized void rethrowException() throws ForeignException {
+    if (exception != null) {
+      // This gets the stack where this is caused, (instead of where it was deserialized).
+      // This is much more useful for debugging
+      throw new ForeignException(exception.getSource(), exception.getCause());
+    }
+  }
+
+  @Override
+  public synchronized boolean hasException() {
+    return exception != null;
+  }
+
+  @Override
+  synchronized public ForeignException getException() {
+    return exception;
+  }
+
+  /**
+   * Sends an exception to all listeners.
+   * @param message human readable message passed to the listener
+   * @param e {@link ForeignException} containing the cause.  Can be null.
+   */
+  private void dispatch(ForeignException e) {
+    // update all the listeners with the passed error
+    for (ForeignExceptionListener l: listeners) {
+      l.receive(e);
+    }
+  }
+
+  /**
+   * Listen for failures to a given process.  This method should only be used during
+   * initialization and not added to after exceptions are accepted.
+   * @param errorable listener for the errors.  may be null.
+   */
+  public synchronized void addListener(ForeignExceptionListener errorable) {
+    this.listeners.add(errorable);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java
new file mode 100644
index 000000000000..014da53fd633
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The ForeignExceptionListener is an interface for objects that can receive a ForeignException.
+ * <p>
+ * Implementations must be thread-safe, because this is expected to be used to propagate exceptions
+ * from foreign threads.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ForeignExceptionListener {
+
+  /**
+   * Receive a ForeignException.
+   * <p>
+   * Implementers must ensure that this method is thread-safe.
+   * @param e exception causing the error. Implementations must accept and handle null here.
+   */
+  public void receive(ForeignException e);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java
new file mode 100644
index 000000000000..47586ddd43a7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This is an interface for a cooperative exception throwing mechanism.  Implementations are
+ * containers that holds an exception from a separate thread. This can be used to receive
+ * exceptions from 'foreign' threads or from separate 'foreign' processes.
+ * <p>
+ * To use, one would pass an implementation of this object to a long running method and
+ * periodically check by calling {@link #rethrowException()}.  If any foreign exceptions have
+ * been received, the calling thread is then responsible for handling the rethrown exception.
+ * <p>
+ * One could use the boolean {@link #hasException()} to determine if there is an exceptoin as well.
+ * <p>
+ * NOTE: This is very similar to the InterruptedException/interrupt/interrupted pattern.  There,
+ * the notification state is bound to a Thread.  Using this, applications receive Exceptions in
+ * the snare.  The snare is referenced and checked by multiple threads which enables exception 
+ * notification in all the involved threads/processes. 
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ForeignExceptionSnare {
+
+  /**
+   * Rethrow an exception currently held by the {@link ForeignExceptionSnare}. If there is
+   * no exception this is a no-op
+   *
+   * @throws ForeignException
+   *           all exceptions from remote sources are procedure exceptions
+   */
+  public void rethrowException() throws ForeignException;
+
+  /**
+   * Non-exceptional form of {@link #rethrowException()}. Checks to see if any
+   * process to which the exception checkers is bound has created an error that
+   * would cause a failure.
+   *
+   * @return <tt>true</tt> if there has been an error,<tt>false</tt> otherwise
+   */
+  public boolean hasException();
+
+  /**
+   * Get the value of the captured exception.
+   *
+   * @return the captured foreign exception or null if no exception captured.
+   */
+  public ForeignException getException();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java
new file mode 100644
index 000000000000..b67d7d494a40
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Exception for timeout of a task.
+ * @see TimeoutExceptionInjector
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public class TimeoutException extends Exception {
+
+  private final String sourceName;
+  private final long start;
+  private final long end;
+  private final long expected;
+
+  /**
+   * Exception indicating that an operation attempt has timed out
+   * @param start time the operation started (ms since epoch)
+   * @param end time the timeout was triggered (ms since epoch)
+   * @param expected expected amount of time for the operation to complete (ms) (ideally, expected <= end-start)
+   */
+  public TimeoutException(String sourceName, long start, long end, long expected) {
+    super("Timeout elapsed! Source:" + sourceName + " Start:" + start + ", End:" + end
+        + ", diff:" + (end - start) + ", max:" + expected + " ms");
+    this.sourceName = sourceName;
+    this.start = start;
+    this.end = end;
+    this.expected = expected;
+  }
+
+  public long getStart() {
+    return start;
+  }
+
+  public long getEnd() {
+    return end;
+  }
+
+  public long getMaxAllowedOperationTime() {
+    return expected;
+  }
+
+  public String getSourceName() {
+    return sourceName;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java
new file mode 100644
index 000000000000..9f40cbfb575e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * Time a given process/operation and report a failure if the elapsed time exceeds the max allowed
+ * time.
+ * <p>
+ * The timer won't start tracking time until calling {@link #start()}. If {@link #complete()} or
+ * {@link #trigger()} is called before {@link #start()}, calls to {@link #start()} will fail.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TimeoutExceptionInjector {
+
+  private static final Log LOG = LogFactory.getLog(TimeoutExceptionInjector.class);
+
+  private final long maxTime;
+  private volatile boolean complete;
+  private final Timer timer;
+  private final TimerTask timerTask;
+  private long start = -1;
+
+  /**
+   * Create a generic timer for a task/process.
+   * @param listener listener to notify if the process times out
+   * @param maxTime max allowed running time for the process. Timer starts on calls to
+   *          {@link #start()}
+   */
+  public TimeoutExceptionInjector(final ForeignExceptionListener listener, final long maxTime) {
+    this.maxTime = maxTime;
+    timer = new Timer();
+    timerTask = new TimerTask() {
+      @Override
+      public void run() {
+        // ensure we don't run this task multiple times
+        synchronized (this) {
+          // quick exit if we already marked the task complete
+          if (TimeoutExceptionInjector.this.complete) return;
+          // mark the task is run, to avoid repeats
+          TimeoutExceptionInjector.this.complete = true;
+        }
+        long end = EnvironmentEdgeManager.currentTimeMillis();
+        TimeoutException tee =  new TimeoutException(
+            "Timeout caused Foreign Exception", start, end, maxTime);
+        String source = "timer-" + timer;
+        listener.receive(new ForeignException(source, tee));
+      }
+    };
+  }
+
+  public long getMaxTime() {
+    return maxTime;
+  }
+
+  /**
+   * For all time forward, do not throw an error because the process has completed.
+   */
+  public void complete() {
+    // warn if the timer is already marked complete. This isn't going to be thread-safe, but should
+    // be good enough and its not worth locking just for a warning.
+    if (this.complete) {
+      LOG.warn("Timer already marked completed, ignoring!");
+      return;
+    }
+    LOG.debug("Marking timer as complete - no error notifications will be received for this timer.");
+    synchronized (this.timerTask) {
+      this.complete = true;
+    }
+    this.timer.cancel();
+  }
+
+  /**
+   * Start a timer to fail a process if it takes longer than the expected time to complete.
+   * <p>
+   * Non-blocking.
+   * @throws IllegalStateException if the timer has already been marked done via {@link #complete()}
+   *           or {@link #trigger()}
+   */
+  public synchronized void start() throws IllegalStateException {
+    if (this.start >= 0) {
+      LOG.warn("Timer already started, can't be started again. Ignoring second request.");
+      return;
+    }
+    LOG.debug("Scheduling process timer to run in: " + maxTime + " ms");
+    timer.schedule(timerTask, maxTime);
+    this.start = EnvironmentEdgeManager.currentTimeMillis();
+  }
+
+  /**
+   * Trigger the timer immediately.
+   * <p>
+   * Exposed for testing.
+   */
+  public void trigger() {
+    synchronized (timerTask) {
+      if (this.complete) {
+        LOG.warn("Timer already completed, not triggering.");
+        return;
+      }
+      LOG.debug("Triggering timer immediately!");
+      this.timer.cancel();
+      this.timerTask.run();
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
index c9acee3ce036..1be8a0fd63eb 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
@@ -128,6 +128,8 @@ public enum EventType {
     C_M_DELETE_FAMILY         (45),   // Client asking Master to delete family of table
     C_M_MODIFY_FAMILY         (46),   // Client asking Master to modify family of table
     C_M_CREATE_TABLE          (47),   // Client asking Master to create a table
+    C_M_SNAPSHOT_TABLE        (48),   // Client asking Master to snapshot an offline table
+    C_M_RESTORE_SNAPSHOT      (49),   // Client asking Master to snapshot an offline table
 
     // Updates from master to ZK. This is done by the master and there is
     // nothing to process by either Master or RS
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
index 012dc0cbc95e..a34dd691adbd 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
@@ -136,6 +136,8 @@ public ExecutorType getExecutorServiceType(final EventHandler.EventType type) {
       case C_M_ENABLE_TABLE:
       case C_M_MODIFY_TABLE:
       case C_M_CREATE_TABLE:
+      case C_M_SNAPSHOT_TABLE:
+      case C_M_RESTORE_SNAPSHOT:
         return ExecutorType.MASTER_TABLE_OPERATIONS;
 
       // RegionServer executor services
diff --git a/src/main/java/org/apache/hadoop/hbase/io/FileLink.java b/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
new file mode 100644
index 000000000000..e04de85ccbed
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
@@ -0,0 +1,455 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.util.Collection;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.FileNotFoundException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+
+/**
+ * The FileLink is a sort of hardlink, that allows to access a file given a set of locations.
+ *
+ * <p><b>The Problem:</b>
+ * <ul>
+ *  <li>
+ *    HDFS doesn't have support for hardlinks, and this make impossible to referencing
+ *    the same data blocks using different names.
+ *  </li>
+ *  <li>
+ *    HBase store files in one location (e.g. table/region/family/) and when the file is not
+ *    needed anymore (e.g. compaction, region deletetion, ...) moves it to an archive directory.
+ *  </li>
+ * </ul>
+ * If we want to create a reference to a file, we need to remember that it can be in its
+ * original location or in the archive folder.
+ * The FileLink class tries to abstract this concept and given a set of locations
+ * it is able to switch between them making this operation transparent for the user.
+ * More concrete implementations of the FileLink are the {@link HFileLink} and the {@link HLogLink}.
+ *
+ * <p><b>Back-references:</b>
+ * To help the {@link CleanerChore} to keep track of the links to a particular file,
+ * during the FileLink creation, a new file is placed inside a back-reference directory.
+ * There's one back-reference directory for each file that has links,
+ * and in the directory there's one file per link.
+ *
+ * <p>HFileLink Example
+ * <ul>
+ *  <li>
+ *      /hbase/table/region-x/cf/file-k
+ *      (Original File)
+ *  </li>
+ *  <li>
+ *      /hbase/table-cloned/region-y/cf/file-k.region-x.table
+ *     (HFileLink to the original file)
+ *  </li>
+ *  <li>
+ *      /hbase/table-2nd-cloned/region-z/cf/file-k.region-x.table
+ *      (HFileLink to the original file)
+ *  </li>
+ *  <li>
+ *      /hbase/.archive/table/region-x/.links-file-k/region-y.table-cloned
+ *      (Back-reference to the link in table-cloned)
+ *  </li>
+ *  <li>
+ *      /hbase/.archive/table/region-x/.links-file-k/region-z.table-cloned
+ *      (Back-reference to the link in table-2nd-cloned)
+ *  </li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+public class FileLink {
+  private static final Log LOG = LogFactory.getLog(FileLink.class);
+
+  /** Define the Back-reference directory name prefix: .links-<hfile>/ */
+  public static final String BACK_REFERENCES_DIRECTORY_PREFIX = ".links-";
+
+  /**
+   * FileLink InputStream that handles the switch between the original path
+   * and the alternative locations, when the file is moved.
+   */
+  private static class FileLinkInputStream extends InputStream
+      implements Seekable, PositionedReadable {
+    private FSDataInputStream in = null;
+    private Path currentPath = null;
+    private long pos = 0;
+
+    private final FileLink fileLink;
+    private final int bufferSize;
+    private final FileSystem fs;
+
+    public FileLinkInputStream(final FileSystem fs, final FileLink fileLink)
+        throws IOException {
+      this(fs, fileLink, fs.getConf().getInt("io.file.buffer.size", 4096));
+    }
+
+    public FileLinkInputStream(final FileSystem fs, final FileLink fileLink, int bufferSize)
+        throws IOException {
+      this.bufferSize = bufferSize;
+      this.fileLink = fileLink;
+      this.fs = fs;
+
+      this.in = tryOpen();
+    }
+
+    @Override
+    public int read() throws IOException {
+      int res;
+      try {
+        res = in.read();
+      } catch (FileNotFoundException e) {
+        res = tryOpen().read();
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().read();
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().read();
+      }
+      if (res > 0) pos += 1;
+      return res;
+    }
+
+    @Override
+    public int read(byte b[]) throws IOException {
+       return read(b, 0, b.length);
+    }
+
+    @Override
+    public int read(byte b[], int off, int len) throws IOException {
+      int n;
+      try {
+        n = in.read(b, off, len);
+      } catch (FileNotFoundException e) {
+        n = tryOpen().read(b, off, len);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(b, off, len);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(b, off, len);
+      }
+      if (n > 0) pos += n;
+      assert(in.getPos() == pos);
+      return n;
+    }
+
+    @Override
+    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+      int n;
+      try {
+        n = in.read(position, buffer, offset, length);
+      } catch (FileNotFoundException e) {
+        n = tryOpen().read(position, buffer, offset, length);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(position, buffer, offset, length);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(position, buffer, offset, length);
+      }
+      return n;
+    }
+
+    @Override
+    public void readFully(long position, byte[] buffer) throws IOException {
+      readFully(position, buffer, 0, buffer.length);
+    }
+
+    @Override
+    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+      try {
+        in.readFully(position, buffer, offset, length);
+      } catch (FileNotFoundException e) {
+        tryOpen().readFully(position, buffer, offset, length);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().readFully(position, buffer, offset, length);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().readFully(position, buffer, offset, length);
+      }
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+      long skipped;
+
+      try {
+        skipped = in.skip(n);
+      } catch (FileNotFoundException e) {
+        skipped = tryOpen().skip(n);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        skipped = tryOpen().skip(n);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        skipped = tryOpen().skip(n);
+      }
+
+      if (skipped > 0) pos += skipped;
+      return skipped;
+    }
+
+    @Override
+    public int available() throws IOException {
+      try {
+        return in.available();
+      } catch (FileNotFoundException e) {
+        return tryOpen().available();
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        return tryOpen().available();
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        return tryOpen().available();
+      }
+    }
+
+    @Override
+    public void seek(long pos) throws IOException {
+      try {
+        in.seek(pos);
+      } catch (FileNotFoundException e) {
+        tryOpen().seek(pos);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().seek(pos);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().seek(pos);
+      }
+      this.pos = pos;
+    }
+
+    @Override
+    public long getPos() throws IOException {
+      return pos;
+    }
+
+    @Override
+    public boolean seekToNewSource(long targetPos) throws IOException {
+      boolean res;
+      try {
+        res = in.seekToNewSource(targetPos);
+      } catch (FileNotFoundException e) {
+        res = tryOpen().seekToNewSource(targetPos);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().seekToNewSource(targetPos);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().seekToNewSource(targetPos);
+      }
+      if (res) pos = targetPos;
+      return res;
+    }
+
+    @Override
+    public void close() throws IOException {
+      in.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+      throw new IOException("mark/reset not supported");
+    }
+
+    @Override
+    public boolean markSupported() {
+      return false;
+    }
+
+    /**
+     * Try to open the file from one of the available locations.
+     *
+     * @return FSDataInputStream stream of the opened file link
+     * @throws IOException on unexpected error, or file not found.
+     */
+    private FSDataInputStream tryOpen() throws IOException {
+      for (Path path: fileLink.getLocations()) {
+        if (path.equals(currentPath)) continue;
+        try {
+          in = fs.open(path, bufferSize);
+          in.seek(pos);
+          assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos;
+          if (LOG.isTraceEnabled()) {
+            if (currentPath != null) {
+              LOG.debug("link open path=" + path);
+            } else {
+              LOG.trace("link switch from path=" + currentPath + " to path=" + path);
+            }
+          }
+          currentPath = path;
+          return(in);
+        } catch (FileNotFoundException e) {
+          // Try another file location
+        }
+      }
+      throw new FileNotFoundException("Unable to open link: " + fileLink);
+    }
+  }
+
+  private Path[] locations = null;
+
+  protected FileLink() {
+    this.locations = null;
+  }
+
+  /**
+   * @param originPath Original location of the file to link
+   * @param alternativePaths Alternative locations to look for the linked file
+   */
+  public FileLink(Path originPath, Path... alternativePaths) {
+    setLocations(originPath, alternativePaths);
+  }
+
+  /**
+   * @param locations locations to look for the linked file
+   */
+  public FileLink(final Collection<Path> locations) {
+    this.locations = locations.toArray(new Path[locations.size()]);
+  }
+
+  /**
+   * @return the locations to look for the linked file.
+   */
+  public Path[] getLocations() {
+    return locations;
+  }
+
+  public String toString() {
+    StringBuilder str = new StringBuilder(getClass().getName());
+    str.append(" locations=[");
+    int i = 0;
+    for (Path location: locations) {
+      if (i++ > 0) str.append(", ");
+      str.append(location.toString());
+    }
+    str.append("]");
+    return str.toString();
+  }
+
+  /**
+   * @return the path of the first available link.
+   */
+  public Path getAvailablePath(FileSystem fs) throws IOException {
+    for (Path path: locations) {
+      if (fs.exists(path)) {
+        return path;
+      }
+    }
+    throw new FileNotFoundException("Unable to open link: " + this);
+  }
+
+  /**
+   * Get the FileStatus of the referenced file.
+   *
+   * @param fs {@link FileSystem} on which to get the file status
+   * @return InputStream for the hfile link.
+   * @throws IOException on unexpected error.
+   */
+  public FileStatus getFileStatus(FileSystem fs) throws IOException {
+    for (Path path: locations) {
+      try {
+        return fs.getFileStatus(path);
+      } catch (FileNotFoundException e) {
+        // Try another file location
+      }
+    }
+    throw new FileNotFoundException("Unable to open link: " + this);
+  }
+
+  /**
+   * Open the FileLink for read.
+   * <p>
+   * It uses a wrapper of FSDataInputStream that is agnostic to the location
+   * of the file, even if the file switches between locations.
+   *
+   * @param fs {@link FileSystem} on which to open the FileLink
+   * @return InputStream for reading the file link.
+   * @throws IOException on unexpected error.
+   */
+  public FSDataInputStream open(final FileSystem fs) throws IOException {
+    return new FSDataInputStream(new FileLinkInputStream(fs, this));
+  }
+
+  /**
+   * Open the FileLink for read.
+   * <p>
+   * It uses a wrapper of FSDataInputStream that is agnostic to the location
+   * of the file, even if the file switches between locations.
+   *
+   * @param fs {@link FileSystem} on which to open the FileLink
+   * @param bufferSize the size of the buffer to be used.
+   * @return InputStream for reading the file link.
+   * @throws IOException on unexpected error.
+   */
+  public FSDataInputStream open(final FileSystem fs, int bufferSize) throws IOException {
+    return new FSDataInputStream(new FileLinkInputStream(fs, this, bufferSize));
+  }
+
+  /**
+   * NOTE: This method must be used only in the constructor!
+   * It creates a List with the specified locations for the link.
+   */
+  protected void setLocations(Path originPath, Path... alternativePaths) {
+    assert this.locations == null : "Link locations already set";
+    this.locations = new Path[1 + alternativePaths.length];
+    this.locations[0] = originPath;
+    for (int i = 0; i < alternativePaths.length; i++) {
+      this.locations[i + 1] = alternativePaths[i];
+    }
+  }
+
+  /**
+   * Get the directory to store the link back references
+   *
+   * <p>To simplify the reference count process, during the FileLink creation
+   * a back-reference is added to the back-reference directory of the specified file.
+   *
+   * @param storeDir Root directory for the link reference folder
+   * @param fileName File Name with links
+   * @return Path for the link back references.
+   */
+  public static Path getBackReferencesDir(final Path storeDir, final String fileName) {
+    return new Path(storeDir, BACK_REFERENCES_DIRECTORY_PREFIX + fileName);
+  }
+
+  /**
+   * Get the referenced file name from the reference link directory path.
+   *
+   * @param dirPath Link references directory path
+   * @return Name of the file referenced
+   */
+  public static String getBackReferenceFileName(final Path dirPath) {
+    return dirPath.getName().substring(BACK_REFERENCES_DIRECTORY_PREFIX.length());
+  }
+
+  /**
+   * Checks if the specified directory path is a back reference links folder.
+   *
+   * @param dirPath Directory path to verify
+   * @return True if the specified directory is a link references folder
+   */
+  public static boolean isBackReferencesDir(final Path dirPath) {
+    if (dirPath == null) return false;
+    return dirPath.getName().startsWith(BACK_REFERENCES_DIRECTORY_PREFIX);
+  }
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
new file mode 100644
index 000000000000..59bf57834048
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * HFileLink describes a link to an hfile.
+ *
+ * An hfile can be served from a region or from the hfile archive directory (/hbase/.archive)
+ * HFileLink allows to access the referenced hfile regardless of the location where it is.
+ *
+ * <p>Searches for hfiles in the following order and locations:
+ * <ul>
+ *  <li>/hbase/table/region/cf/hfile</li>
+ *  <li>/hbase/.archive/table/region/cf/hfile</li>
+ * </ul>
+ *
+ * The link checks first in the original path if it is not present
+ * it fallbacks to the archived path.
+ */
+@InterfaceAudience.Private
+public class HFileLink extends FileLink {
+  private static final Log LOG = LogFactory.getLog(HFileLink.class);
+
+  /**
+   * A non-capture group, for HFileLink, so that this can be embedded.
+   * The HFileLink describe a link to an hfile in a different table/region
+   * and the name is in the form: table=region-hfile.
+   * <p>
+   * Table name is ([a-zA-Z_0-9][a-zA-Z_0-9.-]*), so '=' is an invalid character for the table name.
+   * Region name is ([a-f0-9]+), so '-' is an invalid character for the region name.
+   * HFile is ([0-9a-f]+(?:_SeqId_[0-9]+_)?) covering the plain hfiles (uuid)
+   * and the bulk loaded (_SeqId_[0-9]+_) hfiles.
+   */
+  public static final String LINK_NAME_REGEX =
+    String.format("%s=%s-%s", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX);
+
+  /** Define the HFile Link name parser in the form of: table=region-hfile */
+  private static final Pattern LINK_NAME_PATTERN =
+    Pattern.compile(String.format("^(%s)=(%s)-(%s)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX));
+
+  /**
+   * The pattern should be used for hfile and reference links
+   * that can be found in /hbase/table/region/family/
+   */
+  private static final Pattern REF_OR_HFILE_LINK_PATTERN =
+    Pattern.compile(String.format("^(%s)=(%s)-(.+)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX));
+
+  private final Path archivePath;
+  private final Path originPath;
+  private final Path tempPath;
+
+  /**
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param path The path of the HFile Link.
+   * @throws IOException on unexpected error.
+   */
+  public HFileLink(Configuration conf, Path path) throws IOException {
+    this(FSUtils.getRootDir(conf), HFileArchiveUtil.getArchivePath(conf), path);
+  }
+
+  /**
+   * @param rootDir Path to the root directory where hbase files are stored
+   * @param archiveDir Path to the hbase archive directory
+   * @param path The path of the HFile Link.
+   */
+  public HFileLink(final Path rootDir, final Path archiveDir, final Path path) {
+    Path hfilePath = getRelativeTablePath(path);
+    this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
+    this.originPath = new Path(rootDir, hfilePath);
+    this.archivePath = new Path(archiveDir, hfilePath);
+    setLocations(originPath, tempPath, archivePath);
+  }
+
+  /**
+   * @return the origin path of the hfile.
+   */
+  public Path getOriginPath() {
+    return this.originPath;
+  }
+
+  /**
+   * @return the path of the archived hfile.
+   */
+  public Path getArchivePath() {
+    return this.archivePath;
+  }
+
+  /**
+   * @param path Path to check.
+   * @return True if the path is a HFileLink.
+   */
+  public static boolean isHFileLink(final Path path) {
+    return isHFileLink(path.getName());
+  }
+
+
+  /**
+   * @param fileName File name to check.
+   * @return True if the path is a HFileLink.
+   */
+  public static boolean isHFileLink(String fileName) {
+    Matcher m = LINK_NAME_PATTERN.matcher(fileName);
+    if (!m.matches()) return false;
+
+    return m.groupCount() > 2 && m.group(3) != null && m.group(2) != null && m.group(1) != null;
+  }
+
+  /**
+   * Convert a HFileLink path to a table relative path.
+   * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd
+   *      becomes: /hbase/testtb/4567/cf/abcd
+   *
+   * @param path HFileLink path
+   * @return Relative table path
+   * @throws IOException on unexpected error.
+   */
+  private static Path getRelativeTablePath(final Path path) {
+    // table=region-hfile
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName());
+    if (!m.matches()) {
+      throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink name!");
+    }
+
+    // Convert the HFileLink name into a real table/region/cf/hfile path.
+    String tableName = m.group(1);
+    String regionName = m.group(2);
+    String hfileName = m.group(3);
+    String familyName = path.getParent().getName();
+    return new Path(new Path(tableName, regionName), new Path(familyName, hfileName));
+  }
+
+  /**
+   * Get the HFile name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced HFile
+   */
+  public static String getReferencedHFileName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(3));
+  }
+
+  /**
+   * Get the Region name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced Region
+   */
+  public static String getReferencedRegionName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(2));
+  }
+
+  /**
+   * Get the Table name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced Table
+   */
+  public static String getReferencedTableName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(1));
+  }
+
+  /**
+   * Create a new HFileLink name
+   *
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @return file name of the HFile Link
+   */
+  public static String createHFileLinkName(final HRegionInfo hfileRegionInfo,
+      final String hfileName) {
+    return createHFileLinkName(hfileRegionInfo.getTableNameAsString(),
+                      hfileRegionInfo.getEncodedName(), hfileName);
+  }
+
+  /**
+   * Create a new HFileLink name
+   *
+   * @param tableName - Linked HFile table name
+   * @param regionName - Linked HFile region name
+   * @param hfileName - Linked HFile name
+   * @return file name of the HFile Link
+   */
+  public static String createHFileLinkName(final String tableName,
+      final String regionName, final String hfileName) {
+    return String.format("%s=%s-%s", tableName, regionName, hfileName);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
+      final String hfileName) throws IOException {
+    String linkedTable = hfileRegionInfo.getTableNameAsString();
+    String linkedRegion = hfileRegionInfo.getEncodedName();
+    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param linkedTable - Linked Table Name
+   * @param linkedRegion - Linked Region Name
+   * @param hfileName - Linked HFile name
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String linkedTable, final String linkedRegion,
+      final String hfileName) throws IOException {
+    String familyName = dstFamilyPath.getName();
+    String regionName = dstFamilyPath.getParent().getName();
+    String tableName = dstFamilyPath.getParent().getParent().getName();
+
+    String name = createHFileLinkName(linkedTable, linkedRegion, hfileName);
+    String refName = createBackReferenceName(tableName, regionName);
+
+    // Make sure the destination directory exists
+    fs.mkdirs(dstFamilyPath);
+
+    // Make sure the FileLink reference directory exists
+    Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          linkedTable, linkedRegion, familyName);
+    Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
+    fs.mkdirs(backRefssDir);
+
+    // Create the reference for the link
+    Path backRefPath = new Path(backRefssDir, refName);
+    fs.createNewFile(backRefPath);
+    try {
+      // Create the link
+      return fs.createNewFile(new Path(dstFamilyPath, name));
+    } catch (IOException e) {
+      LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
+      // Revert the reference if the link creation failed
+      fs.delete(backRefPath, false);
+      throw e;
+    }
+  }
+
+  /**
+   * Create a new HFileLink starting from a hfileLink name
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String hfileLinkName) throws IOException {
+    Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
+    }
+    return create(conf, fs, dstFamilyPath, m.group(1), m.group(2), m.group(3));
+  }
+
+  /**
+   * Create the back reference name
+   */
+  private static String createBackReferenceName(final String tableName, final String regionName) {
+    return regionName + "." + tableName;
+  }
+
+  /**
+   * Get the full path of the HFile referenced by the back reference
+   *
+   * @param rootDir root hbase directory
+   * @param linkRefPath Link Back Reference path
+   * @return full path of the referenced hfile
+   * @throws IOException on unexpected error.
+   */
+  public static Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath) {
+    int separatorIndex = linkRefPath.getName().indexOf('.');
+    String linkRegionName = linkRefPath.getName().substring(0, separatorIndex);
+    String linkTableName = linkRefPath.getName().substring(separatorIndex + 1);
+    String hfileName = getBackReferenceFileName(linkRefPath.getParent());
+    Path familyPath = linkRefPath.getParent().getParent();
+    Path regionPath = familyPath.getParent();
+    Path tablePath = regionPath.getParent();
+
+    String linkName = createHFileLinkName(tablePath.getName(), regionPath.getName(), hfileName);
+    Path linkTableDir = FSUtils.getTablePath(rootDir, linkTableName);
+    Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName);
+    return new Path(new Path(regionDir, familyPath.getName()), linkName);
+  }
+
+  /**
+   * Get the full path of the HFile referenced by the back reference
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param linkRefPath Link Back Reference path
+   * @return full path of the referenced hfile
+   * @throws IOException on unexpected error.
+   */
+  public static Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath)
+      throws IOException {
+    return getHFileFromBackReference(FSUtils.getRootDir(conf), linkRefPath);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java b/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java
new file mode 100644
index 000000000000..8feb8cbb0d94
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * HLogLink describes a link to a WAL.
+ *
+ * An hlog can be in /hbase/.logs/<server>/<hlog>
+ * or it can be in /hbase/.oldlogs/<hlog>
+ *
+ * The link checks first in the original path,
+ * if it is not present it fallbacks to the archived path.
+ */
+@InterfaceAudience.Private
+public class HLogLink extends FileLink {
+  /**
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param serverName Region Server owner of the log
+   * @param logName WAL file name
+   * @throws IOException on unexpected error.
+   */
+  public HLogLink(final Configuration conf,
+      final String serverName, final String logName) throws IOException {
+    this(FSUtils.getRootDir(conf), serverName, logName);
+  }
+
+  /**
+   * @param rootDir Path to the root directory where hbase files are stored
+   * @param serverName Region Server owner of the log
+   * @param logName WAL file name
+   */
+  public HLogLink(final Path rootDir, final String serverName, final String logName) {
+    final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    final Path logDir = new Path(new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), serverName);
+    setLocations(new Path(logDir, logName), new Path(oldLogDir, logName));
+  }
+
+  /**
+   * @param originPath Path to the wal in the log directory
+   * @param archivePath Path to the wal in the archived log directory
+   */
+  public HLogLink(final Path originPath, final Path archivePath) {
+    setLocations(originPath, archivePath);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index da8f3f088498..d3a82eaa427e 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -58,10 +58,12 @@ public class HalfStoreFileReader extends StoreFile.Reader {
   private boolean firstKeySeeked = false;
 
   /**
-   * @param fs
-   * @param p
+   * Creates a half file reader for a normal hfile.
+   * @param fs fileystem to read from
+   * @param p path to hfile
    * @param cacheConf
-   * @param r
+   * @param r original reference file (contains top or bottom)
+   * @param preferredEncodingInCache
    * @throws IOException
    */
   public HalfStoreFileReader(final FileSystem fs, final Path p,
@@ -78,6 +80,30 @@ public HalfStoreFileReader(final FileSystem fs, final Path p,
     this.top = Reference.isTopFileRegion(r.getFileRegion());
   }
 
+  /**
+   * Creates a half file reader for a hfile referred to by an hfilelink.
+   * @param fs fileystem to read from
+   * @param p path to hfile
+   * @param link
+   * @param cacheConf
+   * @param r original reference file (contains top or bottom)
+   * @param preferredEncodingInCache
+   * @throws IOException
+   */
+  public HalfStoreFileReader(final FileSystem fs, final Path p, final HFileLink link,
+      final CacheConfig cacheConf, final Reference r,
+      DataBlockEncoding preferredEncodingInCache) throws IOException {
+    super(fs, p, link, link.getFileStatus(fs).getLen(), cacheConf, preferredEncodingInCache, true);
+    // This is not actual midkey for this half-file; its just border
+    // around which we split top and bottom.  Have to look in files to find
+    // actual last and first keys for bottom and top halves.  Half-files don't
+    // have an actual midkey themselves. No midkey is how we indicate file is
+    // not splittable.
+    this.splitkey = r.getSplitKey();
+    // Is it top or bottom half?
+    this.top = Reference.isTopFileRegion(r.getFileRegion());
+  }
+
   protected boolean isTop() {
     return this.top;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
index 98a65959288f..317601e36313 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -90,6 +90,7 @@
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ProtoUtil;
 import org.apache.hadoop.io.MapWritable;
@@ -269,6 +270,10 @@ public class HbaseObjectWritable implements Writable, WritableWithSize, Configur
     
     addToMap(FuzzyRowFilter.class, code++);
 
+    // we aren't going to bump the rpc version number.
+    // we don't want to cause incompatiblity with older 0.94/0.92 clients.
+    addToMap(HSnapshotDescription.class, code);
+
     // make sure that this is the last statement in this static block
     NEXT_CLASS_CODE = code;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index c46fcb215b2c..c99caba35a0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -180,18 +180,18 @@ public class HFile {
   static final AtomicLong checksumFailures = new AtomicLong();
 
   // For getting more detailed stats on FS latencies
-  // If, for some reason, the metrics subsystem stops polling for latencies, 
+  // If, for some reason, the metrics subsystem stops polling for latencies,
   // I don't want data to pile up in a memory leak
   // so, after LATENCY_BUFFER_SIZE items have been enqueued for processing,
   // fs latency stats will be dropped (and this behavior will be logged)
   private static final int LATENCY_BUFFER_SIZE = 5000;
-  private static final BlockingQueue<Long> fsReadLatenciesNanos = 
+  private static final BlockingQueue<Long> fsReadLatenciesNanos =
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  private static final BlockingQueue<Long> fsWriteLatenciesNanos = 
+  private static final BlockingQueue<Long> fsWriteLatenciesNanos =
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  private static final BlockingQueue<Long> fsPreadLatenciesNanos = 
+  private static final BlockingQueue<Long> fsPreadLatenciesNanos =
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  
+
   public static final void offerReadLatency(long latencyNanos, boolean pread) {
     if (pread) {
       fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
@@ -203,30 +203,30 @@ public static final void offerReadLatency(long latencyNanos, boolean pread) {
       readOps.incrementAndGet();
     }
   }
-  
+
   public static final void offerWriteLatency(long latencyNanos) {
     fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
-    
+
     writeTimeNano.addAndGet(latencyNanos);
     writeOps.incrementAndGet();
   }
-  
+
   public static final Collection<Long> getReadLatenciesNanos() {
-    final List<Long> latencies = 
+    final List<Long> latencies =
         Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size());
     fsReadLatenciesNanos.drainTo(latencies);
     return latencies;
   }
 
   public static final Collection<Long> getPreadLatenciesNanos() {
-    final List<Long> latencies = 
+    final List<Long> latencies =
         Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size());
     fsPreadLatenciesNanos.drainTo(latencies);
     return latencies;
   }
-  
+
   public static final Collection<Long> getWriteLatenciesNanos() {
-    final List<Long> latencies = 
+    final List<Long> latencies =
         Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size());
     fsWriteLatenciesNanos.drainTo(latencies);
     return latencies;
@@ -572,7 +572,7 @@ public static Reader createReaderWithEncoding(
     HFileSystem hfs = null;
     FSDataInputStream fsdis = fs.open(path);
     FSDataInputStream fsdisNoFsChecksum = fsdis;
-    // If the fs is not an instance of HFileSystem, then create an 
+    // If the fs is not an instance of HFileSystem, then create an
     // instance of HFileSystem that wraps over the specified fs.
     // In this case, we will not be able to avoid checksumming inside
     // the filesystem.
@@ -591,6 +591,39 @@ public static Reader createReaderWithEncoding(
         preferredEncodingInCache, hfs);
   }
 
+  /**
+   * @param fs A file system
+   * @param path Path to HFile
+   * @param fsdis an open checksummed stream of path's file
+   * @param fsdisNoFsChecksum an open unchecksummed stream of path's file
+   * @param size max size of the trailer.
+   * @param cacheConf Cache configuration for hfile's contents
+   * @param preferredEncodingInCache Preferred in-cache data encoding algorithm.
+   * @param closeIStream boolean for closing file after the getting the reader version.
+   * @return A version specific Hfile Reader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
+  public static Reader createReaderWithEncoding(
+      FileSystem fs, Path path, FSDataInputStream fsdis,
+      FSDataInputStream fsdisNoFsChecksum, long size, CacheConfig cacheConf,
+      DataBlockEncoding preferredEncodingInCache, boolean closeIStream)
+      throws IOException {
+    HFileSystem hfs = null;
+
+    // If the fs is not an instance of HFileSystem, then create an
+    // instance of HFileSystem that wraps over the specified fs.
+    // In this case, we will not be able to avoid checksumming inside
+    // the filesystem.
+    if (!(fs instanceof HFileSystem)) {
+      hfs = new HFileSystem(fs);
+    } else {
+      hfs = (HFileSystem)fs;
+    }
+    return pickReaderVersion(path, fsdis, fsdisNoFsChecksum, size,
+                             closeIStream, cacheConf,
+                             preferredEncodingInCache, hfs);
+  }
+
   /**
    * @param fs filesystem
    * @param path Path to file to read
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 5f6f652eec14..3812e9cafca0 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -581,15 +581,21 @@ public long getEvictedCount() {
     return this.stats.getEvictedCount();
   }
 
+  EvictionThread getEvictionThread() {
+    return this.evictionThread;
+  }
+
   /*
    * Eviction thread.  Sits in waiting state until an eviction is triggered
    * when the cache size grows above the acceptable level.<p>
    *
    * Thread is triggered into action by {@link LruBlockCache#runEviction()}
    */
-  private static class EvictionThread extends HasThread {
+  static class EvictionThread extends HasThread {
     private WeakReference<LruBlockCache> cache;
     private boolean go = true;
+    // flag set after enter the run method, used for test
+    private boolean enteringRun = false;
 
     public EvictionThread(LruBlockCache cache) {
       super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
@@ -599,6 +605,7 @@ public EvictionThread(LruBlockCache cache) {
 
     @Override
     public void run() {
+      enteringRun = true;
       while (this.go) {
         synchronized(this) {
           try {
@@ -621,6 +628,13 @@ void shutdown() {
       this.go = false;
       interrupt();
     }
+
+    /**
+     * Used for the test.
+     */
+    boolean isEnteringRun() {
+      return this.enteringRun;
+    }
   }
 
   /*
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index c17da9754ab4..5cd7a69eac82 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.security.TokenInfo;
 import org.apache.hadoop.hbase.security.KerberosInfo;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -286,4 +287,22 @@ public void unassign(final byte [] regionName, final boolean force)
    */
   public ExecResult execCoprocessor(Exec call)
       throws IOException;
+
+  public long snapshot(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public List<HSnapshotDescription> getCompletedSnapshots()
+    throws IOException;
+
+  public void deleteSnapshot(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public boolean isSnapshotDone(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public void restoreSnapshot(final HSnapshotDescription request)
+    throws IOException;
+
+  public boolean isRestoreSnapshotDone(final HSnapshotDescription request)
+    throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 035775961ef0..ff509b85dabb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -44,6 +44,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ClusterStatus;
@@ -92,13 +93,18 @@
 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.util.InfoServer;
@@ -225,6 +231,9 @@ public class HMaster extends HasThread
   private long masterStartTime;
   private long masterActiveTime;
 
+  // monitor for snapshot of hbase tables
+  private SnapshotManager snapshotManager;
+
   /**
    * MX Bean for MasterInfo
    */
@@ -406,6 +415,7 @@ public void run() {
       if (this.serverManager != null) this.serverManager.stop();
       if (this.assignmentManager != null) this.assignmentManager.stop();
       if (this.fileSystemManager != null) this.fileSystemManager.stop();
+      if (this.snapshotManager != null) this.snapshotManager.stop("server shutting down.");
       this.zooKeeper.close();
     }
     LOG.info("HMaster main thread exiting");
@@ -467,6 +477,9 @@ private void initializeZKBasedSystemTrackers() throws IOException,
         ", sessionid=0x" +
         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
         ", cluster-up flag was=" + wasUp);
+
+    // create the snapshot manager
+    this.snapshotManager = new SnapshotManager(this);
   }
 
   // Check if we should stop every second.
@@ -1989,4 +2002,125 @@ private boolean isHealthCheckerConfigured() {
     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
   }
+
+  /**
+   * Exposed for TESTING!
+   * @return the underlying snapshot manager
+   */
+  public SnapshotManager getSnapshotManagerForTesting() {
+    return this.snapshotManager;
+   }
+
+
+  /**
+   * Triggers an asynchronous attempt to take a snapshot.
+   * {@inheritDoc}
+   */
+  @Override
+  public long snapshot(final HSnapshotDescription request) throws IOException {
+    LOG.debug("Submitting snapshot request for:" +
+        SnapshotDescriptionUtils.toString(request.getProto()));
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    // get the snapshot information
+    SnapshotDescription snapshot = SnapshotDescriptionUtils.validate(request.getProto(),
+      this.conf);
+
+    snapshotManager.takeSnapshot(snapshot);
+
+    // send back the max amount of time the client should wait for the snapshot to complete
+    long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
+      SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
+    return waitTime;
+  }
+
+  /**
+   * List the currently available/stored snapshots. Any in-progress snapshots are ignored
+   */
+  @Override
+  public List<HSnapshotDescription> getCompletedSnapshots() throws IOException {
+    List<HSnapshotDescription> availableSnapshots = new ArrayList<HSnapshotDescription>();
+    List<SnapshotDescription> snapshots = snapshotManager.getCompletedSnapshots();
+
+    // convert to writables
+    for (SnapshotDescription snapshot: snapshots) {
+      availableSnapshots.add(new HSnapshotDescription(snapshot));
+    }
+
+    return availableSnapshots;
+  }
+
+  /**
+   * Execute Delete Snapshot operation.
+   * @throws ServiceException wrapping SnapshotDoesNotExistException if specified snapshot did not
+   * exist.
+   */
+  @Override
+  public void deleteSnapshot(final HSnapshotDescription request) throws IOException {
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    snapshotManager.deleteSnapshot(request.getProto());
+  }
+
+  /**
+   * Checks if the specified snapshot is done.
+   * @return true if the snapshot is in file system ready to use,
+   * false if the snapshot is in the process of completing
+   * @throws ServiceException wrapping UnknownSnapshotException if invalid snapshot, or
+   * a wrapped HBaseSnapshotException with progress failure reason.
+   */
+  @Override
+  public boolean isSnapshotDone(final HSnapshotDescription request) throws IOException {
+    LOG.debug("Checking to see if snapshot from request:" +
+      SnapshotDescriptionUtils.toString(request.getProto()) + " is done");
+    return snapshotManager.isSnapshotDone(request.getProto());
+  }
+
+  /**
+   * Execute Restore/Clone snapshot operation.
+   *
+   * <p>If the specified table exists a "Restore" is executed, replacing the table
+   * schema and directory data with the content of the snapshot.
+   * The table must be disabled, or a UnsupportedOperationException will be thrown.
+   *
+   * <p>If the table doesn't exist a "Clone" is executed, a new table is created
+   * using the schema at the time of the snapshot, and the content of the snapshot.
+   *
+   * <p>The restore/clone operation does not require copying HFiles. Since HFiles
+   * are immutable the table can point to and use the same files as the original one.
+   */
+  @Override
+  public void restoreSnapshot(final HSnapshotDescription request) throws IOException {
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    snapshotManager.restoreSnapshot(request.getProto());
+  }
+
+  /**
+   * Returns the status of the requested snapshot restore/clone operation.
+   * This method is not exposed to the user, it is just used internally by HBaseAdmin
+   * to verify if the restore is completed.
+   *
+   * No exceptions are thrown if the restore is not running, the result will be "done".
+   *
+   * @return done <tt>true</tt> if the restore/clone operation is completed.
+   * @throws RestoreSnapshotExcepton if the operation failed.
+   */
+  @Override
+  public boolean isRestoreSnapshotDone(final HSnapshotDescription request) throws IOException {
+    return !snapshotManager.isRestoringTable(request.getProto());
+  }
 }
+
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 5f21a9c606b1..a158085acf41 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.coprocessor.*;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
 
@@ -629,4 +630,146 @@ void postStartMaster() throws IOException {
       }
     }
   }
+
+  public void preSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preCloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preCloneSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postCloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postCloneSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preRestoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preRestoreSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postRestoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postRestoreSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preDeleteSnapshot(ctx, snapshot);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postDeleteSnapshot(ctx, snapshot);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 5c7b8aa1e778..5ed1559f8fd5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -76,6 +76,8 @@ public class MasterFileSystem {
   private final Path oldLogDir;
   // root hbase directory on the FS
   private final Path rootdir;
+  // hbase temp directory used for table construction and deletion
+  private final Path tempdir;
   // create the split log lock
   final Lock splitLogLock = new ReentrantLock();
   final boolean distributedLogSplitting;
@@ -94,6 +96,7 @@ public MasterFileSystem(Server master, MasterServices services,
     // default localfs.  Presumption is that rootdir is fully-qualified before
     // we get to here with appropriate fs scheme.
     this.rootdir = FSUtils.getRootDir(conf);
+    this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
     // Cover both bases, the old way of setting default fs and the new.
     // We're supposed to run on 0.20 and 0.21 anyways.
     this.fs = this.rootdir.getFileSystem(conf);
@@ -130,6 +133,9 @@ private Path createInitialFileSystemLayout() throws IOException {
     // check if the root directory exists
     checkRootDir(this.rootdir, conf, this.fs);
 
+    // check if temp directory exists and clean it
+    checkTempDir(this.tempdir, conf, this.fs);
+
     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
 
     // Make sure the region servers can archive their old logs
@@ -177,6 +183,13 @@ public Path getRootDir() {
     return this.rootdir;
   }
 
+  /**
+   * @return HBase temp dir.
+   */
+  public Path getTempDir() {
+    return this.tempdir;
+  }
+
   /**
    * @return The unique identifier generated for this cluster
    */
@@ -385,6 +398,32 @@ private void createRootTableInfo(Path rd) throws IOException {
     }
   }
 
+  /**
+   * Make sure the hbase temp directory exists and is empty.
+   * NOTE that this method is only executed once just after the master becomes the active one.
+   */
+  private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
+      throws IOException {
+    // If the temp directory exists, clear the content (left over, from the previous run)
+    if (fs.exists(tmpdir)) {
+      // Archive table in temp, maybe left over from failed deletion,
+      // if not the cleaner will take care of them.
+      for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
+        for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
+          HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
+        }
+      }
+      if (!fs.delete(tmpdir, true)) {
+        throw new IOException("Unable to clean the temp directory: " + tmpdir);
+      }
+    }
+
+    // Create the temp directory
+    if (!fs.mkdirs(tmpdir)) {
+      throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
+    }
+  }
+
   private static void bootstrap(final Path rd, final Configuration c)
   throws IOException {
     LOG.info("BOOTSTRAP: creating ROOT and first META regions");
@@ -451,6 +490,37 @@ public void deleteTable(byte[] tableName) throws IOException {
     fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
   }
 
+  /**
+   * Move the specified file/directory to the hbase temp directory.
+   * @param path The path of the file/directory to move
+   * @return The temp location of the file/directory moved
+   * @throws IOException in case of file-system failure
+   */
+  public Path moveToTemp(final Path path) throws IOException {
+    Path tempPath = new Path(this.tempdir, path.getName());
+
+    // Ensure temp exists
+    if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
+      throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
+    }
+
+    if (!fs.rename(path, tempPath)) {
+      throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
+    }
+
+    return tempPath;
+  }
+
+  /**
+   * Move the specified table to the hbase temp directory
+   * @param tableName Table name to move
+   * @return The temp location of the table moved
+   * @throws IOException in case of file-system failure
+   */
+  public Path moveTableToTemp(byte[] tableName) throws IOException {
+    return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
+  }
+
   public void updateRegionInfo(HRegionInfo region) {
     // TODO implement this.  i think this is currently broken in trunk i don't
     //      see this getting updated.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java b/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
new file mode 100644
index 000000000000..bb2d7169c789
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * Watch the current snapshot under process
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface SnapshotSentinel {
+
+  /**
+   * Check to see if the snapshot is finished, where finished may be success or failure.
+   * @return <tt>false</tt> if the snapshot is still in progress, <tt>true</tt> if the snapshot has
+   *         finished
+   */
+  public boolean isFinished();
+
+  /**
+   * Actively cancel a running snapshot.
+   * @param why Reason for cancellation.
+   */
+  public void cancel(String why);
+
+  /**
+   * @return the description of the snapshot being run
+   */
+  public SnapshotDescription getSnapshot();
+
+  /**
+   * Get the exception that caused the snapshot to fail, if the snapshot has failed.
+   * @return {@link ForeignException} that caused the snapshot to fail, or <tt>null</tt> if the
+   *  snapshot is still in progress or has succeeded
+   */
+  public ForeignException getExceptionIfFailed();
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index b815b3b985f8..e58634251c5a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -82,7 +82,10 @@ private void initCleanerChain(String confKey) {
     if (logCleaners != null) {
       for (String className : logCleaners) {
         T logCleaner = newFileCleaner(className, conf);
-        if (logCleaner != null) this.cleanersChain.add(logCleaner);
+        if (logCleaner != null) {
+          LOG.debug("initialize cleaner=" + className);
+          this.cleanersChain.add(logCleaner);
+        }
       }
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 06a612d5ba52..6773fbd98efb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 /**
  * This Chore, every time it runs, will clear the HFiles in the hfile archive
@@ -46,6 +47,9 @@ public HFileCleaner(final int period, final Stoppable stopper, Configuration con
 
   @Override
   protected boolean validate(Path file) {
+    if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) {
+      return true;
+    }
     return StoreFile.validateStoreFileName(file.getName());
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
new file mode 100644
index 000000000000..4da1b9b6fe72
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+
+/**
+ * HFileLink cleaner that determines if a hfile should be deleted.
+ * HFiles can be deleted only if there're no links to them.
+ *
+ * When a HFileLink is created a back reference file is created in:
+ *      /hbase/archive/table/region/cf/.links-hfile/ref-region.ref-table
+ * To check if the hfile can be deleted the back references folder must be empty.
+ */
+@InterfaceAudience.Private
+public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(HFileLinkCleaner.class);
+
+  private FileSystem fs = null;
+
+  @Override
+  public synchronized boolean isFileDeletable(Path filePath) {
+    if (this.fs == null) return false;
+
+    // HFile Link is always deletable
+    if (HFileLink.isHFileLink(filePath)) return true;
+
+    // If the file is inside a link references directory, means that is a back ref link.
+    // The back ref can be deleted only if the referenced file doesn't exists.
+    Path parentDir = filePath.getParent();
+    if (HFileLink.isBackReferencesDir(parentDir)) {
+      try {
+        Path hfilePath = HFileLink.getHFileFromBackReference(getConf(), filePath);
+        return !fs.exists(hfilePath);
+      } catch (IOException e) {
+        LOG.error("Couldn't verify if the referenced file still exists, keep it just in case");
+        return false;
+      }
+    }
+
+    // HFile is deletable only if has no links
+    try {
+      Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
+      return FSUtils.listStatus(fs, backRefDir) == null;
+    } catch (IOException e) {
+      LOG.error("Couldn't get the references, not deleting file, just in case");
+      return false;
+    }
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+
+    // setup filesystem
+    try {
+      this.fs = FileSystem.get(this.getConf());
+    } catch (IOException e) {
+      LOG.error("Couldn't instantiate the file system, not deleting file, just in case");
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index af25defc0d1f..717880de4871 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -20,13 +19,25 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
@@ -41,21 +52,23 @@
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.zookeeper.KeeperException;
 
 /**
  * Handler to create a table.
  */
+@InterfaceAudience.Private
 public class CreateTableHandler extends EventHandler {
   private static final Log LOG = LogFactory.getLog(CreateTableHandler.class);
-  private MasterFileSystem fileSystemManager;
-  private final HTableDescriptor hTableDescriptor;
-  private Configuration conf;
-  private final AssignmentManager assignmentManager;
-  private final CatalogTracker catalogTracker;
-  private final ServerManager serverManager;
+  protected MasterFileSystem fileSystemManager;
+  protected final HTableDescriptor hTableDescriptor;
+  protected Configuration conf;
+  protected final AssignmentManager assignmentManager;
+  protected final CatalogTracker catalogTracker;
+  protected final ServerManager serverManager;
   private final HRegionInfo [] newRegions;
 
   public CreateTableHandler(Server server, MasterFileSystem fileSystemManager,
@@ -98,8 +111,7 @@ public CreateTableHandler(Server server, MasterFileSystem fileSystemManager,
     // table in progress. This will introduce a new zookeeper call. Given
     // createTable isn't a frequent operation, that should be ok.
     try {
-      if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(
-        tableName))
+      if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(tableName))
         throw new TableExistsException(tableName);
     } catch (KeeperException e) {
       throw new IOException("Unable to ensure that the table will be" +
@@ -122,66 +134,91 @@ public String toString() {
   public void process() {
     String tableName = this.hTableDescriptor.getNameAsString();
     try {
-      LOG.info("Attemping to create the table " + tableName);
-      handleCreateTable();
-    } catch (IOException e) {
-      LOG.error("Error trying to create the table " + tableName, e);
-    } catch (KeeperException e) {
+      LOG.info("Attempting to create the table " + tableName);
+      handleCreateTable(tableName);
+      completed(null);
+    } catch (Throwable e) {
       LOG.error("Error trying to create the table " + tableName, e);
+      completed(e);
     }
   }
 
-  private void handleCreateTable() throws IOException, KeeperException {
-
-    // TODO: Currently we make the table descriptor and as side-effect the
-    // tableDir is created.  Should we change below method to be createTable
-    // where we create table in tmp dir with its table descriptor file and then
-    // do rename to move it into place?
-    FSTableDescriptors.createTableDescriptor(this.hTableDescriptor, this.conf);
-
-    List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
-    final int batchSize =
-      this.conf.getInt("hbase.master.createtable.batchsize", 100);
-    for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
-      HRegionInfo newRegion = this.newRegions[regionIdx];
-      // 1. Create HRegion
-      HRegion region = HRegion.createHRegion(newRegion,
-        this.fileSystemManager.getRootDir(), this.conf,
-        this.hTableDescriptor, null, false, true);
-
-      regionInfos.add(region.getRegionInfo());
-      if (regionIdx % batchSize == 0) {
-        // 2. Insert into META
-        MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
-        regionInfos.clear();
-      }
+  /**
+   * Called after that process() is completed.
+   * @param exception null if process() is successful or not null if something has failed.
+   */
+  protected void completed(final Throwable exception) {
+  }
 
-      // 3. Close the new region to flush to disk.  Close log file too.
-      region.close();
+  /**
+   * Responsible of table creation (on-disk and META) and assignment.
+   * - Create the table directory and descriptor (temp folder)
+   * - Create the on-disk regions (temp folder)
+   *   [If something fails here: we've just some trash in temp]
+   * - Move the table from temp to the root directory
+   *   [If something fails here: we've the table in place but some of the rows required
+   *    present in META. (hbck needed)]
+   * - Add regions to META
+   *   [If something fails here: we don't have regions assigned: table disabled]
+   * - Assign regions to Region Servers
+   *   [If something fails here: we still have the table in disabled state]
+   * - Update ZooKeeper with the enabled state
+   */
+  private void handleCreateTable(String tableName) throws IOException, KeeperException {
+    Path tempdir = fileSystemManager.getTempDir();
+    FileSystem fs = fileSystemManager.getFileSystem();
+
+    // 1. Create Table Descriptor
+    FSTableDescriptors.createTableDescriptor(fs, tempdir, this.hTableDescriptor);
+    Path tempTableDir = new Path(tempdir, tableName);
+    Path tableDir = new Path(fileSystemManager.getRootDir(), tableName);
+
+    // 2. Create Regions
+    List<HRegionInfo> regionInfos = handleCreateHdfsRegions(tempdir, tableName);
+
+    // 3. Move Table temp directory to the hbase root location
+    if (!fs.rename(tempTableDir, tableDir)) {
+      throw new IOException("Unable to move table from temp=" + tempTableDir +
+        " to hbase root=" + tableDir);
     }
-    if (regionInfos.size() > 0) {
+
+    if (regionInfos != null && regionInfos.size() > 0) {
+      // 4. Add regions to META
       MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
-    }
 
-    // 4. Trigger immediate assignment of the regions in round-robin fashion
-    List<ServerName> servers = serverManager.getOnlineServersList();
-    // Remove the deadNotExpired servers from the server list.
-    assignmentManager.removeDeadNotExpiredServers(servers);
-    try {
-      this.assignmentManager.assignUserRegions(Arrays.asList(newRegions),
-        servers);
-    } catch (InterruptedException ie) {
-      LOG.error("Caught " + ie + " during round-robin assignment");
-      throw new IOException(ie);
+      // 5. Trigger immediate assignment of the regions in round-robin fashion
+      List<ServerName> servers = serverManager.getOnlineServersList();
+      // Remove the deadNotExpired servers from the server list.
+      assignmentManager.removeDeadNotExpiredServers(servers);
+      try {
+        this.assignmentManager.assignUserRegions(regionInfos, servers);
+      } catch (InterruptedException e) {
+        LOG.error("Caught " + e + " during round-robin assignment");
+        InterruptedIOException ie = new InterruptedIOException(e.getMessage());
+        ie.initCause(e);
+        throw ie;
+      }
     }
 
-    // 5. Set table enabled flag up in zk.
+    // 6. Set table enabled flag up in zk.
     try {
-      assignmentManager.getZKTable().
-        setEnabledTable(this.hTableDescriptor.getNameAsString());
+      assignmentManager.getZKTable().setEnabledTable(tableName);
     } catch (KeeperException e) {
-      throw new IOException("Unable to ensure that the table will be" +
+      throw new IOException("Unable to ensure that " + tableName + " will be" +
         " enabled because of a ZooKeeper issue", e);
     }
   }
-}
\ No newline at end of file
+
+  /**
+   * Create the on-disk structure for the table, and returns the regions info.
+   * @param tableRootDir directory where the table is being created
+   * @param tableName name of the table under construction
+   * @return the list of regions created
+   */
+  protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
+    final String tableName)
+      throws IOException {
+    return ModifyRegionUtils.createRegions(conf, tableRootDir,
+        hTableDescriptor, newRegions, null);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index b299bee4a8fd..2781671b826b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -24,10 +24,14 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
@@ -47,6 +51,7 @@ public DeleteTableHandler(byte [] tableName, Server server,
   @Override
   protected void handleTableOperation(List<HRegionInfo> regions)
   throws IOException, KeeperException {
+    // 1. Wait because of region in transition
     AssignmentManager am = this.masterServices.getAssignmentManager();
     long waitTime = server.getConfiguration().
       getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
@@ -63,23 +68,39 @@ protected void handleTableOperation(List<HRegionInfo> regions)
           waitTime + "ms) for region to leave region " +
           region.getRegionNameAsString() + " in transitions");
       }
-      LOG.debug("Deleting region " + region.getRegionNameAsString() +
-        " from META and FS");
-      // Remove region from META
-      MetaEditor.deleteRegion(this.server.getCatalogTracker(), region);
-      // Delete region from FS
-      this.masterServices.getMasterFileSystem().deleteRegion(region);
     }
-    // Delete table from FS
-    this.masterServices.getMasterFileSystem().deleteTable(tableName);
-    // Update table descriptor cache
-    this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
 
-    // If entry for this table in zk, and up in AssignmentManager, remove it.
+    // 2. Remove regions from META
+    LOG.debug("Deleting regions from META");
+    MetaEditor.deleteRegions(this.server.getCatalogTracker(), regions);
 
-    am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+    // 3. Move the table in /hbase/.tmp
+    LOG.debug("Moving table directory to a temp directory");
+    MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+    Path tempTableDir = mfs.moveTableToTemp(tableName);
+
+    try {
+      // 4. Delete regions from FS (temp directory)
+      FileSystem fs = mfs.getFileSystem();
+      for (HRegionInfo hri: regions) {
+        LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
+        HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+            tempTableDir, new Path(tempTableDir, hri.getEncodedName()));
+      }
+
+      // 5. Delete table from FS (temp directory)
+      if (!fs.delete(tempTableDir, true)) {
+        LOG.error("Couldn't delete " + tempTableDir);
+      }
+    } finally {
+      // 6. Update table descriptor cache
+      this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
+
+      // 7. If entry for this table in zk, and up in AssignmentManager, remove it.
+      am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+    }
   }
-  
+
   @Override
   public String toString() {
     String name = "UnknownServerName";
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
index 5af0690207e2..fcbfb82dc69a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
@@ -170,6 +170,7 @@ protected boolean waitUntilDone(long timeout)
       while (!server.isStopped() && remaining > 0) {
         Thread.sleep(waitingTimeForEvents);
         regions = assignmentManager.getRegionsOfTable(tableName);
+        LOG.debug("Disable waiting until done; " + remaining + " ms remaining; " + regions);
         if (regions.isEmpty()) break;
         remaining = timeout - (System.currentTimeMillis() - startTime);
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
index 4fd784dd9aa9..aeaa4adf976c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -160,12 +160,14 @@ public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
   }
 
   /**
+   * Gets a TableDescriptor from the masterServices.  Can Throw exceptions.
+   *
    * @return Table descriptor for this table
    * @throws TableExistsException
    * @throws FileNotFoundException
    * @throws IOException
    */
-  HTableDescriptor getTableDescriptor()
+  public HTableDescriptor getTableDescriptor()
   throws FileNotFoundException, IOException {
     final String name = Bytes.toString(tableName);
     HTableDescriptor htd =
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
new file mode 100644
index 000000000000..58e6520b5e91
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -0,0 +1,150 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Handler to Clone a snapshot.
+ *
+ * <p>Uses {@link RestoreSnapshotHelper} to create a new table with the same
+ * content of the specified snapshot.
+ */
+@InterfaceAudience.Private
+public class CloneSnapshotHandler extends CreateTableHandler implements SnapshotSentinel {
+  private static final Log LOG = LogFactory.getLog(CloneSnapshotHandler.class);
+
+  private final static String NAME = "Master CloneSnapshotHandler";
+
+  private final SnapshotDescription snapshot;
+
+  private final ForeignExceptionDispatcher monitor;
+
+  private volatile boolean stopped = false;
+
+  public CloneSnapshotHandler(final MasterServices masterServices,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+    super(masterServices, masterServices.getMasterFileSystem(), 
+      masterServices.getServerManager(), hTableDescriptor,
+      masterServices.getConfiguration(), null, masterServices.getCatalogTracker(),
+      masterServices.getAssignmentManager());
+
+    // Snapshot information
+    this.snapshot = snapshot;
+
+    // Monitor
+    this.monitor = new ForeignExceptionDispatcher();
+  }
+
+  /**
+   * Create the on-disk regions, using the tableRootDir provided by the CreateTableHandler.
+   * The cloned table will be created in a temp directory, and then the CreateTableHandler
+   * will be responsible to add the regions returned by this method to META and do the assignment.
+   */
+  @Override
+  protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, final String tableName)
+      throws IOException {
+    FileSystem fs = fileSystemManager.getFileSystem();
+    Path rootDir = fileSystemManager.getRootDir();
+    Path tableDir = new Path(tableRootDir, tableName);
+
+    try {
+      // 1. Execute the on-disk Clone
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+      // Clone operation should not have stuff to restore or remove
+      Preconditions.checkArgument(!metaChanges.hasRegionsToRestore(),
+          "A clone should not have regions to restore");
+      Preconditions.checkArgument(!metaChanges.hasRegionsToRemove(),
+          "A clone should not have regions to remove");
+
+      // At this point the clone is complete. Next step is enabling the table.
+      LOG.info("Clone snapshot=" + snapshot.getName() + " on table=" + tableName + " completed!");
+
+      // 2. let the CreateTableHandler add the regions to meta
+      return metaChanges.getRegionsToAdd();
+    } catch (Exception e) {
+      String msg = "clone snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " failed";
+      LOG.error(msg, e);
+      IOException rse = new RestoreSnapshotException(msg, e, snapshot);
+
+      // these handlers aren't futures so we need to register the error here.
+      this.monitor.receive(new ForeignException(NAME, rse));
+      throw rse;
+    }
+  }
+
+  @Override
+  protected void completed(final Throwable exception) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isFinished() {
+    return this.stopped;
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public void cancel(String why) {
+    if (this.stopped) return;
+    this.stopped = true;
+    LOG.info("Stopping clone snapshot=" + snapshot + " because: " + why);
+    this.monitor.receive(new ForeignException(NAME, new CancellationException(why)));
+  }
+
+  @Override
+  public ForeignException getExceptionIfFailed() {
+    return this.monitor.getException();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
new file mode 100644
index 000000000000..739284fcfdb3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
+import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Take a snapshot of a disabled table.
+ * <p>
+ * Table must exist when taking the snapshot, or results are undefined.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
+  private static final Log LOG = LogFactory.getLog(DisabledTableSnapshotHandler.class);
+  private final TimeoutExceptionInjector timeoutInjector;
+
+  /**
+   * @param snapshot descriptor of the snapshot to take
+   * @param masterServices master services provider
+   * @throws IOException on unexpected error
+   */
+  public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
+      final MasterServices masterServices) throws IOException {
+    super(snapshot, masterServices);
+
+    // setup the timer
+    timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
+  }
+
+  // TODO consider parallelizing these operations since they are independent. Right now its just
+  // easier to keep them serial though
+  @Override
+  public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException,
+  KeeperException {
+    try {
+      timeoutInjector.start();
+
+      // 1. get all the regions hosting this table.
+
+      // extract each pair to separate lists
+      Set<String> serverNames = new HashSet<String>();
+      Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
+      for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
+        regions.add(p.getFirst());
+        serverNames.add(p.getSecond().toString());
+      }
+
+      // 2. for each region, write all the info to disk
+      LOG.info("Starting to write region info and WALs for regions for offline snapshot:"
+          + SnapshotDescriptionUtils.toString(snapshot));
+      for (HRegionInfo regionInfo : regions) {
+        // 2.1 copy the regionInfo files to the snapshot
+        Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
+          regionInfo.getEncodedName());
+        HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
+        // check for error for each region
+        monitor.rethrowException();
+
+        // 2.2 for each region, copy over its recovered.edits directory
+        Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+        new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
+        monitor.rethrowException();
+
+        // 2.3 reference all the files in the region
+        new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+        monitor.rethrowException();
+      }
+
+      // 3. write the table info to disk
+      LOG.info("Starting to copy tableinfo for offline snapshot: " +
+      SnapshotDescriptionUtils.toString(snapshot));
+      TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs,
+          FSUtils.getRootDir(conf));
+      tableInfoCopyTask.call();
+      monitor.rethrowException();
+    } catch (Exception e) {
+      // make sure we capture the exception to propagate back to the client later
+      String reason = "Failed snapshot " + SnapshotDescriptionUtils.toString(snapshot)
+          + " due to exception:" + e.getMessage();
+      ForeignException ee = new ForeignException(reason, e);
+      monitor.receive(ee);
+    } finally {
+      LOG.debug("Marking snapshot" + SnapshotDescriptionUtils.toString(snapshot)
+          + " as finished.");
+
+      // 6. mark the timer as finished - even if we got an exception, we don't need to time the
+      // operation any further
+      timeoutInjector.complete();
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
new file mode 100644
index 000000000000..17d022163713
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.util.Pair;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Handle the master side of taking a snapshot of an online table, regardless of snapshot type.
+ * Uses a {@link Procedure} to run the snapshot across all the involved region servers.
+ * @see ProcedureCoordinator
+ */
+@InterfaceAudience.Private
+public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
+
+  private static final Log LOG = LogFactory.getLog(EnabledTableSnapshotHandler.class);
+  private final ProcedureCoordinator coordinator;
+
+  public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
+      SnapshotManager manager) throws IOException {
+    super(snapshot, master);
+    this.coordinator = manager.getCoordinator();
+  }
+
+  // TODO consider switching over to using regionnames, rather than server names. This would allow
+  // regions to migrate during a snapshot, and then be involved when they are ready. Still want to
+  // enforce a snapshot time constraints, but lets us be potentially a bit more robust.
+
+  /**
+   * This method kicks off a snapshot procedure.  Other than that it hangs around for various
+   * phases to complete.
+   */
+  @Override
+  protected void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regions)
+      throws HBaseSnapshotException {
+    Set<String> regionServers = new HashSet<String>(regions.size());
+    for (Pair<HRegionInfo, ServerName> region : regions) {
+      regionServers.add(region.getSecond().toString());
+    }
+
+    // start the snapshot on the RS
+    Procedure proc = coordinator.startProcedure(this.monitor, this.snapshot.getName(),
+      this.snapshot.toByteArray(), Lists.newArrayList(regionServers));
+    if (proc == null) {
+      String msg = "Failed to submit distributed procedure for snapshot '"
+          + snapshot.getName() + "'";
+      LOG.error(msg);
+      throw new HBaseSnapshotException(msg);
+    }
+
+    try {
+      // wait for the snapshot to complete.  A timer thread is kicked off that should cancel this
+      // if it takes too long.
+      proc.waitForCompleted();
+      LOG.info("Done waiting - snapshot for " + this.snapshot.getName() + " finished!");
+    } catch (InterruptedException e) {
+      ForeignException ee =
+          new ForeignException("Interrupted while waiting for snapshot to finish", e);
+      monitor.receive(ee);
+      Thread.currentThread().interrupt();
+    } catch (ForeignException e) {
+      monitor.receive(e);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
new file mode 100644
index 000000000000..4a4457eeb7e9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * General snapshot verification on the master.
+ * <p>
+ * This is a light-weight verification mechanism for all the files in a snapshot. It doesn't
+ * attempt to verify that the files are exact copies (that would be paramount to taking the
+ * snapshot again!), but instead just attempts to ensure that the files match the expected
+ * files and are the same length.
+ * <p>
+ * Taking an online snapshots can race against other operations and this is an last line of
+ * defense.  For example, if meta changes between when snapshots are taken not all regions of a
+ * table may be present.  This can be caused by a region split (daughters present on this scan,
+ * but snapshot took parent), or move (snapshots only checks lists of region servers, a move could
+ * have caused a region to be skipped or done twice).
+ * <p>
+ * Current snapshot files checked:
+ * <ol>
+ * <li>SnapshotDescription is readable</li>
+ * <li>Table info is readable</li>
+ * <li>Regions</li>
+ * <ul>
+ * <li>Matching regions in the snapshot as currently in the table</li>
+ * <li>{@link HRegionInfo} matches the current and stored regions</li>
+ * <li>All referenced hfiles have valid names</li>
+ * <li>All the hfiles are present (either in .archive directory in the region)</li>
+ * <li>All recovered.edits files are present (by name) and have the correct file size</li>
+ * </ul>
+ * </ol>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class MasterSnapshotVerifier {
+
+  private SnapshotDescription snapshot;
+  private FileSystem fs;
+  private Path rootDir;
+  private String tableName;
+  private MasterServices services;
+
+  /**
+   * @param services services for the master
+   * @param snapshot snapshot to check
+   * @param rootDir root directory of the hbase installation.
+   */
+  public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
+    this.fs = services.getMasterFileSystem().getFileSystem();
+    this.services = services;
+    this.snapshot = snapshot;
+    this.rootDir = rootDir;
+    this.tableName = snapshot.getTable();
+  }
+
+  /**
+   * Verify that the snapshot in the directory is a valid snapshot
+   * @param snapshotDir snapshot directory to check
+   * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot
+   * @throws CorruptedSnapshotException if the snapshot is invalid
+   * @throws IOException if there is an unexpected connection issue to the filesystem
+   */
+  public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
+      throws CorruptedSnapshotException, IOException {
+    // verify snapshot info matches
+    verifySnapshotDescription(snapshotDir);
+
+    // check that tableinfo is a valid table description
+    verifyTableInfo(snapshotDir);
+
+    // check that each region is valid
+    verifyRegions(snapshotDir);
+  }
+
+  /**
+   * Check that the snapshot description written in the filesystem matches the current snapshot
+   * @param snapshotDir snapshot directory to check
+   */
+  private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
+    SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    if (!this.snapshot.equals(found)) {
+      throw new CorruptedSnapshotException("Snapshot read (" + found
+          + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot);
+    }
+  }
+
+  /**
+   * Check that the table descriptor for the snapshot is a valid table descriptor
+   * @param snapshotDir snapshot directory to check
+   */
+  private void verifyTableInfo(Path snapshotDir) throws IOException {
+    FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+  }
+
+  /**
+   * Check that all the regions in the snapshot are valid, and accounted for.
+   * @param snapshotDir snapshot directory to check
+   * @throws IOException if we can't reach .META. or read the files from the FS
+   */
+  private void verifyRegions(Path snapshotDir) throws IOException {
+    List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
+      Bytes.toBytes(tableName));
+    for (HRegionInfo region : regions) {
+      // if offline split parent, skip it
+      if (region.isOffline() && (region.isSplit() || region.isSplitParent())) {
+        continue;
+      }
+
+      verifyRegion(fs, snapshotDir, region);
+    }
+  }
+
+  /**
+   * Verify that the region (regioninfo, hfiles) are valid
+   * @param fs the FileSystem instance
+   * @param snapshotDir snapshot directory to check
+   * @param region the region to check
+   */
+  private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) throws IOException {
+    // make sure we have region in the snapshot
+    Path regionDir = new Path(snapshotDir, region.getEncodedName());
+    if (!fs.exists(regionDir)) {
+      // could happen due to a move or split race.
+      throw new CorruptedSnapshotException("No region directory found for region:" + region,
+          snapshot);
+    }
+    // make sure we have the region info in the snapshot
+    Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
+    // make sure the file exists
+    if (!fs.exists(regionInfo)) {
+      throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
+    }
+    FSDataInputStream in = fs.open(regionInfo);
+    HRegionInfo found = new HRegionInfo();
+    try {
+      found.readFields(in);
+      if (!region.equals(found)) {
+        throw new CorruptedSnapshotException("Found region info (" + found
+           + ") doesn't match expected region:" + region, snapshot);
+      }
+    } finally {
+      in.close();
+    }
+
+    // make sure we have the expected recovered edits files
+    TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
+
+    // check for the existance of each hfile
+    PathFilter familiesDirs = new FSUtils.FamilyDirFilter(fs);
+    FileStatus[] columnFamilies = FSUtils.listStatus(fs, regionDir, familiesDirs);
+    // should we do some checking here to make sure the cfs are correct?
+    if (columnFamilies == null) return;
+
+    // setup the suffixes for the snapshot directories
+    Path tableNameSuffix = new Path(tableName);
+    Path regionNameSuffix = new Path(tableNameSuffix, region.getEncodedName());
+
+    // get the potential real paths
+    Path archivedRegion = new Path(HFileArchiveUtil.getArchivePath(services.getConfiguration()),
+        regionNameSuffix);
+    Path realRegion = new Path(rootDir, regionNameSuffix);
+
+    // loop through each cf and check we can find each of the hfiles
+    for (FileStatus cf : columnFamilies) {
+      FileStatus[] hfiles = FSUtils.listStatus(fs, cf.getPath(), null);
+      // should we check if there should be hfiles?
+      if (hfiles == null || hfiles.length == 0) continue;
+
+      Path realCfDir = new Path(realRegion, cf.getPath().getName());
+      Path archivedCfDir = new Path(archivedRegion, cf.getPath().getName());
+      for (FileStatus hfile : hfiles) {
+        // make sure the name is correct
+        if (!StoreFile.validateStoreFileName(hfile.getPath().getName())) {
+          throw new CorruptedSnapshotException("HFile: " + hfile.getPath()
+              + " is not a valid hfile name.", snapshot);
+        }
+
+        // check to see if hfile is present in the real table
+        String fileName = hfile.getPath().getName();
+        Path file = new Path(realCfDir, fileName);
+        Path archived = new Path(archivedCfDir, fileName);
+        if (!fs.exists(file) && !file.equals(archived)) {
+          throw new CorruptedSnapshotException("Can't find hfile: " + hfile.getPath()
+              + " in the real (" + realCfDir + ") or archive (" + archivedCfDir
+              + ") directory for the primary table.", snapshot);
+        }
+      }
+    }
+  }
+
+  /**
+   * Check that the logs stored in the log directory for the snapshot are valid - it contains all
+   * the expected logs for all servers involved in the snapshot.
+   * @param snapshotDir snapshot directory to check
+   * @param snapshotServers list of the names of servers involved in the snapshot.
+   * @throws CorruptedSnapshotException if the hlogs in the snapshot are not correct
+   * @throws IOException if we can't reach the filesystem
+   */
+  private void verifyLogs(Path snapshotDir, Set<String> snapshotServers)
+      throws CorruptedSnapshotException, IOException {
+    Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
+    Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+    TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers, snapshot,
+      snapshotLogDir);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
new file mode 100644
index 000000000000..310347cb4c67
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -0,0 +1,155 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handler to Restore a snapshot.
+ *
+ * <p>Uses {@link RestoreSnapshotHelper} to replace the table content with the
+ * data available in the snapshot.
+ */
+@InterfaceAudience.Private
+public class RestoreSnapshotHandler extends TableEventHandler implements SnapshotSentinel {
+  private static final Log LOG = LogFactory.getLog(RestoreSnapshotHandler.class);
+
+  private final HTableDescriptor hTableDescriptor;
+  private final SnapshotDescription snapshot;
+
+  private final ForeignExceptionDispatcher monitor;
+  private volatile boolean stopped = false;
+
+  public RestoreSnapshotHandler(final MasterServices masterServices,
+      final SnapshotDescription snapshot, final HTableDescriptor htd)
+      throws IOException {
+    super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices);
+
+    // Snapshot information
+    this.snapshot = snapshot;
+
+    // Monitor
+    this.monitor = new ForeignExceptionDispatcher();
+
+    // Check table exists.
+    getTableDescriptor();
+
+    // This is the new schema we are going to write out as this modification.
+    this.hTableDescriptor = htd;
+  }
+
+  /**
+   * The restore table is executed in place.
+   *  - The on-disk data will be restored - reference files are put in place without moving data
+   *  -  [if something fail here: you need to delete the table and re-run the restore]
+   *  - META will be updated
+   *  -  [if something fail here: you need to run hbck to fix META entries]
+   * The passed in list gets changed in this method
+   */
+  @Override
+  protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
+    MasterFileSystem fileSystemManager = masterServices.getMasterFileSystem();
+    CatalogTracker catalogTracker = masterServices.getCatalogTracker();
+    FileSystem fs = fileSystemManager.getFileSystem();
+    Path rootDir = fileSystemManager.getRootDir();
+    byte[] tableName = hTableDescriptor.getName();
+    Path tableDir = HTableDescriptor.getTableDir(rootDir, tableName);
+
+    try {
+      // 1. Update descriptor
+      this.masterServices.getTableDescriptors().add(hTableDescriptor);
+
+      // 2. Execute the on-disk Restore
+      LOG.debug("Starting restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot));
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
+          masterServices.getConfiguration(), fs,
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+      // 3. Applies changes to .META.
+      hris.clear();
+      if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
+      if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
+      List<HRegionInfo> hrisToRemove = metaChanges.getRegionsToRemove();
+      MetaEditor.mutateRegions(catalogTracker, hrisToRemove, hris);
+
+      // At this point the restore is complete. Next step is enabling the table.
+      LOG.info("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " on table=" +
+        Bytes.toString(tableName) + " completed!");
+    } catch (IOException e) {
+      String msg = "restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+          + " failed. Try re-running the restore command.";
+      LOG.error(msg, e);
+      monitor.receive(new ForeignException(masterServices.getServerName().toString(), e));
+      throw new RestoreSnapshotException(msg, e);
+    } finally {
+      this.stopped = true;
+    }
+  }
+
+  @Override
+  public boolean isFinished() {
+    return this.stopped;
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public void cancel(String why) {
+    if (this.stopped) return;
+    this.stopped = true;
+    String msg = "Stopping restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+        + " because: " + why;
+    LOG.info(msg);
+    CancellationException ce = new CancellationException(why);
+    this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
+  }
+
+  public ForeignException getExceptionIfFailed() {
+    return this.monitor.getException();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
new file mode 100644
index 000000000000..0e998f706463
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -0,0 +1,308 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Intelligently keep track of all the files for all the snapshots.
+ * <p>
+ * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
+ * miss the directory modification time is used to ensure that we don't rescan directories that we
+ * already have in cache. We only check the modification times of the snapshot directories
+ * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
+ * <p>
+ * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
+ * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
+ * we will ignore updates to that snapshot's files.
+ * <p>
+ * This is sufficient because each snapshot has its own directory and is added via an atomic rename
+ * <i>once</i>, when the snapshot is created. We don't need to worry about the data in the snapshot
+ * being run.
+ * <p>
+ * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
+ * also removed from the cache.
+ * <p>
+ * A SnapshotFileInspector must be passed when creating <tt>this</tt> to allow extraction
+ * of files under the /hbase/.snapshot/[snapshot name] directory, for each snapshot.
+ * This allows you to only cache files under, for instance, all the logs in the .logs directory or
+ * all the files under all the regions.
+ * <p>
+ * <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
+ * snapshots and will attempt to cache files from those snapshots as well.
+ * <p>
+ * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotFileCache implements Stoppable {
+  interface SnapshotFileInspector {
+    /**
+     * Returns a collection of file names needed by the snapshot.
+     * @param snapshotDir {@link Path} to the snapshot directory to scan.
+     * @return the collection of file names needed by the snapshot.
+     */
+    Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException;
+  }
+
+  private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
+  private volatile boolean stop = false;
+  private final FileSystem fs;
+  private final SnapshotFileInspector fileInspector;
+  private final Path snapshotDir;
+  private final Set<String> cache = new HashSet<String>();
+  /**
+   * This is a helper map of information about the snapshot directories so we don't need to rescan
+   * them if they haven't changed since the last time we looked.
+   */
+  private final Map<String, SnapshotDirectoryInfo> snapshots =
+      new HashMap<String, SnapshotDirectoryInfo>();
+  private final Timer refreshTimer;
+
+  private long lastModifiedTime = Long.MIN_VALUE;
+
+  /**
+   * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+   * filesystem.
+   * <p>
+   * Immediately loads the file cache.
+   * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and
+   *          hbase root directory
+   * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
+   * @param refreshThreadName name of the cache refresh thread
+   * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+   * @throws IOException if the {@link FileSystem} or root directory cannot be loaded
+   */
+  public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
+      SnapshotFileInspector inspectSnapshotFiles) throws IOException {
+    this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
+        refreshThreadName, inspectSnapshotFiles);
+  }
+
+  /**
+   * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+   * filesystem
+   * @param fs {@link FileSystem} where the snapshots are stored
+   * @param rootDir hbase root directory
+   * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
+   * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
+   * @param refreshThreadName name of the cache refresh thread
+   * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+   */
+  public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
+      long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
+    this.fs = fs;
+    this.fileInspector = inspectSnapshotFiles;
+    this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    // periodically refresh the file cache to make sure we aren't superfluously saving files.
+    this.refreshTimer = new Timer(refreshThreadName, true);
+    this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
+      cacheRefreshPeriod);
+  }
+
+  /**
+   * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending
+   * cache refreshes.
+   * <p>
+   * Blocks until the cache is refreshed.
+   * <p>
+   * Exposed for TESTING.
+   */
+  public void triggerCacheRefreshForTesting() {
+    try {
+      SnapshotFileCache.this.refreshCache();
+    } catch (IOException e) {
+      LOG.warn("Failed to refresh snapshot hfile cache!", e);
+    }
+    LOG.debug("Current cache:" + cache);
+  }
+
+  /**
+   * Check to see if the passed file name is contained in any of the snapshots. First checks an
+   * in-memory cache of the files to keep. If its not in the cache, then the cache is refreshed and
+   * the cache checked again for that file. This ensures that we always return <tt>true</tt> for a
+   * files that exists.
+   * <p>
+   * Note this may lead to periodic false positives for the file being referenced. Periodically, the
+   * cache is refreshed even if there are no requests to ensure that the false negatives get removed
+   * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the
+   * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed
+   * at that point, cache will still think the file system contains that file and return
+   * <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
+   * on the filesystem, we will never find it and always return <tt>false</tt>.
+   * @param fileName file to check
+   * @return <tt>false</tt> if the file is not referenced in any current or running snapshot,
+   *         <tt>true</tt> if the file is in the cache.
+   * @throws IOException if there is an unexpected error reaching the filesystem.
+   */
+  // XXX this is inefficient to synchronize on the method, when what we really need to guard against
+  // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
+  // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
+  public synchronized boolean contains(String fileName) throws IOException {
+    if (this.cache.contains(fileName)) return true;
+
+    refreshCache();
+
+    // then check again
+    return this.cache.contains(fileName);
+  }
+
+  private synchronized void refreshCache() throws IOException {
+    // get the status of the snapshots directory
+    FileStatus status;
+    try {
+      status = fs.getFileStatus(snapshotDir);
+    } catch (FileNotFoundException e) {
+      LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
+      return;
+    }
+    // if the snapshot directory wasn't modified since we last check, we are done
+    if (status.getModificationTime() <= lastModifiedTime) return;
+
+    // directory was modified, so we need to reload our cache
+    // there could be a slight race here where we miss the cache, check the directory modification
+    // time, then someone updates the directory, causing us to not scan the directory again.
+    // However, snapshot directories are only created once, so this isn't an issue.
+
+    // 1. update the modified time
+    this.lastModifiedTime = status.getModificationTime();
+
+    // 2.clear the cache
+    this.cache.clear();
+    Map<String, SnapshotDirectoryInfo> known = new HashMap<String, SnapshotDirectoryInfo>();
+
+    // 3. check each of the snapshot directories
+    FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
+    if (snapshots == null) {
+      // remove all the remembered snapshots because we don't have any left
+      LOG.debug("No snapshots on-disk, cache empty");
+      this.snapshots.clear();
+      return;
+    }
+
+    // 3.1 iterate through the on-disk snapshots
+    for (FileStatus snapshot : snapshots) {
+      String name = snapshot.getPath().getName();
+      // its the tmp dir
+      if (name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+        // only add those files to the cache, but not to the known snapshots
+        FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
+        if (running == null) continue;
+        for (FileStatus run : running) {
+          this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+        }
+      } else {
+        SnapshotDirectoryInfo files = this.snapshots.remove(name);
+        // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
+        // the latter could occur where I create a snapshot, then delete it, and then make a new
+        // snapshot with the same name. We will need to update the cache the information from that new
+        // snapshot, even though it has the same name as the files referenced have probably changed.
+        if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
+          // get all files for the snapshot and create a new info
+          Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
+          files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
+        }
+        // 3.2 add all the files to cache
+        this.cache.addAll(files.getFiles());
+        known.put(name, files);
+      }
+    }
+
+    // 4. set the snapshots we are tracking
+    this.snapshots.clear();
+    this.snapshots.putAll(known);
+  }
+
+  /**
+   * Simple helper task that just periodically attempts to refresh the cache
+   */
+  public class RefreshCacheTask extends TimerTask {
+    @Override
+    public void run() {
+      try {
+        SnapshotFileCache.this.refreshCache();
+      } catch (IOException e) {
+        LOG.warn("Failed to refresh snapshot hfile cache!", e);
+      }
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    if (!this.stop) {
+      this.stop = true;
+      this.refreshTimer.cancel();
+    }
+
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stop;
+  }
+
+  /**
+   * Information about a snapshot directory
+   */
+  private static class SnapshotDirectoryInfo {
+    long lastModified;
+    Collection<String> files;
+
+    public SnapshotDirectoryInfo(long mtime, Collection<String> files) {
+      this.lastModified = mtime;
+      this.files = files;
+    }
+
+    /**
+     * @return the hfiles in the snapshot when <tt>this</tt> was made.
+     */
+    public Collection<String> getFiles() {
+      return this.files;
+    }
+
+    /**
+     * Check if the snapshot directory has been modified
+     * @param mtime current modification time of the directory
+     * @return <tt>true</tt> if it the modification time of the directory is newer time when we
+     *         created <tt>this</tt>
+     */
+    public boolean hasBeenModified(long mtime) {
+      return this.lastModified < mtime;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
new file mode 100644
index 000000000000..7b75fd217211
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
+ * tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(SnapshotHFileCleaner.class);
+
+  /**
+   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+   * snapshots (ms)
+   */
+  public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
+      "hbase.master.hfilecleaner.plugins.snapshot.period";
+
+  /** Refresh cache, by default, every 5 minutes */
+  private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
+
+  /** File cache for HFiles in the completed and currently running snapshots */
+  private SnapshotFileCache cache;
+
+  @Override
+  public synchronized boolean isFileDeletable(Path filePath) {
+    try {
+      return !cache.contains(filePath.getName());
+    } catch (IOException e) {
+      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+        e);
+      return false;
+    }
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    try {
+      long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
+        DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
+      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+      Path rootDir = FSUtils.getRootDir(conf);
+      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+          "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+            }
+          });
+    } catch (IOException e) {
+      LOG.error("Failed to create cleaner util", e);
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    this.cache.stop(why);
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.cache.isStopped();
+  }
+
+  /**
+   * Exposed for Testing!
+   * @return the cache of all hfiles
+   */
+  public SnapshotFileCache getFileCacheForTesting() {
+    return this.cache;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
new file mode 100644
index 000000000000..12af8c429909
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a log cleaner that checks if a log is still used by
+ * snapshots of HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(SnapshotLogCleaner.class);
+
+  /**
+   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+   * snapshots (ms)
+   */
+  static final String HLOG_CACHE_REFRESH_PERIOD_CONF_KEY =
+      "hbase.master.hlogcleaner.plugins.snapshot.period";
+
+  /** Refresh cache, by default, every 5 minutes */
+  private static final long DEFAULT_HLOG_CACHE_REFRESH_PERIOD = 300000;
+
+  private SnapshotFileCache cache;
+
+  @Override
+  public synchronized boolean isLogDeletable(Path filePath) {
+    try {
+      if (null == cache) return false;
+      return !cache.contains(filePath.getName());
+    } catch (IOException e) {
+      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+        e);
+      return false;
+    }
+  }
+
+  /**
+   * This method should only be called <b>once</b>, as it starts a thread to keep the cache
+   * up-to-date.
+   * <p>
+   * {@inheritDoc}
+   */
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    try {
+      long cacheRefreshPeriod = conf.getLong(
+        HLOG_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HLOG_CACHE_REFRESH_PERIOD);
+      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+      Path rootDir = FSUtils.getRootDir(conf);
+      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+          "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+            }
+          });
+    } catch (IOException e) {
+      LOG.error("Failed to create snapshot log cleaner", e);
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    this.cache.stop(why);
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.cache.isStopped();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
new file mode 100644
index 000000000000..a95d0c67b9c8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -0,0 +1,916 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
+import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
+import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This class manages the procedure of taking and restoring snapshots. There is only one
+ * SnapshotManager for the master.
+ * <p>
+ * The class provides methods for monitoring in-progress snapshot actions.
+ * <p>
+ * Note: Currently there can only be one snapshot being taken at a time over the cluster. This is a
+ * simplification in the current implementation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class SnapshotManager implements Stoppable {
+  private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
+
+  /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
+  private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
+
+  /** Enable or disable snapshot support */
+  public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
+
+  /**
+   * Conf key for # of ms elapsed between checks for snapshot errors while waiting for
+   * completion.
+   */
+  private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
+
+  /** By default, check to see if the snapshot is complete (ms) */
+  private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 5000;
+
+  /**
+   * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
+   * completion.
+   */
+  private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
+
+  /** Name of the operation to use in the controller */
+  public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
+
+  // TODO - enable having multiple snapshots with multiple monitors/threads
+  // this needs to be configuration based when running multiple snapshots is implemented
+  /** number of current operations running on the master */
+  private static final int opThreads = 1;
+
+  private boolean stopped;
+  private final long wakeFrequency;
+  private final MasterServices master;  // Needed by TableEventHandlers
+  private final ProcedureCoordinator coordinator;
+
+  // Is snapshot feature enabled?
+  private boolean isSnapshotSupported = false;
+
+  // A reference to a handler.  If the handler is non-null, then it is assumed that a snapshot is
+  // in progress currently
+  // TODO: this is a bad smell;  likely replace with a collection in the future.  Also this gets
+  // reset by every operation.
+  private TakeSnapshotHandler handler;
+
+  private final Path rootDir;
+  private final ExecutorService executorService;
+
+  // Restore Sentinels map, with table name as key
+  private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
+
+  /**
+   * Construct a snapshot manager.
+   * @param master
+   */
+  public SnapshotManager(final MasterServices master) throws KeeperException, IOException,
+    UnsupportedOperationException {
+    this.master = master;
+    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+
+    // get the configuration for the coordinator
+    Configuration conf = master.getConfiguration();
+    this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
+    long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+
+    // setup the default procedure coordinator
+    String name = master.getServerName().toString();
+    ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, keepAliveTime, opThreads, wakeFrequency);
+    ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
+        master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
+    this.coordinator = new ProcedureCoordinator(comms, tpool);
+    this.rootDir = master.getMasterFileSystem().getRootDir();
+    this.executorService = master.getExecutorService();
+    resetTempDir();
+  }
+
+  /**
+   * Fully specify all necessary components of a snapshot manager. Exposed for testing.
+   * @param master services for the master where the manager is running
+   * @param coordinator procedure coordinator instance.  exposed for testing.
+   * @param pool HBase ExecutorServcie instance, exposed for testing.
+   */
+  public SnapshotManager(final MasterServices master, ProcedureCoordinator coordinator, ExecutorService pool)
+      throws IOException, UnsupportedOperationException {
+    this.master = master;
+    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+
+    this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
+      SNAPSHOT_WAKE_MILLIS_DEFAULT);
+    this.coordinator = coordinator;
+    this.rootDir = master.getMasterFileSystem().getRootDir();
+    this.executorService = pool;
+    resetTempDir();
+  }
+
+  /**
+   * Gets the list of all completed snapshots.
+   * @return list of SnapshotDescriptions
+   * @throws IOException File system exception
+   */
+  public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
+    List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
+    // first create the snapshot root path and check to see if it exists
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+
+    // if there are no snapshots, return an empty list
+    if (!fs.exists(snapshotDir)) {
+      return snapshotDescs;
+    }
+
+    // ignore all the snapshots in progress
+    FileStatus[] snapshots = fs.listStatus(snapshotDir,
+      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+    // loop through all the completed snapshots
+    for (FileStatus snapshot : snapshots) {
+      Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
+      // if the snapshot is bad
+      if (!fs.exists(info)) {
+        LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
+        continue;
+      }
+      FSDataInputStream in = null;
+      try {
+        in = fs.open(info);
+        SnapshotDescription desc = SnapshotDescription.parseFrom(in);
+        snapshotDescs.add(desc);
+      } catch (IOException e) {
+        LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
+      } finally {
+        if (in != null) {
+          in.close();
+        }
+      }
+    }
+    return snapshotDescs;
+  }
+
+  /**
+   * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
+   * snapshot attempts.
+   *
+   * @throws IOException if we can't reach the filesystem
+   */
+  void resetTempDir() throws IOException {
+    // cleanup any existing snapshots.
+    Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
+    if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
+      LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
+    }
+  }
+
+  /**
+   * Delete the specified snapshot
+   * @param snapshot
+   * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
+   * @throws IOException For filesystem IOExceptions
+   */
+  public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
+
+    // call coproc pre hook
+    MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preDeleteSnapshot(snapshot);
+    }
+
+    // check to see if it is completed
+    if (!isSnapshotCompleted(snapshot)) {
+      throw new SnapshotDoesNotExistException(snapshot);
+    }
+
+    String snapshotName = snapshot.getName();
+    LOG.debug("Deleting snapshot: " + snapshotName);
+    // first create the snapshot description and check to see if it exists
+    MasterFileSystem fs = master.getMasterFileSystem();
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+
+    // delete the existing snapshot
+    if (!fs.getFileSystem().delete(snapshotDir, true)) {
+      throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
+    }
+
+    // call coproc post hook
+    if (cpHost != null) {
+      cpHost.postDeleteSnapshot(snapshot);
+    }
+
+  }
+
+  /**
+   * Return the handler if it is currently running and has the same snapshot target name.
+   * @param snapshot
+   * @return null if doesn't match, else a live handler.
+   */
+  private synchronized TakeSnapshotHandler getTakeSnapshotHandler(SnapshotDescription snapshot) {
+    TakeSnapshotHandler h = this.handler;
+    if (h == null) {
+      return null;
+    }
+
+    if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+      // specified snapshot is to the one currently running
+      return null;
+    }
+
+    return h;
+  }
+
+  /**
+   * Check if the specified snapshot is done
+   * @param expected
+   * @return true if snapshot is ready to be restored, false if it is still being taken.
+   * @throws IOException IOException if error from HDFS or RPC
+   * @throws UnknownSnapshotException if snapshot is invalid or does not exist.
+   */
+  public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
+    // check the request to make sure it has a snapshot
+    if (expected == null) {
+      throw new UnknownSnapshotException(
+         "No snapshot name passed in request, can't figure out which snapshot you want to check.");
+    }
+
+    String ssString = SnapshotDescriptionUtils.toString(expected);
+
+    // check to see if the sentinel exists
+    TakeSnapshotHandler handler = getTakeSnapshotHandler(expected);
+    if (handler == null) {
+      // doesn't exist, check if it is already completely done.
+      if (!isSnapshotCompleted(expected)) {
+        throw new UnknownSnapshotException("Snapshot " + ssString
+            + " is not currently running or one of the known completed snapshots.");
+      }
+      // was done, return true;
+      return true;
+    }
+
+    // pass on any failure we find in the sentinel
+    try {
+      handler.rethrowException();
+    } catch (ForeignException e) {
+      // Give some procedure info on an exception.
+      String status;
+      Procedure p = coordinator.getProcedure(expected.getName());
+      if (p != null) {
+        status = p.getStatus();
+      } else {
+        status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
+      }
+      throw new HBaseSnapshotException("Snapshot " + ssString +  " had an error.  " + status, e,
+          expected);
+    }
+
+    // check to see if we are done
+    if (handler.isFinished()) {
+      LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
+      return true;
+    } else if (LOG.isDebugEnabled()) {
+      LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
+    }
+    return false;
+  }
+
+  /**
+   * Check to see if there are any snapshots in progress currently.  Currently we have a
+   * limitation only allowing a single snapshot attempt at a time.
+   * @return <tt>true</tt> if there any snapshots in progress, <tt>false</tt> otherwise
+   * @throws SnapshotCreationException if the snapshot failed
+   */
+  synchronized boolean isTakingSnapshot() throws SnapshotCreationException {
+    // TODO later when we handle multiple there would be a map with ssname to handler.
+    return handler != null && !handler.isFinished();
+  }
+
+  /**
+   * Check to see if the specified table has a snapshot in progress.  Currently we have a
+   * limitation only allowing a single snapshot attempt at a time.
+   * @param tableName name of the table being snapshotted.
+   * @return <tt>true</tt> if there is a snapshot in progress on the specified table.
+   */
+  private boolean isTakingSnapshot(final String tableName) {
+    if (handler != null && handler.getSnapshot().getTable().equals(tableName)) {
+      return !handler.isFinished();
+    }
+    return false;
+  }
+
+  /**
+   * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
+   * aren't already running a snapshot.
+   * @param snapshot description of the snapshot we want to start
+   * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
+   */
+  private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+
+    // make sure we aren't already running a snapshot
+    if (isTakingSnapshot()) {
+      throw new SnapshotCreationException("Rejected taking "
+          + SnapshotDescriptionUtils.toString(snapshot)
+          + " because we are already running another snapshot "
+          + SnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+    }
+
+    // make sure we aren't running a restore on the same table
+    if (isRestoringTable(snapshot.getTable())) {
+      throw new SnapshotCreationException("Rejected taking "
+          + SnapshotDescriptionUtils.toString(snapshot)
+          + " because we are already have a restore in progress on the same snapshot "
+          + SnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+    }
+
+    try {
+      // delete the working directory, since we aren't running the snapshot. Likely leftovers
+      // from a failed attempt.
+      fs.delete(workingDir, true);
+
+      // recreate the working directory for the snapshot
+      if (!fs.mkdirs(workingDir)) {
+        throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
+            + ") for snapshot" , snapshot);
+      }
+    } catch (HBaseSnapshotException e) {
+      throw e;
+    } catch (IOException e) {
+      throw new SnapshotCreationException(
+          "Exception while checking to see if snapshot could be started.", e, snapshot);
+    }
+  }
+
+  /**
+   * Take a snapshot of an enabled table.
+   * <p>
+   * The thread limitation on the executorService's thread pool for snapshots ensures the
+   * snapshot won't be started if there is another snapshot already running. Does
+   * <b>not</b> check to see if another snapshot of the same name already exists.
+   * @param snapshot description of the snapshot to take.
+   * @throws HBaseSnapshotException if the snapshot could not be started
+   */
+  private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    TakeSnapshotHandler handler;
+    try {
+      handler = new EnabledTableSnapshotHandler(snapshot, master, this);
+      this.executorService.submit(handler);
+      this.handler = handler;
+    } catch (IOException e) {
+      // cleanup the working directory by trying to delete it from the fs.
+      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+      try {
+        if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
+          LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:"
+              + SnapshotDescriptionUtils.toString(snapshot));
+        }
+      } catch (IOException e1) {
+        LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+            SnapshotDescriptionUtils.toString(snapshot));
+      }
+      // fail the snapshot
+      throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
+    }
+  }
+
+  /**
+   * Take a snapshot based on the enabled/disabled state of the table.
+   *
+   * @param snapshot
+   * @throws HBaseSnapshotException when a snapshot specific exception occurs.
+   * @throws IOException when some sort of generic IO exception occurs.
+   */
+  public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
+    // check to see if we already completed the snapshot
+    if (isSnapshotCompleted(snapshot)) {
+      throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
+          + "' already stored on the filesystem.", snapshot);
+    }
+
+    LOG.debug("No existing snapshot, attempting snapshot...");
+
+    // check to see if the table exists
+    HTableDescriptor desc = null;
+    try {
+      desc = master.getTableDescriptors().get(snapshot.getTable());
+    } catch (FileNotFoundException e) {
+      String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
+      LOG.error(msg);
+      throw new SnapshotCreationException(msg, e, snapshot);
+    } catch (IOException e) {
+      throw new SnapshotCreationException("Error while geting table description for table "
+          + snapshot.getTable(), e, snapshot);
+    }
+    if (desc == null) {
+      throw new SnapshotCreationException("Table '" + snapshot.getTable()
+          + "' doesn't exist, can't take snapshot.", snapshot);
+    }
+
+    // set the snapshot version, now that we are ready to take it
+    snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
+        .build();
+
+    // call pre coproc hook
+    MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preSnapshot(snapshot, desc);
+    }
+
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // if the table is enabled, then have the RS run actually the snapshot work
+    AssignmentManager assignmentMgr = master.getAssignmentManager();
+    if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
+      LOG.debug("Table enabled, starting distributed snapshot.");
+      snapshotEnabledTable(snapshot);
+      LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
+    }
+    // For disabled table, snapshot is created by the master
+    else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
+      LOG.debug("Table is disabled, running snapshot entirely on master.");
+      snapshotDisabledTable(snapshot);
+      LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
+    } else {
+      LOG.error("Can't snapshot table '" + snapshot.getTable()
+          + "', isn't open or closed, we don't know what to do!");
+      TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
+          + " isn't fully open.");
+      throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
+    }
+
+    // call post coproc hook
+    if (cpHost != null) {
+      cpHost.postSnapshot(snapshot, desc);
+    }
+  }
+
+  /**
+   * Take a snapshot of a disabled table.
+   * <p>
+   * The thread limitation on the executorService's thread pool for snapshots ensures the
+   * snapshot won't be started if there is another snapshot already running. Does
+   * <b>not</b> check to see if another snapshot of the same name already exists.
+   * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
+   * @throws HBaseSnapshotException if the snapshot could not be started
+   */
+  private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+
+    // set the snapshot to be a disabled snapshot, since the client doesn't know about that
+    snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
+
+    DisabledTableSnapshotHandler handler;
+    try {
+      handler = new DisabledTableSnapshotHandler(snapshot, this.master);
+      this.executorService.submit(handler);
+      this.handler = handler;
+    } catch (IOException e) {
+      // cleanup the working directory by trying to delete it from the fs.
+      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+      try {
+        if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
+          LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+              SnapshotDescriptionUtils.toString(snapshot));
+        }
+      } catch (IOException e1) {
+        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+            SnapshotDescriptionUtils.toString(snapshot));
+      }
+      // fail the snapshot
+      throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
+    }
+  }
+
+  /**
+   * Set the handler for the current snapshot
+   * <p>
+   * Exposed for TESTING
+   * @param handler handler the master should use
+   *
+   * TODO get rid of this if possible, repackaging, modify tests.
+   */
+  public synchronized void setSnapshotHandlerForTesting(TakeSnapshotHandler handler) {
+    this.handler = handler;
+  }
+
+  /**
+   * @return distributed commit coordinator for all running snapshots
+   */
+  ProcedureCoordinator getCoordinator() {
+    return coordinator;
+  }
+
+  /**
+   * Check to see if the snapshot is one of the currently completed snapshots
+   * @param expected snapshot to check
+   * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
+   *         not stored
+   * @throws IOException if the filesystem throws an unexpected exception,
+   * @throws IllegalArgumentException if snapshot name is invalid.
+   */
+  private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
+    try {
+      final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+
+      // check to see if the snapshot already exists
+      return fs.exists(snapshotDir);
+    } catch (IllegalArgumentException iae) {
+      throw new UnknownSnapshotException("Unexpected exception thrown", iae);
+    }
+  }
+
+  /**
+   * Clone the specified snapshot into a new table.
+   * The operation will fail if the destination table has a snapshot or restore in progress.
+   *
+   * @param snapshot Snapshot Descriptor
+   * @param hTableDescriptor Table Descriptor of the table to create
+   * @param waitTime timeout before considering the clone failed
+   */
+  synchronized void cloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
+    String tableName = hTableDescriptor.getNameAsString();
+
+    // make sure we aren't running a snapshot on the same table
+    if (isTakingSnapshot(tableName)) {
+      throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
+    }
+
+    // make sure we aren't running a restore on the same table
+    if (isRestoringTable(tableName)) {
+      throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
+    }
+
+    try {
+      CloneSnapshotHandler handler =
+        new CloneSnapshotHandler(master, snapshot, hTableDescriptor);
+      this.executorService.submit(handler);
+      restoreHandlers.put(tableName, handler);
+    } catch (Exception e) {
+      String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
+        " on table=" + tableName;
+      LOG.error(msg, e);
+      throw new RestoreSnapshotException(msg, e);
+    }
+  }
+
+  /**
+   * Restore the specified snapshot
+   * @param reqSnapshot
+   * @throws IOException
+   */
+  public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
+    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
+    MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+
+    // check if the snapshot exists
+    if (!fs.exists(snapshotDir)) {
+      LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
+      throw new SnapshotDoesNotExistException(reqSnapshot);
+    }
+
+    // read snapshot information
+    SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+    String tableName = reqSnapshot.getTable();
+
+    // stop tracking completed restores
+    cleanupRestoreSentinels();
+
+    // Execute the restore/clone operation
+    if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
+      if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
+        throw new UnsupportedOperationException("Table '" +
+          fsSnapshot.getTable() + "' must be disabled in order to perform a restore operation.");
+      }
+
+      // call coproc pre hook
+      if (cpHost != null) {
+        cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
+      }
+      restoreSnapshot(fsSnapshot, snapshotTableDesc);
+      LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
+
+      if (cpHost != null) {
+        cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
+      }
+    } else {
+      HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc,
+                                                         Bytes.toBytes(tableName));
+      if (cpHost != null) {
+        cpHost.preCloneSnapshot(reqSnapshot, htd);
+      }
+      cloneSnapshot(fsSnapshot, htd);
+      LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
+
+      if (cpHost != null) {
+        cpHost.postCloneSnapshot(reqSnapshot, htd);
+      }
+    }
+  }
+
+  /**
+   * Restore the specified snapshot.
+   * The restore will fail if the destination table has a snapshot or restore in progress.
+   *
+   * @param snapshot Snapshot Descriptor
+   * @param hTableDescriptor Table Descriptor
+   * @param waitTime timeout before considering the restore failed
+   */
+  private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
+    String tableName = hTableDescriptor.getNameAsString();
+
+    // make sure we aren't running a snapshot on the same table
+    if (isTakingSnapshot(tableName)) {
+      throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
+    }
+
+    // make sure we aren't running a restore on the same table
+    if (isRestoringTable(tableName)) {
+      throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
+    }
+
+    try {
+      RestoreSnapshotHandler handler =
+        new RestoreSnapshotHandler(master, snapshot, hTableDescriptor);
+      this.executorService.submit(handler);
+      restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
+    } catch (Exception e) {
+      String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(snapshot)  +
+          " on table=" + tableName;
+      LOG.error(msg, e);
+      throw new RestoreSnapshotException(msg, e);
+    }
+  }
+
+  /**
+   * Verify if the restore of the specified table is in progress.
+   *
+   * @param tableName table under restore
+   * @return <tt>true</tt> if there is a restore in progress of the specified table.
+   */
+  private boolean isRestoringTable(final String tableName) {
+    SnapshotSentinel sentinel = restoreHandlers.get(tableName);
+    return(sentinel != null && !sentinel.isFinished());
+  }
+
+  /**
+   * Returns status of a restore request, specifically comparing source snapshot and target table
+   * names.  Throws exception if not a known snapshot.
+   * @param snapshot
+   * @return true if in progress, false if snapshot is completed.
+   * @throws UnknownSnapshotException if specified source snapshot does not exit.
+   * @throws IOException if there was some sort of IO failure
+   */
+  public boolean isRestoringTable(final SnapshotDescription snapshot) throws IOException {
+    // check to see if the snapshot is already on the fs
+    if (!isSnapshotCompleted(snapshot)) {
+      throw new UnknownSnapshotException("Snapshot:" + snapshot.getName()
+          + " is not one of the known completed snapshots.");
+    }
+
+    SnapshotSentinel sentinel = getRestoreSnapshotSentinel(snapshot.getTable());
+    if (sentinel == null) {
+      // there is no sentinel so restore is not in progress.
+      return false;
+    }
+    if (!sentinel.getSnapshot().getName().equals(snapshot.getName())) {
+      // another handler is trying to restore to the table, but it isn't the same snapshot source.
+      return false;
+    }
+
+    LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
+        + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
+    ForeignException e = sentinel.getExceptionIfFailed();
+    if (e != null) throw e;
+
+    // check to see if we are done
+    if (sentinel.isFinished()) {
+      LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
+          " has completed. Notifying the client.");
+      return false;
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
+          SnapshotDescriptionUtils.toString(snapshot));
+    }
+    return true;
+  }
+
+  /**
+   * Get the restore snapshot sentinel for the specified table
+   * @param tableName table under restore
+   * @return the restore snapshot handler
+   */
+  private synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName) {
+    try {
+      return restoreHandlers.get(tableName);
+    } finally {
+      cleanupRestoreSentinels();
+    }
+  }
+
+  /**
+   * Scan the restore handlers and remove the finished ones.
+   */
+  private synchronized void cleanupRestoreSentinels() {
+    Iterator<Map.Entry<String, SnapshotSentinel>> it = restoreHandlers.entrySet().iterator();
+    while (it.hasNext()) {
+        Map.Entry<String, SnapshotSentinel> entry = it.next();
+        SnapshotSentinel sentinel = entry.getValue();
+        if (sentinel.isFinished()) {
+          it.remove();
+        }
+    }
+  }
+
+  //
+  // Implementing Stoppable interface
+  //
+
+  @Override
+  public void stop(String why) {
+    // short circuit
+    if (this.stopped) return;
+    // make sure we get stop
+    this.stopped = true;
+    // pass the stop onto take snapshot handlers
+    if (this.handler != null) this.handler.cancel(why);
+
+    // pass the stop onto all the restore handlers
+    for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
+      restoreHandler.cancel(why);
+    }
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
+  /**
+   * Throws an exception if snapshot operations (take a snapshot, restore, clone) are not supported.
+   * Called at the beginning of snapshot() and restoreSnapshot() methods.
+   * @throws UnsupportedOperationException if snapshot are not supported
+   */
+  public void checkSnapshotSupport() throws UnsupportedOperationException {
+    if (!this.isSnapshotSupported) {
+      throw new UnsupportedOperationException(
+        "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
+          HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
+    }
+  }
+
+  /**
+   * Called at startup, to verify if snapshot operation is supported, and to avoid
+   * starting the master if there're snapshots present but the cleaners needed are missing.
+   * Otherwise we can end up with snapshot data loss.
+   * @param conf The {@link Configuration} object to use
+   * @param mfs The MasterFileSystem to use
+   * @throws IOException in case of file-system operation failure
+   * @throws UnsupportedOperationException in case cleaners are missing and
+   *         there're snapshot in the system
+   */
+  private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
+      throws IOException, UnsupportedOperationException {
+    // Verify if snapshot is disabled by the user
+    String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
+    boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
+    boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
+
+    // Extract cleaners from conf
+    Set<String> hfileCleaners = new HashSet<String>();
+    String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
+    if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
+
+    Set<String> logCleaners = new HashSet<String>();
+    cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
+    if (cleaners != null) Collections.addAll(logCleaners, cleaners);
+
+    // If the user has enabled the snapshot, we force the cleaners to be present
+    // otherwise we still need to check if cleaners are enabled or not and verify
+    // that there're no snapshot in the .snapshot folder.
+    if (snapshotEnabled) {
+      // Inject snapshot cleaners, if snapshot.enable is true
+      hfileCleaners.add(SnapshotHFileCleaner.class.getName());
+      hfileCleaners.add(HFileLinkCleaner.class.getName());
+      logCleaners.add(SnapshotLogCleaner.class.getName());
+
+      // Set cleaners conf
+      conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+        hfileCleaners.toArray(new String[hfileCleaners.size()]));
+      conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
+        logCleaners.toArray(new String[logCleaners.size()]));
+    } else {
+      // Verify if cleaners are present
+      snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
+        hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
+        hfileCleaners.contains(HFileLinkCleaner.class.getName());
+
+      // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
+      if (snapshotEnabled) {
+        LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
+          "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
+          (userDisabled ? "is set to 'false'." : "is not set."));
+      }
+    }
+
+    // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
+    this.isSnapshotSupported = snapshotEnabled && !userDisabled;
+
+    // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder
+    // otherwise we end up with snapshot data loss.
+    if (!snapshotEnabled) {
+      LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
+      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
+      FileSystem fs = mfs.getFileSystem();
+      if (fs.exists(snapshotDir)) {
+        FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
+          new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+        if (snapshots != null) {
+          LOG.error("Snapshots are present, but cleaners are not enabled.");
+          checkSnapshotSupport();
+        }
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
new file mode 100644
index 000000000000..3ab4369b04d0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -0,0 +1,237 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * A handler for taking snapshots from the master.
+ *
+ * This is not a subclass of TableEventHandler because using that would incur an extra META scan.
+ *
+ * The {@link #snapshotRegions(List)} call should get implemented for each snapshot flavor.
+ */
+@InterfaceAudience.Private
+public abstract class TakeSnapshotHandler extends EventHandler implements SnapshotSentinel,
+    ForeignExceptionSnare {
+  private static final Log LOG = LogFactory.getLog(TakeSnapshotHandler.class);
+
+  private volatile boolean finished;
+
+  // none of these should ever be null
+  protected final MasterServices master;
+  protected final SnapshotDescription snapshot;
+  protected final Configuration conf;
+  protected final FileSystem fs;
+  protected final Path rootDir;
+  private final Path snapshotDir;
+  protected final Path workingDir;
+  private final MasterSnapshotVerifier verifier;
+  protected final ForeignExceptionDispatcher monitor;
+
+  /**
+   * @param snapshot descriptor of the snapshot to take
+   * @param masterServices master services provider
+   * @throws IOException on unexpected error
+   */
+  public TakeSnapshotHandler(SnapshotDescription snapshot,
+      final MasterServices masterServices) throws IOException {
+    super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
+    assert snapshot != null : "SnapshotDescription must not be nul1";
+    assert masterServices != null : "MasterServices must not be nul1";
+
+    this.master = masterServices;
+    this.snapshot = snapshot;
+    this.conf = this.master.getConfiguration();
+    this.fs = this.master.getMasterFileSystem().getFileSystem();
+    this.rootDir = this.master.getMasterFileSystem().getRootDir();
+    this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+    this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+    this.monitor =  new ForeignExceptionDispatcher();
+
+    loadTableDescriptor(); // check that .tableinfo is present
+
+    // prepare the verify
+    this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
+  }
+
+  private HTableDescriptor loadTableDescriptor()
+      throws FileNotFoundException, IOException {
+    final String name = snapshot.getTable();
+    HTableDescriptor htd =
+      this.master.getTableDescriptors().get(name);
+    if (htd == null) {
+      throw new IOException("HTableDescriptor missing for " + name);
+    }
+    return htd;
+  }
+
+  /**
+   * Execute the core common portions of taking a snapshot. The {@link #snapshotRegions(List)}
+   * call should get implemented for each snapshot flavor.
+   */
+  @Override
+  public void process() {
+    LOG.info("Running table snapshot operation " + eventType + " on table " + snapshot.getTable());
+    try {
+      // If regions move after this meta scan, the region specific snapshot should fail, triggering
+      // an external exception that gets captured here.
+
+      // write down the snapshot info in the working directory
+      SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, this.fs);
+      new TableInfoCopyTask(monitor, snapshot, fs, rootDir).call();
+      monitor.rethrowException();
+
+      List<Pair<HRegionInfo, ServerName>> regionsAndLocations =
+          MetaReader.getTableRegionsAndLocations(this.server.getCatalogTracker(),
+            Bytes.toBytes(snapshot.getTable()), true);
+
+      // run the snapshot
+      snapshotRegions(regionsAndLocations);
+
+      // extract each pair to separate lists
+      Set<String> serverNames = new HashSet<String>();
+      for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
+        serverNames.add(p.getSecond().toString());
+      }
+
+      // verify the snapshot is valid
+      verifier.verifySnapshot(this.workingDir, serverNames);
+
+      // complete the snapshot, atomically moving from tmp to .snapshot dir.
+      completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
+    } catch (Exception e) {
+      String reason = "Failed taking snapshot " + SnapshotDescriptionUtils.toString(snapshot)
+          + " due to exception:" + e.getMessage();
+      LOG.error(reason, e);
+      ForeignException ee = new ForeignException(reason, e);
+      monitor.receive(ee);
+      // need to mark this completed to close off and allow cleanup to happen.
+      cancel("Failed to take snapshot '" + SnapshotDescriptionUtils.toString(snapshot)
+          + "' due to exception");
+    } finally {
+      LOG.debug("Launching cleanup of working dir:" + workingDir);
+      try {
+        // if the working dir is still present, the snapshot has failed.  it is present we delete
+        // it.
+        if (fs.exists(workingDir) && !this.fs.delete(workingDir, true)) {
+          LOG.error("Couldn't delete snapshot working directory:" + workingDir);
+        }
+      } catch (IOException e) {
+        LOG.error("Couldn't delete snapshot working directory:" + workingDir);
+      }
+    }
+  }
+
+  /**
+   * Reset the manager to allow another snapshot to proceed
+   *
+   * @param snapshotDir final path of the snapshot
+   * @param workingDir directory where the in progress snapshot was built
+   * @param fs {@link FileSystem} where the snapshot was built
+   * @throws SnapshotCreationException if the snapshot could not be moved
+   * @throws IOException the filesystem could not be reached
+   */
+  public void completeSnapshot(Path snapshotDir, Path workingDir, FileSystem fs)
+      throws SnapshotCreationException, IOException {
+    LOG.debug("Sentinel is done, just moving the snapshot from " + workingDir + " to "
+        + snapshotDir);
+    if (!fs.rename(workingDir, snapshotDir)) {
+      throw new SnapshotCreationException("Failed to move working directory(" + workingDir
+          + ") to completed directory(" + snapshotDir + ").");
+    }
+    finished = true;
+  }
+
+  /**
+   * Snapshot the specified regions
+   */
+  protected abstract void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regions)
+      throws IOException, KeeperException;
+
+  @Override
+  public void cancel(String why) {
+    if (finished) return;
+
+    this.finished = true;
+    LOG.info("Stop taking snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " because: "
+        + why);
+    CancellationException ce = new CancellationException(why);
+    monitor.receive(new ForeignException(master.getServerName().toString(), ce));
+  }
+
+  @Override
+  public boolean isFinished() {
+    return finished;
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public ForeignException getExceptionIfFailed() {
+    return monitor.getException();
+  }
+
+  @Override
+  public void rethrowException() throws ForeignException {
+    monitor.rethrowException();
+  }
+
+  @Override
+  public boolean hasException() {
+    return monitor.hasException();
+  }
+
+  @Override
+  public ForeignException getException() {
+    return monitor.getException();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java b/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
new file mode 100644
index 000000000000..533a89673b3a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
@@ -0,0 +1,377 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+
+import com.google.common.collect.Lists;
+
+/**
+ * A globally-barriered distributed procedure.  This class encapsulates state and methods for
+ * tracking and managing a distributed procedure, as well as aborting if any member encounters
+ * a problem or if a cancellation is requested.
+ * <p>
+ * All procedures first attempt to reach a barrier point with the {@link #sendGlobalBarrierStart()}
+ * method.  The procedure contacts all members and waits for all subprocedures to execute
+ * {@link Subprocedure#acquireBarrier} to acquire its local piece of the global barrier and then
+ * send acquisition info back to the coordinator.  If all acquisitions at subprocedures succeed,
+ * the coordinator then will call {@link #sendGlobalBarrierReached()}.  This notifies members to
+ * execute the {@link Subprocedure#insideBarrier()} method.  The procedure is blocked until all
+ * {@link Subprocedure#insideBarrier} executions complete at the members.  When
+ * {@link Subprocedure#insideBarrier} completes at each member, the member sends notification to
+ * the coordinator.  Once all members complete, the coordinator calls
+ * {@link #sendGlobalBarrierComplete()}.
+ * <p>
+ * If errors are encountered remotely, they are forwarded to the coordinator, and
+ * {@link Subprocedure#cleanup(Exception)} is called.
+ * <p>
+ * Each Procedure and each Subprocedure enforces a time limit on the execution time. If the time
+ * limit expires before the procedure completes the {@link TimeoutExceptionInjector} will trigger
+ * an {@link ForeignException} to abort the procedure.  This is particularly useful for situations
+ * when running a distributed {@link Subprocedure} so participants can avoid blocking for extreme
+ * amounts of time if one of the participants fails or takes a really long time (e.g. GC pause).
+ * <p>
+ * Users should generally not directly create or subclass instances of this.  They are created
+ * for them implicitly via {@link ProcedureCoordinator#startProcedure(ForeignExceptionDispatcher,
+ * String, byte[], List)}}
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Procedure implements Callable<Void>, ForeignExceptionListener {
+  private static final Log LOG = LogFactory.getLog(Procedure.class);
+
+  //
+  // Arguments and naming
+  //
+
+  // Name of the procedure
+  final private String procName;
+  // Arguments for this procedure execution
+  final private byte[] args;
+
+  //
+  // Execution State
+  //
+  /** latch for waiting until all members have acquire in barrier state */
+  final CountDownLatch acquiredBarrierLatch;
+  /** latch for waiting until all members have executed and released their in barrier state */
+  final CountDownLatch releasedBarrierLatch;
+  /** latch for waiting until a procedure has completed */
+  final CountDownLatch completedLatch;
+  /** monitor to check for errors */
+  private final ForeignExceptionDispatcher monitor;
+
+  //
+  // Execution Timeout Handling.
+  //
+
+  /** frequency to check for errors (ms) */
+  protected final long wakeFrequency;
+  protected final TimeoutExceptionInjector timeoutInjector;
+
+  //
+  // Members' and Coordinator's state
+  //
+
+  /** lock to prevent nodes from acquiring and then releasing before we can track them */
+  private Object joinBarrierLock = new Object();
+  private final List<String> acquiringMembers;
+  private final List<String> inBarrierMembers;
+  private ProcedureCoordinator coord;
+
+  /**
+   * Creates a procedure. (FOR TESTING)
+   *
+   * {@link Procedure} state to be run by a {@link ProcedureCoordinator}.
+   * @param coord coordinator to call back to for general errors (e.g.
+   *          {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}).
+   * @param monitor error monitor to check for external errors
+   * @param wakeFreq frequency to check for errors while waiting
+   * @param timeout amount of time to allow the procedure to run before cancelling
+   * @param procName name of the procedure instance
+   * @param args argument data associated with the procedure instance
+   * @param expectedMembers names of the expected members
+   */
+  public Procedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor, long wakeFreq,
+      long timeout, String procName, byte[] args, List<String> expectedMembers) {
+    this.coord = coord;
+    this.acquiringMembers = new ArrayList<String>(expectedMembers);
+    this.inBarrierMembers = new ArrayList<String>(acquiringMembers.size());
+    this.procName = procName;
+    this.args = args;
+    this.monitor = monitor;
+    this.wakeFrequency = wakeFreq;
+
+    int count = expectedMembers.size();
+    this.acquiredBarrierLatch = new CountDownLatch(count);
+    this.releasedBarrierLatch = new CountDownLatch(count);
+    this.completedLatch = new CountDownLatch(1);
+    this.timeoutInjector = new TimeoutExceptionInjector(monitor, timeout);
+  }
+
+  /**
+   * Create a procedure.
+   *
+   * Users should generally not directly create instances of this.  They are created them
+   * implicitly via {@link ProcedureCoordinator#createProcedure(ForeignExceptionDispatcher,
+   * String, byte[], List)}}
+   *
+   * @param coord coordinator to call back to for general errors (e.g.
+   *          {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}).
+   * @param wakeFreq frequency to check for errors while waiting
+   * @param timeout amount of time to allow the procedure to run before cancelling
+   * @param procName name of the procedure instance
+   * @param args argument data associated with the procedure instance
+   * @param expectedMembers names of the expected members
+   */
+  public Procedure(ProcedureCoordinator coord, long wakeFreq, long timeout,
+      String procName, byte[] args, List<String> expectedMembers) {
+    this(coord, new ForeignExceptionDispatcher(), wakeFreq, timeout, procName, args,
+        expectedMembers);
+  }
+
+  public String getName() {
+    return procName;
+  }
+
+  /**
+   * @return String of the procedure members both trying to enter the barrier and already in barrier
+   */
+  public String getStatus() {
+    String waiting, done;
+    synchronized (joinBarrierLock) {
+      waiting = acquiringMembers.toString();
+      done = inBarrierMembers.toString();
+    }
+    return "Procedure " + procName + " { waiting=" + waiting + " done="+ done + " }";
+  }
+  
+  /**
+   * Get the ForeignExceptionDispatcher
+   * @return the Procedure's monitor.
+   */
+  public ForeignExceptionDispatcher getErrorMonitor() {
+    return monitor;
+  }
+
+  /**
+   * This call is the main execution thread of the barriered procedure.  It sends messages and
+   * essentially blocks until all procedure members acquire or later complete but periodically
+   * checks for foreign exceptions.
+   */
+  @Override
+  @SuppressWarnings("finally")
+  final public Void call() {
+    LOG.info("Starting procedure '" + procName + "'");
+    // start the timer
+    timeoutInjector.start();
+
+    // run the procedure
+    try {
+      // start by checking for error first
+      monitor.rethrowException();
+      LOG.debug("Procedure '" + procName + "' starting 'acquire'");
+      sendGlobalBarrierStart();
+
+      // wait for all the members to report acquisition
+      LOG.debug("Waiting for all members to 'acquire'");
+      waitForLatch(acquiredBarrierLatch, monitor, wakeFrequency, "acquired");
+      monitor.rethrowException();
+
+      LOG.debug("Procedure '" + procName + "' starting 'in-barrier' execution.");
+      sendGlobalBarrierReached();
+
+      // wait for all members to report barrier release
+      waitForLatch(releasedBarrierLatch, monitor, wakeFrequency, "released");
+
+      // make sure we didn't get an error during in barrier execution and release
+      monitor.rethrowException();
+      LOG.info("Procedure '" + procName + "' execution completed");
+    } catch (Exception e) {
+      if (e instanceof InterruptedException) {
+        Thread.currentThread().interrupt();
+      }
+      String msg = "Procedure '" + procName +"' execution failed!";
+      LOG.error(msg, e);
+      receive(new ForeignException(getName(), e));
+    } finally {
+      LOG.debug("Running finish phase.");
+      sendGlobalBarrierComplete();
+      completedLatch.countDown();
+
+      // tell the timer we are done, if we get here successfully
+      timeoutInjector.complete();
+      return null;
+    }
+  }
+
+  /**
+   * Sends a message to Members to create a new {@link Subprocedure} for this Procedure and execute
+   * the {@link Subprocedure#acquireBarrier} step.
+   * @throws ForeignException
+   */
+  public void sendGlobalBarrierStart() throws ForeignException {
+    // start the procedure
+    LOG.debug("Starting procedure '" + procName + "', kicking off acquire phase on members.");
+    try {
+      // send procedure barrier start to specified list of members. cloning the list to avoid
+      // concurrent modification from the controller setting the prepared nodes
+      coord.getRpcs().sendGlobalBarrierAcquire(this, args, Lists.newArrayList(this.acquiringMembers));
+    } catch (IOException e) {
+      coord.rpcConnectionFailure("Can't reach controller.", e);
+    } catch (IllegalArgumentException e) {
+      throw new ForeignException(getName(), e);
+    }
+  }
+
+  /**
+   * Sends a message to all members that the global barrier condition has been satisfied.  This
+   * should only be executed after all members have completed its
+   * {@link Subprocedure#acquireBarrier()} call successfully.  This triggers the member
+   * {@link Subprocedure#insideBarrier} method.
+   * @throws ForeignException
+   */
+  public void sendGlobalBarrierReached() throws ForeignException {
+    try {
+      // trigger to have member run {@link Subprocedure#insideBarrier}
+      coord.getRpcs().sendGlobalBarrierReached(this, Lists.newArrayList(inBarrierMembers));
+    } catch (IOException e) {
+      coord.rpcConnectionFailure("Can't reach controller.", e);
+    }
+  }
+
+  /**
+   * Sends a message to members that all {@link Subprocedure#insideBarrier} calls have completed.
+   * After this executes, the coordinator can assume that any state resources about this barrier
+   * procedure state has been released.
+   */
+  public void sendGlobalBarrierComplete() {
+    LOG.debug("Finished coordinator procedure - removing self from list of running procedures");
+    try {
+      coord.getRpcs().resetMembers(this);
+    } catch (IOException e) {
+      coord.rpcConnectionFailure("Failed to reset procedure:" + procName, e);
+    }
+  }
+
+  //
+  // Call backs from other external processes.
+  //
+
+  /**
+   * Call back triggered by an individual member upon successful local barrier acquisition
+   * @param member
+   */
+  public void barrierAcquiredByMember(String member) {
+    LOG.debug("member: '" + member + "' joining prepared barrier for procedure '" + procName
+        + "' on coordinator");
+    if (this.acquiringMembers.contains(member)) {
+      synchronized (joinBarrierLock) {
+        if (this.acquiringMembers.remove(member)) {
+          this.inBarrierMembers.add(member);
+          acquiredBarrierLatch.countDown();
+        }
+      }
+      LOG.debug("Waiting on: " + acquiredBarrierLatch + " remaining members to acquire global barrier");
+    } else {
+      LOG.warn("Member " + member + " joined barrier, but we weren't waiting on it to join." +
+          " Continuing on.");
+    }
+  }
+
+  /**
+   * Call back triggered by a individual member upon successful local in-barrier execution and
+   * release
+   * @param member
+   */
+  public void barrierReleasedByMember(String member) {
+    boolean removed = false;
+    synchronized (joinBarrierLock) {
+      removed = this.inBarrierMembers.remove(member);
+      if (removed) {
+        releasedBarrierLatch.countDown();
+      }
+    }
+    if (removed) {
+      LOG.debug("Member: '" + member + "' released barrier for procedure'" + procName
+          + "', counting down latch.  Waiting for " + releasedBarrierLatch.getCount()
+          + " more");
+    } else {
+      LOG.warn("Member: '" + member + "' released barrier for procedure'" + procName
+          + "', but we weren't waiting on it to release!");
+    }
+  }
+
+  /**
+   * Waits until the entire procedure has globally completed, or has been aborted.
+   * @throws ForeignException
+   * @throws InterruptedException
+   */
+  public void waitForCompleted() throws ForeignException, InterruptedException {
+    waitForLatch(completedLatch, monitor, wakeFrequency, procName + " completed");
+  }
+
+  /**
+   * A callback that handles incoming ForeignExceptions.
+   */
+  @Override
+  public void receive(ForeignException e) {
+    monitor.receive(e);
+  }
+
+  /**
+   * Wait for latch to count to zero, ignoring any spurious wake-ups, but waking periodically to
+   * check for errors
+   * @param latch latch to wait on
+   * @param monitor monitor to check for errors while waiting
+   * @param wakeFrequency frequency to wake up and check for errors (in
+   *          {@link TimeUnit#MILLISECONDS})
+   * @param latchDescription description of the latch, for logging
+   * @throws ForeignException type of error the monitor can throw, if the task fails
+   * @throws InterruptedException if we are interrupted while waiting on latch
+   */
+  public static void waitForLatch(CountDownLatch latch, ForeignExceptionSnare monitor,
+      long wakeFrequency, String latchDescription) throws ForeignException,
+      InterruptedException {
+    boolean released = false;
+    while (!released) {
+      if (monitor != null) {
+        monitor.rethrowException();
+      }
+      /*
+      ForeignExceptionDispatcher.LOG.debug("Waiting for '" + latchDescription + "' latch. (sleep:"
+          + wakeFrequency + " ms)"); */
+      released = latch.await(wakeFrequency, TimeUnit.MILLISECONDS);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
new file mode 100644
index 000000000000..f9c62590a276
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.DaemonThreadFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * This is the master side of a distributed complex procedure execution.
+ * <p>
+ * The {@link Procedure} is generic and subclassing or customization shouldn't be
+ * necessary -- any customization should happen just in {@link Subprocedure}s.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureCoordinator {
+  private static final Log LOG = LogFactory.getLog(ProcedureCoordinator.class);
+
+  final static long TIMEOUT_MILLIS_DEFAULT = 60000;
+  final static long WAKE_MILLIS_DEFAULT = 500;
+
+  private final ProcedureCoordinatorRpcs rpcs;
+  private final ExecutorService pool;
+
+  // Running procedure table.  Maps procedure name to running procedure reference
+  private final ConcurrentMap<String, Procedure> procedures =
+      new MapMaker().concurrencyLevel(4).weakValues().makeMap();
+
+  /**
+   * Create and start a ProcedureCoordinator.
+   *
+   * The rpc object registers the ProcedureCoordinator and starts any threads in this
+   * constructor.
+   *
+   * @param rpcs
+   * @param pool Used for executing procedures.
+   */
+  public ProcedureCoordinator(ProcedureCoordinatorRpcs rpcs, ThreadPoolExecutor pool) {
+    this.rpcs = rpcs;
+    this.pool = pool;
+    this.rpcs.start(this);
+  }
+
+  /**
+   * Default thread pool for the procedure
+   */
+  public static ThreadPoolExecutor defaultPool(String coordName, long keepAliveTime, int opThreads,
+      long wakeFrequency) {
+    return new ThreadPoolExecutor(1, opThreads, keepAliveTime, TimeUnit.SECONDS,
+        new SynchronousQueue<Runnable>(),
+        new DaemonThreadFactory("(" + coordName + ")-proc-coordinator-pool"));
+  }
+
+  /**
+   * Shutdown the thread pools and release rpc resources
+   * @throws IOException
+   */
+  public void close() throws IOException {
+    // have to use shutdown now to break any latch waiting
+    pool.shutdownNow();
+    rpcs.close();
+  }
+
+  /**
+   * Submit an procedure to kick off its dependent subprocedures.
+   * @param proc Procedure to execute
+   * @return <tt>true</tt> if the procedure was started correctly, <tt>false</tt> if the
+   *         procedure or any subprocedures could not be started.  Failure could be due to
+   *         submitting a procedure multiple times (or one with the same name), or some sort
+   *         of IO problem.  On errors, the procedure's monitor holds a reference to the exception
+   *         that caused the failure.
+   */
+  boolean submitProcedure(Procedure proc) {
+    // if the submitted procedure was null, then we don't want to run it
+    if (proc == null) {
+      return false;
+    }
+    String procName = proc.getName();
+
+    // make sure we aren't already running a procedure of that name
+    synchronized (procedures) {
+      Procedure oldProc = procedures.get(procName);
+      if (oldProc != null) {
+        // procedures are always eventually completed on both successful and failed execution
+        if (oldProc.completedLatch.getCount() != 0) {
+          LOG.warn("Procedure " + procName + " currently running.  Rejecting new request");
+          return false;
+        }
+        LOG.debug("Procedure " + procName + " was in running list but was completed.  Accepting new attempt.");
+        procedures.remove(procName);
+      }
+    }
+
+    // kick off the procedure's execution in a separate thread
+    Future<Void> f = null;
+    try {
+      synchronized (procedures) {
+        f = this.pool.submit(proc);
+        // if everything got started properly, we can add it known running procedures
+        this.procedures.put(procName, proc);
+      }
+      return true;
+    } catch (RejectedExecutionException e) {
+      LOG.warn("Procedure " + procName + " rejected by execution pool.  Propagating error and " +
+          "cancelling operation.", e);
+      // the thread pool is full and we can't run the procedure
+      proc.receive(new ForeignException(procName, e));
+
+      // cancel procedure proactively
+      if (f != null) {
+        f.cancel(true);
+      }
+    }
+    return false;
+  }
+
+  /**
+   * The connection to the rest of the procedure group (members and coordinator) has been
+   * broken/lost/failed. This should fail any interested procedures, but not attempt to notify other
+   * members since we cannot reach them anymore.
+   * @param message description of the error
+   * @param cause the actual cause of the failure
+   */
+  void rpcConnectionFailure(final String message, final IOException cause) {
+    Collection<Procedure> toNotify = procedures.values();
+
+    for (Procedure proc : toNotify) {
+      if (proc == null) {
+        continue;
+      }
+      // notify the elements, if they aren't null
+      proc.receive(new ForeignException(proc.getName(), cause));
+    }
+  }
+
+  /**
+   * Abort the procedure with the given name
+   * @param procName name of the procedure to abort
+   * @param reason serialized information about the abort
+   */
+  public void abortProcedure(String procName, ForeignException reason) {
+    // if we know about the Procedure, notify it
+    synchronized(procedures) {
+      Procedure proc = procedures.get(procName);
+      if (proc == null) {
+        return;
+      }
+      proc.receive(reason);
+    }
+  }
+
+  /**
+   * Exposed for hooking with unit tests.
+   * @param procName
+   * @param procArgs
+   * @param expectedMembers
+   * @return
+   */
+  Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
+      List<String> expectedMembers) {
+    // build the procedure
+    return new Procedure(this, fed, WAKE_MILLIS_DEFAULT, TIMEOUT_MILLIS_DEFAULT,
+        procName, procArgs, expectedMembers);
+  }
+
+  /**
+   * Kick off the named procedure
+   * @param procName name of the procedure to start
+   * @param procArgs arguments for the procedure
+   * @param expectedMembers expected members to start
+   * @return handle to the running procedure, if it was started correctly, <tt>null</tt> otherwise
+   * @throws RejectedExecutionException if there are no more available threads to run the procedure
+   */
+  public Procedure startProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
+      List<String> expectedMembers) throws RejectedExecutionException {
+    Procedure proc = createProcedure(fed, procName, procArgs, expectedMembers);
+    if (!this.submitProcedure(proc)) {
+      LOG.error("Failed to submit procedure '" + procName + "'");
+      return null;
+    }
+    return proc;
+  }
+
+  /**
+   * Notification that the procedure had the specified member acquired its part of the barrier
+   * via {@link Subprocedure#acquireBarrier()}.
+   * @param procName name of the procedure that acquired
+   * @param member name of the member that acquired
+   */
+  void memberAcquiredBarrier(String procName, final String member) {
+    Procedure proc = procedures.get(procName);
+    if (proc != null) {
+      proc.barrierAcquiredByMember(member);
+    }
+  }
+
+  /**
+   * Notification that the procedure had another member finished executing its in-barrier subproc
+   * via {@link Subprocedure#insideBarrier()}.
+   * @param procName name of the subprocedure that finished
+   * @param member name of the member that executed and released its barrier
+   */
+  void memberFinishedBarrier(String procName, final String member) {
+    Procedure proc = procedures.get(procName);
+    if (proc != null) {
+      proc.barrierReleasedByMember(member);
+    }
+  }
+
+  /**
+   * @return the rpcs implementation for all current procedures
+   */
+  ProcedureCoordinatorRpcs getRpcs() {
+    return rpcs;
+  }
+
+  /**
+   * Returns the procedure.  This Procedure is a live instance so should not be modified but can
+   * be inspected.
+   * @param name Name of the procedure
+   * @return Procedure or null if not present any more
+   */
+  public Procedure getProcedure(String name) {
+    return procedures.get(name);
+  }
+
+  /**
+   * @return Return set of all procedure names.
+   */
+  public Set<String> getProcedureNames() {
+    return new HashSet<String>(procedures.keySet());
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java
new file mode 100644
index 000000000000..209c67107f3a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+/**
+ * RPCs for the coordinator to run a barriered procedure with subprocedures executed at
+ * distributed members.
+ * @see ProcedureCoordinator
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ProcedureCoordinatorRpcs extends Closeable {
+
+  /**
+   * Initialize and start threads necessary to connect an implementation's rpc mechanisms.
+   * @param listener
+   * @return true if succeed, false if encountered initialization errors.
+   */
+  public boolean start(final ProcedureCoordinator listener);
+
+  /**
+   * Notify the members that the coordinator has aborted the procedure and that it should release
+   * barrier resources.
+   *
+   * @param procName name of the procedure that was aborted
+   * @param cause the reason why the procedure needs to be aborted
+   * @throws IOException if the rpcs can't reach the other members of the procedure (and can't
+   *           recover).
+   */
+  public void sendAbortToMembers(Procedure procName, ForeignException cause) throws IOException;
+
+  /**
+   * Notify the members to acquire barrier for the procedure
+   *
+   * @param procName name of the procedure to start
+   * @param info information that should be passed to all members
+   * @param members names of the members requested to reach the acquired phase
+   * @throws IllegalArgumentException if the procedure was already marked as failed
+   * @throws IOException if we can't reach the remote notification mechanism
+   */
+  public void sendGlobalBarrierAcquire(Procedure procName, byte[] info, List<String> members)
+      throws IOException, IllegalArgumentException;
+
+  /**
+   * Notify members that all members have acquired their parts of the barrier and that they can
+   * now execute under the global barrier.
+   *
+   * Must come after calling {@link #sendGlobalBarrierAcquire(Procedure, byte[], List)}
+   *
+   * @param procName name of the procedure to start
+   * @param members members to tell we have reached in-barrier phase
+   * @throws IOException if we can't reach the remote notification mechanism
+   */
+  public void sendGlobalBarrierReached(Procedure procName, List<String> members) throws IOException;
+
+  /**
+   * Notify Members to reset the distributed state for procedure
+   * @param procName name of the procedure to reset
+   * @throws IOException if the remote notification mechanism cannot be reached
+   */
+  public void resetMembers(Procedure procName) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
new file mode 100644
index 000000000000..876f5c4a7063
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.DaemonThreadFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * Process to kick off and manage a running {@link Subprocedure} on a member. This is the
+ * specialized part of a {@link Procedure} that actually does procedure type-specific work
+ * and reports back to the coordinator as it completes each phase.
+ * <p>
+ * If there is a connection error ({@link #controllerConnectionFailure(String, IOException)}), all
+ * currently running subprocedures are notify to failed since there is no longer a way to reach any
+ * other members or coordinators since the rpcs are down.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureMember implements Closeable {
+  private static final Log LOG = LogFactory.getLog(ProcedureMember.class);
+
+  private final SubprocedureFactory builder;
+  private final ProcedureMemberRpcs rpcs;
+
+  private final ConcurrentMap<String,Subprocedure> subprocs =
+      new MapMaker().concurrencyLevel(4).weakValues().makeMap();
+  private final ExecutorService pool;
+
+  /**
+   * Instantiate a new ProcedureMember.  This is a slave that executes subprocedures.
+   *
+   * @param rpcs controller used to send notifications to the procedure coordinator
+   * @param pool thread pool to submit subprocedures
+   * @param factory class that creates instances of a subprocedure.
+   */
+  public ProcedureMember(ProcedureMemberRpcs rpcs, ThreadPoolExecutor pool,
+      SubprocedureFactory factory) {
+    this.pool = pool;
+    this.rpcs = rpcs;
+    this.builder = factory;
+  }
+
+  public static ThreadPoolExecutor defaultPool(long wakeFrequency, long keepAlive,
+      int procThreads, String memberName) {
+    return new ThreadPoolExecutor(1, procThreads, keepAlive, TimeUnit.SECONDS,
+        new SynchronousQueue<Runnable>(),
+        new DaemonThreadFactory("member: '" + memberName + "' subprocedure-pool"));
+  }
+
+  /**
+   * Package exposed.  Not for public use.
+   *
+   * @return reference to the Procedure member's rpcs object
+   */
+  ProcedureMemberRpcs getRpcs() {
+     return rpcs;
+  }
+
+
+  /**
+   * This is separated from execution so that we can detect and handle the case where the
+   * subprocedure is invalid and inactionable due to bad info (like DISABLED snapshot type being
+   * sent here)
+   * @param opName
+   * @param data
+   * @return subprocedure
+   */
+  public Subprocedure createSubprocedure(String opName, byte[] data) {
+    return builder.buildSubprocedure(opName, data);
+  }
+
+  /**
+   * Submit an subprocedure for execution.  This starts the local acquire phase.
+   * @param subproc the subprocedure to execute.
+   * @return <tt>true</tt> if the subprocedure was started correctly, <tt>false</tt> if it
+   *         could not be started. In the latter case, the subprocedure holds a reference to
+   *         the exception that caused the failure.
+   */
+  public boolean submitSubprocedure(Subprocedure subproc) {
+     // if the submitted subprocedure was null, bail.
+    if (subproc == null) {
+      LOG.warn("Submitted null subprocedure, nothing to run here.");
+      return false;
+    }
+
+    String procName = subproc.getName();
+    if (procName == null || procName.length() == 0) {
+      LOG.error("Subproc name cannot be null or the empty string");
+      return false;
+    }
+
+    // make sure we aren't already running an subprocedure of that name
+    Subprocedure rsub;
+    synchronized (subprocs) {
+      rsub = subprocs.get(procName);
+    }
+    if (rsub != null) {
+      if (!rsub.isComplete()) {
+        LOG.error("Subproc '" + procName + "' is already running. Bailing out");
+        return false;
+      }
+      LOG.warn("A completed old subproc "  +  procName + " is still present, removing");
+      subprocs.remove(procName);
+    }
+
+    LOG.debug("Submitting new Subprocedure:" + procName);
+
+    // kick off the subprocedure
+    Future<Void> future = null;
+    try {
+      future = this.pool.submit(subproc);
+      synchronized (subprocs) {
+        subprocs.put(procName, subproc);
+      }
+      return true;
+    } catch (RejectedExecutionException e) {
+      // the thread pool is full and we can't run the subprocedure
+      String msg = "Subprocedure pool is full!";
+      subproc.cancel(msg, e.getCause());
+
+      // cancel all subprocedures proactively
+      if (future != null) {
+        future.cancel(true);
+      }
+    }
+
+    LOG.error("Failed to start subprocedure '" + procName + "'");
+    return false;
+  }
+
+   /**
+    * Notification that procedure coordinator has reached the global barrier
+    * @param procName name of the subprocedure that should start running the the in-barrier phase
+    */
+   public void receivedReachedGlobalBarrier(String procName) {
+     Subprocedure subproc = subprocs.get(procName);
+     if (subproc == null) {
+       LOG.warn("Unexpected reached glabal barrier message for Sub-Procedure '" + procName + "'");
+       return;
+     }
+     subproc.receiveReachedGlobalBarrier();
+   }
+
+  /**
+   * Best effort attempt to close the threadpool via Thread.interrupt.
+   */
+  @Override
+  public void close() throws IOException {
+    // have to use shutdown now to break any latch waiting
+    pool.shutdownNow();
+  }
+
+  /**
+   * Shutdown the threadpool, and wait for upto timeoutMs millis before bailing
+   * @param timeoutMs timeout limit in millis
+   * @return true if successfully, false if bailed due to timeout.
+   * @throws InterruptedException
+   */
+  boolean closeAndWait(long timeoutMs) throws InterruptedException {
+    pool.shutdown();
+    return pool.awaitTermination(timeoutMs, TimeUnit.MILLISECONDS);
+  }
+
+  /**
+   * The connection to the rest of the procedure group (member and coordinator) has been
+   * broken/lost/failed. This should fail any interested subprocedure, but not attempt to notify
+   * other members since we cannot reach them anymore.
+   * @param message description of the error
+   * @param cause the actual cause of the failure
+   *
+   * TODO i'm tempted to just remove this code completely and treat it like any other abort.
+   * Implementation wise, if this happens it is a ZK failure which means the RS will abort.
+   */
+  public void controllerConnectionFailure(final String message, final IOException cause) {
+    Collection<Subprocedure> toNotify = subprocs.values();
+    LOG.error(message, cause);
+    for (Subprocedure sub : toNotify) {
+      // TODO notify the elements, if they aren't null
+      sub.cancel(message, cause);
+    }
+  }
+
+  /**
+   * Send abort to the specified procedure
+   * @param procName name of the procedure to about
+   * @param ee exception information about the abort
+   */
+  public void receiveAbortProcedure(String procName, ForeignException ee) {
+    LOG.debug("Request received to abort procedure " + procName, ee);
+    // if we know about the procedure, notify it
+    Subprocedure sub = subprocs.get(procName);
+    if (sub == null) {
+      LOG.info("Received abort on procedure with no local subprocedure " + procName +
+          ", ignoring it.", ee);
+      return; // Procedure has already completed
+    }
+    LOG.error("Propagating foreign exception to subprocedure " + sub.getName(), ee);
+    sub.monitor.receive(ee);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
new file mode 100644
index 000000000000..72cec2bdb2e7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+/**
+ * This is the notification interface for Procedures that encapsulates message passing from
+ * members to a coordinator.  Each of these calls should send a message to the coordinator.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ProcedureMemberRpcs extends Closeable {
+
+  /**
+   * Initialize and start any threads or connections the member needs.
+   */
+  public void start(ProcedureMember member);
+
+  /**
+   * Each subprocedure is being executed on a member.  This is the identifier for the member.
+   * @return the member name
+   */
+  public String getMemberName();
+
+  /**
+   * Notify the coordinator that we aborted the specified {@link Subprocedure}
+   *
+   * @param sub the {@link Subprocedure} we are aborting
+   * @param cause the reason why the member's subprocedure aborted
+   * @throws IOException thrown when the rpcs can't reach the other members of the procedure (and
+   *  thus can't recover).
+   */
+  public void sendMemberAborted(Subprocedure sub, ForeignException cause) throws IOException;
+
+  /**
+   * Notify the coordinator that the specified {@link Subprocedure} has acquired the locally required
+   * barrier condition.
+   *
+   * @param sub the specified {@link Subprocedure}
+   * @throws IOException if we can't reach the coordinator
+   */
+  public void sendMemberAcquired(Subprocedure sub) throws IOException;
+
+  /**
+   * Notify the coordinator that the specified {@link Subprocedure} has completed the work that
+   * needed to be done under the global barrier.
+   *
+   * @param sub the specified {@link Subprocedure}
+   * @throws IOException if we can't reach the coordinator
+   */
+  public void sendMemberCompleted(Subprocedure sub) throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java b/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
new file mode 100644
index 000000000000..ea669a0520c1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
@@ -0,0 +1,331 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+
+/**
+ * Distributed procedure member's Subprocedure.  A procedure is sarted on a ProcedureCoordinator
+ * which communicates with ProcedureMembers who create and start its part of the Procedure.  This
+ * sub part is called a Subprocedure
+ *
+ * Users should subclass this and implement {@link #acquireBarrier()} (get local barrier for this
+ * member), {@link #insideBarrier()} (execute while globally barriered and release barrier) and
+ * {@link #cleanup(Exception)} (release state associated with subprocedure.)
+ *
+ * When submitted to a ProcedureMemeber, the call method is executed in a separate thread.
+ * Latches are use too block its progress and trigger continuations when barrier conditions are
+ * met.
+ *
+ * Exception that makes it out of calls to {@link #acquireBarrier()} or {@link #insideBarrier()}
+ * gets converted into {@link ForeignException}, which will get propagated to the
+ * {@link ProcedureCoordinator}.
+ *
+ * There is a category of procedure (ex: online-snapshots), and a user-specified instance-specific
+ * barrierName. (ex: snapshot121126).
+ */
+abstract public class Subprocedure implements Callable<Void> {
+  private static final Log LOG = LogFactory.getLog(Subprocedure.class);
+
+  // Name of the procedure
+  final private String barrierName;
+
+  //
+  // Execution state
+  //
+
+  /** wait on before allowing the in barrier phase to proceed */
+  private final CountDownLatch inGlobalBarrier;
+  /** counted down when the Subprocedure has completed */
+  private final CountDownLatch releasedLocalBarrier;
+
+  //
+  // Error handling
+  //
+  /** monitor to check for errors */
+  protected final ForeignExceptionDispatcher monitor;
+  /** frequency to check for errors (ms) */
+  protected final long wakeFrequency;
+  protected final TimeoutExceptionInjector executionTimeoutTimer;
+  protected final ProcedureMemberRpcs rpcs;
+
+  private volatile boolean complete = false;
+
+  /**
+   * @param member reference to the member managing this subprocedure
+   * @param procName name of the procedure this subprocedure is associated with
+   * @param monitor notified if there is an error in the subprocedure
+   * @param wakeFrequency time in millis to wake to check if there is an error via the monitor (in
+   *          milliseconds).
+   * @param timeout time in millis that will trigger a subprocedure abort if it has not completed
+   */
+  public Subprocedure(ProcedureMember member, String procName, ForeignExceptionDispatcher monitor,
+      long wakeFrequency, long timeout) {
+    // Asserts should be caught during unit testing
+    assert member != null : "procedure member should be non-null";
+    assert member.getRpcs() != null : "rpc handlers should be non-null";
+    assert procName != null : "procedure name should be non-null";
+    assert monitor != null : "monitor should be non-null";
+
+    // Default to a very large timeout
+    this.rpcs = member.getRpcs();
+    this.barrierName = procName;
+    this.monitor = monitor;
+    // forward any failures to coordinator.  Since this is a dispatcher, resend loops should not be
+    // possible.
+    this.monitor.addListener(new ForeignExceptionListener() {
+      @Override
+      public void receive(ForeignException ee) {
+        // if this is a notification from a remote source, just log
+        if (ee.isRemote()) {
+          LOG.debug("Was remote foreign exception, not redispatching error", ee);
+          return;
+        }
+
+        // if it is local, then send it to the coordinator
+        try {
+          rpcs.sendMemberAborted(Subprocedure.this, ee);
+        } catch (IOException e) {
+          // this will fail all the running procedures, since the connection is down
+          LOG.error("Can't reach controller, not propagating error", e);
+        }
+      }
+    });
+
+    this.wakeFrequency = wakeFrequency;
+    this.inGlobalBarrier = new CountDownLatch(1);
+    this.releasedLocalBarrier = new CountDownLatch(1);
+
+    // accept error from timer thread, this needs to be started.
+    this.executionTimeoutTimer = new TimeoutExceptionInjector(monitor, timeout);
+  }
+
+  public String getName() {
+     return barrierName;
+  }
+
+  public String getMemberName() {
+    return rpcs.getMemberName();
+  }
+
+  private void rethrowException() throws ForeignException {
+    monitor.rethrowException();
+  }
+
+  /**
+   * Execute the Subprocedure {@link #acquireBarrier()} and {@link #insideBarrier()} methods
+   * while keeping some state for other threads to access.
+   *
+   * This would normally be executed by the ProcedureMemeber when a acquire message comes from the
+   * coordinator.  Rpcs are used to spend message back to the coordinator after different phases
+   * are executed.  Any exceptions caught during the execution (except for InterrupedException) get
+   * converted and propagated to coordinator via {@link ProcedureMemberRpcs#sendMemberAborted(
+   * Subprocedure, ForeignException)}.
+   */
+  @SuppressWarnings("finally")
+  final public Void call() {
+    LOG.debug("Starting subprocedure '" + barrierName + "' with timeout " +
+        executionTimeoutTimer.getMaxTime() + "ms");
+    // start the execution timeout timer
+    executionTimeoutTimer.start();
+
+    try {
+      // start by checking for error first
+      rethrowException();
+      LOG.debug("Subprocedure '" + barrierName + "' starting 'acquire' stage");
+      acquireBarrier();
+      LOG.debug("Subprocedure '" + barrierName + "' locally acquired");
+
+      // vote yes to coordinator about being prepared
+      rpcs.sendMemberAcquired(this);
+      LOG.debug("Subprocedure '" + barrierName + "' coordinator notified of 'acquire', waiting on" +
+          " 'reached' or 'abort' from coordinator");
+
+      // wait for the procedure to reach global barrier before proceding
+      waitForReachedGlobalBarrier();
+      rethrowException(); // if Coordinator aborts, will bail from here with exception
+
+      // In traditional 2PC, if a member reaches this state the TX has been committed and the
+      // member is responsible for rolling forward and recovering and completing the subsequent
+      // operations in the case of failure.  It cannot rollback.
+      //
+      // This implementation is not 2PC since it can still rollback here, and thus has different
+      // semantics.
+
+      LOG.debug("Subprocedure '" + barrierName + "' received 'reached' from coordinator.");
+      insideBarrier();
+      LOG.debug("Subprocedure '" + barrierName + "' locally completed");
+
+      // Ack that the member has executed and released local barrier
+      rpcs.sendMemberCompleted(this);
+      LOG.debug("Subprocedure '" + barrierName + "' has notified controller of completion");
+
+      // make sure we didn't get an external exception
+      rethrowException();
+    } catch (Exception e) {
+      String msg = null;
+      if (e instanceof InterruptedException) {
+        msg = "Procedure '" + barrierName + "' aborting due to interrupt!" +
+            " Likely due to pool shutdown.";
+        Thread.currentThread().interrupt();
+      } else if (e instanceof ForeignException) {
+        msg = "Subprocedure '" + barrierName + "' aborting due to a ForeignException!";
+      } else {
+        msg = "Subprocedure '" + barrierName + "' failed!";
+      }
+      LOG.error(msg , e);
+      cancel(msg, e);
+
+      LOG.debug("Subprocedure '" + barrierName + "' running cleanup.");
+      cleanup(e);
+    } finally {
+      releasedLocalBarrier.countDown();
+
+      // tell the timer we are done, if we get here successfully
+      executionTimeoutTimer.complete();
+      complete = true;
+      LOG.debug("Subprocedure '" + barrierName + "' completed.");
+      return null;
+    }
+  }
+
+  boolean isComplete() {
+    return complete;
+  }
+
+  /**
+   * exposed for testing.
+   */
+  ForeignExceptionSnare getErrorCheckable() {
+    return this.monitor;
+  }
+
+  /**
+   * The implementation of this method should gather and hold required resources (locks, disk
+   * space, etc) to satisfy the Procedures barrier condition.  For example, this would be where
+   * to make all the regions on a RS on the quiescent for an procedure that required all regions
+   * to be globally quiesed.
+   *
+   * Users should override this method.  If a quiescent is not required, this is overkill but
+   * can still be used to execute a procedure on all members and to propagate any exceptions.
+   *
+   * @throws ForeignException
+   */
+  abstract public void acquireBarrier() throws ForeignException;
+
+  /**
+   * The implementation of this method should act with the assumption that the barrier condition
+   * has been satisfied.  Continuing the previous example, a condition could be that all RS's
+   * globally have been quiesced, and procedures that require this precondition could be
+   * implemented here.
+   *
+   * Users should override this method.  If quiescense is not required, this can be a no-op
+   *
+   * @throws ForeignException
+   */
+  abstract public void insideBarrier() throws ForeignException;
+
+  /**
+   * Users should override this method. This implementation of this method should rollback and
+   * cleanup any temporary or partially completed state that the {@link #acquireBarrier()} may have
+   * created.
+   * @param e
+   */
+  abstract public void cleanup(Exception e);
+
+  /**
+   * Method to cancel the Subprocedure by injecting an exception from and external source.
+   * @param cause
+   */
+  public void cancel(String msg, Throwable cause) {
+    LOG.error(msg, cause);
+    if (cause instanceof ForeignException) {
+      monitor.receive((ForeignException) cause);
+    } else {
+      monitor.receive(new ForeignException(getMemberName(), cause));
+    }
+  }
+
+  /**
+   * Callback for the member rpcs to call when the global barrier has been reached.  This
+   * unblocks the main subprocedure exectuion thread so that the Subprocedure's
+   * {@link #insideBarrier()} method can be run.
+   */
+  public void receiveReachedGlobalBarrier() {
+    inGlobalBarrier.countDown();
+  }
+
+  //
+  // Subprocedure Internal State interface
+  //
+
+  /**
+   * Wait for the reached global barrier notification.
+   *
+   * Package visibility for testing
+   *
+   * @throws ForeignException
+   * @throws InterruptedException
+   */
+  void waitForReachedGlobalBarrier() throws ForeignException, InterruptedException {
+    Procedure.waitForLatch(inGlobalBarrier, monitor, wakeFrequency,
+        barrierName + ":remote acquired");
+  }
+
+  /**
+   * Waits until the entire procedure has globally completed, or has been aborted.
+   * @throws ForeignException
+   * @throws InterruptedException
+   */
+  public void waitForLocallyCompleted() throws ForeignException, InterruptedException {
+    Procedure.waitForLatch(releasedLocalBarrier, monitor, wakeFrequency,
+        barrierName + ":completed");
+  }
+
+  /**
+   * Empty Subprocedure for testing.
+   *
+   * Must be public for stubbing used in testing to work.
+   */
+  public static class SubprocedureImpl extends Subprocedure {
+
+    public SubprocedureImpl(ProcedureMember member, String opName,
+        ForeignExceptionDispatcher monitor, long wakeFrequency, long timeout) {
+      super(member, opName, monitor, wakeFrequency, timeout);
+    }
+
+    @Override
+    public void acquireBarrier() throws ForeignException {}
+
+    @Override
+    public void insideBarrier() throws ForeignException {}
+
+    @Override
+    public void cleanup(Exception e) {}
+  };
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java b/src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java
new file mode 100644
index 000000000000..0b94c89daed9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Task builder to build instances of a {@link ProcedureMember}'s {@link Subprocedure}s.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface SubprocedureFactory {
+
+  /**
+   * Build {@link Subprocedure} when requested.
+   * @param procName name of the procedure associated with this subprocedure
+   * @param procArgs  arguments passed from the coordinator about the procedure
+   * @return {@link Subprocedure} to run or <tt>null</tt> if the no operation should be run
+   * @throws IllegalArgumentException if the operation could not be run because of errors in the
+   *           request
+   * @throws IllegalStateException if the current runner cannot accept any more new requests
+   */
+  public Subprocedure buildSubprocedure(String procName, byte[] procArgs);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
new file mode 100644
index 000000000000..4595335be912
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * ZooKeeper based {@link ProcedureCoordinatorRpcs} for a {@link ProcedureCoordinator}
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ZKProcedureCoordinatorRpcs implements ProcedureCoordinatorRpcs {
+  public static final Log LOG = LogFactory.getLog(ZKProcedureCoordinatorRpcs.class);
+  private ZKProcedureUtil zkProc = null;
+  protected ProcedureCoordinator coordinator = null;  // if started this should be non-null
+
+  ZooKeeperWatcher watcher;
+  String procedureType;
+  String coordName;
+
+  /**
+   * @param watcher zookeeper watcher. Owned by <tt>this</tt> and closed via {@link #close()}
+   * @param procedureClass procedure type name is a category for when there are multiple kinds of
+   *    procedures.-- this becomes a znode so be aware of the naming restrictions
+   * @param coordName name of the node running the coordinator
+   * @throws KeeperException if an unexpected zk error occurs
+   */
+  public ZKProcedureCoordinatorRpcs(ZooKeeperWatcher watcher,
+      String procedureClass, String coordName) throws KeeperException {
+    this.watcher = watcher;
+    this.procedureType = procedureClass;
+    this.coordName = coordName;
+  }
+
+  /**
+   * The "acquire" phase.  The coordinator creates a new procType/acquired/ znode dir. If znodes
+   * appear, first acquire to relevant listener or sets watch waiting for notification of
+   * the acquire node
+   *
+   * @param proc the Procedure
+   * @param info data to be stored in the acquire node
+   * @param nodeNames children of the acquire phase
+   * @throws IOException if any failure occurs.
+   */
+  @Override
+  final public void sendGlobalBarrierAcquire(Procedure proc, byte[] info, List<String> nodeNames)
+      throws IOException, IllegalArgumentException {
+    String procName = proc.getName();
+    // start watching for the abort node
+    String abortNode = zkProc.getAbortZNode(procName);
+    try {
+      // check to see if the abort node already exists
+      if (ZKUtil.watchAndCheckExists(zkProc.getWatcher(), abortNode)) {
+        abort(abortNode);
+      }
+      // If we get an abort node watch triggered here, we'll go complete creating the acquired
+      // znode but then handle the acquire znode and bail out
+    } catch (KeeperException e) {
+      LOG.error("Failed to watch abort", e);
+      throw new IOException("Failed while watching abort node:" + abortNode, e);
+    }
+
+    // create the acquire barrier
+    String acquire = zkProc.getAcquiredBarrierNode(procName);
+    LOG.debug("Creating acquire znode:" + acquire);
+    try {
+      // notify all the procedure listeners to look for the acquire node
+      byte[] data = ProtobufUtil.prependPBMagic(info);
+      ZKUtil.createWithParents(zkProc.getWatcher(), acquire, data);
+      // loop through all the children of the acquire phase and watch for them
+      for (String node : nodeNames) {
+        String znode = ZKUtil.joinZNode(acquire, node);
+        LOG.debug("Watching for acquire node:" + znode);
+        if (ZKUtil.watchAndCheckExists(zkProc.getWatcher(), znode)) {
+          coordinator.memberAcquiredBarrier(procName, node);
+        }
+      }
+    } catch (KeeperException e) {
+      throw new IOException("Failed while creating acquire node:" + acquire, e);
+    }
+  }
+
+  @Override
+  public void sendGlobalBarrierReached(Procedure proc, List<String> nodeNames) throws IOException {
+    String procName = proc.getName();
+    String reachedNode = zkProc.getReachedBarrierNode(procName);
+    LOG.debug("Creating reached barrier zk node:" + reachedNode);
+    try {
+      // create the reached znode and watch for the reached znodes
+      ZKUtil.createWithParents(zkProc.getWatcher(), reachedNode);
+      // loop through all the children of the acquire phase and watch for them
+      for (String node : nodeNames) {
+        String znode = ZKUtil.joinZNode(reachedNode, node);
+        if (ZKUtil.watchAndCheckExists(zkProc.getWatcher(), znode)) {
+          coordinator.memberFinishedBarrier(procName, node);
+        }
+      }
+    } catch (KeeperException e) {
+      throw new IOException("Failed while creating reached node:" + reachedNode, e);
+    }
+  }
+
+
+  /**
+   * Delete znodes that are no longer in use.
+   */
+  @Override
+  final public void resetMembers(Procedure proc) throws IOException {
+    String procName = proc.getName();
+    boolean stillGettingNotifications = false;
+    do {
+      try {
+        LOG.debug("Attempting to clean out zk node for op:" + procName);
+        zkProc.clearZNodes(procName);
+        stillGettingNotifications = false;
+      } catch (KeeperException.NotEmptyException e) {
+        // recursive delete isn't transactional (yet) so we need to deal with cases where we get
+        // children trickling in
+        stillGettingNotifications = true;
+      } catch (KeeperException e) {
+        throw new IOException("Failed to complete reset procedure " + procName, e);
+      }
+    } while (stillGettingNotifications);
+  }
+
+  /**
+   * Start monitoring znodes in ZK - subclass hook to start monitoring znodes they are about.
+   * @return true if succeed, false if encountered initialization errors.
+   */
+  final public boolean start(final ProcedureCoordinator coordinator) {
+    if (this.coordinator != null) {
+      throw new IllegalStateException(
+        "ZKProcedureCoordinator already started and already has listener installed");
+    }
+    this.coordinator = coordinator;
+
+    try {
+      this.zkProc = new ZKProcedureUtil(watcher, procedureType, coordName) {
+        @Override
+        public void nodeCreated(String path) {
+          if (!isInProcedurePath(path)) return;
+          LOG.debug("Node created: " + path);
+          logZKTree(this.baseZNode);
+          if (isAcquiredPathNode(path)) {
+            // node wasn't present when we created the watch so zk event triggers acquire
+            coordinator.memberAcquiredBarrier(ZKUtil.getNodeName(ZKUtil.getParent(path)),
+              ZKUtil.getNodeName(path));
+          } else if (isReachedPathNode(path)) {
+            // node was absent when we created the watch so zk event triggers the finished barrier.
+
+            // TODO Nothing enforces that acquire and reached znodes from showing up in wrong order.
+            coordinator.memberFinishedBarrier(ZKUtil.getNodeName(ZKUtil.getParent(path)),
+              ZKUtil.getNodeName(path));
+          } else if (isAbortPathNode(path)) {
+            abort(path);
+          }
+        }
+      };
+      zkProc.clearChildZNodes();
+    } catch (KeeperException e) {
+      LOG.error("Unable to start the ZK-based Procedure Coordinator rpcs.", e);
+      return false;
+    }
+
+    LOG.debug("Starting the controller for procedure member:" + zkProc.getMemberName());
+    return true;
+  }
+
+  /**
+   * This is the abort message being sent by the coordinator to member
+   *
+   * TODO this code isn't actually used but can be used to issue a cancellation from the
+   * coordinator.
+   */
+  @Override
+  final public void sendAbortToMembers(Procedure proc, ForeignException ee) {
+    String procName = proc.getName();
+    LOG.debug("Aborting procedure '" + procName + "' in zk");
+    String procAbortNode = zkProc.getAbortZNode(procName);
+    try {
+      LOG.debug("Creating abort znode:" + procAbortNode);
+      String source = (ee.getSource() == null) ? coordName : ee.getSource();
+      byte[] errorInfo = ProtobufUtil.prependPBMagic(ForeignException.serialize(source, ee));
+      // first create the znode for the procedure
+      ZKUtil.createAndFailSilent(zkProc.getWatcher(), procAbortNode, errorInfo);
+      LOG.debug("Finished creating abort node:" + procAbortNode);
+    } catch (KeeperException e) {
+      // possible that we get this error for the procedure if we already reset the zk state, but in
+      // that case we should still get an error for that procedure anyways
+      zkProc.logZKTree(zkProc.baseZNode);
+      coordinator.rpcConnectionFailure("Failed to post zk node:" + procAbortNode
+          + " to abort procedure '" + procName + "'", new IOException(e));
+    }
+  }
+
+  /**
+   * Receive a notification and propagate it to the local coordinator
+   * @param abortNode full znode path to the failed procedure information
+   */
+  protected void abort(String abortNode) {
+    String procName = ZKUtil.getNodeName(abortNode);
+    ForeignException ee = null;
+    try {
+      byte[] data = ZKUtil.getData(zkProc.getWatcher(), abortNode);
+      if (!ProtobufUtil.isPBMagicPrefix(data)) {
+        LOG.warn("Got an error notification for op:" + abortNode
+            + " but we can't read the information. Killing the procedure.");
+        // we got a remote exception, but we can't describe it
+        ee = new ForeignException(coordName, "Data in abort node is illegally formatted.  ignoring content.");
+      } else {
+
+        data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
+        ee = ForeignException.deserialize(data);
+      }
+    } catch (InvalidProtocolBufferException e) {
+      LOG.warn("Got an error notification for op:" + abortNode
+          + " but we can't read the information. Killing the procedure.");
+      // we got a remote exception, but we can't describe it
+      ee = new ForeignException(coordName, e);
+    } catch (KeeperException e) {
+      coordinator.rpcConnectionFailure("Failed to get data for abort node:" + abortNode
+          + zkProc.getAbortZnode(), new IOException(e));
+    }
+    coordinator.abortProcedure(procName, ee);
+  }
+
+  @Override
+  final public void close() throws IOException {
+    zkProc.close();
+  }
+
+  /**
+   * Used in testing
+   */
+  final ZKProcedureUtil getZkProcedureUtil() {
+    return zkProc;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
new file mode 100644
index 000000000000..f5a552a88306
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * ZooKeeper based controller for a procedure member.
+ * <p>
+ * There can only be one {@link ZKProcedureMemberRpcs} per procedure type per member,
+ * since each procedure type is bound to a single set of znodes. You can have multiple
+ * {@link ZKProcedureMemberRpcs} on the same server, each serving a different member
+ * name, but each individual rpcs is still bound to a single member name (and since they are
+ * used to determine global progress, its important to not get this wrong).
+ * <p>
+ * To make this slightly more confusing, you can run multiple, concurrent procedures at the same
+ * time (as long as they have different types), from the same controller, but the same node name
+ * must be used for each procedure (though there is no conflict between the two procedure as long
+ * as they have distinct names).
+ * <p>
+ * There is no real error recovery with this mechanism currently -- if any the coordinator fails,
+ * its re-initialization will delete the znodes and require all in progress subprocedures to start
+ * anew.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
+
+  private static final Log LOG = LogFactory.getLog(ZKProcedureMemberRpcs.class);
+  private final String memberName;
+
+  protected ProcedureMember member;
+  private ZKProcedureUtil zkController;
+
+  /**
+   * Must call {@link #start(ProcedureMember)} before this can be used.
+   * @param watcher {@link ZooKeeperWatcher} to be owned by <tt>this</tt>. Closed via
+   *          {@link #close()}.
+   * @param procType name of the znode describing the procedure type
+   * @param memberName name of the member to join the procedure
+   * @throws KeeperException if we can't reach zookeeper
+   */
+  public ZKProcedureMemberRpcs(ZooKeeperWatcher watcher,
+      String procType, String memberName) throws KeeperException {
+    this.zkController = new ZKProcedureUtil(watcher, procType, memberName) {
+      @Override
+      public void nodeCreated(String path) {
+        if (!isInProcedurePath(path)) {
+          return;
+        }
+
+        LOG.info("Received created event:" + path);
+        // if it is a simple start/end/abort then we just rewatch the node
+        if (isAcquiredNode(path)) {
+          waitForNewProcedures();
+          return;
+        } else if (isAbortNode(path)) {
+          watchForAbortedProcedures();
+          return;
+        }
+        String parent = ZKUtil.getParent(path);
+        // if its the end barrier, the procedure can be completed
+        if (isReachedNode(parent)) {
+          receivedReachedGlobalBarrier(path);
+          return;
+        } else if (isAbortNode(parent)) {
+          abort(path);
+          return;
+        } else if (isAcquiredNode(parent)) {
+          startNewSubprocedure(path);
+        } else {
+          LOG.debug("Ignoring created notification for node:" + path);
+        }
+      }
+
+      @Override
+      public void nodeChildrenChanged(String path) {
+        if (path.equals(this.acquiredZnode)) {
+          LOG.info("Received procedure start children changed event: " + path);
+          waitForNewProcedures();
+        } else if (path.equals(this.abortZnode)) {
+          LOG.info("Received procedure abort children changed event: " + path);
+          watchForAbortedProcedures();
+        }
+      }
+    };
+    this.memberName = memberName;
+  }
+
+  public ZKProcedureUtil getZkController() {
+    return zkController;
+  }
+
+  @Override
+  public String getMemberName() {
+    return memberName;
+  }
+
+  /**
+   * Pass along the procedure global barrier notification to any listeners
+   * @param path full znode path that cause the notification
+   */
+  private void receivedReachedGlobalBarrier(String path) {
+    LOG.debug("Recieved reached global barrier:" + path);
+    String procName = ZKUtil.getNodeName(path);
+    this.member.receivedReachedGlobalBarrier(procName);
+  }
+
+  private void watchForAbortedProcedures() {
+    LOG.debug("Checking for aborted procedures on node: '" + zkController.getAbortZnode() + "'");
+    try {
+      // this is the list of the currently aborted procedues
+      for (String node : ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
+        zkController.getAbortZnode())) {
+        String abortNode = ZKUtil.joinZNode(zkController.getAbortZnode(), node);
+        abort(abortNode);
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to list children for abort node:"
+          + zkController.getAbortZnode(), new IOException(e));
+    }
+  }
+
+  private void waitForNewProcedures() {
+    // watch for new procedues that we need to start subprocedures for
+    LOG.debug("Looking for new procedures under znode:'" + zkController.getAcquiredBarrier() + "'");
+    List<String> runningProcedures = null;
+    try {
+      runningProcedures = ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
+        zkController.getAcquiredBarrier());
+      if (runningProcedures == null) {
+        LOG.debug("No running procedures.");
+        return;
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("General failure when watching for new procedures",
+        new IOException(e));
+    }
+    if (runningProcedures == null) {
+      LOG.debug("No running procedures.");
+      return;
+    }
+    for (String procName : runningProcedures) {
+      // then read in the procedure information
+      String path = ZKUtil.joinZNode(zkController.getAcquiredBarrier(), procName);
+      startNewSubprocedure(path);
+    }
+  }
+
+  /**
+   * Kick off a new sub-procedure on the listener with the data stored in the passed znode.
+   * <p>
+   * Will attempt to create the same procedure multiple times if an procedure znode with the same
+   * name is created. It is left up the coordinator to ensure this doesn't occur.
+   * @param path full path to the znode for the procedure to start
+   */
+  private synchronized void startNewSubprocedure(String path) {
+    LOG.debug("Found procedure znode: " + path);
+    String opName = ZKUtil.getNodeName(path);
+    // start watching for an abort notification for the procedure
+    String abortZNode = zkController.getAbortZNode(opName);
+    try {
+      if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), abortZNode)) {
+        LOG.debug("Not starting:" + opName + " because we already have an abort notification.");
+        return;
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to get the abort znode (" + abortZNode
+          + ") for procedure :" + opName, new IOException(e));
+      return;
+    }
+
+    // get the data for the procedure
+    Subprocedure subproc = null;
+    try {
+      byte[] data = ZKUtil.getData(zkController.getWatcher(), path);
+      LOG.debug("start proc data length is " + data.length);
+      if (!ProtobufUtil.isPBMagicPrefix(data)) {
+        String msg = "Data in for starting procuedure " + opName + " is illegally formatted. "
+            + "Killing the procedure.";
+        LOG.error(msg);
+        throw new IllegalArgumentException(msg);
+      }
+      data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
+      LOG.debug("Found data for znode:" + path);
+      subproc = member.createSubprocedure(opName, data);
+      member.submitSubprocedure(subproc);
+    } catch (IllegalArgumentException iae ) {
+      LOG.error("Illegal argument exception", iae);
+      sendMemberAborted(subproc, new ForeignException(getMemberName(), iae));
+    } catch (IllegalStateException ise) {
+      LOG.error("Illegal state exception ", ise);
+      sendMemberAborted(subproc, new ForeignException(getMemberName(), ise));
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to get data for new procedure:" + opName,
+        new IOException(e));
+    }
+  }
+
+  /**
+   * This attempts to create an acquired state znode for the procedure (snapshot name).
+   *
+   * It then looks for the reached znode to trigger in-barrier execution.  If not present we
+   * have a watcher, if present then trigger the in-barrier action.
+   */
+  @Override
+  public void sendMemberAcquired(Subprocedure sub) throws IOException {
+    String procName = sub.getName();
+    try {
+      LOG.debug("Member: '" + memberName + "' joining acquired barrier for procedure (" + procName
+          + ") in zk");
+      String acquiredZNode = ZKUtil.joinZNode(ZKProcedureUtil.getAcquireBarrierNode(
+        zkController, procName), memberName);
+      ZKUtil.createAndFailSilent(zkController.getWatcher(), acquiredZNode);
+
+      // watch for the complete node for this snapshot
+      String reachedBarrier = zkController.getReachedBarrierNode(procName);
+      LOG.debug("Watch for global barrier reached:" + reachedBarrier);
+      if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), reachedBarrier)) {
+        receivedReachedGlobalBarrier(reachedBarrier);
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to acquire barrier for procedure: "
+          + procName + " and member: " + memberName, new IOException(e));
+    }
+  }
+
+  /**
+   * This acts as the ack for a completed snapshot
+   */
+  @Override
+  public void sendMemberCompleted(Subprocedure sub) throws IOException {
+    String procName = sub.getName();
+    LOG.debug("Marking procedure  '" + procName + "' completed for member '" + memberName
+        + "' in zk");
+    String joinPath = ZKUtil.joinZNode(zkController.getReachedBarrierNode(procName), memberName);
+    try {
+      ZKUtil.createAndFailSilent(zkController.getWatcher(), joinPath);
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to post zk node:" + joinPath
+          + " to join procedure barrier.", new IOException(e));
+    }
+  }
+
+  /**
+   * This should be called by the member and should write a serialized root cause exception as
+   * to the abort znode.
+   */
+  @Override
+  public void sendMemberAborted(Subprocedure sub, ForeignException ee) {
+    if (sub == null) {
+      LOG.error("Failed due to null subprocedure", ee);
+      return;
+    }
+    String procName = sub.getName();
+    LOG.debug("Aborting procedure (" + procName + ") in zk");
+    String procAbortZNode = zkController.getAbortZNode(procName);
+    try {
+      String source = (ee.getSource() == null) ? memberName: ee.getSource();
+      byte[] errorInfo = ProtobufUtil.prependPBMagic(ForeignException.serialize(source, ee));
+      ZKUtil.createAndFailSilent(zkController.getWatcher(), procAbortZNode, errorInfo);
+      LOG.debug("Finished creating abort znode:" + procAbortZNode);
+    } catch (KeeperException e) {
+      // possible that we get this error for the procedure if we already reset the zk state, but in
+      // that case we should still get an error for that procedure anyways
+      zkController.logZKTree(zkController.getBaseZnode());
+      member.controllerConnectionFailure("Failed to post zk node:" + procAbortZNode
+          + " to abort procedure", new IOException(e));
+    }
+  }
+
+  /**
+   * Pass along the found abort notification to the listener
+   * @param abortZNode full znode path to the failed procedure information
+   */
+  protected void abort(String abortZNode) {
+    LOG.debug("Aborting procedure member for znode " + abortZNode);
+    String opName = ZKUtil.getNodeName(abortZNode);
+    try {
+      byte[] data = ZKUtil.getData(zkController.getWatcher(), abortZNode);
+
+      // figure out the data we need to pass
+      ForeignException ee;
+      try {
+        if (!ProtobufUtil.isPBMagicPrefix(data)) {
+          String msg = "Illegally formatted data in abort node for proc " + opName
+              + ".  Killing the procedure.";
+          LOG.error(msg);
+          // we got a remote exception, but we can't describe it so just return exn from here
+          ee = new ForeignException(getMemberName(), new IllegalArgumentException(msg));
+        } else {
+          data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
+          ee = ForeignException.deserialize(data);
+        }
+      } catch (InvalidProtocolBufferException e) {
+        LOG.warn("Got an error notification for op:" + opName
+            + " but we can't read the information. Killing the procedure.");
+        // we got a remote exception, but we can't describe it so just return exn from here
+        ee = new ForeignException(getMemberName(), e);
+      }
+
+      this.member.receiveAbortProcedure(opName, ee);
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to get data for abort znode:" + abortZNode
+          + zkController.getAbortZnode(), new IOException(e));
+    }
+  }
+
+  public void start(ProcedureMember listener) {
+    LOG.debug("Starting procedure member '" + this.memberName + "'");
+    this.member = listener;
+    watchForAbortedProcedures();
+    waitForNewProcedures();
+  }
+
+  @Override
+  public void close() throws IOException {
+    zkController.close();
+  }
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
new file mode 100644
index 000000000000..afc87f5de31a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This is a shared ZooKeeper-based znode management utils for distributed procedure.  All znode
+ * operations should go through the provided methods in coordinators and members.
+ *
+ * Layout of nodes in ZK is
+ * /hbase/[op name]/acquired/
+ *                    [op instance] - op data/
+ *                        /[nodes that have acquired]
+ *                 /reached/
+ *                    [op instance]/
+ *                        /[nodes that have completed]
+ *                 /abort/
+ *                    [op instance] - failure data
+ *
+ * NOTE: while acquired and completed are znode dirs, abort is actually just a znode.
+ *
+ * Assumption here that procedure names are unique
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class ZKProcedureUtil
+    extends ZooKeeperListener implements Closeable {
+
+  private static final Log LOG = LogFactory.getLog(ZKProcedureUtil.class);
+
+  public static final String ACQUIRED_BARRIER_ZNODE_DEFAULT = "acquired";
+  public static final String REACHED_BARRIER_ZNODE_DEFAULT = "reached";
+  public static final String ABORT_ZNODE_DEFAULT = "abort";
+
+  public final String baseZNode;
+  protected final String acquiredZnode;
+  protected final String reachedZnode;
+  protected final String abortZnode;
+
+  protected final String memberName;
+
+  /**
+   * Top-level watcher/controller for procedures across the cluster.
+   * <p>
+   * On instantiation, this ensures the procedure znodes exist.  This however requires the passed in
+   *  watcher has been started.
+   * @param watcher watcher for the cluster ZK. Owned by <tt>this</tt> and closed via
+   *          {@link #close()}
+   * @param procDescription name of the znode describing the procedure to run
+   * @param memberName name of the member from which we are interacting with running procedures
+   * @throws KeeperException when the procedure znodes cannot be created
+   */
+  public ZKProcedureUtil(ZooKeeperWatcher watcher, String procDescription,
+      String memberName) throws KeeperException {
+    super(watcher);
+    this.memberName = memberName;
+    // make sure we are listening for events
+    watcher.registerListener(this);
+    // setup paths for the zknodes used in procedures
+    this.baseZNode = ZKUtil.joinZNode(watcher.baseZNode, procDescription);
+    acquiredZnode = ZKUtil.joinZNode(baseZNode, ACQUIRED_BARRIER_ZNODE_DEFAULT);
+    reachedZnode = ZKUtil.joinZNode(baseZNode, REACHED_BARRIER_ZNODE_DEFAULT);
+    abortZnode = ZKUtil.joinZNode(baseZNode, ABORT_ZNODE_DEFAULT);
+
+    // first make sure all the ZK nodes exist
+    // make sure all the parents exist (sometimes not the case in tests)
+    ZKUtil.createWithParents(watcher, acquiredZnode);
+    // regular create because all the parents exist
+    ZKUtil.createAndFailSilent(watcher, reachedZnode);
+    ZKUtil.createAndFailSilent(watcher, abortZnode);
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (watcher != null) {
+      watcher.close();
+    }
+  }
+
+  public String getAcquiredBarrierNode(String opInstanceName) {
+    return ZKProcedureUtil.getAcquireBarrierNode(this, opInstanceName);
+  }
+
+  public String getReachedBarrierNode(String opInstanceName) {
+    return ZKProcedureUtil.getReachedBarrierNode(this, opInstanceName);
+  }
+
+  public String getAbortZNode(String opInstanceName) {
+    return ZKProcedureUtil.getAbortNode(this, opInstanceName);
+  }
+
+  public String getAbortZnode() {
+    return abortZnode;
+  }
+
+  public String getBaseZnode() {
+    return baseZNode;
+  }
+
+  public String getAcquiredBarrier() {
+    return acquiredZnode;
+  }
+
+  public String getMemberName() {
+    return memberName;
+  }
+
+  /**
+   * Get the full znode path for the node used by the coordinator to trigger a global barrier
+   * acquire on each subprocedure.
+   * @param controller controller running the procedure
+   * @param opInstanceName name of the running procedure instance (not the procedure description).
+   * @return full znode path to the prepare barrier/start node
+   */
+  public static String getAcquireBarrierNode(ZKProcedureUtil controller,
+      String opInstanceName) {
+    return ZKUtil.joinZNode(controller.acquiredZnode, opInstanceName);
+  }
+
+  /**
+   * Get the full znode path for the node used by the coordinator to trigger a global barrier
+   * execution and release on each subprocedure.
+   * @param controller controller running the procedure
+   * @param opInstanceName name of the running procedure instance (not the procedure description).
+   * @return full znode path to the commit barrier
+   */
+  public static String getReachedBarrierNode(ZKProcedureUtil controller,
+      String opInstanceName) {
+    return ZKUtil.joinZNode(controller.reachedZnode, opInstanceName);
+  }
+
+  /**
+   * Get the full znode path for the node used by the coordinator or member to trigger an abort
+   * of the global barrier acquisition or execution in subprocedures.
+   * @param controller controller running the procedure
+   * @param opInstanceName name of the running procedure instance (not the procedure description).
+   * @return full znode path to the abort znode
+   */
+  public static String getAbortNode(ZKProcedureUtil controller, String opInstanceName) {
+    return ZKUtil.joinZNode(controller.abortZnode, opInstanceName);
+  }
+
+  public ZooKeeperWatcher getWatcher() {
+    return watcher;
+  }
+
+  /**
+   * Is this a procedure related znode path?
+   *
+   * TODO: this is not strict, can return true if had name just starts with same prefix but is
+   * different zdir.
+   *
+   * @return true if starts with baseZnode
+   */
+  boolean isInProcedurePath(String path) {
+    return path.startsWith(baseZNode);
+  }
+
+  /**
+   * Is this the exact procedure barrier acquired znode
+   */
+  boolean isAcquiredNode(String path) {
+    return path.equals(acquiredZnode);
+  }
+
+  
+  /**
+   * Is this in the procedure barrier acquired znode path
+   */
+  boolean isAcquiredPathNode(String path) {
+    return path.startsWith(this.acquiredZnode) && !path.equals(acquiredZnode);
+  }
+
+  /**
+   * Is this the exact procedure barrier reached znode
+   */
+  boolean isReachedNode(String path) {
+    return path.equals(reachedZnode);
+  }
+
+  /**
+   * Is this in the procedure barrier reached znode path
+   */
+  boolean isReachedPathNode(String path) {
+    return path.startsWith(this.reachedZnode) && !path.equals(reachedZnode);
+  }
+
+
+  /**
+   * Is this in the procedure barrier abort znode path
+   */
+  boolean isAbortNode(String path) {
+    return path.equals(abortZnode);
+  }
+
+  /**
+   * Is this in the procedure barrier abort znode path
+   */
+  public boolean isAbortPathNode(String path) {
+    return path.startsWith(this.abortZnode) && !path.equals(abortZnode);
+  }
+
+  // --------------------------------------------------------------------------
+  // internal debugging methods
+  // --------------------------------------------------------------------------
+  /**
+   * Recursively print the current state of ZK (non-transactional)
+   * @param root name of the root directory in zk to print
+   * @throws KeeperException
+   */
+  void logZKTree(String root) {
+    if (!LOG.isDebugEnabled()) return;
+    LOG.debug("Current zk system:");
+    String prefix = "|-";
+    LOG.debug(prefix + root);
+    try {
+      logZKTree(root, prefix);
+    } catch (KeeperException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Helper method to print the current state of the ZK tree.
+   * @see #logZKTree(String)
+   * @throws KeeperException if an unexpected exception occurs
+   */
+  protected void logZKTree(String root, String prefix) throws KeeperException {
+    List<String> children = ZKUtil.listChildrenNoWatch(watcher, root);
+    if (children == null) return;
+    for (String child : children) {
+      LOG.debug(prefix + child);
+      String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
+      logZKTree(node, prefix + "---");
+    }
+  }
+
+  public void clearChildZNodes() throws KeeperException {
+    // TODO This is potentially racy since not atomic. update when we support zk that has multi
+    LOG.info("Clearing all procedure znodes: " + acquiredZnode + " " + reachedZnode + " "
+        + abortZnode);
+
+    // If the coordinator was shutdown mid-procedure, then we are going to lose
+    // an procedure that was previously started by cleaning out all the previous state. Its much
+    // harder to figure out how to keep an procedure going and the subject of HBASE-5487.
+    ZKUtil.deleteChildrenRecursively(watcher, acquiredZnode);
+    ZKUtil.deleteChildrenRecursively(watcher, reachedZnode);
+    ZKUtil.deleteChildrenRecursively(watcher, abortZnode);
+  }
+
+  public void clearZNodes(String procedureName) throws KeeperException {
+    // TODO This is potentially racy since not atomic. update when we support zk that has multi
+    LOG.info("Clearing all znodes for procedure " + procedureName + "including nodes "
+        + acquiredZnode + " " + reachedZnode + " " + abortZnode);
+    ZKUtil.deleteNodeRecursively(watcher, getAcquiredBarrierNode(procedureName));
+    ZKUtil.deleteNodeRecursively(watcher, getReachedBarrierNode(procedureName));
+    ZKUtil.deleteNodeRecursively(watcher, getAbortZNode(procedureName));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
new file mode 100644
index 000000000000..65c87d34ff1f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.protobuf;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Protobufs utility.
+ */
+@SuppressWarnings("deprecation")
+public final class ProtobufUtil {
+
+  private ProtobufUtil() {
+  }
+
+  /**
+   * Magic we put ahead of a serialized protobuf message.
+   * For example, all znode content is protobuf messages with the below magic
+   * for preamble.
+   */
+  public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
+  private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
+
+  /**
+   * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
+   * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
+   * @param bytes Bytes to decorate
+   * @return The passed <code>bytes</codes> with magic prepended (Creates a new
+   * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
+   */
+  public static byte [] prependPBMagic(final byte [] bytes) {
+    return Bytes.add(PB_MAGIC, bytes);
+  }
+
+  /**
+   * @param bytes Bytes to check.
+   * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
+   */
+  public static boolean isPBMagicPrefix(final byte [] bytes) {
+    if (bytes == null || bytes.length < PB_MAGIC.length) return false;
+    return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, 0, PB_MAGIC.length) == 0;
+  }
+
+  /**
+   * @return Length of {@link #PB_MAGIC}
+   */
+  public static int lengthOfPBMagic() {
+    return PB_MAGIC.length;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java
new file mode 100644
index 000000000000..067321f6ddc0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java
@@ -0,0 +1,2185 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: ErrorHandling.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class ErrorHandlingProtos {
+  private ErrorHandlingProtos() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public interface StackTraceElementMessageOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string declaringClass = 1;
+    boolean hasDeclaringClass();
+    String getDeclaringClass();
+    
+    // optional string methodName = 2;
+    boolean hasMethodName();
+    String getMethodName();
+    
+    // optional string fileName = 3;
+    boolean hasFileName();
+    String getFileName();
+    
+    // optional int32 lineNumber = 4;
+    boolean hasLineNumber();
+    int getLineNumber();
+  }
+  public static final class StackTraceElementMessage extends
+      com.google.protobuf.GeneratedMessage
+      implements StackTraceElementMessageOrBuilder {
+    // Use StackTraceElementMessage.newBuilder() to construct.
+    private StackTraceElementMessage(Builder builder) {
+      super(builder);
+    }
+    private StackTraceElementMessage(boolean noInit) {}
+    
+    private static final StackTraceElementMessage defaultInstance;
+    public static StackTraceElementMessage getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public StackTraceElementMessage getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string declaringClass = 1;
+    public static final int DECLARINGCLASS_FIELD_NUMBER = 1;
+    private java.lang.Object declaringClass_;
+    public boolean hasDeclaringClass() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDeclaringClass() {
+      java.lang.Object ref = declaringClass_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          declaringClass_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDeclaringClassBytes() {
+      java.lang.Object ref = declaringClass_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        declaringClass_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string methodName = 2;
+    public static final int METHODNAME_FIELD_NUMBER = 2;
+    private java.lang.Object methodName_;
+    public boolean hasMethodName() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getMethodName() {
+      java.lang.Object ref = methodName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          methodName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getMethodNameBytes() {
+      java.lang.Object ref = methodName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        methodName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string fileName = 3;
+    public static final int FILENAME_FIELD_NUMBER = 3;
+    private java.lang.Object fileName_;
+    public boolean hasFileName() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public String getFileName() {
+      java.lang.Object ref = fileName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          fileName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getFileNameBytes() {
+      java.lang.Object ref = fileName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        fileName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int32 lineNumber = 4;
+    public static final int LINENUMBER_FIELD_NUMBER = 4;
+    private int lineNumber_;
+    public boolean hasLineNumber() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public int getLineNumber() {
+      return lineNumber_;
+    }
+    
+    private void initFields() {
+      declaringClass_ = "";
+      methodName_ = "";
+      fileName_ = "";
+      lineNumber_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDeclaringClassBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getMethodNameBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, getFileNameBytes());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeInt32(4, lineNumber_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDeclaringClassBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getMethodNameBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, getFileNameBytes());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(4, lineNumber_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage other = (org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage) obj;
+      
+      boolean result = true;
+      result = result && (hasDeclaringClass() == other.hasDeclaringClass());
+      if (hasDeclaringClass()) {
+        result = result && getDeclaringClass()
+            .equals(other.getDeclaringClass());
+      }
+      result = result && (hasMethodName() == other.hasMethodName());
+      if (hasMethodName()) {
+        result = result && getMethodName()
+            .equals(other.getMethodName());
+      }
+      result = result && (hasFileName() == other.hasFileName());
+      if (hasFileName()) {
+        result = result && getFileName()
+            .equals(other.getFileName());
+      }
+      result = result && (hasLineNumber() == other.hasLineNumber());
+      if (hasLineNumber()) {
+        result = result && (getLineNumber()
+            == other.getLineNumber());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasDeclaringClass()) {
+        hash = (37 * hash) + DECLARINGCLASS_FIELD_NUMBER;
+        hash = (53 * hash) + getDeclaringClass().hashCode();
+      }
+      if (hasMethodName()) {
+        hash = (37 * hash) + METHODNAME_FIELD_NUMBER;
+        hash = (53 * hash) + getMethodName().hashCode();
+      }
+      if (hasFileName()) {
+        hash = (37 * hash) + FILENAME_FIELD_NUMBER;
+        hash = (53 * hash) + getFileName().hashCode();
+      }
+      if (hasLineNumber()) {
+        hash = (37 * hash) + LINENUMBER_FIELD_NUMBER;
+        hash = (53 * hash) + getLineNumber();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        declaringClass_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        methodName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        fileName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000004);
+        lineNumber_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage build() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage result = new org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.declaringClass_ = declaringClass_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.methodName_ = methodName_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.fileName_ = fileName_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.lineNumber_ = lineNumber_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance()) return this;
+        if (other.hasDeclaringClass()) {
+          setDeclaringClass(other.getDeclaringClass());
+        }
+        if (other.hasMethodName()) {
+          setMethodName(other.getMethodName());
+        }
+        if (other.hasFileName()) {
+          setFileName(other.getFileName());
+        }
+        if (other.hasLineNumber()) {
+          setLineNumber(other.getLineNumber());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              declaringClass_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              methodName_ = input.readBytes();
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              fileName_ = input.readBytes();
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              lineNumber_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string declaringClass = 1;
+      private java.lang.Object declaringClass_ = "";
+      public boolean hasDeclaringClass() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDeclaringClass() {
+        java.lang.Object ref = declaringClass_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          declaringClass_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDeclaringClass(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        declaringClass_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDeclaringClass() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        declaringClass_ = getDefaultInstance().getDeclaringClass();
+        onChanged();
+        return this;
+      }
+      void setDeclaringClass(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        declaringClass_ = value;
+        onChanged();
+      }
+      
+      // optional string methodName = 2;
+      private java.lang.Object methodName_ = "";
+      public boolean hasMethodName() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getMethodName() {
+        java.lang.Object ref = methodName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          methodName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setMethodName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        methodName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearMethodName() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        methodName_ = getDefaultInstance().getMethodName();
+        onChanged();
+        return this;
+      }
+      void setMethodName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        methodName_ = value;
+        onChanged();
+      }
+      
+      // optional string fileName = 3;
+      private java.lang.Object fileName_ = "";
+      public boolean hasFileName() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public String getFileName() {
+        java.lang.Object ref = fileName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          fileName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setFileName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        fileName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFileName() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        fileName_ = getDefaultInstance().getFileName();
+        onChanged();
+        return this;
+      }
+      void setFileName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000004;
+        fileName_ = value;
+        onChanged();
+      }
+      
+      // optional int32 lineNumber = 4;
+      private int lineNumber_ ;
+      public boolean hasLineNumber() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public int getLineNumber() {
+        return lineNumber_;
+      }
+      public Builder setLineNumber(int value) {
+        bitField0_ |= 0x00000008;
+        lineNumber_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearLineNumber() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        lineNumber_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:StackTraceElementMessage)
+    }
+    
+    static {
+      defaultInstance = new StackTraceElementMessage(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:StackTraceElementMessage)
+  }
+  
+  public interface GenericExceptionMessageOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string className = 1;
+    boolean hasClassName();
+    String getClassName();
+    
+    // optional string message = 2;
+    boolean hasMessage();
+    String getMessage();
+    
+    // optional bytes errorInfo = 3;
+    boolean hasErrorInfo();
+    com.google.protobuf.ByteString getErrorInfo();
+    
+    // repeated .StackTraceElementMessage trace = 4;
+    java.util.List<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage> 
+        getTraceList();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getTrace(int index);
+    int getTraceCount();
+    java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> 
+        getTraceOrBuilderList();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder getTraceOrBuilder(
+        int index);
+  }
+  public static final class GenericExceptionMessage extends
+      com.google.protobuf.GeneratedMessage
+      implements GenericExceptionMessageOrBuilder {
+    // Use GenericExceptionMessage.newBuilder() to construct.
+    private GenericExceptionMessage(Builder builder) {
+      super(builder);
+    }
+    private GenericExceptionMessage(boolean noInit) {}
+    
+    private static final GenericExceptionMessage defaultInstance;
+    public static GenericExceptionMessage getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public GenericExceptionMessage getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string className = 1;
+    public static final int CLASSNAME_FIELD_NUMBER = 1;
+    private java.lang.Object className_;
+    public boolean hasClassName() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getClassName() {
+      java.lang.Object ref = className_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          className_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClassNameBytes() {
+      java.lang.Object ref = className_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        className_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string message = 2;
+    public static final int MESSAGE_FIELD_NUMBER = 2;
+    private java.lang.Object message_;
+    public boolean hasMessage() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getMessage() {
+      java.lang.Object ref = message_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          message_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getMessageBytes() {
+      java.lang.Object ref = message_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        message_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional bytes errorInfo = 3;
+    public static final int ERRORINFO_FIELD_NUMBER = 3;
+    private com.google.protobuf.ByteString errorInfo_;
+    public boolean hasErrorInfo() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public com.google.protobuf.ByteString getErrorInfo() {
+      return errorInfo_;
+    }
+    
+    // repeated .StackTraceElementMessage trace = 4;
+    public static final int TRACE_FIELD_NUMBER = 4;
+    private java.util.List<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage> trace_;
+    public java.util.List<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage> getTraceList() {
+      return trace_;
+    }
+    public java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> 
+        getTraceOrBuilderList() {
+      return trace_;
+    }
+    public int getTraceCount() {
+      return trace_.size();
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getTrace(int index) {
+      return trace_.get(index);
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder getTraceOrBuilder(
+        int index) {
+      return trace_.get(index);
+    }
+    
+    private void initFields() {
+      className_ = "";
+      message_ = "";
+      errorInfo_ = com.google.protobuf.ByteString.EMPTY;
+      trace_ = java.util.Collections.emptyList();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getClassNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getMessageBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, errorInfo_);
+      }
+      for (int i = 0; i < trace_.size(); i++) {
+        output.writeMessage(4, trace_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getClassNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getMessageBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, errorInfo_);
+      }
+      for (int i = 0; i < trace_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(4, trace_.get(i));
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage other = (org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage) obj;
+      
+      boolean result = true;
+      result = result && (hasClassName() == other.hasClassName());
+      if (hasClassName()) {
+        result = result && getClassName()
+            .equals(other.getClassName());
+      }
+      result = result && (hasMessage() == other.hasMessage());
+      if (hasMessage()) {
+        result = result && getMessage()
+            .equals(other.getMessage());
+      }
+      result = result && (hasErrorInfo() == other.hasErrorInfo());
+      if (hasErrorInfo()) {
+        result = result && getErrorInfo()
+            .equals(other.getErrorInfo());
+      }
+      result = result && getTraceList()
+          .equals(other.getTraceList());
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasClassName()) {
+        hash = (37 * hash) + CLASSNAME_FIELD_NUMBER;
+        hash = (53 * hash) + getClassName().hashCode();
+      }
+      if (hasMessage()) {
+        hash = (37 * hash) + MESSAGE_FIELD_NUMBER;
+        hash = (53 * hash) + getMessage().hashCode();
+      }
+      if (hasErrorInfo()) {
+        hash = (37 * hash) + ERRORINFO_FIELD_NUMBER;
+        hash = (53 * hash) + getErrorInfo().hashCode();
+      }
+      if (getTraceCount() > 0) {
+        hash = (37 * hash) + TRACE_FIELD_NUMBER;
+        hash = (53 * hash) + getTraceList().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getTraceFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        className_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        message_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        errorInfo_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        if (traceBuilder_ == null) {
+          trace_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000008);
+        } else {
+          traceBuilder_.clear();
+        }
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage build() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage result = new org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.className_ = className_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.message_ = message_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.errorInfo_ = errorInfo_;
+        if (traceBuilder_ == null) {
+          if (((bitField0_ & 0x00000008) == 0x00000008)) {
+            trace_ = java.util.Collections.unmodifiableList(trace_);
+            bitField0_ = (bitField0_ & ~0x00000008);
+          }
+          result.trace_ = trace_;
+        } else {
+          result.trace_ = traceBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance()) return this;
+        if (other.hasClassName()) {
+          setClassName(other.getClassName());
+        }
+        if (other.hasMessage()) {
+          setMessage(other.getMessage());
+        }
+        if (other.hasErrorInfo()) {
+          setErrorInfo(other.getErrorInfo());
+        }
+        if (traceBuilder_ == null) {
+          if (!other.trace_.isEmpty()) {
+            if (trace_.isEmpty()) {
+              trace_ = other.trace_;
+              bitField0_ = (bitField0_ & ~0x00000008);
+            } else {
+              ensureTraceIsMutable();
+              trace_.addAll(other.trace_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.trace_.isEmpty()) {
+            if (traceBuilder_.isEmpty()) {
+              traceBuilder_.dispose();
+              traceBuilder_ = null;
+              trace_ = other.trace_;
+              bitField0_ = (bitField0_ & ~0x00000008);
+              traceBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getTraceFieldBuilder() : null;
+            } else {
+              traceBuilder_.addAllMessages(other.trace_);
+            }
+          }
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              className_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              message_ = input.readBytes();
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              errorInfo_ = input.readBytes();
+              break;
+            }
+            case 34: {
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder subBuilder = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addTrace(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string className = 1;
+      private java.lang.Object className_ = "";
+      public boolean hasClassName() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getClassName() {
+        java.lang.Object ref = className_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          className_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClassName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        className_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClassName() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        className_ = getDefaultInstance().getClassName();
+        onChanged();
+        return this;
+      }
+      void setClassName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        className_ = value;
+        onChanged();
+      }
+      
+      // optional string message = 2;
+      private java.lang.Object message_ = "";
+      public boolean hasMessage() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getMessage() {
+        java.lang.Object ref = message_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          message_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setMessage(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        message_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearMessage() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        message_ = getDefaultInstance().getMessage();
+        onChanged();
+        return this;
+      }
+      void setMessage(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        message_ = value;
+        onChanged();
+      }
+      
+      // optional bytes errorInfo = 3;
+      private com.google.protobuf.ByteString errorInfo_ = com.google.protobuf.ByteString.EMPTY;
+      public boolean hasErrorInfo() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public com.google.protobuf.ByteString getErrorInfo() {
+        return errorInfo_;
+      }
+      public Builder setErrorInfo(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        errorInfo_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearErrorInfo() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        errorInfo_ = getDefaultInstance().getErrorInfo();
+        onChanged();
+        return this;
+      }
+      
+      // repeated .StackTraceElementMessage trace = 4;
+      private java.util.List<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage> trace_ =
+        java.util.Collections.emptyList();
+      private void ensureTraceIsMutable() {
+        if (!((bitField0_ & 0x00000008) == 0x00000008)) {
+          trace_ = new java.util.ArrayList<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage>(trace_);
+          bitField0_ |= 0x00000008;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> traceBuilder_;
+      
+      public java.util.List<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage> getTraceList() {
+        if (traceBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(trace_);
+        } else {
+          return traceBuilder_.getMessageList();
+        }
+      }
+      public int getTraceCount() {
+        if (traceBuilder_ == null) {
+          return trace_.size();
+        } else {
+          return traceBuilder_.getCount();
+        }
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getTrace(int index) {
+        if (traceBuilder_ == null) {
+          return trace_.get(index);
+        } else {
+          return traceBuilder_.getMessage(index);
+        }
+      }
+      public Builder setTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage value) {
+        if (traceBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTraceIsMutable();
+          trace_.set(index, value);
+          onChanged();
+        } else {
+          traceBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder builderForValue) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          traceBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addTrace(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage value) {
+        if (traceBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTraceIsMutable();
+          trace_.add(value);
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage value) {
+        if (traceBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTraceIsMutable();
+          trace_.add(index, value);
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addTrace(
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder builderForValue) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.add(builderForValue.build());
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder builderForValue) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllTrace(
+          java.lang.Iterable<? extends org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage> values) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          super.addAll(values, trace_);
+          onChanged();
+        } else {
+          traceBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearTrace() {
+        if (traceBuilder_ == null) {
+          trace_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000008);
+          onChanged();
+        } else {
+          traceBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeTrace(int index) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.remove(index);
+          onChanged();
+        } else {
+          traceBuilder_.remove(index);
+        }
+        return this;
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder getTraceBuilder(
+          int index) {
+        return getTraceFieldBuilder().getBuilder(index);
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder getTraceOrBuilder(
+          int index) {
+        if (traceBuilder_ == null) {
+          return trace_.get(index);  } else {
+          return traceBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> 
+           getTraceOrBuilderList() {
+        if (traceBuilder_ != null) {
+          return traceBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(trace_);
+        }
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder addTraceBuilder() {
+        return getTraceFieldBuilder().addBuilder(
+            org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance());
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder addTraceBuilder(
+          int index) {
+        return getTraceFieldBuilder().addBuilder(
+            index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance());
+      }
+      public java.util.List<org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder> 
+           getTraceBuilderList() {
+        return getTraceFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> 
+          getTraceFieldBuilder() {
+        if (traceBuilder_ == null) {
+          traceBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder>(
+                  trace_,
+                  ((bitField0_ & 0x00000008) == 0x00000008),
+                  getParentForChildren(),
+                  isClean());
+          trace_ = null;
+        }
+        return traceBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:GenericExceptionMessage)
+    }
+    
+    static {
+      defaultInstance = new GenericExceptionMessage(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:GenericExceptionMessage)
+  }
+  
+  public interface ForeignExceptionMessageOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string source = 1;
+    boolean hasSource();
+    String getSource();
+    
+    // optional .GenericExceptionMessage genericException = 2;
+    boolean hasGenericException();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getGenericException();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder getGenericExceptionOrBuilder();
+  }
+  public static final class ForeignExceptionMessage extends
+      com.google.protobuf.GeneratedMessage
+      implements ForeignExceptionMessageOrBuilder {
+    // Use ForeignExceptionMessage.newBuilder() to construct.
+    private ForeignExceptionMessage(Builder builder) {
+      super(builder);
+    }
+    private ForeignExceptionMessage(boolean noInit) {}
+    
+    private static final ForeignExceptionMessage defaultInstance;
+    public static ForeignExceptionMessage getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public ForeignExceptionMessage getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string source = 1;
+    public static final int SOURCE_FIELD_NUMBER = 1;
+    private java.lang.Object source_;
+    public boolean hasSource() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getSource() {
+      java.lang.Object ref = source_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          source_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getSourceBytes() {
+      java.lang.Object ref = source_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        source_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional .GenericExceptionMessage genericException = 2;
+    public static final int GENERICEXCEPTION_FIELD_NUMBER = 2;
+    private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage genericException_;
+    public boolean hasGenericException() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getGenericException() {
+      return genericException_;
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder getGenericExceptionOrBuilder() {
+      return genericException_;
+    }
+    
+    private void initFields() {
+      source_ = "";
+      genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getSourceBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeMessage(2, genericException_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getSourceBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(2, genericException_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage other = (org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage) obj;
+      
+      boolean result = true;
+      result = result && (hasSource() == other.hasSource());
+      if (hasSource()) {
+        result = result && getSource()
+            .equals(other.getSource());
+      }
+      result = result && (hasGenericException() == other.hasGenericException());
+      if (hasGenericException()) {
+        result = result && getGenericException()
+            .equals(other.getGenericException());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasSource()) {
+        hash = (37 * hash) + SOURCE_FIELD_NUMBER;
+        hash = (53 * hash) + getSource().hashCode();
+      }
+      if (hasGenericException()) {
+        hash = (37 * hash) + GENERICEXCEPTION_FIELD_NUMBER;
+        hash = (53 * hash) + getGenericException().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getGenericExceptionFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        source_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        if (genericExceptionBuilder_ == null) {
+          genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+        } else {
+          genericExceptionBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage build() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage result = new org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.source_ = source_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        if (genericExceptionBuilder_ == null) {
+          result.genericException_ = genericException_;
+        } else {
+          result.genericException_ = genericExceptionBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance()) return this;
+        if (other.hasSource()) {
+          setSource(other.getSource());
+        }
+        if (other.hasGenericException()) {
+          mergeGenericException(other.getGenericException());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              source_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder subBuilder = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.newBuilder();
+              if (hasGenericException()) {
+                subBuilder.mergeFrom(getGenericException());
+              }
+              input.readMessage(subBuilder, extensionRegistry);
+              setGenericException(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string source = 1;
+      private java.lang.Object source_ = "";
+      public boolean hasSource() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getSource() {
+        java.lang.Object ref = source_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          source_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setSource(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        source_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSource() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        source_ = getDefaultInstance().getSource();
+        onChanged();
+        return this;
+      }
+      void setSource(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        source_ = value;
+        onChanged();
+      }
+      
+      // optional .GenericExceptionMessage genericException = 2;
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder> genericExceptionBuilder_;
+      public boolean hasGenericException() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getGenericException() {
+        if (genericExceptionBuilder_ == null) {
+          return genericException_;
+        } else {
+          return genericExceptionBuilder_.getMessage();
+        }
+      }
+      public Builder setGenericException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage value) {
+        if (genericExceptionBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          genericException_ = value;
+          onChanged();
+        } else {
+          genericExceptionBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder setGenericException(
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder builderForValue) {
+        if (genericExceptionBuilder_ == null) {
+          genericException_ = builderForValue.build();
+          onChanged();
+        } else {
+          genericExceptionBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder mergeGenericException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage value) {
+        if (genericExceptionBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002) &&
+              genericException_ != org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance()) {
+            genericException_ =
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.newBuilder(genericException_).mergeFrom(value).buildPartial();
+          } else {
+            genericException_ = value;
+          }
+          onChanged();
+        } else {
+          genericExceptionBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder clearGenericException() {
+        if (genericExceptionBuilder_ == null) {
+          genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+          onChanged();
+        } else {
+          genericExceptionBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder getGenericExceptionBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getGenericExceptionFieldBuilder().getBuilder();
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder getGenericExceptionOrBuilder() {
+        if (genericExceptionBuilder_ != null) {
+          return genericExceptionBuilder_.getMessageOrBuilder();
+        } else {
+          return genericException_;
+        }
+      }
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder> 
+          getGenericExceptionFieldBuilder() {
+        if (genericExceptionBuilder_ == null) {
+          genericExceptionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder>(
+                  genericException_,
+                  getParentForChildren(),
+                  isClean());
+          genericException_ = null;
+        }
+        return genericExceptionBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:ForeignExceptionMessage)
+    }
+    
+    static {
+      defaultInstance = new ForeignExceptionMessage(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:ForeignExceptionMessage)
+  }
+  
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_StackTraceElementMessage_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_StackTraceElementMessage_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_GenericExceptionMessage_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_GenericExceptionMessage_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_ForeignExceptionMessage_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_ForeignExceptionMessage_fieldAccessorTable;
+  
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\023ErrorHandling.proto\"l\n\030StackTraceEleme" +
+      "ntMessage\022\026\n\016declaringClass\030\001 \001(\t\022\022\n\nmet" +
+      "hodName\030\002 \001(\t\022\020\n\010fileName\030\003 \001(\t\022\022\n\nlineN" +
+      "umber\030\004 \001(\005\"z\n\027GenericExceptionMessage\022\021" +
+      "\n\tclassName\030\001 \001(\t\022\017\n\007message\030\002 \001(\t\022\021\n\ter" +
+      "rorInfo\030\003 \001(\014\022(\n\005trace\030\004 \003(\0132\031.StackTrac" +
+      "eElementMessage\"]\n\027ForeignExceptionMessa" +
+      "ge\022\016\n\006source\030\001 \001(\t\0222\n\020genericException\030\002" +
+      " \001(\0132\030.GenericExceptionMessageBF\n*org.ap" +
+      "ache.hadoop.hbase.protobuf.generatedB\023Er",
+      "rorHandlingProtosH\001\240\001\001"
+    };
+    com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+      new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+        public com.google.protobuf.ExtensionRegistry assignDescriptors(
+            com.google.protobuf.Descriptors.FileDescriptor root) {
+          descriptor = root;
+          internal_static_StackTraceElementMessage_descriptor =
+            getDescriptor().getMessageTypes().get(0);
+          internal_static_StackTraceElementMessage_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_StackTraceElementMessage_descriptor,
+              new java.lang.String[] { "DeclaringClass", "MethodName", "FileName", "LineNumber", },
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.class,
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder.class);
+          internal_static_GenericExceptionMessage_descriptor =
+            getDescriptor().getMessageTypes().get(1);
+          internal_static_GenericExceptionMessage_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_GenericExceptionMessage_descriptor,
+              new java.lang.String[] { "ClassName", "Message", "ErrorInfo", "Trace", },
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.class,
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder.class);
+          internal_static_ForeignExceptionMessage_descriptor =
+            getDescriptor().getMessageTypes().get(2);
+          internal_static_ForeignExceptionMessage_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_ForeignExceptionMessage_descriptor,
+              new java.lang.String[] { "Source", "GenericException", },
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.class,
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder.class);
+          return null;
+        }
+      };
+    com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        }, assigner);
+  }
+  
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
new file mode 100644
index 000000000000..84abf0151d7f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
@@ -0,0 +1,851 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: hbase.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class HBaseProtos {
+  private HBaseProtos() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public interface SnapshotDescriptionOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // required string name = 1;
+    boolean hasName();
+    String getName();
+    
+    // optional string table = 2;
+    boolean hasTable();
+    String getTable();
+    
+    // optional int64 creationTime = 3 [default = 0];
+    boolean hasCreationTime();
+    long getCreationTime();
+    
+    // optional .SnapshotDescription.Type type = 4 [default = FLUSH];
+    boolean hasType();
+    org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type getType();
+    
+    // optional int32 version = 5;
+    boolean hasVersion();
+    int getVersion();
+  }
+  public static final class SnapshotDescription extends
+      com.google.protobuf.GeneratedMessage
+      implements SnapshotDescriptionOrBuilder {
+    // Use SnapshotDescription.newBuilder() to construct.
+    private SnapshotDescription(Builder builder) {
+      super(builder);
+    }
+    private SnapshotDescription(boolean noInit) {}
+    
+    private static final SnapshotDescription defaultInstance;
+    public static SnapshotDescription getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public SnapshotDescription getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_fieldAccessorTable;
+    }
+    
+    public enum Type
+        implements com.google.protobuf.ProtocolMessageEnum {
+      DISABLED(0, 0),
+      FLUSH(1, 1),
+      ;
+      
+      public static final int DISABLED_VALUE = 0;
+      public static final int FLUSH_VALUE = 1;
+      
+      
+      public final int getNumber() { return value; }
+      
+      public static Type valueOf(int value) {
+        switch (value) {
+          case 0: return DISABLED;
+          case 1: return FLUSH;
+          default: return null;
+        }
+      }
+      
+      public static com.google.protobuf.Internal.EnumLiteMap<Type>
+          internalGetValueMap() {
+        return internalValueMap;
+      }
+      private static com.google.protobuf.Internal.EnumLiteMap<Type>
+          internalValueMap =
+            new com.google.protobuf.Internal.EnumLiteMap<Type>() {
+              public Type findValueByNumber(int number) {
+                return Type.valueOf(number);
+              }
+            };
+      
+      public final com.google.protobuf.Descriptors.EnumValueDescriptor
+          getValueDescriptor() {
+        return getDescriptor().getValues().get(index);
+      }
+      public final com.google.protobuf.Descriptors.EnumDescriptor
+          getDescriptorForType() {
+        return getDescriptor();
+      }
+      public static final com.google.protobuf.Descriptors.EnumDescriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDescriptor().getEnumTypes().get(0);
+      }
+      
+      private static final Type[] VALUES = {
+        DISABLED, FLUSH, 
+      };
+      
+      public static Type valueOf(
+          com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+        if (desc.getType() != getDescriptor()) {
+          throw new java.lang.IllegalArgumentException(
+            "EnumValueDescriptor is not for this type.");
+        }
+        return VALUES[desc.getIndex()];
+      }
+      
+      private final int index;
+      private final int value;
+      
+      private Type(int index, int value) {
+        this.index = index;
+        this.value = value;
+      }
+      
+      // @@protoc_insertion_point(enum_scope:SnapshotDescription.Type)
+    }
+    
+    private int bitField0_;
+    // required string name = 1;
+    public static final int NAME_FIELD_NUMBER = 1;
+    private java.lang.Object name_;
+    public boolean hasName() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getName() {
+      java.lang.Object ref = name_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          name_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getNameBytes() {
+      java.lang.Object ref = name_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        name_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string table = 2;
+    public static final int TABLE_FIELD_NUMBER = 2;
+    private java.lang.Object table_;
+    public boolean hasTable() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getTable() {
+      java.lang.Object ref = table_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          table_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getTableBytes() {
+      java.lang.Object ref = table_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        table_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 creationTime = 3 [default = 0];
+    public static final int CREATIONTIME_FIELD_NUMBER = 3;
+    private long creationTime_;
+    public boolean hasCreationTime() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getCreationTime() {
+      return creationTime_;
+    }
+    
+    // optional .SnapshotDescription.Type type = 4 [default = FLUSH];
+    public static final int TYPE_FIELD_NUMBER = 4;
+    private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type type_;
+    public boolean hasType() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type getType() {
+      return type_;
+    }
+    
+    // optional int32 version = 5;
+    public static final int VERSION_FIELD_NUMBER = 5;
+    private int version_;
+    public boolean hasVersion() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public int getVersion() {
+      return version_;
+    }
+    
+    private void initFields() {
+      name_ = "";
+      table_ = "";
+      creationTime_ = 0L;
+      type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+      version_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      if (!hasName()) {
+        memoizedIsInitialized = 0;
+        return false;
+      }
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getTableBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(3, creationTime_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeEnum(4, type_.getNumber());
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeInt32(5, version_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getTableBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(3, creationTime_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(4, type_.getNumber());
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(5, version_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription other = (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription) obj;
+      
+      boolean result = true;
+      result = result && (hasName() == other.hasName());
+      if (hasName()) {
+        result = result && getName()
+            .equals(other.getName());
+      }
+      result = result && (hasTable() == other.hasTable());
+      if (hasTable()) {
+        result = result && getTable()
+            .equals(other.getTable());
+      }
+      result = result && (hasCreationTime() == other.hasCreationTime());
+      if (hasCreationTime()) {
+        result = result && (getCreationTime()
+            == other.getCreationTime());
+      }
+      result = result && (hasType() == other.hasType());
+      if (hasType()) {
+        result = result &&
+            (getType() == other.getType());
+      }
+      result = result && (hasVersion() == other.hasVersion());
+      if (hasVersion()) {
+        result = result && (getVersion()
+            == other.getVersion());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasName()) {
+        hash = (37 * hash) + NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getName().hashCode();
+      }
+      if (hasTable()) {
+        hash = (37 * hash) + TABLE_FIELD_NUMBER;
+        hash = (53 * hash) + getTable().hashCode();
+      }
+      if (hasCreationTime()) {
+        hash = (37 * hash) + CREATIONTIME_FIELD_NUMBER;
+        hash = (53 * hash) + hashLong(getCreationTime());
+      }
+      if (hasType()) {
+        hash = (37 * hash) + TYPE_FIELD_NUMBER;
+        hash = (53 * hash) + hashEnum(getType());
+      }
+      if (hasVersion()) {
+        hash = (37 * hash) + VERSION_FIELD_NUMBER;
+        hash = (53 * hash) + getVersion();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptionOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        name_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        table_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        creationTime_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        version_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000010);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription build() {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription result = new org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.name_ = name_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.table_ = table_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.creationTime_ = creationTime_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.type_ = type_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.version_ = version_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDefaultInstance()) return this;
+        if (other.hasName()) {
+          setName(other.getName());
+        }
+        if (other.hasTable()) {
+          setTable(other.getTable());
+        }
+        if (other.hasCreationTime()) {
+          setCreationTime(other.getCreationTime());
+        }
+        if (other.hasType()) {
+          setType(other.getType());
+        }
+        if (other.hasVersion()) {
+          setVersion(other.getVersion());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        if (!hasName()) {
+          
+          return false;
+        }
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              name_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              table_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              creationTime_ = input.readInt64();
+              break;
+            }
+            case 32: {
+              int rawValue = input.readEnum();
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type value = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(4, rawValue);
+              } else {
+                bitField0_ |= 0x00000008;
+                type_ = value;
+              }
+              break;
+            }
+            case 40: {
+              bitField0_ |= 0x00000010;
+              version_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // required string name = 1;
+      private java.lang.Object name_ = "";
+      public boolean hasName() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getName() {
+        java.lang.Object ref = name_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          name_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        name_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearName() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        name_ = getDefaultInstance().getName();
+        onChanged();
+        return this;
+      }
+      void setName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        name_ = value;
+        onChanged();
+      }
+      
+      // optional string table = 2;
+      private java.lang.Object table_ = "";
+      public boolean hasTable() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getTable() {
+        java.lang.Object ref = table_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          table_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setTable(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        table_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTable() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        table_ = getDefaultInstance().getTable();
+        onChanged();
+        return this;
+      }
+      void setTable(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        table_ = value;
+        onChanged();
+      }
+      
+      // optional int64 creationTime = 3 [default = 0];
+      private long creationTime_ ;
+      public boolean hasCreationTime() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public long getCreationTime() {
+        return creationTime_;
+      }
+      public Builder setCreationTime(long value) {
+        bitField0_ |= 0x00000004;
+        creationTime_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearCreationTime() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        creationTime_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional .SnapshotDescription.Type type = 4 [default = FLUSH];
+      private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+      public boolean hasType() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type getType() {
+        return type_;
+      }
+      public Builder setType(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000008;
+        type_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearType() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 version = 5;
+      private int version_ ;
+      public boolean hasVersion() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public int getVersion() {
+        return version_;
+      }
+      public Builder setVersion(int value) {
+        bitField0_ |= 0x00000010;
+        version_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearVersion() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        version_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:SnapshotDescription)
+    }
+    
+    static {
+      defaultInstance = new SnapshotDescription(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:SnapshotDescription)
+  }
+  
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_SnapshotDescription_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_SnapshotDescription_fieldAccessorTable;
+  
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\013hbase.proto\"\255\001\n\023SnapshotDescription\022\014\n" +
+      "\004name\030\001 \002(\t\022\r\n\005table\030\002 \001(\t\022\027\n\014creationTi" +
+      "me\030\003 \001(\003:\0010\022.\n\004type\030\004 \001(\0162\031.SnapshotDesc" +
+      "ription.Type:\005FLUSH\022\017\n\007version\030\005 \001(\005\"\037\n\004" +
+      "Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001B>\n*org.apa" +
+      "che.hadoop.hbase.protobuf.generatedB\013HBa" +
+      "seProtosH\001\240\001\001"
+    };
+    com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+      new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+        public com.google.protobuf.ExtensionRegistry assignDescriptors(
+            com.google.protobuf.Descriptors.FileDescriptor root) {
+          descriptor = root;
+          internal_static_SnapshotDescription_descriptor =
+            getDescriptor().getMessageTypes().get(0);
+          internal_static_SnapshotDescription_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_SnapshotDescription_descriptor,
+              new java.lang.String[] { "Name", "Table", "CreationTime", "Type", "Version", },
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.class,
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Builder.class);
+          return null;
+        }
+      };
+    com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        }, assigner);
+  }
+  
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index db72c084bbca..8f85fcc6713c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -96,6 +96,7 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.coprocessor.Exec;
 import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterBase;
@@ -111,12 +112,14 @@
 import org.apache.hadoop.hbase.ipc.RpcCallContext;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.metrics.OperationMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -751,9 +754,32 @@ public long addAndGetGlobalMemstoreSize(long memStoreSize) {
    * @throws IOException
    */
   private void checkRegioninfoOnFilesystem() throws IOException {
-    Path regioninfoPath = new Path(this.regiondir, REGIONINFO_FILE);
-    if (this.fs.exists(regioninfoPath) &&
-        this.fs.getFileStatus(regioninfoPath).getLen() > 0) {
+    checkRegioninfoOnFilesystem(this.regiondir);
+  }
+
+  /**
+   * Write out an info file under the region directory. Useful recovering mangled regions.
+   * @param regiondir directory under which to write out the region info
+   * @throws IOException
+   */
+  private void checkRegioninfoOnFilesystem(Path regiondir) throws IOException {
+    writeRegioninfoOnFilesystem(regionInfo, regiondir, getFilesystem(), conf);
+  }
+
+  /**
+   * Write out an info file under the region directory. Useful recovering mangled regions. If the
+   * regioninfo already exists on disk and there is information in the file, then we fast exit.
+   * @param regionInfo information about the region
+   * @param regiondir directory under which to write out the region info
+   * @param fs {@link FileSystem} on which to write the region info
+   * @param conf {@link Configuration} from which to extract specific file locations
+   * @throws IOException on unexpected error.
+   */
+  public static void writeRegioninfoOnFilesystem(HRegionInfo regionInfo, Path regiondir,
+      FileSystem fs, Configuration conf) throws IOException {
+    Path regioninfoPath = new Path(regiondir, REGIONINFO_FILE);
+    if (fs.exists(regioninfoPath) &&
+        fs.getFileStatus(regioninfoPath).getLen() > 0) {
       return;
     }
     // Create in tmpdir and then move into place in case we crash after
@@ -766,7 +792,7 @@ private void checkRegioninfoOnFilesystem() throws IOException {
         HConstants.DATA_FILE_UMASK_KEY);
 
     // and then create the file
-    Path tmpPath = new Path(getTmpDir(), REGIONINFO_FILE);
+    Path tmpPath = new Path(getTmpDir(regiondir), REGIONINFO_FILE);
 
     // if datanode crashes or if the RS goes down just before the close is called while trying to
     // close the created regioninfo file in the .tmp directory then on next
@@ -779,10 +805,10 @@ private void checkRegioninfoOnFilesystem() throws IOException {
     FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
 
     try {
-      this.regionInfo.write(out);
+      regionInfo.write(out);
       out.write('\n');
       out.write('\n');
-      out.write(Bytes.toBytes(this.regionInfo.toString()));
+      out.write(Bytes.toBytes(regionInfo.toString()));
     } finally {
       out.close();
     }
@@ -1199,7 +1225,11 @@ private void cleanupTmpDir() throws IOException {
    * will have its contents removed when the region is reopened.
    */
   Path getTmpDir() {
-    return new Path(getRegionDir(), REGION_TEMP_SUBDIR);
+    return getTmpDir(getRegionDir());
+  }
+
+  static Path getTmpDir(Path regionDir) {
+    return new Path(regionDir, REGION_TEMP_SUBDIR);
   }
 
   void triggerMajorCompaction() {
@@ -2566,8 +2596,71 @@ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
 
 
   /**
-   * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP}
-   * with the provided current timestamp.
+   * Complete taking the snapshot on the region. Writes the region info and adds references to the
+   * working snapshot directory.
+   *
+   * TODO for api consistency, consider adding another version with no {@link ForeignExceptionSnare}
+   * arg.  (In the future other cancellable HRegion methods could eventually add a
+   * {@link ForeignExceptionSnare}, or we could do something fancier).
+   *
+   * @param desc snasphot description object
+   * @param exnSnare ForeignExceptionSnare that captures external exeptions in case we need to
+   *   bail out.  This is allowed to be null and will just be ignored in that case.
+   * @throws IOException if there is an external or internal error causing the snapshot to fail
+   */
+  public void addRegionToSnapshot(SnapshotDescription desc,
+      ForeignExceptionSnare exnSnare) throws IOException {
+    // This should be "fast" since we don't rewrite store files but instead
+    // back up the store files by creating a reference
+    Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
+    Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(desc, rootDir,
+      regionInfo.getEncodedName());
+
+    // 1. dump region meta info into the snapshot directory
+    LOG.debug("Storing region-info for snapshot.");
+    checkRegioninfoOnFilesystem(snapshotRegionDir);
+
+    // 2. iterate through all the stores in the region
+    LOG.debug("Creating references for hfiles");
+
+    // This ensures that we have an atomic view of the directory as long as we have < ls limit
+    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
+    // batches and may miss files being added/deleted. This could be more robust (iteratively
+    // checking to see if we have all the files until we are sure), but the limit is currently 1000
+    // files/batch, far more than the number of store files under a single column family.
+    for (Store store : stores.values()) {
+      // 2.1. build the snapshot reference directory for the store
+      Path dstStoreDir = TakeSnapshotUtils.getStoreSnapshotDirectory(snapshotRegionDir,
+        Bytes.toString(store.getFamily().getName()));
+      List<StoreFile> storeFiles = store.getStorefiles();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
+      }
+
+      // 2.2. iterate through all the store's files and create "references".
+      int sz = storeFiles.size();
+      for (int i = 0; i < sz; i++) {
+        if (exnSnare != null) {
+          exnSnare.rethrowException();
+        }
+        Path file = storeFiles.get(i).getPath();
+        // create "reference" to this store file.  It is intentionally an empty file -- all
+        // necessary infomration is captured by its fs location and filename.  This allows us to
+        // only figure out what needs to be done via a single nn operation (instead of having to
+        // open and read the files as well).
+        LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
+        Path referenceFile = new Path(dstStoreDir, file.getName());
+        boolean success = fs.createNewFile(referenceFile);
+        if (!success) {
+          throw new IOException("Failed to create reference file:" + referenceFile);
+        }
+      }
+    }
+  }
+
+  /**
+   * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the provided current
+   * timestamp.
    */
   private void updateKVTimestamps(
       final Iterable<List<KeyValue>> keyLists, final byte[] now) {
@@ -4071,6 +4164,8 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
     Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
     FileSystem fs = FileSystem.get(conf);
     fs.mkdirs(regionDir);
+    // Write HRI to a file in case we need to recover .META.
+    writeRegioninfoOnFilesystem(info, regionDir, fs, conf);
     HLog effectiveHLog = hlog;
     if (hlog == null && !ignoreHLog) {
       effectiveHLog = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
@@ -4470,11 +4565,11 @@ public static HRegion merge(HRegion a, HRegion b)
     }
 
     // delete out the 'A' region
-    HFileArchiver.archiveRegion(a.getConf(), fs, FSUtils.getRootDir(a.getConf()), a.getTableDir(),
-      a.getRegionDir());
+    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(a.getConf()),
+        a.getTableDir(), a.getRegionDir());
     // delete out the 'B' region
-    HFileArchiver.archiveRegion(b.getConf(), fs, FSUtils.getRootDir(b.getConf()), b.getTableDir(),
-      b.getRegionDir());
+    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(b.getConf()),
+        b.getTableDir(), b.getRegionDir());
 
     LOG.info("merge completed. New region is " + dstRegion);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 158cd7cd3084..5371ad54df58 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -141,6 +141,7 @@
 import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.StoreMetricType;
+import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -459,6 +460,9 @@ public HRegionServer(Configuration conf)
     cacheConfig = new CacheConfig(conf);
   }
 
+  /** Handle all the snapshot requests to this server */
+  RegionServerSnapshotManager snapshotManager;
+
   /**
    * Run test on configured codecs to make sure supporting libs are in place.
    * @param c
@@ -631,6 +635,13 @@ private void initializeZooKeeper() throws IOException, InterruptedException {
     // Create the catalog tracker and start it;
     this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
     catalogTracker.start();
+
+    // watch for snapshots
+    try {
+      this.snapshotManager = new RegionServerSnapshotManager(this);
+    } catch (KeeperException e) {
+      this.abort("Failed to reach zk cluster when creating snapshot handler.");
+    }
   }
 
   /**
@@ -717,6 +728,9 @@ public void run() {
       }
       registerMBean();
 
+      // start the snapshot handler, since the server is ready to run
+      this.snapshotManager.start();
+
       // We registered with the Master.  Go into run mode.
       long lastMsg = 0;
       long oldRequestCount = -1;
@@ -796,6 +810,12 @@ public void run() {
       this.healthCheckChore.interrupt();
     }
 
+    try {
+      if (snapshotManager != null) snapshotManager.stop(this.abortRequested);
+    } catch (IOException e) {
+      LOG.warn("Failed to close snapshot handler cleanly", e);
+    }
+
     if (this.killed) {
       // Just skip out w/o closing regions.  Used when testing.
     } else if (abortRequested) {
@@ -812,6 +832,13 @@ public void run() {
     // handlers are stuck waiting on meta or root.
     if (this.catalogTracker != null) this.catalogTracker.stop();
 
+    // stop the snapshot handler, forcefully killing all running tasks
+    try {
+      if (snapshotManager != null) snapshotManager.stop(this.abortRequested || this.killed);
+    } catch (IOException e) {
+      LOG.warn("Failed to close snapshot handler cleanly", e);
+    }
+
     // Closing the compactSplit thread before closing meta regions
     if (!this.killed && containsMetaTableRegions()) {
       if (!abortRequested || this.fsOk) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 442dbdf3e66c..90f7fd010ddc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -54,6 +54,7 @@
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
@@ -336,8 +337,27 @@ public long getMaxMemstoreTS() {
    */
   public static Path getStoreHomedir(final Path tabledir,
       final String encodedName, final byte [] family) {
-    return new Path(tabledir, new Path(encodedName,
-      new Path(Bytes.toString(family))));
+     return getStoreHomedir(tabledir, encodedName, Bytes.toString(family));
+   }
+
+  /**
+   * @param tabledir
+   * @param encodedName Encoded region name.
+   * @param family
+   * @return Path to family/Store home directory.
+   */
+  public static Path getStoreHomedir(final Path tabledir,
+      final String encodedName, final String family) {
+    return new Path(tabledir, new Path(encodedName, new Path(family)));
+  }
+
+  /**
+   * @param parentRegionDirectory directory for the parent region
+   * @param family family name of this store
+   * @return Path to the family/Store home directory
+   */
+  public static Path getStoreHomedir(final Path parentRegionDirectory, final byte[] family) {
+    return new Path(parentRegionDirectory, new Path(Bytes.toString(family)));
   }
 
   /**
@@ -393,9 +413,10 @@ private List<StoreFile> loadStoreFiles() throws IOException {
         continue;
       }
       final Path p = files[i].getPath();
-      // Check for empty file. Should never be the case but can happen
+      // Check for empty hfile. Should never be the case but can happen
       // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
-      if (this.fs.getFileStatus(p).getLen() <= 0) {
+      // NOTE: that the HFileLink is just a name, so it's an empty file.
+      if (!HFileLink.isHFileLink(p) && this.fs.getFileStatus(p).getLen() <= 0) {
         LOG.warn("Skipping " + p + " because its empty. HBASE-646 DATA LOSS?");
         continue;
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index bf31c32cf449..26b31afc1735 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -37,33 +37,37 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.io.hfile.HFileWriterV1;
 import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
-import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
-import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
-import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilter;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.io.RawComparator;
@@ -150,6 +154,9 @@ public static enum BloomType {
   // If this StoreFile references another, this is the other files path.
   private Path referencePath;
 
+  // If this storefile is a link to another, this is the link instance.
+  private HFileLink link;
+
   // Block cache configuration and reference.
   private final CacheConfig cacheConf;
 
@@ -194,13 +201,26 @@ public void setMaxMemstoreTS(long maxMemstoreTS) {
    */
   private Map<byte[], byte[]> metadataMap;
 
-  /*
-   * Regex that will work for straight filenames and for reference names.
-   * If reference, then the regex has more than just one group.  Group 1 is
-   * this files id.  Group 2 the referenced region name, etc.
+  /**
+   * A non-capture group, for hfiles, so that this can be embedded.
+   * HFiles are uuid ([0-9a-z]+). Bulk loaded hfiles has (_SeqId_[0-9]+_) has suffix.
+   */
+  public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
+
+  /** Regex that will work for hfiles */
+  private static final Pattern HFILE_NAME_PATTERN =
+    Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
+
+  /**
+   * Regex that will work for straight reference names (<hfile>.<parentEncRegion>)
+   * and hfilelink reference names (<table>=<region>-<hfile>.<parentEncRegion>)
+   * If reference, then the regex has more than just one group.
+   * Group 1, hfile/hfilelink pattern, is this file's id.
+   * Group 2 '(.+)' is the reference's parent region name.
    */
-  private static final Pattern REF_NAME_PARSER =
-    Pattern.compile("^([0-9a-f]+)(?:\\.(.+))?$");
+  private static final Pattern REF_NAME_PATTERN =
+    Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
+      HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
 
   // StoreFile.Reader
   private volatile Reader reader;
@@ -244,9 +264,20 @@ public StoreFile(final FileSystem fs,
     this.dataBlockEncoder =
         dataBlockEncoder == null ? NoOpDataBlockEncoder.INSTANCE
             : dataBlockEncoder;
-    if (isReference(p)) {
+
+    if (HFileLink.isHFileLink(p)) {
+      this.link = new HFileLink(conf, p);
+      LOG.debug("Store file " + p + " is a link");
+    } else if (isReference(p)) {
       this.reference = Reference.read(fs, p);
       this.referencePath = getReferredToFile(this.path);
+      if (HFileLink.isHFileLink(this.referencePath)) {
+        this.link = new HFileLink(conf, this.referencePath);
+      }
+      LOG.debug("Store file " + p + " is a " + reference.getFileRegion() +
+        " reference to " + this.referencePath);
+    } else if (!isHFile(p)) {
+      throw new IOException("path=" + path + " doesn't look like a valid StoreFile");
     }
 
     if (BloomFilterFactory.isGeneralBloomEnabled(conf)) {
@@ -290,27 +321,33 @@ boolean isReference() {
     return this.reference != null;
   }
 
+  /**
+   * @return <tt>true</tt> if this StoreFile is an HFileLink
+   */
+  boolean isLink() {
+    return this.link != null && this.reference == null;
+  }
+
+  private static boolean isHFile(final Path path) {
+    Matcher m = HFILE_NAME_PATTERN.matcher(path.getName());
+    return m.matches() && m.groupCount() > 0;
+  }
+
   /**
    * @param p Path to check.
    * @return True if the path has format of a HStoreFile reference.
    */
   public static boolean isReference(final Path p) {
-    return !p.getName().startsWith("_") &&
-      isReference(p, REF_NAME_PARSER.matcher(p.getName()));
+    return isReference(p.getName());
   }
 
   /**
-   * @param p Path to check.
-   * @param m Matcher to use.
+   * @param name file name to check.
    * @return True if the path has format of a HStoreFile reference.
    */
-  public static boolean isReference(final Path p, final Matcher m) {
-    if (m == null || !m.matches()) {
-      LOG.warn("Failed match of store file name " + p.toString());
-      throw new RuntimeException("Failed match of store file name " +
-          p.toString());
-    }
-    return m.groupCount() > 1 && m.group(2) != null;
+  public static boolean isReference(final String name) {
+    Matcher m = REF_NAME_PATTERN.matcher(name);
+    return m.matches() && m.groupCount() > 1;
   }
 
   /*
@@ -318,13 +355,13 @@ public static boolean isReference(final Path p, final Matcher m) {
    * hierarchy of <code>${hbase.rootdir}/tablename/regionname/familyname</code>.
    * @param p Path to a Reference file.
    * @return Calculated path to parent region file.
-   * @throws IOException
+   * @throws IllegalArgumentException when path regex fails to match.
    */
   public static Path getReferredToFile(final Path p) {
-    Matcher m = REF_NAME_PARSER.matcher(p.getName());
+    Matcher m = REF_NAME_PATTERN.matcher(p.getName());
     if (m == null || !m.matches()) {
       LOG.warn("Failed match of store file name " + p.toString());
-      throw new RuntimeException("Failed match of store file name " +
+      throw new IllegalArgumentException("Failed match of store file name " +
           p.toString());
     }
     // Other region name is suffix on the passed Reference file name
@@ -332,6 +369,8 @@ public static Path getReferredToFile(final Path p) {
     // Tabledir is up two directories from where Reference was written.
     Path tableDir = p.getParent().getParent().getParent();
     String nameStrippedOfSuffix = m.group(1);
+    LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
+
     // Build up new path with the referenced region in place of our current
     // region in the reference path.  Also strip regionname suffix from name.
     return new Path(new Path(new Path(tableDir, otherRegion),
@@ -435,16 +474,15 @@ public HDFSBlocksDistribution getHDFSBlockDistribution() {
    * If this estimate isn't good enough, we can improve it later.
    * @param fs  The FileSystem
    * @param reference  The reference
-   * @param reference  The referencePath
+   * @param status  The reference FileStatus
    * @return HDFS blocks distribution
    */
   static private HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
-    FileSystem fs, Reference reference, Path referencePath) throws IOException {
-    if ( referencePath == null) {
+    FileSystem fs, Reference reference, FileStatus status) throws IOException {
+    if (status == null) {
       return null;
     }
 
-    FileStatus status = fs.getFileStatus(referencePath);
     long start = 0;
     long length = 0;
 
@@ -458,36 +496,26 @@ static private HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
     return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
   }
 
-  /**
-   * helper function to compute HDFS blocks distribution of a given file.
-   * For reference file, it is an estimate
-   * @param fs  The FileSystem
-   * @param p  The path of the file
-   * @return HDFS blocks distribution
-   */
-  static public HDFSBlocksDistribution computeHDFSBlockDistribution(
-    FileSystem fs, Path p) throws IOException {
-    if (isReference(p)) {
-      Reference reference = Reference.read(fs, p);
-      Path referencePath = getReferredToFile(p);
-      return computeRefFileHDFSBlockDistribution(fs, reference, referencePath);
-    } else {
-      FileStatus status = fs.getFileStatus(p);
-      long length = status.getLen();
-      return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, length);
-    }
-  }
-
-
   /**
    * compute HDFS block distribution, for reference file, it is an estimate
    */
   private void computeHDFSBlockDistribution() throws IOException {
     if (isReference()) {
+      FileStatus status;
+      if (this.link != null) {
+        status = this.link.getFileStatus(fs);
+      } else {
+        status = fs.getFileStatus(this.referencePath);
+      }
       this.hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(
-        this.fs, this.reference, this.referencePath);
+        this.fs, this.reference, status);
     } else {
-      FileStatus status = this.fs.getFileStatus(this.path);
+      FileStatus status;
+      if (isLink()) {
+        status = link.getFileStatus(fs);
+      } else {
+        status = this.fs.getFileStatus(path);
+      }
       long length = status.getLen();
       this.hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(
         this.fs, status, 0, length);
@@ -505,9 +533,17 @@ private Reader open() throws IOException {
       throw new IllegalAccessError("Already open");
     }
     if (isReference()) {
-      this.reader = new HalfStoreFileReader(this.fs, this.referencePath,
-          this.cacheConf, this.reference,
-          dataBlockEncoder.getEncodingInCache());
+      if (this.link != null) {
+        this.reader = new HalfStoreFileReader(this.fs, this.referencePath, this.link,
+          this.cacheConf, this.reference, dataBlockEncoder.getEncodingInCache());
+      } else {
+        this.reader = new HalfStoreFileReader(this.fs, this.referencePath,
+          this.cacheConf, this.reference, dataBlockEncoder.getEncodingInCache());
+      }
+    } else if (isLink()) {
+      long size = link.getFileStatus(fs).getLen();
+      this.reader = new Reader(this.fs, this.path, link, size, this.cacheConf,
+          dataBlockEncoder.getEncodingInCache(), true);
     } else {
       this.reader = new Reader(this.fs, this.path, this.cacheConf,
           dataBlockEncoder.getEncodingInCache());
@@ -875,6 +911,10 @@ static Path getRandomFilename(final FileSystem fs,
    * @return <tt>true</tt> if the file could be a valid store file, <tt>false</tt> otherwise
    */
   public static boolean validateStoreFileName(String fileName) {
+    if (HFileLink.isHFileLink(fileName))
+      return true;
+    if (isReference(fileName))
+      return true;
     return !fileName.contains("-");
   }
 
@@ -899,7 +939,7 @@ static Path split(final FileSystem fs,
     // A reference to the bottom half of the hsf store file.
     Reference r = new Reference(splitRow, range);
     // Add the referred-to regions name as a dot separated suffix.
-    // See REF_NAME_PARSER regex above.  The referred-to regions name is
+    // See REF_NAME_REGEX regex above.  The referred-to regions name is
     // up in the path of the passed in <code>f</code> -- parentdir is family,
     // then the directory above is the region name.
     String parentRegionName = f.getPath().getParent().getParent().getName();
@@ -1263,6 +1303,23 @@ public Reader(FileSystem fs, Path path, CacheConfig cacheConf,
       bloomFilterType = BloomType.NONE;
     }
 
+    public Reader(FileSystem fs, Path path, HFileLink hfileLink, long size,
+        CacheConfig cacheConf, DataBlockEncoding preferredEncodingInCache,
+        boolean closeIStream) throws IOException {
+      super(path);
+
+      FSDataInputStream in = hfileLink.open(fs);
+      FSDataInputStream inNoChecksum = in;
+      if (fs instanceof HFileSystem) {
+        FileSystem noChecksumFs = ((HFileSystem)fs).getNoChecksumFs();
+        inNoChecksum = hfileLink.open(noChecksumFs);
+      }
+
+      reader = HFile.createReaderWithEncoding(fs, path, in, inNoChecksum,
+                  size, cacheConf, preferredEncodingInCache, closeIStream);
+      bloomFilterType = BloomType.NONE;
+    }
+
     /**
      * ONLY USE DEFAULT CONSTRUCTOR FOR UNIT TESTS
      */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
new file mode 100644
index 000000000000..b5d194ce9483
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.snapshot;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.procedure.ProcedureMember;
+import org.apache.hadoop.hbase.procedure.Subprocedure;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager.SnapshotSubprocedurePool;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+
+/**
+ * This online snapshot implementation uses the distributed procedure framework to force a
+ * store flush and then records the hfiles.  Its enter stage does nothing.  Its leave stage then
+ * flushes the memstore, builds the region server's snapshot manifest from its hfiles list, and
+ * copies .regioninfos into the snapshot working directory.  At the master side, there is an atomic
+ * rename of the working dir into the proper snapshot directory.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class FlushSnapshotSubprocedure extends Subprocedure {
+  private static final Log LOG = LogFactory.getLog(FlushSnapshotSubprocedure.class);
+
+  private final List<HRegion> regions;
+  private final SnapshotDescription snapshot;
+  private final SnapshotSubprocedurePool taskManager;
+
+  public FlushSnapshotSubprocedure(ProcedureMember member,
+      ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout,
+      List<HRegion> regions, SnapshotDescription snapshot,
+      SnapshotSubprocedurePool taskManager) {
+    super(member, snapshot.getName(), errorListener, wakeFrequency, timeout);
+    this.snapshot = snapshot;
+    this.regions = regions;
+    this.taskManager = taskManager;
+  }
+
+  /**
+   * Callable for adding files to snapshot manifest working dir.  Ready for multithreading.
+   */
+  private class RegionSnapshotTask implements Callable<Void> {
+    HRegion region;
+    RegionSnapshotTask(HRegion region) {
+      this.region = region;
+    }
+
+    @Override
+    public Void call() throws Exception {
+      // Taking the region read lock prevents the individual region from being closed while a
+      // snapshot is in progress.  This is helpful but not sufficient for preventing races with
+      // snapshots that involve multiple regions and regionservers.  It is still possible to have
+      // an interleaving such that globally regions are missing, so we still need the verification
+      // step.
+      LOG.debug("Starting region operation on " + region);
+      region.startRegionOperation();
+      try {
+        LOG.debug("Flush Snapshotting region " + region.toString() + " started...");
+        region.flushcache();
+        region.addRegionToSnapshot(snapshot, monitor);
+        LOG.debug("... Flush Snapshotting region " + region.toString() + " completed.");
+      } finally {
+        LOG.debug("Closing region operation on " + region);
+        region.closeRegionOperation();
+      }
+      return null;
+    }
+  }
+
+  private void flushSnapshot() throws ForeignException {
+    if (regions.isEmpty()) {
+      // No regions on this RS, we are basically done.
+      return;
+    }
+
+    monitor.rethrowException();
+
+    // assert that the taskManager is empty.
+    if (taskManager.hasTasks()) {
+      throw new IllegalStateException("Attempting to take snapshot "
+          + SnapshotDescriptionUtils.toString(snapshot)
+          + " but we currently have outstanding tasks");
+    }
+    
+    // Add all hfiles already existing in region.
+    for (HRegion region : regions) {
+      // submit one task per region for parallelize by region.
+      taskManager.submitTask(new RegionSnapshotTask(region));
+      monitor.rethrowException();
+    }
+
+    // wait for everything to complete.
+    LOG.debug("Flush Snapshot Tasks submitted for " + regions.size() + " regions");
+    try {
+      taskManager.waitForOutstandingTasks();
+    } catch (InterruptedException e) {
+      throw new ForeignException(getMemberName(), e);
+    }
+  }
+
+  /**
+   * do nothing, core of snapshot is executed in {@link #insideBarrier} step.
+   */
+  @Override
+  public void acquireBarrier() throws ForeignException {
+    // NO OP
+  }
+
+  /**
+   * do a flush snapshot of every region on this rs from the target table.
+   */
+  @Override
+  public void insideBarrier() throws ForeignException {
+    flushSnapshot();
+  }
+
+  /**
+   * Cancel threads if they haven't finished.
+   */
+  @Override
+  public void cleanup(Exception e) {
+    LOG.info("Aborting all online FLUSH snapshot subprocedure task threads for '"
+        + snapshot.getName() + "' due to error", e);
+    try {
+      taskManager.cancelTasks();
+    } catch (InterruptedException e1) {
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  /**
+   * Hooray!
+   */
+  public void releaseBarrier() {
+    // NO OP
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
new file mode 100644
index 000000000000..987a713ca289
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -0,0 +1,377 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.snapshot;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DaemonThreadFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.snapshot.MasterSnapshotVerifier;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.procedure.ProcedureMember;
+import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
+import org.apache.hadoop.hbase.procedure.Subprocedure;
+import org.apache.hadoop.hbase.procedure.SubprocedureFactory;
+import org.apache.hadoop.hbase.procedure.ZKProcedureMemberRpcs;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * This manager class handles the work dealing with snapshots for a {@link HRegionServer}.
+ * <p>
+ * This provides the mechanism necessary to kick off a online snapshot specific
+ * {@link Subprocedure} that is responsible for the regions being served by this region server.
+ * If any failures occur with the subprocedure, the RegionSeverSnapshotManager's subprocedure
+ * handler, {@link ProcedureMember}, notifies the master's ProcedureCoordinator to abort all
+ * others.
+ * <p>
+ * On startup, requires {@link #start()} to be called.
+ * <p>
+ * On shutdown, requires {@link #stop(boolean)} to be called
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class RegionServerSnapshotManager {
+  private static final Log LOG = LogFactory.getLog(RegionServerSnapshotManager.class);
+
+  /** Maximum number of snapshot region tasks that can run concurrently */
+  private static final String CONCURENT_SNAPSHOT_TASKS_KEY = "hbase.snapshot.region.concurrentTasks";
+  private static final int DEFAULT_CONCURRENT_SNAPSHOT_TASKS = 3;
+
+  /** Conf key for number of request threads to start snapshots on regionservers */
+  public static final String SNAPSHOT_REQUEST_THREADS_KEY = "hbase.snapshot.region.pool.threads";
+  /** # of threads for snapshotting regions on the rs. */
+  public static final int SNAPSHOT_REQUEST_THREADS_DEFAULT = 10;
+
+  /** Conf key for max time to keep threads in snapshot request pool waiting */
+  public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.region.timeout";
+  /** Keep threads alive in request pool for max of 60 seconds */
+  public static final long SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
+
+  /** Conf key for millis between checks to see if snapshot completed or if there are errors*/
+  public static final String SNAPSHOT_REQUEST_WAKE_MILLIS_KEY = "hbase.snapshot.region.wakefrequency";
+  /** Default amount of time to check for errors while regions finish snapshotting */
+  private static final long SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT = 500;
+
+  private final RegionServerServices rss;
+  private final ProcedureMemberRpcs memberRpcs;
+  private final ProcedureMember member;
+
+  /**
+   * Exposed for testing.
+   * @param conf HBase configuration.
+   * @param parent parent running the snapshot handler
+   * @param memberRpc use specified memberRpc instance
+   * @param procMember use specified ProcedureMember
+   */
+   RegionServerSnapshotManager(Configuration conf, HRegionServer parent,
+      ProcedureMemberRpcs memberRpc, ProcedureMember procMember) {
+    this.rss = parent;
+    this.memberRpcs = memberRpc;
+    this.member = procMember;
+  }
+
+  /**
+   * Create a default snapshot handler - uses a zookeeper based member controller.
+   * @param rss region server running the handler
+   * @throws KeeperException if the zookeeper cluster cannot be reached
+   */
+  public RegionServerSnapshotManager(RegionServerServices rss)
+      throws KeeperException {
+    this.rss = rss;
+    ZooKeeperWatcher zkw = rss.getZooKeeper();
+    String nodeName = rss.getServerName().toString();
+    this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
+        SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, nodeName);
+
+    // read in the snapshot request configuration properties
+    Configuration conf = rss.getConfiguration();
+    long wakeMillis = conf.getLong(SNAPSHOT_REQUEST_WAKE_MILLIS_KEY, SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT);
+    long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+    int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT);
+
+    // create the actual snapshot procedure member
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(wakeMillis, keepAlive, opThreads, nodeName);
+    this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder());
+  }
+
+  /**
+   * Start accepting snapshot requests.
+   */
+  public void start() {
+    this.memberRpcs.start(member);
+  }
+
+  /**
+   * Close <tt>this</tt> and all running snapshot tasks
+   * @param force forcefully stop all running tasks
+   * @throws IOException
+   */
+  public void stop(boolean force) throws IOException {
+    String mode = force ? "abruptly" : "gracefully";
+    LOG.info("Stopping RegionServerSnapshotManager " + mode + ".");
+
+    try {
+      this.member.close();
+    } finally {
+      this.memberRpcs.close();
+    }
+  }
+
+  /**
+   * If in a running state, creates the specified subprocedure for handling an online snapshot.
+   *
+   * Because this gets the local list of regions to snapshot and not the set the master had,
+   * there is a possibility of a race where regions may be missed.  This detected by the master in
+   * the snapshot verification step.
+   *
+   * @param snapshot
+   * @return Subprocedure to submit to the ProcedureMemeber.
+   */
+  public Subprocedure buildSubprocedure(SnapshotDescription snapshot) {
+
+    // don't run a snapshot if the parent is stop(ping)
+    if (rss.isStopping() || rss.isStopped()) {
+      throw new IllegalStateException("Can't start snapshot on RS: " + rss.getServerName()
+          + ", because stopping/stopped!");
+    }
+
+    // check to see if this server is hosting any regions for the snapshots
+    // check to see if we have regions for the snapshot
+    List<HRegion> involvedRegions;
+    try {
+      involvedRegions = getRegionsToSnapshot(snapshot);
+    } catch (IOException e1) {
+      throw new IllegalStateException("Failed to figure out if we should handle a snapshot - "
+          + "something has gone awry with the online regions.", e1);
+    }
+
+    // We need to run the subprocedure even if we have no relevant regions.  The coordinator
+    // expects participation in the procedure and without sending message the snapshot attempt
+    // will hang and fail.
+
+    LOG.debug("Launching subprocedure for snapshot " + snapshot.getName() + " from table "
+        + snapshot.getTable());
+    ForeignExceptionDispatcher exnDispatcher = new ForeignExceptionDispatcher();
+    Configuration conf = rss.getConfiguration();
+    long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY,
+        SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+    long wakeMillis = conf.getLong(SNAPSHOT_REQUEST_WAKE_MILLIS_KEY,
+        SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT);
+
+    switch (snapshot.getType()) {
+    case FLUSH:
+      SnapshotSubprocedurePool taskManager =
+        new SnapshotSubprocedurePool(rss.getServerName().toString(), conf);
+      return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis,
+          timeoutMillis, involvedRegions, snapshot, taskManager);
+    default:
+      throw new UnsupportedOperationException("Unrecognized snapshot type:" + snapshot.getType());
+    }
+  }
+
+  /**
+   * Determine if the snapshot should be handled on this server
+   *
+   * NOTE: This is racy -- the master expects a list of regionservers.
+   * This means if a region moves somewhere between the calls we'll miss some regions.
+   * For example, a region move during a snapshot could result in a region to be skipped or done
+   * twice.  This is manageable because the {@link MasterSnapshotVerifier} will double check the
+   * region lists after the online portion of the snapshot completes and will explicitly fail the
+   * snapshot.
+   *
+   * @param snapshot
+   * @return the list of online regions. Empty list is returned if no regions are responsible for
+   *         the given snapshot.
+   * @throws IOException
+   */
+  private List<HRegion> getRegionsToSnapshot(SnapshotDescription snapshot) throws IOException {
+    byte[] table = Bytes.toBytes(snapshot.getTable());
+    return rss.getOnlineRegions(table);
+  }
+
+  /**
+   * Build the actual snapshot runner that will do all the 'hard' work
+   */
+  public class SnapshotSubprocedureBuilder implements SubprocedureFactory {
+
+    @Override
+    public Subprocedure buildSubprocedure(String name, byte[] data) {
+      try {
+        // unwrap the snapshot information
+        SnapshotDescription snapshot = SnapshotDescription.parseFrom(data);
+        return RegionServerSnapshotManager.this.buildSubprocedure(snapshot);
+      } catch (InvalidProtocolBufferException e) {
+        throw new IllegalArgumentException("Could not read snapshot information from request.");
+      }
+    }
+
+  }
+
+  /**
+   * We use the SnapshotSubprocedurePool, a class specific thread pool instead of
+   * {@link org.apache.hadoop.hbase.executor.ExecutorService}.
+   *
+   * It uses a {@link java.util.concurrent.ExecutorCompletionService} which provides queuing of
+   * completed tasks which lets us efficiently cancel pending tasks upon the earliest operation
+   * failures.
+   *
+   * HBase's ExecutorService (different from {@link java.util.concurrent.ExecutorService}) isn't
+   * really built for coordinated tasks where multiple threads as part of one larger task.  In
+   * RS's the HBase Executor services are only used for open and close and not other threadpooled
+   * operations such as compactions and replication  sinks.
+   */
+  static class SnapshotSubprocedurePool {
+    private final ExecutorCompletionService<Void> taskPool;
+    private final ThreadPoolExecutor executor;
+    private volatile boolean stopped;
+    private final List<Future<Void>> futures = new ArrayList<Future<Void>>();
+    private final String name;
+
+    SnapshotSubprocedurePool(String name, Configuration conf) {
+      // configure the executor service
+      long keepAlive = conf.getLong(
+        RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY,
+        RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+      int threads = conf.getInt(CONCURENT_SNAPSHOT_TASKS_KEY, DEFAULT_CONCURRENT_SNAPSHOT_TASKS);
+      this.name = name;
+      executor = new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.MILLISECONDS,
+          new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory("rs("
+              + name + ")-snapshot-pool"));
+      taskPool = new ExecutorCompletionService<Void>(executor);
+    }
+
+    boolean hasTasks() {
+      return futures.size() != 0;
+    }
+    
+    /**
+     * Submit a task to the pool.
+     *
+     * NOTE: all must be submitted before you can safely {@link #waitForOutstandingTasks()}. This
+     * version does not support issuing tasks from multiple concurrent table snapshots requests.
+     */
+    void submitTask(final Callable<Void> task) {
+      Future<Void> f = this.taskPool.submit(task);
+      futures.add(f);
+    }
+
+    /**
+     * Wait for all of the currently outstanding tasks submitted via {@link #submitTask(Callable)}.
+     * This *must* be called after all tasks are submitted via submitTask.
+     *
+     * @return <tt>true</tt> on success, <tt>false</tt> otherwise
+     * @throws InterruptedException
+     * @throws SnapshotCreationException if the snapshot failed while we were waiting
+     */
+    boolean waitForOutstandingTasks() throws ForeignException, InterruptedException {
+      LOG.debug("Waiting for local region snapshots to finish.");
+
+      int sz = futures.size();
+      try {
+        // Using the completion service to process the futures that finish first first.
+        for (int i = 0; i < sz; i++) {
+          Future<Void> f = taskPool.take();
+          f.get();
+          if (!futures.remove(f)) {
+            LOG.warn("unexpected future" + f);
+          }
+          LOG.debug("Completed " + (i+1) + "/" + sz +  " local region snapshots.");
+        }
+        LOG.debug("Completed " + sz +  " local region snapshots.");
+        return true;
+      } catch (InterruptedException e) {
+        LOG.warn("Got InterruptedException in SnapshotSubprocedurePool", e);
+        if (!stopped) {
+          Thread.currentThread().interrupt();
+          throw new ForeignException("SnapshotSubprocedurePool", e);
+        }
+        // we are stopped so we can just exit.
+      } catch (ExecutionException e) {
+        if (e.getCause() instanceof ForeignException) {
+          LOG.warn("Rethrowing ForeignException from SnapshotSubprocedurePool", e);
+          throw (ForeignException)e.getCause();
+        }
+        LOG.warn("Got Exception in SnapshotSubprocedurePool", e);
+        throw new ForeignException(name, e.getCause());
+      } finally {
+        cancelTasks();
+      }
+      return false;
+    }
+
+    /**
+     * This attempts to cancel out all pending and in progress tasks (interruptions issues)
+     * @throws InterruptedException
+     */
+    void cancelTasks() throws InterruptedException {
+      Collection<Future<Void>> tasks = futures;
+      LOG.debug("cancelling " + tasks.size() + " tasks for snapshot " + name);
+      for (Future<Void> f: tasks) {
+        // TODO Ideally we'd interrupt hbase threads when we cancel.  However it seems that there
+        // are places in the HBase code where row/region locks are taken and not released in a
+        // finally block.  Thus we cancel without interrupting.  Cancellations will be slower to
+        // complete but we won't suffer from unreleased locks due to poor code discipline.
+        f.cancel(false);
+      }
+
+      // evict remaining tasks and futures from taskPool.
+      LOG.debug(taskPool);
+      while (!futures.isEmpty()) {
+        // block to remove cancelled futures;
+        LOG.warn("Removing cancelled elements from taskPool");
+        futures.remove(taskPool.take());
+      }
+      stop();
+    }
+
+    /**
+     * Abruptly shutdown the thread pool.  Call when exiting a region server.
+     */
+    void stop() {
+      if (this.stopped) return;
+
+      this.stopped = true;
+      this.executor.shutdownNow();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 33cb56228a00..57fc520545cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -124,10 +124,10 @@ public class HLog implements Syncable {
    * Name of directory that holds recovered edits written by the wal log
    * splitting code, one per region
    */
-  private static final String RECOVERED_EDITS_DIR = "recovered.edits";
+  public static final String RECOVERED_EDITS_DIR = "recovered.edits";
   private static final Pattern EDITFILES_NAME_PATTERN =
     Pattern.compile("-?[0-9]+");
-  static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
+  public static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
   
   private final FileSystem fs;
   private final Path dir;
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java
new file mode 100644
index 000000000000..88a2e66ed72e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.NavigableSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+/**
+ * Copy over each of the files in a region's recovered.edits directory to the region's snapshot
+ * directory.
+ * <p>
+ * This is a serial operation over each of the files in the recovered.edits directory and also
+ * streams all the bytes to the client and then back to the filesystem, so the files being copied
+ * should be <b>small</b> or it will (a) suck up a lot of bandwidth, and (b) take a long time.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class CopyRecoveredEditsTask extends SnapshotTask {
+
+  private static final Log LOG = LogFactory.getLog(CopyRecoveredEditsTask.class);
+  private final FileSystem fs;
+  private final Path regiondir;
+  private final Path outputDir;
+
+  /**
+   * @param snapshot Snapshot being taken
+   * @param monitor error monitor for the snapshot
+   * @param fs {@link FileSystem} where the snapshot is being taken
+   * @param regionDir directory for the region to examine for edits
+   * @param snapshotRegionDir directory for the region in the snapshot
+   */
+  public CopyRecoveredEditsTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor,
+      FileSystem fs, Path regionDir, Path snapshotRegionDir) {
+    super(snapshot, monitor);
+    this.fs = fs;
+    this.regiondir = regionDir;
+    this.outputDir = HLog.getRegionDirRecoveredEditsDir(snapshotRegionDir);
+  }
+
+  @Override
+  public Void call() throws IOException {
+    NavigableSet<Path> files = HLog.getSplitEditFilesSorted(this.fs, regiondir);
+    if (files == null || files.size() == 0) return null;
+
+    // copy over each file.
+    // this is really inefficient (could be trivially parallelized), but is
+    // really simple to reason about.
+    for (Path source : files) {
+      // check to see if the file is zero length, in which case we can skip it
+      FileStatus stat = fs.getFileStatus(source);
+      if (stat.getLen() <= 0) continue;
+
+      // its not zero length, so copy over the file
+      Path out = new Path(outputDir, source.getName());
+      LOG.debug("Copying " + source + " to " + out);
+      FileUtil.copy(fs, source, fs, out, true, fs.getConf());
+
+      // check for errors to the running operation after each file
+      this.rethrowException();
+    }
+    return null;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
new file mode 100644
index 000000000000..2e16c1b37a47
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+
+/**
+ * Exception thrown when the found snapshot info from the filesystem is not valid
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class CorruptedSnapshotException extends HBaseSnapshotException {
+
+  /**
+   * @param message message describing the exception
+   * @param e cause
+   */
+  public CorruptedSnapshotException(String message, Exception e) {
+    super(message, e);
+  }
+
+  /**
+   * Snapshot was corrupt for some reason
+   * @param message full description of the failure
+   * @param snapshot snapshot that was expected
+   */
+  public CorruptedSnapshotException(String message, SnapshotDescription snapshot) {
+    super(message, snapshot);
+  }
+
+  /**
+   * @param message message describing the exception
+   */
+  public CorruptedSnapshotException(String message) {
+    super(message, (SnapshotDescription)null);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
new file mode 100644
index 000000000000..31c3cdc9f2c7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -0,0 +1,701 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.HLogLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.ExportSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Export the specified snapshot to a given FileSystem.
+ *
+ * The .snapshot/name folder is copied to the destination cluster
+ * and then all the hfiles/hlogs are copied using a Map-Reduce Job in the .archive/ location.
+ * When everything is done, the second cluster can restore the snapshot.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class ExportSnapshot extends Configured implements Tool {
+  private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
+
+  private static final String CONF_TMP_DIR = "hbase.tmp.dir";
+  private static final String CONF_FILES_USER = "snapshot.export.files.attributes.user";
+  private static final String CONF_FILES_GROUP = "snapshot.export.files.attributes.group";
+  private static final String CONF_FILES_MODE = "snapshot.export.files.attributes.mode";
+  private static final String CONF_CHECKSUM_VERIFY = "snapshot.export.checksum.verify";
+  private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root";
+  private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
+
+  private static final String INPUT_FOLDER_PREFIX = "export-files.";
+
+  // Export Map-Reduce Counters, to keep track of the progress
+  public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED };
+
+  private static class ExportMapper extends Mapper<Text, NullWritable, NullWritable, NullWritable> {
+    final static int REPORT_SIZE = 1 * 1024 * 1024;
+    final static int BUFFER_SIZE = 64 * 1024;
+
+    private boolean verifyChecksum;
+    private String filesGroup;
+    private String filesUser;
+    private short filesMode;
+
+    private FileSystem outputFs;
+    private Path outputArchive;
+    private Path outputRoot;
+
+    private FileSystem inputFs;
+    private Path inputArchive;
+    private Path inputRoot;
+
+    @Override
+    public void setup(Context context) {
+      Configuration conf = context.getConfiguration();
+      verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
+
+      filesGroup = conf.get(CONF_FILES_GROUP);
+      filesUser = conf.get(CONF_FILES_USER);
+      filesMode = (short)conf.getInt(CONF_FILES_MODE, 0);
+      outputRoot = new Path(conf.get(CONF_OUTPUT_ROOT));
+      inputRoot = new Path(conf.get(CONF_INPUT_ROOT));
+
+      inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
+      outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+      try {
+        inputFs = FileSystem.get(inputRoot.toUri(), conf);
+      } catch (IOException e) {
+        throw new RuntimeException("Could not get the input FileSystem with root=" + inputRoot, e);
+      }
+
+      try {
+        outputFs = FileSystem.get(outputRoot.toUri(), conf);
+      } catch (IOException e) {
+        throw new RuntimeException("Could not get the output FileSystem with root="+ outputRoot, e);
+      }
+    }
+
+    @Override
+    public void map(Text key, NullWritable value, Context context)
+        throws InterruptedException, IOException {
+      Path inputPath = new Path(key.toString());
+      Path outputPath = getOutputPath(inputPath);
+
+      LOG.info("copy file input=" + inputPath + " output=" + outputPath);
+      if (copyFile(context, inputPath, outputPath)) {
+        LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
+      }
+    }
+
+    /**
+     * Returns the location where the inputPath will be copied.
+     *  - hfiles are encoded as hfile links hfile-region-table
+     *  - logs are encoded as serverName/logName
+     */
+    private Path getOutputPath(final Path inputPath) throws IOException {
+      Path path;
+      if (HFileLink.isHFileLink(inputPath)) {
+        String family = inputPath.getParent().getName();
+        String table = HFileLink.getReferencedTableName(inputPath.getName());
+        String region = HFileLink.getReferencedRegionName(inputPath.getName());
+        String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
+        path = new Path(table, new Path(region, new Path(family, hfile)));
+      } else if (isHLogLinkPath(inputPath)) {
+        String logName = inputPath.getName();
+        path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName);
+      } else {
+        path = inputPath;
+      }
+      return new Path(outputArchive, path);
+    }
+
+    private boolean copyFile(final Context context, final Path inputPath, final Path outputPath)
+        throws IOException {
+      FSDataInputStream in = openSourceFile(inputPath);
+      if (in == null) {
+        context.getCounter(Counter.MISSING_FILES).increment(1);
+        return false;
+      }
+
+      try {
+        // Verify if the input file exists
+        FileStatus inputStat = getFileStatus(inputFs, inputPath);
+        if (inputStat == null) return false;
+
+        // Verify if the output file exists and is the same that we want to copy
+        FileStatus outputStat = getFileStatus(outputFs, outputPath);
+        if (outputStat != null && sameFile(inputStat, outputStat)) {
+          LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+          return true;
+        }
+
+        context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
+
+        // Ensure that the output folder is there and copy the file
+        outputFs.mkdirs(outputPath.getParent());
+        FSDataOutputStream out = outputFs.create(outputPath, true);
+        try {
+          if (!copyData(context, inputPath, in, outputPath, out, inputStat.getLen()))
+            return false;
+        } finally {
+          out.close();
+        }
+
+        // Preserve attributes
+        return preserveAttributes(outputPath, inputStat);
+      } finally {
+        in.close();
+      }
+    }
+
+    /**
+     * Preserve the files attribute selected by the user copying them from the source file
+     */
+    private boolean preserveAttributes(final Path path, final FileStatus refStat) {
+      FileStatus stat;
+      try {
+        stat = outputFs.getFileStatus(path);
+      } catch (IOException e) {
+        LOG.warn("Unable to get the status for file=" + path);
+        return false;
+      }
+
+      try {
+        if (filesMode > 0 && stat.getPermission().toShort() != filesMode) {
+          outputFs.setPermission(path, new FsPermission(filesMode));
+        } else if (!stat.getPermission().equals(refStat.getPermission())) {
+          outputFs.setPermission(path, refStat.getPermission());
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to set the permission for file=" + path, e);
+        return false;
+      }
+
+      try {
+        String user = (filesUser != null) ? filesUser : refStat.getOwner();
+        String group = (filesGroup != null) ? filesGroup : refStat.getGroup();
+        if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
+          outputFs.setOwner(path, user, group);
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to set the owner/group for file=" + path, e);
+        return false;
+      }
+
+      return true;
+    }
+
+    private boolean copyData(final Context context,
+        final Path inputPath, final FSDataInputStream in,
+        final Path outputPath, final FSDataOutputStream out,
+        final long inputFileSize) {
+      final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
+                                   " (%.3f%%) from " + inputPath + " to " + outputPath;
+
+      try {
+        byte[] buffer = new byte[BUFFER_SIZE];
+        long totalBytesWritten = 0;
+        int reportBytes = 0;
+        int bytesRead;
+
+        while ((bytesRead = in.read(buffer)) > 0) {
+          out.write(buffer, 0, bytesRead);
+          totalBytesWritten += bytesRead;
+          reportBytes += bytesRead;
+
+          if (reportBytes >= REPORT_SIZE) {
+            context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
+            context.setStatus(String.format(statusMessage,
+                              StringUtils.humanReadableInt(totalBytesWritten),
+                              reportBytes/(float)inputFileSize));
+            reportBytes = 0;
+          }
+        }
+
+        context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
+        context.setStatus(String.format(statusMessage,
+                          StringUtils.humanReadableInt(totalBytesWritten),
+                          reportBytes/(float)inputFileSize));
+
+        // Verify that the written size match
+        if (totalBytesWritten != inputFileSize) {
+          LOG.error("number of bytes copied not matching copied=" + totalBytesWritten +
+                    " expected=" + inputFileSize + " for file=" + inputPath);
+          context.getCounter(Counter.COPY_FAILED).increment(1);
+          return false;
+        }
+
+        return true;
+      } catch (IOException e) {
+        LOG.error("Error copying " + inputPath + " to " + outputPath, e);
+        context.getCounter(Counter.COPY_FAILED).increment(1);
+        return false;
+      }
+    }
+
+    private FSDataInputStream openSourceFile(final Path path) {
+      try {
+        if (HFileLink.isHFileLink(path)) {
+          return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
+        } else if (isHLogLinkPath(path)) {
+          String serverName = path.getParent().getName();
+          String logName = path.getName();
+          return new HLogLink(inputRoot, serverName, logName).open(inputFs);
+        }
+        return inputFs.open(path);
+      } catch (IOException e) {
+        LOG.error("Unable to open source file=" + path, e);
+        return null;
+      }
+    }
+
+    private FileStatus getFileStatus(final FileSystem fs, final Path path) {
+      try {
+        if (HFileLink.isHFileLink(path)) {
+          HFileLink link = new HFileLink(inputRoot, inputArchive, path);
+          return link.getFileStatus(fs);
+        } else if (isHLogLinkPath(path)) {
+          String serverName = path.getParent().getName();
+          String logName = path.getName();
+          return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs);
+        }
+        return fs.getFileStatus(path);
+      } catch (IOException e) {
+        LOG.warn("Unable to get the status for file=" + path);
+        return null;
+      }
+    }
+
+    private FileChecksum getFileChecksum(final FileSystem fs, final Path path) {
+      try {
+        return fs.getFileChecksum(path);
+      } catch (IOException e) {
+        LOG.warn("Unable to get checksum for file=" + path, e);
+        return null;
+      }
+    }
+
+    /**
+     * Check if the two files are equal by looking at the file length,
+     * and at the checksum (if user has specified the verifyChecksum flag).
+     */
+    private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) {
+      // Not matching length
+      if (inputStat.getLen() != outputStat.getLen()) return false;
+
+      // Mark files as equals, since user asked for no checksum verification
+      if (!verifyChecksum) return true;
+
+      // If checksums are not available, files are not the same.
+      FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath());
+      if (inChecksum == null) return false;
+
+      FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath());
+      if (outChecksum == null) return false;
+
+      return inChecksum.equals(outChecksum);
+    }
+
+    /**
+     * HLog files are encoded as serverName/logName
+     * and since all the other files should be in /hbase/table/..path..
+     * we can rely on the depth, for now.
+     */
+    private static boolean isHLogLinkPath(final Path path) {
+      return path.depth() == 2;
+    }
+  }
+
+  /**
+   * Extract the list of files (HFiles/HLogs) to copy using Map-Reduce.
+   * @return list of files referenced by the snapshot (pair of path and size)
+   */
+  private List<Pair<Path, Long>> getSnapshotFiles(final FileSystem fs, final Path snapshotDir)
+      throws IOException {
+    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+
+    final List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
+    final String table = snapshotDesc.getTable();
+    final Configuration conf = getConf();
+
+    // Get snapshot files
+    SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
+      new SnapshotReferenceUtil.FileVisitor() {
+        public void storeFile (final String region, final String family, final String hfile)
+            throws IOException {
+          Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
+          long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
+          files.add(new Pair<Path, Long>(path, size));
+        }
+
+        public void recoveredEdits (final String region, final String logfile)
+            throws IOException {
+          // copied with the snapshot referenecs
+        }
+
+        public void logFile (final String server, final String logfile)
+            throws IOException {
+          long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
+          files.add(new Pair<Path, Long>(new Path(server, logfile), size));
+        }
+    });
+
+    return files;
+  }
+
+  /**
+   * Given a list of file paths and sizes, create around ngroups in as balanced a way as possible.
+   * The groups created will have similar amounts of bytes.
+   * <p>
+   * The algorithm used is pretty straightforward; the file list is sorted by size,
+   * and then each group fetch the bigger file available, iterating through groups
+   * alternating the direction.
+   */
+  static List<List<Path>> getBalancedSplits(final List<Pair<Path, Long>> files, int ngroups) {
+    // Sort files by size, from small to big
+    Collections.sort(files, new Comparator<Pair<Path, Long>>() {
+      public int compare(Pair<Path, Long> a, Pair<Path, Long> b) {
+        long r = a.getSecond() - b.getSecond();
+        return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
+      }
+    });
+
+    // create balanced groups
+    List<List<Path>> fileGroups = new LinkedList<List<Path>>();
+    long[] sizeGroups = new long[ngroups];
+    int hi = files.size() - 1;
+    int lo = 0;
+
+    List<Path> group;
+    int dir = 1;
+    int g = 0;
+
+    while (hi >= lo) {
+      if (g == fileGroups.size()) {
+        group = new LinkedList<Path>();
+        fileGroups.add(group);
+      } else {
+        group = fileGroups.get(g);
+      }
+
+      Pair<Path, Long> fileInfo = files.get(hi--);
+
+      // add the hi one
+      sizeGroups[g] += fileInfo.getSecond();
+      group.add(fileInfo.getFirst());
+
+      // change direction when at the end or the beginning
+      g += dir;
+      if (g == ngroups) {
+        dir = -1;
+        g = ngroups - 1;
+      } else if (g < 0) {
+        dir = 1;
+        g = 0;
+      }
+    }
+
+    if (LOG.isDebugEnabled()) {
+      for (int i = 0; i < sizeGroups.length; ++i) {
+        LOG.debug("export split=" + i + " size=" + StringUtils.humanReadableInt(sizeGroups[i]));
+      }
+    }
+
+    return fileGroups;
+  }
+
+  private static Path getInputFolderPath(final FileSystem fs, final Configuration conf)
+      throws IOException, InterruptedException {
+    String stagingName = "exportSnapshot-" + EnvironmentEdgeManager.currentTimeMillis();
+    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
+    fs.mkdirs(stagingDir);
+    return new Path(stagingDir, INPUT_FOLDER_PREFIX +
+      String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
+  }
+
+  /**
+   * Create the input files, with the path to copy, for the MR job.
+   * Each input files contains n files, and each input file has a similar amount data to copy.
+   * The number of input files created are based on the number of mappers provided as argument
+   * and the number of the files to copy.
+   */
+  private static Path[] createInputFiles(final Configuration conf,
+      final List<Pair<Path, Long>> snapshotFiles, int mappers)
+      throws IOException, InterruptedException {
+    FileSystem fs = FileSystem.get(conf);
+    Path inputFolderPath = getInputFolderPath(fs, conf);
+    LOG.debug("Input folder location: " + inputFolderPath);
+
+    List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);
+    Path[] inputFiles = new Path[splits.size()];
+
+    Text key = new Text();
+    for (int i = 0; i < inputFiles.length; i++) {
+      List<Path> files = splits.get(i);
+      inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i));
+      SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i],
+        Text.class, NullWritable.class);
+      LOG.debug("Input split: " + i);
+      try {
+        for (Path file: files) {
+          LOG.debug(file.toString());
+          key.set(file.toString());
+          writer.append(key, NullWritable.get());
+        }
+      } finally {
+        writer.close();
+      }
+    }
+
+    return inputFiles;
+  }
+
+  /**
+   * Run Map-Reduce Job to perform the files copy.
+   */
+  private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
+      final List<Pair<Path, Long>> snapshotFiles, final boolean verifyChecksum,
+      final String filesUser, final String filesGroup, final int filesMode,
+      final int mappers) throws IOException, InterruptedException, ClassNotFoundException {
+    Configuration conf = getConf();
+    if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup);
+    if (filesUser != null) conf.set(CONF_FILES_USER, filesUser);
+    conf.setInt(CONF_FILES_MODE, filesMode);
+    conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum);
+    conf.set(CONF_OUTPUT_ROOT, outputRoot.toString());
+    conf.set(CONF_INPUT_ROOT, inputRoot.toString());
+    conf.setInt("mapreduce.job.maps", mappers);
+
+    // job.setMapSpeculativeExecution(false)
+    conf.setBoolean("mapreduce.map.speculative", false);
+    conf.setBoolean("mapreduce.reduce.speculative", false);
+    conf.setBoolean("mapred.map.tasks.speculative.execution", false);
+    conf.setBoolean("mapred.reduce.tasks.speculative.execution", false);
+
+    Job job = new Job(conf);
+    job.setJobName("ExportSnapshot");
+    job.setJarByClass(ExportSnapshot.class);
+    job.setMapperClass(ExportMapper.class);
+    job.setInputFormatClass(SequenceFileInputFormat.class);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    job.setNumReduceTasks(0);
+    for (Path path: createInputFiles(conf, snapshotFiles, mappers)) {
+      LOG.debug("Add Input Path=" + path);
+      SequenceFileInputFormat.addInputPath(job, path);
+    }
+
+    return job.waitForCompletion(true);
+  }
+
+  /**
+   * Execute the export snapshot by copying the snapshot metadata, hfiles and hlogs.
+   * @return 0 on success, and != 0 upon failure.
+   */
+  @Override
+  public int run(String[] args) throws Exception {
+    boolean verifyChecksum = true;
+    String snapshotName = null;
+    String filesGroup = null;
+    String filesUser = null;
+    Path outputRoot = null;
+    int filesMode = 0;
+    int mappers = getConf().getInt("mapreduce.job.maps", 1);
+
+    // Process command line args
+    for (int i = 0; i < args.length; i++) {
+      String cmd = args[i];
+      try {
+        if (cmd.equals("-snapshot")) {
+          snapshotName = args[++i];
+        } else if (cmd.equals("-copy-to")) {
+          outputRoot = new Path(args[++i]);
+        } else if (cmd.equals("-no-checksum-verify")) {
+          verifyChecksum = false;
+        } else if (cmd.equals("-mappers")) {
+          mappers = Integer.parseInt(args[++i]);
+        } else if (cmd.equals("-chuser")) {
+          filesUser = args[++i];
+        } else if (cmd.equals("-chgroup")) {
+          filesGroup = args[++i];
+        } else if (cmd.equals("-chmod")) {
+          filesMode = Integer.parseInt(args[++i], 8);
+        } else if (cmd.equals("-h") || cmd.equals("--help")) {
+          printUsageAndExit();
+        } else {
+          System.err.println("UNEXPECTED: " + cmd);
+          printUsageAndExit();
+        }
+      } catch (Exception e) {
+        printUsageAndExit();
+      }
+    }
+
+    // Check user options
+    if (snapshotName == null) {
+      System.err.println("Snapshot name not provided.");
+      printUsageAndExit();
+    }
+
+    if (outputRoot == null) {
+      System.err.println("Destination file-system not provided.");
+      printUsageAndExit();
+    }
+
+    Configuration conf = getConf();
+    Path inputRoot = FSUtils.getRootDir(conf);
+    FileSystem inputFs = FileSystem.get(conf);
+    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
+
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
+    Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
+    Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+
+    // Check if the snapshot already exists
+    if (outputFs.exists(outputSnapshotDir)) {
+      System.err.println("The snapshot '" + snapshotName +
+        "' already exists in the destination: " + outputSnapshotDir);
+      return 1;
+    }
+
+    // Check if the snapshot already in-progress
+    if (outputFs.exists(snapshotTmpDir)) {
+      System.err.println("A snapshot with the same name '" + snapshotName + "' is in-progress");
+      return 1;
+    }
+
+    // Step 0 - Extract snapshot files to copy
+    final List<Pair<Path, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
+
+    // Step 1 - Copy fs1:/.snapshot/<snapshot> to  fs2:/.snapshot/.tmp/<snapshot>
+    // The snapshot references must be copied before the hfiles otherwise the cleaner
+    // will remove them because they are unreferenced.
+    try {
+      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+    } catch (IOException e) {
+      System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
+        " to=" + snapshotTmpDir);
+      e.printStackTrace(System.err);
+      return 1;
+    }
+
+    // Step 2 - Start MR Job to copy files
+    // The snapshot references must be copied before the files otherwise the files gets removed
+    // by the HFileArchiver, since they have no references.
+    try {
+      if (!runCopyJob(inputRoot, outputRoot, files, verifyChecksum,
+          filesUser, filesGroup, filesMode, mappers)) {
+        throw new ExportSnapshotException("Snapshot export failed!");
+      }
+
+      // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
+      if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
+        System.err.println("Snapshot export failed!");
+        System.err.println("Unable to rename snapshot directory from=" +
+                           snapshotTmpDir + " to=" + outputSnapshotDir);
+        return 1;
+      }
+
+      return 0;
+    } catch (Exception e) {
+      System.err.println("Snapshot export failed!");
+      e.printStackTrace(System.err);
+      outputFs.delete(outputSnapshotDir, true);
+      return 1;
+    }
+  }
+
+  // ExportSnapshot
+  private void printUsageAndExit() {
+    System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
+    System.err.println(" where [options] are:");
+    System.err.println("  -h|-help                Show this help and exit.");
+    System.err.println("  -snapshot NAME          Snapshot to restore.");
+    System.err.println("  -copy-to NAME           Remote destination hdfs://");
+    System.err.println("  -no-checksum-verify     Do not verify checksum.");
+    System.err.println("  -chuser USERNAME        Change the owner of the files to the specified one.");
+    System.err.println("  -chgroup GROUP          Change the group of the files to the specified one.");
+    System.err.println("  -chmod MODE             Change the permission of the files to the specified one.");
+    System.err.println("  -mappers                Number of mappers to use during the copy (mapreduce.job.maps).");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println("  hbase " + getClass() + " \\");
+    System.err.println("    -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\");
+    System.err.println("    -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
+    System.exit(1);
+  }
+
+  /**
+   * The guts of the {@link #main} method.
+   * Call this method to avoid the {@link #main(String[])} System.exit.
+   * @param args
+   * @return errCode
+   * @throws Exception
+   */
+  static int innerMain(final Configuration conf, final String [] args) throws Exception {
+    return ToolRunner.run(conf, new ExportSnapshot(), args);
+  }
+
+  public static void main(String[] args) throws Exception {
+     System.exit(innerMain(HBaseConfiguration.create(), args));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
new file mode 100644
index 000000000000..76a83cd55e38
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Thrown when a snapshot could not be exported due to an error during the operation.
+ */
+@InterfaceAudience.Public
+@SuppressWarnings("serial")
+public class ExportSnapshotException extends HBaseSnapshotException {
+
+  /**
+   * @param msg message describing the exception
+   */
+  public ExportSnapshotException(String msg) {
+    super(msg);
+  }
+
+  /**
+   * @param message message describing the exception
+   * @param e cause
+   */
+  public ExportSnapshotException(String message, Exception e) {
+    super(message, e);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
new file mode 100644
index 000000000000..70a884255a91
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * General exception base class for when a snapshot fails
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class HBaseSnapshotException extends HBaseIOException {
+
+  private SnapshotDescription description;
+
+  /**
+   * Some exception happened for a snapshot and don't even know the snapshot that it was about
+   * @param msg Full description of the failure
+   */
+  public HBaseSnapshotException(String msg) {
+    super(msg);
+  }
+
+  /**
+   * Exception for the given snapshot that has no previous root cause
+   * @param msg reason why the snapshot failed
+   * @param desc description of the snapshot that is being failed
+   */
+  public HBaseSnapshotException(String msg, SnapshotDescription desc) {
+    super(msg);
+    this.description = desc;
+  }
+
+  /**
+   * Exception for the given snapshot due to another exception
+   * @param msg reason why the snapshot failed
+   * @param cause root cause of the failure
+   * @param desc description of the snapshot that is being failed
+   */
+  public HBaseSnapshotException(String msg, Throwable cause, SnapshotDescription desc) {
+    super(msg, cause);
+    this.description = desc;
+  }
+
+  /**
+   * Exception when the description of the snapshot cannot be determined, due to some root other
+   * root cause
+   * @param message description of what caused the failure
+   * @param e root cause
+   */
+  public HBaseSnapshotException(String message, Exception e) {
+    super(message, e);
+  }
+
+  public SnapshotDescription getSnapshotDescription() {
+    return this.description;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java b/src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java
new file mode 100644
index 000000000000..1e321b7187a3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Writable version of the SnapshotDescription used by the rpc
+ */
+public class HSnapshotDescription implements Writable {
+  private SnapshotDescription proto;
+
+  public HSnapshotDescription() {
+  }
+
+  public HSnapshotDescription(final SnapshotDescription proto) {
+    assert proto != null : "proto must be non-null";
+    this.proto = proto;
+  }
+
+  public String getName() {
+    return this.proto.getName();
+  }
+
+  public SnapshotDescription getProto() {
+    return this.proto;
+  }
+
+  public SnapshotDescription.Type getType() {
+    return this.proto.getType();
+  }
+
+  public String getTable() {
+    return this.proto.getTable();
+  }
+
+  public boolean hasTable() {
+    return this.proto.hasTable();
+  }
+
+  public long getCreationTime() {
+    return this.proto.getCreationTime();
+  }
+
+  public int getVersion() {
+    return this.proto.getVersion();
+  }
+
+  public String toString() {
+    if (this.proto != null) {
+      return this.proto.toString();
+    }
+    return "(no snapshot)";
+  }
+
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof HSnapshotDescription)) {
+      return false;
+    }
+    SnapshotDescription oproto = ((HSnapshotDescription)obj).getProto();
+    if (this.proto == oproto) {
+      return true;
+    }
+    if (this.proto == null && oproto != null) {
+      return false;
+    }
+    return this.proto.equals(oproto);
+  }
+
+  // Writable
+  /**
+   * <em> INTERNAL </em> This method is a part of {@link Writable} interface
+   * and is used for de-serialization of the HTableDescriptor over RPC
+   */
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    byte[] data = Bytes.readByteArray(in);
+    if (data.length > 0) {
+      this.proto = SnapshotDescription.parseFrom(data);
+    } else {
+      this.proto = null;
+    }
+  }
+
+  /**
+   * <em> INTERNAL </em> This method is a part of {@link Writable} interface
+   * and is used for serialization of the HTableDescriptor over RPC
+   */
+  @Override
+  public void write(DataOutput out) throws IOException {
+    if (this.proto != null) {
+      Bytes.writeByteArray(out, this.proto.toByteArray());
+    } else {
+      Bytes.writeByteArray(out, new byte[0]);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java
new file mode 100644
index 000000000000..60d48d9c6889
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Reference all the hfiles in a region for a snapshot.
+ * <p>
+ * Doesn't take into acccount if the hfiles are valid or not, just keeps track of what's in the
+ * region's directory.
+ */
+public class ReferenceRegionHFilesTask extends SnapshotTask {
+
+  public static final Log LOG = LogFactory.getLog(ReferenceRegionHFilesTask.class);
+  private final Path regiondir;
+  private final FileSystem fs;
+  private final PathFilter fileFilter;
+  private final Path snapshotDir;
+
+  /**
+   * Reference all the files in the given region directory
+   * @param snapshot snapshot for which to add references
+   * @param monitor to check/send error
+   * @param regionDir region directory to look for errors
+   * @param fs {@link FileSystem} where the snapshot/region live
+   * @param regionSnapshotDir directory in the snapshot to store region files
+   */
+  public ReferenceRegionHFilesTask(final SnapshotDescription snapshot,
+      ForeignExceptionDispatcher monitor, Path regionDir, final FileSystem fs, Path regionSnapshotDir) {
+    super(snapshot, monitor);
+    this.regiondir = regionDir;
+    this.fs = fs;
+
+    this.fileFilter = new PathFilter() {
+      @Override
+      public boolean accept(Path path) {
+        try {
+          return fs.isFile(path);
+        } catch (IOException e) {
+          LOG.error("Failed to reach fs to check file:" + path + ", marking as not file");
+          ReferenceRegionHFilesTask.this.snapshotFailure("Failed to reach fs to check file status",
+            e);
+          return false;
+        }
+      }
+    };
+    this.snapshotDir = regionSnapshotDir;
+  }
+
+  @Override
+  public Void call() throws IOException {
+    FileStatus[] families = FSUtils.listStatus(fs, regiondir, new FSUtils.FamilyDirFilter(fs));
+
+    // if no families, then we are done again
+    if (families == null || families.length == 0) {
+      LOG.info("No families under region directory:" + regiondir
+          + ", not attempting to add references.");
+      return null;
+    }
+
+    // snapshot directories to store the hfile reference
+    List<Path> snapshotFamilyDirs = TakeSnapshotUtils.getFamilySnapshotDirectories(snapshot,
+      snapshotDir, families);
+
+    LOG.debug("Add hfile references to snapshot directories:" + snapshotFamilyDirs);
+    for (int i = 0; i < families.length; i++) {
+      FileStatus family = families[i];
+      Path familyDir = family.getPath();
+      // get all the hfiles in the family
+      FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir, fileFilter);
+
+      // if no hfiles, then we are done with this family
+      if (hfiles == null || hfiles.length == 0) {
+        LOG.debug("Not hfiles found for family: " + familyDir + ", skipping.");
+        continue;
+      }
+
+      // make the snapshot's family directory
+      Path snapshotFamilyDir = snapshotFamilyDirs.get(i);
+      fs.mkdirs(snapshotFamilyDir);
+
+      // create a reference for each hfile
+      for (FileStatus hfile : hfiles) {
+        // references are 0-length files, relying on file name.
+        Path referenceFile = new Path(snapshotFamilyDir, hfile.getPath().getName());
+        LOG.debug("Creating reference for:" + hfile.getPath() + " at " + referenceFile);
+        if (!fs.createNewFile(referenceFile)) {
+          throw new IOException("Failed to create reference file:" + referenceFile);
+        }
+      }
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Finished referencing hfiles, current region state:");
+      FSUtils.logFileSystemState(fs, regiondir, LOG);
+      LOG.debug("and the snapshot directory:");
+      FSUtils.logFileSystemState(fs, snapshotDir, LOG);
+    }
+    return null;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java
new file mode 100644
index 000000000000..9c987ab20e08
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Reference all the WAL files under a server's WAL directory
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ReferenceServerWALsTask extends SnapshotTask {
+  private static final Log LOG = LogFactory.getLog(ReferenceServerWALsTask.class);
+  private final FileSystem fs;
+  private final Configuration conf;
+  private final String serverName;
+  private Path logDir;
+
+  /**
+   * @param snapshot snapshot being run
+   * @param failureListener listener to check for errors while running the operation and to
+   *          propagate errors found while running the task
+   * @param logDir log directory for the server. Name of the directory is taken as the name of the
+   *          server
+   * @param conf {@link Configuration} to extract filesystem information
+   * @param fs filesystem where the log files are stored and should be referenced
+   */
+  public ReferenceServerWALsTask(SnapshotDescription snapshot,
+      ForeignExceptionDispatcher failureListener, final Path logDir, final Configuration conf,
+      final FileSystem fs) {
+    super(snapshot, failureListener);
+    this.fs = fs;
+    this.conf = conf;
+    this.serverName = logDir.getName();
+    this.logDir = logDir;
+  }
+
+  /**
+   * Create reference files (empty files with the same path and file name as original).
+   * @throws IOException exception from hdfs or network problems
+   * @throws ForeignException exception from an external procedure
+   */
+  @Override
+  public Void call() throws IOException, ForeignException {
+    // TODO switch to using a single file to reference all required WAL files
+
+    // Iterate through each of the log files and add a reference to it.
+    // assumes that all the files under the server's logs directory is a log
+    FileStatus[] serverLogs = FSUtils.listStatus(fs, logDir, null);
+    if (serverLogs == null) {
+      LOG.debug("No logs for server directory:" + logDir + ", done referencing files.");
+      return null;
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Adding references for WAL files:" + Arrays.toString(serverLogs));
+    }
+
+    for (FileStatus file : serverLogs) {
+      this.rethrowException();
+
+      // add the reference to the file. ex: hbase/.snapshots/.logs/<serverName>/<hlog>
+      Path rootDir = FSUtils.getRootDir(conf);
+      Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(this.snapshot, rootDir);
+      Path snapshotLogDir = TakeSnapshotUtils.getSnapshotHLogsDir(snapshotDir, serverName);
+      // actually store the reference on disk (small file)
+      Path ref = new Path(snapshotLogDir, file.getPath().getName());
+      if (!fs.createNewFile(ref)) {
+        if (!fs.exists(ref)) {
+          throw new IOException("Couldn't create reference for:" + file.getPath());
+        }
+      }
+      LOG.debug("Completed WAL referencing for: " + file.getPath() + " to " + ref);
+    }
+
+    LOG.debug("Successfully completed WAL referencing for ALL files");
+    return null;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
new file mode 100644
index 000000000000..ff40783844aa
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * Thrown when a snapshot could not be restored due to a server-side error when restoring it.
+ */
+@SuppressWarnings("serial")
+public class RestoreSnapshotException extends HBaseSnapshotException {
+  public RestoreSnapshotException(String msg, SnapshotDescription desc) {
+    super(msg, desc);
+  }
+
+  public RestoreSnapshotException(String msg, Throwable cause, SnapshotDescription desc) {
+    super(msg, cause, desc);
+  }
+
+  public RestoreSnapshotException(String msg) {
+    super(msg);
+  }
+
+  public RestoreSnapshotException(String message, Exception e) {
+    super(message, e);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
new file mode 100644
index 000000000000..16f71b39d8a6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -0,0 +1,588 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSVisitor;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Helper to Restore/Clone a Snapshot
+ *
+ * <p>The helper assumes that a table is already created, and by calling restore()
+ * the content present in the snapshot will be restored as the new content of the table.
+ *
+ * <p>Clone from Snapshot: If the target table is empty, the restore operation
+ * is just a "clone operation", where the only operations are:
+ * <ul>
+ *  <li>for each region in the snapshot create a new region
+ *    (note that the region will have a different name, since the encoding contains the table name)
+ *  <li>for each file in the region create a new HFileLink to point to the original file.
+ *  <li>restore the logs, if any
+ * </ul>
+ *
+ * <p>Restore from Snapshot:
+ * <ul>
+ *  <li>for each region in the table verify which are available in the snapshot and which are not
+ *    <ul>
+ *    <li>if the region is not present in the snapshot, remove it.
+ *    <li>if the region is present in the snapshot
+ *      <ul>
+ *      <li>for each file in the table region verify which are available in the snapshot
+ *        <ul>
+ *          <li>if the hfile is not present in the snapshot, remove it
+ *          <li>if the hfile is present, keep it (nothing to do)
+ *        </ul>
+ *      <li>for each file in the snapshot region but not in the table
+ *        <ul>
+ *          <li>create a new HFileLink that point to the original file
+ *        </ul>
+ *      </ul>
+ *    </ul>
+ *  <li>for each region in the snapshot not present in the current table state
+ *    <ul>
+ *    <li>create a new region and for each file in the region create a new HFileLink
+ *      (This is the same as the clone operation)
+ *    </ul>
+ *  <li>restore the logs, if any
+ * </ul>
+ */
+@InterfaceAudience.Private
+public class RestoreSnapshotHelper {
+  private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
+
+  private final Map<byte[], byte[]> regionsMap =
+        new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
+
+  private final ForeignExceptionDispatcher monitor;
+
+  private final SnapshotDescription snapshotDesc;
+  private final Path snapshotDir;
+
+  private final HTableDescriptor tableDesc;
+  private final Path tableDir;
+
+  private final Configuration conf;
+  private final FileSystem fs;
+
+  public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
+      final SnapshotDescription snapshotDescription, final Path snapshotDir,
+      final HTableDescriptor tableDescriptor, final Path tableDir,
+      final ForeignExceptionDispatcher monitor)
+  {
+    this.fs = fs;
+    this.conf = conf;
+    this.snapshotDesc = snapshotDescription;
+    this.snapshotDir = snapshotDir;
+    this.tableDesc = tableDescriptor;
+    this.tableDir = tableDir;
+    this.monitor = monitor;
+  }
+
+  /**
+   * Restore the on-disk table to a specified snapshot state.
+   * @return the set of regions touched by the restore operation
+   */
+  public RestoreMetaChanges restoreHdfsRegions() throws IOException {
+    LOG.debug("starting restore");
+    Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
+    if (snapshotRegionNames == null) {
+      LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
+      return null;
+    }
+
+    RestoreMetaChanges metaChanges = new RestoreMetaChanges();
+
+    // Identify which region are still available and which not.
+    // NOTE: we rely upon the region name as: "table name, start key, end key"
+    List<HRegionInfo> tableRegions = getTableRegions();
+    if (tableRegions != null) {
+      monitor.rethrowException();
+      for (HRegionInfo regionInfo: tableRegions) {
+        String regionName = regionInfo.getEncodedName();
+        if (snapshotRegionNames.contains(regionName)) {
+          LOG.info("region to restore: " + regionName);
+          snapshotRegionNames.remove(regionName);
+          metaChanges.addRegionToRestore(regionInfo);
+        } else {
+          LOG.info("region to remove: " + regionName);
+          metaChanges.addRegionToRemove(regionInfo);
+        }
+      }
+
+      // Restore regions using the snapshot data
+      monitor.rethrowException();
+      restoreHdfsRegions(metaChanges.getRegionsToRestore());
+
+      // Remove regions from the current table
+      monitor.rethrowException();
+      removeHdfsRegions(metaChanges.getRegionsToRemove());
+    }
+
+    // Regions to Add: present in the snapshot but not in the current table
+    if (snapshotRegionNames.size() > 0) {
+      List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
+
+      monitor.rethrowException();
+      for (String regionName: snapshotRegionNames) {
+        LOG.info("region to add: " + regionName);
+        Path regionDir = new Path(snapshotDir, regionName);
+        regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir));
+      }
+
+      // Create new regions cloning from the snapshot
+      monitor.rethrowException();
+      HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
+      metaChanges.setNewRegions(clonedRegions);
+    }
+
+    // Restore WALs
+    monitor.rethrowException();
+    restoreWALs();
+
+    return metaChanges;
+  }
+
+  /**
+   * Describe the set of operations needed to update META after restore.
+   */
+  public static class RestoreMetaChanges {
+    private List<HRegionInfo> regionsToRestore = null;
+    private List<HRegionInfo> regionsToRemove = null;
+    private List<HRegionInfo> regionsToAdd = null;
+
+    /**
+     * @return true if there're new regions
+     */
+    public boolean hasRegionsToAdd() {
+      return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
+    }
+
+    /**
+     * Returns the list of new regions added during the on-disk restore.
+     * The caller is responsible to add the regions to META.
+     * e.g MetaEditor.addRegionsToMeta(...)
+     * @return the list of regions to add to META
+     */
+    public List<HRegionInfo> getRegionsToAdd() {
+      return this.regionsToAdd;
+    }
+
+    /**
+     * @return true if there're regions to restore
+     */
+    public boolean hasRegionsToRestore() {
+      return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
+    }
+
+    /**
+     * Returns the list of 'restored regions' during the on-disk restore.
+     * The caller is responsible to add the regions to META if not present.
+     * @return the list of regions restored
+     */
+    public List<HRegionInfo> getRegionsToRestore() {
+      return this.regionsToRestore;
+    }
+
+    /**
+     * @return true if there're regions to remove
+     */
+    public boolean hasRegionsToRemove() {
+      return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
+    }
+
+    /**
+     * Returns the list of regions removed during the on-disk restore.
+     * The caller is responsible to remove the regions from META.
+     * e.g. MetaEditor.deleteRegions(...)
+     * @return the list of regions to remove from META
+     */
+    public List<HRegionInfo> getRegionsToRemove() {
+      return this.regionsToRemove;
+    }
+
+    void setNewRegions(final HRegionInfo[] hris) {
+      if (hris != null) {
+        regionsToAdd = Arrays.asList(hris);
+      } else {
+        regionsToAdd = null;
+      }
+    }
+
+    void addRegionToRemove(final HRegionInfo hri) {
+      if (regionsToRemove == null) {
+        regionsToRemove = new LinkedList<HRegionInfo>();
+      }
+      regionsToRemove.add(hri);
+    }
+
+    void addRegionToRestore(final HRegionInfo hri) {
+      if (regionsToRestore == null) {
+        regionsToRestore = new LinkedList<HRegionInfo>();
+      }
+      regionsToRestore.add(hri);
+    }
+  }
+
+  /**
+   * Remove specified regions from the file-system, using the archiver.
+   */
+  private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+    if (regions != null && regions.size() > 0) {
+      for (HRegionInfo hri: regions) {
+        HFileArchiver.archiveRegion(conf, fs, hri);
+      }
+    }
+  }
+
+  /**
+   * Restore specified regions by restoring content to the snapshot state.
+   */
+  private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+    if (regions == null || regions.size() == 0) return;
+    for (HRegionInfo hri: regions) restoreRegion(hri);
+  }
+
+  /**
+   * Restore region by removing files not in the snapshot
+   * and adding the missing ones from the snapshot.
+   */
+  private void restoreRegion(HRegionInfo regionInfo) throws IOException {
+    Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
+    Map<String, List<String>> snapshotFiles =
+                SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
+    Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
+    String tableName = tableDesc.getNameAsString();
+
+    // Restore families present in the table
+    for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+      byte[] family = Bytes.toBytes(familyDir.getName());
+      Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
+      List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
+      if (snapshotFamilyFiles != null) {
+        List<String> hfilesToAdd = new LinkedList<String>();
+        for (String hfileName: snapshotFamilyFiles) {
+          if (familyFiles.contains(hfileName)) {
+            // HFile already present
+            familyFiles.remove(hfileName);
+          } else {
+            // HFile missing
+            hfilesToAdd.add(hfileName);
+          }
+        }
+
+        // Restore Missing files
+        for (String hfileName: hfilesToAdd) {
+          LOG.trace("Adding HFileLink " + hfileName +
+            " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
+          restoreStoreFile(familyDir, regionInfo, hfileName);
+        }
+
+        // Remove hfiles not present in the snapshot
+        for (String hfileName: familyFiles) {
+          Path hfile = new Path(familyDir, hfileName);
+          LOG.trace("Removing hfile=" + hfile +
+            " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
+          HFileArchiver.archiveStoreFile(fs, regionInfo, conf, tableDir, family, hfile);
+        }
+      } else {
+        // Family doesn't exists in the snapshot
+        LOG.trace("Removing family=" + Bytes.toString(family) +
+          " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
+        HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
+        fs.delete(familyDir, true);
+      }
+    }
+
+    // Add families not present in the table
+    for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
+      Path familyDir = new Path(regionDir, familyEntry.getKey());
+      if (!fs.mkdirs(familyDir)) {
+        throw new IOException("Unable to create familyDir=" + familyDir);
+      }
+
+      for (String hfileName: familyEntry.getValue()) {
+        LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
+        restoreStoreFile(familyDir, regionInfo, hfileName);
+      }
+    }
+  }
+
+  /**
+   * @return The set of files in the specified family directory.
+   */
+  private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
+    Set<String> familyFiles = new HashSet<String>();
+
+    FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
+    if (hfiles == null) return familyFiles;
+
+    for (FileStatus hfileRef: hfiles) {
+      String hfileName = hfileRef.getPath().getName();
+      familyFiles.add(hfileName);
+    }
+
+    return familyFiles;
+  }
+
+  /**
+   * Clone specified regions. For each region create a new region
+   * and create a HFileLink for each hfile.
+   */
+  private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+    if (regions == null || regions.size() == 0) return null;
+
+    final Map<String, HRegionInfo> snapshotRegions =
+      new HashMap<String, HRegionInfo>(regions.size());
+
+    // clone region info (change embedded tableName with the new one)
+    HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
+    for (int i = 0; i < clonedRegionsInfo.length; ++i) {
+      // clone the region info from the snapshot region info
+      HRegionInfo snapshotRegionInfo = regions.get(i);
+      clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
+
+      // add the region name mapping between snapshot and cloned
+      String snapshotRegionName = snapshotRegionInfo.getEncodedName();
+      String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
+      regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
+      LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
+
+      // Add mapping between cloned region name and snapshot region info
+      snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
+    }
+
+    // create the regions on disk
+    ModifyRegionUtils.createRegions(conf, tableDir.getParent(),
+      tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
+        public void fillRegion(final HRegion region) throws IOException {
+          cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
+        }
+      });
+
+    return clonedRegionsInfo;
+  }
+
+  /**
+   * Clone region directory content from the snapshot info.
+   *
+   * Each region is encoded with the table name, so the cloned region will have
+   * a different region name.
+   *
+   * Instead of copying the hfiles a HFileLink is created.
+   *
+   * @param region {@link HRegion} cloned
+   * @param snapshotRegionInfo
+   */
+  private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
+      throws IOException {
+    final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
+    final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
+    final String tableName = tableDesc.getNameAsString();
+    SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
+      new FSVisitor.StoreFileVisitor() {
+        public void storeFile (final String region, final String family, final String hfile)
+            throws IOException {
+          LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
+          Path familyDir = new Path(regionDir, family);
+          restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
+        }
+    });
+  }
+
+  /**
+   * Create a new {@link HFileLink} to reference the store file.
+   * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
+   * <ul>
+   *   <li>hfile: abc -> table=region-abc
+   *   <li>reference: abc.1234 -> table=region-abc.1234
+   *   <li>hfilelink: table=region-hfile -> table=region-hfile
+   * </ul>
+   * @param familyDir destination directory for the store file
+   * @param regionInfo destination region info for the table
+   * @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
+   */
+  private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
+      final String hfileName) throws IOException {
+    if (HFileLink.isHFileLink(hfileName)) {
+      HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
+    } else if (StoreFile.isReference(hfileName)) {
+      restoreReferenceFile(familyDir, regionInfo, hfileName);
+    } else {
+      HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
+    }
+  }
+
+  /**
+   * Create a new {@link Reference} as copy of the source one.
+   * <p><blockquote><pre>
+   * The source table looks like:
+   *    1234/abc      (original file)
+   *    5678/abc.1234 (reference file)
+   *
+   * After the clone operation looks like:
+   *   wxyz/table=1234-abc
+   *   stuv/table=1234-abc.wxyz
+   *
+   * NOTE that the region name in the clone changes (md5 of regioninfo)
+   * and the reference should reflect that change.
+   * </pre></blockquote>
+   * @param familyDir destination directory for the store file
+   * @param regionInfo destination region info for the table
+   * @param hfileName reference file name
+   */
+  private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
+      final String hfileName) throws IOException {
+    // Extract the referred information (hfile name and parent region)
+    String tableName = snapshotDesc.getTable();
+    Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(tableName,
+        regionInfo.getEncodedName()), familyDir.getName()), hfileName));
+    String snapshotRegionName = refPath.getParent().getParent().getName();
+    String fileName = refPath.getName();
+
+    // The new reference should have the cloned region name as parent, if it is a clone.
+    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
+    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
+
+    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
+    String refLink = fileName;
+    if (!HFileLink.isHFileLink(fileName)) {
+      refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
+    }
+    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
+
+    // Create the new reference
+    Path linkPath = new Path(familyDir,
+      HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
+    InputStream in = new HFileLink(conf, linkPath).open(fs);
+    OutputStream out = fs.create(outPath);
+    IOUtils.copyBytes(in, out, conf);
+  }
+
+  /**
+   * Create a new {@link HRegionInfo} from the snapshot region info.
+   * Keep the same startKey, endKey, regionId and split information but change
+   * the table name.
+   *
+   * @param snapshotRegionInfo Info for region to clone.
+   * @return the new HRegion instance
+   */
+  public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
+    return new HRegionInfo(tableDesc.getName(),
+                      snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
+                      snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
+  }
+
+  /**
+   * Restore snapshot WALs.
+   *
+   * Global Snapshot keep a reference to region servers logs present during the snapshot.
+   * (/hbase/.snapshot/snapshotName/.logs/hostName/logName)
+   *
+   * Since each log contains different tables data, logs must be split to
+   * extract the table that we are interested in.
+   */
+  private void restoreWALs() throws IOException {
+    final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
+                                Bytes.toBytes(snapshotDesc.getTable()), regionsMap);
+    try {
+      // Recover.Edits
+      SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
+          new FSVisitor.RecoveredEditsVisitor() {
+        public void recoveredEdits (final String region, final String logfile) throws IOException {
+          Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
+          logSplitter.splitRecoveredEdit(path);
+        }
+      });
+
+      // Region Server Logs
+      SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
+        public void logFile (final String server, final String logfile) throws IOException {
+          logSplitter.splitLog(server, logfile);
+        }
+      });
+    } finally {
+      logSplitter.close();
+    }
+  }
+
+  /**
+   * @return the set of the regions contained in the table
+   */
+  private List<HRegionInfo> getTableRegions() throws IOException {
+    LOG.debug("get table regions: " + tableDir);
+    FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
+    if (regionDirs == null) return null;
+
+    List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
+    for (FileStatus regionDir: regionDirs) {
+      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir.getPath());
+      regions.add(hri);
+    }
+    LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString());
+    return regions;
+  }
+
+  /**
+   * Create a new table descriptor cloning the snapshot table schema.
+   *
+   * @param snapshotTableDescriptor
+   * @param tableName
+   * @return cloned table descriptor
+   * @throws IOException
+   */
+  public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
+      final byte[] tableName) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
+      htd.addFamily(hcd);
+    }
+    return htd;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
new file mode 100644
index 000000000000..69dc3d031dc3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * Thrown when a snapshot could not be created due to a server-side error when taking the snapshot.
+ */
+@SuppressWarnings("serial")
+public class SnapshotCreationException extends HBaseSnapshotException {
+
+  /**
+   * Used internally by the RPC engine to pass the exception back to the client.
+   * @param msg error message to pass back
+   */
+  public SnapshotCreationException(String msg) {
+    super(msg);
+  }
+
+  /**
+   * Failure to create the specified snapshot
+   * @param msg reason why the snapshot couldn't be completed
+   * @param desc description of the snapshot attempted
+   */
+  public SnapshotCreationException(String msg, SnapshotDescription desc) {
+    super(msg, desc);
+  }
+
+  /**
+   * Failure to create the specified snapshot due to an external cause
+   * @param msg reason why the snapshot couldn't be completed
+   * @param cause root cause of the failure
+   * @param desc description of the snapshot attempted
+   */
+  public SnapshotCreationException(String msg, Throwable cause, SnapshotDescription desc) {
+    super(msg, cause, desc);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
new file mode 100644
index 000000000000..e04876871ee1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -0,0 +1,360 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
+ * <p>
+ * Snapshots are laid out on disk like this:
+ *
+ * <pre>
+ * /hbase/.snapshots
+ *          /.tmp                <---- working directory
+ *          /[snapshot name]     <----- completed snapshot
+ * </pre>
+ *
+ * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
+ * signified by '...' on the same directory depth).
+ *
+ * <pre>
+ * /hbase/.snapshots/completed
+ *                   .snapshotinfo          <--- Description of the snapshot
+ *                   .tableinfo             <--- Copy of the tableinfo
+ *                    /.logs
+ *                        /[server_name]
+ *                            /... [log files]
+ *                         ...
+ *                   /[region name]           <---- All the region's information
+ *                   .regioninfo              <---- Copy of the HRegionInfo
+ *                      /[column family name]
+ *                          /[hfile name]     <--- name of the hfile in the real region
+ *                          ...
+ *                      ...
+ *                    ...
+ * </pre>
+ *
+ * Utility methods in this class are useful for getting the correct locations for different parts of
+ * the snapshot, as well as moving completed snapshots into place (see
+ * {@link #completeSnapshot}, and writing the
+ * {@link SnapshotDescription} to the working snapshot directory.
+ */
+public class SnapshotDescriptionUtils {
+
+  /**
+   * Filter that only accepts completed snapshot directories
+   */
+  public static class CompletedSnaphotDirectoriesFilter extends FSUtils.DirFilter {
+
+    /**
+     * @param fs
+     */
+    public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
+      super(fs);
+    }
+
+    @Override
+    public boolean accept(Path path) {
+      // only accept directories that aren't the tmp directory
+      if (super.accept(path)) {
+        return !path.getName().equals(SNAPSHOT_TMP_DIR_NAME);
+      }
+      return false;
+    }
+
+  }
+
+  private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
+  /**
+   * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
+   * which we may need to read in differently.
+   */
+  public static final int SNAPSHOT_LAYOUT_VERSION = 0;
+
+  // snapshot directory constants
+  /**
+   * The file contains the snapshot basic information and it is under the directory of a snapshot.
+   */
+  public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
+
+  /** Temporary directory under the snapshot directory to store in-progress snapshots */
+  public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
+  // snapshot operation values
+  /** Default value if no start time is specified */
+  public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
+
+  public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis";
+
+  /** By default, wait 60 seconds for a snapshot to complete */
+  public static final long DEFAULT_MAX_WAIT_TIME = 60000;
+
+  private SnapshotDescriptionUtils() {
+    // private constructor for utility class
+  }
+
+  /**
+   * Check to make sure that the description of the snapshot requested is valid
+   * @param snapshot description of the snapshot
+   * @throws IllegalArgumentException if the name of the snapshot or the name of the table to
+   *           snapshot are not valid names.
+   */
+  public static void assertSnapshotRequestIsValid(SnapshotDescription snapshot)
+      throws IllegalArgumentException {
+    // FIXME these method names is really bad - trunk will probably change
+    // .META. and -ROOT- snapshots are not allowed
+    if (HTableDescriptor.isMetaTable(Bytes.toBytes(snapshot.getTable()))) {
+      throw new IllegalArgumentException(".META. and -ROOT- snapshots are not allowed");
+    }
+    // make sure the snapshot name is valid
+    HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getName()));
+    // make sure the table name is valid
+    HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getTable()));
+  }
+
+  /**
+   * @param conf {@link Configuration} from which to check for the timeout
+   * @param type type of snapshot being taken
+   * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration
+   * @return the max amount of time the master should wait for a snapshot to complete
+   */
+  public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type,
+      long defaultMaxWaitTime) {
+    String confKey;
+    switch (type) {
+    case DISABLED:
+    default:
+      confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS;
+    }
+    return conf.getLong(confKey, defaultMaxWaitTime);
+  }
+
+  /**
+   * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
+   * ${hbase.rootdir}/.snapshot
+   * @param rootDir hbase root directory
+   * @return the base directory in which all snapshots are kept
+   */
+  public static Path getSnapshotRootDir(final Path rootDir) {
+    return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
+  }
+
+  /**
+   * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
+   * directory and all the data files for a snapshot are kept under this directory.
+   * @param snapshot snapshot being taken
+   * @param rootDir hbase root directory
+   * @return the final directory for the completed snapshot
+   */
+  public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
+    return getCompletedSnapshotDir(snapshot.getName(), rootDir);
+  }
+
+  /**
+   * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
+   * directory and all the data files for a snapshot are kept under this directory.
+   * @param snapshotName name of the snapshot being taken
+   * @param rootDir hbase root directory
+   * @return the final directory for the completed snapshot
+   */
+  public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
+    return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
+  }
+
+  /**
+   * Get the general working directory for snapshots - where they are built, where they are
+   * temporarily copied on export, etc.
+   * @param rootDir root directory of the HBase installation
+   * @return Path to the snapshot tmp directory, relative to the passed root directory
+   */
+  public static Path getWorkingSnapshotDir(final Path rootDir) {
+    return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
+  }
+
+  /**
+   * Get the directory to build a snapshot, before it is finalized
+   * @param snapshot snapshot that will be built
+   * @param rootDir root directory of the hbase installation
+   * @return {@link Path} where one can build a snapshot
+   */
+  public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
+    return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
+  }
+
+  /**
+   * Get the directory to build a snapshot, before it is finalized
+   * @param snapshotName name of the snapshot
+   * @param rootDir root directory of the hbase installation
+   * @return {@link Path} where one can build a snapshot
+   */
+  public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
+    return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
+  }
+
+  /**
+   * Get the directory to store the snapshot instance
+   * @param snapshotsDir hbase-global directory for storing all snapshots
+   * @param snapshotName name of the snapshot to take
+   * @return
+   */
+  private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
+    return new Path(snapshotsDir, snapshotName);
+  }
+
+  /**
+   * @param rootDir hbase root directory
+   * @return the directory for all completed snapshots;
+   */
+  public static final Path getSnapshotsDir(Path rootDir) {
+    return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
+  }
+
+  /**
+   * Convert the passed snapshot description into a 'full' snapshot description based on default
+   * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
+   * supplied to their default values.
+   * @param snapshot general snapshot descriptor
+   * @param conf Configuration to read configured snapshot defaults if snapshot is not complete
+   * @return a valid snapshot description
+   * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete
+   *           {@link SnapshotDescription}.
+   */
+  public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf)
+      throws IllegalArgumentException {
+    if (!snapshot.hasTable()) {
+      throw new IllegalArgumentException(
+        "Descriptor doesn't apply to a table, so we can't build it.");
+    }
+
+    // set the creation time, if one hasn't been set
+    long time = snapshot.getCreationTime();
+    if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) {
+      time = EnvironmentEdgeManager.currentTimeMillis();
+      LOG.debug("Creation time not specified, setting to:" + time + " (current time:"
+          + EnvironmentEdgeManager.currentTimeMillis() + ").");
+      SnapshotDescription.Builder builder = snapshot.toBuilder();
+      builder.setCreationTime(time);
+      snapshot = builder.build();
+    }
+    return snapshot;
+  }
+
+  /**
+   * Write the snapshot description into the working directory of a snapshot
+   * @param snapshot description of the snapshot being taken
+   * @param workingDir working directory of the snapshot
+   * @param fs {@link FileSystem} on which the snapshot should be taken
+   * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
+   *           failure
+   */
+  public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
+      throws IOException {
+    FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
+      HConstants.DATA_FILE_UMASK_KEY);
+    Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
+    try {
+      FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
+      try {
+        snapshot.writeTo(out);
+      } finally {
+        out.close();
+      }
+    } catch (IOException e) {
+      // if we get an exception, try to remove the snapshot info
+      if (!fs.delete(snapshotInfo, false)) {
+        String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
+        LOG.error(msg);
+        throw new IOException(msg);
+      }
+    }
+  }
+
+  /**
+   * Read in the {@link SnapshotDescription} stored for the snapshot in the passed directory
+   * @param fs filesystem where the snapshot was taken
+   * @param snapshotDir directory where the snapshot was stored
+   * @return the stored snapshot description
+   * @throws CorruptedSnapshotException if the snapshot cannot be read
+   */
+  public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
+      throws CorruptedSnapshotException {
+    Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
+    try {
+      FSDataInputStream in = null;
+      try {
+        in = fs.open(snapshotInfo);
+        return SnapshotDescription.parseFrom(in);
+      } finally {
+        if (in != null) in.close();
+      }
+    } catch (IOException e) {
+      throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
+    }
+  }
+
+  /**
+   * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
+   * as 'complete'.
+   * @param snapshot description of the snapshot being tabken
+   * @param rootdir root directory of the hbase installation
+   * @param workingDir directory where the in progress snapshot was built
+   * @param fs {@link FileSystem} where the snapshot was built
+   * @throws SnapshotCreationException if the snapshot could not be moved
+   * @throws IOException the filesystem could not be reached
+   */
+  public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
+      FileSystem fs) throws SnapshotCreationException, IOException {
+    Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
+    LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
+        + finishedDir);
+    if (!fs.rename(workingDir, finishedDir)) {
+      throw new SnapshotCreationException("Failed to move working directory(" + workingDir
+          + ") to completed directory(" + finishedDir + ").", snapshot);
+    }
+  }
+
+  /**
+   * Returns a single line (no \n) representation of snapshot metadata.  Use this instead of
+   * {@link SnapshotDescription#toString()}.  We don't replace SnapshotDescrpition's toString
+   * because it is auto-generated by protoc.
+   * @param ssd
+   * @return Single line string with a summary of the snapshot parameters
+   */
+  public static String toString(SnapshotDescription ssd) {
+    if (ssd == null) {
+      return null;
+    }
+    return "{ ss=" + ssd.getName() + " table=" + ssd.getTable()
+        + " type=" + ssd.getType() + " }";
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
new file mode 100644
index 000000000000..eb02ece2c51d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+
+/**
+ * Thrown when the server is looking for a snapshot but can't find the snapshot on the filesystem
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class SnapshotDoesNotExistException extends HBaseSnapshotException {
+  /**
+   * @param msg full description of the failure
+   */
+  public SnapshotDoesNotExistException(String msg) {
+    super(msg);
+  }
+
+  /**
+   * @param desc expected snapshot to find
+   */
+  public SnapshotDoesNotExistException(SnapshotDescription desc) {
+    super("Snapshot '" + desc.getName() +"' doesn't exist on the filesystem", desc);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
new file mode 100644
index 000000000000..2ce2d318308c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * Thrown when a snapshot exists but should not
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class SnapshotExistsException extends HBaseSnapshotException {
+
+  /**
+   * Failure due to the snapshot already existing
+   * @param msg full description of the failure
+   * @param desc snapshot that was attempted
+   */
+  public SnapshotExistsException(String msg, SnapshotDescription desc) {
+    super(msg, desc);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
new file mode 100644
index 000000000000..d0256e2ccdc0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.text.SimpleDateFormat;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.Date;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.HLogLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+
+/**
+ * Tool for dumping snapshot information.
+ * <ol>
+ * <li> Table Descriptor
+ * <li> Snapshot creation time, type, format version, ...
+ * <li> List of hfiles and hlogs
+ * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
+ * </ol>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class SnapshotInfo extends Configured implements Tool {
+  private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
+
+  private FileSystem fs;
+  private Path rootDir;
+
+  private HTableDescriptor snapshotTableDesc;
+  private SnapshotDescription snapshotDesc;
+  private Path snapshotDir;
+
+  @Override
+  public int run(String[] args) throws IOException, InterruptedException {
+    String snapshotName = null;
+    boolean showSchema = false;
+    boolean showFiles = false;
+    boolean showStats = false;
+
+    // Process command line args
+    for (int i = 0; i < args.length; i++) {
+      String cmd = args[i];
+      try {
+        if (cmd.equals("-snapshot")) {
+          snapshotName = args[++i];
+        } else if (cmd.equals("-files")) {
+          showFiles = true;
+        } else if (cmd.equals("-stats")) {
+          showStats = true;
+        } else if (cmd.equals("-schema")) {
+          showSchema = true;
+        } else if (cmd.equals("-h") || cmd.equals("--help")) {
+          printUsageAndExit();
+        } else {
+          System.err.println("UNEXPECTED: " + cmd);
+          printUsageAndExit();
+        }
+      } catch (Exception e) {
+        printUsageAndExit();
+      }
+    }
+
+    if (snapshotName == null) {
+      System.err.println("Missing snapshot name!");
+      printUsageAndExit();
+      return 1;
+    }
+
+    Configuration conf = getConf();
+    fs = FileSystem.get(conf);
+    rootDir = FSUtils.getRootDir(conf);
+
+    // Load snapshot information
+    if (!loadSnapshotInfo(snapshotName)) {
+      System.err.println("Snapshot '" + snapshotName + "' not found!");
+      return 1;
+    }
+
+    printInfo();
+    if (showSchema) printSchema();
+    if (showFiles || showStats) printFiles(showFiles);
+
+    return 0;
+  }
+
+  /**
+   * Load snapshot info and table descriptor for the specified snapshot
+   * @param snapshotName name of the snapshot to load
+   * @return false if snapshot is not found
+   */
+  private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
+    snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    if (!fs.exists(snapshotDir)) {
+      LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
+      return false;
+    }
+
+    snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+    return true;
+  }
+
+  /**
+   * Dump the {@link SnapshotDescription}
+   */
+  private void printInfo() {
+    SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+    System.out.println("Snapshot Info");
+    System.out.println("----------------------------------------");
+    System.out.println("   Name: " + snapshotDesc.getName());
+    System.out.println("   Type: " + snapshotDesc.getType());
+    System.out.println("  Table: " + snapshotDesc.getTable());
+    System.out.println(" Format: " + snapshotDesc.getVersion());
+    System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
+    System.out.println();
+  }
+
+  /**
+   * Dump the {@link HTableDescriptor}
+   */
+  private void printSchema() {
+    System.out.println("Table Descriptor");
+    System.out.println("----------------------------------------");
+    System.out.println(snapshotTableDesc.toString());
+    System.out.println();
+  }
+
+  /**
+   * Collect the hfiles and logs statistics of the snapshot and
+   * dump the file list if requested and the collected information.
+   */
+  private void printFiles(final boolean showFiles) throws IOException {
+    final String table = snapshotDesc.getTable();
+    final Configuration conf = getConf();
+
+    if (showFiles) {
+      System.out.println("Snapshot Files");
+      System.out.println("----------------------------------------");
+    }
+
+    // Collect information about hfiles and logs in the snapshot
+    final AtomicInteger hfileArchiveCount = new AtomicInteger();
+    final AtomicInteger hfilesMissing = new AtomicInteger();
+    final AtomicInteger hfilesCount = new AtomicInteger();
+    final AtomicInteger logsMissing = new AtomicInteger();
+    final AtomicInteger logsCount = new AtomicInteger();
+    final AtomicLong hfileArchiveSize = new AtomicLong();
+    final AtomicLong hfileSize = new AtomicLong();
+    final AtomicLong logSize = new AtomicLong();
+    SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
+      new SnapshotReferenceUtil.FileVisitor() {
+        public void storeFile (final String region, final String family, final String hfile)
+            throws IOException {
+          Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
+          HFileLink link = new HFileLink(conf, path);
+          boolean inArchive = false;
+          long size = -1;
+          try {
+            if ((inArchive = fs.exists(link.getArchivePath()))) {
+              size = fs.getFileStatus(link.getArchivePath()).getLen();
+              hfileArchiveSize.addAndGet(size);
+              hfileArchiveCount.addAndGet(1);
+            } else {
+              size = link.getFileStatus(fs).getLen();
+              hfileSize.addAndGet(size);
+              hfilesCount.addAndGet(1);
+            }
+          } catch (FileNotFoundException e) {
+            hfilesMissing.addAndGet(1);
+          }
+
+          if (showFiles) {
+            System.out.printf("%8s %s/%s/%s/%s %s%n",
+              (size < 0 ? "-" : StringUtils.humanReadableInt(size)),
+              table, region, family, hfile,
+              (inArchive ? "(archive)" : (size < 0) ? "(NOT FOUND)" : ""));
+          }
+        }
+
+        public void recoveredEdits (final String region, final String logfile)
+            throws IOException {
+          Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
+          long size = fs.getFileStatus(path).getLen();
+          logSize.addAndGet(size);
+          logsCount.addAndGet(1);
+
+          if (showFiles) {
+            System.out.printf("%8s recovered.edits %s on region %s%n",
+              StringUtils.humanReadableInt(size), logfile, region);
+          }
+        }
+
+        public void logFile (final String server, final String logfile)
+            throws IOException {
+          HLogLink logLink = new HLogLink(conf, server, logfile);
+          long size = -1;
+          try {
+            size = logLink.getFileStatus(fs).getLen();
+            logSize.addAndGet(size);
+            logsCount.addAndGet(1);
+          } catch (FileNotFoundException e) {
+            logsMissing.addAndGet(1);
+          }
+
+          if (showFiles) {
+            System.out.printf("%8s log %s on server %s %s%n",
+              (size < 0 ? "-" : StringUtils.humanReadableInt(size)),
+              logfile, server,
+              (size < 0 ? "(NOT FOUND)" : ""));
+          }
+        }
+    });
+
+    // Dump the stats
+    System.out.println();
+    if (hfilesMissing.get() > 0 || logsMissing.get() > 0) {
+      System.out.println("**************************************************************");
+      System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
+        hfilesMissing.get(), logsMissing.get());
+      System.out.println("**************************************************************");
+    }
+
+    System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
+      hfilesCount.get() + hfileArchiveCount.get(), hfileArchiveCount.get(),
+      StringUtils.humanReadableInt(hfileSize.get() + hfileArchiveSize.get()),
+      ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100,
+      StringUtils.humanReadableInt(hfileSize.get())
+    );
+    System.out.printf("%d Logs, total size %s%n",
+      logsCount.get(), StringUtils.humanReadableInt(logSize.get()));
+    System.out.println();
+  }
+
+  private void printUsageAndExit() {
+    System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
+    System.err.println(" where [options] are:");
+    System.err.println("  -h|-help                Show this help and exit.");
+    System.err.println("  -snapshot NAME          Snapshot to examine.");
+    System.err.println("  -files                  Files and logs list.");
+    System.err.println("  -stats                  Files and logs stats.");
+    System.err.println("  -schema                 Describe the snapshotted table.");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println("  hbase " + getClass() + " \\");
+    System.err.println("    -snapshot MySnapshot -files");
+    System.exit(1);
+  }
+
+  /**
+   * The guts of the {@link #main} method.
+   * Call this method to avoid the {@link #main(String[])} System.exit.
+   * @param args
+   * @return errCode
+   * @throws Exception
+   */
+  static int innerMain(final String [] args) throws Exception {
+    return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
+  }
+
+  public static void main(String[] args) throws Exception {
+     System.exit(innerMain(args));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java
new file mode 100644
index 000000000000..788435c79a99
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.HLogLink;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * If the snapshot has references to one or more log files,
+ * those must be split (each log contains multiple tables and regions)
+ * and must be placed in the region/recovered.edits folder.
+ * (recovered.edits files will be played on region startup)
+ *
+ * In case of Restore: the log can just be split in the recovered.edits folder.
+ * In case of Clone: each entry in the log must be modified to use the new region name.
+ * (region names are encoded with: tableName, startKey, regionIdTimeStamp)
+ *
+ * We can't use the normal split code, because the HLogKey contains the
+ * table name and the region name, and in case of "clone from snapshot"
+ * region name and table name will be different and must be replaced in
+ * the recovered.edits.
+ */
+@InterfaceAudience.Private
+class SnapshotLogSplitter implements Closeable {
+  static final Log LOG = LogFactory.getLog(SnapshotLogSplitter.class);
+
+  private final class LogWriter implements Closeable {
+    private HLog.Writer writer;
+    private Path logFile;
+    private long seqId;
+
+    public LogWriter(final Configuration conf, final FileSystem fs,
+        final Path logDir, long seqId) throws IOException {
+      logFile = new Path(logDir, logFileName(seqId, true));
+      this.writer = HLog.createWriter(fs, logFile, conf);
+      this.seqId = seqId;
+    }
+
+    public void close() throws IOException {
+      writer.close();
+
+      Path finalFile = new Path(logFile.getParent(), logFileName(seqId, false));
+      LOG.debug("LogWriter tmpLogFile=" + logFile + " -> logFile=" + finalFile);
+      fs.rename(logFile, finalFile);
+    }
+
+    public void append(final HLog.Entry entry) throws IOException {
+      writer.append(entry);
+      if (seqId < entry.getKey().getLogSeqNum()) {
+        seqId = entry.getKey().getLogSeqNum();
+      }
+    }
+
+    private String logFileName(long seqId, boolean temp) {
+      String fileName = String.format("%019d", seqId);
+      if (temp) fileName += HLog.RECOVERED_LOG_TMPFILE_SUFFIX;
+      return fileName;
+    }
+  }
+
+  private final Map<byte[], LogWriter> regionLogWriters =
+      new TreeMap<byte[], LogWriter>(Bytes.BYTES_COMPARATOR);
+
+  private final Map<byte[], byte[]> regionsMap;
+  private final Configuration conf;
+  private final byte[] snapshotTableName;
+  private final byte[] tableName;
+  private final Path tableDir;
+  private final FileSystem fs;
+
+  /**
+   * @params tableName snapshot table name
+   * @params regionsMap maps original region names to the new ones.
+   */
+  public SnapshotLogSplitter(final Configuration conf, final FileSystem fs,
+      final Path tableDir, final byte[] snapshotTableName,
+      final Map<byte[], byte[]> regionsMap) {
+    this.regionsMap = regionsMap;
+    this.snapshotTableName = snapshotTableName;
+    this.tableName = Bytes.toBytes(tableDir.getName());
+    this.tableDir = tableDir;
+    this.conf = conf;
+    this.fs = fs;
+  }
+
+  public void close() throws IOException {
+    for (LogWriter writer: regionLogWriters.values()) {
+      writer.close();
+    }
+  }
+
+  public void splitLog(final String serverName, final String logfile) throws IOException {
+    LOG.debug("Restore log=" + logfile + " server=" + serverName +
+              " for snapshotTable=" + Bytes.toString(snapshotTableName) +
+              " to table=" + Bytes.toString(tableName));
+    splitLog(new HLogLink(conf, serverName, logfile).getAvailablePath(fs));
+  }
+
+  public void splitRecoveredEdit(final Path editPath) throws IOException {
+    LOG.debug("Restore recover.edits=" + editPath +
+              " for snapshotTable=" + Bytes.toString(snapshotTableName) +
+              " to table=" + Bytes.toString(tableName));
+    splitLog(editPath);
+  }
+
+  /**
+   * Split the snapshot HLog reference into regions recovered.edits.
+   *
+   * The HLogKey contains the table name and the region name,
+   * and they must be changed to the restored table names.
+   *
+   * @param logPath Snapshot HLog reference path
+   */
+  public void splitLog(final Path logPath) throws IOException {
+    HLog.Reader log = HLog.getReader(fs, logPath, conf);
+    try {
+      HLog.Entry entry;
+      LogWriter writer = null;
+      byte[] regionName = null;
+      byte[] newRegionName = null;
+      while ((entry = log.next()) != null) {
+        HLogKey key = entry.getKey();
+
+        // We're interested only in the snapshot table that we're restoring
+        if (!Bytes.equals(key.getTablename(), snapshotTableName)) continue;
+
+        // Writer for region.
+        if (!Bytes.equals(regionName, key.getEncodedRegionName())) {
+          regionName = key.getEncodedRegionName().clone();
+
+          // Get the new region name in case of clone, or use the original one
+          newRegionName = regionsMap.get(regionName);
+          if (newRegionName == null) newRegionName = regionName;
+
+          writer = getOrCreateWriter(newRegionName, key.getLogSeqNum());
+          LOG.debug("+ regionName=" + Bytes.toString(regionName));
+        }
+
+        // Append Entry
+        key = new HLogKey(newRegionName, tableName,
+                          key.getLogSeqNum(), key.getWriteTime(), key.getClusterId());
+        writer.append(new HLog.Entry(key, entry.getEdit()));
+      }
+    } catch (IOException e) {
+      LOG.warn("Something wrong during the log split", e);
+    } finally {
+      log.close();
+    }
+  }
+
+  /**
+   * Create a LogWriter for specified region if not already created.
+   */
+  private LogWriter getOrCreateWriter(final byte[] regionName, long seqId) throws IOException {
+    LogWriter writer = regionLogWriters.get(regionName);
+    if (writer == null) {
+      Path regionDir = HRegion.getRegionDir(tableDir, Bytes.toString(regionName));
+      Path dir = HLog.getRegionDirRecoveredEditsDir(regionDir);
+      fs.mkdirs(dir);
+
+      writer = new LogWriter(conf, fs, dir, seqId);
+      regionLogWriters.put(regionName, writer);
+    }
+    return(writer);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
new file mode 100644
index 000000000000..eee205792a56
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.TreeMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSVisitor;
+
+/**
+ * Utility methods for interacting with the snapshot referenced files.
+ */
+@InterfaceAudience.Private
+public final class SnapshotReferenceUtil {
+  public interface FileVisitor extends FSVisitor.StoreFileVisitor,
+    FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
+  }
+
+  private SnapshotReferenceUtil() {
+    // private constructor for utility class
+  }
+
+  /**
+   * Get log directory for a server in a snapshot.
+   *
+   * @param snapshotDir directory where the specific snapshot is stored
+   * @param serverName name of the parent regionserver for the log files
+   * @return path to the log home directory for the archive files.
+   */
+  public static Path getLogsDir(Path snapshotDir, String serverName) {
+    return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName));
+  }
+
+  /**
+   * Get the snapshotted recovered.edits dir for the specified region.
+   *
+   * @param snapshotDir directory where the specific snapshot is stored
+   * @param regionName name of the region
+   * @return path to the recovered.edits directory for the specified region files.
+   */
+  public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
+    return HLog.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
+  }
+
+  /**
+   * Get the snapshot recovered.edits file
+   *
+   * @param snapshotDir directory where the specific snapshot is stored
+   * @param regionName name of the region
+   * @param logfile name of the edit file
+   * @return full path of the log file for the specified region files.
+   */
+  public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
+    return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
+  }
+
+  /**
+   * Iterate over the snapshot store files, restored.edits and logs
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param visitor callback object to get the referenced files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
+      final FileVisitor visitor) throws IOException {
+    visitTableStoreFiles(fs, snapshotDir, visitor);
+    visitRecoveredEdits(fs, snapshotDir, visitor);
+    visitLogFiles(fs, snapshotDir, visitor);
+  }
+
+  /**
+   * Iterate over the snapshot store files
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param visitor callback object to get the store files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
+      final FSVisitor.StoreFileVisitor visitor) throws IOException {
+    FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
+  }
+
+  /**
+   * Iterate over the snapshot store files in the specified region
+   *
+   * @param fs {@link FileSystem}
+   * @param regionDir {@link Path} to the Snapshot region directory
+   * @param visitor callback object to get the store files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
+      final FSVisitor.StoreFileVisitor visitor) throws IOException {
+    FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
+  }
+
+  /**
+   * Iterate over the snapshot recovered.edits
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param visitor callback object to get the recovered.edits files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
+      final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
+    FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
+  }
+
+  /**
+   * Iterate over the snapshot log files
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param visitor callback object to get the log files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
+      final FSVisitor.LogFileVisitor visitor) throws IOException {
+    FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
+  }
+
+  /**
+   * Returns the set of region names available in the snapshot.
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @throws IOException if an error occurred while scanning the directory
+   * @return the set of the regions contained in the snapshot
+   */
+  public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
+      throws IOException {
+    FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
+    if (regionDirs == null) return null;
+
+    Set<String> regions = new HashSet<String>();
+    for (FileStatus regionDir: regionDirs) {
+      regions.add(regionDir.getPath().getName());
+    }
+    return regions;
+  }
+
+  /**
+   * Get the list of hfiles for the specified snapshot region.
+   * NOTE: The current implementation keeps one empty file per HFile in the region.
+   * The file name matches the one in the original table, and by reconstructing
+   * the path you can quickly jump to the referenced file.
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotRegionDir {@link Path} to the Snapshot region directory
+   * @return Map of hfiles per family, the key is the family name and values are hfile names
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
+      final Path snapshotRegionDir) throws IOException {
+    final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
+
+    visitRegionStoreFiles(fs, snapshotRegionDir,
+      new FSVisitor.StoreFileVisitor() {
+        public void storeFile (final String region, final String family, final String hfile)
+            throws IOException {
+          List<String> hfiles = familyFiles.get(family);
+          if (hfiles == null) {
+            hfiles = new LinkedList<String>();
+            familyFiles.put(family, hfiles);
+          }
+          hfiles.add(hfile);
+        }
+    });
+
+    return familyFiles;
+  }
+
+  /**
+   * Returns the store file names in the snapshot.
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @throws IOException if an error occurred while scanning the directory
+   * @return the names of hfiles in the specified snaphot
+   */
+  public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
+      throws IOException {
+    final Set<String> names = new HashSet<String>();
+    visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile (final String region, final String family, final String hfile)
+          throws IOException {
+        if (HFileLink.isHFileLink(hfile)) {
+          names.add(HFileLink.getReferencedHFileName(hfile));
+        } else {
+          names.add(hfile);
+        }
+      }
+    });
+    return names;
+  }
+
+  /**
+   * Returns the log file names available in the snapshot.
+   *
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @throws IOException if an error occurred while scanning the directory
+   * @return the names of hlogs in the specified snaphot
+   */
+  public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
+      throws IOException {
+    final Set<String> names = new HashSet<String>();
+    visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
+      public void logFile (final String server, final String logfile) throws IOException {
+        names.add(logfile);
+      }
+    });
+    return names;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java
new file mode 100644
index 000000000000..ede2d85ecd61
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * General snapshot operation taken on a regionserver
+ */
+@InterfaceAudience.Private
+public abstract class SnapshotTask implements ForeignExceptionSnare, Callable<Void>{
+
+  protected final SnapshotDescription snapshot;
+  protected final ForeignExceptionDispatcher errorMonitor;
+
+  /**
+   * @param snapshot Description of the snapshot we are going to operate on
+   * @param monitor listener interested in failures to the snapshot caused by this operation
+   */
+  public SnapshotTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor) {
+    assert monitor != null : "ForeignExceptionDispatcher must not be null!";
+    assert snapshot != null : "SnapshotDescription must not be null!";
+    this.snapshot = snapshot;
+    this.errorMonitor = monitor;
+  }
+
+  public void snapshotFailure(String message, Exception e) {
+    ForeignException ee = new ForeignException(message, e);
+    errorMonitor.receive(ee);
+  }
+
+  @Override
+  public void rethrowException() throws ForeignException {
+    this.errorMonitor.rethrowException();
+  }
+
+  @Override
+  public boolean hasException() {
+    return this.errorMonitor.hasException();
+  }
+
+  @Override
+  public ForeignException getException() {
+    return this.errorMonitor.getException();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java
new file mode 100644
index 000000000000..9d431b48d9cd
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+
+/**
+ * Copy the table info into the snapshot directory
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class TableInfoCopyTask extends SnapshotTask {
+
+  public static final Log LOG = LogFactory.getLog(TableInfoCopyTask.class);
+  private final FileSystem fs;
+  private final Path rootDir;
+
+  /**
+   * Copy the table info for the given table into the snapshot
+   * @param monitor listen for errors while running the snapshot
+   * @param snapshot snapshot for which we are copying the table info
+   * @param fs {@link FileSystem} where the tableinfo is stored (and where the copy will be written)
+   * @param rootDir root of the {@link FileSystem} where the tableinfo is stored
+   */
+  public TableInfoCopyTask(ForeignExceptionDispatcher monitor,
+      SnapshotDescription snapshot, FileSystem fs, Path rootDir) {
+    super(snapshot, monitor);
+    this.rootDir = rootDir;
+    this.fs = fs;
+  }
+
+  @Override
+  public Void call() throws Exception {
+    LOG.debug("Running table info copy.");
+    this.rethrowException();
+    LOG.debug("Attempting to copy table info for snapshot:"
+        + SnapshotDescriptionUtils.toString(this.snapshot));
+    // get the HTable descriptor
+    HTableDescriptor orig = FSTableDescriptors.getTableDescriptor(fs, rootDir,
+      Bytes.toBytes(this.snapshot.getTable()));
+    this.rethrowException();
+    // write a copy of descriptor to the snapshot directory
+    Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+    FSTableDescriptors.createTableDescriptorForTableDirectory(fs, snapshotDir, orig, false);
+    LOG.debug("Finished copying tableinfo.");
+    return null;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java
new file mode 100644
index 000000000000..6b27be8ab063
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Thrown if a table should be online/offline but is partially open
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TablePartiallyOpenException extends IOException {
+  private static final long serialVersionUID = 3571982660065058361L;
+
+  public TablePartiallyOpenException() {
+    super();
+  }
+
+  /**
+   * @param s message
+   */
+  public TablePartiallyOpenException(String s) {
+    super(s);
+  }
+
+  /**
+   * @param tableName Name of table that is partial open
+   */
+  public TablePartiallyOpenException(byte[] tableName) {
+    this(Bytes.toString(tableName));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java b/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java
new file mode 100644
index 000000000000..220e9642dc4f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java
@@ -0,0 +1,323 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+/**
+ * Utilities for useful when taking a snapshot
+ */
+public class TakeSnapshotUtils {
+
+  private static final Log LOG = LogFactory.getLog(TakeSnapshotUtils.class);
+
+  private TakeSnapshotUtils() {
+    // private constructor for util class
+  }
+
+  /**
+   * Get the per-region snapshot description location.
+   * <p>
+   * Under the per-snapshot directory, specific files per-region are kept in a similar layout as per
+   * the current directory layout.
+   * @param desc description of the snapshot
+   * @param rootDir root directory for the hbase installation
+   * @param regionName encoded name of the region (see {@link HRegionInfo#encodeRegionName(byte[])})
+   * @return path to the per-region directory for the snapshot
+   */
+  public static Path getRegionSnapshotDirectory(SnapshotDescription desc, Path rootDir,
+      String regionName) {
+    Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+    return HRegion.getRegionDir(snapshotDir, regionName);
+  }
+
+  /**
+   * Get the home directory for store-level snapshot files.
+   * <p>
+   * Specific files per store are kept in a similar layout as per the current directory layout.
+   * @param regionDir snapshot directory for the parent region, <b>not</b> the standard region
+   *          directory. See {@link #getRegionSnapshotDirectory}
+   * @param family name of the store to snapshot
+   * @return path to the snapshot home directory for the store/family
+   */
+  public static Path getStoreSnapshotDirectory(Path regionDir, String family) {
+    return Store.getStoreHomedir(regionDir, Bytes.toBytes(family));
+  }
+
+  /**
+   * Get the snapshot directory for each family to be added to the the snapshot
+   * @param snapshot description of the snapshot being take
+   * @param snapshotRegionDir directory in the snapshot where the region directory information
+   *          should be stored
+   * @param families families to be added (can be null)
+   * @return paths to the snapshot directory for each family, in the same order as the families
+   *         passed in
+   */
+  public static List<Path> getFamilySnapshotDirectories(SnapshotDescription snapshot,
+      Path snapshotRegionDir, FileStatus[] families) {
+    if (families == null || families.length == 0) return Collections.emptyList();
+
+    List<Path> familyDirs = new ArrayList<Path>(families.length);
+    for (FileStatus family : families) {
+      // build the reference directory name
+      familyDirs.add(getStoreSnapshotDirectory(snapshotRegionDir, family.getPath().getName()));
+    }
+    return familyDirs;
+  }
+
+  /**
+   * Create a snapshot timer for the master which notifies the monitor when an error occurs
+   * @param snapshot snapshot to monitor
+   * @param conf configuration to use when getting the max snapshot life
+   * @param monitor monitor to notify when the snapshot life expires
+   * @return the timer to use update to signal the start and end of the snapshot
+   */
+  public static TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot,
+      Configuration conf, ForeignExceptionListener monitor) {
+    long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
+      SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
+    return new TimeoutExceptionInjector(monitor, maxTime);
+  }
+
+  /**
+   * Verify that all the expected logs got referenced
+   * @param fs filesystem where the logs live
+   * @param logsDir original logs directory
+   * @param serverNames names of the servers that involved in the snapshot
+   * @param snapshot description of the snapshot being taken
+   * @param snapshotLogDir directory for logs in the snapshot
+   * @throws IOException
+   */
+  public static void verifyAllLogsGotReferenced(FileSystem fs, Path logsDir,
+      Set<String> serverNames, SnapshotDescription snapshot, Path snapshotLogDir)
+      throws IOException {
+    assertTrue(snapshot, "Logs directory doesn't exist in snapshot", fs.exists(logsDir));
+    // for each of the server log dirs, make sure it matches the main directory
+    Multimap<String, String> snapshotLogs = getMapOfServersAndLogs(fs, snapshotLogDir, serverNames);
+    Multimap<String, String> realLogs = getMapOfServersAndLogs(fs, logsDir, serverNames);
+    if (realLogs != null) {
+      assertNotNull(snapshot, "No server logs added to snapshot", snapshotLogs);
+    } else {
+      assertNull(snapshot, "Snapshotted server logs that don't exist", snapshotLogs);
+    }
+
+    // check the number of servers
+    Set<Entry<String, Collection<String>>> serverEntries = realLogs.asMap().entrySet();
+    Set<Entry<String, Collection<String>>> snapshotEntries = snapshotLogs.asMap().entrySet();
+    assertEquals(snapshot, "Not the same number of snapshot and original server logs directories",
+      serverEntries.size(), snapshotEntries.size());
+
+    // verify we snapshotted each of the log files
+    for (Entry<String, Collection<String>> serverLogs : serverEntries) {
+      // if the server is not the snapshot, skip checking its logs
+      if (!serverNames.contains(serverLogs.getKey())) continue;
+      Collection<String> snapshotServerLogs = snapshotLogs.get(serverLogs.getKey());
+      assertNotNull(snapshot, "Snapshots missing logs for server:" + serverLogs.getKey(),
+        snapshotServerLogs);
+
+      // check each of the log files
+      assertEquals(snapshot,
+        "Didn't reference all the log files for server:" + serverLogs.getKey(), serverLogs
+            .getValue().size(), snapshotServerLogs.size());
+      for (String log : serverLogs.getValue()) {
+        assertTrue(snapshot, "Snapshot logs didn't include " + log,
+          snapshotServerLogs.contains(log));
+      }
+    }
+  }
+
+  /**
+   * Verify one of a snapshot's region's recovered.edits, has been at the surface (file names,
+   * length), match the original directory.
+   * @param fs filesystem on which the snapshot had been taken
+   * @param rootDir full path to the root hbase directory
+   * @param regionInfo info for the region
+   * @param snapshot description of the snapshot that was taken
+   * @throws IOException if there is an unexpected error talking to the filesystem
+   */
+  public static void verifyRecoveredEdits(FileSystem fs, Path rootDir, HRegionInfo regionInfo,
+      SnapshotDescription snapshot) throws IOException {
+    Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+    Path editsDir = HLog.getRegionDirRecoveredEditsDir(regionDir);
+    Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
+      regionInfo.getEncodedName());
+    Path snapshotEditsDir = HLog.getRegionDirRecoveredEditsDir(snapshotRegionDir);
+
+    FileStatus[] edits = FSUtils.listStatus(fs, editsDir);
+    FileStatus[] snapshotEdits = FSUtils.listStatus(fs, snapshotEditsDir);
+    if (edits == null) {
+      assertNull(snapshot, "Snapshot has edits but table doesn't", snapshotEdits);
+      return;
+    }
+
+    assertNotNull(snapshot, "Table has edits, but snapshot doesn't", snapshotEdits);
+
+    // check each of the files
+    assertEquals(snapshot, "Not same number of edits in snapshot as table", edits.length,
+      snapshotEdits.length);
+
+    // make sure we have a file with the same name as the original
+    // it would be really expensive to verify the content matches the original
+    for (FileStatus edit : edits) {
+      for (FileStatus sEdit : snapshotEdits) {
+        if (sEdit.getPath().equals(edit.getPath())) {
+          assertEquals(snapshot, "Snapshot file" + sEdit.getPath()
+              + " length not equal to the original: " + edit.getPath(), edit.getLen(),
+            sEdit.getLen());
+          break;
+        }
+      }
+      assertTrue(snapshot, "No edit in snapshot with name:" + edit.getPath(), false);
+    }
+  }
+
+  private static void assertNull(SnapshotDescription snapshot, String msg, Object isNull)
+      throws CorruptedSnapshotException {
+    if (isNull != null) {
+      throw new CorruptedSnapshotException(msg + ", Expected " + isNull + " to be null.", snapshot);
+    }
+  }
+
+  private static void assertNotNull(SnapshotDescription snapshot, String msg, Object notNull)
+      throws CorruptedSnapshotException {
+    if (notNull == null) {
+      throw new CorruptedSnapshotException(msg + ", Expected object to not be null, but was null.",
+          snapshot);
+    }
+  }
+
+  private static void assertTrue(SnapshotDescription snapshot, String msg, boolean isTrue)
+      throws CorruptedSnapshotException {
+    if (!isTrue) {
+      throw new CorruptedSnapshotException(msg + ", Expected true, but was false", snapshot);
+    }
+  }
+
+  /**
+   * Assert that the expect matches the gotten amount
+   * @param msg message to add the to exception
+   * @param expected
+   * @param gotten
+   * @throws CorruptedSnapshotException thrown if the two elements don't match
+   */
+  private static void assertEquals(SnapshotDescription snapshot, String msg, int expected,
+      int gotten) throws CorruptedSnapshotException {
+    if (expected != gotten) {
+      throw new CorruptedSnapshotException(msg + ". Expected:" + expected + ", got:" + gotten,
+          snapshot);
+    }
+  }
+
+  /**
+   * Assert that the expect matches the gotten amount
+   * @param msg message to add the to exception
+   * @param expected
+   * @param gotten
+   * @throws CorruptedSnapshotException thrown if the two elements don't match
+   */
+  private static void assertEquals(SnapshotDescription snapshot, String msg, long expected,
+      long gotten) throws CorruptedSnapshotException {
+    if (expected != gotten) {
+      throw new CorruptedSnapshotException(msg + ". Expected:" + expected + ", got:" + gotten,
+          snapshot);
+    }
+  }
+
+  /**
+   * @param logdir
+   * @param toInclude list of servers to include. If empty or null, returns all servers
+   * @return maps of servers to all their log files. If there is no log directory, returns
+   *         <tt>null</tt>
+   */
+  private static Multimap<String, String> getMapOfServersAndLogs(FileSystem fs, Path logdir,
+      Collection<String> toInclude) throws IOException {
+    // create a path filter based on the passed directories to include
+    PathFilter filter = toInclude == null || toInclude.size() == 0 ? null
+        : new MatchesDirectoryNames(toInclude);
+
+    // get all the expected directories
+    FileStatus[] serverLogDirs = FSUtils.listStatus(fs, logdir, filter);
+    if (serverLogDirs == null) return null;
+
+    // map those into a multimap of servername -> [log files]
+    Multimap<String, String> map = HashMultimap.create();
+    for (FileStatus server : serverLogDirs) {
+      FileStatus[] serverLogs = FSUtils.listStatus(fs, server.getPath(), null);
+      if (serverLogs == null) continue;
+      for (FileStatus log : serverLogs) {
+        map.put(server.getPath().getName(), log.getPath().getName());
+      }
+    }
+    return map;
+  }
+
+  /**
+   * Path filter that only accepts paths where that have a {@link Path#getName()} that is contained
+   * in the specified collection.
+   */
+  private static class MatchesDirectoryNames implements PathFilter {
+
+    Collection<String> paths;
+
+    public MatchesDirectoryNames(Collection<String> dirNames) {
+      this.paths = dirNames;
+    }
+
+    @Override
+    public boolean accept(Path path) {
+      return paths.contains(path.getName());
+    }
+  }
+
+  /**
+   * Get the log directory for a specific snapshot
+   * @param snapshotDir directory where the specific snapshot will be store
+   * @param serverName name of the parent regionserver for the log files
+   * @return path to the log home directory for the archive files.
+   */
+  public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) {
+    return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java
new file mode 100644
index 000000000000..a6b381fe3b3f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Exception thrown when we get a request for a snapshot we don't recognize.
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class UnknownSnapshotException extends HBaseSnapshotException {
+
+  /**
+   * @param msg full information about the failure
+   */
+  public UnknownSnapshotException(String msg) {
+    super(msg);
+  }
+
+  public UnknownSnapshotException(String msg, Exception  e) {
+    super(msg, e);
+  }
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 51411219a313..786b334bff7f 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -597,8 +597,25 @@ public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
   public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
       HTableDescriptor htableDescriptor, boolean forceCreation)
   throws IOException {
-    FileStatus status =
-      getTableInfoPath(fs, rootdir, htableDescriptor.getNameAsString());
+    Path tabledir = FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString());
+    return createTableDescriptorForTableDirectory(fs, tabledir, htableDescriptor, forceCreation);
+  }
+
+  /**
+   * Create a new HTableDescriptor in HDFS in the specified table directory. Happens when we create
+   * a new table or snapshot a table.
+   * @param fs filesystem where the descriptor should be written
+   * @param tabledir directory under which we should write the file
+   * @param htableDescriptor description of the table to write
+   * @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
+   *          be overwritten
+   * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
+   *         already exists and we weren't forcing the descriptor creation.
+   * @throws IOException if a filesystem error occurs
+   */
+  public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tabledir,
+      HTableDescriptor htableDescriptor, boolean forceCreation) throws IOException {
+    FileStatus status = getTableInfoPath(fs, tabledir);
     if (status != null) {
       LOG.info("Current tableInfoPath = " + status.getPath());
       if (!forceCreation) {
@@ -608,8 +625,7 @@ public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
         }
       }
     }
-    Path p = writeTableDescriptor(fs, htableDescriptor,
-      FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString()), status);
+    Path p = writeTableDescriptor(fs, htableDescriptor, tabledir, status);
     return p != null;
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 62f7ed5cd604..1242088abbeb 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -75,7 +75,7 @@ protected FSUtils() {
   public static FSUtils getInstance(FileSystem fs, Configuration conf) {
     String scheme = fs.getUri().getScheme();
     if (scheme == null) {
-      LOG.warn("Could not find scheme for uri " + 
+      LOG.warn("Could not find scheme for uri " +
           fs.getUri() + ", default to hdfs");
       scheme = "hdfs";
     }
@@ -122,7 +122,7 @@ public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
    * <li>use the default block size</li>
    * <li>not track progress</li>
    * </ol>
-   * 
+   *
    * @param fs {@link FileSystem} on which to write the file
    * @param path {@link Path} to the file to write
    * @return output stream to the created file
@@ -143,7 +143,7 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
    * <li>use the default block size</li>
    * <li>not track progress</li>
    * </ol>
-   * 
+   *
    * @param fs {@link FileSystem} on which to write the file
    * @param path {@link Path} to the file to write
    * @param perm
@@ -163,7 +163,7 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
   /**
    * Get the file permissions specified in the configuration, if they are
    * enabled.
-   * 
+   *
    * @param fs filesystem that the file will be created on.
    * @param conf configuration to read for determining if permissions are
    *          enabled and which to use
@@ -222,7 +222,7 @@ public static void checkFileSystemAvailable(final FileSystem fs)
     try {
       fs.close();
     } catch (Exception e) {
-        LOG.error("file system close failed: ", e);
+      LOG.error("file system close failed: ", e);
     }
     IOException io = new IOException("File system is not available");
     io.initCause(exception);
@@ -255,11 +255,11 @@ private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOExceptio
   }
   
   /**
-   * Check whether dfs is in safemode. 
+   * Check whether dfs is in safemode.
    * @param conf
    * @throws IOException
    */
-  public static void checkDfsSafeMode(final Configuration conf) 
+  public static void checkDfsSafeMode(final Configuration conf)
   throws IOException {
     boolean isInSafeMode = false;
     FileSystem fs = FileSystem.get(conf);
@@ -271,7 +271,7 @@ public static void checkDfsSafeMode(final Configuration conf)
       throw new IOException("File system is in safemode, it can't be written now");
     }
   }
-  
+
   /**
    * Verifies current version of file system
    *
@@ -309,7 +309,7 @@ public static String getVersion(FileSystem fs, Path rootdir)
    */
   public static void checkVersion(FileSystem fs, Path rootdir,
       boolean message) throws IOException {
-    checkVersion(fs, rootdir, message, 0, 
+    checkVersion(fs, rootdir, message, 0,
     		HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
   }
 
@@ -359,7 +359,7 @@ public static void checkVersion(FileSystem fs, Path rootdir,
    */
   public static void setVersion(FileSystem fs, Path rootdir)
   throws IOException {
-    setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0, 
+    setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
     		HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
   }
 
@@ -406,7 +406,7 @@ public static void setVersion(FileSystem fs, Path rootdir, String version,
           fs.delete(versionFile, false);
           try {
             if (wait > 0) {
-              Thread.sleep(wait);  						
+              Thread.sleep(wait);
             }
           } catch (InterruptedException ex) {
             // ignore
@@ -612,9 +612,9 @@ public static boolean rootRegionExists(FileSystem fs, Path rootdir)
    * @param fs file system
    * @param status file status of the file
    * @param start start position of the portion
-   * @param length length of the portion 
+   * @param length length of the portion
    * @return The HDFS blocks distribution
-   */  
+   */
   static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
     final FileSystem fs, FileStatus status, long start, long length)
     throws IOException {
@@ -626,12 +626,12 @@ static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
       long len = bl.getLength();
       blocksDistribution.addHostsAndBlockWeight(hosts, len);
     }
-    
+
     return blocksDistribution;
   }
-  
 
-  
+
+
   /**
    * Runs through the hbase rootdir and checks all stores have only
    * one file in them -- that is, they've been major compacted.  Looks
@@ -850,6 +850,27 @@ public static boolean isMajorCompactedPre020(final FileSystem fs,
     return true;
   }
 
+  /**
+   * A {@link PathFilter} that returns only regular files.
+   */
+  static class FileFilter implements PathFilter {
+    private final FileSystem fs;
+
+    public FileFilter(final FileSystem fs) {
+      this.fs = fs;
+    }
+
+    @Override
+    public boolean accept(Path p) {
+      try {
+        return fs.isFile(p);
+      } catch (IOException e) {
+        LOG.debug("unable to verify if path=" + p + " is a regular file", e);
+        return false;
+      }
+    }
+  }
+
   /**
    * A {@link PathFilter} that returns directories.
    */
@@ -860,13 +881,14 @@ public DirFilter(final FileSystem fs) {
       this.fs = fs;
     }
 
+    @Override
     public boolean accept(Path p) {
       boolean isValid = false;
       try {
         if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(p)) {
           isValid = false;
         } else {
-            isValid = this.fs.getFileStatus(p).isDir();
+          isValid = this.fs.getFileStatus(p).isDir();
         }
       } catch (IOException e) {
         e.printStackTrace();
@@ -920,7 +942,7 @@ public static boolean isHDFS(final Configuration conf) throws IOException {
   }
 
   /**
-   * Recover file lease. Used when a file might be suspect 
+   * Recover file lease. Used when a file might be suspect
    * to be had been left open by another process.
    * @param fs FileSystem handle
    * @param p Path of file to recover lease
@@ -929,7 +951,7 @@ public static boolean isHDFS(final Configuration conf) throws IOException {
    */
   public abstract void recoverFileLease(final FileSystem fs, final Path p,
       Configuration conf) throws IOException;
-  
+
   /**
    * @param fs
    * @param rootdir
@@ -1096,10 +1118,10 @@ public static FileSystem getCurrentFileSystem(Configuration conf)
   throws IOException {
     return getRootDir(conf).getFileSystem(conf);
   }
-  
+
   /**
-   * Runs through the HBase rootdir and creates a reverse lookup map for 
-   * table StoreFile names to the full Path. 
+   * Runs through the HBase rootdir and creates a reverse lookup map for
+   * table StoreFile names to the full Path.
    * <br>
    * Example...<br>
    * Key = 3944417774205889744  <br>
@@ -1146,17 +1168,17 @@ public static Map<String, Path> getTableStoreFilePathMap(
             Path sf = sfStatus.getPath();
             map.put( sf.getName(), sf);
           }
-          
+
         }
       }
     }
       return map;
   }
-  
+
   /**
    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
-   * This would accommodate difference in various hadoop versions
-   * 
+   * This accommodates differences between hadoop versions
+   *
    * @param fs file system
    * @param dir directory
    * @param filter path filter
@@ -1169,15 +1191,27 @@ public static Map<String, Path> getTableStoreFilePathMap(
       status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
     } catch (FileNotFoundException fnfe) {
       // if directory doesn't exist, return null
-      LOG.info(dir + " doesn't exist");
+      LOG.debug(dir + " doesn't exist");
     }
     if (status == null || status.length < 1) return null;
     return status;
   }
-  
+
+  /**
+   * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
+   * This would accommodates differences between hadoop versions
+   *
+   * @param fs file system
+   * @param dir directory
+   * @return null if tabledir doesn't exist, otherwise FileStatus array
+   */
+  public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
+    return listStatus(fs, dir, null);
+  }
+
   /**
    * Calls fs.delete() and returns the value returned by the fs.delete()
-   * 
+   *
    * @param fs
    * @param path
    * @param recursive
@@ -1229,7 +1263,7 @@ private static boolean contains(String[] groups, String user) {
 
   /**
    * Calls fs.exists(). Checks if the specified path exists
-   * 
+   *
    * @param fs
    * @param path
    * @return
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java b/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
new file mode 100644
index 000000000000..a945c039e82a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
@@ -0,0 +1,194 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.NavigableSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Utility methods for interacting with the hbase.root file system.
+ */
+@InterfaceAudience.Private
+public final class FSVisitor {
+  private static final Log LOG = LogFactory.getLog(FSVisitor.class);
+
+  public interface StoreFileVisitor {
+    void storeFile(final String region, final String family, final String hfileName)
+       throws IOException;
+  }
+
+  public interface RecoveredEditsVisitor {
+    void recoveredEdits (final String region, final String logfile)
+      throws IOException;
+  }
+
+  public interface LogFileVisitor {
+    void logFile (final String server, final String logfile)
+      throws IOException;
+  }
+
+  private FSVisitor() {
+    // private constructor for utility class
+  }
+
+  /**
+   * Iterate over the table store files
+   *
+   * @param fs {@link FileSystem}
+   * @param tableDir {@link Path} to the table directory
+   * @param visitor callback object to get the store files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
+      final StoreFileVisitor visitor) throws IOException {
+    FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
+    if (regions == null) {
+      LOG.info("No regions under directory:" + tableDir);
+      return;
+    }
+
+    for (FileStatus region: regions) {
+      visitRegionStoreFiles(fs, region.getPath(), visitor);
+    }
+  }
+
+  /**
+   * Iterate over the region store files
+   *
+   * @param fs {@link FileSystem}
+   * @param regionDir {@link Path} to the region directory
+   * @param visitor callback object to get the store files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
+      final StoreFileVisitor visitor) throws IOException {
+    FileStatus[] families = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
+    if (families == null) {
+      LOG.info("No families under region directory:" + regionDir);
+      return;
+    }
+
+    PathFilter fileFilter = new FSUtils.FileFilter(fs);
+    for (FileStatus family: families) {
+      Path familyDir = family.getPath();
+      String familyName = familyDir.getName();
+
+      // get all the storeFiles in the family
+      FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter);
+      if (storeFiles == null) {
+        LOG.debug("No hfiles found for family: " + familyDir + ", skipping.");
+        continue;
+      }
+
+      for (FileStatus hfile: storeFiles) {
+        Path hfilePath = hfile.getPath();
+        visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName());
+      }
+    }
+  }
+
+  /**
+   * Iterate over each region in the table and inform about recovered.edits
+   *
+   * @param fs {@link FileSystem}
+   * @param tableDir {@link Path} to the table directory
+   * @param visitor callback object to get the recovered.edits files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir,
+      final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
+    FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
+    if (regions == null) {
+      LOG.info("No regions under directory:" + tableDir);
+      return;
+    }
+
+    for (FileStatus region: regions) {
+      visitRegionRecoveredEdits(fs, region.getPath(), visitor);
+    }
+  }
+
+  /**
+   * Iterate over recovered.edits of the specified region
+   *
+   * @param fs {@link FileSystem}
+   * @param regionDir {@link Path} to the Region directory
+   * @param visitor callback object to get the recovered.edits files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir,
+      final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
+    NavigableSet<Path> files = HLog.getSplitEditFilesSorted(fs, regionDir);
+    if (files == null || files.size() == 0) return;
+
+    for (Path source: files) {
+      // check to see if the file is zero length, in which case we can skip it
+      FileStatus stat = fs.getFileStatus(source);
+      if (stat.getLen() <= 0) continue;
+
+      visitor.recoveredEdits(regionDir.getName(), source.getName());
+    }
+  }
+
+  /**
+   * Iterate over hbase log files
+   *
+   * @param fs {@link FileSystem}
+   * @param rootDir {@link Path} to the HBase root folder
+   * @param visitor callback object to get the log files
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void visitLogFiles(final FileSystem fs, final Path rootDir,
+      final LogFileVisitor visitor) throws IOException {
+    Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+    FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir);
+    if (logServerDirs == null) {
+      LOG.info("No logs under directory:" + logsDir);
+      return;
+    }
+
+    for (FileStatus serverLogs: logServerDirs) {
+      String serverName = serverLogs.getPath().getName();
+
+      FileStatus[] hlogs = FSUtils.listStatus(fs, serverLogs.getPath());
+      if (hlogs == null) {
+        LOG.debug("No hfiles found for server: " + serverName + ", skipping.");
+        continue;
+      }
+
+      for (FileStatus hlogRef: hlogs) {
+        visitor.logFile(serverName, hlogRef.getPath().getName());
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
index 266fd1a00137..92a2408a4435 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
@@ -38,6 +38,21 @@ private HFileArchiveUtil() {
     // non-external instantiation - util class
   }
 
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param tableName table name under which the store currently lives
+   * @param regionName region encoded name under which the store currently lives
+   * @param family name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or
+   *         <tt>null</tt> if it should not be archived
+   */
+  public static Path getStoreArchivePath(final Configuration conf, final String tableName,
+      final String regionName, final String familyName) throws IOException {
+    Path tableArchiveDir = getTableArchivePath(conf, tableName);
+    return Store.getStoreHomedir(tableArchiveDir, regionName, familyName);
+  }
+
   /**
    * Get the directory to archive a store directory
    * @param conf {@link Configuration} to read for the archive directory name
@@ -84,6 +99,24 @@ public static Path getRegionArchiveDir(Configuration conf, Path tabledir, Path r
     return HRegion.getRegionDir(archiveDir, encodedRegionName);
   }
 
+  /**
+   * Get the archive directory for a given region under the specified table
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tabledir the original table directory. Cannot be null.
+   * @param regiondir the path to the region directory. Cannot be null.
+   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
+   *         should not be archived
+   */
+  public static Path getRegionArchiveDir(Path rootdir, Path tabledir, Path regiondir) {
+    // get the archive directory for a table
+    Path archiveDir = getTableArchivePath(rootdir, tabledir.getName());
+
+    // then add on the region path under the archive
+    String encodedRegionName = regiondir.getName();
+    return HRegion.getRegionDir(archiveDir, encodedRegionName);
+  }
+
   /**
    * Get the path to the table archive directory based on the configured archive directory.
    * <p>
@@ -95,7 +128,35 @@ public static Path getRegionArchiveDir(Configuration conf, Path tabledir, Path r
    */
   public static Path getTableArchivePath(Path tabledir) {
     Path root = tabledir.getParent();
-    return new Path(new Path(root,HConstants.HFILE_ARCHIVE_DIRECTORY), tabledir.getName());
+    return getTableArchivePath(root, tabledir.getName());
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Get the path to the table's archive directory.
+   * <p>
+   * Generally of the form: /hbase/.archive/[tablename]
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableName Name of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(final Path rootdir, final String tableName) {
+    return new Path(getArchivePath(rootdir), tableName);
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Assumed that the table should already be archived.
+   * @param conf {@link Configuration} to read the archive directory property. Can be null
+   * @param tableName Name of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(final Configuration conf, final String tableName)
+      throws IOException {
+    return new Path(getArchivePath(conf), tableName);
   }
 
   /**
@@ -106,6 +167,16 @@ public static Path getTableArchivePath(Path tabledir) {
    * @throws IOException if an unexpected error occurs
    */
   public static Path getArchivePath(Configuration conf) throws IOException {
-    return new Path(FSUtils.getRootDir(conf), HConstants.HFILE_ARCHIVE_DIRECTORY);
+    return getArchivePath(FSUtils.getRootDir(conf));
+  }
+
+  /**
+   * Get the full path to the archive directory on the configured {@link FileSystem}
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @return the full {@link Path} to the archive directory, as defined by the configuration
+   */
+  private static Path getArchivePath(final Path rootdir) {
+    return new Path(rootdir, HConstants.HFILE_ARCHIVE_DIRECTORY);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
new file mode 100644
index 000000000000..ba72227587b8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -0,0 +1,176 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+
+/**
+ * Utility methods for interacting with the regions.
+ */
+@InterfaceAudience.Private
+public abstract class ModifyRegionUtils {
+  private static final Log LOG = LogFactory.getLog(ModifyRegionUtils.class);
+
+  private ModifyRegionUtils() {
+  }
+
+  public interface RegionFillTask {
+    public void fillRegion(final HRegion region) throws IOException;
+  }
+
+  /**
+   * Create new set of regions on the specified file-system.
+   * NOTE: that you should add the regions to .META. after this operation.
+   *
+   * @param conf {@link Configuration}
+   * @param rootDir Root directory for HBase instance
+   * @param hTableDescriptor description of the table
+   * @param newRegions {@link HRegionInfo} that describes the regions to create
+   * @throws IOException
+   */
+  public static List<HRegionInfo> createRegions(final Configuration conf, final Path rootDir,
+      final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions) throws IOException {
+    return createRegions(conf, rootDir, hTableDescriptor, newRegions, null);
+  }
+
+  /**
+   * Create new set of regions on the specified file-system.
+   * NOTE: that you should add the regions to .META. after this operation.
+   *
+   * @param conf {@link Configuration}
+   * @param rootDir Root directory for HBase instance
+   * @param hTableDescriptor description of the table
+   * @param newRegions {@link HRegionInfo} that describes the regions to create
+   * @param task {@link RegionFillTask} custom code to populate region after creation
+   * @throws IOException
+   */
+  public static List<HRegionInfo> createRegions(final Configuration conf, final Path rootDir,
+      final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions,
+      final RegionFillTask task) throws IOException {
+    if (newRegions == null) return null;
+    int regionNumber = newRegions.length;
+    ThreadPoolExecutor regionOpenAndInitThreadPool = getRegionOpenAndInitThreadPool(conf,
+        "RegionOpenAndInitThread-" + hTableDescriptor.getNameAsString(), regionNumber);
+    CompletionService<HRegionInfo> completionService = new ExecutorCompletionService<HRegionInfo>(
+        regionOpenAndInitThreadPool);
+    List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
+    for (final HRegionInfo newRegion : newRegions) {
+      completionService.submit(new Callable<HRegionInfo>() {
+        public HRegionInfo call() throws IOException {
+          // 1. Create HRegion
+          HRegion region = HRegion.createHRegion(newRegion,
+              rootDir, conf, hTableDescriptor, null,
+              false, true);
+          try {
+            // 2. Custom user code to interact with the created region
+            if (task != null) {
+              task.fillRegion(region);
+            }
+          } finally {
+            // 3. Close the new region to flush to disk. Close log file too.
+            region.close();
+          }
+          return region.getRegionInfo();
+        }
+      });
+    }
+    try {
+      // 4. wait for all regions to finish creation
+      for (int i = 0; i < regionNumber; i++) {
+        Future<HRegionInfo> future = completionService.take();
+        HRegionInfo regionInfo = future.get();
+        regionInfos.add(regionInfo);
+      }
+    } catch (InterruptedException e) {
+      LOG.error("Caught " + e + " during region creation");
+      throw new InterruptedIOException(e.getMessage());
+    } catch (ExecutionException e) {
+      throw new IOException(e);
+    } finally {
+      regionOpenAndInitThreadPool.shutdownNow();
+    }
+    return regionInfos;
+  }
+
+  /*
+   * used by createRegions() to get the thread pool executor based on the
+   * "hbase.hregion.open.and.init.threads.max" property.
+   */
+  static ThreadPoolExecutor getRegionOpenAndInitThreadPool(final Configuration conf,
+      final String threadNamePrefix, int regionNumber) {
+    int maxThreads = Math.min(regionNumber, conf.getInt(
+        "hbase.hregion.open.and.init.threads.max", 10));
+    ThreadPoolExecutor regionOpenAndInitThreadPool = Threads
+    .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+        new ThreadFactory() {
+          private int count = 1;
+
+          public Thread newThread(Runnable r) {
+            Thread t = new Thread(r, threadNamePrefix + "-" + count++);
+            return t;
+          }
+        });
+    return regionOpenAndInitThreadPool;
+  }
+
+  /**
+   * Trigger immediate assignment of the regions in round-robin fashion
+   *
+   * @param assignmentManager
+   * @param regions
+   */
+  public static void assignRegions(final AssignmentManager assignmentManager,
+      final List<HRegionInfo> regions) throws IOException {
+    try {
+      assignmentManager.assignUserRegionsToOnlineServers(regions);
+    } catch (InterruptedException ie) {
+      LOG.error("Caught " + ie + " during round-robin assignment");
+      throw new InterruptedIOException(ie.getMessage());
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index d3206c6b3218..2bc3e66188bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -61,9 +61,9 @@
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
 import org.apache.zookeeper.server.ZooKeeperSaslServer;
 import org.apache.zookeeper.proto.CreateRequest;
 import org.apache.zookeeper.proto.DeleteRequest;
@@ -855,6 +855,10 @@ public static boolean setData(ZooKeeperWatcher zkw, String znode,
   /**
    * Set data into node creating node if it doesn't yet exist.
    * Does not set watch.
+   *
+   * WARNING: this is not atomic -- it is possible to get a 0-byte data value in the znode before
+   * data is written
+   *
    * @param zkw zk reference
    * @param znode path of node
    * @param data data to set for node
@@ -1067,7 +1071,7 @@ public static void asyncCreate(ZooKeeperWatcher zkw,
   }
 
   /**
-   * Creates the specified node, if the node does not exist.  Does not set a
+   * Creates the specified node, iff the node does not exist.  Does not set a
    * watch and fails silently if the node already exists.
    *
    * The node created is persistent and open access.
@@ -1078,8 +1082,24 @@ public static void asyncCreate(ZooKeeperWatcher zkw,
    */
   public static void createAndFailSilent(ZooKeeperWatcher zkw,
       String znode) throws KeeperException {
+    createAndFailSilent(zkw, znode, new byte[0]);
+  }
+
+  /**
+   * Creates the specified node containing specified data, iff the node does not exist.  Does
+   * not set a watch and fails silently if the node already exists.
+   *
+   * The node created is persistent and open access.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param data a byte array data to store in the znode
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static void createAndFailSilent(ZooKeeperWatcher zkw,
+      String znode, byte[] data) throws KeeperException {
     createAndFailSilent(zkw,
-      (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, new byte[0]));
+      (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
   }
 
   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
@@ -1119,12 +1139,30 @@ private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilen
    * @throws KeeperException if unexpected zookeeper exception
    */
   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    createWithParents(zkw, znode, new byte[0]);
+  }
+
+  /**
+   * Creates the specified node and all parent nodes required for it to exist.  The creation of
+   * parent znodes is not atomic with the leafe znode creation but the data is written atomically
+   * when the leaf node is created.
+   *
+   * No watches are set and no errors are thrown if the node already exists.
+   *
+   * The nodes created are persistent and open access.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
   throws KeeperException {
     try {
       if(znode == null) {
         return;
       }
-      zkw.getRecoverableZooKeeper().create(znode, new byte[0], createACL(zkw, znode),
+      zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
           CreateMode.PERSISTENT);
     } catch(KeeperException.NodeExistsException nee) {
       return;
@@ -1606,4 +1644,37 @@ public static void waitForBaseZNode(Configuration conf) throws IOException {
       throw new IOException(keeperEx);
     }
   }
+
+  /**
+   * Recursively print the current state of ZK (non-transactional)
+   * @param root name of the root directory in zk to print
+   * @throws KeeperException
+   */
+  public static void logZKTree(ZooKeeperWatcher zkw, String root) {
+    if (!LOG.isDebugEnabled()) return;
+    LOG.debug("Current zk system:");
+    String prefix = "|-";
+    LOG.debug(prefix + root);
+    try {
+      logZKTree(zkw, root, prefix);
+    } catch (KeeperException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Helper method to print the current state of the ZK tree.
+   * @see #logZKTree(ZooKeeperWatcher, String)
+   * @throws KeeperException if an unexpected exception occurs
+   */
+  protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
+    List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
+    if (children == null) return;
+    for (String child : children) {
+      LOG.debug(prefix + child);
+      String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
+      logZKTree(zkw, node, prefix + "---");
+    }
+  }
+
 }
diff --git a/src/main/protobuf/ErrorHandling.proto b/src/main/protobuf/ErrorHandling.proto
new file mode 100644
index 000000000000..53c4165a90f9
--- /dev/null
+++ b/src/main/protobuf/ErrorHandling.proto
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for error handling
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ErrorHandlingProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * Protobuf version of a java.lang.StackTraceElement
+ * so we can serialize exceptions.
+ */
+message StackTraceElementMessage {
+  optional string declaringClass = 1;
+  optional string methodName = 2;
+  optional string fileName = 3;
+  optional int32 lineNumber = 4;
+}
+
+/**
+ * Cause of a remote failure for a generic exception. Contains
+ * all the information for a generic exception as well as
+ * optional info about the error for generic info passing
+ * (which should be another protobuffed class).
+ */
+message GenericExceptionMessage {
+  optional string className = 1;
+  optional string message = 2;
+  optional bytes errorInfo = 3;
+  repeated StackTraceElementMessage trace = 4;
+}
+
+/**
+ * Exception sent across the wire when a remote task needs
+ * to notify other tasks that it failed and why
+ */
+message ForeignExceptionMessage {
+  optional string source = 1;
+  optional GenericExceptionMessage genericException = 2;
+
+}
diff --git a/src/main/protobuf/hbase.proto b/src/main/protobuf/hbase.proto
new file mode 100644
index 000000000000..93d4232c699e
--- /dev/null
+++ b/src/main/protobuf/hbase.proto
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are shared throughout HBase
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "HBaseProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * Description of the snapshot to take
+ */
+message SnapshotDescription {
+  required string name = 1;
+  optional string table = 2; // not needed for delete, but checked for in taking snapshot
+  optional int64 creationTime = 3 [default = 0];
+  enum Type {
+    DISABLED = 0;
+    FLUSH = 1;
+  }
+  optional Type type = 4 [default = FLUSH];
+  optional int32 version = 5;
+}
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 85367bfec6c7..73e8552bc5dd 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -586,5 +586,35 @@ def online(region_name, on_off)
       put.add(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, org.apache.hadoop.hbase.util.Writables.getBytes(hri))
       meta.put(put)
     end
+
+    #----------------------------------------------------------------------------------------------
+    # Take a snapshot of specified table
+    def snapshot(table, snapshot_name)
+      @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+    end
+
+    #----------------------------------------------------------------------------------------------
+    # Restore specified snapshot
+    def restore_snapshot(snapshot_name)
+      @admin.restoreSnapshot(snapshot_name.to_java_bytes)
+    end
+
+    #----------------------------------------------------------------------------------------------
+    # Create a new table by cloning the snapshot content
+    def clone_snapshot(snapshot_name, table)
+      @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+    end
+
+    #----------------------------------------------------------------------------------------------
+    # Delete specified snapshot
+    def delete_snapshot(snapshot_name)
+      @admin.deleteSnapshot(snapshot_name.to_java_bytes)
+    end
+
+    #----------------------------------------------------------------------------------------------
+    # Returns a list of snapshots
+    def list_snapshot
+      @admin.listSnapshots
+    end
   end
 end
diff --git a/src/main/ruby/shell.rb b/src/main/ruby/shell.rb
index f0d96e6fcc4e..d1ec550d1797 100644
--- a/src/main/ruby/shell.rb
+++ b/src/main/ruby/shell.rb
@@ -289,6 +289,18 @@ def help_footer
   ]
 )
 
+Shell.load_command_group(
+  'snapshot',
+  :full_name => 'CLUSTER SNAPSHOT TOOLS',
+  :commands => %w[
+    snapshot
+    clone_snapshot
+    restore_snapshot
+    delete_snapshot
+    list_snapshots
+  ]
+)
+
 Shell.load_command_group(
   'security',
   :full_name => 'SECURITY TOOLS',
diff --git a/src/main/ruby/shell/commands/clone_snapshot.rb b/src/main/ruby/shell/commands/clone_snapshot.rb
new file mode 100644
index 000000000000..43d240f12f34
--- /dev/null
+++ b/src/main/ruby/shell/commands/clone_snapshot.rb
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class CloneSnapshot < Command
+      def help
+        return <<-EOF
+Create a new table by cloning the snapshot content.
+There're no copies of data involved.
+And writing on the newly created table will not influence the snapshot data.
+
+Examples:
+  hbase> clone_snapshot 'snapshotName', 'tableName'
+EOF
+      end
+
+      def command(snapshot_name, table)
+        format_simple_command do
+          admin.clone_snapshot(snapshot_name, table)
+        end
+      end
+    end
+  end
+end
diff --git a/src/main/ruby/shell/commands/delete_snapshot.rb b/src/main/ruby/shell/commands/delete_snapshot.rb
new file mode 100644
index 000000000000..b8c3791a5401
--- /dev/null
+++ b/src/main/ruby/shell/commands/delete_snapshot.rb
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class DeleteSnapshot < Command
+      def help
+        return <<-EOF
+Delete a specified snapshot. Examples:
+
+  hbase> delete_snapshot 'snapshotName',
+EOF
+      end
+
+      def command(snapshot_name)
+        format_simple_command do
+          admin.delete_snapshot(snapshot_name)
+        end
+      end
+    end
+  end
+end
diff --git a/src/main/ruby/shell/commands/list_snapshots.rb b/src/main/ruby/shell/commands/list_snapshots.rb
new file mode 100644
index 000000000000..95135185c2ee
--- /dev/null
+++ b/src/main/ruby/shell/commands/list_snapshots.rb
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'time'
+
+module Shell
+  module Commands
+    class ListSnapshots < Command
+      def help
+        return <<-EOF
+List all snapshots taken (by printing the names and relative information).
+Optional regular expression parameter could be used to filter the output
+by snapshot name.
+
+Examples:
+  hbase> list_snapshots
+  hbase> list_snapshots 'abc.*'
+EOF
+      end
+
+      def command(regex = ".*")
+        now = Time.now
+        formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"])
+
+        regex = /#{regex}/ unless regex.is_a?(Regexp)
+        list = admin.list_snapshot.select {|s| regex.match(s.getName)}
+        list.each do |snapshot|
+          creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s
+          formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ])
+        end
+
+        formatter.footer(now, list.size)
+        return list.map { |s| s.getName() }
+      end
+    end
+  end
+end
diff --git a/src/main/ruby/shell/commands/restore_snapshot.rb b/src/main/ruby/shell/commands/restore_snapshot.rb
new file mode 100644
index 000000000000..4d531711bca6
--- /dev/null
+++ b/src/main/ruby/shell/commands/restore_snapshot.rb
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class RestoreSnapshot < Command
+      def help
+        return <<-EOF
+Restore a specified snapshot.
+The restore will replace the content of the original table,
+bringing back the content to the snapshot state.
+The table must be disabled.
+
+Examples:
+  hbase> restore_snapshot 'snapshotName'
+EOF
+      end
+
+      def command(snapshot_name)
+        format_simple_command do
+          admin.restore_snapshot(snapshot_name)
+        end
+      end
+    end
+  end
+end
diff --git a/src/main/ruby/shell/commands/snapshot.rb b/src/main/ruby/shell/commands/snapshot.rb
new file mode 100644
index 000000000000..1c4ecfec0e03
--- /dev/null
+++ b/src/main/ruby/shell/commands/snapshot.rb
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class Snapshot < Command
+      def help
+        return <<-EOF
+Take a snapshot of specified table. Examples:
+
+  hbase> snapshot 'sourceTable', 'snapshotName'
+EOF
+      end
+
+      def command(table, snapshot_name)
+        format_simple_command do
+          admin.snapshot(table, snapshot_name)
+        end
+      end
+    end
+  end
+end
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index dc0883bc86b1..03161a010278 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1089,6 +1089,20 @@ public int countRows(final HTable table) throws IOException {
     return count;
   }
 
+  public int countRows(final HTable table, final byte[]... families) throws IOException {
+    Scan scan = new Scan();
+    for (byte[] family: families) {
+      scan.addFamily(family);
+    }
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (@SuppressWarnings("unused") Result res : results) {
+      count++;
+    }
+    results.close();
+    return count;
+  }
+
   /**
    * Return an md5 digest of the entire contents of a table.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
index 186d5d29faad..586b7af50c6d 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
@@ -20,9 +20,15 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+ 
+import java.util.regex.Pattern;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -31,6 +37,7 @@
  */
 @Category(SmallTests.class)
 public class TestHTableDescriptor {
+  final static Log LOG = LogFactory.getLog(TestHTableDescriptor.class);
 
   /**
    * Test cps in the table description
@@ -122,4 +129,42 @@ public void testGetMemStoreFlushSize() {
     desc.setMemStoreFlushSize(1111L);
     assertEquals(1111L, desc.getMemStoreFlushSize());
   }
+
+  String legalTableNames[] = { "foo", "with-dash_under.dot", "_under_start_ok",  };
+  String illegalTableNames[] = { ".dot_start_illegal", "-dash_start_illegal", "spaces not ok" };
+
+  @Test
+  public void testLegalHTableNames() {
+    for (String tn : legalTableNames) {
+      HTableDescriptor.isLegalTableName(Bytes.toBytes(tn));
+    }
+  }
+
+  @Test
+  public void testIllegalHTableNames() {
+    for (String tn : illegalTableNames) {
+      try {
+        HTableDescriptor.isLegalTableName(Bytes.toBytes(tn));
+        fail("invalid tablename " + tn + " should have failed");
+      } catch (Exception e) {
+        // expected
+      }
+    }
+  }
+
+  @Test
+  public void testLegalHTableNamesRegex() {
+    for (String tn : legalTableNames) {
+      LOG.info("Testing: '" + tn + "'");
+      assertTrue(Pattern.matches(HTableDescriptor.VALID_USER_TABLE_REGEX, tn));
+    }
+  }
+
+  @Test
+  public void testIllegalHTableNamesRegex() {
+    for (String tn : illegalTableNames) {
+      LOG.info("Testing: '" + tn + "'");
+      assertFalse(Pattern.matches(HTableDescriptor.VALID_USER_TABLE_REGEX, tn));
+    }
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index 2d177f8c10c1..c2b3d76f4de8 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -360,7 +360,7 @@ public void testCleaningRace() throws Exception {
 
         try {
           // Try to archive the file
-          HFileArchiver.archiveRegion(conf, fs, rootDir,
+          HFileArchiver.archiveRegion(fs, rootDir,
               sourceRegionDir.getParent(), sourceRegionDir);
 
           // The archiver succeded, the file is no longer in the original location
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
new file mode 100644
index 000000000000..dbd0a711864d
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -0,0 +1,391 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test clone/restore snapshots from the client
+ */
+@Category(LargeTests.class)
+public class TestRestoreSnapshotFromClient {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final byte[] FAMILY = Bytes.toBytes("cf");
+
+  private byte[] emptySnapshot;
+  private byte[] snapshotName0;
+  private byte[] snapshotName1;
+  private byte[] snapshotName2;
+  private int snapshot0Rows;
+  private int snapshot1Rows;
+  private byte[] tableName;
+  private HBaseAdmin admin;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
+    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
+    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+    TEST_UTIL.getConfiguration().setBoolean(
+        "hbase.master.enabletable.roundrobin", true);
+    TEST_UTIL.startMiniCluster(3);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Initialize the tests with a table filled with some data
+   * and two snapshots (snapshotName0, snapshotName1) of different states.
+   * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
+   */
+  @Before
+  public void setup() throws Exception {
+    this.admin = TEST_UTIL.getHBaseAdmin();
+
+    long tid = System.currentTimeMillis();
+    tableName = Bytes.toBytes("testtb-" + tid);
+    emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
+    snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
+    snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
+    snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
+
+    // create Table and disable it
+    createTable(tableName, FAMILY);
+    admin.disableTable(tableName);
+
+    // take an empty snapshot
+    admin.snapshot(emptySnapshot, tableName);
+
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    try {
+      // enable table and insert data
+      admin.enableTable(tableName);
+      loadData(table, 500, FAMILY);
+      snapshot0Rows = TEST_UTIL.countRows(table);
+      admin.disableTable(tableName);
+
+      // take a snapshot
+      admin.snapshot(snapshotName0, tableName);
+
+      // enable table and insert more data
+      admin.enableTable(tableName);
+      loadData(table, 500, FAMILY);
+      snapshot1Rows = TEST_UTIL.countRows(table);
+      admin.disableTable(tableName);
+
+      // take a snapshot of the updated table
+      admin.snapshot(snapshotName1, tableName);
+
+      // re-enable table
+      admin.enableTable(tableName);
+    } finally {
+      table.close();
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (admin.tableExists(tableName)) {
+      TEST_UTIL.deleteTable(tableName);
+    }
+    admin.deleteSnapshot(snapshotName0);
+    admin.deleteSnapshot(snapshotName1);
+
+    // Ensure the archiver to be empty
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    mfs.getFileSystem().delete(
+      new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+  }
+
+  @Test
+  public void testRestoreSnapshot() throws IOException {
+    verifyRowCount(tableName, snapshot1Rows);
+
+    // Restore from snapshot-0
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(snapshotName0);
+    admin.enableTable(tableName);
+    verifyRowCount(tableName, snapshot0Rows);
+
+    // Restore from emptySnapshot
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(emptySnapshot);
+    admin.enableTable(tableName);
+    verifyRowCount(tableName, 0);
+
+    // Restore from snapshot-1
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(snapshotName1);
+    admin.enableTable(tableName);
+    verifyRowCount(tableName, snapshot1Rows);
+  }
+
+  @Test
+  public void testRestoreSchemaChange() throws IOException {
+    byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
+
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+
+    // Add one column family and put some data in it
+    admin.disableTable(tableName);
+    admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
+    admin.enableTable(tableName);
+    assertEquals(2, table.getTableDescriptor().getFamilies().size());
+    HTableDescriptor htd = admin.getTableDescriptor(tableName);
+    assertEquals(2, htd.getFamilies().size());
+    loadData(table, 500, TEST_FAMILY2);
+    long snapshot2Rows = snapshot1Rows + 500;
+    assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
+    assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
+    Set<String> fsFamilies = getFamiliesFromFS(tableName);
+    assertEquals(2, fsFamilies.size());
+    table.close();
+
+    // Take a snapshot
+    admin.disableTable(tableName);
+    admin.snapshot(snapshotName2, tableName);
+
+    // Restore the snapshot (without the cf)
+    admin.restoreSnapshot(snapshotName0);
+    assertEquals(1, table.getTableDescriptor().getFamilies().size());
+    admin.enableTable(tableName);
+    try {
+      TEST_UTIL.countRows(table, TEST_FAMILY2);
+      fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
+    } catch (NoSuchColumnFamilyException e) {
+      // expected
+    }
+    assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+    htd = admin.getTableDescriptor(tableName);
+    assertEquals(1, htd.getFamilies().size());
+    fsFamilies = getFamiliesFromFS(tableName);
+    assertEquals(1, fsFamilies.size());
+    table.close();
+
+    // Restore back the snapshot (with the cf)
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(snapshotName2);
+    admin.enableTable(tableName);
+    htd = admin.getTableDescriptor(tableName);
+    assertEquals(2, htd.getFamilies().size());
+    assertEquals(2, table.getTableDescriptor().getFamilies().size());
+    assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
+    assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
+    fsFamilies = getFamiliesFromFS(tableName);
+    assertEquals(2, fsFamilies.size());
+    table.close();
+  }
+
+  @Test(expected=SnapshotDoesNotExistException.class)
+  public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
+    String snapshotName = "random-snapshot-" + System.currentTimeMillis();
+    String tableName = "random-table-" + System.currentTimeMillis();
+    admin.cloneSnapshot(snapshotName, tableName);
+  }
+
+  @Test
+  public void testCloneSnapshot() throws IOException, InterruptedException {
+    byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
+    testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
+    testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
+    testCloneSnapshot(clonedTableName, emptySnapshot, 0);
+  }
+
+  private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
+      int snapshotRows) throws IOException, InterruptedException {
+    // create a new table from snapshot
+    admin.cloneSnapshot(snapshotName, tableName);
+    verifyRowCount(tableName, snapshotRows);
+
+    admin.disableTable(tableName);
+    admin.deleteTable(tableName);
+  }
+
+  @Test
+  public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
+    byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName0, clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+    admin.disableTable(clonedTableName);
+    admin.snapshot(snapshotName2, clonedTableName);
+    admin.deleteTable(clonedTableName);
+    waitCleanerRun();
+
+    admin.cloneSnapshot(snapshotName2, clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+    admin.disableTable(clonedTableName);
+    admin.deleteTable(clonedTableName);
+  }
+
+  /**
+   * Verify that tables created from the snapshot are still alive after source table deletion.
+   */
+  @Test
+  public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
+    // Clone a table from the first snapshot
+    byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName0, clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+
+    // Take a snapshot of this cloned table.
+    admin.disableTable(clonedTableName);
+    admin.snapshot(snapshotName2, clonedTableName);
+
+    // Clone the snapshot of the cloned table
+    byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName2, clonedTableName2);
+    verifyRowCount(clonedTableName2, snapshot0Rows);
+    admin.disableTable(clonedTableName2);
+
+    // Remove the original table
+    admin.disableTable(tableName);
+    admin.deleteTable(tableName);
+    waitCleanerRun();
+
+    // Verify the first cloned table
+    admin.enableTable(clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+
+    // Verify the second cloned table
+    admin.enableTable(clonedTableName2);
+    verifyRowCount(clonedTableName2, snapshot0Rows);
+    admin.disableTable(clonedTableName2);
+
+    // Delete the first cloned table
+    admin.disableTable(clonedTableName);
+    admin.deleteTable(clonedTableName);
+    waitCleanerRun();
+
+    // Verify the second cloned table
+    admin.enableTable(clonedTableName2);
+    verifyRowCount(clonedTableName2, snapshot0Rows);
+
+    // Clone a new table from cloned
+    byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName2, clonedTableName3);
+    verifyRowCount(clonedTableName3, snapshot0Rows);
+
+    // Delete the cloned tables
+    admin.disableTable(clonedTableName2);
+    admin.deleteTable(clonedTableName2);
+    admin.disableTable(clonedTableName3);
+    admin.deleteTable(clonedTableName3);
+    admin.deleteSnapshot(snapshotName2);
+  }
+
+  // ==========================================================================
+  //  Helpers
+  // ==========================================================================
+  private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (byte[] family: families) {
+      HColumnDescriptor hcd = new HColumnDescriptor(family);
+      htd.addFamily(hcd);
+    }
+    byte[][] splitKeys = new byte[16][];
+    byte[] hex = Bytes.toBytes("0123456789abcdef");
+    for (int i = 0; i < 16; ++i) {
+      splitKeys[i] = new byte[] { hex[i] };
+    }
+    admin.createTable(htd, splitKeys);
+  }
+
+  public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
+    byte[] qualifier = Bytes.toBytes("q");
+    table.setAutoFlush(false);
+    while (rows-- > 0) {
+      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
+      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
+      Put put = new Put(key);
+      put.setWriteToWAL(false);
+      for (byte[] family: families) {
+        put.add(family, qualifier, value);
+      }
+      table.put(put);
+    }
+    table.flushCommits();
+  }
+
+  private void waitCleanerRun() throws InterruptedException {
+    TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
+  }
+
+  private Set<String> getFamiliesFromFS(final byte[] tableName) throws IOException {
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    Set<String> families = new HashSet<String>();
+    Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName);
+    for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
+      for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
+        families.add(familyDir.getName());
+      }
+    }
+    return families;
+  }
+
+  private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    assertEquals(expectedRows, TEST_UTIL.countRows(table));
+    table.close();
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
new file mode 100644
index 000000000000..ced199e66688
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import com.google.protobuf.RpcController;
+
+/**
+ * Test snapshot logic from the client
+ */
+@Category(SmallTests.class)
+public class TestSnapshotFromAdmin {
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotFromAdmin.class);
+
+  /**
+   * Test that the logic for doing 'correct' back-off based on exponential increase and the max-time
+   * passed from the server ensures the correct overall waiting for the snapshot to finish.
+   * @throws Exception
+   */
+  @Test(timeout = 10000)
+  public void testBackoffLogic() throws Exception {
+    final int maxWaitTime = 7500;
+    final int numRetries = 10;
+    final int pauseTime = 500;
+    // calculate the wait time, if we just do straight backoff (ignoring the expected time from
+    // master)
+    long ignoreExpectedTime = 0;
+    for (int i = 0; i < 6; i++) {
+      ignoreExpectedTime += HConstants.RETRY_BACKOFF[i] * pauseTime;
+    }
+    // the correct wait time, capping at the maxTime/tries + fudge room
+    final long time = pauseTime * 3 + ((maxWaitTime / numRetries) * 3) + 300;
+    assertTrue("Capped snapshot wait time isn't less that the uncapped backoff time "
+        + "- further testing won't prove anything.", time < ignoreExpectedTime);
+
+    // setup the mocks
+    HConnectionManager.HConnectionImplementation mockConnection = Mockito
+        .mock(HConnectionManager.HConnectionImplementation.class);
+    Configuration conf = HBaseConfiguration.create();
+    // setup the conf to match the expected properties
+    conf.setInt("hbase.client.retries.number", numRetries);
+    conf.setLong("hbase.client.pause", pauseTime);
+    // mock the master admin to our mock
+    HMasterInterface mockMaster = Mockito.mock(HMasterInterface.class);
+    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
+    Mockito.when(mockConnection.getMaster()).thenReturn(mockMaster);
+    // set the max wait time for the snapshot to complete
+    Mockito
+        .when(
+          mockMaster.snapshot(
+            Mockito.any(HSnapshotDescription.class))).thenReturn((long)maxWaitTime);
+
+    // first five times, we return false, last we get success
+    Mockito.when(
+      mockMaster.isSnapshotDone(
+        Mockito.any(HSnapshotDescription.class))).thenReturn(false, false,
+      false, false, false, true);
+
+    // setup the admin and run the test
+    HBaseAdmin admin = new HBaseAdmin(mockConnection);
+    String snapshot = "snapshot";
+    String table = "table";
+    // get start time
+    long start = System.currentTimeMillis();
+    admin.snapshot(snapshot, table);
+    long finish = System.currentTimeMillis();
+    long elapsed = (finish - start);
+    assertTrue("Elapsed time:" + elapsed + " is more than expected max:" + time, elapsed <= time);
+    admin.close();
+  }
+
+  /**
+   * Make sure that we validate the snapshot name and the table name before we pass anything across
+   * the wire
+   * @throws Exception on failure
+   */
+  @Test
+  public void testValidateSnapshotName() throws Exception {
+    HConnectionManager.HConnectionImplementation mockConnection = Mockito
+        .mock(HConnectionManager.HConnectionImplementation.class);
+    Configuration conf = HBaseConfiguration.create();
+    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
+    HBaseAdmin admin = new HBaseAdmin(mockConnection);
+    SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+    // check that invalid snapshot names fail
+    failSnapshotStart(admin, builder.setName(".snapshot").build());
+    failSnapshotStart(admin, builder.setName("-snapshot").build());
+    failSnapshotStart(admin, builder.setName("snapshot fails").build());
+    failSnapshotStart(admin, builder.setName("snap$hot").build());
+    // check the table name also get verified
+    failSnapshotStart(admin, builder.setName("snapshot").setTable(".table").build());
+    failSnapshotStart(admin, builder.setName("snapshot").setTable("-table").build());
+    failSnapshotStart(admin, builder.setName("snapshot").setTable("table fails").build());
+    failSnapshotStart(admin, builder.setName("snapshot").setTable("tab%le").build());
+
+    // mock the master connection
+    HMasterInterface master = Mockito.mock(HMasterInterface.class);
+    Mockito.when(mockConnection.getMaster()).thenReturn(master);
+
+    Mockito.when(
+      master.snapshot(Mockito.any(HSnapshotDescription.class))).thenReturn((long)0);
+    Mockito.when(
+      master.isSnapshotDone(
+        Mockito.any(HSnapshotDescription.class))).thenReturn(true);
+
+      // make sure that we can use valid names
+    admin.snapshot(builder.setName("snapshot").setTable("table").build());
+  }
+
+  private void failSnapshotStart(HBaseAdmin admin, SnapshotDescription snapshot) throws IOException {
+    try {
+      admin.snapshot(snapshot);
+      fail("Snapshot should not have succeed with name:" + snapshot.getName());
+    } catch (IllegalArgumentException e) {
+      LOG.debug("Correctly failed to start snapshot:" + e.getMessage());
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
new file mode 100644
index 000000000000..e1b5ff8e322b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test create/using/deleting snapshots from the client
+ * <p>
+ * This is an end-to-end test for the snapshot utility
+ */
+@Category(LargeTests.class)
+public class TestSnapshotFromClient {
+  private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final int NUM_RS = 2;
+  private static final String STRING_TABLE_NAME = "test";
+  private static final byte[] TEST_FAM = Bytes.toBytes("fam");
+  private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
+
+  /**
+   * Setup the config for the cluster
+   * @throws Exception on failure
+   */
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster(NUM_RS);
+  }
+
+  private static void setupConf(Configuration conf) {
+    // disable the ui
+    conf.setInt("hbase.regionsever.info.port", -1);
+    // change the flush size to a small amount, regulating number of store files
+    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+    // so make sure we get a compaction when doing a load, but keep around some
+    // files in the store
+    conf.setInt("hbase.hstore.compaction.min", 10);
+    conf.setInt("hbase.hstore.compactionThreshold", 10);
+    // block writes if we get to 12 store files
+    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
+    // drop the number of attempts for the hbase admin
+    conf.setInt("hbase.client.retries.number", 1);
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    // prevent aggressive region split
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      ConstantSizeRegionSplitPolicy.class.getName());
+  }
+
+  @Before
+  public void setup() throws Exception {
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    UTIL.deleteTable(TABLE_NAME);
+    // and cleanup the archive directory
+    try {
+      UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
+    } catch (IOException e) {
+      LOG.warn("Failure to delete archive directory", e);
+    }
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    try {
+      UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      LOG.warn("failure shutting down cluster", e);
+    }
+  }
+
+  /**
+   * Test snapshotting not allowed .META. and -ROOT-
+   * @throws Exception
+   */
+  @Test
+  public void testMetaTablesSnapshot() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    byte[] snapshotName = Bytes.toBytes("metaSnapshot");
+
+    try {
+      admin.snapshot(snapshotName, HConstants.META_TABLE_NAME);
+      fail("taking a snapshot of .META. should not be allowed");
+    } catch (IllegalArgumentException e) {
+      // expected
+    }
+
+    try {
+      admin.snapshot(snapshotName, HConstants.ROOT_TABLE_NAME);
+      fail("taking a snapshot of -ROOT- should not be allowed");
+    } catch (IllegalArgumentException e) {
+      // expected
+    }
+  }
+
+  /**
+   * Test snapshotting a table that is offline
+   * @throws Exception
+   */
+  @Test
+  public void testOfflineTableSnapshot() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // put some stuff in the table
+    HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
+    UTIL.loadTable(table, TEST_FAM);
+
+    // get the name of all the regionservers hosting the snapshotted table
+    Set<String> snapshotServers = new HashSet<String>();
+    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
+    for (RegionServerThread server : servers) {
+      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
+        snapshotServers.add(server.getRegionServer().getServerName().toString());
+      }
+    }
+
+    LOG.debug("FS state before disable:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+    // XXX if this is flakey, might want to consider using the async version and looping as
+    // disableTable can succeed and still timeout.
+    admin.disableTable(TABLE_NAME);
+
+    LOG.debug("FS state before snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    // take a snapshot of the disabled table
+    byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
+    admin.snapshot(snapshot, TABLE_NAME);
+    LOG.debug("Snapshot completed.");
+
+    // make sure we have the snapshot
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+      snapshot, TABLE_NAME);
+
+    // make sure its a valid snapshot
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    LOG.debug("FS state after snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+      admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+
+    admin.deleteSnapshot(snapshot);
+    snapshots = admin.listSnapshots();
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+  }
+
+  @Test
+  public void testSnapshotFailsOnNonExistantTable() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    String tableName = "_not_a_table";
+
+    // make sure the table doesn't exist
+    boolean fail = false;
+    do {
+    try {
+      admin.getTableDescriptor(Bytes.toBytes(tableName));
+      fail = true;
+          LOG.error("Table:" + tableName + " already exists, checking a new name");
+      tableName = tableName+"!";
+    } catch (TableNotFoundException e) {
+      fail = false;
+      }
+    } while (fail);
+
+    // snapshot the non-existant table
+    try {
+      admin.snapshot("fail", tableName);
+      fail("Snapshot succeeded even though there is not table.");
+    } catch (SnapshotCreationException e) {
+      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
new file mode 100644
index 000000000000..12c575398b99
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import com.google.protobuf.RpcController;
+
+/**
+ * Test snapshot logic from the client
+ */
+@Category(SmallTests.class)
+public class TestSnapshotsFromAdmin {
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotsFromAdmin.class);
+
+  /**
+   * Test that the logic for doing 'correct' back-off based on exponential increase and the max-time
+   * passed from the server ensures the correct overall waiting for the snapshot to finish.
+   * @throws Exception
+   */
+  @Test(timeout = 10000)
+  public void testBackoffLogic() throws Exception {
+    final int maxWaitTime = 7500;
+    final int numRetries = 10;
+    final int pauseTime = 500;
+    // calculate the wait time, if we just do straight backoff (ignoring the expected time from
+    // master)
+    long ignoreExpectedTime = 0;
+    for (int i = 0; i < 6; i++) {
+      ignoreExpectedTime += HConstants.RETRY_BACKOFF[i] * pauseTime;
+    }
+    // the correct wait time, capping at the maxTime/tries + fudge room
+    final long time = pauseTime * 3 + ((maxWaitTime / numRetries) * 3) + 300;
+    assertTrue("Capped snapshot wait time isn't less that the uncapped backoff time "
+        + "- further testing won't prove anything.", time < ignoreExpectedTime);
+
+    // setup the mocks
+    HConnectionManager.HConnectionImplementation mockConnection = Mockito
+        .mock(HConnectionManager.HConnectionImplementation.class);
+    Configuration conf = HBaseConfiguration.create();
+    // setup the conf to match the expected properties
+    conf.setInt("hbase.client.retries.number", numRetries);
+    conf.setLong("hbase.client.pause", pauseTime);
+    // mock the master admin to our mock
+    HMasterInterface mockMaster = Mockito.mock(HMasterInterface.class);
+    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
+    Mockito.when(mockConnection.getMaster()).thenReturn(mockMaster);
+    // set the max wait time for the snapshot to complete
+    Mockito
+        .when(
+          mockMaster.snapshot(
+            Mockito.any(HSnapshotDescription.class))).thenReturn((long)maxWaitTime);
+    // first five times, we return false, last we get success
+    Mockito.when(
+      mockMaster.isSnapshotDone(
+        Mockito.any(HSnapshotDescription.class))).thenReturn(false, false,
+          false, false, false, true);
+
+    // setup the admin and run the test
+    HBaseAdmin admin = new HBaseAdmin(mockConnection);
+    String snapshot = "snasphot";
+    String table = "table";
+    // get start time
+    long start = System.currentTimeMillis();
+    admin.snapshot(snapshot, table);
+    long finish = System.currentTimeMillis();
+    long elapsed = (finish - start);
+    assertTrue("Elapsed time:" + elapsed + " is more than expected max:" + time, elapsed <= time);
+  }
+
+  /**
+   * Make sure that we validate the snapshot name and the table name before we pass anything across
+   * the wire
+   * @throws IOException on failure
+   */
+  @Test
+  public void testValidateSnapshotName() throws IOException {
+    HConnectionManager.HConnectionImplementation mockConnection = Mockito
+        .mock(HConnectionManager.HConnectionImplementation.class);
+    Configuration conf = HBaseConfiguration.create();
+    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
+    HBaseAdmin admin = new HBaseAdmin(mockConnection);
+    SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+    // check that invalid snapshot names fail
+    failSnapshotStart(admin, builder.setName(".snapshot").build());
+    failSnapshotStart(admin, builder.setName("-snapshot").build());
+    failSnapshotStart(admin, builder.setName("snapshot fails").build());
+    failSnapshotStart(admin, builder.setName("snap$hot").build());
+    // check the table name also get verified
+    failSnapshotStart(admin, builder.setName("snapshot").setTable(".table").build());
+    failSnapshotStart(admin, builder.setName("snapshot").setTable("-table").build());
+    failSnapshotStart(admin, builder.setName("snapshot").setTable("table fails").build());
+    failSnapshotStart(admin, builder.setName("snapshot").setTable("tab%le").build());
+  }
+
+  private void failSnapshotStart(HBaseAdmin admin, SnapshotDescription snapshot) throws IOException {
+    try {
+      admin.snapshot(snapshot);
+      fail("Snapshot should not have succeed with name:" + snapshot.getName());
+    } catch (IllegalArgumentException e) {
+      LOG.debug("Correctly failed to start snapshot:" + e.getMessage());
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 4a68a94ab8c2..00aefe08d4d7 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -42,6 +42,8 @@
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
@@ -92,6 +94,14 @@ public static class CPMasterObserver implements MasterObserver {
     private boolean postStartMasterCalled;
     private boolean startCalled;
     private boolean stopCalled;
+    private boolean preSnapshotCalled;
+    private boolean postSnapshotCalled;
+    private boolean preCloneSnapshotCalled;
+    private boolean postCloneSnapshotCalled;
+    private boolean preRestoreSnapshotCalled;
+    private boolean postRestoreSnapshotCalled;
+    private boolean preDeleteSnapshotCalled;
+    private boolean postDeleteSnapshotCalled;
 
     public void enableBypass(boolean bypass) {
       this.bypass = bypass;
@@ -124,6 +134,14 @@ public void resetStates() {
       postBalanceCalled = false;
       preBalanceSwitchCalled = false;
       postBalanceSwitchCalled = false;
+      preSnapshotCalled = false;
+      postSnapshotCalled = false;
+      preCloneSnapshotCalled = false;
+      postCloneSnapshotCalled = false;
+      preRestoreSnapshotCalled = false;
+      postRestoreSnapshotCalled = false;
+      preDeleteSnapshotCalled = false;
+      postDeleteSnapshotCalled = false;
     }
 
     @Override
@@ -463,10 +481,82 @@ public void stop(CoprocessorEnvironment env) throws IOException {
     public boolean wasStarted() { return startCalled; }
 
     public boolean wasStopped() { return stopCalled; }
+
+    @Override
+    public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      preSnapshotCalled = true;
+    }
+
+    @Override
+    public void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      postSnapshotCalled = true;
+    }
+
+    public boolean wasSnapshotCalled() {
+      return preSnapshotCalled && postSnapshotCalled;
+    }
+
+    @Override
+    public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      preCloneSnapshotCalled = true;
+    }
+
+    @Override
+    public void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      postCloneSnapshotCalled = true;
+    }
+
+    public boolean wasCloneSnapshotCalled() {
+      return preCloneSnapshotCalled && postCloneSnapshotCalled;
+    }
+
+    @Override
+    public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      preRestoreSnapshotCalled = true;
+    }
+
+    @Override
+    public void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      postRestoreSnapshotCalled = true;
+    }
+
+    public boolean wasRestoreSnapshotCalled() {
+      return preRestoreSnapshotCalled && postRestoreSnapshotCalled;
+    }
+
+    @Override
+    public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot) throws IOException {
+      preDeleteSnapshotCalled = true;
+    }
+
+    @Override
+    public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot) throws IOException {
+      postDeleteSnapshotCalled = true;
+    }
+
+    public boolean wasDeleteSnapshotCalled() {
+      return preDeleteSnapshotCalled && postDeleteSnapshotCalled;
+    }
   }
 
   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static byte[] TEST_SNAPSHOT = Bytes.toBytes("observed_snapshot");
   private static byte[] TEST_TABLE = Bytes.toBytes("observed_table");
+  private static byte[] TEST_CLONE = Bytes.toBytes("observed_clone");
   private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
 
@@ -475,6 +565,8 @@ public static void setupBeforeClass() throws Exception {
     Configuration conf = UTIL.getConfiguration();
     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
         CPMasterObserver.class.getName());
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
     // We need more than one data server on this test
     UTIL.startMiniCluster(2);
   }
@@ -719,6 +811,63 @@ public void testRegionTransitionOperations() throws Exception {
         cp.wasBalanceCalled());
   }
 
+  @Test
+  public void testSnapshotOperations() throws Exception {
+    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+    HMaster master = cluster.getMaster();
+    MasterCoprocessorHost host = master.getCoprocessorHost();
+    CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
+        CPMasterObserver.class.getName());
+    cp.resetStates();
+
+    // create a table
+    HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+
+    // delete table if exists
+    if (admin.tableExists(TEST_TABLE)) {
+      UTIL.deleteTable(TEST_TABLE);
+    }
+
+    admin.createTable(htd);
+    admin.disableTable(TEST_TABLE);
+    assertTrue(admin.isTableDisabled(TEST_TABLE));
+
+    try {
+      // Test snapshot operation
+      assertFalse("Coprocessor should not have been called yet",
+        cp.wasSnapshotCalled());
+      admin.snapshot(TEST_SNAPSHOT, TEST_TABLE);
+      assertTrue("Coprocessor should have been called on snapshot",
+        cp.wasSnapshotCalled());
+
+      // Test clone operation
+      admin.cloneSnapshot(TEST_SNAPSHOT, TEST_CLONE);
+      assertTrue("Coprocessor should have been called on snapshot clone",
+        cp.wasCloneSnapshotCalled());
+      assertFalse("Coprocessor restore should not have been called on snapshot clone",
+        cp.wasRestoreSnapshotCalled());
+      admin.disableTable(TEST_CLONE);
+      assertTrue(admin.isTableDisabled(TEST_TABLE));
+      admin.deleteTable(TEST_CLONE);
+
+      // Test restore operation
+      cp.resetStates();
+      admin.restoreSnapshot(TEST_SNAPSHOT);
+      assertTrue("Coprocessor should have been called on snapshot restore",
+        cp.wasRestoreSnapshotCalled());
+      assertFalse("Coprocessor clone should not have been called on snapshot restore",
+        cp.wasCloneSnapshotCalled());
+
+      admin.deleteSnapshot(TEST_SNAPSHOT);
+      assertTrue("Coprocessor should have been called on snapshot delete",
+        cp.wasDeleteSnapshotCalled());
+    } finally {
+      admin.deleteTable(TEST_TABLE);
+    }
+  }
+
   private void waitForRITtoBeZero(HMaster master) throws IOException {
     // wait for assignments to finish
     AssignmentManager mgr = master.getAssignmentManager();
diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java
new file mode 100644
index 000000000000..e5c47b13996a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test that we propagate errors through an dispatcher exactly once via different failure
+ * injection mechanisms.
+ */
+@Category(SmallTests.class)
+public class TestForeignExceptionDispatcher {
+  private static final Log LOG = LogFactory.getLog(TestForeignExceptionDispatcher.class);
+
+  /**
+   * Exception thrown from the test
+   */
+  final ForeignException EXTEXN = new ForeignException("FORTEST", new IllegalArgumentException("FORTEST"));
+  final ForeignException EXTEXN2 = new ForeignException("FORTEST2", new IllegalArgumentException("FORTEST2"));
+
+  /**
+   * Tests that a dispatcher only dispatches only the first exception, and does not propagate
+   * subsequent exceptions.
+   */
+  @Test
+  public void testErrorPropagation() {
+    ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionDispatcher dispatcher = new ForeignExceptionDispatcher();
+
+    // add the listeners
+    dispatcher.addListener(listener1);
+    dispatcher.addListener(listener2);
+
+    // create an artificial error
+    dispatcher.receive(EXTEXN);
+
+    // make sure the listeners got the error
+    Mockito.verify(listener1, Mockito.times(1)).receive(EXTEXN);
+    Mockito.verify(listener2, Mockito.times(1)).receive(EXTEXN);
+
+    // make sure that we get an exception
+    try {
+      dispatcher.rethrowException();
+      fail("Monitor should have thrown an exception after getting error.");
+    } catch (ForeignException ex) {
+      assertTrue("Got an unexpected exception:" + ex, ex.getCause() == EXTEXN.getCause());
+      LOG.debug("Got the testing exception!");
+    }
+
+    // push another error, which should be not be passed to listeners
+    dispatcher.receive(EXTEXN2);
+    Mockito.verify(listener1, Mockito.never()).receive(EXTEXN2);
+    Mockito.verify(listener2, Mockito.never()).receive(EXTEXN2);
+  }
+
+  @Test
+  public void testSingleDispatcherWithTimer() {
+    ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class);
+
+    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
+
+    // add the listeners
+    monitor.addListener(listener1);
+    monitor.addListener(listener2);
+
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(monitor, 1000);
+    timer.start();
+    timer.trigger();
+
+    assertTrue("Monitor didn't get timeout", monitor.hasException());
+
+    // verify that that we propagated the error
+    Mockito.verify(listener1).receive(Mockito.any(ForeignException.class));
+    Mockito.verify(listener2).receive(Mockito.any(ForeignException.class));
+  }
+
+  /**
+   * Test that the dispatcher can receive an error via the timer mechanism.
+   */
+  @Test
+  public void testAttemptTimer() {
+    ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionDispatcher orchestrator = new ForeignExceptionDispatcher();
+
+    // add the listeners
+    orchestrator.addListener(listener1);
+    orchestrator.addListener(listener2);
+
+    // now create a timer and check for that error
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(orchestrator, 1000);
+    timer.start();
+    timer.trigger();
+    // make sure that we got the timer error
+    Mockito.verify(listener1, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+    Mockito.verify(listener2, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java
new file mode 100644
index 000000000000..11363fed437f
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Test that we correctly serialize exceptions from a remote source
+ */
+@Category(SmallTests.class)
+public class TestForeignExceptionSerialization {
+  private static final String srcName = "someNode";
+
+  /**
+   * Verify that we get back similar stack trace information before an after serialization.
+   * @throws InvalidProtocolBufferException
+   */
+  @Test
+  public void testSimpleException() throws InvalidProtocolBufferException {
+    String data = "some bytes";
+    ForeignException in = new ForeignException("SRC", new IllegalArgumentException(data));
+    // check that we get the data back out
+    ForeignException e = ForeignException.deserialize(ForeignException.serialize(srcName, in));
+    assertNotNull(e);
+
+    // now check that we get the right stack trace
+    StackTraceElement elem = new StackTraceElement(this.getClass().toString(), "method", "file", 1);
+    in.setStackTrace(new StackTraceElement[] { elem });
+    e = ForeignException.deserialize(ForeignException.serialize(srcName, in));
+
+    assertNotNull(e);
+    assertEquals("Stack trace got corrupted", elem, e.getCause().getStackTrace()[0]);
+    assertEquals("Got an unexpectedly long stack trace", 1, e.getCause().getStackTrace().length);
+  }
+
+  /**
+   * Compare that a generic exception's stack trace has the same stack trace elements after
+   * serialization and deserialization
+   * @throws InvalidProtocolBufferException
+   */
+  @Test
+  public void testRemoteFromLocal() throws InvalidProtocolBufferException {
+    String errorMsg = "some message";
+    Exception generic = new Exception(errorMsg);
+    generic.printStackTrace();
+    assertTrue(generic.getMessage().contains(errorMsg));
+
+    ForeignException e = ForeignException.deserialize(ForeignException.serialize(srcName, generic));
+    assertArrayEquals("Local stack trace got corrupted", generic.getStackTrace(), e.getCause().getStackTrace());
+
+    e.printStackTrace(); // should have ForeignException and source node in it.
+    assertTrue(e.getCause().getCause() == null);
+
+    // verify that original error message is present in Foreign exception message
+    assertTrue(e.getCause().getMessage().contains(errorMsg));
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
new file mode 100644
index 000000000000..ff72d9174371
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import static org.junit.Assert.fail;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test the {@link TimeoutExceptionInjector} to ensure we fulfill contracts
+ */
+@Category(SmallTests.class)
+public class TestTimeoutExceptionInjector {
+
+  private static final Log LOG = LogFactory.getLog(TestTimeoutExceptionInjector.class);
+
+  /**
+   * Test that a manually triggered timer fires an exception.
+   */
+  @Test(timeout = 1000)
+  public void testTimerTrigger() {
+    final long time = 10000000; // pick a value that is very far in the future
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.start();
+    timer.trigger();
+    Mockito.verify(listener, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+  }
+
+  /**
+   * Test that a manually triggered exception with data fires with the data in receiveError.
+   */
+  @Test
+  public void testTimerPassesOnErrorInfo() {
+    final long time = 1000000;
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.start();
+    timer.trigger();
+    Mockito.verify(listener).receive(Mockito.any(ForeignException.class));
+  }
+
+  /**
+   * Demonstrate TimeoutExceptionInjector semantics -- completion means no more exceptions passed to
+   * error listener.
+   */
+  @Test(timeout = 1000)
+  public void testStartAfterComplete() throws InterruptedException {
+    final long time = 10;
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.complete();
+    try {
+      timer.start();
+      fail("Timer should fail to start after complete.");
+    } catch (IllegalStateException e) {
+      LOG.debug("Correctly failed timer: " + e.getMessage());
+    }
+    Thread.sleep(time + 1);
+    Mockito.verifyZeroInteractions(listener);
+  }
+
+  /**
+   * Demonstrate TimeoutExceptionInjector semantics -- triggering fires exception and completes
+   * the timer.
+   */
+  @Test(timeout = 1000)
+  public void testStartAfterTrigger() throws InterruptedException {
+    final long time = 10;
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.trigger();
+    try {
+      timer.start();
+      fail("Timer should fail to start after complete.");
+    } catch (IllegalStateException e) {
+      LOG.debug("Correctly failed timer: " + e.getMessage());
+    }
+    Thread.sleep(time * 2);
+    Mockito.verify(listener, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+    Mockito.verifyNoMoreInteractions(listener);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java b/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java
new file mode 100644
index 000000000000..88310ef900f6
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import junit.framework.TestCase;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.io.FileLink;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test that FileLink switches between alternate locations
+ * when the current location moves or gets deleted.
+ */
+@Category(MediumTests.class)
+public class TestFileLink {
+  /**
+   * Test, on HDFS, that the FileLink is still readable
+   * even when the current file gets renamed.
+   */
+  @Test
+  public void testHDFSLinkReadDuringRename() throws Exception {
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    Configuration conf = testUtil.getConfiguration();
+    conf.setInt("dfs.blocksize", 1024 * 1024);
+    conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);
+
+    testUtil.startMiniDFSCluster(1);
+    MiniDFSCluster cluster = testUtil.getDFSCluster();
+    FileSystem fs = cluster.getFileSystem();
+    assertEquals("hdfs", fs.getUri().getScheme());
+
+    try {
+      testLinkReadDuringRename(fs, testUtil.getDefaultRootDirPath());
+    } finally {
+      testUtil.shutdownMiniCluster();
+    }
+  }
+
+  /**
+   * Test, on a local filesystem, that the FileLink is still readable
+   * even when the current file gets renamed.
+   */
+  @Test
+  public void testLocalLinkReadDuringRename() throws IOException {
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    FileSystem fs = testUtil.getTestFileSystem();
+    assertEquals("file", fs.getUri().getScheme());
+    testLinkReadDuringRename(fs, testUtil.getDataTestDir());
+  }
+
+  /**
+   * Test that link is still readable even when the current file gets renamed.
+   */
+  private void testLinkReadDuringRename(FileSystem fs, Path rootDir) throws IOException {
+    Path originalPath = new Path(rootDir, "test.file");
+    Path archivedPath = new Path(rootDir, "archived.file");
+
+    writeSomeData(fs, originalPath, 256 << 20, (byte)2);
+
+    List<Path> files = new ArrayList<Path>();
+    files.add(originalPath);
+    files.add(archivedPath);
+
+    FileLink link = new FileLink(files);
+    FSDataInputStream in = link.open(fs);
+    try {
+      byte[] data = new byte[8192];
+      long size = 0;
+
+      // Read from origin
+      int n = in.read(data);
+      dataVerify(data, n, (byte)2);
+      size += n;
+
+      // Move origin to archive
+      assertFalse(fs.exists(archivedPath));
+      fs.rename(originalPath, archivedPath);
+      assertFalse(fs.exists(originalPath));
+      assertTrue(fs.exists(archivedPath));
+
+      // Try to read to the end
+      while ((n = in.read(data)) > 0) {
+        dataVerify(data, n, (byte)2);
+        size += n;
+      }
+
+      assertEquals(256 << 20, size);
+    } finally {
+      in.close();
+      if (fs.exists(originalPath)) fs.delete(originalPath);
+      if (fs.exists(archivedPath)) fs.delete(archivedPath);
+    }
+  }
+
+  /**
+   * Test that link is still readable even when the current file gets deleted.
+   *
+   * NOTE: This test is valid only on HDFS.
+   * When a file is deleted from a local file-system, it is simply 'unlinked'.
+   * The inode, which contains the file's data, is not deleted until all
+   * processes have finished with it.
+   * In HDFS when the request exceed the cached block locations,
+   * a query to the namenode is performed, using the filename,
+   * and the deleted file doesn't exists anymore (FileNotFoundException).
+   */
+  @Test
+  public void testHDFSLinkReadDuringDelete() throws Exception {
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    Configuration conf = testUtil.getConfiguration();
+    conf.setInt("dfs.blocksize", 1024 * 1024);
+    conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);
+
+    testUtil.startMiniDFSCluster(1);
+    MiniDFSCluster cluster = testUtil.getDFSCluster();
+    FileSystem fs = cluster.getFileSystem();
+    assertEquals("hdfs", fs.getUri().getScheme());
+
+    try {
+      List<Path> files = new ArrayList<Path>();
+      for (int i = 0; i < 3; i++) {
+        Path path = new Path(String.format("test-data-%d", i));
+        writeSomeData(fs, path, 1 << 20, (byte)i);
+        files.add(path);
+      }
+
+      FileLink link = new FileLink(files);
+      FSDataInputStream in = link.open(fs);
+      try {
+        byte[] data = new byte[8192];
+        int n;
+
+        // Switch to file 1
+        n = in.read(data);
+        dataVerify(data, n, (byte)0);
+        fs.delete(files.get(0));
+        skipBuffer(in, (byte)0);
+
+        // Switch to file 2
+        n = in.read(data);
+        dataVerify(data, n, (byte)1);
+        fs.delete(files.get(1));
+        skipBuffer(in, (byte)1);
+
+        // Switch to file 3
+        n = in.read(data);
+        dataVerify(data, n, (byte)2);
+        fs.delete(files.get(2));
+        skipBuffer(in, (byte)2);
+
+        // No more files available
+        try {
+          n = in.read(data);
+          assert(n <= 0);
+        } catch (FileNotFoundException e) {
+          assertTrue(true);
+        }
+      } finally {
+        in.close();
+      }
+    } finally {
+      testUtil.shutdownMiniCluster();
+    }
+  }
+
+  /**
+   * Write up to 'size' bytes with value 'v' into a new file called 'path'.
+   */
+  private void writeSomeData (FileSystem fs, Path path, long size, byte v) throws IOException {
+    byte[] data = new byte[4096];
+    for (int i = 0; i < data.length; i++) {
+      data[i] = v;
+    }
+
+    FSDataOutputStream stream = fs.create(path);
+    try {
+      long written = 0;
+      while (written < size) {
+        stream.write(data, 0, data.length);
+        written += data.length;
+      }
+    } finally {
+      stream.close();
+    }
+  }
+
+  /**
+   * Verify that all bytes in 'data' have 'v' as value.
+   */
+  private static void dataVerify(byte[] data, int n, byte v) {
+    for (int i = 0; i < n; ++i) {
+      assertEquals(v, data[i]);
+    }
+  }
+
+  private static void skipBuffer(FSDataInputStream in, byte v) throws IOException {
+    byte[] data = new byte[8192];
+    try {
+      int n;
+      while ((n = in.read(data)) == data.length) {
+        for (int i = 0; i < data.length; ++i) {
+          if (data[i] != v)
+            throw new Exception("File changed");
+        }
+      }
+    } catch (Exception e) {
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index 4554a0e84a3f..3a581ef7ffa4 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -19,6 +19,9 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Map;
@@ -28,19 +31,18 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.TestSchemaMetrics;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-import org.junit.experimental.categories.Category;
-import static org.junit.Assert.*;
-
 /**
  * Tests the concurrent LruBlockCache.<p>
  *
@@ -77,7 +79,6 @@ public void tearDown() throws Exception {
 
   @Test
   public void testBackgroundEvictionThread() throws Exception {
-
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 9); // room for 9, will evict
 
@@ -85,6 +86,14 @@ public void testBackgroundEvictionThread() throws Exception {
 
     CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
 
+    EvictionThread evictionThread = cache.getEvictionThread();
+    assertTrue(evictionThread != null);
+
+    // Make sure eviction thread has entered run method
+    while (!evictionThread.isEnteringRun()) {
+      Thread.sleep(1);
+    }
+
     // Add all the blocks
     for (CachedItem block : blocks) {
       cache.cacheBlock(block.cacheKey, block);
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
new file mode 100644
index 000000000000..c55084db99cd
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the HFileLink Cleaner.
+ * HFiles with links cannot be deleted until a link is present.
+ */
+@Category(SmallTests.class)
+public class TestHFileLinkCleaner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @Test
+  public void testHFileLinkCleaning() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir().toString());
+    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+
+    final String tableName = "test-table";
+    final String tableLinkName = "test-link";
+    final String hfileName = "1234567890";
+    final String familyName = "cf";
+
+    HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tableName));
+    HRegionInfo hriLink = new HRegionInfo(Bytes.toBytes(tableLinkName));
+
+    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
+    Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          tableName, hri.getEncodedName(), familyName);
+    Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          tableLinkName, hriLink.getEncodedName(), familyName);
+
+    // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
+    Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
+    fs.mkdirs(familyPath);
+    Path hfilePath = new Path(familyPath, hfileName);
+    fs.createNewFile(hfilePath);
+
+    // Create link to hfile
+    Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
+                                        hriLink.getEncodedName(), familyName);
+    fs.mkdirs(familyLinkPath);
+    HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
+    Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
+    assertTrue(fs.exists(linkBackRefDir));
+    FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
+    assertEquals(1, backRefs.length);
+    Path linkBackRef = backRefs[0].getPath();
+
+    // Initialize cleaner
+    final long ttl = 1000;
+    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
+    Server server = new DummyServer();
+    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
+
+    // Link backref cannot be removed
+    cleaner.chore();
+    assertTrue(fs.exists(linkBackRef));
+    assertTrue(fs.exists(hfilePath));
+
+    // Link backref can be removed
+    fs.rename(new Path(rootDir, tableLinkName), new Path(archiveDir, tableLinkName));
+    cleaner.chore();
+    assertFalse("Link should be deleted", fs.exists(linkBackRef));
+
+    // HFile can be removed
+    Thread.sleep(ttl * 2);
+    cleaner.chore();
+    assertFalse("HFile should be deleted", fs.exists(hfilePath));
+
+    // Remove everything
+    for (int i = 0; i < 4; ++i) {
+      Thread.sleep(ttl * 2);
+      cleaner.chore();
+    }
+    assertFalse("HFile should be deleted", fs.exists(new Path(archiveDir, tableName)));
+    assertFalse("Link should be deleted", fs.exists(new Path(archiveDir, tableLinkName)));
+
+    cleaner.interrupt();
+  }
+
+  private static Path getFamilyDirPath (final Path rootDir, final String table,
+    final String region, final String family) {
+    return new Path(new Path(new Path(rootDir, table), region), family);
+  }
+
+  static class DummyServer implements Server {
+
+    @Override
+    public Configuration getConfiguration() {
+      return TEST_UTIL.getConfiguration();
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      try {
+        return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+      return null;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+
+    @Override
+    public ServerName getServerName() {
+      return new ServerName("regionserver,60020,000000");
+    }
+
+    @Override
+    public void abort(String why, Throwable e) {}
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {}
+
+    @Override
+    public boolean isStopped() {
+      return false;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
new file mode 100644
index 000000000000..56db35826a3a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -0,0 +1,382 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Test the master-related aspects of a snapshot
+ */
+@Category(MediumTests.class)
+public class TestSnapshotFromMaster {
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotFromMaster.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final int NUM_RS = 2;
+  private static Path rootDir;
+  private static Path snapshots;
+  private static FileSystem fs;
+  private static HMaster master;
+
+  // for hfile archiving test.
+  private static Path archiveDir;
+  private static final String STRING_TABLE_NAME = "test";
+  private static final byte[] TEST_FAM = Bytes.toBytes("fam");
+  private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
+  // refresh the cache every 1/2 second
+  private static final long cacheRefreshPeriod = 500;
+
+  /**
+   * Setup the config for the cluster
+   */
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster(NUM_RS);
+    fs = UTIL.getDFSCluster().getFileSystem();
+    master = UTIL.getMiniHBaseCluster().getMaster();
+    rootDir = master.getMasterFileSystem().getRootDir();
+    snapshots = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+  }
+
+  private static void setupConf(Configuration conf) {
+    // disable the ui
+    conf.setInt("hbase.regionsever.info.port", -1);
+    // change the flush size to a small amount, regulating number of store files
+    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+    // so make sure we get a compaction when doing a load, but keep around some
+    // files in the store
+    conf.setInt("hbase.hstore.compaction.min", 3);
+    conf.setInt("hbase.hstore.compactionThreshold", 5);
+    // block writes if we get to 12 store files
+    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
+    // drop the number of attempts for the hbase admin
+    conf.setInt("hbase.client.retries.number", 1);
+    // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
+    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
+    conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, "");
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod);
+
+    // prevent aggressive region split
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      ConstantSizeRegionSplitPolicy.class.getName());
+  }
+
+  @Before
+  public void setup() throws Exception {
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    UTIL.deleteTable(TABLE_NAME);
+
+    // delete the archive directory, if its exists
+    if (fs.exists(archiveDir)) {
+      if (!fs.delete(archiveDir, true)) {
+        throw new IOException("Couldn't delete archive directory (" + archiveDir
+            + " for an unknown reason");
+      }
+    }
+
+    // delete the snapshot directory, if its exists
+    if (fs.exists(snapshots)) {
+      if (!fs.delete(snapshots, true)) {
+        throw new IOException("Couldn't delete snapshots directory (" + snapshots
+            + " for an unknown reason");
+      }
+    }
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    try {
+      UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      // NOOP;
+    }
+  }
+
+  /**
+   * Test that the contract from the master for checking on a snapshot are valid.
+   * <p>
+   * <ol>
+   * <li>If a snapshot fails with an error, we expect to get the source error.</li>
+   * <li>If there is no snapshot name supplied, we should get an error.</li>
+   * <li>If asking about a snapshot has hasn't occurred, you should get an error.</li>
+   * </ol>
+   */
+  @Test(timeout = 15000)
+  public void testIsDoneContract() throws Exception {
+
+    String snapshotName = "asyncExpectedFailureTest";
+
+    // check that we get an exception when looking up snapshot where one hasn't happened
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
+      UnknownSnapshotException.class);
+
+    // and that we get the same issue, even if we specify a name
+    SnapshotDescription desc = SnapshotDescription.newBuilder()
+      .setName(snapshotName).build();
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
+      UnknownSnapshotException.class);
+
+    // set a mock handler to simulate a snapshot
+    DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class);
+    Mockito.when(mockHandler.getException()).thenReturn(null);
+    Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
+    Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+
+    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
+
+    // if we do a lookup without a snapshot name, we should fail - you should always know your name
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
+      UnknownSnapshotException.class);
+
+    // then do the lookup for the snapshot that it is done
+    boolean isDone = master.isSnapshotDone(new HSnapshotDescription(desc));
+    assertTrue("Snapshot didn't complete when it should have.", isDone);
+
+    // now try the case where we are looking for a snapshot we didn't take
+    desc = SnapshotDescription.newBuilder().setName("Not A Snapshot").build();
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
+      UnknownSnapshotException.class);
+
+    // then create a snapshot to the fs and make sure that we can find it when checking done
+    snapshotName = "completed";
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    desc = desc.toBuilder().setName(snapshotName).build();
+    SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshotDir, fs);
+
+    isDone = master.isSnapshotDone(new HSnapshotDescription(desc));
+    assertTrue("Completed, on-disk snapshot not found", isDone);
+  }
+
+  @Test
+  public void testGetCompletedSnapshots() throws Exception {
+    // first check when there are no snapshots
+    List<HSnapshotDescription> snapshots = master.getCompletedSnapshots();
+    assertEquals("Found unexpected number of snapshots", 0, snapshots.size());
+
+    // write one snapshot to the fs
+    String snapshotName = "completed";
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
+    SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
+
+    // check that we get one snapshot
+    snapshots = master.getCompletedSnapshots();
+    assertEquals("Found unexpected number of snapshots", 1, snapshots.size());
+    List<HSnapshotDescription> expected = Lists.newArrayList(new HSnapshotDescription(snapshot));
+    assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
+
+    // write a second snapshot
+    snapshotName = "completed_two";
+    snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
+    SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
+    expected.add(new HSnapshotDescription(snapshot));
+
+    // check that we get one snapshot
+    snapshots = master.getCompletedSnapshots();
+    assertEquals("Found unexpected number of snapshots", 2, snapshots.size());
+    assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
+  }
+
+  @Test
+  public void testDeleteSnapshot() throws Exception {
+
+    String snapshotName = "completed";
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
+
+    try {
+      master.deleteSnapshot(new HSnapshotDescription(snapshot));
+      fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
+    } catch (IOException e) {
+      LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage());
+    }
+
+    // write one snapshot to the fs
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
+
+    // then delete the existing snapshot,which shouldn't cause an exception to be thrown
+    master.deleteSnapshot(new HSnapshotDescription(snapshot));
+  }
+
+  /**
+   * Test that the snapshot hfile archive cleaner works correctly. HFiles that are in snapshots
+   * should be retained, while those that are not in a snapshot should be deleted.
+   * @throws Exception on failure
+   */
+  @Test
+  public void testSnapshotHFileArchiving() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    // load the table
+    UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+
+    // disable the table so we can take a snapshot
+    admin.disableTable(TABLE_NAME);
+
+    // take a snapshot of the table
+    String snapshotName = "snapshot";
+    byte[] snapshotNameBytes = Bytes.toBytes(snapshotName);
+    admin.snapshot(snapshotNameBytes, TABLE_NAME);
+
+    Configuration conf = master.getConfiguration();
+    LOG.info("After snapshot File-System state");
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // ensure we only have one snapshot
+    SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME);
+
+    // renable the table so we can compact the regions
+    admin.enableTable(TABLE_NAME);
+
+    // compact the files so we get some archived files for the table we just snapshotted
+    List<HRegion> regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
+    for (HRegion region : regions) {
+      region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it.
+      region.compactStores();
+    }
+    LOG.info("After compaction File-System state");
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // make sure the cleaner has run
+    LOG.debug("Running hfile cleaners");
+    ensureHFileCleanersRun();
+    LOG.info("After cleaners File-System state: " + rootDir);
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // get the snapshot files for the table
+    Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    FileStatus[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+    // check that the files in the archive contain the ones that we need for the snapshot
+    LOG.debug("Have snapshot hfiles:");
+    for (FileStatus file : snapshotHFiles) {
+      LOG.debug(file.getPath());
+    }
+    // get the archived files for the table
+    Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
+
+    // and make sure that there is a proper subset
+    for (FileStatus file : snapshotHFiles) {
+      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file.getPath(),
+        files.contains(file.getPath().getName()));
+    }
+
+    // delete the existing snapshot
+    admin.deleteSnapshot(snapshotNameBytes);
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // make sure that we don't keep around the hfiles that aren't in a snapshot
+    // make sure we wait long enough to refresh the snapshot hfile
+    List<BaseHFileCleanerDelegate> delegates = UTIL.getMiniHBaseCluster().getMaster()
+        .getHFileCleaner().cleanersChain;
+    for (BaseHFileCleanerDelegate delegate: delegates) {
+      if (delegate instanceof SnapshotHFileCleaner) {
+        ((SnapshotHFileCleaner)delegate).getFileCacheForTesting().triggerCacheRefreshForTesting();
+      }
+    }
+    // run the cleaner again
+    LOG.debug("Running hfile cleaners");
+    ensureHFileCleanersRun();
+    LOG.info("After delete snapshot cleaners run File-System state");
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
+    assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0,
+      files.size());
+  }
+
+  /**
+   * @return all the HFiles for a given table that have been archived
+   * @throws IOException on expected failure
+   */
+  private final Collection<String> getArchivedHFiles(Path archiveDir, Path rootDir,
+      FileSystem fs, String tableName) throws IOException {
+    Path tableArchive = new Path(archiveDir, tableName);
+    FileStatus[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
+    List<String> files = new ArrayList<String>(archivedHFiles.length);
+    LOG.debug("Have archived hfiles: " + tableArchive);
+    for (FileStatus file : archivedHFiles) {
+      LOG.debug(file.getPath());
+      files.add(file.getPath().getName());
+    }
+    // sort the archived files
+
+    Collections.sort(files);
+    return files;
+  }
+
+  /**
+   * Make sure the {@link HFileCleaner HFileCleaners} run at least once
+   */
+  private static void ensureHFileCleanersRun() {
+    UTIL.getHBaseCluster().getMaster().getHFileCleaner().chore();
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
new file mode 100644
index 000000000000..a6e627cde729
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that we correctly reload the cache, filter directories, etc.
+ */
+@Category(MediumTests.class)
+public class TestSnapshotFileCache {
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static FileSystem fs;
+  private static Path rootDir;
+
+  @BeforeClass
+  public static void startCluster() throws Exception {
+    UTIL.startMiniDFSCluster(1);
+    fs = UTIL.getDFSCluster().getFileSystem();
+    rootDir = UTIL.getDefaultRootDirPath();
+  }
+
+  @AfterClass
+  public static void stopCluster() throws Exception {
+    UTIL.shutdownMiniDFSCluster();
+  }
+
+  @After
+  public void cleanupFiles() throws Exception {
+    // cleanup the snapshot directory
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    fs.delete(snapshotDir, true);
+  }
+
+  @Test(timeout = 10000000)
+  public void testLoadAndDelete() throws Exception {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    Path file2 = new Path(family, "file2");
+
+    // create two hfiles under the snapshot
+    fs.create(file1);
+    fs.create(file2);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // then make sure the cache finds them
+    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+    String not = "file-shouldn't-be-found";
+    assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
+
+    // make sure we get a little bit of separation in the modification times
+    // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
+    Thread.sleep(10);
+
+    LOG.debug("Deleting snapshot.");
+    // then delete the snapshot and make sure that we can still find the files
+    if (!fs.delete(snapshot, true)) {
+      throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
+    }
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+
+    LOG.debug("Checking to see if file is deleted.");
+    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+
+    // then trigger a refresh
+    cache.triggerCacheRefreshForTesting();
+    // and not it shouldn't find those files
+    assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
+      cache.contains(file1.getName()));
+    assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
+      cache.contains(file2.getName()));
+
+    fs.delete(snapshotDir, true);
+  }
+
+  @Test
+  public void testLoadsTmpDir() throws Exception {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    // create a file in a 'completed' snapshot
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    fs.create(file1);
+
+    // create an 'in progress' snapshot
+    SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
+    snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+    region = new Path(snapshot, "7e91021");
+    family = new Path(region, "fam");
+    Path file2 = new Path(family, "file2");
+    fs.create(file2);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // then make sure the cache finds both files
+    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+  }
+
+  @Test
+  public void testJustFindLogsDirectory() throws Exception {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+            }
+        });
+
+    // create a file in a 'completed' snapshot
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    fs.create(file1);
+
+    // and another file in the logs directory
+    Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
+    Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
+    fs.create(log);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // then make sure the cache only finds the log files
+    assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
+      cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
+  }
+
+  @Test
+  public void testReloadModifiedDirectory() throws IOException {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    Path file2 = new Path(family, "file2");
+
+    // create two hfiles under the snapshot
+    fs.create(file1);
+    fs.create(file2);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
+
+    // now delete the snapshot and add a file with a different name
+    fs.delete(snapshot, true);
+    Path file3 = new Path(family, "new_file");
+    fs.create(file3);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+    assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
+  }
+
+  class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
+    public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
+      Collection<String> files =  new HashSet<String>();
+      files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
+      files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
+      return files;
+    }
+  };
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
new file mode 100644
index 000000000000..03847c4e5bf4
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot
+ */
+@Category(SmallTests.class)
+public class TestSnapshotHFileCleaner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @AfterClass
+  public static void cleanup() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    // cleanup
+    fs.delete(rootDir, true);
+  }
+
+  @Test
+  public void testFindsSnapshotFilesWhenCleaning() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
+    Path rootDir = FSUtils.getRootDir(conf);
+    Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+    FileSystem fs = FileSystem.get(conf);
+    SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
+    cleaner.setConf(conf);
+
+    // write an hfile to the snapshot directory
+    String snapshotName = "snapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotName);
+    String table = "table";
+    byte[] tableName = Bytes.toBytes(table);
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    HRegionInfo mockRegion = new HRegionInfo(tableName);
+    Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
+    Path familyDir = new Path(regionSnapshotDir, "family");
+    // create a reference to a supposedly valid hfile
+    String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
+    Path refFile = new Path(familyDir, hfile);
+
+    // make sure the reference file exists
+    fs.create(refFile);
+
+    // create the hfile in the archive
+    fs.mkdirs(archivedHfileDir);
+    fs.createNewFile(new Path(archivedHfileDir, hfile));
+
+    // make sure that the file isn't deletable
+    assertFalse(cleaner.isFileDeletable(new Path(hfile)));
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
new file mode 100644
index 000000000000..f0f2ebd620ca
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that the snapshot log cleaner finds logs referenced in a snapshot
+ */
+@Category(SmallTests.class)
+public class TestSnapshotLogCleaner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @AfterClass
+  public static void cleanup() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    // cleanup
+    fs.delete(rootDir, true);
+  }
+
+  @Test
+  public void testFindsSnapshotFilesWhenCleaning() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    SnapshotLogCleaner cleaner = new SnapshotLogCleaner();
+    cleaner.setConf(conf);
+
+    // write an hfile to the snapshot directory
+    String snapshotName = "snapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotName);
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
+    String timestamp = "1339643343027";
+    String hostFromMaster = "localhost%2C59648%2C1339643336601";
+
+    Path hostSnapshotLogDir = new Path(snapshotLogDir, hostFromMaster);
+    String snapshotlogfile = hostFromMaster + "." + timestamp + ".hbase";
+
+    // add the reference to log in the snapshot
+    fs.create(new Path(hostSnapshotLogDir, snapshotlogfile));
+
+    // now check to see if that log file would get deleted.
+    Path oldlogDir = new Path(rootDir, ".oldlogs");
+    Path logFile = new Path(oldlogDir, snapshotlogfile);
+    fs.create(logFile);
+
+    // make sure that the file isn't deletable
+    assertFalse(cleaner.isFileDeletable(logFile));
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
new file mode 100644
index 000000000000..c84b6c03f517
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.zookeeper.KeeperException;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test basic snapshot manager functionality
+ */
+@Category(SmallTests.class)
+public class TestSnapshotManager {
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  MasterServices services = Mockito.mock(MasterServices.class);
+  ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class);
+  ExecutorService pool = Mockito.mock(ExecutorService.class);
+  MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
+  FileSystem fs;
+  {
+    try {
+      fs = UTIL.getTestFileSystem();
+    } catch (IOException e) {
+      throw new RuntimeException("Couldn't get test filesystem", e);
+    }
+  }
+
+   private SnapshotManager getNewManager() throws IOException, KeeperException {
+    return getNewManager(UTIL.getConfiguration());
+  }
+
+  private SnapshotManager getNewManager(final Configuration conf) throws IOException, KeeperException {
+    Mockito.reset(services);
+    Mockito.when(services.getConfiguration()).thenReturn(conf);
+    Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
+    Mockito.when(mfs.getFileSystem()).thenReturn(fs);
+    Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
+    return new SnapshotManager(services, coordinator, pool);
+  }
+
+  @Test
+  public void testInProcess() throws KeeperException, IOException {
+    SnapshotManager manager = getNewManager();
+    TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
+    assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
+    manager.setSnapshotHandlerForTesting(handler);
+    Mockito.when(handler.isFinished()).thenReturn(false);
+    assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot());
+    Mockito.when(handler.isFinished()).thenReturn(true);
+    assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
+  }
+
+  /**
+   * Verify the snapshot support based on the configuration.
+   */
+  @Test
+  public void testSnapshotSupportConfiguration() throws Exception {
+    // No configuration (no cleaners, not enabled): snapshot feature disabled
+    Configuration conf = new Configuration();
+    SnapshotManager manager = getNewManager(conf);
+    assertFalse("Snapshot should be disabled with no configuration", isSnapshotSupported(manager));
+
+    // force snapshot feature to be enabled
+    conf = new Configuration();
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    manager = getNewManager(conf);
+    assertTrue("Snapshot should be enabled", isSnapshotSupported(manager));
+
+    // force snapshot feature to be disabled
+    conf = new Configuration();
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
+    manager = getNewManager(conf);
+    assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));
+
+    // force snapshot feature to be disabled, even if cleaners are present
+    conf = new Configuration();
+    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+      SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
+    conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName());
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
+    manager = getNewManager(conf);
+    assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));
+
+    // cleaners are present, but missing snapshot enabled property
+    conf = new Configuration();
+    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+      SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
+    conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName());
+    manager = getNewManager(conf);
+    assertTrue("Snapshot should be enabled, because cleaners are present",
+      isSnapshotSupported(manager));
+
+    // Create a "test snapshot"
+    Path rootDir = UTIL.getDataTestDir();
+    Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+      "testSnapshotSupportConfiguration", rootDir);
+    fs.mkdirs(testSnapshotDir);
+    try {
+      // force snapshot feature to be disabled, but snapshots are present
+      conf = new Configuration();
+      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
+      manager = getNewManager(conf);
+      fail("Master should not start when snapshot is disabled, but snapshots are present");
+    } catch (UnsupportedOperationException e) {
+      // expected
+    } finally {
+      fs.delete(testSnapshotDir, true);
+    }
+  }
+
+  private boolean isSnapshotSupported(final SnapshotManager manager) {
+    try {
+      manager.checkSnapshotSupport();
+      return true;
+    } catch (UnsupportedOperationException e) {
+      return false;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
new file mode 100644
index 000000000000..4a249ba55f5f
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Demonstrate how Procedure handles single members, multiple members, and errors semantics
+ */
+@Category(SmallTests.class)
+public class TestProcedure {
+
+  ProcedureCoordinator coord;
+
+  @Before
+  public void setup() {
+    coord = mock(ProcedureCoordinator.class);
+    final ProcedureCoordinatorRpcs comms = mock(ProcedureCoordinatorRpcs.class);
+    when(coord.getRpcs()).thenReturn(comms); // make it not null
+  }
+
+  class LatchedProcedure extends Procedure {
+    CountDownLatch startedAcquireBarrier = new CountDownLatch(1);
+    CountDownLatch startedDuringBarrier = new CountDownLatch(1);
+    CountDownLatch completedProcedure = new CountDownLatch(1);
+
+    public LatchedProcedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor,
+        long wakeFreq, long timeout, String opName, byte[] data,
+        List<String> expectedMembers) {
+      super(coord, monitor, wakeFreq, timeout, opName, data, expectedMembers);
+    }
+
+    @Override
+    public void sendGlobalBarrierStart() {
+      startedAcquireBarrier.countDown();
+    }
+
+    @Override
+    public void sendGlobalBarrierReached() {
+      startedDuringBarrier.countDown();
+    }
+
+    @Override
+    public void sendGlobalBarrierComplete() {
+      completedProcedure.countDown();
+    }
+  };
+
+  /**
+   * With a single member, verify ordered execution.  The Coordinator side is run in a separate
+   * thread so we can only trigger from members and wait for particular state latches.
+   */
+  @Test(timeout = 1000)
+  public void testSingleMember() throws Exception {
+    // The member
+    List<String> members =  new ArrayList<String>();
+    members.add("member");
+    LatchedProcedure proc = new LatchedProcedure(coord, new ForeignExceptionDispatcher(), 100,
+        Integer.MAX_VALUE, "op", null, members);
+    final LatchedProcedure procspy = spy(proc);
+    // coordinator: start the barrier procedure
+    new Thread() {
+      public void run() {
+        procspy.call();
+      }
+    }.start();
+
+    // coordinator: wait for the barrier to be acquired, then send start barrier
+    proc.startedAcquireBarrier.await();
+
+    // we only know that {@link Procedure#sendStartBarrier()} was called, and others are blocked.
+    verify(procspy).sendGlobalBarrierStart();
+    verify(procspy, never()).sendGlobalBarrierReached();
+    verify(procspy, never()).sendGlobalBarrierComplete();
+    verify(procspy, never()).barrierAcquiredByMember(anyString());
+
+    // member: trigger global barrier acquisition
+    proc.barrierAcquiredByMember(members.get(0));
+
+    // coordinator: wait for global barrier to be acquired.
+    proc.acquiredBarrierLatch.await();
+    verify(procspy).sendGlobalBarrierStart(); // old news
+
+    // since two threads, we cannot guarantee that {@link Procedure#sendSatsifiedBarrier()} was
+    // or was not called here.
+
+    // member: trigger global barrier release
+    proc.barrierReleasedByMember(members.get(0));
+
+    // coordinator: wait for procedure to be completed
+    proc.completedProcedure.await();
+    verify(procspy).sendGlobalBarrierReached();
+    verify(procspy).sendGlobalBarrierComplete();
+    verify(procspy, never()).receive(any(ForeignException.class));
+  }
+
+  @Test(timeout=1000)
+  public void testMultipleMember() throws Exception {
+    // 2 members
+    List<String> members =  new ArrayList<String>();
+    members.add("member1");
+    members.add("member2");
+
+    LatchedProcedure proc = new LatchedProcedure(coord, new ForeignExceptionDispatcher(), 100,
+        Integer.MAX_VALUE, "op", null, members);
+    final LatchedProcedure procspy = spy(proc);
+    // start the barrier procedure
+    new Thread() {
+      public void run() {
+        procspy.call();
+      }
+    }.start();
+
+    // coordinator: wait for the barrier to be acquired, then send start barrier
+    procspy.startedAcquireBarrier.await();
+
+    // we only know that {@link Procedure#sendStartBarrier()} was called, and others are blocked.
+    verify(procspy).sendGlobalBarrierStart();
+    verify(procspy, never()).sendGlobalBarrierReached();
+    verify(procspy, never()).sendGlobalBarrierComplete();
+    verify(procspy, never()).barrierAcquiredByMember(anyString()); // no externals
+
+    // member0: [1/2] trigger global barrier acquisition.
+    procspy.barrierAcquiredByMember(members.get(0));
+
+    // coordinator not satisified.
+    verify(procspy).sendGlobalBarrierStart();
+    verify(procspy, never()).sendGlobalBarrierReached();
+    verify(procspy, never()).sendGlobalBarrierComplete();
+
+    // member 1: [2/2] trigger global barrier acquisition.
+    procspy.barrierAcquiredByMember(members.get(1));
+
+    // coordinator: wait for global barrier to be acquired.
+    procspy.startedDuringBarrier.await();
+    verify(procspy).sendGlobalBarrierStart(); // old news
+
+    // member 1, 2: trigger global barrier release
+    procspy.barrierReleasedByMember(members.get(0));
+    procspy.barrierReleasedByMember(members.get(1));
+
+    // coordinator wait for procedure to be completed
+    procspy.completedProcedure.await();
+    verify(procspy).sendGlobalBarrierReached();
+    verify(procspy).sendGlobalBarrierComplete();
+    verify(procspy, never()).receive(any(ForeignException.class));
+  }
+
+  @Test(timeout = 1000)
+  public void testErrorPropagation() throws Exception {
+    List<String> members =  new ArrayList<String>();
+    members.add("member");
+    Procedure proc = new Procedure(coord, new ForeignExceptionDispatcher(), 100,
+        Integer.MAX_VALUE, "op", null, members);
+    final Procedure procspy = spy(proc);
+
+    ForeignException cause = new ForeignException("SRC", "External Exception");
+    proc.receive(cause);
+
+    // start the barrier procedure
+    Thread t = new Thread() {
+      public void run() {
+        procspy.call();
+      }
+    };
+    t.start();
+    t.join();
+
+    verify(procspy, never()).sendGlobalBarrierStart();
+    verify(procspy, never()).sendGlobalBarrierReached();
+    verify(procspy).sendGlobalBarrierComplete();
+  }
+
+  @Test(timeout = 1000)
+  public void testBarrieredErrorPropagation() throws Exception {
+    List<String> members =  new ArrayList<String>();
+    members.add("member");
+    LatchedProcedure proc = new LatchedProcedure(coord, new ForeignExceptionDispatcher(), 100,
+        Integer.MAX_VALUE, "op", null, members);
+    final LatchedProcedure procspy = spy(proc);
+
+    // start the barrier procedure
+    Thread t = new Thread() {
+      public void run() {
+        procspy.call();
+      }
+    };
+    t.start();
+
+    // now test that we can put an error in before the commit phase runs
+    procspy.startedAcquireBarrier.await();
+    ForeignException cause = new ForeignException("SRC", "External Exception");
+    procspy.receive(cause);
+    procspy.barrierAcquiredByMember(members.get(0));
+    t.join();
+
+    // verify state of all the object
+    verify(procspy).sendGlobalBarrierStart();
+    verify(procspy).sendGlobalBarrierComplete();
+    verify(procspy, never()).sendGlobalBarrierReached();
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
new file mode 100644
index 000000000000..6bd46cce1c72
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.InOrder;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Test Procedure coordinator operation.
+ * <p>
+ * This only works correctly when we do <i>class level parallelization</i> of tests. If we do method
+ * level serialization this class will likely throw all kinds of errors.
+ */
+@Category(SmallTests.class)
+public class TestProcedureCoordinator {
+  // general test constants
+  private static final long WAKE_FREQUENCY = 1000;
+  private static final long TIMEOUT = 100000;
+  private static final long POOL_KEEP_ALIVE = 1;
+  private static final String nodeName = "node";
+  private static final String procName = "some op";
+  private static final byte[] procData = new byte[0];
+  private static final List<String> expected = Lists.newArrayList("remote1", "remote2");
+
+  // setup the mocks
+  private final ProcedureCoordinatorRpcs controller = mock(ProcedureCoordinatorRpcs.class);
+  private final Procedure task = mock(Procedure.class);
+  private final ForeignExceptionDispatcher monitor = mock(ForeignExceptionDispatcher.class);
+
+  // handle to the coordinator for each test
+  private ProcedureCoordinator coordinator;
+
+  @After
+  public void resetTest() throws IOException {
+    // reset all the mocks used for the tests
+    reset(controller, task, monitor);
+    // close the open coordinator, if it was used
+    if (coordinator != null) coordinator.close();
+  }
+
+  private ProcedureCoordinator buildNewCoordinator() {
+    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(nodeName, POOL_KEEP_ALIVE, 1, WAKE_FREQUENCY);
+    return spy(new ProcedureCoordinator(controller, pool));
+  }
+
+  /**
+   * Currently we can only handle one procedure at a time.  This makes sure we handle that and
+   * reject submitting more.
+   */
+  @Test
+  public void testThreadPoolSize() throws Exception {
+    ProcedureCoordinator coordinator = buildNewCoordinator();
+    Procedure proc = new Procedure(coordinator,  monitor,
+        WAKE_FREQUENCY, TIMEOUT, procName, procData, expected);
+    Procedure procSpy = spy(proc);
+
+    Procedure proc2 = new Procedure(coordinator,  monitor,
+        WAKE_FREQUENCY, TIMEOUT, procName +"2", procData, expected);
+    Procedure procSpy2 = spy(proc2);
+    when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
+    .thenReturn(procSpy, procSpy2);
+
+    coordinator.startProcedure(procSpy.getErrorMonitor(), procName, procData, expected);
+    // null here means second procedure failed to start.
+    assertNull("Coordinator successfully ran two tasks at once with a single thread pool.",
+      coordinator.startProcedure(proc2.getErrorMonitor(), "another op", procData, expected));
+  }
+
+  /**
+   * Check handling a connection failure correctly if we get it during the acquiring phase
+   */
+  @Test(timeout = 5000)
+  public void testUnreachableControllerDuringPrepare() throws Exception {
+    coordinator = buildNewCoordinator();
+    // setup the proc
+    List<String> expected = Arrays.asList("cohort");
+    Procedure proc = new Procedure(coordinator, WAKE_FREQUENCY,
+        TIMEOUT, procName, procData, expected);
+    final Procedure procSpy = spy(proc);
+
+    when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
+        .thenReturn(procSpy);
+
+    // use the passed controller responses
+    IOException cause = new IOException("Failed to reach comms during acquire");
+    doThrow(cause).when(controller)
+        .sendGlobalBarrierAcquire(eq(procSpy), eq(procData), anyListOf(String.class));
+
+    // run the operation
+    proc = coordinator.startProcedure(proc.getErrorMonitor(), procName, procData, expected);
+    // and wait for it to finish
+    proc.waitForCompleted();
+    verify(procSpy, atLeastOnce()).receive(any(ForeignException.class));
+    verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
+    verify(controller, times(1)).sendGlobalBarrierAcquire(procSpy, procData, expected);
+    verify(controller, never()).sendGlobalBarrierReached(any(Procedure.class),
+        anyListOf(String.class));
+  }
+
+  /**
+   * Check handling a connection failure correctly if we get it during the barrier phase
+   */
+  @Test(timeout = 5000)
+  public void testUnreachableControllerDuringCommit() throws Exception {
+    coordinator = buildNewCoordinator();
+
+    // setup the task and spy on it
+    List<String> expected = Arrays.asList("cohort");
+    final Procedure spy = spy(new Procedure(coordinator,
+        WAKE_FREQUENCY, TIMEOUT, procName, procData, expected));
+
+    when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
+    .thenReturn(spy);
+
+    // use the passed controller responses
+    IOException cause = new IOException("Failed to reach controller during prepare");
+    doAnswer(new AcquireBarrierAnswer(procName, new String[] { "cohort" }))
+        .when(controller).sendGlobalBarrierAcquire(eq(spy), eq(procData), anyListOf(String.class));
+    doThrow(cause).when(controller).sendGlobalBarrierReached(eq(spy), anyListOf(String.class));
+
+    // run the operation
+    Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
+    // and wait for it to finish
+    task.waitForCompleted();
+    verify(spy, atLeastOnce()).receive(any(ForeignException.class));
+    verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
+    verify(controller, times(1)).sendGlobalBarrierAcquire(eq(spy),
+        eq(procData), anyListOf(String.class));
+    verify(controller, times(1)).sendGlobalBarrierReached(any(Procedure.class),
+        anyListOf(String.class));
+  }
+
+  @Test(timeout = 1000)
+  public void testNoCohort() throws Exception {
+    runSimpleProcedure();
+  }
+
+  @Test(timeout = 1000)
+  public void testSingleCohortOrchestration() throws Exception {
+    runSimpleProcedure("one");
+  }
+
+  @Test(timeout = 1000)
+  public void testMultipleCohortOrchestration() throws Exception {
+    runSimpleProcedure("one", "two", "three", "four");
+  }
+
+  public void runSimpleProcedure(String... members) throws Exception {
+    coordinator = buildNewCoordinator();
+    Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY,
+        TIMEOUT, procName, procData, Arrays.asList(members));
+    final Procedure spy = spy(task);
+    runCoordinatedProcedure(spy, members);
+  }
+
+  /**
+   * Test that if nodes join the barrier early we still correctly handle the progress
+   */
+  @Test(timeout = 1000)
+  public void testEarlyJoiningBarrier() throws Exception {
+    final String[] cohort = new String[] { "one", "two", "three", "four" };
+    coordinator = buildNewCoordinator();
+    final ProcedureCoordinator ref = coordinator;
+    Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY,
+        TIMEOUT, procName, procData, Arrays.asList(cohort));
+    final Procedure spy = spy(task);
+
+    AcquireBarrierAnswer prepare = new AcquireBarrierAnswer(procName, cohort) {
+      public void doWork() {
+        // then do some fun where we commit before all nodes have prepared
+        // "one" commits before anyone else is done
+        ref.memberAcquiredBarrier(this.opName, this.cohort[0]);
+        ref.memberFinishedBarrier(this.opName, this.cohort[0]);
+        // but "two" takes a while
+        ref.memberAcquiredBarrier(this.opName, this.cohort[1]);
+        // "three"jumps ahead
+        ref.memberAcquiredBarrier(this.opName, this.cohort[2]);
+        ref.memberFinishedBarrier(this.opName, this.cohort[2]);
+        // and "four" takes a while
+        ref.memberAcquiredBarrier(this.opName, this.cohort[3]);
+      }
+    };
+
+    BarrierAnswer commit = new BarrierAnswer(procName, cohort) {
+      @Override
+      public void doWork() {
+        ref.memberFinishedBarrier(opName, this.cohort[1]);
+        ref.memberFinishedBarrier(opName, this.cohort[3]);
+      }
+    };
+    runCoordinatedOperation(spy, prepare, commit, cohort);
+  }
+
+  /**
+   * Just run a procedure with the standard name and data, with not special task for the mock
+   * coordinator (it works just like a regular coordinator). For custom behavior see
+   * {@link #runCoordinatedOperation(Procedure, AcquireBarrierAnswer, BarrierAnswer, String[])}
+   * .
+   * @param spy Spy on a real {@link Procedure}
+   * @param cohort expected cohort members
+   * @throws Exception on failure
+   */
+  public void runCoordinatedProcedure(Procedure spy, String... cohort) throws Exception {
+    runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort),
+      new BarrierAnswer(procName, cohort), cohort);
+  }
+
+  public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepare,
+      String... cohort) throws Exception {
+    runCoordinatedOperation(spy, prepare, new BarrierAnswer(procName, cohort), cohort);
+  }
+
+  public void runCoordinatedOperation(Procedure spy, BarrierAnswer commit,
+      String... cohort) throws Exception {
+    runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort), commit, cohort);
+  }
+
+  public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepareOperation,
+      BarrierAnswer commitOperation, String... cohort) throws Exception {
+    List<String> expected = Arrays.asList(cohort);
+    when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
+      .thenReturn(spy);
+
+    // use the passed controller responses
+    doAnswer(prepareOperation).when(controller).sendGlobalBarrierAcquire(spy, procData, expected);
+    doAnswer(commitOperation).when(controller)
+        .sendGlobalBarrierReached(eq(spy), anyListOf(String.class));
+
+    // run the operation
+    Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
+    // and wait for it to finish
+    task.waitForCompleted();
+
+    // make sure we mocked correctly
+    prepareOperation.ensureRan();
+    // we never got an exception
+    InOrder inorder = inOrder(spy, controller);
+    inorder.verify(spy).sendGlobalBarrierStart();
+    inorder.verify(controller).sendGlobalBarrierAcquire(task, procData, expected);
+    inorder.verify(spy).sendGlobalBarrierReached();
+    inorder.verify(controller).sendGlobalBarrierReached(eq(task), anyListOf(String.class));
+  }
+
+  private abstract class OperationAnswer implements Answer<Void> {
+    private boolean ran = false;
+
+    public void ensureRan() {
+      assertTrue("Prepare mocking didn't actually run!", ran);
+    }
+
+    @Override
+    public final Void answer(InvocationOnMock invocation) throws Throwable {
+      this.ran = true;
+      doWork();
+      return null;
+    }
+
+    protected abstract void doWork() throws Throwable;
+  }
+
+  /**
+   * Just tell the current coordinator that each of the nodes has prepared
+   */
+  private class AcquireBarrierAnswer extends OperationAnswer {
+    protected final String[] cohort;
+    protected final String opName;
+
+    public AcquireBarrierAnswer(String opName, String... cohort) {
+      this.cohort = cohort;
+      this.opName = opName;
+    }
+
+    @Override
+    public void doWork() {
+      if (cohort == null) return;
+      for (String member : cohort) {
+        TestProcedureCoordinator.this.coordinator.memberAcquiredBarrier(opName, member);
+      }
+    }
+  }
+
+  /**
+   * Just tell the current coordinator that each of the nodes has committed
+   */
+  private class BarrierAnswer extends OperationAnswer {
+    protected final String[] cohort;
+    protected final String opName;
+
+    public BarrierAnswer(String opName, String... cohort) {
+      this.cohort = cohort;
+      this.opName = opName;
+    }
+
+    @Override
+    public void doWork() {
+      if (cohort == null) return;
+      for (String member : cohort) {
+        TestProcedureCoordinator.this.coordinator.memberFinishedBarrier(opName, member);
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
new file mode 100644
index 000000000000..b4c1b27f3510
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
@@ -0,0 +1,444 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.TimeoutException;
+import org.apache.hadoop.hbase.procedure.Subprocedure.SubprocedureImpl;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test the procedure member, and it's error handling mechanisms.
+ */
+@Category(SmallTests.class)
+public class TestProcedureMember {
+  private static final long WAKE_FREQUENCY = 100;
+  private static final long TIMEOUT = 100000;
+  private static final long POOL_KEEP_ALIVE = 1;
+
+  private final String op = "some op";
+  private final byte[] data = new byte[0];
+  private final ForeignExceptionDispatcher mockListener = Mockito
+      .spy(new ForeignExceptionDispatcher());
+  private final SubprocedureFactory mockBuilder = mock(SubprocedureFactory.class);
+  private final ProcedureMemberRpcs mockMemberComms = Mockito
+      .mock(ProcedureMemberRpcs.class);
+  private ProcedureMember member;
+  private ForeignExceptionDispatcher dispatcher;
+  Subprocedure spySub;
+
+  /**
+   * Reset all the mock objects
+   */
+  @After
+  public void resetTest() {
+    reset(mockListener, mockBuilder, mockMemberComms);
+    if (member != null)
+      try {
+        member.close();
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+  }
+
+  /**
+   * Build a member using the class level mocks
+   * @return member to use for tests
+   */
+  private ProcedureMember buildCohortMember() {
+    String name = "node";
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(WAKE_FREQUENCY, POOL_KEEP_ALIVE, 1, name);
+    return new ProcedureMember(mockMemberComms, pool, mockBuilder);
+  }
+
+  /**
+   * Setup a procedure member that returns the spied-upon {@link Subprocedure}.
+   */
+  private void buildCohortMemberPair() throws IOException {
+    dispatcher = new ForeignExceptionDispatcher();
+    String name = "node";
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(WAKE_FREQUENCY, POOL_KEEP_ALIVE, 1, name);
+    member = new ProcedureMember(mockMemberComms, pool, mockBuilder);
+    when(mockMemberComms.getMemberName()).thenReturn("membername"); // needed for generating exception
+    Subprocedure subproc = new EmptySubprocedure(member, dispatcher);
+    spySub = spy(subproc);
+    when(mockBuilder.buildSubprocedure(op, data)).thenReturn(spySub);
+    addCommitAnswer();
+  }
+
+
+  /**
+   * Add a 'in barrier phase' response to the mock controller when it gets a acquired notification
+   */
+  private void addCommitAnswer() throws IOException {
+    doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        member.receivedReachedGlobalBarrier(op);
+        return null;
+      }
+    }).when(mockMemberComms).sendMemberAcquired(any(Subprocedure.class));
+  }
+
+  /**
+   * Test the normal sub procedure execution case.
+   */
+  @Test(timeout = 500)
+  public void testSimpleRun() throws Exception {
+    member = buildCohortMember();
+    EmptySubprocedure subproc = new EmptySubprocedure(member, mockListener);
+    EmptySubprocedure spy = spy(subproc);
+    when(mockBuilder.buildSubprocedure(op, data)).thenReturn(spy);
+
+    // when we get a prepare, then start the commit phase
+    addCommitAnswer();
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc1 = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc1);
+    // and wait for it to finish
+    subproc.waitForLocallyCompleted();
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spy);
+    order.verify(spy).acquireBarrier();
+    order.verify(mockMemberComms).sendMemberAcquired(eq(spy));
+    order.verify(spy).insideBarrier();
+    order.verify(mockMemberComms).sendMemberCompleted(eq(spy));
+    order.verify(mockMemberComms, never()).sendMemberAborted(eq(spy),
+        any(ForeignException.class));
+  }
+
+  /**
+   * Make sure we call cleanup etc, when we have an exception during
+   * {@link Subprocedure#acquireBarrier()}.
+   */
+  @Test(timeout = 1000)
+  public void testMemberPrepareException() throws Exception {
+    buildCohortMemberPair();
+
+    // mock an exception on Subprocedure's prepare
+    doAnswer(
+        new Answer<Void>() {
+          @Override
+          public Void answer(InvocationOnMock invocation) throws Throwable {
+            throw new IOException("Forced IOException in member acquireBarrier");
+          }
+        }).when(spySub).acquireBarrier();
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc);
+    // if the operation doesn't die properly, then this will timeout
+    member.closeAndWait(TIMEOUT);
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spySub);
+    order.verify(spySub).acquireBarrier();
+    // Later phases not run
+    order.verify(mockMemberComms, never()).sendMemberAcquired(eq(spySub));
+    order.verify(spySub, never()).insideBarrier();
+    order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub));
+    // error recovery path exercised
+    order.verify(spySub).cancel(anyString(), any(Exception.class));
+    order.verify(spySub).cleanup(any(Exception.class));
+  }
+
+  /**
+   * Make sure we call cleanup etc, when we have an exception during prepare.
+   */
+  @Test(timeout = 1000)
+  public void testSendMemberAcquiredCommsFailure() throws Exception {
+    buildCohortMemberPair();
+
+    // mock an exception on Subprocedure's prepare
+    doAnswer(
+        new Answer<Void>() {
+          @Override
+          public Void answer(InvocationOnMock invocation) throws Throwable {
+            throw new IOException("Forced IOException in memeber prepare");
+          }
+        }).when(mockMemberComms).sendMemberAcquired(any(Subprocedure.class));
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc);
+    // if the operation doesn't die properly, then this will timeout
+    member.closeAndWait(TIMEOUT);
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spySub);
+    order.verify(spySub).acquireBarrier();
+    order.verify(mockMemberComms).sendMemberAcquired(eq(spySub));
+
+    // Later phases not run
+    order.verify(spySub, never()).insideBarrier();
+    order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub));
+    // error recovery path exercised
+    order.verify(spySub).cancel(anyString(), any(Exception.class));
+    order.verify(spySub).cleanup(any(Exception.class));
+  }
+
+  /**
+   * Fail correctly if coordinator aborts the procedure.  The subprocedure will not interrupt a
+   * running {@link Subprocedure#prepare} -- prepare needs to finish first, and the the abort
+   * is checked.  Thus, the {@link Subprocedure#prepare} should succeed but later get rolled back
+   * via {@link Subprocedure#cleanup}.
+   */
+  @Test(timeout = 1000)
+  public void testCoordinatorAbort() throws Exception {
+    buildCohortMemberPair();
+
+    // mock that another node timed out or failed to prepare
+    final TimeoutException oate = new TimeoutException("bogus timeout", 1,2,0);
+    doAnswer(
+        new Answer<Void>() {
+          @Override
+          public Void answer(InvocationOnMock invocation) throws Throwable {
+            // inject a remote error (this would have come from an external thread)
+            spySub.cancel("bogus message", oate);
+            // sleep the wake frequency since that is what we promised
+            Thread.sleep(WAKE_FREQUENCY);
+            return null;
+          }
+        }).when(spySub).waitForReachedGlobalBarrier();
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc);
+    // if the operation doesn't die properly, then this will timeout
+    member.closeAndWait(TIMEOUT);
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spySub);
+    order.verify(spySub).acquireBarrier();
+    order.verify(mockMemberComms).sendMemberAcquired(eq(spySub));
+    // Later phases not run
+    order.verify(spySub, never()).insideBarrier();
+    order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub));
+    // error recovery path exercised
+    order.verify(spySub).cancel(anyString(), any(Exception.class));
+    order.verify(spySub).cleanup(any(Exception.class));
+  }
+
+  /**
+   * Handle failures if a member's commit phase fails.
+   *
+   * NOTE: This is the core difference that makes this different from traditional 2PC.  In true
+   * 2PC the transaction is committed just before the coordinator sends commit messages to the
+   * member.  Members are then responsible for reading its TX log.  This implementation actually
+   * rolls back, and thus breaks the normal TX guarantees.
+  */
+  @Test(timeout = 1000)
+  public void testMemberCommitException() throws Exception {
+    buildCohortMemberPair();
+
+    // mock an exception on Subprocedure's prepare
+    doAnswer(
+        new Answer<Void>() {
+          @Override
+          public Void answer(InvocationOnMock invocation) throws Throwable {
+            throw new IOException("Forced IOException in memeber prepare");
+          }
+        }).when(spySub).insideBarrier();
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc);
+    // if the operation doesn't die properly, then this will timeout
+    member.closeAndWait(TIMEOUT);
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spySub);
+    order.verify(spySub).acquireBarrier();
+    order.verify(mockMemberComms).sendMemberAcquired(eq(spySub));
+    order.verify(spySub).insideBarrier();
+
+    // Later phases not run
+    order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub));
+    // error recovery path exercised
+    order.verify(spySub).cancel(anyString(), any(Exception.class));
+    order.verify(spySub).cleanup(any(Exception.class));
+  }
+
+  /**
+   * Handle Failures if a member's commit phase succeeds but notification to coordinator fails
+   *
+   * NOTE: This is the core difference that makes this different from traditional 2PC.  In true
+   * 2PC the transaction is committed just before the coordinator sends commit messages to the
+   * member.  Members are then responsible for reading its TX log.  This implementation actually
+   * rolls back, and thus breaks the normal TX guarantees.
+  */
+  @Test(timeout = 1000)
+  public void testMemberCommitCommsFailure() throws Exception {
+    buildCohortMemberPair();
+    final TimeoutException oate = new TimeoutException("bogus timeout",1,2,0);
+    doAnswer(
+        new Answer<Void>() {
+          @Override
+          public Void answer(InvocationOnMock invocation) throws Throwable {
+            // inject a remote error (this would have come from an external thread)
+            spySub.cancel("commit comms fail", oate);
+            // sleep the wake frequency since that is what we promised
+            Thread.sleep(WAKE_FREQUENCY);
+            return null;
+          }
+        }).when(mockMemberComms).sendMemberCompleted(any(Subprocedure.class));
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc);
+    // if the operation doesn't die properly, then this will timeout
+    member.closeAndWait(TIMEOUT);
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spySub);
+    order.verify(spySub).acquireBarrier();
+    order.verify(mockMemberComms).sendMemberAcquired(eq(spySub));
+    order.verify(spySub).insideBarrier();
+    order.verify(mockMemberComms).sendMemberCompleted(eq(spySub));
+    // error recovery path exercised
+    order.verify(spySub).cancel(anyString(), any(Exception.class));
+    order.verify(spySub).cleanup(any(Exception.class));
+  }
+
+  /**
+   * Fail correctly on getting an external error while waiting for the prepared latch
+   * @throws Exception on failure
+   */
+  @Test(timeout = 1000)
+  public void testPropagateConnectionErrorBackToManager() throws Exception {
+    // setup the operation
+    member = buildCohortMember();
+    ProcedureMember memberSpy = spy(member);
+
+    // setup the commit and the spy
+    final ForeignExceptionDispatcher dispatcher = new ForeignExceptionDispatcher();
+    ForeignExceptionDispatcher dispSpy = spy(dispatcher);
+    Subprocedure commit = new EmptySubprocedure(member, dispatcher);
+    Subprocedure spy = spy(commit);
+    when(mockBuilder.buildSubprocedure(op, data)).thenReturn(spy);
+
+    // fail during the prepare phase
+    doThrow(new ForeignException("SRC", "prepare exception")).when(spy).acquireBarrier();
+    // and throw a connection error when we try to tell the controller about it
+    doThrow(new IOException("Controller is down!")).when(mockMemberComms)
+        .sendMemberAborted(eq(spy), any(ForeignException.class));
+
+
+    // run the operation
+    // build a new operation
+    Subprocedure subproc = memberSpy.createSubprocedure(op, data);
+    memberSpy.submitSubprocedure(subproc);
+    // if the operation doesn't die properly, then this will timeout
+    memberSpy.closeAndWait(TIMEOUT);
+
+    // make sure everything ran in order
+    InOrder order = inOrder(mockMemberComms, spy, dispSpy);
+    // make sure we acquire.
+    order.verify(spy).acquireBarrier();
+    order.verify(mockMemberComms, never()).sendMemberAcquired(spy);
+
+    // TODO Need to do another refactor to get this to propagate to the coordinator.
+    // make sure we pass a remote exception back the controller
+//    order.verify(mockMemberComms).sendMemberAborted(eq(spy),
+//      any(ExternalException.class));
+//    order.verify(dispSpy).receiveError(anyString(),
+//        any(ExternalException.class), any());
+  }
+
+  /**
+   * Test that the cohort member correctly doesn't attempt to start a task when the builder cannot
+   * correctly build a new task for the requested operation
+   * @throws Exception on failure
+   */
+  @Test
+  public void testNoTaskToBeRunFromRequest() throws Exception {
+    ThreadPoolExecutor pool = mock(ThreadPoolExecutor.class);
+    when(mockBuilder.buildSubprocedure(op, data)).thenReturn(null)
+      .thenThrow(new IllegalStateException("Wrong state!"), new IllegalArgumentException("can't understand the args"));
+    member = new ProcedureMember(mockMemberComms, pool, mockBuilder);
+    // builder returns null
+    // build a new operation
+    Subprocedure subproc = member.createSubprocedure(op, data);
+    member.submitSubprocedure(subproc);
+    // throws an illegal state exception
+    try {
+      // build a new operation
+      Subprocedure subproc2 = member.createSubprocedure(op, data);
+      member.submitSubprocedure(subproc2);
+    } catch (IllegalStateException ise) {
+    }
+    // throws an illegal argument exception
+    try {
+      // build a new operation
+      Subprocedure subproc3 = member.createSubprocedure(op, data);
+      member.submitSubprocedure(subproc3);
+    } catch (IllegalArgumentException iae) {
+    }
+
+    // no request should reach the pool
+    verifyZeroInteractions(pool);
+    // get two abort requests
+    // TODO Need to do another refactor to get this to propagate to the coordinator.
+    // verify(mockMemberComms, times(2)).sendMemberAborted(any(Subprocedure.class), any(ExternalException.class));
+  }
+
+  /**
+   * Helper {@link Procedure} who's phase for each step is just empty
+   */
+  public class EmptySubprocedure extends SubprocedureImpl {
+    public EmptySubprocedure(ProcedureMember member, ForeignExceptionDispatcher dispatcher) {
+      super( member, op, dispatcher,
+      // TODO 1000000 is an arbitrary number that I picked.
+          WAKE_FREQUENCY, TIMEOUT);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
new file mode 100644
index 000000000000..a0fc5491fda3
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
@@ -0,0 +1,405 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.TimeoutException;
+import org.apache.hadoop.hbase.procedure.Subprocedure.SubprocedureImpl;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.mockito.internal.matchers.ArrayEquals;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.mockito.verification.VerificationMode;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Cluster-wide testing of a distributed three-phase commit using a 'real' zookeeper cluster
+ */
+@Category(MediumTests.class)
+public class TestZKProcedure {
+
+  private static final Log LOG = LogFactory.getLog(TestZKProcedure.class);
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final String COORDINATOR_NODE_NAME = "coordinator";
+  private static final long KEEP_ALIVE = 100; // seconds
+  private static final int POOL_SIZE = 1;
+  private static final long TIMEOUT = 10000; // when debugging make this larger for debugging
+  private static final long WAKE_FREQUENCY = 500;
+  private static final String opName = "op";
+  private static final byte[] data = new byte[] { 1, 2 }; // TODO what is this used for?
+  private static final VerificationMode once = Mockito.times(1);
+
+  @BeforeClass
+  public static void setupTest() throws Exception {
+    UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    UTIL.shutdownMiniZKCluster();
+  }
+
+  private static ZooKeeperWatcher newZooKeeperWatcher() throws IOException {
+    return new ZooKeeperWatcher(UTIL.getConfiguration(), "testing utility", new Abortable() {
+      @Override
+      public void abort(String why, Throwable e) {
+        throw new RuntimeException(
+            "Unexpected abort in distributed three phase commit test:" + why, e);
+      }
+
+      @Override
+      public boolean isAborted() {
+        return false;
+      }
+    });
+  }
+
+  @Test
+  public void testEmptyMemberSet() throws Exception {
+    runCommit();
+  }
+
+  @Test
+  public void testSingleMember() throws Exception {
+    runCommit("one");
+  }
+
+  @Test
+  public void testMultipleMembers() throws Exception {
+    runCommit("one", "two", "three", "four" );
+  }
+
+  private void runCommit(String... members) throws Exception {
+    // make sure we just have an empty list
+    if (members == null) {
+      members = new String[0];
+    }
+    List<String> expected = Arrays.asList(members);
+
+    // setup the constants
+    ZooKeeperWatcher coordZkw = newZooKeeperWatcher();
+    String opDescription = "coordination test - " + members.length + " cohort members";
+
+    // start running the controller
+    ZKProcedureCoordinatorRpcs coordinatorComms = new ZKProcedureCoordinatorRpcs(
+        coordZkw, opDescription, COORDINATOR_NODE_NAME);
+    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, KEEP_ALIVE, POOL_SIZE, WAKE_FREQUENCY);
+    ProcedureCoordinator coordinator = new ProcedureCoordinator(coordinatorComms, pool) {
+      @Override
+      public Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
+          List<String> expectedMembers) {
+        return Mockito.spy(super.createProcedure(fed, procName, procArgs, expectedMembers));
+      }
+    };
+
+    // build and start members
+    // NOTE: There is a single subprocedure builder for all members here.
+    SubprocedureFactory subprocFactory = Mockito.mock(SubprocedureFactory.class);
+    List<Pair<ProcedureMember, ZKProcedureMemberRpcs>> procMembers = new ArrayList<Pair<ProcedureMember, ZKProcedureMemberRpcs>>(
+        members.length);
+    // start each member
+    for (String member : members) {
+      ZooKeeperWatcher watcher = newZooKeeperWatcher();
+      ZKProcedureMemberRpcs comms = new ZKProcedureMemberRpcs(watcher, opDescription, member);
+      ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member);
+      ProcedureMember procMember = new ProcedureMember(comms, pool2, subprocFactory);
+      procMembers.add(new Pair<ProcedureMember, ZKProcedureMemberRpcs>(procMember, comms));
+      comms.start(procMember);
+    }
+
+    // setup mock member subprocedures
+    final List<Subprocedure> subprocs = new ArrayList<Subprocedure>();
+    for (int i = 0; i < procMembers.size(); i++) {
+      ForeignExceptionDispatcher cohortMonitor = new ForeignExceptionDispatcher();
+      Subprocedure commit = Mockito
+      .spy(new SubprocedureImpl(procMembers.get(i).getFirst(), opName, cohortMonitor,
+          WAKE_FREQUENCY, TIMEOUT));
+      subprocs.add(commit);
+    }
+
+    // link subprocedure to buildNewOperation invocation.
+    final AtomicInteger i = new AtomicInteger(0); // NOTE: would be racy if not an AtomicInteger
+    Mockito.when(subprocFactory.buildSubprocedure(Mockito.eq(opName),
+        (byte[]) Mockito.argThat(new ArrayEquals(data)))).thenAnswer(
+      new Answer<Subprocedure>() {
+        @Override
+        public Subprocedure answer(InvocationOnMock invocation) throws Throwable {
+          int index = i.getAndIncrement();
+          LOG.debug("Task size:" + subprocs.size() + ", getting:" + index);
+          Subprocedure commit = subprocs.get(index);
+          return commit;
+        }
+      });
+
+    // setup spying on the coordinator
+//    Procedure proc = Mockito.spy(procBuilder.createProcedure(coordinator, opName, data, expected));
+//    Mockito.when(procBuilder.build(coordinator, opName, data, expected)).thenReturn(proc);
+
+    // start running the operation
+    Procedure task = coordinator.startProcedure(new ForeignExceptionDispatcher(), opName, data, expected);
+//    assertEquals("Didn't mock coordinator task", proc, task);
+
+    // verify all things ran as expected
+//    waitAndVerifyProc(proc, once, once, never(), once, false);
+    waitAndVerifyProc(task, once, once, never(), once, false);
+    verifyCohortSuccessful(expected, subprocFactory, subprocs, once, once, never(), once, false);
+
+    // close all the things
+    closeAll(coordinator, coordinatorComms, procMembers);
+  }
+
+  /**
+   * Test a distributed commit with multiple cohort members, where one of the cohort members has a
+   * timeout exception during the prepare stage.
+   */
+  @Test
+  public void testMultiCohortWithMemberTimeoutDuringPrepare() throws Exception {
+    String opDescription = "error injection coordination";
+    String[] cohortMembers = new String[] { "one", "two", "three" };
+    List<String> expected = Lists.newArrayList(cohortMembers);
+    // error constants
+    final int memberErrorIndex = 2;
+    final CountDownLatch coordinatorReceivedErrorLatch = new CountDownLatch(1);
+
+    // start running the coordinator and its controller
+    ZooKeeperWatcher coordinatorWatcher = newZooKeeperWatcher();
+    ZKProcedureCoordinatorRpcs coordinatorController = new ZKProcedureCoordinatorRpcs(
+        coordinatorWatcher, opDescription, COORDINATOR_NODE_NAME);
+    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, KEEP_ALIVE, POOL_SIZE, WAKE_FREQUENCY);
+    ProcedureCoordinator coordinator = spy(new ProcedureCoordinator(coordinatorController, pool));
+
+    // start a member for each node
+    SubprocedureFactory subprocFactory = Mockito.mock(SubprocedureFactory.class);
+    List<Pair<ProcedureMember, ZKProcedureMemberRpcs>> members = new ArrayList<Pair<ProcedureMember, ZKProcedureMemberRpcs>>(
+        expected.size());
+    for (String member : expected) {
+      ZooKeeperWatcher watcher = newZooKeeperWatcher();
+      ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(watcher, opDescription, member);
+      ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member);
+      ProcedureMember mem = new ProcedureMember(controller, pool2, subprocFactory);
+      members.add(new Pair<ProcedureMember, ZKProcedureMemberRpcs>(mem, controller));
+      controller.start(mem);
+    }
+
+    // setup mock subprocedures
+    final List<Subprocedure> cohortTasks = new ArrayList<Subprocedure>();
+    final int[] elem = new int[1];
+    for (int i = 0; i < members.size(); i++) {
+      ForeignExceptionDispatcher cohortMonitor = new ForeignExceptionDispatcher();
+      ProcedureMember comms = members.get(i).getFirst();
+      Subprocedure commit = Mockito
+      .spy(new SubprocedureImpl(comms, opName, cohortMonitor, WAKE_FREQUENCY, TIMEOUT));
+      // This nasty bit has one of the impls throw a TimeoutException
+      Mockito.doAnswer(new Answer<Void>() {
+        @Override
+        public Void answer(InvocationOnMock invocation) throws Throwable {
+          int index = elem[0];
+          if (index == memberErrorIndex) {
+            LOG.debug("Sending error to coordinator");
+            ForeignException remoteCause = new ForeignException("TIMER",
+                new TimeoutException("subprocTimeout" , 1, 2, 0));
+            Subprocedure r = ((Subprocedure) invocation.getMock());
+            LOG.error("Remote commit failure, not propagating error:" + remoteCause);
+            r.monitor.receive(remoteCause);
+            // don't complete the error phase until the coordinator has gotten the error
+            // notification (which ensures that we never progress past prepare)
+            try {
+              Procedure.waitForLatch(coordinatorReceivedErrorLatch, new ForeignExceptionDispatcher(),
+                  WAKE_FREQUENCY, "coordinator received error");
+            } catch (InterruptedException e) {
+              LOG.debug("Wait for latch interrupted, done:" + (coordinatorReceivedErrorLatch.getCount() == 0));
+              // reset the interrupt status on the thread
+              Thread.currentThread().interrupt();
+            }
+          }
+          elem[0] = ++index;
+          return null;
+        }
+      }).when(commit).acquireBarrier();
+      cohortTasks.add(commit);
+    }
+
+    // pass out a task per member
+    final int[] i = new int[] { 0 };
+    Mockito.when(
+      subprocFactory.buildSubprocedure(Mockito.eq(opName),
+        (byte[]) Mockito.argThat(new ArrayEquals(data)))).thenAnswer(
+      new Answer<Subprocedure>() {
+        @Override
+        public Subprocedure answer(InvocationOnMock invocation) throws Throwable {
+          int index = i[0];
+          Subprocedure commit = cohortTasks.get(index);
+          index++;
+          i[0] = index;
+          return commit;
+        }
+      });
+
+    // setup spying on the coordinator
+    ForeignExceptionDispatcher coordinatorTaskErrorMonitor = Mockito
+        .spy(new ForeignExceptionDispatcher());
+    Procedure coordinatorTask = Mockito.spy(new Procedure(coordinator,
+        coordinatorTaskErrorMonitor, WAKE_FREQUENCY, TIMEOUT,
+        opName, data, expected));
+    when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(opName), eq(data), anyListOf(String.class)))
+      .thenReturn(coordinatorTask);
+    // count down the error latch when we get the remote error
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        // pass on the error to the master
+        invocation.callRealMethod();
+        // then count down the got error latch
+        coordinatorReceivedErrorLatch.countDown();
+        return null;
+      }
+    }).when(coordinatorTask).receive(Mockito.any(ForeignException.class));
+
+    // ----------------------------
+    // start running the operation
+    // ----------------------------
+
+    Procedure task = coordinator.startProcedure(coordinatorTaskErrorMonitor, opName, data, expected);
+    assertEquals("Didn't mock coordinator task", coordinatorTask, task);
+
+    // wait for the task to complete
+    try {
+      task.waitForCompleted();
+    } catch (ForeignException fe) {
+      // this may get caught or may not 
+    }
+
+    // -------------
+    // verification
+    // -------------
+    waitAndVerifyProc(coordinatorTask, once, never(), once, once, true);
+    verifyCohortSuccessful(expected, subprocFactory, cohortTasks, once, never(), once,
+      once, true);
+
+    // close all the open things
+    closeAll(coordinator, coordinatorController, members);
+  }
+
+  /**
+   * Wait for the coordinator task to complete, and verify all the mocks
+   * @param task to wait on
+   * @throws Exception on unexpected failure
+   */
+  private void waitAndVerifyProc(Procedure proc, VerificationMode prepare,
+      VerificationMode commit, VerificationMode cleanup, VerificationMode finish, boolean opHasError)
+      throws Exception {
+    boolean caughtError = false;
+    try {
+      proc.waitForCompleted();
+    } catch (ForeignException fe) {
+      caughtError = true;
+    }
+    // make sure that the task called all the expected phases
+    Mockito.verify(proc, prepare).sendGlobalBarrierStart();
+    Mockito.verify(proc, commit).sendGlobalBarrierReached();
+    Mockito.verify(proc, finish).sendGlobalBarrierComplete();
+    assertEquals("Operation error state was unexpected", opHasError, proc.getErrorMonitor()
+        .hasException());
+    assertEquals("Operation error state was unexpected", opHasError, caughtError);
+
+  }
+
+  /**
+   * Wait for the coordinator task to complete, and verify all the mocks
+   * @param task to wait on
+   * @throws Exception on unexpected failure
+   */
+  private void waitAndVerifySubproc(Subprocedure op, VerificationMode prepare,
+      VerificationMode commit, VerificationMode cleanup, VerificationMode finish, boolean opHasError)
+      throws Exception {
+    boolean caughtError = false;
+    try {
+      op.waitForLocallyCompleted();
+    } catch (ForeignException fe) {
+      caughtError = true;
+    }
+    // make sure that the task called all the expected phases
+    Mockito.verify(op, prepare).acquireBarrier();
+    Mockito.verify(op, commit).insideBarrier();
+    // We cannot guarantee that cleanup has run so we don't check it.
+
+    assertEquals("Operation error state was unexpected", opHasError, op.getErrorCheckable()
+        .hasException());
+    assertEquals("Operation error state was unexpected", opHasError, caughtError);
+
+  }
+
+  private void verifyCohortSuccessful(List<String> cohortNames,
+      SubprocedureFactory subprocFactory, Iterable<Subprocedure> cohortTasks,
+      VerificationMode prepare, VerificationMode commit, VerificationMode cleanup,
+      VerificationMode finish, boolean opHasError) throws Exception {
+
+    // make sure we build the correct number of cohort members
+    Mockito.verify(subprocFactory, Mockito.times(cohortNames.size())).buildSubprocedure(
+      Mockito.eq(opName), (byte[]) Mockito.argThat(new ArrayEquals(data)));
+    // verify that we ran each of the operations cleanly
+    int j = 0;
+    for (Subprocedure op : cohortTasks) {
+      LOG.debug("Checking mock:" + (j++));
+      waitAndVerifySubproc(op, prepare, commit, cleanup, finish, opHasError);
+    }
+  }
+
+  private void closeAll(
+      ProcedureCoordinator coordinator,
+      ZKProcedureCoordinatorRpcs coordinatorController,
+      List<Pair<ProcedureMember, ZKProcedureMemberRpcs>> cohort)
+      throws IOException {
+    // make sure we close all the resources
+    for (Pair<ProcedureMember, ZKProcedureMemberRpcs> member : cohort) {
+      member.getFirst().close();
+      member.getSecond().close();
+    }
+    coordinator.close();
+    coordinatorController.close();
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
new file mode 100644
index 000000000000..66e4e901cc23
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
@@ -0,0 +1,429 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.mockito.verification.VerificationMode;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Test zookeeper-based, procedure controllers
+ */
+@Category(MediumTests.class)
+public class TestZKProcedureControllers {
+
+  static final Log LOG = LogFactory.getLog(TestZKProcedureControllers.class);
+  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final String COHORT_NODE_NAME = "expected";
+  private static final String CONTROLLER_NODE_NAME = "controller";
+  private static final VerificationMode once = Mockito.times(1);
+
+  @BeforeClass
+  public static void setupTest() throws Exception {
+    UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    UTIL.shutdownMiniZKCluster();
+  }
+
+  /**
+   * Smaller test to just test the actuation on the cohort member
+   * @throws Exception on failure
+   */
+  @Test(timeout = 15000)
+  public void testSimpleZKCohortMemberController() throws Exception {
+    ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
+    final String operationName = "instanceTest";
+
+    final Subprocedure sub = Mockito.mock(Subprocedure.class);
+    Mockito.when(sub.getName()).thenReturn(operationName);
+
+    final byte[] data = new byte[] { 1, 2, 3 };
+    final CountDownLatch prepared = new CountDownLatch(1);
+    final CountDownLatch committed = new CountDownLatch(1);
+
+    final ForeignExceptionDispatcher monitor = spy(new ForeignExceptionDispatcher());
+    final ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(
+        watcher, "testSimple", COHORT_NODE_NAME);
+
+    // mock out cohort member callbacks
+    final ProcedureMember member = Mockito
+        .mock(ProcedureMember.class);
+    Mockito.doReturn(sub).when(member).createSubprocedure(operationName, data);
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        controller.sendMemberAcquired(sub);
+        prepared.countDown();
+        return null;
+      }
+    }).when(member).submitSubprocedure(sub);
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        controller.sendMemberCompleted(sub);
+        committed.countDown();
+        return null;
+      }
+    }).when(member).receivedReachedGlobalBarrier(operationName);
+
+    // start running the listener
+    controller.start(member);
+
+    // set a prepare node from a 'coordinator'
+    String prepare = ZKProcedureUtil.getAcquireBarrierNode(controller.getZkController(), operationName);
+    ZKUtil.createSetData(watcher, prepare, ProtobufUtil.prependPBMagic(data));
+    // wait for the operation to be prepared
+    prepared.await();
+
+    // create the commit node so we update the operation to enter the commit phase
+    String commit = ZKProcedureUtil.getReachedBarrierNode(controller.getZkController(), operationName);
+    LOG.debug("Found prepared, posting commit node:" + commit);
+    ZKUtil.createAndFailSilent(watcher, commit);
+    LOG.debug("Commit node:" + commit + ", exists:" + ZKUtil.checkExists(watcher, commit));
+    committed.await();
+
+    verify(monitor, never()).receive(Mockito.any(ForeignException.class));
+    // XXX: broken due to composition.
+//    verify(member, never()).getManager().controllerConnectionFailure(Mockito.anyString(),
+//      Mockito.any(IOException.class));
+    // cleanup after the test
+    ZKUtil.deleteNodeRecursively(watcher, controller.getZkController().getBaseZnode());
+    assertEquals("Didn't delete prepare node", -1, ZKUtil.checkExists(watcher, prepare));
+    assertEquals("Didn't delete commit node", -1, ZKUtil.checkExists(watcher, commit));
+  }
+
+  @Test(timeout = 15000)
+  public void testZKCoordinatorControllerWithNoCohort() throws Exception {
+    final String operationName = "no cohort controller test";
+    final byte[] data = new byte[] { 1, 2, 3 };
+
+    runMockCommitWithOrchestratedControllers(startCoordinatorFirst, operationName, data);
+    runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data);
+  }
+
+  @Test(timeout = 15000)
+  public void testZKCoordinatorControllerWithSingleMemberCohort() throws Exception {
+    final String operationName = "single member controller test";
+    final byte[] data = new byte[] { 1, 2, 3 };
+
+    runMockCommitWithOrchestratedControllers(startCoordinatorFirst, operationName, data, "cohort");
+    runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data, "cohort");
+  }
+
+  @Test(timeout = 15000)
+  public void testZKCoordinatorControllerMultipleCohort() throws Exception {
+    final String operationName = "multi member controller test";
+    final byte[] data = new byte[] { 1, 2, 3 };
+
+    runMockCommitWithOrchestratedControllers(startCoordinatorFirst, operationName, data, "cohort",
+      "cohort2", "cohort3");
+    runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data, "cohort",
+      "cohort2", "cohort3");
+  }
+
+  private void runMockCommitWithOrchestratedControllers(StartControllers controllers,
+      String operationName, byte[] data, String... cohort) throws Exception {
+    ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
+    List<String> expected = Lists.newArrayList(cohort);
+
+    final Subprocedure sub = Mockito.mock(Subprocedure.class);
+    Mockito.when(sub.getName()).thenReturn(operationName);
+
+    CountDownLatch prepared = new CountDownLatch(expected.size());
+    CountDownLatch committed = new CountDownLatch(expected.size());
+    // mock out coordinator so we can keep track of zk progress
+    ProcedureCoordinator coordinator = setupMockCoordinator(operationName,
+      prepared, committed);
+
+    ProcedureMember member = Mockito.mock(ProcedureMember.class);
+
+    Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> pair = controllers
+        .start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected);
+    ZKProcedureCoordinatorRpcs controller = pair.getFirst();
+    List<ZKProcedureMemberRpcs> cohortControllers = pair.getSecond();
+    // start the operation
+    Procedure p = Mockito.mock(Procedure.class);
+    Mockito.when(p.getName()).thenReturn(operationName);
+
+    controller.sendGlobalBarrierAcquire(p, data, expected);
+
+    // post the prepare node for each expected node
+    for (ZKProcedureMemberRpcs cc : cohortControllers) {
+      cc.sendMemberAcquired(sub);
+    }
+
+    // wait for all the notifications to reach the coordinator
+    prepared.await();
+    // make sure we got the all the nodes and no more
+    Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName),
+      Mockito.anyString());
+
+    // kick off the commit phase
+    controller.sendGlobalBarrierReached(p, expected);
+
+    // post the committed node for each expected node
+    for (ZKProcedureMemberRpcs cc : cohortControllers) {
+      cc.sendMemberCompleted(sub);
+    }
+
+    // wait for all commit notifications to reach the coordinator
+    committed.await();
+    // make sure we got the all the nodes and no more
+    Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName),
+      Mockito.anyString());
+
+    controller.resetMembers(p);
+
+    // verify all behavior
+    verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil());
+    verifyCohort(member, cohortControllers.size(), operationName, data);
+    verifyCoordinator(operationName, coordinator, expected);
+  }
+
+  // TODO Broken by composition.
+//  @Test
+//  public void testCoordinatorControllerHandlesEarlyPrepareNodes() throws Exception {
+//    runEarlyPrepareNodes(startCoordinatorFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 },
+//      "cohort1", "cohort2");
+//    runEarlyPrepareNodes(startCohortFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 },
+//      "cohort1", "cohort2");
+//  }
+
+  public void runEarlyPrepareNodes(StartControllers controllers, String operationName, byte[] data,
+      String... cohort) throws Exception {
+    ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
+    List<String> expected = Lists.newArrayList(cohort);
+
+    final Subprocedure sub = Mockito.mock(Subprocedure.class);
+    Mockito.when(sub.getName()).thenReturn(operationName);
+
+    final CountDownLatch prepared = new CountDownLatch(expected.size());
+    final CountDownLatch committed = new CountDownLatch(expected.size());
+    // mock out coordinator so we can keep track of zk progress
+    ProcedureCoordinator coordinator = setupMockCoordinator(operationName,
+      prepared, committed);
+
+    ProcedureMember member = Mockito.mock(ProcedureMember.class);
+    Procedure p = Mockito.mock(Procedure.class);
+    Mockito.when(p.getName()).thenReturn(operationName);
+
+    Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> pair = controllers
+        .start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected);
+    ZKProcedureCoordinatorRpcs controller = pair.getFirst();
+    List<ZKProcedureMemberRpcs> cohortControllers = pair.getSecond();
+
+    // post 1/2 the prepare nodes early
+    for (int i = 0; i < cohortControllers.size() / 2; i++) {
+      cohortControllers.get(i).sendMemberAcquired(sub);
+    }
+
+    // start the operation
+    controller.sendGlobalBarrierAcquire(p, data, expected);
+
+    // post the prepare node for each expected node
+    for (ZKProcedureMemberRpcs cc : cohortControllers) {
+      cc.sendMemberAcquired(sub);
+    }
+
+    // wait for all the notifications to reach the coordinator
+    prepared.await();
+    // make sure we got the all the nodes and no more
+    Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName),
+      Mockito.anyString());
+
+    // kick off the commit phase
+    controller.sendGlobalBarrierReached(p, expected);
+
+    // post the committed node for each expected node
+    for (ZKProcedureMemberRpcs cc : cohortControllers) {
+      cc.sendMemberCompleted(sub);
+    }
+
+    // wait for all commit notifications to reach the coordiantor
+    committed.await();
+    // make sure we got the all the nodes and no more
+    Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName),
+      Mockito.anyString());
+
+    controller.resetMembers(p);
+
+    // verify all behavior
+    verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil());
+    verifyCohort(member, cohortControllers.size(), operationName, data);
+    verifyCoordinator(operationName, coordinator, expected);
+  }
+
+  /**
+   * @return a mock {@link ProcedureCoordinator} that just counts down the
+   *         prepared and committed latch for called to the respective method
+   */
+  private ProcedureCoordinator setupMockCoordinator(String operationName,
+      final CountDownLatch prepared, final CountDownLatch committed) {
+    ProcedureCoordinator coordinator = Mockito
+        .mock(ProcedureCoordinator.class);
+    Mockito.mock(ProcedureCoordinator.class);
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        prepared.countDown();
+        return null;
+      }
+    }).when(coordinator).memberAcquiredBarrier(Mockito.eq(operationName), Mockito.anyString());
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        committed.countDown();
+        return null;
+      }
+    }).when(coordinator).memberFinishedBarrier(Mockito.eq(operationName), Mockito.anyString());
+    return coordinator;
+  }
+
+  /**
+   * Verify that the prepare, commit and abort nodes for the operation are removed from zookeeper
+   */
+  private void verifyZooKeeperClean(String operationName, ZooKeeperWatcher watcher,
+      ZKProcedureUtil controller) throws Exception {
+    String prepare = ZKProcedureUtil.getAcquireBarrierNode(controller, operationName);
+    String commit = ZKProcedureUtil.getReachedBarrierNode(controller, operationName);
+    String abort = ZKProcedureUtil.getAbortNode(controller, operationName);
+    assertEquals("Didn't delete prepare node", -1, ZKUtil.checkExists(watcher, prepare));
+    assertEquals("Didn't delete commit node", -1, ZKUtil.checkExists(watcher, commit));
+    assertEquals("Didn't delete abort node", -1, ZKUtil.checkExists(watcher, abort));
+  }
+
+  /**
+   * Verify the cohort controller got called once per expected node to start the operation
+   */
+  private void verifyCohort(ProcedureMember member, int cohortSize,
+      String operationName, byte[] data) {
+//    verify(member, Mockito.times(cohortSize)).submitSubprocedure(Mockito.eq(operationName),
+//      (byte[]) Mockito.argThat(new ArrayEquals(data)));
+    verify(member, Mockito.times(cohortSize)).submitSubprocedure(Mockito.any(Subprocedure.class));
+
+  }
+
+  /**
+   * Verify that the coordinator only got called once for each expected node
+   */
+  private void verifyCoordinator(String operationName,
+      ProcedureCoordinator coordinator, List<String> expected) {
+    // verify that we got all the expected nodes
+    for (String node : expected) {
+      verify(coordinator, once).memberAcquiredBarrier(operationName, node);
+      verify(coordinator, once).memberFinishedBarrier(operationName, node);
+    }
+  }
+
+  /**
+   * Specify how the controllers that should be started (not spy/mockable) for the test.
+   */
+  private abstract class StartControllers {
+    public abstract Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> start(
+        ZooKeeperWatcher watcher, String operationName,
+        ProcedureCoordinator coordinator, String controllerName,
+        ProcedureMember member, List<String> cohortNames) throws Exception;
+  }
+
+  private final StartControllers startCoordinatorFirst = new StartControllers() {
+
+    @Override
+    public Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> start(
+        ZooKeeperWatcher watcher, String operationName,
+        ProcedureCoordinator coordinator, String controllerName,
+        ProcedureMember member, List<String> expected) throws Exception {
+      // start the controller
+      ZKProcedureCoordinatorRpcs controller = new ZKProcedureCoordinatorRpcs(
+          watcher, operationName, CONTROLLER_NODE_NAME);
+      controller.start(coordinator);
+
+      // make a cohort controller for each expected node
+
+      List<ZKProcedureMemberRpcs> cohortControllers = new ArrayList<ZKProcedureMemberRpcs>();
+      for (String nodeName : expected) {
+        ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(
+            watcher, operationName, nodeName);
+        cc.start(member);
+        cohortControllers.add(cc);
+      }
+      return new Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>>(
+          controller, cohortControllers);
+    }
+  };
+
+  /**
+   * Check for the possible race condition where a cohort member starts after the controller and
+   * therefore could miss a new operation
+   */
+  private final StartControllers startCohortFirst = new StartControllers() {
+
+    @Override
+    public Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> start(
+        ZooKeeperWatcher watcher, String operationName,
+        ProcedureCoordinator coordinator, String controllerName,
+        ProcedureMember member, List<String> expected) throws Exception {
+
+      // make a cohort controller for each expected node
+      List<ZKProcedureMemberRpcs> cohortControllers = new ArrayList<ZKProcedureMemberRpcs>();
+      for (String nodeName : expected) {
+        ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(
+            watcher, operationName, nodeName);
+        cc.start(member);
+        cohortControllers.add(cc);
+      }
+
+      // start the controller
+      ZKProcedureCoordinatorRpcs controller = new ZKProcedureCoordinatorRpcs(
+          watcher, operationName, CONTROLLER_NODE_NAME);
+      controller.start(coordinator);
+
+      return new Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>>(
+          controller, cohortControllers);
+    }
+  };
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 3f63e3d29daf..a07fff1133cf 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -28,16 +28,22 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;
+import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.HalfStoreFileReader;
+import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.io.Reference.Range;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -53,6 +59,7 @@
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
@@ -92,8 +99,8 @@ public void tearDown() throws Exception {
    * @throws Exception
    */
   public void testBasicHalfMapFile() throws Exception {
-    // Make up a directory hierarchy that has a regiondir and familyname.
-    Path outputDir = new Path(new Path(this.testDir, "regionname"),
+    // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
+    Path outputDir = new Path(new Path(this.testDir, "7e0102"),
         "familyname");
     StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
         this.fs, 2 * 1024)
@@ -107,6 +114,10 @@ public void testBasicHalfMapFile() throws Exception {
   private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
     writeStoreFile(writer, Bytes.toBytes(getName()), Bytes.toBytes(getName()));
   }
+
+  // pick an split point (roughly halfway)
+  byte[] SPLITKEY = new byte[] { (LAST_CHAR-FIRST_CHAR)/2, FIRST_CHAR};
+
   /*
    * Writes HStoreKey and ImmutableBytes data to passed writer and
    * then closes it.
@@ -135,12 +146,12 @@ public static void writeStoreFile(final StoreFile.Writer writer, byte[] fam, byt
    */
   public void testReference()
   throws IOException {
-    Path storedir = new Path(new Path(this.testDir, "regionname"), "familyname");
-    Path dir = new Path(storedir, "1234567890");
+    // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
+    Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname");
     // Make a store file and write data to it.
     StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
         this.fs, 8 * 1024)
-            .withOutputDir(dir)
+            .withOutputDir(storedir)
             .build();
     writeStoreFile(writer);
     StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
@@ -154,7 +165,7 @@ public void testReference()
     kv = KeyValue.createKeyValueFromKey(reader.getLastKey());
     byte [] finalRow = kv.getRow();
     // Make a reference
-    Path refPath = StoreFile.split(fs, dir, hsf, midRow, Range.top);
+    Path refPath = StoreFile.split(fs, storedir, hsf, midRow, Range.top);
     StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf,
         StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     // Now confirm that I can read from the reference and that it only gets
@@ -171,6 +182,147 @@ public void testReference()
     assertTrue(Bytes.equals(kv.getRow(), finalRow));
   }
 
+  public void testHFileLink() throws IOException {
+    final String columnFamily = "f";
+
+    Configuration testConf = new Configuration(this.conf);
+    FSUtils.setRootDir(testConf, this.testDir);
+
+    HRegionInfo hri = new HRegionInfo(Bytes.toBytes("table-link"));
+    Path storedir = new Path(new Path(this.testDir,
+      new Path(hri.getTableNameAsString(), hri.getEncodedName())), columnFamily);
+
+    // Make a store file and write data to it.
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(testConf, cacheConf,
+         this.fs, 8 * 1024)
+            .withOutputDir(storedir)
+            .build();
+    Path storeFilePath = writer.getPath();
+    writeStoreFile(writer);
+    writer.close();
+
+    Path dstPath = new Path(this.testDir, new Path("test-region", columnFamily));
+    HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
+    Path linkFilePath = new Path(dstPath,
+                  HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
+
+    // Try to open store file from link
+    StoreFile hsf = new StoreFile(this.fs, linkFilePath, testConf, cacheConf,
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
+    assertTrue(hsf.isLink());
+
+    // Now confirm that I can read from the link
+    int count = 1;
+    HFileScanner s = hsf.createReader().getScanner(false, false);
+    s.seekTo();
+    while (s.next()) {
+      count++;
+    }
+    assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
+  }
+
+  /**
+   * Validate that we can handle valid tables with '.', '_', and '-' chars.
+   */
+  public void testStoreFileNames() {
+    String[] legalHFileLink = { "MyTable_02=abc012-def345", "MyTable_02.300=abc012-def345",
+      "MyTable_02-400=abc012-def345", "MyTable_02-400.200=abc012-def345",
+      "MyTable_02=abc012-def345_SeqId_1_", "MyTable_02=abc012-def345_SeqId_20_" };
+    for (String name: legalHFileLink) {
+      assertTrue("should be a valid link: " + name, HFileLink.isHFileLink(name));
+      assertTrue("should be a valid StoreFile" + name, StoreFile.validateStoreFileName(name));
+      assertFalse("should not be a valid reference: " + name, StoreFile.isReference(name));
+
+      String refName = name + ".6789";
+      assertTrue("should be a valid link reference: " + refName, StoreFile.isReference(refName));
+      assertTrue("should be a valid StoreFile" + refName, StoreFile.validateStoreFileName(refName));
+    }
+
+    String[] illegalHFileLink = { ".MyTable_02=abc012-def345", "-MyTable_02.300=abc012-def345",
+      "MyTable_02-400=abc0_12-def345", "MyTable_02-400.200=abc012-def345...." };
+    for (String name: illegalHFileLink) {
+      assertFalse("should not be a valid link: " + name, HFileLink.isHFileLink(name));
+    }
+  }
+
+  /**
+   * This test creates an hfile and then the dir structures and files to verify that references
+   * to hfilelinks (created by snapshot clones) can be properly interpreted.
+   */
+  public void testReferenceToHFileLink() throws IOException {
+    final String columnFamily = "f";
+
+    Path rootDir = FSUtils.getRootDir(conf);
+
+    String tablename = "_original-evil-name"; // adding legal table name chars to verify regex handles it.
+    HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tablename));
+    // store dir = <root>/<tablename>/<rgn>/<cf>
+    Path storedir = new Path(new Path(rootDir,
+      new Path(hri.getTableNameAsString(), hri.getEncodedName())), columnFamily);
+
+    // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+         this.fs, 8 * 1024)
+            .withOutputDir(storedir)
+            .build();
+    Path storeFilePath = writer.getPath();
+    writeStoreFile(writer);
+    writer.close();
+
+    // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
+    String target = "clone";
+    Path dstPath = new Path(rootDir, new Path(new Path(target, "7e0102"), columnFamily));
+    HFileLink.create(conf, this.fs, dstPath, hri, storeFilePath.getName());
+    Path linkFilePath = new Path(dstPath,
+                  HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
+
+    // create splits of the link.
+    // <root>/clone/splitA/<cf>/<reftohfilelink>,
+    // <root>/clone/splitB/<cf>/<reftohfilelink>
+    Path splitDirA = new Path(new Path(rootDir,
+        new Path(target, "571A")), columnFamily);
+    Path splitDirB = new Path(new Path(rootDir,
+        new Path(target, "571B")), columnFamily);
+    StoreFile f = new StoreFile(fs, linkFilePath, conf, cacheConf, BloomType.NONE,
+        NoOpDataBlockEncoder.INSTANCE);
+    byte[] splitRow = SPLITKEY;
+    Path pathA = StoreFile.split(fs, splitDirA, f, splitRow, Range.top); // top
+    Path pathB = StoreFile.split(fs, splitDirB, f, splitRow, Range.bottom); // bottom
+
+    // OK test the thing
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // There is a case where a file with the hfilelink pattern is actually a daughter
+    // reference to a hfile link.  This code in StoreFile that handles this case.
+
+    // Try to open store file from link
+    StoreFile hsfA = new StoreFile(this.fs, pathA,  conf, cacheConf,
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
+
+    // Now confirm that I can read from the ref to link
+    int count = 1;
+    HFileScanner s = hsfA.createReader().getScanner(false, false);
+    s.seekTo();
+    while (s.next()) {
+      count++;
+    }
+    assertTrue(count > 0); // read some rows here
+
+    // Try to open store file from link
+    StoreFile hsfB = new StoreFile(this.fs, pathB,  conf, cacheConf,
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
+
+    // Now confirm that I can read from the ref to link
+    HFileScanner sB = hsfB.createReader().getScanner(false, false);
+    sB.seekTo();
+    while (sB.next()) {
+      count++;
+    }
+
+    // read the rest of the rows
+    assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
+  }
+
   private void checkHalfHFile(final StoreFile f)
   throws IOException {
     byte [] midkey = f.createReader().midkey();
@@ -694,8 +846,8 @@ public void testMultipleTimestamps() throws IOException {
     long[] timestamps = new long[] {20,10,5,1};
     Scan scan = new Scan();
 
-    Path storedir = new Path(new Path(this.testDir, "regionname"),
-    "familyname");
+    // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
+    Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname");
     Path dir = new Path(storedir, "1234567890");
     StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
         this.fs, 8 * 1024)
@@ -738,8 +890,8 @@ public void testMultipleTimestamps() throws IOException {
   public void testCacheOnWriteEvictOnClose() throws Exception {
     Configuration conf = this.conf;
 
-    // Find a home for our files
-    Path baseDir = new Path(new Path(this.testDir, "regionname"),"twoCOWEOC");
+    // Find a home for our files (regiondir ("7e0102") and familyname).
+    Path baseDir = new Path(new Path(this.testDir, "7e0102"),"twoCOWEOC");
 
     // Grab the block cache and get the initial hit/miss counts
     BlockCache bc = new CacheConfig(conf).getBlockCache();
@@ -810,7 +962,7 @@ public void testCacheOnWriteEvictOnClose() throws Exception {
       kv2 = scannerTwo.next();
       assertTrue(kv1.equals(kv2));
       assertTrue(Bytes.compareTo(
-          kv1.getBuffer(), kv1.getKeyOffset(), kv1.getKeyLength(), 
+          kv1.getBuffer(), kv1.getKeyOffset(), kv1.getKeyLength(),
           kv2.getBuffer(), kv2.getKeyOffset(), kv2.getKeyLength()) == 0);
       assertTrue(Bytes.compareTo(
           kv1.getBuffer(), kv1.getValueOffset(), kv1.getValueLength(),
@@ -891,7 +1043,8 @@ private StoreFile.Writer writeStoreFile(Configuration conf,
    * file info.
    */
   public void testDataBlockEncodingMetaData() throws IOException {
-    Path dir = new Path(new Path(this.testDir, "regionname"), "familyname");
+    // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
+    Path dir = new Path(new Path(this.testDir, "7e0102"), "familyname");
     Path path = new Path(dir, "1234567890");
 
     DataBlockEncoding dataBlockEncoderAlgo =
@@ -910,11 +1063,11 @@ public void testDataBlockEncodingMetaData() throws IOException {
             .withBytesPerChecksum(CKBYTES)
             .build();
     writer.close();
-    
+
     StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf,
         cacheConf, BloomType.NONE, dataBlockEncoder);
     StoreFile.Reader reader = storeFile.createReader();
-    
+
     Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
     byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
 
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
new file mode 100644
index 000000000000..b04fb55f54f7
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Assert;
+
+/**
+ * Utilities class for snapshots
+ */
+public class SnapshotTestingUtils {
+
+  private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
+
+  /**
+   * Assert that we don't have any snapshots lists
+   * @throws IOException if the admin operation fails
+   */
+  public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
+    assertEquals("Have some previous snapshots", 0, admin.listSnapshots().size());
+  }
+
+  /**
+   * Make sure that there is only one snapshot returned from the master and its name and table match
+   * the passed in parameters.
+   */
+  public static void assertOneSnapshotThatMatches(HBaseAdmin admin, HSnapshotDescription snapshot)
+      throws IOException {
+    assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
+  }
+
+  /**
+   * Make sure that there is only one snapshot returned from the master and its name and table match
+   * the passed in parameters.
+   */
+  public static void assertOneSnapshotThatMatches(HBaseAdmin admin, SnapshotDescription snapshot)
+      throws IOException {
+    assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
+  }
+
+  /**
+   * Make sure that there is only one snapshot returned from the master and its name and table match
+   * the passed in parameters.
+   */
+  public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
+      String snapshotName, String tableName) throws IOException {
+    // list the snapshot
+    List<SnapshotDescription> snapshots = admin.listSnapshots();
+
+    assertEquals("Should only have 1 snapshot", 1, snapshots.size());
+    assertEquals(snapshotName, snapshots.get(0).getName());
+    assertEquals(tableName, snapshots.get(0).getTable());
+
+    return snapshots;
+  }
+
+  /**
+   * Make sure that there is only one snapshot returned from the master and its name and table match
+   * the passed in parameters.
+   */
+  public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
+      byte[] snapshot, byte[] tableName) throws IOException {
+    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), Bytes.toString(tableName));
+  }
+
+  /**
+   * Confirm that the snapshot contains references to all the files that should be in the snapshot
+   */
+  public static void confirmSnapshotValid(SnapshotDescription snapshotDescriptor,
+      byte[] tableName, byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
+      boolean requireLogs, Path logsDir, Set<String> snapshotServers) throws IOException {
+    Path snapshotDir = SnapshotDescriptionUtils
+        .getCompletedSnapshotDir(snapshotDescriptor, rootDir);
+    assertTrue(fs.exists(snapshotDir));
+    Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
+    assertTrue(fs.exists(snapshotinfo));
+    // check the logs dir
+    if (requireLogs) {
+      TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers,
+        snapshotDescriptor, new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME));
+    }
+    // check the table info
+    HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, tableName);
+    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+    assertEquals(desc, snapshotDesc);
+
+    // check the region snapshot for all the regions
+    List<HRegionInfo> regions = admin.getTableRegions(tableName);
+    for (HRegionInfo info : regions) {
+      String regionName = info.getEncodedName();
+      Path regionDir = new Path(snapshotDir, regionName);
+      HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+      assertEquals(info, snapshotRegionInfo);
+      // check to make sure we have the family
+      Path familyDir = new Path(regionDir, Bytes.toString(testFamily));
+      assertTrue("Expected to find: " + familyDir + ", but it doesn't exist", fs.exists(familyDir));
+      // make sure we have some files references
+      assertTrue(fs.listStatus(familyDir).length > 0);
+    }
+  }
+
+  /**
+   * Helper method for testing async snapshot operations. Just waits for the given snapshot to
+   * complete on the server by repeatedly checking the master.
+   * @param master running the snapshot
+   * @param snapshot to check
+   * @param sleep amount to sleep between checks to see if the snapshot is done
+   * @throws IOException if the snapshot fails
+   */
+  public static void waitForSnapshotToComplete(HMaster master, HSnapshotDescription snapshot,
+      long sleep) throws IOException {
+    boolean done = false;
+    while (!done) {
+      done = master.isSnapshotDone(snapshot);
+      try {
+        Thread.sleep(sleep);
+      } catch (InterruptedException e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
+  public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName) throws IOException {
+    SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
+  }
+
+  public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName) throws IOException {
+    // delete the taken snapshot
+    admin.deleteSnapshot(snapshotName);
+    assertNoSnapshots(admin);
+  }
+
+  /**
+   * Expect the snapshot to throw an error when checking if the snapshot is complete
+   * @param master master to check
+   * @param snapshot the {@link HSnapshotDescription} request to pass to the master
+   * @param clazz expected exception from the master
+   */
+  public static void expectSnapshotDoneException(HMaster master, HSnapshotDescription snapshot,
+      Class<? extends HBaseSnapshotException> clazz) {
+    try {
+      boolean res = master.isSnapshotDone(snapshot);
+      Assert.fail("didn't fail to lookup a snapshot: res=" + res);
+    } catch (HBaseSnapshotException e) {
+      assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
+    } catch (Throwable t) {
+      Assert.fail("Threw an unexpected exception:" + t);
+    }
+  }
+
+  /**
+   * List all the HFiles in the given table
+   * @param fs FileSystem where the table lives
+   * @param tableDir directory of the table
+   * @return array of the current HFiles in the table (could be a zero-length array)
+   * @throws IOException on unexecpted error reading the FS
+   */
+  public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir) throws IOException {
+    // setup the filters we will need based on the filesystem
+    PathFilter regionFilter = new FSUtils.RegionDirFilter(fs);
+    PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs);
+    final PathFilter fileFilter = new PathFilter() {
+      @Override
+      public boolean accept(Path file) {
+        try {
+          return fs.isFile(file);
+        } catch (IOException e) {
+          return false;
+        }
+      }
+    };
+
+    FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter);
+    // if no regions, then we are done
+    if (regionDirs == null || regionDirs.length == 0) return new FileStatus[0];
+
+    // go through each of the regions, and add al the hfiles under each family
+    List<FileStatus> regionFiles = new ArrayList<FileStatus>(regionDirs.length);
+    for (FileStatus regionDir : regionDirs) {
+      FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(), familyFilter);
+      // if no families, then we are done again
+      if (fams == null || fams.length == 0) continue;
+      // add all the hfiles under the family
+      regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs, fileFilter));
+    }
+    FileStatus[] files = new FileStatus[regionFiles.size()];
+    regionFiles.toArray(files);
+    return files;
+  }
+
+  /**
+   * Get all the hfiles in the region, under the passed set of families
+   * @param families all the family directories under the region
+   * @param fs filesystem where the families live
+   * @param fileFilter filter to only include files
+   * @return collection of all the hfiles under all the passed in families (non-null)
+   * @throws IOException on unexecpted error reading the FS
+   */
+  public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families, FileSystem fs,
+      PathFilter fileFilter) throws IOException {
+    Set<FileStatus> files = new TreeSet<FileStatus>();
+    for (FileStatus family : families) {
+      // get all the hfiles in the family
+      FileStatus[] hfiles = FSUtils.listStatus(fs, family.getPath(), fileFilter);
+      // if no hfiles, then we are done with this family
+      if (hfiles == null || hfiles.length == 0) continue;
+      files.addAll(Arrays.asList(hfiles));
+    }
+    return files;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java
new file mode 100644
index 000000000000..b68c3b94ab1c
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test that we correctly copy the recovered edits from a directory
+ */
+@Category(SmallTests.class)
+public class TestCopyRecoveredEditsTask {
+
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @Test
+  public void testCopyFiles() throws Exception {
+
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
+    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+    FileSystem fs = UTIL.getTestFileSystem();
+    Path root = UTIL.getDataTestDir();
+    String regionName = "regionA";
+    Path regionDir = new Path(root, regionName);
+    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root);
+
+    try {
+      // doesn't really matter where the region's snapshot directory is, but this is pretty close
+      Path snapshotRegionDir = new Path(workingDir, regionName);
+      fs.mkdirs(snapshotRegionDir);
+
+      // put some stuff in the recovered.edits directory
+      Path edits = HLog.getRegionDirRecoveredEditsDir(regionDir);
+      fs.mkdirs(edits);
+      // make a file with some data
+      Path file1 = new Path(edits, "0000000000000002352");
+      FSDataOutputStream out = fs.create(file1);
+      byte[] data = new byte[] { 1, 2, 3, 4 };
+      out.write(data);
+      out.close();
+      // make an empty file
+      Path empty = new Path(edits, "empty");
+      fs.createNewFile(empty);
+
+      CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
+          snapshotRegionDir);
+      CopyRecoveredEditsTask taskSpy = Mockito.spy(task);
+      taskSpy.call();
+
+      Path snapshotEdits = HLog.getRegionDirRecoveredEditsDir(snapshotRegionDir);
+      FileStatus[] snapshotEditFiles = FSUtils.listStatus(fs, snapshotEdits);
+      assertEquals("Got wrong number of files in the snapshot edits", 1, snapshotEditFiles.length);
+      FileStatus file = snapshotEditFiles[0];
+      assertEquals("Didn't copy expected file", file1.getName(), file.getPath().getName());
+
+      Mockito.verify(monitor, Mockito.never()).receive(Mockito.any(ForeignException.class));
+      Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(),
+           Mockito.any(Exception.class));
+    } finally {
+      // cleanup the working directory
+      FSUtils.delete(fs, regionDir, true);
+      FSUtils.delete(fs, workingDir, true);
+    }
+  }
+
+  /**
+   * Check that we don't get an exception if there is no recovered edits directory to copy
+   * @throws Exception on failure
+   */
+  @Test
+  public void testNoEditsDir() throws Exception {
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
+    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+    FileSystem fs = UTIL.getTestFileSystem();
+    Path root = UTIL.getDataTestDir();
+    String regionName = "regionA";
+    Path regionDir = new Path(root, regionName);
+    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root);
+    try {
+      // doesn't really matter where the region's snapshot directory is, but this is pretty close
+      Path snapshotRegionDir = new Path(workingDir, regionName);
+      fs.mkdirs(snapshotRegionDir);
+      Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionDir);
+      assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits));
+
+      CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
+          snapshotRegionDir);
+      task.call();
+    } finally {
+      // cleanup the working directory
+      FSUtils.delete(fs, regionDir, true);
+      FSUtils.delete(fs, workingDir, true);
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
new file mode 100644
index 000000000000..f329c60259ab
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test Export Snapshot Tool
+ */
+@Category(MediumTests.class)
+public class TestExportSnapshot {
+  private final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final static byte[] FAMILY = Bytes.toBytes("cf");
+
+  private byte[] snapshotName;
+  private byte[] tableName;
+  private HBaseAdmin admin;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
+    TEST_UTIL.startMiniCluster(3);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Create a table and take a snapshot of the table used by the export test.
+   */
+  @Before
+  public void setUp() throws Exception {
+    this.admin = TEST_UTIL.getHBaseAdmin();
+
+    long tid = System.currentTimeMillis();
+    tableName = Bytes.toBytes("testtb-" + tid);
+    snapshotName = Bytes.toBytes("snaptb0-" + tid);
+
+    // create Table
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(FAMILY));
+    admin.createTable(htd, null);
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    TEST_UTIL.loadTable(table, FAMILY);
+
+    // take a snapshot
+    admin.disableTable(tableName);
+    admin.snapshot(snapshotName, tableName);
+    admin.enableTable(tableName);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    this.admin.close();
+  }
+
+  /**
+   * Verfy the result of getBalanceSplits() method.
+   * The result are groups of files, used as input list for the "export" mappers.
+   * All the groups should have similar amount of data.
+   *
+   * The input list is a pair of file path and length.
+   * The getBalanceSplits() function sort it by length,
+   * and assign to each group a file, going back and forth through the groups.
+   */
+  @Test
+  public void testBalanceSplit() throws Exception {
+    // Create a list of files
+    List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
+    for (long i = 0; i <= 20; i++) {
+      files.add(new Pair<Path, Long>(new Path("file-" + i), i));
+    }
+
+    // Create 5 groups (total size 210)
+    //    group 0: 20, 11, 10,  1 (total size: 42)
+    //    group 1: 19, 12,  9,  2 (total size: 42)
+    //    group 2: 18, 13,  8,  3 (total size: 42)
+    //    group 3: 17, 12,  7,  4 (total size: 42)
+    //    group 4: 16, 11,  6,  5 (total size: 42)
+    List<List<Path>> splits = ExportSnapshot.getBalancedSplits(files, 5);
+    assertEquals(5, splits.size());
+    assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"),
+      new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0));
+    assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"),
+      new Path("file-9"), new Path("file-2")), splits.get(1));
+    assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"),
+      new Path("file-8"), new Path("file-3")), splits.get(2));
+    assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"),
+      new Path("file-7"), new Path("file-4")), splits.get(3));
+    assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"),
+      new Path("file-6"), new Path("file-5")), splits.get(4));
+  }
+
+  /**
+   * Verify if exported snapshot and copied files matches the original one.
+   */
+  @Test
+  public void testExportFileSystemState() throws Exception {
+    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+    URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
+    FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
+    copyDir = copyDir.makeQualified(fs);
+
+    // Export Snapshot
+    int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
+      "-snapshot", Bytes.toString(snapshotName),
+      "-copy-to", copyDir.toString()
+    });
+    assertEquals(0, res);
+
+    // Verify File-System state
+    FileStatus[] rootFiles = fs.listStatus(copyDir);
+    assertEquals(2, rootFiles.length);
+    for (FileStatus fileStatus: rootFiles) {
+      String name = fileStatus.getPath().getName();
+      assertTrue(fileStatus.isDir());
+      assertTrue(name.equals(".snapshot") || name.equals(".archive"));
+    }
+
+    // compare the snapshot metadata and verify the hfiles
+    final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
+    final Path snapshotDir = new Path(".snapshot", Bytes.toString(snapshotName));
+    verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
+        fs, new Path(copyDir, snapshotDir));
+    verifyArchive(fs, copyDir, Bytes.toString(snapshotName));
+
+    // Remove the exported dir
+    fs.delete(copyDir, true);
+  }
+
+  /*
+   * verify if the snapshot folder on file-system 1 match the one on file-system 2
+   */
+  private void verifySnapshot(final FileSystem fs1, final Path root1,
+      final FileSystem fs2, final Path root2) throws IOException {
+    Set<String> s = new HashSet<String>();
+    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
+  }
+
+  /*
+   * Verify if the files exists
+   */
+  private void verifyArchive(final FileSystem fs, final Path rootDir, final String snapshotName)
+      throws IOException {
+    final Path exportedSnapshot = new Path(rootDir, new Path(".snapshot", snapshotName));
+    final Path exportedArchive = new Path(rootDir, ".archive");
+    LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
+    SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
+        new SnapshotReferenceUtil.FileVisitor() {
+        public void storeFile (final String region, final String family, final String hfile)
+            throws IOException {
+          verifyNonEmptyFile(new Path(exportedArchive,
+            new Path(Bytes.toString(tableName), new Path(region, new Path(family, hfile)))));
+        }
+
+        public void recoveredEdits (final String region, final String logfile)
+            throws IOException {
+          verifyNonEmptyFile(new Path(exportedSnapshot,
+            new Path(Bytes.toString(tableName), new Path(region, logfile))));
+        }
+
+        public void logFile (final String server, final String logfile)
+            throws IOException {
+          verifyNonEmptyFile(new Path(exportedSnapshot, new Path(server, logfile)));
+        }
+
+        private void verifyNonEmptyFile(final Path path) throws IOException {
+          LOG.debug(path);
+          assertTrue(fs.exists(path));
+          assertTrue(fs.getFileStatus(path).getLen() > 0);
+        }
+    });
+  }
+
+  private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
+      throws IOException {
+    Set<String> files = new HashSet<String>();
+    int rootPrefix = root.toString().length();
+    FileStatus[] list = FSUtils.listStatus(fs, dir);
+    if (list != null) {
+      for (FileStatus fstat: list) {
+        LOG.debug(fstat.getPath());
+        if (fstat.isDir()) {
+          files.addAll(listFiles(fs, root, fstat.getPath()));
+        } else {
+          files.add(fstat.getPath().toString().substring(rootPrefix));
+        }
+      }
+    }
+    return files;
+  }
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
new file mode 100644
index 000000000000..feae120460cf
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -0,0 +1,369 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HBaseFsck;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test creating/using/deleting snapshots from the client
+ * <p>
+ * This is an end-to-end test for the snapshot utility
+ *
+ * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
+ * because there will be a few more flavors of snapshots that need to run these tests.
+ */
+@Category(LargeTests.class)
+public class TestFlushSnapshotFromClient {
+  private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final int NUM_RS = 2;
+  private static final String STRING_TABLE_NAME = "test";
+  private static final byte[] TEST_FAM = Bytes.toBytes("fam");
+  private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
+
+  /**
+   * Setup the config for the cluster
+   * @throws Exception on failure
+   */
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster(NUM_RS);
+  }
+
+  private static void setupConf(Configuration conf) {
+    // disable the ui
+    conf.setInt("hbase.regionsever.info.port", -1);
+    // change the flush size to a small amount, regulating number of store files
+    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+    // so make sure we get a compaction when doing a load, but keep around some
+    // files in the store
+    conf.setInt("hbase.hstore.compaction.min", 10);
+    conf.setInt("hbase.hstore.compactionThreshold", 10);
+    // block writes if we get to 12 store files
+    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
+    // drop the number of attempts for the hbase admin
+    conf.setInt("hbase.client.retries.number", 1);
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    // prevent aggressive region split
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      ConstantSizeRegionSplitPolicy.class.getName());
+  }
+
+  @Before
+  public void setup() throws Exception {
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    UTIL.deleteTable(TABLE_NAME);
+    // and cleanup the archive directory
+    try {
+      UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
+    } catch (IOException e) {
+      LOG.warn("Failure to delete archive directory", e);
+    }
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    try {
+      UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      LOG.warn("failure shutting down cluster", e);
+    }
+  }
+
+  /**
+   * Test simple flush snapshotting a table that is online
+   * @throws Exception
+   */
+  @Test
+  public void testFlushTableSnapshot() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // put some stuff in the table
+    HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
+    UTIL.loadTable(table, TEST_FAM);
+
+    // get the name of all the regionservers hosting the snapshotted table
+    Set<String> snapshotServers = new HashSet<String>();
+    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
+    for (RegionServerThread server : servers) {
+      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
+        snapshotServers.add(server.getRegionServer().getServerName().toString());
+      }
+    }
+
+    LOG.debug("FS state before snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    // take a snapshot of the enabled table
+    String snapshotString = "offlineTableSnapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotString);
+    admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
+    LOG.debug("Snapshot completed.");
+
+    // make sure we have the snapshot
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+      snapshot, TABLE_NAME);
+
+    // make sure its a valid snapshot
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    LOG.debug("FS state after snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+      admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+
+    admin.deleteSnapshot(snapshot);
+    snapshots = admin.listSnapshots();
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+  }
+
+  @Test
+  public void testSnapshotFailsOnNonExistantTable() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    String tableName = "_not_a_table";
+
+    // make sure the table doesn't exist
+    boolean fail = false;
+    do {
+    try {
+      admin.getTableDescriptor(Bytes.toBytes(tableName));
+      fail = true;
+          LOG.error("Table:" + tableName + " already exists, checking a new name");
+      tableName = tableName+"!";
+    } catch (TableNotFoundException e) {
+      fail = false;
+      }
+    } while (fail);
+
+    // snapshot the non-existant table
+    try {
+      admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
+      fail("Snapshot succeeded even though there is not table.");
+    } catch (SnapshotCreationException e) {
+      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
+    }
+  }
+
+  /**
+   * Basic end-to-end test of simple-flush-based snapshots
+   */
+  @Test
+  public void testFlushCreateListDestroy() throws Exception {
+    LOG.debug("------- Starting Snapshot test -------------");
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    // load the table so we have some data
+    UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+    // and wait until everything stabilizes
+    HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
+    List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
+    for (HRegion region : onlineRegions) {
+      region.waitForFlushesAndCompactions();
+    }
+    String snapshotName = "flushSnapshotCreateListDestroy";
+    // test creating the snapshot
+    admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
+    logFSTree(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR)));
+
+    // make sure we only have 1 matching snapshot
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+      snapshotName, STRING_TABLE_NAME);
+
+    // check the directory structure
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
+    assertTrue(fs.exists(snapshotDir));
+    HBaseFsck.debugLsr(UTIL.getHBaseCluster().getConfiguration(),  snapshotDir);
+    Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
+    assertTrue(fs.exists(snapshotinfo));
+
+    // check the table info
+    HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, TABLE_NAME);
+    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs,
+      SnapshotDescriptionUtils.getSnapshotsDir(rootDir), Bytes.toBytes(snapshotName));
+    assertEquals(desc, snapshotDesc);
+
+    // check the region snapshot for all the regions
+    List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
+    for (HRegionInfo info : regions) {
+      String regionName = info.getEncodedName();
+      Path regionDir = new Path(snapshotDir, regionName);
+      HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+      assertEquals(info, snapshotRegionInfo);
+      // check to make sure we have the family
+      Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
+      assertTrue(fs.exists(familyDir));
+      // make sure we have some file references
+      assertTrue(fs.listStatus(familyDir).length > 0);
+    }
+
+    // test that we can delete the snapshot
+    admin.deleteSnapshot(snapshotName);
+    HBaseFsck.debugLsr(UTIL.getHBaseCluster().getConfiguration(), FSUtils.getRootDir(UTIL.getConfiguration()));
+
+    // make sure we don't have any snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    LOG.debug("------- Flush-Snapshot Create List Destroy-------------");
+  }
+
+  /**
+   * Demonstrate that we reject snapshot requests if there is a snapshot currently running.
+   */
+  @Test(timeout=60000)
+  public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
+    int ssNum = 10;
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    // load the table so we have some data
+    UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+    // and wait until everything stabilizes
+    HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
+    List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
+    for (HRegion region : onlineRegions) {
+      region.waitForFlushesAndCompactions();
+    }
+
+    // build descriptions
+    SnapshotDescription[] descs = new SnapshotDescription[ssNum];
+    for (int i = 0; i < ssNum; i++) {
+      SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+      builder.setTable(STRING_TABLE_NAME);
+      builder.setName("ss"+i);
+      builder.setType(SnapshotDescription.Type.FLUSH);
+      descs[i] = builder.build();
+    }
+
+    final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
+    // We'll have one of these per thread
+    class SSRunnable implements Runnable {
+      SnapshotDescription ss;
+      SSRunnable(SnapshotDescription ss) {
+        this.ss = ss;
+      }
+
+      @Override
+      public void run() {
+        try {
+          HBaseAdmin admin = UTIL.getHBaseAdmin();
+          LOG.info("Submitting snapshot request: " + SnapshotDescriptionUtils.toString(ss));
+          admin.takeSnapshotAsync(ss);
+        } catch (Exception e) {
+          LOG.info("Exception during snapshot request: " + SnapshotDescriptionUtils.toString(ss)
+              + ".  This is ok, we expect some", e);
+        }
+        LOG.info("Submitted snapshot request: " + SnapshotDescriptionUtils.toString(ss));
+        toBeSubmitted.countDown();
+      }
+    };
+
+    // kick each off its own thread
+    for (int i=0 ; i < ssNum; i++) {
+      new Thread(new SSRunnable(descs[i])).start();
+    }
+
+    // wait until all have been submitted
+    toBeSubmitted.await();
+
+    // loop until all are done.
+    while (true) {
+      int doneCount = 0;
+      for (SnapshotDescription ss : descs) {
+        try {
+          if (admin.isSnapshotFinished(ss)) {
+            doneCount++;
+          }
+        } catch (Exception e) {
+          LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
+          doneCount++;
+        }
+      }
+      if (doneCount == descs.length) {
+        break;
+      }
+      Thread.sleep(100);
+    }
+
+    // dump for debugging
+    logFSTree(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR)));
+
+    List<SnapshotDescription> taken = admin.listSnapshots();
+    int takenSize = taken.size();
+    LOG.info("Taken " + takenSize + " snapshots:  " + taken);
+    assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
+        " issued many requests", takenSize < ssNum && takenSize > 0);
+  }
+
+  private void logFSTree(Path root) throws IOException {
+    FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java
new file mode 100644
index 000000000000..73c2aba863b3
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+@Category(SmallTests.class)
+public class TestReferenceRegionHFilesTask {
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @Test
+  public void testRun() throws IOException {
+    FileSystem fs = UTIL.getTestFileSystem();
+    // setup the region internals
+    Path testdir = UTIL.getDataTestDir();
+    Path regionDir = new Path(testdir, "region");
+    Path family1 = new Path(regionDir, "fam1");
+    // make an empty family
+    Path family2 = new Path(regionDir, "fam2");
+    fs.mkdirs(family2);
+
+    // add some files to family 1
+    Path file1 = new Path(family1, "05f99689ae254693836613d1884c6b63");
+    fs.createNewFile(file1);
+    Path file2 = new Path(family1, "7ac9898bf41d445aa0003e3d699d5d26");
+    fs.createNewFile(file2);
+
+    // create the snapshot directory
+    Path snapshotRegionDir = new Path(testdir, HConstants.SNAPSHOT_DIR_NAME);
+    fs.mkdirs(snapshotRegionDir);
+
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("name")
+        .setTable("table").build();
+    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+    ReferenceRegionHFilesTask task = new ReferenceRegionHFilesTask(snapshot, monitor, regionDir,
+        fs, snapshotRegionDir);
+    ReferenceRegionHFilesTask taskSpy = Mockito.spy(task);
+    task.call();
+
+    // make sure we never get an error
+    Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(),
+        Mockito.any(Exception.class));
+
+    // verify that all the hfiles get referenced
+    List<String> hfiles = new ArrayList<String>(2);
+    FileStatus[] regions = FSUtils.listStatus(fs, snapshotRegionDir);
+    for (FileStatus region : regions) {
+      FileStatus[] fams = FSUtils.listStatus(fs, region.getPath());
+      for (FileStatus fam : fams) {
+        FileStatus[] files = FSUtils.listStatus(fs, fam.getPath());
+        for (FileStatus file : files) {
+          hfiles.add(file.getPath().getName());
+        }
+      }
+    }
+    assertTrue("Didn't reference :" + file1, hfiles.contains(file1.getName()));
+    assertTrue("Didn't reference :" + file1, hfiles.contains(file2.getName()));
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
new file mode 100644
index 000000000000..498d50885aad
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test clone/restore snapshots from the client
+ *
+ * TODO This is essentially a clone of TestRestoreSnapshotFromClient.  This is worth refactoring
+ * this because there will be a few more flavors of snapshots that need to run these tests.
+ */
+@Category(LargeTests.class)
+public class TestRestoreFlushSnapshotFromClient {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final byte[] FAMILY = Bytes.toBytes("cf");
+
+  private byte[] snapshotName0;
+  private byte[] snapshotName1;
+  private byte[] snapshotName2;
+  private int snapshot0Rows;
+  private int snapshot1Rows;
+  private byte[] tableName;
+  private HBaseAdmin admin;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
+    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+    TEST_UTIL.getConfiguration().setBoolean(
+        "hbase.master.enabletable.roundrobin", true);
+
+    // Enable snapshot
+    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+
+    TEST_UTIL.startMiniCluster(3);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Initialize the tests with a table filled with some data
+   * and two snapshots (snapshotName0, snapshotName1) of different states.
+   * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
+   */
+  @Before
+  public void setup() throws Exception {
+    this.admin = TEST_UTIL.getHBaseAdmin();
+
+    long tid = System.currentTimeMillis();
+    tableName = Bytes.toBytes("testtb-" + tid);
+    snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
+    snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
+    snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
+
+    // create Table and disable it
+    createTable(tableName, FAMILY);
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    try {
+      loadData(table, 500, FAMILY);
+      snapshot0Rows = TEST_UTIL.countRows(table);
+      LOG.info("=== before snapshot with 500 rows");
+      logFSTree();
+
+      // take a snapshot
+      admin.snapshot(Bytes.toString(snapshotName0), Bytes.toString(tableName),
+          SnapshotDescription.Type.FLUSH);
+
+      LOG.info("=== after snapshot with 500 rows");
+      logFSTree();
+
+      // insert more data
+      loadData(table, 500, FAMILY);
+      snapshot1Rows = TEST_UTIL.countRows(table);
+      LOG.info("=== before snapshot with 1000 rows");
+      logFSTree();
+
+      // take a snapshot of the updated table
+      admin.snapshot(Bytes.toString(snapshotName1), Bytes.toString(tableName),
+          SnapshotDescription.Type.FLUSH);
+      LOG.info("=== after snapshot with 1000 rows");
+      logFSTree();
+    } finally {
+      table.close();
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    TEST_UTIL.deleteTable(tableName);
+    admin.deleteSnapshot(snapshotName0);
+    admin.deleteSnapshot(snapshotName1);
+
+    // Ensure the archiver to be empty
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    mfs.getFileSystem().delete(
+      new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+  }
+
+  @Test
+  public void testTakeFlushSnapshot() throws IOException {
+    // taking happens in setup.
+  }
+
+  @Test
+  public void testRestoreSnapshot() throws IOException {
+    verifyRowCount(tableName, snapshot1Rows);
+
+    // Restore from snapshot-0
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(snapshotName0);
+    logFSTree();
+    admin.enableTable(tableName);
+    LOG.info("=== after restore with 500 row snapshot");
+    logFSTree();
+    verifyRowCount(tableName, snapshot0Rows);
+
+    // Restore from snapshot-1
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(snapshotName1);
+    admin.enableTable(tableName);
+    verifyRowCount(tableName, snapshot1Rows);
+  }
+
+  @Test(expected=SnapshotDoesNotExistException.class)
+  public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
+    String snapshotName = "random-snapshot-" + System.currentTimeMillis();
+    String tableName = "random-table-" + System.currentTimeMillis();
+    admin.cloneSnapshot(snapshotName, tableName);
+  }
+
+  @Test
+  public void testCloneSnapshot() throws IOException, InterruptedException {
+    byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
+    testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
+    testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
+  }
+
+  private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
+      int snapshotRows) throws IOException, InterruptedException {
+    // create a new table from snapshot
+    admin.cloneSnapshot(snapshotName, tableName);
+    verifyRowCount(tableName, snapshotRows);
+
+    TEST_UTIL.deleteTable(tableName);
+  }
+
+  @Test
+  public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
+    byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName0, clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+    admin.snapshot(Bytes.toString(snapshotName2), Bytes.toString(clonedTableName), SnapshotDescription.Type.FLUSH);
+    TEST_UTIL.deleteTable(clonedTableName);
+
+    admin.cloneSnapshot(snapshotName2, clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+    TEST_UTIL.deleteTable(clonedTableName);
+  }
+
+  // ==========================================================================
+  //  Helpers
+  // ==========================================================================
+  private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (byte[] family: families) {
+      HColumnDescriptor hcd = new HColumnDescriptor(family);
+      htd.addFamily(hcd);
+    }
+    byte[][] splitKeys = new byte[16][];
+    byte[] hex = Bytes.toBytes("0123456789abcdef");
+    for (int i = 0; i < 16; ++i) {
+      splitKeys[i] = new byte[] { hex[i] };
+    }
+    admin.createTable(htd, splitKeys);
+  }
+
+  public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
+    byte[] qualifier = Bytes.toBytes("q");
+    table.setAutoFlush(false);
+    while (rows-- > 0) {
+      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
+      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
+      Put put = new Put(key);
+      put.setWriteToWAL(false);
+      for (byte[] family: families) {
+        put.add(family, qualifier, value);
+      }
+      table.put(put);
+    }
+    table.flushCommits();
+  }
+
+  private void logFSTree() throws IOException {
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
+  }
+
+  private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    assertEquals(expectedRows, TEST_UTIL.countRows(table));
+    table.close();
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
new file mode 100644
index 000000000000..e5cb149b8a72
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test the restore/clone operation from a file-system point of view.
+ */
+@Category(SmallTests.class)
+public class TestRestoreSnapshotHelper {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static String TEST_FAMILY = "cf";
+  private final static String TEST_HFILE = "abc";
+
+  private Configuration conf;
+  private Path archiveDir;
+  private FileSystem fs;
+  private Path rootDir;
+
+  @Before
+  public void setup() throws Exception {
+    rootDir = TEST_UTIL.getDataTestDir("testRestore");
+    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+    fs = TEST_UTIL.getTestFileSystem();
+    conf = TEST_UTIL.getConfiguration();
+    FSUtils.setRootDir(conf, rootDir);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    fs.delete(TEST_UTIL.getDataTestDir(), true);
+  }
+
+  @Test
+  public void testRestore() throws IOException {
+    HTableDescriptor htd = createTableDescriptor("testtb");
+
+    Path snapshotDir = new Path(rootDir, "snapshot");
+    createSnapshot(rootDir, snapshotDir, htd);
+
+    // Test clone a snapshot
+    HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
+    testRestore(snapshotDir, htd.getNameAsString(), htdClone);
+    verifyRestore(rootDir, htd, htdClone);
+
+    // Test clone a clone ("link to link")
+    Path cloneDir = HTableDescriptor.getTableDir(rootDir, htdClone.getName());
+    HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
+    testRestore(cloneDir, htdClone.getNameAsString(), htdClone2);
+    verifyRestore(rootDir, htd, htdClone2);
+  }
+
+  private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
+      final HTableDescriptor htdClone) throws IOException {
+    String[] files = getHFiles(HTableDescriptor.getTableDir(rootDir, htdClone.getName()));
+    assertEquals(2, files.length);
+    assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
+    assertTrue(files[1] + " should be a Referene", StoreFile.isReference(files[1]));
+    assertEquals(sourceHtd.getNameAsString(), HFileLink.getReferencedTableName(files[0]));
+    assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
+    Path refPath = getReferredToFile(files[1]);
+    assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
+    assertEquals(files[0], refPath.getName());
+  }
+
+  /**
+   * Execute the restore operation
+   * @param snapshotDir The snapshot directory to use as "restore source"
+   * @param sourceTableName The name of the snapshotted table
+   * @param htdClone The HTableDescriptor of the table to restore/clone.
+   */
+  public void testRestore(final Path snapshotDir, final String sourceTableName,
+      final HTableDescriptor htdClone) throws IOException {
+    LOG.debug("pre-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir);
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    FSTableDescriptors.createTableDescriptor(htdClone, conf);
+    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
+    helper.restoreHdfsRegions();
+
+    LOG.debug("post-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir);
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+  }
+
+  /**
+   * Initialize the restore helper, based on the snapshot and table information provided.
+   */
+  private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
+      final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
+    CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
+    HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
+    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+
+    SnapshotDescription sd = SnapshotDescription.newBuilder()
+      .setName("snapshot").setTable(sourceTableName).build();
+
+    return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
+      htdClone, HTableDescriptor.getTableDir(rootDir, htdClone.getName()), monitor);
+  }
+
+  private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
+      throws IOException {
+    // First region, simple with one plain hfile.
+    HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+        conf, htd, null, true, true);
+    Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE);
+    fs.createNewFile(storeFile);
+    r0.close();
+
+    // Second region, used to test the split case.
+    // This region contains a reference to the hfile in the first region.
+    HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+        conf, htd, null, true, true);
+    fs.createNewFile(new Path(new Path(r1.getRegionDir(), TEST_FAMILY),
+        storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName()));
+    r1.close();
+
+    Path tableDir = HTableDescriptor.getTableDir(archiveDir, htd.getName());
+    FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
+  }
+
+  private HTableDescriptor createTableDescriptor(final String tableName) {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    return htd;
+  }
+
+  private Path getReferredToFile(final String referenceName) {
+    Path fakeBasePath = new Path(new Path("table", "region"), "cf");
+    return StoreFile.getReferredToFile(new Path(fakeBasePath, referenceName));
+  }
+
+  private String[] getHFiles(final Path tableDir) throws IOException {
+    List<String> files = new ArrayList<String>();
+    for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
+      for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+        for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
+          files.add(file.getPath().getName());
+        }
+      }
+    }
+    Collections.sort(files);
+    return files.toArray(new String[files.size()]);
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
new file mode 100644
index 000000000000..3c452f7eae98
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.util.EnvironmentEdge;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that the {@link SnapshotDescription} helper is helping correctly.
+ */
+@Category(MediumTests.class)
+public class TestSnapshotDescriptionUtils {
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static FileSystem fs;
+  private static Path root;
+
+  @BeforeClass
+  public static void setupFS() throws Exception {
+    fs = UTIL.getTestFileSystem();
+    root = new Path(UTIL.getDataTestDir(), "hbase");
+  }
+
+  @After
+  public void cleanupFS() throws Exception {
+    if (fs.exists(root)) {
+      if (!fs.delete(root, true)) {
+        throw new IOException("Failed to delete root test dir: " + root);
+      }
+      if (!fs.mkdirs(root)) {
+        throw new IOException("Failed to create root test dir: " + root);
+      }
+    }
+    EnvironmentEdgeManagerTestHelper.reset();
+  }
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotDescriptionUtils.class);
+
+  @Test
+  public void testValidateMissingTableName() {
+    Configuration conf = new Configuration(false);
+    try {
+      SnapshotDescriptionUtils.validate(SnapshotDescription.newBuilder().setName("fail").build(),
+        conf);
+      fail("Snapshot was considered valid without a table name");
+    } catch (IllegalArgumentException e) {
+      LOG.debug("Correctly failed when snapshot doesn't have a tablename");
+    }
+  }
+
+  /**
+   * Test that we throw an exception if there is no working snapshot directory when we attempt to
+   * 'complete' the snapshot
+   * @throws Exception on failure
+   */
+  @Test
+  public void testCompleteSnapshotWithNoSnapshotDirectoryFailure() throws Exception {
+    Path snapshotDir = new Path(root, ".snapshot");
+    Path tmpDir = new Path(snapshotDir, ".tmp");
+    Path workingDir = new Path(tmpDir, "not_a_snapshot");
+    assertFalse("Already have working snapshot dir: " + workingDir
+        + " but shouldn't. Test file leak?", fs.exists(workingDir));
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
+    try {
+      SnapshotDescriptionUtils.completeSnapshot(snapshot, root, workingDir, fs);
+      fail("Shouldn't successfully complete move of a non-existent directory.");
+    } catch (IOException e) {
+      LOG.info("Correctly failed to move non-existant directory: " + e.getMessage());
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java
new file mode 100644
index 000000000000..66b941a1c91b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test snapshot log splitter
+ */
+@Category(SmallTests.class)
+public class TestSnapshotLogSplitter {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private byte[] TEST_QUALIFIER = Bytes.toBytes("q");
+  private byte[] TEST_FAMILY = Bytes.toBytes("f");
+
+  private Configuration conf;
+  private FileSystem fs;
+  private Path logFile;
+
+  @Before
+  public void setup() throws Exception {
+    conf = TEST_UTIL.getConfiguration();
+    fs = FileSystem.get(conf);
+    logFile = new Path(TEST_UTIL.getDataTestDir(), "test.log");
+    writeTestLog(logFile);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    fs.delete(logFile, false);
+  }
+
+  @Test
+  public void testSplitLogs() throws IOException {
+    Map<byte[], byte[]> regionsMap = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
+    splitTestLogs(getTableName(5), regionsMap);
+  }
+
+  @Test
+  public void testSplitLogsOnDifferentTable() throws IOException {
+    byte[] tableName = getTableName(1);
+    Map<byte[], byte[]> regionsMap = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
+    for (int j = 0; j < 10; ++j) {
+      byte[] regionName = getRegionName(tableName, j);
+      byte[] newRegionName = getNewRegionName(tableName, j);
+      regionsMap.put(regionName, newRegionName);
+    }
+    splitTestLogs(tableName, regionsMap);
+  }
+
+  /*
+   * Split and verify test logs for the specified table
+   */
+  private void splitTestLogs(final byte[] tableName, final Map<byte[], byte[]> regionsMap)
+      throws IOException {
+    Path tableDir = new Path(TEST_UTIL.getDataTestDir(), Bytes.toString(tableName));
+    SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
+      tableName, regionsMap);
+    try {
+      logSplitter.splitLog(logFile);
+    } finally {
+      logSplitter.close();
+    }
+    verifyRecoverEdits(tableDir, tableName, regionsMap);
+  }
+
+  /*
+   * Verify that every logs in the table directory has just the specified table and regions.
+   */
+  private void verifyRecoverEdits(final Path tableDir, final byte[] tableName,
+      final Map<byte[], byte[]> regionsMap) throws IOException {
+    for (FileStatus regionStatus: FSUtils.listStatus(fs, tableDir)) {
+      assertTrue(regionStatus.getPath().getName().startsWith(Bytes.toString(tableName)));
+      Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionStatus.getPath());
+      byte[] regionName = Bytes.toBytes(regionStatus.getPath().getName());
+      assertFalse(regionsMap.containsKey(regionName));
+      for (FileStatus logStatus: FSUtils.listStatus(fs, regionEdits)) {
+        HLog.Reader reader = HLog.getReader(fs, logStatus.getPath(), conf);
+        try {
+          HLog.Entry entry;
+          while ((entry = reader.next()) != null) {
+            HLogKey key = entry.getKey();
+            assertArrayEquals(tableName, key.getTablename());
+            assertArrayEquals(regionName, key.getEncodedRegionName());
+          }
+        } finally {
+          reader.close();
+        }
+      }
+    }
+  }
+
+  /*
+   * Write some entries in the log file.
+   * 7 different tables with name "testtb-%d"
+   * 10 region per table with name "tableName-region-%d"
+   * 50 entry with row key "row-%d"
+   */
+  private void writeTestLog(final Path logFile) throws IOException {
+    fs.mkdirs(logFile.getParent());
+    HLog.Writer writer = HLog.createWriter(fs, logFile, conf);
+    try {
+      for (int i = 0; i < 7; ++i) {
+        byte[] tableName = getTableName(i);
+        for (int j = 0; j < 10; ++j) {
+          byte[] regionName = getRegionName(tableName, j);
+          for (int k = 0; k < 50; ++k) {
+            byte[] rowkey = Bytes.toBytes("row-" + k);
+            HLogKey key = new HLogKey(regionName, tableName, (long)k,
+              System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
+            WALEdit edit = new WALEdit();
+            edit.add(new KeyValue(rowkey, TEST_FAMILY, TEST_QUALIFIER, rowkey));
+            writer.append(new HLog.Entry(key, edit));
+          }
+        }
+      }
+    } finally {
+      writer.close();
+    }
+  }
+
+  private byte[] getTableName(int tableId) {
+    return Bytes.toBytes("testtb-" + tableId);
+  }
+
+  private byte[] getRegionName(final byte[] tableName, int regionId) {
+    return Bytes.toBytes(Bytes.toString(tableName) + "-region-" + regionId);
+  }
+
+  private byte[] getNewRegionName(final byte[] tableName, int regionId) {
+    return Bytes.toBytes(Bytes.toString(tableName) + "-new-region-" + regionId);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java
new file mode 100644
index 000000000000..36b70508df74
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotTask;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+@Category(SmallTests.class)
+public class TestSnapshotTask {
+
+  /**
+   * Check that errors from running the task get propagated back to the error listener.
+   */
+  @Test
+  public void testErrorPropagation() throws Exception {
+    ForeignExceptionDispatcher error = mock(ForeignExceptionDispatcher.class);
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot")
+        .setTable("table").build();
+    final Exception thrown = new Exception("Failed!");
+    SnapshotTask fail = new SnapshotTask(snapshot, error) {
+      @Override
+      public Void call() {
+        snapshotFailure("Injected failure", thrown);
+        return null;
+      }
+    };
+    fail.call();
+
+    verify(error, Mockito.times(1)).receive(any(ForeignException.class));
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java
new file mode 100644
index 000000000000..a813ba78f9b6
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.ReferenceServerWALsTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test that the WAL reference task works as expected
+ */
+@Category(SmallTests.class)
+public class TestWALReferenceTask {
+
+  private static final Log LOG = LogFactory.getLog(TestWALReferenceTask.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @Test
+  public void testRun() throws IOException {
+    Configuration conf = UTIL.getConfiguration();
+    FileSystem fs = UTIL.getTestFileSystem();
+    // setup the log dir
+    Path testDir = UTIL.getDataTestDir();
+    Set<String> servers = new HashSet<String>();
+    Path logDir = new Path(testDir, ".logs");
+    Path server1Dir = new Path(logDir, "Server1");
+    servers.add(server1Dir.getName());
+    Path server2Dir = new Path(logDir, "me.hbase.com,56073,1348618509968");
+    servers.add(server2Dir.getName());
+    // logs under server 1
+    Path log1_1 = new Path(server1Dir, "me.hbase.com%2C56073%2C1348618509968.1348618520536");
+    Path log1_2 = new Path(server1Dir, "me.hbase.com%2C56073%2C1348618509968.1234567890123");
+    // logs under server 2
+    Path log2_1 = new Path(server2Dir, "me.hbase.com%2C56074%2C1348618509998.1348618515589");
+    Path log2_2 = new Path(server2Dir, "me.hbase.com%2C56073%2C1348618509968.1234567890123");
+
+    // create all the log files
+    fs.createNewFile(log1_1);
+    fs.createNewFile(log1_2);
+    fs.createNewFile(log2_1);
+    fs.createNewFile(log2_2);
+
+    FSUtils.logFileSystemState(fs, testDir, LOG);
+    FSUtils.setRootDir(conf, testDir);
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder()
+        .setName("testWALReferenceSnapshot").build();
+    ForeignExceptionDispatcher listener = Mockito.mock(ForeignExceptionDispatcher.class);
+
+    // reference all the files in the first server directory
+    ReferenceServerWALsTask task = new ReferenceServerWALsTask(snapshot, listener, server1Dir,
+        conf, fs);
+    task.call();
+
+    // reference all the files in the first server directory
+    task = new ReferenceServerWALsTask(snapshot, listener, server2Dir, conf, fs);
+    task.call();
+
+    // verify that we got everything
+    FSUtils.logFileSystemState(fs, testDir, LOG);
+    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, testDir);
+    Path snapshotLogDir = new Path(workingDir, HConstants.HREGION_LOGDIR_NAME);
+
+    // make sure we reference the all the wal files
+    TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logDir, servers, snapshot, snapshotLogDir);
+
+    // make sure we never got an error
+    Mockito.verify(listener, Mockito.atLeastOnce()).rethrowException();
+    Mockito.verifyNoMoreInteractions(listener);
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
new file mode 100644
index 000000000000..c2c95b153365
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
@@ -0,0 +1,225 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+import java.util.Set;
+import java.util.HashSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test {@link FSUtils}.
+ */
+@Category(MediumTests.class)
+public class TestFSVisitor {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final String TABLE_NAME = "testtb";
+
+  private Set<String> tableFamilies;
+  private Set<String> tableRegions;
+  private Set<String> recoveredEdits;
+  private Set<String> tableHFiles;
+  private Set<String> regionServers;
+  private Set<String> serverLogs;
+
+  private FileSystem fs;
+  private Path tableDir;
+  private Path logsDir;
+  private Path rootDir;
+
+  @Before
+  public void setUp() throws Exception {
+    fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    rootDir = TEST_UTIL.getDataTestDir("hbase");
+    logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+
+    tableFamilies = new HashSet<String>();
+    tableRegions = new HashSet<String>();
+    recoveredEdits = new HashSet<String>();
+    tableHFiles = new HashSet<String>();
+    regionServers = new HashSet<String>();
+    serverLogs = new HashSet<String>();
+    tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
+    createRecoverEdits(tableDir, tableRegions, recoveredEdits);
+    createLogs(logsDir, regionServers, serverLogs);
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    fs.delete(rootDir);
+  }
+
+  @Test
+  public void testVisitStoreFiles() throws IOException {
+    final Set<String> regions = new HashSet<String>();
+    final Set<String> families = new HashSet<String>();
+    final Set<String> hfiles = new HashSet<String>();
+    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile(final String region, final String family, final String hfileName)
+          throws IOException {
+        regions.add(region);
+        families.add(family);
+        hfiles.add(hfileName);
+      }
+    });
+    assertEquals(tableRegions, regions);
+    assertEquals(tableFamilies, families);
+    assertEquals(tableHFiles, hfiles);
+  }
+
+  @Test
+  public void testVisitRecoveredEdits() throws IOException {
+    final Set<String> regions = new HashSet<String>();
+    final Set<String> edits = new HashSet<String>();
+    FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() {
+      public void recoveredEdits (final String region, final String logfile)
+          throws IOException {
+        regions.add(region);
+        edits.add(logfile);
+      }
+    });
+    assertEquals(tableRegions, regions);
+    assertEquals(recoveredEdits, edits);
+  }
+
+  @Test
+  public void testVisitLogFiles() throws IOException {
+    final Set<String> servers = new HashSet<String>();
+    final Set<String> logs = new HashSet<String>();
+    FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() {
+      public void logFile (final String server, final String logfile) throws IOException {
+        servers.add(server);
+        logs.add(logfile);
+      }
+    });
+    assertEquals(regionServers, servers);
+    assertEquals(serverLogs, logs);
+  }
+
+
+  /*
+   * |-testtb/
+   * |----f1d3ff8443297732862df21dc4e57262/
+   * |-------f1/
+   * |----------d0be84935ba84b66b1e866752ec5d663
+   * |----------9fc9d481718f4878b29aad0a597ecb94
+   * |-------f2/
+   * |----------4b0fe6068c564737946bcf4fd4ab8ae1
+   */
+  private Path createTableFiles(final Path rootDir, final String tableName,
+      final Set<String> tableRegions, final Set<String> tableFamilies,
+      final Set<String> tableHFiles) throws IOException {
+    Path tableDir = new Path(rootDir, tableName);
+    for (int r = 0; r < 10; ++r) {
+      String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
+      tableRegions.add(regionName);
+      Path regionDir = new Path(tableDir, regionName);
+      for (int f = 0; f < 3; ++f) {
+        String familyName = "f" + f;
+        tableFamilies.add(familyName);
+        Path familyDir = new Path(regionDir, familyName);
+        fs.mkdirs(familyDir);
+        for (int h = 0; h < 5; ++h) {
+         String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
+         tableHFiles.add(hfileName);
+         fs.createNewFile(new Path(familyDir, hfileName));
+        }
+      }
+    }
+    return tableDir;
+  }
+
+  /*
+   * |-testtb/
+   * |----f1d3ff8443297732862df21dc4e57262/
+   * |-------recovered.edits/
+   * |----------0000001351969633479
+   * |----------0000001351969633481
+   */
+  private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions,
+      final Set<String> recoverEdits) throws IOException {
+    for (String region: tableRegions) {
+      Path regionEditsDir = HLog.getRegionDirRecoveredEditsDir(new Path(tableDir, region));
+      long seqId = System.currentTimeMillis();
+      for (int i = 0; i < 3; ++i) {
+        String editName = String.format("%019d", seqId + i);
+        recoverEdits.add(editName);
+        FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName));
+        stream.write(Bytes.toBytes("test"));
+        stream.close();
+      }
+    }
+  }
+
+  /*
+   * |-.logs/
+   * |----server5,5,1351969633508/
+   * |-------server5,5,1351969633508.0
+   * |----server6,6,1351969633512/
+   * |-------server6,6,1351969633512.0
+   * |-------server6,6,1351969633512.3
+   */
+  private void createLogs(final Path logDir, final Set<String> servers,
+      final Set<String> logs) throws IOException {
+    for (int s = 0; s < 7; ++s) {
+      String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis());
+      servers.add(server);
+      Path serverLogDir = new Path(logDir, server);
+      fs.mkdirs(serverLogDir);
+      for (int i = 0; i < 5; ++i) {
+        String logfile = server + '.' + i;
+        logs.add(logfile);
+        FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile));
+        stream.write(Bytes.toBytes("test"));
+        stream.close();
+      }
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java b/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java
index 9175bc27d046..82f96d912e46 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java
@@ -50,9 +50,10 @@ public void testGetArchivePath() throws Exception {
   
   @Test
   public void testRegionArchiveDir() {
+    Configuration conf = null;
     Path tableDir = new Path("table");
     Path regionDir = new Path("region");
-    assertNotNull(HFileArchiveUtil.getRegionArchiveDir(null, tableDir, regionDir));
+    assertNotNull(HFileArchiveUtil.getRegionArchiveDir(conf, tableDir, regionDir));
   }
   
   @Test

From 5064ec039ccd875c43177032dde67f1ee847908e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 4 Mar 2013 19:00:18 +0000
Subject: [PATCH 0816/1540] HBASE-7990 Backport HBASE-5837 'hbase shell
 deleteall to .META. allows insertion of malformed rowkey' to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452443 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/table.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index ed4113228565..082214faa2ee 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -53,6 +53,7 @@ def delete(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_T
     #----------------------------------------------------------------------------------------------
     # Delete a row
     def deleteall(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP)
+      raise ArgumentError, "Row Not Found" if _get_internal(row).nil?
       d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp, nil)
       if column
         family, qualifier = parse_column_name(column)

From 731410b6c7c0c57386d0f1c8739e69344607469b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 4 Mar 2013 19:15:05 +0000
Subject: [PATCH 0817/1540] HBASE-7824 Revert until TestMasterFailover passes
 reliably

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452452 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  |   2 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 110 ++++-------
 .../hadoop/hbase/master/MasterFileSystem.java |  33 ++--
 .../hadoop/hbase/master/ServerManager.java    |  10 -
 .../master/handler/ServerShutdownHandler.java |   4 -
 .../hbase/regionserver/HRegionServer.java     |   7 +-
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |   2 +-
 .../master/TestDistributedLogSplitting.java   | 178 ++----------------
 .../TestRSKilledWhenMasterInitializing.java   |  25 ++-
 9 files changed, 96 insertions(+), 275 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 58c61ee1703d..5f2f1482c7f2 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -118,7 +118,7 @@ public class CatalogTracker {
    */
   private ServerName metaLocation;
 
-  private volatile boolean stopped = false;
+  private boolean stopped = false;
 
   static final byte [] ROOT_REGION_NAME =
     HRegionInfo.ROOT_REGIONINFO.getRegionName();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index ff509b85dabb..657e3717b9ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -41,6 +41,9 @@
 
 import javax.management.ObjectName;
 
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -126,10 +129,6 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 
-import com.google.common.collect.ClassToInstanceMap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MutableClassToInstanceMap;
-
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
  * master.  If many masters are started, all compete.  Whichever wins goes on to
@@ -566,37 +565,14 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     if (!masterRecovery) {
       this.assignmentManager.startTimeOutMonitor();
     }
+    // TODO: Should do this in background rather than block master startup
+    status.setStatus("Splitting logs after master startup");
+    splitLogAfterStartup(this.fileSystemManager);
 
-    // get a list for previously failed RS which need recovery work
-    Set<ServerName> failedServers = this.fileSystemManager.getFailedServersFromLogFolders();
-    ServerName preRootServer = this.catalogTracker.getRootLocation();
-    if (preRootServer != null && failedServers.contains(preRootServer)) {
-      // create recovered edits file for _ROOT_ server
-      this.fileSystemManager.splitLog(preRootServer);
-      failedServers.remove(preRootServer);
-    }
-
-    // Make sure root assigned before proceeding.
-    assignRoot(status);
-
-    // log splitting for .META. server
-    ServerName preMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-    if (preMetaServer != null && failedServers.contains(preMetaServer)) {
-      // create recovered edits file for .META. server
-      this.fileSystemManager.splitLog(preMetaServer);
-      failedServers.remove(preMetaServer);
-    }
-    // Make sure meta assigned before proceeding.
-    assignMeta(status, preRootServer);
-
+    // Make sure root and meta assigned before proceeding.
+    assignRootAndMeta(status);
     enableServerShutdownHandler();
 
-    // handle other dead servers in SSH
-    status.setStatus("Submit log splitting work of non-meta region servers");
-    for (ServerName curServer : failedServers) {
-      this.serverManager.processDeadServer(curServer);
-    }
-
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
     // before we assign all user regions or else the assignment will fail.
@@ -668,13 +644,22 @@ protected void startCatalogJanitorChore() {
   }
 
   /**
-   * Check <code>-ROOT-</code> is assigned. If not, assign it.
-   * @param status MonitoredTask
+   * Override to change master's splitLogAfterStartup. Used testing
+   * @param mfs
+   */
+  protected void splitLogAfterStartup(final MasterFileSystem mfs) {
+    mfs.splitLogAfterStartup();
+  }
+
+  /**
+   * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,
+   * assign them.
    * @throws InterruptedException
    * @throws IOException
    * @throws KeeperException
+   * @return Count of regions we assigned.
    */
-  private void assignRoot(MonitoredTask status)
+  int assignRootAndMeta(MonitoredTask status)
   throws InterruptedException, IOException, KeeperException {
     int assigned = 0;
     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
@@ -705,32 +690,16 @@ private void assignRoot(MonitoredTask status)
     LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getRootLocation());
 
-    status.setStatus("ROOT assigned.");
-  }
-
-  /**
-   * Check <code>.META.</code> is assigned. If not, assign it.
-   * @param status MonitoredTask
-   * @param previousRootServer ServerName of previous root region server before current start up
-   * @return
-   * @throws InterruptedException
-   * @throws IOException
-   * @throws KeeperException
-   */
-  private void assignMeta(MonitoredTask status, ServerName previousRootServer)
-      throws InterruptedException,
-      IOException, KeeperException {
-    int assigned = 0;
-    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
-
+    // Work on meta region
     status.setStatus("Assigning META region");
-    boolean rit =
-        this.assignmentManager
-            .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
+    rit = this.assignmentManager.
+      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
     boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
     if (!rit && !metaRegionLocation) {
-      ServerName currentMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-      if (currentMetaServer != null && !currentMetaServer.equals(previousRootServer)) {
+      ServerName currentMetaServer =
+        this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+      if (currentMetaServer != null
+          && !currentMetaServer.equals(currentRootServer)) {
         splitLogAndExpireIfOnline(currentMetaServer);
       }
       assignmentManager.assignMeta();
@@ -740,14 +709,15 @@ private void assignMeta(MonitoredTask status, ServerName previousRootServer)
       enableSSHandWaitForMeta();
       assigned++;
     } else {
-      // Region already assigned. We didnt' assign it. Add to in-memory state.
+      // Region already assigned.  We didnt' assign it.  Add to in-memory state.
       this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
         this.catalogTracker.getMetaLocation());
     }
     enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
-    LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
-        + catalogTracker.getMetaLocation());
-    status.setStatus("META assigned.");
+    LOG.info(".META. assigned=" + assigned + ", rit=" + rit +
+      ", location=" + catalogTracker.getMetaLocation());
+    status.setStatus("META and ROOT assigned.");
+    return assigned;
   }
 
   private void enableSSHandWaitForMeta() throws IOException,
@@ -806,7 +776,8 @@ public boolean visit(Result r) throws IOException {
   }
 
   /**
-   * Expire a server if we find it is one of the online servers.
+   * Split a server's log and expire it if we find it is one of the online
+   * servers.
    * @param sn ServerName to check.
    * @throws IOException
    */
@@ -1668,23 +1639,12 @@ public void shutdown() {
     }
     if (this.assignmentManager != null) this.assignmentManager.shutdown();
     if (this.serverManager != null) this.serverManager.shutdownCluster();
-
     try {
       if (this.clusterStatusTracker != null){
         this.clusterStatusTracker.setClusterDown();
       }
     } catch (KeeperException e) {
-      if (e instanceof KeeperException.SessionExpiredException) {
-        LOG.warn("ZK session expired. Retry a new connection...");
-        try {
-          this.zooKeeper.reconnectAfterExpiration();
-          this.clusterStatusTracker.setClusterDown();
-        } catch (Exception ex) {
-          LOG.warn("Retry setClusterDown failed", ex);
-        }
-      } else {
-        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
-      }
+      LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 5ed1559f8fd5..54ce8c59ab5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -21,7 +21,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
@@ -198,31 +197,30 @@ public String getClusterId() {
   }
 
   /**
-   * Inspect the log directory to find dead servers which need log splitting
+   * Inspect the log directory to recover any log file without
+   * an active region server.
    */
-  Set<ServerName> getFailedServersFromLogFolders() {
+  void splitLogAfterStartup() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
-      HLog.SPLIT_SKIP_ERRORS_DEFAULT);
-    
-    Set<ServerName> serverNames = new HashSet<ServerName>();
+        HLog.SPLIT_SKIP_ERRORS_DEFAULT);
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
-
     do {
       if (master.isStopped()) {
-        LOG.warn("Master stopped while trying to get failed servers.");
+        LOG.warn("Master stopped while splitting logs");
         break;
       }
+      List<ServerName> serverNames = new ArrayList<ServerName>();
       try {
-        if (!this.fs.exists(logsDirPath)) return serverNames;
+        if (!this.fs.exists(logsDirPath)) return;
         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
         // Get online servers after getting log folders to avoid log folder deletion of newly
         // checked in region servers . see HBASE-5916
-        Set<ServerName> onlineServers =
-            ((HMaster) master).getServerManager().getOnlineServers().keySet();
+        Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
+            .keySet();
 
         if (logFolders == null || logFolders.length == 0) {
           LOG.debug("No log files to split, proceeding...");
-          return serverNames;
+          return;
         }
         for (FileStatus status : logFolders) {
           String sn = status.getPath().getName();
@@ -236,19 +234,22 @@ Set<ServerName> getFailedServersFromLogFolders() {
                 + "to a known region server, splitting");
             serverNames.add(serverName);
           } else {
-            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
+            LOG.info("Log folder " + status.getPath()
+                + " belongs to an existing region server");
           }
         }
+        splitLog(serverNames);
         retrySplitting = false;
       } catch (IOException ioe) {
-        LOG.warn("Failed getting failed servers to be recovered.", ioe);
+        LOG.warn("Failed splitting of " + serverNames, ioe);
         if (!checkFileSystem()) {
           LOG.warn("Bad Filesystem, exiting");
           Runtime.getRuntime().halt(1);
         }
         try {
           if (retrySplitting) {
-            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
+            Thread.sleep(conf.getInt(
+              "hbase.hlog.split.failure.retry.interval", 30 * 1000));
           }
         } catch (InterruptedException e) {
           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
@@ -258,8 +259,6 @@ Set<ServerName> getFailedServersFromLogFolders() {
         }
       }
     } while (retrySplitting);
-
-    return serverNames;
   }
 
   public void splitLog(final ServerName serverName) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 6d5c45a3cdc7..039702007613 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -424,16 +424,6 @@ public synchronized void expireServer(final ServerName serverName) {
         carryingRoot + ", meta=" + carryingMeta);
   }
 
-  /**
-   * The function is to allow master to submit known dead servers into SSH
-   * @param serverName
-   */
-  void processDeadServer(final ServerName serverName) {
-    this.deadservers.add(serverName);
-    this.services.getExecutorService().submit(
-      new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName, true));
-  }
-
   /**
    * Expire the servers which died during master's initialization. It will be
    * called after HMaster#assignRootAndMeta.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index b545891c1d2f..8b889f7db921 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -179,10 +179,6 @@ public String toString() {
   public void process() throws IOException {
     final ServerName serverName = this.serverName;
     try {
-      if (this.server.isStopped()) {
-        throw new IOException("Server is stopped");
-      }
-
       try {
         if (this.shouldSplitHlog) {
           LOG.info("Splitting logs for " + serverName);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5371ad54df58..23b52c9962ee 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,12 +66,12 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterAddressTracker;
 import org.apache.hadoop.hbase.NotServingRegionException;
@@ -172,6 +172,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 import org.codehaus.jackson.map.ObjectMapper;
+import org.joda.time.field.MillisDurationField;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
@@ -1704,9 +1705,7 @@ public CatalogTracker getCatalogTracker() {
   @Override
   public void stop(final String msg) {
     try {
-      if (this.rsHost != null) {
-        this.rsHost.preStop(msg);
-      }
+      this.rsHost.preStop(msg);
       this.stopped = true;
       LOG.info("STOPPED: " + msg);
       // Wakes run() if it is sleeping
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index 4365f78e7172..c6e607ef1334 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -45,7 +45,7 @@ public abstract class ZooKeeperNodeTracker extends ZooKeeperListener {
   /** Used to abort if a fatal error occurs */
   protected final Abortable abortable;
 
-  private volatile boolean stopped = false;
+  private boolean stopped = false;
 
   /**
    * Constructs a new ZK node tracker.
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 36a3c547e137..823d7da933de 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -19,29 +19,22 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_mgr_wait_for_zk_delete;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_final_transistion_failed;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_done;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_err;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_resigned;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.TreeSet;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -50,13 +43,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -67,9 +54,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -96,21 +82,15 @@ public class TestDistributedLogSplitting {
   Configuration conf;
   HBaseTestingUtility TEST_UTIL;
 
-
   private void startCluster(int num_rs) throws Exception{
-    conf = HBaseConfiguration.create();
-    startCluster(NUM_MASTERS, num_rs, conf);
-  }
-
-  private void startCluster(int num_master, int num_rs, Configuration inConf) throws Exception {
     ZKSplitLog.Counters.resetCounters();
     LOG.info("Starting cluster");
-    this.conf = inConf;
+    conf = HBaseConfiguration.create();
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
     TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(num_master, num_rs);
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
@@ -122,10 +102,6 @@ private void startCluster(int num_master, int num_rs, Configuration inConf) thro
 
   @After
   public void after() throws Exception {
-    for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
-      mt.getMaster().abort("closing...", new Exception("Trace info"));
-    }
-
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -229,89 +205,6 @@ public void testRecoveredEdits() throws Exception {
     assertEquals(NUM_LOG_LINES, count);
   }
 
-  @Test(timeout = 300000)
-  public void testMasterStartsUpWithLogSplittingWork() throws Exception {
-    LOG.info("testMasterStartsUpWithLogSplittingWork");
-    Configuration curConf = HBaseConfiguration.create();
-    curConf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
-    startCluster(2, NUM_RS, curConf);
-
-    final int NUM_REGIONS_TO_CREATE = 40;
-    final int NUM_LOG_LINES = 1000;
-    // turn off load balancing to prevent regions from moving around otherwise
-    // they will consume recovered.edits
-    master.balanceSwitch(false);
-
-    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
-    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
-    HTable ht = installTable(zkw, "table", "f", NUM_REGIONS_TO_CREATE);
-
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = hrs.getOnlineRegions();
-      for (HRegionInfo region : regions) {
-        if (region.isRootRegion() || region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
-    makeHLog(hrs.getWAL(), regions, "table", NUM_LOG_LINES, 100);
-
-    // abort master
-    abortMaster(cluster);
-
-    // abort RS
-    int numRS = cluster.getLiveRegionServerThreads().size();
-    LOG.info("Aborting region server: " + hrs.getServerName());
-    hrs.abort("testing");
-
-    // wait for the RS dies
-    long start = EnvironmentEdgeManager.currentTimeMillis();
-    while (cluster.getLiveRegionServerThreads().size() > (numRS - 1)) {
-      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
-        assertTrue(false);
-      }
-      Thread.sleep(200);
-    }
-
-    Thread.sleep(2000);
-    LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
-    
-    startMasterAndWaitUntilLogSplit(cluster);
-    
-    start = EnvironmentEdgeManager.currentTimeMillis();
-    while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
-      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
-        assertTrue("Timedout", false);
-      }
-      Thread.sleep(200);
-    }
-
-    LOG.info("Current Open Regions After Master Node Starts Up:"
-        + getAllOnlineRegions(cluster).size());
-
-    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
-
-    ht.close();
-  }
-
   /**
    * The original intention of this test was to force an abort of a region
    * server and to make sure that the failure path in the region servers is
@@ -500,40 +393,33 @@ public void makeHLog(HLog log,
       List<HRegionInfo> hris, String tname,
       int num_edits, int edit_size) throws IOException {
 
-    // remove root and meta region
-    hris.remove(HRegionInfo.ROOT_REGIONINFO);
-    hris.remove(HRegionInfo.FIRST_META_REGIONINFO);
     byte[] table = Bytes.toBytes(tname);
     HTableDescriptor htd = new HTableDescriptor(tname);
     byte[] value = new byte[edit_size];
     for (int i = 0; i < edit_size; i++) {
-      value[i] = (byte) ('a' + (i % 26));
+      value[i] = (byte)('a' + (i % 26));
     }
     int n = hris.size();
     int[] counts = new int[n];
+    int j = 0;
     if (n > 0) {
       for (int i = 0; i < num_edits; i += 1) {
         WALEdit e = new WALEdit();
-        HRegionInfo curRegionInfo = hris.get(i % n);
-        byte[] startRow = curRegionInfo.getStartKey();
-        if (startRow == null || startRow.length == 0) {
-          startRow = new byte[] { 0, 0, 0, 0, 1 };
-        }
-        byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
-        row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because
-                                             // HBaseTestingUtility.createMultiRegions use 5 bytes
-                                             // key
-        byte[] family = Bytes.toBytes("f");
-        byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
-        e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
-        log.append(curRegionInfo, table, e, System.currentTimeMillis(), htd);
-        counts[i % n] += 1;
+        byte [] row = Bytes.toBytes("r" + Integer.toString(i));
+        byte [] family = Bytes.toBytes("f");
+        byte [] qualifier = Bytes.toBytes("c" + Integer.toString(i));
+        e.add(new KeyValue(row, family, qualifier,
+            System.currentTimeMillis(), value));
+        j++;
+        log.append(hris.get(j % n), table, e, System.currentTimeMillis(), htd);
+        counts[j % n] += 1;
       }
     }
     log.sync();
     log.close();
     for (int i = 0; i < n; i++) {
-      LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
+      LOG.info("region " + hris.get(i).getRegionNameAsString() +
+          " has " + counts[i] + " edits");
     }
     return;
   }
@@ -593,30 +479,6 @@ private void waitForCounter(AtomicLong ctr, long oldval, long newval,
     assertTrue(false);
   }
 
-  private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException {
-    for (MasterThread mt : cluster.getLiveMasterThreads()) {
-      if (mt.getMaster().isActiveMaster()) {
-        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
-        mt.join();
-        break;
-      }
-    }
-    LOG.debug("Master is aborted");
-  }
-
-  private void startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
-      throws IOException, InterruptedException {
-    cluster.startMaster();
-    HMaster master = cluster.getMaster();
-    while (!master.isInitialized()) {
-      Thread.sleep(100);
-    }
-    ServerManager serverManager = master.getServerManager();
-    while (serverManager.areDeadServersInProgress()) {
-      Thread.sleep(100);
-    }
-  }
-
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index eb2f42dccc87..a7287a1b84a2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -36,16 +37,19 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.TestMasterFailover;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -98,6 +102,19 @@ public TestingMaster(Configuration conf) throws IOException,
       super(conf);
     }
 
+    @Override
+    protected void splitLogAfterStartup(MasterFileSystem mfs) {
+      super.splitLogAfterStartup(mfs);
+      logSplit = true;
+      // If "TestingMaster.sleep" is set, sleep after log split.
+      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
+        int duration = getConfiguration().getInt(
+            "TestingMaster.sleep.duration", 0);
+        Threads.sleep(duration);
+      }
+    }
+
+
     public boolean isLogSplitAfterStartup() {
       return logSplit;
     }
@@ -232,13 +249,11 @@ private void abortMaster(MiniHBaseCluster cluster)
   private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
       throws IOException, InterruptedException {
     TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
-    while (!master.isInitialized()) {
-      Thread.sleep(100);
-    }
-    ServerManager serverManager = cluster.getMaster().getServerManager();
-    while (serverManager.areDeadServersInProgress()) {
+    while (!master.isLogSplitAfterStartup()) {
       Thread.sleep(100);
     }
+    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
+        + master.isInitialized());
     return master;
   }
 

From 57b434ddb1e7a90180a081863b91706792f043e0 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 4 Mar 2013 19:41:12 +0000
Subject: [PATCH 0818/1540] HBASE-7991 Backport HBASE-6479 'HFileReaderV1
 caching the same parent META block could cause server abort when splitting'
 to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452465 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |  5 ++-
 .../hadoop/hbase/HBaseTestingUtility.java     | 16 ++++++++
 .../regionserver/TestSplitTransaction.java    | 38 +++++++++++++++++--
 3 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index f642e3dbc222..2f40db9a7679 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -679,8 +679,9 @@ public HFileBlock readBlock(long offset, long onDiskBlockSize,
 
   @Override
   public DataInput getGeneralBloomFilterMetadata() throws IOException {
-    // Always cache Bloom filter blocks.
-    ByteBuffer buf = getMetaBlock(HFileWriterV1.BLOOM_FILTER_META_KEY, true);
+    // Shouldn't cache Bloom filter blocks, otherwise server would abort when
+    // splitting, see HBASE-6479
+    ByteBuffer buf = getMetaBlock(HFileWriterV1.BLOOM_FILTER_META_KEY, false);
     if (buf == null)
       return null;
     ByteArrayInputStream bais = new ByteArrayInputStream(buf.array(),
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 03161a010278..7e4bffa5ea47 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1056,6 +1056,19 @@ public int loadTable(final HTable t, final byte[][] f) throws IOException {
    */
   public int loadRegion(final HRegion r, final byte[] f)
   throws IOException {
+    return loadRegion(r, f, false);
+  }
+  
+  /**
+   * Load region with rows from 'aaa' to 'zzz'.
+   * @param r Region
+   * @param f Family
+   * @param flush flush the cache if true
+   * @return Count of rows loaded.
+   * @throws IOException
+   */
+  public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
+      throws IOException {
     byte[] k = new byte[3];
     int rowCount = 0;
     for (byte b1 = 'a'; b1 <= 'z'; b1++) {
@@ -1071,6 +1084,9 @@ public int loadRegion(final HRegion r, final byte[] f)
           rowCount++;
         }
       }
+      if (flush) {
+        r.flushcache();
+      }
     }
     return rowCount;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index b2b7cceb9361..b935cd2d012b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -19,8 +19,6 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import com.google.common.collect.ImmutableList;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -33,8 +31,18 @@
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.PairOfSameType;
@@ -45,6 +53,8 @@
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
+import com.google.common.collect.ImmutableList;
+
 /**
  * Test the {@link SplitTransaction} class against an HRegion (as opposed to
  * running cluster).
@@ -182,12 +192,32 @@ private SplitTransaction prepareGOOD_SPLIT_ROW() {
     assertFalse(st.prepare());
   }
 
+  @Test public void testWholesomeSplitWithHFileV1() throws IOException {
+    int defaultVersion = TEST_UTIL.getConfiguration().getInt(
+        HFile.FORMAT_VERSION_KEY, 2);
+    TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 1);
+    try {
+      for (Store store : this.parent.stores.values()) {
+        store.getFamily().setBloomFilterType(StoreFile.BloomType.ROW);
+      }
+      testWholesomeSplit();
+    } finally {
+      TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY,
+          defaultVersion);
+    }
+  }
+
   @Test public void testWholesomeSplit() throws IOException {
-    final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
+    final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true);
     assertTrue(rowcount > 0);
     int parentRowCount = countRows(this.parent);
     assertEquals(rowcount, parentRowCount);
 
+    // Pretend region's blocks are not in the cache, used for
+    // testWholesomeSplitWithHFileV1
+    CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
+    ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
+
     // Start transaction.
     SplitTransaction st = prepareGOOD_SPLIT_ROW();
 

From c7f0b83104a2a82aefd1857751d8f951542eafa7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 4 Mar 2013 21:36:47 +0000
Subject: [PATCH 0819/1540] HBASE-7360 Addendum, test fix for TestMultiParallel
 (Matteo)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452537 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestMultiParallel.java       | 33 +++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
index c36272f4bcc4..ae5ff1f80c2f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
@@ -57,6 +57,7 @@ public class TestMultiParallel {
     UTIL.startMiniCluster(slaves);
     HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
     UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
+    UTIL.waitTableAvailable(Bytes.toBytes(TEST_TABLE), 15 * 1000);
     t.close();
   }
 
@@ -218,6 +219,11 @@ public void testFlushCommitsNoAbort() throws Exception {
     doTestFlushCommits(false);
   }
 
+  /**
+   * Set table auto flush to false and test flushing commits
+   * @param doAbort true if abort one regionserver in the testing
+   * @throws Exception
+   */
   private void doTestFlushCommits(boolean doAbort) throws Exception {
     // Load the data
     LOG.info("get new table");
@@ -232,9 +238,25 @@ private void doTestFlushCommits(boolean doAbort) throws Exception {
     }
     LOG.info("puts");
     table.flushCommits();
+    int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
+        .size();
+    assert liveRScount > 0;
+    JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
+        .getLiveRegionServerThreads().get(0);
     if (doAbort) {
       LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
 
+      // If we waiting for no regions being online after we abort the server, we
+      // could ensure the master has re-assigned the regions on killed server
+      // after putting keys successfully, it means the server we abort is dead
+      // and detected by matser
+      while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
+        Thread.sleep(100);
+      }
+      while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == liveRScount) {
+        Thread.sleep(100);
+      }
+
       // try putting more keys after the abort. same key/qual... just validating
       // no exceptions thrown
       puts = constructPutRequests();
@@ -258,7 +280,7 @@ private void doTestFlushCommits(boolean doAbort) throws Exception {
     }
     LOG.info("Count=" + count);
     Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
-      (doAbort ? 1 : 2), count);
+        (doAbort ? (liveRScount - 1) : liveRScount), count);
     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
       int regions = t.getRegionServer().getOnlineRegions().size();
       Assert.assertTrue("Count of regions=" + regions, regions > 10);
@@ -279,7 +301,13 @@ public void testBatchWithPut() throws Exception {
     validateSizeAndEmpty(results, KEYS.length);
 
     if (true) {
-      UTIL.getMiniHBaseCluster().abortRegionServer(0);
+      int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
+          .size();
+      assert liveRScount > 0;
+      JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
+          .getLiveRegionServerThreads().get(0);
+      liveRS.getRegionServer().abort("Aborting for tests",
+          new Exception("testBatchWithPut"));
 
       puts = constructPutRequests();
       results = table.batch(puts);
@@ -416,6 +444,7 @@ public void testBatchWithIncrementAndAppend() throws Exception {
     validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
     validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
     validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
+    table.close();
   }
 
   @Test(timeout=300000)

From 22ea1fd9301414c05c3986431b7236c1251c0e1e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 4 Mar 2013 21:46:12 +0000
Subject: [PATCH 0820/1540] HBASE-6347 -ROOT- and .META. are stale in table.jsp
 if they moved

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452546 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/resources/hbase-webapps/master/table.jsp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 69a2e34bc402..a2e20103f3a2 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -131,7 +131,7 @@
 <%
   // NOTE: Presumes one meta region only.
   HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
-  ServerName metaLocation = master.getCatalogTracker().getMetaLocation();
+  ServerName metaLocation = master.getCatalogTracker().waitForMeta(1);
   for (int i = 0; i < 1; i++) {
     String url = "http://" + metaLocation.getHostname() + ":" + infoPort + "/";
 %>

From b2e1ab7e0e4df406633fd1aac768ae8b491b8e2f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 4 Mar 2013 21:54:26 +0000
Subject: [PATCH 0821/1540] HBASE-7111 hbase zkcli will not start if the
 zookeeper server chosen to connect to is unavailable (Zhou wenjian)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452552 13f79535-47bb-0310-9956-ffa450edef68
---
 .../zookeeper/ZooKeeperMainServerArg.java     | 20 ++++++++++++++-----
 .../zookeeper/TestZooKeeperMainServerArg.java |  3 ++-
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java
index c662a5b8c2fe..e6f7f16a773d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java
@@ -20,8 +20,10 @@
 
 package org.apache.hadoop.hbase.zookeeper;
 
-import java.util.Properties;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map.Entry;
+import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -41,18 +43,26 @@ public String parse(final Configuration c) {
     Properties zkProps = ZKConfig.makeZKProps(c);
     String host = null;
     String clientPort = null;
+    List<String> hosts = new ArrayList<String>();
     for (Entry<Object, Object> entry: zkProps.entrySet()) {
       String key = entry.getKey().toString().trim();
       String value = entry.getValue().toString().trim();
-      if (key.startsWith("server.") && host == null) {
+      if (key.startsWith("server.")) {
         String[] parts = value.split(":");
-        host = parts[0];
+        hosts.add(parts[0]);
       } else if (key.endsWith("clientPort")) {
         clientPort = value;
       }
-      if (host != null && clientPort != null) break;
     }
-    return host != null && clientPort != null? host + ":" + clientPort: null;
+    if (hosts.isEmpty() || clientPort == null)
+      return null;
+    for (int i = 0; i < hosts.size(); i++) {
+      if (i > 0)
+        host += "," + hosts.get(i);
+      else
+        host = hosts.get(i);
+    }
+    return host != null ? host + ":" + clientPort : null;
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java
index 96073ce46158..ddc8dc4bac79 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java
@@ -40,7 +40,8 @@ public class TestZooKeeperMainServerArg {
     c.set("hbase.zookeeper.quorum", "example.com");
     assertEquals("example.com:" + port, parser.parse(c));
     c.set("hbase.zookeeper.quorum", "example1.com,example2.com,example3.com");
-    assertTrue(port, parser.parse(c).matches("example[1-3]\\.com:" + port));
+    assertTrue(port,
+        parser.parse(c).matches("(example[1-3]\\.com,){2}example[1-3]\\.com:" + port));
   }
 
   @org.junit.Rule

From c8428216d577843f68bed01d2cd7309067fd329e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 5 Mar 2013 01:59:31 +0000
Subject: [PATCH 0822/1540] HBASE-7990 Revert. Needs more refactoring in HBase
 ruby client.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452630 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/table.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index 082214faa2ee..ed4113228565 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -53,7 +53,6 @@ def delete(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_T
     #----------------------------------------------------------------------------------------------
     # Delete a row
     def deleteall(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP)
-      raise ArgumentError, "Row Not Found" if _get_internal(row).nil?
       d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp, nil)
       if column
         family, qualifier = parse_column_name(column)

From e1430073c6519235e889569ee53c238990dad290 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 5 Mar 2013 03:20:04 +0000
Subject: [PATCH 0823/1540] HBASE-7818 add region level metrics
 readReqeustCount and writeRequestCount (Tianying Chang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452648 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 42 +++++++++++++++++--
 .../hbase/regionserver/HRegionServer.java     |  1 +
 .../hbase/regionserver/SplitTransaction.java  |  8 +++-
 .../metrics/OperationMetrics.java             | 33 +++++++++++++--
 .../metrics/RegionMetricsStorage.java         | 17 ++++++--
 5 files changed, 90 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 8f85fcc6713c..6eb5f5e9e872 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1040,7 +1040,7 @@ public ImmutableList<StoreFile> call() throws IOException {
         status.setStatus("Running coprocessor post-close hooks");
         this.coprocessorHost.postClose(abort);
       }
-      this.opMetrics.closeMetrics();
+      this.opMetrics.closeMetrics(this.getRegionInfo().getEncodedName());
       status.markComplete("Closed");
       LOG.info("Closed " + this);
       return result;
@@ -1688,6 +1688,7 @@ public Result getClosestRowBefore(final byte [] row, final byte [] family)
     checkRow(row, "getClosestRowBefore");
     startRegionOperation();
     this.readRequestsCount.increment();
+    this.opMetrics.setReadRequestCountMetrics(this.readRequestsCount.get());   
     try {
       Store store = getStore(family);
       // get the closest key. (HStore.getRowKeyAtOrBefore can return null)
@@ -1734,6 +1735,7 @@ protected RegionScanner getScanner(Scan scan,
       List<KeyValueScanner> additionalScanners) throws IOException {
     startRegionOperation();
     this.readRequestsCount.increment();
+    this.opMetrics.setReadRequestCountMetrics(this.readRequestsCount.get());
     try {
       // Verify families are all valid
       prepareScanner(scan);
@@ -1801,6 +1803,7 @@ public void delete(Delete delete, Integer lockid, boolean writeToWAL)
     Integer lid = null;
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
     try {
       byte [] row = delete.getRow();
       // If we did not pass an existing row lock, obtain a new one
@@ -1993,6 +1996,7 @@ public void put(Put put, Integer lockid, boolean writeToWAL)
     checkResources();
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
     try {
       // We obtain a per-row lock, so other clients will block while one client
       // performs an update. The read lock is released by the client calling
@@ -2093,6 +2097,7 @@ public OperationStatus[] batchMutate(
       try {
         if (!initialized) {
           this.writeRequestsCount.increment();
+          this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
           doPreMutationHook(batchOp);
           initialized = true;
         }
@@ -2519,6 +2524,7 @@ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
 
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
     try {
       RowLock lock = isPut ? ((Put)w).getRowLock() : ((Delete)w).getRowLock();
       Get get = new Get(row, lock);
@@ -3341,6 +3347,7 @@ void checkRow(final byte [] row, String op) throws IOException {
   public Integer obtainRowLock(final byte [] row) throws IOException {
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics( this.writeRequestsCount.get());
     try {
       return internalObtainRowLock(row, true);
     } finally {
@@ -3513,6 +3520,7 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
     startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));
     try {
       this.writeRequestsCount.increment();
+      this.opMetrics.setWriteRequestCountMetrics( this.writeRequestsCount.get());
 
       // There possibly was a split that happend between when the split keys
       // were gathered and before the HReiogn's write lock was taken.  We need
@@ -3737,6 +3745,7 @@ public synchronized boolean next(List<KeyValue> outResults, int limit,
       }
       startRegionOperation();
       readRequestsCount.increment();
+      opMetrics.setReadRequestCountMetrics(readRequestsCount.get());
       try {
 
         // This could be a new thread from the last time we called next().
@@ -4555,8 +4564,14 @@ public static HRegion merge(HRegion a, HRegion b)
     }
     HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf,
         newRegionInfo, a.getTableDesc(), null);
-    dstRegion.readRequestsCount.set(a.readRequestsCount.get() + b.readRequestsCount.get());
-    dstRegion.writeRequestsCount.set(a.writeRequestsCount.get() + b.writeRequestsCount.get());
+    long totalReadRequestCount = a.readRequestsCount.get() + b.readRequestsCount.get();
+    dstRegion.readRequestsCount.set(totalReadRequestCount);
+    dstRegion.opMetrics.setReadRequestCountMetrics(totalReadRequestCount);
+    
+    long totalWriteRequestCount = a.writeRequestsCount.get() + b.writeRequestsCount.get();
+    dstRegion.writeRequestsCount.set(totalWriteRequestCount);
+    dstRegion.opMetrics.setWriteRequestCountMetrics(totalWriteRequestCount);
+    
     dstRegion.initialize();
     dstRegion.compactStores();
     if (LOG.isDebugEnabled()) {
@@ -4946,6 +4961,7 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
     // Lock row
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
     try {
       Integer lid = getLock(lockid, row, true);
       lock(this.updatesLock.readLock());
@@ -5116,6 +5132,7 @@ public Result increment(Increment increment, Integer lockid,
     // Lock row
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
     try {
       Integer lid = getLock(lockid, row, true);
       lock(this.updatesLock.readLock());
@@ -5234,6 +5251,7 @@ public long incrementColumnValue(byte [] row, byte [] family,
     long result = amount;
     startRegionOperation();
     this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
     try {
       Integer lid = obtainRowLock(row);
       lock(this.updatesLock.readLock());
@@ -5619,6 +5637,24 @@ public RegionCoprocessorHost getCoprocessorHost() {
     return coprocessorHost;
   }
 
+  /*
+   * Set the read request count defined in opMetrics
+   * @param value absolute value of read request count
+   */
+  public void setOpMetricsReadRequestCount(long value)
+  {
+    this.opMetrics.setReadRequestCountMetrics(value);
+  }
+  
+  /*
+   * Set the write request count defined in opMetrics
+   * @param value absolute value of write request count
+   */
+  public void setOpMetricsWriteRequestCount(long value)
+  {
+    this.opMetrics.setWriteRequestCountMetrics(value);
+  }
+  
   /** @param coprocessorHost the new coprocessor host */
   public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {
     this.coprocessorHost = coprocessorHost;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 23b52c9962ee..455da6f5cc0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2524,6 +2524,7 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
         }
         requestCount.addAndGet(i);
         region.readRequestsCount.add(i);
+        region.setOpMetricsReadRequestCount(region.readRequestsCount.get());
       } finally {
         region.closeRegionOperation();
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 044c8f096c88..cae5d48d639d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -700,8 +700,12 @@ HRegion createDaughterRegion(final HRegionInfo hri,
     HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
       this.parent.getLog(), fs, this.parent.getConf(),
       hri, this.parent.getTableDesc(), rsServices);
-    r.readRequestsCount.set(this.parent.getReadRequestsCount() / 2);
-    r.writeRequestsCount.set(this.parent.getWriteRequestsCount() / 2);
+    long halfParentReadRequestCount = this.parent.getReadRequestsCount() / 2;
+    r.readRequestsCount.set(halfParentReadRequestCount);
+    r.setOpMetricsReadRequestCount(halfParentReadRequestCount);
+    long halfParentWriteRequest = this.parent.getWriteRequestsCount() / 2;
+    r.writeRequestsCount.set(halfParentWriteRequest);
+    r.setOpMetricsWriteRequestCount(halfParentWriteRequest);    
     HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
     return r;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
index eacb702d9f08..9e9985731f65 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java
@@ -46,6 +46,8 @@ public class OperationMetrics {
   private static final String MULTIPUT_KEY = "multiput_";
   private static final String MULTIDELETE_KEY = "multidelete_";
   private static final String APPEND_KEY = "append_";
+  private static final String READREQUESTCOUNT_KEY = "readrequestcount";
+  private static final String WRITEREQUESTCOUNT_KEY = "writerequestcount";
   
   /** Conf key controlling whether we should expose metrics.*/
   private static final String CONF_KEY =
@@ -98,6 +100,27 @@ public OperationMetrics() {
     this(null, null);
   }
 
+    /*
+     * This is used in set the read request count that is going to be exposed to 
+     * hadoop metric framework.
+     * @param value absolute value of read account
+     */
+    public void setReadRequestCountMetrics(long value) {
+      doSetNumericPersistentMetrics(READREQUESTCOUNT_KEY, value);
+    }
+
+    /*
+     * This is used in set the read request count that is going to be exposed to 
+     * hadoop metric framework.
+     * @param value absolute value of write account
+     */
+    public void setWriteRequestCountMetrics(long value) {
+      doSetNumericPersistentMetrics(WRITEREQUESTCOUNT_KEY, value);
+    }
+    
+    private void doSetNumericPersistentMetrics(String key, long value) {      
+       RegionMetricsStorage.setNumericPersistentMetric(this.regionMetrixPrefix+key, value); 
+    }    
 
   /**
    * Update the stats associated with {@link HTable#put(java.util.List)}.
@@ -190,11 +213,15 @@ public void updateDeleteMetrics(Set<byte[]> columnFamilies, long value) {
     doUpdateTimeVarying(columnFamilies, DELETE_KEY, value);
   }
   
+
+
   /**
-   * This deletes all old metrics this instance has ever created or updated.
+   * This deletes all old non-persistent metrics this instance has ever created or updated.
+   * for persistent metrics, only delete for the region to be closed
+   * @param regionEncodedName the region that is to be closed
    */
-  public void closeMetrics() {
-    RegionMetricsStorage.clear();
+  public void closeMetrics(String regionEncodedName) {
+    RegionMetricsStorage.clear(regionEncodedName);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
index 32395407ca58..6e65cb4008c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver.metrics;
 
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -88,6 +89,9 @@ public static void incrTimeVaryingMetric(String key, long amount) {
     oldVal.getSecond().incrementAndGet(); // increment ops by 1
   }
 
+  public static void setNumericPersistentMetric(String key, long amount) {
+    numericPersistentMetrics.put(key, new AtomicLong(amount));
+  }
   public static void incrNumericPersistentMetric(String key, long amount) {
     AtomicLong oldVal = numericPersistentMetrics.get(key);
     if (oldVal == null) {
@@ -126,11 +130,18 @@ public static long getNumericPersistentMetric(String key) {
   }
 
   /**
-   * Clear all copies of the metrics this stores.
+   * Clear the timevarying and numeric metrics for all regions in this region server
+   * Clear the numericPersistentMerics for only the region being closed.
    */
-  public static void clear() {
+  public static void clear(String regionEncodedName) {
     timeVaryingMetrics.clear();
     numericMetrics.clear();
-    numericPersistentMetrics.clear();
+    for (Entry<String, AtomicLong> entry : RegionMetricsStorage.getNumericPersistentMetrics().entrySet()) {
+     if (entry.getKey().contains(regionEncodedName))
+     {
+       String keyName = entry.getKey();
+       numericPersistentMetrics.remove(keyName);
+     }
+    }
   }
 }

From b93b12234173792ed5f21c16b32031043ff90124 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 5 Mar 2013 12:47:06 +0000
Subject: [PATCH 0824/1540] HBASE-7986. [REST] Make HTablePool size
 configurable (binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1452772 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
index 89c32a3c9ea9..91dca78d21eb 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
@@ -70,7 +70,8 @@ public synchronized static void stop() {
    */
   RESTServlet(Configuration conf) throws IOException {
     this.conf = conf;
-    this.pool = new HTablePool(conf, 10);
+    int maxSize = conf.getInt("hbase.rest.htablepool.size", 10);
+    this.pool = new HTablePool(conf, maxSize);
     this.admin = new HBaseAdmin(conf);
   }
 

From 79db49acf78977b13be8b5294d1d15f1e494a921 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 5 Mar 2013 22:49:45 +0000
Subject: [PATCH 0825/1540] HBASE-7944 Addendum (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1453064 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/ReplicationHLogReaderManager.java   | 1 +
 .../hadoop/hbase/replication/regionserver/ReplicationSource.java | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
index ae5c3bdc839e..06b4935caf27 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -63,6 +63,7 @@ public HLog.Reader openReader(Path path) throws IOException {
     // Detect if this is a new file, if so get a new reader else
     // reset the current reader so that we see the new data
     if (this.reader == null || !this.lastPath.equals(path)) {
+      this.closeReader();
       this.reader = HLog.getReader(this.fs, path, this.conf);
       this.lastPath = path;
     } else {
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 138dfd260bf5..84648fb48a6c 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -363,6 +363,7 @@ public void run() {
         }
       } finally {
         try {
+          this.reader = null;
           this.repLogReader.closeReader();
         } catch (IOException e) {
           gotIOE = true;

From 60c2502408fb9834ee2559e0438c7fff0aad207d Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 6 Mar 2013 00:22:13 +0000
Subject: [PATCH 0826/1540] HBASE-8007 Adopt TestLoadAndVerify from BigTop

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1453104 13f79535-47bb-0310-9956-ffa450edef68
---
 .../test/IntegrationTestLoadAndVerify.java    | 460 ++++++++++++++++++
 1 file changed, 460 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java

diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
new file mode 100644
index 000000000000..3bf7f24ed4d8
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
@@ -0,0 +1,460 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.IntegrationTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.NMapInputFormat;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+/**
+ * A large test which loads a lot of data that has internal references, and
+ * verifies the data.
+ *
+ * In load step, 200 map tasks are launched, which in turn write loadmapper.num_to_write
+ * (default 100K) rows to an hbase table. Rows are written in blocks, for a total of
+ * 100 blocks. Each row in a block, contains loadmapper.backrefs (default 50) references
+ * to random rows in the prev block.
+ *
+ * Verify step is scans the table, and verifies that for every referenced row, the row is
+ * actually there (no data loss). Failed rows are output from reduce to be saved in the
+ * job output dir in hdfs and inspected later.
+ *
+ * This class can be run as a unit test, as an integration test, or from the command line
+ *
+ * Originally taken from Apache Bigtop.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestLoadAndVerify  extends Configured implements Tool {
+  private static final String TEST_NAME = "IntegrationTestLoadAndVerify";
+  private static final byte[] TEST_FAMILY = Bytes.toBytes("f1");
+  private static final byte[] TEST_QUALIFIER = Bytes.toBytes("q1");
+
+  private static final String NUM_TO_WRITE_KEY =
+    "loadmapper.num_to_write";
+  private static final long NUM_TO_WRITE_DEFAULT = 100*1000;
+
+  private static final String TABLE_NAME_KEY = "loadmapper.table";
+  private static final String TABLE_NAME_DEFAULT = "table";
+
+  private static final String NUM_BACKREFS_KEY = "loadmapper.backrefs";
+  private static final int NUM_BACKREFS_DEFAULT = 50;
+
+  private static final String NUM_MAP_TASKS_KEY = "loadmapper.map.tasks";
+  private static final String NUM_REDUCE_TASKS_KEY = "verify.reduce.tasks";
+  private static final int NUM_MAP_TASKS_DEFAULT = 200;
+  private static final int NUM_REDUCE_TASKS_DEFAULT = 35;
+
+  private static final int SCANNER_CACHING = 500;
+
+  private IntegrationTestingUtility util;
+
+  private enum Counters {
+    ROWS_WRITTEN,
+    REFERENCES_WRITTEN,
+    REFERENCES_CHECKED;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    util = getTestingUtil();
+    util.initializeCluster(3);
+    this.setConf(util.getConfiguration());
+    getConf().setLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT / 100);
+    getConf().setInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT / 100);
+    getConf().setInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT / 10);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    util.restoreCluster();
+  }
+
+  /**
+   * Converts a "long" value between endian systems.
+   * Borrowed from Apache Commons IO
+   * @param value value to convert
+   * @return the converted value
+   */
+  public static long swapLong(long value)
+  {
+    return
+      ( ( ( value >> 0 ) & 0xff ) << 56 ) +
+      ( ( ( value >> 8 ) & 0xff ) << 48 ) +
+      ( ( ( value >> 16 ) & 0xff ) << 40 ) +
+      ( ( ( value >> 24 ) & 0xff ) << 32 ) +
+      ( ( ( value >> 32 ) & 0xff ) << 24 ) +
+      ( ( ( value >> 40 ) & 0xff ) << 16 ) +
+      ( ( ( value >> 48 ) & 0xff ) << 8 ) +
+      ( ( ( value >> 56 ) & 0xff ) << 0 );
+  }
+
+  public static class LoadMapper
+      extends Mapper<NullWritable, NullWritable, NullWritable, NullWritable>
+  {
+    private long recordsToWrite;
+    private HTable table;
+    private Configuration conf;
+    private int numBackReferencesPerRow;
+    private String shortTaskId;
+
+    private Random rand = new Random();
+
+    private Counter rowsWritten, refsWritten;
+
+    @Override
+    public void setup(Context context) throws IOException {
+      conf = context.getConfiguration();
+      recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT);
+      String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT);
+      numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT);
+      table = new HTable(conf, tableName);
+      table.setWriteBufferSize(4*1024*1024);
+      table.setAutoFlush(false);
+
+      String taskId = conf.get("mapred.task.id");
+      Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId);
+      if (!matcher.matches()) {
+        throw new RuntimeException("Strange task ID: " + taskId);
+      }
+      shortTaskId = matcher.group(1);
+
+      rowsWritten = context.getCounter(Counters.ROWS_WRITTEN);
+      refsWritten = context.getCounter(Counters.REFERENCES_WRITTEN);
+    }
+
+    @Override
+    public void cleanup(Context context) throws IOException {
+      table.flushCommits();
+      table.close();
+    }
+
+    @Override
+    protected void map(NullWritable key, NullWritable value,
+        Context context) throws IOException, InterruptedException {
+
+      String suffix = "/" + shortTaskId;
+      byte[] row = Bytes.add(new byte[8], Bytes.toBytes(suffix));
+
+      int BLOCK_SIZE = (int)(recordsToWrite / 100);
+
+      for (long i = 0; i < recordsToWrite;) {
+        long blockStart = i;
+        for (long idxInBlock = 0;
+             idxInBlock < BLOCK_SIZE && i < recordsToWrite;
+             idxInBlock++, i++) {
+
+          long byteSwapped = swapLong(i);
+          Bytes.putLong(row, 0, byteSwapped);
+
+          Put p = new Put(row);
+          p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY);
+          if (blockStart > 0) {
+            for (int j = 0; j < numBackReferencesPerRow; j++) {
+              long referredRow = blockStart - BLOCK_SIZE + rand.nextInt(BLOCK_SIZE);
+              Bytes.putLong(row, 0, swapLong(referredRow));
+              p.add(TEST_FAMILY, row, HConstants.EMPTY_BYTE_ARRAY);
+            }
+            refsWritten.increment(1);
+          }
+          rowsWritten.increment(1);
+          table.put(p);
+
+          if (i % 100 == 0) {
+            context.setStatus("Written " + i + "/" + recordsToWrite + " records");
+            context.progress();
+          }
+        }
+        // End of block, flush all of them before we start writing anything
+        // pointing to these!
+        table.flushCommits();
+      }
+    }
+  }
+
+  public static class VerifyMapper extends TableMapper<BytesWritable, BytesWritable> {
+    static final BytesWritable EMPTY = new BytesWritable(HConstants.EMPTY_BYTE_ARRAY);
+
+    @Override
+    protected void map(ImmutableBytesWritable key, Result value, Context context)
+        throws IOException, InterruptedException {
+      BytesWritable bwKey = new BytesWritable(key.get());
+      BytesWritable bwVal = new BytesWritable();
+      for (KeyValue kv : value.list()) {
+        if (Bytes.compareTo(TEST_QUALIFIER, 0, TEST_QUALIFIER.length,
+                            kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()) == 0) {
+          context.write(bwKey, EMPTY);
+        } else {
+          bwVal.set(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
+          context.write(bwVal, bwKey);
+        }
+      }
+    }
+  }
+
+  public static class VerifyReducer extends Reducer<BytesWritable, BytesWritable, Text, Text> {
+    private Counter refsChecked;
+    private Counter rowsWritten;
+
+    @Override
+    public void setup(Context context) throws IOException {
+      refsChecked = context.getCounter(Counters.REFERENCES_CHECKED);
+      rowsWritten = context.getCounter(Counters.ROWS_WRITTEN);
+    }
+
+    @Override
+    protected void reduce(BytesWritable referredRow, Iterable<BytesWritable> referrers,
+        VerifyReducer.Context ctx) throws IOException, InterruptedException {
+      boolean gotOriginalRow = false;
+      int refCount = 0;
+
+      for (BytesWritable ref : referrers) {
+        if (ref.getLength() == 0) {
+          assert !gotOriginalRow;
+          gotOriginalRow = true;
+        } else {
+          refCount++;
+        }
+      }
+      refsChecked.increment(refCount);
+
+      if (!gotOriginalRow) {
+        String parsedRow = makeRowReadable(referredRow.getBytes(), referredRow.getLength());
+        String binRow = Bytes.toStringBinary(referredRow.getBytes(), 0, referredRow.getLength());
+        ctx.write(new Text(binRow), new Text(parsedRow));
+        rowsWritten.increment(1);
+      }
+    }
+
+    private String makeRowReadable(byte[] bytes, int length) {
+      long rowIdx = swapLong(Bytes.toLong(bytes, 0));
+      String suffix = Bytes.toString(bytes, 8, length - 8);
+
+      return "Row #" + rowIdx + " suffix " + suffix;
+    }
+  }
+
+  private void doLoad(Configuration conf, HTableDescriptor htd) throws Exception {
+    Path outputDir = getTestDir(TEST_NAME, "load-output");
+
+    NMapInputFormat.setNumMapTasks(conf, conf.getInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT));
+    conf.set(TABLE_NAME_KEY, htd.getNameAsString());
+
+    Job job = new Job(conf);
+    job.setJobName(TEST_NAME + " Load for " + htd.getNameAsString());
+    job.setJarByClass(this.getClass());
+    job.setMapperClass(LoadMapper.class);
+    job.setInputFormatClass(NMapInputFormat.class);
+    job.setNumReduceTasks(0);
+    FileOutputFormat.setOutputPath(job, outputDir);
+
+    TableMapReduceUtil.addDependencyJars(job);
+    TableMapReduceUtil.addDependencyJars(
+        job.getConfiguration(), HTable.class, Lists.class);
+    TableMapReduceUtil.initCredentials(job);
+    assertTrue(job.waitForCompletion(true));
+  }
+
+  private void doVerify(Configuration conf, HTableDescriptor htd) throws Exception {
+    Path outputDir = getTestDir(TEST_NAME, "verify-output");
+
+    Job job = new Job(conf);
+    job.setJarByClass(this.getClass());
+    job.setJobName(TEST_NAME + " Verification for " + htd.getNameAsString());
+
+    Scan scan = new Scan();
+
+    TableMapReduceUtil.initTableMapperJob(
+        htd.getNameAsString(), scan, VerifyMapper.class,
+        BytesWritable.class, BytesWritable.class, job);
+    int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
+    TableMapReduceUtil.setScannerCaching(job, scannerCaching);
+
+    job.setReducerClass(VerifyReducer.class);
+    job.setNumReduceTasks(conf.getInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT));
+    FileOutputFormat.setOutputPath(job, outputDir);
+    assertTrue(job.waitForCompletion(true));
+
+    long numOutputRecords = job.getCounters().findCounter(Counters.ROWS_WRITTEN).getValue();
+    assertEquals(0, numOutputRecords);
+  }
+
+  public Path getTestDir(String testName, String subdir) throws IOException {
+    //HBaseTestingUtility.getDataTestDirOnTestFs() has not been backported.
+    FileSystem fs = FileSystem.get(getConf());
+    Path base = new Path(fs.getWorkingDirectory(), "test-data");
+    String randomStr = UUID.randomUUID().toString();
+    Path testDir = new Path(base, randomStr);
+    fs.deleteOnExit(testDir);
+
+    return new Path(new Path(testDir, testName), subdir);
+  }
+
+  @Test
+  public void testLoadAndVerify() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor(TEST_NAME);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+    HBaseAdmin admin = getTestingUtil().getHBaseAdmin();
+    int numPreCreate = 40;
+    admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPreCreate);
+
+    doLoad(getConf(), htd);
+    doVerify(getConf(), htd);
+
+    // Only disable and drop if we succeeded to verify - otherwise it's useful
+    // to leave it around for post-mortem
+    deleteTable(admin, htd);
+  }
+
+  private void deleteTable(HBaseAdmin admin, HTableDescriptor htd)
+    throws IOException, InterruptedException {
+    // Use disableTestAsync because disable can take a long time to complete
+    System.out.print("Disabling table " + htd.getNameAsString() +" ");
+    admin.disableTableAsync(htd.getName());
+
+    long start = System.currentTimeMillis();
+    // NOTE tables can be both admin.isTableEnabled=false and
+    // isTableDisabled=false, when disabling must use isTableDisabled!
+    while (!admin.isTableDisabled(htd.getName())) {
+      System.out.print(".");
+      Thread.sleep(1000);
+    }
+    long delta = System.currentTimeMillis() - start;
+    System.out.println(" " + delta +" ms");
+    System.out.println("Deleting table " + htd.getNameAsString() +" ");
+    admin.deleteTable(htd.getName());
+  }
+
+  public void usage() {
+    System.err.println(this.getClass().getSimpleName() + " [-Doptions] <load|verify|loadAndVerify>");
+    System.err.println("  Loads a table with row dependencies and verifies the dependency chains");
+    System.err.println("Options");
+    System.err.println("  -Dloadmapper.table=<name>        Table to write/verify (default autogen)");
+    System.err.println("  -Dloadmapper.backrefs=<n>        Number of backreferences per row (default 50)");
+    System.err.println("  -Dloadmapper.num_to_write=<n>    Number of rows per mapper (default 100,000 per mapper)");
+    System.err.println("  -Dloadmapper.deleteAfter=<bool>  Delete after a successful verify (default true)");
+    System.err.println("  -Dloadmapper.numPresplits=<n>    Number of presplit regions to start with (default 40)");
+    System.err.println("  -Dloadmapper.map.tasks=<n>       Number of map tasks for load (default 200)");
+    System.err.println("  -Dverify.reduce.tasks=<n>        Number of reduce tasks for verify (default 35)");
+    System.err.println("  -Dverify.scannercaching=<n>      Number hbase scanner caching rows to read (default 50)");
+  }
+
+  public int run(String argv[]) throws Exception {
+    if (argv.length < 1 || argv.length > 1) {
+      usage();
+      return 1;
+    }
+
+    IntegrationTestingUtility.setUseDistributedCluster(getConf());
+    boolean doLoad = false;
+    boolean doVerify = false;
+    boolean doDelete = getConf().getBoolean("loadmapper.deleteAfter",true);
+    int numPresplits = getConf().getInt("loadmapper.numPresplits", 40);
+
+    if (argv[0].equals("load")) {
+      doLoad = true;
+    } else if (argv[0].equals("verify")) {
+      doVerify= true;
+    } else if (argv[0].equals("loadAndVerify")) {
+      doLoad=true;
+      doVerify= true;
+    } else {
+      System.err.println("Invalid argument " + argv[0]);
+      usage();
+      return 1;
+    }
+
+    // create HTableDescriptor for specified table
+    String table = getConf().get(TABLE_NAME_KEY, TEST_NAME);
+    HTableDescriptor htd = new HTableDescriptor(table);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+    HBaseAdmin admin = new HBaseAdmin(getConf());
+    if (doLoad) {
+      admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPresplits);
+      doLoad(getConf(), htd);
+    }
+    if (doVerify) {
+      doVerify(getConf(), htd);
+      if (doDelete) {
+        deleteTable(admin, htd);
+      }
+    }
+    return 0;
+  }
+
+  private IntegrationTestingUtility getTestingUtil() {
+    if (this.util == null) {
+      if (getConf() == null) {
+        this.util = new IntegrationTestingUtility();
+      } else {
+        this.util = new IntegrationTestingUtility(getConf());
+      }
+    }
+    return util;
+  }
+
+  public static void main(String argv[]) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    int ret = ToolRunner.run(conf, new IntegrationTestLoadAndVerify(), argv);
+    System.exit(ret);
+  }
+}

From ab45aa69588c51a7b95583fafc49aaa2e2aa7d33 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 6 Mar 2013 01:04:08 +0000
Subject: [PATCH 0827/1540] CHANGES.txt and pom.xml for 0.94.6RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1453108 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 pom.xml     |  2 +-
 2 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 43da66851228..194d7893a3a8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,59 @@
 HBase Change Log
 
-Release 0.94.5 - 2/7/2012
+Release 0.94.6 - 3/5/2013
+Sub-task
+
+    [HBASE-7944] - Replication leaks file reader resource & not reset currentNbOperations
+
+Bug
+
+    [HBASE-6132] - ColumnCountGetFilter & PageFilter not working with FilterList
+    [HBASE-6347] - -ROOT- and .META. are stale in table.jsp if they moved
+    [HBASE-6748] - Endless recursive of deleteNode happened in SplitLogManager#DeleteAsyncCallback
+    [HBASE-7111] - hbase zkcli will not start if the zookeeper server chosen to connect to is unavailable
+    [HBASE-7507] - Make memstore flush be able to retry after exception
+    [HBASE-7521] - fix HBASE-6060 (regions stuck in opening state) in 0.94
+    [HBASE-7671] - Flushing memstore again after last failure could cause data loss
+    [HBASE-7700] - TestColumnSeeking is mathematically bound to fail
+    [HBASE-7723] - Remove NameNode URI from ZK splitlogs
+    [HBASE-7725] - Add ability to create custom compaction request
+    [HBASE-7761] - MemStore.USEMSLAB_DEFAULT is false, hbase-default.xml says it's true
+    [HBASE-7763] - Compactions not sorting based on size anymore.
+    [HBASE-7768] - zkcluster in local mode not seeing configurations in hbase-{site|default}.xml
+    [HBASE-7777] - HBCK check for lingering split parents should check for child regions
+    [HBASE-7813] - Bug in BulkDeleteEndpoint kills entire rows on COLUMN/VERSION Deletes
+    [HBASE-7814] - Port HBASE-6963 'unable to run hbck on a secure cluster' to 0.94
+    [HBASE-7829] - zookeeper kerberos conf keytab and principal parameters interchanged
+    [HBASE-7832] - Use User.getShortName() in FSUtils
+    [HBASE-7833] - 0.94 does not compile with Hadoop-0.20.205 and 0.22.0
+    [HBASE-7851] - Include the guava classes as a dependency for jobs using mapreduce.TableMapReduceUtil
+    [HBASE-7866] - TestSplitTransactionOnCluster.testSplitBeforeSettingSplittingInZK failed 3 times in a row
+    [HBASE-7867] - setPreallocSize is different with COMMENT in setupTestEnv in MiniZooKeeperCluster.java
+    [HBASE-7869] - Provide way to not start LogSyncer thread
+    [HBASE-7883] - Update memstore size when removing the entries in append operation
+    [HBASE-7884] - ByteBloomFilter's performance can be improved by avoiding multiplication when generating hash
+    [HBASE-7913] - Secure Rest server should login before getting an instance of Rest servlet
+    [HBASE-7914] - Port the fix of HBASE-6748 into 0.94 branch
+    [HBASE-7915] - Secure ThriftServer needs to login before calling HBaseHandler
+    [HBASE-7916] - HMaster uses wrong InetSocketAddress parameter to throw exception
+    [HBASE-7919] - Wrong key is used in ServerManager#getServerConnection() to retrieve from Map serverConnections
+    [HBASE-7920] - Move isFamilyEssential(byte[] name) out of Filter interface in 0.94
+    [HBASE-7945] - Remove flaky TestCatalogTrackerOnCluster
+    [HBASE-7986] - [REST] Make HTablePool size configurable
+    [HBASE-7991] - Backport HBASE-6479 'HFileReaderV1 caching the same parent META block could cause server abort when splitting' to 0.94
+    [HBASE-8007] - Adopt TestLoadAndVerify from BigTop
+
+Improvement
+
+    [HBASE-7818] - add region level metrics readReqeustCount and writeRequestCount
+
+New Feature
+
+    [HBASE-4210] - Allow coprocessor to interact with batches per region sent from a client
+    [HBASE-7360] - Snapshot 0.94 Backport
+
+
+Release 0.94.5 - 2/7/2013
 Sub-task
 
     [HBASE-2611] - Handle RS that fails while processing the failure of another one
@@ -93,7 +146,7 @@ Wish
     [HBASE-7705] - Make the method getCurrentPoolSize of HTablePool public
 
 
-Release 0.94.4 - 1/2/2012
+Release 0.94.4 - 1/2/2013
 Sub-task
 
     [HBASE-3776] - Add Bloom Filter Support to HFileOutputFormat
diff --git a/pom.xml b/pom.xml
index 14e54e0c77b4..5e8f270f8386 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.6-SNAPSHOT</version>
+  <version>0.94.6</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 758f22208e6a26c9d1f6ddf1ed558b042fe79352 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 7 Mar 2013 04:08:57 +0000
Subject: [PATCH 0828/1540] HBASE-8019 Port HBASE-7779 '[snapshot 130201 merge]
 Fix TestMultiParallel' to 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1453679 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
index afc87f5de31a..0a4dd4a237aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -99,9 +99,8 @@ public ZKProcedureUtil(ZooKeeperWatcher watcher, String procDescription,
 
   @Override
   public void close() throws IOException {
-    if (watcher != null) {
-      watcher.close();
-    }
+    // the watcher is passed from either Master or Region Server
+    // watcher.close() will be called by the owner so no need to call close() here
   }
 
   public String getAcquiredBarrierNode(String opInstanceName) {
@@ -283,4 +282,4 @@ public void clearZNodes(String procedureName) throws KeeperException {
     ZKUtil.deleteNodeRecursively(watcher, getReachedBarrierNode(procedureName));
     ZKUtil.deleteNodeRecursively(watcher, getAbortZNode(procedureName));
   }
-}
\ No newline at end of file
+}

From 57ae7328a1defbca2cb45544067971d5a5d2f54b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 7 Mar 2013 05:49:58 +0000
Subject: [PATCH 0829/1540] HBASE-7153 print gc option in hbase-env.sh affects
 hbase zkcli (Dave Latham and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1453684 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/hbase b/bin/hbase
index f23160580f6e..21fb6f8c9d9f 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -250,7 +250,7 @@ unset IFS
 
 #Set the right GC options based on the what we are running
 declare -a client_cmds=("shell" "hbck" "hlog" "hfile" "zkcli")
-for cmd in $client_cmds; do
+for cmd in ${client_cmds[@]}; do
 	if [[ $cmd == $COMMAND ]]; then
 		client=true
 		break

From 41149ae59137ba62692bca1fa9c60276c9e0d90f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 7 Mar 2013 05:53:39 +0000
Subject: [PATCH 0830/1540] CHANGES.txt for 0.94.6rc1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1453688 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 194d7893a3a8..90ae95595e69 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.6 - 3/5/2013
+Release 0.94.6 - 3/6/2013
 Sub-task
 
     [HBASE-7944] - Replication leaks file reader resource & not reset currentNbOperations
@@ -11,6 +11,7 @@ Bug
     [HBASE-6347] - -ROOT- and .META. are stale in table.jsp if they moved
     [HBASE-6748] - Endless recursive of deleteNode happened in SplitLogManager#DeleteAsyncCallback
     [HBASE-7111] - hbase zkcli will not start if the zookeeper server chosen to connect to is unavailable
+    [HBASE-7153] - print gc option in hbase-env.sh affects hbase zkcli
     [HBASE-7507] - Make memstore flush be able to retry after exception
     [HBASE-7521] - fix HBASE-6060 (regions stuck in opening state) in 0.94
     [HBASE-7671] - Flushing memstore again after last failure could cause data loss
@@ -42,6 +43,7 @@ Bug
     [HBASE-7986] - [REST] Make HTablePool size configurable
     [HBASE-7991] - Backport HBASE-6479 'HFileReaderV1 caching the same parent META block could cause server abort when splitting' to 0.94
     [HBASE-8007] - Adopt TestLoadAndVerify from BigTop
+    [HBASE-8019] - Port HBASE-7779 '[snapshot 130201 merge] Fix TestMultiParallel' to 0.94
 
 Improvement
 

From b840aa368e9e6a440f084aff567581f413203bd0 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 7 Mar 2013 20:41:47 +0000
Subject: [PATCH 0831/1540] HBASE-8019 Addendum breaks
 TestRestoreSnapshotFromClient into TestRestoreSnapshotFromClient and
 TestCloneSnapshotFromClient (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1454089 13f79535-47bb-0310-9956-ffa450edef68
---
 .../client/TestCloneSnapshotFromClient.java   | 269 ++++++++++++++++++
 .../client/TestRestoreSnapshotFromClient.java | 104 +------
 2 files changed, 275 insertions(+), 98 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
new file mode 100644
index 000000000000..66a8b3de8e62
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test clone snapshots from the client
+ */
+@Category(LargeTests.class)
+public class TestCloneSnapshotFromClient {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final byte[] FAMILY = Bytes.toBytes("cf");
+
+  private byte[] emptySnapshot;
+  private byte[] snapshotName0;
+  private byte[] snapshotName1;
+  private byte[] snapshotName2;
+  private int snapshot0Rows;
+  private int snapshot1Rows;
+  private byte[] tableName;
+  private HBaseAdmin admin;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
+    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
+    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+    TEST_UTIL.getConfiguration().setBoolean(
+        "hbase.master.enabletable.roundrobin", true);
+    TEST_UTIL.startMiniCluster(3);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Initialize the tests with a table filled with some data
+   * and two snapshots (snapshotName0, snapshotName1) of different states.
+   * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
+   */
+  @Before
+  public void setup() throws Exception {
+    this.admin = TEST_UTIL.getHBaseAdmin();
+
+    long tid = System.currentTimeMillis();
+    tableName = Bytes.toBytes("testtb-" + tid);
+    emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
+    snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
+    snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
+    snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
+
+    // create Table and disable it
+    createTable(tableName, FAMILY);
+    admin.disableTable(tableName);
+
+    // take an empty snapshot
+    admin.snapshot(emptySnapshot, tableName);
+
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    try {
+      // enable table and insert data
+      admin.enableTable(tableName);
+      loadData(table, 500, FAMILY);
+      snapshot0Rows = TEST_UTIL.countRows(table);
+      admin.disableTable(tableName);
+
+      // take a snapshot
+      admin.snapshot(snapshotName0, tableName);
+
+      // enable table and insert more data
+      admin.enableTable(tableName);
+      loadData(table, 500, FAMILY);
+      snapshot1Rows = TEST_UTIL.countRows(table);
+      admin.disableTable(tableName);
+
+      // take a snapshot of the updated table
+      admin.snapshot(snapshotName1, tableName);
+
+      // re-enable table
+      admin.enableTable(tableName);
+    } finally {
+      table.close();
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (admin.tableExists(tableName)) {
+      TEST_UTIL.deleteTable(tableName);
+    }
+    admin.deleteSnapshot(snapshotName0);
+    admin.deleteSnapshot(snapshotName1);
+
+    // Ensure the archiver to be empty
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    mfs.getFileSystem().delete(
+      new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+  }
+
+  @Test(expected=SnapshotDoesNotExistException.class)
+  public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
+    String snapshotName = "random-snapshot-" + System.currentTimeMillis();
+    String tableName = "random-table-" + System.currentTimeMillis();
+    admin.cloneSnapshot(snapshotName, tableName);
+  }
+
+  @Test
+  public void testCloneSnapshot() throws IOException, InterruptedException {
+    byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
+    testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
+    testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
+    testCloneSnapshot(clonedTableName, emptySnapshot, 0);
+  }
+
+  private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
+      int snapshotRows) throws IOException, InterruptedException {
+    // create a new table from snapshot
+    admin.cloneSnapshot(snapshotName, tableName);
+    verifyRowCount(tableName, snapshotRows);
+
+    admin.disableTable(tableName);
+    admin.deleteTable(tableName);
+  }
+
+  /**
+   * Verify that tables created from the snapshot are still alive after source table deletion.
+   */
+  @Test
+  public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
+    // Clone a table from the first snapshot
+    byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName0, clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+
+    // Take a snapshot of this cloned table.
+    admin.disableTable(clonedTableName);
+    admin.snapshot(snapshotName2, clonedTableName);
+
+    // Clone the snapshot of the cloned table
+    byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName2, clonedTableName2);
+    verifyRowCount(clonedTableName2, snapshot0Rows);
+    admin.disableTable(clonedTableName2);
+
+    // Remove the original table
+    admin.disableTable(tableName);
+    admin.deleteTable(tableName);
+    waitCleanerRun();
+
+    // Verify the first cloned table
+    admin.enableTable(clonedTableName);
+    verifyRowCount(clonedTableName, snapshot0Rows);
+
+    // Verify the second cloned table
+    admin.enableTable(clonedTableName2);
+    verifyRowCount(clonedTableName2, snapshot0Rows);
+    admin.disableTable(clonedTableName2);
+
+    // Delete the first cloned table
+    admin.disableTable(clonedTableName);
+    admin.deleteTable(clonedTableName);
+    waitCleanerRun();
+
+    // Verify the second cloned table
+    admin.enableTable(clonedTableName2);
+    verifyRowCount(clonedTableName2, snapshot0Rows);
+
+    // Clone a new table from cloned
+    byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
+    admin.cloneSnapshot(snapshotName2, clonedTableName3);
+    verifyRowCount(clonedTableName3, snapshot0Rows);
+
+    // Delete the cloned tables
+    admin.disableTable(clonedTableName2);
+    admin.deleteTable(clonedTableName2);
+    admin.disableTable(clonedTableName3);
+    admin.deleteTable(clonedTableName3);
+    admin.deleteSnapshot(snapshotName2);
+  }
+
+  // ==========================================================================
+  //  Helpers
+  // ==========================================================================
+  private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (byte[] family: families) {
+      HColumnDescriptor hcd = new HColumnDescriptor(family);
+      htd.addFamily(hcd);
+    }
+    byte[][] splitKeys = new byte[16][];
+    byte[] hex = Bytes.toBytes("0123456789abcdef");
+    for (int i = 0; i < 16; ++i) {
+      splitKeys[i] = new byte[] { hex[i] };
+    }
+    admin.createTable(htd, splitKeys);
+  }
+
+  public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
+    byte[] qualifier = Bytes.toBytes("q");
+    table.setAutoFlush(false);
+    while (rows-- > 0) {
+      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
+      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
+      Put put = new Put(key);
+      put.setWriteToWAL(false);
+      for (byte[] family: families) {
+        put.add(family, qualifier, value);
+      }
+      table.put(put);
+    }
+    table.flushCommits();
+  }
+
+  private void waitCleanerRun() throws InterruptedException {
+    TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
+  }
+
+  private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    assertEquals(expectedRows, TEST_UTIL.countRows(table));
+    table.close();
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index dbd0a711864d..f3af219fb749 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -17,43 +17,32 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.MD5Hash;
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 /**
@@ -237,31 +226,6 @@ public void testRestoreSchemaChange() throws IOException {
     table.close();
   }
 
-  @Test(expected=SnapshotDoesNotExistException.class)
-  public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
-    String snapshotName = "random-snapshot-" + System.currentTimeMillis();
-    String tableName = "random-table-" + System.currentTimeMillis();
-    admin.cloneSnapshot(snapshotName, tableName);
-  }
-
-  @Test
-  public void testCloneSnapshot() throws IOException, InterruptedException {
-    byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
-    testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
-    testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
-    testCloneSnapshot(clonedTableName, emptySnapshot, 0);
-  }
-
-  private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
-      int snapshotRows) throws IOException, InterruptedException {
-    // create a new table from snapshot
-    admin.cloneSnapshot(snapshotName, tableName);
-    verifyRowCount(tableName, snapshotRows);
-
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-  }
-
   @Test
   public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
@@ -278,62 +242,6 @@ public void testRestoreSnapshotOfCloned() throws IOException, InterruptedExcepti
     admin.deleteTable(clonedTableName);
   }
 
-  /**
-   * Verify that tables created from the snapshot are still alive after source table deletion.
-   */
-  @Test
-  public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
-    // Clone a table from the first snapshot
-    byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
-    admin.cloneSnapshot(snapshotName0, clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
-
-    // Take a snapshot of this cloned table.
-    admin.disableTable(clonedTableName);
-    admin.snapshot(snapshotName2, clonedTableName);
-
-    // Clone the snapshot of the cloned table
-    byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
-    admin.cloneSnapshot(snapshotName2, clonedTableName2);
-    verifyRowCount(clonedTableName2, snapshot0Rows);
-    admin.disableTable(clonedTableName2);
-
-    // Remove the original table
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    waitCleanerRun();
-
-    // Verify the first cloned table
-    admin.enableTable(clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
-
-    // Verify the second cloned table
-    admin.enableTable(clonedTableName2);
-    verifyRowCount(clonedTableName2, snapshot0Rows);
-    admin.disableTable(clonedTableName2);
-
-    // Delete the first cloned table
-    admin.disableTable(clonedTableName);
-    admin.deleteTable(clonedTableName);
-    waitCleanerRun();
-
-    // Verify the second cloned table
-    admin.enableTable(clonedTableName2);
-    verifyRowCount(clonedTableName2, snapshot0Rows);
-
-    // Clone a new table from cloned
-    byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
-    admin.cloneSnapshot(snapshotName2, clonedTableName3);
-    verifyRowCount(clonedTableName3, snapshot0Rows);
-
-    // Delete the cloned tables
-    admin.disableTable(clonedTableName2);
-    admin.deleteTable(clonedTableName2);
-    admin.disableTable(clonedTableName3);
-    admin.deleteTable(clonedTableName3);
-    admin.deleteSnapshot(snapshotName2);
-  }
-
   // ==========================================================================
   //  Helpers
   // ==========================================================================

From 7c25efc3136a95f452d0dc85c6f880b57e413016 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sun, 10 Mar 2013 16:37:34 +0000
Subject: [PATCH 0832/1540] HBASE-8061 Missing test from
 TestFlushSnapshotFromClient in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1454875 13f79535-47bb-0310-9956-ffa450edef68
---
 .../snapshot/TestFlushSnapshotFromClient.java | 39 +++++++++++++++++--
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index feae120460cf..d22fb73fa67e 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -49,7 +49,6 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -211,6 +210,35 @@ public void testSnapshotFailsOnNonExistantTable() throws Exception {
     }
   }
 
+  @Test(timeout = 15000)
+  public void testAsyncFlushSnapshot() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
+        .setTable(STRING_TABLE_NAME).setType(SnapshotDescription.Type.FLUSH).build();
+
+    // take the snapshot async
+    admin.takeSnapshotAsync(snapshot);
+
+    // constantly loop, looking for the snapshot to complete
+    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+    SnapshotTestingUtils.waitForSnapshotToComplete(master, new HSnapshotDescription(snapshot), 200);
+    LOG.info(" === Async Snapshot Completed ===");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+    // make sure we get the snapshot
+    SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
+
+    // test that we can delete the snapshot
+    admin.deleteSnapshot(snapshot.getName());
+    LOG.info(" === Async Snapshot Deleted ===");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+    // make sure we don't have any snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    LOG.info(" === Async Snapshot Test Completed ===");
+
+  }
+
   /**
    * Basic end-to-end test of simple-flush-based snapshots
    */
@@ -242,7 +270,7 @@ public void testFlushCreateListDestroy() throws Exception {
     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
     assertTrue(fs.exists(snapshotDir));
-    HBaseFsck.debugLsr(UTIL.getHBaseCluster().getConfiguration(),  snapshotDir);
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
     assertTrue(fs.exists(snapshotinfo));
 
@@ -268,7 +296,8 @@ public void testFlushCreateListDestroy() throws Exception {
 
     // test that we can delete the snapshot
     admin.deleteSnapshot(snapshotName);
-    HBaseFsck.debugLsr(UTIL.getHBaseCluster().getConfiguration(), FSUtils.getRootDir(UTIL.getConfiguration()));
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 
     // make sure we don't have any snapshots
     SnapshotTestingUtils.assertNoSnapshots(admin);
@@ -361,6 +390,10 @@ public void run() {
     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
         " issued many requests", takenSize < ssNum && takenSize > 0);
+    // delete snapshots so subsequent tests are clean.
+    for (SnapshotDescription ss : taken) {
+      admin.deleteSnapshot(ss.getName());
+    }
   }
 
   private void logFSTree(Path root) throws IOException {

From e2d84c1ede95411e726ca1b75db14608e7457265 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 11 Mar 2013 23:49:52 +0000
Subject: [PATCH 0833/1540] HBASE-8069 TestHLog is dependent on the execution
 order

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1455365 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/wal/TestHLog.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 8feb5d6db2d1..60f77a025db6 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -485,6 +485,9 @@ public void run() {
     }
     assertEquals(total, count);
     reader.close();
+
+    // Reset the lease period
+    setLeasePeriod.invoke(cluster, new Object[]{new Long(60000), new Long(3600000)});
   }
 
   /**

From f3e8df2926b1bcac1e341c3095caa9a28a5c3ee6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 12 Mar 2013 03:54:04 +0000
Subject: [PATCH 0834/1540] HBASE-8055 Null check missing in
 StoreFile.Reader.getMaxTimestamp()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1455400 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/StoreFile.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 26b31afc1735..bc0bd9d7ac5d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -1772,7 +1772,7 @@ void disableBloomFilterForTesting() {
     }
 
     public long getMaxTimestamp() {
-      return timeRangeTracker.maximumTimestamp;
+      return timeRangeTracker == null ? Long.MAX_VALUE : timeRangeTracker.maximumTimestamp;
     }
 
     @Override

From 52af451f0223452c9a6b52cc37dd1dd2715a3eb4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 12 Mar 2013 03:54:05 +0000
Subject: [PATCH 0835/1540] HBASE-7827 Improve the speed of Hbase Thirft Batch
 mutation for deletes (Shivendra)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1455401 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/thrift/ThriftServerRunner.java    | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 98ee569410ae..cf170e4ab75d 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -1047,9 +1047,7 @@ public void mutateRowsTs(
         table = getTable(tableName);
         if (!puts.isEmpty())
           table.put(puts);
-        for (Delete del : deletes) {
-          table.delete(del);
-        }
+        if (!deletes.isEmpty()) table.delete(deletes);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());

From 44aa17c933524b2f787ee192bb88904a7f92f8aa Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 12 Mar 2013 22:01:28 +0000
Subject: [PATCH 0836/1540] HBASE-7624 Backport HBASE-5359 and HBASE-7596 to
 0.94 (Jefferey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1455730 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |   3 +-
 .../apache/hadoop/hbase/master/HMaster.java   |   6 +-
 .../master/handler/TableAddFamilyHandler.java |   7 +-
 .../handler/TableDeleteFamilyHandler.java     |   9 +-
 .../master/handler/TableEventHandler.java     |  44 ++++-
 .../TestTableDescriptorModification.java      | 160 ++++++++++++++++++
 6 files changed, 212 insertions(+), 17 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 03bd313df7de..ef762f98d88b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -341,7 +341,8 @@ public Pair<Integer, Integer> getReopenStatus(byte[] tableName)
       MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName);
     Integer pending = 0;
     for(HRegionInfo hri : hris) {
-      if(regionsToReopen.get(hri.getEncodedName()) != null) {
+      String name = hri.getEncodedName();
+      if (regionsToReopen.containsKey(name) || regionsInTransition.containsKey(name)) {
         pending++;
       }
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 657e3717b9ea..c414adfea451 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -94,6 +94,7 @@
 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
 import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
+import org.apache.hadoop.hbase.master.handler.TableEventHandler;
 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
@@ -1387,7 +1388,10 @@ public void modifyTable(final byte[] tableName, HTableDescriptor htd)
     if (cpHost != null) {
       cpHost.preModifyTable(tableName, htd);
     }
-    this.executorService.submit(new ModifyTableHandler(tableName, htd, this, this));
+    TableEventHandler tblHandler = new ModifyTableHandler(tableName, htd, this, this);
+    this.executorService.submit(tblHandler);
+    // prevent client from querying status even before the event is being handled.
+    tblHandler.waitForEventBeingHandled();
     if (cpHost != null) {
       cpHost.postModifyTable(tableName, htd);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
index 3fb50b2efeb8..cfad4037b437 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
@@ -50,11 +50,8 @@ public TableAddFamilyHandler(byte[] tableName, HColumnDescriptor familyDesc,
   @Override
   protected void handleTableOperation(List<HRegionInfo> hris)
   throws IOException {
-    // Update table descriptor in HDFS
-    HTableDescriptor htd = this.masterServices.getMasterFileSystem()
-        .addColumn(tableName, familyDesc);
-    // Update in-memory descriptor cache
-    this.masterServices.getTableDescriptors().add(htd);
+    // Update table descriptor
+    this.masterServices.getMasterFileSystem().addColumn(tableName, familyDesc);
   }
   @Override
   public String toString() {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
index 9322ba6479b2..b520762a31a1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
@@ -45,13 +45,10 @@ public TableDeleteFamilyHandler(byte[] tableName, byte [] familyName,
 
   @Override
   protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
-    // Update table descriptor in HDFS
-    HTableDescriptor htd =
-      this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName);
-    // Update in-memory descriptor cache
-    this.masterServices.getTableDescriptors().add(htd);
-    // Remove the column family from the file system
     MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+    // Update table descriptor
+    mfs.deleteColumn(tableName, familyName);
+    // Remove the column family from the file system
     for (HRegionInfo hri : hris) {
       // Delete the family directory in FS for all the regions one by one
       mfs.deleteFamilyFromFS(hri, familyName);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
index aeaa4adf976c..354c59397a35 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -21,6 +21,7 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -59,6 +60,7 @@ public abstract class TableEventHandler extends EventHandler {
   protected final MasterServices masterServices;
   protected final byte [] tableName;
   protected final String tableNameStr;
+  protected boolean isEventBeingHandled = false;
 
   public TableEventHandler(EventType eventType, byte [] tableName, Server server,
       MasterServices masterServices)
@@ -108,16 +110,27 @@ public void process() {
       LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
     } catch (KeeperException e) {
       LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
+    } finally {
+      notifyEventBeingHandled();
     }
   }
 
   public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
     boolean done = false;
+    HTable table = null;
+    TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps.newTreeMap();
+    NavigableMap<HRegionInfo, ServerName> hriHserverMapping;
+
     LOG.info("Bucketing regions by region server...");
-    HTable table = new HTable(masterServices.getConfiguration(), tableName);
-    TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps
-        .newTreeMap();
-    NavigableMap<HRegionInfo, ServerName> hriHserverMapping = table.getRegionLocations();
+
+    try {
+      table = new HTable(masterServices.getConfiguration(), tableName);
+      hriHserverMapping = table.getRegionLocations();
+    } finally {
+      if (table != null) {
+        table.close();
+      }
+    }
     List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
     for (HRegionInfo hri : regions) {
       ServerName rsLocation = hriHserverMapping.get(hri);
@@ -139,6 +152,7 @@ public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
     LOG.info("Reopening " + reRegions.size() + " regions on "
         + serverToRegions.size() + " region servers.");
     this.masterServices.getAssignmentManager().setRegionsToReopen(reRegions);
+    notifyEventBeingHandled();
     BulkReOpen bulkReopen = new BulkReOpen(this.server, serverToRegions,
         this.masterServices.getAssignmentManager());
     while (true) {
@@ -189,4 +203,26 @@ public HTableDescriptor getTableDescriptor()
 
   protected abstract void handleTableOperation(List<HRegionInfo> regions)
   throws IOException, KeeperException;
+
+  /**
+   * Table modifications are processed asynchronously, but provide an API for you to query their
+   * status.
+   * @throws IOException
+   */
+  public synchronized void waitForEventBeingHandled() throws IOException {
+    if (!this.isEventBeingHandled) {
+      try {
+        wait();
+      } catch (InterruptedException ie) {
+        throw (IOException) new InterruptedIOException().initCause(ie);
+      }
+    }
+  }
+
+  private synchronized void notifyEventBeingHandled() {
+    if (!this.isEventBeingHandled) {
+      isEventBeingHandled = true;
+      notify();
+    }
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java
new file mode 100644
index 000000000000..8b6e8b991dcc
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java
@@ -0,0 +1,160 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Verify that the HTableDescriptor is updated after
+ * addColumn(), deleteColumn() and modifyTable() operations.
+ */
+@Category(LargeTests.class)
+public class TestTableDescriptorModification {
+
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final byte[] TABLE_NAME = Bytes.toBytes("table");
+  private static final byte[] FAMILY_0 = Bytes.toBytes("cf0");
+  private static final byte[] FAMILY_1 = Bytes.toBytes("cf1");
+
+  /**
+   * Start up a mini cluster and put a small table of empty regions into it.
+   *
+   * @throws Exception
+   */
+  @BeforeClass
+  public static void beforeAllTests() throws Exception {
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void afterAllTests() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testModifyTable() throws IOException {
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    // Create a table with one family
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    admin.createTable(baseHtd);
+    admin.disableTable(TABLE_NAME);
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      // Modify the table adding another family and verify the descriptor
+      HTableDescriptor modifiedHtd = new HTableDescriptor(TABLE_NAME);
+      modifiedHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+      modifiedHtd.addFamily(new HColumnDescriptor(FAMILY_1));
+      admin.modifyTable(TABLE_NAME, modifiedHtd);
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+    } finally {
+      admin.deleteTable(TABLE_NAME);
+    }
+  }
+
+  @Test
+  public void testAddColumn() throws IOException {
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    // Create a table with two families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    admin.createTable(baseHtd);
+    admin.disableTable(TABLE_NAME);
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      // Modify the table removing one family and verify the descriptor
+      admin.addColumn(TABLE_NAME, new HColumnDescriptor(FAMILY_1));
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+    } finally {
+      admin.deleteTable(TABLE_NAME);
+    }
+  }
+
+  @Test
+  public void testDeleteColumn() throws IOException {
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    // Create a table with two families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_1));
+    admin.createTable(baseHtd);
+    admin.disableTable(TABLE_NAME);
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+
+      // Modify the table removing one family and verify the descriptor
+      admin.deleteColumn(TABLE_NAME, FAMILY_1);
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+    } finally {
+      admin.deleteTable(TABLE_NAME);
+    }
+  }
+
+  private void verifyTableDescriptor(final byte[] tableName, final byte[]... families)
+      throws IOException {
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+
+    // Verify descriptor from master
+    HTableDescriptor htd = admin.getTableDescriptor(tableName);
+    verifyTableDescriptor(htd, tableName, families);
+
+    // Verify descriptor from HDFS
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName);
+    htd = FSTableDescriptors.getTableDescriptor(mfs.getFileSystem(), tableDir);
+    verifyTableDescriptor(htd, tableName, families);
+  }
+
+  private void verifyTableDescriptor(final HTableDescriptor htd,
+      final byte[] tableName, final byte[]... families) {
+    Set<byte[]> htdFamilies = htd.getFamiliesKeys();
+    assertTrue(Bytes.equals(tableName, htd.getName()));
+    assertEquals(families.length, htdFamilies.size());
+    for (byte[] familyName: families) {
+      assertTrue("Expected family " + Bytes.toString(familyName), htdFamilies.contains(familyName));
+    }
+  }
+}
\ No newline at end of file

From 209e088e509132a83a01d35cc5e0e486adcfc030 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 13 Mar 2013 14:56:40 +0000
Subject: [PATCH 0837/1540] HBASE-7824 Improve master start up time when there
 is log splitting work (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1455976 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  |   2 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 110 +++++++----
 .../hadoop/hbase/master/MasterFileSystem.java |  33 ++--
 .../hadoop/hbase/master/ServerManager.java    |  10 +
 .../master/handler/ServerShutdownHandler.java |   4 +
 .../hbase/regionserver/HRegionServer.java     |   7 +-
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |   2 +-
 .../apache/hadoop/hbase/MiniHBaseCluster.java |   5 +-
 .../master/TestDistributedLogSplitting.java   | 178 ++++++++++++++++--
 .../TestRSKilledWhenMasterInitializing.java   |  25 +--
 10 files changed, 279 insertions(+), 97 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 5f2f1482c7f2..58c61ee1703d 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -118,7 +118,7 @@ public class CatalogTracker {
    */
   private ServerName metaLocation;
 
-  private boolean stopped = false;
+  private volatile boolean stopped = false;
 
   static final byte [] ROOT_REGION_NAME =
     HRegionInfo.ROOT_REGIONINFO.getRegionName();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c414adfea451..c336bc8286d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -41,9 +41,6 @@
 
 import javax.management.ObjectName;
 
-import com.google.common.collect.ClassToInstanceMap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MutableClassToInstanceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -130,6 +127,10 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
+
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
  * master.  If many masters are started, all compete.  Whichever wins goes on to
@@ -566,14 +567,37 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     if (!masterRecovery) {
       this.assignmentManager.startTimeOutMonitor();
     }
-    // TODO: Should do this in background rather than block master startup
-    status.setStatus("Splitting logs after master startup");
-    splitLogAfterStartup(this.fileSystemManager);
 
-    // Make sure root and meta assigned before proceeding.
-    assignRootAndMeta(status);
+    // get a list for previously failed RS which need recovery work
+    Set<ServerName> failedServers = this.fileSystemManager.getFailedServersFromLogFolders();
+    ServerName preRootServer = this.catalogTracker.getRootLocation();
+    if (preRootServer != null && failedServers.contains(preRootServer)) {
+      // create recovered edits file for _ROOT_ server
+      this.fileSystemManager.splitLog(preRootServer);
+      failedServers.remove(preRootServer);
+    }
+
+    // Make sure root assigned before proceeding.
+    assignRoot(status);
+
+    // log splitting for .META. server
+    ServerName preMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+    if (preMetaServer != null && failedServers.contains(preMetaServer)) {
+      // create recovered edits file for .META. server
+      this.fileSystemManager.splitLog(preMetaServer);
+      failedServers.remove(preMetaServer);
+    }
+    // Make sure meta assigned before proceeding.
+    assignMeta(status, preRootServer);
+
     enableServerShutdownHandler();
 
+    // handle other dead servers in SSH
+    status.setStatus("Submit log splitting work of non-meta region servers");
+    for (ServerName curServer : failedServers) {
+      this.serverManager.processDeadServer(curServer);
+    }
+
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
     // before we assign all user regions or else the assignment will fail.
@@ -645,22 +669,13 @@ protected void startCatalogJanitorChore() {
   }
 
   /**
-   * Override to change master's splitLogAfterStartup. Used testing
-   * @param mfs
-   */
-  protected void splitLogAfterStartup(final MasterFileSystem mfs) {
-    mfs.splitLogAfterStartup();
-  }
-
-  /**
-   * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,
-   * assign them.
+   * Check <code>-ROOT-</code> is assigned. If not, assign it.
+   * @param status MonitoredTask
    * @throws InterruptedException
    * @throws IOException
    * @throws KeeperException
-   * @return Count of regions we assigned.
    */
-  int assignRootAndMeta(MonitoredTask status)
+  private void assignRoot(MonitoredTask status)
   throws InterruptedException, IOException, KeeperException {
     int assigned = 0;
     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
@@ -691,16 +706,32 @@ int assignRootAndMeta(MonitoredTask status)
     LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getRootLocation());
 
-    // Work on meta region
+    status.setStatus("ROOT assigned.");
+  }
+
+  /**
+   * Check <code>.META.</code> is assigned. If not, assign it.
+   * @param status MonitoredTask
+   * @param previousRootServer ServerName of previous root region server before current start up
+   * @return
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private void assignMeta(MonitoredTask status, ServerName previousRootServer)
+      throws InterruptedException,
+      IOException, KeeperException {
+    int assigned = 0;
+    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
+
     status.setStatus("Assigning META region");
-    rit = this.assignmentManager.
-      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
+    boolean rit =
+        this.assignmentManager
+            .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
     boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
     if (!rit && !metaRegionLocation) {
-      ServerName currentMetaServer =
-        this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-      if (currentMetaServer != null
-          && !currentMetaServer.equals(currentRootServer)) {
+      ServerName currentMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+      if (currentMetaServer != null && !currentMetaServer.equals(previousRootServer)) {
         splitLogAndExpireIfOnline(currentMetaServer);
       }
       assignmentManager.assignMeta();
@@ -710,15 +741,14 @@ int assignRootAndMeta(MonitoredTask status)
       enableSSHandWaitForMeta();
       assigned++;
     } else {
-      // Region already assigned.  We didnt' assign it.  Add to in-memory state.
+      // Region already assigned. We didnt' assign it. Add to in-memory state.
       this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
         this.catalogTracker.getMetaLocation());
     }
     enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
-    LOG.info(".META. assigned=" + assigned + ", rit=" + rit +
-      ", location=" + catalogTracker.getMetaLocation());
-    status.setStatus("META and ROOT assigned.");
-    return assigned;
+    LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
+        + catalogTracker.getMetaLocation());
+    status.setStatus("META assigned.");
   }
 
   private void enableSSHandWaitForMeta() throws IOException,
@@ -777,8 +807,7 @@ public boolean visit(Result r) throws IOException {
   }
 
   /**
-   * Split a server's log and expire it if we find it is one of the online
-   * servers.
+   * Expire a server if we find it is one of the online servers.
    * @param sn ServerName to check.
    * @throws IOException
    */
@@ -1643,12 +1672,23 @@ public void shutdown() {
     }
     if (this.assignmentManager != null) this.assignmentManager.shutdown();
     if (this.serverManager != null) this.serverManager.shutdownCluster();
+
     try {
       if (this.clusterStatusTracker != null){
         this.clusterStatusTracker.setClusterDown();
       }
     } catch (KeeperException e) {
-      LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      if (e instanceof KeeperException.SessionExpiredException) {
+        LOG.warn("ZK session expired. Retry a new connection...");
+        try {
+          this.zooKeeper.reconnectAfterExpiration();
+          this.clusterStatusTracker.setClusterDown();
+        } catch (Exception ex) {
+          LOG.warn("Retry setClusterDown failed", ex);
+        }
+      } else {
+        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 54ce8c59ab5b..5ed1559f8fd5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
@@ -197,30 +198,31 @@ public String getClusterId() {
   }
 
   /**
-   * Inspect the log directory to recover any log file without
-   * an active region server.
+   * Inspect the log directory to find dead servers which need log splitting
    */
-  void splitLogAfterStartup() {
+  Set<ServerName> getFailedServersFromLogFolders() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
-        HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+      HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+    
+    Set<ServerName> serverNames = new HashSet<ServerName>();
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
+
     do {
       if (master.isStopped()) {
-        LOG.warn("Master stopped while splitting logs");
+        LOG.warn("Master stopped while trying to get failed servers.");
         break;
       }
-      List<ServerName> serverNames = new ArrayList<ServerName>();
       try {
-        if (!this.fs.exists(logsDirPath)) return;
+        if (!this.fs.exists(logsDirPath)) return serverNames;
         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
         // Get online servers after getting log folders to avoid log folder deletion of newly
         // checked in region servers . see HBASE-5916
-        Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
-            .keySet();
+        Set<ServerName> onlineServers =
+            ((HMaster) master).getServerManager().getOnlineServers().keySet();
 
         if (logFolders == null || logFolders.length == 0) {
           LOG.debug("No log files to split, proceeding...");
-          return;
+          return serverNames;
         }
         for (FileStatus status : logFolders) {
           String sn = status.getPath().getName();
@@ -234,22 +236,19 @@ void splitLogAfterStartup() {
                 + "to a known region server, splitting");
             serverNames.add(serverName);
           } else {
-            LOG.info("Log folder " + status.getPath()
-                + " belongs to an existing region server");
+            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
           }
         }
-        splitLog(serverNames);
         retrySplitting = false;
       } catch (IOException ioe) {
-        LOG.warn("Failed splitting of " + serverNames, ioe);
+        LOG.warn("Failed getting failed servers to be recovered.", ioe);
         if (!checkFileSystem()) {
           LOG.warn("Bad Filesystem, exiting");
           Runtime.getRuntime().halt(1);
         }
         try {
           if (retrySplitting) {
-            Thread.sleep(conf.getInt(
-              "hbase.hlog.split.failure.retry.interval", 30 * 1000));
+            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
           }
         } catch (InterruptedException e) {
           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
@@ -259,6 +258,8 @@ void splitLogAfterStartup() {
         }
       }
     } while (retrySplitting);
+
+    return serverNames;
   }
 
   public void splitLog(final ServerName serverName) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 039702007613..6d5c45a3cdc7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -424,6 +424,16 @@ public synchronized void expireServer(final ServerName serverName) {
         carryingRoot + ", meta=" + carryingMeta);
   }
 
+  /**
+   * The function is to allow master to submit known dead servers into SSH
+   * @param serverName
+   */
+  void processDeadServer(final ServerName serverName) {
+    this.deadservers.add(serverName);
+    this.services.getExecutorService().submit(
+      new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName, true));
+  }
+
   /**
    * Expire the servers which died during master's initialization. It will be
    * called after HMaster#assignRootAndMeta.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 8b889f7db921..b545891c1d2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -179,6 +179,10 @@ public String toString() {
   public void process() throws IOException {
     final ServerName serverName = this.serverName;
     try {
+      if (this.server.isStopped()) {
+        throw new IOException("Server is stopped");
+      }
+
       try {
         if (this.shouldSplitHlog) {
           LOG.info("Splitting logs for " + serverName);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 455da6f5cc0c..e58cf9454d2a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,12 +66,12 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
-import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterAddressTracker;
 import org.apache.hadoop.hbase.NotServingRegionException;
@@ -172,7 +172,6 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.joda.time.field.MillisDurationField;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
@@ -1705,7 +1704,9 @@ public CatalogTracker getCatalogTracker() {
   @Override
   public void stop(final String msg) {
     try {
-      this.rsHost.preStop(msg);
+      if (this.rsHost != null) {
+        this.rsHost.preStop(msg);
+      }
       this.stopped = true;
       LOG.info("STOPPED: " + msg);
       // Wakes run() if it is sleeping
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index c6e607ef1334..4365f78e7172 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -45,7 +45,7 @@ public abstract class ZooKeeperNodeTracker extends ZooKeeperListener {
   /** Used to abort if a fatal error occurs */
   protected final Abortable abortable;
 
-  private boolean stopped = false;
+  private volatile boolean stopped = false;
 
   /**
    * Constructs a new ZK node tracker.
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 288bd6ccbd41..109d94eeae49 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.security.User;
@@ -452,7 +453,9 @@ public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
     while (!(mts = getMasterThreads()).isEmpty()
         && (System.currentTimeMillis() - start) < timeout) {
       for (JVMClusterUtil.MasterThread mt : mts) {
-        if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) {
+        ServerManager serverManager = mt.getMaster().getServerManager();
+        if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()
+            && !serverManager.areDeadServersInProgress()) {
           return true;
         }
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 823d7da933de..36a3c547e137 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -19,22 +19,29 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_mgr_wait_for_zk_delete;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_final_transistion_failed;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_done;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_err;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_resigned;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,7 +50,13 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -54,8 +67,9 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -82,15 +96,21 @@ public class TestDistributedLogSplitting {
   Configuration conf;
   HBaseTestingUtility TEST_UTIL;
 
+
   private void startCluster(int num_rs) throws Exception{
+    conf = HBaseConfiguration.create();
+    startCluster(NUM_MASTERS, num_rs, conf);
+  }
+
+  private void startCluster(int num_master, int num_rs, Configuration inConf) throws Exception {
     ZKSplitLog.Counters.resetCounters();
     LOG.info("Starting cluster");
-    conf = HBaseConfiguration.create();
+    this.conf = inConf;
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
     TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(NUM_MASTERS, num_rs);
+    TEST_UTIL.startMiniCluster(num_master, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
@@ -102,6 +122,10 @@ private void startCluster(int num_rs) throws Exception{
 
   @After
   public void after() throws Exception {
+    for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
+      mt.getMaster().abort("closing...", new Exception("Trace info"));
+    }
+
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -205,6 +229,89 @@ public void testRecoveredEdits() throws Exception {
     assertEquals(NUM_LOG_LINES, count);
   }
 
+  @Test(timeout = 300000)
+  public void testMasterStartsUpWithLogSplittingWork() throws Exception {
+    LOG.info("testMasterStartsUpWithLogSplittingWork");
+    Configuration curConf = HBaseConfiguration.create();
+    curConf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
+    startCluster(2, NUM_RS, curConf);
+
+    final int NUM_REGIONS_TO_CREATE = 40;
+    final int NUM_LOG_LINES = 1000;
+    // turn off load balancing to prevent regions from moving around otherwise
+    // they will consume recovered.edits
+    master.balanceSwitch(false);
+
+    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
+    HTable ht = installTable(zkw, "table", "f", NUM_REGIONS_TO_CREATE);
+
+    List<HRegionInfo> regions = null;
+    HRegionServer hrs = null;
+    for (int i = 0; i < NUM_RS; i++) {
+      boolean isCarryingMeta = false;
+      hrs = rsts.get(i).getRegionServer();
+      regions = hrs.getOnlineRegions();
+      for (HRegionInfo region : regions) {
+        if (region.isRootRegion() || region.isMetaRegion()) {
+          isCarryingMeta = true;
+          break;
+        }
+      }
+      if (isCarryingMeta) {
+        continue;
+      }
+      break;
+    }
+
+    LOG.info("#regions = " + regions.size());
+    Iterator<HRegionInfo> it = regions.iterator();
+    while (it.hasNext()) {
+      HRegionInfo region = it.next();
+      if (region.isMetaTable()) {
+        it.remove();
+      }
+    }
+    makeHLog(hrs.getWAL(), regions, "table", NUM_LOG_LINES, 100);
+
+    // abort master
+    abortMaster(cluster);
+
+    // abort RS
+    int numRS = cluster.getLiveRegionServerThreads().size();
+    LOG.info("Aborting region server: " + hrs.getServerName());
+    hrs.abort("testing");
+
+    // wait for the RS dies
+    long start = EnvironmentEdgeManager.currentTimeMillis();
+    while (cluster.getLiveRegionServerThreads().size() > (numRS - 1)) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue(false);
+      }
+      Thread.sleep(200);
+    }
+
+    Thread.sleep(2000);
+    LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
+    
+    startMasterAndWaitUntilLogSplit(cluster);
+    
+    start = EnvironmentEdgeManager.currentTimeMillis();
+    while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue("Timedout", false);
+      }
+      Thread.sleep(200);
+    }
+
+    LOG.info("Current Open Regions After Master Node Starts Up:"
+        + getAllOnlineRegions(cluster).size());
+
+    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+
+    ht.close();
+  }
+
   /**
    * The original intention of this test was to force an abort of a region
    * server and to make sure that the failure path in the region servers is
@@ -393,33 +500,40 @@ public void makeHLog(HLog log,
       List<HRegionInfo> hris, String tname,
       int num_edits, int edit_size) throws IOException {
 
+    // remove root and meta region
+    hris.remove(HRegionInfo.ROOT_REGIONINFO);
+    hris.remove(HRegionInfo.FIRST_META_REGIONINFO);
     byte[] table = Bytes.toBytes(tname);
     HTableDescriptor htd = new HTableDescriptor(tname);
     byte[] value = new byte[edit_size];
     for (int i = 0; i < edit_size; i++) {
-      value[i] = (byte)('a' + (i % 26));
+      value[i] = (byte) ('a' + (i % 26));
     }
     int n = hris.size();
     int[] counts = new int[n];
-    int j = 0;
     if (n > 0) {
       for (int i = 0; i < num_edits; i += 1) {
         WALEdit e = new WALEdit();
-        byte [] row = Bytes.toBytes("r" + Integer.toString(i));
-        byte [] family = Bytes.toBytes("f");
-        byte [] qualifier = Bytes.toBytes("c" + Integer.toString(i));
-        e.add(new KeyValue(row, family, qualifier,
-            System.currentTimeMillis(), value));
-        j++;
-        log.append(hris.get(j % n), table, e, System.currentTimeMillis(), htd);
-        counts[j % n] += 1;
+        HRegionInfo curRegionInfo = hris.get(i % n);
+        byte[] startRow = curRegionInfo.getStartKey();
+        if (startRow == null || startRow.length == 0) {
+          startRow = new byte[] { 0, 0, 0, 0, 1 };
+        }
+        byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
+        row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because
+                                             // HBaseTestingUtility.createMultiRegions use 5 bytes
+                                             // key
+        byte[] family = Bytes.toBytes("f");
+        byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
+        e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
+        log.append(curRegionInfo, table, e, System.currentTimeMillis(), htd);
+        counts[i % n] += 1;
       }
     }
     log.sync();
     log.close();
     for (int i = 0; i < n; i++) {
-      LOG.info("region " + hris.get(i).getRegionNameAsString() +
-          " has " + counts[i] + " edits");
+      LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
     }
     return;
   }
@@ -479,6 +593,30 @@ private void waitForCounter(AtomicLong ctr, long oldval, long newval,
     assertTrue(false);
   }
 
+  private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException {
+    for (MasterThread mt : cluster.getLiveMasterThreads()) {
+      if (mt.getMaster().isActiveMaster()) {
+        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
+        mt.join();
+        break;
+      }
+    }
+    LOG.debug("Master is aborted");
+  }
+
+  private void startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
+      throws IOException, InterruptedException {
+    cluster.startMaster();
+    HMaster master = cluster.getMaster();
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    ServerManager serverManager = master.getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
+      Thread.sleep(100);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index a7287a1b84a2..eb2f42dccc87 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -25,7 +25,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -37,19 +36,16 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.TestMasterFailover;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -102,19 +98,6 @@ public TestingMaster(Configuration conf) throws IOException,
       super(conf);
     }
 
-    @Override
-    protected void splitLogAfterStartup(MasterFileSystem mfs) {
-      super.splitLogAfterStartup(mfs);
-      logSplit = true;
-      // If "TestingMaster.sleep" is set, sleep after log split.
-      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
-        int duration = getConfiguration().getInt(
-            "TestingMaster.sleep.duration", 0);
-        Threads.sleep(duration);
-      }
-    }
-
-
     public boolean isLogSplitAfterStartup() {
       return logSplit;
     }
@@ -249,11 +232,13 @@ private void abortMaster(MiniHBaseCluster cluster)
   private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
       throws IOException, InterruptedException {
     TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
-    while (!master.isLogSplitAfterStartup()) {
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    ServerManager serverManager = cluster.getMaster().getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
       Thread.sleep(100);
     }
-    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
-        + master.isInitialized());
     return master;
   }
 

From eeb0555668a719e895e880bd3a136b0b78472fc8 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 13 Mar 2013 15:20:19 +0000
Subject: [PATCH 0838/1540] HBASE-8088 Versioning site: part one, put stake in
 the ground for 0.94 by copying current versions of book and site

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1455996 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/book.xml                      | 1152 ++++++++++++++++------
 src/docbkx/configuration.xml             | 1050 ++++++--------------
 src/docbkx/customization.xsl             |   18 +-
 src/docbkx/developer.xml                 |  875 ++++++++++++----
 src/docbkx/external_apis.xml             |   32 +-
 src/docbkx/getting_started.xml           |   68 +-
 src/docbkx/ops_mgt.xml                   |  281 +++++-
 src/docbkx/performance.xml               |  322 ++++--
 src/docbkx/preface.xml                   |    2 +-
 src/docbkx/shell.xml                     |   19 +-
 src/docbkx/troubleshooting.xml           |  316 +++---
 src/docbkx/upgrading.xml                 |  104 +-
 src/site/resources/css/site.css          |    5 +
 src/site/resources/images/hbase_logo.png |  Bin 1733 -> 2997 bytes
 src/site/resources/images/hbase_logo.svg |  119 ++-
 src/site/site.vm                         |   19 +-
 src/site/site.xml                        |   20 +-
 src/site/xdoc/acid-semantics.xml         |   11 +-
 src/site/xdoc/bulk-loads.xml             |  151 +--
 src/site/xdoc/cygwin.xml                 |   13 +-
 src/site/xdoc/index.xml                  |   43 +-
 src/site/xdoc/metrics.xml                |    7 +-
 src/site/xdoc/old_news.xml               |   31 +-
 src/site/xdoc/pseudo-distributed.xml     |   49 +-
 src/site/xdoc/replication.xml            |  166 +++-
 src/site/xdoc/sponsors.xml               |   13 +-
 26 files changed, 2968 insertions(+), 1918 deletions(-)

diff --git a/src/docbkx/book.xml b/src/docbkx/book.xml
index dbc43bf8b04a..4e19d912eb63 100644
--- a/src/docbkx/book.xml
+++ b/src/docbkx/book.xml
@@ -1,7 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -26,11 +25,11 @@
       xmlns:svg="http://www.w3.org/2000/svg"
       xmlns:m="http://www.w3.org/1998/Math/MathML"
       xmlns:html="http://www.w3.org/1999/xhtml"
-      xmlns:db="http://docbook.org/ns/docbook">
+      xmlns:db="http://docbook.org/ns/docbook" xml:id="book">
   <info>
-    
+
     <title><link xlink:href="http://www.hbase.org">
-    Apache HBase Reference Guide
+    The Apache HBase&#153; Reference Guide
     </link></title>
     <subtitle><link xlink:href="http://www.hbase.org">
            <inlinemediaobject>
@@ -40,10 +39,13 @@
            </inlinemediaobject>
        </link>
     </subtitle>
-      <copyright><year>2012</year><holder>Apache Software Foundation</holder></copyright>
+    <copyright><year>2012</year><holder>Apache Software Foundation.
+        All Rights Reserved.  Apache Hadoop, Hadoop, MapReduce, HDFS, Zookeeper, HBase, and the HBase project logo are trademarks of the Apache Software Foundation.
+        </holder>
+    </copyright>
       <abstract>
     <para>This is the official reference guide of
-    <link xlink:href="http://www.hbase.org">Apache HBase</link>,
+    <link xlink:href="http://www.hbase.org">Apache HBase (TM)</link>,
     a distributed, versioned, column-oriented database built on top of
     <link xlink:href="http://hadoop.apache.org/">Apache Hadoop</link> and
     <link xlink:href="http://zookeeper.apache.org/">Apache ZooKeeper</link>.
@@ -131,7 +133,7 @@
         Although at a conceptual level tables may be viewed as a sparse set of rows.
         Physically they are stored on a per-column family basis.  New columns
         (i.e., <varname>columnfamily:column</varname>) can be added to any
-        column family without pre-announcing them. 
+        column family without pre-announcing them.
         <table frame='all'><title>ColumnFamily <varname>anchor</varname></title>
 	<tgroup cols='3' align='left' colsep='1' rowsep='1'>
 	<colspec colname='c1'/>
@@ -173,10 +175,10 @@
     <varname>com.cnn.www</varname> if no timestamp is specified would be:
     the value of <varname>contents:html</varname> from time stamp
     <literal>t6</literal>, the value of <varname>anchor:cnnsi.com</varname>
-    from time stamp <literal>t9</literal>, the value of 
+    from time stamp <literal>t9</literal>, the value of
     <varname>anchor:my.look.ca</varname> from time stamp <literal>t8</literal>.
 	</para>
-	<para>For more information about the internals of how HBase stores data, see <xref linkend="regions.arch" />.
+	<para>For more information about the internals of how Apache HBase stores data, see <xref linkend="regions.arch" />.
 	</para>
 	</section>
 
@@ -198,7 +200,7 @@
     <section xml:id="columnfamily">
       <title>Column Family<indexterm><primary>Column Family</primary></indexterm></title>
         <para>
-      Columns in HBase are grouped into <emphasis>column families</emphasis>.
+      Columns in Apache HBase are grouped into <emphasis>column families</emphasis>.
       All column members of a column family have the same prefix.  For example, the
       columns <emphasis>courses:history</emphasis> and
       <emphasis>courses:math</emphasis> are both members of the
@@ -224,43 +226,43 @@
     <section xml:id="cells">
       <title>Cells<indexterm><primary>Cells</primary></indexterm></title>
       <para>A <emphasis>{row, column, version} </emphasis>tuple exactly
-      specifies a <literal>cell</literal> in HBase. 
+      specifies a <literal>cell</literal> in HBase.
       Cell content is uninterrpreted bytes</para>
     </section>
     <section xml:id="data_model_operations">
        <title>Data Model Operations</title>
-       <para>The four primary data model operations are Get, Put, Scan, and Delete.  Operations are applied via 
-       <link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/HTable.html">HTable</link> instances.
+       <para>The four primary data model operations are Get, Put, Scan, and Delete.  Operations are applied via
+       <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html">HTable</link> instances.
        </para>
       <section xml:id="get">
         <title>Get</title>
-        <para><link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Get.html">Get</link> returns
-        attributes for a specified row.  Gets are executed via 
-        <link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/HTable.html#get%28org.apache.hadoop.hbase.client.Get%29">
+        <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Get.html">Get</link> returns
+        attributes for a specified row.  Gets are executed via
+        <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#get%28org.apache.hadoop.hbase.client.Get%29">
         HTable.get</link>.
         </para>
       </section>
       <section xml:id="put">
         <title>Put</title>
-        <para><link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Put.html">Put</link> either 
+        <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Put.html">Put</link> either
         adds new rows to a table (if the key is new) or can update existing rows (if the key already exists).  Puts are executed via
-        <link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/HTable.html#put%28org.apache.hadoop.hbase.client.Put%29">
-        HTable.put</link> (writeBuffer) or <link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/HTable.html#batch%28java.util.List%29">
-        HTable.batch</link> (non-writeBuffer).  
+        <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#put%28org.apache.hadoop.hbase.client.Put%29">
+        HTable.put</link> (writeBuffer) or <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#batch%28java.util.List%29">
+        HTable.batch</link> (non-writeBuffer).
         </para>
       </section>
       <section xml:id="scan">
           <title>Scans</title>
-          <para><link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Scan.html">Scan</link> allow
+          <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html">Scan</link> allow
           iteration over multiple rows for specified attributes.
           </para>
-          <para>The following is an example of a 
-           on an HTable table instance.  Assume that a table is populated with rows with keys "row1", "row2", "row3", 
-           and then another set of rows with the keys "abc1", "abc2", and "abc3".  The following example shows how startRow and stopRow 
-           can be applied to a Scan instance to return the rows beginning with "row".        
+          <para>The following is an example of a
+           on an HTable table instance.  Assume that a table is populated with rows with keys "row1", "row2", "row3",
+           and then another set of rows with the keys "abc1", "abc2", and "abc3".  The following example shows how startRow and stopRow
+           can be applied to a Scan instance to return the rows beginning with "row".
 <programlisting>
 HTable htable = ...      // instantiate HTable
-    
+
 Scan scan = new Scan();
 scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("attr"));
 scan.setStartRow( Bytes.toBytes("row"));                   // start key is inclusive
@@ -273,24 +275,24 @@ try {
   rs.close();  // always close the ResultScanner!
 }
 </programlisting>
-         </para>        
+         </para>
         </section>
       <section xml:id="delete">
         <title>Delete</title>
-        <para><link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Delete.html">Delete</link> removes
-        a row from a table.  Deletes are executed via 
-        <link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/HTable.html#delete%28org.apache.hadoop.hbase.client.Delete%29">
+        <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Delete.html">Delete</link> removes
+        a row from a table.  Deletes are executed via
+        <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#delete%28org.apache.hadoop.hbase.client.Delete%29">
         HTable.delete</link>.
         </para>
         <para>HBase does not modify data in place, and so deletes are handled by creating new markers called <emphasis>tombstones</emphasis>.
         These tombstones, along with the dead values, are cleaned up on major compactions.
         </para>
-        <para>See <xref linkend="version.delete"/> for more information on deleting versions of columns, and see 
-        <xref linkend="compaction"/> for more information on compactions.         
+        <para>See <xref linkend="version.delete"/> for more information on deleting versions of columns, and see
+        <xref linkend="compaction"/> for more information on compactions.
         </para>
- 
+
       </section>
-            
+
     </section>
 
 
@@ -298,7 +300,7 @@ try {
       <title>Versions<indexterm><primary>Versions</primary></indexterm></title>
 
       <para>A <emphasis>{row, column, version} </emphasis>tuple exactly
-      specifies a <literal>cell</literal> in HBase. Its possible to have an
+      specifies a <literal>cell</literal> in HBase. It's possible to have an
       unbounded number of cells where the row and column are the same but the
       cell address differs only in its version dimension.</para>
 
@@ -356,8 +358,8 @@ try {
           <title>Get/Scan</title>
 
           <para>Gets are implemented on top of Scans. The below discussion of
-            <link xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Get.html">Get</link> applies equally to <link
-            xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Scan.html">Scans</link>.</para>
+            <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Get.html">Get</link> applies equally to <link
+            xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html">Scans</link>.</para>
 
           <para>By default, i.e. if you specify no explicit version, when
           doing a <literal>get</literal>, the cell whose version has the
@@ -368,7 +370,7 @@ try {
           <itemizedlist>
             <listitem>
               <para>to return more than one version, see <link
-              xlink:href="http://hbase.apache.org/docs/current/api/org/apache/hadoop/hbase/client/Get.html#setMaxVersions()">Get.setMaxVersions()</link></para>
+              xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Get.html#setMaxVersions()">Get.setMaxVersions()</link></para>
             </listitem>
 
             <listitem>
@@ -389,7 +391,7 @@ try {
 <programlisting>
 Get get = new Get(Bytes.toBytes("row1"));
 Result r = htable.get(get);
-byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr"));  // returns current version of value          
+byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr"));  // returns current version of value
 </programlisting>
         </para>
         </section>
@@ -401,11 +403,11 @@ Get get = new Get(Bytes.toBytes("row1"));
 get.setMaxVersions(3);  // will return last 3 versions of row
 Result r = htable.get(get);
 byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr"));  // returns current version of value
-List&lt;KeyValue&gt; kv = r.getColumn(Bytes.toBytes("cf"), Bytes.toBytes("attr"));  // returns all versions of this column       
+List&lt;KeyValue&gt; kv = r.getColumn(Bytes.toBytes("cf"), Bytes.toBytes("attr"));  // returns all versions of this column
 </programlisting>
         </para>
         </section>
-     
+
         <section>
           <title>Put</title>
 
@@ -438,12 +440,12 @@ long explicitTimeInMs = 555;  // just an example
 put.add(Bytes.toBytes("cf"), Bytes.toBytes("attr1"), explicitTimeInMs, Bytes.toBytes(data));
 htable.put(put);
 </programlisting>
-          Caution:  the version timestamp is internally by HBase for things like time-to-live calculations.  
-          It's usually best to avoid setting this timestamp yourself.  Prefer using a separate 
+          Caution:  the version timestamp is internally by HBase for things like time-to-live calculations.
+          It's usually best to avoid setting this timestamp yourself.  Prefer using a separate
           timestamp attribute of the row, or have the timestamp a part of the rowkey, or both.
           </para>
           </section>
-          
+
         </section>
 
         <section xml:id="version.delete">
@@ -451,7 +453,7 @@ htable.put(put);
 
           <para>There are three different types of internal delete markers
             <footnote><para>See Lars Hofhansl's blog for discussion of his attempt
-            adding another, <link xlink:href="http://hadoop-hbase.blogspot.com/2012/01/scanning-in-hbase.html">Scanning in HBase: Prefix Delete Marker</link></para></footnote>: 
+            adding another, <link xlink:href="http://hadoop-hbase.blogspot.com/2012/01/scanning-in-hbase.html">Scanning in HBase: Prefix Delete Marker</link></para></footnote>:
             <itemizedlist>
             <listitem><para>Delete:  for a specific version of a column.</para>
             </listitem>
@@ -478,6 +480,9 @@ htable.put(put);
             </footnote>. If the version you specified when deleting a row is
           larger than the version of any value in the row, then you can
           consider the complete row to be deleted.</para>
+          <para>For an informative discussion on how deletes and versioning interact, see
+          the thread <link xlink:href="http://comments.gmane.org/gmane.comp.java.hadoop.hbase.user/28421">Put w/ timestamp -> Deleteall -> Put w/ timestamp fails</link>
+          up on the user mailing list.</para>
           <para>Also see <xref linkend="keyvalue"/> for more information on the internal KeyValue format.
           </para>
         </section>
@@ -486,10 +491,6 @@ htable.put(put);
       <section>
         <title>Current Limitations</title>
 
-        <para>There are still some bugs (or at least 'undecided behavior')
-        with the version dimension that will be addressed by later HBase
-        releases.</para>
-
         <section>
           <title>Deletes mask Puts</title>
 
@@ -529,7 +530,7 @@ htable.put(put);
     </section>
     <section xml:id="dm.sort">
       <title>Sort Order</title>
-      <para>All data model operations HBase return data in sorted order.  First by row, 
+      <para>All data model operations HBase return data in sorted order.  First by row,
       then by ColumnFamily, followed by column qualifier, and finally timestamp (sorted
       in reverse, so newest records are returned first).
       </para>
@@ -537,32 +538,37 @@ htable.put(put);
     <section xml:id="dm.column.metadata">
       <title>Column Metadata</title>
       <para>There is no store of column metadata outside of the internal KeyValue instances for a ColumnFamily.
-      Thus, while HBase can support not only a wide number of columns per row, but a heterogenous set of columns 
-      between rows as well, it is your responsibility to keep track of the column names.        
+      Thus, while HBase can support not only a wide number of columns per row, but a heterogenous set of columns
+      between rows as well, it is your responsibility to keep track of the column names.
       </para>
-      <para>The only way to get a complete set of columns that exist for a ColumnFamily is to process all the rows. 
+      <para>The only way to get a complete set of columns that exist for a ColumnFamily is to process all the rows.
       For more information about how HBase stores data internally, see <xref linkend="keyvalue" />.
-	  </para>         
+	  </para>
     </section>
     <section xml:id="joins"><title>Joins</title>
       <para>Whether HBase supports joins is a common question on the dist-list, and there is a simple answer:  it doesn't,
       at not least in the way that RDBMS' support them (e.g., with equi-joins or outer-joins in SQL).  As has been illustrated
-      in this chapter, the read data model operations in HBase are Get and Scan.       
+      in this chapter, the read data model operations in HBase are Get and Scan.
       </para>
       <para>However, that doesn't mean that equivalent join functionality can't be supported in your application, but
       you have to do it yourself.  The two primary strategies are either denormalizing the data upon writing to HBase,
-      or to have lookup tables and do the join between HBase tables in your application or MapReduce code (and as RDBMS' 
+      or to have lookup tables and do the join between HBase tables in your application or MapReduce code (and as RDBMS'
       demonstrate, there are several strategies for this depending on the size of the tables, e.g., nested loops vs.
       hash-joins).  So which is the best approach?  It depends on what you are trying to do, and as such there isn't a single
       answer that works for every use case.
       </para>
     </section>
+    <section xml:id="acid"><title>ACID</title>
+        <pre>See <link xlink:href="http://hbase.apache.org/acid-semantics.html">ACID Semantics</link>.
+            Lars Hofhansl has also written a note on
+            <link xlink:href="http://hadoop-hbase.blogspot.com/2012/03/acid-in-hbase.html">ACID in HBase</link>.</pre>
+    </section>
   </chapter>  <!-- data model -->
 
  <chapter xml:id="schema">
   <title>HBase and Schema Design</title>
       <para>A good general introduction on the strength and weaknesses modelling on
-          the various non-rdbms datastores is Ian Varleys' Master thesis,
+          the various non-rdbms datastores is Ian Varley's Master thesis,
           <link xlink:href="http://ianvarley.com/UT/MR/Varley_MastersReport_Full_2009-08-07.pdf">No Relation: The Mixed Blessings of Non-Relational Databases</link>.
           Recommended.  Also, read <xref linkend="keyvalue"/> for how HBase stores data internally.
       </para>
@@ -575,31 +581,31 @@ htable.put(put);
       </para>
       <para>Tables must be disabled when making ColumnFamily modifications, for example..
       <programlisting>
-Configuration config = HBaseConfiguration.create();  
-HBaseAdmin admin = new HBaseAdmin(conf);    
+Configuration config = HBaseConfiguration.create();
+HBaseAdmin admin = new HBaseAdmin(conf);
 String table = "myTable";
 
-admin.disableTable(table);           
+admin.disableTable(table);
 
 HColumnDescriptor cf1 = ...;
 admin.addColumn(table, cf1);      // adding new ColumnFamily
 HColumnDescriptor cf2 = ...;
 admin.modifyColumn(table, cf2);    // modifying existing ColumnFamily
 
-admin.enableTable(table);                
+admin.enableTable(table);
       </programlisting>
       </para>See <xref linkend="client_dependencies"/> for more information about configuring client connections.
       <para>Note:  online schema changes are supported in the 0.92.x codebase, but the 0.90.x codebase requires the table
       to be disabled.
       </para>
-    <section xml:id="schema.updates"><title>Schema Updates</title>  
+    <section xml:id="schema.updates"><title>Schema Updates</title>
       <para>When changes are made to either Tables or ColumnFamilies (e.g., region size, block size), these changes
       take effect the next time there is a major compaction and the StoreFiles get re-written.
       </para>
       <para>See <xref linkend="store"/> for more information on StoreFiles.
       </para>
     </section>
-  </section>   
+  </section>
   <section xml:id="number.of.cfs">
   <title>
       On the number of column families
@@ -610,7 +616,7 @@ admin.enableTable(table);
       if one column family is carrying the bulk of the data bringing on flushes, the adjacent families
       will also be flushed though the amount of data they carry is small.  When many column families the
       flushing and compaction interaction can make for a bunch of needless i/o loading (To be addressed by
-      changing flushing and compaction to work on a per column family basis).  For more information 
+      changing flushing and compaction to work on a per column family basis).  For more information
       on compactions, see <xref linkend="compaction"/>.
     </para>
     <para>Try to make do with one column family if you can in your schemas.  Only introduce a
@@ -618,9 +624,9 @@ admin.enableTable(table);
         i.e. you query one column family or the other but usually not both at the one time.
     </para>
     <section xml:id="number.of.cfs.card"><title>Cardinality of ColumnFamilies</title>
-      <para>Where multiple ColumnFamilies exist in a single table, be aware of the cardinality (i.e., number of rows).  
-      If ColumnFamilyA has 1 million rows and ColumnFamilyB has 1 billion rows, ColumnFamilyA's data will likely be spread 
-      across many, many regions (and RegionServers).  This makes mass scans for ColumnFamilyA less efficient.  
+      <para>Where multiple ColumnFamilies exist in a single table, be aware of the cardinality (i.e., number of rows).
+      If ColumnFamilyA has 1 million rows and ColumnFamilyB has 1 billion rows, ColumnFamilyA's data will likely be spread
+      across many, many regions (and RegionServers).  This makes mass scans for ColumnFamilyA less efficient.
       </para>
     </section>
   </section>
@@ -632,7 +638,7 @@ admin.enableTable(table);
     <para>
       In the HBase chapter of Tom White's book <link xlink:url="http://oreilly.com/catalog/9780596521981">Hadoop: The Definitive Guide</link> (O'Reilly) there is a an optimization note on watching out for a phenomenon where an import process walks in lock-step with all clients in concert pounding one of the table's regions (and thus, a single node), then moving onto the next region, etc.  With monotonically increasing row-keys (i.e., using a timestamp), this will happen.  See this comic by IKai Lan on why monotonically increasing row keys are problematic in BigTable-like datastores:
       <link xlink:href="http://ikaisays.com/2011/01/25/app-engine-datastore-tip-monotonically-increasing-values-are-bad/">monotonically increasing values are bad</link>.  The pile-up on a single region brought on
-      by monotonically increasing keys can be mitigated by randomizing the input records to not be in sorted order, but in general its best to avoid using a timestamp or a sequence (e.g. 1, 2, 3) as the row-key. 
+      by monotonically increasing keys can be mitigated by randomizing the input records to not be in sorted order, but in general it's best to avoid using a timestamp or a sequence (e.g. 1, 2, 3) as the row-key.
     </para>
 
 
@@ -670,20 +676,20 @@ admin.enableTable(table);
        <para>See <xref linkend="keyvalue"/> for more information on HBase stores data internally to see why this is important.</para>
        <section xml:id="keysize.cf"><title>Column Families</title>
          <para>Try to keep the ColumnFamily names as small as possible, preferably one character (e.g. "d" for data/default).
-         </para> 
+         </para>
        <para>See <xref linkend="keyvalue"/> for more information on HBase stores data internally to see why this is important.</para>
        </section>
        <section xml:id="keysize.atttributes"><title>Attributes</title>
          <para>Although verbose attribute names (e.g., "myVeryImportantAttribute") are easier to read, prefer shorter attribute names (e.g., "via")
          to store in HBase.
-         </para> 
+         </para>
        <para>See <xref linkend="keyvalue"/> for more information on HBase stores data internally to see why this is important.</para>
        </section>
        <section xml:id="keysize.row"><title>Rowkey Length</title>
-         <para>Keep them as short as is reasonable such that they can still be useful for required data access (e.g., Get vs. Scan). 
+         <para>Keep them as short as is reasonable such that they can still be useful for required data access (e.g., Get vs. Scan).
          A short key that is useless for data access is not better than a longer key with better get/scan properties.  Expect tradeoffs
          when designing rowkeys.
-         </para> 
+         </para>
        </section>
        <section xml:id="keysize.patterns"><title>Byte Patterns</title>
          <para>A long is 8 bytes.  You can store an unsigned number up to 18,446,744,073,709,551,615 in those eight bytes.
@@ -696,28 +702,28 @@ admin.enableTable(table);
 long l = 1234567890L;
 byte[] lb = Bytes.toBytes(l);
 System.out.println("long bytes length: " + lb.length);   // returns 8
-		
+
 String s = "" + l;
 byte[] sb = Bytes.toBytes(s);
 System.out.println("long as string length: " + sb.length);    // returns 10
-			
-// hash 
+
+// hash
 //
 MessageDigest md = MessageDigest.getInstance("MD5");
 byte[] digest = md.digest(Bytes.toBytes(s));
 System.out.println("md5 digest bytes length: " + digest.length);    // returns 16
-		
+
 String sDigest = new String(digest);
 byte[] sbDigest = Bytes.toBytes(sDigest);
-System.out.println("md5 digest as string length: " + sbDigest.length);    // returns 26		
-</programlisting>               
+System.out.println("md5 digest as string length: " + sbDigest.length);    // returns 26
+</programlisting>
          </para>
        </section>
-       
+
     </section>
     <section xml:id="reverse.timestamp"><title>Reverse Timestamps</title>
     <para>A common problem in database processing is quickly finding the most recent version of a value.  A technique using reverse timestamps
-    as a part of the key can help greatly with a special case of this problem.  Also found in the HBase chapter of Tom White's book Hadoop:  The Definitive Guide (O'Reilly), 
+    as a part of the key can help greatly with a special case of this problem.  Also found in the HBase chapter of Tom White's book Hadoop:  The Definitive Guide (O'Reilly),
     the technique involves appending (<code>Long.MAX_VALUE - timestamp</code>) to the end of any key, e.g., [key][reverse_timestamp].
     </para>
     <para>The most recent value for [key] in a table can be found by performing a Scan for [key] and obtaining the first record.  Since HBase keys
@@ -734,11 +740,76 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
     </section>
     <section xml:id="changing.rowkeys"><title>Immutability of Rowkeys</title>
     <para>Rowkeys cannot be changed.  The only way they can be "changed" in a table is if the row is deleted and then re-inserted.
-    This is a fairly common question on the HBase dist-list so it pays to get the rowkeys right the first time (and/or before you've 
+    This is a fairly common question on the HBase dist-list so it pays to get the rowkeys right the first time (and/or before you've
     inserted a lot of data).
     </para>
     </section>
-    </section>  <!--  rowkey design -->  
+    <section xml:id="rowkey.regionsplits"><title>Relationship Between RowKeys and Region Splits</title>
+    <para>If you pre-split your table, it is <emphasis>critical</emphasis> to understand how your rowkey will be distributed across
+    the region boundaries.  As an example of why this is important, consider the example of using displayable hex characters as the
+    lead position of the key (e.g., ""0000000000000000" to "ffffffffffffffff").  Running those key ranges through <code>Bytes.split</code>
+    (which is the split strategy used when creating regions in <code>HBaseAdmin.createTable(byte[] startKey, byte[] endKey, numRegions)</code>
+    for 10 regions will generate the following splits...
+    </para>
+    <para>
+    <programlisting>
+48 48 48 48 48 48 48 48 48 48 48 48 48 48 48 48                                // 0
+54 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10                 // 6
+61 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -68                 // =
+68 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -126  // D
+75 75 75 75 75 75 75 75 75 75 75 75 75 75 75 72                                // K
+82 18 18 18 18 18 18 18 18 18 18 18 18 18 18 14                                // R
+88 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -44                 // X
+95 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -102                // _
+102 102 102 102 102 102 102 102 102 102 102 102 102 102 102 102                // f
+    </programlisting>
+    ... (note:  the lead byte is listed to the right as a comment.)  Given that the first split is a '0' and the last split is an 'f',
+    everything is great, right?  Not so fast.
+    </para>
+    <para>The problem is that all the data is going to pile up in the first 2 regions and the last region thus creating a "lumpy" (and
+    possibly "hot") region problem.  To understand why, refer to an  <link xlink:href="http://www.asciitable.com">ASCII Table</link>.
+    '0' is byte 48, and 'f' is byte 102, but there is a huge gap in byte values (bytes 58 to 96) that will <emphasis>never appear in this
+    keyspace</emphasis> because the only values are [0-9] and [a-f].  Thus, the middle regions regions will
+    never be used.  To make pre-spliting work with this example keyspace, a custom definition of splits (i.e., and not relying on the
+    built-in split method) is required.
+    </para>
+    <para>Lesson #1:  Pre-splitting tables is generally a best practice, but you need to pre-split them in such a way that all the
+    regions are accessible in the keyspace.  While this example demonstrated the problem with a hex-key keyspace, the same problem can happen
+     with <emphasis>any</emphasis> keyspace.  Know your data.
+    </para>
+    <para>Lesson #2:  While generally not advisable, using hex-keys (and more generally, displayable data) can still work with pre-split
+    tables as long as all the created regions are accessible in the keyspace.
+    </para>
+	 <para>To conclude this example, the following is an example of  how appropriate splits can be pre-created for hex-keys:.
+	    </para>
+<programlisting>public static boolean createTable(HBaseAdmin admin, HTableDescriptor table, byte[][] splits)
+throws IOException {
+  try {
+    admin.createTable( table, splits );
+    return true;
+  } catch (TableExistsException e) {
+    logger.info("table " + table.getNameAsString() + " already exists");
+    // the table already exists...
+    return false;
+  }
+}
+
+public static byte[][] getHexSplits(String startKey, String endKey, int numRegions) {
+  byte[][] splits = new byte[numRegions-1][];
+  BigInteger lowestKey = new BigInteger(startKey, 16);
+  BigInteger highestKey = new BigInteger(endKey, 16);
+  BigInteger range = highestKey.subtract(lowestKey);
+  BigInteger regionIncrement = range.divide(BigInteger.valueOf(numRegions));
+  lowestKey = lowestKey.add(regionIncrement);
+  for(int i=0; i &lt; numRegions-1;i++) {
+    BigInteger key = lowestKey.add(regionIncrement.multiply(BigInteger.valueOf(i)));
+    byte[] b = String.format("%016x", key).getBytes();
+    splits[i] = b;
+  }
+  return splits;
+}</programlisting>
+    </section>
+    </section>  <!--  rowkey design -->
     <section xml:id="schema.versions">
   <title>
   Number of Versions
@@ -752,8 +823,8 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
       stores different values per row by time (and qualifier).  Excess versions are removed during major
       compactions.  The number of max versions may need to be increased or decreased depending on application needs.
       </para>
-      <para>It is not recommended setting the number of max versions to an exceedingly high level (e.g., hundreds or more) unless those old values are 
-      very dear to you because this will greatly increase StoreFile size. 
+      <para>It is not recommended setting the number of max versions to an exceedingly high level (e.g., hundreds or more) unless those old values are
+      very dear to you because this will greatly increase StoreFile size.
       </para>
      </section>
     <section xml:id="schema.minversions">
@@ -778,24 +849,24 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
   </title>
   <para>HBase supports a "bytes-in/bytes-out" interface via <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Put.html">Put</link> and
   <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Result.html">Result</link>, so anything that can be
-  converted to an array of bytes can be stored as a value.  Input could be strings, numbers, complex objects, or even images as long as they can rendered as bytes.  
+  converted to an array of bytes can be stored as a value.  Input could be strings, numbers, complex objects, or even images as long as they can rendered as bytes.
   </para>
   <para>There are practical limits to the size of values (e.g., storing 10-50MB objects in HBase would probably be too much to ask);
-  search the mailling list for conversations on this topic. All rows in HBase conform to the <xref linkend="datamodel">datamodel</xref>, and 
-  that includes versioning.  Take that into consideration when making your design, as well as block size for the ColumnFamily.  
+  search the mailling list for conversations on this topic. All rows in HBase conform to the <xref linkend="datamodel">datamodel</xref>, and
+  that includes versioning.  Take that into consideration when making your design, as well as block size for the ColumnFamily.
   </para>
     <section xml:id="counters">
       <title>Counters</title>
       <para>
-      One supported datatype that deserves special mention are "counters" (i.e., the ability to do atomic increments of numbers).  See 
+      One supported datatype that deserves special mention are "counters" (i.e., the ability to do atomic increments of numbers).  See
       <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#increment%28org.apache.hadoop.hbase.client.Increment%29">Increment</link> in HTable.
       </para>
       <para>Synchronization on counters are done on the RegionServer, not in the client.
       </para>
-    </section> 
+    </section>
   </section>
   <section xml:id="schema.joins"><title>Joins</title>
-    <para>If you have multiple tables, don't forget to factor in the potential for <xref linkend="joins"/> into the schema design. 
+    <para>If you have multiple tables, don't forget to factor in the potential for <xref linkend="joins"/> into the schema design.
     </para>
   </section>
   <section xml:id="ttl">
@@ -828,22 +899,22 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
   Secondary Indexes and Alternate Query Paths
   </title>
   <para>This section could also be titled "what if my table rowkey looks like <emphasis>this</emphasis> but I also want to query my table like <emphasis>that</emphasis>."
-  A common example on the dist-list is where a row-key is of the format "user-timestamp" but there are are reporting requirements on activity across users for certain 
+  A common example on the dist-list is where a row-key is of the format "user-timestamp" but there are reporting requirements on activity across users for certain
   time ranges.  Thus, selecting by user is easy because it is in the lead position of the key, but time is not.
   </para>
   <para>There is no single answer on the best way to handle this because it depends on...
    <itemizedlist>
-       <listitem>Number of users</listitem>  
+       <listitem>Number of users</listitem>
        <listitem>Data size and data arrival rate</listitem>
-       <listitem>Flexibility of reporting requirements (e.g., completely ad-hoc date selection vs. pre-configured ranges) </listitem>  
-       <listitem>Desired execution speed of query (e.g., 90 seconds may be reasonable to some for an ad-hoc report, whereas it may be too long for others) </listitem>  
+       <listitem>Flexibility of reporting requirements (e.g., completely ad-hoc date selection vs. pre-configured ranges) </listitem>
+       <listitem>Desired execution speed of query (e.g., 90 seconds may be reasonable to some for an ad-hoc report, whereas it may be too long for others) </listitem>
    </itemizedlist>
-   ... and solutions are also influenced by the size of the cluster and how much processing power you have to throw at the solution.  
-   Common techniques are in sub-sections below.  This is a comprehensive, but not exhaustive, list of approaches.   
+   ... and solutions are also influenced by the size of the cluster and how much processing power you have to throw at the solution.
+   Common techniques are in sub-sections below.  This is a comprehensive, but not exhaustive, list of approaches.
   </para>
-  <para>It should not be a surprise that secondary indexes require additional cluster space and processing.  
+  <para>It should not be a surprise that secondary indexes require additional cluster space and processing.
   This is precisely what happens in an RDBMS because the act of creating an alternate index requires both space and processing cycles to update.  RBDMS products
-  are more advanced in this regard to handle alternative index management out of the box.  However, HBase scales better at larger data volumes, so this is a feature trade-off. 
+  are more advanced in this regard to handle alternative index management out of the box.  However, HBase scales better at larger data volumes, so this is a feature trade-off.
   </para>
   <para>Pay attention to <xref linkend="performance"/> when implementing any of these approaches.</para>
   <para>Additionally, see the David Butler response in this dist-list thread <link xlink:href="http://search-hadoop.com/m/nvbiBp2TDP/Stargate%252Bhbase&amp;subj=Stargate+hbase">HBase, mail # user - Stargate+hbase</link>
@@ -860,7 +931,7 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
       <title>
        Periodic-Update Secondary Index
       </title>
-      <para>A secondary index could be created in an other table which is periodically updated via a MapReduce job.  The job could be executed intra-day, but depending on 
+      <para>A secondary index could be created in an other table which is periodically updated via a MapReduce job.  The job could be executed intra-day, but depending on
       load-strategy it could still potentially be out of sync with the main data table.</para>
       <para>See <xref linkend="mapreduce.example.readwrite"/> for more information.</para>
     </section>
@@ -868,7 +939,7 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
       <title>
        Dual-Write Secondary Index
       </title>
-      <para>Another strategy is to build the secondary index while publishing data to the cluster (e.g., write to data table, write to index table). 
+      <para>Another strategy is to build the secondary index while publishing data to the cluster (e.g., write to data table, write to index table).
       If this is approach is taken after a data table already exists, then bootstrapping will be needed for the secondary index with a MapReduce job (see <xref linkend="secondary.indexes.periodic"/>).</para>
     </section>
     <section xml:id="secondary.indexes.summary">
@@ -888,12 +959,12 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
     </section>
   </section>
   <section xml:id="schema.smackdown"><title>Schema Design Smackdown</title>
-    <para>This section will describe common schema design questions that appear on the dist-list.  These are 
-    general guidelines and not laws - each application must consider it's own needs.  
+    <para>This section will describe common schema design questions that appear on the dist-list.  These are
+    general guidelines and not laws - each application must consider its own needs.
     </para>
     <section xml:id="schema.smackdown.rowsversions"><title>Rows vs. Versions</title>
       <para>A common question is whether one should prefer rows or HBase's built-in-versioning.  The context is typically where there are
-      "a lot" of versions of a row to be retained (e.g., where it is significantly above the HBase default of 3 max versions).  The 
+      "a lot" of versions of a row to be retained (e.g., where it is significantly above the HBase default of 3 max versions).  The
       rows-approach would require storing a timstamp in some portion of the rowkey so that they would not overwite with each successive update.
       </para>
       <para>Preference:  Rows (generally speaking).
@@ -901,18 +972,29 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
     </section>
     <section xml:id="schema.smackdown.rowscols"><title>Rows vs. Columns</title>
       <para>Another common question is whether one should prefer rows or columns.  The context is typically in extreme cases of wide
-      tables, such as having 1 row with 1 million attributes, or 1 million rows with 1 columns apiece.  
+      tables, such as having 1 row with 1 million attributes, or 1 million rows with 1 columns apiece.
       </para>
-      <para>Preference:  Rows (generally speaking).  To be clear, this guideline is in the context is in extremely wide cases, not in the 
-      standard use-case where one needs to store a few dozen or hundred columns.
+      <para>Preference:  Rows (generally speaking).  To be clear, this guideline is in the context is in extremely wide cases, not in the
+      standard use-case where one needs to store a few dozen or hundred columns.  But there is also a middle path between these two
+      options, and that is "Rows as Columns."
       </para>
     </section>
+    <section xml:id="schema.smackdown.rowsascols"><title>Rows as Columns</title>
+      <para>The middle path between Rows vs. Columns is packing data that would be a separate row into columns, for certain rows.
+      OpenTSDB is the best example of this case where a single row represents a defined time-range, and then discrete events are treated as
+      columns.  This approach is often more complex, and may require the additional complexity of re-writing your data, but has the
+      advantage of being I/O efficient.  For an overview of this approach, see
+      <link xlink:href="http://www.cloudera.com/content/cloudera/en/resources/library/hbasecon/video-hbasecon-2012-lessons-learned-from-opentsdb.html">Lessons Learned from OpenTSDB</link>
+      from HBaseCon2012.
+      </para>
+    </section>
+
   </section>
   <section xml:id="schema.ops"><title>Operational and Performance Configuration Options</title>
     <para>See the Performance section <xref linkend="perf.schema"/> for more information operational and performance
     schema design options, such as Bloom Filters, Table-configured regionsizes, compression, and blocksizes.
     </para>
-  </section>  
+  </section>
 
   <section xml:id="constraints"><title>Constraints</title>
     <para>HBase currently supports 'constraints' in traditional (SQL) database parlance. The advised usage for Constraints is in enforcing business rules for attributes in the table (eg. make sure values are in the range 1-10).
@@ -942,9 +1024,9 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
     </section>
     <section xml:id="splitter.custom">
     <title>Custom Splitters</title>
-    <para>For those interested in implementing custom splitters, see the method <code>getSplits</code> in 
+    <para>For those interested in implementing custom splitters, see the method <code>getSplits</code> in
     <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.html">TableInputFormatBase</link>.
-    That is where the logic for map-task assignment resides.  
+    That is where the logic for map-task assignment resides.
     </para>
     </section>
   </section>
@@ -959,22 +1041,22 @@ System.out.println("md5 digest as string length: " + sbDigest.length);    // ret
 Configuration config = HBaseConfiguration.create();
 Job job = new Job(config, "ExampleRead");
 job.setJarByClass(MyReadJob.class);     // class that contains mapper
-	
+
 Scan scan = new Scan();
 scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
 scan.setCacheBlocks(false);  // don't set to true for MR jobs
 // set other scan attrs
 ...
-  
+
 TableMapReduceUtil.initTableMapperJob(
   tableName,        // input HBase table name
   scan,             // Scan instance to control CF and attribute selection
   MyMapper.class,   // mapper
-  null,             // mapper output key 
+  null,             // mapper output key
   null,             // mapper output value
   job);
 job.setOutputFormatClass(NullOutputFormat.class);   // because we aren't emitting anything from mapper
-	    
+
 boolean b = job.waitForCompletion(true);
 if (!b) {
   throw new IOException("error with job!");
@@ -987,24 +1069,24 @@ public static class MyMapper extends TableMapper&lt;Text, Text&gt; {
   public void map(ImmutableBytesWritable row, Result value, Context context) throws InterruptedException, IOException {
     // process data for the row from the Result instance.
    }
-}    
+}
     </programlisting>
   	  </para>
   	 </section>
     <section xml:id="mapreduce.example.readwrite">
     <title>HBase MapReduce Read/Write Example</title>
-    <para>The following is an example of using HBase both as a source and as a sink with MapReduce. 
+    <para>The following is an example of using HBase both as a source and as a sink with MapReduce.
     This example will simply copy data from one table to another.
     <programlisting>
 Configuration config = HBaseConfiguration.create();
 Job job = new Job(config,"ExampleReadWrite");
 job.setJarByClass(MyReadWriteJob.class);    // class that contains mapper
-	        	        
+
 Scan scan = new Scan();
 scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
 scan.setCacheBlocks(false);  // don't set to true for MR jobs
 // set other scan attrs
-	        
+
 TableMapReduceUtil.initTableMapperJob(
 	sourceTable,      // input table
 	scan,	          // Scan instance to control CF and attribute selection
@@ -1017,17 +1099,17 @@ TableMapReduceUtil.initTableReducerJob(
 	null,             // reducer class
 	job);
 job.setNumReduceTasks(0);
-	        
+
 boolean b = job.waitForCompletion(true);
 if (!b) {
     throw new IOException("error with job!");
 }
     </programlisting>
-	An explanation is required of what <classname>TableMapReduceUtil</classname> is doing, especially with the reducer.  
+	An explanation is required of what <classname>TableMapReduceUtil</classname> is doing, especially with the reducer.
 	<link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.html">TableOutputFormat</link> is being used
 	as the outputFormat class, and several parameters are being set on the config (e.g., TableOutputFormat.OUTPUT_TABLE), as
 	well as setting the reducer output key to <classname>ImmutableBytesWritable</classname> and reducer value to <classname>Writable</classname>.
-	These could be set by the programmer on the job and conf, but <classname>TableMapReduceUtil</classname> tries to make things easier.    
+	These could be set by the programmer on the job and conf, but <classname>TableMapReduceUtil</classname> tries to make things easier.
 	<para>The following is the example mapper, which will create a <classname>Put</classname> and matching the input <classname>Result</classname>
 	and emit it.  Note:  this is what the CopyTable utility does.
 	</para>
@@ -1038,7 +1120,7 @@ public static class MyMapper extends TableMapper&lt;ImmutableBytesWritable, Put&
 		// this example is just copying the data from the source table...
    		context.write(row, resultToPut(row,value));
    	}
-        
+
   	private static Put resultToPut(ImmutableBytesWritable key, Result result) throws IOException {
   		Put put = new Put(key.get());
  		for (KeyValue kv : result.raw()) {
@@ -1049,9 +1131,9 @@ public static class MyMapper extends TableMapper&lt;ImmutableBytesWritable, Put&
 }
     </programlisting>
     <para>There isn't actually a reducer step, so <classname>TableOutputFormat</classname> takes care of sending the <classname>Put</classname>
-    to the target table. 
+    to the target table.
     </para>
-    <para>This is just an example, developers could choose not to use <classname>TableOutputFormat</classname> and connect to the 
+    <para>This is just an example, developers could choose not to use <classname>TableOutputFormat</classname> and connect to the
     target table themselves.
     </para>
     </para>
@@ -1063,18 +1145,18 @@ public static class MyMapper extends TableMapper&lt;ImmutableBytesWritable, Put&
     </section>
     <section xml:id="mapreduce.example.summary">
     <title>HBase MapReduce Summary to HBase Example</title>
-    <para>The following example uses HBase as a MapReduce source and sink with a summarization step.  This example will 
+    <para>The following example uses HBase as a MapReduce source and sink with a summarization step.  This example will
     count the number of distinct instances of a value in a table and write those summarized counts in another table.
     <programlisting>
 Configuration config = HBaseConfiguration.create();
 Job job = new Job(config,"ExampleSummary");
 job.setJarByClass(MySummaryJob.class);     // class that contains mapper and reducer
-	        
+
 Scan scan = new Scan();
 scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
 scan.setCacheBlocks(false);  // don't set to true for MR jobs
 // set other scan attrs
-	        
+
 TableMapReduceUtil.initTableMapperJob(
 	sourceTable,        // input table
 	scan,               // Scan instance to control CF and attribute selection
@@ -1087,20 +1169,20 @@ TableMapReduceUtil.initTableReducerJob(
 	MyTableReducer.class,    // reducer class
 	job);
 job.setNumReduceTasks(1);   // at least one, adjust as required
-	    
+
 boolean b = job.waitForCompletion(true);
 if (!b) {
 	throw new IOException("error with job!");
-}    
+}
     </programlisting>
-    In this example mapper a column with a String-value is chosen as the value to summarize upon.  
+    In this example mapper a column with a String-value is chosen as the value to summarize upon.
     This value is used as the key to emit from the mapper, and an <classname>IntWritable</classname> represents an instance counter.
     <programlisting>
 public static class MyMapper extends TableMapper&lt;Text, IntWritable&gt;  {
 
 	private final IntWritable ONE = new IntWritable(1);
    	private Text text = new Text();
-    	
+
    	public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException {
         	String val = new String(value.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr1")));
           	text.set(val);     // we can only emit Writables...
@@ -1112,7 +1194,7 @@ public static class MyMapper extends TableMapper&lt;Text, IntWritable&gt;  {
     In the reducer, the "ones" are counted (just like any other MR example that does this), and then emits a <classname>Put</classname>.
     <programlisting>
 public static class MyTableReducer extends TableReducer&lt;Text, IntWritable, ImmutableBytesWritable&gt;  {
-        
+
  	public void reduce(Text key, Iterable&lt;IntWritable&gt; values, Context context) throws IOException, InterruptedException {
     		int i = 0;
     		for (IntWritable val : values) {
@@ -1131,17 +1213,17 @@ public static class MyTableReducer extends TableReducer&lt;Text, IntWritable, Im
     <title>HBase MapReduce Summary to File Example</title>
        <para>This very similar to the summary example above, with exception that this is using HBase as a MapReduce source
        but HDFS as the sink.  The differences are in the job setup and in the reducer.  The mapper remains the same.
-       </para> 
+       </para>
     <programlisting>
 Configuration config = HBaseConfiguration.create();
 Job job = new Job(config,"ExampleSummaryToFile");
 job.setJarByClass(MySummaryFileJob.class);     // class that contains mapper and reducer
-	        
+
 Scan scan = new Scan();
 scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
 scan.setCacheBlocks(false);  // don't set to true for MR jobs
 // set other scan attrs
-	        
+
 TableMapReduceUtil.initTableMapperJob(
 	sourceTable,        // input table
 	scan,               // Scan instance to control CF and attribute selection
@@ -1152,22 +1234,22 @@ TableMapReduceUtil.initTableMapperJob(
 job.setReducerClass(MyReducer.class);    // reducer class
 job.setNumReduceTasks(1);    // at least one, adjust as required
 FileOutputFormat.setOutputPath(job, new Path("/tmp/mr/mySummaryFile"));  // adjust directories as required
-	    
+
 boolean b = job.waitForCompletion(true);
 if (!b) {
 	throw new IOException("error with job!");
-}    
+}
     </programlisting>
-    As stated above, the previous Mapper can run unchanged with this example.  
+    As stated above, the previous Mapper can run unchanged with this example.
     As for the Reducer, it is a "generic" Reducer instead of extending TableMapper and emitting Puts.
     <programlisting>
  public static class MyReducer extends Reducer&lt;Text, IntWritable, Text, IntWritable&gt;  {
-        
+
 	public void reduce(Text key, Iterable&lt;IntWritable&gt; values, Context context) throws IOException, InterruptedException {
 		int i = 0;
 		for (IntWritable val : values) {
 			i += val.get();
-		}	
+		}
 		context.write(key, new IntWritable(i));
 	}
 }
@@ -1176,11 +1258,11 @@ if (!b) {
    <section xml:id="mapreduce.example.summary.noreducer">
     <title>HBase MapReduce Summary to HBase Without Reducer</title>
        <para>It is also possible to perform summaries without a reducer - if you use HBase as the reducer.
-       </para> 
+       </para>
        <para>An HBase target table would need to exist for the job summary.  The HTable method <code>incrementColumnValue</code>
-       would be used to atomically increment values.  From a performance perspective, it might make sense to keep a Map 
+       would be used to atomically increment values.  From a performance perspective, it might make sense to keep a Map
        of values with their values to be incremeneted for each map-task, and make one update per key at during the <code>
-       cleanup</code> method of the mapper.  However, your milage may vary depending on the number of rows to be processed and 
+       cleanup</code> method of the mapper.  However, your milage may vary depending on the number of rows to be processed and
        unique keys.
        </para>
        <para>In the end, the summary results are in HBase.
@@ -1192,41 +1274,41 @@ if (!b) {
        to generate summaries directly to an RDBMS via a custom reducer.  The <code>setup</code> method
        can connect to an RDBMS (the connection information can be passed via custom parameters in the context) and the
        cleanup method can close the connection.
-       </para> 
+       </para>
        <para>It is critical to understand that number of reducers for the job affects the summarization implementation, and
        you'll have to design this into your reducer.  Specifically, whether it is designed to run as a singleton (one reducer)
        or multiple reducers.  Neither is right or wrong, it depends on your use-case.  Recognize that the more reducers that
-       are assigned to the job, the more simultaneous connections to the RDBMS will be created - this will scale, but only to a point. 
+       are assigned to the job, the more simultaneous connections to the RDBMS will be created - this will scale, but only to a point.
        </para>
     <programlisting>
  public static class MyRdbmsReducer extends Reducer&lt;Text, IntWritable, Text, IntWritable&gt;  {
 
 	private Connection c = null;
-	
+
 	public void setup(Context context) {
   		// create DB connection...
   	}
-        
+
 	public void reduce(Text key, Iterable&lt;IntWritable&gt; values, Context context) throws IOException, InterruptedException {
 		// do summarization
 		// in this example the keys are Text, but this is just an example
 	}
-	
+
 	public void cleanup(Context context) {
   		// close db connection
   	}
-	
+
 }
     </programlisting>
        <para>In the end, the summary results are written to your RDBMS table/s.
        </para>
    </section>
-   
+
    </section> <!--  mr examples -->
    <section xml:id="mapreduce.htable.access">
    <title>Accessing Other HBase Tables in a MapReduce Job</title>
 	<para>Although the framework currently allows one HBase table as input to a
-    MapReduce job, other HBase tables can 
+    MapReduce job, other HBase tables can
 	be accessed as lookup tables, etc., in a
     MapReduce job via creating an HTable instance in the setup method of the Mapper.
 	<programlisting>public class MyMapper extends TableMapper&lt;Text, LongWritable&gt; {
@@ -1235,12 +1317,12 @@ if (!b) {
   public void setup(Context context) {
     myOtherTable = new HTable("myOtherTable");
   }
-  
+
   public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException {
 	// process Result...
 	// use 'myOtherTable' for lookups
   }
-  
+
   </programlisting>
    </para>
     </section>
@@ -1253,10 +1335,13 @@ if (!b) {
       map-tasks which will double-write your data to HBase; this is probably
       not what you want.
   </para>
+  <para>See <xref linkend="spec.ex"/> for more information.
+  </para>
   </section>
   </chapter>  <!--  mapreduce -->
 
- 
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="security.xml" />
+
   <chapter xml:id="architecture">
     <title>Architecture</title>
 	<section xml:id="arch.overview">
@@ -1264,24 +1349,24 @@ if (!b) {
 	  <section xml:id="arch.overview.nosql">
 	  <title>NoSQL?</title>
 	  <para>HBase is a type of "NoSQL" database.  "NoSQL" is a general term meaning that the database isn't an RDBMS which
-	  supports SQL as it's primary access language, but there are many types of NoSQL databases:  BerkeleyDB is an 
+	  supports SQL as its primary access language, but there are many types of NoSQL databases:  BerkeleyDB is an
 	  example of a local NoSQL database, whereas HBase is very much a distributed database.  Technically speaking,
 	  HBase is really more a "Data Store" than "Data Base" because it lacks many of the features you find in an RDBMS,
 	  such as typed columns, secondary indexes, triggers, and advanced query languages, etc.
 	  </para>
 	  <para>However, HBase has many features which supports both linear and modular scaling.  HBase clusters expand
-	  by adding RegionServers that are hosted on commodity class servers. If a cluster expands from 10 to 20 
+	  by adding RegionServers that are hosted on commodity class servers. If a cluster expands from 10 to 20
 	  RegionServers, for example, it doubles both in terms of storage and as well as processing capacity.
 	  RDBMS can scale well, but only up to a point - specifically, the size of a single database server - and for the best
 	  performance requires specialized hardware and storage devices.  HBase features of note are:
 	        <itemizedlist>
-              <listitem>Strongly consistent reads/writes:  HBase is not an "eventually consistent" DataStore.  This 
+              <listitem>Strongly consistent reads/writes:  HBase is not an "eventually consistent" DataStore.  This
               makes it very suitable for tasks such as high-speed counter aggregation.  </listitem>
               <listitem>Automatic sharding:  HBase tables are distributed on the cluster via regions, and regions are
               automatically split and re-distributed as your data grows.</listitem>
               <listitem>Automatic RegionServer failover</listitem>
-              <listitem>Hadoop/HDFS Integration:  HBase supports HDFS out of the box as it's distributed file system.</listitem>
-              <listitem>MapReduce:  HBase supports massively parallelized processing via MapReduce for using HBase as both 
+              <listitem>Hadoop/HDFS Integration:  HBase supports HDFS out of the box as its distributed file system.</listitem>
+              <listitem>MapReduce:  HBase supports massively parallelized processing via MapReduce for using HBase as both
               source and sink.</listitem>
               <listitem>Java Client API:  HBase supports an easy to use Java API for programmatic access.</listitem>
               <listitem>Thrift/REST API:  HBase also supports Thrift and REST for non-Java front-ends.</listitem>
@@ -1289,12 +1374,12 @@ if (!b) {
               <listitem>Operational Management:  HBase provides build-in web-pages for operational insight as well as JMX metrics.</listitem>
             </itemizedlist>
 	  </para>
-      </section>      
-	
+      </section>
+
 	  <section xml:id="arch.overview.when">
 	    <title>When Should I Use HBase?</title>
 	    	  <para>HBase isn't suitable for every problem.</para>
-	          <para>First, make sure you have enough data.  If you have hundreds of millions or billions of rows, then 
+	          <para>First, make sure you have enough data.  If you have hundreds of millions or billions of rows, then
 	            HBase is a good candidate.  If you only have a few thousand/million rows, then using a traditional RDBMS
 	            might be a better choice due to the fact that all of your data might wind up on a single node (or two) and
 	            the rest of the cluster may be sitting idle.
@@ -1302,7 +1387,7 @@ if (!b) {
 	          <para>Second, make sure you can live without all the extra features that an RDBMS provides (e.g., typed columns,
 	          secondary indexes, transactions, advanced query languages, etc.)  An application built against an RDBMS cannot be
 	          "ported" to HBase by simply changing a JDBC driver, for example.  Consider moving from an RDBMS to HBase as a
-	          complete redesign as opposed to a port.	          
+	          complete redesign as opposed to a port.
               </para>
 	          <para>Third, make sure you have enough hardware.  Even HDFS doesn't do well with anything less than
                 5 DataNodes (due to things such as HDFS block replication which has a default of 3), plus a NameNode.
@@ -1313,9 +1398,9 @@ if (!b) {
       </section>
       <section xml:id="arch.overview.hbasehdfs">
         <title>What Is The Difference Between HBase and Hadoop/HDFS?</title>
-          <para><link xlink:href="http://hadoop.apache.org/hdfs/">HDFS</link> is a distributed file system that is well suited for the storage of large files. 
-          It's documentation states that it is not, however, a general purpose file system, and does not provide fast individual record lookups in files. 
-          HBase, on the other hand, is built on top of HDFS and provides fast record lookups (and updates) for large tables. 
+          <para><link xlink:href="http://hadoop.apache.org/hdfs/">HDFS</link> is a distributed file system that is well suited for the storage of large files.
+          It's documentation states that it is not, however, a general purpose file system, and does not provide fast individual record lookups in files.
+          HBase, on the other hand, is built on top of HDFS and provides fast record lookups (and updates) for large tables.
           This can sometimes be a point of conceptual confusion.  HBase internally puts your data in indexed "StoreFiles" that exist
           on HDFS for high-speed lookups.  See the <xref linkend="datamodel" /> and the rest of this chapter for more information on how HBase achieves its goals.
          </para>
@@ -1324,19 +1409,19 @@ if (!b) {
 
 	<section xml:id="arch.catalog">
 	 <title>Catalog Tables</title>
-	  <para>The catalog tables -ROOT- and .META. exist as HBase tables.  They are are filtered out 
+	  <para>The catalog tables -ROOT- and .META. exist as HBase tables.  They are filtered out
 	  of the HBase shell's <code>list</code> command, but they are in fact tables just like any other.
      </para>
 	  <section xml:id="arch.catalog.root">
 	   <title>ROOT</title>
-	   <para>-ROOT- keeps track of where the .META. table is.  The -ROOT- table structure is as follows: 
+	   <para>-ROOT- keeps track of where the .META. table is.  The -ROOT- table structure is as follows:
        </para>
-       <para>Key:   
+       <para>Key:
             <itemizedlist>
               <listitem>.META. region key (<code>.META.,,1</code>)</listitem>
             </itemizedlist>
        </para>
-       <para>Values:   
+       <para>Values:
             <itemizedlist>
               <listitem><code>info:regioninfo</code> (serialized <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HRegionInfo.html">HRegionInfo</link>
                instance of .META.)</listitem>
@@ -1347,14 +1432,14 @@ if (!b) {
 	   </section>
 	  <section xml:id="arch.catalog.meta">
 	   <title>META</title>
-	   <para>The .META. table keeps a list of all regions in the system. The .META. table structure is as follows: 
+	   <para>The .META. table keeps a list of all regions in the system. The .META. table structure is as follows:
        </para>
-       <para>Key:   
+       <para>Key:
             <itemizedlist>
               <listitem>Region key of the format (<code>[table],[region start key],[region id]</code>)</listitem>
             </itemizedlist>
        </para>
-       <para>Values:   
+       <para>Values:
             <itemizedlist>
               <listitem><code>info:regioninfo</code> (serialized <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HRegionInfo.html">
               HRegionInfo</link> instance for this region)
@@ -1363,12 +1448,12 @@ if (!b) {
               <listitem><code>info:serverstartcode</code> (start-time of the RegionServer process containing this region)</listitem>
             </itemizedlist>
        </para>
-       <para>When a table is in the process of splitting two other columns will be created, <code>info:splitA</code> and <code>info:splitB</code> 
+       <para>When a table is in the process of splitting two other columns will be created, <code>info:splitA</code> and <code>info:splitB</code>
        which represent the two daughter regions.  The values for these columns are also serialized HRegionInfo instances.
        After the region has been split eventually this row will be deleted.
        </para>
        <para>Notes on HRegionInfo:  the empty key is used to denote table start and table end.  A region with an empty start key
-       is the first region in a table.  If region has both an empty start and an empty end key, its the only region in the table
+       is the first region in a table.  If region has both an empty start and an empty end key, it's the only region in the table
        </para>
        <para>In the (hopefully unlikely) event that programmatic processing of catalog metadata is required, see the
          <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/util/Writables.html#getHRegionInfo%28byte[]%29">Writables</link> utility.
@@ -1380,9 +1465,9 @@ if (!b) {
 	    </para>
 	    <para>For information on region-RegionServer assignment, see <xref linkend="regions.arch.assignment"/>.
 	    </para>
-	    </section>	   
+	    </section>
      </section>  <!--  catalog -->
-     
+
 	<section xml:id="client">
 	 <title>Client</title>
      <para>The HBase client
@@ -1398,7 +1483,7 @@ if (!b) {
          need not go through the lookup process.  Should a region be reassigned
          either by the master load balancer or because a RegionServer has died,
          the client will requery the catalog tables to determine the new
-         location of the user region. 
+         location of the user region.
     </para>
     <para>See <xref linkend="master.runtime"/> for more information about the impact of the Master on HBase Client
     communication.
@@ -1406,10 +1491,11 @@ if (!b) {
     <para>Administrative functions are handled through <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HBaseAdmin.html">HBaseAdmin</link>
     </para>
 	   <section xml:id="client.connections"><title>Connections</title>
-           <para>For connection configuration information, see <xref linkend="client_dependencies" />. 
+           <para>For connection configuration information, see <xref linkend="client_dependencies" />.
          </para>
-         <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html">HTable</link>
-instances are not thread-safe.  When creating HTable instances, it is advisable to use the same <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HBaseConfiguration">HBaseConfiguration</link>
+         <para><emphasis><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html">HTable</link>
+                 instances are not thread-safe</emphasis>.  Only one thread use an instance of HTable at any given
+             time.  When creating HTable instances, it is advisable to use the same <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HBaseConfiguration">HBaseConfiguration</link>
 instance.  This will ensure sharing of ZooKeeper and socket instances to the RegionServers
 which is usually what you want.  For example, this is preferred:
 		<programlisting>HBaseConfiguration conf = HBaseConfiguration.create();
@@ -1425,7 +1511,16 @@ HTable table2 = new HTable(conf2, "myTable");</programlisting>
           </para>
           <section xml:id="client.connection.pooling"><title>Connection Pooling</title>
             <para>For applications which require high-end multithreaded access (e.g., web-servers or application servers that may serve many application threads
-            in a single JVM), see <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTablePool.html">HTablePool</link>.
+            in a single JVM), one solution is <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTablePool.html">HTablePool</link>.
+            But as written currently, it is difficult to control client resource consumption when using HTablePool.
+            </para>
+            <para>
+                Another solution is to precreate an <classname>HConnection</classname> using
+                <programlisting>HConnectionManager.createConnection(Configuration)</programlisting> as
+                well as an <classname>ExecutorService</classname>; then use the
+                <programlisting>HTable(byte[], HConnection, ExecutorService)</programlisting>
+                constructor to create <classname>HTable</classname> instances on demand.
+                This construction is very lightweight and resources are controlled/shared if you go this route.
             </para>
           </section>
    	  </section>
@@ -1436,9 +1531,9 @@ HTable table2 = new HTable(conf2, "myTable");</programlisting>
                is filled.  The writebuffer is 2MB by default.  Before an HTable instance is
                discarded, either <methodname>close()</methodname> or
                <methodname>flushCommits()</methodname> should be invoked so Puts
-               will not be lost.   
-	      </para> 
-	      <para>Note: <code>htable.delete(Delete);</code> does not go in the writebuffer!  This only applies to Puts.   
+               will not be lost.
+	      </para>
+	      <para>Note: <code>htable.delete(Delete);</code> does not go in the writebuffer!  This only applies to Puts.
 	      </para>
 	      <para>For additional information on write durability, review the <link xlink:href="acid-semantics.html">ACID semantics</link> page.
 	      </para>
@@ -1456,15 +1551,15 @@ HTable table2 = new HTable(conf2, "myTable");</programlisting>
            in the client API <emphasis>however</emphasis> they are discouraged because if not managed properly these can
            lock up the RegionServers.
            </para>
-           <para>There is an oustanding ticket <link xlink:href="https://issues.apache.org/jira/browse/HBASE-2332">HBASE-2332</link> to 
+           <para>There is an oustanding ticket <link xlink:href="https://issues.apache.org/jira/browse/HBASE-2332">HBASE-2332</link> to
            remove this feature from the client.
            </para>
 		</section>
 	</section>
-	
+
     <section xml:id="client.filter"><title>Client Request Filters</title>
       <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Get.html">Get</link> and <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html">Scan</link> instances can be
-       optionally configured with <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/Filter.html">filters</link> which are applied on the RegionServer. 
+       optionally configured with <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/Filter.html">filters</link> which are applied on the RegionServer.
       </para>
       <para>Filters can be confusing because there are many different types, and it is best to approach them by understanding the groups
       of Filter functionality.
@@ -1473,8 +1568,8 @@ HTable table2 = new HTable(conf2, "myTable");</programlisting>
         <para>Structural Filters contain other Filters.</para>
         <section xml:id="client.filter.structural.fl"><title>FilterList</title>
           <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/FilterList.html">FilterList</link>
-          represents a list of Filters with a relationship of <code>FilterList.Operator.MUST_PASS_ALL</code> or 
-          <code>FilterList.Operator.MUST_PASS_ONE</code> between the Filters.  The following example shows an 'or' between two 
+          represents a list of Filters with a relationship of <code>FilterList.Operator.MUST_PASS_ALL</code> or
+          <code>FilterList.Operator.MUST_PASS_ONE</code> between the Filters.  The following example shows an 'or' between two
           Filters (checking for either 'my value' or 'my other value' on the same attribute).
 <programlisting>
 FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ONE);
@@ -1521,7 +1616,7 @@ scan.setFilter(filter);
         </para>
         <section xml:id="client.filter.cvp.rcs"><title>RegexStringComparator</title>
           <para><link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/RegexStringComparator.html">RegexStringComparator</link>
-          supports regular expressions for value comparisons. 
+          supports regular expressions for value comparisons.
 <programlisting>
 RegexStringComparator comp = new RegexStringComparator("my.");   // any value that starts with 'my'
 SingleColumnValueFilter filter = new SingleColumnValueFilter(
@@ -1532,7 +1627,7 @@ SingleColumnValueFilter filter = new SingleColumnValueFilter(
 	);
 scan.setFilter(filter);
 </programlisting>
-          See the Oracle JavaDoc for <link xlink:href="http://download.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html">supported RegEx patterns in Java</link>. 
+          See the Oracle JavaDoc for <link xlink:href="http://download.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html">supported RegEx patterns in Java</link>.
           </para>
         </section>
         <section xml:id="client.filter.cvp.rcs"><title>SubstringComparator</title>
@@ -1663,36 +1758,40 @@ rs.close();
       </section>
       <section xml:id="client.filter.row"><title>RowKey</title>
         <section xml:id="client.filter.row.rf"><title>RowFilter</title>
-          <para>It is generally a better idea to use the startRow/stopRow methods on Scan for row selection, however 
+          <para>It is generally a better idea to use the startRow/stopRow methods on Scan for row selection, however
           <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/RowFilter.html">RowFilter</link> can also be used.</para>
         </section>
       </section>
       <section xml:id="client.filter.utility"><title>Utility</title>
         <section xml:id="client.filter.utility.fkof"><title>FirstKeyOnlyFilter</title>
-          <para>This is primarily used for rowcount jobs.  
+          <para>This is primarily used for rowcount jobs.
           See <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.html">FirstKeyOnlyFilter</link>.</para>
         </section>
       </section>
 	</section>  <!--  client.filter -->
- 
+
     <section xml:id="master"><title>Master</title>
        <para><code>HMaster</code> is the implementation of the Master Server.  The Master server
        is responsible for monitoring all RegionServer instances in the cluster, and is
-       the interface for all metadata changes.  In a distributed cluster, the Master typically runs on the <xref linkend="arch.hdfs.nn" />.
+       the interface for all metadata changes.  In a distributed cluster, the Master typically runs on the <xref linkend="arch.hdfs.nn" /><footnote>
+            <para>J Mohamed Zahoor goes into some more detail on the Master Architecture in this blog posting, <link
+            xlink:href="http://blog.zahoor.in/2012/08/hbase-hmaster-architecture/">HBase HMaster Architecture
+            </link>.</para>
+          </footnote>
        </para>
        <section xml:id="master.startup"><title>Startup Behavior</title>
          <para>If run in a multi-Master environment, all Masters compete to run the cluster.  If the active
-         Master loses it's lease in ZooKeeper (or the Master shuts down), then then the remaining Masters jostle to 
+         Master loses its lease in ZooKeeper (or the Master shuts down), then then the remaining Masters jostle to
          take over the Master role.
          </para>
        </section>
        <section xml:id="master.runtime"><title>Runtime Impact</title>
          <para>A common dist-list question is what happens to an HBase cluster when the Master goes down.  Because the
-         HBase client talks directly to the RegionServers, the cluster can still function in a "steady 
+         HBase client talks directly to the RegionServers, the cluster can still function in a "steady
          state."  Additionally, per <xref linkend="arch.catalog"/> ROOT and META exist as HBase tables (i.e., are
-         not resident in the Master).  However, the Master controls critical functions such as RegionServer failover and 
-         completing region splits.  So while the cluster can still run <emphasis>for a time</emphasis> without the Master, 
-         the Master should be restarted as soon as possible.     
+         not resident in the Master).  However, the Master controls critical functions such as RegionServer failover and
+         completing region splits.  So while the cluster can still run <emphasis>for a time</emphasis> without the Master,
+         the Master should be restarted as soon as possible.
          </para>
        </section>
        <section xml:id="master.api"><title>Interface</title>
@@ -1700,20 +1799,20 @@ rs.close();
          <itemizedlist>
             <listitem>Table (createTable, modifyTable, removeTable, enable, disable)
             </listitem>
-            <listitem>ColumnFamily (addColumn, modifyColumn, removeColumn) 
+            <listitem>ColumnFamily (addColumn, modifyColumn, removeColumn)
             </listitem>
             <listitem>Region (move, assign, unassign)
             </listitem>
          </itemizedlist>
-         For example, when the <code>HBaseAdmin</code> method <code>disableTable</code> is invoked, it is serviced by the Master server. 
+         For example, when the <code>HBaseAdmin</code> method <code>disableTable</code> is invoked, it is serviced by the Master server.
          </para>
        </section>
        <section xml:id="master.processes"><title>Processes</title>
          <para>The Master runs several background threads:
          </para>
          <section xml:id="master.processes.loadbalancer"><title>LoadBalancer</title>
-           <para>Periodically, and when there are not any regions in transition,
-             a load balancer will run and move regions around to balance cluster load.
+           <para>Periodically, and when there are no regions in transition,
+             a load balancer will run and move regions around to balance the cluster's load.
              See <xref linkend="balancer_config" /> for configuring this property.</para>
              <para>See <xref linkend="regions.arch.assignment"/> for more information on region assignment.
              </para>
@@ -1726,18 +1825,18 @@ rs.close();
      </section>
      <section xml:id="regionserver.arch"><title>RegionServer</title>
        <para><code>HRegionServer</code> is the RegionServer implementation.  It is responsible for serving and managing regions.
-       In a distributed cluster, a RegionServer runs on a <xref linkend="arch.hdfs.dn" />.  
+       In a distributed cluster, a RegionServer runs on a <xref linkend="arch.hdfs.dn" />.
        </para>
        <section xml:id="regionserver.arch.api"><title>Interface</title>
          <para>The methods exposed by <code>HRegionRegionInterface</code> contain both data-oriented and region-maintenance methods:
          <itemizedlist>
             <listitem>Data (get, put, delete, next, etc.)
             </listitem>
-            <listitem>Region (splitRegion, compactRegion, etc.)  
+            <listitem>Region (splitRegion, compactRegion, etc.)
             </listitem>
          </itemizedlist>
          For example, when the <code>HBaseAdmin</code> method <code>majorCompact</code> is invoked on a table, the client is actually iterating through
-         all regions for the specified table and requesting a major compaction directly to each region. 
+         all regions for the specified table and requesting a major compaction directly to each region.
          </para>
        </section>
        <section xml:id="regionserver.arch.processes"><title>Processes</title>
@@ -1761,7 +1860,7 @@ rs.close();
          posted.  Documentation will eventually move to this reference guide, but the blog is the most current information available at this time.
          </para>
        </section>
-       
+
      <section xml:id="block.cache">
        <title>Block Cache</title>
        <section xml:id="block.cache.design">
@@ -1849,9 +1948,9 @@ rs.close();
          <title>Purpose</title>
 
         <para>Each RegionServer adds updates (Puts, Deletes) to its write-ahead log (WAL)
-            first, and then to the <xref linkend="store.memstore"/> for the affected <xref linkend="store" />.  
-        This ensures that HBase has durable writes. Without WAL, there is the possibility of data loss in the case of a RegionServer failure 
-        before each MemStore is flushed and new StoreFiles are written.  <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/wal/HLog.html">HLog</link> 
+            first, and then to the <xref linkend="store.memstore"/> for the affected <xref linkend="store" />.
+        This ensures that HBase has durable writes. Without WAL, there is the possibility of data loss in the case of a RegionServer failure
+        before each MemStore is flushed and new StoreFiles are written.  <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/wal/HLog.html">HLog</link>
         is the HBase WAL implementation, and there is one HLog instance per RegionServer.
        </para>The WAL is in HDFS in <filename>/hbase/.logs/</filename> with subdirectories per region.
        <para>
@@ -1875,11 +1974,11 @@ rs.close();
          <section>
          <title><varname>hbase.hlog.split.skip.errors</varname></title>
 
-        <para>When set to <constant>true</constant>, the default, any error
+        <para>When set to <constant>true</constant>, any error
         encountered splitting will be logged, the problematic WAL will be
         moved into the <filename>.corrupt</filename> directory under the hbase
         <varname>rootdir</varname>, and processing will continue. If set to
-        <constant>false</constant>, the exception will be propagated and the
+        <constant>false</constant>, the default, the exception will be propagated and the
         split logged as failed.<footnote>
             <para>See <link
             xlink:href="https://issues.apache.org/jira/browse/HBASE-2958">HBASE-2958
@@ -1912,10 +2011,10 @@ rs.close();
     <section xml:id="regions.arch">
     <title>Regions</title>
     <para>Regions are the basic element of availability and
-     distribution for tables, and are comprised of a Store per Column Family. The heirarchy of objects 
+     distribution for tables, and are comprised of a Store per Column Family. The heirarchy of objects
      is as follows:
 <programlisting>
-<filename>Table</filename>       (HBase table)      
+<filename>Table</filename>       (HBase table)
     <filename>Region</filename>       (Regions for the table)
          <filename>Store</filename>          (Store per ColumnFamily for each Region for the table)
               <filename>MemStore</filename>           (MemStore for each Store for each Region for the table)
@@ -1924,7 +2023,7 @@ rs.close();
  </programlisting>
      For a description of what HBase files look like when written to HDFS, see <xref linkend="trouble.namenode.hbase.objects"/>.
             </para>
-    
+
     <section xml:id="arch.regions.size">
       <title>Region Size</title>
 
@@ -1936,13 +2035,13 @@ rs.close();
           <para>HBase scales by having regions across many servers. Thus if
           you have 2 regions for 16GB data, on a 20 node machine your data
           will be concentrated on just a few machines - nearly the entire
-          cluster will be idle.  This really cant be stressed enough, since a 
-          common problem is loading 200MB data into HBase then wondering why 
+          cluster will be idle.  This really cant be stressed enough, since a
+          common problem is loading 200MB data into HBase then wondering why
           your awesome 10 node cluster isn't doing anything.</para>
         </listitem>
 
         <listitem>
-          <para>On the other hand, high region count has been known to make things slow. 
+          <para>On the other hand, high region count has been known to make things slow.
           This is getting better with each release of HBase, but it is probably better to have
           700 regions than 3000 for the same amount of data.</para>
         </listitem>
@@ -1953,7 +2052,7 @@ rs.close();
         </listitem>
       </itemizedlist>
 
-      <para>When starting off, its probably best to stick to the default region-size, perhaps going
+      <para>When starting off, it's probably best to stick to the default region-size, perhaps going
       smaller for hot tables (or manually split hot regions to spread the load over
       the cluster), or go with larger region sizes if your cell sizes tend to be
       largish (100k and up).</para>
@@ -1977,10 +2076,10 @@ rs.close();
               <listitem>If the region assignment is still valid (i.e., if the RegionServer is still online)
                 then the assignment is kept.
               </listitem>
-              <listitem>If the assignment is invalid, then the <code>LoadBalancerFactory</code> is invoked to assign the 
+              <listitem>If the assignment is invalid, then the <code>LoadBalancerFactory</code> is invoked to assign the
                 region.  The <code>DefaultLoadBalancer</code> will randomly assign the region to a RegionServer.
               </listitem>
-              <listitem>META is updated with the RegionServer assignment (if needed) and the RegionServer start codes 
+              <listitem>META is updated with the RegionServer assignment (if needed) and the RegionServer start codes
               (start time of the RegionServer process) upon region opening by the RegionServer.
               </listitem>
            </orderedlist>
@@ -1996,7 +2095,7 @@ rs.close();
               <listitem>The Master will detect that the RegionServer has failed.
               </listitem>
               <listitem>The region assignments will be considered invalid and will be re-assigned just
-                like the startup sequence.    
+                like the startup sequence.
               </listitem>
             </orderedlist>
            </para>
@@ -2023,14 +2122,14 @@ rs.close();
              <listitem>Third replica is written to a node in another rack (if sufficient nodes)
              </listitem>
            </orderedlist>
-          Thus, HBase eventually achieves locality for a region after a flush or a compaction. 
+          Thus, HBase eventually achieves locality for a region after a flush or a compaction.
           In a RegionServer failover situation a RegionServer may be assigned regions with non-local
           StoreFiles (because none of the replicas are local), however as new data is written
           in the region, or the table is compacted and StoreFiles are re-written, they will become "local"
-          to the RegionServer.  
+          to the RegionServer.
         </para>
         <para>For more information, see <link xlink:href="http://hadoop.apache.org/common/docs/r0.20.205.0/hdfs_design.html#Replica+Placement%3A+The+First+Baby+Steps">HDFS Design on Replica Placement</link>
-        and also Lars George's blog on <link xlink:href="http://www.larsgeorge.com/2010/05/hbase-file-locality-in-hdfs.html">HBase and HDFS locality</link>.      
+        and also Lars George's blog on <link xlink:href="http://www.larsgeorge.com/2010/05/hbase-file-locality-in-hdfs.html">HBase and HDFS locality</link>.
         </para>
       </section>
 
@@ -2048,7 +2147,7 @@ rs.close();
           <para>The default split policy can be overwritten using a custom <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.html">RegionSplitPolicy</link> (HBase 0.94+).
           Typically a custom split policy should extend HBase's default split policy: <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html">ConstantSizeRegionSplitPolicy</link>.
           </para>
-          <para>The policy can set globally through the HBaseConfiguration used or on a per table basis: 
+          <para>The policy can set globally through the HBaseConfiguration used or on a per table basis:
 <programlisting>
 HTableDescriptor myHtd = ...;
 myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName());
@@ -2064,8 +2163,8 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
     <section xml:id="store.memstore">
       <title>MemStore</title>
       <para>The MemStore holds in-memory modifications to the Store.  Modifications are KeyValues.
-       When asked to flush, current memstore is moved to snapshot and is cleared. 
-       HBase continues to serve edits out of new memstore and backing snapshot until flusher reports in that the 
+       When asked to flush, current memstore is moved to snapshot and is cleared.
+       HBase continues to serve edits out of new memstore and backing snapshot until flusher reports in that the
        flush succeeded. At this point the snapshot is let go.</para>
       </section>
     <section xml:id="hfile">
@@ -2076,7 +2175,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
           <para>The <emphasis>hfile</emphasis> file format is based on
               the SSTable file described in the <link xlink:href="http://research.google.com/archive/bigtable.html">BigTable [2006]</link> paper and on
               Hadoop's <link xlink:href="http://hadoop.apache.org/common/docs/current/api/org/apache/hadoop/io/file/tfile/TFile.html">tfile</link>
-              (The unit test suite and the compression harness were taken directly from tfile). 
+              (The unit test suite and the compression harness were taken directly from tfile).
               Schubert Zhang's blog post on <link xlink:ref="http://cloudepr.blogspot.com/2009/09/hfile-block-indexed-file-format-to.html">HFile: A Block-Indexed File Format to Store Sorted Key-Value Pairs</link> makes for a thorough introduction to HBase's hfile.  Matteo Bertozzi has also put up a
               helpful description, <link xlink:href="http://th30z.blogspot.com/2011/02/hbase-io-hfile.html?spref=tw">HBase I/O: HFile</link>.
           </para>
@@ -2103,7 +2202,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
         </para>
       </section>
       </section> <!--  hfile -->
-      
+
       <section xml:id="hfile.blocks">
         <title>Blocks</title>
         <para>StoreFiles are composed of blocks.  The blocksize is configured on a per-ColumnFamily basis.
@@ -2116,7 +2215,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
       <section xml:id="keyvalue">
         <title>KeyValue</title>
         <para>The KeyValue class is the heart of data storage in HBase.  KeyValue wraps a byte array and takes offsets and lengths into passed array
-         at where to start interpreting the content as KeyValue.  
+         at where to start interpreting the content as KeyValue.
         </para>
         <para>The KeyValue format inside a byte array is:
            <itemizedlist>
@@ -2180,7 +2279,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
         <title>Compaction</title>
         <para>There are two types of compactions:  minor and major.  Minor compactions will usually pick up a couple of the smaller adjacent
          StoreFiles and rewrite them as one.  Minors do not drop deletes or expired cells, only major compactions do this.  Sometimes a minor compaction
-         will pick up all the StoreFiles in the Store and in this case it actually promotes itself to being a major compaction.  
+         will pick up all the StoreFiles in the Store and in this case it actually promotes itself to being a major compaction.
          </para>
          <para>After a major compaction runs there will be a single StoreFile per Store, and this will help performance usually.  Caution:  major compactions rewrite all of the Stores data and on a loaded system, this may not be tenable;
              major compactions will usually have to be done manually on large systems.  See <xref linkend="managed.compactions" />.
@@ -2189,7 +2288,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
         </para>
         <section xml:id="compaction.file.selection">
           <title>Compaction File Selection</title>
-          <para>To understand the core algorithm for StoreFile selection, there is some ASCII-art in the <link xlink:href="http://hbase.apache.org/xref/org/apache/hadoop/hbase/regionserver/Store.html#836">Store source code</link> that 
+          <para>To understand the core algorithm for StoreFile selection, there is some ASCII-art in the <link xlink:href="http://hbase.apache.org/xref/org/apache/hadoop/hbase/regionserver/Store.html#836">Store source code</link> that
           will serve as useful reference.  It has been copied below:
 <programlisting>
 /* normal skew:
@@ -2211,16 +2310,16 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
             <listitem><code>hbase.hstore.compaction.min</code> (.90 hbase.hstore.compactionThreshold) (files) Minimum number
             of StoreFiles per Store to be selected for a compaction to occur (default 2).</listitem>
             <listitem><code>hbase.hstore.compaction.max</code> (files) Maximum number of StoreFiles to compact per minor compaction (default 10).</listitem>
-            <listitem><code>hbase.hstore.compaction.min.size</code> (bytes) 
-            Any StoreFile smaller than this setting with automatically be a candidate for compaction.  Defaults to 
+            <listitem><code>hbase.hstore.compaction.min.size</code> (bytes)
+            Any StoreFile smaller than this setting with automatically be a candidate for compaction.  Defaults to
             <code>hbase.hregion.memstore.flush.size</code> (128 mb). </listitem>
-            <listitem><code>hbase.hstore.compaction.max.size</code> (.92) (bytes) 
+            <listitem><code>hbase.hstore.compaction.max.size</code> (.92) (bytes)
             Any StoreFile larger than this setting with automatically be excluded from compaction (default Long.MAX_VALUE). </listitem>
             </itemizedlist>
           </para>
           <para>The minor compaction StoreFile selection logic is size based, and selects a file for compaction when the file
            &lt;= sum(smaller_files) * <code>hbase.hstore.compaction.ratio</code>.
-          </para>                
+          </para>
         </section>
         <section xml:id="compaction.file.selection.example1">
           <title>Minor Compaction File Selection - Example #1 (Basic Example)</title>
@@ -2228,21 +2327,21 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
           <itemizedlist>
             <listitem><code>hbase.store.compaction.ratio</code> = 1.0f </listitem>
             <listitem><code>hbase.hstore.compaction.min</code> = 3 (files) </listitem>>
-            <listitem><code>hbase.hstore.compaction.max</code> = 5 (files) </listitem>>        
+            <listitem><code>hbase.hstore.compaction.max</code> = 5 (files) </listitem>>
             <listitem><code>hbase.hstore.compaction.min.size</code> = 10 (bytes) </listitem>>
             <listitem><code>hbase.hstore.compaction.max.size</code> = 1000 (bytes) </listitem>>
           </itemizedlist>
           The following StoreFiles exist: 100, 50, 23, 12, and 12 bytes apiece (oldest to newest).
           With the above parameters, the files that would be selected for minor compaction are 23, 12, and 12.
-          </para>           
+          </para>
           <para>Why?
           <itemizedlist>
             <listitem>100 --&gt;  No, because sum(50, 23, 12, 12) * 1.0 = 97. </listitem>
             <listitem>50 --&gt;  No, because sum(23, 12, 12) * 1.0 = 47. </listitem>
             <listitem>23 --&gt;  Yes, because sum(12, 12) * 1.0 = 24. </listitem>
-            <listitem>12 --&gt;  Yes, because the previous file has been included, and because this 
+            <listitem>12 --&gt;  Yes, because the previous file has been included, and because this
           does not exceed the the max-file limit of 5  </listitem>
-            <listitem>12 --&gt;  Yes, because the previous file had been included, and because this 
+            <listitem>12 --&gt;  Yes, because the previous file had been included, and because this
           does not exceed the the max-file limit of 5.</listitem>
           </itemizedlist>
           </para>
@@ -2253,19 +2352,19 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
           <itemizedlist>
             <listitem><code>hbase.store.compaction.ratio</code> = 1.0f </listitem>
             <listitem><code>hbase.hstore.compaction.min</code> = 3 (files) </listitem>>
-            <listitem><code>hbase.hstore.compaction.max</code> = 5 (files) </listitem>>        
+            <listitem><code>hbase.hstore.compaction.max</code> = 5 (files) </listitem>>
             <listitem><code>hbase.hstore.compaction.min.size</code> = 10 (bytes) </listitem>>
             <listitem><code>hbase.hstore.compaction.max.size</code> = 1000 (bytes) </listitem>>
           </itemizedlist>
-          </para>          
+          </para>
           <para>The following StoreFiles exist: 100, 25, 12, and 12 bytes apiece (oldest to newest).
-          With the above parameters, the files that would be selected for minor compaction are 23, 12, and 12.         
-          </para>  
+          With the above parameters, no compaction will be started.
+          </para>
           <para>Why?
           <itemizedlist>
             <listitem>100 --&gt; No, because sum(25, 12, 12) * 1.0 = 47</listitem>
             <listitem>25 --&gt;  No, because sum(12, 12) * 1.0 = 24</listitem>
-            <listitem>12 --&gt;  No. Candidate because sum(12) * 1.0 = 12, there are only 2 files to compact and that is less than the threshold of 3</listitem> 
+            <listitem>12 --&gt;  No. Candidate because sum(12) * 1.0 = 12, there are only 2 files to compact and that is less than the threshold of 3</listitem>
             <listitem>12 --&gt;  No. Candidate because the previous StoreFile was, but there are not enough files to compact</listitem>
           </itemizedlist>
           </para>
@@ -2276,13 +2375,13 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
           <itemizedlist>
             <listitem><code>hbase.store.compaction.ratio</code> = 1.0f </listitem>
             <listitem><code>hbase.hstore.compaction.min</code> = 3 (files) </listitem>>
-            <listitem><code>hbase.hstore.compaction.max</code> = 5 (files) </listitem>>        
+            <listitem><code>hbase.hstore.compaction.max</code> = 5 (files) </listitem>>
             <listitem><code>hbase.hstore.compaction.min.size</code> = 10 (bytes) </listitem>>
             <listitem><code>hbase.hstore.compaction.max.size</code> = 1000 (bytes) </listitem>>
           </itemizedlist>
           The following StoreFiles exist: 7, 6, 5, 4, 3, 2, and 1 bytes apiece (oldest to newest).
-          With the above parameters, the files that would be selected for minor compaction are 7, 6, 5, 4, 3.         
-          </para>  
+          With the above parameters, the files that would be selected for minor compaction are 7, 6, 5, 4, 3.
+          </para>
           <para>Why?
           <itemizedlist>
             <listitem>7 --&gt;  Yes, because sum(6, 5, 4, 3, 2, 1) * 1.0 = 21.  Also, 7 is less than the min-size</listitem>
@@ -2303,74 +2402,126 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
           <para><code>hbase.hstore.compaction.min.size</code>.  Because
           this limit represents the "automatic include" limit for all StoreFiles smaller than this value, this value may need to
           be adjusted downwards in write-heavy environments where many 1 or 2 mb StoreFiles are being flushed, because every file
-          will be targeted for compaction and the resulting files may still be under the min-size and require further compaction, etc. 
+          will be targeted for compaction and the resulting files may still be under the min-size and require further compaction, etc.
           </para>
         </section>
       </section>  <!--  compaction -->
 
      </section>  <!--  store -->
-      
-     <section xml:id="blooms">
-     <title>Bloom Filters</title>
-         <para><link xlink:href="http://en.wikipedia.org/wiki/Bloom_filter">Bloom filters</link> were developed over in <link
-    xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBase-1200
-    Add bloomfilters</link>.<footnote>
-        <para>For description of the development process -- why static blooms
-        rather than dynamic -- and for an overview of the unique properties
-        that pertain to blooms in HBase, as well as possible future
-        directions, see the <emphasis>Development Process</emphasis> section
-        of the document <link
-        xlink:href="https://issues.apache.org/jira/secure/attachment/12444007/Bloom_Filters_in_HBase.pdf">BloomFilters
-        in HBase</link> attached to <link
-        xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBase-1200</link>.</para>
-      </footnote><footnote>
-        <para>The bloom filters described here are actually version two of
-        blooms in HBase. In versions up to 0.19.x, HBase had a dynamic bloom
-        option based on work done by the <link
-        xlink:href="http://www.one-lab.org">European Commission One-Lab
-        Project 034819</link>. The core of the HBase bloom work was later
-        pulled up into Hadoop to implement org.apache.hadoop.io.BloomMapFile.
-        Version 1 of HBase blooms never worked that well. Version 2 is a
-        rewrite from scratch though again it starts with the one-lab
-        work.</para>
-      </footnote></para>
-        <para>See also <xref linkend="schema.bloom" /> and <xref linkend="config.bloom" />.
-        </para>
-     
-     <section xml:id="bloom_footprint">
-      <title>Bloom StoreFile footprint</title>
 
-      <para>Bloom filters add an entry to the <classname>StoreFile</classname>
-      general <classname>FileInfo</classname> data structure and then two
-      extra entries to the <classname>StoreFile</classname> metadata
-      section.</para>
-
-      <section>
-        <title>BloomFilter in the <classname>StoreFile</classname>
-        <classname>FileInfo</classname> data structure</title>
+    </section>  <!--  regions -->
 
-          <para><classname>FileInfo</classname> has a
-          <varname>BLOOM_FILTER_TYPE</varname> entry which is set to
-          <varname>NONE</varname>, <varname>ROW</varname> or
-          <varname>ROWCOL.</varname></para>
+	<section xml:id="arch.bulk.load"><title>Bulk Loading</title>
+      <section xml:id="arch.bulk.load.overview"><title>Overview</title>
+      <para>
+        HBase includes several methods of loading data into tables.
+        The most straightforward method is to either use the <code>TableOutputFormat</code>
+        class from a MapReduce job, or use the normal client APIs; however,
+        these are not always the most efficient methods.
+      </para>
+      <para>
+        The bulk load feature uses a MapReduce job to output table data in HBase's internal
+        data format, and then directly loads the generated StoreFiles into a running
+        cluster. Using bulk load will use less CPU and network resources than
+        simply using the HBase API.
+      </para>
+    </section>
+    <section xml:id="arch.bulk.load.arch"><title>Bulk Load Architecture</title>
+      <para>
+        The HBase bulk load process consists of two main steps.
+      </para>
+      <section xml:id="arch.bulk.load.prep"><title>Preparing data via a MapReduce job</title>
+        <para>
+          The first step of a bulk load is to generate HBase data files (StoreFiles) from
+          a MapReduce job using <code>HFileOutputFormat</code>. This output format writes
+          out data in HBase's internal storage format so that they can be
+          later loaded very efficiently into the cluster.
+        </para>
+        <para>
+          In order to function efficiently, <code>HFileOutputFormat</code> must be
+          configured such that each output HFile fits within a single region.
+          In order to do this, jobs whose output will be bulk loaded into HBase
+          use Hadoop's <code>TotalOrderPartitioner</code> class to partition the map output
+          into disjoint ranges of the key space, corresponding to the key
+          ranges of the regions in the table.
+        </para>
+        <para>
+          <code>HFileOutputFormat</code> includes a convenience function,
+          <code>configureIncrementalLoad()</code>, which automatically sets up
+          a <code>TotalOrderPartitioner</code> based on the current region boundaries of a
+          table.
+        </para>
       </section>
-
-      <section>
-        <title>BloomFilter entries in <classname>StoreFile</classname>
-        metadata</title>
-
-          <para><varname>BLOOM_FILTER_META</varname> holds Bloom Size, Hash
-          Function used, etc. Its small in size and is cached on
-          <classname>StoreFile.Reader</classname> load</para>
-          <para><varname>BLOOM_FILTER_DATA</varname> is the actual bloomfilter
-          data. Obtained on-demand. Stored in the LRU cache, if it is enabled
-          (Its enabled by default).</para>
+      <section xml:id="arch.bulk.load.complete"><title>Completing the data load</title>
+        <para>
+          After the data has been prepared using
+          <code>HFileOutputFormat</code>, it is loaded into the cluster using
+          <code>completebulkload</code>. This command line tool iterates
+          through the prepared data files, and for each one determines the
+          region the file belongs to. It then contacts the appropriate Region
+          Server which adopts the HFile, moving it into its storage directory
+          and making the data available to clients.
+        </para>
+        <para>
+          If the region boundaries have changed during the course of bulk load
+          preparation, or between the preparation and completion steps, the
+          <code>completebulkloads</code> utility will automatically split the
+          data files into pieces corresponding to the new boundaries. This
+          process is not optimally efficient, so users should take care to
+          minimize the delay between preparing a bulk load and importing it
+          into the cluster, especially if other clients are simultaneously
+          loading data through other means.
+        </para>
       </section>
-     </section>   
-     </section>   <!--  bloom  -->  
-     
     </section>
-    
+    <section xml:id="arch.bulk.load.import"><title>Importing the prepared data using the completebulkload tool</title>
+      <para>
+        After a data import has been prepared, either by using the
+        <code>importtsv</code> tool with the
+        "<code>importtsv.bulk.output</code>" option or by some other MapReduce
+        job using the <code>HFileOutputFormat</code>, the
+        <code>completebulkload</code> tool is used to import the data into the
+        running cluster.
+      </para>
+      <para>
+        The <code>completebulkload</code> tool simply takes the output path
+        where <code>importtsv</code> or your MapReduce job put its results, and
+        the table name to import into. For example:
+      </para>
+      <code>$ hadoop jar hbase-VERSION.jar completebulkload [-c /path/to/hbase/config/hbase-site.xml] /user/todd/myoutput mytable</code>
+      <para>
+        The <code>-c config-file</code> option can be used to specify a file
+        containing the appropriate hbase parameters (e.g., hbase-site.xml) if
+        not supplied already on the CLASSPATH (In addition, the CLASSPATH must
+        contain the directory that has the zookeeper configuration file if
+        zookeeper is NOT managed by HBase).
+      </para>
+      <para>
+        Note: If the target table does not already exist in HBase, this
+        tool will create the table automatically.</para>
+      <para>
+        This tool will run quickly, after which point the new data will be visible in
+        the cluster.
+      </para>
+    </section>
+    <section xml:id="arch.bulk.load.also"><title>See Also</title>
+      <para>For more information about the referenced utilities, see <xref linkend="importtsv"/> and  <xref linkend="completebulkload"/>.
+      </para>
+    </section>
+    <section xml:id="arch.bulk.load.adv"><title>Advanced Usage</title>
+      <para>
+        Although the <code>importtsv</code> tool is useful in many cases, advanced users may
+        want to generate data programatically, or import data from other formats. To get
+        started doing so, dig into <code>ImportTsv.java</code> and check the JavaDoc for
+        HFileOutputFormat.
+      </para>
+      <para>
+        The import step of the bulk load can also be done programatically. See the
+        <code>LoadIncrementalHFiles</code> class for more information.
+      </para>
+    </section>
+	</section>  <!--  bulk loading -->
+
     <section xml:id="arch.hdfs"><title>HDFS</title>
        <para>As HBase runs on HDFS (and each StoreFile is written as a file on HDFS),
         it is important to have an understanding of the HDFS Architecture
@@ -2389,15 +2540,18 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
          for more information.
          </para>
        </section>
-    </section>       
-    
+    </section>
+
   </chapter>   <!--  architecture -->
-  
+
   <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="external_apis.xml" />
   <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="performance.xml" />
   <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="troubleshooting.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="case_studies.xml" />
   <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="ops_mgt.xml" />
   <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="developer.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="zookeeper.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="community.xml" />
 
 <appendix xml:id="faq">
     <title >FAQ</title>
@@ -2427,6 +2581,21 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
                 </para>
             </answer>
         </qandaentry>
+        <qandaentry>
+            <question><para>How can I find examples of NoSQL/HBase?</para></question>
+            <answer>
+                <para>See the link to the BigTable paper in <xref linkend="other.info" /> in the appendix, as
+                well as the other papers.
+                </para>
+            </answer>
+        </qandaentry>
+        <qandaentry>
+            <question><para>What is the history of HBase?</para></question>
+            <answer>
+                <para>See <xref linkend="hbase.history"/>.
+                </para>
+            </answer>
+        </qandaentry>
     </qandadiv>
     <qandadiv xml:id="faq.arch"><title>Architecture</title>
         <qandaentry xml:id="faq.arch.regions">
@@ -2541,7 +2710,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
             </para></question>
             <answer>
                 <para>
- 	            EC2 issues are a special case.  See Troubleshooting <xref linkend="trouble.ec2" /> and Performance <xref linkend="perf.ec2" /> sections.                
+ 	            EC2 issues are a special case.  See Troubleshooting <xref linkend="trouble.ec2" /> and Performance <xref linkend="perf.ec2" /> sections.
                </para>
             </answer>
         </qandaentry>
@@ -2581,6 +2750,214 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
     </qandaset>
   </appendix>
 
+  <appendix xml:id="hbck.in.depth">
+    <title>hbck In Depth</title>
+	<para>HBaseFsck (hbck) is a tool for checking for region consistency and table integrity problems
+and repairing a corrupted HBase. It works in two basic modes -- a read-only inconsistency
+identifying mode and a multi-phase read-write repair mode.
+	</para>
+	<section>
+	  <title>Running hbck to identify inconsistencies</title>
+To check to see if your HBase cluster has corruptions, run hbck against your HBase cluster:
+<programlisting>
+$ ./bin/hbase hbck
+</programlisting>
+	<para>
+At the end of the commands output it prints OK or tells you the number of INCONSISTENCIES
+present. You may also want to run run hbck a few times because some inconsistencies can be
+transient (e.g. cluster is starting up or a region is splitting). Operationally you may want to run
+hbck regularly and setup alert (e.g. via nagios) if it repeatedly reports inconsistencies .
+A run of hbck will report a list of inconsistencies along with a brief description of the regions and
+tables affected. The using the <code>-details</code> option will report more details including a representative
+listing of all the splits present in all the tables.
+	</para>
+<programlisting>
+$ ./bin/hbase hbck -details
+</programlisting>
+If you just want to know if some tables are corrupted, you can limit hbck to identify inconsistencies
+in only specific tables. For example the following command would only attempt to check table
+TableFoo and TableBar. The benefit is that hbck will run in less time.
+<programlisting>
+$ ./bin/hbase hbck TableFoo TableBar
+</programlisting>
+	</section>
+	<section><title>Inconsistencies</title>
+	<para>
+	If after several runs, inconsistencies continue to be reported, you may have encountered a
+corruption. These should be rare, but in the event they occur newer versions of HBase include
+the hbck tool enabled with automatic repair options.
+	</para>
+	<para>
+	There are two invariants that when violated create inconsistencies in HBase:
+	</para>
+	<itemizedlist>
+	  <listitem>HBase’s region consistency invariant is satisfied if every region is assigned and
+deployed on exactly one region server, and all places where this state kept is in
+accordance.
+	</listitem>
+	<listitem>HBase’s table integrity invariant is satisfied if for each table, every possible row key
+resolves to exactly one region.
+	</listitem>
+	</itemizedlist>
+	<para>
+Repairs generally work in three phases -- a read-only information gathering phase that identifies
+inconsistencies, a table integrity repair phase that restores the table integrity invariant, and then
+finally a region consistency repair phase that restores the region consistency invariant.
+Starting from version 0.90.0, hbck could detect region consistency problems report on a subset
+of possible table integrity problems. It also included the ability to automatically fix the most
+common inconsistency, region assignment and deployment consistency problems. This repair
+could be done by using the <code>-fix</code> command line option. These problems close regions if they are
+open on the wrong server or on multiple region servers and also assigns regions to region
+servers if they are not open.
+</para>
+<para>
+Starting from HBase versions 0.90.7, 0.92.2 and 0.94.0, several new command line options are
+introduced to aid repairing a corrupted HBase. This hbck sometimes goes by the nickname
+“uberhbck”. Each particular version of uber hbck is compatible with the HBase’s of the same
+major version (0.90.7 uberhbck can repair a 0.90.4). However, versions &lt;=0.90.6 and versions
+&lt;=0.92.1 may require restarting the master or failing over to a backup master.
+</para>
+	</section>
+	<section><title>Localized repairs</title>
+	<para>
+	When repairing a corrupted HBase, it is best to repair the lowest risk inconsistencies first.
+These are generally region consistency repairs -- localized single region repairs, that only modify
+in-memory data, ephemeral zookeeper data, or patch holes in the META table.
+Region consistency requires that the HBase instance has the state of the region’s data in HDFS
+(.regioninfo files), the region’s row in the .META. table., and region’s deployment/assignments on
+region servers and the master in accordance. Options for repairing region consistency include:
+	<itemizedlist>
+		<listitem><code>-fixAssignments</code> (equivalent to the 0.90 <code>-fix</code> option) repairs unassigned, incorrectly
+assigned or multiply assigned regions.
+		</listitem>
+		<listitem><code>-fixMeta</code> which removes meta rows when corresponding regions are not present in
+HDFS and adds new meta rows if they regions are present in HDFS while not in META.
+		</listitem>
+	</itemizedlist>
+	To fix deployment and assignment problems you can run this command:
+</para>
+<programlisting>
+$ ./bin/hbase hbck -fixAssignments
+</programlisting>
+To fix deployment and assignment problems as well as repairing incorrect meta rows you can
+run this command:.
+<programlisting>
+$ ./bin/hbase hbck -fixAssignments -fixMeta
+</programlisting>
+There are a few classes of table integrity problems that are low risk repairs. The first two are
+degenerate (startkey == endkey) regions and backwards regions (startkey > endkey). These are
+automatically handled by sidelining the data to a temporary directory (/hbck/xxxx).
+The third low-risk class is hdfs region holes. This can be repaired by using the:
+	<itemizedlist>
+		<listitem><code>-fixHdfsHoles</code> option for fabricating new empty regions on the file system.
+If holes are detected you can use -fixHdfsHoles and should include -fixMeta and -fixAssignments to make the new region consistent.
+		</listitem>
+	</itemizedlist>
+<programlisting>
+$ ./bin/hbase hbck -fixAssignments -fixMeta -fixHdfsHoles
+</programlisting>
+Since this is a common operation, we’ve added a the <code>-repairHoles</code> flag that is equivalent to the
+previous command:
+<programlisting>
+$ ./bin/hbase hbck -repairHoles
+</programlisting>
+If inconsistencies still remain after these steps, you most likely have table integrity problems
+related to orphaned or overlapping regions.
+	</section>
+	<section><title>Region Overlap Repairs</title>
+Table integrity problems can require repairs that deal with overlaps. This is a riskier operation
+because it requires modifications to the file system, requires some decision making, and may
+require some manual steps. For these repairs it is best to analyze the output of a <code>hbck -details</code>
+run so that you isolate repairs attempts only upon problems the checks identify. Because this is
+riskier, there are safeguard that should be used to limit the scope of the repairs.
+WARNING: This is a relatively new and have only been tested on online but idle HBase instances
+(no reads/writes). Use at your own risk in an active production environment!
+The options for repairing table integrity violations include:
+	<itemizedlist>
+		<listitem><code>-fixHdfsOrphans</code> option for “adopting” a region directory that is missing a region
+metadata file (the .regioninfo file).
+		</listitem>
+		<listitem><code>-fixHdfsOverlaps</code> ability for fixing overlapping regions
+		</listitem>
+	</itemizedlist>
+When repairing overlapping regions, a region’s data can be modified on the file system in two
+ways: 1) by merging regions into a larger region or 2) by sidelining regions by moving data to
+“sideline” directory where data could be restored later. Merging a large number of regions is
+technically correct but could result in an extremely large region that requires series of costly
+compactions and splitting operations. In these cases, it is probably better to sideline the regions
+that overlap with the most other regions (likely the largest ranges) so that merges can happen on
+a more reasonable scale. Since these sidelined regions are already laid out in HBase’s native
+directory and HFile format, they can be restored by using HBase’s bulk load mechanism.
+The default safeguard thresholds are conservative. These options let you override the default
+thresholds and to enable the large region sidelining feature.
+	<itemizedlist>
+		<listitem><code>-maxMerge &lt;n&gt;</code> maximum number of overlapping regions to merge
+		</listitem>
+		<listitem><code>-sidelineBigOverlaps</code> if more than maxMerge regions are overlapping, sideline attempt
+to sideline the regions overlapping with the most other regions.
+		</listitem>
+		<listitem><code>-maxOverlapsToSideline &lt;n&gt;</code> if sidelining large overlapping regions, sideline at most n
+regions.
+		</listitem>
+	</itemizedlist>
+
+Since often times you would just want to get the tables repaired, you can use this option to turn
+on all repair options:
+	<itemizedlist>
+		<listitem><code>-repair</code> includes all the region consistency options and only the hole repairing table
+integrity options.
+		</listitem>
+	</itemizedlist>
+Finally, there are safeguards to limit repairs to only specific tables. For example the following
+command would only attempt to check and repair table TableFoo and TableBar.
+<programlisting>
+$ ./bin/hbase hbck -repair TableFoo TableBar
+</programlisting>
+	<section><title>Special cases: Meta is not properly assigned</title>
+There are a few special cases that hbck can handle as well.
+Sometimes the meta table’s only region is inconsistently assigned or deployed. In this case
+there is a special <code>-fixMetaOnly</code> option that can try to fix meta assignments.
+<programlisting>
+$ ./bin/hbase hbck -fixMetaOnly -fixAssignments
+</programlisting>
+	</section>
+	<section><title>Special cases: HBase version file is missing</title>
+HBase’s data on the file system requires a version file in order to start. If this flie is missing, you
+can use the <code>-fixVersionFile</code> option to fabricating a new HBase version file. This assumes that
+the version of hbck you are running is the appropriate version for the HBase cluster.
+	</section>
+	<section><title>Special case: Root and META are corrupt.</title>
+The most drastic corruption scenario is the case where the ROOT or META is corrupted and
+HBase will not start. In this case you can use the OfflineMetaRepair tool create new ROOT
+and META regions and tables.
+This tool assumes that HBase is offline. It then marches through the existing HBase home
+directory, loads as much information from region metadata files (.regioninfo files) as possible
+from the file system. If the region metadata has proper table integrity, it sidelines the original root
+and meta table directories, and builds new ones with pointers to the region directories and their
+data.
+<programlisting>
+$ ./bin/hbase org.apache.hadoop.hbase.util.hbck.OfflineMetaRepair
+</programlisting>
+NOTE: This tool is not as clever as uberhbck but can be used to bootstrap repairs that uberhbck
+can complete.
+If the tool succeeds you should be able to start hbase and run online repairs if necessary.
+	</section>
+	<section><title>Special cases: Offline split parent</title>
+    <para>
+Once a region is split, the offline parent will be cleaned up automatically. Sometimes, daughter regions
+are split again before their parents are cleaned up. HBase can clean up parents in the right order. However,
+there could be some lingering offline split parents sometimes. They are in META, in HDFS, and not deployed.
+But HBase can't clean them up. In this case, you can use the <code>-fixSplitParents</code> option to reset
+them in META to be online and not split. Therefore, hbck can merge them with other regions if fixing
+overlapping regions option is used.
+    </para>
+    <para>
+This option should not normally be used, and it is not in <code>-fixAll</code>.
+    </para>
+	</section>
+	</section>
+  </appendix>
+
   <appendix xml:id="compression">
 
     <title >Compression In HBase<indexterm><primary>Compression</primary></indexterm></title>
@@ -2589,9 +2966,15 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
     <title>CompressionTest Tool</title>
     <para>
     HBase includes a tool to test compression is set up properly.
-    To run it, type <code>/bin/hbase org.apache.hadoop.hbase.util.CompressionTest</code>. 
+    To run it, type <code>/bin/hbase org.apache.hadoop.hbase.util.CompressionTest</code>.
     This will emit usage on how to run the tool.
     </para>
+    <note><title>You need to restart regionserver for it to pick up fixed codecs!</title>
+        <para>Be aware that the regionserver caches the result of the compression check it runs
+            ahead of each region open.  This means
+        that you will have to restart the regionserver for it to notice that you have fixed
+    any codec issues.</para>
+    </note>
     </section>
 
     <section xml:id="hbase.regionserver.codecs">
@@ -2607,7 +2990,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
     hbase.regionserver.codecs
     </varname>
     to your <filename>hbase-site.xml</filename> with a value of
-    codecs to test on startup.  For example if the 
+    codecs to test on startup.  For example if the
     <varname>
     hbase.regionserver.codecs
     </varname> value is <code>lzo,gz</code> and if lzo is not present
@@ -2668,7 +3051,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
             <listitem>
                 <para>
                     Build and install <link xlink:href="http://code.google.com/p/snappy/">snappy</link> on all nodes
-                    of your cluster.
+                    of your cluster (see below)
                 </para>
             </listitem>
             <listitem>
@@ -2689,15 +3072,54 @@ hbase> describe 't1'</programlisting>
         </orderedlist>
 
     </para>
-    </section>
+    <section xml:id="snappy.compression.installation">
+    <title>
+    Installation
+    </title>
+    <para>
+        You will find the snappy library file under the .libs directory from your Snappy build (For example
+        /home/hbase/snappy-1.0.5/.libs/). The file is called libsnappy.so.1.x.x where 1.x.x is the version of the snappy
+        code you are building. You can either copy this file into your hbase directory under libsnappy.so name, or simply
+        create a symbolic link to it.
+    </para>
+
+    <para>
+        The second file you need is the hadoop native library. You will find this file in your hadoop installation directory
+        under lib/native/Linux-amd64-64/ or lib/native/Linux-i386-32/. The file you are looking for is libhadoop.so.1.x.x.
+        Again, you can simply copy this file or link to it, under the name libhadoop.so.
+    </para>
 
+    <para>
+        At the end of the installation, you should have both libsnappy.so and libhadoop.so links or files present into
+        lib/native/Linux-amd64-64 or into lib/native/Linux-i386-32
+    </para>
+    <para>To point hbase at snappy support, in hbase-env.sh set
+        <programlisting>export HBASE_LIBRARY_PATH=/pathtoyourhadoop/lib/native/Linux-amd64-64</programlisting>
+        In <filename>/pathtoyourhadoop/lib/native/Linux-amd64-64</filename> you should have something like:
+        <programlisting>
+        libsnappy.a
+        libsnappy.so
+        libsnappy.so.1
+        libsnappy.so.1.1.2
+    </programlisting>
+    </para>
+    </section>
+    </section>
+    <section xml:id="changing.compression">
+      <title>Changing Compression Schemes</title>
+      <para>A frequent question on the dist-list is how to change compression schemes for ColumnFamilies.  This is actually quite simple,
+      and can be done via an alter command.  Because the compression scheme is encoded at the block-level in StoreFiles, the table does
+      <emphasis>not</emphasis> need to be re-created and the data does <emphasis>not</emphasis> copied somewhere else.  Just make sure
+      the old codec is still available until you are sure that all of the old StoreFiles have been compacted.
+      </para>
+    </section>
   </appendix>
 
   <appendix>
       <title xml:id="ycsb"><link xlink:href="https://github.com/brianfrankcooper/YCSB/">YCSB: The Yahoo! Cloud Serving Benchmark</link> and HBase</title>
       <para>TODO: Describe how YCSB is poor for putting up a decent cluster load.</para>
       <para>TODO: Describe setup of YCSB for HBase</para>
-      <para>Ted Dunning redid YCSB so its mavenized and added facility for verifying workloads.  See <link xlink:href="https://github.com/tdunning/YCSB">Ted Dunning's YCSB</link>.</para>
+      <para>Ted Dunning redid YCSB so it's mavenized and added facility for verifying workloads.  See <link xlink:href="https://github.com/tdunning/YCSB">Ted Dunning's YCSB</link>.</para>
 
   </appendix>
 
@@ -2719,7 +3141,7 @@ hbase> describe 't1'</programlisting>
                  <phrase>HFile Version 1</phrase>
                </textobject>
                <caption>
-                   <para>HFile Version 1 
+                   <para>HFile Version 1
                  </para>
                </caption>
            </inlinemediaobject>
@@ -2762,7 +3184,7 @@ hbase> describe 't1'</programlisting>
                  <phrase>HFile Version 2</phrase>
                </textobject>
                <caption>
-                   <para>HFile Version 2 
+                   <para>HFile Version 2
                  </para>
                </caption>
            </inlinemediaobject>
@@ -2791,7 +3213,7 @@ hbase> describe 't1'</programlisting>
               </listitem>
              <listitem>
                  <para>
-                     META – meta blocks (not used for Bloom filters in version 2 anymore) 
+                     META – meta blocks (not used for Bloom filters in version 2 anymore)
                   </para>
               </listitem>
              <listitem>
@@ -2816,7 +3238,7 @@ hbase> describe 't1'</programlisting>
               </listitem>
              <listitem>
                  <para>
-                     TRAILER – a fixed>size file trailer. As opposed to the above, this is not an 
+                     TRAILER – a fixed>size file trailer. As opposed to the above, this is not an
                      HFile v2 block but a fixed>size (for each HFile version) data structure
                   </para>
               </listitem>
@@ -2831,7 +3253,7 @@ hbase> describe 't1'</programlisting>
          <para>Compressed size of the block's data, not including the header (int).
          </para>
                  <para>
-Can be used for skipping the current data block when scanning HFile data. 
+Can be used for skipping the current data block when scanning HFile data.
                   </para>
       </listitem>
       <listitem>
@@ -2961,12 +3383,12 @@ This offset may point to a data block or to a deeper>level index block.
          <orderedlist>
           <listitem>
              <para>
-Offset of the block referenced by this entry in the file (long) 
+Offset of the block referenced by this entry in the file (long)
              </para>
           </listitem>
           <listitem>
              <para>
-On>disk size of the referenced block (int) 
+On>disk size of the referenced block (int)
              </para>
           </listitem>
           <listitem>
@@ -3207,6 +3629,8 @@ Comparator class used for Bloom filter keys, a UTF>8 encoded string stored   usi
          </para>
          <para><link xlink:href="http://wiki.apache.org/hadoop/HBase/HBasePresentations">HBase Wiki</link> has a page with a number of presentations.
          </para>
+         <para><link xlink:href="http://refcardz.dzone.com/refcardz/hbase">HBase RefCard</link> from DZone.
+         </para>
        </section>
        <section xml:id="other.info.books"><title>HBase Books</title>
          <para><link xlink:href="http://shop.oreilly.com/product/0636920014348.do">HBase:  The Definitive Guide</link> by Lars George.
@@ -3216,16 +3640,29 @@ Comparator class used for Bloom filter keys, a UTF>8 encoded string stored   usi
          <para><link xlink:href="http://shop.oreilly.com/product/9780596521981.do">Hadoop:  The Definitive Guide</link> by Tom White.
          </para>
        </section>
-       
+
+  </appendix>
+
+  <appendix xml:id="hbase.history"><title>HBase History</title>
+    <itemizedlist>
+	  <listitem>2006:  <link xlink:href="http://research.google.com/archive/bigtable.html">BigTable</link> paper published by Google.
+	  </listitem>
+	  <listitem>2006 (end of year):  HBase development starts.
+	  </listitem>
+	  <listitem>2008:  HBase becomes Hadoop sub-project.
+	  </listitem>
+	  <listitem>2010:  HBase becomes Apache top-level project.
+	  </listitem>
+	</itemizedlist>
   </appendix>
 
   <appendix xml:id="asf" ><title>HBase and the Apache Software Foundation</title>
     <para>HBase is a project in the Apache Software Foundation and as such there are responsibilities to the ASF to ensure
     a healthy project.</para>
        <section xml:id="asf.devprocess"><title>ASF Development Process</title>
-        <para>See the <link xlink:href="http://www.apache.org/dev/#committers">Apache Development Process page</link> 
+        <para>See the <link xlink:href="http://www.apache.org/dev/#committers">Apache Development Process page</link>
         for all sorts of information on how the ASF is structured (e.g., PMC, committers, contributors), to tips on contributing
-        and getting involved, and how open-source works at ASF.     
+        and getting involved, and how open-source works at ASF.
         </para>
        </section>
        <section xml:id="asf.reporting"><title>ASF Board Reporting</title>
@@ -3235,6 +3672,67 @@ Comparator class used for Bloom filter keys, a UTF>8 encoded string stored   usi
        </section>
   </appendix>
 
+  <appendix xml:id="tracing" ><title>Enabling Dapper-like Tracing in HBase</title>
+<para><link xlink:href="https://issues.apache.org/jira/browse/HBASE-6449">HBASE-6449</link> added support
+for tracing requests through HBase, using the open source tracing library,
+<link xlink:href="http://github.com/cloudera/htrace">HTrace</link>. Setting up tracing is quite simple,
+however it currently requires some very minor changes to your client code (it would not be very difficult to remove this requirement).
+</para>
+<section xml:id="tracing.spanreceivers"><title>SpanReceivers</title>
+<para>The tracing system works by collecting information in structs called ‘Spans’.
+It is up to you to choose how you want to receive this information by implementing the
+<classname>SpanReceiver</classname> interface, which defines one method:
+<programlisting>public void receiveSpan(Span span);</programlisting>
+This method serves as a callback whenever a span is completed. HTrace allows you to use
+as many SpanReceivers as you want so you can easily send trace information to multiple destinations.
+</para>
+
+<para>Configure what SpanReceivers you’d like to use by putting a comma separated list of the
+fully-qualified class name of classes implementing <classname>SpanReceiver</classname> in
+<filename>hbase-site.xml</filename> property: <varname>hbase.trace.spanreceiver.classes</varname>.
+</para>
+
+<para>HBase includes a <classname>HBaseLocalFileSpanReceiver</classname> that writes all span
+information to local files in a JSON-based format. The <classname>HBaseLocalFileSpanReceiver</classname>
+looks in <filename>hbase-site.xml</filename> for a <varname>hbase.trace.spanreceiver.localfilespanreceiver.filename</varname>
+property with a value describing the name of the file to which nodes should write their span information.
+</para>
+
+<para>If you do not want to use the included <classname>HBaseLocalFileSpanReceiver</classname>,
+you are encouraged to write your own receiver (take a look at <classname>HBaseLocalFileSpanReceiver</classname>
+for an example). If you think others would benefit from your receiver, file a JIRA or send a pull request to
+<link xlink:href="http://github.com/cloudera/htrace">HTrace</link>.
+</para>
+</section>
+<section xml:id="tracing.client.modifications">
+<title>Client Modifications</title>
+<para>Currently, you must turn on tracing in your client code. To do this, you simply turn on tracing for
+requests you think are interesting, and turn it off when the request is done.
+</para>
+
+<para>For example, if you wanted to trace all of your get operations, you change this:
+<programlisting>HTable table = new HTable(...);
+Get get = new Get(...);</programlisting>
+
+into:
+
+<programlisting>Span getSpan = Trace.startSpan(“doing get”, Sampler.ALWAYS);
+try {
+  HTable table = new HTable(...);
+  Get get = new Get(...);
+...
+} finally {
+  getSpan.stop();
+}</programlisting>
+
+If you wanted to trace half of your ‘get’ operations, you would pass in:
+<programlisting>new ProbabilitySampler(0.5)</programlisting> in lieu of <varname>Sampler.ALWAYS</varname> to <classname>Trace.startSpan()</classname>.
+See the HTrace <filename>README</filename> for more information on Samplers.
+</para>
+</section>
+
+  </appendix>
+
   <index xml:id="book_index">
   <title>Index</title>
   </index>
diff --git a/src/docbkx/configuration.xml b/src/docbkx/configuration.xml
index 44936e19e98e..02c49a3482e0 100644
--- a/src/docbkx/configuration.xml
+++ b/src/docbkx/configuration.xml
@@ -26,14 +26,16 @@
  * limitations under the License.
  */
 -->
-    <title>Configuration</title>
-    <para>This chapter is the Not-So-Quick start guide to HBase configuration.</para>
-    <para>Please read this chapter carefully and ensure that all requirements have 
+    <title>Apache HBase (TM) Configuration</title>
+    <para>This chapter is the Not-So-Quick start guide to Apache HBase (TM) configuration.  It goes
+    over system requirements, Hadoop setup, the different Apache HBase run modes, and the
+    various configurations in HBase.  Please read this chapter carefully.  At a mimimum
+    ensure that all <xref linkend="basic.prerequisites" /> have
       been satisfied.  Failure to do so will cause you (and us) grief debugging strange errors
       and/or data loss.</para>
-    
+
     <para>
-        HBase uses the same configuration system as Hadoop.
+        Apache HBase uses the same configuration system as Apache Hadoop.
         To configure a deploy, edit a file of environment variables
         in <filename>conf/hbase-env.sh</filename> -- this configuration
         is used mostly by the launcher shell scripts getting the cluster
@@ -55,17 +57,20 @@ to ensure well-formedness of your document after an edit session.
     content of the <filename>conf</filename> directory to
     all nodes of the cluster.  HBase will not do this for you.
     Use <command>rsync</command>.</para>
-    
+
+    <section xml:id="basic.prerequisites">
+    <title>Basic Prerequisites</title>
+    <para>This section lists required services and some required system configuration.
+    </para>
+
     <section xml:id="java">
         <title>Java</title>
-
-        <para>Just like Hadoop, HBase requires java 6 from <link
-        xlink:href="http://www.java.com/download/">Oracle</link>. Usually
-        you'll want to use the latest version available except the problematic
-        u18 (u24 is the latest version as of this writing).</para>
+        <para>Just like Hadoop, HBase requires at least java 6 from
+        <link xlink:href="http://www.java.com/download/">Oracle</link>.</para>
     </section>
+
     <section xml:id="os">
-        <title>Operating System</title>        
+        <title>Operating System</title>
       <section xml:id="ssh">
         <title>ssh</title>
 
@@ -73,14 +78,20 @@ to ensure well-formedness of your document after an edit session.
         <command>sshd</command> must be running to use Hadoop's scripts to
         manage remote Hadoop and HBase daemons. You must be able to ssh to all
         nodes, including your local node, using passwordless login (Google
-        "ssh passwordless login").</para>
+        "ssh passwordless login").  If on mac osx, see the section,
+        <link xlink:href="http://wiki.apache.org/hadoop/Running_Hadoop_On_OS_X_10.5_64-bit_%28Single-Node_Cluster%29">SSH: Setting up Remote Desktop and Enabling Self-Login</link>
+        on the hadoop wiki.</para>
       </section>
 
       <section xml:id="dns">
         <title>DNS</title>
 
-        <para>HBase uses the local hostname to self-report it's IP address.
-        Both forward and reverse DNS resolving should work.</para>
+        <para>HBase uses the local hostname to self-report its IP address.
+        Both forward and reverse DNS resolving must work in versions of
+        HBase previous to 0.92.0
+        <footnote><para>The <link xlink:href="https://github.com/sujee/hadoop-dns-checker">hadoop-dns-checker</link> tool can be used to verify
+        DNS is working correctly on the cluster.  The project README file provides detailed instructions on usage.
+</para></footnote>.</para>
 
         <para>If your machine has multiple interfaces, HBase will use the
         interface that the primary hostname resolves to.</para>
@@ -97,15 +108,7 @@ to ensure well-formedness of your document after an edit session.
       </section>
       <section xml:id="loopback.ip">
         <title>Loopback IP</title>
-        <para>HBase expects the loopback IP address to be 127.0.0.1.  Ubuntu and some other distributions,
-        for example, will default to 127.0.1.1 and this will cause problems for you.
-        </para>
-        <para><filename>/etc/hosts</filename> should look something like this:
-<programlisting>
-            127.0.0.1 localhost
-            127.0.0.1 ubuntu.ubuntu-domain ubuntu
-</programlisting>
-        </para>
+        <para>HBase expects the loopback IP address to be 127.0.0.1.  See <xref linkend="loopback.ip"/></para>
        </section>
 
       <section xml:id="ntp">
@@ -132,7 +135,7 @@ to ensure well-formedness of your document after an edit session.
             </indexterm>
         </title>
 
-        <para>HBase is a database.  It uses a lot of files all at the same time.
+        <para>Apache HBase is a database.  It uses a lot of files all at the same time.
         The default ulimit -n -- i.e. user file limit -- of 1024 on most *nix systems
         is insufficient (On mac os x its 256). Any significant amount of loading will
         lead you to <xref linkend="trouble.rs.runtime.filehandles"/>.
@@ -141,9 +144,9 @@ to ensure well-formedness of your document after an edit session.
       2010-04-06 03:04:37,542 INFO org.apache.hadoop.hdfs.DFSClient: Abandoning block blk_-6935524980745310745_1391901
       </programlisting> Do yourself a favor and change the upper bound on the
         number of file descriptors. Set it to north of 10k.  The math runs roughly as follows:  per ColumnFamily
-        there is at least one StoreFile and possibly up to 5 or 6 if the region is under load.  Multiply the 
+        there is at least one StoreFile and possibly up to 5 or 6 if the region is under load.  Multiply the
         average number of StoreFiles per ColumnFamily times the number of regions per RegionServer.  For example, assuming
-        that a schema had 3 ColumnFamilies per region with an average of 3 StoreFiles per ColumnFamily, 
+        that a schema had 3 ColumnFamilies per region with an average of 3 StoreFiles per ColumnFamily,
         and there are 100 regions per RegionServer, the JVM will open 3 * 3 * 100 = 900 file descriptors
         (not counting open jar files, config files, etc.)
         </para>
@@ -153,7 +156,7 @@ to ensure well-formedness of your document after an edit session.
         <footnote><para>See Jack Levin's <link xlink:href="">major hdfs issues</link>
                 note up on the user list.</para></footnote>
         <footnote><para>The requirement that a database requires upping of system limits
-        is not peculiar to HBase.  See for example the section
+        is not peculiar to Apache HBase.  See for example the section
         <emphasis>Setting Shell Limits for the Oracle User</emphasis> in
         <link xlink:href="http://www.akadia.com/services/ora_linux_install_10g.html">
         Short Guide to install Oracle 10 on Linux</link>.</para></footnote>.
@@ -198,7 +201,7 @@ to ensure well-formedness of your document after an edit session.
       <section xml:id="windows">
         <title>Windows</title>
 
-        <para>HBase has been little tested running on Windows. Running a
+        <para>Apache HBase has been little tested running on Windows. Running a
         production install of HBase on top of Windows is not
         recommended.</para>
 
@@ -206,32 +209,61 @@ to ensure well-formedness of your document after an edit session.
         xlink:href="http://cygwin.com/">Cygwin</link> to have a *nix-like
         environment for the shell scripts. The full details are explained in
         the <link xlink:href="http://hbase.apache.org/cygwin.html">Windows
-        Installation</link> guide. Also 
+        Installation</link> guide. Also
         <link xlink:href="http://search-hadoop.com/?q=hbase+windows&amp;fc_project=HBase&amp;fc_type=mail+_hash_+dev">search our user mailing list</link> to pick
         up latest fixes figured by Windows users.</para>
       </section>
 
     </section>   <!--  OS -->
-    
+
     <section xml:id="hadoop">
         <title><link
         xlink:href="http://hadoop.apache.org">Hadoop</link><indexterm>
             <primary>Hadoop</primary>
           </indexterm></title>
-         <note><title>Please read all of this section</title>
-         <para>Please read this section to the end.  Up front we
-         wade through the weeds of Hadoop versions.  Later we talk of what you must do in HBase
-         to make it work w/ a particular Hadoop version.</para>
-         </note>
-
-          <para>
-        HBase will lose data unless it is running on an HDFS that has a durable
-        <code>sync</code> implementation. Hadoop 0.20.2, Hadoop 0.20.203.0, and Hadoop 0.20.204.0
-	DO NOT have this attribute.
-        Currently only Hadoop versions 0.20.205.x or any release in excess of this
-        version -- this includes hadoop 1.0.0 -- have a working, durable sync
+         <para>Selecting a Hadoop version is critical for your HBase deployment. Below table shows some information about what versions of Hadoop are supported by various HBase versions. Based on the version of HBase, you should select the most appropriate version of Hadoop. We are not in the Hadoop distro selection business. You can use Hadoop distributions from Apache, or learn about vendor distributions of Hadoop at <link xlink:href="http://wiki.apache.org/hadoop/Distributions%20and%20Commercial%20Support"/></para>
+         <para>
+	     <table>
+		 <title>Hadoop version support matrix</title>
+		 <tgroup cols='4' align='left' colsep='1' rowsep='1'><colspec colname='c1' align='left'/><colspec colname='c2' align='center'/><colspec colname='c3' align='center'/><colspec colname='c4' align='center'/>
+         <thead>
+	     <row><entry>               </entry><entry>HBase-0.92.x</entry><entry>HBase-0.94.x</entry><entry>HBase-0.96</entry></row>
+	     </thead><tbody>
+         <row><entry>Hadoop-0.20.205</entry><entry>S</entry>          <entry>X</entry>           <entry>X</entry></row>
+         <row><entry>Hadoop-0.22.x  </entry><entry>S</entry>          <entry>X</entry>           <entry>X</entry></row>
+         <row><entry>Hadoop-1.0.x   </entry><entry>S</entry>          <entry>S</entry>           <entry>S</entry></row>
+         <row><entry>Hadoop-1.1.x   </entry><entry>NT</entry>         <entry>S</entry>           <entry>S</entry></row>
+         <row><entry>Hadoop-0.23.x  </entry><entry>X</entry>          <entry>S</entry>           <entry>NT</entry></row>
+         <row><entry>Hadoop-2.x     </entry><entry>X</entry>          <entry>S</entry>           <entry>S</entry></row>
+		 </tbody></tgroup></table>
+
+        Where
+		<simplelist type='vert' columns='1'>
+		<member>S = supported and tested,</member>
+		<member>X = not supported,</member>
+		<member>NT = it should run, but not tested enough.</member>
+		</simplelist>
+        </para>
+        <para>
+	Because HBase depends on Hadoop, it bundles an instance of the Hadoop jar under its <filename>lib</filename> directory. The bundled jar is ONLY for use in standalone mode. In distributed mode, it is <emphasis>critical</emphasis> that the version of Hadoop that is out on your cluster match what is under HBase. Replace the hadoop jar found in the HBase lib directory with the hadoop jar you are running on your cluster to avoid version mismatch issues. Make sure you replace the jar in HBase everywhere on your cluster. Hadoop version mismatch issues have various manifestations but often all looks like its hung up.
+    </para>
+    <section xml:id="hadoop.hbase-0.94">
+	<title>Apache HBase 0.92 and 0.94</title>
+	<para>HBase 0.92 and 0.94 versions can work with Hadoop versions, 0.20.205, 0.22.x, 1.0.x, and 1.1.x. HBase-0.94 can additionally work with Hadoop-0.23.x and 2.x, but you may have to recompile the code using the specific maven profile (see top level pom.xml)</para>
+   </section>
+
+    <section xml:id="hadoop.hbase-0.96">
+	<title>Apache HBase 0.96</title>
+	<para>Apache HBase 0.96.0 requires Apache Hadoop 1.x at a minimum, and it can run equally well on hadoop-2.0.
+	As of Apache HBase 0.96.x, Apache Hadoop 1.0.x at least is required. We will no longer run properly on older Hadoops such as 0.20.205 or branch-0.20-append. Do not move to Apache HBase 0.96.x if you cannot upgrade your Hadoop<footnote><para>See <link xlink:href="http://search-hadoop.com/m/7vFVx4EsUb2">HBase, mail # dev - DISCUSS: Have hbase require at least hadoop 1.0.0 in hbase 0.96.0?</link></para></footnote>.</para>
+   </section>
+
+    <section xml:id="hadoop.older.versions">
+	<title>Hadoop versions 0.20.x - 1.x</title>
+	<para>
+     HBase will lose data unless it is running on an HDFS that has a durable
+        <code>sync</code> implementation.  DO NOT use Hadoop 0.20.2, Hadoop 0.20.203.0, and Hadoop 0.20.204.0 which DO NOT have this attribute. Currently only Hadoop versions 0.20.205.x or any release in excess of this version -- this includes hadoop-1.0.0 -- have a working, durable sync
           <footnote>
-          <title>On Hadoop Versions</title>
           <para>The Cloudera blog post <link xlink:href="http://www.cloudera.com/blog/2012/01/an-update-on-apache-hadoop-1-0/">An update on Apache Hadoop 1.0</link>
           by Charles Zedlweski has a nice exposition on how all the Hadoop versions relate.
           Its worth checking out if you are having trouble making sense of the
@@ -250,57 +282,18 @@ to ensure well-formedness of your document after an edit session.
         </programlisting>
         You will have to restart your cluster after making this edit.  Ignore the chicken-little
         comment you'll find in the <filename>hdfs-default.xml</filename> in the
-        description for the <varname>dfs.support.append</varname> configuration; it says it is not enabled because there
-        are <quote>... bugs in the 'append code' and is not supported in any production
-        cluster.</quote>. This comment is stale, from another era, and while I'm sure there
-        are bugs, the sync/append code has been running
-        in production at large scale deploys and is on
-        by default in the offerings of hadoop by commercial vendors
-        <footnote><para>Until recently only the
-        <link xlink:href="http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/">branch-0.20-append</link>
-        branch had a working sync but no official release was ever made from this branch.
-        You had to build it yourself. Michael Noll wrote a detailed blog,
-        <link xlink:href="http://www.michael-noll.com/blog/2011/04/14/building-an-hadoop-0-20-x-version-for-hbase-0-90-2/">Building
-        an Hadoop 0.20.x version for HBase 0.90.2</link>, on how to build an
-    Hadoop from branch-0.20-append.  Recommended.</para></footnote>
-    <footnote><para>Praveen Kumar has written
-            a complimentary article,
-            <link xlink:href="http://praveen.kumar.in/2011/06/20/building-hadoop-and-hbase-for-hbase-maven-application-development/">Building Hadoop and HBase for HBase Maven application development</link>.
-</para></footnote><footnote>Cloudera have <varname>dfs.support.append</varname> set to true by default.</footnote>.</para>
-
-<para>Or use the
-    <link xlink:href="http://www.cloudera.com/">Cloudera</link> or
-    <link xlink:href="http://www.mapr.com/">MapR</link> distributions.
-    Cloudera' <link xlink:href="http://archive.cloudera.com/docs/">CDH3</link>
-    is Apache Hadoop 0.20.x plus patches including all of the 
-    <link xlink:href="http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/">branch-0.20-append</link>
-    additions needed to add a durable sync. Use the released, most recent version of CDH3.</para>
-    <para>
-    <link xlink:href="http://www.mapr.com/">MapR</link>
-    includes a commercial, reimplementation of HDFS.
-    It has a durable sync as well as some other interesting features that are not
-    yet in Apache Hadoop.  Their <link xlink:href="http://www.mapr.com/products/mapr-editions/m3-edition">M3</link>
-    product is free to use and unlimited.
-    </para>
-
-        <para>Because HBase depends on Hadoop, it bundles an instance of the
-        Hadoop jar under its <filename>lib</filename> directory. The bundled jar is ONLY for use in standalone mode.
-        In distributed mode, it is <emphasis>critical</emphasis> that the version of Hadoop that is out
-        on your cluster match what is under HBase.  Replace the hadoop jar found in the HBase
-        <filename>lib</filename> directory with the hadoop jar you are running on
-        your cluster to avoid version mismatch issues. Make sure you
-        replace the jar in HBase everywhere on your cluster.  Hadoop version
-        mismatch issues have various manifestations but often all looks like
-        its hung up.</para>
-
+        description for the <varname>dfs.support.append</varname> configuration.
+     </para>
+     </section>
        <section xml:id="hadoop.security">
-          <title>Hadoop Security</title>
-          <para>HBase will run on any Hadoop 0.20.x that incorporates Hadoop
-          security features -- e.g. Y! 0.20S or CDH3B3 -- as long as you do as
+          <title>Apache HBase on Secure Hadoop</title>
+          <para>Apache HBase will run on any Hadoop 0.20.x that incorporates Hadoop
+          security features as long as you do as
           suggested above and replace the Hadoop jar that ships with HBase
-          with the secure version.</para>
+          with the secure version.  If you want to read more about how to setup
+          Secure HBase, see <xref linkend="hbase.secure.configuration" />.</para>
        </section>
-           
+
        <section xml:id="dfs.datanode.max.xcievers">
         <title><varname>dfs.datanode.max.xcievers</varname><indexterm>
             <primary>xcievers</primary>
@@ -331,9 +324,12 @@ to ensure well-formedness of your document after an edit session.
         java.io.IOException: No live nodes contain current block. Will get new
         block locations from namenode and retry...</code>
         <footnote><para>See <link xlink:href="http://ccgtech.blogspot.com/2010/02/hadoop-hdfs-deceived-by-xciever.html">Hadoop HDFS: Deceived by Xciever</link> for an informative rant on xceivering.</para></footnote></para>
+       <para>See also <xref linkend="casestudies.xceivers"/>
+       </para>
       </section>
-     
+
      </section>    <!--  hadoop -->
+     </section>
 
     <section xml:id="standalone_dist">
       <title>HBase run modes: Standalone and Distributed</title>
@@ -376,7 +372,7 @@ to ensure well-formedness of your document after an edit session.
 
         <para>Distributed modes require an instance of the <emphasis>Hadoop
         Distributed File System</emphasis> (HDFS). See the Hadoop <link
-        xlink:href="http://hadoop.apache.org/common/docs/current/api/overview-summary.html#overview_description">
+        xlink:href="http://hadoop.apache.org/common/docs/r1.1.1/api/overview-summary.html#overview_description">
         requirements and instructions</link> for how to set up a HDFS. Before
         proceeding, ensure you have an appropriate, working HDFS.</para>
 
@@ -395,57 +391,92 @@ to ensure well-formedness of your document after an edit session.
           HBase. Do not use this configuration for production nor for
           evaluating HBase performance.</para>
 
-          <para>Once you have confirmed your HDFS setup, edit
-          <filename>conf/hbase-site.xml</filename>. This is the file into
+	      <para>First, setup your HDFS in <link xlink:href="http://hadoop.apache.org/docs/r1.0.3/single_node_setup.html">pseudo-distributed mode</link>.
+   	      </para>
+	      <para>Next, configure HBase.  Below is an example <filename>conf/hbase-site.xml</filename>.
+          This is the file into
           which you add local customizations and overrides for
-          <xreg linkend="hbase_default_configurations" /> and <xref linkend="hdfs_client_conf" />. Point HBase at the running Hadoop HDFS
-          instance by setting the <varname>hbase.rootdir</varname> property.
-          This property points HBase at the Hadoop filesystem instance to use.
-          For example, adding the properties below to your
-          <filename>hbase-site.xml</filename> says that HBase should use the
-          <filename>/hbase</filename> directory in the HDFS whose namenode is
-          at port 8020 on your local machine, and that it should run with one
-          replica only (recommended for pseudo-distributed mode):</para>
+          <xref linkend="hbase_default_configurations" /> and <xref linkend="hdfs_client_conf" />.
+              Note that the <varname>hbase.rootdir</varname> property points to the
+              local HDFS instance.
+   		  </para>
 
-          <programlisting>
+          <para>Now skip to <xref linkend="confirm" /> for how to start and verify your
+          pseudo-distributed install. <footnote>
+              <para>See <xref linkend="pseudo.extras">Pseudo-distributed
+              mode extras</xref> for notes on how to start extra Masters and
+              RegionServers when running pseudo-distributed.</para>
+            </footnote></para>
+
+          <note>
+            <para>Let HBase create the <varname>hbase.rootdir</varname>
+            directory. If you don't, you'll get warning saying HBase needs a
+            migration run because the directory is missing files expected by
+            HBase (it'll create them if you let it).</para>
+          </note>
+
+  		  <section xml:id="pseudo.config">
+  		  	<title>Pseudo-distributed Configuration File</title>
+			<para>Below is a sample pseudo-distributed file for the node <varname>h-24-30.example.com</varname>.
+<filename>hbase-site.xml</filename>
+<programlisting>
 &lt;configuration&gt;
   ...
   &lt;property&gt;
     &lt;name&gt;hbase.rootdir&lt;/name&gt;
-    &lt;value&gt;hdfs://localhost:8020/hbase&lt;/value&gt;
-    &lt;description&gt;The directory shared by RegionServers.
-    &lt;/description&gt;
+    &lt;value&gt;hdfs://h-24-30.sfo.stumble.net:8020/hbase&lt;/value&gt;
   &lt;/property&gt;
   &lt;property&gt;
-    &lt;name&gt;dfs.replication&lt;/name&gt;
-    &lt;value&gt;1&lt;/value&gt;
-    &lt;description&gt;The replication count for HLog and HFile storage. Should not be greater than HDFS datanode count.
-    &lt;/description&gt;
+    &lt;name&gt;hbase.cluster.distributed&lt;/name&gt;
+    &lt;value&gt;true&lt;/value&gt;
+  &lt;/property&gt;
+  &lt;property&gt;
+    &lt;name&gt;hbase.zookeeper.quorum&lt;/name&gt;
+    &lt;value&gt;h-24-30.sfo.stumble.net&lt;/value&gt;
   &lt;/property&gt;
   ...
 &lt;/configuration&gt;
 </programlisting>
+</para>
 
-          <note>
-            <para>Let HBase create the <varname>hbase.rootdir</varname>
-            directory. If you don't, you'll get warning saying HBase needs a
-            migration run because the directory is missing files expected by
-            HBase (it'll create them if you let it).</para>
-          </note>
+  		  </section>
 
-          <note>
-            <para>Above we bind to <varname>localhost</varname>. This means
-            that a remote client cannot connect. Amend accordingly, if you
-            want to connect from a remote location.</para>
-          </note>
+		  <section xml:id="pseudo.extras">
+		    <title>Pseudo-distributed Extras</title>
+
+		  <section xml:id="pseudo.extras.start">
+		  	<title>Startup</title>
+		    	<para>To start up the initial HBase cluster...
+                   <programlisting>% bin/start-hbase.sh</programlisting>
+                </para>
+            	<para>To start up an extra backup master(s) on the same server run...
+                       <programlisting>% bin/local-master-backup.sh start 1</programlisting>
+                       ... the '1' means use ports 60001 &amp; 60011, and this backup master's logfile will be at <filename>logs/hbase-${USER}-1-master-${HOSTNAME}.log</filename>.
+                </para>
+                <para>To startup multiple backup masters run... <programlisting>% bin/local-master-backup.sh start 2 3</programlisting> You can start up to 9 backup masters (10 total).
+ 				</para>
+				<para>To start up more regionservers...
+     			  <programlisting>% bin/local-regionservers.sh start 1</programlisting>
+     			where '1' means use ports 60201 &amp; 60301 and its logfile will be at <filename>logs/hbase-${USER}-1-regionserver-${HOSTNAME}.log</filename>.
+     			</para>
+     			<para>To add 4 more regionservers in addition to the one you just started by running... <programlisting>% bin/local-regionservers.sh start 2 3 4 5</programlisting>
+     			This supports up to 99 extra regionservers (100 total).
+				</para>
+			</section>
+			<section xml:id="pseudo.options.stop">
+		  	<title>Stop</title>
+    			<para>Assuming you want to stop master backup # 1, run...
+            	<programlisting>% cat /tmp/hbase-${USER}-1-master.pid |xargs kill -9</programlisting>
+            	Note that bin/local-master-backup.sh stop 1 will try to stop the cluster along with the master.
+            	</para>
+            	<para>To stop an individual regionserver, run...
+                	<programlisting>% bin/local-regionservers.sh stop 1
+	                </programlisting>
+				</para>
+			</section>
+
+		  </section>
 
-          <para>Now skip to <xref linkend="confirm" /> for how to start and verify your
-          pseudo-distributed install. <footnote>
-              <para>See <link
-              xlink:href="http://hbase.apache.org/pseudo-distributed.html">Pseudo-distributed
-              mode extras</link> for notes on how to start extra Masters and
-              RegionServers when running pseudo-distributed.</para>
-            </footnote></para>
         </section>
 
         <section xml:id="fully_dist">
@@ -542,7 +573,7 @@ to ensure well-formedness of your document after an edit session.
       <section xml:id="confirm">
         <title>Running and Confirming Your Installation</title>
 
-         
+
 
         <para>Make sure HDFS is running first. Start and stop the Hadoop HDFS
         daemons by running <filename>bin/start-hdfs.sh</filename> over in the
@@ -552,31 +583,31 @@ to ensure well-formedness of your document after an edit session.
         not normally use the mapreduce daemons. These do not need to be
         started.</para>
 
-         
+
 
         <para><emphasis>If</emphasis> you are managing your own ZooKeeper,
         start it and confirm its running else, HBase will start up ZooKeeper
         for you as part of its start process.</para>
 
-         
+
 
         <para>Start HBase with the following command:</para>
 
-         
+
 
         <programlisting>bin/start-hbase.sh</programlisting>
 
-         Run the above from the 
+         Run the above from the
 
         <varname>HBASE_HOME</varname>
 
-         directory. 
+         directory.
 
         <para>You should now have a running HBase instance. HBase logs can be
         found in the <filename>logs</filename> subdirectory. Check them out
         especially if HBase had trouble starting.</para>
 
-         
+
 
         <para>HBase also puts up a UI listing vital attributes. By default its
         deployed on the Master host at port 60010 (HBase RegionServers listen
@@ -586,13 +617,13 @@ to ensure well-formedness of your document after an edit session.
         Master's homepage you'd point your browser at
         <filename>http://master.example.org:60010</filename>.</para>
 
-         
+
 
     <para>Once HBase has started, see the <xref linkend="shell_exercises" /> for how to
         create tables, add data, scan your insertions, and finally disable and
         drop your tables.</para>
 
-         
+
 
         <para>To stop HBase after exiting the HBase shell enter
         <programlisting>$ ./bin/stop-hbase.sh
@@ -602,574 +633,15 @@ stopping hbase...............</programlisting> Shutdown can take a moment to
         until HBase has shut down completely before stopping the Hadoop
         daemons.</para>
 
-         
+
       </section>
      </section>    <!--  run modes -->
-    
-     <section xml:id="zookeeper">
-            <title>ZooKeeper<indexterm>
-                <primary>ZooKeeper</primary>
-              </indexterm></title>
-
-            <para>A distributed HBase depends on a running ZooKeeper cluster.
-            All participating nodes and clients need to be able to access the
-            running ZooKeeper ensemble. HBase by default manages a ZooKeeper
-            "cluster" for you. It will start and stop the ZooKeeper ensemble
-            as part of the HBase start/stop process. You can also manage the
-            ZooKeeper ensemble independent of HBase and just point HBase at
-            the cluster it should use. To toggle HBase management of
-            ZooKeeper, use the <varname>HBASE_MANAGES_ZK</varname> variable in
-            <filename>conf/hbase-env.sh</filename>. This variable, which
-            defaults to <varname>true</varname>, tells HBase whether to
-            start/stop the ZooKeeper ensemble servers as part of HBase
-            start/stop.</para>
-
-            <para>When HBase manages the ZooKeeper ensemble, you can specify
-            ZooKeeper configuration using its native
-            <filename>zoo.cfg</filename> file, or, the easier option is to
-            just specify ZooKeeper options directly in
-            <filename>conf/hbase-site.xml</filename>. A ZooKeeper
-            configuration option can be set as a property in the HBase
-            <filename>hbase-site.xml</filename> XML configuration file by
-            prefacing the ZooKeeper option name with
-            <varname>hbase.zookeeper.property</varname>. For example, the
-            <varname>clientPort</varname> setting in ZooKeeper can be changed
-            by setting the
-            <varname>hbase.zookeeper.property.clientPort</varname> property.
-            For all default values used by HBase, including ZooKeeper
-            configuration, see <xref linkend="hbase_default_configurations" />. Look for the
-            <varname>hbase.zookeeper.property</varname> prefix <footnote>
-                <para>For the full list of ZooKeeper configurations, see
-                ZooKeeper's <filename>zoo.cfg</filename>. HBase does not ship
-                with a <filename>zoo.cfg</filename> so you will need to browse
-                the <filename>conf</filename> directory in an appropriate
-                ZooKeeper download.</para>
-              </footnote></para>
-
-            <para>You must at least list the ensemble servers in
-            <filename>hbase-site.xml</filename> using the
-            <varname>hbase.zookeeper.quorum</varname> property. This property
-            defaults to a single ensemble member at
-            <varname>localhost</varname> which is not suitable for a fully
-            distributed HBase. (It binds to the local machine only and remote
-            clients will not be able to connect). <note xml:id="how_many_zks">
-                <title>How many ZooKeepers should I run?</title>
-
-                <para>You can run a ZooKeeper ensemble that comprises 1 node
-                only but in production it is recommended that you run a
-                ZooKeeper ensemble of 3, 5 or 7 machines; the more members an
-                ensemble has, the more tolerant the ensemble is of host
-                failures. Also, run an odd number of machines. In ZooKeeper, 
-                an even number of peers is supported, but it is normally not used 
-                because an even sized ensemble requires, proportionally, more peers 
-                to form a quorum than an odd sized ensemble requires. For example, an 
-                ensemble with 4 peers requires 3 to form a quorum, while an ensemble with 
-                5 also requires 3 to form a quorum. Thus, an ensemble of 5 allows 2 peers to 
-                fail, and thus is more fault tolerant than the ensemble of 4, which allows 
-                only 1 down peer.                 
-                </para>
-                <para>Give each ZooKeeper server around 1GB of RAM, and if possible, its own
-                dedicated disk (A dedicated disk is the best thing you can do
-                to ensure a performant ZooKeeper ensemble). For very heavily
-                loaded clusters, run ZooKeeper servers on separate machines
-                from RegionServers (DataNodes and TaskTrackers).</para>
-              </note></para>
-
-            <para>For example, to have HBase manage a ZooKeeper quorum on
-            nodes <emphasis>rs{1,2,3,4,5}.example.com</emphasis>, bound to
-            port 2222 (the default is 2181) ensure
-            <varname>HBASE_MANAGE_ZK</varname> is commented out or set to
-            <varname>true</varname> in <filename>conf/hbase-env.sh</filename>
-            and then edit <filename>conf/hbase-site.xml</filename> and set
-            <varname>hbase.zookeeper.property.clientPort</varname> and
-            <varname>hbase.zookeeper.quorum</varname>. You should also set
-            <varname>hbase.zookeeper.property.dataDir</varname> to other than
-            the default as the default has ZooKeeper persist data under
-            <filename>/tmp</filename> which is often cleared on system
-            restart. In the example below we have ZooKeeper persist to
-            <filename>/user/local/zookeeper</filename>. <programlisting>
-  &lt;configuration&gt;
-    ...
-    &lt;property&gt;
-      &lt;name&gt;hbase.zookeeper.property.clientPort&lt;/name&gt;
-      &lt;value&gt;2222&lt;/value&gt;
-      &lt;description&gt;Property from ZooKeeper's config zoo.cfg.
-      The port at which the clients will connect.
-      &lt;/description&gt;
-    &lt;/property&gt;
-    &lt;property&gt;
-      &lt;name&gt;hbase.zookeeper.quorum&lt;/name&gt;
-      &lt;value&gt;rs1.example.com,rs2.example.com,rs3.example.com,rs4.example.com,rs5.example.com&lt;/value&gt;
-      &lt;description&gt;Comma separated list of servers in the ZooKeeper Quorum.
-      For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
-      By default this is set to localhost for local and pseudo-distributed modes
-      of operation. For a fully-distributed setup, this should be set to a full
-      list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
-      this is the list of servers which we will start/stop ZooKeeper on.
-      &lt;/description&gt;
-    &lt;/property&gt;
-    &lt;property&gt;
-      &lt;name&gt;hbase.zookeeper.property.dataDir&lt;/name&gt;
-      &lt;value&gt;/usr/local/zookeeper&lt;/value&gt;
-      &lt;description&gt;Property from ZooKeeper's config zoo.cfg.
-      The directory where the snapshot is stored.
-      &lt;/description&gt;
-    &lt;/property&gt;
-    ...
-  &lt;/configuration&gt;</programlisting></para>
-
-            <section>
-              <title>Using existing ZooKeeper ensemble</title>
-
-              <para>To point HBase at an existing ZooKeeper cluster, one that
-              is not managed by HBase, set <varname>HBASE_MANAGES_ZK</varname>
-              in <filename>conf/hbase-env.sh</filename> to false
-              <programlisting>
-  ...
-  # Tell HBase whether it should manage it's own instance of Zookeeper or not.
-  export HBASE_MANAGES_ZK=false</programlisting> Next set ensemble locations
-              and client port, if non-standard, in
-              <filename>hbase-site.xml</filename>, or add a suitably
-              configured <filename>zoo.cfg</filename> to HBase's
-              <filename>CLASSPATH</filename>. HBase will prefer the
-              configuration found in <filename>zoo.cfg</filename> over any
-              settings in <filename>hbase-site.xml</filename>.</para>
-
-              <para>When HBase manages ZooKeeper, it will start/stop the
-              ZooKeeper servers as a part of the regular start/stop scripts.
-              If you would like to run ZooKeeper yourself, independent of
-              HBase start/stop, you would do the following</para>
-
-              <programlisting>
-${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
-</programlisting>
-
-              <para>Note that you can use HBase in this manner to spin up a
-              ZooKeeper cluster, unrelated to HBase. Just make sure to set
-              <varname>HBASE_MANAGES_ZK</varname> to <varname>false</varname>
-              if you want it to stay up across HBase restarts so that when
-              HBase shuts down, it doesn't take ZooKeeper down with it.</para>
-
-              <para>For more information about running a distinct ZooKeeper
-              cluster, see the ZooKeeper <link
-              xlink:href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">Getting
-              Started Guide</link>.  Additionally, see the <link xlink:href="http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A7">ZooKeeper Wiki</link> or the 
-          <link xlink:href="http://zookeeper.apache.org/doc/r3.3.3/zookeeperAdmin.html#sc_zkMulitServerSetup">ZooKeeper documentation</link> 
-          for more information on ZooKeeper sizing.
-            </para>
-            </section>
-            
-
-            <section xml:id="zk.sasl.auth">
-              <title>SASL Authentication with ZooKeeper</title>
-              <para>Newer releases of HBase (&gt;= 0.92) will
-              support connecting to a ZooKeeper Quorum that supports
-              SASL authentication (which is available in Zookeeper
-              versions 3.4.0 or later).</para>
-              
-              <para>This describes how to set up HBase to mutually
-              authenticate with a ZooKeeper Quorum. ZooKeeper/HBase
-              mutual authentication (<link
-              xlink:href="https://issues.apache.org/jira/browse/HBASE-2418">HBASE-2418</link>)
-              is required as part of a complete secure HBase configuration
-              (<link
-              xlink:href="https://issues.apache.org/jira/browse/HBASE-3025">HBASE-3025</link>).
-
-              For simplicity of explication, this section ignores
-              additional configuration required (Secure HDFS and Coprocessor
-              configuration).  It's recommended to begin with an
-              HBase-managed Zookeeper configuration (as opposed to a
-              standalone Zookeeper quorum) for ease of learning.
-              </para>
-
-              <section><title>Operating System Prerequisites</title></section>
-
-              <para>
-                  You need to have a working Kerberos KDC setup. For
-                  each <code>$HOST</code> that will run a ZooKeeper
-                  server, you should have a principle
-                  <code>zookeeper/$HOST</code>.  For each such host,
-                  add a service key (using the <code>kadmin</code> or
-                  <code>kadmin.local</code> tool's <code>ktadd</code>
-                  command) for <code>zookeeper/$HOST</code> and copy
-                  this file to <code>$HOST</code>, and make it
-                  readable only to the user that will run zookeeper on
-                  <code>$HOST</code>. Note the location of this file,
-                  which we will use below as
-                  <filename>$PATH_TO_ZOOKEEPER_KEYTAB</filename>.
-              </para>
-
-              <para>
-                Similarly, for each <code>$HOST</code> that will run
-                an HBase server (master or regionserver), you should
-                have a principle: <code>hbase/$HOST</code>. For each
-                host, add a keytab file called
-                <filename>hbase.keytab</filename> containing a service
-                key for <code>hbase/$HOST</code>, copy this file to
-                <code>$HOST</code>, and make it readable only to the
-                user that will run an HBase service on
-                <code>$HOST</code>. Note the location of this file,
-                which we will use below as
-                <filename>$PATH_TO_HBASE_KEYTAB</filename>.
-              </para>
-
-              <para>
-                Each user who will be an HBase client should also be
-                given a Kerberos principal. This principal should
-                usually have a password assigned to it (as opposed to,
-                as with the HBase servers, a keytab file) which only
-                this user knows. The client's principal's
-                <code>maxrenewlife</code> should be set so that it can
-                be renewed enough so that the user can complete their
-                HBase client processes. For example, if a user runs a
-                long-running HBase client process that takes at most 3
-                days, we might create this user's principal within
-                <code>kadmin</code> with: <code>addprinc -maxrenewlife
-                3days</code>. The Zookeeper client and server
-                libraries manage their own ticket refreshment by
-                running threads that wake up periodically to do the
-                refreshment.
-              </para>
-
-                <para>On each host that will run an HBase client
-                (e.g. <code>hbase shell</code>), add the following
-                file to the HBase home directory's <filename>conf</filename>
-                directory:</para>
-
-                <programlisting>
-                  Client {
-                    com.sun.security.auth.module.Krb5LoginModule required
-                    useKeyTab=false
-                    useTicketCache=true;
-                  };
-                </programlisting>
-
-                <para>We'll refer to this JAAS configuration file as
-                <filename>$CLIENT_CONF</filename> below.</para>
-
-              <section>
-                <title>HBase-managed Zookeeper Configuration</title>
-
-                <para>On each node that will run a zookeeper, a
-                master, or a regionserver, create a <link
-                xlink:href="http://docs.oracle.com/javase/1.4.2/docs/guide/security/jgss/tutorials/LoginConfigFile.html">JAAS</link>
-                configuration file in the conf directory of the node's
-                <filename>HBASE_HOME</filename> directory that looks like the
-                following:</para>
-
-                <programlisting>
-                  Server {
-                    com.sun.security.auth.module.Krb5LoginModule required
-                    useKeyTab=true
-                    keyTab="$PATH_TO_ZOOKEEPER_KEYTAB"
-                    storeKey=true
-                    useTicketCache=false
-                    principal="zookeeper/$HOST";
-                  };
-                  Client {
-                    com.sun.security.auth.module.Krb5LoginModule required
-                    useKeyTab=true
-                    useTicketCache=false
-                    keyTab="$PATH_TO_HBASE_KEYTAB"
-                    principal="hbase/$HOST";
-                  };
-                </programlisting>
-                
-                where the <filename>$PATH_TO_HBASE_KEYTAB</filename> and
-                <filename>$PATH_TO_ZOOKEEPER_KEYTAB</filename> files are what
-                you created above, and <code>$HOST</code> is the hostname for that
-                node.
-
-                <para>The <code>Server</code> section will be used by
-                the Zookeeper quorum server, while the
-                <code>Client</code> section will be used by the HBase
-                master and regionservers. The path to this file should
-                be substituted for the text <filename>$HBASE_SERVER_CONF</filename>
-                in the <filename>hbase-env.sh</filename>
-                listing below.</para>
-
-                <para>
-                  The path to this file should be substituted for the
-                  text <filename>$CLIENT_CONF</filename> in the
-                  <filename>hbase-env.sh</filename> listing below.
-                </para>
-
-                <para>Modify your <filename>hbase-env.sh</filename> to include the
-                following:</para>
-
-                <programlisting>
-                  export HBASE_OPTS="-Djava.security.auth.login.config=$CLIENT_CONF"
-                  export HBASE_MANAGES_ZK=true
-                  export HBASE_ZOOKEEPER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
-                  export HBASE_MASTER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
-                  export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
-                </programlisting>
-
-                where <filename>$HBASE_SERVER_CONF</filename> and
-                <filename>$CLIENT_CONF</filename> are the full paths to the
-                JAAS configuration files created above.
-
-                <para>Modify your <filename>hbase-site.xml</filename> on each node
-                that will run zookeeper, master or regionserver to contain:</para>
-
-                <programlisting><![CDATA[
-                  <configuration>
-                    <property>
-                      <name>hbase.zookeeper.quorum</name>
-                      <value>$ZK_NODES</value>
-                    </property>
-                    <property>
-                      <name>hbase.cluster.distributed</name>
-                      <value>true</value>
-                    </property>
-                    <property>
-                      <name>hbase.zookeeper.property.authProvider.1</name>
-                      <value>org.apache.zookeeper.server.auth.SASLAuthenticationProvider</value>
-                    </property>
-                    <property>
-                      <name>hbase.zookeeper.property.kerberos.removeHostFromPrincipal</name>
-                      <value>true</value>
-                    </property>
-                    <property>
-                      <name>hbase.zookeeper.property.kerberos.removeRealmFromPrincipal</name>
-                      <value>true</value>
-                    </property>
-                  </configuration>
-                  ]]></programlisting>
-
-                <para>where <code>$ZK_NODES</code> is the
-                comma-separated list of hostnames of the Zookeeper
-                Quorum hosts.</para>
-
-                <para>Start your hbase cluster by running one or more
-                of the following set of commands on the appropriate
-                hosts:
-                </para>
-
-                <programlisting>
-                  bin/hbase zookeeper start
-                  bin/hbase master start
-                  bin/hbase regionserver start
-                </programlisting>
-
-              </section>
-
-              <section><title>External Zookeeper Configuration</title>
-                <para>Add a JAAS configuration file that looks like:
-
-                <programlisting>
-                  Client {
-                    com.sun.security.auth.module.Krb5LoginModule required
-                    useKeyTab=true
-                    useTicketCache=false
-                    keyTab="$PATH_TO_HBASE_KEYTAB"
-                    principal="hbase/$HOST";
-                  };
-                </programlisting>
-
-                where the <filename>$PATH_TO_HBASE_KEYTAB</filename> is the keytab 
-                created above for HBase services to run on this host, and <code>$HOST</code> is the
-                hostname for that node. Put this in the HBase home's
-                configuration directory. We'll refer to this file's
-                full pathname as <filename>$HBASE_SERVER_CONF</filename> below.</para>
-
-                <para>Modify your hbase-env.sh to include the following:</para>
-
-                <programlisting>
-                  export HBASE_OPTS="-Djava.security.auth.login.config=$CLIENT_CONF"
-                  export HBASE_MANAGES_ZK=false
-                  export HBASE_MASTER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
-                  export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
-                </programlisting>
-
-
-                <para>Modify your <filename>hbase-site.xml</filename> on each node
-                that will run a master or regionserver to contain:</para>
-
-                <programlisting><![CDATA[
-                  <configuration>
-                    <property>
-                      <name>hbase.zookeeper.quorum</name>
-                      <value>$ZK_NODES</value>
-                    </property>
-                    <property>
-                      <name>hbase.cluster.distributed</name>
-                      <value>true</value>
-                    </property>
-                  </configuration>
-                  ]]>
-                </programlisting>
-
-                <para>where <code>$ZK_NODES</code> is the
-                comma-separated list of hostnames of the Zookeeper
-                Quorum hosts.</para>
-
-                <para>
-                  Add a <filename>zoo.cfg</filename> for each Zookeeper Quorum host containing:
-                  <programlisting>
-                      authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
-                      kerberos.removeHostFromPrincipal=true
-                      kerberos.removeRealmFromPrincipal=true
-                  </programlisting>
-
-                  Also on each of these hosts, create a JAAS configuration file containing:
-
-                  <programlisting>
-                  Server {
-                    com.sun.security.auth.module.Krb5LoginModule required
-                    useKeyTab=true
-                    keyTab="$PATH_TO_ZOOKEEPER_KEYTAB"
-                    storeKey=true
-                    useTicketCache=false
-                    principal="zookeeper/$HOST";
-                  };
-                  </programlisting>
-
-                  where <code>$HOST</code> is the hostname of each
-                  Quorum host. We will refer to the full pathname of
-                  this file as <filename>$ZK_SERVER_CONF</filename> below.
-
-                </para>
-
-                <para>
-                  Start your Zookeepers on each Zookeeper Quorum host with:
-                  
-                  <programlisting>
-                    SERVER_JVMFLAGS="-Djava.security.auth.login.config=$ZK_SERVER_CONF" bin/zkServer start
-                  </programlisting>
-
-                </para>
-
-                <para>
-                  Start your HBase cluster by running one or more of the following set of commands on the appropriate nodes:
-                </para>
-
-                <programlisting>
-                  bin/hbase master start
-                  bin/hbase regionserver start
-                </programlisting>
-
-
-              </section>
-
-              <section>
-                <title>Zookeeper Server Authentication Log Output</title>
-                <para>If the configuration above is successful,
-                you should see something similar to the following in
-                your Zookeeper server logs:
-                <programlisting>
-11/12/05 22:43:39 INFO zookeeper.Login: successfully logged in.
-11/12/05 22:43:39 INFO server.NIOServerCnxnFactory: binding to port 0.0.0.0/0.0.0.0:2181
-11/12/05 22:43:39 INFO zookeeper.Login: TGT refresh thread started.
-11/12/05 22:43:39 INFO zookeeper.Login: TGT valid starting at:        Mon Dec 05 22:43:39 UTC 2011
-11/12/05 22:43:39 INFO zookeeper.Login: TGT expires:                  Tue Dec 06 22:43:39 UTC 2011
-11/12/05 22:43:39 INFO zookeeper.Login: TGT refresh sleeping until: Tue Dec 06 18:36:42 UTC 2011
-..
-11/12/05 22:43:59 INFO auth.SaslServerCallbackHandler: 
-  Successfully authenticated client: authenticationID=hbase/ip-10-166-175-249.us-west-1.compute.internal@HADOOP.LOCALDOMAIN; 
-  authorizationID=hbase/ip-10-166-175-249.us-west-1.compute.internal@HADOOP.LOCALDOMAIN.
-11/12/05 22:43:59 INFO auth.SaslServerCallbackHandler: Setting authorizedID: hbase
-11/12/05 22:43:59 INFO server.ZooKeeperServer: adding SASL authorization for authorizationID: hbase
-                </programlisting>
-                  
-                </para>
-
-              </section>
-
-              <section>
-                <title>Zookeeper Client Authentication Log Output</title>
-                <para>On the Zookeeper client side (HBase master or regionserver),
-                you should see something similar to the following:
-
-                <programlisting>
-11/12/05 22:43:59 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=ip-10-166-175-249.us-west-1.compute.internal:2181 sessionTimeout=180000 watcher=master:60000
-11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Opening socket connection to server /10.166.175.249:2181
-11/12/05 22:43:59 INFO zookeeper.RecoverableZooKeeper: The identifier of this process is 14851@ip-10-166-175-249
-11/12/05 22:43:59 INFO zookeeper.Login: successfully logged in.
-11/12/05 22:43:59 INFO client.ZooKeeperSaslClient: Client will use GSSAPI as SASL mechanism.
-11/12/05 22:43:59 INFO zookeeper.Login: TGT refresh thread started.
-11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Socket connection established to ip-10-166-175-249.us-west-1.compute.internal/10.166.175.249:2181, initiating session
-11/12/05 22:43:59 INFO zookeeper.Login: TGT valid starting at:        Mon Dec 05 22:43:59 UTC 2011
-11/12/05 22:43:59 INFO zookeeper.Login: TGT expires:                  Tue Dec 06 22:43:59 UTC 2011
-11/12/05 22:43:59 INFO zookeeper.Login: TGT refresh sleeping until: Tue Dec 06 18:30:37 UTC 2011
-11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Session establishment complete on server ip-10-166-175-249.us-west-1.compute.internal/10.166.175.249:2181, sessionid = 0x134106594320000, negotiated timeout = 180000
-                </programlisting>
-                </para>
-              </section>
-
-              <section>
-                <title>Configuration from Scratch</title>
-
-                This has been tested on the current standard Amazon
-                Linux AMI.  First setup KDC and principals as
-                described above. Next checkout code and run a sanity
-                check.
-                
-                <programlisting>
-                git clone git://git.apache.org/hbase.git
-                cd hbase
-                mvn -Psecurity,localTests clean test -Dtest=TestZooKeeperACL
-                </programlisting>
-
-                Then configure HBase as described above.
-                Manually edit target/cached_classpath.txt (see below)..
-
-                <programlisting>
-                bin/hbase zookeeper &amp;
-                bin/hbase master &amp;
-                bin/hbase regionserver &amp;
-                </programlisting>
-              </section>
-
-
-              <section>
-                <title>Future improvements</title>
-
-                <section><title>Fix target/cached_classpath.txt</title>
-                <para>
-                You must override the standard hadoop-core jar file from the
-                <code>target/cached_classpath.txt</code>
-                file with the version containing the HADOOP-7070 fix. You can use the following script to do this:
-
-                <programlisting>
-                  echo `find ~/.m2 -name "*hadoop-core*7070*SNAPSHOT.jar"` ':' `cat target/cached_classpath.txt` | sed 's/ //g' > target/tmp.txt 
-                  mv target/tmp.txt target/cached_classpath.txt
-                </programlisting>
-
-                </para>
-
-                </section>
-
-                <section>
-                  <title>Set JAAS configuration
-                  programmatically</title> 
-
-
-                  This would avoid the need for a separate Hadoop jar
-                  that fixes <link xlink:href="https://issues.apache.org/jira/browse/HADOOP-7070">HADOOP-7070</link>.
-                </section>
-                
-                <section>
-                  <title>Elimination of 
-                  <code>kerberos.removeHostFromPrincipal</code> and 
-                  <code>kerberos.removeRealmFromPrincipal</code></title>
-                </section>
-                
-              </section>
-
-
-            </section> <!-- SASL Authentication with ZooKeeper -->
-
-
 
 
 
-     </section>     <!--  zookeeper -->        
-    
-    
-    <section xml:id="config.files">    
+    <section xml:id="config.files">
          <title>Configuration Files</title>
-         
+
     <section xml:id="hbase.site">
     <title><filename>hbase-site.xml</filename> and <filename>hbase-default.xml</filename></title>
     <para>Just as in Hadoop where you add site-specific HDFS configuration
@@ -1197,7 +669,7 @@ ${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
     The generated file is a docbook section with a glossary
     in it-->
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude"
-      href="../../target/site/hbase-default.xml" />
+        href="../../hbase-common/src/main/resources/hbase-default.xml" />
     </section>
 
       <section xml:id="hbase.env.sh">
@@ -1242,8 +714,17 @@ ${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
             used by tests).
       </para>
       <para>
-          Minimally, a client of HBase needs the hbase, hadoop, log4j, commons-logging, commons-lang,
-          and ZooKeeper jars in its <varname>CLASSPATH</varname> connecting to a cluster.
+          Minimally, a client of HBase needs several libraries in its <varname>CLASSPATH</varname> when connecting to a cluster, including:
+          <programlisting>
+commons-configuration (commons-configuration-1.6.jar)
+commons-lang (commons-lang-2.5.jar)
+commons-logging (commons-logging-1.1.1.jar)
+hadoop-core (hadoop-core-1.0.0.jar)
+hbase (hbase-0.92.0.jar)
+log4j (log4j-1.2.16.jar)
+slf4j-api (slf4j-api-1.5.8.jar)
+slf4j-log4j (slf4j-log4j12-1.5.8.jar)
+zookeeper (zookeeper-3.4.2.jar)</programlisting>
       </para>
         <para>
           An example basic <filename>hbase-site.xml</filename> for client only
@@ -1261,7 +742,7 @@ ${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
 </configuration>
 ]]></programlisting>
         </para>
-        
+
         <section xml:id="java.client.config">
         <title>Java client configuration</title>
         <para>The configuration used by a Java client is kept
@@ -1270,15 +751,15 @@ ${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
         on invocation, will read in the content of the first <filename>hbase-site.xml</filename> found on
         the client's <varname>CLASSPATH</varname>, if one is present
         (Invocation will also factor in any <filename>hbase-default.xml</filename> found;
-        an hbase-default.xml ships inside the <filename>hbase.X.X.X.jar</filename>). 
+        an hbase-default.xml ships inside the <filename>hbase.X.X.X.jar</filename>).
         It is also possible to specify configuration directly without having to read from a
         <filename>hbase-site.xml</filename>.  For example, to set the ZooKeeper
         ensemble for the cluster programmatically do as follows:
         <programlisting>Configuration config = HBaseConfiguration.create();
-config.set("hbase.zookeeper.quorum", "localhost");  // Here we are running zookeeper locally</programlisting>    
+config.set("hbase.zookeeper.quorum", "localhost");  // Here we are running zookeeper locally</programlisting>
         If multiple ZooKeeper instances make up your ZooKeeper ensemble,
         they may be specified in a comma-separated list (just as in the <filename>hbase-site.xml</filename> file).
-        This populated <classname>Configuration</classname> instance can then be passed to an 
+        This populated <classname>Configuration</classname> instance can then be passed to an
         <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html">HTable</link>,
         and so on.
         </para>
@@ -1286,7 +767,7 @@ config.set("hbase.zookeeper.quorum", "localhost");  // Here we are running zooke
         </section>
 
       </section>  <!--  config files -->
-      
+
       <section xml:id="example_config">
       <title>Example Configurations</title>
 
@@ -1378,7 +859,7 @@ config.set("hbase.zookeeper.quorum", "localhost");  // Here we are running zooke
           1G.</para>
 
           <programlisting>
-    
+
 $ git diff hbase-env.sh
 diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
 index e70ebc6..96f8c27 100644
@@ -1386,11 +867,11 @@ index e70ebc6..96f8c27 100644
 +++ b/conf/hbase-env.sh
 @@ -31,7 +31,7 @@ export JAVA_HOME=/usr/lib//jvm/java-6-sun/
  # export HBASE_CLASSPATH=
- 
+
  # The maximum amount of heap to use, in MB. Default is 1000.
 -# export HBASE_HEAPSIZE=1000
 +export HBASE_HEAPSIZE=4096
- 
+
  # Extra Java runtime options.
  # Below are what we set by default.  May only work with SUN JVM.
 
@@ -1402,8 +883,8 @@ index e70ebc6..96f8c27 100644
         </section>
       </section>
      </section>       <!-- example config -->
-      
-      
+
+
       <section xml:id="important_configurations">
       <title>The Important Configurations</title>
       <para>Below we list what the <emphasis>important</emphasis>
@@ -1415,9 +896,23 @@ index e70ebc6..96f8c27 100644
       <section xml:id="required_configuration"><title>Required Configurations</title>
           <para>Review the <xref linkend="os" /> and <xref linkend="hadoop" /> sections.
       </para>
+      <section xml:id="big.cluster.config"><title>Big Cluster Configurations</title>
+        <para>If a cluster with a lot of regions, it is possible if an eager beaver
+            regionserver checks in soon after master start while all the rest in the
+            cluster are laggardly, this first server to checkin will be assigned all
+            regions.  If lots of regions, this first server could buckle under the
+            load.  To prevent the above scenario happening up the
+            <varname>hbase.master.wait.on.regionservers.mintostart</varname> from its
+            default value of 1.  See
+            <link xlink:href="https://issues.apache.org/jira/browse/HBASE-6389">HBASE-6389 Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments</link>
+            for more detail.
+        </para>
+      </section>
       </section>
 
       <section xml:id="recommended_configurations"><title>Recommended Configurations</title>
+          <section xml:id="recommended_configurations.zk">
+              <title>ZooKeeper Configuration</title>
           <section xml:id="zookeeper.session.timeout"><title><varname>zookeeper.session.timeout</varname></title>
           <para>The default timeout is three minutes (specified in milliseconds). This means
               that if a server crashes, it will be three minutes before the Master notices
@@ -1427,7 +922,7 @@ index e70ebc6..96f8c27 100644
               configuration under control otherwise, a long garbage collection that lasts
               beyond the ZooKeeper session timeout will take out
               your RegionServer (You might be fine with this -- you probably want recovery to start
-          on the server if a RegionServer has been in GC for a long period of time).</para> 
+          on the server if a RegionServer has been in GC for a long period of time).</para>
 
       <para>To change this configuration, edit <filename>hbase-site.xml</filename>,
           copy the changed file around the cluster and restart.</para>
@@ -1443,6 +938,18 @@ index e70ebc6..96f8c27 100644
           <section xml:id="zookeeper.instances"><title>Number of ZooKeeper Instances</title>
           <para>See <xref linkend="zookeeper"/>.
           </para>
+      </section>
+      </section>
+      <section xml:id="recommended.configurations.hdfs">
+          <title>HDFS Configurations</title>
+          <section xml:id="dfs.datanode.failed.volumes.tolerated">
+              <title>dfs.datanode.failed.volumes.tolerated</title>
+              <para>This is the "...number of volumes that are allowed to fail before a datanode stops offering service. By default
+                  any volume failure will cause a datanode to shutdown" from the <filename>hdfs-default.xml</filename>
+                  description.  If you have > three or four disks, you might want to set this to 1 or if you have many disks,
+                  two or more.
+              </para>
+          </section>
       </section>
           <section xml:id="hbase.regionserver.handler.count"><title><varname>hbase.regionserver.handler.count</varname></title>
           <para>
@@ -1503,7 +1010,7 @@ index e70ebc6..96f8c27 100644
       cluster (You can always later manually split the big Regions should one prove
       hot and you want to spread the request load over the cluster).  A lower number of regions is
        preferred, generally in the range of 20 to low-hundreds
-       per RegionServer.  Adjust the regionsize as appropriate to achieve this number. 
+       per RegionServer.  Adjust the regionsize as appropriate to achieve this number.
        </para>
        <para>For the 0.90.x codebase, the upper-bound of regionsize is about 4Gb, with a default of 256Mb.
        For 0.92.x codebase, due to the HFile v2 change much larger regionsizes can be supported (e.g., 20Gb).
@@ -1511,10 +1018,58 @@ index e70ebc6..96f8c27 100644
        <para>You may need to experiment with this setting based on your hardware configuration and application needs.
        </para>
        <para>Adjust <code>hbase.hregion.max.filesize</code> in your <filename>hbase-site.xml</filename>.
-       RegionSize can also be set on a per-table basis via 
+       RegionSize can also be set on a per-table basis via
        <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html">HTableDescriptor</link>.
       </para>
-      
+      <section xml:id="too_many_regions">
+          <title>How many regions per RegionServer?</title>
+          <para>
+              Typically you want to keep your region count low on HBase for numerous reasons.
+              Usually right around 100 regions per RegionServer has yielded the best results.
+              Here are some of the reasons below for keeping region count low:
+              <unorderedlist>
+                  <listitem><para>
+                          MSLAB requires 2mb per memstore (that's 2mb per family per region).
+                          1000 regions that have 2 families each is 3.9GB of heap used, and it's not even storing data yet. NB: the 2MB value is configurable.
+                  </para></listitem>
+                  <listitem><para>If you fill all the regions at somewhat the same rate, the global memory usage makes it that it forces tiny
+                          flushes when you have too many regions which in turn generates compactions.
+                          Rewriting the same data tens of times is the last thing you want.
+                          An example is filling 1000 regions (with one family) equally and let's consider a lower bound for global memstore
+                          usage of 5GB (the region server would have a big heap).
+                          Once it reaches 5GB it will force flush the biggest region,
+                          at that point they should almost all have about 5MB of data so
+                          it would flush that amount. 5MB inserted later, it would flush another
+                          region that will now have a bit over 5MB of data, and so on.
+                          A basic formula for the amount of regions to have per region server would
+                          look like this:
+                          Heap * upper global memstore limit = amount of heap devoted to memstore
+                          then the amount of heap devoted to memstore / (Number of regions per RS * CFs).
+                          This will give you the rough memstore size if everything is being written to.
+                          A more accurate formula is
+                          Heap * upper global memstore limit = amount of heap devoted to memstore then the
+                          amount of heap devoted to memstore / (Number of actively written regions per RS * CFs).
+                          This can allot you a higher region count from the write perspective if you know how many
+                          regions you will be writing to at one time.
+                  </para></listitem>
+                  <listitem><para>The master as is is allergic to tons of regions, and will
+                          take a lot of time assigning them and moving them around in batches.
+                          The reason is that it's heavy on ZK usage, and it's not very async
+                          at the moment (could really be improved -- and has been imporoved a bunch
+                          in 0.96 hbase).
+                  </para></listitem>
+                  <listitem><para>
+                          In older versions of HBase (pre-v2 hfile, 0.90 and previous), tons of regions
+                          on a few RS can cause the store file index to rise raising heap usage and can
+                          create memory pressure or OOME on the RSs
+                  </para></listitem>
+          </unorderedlist>
+      </para>
+      <para>Another issue is the effect of the number of regions on mapreduce jobs.
+          Keeping 5 regions per RS would be too low for a job, whereas 1000 will generate too many maps.
+      </para>
+      </section>
+
       </section>
       <section xml:id="disable.splitting">
       <title>Managed Splitting</title>
@@ -1567,23 +1122,30 @@ of all regions.
 </para>
       </section>
       <section xml:id="managed.compactions"><title>Managed Compactions</title>
-      <para>A common administrative technique is to manage major compactions manually, rather than letting 
+      <para>A common administrative technique is to manage major compactions manually, rather than letting
       HBase do it.  By default, <varname>HConstants.MAJOR_COMPACTION_PERIOD</varname> is one day and major compactions
       may kick in when you least desire it - especially on a busy system.  To turn off automatic major compactions set
-      the value to <varname>0</varname>. 
+      the value to <varname>0</varname>.
       </para>
       <para>It is important to stress that major compactions are absolutely necessary for StoreFile cleanup, the only variant is when
-      they occur.  They can be administered through the HBase shell, or via 
+      they occur.  They can be administered through the HBase shell, or via
       <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HBaseAdmin.html#majorCompact%28java.lang.String%29">HBaseAdmin</link>.
       </para>
       <para>For more information about compactions and the compaction file selection process, see <xref linkend="compaction"/></para>
       </section>
-      
+
+      <section xml:id="spec.ex"><title>Speculative Execution</title>
+        <para>Speculative Execution of MapReduce tasks is on by default, and for HBase clusters it is generally advised to turn off
+        Speculative Execution at a system-level unless you need it for a specific case, where it can be configured per-job.
+        Set the properties <varname>mapred.map.tasks.speculative.execution</varname> and
+        <varname>mapred.reduce.tasks.speculative.execution</varname> to false.
+        </para>
+      </section>
       </section>
 
       <section xml:id="other_configuration"><title>Other Configurations</title>
          <section xml:id="balancer_config"><title>Balancer</title>
-           <para>The balancer is periodic operation run on the master to redistribute regions on the cluster.  It is configured via
+           <para>The balancer is a periodic operation which is run on the master to redistribute regions on the cluster.  It is configured via
            <varname>hbase.balancer.period</varname> and defaults to 300000 (5 minutes). </para>
            <para>See <xref linkend="master.processes.loadbalancer" /> for more information on the LoadBalancer.
            </para>
@@ -1596,38 +1158,18 @@ of all regions.
            on the size you need by surveying regionserver UIs; you'll see index block size accounted near the
            top of the webpage).</para>
          </section>
-      </section>
-      
-      </section> <!--  important config -->
-
-	  <section xml:id="config.bloom">
-	    <title>Bloom Filter Configuration</title>
-        <section>
-        <title><varname>io.hfile.bloom.enabled</varname> global kill
-        switch</title>
-
-        <para><code>io.hfile.bloom.enabled</code> in
-        <classname>Configuration</classname> serves as the kill switch in case
-        something goes wrong. Default = <varname>true</varname>.</para>
-        </section>
-
-        <section>
-        <title><varname>io.hfile.bloom.error.rate</varname></title>
+    <section xml:id="nagles">
+      <title><link xlink:href="http://en.wikipedia.org/wiki/Nagle's_algorithm">Nagle's</link> or the small package problem</title>
+      <para>If a big 40ms or so occasional delay is seen in operations against HBase,
+      try the Nagles' setting.  For example, see the user mailing list thread,
+      <link xlink:href="http://search-hadoop.com/m/pduLg2fydtE/Inconsistent+scan+performance+with+caching+set+&amp;subj=Re+Inconsistent+scan+performance+with+caching+set+to+1">Inconsistent scan performance with caching set to 1</link>
+      and the issue cited therein where setting notcpdelay improved scan speeds.  You might also
+      see the graphs on the tail of <link xlink:href="https://issues.apache.org/jira/browse/HBASE-7008">HBASE-7008 Set scanner caching to a better default</link>
+      where our Lars Hofhansl tries various data sizes w/ Nagle's on and off measuring the effect.</para>
+    </section>
 
-        <para><varname>io.hfile.bloom.error.rate</varname> = average false
-        positive rate. Default = 1%. Decrease rate by ½ (e.g. to .5%) == +1
-        bit per bloom entry.</para>
-        </section>
+      </section>
 
-        <section>
-        <title><varname>io.hfile.bloom.max.fold</varname></title>
+      </section> <!--  important config -->
 
-        <para><varname>io.hfile.bloom.max.fold</varname> = guaranteed minimum
-        fold rate. Most people should leave this alone. Default = 7, or can
-        collapse to at least 1/128th of original size. See the
-        <emphasis>Development Process</emphasis> section of the document <link
-        xlink:href="https://issues.apache.org/jira/secure/attachment/12444007/Bloom_Filters_in_HBase.pdf">BloomFilters
-        in HBase</link> for more on what this option means.</para>
-        </section>
-      </section>
   </chapter>
diff --git a/src/docbkx/customization.xsl b/src/docbkx/customization.xsl
index d80a2b5abd61..a5065a48ff93 100644
--- a/src/docbkx/customization.xsl
+++ b/src/docbkx/customization.xsl
@@ -20,15 +20,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-This stylesheet is used making an html version of hbase-default.xml.
 -->
   <xsl:import href="urn:docbkx:stylesheet"/>
+  <xsl:output method="html" encoding="UTF-8" indent="no"/>
 
   <xsl:template name="user.header.content">
   </xsl:template>
 
   <xsl:template name="user.footer.content">
+<div id="disqus_thread"></div>
+<script type="text/javascript">
+    var disqus_shortname = 'hbase'; // required: replace example with your forum shortname
+    var disqus_url = 'http://hbase.apache.org/book';
+    var disqus_identifier = '<xsl:value-of select="@xml:id" />';
+
+    /* * * DON'T EDIT BELOW THIS LINE * * */
+    (function() {
+        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+        dsq.src = 'http://' + disqus_shortname + '.disqus.com/embed.js';
+        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+    })();
+</script>
+<noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
+<a href="http://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
   </xsl:template>
 
 </xsl:stylesheet>
diff --git a/src/docbkx/developer.xml b/src/docbkx/developer.xml
index 0c139b843ed0..d23e276518af 100644
--- a/src/docbkx/developer.xml
+++ b/src/docbkx/developer.xml
@@ -26,126 +26,266 @@
  * limitations under the License.
  */
 -->
-    <title>Building and Developing HBase</title>
-    <para>This chapter will be of interest only to those building and developing HBase (i.e., as opposed to
+    <title>Building and Developing Apache HBase (TM)</title>
+    <para>This chapter will be of interest only to those building and developing Apache HBase (TM) (i.e., as opposed to
     just downloading the latest distribution).
     </para>
     <section xml:id="repos">
-      <title>HBase Repositories</title>
+      <title>Apache HBase Repositories</title>
+      <para>There are two different repositories for Apache HBase: Subversion (SVN) and Git. The former is the system of record for committers, but the latter is easier to work with to build and contribute. SVN updates get automatically propagated to the Git repo.</para>
       <section xml:id="svn">
         <title>SVN</title>
         <programlisting>
-svn co http://svn.apache.org/repos/asf/hbase/trunk hbase-core-trunk 
+svn co http://svn.apache.org/repos/asf/hbase/trunk hbase-core-trunk
         </programlisting>
-      </section>         
+      </section>
       <section xml:id="git">
         <title>Git</title>
         <programlisting>
 git clone git://git.apache.org/hbase.git
         </programlisting>
-      </section>         
-    </section>    
-             
-    <section xml:id="ides"> 
+      </section>
+    </section>
+
+    <section xml:id="ides">
         <title>IDEs</title>
         <section xml:id="eclipse">
           <title>Eclipse</title>
             <section xml:id="eclipse.code.formatting">
             <title>Code Formatting</title>
-            <para>See <link xlink:href="https://issues.apache.org/jira/browse/HBASE-3678">HBASE-3678 Add Eclipse-based Apache Formatter to HBase Wiki</link>
-              for an Eclipse formatter to help ensure your code conforms to HBase'y coding convention.
-            The issue includes instructions for loading the attached formatter.</para>
+            <para>Under the <filename>dev-support</filename> folder, you will find <filename>hbase_eclipse_formatter.xml</filename>.
+            We encourage you to have this formatter in place in eclipse when editing HBase code.  To load it into eclipse:
+<orderedlist>
+<listitem><para>Go to Eclipse->Preferences...</para></listitem>
+<listitem><para>In Preferences, Go to Java-&gt;Code Style-&gt;Formatter</para></listitem>
+<listitem><para>Import... <filename>hbase_eclipse_formatter.xml</filename></para></listitem>
+<listitem><para>Click Apply</para></listitem>
+<listitem><para>Still in Preferences, Go to Java->Editor->Save Actions</para></listitem>
+<listitem><para>Check the following:
+<orderedlist>
+<listitem><para>Perform the selected actions on save</para></listitem>
+<listitem><para>Format source code</para></listitem>
+<listitem><para>Format edited lines</para></listitem>
+</orderedlist>
+</para></listitem>
+<listitem><para>Click Apply</para></listitem>
+</orderedlist>
+</para>
+            <para>In addition to the automatic formatting, make sure you follow the style guidelines explained in <xref linkend="common.patch.feedback"/></para>
             <para>Also, no @author tags - that's a rule.  Quality Javadoc comments are appreciated.  And include the Apache license.</para>
-            </section>         
+            </section>
             <section xml:id="eclipse.svn">
             <title>Subversive Plugin</title>
             <para>Download and install the Subversive plugin.</para>
             <para>Set up an SVN Repository target from <xref linkend="svn"/>, then check out the code.</para>
-            </section>         
+            </section>
+            <section xml:id="eclipse.git.plugin">
+            <title>Git Plugin</title>
+            <para>If you cloned the project via git, download and install the Git plugin (EGit). Attach to your local git repo (via the Git Repositories window) and you'll be able to see file revision history, generate patches, etc.</para>
+            </section>
             <section xml:id="eclipse.maven.setup">
-            <title>HBase Project Setup</title>
-            To set up your Eclipse environment for HBase, close Eclipse and execute...
-            <programlisting>
-mvn eclipse:eclipse
-            </programlisting>
-            ... from your local HBase project directory in your workspace to generate some new <filename>.project</filename> 
-            and <filename>.classpath</filename>files.  Then reopen Eclipse.
-            </section>         
-            <section xml:id="eclipse.maven.plugin">
-            <title>Maven Plugin</title>
-            <para>Download and install the Maven plugin.  For example, Help -&gt; Install New Software -&gt; (search for Maven Plugin)</para>
-            </section>         
+            <title>HBase Project Setup in Eclipse</title>
+            <para>The easiest way is to use the m2eclipse plugin for Eclipse. Eclipse Indigo or newer has m2eclipse built-in, or it can be found here:http://www.eclipse.org/m2e/. M2Eclipse provides Maven integration for Eclipse - it even lets you use the direct Maven commands from within Eclipse to compile and test your project.</para>
+            <para>To import the project, you merely need to go to File->Import...Maven->Existing Maven Projects and then point Eclipse at the HBase root directory; m2eclipse will automatically find all the hbase modules for you.</para>
+            <para>If you install m2eclipse and import HBase in your workspace, you will have to fix your eclipse Build Path.
+            Remove <filename>target</filename> folder, add <filename>target/generated-jamon</filename>
+            and <filename>target/generated-sources/java</filename> folders. You may also remove from your Build Path
+            the exclusions on the <filename>src/main/resources</filename> and <filename>src/test/resources</filename>
+            to avoid error message in the console 'Failed to execute goal org.apache.maven.plugins:maven-antrun-plugin:1.6:run (default) on project hbase:
+            'An Ant BuildException has occured: Replace: source file .../target/classes/hbase-default.xml doesn't exist'. This will also
+            reduce the eclipse build cycles and make your life easier when developing.</para>
+            </section>
+            <section xml:id="eclipse.commandline">
+            <title>Import into eclipse with the command line</title>
+            <para>For those not inclined to use m2eclipse, you can generate the Eclipse files from the command line. First, run (you should only have to do this once):
+            <programlisting>mvn clean install -DskipTests</programlisting>
+            and then close Eclipse and execute...
+            <programlisting>mvn eclipse:eclipse</programlisting>
+            ... from your local HBase project directory in your workspace to generate some new <filename>.project</filename>
+            and <filename>.classpath</filename>files.  Then reopen Eclipse, or refresh your eclipse project (F5), and import
+            the .project file in the HBase directory to a workspace.
+            </para>
+            </section>
             <section xml:id="eclipse.maven.class">
             <title>Maven Classpath Variable</title>
-            <para>The <varname>M2_REPO</varname> classpath variable needs to be set up for the project.  This needs to be set to 
+            <para>The <varname>M2_REPO</varname> classpath variable needs to be set up for the project.  This needs to be set to
             your local Maven repository, which is usually <filename>~/.m2/repository</filename></para>
             If this classpath variable is not configured, you will see compile errors in Eclipse like this...
             <programlisting>
 Description	Resource	Path	Location	Type
-The project cannot be built until build path errors are resolved	hbase		Unknown	Java Problem 
+The project cannot be built until build path errors are resolved	hbase		Unknown	Java Problem
 Unbound classpath variable: 'M2_REPO/asm/asm/3.1/asm-3.1.jar' in project 'hbase'	hbase		Build path	Build Path Problem
-Unbound classpath variable: 'M2_REPO/com/github/stephenc/high-scale-lib/high-scale-lib/1.1.1/high-scale-lib-1.1.1.jar' in project 'hbase'	hbase		Build path	Build Path Problem 
+Unbound classpath variable: 'M2_REPO/com/github/stephenc/high-scale-lib/high-scale-lib/1.1.1/high-scale-lib-1.1.1.jar' in project 'hbase'	hbase		Build path	Build Path Problem
 Unbound classpath variable: 'M2_REPO/com/google/guava/guava/r09/guava-r09.jar' in project 'hbase'	hbase		Build path	Build Path Problem
 Unbound classpath variable: 'M2_REPO/com/google/protobuf/protobuf-java/2.3.0/protobuf-java-2.3.0.jar' in project 'hbase'	hbase		Build path	Build Path Problem Unbound classpath variable:
-            </programlisting>            
+            </programlisting>
             </section>
-            <section xml:id="eclipse.m2eclipse">
-            <title>Import via m2eclipse</title>
-            <para>If you install the m2eclipse and import the HBase pom.xml in your workspace, you will have to fix your eclipse Build Path.
-            Remove <filename>target</filename> folder, add <filename>target/generated-jamon</filename>
-            and <filename>target/generated-sources/java</filename> folders. You may also remove from your Build Path
-            the exclusions on the <filename>src/main/resources</filename> and <filename>src/test/resources</filename>
-            to avoid error message in the console 'Failed to execute goal org.apache.maven.plugins:maven-antrun-plugin:1.6:run (default) on project hbase: 
-            'An Ant BuildException has occured: Replace: source file .../target/classes/hbase-default.xml doesn't exist'. This will also
-            reduce the eclipse build cycles and make your life easier when developing.</para>
-            </section>     
             <section xml:id="eclipse.issues">
             <title>Eclipse Known Issues</title>
             <para>Eclipse will currently complain about <filename>Bytes.java</filename>.  It is not possible to turn these errors off.</para>
-            <programlisting>            
+            <programlisting>
 Description	Resource	Path	Location	Type
 Access restriction: The method arrayBaseOffset(Class) from the type Unsafe is not accessible due to restriction on required library /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar	Bytes.java	/hbase/src/main/java/org/apache/hadoop/hbase/util	line 1061	Java Problem
 Access restriction: The method arrayIndexScale(Class) from the type Unsafe is not accessible due to restriction on required library /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar	Bytes.java	/hbase/src/main/java/org/apache/hadoop/hbase/util	line 1064	Java Problem
 Access restriction: The method getLong(Object, long) from the type Unsafe is not accessible due to restriction on required library /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar	Bytes.java	/hbase/src/main/java/org/apache/hadoop/hbase/util	line 1111	Java Problem
-             </programlisting>   
+             </programlisting>
              </section>
              <section xml:id="eclipse.more">
              <title>Eclipse - More Information</title>
-             <para>For additional information on setting up Eclipse for HBase development on Windows, see 
+             <para>For additional information on setting up Eclipse for HBase development on Windows, see
              <link xlink:href="http://michaelmorello.blogspot.com/2011/09/hbase-subversion-eclipse-windows.html">Michael Morello's blog</link> on the topic.
              </para>
              </section>
         </section>
-    </section> 
-    
+    </section>
+
         <section xml:id="build">
-       <title>Building HBase</title>
-      <para>This section will be of interest only to those building HBase from source.
-      </para>
+       <title>Building Apache HBase</title>
+      <section xml:id="build.basic">
+       <title>Basic Compile</title>
+       <para>Thanks to maven, building HBase is pretty easy. You can read about the various maven commands in <xref linkend="maven.build.commands"/>, but the simplest command to compile HBase from its java source code is:
+       <programlisting>
+mvn package -DskipTests
+       </programlisting>
+       Or, to clean up before compiling:
+       <programlisting>
+mvn clean package -DskipTests
+       </programlisting>
+       With Eclipse set up as explained above in <xref linkend="eclipse"/>, you can also simply use the build command in Eclipse. To create the full installable HBase package takes a little bit more work, so read on.
+       </para>
+      </section>
       <section xml:id="build.snappy">
         <title>Building in snappy compression support</title>
         <para>Pass <code>-Dsnappy</code> to trigger the snappy maven profile for building
-            snappy native libs into hbase.</para>
+            snappy native libs into hbase.  See also <xref linkend="snappy.compression" /></para>
       </section>
 
       <section xml:id="build.tgz">
         <title>Building the HBase tarball</title>
         <para>Do the following to build the HBase tarball.
-        Passing the -Drelease will generate javadoc and run the RAT plugin to verify licenses on source.
-        <programlisting>% MAVEN_OPTS="-Xmx2g" mvn clean site install assembly:single -Dmaven.test.skip -Prelease</programlisting>
+        Passing the -Prelease will generate javadoc and run the RAT plugin to verify licenses on source.
+        <programlisting>% MAVEN_OPTS="-Xmx2g" mvn clean site install assembly:assembly -DskipTests -Prelease</programlisting>
 </para>
       </section>
 
+
+      <section xml:id="build.gotchas"><title>Build Gotchas</title>
+			<para>If you see <code>Unable to find resource 'VM_global_library.vm'</code>, ignore it.
+			Its not an error.  It is <link xlink:href="http://jira.codehaus.org/browse/MSITE-286">officially ugly</link> though.
+           </para>
+      </section>
+    </section> <!--  build -->
       <section xml:id="mvn_repo">
-        <title>Adding an HBase release to Apache's Maven Repository</title>
+        <title>Adding an Apache HBase release to Apache's Maven Repository</title>
         <para>Follow the instructions at
-        <link xlink:href="http://www.apache.org/dev/publishing-maven-artifacts.html">Publishing Maven Artifacts</link>.
-            The 'trick' to making it all work is answering the questions put to you by the mvn release plugin properly,
-            making sure it is using the actual branch AND before doing the <command>mvn release:perform</command> step,
-            VERY IMPORTANT, hand edit the release.properties file that was put under <varname>${HBASE_HOME}</varname>
-            by the previous step, <command>release:perform</command>. You need to edit it to make it point at
-            right locations in SVN.
+        <link xlink:href="http://www.apache.org/dev/publishing-maven-artifacts.html">Publishing Maven Artifacts</link> after
+        reading the below miscellaney.
+        </para>
+        <para>You must use maven 3.0.x (Check by running <command>mvn -version</command>).
+        </para>
+        <para>Let me list out the commands I used first.  The sections that follow dig in more
+        on what is going on.  In this example, we are releasing the 0.92.2 jar to the apache
+        maven repository.
+        <programlisting>
+  # First make a copy of the tag we want to release; presumes the release has been tagged already
+  # We do this because we need to make some commits for the mvn release plugin to work.
+  853  svn copy -m "Publishing 0.92.2 to mvn"  https://svn.apache.org/repos/asf/hbase/tags/0.92.2 https://svn.apache.org/repos/asf/hbase/tags/0.92.2mvn
+  857  svn checkout https://svn.apache.org/repos/asf/hbase/tags/0.92.2mvn
+  858  cd 0.92.2mvn/
+  # Edit the version making it release version with a '-SNAPSHOT' suffix (See below for more on this)
+  860  vi pom.xml
+  861  svn commit -m "Add SNAPSHOT to the version" pom.xml
+  862  ~/bin/mvn/bin/mvn release:clean
+  865  ~/bin/mvn/bin/mvn release:prepare
+  866  # Answer questions and then ^C to kill the build after the last question. See below for more on this.
+  867  vi release.properties
+       # Change the references to trunk svn to be 0.92.2mvn; the release plugin presumes trunk
+       # Then restart the release:prepare -- it won't ask questions
+       # because the properties file exists.
+  868  ~/bin/mvn/bin/mvn release:prepare
+  # The apache-release profile comes from the apache parent pom and does signing of artifacts published
+  869  ~/bin/mvn/bin/mvn release:perform  -Papache-release
+       # When done copying up to apache staging repository,
+       # browse to repository.apache.org, login and finish
+       # the release as according to the above
+       # "Publishing Maven Artifacts.
+        </programlisting>
+        </para>
+        <para>Below is more detail on the commmands listed above.</para>
+        <para>At the <command>mvn release:perform</command> step, before starting, if you are for example
+        releasing hbase 0.92.2, you need to make sure the pom.xml version is 0.92.2-SNAPSHOT.  This needs
+        to be checked in.  Since we do the maven release after actual release, I've been doing this
+        checkin into a copy of the release tag rather than into the actual release tag itself (presumes the release has been properly tagged in svn).
+        So, say we released hbase 0.92.2 and now we want to do the release to the maven repository, in svn, the 0.92.2
+        release will be tagged 0.92.2.  Making the maven release, copy the 0.92.2 tag to 0.92.2mvn.
+        Check out this tag and change the version therein and commit.
+        </para>
+        <para>
+            Currently, the mvn release wants to go against trunk.  I haven't figured how to tell it to do otherwise
+            so I do the below hack.  The hack comprises answering the questions put to you by the mvn release plugin properly,
+            then immediately control-C'ing the build after the last question asked as the build release step starts to run.
+            After control-C'ing it, You'll notice a release.properties in your build dir.  Review it.
+            Make sure it is using the proper branch -- it tends to use trunk rather than the 0.92.2mvn or whatever
+            that you want it to use -- so hand edit the release.properties file that was put under <varname>${HBASE_HOME}</varname>
+            by the <command>release:perform</command> invocation.  When done, resstart the
+            <command>release:perform</command>.
         </para>
+        <para>Here is how I'd answer the questions at <command>release:prepare</command> time:
+        <programlisting>What is the release version for "HBase"? (org.apache.hbase:hbase) 0.92.2: :
+What is SCM release tag or label for "HBase"? (org.apache.hbase:hbase) hbase-0.92.2: : 0.92.2mvn
+What is the new development version for "HBase"? (org.apache.hbase:hbase) 0.92.3-SNAPSHOT: :
+[INFO] Transforming 'HBase'...</programlisting>
+        </para>
+        <para>When you run <command>release:perform</command>, pass <command>-Papache-release</command>
+        else it will not 'sign' the artifacts it uploads.
+        </para>
+        <para>A strange issue I ran into was the one where the upload into the apache
+        repository was being sprayed across multiple apache machines making it so I could
+        not release.  See <link xlink:href="https://issues.apache.org/jira/browse/INFRA-4482">INFRA-4482 Why is my upload to mvn spread across multiple repositories?</link>.</para>
+
+        <para xml:id="mvn.settings.file">Here is my <filename>~/.m2/settings.xml</filename>.
+        This is read by the release plugin.  The apache-release profile will pick up your
+        gpg key setup from here if you've specified it into the file.  The password
+        can be maven encrypted as suggested in the "Publishing Maven Artifacts" but plain
+        text password works too (just don't let anyone see your local settings.xml).
+        <programlisting>&lt;settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+                      http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  &lt;servers>
+    &lt;!- To publish a snapshot of some part of Maven -->
+    &lt;server>
+      &lt;id>apache.snapshots.https&lt;/id>
+      &lt;username>YOUR_APACHE_ID
+      &lt;/username>
+      &lt;password>YOUR_APACHE_PASSWORD
+      &lt;/password>
+    &lt;/server>
+    &lt;!-- To publish a website using Maven -->
+    &lt;!-- To stage a release of some part of Maven -->
+    &lt;server>
+      &lt;id>apache.releases.https&lt;/id>
+      &lt;username>YOUR_APACHE_ID
+      &lt;/username>
+      &lt;password>YOUR_APACHE_PASSWORD
+      &lt;/password>
+    &lt;/server>
+  &lt;/servers>
+  &lt;profiles>
+    &lt;profile>
+      &lt;id>apache-release&lt;/id>
+      &lt;properties>
+    &lt;gpg.keyname>YOUR_KEYNAME&lt;/gpg.keyname>
+    &lt;!--Keyname is something like this ... 00A5F21E... do gpg --list-keys to find it-->
+    &lt;gpg.passphrase>YOUR_KEY_PASSWORD
+    &lt;/gpg.passphrase>
+      &lt;/properties>
+    &lt;/profile>
+  &lt;/profiles>
+&lt;/settings>
+        </programlisting>
+        </para>
+
         <para>If you see run into the below, its because you need to edit version in the pom.xml and add
         <code>-SNAPSHOT</code> to the version (and commit).
         <programlisting>[INFO] Scanning for projects...
@@ -168,73 +308,163 @@ Access restriction: The method getLong(Object, long) from the type Unsafe is not
 [INFO] -----------------------------------------------------------------------</programlisting>
         </para>
       </section>
-      <section xml:id="build.gotchas"><title>Build Gotchas</title>
-			<para>If you see <code>Unable to find resource 'VM_global_library.vm'</code>, ignore it.  
-			Its not an error.  It is <link xlink:href="http://jira.codehaus.org/browse/MSITE-286">officially ugly</link> though.
-           </para>
+      <section xml:id="documentation">
+          <title>Generating the HBase Reference Guide</title>
+          <para>The manual is marked up using <link xlink:href="http://www.docbook.org/">docbook</link>.
+              We then use the <link xlink:href="http://code.google.com/p/docbkx-tools/">docbkx maven plugin</link>
+              to transform the markup to html.  This plugin is run when you specify the <command>site</command>
+              goal as in when you run <command>mvn site</command> or you can call the plugin explicitly to
+              just generate the manual by doing <command>mvn docbkx:generate-html</command>
+              (TODO: It looks like you have to run <command>mvn site</command> first because docbkx wants to
+              include a transformed <filename>hbase-default.xml</filename>.  Fix).
+              When you run mvn site, we do the document generation twice, once to generate the multipage
+              manual and then again for the single page manual (the single page version is easier to search).
+          </para>
       </section>
-    </section> <!--  build -->
-
+    <section xml:id="hbase.org">
+    <title>Updating hbase.apache.org</title>
+      <section xml:id="hbase.org.site.contributing">
+      <title>Contributing to hbase.apache.org</title>
+      <para>The Apache HBase apache web site (including this reference guide) is maintained as part of the main Apache HBase source tree, under <filename>/src/docbkx</filename> and <filename>/src/site</filename>. The former is this reference guide; the latter, in most cases, are legacy pages that are in the process of being merged into the docbkx tree.</para>
+      <para>To contribute to the reference guide, edit these files and submit them as a patch (see <xref linkend="submitting.patches"/>). Your Jira should contain a summary of the changes in each section (see <link xlink:href="https://issues.apache.org/jira/browse/HBASE-6081">HBASE-6081</link> for an example).</para>
+      <para>To generate the site locally while you're working on it, run:
+      <programlisting>mvn site</programlisting>
+      Then you can load up the generated HTML files in your browser (file are under <filename>/target/site</filename>).</para>
+      </section>
+      <section xml:id="hbase.org.site.publishing">
+      <title>Publishing hbase.apache.org</title>
+      <para>As of <link xlink:href="https://issues.apache.org/jira/browse/INFRA-5680">INFRA-5680 Migrate apache hbase website</link>,
+          to publish the website, build it, and then deploy it over a checkout of <filename>https://svn.apache.org/repos/asf/hbase/hbase.apache.org/trunk</filename>,
+          and then check it in.  For example, if trunk is checked out out at <filename>/Users/stack/checkouts/trunk</filename>
+          and hbase.apache.org is checked out at <filename>/Users/stack/checkouts/hbase.apache.org/trunk</filename>, to update
+          the site, do the following:
+          <programlisting>
+              # Build the site and deploy it to the checked out directory
+              # Getting the javadoc into site is a little tricky.  You have to build it independent, then
+              # 'aggregate' it at top-level so the pre-site site lifecycle step can find it; that is
+              # what the javadoc:javadoc and javadoc:aggregate is about.
+              $ MAVEN_OPTS=" -Xmx3g" mvn clean -DskipTests javadoc:javadoc javadoc:aggregate site  site:stage -DstagingDirectory=/Users/stack/checkouts/hbase.apache.org/trunk
+              # Check the deployed site by viewing in a brower.
+              # If all is good, commit it and it will show up at http://hbase.apache.org
+              #
+              $ cd /Users/stack/checkouts/hbase.apache.org/trunk
+              $ svn commit -m 'Committing latest version of website...'
+          </programlisting>
+      </para>
+      </section>
+    </section>
     <section xml:id="hbase.tests">
     <title>Tests</title>
 
-<para>HBase tests are divided into two groups: <xref linkend="hbase.unittests"/> and
-<xref linkend="integration.tests" />.
-Unit tests are run by the Apache Continuous Integration server and by developers
-when they are verifying a fix does not cause breakage elsewhere in the code base.
-Integration tests are generally long-running tests that are invoked out-of-bound of
-the CI server when you want to do more intensive testing beyond the unit test set.
-Integration tests, for example, are run proving a release candidate or a production
-deploy. Below we go into more detail on each of these test types.  Developers at a
-minimum should familiarize themselves with the unit test detail; unit tests in
-HBase have a character not usually seen in other projects.</para> 
+<para> Developers, at a minimum, should familiarize themselves with the unit test detail; unit tests in
+HBase have a character not usually seen in other projects.</para>
+
+<section xml:id="hbase.moduletests">
+<title>Apache HBase Modules</title>
+<para>As of 0.96, Apache HBase is split into multiple modules which creates "interesting" rules for
+how and where tests are written. If you are writting code for <classname>hbase-server</classname>, see
+<xref linkend="hbase.unittests"/> for how to write your tests; these tests can spin
+up a minicluster and will need to be categorized. For any other module, for example
+<classname>hbase-common</classname>, the tests must be strict unit tests and just test the class
+under test - no use of the HBaseTestingUtility or minicluster is allowed (or even possible
+given the dependency tree).</para>
+  <section xml:id="hbase.moduletest.run">
+  <title>Running Tests in other Modules</title>
+  If the module you are developing in has no other dependencies on other HBase modules, then
+  you can cd into that module and just run:
+  <programlisting>mvn test</programlisting>
+  which will just run the tests IN THAT MODULE. If there are other dependencies on other modules,
+  then you will have run the command from the ROOT HBASE DIRECTORY. This will run the tests in the other
+  modules, unless you specify to skip the tests in that module. For instance, to skip the tests in the hbase-server module,
+  you would run:
+  <programlisting>mvn clean test -PskipServerTests</programlisting>
+  from the top level directory to run all the tests in modules other than hbase-server. Note that you
+  can specify to skip tests in multiple modules as well as just for a single module. For example, to skip
+  the tests in <classname>hbase-server</classname> and <classname>hbase-common</classname>, you would run:
+  <programlisting>mvn clean test -PskipServerTests -PskipCommonTests</programlisting>
+  <para>Also, keep in mind that if you are running tests in the <classname>hbase-server</classname> module you will need to
+  apply the maven profiles discussed in <xref linkend="hbase.unittests.cmds"/> to get the tests to run properly.</para>
+  </section>
+</section>
 
 <section xml:id="hbase.unittests">
 <title>Unit Tests</title>
-<para>HBase unit tests are subdivided into three categories: small, medium and large, with
-corresponding JUnit <link xlink:href="http://www.junit.org/node/581">categories</link>:
+<para>Apache HBase unit tests are subdivided into four categories: small, medium, large, and
+integration with corresponding JUnit <link xlink:href="http://www.junit.org/node/581">categories</link>:
 <classname>SmallTests</classname>, <classname>MediumTests</classname>,
-<classname>LargeTests</classname>.  JUnit categories are denoted using java annotations
-and look like this in your unit test code.
+<classname>LargeTests</classname>, <classname>IntegrationTests</classname>.
+JUnit categories are denoted using java annotations and look like this in your unit test code.
 <programlisting>...
 @Category(SmallTests.class)
 public class TestHRegionInfo {
-
   @Test
   public void testCreateHRegionInfoName() throws Exception {
     // ...
   }
 }</programlisting>
-The above example shows how to mark a test as belonging to the small category.
+The above example shows how to mark a unit test as belonging to the small category.
+All unit tests in HBase have a categorization.
 </para>
 <para>
+The first three categories, small, medium, and large are for tests run when
+you type <code>$ mvn test</code>; i.e. these three categorizations are for
+HBase unit tests. The integration category is for not for unit tests but for integration
+tests.  These are run when you invoke <code>$ mvn verify</code>.  Integration tests
+are described in <xref linkend="integration.tests">integration tests section</xref> and will not be discussed further
+in this section on HBase unit tests.</para>
+<para>
+Apache HBase uses a patched maven surefire plugin and maven profiles to implement
+its unit test characterizations.
+</para>
+<para>Read the below to figure which annotation of the set small, medium, and large to
+put on your new HBase unit test.
+</para>
+
+<section xml:id="hbase.unittests.small">
+<title>Small Tests<indexterm><primary>SmallTests</primary></indexterm></title>
+<para>
 <emphasis>Small</emphasis> tests are executed in a shared JVM. We put in this category all the tests that can
-be executed quickly in a shared JVM.  The maximum execution time for a test is 15 seconds,
-and they do not use a cluster.  <emphasis>Medium</emphasis> tests represent tests that must be executed
+be executed quickly in a shared JVM.  The maximum execution time for a small test is 15 seconds,
+and small tests should not use a (mini)cluster.</para>
+</section>
+
+<section xml:id="hbase.unittests.medium">
+<title>Medium Tests<indexterm><primary>MediumTests</primary></indexterm></title>
+<para><emphasis>Medium</emphasis> tests represent tests that must be executed
 before proposing a patch. They are designed to run in less than 30 minutes altogether,
 and are quite stable in their results. They are designed to last less than 50 seconds
 individually. They can use a cluster, and each of them is executed in a separate JVM.
-<emphasis>Large</emphasis> tests are everything else. They are typically integration-like
-tests (yes, some large tests should be moved out to be HBase <xref linkend="integration.tests" />),
-regression tests for specific bugs, timeout tests, performance tests.
+</para>
+</section>
+
+<section xml:id="hbase.unittests.large">
+<title>Large Tests<indexterm><primary>LargeTests</primary></indexterm></title>
+<para><emphasis>Large</emphasis> tests are everything else. They are typically large-scale
+tests, regression tests for specific bugs, timeout tests, performance tests.
 They are executed before a commit on the pre-integration machines. They can be run on
 the developer machine as well.
 </para>
-<para>HBase uses a patched maven surefire plugin and maven profiles to implement its
-unit test characterizations.</para>
+</section>
+<section xml:id="hbase.unittests.integration">
+<title>Integration Tests<indexterm><primary>IntegrationTests</primary></indexterm></title>
+<para><emphasis>Integration</emphasis> tests are system level tests. See
+<xref linkend="integration.tests">integration tests section</xref> for more info.
+</para>
+</section>
+</section>
 
 <section xml:id="hbase.unittests.cmds">
 <title>Running tests</title>
-<para>Below we describe how to run the HBase junit categories.</para>
+<para>Below we describe how to run the Apache HBase junit categories.</para>
 
 <section xml:id="hbase.unittests.cmds.test">
 <title>Default: small and medium category tests
 </title>
-<para>Running <programlisting>mvn test</programlisting> will execute all small tests in a single JVM and medium tests in a separate JVM for
-each test instance. Medium tests are NOT executed if there is an error in a small test.
+<para>Running <programlisting>mvn test</programlisting> will execute all small tests in a single JVM
+(no fork) and then medium tests in a separate JVM for each test instance.
+Medium tests are NOT executed if there is an error in a small test.
 Large tests are NOT executed.  There is one report for small tests, and one report for
-medium tests if they are executed. To run small and medium tests with the security
-profile enabled, do <programlisting>mvn test -P security</programlisting>
+medium tests if they are executed.
 </para>
 </section>
 
@@ -244,42 +474,69 @@ profile enabled, do <programlisting>mvn test -P security</programlisting>
 will execute small tests in a single JVM then medium and large tests in a separate JVM for each test.
 Medium and large tests are NOT executed if there is an error in a small test.
 Large tests are NOT executed if there is an error in a small or medium test.
-There is one report for small tests, and one report for medium and large tests if they are executed
+There is one report for small tests, and one report for medium and large tests if they are executed.
 </para>
 </section>
 
 <section xml:id="hbase.unittests.cmds.test.localtests.mytest">
 <title>Running a single test or all tests in a package</title>
 <para>To run an individual test, e.g. <classname>MyTest</classname>, do
-<programlisting>mvn test -P localTests -Dtest=MyTest</programlisting>  You can also
+<programlisting>mvn test -Dtest=MyTest</programlisting>  You can also
 pass multiple, individual tests as a comma-delimited list:
-<programlisting>mvn test -P localTests -Dtest=MyTest1,MyTest2,MyTest3</programlisting>
+<programlisting>mvn test -Dtest=MyTest1,MyTest2,MyTest3</programlisting>
 You can also pass a package, which will run all tests under the package:
-<programlisting>mvn test -P localTests -Dtest=org.apache.hadoop.hbase.client.*</programlisting>
-To run a single test with the security profile enabled:
-<programlisting>mvn test -P security,localTests -Dtest=TestGet</programlisting>
+<programlisting>mvn test -Dtest=org.apache.hadoop.hbase.client.*</programlisting>
 </para>
 
 <para>
-The <code>-P localTests</code>  will remove the JUnit category effect (without this specific profile,
-the profiles are taken into account).  It will actually use the official release of surefire
-and the old connector (The HBase build uses a patched version of the maven surefire plugin).
-junit tests are executed in separated JVM.  You will see a new message at the end of the
-report: "[INFO] Tests are skipped". It's harmless.
+When <code>-Dtest</code> is specified, <code>localTests</code> profile will be used. It will use the official release
+of maven surefire, rather than our custom surefire plugin, and the old connector (The HBase build uses a patched
+version of the maven surefire plugin). Each junit tests is executed in a separate JVM (A fork per test class).
+There is no parallelization when tests are running in this mode. You will see a new message at the end of the
+-report: "[INFO] Tests are skipped". It's harmless. While you need to make sure the sum of <code>Tests run:</code> in
+the <code>Results :</code> section of test reports matching the number of tests you specified because no
+error will be reported when a non-existent test case is specified.
 </para>
 </section>
 
 <section xml:id="hbase.unittests.cmds.test.profiles">
 <title>Other test invocation permutations</title>
-<para>Running <programlisting>mvn test -P runSmallTests</programlisting> will execute small tests only, in a single JVM.
+<para>Running <programlisting>mvn test -P runSmallTests</programlisting> will execute "small" tests only, using a single JVM.
+</para>
+<para>Running <programlisting>mvn test -P runMediumTests</programlisting> will execute "medium" tests only, launching a new JVM for each test-class.
 </para>
-<para>Running <programlisting>mvn test -P runMediumTests</programlisting> will execute medium tests in a single JVM.
+<para>Running <programlisting>mvn test -P runLargeTests</programlisting> will execute "large" tests only, launching a new JVM for each test-class.
 </para>
-<para>Running <programlisting>mvn test -P runLargeTests</programlisting> execute medium tests in a single JVM.
+<para>For convenience, you can run <programlisting>mvn test -P runDevTests</programlisting> to execute both small and medium tests, using a single JVM.
 </para>
+</section>
+
+<section xml:id="hbase.unittests.test.faster">
+<title>Running tests faster</title>
+<para>
+By default, <code>$ mvn test -P runAllTests</code> runs 5 tests in parallel.
+It can be increased on a developer's machine. Allowing that you can have 2
+tests in parallel per core, and you need about 2Gb of memory per test (at the
+extreme), if you have an 8 core, 24Gb box, you can have 16 tests in parallel.
+but the memory available limits it to 12 (24/2), To run all tests with 12 tests
+in parallell, do this:
+<command>mvn test -P runAllTests -Dsurefire.secondPartThreadCount=12</command>.
+To increase the speed, you can as well use a ramdisk. You will need 2Gb of memory
+to run all tests. You will also need to delete the files between two test run.
+The typical way to configure a ramdisk on Linux is:
+<programlisting>$ sudo mkdir /ram2G
+sudo mount -t tmpfs -o size=2048M tmpfs /ram2G</programlisting>
+You can then use it to run all HBase tests with the command:
+<command>mvn test -P runAllTests -Dsurefire.secondPartThreadCount=12 -Dtest.build.data.basedirectory=/ram2G</command>
+</para>
+</section>
 
+<section xml:id="hbase.unittests.cmds.test.hbasetests">
+<title><command>hbasetests.sh</command></title>
 <para>It's also possible to use the script <command>hbasetests.sh</command>. This script runs the medium and
-large tests in parallel with two maven instances, and provide a single report.
+large tests in parallel with two maven instances, and provides a single report.  This script does not use
+the hbase version of surefire so no parallelization is being done other than the two maven instances the
+script sets up.
 It must be executed from the directory which contains the <filename>pom.xml</filename>.</para>
 <para>For example running
 <programlisting>./dev-support/hbasetests.sh</programlisting> will execute small and medium tests.
@@ -288,6 +545,26 @@ Running <programlisting>./dev-support/hbasetests.sh replayFailed</programlisting
 second time, in a separate jvm and without parallelisation.
 </para>
 </section>
+<section xml:id="hbase.unittests.resource.checker">
+<title>Test Resource Checker<indexterm><primary>Test Resource Checker</primary></indexterm></title>
+<para>
+A custom Maven SureFire plugin listener checks a  number of resources before
+and after each HBase unit test runs and logs its findings at the end of the test
+output files which can be found in <filename>target/surefire-reports</filename>
+per Maven module (Tests write test reports named for the test class into this directory.
+Check the <filename>*-out.txt</filename> files).  The resources counted are the number
+of threads, the number of file descriptors, etc. If the number has increased, it adds
+a <emphasis>LEAK?</emphasis> comment in the logs. As you can have an HBase instance
+running in the background, some threads can be deleted/created without any specific
+action in the test. However, if the test does not work as expected, or if the test
+should not impact these resources, it's worth checking these log lines
+<computeroutput>...hbase.ResourceChecker(157): before...</computeroutput> and
+<computeroutput>...hbase.ResourceChecker(157): after...</computeroutput>. For example:
+<computeroutput>
+2012-09-26 09:22:15,315 INFO  [pool-1-thread-1] hbase.ResourceChecker(157): after: regionserver.TestColumnSeeking#testReseeking Thread=65 (was 65), OpenFileDescriptor=107 (was 107), MaxFileDescriptor=10240 (was 10240), ConnectionCount=1 (was 1)
+</computeroutput>
+</para>
+</section>
 </section>
 
 <section xml:id="hbase.tests.writing">
@@ -307,8 +584,12 @@ Tests should not overlog. More than 100 lines/second makes the logs complex to r
 <listitem>
 Tests can be written with <classname>HBaseTestingUtility</classname>.
 This class offers helper functions to create a temp directory and do the cleanup, or to start a cluster.
-Categories and execution time
 </listitem>
+</itemizedlist>
+</section>
+<section xml:id="hbase.tests.categories">
+<title>Categories and execution time</title>
+<itemizedlist>
 <listitem>
 All tests must be categorized, if not they could be skipped.
 </listitem>
@@ -345,30 +626,50 @@ As most as possible, tests should use the default settings for the cluster. When
 </para>
 </section>
 </section>
-</section>
 
 <section xml:id="integration.tests">
 <title>Integration Tests</title>
-<para>HBase integration Tests are tests that are beyond HBase unit tests.  They
+<para>HBase integration/system tests are tests that are beyond HBase unit tests.  They
 are generally long-lasting, sizeable (the test can be asked to 1M rows or 1B rows),
 targetable (they can take configuration that will point them at the ready-made cluster
 they are to run against; integration tests do not include cluster start/stop code),
 and verifying success, integration tests rely on public APIs only; they do not
-attempt to examine server internals asserring success/fail. Integration tests
+attempt to examine server internals asserting success/fail. Integration tests
 are what you would run when you need to more elaborate proofing of a release candidate
 beyond what unit tests can do. They are not generally run on the Apache Continuous Integration
-build server.
+build server, however, some sites opt to run integration tests as a part of their
+continuous testing on an actual cluster.
 </para>
 <para>
-Integration tests currently live under the <filename>src/test</filename> directory and
-will match the regex: <filename>**/IntegrationTest*.java</filename>. 
+Integration tests currently live under the <filename>src/test</filename> directory
+in the hbase-it submodule and will match the regex: <filename>**/IntegrationTest*.java</filename>.
+All integration tests are also annotated with <code>@Category(IntegrationTests.class)</code>.
 </para>
+
+<para>
+Integration tests can be run in two modes: using a mini cluster, or against an actual distributed cluster.
+Maven failsafe is used to run the tests using the mini cluster. IntegrationTestsDriver class is used for
+executing the tests against a distributed cluster. Integration tests SHOULD NOT assume that they are running against a
+mini cluster, and SHOULD NOT use private API's to access cluster state. To interact with the distributed or mini
+cluster uniformly, <code>IntegrationTestingUtility</code>, and <code>HBaseCluster</code> classes,
+and public client API's can be used.
+</para>
+
+<section xml:id="maven.build.commands.integration.tests.mini">
+<title>Running integration tests against mini cluster</title>
 <para>HBase 0.92 added a <varname>verify</varname> maven target.
 Invoking it, for example by doing <code>mvn verify</code>, will
 run all the phases up to and including the verify phase via the
 maven <link xlink:href="http://maven.apache.org/plugins/maven-failsafe-plugin/">failsafe plugin</link>,
 running all the above mentioned HBase unit tests as well as tests that are in the HBase integration test group.
-If you just want to run the integration tests, you need to run two commands. First:
+After you have completed
+          <programlisting>mvn install -DskipTests</programlisting>
+You can run just the integration tests by invoking:
+          <programlisting>
+cd hbase-it
+mvn verify</programlisting>
+
+If you just want to run the integration tests in top-level, you need to run two commands. First:
           <programlisting>mvn failsafe:integration-test</programlisting>
 This actually runs ALL the integration tests.
           <note><para>This command will always output <code>BUILD SUCCESS</code> even if there are test failures.
@@ -379,75 +680,170 @@ This actually runs ALL the integration tests.
 
       <section xml:id="maven.build.commanas.integration.tests2">
           <title>Running a subset of Integration tests</title>
-          <para>This is very similar to how you specify running a subset of unit tests (see above).
+          <para>This is very similar to how you specify running a subset of unit tests (see above), but use the property
+	      <code>it.test</code> instead of <code>test</code>.
 To just run <classname>IntegrationTestClassXYZ.java</classname>, use:
-          <programlisting>mvn failsafe:integration-test -Dtest=IntegrationTestClassXYZ</programlisting> 
-          Pretty similar, right?
+          <programlisting>mvn failsafe:integration-test -Dit.test=IntegrationTestClassXYZ</programlisting>
           The next thing you might want to do is run groups of integration tests, say all integration tests that are named IntegrationTestClassX*.java:
-          <programlisting>mvn failsafe:integration-test -Dtest=*ClassX*</programlisting> 
+          <programlisting>mvn failsafe:integration-test -Dit.test=*ClassX*</programlisting>
           This runs everything that is an integration test that matches *ClassX*. This means anything matching: "**/IntegrationTest*ClassX*".
           You can also run multiple groups of integration tests using comma-delimited lists (similar to unit tests). Using a list of matches still supports full regex matching for each of the groups.This would look something like:
-          <programlisting>mvn failsafe:integration-test -Dtest=*ClassX*, *ClassY</programlisting>  
+          <programlisting>mvn failsafe:integration-test -Dit.test=*ClassX*, *ClassY</programlisting>
           </para>
       </section>
-    </section>
+</section>
+<section xml:id="maven.build.commands.integration.tests.distributed">
+<title>Running integration tests against distributed cluster</title>
+<para>
+If you have an already-setup HBase cluster, you can launch the integration tests by invoking the class <code>IntegrationTestsDriver</code>. You may have to
+run test-compile first. The configuration will be picked by the bin/hbase script.
+<programlisting>mvn test-compile</programlisting>
+Then launch the tests with:
+<programlisting>bin/hbase [--config config_dir] org.apache.hadoop.hbase.IntegrationTestsDriver [-test=class_regex]</programlisting>
+
+This execution will launch the tests under <code>hbase-it/src/test</code>, having <code>@Category(IntegrationTests.class)</code> annotation,
+and a name starting with <code>IntegrationTests</code>. If specified, class_regex will be   used to filter test classes. The regex is checked against full class name; so, part of class name can be used.
+IntegrationTestsDriver uses Junit to run the tests. Currently there is no support for running integration tests against a distributed cluster using maven (see <link xlink:href="https://issues.apache.org/jira/browse/HBASE-6201">HBASE-6201</link>).
+</para>
+
+<para>
+The tests interact with the distributed cluster by using the methods in the <code>DistributedHBaseCluster</code> (implementing <code>HBaseCluster</code>) class, which in turn uses a pluggable <code>ClusterManager</code>. Concrete implementations provide actual functionality for carrying out deployment-specific and environment-dependent tasks (SSH, etc). The default <code>ClusterManager</code> is <code>HBaseClusterManager</code>, which uses SSH to remotely execute start/stop/kill/signal commands, and assumes some posix commands (ps, etc). Also assumes the user running the test has enough "power" to start/stop servers on the remote machines. By default, it picks up <code>HBASE_SSH_OPTS, HBASE_HOME, HBASE_CONF_DIR</code> from the env, and uses <code>bin/hbase-daemon.sh</code> to carry out the actions. Currently tarball deployments, deployments which uses hbase-daemons.sh, and <link xlink:href="http://incubator.apache.org/ambari/">Apache Ambari</link> deployments are supported. /etc/init.d/ scripts are not supported for now, but it can be easily added. For other deployment options, a ClusterManager can be implemented and plugged in.
+</para>
+</section>
+
+<section xml:id="maven.build.commands.integration.tests.destructive">
+<title>Destructive integration / system tests</title>
+<para>
+	In 0.96, a tool named <code>ChaosMonkey</code> has been introduced. It is modeled after the <link xlink:href="http://techblog.netflix.com/2012/07/chaos-monkey-released-into-wild.html">same-named tool by Netflix</link>.
+Some of the tests use ChaosMonkey to simulate faults in the running cluster in the way of killing random servers,
+disconnecting servers, etc. ChaosMonkey can also be used as a stand-alone tool to run a (misbehaving) policy while you
+are running other tests.
+</para>
+
+<para>
+ChaosMonkey defines Action's and Policy's. Actions are sequences of events. We have at least the following actions:
+<itemizedlist>
+<listitem>Restart active master (sleep 5 sec)</listitem>
+<listitem>Restart random regionserver (sleep 5 sec)</listitem>
+<listitem>Restart random regionserver (sleep 60 sec)</listitem>
+<listitem>Restart META regionserver (sleep 5 sec)</listitem>
+<listitem>Restart ROOT regionserver (sleep 5 sec)</listitem>
+<listitem>Batch restart of 50% of regionservers (sleep 5 sec)</listitem>
+<listitem>Rolling restart of 100% of regionservers (sleep 5 sec)</listitem>
+</itemizedlist>
+
+Policies on the other hand are responsible for executing the actions based on a strategy.
+The default policy is to execute a random action every minute based on predefined action
+weights. ChaosMonkey executes predefined named policies until it is stopped. More than one
+policy can be active at any time.
+</para>
+
+<para>
+  To run ChaosMonkey as a standalone tool deploy your HBase cluster as usual. ChaosMonkey uses the configuration
+from the bin/hbase script, thus no extra configuration needs to be done. You can invoke the ChaosMonkey by running:
+<programlisting>bin/hbase org.apache.hadoop.hbase.util.ChaosMonkey</programlisting>
+
+This will output smt like:
+<programlisting>
+12/11/19 23:21:57 INFO util.ChaosMonkey: Using ChaosMonkey Policy: class org.apache.hadoop.hbase.util.ChaosMonkey$PeriodicRandomActionPolicy, period:60000
+12/11/19 23:21:57 INFO util.ChaosMonkey: Sleeping for 26953 to add jitter
+12/11/19 23:22:24 INFO util.ChaosMonkey: Performing action: Restart active master
+12/11/19 23:22:24 INFO util.ChaosMonkey: Killing master:master.example.com,60000,1353367210440
+12/11/19 23:22:24 INFO hbase.HBaseCluster: Aborting Master: master.example.com,60000,1353367210440
+12/11/19 23:22:24 INFO hbase.ClusterManager: Executing remote command: ps aux | grep master | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:master.example.com
+12/11/19 23:22:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:
+12/11/19 23:22:25 INFO hbase.HBaseCluster: Waiting service:master to stop: master.example.com,60000,1353367210440
+12/11/19 23:22:25 INFO hbase.ClusterManager: Executing remote command: ps aux | grep master | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:master.example.com
+12/11/19 23:22:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:
+12/11/19 23:22:25 INFO util.ChaosMonkey: Killed master server:master.example.com,60000,1353367210440
+12/11/19 23:22:25 INFO util.ChaosMonkey: Sleeping for:5000
+12/11/19 23:22:30 INFO util.ChaosMonkey: Starting master:master.example.com
+12/11/19 23:22:30 INFO hbase.HBaseCluster: Starting Master on: master.example.com
+12/11/19 23:22:30 INFO hbase.ClusterManager: Executing remote command: /homes/enis/code/hbase-0.94/bin/../bin/hbase-daemon.sh --config /homes/enis/code/hbase-0.94/bin/../conf start master , hostname:master.example.com
+12/11/19 23:22:31 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:starting master, logging to /homes/enis/code/hbase-0.94/bin/../logs/hbase-enis-master-master.example.com.out
+....
+12/11/19 23:22:33 INFO util.ChaosMonkey: Started master: master.example.com,60000,1353367210440
+12/11/19 23:22:33 INFO util.ChaosMonkey: Sleeping for:51321
+12/11/19 23:23:24 INFO util.ChaosMonkey: Performing action: Restart random region server
+12/11/19 23:23:24 INFO util.ChaosMonkey: Killing region server:rs3.example.com,60020,1353367027826
+12/11/19 23:23:24 INFO hbase.HBaseCluster: Aborting RS: rs3.example.com,60020,1353367027826
+12/11/19 23:23:24 INFO hbase.ClusterManager: Executing remote command: ps aux | grep regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:rs3.example.com
+12/11/19 23:23:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:
+12/11/19 23:23:25 INFO hbase.HBaseCluster: Waiting service:regionserver to stop: rs3.example.com,60020,1353367027826
+12/11/19 23:23:25 INFO hbase.ClusterManager: Executing remote command: ps aux | grep regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:rs3.example.com
+12/11/19 23:23:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:
+12/11/19 23:23:25 INFO util.ChaosMonkey: Killed region server:rs3.example.com,60020,1353367027826. Reported num of rs:6
+12/11/19 23:23:25 INFO util.ChaosMonkey: Sleeping for:60000
+12/11/19 23:24:25 INFO util.ChaosMonkey: Starting region server:rs3.example.com
+12/11/19 23:24:25 INFO hbase.HBaseCluster: Starting RS on: rs3.example.com
+12/11/19 23:24:25 INFO hbase.ClusterManager: Executing remote command: /homes/enis/code/hbase-0.94/bin/../bin/hbase-daemon.sh --config /homes/enis/code/hbase-0.94/bin/../conf start regionserver , hostname:rs3.example.com
+12/11/19 23:24:26 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /homes/enis/code/hbase-0.94/bin/../logs/hbase-enis-regionserver-rs3.example.com.out
+
+12/11/19 23:24:27 INFO util.ChaosMonkey: Started region server:rs3.example.com,60020,1353367027826. Reported num of rs:6
+</programlisting>
+
+As you can see from the log, ChaosMonkey started the default PeriodicRandomActionPolicy, which is configured with all the available actions, and ran RestartActiveMaster and RestartRandomRs actions. ChaosMonkey tool, if run from command line, will keep on running until the process is killed.
+</para>
+</section>
+</section>
 </section> <!-- tests -->
-    
-    <section xml:id="maven.build.commands"> 
+
+    <section xml:id="maven.build.commands">
        <title>Maven Build Commands</title>
        <para>All commands executed from the local HBase project directory.
        </para>
        <para>Note: use Maven 3 (Maven 2 may work but we suggest you use Maven 3).
        </para>
-       <section xml:id="maven.build.commands.compile"> 
+       <section xml:id="maven.build.commands.compile">
           <title>Compile</title>
           <programlisting>
 mvn compile
           </programlisting>
-       </section>       
+       </section>
 
-       <section xml:id="maven.build.commands.unitall"> 
+       <section xml:id="maven.build.commands.unitall">
           <title>Running all or individual Unit Tests</title>
           <para>See the <xref linkend="hbase.unittests.cmds" /> section
           above in <xref linkend="hbase.unittests" /></para>
-       </section>       
-
-       <section xml:id="maven.build.commanas.integration.tests"> 
-          <title>Running all or individual Integration Tests</title>
-          <para>See <xref linkend="integration.tests" />
-          </para>
-      </section>
+       </section>
 
-       <section xml:id="maven.build.hadoop"> 
-          <title>To build against hadoop 0.22.x or 0.23.x</title>
-          <programlisting>
-mvn -Dhadoop.profile=22 ...
-          </programlisting>
-<para>That is, designate build with hadoop.profile 22.  Pass 23 for hadoop.profile to build against hadoop 0.23.
-Tests do not all pass as of this writing so you may need ot pass <code>-DskipTests</code> unless you are inclined
-to fix the failing tests.
-</para>
+       <section xml:id="maven.build.hadoop">
+          <title>Building against various hadoop versions.</title>
+          <para>As of 0.96, Apache HBase supports building against Apache Hadoop versions: 1.0.3, 2.0.0-alpha and 3.0.0-SNAPSHOT.
+          By default, we will build with Hadoop-1.0.3. To change the version to run with Hadoop-2.0.0-alpha, you would run:</para>
+         <programlisting>mvn -Dhadoop.profile=2.0 ...</programlisting>
+         <para>
+         That is, designate build with hadoop.profile 2.0.  Pass 2.0 for hadoop.profile to build against hadoop 2.0.
+         Tests may not all pass as of this writing so you may need to pass <code>-DskipTests</code> unless you are inclined
+          to fix the failing tests.</para>
+          <para>
+         Similarly, for 3.0, you would just replace the profile value. Note that Hadoop-3.0.0-SNAPSHOT does not currently have a deployed maven artificat - you will need to build and install your own in your local maven repository if you want to run against this profile.
+         </para>
+         <para>
+         In earilier verions of Apache HBase, you can build against older versions of Apache Hadoop, notably, Hadoop 0.22.x and 0.23.x.
+         If you are running, for example HBase-0.94 and wanted to build against Hadoop 0.23.x, you would run with:</para>
+          <programlisting>mvn -Dhadoop.profile=22 ...</programlisting>
       </section>
     </section>
-  
-    <section xml:id="getting.involved"> 
+
+    <section xml:id="getting.involved">
         <title>Getting Involved</title>
-        <para>HBase gets better only when people contribute!
+        <para>Apache HBase gets better only when people contribute!
         </para>
-        <para>As HBase is an Apache Software Foundation project, see <xref linkend="asf"/> for more information about how the ASF functions.
+        <para>As Apache HBase is an Apache Software Foundation project, see <xref linkend="asf"/> for more information about how the ASF functions.
         </para>
         <section xml:id="mailing.list">
           <title>Mailing Lists</title>
-          <para>Sign up for the dev-list and the user-list.  See the 
+          <para>Sign up for the dev-list and the user-list.  See the
           <link xlink:href="http://hbase.apache.org/mail-lists.html">mailing lists</link> page.
-          Posing questions - and helping to answer other people's questions - is encouraged!  
-          There are varying levels of experience on both lists so patience and politeness are encouraged (and please 
-          stay on topic.)  
+          Posing questions - and helping to answer other people's questions - is encouraged!
+          There are varying levels of experience on both lists so patience and politeness are encouraged (and please
+          stay on topic.)
           </para>
         </section>
         <section xml:id="jira">
           <title>Jira</title>
-          <para>Check for existing issues in <link xlink:href="https://issues.apache.org/jira/browse/HBASE">Jira</link>.  
+          <para>Check for existing issues in <link xlink:href="https://issues.apache.org/jira/browse/HBASE">Jira</link>.
           If it's either a new feature request, enhancement, or a bug, file a ticket.
           </para>
           <section xml:id="jira.priorities"><title>Jira Priorities</title>
@@ -457,10 +853,10 @@ to fix the failing tests.
                 <listitem>Critical: The issue described can cause data loss or cluster instability in some cases.</listitem>
                 <listitem>Major: Important but not tragic issues, like updates to the client API that will add a lot of much-needed functionality or significant
                 bugs that need to be fixed but that don't cause data loss.</listitem>
-                <listitem>Minor: Useful enhancements and annoying but not damaging bugs.</listitem> 
-                <listitem>Trivial: Useful enhancements but generally cosmetic.</listitem> 
-                </itemizedlist>  
-             </para> 
+                <listitem>Minor: Useful enhancements and annoying but not damaging bugs.</listitem>
+                <listitem>Trivial: Useful enhancements but generally cosmetic.</listitem>
+                </itemizedlist>
+             </para>
         </section>
         <section xml:id="submitting.patches.jira.code">
           <title>Code Blocks in Jira Comments</title>
@@ -475,15 +871,15 @@ to fix the failing tests.
         </section>
        </section>  <!--  jira -->
       </section>  <!--  getting involved -->
-      
+
       <section xml:id="developing">
         <title>Developing</title>
         <section xml:id="codelines"><title>Codelines</title>
           <para>Most development is done on TRUNK.  However, there are branches for minor releases (e.g., 0.90.1, 0.90.2, and 0.90.3 are on the 0.90 branch).</para>
           <para>If you have any questions on this just send an email to the dev dist-list.</para>
         </section>
-  
-        <section xml:id="unit.tests"> 
+
+        <section xml:id="unit.tests">
           <title>Unit Tests</title>
           <para>In HBase we use <link xlink:href="http://junit.org">JUnit</link> 4.
             If you need to run miniclusters of HDFS, ZooKeeper, HBase, or MapReduce testing,
@@ -506,30 +902,82 @@ to fix the failing tests.
               </programlisting>
            </para>
          </section>
+        </section>   <!--  unit tests  -->
+
           <section xml:id="code.standards">
            <title>Code Standards</title>
            <para>See <xref linkend="eclipse.code.formatting"/> and <xref linkend="common.patch.feedback"/>.
            </para>
-           </section>
-        </section>   <!--  unit tests  -->       
+           <para>Also, please pay attention to the interface stability/audience classifications that you
+           will see all over our code base.   They look like this at the head of the class:
+           <programlisting>@InterfaceAudience.Public
+@InterfaceStability.Stable</programlisting>
+           </para>
+           <para>If the <classname>InterfaceAudience</classname> is <varname>Private</varname>,
+           we can change the class (and we do not need to include a <classname>InterfaceStability</classname> mark).
+           If a class is marked <varname>Public</varname> but its <classname>InterfaceStability</classname>
+           is marked <varname>Unstable</varname>, we can change it. If it's
+           marked <varname>Public</varname>/<varname>Evolving</varname>, we're allowed to change it
+           but should try not to. If it's <varname>Public</varname> and <varname>Stable</varname>
+           we can't change it without a deprecation path or with a really GREAT reason.</para>
+           <para>When you add new classes, mark them with the annotations above if publically accessible.
+           If you are not cleared on how to mark your additions, ask up on the dev list.
+           </para>
+           <para>This convention comes from our parent project Hadoop.</para>
+           </section> <!-- code.standards -->
+
+          <section xml:id="design.invariants">
+           <title>Invariants</title>
+           <para>We don't have many but what we have we list below.  All are subject to challenge of
+           course but until then, please hold to the rules of the road.
+           </para>
+          <section xml:id="design.invariants.zk.data">
+            <title>No permanent state in ZooKeeper</title>
+          <para>ZooKeeper state should transient (treat it like memory). If deleted, hbase
+          should be able to recover and essentially be in the same state<footnote><para>There are currently
+          a few exceptions that we need to fix around whether a table is enabled or disabled</para></footnote>.
+          </para>
+          </section>
+
+           </section> <!-- design.invariants -->
+
+          <section xml:id="run.insitu">
+           <title>Running In-Situ</title>
+           <para>If you are developing Apache HBase, frequently it is useful to test your changes against a more-real cluster than what you find in unit tests. In this case, HBase can be run directly from the source in local-mode.
+           All you need to do is run:
+           </para>
+           <programlisting>${HBASE_HOME}/bin/start-hbase.sh</programlisting>
+           <para>
+           This will spin up a full local-cluster, just as if you had packaged up HBase and installed it on your machine.
+           </para>
+           <para>Keep in mind that you will need to have installed HBase into your local maven repository for the in-situ cluster to work properly. That is, you will need to run:</para>
+           <programlisting>mvn clean install -DskipTests</programlisting>
+           <para>to ensure that maven can find the correct classpath and dependencies. Generally, the above command
+           is just a good thing to try running first, if maven is acting oddly.</para>
+           </section> <!-- run.insitu -->
 
       </section>  <!--  developing -->
 
        <section xml:id="submitting.patches">
           <title>Submitting Patches</title>
+          <para>If you are new to submitting patches to open source or new to submitting patches to Apache,
+          I'd suggest you start by reading the <link xlink:href="http://commons.apache.org/patches.html">On Contributing Patches</link>
+          page from <link xlink:href="http://commons.apache.org/">Apache Commons Project</link>.  Its a nice overview that
+          applies equally to the Apache HBase Project.</para>
           <section xml:id="submitting.patches.create">
             <title>Create Patch</title>
-          <para>Patch files can be easily generated from Eclipse, for example by selecting "Team -&gt; Create Patch".
+          <para>See the aforementioned Apache Commons link for how to make patches against a checked out subversion
+          repository.  Patch files can also be easily generated from Eclipse, for example by selecting "Team -&gt; Create Patch".
           Patches can also be created by git diff and svn diff.
           </para>
-          <para>Please submit one patch-file per Jira.  For example, if multiple files are changed make sure the 
+          <para>Please submit one patch-file per Jira.  For example, if multiple files are changed make sure the
           selected resource when generating the patch is a directory.  Patch files can reflect changes in multiple files. </para>
           <para>Make sure you review <xref linkend="eclipse.code.formatting"/> for code style. </para>
           </section>
           <section xml:id="submitting.patches.naming">
             <title>Patch File Naming</title>
-          <para>The patch file should have the HBase Jira ticket in the name.  For example, if a patch was submitted for <filename>Foo.java</filename>, then
-          a patch file called <filename>Foo_HBASE_XXXX.patch</filename> would be acceptable where XXXX is the HBase Jira number.
+          <para>The patch file should have the Apache HBase Jira ticket in the name.  For example, if a patch was submitted for <filename>Foo.java</filename>, then
+          a patch file called <filename>Foo_HBASE_XXXX.patch</filename> would be acceptable where XXXX is the Apache HBase Jira number.
           </para>
           <para>If you generating from a branch, then including the target branch in the filename is advised, e.g., <filename>HBASE-XXXX-0.90.patch</filename>.
           </para>
@@ -539,26 +987,30 @@ to fix the failing tests.
             <para>Yes, please.  Please try to include unit tests with every code patch (and especially new classes and large changes).
             Make sure unit tests pass locally before submitting the patch.</para>
             <para>Also, see <xref linkend="mockito"/>.</para>
+            <para>If you are creating a new unit test class, notice how other unit test classes have classification/sizing
+            annotations at the top and a static method on the end.  Be sure to include these in any new unit test files
+            you generate.  See <xref linkend="hbase.tests" /> for more on how the annotations work.
+            </para>
           </section>
           <section xml:id="submitting.patches.jira">
             <title>Attach Patch to Jira</title>
             <para>The patch should be attached to the associated Jira ticket "More Actions -&gt; Attach Files".  Make sure you click the
             ASF license inclusion, otherwise the patch can't be considered for inclusion.
             </para>
-            <para>Once attached to the ticket, click "Submit Patch" and 
+            <para>Once attached to the ticket, click "Submit Patch" and
             the status of the ticket will change.  Committers will review submitted patches for inclusion into the codebase.  Please
             understand that not every patch may get committed, and that feedback will likely be provided on the patch.  Fear not, though,
-            because the HBase community is helpful!
+            because the Apache HBase community is helpful!
             </para>
          </section>
-          
+
         <section xml:id="common.patch.feedback">
             <title>Common Patch Feedback</title>
           <para>The following items are representative of common patch feedback. Your patch process will go faster if these are
           taken into account <emphasis>before</emphasis> submission.
           </para>
           <para>
-          See the <link xlink:href="http://www.oracle.com/technetwork/java/codeconv-138413.html">Java coding standards</link> 
+          See the <link xlink:href="http://www.oracle.com/technetwork/java/codeconv-138413.html">Java coding standards</link>
           for more information on coding conventions in Java.
           </para>
           <section xml:id="common.patch.feedback.space.invaders">
@@ -567,7 +1019,7 @@ to fix the failing tests.
 <programlisting>
 if ( foo.equals( bar ) ) {     // don't do this
 </programlisting>
-			... do this instead...        
+			... do this instead...
 <programlisting>
 if (foo.equals(bar)) {
 </programlisting>
@@ -576,9 +1028,9 @@ if (foo.equals(bar)) {
 <programlisting>
 foo = barArray[ i ];     // don't do this
 </programlisting>
-			... do this instead...        
+			... do this instead...
 <programlisting>
-foo = barArray[i];   
+foo = barArray[i];
 </programlisting>
           </para>
           </section>
@@ -589,7 +1041,7 @@ foo = barArray[i];
  public void readFields(DataInput arg0) throws IOException {    // don't do this
    foo = arg0.readUTF();                                       // don't do this
 </programlisting>
-			... do this instead ...        
+			... do this instead ...
 <programlisting>
  public void readFields(DataInput di) throws IOException {
    foo = di.readUTF();
@@ -600,19 +1052,14 @@ foo = barArray[i];
           <section xml:id="common.patch.feedback.longlines">
             <title>Long Lines</title>
             <para>
-            Keep lines less than 80 characters.
-<programlisting>
-Bar bar = foo.veryLongMethodWithManyArguments(argument1, argument2, argument3, argument4, argument5);  // don't do this
-</programlisting>
-			... do this instead ...        
+            Keep lines less than 100 characters.
 <programlisting>
-Bar bar = foo.veryLongMethodWithManyArguments(argument1,
- argument2, argument3,argument4, argument5); 
+Bar bar = foo.veryLongMethodWithManyArguments(argument1, argument2, argument3, argument4, argument5, argument6, argument7, argument8, argument9);  // don't do this
 </programlisting>
-           ... or this, whichever looks better ...
+			... do something like this instead ...
 <programlisting>
 Bar bar = foo.veryLongMethodWithManyArguments(
- argument1, argument2, argument3,argument4, argument5); 
+ argument1, argument2, argument3,argument4, argument5, argument6, argument7, argument8, argument9);
 </programlisting>
            </para>
           </section>
@@ -624,11 +1071,17 @@ Bar bar = foo.veryLongMethodWithManyArguments(
 Bar bar = foo.getBar();     &lt;--- imagine there's an extra space(s) after the semicolon instead of a line break.
 </programlisting>
             Make sure there's a line-break after the end of your code, and also avoid lines that have nothing
-            but whitespace. 
+            but whitespace.
             </para>
-         </section> 
+         </section>
           <section xml:id="common.patch.feedback.writable">
             <title>Implementing Writable</title>
+            <note>
+                <title>Applies pre-0.96 only</title>
+                <para>In 0.96, HBase moved to protobufs.  The below section on Writables
+                    applies to 0.94.x and previous, not to 0.96 and beyond.
+                </para>
+            </note>
             <para>Every class returned by RegionServers must implement <code>Writable</code>.  If you
             are creating a new class that needs to implement this interface, don't forget the default constructor.
             </para>
@@ -636,39 +1089,60 @@ Bar bar = foo.getBar();     &lt;--- imagine there's an extra space(s) after the
           <section xml:id="common.patch.feedback.javadoc">
             <title>Javadoc</title>
             <para>This is also a very common feedback item.  Don't forget Javadoc!
+                <para>Javadoc warnings are checked during precommit. If the precommit tool gives you a '-1',
+                    please fix the javadoc issue. Your patch won't be committed if it adds such warnings.
+                </para>
+            </para>
+          </section>
+          <section xml:id="common.patch.feedback.findbugs">
+            <title>Findbugs</title>
+                <para>
+                    Findbugs is used to detect common bugs pattern. As Javadoc, it is checked during
+                    the precommit build up on Apache's Jenkins, and as with Javadoc, please fix them.
+                    You can run findbugs locally with 'mvn findbugs:findbugs': it will generate the
+                    findbugs files locally.  Sometimes, you may have to write code smarter than
+                    Findbugs. You can annotate your code to tell Findbugs you know what you're
+                    doing, by annotating your class with:
+                    <programlisting>@edu.umd.cs.findbugs.annotations.SuppressWarnings(
+                    value="HE_EQUALS_USE_HASHCODE",
+                    justification="I know what I'm doing")</programlisting>
+            </para>
+            <para>
+                    Note that we're using the apache licensed version of the annotations.
             </para>
           </section>
+
           <section xml:id="common.patch.feedback.javadoc.defaults">
             <title>Javadoc - Useless Defaults</title>
             <para>Don't just leave the @param arguments the way your IDE generated them.  Don't do this...
 <programlisting>
   /**
-   * 
+   *
    * @param bar             &lt;---- don't do this!!!!
    * @return                &lt;---- or this!!!!
    */
   public Foo getFoo(Bar bar);
-</programlisting> 
-            ... either add something descriptive to the @param and @return lines, or just remove them. 
-            But the preference is to add something descriptive and useful.          
+</programlisting>
+            ... either add something descriptive to the @param and @return lines, or just remove them.
+            But the preference is to add something descriptive and useful.
             </para>
           </section>
           <section xml:id="common.patch.feedback.onething">
             <title>One Thing At A Time, Folks</title>
             <para>If you submit a patch for one thing, don't do auto-reformatting or unrelated reformatting of code on a completely
-            different area of code. 
+            different area of code.
             </para>
-            <para>Likewise, don't add unrelated cleanup or refactorings outside the scope of your Jira. 
+            <para>Likewise, don't add unrelated cleanup or refactorings outside the scope of your Jira.
             </para>
           </section>
           <section xml:id="common.patch.feedback.tests">
             <title>Ambigious Unit Tests</title>
-            <para>Make sure that you're clear about what you are testing in your unit tests and why. 
+            <para>Make sure that you're clear about what you are testing in your unit tests and why.
             </para>
           </section>
 
         </section>   <!--  patch feedback -->
- 
+
          <section xml:id="reviewboard">
             <title>ReviewBoard</title>
           <para>Larger patches should go through <link xlink:href="http://reviews.apache.org">ReviewBoard</link>.
@@ -676,16 +1150,29 @@ Bar bar = foo.getBar();     &lt;--- imagine there's an extra space(s) after the
           <para>For more information on how to use ReviewBoard, see
            <link xlink:href="http://www.reviewboard.org/docs/manual/1.5/">the ReviewBoard documentation</link>.
           </para>
-        </section> 
+        </section>
         <section xml:id="committing.patches">
           <title>Committing Patches</title>
           <para>
-          Committers do this.  See <link xlink:href="http://wiki.apache.org/hadoop/Hbase/HowToCommit">How To Commit</link> in the HBase wiki.
+          Committers do this.  See <link xlink:href="http://wiki.apache.org/hadoop/Hbase/HowToCommit">How To Commit</link> in the Apache HBase wiki.
           </para>
           <para>Commiters will also resolve the Jira, typically after the patch passes a build.
           </para>
+          <section xml:id="committer.tests">
+              <title>Committers are responsible for making sure commits do not break the build or tests</title>
+          <para>
+              If a committer commits a patch it is their responsibility
+              to make sure it passes the test suite.  It is helpful
+              if contributors keep an eye out that their patch
+              does not break the hbase build and/or tests but ultimately,
+              a contributor cannot be expected to be up on the
+              particular vagaries and interconnections that occur
+              in a project like hbase.  A committer should.
+            </para>
+        </section>
         </section>
 
        </section>   <!--  submitting patches -->
-    
+
+
     </chapter>
diff --git a/src/docbkx/external_apis.xml b/src/docbkx/external_apis.xml
index 155a964862f3..6380b6e7b801 100644
--- a/src/docbkx/external_apis.xml
+++ b/src/docbkx/external_apis.xml
@@ -26,31 +26,34 @@
  * limitations under the License.
  */
 -->
-  <title>External APIs</title>
-  This chapter will cover access to HBase either through non-Java languages, or through custom protocols.
-  
+  <title>Apache HBase (TM) External APIs</title>
+  This chapter will cover access to Apache HBase (TM) either through non-Java languages, or through custom protocols.
+
   <section xml:id="nonjava.jvm">
     <title>Non-Java Languages Talking to the JVM</title>
-    <para>Currently the documentation on this topic in the 
-      <link xlink:href="http://wiki.apache.org/hadoop/Hbase">HBase Wiki</link>.
+    <para>Currently the documentation on this topic in the
+      <link xlink:href="http://wiki.apache.org/hadoop/Hbase">Apache HBase Wiki</link>.
+      See also the <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/thrift/package-summary.html#package_description">Thrift API Javadoc</link>.
     </para>
   </section>
 
   <section xml:id="rest">
     <title>REST</title>
-    <para>Currently most of the documentation on REST exists in the 
-      <link xlink:href="http://wiki.apache.org/hadoop/Hbase/Stargate">HBase Wiki on REST</link>.
+    <para>Currently most of the documentation on REST exists in the
+        <link xlink:href="http://wiki.apache.org/hadoop/Hbase/Stargate">Apache HBase Wiki on REST</link> (The REST gateway used to be
+        called 'Stargate').  There are also a nice set of blogs on <link xlink:href="http://blog.cloudera.com/blog/2013/03/how-to-use-the-apache-hbase-rest-interface-part-1/">How-to: Use the Apache HBase REST Interface</link>
+        by Jesse Anderson.
     </para>
   </section>   <!-- rest -->
 
   <section xml:id="thrift">
     <title>Thrift</title>
-      <para>Currently most of the documentation on Thrift exists in the 
-      <link xlink:href="http://wiki.apache.org/hadoop/Hbase/ThriftApi">HBase Wiki on Thrift</link>.
+      <para>Currently most of the documentation on Thrift exists in the
+      <link xlink:href="http://wiki.apache.org/hadoop/Hbase/ThriftApi">Apache HBase Wiki on Thrift</link>.
     </para>
           <section xml:id="thrift.filter-language"><title>Filter Language</title>
              <section xml:id="use-case"><title>Use Case</title>
-               <para>Note:  this feature was introduced in HBase 0.92</para>
+               <para>Note:  this feature was introduced in Apache HBase 0.92</para>
                <para>This allows the user to perform server-side filtering when accessing HBase over Thrift. The user specifies a filter via a string. The string is parsed on the server to construct the filter</para>
              </section>
 
@@ -407,10 +410,15 @@
 
           </orderedlist>
         </section>
-        
+
         </section>
-    
+
   </section>  <!-- thrift -->
 
+  <section xml:id="c">
+    <title>C/C++ Apache HBase Client</title>
+    <para>FB's Chip Turner wrote a pure C/C++ client.  <link xlink:href="https://github.com/facebook/native-cpp-hbase-client">Check it out</link>.
+    </para>
+  </section>
 
 </chapter>
diff --git a/src/docbkx/getting_started.xml b/src/docbkx/getting_started.xml
index 3aa392b810bf..e1c4344ef074 100644
--- a/src/docbkx/getting_started.xml
+++ b/src/docbkx/getting_started.xml
@@ -32,9 +32,8 @@
     <title>Introduction</title>
 
     <para><xref linkend="quickstart" /> will get you up and
-    running on a single-node instance of HBase using the local filesystem. 
-    <xref linkend="configuration" /> describes setup
-    of HBase in distributed mode running on top of HDFS.</para>
+    running on a single-node instance of HBase using the local filesystem.
+    </para>
   </section>
 
   <section xml:id="quickstart">
@@ -45,17 +44,31 @@
     rows via the HBase <command>shell</command>, and then cleaning
     up and shutting down your standalone HBase instance. The below exercise
     should take no more than ten minutes (not including download time).</para>
+    <para>Before we proceed, make sure you are good on the below loopback prerequisite.</para>
+    <note xml:id="loopback.ip.getting.started">
+        <title>Loopback IP</title>
+        <para>HBase expects the loopback IP address to be 127.0.0.1.  Ubuntu and some other distributions,
+        for example, will default to 127.0.1.1 and this will cause problems for you.
+        </para>
+        <para><filename>/etc/hosts</filename> should look something like this:
+<programlisting>
+            127.0.0.1 localhost
+            127.0.0.1 ubuntu.ubuntu-domain ubuntu
+</programlisting>
+        </para>
+       </note>
+
 
     <section>
       <title>Download and unpack the latest stable release.</title>
 
       <para>Choose a download site from this list of <link
       xlink:href="http://www.apache.org/dyn/closer.cgi/hbase/">Apache Download
-      Mirrors</link>. Click on suggested top link. This will take you to a
+      Mirrors</link>. Click on the suggested top link. This will take you to a
       mirror of <emphasis>HBase Releases</emphasis>. Click on the folder named
       <filename>stable</filename> and then download the file that ends in
       <filename>.tar.gz</filename> to your local filesystem; e.g.
-      <filename>hbase-<?eval ${project.version}?>.tar.gz</filename>.</para>
+      <filename>hbase-0.94.2.tar.gz</filename>.</para>
 
       <para>Decompress and untar your download and then change into the
       unpacked directory.</para>
@@ -65,24 +78,27 @@ $ cd hbase-<?eval ${project.version}?>
 </programlisting></para>
 
       <para>At this point, you are ready to start HBase. But before starting
-      it, you might want to edit <filename>conf/hbase-site.xml</filename> and
-      set the directory you want HBase to write to,
-      <varname>hbase.rootdir</varname>. <programlisting>
-
-&lt;?xml version="1.0"?&gt;
+      it, edit <filename>conf/hbase-site.xml</filename>, the file you write
+      your site-specific configurations into. Set
+      <varname>hbase.rootdir</varname>, the directory HBase writes data to,
+      and <varname>hbase.zookeeper.property.dataDir</varname>, the director
+      ZooKeeper writes its data too:
+<programlisting>&lt;?xml version="1.0"?&gt;
 &lt;?xml-stylesheet type="text/xsl" href="configuration.xsl"?&gt;
 &lt;configuration&gt;
   &lt;property&gt;
     &lt;name&gt;hbase.rootdir&lt;/name&gt;
     &lt;value&gt;file:///DIRECTORY/hbase&lt;/value&gt;
   &lt;/property&gt;
-&lt;/configuration&gt;
-
-</programlisting> Replace <varname>DIRECTORY</varname> in the above with a
-      path to a directory where you want HBase to store its data. By default,
-      <varname>hbase.rootdir</varname> is set to
-      <filename>/tmp/hbase-${user.name}</filename> which means you'll lose all
-      your data whenever your server reboots (Most operating systems clear
+  &lt;property&gt;
+    &lt;name&gt;hbase.zookeeper.property.dataDir&lt;/name&gt;
+    &lt;value&gt;/DIRECTORY/zookeeper&lt;/value&gt;
+  &lt;/property&gt;
+&lt;/configuration&gt;</programlisting> Replace <varname>DIRECTORY</varname> in the above with the
+      path to the directory you would have HBase and ZooKeeper write their data. By default,
+      <varname>hbase.rootdir</varname> is set to <filename>/tmp/hbase-${user.name}</filename>
+      and similarly so for the default ZooKeeper data location which means you'll lose all
+      your data whenever your server reboots unless you change it (Most operating systems clear
       <filename>/tmp</filename> on restart).</para>
     </section>
 
@@ -96,19 +112,19 @@ starting Master, logging to logs/hbase-user-master-example.org.out</programlisti
       standalone mode, HBase runs all daemons in the the one JVM; i.e. both
       the HBase and ZooKeeper daemons. HBase logs can be found in the
       <filename>logs</filename> subdirectory. Check them out especially if
-      HBase had trouble starting.</para>
+      it seems HBase had trouble starting.</para>
 
       <note>
         <title>Is <application>java</application> installed?</title>
 
         <para>All of the above presumes a 1.6 version of Oracle
         <application>java</application> is installed on your machine and
-        available on your path; i.e. when you type
+        available on your path (See <xref linkend="java" />); i.e. when you type
         <application>java</application>, you see output that describes the
         options the java program takes (HBase requires java 6). If this is not
         the case, HBase will not start. Install java, edit
         <filename>conf/hbase-env.sh</filename>, uncommenting the
-        <envar>JAVA_HOME</envar> line pointing it to your java install. Then,
+        <envar>JAVA_HOME</envar> line pointing it to your java install, then,
         retry the steps above.</para>
       </note>
     </section>
@@ -154,9 +170,7 @@ hbase(main):006:0&gt; put 'test', 'row3', 'cf:c', 'value3'
       <varname>cf</varname> in this example -- followed by a colon and then a
       column qualifier suffix (<varname>a</varname> in this case).</para>
 
-      <para>Verify the data insert.</para>
-
-      <para>Run a scan of the table by doing the following</para>
+      <para>Verify the data insert by running a scan of the table as follows</para>
 
       <para><programlisting>hbase(main):007:0&gt; scan 'test'
 ROW        COLUMN+CELL
@@ -165,7 +179,7 @@ row2       column=cf:b, timestamp=1288380738440, value=value2
 row3       column=cf:c, timestamp=1288380747365, value=value3
 3 row(s) in 0.0590 seconds</programlisting></para>
 
-      <para>Get a single row as follows</para>
+      <para>Get a single row</para>
 
       <para><programlisting>hbase(main):008:0&gt; get 'test', 'row1'
 COLUMN      CELL
@@ -198,9 +212,9 @@ stopping hbase...............</programlisting></para>
       <title>Where to go next</title>
 
       <para>The above described standalone setup is good for testing and
-          experiments only. Next move on to <xref linkend="configuration" /> where we'll go into
-      depth on the different HBase run modes, requirements and critical
-      configurations needed setting up a distributed HBase deploy.</para>
+          experiments only. In the next chapter, <xref linkend="configuration" />,
+      we'll go into depth on the different HBase run modes, system requirements
+      running HBase, and critical configurations setting up a distributed HBase deploy.</para>
     </section>
   </section>
 
diff --git a/src/docbkx/ops_mgt.xml b/src/docbkx/ops_mgt.xml
index 2797049000f4..a1bde700aabe 100644
--- a/src/docbkx/ops_mgt.xml
+++ b/src/docbkx/ops_mgt.xml
@@ -26,16 +26,35 @@
  * limitations under the License.
  */
 -->
-  <title>HBase Operational Management</title>
-  This chapter will cover operational tools and practices required of a running HBase cluster.
+  <title>Apache HBase (TM) Operational Management</title>
+  This chapter will cover operational tools and practices required of a running Apache HBase cluster.
   The subject of operations is related to the topics of <xref linkend="trouble" />, <xref linkend="performance"/>,
-  and <xref linkend="configuration" /> but is a distinct topic in itself.  
-  
+  and <xref linkend="configuration" /> but is a distinct topic in itself.
+
   <section xml:id="tools">
     <title >HBase Tools and Utilities</title>
 
     <para>Here we list HBase tools for administration, analysis, fixup, and
     debugging.</para>
+    <section xml:id="driver"><title>Driver</title>
+      <para>There is a <code>Driver</code> class that is executed by the HBase jar can be used to invoke frequently accessed utilities.  For example,
+<programlisting>HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-VERSION.jar
+</programlisting>
+... will return...
+<programlisting>
+An example program must be given as the first argument.
+Valid program names are:
+  completebulkload: Complete a bulk data load.
+  copytable: Export a table from local cluster to peer cluster
+  export: Write table data to HDFS.
+  import: Import data written by Export.
+  importtsv: Import data in TSV format.
+  rowcounter: Count rows in HBase table
+  verifyrep: Compare the data from tables in two different clusters. WARNING: It doesn't work for incrementColumnValues'd cells since the timestamp is chan
+</programlisting>
+... for allowable program names.
+      </para>
+    </section>
     <section xml:id="hbck">
         <title>HBase <application>hbck</application></title>
         <subtitle>An <emphasis>fsck</emphasis> for your HBase install</subtitle>
@@ -50,6 +69,8 @@
         Passing <command>-fix</command> may correct the inconsistency (This latter
         is an experimental feature).
         </para>
+        <para>For more information, see <xref linkend="hbck.in.depth"/>.
+        </para>
     </section>
     <section xml:id="hfile_tool2"><title>HFile Tool</title>
         <para>See <xref linkend="hfile_tool" />.</para>
@@ -72,10 +93,17 @@
 
         <para>Similarly you can force a split of a log file directory by
         doing:<programlisting> $ ./<code>bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLog --split hdfs://example.org:8020/hbase/.logs/example.org,60020,1283516293161/</code></programlisting></para>
+
+        <section xml:id="hlog_tool.prettyprint">
+          <title><classname>HLogPrettyPrinter</classname></title>
+          <para><classname>HLogPrettyPrinter</classname> is a tool with configurable options to print the contents of an HLog.
+          </para>
+        </section>
+
       </section>
     </section>
     <section xml:id="compression.tool"><title>Compression Tool</title>
-        <para>See <xref linkend="compression.tool" />.</para>
+        <para>See <xref linkend="compression.test" />.</para>
     </section>
         <section xml:id="copytable">
         <title>CopyTable</title>
@@ -105,7 +133,12 @@
 --starttime=1265875194289 --endtime=1265878794289
 --peer.adr=server1,server2,server3:2181:/hbase TestTable</programlisting>
         </para>
-        <para>Note:  caching for the input Scan is configured via <code>hbase.client.scanner.caching</code> in the job configuration.
+        <note><title>Scanner Caching</title>
+        <para>Caching for the input Scan is configured via <code>hbase.client.scanner.caching</code> in the job configuration.
+        </para>
+        </note>
+        <para>
+        See Jonathan Hsieh's <link xlink:href="http://www.cloudera.com/blog/2012/06/online-hbase-backups-with-copytable-2/">Online HBase Backups with CopyTable</link> blog post for more on <command>CopyTable</command>.
         </para>
     </section>
     <section xml:id="export">
@@ -124,13 +157,110 @@
 </programlisting>
        </para>
     </section>
+    <section xml:id="importtsv">
+       <title>ImportTsv</title>
+       <para>ImportTsv is a utility that will load data in TSV format into HBase.  It has two distinct usages:  loading data from TSV format in HDFS
+       into HBase via Puts, and preparing StoreFiles to be loaded via the <code>completebulkload</code>.
+       </para>
+       <para>To load data via Puts (i.e., non-bulk loading):
+<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.columns=a,b,c &lt;tablename&gt; &lt;hdfs-inputdir&gt;
+</programlisting>
+       </para>
+       <para>To generate StoreFiles for bulk-loading:
+<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.columns=a,b,c -Dimporttsv.bulk.output=hdfs://storefile-outputdir &lt;tablename&gt; &lt;hdfs-data-inputdir&gt;
+</programlisting>
+       </para>
+       <para>These generated StoreFiles can be loaded into HBase via <xref linkend="completebulkload"/>.
+       </para>
+       <section xml:id="importtsv.options"><title>ImportTsv Options</title>
+       Running ImportTsv with no arguments prints brief usage information:
+<programlisting>
+Usage: importtsv -Dimporttsv.columns=a,b,c &lt;tablename&gt; &lt;inputdir&gt;
+
+Imports the given input directory of TSV data into the specified table.
+
+The column names of the TSV data must be specified using the -Dimporttsv.columns
+option. This option takes the form of comma-separated column names, where each
+column name is either a simple column family, or a columnfamily:qualifier. The special
+column name HBASE_ROW_KEY is used to designate that this column should be used
+as the row key for each imported record. You must specify exactly one column
+to be the row key, and you must specify a column name for every column that exists in the
+input data.
+
+By default importtsv will load data directly into HBase. To instead generate
+HFiles of data to prepare for a bulk data load, pass the option:
+  -Dimporttsv.bulk.output=/path/for/output
+  Note: the target table will be created with default column family descriptors if it does not already exist.
+
+Other options that may be specified with -D include:
+  -Dimporttsv.skip.bad.lines=false - fail if encountering an invalid line
+  '-Dimporttsv.separator=|' - eg separate on pipes instead of tabs
+  -Dimporttsv.timestamp=currentTimeAsLong - use the specified timestamp for the import
+  -Dimporttsv.mapper.class=my.Mapper - A user-defined Mapper to use instead of org.apache.hadoop.hbase.mapreduce.TsvImporterMapper
+</programlisting>
+       </section>
+       <section xml:id="importtsv.example"><title>ImportTsv Example</title>
+         <para>For example, assume that we are loading data into a table called 'datatsv' with a ColumnFamily called 'd' with two columns "c1" and "c2".
+         </para>
+         <para>Assume that an input file exists as follows:
+<programlisting>
+row1	c1	c2
+row2	c1	c2
+row3	c1	c2
+row4	c1	c2
+row5	c1	c2
+row6	c1	c2
+row7	c1	c2
+row8	c1	c2
+row9	c1	c2
+row10	c1	c2
+</programlisting>
+         </para>
+         <para>For ImportTsv to use this imput file, the command line needs to look like this:
+ <programlisting>
+ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-VERSION.jar importtsv -Dimporttsv.columns=HBASE_ROW_KEY,d:c1,d:c2 -Dimporttsv.bulk.output=hdfs://storefileoutput datatsv hdfs://inputfile
+ </programlisting>
+         ... and in this example the first column is the rowkey, which is why the HBASE_ROW_KEY is used.  The second and third columns in the file will be imported as "d:c1" and "d:c2", respectively.
+         </para>
+       </section>
+       <section xml:id="importtsv.warning"><title>ImportTsv Warning</title>
+         <para>If you have preparing a lot of data for bulk loading, make sure the target HBase table is pre-split appropriately.
+         </para>
+       </section>
+       <section xml:id="importtsv.also"><title>See Also</title>
+       For more information about bulk-loading HFiles into HBase, see <xref linkend="arch.bulk.load"/>
+       </section>
+    </section>
+
+    <section xml:id="completebulkload">
+       <title>CompleteBulkLoad</title>
+	   <para>The <code>completebulkload</code> utility will move generated StoreFiles into an HBase table.  This utility is often used
+	   in conjunction with output from <xref linkend="importtsv"/>.
+	   </para>
+	   <para>There are two ways to invoke this utility, with explicit classname and via the driver:
+<programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles &lt;hdfs://storefileoutput&gt; &lt;tablename&gt;
+</programlisting>
+.. and via the Driver..
+<programlisting>HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-VERSION.jar completebulkload &lt;hdfs://storefileoutput&gt; &lt;tablename&gt;
+</programlisting>
+	  </para>
+          <section xml:id="completebulkload.warning"><title>CompleteBulkLoad Warning</title>
+          <para>Data generated via MapReduce is often created with file permissions that are not compatible with the running HBase process. Assuming you're running HDFS with permissions enabled, those permissions will need to be updated before you run CompleteBulkLoad.
+          </para>
+          </section>
+       <para>For more information about bulk-loading HFiles into HBase, see <xref linkend="arch.bulk.load"/>.
+       </para>
+    </section>
     <section xml:id="walplayer">
        <title>WALPlayer</title>
        <para>WALPlayer is a utility to replay WAL files into HBase.
        </para>
-       <para>The WAL can be replayed for a set of tables or all tables, and a timerange can be provided (in milliseconds). The WAL is filtered to this set of tables. The output can optionally be mapped to another set of tables.
+       <para>The WAL can be replayed for a set of tables or all tables, and a
+           timerange can be provided (in milliseconds). The WAL is filtered to
+           this set of tables. The output can optionally be mapped to another set of tables.
        </para>
-       <para>WALPlayer can also generate HFiles for later bulk importing, in that case only a single table and no mapping can be specified.
+       <para>WALPlayer can also generate HFiles for later bulk importing, in that case
+           only a single table and no mapping can be specified.
        </para>
        <para>Invoke via:
 <programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.WALPlayer [options] &lt;wal inputdir&gt; &lt;tables&gt; [&lt;tableMappings>]&gt;
@@ -140,18 +270,43 @@
 <programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.WALPlayer /backuplogdir oldTable1,oldTable2 newTable1,newTable2
 </programlisting>
        </para>
+       <para>
+           WALPlayer, by default, runs as a mapreduce job.  To NOT run WALPlayer as a mapreduce job on your cluster,
+           force it to run all in the local process by adding the flags <code>-Dmapred.job.tracker=local</code> on the command line.
+       </para>
     </section>
     <section xml:id="rowcounter">
-       <title>RowCounter</title>
-       <para>RowCounter is a utility that will count all the rows of a table.  This is a good utility to use
-       as a sanity check to ensure that HBase can read all the blocks of a table if there are any concerns of metadata inconsistency.
+       <title>RowCounter and CellCounter</title>
+       <para><ulink url="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/RowCounter.html">RowCounter</ulink> is a
+       mapreduce job to count all the rows of a table.  This is a good utility to use as a sanity check to ensure that HBase can read
+       all the blocks of a table if there are any concerns of metadata inconsistency. It will run the mapreduce all in a single
+       process but it will run faster if you have a MapReduce cluster in place for it to exploit.
 <programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.RowCounter &lt;tablename&gt; [&lt;column1&gt; &lt;column2&gt;...]
 </programlisting>
        </para>
-       <para>Note:  caching for the input Scan is configured via <code>hbase.client.scanner.caching</code> in the job configuration.
+       <para>Note: caching for the input Scan is configured via <code>hbase.client.scanner.caching</code> in the job configuration.
+       </para>
+       <para>HBase ships another diagnostic mapreduce job called
+         <ulink url="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/CellCounter.html">CellCounter</ulink>. Like
+         RowCounter, it gathers more fine-grained statistics about your table. The statistics gathered by RowCounter are more fine-grained
+         and include:
+         <itemizedlist>
+           <listitem>Total number of rows in the table.</listitem>
+           <listitem>Total number of CFs across all rows.</listitem>
+           <listitem>Total qualifiers across all rows.</listitem>
+           <listitem>Total occurrence of each CF.</listitem>
+           <listitem>Total occurrence of each qualifier.</listitem>
+           <listitem>Total number of versions of each qualifier.</listitem>
+         </itemizedlist>
+       </para>
+       <para>The program allows you to limit the scope of the run. Provide a row regex or prefix to limit the rows to analyze. Use
+         <code>hbase.mapreduce.scan.column.family</code> to specify scanning a single column family.
+         <programlisting>$ bin/hbase org.apache.hadoop.hbase.mapreduce.CellCounter &lt;tablename&gt; &lt;outputDir&gt; [regex or prefix]</programlisting>
        </para>
+       <para>Note: just like RowCounter, caching for the input Scan is configured via <code>hbase.client.scanner.caching</code> in the
+       job configuration. </para>
     </section>
-           
+
     </section>  <!--  tools -->
 
   <section xml:id="ops.regionmgt">
@@ -161,7 +316,7 @@
       <para>Major compactions can be requested via the HBase shell or <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HBaseAdmin.html#majorCompact%28java.lang.String%29">HBaseAdmin.majorCompact</link>.
       </para>
       <para>Note:  major compactions do NOT do region merges.  See <xref linkend="compaction"/> for more information about compactions.
-      
+
       </para>
     </section>
     <section xml:id="ops.regionmgt.merge">
@@ -170,16 +325,16 @@
 <programlisting>$ bin/hbase org.apache.hbase.util.Merge &lt;tablename&gt; &lt;region1&gt; &lt;region2&gt;
 </programlisting>
       <para>If you feel you have too many regions and want to consolidate them, Merge is the utility you need.  Merge must
-      run be done when the cluster is down.  
+      run be done when the cluster is down.
       See the <link xlink:href="http://ofps.oreilly.com/titles/9781449396107/performance.html">O'Reilly HBase Book</link> for
       an example of usage.
       </para>
-      <para>Additionally, there is a Ruby script attached to <link xlink:href="https://issues.apache.org/jira/browse/HBASE-1621">HBASE-1621</link> 
+      <para>Additionally, there is a Ruby script attached to <link xlink:href="https://issues.apache.org/jira/browse/HBASE-1621">HBASE-1621</link>
       for region merging.
       </para>
     </section>
   </section>
-    
+
     <section xml:id="node.management"><title>Node Management</title>
      <section xml:id="decommission"><title>Node Decommission</title>
         <para>You can stop an individual RegionServer by running the following
@@ -202,10 +357,10 @@
         A downside to the above stop of a RegionServer is that regions could be offline for
         a good period of time.  Regions are closed in order.  If many regions on the server, the
         first region to close may not be back online until all regions close and after the master
-        notices the RegionServer's znode gone.  In HBase 0.90.2, we added facility for having
-        a node gradually shed its load and then shutdown itself down.  HBase 0.90.2 added the
+        notices the RegionServer's znode gone.  In Apache HBase 0.90.2, we added facility for having
+        a node gradually shed its load and then shutdown itself down. Apache HBase 0.90.2 added the
             <filename>graceful_stop.sh</filename> script.  Here is its usage:
-            <programlisting>$ ./bin/graceful_stop.sh 
+            <programlisting>$ ./bin/graceful_stop.sh
 Usage: graceful_stop.sh [--config &amp;conf-dir>] [--restart] [--reload] [--thrift] [--rest] &amp;hostname>
  thrift      If we should stop/start thrift before/after the hbase stop/start
  rest        If we should stop/start rest before/after the hbase stop/start
@@ -218,7 +373,7 @@ Usage: graceful_stop.sh [--config &amp;conf-dir>] [--restart] [--reload] [--thri
             To decommission a loaded RegionServer, run the following:
             <programlisting>$ ./bin/graceful_stop.sh HOSTNAME</programlisting>
             where <varname>HOSTNAME</varname> is the host carrying the RegionServer
-            you would decommission.  
+            you would decommission.
             <note><title>On <varname>HOSTNAME</varname></title>
                 <para>The <varname>HOSTNAME</varname> passed to <filename>graceful_stop.sh</filename>
             must match the hostname that hbase is using to identify RegionServers.
@@ -240,7 +395,7 @@ Usage: graceful_stop.sh [--config &amp;conf-dir>] [--restart] [--reload] [--thri
             and because the RegionServer went down cleanly, there will be no
             WAL logs to split.
             <note xml:id="lb"><title>Load Balancer</title>
-            <para> 
+            <para>
                 It is assumed that the Region Load Balancer is disabled while the
                 <command>graceful_stop</command> script runs (otherwise the balancer
                 and the decommission script will end up fighting over region deployments).
@@ -252,10 +407,31 @@ This turns the balancer OFF.  To reenable, do:
                 <programlisting>hbase(main):001:0> balance_switch true
 false
 0 row(s) in 0.3590 seconds</programlisting>
-            </para> 
+            </para>
         </note>
         </para>
-        </section>  
+        <section xml:id="bad.disk">
+            <title>Bad or Failing Disk</title>
+            <para>It is good having <xref linkend="dfs.datanode.failed.volumes.tolerated" /> set if you have a decent number of disks
+            per machine for the case where a disk plain dies.  But usually disks do the "John Wayne" -- i.e. take a while
+            to go down spewing errors in <filename>dmesg</filename> -- or for some reason, run much slower than their
+            companions.  In this case you want to decommission the disk.  You have two options.  You can
+            <xlink href="http://wiki.apache.org/hadoop/FAQ#I_want_to_make_a_large_cluster_smaller_by_taking_out_a_bunch_of_nodes_simultaneously._How_can_this_be_done.3F">decommission the datanode</xlink>
+            or, less disruptive in that only the bad disks data will be rereplicated, can stop the datanode,
+            unmount the bad volume (You can't umount a volume while the datanode is using it), and then restart the
+            datanode (presuming you have set dfs.datanode.failed.volumes.tolerated > 0).  The regionserver will
+            throw some errors in its logs as it recalibrates where to get its data from -- it will likely
+            roll its WAL log too -- but in general but for some latency spikes, it should keep on chugging.
+            <note>
+                <para>If you are doing short-circuit reads, you will have to move the regions off the regionserver
+                    before you stop the datanode; when short-circuiting reading, though chmod'd so regionserver cannot
+                    have access, because it already has the files open, it will be able to keep reading the file blocks
+                    from the bad disk even though the datanode is down.  Move the regions back after you restart the
+                datanode.</para>
+            </note>
+            </para>
+        </section>
+        </section>
         <section xml:id="rolling">
             <title>Rolling Restart</title>
         <para>
@@ -313,7 +489,7 @@ false
     </section>  <!--  node mgt -->
 
   <section xml:id="hbase_metrics">
-  <title>Metrics</title>
+  <title>HBase Metrics</title>
   <section xml:id="metric_setup">
   <title>Metric Setup</title>
   <para>See <link xlink:href="http://hbase.apache.org/metrics.html">Metrics</link> for
@@ -394,8 +570,37 @@ false
 
   <section xml:id="ops.monitoring">
     <title >HBase Monitoring</title>
-    <para>TODO
-    </para>
+    <section xml:id="ops.monitoring.overview">
+    <title>Overview</title>
+      <para>The following metrics are arguably the most important to monitor for each RegionServer for
+      "macro monitoring", preferably with a system like <link xlink:href="http://opentsdb.net/">OpenTSDB</link>.
+      If your cluster is having performance issues it's likely that you'll see something unusual with
+      this group.
+      </para>
+      <para>HBase:
+      <itemizedlist>
+      <listitem>Requests</listitem>
+      <listitem>Compactions queue</listitem>
+      </itemizedlist>
+      </para>
+      <para>OS:
+      <itemizedlist>
+      <listitem>IO Wait</listitem>
+      <listitem>User CPU</listitem>
+      </itemizedlist>
+      </para>
+      <para>Java:
+      <itemizedlist>
+      <listitem>GC</listitem>
+      </itemizedlist>
+      </para>
+      <para>
+      </para>
+      <para>
+      For more information on HBase metrics, see <xref linkend="hbase_metrics"/>.
+      </para>
+    </section>
+
     <section xml:id="ops.slow.query">
     <title>Slow Query Log</title>
 <para>The HBase slow query log consists of parseable JSON structures describing the properties of those client operations (Gets, Puts, Deletes, etc.) that either took too long to run, or produced too much output. The thresholds for "too long to run" and "too much output" are configurable, as described below. The output is produced inline in the main region server logs so that it is easy to discover further details from context with other logged events. It is also prepended with identifying tags <constant>(responseTooSlow)</constant>, <constant>(responseTooLarge)</constant>, <constant>(operationTooSlow)</constant>, and <constant>(operationTooLarge)</constant> in order to enable easy filtering with grep, in case the user desires to see only slow queries.
@@ -442,7 +647,7 @@ false
 
 
   </section>
-  
+
   <section xml:id="cluster_replication">
     <title>Cluster Replication</title>
     <para>See <link xlink:href="http://hbase.apache.org/replication.html">Cluster Replication</link>.
@@ -450,8 +655,8 @@ false
   </section>
   <section xml:id="ops.backup">
     <title >HBase Backup</title>
-    <para>There are two broad strategies for performing HBase backups: backing up with a full cluster shutdown, and backing up on a live cluster. 
-    Each approach has pros and cons.   
+    <para>There are two broad strategies for performing HBase backups: backing up with a full cluster shutdown, and backing up on a live cluster.
+    Each approach has pros and cons.
     </para>
     <para>For additional information, see <link xlink:href="http://blog.sematext.com/2011/03/11/hbase-backup-options/">HBase Backup Options</link> over on the Sematext Blog.
     </para>
@@ -465,27 +670,27 @@ false
         </para>
       </section>
       <section xml:id="ops.backup.fullshutdown.distcp"><title>Distcp</title>
-        <para>Distcp could be used to either copy the contents of the HBase directory in HDFS to either the same cluster in another directory, or 
+        <para>Distcp could be used to either copy the contents of the HBase directory in HDFS to either the same cluster in another directory, or
         to a different cluster.
         </para>
-        <para>Note:  Distcp works in this situation because the cluster is down and there are no in-flight edits to files.  
+        <para>Note:  Distcp works in this situation because the cluster is down and there are no in-flight edits to files.
         Distcp-ing of files in the HBase directory is not generally recommended on a live cluster.
         </para>
       </section>
       <section xml:id="ops.backup.fullshutdown.restore"><title>Restore (if needed)</title>
-        <para>The backup of the hbase directory from HDFS is copied onto the 'real' hbase directory via distcp.  The act of copying these files 
+        <para>The backup of the hbase directory from HDFS is copied onto the 'real' hbase directory via distcp.  The act of copying these files
         creates new HDFS metadata, which is why a restore of the NameNode edits from the time of the HBase backup isn't required for this kind of
         restore, because it's a restore (via distcp) of a specific HDFS directory (i.e., the HBase part) not the entire HDFS file-system.
         </para>
       </section>
     </section>
     <section xml:id="ops.backup.live.replication"><title>Live Cluster Backup - Replication</title>
-      <para>This approach assumes that there is a second cluster.  
+      <para>This approach assumes that there is a second cluster.
       See the HBase page on <link xlink:href="http://hbase.apache.org/replication.html">replication</link> for more information.
       </para>
     </section>
     <section xml:id="ops.backup.live.copytable"><title>Live Cluster Backup - CopyTable</title>
-      <para>The <xref linkend="copytable" /> utility could either be used to copy data from one table to another on the 
+      <para>The <xref linkend="copytable" /> utility could either be used to copy data from one table to another on the
       same cluster, or to copy data to another table on another cluster.
       </para>
       <para>Since the cluster is up, there is a risk that edits could be missed in the copy process.
@@ -506,10 +711,10 @@ false
       with a solid understanding of how HBase handles data internally (KeyValue).
       </para>
       <section xml:id="ops.capacity.storage.kv"><title>KeyValue</title>
-        <para>HBase storage will be dominated by KeyValues.  See <xref linkend="keyvalue" /> and <xref linkend="keysize" /> for 
-        how HBase stores data internally.  
+        <para>HBase storage will be dominated by KeyValues.  See <xref linkend="keyvalue" /> and <xref linkend="keysize" /> for
+        how HBase stores data internally.
         </para>
-        <para>It is critical to understand that there is a KeyValue instance for every attribute stored in a row, and the 
+        <para>It is critical to understand that there is a KeyValue instance for every attribute stored in a row, and the
         rowkey-length, ColumnFamily name-length and attribute lengths will drive the size of the database more than any other
         factor.
         </para>
diff --git a/src/docbkx/performance.xml b/src/docbkx/performance.xml
index 3ae843232698..b65dab02fc01 100644
--- a/src/docbkx/performance.xml
+++ b/src/docbkx/performance.xml
@@ -26,7 +26,7 @@
  * limitations under the License.
  */
 -->
-  <title>Performance Tuning</title>
+  <title>Apache HBase (TM) Performance Tuning</title>
 
   <section xml:id="perf.os">
     <title>Operating System</title>
@@ -47,7 +47,7 @@
     <title>Network</title>
     <para>
     Perhaps the most important factor in avoiding network issues degrading Hadoop and HBbase performance is the switching hardware
-    that is used, decisions made early in the scope of the project can cause major problems when you double or triple the size of your cluster (or more). 
+    that is used, decisions made early in the scope of the project can cause major problems when you double or triple the size of your cluster (or more).
     </para>
     <para>
     Important items to consider:
@@ -59,15 +59,15 @@
     </para>
     <section xml:id="perf.network.1switch">
       <title>Single Switch</title>
-      <para>The single most important factor in this configuration is that the switching capacity of the hardware is capable of 
+      <para>The single most important factor in this configuration is that the switching capacity of the hardware is capable of
       handling the traffic which can be generated by all systems connected to the switch. Some lower priced commodity hardware
-      can have a slower switching capacity than could be utilized by a full switch. 
+      can have a slower switching capacity than could be utilized by a full switch.
       </para>
     </section>
     <section xml:id="perf.network.2switch">
       <title>Multiple Switches</title>
       <para>Multiple switches are a potential pitfall in the architecture.   The most common configuration of lower priced hardware is a
-      simple 1Gbps uplink from one switch to another. This often overlooked pinch point can easily become a bottleneck for cluster communication. 
+      simple 1Gbps uplink from one switch to another. This often overlooked pinch point can easily become a bottleneck for cluster communication.
       Especially with MapReduce jobs that are both reading and writing a lot of data the communication across this uplink could be saturated.
       </para>
       <para>Mitigation of this issue is fairly simple and can be accomplished in multiple ways:
@@ -85,22 +85,27 @@
            <listitem>Poor switch capacity performance</listitem>
            <listitem>Insufficient uplink to another rack</listitem>
          </itemizedlist>
-      If the the switches in your rack have appropriate switching capacity to handle all the hosts at full speed, the next most likely issue will be caused by homing 
+      If the the switches in your rack have appropriate switching capacity to handle all the hosts at full speed, the next most likely issue will be caused by homing
       more of your cluster across racks.  The easiest way to avoid issues when spanning multiple racks is to use port trunking to create a bonded uplink to other racks.
       The downside of this method however, is in the overhead of ports that could potentially be used. An example of this is, creating an 8Gbps port channel from rack
-      A to rack B, using 8 of your 24 ports to communicate between racks gives you a poor ROI, using too few however can mean you're not getting the most out of your cluster. 
+      A to rack B, using 8 of your 24 ports to communicate between racks gives you a poor ROI, using too few however can mean you're not getting the most out of your cluster.
       </para>
       <para>Using 10Gbe links between racks will greatly increase performance, and assuming your switches support a 10Gbe uplink or allow for an expansion card will allow you to
       save your ports for machines as opposed to uplinks.
       </para>
-      
+    </section>
+    <section xml:id="perf.network.ints">
+      <title>Network Interfaces</title>
+      <para>Are all the network interfaces functioning correctly?  Are you sure?  See the Troubleshooting Case Study in <xref linkend="casestudies.slownode"/>.
+      </para>
     </section>
   </section>  <!-- network -->
+
   <section xml:id="jvm">
     <title>Java</title>
 
     <section xml:id="gc">
-      <title>The Garbage Collector and HBase</title>
+      <title>The Garbage Collector and Apache HBase</title>
 
       <section xml:id="gcpause">
         <title>Long GC pauses</title>
@@ -117,13 +122,20 @@
         threshold, the more GCing is done, the more CPU used). To address the
         second fragmentation issue, Todd added an experimental facility,
         <indexterm><primary>MSLAB</primary></indexterm>, that
-        must be explicitly enabled in HBase 0.90.x (Its defaulted to be on in
-        0.92.x HBase). See <code>hbase.hregion.memstore.mslab.enabled</code>
+        must be explicitly enabled in Apache HBase 0.90.x (Its defaulted to be on in
+        Apache 0.92.x HBase). See <code>hbase.hregion.memstore.mslab.enabled</code>
         to true in your <classname>Configuration</classname>. See the cited
         slides for background and detail<footnote><para>The latest jvms do better
         regards fragmentation so make sure you are running a recent release.
         Read down in the message,
-        <link xlink:href="http://osdir.com/ml/hotspot-gc-use/2011-11/msg00002.html">Identifying concurrent mode failures caused by fragmentation</link>.</para></footnote>.</para>
+        <link xlink:href="http://osdir.com/ml/hotspot-gc-use/2011-11/msg00002.html">Identifying concurrent mode failures caused by fragmentation</link>.</para></footnote>.
+        Be aware that when enabled, each MemStore instance will occupy at least
+        an MSLAB instance of memory.  If you have thousands of regions or lots
+        of regions each with many column families, this allocation of MSLAB
+        may be responsible for a good portion of your heap allocation and in
+        an extreme case cause you to OOME.  Disable MSLAB in this case, or
+        lower the amount of memory it uses or float less regions per server.
+        </para>
         <para>For more information about GC logs, see <xref linkend="trouble.log.gc" />.
         </para>
       </section>
@@ -135,6 +147,7 @@
 
     <para>See <xref linkend="recommended_configurations" />.</para>
 
+
     <section xml:id="perf.number.of.regions">
       <title>Number of Regions</title>
 
@@ -153,41 +166,52 @@
 
     <section xml:id="perf.handlers">
         <title><varname>hbase.regionserver.handler.count</varname></title>
-        <para>See <xref linkend="hbase.regionserver.handler.count"/>. 
+        <para>See <xref linkend="hbase.regionserver.handler.count"/>.
 	    </para>
     </section>
     <section xml:id="perf.hfile.block.cache.size">
         <title><varname>hfile.block.cache.size</varname></title>
-        <para>See <xref linkend="hfile.block.cache.size"/>. 
+        <para>See <xref linkend="hfile.block.cache.size"/>.
         A memory setting for the RegionServer process.
         </para>
-    </section>    
+    </section>
     <section xml:id="perf.rs.memstore.upperlimit">
         <title><varname>hbase.regionserver.global.memstore.upperLimit</varname></title>
-        <para>See <xref linkend="hbase.regionserver.global.memstore.upperLimit"/>.  
+        <para>See <xref linkend="hbase.regionserver.global.memstore.upperLimit"/>.
         This memory setting is often adjusted for the RegionServer process depending on needs.
         </para>
-    </section>    
+    </section>
     <section xml:id="perf.rs.memstore.lowerlimit">
         <title><varname>hbase.regionserver.global.memstore.lowerLimit</varname></title>
-        <para>See <xref linkend="hbase.regionserver.global.memstore.lowerLimit"/>.  
+        <para>See <xref linkend="hbase.regionserver.global.memstore.lowerLimit"/>.
         This memory setting is often adjusted for the RegionServer process depending on needs.
         </para>
     </section>
     <section xml:id="perf.hstore.blockingstorefiles">
         <title><varname>hbase.hstore.blockingStoreFiles</varname></title>
-        <para>See <xref linkend="hbase.hstore.blockingStoreFiles"/>.  
+        <para>See <xref linkend="hbase.hstore.blockingStoreFiles"/>.
         If there is blocking in the RegionServer logs, increasing this can help.
         </para>
     </section>
     <section xml:id="perf.hregion.memstore.block.multiplier">
         <title><varname>hbase.hregion.memstore.block.multiplier</varname></title>
-        <para>See <xref linkend="hbase.hregion.memstore.block.multiplier"/>.  
-        If there is enough RAM, increasing this can help.  
+        <para>See <xref linkend="hbase.hregion.memstore.block.multiplier"/>.
+        If there is enough RAM, increasing this can help.
+        </para>
+    </section>
+    <section xml:id="hbase.regionserver.checksum.verify">
+        <title><varname>hbase.regionserver.checksum.verify</varname></title>
+        <para>Have HBase write the checksum into the datablock and save
+        having to do the checksum seek whenever you read. See the
+        release note on <link xlink:href="https://issues.apache.org/jira/browse/HBASE-5074">HBASE-5074 support checksums in HBase block cache</link>.
         </para>
     </section>
 
   </section>
+
+
+
+
   <section xml:id="perf.zookeeper">
     <title>ZooKeeper</title>
     <para>See <xref linkend="zookeeper"/> for information on configuring ZooKeeper, and see the part
@@ -196,19 +220,19 @@
   </section>
   <section xml:id="perf.schema">
       <title>Schema Design</title>
-  
+
     <section xml:id="perf.number.of.cfs">
       <title>Number of Column Families</title>
       <para>See <xref linkend="number.of.cfs" />.</para>
     </section>
     <section xml:id="perf.schema.keys">
       <title>Key and Attribute Lengths</title>
-      <para>See <xref linkend="keysize" />.  See also <xref linkend="perf.compression.however" /> for 
+      <para>See <xref linkend="keysize" />.  See also <xref linkend="perf.compression.however" /> for
       compression caveats.</para>
     </section>
     <section xml:id="schema.regionsize"><title>Table RegionSize</title>
     <para>The regionsize can be set on a per-table basis via <code>setFileSize</code> on
-    <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html">HTableDescriptor</link> in the 
+    <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html">HTableDescriptor</link> in the
     event where certain tables require different regionsizes than the configured default regionsize.
     </para>
     <para>See <xref linkend="perf.number.of.regions"/> for more information.
@@ -224,22 +248,23 @@
         on each insert. If <varname>ROWCOL</varname>, the hash of the row +
         column family + column family qualifier will be added to the bloom on
         each key insert.</para>
-    <para>See <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link> and 
-    <xref linkend="blooms"/> for more information.
+    <para>See <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link> and
+    <xref linkend="blooms"/> for more information or this answer up in quora,
+<link xlink:href="http://www.quora.com/How-are-bloom-filters-used-in-HBase">How are bloom filters used in HBase?</link>.
     </para>
     </section>
     <section xml:id="schema.cf.blocksize"><title>ColumnFamily BlockSize</title>
-    <para>The blocksize can be configured for each ColumnFamily in a table, and this defaults to 64k.  Larger cell values require larger blocksizes. 
+    <para>The blocksize can be configured for each ColumnFamily in a table, and this defaults to 64k.  Larger cell values require larger blocksizes.
     There is an inverse relationship between blocksize and the resulting StoreFile indexes (i.e., if the blocksize is doubled then the resulting
     indexes should be roughly halved).
     </para>
-    <para>See <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link> 
+    <para>See <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link>
     and <xref linkend="store"/>for more information.
     </para>
     </section>
     <section xml:id="cf.in.memory">
     <title>In-Memory ColumnFamilies</title>
-    <para>ColumnFamilies can optionally be defined as in-memory.  Data is still persisted to disk, just like any other ColumnFamily.  
+    <para>ColumnFamilies can optionally be defined as in-memory.  Data is still persisted to disk, just like any other ColumnFamily.
     In-memory blocks have the highest priority in the <xref linkend="block.cache" />, but it is not a guarantee that the entire table
     will be in memory.
     </para>
@@ -251,24 +276,24 @@
       <para>Production systems should use compression with their ColumnFamily definitions.  See <xref linkend="compression" /> for more information.
       </para>
       <section xml:id="perf.compression.however"><title>However...</title>
-         <para>Compression deflates data <emphasis>on disk</emphasis>.  When it's in-memory (e.g., in the 
+         <para>Compression deflates data <emphasis>on disk</emphasis>.  When it's in-memory (e.g., in the
          MemStore) or on the wire (e.g., transferring between RegionServer and Client) it's inflated.
          So while using ColumnFamily compression is a best practice, but it's not going to completely eliminate
-         the impact of over-sized Keys, over-sized ColumnFamily names, or over-sized Column names. 
+         the impact of over-sized Keys, over-sized ColumnFamily names, or over-sized Column names.
          </para>
          <para>See <xref linkend="keysize" /> on for schema design tips, and <xref linkend="keyvalue"/> for more information on HBase stores data internally.
-         </para> 
+         </para>
       </section>
     </section>
   </section>  <!--  perf schema -->
-  
+
   <section xml:id="perf.writing">
     <title>Writing to HBase</title>
 
     <section xml:id="perf.batch.loading">
       <title>Batch Loading</title>
       <para>Use the bulk load tool if you can.  See
-        <link xlink:href="http://hbase.apache.org/bulk-loads.html">Bulk Loads</link>.
+        <xref linkend="arch.bulk.load"/>.
         Otherwise, pay attention to the below.
       </para>
     </section>  <!-- batch loading -->
@@ -278,35 +303,27 @@
     Table Creation: Pre-Creating Regions
     </title>
 <para>
-Tables in HBase are initially created with one region by default.  For bulk imports, this means that all clients will write to the same region until it is large enough to split and become distributed across the cluster.  A useful pattern to speed up the bulk import process is to pre-create empty regions.  Be somewhat conservative in this, because too-many regions can actually degrade performance.  An example of pre-creation using hex-keys is as follows (note:  this example may need to be tweaked to the individual applications keys):
+Tables in HBase are initially created with one region by default.  For bulk imports, this means that all clients will write to the same region 
+until it is large enough to split and become distributed across the cluster.  A useful pattern to speed up the bulk import process is to pre-create empty regions. 
+ Be somewhat conservative in this, because too-many regions can actually degrade performance.  
 </para>
+	<para>There are two different approaches to pre-creating splits.  The first approach is to rely on the default <code>HBaseAdmin</code> strategy 
+	(which is implemented in <code>Bytes.split</code>)...
+	</para>
+<programlisting>
+byte[] startKey = ...;   	// your lowest keuy
+byte[] endKey = ...;   		// your highest key
+int numberOfRegions = ...;	// # of regions to create
+admin.createTable(table, startKey, endKey, numberOfRegions);
+</programlisting>
+	<para>And the other approach is to define the splits yourself...
+	</para>
+<programlisting>
+byte[][] splits = ...;   // create your own splits
+admin.createTable(table, splits);
+</programlisting>
 <para>
-<programlisting>public static boolean createTable(HBaseAdmin admin, HTableDescriptor table, byte[][] splits)
-throws IOException {
-  try {
-    admin.createTable( table, splits );
-    return true;
-  } catch (TableExistsException e) {
-    logger.info("table " + table.getNameAsString() + " already exists");
-    // the table already exists...
-    return false;  
-  }
-}
-
-public static byte[][] getHexSplits(String startKey, String endKey, int numRegions) {
-  byte[][] splits = new byte[numRegions-1][];
-  BigInteger lowestKey = new BigInteger(startKey, 16);
-  BigInteger highestKey = new BigInteger(endKey, 16);
-  BigInteger range = highestKey.subtract(lowestKey);
-  BigInteger regionIncrement = range.divide(BigInteger.valueOf(numRegions));
-  lowestKey = lowestKey.add(regionIncrement);
-  for(int i=0; i &lt; numRegions-1;i++) {
-    BigInteger key = lowestKey.add(regionIncrement.multiply(BigInteger.valueOf(i)));
-    byte[] b = String.format("%016x", key).getBytes();
-    splits[i] = b;
-  }
-  return splits;
-}</programlisting>
+   See <xref linkend="rowkey.regionsplits"/> for issues related to understanding your keyspace and pre-creating regions.
   </para>
   </section>
     <section xml:id="def.log.flush">
@@ -314,7 +331,7 @@ public static byte[][] getHexSplits(String startKey, String endKey, int numRegio
     Table Creation: Deferred Log Flush
     </title>
 <para>
-The default behavior for Puts using the Write Ahead Log (WAL) is that <classname>HLog</classname> edits will be written immediately.  If deferred log flush is used, 
+The default behavior for Puts using the Write Ahead Log (WAL) is that <classname>HLog</classname> edits will be written immediately.  If deferred log flush is used,
 WAL edits are kept in memory until the flush period.  The benefit is aggregated and asynchronous <classname>HLog</classname>- writes, but the potential downside is that if
  the RegionServer goes down the yet-to-be-flushed edits are lost.  This is safer, however, than not using WAL at all with Puts.
 </para>
@@ -322,7 +339,7 @@ WAL edits are kept in memory until the flush period.  The benefit is aggregated
 Deferred log flush can be configured on tables via <link
       xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html">HTableDescriptor</link>.  The default value of <varname>hbase.regionserver.optionallogflushinterval</varname> is 1000ms.
 </para>
-    </section>  
+    </section>
 
     <section xml:id="perf.hbase.client.autoflush">
       <title>HBase Client:  AutoFlush</title>
@@ -348,25 +365,25 @@ Deferred log flush can be configured on tables via <link
           it makes little difference if your load is well distributed across the cluster.
       </para>
       <para>In general, it is best to use WAL for Puts, and where loading throughput
-          is a concern to use <link linkend="perf.batch.loading">bulk loading</link> techniques instead.  
+          is a concern to use <link linkend="perf.batch.loading">bulk loading</link> techniques instead.
       </para>
     </section>
     <section xml:id="perf.hbase.client.regiongroup">
       <title>HBase Client: Group Puts by RegionServer</title>
-      <para>In addition to using the writeBuffer, grouping <classname>Put</classname>s by RegionServer can reduce the number of client RPC calls per writeBuffer flush. 
+      <para>In addition to using the writeBuffer, grouping <classname>Put</classname>s by RegionServer can reduce the number of client RPC calls per writeBuffer flush.
       There is a utility <classname>HTableUtil</classname> currently on TRUNK that does this, but you can either copy that or implement your own verison for
       those still on 0.90.x or earlier.
       </para>
-    </section>    
+    </section>
     <section xml:id="perf.hbase.write.mr.reducer">
       <title>MapReduce:  Skip The Reducer</title>
       <para>When writing a lot of data to an HBase table from a MR job (e.g., with <link
       xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.html">TableOutputFormat</link>), and specifically where Puts are being emitted
-      from the Mapper, skip the Reducer step.  When a Reducer step is used, all of the output (Puts) from the Mapper will get spooled to disk, then sorted/shuffled to other 
-      Reducers that will most likely be off-node.  It's far more efficient to just write directly to HBase.   
+      from the Mapper, skip the Reducer step.  When a Reducer step is used, all of the output (Puts) from the Mapper will get spooled to disk, then sorted/shuffled to other
+      Reducers that will most likely be off-node.  It's far more efficient to just write directly to HBase.
       </para>
-      <para>For summary jobs where HBase is used as a source and a sink, then writes will be coming from the Reducer step (e.g., summarize values then write out result). 
-      This is a different processing problem than from the the above case. 
+      <para>For summary jobs where HBase is used as a source and a sink, then writes will be coming from the Reducer step (e.g., summarize values then write out result).
+      This is a different processing problem than from the the above case.
       </para>
     </section>
 
@@ -375,16 +392,16 @@ Deferred log flush can be configured on tables via <link
     <para>If all your data is being written to one region at a time, then re-read the
     section on processing <link linkend="timeseries">timeseries</link> data.</para>
     <para>Also, if you are pre-splitting regions and all your data is <emphasis>still</emphasis> winding up in a single region even though
-    your keys aren't monotonically increasing, confirm that your keyspace actually works with the split strategy.  There are a 
+    your keys aren't monotonically increasing, confirm that your keyspace actually works with the split strategy.  There are a
     variety of reasons that regions may appear "well split" but won't work with your data.   As
-    the HBase client communicates directly with the RegionServers, this can be obtained via 
+    the HBase client communicates directly with the RegionServers, this can be obtained via
     <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#getRegionLocation%28byte[]%29">HTable.getRegionLocation</link>.
     </para>
-    <para>See <xref linkend="precreate.regions"/>, as well as <xref linkend="perf.configurations"/> </para>   
+    <para>See <xref linkend="precreate.regions"/>, as well as <xref linkend="perf.configurations"/> </para>
   </section>
 
   </section>  <!--  writing -->
-  
+
   <section xml:id="perf.reading">
     <title>Reading from HBase</title>
 
@@ -406,7 +423,7 @@ Deferred log flush can be configured on tables via <link
         <para>Scan settings in MapReduce jobs deserve special attention.  Timeouts can result (e.g., UnknownScannerException)
         in Map tasks if it takes longer to process a batch of records before the client goes back to the RegionServer for the
         next set of data.  This problem can occur because there is non-trivial processing occuring per row.  If you process
-        rows quickly, set caching higher.  If you process rows more slowly (e.g., lots of transformations per row, writes), 
+        rows quickly, set caching higher.  If you process rows more slowly (e.g., lots of transformations per row, writes),
         then set caching lower.
         </para>
         <para>Timeouts can also happen in a non-MapReduce use case (i.e., single threaded HBase client doing a Scan), but the
@@ -424,6 +441,13 @@ Deferred log flush can be configured on tables via <link
       in the input scan because attribute over-selection is a non-trivial performance penalty over large datasets.
       </para>
     </section>
+    <section xml:id="perf.hbase.mr.input">
+        <title>MapReduce - Input Splits</title>
+        <para>For MapReduce jobs that use HBase tables as a source, if there a pattern where the "slow" map tasks seem to
+        have the same Input Split (i.e., the RegionServer serving the data), see the
+        Troubleshooting Case Study in <xref linkend="casestudies.slownode"/>.
+        </para>
+    </section>
 
     <section xml:id="perf.hbase.client.scannerclose">
       <title>Close ResultScanners</title>
@@ -469,13 +493,103 @@ htable.close();</programlisting></para>
     </section>
    <section xml:id="perf.hbase.read.dist">
       <title>Concurrency:  Monitor Data Spread</title>
-      <para>When performing a high number of concurrent reads, monitor the data spread of the target tables.  If the target table(s) have 
+      <para>When performing a high number of concurrent reads, monitor the data spread of the target tables.  If the target table(s) have
       too few regions then the reads could likely be served from too few nodes.  </para>
-      <para>See <xref linkend="precreate.regions"/>, as well as <xref linkend="perf.configurations"/> </para>   
+      <para>See <xref linkend="precreate.regions"/>, as well as <xref linkend="perf.configurations"/> </para>
    </section>
-    
+     <section xml:id="blooms">
+     <title>Bloom Filters</title>
+         <para>Enabling Bloom Filters can save your having to go to disk and
+         can help improve read latencys.</para>
+         <para><link xlink:href="http://en.wikipedia.org/wiki/Bloom_filter">Bloom filters</link> were developed over in <link
+    xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBase-1200
+    Add bloomfilters</link>.<footnote>
+        <para>For description of the development process -- why static blooms
+        rather than dynamic -- and for an overview of the unique properties
+        that pertain to blooms in HBase, as well as possible future
+        directions, see the <emphasis>Development Process</emphasis> section
+        of the document <link
+        xlink:href="https://issues.apache.org/jira/secure/attachment/12444007/Bloom_Filters_in_HBase.pdf">BloomFilters
+        in HBase</link> attached to <link
+        xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBase-1200</link>.</para>
+      </footnote><footnote>
+        <para>The bloom filters described here are actually version two of
+        blooms in HBase. In versions up to 0.19.x, HBase had a dynamic bloom
+        option based on work done by the <link
+        xlink:href="http://www.one-lab.org">European Commission One-Lab
+        Project 034819</link>. The core of the HBase bloom work was later
+        pulled up into Hadoop to implement org.apache.hadoop.io.BloomMapFile.
+        Version 1 of HBase blooms never worked that well. Version 2 is a
+        rewrite from scratch though again it starts with the one-lab
+        work.</para>
+      </footnote></para>
+        <para>See also <xref linkend="schema.bloom" />.
+        </para>
+
+     <section xml:id="bloom_footprint">
+      <title>Bloom StoreFile footprint</title>
+
+      <para>Bloom filters add an entry to the <classname>StoreFile</classname>
+      general <classname>FileInfo</classname> data structure and then two
+      extra entries to the <classname>StoreFile</classname> metadata
+      section.</para>
+
+      <section>
+        <title>BloomFilter in the <classname>StoreFile</classname>
+        <classname>FileInfo</classname> data structure</title>
+
+          <para><classname>FileInfo</classname> has a
+          <varname>BLOOM_FILTER_TYPE</varname> entry which is set to
+          <varname>NONE</varname>, <varname>ROW</varname> or
+          <varname>ROWCOL.</varname></para>
+      </section>
+
+      <section>
+        <title>BloomFilter entries in <classname>StoreFile</classname>
+        metadata</title>
+
+          <para><varname>BLOOM_FILTER_META</varname> holds Bloom Size, Hash
+          Function used, etc. Its small in size and is cached on
+          <classname>StoreFile.Reader</classname> load</para>
+          <para><varname>BLOOM_FILTER_DATA</varname> is the actual bloomfilter
+          data. Obtained on-demand. Stored in the LRU cache, if it is enabled
+          (Its enabled by default).</para>
+      </section>
+     </section>
+	  <section xml:id="config.bloom">
+	    <title>Bloom Filter Configuration</title>
+        <section>
+        <title><varname>io.hfile.bloom.enabled</varname> global kill
+        switch</title>
+
+        <para><code>io.hfile.bloom.enabled</code> in
+        <classname>Configuration</classname> serves as the kill switch in case
+        something goes wrong. Default = <varname>true</varname>.</para>
+        </section>
+
+        <section>
+        <title><varname>io.hfile.bloom.error.rate</varname></title>
+
+        <para><varname>io.hfile.bloom.error.rate</varname> = average false
+        positive rate. Default = 1%. Decrease rate by ½ (e.g. to .5%) == +1
+        bit per bloom entry.</para>
+        </section>
+
+        <section>
+        <title><varname>io.hfile.bloom.max.fold</varname></title>
+
+        <para><varname>io.hfile.bloom.max.fold</varname> = guaranteed minimum
+        fold rate. Most people should leave this alone. Default = 7, or can
+        collapse to at least 1/128th of original size. See the
+        <emphasis>Development Process</emphasis> section of the document <link
+        xlink:href="https://issues.apache.org/jira/secure/attachment/12444007/Bloom_Filters_in_HBase.pdf">BloomFilters
+        in HBase</link> for more on what this option means.</para>
+        </section>
+        </section>
+     </section>   <!--  bloom  -->
+
   </section>  <!--  reading -->
-  
+
   <section xml:id="perf.deleting">
     <title>Deleting from HBase</title>
      <section xml:id="perf.deleting.queue">
@@ -503,21 +617,54 @@ htable.close();</programlisting></para>
    </para>
     <section xml:id="perf.hdfs.curr"><title>Current Issues With Low-Latency Reads</title>
       <para>The original use-case for HDFS was batch processing.  As such, there low-latency reads were historically not a priority.
-      With the increased adoption of HBase this is changing, and several improvements are already in development.
-      See the 
+      With the increased adoption of Apache HBase this is changing, and several improvements are already in development.
+      See the
       <link xlink:href="https://issues.apache.org/jira/browse/HDFS-1599">Umbrella Jira Ticket for HDFS Improvements for HBase</link>.
       </para>
     </section>
+    <section xml:id="perf.hdfs.configs.localread">
+    <title>Leveraging local data</title>
+<para>Since Hadoop 1.0.0 (also 0.22.1, 0.23.1, CDH3u3 and HDP 1.0) via
+<link xlink:href="https://issues.apache.org/jira/browse/HDFS-2246">HDFS-2246</link>,
+it is possible for the DFSClient to take a "short circuit" and
+read directly from disk instead of going through the DataNode when the
+data is local. What this means for HBase is that the RegionServers can
+read directly off their machine's disks instead of having to open a
+socket to talk to the DataNode, the former being generally much
+faster<footnote><para>See JD's <link xlink:href="http://files.meetup.com/1350427/hug_ebay_jdcryans.pdf">Performance Talk</link></para></footnote>.
+Also see <link xlink:href="http://search-hadoop.com/m/zV6dKrLCVh1">HBase, mail # dev - read short circuit</link> thread for
+more discussion around short circuit reads.
+</para>
+<para>To enable "short circuit" reads, you must set two configurations.
+First, the hdfs-site.xml needs to be amended. Set
+the property  <varname>dfs.block.local-path-access.user</varname>
+to be the <emphasis>only</emphasis> user that can use the shortcut.
+This has to be the user that started HBase.  Then in hbase-site.xml,
+set <varname>dfs.client.read.shortcircuit</varname> to be <varname>true</varname>
+</para>
+<para>
+    For optimal performance when short-circuit reads are enabled, it is recommended that HDFS checksums are disabled.
+    To maintain data integrity with HDFS checksums disabled, HBase can be configured to write its own checksums into
+    its datablocks and verify against these. See <xref linkend="hbase.regionserver.checksum.verify" />.
+</para>
+<para>
+The DataNodes need to be restarted in order to pick up the new
+configuration. Be aware that if a process started under another
+username than the one configured here also has the shortcircuit
+enabled, it will get an Exception regarding an unauthorized access but
+the data will still be read.
+</para>
+  </section>
     <section xml:id="perf.hdfs.comp"><title>Performance Comparisons of HBase vs. HDFS</title>
-     <para>A fairly common question on the dist-list is why HBase isn't as performant as HDFS files in a batch context (e.g., as 
-     a MapReduce source or sink).  The short answer is that HBase is doing a lot more than HDFS (e.g., reading the KeyValues, 
-     returning the most current row or specified timestamps, etc.), and as such HBase is 4-5 times slower than HDFS in this 
+     <para>A fairly common question on the dist-list is why HBase isn't as performant as HDFS files in a batch context (e.g., as
+     a MapReduce source or sink).  The short answer is that HBase is doing a lot more than HDFS (e.g., reading the KeyValues,
+     returning the most current row or specified timestamps, etc.), and as such HBase is 4-5 times slower than HDFS in this
      processing context.  Not that there isn't room for improvement (and this gap will, over time, be reduced), but HDFS
       will always be faster in this use-case.
      </para>
     </section>
   </section>
-  
+
   <section xml:id="perf.ec2"><title>Amazon EC2</title>
    <para>Performance questions are common on Amazon EC2 environments because it is a shared environment.  You will
    not see the same throughput as a dedicated server.  In terms of running tests on EC2, run them several times for the same
@@ -527,4 +674,9 @@ htable.close();</programlisting></para>
     because EC2 issues are practically a separate class of performance issues.
    </para>
   </section>
+
+  <section xml:id="perf.casestudy"><title>Case Studies</title>
+      <para>For Performance and Troubleshooting Case Studies, see <xref linkend="casestudies"/>.
+      </para>
+  </section>
 </chapter>
diff --git a/src/docbkx/preface.xml b/src/docbkx/preface.xml
index 2d9f39d1c678..af54aa29749a 100644
--- a/src/docbkx/preface.xml
+++ b/src/docbkx/preface.xml
@@ -33,7 +33,7 @@
   Herein you will find either the definitive documentation on an HBase topic
   as of its standing when the referenced HBase version shipped, or it
   will point to the location in <link
-  xlink:href="http://hbase.apache.org/docs/current/api/index.html">javadoc</link>,
+  xlink:href="http://hbase.apache.org/apidocs/index.html">javadoc</link>,
   <link xlink:href="https://issues.apache.org/jira/browse/HBASE">JIRA</link>
   or <link xlink:href="http://wiki.apache.org/hadoop/Hbase">wiki</link> where
   the pertinent information can be found.</para>
diff --git a/src/docbkx/shell.xml b/src/docbkx/shell.xml
index 4fbab08d2236..2a1535336189 100644
--- a/src/docbkx/shell.xml
+++ b/src/docbkx/shell.xml
@@ -26,13 +26,13 @@
  * limitations under the License.
  */
 -->
-    <title>The HBase Shell</title>
+    <title>The Apache HBase Shell</title>
 
     <para>
-        The HBase Shell is <link xlink:href="http://jruby.org">(J)Ruby</link>'s
+        The Apache HBase (TM) Shell is <link xlink:href="http://jruby.org">(J)Ruby</link>'s
         IRB with some HBase particular commands added.  Anything you can do in
         IRB, you should be able to do in the HBase Shell.</para>
-        <para>To run the HBase shell, 
+        <para>To run the HBase shell,
         do as follows:
         <programlisting>$ ./bin/hbase shell</programlisting>
         </para>
@@ -47,7 +47,7 @@
             for example basic shell operation.</para>
 
     <section xml:id="scripting"><title>Scripting</title>
-        <para>For examples scripting HBase, look in the
+        <para>For examples scripting Apache HBase, look in the
             HBase <filename>bin</filename> directory.  Look at the files
             that end in <filename>*.rb</filename>.  To run one of these
             files, do as follows:
@@ -104,5 +104,16 @@
                </para>
             </section>
          </section>
+        <section><title>Commands</title>
+            <section><title>count</title>
+                <para>Count command returns the number of rows in a table.
+		    It's quite fast when configured with the right CACHE
+            <programlisting>hbase> count '&lt;tablename&gt;', CACHE => 1000</programlisting>
+            The above count fetches 1000 rows at a time.  Set CACHE lower if your rows are big.
+            Default is to fetch one row at a time.
+                 </para>
+            </section>
+         </section>
+
     </section>
   </chapter>
diff --git a/src/docbkx/troubleshooting.xml b/src/docbkx/troubleshooting.xml
index a1e382969406..5967b03a3d65 100644
--- a/src/docbkx/troubleshooting.xml
+++ b/src/docbkx/troubleshooting.xml
@@ -26,7 +26,7 @@
  * limitations under the License.
  */
 -->
-  <title>Troubleshooting and Debugging HBase</title>
+  <title>Troubleshooting and Debugging Apache HBase (TM)</title>
     <section xml:id="trouble.general">
       <title>General Guidelines</title>
       <para>
@@ -37,7 +37,7 @@
           should return some hits for those exceptions you’re seeing.
       </para>
       <para>
-          An error rarely comes alone in HBase, usually when something gets screwed up what will
+          An error rarely comes alone in Apache HBase (TM), usually when something gets screwed up what will
           follow may be hundreds of exceptions and stack traces coming from all over the place.
           The best way to approach this type of problem is to walk the log up to where it all
           began, for example one trick with RegionServers is that they will print some
@@ -54,7 +54,7 @@
           prolonged garbage collection pauses that last longer than the default ZooKeeper session timeout.
           For more information on GC pauses, see the
           <link xlink:href="http://www.cloudera.com/blog/2011/02/avoiding-full-gcs-in-hbase-with-memstore-local-allocation-buffers-part-1/">3 part blog post</link>  by Todd Lipcon
-          and <xref linkend="gcpause" /> above. 
+          and <xref linkend="gcpause" /> above.
       </para>
     </section>
     <section xml:id="trouble.log">
@@ -72,7 +72,7 @@
       JobTracker:  <filename>$HADOOP_HOME/logs/hadoop-&lt;user&gt;-jobtracker-&lt;hostname&gt;.log</filename>
       </para>
       <para>
-      TaskTracker:  <filename>$HADOOP_HOME/logs/hadoop-&lt;user&gt;-jobtracker-&lt;hostname&gt;.log</filename>
+      TaskTracker:  <filename>$HADOOP_HOME/logs/hadoop-&lt;user&gt;-tasktracker-&lt;hostname&gt;.log</filename>
       </para>
       <para>
       HMaster:  <filename>$HBASE_HOME/logs/hbase-&lt;user&gt;-master-&lt;hostname&gt;.log</filename>
@@ -91,7 +91,7 @@
           <title>NameNode</title>
           <para>The NameNode log is on the NameNode server.  The HBase Master is typically run on the NameNode server, and well as ZooKeeper.</para>
           <para>For smaller clusters the JobTracker is typically run on the NameNode server as well.</para>
-         </section>        
+         </section>
         <section xml:id="trouble.log.locations.datanode">
           <title>DataNode</title>
           <para>Each DataNode server will have a DataNode log for HDFS, as well as a RegionServer log for HBase.</para>
@@ -105,32 +105,32 @@
            insight on timings at the server.  Once enabled, the amount of log
            spewed is voluminous.  It is not recommended that you leave this
            logging on for more than short bursts of time.  To enable RPC-level
-           logging, browse to the RegionServer UI and click on 
+           logging, browse to the RegionServer UI and click on
            <emphasis>Log Level</emphasis>.  Set the log level to <varname>DEBUG</varname> for the package
            <classname>org.apache.hadoop.ipc</classname> (Thats right, for
            <classname>hadoop.ipc</classname>, NOT, <classname>hbase.ipc</classname>).  Then tail the RegionServers log.  Analyze.</para>
            <para>To disable, set the logging level back to <varname>INFO</varname> level.
            </para>
-         </section>                 
-       </section>      
+         </section>
+       </section>
       <section xml:id="trouble.log.gc">
         <title>JVM Garbage Collection Logs</title>
-          <para>HBase is memory intensive, and using the default GC you can see long pauses in all threads including the <emphasis>Juliet Pause</emphasis> aka "GC of Death". 
-           To help debug this or confirm this is happening GC logging can be turned on in the Java virtual machine.  
+          <para>HBase is memory intensive, and using the default GC you can see long pauses in all threads including the <emphasis>Juliet Pause</emphasis> aka "GC of Death".
+           To help debug this or confirm this is happening GC logging can be turned on in the Java virtual machine.
           </para>
           <para>
           To enable, in <filename>hbase-env.sh</filename> add:
-          <programlisting> 
+          <programlisting>
 export HBASE_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/home/hadoop/hbase/logs/gc-hbase.log"
           </programlisting>
-           Adjust the log directory to wherever you log.  Note:  The GC log does NOT roll automatically, so you'll have to keep an eye on it so it doesn't fill up the disk. 
+           Adjust the log directory to wherever you log.  Note:  The GC log does NOT roll automatically, so you'll have to keep an eye on it so it doesn't fill up the disk.
           </para>
           <para>
          At this point you should see logs like so:
           <programlisting>
-64898.952: [GC [1 CMS-initial-mark: 2811538K(3055704K)] 2812179K(3061272K), 0.0007360 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] 
+64898.952: [GC [1 CMS-initial-mark: 2811538K(3055704K)] 2812179K(3061272K), 0.0007360 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]
 64898.953: [CMS-concurrent-mark-start]
-64898.971: [GC 64898.971: [ParNew: 5567K->576K(5568K), 0.0101110 secs] 2817105K->2812715K(3061272K), 0.0102200 secs] [Times: user=0.07 sys=0.00, real=0.01 secs] 
+64898.971: [GC 64898.971: [ParNew: 5567K->576K(5568K), 0.0101110 secs] 2817105K->2812715K(3061272K), 0.0102200 secs] [Times: user=0.07 sys=0.00, real=0.01 secs]
           </programlisting>
           </para>
           <para>
@@ -139,20 +139,20 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+
             <para>
            The third line indicates a "minor GC", which pauses the VM for 0.0101110 seconds - aka 10 milliseconds. It has reduced the "ParNew" from about 5.5m to 576k.
            Later on in this cycle we see:
-           <programlisting> 
-64901.445: [CMS-concurrent-mark: 1.542/2.492 secs] [Times: user=10.49 sys=0.33, real=2.49 secs] 
+           <programlisting>
+64901.445: [CMS-concurrent-mark: 1.542/2.492 secs] [Times: user=10.49 sys=0.33, real=2.49 secs]
 64901.445: [CMS-concurrent-preclean-start]
-64901.453: [GC 64901.453: [ParNew: 5505K->573K(5568K), 0.0062440 secs] 2868746K->2864292K(3061272K), 0.0063360 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] 
-64901.476: [GC 64901.476: [ParNew: 5563K->575K(5568K), 0.0072510 secs] 2869283K->2864837K(3061272K), 0.0073320 secs] [Times: user=0.05 sys=0.01, real=0.01 secs] 
-64901.500: [GC 64901.500: [ParNew: 5517K->573K(5568K), 0.0120390 secs] 2869780K->2865267K(3061272K), 0.0121150 secs] [Times: user=0.09 sys=0.00, real=0.01 secs] 
-64901.529: [GC 64901.529: [ParNew: 5507K->569K(5568K), 0.0086240 secs] 2870200K->2865742K(3061272K), 0.0087180 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] 
-64901.554: [GC 64901.555: [ParNew: 5516K->575K(5568K), 0.0107130 secs] 2870689K->2866291K(3061272K), 0.0107820 secs] [Times: user=0.06 sys=0.00, real=0.01 secs] 
-64901.578: [CMS-concurrent-preclean: 0.070/0.133 secs] [Times: user=0.48 sys=0.01, real=0.14 secs] 
+64901.453: [GC 64901.453: [ParNew: 5505K->573K(5568K), 0.0062440 secs] 2868746K->2864292K(3061272K), 0.0063360 secs] [Times: user=0.05 sys=0.00, real=0.01 secs]
+64901.476: [GC 64901.476: [ParNew: 5563K->575K(5568K), 0.0072510 secs] 2869283K->2864837K(3061272K), 0.0073320 secs] [Times: user=0.05 sys=0.01, real=0.01 secs]
+64901.500: [GC 64901.500: [ParNew: 5517K->573K(5568K), 0.0120390 secs] 2869780K->2865267K(3061272K), 0.0121150 secs] [Times: user=0.09 sys=0.00, real=0.01 secs]
+64901.529: [GC 64901.529: [ParNew: 5507K->569K(5568K), 0.0086240 secs] 2870200K->2865742K(3061272K), 0.0087180 secs] [Times: user=0.05 sys=0.00, real=0.01 secs]
+64901.554: [GC 64901.555: [ParNew: 5516K->575K(5568K), 0.0107130 secs] 2870689K->2866291K(3061272K), 0.0107820 secs] [Times: user=0.06 sys=0.00, real=0.01 secs]
+64901.578: [CMS-concurrent-preclean: 0.070/0.133 secs] [Times: user=0.48 sys=0.01, real=0.14 secs]
 64901.578: [CMS-concurrent-abortable-preclean-start]
-64901.584: [GC 64901.584: [ParNew: 5504K->571K(5568K), 0.0087270 secs] 2871220K->2866830K(3061272K), 0.0088220 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] 
-64901.609: [GC 64901.609: [ParNew: 5512K->569K(5568K), 0.0063370 secs] 2871771K->2867322K(3061272K), 0.0064230 secs] [Times: user=0.06 sys=0.00, real=0.01 secs] 
-64901.615: [CMS-concurrent-abortable-preclean: 0.007/0.037 secs] [Times: user=0.13 sys=0.00, real=0.03 secs] 
-64901.616: [GC[YG occupancy: 645 K (5568 K)]64901.616: [Rescan (parallel) , 0.0020210 secs]64901.618: [weak refs processing, 0.0027950 secs] [1 CMS-remark: 2866753K(3055704K)] 2867399K(3061272K), 0.0049380 secs] [Times: user=0.00 sys=0.01, real=0.01 secs] 
+64901.584: [GC 64901.584: [ParNew: 5504K->571K(5568K), 0.0087270 secs] 2871220K->2866830K(3061272K), 0.0088220 secs] [Times: user=0.05 sys=0.00, real=0.01 secs]
+64901.609: [GC 64901.609: [ParNew: 5512K->569K(5568K), 0.0063370 secs] 2871771K->2867322K(3061272K), 0.0064230 secs] [Times: user=0.06 sys=0.00, real=0.01 secs]
+64901.615: [CMS-concurrent-abortable-preclean: 0.007/0.037 secs] [Times: user=0.13 sys=0.00, real=0.03 secs]
+64901.616: [GC[YG occupancy: 645 K (5568 K)]64901.616: [Rescan (parallel) , 0.0020210 secs]64901.618: [weak refs processing, 0.0027950 secs] [1 CMS-remark: 2866753K(3055704K)] 2867399K(3061272K), 0.0049380 secs] [Times: user=0.00 sys=0.01, real=0.01 secs]
 64901.621: [CMS-concurrent-sweep-start]
             </programlisting>
             </para>
@@ -161,20 +161,20 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+
             </para>
             <para>
             There are a few more minor GCs, then there is a pause at the 2nd last line:
-            <programlisting>  
-64901.616: [GC[YG occupancy: 645 K (5568 K)]64901.616: [Rescan (parallel) , 0.0020210 secs]64901.618: [weak refs processing, 0.0027950 secs] [1 CMS-remark: 2866753K(3055704K)] 2867399K(3061272K), 0.0049380 secs] [Times: user=0.00 sys=0.01, real=0.01 secs] 
+            <programlisting>
+64901.616: [GC[YG occupancy: 645 K (5568 K)]64901.616: [Rescan (parallel) , 0.0020210 secs]64901.618: [weak refs processing, 0.0027950 secs] [1 CMS-remark: 2866753K(3055704K)] 2867399K(3061272K), 0.0049380 secs] [Times: user=0.00 sys=0.01, real=0.01 secs]
             </programlisting>
             </para>
             <para>
-            The pause here is 0.0049380 seconds (aka 4.9 milliseconds) to 'remark' the heap.  
+            The pause here is 0.0049380 seconds (aka 4.9 milliseconds) to 'remark' the heap.
             </para>
             <para>
             At this point the sweep starts, and you can watch the heap size go down:
             <programlisting>
-64901.637: [GC 64901.637: [ParNew: 5501K->569K(5568K), 0.0097350 secs] 2871958K->2867441K(3061272K), 0.0098370 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] 
+64901.637: [GC 64901.637: [ParNew: 5501K->569K(5568K), 0.0097350 secs] 2871958K->2867441K(3061272K), 0.0098370 secs] [Times: user=0.05 sys=0.00, real=0.01 secs]
 ...  lines removed ...
-64904.936: [GC 64904.936: [ParNew: 5532K->568K(5568K), 0.0070720 secs] 1365024K->1360689K(3061272K), 0.0071930 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] 
-64904.953: [CMS-concurrent-sweep: 2.030/3.332 secs] [Times: user=9.57 sys=0.26, real=3.33 secs] 
+64904.936: [GC 64904.936: [ParNew: 5532K->568K(5568K), 0.0070720 secs] 1365024K->1360689K(3061272K), 0.0071930 secs] [Times: user=0.05 sys=0.00, real=0.01 secs]
+64904.953: [CMS-concurrent-sweep: 2.030/3.332 secs] [Times: user=9.57 sys=0.26, real=3.33 secs]
             </programlisting>
             At this point, the CMS sweep took 3.332 seconds, and heap went from about ~ 2.8 GB to 1.3 GB (approximate).
             </para>
@@ -186,14 +186,14 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+
             </para>
             <para>
              Add this to HBASE_OPTS:
-            <programlisting> 
+            <programlisting>
 export HBASE_OPTS="-XX:NewSize=64m -XX:MaxNewSize=64m &lt;cms options from above&gt; &lt;gc logging options from above&gt;"
             </programlisting>
             </para>
             <para>
             For more information on GC pauses, see the <link xlink:href="http://www.cloudera.com/blog/2011/02/avoiding-full-gcs-in-hbase-with-memstore-local-allocation-buffers-part-1/">3 part blog post</link>  by Todd Lipcon
             and <xref linkend="gcpause" /> above.
-            </para>                  
+            </para>
       </section>
     </section>
     <section xml:id="trouble.resources">
@@ -201,18 +201,18 @@ export HBASE_OPTS="-XX:NewSize=64m -XX:MaxNewSize=64m &lt;cms options from above
       <section xml:id="trouble.resources.searchhadoop">
         <title>search-hadoop.com</title>
         <para>
-        <link xlink:href="http://search-hadoop.com">search-hadoop.com</link> indexes all the mailing lists and is great for historical searches.  
+        <link xlink:href="http://search-hadoop.com">search-hadoop.com</link> indexes all the mailing lists and is great for historical searches.
         Search here first when you have an issue as its more than likely someone has already had your problem.
         </para>
       </section>
       <section xml:id="trouble.resources.lists">
         <title>Mailing Lists</title>
-        <para>Ask a question on the <link xlink:href="http://hbase.apache.org/mail-lists.html">HBase mailing lists</link>.
-        The 'dev' mailing list is aimed at the community of developers actually building HBase and for features currently under development, and 'user'
-        is generally used for questions on released versions of HBase.  Before going to the mailing list, make sure your
+        <para>Ask a question on the <link xlink:href="http://hbase.apache.org/mail-lists.html">Apache HBase mailing lists</link>.
+        The 'dev' mailing list is aimed at the community of developers actually building Apache HBase and for features currently under development, and 'user'
+        is generally used for questions on released versions of Apache HBase.  Before going to the mailing list, make sure your
         question has not already been answered by searching the mailing list archives first.  Use
         <xref linkend="trouble.resources.searchhadoop" />.
-        Take some time crafting your question<footnote><para>See <link xlink="http://www.mikeash.com/getting_answers.html">Getting Answers</link></para></footnote>; a quality question that includes all context and 
+        Take some time crafting your question<footnote><para>See <link xlink="http://www.mikeash.com/getting_answers.html">Getting Answers</link></para></footnote>; a quality question that includes all context and
         exhibits evidence the author has tried to find answers in the manual and out on lists
         is more likely to get a prompt response.
         </para>
@@ -236,7 +236,7 @@ export HBASE_OPTS="-XX:NewSize=64m -XX:MaxNewSize=64m &lt;cms options from above
               <title>Master Web Interface</title>
               <para>The Master starts a web-interface on port 60010 by default.
               </para>
-              <para>The Master web UI lists created tables and their definition (e.g., ColumnFamilies, blocksize, etc.).  Additionally, 
+              <para>The Master web UI lists created tables and their definition (e.g., ColumnFamilies, blocksize, etc.).  Additionally,
               the available RegionServers in the cluster are listed along with selected high-level metrics (requests, number of regions, usedHeap, maxHeap).
               The Master web UI allows navigation to each RegionServer's web UI.
               </para>
@@ -263,13 +263,13 @@ export HBASE_OPTS="-XX:NewSize=64m -XX:MaxNewSize=64m &lt;cms options from above
 	ls path [watch]
 	set path data [version]
 	delquota [-n|-b] path
-	quit 
+	quit
 	printwatches on|off
 	create [-s] [-e] path data acl
 	stat path [watch]
-	close 
+	close
 	ls2 path [watch]
-	history 
+	history
 	listquota path
 	setAcl path acl
 	getAcl path
@@ -292,7 +292,7 @@ export HBASE_OPTS="-XX:NewSize=64m -XX:MaxNewSize=64m &lt;cms options from above
       </section>
       <section xml:id="trouble.tools.top">
         <title>top</title>
-        <para>         
+        <para>
         <code>top</code> is probably one of the most important tool when first trying to see what’s running on a machine and how the resources are consumed. Here’s an example from production system:
         <programlisting>
 top - 14:46:59 up 39 days, 11:55,  1 user,  load average: 3.75, 3.57, 3.84
@@ -300,10 +300,10 @@ Tasks: 309 total,   1 running, 308 sleeping,   0 stopped,   0 zombie
 Cpu(s):  4.5%us,  1.6%sy,  0.0%ni, 91.7%id,  1.4%wa,  0.1%hi,  0.6%si,  0.0%st
 Mem:  24414432k total, 24296956k used,   117476k free,     7196k buffers
 Swap: 16008732k total,	14348k used, 15994384k free, 11106908k cached
- 
-  PID USER  	PR  NI  VIRT  RES  SHR S %CPU %MEM	TIME+  COMMAND                                                                                                                                                                      
-15558 hadoop	18  -2 3292m 2.4g 3556 S   79 10.4   6523:52 java                                                                                                                                                                          
-13268 hadoop	18  -2 8967m 8.2g 4104 S   21 35.1   5170:30 java                                                                                                                                                                          
+
+  PID USER  	PR  NI  VIRT  RES  SHR S %CPU %MEM	TIME+  COMMAND
+15558 hadoop	18  -2 3292m 2.4g 3556 S   79 10.4   6523:52 java
+13268 hadoop	18  -2 8967m 8.2g 4104 S   21 35.1   5170:30 java
  8895 hadoop	18  -2 1581m 497m 3420 S   11  2.1   4002:32 java
 …
         </programlisting>
@@ -351,7 +351,7 @@ hadoop@sv4borg12:~$ jps
         <programlisting>
 hadoop@sv4borg12:~$ ps aux | grep HRegionServer
 hadoop   17789  155 35.2 9067824 8604364 ?     S&lt;l  Mar04 9855:48 /usr/java/jdk1.6.0_14/bin/java -Xmx8000m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -XX:+UseConcMarkSweepGC -XX:NewSize=64m -XX:MaxNewSize=64m -XX:CMSInitiatingOccupancyFraction=88 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/export1/hadoop/logs/gc-hbase.log -Dcom.sun.management.jmxremote.port=10102 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=/home/hadoop/hbase/conf/jmxremote.password -Dcom.sun.management.jmxremote -Dhbase.log.dir=/export1/hadoop/logs -Dhbase.log.file=hbase-hadoop-regionserver-sv4borg12.log -Dhbase.home.dir=/home/hadoop/hbase -Dhbase.id.str=hadoop -Dhbase.root.logger=INFO,DRFA -Djava.library.path=/home/hadoop/hbase/lib/native/Linux-amd64-64 -classpath /home/hadoop/hbase/bin/../conf:[many jars]:/home/hadoop/hadoop/conf org.apache.hadoop.hbase.regionserver.HRegionServer start
-        </programlisting>      
+        </programlisting>
         </para>
       </section>
       <section xml:id="trouble.tools.jstack">
@@ -371,7 +371,7 @@ hadoop   17789  155 35.2 9067824 8604364 ?     S&lt;l  Mar04 9855:48 /usr/java/j
         	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:395)
         	at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:647)
         	at java.lang.Thread.run(Thread.java:619)
- 
+
         	The MemStore flusher thread that is currently flushing to a file:
 "regionserver60020.cacheFlusher" daemon prio=10 tid=0x0000000040f4e000 nid=0x45eb in Object.wait() [0x00007f16b5b86000..0x00007f16b5b87af0]
    java.lang.Thread.State: WAITING (on object monitor)
@@ -444,7 +444,7 @@ hadoop   17789  155 35.2 9067824 8604364 ?     S&lt;l  Mar04 9855:48 /usr/java/j
         </para>
         <para>
         	A thread that receives data from HDFS:
-        <programlisting>        	
+        <programlisting>
 "IPC Client (47) connection to sv4borg9/10.4.24.40:9000 from hadoop" daemon prio=10 tid=0x00007f16a02d0000 nid=0x4fa3 runnable [0x00007f16b517d000..0x00007f16b517dbf0]
    java.lang.Thread.State: RUNNABLE
         	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
@@ -498,63 +498,75 @@ hadoop   17789  155 35.2 9067824 8604364 ?     S&lt;l  Mar04 9855:48 /usr/java/j
         <section xml:id="trouble.tools.opentsdb">
           <title>OpenTSDB</title>
           <para>
-          <link xlink:href="http://opentsdb.net">OpenTSDB</link> is an excellent alternative to Ganglia as it uses HBase to store all the time series and doesn’t have to downsample. Monitoring your own HBase cluster that hosts OpenTSDB is a good exercise.
+          <link xlink:href="http://opentsdb.net">OpenTSDB</link> is an excellent alternative to Ganglia as it uses Apache HBase to store all the time series and doesn’t have to downsample. Monitoring your own HBase cluster that hosts OpenTSDB is a good exercise.
           </para>
           <para>
           Here’s an example of a cluster that’s suffering from hundreds of compactions launched almost all around the same time, which severely affects the IO performance:  (TODO:  insert graph plotting compactionQueueSize)
           </para>
           <para>
-          It’s a good practice to build dashboards with all the important graphs per machine and per cluster so that debugging issues can be done with a single quick look. For example, at StumbleUpon there’s one dashboard per cluster with the most important metrics from both the OS and HBase. You can then go down at the machine level and get even more detailed metrics.
+          It’s a good practice to build dashboards with all the important graphs per machine and per cluster so that debugging issues can be done with a single quick look. For example, at StumbleUpon there’s one dashboard per cluster with the most important metrics from both the OS and Apache HBase. You can then go down at the machine level and get even more detailed metrics.
           </para>
        </section>
        <section xml:id="trouble.tools.clustersshtop">
         <title>clusterssh+top</title>
-         <para> 
-          clusterssh+top, it’s like a poor man’s monitoring system and it can be quite useful when you have only a few machines as it’s very easy to setup. Starting clusterssh will give you one terminal per machine and another terminal in which whatever you type will be retyped in every window. This means that you can type “top” once and it will start it for all of your machines at the same time giving you full view of the current state of your cluster. You can also tail all the logs at the same time, edit files, etc.      
+         <para>
+          clusterssh+top, it’s like a poor man’s monitoring system and it can be quite useful when you have only a few machines as it’s very easy to setup. Starting clusterssh will give you one terminal per machine and another terminal in which whatever you type will be retyped in every window. This means that you can type “top” once and it will start it for all of your machines at the same time giving you full view of the current state of your cluster. You can also tail all the logs at the same time, edit files, etc.
           </para>
        </section>
     </section>
     </section>
-    
+
     <section xml:id="trouble.client">
       <title>Client</title>
-       <para>For more information on the HBase client, see <xref linkend="client"/>. 
+       <para>For more information on the HBase client, see <xref linkend="client"/>.
        </para>
        <section xml:id="trouble.client.scantimeout">
             <title>ScannerTimeoutException or UnknownScannerException</title>
-            <para>This is thrown if the time between RPC calls from the client to RegionServer exceeds the scan timeout.  
+            <para>This is thrown if the time between RPC calls from the client to RegionServer exceeds the scan timeout.
             For example, if <code>Scan.setCaching</code> is set to 500, then there will be an RPC call to fetch the next batch of rows every 500 <code>.next()</code> calls on the ResultScanner
             because data is being transferred in blocks of 500 rows to the client.  Reducing the setCaching value may be an option, but setting this value too low makes for inefficient
             processing on numbers of rows.
             </para>
             <para>See <xref linkend="perf.hbase.client.caching"/>.
             </para>
-       </section>    
+       </section>
+       <section xml:id="trouble.client.lease.exception">
+            <title><classname>LeaseException</classname> when calling <classname>Scanner.next</classname></title>
+            <para>
+In some situations clients that fetch data from a RegionServer get a LeaseException instead of the usual
+<xref linkend="trouble.client.scantimeout" />.  Usually the source of the exception is
+<classname>org.apache.hadoop.hbase.regionserver.Leases.removeLease(Leases.java:230)</classname> (line number may vary).
+It tends to happen in the context of a slow/freezing RegionServer#next call.
+It can be prevented by having <varname>hbase.rpc.timeout</varname> > <varname>hbase.regionserver.lease.period</varname>.
+Harsh J investigated the issue as part of the mailing list thread
+<link xlink:href="http://mail-archives.apache.org/mod_mbox/hbase-user/201209.mbox/%3CCAOcnVr3R-LqtKhFsk8Bhrm-YW2i9O6J6Fhjz2h7q6_sxvwd2yw%40mail.gmail.com%3E">HBase, mail # user - Lease does not exist exceptions</link>
+            </para>
+       </section>
        <section xml:id="trouble.client.scarylogs">
             <title>Shell or client application throws lots of scary exceptions during normal operation</title>
             <para>Since 0.20.0 the default log level for <code>org.apache.hadoop.hbase.*</code>is DEBUG. </para>
             <para>
-            On your clients, edit <filename>$HBASE_HOME/conf/log4j.properties</filename> and change this: <code>log4j.logger.org.apache.hadoop.hbase=DEBUG</code> to this: <code>log4j.logger.org.apache.hadoop.hbase=INFO</code>, or even <code>log4j.logger.org.apache.hadoop.hbase=WARN</code>. 
+            On your clients, edit <filename>$HBASE_HOME/conf/log4j.properties</filename> and change this: <code>log4j.logger.org.apache.hadoop.hbase=DEBUG</code> to this: <code>log4j.logger.org.apache.hadoop.hbase=INFO</code>, or even <code>log4j.logger.org.apache.hadoop.hbase=WARN</code>.
             </para>
-       </section>    
+       </section>
        <section xml:id="trouble.client.longpauseswithcompression">
             <title>Long Client Pauses With Compression</title>
-            <para>This is a fairly frequent question on the HBase dist-list.  The scenario is that a client is typically inserting a lot of data into a 
+            <para>This is a fairly frequent question on the Apache HBase dist-list.  The scenario is that a client is typically inserting a lot of data into a
             relatively un-optimized HBase cluster.  Compression can exacerbate the pauses, although it is not the source of the problem.</para>
             <para>See <xref linkend="precreate.regions"/> on the pattern for pre-creating regions and confirm that the table isn't starting with a single region.</para>
-            <para>See <xref linkend="perf.configurations"/> for cluster configuration, particularly <code>hbase.hstore.blockingStoreFiles</code>, <code>hbase.hregion.memstore.block.multiplier</code>, 
+            <para>See <xref linkend="perf.configurations"/> for cluster configuration, particularly <code>hbase.hstore.blockingStoreFiles</code>, <code>hbase.hregion.memstore.block.multiplier</code>,
             <code>MAX_FILESIZE</code> (region size), and <code>MEMSTORE_FLUSHSIZE.</code>  </para>
-            <para>A slightly longer explanation of why pauses can happen is as follows:  Puts are sometimes blocked on the MemStores which are blocked by the flusher thread which is blocked because there are 
+            <para>A slightly longer explanation of why pauses can happen is as follows:  Puts are sometimes blocked on the MemStores which are blocked by the flusher thread which is blocked because there are
             too many files to compact because the compactor is given too many small files to compact and has to compact the same data repeatedly.  This situation can occur even with minor compactions.
-            Compounding this situation, HBase doesn't compress data in memory.  Thus, the 64MB that lives in the MemStore could become a 6MB file after compression - which results in a smaller StoreFile.  The upside is that
+            Compounding this situation, Apache HBase doesn't compress data in memory.  Thus, the 64MB that lives in the MemStore could become a 6MB file after compression - which results in a smaller StoreFile.  The upside is that
             more data is packed into the same region, but performance is achieved by being able to write larger files - which is why HBase waits until the flushize before writing a new StoreFile.  And smaller StoreFiles
-            become targets for compaction.  Without compression the files are much bigger and don't need as much compaction, however this is at the expense of I/O.   
+            become targets for compaction.  Without compression the files are much bigger and don't need as much compaction, however this is at the expense of I/O.
             </para>
             <para>
             For additional information, see this thread on <link xlink:href="http://search-hadoop.com/m/WUnLM6ojHm1/Long+client+pauses+with+compression&amp;subj=Long+client+pauses+with+compression">Long client pauses with compression</link>.
             </para>
-            
-       </section>    
+
+       </section>
        <section xml:id="trouble.client.zookeeper">
             <title>ZooKeeper Client Connection Errors</title>
             <para>Errors like this...
@@ -576,11 +588,11 @@ hadoop   17789  155 35.2 9067824 8604364 ?     S&lt;l  Mar04 9855:48 /usr/java/j
  11/07/05 11:26:45 INFO zookeeper.ClientCnxn: Opening socket connection to
  server localhost/127.0.0.1:2181
 </programlisting>
-            ... are either due to ZooKeeper being down, or unreachable due to network issues.            
+            ... are either due to ZooKeeper being down, or unreachable due to network issues.
             </para>
             <para>The utility <xref linkend="trouble.tools.builtin.zkcli"/> may help investigate ZooKeeper issues.
             </para>
-       </section>    
+       </section>
        <section xml:id="trouble.client.oome.directmemory.leak">
             <title>Client running out of memory though heap size seems to be stable (but the off-heap/direct heap keeps growing)</title>
             <para>
@@ -595,15 +607,15 @@ it  a bit hefty.  You want to make this setting client-side only especially if y
 server-side off-heap cache since this feature depends on being able to use big direct buffers (You may have to keep
 separate client-side and server-side config dirs).
             </para>
-       </section>    
+       </section>
        <section xml:id="trouble.client.slowdown.admin">
             <title>Client Slowdown When Calling Admin Methods (flush, compact, etc.)</title>
             <para>
 This is a client issue fixed by <link xlink:href="https://issues.apache.org/jira/browse/HBASE-5073">HBASE-5073</link> in 0.90.6.
-There was a ZooKeeper leak in the client and the client was getting pummeled by ZooKeeper events with each additional 
-invocation of the admin API. 
+There was a ZooKeeper leak in the client and the client was getting pummeled by ZooKeeper events with each additional
+invocation of the admin API.
             </para>
-       </section>    
+       </section>
 
        <section xml:id="trouble.client.security.rpc">
            <title>Secure Client Cannot Connect ([Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)])</title>
@@ -611,7 +623,7 @@ invocation of the admin API.
 There can be several causes that produce this symptom.
            </para>
            <para>
-First, check that you have a valid Kerberos ticket. One is required in order to set up communication with a secure HBase cluster. Examine the ticket currently in the credential cache, if any, by running the klist command line utility. If no ticket is listed, you must obtain a ticket by running the kinit command with either a keytab specified, or by interactively entering a password for the desired principal.
+First, check that you have a valid Kerberos ticket. One is required in order to set up communication with a secure Apache HBase cluster. Examine the ticket currently in the credential cache, if any, by running the klist command line utility. If no ticket is listed, you must obtain a ticket by running the kinit command with either a keytab specified, or by interactively entering a password for the desired principal.
            </para>
            <para>
 Then, consult the <link xlink:href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jgss/tutorials/Troubleshooting.html">Java Security Guide troubleshooting section</link>. The most common problem addressed there is resolved by setting javax.security.auth.useSubjectCredsOnly system property value to false.
@@ -625,16 +637,16 @@ Finally, depending on your Kerberos configuration, you may need to install the <
            <para>
 You may also need to download the <link xlink:href="http://www.oracle.com/technetwork/java/javase/downloads/jce-6-download-429243.html">unlimited strength JCE policy files</link>. Uncompress and extract the downloaded file, and install the policy jars into &lt;java-home&gt;/lib/security.
            </para>
-       </section>    
+       </section>
 
     </section>
-    
+
     <section xml:id="trouble.mapreduce">
       <title>MapReduce</title>
       <section xml:id="trouble.mapreduce.local">
         <title>You Think You're On The Cluster, But You're Actually Local</title>
         <para>This following stacktrace happened using <code>ImportTsv</code>, but things like this
-        can happen on any job with a mis-configuration.        
+        can happen on any job with a mis-configuration.
 <programlisting>
     WARN mapred.LocalJobRunner: job_local_0001
 java.lang.IllegalArgumentException: Can't read partitions file
@@ -659,17 +671,17 @@ Caused by: java.io.FileNotFoundException: File _partition.lst does not exist.
 </programlisting>
        LocalJobRunner means the job is running locally, not on the cluster.
       </para>
-      <para>See 
+      <para>See
       <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#classpath">
-      http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#classpath</link> for more 
+      http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#classpath</link> for more
       information on HBase MapReduce jobs and classpaths.
-      </para>        
+      </para>
       </section>
     </section>
-    
+
     <section xml:id="trouble.namenode">
       <title>NameNode</title>
-       <para>For more information on the NameNode, see <xref linkend="arch.hdfs"/>. 
+       <para>For more information on the NameNode, see <xref linkend="arch.hdfs"/>.
        </para>
        <section xml:id="trouble.namenode.disk">
             <title>HDFS Utilization of Tables and Regions</title>
@@ -679,7 +691,7 @@ Caused by: java.io.FileNotFoundException: File _partition.lst does not exist.
             <para><programlisting>hadoop fs -du /hbase/myTable</programlisting> ...returns a list of the regions under the HBase table 'myTable' and their disk utilization. </para>
             <para>For more information on HDFS shell commands, see the <link xlink:href="http://hadoop.apache.org/common/docs/current/file_system_shell.html">HDFS FileSystem Shell documentation</link>.
             </para>
-       </section>    
+       </section>
        <section xml:id="trouble.namenode.hbase.objects">
             <title>Browsing HDFS for HBase Objects</title>
             <para>Somtimes it will be necessary to explore the HBase objects that exist on HDFS.  These objects could include the WALs (Write Ahead Logs), tables, regions, StoreFiles, etc.
@@ -697,30 +709,30 @@ Caused by: java.io.FileNotFoundException: File _partition.lst does not exist.
             <para>The HDFS directory structure of HBase WAL is..
             <programlisting>
 <filename>/hbase</filename>
-     <filename>/.logs</filename>     
+     <filename>/.logs</filename>
           <filename>/&lt;RegionServer&gt;</filename>    (RegionServers)
                <filename>/&lt;HLog&gt;</filename>           (WAL HLog files for the RegionServer)
             </programlisting>
             </para>
-		    <para>See the <link xlink:href="see http://hadoop.apache.org/common/docs/current/hdfs_user_guide.html">HDFS User Guide</link> for other non-shell diagnostic 
-		    utilities like <code>fsck</code>. 
+		    <para>See the <link xlink:href="see http://hadoop.apache.org/common/docs/current/hdfs_user_guide.html">HDFS User Guide</link> for other non-shell diagnostic
+		    utilities like <code>fsck</code>.
             </para>
           <section xml:id="trouble.namenode.uncompaction">
             <title>Use Cases</title>
-              <para>Two common use-cases for querying HDFS for HBase objects is research the degree of uncompaction of a table.  If there are a large number of StoreFiles for each ColumnFamily it could 
+              <para>Two common use-cases for querying HDFS for HBase objects is research the degree of uncompaction of a table.  If there are a large number of StoreFiles for each ColumnFamily it could
               indicate the need for a major compaction.  Additionally, after a major compaction if the resulting StoreFile is "small" it could indicate the need for a reduction of ColumnFamilies for
               the table.
 		    </para>
 		  </section>
 
-       </section>    
+       </section>
      </section>
-        
+
     <section xml:id="trouble.network">
       <title>Network</title>
       <section xml:id="trouble.network.spikes">
         <title>Network Spikes</title>
-        <para>If you are seeing periodic network spikes you might want to check the <code>compactionQueues</code> to see if major 
+        <para>If you are seeing periodic network spikes you might want to check the <code>compactionQueues</code> to see if major
         compactions are happening.
         </para>
         <para>See <xref linkend="managed.compactions"/> for more information on managing compactions.
@@ -731,11 +743,17 @@ Caused by: java.io.FileNotFoundException: File _partition.lst does not exist.
         <para>HBase expects the loopback IP Address to be 127.0.0.1.  See the Getting Started section on <xref linkend="loopback.ip" />.
         </para>
        </section>
+      <section xml:id="trouble.network.ints">
+        <title>Network Interfaces</title>
+        <para>Are all the network interfaces functioning correctly?  Are you sure?  See the Troubleshooting Case Study in <xref linkend="trouble.casestudy"/>.
+        </para>
+      </section>
+
     </section>
-        
+
     <section xml:id="trouble.rs">
       <title>RegionServer</title>
-        <para>For more information on the RegionServers, see <xref linkend="regionserver.arch"/>. 
+        <para>For more information on the RegionServers, see <xref linkend="regionserver.arch"/>.
        </para>
       <section xml:id="trouble.rs.startup">
         <title>Startup Errors</title>
@@ -743,9 +761,9 @@ Caused by: java.io.FileNotFoundException: File _partition.lst does not exist.
             <title>Master Starts, But RegionServers Do Not</title>
             <para>The Master believes the RegionServers have the IP of 127.0.0.1 - which is localhost and resolves to the master's own localhost.
             </para>
-            <para>The RegionServers are erroneously informing the Master that their IP addresses are 127.0.0.1. 
+            <para>The RegionServers are erroneously informing the Master that their IP addresses are 127.0.0.1.
             </para>
-            <para>Modify <filename>/etc/hosts</filename> on the region servers, from...  
+            <para>Modify <filename>/etc/hosts</filename> on the region servers, from...
             <programlisting>
 # Do not remove the following line, or various programs
 # that require network functionality will fail.
@@ -761,7 +779,7 @@ Caused by: java.io.FileNotFoundException: File _partition.lst does not exist.
             </programlisting>
             </para>
           </section>
-          
+
           <section xml:id="trouble.rs.startup.compression">
             <title>Compression Link Errors</title>
             <para>
@@ -775,8 +793,8 @@ java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path
             </programlisting>
             .. then there is a path issue with the compression libraries.  See the Configuration section on <link linkend="lzo.compression">LZO compression configuration</link>.
             </para>
-          </section> 
-      </section>    
+          </section>
+      </section>
       <section xml:id="trouble.rs.runtime">
         <title>Runtime Errors</title>
 
@@ -789,7 +807,7 @@ java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path
             Adding <code>-XX:+UseMembar</code> to the HBase <varname>HBASE_OPTS</varname> in <filename>conf/hbase-env.sh</filename>
             may fix it.
             </para>
-            <para>Also, are you using <xref linkend="client.rowlocks"/>?  These are discouraged because they can lock up the 
+            <para>Also, are you using <xref linkend="client.rowlocks"/>?  These are discouraged because they can lock up the
             RegionServers if not managed properly.
             </para>
         </section>
@@ -798,7 +816,7 @@ java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path
            <para>
            If you see log messages like this...
 <programlisting>
-2010-09-13 01:24:17,336 WARN org.apache.hadoop.hdfs.server.datanode.DataNode: 
+2010-09-13 01:24:17,336 WARN org.apache.hadoop.hdfs.server.datanode.DataNode:
 Disk-related IOException in BlockReceiver constructor. Cause is java.io.IOException: Too many open files
         at java.io.UnixFileSystem.createFileExclusively(Native Method)
         at java.io.File.createNewFile(File.java:883)
@@ -829,7 +847,7 @@ Disk-related IOException in BlockReceiver constructor. Cause is java.io.IOExcept
            <programlisting>
 2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 10000
 2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 15000
-2009-02-24 10:01:36,472 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: unable to report to master for xxx milliseconds - retrying      
+2009-02-24 10:01:36,472 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: unable to report to master for xxx milliseconds - retrying
            </programlisting>
            ... or see full GC compactions then you may be experiencing full GC's.
            </para>
@@ -860,12 +878,12 @@ java.io.IOException: Session Expired
        at org.apache.zookeeper.ClientCnxn$SendThread.readConnectResult(ClientCnxn.java:589)
        at org.apache.zookeeper.ClientCnxn$SendThread.doIO(ClientCnxn.java:709)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:945)
-ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expired           
+ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expired
            </programlisting>
            <para>
            The JVM is doing a long running garbage collecting which is pausing every threads (aka "stop the world").
            Since the RegionServer's local ZooKeeper client cannot send heartbeats, the session times out.
-           By design, we shut down any node that isn't able to contact the ZooKeeper ensemble after getting a timeout so that it stops serving data that may already be assigned elsewhere.  
+           By design, we shut down any node that isn't able to contact the ZooKeeper ensemble after getting a timeout so that it stops serving data that may already be assigned elsewhere.
            </para>
            <para>
             <itemizedlist>
@@ -874,7 +892,7 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
               <listitem>Make sure you are not CPU starving the RegionServer thread. For example, if you are running a MapReduce job using 6 CPU-intensive tasks on a machine with 4 cores, you are probably starving the RegionServer enough to create longer garbage collection pauses.</listitem>
               <listitem>Increase the ZooKeeper session timeout</listitem>
            </itemizedlist>
-           If you wish to increase the session timeout, add the following to your <filename>hbase-site.xml</filename> to increase the timeout from the default of 60 seconds to 120 seconds. 
+           If you wish to increase the session timeout, add the following to your <filename>hbase-site.xml</filename> to increase the timeout from the default of 60 seconds to 120 seconds.
            <programlisting>
 &lt;property&gt;
     &lt;name&gt;zookeeper.session.timeout&lt;/name&gt;
@@ -888,8 +906,8 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
            </para>
            <para>
            Be aware that setting a higher timeout means that the regions served by a failed RegionServer will take at least
-           that amount of time to be transfered to another RegionServer. For a production system serving live requests, we would instead 
-           recommend setting it lower than 1 minute and over-provision your cluster in order the lower the memory load on each machines (hence having 
+           that amount of time to be transfered to another RegionServer. For a production system serving live requests, we would instead
+           recommend setting it lower than 1 minute and over-provision your cluster in order the lower the memory load on each machines (hence having
            less garbage to collect per machine).
            </para>
            <para>
@@ -906,7 +924,7 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
         <section xml:id="trouble.rs.runtime.double_listed_regions">
            <title>Regions listed by domain name, then IP</title>
            <para>
-           Fix your DNS.  In versions of HBase before 0.92.x, reverse DNS needs to give same answer
+           Fix your DNS.  In versions of Apache HBase before 0.92.x, reverse DNS needs to give same answer
            as forward lookup. See <link xlink:href="https://issues.apache.org/jira/browse/HBASE-3431">HBASE 3431
            RegionServer is not using the name given it by the master; double entry in master listing of servers</link> for gorey details.
           </para>
@@ -930,35 +948,41 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
            </para>
         </section>
 
-      </section>    
+      </section>
       <section xml:id="trouble.rs.shutdown">
         <title>Shutdown Errors</title>
 
-      </section>    
+      </section>
 
-    </section>    
+    </section>
 
     <section xml:id="trouble.master">
       <title>Master</title>
-       <para>For more information on the Master, see <xref linkend="master"/>. 
+       <para>For more information on the Master, see <xref linkend="master"/>.
        </para>
       <section xml:id="trouble.master.startup">
         <title>Startup Errors</title>
           <section xml:id="trouble.master.startup.migration">
              <title>Master says that you need to run the hbase migrations script</title>
              <para>Upon running that, the hbase migrations script says no files in root directory.</para>
-             <para>HBase expects the root directory to either not exist, or to have already been initialized by hbase running a previous time. If you create a new directory for HBase using Hadoop DFS, this error will occur. 
-             Make sure the HBase root directory does not currently exist or has been initialized by a previous run of HBase. Sure fire solution is to just use Hadoop dfs to delete the HBase root and let HBase create and initialize the directory itself. 
-             </para>          
+             <para>HBase expects the root directory to either not exist, or to have already been initialized by hbase running a previous time. If you create a new directory for HBase using Hadoop DFS, this error will occur.
+             Make sure the HBase root directory does not currently exist or has been initialized by a previous run of HBase. Sure fire solution is to just use Hadoop dfs to delete the HBase root and let HBase create and initialize the directory itself.
+             </para>
+          </section>
+          <section xml:id="trouble.master.startup.zk.buffer">
+              <title>Packet len6080218 is out of range!</title>
+              <para>If you have many regions on your cluster and you see an error
+                  like that reported above in this sections title in your logs, see
+                  <link xlink:href="https://issues.apache.org/jira/browse/HBASE-4246">HBASE-4246 Cluster with too many regions cannot withstand some master failover scenarios</link>.</para>
           </section>
-          
-      </section>    
+
+      </section>
       <section xml:id="trouble.master.shutdown">
         <title>Shutdown Errors</title>
 
-      </section>    
+      </section>
 
-    </section>    
+    </section>
 
     <section xml:id="trouble.zookeeper">
       <title>ZooKeeper</title>
@@ -967,28 +991,28 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
           <section xml:id="trouble.zookeeper.startup.address">
              <title>Could not find my address: xyz in list of ZooKeeper quorum servers</title>
              <para>A ZooKeeper server wasn't able to start, throws that error. xyz is the name of your server.</para>
-             <para>This is a name lookup problem. HBase tries to start a ZooKeeper server on some machine but that machine isn't able to find itself in the <varname>hbase.zookeeper.quorum</varname> configuration.  
-             </para>          
-             <para>Use the hostname presented in the error message instead of the value you used. If you have a DNS server, you can set <varname>hbase.zookeeper.dns.interface</varname> and <varname>hbase.zookeeper.dns.nameserver</varname> in <filename>hbase-site.xml</filename> to make sure it resolves to the correct FQDN.   
-             </para>          
+             <para>This is a name lookup problem. HBase tries to start a ZooKeeper server on some machine but that machine isn't able to find itself in the <varname>hbase.zookeeper.quorum</varname> configuration.
+             </para>
+             <para>Use the hostname presented in the error message instead of the value you used. If you have a DNS server, you can set <varname>hbase.zookeeper.dns.interface</varname> and <varname>hbase.zookeeper.dns.nameserver</varname> in <filename>hbase-site.xml</filename> to make sure it resolves to the correct FQDN.
+             </para>
           </section>
-          
-      </section>    
+
+      </section>
       <section xml:id="trouble.zookeeper.general">
           <title>ZooKeeper, The Cluster Canary</title>
           <para>ZooKeeper is the cluster's "canary in the mineshaft". It'll be the first to notice issues if any so making sure its happy is the short-cut to a humming cluster.
-          </para> 
+          </para>
           <para>
           See the <link xlink:href="http://wiki.apache.org/hadoop/ZooKeeper/Troubleshooting">ZooKeeper Operating Environment Troubleshooting</link> page. It has suggestions and tools for checking disk and networking performance; i.e. the operating environment your ZooKeeper and HBase are running in.
           </para>
          <para>Additionally, the utility <xref linkend="trouble.tools.builtin.zkcli"/> may help investigate ZooKeeper issues.
          </para>
-      </section>  
+      </section>
 
-    </section>    
+    </section>
 
     <section xml:id="trouble.ec2">
-       <title>Amazon EC2</title>      
+       <title>Amazon EC2</title>
           <section xml:id="trouble.ec2.zookeeper">
              <title>ZooKeeper does not seem to work on Amazon EC2</title>
              <para>HBase does not start when deployed as Amazon EC2 instances.  Exceptions like the below appear in the Master and/or RegionServer logs: </para>
@@ -1000,8 +1024,8 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
   java.net.ConnectException: Connection refused
              </programlisting>
              <para>
-             Security group policy is blocking the ZooKeeper port on a public address. 
-             Use the internal EC2 host names when configuring the ZooKeeper quorum peer list. 
+             Security group policy is blocking the ZooKeeper port on a public address.
+             Use the internal EC2 host names when configuring the ZooKeeper quorum peer list.
              </para>
           </section>
           <section xml:id="trouble.ec2.instability">
@@ -1015,15 +1039,15 @@ ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expi
              See Andrew's answer here, up on the user list: <link xlink:href="http://search-hadoop.com/m/sPdqNFAwyg2">Remote Java client connection into EC2 instance</link>.
              </para>
           </section>
-          
+
     </section>
-    
+
     <section xml:id="trouble.versions">
-       <title>HBase and Hadoop version issues</title>      
+       <title>HBase and Hadoop version issues</title>
           <section xml:id="trouble.versions.205">
              <title><code>NoClassDefFoundError</code> when trying to run 0.90.x on hadoop-0.20.205.x (or hadoop-1.0.x)</title>
-             <para>HBase 0.90.x does not ship with hadoop-0.20.205.x, etc.  To make it run, you need to replace the hadoop
-             jars that HBase shipped with in its <filename>lib</filename> directory with those of the Hadoop you want to
+             <para>Apache HBase 0.90.x does not ship with hadoop-0.20.205.x, etc.  To make it run, you need to replace the hadoop
+             jars that Apache HBase shipped with in its <filename>lib</filename> directory with those of the Hadoop you want to
              run HBase on.  If even after replacing Hadoop jars you get the below exception:
 <programlisting>
 sv4r6s38: Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/configuration/Configuration
@@ -1040,7 +1064,27 @@ sv4r6s38:       at org.apache.hadoop.security.UserGroupInformation.ensureInitial
 you need to copy under <filename>hbase/lib</filename>, the <filename>commons-configuration-X.jar</filename> you find
 in your Hadoop's <filename>lib</filename> directory.  That should fix the above complaint.
 </para>
+</section>
+
+          <section xml:id="trouble.versions.205">
+             <title>...cannot communicate with client version...</title>
+<para>If you see something like the following in your logs
+<computeroutput>...
+2012-09-24 10:20:52,168 FATAL org.apache.hadoop.hbase.master.HMaster: Unhandled exception. Starting shutdown.
+org.apache.hadoop.ipc.RemoteException: Server IPC version 7 cannot communicate with client version 4
+...</computeroutput>
+...are you trying to talk to an Hadoop 2.0.x from an HBase that has an Hadoop 1.0.x client?
+Use the HBase built against Hadoop 2.0 or rebuild your HBase passing the <command>-Dhadoop.profile=2.0</command>
+attribute to Maven (See <xref linkend="maven.build.hadoop" /> for more).
+</para>
+
 </section>
 </section>
-    
+
+    <section xml:id="trouble.casestudy">
+      <title>Case Studies</title>
+      <para>For Performance and Troubleshooting Case Studies, see <xref linkend="casestudies"/>.
+      </para>
+    </section>
+
   </chapter>
diff --git a/src/docbkx/upgrading.xml b/src/docbkx/upgrading.xml
index 5a1887284cf4..d1dcdd8c0e36 100644
--- a/src/docbkx/upgrading.xml
+++ b/src/docbkx/upgrading.xml
@@ -27,49 +27,29 @@
  */
 -->
     <title>Upgrading</title>
+    <para>You cannot skip major verisons upgrading.  If you are upgrading from
+    version 0.20.x to 0.92.x, you must first go from 0.20.x to 0.90.x and then go
+    from 0.90.x to 0.92.x.</para>
     <para>
         Review <xref linkend="configuration" />, in particular the section on Hadoop version.
     </para>
-    <section xml:id="upgrade0.90">
-    <title>Upgrading to HBase 0.90.x from 0.20.x or 0.89.x</title>
-          <para>This version of 0.90.x HBase can be started on data written by
-              HBase 0.20.x or HBase 0.89.x.  There is no need of a migration step.
-              HBase 0.89.x and 0.90.x does write out the name of region directories
-              differently -- it names them with a md5 hash of the region name rather
-              than a jenkins hash -- so this means that once started, there is no
-              going back to HBase 0.20.x.
-          </para>
-          <para>
-             Be sure to remove the <filename>hbase-default.xml</filename> from
-             your <filename>conf</filename>
-             directory on upgrade.  A 0.20.x version of this file will have
-             sub-optimal configurations for 0.90.x HBase.  The
-             <filename>hbase-default.xml</filename> file is now bundled into the
-             HBase jar and read from there.  If you would like to review
-             the content of this file, see it in the src tree at
-             <filename>src/main/resources/hbase-default.xml</filename> or
-             see <xref linkend="hbase_default_configurations" />.
-          </para>
-          <para>
-            Finally, if upgrading from 0.20.x, check your 
-            <varname>.META.</varname> schema in the shell.  In the past we would
-            recommend that users run with a 16kb
-            <varname>MEMSTORE_FLUSHSIZE</varname>.
-            Run <code>hbase> scan '-ROOT-'</code> in the shell. This will output
-            the current <varname>.META.</varname> schema.  Check
-            <varname>MEMSTORE_FLUSHSIZE</varname> size.  Is it 16kb (16384)?  If so, you will
-            need to change this (The 'normal'/default value is 64MB (67108864)).
-            Run the script <filename>bin/set_meta_memstore_size.rb</filename>.
-            This will make the necessary edit to your <varname>.META.</varname> schema.
-            Failure to run this change will make for a slow cluster <footnote>
-            <para>
-            See <link xlink:href="https://issues.apache.org/jira/browse/HBASE-3499">HBASE-3499 Users upgrading to 0.90.0 need to have their .META. table updated with the right MEMSTORE_SIZE</link>
-            </para>
-            </footnote>
-            .
-
-          </para>
-          </section>
+    <section xml:id="upgrade0.96">
+      <title>Upgrading from 0.94.x to 0.96.x</title>
+      <subtitle>The Singularity</subtitle>
+      <para>You will have to stop your old 0.94 cluster completely to upgrade.  If you are replicating
+     between clusters, both clusters will have to go down to upgrade.  Make sure it is a clean shutdown
+     so there are no WAL files laying around (TODO: Can 0.96 read 0.94 WAL files?).  Make sure
+     zookeeper is cleared of state.  All clients must be upgraded to 0.96 too.
+ </para>
+ <para>The API has changed in a few areas; in particular how you use coprocessors (TODO: MapReduce too?)
+ </para>
+ <para>TODO: Write about 3.4 zk ensemble and multi support</para>
+    </section>
+    <section xml:id="upgrade0.94">
+      <title>Upgrading from 0.92.x to 0.94.x</title>
+    <para>0.92 and 0.94 are interface compatible.  You can do a rolling upgrade between these versions.
+    </para>
+    </section>
     <section xml:id="upgrade0.92">
       <title>Upgrading from 0.90.x to 0.92.x</title>
       <subtitle>Upgrade Guide</subtitle>
@@ -170,7 +150,7 @@ The block size default size has been changed in 0.92.0 from 0.2 (20 percent of h
 <section><title>Experimental off-heap cache
 </title>
 <para>
-A new cache was contributed to 0.92.0 to act as a solution between using the “on-heap” cache which is the current LRU cache the region servers have and the operating system cache which is out of our control. 
+A new cache was contributed to 0.92.0 to act as a solution between using the “on-heap” cache which is the current LRU cache the region servers have and the operating system cache which is out of our control.
 To enable, set “-XX:MaxDirectMemorySize” in hbase-env.sh to the value for maximum direct memory size and specify hbase.offheapcache.percentage in hbase-site.xml with the percentage that you want to dedicate to off-heap cache. This should only be set for servers and not for clients. Use at your own risk.
 See this blog post for additional information on this new experimental feature: http://www.cloudera.com/blog/2012/01/caching-in-hbase-slabcache/
 </para>
@@ -194,8 +174,48 @@ See this blog post for additional information on this new experimental feature:
 </title>
 <para>0.92.0 stores data in a new format, <xref linkend="hfilev2" />.   As HBase runs, it will move all your data from HFile v1 to HFile v2 format.  This auto-migration will run in the background as flushes and compactions run.
 HFile V2 allows HBase run with larger regions/files.  In fact, we encourage that all HBasers going forward tend toward Facebook axiom #1, run with larger, fewer regions.
-If you have lots of regions now -- more than 100s per host -- you should look into setting your region size up after you move to 0.92.0 (In 0.92.0, default size is not 1G, up from 256M), and then running online merge tool (See “HBASE-1621 merge tool should work on online cluster, but disabled table”).
+If you have lots of regions now -- more than 100s per host -- you should look into setting your region size up after you move to 0.92.0 (In 0.92.0, default size is now 1G, up from 256M), and then running online merge tool (See “HBASE-1621 merge tool should work on online cluster, but disabled table”).
 </para>
 </section>
     </section>
+    <section xml:id="upgrade0.90">
+    <title>Upgrading to HBase 0.90.x from 0.20.x or 0.89.x</title>
+          <para>This version of 0.90.x HBase can be started on data written by
+              HBase 0.20.x or HBase 0.89.x.  There is no need of a migration step.
+              HBase 0.89.x and 0.90.x does write out the name of region directories
+              differently -- it names them with a md5 hash of the region name rather
+              than a jenkins hash -- so this means that once started, there is no
+              going back to HBase 0.20.x.
+          </para>
+          <para>
+             Be sure to remove the <filename>hbase-default.xml</filename> from
+             your <filename>conf</filename>
+             directory on upgrade.  A 0.20.x version of this file will have
+             sub-optimal configurations for 0.90.x HBase.  The
+             <filename>hbase-default.xml</filename> file is now bundled into the
+             HBase jar and read from there.  If you would like to review
+             the content of this file, see it in the src tree at
+             <filename>src/main/resources/hbase-default.xml</filename> or
+             see <xref linkend="hbase_default_configurations" />.
+          </para>
+          <para>
+            Finally, if upgrading from 0.20.x, check your
+            <varname>.META.</varname> schema in the shell.  In the past we would
+            recommend that users run with a 16kb
+            <varname>MEMSTORE_FLUSHSIZE</varname>.
+            Run <code>hbase> scan '-ROOT-'</code> in the shell. This will output
+            the current <varname>.META.</varname> schema.  Check
+            <varname>MEMSTORE_FLUSHSIZE</varname> size.  Is it 16kb (16384)?  If so, you will
+            need to change this (The 'normal'/default value is 64MB (67108864)).
+            Run the script <filename>bin/set_meta_memstore_size.rb</filename>.
+            This will make the necessary edit to your <varname>.META.</varname> schema.
+            Failure to run this change will make for a slow cluster <footnote>
+            <para>
+            See <link xlink:href="https://issues.apache.org/jira/browse/HBASE-3499">HBASE-3499 Users upgrading to 0.90.0 need to have their .META. table updated with the right MEMSTORE_SIZE</link>
+            </para>
+            </footnote>
+            .
+
+          </para>
+          </section>
     </chapter>
diff --git a/src/site/resources/css/site.css b/src/site/resources/css/site.css
index 7978259ab985..f26d03c39b5b 100644
--- a/src/site/resources/css/site.css
+++ b/src/site/resources/css/site.css
@@ -109,6 +109,11 @@ h4 {
   background-repeat: repeat-x;
 }
 
+.section {
+  padding-bottom: 0;
+  padding-top: 0;
+}
+
 /*
 #leftColumn {
   display: none !important
diff --git a/src/site/resources/images/hbase_logo.png b/src/site/resources/images/hbase_logo.png
index 615b0a80c8919bb8ec890cecaa62c49eebbb25fe..e962ce04975b3c487454da743cf746c975a5e004 100644
GIT binary patch
literal 2997
zcmV;m3rh5fP)<h;3K|Lk000e1NJLTq009dC002S=1^@s6Ky~v%00004b3#c}2nYxW
zd<bNS00009a7bBm000K`000K`0dwmhng9R*8FWQhbW?9;ba!ELWdL_~cP?peYja~^
zaAhuUa%Y?FJQ@H13o=PWK~#90?VWk79aSC2Kkv1Vvh;{h+f#)?AuQ5jK^7?l0tQDw
zP+2rYga83^h(-}Ns_eKx4H$@0JZL0@tsx-6R6`jzHlamUQ$R(npunJ&(xE_G`ksHB
zx#aftzP>Z-nVEM!$)vq=&-u;w-8=KUXZ`&KMn*;eP)bb)E(YcTvw?}EV+HUSaHrP#
z+LU#3qm;TEIHDkP{{S8auGL!KUy#kzD5bUmE&&b#J^*Z%(!^_+?_?2hmDYN=ScUc|
zrFI170sCU^uPrm0ny<B<SD3I(pp^POaA?=5#ainlbJSm=wVqpyx-qYPIdE=~HU|^K
z2Ln$43fPPEjsjo`5C995Qg<n(wkk|z_3Q+E2Kcp7>LeEelu~mr5Be7|4_I190kAdj
zVc<$&u~O>Y#VNE$DfM|^8E_V`KiM6H$038sL%k4?1={@taDIXIT(KE&lTvEmDle*(
zngHAk%q-CCM}W%;v}c7<>R8|!pvgZT9!$&PtH5?8*gi)owL=MZt7ZWBZpGI;1K6_!
zyPs1^%__leUzAc)fS*=%Jx{qz#QfrP^D3|acs-@9{eTm?UQYrJ25!l$E*~#o{*El1
z2JGJT`Us`eG_Cb3xmDTG`<=W2{0Q^iq+u3tO4sWNz~R8t*`=l85KQ;9n`eMu04p=v
zd?=Tq@ypx51$hii#0)ldvrucDx(lI{8UkXBljzWEyGmZvT3?=}qEc!p@SC2~y8=}?
zi{8&|k=8oXV3ktp2w+;*sX3WdrDva>*TcZ!TI-i8)Zj6!&-y%c7zDQMIhW~y&|0r!
z<6n#3UzMh?ywABZx@x1*`?=+LxO(eO&0#AoGkRXX;2lMC$ujFP9+=tZ+`p<8(E<is
z3n-Yqlu`>*l&@@j$bzAi+Phh;wLYVtv)EfHby`NFTV$+P1~Zbm<}7@=&$++m5tEZy
zIlhzLmZK*-(}5FRi^#|z*|on0^mP`WOpd1WK8jnE@=LAte+sgdg&;XR9%l@?7SI5O
zf%9C8@OWI~$1FC@8^Fn0>nB|c@zjFVq#L{1f%<gKM}WU%th)-@?JP!swHfRE4R`>!
zPHX*Gp{hJh=7F4hLtL^}YyJK_2KqURtAVRB+Is|ePj)$lvb3sAI~8y}aB0T6zXJ9s
zP)i#A*YkR+Qfji+I_-i#PYvkjELLi*muE4ilv<xpj%O5B<@iprw62Pxw^g<!aJ^FM
zX0or~d|qq)_k6O(C4)+-!*bLgZgV%p<DLhAb-+;9xlbiWRxSs=nMc$(WeE6Pj{0l-
zY>1~clGf101)FW#La)Z)orR}t<^#967U9W{cNU&f(OR$9TAzsNdOVAnZSpjbiI_-A
z-7L>x?|3$h==5R^TWR?Qra#+ZRfX3sn%>Vb&%Ts1fyrI3|C_^BX0+CK0Cy;*CSzu2
zvIfI>zBpB5@@1eqx>)b*PTKla-jmk>cNb`Hc%T!5)8p}YHihqocsw2}ytD9lJXUyT
z;qiE^7`SKJ<hLx!EeWF7*&6pyiVp$zSrolDh~js0hlEnh0hX2~EI%Wd0lTMwKPAVL
zK@=CVLNSzLDkj{($AO(O5!?mUtyLJogj#$BlZ#~$a90q;FBf4vlwv2$VDOQ^_L#q~
zZfOSQ1W~-v2Ch(wC784ujbS1i#>^ZIn=)lGXsMw$x5j{FJ(uj39Pg}Z45e5E%nPD;
zSr&t#6q7IsedYn17a=kSn`1)x%?6GEei%w|HE>A~#hLbqBu;QXCa`kd7io>wdY?r7
z@68QpTQ{V4YNx4FJ;wk~gi@TDwqhv749uR4i*fHL`Y{B24OkXRk$M+oD8(VbpMle<
zzoTdsZF3fFVj}R9PzvjP5TO*i0uNHJ*pH*J18`p`g>_Rrl;S90F|bW(syw68HfPb6
zmPI$;PR3p~Nhrkta3dQ>)UGpAfm=f<hQ=zGbUkjNKEY(h9c^<KZDax_{oh!e3LH?f
zmNM7__>x7r`E0lv(o->Qa~5rESpXayN^#0)Z-!F53kP{j3*@3uis_@h8A@>=W-!y^
zkhVFCHuJ4bymtX6Onp-k!1w#T5lYc%S^2?C$F$8^w6$dcuv;j_X#I6nsgiOyIh3N`
z=CT8T4^}OvKw5JT`bb%Z6}ZVbHi%h&VB@q0F|!<xcApHT*b(!~Y{PnB8P(pvv$4C+
z1~B6nf9yG*W@cwK@LY=WRj?*!{n|v9Z=18o+OoJkh~l$~7Gp!Q*5GHrffmL0q{mq#
z=WKWgI5mjkN;9j4QtXV`E`7K~@!4$bENqi&-wK=^M6t_cP=ymj@!qaAbZA>QBrDE7
zhaifd1#Yk`zI&f@w$9>)svQMDvUL5&mc{$EB5Ye0KWLkys2go_7Fk;st_RO-_pkH|
zo?`1PT<($igJrpXtq7Zz#hbtr_HsN~Xq&Uhifd;v!LnSx$YWD2>%L@Ju5f}VwvCw9
zFXxO+dWk{&y<9qN`+%F;ve@6UoXbq^u6Wv_%p_o+vW1Ly+U6{>;_P!s_C}m;QS?P%
zNy&2Ri0#hX3zNMuo|3Sg4cRA@qL3H_85zQ~5=Jmt%{%`NBuA$i+Pg))_w{HTm#h-3
znn7R}>{>Rr2T?p+vYt9()3N|K3ixO!MO6Wb*J9#yyo^cSb4L)x%Zt8<<L3_Gw?P!2
z1oqBi2oqSuH66z=8olq?wivcfj=HjvW-sT?Lz<W#6R7q+?QGgR8n_~q;^)9uf++rH
zf!A>T+-=+WPutd6w3TxUh%U8DC`A)DVlBn25pmJ-OUv=VQK1wnh~gVvizyoKEb7nq
zf+)VO06X^=V54$^C>{x=_yX_<_OnJV*#fvBltKhiyuh`XBJl&9_2!BoiqCT`!Zkq@
zKaTyvt~xLPToX!hkZU1D;+;jkIX#rZW|{PO;C!HID>Sc8f)cd??=0%g6ku^E#Tl-J
zxF(3=m6#ci6|S|UU_VSyqE_IYMg5roEC{98*R>GW1X27ba41kns)UNTER<sM*v0CP
zcNWcHGhksTMcr~@R8Eo}VHPIfPo`7?6*C<;!L^7Yu$>KAi^IJInB}}>MuK`Bcg@=^
zvD@FV3ubEWrgG)fpCF3Y0N)9vxDu0K=wrZ4;Js{^F}_Tuq_~#tHv4`Qxn?5^QRhFl
z&f@VPiszOjW=ujU=3?gB%i?B-@hk>`6M<_PdtumT@y_BMf+&6yxHCB_C5d}_KJaym
z;+a!31W`OQYu%#p-4MqFQ9K+(@vWF8!($frU6FbaMDZ%%LSUUm@jNYy+Ta&u)rW_G
zW5*~n0h65P*$nn?!^C}XMQdgSIs{StRw%_&z*vLNJBCth6-4psDs+{FcNX>GZOd{q
z$mT50E-E2$7D6dn=PVLzzPGXVG&O?8{{bz+cJG5v-B%NmOlmeUGM^8kI4)aH*?4DB
z141c23>;=zd^`rgsU3-<I2*f6=(gMJYmM)Q7>*02a1}*aU_V;E9(bliEwv?-BE&xB
zK^1HdY-d?E%{KcD2&H&9Yu$ob;hlwHT5{B#=YlAH%e4qc>`%R8j%<oNEsMJ3uZ?^+
zq*>G^;f%+c6*yQ2QU=~xG>5ktztOUq%tb*Izv@~<5qM|O9IgtY*yqM*7Ec4(wTPnd
z&Y~GC2QG3g!jqbFf+)6KKT~_Wv#39>V{(+f;aZ3%6_*B4e2;4pMdD{e>di~Q(Mec2
zPcs<C1X#YpwUDCmEsJ`x2DlQK?Hxs<co5jHjgA6*%c2gf!|e5V3NtnOB=Beu#Z3wy
rHy-K4c=<PQFK}lN#gDj^R4)GqpN^YDo8^9N00000NkvXXu0mjfB=wO-

literal 1733
zcmV;$20HnPP)<h;3K|Lk000e1NJLTq009aB002S=1^@s6ueX?d00009a7bBm000XU
z000XU0RWnu7ytkO8FWQhbW?9;ba!ELWdK2BZ(?O2No`?gWm08fWO;GPWjp`?22x2x
zK~#9!?47Z4<2De6A<va5RHY5`1kpXgQB7JaPjK`TSn4j7<SN%$ege@?AYGfr%oCjM
z369sMQfAU8yAtGa7)k~Kiok;ZH^b44#-gzR_UA6R1P4m<+sBVX`k2yjO~**tQPNS+
zarf(o52fl5YX1{`q%EG3?RX$NHp&xZ6WepE?HW4R@kn;$9_?sjZtWpE2EOJ~lmG4d
z`@nobUA-7cKKa~nAQL(=IzH1mM@lx_(QC3}<;l4+Jth+ekvQ71@@NOogRd>jE#Q$W
z{kd*=5ck5P1xPv>*)j1DNvtkl$D(695-oN^xSDXG(33K~z`2L|DiQ<t9sj>e_1kU7
z;(ZU1h=VsLM55G3Q(>q^C442Gd9<&v{N1QB!4no#$}x*eH&{Dz+LA3*JC5hxbY#b*
zrMU(C`NFeMysP<sY_k|iljCzmEE1nRLpF>Z)(2KKPFHl$?79?iNw&-^?3n1ZvU>iF
zCp$J|$1m_%IlfR`qo|(LR`GX{^lNz{V)Dgbk&MR|xUAgN1B4|CAW-yeN0*37O56Vz
zjl4N=(uL(YlfXL1){g78b38}hS=#ag+`CoMzP9~TL=nr~ini?F5Ps`<kW=_6@WmCn
zzzD9;+@Tu)gaVvmOCqsAH-J8a#mw{RMoz~ax&kn+&w5B{dlR9sz30l6cp<I!jvYF-
z2*?f=tVfIAja2tlrJyeng{_m2iUyU(2j=W9e6z8I#=~cu%bM@FguGM^m6;KZX{YE%
zHIRCM;KKA$i$e0^Ypzdmeb{Y)-nHV%<;Pge1eXNsJCOjP0GEs^a&=f=0T2oR&Wgp2
zIy^y)Iuyb18U?SW|IyO<+B}z}|J^2KO~_sZa4r|ISklMxf_tBYIG^AP0000000000
z001pB`2G9uOl#7|KmYl6;iHk%MsA-`!3ec+VDq~G;K$gxe?PU=&tyMug?FvTH}PoO
z?r0kw8?tZLf^BRMzw>0I!d)P}BU=|WZGTe2whcgKW;l%WF{O!A=t)*M2*kY#XCtBG
zM!WlVQTrb19LUhNp|&l<l<5xCQH04p5+bKqSU3_p(OLzz+gL?3=6qGvPT$0~<I98~
z5)caTg1L5lQ%B<GD5k>1aM(a3@G>C?1vtS}rxLF@QYP?9YxEer788gBgaW+6>fDuA
zFYHO(cOo+o!WBdULIKS3`g1A8T(~3MpAkV$Q6`uI5DH+%`!mr*Cfv3}r%rX;V(1|h
zpqVozHQ&c_wY{zp!6Bjv2nDbZ3lj$uN}V9281gU>QE;UTj|N@+u1gx;_G8B>D&LuI
zW}H)h+<35$Q;L;oUr-yj7PfJv=eY9Q81PmNjff&V5#-68JDkFF5p(9rKDKRZY0FOy
zL~x7Q1fhtXZyh+S&mE?0|MxJI{)12eN#LppAQU~u^sVD%&-!K8a{mDcg-mooQD)*a
zJ<P+u82+6}btiD3$z2bAfG2o^1n)=j8aYy}lki%`+SB|j)M1~E`~vGhzJ0DBo5qMJ
zWQE`lJ+!+J&_=!(aS8fYn{5GNj3|IVcoUnlg%rmZR?ngE6`I<eD;Xu~$9*=q0B-F0
zE)#^Jhj2){Y$J>nYX75>^(@Osrwmbv1yYZ&+4!iY3<+JCxD?iG7K?RE;{c%un@P-h
zZ<EwZ$*UnrY3!!43;={8d>U<s7im<nf0jeZt0S2=-4_%>0bH2&y-q-j#am&KMKq7e
zLn!31iskC$A{KPK)v1V-g~Im`3efe{DML5x1!F-UKhnVqy}3HnTZAecX$*bd<o2kD
zH-%a9T!H4vJ{ACzJsbGyw1iYVry{xXRI%D{pN-Vgcd#D?_|K%z4Rh}idQ04xMF9$~
z!oetXuGt03IBN1}+epQ^-TtmGlM_NA58HBRpQ)9&m+}In9-)$;3ovF73UKa<yMR!L
z8?$MHF%zGnAoV!F<+lF_X=~X-rxcyKIBYl+${sKUA_`gTTpu?FowGse0Wj&nYkY!G
zfUbE&7l@+N^%f8c&|NeUN|hckW)_Pf5DFRas`(ANK@_|*Sv094ghGZ$M7IZH<ZNS#
zCLk07a$FVX%Xg!WBosE+1s%dqiQXZS5GCrfn9>ahMY!QAPd2`twt+y1pg4|MtO9Z5
bUjYUHO*I@g#JC^K00000NkvXXu0mjfLP;qH

diff --git a/src/site/resources/images/hbase_logo.svg b/src/site/resources/images/hbase_logo.svg
index c4b3343ecc2c..2cc26d934100 100644
--- a/src/site/resources/images/hbase_logo.svg
+++ b/src/site/resources/images/hbase_logo.svg
@@ -1,41 +1,78 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 15.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 width="792px" height="612px" viewBox="0 0 792 612" enable-background="new 0 0 792 612" xml:space="preserve">
-<path fill="#BA160C" d="M233.586,371.672h-9.895v-51.583h9.895V371.672L233.586,371.672z M223.691,307.6v-19.465h9.895V307.6
-	H223.691z M223.691,371.672h-9.896v-32.117h-63.584v32.117h-19.466v-83.537h19.466v31.954h55.128h8.457h9.896V371.672
-	L223.691,371.672z M223.691,288.135h-9.896V307.6h9.896V288.135z"/>
-<path fill="#BA160C" d="M335.939,329.334c6.812,4.218,10.219,10.652,10.219,19.303c0,6.272-2,11.571-6.002,15.897
-	c-4.325,4.758-10.165,7.137-17.519,7.137h-28.629v-19.465h28.629c2.812,0,4.218-2.109,4.218-6.327c0-4.216-1.406-6.325-4.218-6.325
-	h-28.629v-19.303h27.17c2.811,0,4.217-2.109,4.217-6.327c0-4.216-1.406-6.326-4.217-6.326h-27.17v-19.464h27.17
-	c7.353,0,13.192,2.379,17.519,7.137c3.892,4.325,5.839,9.625,5.839,15.896C344.536,318.954,341.67,325.009,335.939,329.334z
-	 M294.008,371.672h-52.312v-51.42h19.466h5.259h27.588v19.303h-32.847v12.652h32.847V371.672L294.008,371.672z M294.008,307.599
-	h-32.847v0h-19.466v-19.465h52.312V307.599z"/>
-<path fill="#878888" d="M355.123,266.419v-8.92h14.532v-5.353c0-1.932-0.644-2.899-1.933-2.899h-12.6v-8.919h12.6
-	c3.223,0,5.836,1.164,7.842,3.494c2.007,2.33,3.011,5.104,3.011,8.325v26.463h-8.921v-12.19H355.123L355.123,266.419z
-	 M473.726,278.61h-29.587c-3.469,0-6.417-1.152-8.845-3.458c-2.429-2.304-3.642-5.191-3.642-8.659v-14.049
-	c0-3.47,1.213-6.356,3.642-8.662c2.428-2.304,5.376-3.455,8.845-3.455h29.587v8.919h-29.587c-2.378,0-3.567,1.066-3.567,3.197
-	v14.049c0,2.131,1.189,3.196,3.567,3.196h29.587V278.61L473.726,278.61z M567.609,278.61h-8.996v-14.718h-22.895v14.718h-8.92
-	v-38.282h8.92v14.644h22.895v-14.644h8.996V278.61L567.609,278.61z M661.494,249.247h-31.889v5.725h29.807v8.92h-29.807v5.797
-	h31.814v8.92h-40.735v-38.282h40.809V249.247z M355.123,240.328v8.919h-12.674c-1.239,0-1.858,0.967-1.858,2.899v5.353h5.575h2.435
-	h6.522v8.92h-6.522h-2.435h-5.575v12.19h-8.92v-26.463c0-3.221,1.004-5.996,3.011-8.325c2.006-2.33,4.596-3.494,7.768-3.494H355.123
-	L355.123,240.328z M254.661,266.122v-8.92h13.083c1.288,0,1.933-1.313,1.933-3.939c0-2.676-0.645-4.015-1.933-4.015h-13.083v-8.919
-	h13.083c3.32,0,5.995,1.363,8.028,4.088c1.883,2.478,2.825,5.425,2.825,8.846c0,3.419-0.942,6.342-2.825,8.771
-	c-2.033,2.725-4.708,4.088-8.028,4.088H254.661z M177.649,278.61h-8.92v-12.19h-14.532v-8.92h14.532v-5.353
-	c0-1.932-0.644-2.899-1.932-2.899h-12.6v-8.919h12.6c3.222,0,5.835,1.164,7.842,3.494c2.007,2.33,3.01,5.104,3.01,8.325V278.61
-	L177.649,278.61z M254.661,240.328v8.919h-15.016v7.954h15.016v8.92h-15.016v12.488h-8.92v-38.282H254.661z M154.198,266.419h-7.604
-	h-1.354h-5.575v12.19h-8.92v-26.463c0-3.221,1.004-5.996,3.01-8.325c2.007-2.33,4.597-3.494,7.768-3.494h12.674v8.919h-12.674
-	c-1.239,0-1.858,0.967-1.858,2.899v5.353h5.575h1.354h7.604V266.419z"/>
-<path fill="#BA160C" d="M456.325,371.672H436.86V345.07h-31.094h-0.618v-19.466h0.618h31.094v-11.68
-	c0-4.216-1.406-6.324-4.218-6.324h-27.494v-19.465h27.494c7.03,0,12.733,2.541,17.114,7.623c4.379,5.083,6.569,11.139,6.569,18.167
-	V371.672z M405.148,345.07h-19.547h-12.165v26.602h-19.466v-57.748c0-7.028,2.19-13.083,6.569-18.167
-	c4.379-5.083,10.03-7.623,16.952-7.623h27.656V307.6h-27.656c-2.704,0-4.055,2.108-4.055,6.324v11.68h12.165h19.547V345.07z"/>
-<path fill="#BA160C" d="M564.329,345.88c0,7.03-2.109,13.031-6.327,18.006c-4.541,5.19-10.273,7.786-17.193,7.786h-72.02v-19.465
-	h72.02c2.704,0,4.055-2.109,4.055-6.327c0-4.216-1.352-6.325-4.055-6.325h-52.394c-6.92,0-12.652-2.596-17.193-7.787
-	c-4.327-4.865-6.49-10.813-6.49-17.843c0-7.028,2.218-13.083,6.651-18.167c4.434-5.083,10.112-7.623,17.032-7.623h72.021v19.464
-	h-72.021c-2.703,0-4.055,2.109-4.055,6.326c0,4.109,1.352,6.164,4.055,6.164h52.394c6.92,0,12.652,2.596,17.193,7.787
-	C562.22,332.85,564.329,338.852,564.329,345.88z"/>
-<polygon fill="#BA160C" points="661.494,307.599 591.906,307.599 591.906,320.089 656.952,320.089 656.952,339.555 591.906,339.555 
-	591.906,352.207 661.331,352.207 661.331,371.672 572.44,371.672 572.44,288.135 661.494,288.135 "/>
-</svg>
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Generator: Adobe Illustrator 15.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   width="792px"
+   height="612px"
+   viewBox="0 0 792 612"
+   enable-background="new 0 0 792 612"
+   xml:space="preserve"
+   inkscape:version="0.48.4 r9939"
+   sodipodi:docname="hbase_banner_logo.png"
+   inkscape:export-filename="hbase_logo_filledin.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"><metadata
+   id="metadata3285"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs3283" /><sodipodi:namedview
+   pagecolor="#ffffff"
+   bordercolor="#666666"
+   borderopacity="1"
+   objecttolerance="10"
+   gridtolerance="10"
+   guidetolerance="10"
+   inkscape:pageopacity="0"
+   inkscape:pageshadow="2"
+   inkscape:window-width="1131"
+   inkscape:window-height="715"
+   id="namedview3281"
+   showgrid="false"
+   inkscape:zoom="4.3628026"
+   inkscape:cx="328.98554"
+   inkscape:cy="299.51695"
+   inkscape:window-x="752"
+   inkscape:window-y="456"
+   inkscape:window-maximized="0"
+   inkscape:current-layer="Layer_1" />
+<path
+   d="m 233.586,371.672 -9.895,0 0,-51.583 9.895,0 0,51.583 z m -9.77344,-51.59213 -0.12156,-31.94487 9.895,0 -0.0405,31.98539 z m -0.12156,51.59213 -9.896,0 0,-32.117 -63.584,0 0,32.117 -19.466,0 0,-83.537 19.466,0 0,31.954 55.128,0 8.457,0 9.896,0 0,51.583 z m 0,-83.537 -9.896,0 0,31.98539 10.01756,-0.0405 z"
+   id="path3269"
+   inkscape:connector-curvature="0"
+   style="fill:#ba160c"
+   sodipodi:nodetypes="cccccccccccccccccccccccccccccc" />
+<path
+   d="m 335.939,329.334 c 6.812,4.218 10.219,10.652 10.219,19.303 0,6.272 -2,11.571 -6.002,15.897 -4.325,4.758 -10.165,7.137 -17.519,7.137 l -28.629,0 0,-19.465 28.629,0 c 2.812,0 4.218,-2.109 4.218,-6.327 0,-4.216 -1.406,-6.325 -4.218,-6.325 l -28.629,0 0,-19.303 27.17,0 c 2.811,0 4.217,-2.109 4.217,-6.327 0,-4.216 -1.406,-6.326 -4.217,-6.326 l -27.17,0 0,-19.464 27.17,0 c 7.353,0 13.192,2.379 17.519,7.137 3.892,4.325 5.839,9.625 5.839,15.896 0,7.787 -2.866,13.842 -8.597,18.167 z m -41.931,42.338 -52.312,0 0,-51.42 19.466,0 5.259,0 27.588,0 0,19.303 -32.847,0 0,12.652 32.847,0 0,19.465 z m 0,-64.073 -32.847,0 0.0405,13.24974 -19.466,-0.48623 -0.0405,-32.22851 52.312,0 0,19.465 z"
+   id="path3271"
+   inkscape:connector-curvature="0"
+   style="fill:#ba160c"
+   sodipodi:nodetypes="cscsccsssccsssccscsccccccccccccccccccccc" />
+<path
+   d="M355.123,266.419v-8.92h14.532v-5.353c0-1.932-0.644-2.899-1.933-2.899h-12.6v-8.919h12.6  c3.223,0,5.836,1.164,7.842,3.494c2.007,2.33,3.011,5.104,3.011,8.325v26.463h-8.921v-12.19H355.123L355.123,266.419z   M473.726,278.61h-29.587c-3.469,0-6.417-1.152-8.845-3.458c-2.429-2.304-3.642-5.191-3.642-8.659v-14.049  c0-3.47,1.213-6.356,3.642-8.662c2.428-2.304,5.376-3.455,8.845-3.455h29.587v8.919h-29.587c-2.378,0-3.567,1.066-3.567,3.197  v14.049c0,2.131,1.189,3.196,3.567,3.196h29.587V278.61L473.726,278.61z M567.609,278.61h-8.996v-14.718h-22.895v14.718h-8.92  v-38.282h8.92v14.644h22.895v-14.644h8.996V278.61L567.609,278.61z M661.494,249.247h-31.889v5.725h29.807v8.92h-29.807v5.797  h31.814v8.92h-40.735v-38.282h40.809V249.247z M355.123,240.328v8.919h-12.674c-1.239,0-1.858,0.967-1.858,2.899v5.353h5.575h2.435  h6.522v8.92h-6.522h-2.435h-5.575v12.19h-8.92v-26.463c0-3.221,1.004-5.996,3.011-8.325c2.006-2.33,4.596-3.494,7.768-3.494H355.123  L355.123,240.328z M254.661,266.122v-8.92h13.083c1.288,0,1.933-1.313,1.933-3.939c0-2.676-0.645-4.015-1.933-4.015h-13.083v-8.919  h13.083c3.32,0,5.995,1.363,8.028,4.088c1.883,2.478,2.825,5.425,2.825,8.846c0,3.419-0.942,6.342-2.825,8.771  c-2.033,2.725-4.708,4.088-8.028,4.088H254.661z M177.649,278.61h-8.92v-12.19h-14.532v-8.92h14.532v-5.353  c0-1.932-0.644-2.899-1.932-2.899h-12.6v-8.919h12.6c3.222,0,5.835,1.164,7.842,3.494c2.007,2.33,3.01,5.104,3.01,8.325V278.61  L177.649,278.61z M254.661,240.328v8.919h-15.016v7.954h15.016v8.92h-15.016v12.488h-8.92v-38.282H254.661z M154.198,266.419h-7.604  h-1.354h-5.575v12.19h-8.92v-26.463c0-3.221,1.004-5.996,3.01-8.325c2.007-2.33,4.597-3.494,7.768-3.494h12.674v8.919h-12.674  c-1.239,0-1.858,0.967-1.858,2.899v5.353h5.575h1.354h7.604V266.419z"
+   id="path3273"
+   style="fill:#666666"
+   fill="#878888" />
+<path
+   fill="#BA160C"
+   d="M456.325,371.672H436.86V345.07h-31.094h-0.618v-19.466h0.618h31.094v-11.68  c0-4.216-1.406-6.324-4.218-6.324h-27.494v-19.465h27.494c7.03,0,12.733,2.541,17.114,7.623c4.379,5.083,6.569,11.139,6.569,18.167  V371.672z M405.148,345.07h-19.547h-12.165v26.602h-19.466v-57.748c0-7.028,2.19-13.083,6.569-18.167  c4.379-5.083,10.03-7.623,16.952-7.623h27.656V307.6h-27.656c-2.704,0-4.055,2.108-4.055,6.324v11.68h12.165h19.547V345.07z"
+   id="path3275" />
+<path
+   fill="#BA160C"
+   d="M564.329,345.88c0,7.03-2.109,13.031-6.327,18.006c-4.541,5.19-10.273,7.786-17.193,7.786h-72.02v-19.465  h72.02c2.704,0,4.055-2.109,4.055-6.327c0-4.216-1.352-6.325-4.055-6.325h-52.394c-6.92,0-12.652-2.596-17.193-7.787  c-4.327-4.865-6.49-10.813-6.49-17.843c0-7.028,2.218-13.083,6.651-18.167c4.434-5.083,10.112-7.623,17.032-7.623h72.021v19.464  h-72.021c-2.703,0-4.055,2.109-4.055,6.326c0,4.109,1.352,6.164,4.055,6.164h52.394c6.92,0,12.652,2.596,17.193,7.787  C562.22,332.85,564.329,338.852,564.329,345.88z"
+   id="path3277" />
+<polygon
+   fill="#BA160C"
+   points="661.494,307.599 591.906,307.599 591.906,320.089 656.952,320.089 656.952,339.555 591.906,339.555   591.906,352.207 661.331,352.207 661.331,371.672 572.44,371.672 572.44,288.135 661.494,288.135 "
+   id="polygon3279" />
+</svg>
\ No newline at end of file
diff --git a/src/site/site.vm b/src/site/site.vm
index 03c2e758ebd6..0e2519528b63 100644
--- a/src/site/site.vm
+++ b/src/site/site.vm
@@ -481,6 +481,20 @@
       #end
     #end
     ## $headContent
+<!--Google Analytics-->
+<script type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-30210968-1']);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</script>
   </head>
   <body class="composite">
     <div id="banner">
@@ -521,7 +535,10 @@
       <hr/>
     </div>
     <div id="footer">
-      <div class="xright"> #publishDate( "right" $decoration.publishDate $decoration.version )&nbsp;| Copyright &#169;#copyright()All Rights Reserved.  </div>
+       <div class="xright">#publishDate( "right" $decoration.publishDate $decoration.version )&nbsp;
+        </div>
+        <div class="xright">Copyright &#169;<a href="http://www.apache.org">#copyright()</a>All Rights Reserved.  Apache Hadoop, Hadoop, HDFS, HBase and the HBase project logo are trademarks of the Apache Software Foundation.
+        </div>
       <div class="clear">
         <hr/>
       </div>
diff --git a/src/site/site.xml b/src/site/site.xml
index 3728283ce09a..210277af5dc8 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <!--
 /**
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -24,7 +24,7 @@
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://maven.apache.org/DECORATION/1.0.0 http://maven.apache.org/xsd/decoration-1.0.0.xsd">
   <bannerLeft>
-    <name>HBase</name>
+    <name>Apache HBase</name>
     <src>images/hbase_logo.png</src>
     <href>http://hbase.apache.org/</href>
   </bannerLeft>
@@ -32,7 +32,7 @@
   <version position="right" />
   <publishDate position="right" />
   <body>
-    <menu name="HBase Project">
+    <menu name="Apache HBase Project">
       <item name="Overview" href="index.html"/>
       <item name="License" href="license.html" />
       <item name="Downloads" href="http://www.apache.org/dyn/closer.cgi/hbase/" />
@@ -40,8 +40,11 @@
       <item name="Issue Tracking" href="issue-tracking.html" />
       <item name="Mailing Lists" href="mail-lists.html" />
       <item name="Source Repository" href="source-repository.html" />
+      <item name="ReviewBoard" href="https://reviews.apache.org"/>
       <item name="Team" href="team-list.html" />
-      <item name="Sponsors" href="sponsors.html" />
+      <item name="Thanks" href="sponsors.html" />
+      <item name="Blog" href="http://blogs.apache.org/hbase/" />
+      <item name="Other resources" href="resources.html" />
     </menu>
     <menu name="Documentation">
         <item name="Getting Started" href="book/quickstart.html" />
@@ -49,15 +52,20 @@
       <item name="X-Ref" href="xref/index.html" />
       <item name="Ref Guide (multi-page)"      href="book/book.html" />
       <item name="Ref Guide (single-page)"      href="book.html" />
+      <item name="中文参考指南(单页)" href="http://abloz.com/hbase/book.html" />
       <item name="FAQ" href="book/faq.html" />
       <item name="Videos/Presentations" href="book.html#other.info" />
       <item name="Wiki" href="http://wiki.apache.org/hadoop/Hbase" />
       <item name="ACID Semantics" href="acid-semantics.html" />
-      <item name="Bulk Loads" href="bulk-loads.html" />
+      <item name="Bulk Loads" href="book.html#arch.bulk.load" />
       <item name="Metrics"      href="metrics.html" />
       <item name="HBase on Windows"      href="cygwin.html" />
       <item name="Cluster replication"      href="replication.html" />
-      <item name="Pseudo-Dist. Extras"      href="pseudo-distributed.html" />
+    </menu>
+    <menu name="ASF">
+        <item name="Apache Software Foundation"      href="http://www.apache.org/foundation/" />
+        <item name="How Apache Works"      href="http://www.apache.org/foundation/how-it-works.html" />
+        <item name="Sponsoring Apache"      href="http://www.apache.org/foundation/sponsorship.html" />
     </menu>
   </body>
     <skin>
diff --git a/src/site/xdoc/acid-semantics.xml b/src/site/xdoc/acid-semantics.xml
index 99b0392a05e4..81f7f04fed4a 100644
--- a/src/site/xdoc/acid-semantics.xml
+++ b/src/site/xdoc/acid-semantics.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -23,13 +22,13 @@
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
     <title> 
-      HBase ACID Properties
+      Apache HBase (TM) ACID Properties
     </title>
   </properties>
 
   <body>
     <section name="About this Document">
-      <p>HBase is not an ACID compliant database. However, it does guarantee certain specific
+      <p>Apache HBase (TM) is not an ACID compliant database. However, it does guarantee certain specific
       properties.</p>
       <p>This specification enumerates the ACID properties of HBase.</p>
     </section>
@@ -197,7 +196,7 @@
         </ol>
       </section>
       <section name="Tunability">
-        <p>All of the above guarantees must be possible within HBase. For users who would like to trade
+        <p>All of the above guarantees must be possible within Apache HBase. For users who would like to trade
         off some guarantees for performance, HBase may offer several tuning options. For example:</p>
         <ul>
           <li>Visibility may be tuned on a per-read basis to allow stale reads or time travel.</li>
@@ -207,7 +206,7 @@
     </section>
     <section name="More Information">
       <p>
-      For more information, see the <a href="book.html#client">client architecture</a> or <a href="book.html#datamodel">data model</a> sections in the HBase book. 
+      For more information, see the <a href="book.html#client">client architecture</a> or <a href="book.html#datamodel">data model</a> sections in the Apache HBase Reference Guide. 
       </p>
     </section>
     
@@ -218,7 +217,7 @@
           (See <a href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html#setBatch(int)">Scan#setBatch(int)</a>).
       </p>
 
-      <p>[2] In the context of HBase, &quot;durably on disk&quot; implies an hflush() call on the transaction
+      <p>[2] In the context of Apache HBase, &quot;durably on disk&quot; implies an hflush() call on the transaction
       log. This does not actually imply an fsync() to magnetic media, but rather just that the data has been
       written to the OS cache on all replicas of the log. In the case of a full datacenter power loss, it is
       possible that the edits are not truly durable.</p>
diff --git a/src/site/xdoc/bulk-loads.xml b/src/site/xdoc/bulk-loads.xml
index 869502a899ae..f4d3d75f9511 100644
--- a/src/site/xdoc/bulk-loads.xml
+++ b/src/site/xdoc/bulk-loads.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -19,153 +18,13 @@
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
     <title> 
-      Bulk Loads in HBase
+      Bulk Loads in Apache HBase (TM)
     </title>
   </properties>
   <body>
-    <section name="Overview">
-      <p>
-        HBase includes several methods of loading data into tables.
-        The most straightforward method is to either use the TableOutputFormat
-        class from a MapReduce job, or use the normal client APIs; however,
-        these are not always the most efficient methods.
-      </p>
-      <p>
-        This document describes HBase's bulk load functionality. The bulk load
-        feature uses a MapReduce job to output table data in HBase's internal
-        data format, and then directly loads the data files into a running
-        cluster. Using bulk load will use less CPU and network resources than
-        simply using the HBase API.
-      </p>
-    </section>
-    <section name="Bulk Load Architecture">
-      <p>
-        The HBase bulk load process consists of two main steps.
-      </p>
-      <section name="Preparing data via a MapReduce job">
-        <p>
-          The first step of a bulk load is to generate HBase data files from
-          a MapReduce job using HFileOutputFormat. This output format writes
-          out data in HBase's internal storage format so that they can be
-          later loaded very efficiently into the cluster.
-        </p>
-        <p>
-          In order to function efficiently, HFileOutputFormat must be
-          configured such that each output HFile fits within a single region.
-          In order to do this, jobs whose output will be bulk loaded into HBase
-          use Hadoop's TotalOrderPartitioner class to partition the map output
-          into disjoint ranges of the key space, corresponding to the key
-          ranges of the regions in the table.
-        </p>
-        <p>
-          HFileOutputFormat includes a convenience function,
-          <code>configureIncrementalLoad()</code>, which automatically sets up
-          a TotalOrderPartitioner based on the current region boundaries of a
-          table.
-        </p>
-      </section>
-      <section name="Completing the data load">
-        <p>
-          After the data has been prepared using
-          <code>HFileOutputFormat</code>, it is loaded into the cluster using
-          <code>completebulkload</code>. This command line tool iterates
-          through the prepared data files, and for each one determines the
-          region the file belongs to. It then contacts the appropriate Region
-          Server which adopts the HFile, moving it into its storage directory
-          and making the data available to clients.
-        </p>
-        <p>
-          If the region boundaries have changed during the course of bulk load
-          preparation, or between the preparation and completion steps, the
-          <code>completebulkloads</code> utility will automatically split the
-          data files into pieces corresponding to the new boundaries. This
-          process is not optimally efficient, so users should take care to
-          minimize the delay between preparing a bulk load and importing it
-          into the cluster, especially if other clients are simultaneously
-          loading data through other means.
-        </p>
-      </section>
-    </section>
-    <section name="Importing the prepared data using the completebulkload tool">
-      <p>
-        After a data import has been prepared, either by using the
-        <code>importtsv</code> tool with the
-        "<code>importtsv.bulk.output</code>" option or by some other MapReduce
-        job using the <code>HFileOutputFormat</code>, the
-        <code>completebulkload</code> tool is used to import the data into the
-        running cluster.
-      </p>
-      <p>
-        The <code>completebulkload</code> tool simply takes the output path
-        where <code>importtsv</code> or your MapReduce job put its results, and
-        the table name to import into. For example:
-      </p>
-      <code>$ hadoop jar hbase-VERSION.jar completebulkload [-c /path/to/hbase/config/hbase-site.xml] /user/todd/myoutput mytable</code>
-      <p>
-        The <code>-c config-file</code> option can be used to specify a file
-        containing the appropriate hbase parameters (e.g., hbase-site.xml) if
-        not supplied already on the CLASSPATH (In addition, the CLASSPATH must
-        contain the directory that has the zookeeper configuration file if
-        zookeeper is NOT managed by HBase).
-      </p>
-      <p>
-        <b>Note:</b> If the target table does not already exist in HBase, this
-        tool will create the table automatically.</p>
-      <p>
-        This tool will run quickly, after which point the new data will be visible in
-        the cluster.
-      </p>
-    </section>
-    <section name="Using the importtsv tool to bulk load data">
-      <p>
-        HBase ships with a command line tool called <code>importtsv</code>
-        which when given files containing data in TSV form can prepare this
-        data for bulk import into HBase. This tool by default uses the HBase
-        <code>put</code> API to insert data into HBase one row at a time, but
-        when the "<code>importtsv.bulk.output</code>" option is used,
-        <code>importtsv</code> will instead generate files using
-        <code>HFileOutputFormat</code> which can subsequently be bulk-loaded
-        into HBase using the <code>completebulkload</code> tool described
-        above. This tool is available by running "<code>hadoop jar
-          /path/to/hbase-VERSION.jar importtsv</code>". Running this command
-        with no arguments prints brief usage information:
-      </p>
-      <code><pre>
-Usage: importtsv -Dimporttsv.columns=a,b,c &lt;tablename&gt; &lt;inputdir&gt;
-
-Imports the given input directory of TSV data into the specified table.
-
-The column names of the TSV data must be specified using the -Dimporttsv.columns
-option. This option takes the form of comma-separated column names, where each
-column name is either a simple column family, or a columnfamily:qualifier. The special
-column name HBASE_ROW_KEY is used to designate that this column should be used
-as the row key for each imported record. You must specify exactly one column
-to be the row key, and you must specify a column name for every column that exists in the
-input data.
-
-By default importtsv will load data directly into HBase. To instead generate
-HFiles of data to prepare for a bulk data load, pass the option:
-  -Dimporttsv.bulk.output=/path/for/output
-  Note: if you do not use this option, then the target table must already exist in HBase
-
-Other options that may be specified with -D include:
-  -Dimporttsv.skip.bad.lines=false - fail if encountering an invalid line
-  '-Dimporttsv.separator=|' - eg separate on pipes instead of tabs
-  -Dimporttsv.timestamp=currentTimeAsLong - use the specified timestamp for the import
-  -Dimporttsv.mapper.class=my.Mapper - A user-defined Mapper to use instead of org.apache.hadoop.hbase.mapreduce.TsvImporterMapper
-</pre></code>
-    </section>
-    <section name="Advanced Usage">
-      <p>
-        Although the <code>importtsv</code> tool is useful in many cases, advanced users may
-        want to generate data programatically, or import data from other formats. To get
-        started doing so, dig into <code>ImportTsv.java</code> and check the JavaDoc for
-        HFileOutputFormat.
-      </p>
-      <p>
-        The import step of the bulk load can also be done programatically. See the
-        <code>LoadIncrementalHFiles</code> class for more information.
-      </p>
-    </section>
+       <p>This page has been retired.  The contents have been moved to the 
+      <a href="http://hbase.apache.org/book.html#arch.bulk.load">Bulk Loading</a> section
+ in the Reference Guide.
+ </p>
   </body>
 </document>
diff --git a/src/site/xdoc/cygwin.xml b/src/site/xdoc/cygwin.xml
index 2bdce12df85b..40e0f72cf853 100644
--- a/src/site/xdoc/cygwin.xml
+++ b/src/site/xdoc/cygwin.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -18,21 +17,21 @@
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
-    <title>Installing HBase on Windows using Cygwin</title>
+    <title>Installing Apache HBase (TM) on Windows using Cygwin</title>
   </properties>
 
 <body>
 <section name="Introduction">
-<p><a title="HBase project" href="http://hadoop.apache.org/hbase" target="_blank">HBase</a> is a distributed, column-oriented store, modeled after Google's <a title="Google's BigTable" href="http://labs.google.com/papers/bigtable.html" target="_blank">BigTable</a>. HBase is built on top of <a title="Hadoop project" href="http://hadoop.apache.org">Hadoop</a> for its <a title="Hadoop MapReduce project" href="http://hadoop.apache.org/mapreduce" target="_blank">MapReduce </a>and <a title="Hadoop DFS project" href="http://hadoop.apache.org/hdfs">distributed file system</a> implementation. All these projects are open-source and part of the <a title="The Apache Software Foundation" href="http://www.apache.org/" target="_blank">Apache Software Foundation</a>.</p>
+<p><a title="HBase project" href="http://hbase.apache.org" target="_blank">Apache HBase (TM)</a> is a distributed, column-oriented store, modeled after Google's <a title="Google's BigTable" href="http://research.google.com/archive/bigtable.html" target="_blank">BigTable</a>. Apache HBase is built on top of <a title="Hadoop project" href="http://hadoop.apache.org">Hadoop</a> for its <a title="Hadoop MapReduce project" href="http://hadoop.apache.org/mapreduce" target="_blank">MapReduce </a>and <a title="Hadoop DFS project" href="http://hadoop.apache.org/hdfs">distributed file system</a> implementation. All these projects are open-source and part of the <a title="The Apache Software Foundation" href="http://www.apache.org/" target="_blank">Apache Software Foundation</a>.</p>
 
 <p style="text-align: justify; ">As being distributed, large scale platforms, the Hadoop and HBase projects mainly focus on <em><strong>*nix</strong></em><strong> environments</strong> for production installations. However, being developed in <strong>Java</strong>, both projects are fully <strong>portable</strong> across platforms and, hence, also to the <strong>Windows operating system</strong>. For ease of development the projects rely on <a title="Cygwin site" href="http://www.cygwin.com/" target="_blank">Cygwin</a> to have a *nix-like environment on Windows to run the shell scripts.</p>
 </section>
 <section name="Purpose">
-<p style="text-align: justify; ">This document explains the <strong>intricacies of running HBase on Windows using Cygwin</strong> as an all-in-one single-node installation for testing and development. The HBase <a title="HBase Overview" href="http://hadoop.apache.org/hbase/docs/current/api/overview-summary.html#overview_description" target="_blank">Overview</a> and <a title="HBase QuickStart" href="http://hadoop.apache.org/common/docs/current/quickstart.html" target="_blank">QuickStart</a> guides on the other hand go a long way in explaning how to setup <a title="HBase project" href="http://hadoop.apache.org/hbase" target="_blank">HBase</a> in more complex deployment scenario's.</p>
+<p style="text-align: justify; ">This document explains the <strong>intricacies of running Apache HBase on Windows using Cygwin</strong> as an all-in-one single-node installation for testing and development. The HBase <a title="HBase Overview" href="http://hbase.apache.org/apidocs/overview-summary.html#overview_description" target="_blank">Overview</a> and <a title="HBase QuickStart" href="http://hbase.apache.org/book/quickstart.html" target="_blank">QuickStart</a> guides on the other hand go a long way in explaning how to setup <a title="HBase project" href="http://hadoop.apache.org/hbase" target="_blank">HBase</a> in more complex deployment scenario's.</p>
 </section>
 
 <section name="Installation">
-<p style="text-align: justify; ">For running HBase on Windows, 3 technologies are required: <strong>Java, Cygwin and SSH</strong>. The following paragraphs detail the installation of each of the aforementioned technologies.</p>
+<p style="text-align: justify; ">For running Apache HBase on Windows, 3 technologies are required: <strong>Java, Cygwin and SSH</strong>. The following paragraphs detail the installation of each of the aforementioned technologies.</p>
 <section name="Java">
 <p style="text-align: justify; ">HBase depends on the <a title="Java Platform, Standard Edition, 6 Release" href="http://java.sun.com/javase/6/" target="_blank">Java Platform, Standard Edition, 6 Release</a>. So the target system has to be provided with at least the Java Runtime Environment (JRE); however if the system will also be used for development, the Jave Development Kit (JDK) is preferred. You can download the latest versions for both from <a title="Java SE Downloads" href="http://java.sun.com/javase/downloads/index.jsp" target="_blank">Sun's download page</a>. Installation is a simple GUI wizard that guides you through the process.</p>
 </section>
@@ -92,7 +91,7 @@
 </ol>
 </section>
 <section name="HBase">
-<p style="text-align: justify; ">Download the <strong>latest release </strong>of HBase from the <a title="HBase Releases" href="http://hadoop.apache.org/hbase/releases.html" target="_blank">website</a>. As the HBase distributable is just a zipped archive, installation is as simple as unpacking the archive so it ends up in its final <strong>installation</strong> directory. Notice that HBase has to be installed in Cygwin and a good directory suggestion is to use <code>/usr/local/</code> (or [<code><strong>Root</strong> directory]\usr\local</code> in Windows slang). You should end up with a <code>/usr/local/hbase-<em>&lt;version&gt;</em></code> installation in Cygwin.</p>
+<p style="text-align: justify; ">Download the <strong>latest release </strong>of Apache HBase from the <a title="HBase Releases" href="http://www.apache.org/dyn/closer.cgi/hbase/" target="_blank">website</a>. As the Apache HBase distributable is just a zipped archive, installation is as simple as unpacking the archive so it ends up in its final <strong>installation</strong> directory. Notice that HBase has to be installed in Cygwin and a good directory suggestion is to use <code>/usr/local/</code> (or [<code><strong>Root</strong> directory]\usr\local</code> in Windows slang). You should end up with a <code>/usr/local/hbase-<em>&lt;version&gt;</em></code> installation in Cygwin.</p>
 
 This finishes installation. We go on with the configuration.
 </section>
@@ -193,7 +192,7 @@ If all previous configurations are working properly, we just need some tinkering
 <section>
 <title>Testing</title>
 <p>
-This should conclude the installation and configuration of HBase on Windows using Cygwin. So it's time <strong>to test it</strong>.
+This should conclude the installation and configuration of Apache HBase on Windows using Cygwin. So it's time <strong>to test it</strong>.
 <ol>
 	<li>Start a Cygwin<strong> terminal</strong>, if you haven't already.</li>
 	<li>Change directory to HBase <strong>installation</strong> using <code>CD /usr/local/hbase-<em>&lt;version&gt;</em></code>, preferably using auto-completion.</li>
diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml
index 2c330fe773ab..b4c1f817defe 100644
--- a/src/site/xdoc/index.xml
+++ b/src/site/xdoc/index.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -18,24 +17,23 @@
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
-    <title>HBase Home</title>
+    <title>Apache HBase&#153; Home</title>
     <link rel="shortcut icon" href="/images/favicon.ico" />
   </properties>
-  
+
   <body>
-    <section name="Welcome to Apache HBase!">
-    <p>HBase is the <a href="http://hadoop.apache.org">Hadoop</a> database.  Think of it as a distributed scalable Big Data store.
+    <section name="Welcome to Apache HBase&#153;">
+        <p><a href="http://www.apache.org/">Apache</a> HBase&#153; is the <a href="http://hadoop.apache.org">Hadoop</a> database, a distributed, scalable, big data store.
     </p>
-    <h4>When Would I Use HBase?</h4>
+    <h4>When Would I Use Apache HBase?</h4>
     <p>
-    Use HBase when you need random, realtime read/write access to your Big Data.
+    Use Apache HBase when you need random, realtime read/write access to your Big Data.
     This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware.
-HBase is an open-source, distributed, versioned, column-oriented store modeled after Google's <a href="http://research.google.com/archive/bigtable.html">Bigtable: A Distributed Storage System for Structured Data</a> by Chang et al.
- Just as Bigtable leverages the distributed data storage provided by the Google File System, HBase provides Bigtable-like capabilities on top of Hadoop and HDFS.
+Apache HBase is an open-source, distributed, versioned, column-oriented store modeled after Google's <a href="http://research.google.com/archive/bigtable.html">Bigtable: A Distributed Storage System for Structured Data</a> by Chang et al.
+ Just as Bigtable leverages the distributed data storage provided by the Google File System, Apache HBase provides Bigtable-like capabilities on top of Hadoop and HDFS.
     </p>
     <h4>Features</h4>
     <p>
-HBase provides:
 <ul>
     <li>Linear and modular scalability.
 </li>
@@ -45,7 +43,7 @@ HBase provides:
 </li>
     <li>Automatic failover support between RegionServers.
 </li>
-    <li>Convenient base classes for backing Hadoop MapReduce jobs with HBase tables.
+    <li>Convenient base classes for backing Hadoop MapReduce jobs with Apache HBase tables.
 </li>
     <li>Easy to use Java API for client access.
 </li>
@@ -67,19 +65,18 @@ HBase provides:
    </p>
  </section>
      <section name="News">
-            <p>May 22nd, 2012 <a href="http://www.hbasecon.com">HBaseCon2012</a> in San Francisco</p>
-            <p>January 23rd, 2012 HBase 0.92.0 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
-            <p>January 19th, 2012 <a href="http://www.meetup.com/hbaseusergroup/events/46702842/">Meetup @ EBay</a></p>
-            <p>December 23rd, 2011 HBase 0.90.5 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
-            <p>November 29th, 2011 <a href="http://www.meetup.com/hackathon/events/41025972/">Developer Pow-Wow in SF</a> at Salesforce HQ</p>
-            <p>November 7th, 2011 <a href="http://www.meetup.com/hbaseusergroup/events/35682812/">HBase Meetup in NYC (6PM)</a> at the AppNexus office</p>
-            <p>August 22nd, 2011 <a href="http://www.meetup.com/hbaseusergroup/events/28518471/">HBase Hackathon (11AM) and Meetup (6PM)</a> at FB in PA</p>
-            <p>June 30th, 2011 <a href="http://www.meetup.com/hbaseusergroup/events/20572251/">HBase Contributor Day</a>, the day after the <a href="http://developer.yahoo.com/events/hadoopsummit2011/">Hadoop Summit</a> hosted by Y!</p>
-            <p>June 8th, 2011 <a href="http://berlinbuzzwords.de/wiki/hbase-hackathon">HBase Hackathon</a> in Berlin to coincide with <a href="http://berlinbuzzwords.de/">Berlin Buzzwords</a></p>
-            <p>May 19th, 2011 HBase 0.90.3 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
-            <p>April 12th, 2011 HBase 0.90.2 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
+         <p>June 13th, 2013 <a href="http://www.hbasecon.com/">HBaseCon2013</a> in San Francisco.  Submit an Abstract!</p>
+         <p>April 11th, 2013 <a href="http://www.meetup.com/hbaseusergroup/events/103587852/">HBase Meetup at AdRoll</a> in San Francisco</p>
+         <p>February 28th, 2013 <a href="http://www.meetup.com/hbaseusergroup/events/96584102/">HBase Meetup at Intel Mission Campus</a></p>
+         <p>February 19th, 2013 <a href="http://www.meetup.com/hackathon/events/103633042/">Developers PowWow</a> at HortonWorks' new digs</p>
+         <p>January 23rd, 2013 <a href="http://www.meetup.com/hbaseusergroup/events/91381312/">HBase Meetup at WibiData World HQ!</a></p>
+            <p>December 4th, 2012 <a href="http://www.meetup.com/hackathon/events/90536432/">0.96 Bug Squashing and Testing Hackathon</a> at Cloudera, SF.</p>
+            <p>October 29th, 2012 <a href="http://www.meetup.com/hbaseusergroup/events/82791572/">HBase User Group Meetup</a> at Wize Commerce in San Mateo.</p>
+            <p>October 25th, 2012 <a href="http://www.meetup.com/HBase-NYC/events/81728932/">Strata/Hadoop World HBase Meetup.</a> in NYC</p>
+            <p>September 11th, 2012 <a href="http://www.meetup.com/hbaseusergroup/events/80621872/">Contributor's Pow-Wow at HortonWorks HQ.</a></p>
+
       <p><small><a href="old_news.html">Old News</a></small></p>
     </section>
   </body>
-  
+
 </document>
diff --git a/src/site/xdoc/metrics.xml b/src/site/xdoc/metrics.xml
index 9ff14e60d2c1..47902b8df49c 100644
--- a/src/site/xdoc/metrics.xml
+++ b/src/site/xdoc/metrics.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -19,14 +18,14 @@
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
     <title> 
-      HBase Metrics
+      Apache HBase (TM) Metrics
     </title>
   </properties>
 
   <body>
     <section name="Introduction">
       <p>
-      HBase emits Hadoop <a href="http://hadoop.apache.org/core/docs/current/api/org/apache/hadoop/metrics/package-summary.html">metrics</a>.
+      Apache HBase (TM) emits Hadoop <a href="http://hadoop.apache.org/core/docs/current/api/org/apache/hadoop/metrics/package-summary.html">metrics</a>.
       </p>
       </section>
       <section name="Setup">
@@ -139,7 +138,7 @@ export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.p
       </section>
       <section name="Understanding HBase Metrics">
       <p>
-      For more information on understanding HBase metrics, see the <a href="book.html#hbase_metrics">metrics section</a> in the HBase book. 
+      For more information on understanding HBase metrics, see the <a href="book.html#hbase_metrics">metrics section</a> in the Apache HBase Reference Guide. 
       </p>
       </section>
     </section>
diff --git a/src/site/xdoc/old_news.xml b/src/site/xdoc/old_news.xml
index 80c388c42b06..1dc3eaeb1b1d 100644
--- a/src/site/xdoc/old_news.xml
+++ b/src/site/xdoc/old_news.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -22,27 +21,43 @@
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
-    <title> 
-      Old News
+    <title>
+      Old Apache HBase (TM) News
     </title>
   </properties>
   <body>
   <section name="Old News">
+            <p>August 8th, 2012 <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Apache HBase 0.94.1 is available for download</a></p>
+            <p>June 15th, 2012 <a href="http://www.meetup.com/hbaseusergroup/events/59829652/">Birds-of-a-feather</a> in San Jose, day after <a href="http://hadoopsummit.org">Hadoop Summit</a></p>
+            <p>May 23rd, 2012 <a href="http://www.meetup.com/hackathon/events/58953522/">HackConAthon</a> in Palo Alto</p>
+            <p>May 22nd, 2012 <a href="http://www.hbasecon.com">HBaseCon2012</a> in San Francisco</p>
+            <p>March 27th, 2012 <a href="http://www.meetup.com/hbaseusergroup/events/56021562/">Meetup @ StumbleUpon</a> in San Francisco</p>
+
+            <p>January 19th, 2012 <a href="http://www.meetup.com/hbaseusergroup/events/46702842/">Meetup @ EBay</a></p>
+            <p>January 23rd, 2012 Apache HBase 0.92.0 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
+            <p>December 23rd, 2011 Apache HBase 0.90.5 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
+            <p>November 29th, 2011 <a href="http://www.meetup.com/hackathon/events/41025972/">Developer Pow-Wow in SF</a> at Salesforce HQ</p>
+            <p>November 7th, 2011 <a href="http://www.meetup.com/hbaseusergroup/events/35682812/">HBase Meetup in NYC (6PM)</a> at the AppNexus office</p>
+            <p>August 22nd, 2011 <a href="http://www.meetup.com/hbaseusergroup/events/28518471/">HBase Hackathon (11AM) and Meetup (6PM)</a> at FB in PA</p>
+            <p>June 30th, 2011 <a href="http://www.meetup.com/hbaseusergroup/events/20572251/">HBase Contributor Day</a>, the day after the <a href="http://developer.yahoo.com/events/hadoopsummit2011/">Hadoop Summit</a> hosted by Y!</p>
+            <p>June 8th, 2011 <a href="http://berlinbuzzwords.de/wiki/hbase-workshop-and-hackathon">HBase Hackathon</a> in Berlin to coincide with <a href="http://berlinbuzzwords.de/">Berlin Buzzwords</a></p>
+            <p>May 19th, 2011 Apache HBase 0.90.3 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
+            <p>April 12th, 2011 Apache HBase 0.90.2 released. <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Download it!</a></p>
             <p>March 21st, <a href="http://www.meetup.com/hackathon/events/16770852/">HBase 0.92 Hackathon at StumbleUpon, SF</a></p>
             <p>February 22nd, <a href="http://www.meetup.com/hbaseusergroup/events/16492913/">HUG12: February HBase User Group at StumbleUpon SF</a></p>
             <p>December 13th, <a href="http://www.meetup.com/hackathon/calendar/15597555/">HBase Hackathon: Coprocessor Edition</a></p>
-      <p>November 19th, <a href="http://huguk.org/">Hadoop HUG in London</a> is all about HBase</p>
+      <p>November 19th, <a href="http://huguk.org/">Hadoop HUG in London</a> is all about Apache HBase</p>
       <p>November 15-19th, <a href="http://www.devoxx.com/display/Devoxx2K10/Home">Devoxx</a> features HBase Training and multiple HBase presentations</p>
       <p>October 12th, HBase-related presentations by core contributors and users at <a href="http://www.cloudera.com/company/press-center/hadoop-world-nyc/">Hadoop World 2010</a></p>
       <p>October 11th, <a href="http://www.meetup.com/hbaseusergroup/calendar/14606174/">HUG-NYC: HBase User Group NYC Edition</a> (Night before Hadoop World)</p>
-      <p>June 30th, <a href="http://www.meetup.com/hbaseusergroup/calendar/13562846/">HBase Contributor Workshop</a> (Day after Hadoop Summit)</p>
-      <p>May 10th, 2010: HBase graduates from Hadoop sub-project to Apache Top Level Project </p>
+      <p>June 30th, <a href="http://www.meetup.com/hbaseusergroup/calendar/13562846/">Apache HBase Contributor Workshop</a> (Day after Hadoop Summit)</p>
+      <p>May 10th, 2010: Apache HBase graduates from Hadoop sub-project to Apache Top Level Project </p>
       <p>Signup for <a href="http://www.meetup.com/hbaseusergroup/calendar/12689490/">HBase User Group Meeting, HUG10</a> hosted by Trend Micro, April 19th, 2010</p>
 
       <p><a href="http://www.meetup.com/hbaseusergroup/calendar/12689351/">HBase User Group Meeting, HUG9</a> hosted by Mozilla, March 10th, 2010</p>
       <p>Sign up for the <a href="http://www.meetup.com/hbaseusergroup/calendar/12241393/">HBase User Group Meeting, HUG8</a>, January 27th, 2010 at StumbleUpon in SF</p>
-      <p>September 8th, 2010: HBase 0.20.0 is faster, stronger, slimmer, and sweeter tasting than any previous HBase release.  Get it off the <a href="releases.html">Releases</a> page.</p>
-      <p><a href="http://dev.us.apachecon.com/c/acus2009/">ApacheCon</a> in Oakland: November 2-6th, 2009: 
+      <p>September 8th, 2010: Apache HBase 0.20.0 is faster, stronger, slimmer, and sweeter tasting than any previous Apache HBase release.  Get it off the <a href="http://www.apache.org/dyn/closer.cgi/hbase/">Releases</a> page.</p>
+      <p><a href="http://dev.us.apachecon.com/c/acus2009/">ApacheCon</a> in Oakland: November 2-6th, 2009:
       The Apache Foundation will be celebrating its 10th anniversary in beautiful Oakland by the Bay. Lots of good talks and meetups including an HBase presentation by a couple of the lads.</p>
       <p>HBase at Hadoop World in NYC: October 2nd, 2009: A few of us will be talking on Practical HBase out east at <a href="http://www.cloudera.com/hadoop-world-nyc">Hadoop World: NYC</a>.</p>
       <p>HUG7 and HBase Hackathon: August 7th-9th, 2009 at StumbleUpon in SF: Sign up for the <a href="http://www.meetup.com/hbaseusergroup/calendar/10950511/">HBase User Group Meeting, HUG7</a> or for the <a href="http://www.meetup.com/hackathon/calendar/10951718/">Hackathon</a> or for both (all are welcome!).</p>
diff --git a/src/site/xdoc/pseudo-distributed.xml b/src/site/xdoc/pseudo-distributed.xml
index 8677ddc21b09..a1b38babb997 100644
--- a/src/site/xdoc/pseudo-distributed.xml
+++ b/src/site/xdoc/pseudo-distributed.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -23,55 +22,17 @@
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
     <title> 
-Running HBase in pseudo-distributed mode
+Running Apache HBase (TM) in pseudo-distributed mode
     </title>
   </properties>
 
   <body>
-      <p>This document augments what is described in the HBase 'Getting Started' in the 
- <a href="http://hbase.apache.org/docs/current/api/overview-summary.html#distributed">Distributed Operation: Pseudo- and Fully-distributed modes</a> section.
- In particular it describes scripts that allow you start extra masters and regionservers when running in pseudo-distributed mode.
+      <p>This page has been retired.  The contents have been moved to the 
+      <a href="http://hbase.apache.org/book.html#distributed">Distributed Operation: Pseudo- and Fully-distributed modes</a> section
+ in the Reference Guide.
  </p>
 
- <ol><li>Copy the pseudo-distributed suggested configuration file (feel free to take a peek and understand what it's doing)
-             <source>% cp conf/hbase-site.xml{.pseudo-distributed.template,}</source>
-    </li>
-    <li>(Optional) Start up <a href="http://hadoop.apache.org/common/docs/r0.20.2/quickstart.html#PseudoDistributed">Pseudo-distributed HDFS</a>.
-             <ol><li>If you do, go to conf/hbase-site.xml.  Uncomment the 'hbase.rootdir' property.
-                 </li>
-               <li>Additionally, if you want to test HBase with high data durability enabled, also uncomment the 'dfs.support.append' property.
-               </li>
-       </ol>
-   </li>
-<li>Start up the initial HBase cluster
-                   <source>% bin/start-hbase.sh</source>
-                   <ol>    <li>To start up an extra backup master(s) on the same server run
-                       <source>% bin/local-master-backup.sh start 1</source>
-                       Here the '1' means use ports 60001 &amp; 60011, and this backup master's logfile will be at <i>logs/hbase-${USER}-1-master-${HOSTNAME}.log</i>.
-                       To startup multiple backup masters run <source>% bin/local-master-backup.sh start 2 3</source> You can start up to 9 backup masters (10 total).
- </li>
- <li>To start up more regionservers
-     <source>% bin/local-regionservers.sh start 1</source>
-     where '1' means use ports 60201 &amp; 60301 and its logfile will be at <i>logs/hbase-${USER}-1-regionserver-${HOSTNAME}.log</i>.
-     To add 4 more regionservers in addition to the one you just started by running <source>% bin/local-regionservers.sh start 2 3 4 5</source>
-     Supports up to 99 extra regionservers (100 total).
-                    </li>
-                </ol>
-</li>
-<li>To stop the cluster
-    <ol>
-        <li>Assuming you want to stop master backup # 1, run
-            <source>% cat /tmp/hbase-${USER}-1-master.pid |xargs kill -9</source>
-            Note that bin/local-master-backup.sh stop 1 will try to stop the cluster along with the master
-                        </li>
-                        <li>To stop an individual regionserver, run
-                            <source>% bin/local-regionservers.sh stop 1
-                            </source>
-                        </li>
-                    </ol>
-</li>
-</ol>
-</body>
+ </body>
 
 </document>
 
diff --git a/src/site/xdoc/replication.xml b/src/site/xdoc/replication.xml
index 8233520eb706..727ca267636c 100644
--- a/src/site/xdoc/replication.xml
+++ b/src/site/xdoc/replication.xml
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright 2010 The Apache Software Foundation
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -23,13 +22,13 @@
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
     <title>
-      HBase Replication
+      Apache HBase (TM) Replication
     </title>
   </properties>
   <body>
     <section name="Overview">
       <p>
-        HBase replication is a way to copy data between HBase deployments. It
+        The replication feature of Apache HBase (TM) provides a way to copy data between HBase deployments. It
         can serve as a disaster recovery solution and can contribute to provide
         higher availability at the HBase layer. It can also serve more practically;
         for example, as a way to easily copy edits from a web-facing cluster to a "MapReduce"
@@ -37,7 +36,7 @@
         automatically.
       </p>
       <p>
-        The basic architecture pattern used for HBase replication is (HBase cluster) master-push;
+        The basic architecture pattern used for Apache HBase replication is (HBase cluster) master-push;
         it is much easier to keep track of what’s currently being replicated since
         each region server has its own write-ahead-log (aka WAL or HLog), just like
         other well known solutions like MySQL master/slave replication where
@@ -75,15 +74,15 @@
         of replication on the slave clusters by relying on randomization.
       </p>
       <p>
-        As of version 0.92 HBase supports master/master and cyclic replication as
-        well as replication to multiple slaves.
+        As of version 0.92, Apache HBase supports master/master and cyclic
+        replication as well as replication to multiple slaves.
       </p>
       <img src="images/replication_overview.png"/>
     </section>
     <section name="Enabling replication">
       <p>
         The guide on enabling and using cluster replication is contained
-        in the API documentation shipped with your HBase distribution.
+        in the API documentation shipped with your Apache HBase distribution.
       </p>
       <p>
         The most up-to-date documentation is
@@ -99,7 +98,7 @@
       </p>
       <section name="Normal processing">
         <p>
-          The client uses a HBase API that sends a Put, Delete or ICV to a region
+          The client uses an API that sends a Put, Delete or ICV to a region
           server. The key values are transformed into a WALEdit by the region
           server and is inspected by the replication code that, for each family
           that is scoped for replication, adds the scope to the edit. The edit
@@ -163,6 +162,157 @@
         This section describes in depth how each of replication's internal
         features operate.
       </p>
+      <section name="Replication Zookeeper State">
+        <p>
+            HBase replication maintains all of its state in Zookeeper. By default, this state is 
+            contained in the base znode:
+        </p>
+        <pre>
+                /hbase/replication
+        </pre>
+        <p>
+            There are three major child znodes in the base replication znode:
+            <ul>
+            <li><b>State znode:</b> /hbase/replication/state</li>
+            <li><b>Peers znode:</b> /hbase/replication/peers</li>
+            <li><b>RS znode:</b> /hbase/replication/rs</li>
+            </ul>
+        </p>
+        <section name="The State znode">
+            <p>
+                The <b>state znode</b> indicates whether or not replication is enabled on the cluster 
+                corresponding to this zookeeper quorum. It does not have any child znodes and simply 
+                contains a boolean value. This value is initialized on startup based on the 
+                <i>hbase.replication</i> config parameter in the <i>hbase-site.xml</i> file. The status 
+                value is read/maintained by the <i>ReplicationZookeeper.ReplicationStatusTracker</i> 
+                class. It is also cached locally using an AtomicBoolean in the <i>ReplicationZookeeper</i> 
+                class. This value can be changed on a live cluster using the <i>stop_replication</i> 
+                command available through the hbase shell.
+            </p>
+            <pre>
+                /hbase/replication/state [VALUE: true]
+            </pre>
+        </section>
+        <section name="The Peers znode">
+            <p>
+                The <b>peers znode</b> contains a list of all peer replication clusters and the 
+                current replication state of those clusters. It has one child <i>peer znode</i> 
+                for each peer cluster. The <i>peer znode</i> is named with the cluster id provided 
+                by the user in the HBase shell. The value of the <i>peer znode</i> contains 
+                the peers cluster key provided by the user in the HBase Shell. The cluster key 
+                contains a list of zookeeper nodes in the clusters quorum, the client port for the 
+                zookeeper quorum, and the base znode for HBase 
+                (i.e. “zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase”).
+            </p>
+            <pre>
+                /hbase/replication/peers
+                    /1 [Value: zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase]
+                    /2 [Value: zk5.host.com,zk6.host.com,zk7.host.com:2181:/hbase]
+            </pre>
+            <p>
+                Each of these <i>peer znodes</i> has a child znode that indicates whether or not 
+                replication is enabled on that peer cluster. These <i>peer-state znodes</i> do not 
+                have child znodes and simply contain a boolean value (i.e. ENABLED or DISABLED). 
+                This value is read/maintained by the <i>ReplicationPeer.PeerStateTracker</i> class. 
+                It is also cached locally using an AtomicBoolean in the <i>ReplicationPeer</i> class.
+            </p>
+            <pre>
+                /hbase/replication/peers
+                    /1/peer-state [Value: ENABLED]
+                    /2/peer-state [Value: DISABLED]
+            </pre>
+        </section>
+        <section name="The RS znode">
+            <p>
+                The <b>rs znode</b> contains a list of all outstanding HLog files in the cluster 
+                that need to be replicated. The list is divided into a set of queues organized by 
+                region server and the peer cluster the region server is shipping the HLogs to. The 
+                <b>rs znode</b> has one child znode for each region server in the cluster. The child 
+                znode name is simply the regionserver name (a concatenation of the region server’s 
+                hostname, client port and start code). These region servers could either be dead or alive.
+            </p>
+            <pre>
+                /hbase/replication/rs
+                    /hostname.example.org,6020,1234
+                    /hostname2.example.org,6020,2856
+            </pre>
+            <p>
+                Within each region server znode, the region server maintains a set of HLog replication 
+                queues. Each region server has one queue for every peer cluster it replicates to. 
+                These queues are represented by child znodes named using the cluster id of the peer 
+                cluster they represent (see the peer znode section).
+            </p>
+            <pre>
+                /hbase/replication/rs
+                    /hostname.example.org,6020,1234
+                        /1
+                        /2
+            </pre>
+            <p>
+                Each queue has one child znode for every HLog that still needs to be replicated. 
+                The value of these HLog child znodes is the latest position that has been replicated. 
+                This position is updated every time a HLog entry is replicated.
+            </p>
+            <pre>
+                /hbase/replication/rs
+                    /hostname.example.org,6020,1234
+                        /1
+                            23522342.23422 [VALUE: 254]
+                            12340993.22342 [VALUE: 0]
+            </pre>
+        </section>
+      </section>
+      <section name="Configuration Parameters">
+        <section name="Zookeeper znode paths">
+            <p>
+                All of the base znode names are configurable through parameters:
+            </p>
+            <table border="1">
+                <tr>
+                    <td><b>Parameter</b></td>
+                    <td><b>Default Value</b></td>
+                </tr>
+                                <tr>
+                    <td>zookeeper.znode.parent</td>
+                    <td>/hbase</td>
+                </tr>
+                <tr>
+                    <td>zookeeper.znode.replication</td>
+                    <td>replication</td>
+                </tr>
+                <tr>
+                    <td>zookeeper.znode.replication.peers</td>
+                    <td>peers</td>
+                </tr>
+                <tr>
+                    <td>zookeeper.znode.replication.peers.state</td>
+                    <td>peer-state</td>
+                </tr>
+                <tr>
+                    <td>zookeeper.znode.replication.rs</td>
+                    <td>rs</td>
+                </tr>
+            </table>
+            <p>
+                The default replication znode structure looks like the following:
+            </p>
+            <pre>
+                /hbase/replication/state
+                /hbase/replication/peers/{peerId}/peer-state
+                /hbase/replication/rs
+            </pre>
+        </section>
+        <section name="Other parameters">
+            <ul>
+                <li><b>hbase.replication</b> (Default: false) - Controls whether replication is enabled 
+                or disabled for the cluster.</li>
+                <li><b>replication.sleep.before.failover</b> (Default: 2000) - The amount of time a failover 
+                worker waits before attempting to replicate a dead region server’s HLog queues.</li>
+                <li><b>replication.executor.workers</b> (Default: 1) - The number of dead region servers 
+                one region server should attempt to failover simultaneously.</li>
+            </ul>
+        </section>
+      </section>
       <section name="Choosing region servers to replicate to">
         <p>
           When a master cluster RS initiates a replication source to a slave cluster,
diff --git a/src/site/xdoc/sponsors.xml b/src/site/xdoc/sponsors.xml
index e39730bbbd2e..639e0c39acbb 100644
--- a/src/site/xdoc/sponsors.xml
+++ b/src/site/xdoc/sponsors.xml
@@ -16,20 +16,29 @@
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
   <properties>
-    <title>Installing HBase on Windows using Cygwin</title>
+    <title>Apache HBase&#153; Sponsors</title>
   </properties>
 
 <body>
 <section name="Sponsors">
-<p>The below companies have been gracious enough to provide their commerical tool offerings free of charge to the Apache HBase project.
+    <p>First off, thanks to <a href="http://www.apache.org/foundation/thanks.html">all who sponsor</a>
+       our parent, the Apache Software Foundation.
+    </p>
+<p>The below companies have been gracious enough to provide their commerical tool offerings free of charge to the Apache HBase&#153; project.
 <ul>
 	<li>The crew at <a href="http://www.ej-technologies.com/">ej-technologies</a> have
         been let us use <a href="http://www.ej-technologies.com/products/jprofiler/overview.html">JProfiler</a> for years now.</li>
 	<li>The lads at <a href="http://headwaysoftware.com/">headway software</a> have
         given us a license for <a href="http://headwaysoftware.com/products/?code=Restructure101">Restructure101</a>
         so we can untangle our interdependency mess.</li>
+	<li><a href="http://www.yourkit.com">YourKit</a> allows us to use their <a href="http://www.yourkit.com/overview/index.jsp">Java Profiler</a>.</li>
+	<li>Some of us use <a href="http://www.jetbrains.com/idea">IntelliJ IDEA</a> thanks to <a href="http://www.jetbrains.com/">JetBrains</a>.</li>
 </ul>
 </p>
 </section>
+<section name="Sponsoring the Apache Software Foundation">
+<p>To contribute to the Apache Software Foundation, a good idea in our opinion, see the <a href="http://www.apache.org/foundation/sponsorship.html">ASF Sponsorship</a> page.
+</p>
+</section>
 </body>
 </document>

From cb2535e7f2324d133bcadf696926e713350b28db Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 13 Mar 2013 16:54:25 +0000
Subject: [PATCH 0839/1540] HBASE_8088 ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456049 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/configuration.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/docbkx/configuration.xml b/src/docbkx/configuration.xml
index 02c49a3482e0..2d182fa6a600 100644
--- a/src/docbkx/configuration.xml
+++ b/src/docbkx/configuration.xml
@@ -669,7 +669,7 @@ stopping hbase...............</programlisting> Shutdown can take a moment to
     The generated file is a docbook section with a glossary
     in it-->
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude"
-        href="../../hbase-common/src/main/resources/hbase-default.xml" />
+        href="../../src/main/resources/hbase-default.xml" />
     </section>
 
       <section xml:id="hbase.env.sh">

From 6a9676b8b38edc28a1a6597931c6dea4625ad456 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 13 Mar 2013 21:05:58 +0000
Subject: [PATCH 0840/1540] Add smart-apply-patch.sh to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456129 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/smart-apply-patch.sh | 87 ++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100755 dev-support/smart-apply-patch.sh

diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
new file mode 100755
index 000000000000..74a7128fb51c
--- /dev/null
+++ b/dev-support/smart-apply-patch.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -e
+
+PATCH_FILE=$1
+if [ -z "$PATCH_FILE" ]; then
+  echo usage: $0 patch-file
+  exit 1
+fi
+
+PATCH=${PATCH:-patch} # allow overriding patch binary
+
+# Cleanup handler for temporary files
+TOCLEAN=""
+cleanup() {
+  rm $TOCLEAN
+  exit $1
+}
+trap "cleanup 1" HUP INT QUIT TERM
+
+# Allow passing "-" for stdin patches
+if [ "$PATCH_FILE" == "-" ]; then
+  PATCH_FILE=/tmp/tmp.in.$$
+  cat /dev/fd/0 > $PATCH_FILE
+  TOCLEAN="$TOCLEAN $PATCH_FILE"
+fi
+
+# Come up with a list of changed files into $TMP
+TMP=/tmp/tmp.paths.$$
+TOCLEAN="$TOCLEAN $TMP"
+
+if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
+  PLEVEL=0
+  #if the patch applied at P0 there is the possability that all we are doing
+  # is adding new files and they would apply anywhere. So try to guess the
+  # correct place to put those files.
+
+  TMP2=/tmp/tmp.paths.2.$$
+  TOCLEAN="$TOCLEAN $TMP2"
+
+  grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
+
+  #first off check that all of the files do not exist
+  FOUND_ANY=0
+  for CHECK_FILE in $(cat $TMP2)
+  do
+    if [[ -f $CHECK_FILE ]]; then
+      FOUND_ANY=1
+    fi
+  done
+
+  if [[ "$FOUND_ANY" = "0" ]]; then
+    #all of the files are new files so we have to guess where the correct place to put it is.
+
+    # if all of the lines start with a/ or b/, then this is a git patch that
+    # was generated without --no-prefix
+    if ! grep -qv '^a/\|^b/' $TMP2 ; then
+      echo Looks like this is a git patch. Stripping a/ and b/ prefixes
+      echo and incrementing PLEVEL
+      PLEVEL=$[$PLEVEL + 1]
+      sed -i -e 's,^[ab]/,,' $TMP2
+    fi
+  fi
+elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+  PLEVEL=1
+elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+  PLEVEL=2
+else
+  echo "The patch does not appear to apply with p0 to p2";
+  cleanup 1;
+fi
+
+echo Going to apply patch with: $PATCH -p$PLEVEL
+$PATCH -p$PLEVEL -E < $PATCH_FILE
+
+cleanup $?

From cbf1e90f81e37dcca9891e9dde9fef97e144445c Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 13 Mar 2013 23:59:53 +0000
Subject: [PATCH 0841/1540] HBASE-8031 Adopt goraci as an Integration test

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456284 13f79535-47bb-0310-9956-ffa450edef68
---
 .../test/IntegrationTestBigLinkedList.java    | 1010 +++++++++++++++++
 1 file changed, 1010 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java

diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
new file mode 100644
index 000000000000..0f4aa4ceae1d
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -0,0 +1,1010 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.test;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.IntegrationTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * This is an integration test borrowed from goraci, written by Keith Turner,
+ * which is in turn inspired by the Accumulo test called continous ingest (ci).
+ * The original source code can be found here:
+ * https://github.com/keith-turner/goraci
+ * https://github.com/enis/goraci/
+ *
+ * Apache Accumulo [0] has a simple test suite that verifies that data is not
+ * lost at scale. This test suite is called continuous ingest. This test runs
+ * many ingest clients that continually create linked lists containing 25
+ * million nodes. At some point the clients are stopped and a map reduce job is
+ * run to ensure no linked list has a hole. A hole indicates data was lost.··
+ *
+ * The nodes in the linked list are random. This causes each linked list to
+ * spread across the table. Therefore if one part of a table loses data, then it
+ * will be detected by references in another part of the table.
+ *
+ * THE ANATOMY OF THE TEST
+ *
+ * Below is rough sketch of how data is written. For specific details look at
+ * the Generator code.
+ *
+ * 1 Write out 1 million nodes· 2 Flush the client· 3 Write out 1 million that
+ * reference previous million· 4 If this is the 25th set of 1 million nodes,
+ * then update 1st set of million to point to last· 5 goto 1
+ *
+ * The key is that nodes only reference flushed nodes. Therefore a node should
+ * never reference a missing node, even if the ingest client is killed at any
+ * point in time.
+ *
+ * When running this test suite w/ Accumulo there is a script running in
+ * parallel called the Aggitator that randomly and continuously kills server
+ * processes.·· The outcome was that many data loss bugs were found in Accumulo
+ * by doing this.· This test suite can also help find bugs that impact uptime
+ * and stability when· run for days or weeks.··
+ *
+ * This test suite consists the following· - a few Java programs· - a little
+ * helper script to run the java programs - a maven script to build it.··
+ *
+ * When generating data, its best to have each map task generate a multiple of
+ * 25 million. The reason for this is that circular linked list are generated
+ * every 25M. Not generating a multiple in 25M will result in some nodes in the
+ * linked list not having references. The loss of an unreferenced node can not
+ * be detected.
+ *
+ *
+ * Below is a description of the Java programs
+ *
+ * Generator - A map only job that generates data. As stated previously,·
+ * its best to generate data in multiples of 25M.
+ *
+ * Verify - A map reduce job that looks for holes. Look at the counts after running. REFERENCED and
+ * UNREFERENCED are· ok, any UNDEFINED counts are bad. Do not run at the· same
+ * time as the Generator.
+ *
+ * Walker - A standalong program that start following a linked list· and emits timing info.··
+ *
+ * Print - A standalone program that prints nodes in the linked list
+ *
+ * Delete - A standalone program that deletes a single node
+ *
+ * This class can be run as a unit test, as an integration test, or from the command line
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestBigLinkedList extends Configured implements Tool {
+
+  private static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
+
+  private static final String DEFAULT_TABLE_NAME = "ci";
+
+  private static byte[] FAMILY_NAME = Bytes.toBytes("meta");
+
+  //link to the id of the prev node in the linked list
+  private static final byte[] COLUMN_PREV = Bytes.toBytes("prev");
+
+  //identifier of the mapred task that generated this row
+  private static final byte[] COLUMN_CLIENT = Bytes.toBytes("client");
+
+  //the id of the row within the same client.
+  private static final byte[] COLUMN_COUNT = Bytes.toBytes("count");
+
+  /** How many rows to write per map task. This has to be a multiple of 25M */
+  private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY
+    = "IntegrationTestBigLinkedList.generator.num_rows";
+
+  private static final String GENERATOR_NUM_MAPPERS_KEY
+    = "IntegrationTestBigLinkedList.generator.map.tasks";
+
+  static class CINode {
+    long key;
+    long prev;
+    String client;
+    long count;
+  }
+
+  /**
+   * A Map only job that generates random linked list and stores them.
+   */
+  static class Generator extends Configured implements Tool {
+
+    private static final Log LOG = LogFactory.getLog(Generator.class);
+
+    private static final int WIDTH = 1000000;
+    private static final int WRAP = WIDTH * 25;
+
+    public static enum Counts {
+      UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+    }
+
+    static class GeneratorInputFormat extends InputFormat<LongWritable,NullWritable> {
+      static class GeneratorInputSplit extends InputSplit implements Writable {
+        @Override
+        public long getLength() throws IOException, InterruptedException {
+          return 1;
+        }
+        @Override
+        public String[] getLocations() throws IOException, InterruptedException {
+          return new String[0];
+        }
+        @Override
+        public void readFields(DataInput arg0) throws IOException {
+        }
+        @Override
+        public void write(DataOutput arg0) throws IOException {
+        }
+      }
+
+      static class GeneratorRecordReader extends RecordReader<LongWritable,NullWritable> {
+        private long count;
+        private long numNodes;
+        private Random rand;
+
+        @Override
+        public void close() throws IOException {
+        }
+
+        @Override
+        public LongWritable getCurrentKey() throws IOException, InterruptedException {
+          return new LongWritable(Math.abs(rand.nextLong()));
+        }
+
+        @Override
+        public NullWritable getCurrentValue() throws IOException, InterruptedException {
+          return NullWritable.get();
+        }
+
+        @Override
+        public float getProgress() throws IOException, InterruptedException {
+          return (float)(count / (double)numNodes);
+        }
+
+        @Override
+        public void initialize(InputSplit arg0, TaskAttemptContext context)
+            throws IOException, InterruptedException {
+          numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
+          rand = new Random();
+        }
+
+        @Override
+        public boolean nextKeyValue() throws IOException, InterruptedException {
+          return count++ < numNodes;
+        }
+
+      }
+
+      @Override
+      public RecordReader<LongWritable,NullWritable> createRecordReader(
+          InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
+        GeneratorRecordReader rr = new GeneratorRecordReader();
+        rr.initialize(split, context);
+        return rr;
+      }
+
+      @Override
+      public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
+        int numMappers = job.getConfiguration().getInt(GENERATOR_NUM_MAPPERS_KEY, 1);
+
+        ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numMappers);
+
+        for (int i = 0; i < numMappers; i++) {
+          splits.add(new GeneratorInputSplit());
+        }
+
+        return splits;
+      }
+    }
+
+    /** Ensure output files from prev-job go to map inputs for current job */
+    static class OneFilePerMapperSFIF<K, V> extends SequenceFileInputFormat<K, V> {
+      @Override
+      protected boolean isSplitable(JobContext context, Path filename) {
+        return false;
+      }
+    }
+
+    /**
+     * Some ASCII art time:
+     * [ . . . ] represents one batch of random longs of length WIDTH
+     *
+     *                _________________________
+     *               |                  ______ |
+     *               |                 |      ||
+     *             __+_________________+_____ ||
+     *             v v                 v     |||
+     * first   = [ . . . . . . . . . . . ]   |||
+     *             ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^     |||
+     *             | | | | | | | | | | |     |||
+     * prev    = [ . . . . . . . . . . . ]   |||
+     *             ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^     |||
+     *             | | | | | | | | | | |     |||
+     * current = [ . . . . . . . . . . . ]   |||
+     *                                       |||
+     * ...                                   |||
+     *                                       |||
+     * last    = [ . . . . . . . . . . . ]   |||
+     *             | | | | | | | | | | |-----|||
+     *             |                 |--------||
+     *             |___________________________|
+     */
+    static class GeneratorMapper
+      extends Mapper<LongWritable, NullWritable, NullWritable, NullWritable> {
+      Random rand = new Random();
+
+      long[] first = null;
+      long[] prev = null;
+      long[] current = new long[WIDTH];
+      byte[] id;
+      long count = 0;
+      int i;
+      HTable table;
+      long numNodes;
+      long wrap = WRAP;
+
+      protected void setup(Context context) throws IOException, InterruptedException {
+        id = Bytes.toBytes(UUID.randomUUID().toString());
+        Configuration conf = context.getConfiguration();
+        table = new HTable(conf, getTableName(conf));
+        table.setAutoFlush(false);
+        table.setWriteBufferSize(4 * 1024 * 1024);
+        numNodes = context.getConfiguration().getLong(GENERATOR_NUM_MAPPERS_KEY, 25000000);
+        if (numNodes < 25000000) {
+          wrap = numNodes;
+        }
+      };
+
+      protected void cleanup(Context context) throws IOException ,InterruptedException {
+        table.close();
+      };
+
+      @Override
+      protected void map(LongWritable key, NullWritable value, Context output) throws IOException {
+        current[i++] = Math.abs(key.get());
+
+        if (i == current.length) {
+          persist(output, count, prev, current, id);
+          i = 0;
+
+          if (first == null)
+            first = current;
+          prev = current;
+          current = new long[WIDTH];
+
+          count += current.length;
+          output.setStatus("Count " + count);
+
+          if (count % wrap == 0) {
+            // this block of code turns the 1 million linked list of length 25 into one giant
+            //circular linked list of 25 million
+            circularLeftShift(first);
+
+            persist(output, -1, prev, first, null);
+
+            first = null;
+            prev = null;
+          }
+        }
+      }
+
+      private static void circularLeftShift(long[] first) {
+        long ez = first[0];
+        for (int i = 0; i < first.length - 1; i++)
+          first[i] = first[i + 1];
+        first[first.length - 1] = ez;
+      }
+
+      private void persist(Context output, long count, long[] prev, long[] current, byte[] id)
+          throws IOException {
+        for (int i = 0; i < current.length; i++) {
+          Put put = new Put(Bytes.toBytes(current[i]));
+          put.add(FAMILY_NAME, COLUMN_PREV, Bytes.toBytes(prev == null ? -1 : prev[i]));
+
+          if (count > 0) {
+            put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + 1));
+          }
+          if (id != null) {
+            put.add(FAMILY_NAME, COLUMN_CLIENT, id);
+          }
+          table.put(put);
+
+          if (i % 1000 == 0) {
+            // Tickle progress every so often else maprunner will think us hung
+            output.progress();
+          }
+        }
+
+        table.flushCommits();
+      }
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+      if (args.length < 3) {
+        System.out.println("Usage : " + Generator.class.getSimpleName() +
+            " <num mappers> <num nodes per map> <tmp output dir>");
+        System.out.println("   where <num nodes per map> should be a multiple of 25M");
+        return 0;
+      }
+
+      int numMappers = Integer.parseInt(args[0]);
+      long numNodes = Long.parseLong(args[1]);
+      Path tmpOutput = new Path(args[2]);
+      return run(numMappers, numNodes, tmpOutput);
+    }
+
+    protected void createSchema() throws IOException {
+      HBaseAdmin admin = new HBaseAdmin(getConf());
+      byte[] tableName = getTableName(getConf());
+      if (!admin.tableExists(tableName)) {
+        HTableDescriptor htd = new HTableDescriptor(getTableName(getConf()));
+        htd.addFamily(new HColumnDescriptor(FAMILY_NAME));
+        admin.createTable(htd);
+      }
+      admin.close();
+    }
+
+    public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput)
+        throws Exception {
+      LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
+          + ", numNodes=" + numNodes);
+      Job job = new Job(getConf());
+
+      job.setJobName("Random Input Generator");
+      job.setNumReduceTasks(0);
+      job.setJarByClass(getClass());
+
+      job.setInputFormatClass(GeneratorInputFormat.class);
+      job.setOutputKeyClass(LongWritable.class);
+      job.setOutputValueClass(NullWritable.class);
+
+      job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
+      job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+
+      job.setMapperClass(Mapper.class); //identity mapper
+
+      FileOutputFormat.setOutputPath(job, tmpOutput);
+      job.setOutputFormatClass(SequenceFileOutputFormat.class);
+
+      boolean success = job.waitForCompletion(true);
+
+      return success ? 0 : 1;
+    }
+
+    public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Exception {
+      LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
+      createSchema();
+
+      Job job = new Job(getConf());
+
+      job.setJobName("Link Generator");
+      job.setNumReduceTasks(0);
+      job.setJarByClass(getClass());
+
+      FileInputFormat.setInputPaths(job, tmpOutput);
+      job.setInputFormatClass(OneFilePerMapperSFIF.class);
+      job.setOutputKeyClass(NullWritable.class);
+      job.setOutputValueClass(NullWritable.class);
+
+      job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
+      job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+
+      job.setMapperClass(GeneratorMapper.class);
+
+      job.setOutputFormatClass(NullOutputFormat.class);
+
+      job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+
+      boolean success = job.waitForCompletion(true);
+
+      return success ? 0 : 1;
+    }
+
+    public int run(int numMappers, long numNodes, Path tmpOutput) throws Exception {
+      int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput);
+      if (ret > 0) {
+        return ret;
+      }
+
+      return runGenerator(numMappers, numNodes, tmpOutput);
+    }
+  }
+
+  /**
+   * A Map Reduce job that verifies that the linked lists generated by
+   * {@link Generator} do not have any holes.
+   */
+  static class Verify extends Configured implements Tool {
+
+    private static final Log LOG = LogFactory.getLog(Verify.class);
+    private static final VLongWritable DEF = new VLongWritable(-1);
+
+    private Job job;
+
+    public static class VerifyMapper extends TableMapper<LongWritable, VLongWritable> {
+      private LongWritable row = new LongWritable();
+      private LongWritable ref = new LongWritable();
+      private VLongWritable vrow = new VLongWritable();
+
+      @Override
+      protected void map(ImmutableBytesWritable key, Result value, Context context)
+          throws IOException ,InterruptedException {
+        row.set(Bytes.toLong(key.get()));
+        context.write(row, DEF);
+
+        long prev = Bytes.toLong(value.getValue(FAMILY_NAME, COLUMN_PREV));
+        if (prev >= 0) {
+          ref.set(prev);
+          vrow.set(Bytes.toLong(key.get()));
+          context.write(ref, vrow);
+        }
+      }
+    }
+
+    public static enum Counts {
+      UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+    }
+
+    public static class VerifyReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+      private ArrayList<Long> refs = new ArrayList<Long>();
+
+      public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context)
+          throws IOException, InterruptedException {
+
+        int defCount = 0;
+
+        refs.clear();
+        for (VLongWritable type : values) {
+          if (type.get() == -1) {
+            defCount++;
+          } else {
+            refs.add(type.get());
+          }
+        }
+
+        // TODO check for more than one def, should not happen
+
+        if (defCount == 0 && refs.size() > 0) {
+          // this is bad, found a node that is referenced but not defined. It must have been
+          //lost, emit some info about this node for debugging purposes.
+
+          StringBuilder sb = new StringBuilder();
+          String comma = "";
+          for (Long ref : refs) {
+            sb.append(comma);
+            comma = ",";
+            sb.append(String.format("%016x", ref));
+          }
+
+          context.write(new Text(String.format("%016x", key.get())), new Text(sb.toString()));
+          context.getCounter(Counts.UNDEFINED).increment(1);
+
+        } else if (defCount > 0 && refs.size() == 0) {
+          // node is defined but not referenced
+          context.getCounter(Counts.UNREFERENCED).increment(1);
+        } else {
+          // node is defined and referenced
+          context.getCounter(Counts.REFERENCED).increment(1);
+        }
+
+      }
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+
+      if (args.length != 2) {
+        System.out.println("Usage : " + Verify.class.getSimpleName() + " <output dir> <num reducers>");
+        return 0;
+      }
+
+      String outputDir = args[0];
+      int numReducers = Integer.parseInt(args[1]);
+
+       return run(outputDir, numReducers);
+    }
+
+    public int run(String outputDir, int numReducers) throws Exception {
+      return run(new Path(outputDir), numReducers);
+    }
+
+    public int run(Path outputDir, int numReducers) throws Exception {
+      LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
+
+      job = new Job(getConf());
+
+      job.setJobName("Link Verifier");
+      job.setNumReduceTasks(numReducers);
+      job.setJarByClass(getClass());
+
+      Scan scan = new Scan();
+      scan.addColumn(FAMILY_NAME, COLUMN_PREV);
+      scan.setCaching(10000);
+      scan.setCacheBlocks(false);
+
+      TableMapReduceUtil.initTableMapperJob(getTableName(getConf()), scan,
+          VerifyMapper.class, LongWritable.class, VLongWritable.class, job);
+
+      job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+
+      job.setReducerClass(VerifyReducer.class);
+      job.setOutputFormatClass(TextOutputFormat.class);
+      TextOutputFormat.setOutputPath(job, outputDir);
+
+      boolean success = job.waitForCompletion(true);
+
+      return success ? 0 : 1;
+    }
+
+    public boolean verify(long expectedReferenced) throws Exception {
+      if (job == null) {
+        throw new IllegalStateException("You should call run() first");
+      }
+
+      Counters counters = job.getCounters();
+
+      Counter referenced = counters.findCounter(Counts.REFERENCED);
+      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
+      Counter undefined = counters.findCounter(Counts.UNDEFINED);
+
+      boolean success = true;
+      //assert
+      if (expectedReferenced != referenced.getValue()) {
+        LOG.error("Expected referenced count does not match with actual referenced count. " +
+            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
+        success = false;
+      }
+
+      if (unreferenced.getValue() > 0) {
+        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
+        success = false;
+      }
+
+      if (undefined.getValue() > 0) {
+        LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
+        success = false;
+      }
+
+      return success;
+    }
+  }
+
+  /**
+   * Executes Generate and Verify in a loop. Data is not cleaned between runs, so each iteration
+   * adds more data.
+   */
+  private static class Loop extends Configured implements Tool {
+
+    private static final Log LOG = LogFactory.getLog(Loop.class);
+
+    protected void runGenerator(int numMappers, long numNodes, String outputDir) throws Exception {
+      Path outputPath = new Path(outputDir);
+      UUID uuid = UUID.randomUUID(); //create a random UUID.
+      Path generatorOutput = new Path(outputPath, uuid.toString());
+
+      Generator generator = new Generator();
+      generator.setConf(getConf());
+      int retCode = generator.run(numMappers, numNodes, generatorOutput);
+
+      if (retCode > 0) {
+        throw new RuntimeException("Generator failed with return code: " + retCode);
+      }
+    }
+
+    protected void runVerify(String outputDir, int numReducers, long expectedNumNodes) throws Exception {
+      Path outputPath = new Path(outputDir);
+      UUID uuid = UUID.randomUUID(); //create a random UUID.
+      Path iterationOutput = new Path(outputPath, uuid.toString());
+
+      Verify verify = new Verify();
+      verify.setConf(getConf());
+      int retCode = verify.run(iterationOutput, numReducers);
+      if (retCode > 0) {
+        throw new RuntimeException("Verify.run failed with return code: " + retCode);
+      }
+
+      boolean verifySuccess = verify.verify(expectedNumNodes);
+      if (!verifySuccess) {
+        throw new RuntimeException("Verify.verify failed");
+      }
+
+      LOG.info("Verify finished with succees. Total nodes=" + expectedNumNodes);
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+      if (args.length < 5) {
+        System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers>");
+        return 1;
+      }
+
+      LOG.info("Running Loop with args:" + Arrays.deepToString(args));
+
+      int numIterations = Integer.parseInt(args[0]);
+      int numMappers = Integer.parseInt(args[1]);
+      long numNodes = Long.parseLong(args[2]);
+      String outputDir = args[3];
+      int numReducers = Integer.parseInt(args[4]);
+
+      long expectedNumNodes = 0;
+
+      if (numIterations < 0) {
+        numIterations = Integer.MAX_VALUE; //run indefinitely (kind of)
+      }
+
+      for (int i=0; i < numIterations; i++) {
+        LOG.info("Starting iteration = " + i);
+        runGenerator(numMappers, numNodes, outputDir);
+        expectedNumNodes += numMappers * numNodes;
+
+        runVerify(outputDir, numReducers, expectedNumNodes);
+      }
+
+      return 0;
+    }
+  }
+
+  /**
+   * A stand alone program that prints out portions of a list created by {@link Generator}
+   */
+  private static class Print extends Configured implements Tool {
+    public int run(String[] args) throws Exception {
+      Options options = new Options();
+      options.addOption("s", "start", true, "start key");
+      options.addOption("e", "end", true, "end key");
+      options.addOption("l", "limit", true, "number to print");
+
+      GnuParser parser = new GnuParser();
+      CommandLine cmd = null;
+      try {
+        cmd = parser.parse(options, args);
+        if (cmd.getArgs().length != 0) {
+          throw new ParseException("Command takes no arguments");
+        }
+      } catch (ParseException e) {
+        System.err.println("Failed to parse command line " + e.getMessage());
+        System.err.println();
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp(getClass().getSimpleName(), options);
+        System.exit(-1);
+      }
+
+      HTable table = new HTable(getConf(), getTableName(getConf()));
+
+      Scan scan = new Scan();
+      scan.setBatch(10000);
+
+      if (cmd.hasOption("s"))
+        scan.setStartRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("s"), 16).longValue()));
+
+      if (cmd.hasOption("e"))
+        scan.setStopRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("e"), 16).longValue()));
+
+      int limit = 0;
+      if (cmd.hasOption("l"))
+        limit = Integer.parseInt(cmd.getOptionValue("l"));
+      else
+        limit = 100;
+
+      ResultScanner scanner = table.getScanner(scan);
+
+      CINode node = new CINode();
+      Result result = scanner.next();
+      int count = 0;
+      while (result != null && count++ < limit) {
+        node = getCINode(result, node);
+        System.out.printf("%016x:%016x:%012d:%s\n", node.key, node.prev, node.count, node.client);
+        result = scanner.next();
+      }
+      scanner.close();
+      table.close();
+
+      return 0;
+    }
+  }
+
+  /**
+   * A stand alone program that deletes a single node.
+   */
+  private static class Delete extends Configured implements Tool {
+    public int run(String[] args) throws Exception {
+      if (args.length != 1) {
+        System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
+        return 0;
+      }
+      long val = new BigInteger(args[0], 16).longValue();
+
+      org.apache.hadoop.hbase.client.Delete delete
+        = new org.apache.hadoop.hbase.client.Delete(Bytes.toBytes(val));
+
+      HTable table = new HTable(getConf(), getTableName(getConf()));
+
+      table.delete(delete);
+      table.flushCommits();
+      table.close();
+
+      System.out.println("Delete successful");
+      return 0;
+    }
+  }
+
+  /**
+   * A stand alone program that follows a linked list created by {@link Generator} and prints timing info.
+   */
+  private static class Walker extends Configured implements Tool {
+    public int run(String[] args) throws IOException {
+      Options options = new Options();
+      options.addOption("n", "num", true, "number of queries");
+
+      GnuParser parser = new GnuParser();
+      CommandLine cmd = null;
+      try {
+        cmd = parser.parse(options, args);
+        if (cmd.getArgs().length != 0) {
+          throw new ParseException("Command takes no arguments");
+        }
+      } catch (ParseException e) {
+        System.err.println("Failed to parse command line " + e.getMessage());
+        System.err.println();
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp(getClass().getSimpleName(), options);
+        System.exit(-1);
+      }
+
+      long maxQueries = Long.MAX_VALUE;
+      if (cmd.hasOption('n')) {
+        maxQueries = Long.parseLong(cmd.getOptionValue("n"));
+      }
+
+      HTable table = new HTable(getConf(), getTableName(getConf()));
+
+      Random rand = new Random();
+
+      long numQueries = 0;
+
+      while (numQueries < maxQueries) {
+        CINode node = findStartNode(rand, table);
+        numQueries++;
+        while (node != null && node.prev >= 0 && numQueries < maxQueries) {
+          long prev = node.prev;
+
+          long t1 = System.currentTimeMillis();
+          node = getNode(prev, table, node);
+          long t2 = System.currentTimeMillis();
+          System.out.printf("CQ %d %016x \n", t2 - t1, prev); //cold cache
+          numQueries++;
+
+          t1 = System.currentTimeMillis();
+          node = getNode(prev, table, node);
+          t2 = System.currentTimeMillis();
+          System.out.printf("HQ %d %016x \n", t2 - t1, prev); //hot cache
+          numQueries++;
+        }
+      }
+
+      table.close();
+      return 0;
+    }
+
+    private static CINode findStartNode(Random rand, HTable table) throws IOException {
+      Scan scan = new Scan();
+      scan.setStartRow(Bytes.toBytes(Math.abs(rand.nextLong())));
+      scan.setBatch(1);
+      scan.addColumn(FAMILY_NAME, COLUMN_PREV);
+
+      long t1 = System.currentTimeMillis();
+      ResultScanner scanner = table.getScanner(scan);
+      Result result = scanner.next();
+      long t2 = System.currentTimeMillis();
+      scanner.close();
+
+      if ( result != null) {
+        CINode node = getCINode(result, new CINode());
+        System.out.printf("FSR %d %016x\n", t2 - t1, node.key);
+        return node;
+      }
+
+      System.out.println("FSR " + (t2 - t1));
+
+      return null;
+    }
+
+    private CINode getNode(long row, HTable table, CINode node) throws IOException {
+      Get get = new Get(Bytes.toBytes(row));
+      get.addColumn(FAMILY_NAME, COLUMN_PREV);
+      Result result = table.get(get);
+      return getCINode(result, node);
+    }
+  }
+
+  private static byte[] getTableName(Configuration conf) {
+    return Bytes.toBytes(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME));
+  }
+
+  private static CINode getCINode(Result result, CINode node) {
+    node.key = Bytes.toLong(result.getRow());
+    if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) {
+      node.prev = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_PREV));
+    }
+    if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) {
+      node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT));
+    }
+    if (result.containsColumn(FAMILY_NAME, COLUMN_CLIENT)) {
+      node.client = Bytes.toString(result.getValue(FAMILY_NAME, COLUMN_CLIENT));
+    }
+    return node;
+  }
+
+  private IntegrationTestingUtility util;
+
+  @Before
+  public void setUp() throws Exception {
+    util = getTestingUtil();
+    util.initializeCluster(3);
+    this.setConf(util.getConfiguration());
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    util.restoreCluster();
+  }
+
+  @Test
+  public void testContinuousIngest() throws IOException, Exception {
+    //Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers>
+    int ret = ToolRunner.run(getTestingUtil().getConfiguration(), new Loop(),
+        new String[] {"1", "1", "2000000",
+                     getTestDir("IntegrationTestBigLinkedList", "testContinuousIngest").toString(), "1"});
+    org.junit.Assert.assertEquals(0, ret);
+  }
+
+  public Path getTestDir(String testName, String subdir) throws IOException {
+    //HBaseTestingUtility.getDataTestDirOnTestFs() has not been backported.
+    FileSystem fs = FileSystem.get(getConf());
+    Path base = new Path(fs.getWorkingDirectory(), "test-data");
+    String randomStr = UUID.randomUUID().toString();
+    Path testDir = new Path(base, randomStr);
+    fs.deleteOnExit(testDir);
+
+    return new Path(new Path(testDir, testName), subdir);
+  }
+
+  private IntegrationTestingUtility getTestingUtil() {
+    if (this.util == null) {
+      if (getConf() == null) {
+        this.util = new IntegrationTestingUtility();
+      } else {
+        this.util = new IntegrationTestingUtility(getConf());
+      }
+    }
+    return util;
+  }
+
+  private int printUsage() {
+    System.err.println("Usage: " + this.getClass().getSimpleName() + " COMMAND [COMMAND options]");
+    System.err.println("  where COMMAND is one of:");
+    System.err.println("");
+    System.err.println("  Generator                  A map only job that generates data.");
+    System.err.println("  Verify                     A map reduce job that looks for holes");
+    System.err.println("                             Look at the counts after running");
+    System.err.println("                             REFERENCED and UNREFERENCED are ok");
+    System.err.println("                             any UNDEFINED counts are bad. Do not");
+    System.err.println("                             run at the same time as the Generator.");
+    System.err.println("  Walker                     A standalong program that starts ");
+    System.err.println("                             following a linked list and emits");
+    System.err.println("                             timing info.");
+    System.err.println("  Print                      A standalone program that prints nodes");
+    System.err.println("                             in the linked list.");
+    System.err.println("  Delete                     A standalone program that deletes a·");
+    System.err.println("                             single node.");
+    System.err.println("  Loop                       A program to Loop through Generator and");
+    System.err.println("                             Verify steps");
+    System.err.println("\t  ");
+    return 1;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    //get the class, run with the conf
+    if (args.length < 1) {
+      return printUsage();
+    }
+    Tool tool = null;
+    if (args[0].equals("Generator")) {
+      tool = new Generator();
+    } else if (args[0].equals("Verify")) {
+      tool = new Verify();
+    } else if (args[0].equals("Loop")) {
+      tool = new Loop();
+    } else if (args[0].equals("Walker")) {
+      tool = new Walker();
+    } else if (args[0].equals("Print")) {
+      tool = new Print();
+    } else if (args[0].equals("Delete")) {
+      tool = new Delete();
+    } else {
+      return printUsage();
+    }
+
+    args = Arrays.copyOfRange(args, 1, args.length);
+    return ToolRunner.run(getConf(), tool, args);
+  }
+
+  public static void main(String[] args) throws Exception {
+    int ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
+    System.exit(ret);
+  }
+}

From 7465b3d0766140171f4890d4c262b6590b8abc33 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 14 Mar 2013 02:26:17 +0000
Subject: [PATCH 0842/1540] HBASE-8085 Backport the fix for
 Bytes.toStringBinary() into 94 (HBASE-6991) (Tianying Chang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456306 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/Bytes.java   | 23 +++++++---------
 .../hadoop/hbase/io/hfile/TestHFileBlock.java |  2 +-
 .../io/hfile/TestHFileBlockCompatibility.java |  2 +-
 .../apache/hadoop/hbase/util/TestBytes.java   | 26 +++++++++++++++++++
 4 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index e10619c89c58..74b390a6b94f 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -339,21 +339,16 @@ public static String toStringBinary(ByteBuffer buf) {
    */
   public static String toStringBinary(final byte [] b, int off, int len) {
     StringBuilder result = new StringBuilder();
-    try {
-      String first = new String(b, off, len, "ISO-8859-1");
-      for (int i = 0; i < first.length() ; ++i ) {
-        int ch = first.charAt(i) & 0xFF;
-        if ( (ch >= '0' && ch <= '9')
-            || (ch >= 'A' && ch <= 'Z')
-            || (ch >= 'a' && ch <= 'z')
-            || " `~!@#$%^&*()-_=+[]{}\\|;:'\",.<>/?".indexOf(ch) >= 0 ) {
-          result.append(first.charAt(i));
-        } else {
-          result.append(String.format("\\x%02X", ch));
-        }
+    for (int i = off; i < off + len ; ++i ) { 
+      int ch = b[i] & 0xFF;
+      if ( (ch >= '0' && ch <= '9')      
+          || (ch >= 'A' && ch <= 'Z')
+          || (ch >= 'a' && ch <= 'z')
+          || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
+          result.append((char)ch);
+      } else {
+        result.append(String.format("\\x%02X", ch));
       }
-    } catch (UnsupportedEncodingException e) {
-      LOG.error("ISO-8859-1 not supported?", e);
     }
     return result.toString();
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index 489a1d2b55de..02a241baf977 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -247,7 +247,7 @@ public void testGzipCompression() throws IOException {
             + "\\x03"
             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
-            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00"
+            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"
             + "\\xAB\\x85g\\x91"; //  4 byte checksum
     final int correctGzipBlockLength = 95;
     assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
index d7b44f4cd365..98ea1805be0e 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
@@ -157,7 +157,7 @@ public void testGzipCompression() throws IOException {
             + "\\x03"
             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
-            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
+            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00";
     final int correctGzipBlockLength = 82;
     assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
         correctGzipBlockLength));
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
index 87f60f5ac5a6..70e707a4fd92 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
@@ -27,6 +27,7 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.Arrays;
+import java.util.Random;
 
 import junit.framework.TestCase;
 import org.apache.hadoop.hbase.SmallTests;
@@ -221,6 +222,31 @@ public void testBinarySearch() throws Exception {
     }
   }
 
+  public void testToStringBytesBinaryReversible() {  
+    //  let's run test with 1000 randomly generated byte arrays
+    Random rand = new Random(System.currentTimeMillis());
+    byte[] randomBytes = new byte[1000];
+    for (int i = 0; i < 1000; i++) {
+      rand.nextBytes(randomBytes);
+      verifyReversibleForBytes(randomBytes); 
+    }
+    
+        
+    //  some specific cases
+    verifyReversibleForBytes(new  byte[] {});
+    verifyReversibleForBytes(new  byte[] {'\\', 'x', 'A', 'D'});
+    verifyReversibleForBytes(new  byte[] {'\\', 'x', 'A', 'D', '\\'});
+  }
+
+  private void verifyReversibleForBytes(byte[] originalBytes) {  
+    String convertedString = Bytes.toStringBinary(originalBytes);
+    byte[] convertedBytes = Bytes.toBytesBinary(convertedString);
+    if (Bytes.compareTo(originalBytes, convertedBytes) != 0) {
+      fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) +
+          ",\nStringBinary: " + convertedString);
+    }
+  }
+
   public void testStartsWith() {
     assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h")));
     assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("")));

From 6ab7671f0833e25b3c49729a95e623e5972311e6 Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Thu, 14 Mar 2013 03:08:25 +0000
Subject: [PATCH 0843/1540] HBASE-8040 - Race condition in AM after HBASE-7521
 (only 0.94) (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456310 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/AssignmentManager.java   | 6 ++++++
 .../hbase/regionserver/handler/OpenRegionHandler.java       | 1 +
 2 files changed, 7 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index ef762f98d88b..3c9973e55103 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1693,10 +1693,16 @@ private void assign(final HRegionInfo region, final RegionState state,
       try {
         LOG.debug("Assigning region " + state.getRegion().getRegionNameAsString() +
           " to " + plan.getDestination().toString());
+        long currentOfflineTimeStamp = state.getStamp();
         RegionOpeningState regionOpenState = serverManager.sendRegionOpen(plan.getDestination(),
             state.getRegion(), versionOfOfflineNode);
         if (regionOpenState == RegionOpeningState.OPENED) {
           // Transition RegionState to PENDING_OPEN
+          // Check if already the offline state has been updated due to a
+          // failure in prev assign
+          if (state.isOffline() && currentOfflineTimeStamp != state.getStamp()) {
+            return;
+          }
           if (state.isOffline() && !state.isOpening()) {
             state.update(RegionState.State.PENDING_OPEN,
                 System.currentTimeMillis(), plan.getDestination());
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 241ad0ba154e..203e03134118 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -100,6 +100,7 @@ public void process() throws IOException {
       region = openRegion();
       if (region == null) {
         tryTransitionToFailedOpen(regionInfo);
+        transitionToFailedOpen = true;
         return;
       }
       boolean failed = true;

From bfec0df88b625f543a011f7eef6db037d30535e8 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 14 Mar 2013 03:38:23 +0000
Subject: [PATCH 0844/1540] HBASE-8088 Versioning site: part one, put stake in
 the ground for 0.94 by copying current versions of book and site; SVN ADD NEW
 FILES

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456317 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/case_studies.xml       | 324 ++++++++++++++++
 src/docbkx/community.xml          | 109 ++++++
 src/docbkx/security.xml           | 532 ++++++++++++++++++++++++++
 src/docbkx/zookeeper.xml          | 595 ++++++++++++++++++++++++++++++
 src/site/resources/doap_Hbase.rdf |  57 +++
 src/site/xdoc/resources.xml       |  37 ++
 6 files changed, 1654 insertions(+)
 create mode 100644 src/docbkx/case_studies.xml
 create mode 100644 src/docbkx/community.xml
 create mode 100644 src/docbkx/security.xml
 create mode 100644 src/docbkx/zookeeper.xml
 create mode 100644 src/site/resources/doap_Hbase.rdf
 create mode 100644 src/site/xdoc/resources.xml

diff --git a/src/docbkx/case_studies.xml b/src/docbkx/case_studies.xml
new file mode 100644
index 000000000000..2e3bba0432f8
--- /dev/null
+++ b/src/docbkx/case_studies.xml
@@ -0,0 +1,324 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<chapter version="5.0" xml:id="casestudies"
+         xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xmlns:svg="http://www.w3.org/2000/svg"
+         xmlns:m="http://www.w3.org/1998/Math/MathML"
+         xmlns:html="http://www.w3.org/1999/xhtml"
+         xmlns:db="http://docbook.org/ns/docbook">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+  <title>Apache HBase (TM) Case Studies</title>
+    <section xml:id="casestudies.overview">
+      <title>Overview</title>
+      <para>This chapter will describe a variety of performance and troubleshooting case studies that can 
+      provide a useful blueprint on diagnosing Apache HBase (TM) cluster issues.</para>
+      <para>For more information on Performance and Troubleshooting, see <xref linkend="performance"/> and <xref linkend="trouble"/>.
+      </para>
+    </section>
+
+    <section xml:id="casestudies.schema">
+    	<title>Schema Design</title>
+
+    	<section xml:id="casestudies.schema.listdata">
+    		<title>List Data</title>
+    		<para>The following is an exchange from the user dist-list regarding a fairly common question:  
+    		how to handle per-user list data in Apache HBase. 
+    		</para>
+    		<para>*** QUESTION ***</para>
+    		<para>
+    		We're looking at how to store a large amount of (per-user) list data in
+HBase, and we were trying to figure out what kind of access pattern made
+the most sense.  One option is store the majority of the data in a key, so
+we could have something like:
+    		</para>
+
+    		<programlisting>
+&lt;FixedWidthUserName&gt;&lt;FixedWidthValueId1&gt;:"" (no value)
+&lt;FixedWidthUserName&gt;&lt;FixedWidthValueId2&gt;:"" (no value)
+&lt;FixedWidthUserName&gt;&lt;FixedWidthValueId3&gt;:"" (no value)
+			</programlisting>
+
+The other option we had was to do this entirely using:
+    		<programlisting>
+&lt;FixedWidthUserName&gt;&lt;FixedWidthPageNum0&gt;:&lt;FixedWidthLength&gt;&lt;FixedIdNextPageNum&gt;&lt;ValueId1&gt;&lt;ValueId2&gt;&lt;ValueId3&gt;...
+&lt;FixedWidthUserName&gt;&lt;FixedWidthPageNum1&gt;:&lt;FixedWidthLength&gt;&lt;FixedIdNextPageNum&gt;&lt;ValueId1&gt;&lt;ValueId2&gt;&lt;ValueId3&gt;...
+    		</programlisting>
+			<para>
+where each row would contain multiple values.
+So in one case reading the first thirty values would be:
+			</para>
+    		<programlisting>
+scan { STARTROW =&gt; 'FixedWidthUsername' LIMIT =&gt; 30}
+    		</programlisting>
+And in the second case it would be
+    		<programlisting>
+get 'FixedWidthUserName\x00\x00\x00\x00'
+    		</programlisting>
+			<para>
+The general usage pattern would be to read only the first 30 values of
+these lists, with infrequent access reading deeper into the lists.  Some
+users would have &lt;= 30 total values in these lists, and some users would
+have millions (i.e. power-law distribution)
+			</para>			
+			<para>
+ The single-value format seems like it would take up more space on HBase,
+but would offer some improved retrieval / pagination flexibility.  Would
+there be any significant performance advantages to be able to paginate via
+gets vs paginating with scans?
+			</para>
+			<para>
+  My initial understanding was that doing a scan should be faster if our
+paging size is unknown (and caching is set appropriately), but that gets
+should be faster if we'll always need the same page size.  I've ended up
+hearing different people tell me opposite things about performance.  I
+assume the page sizes would be relatively consistent, so for most use cases
+we could guarantee that we only wanted one page of data in the
+fixed-page-length case.  I would also assume that we would have infrequent
+updates, but may have inserts into the middle of these lists (meaning we'd
+need to update all subsequent rows).
+			</para>
+			<para>
+Thanks for help / suggestions / follow-up questions.
+			</para>
+			<para>*** ANSWER ***</para>
+			<para>
+If I understand you correctly, you're ultimately trying to store
+triples in the form "user, valueid, value", right? E.g., something
+like:
+			</para>
+			<programlisting>
+"user123, firstname, Paul",
+"user234, lastname, Smith"
+			</programlisting>
+			<para>
+(But the usernames are fixed width, and the valueids are fixed width).
+			</para>
+			<para>
+And, your access pattern is along the lines of: "for user X, list the
+next 30 values, starting with valueid Y". Is that right? And these
+values should be returned sorted by valueid?
+			</para>
+			<para>
+The tl;dr version is that you should probably go with one row per
+user+value, and not build a complicated intra-row pagination scheme on
+your own unless you're really sure it is needed.
+			</para>
+			<para>
+Your two options mirror a common question people have when designing
+HBase schemas: should I go "tall" or "wide"? Your first schema is
+"tall": each row represents one value for one user, and so there are
+many rows in the table for each user; the row key is user + valueid,
+and there would be (presumably) a single column qualifier that means
+"the value". This is great if you want to scan over rows in sorted
+order by row key (thus my question above, about whether these ids are
+sorted correctly). You can start a scan at any user+valueid, read the
+next 30, and be done. What you're giving up is the ability to have
+transactional guarantees around all the rows for one user, but it
+doesn't sound like you need that. Doing it this way is generally
+recommended (see
+here <link xlink:href="http://hbase.apache.org/book.html#schema.smackdown">http://hbase.apache.org/book.html#schema.smackdown</link>).
+			</para>
+			<para>
+Your second option is "wide": you store a bunch of values in one row,
+using different qualifiers (where the qualifier is the valueid). The
+simple way to do that would be to just store ALL values for one user
+in a single row. I'm guessing you jumped to the "paginated" version
+because you're assuming that storing millions of columns in a single
+row would be bad for performance, which may or may not be true; as
+long as you're not trying to do too much in a single request, or do
+things like scanning over and returning all of the cells in the row,
+it shouldn't be fundamentally worse. The client has methods that allow
+you to get specific slices of columns.
+			</para>
+			<para>
+Note that neither case fundamentally uses more disk space than the
+other; you're just "shifting" part of the identifying information for
+a value either to the left (into the row key, in option one) or to the
+right (into the column qualifiers in option 2). Under the covers,
+every key/value still stores the whole row key, and column family
+name. (If this is a bit confusing, take an hour and watch Lars
+George's excellent video about understanding HBase schema design:
+<link xlink:href="http://www.youtube.com/watch?v=_HLoH_PgrLk)">http://www.youtube.com/watch?v=_HLoH_PgrLk)</link>.
+			</para>
+			<para>
+A manually paginated version has lots more complexities, as you note,
+like having to keep track of how many things are in each page,
+re-shuffling if new values are inserted, etc. That seems significantly
+more complex. It might have some slight speed advantages (or
+disadvantages!) at extremely high throughput, and the only way to
+really know that would be to try it out. If you don't have time to
+build it both ways and compare, my advice would be to start with the
+simplest option (one row per user+value). Start simple and iterate! :)
+			</para>
+    		
+		</section>  <!--  listdata -->
+
+		
+	</section>   <!--  schema design -->
+
+    <section xml:id="casestudies.perftroub">
+    	<title>Performance/Troubleshooting</title>
+   
+    <section xml:id="casestudies.slownode">
+      <title>Case Study #1 (Performance Issue On A Single Node)</title>
+      <section><title>Scenario</title>
+        <para>Following a scheduled reboot, one data node began exhibiting unusual behavior.  Routine MapReduce 
+         jobs run against HBase tables which regularly completed in five or six minutes began taking 30 or 40 minutes 
+         to finish. These jobs were consistently found to be waiting on map and reduce tasks assigned to the troubled data node 
+         (e.g., the slow map tasks all had the same Input Split).           
+         The situation came to a head during a distributed copy, when the copy was severely prolonged by the lagging node.
+		</para>
+       </section>
+      <section><title>Hardware</title>
+        <para>Datanodes:
+        <itemizedlist>
+          <listitem>Two 12-core processors</listitem>
+          <listitem>Six Enerprise SATA disks</listitem>
+          <listitem>24GB of RAM</listitem>
+          <listitem>Two bonded gigabit NICs</listitem>
+        </itemizedlist>
+        </para>		
+        <para>Network:
+        <itemizedlist>
+          <listitem>10 Gigabit top-of-rack switches</listitem>
+          <listitem>20 Gigabit bonded interconnects between racks.</listitem>
+        </itemizedlist>
+        </para>
+      </section>
+      <section><title>Hypotheses</title>
+		<section><title>HBase "Hot Spot" Region</title>
+		  <para>We hypothesized that we were experiencing a familiar point of pain: a "hot spot" region in an HBase table, 
+		  where uneven key-space distribution can funnel a huge number of requests to a single HBase region, bombarding the RegionServer 
+		  process and cause slow response time. Examination of the HBase Master status page showed that the number of HBase requests to the 
+		  troubled node was almost zero.  Further, examination of the HBase logs showed that there were no region splits, compactions, or other region transitions 
+		  in progress.  This effectively ruled out a "hot spot" as the root cause of the observed slowness.
+          </para>		
+        </section>
+		<section><title>HBase Region With Non-Local Data</title>
+		  <para>Our next hypothesis was that one of the MapReduce tasks was requesting data from HBase that was not local to the datanode, thus 
+		  forcing HDFS to request data blocks from other servers over the network.  Examination of the datanode logs showed that there were very 
+		  few blocks being requested over the network, indicating that the HBase region was correctly assigned, and that the majority of the necessary 
+		  data was located on the node. This ruled out the possibility of non-local data causing a slowdown.
+          </para>
+        </section>		
+		<section><title>Excessive I/O Wait Due To Swapping Or An Over-Worked Or Failing Hard Disk</title>
+          <para>After concluding that the Hadoop and HBase were not likely to be the culprits, we moved on to troubleshooting the datanode's hardware. 
+          Java, by design, will periodically scan its entire memory space to do garbage collection.  If system memory is heavily overcommitted, the Linux 
+          kernel may enter a vicious cycle, using up all of its resources swapping Java heap back and forth from disk to RAM as Java tries to run garbage 
+          collection.  Further, a failing hard disk will often retry reads and/or writes many times before giving up and returning an error. This can manifest 
+          as high iowait, as running processes wait for reads and writes to complete.  Finally, a disk nearing the upper edge of its performance envelope will 
+          begin to cause iowait as it informs the kernel that it cannot accept any more data, and the kernel queues incoming data into the dirty write pool in memory.  
+          However, using <code>vmstat(1)</code> and <code>free(1)</code>, we could see that no swap was being used, and the amount of disk IO was only a few kilobytes per second.
+          </para>		
+        </section>
+		<section><title>Slowness Due To High Processor Usage</title>
+          <para>Next, we checked to see whether the system was performing slowly simply due to very high computational load.  <code>top(1)</code> showed that the system load 
+          was higher than normal, but <code>vmstat(1)</code> and <code>mpstat(1)</code> showed that the amount of processor being used for actual computation was low.
+          </para>	
+        </section>	
+		<section><title>Network Saturation (The Winner)</title>
+          <para>Since neither the disks nor the processors were being utilized heavily, we moved on to the performance of the network interfaces.  The datanode had two 
+          gigabit ethernet adapters, bonded to form an active-standby interface.  <code>ifconfig(8)</code> showed some unusual anomalies, namely interface errors, overruns, framing errors. 
+          While not unheard of, these kinds of errors are exceedingly rare on modern hardware which is operating as it should:
+<programlisting>		
+$ /sbin/ifconfig bond0
+bond0  Link encap:Ethernet  HWaddr 00:00:00:00:00:00  
+inet addr:10.x.x.x  Bcast:10.x.x.255  Mask:255.255.255.0
+UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
+RX packets:2990700159 errors:12 dropped:0 overruns:1 frame:6          &lt;--- Look Here! Errors!
+TX packets:3443518196 errors:0 dropped:0 overruns:0 carrier:0
+collisions:0 txqueuelen:0 
+RX bytes:2416328868676 (2.4 TB)  TX bytes:3464991094001 (3.4 TB)
+</programlisting>
+          </para>		
+          <para>These errors immediately lead us to suspect that one or more of the ethernet interfaces might have negotiated the wrong line speed.  This was confirmed both by running an ICMP ping 
+          from an external host and observing round-trip-time in excess of 700ms, and by running <code>ethtool(8)</code> on the members of the bond interface and discovering that the active interface 
+          was operating at 100Mbs/, full duplex.
+<programlisting>		
+$ sudo ethtool eth0
+Settings for eth0:
+Supported ports: [ TP ]
+Supported link modes:   10baseT/Half 10baseT/Full 
+                       100baseT/Half 100baseT/Full 
+                       1000baseT/Full 
+Supports auto-negotiation: Yes
+Advertised link modes:  10baseT/Half 10baseT/Full 
+                       100baseT/Half 100baseT/Full 
+                       1000baseT/Full 
+Advertised pause frame use: No
+Advertised auto-negotiation: Yes
+Link partner advertised link modes:  Not reported
+Link partner advertised pause frame use: No
+Link partner advertised auto-negotiation: No
+Speed: 100Mb/s                                     &lt;--- Look Here!  Should say 1000Mb/s!
+Duplex: Full
+Port: Twisted Pair
+PHYAD: 1
+Transceiver: internal
+Auto-negotiation: on
+MDI-X: Unknown
+Supports Wake-on: umbg
+Wake-on: g
+Current message level: 0x00000003 (3)
+Link detected: yes
+</programlisting>		
+		  </para>
+		  <para>In normal operation, the ICMP ping round trip time should be around 20ms, and the interface speed and duplex should read, "1000MB/s", and, "Full", respectively.  
+		  </para>
+	    </section>
+     </section>  
+   	<section><title>Resolution</title>
+   	  <para>After determining that the active ethernet adapter was at the incorrect speed, we used the <code>ifenslave(8)</code> command to make the standby interface 
+   	  the active interface, which yielded an immediate improvement in MapReduce performance, and a 10 times improvement in network throughput:
+	  </para>
+	  <para>On the next trip to the datacenter, we determined that the line speed issue was ultimately caused by a bad network cable, which was replaced.
+	  </para>
+	</section>
+   </section>  <!--  case study -->
+    <section xml:id="casestudies.perf.1">
+      <title>Case Study #2 (Performance Research 2012)</title>
+      <para>Investigation results of a self-described "we're not sure what's wrong, but it seems slow" problem. 
+      <link xlink:href="http://gbif.blogspot.com/2012/03/hbase-performance-evaluation-continued.html">http://gbif.blogspot.com/2012/03/hbase-performance-evaluation-continued.html</link>
+      </para>
+    </section>
+
+    <section xml:id="casestudies.perf.2">
+      <title>Case Study #3 (Performance Research 2010))</title>
+      <para>
+      Investigation results of general cluster performance from 2010.  Although this research is on an older version of the codebase, this writeup
+      is still very useful in terms of approach.
+      <link xlink:href="http://hstack.org/hbase-performance-testing/">http://hstack.org/hbase-performance-testing/</link>
+      </para>
+    </section>
+
+    <section xml:id="casestudies.xceivers">
+      <title>Case Study #4 (xcievers Config)</title>
+      <para>Case study of configuring <code>xceivers</code>, and diagnosing errors from mis-configurations.
+      <link xlink:href="http://www.larsgeorge.com/2012/03/hadoop-hbase-and-xceivers.html">http://www.larsgeorge.com/2012/03/hadoop-hbase-and-xceivers.html</link>
+      </para>
+      <para>See also <xref linkend="dfs.datanode.max.xcievers"/>.
+      </para>
+    </section>
+
+    </section>    <!--  performance/troubleshooting -->
+	
+  </chapter>
diff --git a/src/docbkx/community.xml b/src/docbkx/community.xml
new file mode 100644
index 000000000000..2c09908aed98
--- /dev/null
+++ b/src/docbkx/community.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0"?>
+    <chapter xml:id="community"
+      version="5.0" xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xmlns:svg="http://www.w3.org/2000/svg"
+      xmlns:m="http://www.w3.org/1998/Math/MathML"
+      xmlns:html="http://www.w3.org/1999/xhtml"
+      xmlns:db="http://docbook.org/ns/docbook">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+    <title>Community</title>
+    <section xml:id="decisions">
+      <title>Decisions</title>
+      <section xml:id="feature_branches">
+        <title>Feature Branches</title>
+        <para>Feature Branches are easy to make.  You do not have to be a committer to make one.  Just request the name of your branch be added to JIRA up on the
+        developer's mailing list and a committer will add it for you.  Thereafter you can file issues against your feature branch in Apache HBase (TM) JIRA.  Your code you
+        keep elsewhere -- it should be public so it can be observed -- and you can update dev mailing list on progress.   When the feature is ready for commit,
+        3 +1s from committers will get your feature merged<footnote><para>See <link xlink:href="http://search-hadoop.com/m/asM982C5FkS1">HBase, mail # dev - Thoughts about large feature dev branches</link></para></footnote>
+        </para>
+      </section>         
+      <section xml:id="patchplusonepolicy">
+        <title>Patch +1 Policy</title>
+        <para>
+The below policy is something we put in place 09/2012.  It is a
+suggested policy rather than a hard requirement.  We want to try it
+first to see if it works before we cast it in stone.
+        </para>
+<para>
+Apache HBase is made of
+<link xlink:href="https://issues.apache.org/jira/browse/HBASE#selectedTab=com.atlassian.jira.plugin.system.project%3Acomponents-panel">components</link>.
+Components have one or more <xref linkend="OWNER" />s.  See the 'Description' field on the 
+<link xlink:href="https://issues.apache.org/jira/browse/HBASE#selectedTab=com.atlassian.jira.plugin.system.project%3Acomponents-panel">components</link>
+JIRA page for who the current owners are by component.
+</para>
+<para>
+Patches that fit within the scope of a single Apache HBase component require,
+at least, a +1 by one of the component's owners before commit. If
+owners are absent -- busy or otherwise -- two +1s by non-owners will
+suffice.
+</para>
+<para>
+Patches that span components need at least two +1s before they can be
+committed, preferably +1s by owners of components touched by the
+x-component patch (TODO: This needs tightening up but I think fine for
+first pass).
+</para>
+<para>
+Any -1 on a patch by anyone vetos a patch; it cannot be committed
+until the justification for the -1 is addressed.
+</para>
+      </section>         
+    </section>    
+    <section xml:id="community.roles">
+      <title>Community Roles</title>
+      <section xml:id="OWNER">
+        <title>Component Owner</title>
+        <para>
+Component owners are listed in the description field on this Apache HBase JIRA <link xlink:href="https://issues.apache.org/jira/browse/HBASE#selectedTab=com.atlassian.jira.plugin.system.project%3Acomponents-panel">components</link>
+page.  The owners are listed in the 'Description' field rather than in the 'Component
+Lead' field because the latter only allows us list one individual
+whereas it is encouraged that components have multiple owners.
+        </para>
+<para>
+Owners are volunteers who are (usually, but not necessarily) expert in
+their component domain and may have an agenda on how they think their
+Apache HBase component should evolve.
+</para>
+<para>
+Duties include:
+<orderedlist>
+<listitem>
+<para>
+Owners will try and review patches that land within their component's scope.
+</para>
+</listitem>
+<listitem>
+<para>
+If applicable, if an owner has an agenda, they will publish their
+goals or the design toward which they are driving their component
+</para>
+</listitem>
+</orderedlist>
+</para>
+<para>
+If you would like to be volunteer as a component owner, just write the
+dev list and we'll sign you up. Owners do not need to be committers.
+</para>
+      </section>         
+    </section>    
+    </chapter>
diff --git a/src/docbkx/security.xml b/src/docbkx/security.xml
new file mode 100644
index 000000000000..ed4a0c2ed638
--- /dev/null
+++ b/src/docbkx/security.xml
@@ -0,0 +1,532 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<chapter version="5.0" xml:id="security"
+         xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xmlns:svg="http://www.w3.org/2000/svg"
+         xmlns:m="http://www.w3.org/1998/Math/MathML"
+         xmlns:html="http://www.w3.org/1999/xhtml"
+         xmlns:db="http://docbook.org/ns/docbook">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<title>Secure Apache HBase (TM)</title>
+<section xml:id="hbase.secure.configuration">
+    <title>Secure Client Access to Apache HBase</title>
+    <para>Newer releases of Apache HBase (TM) (&gt;= 0.92) support optional SASL authentication of clients<footnote><para>See
+    also Matteo Bertozzi's article on <link xlink:href="http://www.cloudera.com/blog/2012/09/understanding-user-authentication-and-authorization-in-apache-hbase/">Understanding User Authentication and Authorization in Apache HBase</link>.</para></footnote>.</para>
+    <para>This describes how to set up Apache HBase and clients for connection to secure HBase resources.</para>
+
+    <section><title>Prerequisites</title>
+    <para>
+        You need to have a working Kerberos KDC.
+    </para>
+    <para>
+        A HBase configured for secure client access is expected to be running
+        on top of a secured HDFS cluster. HBase must be able to authenticate
+        to HDFS services. HBase needs Kerberos credentials to interact with
+        the Kerberos-enabled HDFS daemons. Authenticating a service should be
+        done using a keytab file. The procedure for creating keytabs for HBase
+        service is the same as for creating keytabs for Hadoop. Those steps
+        are omitted here. Copy the resulting keytab files to wherever HBase
+        Master and RegionServer processes are deployed and make them readable
+        only to the user account under which the HBase daemons will run.
+    </para>
+    <para>
+        A Kerberos principal has three parts, with the form 
+        <code>username/fully.qualified.domain.name@YOUR-REALM.COM</code>. We
+        recommend using <code>hbase</code> as the username portion.
+    </para>
+    <para>
+        The following is an example of the configuration properties for
+        Kerberos operation that must be added to the 
+        <code>hbase-site.xml</code> file on every server machine in the
+        cluster. Required for even the most basic interactions with a
+        secure Hadoop configuration, independent of HBase security.
+    </para>
+    <programlisting><![CDATA[
+      <property>
+        <name>hbase.regionserver.kerberos.principal</name>
+        <value>hbase/_HOST@YOUR-REALM.COM</value>
+      </property>
+      <property>
+        <name>hbase.regionserver.keytab.file</name>
+        <value>/etc/hbase/conf/keytab.krb5</value>
+      </property>
+      <property>
+        <name>hbase.master.kerberos.principal</name>
+        <value>hbase/_HOST@YOUR-REALM.COM</value>
+      </property>
+      <property>
+        <name>hbase.master.keytab.file</name>
+        <value>/etc/hbase/conf/keytab.krb5</value>
+      </property>
+    ]]></programlisting>
+    <para>
+        Each HBase client user should also be given a Kerberos principal. This
+        principal should have a password assigned to it (as opposed to a
+        keytab file). The client principal's <code>maxrenewlife</code> should
+        be set so that it can be renewed enough times for the HBase client
+        process to complete. For example, if a user runs a long-running HBase
+        client process that takes at most 3 days, we might create this user's
+        principal within <code>kadmin</code> with: <code>addprinc -maxrenewlife
+        3days</code>
+    </para>
+    <para>
+        Long running daemons with indefinite lifetimes that require client
+        access to HBase can instead be configured to log in from a keytab. For
+        each host running such daemons, create a keytab with 
+        <code>kadmin</code> or <code>kadmin.local</code>. The procedure for
+        creating keytabs for HBase service is the same as for creating
+        keytabs for Hadoop. Those steps are omitted here. Copy the resulting
+        keytab files to where the client daemon will execute and make them
+        readable only to the user account under which the daemon will run.
+    </para>
+    </section>
+
+    <section><title>Server-side Configuration for Secure Operation</title>
+    <para>
+        Add the following to the <code>hbase-site.xml</code> file on every server machine in the cluster:
+    </para>
+    <programlisting><![CDATA[
+      <property>
+        <name>hbase.security.authentication</name>
+        <value>kerberos</value> 
+      </property> 
+      <property>
+        <name>hbase.security.authorization</name>
+        <value>true</value>
+      </property>
+      <property>
+      <name>hbase.coprocessor.region.classes</name>
+        <value>org.apache.hadoop.hbase.security.token.TokenProvider</value>
+      </property>
+    ]]></programlisting>
+    <para>
+       A full shutdown and restart of HBase service is required when deploying
+       these configuration changes.
+    </para>
+    </section>
+
+    <section><title>Client-side Configuration for Secure Operation</title>
+    <para>
+        Add the following to the <code>hbase-site.xml</code> file on every client:
+    </para>
+    <programlisting><![CDATA[
+      <property>
+        <name>hbase.security.authentication</name>
+        <value>kerberos</value>
+      </property> 
+    ]]></programlisting>
+    <para>
+        The client environment must be logged in to Kerberos from KDC or
+        keytab via the <code>kinit</code> command before communication with
+        the HBase cluster will be possible.
+    </para>
+    <para>
+        Be advised that if the <code>hbase.security.authentication</code>
+        in the client- and server-side site files do not match, the client will 
+        not be able to communicate with the cluster.
+    </para>
+    <para>
+        Once HBase is configured for secure RPC it is possible to optionally
+        configure encrypted communication. To do so, add the following to the
+        <code>hbase-site.xml</code> file on every client:
+    </para>
+    <programlisting><![CDATA[
+      <property>
+        <name>hbase.rpc.protection</name>
+        <value>privacy</value>
+      </property>
+    ]]></programlisting>
+    <para>
+        This configuration property can also be set on a per connection basis.
+        Set it in the <code>Configuration</code> supplied to
+        <code>HTable</code>:
+    </para>
+    <programlisting>
+      Configuration conf = HBaseConfiguration.create();
+      conf.set("hbase.rpc.protection", "privacy");
+      HTable table = new HTable(conf, tablename);
+    </programlisting>
+    <para>
+        Expect a ~10% performance penalty for encrypted communication.
+    </para>
+    </section>
+
+    <section><title>Client-side Configuration for Secure Operation - Thrift Gateway</title>
+    <para>
+        Add the following to the <code>hbase-site.xml</code> file for every Thrift gateway:
+    <programlisting><![CDATA[
+    <property>
+      <name>hbase.thrift.keytab.file</name>
+      <value>/etc/hbase/conf/hbase.keytab</value>
+    </property>
+    <property>
+      <name>hbase.thrift.kerberos.principal</name>
+      <value>$USER/_HOST@HADOOP.LOCALDOMAIN</value>
+    </property>
+    ]]></programlisting>
+    </para>
+    <para>
+        Substitute the appropriate credential and keytab for $USER and $KEYTAB
+        respectively.
+    </para>
+    <para>
+        The Thrift gateway will authenticate with HBase using the supplied
+        credential. No authentication will be performed by the Thrift gateway
+        itself. All client access via the Thrift gateway will use the Thrift
+        gateway's credential and have its privilege.
+    </para>
+    </section>
+
+    <section><title>Client-side Configuration for Secure Operation - REST Gateway</title>
+    <para>
+        Add the following to the <code>hbase-site.xml</code> file for every REST gateway:
+    <programlisting><![CDATA[
+    <property>
+      <name>hbase.rest.keytab.file</name>
+      <value>$KEYTAB</value>
+    </property>
+    <property>
+      <name>hbase.rest.kerberos.principal</name>
+      <value>$USER/_HOST@HADOOP.LOCALDOMAIN</value>
+    </property>
+    ]]></programlisting>
+    </para>
+    <para>
+        Substitute the appropriate credential and keytab for $USER and $KEYTAB
+        respectively.
+    </para>
+    <para>
+        The REST gateway will authenticate with HBase using the supplied
+        credential. No authentication will be performed by the REST gateway
+        itself. All client access via the REST gateway will use the REST
+        gateway's credential and have its privilege.
+    </para>
+    <para>
+        It should be possible for clients to authenticate with the HBase
+        cluster through the REST gateway in a pass-through manner via SPEGNO
+        HTTP authentication. This is future work.
+    </para>
+   </section>
+
+</section>  <!-- Secure Client Access to HBase -->
+
+
+<section xml:id="hbase.accesscontrol.configuration">
+    <title>Access Control</title>
+    <para>
+        Newer releases of Apache HBase (&gt;= 0.92) support optional access control
+        list (ACL-) based protection of resources on a column family and/or
+        table basis.
+    </para>
+    <para>
+        This describes how to set up Secure HBase for access control, with an
+        example of granting and revoking user permission on table resources
+        provided.
+    </para>
+
+    <section><title>Prerequisites</title>
+    <para>
+       You must configure HBase for secure operation. Refer to the section
+       "Secure Client Access to HBase" and complete all of the steps described
+       there.
+    </para>
+    <para>
+       You must also configure ZooKeeper for secure operation. Changes to ACLs
+       are synchronized throughout the cluster using ZooKeeper. Secure 
+       authentication to ZooKeeper must be enabled or otherwise it will be
+       possible to subvert HBase access control via direct client access to
+       ZooKeeper. Refer to the section on secure ZooKeeper configuration and
+       complete all of the steps described there.
+    </para>
+    </section>
+
+    <section><title>Overview</title>
+    <para>
+        With Secure RPC and Access Control enabled, client access to HBase is
+        authenticated and user data is private unless access has been
+        explicitly granted. Access to data can be granted at a table or per
+        column family basis. 
+    </para>
+    <para>
+        However, the following items have been left out of the initial
+        implementation for simplicity:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>Row-level or per value (cell): This would require broader changes for storing the ACLs inline with rows. It is a future goal.</para>
+      </listitem>
+      <listitem>
+        <para>Push down of file ownership to HDFS: HBase is not designed for the case where files may have different permissions than the HBase system principal. Pushing file ownership down into HDFS would necessitate changes to core code. Also, while HDFS file ownership would make applying quotas easy, and possibly make bulk imports more straightforward, it is not clear that it would offer a more secure setup.</para>
+      </listitem>
+      <listitem>
+        <para>HBase managed "roles" as collections of permissions: We will not model "roles" internally in HBase to begin with. We instead allow group names to be granted permissions, which allows external modeling of roles via group membership. Groups are created and manipulated externally to HBase, via the Hadoop group mapping service.</para>
+      </listitem>
+    </orderedlist>
+    <para>
+Access control mechanisms are mature and fairly standardized in the relational database world. The HBase implementation approximates current convention, but HBase has a simpler feature set than relational databases, especially in terms of client operations. We don't distinguish between an insert (new record) and update (of existing record), for example, as both collapse down into a Put. Accordingly, the important operations condense to four permissions: READ, WRITE, CREATE, and ADMIN.
+    </para>
+    <table>
+      <title>Operation To Permission Mapping</title>
+      <thead>
+        <row>
+          <entry>Permission</entry>
+          <entry>Operation</entry>
+        </row>
+      </thead>
+      <tbody>
+        <!-- READ -->
+        <row>
+          <entry>Read</entry>
+          <entry>Get</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Exists</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Scan</entry>
+        </row>
+        <!-- WRITE -->
+        <row>
+          <entry>Write</entry>
+          <entry>Put</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Delete</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Lock/UnlockRow</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>IncrementColumnValue</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>CheckAndDelete/Put</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Flush</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Compact</entry>
+        </row>
+        <!-- CREATE -->
+        <row>
+          <entry>Create</entry>
+          <entry>Create</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Alter</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Drop</entry>
+        </row>
+        <!-- ADMIN -->
+        <row>
+          <entry>Admin</entry>
+          <entry>Enable/Disable</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Split</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Major Compact</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Grant</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Revoke</entry>
+        </row>
+        <row>
+          <entry></entry>
+          <entry>Shutdown</entry>
+        </row>
+      </tbody>
+    </table>
+    <para>
+        Permissions can be granted in any of the following scopes, though
+        CREATE and ADMIN permissions are effective only at table scope.
+    </para>
+    <para>
+    <itemizedlist>
+      <listitem>
+        <para>Table</para>
+        <para>
+        <itemizedlist>
+          <listitem><para>Read: User can read from any column family in table</para></listitem>
+          <listitem><para>Write: User can write to any column family in table</para></listitem>
+          <listitem><para>Create: User can alter table attributes; add, alter, or drop column families; and drop the table.</para></listitem>
+          <listitem><para>Admin: User can alter table attributes; add, alter, or drop column families; and enable, disable, or drop the table. User can also trigger region (re)assignments or relocation.</para></listitem>
+        </itemizedlist>
+        </para>
+      </listitem>
+      <listitem>
+        <para>Column Family</para>
+        <para>
+        <itemizedlist>
+          <listitem><para>Read: User can read from the column family</para></listitem>
+          <listitem><para>Write: User can write to the column family</para></listitem>
+        </itemizedlist>
+        </para>
+      </listitem>
+    </itemizedlist>
+    </para>
+    <para>
+       There is also an implicit global scope for the superuser.
+    </para>
+    <para>
+       The superuser is a principal, specified in the HBase site configuration
+       file, that has equivalent access to HBase as the 'root' user would on a
+       UNIX derived system. Normally this is the principal that the HBase
+       processes themselves authenticate as. Although future versions of HBase
+       Access Control may support multiple superusers, the superuser privilege
+       will always include the principal used to run the HMaster process. Only
+       the superuser is allowed to create tables, switch the balancer on or
+       off, or take other actions with global consequence. Furthermore, the
+       superuser has an implicit grant of all permissions to all resources.
+    </para>
+    <para>
+       Tables have a new metadata attribute: OWNER, the user principal who owns
+       the table. By default this will be set to the user principal who creates
+       the table, though it may be changed at table creation time or during an
+       alter operation by setting or changing the OWNER table attribute. Only a
+       single user principal can own a table at a given time. A table owner will
+       have all permissions over a given table. 
+    </para>
+    </section>
+
+    <section><title>Server-side Configuration for Access Control</title>
+    <para>
+        Enable the AccessController coprocessor in the cluster configuration
+        and restart HBase. The restart can be a rolling one. Complete the
+        restart of all Master and RegionServer processes before setting up
+        ACLs.
+    </para>
+    <para>
+        To enable the AccessController, modify the <code>hbase-site.xml</code> file on every server machine in the cluster to look like:
+    </para>
+    <programlisting><![CDATA[
+      <property>
+        <name>hbase.coprocessor.master.classes</name>
+        <value>org.apache.hadoop.hbase.security.access.AccessController</value>
+      </property>
+      <property>
+      <name>hbase.coprocessor.region.classes</name>
+        <value>org.apache.hadoop.hbase.security.token.TokenProvider,
+        org.apache.hadoop.hbase.security.access.AccessController</value>
+      </property>
+    ]]></programlisting>
+    </section>
+
+    <section><title>Shell Enhancements for Access Control</title>
+    <para>
+The HBase shell has been extended to provide simple commands for editing and updating user permissions. The following commands have been added for access control list management: 
+    </para>
+    Grant
+    <para>
+    <programlisting>
+    grant &lt;user&gt; &lt;permissions&gt; &lt;table&gt; [ &lt;column family&gt; [ &lt;column qualifier&gt; ] ]
+    </programlisting>
+    </para>
+    <para>
+    <code>&lt;permissions&gt;</code> is zero or more letters from the set "RWCA": READ('R'), WRITE('W'), CREATE('C'), ADMIN('A'). 
+    </para>
+    <para>
+    Note: Grants and revocations of individual permissions on a resource are both accomplished using the <code>grant</code> command. A separate <code>revoke</code> command is also provided by the shell, but this is for fast revocation of all of a user's access rights to a given resource only.
+    </para>
+    <para>
+    Revoke
+    </para>
+    <para>
+    <programlisting>
+    revoke &lt;user&gt; &lt;table&gt; [ &lt;column family&gt; [ &lt;column qualifier&gt; ] ]
+    </programlisting>
+    </para>
+    <para>
+    Alter
+    </para>
+    <para>
+    The <code>alter</code> command has been extended to allow ownership assignment:
+    <programlisting>
+      alter 'tablename', {OWNER => 'username'}
+    </programlisting>
+    </para>
+    <para>
+    User Permission
+    </para>
+    <para>
+    The <code>user_permission</code> command shows all access permissions for the current user for a given table:
+    <programlisting>
+      user_permission &lt;table&gt;
+    </programlisting>
+    </para>
+  </section>
+
+</section>  <!-- Access Control -->
+
+<section xml:id="hbase.secure.bulkload">
+    <title>Secure Bulk Load</title>
+    <para>
+	Bulk loading in secure mode is a bit more involved than normal setup, since the client has to transfer the ownership of the files generated from the mapreduce job to HBase. Secure bulk loading is implemented by a coprocessor, named <link xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.html">SecureBulkLoadEndpoint</link>. SecureBulkLoadEndpoint uses a staging directory <code>"hbase.bulkload.staging.dir"</code>, which defaults to <code>/tmp/hbase-staging/</code>. The algorithm is as follows.
+	<itemizedlist>
+      <listitem>Create an hbase owned staging directory which is world traversable (<code>-rwx--x--x, 711</code>) <code>/tmp/hbase-staging</code>. </listitem>
+      <listitem>A user writes out data to his secure output directory: /user/foo/data </listitem>
+      <listitem>A call is made to hbase to create a secret staging directory
+  which is globally readable/writable (<code>-rwxrwxrwx, 777</code>): /tmp/hbase-staging/averylongandrandomdirectoryname</listitem>
+  <listitem>The user makes the data world readable and writable, then moves it
+  into the random staging directory, then calls bulkLoadHFiles()</listitem>
+  </itemizedlist>
+  </para>
+  <para>
+  Like delegation tokens the strength of the security lies in the length
+  and randomness of the secret directory.
+    </para>
+
+	<para>
+        You have to enable the secure bulk load to work properly. You can modify the <code>hbase-site.xml</code> file on every server machine in the cluster and add the SecureBulkLoadEndpoint class to the list of regionserver coprocessors:
+    </para>
+    <programlisting><![CDATA[
+      <property>
+        <name>hbase.bulkload.staging.dir</name>
+        <value>/tmp/hbase-staging</value>
+      </property>
+      <property>
+        <name>hbase.coprocessor.region.classes</name>
+        <value>org.apache.hadoop.hbase.security.token.TokenProvider,
+        org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value>
+      </property>
+    ]]></programlisting>
+</section>
+</chapter>
diff --git a/src/docbkx/zookeeper.xml b/src/docbkx/zookeeper.xml
new file mode 100644
index 000000000000..d6301e26fa1c
--- /dev/null
+++ b/src/docbkx/zookeeper.xml
@@ -0,0 +1,595 @@
+<?xml version="1.0"?>
+    <chapter xml:id="zookeeper"
+      version="5.0" xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xmlns:svg="http://www.w3.org/2000/svg"
+      xmlns:m="http://www.w3.org/1998/Math/MathML"
+      xmlns:html="http://www.w3.org/1999/xhtml"
+      xmlns:db="http://docbook.org/ns/docbook">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+            <title>ZooKeeper<indexterm>
+                <primary>ZooKeeper</primary>
+              </indexterm></title>
+
+            <para>A distributed Apache HBase (TM) installation depends on a running ZooKeeper cluster.
+            All participating nodes and clients need to be able to access the
+            running ZooKeeper ensemble. Apache HBase by default manages a ZooKeeper
+            "cluster" for you. It will start and stop the ZooKeeper ensemble
+            as part of the HBase start/stop process. You can also manage the
+            ZooKeeper ensemble independent of HBase and just point HBase at
+            the cluster it should use. To toggle HBase management of
+            ZooKeeper, use the <varname>HBASE_MANAGES_ZK</varname> variable in
+            <filename>conf/hbase-env.sh</filename>. This variable, which
+            defaults to <varname>true</varname>, tells HBase whether to
+            start/stop the ZooKeeper ensemble servers as part of HBase
+            start/stop.</para>
+
+            <para>When HBase manages the ZooKeeper ensemble, you can specify
+            ZooKeeper configuration using its native
+            <filename>zoo.cfg</filename> file, or, the easier option is to
+            just specify ZooKeeper options directly in
+            <filename>conf/hbase-site.xml</filename>. A ZooKeeper
+            configuration option can be set as a property in the HBase
+            <filename>hbase-site.xml</filename> XML configuration file by
+            prefacing the ZooKeeper option name with
+            <varname>hbase.zookeeper.property</varname>. For example, the
+            <varname>clientPort</varname> setting in ZooKeeper can be changed
+            by setting the
+            <varname>hbase.zookeeper.property.clientPort</varname> property.
+            For all default values used by HBase, including ZooKeeper
+            configuration, see <xref linkend="hbase_default_configurations" />. Look for the
+            <varname>hbase.zookeeper.property</varname> prefix <footnote>
+                <para>For the full list of ZooKeeper configurations, see
+                ZooKeeper's <filename>zoo.cfg</filename>. HBase does not ship
+                with a <filename>zoo.cfg</filename> so you will need to browse
+                the <filename>conf</filename> directory in an appropriate
+                ZooKeeper download.</para>
+              </footnote></para>
+
+            <para>You must at least list the ensemble servers in
+            <filename>hbase-site.xml</filename> using the
+            <varname>hbase.zookeeper.quorum</varname> property. This property
+            defaults to a single ensemble member at
+            <varname>localhost</varname> which is not suitable for a fully
+            distributed HBase. (It binds to the local machine only and remote
+            clients will not be able to connect). <note xml:id="how_many_zks">
+                <title>How many ZooKeepers should I run?</title>
+
+                <para>You can run a ZooKeeper ensemble that comprises 1 node
+                only but in production it is recommended that you run a
+                ZooKeeper ensemble of 3, 5 or 7 machines; the more members an
+                ensemble has, the more tolerant the ensemble is of host
+                failures. Also, run an odd number of machines. In ZooKeeper,
+                an even number of peers is supported, but it is normally not used
+                because an even sized ensemble requires, proportionally, more peers
+                to form a quorum than an odd sized ensemble requires. For example, an
+                ensemble with 4 peers requires 3 to form a quorum, while an ensemble with
+                5 also requires 3 to form a quorum. Thus, an ensemble of 5 allows 2 peers to
+                fail, and thus is more fault tolerant than the ensemble of 4, which allows
+                only 1 down peer.
+                </para>
+                <para>Give each ZooKeeper server around 1GB of RAM, and if possible, its own
+                dedicated disk (A dedicated disk is the best thing you can do
+                to ensure a performant ZooKeeper ensemble). For very heavily
+                loaded clusters, run ZooKeeper servers on separate machines
+                from RegionServers (DataNodes and TaskTrackers).</para>
+              </note></para>
+
+            <para>For example, to have HBase manage a ZooKeeper quorum on
+            nodes <emphasis>rs{1,2,3,4,5}.example.com</emphasis>, bound to
+            port 2222 (the default is 2181) ensure
+            <varname>HBASE_MANAGE_ZK</varname> is commented out or set to
+            <varname>true</varname> in <filename>conf/hbase-env.sh</filename>
+            and then edit <filename>conf/hbase-site.xml</filename> and set
+            <varname>hbase.zookeeper.property.clientPort</varname> and
+            <varname>hbase.zookeeper.quorum</varname>. You should also set
+            <varname>hbase.zookeeper.property.dataDir</varname> to other than
+            the default as the default has ZooKeeper persist data under
+            <filename>/tmp</filename> which is often cleared on system
+            restart. In the example below we have ZooKeeper persist to
+            <filename>/user/local/zookeeper</filename>. <programlisting>
+  &lt;configuration&gt;
+    ...
+    &lt;property&gt;
+      &lt;name&gt;hbase.zookeeper.property.clientPort&lt;/name&gt;
+      &lt;value&gt;2222&lt;/value&gt;
+      &lt;description&gt;Property from ZooKeeper's config zoo.cfg.
+      The port at which the clients will connect.
+      &lt;/description&gt;
+    &lt;/property&gt;
+    &lt;property&gt;
+      &lt;name&gt;hbase.zookeeper.quorum&lt;/name&gt;
+      &lt;value&gt;rs1.example.com,rs2.example.com,rs3.example.com,rs4.example.com,rs5.example.com&lt;/value&gt;
+      &lt;description&gt;Comma separated list of servers in the ZooKeeper Quorum.
+      For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+      By default this is set to localhost for local and pseudo-distributed modes
+      of operation. For a fully-distributed setup, this should be set to a full
+      list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+      this is the list of servers which we will start/stop ZooKeeper on.
+      &lt;/description&gt;
+    &lt;/property&gt;
+    &lt;property&gt;
+      &lt;name&gt;hbase.zookeeper.property.dataDir&lt;/name&gt;
+      &lt;value&gt;/usr/local/zookeeper&lt;/value&gt;
+      &lt;description&gt;Property from ZooKeeper's config zoo.cfg.
+      The directory where the snapshot is stored.
+      &lt;/description&gt;
+    &lt;/property&gt;
+    ...
+  &lt;/configuration&gt;</programlisting></para>
+  <caution>
+      <title>ZooKeeper Maintenance</title>
+      <para>Be sure to set up the data dir cleaner described under
+          <link xlink:href="http://zookeeper.apache.org/doc/r3.1.2/zookeeperAdmin.html#sc_maintenance">Zookeeper Maintenance</link> else you could
+          have 'interesting' problems a couple of months in; i.e. zookeeper could start
+          dropping sessions if it has to run through a directory of hundreds of thousands of
+          logs which is wont to do around leader reelection time -- a process rare but run on
+      occasion whether because a machine is dropped or happens to hiccup.</para>
+  </caution>
+
+            <section>
+              <title>Using existing ZooKeeper ensemble</title>
+
+              <para>To point HBase at an existing ZooKeeper cluster, one that
+              is not managed by HBase, set <varname>HBASE_MANAGES_ZK</varname>
+              in <filename>conf/hbase-env.sh</filename> to false
+              <programlisting>
+  ...
+  # Tell HBase whether it should manage its own instance of Zookeeper or not.
+  export HBASE_MANAGES_ZK=false</programlisting> Next set ensemble locations
+              and client port, if non-standard, in
+              <filename>hbase-site.xml</filename>, or add a suitably
+              configured <filename>zoo.cfg</filename> to HBase's
+              <filename>CLASSPATH</filename>. HBase will prefer the
+              configuration found in <filename>zoo.cfg</filename> over any
+              settings in <filename>hbase-site.xml</filename>.</para>
+
+              <para>When HBase manages ZooKeeper, it will start/stop the
+              ZooKeeper servers as a part of the regular start/stop scripts.
+              If you would like to run ZooKeeper yourself, independent of
+              HBase start/stop, you would do the following</para>
+
+              <programlisting>
+${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
+</programlisting>
+
+              <para>Note that you can use HBase in this manner to spin up a
+              ZooKeeper cluster, unrelated to HBase. Just make sure to set
+              <varname>HBASE_MANAGES_ZK</varname> to <varname>false</varname>
+              if you want it to stay up across HBase restarts so that when
+              HBase shuts down, it doesn't take ZooKeeper down with it.</para>
+
+              <para>For more information about running a distinct ZooKeeper
+              cluster, see the ZooKeeper <link
+              xlink:href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">Getting
+              Started Guide</link>.  Additionally, see the <link xlink:href="http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A7">ZooKeeper Wiki</link> or the
+          <link xlink:href="http://zookeeper.apache.org/doc/r3.3.3/zookeeperAdmin.html#sc_zkMulitServerSetup">ZooKeeper documentation</link>
+          for more information on ZooKeeper sizing.
+            </para>
+            </section>
+
+
+            <section xml:id="zk.sasl.auth">
+              <title>SASL Authentication with ZooKeeper</title>
+              <para>Newer releases of Apache HBase (&gt;= 0.92) will
+              support connecting to a ZooKeeper Quorum that supports
+              SASL authentication (which is available in Zookeeper
+              versions 3.4.0 or later).</para>
+
+              <para>This describes how to set up HBase to mutually
+              authenticate with a ZooKeeper Quorum. ZooKeeper/HBase
+              mutual authentication (<link
+              xlink:href="https://issues.apache.org/jira/browse/HBASE-2418">HBASE-2418</link>)
+              is required as part of a complete secure HBase configuration
+              (<link
+              xlink:href="https://issues.apache.org/jira/browse/HBASE-3025">HBASE-3025</link>).
+
+              For simplicity of explication, this section ignores
+              additional configuration required (Secure HDFS and Coprocessor
+              configuration).  It's recommended to begin with an
+              HBase-managed Zookeeper configuration (as opposed to a
+              standalone Zookeeper quorum) for ease of learning.
+              </para>
+
+              <section><title>Operating System Prerequisites</title></section>
+
+              <para>
+                  You need to have a working Kerberos KDC setup. For
+                  each <code>$HOST</code> that will run a ZooKeeper
+                  server, you should have a principle
+                  <code>zookeeper/$HOST</code>.  For each such host,
+                  add a service key (using the <code>kadmin</code> or
+                  <code>kadmin.local</code> tool's <code>ktadd</code>
+                  command) for <code>zookeeper/$HOST</code> and copy
+                  this file to <code>$HOST</code>, and make it
+                  readable only to the user that will run zookeeper on
+                  <code>$HOST</code>. Note the location of this file,
+                  which we will use below as
+                  <filename>$PATH_TO_ZOOKEEPER_KEYTAB</filename>.
+              </para>
+
+              <para>
+                Similarly, for each <code>$HOST</code> that will run
+                an HBase server (master or regionserver), you should
+                have a principle: <code>hbase/$HOST</code>. For each
+                host, add a keytab file called
+                <filename>hbase.keytab</filename> containing a service
+                key for <code>hbase/$HOST</code>, copy this file to
+                <code>$HOST</code>, and make it readable only to the
+                user that will run an HBase service on
+                <code>$HOST</code>. Note the location of this file,
+                which we will use below as
+                <filename>$PATH_TO_HBASE_KEYTAB</filename>.
+              </para>
+
+              <para>
+                Each user who will be an HBase client should also be
+                given a Kerberos principal. This principal should
+                usually have a password assigned to it (as opposed to,
+                as with the HBase servers, a keytab file) which only
+                this user knows. The client's principal's
+                <code>maxrenewlife</code> should be set so that it can
+                be renewed enough so that the user can complete their
+                HBase client processes. For example, if a user runs a
+                long-running HBase client process that takes at most 3
+                days, we might create this user's principal within
+                <code>kadmin</code> with: <code>addprinc -maxrenewlife
+                3days</code>. The Zookeeper client and server
+                libraries manage their own ticket refreshment by
+                running threads that wake up periodically to do the
+                refreshment.
+              </para>
+
+                <para>On each host that will run an HBase client
+                (e.g. <code>hbase shell</code>), add the following
+                file to the HBase home directory's <filename>conf</filename>
+                directory:</para>
+
+                <programlisting>
+                  Client {
+                    com.sun.security.auth.module.Krb5LoginModule required
+                    useKeyTab=false
+                    useTicketCache=true;
+                  };
+                </programlisting>
+
+                <para>We'll refer to this JAAS configuration file as
+                <filename>$CLIENT_CONF</filename> below.</para>
+
+              <section>
+                <title>HBase-managed Zookeeper Configuration</title>
+
+                <para>On each node that will run a zookeeper, a
+                master, or a regionserver, create a <link
+                xlink:href="http://docs.oracle.com/javase/1.4.2/docs/guide/security/jgss/tutorials/LoginConfigFile.html">JAAS</link>
+                configuration file in the conf directory of the node's
+                <filename>HBASE_HOME</filename> directory that looks like the
+                following:</para>
+
+                <programlisting>
+                  Server {
+                    com.sun.security.auth.module.Krb5LoginModule required
+                    useKeyTab=true
+                    keyTab="$PATH_TO_ZOOKEEPER_KEYTAB"
+                    storeKey=true
+                    useTicketCache=false
+                    principal="zookeeper/$HOST";
+                  };
+                  Client {
+                    com.sun.security.auth.module.Krb5LoginModule required
+                    useKeyTab=true
+                    useTicketCache=false
+                    keyTab="$PATH_TO_HBASE_KEYTAB"
+                    principal="hbase/$HOST";
+                  };
+                </programlisting>
+
+                where the <filename>$PATH_TO_HBASE_KEYTAB</filename> and
+                <filename>$PATH_TO_ZOOKEEPER_KEYTAB</filename> files are what
+                you created above, and <code>$HOST</code> is the hostname for that
+                node.
+
+                <para>The <code>Server</code> section will be used by
+                the Zookeeper quorum server, while the
+                <code>Client</code> section will be used by the HBase
+                master and regionservers. The path to this file should
+                be substituted for the text <filename>$HBASE_SERVER_CONF</filename>
+                in the <filename>hbase-env.sh</filename>
+                listing below.</para>
+
+                <para>
+                  The path to this file should be substituted for the
+                  text <filename>$CLIENT_CONF</filename> in the
+                  <filename>hbase-env.sh</filename> listing below.
+                </para>
+
+                <para>Modify your <filename>hbase-env.sh</filename> to include the
+                following:</para>
+
+                <programlisting>
+                  export HBASE_OPTS="-Djava.security.auth.login.config=$CLIENT_CONF"
+                  export HBASE_MANAGES_ZK=true
+                  export HBASE_ZOOKEEPER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
+                  export HBASE_MASTER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
+                  export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
+                </programlisting>
+
+                where <filename>$HBASE_SERVER_CONF</filename> and
+                <filename>$CLIENT_CONF</filename> are the full paths to the
+                JAAS configuration files created above.
+
+                <para>Modify your <filename>hbase-site.xml</filename> on each node
+                that will run zookeeper, master or regionserver to contain:</para>
+
+                <programlisting><![CDATA[
+                  <configuration>
+                    <property>
+                      <name>hbase.zookeeper.quorum</name>
+                      <value>$ZK_NODES</value>
+                    </property>
+                    <property>
+                      <name>hbase.cluster.distributed</name>
+                      <value>true</value>
+                    </property>
+                    <property>
+                      <name>hbase.zookeeper.property.authProvider.1</name>
+                      <value>org.apache.zookeeper.server.auth.SASLAuthenticationProvider</value>
+                    </property>
+                    <property>
+                      <name>hbase.zookeeper.property.kerberos.removeHostFromPrincipal</name>
+                      <value>true</value>
+                    </property>
+                    <property>
+                      <name>hbase.zookeeper.property.kerberos.removeRealmFromPrincipal</name>
+                      <value>true</value>
+                    </property>
+                  </configuration>
+                  ]]></programlisting>
+
+                <para>where <code>$ZK_NODES</code> is the
+                comma-separated list of hostnames of the Zookeeper
+                Quorum hosts.</para>
+
+                <para>Start your hbase cluster by running one or more
+                of the following set of commands on the appropriate
+                hosts:
+                </para>
+
+                <programlisting>
+                  bin/hbase zookeeper start
+                  bin/hbase master start
+                  bin/hbase regionserver start
+                </programlisting>
+
+              </section>
+
+              <section><title>External Zookeeper Configuration</title>
+                <para>Add a JAAS configuration file that looks like:
+
+                <programlisting>
+                  Client {
+                    com.sun.security.auth.module.Krb5LoginModule required
+                    useKeyTab=true
+                    useTicketCache=false
+                    keyTab="$PATH_TO_HBASE_KEYTAB"
+                    principal="hbase/$HOST";
+                  };
+                </programlisting>
+
+                where the <filename>$PATH_TO_HBASE_KEYTAB</filename> is the keytab
+                created above for HBase services to run on this host, and <code>$HOST</code> is the
+                hostname for that node. Put this in the HBase home's
+                configuration directory. We'll refer to this file's
+                full pathname as <filename>$HBASE_SERVER_CONF</filename> below.</para>
+
+                <para>Modify your hbase-env.sh to include the following:</para>
+
+                <programlisting>
+                  export HBASE_OPTS="-Djava.security.auth.login.config=$CLIENT_CONF"
+                  export HBASE_MANAGES_ZK=false
+                  export HBASE_MASTER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
+                  export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF"
+                </programlisting>
+
+
+                <para>Modify your <filename>hbase-site.xml</filename> on each node
+                that will run a master or regionserver to contain:</para>
+
+                <programlisting><![CDATA[
+                  <configuration>
+                    <property>
+                      <name>hbase.zookeeper.quorum</name>
+                      <value>$ZK_NODES</value>
+                    </property>
+                    <property>
+                      <name>hbase.cluster.distributed</name>
+                      <value>true</value>
+                    </property>
+                  </configuration>
+                  ]]>
+                </programlisting>
+
+                <para>where <code>$ZK_NODES</code> is the
+                comma-separated list of hostnames of the Zookeeper
+                Quorum hosts.</para>
+
+                <para>
+                  Add a <filename>zoo.cfg</filename> for each Zookeeper Quorum host containing:
+                  <programlisting>
+                      authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
+                      kerberos.removeHostFromPrincipal=true
+                      kerberos.removeRealmFromPrincipal=true
+                  </programlisting>
+
+                  Also on each of these hosts, create a JAAS configuration file containing:
+
+                  <programlisting>
+                  Server {
+                    com.sun.security.auth.module.Krb5LoginModule required
+                    useKeyTab=true
+                    keyTab="$PATH_TO_ZOOKEEPER_KEYTAB"
+                    storeKey=true
+                    useTicketCache=false
+                    principal="zookeeper/$HOST";
+                  };
+                  </programlisting>
+
+                  where <code>$HOST</code> is the hostname of each
+                  Quorum host. We will refer to the full pathname of
+                  this file as <filename>$ZK_SERVER_CONF</filename> below.
+
+                </para>
+
+                <para>
+                  Start your Zookeepers on each Zookeeper Quorum host with:
+
+                  <programlisting>
+                    SERVER_JVMFLAGS="-Djava.security.auth.login.config=$ZK_SERVER_CONF" bin/zkServer start
+                  </programlisting>
+
+                </para>
+
+                <para>
+                  Start your HBase cluster by running one or more of the following set of commands on the appropriate nodes:
+                </para>
+
+                <programlisting>
+                  bin/hbase master start
+                  bin/hbase regionserver start
+                </programlisting>
+
+
+              </section>
+
+              <section>
+                <title>Zookeeper Server Authentication Log Output</title>
+                <para>If the configuration above is successful,
+                you should see something similar to the following in
+                your Zookeeper server logs:
+                <programlisting>
+11/12/05 22:43:39 INFO zookeeper.Login: successfully logged in.
+11/12/05 22:43:39 INFO server.NIOServerCnxnFactory: binding to port 0.0.0.0/0.0.0.0:2181
+11/12/05 22:43:39 INFO zookeeper.Login: TGT refresh thread started.
+11/12/05 22:43:39 INFO zookeeper.Login: TGT valid starting at:        Mon Dec 05 22:43:39 UTC 2011
+11/12/05 22:43:39 INFO zookeeper.Login: TGT expires:                  Tue Dec 06 22:43:39 UTC 2011
+11/12/05 22:43:39 INFO zookeeper.Login: TGT refresh sleeping until: Tue Dec 06 18:36:42 UTC 2011
+..
+11/12/05 22:43:59 INFO auth.SaslServerCallbackHandler:
+  Successfully authenticated client: authenticationID=hbase/ip-10-166-175-249.us-west-1.compute.internal@HADOOP.LOCALDOMAIN;
+  authorizationID=hbase/ip-10-166-175-249.us-west-1.compute.internal@HADOOP.LOCALDOMAIN.
+11/12/05 22:43:59 INFO auth.SaslServerCallbackHandler: Setting authorizedID: hbase
+11/12/05 22:43:59 INFO server.ZooKeeperServer: adding SASL authorization for authorizationID: hbase
+                </programlisting>
+
+                </para>
+
+              </section>
+
+              <section>
+                <title>Zookeeper Client Authentication Log Output</title>
+                <para>On the Zookeeper client side (HBase master or regionserver),
+                you should see something similar to the following:
+
+                <programlisting>
+11/12/05 22:43:59 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=ip-10-166-175-249.us-west-1.compute.internal:2181 sessionTimeout=180000 watcher=master:60000
+11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Opening socket connection to server /10.166.175.249:2181
+11/12/05 22:43:59 INFO zookeeper.RecoverableZooKeeper: The identifier of this process is 14851@ip-10-166-175-249
+11/12/05 22:43:59 INFO zookeeper.Login: successfully logged in.
+11/12/05 22:43:59 INFO client.ZooKeeperSaslClient: Client will use GSSAPI as SASL mechanism.
+11/12/05 22:43:59 INFO zookeeper.Login: TGT refresh thread started.
+11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Socket connection established to ip-10-166-175-249.us-west-1.compute.internal/10.166.175.249:2181, initiating session
+11/12/05 22:43:59 INFO zookeeper.Login: TGT valid starting at:        Mon Dec 05 22:43:59 UTC 2011
+11/12/05 22:43:59 INFO zookeeper.Login: TGT expires:                  Tue Dec 06 22:43:59 UTC 2011
+11/12/05 22:43:59 INFO zookeeper.Login: TGT refresh sleeping until: Tue Dec 06 18:30:37 UTC 2011
+11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Session establishment complete on server ip-10-166-175-249.us-west-1.compute.internal/10.166.175.249:2181, sessionid = 0x134106594320000, negotiated timeout = 180000
+                </programlisting>
+                </para>
+              </section>
+
+              <section>
+                <title>Configuration from Scratch</title>
+
+                This has been tested on the current standard Amazon
+                Linux AMI.  First setup KDC and principals as
+                described above. Next checkout code and run a sanity
+                check.
+
+                <programlisting>
+                git clone git://git.apache.org/hbase.git
+                cd hbase
+                mvn clean test -Dtest=TestZooKeeperACL
+                </programlisting>
+
+                Then configure HBase as described above.
+                Manually edit target/cached_classpath.txt (see below)..
+
+                <programlisting>
+                bin/hbase zookeeper &amp;
+                bin/hbase master &amp;
+                bin/hbase regionserver &amp;
+                </programlisting>
+              </section>
+
+
+              <section>
+                <title>Future improvements</title>
+
+                <section><title>Fix target/cached_classpath.txt</title>
+                <para>
+                You must override the standard hadoop-core jar file from the
+                <code>target/cached_classpath.txt</code>
+                file with the version containing the HADOOP-7070 fix. You can use the following script to do this:
+
+                <programlisting>
+                  echo `find ~/.m2 -name "*hadoop-core*7070*SNAPSHOT.jar"` ':' `cat target/cached_classpath.txt` | sed 's/ //g' > target/tmp.txt
+                  mv target/tmp.txt target/cached_classpath.txt
+                </programlisting>
+
+                </para>
+
+                </section>
+
+                <section>
+                  <title>Set JAAS configuration
+                  programmatically</title>
+
+
+                  This would avoid the need for a separate Hadoop jar
+                  that fixes <link xlink:href="https://issues.apache.org/jira/browse/HADOOP-7070">HADOOP-7070</link>.
+                </section>
+
+                <section>
+                  <title>Elimination of
+                  <code>kerberos.removeHostFromPrincipal</code> and
+                  <code>kerberos.removeRealmFromPrincipal</code></title>
+                </section>
+
+              </section>
+
+
+            </section> <!-- SASL Authentication with ZooKeeper -->
+
+
+
+
+    </chapter>
diff --git a/src/site/resources/doap_Hbase.rdf b/src/site/resources/doap_Hbase.rdf
new file mode 100644
index 000000000000..08e9bc07cf14
--- /dev/null
+++ b/src/site/resources/doap_Hbase.rdf
@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl"?>
+<rdf:RDF xml:lang="en"
+         xmlns="http://usefulinc.com/ns/doap#" 
+         xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" 
+         xmlns:asfext="http://projects.apache.org/ns/asfext#"
+         xmlns:foaf="http://xmlns.com/foaf/0.1/">
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+   
+         http://www.apache.org/licenses/LICENSE-2.0
+   
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+  <Project rdf:about="http://hbase.apache.org">
+    <created>2012-04-14</created>
+    <license rdf:resource="http://usefulinc.com/doap/licenses/asl20" />
+    <name>Apache HBase</name>
+    <homepage rdf:resource="http://hbase.apache.org" />
+    <asfext:pmc rdf:resource="http://hbase.apache.org" />
+    <shortdesc>Apache HBase software is the Hadoop database. Think of it as a distributed, scalable, big data store.</shortdesc>
+    <description>Use Apache HBase software when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. HBase is an open-source, distributed, versioned, column-oriented store modeled after Google's Bigtable: A Distributed Storage System for Structured Data by Chang et al. Just as Bigtable leverages the distributed data storage provided by the Google File System, HBase provides Bigtable-like capabilities on top of Hadoop and HDFS. </description>
+    <bug-database rdf:resource="http://issues.apache.org/jira/browse/HBASE" />
+    <mailing-list rdf:resource="http://hbase.apache.org/mail-lists.html" />
+    <download-page rdf:resource="http://www.apache.org/dyn/closer.cgi/hbase/" />
+    <programming-language>Java</programming-language>
+    <category rdf:resource="http://projects.apache.org/category/database" />
+    <release>
+      <Version>
+        <name>Apache hbase 0.92.1</name>
+        <created>2012-03-19</created>
+        <revision>0.92.1</revision>
+      </Version>
+    </release>
+    <repository>
+      <SVNRepository>
+        <location rdf:resource="http://svn.apache.org/repos/asf/hbase"/>
+        <browse rdf:resource="http://svn.apache.org/viewvc/hbase"/>
+      </SVNRepository>
+    </repository>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Apache HBase PMC</foaf:name>
+          <foaf:mbox rdf:resource="mailto:dev@hbase.apache.org"/>
+      </foaf:Person>
+    </maintainer>
+  </Project>
+</rdf:RDF>
diff --git a/src/site/xdoc/resources.xml b/src/site/xdoc/resources.xml
new file mode 100644
index 000000000000..214fc79aa372
--- /dev/null
+++ b/src/site/xdoc/resources.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<document xmlns="http://maven.apache.org/XDOC/2.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
+  <properties>
+    <title>Other Apache HBase (TM) Resources</title>
+  </properties>
+
+<body>
+<section name="Other Apache HBase Resources">
+<section name="Books">
+<section name="HBase: The Definitive Guide">
+<p><a href="http://shop.oreilly.com/product/0636920014348.do">HBase: The Definitive Guide <i>Random Access to Your Planet-Size Data</i></a> by Lars George. Publisher: O'Reilly Media, Released: August 2011, Pages: 556.</p>
+</section>
+<section name="HBase In Action">
+<p><a href="http://www.manning.com/dimidukkhurana/">HBase In Action</a> By Nick Dimiduk and Amandeep Khurana.  Publisher: Manning, MEAP Began: January 2012, Softbound print: Fall 2012, Pages: 350.</p>
+</section>
+<section name="HBase Administration Cookbook">
+<p><a href="http://www.packtpub.com/hbase-administration-for-optimum-database-performance-cookbook/book">HBase Administration Cookbook</a> by Yifeng Jiang.  Publisher: PACKT Publishing, Release: Expected August 2012, Pages: 335.</p>
+</section>
+</section>
+</section>
+</body>
+</document>

From a740ff64603d20195a9d94dfd0f1ecfad249def1 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 14 Mar 2013 05:49:46 +0000
Subject: [PATCH 0845/1540] HBASE-8088 Versioning site: part one, put stake in
 the ground for 0.94 by copying current versions of book and site; SVN ADD NEW
 FILES

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456334 13f79535-47bb-0310-9956-ffa450edef68
---
 src/site/resources/images/big_h_logo.png | Bin 0 -> 2286 bytes
 src/site/resources/images/big_h_logo.svg | 139 +++++++++++++++++++++++
 2 files changed, 139 insertions(+)
 create mode 100644 src/site/resources/images/big_h_logo.png
 create mode 100644 src/site/resources/images/big_h_logo.svg

diff --git a/src/site/resources/images/big_h_logo.png b/src/site/resources/images/big_h_logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..5256094e46b6b8b756bf724fe27306acf259496c
GIT binary patch
literal 2286
zcmaJ@X*3&%7EVHJMJZ}2I#N|nOVk!oODeI~GKgKRL2Zd8LW@|2##W73I@VFGNLstv
z*UqS5Y&G`UsT#F6wbb&wU+?{RXYM)oe&>GQJ@=k_f88VtGXowjF)jcAz++^Hvb;e1
z3wPjPyNIJj$G{6<4be5S=D4s(4o}iW&WSU$4*>u`+y8MOA~`GPf)owavkSGtdW90)
zf-wLBfuQgx&@aTp4Tn*{27BjiX^H^=9DYWq`_|$4n*{;>j^08&#GZ-3obgAaIqHu@
zE$D4^0>jnCHQCS--%<R#wt86$?NBlwi?y1J8ehS4R{?APRvw!dYOHZvNOs>bdYBk=
z>tPI!ii*rM{(&5Fcw|Bv<7d-S?YAbWtpa@(wK=&vfCde1r-T%R6z=DrL~SpJP&U^1
z5$_jwWcZteeO_|#!*hq$2w`~qY1<?lAMeu+X$S#6l~@&PtGCi_cPFemH62wlGC+R}
zRh;OGMxhB|VPWm|fzN$Z9;eZ@S?{n%@+x>Q?;f%}jlOO(Mn$do+_}0TTc-yhCv?`*
zrzfVSng)xvne^kCU^tM(XgM>K-m_m2Ud^}0Ct&Q00<l=^R0{#yfqbslnQY7gybcsR
zY^B^4=Ovn<i4Tj<x%j7=-F{EBfRMu*CMW0Sv|*IqR@ORe(%pI5Nd4v%q(&DF6`BFX
zB7Vy0FXtoInnmp&MaE<7g@^=n+XDNRh(@$n_&`@Bi(PiFryA&q+#e<&sCpd?=wdsT
zwyKeAh=w1Yo~0qRJw!xACLs`cgg#30=dn~|Y6h%FZxEE8RgF36G>+-&>iW^#+&n1s
z1O*|Rbedb^g|hU{keZUj$zHxBltxVVh)-zHovZhj7iz8`Q0-Szx*sQ}U2q6Ak^56z
zJVUz(OQVBM6tAXe*-7;n+vTYRh$%aN4u(@{{Ym@T7_CK5hF2|1bmC`de~sib{gSr&
zfb-g6BeZQ22V_ZQNT^0eER8MsTEI*o$RTzV!|J8Zj*gPPlJpAAv<u=zRz8CZ*5x?u
zsa6H!a61K@-%l-?+M_#&PdgSGk!W)gBZHH-P9HrX2uo({__ZdBNFdTl{zb05RvEhF
zB4AvM;E{^uXJiQ<Km0YFD)FG6-kv+)MX)BDgX}m2hiK?ki>(Kh86KI@icnTM96rao
z6{<vHbQqup*Y*iASCZP@sfN-u5{FaaC1rmL?1o2r3vp(3Zy#Xq;TAfH$ulJ*UuNme
zZ=5=cVIurZx5RnR2iEp^)K_49={-ajw?pfTTQ*g>^=Hec1UEUU(i|a{mS?;p3s8?2
zHY7tT{S3g7#5gOy+~sZt+Rl3=m`(S6nY!tul{1mKxX=}UD|VpsniH*u=NxOOB<N)6
zo$BO>?OSKTPE)wS-hh-1Zps16IbE3%<Z54Vw^R-WAeJkCIRB8Z*-d*>p<<oH-=x$H
z^AhOF@0D~#NwSB?m<y*eDCuxYPuBzCQSgnu6_3G{AjQ#cl`hg(u>MNli*@(h|5pi<
z3?zHXhG$(0DupOJlZpdLkrEbrcA0L!$S6C9$M8?Fq%tl>d<Wd9VHY85b&|dU^*t{u
zLZ1AO1Up0g#eTv_P`4uMb>?jFEG06q4a`#Ujz;ZSqr^DaQXIppdFP)x{6q8qO3@3h
z1$WJw$PB^PMaRrvzr3Gd`&|&(jO>aE#(;M?i%H4TRH29feZ=?3r9Zp(&7zaWOK)Zu
z{E%i0SXa{tKzsDF#pNTW7HzE`F2M_KklhRV1yknm?o1N!Qrux8XBs>#;rmJ6=e4{l
zJw1;b3|(lA?L(=j40uluQa`+HjB0$n%Zr|M-d30U>M%@cJ}*(M(L7j;*#A5o`zcU#
zwy#98(0{`;ivK53Urpd~w6J^gL+3m1mk_poRS}I+Q<`fI=-HG@*jKU#t+O8w-@RAk
zSs^q@@4Q-!=A{0)|7K1nQr4Y#S$0ayV`-Op4XQE}2o5jB4;l{#RCE_g8y6;6VW@@!
zm?<?^rVn>h#EQ2_i+tjjN{geAF)})rFDho_wwujEPt~e<YI8l~N2c7ou#}_68DLTl
zn_0`bSN8JR50^B9Bu3np--hXR+D${k{@Vsrq;tt#AHbK~d7P>uwR3FHVZG&sx9?2o
z%_;F#ezy%nzC|`y9fwx+`YrqU9EAogi>O1D@h}Bu3nw1$5xLuoPw1hyf?Sh28hv}m
z<aV-vU#>qX>$<({GZH2mycut`qfq~C#GdR;DRylymDTV%P;?}p;y<G$v?B_tXPo^X
zN^pI@RZKq-QUV$1!RzXwOe<PdZT$4KeXRl_+Kk@4ezXF9$0nAdiF2Iq7Va1EmPXdP
zD0sa2gFQdb`;^U;cxkZ>oQ9q_LEcWRqgS2bOqK(JHr$^a-8S{Iz1HwNQFH0oO9A3Q
z%iw#S-lRHyTv!--u{o5Z{S*=97g#o@t$h6xkcMrh%*s{_xzHo2_XNNB$G>Gp<y<MN
z&|MXBts2C+XlY^#9bsNKGP7~-ZK|2!^7jSjO(DnqiypC?oy!U<wn70>>o+?4bev6*
z8n7Q??`T`~4jh~{=9T^n;Njn5PuOF1vXA?mC%?P+4s4DKc$%1~JEmX0>3qo`qj*8V
zb$obtui|KbP4LqxFU8>Y7S(}m6VZs*tQ@0?#C)sXJ>gpC1-l5_sZP5?yYGAQpY}9Q
z<ALHGFG^Ab%1woRx!l*VAAgPoM#jEj%l-QC09RGMy;SLEswB!a2O5OvH2k)xmBXTZ
zE7ag$lTy{2s|f-0y=+Qld7X?yUiM!aAGmwf-`<yQM`k<wfo*ktH-C+#^n%kSCUULI
zC$pLPn6+84wcxFq)$p67y}H+`wm9ykAv4Rxiga5#oGO>Bd39@>aH=Cpk=iq{Cmt6Y
zk?zqrTGd}%j(noz-h#e$z|zC2b!dh?9x)$tF^<q<`zFhfB4Yn;*`T*tz0EoY8C~tk
VU*nKWy7)~1MtWwb3SHN@e**}mMfLyy

literal 0
HcmV?d00001

diff --git a/src/site/resources/images/big_h_logo.svg b/src/site/resources/images/big_h_logo.svg
new file mode 100644
index 000000000000..ab241986fc37
--- /dev/null
+++ b/src/site/resources/images/big_h_logo.svg
@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Generator: Adobe Illustrator 15.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   width="792px"
+   height="612px"
+   viewBox="0 0 792 612"
+   enable-background="new 0 0 792 612"
+   xml:space="preserve"
+   inkscape:version="0.48.4 r9939"
+   sodipodi:docname="big_h_same_font_hbase3_logo.png"
+   inkscape:export-filename="big_h_bitmap.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"><metadata
+   id="metadata3693"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs3691" /><sodipodi:namedview
+   pagecolor="#000000"
+   bordercolor="#666666"
+   borderopacity="1"
+   objecttolerance="10"
+   gridtolerance="10"
+   guidetolerance="10"
+   inkscape:pageopacity="0"
+   inkscape:pageshadow="2"
+   inkscape:window-width="1440"
+   inkscape:window-height="856"
+   id="namedview3689"
+   showgrid="false"
+   inkscape:zoom="2.1814013"
+   inkscape:cx="415.39305"
+   inkscape:cy="415.72702"
+   inkscape:window-x="1164"
+   inkscape:window-y="22"
+   inkscape:window-maximized="0"
+   inkscape:current-layer="Layer_1" />
+
+
+
+
+
+
+<text
+   xml:space="preserve"
+   style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;font-family:Bitsumishi;-inkscape-font-specification:Bitsumishi"
+   x="311.18643"
+   y="86.224579"
+   id="text3082"
+   sodipodi:linespacing="125%"><tspan
+     sodipodi:role="line"
+     id="tspan3084"
+     x="311.18643"
+     y="86.224579" /></text>
+<text
+   xml:space="preserve"
+   style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;font-family:Bitsumishi;-inkscape-font-specification:Bitsumishi"
+   x="283.95764"
+   y="87.845337"
+   id="text3086"
+   sodipodi:linespacing="125%"><tspan
+     sodipodi:role="line"
+     id="tspan3088"
+     x="283.95764"
+     y="87.845337" /></text>
+<g
+   id="g3105"
+   transform="translate(14.669469,-80.682082)"
+   inkscape:export-filename="/Users/stack/Documents/big_h_base.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"><path
+     sodipodi:nodetypes="ccccccccccccccccccccccccccccc"
+     style="fill:#ba160c"
+     inkscape:connector-curvature="0"
+     id="path3677"
+     d="m 589.08202,499.77746 -40.3716,0 0,-168.36691 40.3716,0 z m -40.20304,-168.35619 -0.1684,-104.30857 40.3716,0 -0.33048,104.26805 z m -0.1684,168.35619 -40.37568,0 0,-104.82988 -259.42272,0 0,104.82988 -79.42128,0 0,-272.66476 79.42128,0 0,104.29785 224.92224,0 34.50456,0 40.37568,0 0,168.36691 z m 0,-272.66476 -40.37568,0 -0.0171,104.30857 40.55802,-0.01 z"
+     inkscape:export-filename="/Users/stack/Documents/polygon3687.png"
+     inkscape:export-xdpi="90"
+     inkscape:export-ydpi="90" /><path
+     sodipodi:nodetypes="cscsccsssccsssccscsccccccccccccccccccccc"
+     style="fill:#ba160c"
+     inkscape:connector-curvature="0"
+     id="path3679"
+     d="m 263.96692,553.27262 c 6.812,4.218 10.219,10.652 10.219,19.303 0,6.272 -2,11.571 -6.002,15.897 -4.325,4.758 -10.165,7.137 -17.519,7.137 l -28.629,0 0,-19.465 28.629,0 c 2.812,0 4.218,-2.109 4.218,-6.327 0,-4.216 -1.406,-6.325 -4.218,-6.325 l -28.629,0 0,-19.303 27.17,0 c 2.811,0 4.217,-2.109 4.217,-6.327 0,-4.216 -1.406,-6.326 -4.217,-6.326 l -27.17,0 0,-19.464 27.17,0 c 7.353,0 13.192,2.379 17.519,7.137 3.892,4.325 5.839,9.625 5.839,15.896 0,7.787 -2.866,13.842 -8.597,18.167 z m -41.931,42.338 -52.312,0 0,-51.42 19.466,0 5.259,0 27.588,0 0,19.303 -32.847,0 0,12.652 32.847,0 0,19.465 z m 0,-64.073 -32.847,0 0.0405,12.76351 -19.466,0.081 -0.0405,-32.30954 52.312,0 0,19.465 z" /><path
+     style="fill:#ba160c"
+     inkscape:connector-curvature="0"
+     id="path3683"
+     d="m 384.35292,595.61062 h -19.465 v -26.602 h -31.094 -0.618 v -19.466 h 0.618 31.094 v -11.68 c 0,-4.216 -1.406,-6.324 -4.218,-6.324 h -27.494 v -19.465 h 27.494 c 7.03,0 12.733,2.541 17.114,7.623 4.379,5.083 6.569,11.139 6.569,18.167 v 57.747 z m -51.177,-26.602 h -19.547 -12.165 v 26.602 h -19.466 v -57.748 c 0,-7.028 2.19,-13.083 6.569,-18.167 4.379,-5.083 10.03,-7.623 16.952,-7.623 h 27.656 v 19.466 h -27.656 c -2.704,0 -4.055,2.108 -4.055,6.324 v 11.68 h 12.165 19.547 v 19.466 z" /><path
+     style="fill:#ba160c"
+     inkscape:connector-curvature="0"
+     id="path3685"
+     d="m 492.35692,569.81862 c 0,7.03 -2.109,13.031 -6.327,18.006 -4.541,5.19 -10.273,7.786 -17.193,7.786 h -72.02 v -19.465 h 72.02 c 2.704,0 4.055,-2.109 4.055,-6.327 0,-4.216 -1.352,-6.325 -4.055,-6.325 h -52.394 c -6.92,0 -12.652,-2.596 -17.193,-7.787 -4.327,-4.865 -6.49,-10.813 -6.49,-17.843 0,-7.028 2.218,-13.083 6.651,-18.167 4.434,-5.083 10.112,-7.623 17.032,-7.623 h 72.021 v 19.464 h -72.021 c -2.703,0 -4.055,2.109 -4.055,6.326 0,4.109 1.352,6.164 4.055,6.164 h 52.394 c 6.92,0 12.652,2.596 17.193,7.787 4.218,4.974 6.327,10.976 6.327,18.004 z" /><polygon
+     style="fill:#ba160c"
+     transform="translate(-71.972085,223.93862)"
+     id="polygon3687"
+     points="656.952,339.555 591.906,339.555 591.906,352.207 661.331,352.207 661.331,371.672 572.44,371.672 572.44,288.135 661.494,288.135 661.494,307.599 591.906,307.599 591.906,320.089 656.952,320.089 "
+     inkscape:export-xdpi="90"
+     inkscape:export-ydpi="90" /><g
+     id="g3349"><g
+       id="g3344"><text
+         transform="scale(0.93350678,1.0712295)"
+         sodipodi:linespacing="125%"
+         id="text3076"
+         y="203.03328"
+         x="181.98402"
+         style="font-size:84.015625px;font-style:italic;font-variant:normal;font-weight:bold;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#4d4d4d;fill-opacity:1;stroke:none;font-family:Bitsumishi;-inkscape-font-specification:Bitsumishi Bold Italic"
+         xml:space="preserve"
+         inkscape:export-xdpi="90"
+         inkscape:export-ydpi="90"
+         inkscape:export-filename="/Users/stack/Documents/polygon3687.png"><tspan
+           style="font-size:84.015625px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:25.64349174px;writing-mode:lr-tb;text-anchor:start;fill:#4d4d4d;font-family:Bitsumishi;-inkscape-font-specification:Bitsumishi"
+           y="203.03328"
+           x="181.98402"
+           id="tspan3080"
+           sodipodi:role="line">APACHE</tspan></text>
+<rect
+         y="191.93103"
+         x="178.85117"
+         height="10.797735"
+         width="7.7796612"
+         id="rect3090"
+         style="fill:#4d4d4d" /></g><rect
+       style="fill:#4d4d4d"
+       id="rect3103"
+       width="8.1443329"
+       height="10.787481"
+       x="334.64697"
+       y="191.93881" /></g></g></svg>
\ No newline at end of file

From 0efdd87688019c0fecef2aa38231e5ba19defbe6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 14 Mar 2013 05:52:13 +0000
Subject: [PATCH 0846/1540] HBASE-8103 Fix pom so 0.94 can generate site
 reports

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456335 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5e8f270f8386..ae7cf6b05f54 100644
--- a/pom.xml
+++ b/pom.xml
@@ -383,7 +383,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-site-plugin</artifactId>
-          <version>2.0.1</version>
+          <version>3.2</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -566,6 +566,14 @@
     </testResources>
 
     <plugins>
+      <plugin>
+        <artifactId>maven-site-plugin</artifactId>
+        <configuration>
+          <inputEncoding>UTF-8</inputEncoding>
+          <outputEncoding>UTF-8</outputEncoding>
+          <templateFile>src/site/site.vm</templateFile>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro-maven-plugin</artifactId>
@@ -2433,7 +2441,7 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
     <plugins>
       <plugin>
         <artifactId>maven-project-info-reports-plugin</artifactId>
-        <version>2.1.2</version>
+        <version>2.6</version>
         <reportSets>
           <reportSet>
             <reports>
@@ -2449,15 +2457,6 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
         </reportSets>
 
       </plugin>
-      <plugin>
-        <artifactId>maven-site-plugin</artifactId>
-        <version>2.0.1</version>
-        <configuration>
-          <inputEncoding>UTF-8</inputEncoding>
-          <outputEncoding>UTF-8</outputEncoding>
-          <templateFile>src/site/site.vm</templateFile>
-        </configuration>
-      </plugin>
       <plugin>
         <artifactId>maven-javadoc-plugin</artifactId>
         <version>2.6.1</version>

From fa52ead83507dde6f29e4e02ca27ad148e2ccffc Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 14 Mar 2013 07:43:07 +0000
Subject: [PATCH 0847/1540] HBASE-8090 Versioning site; part two, publish 0.94
 site and add link from main site

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456356 13f79535-47bb-0310-9956-ffa450edef68
---
 src/site/site.vm  | 2 --
 src/site/site.xml | 8 +++++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/site/site.vm b/src/site/site.vm
index 0e2519528b63..f651010401cb 100644
--- a/src/site/site.vm
+++ b/src/site/site.vm
@@ -89,11 +89,9 @@
         #if ( $banner.alt )
             #set ( $alt = $banner.alt )
         #else
-            #set ( $alt = $banner.name )
         #end
         <img src="$src" alt="$alt" />
     #else
-        $banner.name
     #end
 ##
     #if( $banner.href )
diff --git a/src/site/site.xml b/src/site/site.xml
index 210277af5dc8..a6b7579798fb 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -28,9 +28,11 @@
     <src>images/hbase_logo.png</src>
     <href>http://hbase.apache.org/</href>
   </bannerLeft>
-  <bannerRight />
+  <bannerRight>
+      <name>Version 0.94</name>
+  </bannerRight>
   <version position="right" />
-  <publishDate position="right" />
+  <publishDate position="right"/>
   <body>
     <menu name="Apache HBase Project">
       <item name="Overview" href="index.html"/>
@@ -46,7 +48,7 @@
       <item name="Blog" href="http://blogs.apache.org/hbase/" />
       <item name="Other resources" href="resources.html" />
     </menu>
-    <menu name="Documentation">
+    <menu name="0.94 Documentation">
         <item name="Getting Started" href="book/quickstart.html" />
       <item name="API" href="apidocs/index.html" />
       <item name="X-Ref" href="xref/index.html" />

From fb756e68c0f84339f3c64323d69647eedd6c87f0 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 14 Mar 2013 07:43:13 +0000
Subject: [PATCH 0848/1540] HBASE-8090 Versioning site; part two, publish 0.94
 site and add link from main site

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456357 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/pom.xml b/pom.xml
index ae7cf6b05f54..6fe7bd6826ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2554,4 +2554,14 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
       </plugin>
     </plugins>
   </reporting>
+  <distributionManagement>
+      <site>
+          <id>hbase.apache.org</id>
+          <name>HBase Website at hbase.apache.org</name>
+          <!-- On why this is the tmp dir and not hbase.apache.org, see
+               https://issues.apache.org/jira/browse/HBASE-7593?focusedCommentId=13555866&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13555866
+               -->
+          <url>file:///tmp</url>
+      </site>
+  </distributionManagement>
 </project>

From 401339606cac4d52821e06219c066693c394a393 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Mar 2013 16:40:21 +0000
Subject: [PATCH 0849/1540] HBASE-8099 
 ReplicationZookeeper.copyQueuesFromRSUsingMulti should not return any queues
 if it failed to execute. (Himanshu and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456519 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/replication/ReplicationZookeeper.java   | 3 ++-
 .../regionserver/ReplicationSourceManager.java           | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index f23c31c62965..290741e239e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -672,7 +672,7 @@ public SortedMap<String, SortedSet<String>> copyQueuesFromRSUsingMulti(String zn
     List<ZKUtilOp> listOfOps = new ArrayList<ZKUtil.ZKUtilOp>();
     try {
       peerIdsToProcess = ZKUtil.listChildrenNoWatch(this.zookeeper, deadRSZnodePath);
-      if (peerIdsToProcess == null) return null; // node already processed
+      if (peerIdsToProcess == null) return queues; // node already processed
       for (String peerId : peerIdsToProcess) {
         String newPeerId = peerId + "-" + znode;
         String newPeerZnode = ZKUtil.joinZNode(this.rsServerNameZnode, newPeerId);
@@ -707,6 +707,7 @@ public SortedMap<String, SortedSet<String>> copyQueuesFromRSUsingMulti(String zn
     } catch (KeeperException e) {
       // Multi call failed; it looks like some other regionserver took away the logs.
       LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
+      queues.clear();
     }
     return queues;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 4c6c897ca682..51d0e61928f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -26,6 +26,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -89,6 +90,9 @@ public class ReplicationSourceManager {
   private final long sleepBeforeFailover;
   // Homemade executer service for replication
   private final ThreadPoolExecutor executor;
+  
+  private final Random rand;
+
 
   /**
    * Creates a replication manager and sets the watch on all the other
@@ -135,6 +139,7 @@ public ReplicationSourceManager(final ReplicationZookeeper zkHelper,
     ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
     tfb.setNameFormat("ReplicationExecutor-%d");
     this.executor.setThreadFactory(tfb.build());
+    this.rand = new Random();
   }
 
   /**
@@ -572,7 +577,7 @@ public void run() {
       // Wait a bit before transferring the queues, we may be shutting down.
       // This sleep may not be enough in some cases.
       try {
-        Thread.sleep(sleepBeforeFailover);
+        Thread.sleep(sleepBeforeFailover + (long) (rand.nextFloat() * sleepBeforeFailover));
       } catch (InterruptedException e) {
         LOG.warn("Interrupted while waiting before transferring a queue.");
         Thread.currentThread().interrupt();
@@ -597,7 +602,7 @@ public void run() {
         zkHelper.deleteRsQueues(rsZnode);
       }
       // process of copying over the failed queue is completed.
-      if (newQueues.size() == 0) {
+      if (newQueues.isEmpty()) {
         return;
       }
 

From c5311ec918ace1c03892416409910259fd9d9746 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Mar 2013 18:11:10 +0000
Subject: [PATCH 0850/1540] HBASE-8025  zkcli fails when SERVER_GC_OPTS is
 enabled (Dave Latham)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456580 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bin/hbase b/bin/hbase
index 21fb6f8c9d9f..15e109bada41 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -249,18 +249,18 @@ fi
 unset IFS
 
 #Set the right GC options based on the what we are running
-declare -a client_cmds=("shell" "hbck" "hlog" "hfile" "zkcli")
-for cmd in ${client_cmds[@]}; do
+declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper")
+for cmd in ${server_cmds[@]}; do
 	if [[ $cmd == $COMMAND ]]; then
-		client=true
+		server=true
 		break
 	fi
 done
 
-if [[ $client ]]; then
-	HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS"
-else
+if [[ $server ]]; then
 	HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS"
+else
+	HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS"
 fi
 
 # figure out which class to run

From 934afa912cfabcd2c8dd369e5d18736c0d60ac15 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Mar 2013 18:14:01 +0000
Subject: [PATCH 0851/1540] HBASE-7876  Got exception when manually triggers a
 split on an empty region (Maryann Xue)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456583 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/SplitTransaction.java  | 11 ++--
 .../TestSplitTransactionOnCluster.java        | 53 -------------------
 2 files changed, 5 insertions(+), 59 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index cae5d48d639d..058393de3155 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -282,12 +282,7 @@ private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
       if (exceptionToThrow instanceof IOException) throw (IOException)exceptionToThrow;
       throw new IOException(exceptionToThrow);
     }
-            
-    if (hstoreFilesToSplit.size() == 0) {
-      String errorMsg = "No store files to split for the region " + this.parent.getRegionInfo();
-      LOG.error(errorMsg);
-      throw new IOException(errorMsg);
-    }
+
     if (!testing) {
       services.removeFromOnlineRegions(this.parent.getRegionInfo().getEncodedName());
     }
@@ -598,6 +593,10 @@ private void splitStoreFiles(final Path splitdir,
     // there's files to split. It then fires up everything, waits for
     // completion and finally checks for any exception
     int nbFiles = hstoreFilesToSplit.size();
+    if (nbFiles == 0) {
+      // no file needs to be splitted.
+      return;
+    }
     ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
     builder.setNameFormat("StoreFileSplitter-%1$d");
     ThreadFactory factory = builder.build();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 5918af811106..eea0221458ee 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -795,60 +795,7 @@ private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) thro
     t.put(p);
     admin.flush(tableName);
   }
-  @Test(timeout = 15000)
-  public void testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfullyExecuteRollback()
-      throws Exception {
-    final byte[] tableName = Bytes
-        .toBytes("testShouldThrowIOExceptionIfStoreFileSizeIsEmptyAndSHouldSuccessfullyExecuteRollback");
-    HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-    try {
-      // Create table then get the single region for our new table.
-      HTableDescriptor htd = new HTableDescriptor(tableName);
-      htd.addFamily(new HColumnDescriptor("cf"));
-      admin.createTable(htd);
-      for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
-        Thread.sleep(100);
-      }
-      assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
-      List<HRegion> regions = cluster.getRegions(tableName);
-      HRegionInfo hri = getAndCheckSingleTableRegion(regions);
-      int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
-      int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
-      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
-      // Turn off balancer so it doesn't cut in and mess up our placements.
-      this.admin.setBalancerRunning(false, false);
-      // Turn off the meta scanner so it don't remove parent on us.
-      cluster.getMaster().setCatalogJanitorEnabled(false);
-      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
-      printOutRegions(server, "Initial regions: ");
-      // Now split.
-      SplitTransaction st = null;
-      st = new MockedSplitTransaction(regions.get(0), null);
-      try {
-        st.execute(regionServer, regionServer);
-      } catch (IOException e) {
-        List<HRegion> daughters = cluster.getRegions(tableName);
-        assertTrue(daughters.size() == 1);
 
-        String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0)
-            .getRegionInfo().getEncodedName());
-        assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-        assertTrue(st.rollback(regionServer, regionServer));
-        for(int i=0; ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1 && i<100; i++) {
-          Thread.sleep(100);
-        }
-        assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
-      }
-    } finally {
-      admin.setBalancerRunning(true, false);
-      cluster.getMaster().setCatalogJanitorEnabled(true);
-    }
-    if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) {
-      admin.disableTable(tableName);
-      admin.deleteTable(tableName);
-      admin.close();
-    }
-  }
   public static class MockedSplitTransaction extends SplitTransaction {
 
     private HRegion currentRegion;

From a91eba17c7921d2a2a018a1c6564d943375b902e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Mar 2013 18:17:46 +0000
Subject: [PATCH 0852/1540] CHANGES.txt for 0.94.6RC2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456584 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 90ae95595e69..1f5e196dc485 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.6 - 3/6/2013
+Release 0.94.6 - 3/14/2013
 Sub-task
 
     [HBASE-7944] - Replication leaks file reader resource & not reset currentNbOperations
@@ -14,6 +14,7 @@ Bug
     [HBASE-7153] - print gc option in hbase-env.sh affects hbase zkcli
     [HBASE-7507] - Make memstore flush be able to retry after exception
     [HBASE-7521] - fix HBASE-6060 (regions stuck in opening state) in 0.94
+    [HBASE-7624] - Backport HBASE-5359 and HBASE-7596 to 0.94
     [HBASE-7671] - Flushing memstore again after last failure could cause data loss
     [HBASE-7700] - TestColumnSeeking is mathematically bound to fail
     [HBASE-7723] - Remove NameNode URI from ZK splitlogs
@@ -24,6 +25,7 @@ Bug
     [HBASE-7777] - HBCK check for lingering split parents should check for child regions
     [HBASE-7813] - Bug in BulkDeleteEndpoint kills entire rows on COLUMN/VERSION Deletes
     [HBASE-7814] - Port HBASE-6963 'unable to run hbck on a secure cluster' to 0.94
+    [HBASE-7824] - Improve master start up time when there is log splitting work
     [HBASE-7829] - zookeeper kerberos conf keytab and principal parameters interchanged
     [HBASE-7832] - Use User.getShortName() in FSUtils
     [HBASE-7833] - 0.94 does not compile with Hadoop-0.20.205 and 0.22.0
@@ -31,6 +33,7 @@ Bug
     [HBASE-7866] - TestSplitTransactionOnCluster.testSplitBeforeSettingSplittingInZK failed 3 times in a row
     [HBASE-7867] - setPreallocSize is different with COMMENT in setupTestEnv in MiniZooKeeperCluster.java
     [HBASE-7869] - Provide way to not start LogSyncer thread
+    [HBASE-7876] - Got exception when manually triggers a split on an empty region
     [HBASE-7883] - Update memstore size when removing the entries in append operation
     [HBASE-7884] - ByteBloomFilter's performance can be improved by avoiding multiplication when generating hash
     [HBASE-7913] - Secure Rest server should login before getting an instance of Rest servlet
@@ -44,16 +47,31 @@ Bug
     [HBASE-7991] - Backport HBASE-6479 'HFileReaderV1 caching the same parent META block could cause server abort when splitting' to 0.94
     [HBASE-8007] - Adopt TestLoadAndVerify from BigTop
     [HBASE-8019] - Port HBASE-7779 '[snapshot 130201 merge] Fix TestMultiParallel' to 0.94
+    [HBASE-8025] - zkcli fails when SERVER_GC_OPTS is enabled
+    [HBASE-8040] - Race condition in AM after HBASE-7521 (only 0.94)
+    [HBASE-8055] - Null check missing in StoreFile.Reader.getMaxTimestamp()
+    [HBASE-8061] - Missing test from TestFlushSnapshotFromClient in 0.94
+    [HBASE-8069] - TestHLog is dependent on the execution order
+    [HBASE-8085] - Backport the fix for Bytes.toStringBinary() into 94 (HBASE-6991)
+    [HBASE-8099] - ReplicationZookeeper.copyQueuesFromRSUsingMulti should not return any queues if it failed to execute.
+    [HBASE-8103] - Fix pom so 0.94 can generate site reports
 
 Improvement
 
     [HBASE-7818] - add region level metrics readReqeustCount and writeRequestCount
+    [HBASE-7827] - Improve the speed of Hbase Thirft Batch mutation for deletes
+    [HBASE-8031] - Adopt goraci as an Integration test
 
 New Feature
 
     [HBASE-4210] - Allow coprocessor to interact with batches per region sent from a client
     [HBASE-7360] - Snapshot 0.94 Backport
 
+Task
+
+    [HBASE-8088] - Versioning site: part one, put stake in the ground for 0.94 by copying current versions of book and site
+    [HBASE-8090] - Versioning site; part two, publish 0.94 site and add link from main site
+
 
 Release 0.94.5 - 2/7/2013
 Sub-task

From 20387a21cd9c2c9acf0259489e05fd1586861490 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 14 Mar 2013 21:28:36 +0000
Subject: [PATCH 0853/1540] HBASE-7824 Improve master start up time when there
 is log splitting work, revert due to
 TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS failure

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456689 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  |   2 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 110 ++++-------
 .../hadoop/hbase/master/MasterFileSystem.java |  33 ++--
 .../hadoop/hbase/master/ServerManager.java    |  10 -
 .../master/handler/ServerShutdownHandler.java |   4 -
 .../hbase/regionserver/HRegionServer.java     |   7 +-
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |   2 +-
 .../apache/hadoop/hbase/MiniHBaseCluster.java |   5 +-
 .../master/TestDistributedLogSplitting.java   | 178 ++----------------
 .../TestRSKilledWhenMasterInitializing.java   |  25 ++-
 10 files changed, 97 insertions(+), 279 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 58c61ee1703d..5f2f1482c7f2 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -118,7 +118,7 @@ public class CatalogTracker {
    */
   private ServerName metaLocation;
 
-  private volatile boolean stopped = false;
+  private boolean stopped = false;
 
   static final byte [] ROOT_REGION_NAME =
     HRegionInfo.ROOT_REGIONINFO.getRegionName();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c336bc8286d6..c414adfea451 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -41,6 +41,9 @@
 
 import javax.management.ObjectName;
 
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -127,10 +130,6 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 
-import com.google.common.collect.ClassToInstanceMap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MutableClassToInstanceMap;
-
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
  * master.  If many masters are started, all compete.  Whichever wins goes on to
@@ -567,37 +566,14 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     if (!masterRecovery) {
       this.assignmentManager.startTimeOutMonitor();
     }
+    // TODO: Should do this in background rather than block master startup
+    status.setStatus("Splitting logs after master startup");
+    splitLogAfterStartup(this.fileSystemManager);
 
-    // get a list for previously failed RS which need recovery work
-    Set<ServerName> failedServers = this.fileSystemManager.getFailedServersFromLogFolders();
-    ServerName preRootServer = this.catalogTracker.getRootLocation();
-    if (preRootServer != null && failedServers.contains(preRootServer)) {
-      // create recovered edits file for _ROOT_ server
-      this.fileSystemManager.splitLog(preRootServer);
-      failedServers.remove(preRootServer);
-    }
-
-    // Make sure root assigned before proceeding.
-    assignRoot(status);
-
-    // log splitting for .META. server
-    ServerName preMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-    if (preMetaServer != null && failedServers.contains(preMetaServer)) {
-      // create recovered edits file for .META. server
-      this.fileSystemManager.splitLog(preMetaServer);
-      failedServers.remove(preMetaServer);
-    }
-    // Make sure meta assigned before proceeding.
-    assignMeta(status, preRootServer);
-
+    // Make sure root and meta assigned before proceeding.
+    assignRootAndMeta(status);
     enableServerShutdownHandler();
 
-    // handle other dead servers in SSH
-    status.setStatus("Submit log splitting work of non-meta region servers");
-    for (ServerName curServer : failedServers) {
-      this.serverManager.processDeadServer(curServer);
-    }
-
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
     // before we assign all user regions or else the assignment will fail.
@@ -669,13 +645,22 @@ protected void startCatalogJanitorChore() {
   }
 
   /**
-   * Check <code>-ROOT-</code> is assigned. If not, assign it.
-   * @param status MonitoredTask
+   * Override to change master's splitLogAfterStartup. Used testing
+   * @param mfs
+   */
+  protected void splitLogAfterStartup(final MasterFileSystem mfs) {
+    mfs.splitLogAfterStartup();
+  }
+
+  /**
+   * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,
+   * assign them.
    * @throws InterruptedException
    * @throws IOException
    * @throws KeeperException
+   * @return Count of regions we assigned.
    */
-  private void assignRoot(MonitoredTask status)
+  int assignRootAndMeta(MonitoredTask status)
   throws InterruptedException, IOException, KeeperException {
     int assigned = 0;
     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
@@ -706,32 +691,16 @@ private void assignRoot(MonitoredTask status)
     LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getRootLocation());
 
-    status.setStatus("ROOT assigned.");
-  }
-
-  /**
-   * Check <code>.META.</code> is assigned. If not, assign it.
-   * @param status MonitoredTask
-   * @param previousRootServer ServerName of previous root region server before current start up
-   * @return
-   * @throws InterruptedException
-   * @throws IOException
-   * @throws KeeperException
-   */
-  private void assignMeta(MonitoredTask status, ServerName previousRootServer)
-      throws InterruptedException,
-      IOException, KeeperException {
-    int assigned = 0;
-    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
-
+    // Work on meta region
     status.setStatus("Assigning META region");
-    boolean rit =
-        this.assignmentManager
-            .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
+    rit = this.assignmentManager.
+      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
     boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
     if (!rit && !metaRegionLocation) {
-      ServerName currentMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-      if (currentMetaServer != null && !currentMetaServer.equals(previousRootServer)) {
+      ServerName currentMetaServer =
+        this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+      if (currentMetaServer != null
+          && !currentMetaServer.equals(currentRootServer)) {
         splitLogAndExpireIfOnline(currentMetaServer);
       }
       assignmentManager.assignMeta();
@@ -741,14 +710,15 @@ private void assignMeta(MonitoredTask status, ServerName previousRootServer)
       enableSSHandWaitForMeta();
       assigned++;
     } else {
-      // Region already assigned. We didnt' assign it. Add to in-memory state.
+      // Region already assigned.  We didnt' assign it.  Add to in-memory state.
       this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
         this.catalogTracker.getMetaLocation());
     }
     enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
-    LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
-        + catalogTracker.getMetaLocation());
-    status.setStatus("META assigned.");
+    LOG.info(".META. assigned=" + assigned + ", rit=" + rit +
+      ", location=" + catalogTracker.getMetaLocation());
+    status.setStatus("META and ROOT assigned.");
+    return assigned;
   }
 
   private void enableSSHandWaitForMeta() throws IOException,
@@ -807,7 +777,8 @@ public boolean visit(Result r) throws IOException {
   }
 
   /**
-   * Expire a server if we find it is one of the online servers.
+   * Split a server's log and expire it if we find it is one of the online
+   * servers.
    * @param sn ServerName to check.
    * @throws IOException
    */
@@ -1672,23 +1643,12 @@ public void shutdown() {
     }
     if (this.assignmentManager != null) this.assignmentManager.shutdown();
     if (this.serverManager != null) this.serverManager.shutdownCluster();
-
     try {
       if (this.clusterStatusTracker != null){
         this.clusterStatusTracker.setClusterDown();
       }
     } catch (KeeperException e) {
-      if (e instanceof KeeperException.SessionExpiredException) {
-        LOG.warn("ZK session expired. Retry a new connection...");
-        try {
-          this.zooKeeper.reconnectAfterExpiration();
-          this.clusterStatusTracker.setClusterDown();
-        } catch (Exception ex) {
-          LOG.warn("Retry setClusterDown failed", ex);
-        }
-      } else {
-        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
-      }
+      LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 5ed1559f8fd5..54ce8c59ab5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -21,7 +21,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
@@ -198,31 +197,30 @@ public String getClusterId() {
   }
 
   /**
-   * Inspect the log directory to find dead servers which need log splitting
+   * Inspect the log directory to recover any log file without
+   * an active region server.
    */
-  Set<ServerName> getFailedServersFromLogFolders() {
+  void splitLogAfterStartup() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
-      HLog.SPLIT_SKIP_ERRORS_DEFAULT);
-    
-    Set<ServerName> serverNames = new HashSet<ServerName>();
+        HLog.SPLIT_SKIP_ERRORS_DEFAULT);
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
-
     do {
       if (master.isStopped()) {
-        LOG.warn("Master stopped while trying to get failed servers.");
+        LOG.warn("Master stopped while splitting logs");
         break;
       }
+      List<ServerName> serverNames = new ArrayList<ServerName>();
       try {
-        if (!this.fs.exists(logsDirPath)) return serverNames;
+        if (!this.fs.exists(logsDirPath)) return;
         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
         // Get online servers after getting log folders to avoid log folder deletion of newly
         // checked in region servers . see HBASE-5916
-        Set<ServerName> onlineServers =
-            ((HMaster) master).getServerManager().getOnlineServers().keySet();
+        Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
+            .keySet();
 
         if (logFolders == null || logFolders.length == 0) {
           LOG.debug("No log files to split, proceeding...");
-          return serverNames;
+          return;
         }
         for (FileStatus status : logFolders) {
           String sn = status.getPath().getName();
@@ -236,19 +234,22 @@ Set<ServerName> getFailedServersFromLogFolders() {
                 + "to a known region server, splitting");
             serverNames.add(serverName);
           } else {
-            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
+            LOG.info("Log folder " + status.getPath()
+                + " belongs to an existing region server");
           }
         }
+        splitLog(serverNames);
         retrySplitting = false;
       } catch (IOException ioe) {
-        LOG.warn("Failed getting failed servers to be recovered.", ioe);
+        LOG.warn("Failed splitting of " + serverNames, ioe);
         if (!checkFileSystem()) {
           LOG.warn("Bad Filesystem, exiting");
           Runtime.getRuntime().halt(1);
         }
         try {
           if (retrySplitting) {
-            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
+            Thread.sleep(conf.getInt(
+              "hbase.hlog.split.failure.retry.interval", 30 * 1000));
           }
         } catch (InterruptedException e) {
           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
@@ -258,8 +259,6 @@ Set<ServerName> getFailedServersFromLogFolders() {
         }
       }
     } while (retrySplitting);
-
-    return serverNames;
   }
 
   public void splitLog(final ServerName serverName) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 6d5c45a3cdc7..039702007613 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -424,16 +424,6 @@ public synchronized void expireServer(final ServerName serverName) {
         carryingRoot + ", meta=" + carryingMeta);
   }
 
-  /**
-   * The function is to allow master to submit known dead servers into SSH
-   * @param serverName
-   */
-  void processDeadServer(final ServerName serverName) {
-    this.deadservers.add(serverName);
-    this.services.getExecutorService().submit(
-      new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName, true));
-  }
-
   /**
    * Expire the servers which died during master's initialization. It will be
    * called after HMaster#assignRootAndMeta.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index b545891c1d2f..8b889f7db921 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -179,10 +179,6 @@ public String toString() {
   public void process() throws IOException {
     final ServerName serverName = this.serverName;
     try {
-      if (this.server.isStopped()) {
-        throw new IOException("Server is stopped");
-      }
-
       try {
         if (this.shouldSplitHlog) {
           LOG.info("Splitting logs for " + serverName);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e58cf9454d2a..455da6f5cc0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,12 +66,12 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterAddressTracker;
 import org.apache.hadoop.hbase.NotServingRegionException;
@@ -172,6 +172,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 import org.codehaus.jackson.map.ObjectMapper;
+import org.joda.time.field.MillisDurationField;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
@@ -1704,9 +1705,7 @@ public CatalogTracker getCatalogTracker() {
   @Override
   public void stop(final String msg) {
     try {
-      if (this.rsHost != null) {
-        this.rsHost.preStop(msg);
-      }
+      this.rsHost.preStop(msg);
       this.stopped = true;
       LOG.info("STOPPED: " + msg);
       // Wakes run() if it is sleeping
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index 4365f78e7172..c6e607ef1334 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -45,7 +45,7 @@ public abstract class ZooKeeperNodeTracker extends ZooKeeperListener {
   /** Used to abort if a fatal error occurs */
   protected final Abortable abortable;
 
-  private volatile boolean stopped = false;
+  private boolean stopped = false;
 
   /**
    * Constructs a new ZK node tracker.
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 109d94eeae49..288bd6ccbd41 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.security.User;
@@ -453,9 +452,7 @@ public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
     while (!(mts = getMasterThreads()).isEmpty()
         && (System.currentTimeMillis() - start) < timeout) {
       for (JVMClusterUtil.MasterThread mt : mts) {
-        ServerManager serverManager = mt.getMaster().getServerManager();
-        if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()
-            && !serverManager.areDeadServersInProgress()) {
+        if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) {
           return true;
         }
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 36a3c547e137..823d7da933de 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -19,29 +19,22 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_mgr_wait_for_zk_delete;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_final_transistion_failed;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_done;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_err;
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_resigned;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.TreeSet;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -50,13 +43,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -67,9 +54,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -96,21 +82,15 @@ public class TestDistributedLogSplitting {
   Configuration conf;
   HBaseTestingUtility TEST_UTIL;
 
-
   private void startCluster(int num_rs) throws Exception{
-    conf = HBaseConfiguration.create();
-    startCluster(NUM_MASTERS, num_rs, conf);
-  }
-
-  private void startCluster(int num_master, int num_rs, Configuration inConf) throws Exception {
     ZKSplitLog.Counters.resetCounters();
     LOG.info("Starting cluster");
-    this.conf = inConf;
+    conf = HBaseConfiguration.create();
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
     TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(num_master, num_rs);
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
@@ -122,10 +102,6 @@ private void startCluster(int num_master, int num_rs, Configuration inConf) thro
 
   @After
   public void after() throws Exception {
-    for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
-      mt.getMaster().abort("closing...", new Exception("Trace info"));
-    }
-
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -229,89 +205,6 @@ public void testRecoveredEdits() throws Exception {
     assertEquals(NUM_LOG_LINES, count);
   }
 
-  @Test(timeout = 300000)
-  public void testMasterStartsUpWithLogSplittingWork() throws Exception {
-    LOG.info("testMasterStartsUpWithLogSplittingWork");
-    Configuration curConf = HBaseConfiguration.create();
-    curConf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
-    startCluster(2, NUM_RS, curConf);
-
-    final int NUM_REGIONS_TO_CREATE = 40;
-    final int NUM_LOG_LINES = 1000;
-    // turn off load balancing to prevent regions from moving around otherwise
-    // they will consume recovered.edits
-    master.balanceSwitch(false);
-
-    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
-    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
-    HTable ht = installTable(zkw, "table", "f", NUM_REGIONS_TO_CREATE);
-
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = hrs.getOnlineRegions();
-      for (HRegionInfo region : regions) {
-        if (region.isRootRegion() || region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
-    makeHLog(hrs.getWAL(), regions, "table", NUM_LOG_LINES, 100);
-
-    // abort master
-    abortMaster(cluster);
-
-    // abort RS
-    int numRS = cluster.getLiveRegionServerThreads().size();
-    LOG.info("Aborting region server: " + hrs.getServerName());
-    hrs.abort("testing");
-
-    // wait for the RS dies
-    long start = EnvironmentEdgeManager.currentTimeMillis();
-    while (cluster.getLiveRegionServerThreads().size() > (numRS - 1)) {
-      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
-        assertTrue(false);
-      }
-      Thread.sleep(200);
-    }
-
-    Thread.sleep(2000);
-    LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
-    
-    startMasterAndWaitUntilLogSplit(cluster);
-    
-    start = EnvironmentEdgeManager.currentTimeMillis();
-    while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
-      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
-        assertTrue("Timedout", false);
-      }
-      Thread.sleep(200);
-    }
-
-    LOG.info("Current Open Regions After Master Node Starts Up:"
-        + getAllOnlineRegions(cluster).size());
-
-    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
-
-    ht.close();
-  }
-
   /**
    * The original intention of this test was to force an abort of a region
    * server and to make sure that the failure path in the region servers is
@@ -500,40 +393,33 @@ public void makeHLog(HLog log,
       List<HRegionInfo> hris, String tname,
       int num_edits, int edit_size) throws IOException {
 
-    // remove root and meta region
-    hris.remove(HRegionInfo.ROOT_REGIONINFO);
-    hris.remove(HRegionInfo.FIRST_META_REGIONINFO);
     byte[] table = Bytes.toBytes(tname);
     HTableDescriptor htd = new HTableDescriptor(tname);
     byte[] value = new byte[edit_size];
     for (int i = 0; i < edit_size; i++) {
-      value[i] = (byte) ('a' + (i % 26));
+      value[i] = (byte)('a' + (i % 26));
     }
     int n = hris.size();
     int[] counts = new int[n];
+    int j = 0;
     if (n > 0) {
       for (int i = 0; i < num_edits; i += 1) {
         WALEdit e = new WALEdit();
-        HRegionInfo curRegionInfo = hris.get(i % n);
-        byte[] startRow = curRegionInfo.getStartKey();
-        if (startRow == null || startRow.length == 0) {
-          startRow = new byte[] { 0, 0, 0, 0, 1 };
-        }
-        byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
-        row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because
-                                             // HBaseTestingUtility.createMultiRegions use 5 bytes
-                                             // key
-        byte[] family = Bytes.toBytes("f");
-        byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
-        e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
-        log.append(curRegionInfo, table, e, System.currentTimeMillis(), htd);
-        counts[i % n] += 1;
+        byte [] row = Bytes.toBytes("r" + Integer.toString(i));
+        byte [] family = Bytes.toBytes("f");
+        byte [] qualifier = Bytes.toBytes("c" + Integer.toString(i));
+        e.add(new KeyValue(row, family, qualifier,
+            System.currentTimeMillis(), value));
+        j++;
+        log.append(hris.get(j % n), table, e, System.currentTimeMillis(), htd);
+        counts[j % n] += 1;
       }
     }
     log.sync();
     log.close();
     for (int i = 0; i < n; i++) {
-      LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
+      LOG.info("region " + hris.get(i).getRegionNameAsString() +
+          " has " + counts[i] + " edits");
     }
     return;
   }
@@ -593,30 +479,6 @@ private void waitForCounter(AtomicLong ctr, long oldval, long newval,
     assertTrue(false);
   }
 
-  private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException {
-    for (MasterThread mt : cluster.getLiveMasterThreads()) {
-      if (mt.getMaster().isActiveMaster()) {
-        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
-        mt.join();
-        break;
-      }
-    }
-    LOG.debug("Master is aborted");
-  }
-
-  private void startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
-      throws IOException, InterruptedException {
-    cluster.startMaster();
-    HMaster master = cluster.getMaster();
-    while (!master.isInitialized()) {
-      Thread.sleep(100);
-    }
-    ServerManager serverManager = master.getServerManager();
-    while (serverManager.areDeadServersInProgress()) {
-      Thread.sleep(100);
-    }
-  }
-
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index eb2f42dccc87..a7287a1b84a2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -36,16 +37,19 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.TestMasterFailover;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
@@ -98,6 +102,19 @@ public TestingMaster(Configuration conf) throws IOException,
       super(conf);
     }
 
+    @Override
+    protected void splitLogAfterStartup(MasterFileSystem mfs) {
+      super.splitLogAfterStartup(mfs);
+      logSplit = true;
+      // If "TestingMaster.sleep" is set, sleep after log split.
+      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
+        int duration = getConfiguration().getInt(
+            "TestingMaster.sleep.duration", 0);
+        Threads.sleep(duration);
+      }
+    }
+
+
     public boolean isLogSplitAfterStartup() {
       return logSplit;
     }
@@ -232,13 +249,11 @@ private void abortMaster(MiniHBaseCluster cluster)
   private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
       throws IOException, InterruptedException {
     TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
-    while (!master.isInitialized()) {
-      Thread.sleep(100);
-    }
-    ServerManager serverManager = cluster.getMaster().getServerManager();
-    while (serverManager.areDeadServersInProgress()) {
+    while (!master.isLogSplitAfterStartup()) {
       Thread.sleep(100);
     }
+    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
+        + master.isInitialized());
     return master;
   }
 

From 43a99c0066351bbe407a368bd87405c732b4ea29 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Mar 2013 21:37:28 +0000
Subject: [PATCH 0854/1540] updated CHANGES.txt

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456690 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 1f5e196dc485..7f105ca39b22 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -25,7 +25,6 @@ Bug
     [HBASE-7777] - HBCK check for lingering split parents should check for child regions
     [HBASE-7813] - Bug in BulkDeleteEndpoint kills entire rows on COLUMN/VERSION Deletes
     [HBASE-7814] - Port HBASE-6963 'unable to run hbck on a secure cluster' to 0.94
-    [HBASE-7824] - Improve master start up time when there is log splitting work
     [HBASE-7829] - zookeeper kerberos conf keytab and principal parameters interchanged
     [HBASE-7832] - Use User.getShortName() in FSUtils
     [HBASE-7833] - 0.94 does not compile with Hadoop-0.20.205 and 0.22.0

From dc99697bb319e267088ea6b9f080086b663ed28e Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 15 Mar 2013 13:38:44 +0000
Subject: [PATCH 0855/1540] HBASE-8118 TestTablePermission depends on the
 execution order

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1456951 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/security/access/TestTablePermissions.java      | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
index 9c04d59d3422..0410e97a79f5 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -104,6 +105,14 @@ public static void afterClass() throws Exception {
     UTIL.shutdownMiniCluster();
   }
 
+  @After
+  public void tearDown() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+    AccessControlLists.removeTablePermissions(conf, TEST_TABLE);
+    AccessControlLists.removeTablePermissions(conf, TEST_TABLE2);
+    AccessControlLists.removeTablePermissions(conf, AccessControlLists.ACL_TABLE_NAME);
+  }
+
   @Test
   public void testBasicWrite() throws Exception {
     Configuration conf = UTIL.getConfiguration();

From eac567f44fd0a915c44d341007ad4a2593e4f4ce Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 16 Mar 2013 07:20:17 +0000
Subject: [PATCH 0856/1540] HBASE-8106 Test to check replication log znodes
 move is done correctly (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1457215 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestReplicationSourceManager.java         | 113 +++++++++++++++++-
 1 file changed, 109 insertions(+), 4 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 33ee4d0942c2..e11703eaf5e7 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -23,7 +23,12 @@
 
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -92,7 +97,10 @@ public class TestReplicationSourceManager {
   private static Path oldLogDir;
 
   private static Path logDir;
+  
+  private static CountDownLatch latch;
 
+  private static List<String> files = new ArrayList<String>();
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
@@ -213,8 +221,105 @@ public void testLogRoll() throws Exception {
 
     // TODO Need a case with only 2 HLogs and we only want to delete the first one
   }
+  
+  @Test
+  public void testNodeFailoverWorkerCopyQueuesFromRSUsingMulti() throws Exception {
+    LOG.debug("testNodeFailoverWorkerCopyQueuesFromRSUsingMulti");
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
+    final Server server = new DummyServer("hostname0.example.org");
+    AtomicBoolean replicating = new AtomicBoolean(true);
+    ReplicationZookeeper rz = new ReplicationZookeeper(server, replicating);
+    // populate some znodes in the peer znode
+    files.add("log1");
+    files.add("log2");
+    for (String file : files) {
+      rz.addLogToList(file, "1");
+    }
+    // create 3 DummyServers
+    Server s1 = new DummyServer("dummyserver1.example.org");
+    Server s2 = new DummyServer("dummyserver2.example.org");
+    Server s3 = new DummyServer("dummyserver3.example.org");
+
+    // create 3 DummyNodeFailoverWorkers
+    DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(
+        server.getServerName().getServerName(), s1);
+    DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(
+        server.getServerName().getServerName(), s2);
+    DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(
+        server.getServerName().getServerName(), s3);
+
+    latch = new CountDownLatch(3);
+    // start the threads
+    w1.start();
+    w2.start();
+    w3.start();
+    // make sure only one is successful
+    int populatedMap = 0;
+    // wait for result now... till all the workers are done.
+    latch.await();
+    populatedMap += w1.isLogZnodesMapPopulated() + w2.isLogZnodesMapPopulated()
+        + w3.isLogZnodesMapPopulated();
+    assertEquals(1, populatedMap);
+    // close out the resources.
+    server.abort("", null);
+  }
+
+  static class DummyNodeFailoverWorker extends Thread {
+    private SortedMap<String, SortedSet<String>> logZnodesMap;
+    Server server;
+    private String deadRsZnode;
+    ReplicationZookeeper rz;
+
+    public DummyNodeFailoverWorker(String znode, Server s) throws Exception {
+      this.deadRsZnode = znode;
+      this.server = s;
+      rz = new ReplicationZookeeper(server, new AtomicBoolean(true));
+    }
+
+    @Override
+    public void run() {
+      try {
+        logZnodesMap = rz.copyQueuesFromRSUsingMulti(deadRsZnode);
+        server.abort("Done with testing", null);
+      } catch (Exception e) {
+        LOG.error("Got exception while running NodeFailoverWorker", e);
+      } finally {
+        latch.countDown();
+      }
+    }
 
+    /**
+     * @return 1 when the map is not empty.
+     */
+    private int isLogZnodesMapPopulated() {
+      Collection<SortedSet<String>> sets = logZnodesMap.values();
+      if (sets.size() > 1) {
+        throw new RuntimeException("unexpected size of logZnodesMap: " + sets.size());
+      }
+      if (sets.size() == 1) {
+        SortedSet<String> s = sets.iterator().next();
+        for (String file : files) {
+          // at least one file was missing
+          if (!s.contains(file)) {
+            return 0;
+          }
+        }
+        return 1; // we found all the files
+      }
+      return 0;
+    }
+  }
+  
   static class DummyServer implements Server {
+    String hostname;
+
+    DummyServer() {
+      hostname = "hostname.example.org";
+    }
+
+    DummyServer(String hostname) {
+      this.hostname = hostname;
+    }
 
     @Override
     public Configuration getConfiguration() {
@@ -228,19 +333,19 @@ public ZooKeeperWatcher getZooKeeper() {
 
     @Override
     public CatalogTracker getCatalogTracker() {
-      return null;  //To change body of implemented methods use File | Settings | File Templates.
+      return null; // To change body of implemented methods use File | Settings | File Templates.
     }
 
     @Override
     public ServerName getServerName() {
-      return new ServerName("hostname.example.org", 1234, -1L);
+      return new ServerName(hostname, 1234, -1L);
     }
 
     @Override
     public void abort(String why, Throwable e) {
-      //To change body of implemented methods use File | Settings | File Templates.
+      // To change body of implemented methods use File | Settings | File Templates.
     }
-    
+
     @Override
     public boolean isAborted() {
       return false;

From 2c684ee552b30cff0c72037f1639681851beadbd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 17 Mar 2013 04:09:31 +0000
Subject: [PATCH 0857/1540] HBASE-8125 HBASE-7435 breaks
 BuiltInGzipDecompressor on Hadoop < 1.0.x (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1457370 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/hfile/Compression.java | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
index 3c06af19d294..186bfc330992 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.lang.annotation.Annotation;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -33,7 +34,6 @@
 import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.Decompressor;
-import org.apache.hadoop.io.compress.DoNotPool;
 import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -349,8 +349,13 @@ public Decompressor getDecompressor() {
     public void returnDecompressor(Decompressor decompressor) {
       if (decompressor != null) {
         CodecPool.returnDecompressor(decompressor);
-        if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
-          decompressor.end();
+        Annotation[] annotations = decompressor.getClass().getAnnotations();
+        if (annotations != null) {
+          for (Annotation annotation : annotations) {
+            if (annotation.getClass().getSimpleName().equals("DoNotPool")) {
+              decompressor.end();              
+            }
+          }
         }
       }
     }

From 9230c45c7e637296f05c0f7d4c56f8198e4870cf Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sun, 17 Mar 2013 15:18:59 +0000
Subject: [PATCH 0858/1540] HBASE-8067
 TestHFileArchiving.testArchiveOnTableDelete sometimes fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1457461 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/backup/TestHFileArchiving.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index c2b3d76f4de8..73912a703d38 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -78,6 +78,9 @@ public class TestHFileArchiving {
   public static void setupCluster() throws Exception {
     setupConf(UTIL.getConfiguration());
     UTIL.startMiniCluster();
+
+    // We don't want the cleaner to remove files. The tests do that.
+    UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
   }
 
   private static void setupConf(Configuration conf) {

From 9df94d4a8226832375e4e6e0de34421b8fdd7d14 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Mon, 18 Mar 2013 19:44:53 +0000
Subject: [PATCH 0859/1540] HBASE-8128 HTable#put improvements

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1457939 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTable.java    | 32 +++++++++----------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index bc5f6e3caa9b..52deff7ae9ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -114,7 +114,6 @@ public class HTable implements HTableInterface {
   private ExecutorService pool;  // For Multi
   private boolean closed;
   private int operationTimeout;
-  private static final int DOPUT_WB_CHECK = 10;    // i.e., doPut checks the writebuffer every X Puts.
   private final boolean cleanupPoolOnClose; // shutdown the pool in close()
   private final boolean cleanupConnectionOnClose; // close the connection in close()
 
@@ -746,7 +745,10 @@ public void delete(final List<Delete> deletes)
    */
   @Override
   public void put(final Put put) throws IOException {
-    doPut(Arrays.asList(put));
+    doPut(put);
+    if (autoFlush) {
+      flushCommits();
+    }
   }
 
   /**
@@ -754,23 +756,19 @@ public void put(final Put put) throws IOException {
    */
   @Override
   public void put(final List<Put> puts) throws IOException {
-    doPut(puts);
-  }
-
-  private void doPut(final List<Put> puts) throws IOException {
-    int n = 0;
     for (Put put : puts) {
-      validatePut(put);
-      writeBuffer.add(put);
-      currentWriteBufferSize += put.heapSize();
-     
-      // we need to periodically see if the writebuffer is full instead of waiting until the end of the List
-      n++;
-      if (n % DOPUT_WB_CHECK == 0 && currentWriteBufferSize > writeBufferSize) {
-        flushCommits();
-      }
+      doPut(put);
+    }
+    if (autoFlush) {
+      flushCommits();
     }
-    if (autoFlush || currentWriteBufferSize > writeBufferSize) {
+  }
+
+  private void doPut(Put put) throws IOException{
+    validatePut(put);
+    writeBuffer.add(put);
+    currentWriteBufferSize += put.heapSize();
+    if (currentWriteBufferSize > writeBufferSize) {
       flushCommits();
     }
   }

From e4d9f448380b036ec1c3c57fbdb5b9500ae25e77 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 18 Mar 2013 21:05:06 +0000
Subject: [PATCH 0860/1540] HBASE-8014 Backport HBASE-6915 to 0.94 (Jean-Marc)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1457994 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/ClassSize.java    | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
index e18473067d59..4d6cdfbb8603 100755
--- a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
@@ -97,6 +97,20 @@ public class ClassSize {
   /** Overhead for CopyOnWriteArrayList */
   public static final int COPYONWRITE_ARRAYLIST;
 
+  /* Are we running on jdk7? */
+  private static final boolean JDK7;
+  static {
+    final String version = System.getProperty("java.version");
+    // Verify String looks like this: 1.6.0_29
+    if (!version.matches("\\d\\.\\d\\..*")) {
+      throw new RuntimeException("Unexpected version format: " + version);
+    }
+    // Convert char to int
+    int major = (int) (version.charAt(0) - '0');
+    int minor = (int) (version.charAt(2) - '0');
+    JDK7 = major == 1 && minor == 7;
+  }
+
   /**
    * Method for reading the arc settings and setting overheads according
    * to 32-bit or 64-bit architecture.
@@ -127,9 +141,9 @@ public class ClassSize {
 
     TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + align(7 * REFERENCE));
 
-    STRING = align(OBJECT + ARRAY + REFERENCE + 3 * Bytes.SIZEOF_INT);
+    STRING = align(OBJECT + ARRAY + REFERENCE + ((JDK7? 2: 3) * Bytes.SIZEOF_INT));
 
-    CONCURRENT_HASHMAP = align((2 * Bytes.SIZEOF_INT) + ARRAY +
+    CONCURRENT_HASHMAP = align(((JDK7? 3: 2) * Bytes.SIZEOF_INT) + ARRAY +
         (6 * REFERENCE) + OBJECT);
 
     CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) +

From bccabcfd9eafd6c8e81e782902cc52e993939d13 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 18 Mar 2013 21:08:15 +0000
Subject: [PATCH 0861/1540] HBASE-8141. Remove accidental uses of
 org.mortbay.log.Log

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1457997 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java     | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
index 8798999b2216..2160e3e5b275 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
@@ -32,6 +32,8 @@
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -44,7 +46,6 @@
 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
 import org.apache.hadoop.io.BytesWritable;
 import org.junit.experimental.categories.Category;
-import org.mortbay.log.Log;
 
 /**
  * test the performance for seek.
@@ -56,6 +57,7 @@
  */
 @Category(MediumTests.class)
 public class TestHFileSeek extends TestCase {
+  private static final Log LOG = LogFactory.getLog(TestHFileSeek.class);
   private static final boolean USE_PREAD = true;
   private MyOptions options;
   private Configuration conf;
@@ -529,7 +531,7 @@ public static void main(String[] argv) throws IOException {
 
     testCase.options = options;
     for (int i = 0; i < options.trialCount; i++) {
-      Log.info("Beginning trial " + (i+1));
+      LOG.info("Beginning trial " + (i+1));
       testCase.setUp();
       testCase.testSeeks();
       testCase.tearDown();

From 8e664d8133009fb14c4d3bd4101971bcbe80a1fe Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 19 Mar 2013 18:06:41 +0000
Subject: [PATCH 0862/1540] HBASE-8125 HBASE-7435 breaks
 BuiltInGzipDecompressor on Hadoop < 1.0.x, real fix (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1458439 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/io/hfile/Compression.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
index 186bfc330992..08bd9032e4b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
@@ -352,7 +352,8 @@ public void returnDecompressor(Decompressor decompressor) {
         Annotation[] annotations = decompressor.getClass().getAnnotations();
         if (annotations != null) {
           for (Annotation annotation : annotations) {
-            if (annotation.getClass().getSimpleName().equals("DoNotPool")) {
+            String annoName = annotation.annotationType().getSimpleName();
+            if (annoName.equals("DoNotPool")) {
               decompressor.end();              
             }
           }

From 3a9e36262af23f84783d557cc54f4024d5cca405 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 20 Mar 2013 00:15:39 +0000
Subject: [PATCH 0863/1540] HBASE-8142 Sporadic TestZKProcedureController
 failures (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1458582 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java    | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 2bc3e66188bc..041f4ae3ab52 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -856,9 +856,6 @@ public static boolean setData(ZooKeeperWatcher zkw, String znode,
    * Set data into node creating node if it doesn't yet exist.
    * Does not set watch.
    *
-   * WARNING: this is not atomic -- it is possible to get a 0-byte data value in the znode before
-   * data is written
-   *
    * @param zkw zk reference
    * @param znode path of node
    * @param data data to set for node
@@ -868,9 +865,10 @@ public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
       final byte [] data)
   throws KeeperException {
     if (checkExists(zkw, znode) == -1) {
-      ZKUtil.createWithParents(zkw, znode);
+      ZKUtil.createWithParents(zkw, znode, data);
+    } else {
+      ZKUtil.setData(zkw, znode, data);
     }
-    ZKUtil.setData(zkw, znode, data);
   }
 
   /**

From f6c2ff88ec7c98da7d76e770be5a634469b90bfa Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 20 Mar 2013 16:53:16 +0000
Subject: [PATCH 0864/1540] HBASE-8131 - Create table handler needs to handle
 failure cases.(Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1458942 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/handler/CreateTableHandler.java    |  23 ++--
 .../hadoop/hbase/zookeeper/ZKTable.java       |  14 +++
 .../handler/TestCreateTableHandler.java       | 116 ++++++++++++++++++
 3 files changed, 140 insertions(+), 13 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 717880de4871..7e6d5bea3127 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -20,17 +20,7 @@
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -51,10 +41,8 @@
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -148,6 +136,15 @@ public void process() {
    * @param exception null if process() is successful or not null if something has failed.
    */
   protected void completed(final Throwable exception) {
+    // Try deleting the enabling node
+    // If this does not happen then if the client tries to create the table
+    // again with the same Active master
+    // It will block the creation saying TableAlreadyExists.
+    if (exception != null) {
+      this.assignmentManager.getZKTable().removeEnablingTable(
+          this.hTableDescriptor.getNameAsString());
+    }
+
   }
 
   /**
@@ -181,7 +178,7 @@ private void handleCreateTable(String tableName) throws IOException, KeeperExcep
       throw new IOException("Unable to move table from temp=" + tempTableDir +
         " to hbase root=" + tableDir);
     }
-
+    
     if (regionInfos != null && regionInfos.size() > 0) {
       // 4. Add regions to META
       MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 41cd25fd7352..6e66b70a581f 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -177,6 +177,20 @@ public boolean checkAndSetEnablingTable(final String tableName)
       return true;
     }
   }
+  
+  /**
+   * If the table is found in ENABLING state the inmemory state is removed.
+   * This helps in cases where CreateTable is to be retried by the client incase of failures
+   * @param tableName
+   */
+  public void removeEnablingTable(final String tableName) {
+    synchronized (this.cache) {
+      if (isEnablingTable(tableName)) {
+        this.cache.remove(tableName);
+      }
+
+    }
+  }
 
   /**
    * Sets the specified table as ENABLING in zookeeper atomically
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
new file mode 100644
index 000000000000..8c0624c269d0
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
@@ -0,0 +1,116 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.TestMaster;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestCreateTableHandler {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class);
+  private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler");
+  private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
+  public static boolean throwException = false;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() throws Exception {
+    final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    final HMaster m = cluster.getMaster();
+    final HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+    desc.addFamily(new HColumnDescriptor(FAMILYNAME));
+    final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null,
+        null) };
+    CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(),
+        m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos,
+        m.getCatalogTracker(), m.getAssignmentManager());
+    throwException = true;
+    handler.process();
+    throwException = false;
+    CustomCreateTableHandler handler1 = new CustomCreateTableHandler(m, m.getMasterFileSystem(),
+        m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos,
+        m.getCatalogTracker(), m.getAssignmentManager());
+    handler1.process();
+    for (int i = 0; i < 100; i++) {
+      if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(TABLENAME)) {
+        Thread.sleep(200);
+      }
+    }
+    assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME));
+
+  }
+
+  private static class CustomCreateTableHandler extends CreateTableHandler {
+    public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager,
+        ServerManager sm, HTableDescriptor hTableDescriptor, Configuration conf,
+        HRegionInfo[] newRegions, CatalogTracker ct, AssignmentManager am)
+        throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+      super(server, fileSystemManager, sm, hTableDescriptor, conf, newRegions, ct, am);
+    }
+
+    @Override
+    protected List<HRegionInfo> handleCreateHdfsRegions(Path tableRootDir, String tableName)
+        throws IOException {
+      if (throwException) {
+        throw new IOException("Test throws exceptions.");
+      }
+      return super.handleCreateHdfsRegions(tableRootDir, tableName);
+    }
+  }
+}

From e909613d4359ff8b0c89d6fdc15a8f2be83a67ef Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 20 Mar 2013 21:26:10 +0000
Subject: [PATCH 0865/1540] HBASE-8146 IntegrationTestBigLinkedList does not
 work on distributed setup

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1459065 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java  | 1 +
 .../hadoop/hbase/test/IntegrationTestBigLinkedList.java    | 7 +++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index e49525920687..53df3bd18929 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -514,6 +514,7 @@ public static void addDependencyJars(Job job) throws IOException {
           org.apache.zookeeper.ZooKeeper.class,
           com.google.protobuf.Message.class,
           com.google.common.collect.ImmutableSet.class,
+          org.apache.hadoop.hbase.util.Bytes.class, //one class from hbase.jar
           job.getMapOutputKeyClass(),
           job.getMapOutputValueClass(),
           job.getInputFormatClass(),
diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 0f4aa4ceae1d..be522185450e 100644
--- a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.IntegrationTestingUtility;
@@ -150,7 +151,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool {
 
   private static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
 
-  private static final String DEFAULT_TABLE_NAME = "ci";
+  private static final String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList";
 
   private static byte[] FAMILY_NAME = Bytes.toBytes("meta");
 
@@ -467,6 +468,8 @@ public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Ex
       job.setOutputFormatClass(NullOutputFormat.class);
 
       job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+      TableMapReduceUtil.addDependencyJars(job);
+      TableMapReduceUtil.initCredentials(job);
 
       boolean success = job.waitForCompletion(true);
 
@@ -1004,7 +1007,7 @@ public int run(String[] args) throws Exception {
   }
 
   public static void main(String[] args) throws Exception {
-    int ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
+    int ret = ToolRunner.run(HBaseConfiguration.create(), new IntegrationTestBigLinkedList(), args);
     System.exit(ret);
   }
 }

From 03da4de1b26b3e0a2fab1965f20b534c66835f67 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 21 Mar 2013 18:11:59 +0000
Subject: [PATCH 0866/1540] HBASE-8151 Decode memstoreTS in HFileReaderV2 only
 when necessary

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1459431 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  | 48 ++++++++++++-------
 1 file changed, 31 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 093d8567e062..f5f1f9b61fdf 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -54,6 +54,7 @@ public class HFileReaderV2 extends AbstractHFileReader {
   private static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
 
   private boolean includesMemstoreTS = false;
+  private boolean decodeMemstoreTS = false;
 
   private boolean shouldIncludeMemstoreTS() {
     return includesMemstoreTS;
@@ -145,6 +146,9 @@ public HFileReaderV2(Path path, FixedFileTrailer trailer,
         Bytes.toInt(keyValueFormatVersion) ==
             HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
     fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
+    if (includesMemstoreTS) {
+      decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
+    }
 
     // Read data block encoding algorithm name from file info.
     dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo,
@@ -785,15 +789,20 @@ private final void readKeyValueLen() {
       currValueLen = blockBuffer.getInt();
       blockBuffer.reset();
       if (this.reader.shouldIncludeMemstoreTS()) {
-        try {
-          int memstoreTSOffset = blockBuffer.arrayOffset()
-              + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen
-              + currValueLen;
-          currMemstoreTS = Bytes.readVLong(blockBuffer.array(),
-              memstoreTSOffset);
-          currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
-        } catch (Exception e) {
-          throw new RuntimeException("Error reading memstore timestamp", e);
+        if (this.reader.decodeMemstoreTS) {
+          try {
+            int memstoreTSOffset = blockBuffer.arrayOffset()
+                + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen
+                + currValueLen;
+            currMemstoreTS = Bytes.readVLong(blockBuffer.array(),
+                memstoreTSOffset);
+            currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
+          } catch (Exception e) {
+            throw new RuntimeException("Error reading memstore timestamp", e);
+          }
+        } else {
+          currMemstoreTS = 0;
+          currMemstoreTSLen = 1;
         }
       }
 
@@ -832,14 +841,19 @@ private int blockSeek(byte[] key, int offset, int length,
         vlen = blockBuffer.getInt();
         blockBuffer.reset();
         if (this.reader.shouldIncludeMemstoreTS()) {
-          try {
-            int memstoreTSOffset = blockBuffer.arrayOffset()
-                + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen;
-            memstoreTS = Bytes.readVLong(blockBuffer.array(),
-                memstoreTSOffset);
-            memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
-          } catch (Exception e) {
-            throw new RuntimeException("Error reading memstore timestamp", e);
+          if (this.reader.decodeMemstoreTS) {
+            try {
+              int memstoreTSOffset = blockBuffer.arrayOffset()
+                  + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen;
+              memstoreTS = Bytes.readVLong(blockBuffer.array(),
+                  memstoreTSOffset);
+              memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
+            } catch (Exception e) {
+              throw new RuntimeException("Error reading memstore timestamp", e);
+            }
+          } else {
+            memstoreTS = 0;
+            memstoreTSLen = 1;
           }
         }
 

From 408f8da5dd0d6cc7c55a6db8df72749b8c664ced Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 21 Mar 2013 23:29:21 +0000
Subject: [PATCH 0867/1540] HBASE-8160 HMaster#move doesn't check if master
 initialized

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1459583 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   | 11 +++++-----
 .../hadoop/hbase/master/TestMaster.java       | 20 ++++++++++++++++---
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c414adfea451..3b4293750a73 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -47,7 +47,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ClusterStatus;
@@ -108,7 +107,6 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.util.InfoServer;
@@ -205,8 +203,11 @@ public class HMaster extends HasThread
   private volatile boolean abort = false;
   // flag set after we become the active master (used for testing)
   private volatile boolean isActiveMaster = false;
-  // flag set after we complete initialization once active (used for testing)
-  private volatile boolean initialized = false;
+
+  // flag set after we complete initialization once active,
+  // it is not private since it's used in unit tests
+  volatile boolean initialized = false;
+
   // flag set after we complete assignRootAndMeta.
   private volatile boolean serverShutdownHandlerEnabled = false;
 
@@ -1186,6 +1187,7 @@ public void move(final byte[] encodedRegionName, final byte[] destServerName)
     RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
     
     try {
+      checkInitialized();
       if (this.cpHost != null) {
         if (this.cpHost.preMove(p.getFirst(), p.getSecond(), dest)) {
           return;
@@ -1202,7 +1204,6 @@ public void move(final byte[] encodedRegionName, final byte[] destServerName)
       ure.initCause(ioe);
       throw ure;
     }
-
   }
 
   public void createTable(HTableDescriptor hTableDescriptor,
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index 3122b1592247..65b4d5962ad7 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -23,7 +23,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.catalog.MetaReader;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
@@ -55,7 +54,7 @@ public class TestMaster {
   @BeforeClass
   public static void beforeAllTests() throws Exception {
     // Start a cluster of two regionservers.
-    TEST_UTIL.startMiniCluster(1);
+    TEST_UTIL.startMiniCluster(2);
   }
 
   @AfterClass
@@ -116,6 +115,22 @@ public void testMasterOpsWhileSplitting() throws Exception {
     }
   }
 
+  @Test
+  public void testMoveRegionWhenNotInitialized() {
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    HMaster m = cluster.getMaster();
+    try {
+      m.initialized = false; // fake it, set back later
+      HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
+      m.move(meta.getEncodedNameAsBytes(), null);
+      fail("Region should not be moved since master is not initialized");
+    } catch (IOException ioe) {
+      assertTrue(ioe.getCause() instanceof PleaseHoldException);
+    } finally {
+      m.initialized = true;
+    }
+  }
+
   static class RegionSplitListener implements EventHandlerListener {
     CountDownLatch split, proceed;
 
@@ -147,4 +162,3 @@ public void beforeProcess(EventHandler event) {
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-

From bb4900da1e1ecbf34e50e428247edcf66f0c3eed Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 21 Mar 2013 23:55:38 +0000
Subject: [PATCH 0868/1540] HBASE-8148  Allow IPC to bind on a specific address

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1459590 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/HMaster.java  | 16 ++++++++++++----
 .../hadoop/hbase/regionserver/HRegionServer.java |  7 ++++---
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3b4293750a73..bb19dc6d97b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -274,16 +274,24 @@ public HMaster(final Configuration conf)
       conf.get("hbase.master.dns.interface", "default"),
       conf.get("hbase.master.dns.nameserver", "default")));
     int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
-    // Creation of a HSA will force a resolve.
+    // Test that the hostname is reachable
     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
     if (initialIsa.getAddress() == null) {
-      throw new IllegalArgumentException("Failed resolve of " + initialIsa);
+      throw new IllegalArgumentException("Failed resolve of hostname " + initialIsa);
+    }
+    // Verify that the bind address is reachable if set
+    String bindAddress = conf.get("hbase.master.ipc.address");
+    if (bindAddress != null) {
+      initialIsa = new InetSocketAddress(bindAddress, port);
+      if (initialIsa.getAddress() == null) {
+        throw new IllegalArgumentException("Failed resolve of bind address " + initialIsa);
+      }
     }
     int numHandlers = conf.getInt("hbase.master.handler.count",
       conf.getInt("hbase.regionserver.handler.count", 25));
     this.rpcServer = HBaseRPC.getServer(this,
       new Class<?>[]{HMasterInterface.class, HMasterRegionInterface.class},
-        initialIsa.getHostName(), // BindAddress is IP we got for this server.
+        initialIsa.getHostName(), // This is bindAddress if set else it's hostname
         initialIsa.getPort(),
         numHandlers,
         0, // we dont use high priority handlers in master
@@ -291,7 +299,7 @@ public HMaster(final Configuration conf)
         0); // this is a DNC w/o high priority handlers
     // Set our address.
     this.isa = this.rpcServer.getListenerAddress();
-    this.serverName = new ServerName(this.isa.getHostName(),
+    this.serverName = new ServerName(hostname,
       this.isa.getPort(), System.currentTimeMillis());
     this.rsFatals = new MemoryBoundedLogMessageBuffer(
         conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 455da6f5cc0c..1d42fea22ce9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -423,9 +423,10 @@ public HRegionServer(Configuration conf)
     this.stopped = false;
 
     // Server to handle client requests.
-    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
-      conf.get("hbase.regionserver.dns.interface", "default"),
-      conf.get("hbase.regionserver.dns.nameserver", "default")));
+    String hostname = conf.get("hbase.regionserver.ipc.address",
+      Strings.domainNamePointerToHostName(DNS.getDefaultHost(
+        conf.get("hbase.regionserver.dns.interface", "default"),
+        conf.get("hbase.regionserver.dns.nameserver", "default"))));
     int port = conf.getInt(HConstants.REGIONSERVER_PORT,
       HConstants.DEFAULT_REGIONSERVER_PORT);
     // Creation of a HSA will force a resolve.

From 1edafac9c24751ca03da795d806090f40242a3d0 Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Fri, 22 Mar 2013 14:24:11 +0000
Subject: [PATCH 0869/1540] HBASE-8030 znode  path of online region servers is
 hard coded in rolling_restart.sh(rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1459796 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/rolling-restart.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index e7329feb3c31..23f8d325e81f 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -149,7 +149,10 @@ else
 
   if [ $RR_GRACEFUL -eq 1 ]; then
     # gracefully restart all online regionservers
-    online_regionservers=`$bin/hbase zkcli ls $zparent/rs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"`
+    zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs`
+    if [ "$zkrs" == "null" ]; then zkrs="rs"; fi
+    zkrs="$zparent/$zkrs"
+    online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"`
     for rs in $online_regionservers
     do
         rs_parts=(${rs//,/ })

From 3b61db3276ef9972389912dbda652893be92b8f1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 22 Mar 2013 20:02:51 +0000
Subject: [PATCH 0870/1540] HBASE-8166 Avoid writing the memstoreTS into HFiles
 when possible

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1459961 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/hfile/HFile.java | 12 +++++++++---
 .../hadoop/hbase/io/hfile/HFileWriterV1.java    |  2 +-
 .../hadoop/hbase/io/hfile/HFileWriterV2.java    | 11 ++++++-----
 .../hadoop/hbase/regionserver/Compactor.java    | 14 ++++++++++++--
 .../apache/hadoop/hbase/regionserver/Store.java |  5 +++--
 .../hadoop/hbase/regionserver/StoreFile.java    | 17 ++++++++++++++---
 .../regionserver/TestCacheOnWriteInSchema.java  |  2 +-
 7 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index c99caba35a0c..002a69a85930 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -323,6 +323,7 @@ public static abstract class WriterFactory {
     protected KeyComparator comparator;
     protected ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
     protected int bytesPerChecksum = DEFAULT_BYTES_PER_CHECKSUM;
+    protected boolean includeMVCCReadpoint = true;
 
     WriterFactory(Configuration conf, CacheConfig cacheConf) {
       this.conf = conf;
@@ -383,6 +384,11 @@ public WriterFactory withBytesPerChecksum(int bytesPerChecksum) {
       return this;
     }
 
+    public WriterFactory includeMVCCReadpoint(boolean includeMVCCReadpoint) {
+      this.includeMVCCReadpoint = includeMVCCReadpoint;
+      return this;
+    }
+
     public Writer create() throws IOException {
       if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
         throw new AssertionError("Please specify exactly one of " +
@@ -391,8 +397,8 @@ public Writer create() throws IOException {
       if (path != null) {
         ostream = AbstractHFileWriter.createOutputStream(conf, fs, path);
       }
-      return createWriter(fs, path, ostream, blockSize,
-          compression, encoder, comparator, checksumType, bytesPerChecksum);
+      return createWriter(fs, path, ostream, blockSize, compression, encoder, comparator,
+          checksumType, bytesPerChecksum, includeMVCCReadpoint);
     }
 
     protected abstract Writer createWriter(FileSystem fs, Path path,
@@ -400,7 +406,7 @@ protected abstract Writer createWriter(FileSystem fs, Path path,
         Compression.Algorithm compress,
         HFileDataBlockEncoder dataBlockEncoder,
         KeyComparator comparator, ChecksumType checksumType,
-        int bytesPerChecksum) throws IOException;
+        int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException;
   }
 
   /** The configuration key for HFile version to use for new files */
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
index ae276c42944a..ed0a1ba28662 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
@@ -92,7 +92,7 @@ public Writer createWriter(FileSystem fs, Path path,
         FSDataOutputStream ostream, int blockSize,
         Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder,
         KeyComparator comparator, final ChecksumType checksumType,
-        final int bytesPerChecksum) throws IOException {
+        final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException {
       // version 1 does not implement checksums
       return new HFileWriterV1(conf, cacheConf, fs, path, ostream, blockSize,
           compressAlgo, dataBlockEncoder, comparator);
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index a299b3cde4f3..ceb474519ec9 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -86,7 +86,7 @@ public class HFileWriterV2 extends AbstractHFileWriter {
   private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
   private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
 
-  private final boolean includeMemstoreTS = true;
+  private final boolean includeMemstoreTS;
   private long maxMemstoreTS = 0;
 
   private int minorVersion = HFileReaderV2.MAX_MINOR_VERSION;
@@ -101,9 +101,9 @@ public Writer createWriter(FileSystem fs, Path path,
         FSDataOutputStream ostream, int blockSize,
         Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder,
         final KeyComparator comparator, final ChecksumType checksumType,
-        final int bytesPerChecksum) throws IOException {
-      return new HFileWriterV2(conf, cacheConf, fs, path, ostream, blockSize,
-          compress, blockEncoder, comparator, checksumType, bytesPerChecksum);
+        final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException {
+      return new HFileWriterV2(conf, cacheConf, fs, path, ostream, blockSize, compress,
+          blockEncoder, comparator, checksumType, bytesPerChecksum, includeMVCCReadpoint);
     }
   }
 
@@ -112,13 +112,14 @@ public HFileWriterV2(Configuration conf, CacheConfig cacheConf,
       FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize,
       Compression.Algorithm compressAlgo, HFileDataBlockEncoder blockEncoder,
       final KeyComparator comparator, final ChecksumType checksumType,
-      final int bytesPerChecksum) throws IOException {
+      final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException {
     super(cacheConf,
         ostream == null ? createOutputStream(conf, fs, path) : ostream,
         path, blockSize, compressAlgo, blockEncoder, comparator);
     SchemaMetrics.configureGlobally(conf);
     this.checksumType = checksumType;
     this.bytesPerChecksum = bytesPerChecksum;
+    this.includeMemstoreTS = includeMVCCReadpoint;
     if (!conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, false)) {
       this.minorVersion = 0;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
index 900a137a6aa8..50ef8b7d8395 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -32,6 +33,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -84,6 +86,7 @@ StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOExcepti
     // Also calculate earliest put timestamp if major compaction
     int maxKeyCount = 0;
     long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+    long maxMVCCReadpoint = 0;
 
     // pull out the interesting things from the CR for ease later
     final Store store = request.getStore();
@@ -102,11 +105,17 @@ StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOExcepti
           .getBloomFilterType()) ?
           r.getFilterEntries() : r.getEntries();
       maxKeyCount += keyCount;
+      // Calculate the maximum MVCC readpoint used in any of the involved files
+      Map<byte[], byte[]> fileInfo = r.loadFileInfo();
+      byte[] tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
+      if (tmp != null) {
+        maxMVCCReadpoint = Math.max(maxMVCCReadpoint, Bytes.toLong(tmp));
+      }
       // For major compactions calculate the earliest put timestamp
       // of all involved storefiles. This is used to remove 
       // family delete marker during the compaction.
       if (majorCompaction) {
-        byte[] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
+        tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
         if (tmp == null) {
           // There's a file with no information, must be an old one
           // assume we have very old puts
@@ -183,7 +192,8 @@ StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOExcepti
         do {
           hasMore = scanner.next(kvs, compactionKVMax);
           if (writer == null && !kvs.isEmpty()) {
-            writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true);
+            writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true,
+                maxMVCCReadpoint >= smallestReadPoint);
           }
           if (writer != null) {
             // output to writer:
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 90f7fd010ddc..f357583c6097 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -901,7 +901,7 @@ private StoreFile commitFile(final Path path,
    */
   private StoreFile.Writer createWriterInTmp(int maxKeyCount)
   throws IOException {
-    return createWriterInTmp(maxKeyCount, this.family.getCompression(), false);
+    return createWriterInTmp(maxKeyCount, this.family.getCompression(), false, true);
   }
 
   /*
@@ -911,7 +911,7 @@ private StoreFile.Writer createWriterInTmp(int maxKeyCount)
    * @return Writer for a new StoreFile in the tmp dir.
    */
   public StoreFile.Writer createWriterInTmp(int maxKeyCount,
-    Compression.Algorithm compression, boolean isCompaction)
+    Compression.Algorithm compression, boolean isCompaction, boolean includeMVCCReadpoint)
   throws IOException {
     final CacheConfig writerCacheConf;
     if (isCompaction) {
@@ -931,6 +931,7 @@ public StoreFile.Writer createWriterInTmp(int maxKeyCount,
             .withChecksumType(checksumType)
             .withBytesPerChecksum(bytesPerChecksum)
             .withCompression(compression)
+            .includeMVCCReadpoint(includeMVCCReadpoint)
             .build();
     // The store file writer's path does not include the CF name, so we need
     // to configure the HFile writer directly.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index bc0bd9d7ac5d..96c693490efa 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -43,7 +43,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.client.Scan;
@@ -741,6 +740,7 @@ public static class WriterBuilder {
     private Path filePath;
     private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
     private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
+    private boolean includeMVCCReadpoint = true;
 
     public WriterBuilder(Configuration conf, CacheConfig cacheConf,
         FileSystem fs, int blockSize) {
@@ -825,6 +825,15 @@ public WriterBuilder withBytesPerChecksum(int bytesPerChecksum) {
       return this;
     }
 
+    /**
+     * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV
+     * @return this (for chained invocation)
+     */
+    public WriterBuilder includeMVCCReadpoint(boolean includeMVCCReadpoint) {
+      this.includeMVCCReadpoint = includeMVCCReadpoint;
+      return this;
+    }
+
     /**
      * Create a store file writer. Client is responsible for closing file when
      * done. If metadata, add BEFORE closing using
@@ -859,7 +868,7 @@ public Writer build() throws IOException {
       }
       return new Writer(fs, filePath, blockSize, compressAlgo, dataBlockEncoder,
           conf, cacheConf, comparator, bloomType, maxKeyCount, checksumType,
-          bytesPerChecksum);
+          bytesPerChecksum, includeMVCCReadpoint);
     }
   }
 
@@ -998,6 +1007,7 @@ public static class Writer {
      *        for Bloom filter size in {@link HFile} format version 1.
      * @param checksumType the checksum type
      * @param bytesPerChecksum the number of bytes per checksum value
+     * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV
      * @throws IOException problem writing to FS
      */
     private Writer(FileSystem fs, Path path, int blocksize,
@@ -1005,7 +1015,7 @@ private Writer(FileSystem fs, Path path, int blocksize,
         HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
         CacheConfig cacheConf,
         final KVComparator comparator, BloomType bloomType, long maxKeys,
-        final ChecksumType checksumType, final int bytesPerChecksum)
+        final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint)
         throws IOException {
       this.dataBlockEncoder = dataBlockEncoder != null ?
           dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
@@ -1017,6 +1027,7 @@ private Writer(FileSystem fs, Path path, int blocksize,
           .withComparator(comparator.getRawComparator())
           .withChecksumType(checksumType)
           .withBytesPerChecksum(bytesPerChecksum)
+          .includeMVCCReadpoint(includeMVCCReadpoint)
           .create();
 
       this.kvComparator = comparator;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
index f230b1ea96bb..863b97ce7000 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
@@ -181,7 +181,7 @@ public void tearDown() {
   public void testCacheOnWriteInSchema() throws IOException {
     // Write some random data into the store
     StoreFile.Writer writer = store.createWriterInTmp(Integer.MAX_VALUE,
-        Compression.Algorithm.NONE, false);
+        Compression.Algorithm.NONE, false, true);
     writeStoreFile(writer);
     writer.close();
     // Verify the block types of interest were cached on write

From e51b1ac36ace669ab95dd220ab67e6915e09dbff Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Tue, 26 Mar 2013 16:46:29 +0000
Subject: [PATCH 0871/1540] HBASE-8170 HbaseAdmin.createTable cannot handle
 creating three regions(jmspaggi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1461223 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       |  4 ++
 .../apache/hadoop/hbase/client/TestAdmin.java | 49 +++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index d69007afc752..3410b81f9a95 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -381,6 +381,10 @@ public void createTable(HTableDescriptor desc, byte [] startKey,
     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
       throw new IllegalArgumentException("Start key must be smaller than end key");
     }
+    if (numRegions == 3) {
+      createTable(desc, new byte[][] { startKey, endKey });
+      return;
+    }
     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
     if(splitKeys == null || splitKeys.length != numRegions - 1) {
       throw new IllegalArgumentException("Unable to split key range into enough regions");
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 66b5dd786d36..9ba1864b23f2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -547,6 +547,55 @@ protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) thro
     }
   }
 
+  @Test
+  public void testCreateTableNumberOfRegions() throws IOException, InterruptedException {
+    byte[] tableName = Bytes.toBytes("testCreateTableNumberOfRegions");
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+    admin.createTable(desc);
+    HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
+    assertEquals("Table should have only 1 region", 1, regions.size());
+    ht.close();
+
+    byte[] TABLE_2 = Bytes.add(tableName, Bytes.toBytes("_2"));
+    desc = new HTableDescriptor(TABLE_2);
+    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+    admin.createTable(desc, new byte[][] { new byte[] { 42 } });
+    HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
+    regions = ht2.getRegionLocations();
+    assertEquals("Table should have only 2 region", 2, regions.size());
+    ht2.close();
+
+    byte[] TABLE_3 = Bytes.add(tableName, Bytes.toBytes("_3"));
+    desc = new HTableDescriptor(TABLE_3);
+    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+    admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3);
+    HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
+    regions = ht3.getRegionLocations();
+    assertEquals("Table should have only 3 region", 3, regions.size());
+    ht3.close();
+
+    byte[] TABLE_4 = Bytes.add(tableName, Bytes.toBytes("_4"));
+    desc = new HTableDescriptor(TABLE_4);
+    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+    try {
+      admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2);
+      fail("Should not be able to create a table with only 2 regions using this API.");
+    } catch (IllegalArgumentException eae) {
+      // Expected
+    }
+
+    byte[] TABLE_5 = Bytes.add(tableName, Bytes.toBytes("_5"));
+    desc = new HTableDescriptor(TABLE_5);
+    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+    admin.createTable(desc, new byte[] { 1 }, new byte[] { 127 }, 16);
+    HTable ht5 = new HTable(TEST_UTIL.getConfiguration(), TABLE_5);
+    regions = ht5.getRegionLocations();
+    assertEquals("Table should have 16 region", 16, regions.size());
+    ht5.close();
+  }
+ 
   @Test
   public void testCreateTableWithRegions() throws IOException, InterruptedException {
 

From 378d6da2c8a79664551c7f6faaca7ba964c183d9 Mon Sep 17 00:00:00 2001
From: Devaraj Das <ddas@apache.org>
Date: Tue, 26 Mar 2013 21:00:11 +0000
Subject: [PATCH 0872/1540] HBASE-8081. Backport HBASE-7213 (separate hlog for
 meta tables) (Devaraj Das).

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1461314 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   |  15 ++-
 .../hadoop/hbase/master/MasterFileSystem.java |  73 ++++++++++-
 .../hadoop/hbase/master/MasterServices.java   |   5 +
 .../hadoop/hbase/master/SplitLogManager.java  |  25 +++-
 .../handler/MetaServerShutdownHandler.java    | 122 +++++++++++++++++-
 .../master/handler/ServerShutdownHandler.java | 105 +--------------
 .../hbase/regionserver/HRegionServer.java     | 109 +++++++++++++---
 .../hadoop/hbase/regionserver/LogRoller.java  |   9 +-
 .../hbase/regionserver/MetaLogRoller.java     |  38 ++++++
 .../regionserver/RegionServerServices.java    |   6 +-
 .../handler/OpenRegionHandler.java            |   6 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |  38 +++++-
 .../hbase/regionserver/wal/HLogSplitter.java  |   6 +
 src/main/resources/hbase-default.xml          |   4 +
 .../hbase/master/TestCatalogJanitor.java      |   5 +
 .../hbase/util/MockRegionServerServices.java  |   2 +-
 16 files changed, 430 insertions(+), 138 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bb19dc6d97b2..506118e33020 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -101,6 +101,7 @@
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
 import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
@@ -210,6 +211,8 @@ public class HMaster extends HasThread
 
   // flag set after we complete assignRootAndMeta.
   private volatile boolean serverShutdownHandlerEnabled = false;
+  // flag to indicate that we should be handling meta hlogs differently for splitting
+  private volatile boolean shouldSplitMetaSeparately;
 
   // Instance of the hbase executor service.
   ExecutorService executorService;
@@ -333,6 +336,7 @@ public HMaster(final Configuration conf)
     if (isHealthCheckerConfigured()) {
       healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
     }
+    this.shouldSplitMetaSeparately = conf.getBoolean(HLog.SEPARATE_HLOG_FOR_META, false);
   }
 
   /**
@@ -797,7 +801,12 @@ private void splitLogAndExpireIfOnline(final ServerName sn)
       return;
     }
     LOG.info("Forcing splitLog and expire of " + sn);
-    fileSystemManager.splitLog(sn);
+    if (this.shouldSplitMetaSeparately) {
+      fileSystemManager.splitMetaLog(sn);
+      fileSystemManager.splitLog(sn);
+    } else {
+      fileSystemManager.splitAllLogs(sn);  
+    }
     serverManager.expireServer(sn);
   }
 
@@ -1736,6 +1745,10 @@ public boolean isServerShutdownHandlerEnabled() {
     return this.serverShutdownHandlerEnabled;
   }
 
+  public boolean shouldSplitMetaSeparately() {
+    return this.shouldSplitMetaSeparately;
+  }
+
   @Override
   @Deprecated
   public void assign(final byte[] regionName, final boolean force)
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 54ce8c59ab5b..3f12660d7e9d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -83,6 +84,18 @@ public class MasterFileSystem {
   final SplitLogManager splitLogManager;
   private final MasterServices services;
 
+  private final static PathFilter META_FILTER = new PathFilter() {
+    public boolean accept(Path p) {
+      return HLog.isMetaFile(p);
+    }
+  };
+
+  private final static PathFilter NON_META_FILTER = new PathFilter() {
+    public boolean accept(Path p) {
+      return !HLog.isMetaFile(p);
+    }
+  };
+
   public MasterFileSystem(Server master, MasterServices services,
       MasterMetrics metrics, boolean masterRecovery)
   throws IOException {
@@ -238,7 +251,12 @@ void splitLogAfterStartup() {
                 + " belongs to an existing region server");
           }
         }
-        splitLog(serverNames);
+        if (services.shouldSplitMetaSeparately()) {
+          splitLog(serverNames, META_FILTER);
+          splitLog(serverNames, NON_META_FILTER);
+        } else {
+          splitAllLogs(serverNames);
+        }
         retrySplitting = false;
       } catch (IOException ioe) {
         LOG.warn("Failed splitting of " + serverNames, ioe);
@@ -267,8 +285,36 @@ public void splitLog(final ServerName serverName) throws IOException {
     splitLog(serverNames);
   }
 
-  public void splitLog(final List<ServerName> serverNames) throws IOException {
+  public void splitAllLogs(final ServerName serverName) throws IOException {
+    List<ServerName> serverNames = new ArrayList<ServerName>();
+    serverNames.add(serverName);
+    splitAllLogs(serverNames);
+  }
+
+  /**
+   * Specialized method to handle the splitting for meta HLog
+   * @param serverName
+   * @throws IOException
+   */
+  public void splitMetaLog(final ServerName serverName) throws IOException {
     long splitTime = 0, splitLogSize = 0;
+    List<ServerName> serverNames = new ArrayList<ServerName>();
+    serverNames.add(serverName);
+    List<Path> logDirs = getLogDirs(serverNames);
+    if (logDirs.isEmpty()) {
+      LOG.info("No meta logs to split");
+      return;
+    }
+    splitLogManager.handleDeadWorkers(serverNames);
+    splitTime = EnvironmentEdgeManager.currentTimeMillis();
+    splitLogSize = splitLogManager.splitLogDistributed(logDirs, META_FILTER);
+    splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
+    if (this.metrics != null) {
+      this.metrics.addSplit(splitTime, splitLogSize);
+    }
+  }
+
+  private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOException {
     List<Path> logDirs = new ArrayList<Path>();
     for(ServerName serverName: serverNames){
       Path logDir = new Path(this.rootdir,
@@ -287,6 +333,27 @@ public void splitLog(final List<ServerName> serverNames) throws IOException {
       }
       logDirs.add(splitDir);
     }
+    return logDirs;
+  }
+
+  public void splitLog(final List<ServerName> serverNames) throws IOException {
+    splitLog(serverNames, NON_META_FILTER);
+  }
+
+  public void splitAllLogs(final List<ServerName> serverNames) throws IOException {
+    splitLog(serverNames, null); //no filter
+  }
+
+  /**
+   * This method is the base split method that splits HLog files matching a filter.
+   * Callers should pass the appropriate filter for meta and non-meta HLogs.
+   * @param serverNames
+   * @param filter
+   * @throws IOException
+   */
+  public void splitLog(final List<ServerName> serverNames, PathFilter filter) throws IOException {
+    long splitTime = 0, splitLogSize = 0;
+    List<Path> logDirs = getLogDirs(serverNames);
 
     if (logDirs.isEmpty()) {
       LOG.info("No logs to split");
@@ -296,7 +363,7 @@ public void splitLog(final List<ServerName> serverNames) throws IOException {
     if (distributedLogSplitting) {
       splitLogManager.handleDeadWorkers(serverNames);
       splitTime = EnvironmentEdgeManager.currentTimeMillis();
-      splitLogSize = splitLogManager.splitLogDistributed(logDirs);
+      splitLogSize = splitLogManager.splitLogDistributed(logDirs,filter);
       splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
     } else {
       for(Path logDir: logDirs){
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index eb0e5400acd0..3eefdfcff2ce 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -135,6 +135,11 @@ public void deleteColumn(final byte[] tableName, final byte[] columnName)
    * @return true if master enables ServerShutdownHandler;
    */
   public boolean isServerShutdownHandlerEnabled();
+  
+  /**
+   * @return true if master thinks that meta hlogs should be split separately
+   */
+  public boolean shouldSplitMetaSeparately();
 
   /**
    * @return returns the master coprocessor host
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 02c96629ce18..b33aa43104b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
@@ -196,7 +197,7 @@ public void finishInitialization(boolean masterRecovery) {
     }
   }
 
-  private FileStatus[] getFileList(List<Path> logDirs) throws IOException {
+  private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws IOException {
     List<FileStatus> fileStatus = new ArrayList<FileStatus>();
     for (Path hLogDir : logDirs) {
       this.fs = hLogDir.getFileSystem(conf);
@@ -204,8 +205,7 @@ private FileStatus[] getFileList(List<Path> logDirs) throws IOException {
         LOG.warn(hLogDir + " doesn't exist. Nothing to do!");
         continue;
       }
-      // TODO filter filenames?
-      FileStatus[] logfiles = FSUtils.listStatus(fs, hLogDir, null);
+      FileStatus[] logfiles = FSUtils.listStatus(fs, hLogDir, filter);
       if (logfiles == null || logfiles.length == 0) {
         LOG.info(hLogDir + " is empty dir, no logs to split");
       } else {
@@ -230,6 +230,7 @@ public long splitLogDistributed(final Path logDir) throws IOException {
     logDirs.add(logDir);
     return splitLogDistributed(logDirs);
   }
+
   /**
    * The caller will block until all the log files of the given region server
    * have been processed - successfully split or an error is encountered - by an
@@ -242,9 +243,25 @@ public long splitLogDistributed(final Path logDir) throws IOException {
    * @return cumulative size of the logfiles split
    */
   public long splitLogDistributed(final List<Path> logDirs) throws IOException {
+    return splitLogDistributed(logDirs, null);
+  }
+
+  /**
+   * The caller will block until all the META log files of the given region server
+   * have been processed - successfully split or an error is encountered - by an
+   * available worker region server. This method must only be called after the
+   * region servers have been brought online.
+   *
+   * @param logDirs List of log dirs to split
+   * @param filter the Path filter to select specific files for considering
+   * @throws IOException If there was an error while splitting any log file
+   * @return cumulative size of the logfiles split
+   */
+  public long splitLogDistributed(final List<Path> logDirs, PathFilter filter) 
+      throws IOException {
     MonitoredTask status = TaskMonitor.get().createStatus(
           "Doing distributed log split in " + logDirs);
-    FileStatus[] logfiles = getFileList(logDirs);
+    FileStatus[] logfiles = getFileList(logDirs, filter);
     status.setStatus("Checking directory contents...");
     LOG.debug("Scheduling batch of logs to split");
     tot_mgr_log_split_batch_start.incrementAndGet();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
index 99d77e4ddb67..5c2ba7b00430 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
@@ -19,10 +19,16 @@
  */
 package org.apache.hadoop.hbase.master.handler;
 
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.master.DeadServer;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * Shutdown handler for the server hosting <code>-ROOT-</code>,
@@ -31,7 +37,7 @@
 public class MetaServerShutdownHandler extends ServerShutdownHandler {
   private final boolean carryingRoot;
   private final boolean carryingMeta;
-
+  private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
   public MetaServerShutdownHandler(final Server server,
       final MasterServices services,
       final DeadServer deadServers, final ServerName serverName,
@@ -43,11 +49,123 @@ public MetaServerShutdownHandler(final Server server,
   }
 
   @Override
+  public void process() throws IOException {
+    try {
+      if (this.shouldSplitHlog) {
+        if (this.services.shouldSplitMetaSeparately()) {
+          LOG.info("Splitting META logs for " + serverName);
+          this.services.getMasterFileSystem().splitMetaLog(serverName);
+        } else {
+          LOG.info("Splitting all logs for " + serverName);
+          this.services.getMasterFileSystem().splitAllLogs(serverName);
+        }
+      }
+    } catch (IOException ioe) {
+      this.services.getExecutorService().submit(this);
+      this.deadServers.add(serverName);
+      throw new IOException("failed log splitting for " +
+          serverName + ", will retry", ioe);
+    }
+
+    // Assign root and meta if we were carrying them.
+    if (isCarryingRoot()) { // -ROOT-
+      // Check again: region may be assigned to other where because of RIT
+      // timeout
+      if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
+        LOG.info("Server " + serverName
+            + " was carrying ROOT. Trying to assign.");
+        this.services.getAssignmentManager().regionOffline(
+            HRegionInfo.ROOT_REGIONINFO);
+        verifyAndAssignRootWithRetries();
+      } else {
+        LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
+      }
+    }
+
+    // Carrying meta?
+    if (isCarryingMeta()) {
+      // Check again: region may be assigned to other where because of RIT
+      // timeout
+      if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
+        LOG.info("Server " + serverName
+            + " was carrying META. Trying to assign.");
+        this.services.getAssignmentManager().regionOffline(
+            HRegionInfo.FIRST_META_REGIONINFO);
+        this.services.getAssignmentManager().assignMeta();
+      } else {
+        LOG.info("META has been assigned to otherwhere, skip assigning.");
+      }
+
+    }
+    super.process();
+  }
+  /**
+   * Before assign the ROOT region, ensure it haven't
+   *  been assigned by other place
+   * <p>
+   * Under some scenarios, the ROOT region can be opened twice, so it seemed online
+   * in two regionserver at the same time.
+   * If the ROOT region has been assigned, so the operation can be canceled.
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private void verifyAndAssignRoot()
+  throws InterruptedException, IOException, KeeperException {
+    long timeout = this.server.getConfiguration().
+      getLong("hbase.catalog.verification.timeout", 1000);
+    if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
+      this.services.getAssignmentManager().assignRoot();
+    } else if (serverName.equals(server.getCatalogTracker().getRootLocation())) {
+      throw new IOException("-ROOT- is onlined on the dead server "
+          + serverName);
+    } else {
+      LOG.info("Skip assigning -ROOT-, because it is online on the "
+          + server.getCatalogTracker().getRootLocation());
+    }
+  }
+
+  /**
+   * Failed many times, shutdown processing
+   * @throws IOException
+   */
+  private void verifyAndAssignRootWithRetries() throws IOException {
+    int iTimes = this.server.getConfiguration().getInt(
+        "hbase.catalog.verification.retries", 10);
+
+    long waitTime = this.server.getConfiguration().getLong(
+        "hbase.catalog.verification.timeout", 1000);
+
+    int iFlag = 0;
+    while (true) {
+      try {
+        verifyAndAssignRoot();
+        break;
+      } catch (KeeperException e) {
+        this.server.abort("In server shutdown processing, assigning root", e);
+        throw new IOException("Aborting", e);
+      } catch (Exception e) {
+        if (iFlag >= iTimes) {
+          this.server.abort("verifyAndAssignRoot failed after" + iTimes
+              + " times retries, aborting", e);
+          throw new IOException("Aborting", e);
+        }
+        try {
+          Thread.sleep(waitTime);
+        } catch (InterruptedException e1) {
+          LOG.warn("Interrupted when is the thread sleep", e1);
+          Thread.currentThread().interrupt();
+          throw new IOException("Interrupted", e1);
+        }
+        iFlag++;
+      }
+    }
+  }
+
   boolean isCarryingRoot() {
     return this.carryingRoot;
   }
 
-  @Override
   boolean isCarryingMeta() {
     return this.carryingMeta;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 8b889f7db921..0c4bc98f743c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -54,10 +54,10 @@
  */
 public class ServerShutdownHandler extends EventHandler {
   private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
-  private final ServerName serverName;
-  private final MasterServices services;
-  private final DeadServer deadServers;
-  private final boolean shouldSplitHlog; // whether to split HLog or not
+  protected final ServerName serverName;
+  protected final MasterServices services;
+  protected final DeadServer deadServers;
+  protected final boolean shouldSplitHlog; // whether to split HLog or not
 
   public ServerShutdownHandler(final Server server, final MasterServices services,
       final DeadServer deadServers, final ServerName serverName,
@@ -88,69 +88,6 @@ public String getInformativeName() {
       return super.getInformativeName();
     }
   }
-
-  /**
-   * Before assign the ROOT region, ensure it haven't 
-   *  been assigned by other place
-   * <p>
-   * Under some scenarios, the ROOT region can be opened twice, so it seemed online
-   * in two regionserver at the same time.
-   * If the ROOT region has been assigned, so the operation can be canceled. 
-   * @throws InterruptedException
-   * @throws IOException
-   * @throws KeeperException
-   */
-  private void verifyAndAssignRoot()
-  throws InterruptedException, IOException, KeeperException {
-    long timeout = this.server.getConfiguration().
-      getLong("hbase.catalog.verification.timeout", 1000);
-    if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
-      this.services.getAssignmentManager().assignRoot();
-    } else if (serverName.equals(server.getCatalogTracker().getRootLocation())) {
-      throw new IOException("-ROOT- is onlined on the dead server "
-          + serverName);
-    } else {
-      LOG.info("Skip assigning -ROOT-, because it is online on the "
-          + server.getCatalogTracker().getRootLocation());
-    }
-  }
-
-  /**
-   * Failed many times, shutdown processing
-   * @throws IOException
-   */
-  private void verifyAndAssignRootWithRetries() throws IOException {
-    int iTimes = this.server.getConfiguration().getInt(
-        "hbase.catalog.verification.retries", 10);
-
-    long waitTime = this.server.getConfiguration().getLong(
-        "hbase.catalog.verification.timeout", 1000);
-
-    int iFlag = 0;
-    while (true) {
-      try {
-        verifyAndAssignRoot();
-        break;
-      } catch (KeeperException e) {
-        this.server.abort("In server shutdown processing, assigning root", e);
-        throw new IOException("Aborting", e);
-      } catch (Exception e) {
-        if (iFlag >= iTimes) {
-          this.server.abort("verifyAndAssignRoot failed after" + iTimes
-              + " times retries, aborting", e);
-          throw new IOException("Aborting", e);
-        }
-        try {
-          Thread.sleep(waitTime);
-        } catch (InterruptedException e1) {
-          LOG.warn("Interrupted when is the thread sleep", e1);
-          Thread.currentThread().interrupt();
-          throw new IOException("Interrupted", e1);
-        }
-        iFlag++;
-      }
-    }
-  }
   
   /**
    * @return True if the server we are processing was carrying <code>-ROOT-</code>
@@ -187,42 +124,14 @@ public void process() throws IOException {
           LOG.info("Skipping log splitting for " + serverName);
         }
       } catch (IOException ioe) {
-        this.services.getExecutorService().submit(this);
+        //typecast to SSH so that we make sure that it is the SSH instance that
+        //gets submitted as opposed to MSSH or some other derived instance of SSH
+        this.services.getExecutorService().submit((ServerShutdownHandler)this);
         this.deadServers.add(serverName);
         throw new IOException("failed log splitting for " +
           serverName + ", will retry", ioe);
       }
 
-      // Assign root and meta if we were carrying them.
-      if (isCarryingRoot()) { // -ROOT-
-        // Check again: region may be assigned to other where because of RIT
-        // timeout
-        if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
-          LOG.info("Server " + serverName
-              + " was carrying ROOT. Trying to assign.");
-          this.services.getAssignmentManager().regionOffline(
-              HRegionInfo.ROOT_REGIONINFO);
-          verifyAndAssignRootWithRetries();
-        } else {
-          LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
-        }
-      }
-
-      // Carrying meta?
-      if (isCarryingMeta()) {
-        // Check again: region may be assigned to other where because of RIT
-        // timeout
-        if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
-          LOG.info("Server " + serverName
-              + " was carrying META. Trying to assign.");
-          this.services.getAssignmentManager().regionOffline(
-              HRegionInfo.FIRST_META_REGIONINFO);
-          this.services.getAssignmentManager().assignMeta();
-        } else {
-          LOG.info("META has been assigned to otherwhere, skip assigning.");
-        }
-      }
-
       // We don't want worker thread in the MetaServerShutdownHandler
       // executor pool to block by waiting availability of -ROOT-
       // and .META. server. Otherwise, it could run into the following issue:
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1d42fea22ce9..7e24e48a11b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -31,6 +31,7 @@
 import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -249,6 +250,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   private HBaseServer server;  
 
   private final InetSocketAddress isa;
+  private UncaughtExceptionHandler uncaughtExceptionHandler;
 
   // Leases
   private Leases leases;
@@ -293,7 +295,14 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   // HLog and HLog roller. log is protected rather than private to avoid
   // eclipse warning when accessed by inner classes
   protected volatile HLog hlog;
+  // The meta updates are written to a different hlog. If this
+  // regionserver holds meta regions, then this field will be non-null.
+  protected volatile HLog hlogForMeta;
+
   LogRoller hlogRoller;
+  LogRoller metaHLogRoller;
+
+  private final boolean separateHLogForMeta;
 
   // flag set after we're done setting up server threads (used for testing)
   protected volatile boolean isOnline;
@@ -401,6 +410,7 @@ public HRegionServer(Configuration conf)
       HConstants.HBASE_CHECKSUM_VERIFICATION, false);
 
     // Config'ed params
+    this.separateHLogForMeta = conf.getBoolean(HLog.SEPARATE_HLOG_FOR_META, false);
     this.numRetries = conf.getInt("hbase.client.retries.number", 10);
     this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
       10 * 1000);
@@ -460,6 +470,11 @@ public HRegionServer(Configuration conf)
       "hbase.regionserver.kerberos.principal", this.isa.getHostName());
     regionServerAccounting = new RegionServerAccounting();
     cacheConfig = new CacheConfig(conf);
+    uncaughtExceptionHandler = new UncaughtExceptionHandler() {
+      public void uncaughtException(Thread t, Throwable e) {
+        abort("Uncaught exception in service thread " + t.getName(), e);
+      }
+    };
   }
 
   /** Handle all the snapshot requests to this server */
@@ -806,6 +821,7 @@ public void run() {
     if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
     if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
     if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();
+    if (this.metaHLogRoller != null) this.metaHLogRoller.interruptIfNecessary();
     if (this.compactionChecker != null)
       this.compactionChecker.interrupt();
     if (this.healthCheckChore != null) {
@@ -994,16 +1010,27 @@ private void waitOnAllRegionsToClose(final boolean abort) {
   }
 
   private void closeWAL(final boolean delete) {
-    try {
-      if (this.hlog != null) {
+    if (this.hlogForMeta != null) {
+      // All hlogs (meta and non-meta) are in the same directory. Don't call
+      // closeAndDelete here since that would delete all hlogs not just the
+      // meta ones. We will just 'close' the hlog for meta here, and leave
+      // the directory cleanup to the follow-on closeAndDelete call.
+      try { //Part of the patch from HBASE-7982 to do with exception handling 
+        this.hlogForMeta.close();
+      } catch (Throwable e) {
+        LOG.error("Metalog close and delete failed", RemoteExceptionHandler.checkThrowable(e));
+      }
+    }
+    if (this.hlog != null) {
+      try {
         if (delete) {
           hlog.closeAndDelete();
         } else {
           hlog.close();
         }
+      } catch (Throwable e) {
+        LOG.error("Close and delete failed", RemoteExceptionHandler.checkThrowable(e));
       }
-    } catch (Throwable e) {
-      LOG.error("Close and delete failed", RemoteExceptionHandler.checkThrowable(e));
     }
   }
 
@@ -1355,6 +1382,24 @@ private HLog setupWALAndReplication() throws IOException {
     return instantiateHLog(logdir, oldLogDir);
   }
 
+  // The method is synchronized to guarantee atomic update to hlogForMeta - 
+  // It is possible that multiple calls could be made to this method almost 
+  // at the same time, one for _ROOT_ and another for .META. (if they happen
+  // to be assigned to the same RS). Also, we want to use the same log for both
+  private synchronized HLog getMetaWAL() throws IOException {
+    if (this.hlogForMeta == null) {
+      final String logName
+      = HLog.getHLogDirectoryName(this.serverNameFromMasterPOV.toString());
+
+      Path logdir = new Path(rootDir, logName);
+      final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+      if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
+      this.hlogForMeta = new HLog(this.fs.getBackingFs(), logdir, oldLogDir, this.conf,
+          getMetaWALActionListeners(), false, this.serverNameFromMasterPOV.toString(), true);
+    }
+    return this.hlogForMeta;
+  }
+
   /**
    * Called by {@link #setupWALAndReplication()} creating WAL instance.
    * @param logdir
@@ -1386,6 +1431,20 @@ protected List<WALActionsListener> getWALActionListeners() {
     return listeners;
   }
 
+  protected List<WALActionsListener> getMetaWALActionListeners() {
+    List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
+    // Using a tmp log roller to ensure metaLogRoller is alive once it is not
+    // null (addendum patch on HBASE-7213)
+    MetaLogRoller tmpLogRoller = new MetaLogRoller(this, this);
+    String n = Thread.currentThread().getName();
+    Threads.setDaemonThreadRunning(tmpLogRoller.getThread(),
+        n + "MetaLogRoller", uncaughtExceptionHandler);
+    this.metaHLogRoller = tmpLogRoller;
+    tmpLogRoller = null;
+    listeners.add(this.metaHLogRoller);
+    return listeners;
+  }
+
   protected LogRoller getLogRoller() {
     return hlogRoller;
   }
@@ -1580,12 +1639,6 @@ public MasterAddressTracker getMasterAddressManager() {
    */
   private void startServiceThreads() throws IOException {
     String n = Thread.currentThread().getName();
-    UncaughtExceptionHandler handler = new UncaughtExceptionHandler() {
-      public void uncaughtException(Thread t, Throwable e) {
-        abort("Uncaught exception in service thread " + t.getName(), e);
-      }
-    };
-
     // Start executor services
     this.service = new ExecutorService(getServerName().toString());
     this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
@@ -1601,14 +1654,15 @@ public void uncaughtException(Thread t, Throwable e) {
     this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
       conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
 
-    Threads.setDaemonThreadRunning(this.hlogRoller.getThread(), n + ".logRoller", handler);
+    Threads.setDaemonThreadRunning(this.hlogRoller.getThread(), n + ".logRoller",
+        uncaughtExceptionHandler);
     Threads.setDaemonThreadRunning(this.cacheFlusher.getThread(), n + ".cacheFlusher",
-      handler);
+        uncaughtExceptionHandler);
     Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), n +
-      ".compactionChecker", handler);
+      ".compactionChecker", uncaughtExceptionHandler);
     if (this.healthCheckChore != null) {
       Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker",
-        handler);
+          uncaughtExceptionHandler);
     }
 
     // Leases is not a Thread. Internally it runs a daemon thread. If it gets
@@ -1690,11 +1744,33 @@ private boolean isHealthy() {
       stop("One or more threads are no longer alive -- stop");
       return false;
     }
+    if (metaHLogRoller != null && !metaHLogRoller.isAlive()) {
+      stop("Meta HLog roller thread is no longer alive -- stop");
+      return false;
+    }
     return true;
   }
 
-  @Override
   public HLog getWAL() {
+    try {
+      return getWAL(null);
+    } catch (IOException e) {
+      LOG.warn("getWAL threw exception " + e);
+      return null; 
+    }
+  }
+
+  @Override
+  public HLog getWAL(HRegionInfo regionInfo) throws IOException {
+    //TODO: at some point this should delegate to the HLogFactory
+    //currently, we don't care about the region as much as we care about the 
+    //table.. (hence checking the tablename below)
+    //_ROOT_ and .META. regions have separate WAL. 
+    if (this.separateHLogForMeta && 
+        regionInfo != null && 
+        regionInfo.isMetaTable()) {
+      return getMetaWAL();
+    }
     return this.hlog;
   }
 
@@ -1846,6 +1922,9 @@ protected void join() {
     if (this.hlogRoller != null) {
       Threads.shutdown(this.hlogRoller.getThread());
     }
+    if (this.metaHLogRoller != null) {
+      Threads.shutdown(this.metaHLogRoller.getThread());
+    }
     if (this.compactSplitThread != null) {
       this.compactSplitThread.join();
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 74429a19beaa..01ae295c818e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -46,7 +47,7 @@ class LogRoller extends HasThread implements WALActionsListener {
   private final ReentrantLock rollLock = new ReentrantLock();
   private final AtomicBoolean rollLog = new AtomicBoolean(false);
   private final Server server;
-  private final RegionServerServices services;
+  protected final RegionServerServices services;
   private volatile long lastrolltime = System.currentTimeMillis();
   // Period to roll log.
   private final long rollperiod;
@@ -91,7 +92,7 @@ public void run() {
       try {
         this.lastrolltime = now;
         // This is array of actual region names.
-        byte [][] regionsToFlush = this.services.getWAL().rollWriter(rollLog.get());
+        byte [][] regionsToFlush = getWAL().rollWriter(rollLog.get());
         if (regionsToFlush != null) {
           for (byte [] r: regionsToFlush) scheduleFlush(r);
         }
@@ -155,6 +156,10 @@ public void interruptIfNecessary() {
     }
   }
 
+  protected HLog getWAL() throws IOException {
+    return this.services.getWAL(null);
+  }
+
   @Override
   public void preLogRoll(Path oldPath, Path newPath) throws IOException {
     // Not interested
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java
new file mode 100644
index 000000000000..d1ebd593c2b7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+@InterfaceAudience.Private
+class MetaLogRoller extends LogRoller {
+  public MetaLogRoller(Server server, RegionServerServices services) {
+    super(server, services);
+  }
+  @Override
+  protected HLog getWAL() throws IOException {
+    //The argument to getWAL below could either be HRegionInfo.FIRST_META_REGIONINFO or
+    //HRegionInfo.ROOT_REGIONINFO. Both these share the same WAL.
+    return services.getWAL(HRegionInfo.FIRST_META_REGIONINFO);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 1c667048b560..c3473c6a1225 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
-import java.util.Map;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -38,8 +37,9 @@ public interface RegionServerServices extends OnlineRegions {
    */
   public boolean isStopping();
 
-  /** @return the HLog */
-  public HLog getWAL();
+  /** @return the HLog for a particular region. Pass null for getting the
+   * default (common) WAL */
+  public HLog getWAL(HRegionInfo regionInfo) throws IOException;
 
   /**
    * @return Implementation of {@link CompactionRequestor} or null.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 203e03134118..1fbf8f0ef95d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.zookeeper.KeeperException;
@@ -43,7 +44,7 @@
 public class OpenRegionHandler extends EventHandler {
   private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
 
-  private final RegionServerServices rsServices;
+  protected final RegionServerServices rsServices;
 
   private final HRegionInfo regionInfo;
   private final HTableDescriptor htd;
@@ -327,7 +328,8 @@ HRegion openRegion() {
       // Instantiate the region.  This also periodically tickles our zk OPENING
       // state so master doesn't timeout this region in transition.
       region = HRegion.openHRegion(this.regionInfo, this.htd,
-          this.rsServices.getWAL(), this.server.getConfiguration(),
+          this.rsServices.getWAL(this.regionInfo), 
+          this.server.getConfiguration(),
           this.rsServices,
         new CancelableProgressable() {
           public boolean progress() {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 57fc520545cb..1030ab0b1297 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -119,6 +119,9 @@ public class HLog implements Syncable {
   /** File Extension used while splitting an HLog into regions (HBASE-2312) */
   public static final String SPLITTING_EXT = "-splitting";
   public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
+  /** The META region's HLog filename extension */
+  public static final String META_HLOG_FILE_EXTN = ".meta";
+  public static final String SEPARATE_HLOG_FOR_META = "hbase.regionserver.separate.hlog.for.meta";
 
   /*
    * Name of directory that holds recovered edits written by the wal log
@@ -200,6 +203,8 @@ public interface Writer {
 
   private final AtomicLong logSeqNum = new AtomicLong(0);
 
+  private boolean forMeta = false;
+
   // The timestamp (in ms) when the log file was created.
   private volatile long filenum = -1;
 
@@ -254,7 +259,8 @@ public interface Writer {
   /**
    * Pattern used to validate a HLog file name
    */
-  private static final Pattern pattern = Pattern.compile(".*\\.\\d*");
+  private static final Pattern pattern = 
+      Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
 
   static byte [] COMPLETE_CACHE_FLUSH;
   static {
@@ -334,7 +340,7 @@ public static Metric getSlowAppendTime() {
   public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
               final Configuration conf)
   throws IOException {
-    this(fs, dir, oldLogDir, conf, null, true, null);
+    this(fs, dir, oldLogDir, conf, null, true, null, false);
   }
 
   /**
@@ -359,7 +365,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
   public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
       final Configuration conf, final List<WALActionsListener> listeners,
       final String prefix) throws IOException {
-    this(fs, dir, oldLogDir, conf, listeners, true, prefix);
+    this(fs, dir, oldLogDir, conf, listeners, true, prefix, false);
   }
 
   /**
@@ -380,11 +386,12 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
    * @param prefix should always be hostname and port in distributed env and
    *        it will be URL encoded before being used.
    *        If prefix is null, "hlog" will be used
+   * @param forMeta if this hlog is meant for meta updates
    * @throws IOException
    */
   public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
       final Configuration conf, final List<WALActionsListener> listeners,
-      final boolean failIfLogDirExists, final String prefix)
+      final boolean failIfLogDirExists, final String prefix, boolean forMeta)
   throws IOException {
     super();
     this.fs = fs;
@@ -402,10 +409,11 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     this.logrollsize = (long)(this.blocksize * multi);
     this.optionalFlushInterval =
       conf.getLong("hbase.regionserver.optionallogflushinterval", 1 * 1000);
-    if (failIfLogDirExists && fs.exists(dir)) {
+    boolean dirExists = false;
+    if (failIfLogDirExists && (dirExists = this.fs.exists(dir))) {
       throw new IOException("Target HLog directory already exists: " + dir);
     }
-    if (!fs.mkdirs(dir)) {
+    if (!dirExists && !fs.mkdirs(dir)) {
       throw new IOException("Unable to mkdir " + dir);
     }
     this.oldLogDir = oldLogDir;
@@ -414,6 +422,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
         throw new IOException("Unable to mkdir " + this.oldLogDir);
       }
     }
+    this.forMeta = forMeta;
     this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32);
     this.minTolerableReplication = conf.getInt(
         "hbase.regionserver.hlog.tolerable.lowreplication",
@@ -623,6 +632,7 @@ OutputStream getOutputStream() {
       long currentFilenum = this.filenum;
       Path oldPath = null;
       if (currentFilenum > 0) {
+        //computeFilename  will take care of meta hlog filename
         oldPath = computeFilename(currentFilenum);
       }
       this.filenum = System.currentTimeMillis();
@@ -698,6 +708,9 @@ OutputStream getOutputStream() {
    */
   protected Writer createWriterInstance(final FileSystem fs, final Path path,
       final Configuration conf) throws IOException {
+    if (forMeta) {
+      //TODO: set a higher replication for the hlog files (HBASE-6773)
+    }
     return createWriter(fs, path, conf);
   }
 
@@ -947,7 +960,18 @@ protected Path computeFilename(long filenum) {
     if (filenum < 0) {
       throw new RuntimeException("hlog file number can't be < 0");
     }
-    return new Path(dir, prefix + "." + filenum);
+    String child = prefix + "." + filenum;
+    if (forMeta) {
+      child += HLog.META_HLOG_FILE_EXTN;
+    }
+    return new Path(dir, child);
+  }
+
+  public static boolean isMetaFile(Path p) {
+    if (p.getName().endsWith(HLog.META_HLOG_FILE_EXTN)) {
+      return true;
+    }
+    return false;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index ecac91d0b382..6f34f75acd2d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
@@ -279,6 +280,11 @@ private List<Path> splitLog(final FileStatus[] logfiles) throws IOException {
             + ": " + logPath + ", length=" + logLength);
         Reader in;
         try {
+          //actually, for meta-only hlogs, we don't need to go thru the process
+          //of parsing and segregating by regions since all the logs are for
+          //meta only. However, there is a sequence number that can be obtained
+          //only by parsing.. so we parse for all files currently
+          //TODO: optimize this part somehow
           in = getReader(fs, log, conf, skipErrors);
           if (in != null) {
             parseHLog(in, logPath, entryBuffers, fs, conf, skipErrors);
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 31a7b802d639..ea1db8a50284 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -231,6 +231,10 @@
     <value>org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter</value>
     <description>The HLog file writer implementation.</description>
   </property>
+  <property>
+    <name>hbase.regionserver.separate.hlog.for.meta</name>
+    <value>false</value>
+  </property>
   <property>
     <name>hbase.regionserver.nbreservationblocks</name>
     <value>4</value>
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 6b17c762864b..0d9aaf32d96a 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -308,6 +308,11 @@ public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor) throws
     @Override
     public void deleteColumn(byte[] tableName, byte[] columnName) throws IOException {
     }
+
+    @Override
+    public boolean shouldSplitMetaSeparately() {
+      return false;
+    }
   }
 
   @Test
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
index f115fb29e016..09c954959e6f 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
@@ -82,7 +82,7 @@ public boolean isStopping() {
   }
 
   @Override
-  public HLog getWAL() {
+  public HLog getWAL(HRegionInfo regionInfo) throws IOException {
     return null;
   }
 

From e3dabdbe18bec88bd8b71571eac554f4f08ee063 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 26 Mar 2013 22:21:07 +0000
Subject: [PATCH 0873/1540] HBASE-8199 Eliminate exception for ExportSnapshot
 against the null table snapshot (Julian Zhou)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1461346 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 10 ++++--
 .../hbase/snapshot/TestExportSnapshot.java    | 32 +++++++++++++++++--
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 31c3cdc9f2c7..62f2bcbaf388 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -642,9 +642,13 @@ public int run(String[] args) throws Exception {
     // The snapshot references must be copied before the files otherwise the files gets removed
     // by the HFileArchiver, since they have no references.
     try {
-      if (!runCopyJob(inputRoot, outputRoot, files, verifyChecksum,
-          filesUser, filesGroup, filesMode, mappers)) {
-        throw new ExportSnapshotException("Snapshot export failed!");
+      if (files.size() == 0) {
+        LOG.warn("There are 0 store file to be copied. There may be no data in the table.");
+      } else {
+        if (!runCopyJob(inputRoot, outputRoot, files, verifyChecksum,
+            filesUser, filesGroup, filesMode, mappers)) {
+          throw new ExportSnapshotException("Snapshot export failed!");
+        }
       }
 
       // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index f329c60259ab..dfd0a7878e24 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
@@ -70,6 +71,7 @@ public class TestExportSnapshot {
 
   private final static byte[] FAMILY = Bytes.toBytes("cf");
 
+  private byte[] emptySnapshotName;
   private byte[] snapshotName;
   private byte[] tableName;
   private HBaseAdmin admin;
@@ -99,11 +101,19 @@ public void setUp() throws Exception {
     long tid = System.currentTimeMillis();
     tableName = Bytes.toBytes("testtb-" + tid);
     snapshotName = Bytes.toBytes("snaptb0-" + tid);
+    emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid);
 
     // create Table
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.addFamily(new HColumnDescriptor(FAMILY));
     admin.createTable(htd, null);
+
+    // Take an empty snapshot
+    admin.disableTable(tableName);
+    admin.snapshot(emptySnapshotName, tableName);
+    admin.enableTable(tableName);
+
+    // Add some rows
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
     TEST_UTIL.loadTable(table, FAMILY);
 
@@ -115,7 +125,11 @@ public void setUp() throws Exception {
 
   @After
   public void tearDown() throws Exception {
-    this.admin.close();
+    admin.disableTable(tableName);
+    admin.deleteSnapshot(snapshotName);
+    admin.deleteSnapshot(emptySnapshotName);
+    admin.deleteTable(tableName);
+    admin.close();
   }
 
   /**
@@ -160,6 +174,19 @@ public void testBalanceSplit() throws Exception {
    */
   @Test
   public void testExportFileSystemState() throws Exception {
+    testExportFileSystemState(tableName, snapshotName, 2);
+  }
+
+  @Test
+  public void testEmptyExportFileSystemState() throws Exception {
+    testExportFileSystemState(tableName, emptySnapshotName, 1);
+  }
+
+  /**
+   * Test ExportSnapshot
+   */
+  private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
+      int filesExpected) throws Exception {
     Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
     URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
     FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
@@ -174,7 +201,7 @@ public void testExportFileSystemState() throws Exception {
 
     // Verify File-System state
     FileStatus[] rootFiles = fs.listStatus(copyDir);
-    assertEquals(2, rootFiles.length);
+    assertEquals(filesExpected, rootFiles.length);
     for (FileStatus fileStatus: rootFiles) {
       String name = fileStatus.getPath().getName();
       assertTrue(fileStatus.isDir());
@@ -254,4 +281,3 @@ private Set<String> listFiles(final FileSystem fs, final Path root, final Path d
     return files;
   }
 }
-

From 8c13ed86d472266c1ef95a0da3e3cbc37d1244f4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Mar 2013 17:39:23 +0000
Subject: [PATCH 0874/1540] HBASE-8198 Backport HBASE-8063(Filter HFiles based
 on first/last key) into 0.94 (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462231 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/HalfStoreFileReader.java  |  6 +++++
 .../hadoop/hbase/regionserver/StoreFile.java  | 22 +++++++++++++++++++
 .../hbase/regionserver/StoreFileScanner.java  |  7 +++---
 .../io/encoding/TestChangingEncoding.java     | 12 ++++++++--
 4 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index d3a82eaa427e..e07f37056da9 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
@@ -282,6 +283,11 @@ public boolean isSeeked() {
     };
   }
 
+  @Override
+  public boolean passesKeyRangeFilter(Scan scan) {
+    return true;
+  }
+  
   @Override
   public byte[] getLastKey() {
     if (top) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 96c693490efa..4cfaaca0129f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -1610,6 +1610,28 @@ public boolean passesGeneralBloomFilter(byte[] row, int rowOffset,
       return true;
     }
 
+    /**
+     * Checks whether the given scan rowkey range overlaps with the current storefile's
+     * @param scan the scan specification. Used to determine the rowkey range.
+     * @return true if there is overlap, false otherwise
+     */
+    public boolean passesKeyRangeFilter(Scan scan) {
+      if (this.getFirstKey() == null || this.getLastKey() == null) {
+        // the file is empty
+        return false;
+      }
+      if (Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW)
+          && Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
+        return true;
+      }
+      KeyValue startKeyValue = KeyValue.createFirstOnRow(scan.getStartRow());
+      KeyValue stopKeyValue = KeyValue.createLastOnRow(scan.getStopRow());
+      boolean nonOverLapping = (getComparator().compare(this.getFirstKey(),
+        stopKeyValue.getKey()) > 0 && !Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW))
+          || getComparator().compare(this.getLastKey(), startKeyValue.getKey()) < 0;
+      return !nonOverLapping;
+    }
+
     public Map<byte[], byte[]> loadFileInfo() throws IOException {
       Map<byte [], byte []> fi = reader.loadFileInfo();
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index 363bb62fc14c..d69fc9fdf5dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -368,9 +368,8 @@ static final long getSeekCount() {
   }
 
   @Override
-  public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns,
-      long oldestUnexpiredTS) {
-    return reader.passesTimerangeFilter(scan, oldestUnexpiredTS) &&
-        reader.passesBloomFilter(scan, columns);
+  public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns, long oldestUnexpiredTS) {
+    return reader.passesTimerangeFilter(scan, oldestUnexpiredTS)
+        && reader.passesKeyRangeFilter(scan) && reader.passesBloomFilter(scan, columns);
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
index d3418a03d0ab..77ce818bab9c 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
@@ -238,10 +238,18 @@ public void testFlippingEncodeOnDisk() throws Exception {
   private void compactAndWait() throws IOException, InterruptedException {
     LOG.debug("Compacting table " + tableName);
     admin.majorCompact(tableName);
-    Threads.sleepWithoutInterrupt(500);
     HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+    
+    // Waiting for the compaction to start, at least .5s.
+    final long maxWaitime = System.currentTimeMillis() + 500;
+    boolean cont;
+    do {
+      cont = rs.compactSplitThread.getCompactionQueueSize() == 0;
+      Threads.sleep(1);
+    } while (cont && System.currentTimeMillis() < maxWaitime);
+    
     while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
-      Threads.sleep(50);
+      Threads.sleep(5);
     }
     LOG.debug("Compaction queue size reached 0, continuing");
   }

From e7894a9c5e5c0b5940e5dc415c9ee74f0469de80 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Mar 2013 17:42:15 +0000
Subject: [PATCH 0875/1540] HBASE-8198 Add the new test

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462233 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestScannerSelectionUsingKeyRange.java    | 143 ++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
new file mode 100644
index 000000000000..269983225dc7
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Test the optimization that does not scan files where all key ranges are excluded.
+ */
+@RunWith(Parameterized.class)
+@Category(SmallTests.class)
+public class TestScannerSelectionUsingKeyRange {
+  private static final Log LOG = LogFactory.getLog(TestScannerSelectionUsingKeyRange.class);
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static String TABLE = "myTable";
+  private static String FAMILY = "myCF";
+  private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
+  private static final int NUM_ROWS = 8;
+  private static final int NUM_COLS_PER_ROW = 5;
+  private static final int NUM_FILES = 2;
+  private static final Map<Object, Integer> TYPE_COUNT = new HashMap<Object, Integer>(3);
+  static {
+    TYPE_COUNT.put(BloomType.ROWCOL, 2);
+    TYPE_COUNT.put(BloomType.ROW, 2);
+    TYPE_COUNT.put(BloomType.NONE, 2);
+  }
+
+  private BloomType bloomType;
+  private int expectedCount;
+
+  @Parameters
+  public static Collection<Object[]> parameters() {
+    List<Object[]> params = new ArrayList<Object[]>();
+    for (Object type : TYPE_COUNT.keySet()) {
+        params.add(new Object[] { type, TYPE_COUNT.get(type) });
+    }
+    return params;
+  }
+
+  public TestScannerSelectionUsingKeyRange(Object expectedType, Object expectedCount) {
+    bloomType = (BloomType)expectedType;
+    expectedCount = expectedCount;
+  }
+  
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.cleanupTestDir();
+  }
+
+  @Test
+  public void testScannerSelection() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setInt("hbase.hstore.compactionThreshold", 10000);
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY_BYTES).setBlockCacheEnabled(true)
+        .setBloomFilterType(bloomType);
+    HTableDescriptor htd = new HTableDescriptor(TABLE);
+    htd.addFamily(hcd);
+    HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE));
+    HRegion region = HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(), conf, htd);
+
+    for (int iFile = 0; iFile < NUM_FILES; ++iFile) {
+      for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
+        Put put = new Put(Bytes.toBytes("row" + iRow));
+        for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
+          put.add(FAMILY_BYTES, Bytes.toBytes("col" + iCol),
+              Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
+        }
+        region.put(put);
+      }
+      region.flushcache();
+    }
+
+    Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz"));
+    CacheConfig cacheConf = new CacheConfig(conf);
+    LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
+    cache.clearCache();
+    Map<String, Long> metricsBefore = SchemaMetrics.getMetricsSnapshot();
+    SchemaMetrics.validateMetricChanges(metricsBefore);
+    InternalScanner scanner = region.getScanner(scan);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    while (scanner.next(results)) {
+    }
+    scanner.close();
+    assertEquals(0, results.size());
+    Set<String> accessedFiles = cache.getCachedFileNamesForTest();
+    assertEquals(accessedFiles.size(), 0);
+    //assertEquals(cache.getBlockCount(), 0);
+    Map<String, Long> diffMetrics = SchemaMetrics.diffMetrics(metricsBefore,
+      SchemaMetrics.getMetricsSnapshot());
+    SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE, FAMILY);
+    long dataBlockRead = SchemaMetrics.getLong(diffMetrics,
+      schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT));
+    assertEquals(dataBlockRead, 0);
+    region.close();
+  }
+}

From a0b68d2cbe92fcd66773cb97e6d7c6fd45f78a82 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 29 Mar 2013 00:07:28 +0000
Subject: [PATCH 0876/1540] HBASE-8215 Removing existing .regioninfo in
 writeRegioninfoOnFilesystem

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462361 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegion.java   | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 6eb5f5e9e872..4e592d9bdfd9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -778,10 +778,17 @@ private void checkRegioninfoOnFilesystem(Path regiondir) throws IOException {
   public static void writeRegioninfoOnFilesystem(HRegionInfo regionInfo, Path regiondir,
       FileSystem fs, Configuration conf) throws IOException {
     Path regioninfoPath = new Path(regiondir, REGIONINFO_FILE);
-    if (fs.exists(regioninfoPath) &&
-        fs.getFileStatus(regioninfoPath).getLen() > 0) {
-      return;
+    if (fs.exists(regioninfoPath)) {
+      if (fs.getFileStatus(regioninfoPath).getLen() > 0) {
+        return;
+      }
+
+      LOG.info("Rewriting .regioninfo file at: " + regioninfoPath);
+      if (!fs.delete(regioninfoPath, false)) {
+        throw new IOException("Unable to remove existing " + regioninfoPath);
+      }
     }
+
     // Create in tmpdir and then move into place in case we crash after
     // create but before close.  If we don't successfully close the file,
     // subsequent region reopens will fail the below because create is

From 35d2464b384bce09afb5643400d25c65311aa5da Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 29 Mar 2013 15:53:50 +0000
Subject: [PATCH 0877/1540] HBASE-8210. Backport the LoadTest portions of
 HBASE-7383

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462511 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/encoding/TestEncodedSeekers.java |  22 ++-
 .../hbase/util/LoadTestDataGenerator.java     |  89 ++++++++++
 .../hbase/util/LoadTestKVGenerator.java       |  44 ++---
 .../hadoop/hbase/util/LoadTestTool.java       |  14 +-
 .../hbase/util/MultiThreadedAction.java       | 158 ++++++++++++++++--
 .../hbase/util/MultiThreadedReader.java       |  86 +++++-----
 .../hbase/util/MultiThreadedWriter.java       | 104 ++++--------
 .../hadoop/hbase/util/RestartMetaTest.java    |  11 +-
 .../hbase/util/TestLoadTestKVGenerator.java   |  14 +-
 .../util/TestMiniClusterLoadSequential.java   |   5 +-
 10 files changed, 366 insertions(+), 181 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java

diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java
index f40afe48b7be..46b55620dc0c 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java
@@ -32,14 +32,11 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
-import org.apache.hadoop.hbase.util.MultiThreadedWriter;
+
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
@@ -101,17 +98,18 @@ TABLE_NAME, new HColumnDescriptor(CF_NAME)
             .setDataBlockEncoding(encoding)
             .setEncodeOnDisk(encodeOnDisk)
     );
+
     LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(
         MIN_VALUE_SIZE, MAX_VALUE_SIZE);
 
     // Write
     for (int i = 0; i < NUM_ROWS; ++i) {
-      byte[] key = MultiThreadedWriter.longToByteArrayKey(i);
+      byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
       for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
         Put put = new Put(key);
-        String colAsStr = String.valueOf(j);
-        byte[] value = dataGenerator.generateRandomSizeValue(i, colAsStr);
-        put.add(CF_BYTES, Bytes.toBytes(colAsStr), value);
+        byte[] col = Bytes.toBytes(String.valueOf(j));
+        byte[] value = dataGenerator.generateRandomSizeValue(key, col);
+        put.add(CF_BYTES, col, value);
         region.put(put);
       }
       if (i % NUM_ROWS_PER_FLUSH == 0) {
@@ -122,7 +120,7 @@ TABLE_NAME, new HColumnDescriptor(CF_NAME)
     for (int doneCompaction = 0; doneCompaction <= 1; ++doneCompaction) {
       // Read
       for (int i = 0; i < NUM_ROWS; ++i) {
-        final byte[] rowKey = MultiThreadedWriter.longToByteArrayKey(i);
+        byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
         for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
           if (VERBOSE) {
             System.err.println("Reading row " + i + ", column " +  j);
@@ -131,10 +129,10 @@ TABLE_NAME, new HColumnDescriptor(CF_NAME)
           final byte[] qualBytes = Bytes.toBytes(qualStr);
           Get get = new Get(rowKey);
           get.addColumn(CF_BYTES, qualBytes);
-          Result result = region.get(get, null);
+          Result result = region.get(get);
           assertEquals(1, result.size());
-          assertTrue(LoadTestKVGenerator.verify(Bytes.toString(rowKey), qualStr,
-              result.getValue(CF_BYTES, qualBytes)));
+          byte[] value = result.getValue(CF_BYTES, qualBytes);
+          assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes));
         }
       }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java
new file mode 100644
index 000000000000..c7f6dd6b6e26
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.util.Set;
+
+/**
+ * A generator of random data (keys/cfs/columns/values) for load testing.
+ * Contains LoadTestKVGenerator as a matter of convenience...
+ */
+public abstract class LoadTestDataGenerator {
+  protected final LoadTestKVGenerator kvGenerator;
+
+  /**
+   * Initializes the object.
+   * @param minValueSize minimum size of the value generated by
+   * {@link #generateValue(byte[], byte[], byte[])}.
+   * @param maxValueSize maximum size of the value generated by
+   * {@link #generateValue(byte[], byte[], byte[])}.
+   */
+  public LoadTestDataGenerator(int minValueSize, int maxValueSize) {
+    this.kvGenerator = new LoadTestKVGenerator(minValueSize, maxValueSize);
+ }
+  /**
+   * Generates a deterministic, unique hashed row key from a number. That way, the user can
+   * keep track of numbers, without messing with byte array and ensuring key distribution.
+   * @param keyBase Base number for a key, such as a loop counter.
+   */
+  public abstract byte[] getDeterministicUniqueKey(long keyBase);
+
+  /**
+   * Gets column families for the load test table.
+   * @return The array of byte[]s representing column family names.
+   */
+  public abstract byte[][] getColumnFamilies();
+
+  /**
+   * Generates an applicable set of columns to be used for a particular key and family.
+   * @param rowKey The row key to generate for.
+   * @param cf The column family name to generate for.
+   * @return The array of byte[]s representing column names.
+   */
+  public abstract byte[][] generateColumnsForCf(byte[] rowKey, byte[] cf);
+
+  /**
+   * Generates a value to be used for a particular row/cf/column.
+   * @param rowKey The row key to generate for.
+   * @param cf The column family name to generate for.
+   * @param column The column name to generate for.
+   * @return The value to use.
+   */
+  public abstract byte[] generateValue(byte[] rowKey, byte[] cf, byte[] column);
+
+  /**
+   * Checks that columns for a rowKey and cf are valid if generated via
+   * {@link #generateColumnsForCf(byte[], byte[])}
+   * @param rowKey The row key to verify for.
+   * @param cf The column family name to verify for.
+   * @param columnSet The column set (for example, encountered by read).
+   * @return True iff valid.
+   */
+  public abstract boolean verify(byte[] rowKey, byte[] cf, Set<byte[]> columnSet);
+
+  /**
+   * Checks that value for a rowKey/cf/column is valid if generated via
+   * {@link #generateValue(byte[], byte[], byte[])}
+   * @param rowKey The row key to verify for.
+   * @param cf The column family name to verify for.
+   * @param column The column name to verify for.
+   * @param value The value (for example, encountered by read).
+   * @return True iff valid.
+   */
+  public abstract boolean verify(byte[] rowKey, byte[] cf, byte[] column, byte[] value);
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java
index ba125a62e9a7..77f5420c60d9 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java
@@ -27,8 +27,6 @@
  * hash. Values are generated by selecting value size in the configured range
  * and generating a pseudo-random sequence of bytes seeded by key, column
  * qualifier, and value size.
- * <p>
- * Not thread-safe, so a separate instance is needed for every writer thread/
  */
 public class LoadTestKVGenerator {
 
@@ -49,13 +47,13 @@ public LoadTestKVGenerator(int minValueSize, int maxValueSize) {
 
   /**
    * Verifies that the given byte array is the same as what would be generated
-   * for the given row key and qualifier. We are assuming that the value size
-   * is correct, and only verify the actual bytes. However, if the min/max
-   * value sizes are set sufficiently high, an accidental match should be
+   * for the given seed strings (row/cf/column/...). We are assuming that the
+   * value size is correct, and only verify the actual bytes. However, if the
+   * min/max value sizes are set sufficiently high, an accidental match should be
    * extremely improbable.
    */
-  public static boolean verify(String rowKey, String qual, byte[] value) {
-    byte[] expectedData = getValueForRowColumn(rowKey, qual, value.length);
+  public static boolean verify(byte[] value, byte[]... seedStrings) {
+    byte[] expectedData = getValueForRowColumn(value.length, seedStrings);
     return Bytes.equals(expectedData, value);
   }
 
@@ -74,25 +72,31 @@ public static String md5PrefixedKey(long key) {
   /**
    * Generates a value for the given key index and column qualifier. Size is
    * selected randomly in the configured range. The generated value depends
-   * only on the combination of the key, qualifier, and the selected value
-   * size. This allows to verify the actual value bytes when reading, as done
-   * in {@link #verify(String, String, byte[])}.
+   * only on the combination of the strings passed (key/cf/column/...) and the selected
+   * value size. This allows to verify the actual value bytes when reading, as done
+   * in {#verify(byte[], byte[]...)}
+   * This method is as thread-safe as Random class. It appears that the worst bug ever
+   * found with the latter is that multiple threads will get some duplicate values, which
+   * we don't care about.
    */
-  public byte[] generateRandomSizeValue(long key, String qual) {
-    String rowKey = md5PrefixedKey(key);
-    int dataSize = minValueSize + randomForValueSize.nextInt(
-        Math.abs(maxValueSize - minValueSize));
-    return getValueForRowColumn(rowKey, qual, dataSize);
-  }
+  public byte[] generateRandomSizeValue(byte[]... seedStrings) {
+    int dataSize = minValueSize;
+    if (minValueSize != maxValueSize) {
+      dataSize = minValueSize + randomForValueSize.nextInt(Math.abs(maxValueSize - minValueSize));
+    }
+    return getValueForRowColumn(dataSize, seedStrings);
+ }
 
   /**
    * Generates random bytes of the given size for the given row and column
    * qualifier. The random seed is fully determined by these parameters.
    */
-  private static byte[] getValueForRowColumn(String rowKey, String qual,
-      int dataSize) {
-    Random seededRandom = new Random(rowKey.hashCode() + qual.hashCode() +
-        dataSize);
+  private static byte[] getValueForRowColumn(int dataSize, byte[]... seedStrings) {
+    long seed = dataSize;
+    for (byte[] str : seedStrings) {
+      seed += Bytes.toString(str).hashCode();
+    }
+    Random seededRandom = new Random(seed);
     byte[] randomBytes = new byte[dataSize];
     seededRandom.nextBytes(randomBytes);
     return randomBytes;
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index 4d73ba8be107..3761242ac80f 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -118,7 +118,7 @@ public class LoadTestTool extends AbstractHBaseTool {
 
   // Writer options
   private int numWriterThreads = DEFAULT_NUM_THREADS;
-  private long minColsPerKey, maxColsPerKey;
+  private int minColsPerKey, maxColsPerKey;
   private int minColDataSize, maxColDataSize;
   private boolean isMultiPut;
 
@@ -259,7 +259,7 @@ protected void processOptions(CommandLine cmd) {
 
       int colIndex = 0;
       minColsPerKey = 1;
-      maxColsPerKey = 2 * Long.parseLong(writeOpts[colIndex++]);
+      maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
       int avgColDataSize =
           parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
       minColDataSize = avgColDataSize / 2;
@@ -341,16 +341,16 @@ protected int doWork() throws IOException {
       initTestTable();
     }
 
+    LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(
+      minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, COLUMN_FAMILY);
+
     if (isWrite) {
-      writerThreads = new MultiThreadedWriter(conf, tableName, COLUMN_FAMILY);
+      writerThreads = new MultiThreadedWriter(dataGen, conf, tableName);
       writerThreads.setMultiPut(isMultiPut);
-      writerThreads.setColumnsPerKey(minColsPerKey, maxColsPerKey);
-      writerThreads.setDataSize(minColDataSize, maxColDataSize);
     }
 
     if (isRead) {
-      readerThreads = new MultiThreadedReader(conf, tableName, COLUMN_FAMILY,
-          verifyPercent);
+      readerThreads = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent);
       readerThreads.setMaxErrors(maxReadErrors);
       readerThreads.setKeyWindow(keyWindow);
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
index b312cca1d842..44b1c9472d38 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
@@ -18,12 +18,16 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -34,7 +38,6 @@ public abstract class MultiThreadedAction {
   private static final Log LOG = LogFactory.getLog(MultiThreadedAction.class);
 
   protected final byte[] tableName;
-  protected final byte[] columnFamily;
   protected final Configuration conf;
 
   protected int numThreads = 1;
@@ -51,8 +54,69 @@ public abstract class MultiThreadedAction {
   protected AtomicLong totalOpTimeMs = new AtomicLong();
   protected boolean verbose = false;
 
-  protected int minDataSize = 256;
-  protected int maxDataSize = 1024;
+  protected LoadTestDataGenerator dataGenerator = null;
+
+  /**
+   * Default implementation of LoadTestDataGenerator that uses LoadTestKVGenerator, fixed
+   * set of column families, and random number of columns in range. The table for it can
+   * be created manually or, for example, via
+   * {@link HBaseTestingUtility#createPreSplitLoadTestTable(
+   * org.apache.hadoop.hbase.Configuration, byte[], byte[], Algorithm, DataBlockEncoding)}
+   */
+  public static class DefaultDataGenerator extends LoadTestDataGenerator {
+    private byte[][] columnFamilies = null;
+    private int minColumnsPerKey;
+    private int maxColumnsPerKey;
+    private final Random random = new Random();
+
+    public DefaultDataGenerator(int minValueSize, int maxValueSize,
+        int minColumnsPerKey, int maxColumnsPerKey, byte[]... columnFamilies) {
+      super(minValueSize, maxValueSize);
+      this.columnFamilies = columnFamilies;
+      this.minColumnsPerKey = minColumnsPerKey;
+      this.maxColumnsPerKey = maxColumnsPerKey;
+    }
+
+    public DefaultDataGenerator(byte[]... columnFamilies) {
+      // Default values for tests that didn't care to provide theirs.
+      this(256, 1024, 1, 10, columnFamilies);
+    }
+
+    @Override
+    public byte[] getDeterministicUniqueKey(long keyBase) {
+      return LoadTestKVGenerator.md5PrefixedKey(keyBase).getBytes();
+    }
+
+    @Override
+    public byte[][] getColumnFamilies() {
+      return columnFamilies;
+    }
+
+    @Override
+    public byte[][] generateColumnsForCf(byte[] rowKey, byte[] cf) {
+      int numColumns = minColumnsPerKey + random.nextInt(maxColumnsPerKey - minColumnsPerKey + 1);
+      byte[][] columns = new byte[numColumns][];
+      for (int i = 0; i < numColumns; ++i) {
+        columns[i] = Integer.toString(i).getBytes();
+      }
+      return columns;
+    }
+
+    @Override
+    public byte[] generateValue(byte[] rowKey, byte[] cf, byte[] column) {
+      return kvGenerator.generateRandomSizeValue(rowKey, cf, column);
+    }
+
+    @Override
+    public boolean verify(byte[] rowKey, byte[] cf, byte[] column, byte[] value) {
+      return LoadTestKVGenerator.verify(value, rowKey, cf, column);
+    }
+
+    @Override
+    public boolean verify(byte[] rowKey, byte[] cf, Set<byte[]> columnSet) {
+      return (columnSet.size() >= minColumnsPerKey) && (columnSet.size() <= maxColumnsPerKey);
+    }
+  }
 
   /** "R" or "W" */
   private String actionLetter;
@@ -62,11 +126,11 @@ public abstract class MultiThreadedAction {
 
   public static final int REPORTING_INTERVAL_MS = 5000;
 
-  public MultiThreadedAction(Configuration conf, byte[] tableName,
-      byte[] columnFamily, String actionLetter) {
+  public MultiThreadedAction(LoadTestDataGenerator dataGen, Configuration conf, byte[] tableName,
+      String actionLetter) {
     this.conf = conf;
     this.tableName = tableName;
-    this.columnFamily = columnFamily;
+    this.dataGenerator = dataGen;
     this.actionLetter = actionLetter;
   }
 
@@ -165,17 +229,16 @@ private void printStreamingCounters(long numKeysDelta,
     }
   }
 
-  public void setDataSize(int minDataSize, int maxDataSize) {
-    this.minDataSize = minDataSize;
-    this.maxDataSize = maxDataSize;
-  }
-
   public void waitForFinish() {
     while (numThreadsWorking.get() != 0) {
       Threads.sleepWithoutInterrupt(1000);
     }
   }
 
+  public boolean isDone() {
+    return (numThreadsWorking.get() == 0);
+  }
+
   protected void startThreads(Collection<? extends Thread> threads) {
     numThreadsWorking.addAndGet(threads.size());
     for (Thread thread : threads) {
@@ -202,4 +265,77 @@ protected static void appendToStatus(StringBuilder sb, String desc,
     sb.append(v);
   }
 
+  /**
+   * See {@link #verifyResultAgainstDataGenerator(Result, boolean, boolean)}.
+   * Does not verify cf/column integrity.
+   */
+  public boolean verifyResultAgainstDataGenerator(Result result, boolean verifyValues) {
+    return verifyResultAgainstDataGenerator(result, verifyValues, false);
+  }
+
+  /**
+   * Verifies the result from get or scan using the dataGenerator (that was presumably
+   * also used to generate said result).
+   * @param verifyValues verify that values in the result make sense for row/cf/column combination
+   * @param verifyCfAndColumnIntegrity verify that cf/column set in the result is complete. Note
+   *                                   that to use this multiPut should be used, or verification
+   *                                   has to happen after writes, otherwise there can be races.
+   * @return
+   */
+  public boolean verifyResultAgainstDataGenerator(Result result, boolean verifyValues,
+      boolean verifyCfAndColumnIntegrity) {
+    String rowKeyStr = Bytes.toString(result.getRow());
+
+    // See if we have any data at all.
+    if (result.isEmpty()) {
+      LOG.error("No data returned for key = [" + rowKeyStr + "]");
+      return false;
+    }
+
+    if (!verifyValues && !verifyCfAndColumnIntegrity) {
+      return true; // as long as we have something, we are good.
+    }
+
+    // See if we have all the CFs.
+    byte[][] expectedCfs = dataGenerator.getColumnFamilies();
+    if (verifyCfAndColumnIntegrity && (expectedCfs.length != result.getMap().size())) {
+      LOG.error("Bad family count for [" + rowKeyStr + "]: " + result.getMap().size());
+      return false;
+    }
+
+    // Verify each column family from get in the result.
+    for (byte[] cf : result.getMap().keySet()) {
+      String cfStr = Bytes.toString(cf);
+      Map<byte[], byte[]> columnValues = result.getFamilyMap(cf);
+      if (columnValues == null) {
+        LOG.error("No data for family [" + cfStr + "] for [" + rowKeyStr + "]");
+        return false;
+      }
+      // See if we have correct columns.
+      if (verifyCfAndColumnIntegrity
+          && !dataGenerator.verify(result.getRow(), cf, columnValues.keySet())) {
+        String colsStr = "";
+        for (byte[] col : columnValues.keySet()) {
+          if (colsStr.length() > 0) {
+            colsStr += ", ";
+          }
+          colsStr += "[" + Bytes.toString(col) + "]";
+        }
+        LOG.error("Bad columns for family [" + cfStr + "] for [" + rowKeyStr + "]: " + colsStr);
+        return false;
+      }
+      // See if values check out.
+      if (verifyValues) {
+        for (Map.Entry<byte[], byte[]> kv : columnValues.entrySet()) {
+          if (!dataGenerator.verify(result.getRow(), cf, kv.getKey(), kv.getValue())) {
+            LOG.error("Error checking data for key [" + rowKeyStr + "], column family ["
+              + cfStr + "], column [" + Bytes.toString(kv.getKey()) + "]; value of length " +
+              + kv.getValue().length);
+            return false;
+          }
+        }
+      }
+    }
+    return true;
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
index a05a121bdb62..e42152f4b721 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
@@ -18,7 +18,6 @@
 
 import java.io.IOException;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
@@ -27,14 +26,13 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 
 /** Creates multiple threads that read and verify previously written data */
-public class MultiThreadedReader extends MultiThreadedAction
-{
+public class MultiThreadedReader extends MultiThreadedAction {
+
   private static final Log LOG = LogFactory.getLog(MultiThreadedReader.class);
 
   private Set<HBaseReaderThread> readers = new HashSet<HBaseReaderThread>();
@@ -71,9 +69,9 @@ public class MultiThreadedReader extends MultiThreadedAction
   private int maxErrors = DEFAULT_MAX_ERRORS;
   private int keyWindow = DEFAULT_KEY_WINDOW;
 
-  public MultiThreadedReader(Configuration conf, byte[] tableName,
-      byte[] columnFamily, double verifyPercent) {
-    super(conf, tableName, columnFamily, "R");
+  public MultiThreadedReader(LoadTestDataGenerator dataGen, Configuration conf,
+      byte[] tableName, double verifyPercent) {
+    super(dataGen, conf, tableName, "R");
     this.verifyPercent = verifyPercent;
   }
 
@@ -222,14 +220,22 @@ private long getNextKeyToRead() {
     }
 
     private Get readKey(long keyToRead) {
-      Get get = new Get(
-          LoadTestKVGenerator.md5PrefixedKey(keyToRead).getBytes());
-      get.addFamily(columnFamily);
+      Get get = new Get(dataGenerator.getDeterministicUniqueKey(keyToRead));
+      String cfsString = "";
+      byte[][] columnFamilies = dataGenerator.getColumnFamilies();
+      for (byte[] cf : columnFamilies) {
+        get.addFamily(cf);
+        if (verbose) {
+          if (cfsString.length() > 0) {
+            cfsString += ", ";
+          }
+          cfsString += "[" + Bytes.toStringBinary(cf) + "]";
+        }
+      }
 
       try {
         if (verbose) {
-          LOG.info("[" + readerId + "] " + "Querying key " + keyToRead
-              + ", cf " + Bytes.toStringBinary(columnFamily));
+          LOG.info("[" + readerId + "] " + "Querying key " + keyToRead + ", cfs " + cfsString);
         }
         queryKey(get, random.nextInt(100) < verifyPercent);
       } catch (IOException e) {
@@ -251,45 +257,37 @@ public void queryKey(Get get, boolean verify) throws IOException {
       numKeys.addAndGet(1);
 
       // if we got no data report error
-      if (result.isEmpty()) {
-         HRegionLocation hloc = table.getRegionLocation(
-             Bytes.toBytes(rowKey));
+      if (!result.isEmpty()) {
+        if (verify) {
+          numKeysVerified.incrementAndGet();
+        }
+      } else {
+        HRegionLocation hloc = table.getRegionLocation(Bytes.toBytes(rowKey));
         LOG.info("Key = " + rowKey + ", RegionServer: "
             + hloc.getHostname());
-        numReadErrors.addAndGet(1);
-        LOG.error("No data returned, tried to get actions for key = "
-            + rowKey + (writer == null ? "" : ", keys inserted by writer: " +
-                writer.numKeys.get() + ")"));
-
-         if (numReadErrors.get() > maxErrors) {
-          LOG.error("Aborting readers -- found more than " + maxErrors
-              + " errors\n");
-           aborted = true;
-         }
       }
 
-      if (result.getFamilyMap(columnFamily) != null) {
-        // increment number of columns read
-        numCols.addAndGet(result.getFamilyMap(columnFamily).size());
-
-        if (verify) {
-          // verify the result
-          List<KeyValue> keyValues = result.list();
-          for (KeyValue kv : keyValues) {
-            String qual = new String(kv.getQualifier());
-
-            // if something does not look right report it
-            if (!LoadTestKVGenerator.verify(rowKey, qual, kv.getValue())) {
-              numReadErrors.addAndGet(1);
-              LOG.error("Error checking data for key = " + rowKey
-                  + ", actionId = " + qual);
-            }
-          }
-          numKeysVerified.addAndGet(1);
+      boolean isOk = verifyResultAgainstDataGenerator(result, verify);
+      long numErrorsAfterThis = 0;
+      if (isOk) {
+        long cols = 0;
+        // Count the columns for reporting purposes.
+        for (byte[] cf : result.getMap().keySet()) {
+          cols += result.getFamilyMap(cf).size();
+        }
+        numCols.addAndGet(cols);
+      } else {
+        if (writer != null) {
+          LOG.error("At the time of failure, writer inserted " + writer.numKeys.get() + " keys");
         }
+        numErrorsAfterThis = numReadErrors.incrementAndGet();
       }
-    }
 
+      if (numErrorsAfterThis > maxErrors) {
+        LOG.error("Aborting readers -- found more than " + maxErrors + " errors");
+        aborted = true;
+      }
+    }
   }
 
   public long getNumReadFailures() {
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
index 4bfc2a959a13..adc5b319474b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
@@ -20,7 +20,6 @@
 import java.util.HashSet;
 import java.util.PriorityQueue;
 import java.util.Queue;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
@@ -38,8 +37,6 @@
 public class MultiThreadedWriter extends MultiThreadedAction {
   private static final Log LOG = LogFactory.getLog(MultiThreadedWriter.class);
 
-  private long minColumnsPerKey = 1;
-  private long maxColumnsPerKey = 10;
   private Set<HBaseWriterThread> writers = new HashSet<HBaseWriterThread>();
 
   private boolean isMultiPut = false;
@@ -50,8 +47,7 @@ public class MultiThreadedWriter extends MultiThreadedAction {
    * {@link #insertedUpToKey}, the maximum key in the contiguous range of keys
    * being inserted. This queue is supposed to stay small.
    */
-  private BlockingQueue<Long> insertedKeys =
-      new ArrayBlockingQueue<Long>(10000);
+  private BlockingQueue<Long> insertedKeys = new ArrayBlockingQueue<Long>(10000);
 
   /**
    * This is the current key to be inserted by any thread. Each thread does an
@@ -77,9 +73,9 @@ public class MultiThreadedWriter extends MultiThreadedAction {
   /** Enable this if used in conjunction with a concurrent reader. */
   private boolean trackInsertedKeys;
 
-  public MultiThreadedWriter(Configuration conf, byte[] tableName,
-      byte[] columnFamily) {
-    super(conf, tableName, columnFamily, "W");
+  public MultiThreadedWriter(LoadTestDataGenerator dataGen, Configuration conf,
+      byte[] tableName) {
+    super(dataGen, conf, tableName, "W");
   }
 
   /** Use multi-puts vs. separate puts for every column in a row */
@@ -87,11 +83,6 @@ public void setMultiPut(boolean isMultiPut) {
     this.isMultiPut = isMultiPut;
   }
 
-  public void setColumnsPerKey(long minColumnsPerKey, long maxColumnsPerKey) {
-    this.minColumnsPerKey = minColumnsPerKey;
-    this.maxColumnsPerKey = maxColumnsPerKey;
-  }
-
   @Override
   public void start(long startKey, long endKey, int numThreads)
       throws IOException {
@@ -117,17 +108,9 @@ public void start(long startKey, long endKey, int numThreads)
     startThreads(writers);
   }
 
-  public static byte[] longToByteArrayKey(long rowKey) {
-    return LoadTestKVGenerator.md5PrefixedKey(rowKey).getBytes();
-  }
-
   private class HBaseWriterThread extends Thread {
     private final HTable table;
 
-    private final Random random = new Random();
-    private final LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(
-        minDataSize, maxDataSize);
-
     public HBaseWriterThread(int writerId) throws IOException {
       setName(getClass().getSimpleName() + "_" + writerId);
       table = new HTable(conf, tableName);
@@ -135,20 +118,35 @@ public HBaseWriterThread(int writerId) throws IOException {
 
     public void run() {
       try {
-        long rowKey;
-        while ((rowKey = nextKeyToInsert.getAndIncrement()) < endKey) {
-          long numColumns = minColumnsPerKey + Math.abs(random.nextLong())
-              % (maxColumnsPerKey - minColumnsPerKey);
+        long rowKeyBase;
+        byte[][] columnFamilies = dataGenerator.getColumnFamilies();
+        while ((rowKeyBase = nextKeyToInsert.getAndIncrement()) < endKey) {
+          byte[] rowKey = dataGenerator.getDeterministicUniqueKey(rowKeyBase);
+          Put put = new Put(rowKey);
           numKeys.addAndGet(1);
+          int columnCount = 0;
+          for (byte[] cf : columnFamilies) {
+            byte[][] columns = dataGenerator.generateColumnsForCf(rowKey, cf);
+            for (byte[] column : columns) {
+              byte[] value = dataGenerator.generateValue(rowKey, cf, column);
+              put.add(cf, column, value);
+              ++columnCount;
+              if (!isMultiPut) {
+                insert(put, rowKeyBase);
+                numCols.addAndGet(1);
+                put = new Put(rowKey);
+              }
+            }
+          }
           if (isMultiPut) {
-            multiPutInsertKey(rowKey, 0, numColumns);
-          } else {
-            for (long col = 0; col < numColumns; ++col) {
-              insert(rowKey, col);
+            if (verbose) {
+              LOG.debug("Preparing put for key = [" + rowKey + "], " + columnCount + " columns");
             }
+            insert(put, rowKeyBase);
+            numCols.addAndGet(columnCount);
           }
           if (trackInsertedKeys) {
-            insertedKeys.add(rowKey);
+            insertedKeys.add(rowKeyBase);
           }
         }
       } finally {
@@ -161,52 +159,14 @@ public void run() {
       }
     }
 
-    public void insert(long rowKey, long col) {
-      Put put = new Put(longToByteArrayKey(rowKey));
-      String colAsStr = String.valueOf(col);
-      put.add(columnFamily, Bytes.toBytes(colAsStr),
-          dataGenerator.generateRandomSizeValue(rowKey, colAsStr));
+    public void insert(Put put, long keyBase) {
       try {
         long start = System.currentTimeMillis();
         table.put(put);
-        numCols.addAndGet(1);
         totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
       } catch (IOException e) {
-        failedKeySet.add(rowKey);
-        LOG.error("Failed to insert: " + rowKey);
-        e.printStackTrace();
-      }
-    }
-
-    public void multiPutInsertKey(long rowKey, long startCol, long endCol) {
-      if (verbose) {
-        LOG.debug("Preparing put for key = " + rowKey + ", cols = ["
-            + startCol + ", " + endCol + ")");
-      }
-
-      if (startCol >= endCol) {
-        return;
-      }
-
-      Put put = new Put(LoadTestKVGenerator.md5PrefixedKey(
-          rowKey).getBytes());
-      byte[] columnQualifier;
-      byte[] value;
-      for (long i = startCol; i < endCol; ++i) {
-        String qualStr = String.valueOf(i);
-        columnQualifier = qualStr.getBytes();
-        value = dataGenerator.generateRandomSizeValue(rowKey, qualStr);
-        put.add(columnFamily, columnQualifier, value);
-      }
-
-      try {
-        long start = System.currentTimeMillis();
-        table.put(put);
-        numCols.addAndGet(endCol - startCol);
-        totalOpTimeMs.addAndGet(
-            System.currentTimeMillis() - start);
-      } catch (IOException e) {
-        failedKeySet.add(rowKey);
+        failedKeySet.add(keyBase);
+        LOG.error("Failed to insert: " + keyBase);
         e.printStackTrace();
       }
     }
@@ -301,7 +261,7 @@ protected String progressInfo() {
    * key, which requires a blocking queue and a consumer thread.
    * @param enable whether to enable tracking the last inserted key
    */
-  void setTrackInsertedKeys(boolean enable) {
+  public void setTrackInsertedKeys(boolean enable) {
     trackInsertedKeys = enable;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java b/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
index b49aaee3785c..3b4b66c72571 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
@@ -62,8 +62,8 @@ public class RestartMetaTest extends AbstractHBaseTool {
   private void loadData() throws IOException {
     long startKey = 0;
     long endKey = 100000;
-    long minColsPerKey = 5;
-    long maxColsPerKey = 15;
+    int minColsPerKey = 5;
+    int maxColsPerKey = 15;
     int minColDataSize = 256;
     int maxColDataSize = 256 * 3;
     int numThreads = 10;
@@ -77,11 +77,10 @@ private void loadData() throws IOException {
     System.out.printf("Client Threads: %d\n", numThreads);
 
     // start the writers
-    MultiThreadedWriter writer = new MultiThreadedWriter(conf, TABLE_NAME,
-        LoadTestTool.COLUMN_FAMILY);
+    LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(
+      minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, LoadTestTool.COLUMN_FAMILY);
+    MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME);
     writer.setMultiPut(true);
-    writer.setColumnsPerKey(minColsPerKey, maxColsPerKey);
-    writer.setDataSize(minColDataSize, maxColDataSize);
     writer.start(startKey, endKey, numThreads);
     System.out.printf("Started loading data...");
     writer.waitForFinish();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
index 43aad3b40ada..6dc07f123eaa 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
@@ -39,8 +39,8 @@ public class TestLoadTestKVGenerator {
   @Test
   public void testValueLength() {
     for (int i = 0; i < 1000; ++i) {
-      byte[] v = gen.generateRandomSizeValue(i,
-          String.valueOf(rand.nextInt()));
+      byte[] v = gen.generateRandomSizeValue(Integer.toString(i).getBytes(),
+        String.valueOf(rand.nextInt()).getBytes());
       assertTrue(MIN_LEN <= v.length);
       assertTrue(v.length <= MAX_LEN);
     }
@@ -50,12 +50,12 @@ public void testValueLength() {
   public void testVerification() {
     for (int i = 0; i < 1000; ++i) {
       for (int qualIndex = 0; qualIndex < 20; ++qualIndex) {
-        String qual = String.valueOf(qualIndex);
-        byte[] v = gen.generateRandomSizeValue(i, qual);
-        String rowKey = LoadTestKVGenerator.md5PrefixedKey(i);
-        assertTrue(LoadTestKVGenerator.verify(rowKey, qual, v));
+        byte[] qual = String.valueOf(qualIndex).getBytes();
+        byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
+        byte[] v = gen.generateRandomSizeValue(rowKey, qual);
+        assertTrue(LoadTestKVGenerator.verify(v, rowKey, qual));
         v[0]++;
-        assertFalse(LoadTestKVGenerator.verify(rowKey, qual, v));
+        assertFalse(LoadTestKVGenerator.verify(v, rowKey, qual));
       }
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
index 90f7c6cd0de1..15fe7eb719c2 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
@@ -136,9 +136,10 @@ protected void prepareForLoadTest() throws IOException {
 
     TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
 
-    writerThreads = new MultiThreadedWriter(conf, TABLE, CF);
+    LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF);
+    writerThreads = new MultiThreadedWriter(dataGen, conf, TABLE);
     writerThreads.setMultiPut(isMultiPut);
-    readerThreads = new MultiThreadedReader(conf, TABLE, CF, 100);
+    readerThreads = new MultiThreadedReader(dataGen, conf, TABLE, 100);
   }
 
   protected int numKeys() {

From 542b8dcc1fced55d6a5a71dcd840af254f3c918a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 29 Mar 2013 16:09:30 +0000
Subject: [PATCH 0878/1540] HBASE-8207 Replication could have data loss when
 machine name contains hyphen "-" (Jeffrey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462518 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/ReplicationZookeeper.java     |  5 +-
 .../regionserver/ReplicationSource.java       | 76 ++++++++++++++++---
 .../replication/TestReplicationBase.java      |  2 +-
 .../TestReplicationSourceManager.java         | 42 +++++++++-
 4 files changed, 111 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 290741e239e4..a7aa59edf22b 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -679,7 +679,10 @@ public SortedMap<String, SortedSet<String>> copyQueuesFromRSUsingMulti(String zn
         // check the logs queue for the old peer cluster
         String oldClusterZnode = ZKUtil.joinZNode(deadRSZnodePath, peerId);
         List<String> hlogs = ZKUtil.listChildrenNoWatch(this.zookeeper, oldClusterZnode);
-        if (hlogs == null || hlogs.size() == 0) continue; // empty log queue.
+        if (hlogs == null || hlogs.size() == 0) {
+          listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
+          continue; // empty log queue.
+        }
         // create the new cluster znode
         SortedSet<String> logQueue = new TreeSet<String>();
         queues.put(newPeerId, logQueue);
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 84648fb48a6c..54019b13e24f 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -26,6 +26,7 @@
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
@@ -122,7 +123,7 @@ public class ReplicationSource extends Thread
   // Indicates if this queue is recovered (and will be deleted when depleted)
   private boolean queueRecovered;
   // List of all the dead region servers that had this queue (if recovered)
-  private String[] deadRegionServers;
+  private List<String> deadRegionServers = new ArrayList<String>();
   // Maximum number of retries before taking bold actions
   private int maxRetriesMultiplier;
   // Socket timeouts require even bolder actions since we don't want to DDOS
@@ -200,17 +201,71 @@ public void init(final Configuration conf,
 
   // The passed znode will be either the id of the peer cluster or
   // the handling story of that queue in the form of id-servername-*
-  private void checkIfQueueRecovered(String peerClusterZnode) {
-    String[] parts = peerClusterZnode.split("-");
+  //
+  // package access for testing
+  void checkIfQueueRecovered(String peerClusterZnode) {
+    String[] parts = peerClusterZnode.split("-", 2);
     this.queueRecovered = parts.length != 1;
     this.peerId = this.queueRecovered ?
         parts[0] : peerClusterZnode;
     this.peerClusterZnode = peerClusterZnode;
-    this.deadRegionServers = new String[parts.length-1];
-    // Extract all the places where we could find the hlogs
-    for (int i = 1; i < parts.length; i++) {
-      this.deadRegionServers[i-1] = parts[i];
+
+    if (parts.length < 2) {
+      // not queue recovered situation
+      return;
     }
+
+    // extract dead servers
+    extracDeadServersFromZNodeString(parts[1], this.deadRegionServers);
+  }
+
+  /**
+   * for tests only
+   */
+  List<String> getDeadRegionServers() {
+    return Collections.unmodifiableList(this.deadRegionServers);
+  }
+
+  /**
+   * Parse dead server names from znode string servername can contain "-" such as
+   * "ip-10-46-221-101.ec2.internal", so we need skip some "-" during parsing for the following
+   * cases: 2-ip-10-46-221-101.ec2.internal,52170,1364333181125-<server name>-...
+   */
+  private static void
+      extracDeadServersFromZNodeString(String deadServerListStr, List<String> result) {
+
+    if (deadServerListStr == null || result == null || deadServerListStr.isEmpty()) return;
+
+    // valid server name delimiter "-" has to be after "," in a server name
+    int seenCommaCnt = 0;
+    int startIndex = 0;
+    int len = deadServerListStr.length();
+
+    for (int i = 0; i < len; i++) {
+      switch (deadServerListStr.charAt(i)) {
+      case ',':
+        seenCommaCnt += 1;
+        break;
+      case '-':
+        if (seenCommaCnt >= 2) {
+          if (i > startIndex) {
+            result.add(deadServerListStr.substring(startIndex, i));
+            startIndex = i + 1;
+          }
+          seenCommaCnt = 0;
+        }
+        break;
+      default:
+        break;
+      }
+    }
+
+    // add tail
+    if (startIndex < len - 1) {
+      result.add(deadServerListStr.substring(startIndex, len));
+    }
+
+    LOG.debug("Found dead servers:" + result);
   }
 
   /**
@@ -510,11 +565,10 @@ protected boolean openReader(int sleepMultiplier) {
           // We didn't find the log in the archive directory, look if it still
           // exists in the dead RS folder (there could be a chain of failures
           // to look at)
-          LOG.info("NB dead servers : " + deadRegionServers.length);
-          for (int i = this.deadRegionServers.length - 1; i >= 0; i--) {
-
+          LOG.info("NB dead servers : " + deadRegionServers.size());
+          for (String curDeadServerName : deadRegionServers) {
             Path deadRsDirectory =
-                new Path(manager.getLogDir().getParent(), this.deadRegionServers[i]);
+                new Path(manager.getLogDir().getParent(), curDeadServerName);
             Path[] locs = new Path[] {
                 new Path(deadRsDirectory, currentPath.getName()),
                 new Path(deadRsDirectory.suffix(HLog.SPLITTING_EXT),
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index 938109f43f6c..aabaedc77b75 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -79,7 +79,7 @@ public static void setUpBeforeClass() throws Exception {
     conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
     // smaller log roll size to trigger more events
     conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
-    conf1.setInt("replication.source.size.capacity", 1024);
+    conf1.setInt("replication.source.size.capacity", 10240);
     conf1.setLong("replication.source.sleepforretries", 100);
     conf1.setInt("hbase.regionserver.maxlogs", 10);
     conf1.setLong("hbase.master.logcleaner.ttl", 10);
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index e11703eaf5e7..f14fed94b15f 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.replication.regionserver;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.net.URLEncoder;
 import java.util.ArrayList;
@@ -264,6 +265,45 @@ public void testNodeFailoverWorkerCopyQueuesFromRSUsingMulti() throws Exception
     server.abort("", null);
   }
 
+  @Test
+  public void testNodeFailoverDeadServerParsing() throws Exception {
+    LOG.debug("testNodeFailoverDeadServerParsing");
+    conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
+    final Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
+    AtomicBoolean replicating = new AtomicBoolean(true);
+    ReplicationZookeeper rz = new ReplicationZookeeper(server, replicating);
+    // populate some znodes in the peer znode
+    files.add("log1");
+    files.add("log2");
+    for (String file : files) {
+      rz.addLogToList(file, "1");
+    }
+    // create 3 DummyServers
+    Server s1 = new DummyServer("ip-10-8-101-114.ec2.internal");
+    Server s2 = new DummyServer("ec2-107-20-52-47.compute-1.amazonaws.com");
+    Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com");
+
+    // simulate three server fail sequentially
+    ReplicationZookeeper rz1 = new ReplicationZookeeper(s1, new AtomicBoolean(true));
+    SortedMap<String, SortedSet<String>> testMap =
+        rz1.copyQueuesFromRSUsingMulti(server.getServerName().getServerName());
+    ReplicationZookeeper rz2 = new ReplicationZookeeper(s2, new AtomicBoolean(true));
+    testMap = rz2.copyQueuesFromRSUsingMulti(s1.getServerName().getServerName());
+    ReplicationZookeeper rz3 = new ReplicationZookeeper(s3, new AtomicBoolean(true));
+    testMap = rz3.copyQueuesFromRSUsingMulti(s2.getServerName().getServerName());
+
+    ReplicationSource s = new ReplicationSource();
+    s.checkIfQueueRecovered(testMap.firstKey());
+    List<String> result = s.getDeadRegionServers();
+
+    // verify
+    assertTrue(result.contains(server.getServerName().getServerName()));
+    assertTrue(result.contains(s1.getServerName().getServerName()));
+    assertTrue(result.contains(s2.getServerName().getServerName()));
+
+    server.abort("", null);
+  }
+
   static class DummyNodeFailoverWorker extends Thread {
     private SortedMap<String, SortedSet<String>> logZnodesMap;
     Server server;
@@ -338,7 +378,7 @@ public CatalogTracker getCatalogTracker() {
 
     @Override
     public ServerName getServerName() {
-      return new ServerName(hostname, 1234, -1L);
+      return new ServerName(hostname, 1234, 1L);
     }
 
     @Override

From 6930ff607f5b54bfe43fc63e4ecb3340d76c9e09 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 29 Mar 2013 17:34:04 +0000
Subject: [PATCH 0879/1540] HBASE-8207 Addendum fixes the typo in
 extractDeadServersFromZNodeString()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462554 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 54019b13e24f..9548f55f8451 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -216,7 +216,7 @@ void checkIfQueueRecovered(String peerClusterZnode) {
     }
 
     // extract dead servers
-    extracDeadServersFromZNodeString(parts[1], this.deadRegionServers);
+    extractDeadServersFromZNodeString(parts[1], this.deadRegionServers);
   }
 
   /**
@@ -232,7 +232,7 @@ List<String> getDeadRegionServers() {
    * cases: 2-ip-10-46-221-101.ec2.internal,52170,1364333181125-<server name>-...
    */
   private static void
-      extracDeadServersFromZNodeString(String deadServerListStr, List<String> result) {
+      extractDeadServersFromZNodeString(String deadServerListStr, List<String> result) {
 
     if (deadServerListStr == null || result == null || deadServerListStr.isEmpty()) return;
 

From 5c25844d83b40af8d430a69d44f9ecae50bb0877 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 29 Mar 2013 19:29:32 +0000
Subject: [PATCH 0880/1540] HBASE-8209. Improve LoadTest extensibility

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462618 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 75 +++++++++++++------
 .../hbase/coprocessor/TestMasterObserver.java |  2 +-
 ...onServerCoprocessorExceptionWithAbort.java |  2 +-
 ...nServerCoprocessorExceptionWithRemove.java |  2 +-
 .../hbase/master/TestMasterTransitions.java   |  2 +-
 .../hbase/util/MultiThreadedAction.java       |  9 +++
 .../hbase/util/MultiThreadedWriter.java       | 24 +++---
 .../hbase/util/TestLoadTestKVGenerator.java   |  2 +-
 .../util/TestMiniClusterLoadSequential.java   | 34 +++++++--
 9 files changed, 105 insertions(+), 47 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 7e4bffa5ea47..699c0dd01ba9 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1835,29 +1835,45 @@ public static void setMaxRecoveryErrorCount(final OutputStream stream,
    * @param countOfRegions How many regions in .META.
    * @throws IOException
    */
-  public void waitUntilAllRegionsAssigned(final int countOfRegions)
+  public void waitUntilAllRegionsAssigned(final byte[] tableName, final int countOfRegions)
   throws IOException {
+    int retries = 30; // We may wait up to 30 seconds
+    int rows = 0;
     HTable meta = new HTable(getConfiguration(), HConstants.META_TABLE_NAME);
-    while (true) {
-      int rows = 0;
-      Scan scan = new Scan();
-      scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-      ResultScanner s = meta.getScanner(scan);
-      for (Result r = null; (r = s.next()) != null;) {
-        byte [] b =
-          r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        if (b == null || b.length <= 0) {
+    try {
+      do {
+        Scan scan = new Scan();
+        scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+        scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+        ResultScanner s = meta.getScanner(scan);
+        try {
+          for (Result r = null; (r = s.next()) != null;) {
+            byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+            HRegionInfo hri = Writables.getHRegionInfoOrNull(b);
+            if (hri != null && Bytes.equals(hri.getTableName(), tableName)) {
+              b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+              if (b == null || b.length <= 0) {
+                continue;
+              }
+              rows++;
+            }
+          }
+        } finally {
+          s.close();
+        }
+        // If I get to here and all rows have a Server, then all have been assigned.
+        if (rows == countOfRegions) {
           break;
         }
-        rows++;
-      }
-      s.close();
-      // If I get to here and all rows have a Server, then all have been assigned.
-      if (rows == countOfRegions) {
-        break;
-      }
-      LOG.info("Found=" + rows);
-      Threads.sleep(200);
+        LOG.info("Found=" + rows);
+        Threads.sleep(1000);
+      } while (--retries > 0);
+    } finally {
+      meta.close();
+    }
+    if (rows != countOfRegions) {
+      throw new IOException("Timed out waiting for " + countOfRegions + " regions of " +
+        Bytes.toStringBinary(tableName) + " to come online");
     }
   }
 
@@ -2138,12 +2154,23 @@ public static int createPreSplitLoadTestTable(Configuration conf,
     HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
     hcd.setDataBlockEncoding(dataBlockEncoding);
     hcd.setCompressionType(compression);
-    desc.addFamily(hcd);
+    return createPreSplitLoadTestTable(conf, desc, hcd);
+  }
+
+  /**
+   * Creates a pre-split table for load testing. If the table already exists,
+   * logs a warning and continues.
+   * @return the number of regions the table was split into
+   */
+  public static int createPreSplitLoadTestTable(Configuration conf,
+      HTableDescriptor desc, HColumnDescriptor hcd) throws IOException {
+    if (!desc.hasFamily(hcd.getName())) {
+      desc.addFamily(hcd);
+    }
 
     int totalNumberOfRegions = 0;
+    HBaseAdmin admin = new HBaseAdmin(conf);
     try {
-      HBaseAdmin admin = new HBaseAdmin(conf);
-
       // create a table a pre-splits regions.
       // The number of splits is set as:
       //    region servers * regions per region server).
@@ -2166,8 +2193,10 @@ public static int createPreSplitLoadTestTable(Configuration conf,
       LOG.error("Master not running", e);
       throw new IOException(e);
     } catch (TableExistsException e) {
-      LOG.warn("Table " + Bytes.toStringBinary(tableName) +
+      LOG.warn("Table " + Bytes.toStringBinary(desc.getName()) +
           " already exists, continuing");
+    } finally {
+      admin.close();
     }
     return totalNumberOfRegions;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 00aefe08d4d7..3c831ba9ad60 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -748,7 +748,7 @@ public void testRegionTransitionOperations() throws Exception {
 
     HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
     int countOfRegions = UTIL.createMultiRegions(table, TEST_FAMILY);
-    UTIL.waitUntilAllRegionsAssigned(countOfRegions);
+    UTIL.waitUntilAllRegionsAssigned(TEST_TABLE, countOfRegions);
     
     NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
     Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
index 33756579719f..811196543804 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
@@ -109,7 +109,7 @@ public void testExceptionFromCoprocessorDuringPut()
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    TEST_UTIL.waitUntilAllRegionsAssigned(
+    TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE,
         TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
 
     // Note which regionServer will abort (after put is attempted).
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
index 10f44a6eb3f3..20b7b145c0a3 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
@@ -92,7 +92,7 @@ public void testExceptionFromCoprocessorDuringPut()
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    TEST_UTIL.waitUntilAllRegionsAssigned(
+    TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE,
         TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
     // Note which regionServer that should survive the buggy coprocessor's
     // prePut().
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
index e0a486134838..b893288988a6 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
@@ -62,7 +62,7 @@ public class TestMasterTransitions {
     TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
     int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
-    TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
+    TEST_UTIL.waitUntilAllRegionsAssigned(Bytes.toBytes(TABLENAME), countOfRegions);
     addToEachStartKey(countOfRegions);
     t.close();
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
index 44b1c9472d38..92eb11c86a72 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
@@ -265,6 +265,14 @@ protected static void appendToStatus(StringBuilder sb, String desc,
     sb.append(v);
   }
 
+  protected static void appendToStatus(StringBuilder sb, String desc,
+      String v) {
+    sb.append(", ");
+    sb.append(desc);
+    sb.append("=");
+    sb.append(v);
+  }
+
   /**
    * See {@link #verifyResultAgainstDataGenerator(Result, boolean, boolean)}.
    * Does not verify cf/column integrity.
@@ -338,4 +346,5 @@ public boolean verifyResultAgainstDataGenerator(Result result, boolean verifyVal
     }
     return true;
   }
+
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
index adc5b319474b..ea372cc84050 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
@@ -132,7 +132,7 @@ public void run() {
               put.add(cf, column, value);
               ++columnCount;
               if (!isMultiPut) {
-                insert(put, rowKeyBase);
+                insert(table, put, rowKeyBase);
                 numCols.addAndGet(1);
                 put = new Put(rowKey);
               }
@@ -142,7 +142,7 @@ public void run() {
             if (verbose) {
               LOG.debug("Preparing put for key = [" + rowKey + "], " + columnCount + " columns");
             }
-            insert(put, rowKeyBase);
+            insert(table, put, rowKeyBase);
             numCols.addAndGet(columnCount);
           }
           if (trackInsertedKeys) {
@@ -158,17 +158,17 @@ public void run() {
         numThreadsWorking.decrementAndGet();
       }
     }
+  }
 
-    public void insert(Put put, long keyBase) {
-      try {
-        long start = System.currentTimeMillis();
-        table.put(put);
-        totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
-      } catch (IOException e) {
-        failedKeySet.add(keyBase);
-        LOG.error("Failed to insert: " + keyBase);
-        e.printStackTrace();
-      }
+  public void insert(HTable table, Put put, long keyBase) {
+    try {
+      long start = System.currentTimeMillis();
+      table.put(put);
+      totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
+    } catch (IOException e) {
+      failedKeySet.add(keyBase);
+      LOG.error("Failed to insert: " + keyBase);
+      e.printStackTrace();
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
index 6dc07f123eaa..f6408b85316e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
@@ -64,7 +64,7 @@ public void testVerification() {
   public void testCorrectAndUniqueKeys() {
     Set<String> keys = new HashSet<String>();
     for (int i = 0; i < 1000; ++i) {
-      String k = LoadTestKVGenerator.md5PrefixedKey(i);
+      String k = gen.md5PrefixedKey(i);
       assertFalse(keys.contains(k));
       assertTrue(k.endsWith("-" + i));
       keys.add(k);
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
index 15fe7eb719c2..12bca7136c51 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -102,6 +103,19 @@ public void tearDown() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
+  protected MultiThreadedReader prepareReaderThreads(LoadTestDataGenerator dataGen,
+      Configuration conf, byte[] tableName, double verifyPercent) {
+    MultiThreadedReader reader = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent);
+    return reader;
+  }
+
+  protected MultiThreadedWriter prepareWriterThreads(LoadTestDataGenerator dataGen,
+      Configuration conf, byte[] tableName) {
+    MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, tableName);
+    writer.setMultiPut(isMultiPut);
+    return writer;
+  }
+
   @Test(timeout=TIMEOUT_MS)
   public void loadTest() throws Exception {
     prepareForLoadTest();
@@ -120,6 +134,12 @@ protected void runLoadTestOnExistingTable() throws IOException {
     assertEquals(numKeys, readerThreads.getNumKeysVerified());
   }
 
+  protected void createPreSplitLoadTestTable(HTableDescriptor htd, HColumnDescriptor hcd)
+      throws IOException {
+    int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf, htd, hcd);
+    TEST_UTIL.waitUntilAllRegionsAssigned(htd.getName(), numRegions);
+  }
+
   protected void prepareForLoadTest() throws IOException {
     LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding +
         ", isMultiPut=" + isMultiPut);
@@ -131,15 +151,15 @@ protected void prepareForLoadTest() throws IOException {
     }
     admin.close();
 
-    int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf,
-        TABLE, CF, compression, dataBlockEncoding);
-
-    TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
+    HTableDescriptor htd = new HTableDescriptor(TABLE);
+    HColumnDescriptor hcd = new HColumnDescriptor(CF)
+      .setCompressionType(compression)
+      .setDataBlockEncoding(dataBlockEncoding);
+    createPreSplitLoadTestTable(htd, hcd);
 
     LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF);
-    writerThreads = new MultiThreadedWriter(dataGen, conf, TABLE);
-    writerThreads.setMultiPut(isMultiPut);
-    readerThreads = new MultiThreadedReader(dataGen, conf, TABLE, 100);
+    writerThreads = prepareWriterThreads(dataGen, conf, TABLE);
+    readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100);
   }
 
   protected int numKeys() {

From 228b5979047056b999323a9b48f01fcb1b6fb6b4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 29 Mar 2013 19:46:43 +0000
Subject: [PATCH 0881/1540] HBASE-8176 Backport HBASE-5335 "Dynamic Schema
 Configurations" to 0.94 (clockfly)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462622 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HColumnDescriptor.java       |  90 +++-
 .../org/apache/hadoop/hbase/HConstants.java   |   1 +
 .../apache/hadoop/hbase/HTableDescriptor.java | 165 ++++---
 .../regionserver/CompoundConfiguration.java   | 466 ++++++++++++++++++
 .../hadoop/hbase/regionserver/HRegion.java    |  42 +-
 .../hbase/regionserver/SplitTransaction.java  |   2 +-
 .../hadoop/hbase/regionserver/Store.java      |  10 +-
 src/main/ruby/hbase.rb                        |   1 +
 src/main/ruby/hbase/admin.rb                  | 101 ++--
 src/main/ruby/shell/commands/alter.rb         |   6 +
 .../apache/hadoop/hbase/HBaseTestCase.java    |   4 +-
 .../hbase/client/TestFromClientSide3.java     | 260 ++++++++++
 .../coprocessor/TestCoprocessorInterface.java |   4 +-
 .../TestCompoundConfiguration.java            | 116 +++++
 .../regionserver/TestSplitTransaction.java    |   4 +-
 15 files changed, 1144 insertions(+), 128 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index b1cb9c317115..dca442d049ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -24,8 +24,9 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
-
+import java.util.Set;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
@@ -179,6 +180,8 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
   public static final boolean DEFAULT_EVICT_BLOCKS_ON_CLOSE = false;
 
   private final static Map<String, String> DEFAULT_VALUES = new HashMap<String, String>();
+  private final static Set<ImmutableBytesWritable> RESERVED_KEYWORDS
+    = new HashSet<ImmutableBytesWritable>();
   static {
       DEFAULT_VALUES.put(BLOOMFILTER, DEFAULT_BLOOMFILTER);
       DEFAULT_VALUES.put(REPLICATION_SCOPE, String.valueOf(DEFAULT_REPLICATION_SCOPE));
@@ -202,13 +205,16 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
           String.valueOf(DEFAULT_CACHE_BLOOMS_ON_WRITE));
       DEFAULT_VALUES.put(EVICT_BLOCKS_ON_CLOSE,
           String.valueOf(DEFAULT_EVICT_BLOCKS_ON_CLOSE));
+      for (String s : DEFAULT_VALUES.keySet()) {
+        RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s)));
+      }
   }
 
   // Column family name
   private byte [] name;
 
   // Column metadata
-  protected Map<ImmutableBytesWritable,ImmutableBytesWritable> values =
+  protected final Map<ImmutableBytesWritable,ImmutableBytesWritable> values =
     new HashMap<ImmutableBytesWritable,ImmutableBytesWritable>();
 
   /*
@@ -469,6 +475,7 @@ public String getValue(String key) {
    * @return All values.
    */
   public Map<ImmutableBytesWritable,ImmutableBytesWritable> getValues() {
+    // shallow pointer copy
     return Collections.unmodifiableMap(values);
   }
 
@@ -496,7 +503,11 @@ public void remove(final byte [] key) {
    * @return this (for chained invocation)
    */
   public HColumnDescriptor setValue(String key, String value) {
-    setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+    if (value == null) {
+      remove(Bytes.toBytes(key));
+    } else {
+      setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+    }
     return this;
   }
 
@@ -863,16 +874,7 @@ public String toString() {
     s.append(" => '");
     s.append(Bytes.toString(name));
     s.append("'");
-    for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
-        values.entrySet()) {
-      String key = Bytes.toString(e.getKey().get());
-      String value = Bytes.toString(e.getValue().get());
-      s.append(", ");
-      s.append(key);
-      s.append(" => '");
-      s.append(value);
-      s.append("'");
-    }
+    s.append(getValues(true));
     s.append('}');
     return s.toString();
   }
@@ -887,20 +889,62 @@ public String toStringCustomizedValues() {
     s.append(" => '");
     s.append(Bytes.toString(name));
     s.append("'");
-    for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
-        values.entrySet()) {
-      String key = Bytes.toString(e.getKey().get());
-      String value = Bytes.toString(e.getValue().get());
-      if(DEFAULT_VALUES.get(key) == null || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) {
+    s.append(getValues(false));
+    s.append('}');
+    return s.toString();
+  }
+
+  private StringBuilder getValues(boolean printDefaults) {
+    StringBuilder s = new StringBuilder();
+
+    boolean hasConfigKeys = false;
+
+    // print all reserved keys first
+    for (ImmutableBytesWritable k : values.keySet()) {
+      if (!RESERVED_KEYWORDS.contains(k)) {
+        hasConfigKeys = true;
+        continue;
+      }
+      String key = Bytes.toString(k.get());
+      String value = Bytes.toString(values.get(k).get());
+      if (printDefaults
+          || !DEFAULT_VALUES.containsKey(key)
+          || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) {
         s.append(", ");
         s.append(key);
-        s.append(" => '");
-        s.append(value);
-        s.append("'");
+
+        s.append(" => ");
+        s.append('\'').append(value).append('\'');
       }
     }
-    s.append('}');
-    return s.toString();
+
+    // print all non-reserved, advanced config keys as a separate subset
+    if (hasConfigKeys) {
+      s.append(", ");
+      s.append(HConstants.CONFIG).append(" => ");
+      s.append('{');
+      boolean printComma = false;
+      for (ImmutableBytesWritable k : values.keySet()) {
+        if (RESERVED_KEYWORDS.contains(k)) {
+          continue;
+        }
+        String key = Bytes.toString(k.get());
+        String value = Bytes.toString(values.get(k).get());
+        if (printComma) {
+          s.append(", ");
+        }
+        printComma = true;
+        s.append('\'').append(key).append('\'');
+        s.append(" => ");
+        s.append('\'').append(value).append('\'');
+      }
+      s.append('}');
+    }
+    return s;
+  }
+    
+  public static Map<String, String> getDefaultValues() {
+    return Collections.unmodifiableMap(DEFAULT_VALUES);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 485006b54a97..53254e530fe6 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -414,6 +414,7 @@ public enum OperationStatusCode {
   public static final String NAME = "NAME";
   public static final String VERSIONS = "VERSIONS";
   public static final String IN_MEMORY = "IN_MEMORY";
+  public static final String CONFIG = "CONFIG";
 
   /**
    * This is a retry backoff multiplier table similar to the BSD TCP syn
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index a20c383fe6d8..e9c17b146c39 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -30,6 +30,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.HashSet;
+import java.util.TreeSet;
 import java.util.TreeMap;
 import java.util.regex.Matcher;
 
@@ -65,7 +67,7 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
    * includes values like IS_ROOT, IS_META, DEFERRED_LOG_FLUSH, SPLIT_POLICY,
    * MAX_FILE_SIZE, READONLY, MEMSTORE_FLUSHSIZE etc...
    */
-  protected Map<ImmutableBytesWritable, ImmutableBytesWritable> values =
+  protected final Map<ImmutableBytesWritable, ImmutableBytesWritable> values =
     new HashMap<ImmutableBytesWritable, ImmutableBytesWritable>();
 
   private static final String FAMILIES = "FAMILIES";
@@ -160,6 +162,26 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
    * the contents are flushed to the store files
    */
   public static final long DEFAULT_MEMSTORE_FLUSH_SIZE = 1024*1024*128L;
+  private final static Map<String, String> DEFAULT_VALUES
+    = new HashMap<String, String>();
+  private final static Set<ImmutableBytesWritable> RESERVED_KEYWORDS
+    = new HashSet<ImmutableBytesWritable>();
+  static {
+    DEFAULT_VALUES.put(MAX_FILESIZE,
+        String.valueOf(HConstants.DEFAULT_MAX_FILE_SIZE));
+    DEFAULT_VALUES.put(READONLY, String.valueOf(DEFAULT_READONLY));
+    DEFAULT_VALUES.put(MEMSTORE_FLUSHSIZE,
+        String.valueOf(DEFAULT_MEMSTORE_FLUSH_SIZE));
+    DEFAULT_VALUES.put(DEFERRED_LOG_FLUSH,
+        String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH));
+    for (String s : DEFAULT_VALUES.keySet()) {
+      RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s)));
+    }
+    RESERVED_KEYWORDS.add(IS_ROOT_KEY);
+    RESERVED_KEYWORDS.add(IS_META_KEY);
+  }
+
+  
 
   private volatile Boolean meta = null;
   private volatile Boolean root = null;
@@ -429,7 +451,8 @@ public String getValue(String key) {
    * @see #values
    */
   public Map<ImmutableBytesWritable,ImmutableBytesWritable> getValues() {
-     return Collections.unmodifiableMap(values);
+    // shallow pointer copy 
+    return Collections.unmodifiableMap(values);
   }
 
   /**
@@ -469,7 +492,11 @@ private void setValue(final ImmutableBytesWritable key,
    * @see #values
    */
   public void setValue(String key, String value) {
-    setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+    if (value == null) {
+      remove(Bytes.toBytes(key));
+    } else {
+      setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+    }
   }
 
   /**
@@ -681,36 +708,11 @@ public boolean hasFamily(final byte [] familyName) {
   @Override
   public String toString() {
     StringBuilder s = new StringBuilder();
-    s.append('{');
-    s.append(HConstants.NAME);
-    s.append(" => '");
-    s.append(Bytes.toString(name));
-    s.append("'");
-    for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
-        values.entrySet()) {
-      String key = Bytes.toString(e.getKey().get());
-      String value = Bytes.toString(e.getValue().get());
-      if (key == null) {
-        continue;
-      }
-      String upperCase = key.toUpperCase();
-      if (upperCase.equals(IS_ROOT) || upperCase.equals(IS_META)) {
-        // Skip. Don't bother printing out read-only values if false.
-        if (value.toLowerCase().equals(Boolean.FALSE.toString())) {
-          continue;
-        }
-      }
-      s.append(", ");
-      s.append(Bytes.toString(e.getKey().get()));
-      s.append(" => '");
-      s.append(Bytes.toString(e.getValue().get()));
-      s.append("'");
+    s.append('\'').append(Bytes.toString(name)).append('\'');
+    s.append(getValues(true));
+    for (HColumnDescriptor f : families.values()) {
+      s.append(", ").append(f);
     }
-    s.append(", ");
-    s.append(FAMILIES);
-    s.append(" => ");
-    s.append(families.values());
-    s.append('}');
     return s.toString();
   }
 
@@ -720,44 +722,87 @@ public String toString() {
    */
   public String toStringCustomizedValues() {
     StringBuilder s = new StringBuilder();
-    s.append('{');
-    s.append(HConstants.NAME);
-    s.append(" => '");
-    s.append(Bytes.toString(name));
-    s.append("'");
-    for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
-        values.entrySet()) {
-      String key = Bytes.toString(e.getKey().get());
-      String value = Bytes.toString(e.getValue().get());
-      if (key == null) {
+    s.append('\'').append(Bytes.toString(name)).append('\'');
+    s.append(getValues(false));
+    for(HColumnDescriptor hcd : families.values()) {
+      s.append(", ").append(hcd.toStringCustomizedValues());
+    }
+    return s.toString();
+  }
+
+  private StringBuilder getValues(boolean printDefaults) {
+    StringBuilder s = new StringBuilder();
+
+    // step 1: set partitioning and pruning
+    Set<ImmutableBytesWritable> reservedKeys = new TreeSet<ImmutableBytesWritable>();
+    Set<ImmutableBytesWritable> configKeys = new TreeSet<ImmutableBytesWritable>();
+    for (ImmutableBytesWritable k : values.keySet()) {
+      if (k == null || k.get() == null) continue;
+      String key = Bytes.toString(k.get());
+      // in this section, print out reserved keywords + coprocessor info
+      if (!RESERVED_KEYWORDS.contains(k) && !key.startsWith("coprocessor$")) {
+        configKeys.add(k);        
         continue;
       }
-      String upperCase = key.toUpperCase();
-      if (upperCase.equals(IS_ROOT) || upperCase.equals(IS_META)) {
+      // only print out IS_ROOT/IS_META if true
+      String value = Bytes.toString(values.get(k).get());
+      if (key.equalsIgnoreCase(IS_ROOT) || key.equalsIgnoreCase(IS_META)) {
         // Skip. Don't bother printing out read-only values if false.
         if (value.toLowerCase().equals(Boolean.FALSE.toString())) {
           continue;
         }
       }
+
+      // see if a reserved key is a default value. may not want to print it out
+      if (printDefaults
+          || !DEFAULT_VALUES.containsKey(key)
+          || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) {
+        reservedKeys.add(k);
+      }
+    }
+
+
+
+    // early exit optimization
+    if (reservedKeys.isEmpty() && configKeys.isEmpty()) return s;
+
+    // step 2: printing
+    s.append(", {METHOD => 'table_att'");
+
+    // print all reserved keys first
+    for (ImmutableBytesWritable k : reservedKeys) {
+      String key = Bytes.toString(k.get());
+      String value = Bytes.toString(values.get(k).get());
+
       s.append(", ");
-      s.append(Bytes.toString(e.getKey().get()));
-      s.append(" => '");
-      s.append(Bytes.toString(e.getValue().get()));
-      s.append("'");
+      s.append(key);
+      s.append(" => ");
+      s.append('\'').append(value).append('\'');
     }
-    s.append(", ");
-    s.append(FAMILIES);
-    s.append(" => [");
-    int size = families.values().size();
-    int i = 0;
-    for(HColumnDescriptor hcd : families.values()) {
-      s.append(hcd.toStringCustomizedValues());
-      i++;
-      if( i != size)
-        s.append(", ");
+    if (!configKeys.isEmpty()) {
+      // print all non-reserved, advanced config keys as a separate subset
+      s.append(", ");
+      s.append(HConstants.CONFIG).append(" => ");
+      s.append("{");
+      boolean printComma = false;
+      for (ImmutableBytesWritable k : configKeys) {
+        String key = Bytes.toString(k.get());
+        String value = Bytes.toString(values.get(k).get());
+        if (printComma) s.append(", ");
+        printComma = true;
+        s.append('\'').append(key).append('\'');
+        s.append(" => ");
+        s.append('\'').append(value).append('\'');
+      }
+      s.append("}");
     }
-    s.append("]}");
-    return s.toString();
+
+    s.append('}'); // end METHOD
+    return s;
+  }
+                
+  public static Map<String, String> getDefaultValues() {
+    return Collections.unmodifiableMap(DEFAULT_VALUES);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
new file mode 100644
index 000000000000..0197a602b3e0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
@@ -0,0 +1,466 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Do a shallow merge of multiple KV configuration pools. This is a very useful
+ * utility class to easily add per-object configurations in addition to wider
+ * scope settings. This is different from Configuration.addResource()
+ * functionality, which performs a deep merge and mutates the common data
+ * structure.
+ * <p>
+ * For clarity: the shallow merge allows the user to mutate either of the
+ * configuration objects and have changes reflected everywhere. In contrast to a
+ * deep merge, that requires you to explicitly know all applicable copies to
+ * propagate changes.
+ * <p>
+ * This class is package private because we expect significant refactoring here
+ * on the HBase side when certain HDFS changes are added & ubiquitous. Will
+ * revisit expanding access at that point.
+ */
+@InterfaceAudience.Private
+class CompoundConfiguration extends Configuration {
+  /**
+   * Default Constructor. Initializes empty configuration
+   */
+  public CompoundConfiguration() {
+  }
+
+  // Devs: these APIs are the same contract as their counterparts in
+  // Configuration.java
+  private static interface ImmutableConfigMap {
+    String get(String key);
+    String getRaw(String key);
+    Class<?> getClassByName(String name) throws ClassNotFoundException;
+    int size();
+  }
+
+  protected List<ImmutableConfigMap> configs
+    = new ArrayList<ImmutableConfigMap>();
+
+  /****************************************************************************
+   * These initial APIs actually required original thought
+   ***************************************************************************/
+
+  /**
+   * Add Hadoop Configuration object to config list
+   * @param conf configuration object
+   * @return this, for builder pattern
+   */
+  public CompoundConfiguration add(final Configuration conf) {
+    if (conf instanceof CompoundConfiguration) {
+      this.configs.addAll(0, ((CompoundConfiguration) conf).configs);
+      return this;
+    }
+    // put new config at the front of the list (top priority)
+    this.configs.add(0, new ImmutableConfigMap() {
+      Configuration c = conf;
+
+      @Override
+      public String get(String key) {
+        return c.get(key);
+      }
+
+      @Override
+      public String getRaw(String key) {
+        return c.getRaw(key);
+      }
+
+      @Override
+      public Class<?> getClassByName(String name)
+          throws ClassNotFoundException {
+        return c.getClassByName(name);
+      }
+
+      @Override
+      public int size() {
+        return c.size();
+      }
+
+      @Override
+      public String toString() {
+        return c.toString();
+      }
+    });
+    return this;
+  }
+
+  /**
+   * Add ImmutableBytesWritable map to config list. This map is generally
+   * created by HTableDescriptor or HColumnDescriptor, but can be abstractly
+   * used.
+   *
+   * @param map
+   *          ImmutableBytesWritable map
+   * @return this, for builder pattern
+   */
+  public CompoundConfiguration add(
+      final Map<ImmutableBytesWritable, ImmutableBytesWritable> map) {
+    // put new map at the front of the list (top priority)
+    this.configs.add(0, new ImmutableConfigMap() {
+      Map<ImmutableBytesWritable, ImmutableBytesWritable> m = map;
+
+      @Override
+      public String get(String key) {
+        ImmutableBytesWritable ibw = new ImmutableBytesWritable(Bytes
+            .toBytes(key));
+        if (!m.containsKey(ibw))
+          return null;
+        ImmutableBytesWritable value = m.get(ibw);
+        if (value == null || value.get() == null)
+          return null;
+        return Bytes.toString(value.get());
+      }
+
+      @Override
+      public String getRaw(String key) {
+        return get(key);
+      }
+
+      @Override
+      public Class<?> getClassByName(String name)
+      throws ClassNotFoundException {
+        return null;
+      }
+
+      @Override
+      public int size() {
+        // TODO Auto-generated method stub
+        return m.size();
+      }
+
+      @Override
+      public String toString() {
+        return m.toString();
+      }
+    });
+    return this;
+  }
+
+  @Override
+  public String toString() {
+    StringBuffer sb = new StringBuffer();
+    sb.append("CompoundConfiguration: " + this.configs.size() + " configs");
+    for (ImmutableConfigMap m : this.configs) {
+      sb.append(this.configs);
+    }
+    return sb.toString();
+  }
+
+  @Override
+  public String get(String key) {
+    for (ImmutableConfigMap m : this.configs) {
+      String value = m.get(key);
+      if (value != null) {
+        return value;
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public String getRaw(String key) {
+    for (ImmutableConfigMap m : this.configs) {
+      String value = m.getRaw(key);
+      if (value != null) {
+        return value;
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public Class<?> getClassByName(String name) throws ClassNotFoundException {
+    for (ImmutableConfigMap m : this.configs) {
+      try {
+        Class<?> value = m.getClassByName(name);
+        if (value != null) {
+          return value;
+        }
+      } catch (ClassNotFoundException e) {
+        // don't propagate an exception until all configs fail
+        continue;
+      }
+    }
+    throw new ClassNotFoundException();
+  }
+
+  @Override
+  public int size() {
+    int ret = 0;
+    for (ImmutableConfigMap m : this.configs) {
+      ret += m.size();
+    }
+    return ret;
+  }
+
+  /***************************************************************************
+   * You should just ignore everything below this line unless there's a bug in
+   * Configuration.java...
+   *
+   * Below get APIs are directly copied from Configuration.java Oh, how I wish
+   * this wasn't so! A tragically-sad example of why you use interfaces instead
+   * of inheritance.
+   *
+   * Why the duplication? We basically need to override Configuration.getProps
+   * or we'd need protected access to Configuration.properties so we can modify
+   * that pointer. There are a bunch of functions in the base Configuration that
+   * call getProps() and we need to use our derived version instead of the base
+   * version. We need to make a generic implementation that works across all
+   * HDFS versions. We should modify Configuration.properties in HDFS 1.0 to be
+   * protected, but we still need to have this code until that patch makes it to
+   * all the HDFS versions we support.
+   ***************************************************************************/
+
+  @Override
+  public String get(String name, String defaultValue) {
+    String ret = get(name);
+    return ret == null ? defaultValue : ret;
+  }
+
+  @Override
+  public int getInt(String name, int defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      String hexString = getHexDigits(valueString);
+      if (hexString != null) {
+        return Integer.parseInt(hexString, 16);
+      }
+      return Integer.parseInt(valueString);
+    } catch (NumberFormatException e) {
+      return defaultValue;
+    }
+  }
+
+  @Override
+  public long getLong(String name, long defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      String hexString = getHexDigits(valueString);
+      if (hexString != null) {
+        return Long.parseLong(hexString, 16);
+      }
+      return Long.parseLong(valueString);
+    } catch (NumberFormatException e) {
+      return defaultValue;
+    }
+  }
+
+  protected String getHexDigits(String value) {
+    boolean negative = false;
+    String str = value;
+    String hexString = null;
+    if (value.startsWith("-")) {
+      negative = true;
+      str = value.substring(1);
+    }
+    if (str.startsWith("0x") || str.startsWith("0X")) {
+      hexString = str.substring(2);
+      if (negative) {
+        hexString = "-" + hexString;
+      }
+      return hexString;
+    }
+    return null;
+  }
+
+  @Override
+  public float getFloat(String name, float defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      return Float.parseFloat(valueString);
+    } catch (NumberFormatException e) {
+      return defaultValue;
+    }
+  }
+
+  @Override
+  public boolean getBoolean(String name, boolean defaultValue) {
+    String valueString = get(name);
+    if ("true".equals(valueString))
+      return true;
+    else if ("false".equals(valueString))
+      return false;
+    else return defaultValue;
+  }
+
+  @Override
+  public IntegerRanges getRange(String name, String defaultValue) {
+    return new IntegerRanges(get(name, defaultValue));
+  }
+
+  @Override
+  public Collection<String> getStringCollection(String name) {
+    String valueString = get(name);
+    return StringUtils.getStringCollection(valueString);
+  }
+
+  @Override
+  public String[] getStrings(String name) {
+    String valueString = get(name);
+    return StringUtils.getStrings(valueString);
+  }
+
+  @Override
+  public String[] getStrings(String name, String... defaultValue) {
+    String valueString = get(name);
+    if (valueString == null) {
+      return defaultValue;
+    } else {
+      return StringUtils.getStrings(valueString);
+    }
+  }
+
+  @Override
+  public Class<?>[] getClasses(String name, Class<?>... defaultValue) {
+    String[] classnames = getStrings(name);
+    if (classnames == null)
+      return defaultValue;
+    try {
+      Class<?>[] classes = new Class<?>[classnames.length];
+      for (int i = 0; i < classnames.length; i++) {
+        classes[i] = getClassByName(classnames[i]);
+      }
+      return classes;
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public Class<?> getClass(String name, Class<?> defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      return getClassByName(valueString);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public <U> Class<? extends U> getClass(String name,
+      Class<? extends U> defaultValue, Class<U> xface) {
+    try {
+      Class<?> theClass = getClass(name, defaultValue);
+      if (theClass != null && !xface.isAssignableFrom(theClass))
+        throw new RuntimeException(theClass + " not " + xface.getName());
+      else if (theClass != null)
+        return theClass.asSubclass(xface);
+      else
+        return null;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /*******************************************************************
+   * This class is immutable. Quickly abort any attempts to alter it *
+   *******************************************************************/
+
+  @Override
+  public void clear() {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public Iterator<Map.Entry<String, String>> iterator() {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void set(String name, String value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setIfUnset(String name, String value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setInt(String name, int value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setLong(String name, long value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setFloat(String name, float value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setBoolean(String name, boolean value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setBooleanIfUnset(String name, boolean value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setStrings(String name, String... values) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setClass(String name, Class<?> theClass, Class<?> xface) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setClassLoader(ClassLoader classLoader) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void writeXml(OutputStream out) throws IOException {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+};
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4e592d9bdfd9..711a6a30f7c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -243,6 +243,7 @@ public class HRegion implements HeapSize { // , Writable{
   private final HLog log;
   private final FileSystem fs;
   private final Configuration conf;
+  final Configuration baseConf;
   private final int rowLockWaitDuration;
   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
 
@@ -378,6 +379,7 @@ public HRegion(){
     this.conf = null;
     this.rowLockWaitDuration = DEFAULT_ROWLOCK_WAIT_DURATION;
     this.rsServices = null;
+    this.baseConf = null;
     this.fs = null;
     this.timestampSlop = HConstants.LATEST_TIMESTAMP;
     this.memstoreFlushSize = 0L;
@@ -396,6 +398,17 @@ public HRegion(){
     this.deferredLogSyncDisabled = false;
   }
 
+  
+  /**
+   * HRegion copy constructor. Useful when reopening a closed region (normally
+   * for unit tests)
+   * @param other original object
+   */
+  public HRegion(HRegion other) {
+    this(other.getTableDir(), other.getLog(), other.getFilesystem(),
+        other.baseConf, other.getRegionInfo(), other.getTableDesc(), null);
+  }
+  
   /**
    * HRegion constructor.  his constructor should only be used for testing and
    * extensions.  Instances of HRegion should be instantiated with the
@@ -419,14 +432,24 @@ public HRegion(){
    *
    * @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, HRegionInfo, HTableDescriptor, RegionServerServices)
    */
-  public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
+  public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration confParam,
     final HRegionInfo regionInfo, final HTableDescriptor htd,
       RegionServerServices rsServices) {
     this.tableDir = tableDir;
     this.comparator = regionInfo.getComparator();
     this.log = log;
     this.fs = fs;
-    this.conf = conf;
+    if (confParam instanceof CompoundConfiguration) {
+       throw new IllegalArgumentException("Need original base configuration");
+    }
+    // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
+    this.baseConf = confParam;
+    if (htd != null) {
+      this.conf = new CompoundConfiguration().add(confParam).add(htd.getValues());
+    }
+    else {
+      this.conf = new CompoundConfiguration().add(confParam);
+    }
     this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
                     DEFAULT_ROWLOCK_WAIT_DURATION);
 
@@ -1159,6 +1182,17 @@ public Configuration getConf() {
     return this.conf;
   }
 
+    /**
+   * A split takes the config from the parent region & passes it to the daughter
+   * region's constructor. If 'conf' was passed, you would end up using the HTD
+   * of the parent region in addition to the new daughter HTD. Pass 'baseConf'
+   * to the daughter regions to avoid this tricky dedupe problem.
+   * @return Configuration object
+   */
+  Configuration getBaseConf() {
+    return this.baseConf;
+  }
+
   /** @return region directory Path */
   public Path getRegionDir() {
     return this.regiondir;
@@ -4493,7 +4527,7 @@ public static HRegion merge(HRegion a, HRegion b)
       listPaths(fs, b.getRegionDir());
     }
 
-    Configuration conf = a.getConf();
+    Configuration conf = a.getBaseConf();
     HTableDescriptor tabledesc = a.getTableDesc();
     HLog log = a.getLog();
     Path tableDir = a.getTableDir();
@@ -5354,7 +5388,7 @@ private void checkFamily(final byte [] family)
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      35 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+      36 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
       (7 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 058393de3155..8744c7419a7c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -697,7 +697,7 @@ HRegion createDaughterRegion(final HRegionInfo hri,
     Path regionDir = getSplitDirForDaughter(this.parent.getFilesystem(),
       this.splitdir, hri);
     HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
-      this.parent.getLog(), fs, this.parent.getConf(),
+      this.parent.getLog(), fs, this.parent.getBaseConf(),
       hri, this.parent.getTableDesc(), rsServices);
     long halfParentReadRequestCount = this.parent.getReadRequestsCount() / 2;
     r.readRequestsCount.set(halfParentReadRequestCount);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index f357583c6097..3a21a273dc43 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -179,9 +179,10 @@ public class Store extends SchemaConfigured implements HeapSize {
    * @throws IOException
    */
   protected Store(Path basedir, HRegion region, HColumnDescriptor family,
-    FileSystem fs, Configuration conf)
+      FileSystem fs, Configuration confParam)
   throws IOException {
-    super(conf, region.getRegionInfo().getTableNameAsString(),
+    super(new CompoundConfiguration().add(confParam).add(
+        family.getValues()), region.getTableDesc().getNameAsString(),
         Bytes.toString(family.getName()));
     HRegionInfo info = region.getRegionInfo();
     this.fs = fs;
@@ -189,7 +190,8 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.homedir = createStoreHomeDir(this.fs, p);
     this.region = region;
     this.family = family;
-    this.conf = conf;
+    // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
+    this.conf = new CompoundConfiguration().add(confParam).add(family.getValues());
     this.blocksize = family.getBlocksize();
 
     this.dataBlockEncoder =
@@ -221,6 +223,8 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       conf.getInt("hbase.hstore.compaction.min",
         /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
 
+    LOG.info("hbase.hstore.compaction.min = " + this.minFilesToCompact);
+    
     // Setting up cache configuration for this family
     this.cacheConf = new CacheConfig(conf, family);
     this.blockingStoreFileCount =
diff --git a/src/main/ruby/hbase.rb b/src/main/ruby/hbase.rb
index 2a4abcb2ed7a..4d97cd09482f 100644
--- a/src/main/ruby/hbase.rb
+++ b/src/main/ruby/hbase.rb
@@ -40,6 +40,7 @@ module HBaseConstants
   NAME = org.apache.hadoop.hbase.HConstants::NAME
   VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS
   IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY
+  CONFIG = org.apache.hadoop.hbase.HConstants::CONFIG
   STOPROW = "STOPROW"
   STARTROW = "STARTROW"
   ENDROW = STOPROW
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 73e8552bc5dd..e26fe6d3b3f9 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -20,6 +20,7 @@
 
 include Java
 java_import org.apache.hadoop.hbase.util.Pair
+java_import org.apache.hadoop.hbase.util.RegionSplitter
 
 # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
 
@@ -189,38 +190,65 @@ def create(table_name, *args)
           raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type")
         end
 
-        if arg.kind_of?(Hash) and (arg.has_key?(SPLITS) or arg.has_key?(SPLITS_FILE))
-          if arg.has_key?(SPLITS_FILE)
-            unless File.exist?(arg[SPLITS_FILE])
-              raise(ArgumentError, "Splits file #{arg[SPLITS_FILE]} doesn't exist")
-            end
-            arg[SPLITS] = []
-            File.foreach(arg[SPLITS_FILE]) do |line|
-              arg[SPLITS].push(line.strip())
+        if arg.kind_of?(String)
+          # the arg is a string, default action is to add a column to the table
+          htd.addFamily(hcd(arg, htd))
+        else
+          # arg is a hash.  4 possibilities:
+          if (arg.has_key?(SPLITS) or arg.has_key?(SPLITS_FILE))
+            if arg.has_key?(SPLITS_FILE)
+              unless File.exist?(arg[SPLITS_FILE])
+                raise(ArgumentError, "Splits file #{arg[SPLITS_FILE]} doesn't exist")
+              end
+              arg[SPLITS] = []
+              File.foreach(arg[SPLITS_FILE]) do |line|
+                arg[SPLITS].push(line.strip())
+              end
             end
-          end
 
-          splits = Java::byte[][arg[SPLITS].size].new
-          idx = 0
-          arg[SPLITS].each do |split|
-            splits[idx] = split.to_java_bytes
-            idx = idx + 1
-          end
-        elsif arg.kind_of?(Hash) and (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO))
-          raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME)
-          raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS)
-          raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO)
-          raise(ArgumentError, "Number of regions must be geter than 1") unless arg[NUMREGIONS] > 1
-          num_regions = arg[NUMREGIONS]
-          split_algo = org.apache.hadoop.hbase.util.RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO])
-          splits = split_algo.split(JInteger.valueOf(num_regions))
-        else
-          # Add column to the table
-          descriptor = hcd(arg, htd)
-          if arg[COMPRESSION_COMPACT]
-            descriptor.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT])
+            splits = Java::byte[][arg[SPLITS].size].new
+            idx = 0
+            arg[SPLITS].each do |split|
+              splits[idx] = split.to_java_bytes
+              idx = idx + 1
+            end
+          elsif (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO))
+            # (1) deprecated region pre-split API
+            raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME)
+            raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS)
+            raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO)
+            raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1
+            num_regions = arg[NUMREGIONS]
+            split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO])
+            splits = split_algo.split(JInteger.valueOf(num_regions))
+          elsif (method = arg.delete(METHOD))
+            # (2) table_att modification
+            raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att'
+            raise(ArgumentError, "NUMREGIONS & SPLITALGO must both be specified") unless arg.has_key?(NUMREGIONS) == arg.has_key?(split_algo)
+            htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE]
+            htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY]
+            htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE]
+            htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH]
+            htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT]
+            if arg[NUMREGIONS]
+              raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1
+              num_regions = arg[NUMREGIONS]
+              split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO])
+              splits = split_algo.split(JInteger.valueOf(num_regions))
+            end
+            if arg[CONFIG]
+              raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash)
+              for k,v in arg[CONFIG]
+                v = v.to_s unless v.nil?
+                htd.setValue(k, v)
+              end
+            end
+          else
+            # (3) column family spec
+            descriptor = hcd(arg, htd)
+            htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT]
+            htd.addFamily(hcd(arg, htd))
           end
-          htd.addFamily(descriptor)
         end
       end
 
@@ -421,6 +449,13 @@ def alter(table_name, wait = true, *args)
             end
           end
 
+          if arg[CONFIG]
+            raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash)
+            for k,v in arg[CONFIG]
+              v = v.to_s unless v.nil?
+              htd.setValue(k, v)
+            end
+          end
           @admin.modifyTable(table_name.to_java_bytes, htd)
           if wait == true
             puts "Updating all regions with the new schema..."
@@ -561,6 +596,14 @@ def hcd(arg, htd)
           family.setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.valueOf(compression))
         end
       end
+
+      if arg[CONFIG]
+        raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash)
+        for k,v in arg[CONFIG]
+          v = v.to_s unless v.nil?
+          family.setValue(k, v)
+        end
+      end
       return family
     end
 
diff --git a/src/main/ruby/shell/commands/alter.rb b/src/main/ruby/shell/commands/alter.rb
index 33fbcb3cdd6b..3be8f0a42789 100644
--- a/src/main/ruby/shell/commands/alter.rb
+++ b/src/main/ruby/shell/commands/alter.rb
@@ -40,6 +40,9 @@ def help
 or a shorter version:
 
   hbase> alter 't1', 'delete' => 'f1'
+  
+You can also change the column family config by set attribute CONFIG like this:
+  hbase> alter 'test',  NAME=>'f', CONFIG => {'hbase.hstore.compaction.min' => '5'}
 
 You can also change table-scope attributes like MAX_FILESIZE
 MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH.
@@ -47,6 +50,9 @@ def help
 For example, to change the max size of a family to 128MB, do:
 
   hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728'
+  
+You can also change the table-scope by set attribute CONFIG like this:
+  hbase> alter 'test', METHOD=>'table_att', CONFIG => {'hbase.hstore.compaction.min' => '5'}  
 
 You can add a table coprocessor by setting a table coprocessor attribute:
 
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index b4b5728a598b..cbbf737c44ce 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -196,9 +196,7 @@ public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
 
   protected HRegion openClosedRegion(final HRegion closedRegion)
   throws IOException {
-    HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
-        closedRegion.getFilesystem(), closedRegion.getConf(),
-        closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
+    HRegion r = new HRegion(closedRegion);
     r.initialize();
     return r;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
new file mode 100644
index 000000000000..7dd60de90915
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -0,0 +1,260 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+@Category(LargeTests.class)
+public class TestFromClientSide3 {
+  final Log LOG = LogFactory.getLog(getClass());
+  private final static HBaseTestingUtility TEST_UTIL
+    = new HBaseTestingUtility();
+  private static byte[] ROW = Bytes.toBytes("testRow");
+  private static byte[] FAMILY = Bytes.toBytes("testFamily");
+  private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
+  private static byte[] VALUE = Bytes.toBytes("testValue");
+  private static Random random = new Random();
+  private static int SLAVES = 3;
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean(
+        "hbase.online.schema.update.enable", true);
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @Before
+  public void setUp() throws Exception {
+    // Nothing to do.
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @After
+  public void tearDown() throws Exception {
+    // Nothing to do.
+  }
+
+  private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
+      throws Exception {
+    Put put = new Put(row);
+    for (int i = 0; i < nPuts; i++) {
+      byte[] qualifier = Bytes.toBytes(random.nextInt());
+      byte[] value = Bytes.toBytes(random.nextInt());
+      put.add(family, qualifier, value);
+    }
+    table.put(put);
+  }
+
+  private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
+      byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
+
+    // connection needed for poll-wait
+    HConnection conn = HConnectionManager.getConnection(TEST_UTIL
+        .getConfiguration());
+    HRegionLocation loc = table.getRegionLocation(row, true);
+    HRegionInterface server = conn.getHRegionConnection(loc.getHostname(), loc
+        .getPort());
+    byte[] regName = loc.getRegionInfo().getRegionName();
+
+    for (int i = 0; i < nFlushes; i++) {
+      randomCFPuts(table, row, family, nPuts);
+      int sfCount = server.getStoreFileList(regName, FAMILY).size();
+
+      // TODO: replace this api with a synchronous flush after HBASE-2949
+      admin.flush(table.getTableName());
+
+      // synchronously poll wait for a new storefile to appear (flush happened)
+      while (server.getStoreFileList(regName, FAMILY).size() == sfCount) {
+        Thread.sleep(40);
+      }
+    }
+  }
+
+  // override the config settings at the CF level and ensure priority
+  @Test(timeout = 60000)
+  public void testAdvancedConfigOverride() throws Exception {
+    /*
+     * Overall idea: (1) create 3 store files and issue a compaction. config's
+     * compaction.min == 3, so should work. (2) Increase the compaction.min
+     * toggle in the HTD to 5 and modify table. If we use the HTD value instead
+     * of the default config value, adding 3 files and issuing a compaction
+     * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
+     * and modify table. The CF schema should override the Table schema and now
+     * cause a minor compaction.
+     */
+    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
+
+    String tableName = "testAdvancedConfigOverride";
+    byte[] TABLE = Bytes.toBytes(tableName);
+    HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
+    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+    HConnection connection = HConnectionManager.getConnection(TEST_UTIL
+        .getConfiguration());
+
+    // Create 3 store files.
+    byte[] row = Bytes.toBytes(random.nextInt());
+    performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
+
+    // Verify we have multiple store files.
+    HRegionLocation loc = hTable.getRegionLocation(row, true);
+    byte[] regionName = loc.getRegionInfo().getRegionName();
+    HRegionInterface server = connection.getHRegionConnection(
+        loc.getHostname(), loc.getPort());
+    assertTrue(server.getStoreFileList(regionName, FAMILY).size() > 1);
+
+    // Issue a compaction request
+    admin.compact(TABLE);
+
+    // poll wait for the compactions to happen
+    for (int i = 0; i < 10 * 1000 / 40; ++i) {
+      // The number of store files after compaction should be lesser.
+      loc = hTable.getRegionLocation(row, true);
+      if (!loc.getRegionInfo().isOffline()) {
+        regionName = loc.getRegionInfo().getRegionName();
+        server = connection.getHRegionConnection(loc.getHostname(), loc
+            .getPort());
+        if (server.getStoreFileList(regionName, FAMILY).size() <= 1) {
+          break;
+        }
+      }
+      Thread.sleep(40);
+    }
+    // verify the compactions took place and that we didn't just time out
+    assertTrue(server.getStoreFileList(regionName, FAMILY).size() <= 1);
+
+    // change the compaction.min config option for this table to 5
+    LOG.info("hbase.hstore.compaction.min should now be 5");
+    HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
+    htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
+    admin.modifyTable(TABLE, htd);
+    Pair<Integer, Integer> st;
+    while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
+      LOG.debug(st.getFirst() + " regions left to update");
+      Thread.sleep(40);
+    }
+    LOG.info("alter status finished");
+
+    // Create 3 more store files.
+    performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
+
+    // Issue a compaction request
+    admin.compact(TABLE);
+
+    // This time, the compaction request should not happen
+    Thread.sleep(10 * 1000);
+    int sfCount = 0;
+    loc = hTable.getRegionLocation(row, true);
+    regionName = loc.getRegionInfo().getRegionName();
+    server = connection.getHRegionConnection(loc.getHostname(), loc.getPort());
+    sfCount = server.getStoreFileList(regionName, FAMILY).size();
+    assertTrue(sfCount > 1);
+
+    // change an individual CF's config option to 2 & online schema update
+    LOG.info("hbase.hstore.compaction.min should now be 2");
+    HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
+    hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
+    htd.addFamily(hcd);
+    admin.modifyTable(TABLE, htd);
+    while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
+      LOG.debug(st.getFirst() + " regions left to update");
+      Thread.sleep(40);
+    }
+    LOG.info("alter status finished");
+
+    // Issue a compaction request
+    admin.compact(TABLE);
+
+    // poll wait for the compactions to happen
+    for (int i = 0; i < 10 * 1000 / 40; ++i) {
+      loc = hTable.getRegionLocation(row, true);
+      regionName = loc.getRegionInfo().getRegionName();
+      try {
+        server = connection.getHRegionConnection(loc.getHostname(), loc
+            .getPort());
+        if (server.getStoreFileList(regionName, FAMILY).size() < sfCount) {
+          break;
+        }
+      } catch (Exception e) {
+        LOG.debug("Waiting for region to come online: " + regionName);
+      }
+      Thread.sleep(40);
+    }
+    // verify the compaction took place and that we didn't just time out
+    assertTrue(server.getStoreFileList(regionName, FAMILY).size() < sfCount);
+
+    // Finally, ensure that we can remove a custom config value after we made it
+    LOG.info("Removing CF config value");
+    LOG.info("hbase.hstore.compaction.min should now be 5");
+    hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
+    hcd.setValue("hbase.hstore.compaction.min", null);
+    htd.addFamily(hcd);
+    admin.modifyTable(TABLE, htd);
+    while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
+      LOG.debug(st.getFirst() + " regions left to update");
+      Thread.sleep(40);
+    }
+    LOG.info("alter status finished");
+    assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
+        "hbase.hstore.compaction.min"));
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+   new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 3dcd8f12c77d..88becfc89008 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -380,9 +380,7 @@ public void testCoprocessorInterface() throws IOException {
   HRegion reopenRegion(final HRegion closedRegion, Class<?> ... implClasses)
       throws IOException {
     //HRegionInfo info = new HRegionInfo(tableName, null, null, false);
-    HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
-        closedRegion.getFilesystem(), closedRegion.getConf(),
-        closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
+    HRegion r = new HRegion(closedRegion);
     r.initialize();
 
     // this following piece is a hack. currently a coprocessorHost
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
new file mode 100644
index 000000000000..12bc40b3fedc
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
@@ -0,0 +1,116 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.CompoundConfiguration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.SmallTests;
+
+import org.junit.experimental.categories.Category;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+@Category(SmallTests.class)
+public class TestCompoundConfiguration extends TestCase {
+  private Configuration baseConf;
+
+  @Override
+  protected void setUp() throws Exception {
+    baseConf = new Configuration();
+    baseConf.set("A", "1");
+    baseConf.setInt("B", 2);
+    baseConf.set("C", "3");
+  }
+
+  @Test
+  public void testBasicFunctionality() throws ClassNotFoundException {
+    CompoundConfiguration compoundConf = new CompoundConfiguration()
+        .add(baseConf);
+    assertEquals("1", compoundConf.get("A"));
+    assertEquals(2, compoundConf.getInt("B", 0));
+    assertEquals(3, compoundConf.getInt("C", 0));
+    assertEquals(0, compoundConf.getInt("D", 0));
+
+    assertEquals(CompoundConfiguration.class, compoundConf
+        .getClassByName(CompoundConfiguration.class.getName()));
+    try {
+      compoundConf.getClassByName("bad_class_name");
+      fail("Trying to load bad_class_name should throw an exception");
+    } catch (ClassNotFoundException e) {
+      // win!
+    }
+  }
+
+  @Test
+  public void testWithConfig() {
+    Configuration conf = new Configuration();
+    conf.set("B", "2b");
+    conf.set("C", "33");
+    conf.set("D", "4");
+
+    CompoundConfiguration compoundConf = new CompoundConfiguration()
+        .add(baseConf)
+        .add(conf);
+    assertEquals("1", compoundConf.get("A"));
+    assertEquals("2b", compoundConf.get("B"));
+    assertEquals(33, compoundConf.getInt("C", 0));
+    assertEquals("4", compoundConf.get("D"));
+    assertEquals(4, compoundConf.getInt("D", 0));
+    assertNull(compoundConf.get("E"));
+    assertEquals(6, compoundConf.getInt("F", 6));
+  }
+
+  private ImmutableBytesWritable strToIbw(String s) {
+    return new ImmutableBytesWritable(Bytes.toBytes(s));
+  }
+
+  @Test
+  public void testWithIbwMap() {
+    Map<ImmutableBytesWritable, ImmutableBytesWritable> map =
+      new HashMap<ImmutableBytesWritable, ImmutableBytesWritable>();
+    map.put(strToIbw("B"), strToIbw("2b"));
+    map.put(strToIbw("C"), strToIbw("33"));
+    map.put(strToIbw("D"), strToIbw("4"));
+    // unlike config, note that IBW Maps can accept null values
+    map.put(strToIbw("G"), null);
+
+    CompoundConfiguration compoundConf = new CompoundConfiguration()
+      .add(baseConf)
+      .add(map);
+    assertEquals("1", compoundConf.get("A"));
+    assertEquals("2b", compoundConf.get("B"));
+    assertEquals(33, compoundConf.getInt("C", 0));
+    assertEquals("4", compoundConf.get("D"));
+    assertEquals(4, compoundConf.getInt("D", 0));
+    assertNull(compoundConf.get("E"));
+    assertEquals(6, compoundConf.getInt("F", 6));
+    assertNull(compoundConf.get("G"));
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index b935cd2d012b..4d6eb90aa465 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -247,7 +247,7 @@ private SplitTransaction prepareGOOD_SPLIT_ROW() {
     for (HRegion r: daughters) {
       // Open so can count its content.
       HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(),
-         r.getTableDesc(), r.getLog(), r.getConf());
+          r.getTableDesc(), r.getLog(), TEST_UTIL.getConfiguration());
       try {
         int count = countRows(openRegion);
         assertTrue(count > 0 && count != rowcount);
@@ -303,7 +303,7 @@ private SplitTransaction prepareGOOD_SPLIT_ROW() {
     for (HRegion r: daughters) {
       // Open so can count its content.
       HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(),
-         r.getTableDesc(), r.getLog(), r.getConf());
+        r.getTableDesc(), r.getLog(), TEST_UTIL.getConfiguration());
       try {
         int count = countRows(openRegion);
         assertTrue(count > 0 && count != rowcount);

From 0b3220e2bf40ac0999ad56afe4db25710e19b3a7 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 29 Mar 2013 20:50:29 +0000
Subject: [PATCH 0882/1540] Amend HBASE-8209. Revert changes to
 HBaseTestingUtility#waitUntilAllRegionsAssigned

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462641 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 54 +++++++------------
 .../hbase/coprocessor/TestMasterObserver.java |  2 +-
 ...onServerCoprocessorExceptionWithAbort.java |  2 +-
 ...nServerCoprocessorExceptionWithRemove.java |  2 +-
 .../hbase/master/TestMasterTransitions.java   |  2 +-
 .../util/TestMiniClusterLoadSequential.java   |  2 +-
 6 files changed, 24 insertions(+), 40 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 699c0dd01ba9..81f494550234 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1835,45 +1835,29 @@ public static void setMaxRecoveryErrorCount(final OutputStream stream,
    * @param countOfRegions How many regions in .META.
    * @throws IOException
    */
-  public void waitUntilAllRegionsAssigned(final byte[] tableName, final int countOfRegions)
+  public void waitUntilAllRegionsAssigned(final int countOfRegions)
   throws IOException {
-    int retries = 30; // We may wait up to 30 seconds
-    int rows = 0;
     HTable meta = new HTable(getConfiguration(), HConstants.META_TABLE_NAME);
-    try {
-      do {
-        Scan scan = new Scan();
-        scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-        scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        ResultScanner s = meta.getScanner(scan);
-        try {
-          for (Result r = null; (r = s.next()) != null;) {
-            byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-            HRegionInfo hri = Writables.getHRegionInfoOrNull(b);
-            if (hri != null && Bytes.equals(hri.getTableName(), tableName)) {
-              b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-              if (b == null || b.length <= 0) {
-                continue;
-              }
-              rows++;
-            }
-          }
-        } finally {
-          s.close();
-        }
-        // If I get to here and all rows have a Server, then all have been assigned.
-        if (rows == countOfRegions) {
+    while (true) {
+      int rows = 0;
+      Scan scan = new Scan();
+      scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+      ResultScanner s = meta.getScanner(scan);
+      for (Result r = null; (r = s.next()) != null;) {
+        byte [] b =
+          r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+        if (b == null || b.length <= 0) {
           break;
         }
-        LOG.info("Found=" + rows);
-        Threads.sleep(1000);
-      } while (--retries > 0);
-    } finally {
-      meta.close();
-    }
-    if (rows != countOfRegions) {
-      throw new IOException("Timed out waiting for " + countOfRegions + " regions of " +
-        Bytes.toStringBinary(tableName) + " to come online");
+        rows++;
+      }
+      s.close();
+      // If I get to here and all rows have a Server, then all have been assigned.
+      if (rows == countOfRegions) {
+        break;
+      }
+      LOG.info("Found=" + rows);
+      Threads.sleep(200);
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 3c831ba9ad60..00aefe08d4d7 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -748,7 +748,7 @@ public void testRegionTransitionOperations() throws Exception {
 
     HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
     int countOfRegions = UTIL.createMultiRegions(table, TEST_FAMILY);
-    UTIL.waitUntilAllRegionsAssigned(TEST_TABLE, countOfRegions);
+    UTIL.waitUntilAllRegionsAssigned(countOfRegions);
     
     NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
     Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
index 811196543804..33756579719f 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
@@ -109,7 +109,7 @@ public void testExceptionFromCoprocessorDuringPut()
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE,
+    TEST_UTIL.waitUntilAllRegionsAssigned(
         TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
 
     // Note which regionServer will abort (after put is attempted).
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
index 20b7b145c0a3..10f44a6eb3f3 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
@@ -92,7 +92,7 @@ public void testExceptionFromCoprocessorDuringPut()
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE,
+    TEST_UTIL.waitUntilAllRegionsAssigned(
         TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
     // Note which regionServer that should survive the buggy coprocessor's
     // prePut().
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
index b893288988a6..e0a486134838 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
@@ -62,7 +62,7 @@ public class TestMasterTransitions {
     TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
     int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
-    TEST_UTIL.waitUntilAllRegionsAssigned(Bytes.toBytes(TABLENAME), countOfRegions);
+    TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
     addToEachStartKey(countOfRegions);
     t.close();
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
index 12bca7136c51..1adc062acfb7 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
@@ -137,7 +137,7 @@ protected void runLoadTestOnExistingTable() throws IOException {
   protected void createPreSplitLoadTestTable(HTableDescriptor htd, HColumnDescriptor hcd)
       throws IOException {
     int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf, htd, hcd);
-    TEST_UTIL.waitUntilAllRegionsAssigned(htd.getName(), numRegions);
+    TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
   }
 
   protected void prepareForLoadTest() throws IOException {

From f13f945fa88d7700e30ca8bd905d72f0638af586 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 29 Mar 2013 23:33:40 +0000
Subject: [PATCH 0883/1540] HBASE-8222 User class should implement equals() and
 hashCode()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462676 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/security/User.java  | 17 +++++++++++++++++
 .../apache/hadoop/hbase/security/TestUser.java  |  2 ++
 2 files changed, 19 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index f15c470012cc..298e3ab3a931 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -127,6 +127,23 @@ public abstract void obtainAuthTokenForJob(Configuration conf, Job job)
   public abstract void obtainAuthTokenForJob(JobConf job)
       throws IOException, InterruptedException;
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    return ugi.equals(((User) o).ugi);
+  }
+  
+  @Override
+  public int hashCode() {
+    return ugi.hashCode();
+  }
+  
+  @Override
   public String toString() {
     return ugi.toString();
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/security/TestUser.java b/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
index 098787c726f4..e90de1f5967e 100644
--- a/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
+++ b/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
@@ -102,6 +102,8 @@ public void testGetCurrent() throws Exception {
       User u = User.getCurrent();
       assertNotNull(u);
       assertEquals(user1.getName(), u.getName());
+      assertEquals(user1, u);
+      assertEquals(user1.hashCode(), u.hashCode());
     }
   }
 

From fe7dea601c9e2e15246a76aba3f0833c15eb5153 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 31 Mar 2013 04:36:52 +0000
Subject: [PATCH 0884/1540] HBASE-8152 Avoid creating empty reference file when
 splitkey is outside the key range of a store file (clockfly)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462880 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  | 24 +++++++++++++
 .../hbase/regionserver/TestStoreFile.java     | 34 ++++++-------------
 2 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 4cfaaca0129f..274018730a2d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -945,6 +945,30 @@ static Path split(final FileSystem fs,
                     final byte [] splitRow,
                     final Reference.Range range)
       throws IOException {
+	    
+    // Check whether the split row lies in the range of the store file
+    // If it is outside the range, return directly.
+    if (range == Reference.Range.bottom) {
+      //check if smaller than first key
+      KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
+      byte[] firstKey = f.createReader().getFirstKey();
+      if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
+          splitKey.getKeyOffset(), splitKey.getKeyLength(), 
+          firstKey, 0, firstKey.length) < 0) {
+        return null;
+      }      
+    }
+    else {
+      //check if larger than last key.
+      KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
+      byte[] lastKey = f.createReader().getLastKey();      
+      if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
+          splitKey.getKeyOffset(), splitKey.getKeyLength(), 
+          lastKey, 0, lastKey.length) > 0) {
+        return null;
+      }
+    }
+    
     // A reference to the bottom half of the hsf store file.
     Reference r = new Reference(splitRow, range);
     // Add the referred-to regions name as a dot separated suffix.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index a07fff1133cf..a8f2f84ab4d4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -116,7 +116,7 @@ private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
   }
 
   // pick an split point (roughly halfway)
-  byte[] SPLITKEY = new byte[] { (LAST_CHAR-FIRST_CHAR)/2, FIRST_CHAR};
+  byte[] SPLITKEY = new byte[] { (LAST_CHAR + FIRST_CHAR)/2, FIRST_CHAR};
 
   /*
    * Writes HStoreKey and ImmutableBytes data to passed writer and
@@ -315,6 +315,9 @@ public void testReferenceToHFileLink() throws IOException {
     // Now confirm that I can read from the ref to link
     HFileScanner sB = hsfB.createReader().getScanner(false, false);
     sB.seekTo();
+    
+    //count++ as seekTo() will advance the scanner
+    count++;
     while (sB.next()) {
       count++;
     }
@@ -404,20 +407,12 @@ private void checkHalfHFile(final StoreFile f)
       topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
       bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
         Range.bottom);
+      
+      assertNull(bottomPath);
+      
       top = new StoreFile(this.fs, topPath, conf, cacheConf,
           StoreFile.BloomType.NONE,
           NoOpDataBlockEncoder.INSTANCE).createReader();
-      bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
-          StoreFile.BloomType.NONE,
-          NoOpDataBlockEncoder.INSTANCE).createReader();
-      bottomScanner = bottom.getScanner(false, false);
-      int count = 0;
-      while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
-          bottomScanner.next()) {
-        count++;
-      }
-      // When badkey is < than the bottom, should return no values.
-      assertTrue(count == 0);
       // Now read from the top.
       first = true;
       topScanner = top.getScanner(false, false);
@@ -444,16 +439,15 @@ private void checkHalfHFile(final StoreFile f)
       }
       // Remove references.
       this.fs.delete(topPath, false);
-      this.fs.delete(bottomPath, false);
 
       // Test when badkey is > than last key in file ('||' > 'zz').
       badmidkey = Bytes.toBytes("|||");
       topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
       bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
         Range.bottom);
-      top = new StoreFile(this.fs, topPath, conf, cacheConf,
-          StoreFile.BloomType.NONE,
-          NoOpDataBlockEncoder.INSTANCE).createReader();
+
+      assertNull(topPath);
+      
       bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
           StoreFile.BloomType.NONE,
           NoOpDataBlockEncoder.INSTANCE).createReader();
@@ -477,14 +471,6 @@ private void checkHalfHFile(final StoreFile f)
       for (int i = 0; i < tmp.length(); i++) {
         assertTrue(Bytes.toString(keyKV.getRow()).charAt(i) == 'z');
       }
-      count = 0;
-      topScanner = top.getScanner(false, false);
-      while ((!topScanner.isSeeked() && topScanner.seekTo()) ||
-          (topScanner.isSeeked() && topScanner.next())) {
-        count++;
-      }
-      // When badkey is < than the bottom, should return no values.
-      assertTrue(count == 0);
     } finally {
       if (top != null) {
         top.close(true); // evict since we are about to delete the file

From d85b102fd157aac4c1123b7c83466f6eb34721cc Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sun, 31 Mar 2013 15:58:00 +0000
Subject: [PATCH 0885/1540] HBASE-8232. TestAccessController occasionally fails
 with IndexOutOfBoundsException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1462998 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/TestAccessController.java       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index f57357826cec..cb27c38a6380 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -149,6 +149,7 @@ public static void setupBeforeClass() throws Exception {
     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
     htd.setOwner(USER_OWNER);
     admin.createTable(htd);
+    TEST_UTIL.waitTableEnabled(TEST_TABLE, 5000);
 
     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0);
     RegionCoprocessorHost rcpHost = region.getCoprocessorHost();

From 03acb3f870f9162946a6ce23f77cb7a27dea30f4 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 1 Apr 2013 00:57:40 +0000
Subject: [PATCH 0886/1540] HBASE-8226.
 HBaseTestingUtility#waitUntilAllRegionsAssigned won't return if it counts
 'too many' regions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463072 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 77 +++++++++++++------
 .../hbase/coprocessor/TestMasterObserver.java |  4 +-
 ...onServerCoprocessorExceptionWithAbort.java |  4 +-
 ...nServerCoprocessorExceptionWithRemove.java |  4 +-
 .../hbase/master/TestMasterTransitions.java   |  5 +-
 .../util/TestMiniClusterLoadSequential.java   |  4 +-
 6 files changed, 64 insertions(+), 34 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 81f494550234..28ae2a92a4b0 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1827,37 +1827,66 @@ public static void setMaxRecoveryErrorCount(final OutputStream stream,
     }
   }
 
+  /**
+   * Wait until all regions for a table in .META. have a non-empty
+   * info:server, up to 60 seconds.  This means all regions have been deployed,
+   * master has been informed and updated .META. with the regions deployed
+   * server.
+   * @param tableName the table name
+   * @throws IOException
+   */
+  public void waitUntilAllRegionsAssigned(final byte[] tableName) throws IOException {
+    waitUntilAllRegionsAssigned(tableName, 60000);
+  }
 
   /**
-   * Wait until <code>countOfRegion</code> in .META. have a non-empty
-   * info:server.  This means all regions have been deployed, master has been
-   * informed and updated .META. with the regions deployed server.
-   * @param countOfRegions How many regions in .META.
+   * Wait until all regions for a table in .META. have a non-empty
+   * info:server, or until timeout.  This means all regions have been
+   * deployed, master has been informed and updated .META. with the regions
+   * deployed server.
+   * @param tableName the table name
+   * @param timeout timeout, in milliseconds
    * @throws IOException
    */
-  public void waitUntilAllRegionsAssigned(final int countOfRegions)
-  throws IOException {
+  public void waitUntilAllRegionsAssigned(final byte[] tableName, final long timeout)
+      throws IOException {
+    long deadline = System.currentTimeMillis() + timeout;
     HTable meta = new HTable(getConfiguration(), HConstants.META_TABLE_NAME);
-    while (true) {
-      int rows = 0;
-      Scan scan = new Scan();
-      scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-      ResultScanner s = meta.getScanner(scan);
-      for (Result r = null; (r = s.next()) != null;) {
-        byte [] b =
-          r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        if (b == null || b.length <= 0) {
-          break;
+    try {
+      while (true) {
+        boolean allRegionsAssigned = true;
+        Scan scan = new Scan();
+        scan.addFamily(HConstants.CATALOG_FAMILY);
+        ResultScanner s = meta.getScanner(scan);
+        try {
+          Result r;
+          while ((r = s.next()) != null) {
+            byte [] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+            HRegionInfo info = Writables.getHRegionInfoOrNull(b);
+            if (info != null && Bytes.equals(info.getTableName(), tableName)) {
+              b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+              allRegionsAssigned &= (b != null);
+            }
+          }
+        } finally {
+          s.close();
+        }
+        if (allRegionsAssigned) {
+          return;
+        }
+        long now = System.currentTimeMillis();
+        if (now > deadline) {
+          throw new IOException("Timeout waiting for all regions of " +
+            Bytes.toStringBinary(tableName) + " to be assigned");
+        }
+        try {
+          Thread.sleep(deadline - now < 200 ? deadline - now : 200);
+        } catch (InterruptedException e) {
+          throw new IOException(e);
         }
-        rows++;
-      }
-      s.close();
-      // If I get to here and all rows have a Server, then all have been assigned.
-      if (rows == countOfRegions) {
-        break;
       }
-      LOG.info("Found=" + rows);
-      Threads.sleep(200);
+    } finally {
+      meta.close();
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 00aefe08d4d7..eec9151d2d28 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -747,8 +747,8 @@ public void testRegionTransitionOperations() throws Exception {
     cp.resetStates();
 
     HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    int countOfRegions = UTIL.createMultiRegions(table, TEST_FAMILY);
-    UTIL.waitUntilAllRegionsAssigned(countOfRegions);
+    UTIL.createMultiRegions(table, TEST_FAMILY);
+    UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
     
     NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
     Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
index 33756579719f..ae5b9fae167b 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
@@ -109,8 +109,8 @@ public void testExceptionFromCoprocessorDuringPut()
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    TEST_UTIL.waitUntilAllRegionsAssigned(
-        TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
+    TEST_UTIL.createMultiRegions(table, TEST_FAMILY);
+    TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
 
     // Note which regionServer will abort (after put is attempted).
     final HRegionServer regionServer =
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
index 10f44a6eb3f3..d1916650b0bb 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
@@ -92,8 +92,8 @@ public void testExceptionFromCoprocessorDuringPut()
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
-    TEST_UTIL.waitUntilAllRegionsAssigned(
-        TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
+    TEST_UTIL.createMultiRegions(table, TEST_FAMILY);
+    TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
     // Note which regionServer that should survive the buggy coprocessor's
     // prePut().
     HRegionServer regionServer =
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
index e0a486134838..762a2b8eeb77 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
@@ -59,10 +59,11 @@ public class TestMasterTransitions {
     TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
     TEST_UTIL.startMiniCluster(2);
     // Create a table of three families.  This will assign a region.
-    TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
+    byte[] tableName = Bytes.toBytes(TABLENAME);
+    TEST_UTIL.createTable(tableName, FAMILIES);
     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
     int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
-    TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
+    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
     addToEachStartKey(countOfRegions);
     t.close();
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
index 1adc062acfb7..64c67d3a169b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
@@ -136,8 +136,8 @@ protected void runLoadTestOnExistingTable() throws IOException {
 
   protected void createPreSplitLoadTestTable(HTableDescriptor htd, HColumnDescriptor hcd)
       throws IOException {
-    int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf, htd, hcd);
-    TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
+    HBaseTestingUtility.createPreSplitLoadTestTable(conf, htd, hcd);
+    TEST_UTIL.waitUntilAllRegionsAssigned(htd.getName());
   }
 
   protected void prepareForLoadTest() throws IOException {

From f58d0dd49a1654247cfa74ebe99a4ebd17bdc19e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 1 Apr 2013 14:53:47 +0000
Subject: [PATCH 0887/1540] HBASE-7925 Back port HBASE-6881 into 0.94
 (Rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463155 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 66 ++++++++++++++-----
 1 file changed, 49 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 3c9973e55103..bca10e406b8a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1651,6 +1651,8 @@ private void assign(final HRegionInfo region, final RegionState state,
       final boolean setOfflineInZK, final boolean forceNewPlan,
       boolean hijack) {
     boolean regionAlreadyInTransitionException = false;
+    boolean serverNotRunningYet = false;
+    long maxRegionServerStartupWaitTime = -1;
     for (int i = 0; i < this.maximumAssignmentAttempts; i++) {
       int versionOfOfflineNode = -1;
       if (setOfflineInZK) {
@@ -1684,7 +1686,8 @@ private void assign(final HRegionInfo region, final RegionState state,
         LOG.debug("Server stopped; skipping assign of " + state);
         return;
       }
-      RegionPlan plan = getRegionPlan(state, !regionAlreadyInTransitionException && forceNewPlan);
+      RegionPlan plan = getRegionPlan(state, !regionAlreadyInTransitionException
+          && !serverNotRunningYet && forceNewPlan);
       if (plan == null) {
         LOG.debug("Unable to determine a plan to assign " + state);
         this.timeoutMonitor.setAllRegionServersOffline(true);
@@ -1740,15 +1743,39 @@ private void assign(final HRegionInfo region, final RegionState state,
       } catch (Throwable t) {
         if (t instanceof RemoteException) {
           t = ((RemoteException) t).unwrapRemoteException();
-          if (t instanceof RegionAlreadyInTransitionException) {
-            regionAlreadyInTransitionException = true;
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Failed assignment in: " + plan.getDestination() + " due to "
-                  + t.getMessage());
+        }
+        regionAlreadyInTransitionException = false;
+        serverNotRunningYet = false;
+        if (t instanceof RegionAlreadyInTransitionException) {
+          regionAlreadyInTransitionException = true;
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Failed assignment in: " + plan.getDestination() + " due to "
+                + t.getMessage());
+          }
+        } else if (t instanceof ServerNotRunningYetException) {
+          if (maxRegionServerStartupWaitTime < 0) {
+            maxRegionServerStartupWaitTime = System.currentTimeMillis()
+                + this.master.getConfiguration().getLong("hbase.regionserver.rpc.startup.waittime",
+                    60000);
+          }
+          try {
+            long now = System.currentTimeMillis();
+            if (now < maxRegionServerStartupWaitTime) {
+              LOG.debug("Server is not yet up; waiting up to "
+                  + (maxRegionServerStartupWaitTime - now) + "ms", t);
+              serverNotRunningYet = true;
+              Thread.sleep(100);
+              i--; // reset the try count
+            } else {
+              LOG.debug("Server is not up for a while; try a new one", t);
             }
+          } catch (InterruptedException ie) {
+            LOG.warn("Failed to assign " + state.getRegion().getRegionNameAsString()
+                + " since interrupted", ie);
+            Thread.currentThread().interrupt();
+            return;
           }
-        }
-        if (t instanceof java.net.SocketTimeoutException 
+        } else if (t instanceof java.net.SocketTimeoutException 
             && this.serverManager.isServerOnline(plan.getDestination())) {
           LOG.warn("Call openRegion() to " + plan.getDestination()
               + " has timed out when trying to assign "
@@ -1758,13 +1785,15 @@ private void assign(final HRegionInfo region, final RegionState state,
           return;
         }
         LOG.warn("Failed assignment of "
-            + state.getRegion().getRegionNameAsString()
-            + " to "
-            + plan.getDestination()
-            + ", trying to assign "
-            + (regionAlreadyInTransitionException ? "to the same region server"
-                + " because of RegionAlreadyInTransitionException;" : "elsewhere instead; ")
-            + "retry=" + i, t);
+          + state.getRegion().getRegionNameAsString()
+          + " to "
+          + plan.getDestination()
+          + ", trying to assign "
+          + (regionAlreadyInTransitionException || serverNotRunningYet
+            ? "to the same region server because of "
+            + "RegionAlreadyInTransitionException/ServerNotRunningYetException;"
+            : "elsewhere instead; ")
+          + "retry=" + i, t);
         // Clean out plan we failed execute and one that doesn't look like it'll
         // succeed anyways; we need a new plan!
         // Transition back to OFFLINE
@@ -1773,9 +1802,12 @@ private void assign(final HRegionInfo region, final RegionState state,
         // RS may cause double assignments. In case of RegionAlreadyInTransitionException
         // reassigning to same RS.
         RegionPlan newPlan = plan;
-        if (!regionAlreadyInTransitionException) {
+        if (!regionAlreadyInTransitionException && !serverNotRunningYet) {
           // Force a new plan and reassign. Will return null if no servers.
-          newPlan = getRegionPlan(state, plan.getDestination(), true);
+          // The new plan could be the same as the existing plan since we don't
+          // exclude the server of the original plan, which should not be
+          // excluded since it could be the only server up now.
+          newPlan = getRegionPlan(state, true);
         }
         if (newPlan == null) {
           this.timeoutMonitor.setAllRegionServersOffline(true);

From f4e0961a5ce0072551fd3e0ee744b5da8c592591 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 1 Apr 2013 16:34:33 +0000
Subject: [PATCH 0888/1540] HBASE-8213. Global authorization may lose efficacy

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463197 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/TableAuthManager.java     |  7 +-
 .../security/access/TestAccessController.java | 95 +++++++++++++++++++
 2 files changed, 99 insertions(+), 3 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
index 9c09b95fecbc..ba73f4efb43e 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
@@ -100,15 +100,16 @@ public ListMultimap<String,T> getAllPermissions() {
   private TableAuthManager(ZooKeeperWatcher watcher, Configuration conf)
       throws IOException {
     this.conf = conf;
+
+    // initialize global permissions based on configuration
+    globalCache = initGlobal(conf);
+
     this.zkperms = new ZKPermissionWatcher(watcher, this, conf);
     try {
       this.zkperms.start();
     } catch (KeeperException ke) {
       LOG.error("ZooKeeper initialization failed", ke);
     }
-
-    // initialize global permissions based on configuration
-    globalCache = initGlobal(conf);
   }
 
   /**
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index cb27c38a6380..212c14f9342d 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -26,7 +26,10 @@
 import java.security.PrivilegedExceptionAction;
 import java.util.List;
 import java.util.Map;
+import java.util.NavigableMap;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -40,6 +43,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.UnknownRowLockException;
 import org.apache.hadoop.hbase.client.Append;
@@ -63,12 +67,18 @@
 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.access.AccessControlLists;
+import org.apache.hadoop.hbase.security.access.AccessControllerProtocol;
+import org.apache.hadoop.hbase.security.access.Permission;
+import org.apache.hadoop.hbase.security.access.UserPermission;
 import org.apache.hadoop.hbase.security.access.Permission.Action;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -81,6 +91,7 @@
 @Category(LargeTests.class)
 @SuppressWarnings("rawtypes")
 public class TestAccessController {
+  private static final Log LOG = LogFactory.getLog(TestAccessController.class);
   private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static Configuration conf;
 
@@ -102,6 +113,7 @@ public class TestAccessController {
   private static User USER_NONE;
 
   private static byte[] TEST_TABLE = Bytes.toBytes("testtable");
+  private static byte[] TEST_TABLE2 = Bytes.toBytes("testtable2");
   private static byte[] TEST_FAMILY = Bytes.toBytes("f1");
 
   private static MasterCoprocessorEnvironment CP_ENV;
@@ -1774,4 +1786,87 @@ public Object run() throws Exception {
     verifyAllowed(deleteAction, SUPERUSER, USER_ADMIN);
     verifyDenied(cloneAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
   }
+
+  @Test
+  public void testGlobalAuthorizationForNewRegisteredRS() throws Exception {
+    LOG.debug("Test for global authorization for a new registered RegionServer.");
+    MiniHBaseCluster hbaseCluster = TEST_UTIL.getHBaseCluster();
+    final HRegionServer oldRs = hbaseCluster.getRegionServer(0);
+
+    // Since each RegionServer running on different user, add global
+    // permissions for the new user.
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(
+          AccessControllerProtocol.class, TEST_TABLE);
+      String currentUser = User.getCurrent().getShortName();
+      // User name for the new RegionServer we plan to add.
+      String activeUserForNewRs = currentUser + ".hfs."
+          + hbaseCluster.getLiveRegionServerThreads().size();
+
+      protocol.grant(new UserPermission(Bytes.toBytes(activeUserForNewRs),
+          Permission.Action.ADMIN, Permission.Action.CREATE,
+          Permission.Action.READ, Permission.Action.WRITE));
+
+    } finally {
+      acl.close();
+    }
+    final HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    HTableDescriptor htd = new HTableDescriptor(TEST_TABLE2);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    htd.setOwner(USER_OWNER);
+    admin.createTable(htd);
+
+    // Starting a new RegionServer.
+    JVMClusterUtil.RegionServerThread newRsThread = hbaseCluster
+        .startRegionServer();
+    final HRegionServer newRs = newRsThread.getRegionServer();
+
+    // Move region to the new RegionServer.
+    final HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE2);
+    try {
+      NavigableMap<HRegionInfo, ServerName> regions = table
+          .getRegionLocations();
+      final Map.Entry<HRegionInfo, ServerName> firstRegion = regions.entrySet()
+          .iterator().next();
+
+      PrivilegedExceptionAction moveAction = new PrivilegedExceptionAction() {
+        public Object run() throws Exception {
+          admin.move(firstRegion.getKey().getEncodedNameAsBytes(),
+              Bytes.toBytes(newRs.getServerName().getServerName()));
+          return null;
+        }
+      };
+      SUPERUSER.runAs(moveAction);
+
+      final int RETRIES_LIMIT = 10;
+      int retries = 0;
+      while (newRs.getOnlineRegions().size() < 1 && retries < RETRIES_LIMIT) {
+        LOG.debug("Waiting for region to be opened. Already retried " + retries
+            + " times.");
+        try {
+          Thread.sleep(200);
+        } catch (InterruptedException e) {
+        }
+        retries++;
+        if (retries == RETRIES_LIMIT - 1) {
+          fail("Retry exhaust for waiting region to be opened.");
+        }
+      }
+      // Verify write permission for user "admin2" who has the global
+      // permissions.
+      PrivilegedExceptionAction putAction = new PrivilegedExceptionAction() {
+        public Object run() throws Exception {
+          Put put = new Put(Bytes.toBytes("test"));
+          put.add(TEST_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
+          table.put(put);
+          return null;
+        }
+      };
+      USER_ADMIN.runAs(putAction);
+    } finally {
+      table.close();
+    }
+  }
+
 }

From 817082b357389224e7e7b71ff89865e60fcf2691 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 1 Apr 2013 16:53:57 +0000
Subject: [PATCH 0889/1540] HBASE-8174 Backport HBASE-8161(setting blocking
 file count on table level doesn't work) to 0.94 (clockfly)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463204 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/MemStoreFlusher.java      | 11 +++--------
 .../apache/hadoop/hbase/regionserver/Store.java  | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index c91404ddb05a..a4a564e5f128 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -82,7 +82,7 @@ class MemStoreFlusher extends HasThread implements FlushRequester {
     "hbase.regionserver.global.memstore.upperLimit";
   private static final String LOWER_KEY =
     "hbase.regionserver.global.memstore.lowerLimit";
-  private long blockingStoreFilesNumber;
+  
   private long blockingWaitTime;
   private final Counter updatesBlockedMsHighWater = new Counter();
 
@@ -106,12 +106,7 @@ public MemStoreFlusher(final Configuration conf,
         "because supplied " + LOWER_KEY + " was > " + UPPER_KEY);
     }
     this.globalMemStoreLimitLowMark = lower;
-    this.blockingStoreFilesNumber =
-      conf.getInt("hbase.hstore.blockingStoreFiles", 7);
-    if (this.blockingStoreFilesNumber == -1) {
-      this.blockingStoreFilesNumber = 1 +
-        conf.getInt("hbase.hstore.compactionThreshold", 3);
-    }
+    
     this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime",
       90000);
     LOG.info("globalMemStoreLimit=" +
@@ -444,7 +439,7 @@ private boolean flushRegion(final HRegion region, final boolean emergencyFlush)
 
   private boolean isTooManyStoreFiles(HRegion region) {
     for (Store hstore: region.stores.values()) {
-      if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
+      if (hstore.hasTooManyStoreFiles()) {
         return true;
       }
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 3a21a273dc43..337487e1e564 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -109,6 +109,9 @@
  */
 public class Store extends SchemaConfigured implements HeapSize {
   static final Log LOG = LogFactory.getLog(Store.class);
+  
+  public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";
+  public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;
 
   protected final MemStore memstore;
   // This stores directory in the filesystem.
@@ -134,6 +137,8 @@ public class Store extends SchemaConfigured implements HeapSize {
   private final Object flushLock = new Object();
   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
   private final boolean verifyBulkLoads;
+  
+  private long blockingFileCount;
 
   /* The default priority for user-specified compaction requests.
    * The user gets top priority unless we have blocking compactions. (Pri <= 0)
@@ -237,7 +242,10 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
 
     this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
-
+    
+    this.blockingFileCount =
+                conf.getInt(BLOCKING_STOREFILES_KEY, DEFAULT_BLOCKING_STOREFILE_COUNT);
+    
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
           "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
@@ -2271,7 +2279,7 @@ public CacheConfig getCacheConfig() {
 
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (17 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (17 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG)
           + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
@@ -2292,6 +2300,10 @@ public KeyValue.KVComparator getComparator() {
   public ScanInfo getScanInfo() {
     return scanInfo;
   }
+  
+  public boolean hasTooManyStoreFiles() {
+    return getStorefilesCount() > this.blockingFileCount;
+  }
 
   /**
    * Immutable information for scans over a store.

From a79a77aa02af535fd74f40a0684dae237dff879e Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Tue, 2 Apr 2013 00:58:33 +0000
Subject: [PATCH 0890/1540] HBASE-8225 [replication] minor code bug when
 registering ReplicationLogCleaner (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463344 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/replication/regionserver/Replication.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 128199349af4..954ebe1c2016 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -216,9 +216,9 @@ public static void decorateMasterConfiguration(Configuration conf) {
       return;
     }
     String plugins = conf.get(HBASE_MASTER_LOGCLEANER_PLUGINS);
-    if (!plugins.contains(ReplicationLogCleaner.class.toString())) {
-      conf.set(HBASE_MASTER_LOGCLEANER_PLUGINS,
-          plugins + "," + ReplicationLogCleaner.class.getCanonicalName());
+    String cleanerClass = ReplicationLogCleaner.class.getCanonicalName();
+    if (!plugins.contains(cleanerClass)) {
+      conf.set(HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
     }
   }
 

From 20a9a19edafe59da19f3587056923a971b0b3b33 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Tue, 2 Apr 2013 18:50:37 +0000
Subject: [PATCH 0891/1540] HBASE-8192 Logic error causes infinite loop in
 HRegion.bulkLoadHFiles(List) (Chenghao Jiang)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463666 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 15 +++----
 .../mapreduce/TestLoadIncrementalHFiles.java  | 43 +++++++++++++++++++
 2 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 711a6a30f7c3..1808efae5525 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3577,7 +3577,6 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
           IOException ioe = new DoNotRetryIOException(
               "No such column family " + Bytes.toStringBinary(familyName));
           ioes.add(ioe);
-          failures.add(p);
         } else {
           try {
             store.assertBulkLoadHFileOk(new Path(path));
@@ -3591,6 +3590,13 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
         }
       }
 
+      // validation failed because of some sort of IO problem.
+      if (ioes.size() != 0) {
+        IOException e = MultipleIOException.createIOException(ioes);
+        LOG.error("There were one or more IO errors when checking if the bulk load is ok.", e);
+        throw e;
+      }
+
       // validation failed, bail out before doing anything permanent.
       if (failures.size() != 0) {
         StringBuilder list = new StringBuilder();
@@ -3604,13 +3610,6 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
         return false;
       }
 
-      // validation failed because of some sort of IO problem.
-      if (ioes.size() != 0) {
-        LOG.error("There were IO errors when checking if bulk load is ok.  " +
-            "throwing exception!");
-        throw MultipleIOException.createIOException(ioes);
-      }
-
       for (Pair<byte[], String> p : familyPaths) {
         byte[] familyName = p.getFirst();
         String path = p.getSecond();
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 11df9c214278..05abd3a76d10 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -157,6 +157,49 @@ private void runTest(String testName, BloomType bloomType,
     assertEquals(expectedRows, util.countRows(table));
   }
 
+  /**
+   * Test loading into a column family that does not exist.
+   */
+  @Test
+  public void testNonexistentColumnFamilyLoad() throws Exception {
+    String testName = "testNonexistentColumnFamilyLoad";
+    byte[][][] hfileRanges = new byte[][][] {
+      new byte[][]{ Bytes.toBytes("aaa"), Bytes.toBytes("ccc") },
+      new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
+    }; 
+
+    Path dir = util.getDataTestDir(testName);
+    FileSystem fs = util.getTestFileSystem();
+    dir = dir.makeQualified(fs);
+    Path familyDir = new Path(dir, Bytes.toString(FAMILY));
+
+    int hfileIdx = 0;
+    for (byte[][] range : hfileRanges) {
+      byte[] from = range[0];
+      byte[] to = range[1];
+      createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
+          + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
+    }
+
+    final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
+
+    HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
+    HTableDescriptor htd = new HTableDescriptor(TABLE);
+    admin.createTable(htd, SPLIT_KEYS);
+
+    HTable table = new HTable(util.getConfiguration(), TABLE);
+    util.waitTableAvailable(TABLE, 30000);
+    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), false);
+    try {
+      loader.doBulkLoad(dir, table);
+      assertTrue("Loading into table with non-existent family should have failed", false);
+    } catch (Exception e) {
+      assertTrue("IOException expected", e instanceof IOException);
+    }
+    table.close();
+    admin.close();
+  }
+
   @Test
   public void testSplitStoreFile() throws IOException {
     Path dir = util.getDataTestDir("testSplitHFile");

From ff5e58b3ec096e3af25c55b6399a76e44b5d12fd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 3 Apr 2013 04:33:44 +0000
Subject: [PATCH 0892/1540] HBASE-8246 Backport HBASE-6318 to 0.94 where
 SplitLogWorker exits due to ConcurrentModificationException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1463821 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/HLogSplitter.java  | 38 ++++++++++++++-----
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 6f34f75acd2d..485ddb33e82f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -1223,18 +1223,36 @@ private List<IOException> closeLogWriters(List<IOException> thrown)
         if (thrown == null) {
           thrown = Lists.newArrayList();
         }
-        for (WriterAndPath wap : logWriters.values()) {
-          try {
-            wap.w.close();
-          } catch (IOException ioe) {
-            LOG.error("Couldn't close log at " + wap.p, ioe);
-            thrown.add(ioe);
-            continue;
+        try {
+          for (WriterThread t : writerThreads) {
+            while (t.isAlive()) {
+              t.shouldStop = true;
+              t.interrupt();
+              try {
+                t.join(10);
+              } catch (InterruptedException e) {
+                IOException iie = new InterruptedIOException();
+                iie.initCause(e);
+                throw iie;
+              }
+            }
+          }
+        } finally {
+          synchronized (logWriters) {
+            for (WriterAndPath wap : logWriters.values()) {
+              try {
+                wap.w.close();
+              } catch (IOException ioe) {
+                LOG.error("Couldn't close log at " + wap.p, ioe);
+                thrown.add(ioe);
+                continue;
+              }
+              LOG.info("Closed path " + wap.p + " (wrote " + wap.editsWritten
+                  + " edits in " + (wap.nanosSpent / 1000 / 1000) + "ms)");
+            }
           }
-          LOG.info("Closed path " + wap.p + " (wrote " + wap.editsWritten
-              + " edits in " + (wap.nanosSpent / 1000 / 1000) + "ms)");
+          logWritersClosed = true;
         }
-        logWritersClosed = true;
       }
       return thrown;
     }

From 4fcb147579b81ab10e7898a3c21d8aff2c265b7e Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 3 Apr 2013 18:00:54 +0000
Subject: [PATCH 0893/1540] HBASE-7401 Remove warning message about running
 'hbase migrate'

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464128 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/FSUtils.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 1242088abbeb..1020bec0e162 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -340,10 +340,11 @@ public static void checkVersion(FileSystem fs, Path rootdir,
 
     // version is deprecated require migration
     // Output on stdout so user sees it in terminal.
-    String msg = "File system needs to be upgraded."
+    String msg = "HBase file layout needs to be upgraded."
       + "  You have version " + version
       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
-      + ".  Run the '${HBASE_HOME}/bin/hbase migrate' script.";
+      + ".  Is your hbase.rootdir valid?  If so, you may need to run "
+      + "'hbase hbck -fixVersionFile'.";
     if (message) {
       System.out.println("WARNING! " + msg);
     }

From 4b34d82bf30b5b46f1beaa24b9a2a9d78fe391c3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Apr 2013 03:49:40 +0000
Subject: [PATCH 0894/1540] HBASE-8229 Replication code logs like crazy if a
 target table cannot be found.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464275 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 9548f55f8451..d4dfbf602522 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -49,6 +49,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -715,6 +716,12 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo) {
         if (ioe instanceof RemoteException) {
           ioe = ((RemoteException) ioe).unwrapRemoteException();
           LOG.warn("Can't replicate because of an error on the remote cluster: ", ioe);
+          if (ioe instanceof TableNotFoundException) {
+            if (sleepForRetries("A table is missing in the peer cluster. "
+                + "Replication cannot proceed without losing data.", sleepMultiplier)) {
+              sleepMultiplier++;
+            }
+          }
         } else {
           if (ioe instanceof SocketTimeoutException) {
             // This exception means we waited for more than 60s and nothing

From d61afe819233734ffedaf1f8e4bbd350c962c526 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 4 Apr 2013 03:54:38 +0000
Subject: [PATCH 0895/1540] HBASE-8230 Possible NPE on regionserver abort if
 replication service has not been started

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464282 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/replication/regionserver/Replication.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 954ebe1c2016..509112a28911 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -126,7 +126,9 @@ public void stopReplicationService() {
   public void join() {
     if (this.replication) {
       this.replicationManager.join();
-      this.replicationSink.stopReplicationSinkServices();
+      if (this.replicationSink != null) {
+        this.replicationSink.stopReplicationSinkServices();
+      }
     }
   }
 

From 8973f94a96b1d1ac1ce0a7affa2e4a668c913dc6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Apr 2013 04:37:16 +0000
Subject: [PATCH 0896/1540] HBASE-8188 Avoid unnecessary row compare in
 StoreScanner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464286 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/StoreScanner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index bed4ad2cfacb..db2bd8d274f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -343,7 +343,7 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
     byte[] row = peeked.getBuffer();
     int offset = peeked.getRowOffset();
     short length = peeked.getRowLength();
-    if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
+    if (limit < 0 || matcher.row == null || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
       matcher.setRow(row, offset, length);
     }
 

From 8a362b1e7147c02b75ca086d154fae00d84d3773 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 4 Apr 2013 04:55:31 +0000
Subject: [PATCH 0897/1540] HBASE-8231 delete tests in
 table_tests.rb(TestShell) always running on empty table

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464289 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/ruby/hbase/table_test.rb | 36 ++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/test/ruby/hbase/table_test.rb b/src/test/ruby/hbase/table_test.rb
index c87921dba992..7dd1dc8558e6 100644
--- a/src/test/ruby/hbase/table_test.rb
+++ b/src/test/ruby/hbase/table_test.rb
@@ -106,6 +106,22 @@ def setup
       @test_name = "hbase_shell_tests_table"
       create_test_table(@test_name)
       @test_table = table(@test_name)
+      
+      # Insert data to perform delete operations
+      @test_table.put("101", "x:a", "1")
+      @test_table.put("101", "x:a", "2", Time.now.to_i)
+      
+      @test_table.put("102", "x:a", "1",1212)
+      @test_table.put("102", "x:a", "2", 1213)
+      
+      @test_table.put(103, "x:a", "3")
+      @test_table.put(103, "x:a", "4")
+      
+      @test_table.put("104", "x:a", 5)
+      @test_table.put("104", "x:b", 6)
+      
+      @test_table.put(105, "x:a", "3")
+      @test_table.put(105, "x:a", "4")
     end
 
     define_test "put should work without timestamp" do
@@ -127,25 +143,35 @@ def setup
     #-------------------------------------------------------------------------------
 
     define_test "delete should work without timestamp" do
-      @test_table.delete("123", "x:a")
+      @test_table.delete("101", "x:a")
+      res = @test_table._get_internal('101', 'x:a')
+      assert_nil(res)
     end
 
     define_test "delete should work with timestamp" do
-      @test_table.delete("123", "x:a", Time.now.to_i)
+      @test_table.delete("102", "x:a", 1214)
+      res = @test_table._get_internal('102', 'x:a')
+      assert_nil(res)
     end
 
     define_test "delete should work with integer keys" do
-      @test_table.delete(123, "x:a")
+      @test_table.delete(103, "x:a")
+      res = @test_table._get_internal('103', 'x:a')
+      assert_nil(res)
     end
 
     #-------------------------------------------------------------------------------
 
     define_test "deleteall should work w/o columns and timestamps" do
-      @test_table.deleteall("123")
+      @test_table.deleteall("104")
+      res = @test_table._get_internal('104', 'x:a', 'x:b')
+      assert_nil(res)
     end
 
     define_test "deleteall should work with integer keys" do
-      @test_table.deleteall(123)
+      @test_table.deleteall(105)
+      res = @test_table._get_internal('105', 'x:a')
+      assert_nil(res)
     end
 
     #-------------------------------------------------------------------------------

From b58685965472f81283564841031e0d96d100e171 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Apr 2013 06:01:10 +0000
Subject: [PATCH 0898/1540] HBASE-7599  Port HBASE-6066 (low hanging read path
 improvements) to 0.94 - retry (Devaraj Das)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464295 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 53 +++++++++++--------
 .../hbase/regionserver/HRegionServer.java     | 12 +++--
 2 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 1808efae5525..2da9ba1bc7d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3687,7 +3687,6 @@ class RegionScannerImpl implements RegionScanner {
     private final KeyValue KV_LIMIT = new KeyValue();
     private final byte [] stopRow;
     private Filter filter;
-    private List<KeyValue> results = new ArrayList<KeyValue>();
     private int batch;
     private int isScan;
     private boolean filterClosed = false;
@@ -3806,11 +3805,16 @@ public boolean nextRaw(List<KeyValue> outResults, String metric)
     @Override
     public boolean nextRaw(List<KeyValue> outResults, int limit,
         String metric) throws IOException {
-      results.clear();
-
-      boolean returnResult = nextInternal(limit, metric);
-
-      outResults.addAll(results);
+      boolean returnResult;
+      if (outResults.isEmpty()) {
+        // Usually outResults is empty. This is true when next is called
+        // to handle scan or get operation.
+        returnResult = nextInternal(outResults, limit, metric);
+      } else {
+        List<KeyValue> tmpList = new ArrayList<KeyValue>();
+        returnResult = nextInternal(tmpList, limit, metric);
+        outResults.addAll(tmpList);
+      }
       resetFilters();
       if (isFilterDone()) {
         return false;
@@ -3832,10 +3836,12 @@ public boolean next(List<KeyValue> outResults, String metric)
       return next(outResults, batch, metric);
     }
 
-    private void populateFromJoinedHeap(int limit, String metric) throws IOException {
+    private void populateFromJoinedHeap(List<KeyValue> results, int limit, String metric)
+        throws IOException {
       assert joinedContinuationRow != null;
-      KeyValue kv = populateResult(this.joinedHeap, limit, joinedContinuationRow.getBuffer(),
-        joinedContinuationRow.getRowOffset(), joinedContinuationRow.getRowLength(), metric);
+      KeyValue kv = populateResult(results, this.joinedHeap, limit,
+        joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(),
+        joinedContinuationRow.getRowLength(), metric);
       if (kv != KV_LIMIT) {
         // We are done with this row, reset the continuation.
         joinedContinuationRow = null;
@@ -3847,6 +3853,7 @@ private void populateFromJoinedHeap(int limit, String metric) throws IOException
 
     /**
      * Fetches records with this row into result list, until next row or limit (if not -1).
+     * @param results
      * @param heap KeyValueHeap to fetch data from. It must be positioned on correct row before call.
      * @param limit Max amount of KVs to place in result list, -1 means no limit.
      * @param currentRow Byte array with key we are fetching.
@@ -3855,8 +3862,8 @@ private void populateFromJoinedHeap(int limit, String metric) throws IOException
      * @param metric Metric key to be passed into KeyValueHeap::next().
      * @return true if limit reached, false otherwise.
      */
-    private KeyValue populateResult(KeyValueHeap heap, int limit, byte[] currentRow, int offset,
-        short length, String metric) throws IOException {
+    private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int limit,
+        byte[] currentRow, int offset, short length, String metric) throws IOException {
       KeyValue nextKv;
       do {
         heap.next(results, limit - results.size(), metric);
@@ -3875,7 +3882,11 @@ public synchronized boolean isFilterDone() {
       return this.filter != null && this.filter.filterAllRemaining();
     }
 
-    private boolean nextInternal(int limit, String metric) throws IOException {
+    private boolean nextInternal(List<KeyValue> results, int limit, String metric)
+    throws IOException {
+      if (!results.isEmpty()) {
+        throw new IllegalArgumentException("First parameter should be an empty list");
+      }
       RpcCallContext rpcCall = HBaseServer.getCurrentCall();
       // The loop here is used only when at some point during the next we determine
       // that due to effects of filters or otherwise, we have an empty row in the result.
@@ -3920,13 +3931,15 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           // Check if rowkey filter wants to exclude this row. If so, loop to next.
           // Techically, if we hit limits before on this row, we don't need this call.
           if (filterRowKey(currentRow, offset, length)) {
+            results.clear();
             boolean moreRows = nextRow(currentRow, offset, length);
             if (!moreRows) return false;
             continue;
           }
 
           // Ok, we are good, let's try to get some results from the main heap.
-          KeyValue nextKv = populateResult(this.storeHeap, limit, currentRow, offset, length, metric);
+          KeyValue nextKv = populateResult(results, this.storeHeap, limit, currentRow, offset,
+              length, metric);
           if (nextKv == KV_LIMIT) {
             if (this.filter != null && filter.hasFilterRow()) {
               throw new IncompatibleFilterException(
@@ -3934,7 +3947,8 @@ private boolean nextInternal(int limit, String metric) throws IOException {
             }
             return true; // We hit the limit.
           }
-          stopRow = nextKv == null || isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
+          stopRow = nextKv == null
+              || isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
           // save that the row was empty before filters applied to it.
           final boolean isEmptyRow = results.isEmpty();
 
@@ -3945,11 +3959,7 @@ private boolean nextInternal(int limit, String metric) throws IOException {
           }
 
           if (isEmptyRow || filterRow()) {
-            // this seems like a redundant step - we already consumed the row
-            // there're no left overs.
-            // the reasons for calling this method are:
-            // 1. reset the filters.
-            // 2. provide a hook to fast forward the row (used by subclasses)
+            results.clear();
             boolean moreRows = nextRow(currentRow, offset, length);
             if (!moreRows) return false;
 
@@ -3973,12 +3983,12 @@ private boolean nextInternal(int limit, String metric) throws IOException {
                   && joinedHeap.peek().matchingRow(currentRow, offset, length));
             if (mayHaveData) {
               joinedContinuationRow = current;
-              populateFromJoinedHeap(limit, metric);
+              populateFromJoinedHeap(results, limit, metric);
             }
           }
         } else {
           // Populating from the joined map was stopped by limits, populate some more.
-          populateFromJoinedHeap(limit, metric);
+          populateFromJoinedHeap(results, limit, metric);
         }
 
         // We may have just called populateFromJoinedMap and hit the limits. If that is
@@ -4015,7 +4025,6 @@ protected boolean nextRow(byte [] currentRow, int offset, short length) throws I
       while((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
         this.storeHeap.next(MOCKED_LIST);       
       }
-      results.clear();
       resetFilters();
       // Calling the hook in CP which allows it to do a fast forward
       if (this.region.getCoprocessorHost() != null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7e24e48a11b4..e45b46106038 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2570,8 +2570,10 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
             results, nbRows);
         if (!results.isEmpty()) {
           for (Result r : results) {
-            for (KeyValue kv : r.raw()) {
-              currentScanResultSize += kv.heapSize();
+            if (maxScannerResultSize < Long.MAX_VALUE){
+              for (KeyValue kv : r.raw()) {
+                currentScanResultSize += kv.heapSize();
+              }
             }
           }
         }
@@ -2591,8 +2593,10 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
             // Collect values to be returned here
             boolean moreRows = s.nextRaw(values, SchemaMetrics.METRIC_NEXTSIZE);
             if (!values.isEmpty()) {
-              for (KeyValue kv : values) {
-                currentScanResultSize += kv.heapSize();
+              if (maxScannerResultSize < Long.MAX_VALUE){
+                for (KeyValue kv : values) {
+                  currentScanResultSize += kv.heapSize();
+                }
               }
               results.add(new Result(values));
             }

From 0c47666dee3f8ade3241b64d89c2b8976abe7074 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 4 Apr 2013 14:14:38 +0000
Subject: [PATCH 0899/1540] HBASE-8259 Snapshot backport in 0.94.6 breaks
 rolling restarts

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464562 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/executor/EventHandler.java    | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
index 1be8a0fd63eb..46dafe0d8009 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
@@ -95,10 +95,10 @@ public interface EventHandlerListener {
    * originated and then where its destined -- e.g. RS2ZK_ prefix means the
    * event came from a regionserver destined for zookeeper -- and then what
    * the even is; e.g. REGION_OPENING.
-   * 
-   * <p>We give the enums indices so we can add types later and keep them
-   * grouped together rather than have to add them always to the end as we
-   * would have to if we used raw enum ordinals.
+   *
+   * <p>WARNING: Please do not insert, remove or swap any line in this enum
+   * Doing so would change or shift all the codes used to serialize
+   * events, which makes backwards compatibility very hard for clients.
    */
   public enum EventType {
     // Messages originating from RS (NOTE: there is NO direct communication from
@@ -128,8 +128,6 @@ public enum EventType {
     C_M_DELETE_FAMILY         (45),   // Client asking Master to delete family of table
     C_M_MODIFY_FAMILY         (46),   // Client asking Master to modify family of table
     C_M_CREATE_TABLE          (47),   // Client asking Master to create a table
-    C_M_SNAPSHOT_TABLE        (48),   // Client asking Master to snapshot an offline table
-    C_M_RESTORE_SNAPSHOT      (49),   // Client asking Master to snapshot an offline table
 
     // Updates from master to ZK. This is done by the master and there is
     // nothing to process by either Master or RS
@@ -138,7 +136,13 @@ public enum EventType {
 
     // Master controlled events to be executed on the master
     M_SERVER_SHUTDOWN         (70),  // Master is processing shutdown of a RS
-    M_META_SERVER_SHUTDOWN    (72);  // Master is processing shutdown of RS hosting a meta region (-ROOT- or .META.).
+    M_META_SERVER_SHUTDOWN    (72),  // Master is processing shutdown of RS hosting a meta region (-ROOT- or .META.).
+
+    // WARNING: Please do not insert, remove or swap any line in this enum.
+    // RegionTransitionData.write() uses eventType.ordinal() that is the enum index
+    // and not the value specified in the enum definition. so we can't add stuff in the middle.
+    C_M_SNAPSHOT_TABLE        (48),   // Client asking Master to snapshot an offline table
+    C_M_RESTORE_SNAPSHOT      (49);   // Client asking Master to snapshot an offline table
 
     /**
      * Constructor

From 3b7c081120cbb3edee94f521485eee8be8ab640f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 4 Apr 2013 16:12:22 +0000
Subject: [PATCH 0900/1540] HBASE-8231 delete tests in
 table_tests.rb(TestShell) always running on empty table; REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464621 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/ruby/hbase/table_test.rb | 36 +++++--------------------------
 1 file changed, 5 insertions(+), 31 deletions(-)

diff --git a/src/test/ruby/hbase/table_test.rb b/src/test/ruby/hbase/table_test.rb
index 7dd1dc8558e6..c87921dba992 100644
--- a/src/test/ruby/hbase/table_test.rb
+++ b/src/test/ruby/hbase/table_test.rb
@@ -106,22 +106,6 @@ def setup
       @test_name = "hbase_shell_tests_table"
       create_test_table(@test_name)
       @test_table = table(@test_name)
-      
-      # Insert data to perform delete operations
-      @test_table.put("101", "x:a", "1")
-      @test_table.put("101", "x:a", "2", Time.now.to_i)
-      
-      @test_table.put("102", "x:a", "1",1212)
-      @test_table.put("102", "x:a", "2", 1213)
-      
-      @test_table.put(103, "x:a", "3")
-      @test_table.put(103, "x:a", "4")
-      
-      @test_table.put("104", "x:a", 5)
-      @test_table.put("104", "x:b", 6)
-      
-      @test_table.put(105, "x:a", "3")
-      @test_table.put(105, "x:a", "4")
     end
 
     define_test "put should work without timestamp" do
@@ -143,35 +127,25 @@ def setup
     #-------------------------------------------------------------------------------
 
     define_test "delete should work without timestamp" do
-      @test_table.delete("101", "x:a")
-      res = @test_table._get_internal('101', 'x:a')
-      assert_nil(res)
+      @test_table.delete("123", "x:a")
     end
 
     define_test "delete should work with timestamp" do
-      @test_table.delete("102", "x:a", 1214)
-      res = @test_table._get_internal('102', 'x:a')
-      assert_nil(res)
+      @test_table.delete("123", "x:a", Time.now.to_i)
     end
 
     define_test "delete should work with integer keys" do
-      @test_table.delete(103, "x:a")
-      res = @test_table._get_internal('103', 'x:a')
-      assert_nil(res)
+      @test_table.delete(123, "x:a")
     end
 
     #-------------------------------------------------------------------------------
 
     define_test "deleteall should work w/o columns and timestamps" do
-      @test_table.deleteall("104")
-      res = @test_table._get_internal('104', 'x:a', 'x:b')
-      assert_nil(res)
+      @test_table.deleteall("123")
     end
 
     define_test "deleteall should work with integer keys" do
-      @test_table.deleteall(105)
-      res = @test_table._get_internal('105', 'x:a')
-      assert_nil(res)
+      @test_table.deleteall(123)
     end
 
     #-------------------------------------------------------------------------------

From ceefc9a6a86296fe9bd68c4538be4ad7a7c8f0bb Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 4 Apr 2013 20:21:17 +0000
Subject: [PATCH 0901/1540] HBASE-8127 Region of a disabling or disabled table
 could be stuck in transition state when RS dies during Master initialization
 (Rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464724 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 24 ++++++-
 .../master/handler/ServerShutdownHandler.java | 34 +++++-----
 .../hbase/master/TestAssignmentManager.java   | 51 ++++++++++-----
 .../TestRSKilledWhenMasterInitializing.java   | 62 ++++++++++++++++++-
 4 files changed, 132 insertions(+), 39 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index bca10e406b8a..532ef2d20f70 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2836,6 +2836,12 @@ private void processDeadServersAndRecoverLostRegions(
         // see HBASE-5916
         if (actualDeadServers.contains(deadServer.getKey())) {
           for (Pair<HRegionInfo, Result> deadRegion : deadServer.getValue()) {
+            HRegionInfo hri = deadRegion.getFirst();
+            // Delete znode of region in transition if table is disabled or disabling. If a region
+            // server went down during master initialization then SSH cannot handle the regions of
+            // partially disabled tables because in memory region state information may not be
+            // available with master.
+            deleteNodeAndOfflineRegion(hri);
             nodes.remove(deadRegion.getFirst().getEncodedName());
           }
           continue;
@@ -2884,6 +2890,22 @@ private void processDeadServersAndRecoverLostRegions(
     }
   }
 
+  /**
+   * Delete znode of region in transition if table is disabling/disabled and offline the region.
+   * @param hri region to offline.
+   */
+  public void deleteNodeAndOfflineRegion(HRegionInfo hri) {
+    if (zkTable.isDisablingOrDisabledTable(hri.getTableNameAsString())) {
+      try {
+        // If table is partially disabled then delete znode if exists in any state.
+        ZKAssign.deleteNodeFailSilent(this.master.getZooKeeper(), hri);
+      } catch (KeeperException ke) {
+        this.master.abort("Unexpected ZK exception deleting unassigned node " + hri, ke);
+      }
+      regionOffline(hri);
+    }
+  }
+
   /*
    * Presumes caller has taken care of necessary locking modifying servers Map.
    * @param hsi
@@ -3580,7 +3602,7 @@ public void write(DataOutput out) throws IOException {
       out.writeLong(stamp.get());
     }
   }
-
+  
   public void stop() {
     this.timeoutMonitor.interrupt();
     this.timerUpdater.interrupt();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 0c4bc98f743c..2be3a84dcfd5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -288,31 +288,29 @@ private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Res
       // If the table was partially disabled and the RS went down, we should clear the RIT
       // and remove the node for the region. The rit that we use may be stale in case the table
       // was in DISABLING state but though we did assign we will not be clearing the znode in
-      // CLOSING state. Doing this will have no harm. See HBASE-5927
-      toAssign = checkForDisablingOrDisabledTables(ritsGoingToServer, toAssign, rit, assignmentManager);
+      // CLOSING state. Doing this will have no harm. The rit can be null if region server went
+      // down during master startup. In that case If any znodes' exists for partially disabled 
+      // table regions deleting them during startup only. See HBASE-8127. 
+      toAssign =
+          checkForDisablingOrDisabledTables(ritsGoingToServer, toAssign, rit, e.getKey(),
+            assignmentManager);
     }
     return toAssign;
   }
 
   private List<HRegionInfo> checkForDisablingOrDisabledTables(Set<HRegionInfo> regionsFromRIT,
-      List<HRegionInfo> toAssign, RegionState rit, AssignmentManager assignmentManager) {
-    if (rit == null) {
-      return toAssign;
+      List<HRegionInfo> toAssign, RegionState rit, HRegionInfo hri,
+      AssignmentManager assignmentManager) {
+    boolean disabled =
+        assignmentManager.getZKTable().isDisablingOrDisabledTable(hri.getTableNameAsString());
+    if (disabled) {
+      // To avoid region assignment if table is in disabling or disabled state.
+      toAssign.remove(hri);
+      regionsFromRIT.remove(hri);
     }
-    if (!rit.isClosing() && !rit.isPendingClose()) {
-      return toAssign;
+    if (rit != null && disabled) {
+      assignmentManager.deleteNodeAndOfflineRegion(hri);
     }
-    if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(
-        rit.getRegion().getTableNameAsString())) {
-      return toAssign;
-    }
-    HRegionInfo hri = rit.getRegion();
-    AssignmentManager am = assignmentManager;
-    am.deleteClosingOrClosedNode(hri);
-    am.regionOffline(hri);
-    // To avoid region assignment if table is in disabling or disabled state.
-    toAssign.remove(hri);
-    regionsFromRIT.remove(hri);
     return toAssign;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 5295dc25df8d..aa29ca346f2a 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -431,10 +431,18 @@ public void testShutdownHandler() throws KeeperException, IOException {
   @Test
   public void testSSHWhenDisableTableInProgress()
       throws KeeperException, IOException {
-    testCaseWithPartiallyDisabledState(TableState.DISABLING);
-    testCaseWithPartiallyDisabledState(TableState.DISABLED);
+    testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
+    testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
   }
 
+  @Test
+  public void testSSHWhenDisablingTableRegionsInOpeningState()
+      throws KeeperException, IOException {
+    testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
+    testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
+  }
+
+  
   /**
    * To test if the split region is removed from RIT if the region was in SPLITTING state
    * but the RS has actually completed the splitting in META but went down. See HBASE-6070
@@ -500,7 +508,8 @@ private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws Keep
     }
   }
 
-  private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperException, IOException, NodeExistsException {
+  private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
+      throws KeeperException, IOException, NodeExistsException {
     // Create and startup an executor. This is used by AssignmentManager
     // handling zk callbacks.
     ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
@@ -511,20 +520,32 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
     // Create an AM.
     AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
         executor);
-    // adding region to regions and servers maps.
-    am.regionOnline(REGIONINFO, SERVERNAME_A);
-    // adding region in pending close.
-    am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
-        State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
-
+    if (opening) {
+      am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
+          State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
+    } else {
+      // adding region to regions and servers maps.
+      am.regionOnline(REGIONINFO, SERVERNAME_A);
+      // adding region in pending close.
+      am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
+          State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
+    }
     if (state == TableState.DISABLING) {
       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
     } else {
       am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
     }
+    RegionTransitionData data = null;
+    if (opening) {
+      data =
+          new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
+              SERVERNAME_A);
 
-    RegionTransitionData data = new RegionTransitionData(EventType.M_ZK_REGION_CLOSING,
-        REGIONINFO.getRegionName(), SERVERNAME_A);
+    } else {
+      data =
+          new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
+              SERVERNAME_A);
+    }
     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
     // create znode in M_ZK_REGION_CLOSING state.
     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
@@ -534,12 +555,8 @@ private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperE
       // check znode deleted or not.
       // In both cases the znode should be deleted.
       assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
-      // check whether in rit or not.  In the DISABLING case also the below assert will be true
-      // but the piece of code added for HBASE-5927 will not do that.
-      if (state == TableState.DISABLED) {
-        assertTrue("Region state of region in pending close should be removed from rit.",
-            am.regionsInTransition.isEmpty());
-      }
+      assertTrue("Region state of region in pending close should be removed from rit.",
+        am.regionsInTransition.isEmpty());
     } finally {
       executor.shutdown();
       am.shutdown();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index a7287a1b84a2..3d1da30d1227 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -25,7 +25,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,6 +42,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
@@ -51,6 +51,7 @@
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
@@ -64,7 +65,7 @@ public class TestRSKilledWhenMasterInitializing {
 
   private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
   private static final int NUM_MASTERS = 1;
-  private static final int NUM_RS = 4;
+  private static final int NUM_RS = 5;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
@@ -72,7 +73,7 @@ public static void setUpBeforeClass() throws Exception {
     Configuration conf = TESTUTIL.getConfiguration();
     conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
-    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 4);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 5);
 
     // Start up the cluster.
     TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
@@ -234,6 +235,61 @@ public void testCorrectnessWhenMasterFailOver() throws Exception {
     assertEquals(3, count);
   }
 
+  @Test (timeout=180000)
+  public void testMasterFailoverWhenDisablingTableRegionsInRITOnDeadRS() throws Exception {
+    MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
+    HMaster master = cluster.getMaster();
+    // disable load balancing on this master
+    master.balanceSwitch(false);
+
+    final String table = "testMasterFailoverWhenDisablingTableRegionsInRITOnDeadRS";
+    byte [] FAMILY = Bytes.toBytes("family");
+    byte[][] SPLIT_KEYS =
+        new byte[][] {Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
+            Bytes.toBytes("d") };
+    HTableDescriptor htd = new HTableDescriptor(table);
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+    htd.addFamily(hcd);
+    TESTUTIL.getHBaseAdmin().createTable(htd, SPLIT_KEYS);
+    AssignmentManager am = cluster.getMaster().getAssignmentManager();
+    List<HRegionInfo> regionsOfTable = null;
+    while ((regionsOfTable = am.getRegionsOfTable(table.getBytes())).size()
+        != (SPLIT_KEYS.length + 1)) {
+      Thread.sleep(10);
+    }
+    HRegionInfo closingRegion = regionsOfTable.get(0);
+    ServerName serverName = am.getRegionServerOfRegion(closingRegion);
+    HRegionServer deadRS = null;
+    for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
+      deadRS = cluster.getRegionServer(i);
+      if (deadRS.getServerName().equals(serverName)) {
+        break;
+      }
+    }
+
+    // Disable the table in ZK
+    ZKTable zkTable = am.getZKTable();
+    zkTable.setDisablingTable(table);
+    ZKAssign.createNodeClosing(master.getZooKeeper(), closingRegion, serverName);
+
+    // Stop the master
+    abortMaster(cluster);
+    master = startMasterAndWaitUntilLogSplit(cluster);
+    deadRS.kill();
+    deadRS.join();
+    waitUntilMasterIsInitialized(master);
+    am = cluster.getMaster().getAssignmentManager();
+    zkTable = am.getZKTable();
+    // wait for no more RIT
+    ZKAssign.blockUntilNoRIT(master.getZooKeeper());
+    while (!master.getAssignmentManager().getZKTable().isDisabledTable(table)) {
+      Thread.sleep(10);
+    }
+    assertTrue("Table should be disabled state.", zkTable.isDisabledTable(table));
+    HBaseAdmin admin = new HBaseAdmin(master.getConfiguration());
+    admin.deleteTable(table);
+  }
+
   private void abortMaster(MiniHBaseCluster cluster)
       throws InterruptedException {
     for (MasterThread mt : cluster.getLiveMasterThreads()) {

From 77df78960827a463421659492ab42e8fac41705d Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 4 Apr 2013 20:55:30 +0000
Subject: [PATCH 0902/1540] HBASE-7817: Suggested JDWP debug options in
 hbase-env.sh are wrong (Vasu Mariyala)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464745 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase-config.sh |  4 +++-
 bin/hbase-daemon.sh |  7 +++++--
 conf/hbase-env.sh   | 31 +++++++++++++++++++++----------
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 1f25713a9436..966744ae52e6 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -77,8 +77,10 @@ HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}"
 HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}"
 
 # Source the hbase-env.sh.  Will have JAVA_HOME defined.
-if [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then
+# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it.
+if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then
   . "${HBASE_CONF_DIR}/hbase-env.sh"
+  export HBASE_ENV_INIT="true"
 fi
 
 # Newer versions of glibc use an arena memory allocator that causes virtual
diff --git a/bin/hbase-daemon.sh b/bin/hbase-daemon.sh
index 569bbb3a385d..6f680905cdad 100755
--- a/bin/hbase-daemon.sh
+++ b/bin/hbase-daemon.sh
@@ -123,8 +123,11 @@ loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc
 loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}"
 pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid
 
-if [ "$HBASE_USE_GC_LOGFILE" = "true" ]; then
-  export HBASE_GC_OPTS=" -Xloggc:${loggc}"
+if [ -n "$SERVER_GC_OPTS" ]; then
+  export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:<FILE-PATH>"/"-Xloggc:${loggc}"}
+fi
+if [ -n "$CLIENT_GC_OPTS" ]; then
+  export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:<FILE-PATH>"/"-Xloggc:${loggc}"}
 fi
 
 # Set default scheduling priority
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index da53a27f7f76..7c740104f761 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -40,20 +40,31 @@
 # see http://wiki.apache.org/hadoop/PerformanceTuning
 export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
 
-# Uncomment below to enable java garbage collection logging for the server-side processes
-# this enables basic gc logging for the server processes to the .out file
-# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes.
 
-# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
-# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
+# This enables basic gc logging to the .out file.
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 
-# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
-# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+# This enables basic gc logging to its own file.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>"
 
-# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
-# This applies to both the server and client GC options above
-# export HBASE_USE_GC_LOGFILE=true
+# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M"
 
+# Uncomment one of the below three options to enable java garbage collection logging for the client processes.
+
+# This enables basic gc logging to the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+
+# This enables basic gc logging to its own file.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>"
+
+# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M"
 
 # Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
 # export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="

From 479d0c7cb3877d2cd491523e617516208aaafe15 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Apr 2013 21:02:59 +0000
Subject: [PATCH 0903/1540] HBASE-8231 delete tests in
 table_tests.rb(TestShell) always running on empty table. (rajeshbabu and
 LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464761 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/ruby/hbase/table_test.rb | 36 ++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/test/ruby/hbase/table_test.rb b/src/test/ruby/hbase/table_test.rb
index c87921dba992..067058d8a01f 100644
--- a/src/test/ruby/hbase/table_test.rb
+++ b/src/test/ruby/hbase/table_test.rb
@@ -106,6 +106,22 @@ def setup
       @test_name = "hbase_shell_tests_table"
       create_test_table(@test_name)
       @test_table = table(@test_name)
+      
+      # Insert data to perform delete operations
+      @test_table.put("101", "x:a", "1")
+      @test_table.put("101", "x:a", "2", Time.now.to_i)
+      
+      @test_table.put("102", "x:a", "1",1212)
+      @test_table.put("102", "x:a", "2", 1213)
+      
+      @test_table.put(103, "x:a", "3")
+      @test_table.put(103, "x:a", "4")
+      
+      @test_table.put("104", "x:a", 5)
+      @test_table.put("104", "x:b", 6)
+      
+      @test_table.put(105, "x:a", "3")
+      @test_table.put(105, "x:a", "4")
     end
 
     define_test "put should work without timestamp" do
@@ -127,25 +143,35 @@ def setup
     #-------------------------------------------------------------------------------
 
     define_test "delete should work without timestamp" do
-      @test_table.delete("123", "x:a")
+      @test_table.delete("101", "x:a")
+      res = @test_table.get('101', 'x:a')
+      assert_nil(res)
     end
 
     define_test "delete should work with timestamp" do
-      @test_table.delete("123", "x:a", Time.now.to_i)
+      @test_table.delete("102", "x:a", 1214)
+      res = @test_table.get('102', 'x:a')
+      assert_nil(res)
     end
 
     define_test "delete should work with integer keys" do
-      @test_table.delete(123, "x:a")
+      @test_table.delete(103, "x:a")
+      res = @test_table.get('103', 'x:a')
+      assert_nil(res)
     end
 
     #-------------------------------------------------------------------------------
 
     define_test "deleteall should work w/o columns and timestamps" do
-      @test_table.deleteall("123")
+      @test_table.deleteall("104")
+      res = @test_table.get('104', 'x:a', 'x:b')
+      assert_nil(res)
     end
 
     define_test "deleteall should work with integer keys" do
-      @test_table.deleteall(123)
+      @test_table.deleteall(105)
+      res = @test_table.get('105', 'x:a')
+      assert_nil(res)
     end
 
     #-------------------------------------------------------------------------------

From 8c55bcfa2c76a8352d017b3daec948436a27b07b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Apr 2013 23:11:13 +0000
Subject: [PATCH 0904/1540] HBASE-8211 Support for NN HA for 0.94 (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464785 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HBaseFileSystem.java  | 263 ++++++++++++++++++
 .../hadoop/hbase/backup/HFileArchiver.java    |  26 +-
 .../org/apache/hadoop/hbase/io/Reference.java |   3 +-
 .../hadoop/hbase/master/MasterFileSystem.java |  19 +-
 .../hadoop/hbase/master/SplitLogManager.java  |   3 +-
 .../hbase/master/cleaner/CleanerChore.java    |   9 +-
 .../master/handler/CreateTableHandler.java    |   3 +-
 .../master/handler/DeleteTableHandler.java    |   3 +-
 .../hadoop/hbase/regionserver/HRegion.java    |  19 +-
 .../hbase/regionserver/HRegionFileSystem.java |  38 +++
 .../hbase/regionserver/SplitTransaction.java  |   8 +-
 .../hadoop/hbase/regionserver/Store.java      |   8 +-
 .../hadoop/hbase/regionserver/StoreFile.java  |   7 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |  21 +-
 .../regionserver/wal/HLogFileSystem.java      |  69 +++++
 .../hbase/regionserver/wal/HLogSplitter.java  |  39 +--
 .../hadoop/hbase/util/FSTableDescriptors.java |   9 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |  12 +-
 .../hadoop/hbase/zookeeper/ZKSplitLog.java    |   3 +-
 .../hadoop/hbase/HBaseTestingUtility.java     |   5 +
 .../hadoop/hbase/TestHBaseFileSystem.java     | 178 ++++++++++++
 .../hbase/master/TestOpenedRegionHandler.java |   1 -
 22 files changed, 657 insertions(+), 89 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
new file mode 100644
index 000000000000..eebac4d790b4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
+import org.apache.hadoop.hbase.util.Threads;
+
+/**
+ * An abstraction of the underlying filesystem. This is used by other entities such as
+ * {@link HLogFileSystem}, to make calls to the underlying filesystem.
+ *
+ */
+public abstract class HBaseFileSystem {
+
+  public static final Log LOG = LogFactory.getLog(HBaseFileSystem.class);
+
+  /**
+   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
+   * client level.
+   */
+  protected static  int hdfsClientRetriesNumber;
+  private static int baseSleepBeforeRetries;
+  private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
+  private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
+
+  
+  /**
+   * Deletes a file. Assumes the user has already checked for this directory existence.
+   * @param dir
+   * @param fs
+   * @param conf
+   * @return true if the directory is deleted.
+   * @throws IOException
+   */
+  public static boolean deleteFileFromFileSystem(FileSystem fs, Configuration conf, Path dir)
+      throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.delete(dir, false);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(dir)) return true;
+        // dir is there, retry deleting after some time.
+        sleepBeforeRetry("Delete File", i + 1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in deleteFileFromFileSystem", lastIOE);
+  }
+  
+  
+  /**
+   * Deletes a directory. Assumes the user has already checked for this directory existence.
+   * @param dir
+   * @param fs
+   * @param conf
+   * @return true if the directory is deleted.
+   * @throws IOException
+   */
+  public static boolean deleteDirFromFileSystem(FileSystem fs, Configuration conf, Path dir)
+      throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.delete(dir, true);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(dir)) return true;
+        // dir is there, retry deleting after some time.
+        sleepBeforeRetry("Delete Dir", i + 1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in deleteDirFromFileSystem", lastIOE);
+  }
+
+  protected static void checkAndSetRetryCounts(Configuration conf) {
+    if (hdfsClientRetriesNumber == 0) {
+      hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
+        DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
+      baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
+        DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
+    }
+  }
+  
+  /**
+   * Creates a directory for a filesystem and configuration object. Assumes the user has already
+   * checked for this directory existence.
+   * @param fs
+   * @param conf
+   * @param dir
+   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
+   *         whether the directory exists or not, and returns true if it exists.
+   * @throws IOException
+   */
+  public static boolean makeDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
+      throws IOException {
+    int i = 0;
+    IOException lastIOE = null;
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.mkdirs(dir);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (fs.exists(dir)) return true; // directory is present
+        sleepBeforeRetry("Create Directory", i+1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in makeDirOnFileSystem", lastIOE);
+  }
+  
+  /**
+   * Renames a directory. Assumes the user has already checked for this directory existence.
+   * @param src
+   * @param fs
+   * @param dst
+   * @param conf
+   * @return true if the directory is renamed.
+   * @throws IOException
+   */
+  public static boolean renameDirForFileSystem(FileSystem fs, Configuration conf, Path src, Path dst)
+      throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.rename(src, dst);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(src) && fs.exists(dst)) return true;
+        // src is there, retry renaming after some time.
+        sleepBeforeRetry("Rename Directory", i + 1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in renameDirForFileSystem", lastIOE);
+  }
+  
+/**
+ * Creates a path on the file system. Checks whether the path exists already or not, and use it
+ * for retrying in case underlying fs throws an exception.
+ * @param fs
+ * @param conf
+ * @param dir
+ * @param overwrite
+ * @return
+ * @throws IOException
+ */
+  public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Configuration conf, Path dir,
+      boolean overwrite) throws IOException {
+    int i = 0;
+    boolean existsBefore = fs.exists(dir);
+    IOException lastIOE = null;
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.create(dir, overwrite);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        // directory is present, don't overwrite
+        if (!existsBefore && fs.exists(dir)) return fs.create(dir, false);
+        sleepBeforeRetry("Create Path", i + 1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in createPathOnFileSystem", lastIOE);
+  }
+
+  /**
+   * Creates the specified file with the given permission.
+   * @param fs
+   * @param path
+   * @param perm
+   * @param overwrite
+   * @return
+   * @throws IOException 
+   */
+  public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
+      Configuration conf, Path path, FsPermission perm, boolean overwrite) throws IOException {
+    int i = 0;
+    IOException lastIOE = null;
+    boolean existsBefore = fs.exists(path);
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.create(path, perm, overwrite, fs.getConf().getInt("io.file.buffer.size", 4096),
+          fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        // path is present now, don't overwrite
+        if (!existsBefore && fs.exists(path)) return fs.create(path, false);
+        // if it existed before let's retry in case we get IOE, as we can't rely on fs.exists()
+        sleepBeforeRetry("Create Path with Perms", i + 1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in createPathWithPermsOnFileSystem", lastIOE);
+  }
+
+/**
+ * Creates the file. Assumes the user has already checked for this file existence.
+ * @param fs
+ * @param conf
+ * @param dir
+ * @return result true if the file is created with this call, false otherwise.
+ * @throws IOException
+ */
+  public static boolean createNewFileOnFileSystem(FileSystem fs, Configuration conf, Path file)
+      throws IOException {
+    int i = 0;
+    IOException lastIOE = null;
+    checkAndSetRetryCounts(conf);
+    do {
+      try {
+        return fs.createNewFile(file);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (fs.exists(file)) return true; // file exists now, return true.
+        sleepBeforeRetry("Create NewFile", i + 1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in createNewFileOnFileSystem", lastIOE);
+  }
+  
+  /**
+   * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
+   * for this to avoid re-looking for the integer values.
+   */
+  protected static void sleepBeforeRetry(String msg, int sleepMultiplier) {
+    if (sleepMultiplier > hdfsClientRetriesNumber) {
+      LOG.warn(msg + ", retries exhausted");
+      return;
+    }
+    LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
+    Threads.sleep(baseSleepBeforeRetries * sleepMultiplier);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index 6083328fddaa..808b1f85c207 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -216,7 +217,7 @@ public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegion
     Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family);
 
     // make sure we don't archive if we can't and that the archive dir exists
-    if (!fs.mkdirs(storeArchiveDir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, conf, storeArchiveDir)) {
       throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
           + Bytes.toString(family) + ", deleting compacted files instead.");
     }
@@ -250,7 +251,7 @@ public static void archiveStoreFile(FileSystem fs, HRegionInfo regionInfo,
       Configuration conf, Path tableDir, byte[] family, Path storeFile) throws IOException {
     Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
     // make sure we don't archive if we can't and that the archive dir exists
-    if (!fs.mkdirs(storeArchiveDir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, conf, storeArchiveDir)) {
       throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
           + Bytes.toString(family) + ", deleting compacted files instead.");
     }
@@ -318,7 +319,7 @@ private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
 
     // make sure the archive directory exists
     if (!fs.exists(baseArchiveDir)) {
-      if (!fs.mkdirs(baseArchiveDir)) {
+      if (!HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), baseArchiveDir)) {
         throw new IOException("Failed to create the archive directory:" + baseArchiveDir
             + ", quitting archive attempt.");
       }
@@ -385,11 +386,12 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
 
       // move the archive file to the stamped backup
       Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
-      if (!fs.rename(archiveFile, backedupArchiveFile)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), archiveFile,
+        backedupArchiveFile)) {
         LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
             + ", deleting existing file in favor of newer.");
         // try to delete the exisiting file, if we can't rename it
-        if (!fs.delete(archiveFile, false)) {
+        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, fs.getConf(), archiveFile)) {
           throw new IOException("Couldn't delete existing archive file (" + archiveFile
               + ") or rename it to the backup file (" + backedupArchiveFile
               + ") to make room for similarly named file.");
@@ -410,10 +412,9 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
         // the cleaner has removed our archive directory (HBASE-7643).
         // (we're in a retry loop, so don't worry too much about the exception)
         try {
-          if (!fs.exists(archiveDir)) {
-            if (fs.mkdirs(archiveDir)) {
-              LOG.debug("Created archive directory:" + archiveDir);
-            }
+          if (!fs.exists(archiveDir)
+              && HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), archiveDir)) {
+            LOG.debug("Created archive directory:" + archiveDir);
           }
         } catch (IOException e) {
           LOG.warn("Failed to create the archive directory: " + archiveDir, e);
@@ -476,7 +477,7 @@ private static void deleteFilesWithoutArchiving(Collection<File> files) throws I
    */
   private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
       throws IOException {
-    if (fs.delete(regionDir, true)) {
+    if (HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), regionDir)) {
       LOG.debug("Deleted all region files in: " + regionDir);
       return true;
     }
@@ -610,7 +611,7 @@ public File(FileSystem fs) {
     public boolean moveAndClose(Path dest) throws IOException {
       this.close();
       Path p = this.getPath();
-      return fs.rename(p, dest);
+      return HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), p, dest);
     }
 
     /**
@@ -641,7 +642,8 @@ public FileablePath(FileSystem fs, Path file) {
 
     @Override
     public void delete() throws IOException {
-      if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file);
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), file)) 
+        throw new IOException("Failed to delete:" + this.file);
     }
 
     @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
index 99ecb7e85fbe..1fcea842fa4f 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/Reference.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -125,7 +126,7 @@ public static boolean isTopFileRegion(final Range r) {
 
   public Path write(final FileSystem fs, final Path p)
   throws IOException {
-    FSDataOutputStream out = fs.create(p, false);
+    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, fs.getConf(), p, false);
     try {
       write(out);
     } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 3f12660d7e9d..31c1e9bc63ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -152,7 +153,7 @@ private Path createInitialFileSystemLayout() throws IOException {
 
     // Make sure the region servers can archive their old logs
     if(!this.fs.exists(oldLogDir)) {
-      this.fs.mkdirs(oldLogDir);
+      HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir);
     }
 
     return oldLogDir;
@@ -322,7 +323,7 @@ private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOExcep
       Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
       // rename the directory so a rogue RS doesn't create more HLogs
       if (fs.exists(logDir)) {
-        if (!this.fs.rename(logDir, splitDir)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, logDir, splitDir)) {
           throw new IOException("Failed fs.rename for log split: " + logDir);
         }
         logDir = splitDir;
@@ -415,7 +416,7 @@ private Path checkRootDir(final Path rd, final Configuration c,
     // Filesystem is good. Go ahead and check for hbase.rootdir.
     try {
       if (!fs.exists(rd)) {
-        fs.mkdirs(rd);
+        HBaseFileSystem.makeDirOnFileSystem(fs, c, rd);
         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
         // We used to handle this by checking the current DN count and waiting until
         // it is nonzero. With security, the check for datanode count doesn't work --
@@ -479,13 +480,13 @@ private void checkTempDir(final Path tmpdir, final Configuration c, final FileSy
           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
         }
       }
-      if (!fs.delete(tmpdir, true)) {
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, c, tmpdir)) {
         throw new IOException("Unable to clean the temp directory: " + tmpdir);
       }
     }
 
     // Create the temp directory
-    if (!fs.mkdirs(tmpdir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, c, tmpdir)) {
       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
     }
   }
@@ -553,7 +554,7 @@ public void deleteRegion(HRegionInfo region) throws IOException {
   }
 
   public void deleteTable(byte[] tableName) throws IOException {
-    fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
+    HBaseFileSystem.deleteDirFromFileSystem(fs, conf, new Path(rootdir, Bytes.toString(tableName)));
   }
 
   /**
@@ -566,11 +567,11 @@ public Path moveToTemp(final Path path) throws IOException {
     Path tempPath = new Path(this.tempdir, path.getName());
 
     // Ensure temp exists
-    if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
+    if (!fs.exists(tempdir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, tempdir)) {
       throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
     }
 
-    if (!fs.rename(path, tempPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, path, tempPath)) {
       throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
     }
 
@@ -602,7 +603,7 @@ public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
     // delete the family folder
     Path familyDir = new Path(tableDir,
       new Path(region.getEncodedName(), Bytes.toString(familyName)));
-    if (fs.delete(familyDir, true) == false) {
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, conf, familyDir)) {
       throw new IOException("Could not delete family "
           + Bytes.toString(familyName) + " from FileSystem for region "
           + region.getRegionNameAsString() + "(" + region.getEncodedName()
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index b33aa43104b7..c74659ecfa5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskFinisher.Status;
@@ -295,7 +296,7 @@ public long splitLogDistributed(final List<Path> logDirs, PathFilter filter)
     for(Path logDir: logDirs){
       status.setStatus("Cleaning up log directory...");
       try {
-        if (fs.exists(logDir) && !fs.delete(logDir, false)) {
+        if (fs.exists(logDir) && !HBaseFileSystem.deleteFileFromFileSystem(fs, conf, logDir)) {
           LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);
         }
       } catch (IOException ioe) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index e58634251c5a..6d6da76c1b5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -153,7 +154,7 @@ public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
     // if the directory doesn't exist, then we are done
     if (children == null) {
       try {
-        return fs.delete(toCheck, false);
+        return HBaseFileSystem.deleteFileFromFileSystem(fs, conf, toCheck);
       } catch (IOException e) {
         if (LOG.isTraceEnabled()) {
           LOG.trace("Couldn't delete directory: " + toCheck, e);
@@ -185,7 +186,7 @@ else if (!checkAndDelete(path)) {
     // delete this directory. However, don't do so recursively so we don't delete files that have
     // been added since we last checked.
     try {
-      return fs.delete(toCheck, false);
+      return HBaseFileSystem.deleteFileFromFileSystem(fs, conf, toCheck);
     } catch (IOException e) {
       if (LOG.isTraceEnabled()) {
         LOG.trace("Couldn't delete directory: " + toCheck, e);
@@ -207,7 +208,7 @@ private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumen
     // first check to see if the path is valid
     if (!validate(filePath)) {
       LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
-      boolean success = this.fs.delete(filePath, true);
+      boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, conf, filePath);
       if (!success) LOG.warn("Attempted to delete:" + filePath
           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
 
@@ -233,7 +234,7 @@ private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumen
     if (LOG.isTraceEnabled()) {
       LOG.trace("Removing:" + filePath + " from archive");
     }
-    boolean success = this.fs.delete(filePath, false);
+    boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, conf, filePath);
     if (!success) {
       LOG.warn("Attempted to delete:" + filePath
           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 7e6d5bea3127..8557a7f22f40 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
@@ -174,7 +175,7 @@ private void handleCreateTable(String tableName) throws IOException, KeeperExcep
     List<HRegionInfo> regionInfos = handleCreateHdfsRegions(tempdir, tableName);
 
     // 3. Move Table temp directory to the hbase root location
-    if (!fs.rename(tempTableDir, tableDir)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, tempTableDir, tableDir)) {
       throw new IOException("Unable to move table from temp=" + tempTableDir +
         " to hbase root=" + tableDir);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index 2781671b826b..7c9219309ff4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -26,6 +26,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
@@ -89,7 +90,7 @@ protected void handleTableOperation(List<HRegionInfo> regions)
       }
 
       // 5. Delete table from FS (temp directory)
-      if (!fs.delete(tempTableDir, true)) {
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), tempTableDir)) {
         LOG.error("Couldn't delete " + tempTableDir);
       }
     } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 2da9ba1bc7d8..47c178eee2bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -71,6 +71,7 @@
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
@@ -680,7 +681,7 @@ static void moveInitialFilesIntoPlace(final FileSystem fs,
     final Path initialFiles, final Path regiondir)
   throws IOException {
     if (initialFiles != null && fs.exists(initialFiles)) {
-      if (!fs.rename(initialFiles, regiondir)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), initialFiles, regiondir)) {
         LOG.warn("Unable to rename " + initialFiles + " to " + regiondir);
       }
     }
@@ -842,7 +843,7 @@ public static void writeRegioninfoOnFilesystem(HRegionInfo regionInfo, Path regi
     } finally {
       out.close();
     }
-    if (!fs.rename(tmpPath, regioninfoPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, tmpPath, regioninfoPath)) {
       throw new IOException("Unable to rename " + tmpPath + " to " +
         regioninfoPath);
     }
@@ -2697,7 +2698,7 @@ public void addRegionToSnapshot(SnapshotDescription desc,
         // open and read the files as well).
         LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
         Path referenceFile = new Path(dstStoreDir, file.getName());
-        boolean success = fs.createNewFile(referenceFile);
+        boolean success = HBaseFileSystem.createNewFileOnFileSystem(fs, conf, referenceFile);
         if (!success) {
           throw new IOException("Failed to create reference file:" + referenceFile);
         }
@@ -3106,7 +3107,7 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
     }
     // Now delete the content of recovered edits.  We're done w/ them.
     for (Path file: files) {
-      if (!this.fs.delete(file, false)) {
+      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, file)) {
         LOG.error("Failed delete of " + file);
       } else {
         LOG.debug("Deleted recovered.edits file=" + file);
@@ -3298,7 +3299,7 @@ private static boolean isZeroLengthThenDelete(final FileSystem fs, final Path p)
     FileStatus stat = fs.getFileStatus(p);
     if (stat.getLen() > 0) return false;
     LOG.warn("File " + p + " is zero-length, deleting.");
-    fs.delete(p, false);
+    HBaseFileSystem.deleteFileFromFileSystem(fs, fs.getConf(), p);
     return true;
   }
 
@@ -4221,7 +4222,7 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
         HTableDescriptor.getTableDir(rootDir, info.getTableName());
     Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
     FileSystem fs = FileSystem.get(conf);
-    fs.mkdirs(regionDir);
+    HBaseFileSystem.makeDirOnFileSystem(fs, conf, regionDir);
     // Write HRI to a file in case we need to recover .META.
     writeRegioninfoOnFilesystem(info, regionDir, fs, conf);
     HLog effectiveHLog = hlog;
@@ -4417,7 +4418,7 @@ private static void deleteRegion(FileSystem fs, Path regiondir)
     if (LOG.isDebugEnabled()) {
       LOG.debug("DELETING region " + regiondir.toString());
     }
-    if (!fs.delete(regiondir, true)) {
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), regiondir)) {
       LOG.warn("Failed delete of " + regiondir);
     }
   }
@@ -4463,7 +4464,7 @@ public static void makeColumnFamilyDirs(FileSystem fs, Path tabledir,
     final HRegionInfo hri, byte [] colFamily)
   throws IOException {
     Path dir = Store.getStoreHomedir(tabledir, hri.getEncodedName(), colFamily);
-    if (!fs.mkdirs(dir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), dir)) {
       LOG.warn("Failed to create " + dir);
     }
   }
@@ -4573,7 +4574,7 @@ public static HRegion merge(HRegion a, HRegion b)
       throw new IOException("Cannot merge; target file collision at " +
           newRegionDir);
     }
-    fs.mkdirs(newRegionDir);
+    HBaseFileSystem.makeDirOnFileSystem(fs, conf, newRegionDir);
 
     LOG.info("starting merge of regions: " + a + " and " + b +
       " into new region " + newRegionInfo.toString() +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
new file mode 100644
index 000000000000..b2a7a6f89580
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+
+/**
+ * Acts as an abstraction layer b/w HBase and underlying fs. Used for making non-idempotent calls.
+ * This is useful as it can have a retry logic for such operations, as they are not retried at
+ * hdfs level.
+ * Region specific methods that access fs should be added here.
+ *
+ */
+public class HRegionFileSystem extends HBaseFileSystem {
+  public static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
+
+  public HRegionFileSystem(Configuration conf) {
+    checkAndSetRetryCounts(conf);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 8744c7419a7c..18234734b772 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
@@ -551,12 +552,13 @@ void createSplitDir(final FileSystem fs, final Path splitdir)
     if (fs.exists(splitdir)) {
       LOG.info("The " + splitdir
           + " directory exists.  Hence deleting it to recreate it");
-      if (!fs.delete(splitdir, true)) {
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), splitdir)) {
         throw new IOException("Failed deletion of " + splitdir
             + " before creating them again.");
       }
     }
-    if (!fs.mkdirs(splitdir)) throw new IOException("Failed create of " + splitdir);
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), splitdir))
+        throw new IOException("Failed create of " + splitdir);
   }
 
   private static void cleanupSplitDir(final FileSystem fs, final Path splitdir)
@@ -577,7 +579,7 @@ private static void deleteDir(final FileSystem fs, final Path dir,
   throws IOException {
     if (!fs.exists(dir)) {
       if (mustPreExist) throw new IOException(dir.toString() + " does not exist!");
-    } else if (!fs.delete(dir, true)) {
+    } else if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), dir)) {
       throw new IOException("Failed delete of " + dir);
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 337487e1e564..88b0267e9082 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -288,8 +289,7 @@ long getTTL(final HColumnDescriptor family) {
    */
   Path createStoreHomeDir(final FileSystem fs,
       final Path homedir) throws IOException {
-    if (!fs.exists(homedir)) {
-      if (!fs.mkdirs(homedir))
+    if (!fs.exists(homedir) && !HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), homedir)) {
         throw new IOException("Failed create of: " + homedir.toString());
     }
     return homedir;
@@ -879,7 +879,7 @@ private StoreFile commitFile(final Path path,
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);
-    if (!fs.rename(path, dstPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, path, dstPath)) {
       LOG.warn("Unable to rename " + path + " to " + dstPath);
     }
 
@@ -1649,7 +1649,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
       Path origPath = compactedFile.getPath();
       Path destPath = new Path(homedir, origPath.getName());
       LOG.info("Renaming compacted file at " + origPath + " to " + destPath);
-      if (!fs.rename(origPath, destPath)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, origPath, destPath)) {
         LOG.error("Failed move of compacted file " + origPath + " to " +
             destPath);
         throw new IOException("Failed move of compacted file " + origPath +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 274018730a2d..cdaabb7501f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.KeyValue;
@@ -674,7 +675,7 @@ public synchronized void closeReader(boolean evictOnClose)
    */
   public void deleteReader() throws IOException {
     closeReader(true);
-    this.fs.delete(getPath(), true);
+    HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), getPath());
   }
 
   @Override
@@ -717,7 +718,7 @@ public static Path rename(final FileSystem fs,
     if (!fs.exists(src)) {
       throw new FileNotFoundException(src.toString());
     }
-    if (!fs.rename(src, tgt)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), src, tgt)) {
       throw new IOException("Failed rename of " + src + " to " + tgt);
     }
     return tgt;
@@ -850,7 +851,7 @@ public Writer build() throws IOException {
       }
 
       if (!fs.exists(dir)) {
-        fs.mkdirs(dir);
+        HBaseFileSystem.makeDirOnFileSystem(fs, conf, dir);
       }
 
       if (filePath == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 1030ab0b1297..91baba4c7ec7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -59,6 +59,7 @@
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.Syncable;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -135,6 +136,7 @@ public class HLog implements Syncable {
   private final FileSystem fs;
   private final Path dir;
   private final Configuration conf;
+  private final HLogFileSystem hlogFs;
   // Listeners that are called on WAL events.
   private List<WALActionsListener> listeners =
     new CopyOnWriteArrayList<WALActionsListener>();
@@ -397,6 +399,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     this.fs = fs;
     this.dir = dir;
     this.conf = conf;
+    this.hlogFs = new HLogFileSystem(conf);
     if (listeners != null) {
       for (WALActionsListener i: listeners) {
         registerWALActionsListener(i);
@@ -413,14 +416,12 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     if (failIfLogDirExists && (dirExists = this.fs.exists(dir))) {
       throw new IOException("Target HLog directory already exists: " + dir);
     }
-    if (!dirExists && !fs.mkdirs(dir)) {
+    if (!dirExists && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, dir)) {
       throw new IOException("Unable to mkdir " + dir);
     }
     this.oldLogDir = oldLogDir;
-    if (!fs.exists(oldLogDir)) {
-      if (!fs.mkdirs(this.oldLogDir)) {
-        throw new IOException("Unable to mkdir " + this.oldLogDir);
-      }
+    if (!fs.exists(oldLogDir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir)) {
+      throw new IOException("Unable to mkdir " + this.oldLogDir);
     }
     this.forMeta = forMeta;
     this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32);
@@ -711,7 +712,7 @@ protected Writer createWriterInstance(final FileSystem fs, final Path path,
     if (forMeta) {
       //TODO: set a higher replication for the hlog files (HBASE-6773)
     }
-    return createWriter(fs, path, conf);
+    return this.hlogFs.createWriter(fs, conf, path);
   }
 
   /**
@@ -930,7 +931,7 @@ private void archiveLogFile(final Path p, final Long seqno) throws IOException {
         i.preLogArchive(p, newPath);
       }
     }
-    if (!this.fs.rename(p, newPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, p, newPath)) {
       throw new IOException("Unable to rename " + p + " to " + newPath);
     }
     // Tell our listeners that a log has been archived.
@@ -993,7 +994,7 @@ public void closeAndDelete() throws IOException {
         }
       }
 
-      if (!fs.rename(file.getPath(),p)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, file.getPath(),p)) {
         throw new IOException("Unable to rename " + file.getPath() + " to " + p);
       }
       // Tell our listeners that a log was archived.
@@ -1005,7 +1006,7 @@ public void closeAndDelete() throws IOException {
     }
     LOG.debug("Moved " + files.length + " log files to " +
       FSUtils.getPath(this.oldLogDir));
-    if (!fs.delete(dir, true)) {
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, conf, dir)) {
       LOG.info("Unable to delete " + dir);
     }
   }
@@ -1887,7 +1888,7 @@ public static Path moveAsideBadEditsFile(final FileSystem fs,
   throws IOException {
     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "." +
       System.currentTimeMillis());
-    if (!fs.rename(edits, moveAsideName)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), edits, moveAsideName)) {
       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
     }
     return moveAsideName;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java
new file mode 100644
index 000000000000..29efe4f31359
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.wal.HLog.Writer;
+
+/**
+ * Acts as an abstraction between the HLog and the underlying filesystem. This is analogous to the
+ * {@link HRegionFileSystem} class.
+ */
+public class HLogFileSystem extends HBaseFileSystem {
+  public static final Log LOG = LogFactory.getLog(HLogFileSystem.class);
+
+  /**
+   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
+   * client level.
+   */
+
+  public HLogFileSystem(Configuration conf) {
+    checkAndSetRetryCounts(conf);
+  }
+
+  /**
+   * Creates writer for the given path.
+   * @param fs
+   * @param conf
+   * @param hlogFile
+   * @return an init'ed writer for the given path.
+   * @throws IOException
+   */
+  public Writer createWriter(FileSystem fs, Configuration conf, Path hlogFile) throws IOException {
+    int i = 0;
+    IOException lastIOE = null;
+    do {
+      try {
+        return HLog.createWriter(fs, hlogFile, conf);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        sleepBeforeRetry("Create Writer", i+1);
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in createWriter", lastIOE);
+
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 485ddb33e82f..59e1215fde5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -41,7 +41,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
@@ -91,6 +91,7 @@ public class HLogSplitter {
   protected final Path oldLogDir;
   protected final FileSystem fs;
   protected final Configuration conf;
+  private final HLogFileSystem hlogFs;
 
   // Major subcomponents of the split process.
   // These are separated into inner classes to make testing easier.
@@ -163,6 +164,7 @@ public HLogSplitter(Configuration conf, Path rootDir, Path srcDir,
         conf.getInt("hbase.regionserver.hlog.splitlog.buffersize",
             128*1024*1024));
     outputSink = new OutputSink();
+    this.hlogFs = new HLogFileSystem(conf);
   }
 
   /**
@@ -482,7 +484,7 @@ public boolean splitLogFile(FileStatus logfile,
             LOG.warn("Found existing old edits file. It could be the "
                 + "result of a previous failed split attempt. Deleting " + dst + ", length="
                 + fs.getFileStatus(dst).getLen());
-            if (!fs.delete(dst, false)) {
+            if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, dst)) {
               LOG.warn("Failed deleting of old " + dst);
               throw new IOException("Failed deleting of old " + dst);
             }
@@ -491,7 +493,7 @@ public boolean splitLogFile(FileStatus logfile,
           // data without touching disk. TestHLogSplit#testThreading is an
           // example.
           if (fs.exists(wap.p)) {
-            if (!fs.rename(wap.p, dst)) {
+            if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, wap.p, dst)) {
               throw new IOException("Failed renaming " + wap.p + " to " + dst);
             }
             LOG.debug("Rename " + wap.p + " to " + dst);
@@ -558,7 +560,7 @@ public static void finishSplitLogFile(Path rootdir, Path oldLogDir,
     }
     archiveLogs(null, corruptedLogs, processedLogs, oldLogDir, fs, conf);
     Path stagingDir = ZKSplitLog.getSplitLogDir(rootdir, logPath.getName());
-    fs.delete(stagingDir, true);
+    HBaseFileSystem.deleteDirFromFileSystem(fs, conf, stagingDir);
   }
 
   /**
@@ -581,17 +583,17 @@ private static void archiveLogs(
     final Path corruptDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get(
         "hbase.regionserver.hlog.splitlog.corrupt.dir",  HConstants.CORRUPT_DIR_NAME));
 
-    if (!fs.mkdirs(corruptDir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, conf, corruptDir)) {
       LOG.info("Unable to mkdir " + corruptDir);
     }
-    fs.mkdirs(oldLogDir);
+    HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir);
 
     // this method can get restarted or called multiple times for archiving
     // the same log files.
     for (Path corrupted : corruptedLogs) {
       Path p = new Path(corruptDir, corrupted.getName());
       if (fs.exists(corrupted)) {
-        if (!fs.rename(corrupted, p)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, corrupted, p)) {
           LOG.warn("Unable to move corrupted log " + corrupted + " to " + p);
         } else {
           LOG.warn("Moving corrupted log " + corrupted + " to " + p);
@@ -602,7 +604,7 @@ private static void archiveLogs(
     for (Path p : processedLogs) {
       Path newPath = HLog.getHLogArchivePath(oldLogDir, p);
       if (fs.exists(p)) {
-        if (!fs.rename(p, newPath)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, p, newPath)) {
           LOG.warn("Unable to move  " + p + " to " + newPath);
         } else {
           LOG.debug("Archived processed log " + p + " to " + newPath);
@@ -612,7 +614,7 @@ private static void archiveLogs(
 
     // distributed log splitting removes the srcDir (region's log dir) later
     // when all the log files in that srcDir have been successfully processed
-    if (srcDir != null && !fs.delete(srcDir, true)) {
+    if (srcDir != null && !HBaseFileSystem.deleteDirFromFileSystem(fs, conf, srcDir)) {
       throw new IOException("Unable to delete src dir: " + srcDir);
     }
   }
@@ -644,8 +646,9 @@ static Path getRegionSplitEditsPath(final FileSystem fs,
           " already split so it's safe to discard those edits.");
       return null;
     }
-    if (isCreate && !fs.exists(dir)) {
-      if (!fs.mkdirs(dir)) LOG.warn("mkdir failed on " + dir);
+    if (isCreate && !fs.exists(dir) && 
+        !HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), dir)) {
+      LOG.warn("mkdir failed on " + dir);
     }
     // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
     // region's replayRecoveredEdits will not delete it
@@ -816,7 +819,7 @@ private void checkForErrors() throws IOException {
    */
   protected Writer createWriter(FileSystem fs, Path logfile, Configuration conf)
       throws IOException {
-    return HLog.createWriter(fs, logfile, conf);
+    return hlogFs.createWriter(fs, conf, logfile);
   }
 
   /**
@@ -1063,7 +1066,7 @@ private WriterAndPath createWAP(byte[] region, Entry entry, Path rootdir,
           + "result of a previous failed split attempt. Deleting "
           + regionedits + ", length="
           + fs.getFileStatus(regionedits).getLen());
-      if (!fs.delete(regionedits, false)) {
+      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, regionedits)) {
         LOG.warn("Failed delete of old " + regionedits);
       }
     }
@@ -1089,13 +1092,13 @@ Path convertRegionEditsToTemp(Path rootdir, Path edits, String tmpname) {
             + "result of a previous failed split attempt. Deleting "
             + ret + ", length="
             + fs.getFileStatus(ret).getLen());
-        if (!fs.delete(ret, false)) {
+        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, ret)) {
           LOG.warn("Failed delete of old " + ret);
         }
       }
       Path dir = ret.getParent();
-      if (!fs.exists(dir)) {
-        if (!fs.mkdirs(dir)) LOG.warn("mkdir failed on " + dir);
+      if (!fs.exists(dir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, dir)) { 
+          LOG.warn("mkdir failed on " + dir);
       }
     } catch (IOException e) {
       LOG.warn("Could not prepare temp staging area ", e);
@@ -1188,7 +1191,7 @@ private List<Path> closeStreams() throws IOException {
             LOG.warn("Found existing old edits file. It could be the "
                 + "result of a previous failed split attempt. Deleting " + dst
                 + ", length=" + fs.getFileStatus(dst).getLen());
-            if (!fs.delete(dst, false)) {
+            if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, dst)) {
               LOG.warn("Failed deleting of old " + dst);
               throw new IOException("Failed deleting of old " + dst);
             }
@@ -1197,7 +1200,7 @@ private List<Path> closeStreams() throws IOException {
           // the data without touching disk. TestHLogSplit#testThreading is an
           // example.
           if (fs.exists(wap.p)) {
-            if (!fs.rename(wap.p, dst)) {
+            if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, wap.p, dst)) {
               throw new IOException("Failed renaming " + wap.p + " to " + dst);
             }
             LOG.debug("Rename " + wap.p + " to " + dst);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 786b334bff7f..fae73dd6faca 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableDescriptors;
@@ -224,7 +225,7 @@ public HTableDescriptor remove(final String tablename)
     if (!this.fsreadonly) {
       Path tabledir = FSUtils.getTablePath(this.rootdir, tablename);
       if (this.fs.exists(tabledir)) {
-        if (!this.fs.delete(tabledir, true)) {
+        if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), tabledir)) {
           throw new IOException("Failed delete of " + tabledir.toString());
         }
       }
@@ -280,7 +281,7 @@ public boolean accept(Path p) {
       for (int i = 1; i < status.length; i++) {
         Path p = status[i].getPath();
         // Clean up old versions
-        if (!fs.delete(p, false)) {
+        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, fs.getConf(), p)) {
           LOG.warn("Failed cleanup of " + status);
         } else {
           LOG.debug("Cleaned up old tableinfo file " + p);
@@ -504,7 +505,7 @@ private static Path writeTableDescriptor(final FileSystem fs,
       try {
         writeHTD(fs, p, hTableDescriptor);
         tableInfoPath = getTableInfoFileName(tableDir, sequenceid);
-        if (!fs.rename(p, tableInfoPath)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), p, tableInfoPath)) {
           throw new IOException("Failed rename of " + p + " to " + tableInfoPath);
         }
       } catch (IOException ioe) {
@@ -530,7 +531,7 @@ private static Path writeTableDescriptor(final FileSystem fs,
   private static void writeHTD(final FileSystem fs, final Path p,
       final HTableDescriptor htd)
   throws IOException {
-    FSDataOutputStream out = fs.create(p, false);
+    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, fs.getConf(), p, false);
     try {
       htd.write(out);
       out.write('\n');
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 1020bec0e162..dde45806e5b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
@@ -106,7 +107,7 @@ public static boolean deleteDirectory(final FileSystem fs, final Path dir)
    */
   public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
     if (!fs.exists(dir)) {
-      fs.mkdirs(dir);
+      HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), dir);
     }
     return dir;
   }
@@ -151,13 +152,10 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
    * @return output stream to the created file
    * @throws IOException if the file cannot be created
    */
-  public static FSDataOutputStream create(FileSystem fs, Path path,
-      FsPermission perm, boolean overwrite) throws IOException {
+  public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm,
+      boolean overwrite) throws IOException {
     LOG.debug("Creating file=" + path + " with permission=" + perm);
-
-    return fs.create(path, perm, overwrite,
-        fs.getConf().getInt("io.file.buffer.size", 4096),
-        fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+    return HBaseFileSystem.createPathWithPermsOnFileSystem(fs, fs.getConf(), path, perm, overwrite);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
index f02c17886f78..f5d5f85d648a 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
@@ -33,6 +33,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.master.SplitLogManager;
 import org.apache.hadoop.hbase.regionserver.SplitLogWorker;
@@ -164,7 +165,7 @@ public static void markCorrupted(Path rootdir, String logFileName,
       FileSystem fs) {
     Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
     try {
-      fs.createNewFile(file);
+      HBaseFileSystem.createNewFileOnFileSystem(fs, fs.getConf(), file);
     } catch (IOException e) {
       LOG.warn("Could not flag a log file as corrupted. Failed to create " +
           file, e);
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 28ae2a92a4b0..bd12dc760b08 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -197,6 +197,11 @@ public HBaseTestingUtility(Configuration conf) {
 
     // a hbase checksum verification failure will cause unit tests to fail
     ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
+    setHDFSClientRetryProperty();
+  }
+
+  private void setHDFSClientRetryProperty() {
+    this.conf.setInt("hdfs.client.retries.number", 1);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
new file mode 100644
index 000000000000..fe86ac5a4b7a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestHBaseFileSystem {
+  public static final Log LOG = LogFactory.getLog(TestHBaseFileSystem.class);
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static Configuration conf;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf = TEST_UTIL.getConfiguration();
+    conf.setBoolean("dfs.support.append", true);
+    // The below config supported by 0.20-append and CDH3b2
+    conf.setInt("dfs.client.block.recovery.retries", 2);
+    TEST_UTIL.startMiniDFSCluster(3);
+    Path hbaseRootDir =
+      TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
+    LOG.info("hbase.rootdir=" + hbaseRootDir);
+    conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
+    conf.setInt("hdfs.client.retries.number", 10);
+  }
+
+  
+  @Test
+  public void testNonIdempotentOpsWithRetries() throws IOException {
+    LOG.info("testNonIdempotentOpsWithRetries");
+
+    Path rootDir = new Path(TestHBaseFileSystem.conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    // Create a Region
+    assertTrue(HBaseFileSystem.createPathOnFileSystem(fs, TestHBaseFileSystem.conf, rootDir, true) != null);
+
+    boolean result = HBaseFileSystem.makeDirOnFileSystem(new MockFileSystemForCreate(), TestHBaseFileSystem.conf, new Path("/a"));
+    assertTrue("Couldn't create the directory", result);
+
+
+    result = HBaseFileSystem.renameDirForFileSystem(new MockFileSystem(), TestHBaseFileSystem.conf, new Path("/a"), new Path("/b"));
+    assertTrue("Couldn't rename the directory", result);
+
+    result = HBaseFileSystem.deleteDirFromFileSystem(new MockFileSystem(), TestHBaseFileSystem.conf, new Path("/a"));
+
+    assertTrue("Couldn't delete the directory", result);
+    fs.delete(rootDir, true);
+  }
+
+  static class MockFileSystemForCreate extends MockFileSystem {
+    @Override
+    public boolean exists(Path path) {
+      return false;
+    }
+  }
+
+  /**
+   * a mock fs which throws exception for first 3 times, and then process the call (returns the
+   * excepted result).
+   */
+  static class MockFileSystem extends FileSystem {
+    int retryCount;
+    final static int successRetryCount = 3;
+
+    public MockFileSystem() {
+      retryCount = 0;
+    }
+
+    @Override
+    public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
+      throw new IOException("");
+    }
+
+    @Override
+    public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
+        short arg4, long arg5, Progressable arg6) throws IOException {
+      LOG.debug("Create, " + retryCount);
+      if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
+      return null;
+    }
+
+    @Override
+    public boolean delete(Path arg0) throws IOException {
+      if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
+      return true;
+    }
+
+    @Override
+    public boolean delete(Path arg0, boolean arg1) throws IOException {
+      if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
+      return true;
+    }
+
+    @Override
+    public FileStatus getFileStatus(Path arg0) throws IOException {
+      FileStatus fs = new FileStatus();
+      return fs;
+    }
+
+    @Override
+    public boolean exists(Path path) {
+      return true;
+    }
+
+    @Override
+    public URI getUri() {
+      throw new RuntimeException("Something bad happen");
+    }
+
+    @Override
+    public Path getWorkingDirectory() {
+      throw new RuntimeException("Something bad happen");
+    }
+
+    @Override
+    public FileStatus[] listStatus(Path arg0) throws IOException {
+      throw new IOException("Something bad happen");
+    }
+
+    @Override
+    public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
+      LOG.debug("mkdirs, " + retryCount);
+      if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
+      return true;
+    }
+
+    @Override
+    public FSDataInputStream open(Path arg0, int arg1) throws IOException {
+      throw new IOException("Something bad happen");
+    }
+
+    @Override
+    public boolean rename(Path arg0, Path arg1) throws IOException {
+      LOG.debug("rename, " + retryCount);
+      if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
+      return true;
+    }
+
+    @Override
+    public void setWorkingDirectory(Path arg0) {
+      throw new RuntimeException("Something bad happen");
+    }
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java
index f0e7e0c4cbc1..a38b49d9d798 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java
@@ -73,7 +73,6 @@ public void setUp() throws Exception {
   public void tearDown() throws Exception {
     // Stop the cluster
     TEST_UTIL.shutdownMiniCluster();
-    TEST_UTIL = new HBaseTestingUtility(resetConf);
   }
 
   @Test

From 517d073cafd5ac97964576e50d1348eef8509dba Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 4 Apr 2013 23:30:12 +0000
Subject: [PATCH 0905/1540] HBASE-8169
 TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS may fail due to
 regions randomly assigned to a RS (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464787 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 19 +++++
 .../hbase/master/TestMasterFailover.java      | 70 ++++++++++++++-----
 2 files changed, 72 insertions(+), 17 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index bd12dc760b08..fe9db5f47a82 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2236,5 +2236,24 @@ public HRegion createTestRegion(String tableName, HColumnDescriptor hcd)
         HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
     return region;
   }
+  
+  /**
+   * Create region split keys between startkey and endKey
+   * 
+   * @param startKey
+   * @param endKey
+   * @param numRegions the number of regions to be created. it has to be greater than 3.
+   * @return
+   */
+  public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){
+    assertTrue(numRegions>3);
+    byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3);
+    byte [][] result = new byte[tmpSplitKeys.length+1][];
+    for (int i=0;i<tmpSplitKeys.length;i++) {
+      result[i+1] = tmpSplitKeys[i];
+    }
+    result[0] = HConstants.EMPTY_BYTE_ARRAY;
+    return result;
+  }
 
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 479d9048fbcd..a75aeeb709a5 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -26,6 +26,8 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
@@ -47,6 +49,7 @@
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
@@ -617,8 +620,8 @@ public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
     // Create config to use for this cluster
     Configuration conf = HBaseConfiguration.create();
     // Need to drop the timeout much lower
-    conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
-    conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
+    conf.setInt("hbase.master.assignment.timeoutmonitor.period", 4000);
+    conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
 
@@ -658,14 +661,10 @@ public boolean isAborted() {
     // disable load balancing on this master
     master.balanceSwitch(false);
 
-    // create two tables in META, each with 10 regions
+    // create two tables in META, each with 30 regions
     byte [] FAMILY = Bytes.toBytes("family");
-    byte [][] SPLIT_KEYS = new byte [][] {
-        new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
-        Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
-        Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
-        Bytes.toBytes("iii"), Bytes.toBytes("jjj")
-    };
+    byte[][] SPLIT_KEYS =
+        TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
 
     byte [] enabledTable = Bytes.toBytes("enabledTable");
     HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable);
@@ -710,11 +709,11 @@ public boolean isAborted() {
 
     // we'll need some regions to already be assigned out properly on live RS
     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
-    enabledAndAssignedRegions.add(enabledRegions.remove(0));
-    enabledAndAssignedRegions.add(enabledRegions.remove(0));
+    enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
+    enabledRegions.removeAll(enabledAndAssignedRegions);
     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
-    disabledAndAssignedRegions.add(disabledRegions.remove(0));
-    disabledAndAssignedRegions.add(disabledRegions.remove(0));
+    disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
+    disabledRegions.removeAll(disabledAndAssignedRegions);
 
     // now actually assign them
     for (HRegionInfo hri : enabledAndAssignedRegions) {
@@ -728,15 +727,20 @@ public boolean isAborted() {
       master.assignRegion(hri);
     }
 
+    log("Waiting for assignment to finish");
+    ZKAssign.blockUntilNoRIT(zkw);
+    master.assignmentManager.waitUntilNoRegionsInTransition(60000);
+    log("Assignment completed");
+
     assertTrue(" Table must be enabled.", master.getAssignmentManager()
         .getZKTable().isEnabledTable("enabledTable"));
     // we also need regions assigned out on the dead server
     List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
-    enabledAndOnDeadRegions.add(enabledRegions.remove(0));
-    enabledAndOnDeadRegions.add(enabledRegions.remove(0));
+    enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
+    enabledRegions.removeAll(enabledAndOnDeadRegions);
     List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
-    disabledAndOnDeadRegions.add(disabledRegions.remove(0));
-    disabledAndOnDeadRegions.add(disabledRegions.remove(0));
+    disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
+    disabledRegions.removeAll(disabledAndOnDeadRegions);
 
     // set region plan to server to be killed and trigger assign
     for (HRegionInfo hri : enabledAndOnDeadRegions) {
@@ -753,8 +757,25 @@ public boolean isAborted() {
     // wait for no more RIT
     log("Waiting for assignment to finish");
     ZKAssign.blockUntilNoRIT(zkw);
+    master.assignmentManager.waitUntilNoRegionsInTransition(60000);
     log("Assignment completed");
 
+    // Due to master.assignRegion(hri) could fail to assign a region to a specified RS
+    // therefore, we need make sure that regions are in the expected RS
+    verifyRegionLocation(hrs, enabledAndAssignedRegions);
+    verifyRegionLocation(hrs, disabledAndAssignedRegions);
+    verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
+    verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
+
+    assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
+      enabledAndAssignedRegions.size() >= 2);
+    assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
+      disabledAndAssignedRegions.size() >= 2);
+    assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
+      enabledAndOnDeadRegions.size() >= 2);
+    assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
+      disabledAndOnDeadRegions.size() >= 2);
+
     // Stop the master
     log("Aborting master");
     cluster.abortMaster(0);
@@ -1009,6 +1030,21 @@ public boolean isAborted() {
     TEST_UTIL.shutdownMiniCluster();
   }
 
+  /**
+   * Verify regions are on the expected region server
+   */
+  private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
+      throws IOException {
+    List<HRegionInfo> tmpOnlineRegions = hrs.getOnlineRegions();
+    Iterator<HRegionInfo> itr = regions.iterator();
+    while (itr.hasNext()) {
+      HRegionInfo tmp = itr.next();
+      if (!tmpOnlineRegions.contains(tmp)) {
+        itr.remove();
+      }
+    }
+  }
+
   HRegion createRegion(final HRegionInfo  hri, final Path rootdir, final Configuration c,
       final HTableDescriptor htd)
   throws IOException {

From b02d721bea879d33b31faef749db051bab368a3b Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Fri, 5 Apr 2013 01:21:14 +0000
Subject: [PATCH 0906/1540] HBASE-8092 bulk assignment in 0.94 doesn't handle
 ZK errors very well

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464804 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 52 ++++++++++++++-----
 1 file changed, 40 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 532ef2d20f70..32ccfff7f465 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1468,13 +1468,21 @@ void assign(final ServerName destination,
     for (int oldCounter = 0; true;) {
       int count = counter.get();
       if (oldCounter != count) {
-        LOG.info(destination.toString() + " unassigned znodes=" + count +
+        LOG.info(destination.toString() + " outstanding calls=" + count +
           " of total=" + total);
         oldCounter = count;
       }
       if (count == total) break;
       Threads.sleep(1);
     }
+    // Check if any failed.
+    if (cb.hasErrors()) {
+      // TODO: createOrForceNodeOffline actually handles this condition; whereas this
+      //       code used to just abort master. Now, it will bail more "gracefully".
+      LOG.error("Error creating nodes for some of the regions we are trying to bulk assign");
+      return;
+    }
+
     // Move on to open regions.
     try {
       // Send OPEN RPC. If it fails on a IOE or RemoteException, the
@@ -1523,6 +1531,7 @@ static class CreateUnassignedAsyncCallback implements AsyncCallback.StringCallba
     private final ZooKeeperWatcher zkw;
     private final ServerName destination;
     private final AtomicInteger counter;
+    private final AtomicInteger errorCount = new AtomicInteger(0);
 
     CreateUnassignedAsyncCallback(final ZooKeeperWatcher zkw,
         final ServerName destination, final AtomicInteger counter) {
@@ -1531,13 +1540,22 @@ static class CreateUnassignedAsyncCallback implements AsyncCallback.StringCallba
       this.counter = counter;
     }
 
+    boolean hasErrors() {
+      return this.errorCount.get() > 0;
+    }
+
     @Override
     public void processResult(int rc, String path, Object ctx, String name) {
+      if (rc == KeeperException.Code.NODEEXISTS.intValue()) {
+        LOG.warn("Node for " + path + " already exists");
+        reportCompletion(false);
+        return;
+      }
       if (rc != 0) {
-     // This is resultcode.  If non-zero, need to resubmit.
-        LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
-          "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
-        this.zkw.abort("Connectionloss writing unassigned at " + path +
+        // This is resultcode. If non-zero, we will abort :(
+        LOG.warn("rc != 0 for " + path + " -- some error, may be retryable connection loss -- "
+            + "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
+        this.zkw.abort("Some error, may be connection loss writing unassigned at " + path +
           ", rc=" + rc, null);
         return;
       }
@@ -1545,7 +1563,14 @@ public void processResult(int rc, String path, Object ctx, String name) {
       // Async exists to set a watcher so we'll get triggered when
       // unassigned node changes.
       this.zkw.getRecoverableZooKeeper().getZooKeeper().exists(path, this.zkw,
-        new ExistsUnassignedAsyncCallback(this.counter, destination), ctx);
+        new ExistsUnassignedAsyncCallback(this, destination), ctx);
+    }
+
+    void reportCompletion(boolean success) {
+      if (!success) {
+        this.errorCount.incrementAndGet();
+      }
+      this.counter.incrementAndGet();
     }
   }
 
@@ -1555,20 +1580,22 @@ public void processResult(int rc, String path, Object ctx, String name) {
    */
   static class ExistsUnassignedAsyncCallback implements AsyncCallback.StatCallback {
     private final Log LOG = LogFactory.getLog(ExistsUnassignedAsyncCallback.class);
-    private final AtomicInteger counter;
     private ServerName destination;
+    private CreateUnassignedAsyncCallback parent;
 
-    ExistsUnassignedAsyncCallback(final AtomicInteger counter, ServerName destination) {
-      this.counter = counter;
+    ExistsUnassignedAsyncCallback(
+        CreateUnassignedAsyncCallback parent, ServerName destination) {
+      this.parent = parent;
       this.destination = destination;
     }
 
     @Override
     public void processResult(int rc, String path, Object ctx, Stat stat) {
       if (rc != 0) {
-        // Thisis resultcode.  If non-zero, need to resubmit.
-        LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
+        // This is resultcode.  If non-zero, need to resubmit.
+        LOG.warn("rc != 0 for " + path + " -- some error, may be connection loss -- " +
           "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
+        parent.reportCompletion(false);
         return;
       }
       RegionState state = (RegionState)ctx;
@@ -1579,7 +1606,7 @@ public void processResult(int rc, String path, Object ctx, Stat stat) {
       // call to open risks our writing PENDING_OPEN after state has been moved
       // to OPENING by the regionserver.
       state.update(RegionState.State.PENDING_OPEN, System.currentTimeMillis(), destination);
-      this.counter.addAndGet(1);
+      parent.reportCompletion(true);
     }
   }
 
@@ -1909,6 +1936,7 @@ boolean asyncSetOfflineInZooKeeper(final RegionState state,
       ZKAssign.asyncCreateNodeOffline(master.getZooKeeper(), state.getRegion(),
         this.master.getServerName(), cb, ctx);
     } catch (KeeperException e) {
+      // TODO: this error handling will never execute, as the callback is async.
       if (e instanceof NodeExistsException) {
         LOG.warn("Node for " + state.getRegion() + " already exists");
       } else { 

From 259649b7d4eda40087e2c371144f6cee0360901d Mon Sep 17 00:00:00 2001
From: Devaraj Das <ddas@apache.org>
Date: Fri, 5 Apr 2013 04:47:03 +0000
Subject: [PATCH 0907/1540] HBASE-8179. Fixes a problem to do with the json
 response for the cluster status

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464821 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/Main.java     | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index cb3134146499..ca3edc5ad547 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -39,6 +39,8 @@
 
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
@@ -47,6 +49,7 @@
 import org.mortbay.jetty.servlet.ServletHolder;
 import org.mortbay.thread.QueuedThreadPool;
 
+import com.sun.jersey.api.json.JSONConfiguration;
 import com.sun.jersey.spi.container.servlet.ServletContainer;
 
 /**
@@ -148,6 +151,20 @@ public static void main(String[] args) throws Exception {
       ResourceConfig.class.getCanonicalName());
     sh.setInitParameter("com.sun.jersey.config.property.packages",
       "jetty");
+    // The servlet holder below is instantiated to only handle the case
+    // of the /status/cluster returning arrays of nodes (live/dead). Without
+    // this servlet holder, the problem is that the node arrays in the response 
+    // are collapsed to single nodes. We want to be able to treat the 
+    // node lists as POJO in the response to /status/cluster servlet call, 
+    // but not change the behavior for any of the other servlets
+    // Hence we don't use the servlet holder for all servlets / paths
+    ServletHolder shPojoMap = new ServletHolder(ServletContainer.class);
+    @SuppressWarnings("unchecked")
+    Map<String, String> shInitMap = sh.getInitParameters();
+    for (Entry<String, String> e : shInitMap.entrySet()) {
+      shPojoMap.setInitParameter(e.getKey(), e.getValue());
+    }
+    shPojoMap.setInitParameter(JSONConfiguration.FEATURE_POJO_MAPPING, "true");
 
     // set up Jetty and run the embedded server
 
@@ -176,6 +193,7 @@ public static void main(String[] args) throws Exception {
 
     // set up context
     Context context = new Context(server, "/", Context.SESSIONS);
+    context.addServlet(shPojoMap, "/status/cluster");
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
 

From 56751d9df189c76c879b8af580d3543ff428a4b2 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 5 Apr 2013 11:25:07 +0000
Subject: [PATCH 0908/1540] HBASE-7415 Add task information to snapshot
 operation (Jesse Yates)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464928 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/handler/TableEventHandler.java     | 10 +++++++
 .../master/snapshot/CloneSnapshotHandler.java | 27 +++++++++++++++----
 .../DisabledTableSnapshotHandler.java         | 14 +++++++---
 .../snapshot/RestoreSnapshotHandler.java      | 22 ++++++++++++++-
 .../master/snapshot/TakeSnapshotHandler.java  | 16 ++++++++++-
 .../hbase/snapshot/RestoreSnapshotHelper.java | 13 ++++++++-
 .../snapshot/TestRestoreSnapshotHelper.java   |  6 +++--
 7 files changed, 95 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
index 354c59397a35..b73f25db2d21 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -106,15 +106,25 @@ public void process() {
           LOG.warn("Error on reopening the regions");
         }
       }
+      completed(null);
     } catch (IOException e) {
       LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
+      completed(e);
     } catch (KeeperException e) {
       LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
+      completed(e);
     } finally {
       notifyEventBeingHandled();
     }
   }
 
+  /**
+   * Called after that process() is completed.
+   * @param exception null if process() is successful or not null if something has failed.
+   */
+  protected void completed(final Throwable exception) {
+  }
+
   public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
     boolean done = false;
     HTable table = null;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
index 58e6520b5e91..b78a7f866483 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -38,6 +38,8 @@
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
 import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
@@ -61,6 +63,7 @@ public class CloneSnapshotHandler extends CreateTableHandler implements Snapshot
   private final SnapshotDescription snapshot;
 
   private final ForeignExceptionDispatcher monitor;
+  private final MonitoredTask status;
 
   private volatile boolean stopped = false;
 
@@ -77,6 +80,8 @@ public CloneSnapshotHandler(final MasterServices masterServices,
 
     // Monitor
     this.monitor = new ForeignExceptionDispatcher();
+    this.status = TaskMonitor.get().createStatus("Cloning  snapshot '" + snapshot.getName() +
+      "' to table " + hTableDescriptor.getNameAsString());
   }
 
   /**
@@ -85,8 +90,9 @@ public CloneSnapshotHandler(final MasterServices masterServices,
    * will be responsible to add the regions returned by this method to META and do the assignment.
    */
   @Override
-  protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, final String tableName)
-      throws IOException {
+  protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
+      final String tableName) throws IOException {
+    status.setStatus("Creating regions for table: " + tableName);
     FileSystem fs = fileSystemManager.getFileSystem();
     Path rootDir = fileSystemManager.getRootDir();
     Path tableDir = new Path(tableRootDir, tableName);
@@ -95,7 +101,7 @@ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, fin
       // 1. Execute the on-disk Clone
       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
       RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
-          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor, status);
       RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
 
       // Clone operation should not have stuff to restore or remove
@@ -105,7 +111,9 @@ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, fin
           "A clone should not have regions to remove");
 
       // At this point the clone is complete. Next step is enabling the table.
-      LOG.info("Clone snapshot=" + snapshot.getName() + " on table=" + tableName + " completed!");
+      String msg = "Clone snapshot=" + snapshot.getName() +" on table=" + tableName + " completed!";
+      LOG.info(msg);
+      status.setStatus(msg + " Waiting for table to be enabled...");
 
       // 2. let the CreateTableHandler add the regions to meta
       return metaChanges.getRegionsToAdd();
@@ -123,6 +131,13 @@ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, fin
   @Override
   protected void completed(final Throwable exception) {
     this.stopped = true;
+    if (exception != null) {
+     status.abort("Snapshot '" + snapshot.getName() + "' clone failed because " +
+        exception.getMessage());
+    } else {
+      status.markComplete("Snapshot '"+ snapshot.getName() +"' clone completed and table enabled!");
+    }
+    super.completed(exception);
   }
 
   @Override
@@ -139,7 +154,9 @@ public SnapshotDescription getSnapshot() {
   public void cancel(String why) {
     if (this.stopped) return;
     this.stopped = true;
-    LOG.info("Stopping clone snapshot=" + snapshot + " because: " + why);
+    String msg = "Stopping clone snapshot=" + snapshot + " because: " + why;
+    LOG.info(msg);
+    status.abort(msg);
     this.monitor.receive(new ForeignException(NAME, new CancellationException(why)));
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
index 739284fcfdb3..ae89f816eddb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -32,6 +32,8 @@
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
@@ -70,8 +72,8 @@ public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
   // TODO consider parallelizing these operations since they are independent. Right now its just
   // easier to keep them serial though
   @Override
-  public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException,
-  KeeperException {
+  public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations)
+      throws IOException, KeeperException {
     try {
       timeoutInjector.start();
 
@@ -100,10 +102,14 @@ public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocati
         Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
         new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
         monitor.rethrowException();
+        status.setStatus("Completed copying recovered edits for offline snapshot of table: "
+            + snapshot.getTable());
 
         // 2.3 reference all the files in the region
         new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
         monitor.rethrowException();
+        status.setStatus("Completed referencing HFiles for offline snapshot of table: " +
+          snapshot.getTable());
       }
 
       // 3. write the table info to disk
@@ -113,12 +119,14 @@ public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocati
           FSUtils.getRootDir(conf));
       tableInfoCopyTask.call();
       monitor.rethrowException();
+      status.setStatus("Finished copying tableinfo for snapshot of table: " + snapshot.getTable());
     } catch (Exception e) {
       // make sure we capture the exception to propagate back to the client later
       String reason = "Failed snapshot " + SnapshotDescriptionUtils.toString(snapshot)
           + " due to exception:" + e.getMessage();
       ForeignException ee = new ForeignException(reason, e);
       monitor.receive(ee);
+      status.abort("Snapshot of table: "+ snapshot.getTable() +" failed because " + e.getMessage());
     } finally {
       LOG.debug("Marking snapshot" + SnapshotDescriptionUtils.toString(snapshot)
           + " as finished.");
@@ -128,4 +136,4 @@ public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocati
       timeoutInjector.complete();
     }
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index 310347cb4c67..247a2e7621d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -38,6 +38,8 @@
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
 import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
@@ -58,6 +60,8 @@ public class RestoreSnapshotHandler extends TableEventHandler implements Snapsho
   private final SnapshotDescription snapshot;
 
   private final ForeignExceptionDispatcher monitor;
+  private final MonitoredTask status;
+
   private volatile boolean stopped = false;
 
   public RestoreSnapshotHandler(final MasterServices masterServices,
@@ -76,6 +80,10 @@ public RestoreSnapshotHandler(final MasterServices masterServices,
 
     // This is the new schema we are going to write out as this modification.
     this.hTableDescriptor = htd;
+
+    this.status = TaskMonitor.get().createStatus(
+      "Restoring  snapshot '" + snapshot.getName() + "' to table "
+          + hTableDescriptor.getNameAsString());
   }
 
   /**
@@ -104,11 +112,12 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
       RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
           masterServices.getConfiguration(), fs,
-          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor, status);
       RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
 
       // 3. Applies changes to .META.
       hris.clear();
+      status.setStatus("Preparing to restore each region");
       if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
       if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
       List<HRegionInfo> hrisToRemove = metaChanges.getRegionsToRemove();
@@ -128,6 +137,17 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
     }
   }
 
+  @Override
+  protected void completed(final Throwable exception) {
+    this.stopped = true;
+    if (exception != null) {
+      status.abort("Restore snapshot '" + snapshot.getName() + "' failed because " +
+          exception.getMessage());
+    } else {
+      status.markComplete("Restore snapshot '"+ snapshot.getName() +"' completed!");
+    }
+  }
+
   @Override
   public boolean isFinished() {
     return this.stopped;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 3ab4369b04d0..3d8b2e6c76ee 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -40,6 +40,8 @@
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
@@ -72,6 +74,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
   protected final Path workingDir;
   private final MasterSnapshotVerifier verifier;
   protected final ForeignExceptionDispatcher monitor;
+  protected final MonitoredTask status;
 
   /**
    * @param snapshot descriptor of the snapshot to take
@@ -97,6 +100,9 @@ public TakeSnapshotHandler(SnapshotDescription snapshot,
 
     // prepare the verify
     this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
+    // update the running tasks
+    this.status = TaskMonitor.get().createStatus(
+      "Taking " + snapshot.getType() + " snapshot on table: " + snapshot.getTable());
   }
 
   private HTableDescriptor loadTableDescriptor()
@@ -116,7 +122,10 @@ private HTableDescriptor loadTableDescriptor()
    */
   @Override
   public void process() {
-    LOG.info("Running table snapshot operation " + eventType + " on table " + snapshot.getTable());
+    String msg = "Running " + snapshot.getType() + " table snapshot " + snapshot.getName() + " "
+        + eventType + " on table " + snapshot.getTable();
+    LOG.info(msg);
+    status.setStatus(msg);
     try {
       // If regions move after this meta scan, the region specific snapshot should fail, triggering
       // an external exception that gets captured here.
@@ -140,11 +149,16 @@ public void process() {
       }
 
       // verify the snapshot is valid
+      status.setStatus("Verifying snapshot: " + snapshot.getName());
       verifier.verifySnapshot(this.workingDir, serverNames);
 
       // complete the snapshot, atomically moving from tmp to .snapshot dir.
       completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
+      status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshot.getTable()
+          + " completed");
     } catch (Exception e) {
+      status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " +
+          snapshot.getTable() + " because " + e.getMessage());
       String reason = "Failed taking snapshot " + SnapshotDescriptionUtils.toString(snapshot)
           + " due to exception:" + e.getMessage();
       LOG.error(reason, e);
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 16f71b39d8a6..5b92060403dd 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -101,6 +102,7 @@ public class RestoreSnapshotHelper {
         new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
 
   private final ForeignExceptionDispatcher monitor;
+  private final MonitoredTask status;
 
   private final SnapshotDescription snapshotDesc;
   private final Path snapshotDir;
@@ -114,7 +116,7 @@ public class RestoreSnapshotHelper {
   public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
       final SnapshotDescription snapshotDescription, final Path snapshotDir,
       final HTableDescriptor tableDescriptor, final Path tableDir,
-      final ForeignExceptionDispatcher monitor)
+      final ForeignExceptionDispatcher monitor, final MonitoredTask status)
   {
     this.fs = fs;
     this.conf = conf;
@@ -123,6 +125,7 @@ public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
     this.tableDesc = tableDescriptor;
     this.tableDir = tableDir;
     this.monitor = monitor;
+    this.status = status;
   }
 
   /**
@@ -158,11 +161,15 @@ public RestoreMetaChanges restoreHdfsRegions() throws IOException {
 
       // Restore regions using the snapshot data
       monitor.rethrowException();
+      status.setStatus("Restoring table regions...");
       restoreHdfsRegions(metaChanges.getRegionsToRestore());
+      status.setStatus("Finished restoring all table regions.");
 
       // Remove regions from the current table
       monitor.rethrowException();
+      status.setStatus("Starting to delete excess regions from table");
       removeHdfsRegions(metaChanges.getRegionsToRemove());
+      status.setStatus("Finished deleting excess regions from table.");
     }
 
     // Regions to Add: present in the snapshot but not in the current table
@@ -178,13 +185,17 @@ public RestoreMetaChanges restoreHdfsRegions() throws IOException {
 
       // Create new regions cloning from the snapshot
       monitor.rethrowException();
+      status.setStatus("Cloning regions...");
       HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
       metaChanges.setNewRegions(clonedRegions);
+      status.setStatus("Finished cloning regions.");
     }
 
     // Restore WALs
     monitor.rethrowException();
+    status.setStatus("Restoring WALs to table...");
     restoreWALs();
+    status.setStatus("Finished restoring WALs to table.");
 
     return metaChanges;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
index e5cb149b8a72..b995a56163fc 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -145,12 +146,13 @@ private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path sn
     CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
     HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
     ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+    MonitoredTask status = Mockito.mock(MonitoredTask.class);
 
     SnapshotDescription sd = SnapshotDescription.newBuilder()
       .setName("snapshot").setTable(sourceTableName).build();
 
     return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
-      htdClone, HTableDescriptor.getTableDir(rootDir, htdClone.getName()), monitor);
+      htdClone, HTableDescriptor.getTableDir(rootDir, htdClone.getName()), monitor, status);
   }
 
   private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
@@ -197,4 +199,4 @@ private String[] getHFiles(final Path tableDir) throws IOException {
     Collections.sort(files);
     return files.toArray(new String[files.size()]);
   }
-}
\ No newline at end of file
+}

From 43533f95968d03aa0d2539f9d9d4fe8780b992e2 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 5 Apr 2013 11:42:31 +0000
Subject: [PATCH 0909/1540] HBASE-7615 Add metrics for snapshots

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1464931 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  |   2 +-
 .../apache/hadoop/hbase/master/HMaster.java   |   2 +-
 .../hbase/master/metrics/MasterMetrics.java   |  39 ++-
 .../master/snapshot/CloneSnapshotHandler.java |   9 +-
 .../DisabledTableSnapshotHandler.java         |   5 +-
 .../snapshot/EnabledTableSnapshotHandler.java |   5 +-
 .../snapshot/RestoreSnapshotHandler.java      |   9 +-
 .../master/snapshot/SnapshotManager.java      |  21 +-
 .../master/snapshot/TakeSnapshotHandler.java  |   8 +-
 .../hadoop/hbase/snapshot/SnapshotInfo.java   | 281 ++++++++++++++----
 .../hbase-webapps/master/snapshot.jsp         | 189 ++++++++++++
 .../master/snapshot/TestSnapshotManager.java  |   4 +-
 12 files changed, 500 insertions(+), 74 deletions(-)
 create mode 100644 src/main/resources/hbase-webapps/master/snapshot.jsp

diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index f7f7a12086db..097934e91b5c 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -212,7 +212,7 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 </tr>
 <%for SnapshotDescription snapshotDesc : snapshots%>
 <tr>
-    <td><% snapshotDesc.getName() %></td>
+    <td><a href="snapshot.jsp?name=<% snapshotDesc.getName() %>"><% snapshotDesc.getName() %></a></td>
     <td><a href="table.jsp?name=<% snapshotDesc.getTable() %>"><% snapshotDesc.getTable() %></a></td>
     <td><% new Date(snapshotDesc.getCreationTime()) %></td>
     <td><% snapshotDesc.getType() %></td>
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 506118e33020..cf6b4fcab3ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -492,7 +492,7 @@ private void initializeZKBasedSystemTrackers() throws IOException,
         ", cluster-up flag was=" + wasUp);
 
     // create the snapshot manager
-    this.snapshotManager = new SnapshotManager(this);
+    this.snapshotManager = new SnapshotManager(this, this.metrics);
   }
 
   // Check if we should stop every second.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
index 9e4cf7351042..ccb8a38eb15d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
@@ -50,7 +50,8 @@ public class MasterMetrics implements Updater {
   private long lastUpdate = System.currentTimeMillis();
   private long lastExtUpdate = System.currentTimeMillis();
   private long extendedPeriod = 0;
-/*
+
+  /*
    * Count of requests to the cluster since last call to metrics update
    */
   private final MetricsRate cluster_requests =
@@ -64,6 +65,18 @@ public class MasterMetrics implements Updater {
   final PersistentMetricsTimeVaryingRate splitSize =
     new PersistentMetricsTimeVaryingRate("splitSize", registry);
 
+  /** Time it takes to finish snapshot() */
+  final PersistentMetricsTimeVaryingRate snapshotTime =
+    new PersistentMetricsTimeVaryingRate("snapshotTime", registry);
+
+  /** Time it takes to finish restoreSnapshot() */
+  final PersistentMetricsTimeVaryingRate snapshotRestoreTime =
+    new PersistentMetricsTimeVaryingRate("snapshotRestoreTime", registry);
+
+  /** Time it takes to finish cloneSnapshotTime() */
+  final PersistentMetricsTimeVaryingRate snapshotCloneTime =
+    new PersistentMetricsTimeVaryingRate("snapshotCloneTime", registry);
+
   public MasterMetrics(final String name) {
     MetricsContext context = MetricsUtil.getContext("hbase");
     metricsRecord = MetricsUtil.createRecord(context, "master");
@@ -146,4 +159,28 @@ public float getRequests() {
   public void incrementRequests(final int inc) {
     this.cluster_requests.inc(inc);
   }
+
+  /**
+   * Record a single instance of a snapshot
+   * @param time time that the snapshot took
+   */
+  public void addSnapshot(long time) {
+    snapshotTime.inc(time);
+  }
+
+  /**
+   * Record a single instance of a snapshot
+   * @param time time that the snapshot restore took
+   */
+  public void addSnapshotRestore(long time) {
+    snapshotRestoreTime.inc(time);
+  }
+
+  /**
+   * Record a single instance of a snapshot cloned table
+   * @param time time that the snapshot clone took
+   */
+  public void addSnapshotClone(long time) {
+    snapshotCloneTime.inc(time);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
index b78a7f866483..eebf99d26b56 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
 import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -63,17 +64,20 @@ public class CloneSnapshotHandler extends CreateTableHandler implements Snapshot
   private final SnapshotDescription snapshot;
 
   private final ForeignExceptionDispatcher monitor;
+  private final MasterMetrics metricsMaster;
   private final MonitoredTask status;
 
   private volatile boolean stopped = false;
 
   public CloneSnapshotHandler(final MasterServices masterServices,
-      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor,
+      final MasterMetrics metricsMaster)
       throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
-    super(masterServices, masterServices.getMasterFileSystem(), 
+    super(masterServices, masterServices.getMasterFileSystem(),
       masterServices.getServerManager(), hTableDescriptor,
       masterServices.getConfiguration(), null, masterServices.getCatalogTracker(),
       masterServices.getAssignmentManager());
+    this.metricsMaster = metricsMaster;
 
     // Snapshot information
     this.snapshot = snapshot;
@@ -137,6 +141,7 @@ protected void completed(final Throwable exception) {
     } else {
       status.markComplete("Snapshot '"+ snapshot.getName() +"' clone completed and table enabled!");
     }
+    metricsMaster.addSnapshotClone(status.getCompletionTimestamp() - status.getStartTime());
     super.completed(exception);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
index ae89f816eddb..a804f07e277a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -62,8 +63,8 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
    * @throws IOException on unexpected error
    */
   public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
-      final MasterServices masterServices) throws IOException {
-    super(snapshot, masterServices);
+      final MasterServices masterServices, final MasterMetrics metricsMaster) throws IOException {
+    super(snapshot, masterServices, metricsMaster);
 
     // setup the timer
     timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
index 17d022163713..c795855374f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.procedure.Procedure;
 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -49,8 +50,8 @@ public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
   private final ProcedureCoordinator coordinator;
 
   public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
-      SnapshotManager manager) throws IOException {
-    super(snapshot, master);
+      final SnapshotManager manager, final MasterMetrics metricsMaster) throws IOException {
+    super(snapshot, master, metricsMaster);
     this.coordinator = manager.getCoordinator();
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index 247a2e7621d4..0dfa2c0fcd66 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
 import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -60,14 +61,16 @@ public class RestoreSnapshotHandler extends TableEventHandler implements Snapsho
   private final SnapshotDescription snapshot;
 
   private final ForeignExceptionDispatcher monitor;
+  private final MasterMetrics metricsMaster;
   private final MonitoredTask status;
 
   private volatile boolean stopped = false;
 
   public RestoreSnapshotHandler(final MasterServices masterServices,
-      final SnapshotDescription snapshot, final HTableDescriptor htd)
-      throws IOException {
+      final SnapshotDescription snapshot, final HTableDescriptor htd,
+      final MasterMetrics metricsMaster) throws IOException {
     super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices);
+    this.metricsMaster = metricsMaster;
 
     // Snapshot information
     this.snapshot = snapshot;
@@ -146,6 +149,8 @@ protected void completed(final Throwable exception) {
     } else {
       status.markComplete("Restore snapshot '"+ snapshot.getName() +"' completed!");
     }
+    metricsMaster.addSnapshotRestore(status.getCompletionTimestamp() - status.getStartTime());
+    super.completed(exception);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index a95d0c67b9c8..f30210f5c070 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.procedure.Procedure;
 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
@@ -117,6 +118,7 @@ public class SnapshotManager implements Stoppable {
   private boolean stopped;
   private final long wakeFrequency;
   private final MasterServices master;  // Needed by TableEventHandlers
+  private final MasterMetrics metricsMaster;
   private final ProcedureCoordinator coordinator;
 
   // Is snapshot feature enabled?
@@ -138,9 +140,11 @@ public class SnapshotManager implements Stoppable {
    * Construct a snapshot manager.
    * @param master
    */
-  public SnapshotManager(final MasterServices master) throws KeeperException, IOException,
-    UnsupportedOperationException {
+  public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster)
+      throws KeeperException, IOException, UnsupportedOperationException {
     this.master = master;
+    this.metricsMaster = metricsMaster;
+
     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
 
     // get the configuration for the coordinator
@@ -165,9 +169,12 @@ public SnapshotManager(final MasterServices master) throws KeeperException, IOEx
    * @param coordinator procedure coordinator instance.  exposed for testing.
    * @param pool HBase ExecutorServcie instance, exposed for testing.
    */
-  public SnapshotManager(final MasterServices master, ProcedureCoordinator coordinator, ExecutorService pool)
+  public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster,
+      ProcedureCoordinator coordinator, ExecutorService pool)
       throws IOException, UnsupportedOperationException {
     this.master = master;
+    this.metricsMaster = metricsMaster;
+
     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
 
     this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
@@ -427,7 +434,7 @@ private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
       throws HBaseSnapshotException {
     TakeSnapshotHandler handler;
     try {
-      handler = new EnabledTableSnapshotHandler(snapshot, master, this);
+      handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
       this.executorService.submit(handler);
       this.handler = handler;
     } catch (IOException e) {
@@ -536,7 +543,7 @@ private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
 
     DisabledTableSnapshotHandler handler;
     try {
-      handler = new DisabledTableSnapshotHandler(snapshot, this.master);
+      handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
       this.executorService.submit(handler);
       this.handler = handler;
     } catch (IOException e) {
@@ -619,7 +626,7 @@ synchronized void cloneSnapshot(final SnapshotDescription snapshot,
 
     try {
       CloneSnapshotHandler handler =
-        new CloneSnapshotHandler(master, snapshot, hTableDescriptor);
+        new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
       this.executorService.submit(handler);
       restoreHandlers.put(tableName, handler);
     } catch (Exception e) {
@@ -710,7 +717,7 @@ private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
 
     try {
       RestoreSnapshotHandler handler =
-        new RestoreSnapshotHandler(master, snapshot, hTableDescriptor);
+        new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
       this.executorService.submit(handler);
       restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
     } catch (Exception e) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 3d8b2e6c76ee..acbc679da412 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -66,6 +67,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
 
   // none of these should ever be null
   protected final MasterServices master;
+  protected final MasterMetrics metricsMaster;
   protected final SnapshotDescription snapshot;
   protected final Configuration conf;
   protected final FileSystem fs;
@@ -81,13 +83,14 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
    * @param masterServices master services provider
    * @throws IOException on unexpected error
    */
-  public TakeSnapshotHandler(SnapshotDescription snapshot,
-      final MasterServices masterServices) throws IOException {
+  public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
+      final MasterMetrics metricsMaster) throws IOException {
     super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
     assert snapshot != null : "SnapshotDescription must not be nul1";
     assert masterServices != null : "MasterServices must not be nul1";
 
     this.master = masterServices;
+    this.metricsMaster = metricsMaster;
     this.snapshot = snapshot;
     this.conf = this.master.getConfiguration();
     this.fs = this.master.getMasterFileSystem().getFileSystem();
@@ -156,6 +159,7 @@ public void process() {
       completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
       status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshot.getTable()
           + " completed");
+      metricsMaster.addSnapshot(status.getCompletionTimestamp() - status.getStartTime());
     } catch (Exception e) {
       status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " +
           snapshot.getTable() + " because " + e.getMessage());
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index d0256e2ccdc0..7caad2b251d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -63,6 +63,187 @@
 public final class SnapshotInfo extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
 
+  /**
+   * Statistics about the snapshot
+   * <ol>
+   * <li> How many store files and logs are in the archive
+   * <li> How many store files and logs are shared with the table
+   * <li> Total store files and logs size and shared amount
+   * </ol>
+   */
+  public static class SnapshotStats {
+    /** Information about the file referenced by the snapshot */
+    static class FileInfo {
+      private final boolean inArchive;
+      private final long size;
+
+      FileInfo(final boolean inArchive, final long size) {
+        this.inArchive = inArchive;
+        this.size = size;
+      }
+
+      /** @return true if the file is in the archive */
+      public boolean inArchive() {
+        return this.inArchive;
+      }
+
+      /** @return true if the file is missing */
+      public boolean isMissing() {
+        return this.size < 0;
+      }
+
+      /** @return the file size */
+      public long getSize() {
+        return this.size;
+      }
+    }
+
+    private int hfileArchiveCount = 0;
+    private int hfilesMissing = 0;
+    private int hfilesCount = 0;
+    private int logsMissing = 0;
+    private int logsCount = 0;
+    private long hfileArchiveSize = 0;
+    private long hfileSize = 0;
+    private long logSize = 0;
+
+    private final SnapshotDescription snapshot;
+    private final Configuration conf;
+    private final FileSystem fs;
+
+    SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
+    {
+      this.snapshot = snapshot;
+      this.conf = conf;
+      this.fs = fs;
+    }
+
+    /** @return the snapshot descriptor */
+    public SnapshotDescription getSnapshotDescription() {
+      return this.snapshot;
+    }
+
+    /** @return true if the snapshot is corrupted */
+    public boolean isSnapshotCorrupted() {
+      return hfilesMissing > 0 || logsMissing > 0;
+    }
+
+    /** @return the number of available store files */
+    public int getStoreFilesCount() {
+      return hfilesCount + hfileArchiveCount;
+    }
+
+    /** @return the number of available store files in the archive */
+    public int getArchivedStoreFilesCount() {
+      return hfileArchiveCount;
+    }
+
+    /** @return the number of available log files */
+    public int getLogsCount() {
+      return logsCount;
+    }
+
+    /** @return the number of missing store files */
+    public int getMissingStoreFilesCount() {
+      return hfilesMissing;
+    }
+
+    /** @return the number of missing log files */
+    public int getMissingLogsCount() {
+      return logsMissing;
+    }
+
+    /** @return the total size of the store files referenced by the snapshot */
+    public long getStoreFilesSize() {
+      return hfileSize + hfileArchiveSize;
+    }
+
+    /** @return the total size of the store files shared */
+    public long getSharedStoreFilesSize() {
+      return hfileSize;
+    }
+
+    /** @return the total size of the store files in the archive */
+    public long getArchivedStoreFileSize() {
+      return hfileArchiveSize;
+    }
+
+    /** @return the percentage of the shared store files */
+    public float getSharedStoreFilePercentage() {
+      return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
+    }
+
+    /** @return the total log size */
+    public long getLogsSize() {
+      return logSize;
+    }
+
+    /**
+     * Add the specified store file to the stats
+     * @param region region encoded Name
+     * @param family family name
+     * @param hfile store file name
+     * @return the store file information
+     */
+    FileInfo addStoreFile(final String region, final String family, final String hfile)
+          throws IOException {
+      String table = this.snapshot.getTable();
+      Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
+      HFileLink link = new HFileLink(conf, path);
+      boolean inArchive = false;
+      long size = -1;
+      try {
+        if ((inArchive = fs.exists(link.getArchivePath()))) {
+          size = fs.getFileStatus(link.getArchivePath()).getLen();
+          hfileArchiveSize += size;
+          hfileArchiveCount++;
+        } else {
+          size = link.getFileStatus(fs).getLen();
+          hfileSize += size;
+          hfilesCount++;
+        }
+      } catch (FileNotFoundException e) {
+        hfilesMissing++;
+      }
+      return new FileInfo(inArchive, size);
+    }
+
+    /**
+     * Add the specified recovered.edits file to the stats
+     * @param region region encoded name
+     * @param logfile log file name
+     * @return the recovered.edits information
+     */
+    FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
+      Path rootDir = FSUtils.getRootDir(conf);
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
+      long size = fs.getFileStatus(path).getLen();
+      logSize += size;
+      logsCount++;
+      return new FileInfo(true, size);
+    }
+
+    /**
+     * Add the specified log file to the stats
+     * @param server server name
+     * @param logfile log file name
+     * @return the log information
+     */
+    FileInfo addLogFile(final String server, final String logfile) throws IOException {
+      HLogLink logLink = new HLogLink(conf, server, logfile);
+      long size = -1;
+      try {
+        size = logLink.getFileStatus(fs).getLen();
+        logSize += size;
+        logsCount++;
+      } catch (FileNotFoundException e) {
+        logsMissing++;
+      }
+      return new FileInfo(false, size);
+    }
+  }
+
   private FileSystem fs;
   private Path rootDir;
 
@@ -170,104 +351,68 @@ private void printSchema() {
    * dump the file list if requested and the collected information.
    */
   private void printFiles(final boolean showFiles) throws IOException {
-    final String table = snapshotDesc.getTable();
-    final Configuration conf = getConf();
-
     if (showFiles) {
       System.out.println("Snapshot Files");
       System.out.println("----------------------------------------");
     }
 
     // Collect information about hfiles and logs in the snapshot
-    final AtomicInteger hfileArchiveCount = new AtomicInteger();
-    final AtomicInteger hfilesMissing = new AtomicInteger();
-    final AtomicInteger hfilesCount = new AtomicInteger();
-    final AtomicInteger logsMissing = new AtomicInteger();
-    final AtomicInteger logsCount = new AtomicInteger();
-    final AtomicLong hfileArchiveSize = new AtomicLong();
-    final AtomicLong hfileSize = new AtomicLong();
-    final AtomicLong logSize = new AtomicLong();
+    final String table = this.snapshotDesc.getTable();
+    final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
       new SnapshotReferenceUtil.FileVisitor() {
         public void storeFile (final String region, final String family, final String hfile)
             throws IOException {
-          Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
-          HFileLink link = new HFileLink(conf, path);
-          boolean inArchive = false;
-          long size = -1;
-          try {
-            if ((inArchive = fs.exists(link.getArchivePath()))) {
-              size = fs.getFileStatus(link.getArchivePath()).getLen();
-              hfileArchiveSize.addAndGet(size);
-              hfileArchiveCount.addAndGet(1);
-            } else {
-              size = link.getFileStatus(fs).getLen();
-              hfileSize.addAndGet(size);
-              hfilesCount.addAndGet(1);
-            }
-          } catch (FileNotFoundException e) {
-            hfilesMissing.addAndGet(1);
-          }
+          SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
 
           if (showFiles) {
             System.out.printf("%8s %s/%s/%s/%s %s%n",
-              (size < 0 ? "-" : StringUtils.humanReadableInt(size)),
+              (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
               table, region, family, hfile,
-              (inArchive ? "(archive)" : (size < 0) ? "(NOT FOUND)" : ""));
+              (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
           }
         }
 
         public void recoveredEdits (final String region, final String logfile)
             throws IOException {
-          Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
-          long size = fs.getFileStatus(path).getLen();
-          logSize.addAndGet(size);
-          logsCount.addAndGet(1);
+          SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
 
           if (showFiles) {
             System.out.printf("%8s recovered.edits %s on region %s%n",
-              StringUtils.humanReadableInt(size), logfile, region);
+              StringUtils.humanReadableInt(info.getSize()), logfile, region);
           }
         }
 
         public void logFile (final String server, final String logfile)
             throws IOException {
-          HLogLink logLink = new HLogLink(conf, server, logfile);
-          long size = -1;
-          try {
-            size = logLink.getFileStatus(fs).getLen();
-            logSize.addAndGet(size);
-            logsCount.addAndGet(1);
-          } catch (FileNotFoundException e) {
-            logsMissing.addAndGet(1);
-          }
+          SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
 
           if (showFiles) {
             System.out.printf("%8s log %s on server %s %s%n",
-              (size < 0 ? "-" : StringUtils.humanReadableInt(size)),
+              (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
               logfile, server,
-              (size < 0 ? "(NOT FOUND)" : ""));
+              (info.isMissing() ? "(NOT FOUND)" : ""));
           }
         }
     });
 
     // Dump the stats
     System.out.println();
-    if (hfilesMissing.get() > 0 || logsMissing.get() > 0) {
+    if (stats.isSnapshotCorrupted()) {
       System.out.println("**************************************************************");
       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
-        hfilesMissing.get(), logsMissing.get());
+        stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
       System.out.println("**************************************************************");
     }
 
     System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
-      hfilesCount.get() + hfileArchiveCount.get(), hfileArchiveCount.get(),
-      StringUtils.humanReadableInt(hfileSize.get() + hfileArchiveSize.get()),
-      ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100,
-      StringUtils.humanReadableInt(hfileSize.get())
+      stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
+      StringUtils.humanReadableInt(stats.getStoreFilesSize()),
+      stats.getSharedStoreFilePercentage(),
+      StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
     );
     System.out.printf("%d Logs, total size %s%n",
-      logsCount.get(), StringUtils.humanReadableInt(logSize.get()));
+      stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
     System.out.println();
   }
 
@@ -286,6 +431,36 @@ private void printUsageAndExit() {
     System.exit(1);
   }
 
+  /**
+   * Returns the snapshot stats
+   * @param conf the {@link Configuration} to use
+   * @param snapshot {@link SnapshotDescription} to get stats from
+   * @return the snapshot stats
+   */
+  public static SnapshotStats getSnapshotStats(final Configuration conf,
+      final SnapshotDescription snapshot) throws IOException {
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+    final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
+    SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
+      new SnapshotReferenceUtil.FileVisitor() {
+        public void storeFile (final String region, final String family, final String hfile)
+            throws IOException {
+          stats.addStoreFile(region, family, hfile);
+        }
+
+        public void recoveredEdits (final String region, final String logfile) throws IOException {
+          stats.addRecoveredEdits(region, logfile);
+        }
+
+        public void logFile (final String server, final String logfile) throws IOException {
+          stats.addLogFile(server, logfile);
+        }
+    });
+    return stats;
+  }
+
   /**
    * The guts of the {@link #main} method.
    * Call this method to avoid the {@link #main(String[])} System.exit.
diff --git a/src/main/resources/hbase-webapps/master/snapshot.jsp b/src/main/resources/hbase-webapps/master/snapshot.jsp
new file mode 100644
index 000000000000..ab007e90c47b
--- /dev/null
+++ b/src/main/resources/hbase-webapps/master/snapshot.jsp
@@ -0,0 +1,189 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<%@ page contentType="text/html;charset=UTF-8"
+  import="java.util.Date"
+  import="java.util.HashMap"
+  import="org.apache.hadoop.conf.Configuration"
+  import="org.apache.hadoop.hbase.client.HBaseAdmin"
+  import="org.apache.hadoop.hbase.client.HConnectionManager"
+  import="org.apache.hadoop.hbase.HRegionInfo"
+  import="org.apache.hadoop.hbase.master.HMaster"
+  import="org.apache.hadoop.hbase.util.Bytes"
+  import="org.apache.hadoop.hbase.util.FSUtils"
+  import="org.apache.hadoop.hbase.protobuf.ProtobufUtil"
+  import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription"
+  import="org.apache.hadoop.hbase.snapshot.SnapshotInfo"
+  import="org.apache.hadoop.util.StringUtils"
+  import="java.util.List"
+  import="java.util.Map"
+  import="org.apache.hadoop.hbase.HConstants"%><%
+  HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
+  Configuration conf = master.getConfiguration();
+  HBaseAdmin hbadmin = new HBaseAdmin(conf);
+  boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false);
+  String snapshotName = request.getParameter("name");
+  SnapshotDescription snapshot = null;
+  SnapshotInfo.SnapshotStats stats = null;
+  for (SnapshotDescription snapshotDesc: hbadmin.listSnapshots()) {
+    if (snapshotName.equals(snapshotDesc.getName())) {
+      snapshot = snapshotDesc;
+      stats = SnapshotInfo.getSnapshotStats(conf, snapshot);
+      break;
+    }
+  }
+
+  String action = request.getParameter("action");
+  String cloneName = request.getParameter("cloneName");
+  boolean isActionResultPage = (!readOnly && action != null);
+%>
+
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Commenting out DOCTYPE so our blue outline shows on hadoop 0.20.205.0, etc.
+     See tail of HBASE-2110 for explaination.
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+-->
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
+<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
+<% if (isActionResultPage) { %>
+  <title>HBase Master: <%= master.getServerName() %></title>
+  <meta http-equiv="refresh" content="5,javascript:history.back()" />
+<% } else { %>
+  <title>Snapshot: <%= snapshotName %></title>
+<% } %>
+</head>
+<body>
+<a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo.png" alt="HBase Logo" title="HBase Logo" /></a>
+<% if (isActionResultPage) { %>
+  <h1>Snapshot action request...</h1>
+<%
+  if (action.equals("restore")) {
+    hbadmin.restoreSnapshot(snapshotName);
+    %> Restore Snapshot request accepted. <%
+  } else if (action.equals("clone")) {
+    if (cloneName != null && cloneName.length() > 0) {
+      hbadmin.cloneSnapshot(snapshotName, cloneName);
+      %> Clone from Snapshot request accepted. <%
+    } else {
+      %> Clone from Snapshot request failed, No table name specified. <%
+    }
+  }
+%>
+  <p>Go <a href="javascript:history.back()">Back</a>, or wait for the redirect.
+</div>
+<% } else if (snapshot == null) { %>
+  <h1>Snapshot "<%= snapshotName %>" does not exists</h1>
+  <p id="links_menu"><a href="/master.jsp">Master</a>, <a href="/logs/">Local logs</a>, <a href="/stacks">Thread Dump</a>, <a href="/logLevel">Log Level</a></p>
+<hr id="head_rule" />
+  <p>Go <a href="javascript:history.back()">Back</a>, or wait for the redirect.
+<% } else { %>
+  <h1>Snapshot: <%= snapshotName %></h1>
+  <p id="links_menu"><a href="/master.jsp">Master</a>, <a href="/logs/">Local logs</a>, <a href="/stacks">Thread Dump</a>, <a href="/logLevel">Log Level</a></p>
+  <hr id="head_rule" />
+  <h2>Snapshot Attributes</h2>
+  <table class="table" width="90%" >
+    <tr>
+        <th>Table</th>
+        <th>Creation Time</th>
+        <th>Type</th>
+        <th>Format Version</th>
+        <th>State</th>
+    </tr>
+    <tr>
+        <td><a href="table.jsp?name=<%= snapshot.getTable() %>"><%= snapshot.getTable() %></a></td>
+        <td><%= new Date(snapshot.getCreationTime()) %></td>
+        <td><%= snapshot.getType() %></td>
+        <td><%= snapshot.getVersion() %></td>
+        <% if (stats.isSnapshotCorrupted()) { %>
+          <td style="font-weight: bold; color: #dd0000;">CORRUPTED</td>
+        <% } else { %>
+          <td>ok</td>
+        <% } %>
+    </tr>
+  </table>
+  <p>
+    <%= stats.getStoreFilesCount() %> HFiles (<%= stats.getArchivedStoreFilesCount() %> in archive),
+    total size <%= StringUtils.humanReadableInt(stats.getStoreFilesSize()) %>
+    (<%= stats.getSharedStoreFilePercentage() %>&#37;
+    <%= StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) %> shared with the source
+    table)
+  </p>
+  <p>
+    <%= stats.getLogsCount() %> Logs, total size
+    <%= StringUtils.humanReadableInt(stats.getLogsSize()) %>
+  </p>
+  <% if (stats.isSnapshotCorrupted()) { %>
+    <h3>CORRUPTED Snapshot</h3>
+    <p>
+      <%= stats.getMissingStoreFilesCount() %> hfile(s) and
+      <%= stats.getMissingLogsCount() %> log(s) missing.
+    <p>
+  <% } %>
+<%
+  } // end else
+
+HConnectionManager.deleteConnection(hbadmin.getConfiguration());
+%>
+
+
+<% if (!readOnly && action == null && snapshot != null) { %>
+<p><hr><p>
+Actions:
+<p>
+<center>
+<table style="border-style: none" width="90%">
+<tr>
+  <form method="get">
+  <input type="hidden" name="action" value="clone">
+  <input type="hidden" name="name" value="<%= snapshotName %>">
+  <td style="border-style: none; text-align: center">
+      <input style="font-size: 12pt; width: 10em" type="submit" value="Clone" class="btn"></td>
+  <td style="border-style: none" width="5%">&nbsp;</td>
+  <td style="border-style: none">New Table Name (clone):<input type="text" name="cloneName" size="40"></td>
+  <td style="border-style: none">
+    This action will create a new table by cloning the snapshot content.
+    There are no copies of data involved.
+    And writing on the newly created table will not influence the snapshot data.
+  </td>
+  </form>
+</tr>
+<tr><td style="border-style: none" colspan="4">&nbsp;</td></tr>
+<tr>
+  <form method="get">
+  <input type="hidden" name="action" value="restore">
+  <input type="hidden" name="name" value="<%= snapshotName %>">
+  <td style="border-style: none; text-align: center">
+      <input style="font-size: 12pt; width: 10em" type="submit" value="Restore" class="btn"></td>
+  <td style="border-style: none" width="5%">&nbsp;</td>
+  <td style="border-style: none">&nbsp;</td>
+  <td style="border-style: none">Restore a specified snapshot.
+  The restore will replace the content of the original table,
+  bringing back the content to the snapshot state.
+  The table must be disabled.</td>
+  </form>
+</tr>
+</table>
+</center>
+<p>
+</div>
+<% } %>
+</body>
+</html>
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
index c84b6c03f517..ffd5672ec9ec 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.zookeeper.KeeperException;
@@ -49,6 +50,7 @@ public class TestSnapshotManager {
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
 
   MasterServices services = Mockito.mock(MasterServices.class);
+  MasterMetrics metrics = Mockito.mock(MasterMetrics.class);
   ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class);
   ExecutorService pool = Mockito.mock(ExecutorService.class);
   MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
@@ -71,7 +73,7 @@ private SnapshotManager getNewManager(final Configuration conf) throws IOExcepti
     Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
     Mockito.when(mfs.getFileSystem()).thenReturn(fs);
     Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
-    return new SnapshotManager(services, coordinator, pool);
+    return new SnapshotManager(services, metrics, coordinator, pool);
   }
 
   @Test

From 4eaf9a3cebd6ef9159ef1acb808cd75489ddfee9 Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Fri, 5 Apr 2013 17:42:30 +0000
Subject: [PATCH 0910/1540] HBASE-8150 server should not produce RAITE for
 already-opening region in 0.94 (because master retry logic handles this case
 poorly)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465063 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     | 28 +++++++++++++------
 .../master/TestZKBasedOpenCloseRegion.java    |  4 +--
 .../handler/TestOpenRegionHandler.java        |  4 +--
 3 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e45b46106038..f94e0f13866a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2942,7 +2942,14 @@ private RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNo
     }
     // Added to in-memory RS RIT that we are trying to open this region.
     // Clear it if we fail queuing an open executor.
-    addRegionsInTransition(region, OPEN);
+    boolean isNewRit = addRegionsInTransition(region, OPEN);
+    if (!isNewRit) {
+      // An open is in progress. This is supported, but let's log this.
+      LOG.info("Receiving OPEN for the region:" +
+          region.getRegionNameAsString() + " , which we are already trying to OPEN" +
+          " - ignoring this new request for this region.");
+      return RegionOpeningState.OPENED;
+    }
     try {
       LOG.info("Received request to open region: " +
         region.getRegionNameAsString());
@@ -3026,16 +3033,19 @@ int transitionZookeeperOfflineToOpening(final HRegionInfo hri, int versionOfOffl
     *          Whether OPEN or CLOSE.
     * @throws RegionAlreadyInTransitionException
     */
-   protected void addRegionsInTransition(final HRegionInfo region, final String currentAction)
+   protected boolean addRegionsInTransition(final HRegionInfo region, final String currentAction)
        throws RegionAlreadyInTransitionException {
-     Boolean action = this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(),
-         currentAction.equals(OPEN));
-     if (action != null) {
-       // The below exception message will be used in master.
-       throw new RegionAlreadyInTransitionException("Received:" + currentAction + " for the region:"
-           + region.getRegionNameAsString() + " for the region:" + region.getRegionNameAsString()
-           + ", which we are already trying to " + (action ? OPEN : CLOSE) + ".");
+     boolean isOpen = currentAction.equals(OPEN);
+     Boolean action = this.regionsInTransitionInRS.putIfAbsent(
+         region.getEncodedNameAsBytes(), isOpen);
+     if (action == null) return true;
+     if (isOpen && action.booleanValue()) {
+       return false;
      }
+     // The below exception message will be used in master.
+     throw new RegionAlreadyInTransitionException("Received:" + currentAction
+         + " for the region:" + region.getRegionNameAsString()
+         + ", which we are already trying to " + (action ? OPEN : CLOSE) + ".");
    }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
index 02555a8d98b1..516c02864ac8 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
@@ -107,9 +107,9 @@ public TestZKBasedOpenCloseRegionRegionServer(Configuration conf)
       super(conf);
     }
     @Override
-    public void addRegionsInTransition(HRegionInfo region,
+    public boolean addRegionsInTransition(HRegionInfo region,
         String currentAction) throws RegionAlreadyInTransitionException {
-      super.addRegionsInTransition(region, currentAction);
+      return super.addRegionsInTransition(region, currentAction);
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
index 3a8f1321fc98..a2b3fef0453c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
@@ -202,9 +202,9 @@ public TestOpenRegionHandlerRegionServer(Configuration conf)
       super(conf);
     }
     @Override
-    public void addRegionsInTransition(HRegionInfo region,
+    public boolean addRegionsInTransition(HRegionInfo region,
         String currentAction) throws RegionAlreadyInTransitionException {
-      super.addRegionsInTransition(region, currentAction);
+      return super.addRegionsInTransition(region, currentAction);
     }
   }
   

From 53cfeb419a311c1620ca43205a4c949c488b1249 Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Fri, 5 Apr 2013 20:27:24 +0000
Subject: [PATCH 0911/1540] HBASE-8260 create generic integration test for
 trunk and 94 that is more deterministic, can be run for longer and is less
 aggressive

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465118 13f79535-47bb-0310-9956-ffa450edef68
---
 ...rationTestDataIngestSlowDeterministic.java |  77 +++++++++
 ...egrationTestDataIngestWithChaosMonkey.java |   5 -
 .../apache/hadoop/hbase/util/ChaosMonkey.java | 146 ++++++++++++++----
 3 files changed, 195 insertions(+), 33 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java

diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java
new file mode 100644
index 000000000000..57415a19962b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.util.ChaosMonkey;
+import org.apache.hadoop.hbase.util.ChaosMonkey.BatchRestartRs;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartActiveMaster;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRandomRs;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRsHoldingMeta;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRsHoldingRoot;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RollingBatchRestartRs;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * A system test which does large data ingestion and verify using {@link LoadTestTool}.
+ * It performs a set of actions deterministically using ChaosMonkey, then starts killing
+ * things randomly. You can configure how long should the load test run by using 
+ * "hbase.IntegrationTestDataIngestSlowDeterministic.runtime" configuration parameter.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestDataIngestSlowDeterministic extends IngestIntegrationTestBase {
+  private static final int SERVER_COUNT = 3; // number of slaves for the smallest cluster
+  private static final long DEFAULT_RUN_TIME = 30 * 60 * 1000;
+  private static final long CHAOS_EVERY_MS = 150 * 1000; // Chaos every 2.5 minutes.
+
+  private ChaosMonkey monkey;
+
+  @Before
+  public void setUp() throws Exception {
+    super.setUp(SERVER_COUNT);
+    ChaosMonkey.Action[] actions = new ChaosMonkey.Action[] {
+        new RestartRandomRs(60000),
+        new BatchRestartRs(5000, 0.5f),
+        new RestartActiveMaster(5000),
+        new RollingBatchRestartRs(5000, 1.0f),
+        new RestartRsHoldingMeta(35000),
+        new RestartRsHoldingRoot(35000)
+    };
+    monkey = new ChaosMonkey(util, new ChaosMonkey.CompositeSequentialPolicy(
+            new ChaosMonkey.DoActionsOncePolicy(CHAOS_EVERY_MS, actions),
+            new ChaosMonkey.PeriodicRandomActionPolicy(CHAOS_EVERY_MS, actions)));
+    monkey.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (monkey != null) {
+      monkey.stop("tearDown");
+      monkey.waitForStop();
+    }
+    super.tearDown();
+  }
+
+  @Test
+  public void testDataIngest() throws Exception {
+    runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 100, 5);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
index 842ecf96bc1e..72891b1849c1 100644
--- a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
@@ -18,11 +18,6 @@
 
 package org.apache.hadoop.hbase;
 
-import java.io.IOException;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChaosMonkey;
 import org.junit.After;
 import org.junit.Before;
diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index ad63e6e956f1..266586f394dc 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -446,16 +446,122 @@ public void init(PolicyContext context) throws Exception {
     }
   }
 
+  /** A policy that runs multiple other policies one after the other */
+  public static class CompositeSequentialPolicy extends Policy {
+    private List<Policy> policies;
+    public CompositeSequentialPolicy(Policy... policies) {
+      this.policies = Arrays.asList(policies);
+    }
+
+    @Override
+    public void stop(String why) {
+      super.stop(why);
+      for (Policy p : policies) {
+        p.stop(why);
+      }
+    }
+
+    @Override
+    public void run() {
+      for (Policy p : policies) {
+        p.run();
+      }
+    }
+
+    @Override
+    public void init(PolicyContext context) throws Exception {
+      super.init(context);
+      for (Policy p : policies) {
+        p.init(context);
+      }
+    }
+  }
+
+  /** A policy which does stuff every time interval. */
+  public static abstract class PeriodicPolicy extends Policy {
+    private long periodMs;
+
+    public PeriodicPolicy(long periodMs) {
+      this.periodMs = periodMs;
+    }
+
+    @Override
+    public void run() {
+      // Add some jitter.
+      int jitter = new Random().nextInt((int)periodMs);
+      LOG.info("Sleeping for " + jitter + " to add jitter");
+      Threads.sleep(jitter);
+
+      while (!isStopped()) {
+        long start = System.currentTimeMillis();
+        runOneIteration();
+
+        if (isStopped()) return;
+        long sleepTime = periodMs - (System.currentTimeMillis() - start);
+        if (sleepTime > 0) {
+          LOG.info("Sleeping for: " + sleepTime);
+          Threads.sleep(sleepTime);
+        }
+      }
+    }
+
+    protected abstract void runOneIteration();
+
+    @Override
+    public void init(PolicyContext context) throws Exception {
+      super.init(context);
+      LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period: " + periodMs);
+    }
+  }
+
+
+  /** A policy which performs a sequence of actions deterministically. */
+  public static class DoActionsOncePolicy extends PeriodicPolicy {
+    private List<Action> actions;
+
+    public DoActionsOncePolicy(long periodMs, List<Action> actions) {
+      super(periodMs);
+      this.actions = new ArrayList<ChaosMonkey.Action>(actions);
+    }
+
+    public DoActionsOncePolicy(long periodMs, Action... actions) {
+      this(periodMs, Arrays.asList(actions));
+    }
+
+    @Override
+    protected void runOneIteration() {
+      if (actions.isEmpty()) {
+        this.stop("done");
+        return;
+      }
+      Action action = actions.remove(0);
+
+      try {
+        action.perform();
+      } catch (Exception ex) {
+        LOG.warn("Exception occured during performing action: "
+            + StringUtils.stringifyException(ex));
+      }
+    }
+
+    @Override
+    public void init(PolicyContext context) throws Exception {
+      super.init(context);
+      for (Action action : actions) {
+        action.init(this.context);
+      }
+    }
+  }
+
   /**
    * A policy, which picks a random action according to the given weights,
    * and performs it every configurable period.
    */
-  public static class PeriodicRandomActionPolicy extends Policy {
-    private long periodMs;
+  public static class PeriodicRandomActionPolicy extends PeriodicPolicy {
     private List<Pair<Action, Integer>> actions;
 
     public PeriodicRandomActionPolicy(long periodMs, List<Pair<Action, Integer>> actions) {
-      this.periodMs = periodMs;
+      super(periodMs);
       this.actions = actions;
     }
 
@@ -465,7 +571,7 @@ public PeriodicRandomActionPolicy(long periodMs, Pair<Action, Integer>... action
     }
 
     public PeriodicRandomActionPolicy(long periodMs, Action... actions) {
-      this.periodMs = periodMs;
+      super(periodMs);
       this.actions = new ArrayList<Pair<Action, Integer>>(actions.length);
       for (Action action : actions) {
         this.actions.add(new Pair<Action, Integer>(action, 1));
@@ -473,35 +579,19 @@ public PeriodicRandomActionPolicy(long periodMs, Action... actions) {
     }
 
     @Override
-    public void run() {
-      //add some jitter
-      int jitter = new Random().nextInt((int)periodMs);
-      LOG.info("Sleeping for " + jitter + " to add jitter");
-      Threads.sleep(jitter);
-
-      while (!isStopped()) {
-        long start = System.currentTimeMillis();
-        Action action = selectWeightedRandomItem(actions);
-
-        try {
-          action.perform();
-        } catch (Exception ex) {
-          LOG.warn("Exception occured during performing action: "
-              + StringUtils.stringifyException(ex));
-        }
-
-        long sleepTime = periodMs - (System.currentTimeMillis() - start);
-        if (sleepTime > 0) {
-          LOG.info("Sleeping for:" + sleepTime);
-          Threads.sleep(sleepTime);
-        }
+    protected void runOneIteration() {
+      Action action = selectWeightedRandomItem(actions);
+      try {
+        action.perform();
+      } catch (Exception ex) {
+        LOG.warn("Exception occured during performing action: "
+            + StringUtils.stringifyException(ex));
       }
     }
 
     @Override
     public void init(PolicyContext context) throws Exception {
       super.init(context);
-      LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period:" + periodMs);
       for (Pair<Action, Integer> action : actions) {
         action.getFirst().init(this.context);
       }
@@ -654,4 +744,4 @@ public static void main(String[] args) throws Exception {
     System.exit(ret);
   }
 
-}
\ No newline at end of file
+}

From 1d13c114aaf92d74023027fc970df3082fd3cbed Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 5 Apr 2013 20:39:52 +0000
Subject: [PATCH 0912/1540] HBASE-8270 Backport HBASE-8097
 'MetaServerShutdownHandler may potentially keep bumping up
 DeadServer.numProcessing' to 0.94 (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465120 13f79535-47bb-0310-9956-ffa450edef68
---
 .../handler/MetaServerShutdownHandler.java    | 91 +++++++++++--------
 1 file changed, 51 insertions(+), 40 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
index 5c2ba7b00430..198d6f42bc54 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
@@ -50,53 +50,64 @@ public MetaServerShutdownHandler(final Server server,
 
   @Override
   public void process() throws IOException {
+
+    boolean gotException = true;
     try {
-      if (this.shouldSplitHlog) {
-        if (this.services.shouldSplitMetaSeparately()) {
-          LOG.info("Splitting META logs for " + serverName);
-          this.services.getMasterFileSystem().splitMetaLog(serverName);
+      try {
+        if (this.shouldSplitHlog) {
+          if (this.services.shouldSplitMetaSeparately()) {
+            LOG.info("Splitting META logs for " + serverName);
+            this.services.getMasterFileSystem().splitMetaLog(serverName);
+          } else {
+            LOG.info("Splitting all logs for " + serverName);
+            this.services.getMasterFileSystem().splitAllLogs(serverName);
+          }
+        }
+      } catch (IOException ioe) {
+        this.services.getExecutorService().submit(this);
+        this.deadServers.add(serverName);
+        throw new IOException("failed log splitting for " +
+            serverName + ", will retry", ioe);
+      }
+  
+      // Assign root and meta if we were carrying them.
+      if (isCarryingRoot()) { // -ROOT-
+        // Check again: region may be assigned to other where because of RIT
+        // timeout
+        if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
+          LOG.info("Server " + serverName
+              + " was carrying ROOT. Trying to assign.");
+          this.services.getAssignmentManager().regionOffline(
+              HRegionInfo.ROOT_REGIONINFO);
+          verifyAndAssignRootWithRetries();
         } else {
-          LOG.info("Splitting all logs for " + serverName);
-          this.services.getMasterFileSystem().splitAllLogs(serverName);
+          LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
         }
       }
-    } catch (IOException ioe) {
-      this.services.getExecutorService().submit(this);
-      this.deadServers.add(serverName);
-      throw new IOException("failed log splitting for " +
-          serverName + ", will retry", ioe);
-    }
-
-    // Assign root and meta if we were carrying them.
-    if (isCarryingRoot()) { // -ROOT-
-      // Check again: region may be assigned to other where because of RIT
-      // timeout
-      if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
-        LOG.info("Server " + serverName
-            + " was carrying ROOT. Trying to assign.");
-        this.services.getAssignmentManager().regionOffline(
-            HRegionInfo.ROOT_REGIONINFO);
-        verifyAndAssignRootWithRetries();
-      } else {
-        LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
+  
+      // Carrying meta?
+      if (isCarryingMeta()) {
+        // Check again: region may be assigned to other where because of RIT
+        // timeout
+        if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
+          LOG.info("Server " + serverName
+              + " was carrying META. Trying to assign.");
+          this.services.getAssignmentManager().regionOffline(
+              HRegionInfo.FIRST_META_REGIONINFO);
+          this.services.getAssignmentManager().assignMeta();
+        } else {
+          LOG.info("META has been assigned to otherwhere, skip assigning.");
+        }
       }
-    }
-
-    // Carrying meta?
-    if (isCarryingMeta()) {
-      // Check again: region may be assigned to other where because of RIT
-      // timeout
-      if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
-        LOG.info("Server " + serverName
-            + " was carrying META. Trying to assign.");
-        this.services.getAssignmentManager().regionOffline(
-            HRegionInfo.FIRST_META_REGIONINFO);
-        this.services.getAssignmentManager().assignMeta();
-      } else {
-        LOG.info("META has been assigned to otherwhere, skip assigning.");
+      
+      gotException = false;
+    } finally {
+      if (gotException){
+        // If we had an exception, this.deadServers.finish will be skipped in super.process()
+        this.deadServers.finish(serverName);
       }
-
     }
+
     super.process();
   }
   /**

From bb03fbaa736feef478eb27e866f98e46b3c87622 Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Fri, 5 Apr 2013 21:56:53 +0000
Subject: [PATCH 0913/1540] HBASE-8274 Backport to 94: HBASE-7488 Implement
 HConnectionManager.locateRegions which is currently returning null

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465142 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HRegionInfo.java  |  9 +++++++
 .../hadoop/hbase/client/HConnection.java      | 16 ++++++++++--
 .../hbase/client/HConnectionManager.java      | 25 +++++++++++++------
 3 files changed, 41 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 942050522588..53f672082e42 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -404,6 +404,15 @@ public HRegionInfo(HRegionInfo other) {
     return tableName;
   }
 
+  /**
+   * Gets the start key from the specified region name.
+   * @param regionName
+   * @return Start key.
+   */
+  public static byte[] getStartKey(final byte[] regionName) throws IOException {
+    return parseRegionName(regionName)[1];
+  }
+
   /**
    * Separate elements of a regionName.
    * @param regionName
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index 576e2d2dede8..f3bd9fa0c5e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -191,8 +191,20 @@ public HRegionLocation locateRegion(final byte [] regionName)
    * @return list of region locations for all regions of table
    * @throws IOException
    */
-  public List<HRegionLocation> locateRegions(byte[] tableName)
-  throws IOException;
+  public List<HRegionLocation> locateRegions(final byte[] tableName)
+      throws IOException;
+
+  /**
+   * Gets the locations of all regions in the specified table, <i>tableName</i>.
+   * @param tableName table to get regions of
+   * @param useCache Should we use the cache to retrieve the region information.
+   * @param offlined True if we are to include offlined regions, false and we'll leave out offlined
+   *          regions from returned list.
+   * @return list of region locations for all regions of table
+   * @throws IOException
+   */
+  public List<HRegionLocation> locateRegions(final byte[] tableName, final boolean useCache,
+      final boolean offlined) throws IOException;
 
   /**
    * Establishes a connection to the region server at the specified address.
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index a988fb64b9f7..a9a51f912dd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NavigableMap;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
@@ -827,17 +828,27 @@ private boolean testTableOnlineState(byte [] tableName, boolean online)
     }
 
     @Override
-    public HRegionLocation locateRegion(final byte [] regionName)
-    throws IOException {
-      // TODO implement.  use old stuff or new stuff?
-      return null;
+    public HRegionLocation locateRegion(final byte[] regionName) throws IOException {
+      return locateRegion(HRegionInfo.getTableName(regionName),
+        HRegionInfo.getStartKey(regionName), false, true);
     }
 
     @Override
-    public List<HRegionLocation> locateRegions(final byte [] tableName)
+    public List<HRegionLocation> locateRegions(final byte[] tableName)
     throws IOException {
-      // TODO implement.  use old stuff or new stuff?
-      return null;
+      return locateRegions(tableName, false, true);
+    }
+
+    @Override
+    public List<HRegionLocation> locateRegions(final byte[] tableName, final boolean useCache,
+        final boolean offlined) throws IOException {
+      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
+        offlined);
+      final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
+      for (HRegionInfo regionInfo : regions.keySet()) {
+        locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
+      }
+      return locations;
     }
 
     public HRegionLocation locateRegion(final byte [] tableName,

From 00259166a5d49575ba40de5123ed487c29177639 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 5 Apr 2013 23:56:08 +0000
Subject: [PATCH 0914/1540] HBASE-8276 Backport hbase-6738 to 0.94 "Too
 aggressive task resubmission from the distributed log manager" (Jeffrey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465161 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/MasterFileSystem.java |  2 +-
 .../hadoop/hbase/master/SplitLogManager.java  | 46 ++++++++--
 .../hadoop/hbase/zookeeper/ZKSplitLog.java    |  2 +-
 .../hbase/master/TestSplitLogManager.java     | 89 +++++++++++--------
 4 files changed, 93 insertions(+), 46 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 31c1e9bc63ea..32abd2302b6b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -122,7 +122,7 @@ public MasterFileSystem(Server master, MasterServices services,
       conf.getBoolean("hbase.master.distributed.log.splitting", true);
     if (this.distributedLogSplitting) {
       this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
-          master.getConfiguration(), master, master.getServerName().toString());
+          master.getConfiguration(), master, this.services, master.getServerName().toString());
       this.splitLogManager.finishInitialization(masterRecovery);
     } else {
       this.splitLogManager = null;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index c74659ecfa5e..2d6881d56365 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -63,6 +63,8 @@
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
 
+import com.google.common.base.Strings;
+
 import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.*;
 import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.*;
 
@@ -99,6 +101,7 @@ public class SplitLogManager extends ZooKeeperListener {
   private static final Log LOG = LogFactory.getLog(SplitLogManager.class);
 
   private final Stoppable stopper;
+  private final MasterServices master;
   private final String serverName;
   private final TaskFinisher taskFinisher;
   private FileSystem fs;
@@ -111,12 +114,12 @@ public class SplitLogManager extends ZooKeeperListener {
   private long lastNodeCreateTime = Long.MAX_VALUE;
   public boolean ignoreZKDeleteForTesting = false;
 
-  private ConcurrentMap<String, Task> tasks =
+  private final ConcurrentMap<String, Task> tasks =
     new ConcurrentHashMap<String, Task>();
   private TimeoutMonitor timeoutMonitor;
 
   private Set<String> deadWorkers = null;
-  private Object deadWorkersLock = new Object();
+  private final Object deadWorkersLock = new Object();
 
   private Set<String> failedDeletions = null;
 
@@ -133,8 +136,8 @@ public class SplitLogManager extends ZooKeeperListener {
    * @param serverName
    */
   public SplitLogManager(ZooKeeperWatcher zkw, final Configuration conf,
-      Stoppable stopper, String serverName) {
-    this(zkw, conf, stopper, serverName, new TaskFinisher() {
+      Stoppable stopper, MasterServices master, String serverName) {
+    this(zkw, conf, stopper, master, serverName, new TaskFinisher() {
       @Override
       public Status finish(String workerName, String logfile) {
         try {
@@ -160,11 +163,12 @@ public Status finish(String workerName, String logfile) {
    * @param tf task finisher 
    */
   public SplitLogManager(ZooKeeperWatcher zkw, Configuration conf,
-      Stoppable stopper, String serverName, TaskFinisher tf) {
+      Stoppable stopper, MasterServices master, String serverName, TaskFinisher tf) {
     super(zkw);
     this.taskFinisher = tf;
     this.conf = conf;
     this.stopper = stopper;
+    this.master = master;
     this.zkretries = conf.getLong("hbase.splitlog.zk.retries",
         ZKSplitLog.DEFAULT_ZK_RETRIES);
     this.resubmit_threshold = conf.getLong("hbase.splitlog.max.resubmit",
@@ -174,8 +178,9 @@ public SplitLogManager(ZooKeeperWatcher zkw, Configuration conf,
     this.unassignedTimeout =
       conf.getInt("hbase.splitlog.manager.unassigned.timeout",
         ZKSplitLog.DEFAULT_UNASSIGNED_TIMEOUT);
-    LOG.debug("timeout = " + timeout);
-    LOG.debug("unassigned timeout = " + unassignedTimeout);
+    LOG.info("timeout = " + timeout);
+    LOG.info("unassigned timeout = " + unassignedTimeout);
+    LOG.info("resubmit threshold = " + this.resubmit_threshold);
 
     this.serverName = serverName;
     this.timeoutMonitor = new TimeoutMonitor(
@@ -598,8 +603,31 @@ private boolean resubmit(String path, Task task,
     }
     int version;
     if (directive != FORCE) {
-      if ((EnvironmentEdgeManager.currentTimeMillis() - task.last_update) <
-          timeout) {
+      // We're going to resubmit:
+      // 1) immediately if the worker server is now marked as dead
+      // 2) after a configurable timeout if the server is not marked as dead but has still not
+      // finished the task. This allows to continue if the worker cannot actually handle it,
+      // for any reason.
+      final long time = EnvironmentEdgeManager.currentTimeMillis() - task.last_update;
+      ServerName curWorker = null;
+      if (!Strings.isNullOrEmpty(task.cur_worker_name)) {
+        try {
+          curWorker = ServerName.parseServerName(task.cur_worker_name);
+        } catch (IllegalArgumentException ie) {
+          LOG.error("Got invalid server name:" + task.cur_worker_name + " - task for path:" + path
+              + " won't be resubmitted before timeout");
+        }
+      } else {
+        LOG.error("Got empty/null server name:" + task.cur_worker_name + " - task for path:" + path
+            + " won't be resubmitted before timeout");
+      }
+      final boolean alive =
+          (master.getServerManager() != null && curWorker != null) ? master.getServerManager()
+              .isServerOnline(curWorker) : true;
+      if (alive && time < timeout) {
+        LOG.trace("Skipping the resubmit of " + task.toString() + "  because the server "
+            + task.cur_worker_name + " is not marked as dead, we waited for " + time
+            + " while the timeout is " + timeout);
         return false;
       }
       if (task.unforcedResubmits >= resubmit_threshold) {
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
index f5d5f85d648a..526c4cc17a5d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
@@ -46,7 +46,7 @@
 public class ZKSplitLog {
   private static final Log LOG = LogFactory.getLog(ZKSplitLog.class);
 
-  public static final int DEFAULT_TIMEOUT = 25000; // 25 sec
+  public static final int DEFAULT_TIMEOUT = 300000; // 5 mins
   public static final int DEFAULT_ZK_RETRIES = 3;
   public static final int DEFAULT_MAX_RESUBMIT = 3;
   public static final int DEFAULT_UNASSIGNED_TIMEOUT = (3 * 60 * 1000); //3 min
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index d76601861eb6..cb1d3e8b2204 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -23,10 +23,11 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLong;
 
+import junit.framework.Assert;
+
 import static org.junit.Assert.*;
 
 import org.apache.commons.logging.Log;
@@ -48,11 +49,10 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 @Category(MediumTests.class)
 public class TestSplitLogManager {
@@ -66,6 +66,8 @@ public class TestSplitLogManager {
   private SplitLogManager slm;
   private Configuration conf;
   private int to;
+  private final ServerManager sm = Mockito.mock(ServerManager.class);
+  private final MasterServices master = Mockito.mock(MasterServices.class);
 
   private final static HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
@@ -83,14 +85,6 @@ public boolean isStopped() {
 
   };
 
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-  }
-
   @Before
   public void setup() throws Exception {
     TEST_UTIL.startMiniZKCluster();
@@ -111,6 +105,11 @@ public void setup() throws Exception {
     conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to);
     conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
     to = to + 4 * 100;
+
+    // By default, we let the test manage the error as before, so the server
+    // does not appear as dead from the master point of view, only from the split log pov.
+    Mockito.when(sm.isServerOnline(Mockito.any(ServerName.class))).thenReturn(true);
+    Mockito.when(master.getServerManager()).thenReturn(sm);
   }
 
   @After
@@ -180,7 +179,7 @@ private String submitTaskAndWait(TaskBatch batch, String name)
   public void testTaskCreation() throws Exception {
     LOG.info("TestTaskCreation - test the creation of a task in zk");
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
 
@@ -200,7 +199,7 @@ public void testOrphanTaskAcquisition() throws Exception {
         TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
     Task task = slm.findOrCreateOrphanTask(tasknode);
@@ -227,7 +226,7 @@ public void testUnassignedOrphan() throws Exception {
         CreateMode.PERSISTENT);
     int version = ZKUtil.checkExists(zkw, tasknode);
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
     Task task = slm.findOrCreateOrphanTask(tasknode);
@@ -251,7 +250,7 @@ public void testMultipleResubmits() throws Exception {
     LOG.info("TestMultipleResbmits - no indefinite resubmissions");
 
     conf.setInt("hbase.splitlog.max.resubmit", 2);
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
 
@@ -279,7 +278,7 @@ public void testMultipleResubmits() throws Exception {
   public void testRescanCleanup() throws Exception {
     LOG.info("TestRescanCleanup - ensure RESCAN nodes are cleaned up");
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
 
@@ -294,26 +293,21 @@ public long eval() {
         return (tot_mgr_resubmit.get() + tot_mgr_resubmit_failed.get());
       }
     }, 0, 1, 5*60000); // wait long enough
-    if (tot_mgr_resubmit_failed.get() == 0) {
-      int version1 = ZKUtil.checkExists(zkw, tasknode);
-      assertTrue(version1 > version);
-      byte[] taskstate = ZKUtil.getData(zkw, tasknode);
-      assertTrue(Arrays.equals(TaskState.TASK_UNASSIGNED.get("dummy-master"),
-          taskstate));
-      
-      waitForCounter(tot_mgr_rescan_deleted, 0, 1, to/2);
-    } else {
-      LOG.warn("Could not run test. Lost ZK connection?");
-    }
+    Assert
+        .assertEquals("Could not run test. Lost ZK connection?", 0, tot_mgr_resubmit_failed.get());
+    int version1 = ZKUtil.checkExists(zkw, tasknode);
+    assertTrue(version1 > version);
+    byte[] taskstate = ZKUtil.getData(zkw, tasknode);
+    assertTrue(Arrays.equals(TaskState.TASK_UNASSIGNED.get("dummy-master"), taskstate));
 
-    return;
+    waitForCounter(tot_mgr_rescan_deleted, 0, 1, to / 2);
   }
 
   @Test
   public void testTaskDone() throws Exception {
     LOG.info("TestTaskDone - cleanup task node once in DONE state");
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
     String tasknode = submitTaskAndWait(batch, "foo/1");
@@ -332,7 +326,7 @@ public void testTaskErr() throws Exception {
     LOG.info("TestTaskErr - cleanup task node once in ERR state");
 
     conf.setInt("hbase.splitlog.max.resubmit", 0);
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
 
@@ -352,7 +346,7 @@ public void testTaskErr() throws Exception {
   public void testTaskResigned() throws Exception {
     LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
     String tasknode = submitTaskAndWait(batch, "foo/1");
@@ -379,7 +373,7 @@ public void testUnassignedTimeout() throws Exception {
         TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
 
@@ -409,7 +403,7 @@ public void testDeadWorker() throws Exception {
     LOG.info("testDeadWorker");
 
     conf.setLong("hbase.splitlog.max.resubmit", 0);
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     TaskBatch batch = new TaskBatch();
 
@@ -433,7 +427,7 @@ public void testDeadWorker() throws Exception {
   @Test
   public void testEmptyLogDir() throws Exception {
     LOG.info("testEmptyLogDir");
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     FileSystem fs = TEST_UTIL.getTestFileSystem();
     Path emptyLogDirPath = new Path(fs.getWorkingDirectory(),
@@ -448,7 +442,7 @@ public void testVanishingTaskZNode() throws Exception {
     LOG.info("testVanishingTaskZNode");
     conf.setInt("hbase.splitlog.manager.unassigned.timeout", 0);
     conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 1000);
-    slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
     slm.finishInitialization();
     FileSystem fs = TEST_UTIL.getTestFileSystem();
     final Path logDir = new Path(fs.getWorkingDirectory(),
@@ -488,6 +482,31 @@ public void run() {
     }
   }
 
+  @Test
+  public void testWorkerCrash() throws Exception {
+    conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLog.DEFAULT_MAX_RESUBMIT);
+    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
+    slm.finishInitialization();
+    TaskBatch batch = new TaskBatch();
+
+    String tasknode = submitTaskAndWait(batch, "foo/1");
+    final ServerName worker1 = new ServerName("worker1,1,1");
+
+    ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get(worker1.getServerName()));
+    if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to / 2);
+
+    // Not yet resubmitted.
+    Assert.assertEquals(0, tot_mgr_resubmit.get());
+
+    // This server becomes dead
+    Mockito.when(sm.isServerOnline(worker1)).thenReturn(false);
+
+    Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded).
+
+    // It has been resubmitted
+    Assert.assertEquals(1, tot_mgr_resubmit.get());
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 9b6c5bbd09ec19e470e00bf99e0f8e9beae67406 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Apr 2013 03:16:47 +0000
Subject: [PATCH 0915/1540] HBASE-8208 In some situations data is not
 replicated to slaves when deferredLogSync is enabled (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465174 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java       | 16 ++++++++++++++--
 .../hadoop/hbase/regionserver/wal/HLog.java      | 10 +++++-----
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 47c178eee2bc..5cdd90d9ba5d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1589,6 +1589,12 @@ protected boolean internalFlushcache(
     status.setStatus(s);
     LOG.debug(s);
 
+    // sync unflushed WAL changes when deferred log sync is enabled
+    // see HBASE-8208 for details
+    if (wal != null && isDeferredLogSyncEnabled()) {
+      wal.sync();
+    }
+
     // wait for all in-progress transactions to commit to HLog before
     // we can start the flush. This prevents
     // uncommitted transactions from being written into HFiles.
@@ -5829,12 +5835,18 @@ private void lock(final Lock lock, final int multiplier)
    * @throws IOException If anything goes wrong with DFS
    */
   private void syncOrDefer(long txid) throws IOException {
-    if (this.regionInfo.isMetaRegion() ||
-      !this.htableDescriptor.isDeferredLogFlush() || this.deferredLogSyncDisabled) {
+    if (this.getRegionInfo().isMetaRegion() || !isDeferredLogSyncEnabled()) {
       this.log.sync(txid);
     }
   }
 
+  /**
+   * check if current region is deferred sync enabled.
+   */
+  private boolean isDeferredLogSyncEnabled() {
+    return (this.htableDescriptor.isDeferredLogFlush() && !this.deferredLogSyncDisabled);
+  }
+
   /**
    * A mocked list implementaion - discards all updates.
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 91baba4c7ec7..8c4df449fd66 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1321,16 +1321,16 @@ private void syncer() throws IOException {
 
   // sync all transactions upto the specified txid
   private void syncer(long txid) throws IOException {
+    // if the transaction that we are interested in is already
+    // synced, then return immediately.
+    if (txid <= this.syncedTillHere) {
+      return;
+    }
     Writer tempWriter;
     synchronized (this.updateLock) {
       if (this.closed) return;
       tempWriter = this.writer; // guaranteed non-null
     }
-    // if the transaction that we are interested in is already 
-    // synced, then return immediately.
-    if (txid <= this.syncedTillHere) {
-      return;
-    }
     try {
       long doneUpto;
       long now = System.currentTimeMillis();

From ad56140782c6a364642fceaaf66ce45695736883 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Apr 2013 05:16:20 +0000
Subject: [PATCH 0916/1540] HBASE-7961 truncate on disabled table should throw
 TableNotEnabledException. (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465186 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/admin.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index e26fe6d3b3f9..a40a37d3aaf0 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -310,11 +310,12 @@ def describe(table_name)
     def truncate(table_name, conf = @conf)
       h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name)
       table_description = h_table.getTableDescriptor()
+      raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name)
       yield 'Disabling table...' if block_given?
-      disable(table_name)
+      @admin.disableTable(table_name)
 
       yield 'Dropping table...' if block_given?
-      drop(table_name)
+      @admin.deleteTable(table_name)
 
       yield 'Creating table...' if block_given?
       @admin.createTable(table_description)

From e0a141184f89b818d2e2d2d09275dc4ca5c22982 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 8 Apr 2013 19:48:21 +0000
Subject: [PATCH 0917/1540] HBASE-8158. Backport HBASE-8140
 "TableMapReduceUtils#addDependencyJar fails when nested inside another MR
 job" (Nick Dimiduk)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465743 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TableMapReduceUtil.java   | 125 +++++++------
 .../mapreduce/hadoopbackport/JarFinder.java   | 170 ++++++++++++++++++
 .../hadoopbackport/TestJarFinder.java         | 132 ++++++++++++++
 3 files changed, 372 insertions(+), 55 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 53df3bd18929..ceacb7182977 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -24,8 +24,8 @@
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
-import java.lang.reflect.Method;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.ArrayList;
@@ -39,11 +39,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -568,43 +569,31 @@ public static void addDependencyJars(Configuration conf,
   }
 
   /**
-   * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop),
-   * finds the Jar for a class or creates it if it doesn't exist. If
-   * the class is in a directory in the classpath, it creates a Jar
-   * on the fly with the contents of the directory and returns the path
-   * to that Jar. If a Jar is created, it is created in
-   * the system temporary directory.
-   *
-   * Otherwise, returns an existing jar that contains a class of the
-   * same name.
-   *
+   * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
+   * the Jar for a class or creates it if it doesn't exist. If the class is in
+   * a directory in the classpath, it creates a Jar on the fly with the
+   * contents of the directory and returns the path to that Jar. If a Jar is
+   * created, it is created in the system temporary directory. Otherwise,
+   * returns an existing jar that contains a class of the same name.
    * @param my_class the class to find.
-   * @return a jar file that contains the class, or null.
+   * @return a jar file that contains the class.
    * @throws IOException
    */
-  private static String findOrCreateJar(Class my_class)
+  private static String findOrCreateJar(Class<?> my_class)
   throws IOException {
-    try {
-      Class<?> jarFinder = Class.forName("org.apache.hadoop.util.JarFinder");
-      // hadoop-0.23 has a JarFinder class that will create the jar
-      // if it doesn't exist.  Note that this is needed to run the mapreduce
-      // unit tests post-0.23, because mapreduce v2 requires the relevant jars
-      // to be in the mr cluster to do output, split, etc.  At unit test time,
-      // the hbase jars do not exist, so we need to create some.  Note that we
-      // can safely fall back to findContainingJars for pre-0.23 mapreduce.
-      Method m = jarFinder.getMethod("getJar", Class.class);
-      return (String)m.invoke(null,my_class);
-    } catch (InvocationTargetException ite) {
-      // function was properly called, but threw it's own exception
-      throw new IOException(ite.getCause());
-    } catch (Exception e) {
-      // ignore all other exceptions. related to reflection failure
-  }
+    // attempt to locate an existing jar for the class.
+    String jar = findContainingJar(my_class);
+    if (null == jar || jar.isEmpty()) {
+      jar = getJar(my_class);
+    }
 
-  LOG.debug("New JarFinder: org.apache.hadoop.util.JarFinder.getJar " +
-	"not available.  Using old findContainingJar");
-  return findContainingJar(my_class);
-}
+    if (null == jar || jar.isEmpty()) {
+      throw new IOException("Cannot locate resource for class " + my_class.getName());
+    }
+
+    LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
+    return jar;
+  }
 
   /**
    * Find a jar that contains a class of the same name, if any.
@@ -617,34 +606,60 @@ private static String findOrCreateJar(Class my_class)
    * @return a jar file that contains the class, or null.
    * @throws IOException
    */
-  private static String findContainingJar(Class my_class) {
+  private static String findContainingJar(Class<?> my_class) throws IOException {
     ClassLoader loader = my_class.getClassLoader();
     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
-    try {
-      for(Enumeration itr = loader.getResources(class_file);
-          itr.hasMoreElements();) {
-        URL url = (URL) itr.nextElement();
-        if ("jar".equals(url.getProtocol())) {
-          String toReturn = url.getPath();
-          if (toReturn.startsWith("file:")) {
-            toReturn = toReturn.substring("file:".length());
-          }
-          // URLDecoder is a misnamed class, since it actually decodes
-          // x-www-form-urlencoded MIME type rather than actual
-          // URL encoding (which the file path has). Therefore it would
-          // decode +s to ' 's which is incorrect (spaces are actually
-          // either unencoded or encoded as "%20"). Replace +s first, so
-          // that they are kept sacred during the decoding process.
-          toReturn = toReturn.replaceAll("\\+", "%2B");
-          toReturn = URLDecoder.decode(toReturn, "UTF-8");
-          return toReturn.replaceAll("!.*$", "");
+    for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
+      URL url = itr.nextElement();
+      if ("jar".equals(url.getProtocol())) {
+        String toReturn = url.getPath();
+        if (toReturn.startsWith("file:")) {
+          toReturn = toReturn.substring("file:".length());
         }
+        // URLDecoder is a misnamed class, since it actually decodes
+        // x-www-form-urlencoded MIME type rather than actual
+        // URL encoding (which the file path has). Therefore it would
+        // decode +s to ' 's which is incorrect (spaces are actually
+        // either unencoded or encoded as "%20"). Replace +s first, so
+        // that they are kept sacred during the decoding process.
+        toReturn = toReturn.replaceAll("\\+", "%2B");
+        toReturn = URLDecoder.decode(toReturn, "UTF-8");
+        return toReturn.replaceAll("!.*$", "");
       }
-    } catch (IOException e) {
-      throw new RuntimeException(e);
     }
     return null;
   }
 
+  /**
+   * Invoke 'getJar' on a JarFinder implementation. Useful for some job configuration
+   * contexts (HBASE-8140) and also for testing on MRv2. First check if we have
+   * HADOOP-9426. Lacking that, fall back to the backport.
+   *
+   * @param my_class the class to find.
+   * @return a jar file that contains the class, or null.
+   */
+  private static String getJar(Class<?> my_class) {
+    String ret = null;
+    String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
+    Class<?> jarFinder = null;
+    try {
+      LOG.debug("Looking for " + hadoopJarFinder + ".");
+      jarFinder = Class.forName(hadoopJarFinder);
+      LOG.debug(hadoopJarFinder + " found.");
+      Method getJar = jarFinder.getMethod("getJar", Class.class);
+      ret = (String) getJar.invoke(null, my_class);
+    } catch (ClassNotFoundException e) {
+      LOG.debug("Using backported JarFinder.");
+      ret = JarFinder.getJar(my_class);
+    } catch (InvocationTargetException e) {
+      // function was properly called, but threw it's own exception. Unwrap it
+      // and pass it on.
+      throw new RuntimeException(e.getCause());
+    } catch (Exception e) {
+      // toss all other exceptions, related to reflection failure
+      throw new RuntimeException("getJar invocation failed.", e);
+    }
 
+    return ret;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
new file mode 100644
index 000000000000..14e7949609de
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import com.google.common.base.Preconditions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.MessageFormat;
+import java.util.Enumeration;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * Finds the Jar for a class. If the class is in a directory in the
+ * classpath, it creates a Jar on the fly with the contents of the directory
+ * and returns the path to that Jar. If a Jar is created, it is created in
+ * the system temporary directory.
+ * 
+ * This file was forked from hadoop/common/branches/branch-2@1377176.
+ */
+public class JarFinder {
+
+  private static void copyToZipStream(InputStream is, ZipEntry entry,
+                              ZipOutputStream zos) throws IOException {
+    zos.putNextEntry(entry);
+    byte[] arr = new byte[4096];
+    int read = is.read(arr);
+    while (read > -1) {
+      zos.write(arr, 0, read);
+      read = is.read(arr);
+    }
+    is.close();
+    zos.closeEntry();
+  }
+
+  public static void jarDir(File dir, String relativePath, ZipOutputStream zos)
+    throws IOException {
+    Preconditions.checkNotNull(relativePath, "relativePath");
+    Preconditions.checkNotNull(zos, "zos");
+
+    // by JAR spec, if there is a manifest, it must be the first entry in the
+    // ZIP.
+    File manifestFile = new File(dir, JarFile.MANIFEST_NAME);
+    ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+    if (!manifestFile.exists()) {
+      zos.putNextEntry(manifestEntry);
+      new Manifest().write(new BufferedOutputStream(zos));
+      zos.closeEntry();
+    } else {
+      InputStream is = new FileInputStream(manifestFile);
+      copyToZipStream(is, manifestEntry, zos);
+    }
+    zos.closeEntry();
+    zipDir(dir, relativePath, zos, true);
+    zos.close();
+  }
+
+  private static void zipDir(File dir, String relativePath, ZipOutputStream zos,
+                             boolean start) throws IOException {
+    String[] dirList = dir.list();
+    for (String aDirList : dirList) {
+      File f = new File(dir, aDirList);
+      if (!f.isHidden()) {
+        if (f.isDirectory()) {
+          if (!start) {
+            ZipEntry dirEntry = new ZipEntry(relativePath + f.getName() + "/");
+            zos.putNextEntry(dirEntry);
+            zos.closeEntry();
+          }
+          String filePath = f.getPath();
+          File file = new File(filePath);
+          zipDir(file, relativePath + f.getName() + "/", zos, false);
+        }
+        else {
+          String path = relativePath + f.getName();
+          if (!path.equals(JarFile.MANIFEST_NAME)) {
+            ZipEntry anEntry = new ZipEntry(path);
+            InputStream is = new FileInputStream(f);
+            copyToZipStream(is, anEntry, zos);
+          }
+        }
+      }
+    }
+  }
+
+  private static void createJar(File dir, File jarFile) throws IOException {
+    Preconditions.checkNotNull(dir, "dir");
+    Preconditions.checkNotNull(jarFile, "jarFile");
+    File jarDir = jarFile.getParentFile();
+    if (!jarDir.exists()) {
+      if (!jarDir.mkdirs()) {
+        throw new IOException(MessageFormat.format("could not create dir [{0}]",
+                                                   jarDir));
+      }
+    }
+    JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile));
+    jarDir(dir, "", zos);
+  }
+
+  /**
+   * Returns the full path to the Jar containing the class. It always return a
+   * JAR.
+   *
+   * @param klass class.
+   *
+   * @return path to the Jar containing the class.
+   */
+  public static String getJar(Class klass) {
+    Preconditions.checkNotNull(klass, "klass");
+    ClassLoader loader = klass.getClassLoader();
+    if (loader != null) {
+      String class_file = klass.getName().replaceAll("\\.", "/") + ".class";
+      try {
+        for (Enumeration itr = loader.getResources(class_file);
+             itr.hasMoreElements(); ) {
+          URL url = (URL) itr.nextElement();
+          String path = url.getPath();
+          if (path.startsWith("file:")) {
+            path = path.substring("file:".length());
+          }
+          path = URLDecoder.decode(path, "UTF-8");
+          if ("jar".equals(url.getProtocol())) {
+            path = URLDecoder.decode(path, "UTF-8");
+            return path.replaceAll("!.*$", "");
+          }
+          else if ("file".equals(url.getProtocol())) {
+            String klassName = klass.getName();
+            klassName = klassName.replace(".", "/") + ".class";
+            path = path.substring(0, path.length() - klassName.length());
+            File baseDir = new File(path);
+            File testDir = new File(System.getProperty("test.build.dir", "target/test-dir"));
+            testDir = testDir.getAbsoluteFile();
+            if (!testDir.exists()) {
+              testDir.mkdirs();
+            }
+            File tempJar = File.createTempFile("hadoop-", "", testDir);
+            tempJar = new File(tempJar.getAbsolutePath() + ".jar");
+            createJar(baseDir, tempJar);
+            return tempJar.getAbsolutePath();
+          }
+        }
+      }
+      catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return null;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java
new file mode 100644
index 000000000000..fb56993c27a1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.text.MessageFormat;
+import java.util.Properties;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+/**
+ * This file was forked from hadoop/common/branches/branch-2@1350012.
+ */
+@Category(SmallTests.class)
+public class TestJarFinder {
+
+  @Test
+  public void testJar() throws Exception {
+
+    //picking a class that is for sure in a JAR in the classpath
+    String jar = JarFinder.getJar(LogFactory.class);
+    Assert.assertTrue(new File(jar).exists());
+  }
+
+  private static void delete(File file) throws IOException {
+    if (file.getAbsolutePath().length() < 5) {
+      throw new IllegalArgumentException(
+        MessageFormat.format("Path [{0}] is too short, not deleting",
+                             file.getAbsolutePath()));
+    }
+    if (file.exists()) {
+      if (file.isDirectory()) {
+        File[] children = file.listFiles();
+        if (children != null) {
+          for (File child : children) {
+            delete(child);
+          }
+        }
+      }
+      if (!file.delete()) {
+        throw new RuntimeException(
+          MessageFormat.format("Could not delete path [{0}]",
+                               file.getAbsolutePath()));
+      }
+    }
+  }
+
+  @Test
+  public void testExpandedClasspath() throws Exception {
+    //picking a class that is for sure in a directory in the classpath
+    //in this case the JAR is created on the fly
+    String jar = JarFinder.getJar(TestJarFinder.class);
+    Assert.assertTrue(new File(jar).exists());
+  }
+
+  @Test
+  public void testExistingManifest() throws Exception {
+    File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+                        TestJarFinder.class.getName() + "-testExistingManifest");
+    delete(dir);
+    dir.mkdirs();
+
+    File metaInfDir = new File(dir, "META-INF");
+    metaInfDir.mkdirs();
+    File manifestFile = new File(metaInfDir, "MANIFEST.MF");
+    Manifest manifest = new Manifest();
+    OutputStream os = new FileOutputStream(manifestFile);
+    manifest.write(os);
+    os.close();
+
+    File propsFile = new File(dir, "props.properties");
+    Writer writer = new FileWriter(propsFile);
+    new Properties().store(writer, "");
+    writer.close();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    JarOutputStream zos = new JarOutputStream(baos);
+    JarFinder.jarDir(dir, "", zos);
+    JarInputStream jis =
+      new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
+    Assert.assertNotNull(jis.getManifest());
+    jis.close();
+  }
+
+  @Test
+  public void testNoManifest() throws Exception {
+    File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+                        TestJarFinder.class.getName() + "-testNoManifest");
+    delete(dir);
+    dir.mkdirs();
+    File propsFile = new File(dir, "props.properties");
+    Writer writer = new FileWriter(propsFile);
+    new Properties().store(writer, "");
+    writer.close();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    JarOutputStream zos = new JarOutputStream(baos);
+    JarFinder.jarDir(dir, "", zos);
+    JarInputStream jis =
+      new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
+    Assert.assertNotNull(jis.getManifest());
+    jis.close();
+  }
+}

From 51a17c53ed4e96eca09df09f6d477d16d3a27184 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 8 Apr 2013 19:51:34 +0000
Subject: [PATCH 0918/1540] HBASE-8288 HBaseFileSystem: Refactoring and correct
 semantics for createPath methods (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465747 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HBaseFileSystem.java  | 81 +++++++++----------
 .../hadoop/hbase/backup/HFileArchiver.java    | 19 +++--
 .../org/apache/hadoop/hbase/io/Reference.java |  2 +-
 .../hadoop/hbase/master/MasterFileSystem.java | 18 ++---
 .../hadoop/hbase/master/SplitLogManager.java  |  2 +-
 .../hbase/master/cleaner/CleanerChore.java    |  8 +-
 .../master/handler/CreateTableHandler.java    |  2 +-
 .../master/handler/DeleteTableHandler.java    |  2 +-
 .../hadoop/hbase/regionserver/HRegion.java    | 18 ++---
 .../hbase/regionserver/HRegionFileSystem.java |  2 +-
 .../hbase/regionserver/SplitTransaction.java  |  6 +-
 .../hadoop/hbase/regionserver/Store.java      |  6 +-
 .../hadoop/hbase/regionserver/StoreFile.java  |  6 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   | 12 +--
 .../regionserver/wal/HLogFileSystem.java      |  2 +-
 .../hbase/regionserver/wal/HLogSplitter.java  | 28 +++----
 .../hadoop/hbase/util/FSTableDescriptors.java |  8 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java |  4 +-
 .../hadoop/hbase/zookeeper/ZKSplitLog.java    |  2 +-
 .../hadoop/hbase/HBaseTestingUtility.java     |  1 +
 .../hadoop/hbase/TestHBaseFileSystem.java     | 21 ++++-
 21 files changed, 127 insertions(+), 123 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
index eebac4d790b4..a1ae5b0f4bc6 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
@@ -42,25 +42,28 @@ public abstract class HBaseFileSystem {
    * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
    * client level.
    */
-  protected static  int hdfsClientRetriesNumber;
+  protected static int hdfsClientRetriesNumber;
   private static int baseSleepBeforeRetries;
   private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
   private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
+  // This static block is added for performance reasons. This is to ensure we are not checking
+  // in the method calls whether retry properties are set or not. Refer to HBase-8288 for more 
+  // context.
+  static {
+    setRetryCounts(HBaseConfiguration.create());
+  }
 
-  
   /**
    * Deletes a file. Assumes the user has already checked for this directory existence.
-   * @param dir
    * @param fs
-   * @param conf
+   * @param dir
    * @return true if the directory is deleted.
    * @throws IOException
    */
-  public static boolean deleteFileFromFileSystem(FileSystem fs, Configuration conf, Path dir)
+  public static boolean deleteFileFromFileSystem(FileSystem fs, Path dir)
       throws IOException {
     IOException lastIOE = null;
     int i = 0;
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.delete(dir, false);
@@ -77,17 +80,15 @@ public static boolean deleteFileFromFileSystem(FileSystem fs, Configuration conf
   
   /**
    * Deletes a directory. Assumes the user has already checked for this directory existence.
-   * @param dir
    * @param fs
-   * @param conf
+   * @param dir
    * @return true if the directory is deleted.
    * @throws IOException
    */
-  public static boolean deleteDirFromFileSystem(FileSystem fs, Configuration conf, Path dir)
+  public static boolean deleteDirFromFileSystem(FileSystem fs, Path dir)
       throws IOException {
     IOException lastIOE = null;
     int i = 0;
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.delete(dir, true);
@@ -101,30 +102,26 @@ public static boolean deleteDirFromFileSystem(FileSystem fs, Configuration conf,
     throw new IOException("Exception in deleteDirFromFileSystem", lastIOE);
   }
 
-  protected static void checkAndSetRetryCounts(Configuration conf) {
-    if (hdfsClientRetriesNumber == 0) {
+  protected static void setRetryCounts(Configuration conf) {
       hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
         DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
       baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
         DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
-    }
   }
   
   /**
    * Creates a directory for a filesystem and configuration object. Assumes the user has already
    * checked for this directory existence.
    * @param fs
-   * @param conf
    * @param dir
    * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
    *         whether the directory exists or not, and returns true if it exists.
    * @throws IOException
    */
-  public static boolean makeDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
+  public static boolean makeDirOnFileSystem(FileSystem fs, Path dir)
       throws IOException {
     int i = 0;
     IOException lastIOE = null;
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.mkdirs(dir);
@@ -139,18 +136,16 @@ public static boolean makeDirOnFileSystem(FileSystem fs, Configuration conf, Pat
   
   /**
    * Renames a directory. Assumes the user has already checked for this directory existence.
-   * @param src
    * @param fs
+   * @param src
    * @param dst
-   * @param conf
    * @return true if the directory is renamed.
    * @throws IOException
    */
-  public static boolean renameDirForFileSystem(FileSystem fs, Configuration conf, Path src, Path dst)
+  public static boolean renameDirForFileSystem(FileSystem fs, Path src, Path dst)
       throws IOException {
     IOException lastIOE = null;
     int i = 0;
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.rename(src, dst);
@@ -163,30 +158,29 @@ public static boolean renameDirForFileSystem(FileSystem fs, Configuration conf,
     } while (++i <= hdfsClientRetriesNumber);
     throw new IOException("Exception in renameDirForFileSystem", lastIOE);
   }
-  
-/**
- * Creates a path on the file system. Checks whether the path exists already or not, and use it
- * for retrying in case underlying fs throws an exception.
- * @param fs
- * @param conf
- * @param dir
- * @param overwrite
- * @return
- * @throws IOException
- */
-  public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Configuration conf, Path dir,
+
+  /**
+   * Creates a path on the file system. Checks whether the path exists already or not, and use it
+   * for retrying in case underlying fs throws an exception. 
+   * If the dir already exists and overwrite flag is false, the underlying FileSystem throws
+   *  an IOE. It is not retried and the IOE is re-thrown to the caller.
+   * @param fs
+   * @param dir
+   * @param overwrite
+   * @return
+   * @throws IOException
+   */
+  public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Path dir,
       boolean overwrite) throws IOException {
     int i = 0;
     boolean existsBefore = fs.exists(dir);
     IOException lastIOE = null;
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.create(dir, overwrite);
       } catch (IOException ioe) {
         lastIOE = ioe;
-        // directory is present, don't overwrite
-        if (!existsBefore && fs.exists(dir)) return fs.create(dir, false);
+        if (existsBefore && !overwrite) throw ioe;// a legitimate exception
         sleepBeforeRetry("Create Path", i + 1);
       }
     } while (++i <= hdfsClientRetriesNumber);
@@ -194,29 +188,28 @@ public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Configura
   }
 
   /**
-   * Creates the specified file with the given permission.
+   * Creates the specified file with the given permission. 
+   * If the dir already exists and the overwrite flag is false, underlying FileSystem throws
+   * an IOE. It is not retried and the IOE is re-thrown to the caller.
    * @param fs
    * @param path
    * @param perm
    * @param overwrite
    * @return
-   * @throws IOException 
+   * @throws IOException
    */
   public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
-      Configuration conf, Path path, FsPermission perm, boolean overwrite) throws IOException {
+      Path path, FsPermission perm, boolean overwrite) throws IOException {
     int i = 0;
     IOException lastIOE = null;
     boolean existsBefore = fs.exists(path);
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.create(path, perm, overwrite, fs.getConf().getInt("io.file.buffer.size", 4096),
           fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
       } catch (IOException ioe) {
         lastIOE = ioe;
-        // path is present now, don't overwrite
-        if (!existsBefore && fs.exists(path)) return fs.create(path, false);
-        // if it existed before let's retry in case we get IOE, as we can't rely on fs.exists()
+        if (existsBefore && !overwrite) throw ioe;// a legitimate exception
         sleepBeforeRetry("Create Path with Perms", i + 1);
       }
     } while (++i <= hdfsClientRetriesNumber);
@@ -226,16 +219,14 @@ public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
 /**
  * Creates the file. Assumes the user has already checked for this file existence.
  * @param fs
- * @param conf
  * @param dir
  * @return result true if the file is created with this call, false otherwise.
  * @throws IOException
  */
-  public static boolean createNewFileOnFileSystem(FileSystem fs, Configuration conf, Path file)
+  public static boolean createNewFileOnFileSystem(FileSystem fs, Path file)
       throws IOException {
     int i = 0;
     IOException lastIOE = null;
-    checkAndSetRetryCounts(conf);
     do {
       try {
         return fs.createNewFile(file);
diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index 808b1f85c207..8a14b1116b80 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -217,7 +217,7 @@ public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegion
     Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family);
 
     // make sure we don't archive if we can't and that the archive dir exists
-    if (!HBaseFileSystem.makeDirOnFileSystem(fs, conf, storeArchiveDir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, storeArchiveDir)) {
       throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
           + Bytes.toString(family) + ", deleting compacted files instead.");
     }
@@ -251,7 +251,7 @@ public static void archiveStoreFile(FileSystem fs, HRegionInfo regionInfo,
       Configuration conf, Path tableDir, byte[] family, Path storeFile) throws IOException {
     Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
     // make sure we don't archive if we can't and that the archive dir exists
-    if (!HBaseFileSystem.makeDirOnFileSystem(fs, conf, storeArchiveDir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, storeArchiveDir)) {
       throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
           + Bytes.toString(family) + ", deleting compacted files instead.");
     }
@@ -319,7 +319,7 @@ private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
 
     // make sure the archive directory exists
     if (!fs.exists(baseArchiveDir)) {
-      if (!HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), baseArchiveDir)) {
+      if (!HBaseFileSystem.makeDirOnFileSystem(fs, baseArchiveDir)) {
         throw new IOException("Failed to create the archive directory:" + baseArchiveDir
             + ", quitting archive attempt.");
       }
@@ -386,12 +386,11 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
 
       // move the archive file to the stamped backup
       Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), archiveFile,
-        backedupArchiveFile)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, archiveFile, backedupArchiveFile)) {
         LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
             + ", deleting existing file in favor of newer.");
         // try to delete the exisiting file, if we can't rename it
-        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, fs.getConf(), archiveFile)) {
+        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, archiveFile)) {
           throw new IOException("Couldn't delete existing archive file (" + archiveFile
               + ") or rename it to the backup file (" + backedupArchiveFile
               + ") to make room for similarly named file.");
@@ -413,7 +412,7 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
         // (we're in a retry loop, so don't worry too much about the exception)
         try {
           if (!fs.exists(archiveDir)
-              && HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), archiveDir)) {
+              && HBaseFileSystem.makeDirOnFileSystem(fs, archiveDir)) {
             LOG.debug("Created archive directory:" + archiveDir);
           }
         } catch (IOException e) {
@@ -477,7 +476,7 @@ private static void deleteFilesWithoutArchiving(Collection<File> files) throws I
    */
   private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
       throws IOException {
-    if (HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), regionDir)) {
+    if (HBaseFileSystem.deleteDirFromFileSystem(fs, regionDir)) {
       LOG.debug("Deleted all region files in: " + regionDir);
       return true;
     }
@@ -611,7 +610,7 @@ public File(FileSystem fs) {
     public boolean moveAndClose(Path dest) throws IOException {
       this.close();
       Path p = this.getPath();
-      return HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), p, dest);
+      return HBaseFileSystem.renameDirForFileSystem(fs, p, dest);
     }
 
     /**
@@ -642,7 +641,7 @@ public FileablePath(FileSystem fs, Path file) {
 
     @Override
     public void delete() throws IOException {
-      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), file)) 
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, file)) 
         throw new IOException("Failed to delete:" + this.file);
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
index 1fcea842fa4f..06dc5041ba25 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/Reference.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -126,7 +126,7 @@ public static boolean isTopFileRegion(final Range r) {
 
   public Path write(final FileSystem fs, final Path p)
   throws IOException {
-    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, fs.getConf(), p, false);
+    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, p, false);
     try {
       write(out);
     } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 32abd2302b6b..7dd33b4e8cf6 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -153,7 +153,7 @@ private Path createInitialFileSystemLayout() throws IOException {
 
     // Make sure the region servers can archive their old logs
     if(!this.fs.exists(oldLogDir)) {
-      HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir);
+      HBaseFileSystem.makeDirOnFileSystem(fs, oldLogDir);
     }
 
     return oldLogDir;
@@ -323,7 +323,7 @@ private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOExcep
       Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
       // rename the directory so a rogue RS doesn't create more HLogs
       if (fs.exists(logDir)) {
-        if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, logDir, splitDir)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, logDir, splitDir)) {
           throw new IOException("Failed fs.rename for log split: " + logDir);
         }
         logDir = splitDir;
@@ -416,7 +416,7 @@ private Path checkRootDir(final Path rd, final Configuration c,
     // Filesystem is good. Go ahead and check for hbase.rootdir.
     try {
       if (!fs.exists(rd)) {
-        HBaseFileSystem.makeDirOnFileSystem(fs, c, rd);
+        HBaseFileSystem.makeDirOnFileSystem(fs, rd);
         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
         // We used to handle this by checking the current DN count and waiting until
         // it is nonzero. With security, the check for datanode count doesn't work --
@@ -480,13 +480,13 @@ private void checkTempDir(final Path tmpdir, final Configuration c, final FileSy
           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
         }
       }
-      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, c, tmpdir)) {
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, tmpdir)) {
         throw new IOException("Unable to clean the temp directory: " + tmpdir);
       }
     }
 
     // Create the temp directory
-    if (!HBaseFileSystem.makeDirOnFileSystem(fs, c, tmpdir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, tmpdir)) {
       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
     }
   }
@@ -554,7 +554,7 @@ public void deleteRegion(HRegionInfo region) throws IOException {
   }
 
   public void deleteTable(byte[] tableName) throws IOException {
-    HBaseFileSystem.deleteDirFromFileSystem(fs, conf, new Path(rootdir, Bytes.toString(tableName)));
+    HBaseFileSystem.deleteDirFromFileSystem(fs, new Path(rootdir, Bytes.toString(tableName)));
   }
 
   /**
@@ -567,11 +567,11 @@ public Path moveToTemp(final Path path) throws IOException {
     Path tempPath = new Path(this.tempdir, path.getName());
 
     // Ensure temp exists
-    if (!fs.exists(tempdir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, tempdir)) {
+    if (!fs.exists(tempdir) && !HBaseFileSystem.makeDirOnFileSystem(fs, tempdir)) {
       throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
     }
 
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, path, tempPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, path, tempPath)) {
       throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
     }
 
@@ -603,7 +603,7 @@ public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
     // delete the family folder
     Path familyDir = new Path(tableDir,
       new Path(region.getEncodedName(), Bytes.toString(familyName)));
-    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, conf, familyDir)) {
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, familyDir)) {
       throw new IOException("Could not delete family "
           + Bytes.toString(familyName) + " from FileSystem for region "
           + region.getRegionNameAsString() + "(" + region.getEncodedName()
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 2d6881d56365..3c9fcc5b75a1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -301,7 +301,7 @@ public long splitLogDistributed(final List<Path> logDirs, PathFilter filter)
     for(Path logDir: logDirs){
       status.setStatus("Cleaning up log directory...");
       try {
-        if (fs.exists(logDir) && !HBaseFileSystem.deleteFileFromFileSystem(fs, conf, logDir)) {
+        if (fs.exists(logDir) && !HBaseFileSystem.deleteFileFromFileSystem(fs, logDir)) {
           LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);
         }
       } catch (IOException ioe) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 6d6da76c1b5c..0fa53c045389 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -154,7 +154,7 @@ public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
     // if the directory doesn't exist, then we are done
     if (children == null) {
       try {
-        return HBaseFileSystem.deleteFileFromFileSystem(fs, conf, toCheck);
+        return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
       } catch (IOException e) {
         if (LOG.isTraceEnabled()) {
           LOG.trace("Couldn't delete directory: " + toCheck, e);
@@ -186,7 +186,7 @@ else if (!checkAndDelete(path)) {
     // delete this directory. However, don't do so recursively so we don't delete files that have
     // been added since we last checked.
     try {
-      return HBaseFileSystem.deleteFileFromFileSystem(fs, conf, toCheck);
+      return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
     } catch (IOException e) {
       if (LOG.isTraceEnabled()) {
         LOG.trace("Couldn't delete directory: " + toCheck, e);
@@ -208,7 +208,7 @@ private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumen
     // first check to see if the path is valid
     if (!validate(filePath)) {
       LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
-      boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, conf, filePath);
+      boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, filePath);
       if (!success) LOG.warn("Attempted to delete:" + filePath
           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
 
@@ -234,7 +234,7 @@ private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumen
     if (LOG.isTraceEnabled()) {
       LOG.trace("Removing:" + filePath + " from archive");
     }
-    boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, conf, filePath);
+    boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, filePath);
     if (!success) {
       LOG.warn("Attempted to delete:" + filePath
           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 8557a7f22f40..e7b4487f4b80 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -175,7 +175,7 @@ private void handleCreateTable(String tableName) throws IOException, KeeperExcep
     List<HRegionInfo> regionInfos = handleCreateHdfsRegions(tempdir, tableName);
 
     // 3. Move Table temp directory to the hbase root location
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, tempTableDir, tableDir)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, tempTableDir, tableDir)) {
       throw new IOException("Unable to move table from temp=" + tempTableDir +
         " to hbase root=" + tableDir);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index 7c9219309ff4..2cb76f3c31c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -90,7 +90,7 @@ protected void handleTableOperation(List<HRegionInfo> regions)
       }
 
       // 5. Delete table from FS (temp directory)
-      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), tempTableDir)) {
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, tempTableDir)) {
         LOG.error("Couldn't delete " + tempTableDir);
       }
     } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 5cdd90d9ba5d..590c70db9422 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -681,7 +681,7 @@ static void moveInitialFilesIntoPlace(final FileSystem fs,
     final Path initialFiles, final Path regiondir)
   throws IOException {
     if (initialFiles != null && fs.exists(initialFiles)) {
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), initialFiles, regiondir)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, initialFiles, regiondir)) {
         LOG.warn("Unable to rename " + initialFiles + " to " + regiondir);
       }
     }
@@ -843,7 +843,7 @@ public static void writeRegioninfoOnFilesystem(HRegionInfo regionInfo, Path regi
     } finally {
       out.close();
     }
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, tmpPath, regioninfoPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, tmpPath, regioninfoPath)) {
       throw new IOException("Unable to rename " + tmpPath + " to " +
         regioninfoPath);
     }
@@ -2704,7 +2704,7 @@ public void addRegionToSnapshot(SnapshotDescription desc,
         // open and read the files as well).
         LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
         Path referenceFile = new Path(dstStoreDir, file.getName());
-        boolean success = HBaseFileSystem.createNewFileOnFileSystem(fs, conf, referenceFile);
+        boolean success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
         if (!success) {
           throw new IOException("Failed to create reference file:" + referenceFile);
         }
@@ -3113,7 +3113,7 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
     }
     // Now delete the content of recovered edits.  We're done w/ them.
     for (Path file: files) {
-      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, file)) {
+      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, file)) {
         LOG.error("Failed delete of " + file);
       } else {
         LOG.debug("Deleted recovered.edits file=" + file);
@@ -3305,7 +3305,7 @@ private static boolean isZeroLengthThenDelete(final FileSystem fs, final Path p)
     FileStatus stat = fs.getFileStatus(p);
     if (stat.getLen() > 0) return false;
     LOG.warn("File " + p + " is zero-length, deleting.");
-    HBaseFileSystem.deleteFileFromFileSystem(fs, fs.getConf(), p);
+    HBaseFileSystem.deleteFileFromFileSystem(fs, p);
     return true;
   }
 
@@ -4228,7 +4228,7 @@ public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
         HTableDescriptor.getTableDir(rootDir, info.getTableName());
     Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
     FileSystem fs = FileSystem.get(conf);
-    HBaseFileSystem.makeDirOnFileSystem(fs, conf, regionDir);
+    HBaseFileSystem.makeDirOnFileSystem(fs, regionDir);
     // Write HRI to a file in case we need to recover .META.
     writeRegioninfoOnFilesystem(info, regionDir, fs, conf);
     HLog effectiveHLog = hlog;
@@ -4424,7 +4424,7 @@ private static void deleteRegion(FileSystem fs, Path regiondir)
     if (LOG.isDebugEnabled()) {
       LOG.debug("DELETING region " + regiondir.toString());
     }
-    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), regiondir)) {
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, regiondir)) {
       LOG.warn("Failed delete of " + regiondir);
     }
   }
@@ -4470,7 +4470,7 @@ public static void makeColumnFamilyDirs(FileSystem fs, Path tabledir,
     final HRegionInfo hri, byte [] colFamily)
   throws IOException {
     Path dir = Store.getStoreHomedir(tabledir, hri.getEncodedName(), colFamily);
-    if (!HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), dir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, dir)) {
       LOG.warn("Failed to create " + dir);
     }
   }
@@ -4580,7 +4580,7 @@ public static HRegion merge(HRegion a, HRegion b)
       throw new IOException("Cannot merge; target file collision at " +
           newRegionDir);
     }
-    HBaseFileSystem.makeDirOnFileSystem(fs, conf, newRegionDir);
+    HBaseFileSystem.makeDirOnFileSystem(fs, newRegionDir);
 
     LOG.info("starting merge of regions: " + a + " and " + b +
       " into new region " + newRegionInfo.toString() +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index b2a7a6f89580..fe4cb6808402 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -33,6 +33,6 @@ public class HRegionFileSystem extends HBaseFileSystem {
   public static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
 
   public HRegionFileSystem(Configuration conf) {
-    checkAndSetRetryCounts(conf);
+    setRetryCounts(conf);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 18234734b772..113e9d927db3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -552,12 +552,12 @@ void createSplitDir(final FileSystem fs, final Path splitdir)
     if (fs.exists(splitdir)) {
       LOG.info("The " + splitdir
           + " directory exists.  Hence deleting it to recreate it");
-      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), splitdir)) {
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, splitdir)) {
         throw new IOException("Failed deletion of " + splitdir
             + " before creating them again.");
       }
     }
-    if (!HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), splitdir))
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, splitdir))
         throw new IOException("Failed create of " + splitdir);
   }
 
@@ -579,7 +579,7 @@ private static void deleteDir(final FileSystem fs, final Path dir,
   throws IOException {
     if (!fs.exists(dir)) {
       if (mustPreExist) throw new IOException(dir.toString() + " does not exist!");
-    } else if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), dir)) {
+    } else if (!HBaseFileSystem.deleteDirFromFileSystem(fs, dir)) {
       throw new IOException("Failed delete of " + dir);
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 88b0267e9082..ab5e1d56fde5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -289,7 +289,7 @@ long getTTL(final HColumnDescriptor family) {
    */
   Path createStoreHomeDir(final FileSystem fs,
       final Path homedir) throws IOException {
-    if (!fs.exists(homedir) && !HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), homedir)) {
+    if (!fs.exists(homedir) && !HBaseFileSystem.makeDirOnFileSystem(fs, homedir)) {
         throw new IOException("Failed create of: " + homedir.toString());
     }
     return homedir;
@@ -879,7 +879,7 @@ private StoreFile commitFile(final Path path,
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, path, dstPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, path, dstPath)) {
       LOG.warn("Unable to rename " + path + " to " + dstPath);
     }
 
@@ -1649,7 +1649,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
       Path origPath = compactedFile.getPath();
       Path destPath = new Path(homedir, origPath.getName());
       LOG.info("Renaming compacted file at " + origPath + " to " + destPath);
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, origPath, destPath)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, origPath, destPath)) {
         LOG.error("Failed move of compacted file " + origPath + " to " +
             destPath);
         throw new IOException("Failed move of compacted file " + origPath +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index cdaabb7501f3..4c3383be3774 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -675,7 +675,7 @@ public synchronized void closeReader(boolean evictOnClose)
    */
   public void deleteReader() throws IOException {
     closeReader(true);
-    HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), getPath());
+    HBaseFileSystem.deleteDirFromFileSystem(fs, getPath());
   }
 
   @Override
@@ -718,7 +718,7 @@ public static Path rename(final FileSystem fs,
     if (!fs.exists(src)) {
       throw new FileNotFoundException(src.toString());
     }
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), src, tgt)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, src, tgt)) {
       throw new IOException("Failed rename of " + src + " to " + tgt);
     }
     return tgt;
@@ -851,7 +851,7 @@ public Writer build() throws IOException {
       }
 
       if (!fs.exists(dir)) {
-        HBaseFileSystem.makeDirOnFileSystem(fs, conf, dir);
+        HBaseFileSystem.makeDirOnFileSystem(fs, dir);
       }
 
       if (filePath == null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 8c4df449fd66..7fbdb6475ff1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -416,11 +416,11 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     if (failIfLogDirExists && (dirExists = this.fs.exists(dir))) {
       throw new IOException("Target HLog directory already exists: " + dir);
     }
-    if (!dirExists && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, dir)) {
+    if (!dirExists && !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) {
       throw new IOException("Unable to mkdir " + dir);
     }
     this.oldLogDir = oldLogDir;
-    if (!fs.exists(oldLogDir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir)) {
+    if (!fs.exists(oldLogDir) && !HBaseFileSystem.makeDirOnFileSystem(fs, oldLogDir)) {
       throw new IOException("Unable to mkdir " + this.oldLogDir);
     }
     this.forMeta = forMeta;
@@ -931,7 +931,7 @@ private void archiveLogFile(final Path p, final Long seqno) throws IOException {
         i.preLogArchive(p, newPath);
       }
     }
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, p, newPath)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) {
       throw new IOException("Unable to rename " + p + " to " + newPath);
     }
     // Tell our listeners that a log has been archived.
@@ -994,7 +994,7 @@ public void closeAndDelete() throws IOException {
         }
       }
 
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, file.getPath(),p)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, file.getPath(), p)) {
         throw new IOException("Unable to rename " + file.getPath() + " to " + p);
       }
       // Tell our listeners that a log was archived.
@@ -1006,7 +1006,7 @@ public void closeAndDelete() throws IOException {
     }
     LOG.debug("Moved " + files.length + " log files to " +
       FSUtils.getPath(this.oldLogDir));
-    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, conf, dir)) {
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, dir)) {
       LOG.info("Unable to delete " + dir);
     }
   }
@@ -1888,7 +1888,7 @@ public static Path moveAsideBadEditsFile(final FileSystem fs,
   throws IOException {
     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "." +
       System.currentTimeMillis());
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), edits, moveAsideName)) {
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, edits, moveAsideName)) {
       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
     }
     return moveAsideName;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java
index 29efe4f31359..9bcf1db7005b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java
@@ -41,7 +41,7 @@ public class HLogFileSystem extends HBaseFileSystem {
    */
 
   public HLogFileSystem(Configuration conf) {
-    checkAndSetRetryCounts(conf);
+    setRetryCounts(conf);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 59e1215fde5c..71c5850b4b8f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -484,7 +484,7 @@ public boolean splitLogFile(FileStatus logfile,
             LOG.warn("Found existing old edits file. It could be the "
                 + "result of a previous failed split attempt. Deleting " + dst + ", length="
                 + fs.getFileStatus(dst).getLen());
-            if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, dst)) {
+            if (!HBaseFileSystem.deleteFileFromFileSystem(fs, dst)) {
               LOG.warn("Failed deleting of old " + dst);
               throw new IOException("Failed deleting of old " + dst);
             }
@@ -493,7 +493,7 @@ public boolean splitLogFile(FileStatus logfile,
           // data without touching disk. TestHLogSplit#testThreading is an
           // example.
           if (fs.exists(wap.p)) {
-            if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, wap.p, dst)) {
+            if (!HBaseFileSystem.renameDirForFileSystem(fs, wap.p, dst)) {
               throw new IOException("Failed renaming " + wap.p + " to " + dst);
             }
             LOG.debug("Rename " + wap.p + " to " + dst);
@@ -560,7 +560,7 @@ public static void finishSplitLogFile(Path rootdir, Path oldLogDir,
     }
     archiveLogs(null, corruptedLogs, processedLogs, oldLogDir, fs, conf);
     Path stagingDir = ZKSplitLog.getSplitLogDir(rootdir, logPath.getName());
-    HBaseFileSystem.deleteDirFromFileSystem(fs, conf, stagingDir);
+    HBaseFileSystem.deleteDirFromFileSystem(fs, stagingDir);
   }
 
   /**
@@ -583,17 +583,17 @@ private static void archiveLogs(
     final Path corruptDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get(
         "hbase.regionserver.hlog.splitlog.corrupt.dir",  HConstants.CORRUPT_DIR_NAME));
 
-    if (!HBaseFileSystem.makeDirOnFileSystem(fs, conf, corruptDir)) {
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, corruptDir)) {
       LOG.info("Unable to mkdir " + corruptDir);
     }
-    HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir);
+    HBaseFileSystem.makeDirOnFileSystem(fs, oldLogDir);
 
     // this method can get restarted or called multiple times for archiving
     // the same log files.
     for (Path corrupted : corruptedLogs) {
       Path p = new Path(corruptDir, corrupted.getName());
       if (fs.exists(corrupted)) {
-        if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, corrupted, p)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, corrupted, p)) {
           LOG.warn("Unable to move corrupted log " + corrupted + " to " + p);
         } else {
           LOG.warn("Moving corrupted log " + corrupted + " to " + p);
@@ -604,7 +604,7 @@ private static void archiveLogs(
     for (Path p : processedLogs) {
       Path newPath = HLog.getHLogArchivePath(oldLogDir, p);
       if (fs.exists(p)) {
-        if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, p, newPath)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) {
           LOG.warn("Unable to move  " + p + " to " + newPath);
         } else {
           LOG.debug("Archived processed log " + p + " to " + newPath);
@@ -614,7 +614,7 @@ private static void archiveLogs(
 
     // distributed log splitting removes the srcDir (region's log dir) later
     // when all the log files in that srcDir have been successfully processed
-    if (srcDir != null && !HBaseFileSystem.deleteDirFromFileSystem(fs, conf, srcDir)) {
+    if (srcDir != null && !HBaseFileSystem.deleteDirFromFileSystem(fs, srcDir)) {
       throw new IOException("Unable to delete src dir: " + srcDir);
     }
   }
@@ -647,7 +647,7 @@ static Path getRegionSplitEditsPath(final FileSystem fs,
       return null;
     }
     if (isCreate && !fs.exists(dir) && 
-        !HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), dir)) {
+        !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) {
       LOG.warn("mkdir failed on " + dir);
     }
     // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
@@ -1066,7 +1066,7 @@ private WriterAndPath createWAP(byte[] region, Entry entry, Path rootdir,
           + "result of a previous failed split attempt. Deleting "
           + regionedits + ", length="
           + fs.getFileStatus(regionedits).getLen());
-      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, regionedits)) {
+      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, regionedits)) {
         LOG.warn("Failed delete of old " + regionedits);
       }
     }
@@ -1092,12 +1092,12 @@ Path convertRegionEditsToTemp(Path rootdir, Path edits, String tmpname) {
             + "result of a previous failed split attempt. Deleting "
             + ret + ", length="
             + fs.getFileStatus(ret).getLen());
-        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, ret)) {
+        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, ret)) {
           LOG.warn("Failed delete of old " + ret);
         }
       }
       Path dir = ret.getParent();
-      if (!fs.exists(dir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, dir)) { 
+      if (!fs.exists(dir) && !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) { 
           LOG.warn("mkdir failed on " + dir);
       }
     } catch (IOException e) {
@@ -1191,7 +1191,7 @@ private List<Path> closeStreams() throws IOException {
             LOG.warn("Found existing old edits file. It could be the "
                 + "result of a previous failed split attempt. Deleting " + dst
                 + ", length=" + fs.getFileStatus(dst).getLen());
-            if (!HBaseFileSystem.deleteFileFromFileSystem(fs, conf, dst)) {
+            if (!HBaseFileSystem.deleteFileFromFileSystem(fs, dst)) {
               LOG.warn("Failed deleting of old " + dst);
               throw new IOException("Failed deleting of old " + dst);
             }
@@ -1200,7 +1200,7 @@ private List<Path> closeStreams() throws IOException {
           // the data without touching disk. TestHLogSplit#testThreading is an
           // example.
           if (fs.exists(wap.p)) {
-            if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, wap.p, dst)) {
+            if (!HBaseFileSystem.renameDirForFileSystem(fs, wap.p, dst)) {
               throw new IOException("Failed renaming " + wap.p + " to " + dst);
             }
             LOG.debug("Rename " + wap.p + " to " + dst);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index fae73dd6faca..6b4c64b1f3d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -225,7 +225,7 @@ public HTableDescriptor remove(final String tablename)
     if (!this.fsreadonly) {
       Path tabledir = FSUtils.getTablePath(this.rootdir, tablename);
       if (this.fs.exists(tabledir)) {
-        if (!HBaseFileSystem.deleteDirFromFileSystem(fs, fs.getConf(), tabledir)) {
+        if (!HBaseFileSystem.deleteDirFromFileSystem(fs, tabledir)) {
           throw new IOException("Failed delete of " + tabledir.toString());
         }
       }
@@ -281,7 +281,7 @@ public boolean accept(Path p) {
       for (int i = 1; i < status.length; i++) {
         Path p = status[i].getPath();
         // Clean up old versions
-        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, fs.getConf(), p)) {
+        if (!HBaseFileSystem.deleteFileFromFileSystem(fs, p)) {
           LOG.warn("Failed cleanup of " + status);
         } else {
           LOG.debug("Cleaned up old tableinfo file " + p);
@@ -505,7 +505,7 @@ private static Path writeTableDescriptor(final FileSystem fs,
       try {
         writeHTD(fs, p, hTableDescriptor);
         tableInfoPath = getTableInfoFileName(tableDir, sequenceid);
-        if (!HBaseFileSystem.renameDirForFileSystem(fs, fs.getConf(), p, tableInfoPath)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, p, tableInfoPath)) {
           throw new IOException("Failed rename of " + p + " to " + tableInfoPath);
         }
       } catch (IOException ioe) {
@@ -531,7 +531,7 @@ private static Path writeTableDescriptor(final FileSystem fs,
   private static void writeHTD(final FileSystem fs, final Path p,
       final HTableDescriptor htd)
   throws IOException {
-    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, fs.getConf(), p, false);
+    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, p, false);
     try {
       htd.write(out);
       out.write('\n');
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index dde45806e5b4..8ae580868f6d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -107,7 +107,7 @@ public static boolean deleteDirectory(final FileSystem fs, final Path dir)
    */
   public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
     if (!fs.exists(dir)) {
-      HBaseFileSystem.makeDirOnFileSystem(fs, fs.getConf(), dir);
+      HBaseFileSystem.makeDirOnFileSystem(fs, dir);
     }
     return dir;
   }
@@ -155,7 +155,7 @@ public static FSDataOutputStream create(FileSystem fs, Path path,
   public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm,
       boolean overwrite) throws IOException {
     LOG.debug("Creating file=" + path + " with permission=" + perm);
-    return HBaseFileSystem.createPathWithPermsOnFileSystem(fs, fs.getConf(), path, perm, overwrite);
+    return HBaseFileSystem.createPathWithPermsOnFileSystem(fs, path, perm, overwrite);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
index 526c4cc17a5d..8e9aa499c78b 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
@@ -165,7 +165,7 @@ public static void markCorrupted(Path rootdir, String logFileName,
       FileSystem fs) {
     Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
     try {
-      HBaseFileSystem.createNewFileOnFileSystem(fs, fs.getConf(), file);
+      HBaseFileSystem.createNewFileOnFileSystem(fs, file);
     } catch (IOException e) {
       LOG.warn("Could not flag a log file as corrupted. Failed to create " +
           file, e);
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index fe9db5f47a82..c95c78375220 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -202,6 +202,7 @@ public HBaseTestingUtility(Configuration conf) {
 
   private void setHDFSClientRetryProperty() {
     this.conf.setInt("hdfs.client.retries.number", 1);
+    HBaseFileSystem.setRetryCounts(conf);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
index fe86ac5a4b7a..67602c5527ee 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
@@ -55,6 +55,7 @@ public static void setUpBeforeClass() throws Exception {
     LOG.info("hbase.rootdir=" + hbaseRootDir);
     conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
     conf.setInt("hdfs.client.retries.number", 10);
+    HBaseFileSystem.setRetryCounts(conf);
   }
 
   
@@ -65,16 +66,27 @@ public void testNonIdempotentOpsWithRetries() throws IOException {
     Path rootDir = new Path(TestHBaseFileSystem.conf.get(HConstants.HBASE_DIR));
     FileSystem fs = TEST_UTIL.getTestFileSystem();
     // Create a Region
-    assertTrue(HBaseFileSystem.createPathOnFileSystem(fs, TestHBaseFileSystem.conf, rootDir, true) != null);
+    assertTrue(HBaseFileSystem.createPathOnFileSystem(fs, rootDir, 
+      true) != null);
 
-    boolean result = HBaseFileSystem.makeDirOnFileSystem(new MockFileSystemForCreate(), TestHBaseFileSystem.conf, new Path("/a"));
+    boolean result = HBaseFileSystem.makeDirOnFileSystem(new MockFileSystemForCreate(), 
+      new Path("/a"));
     assertTrue("Couldn't create the directory", result);
 
+    try {
+      HBaseFileSystem.createPathOnFileSystem(new MockFileSystemForCreate(), 
+        new Path("/A"), false);
+     assertTrue(false);// control should not come here.
+    } catch (Exception e) {
+      LOG.info(e);
+    }
 
-    result = HBaseFileSystem.renameDirForFileSystem(new MockFileSystem(), TestHBaseFileSystem.conf, new Path("/a"), new Path("/b"));
+    result = HBaseFileSystem.renameDirForFileSystem(new MockFileSystem(), new Path("/a"),
+      new Path("/b"));
     assertTrue("Couldn't rename the directory", result);
 
-    result = HBaseFileSystem.deleteDirFromFileSystem(new MockFileSystem(), TestHBaseFileSystem.conf, new Path("/a"));
+    result = HBaseFileSystem.deleteDirFromFileSystem(new MockFileSystem(), 
+      new Path("/a"));
 
     assertTrue("Couldn't delete the directory", result);
     fs.delete(rootDir, true);
@@ -83,6 +95,7 @@ public void testNonIdempotentOpsWithRetries() throws IOException {
   static class MockFileSystemForCreate extends MockFileSystem {
     @Override
     public boolean exists(Path path) {
+      if ("/A".equals(path.toString())) return true;
       return false;
     }
   }

From a64cdae9c2fd46c3df3f2fb6132e564a05ce4381 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 9 Apr 2013 04:27:44 +0000
Subject: [PATCH 0919/1540] HBASE-8017 Upgrade hadoop 1 dependency to 1.1.2 for
 hadoop 1.1 profile

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1465877 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 6fe7bd6826ae..1f9dbbb4205d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1736,7 +1736,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>1.1.1</hadoop.version>
+        <hadoop.version>1.1.2</hadoop.version>
         <slf4j.version>1.4.3</slf4j.version>
       </properties>
       <dependencies>

From 315f03efb674599da27d77eb96923bdb50a485fe Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 10 Apr 2013 10:36:56 +0000
Subject: [PATCH 0920/1540] HBASE-8313 Add Bloom filter testing for
 HFileOutputFormat

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1466418 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |   2 +-
 .../hadoop/hbase/HBaseTestingUtility.java     |  53 +++++++++
 .../mapreduce/TestHFileOutputFormat.java      | 103 ++++++------------
 3 files changed, 89 insertions(+), 69 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 4c3383be3774..c73935eb47a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -123,7 +123,7 @@ public static enum BloomType {
       Bytes.toBytes("EXCLUDE_FROM_MINOR_COMPACTION");
 
   /** Bloom filter Type in FileInfo */
-  static final byte[] BLOOM_FILTER_TYPE_KEY =
+  public static final byte[] BLOOM_FILTER_TYPE_KEY =
       Bytes.toBytes("BLOOM_FILTER_TYPE");
 
   /** Delete Family Count in FileInfo */
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index c95c78375220..dc9e0c9d8cff 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -61,6 +61,7 @@
 import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -2257,4 +2258,56 @@ public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numR
     return result;
   }
 
+  /**
+   * Create a set of column descriptors with the combination of compression,
+   * encoding, bloom codecs available.
+   * @return the list of column descriptors
+   */
+  public static List<HColumnDescriptor> generateColumnDescriptors() {
+    return generateColumnDescriptors("");
+  }
+
+  /**
+   * Create a set of column descriptors with the combination of compression,
+   * encoding, bloom codecs available.
+   * @param prefix family names prefix
+   * @return the list of column descriptors
+   */
+  public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) {
+    List<HColumnDescriptor> htds = new ArrayList<HColumnDescriptor>();
+    long familyId = 0;
+    for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
+      for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
+        for (StoreFile.BloomType bloomType: StoreFile.BloomType.values()) {
+          String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId);
+          HColumnDescriptor htd = new HColumnDescriptor(name);
+          htd.setCompressionType(compressionType);
+          htd.setDataBlockEncoding(encodingType);
+          htd.setBloomFilterType(bloomType);
+          htds.add(htd);
+          familyId++;
+        }
+      }
+    }
+    return htds;
+  }
+
+  /**
+   * Get supported compression algorithms.
+   * @return supported compression algorithms.
+   */
+  public static Compression.Algorithm[] getSupportedCompressionAlgorithms() {
+    String[] allAlgos = HFile.getSupportedCompressionAlgorithms();
+    List<Compression.Algorithm> supportedAlgos = new ArrayList<Compression.Algorithm>();
+    for (String algoName : allAlgos) {
+      try {
+        Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName);
+        algo.getCompressor();
+        supportedAlgos.add(algo);
+      } catch (Throwable t) {
+        // this algo is not available
+      }
+    }
+    return supportedAlgos.toArray(new Compression.Algorithm[0]);
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index c21cd056dec9..5bf8bbd73af7 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -33,6 +33,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.concurrent.Callable;
 import java.util.Random;
 
@@ -57,6 +59,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
 import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -553,29 +556,23 @@ private Map<String, Compression.Algorithm> getMockColumnFamilies(int numCfs) {
   }
 
   /**
-   * Test that {@link HFileOutputFormat} RecordWriter uses compression settings
-   * from the column family descriptor
+   * Test that {@link HFileOutputFormat} RecordWriter uses compression and
+   * bloom filter settings from the column family descriptor
    */
   @Test
-  public void testColumnFamilyCompression() throws Exception {
+  public void testColumnFamilySettings() throws Exception {
     Configuration conf = new Configuration(this.util.getConfiguration());
     RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
     TaskAttemptContext context = null;
-    Path dir =
-        util.getDataTestDir("testColumnFamilyCompression");
+    Path dir = util.getDataTestDir("testColumnFamilySettings");
 
+    // Setup table descriptor
     HTable table = Mockito.mock(HTable.class);
-
-    Map<String, Compression.Algorithm> configuredCompression =
-      new HashMap<String, Compression.Algorithm>();
-    Compression.Algorithm[] supportedAlgos = getSupportedCompressionAlgorithms();
-
-    int familyIndex = 0;
-    for (byte[] family : FAMILIES) {
-      configuredCompression.put(Bytes.toString(family),
-                                supportedAlgos[familyIndex++ % supportedAlgos.length]);
+    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
+    Mockito.doReturn(htd).when(table).getTableDescriptor();
+    for (HColumnDescriptor hcd: this.util.generateColumnDescriptors()) {
+      htd.addFamily(hcd);
     }
-    setupMockColumnFamilies(table, configuredCompression);
 
     // set up the table to return some mock keys
     setupMockStartKeys(table);
@@ -594,75 +591,45 @@ public void testColumnFamilyCompression() throws Exception {
       writer = hof.getRecordWriter(context);
 
       // write out random rows
-      writeRandomKeyValues(writer, context, ROWSPERSPLIT);
+      writeRandomKeyValues(writer, context, htd.getFamiliesKeys(), ROWSPERSPLIT);
       writer.close(context);
 
       // Make sure that a directory was created for every CF
-      FileSystem fileSystem = dir.getFileSystem(conf);
+      FileSystem fs = dir.getFileSystem(conf);
 
       // commit so that the filesystem has one directory per column family
       hof.getOutputCommitter(context).commitTask(context);
       hof.getOutputCommitter(context).commitJob(context);
-      for (byte[] family : FAMILIES) {
-        String familyStr = new String(family);
-        boolean found = false;
-        for (FileStatus f : fileSystem.listStatus(dir)) {
-
-          if (Bytes.toString(family).equals(f.getPath().getName())) {
-            // we found a matching directory
-            found = true;
-
-            // verify that the compression on this file matches the configured
-            // compression
-            Path dataFilePath = fileSystem.listStatus(f.getPath())[0].getPath();
-            Reader reader = HFile.createReader(fileSystem, dataFilePath,
-                new CacheConfig(conf));
-            reader.loadFileInfo();
-            assertEquals("Incorrect compression used for column family " + familyStr
-                         + "(reader: " + reader + ")",
-                         configuredCompression.get(familyStr), reader.getCompressionAlgorithm());
-            break;
-          }
-        }
-
-        if (!found) {
-          fail("HFile for column family " + familyStr + " not found");
-        }
+      FileStatus[] families = FSUtils.listStatus(fs, dir, new FSUtils.FamilyDirFilter(fs));
+      assertEquals(htd.getFamilies().size(), families.length);
+      for (FileStatus f : families) {
+        String familyStr = f.getPath().getName();
+        HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes(familyStr));
+        // verify that the compression on this file matches the configured
+        // compression
+        Path dataFilePath = fs.listStatus(f.getPath())[0].getPath();
+        Reader reader = HFile.createReader(fs, dataFilePath, new CacheConfig(conf));
+        Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
+
+        byte[] bloomFilter = fileInfo.get(StoreFile.BLOOM_FILTER_TYPE_KEY);
+        if (bloomFilter == null) bloomFilter = Bytes.toBytes("NONE");
+        assertEquals("Incorrect bloom filter used for column family " + familyStr +
+          "(reader: " + reader + ")",
+          hcd.getBloomFilterType(), StoreFile.BloomType.valueOf(Bytes.toString(bloomFilter)));
+        assertEquals("Incorrect compression used for column family " + familyStr +
+          "(reader: " + reader + ")", hcd.getCompression(), reader.getCompressionAlgorithm());
       }
-
     } finally {
       dir.getFileSystem(conf).delete(dir, true);
     }
   }
 
-
-  /**
-   * @return
-   */
-  private Compression.Algorithm[] getSupportedCompressionAlgorithms() {
-    String[] allAlgos = HFile.getSupportedCompressionAlgorithms();
-    List<Compression.Algorithm> supportedAlgos = Lists.newArrayList();
-
-    for (String algoName : allAlgos) {
-      try {
-        Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName);
-        algo.getCompressor();
-        supportedAlgos.add(algo);
-      }catch (Exception e) {
-        // this algo is not available
-      }
-    }
-
-    return supportedAlgos.toArray(new Compression.Algorithm[0]);
-  }
-
-
   /**
    * Write random values to the writer assuming a table created using
    * {@link #FAMILIES} as column family descriptors
    */
-  private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue> writer, TaskAttemptContext context,
-      int numRows)
+  private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue> writer,
+      TaskAttemptContext context, Set<byte[]> families, int numRows)
       throws IOException, InterruptedException {
     byte keyBytes[] = new byte[Bytes.SIZEOF_INT];
     int valLength = 10;
@@ -678,7 +645,7 @@ private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue>
       random.nextBytes(valBytes);
       ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes);
 
-      for (byte[] family : TestHFileOutputFormat.FAMILIES) {
+      for (byte[] family : families) {
         KeyValue kv = new KeyValue(keyBytes, family,
             PerformanceEvaluation.QUALIFIER_NAME, valBytes);
         writer.write(key, kv);

From fce3ca9bbf5b0ad5acc225c59ddc103e560ad63c Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Wed, 10 Apr 2013 17:02:55 +0000
Subject: [PATCH 0921/1540] HBASE-8266-Master cannot start if
 TableNotFoundException is thrown while partial table recovery (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1466567 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/handler/CreateTableHandler.java    | 10 ++++-
 .../master/handler/EnableTableHandler.java    | 13 +++++-
 .../hadoop/hbase/zookeeper/ZKTable.java       | 12 +++++-
 .../handler/TestCreateTableHandler.java       | 41 ++++++++++++++++---
 4 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index e7b4487f4b80..2e2ebfbb6149 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -142,8 +142,14 @@ protected void completed(final Throwable exception) {
     // again with the same Active master
     // It will block the creation saying TableAlreadyExists.
     if (exception != null) {
-      this.assignmentManager.getZKTable().removeEnablingTable(
-          this.hTableDescriptor.getNameAsString());
+      try {
+        this.assignmentManager.getZKTable().removeEnablingTable(
+            this.hTableDescriptor.getNameAsString(), false);
+      } catch (KeeperException e) {
+        // Keeper exception should not happen here
+        LOG.error("Got a keeper exception while removing the ENABLING table znode "
+            + this.hTableDescriptor.getNameAsString(), e);
+      }
     }
 
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
index aa147b532399..7e348107507c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -66,7 +66,18 @@ public EnableTableHandler(Server server, byte [] tableName,
     this.retainAssignment = skipTableStateCheck;
     // Check if table exists
     if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
-      throw new TableNotFoundException(Bytes.toString(tableName));
+      // retainAssignment is true only during recovery. In normal case it is
+      // false
+      if (!this.retainAssignment) {
+        throw new TableNotFoundException(tableNameStr);
+      }
+      try {
+        this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true);
+      } catch (KeeperException e) {
+        // TODO : Use HBCK to clear such nodes
+        LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr
+            + ".  The table will remain unusable. Run HBCK to manually fix the problem.");
+      }
     }
 
     // There could be multiple client requests trying to disable or enable
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 6e66b70a581f..6aa502798b1d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -180,13 +180,21 @@ public boolean checkAndSetEnablingTable(final String tableName)
   
   /**
    * If the table is found in ENABLING state the inmemory state is removed.
-   * This helps in cases where CreateTable is to be retried by the client incase of failures
+   * This helps in cases where CreateTable is to be retried by the client incase of failures.
+   * If deleteZNode is true - the znode is also deleted
    * @param tableName
+   * @param deleteZNode
+   * @throws KeeperException
    */
-  public void removeEnablingTable(final String tableName) {
+  public void removeEnablingTable(final String tableName, boolean deleteZNode)
+      throws KeeperException {
     synchronized (this.cache) {
       if (isEnablingTable(tableName)) {
         this.cache.remove(tableName);
+        if (deleteZNode) {
+          ZKUtil.deleteNodeFailSilent(this.watcher,
+              ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName));
+        }
       }
 
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
index 8c0624c269d0..6900352d714b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
@@ -46,7 +46,9 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -57,16 +59,17 @@ public class TestCreateTableHandler {
   private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class);
   private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler");
   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
-  public static boolean throwException = false;
+  private static boolean throwException = false;
 
-  @BeforeClass
-  public static void setUp() throws Exception {
+  @Before
+  public void setUp() throws Exception {
     TEST_UTIL.startMiniCluster(1);
   }
 
-  @AfterClass
-  public static void tearDown() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
+    throwException = true;
   }
 
   @Test
@@ -95,6 +98,34 @@ public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() t
     assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME));
 
   }
+  
+  @Test (timeout=10000)
+  public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
+    byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated");
+    final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    final HMaster m = cluster.getMaster();
+    final HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor(FAMILYNAME));
+    final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null,
+        null) };
+    CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(),
+        m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos,
+        m.getCatalogTracker(), m.getAssignmentManager());
+    throwException = true;
+    handler.process();
+    abortAndStartNewMaster(cluster);
+    assertTrue(cluster.getLiveMasterThreads().size() == 1);
+    
+  }
+  
+  private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException {
+    cluster.abortMaster(0);
+    cluster.waitOnMaster(0);
+    LOG.info("Starting new master");
+    cluster.startMaster();
+    LOG.info("Waiting for master to become active.");
+    cluster.waitForActiveAndReadyMaster();
+  }
 
   private static class CustomCreateTableHandler extends CreateTableHandler {
     public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager,

From 07cd5a77d36aca0947513aebec1c2b9ea7a56cd3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 10 Apr 2013 22:12:53 +0000
Subject: [PATCH 0922/1540] HBASE-8316 JoinedHeap for non essential column
 families should reseek instead of seek

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1466708 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 590c70db9422..bef10b2bccaf 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3985,7 +3985,8 @@ private boolean nextInternal(List<KeyValue> results, int limit, String metric)
             // If joinedHeap is pointing to some other row, try to seek to a correct one.
             boolean mayHaveData =
               (nextJoinedKv != null && nextJoinedKv.matchingRow(currentRow, offset, length))
-              || (this.joinedHeap.seek(KeyValue.createFirstOnRow(currentRow, offset, length))
+                || (this.joinedHeap.requestSeek(
+                    KeyValue.createFirstOnRow(currentRow, offset, length), true, true)
                   && joinedHeap.peek() != null
                   && joinedHeap.peek().matchingRow(currentRow, offset, length));
             if (mayHaveData) {

From 0013f392de1b40187744328050b56c8532bb2adc Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 10 Apr 2013 22:53:45 +0000
Subject: [PATCH 0923/1540] HBASE-7658 grant with an empty string as permission
 should throw an exception

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1466723 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/AccessControlLists.java     | 5 +++--
 src/main/ruby/hbase/security.rb                              | 5 +++++
 src/main/ruby/shell/commands/grant.rb                        | 2 +-
 src/main/ruby/shell/commands/revoke.rb                       | 3 ++-
 4 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
index 345fde07fcb0..5b4b53d32ff9 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
@@ -134,8 +134,9 @@ static void addUserPermission(Configuration conf, UserPermission userPerm)
     byte[] key = userPermissionKey(userPerm);
 
     if ((actions == null) || (actions.length == 0)) {
-      LOG.warn("No actions associated with user '"+Bytes.toString(userPerm.getUser())+"'");
-      return;
+      String msg = "No actions associated with user '" + Bytes.toString(userPerm.getUser()) + "'";
+      LOG.warn(msg);
+      throw new IOException(msg);
     }
 
     byte[] value = new byte[actions.length];
diff --git a/src/main/ruby/hbase/security.rb b/src/main/ruby/hbase/security.rb
index e3b99038ba54..778e20ec9a89 100644
--- a/src/main/ruby/hbase/security.rb
+++ b/src/main/ruby/hbase/security.rb
@@ -36,6 +36,11 @@ def grant(user, permissions, table_name=nil, family=nil, qualifier=nil)
 
       # TODO: need to validate user name
 
+      # Verify that the specified permission is valid
+      if (permissions == nil || permissions.length == 0)
+        raise(ArgumentError, "Ivalid permission: no actions associated with user")
+      end
+
       if (table_name != nil)
         # Table should exist
         raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name)
diff --git a/src/main/ruby/shell/commands/grant.rb b/src/main/ruby/shell/commands/grant.rb
index a4e9330126ce..7bdc95943b93 100644
--- a/src/main/ruby/shell/commands/grant.rb
+++ b/src/main/ruby/shell/commands/grant.rb
@@ -22,7 +22,7 @@ class Grant < Command
       def help
         return <<-EOF
 Grant users specific rights.
-Syntax : grant <user> <permissions> <table> <column family> <column qualifier>
+Syntax : grant <user> <permissions> [<table> [<column family> [<column qualifier>]]
 
 permissions is either zero or more letters from the set "RWXCA".
 READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A')
diff --git a/src/main/ruby/shell/commands/revoke.rb b/src/main/ruby/shell/commands/revoke.rb
index fe9391c80582..721519e7d6ca 100644
--- a/src/main/ruby/shell/commands/revoke.rb
+++ b/src/main/ruby/shell/commands/revoke.rb
@@ -22,9 +22,10 @@ class Revoke < Command
       def help
         return <<-EOF
 Revoke a user's access rights.
-Syntax : revoke <user> <table> <column family> <column qualifier>
+Syntax : revoke <user> [<table> [<column family> [<column qualifier>]]
 For example:
 
+    hbase> revoke 'bobsmith'
     hbase> revoke 'bobsmith', 't1', 'f1', 'col1'
 EOF
       end

From ab944c690fc9c876a1160bdc54f3bcc483d97ed3 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 10 Apr 2013 22:58:05 +0000
Subject: [PATCH 0924/1540] HBASE-7824 Improve master start up time when there
 is log splitting work (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1466725 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/CatalogTracker.java  |   2 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 150 +++++++++++----
 .../hadoop/hbase/master/MasterFileSystem.java |  59 +++---
 .../hadoop/hbase/master/ServerManager.java    |  38 +++-
 .../handler/MetaServerShutdownHandler.java    |   7 +
 .../master/handler/ServerShutdownHandler.java |  48 +++--
 .../hbase/regionserver/HRegionServer.java     |   7 +-
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |   2 +-
 .../apache/hadoop/hbase/MiniHBaseCluster.java |   5 +-
 .../master/TestDistributedLogSplitting.java   | 178 ++++++++++++++++--
 .../hbase/master/TestMasterFailover.java      | 137 +++++++++++++-
 .../TestRSKilledWhenMasterInitializing.java   |  33 +---
 12 files changed, 533 insertions(+), 133 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 5f2f1482c7f2..58c61ee1703d 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -118,7 +118,7 @@ public class CatalogTracker {
    */
   private ServerName metaLocation;
 
-  private boolean stopped = false;
+  private volatile boolean stopped = false;
 
   static final byte [] ROOT_REGION_NAME =
     HRegionInfo.ROOT_REGIONINFO.getRegionName();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index cf6b4fcab3ac..7cbcf279647c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -41,9 +41,6 @@
 
 import javax.management.ObjectName;
 
-import com.google.common.collect.ClassToInstanceMap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MutableClassToInstanceMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -129,6 +126,10 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
+
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
  * master.  If many masters are started, all compete.  Whichever wins goes on to
@@ -253,6 +254,12 @@ public class HMaster extends HasThread
   /** The health check chore. */
   private HealthCheckChore healthCheckChore;
 
+  /** flag when true, Master waits for log splitting complete before start up */
+  private boolean waitingOnLogSplitting = false;
+
+  /** flag used in test cases in order to simulate RS failures during master initialization */
+  private volatile boolean initializationBeforeMetaAssignment = false;
+
   /**
    * Initializes the HMaster. The steps are as follows:
    * <p>
@@ -336,7 +343,9 @@ public HMaster(final Configuration conf)
     if (isHealthCheckerConfigured()) {
       healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
     }
+
     this.shouldSplitMetaSeparately = conf.getBoolean(HLog.SEPARATE_HLOG_FOR_META, false);
+    waitingOnLogSplitting = this.conf.getBoolean("hbase.master.wait.for.log.splitting", false);
   }
 
   /**
@@ -579,14 +588,49 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     if (!masterRecovery) {
       this.assignmentManager.startTimeOutMonitor();
     }
-    // TODO: Should do this in background rather than block master startup
-    status.setStatus("Splitting logs after master startup");
-    splitLogAfterStartup(this.fileSystemManager);
 
-    // Make sure root and meta assigned before proceeding.
-    assignRootAndMeta(status);
+    // get a list for previously failed RS which need recovery work
+    Set<ServerName> failedServers = this.fileSystemManager.getFailedServersFromLogFolders();
+    if (waitingOnLogSplitting) {
+      List<ServerName> servers = new ArrayList<ServerName>(failedServers);
+      this.fileSystemManager.splitAllLogs(servers);
+      failedServers.clear();
+    }
+
+    ServerName preRootServer = this.catalogTracker.getRootLocation();
+    if (preRootServer != null && failedServers.contains(preRootServer)) {
+      // create recovered edits file for _ROOT_ server
+      this.fileSystemManager.splitAllLogs(preRootServer);
+      failedServers.remove(preRootServer);
+    }
+
+    this.initializationBeforeMetaAssignment = true;
+    // Make sure root assigned before proceeding.
+    assignRoot(status);
+
+    // SSH should enabled for ROOT before META region assignment
+    // because META region assignment is depending on ROOT server online.
+    this.serverManager.enableSSHForRoot();
+
+    // log splitting for .META. server
+    ServerName preMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+    if (preMetaServer != null && failedServers.contains(preMetaServer)) {
+      // create recovered edits file for .META. server
+      this.fileSystemManager.splitAllLogs(preMetaServer);
+      failedServers.remove(preMetaServer);
+    }
+
+    // Make sure meta assigned before proceeding.
+    assignMeta(status, ((masterRecovery) ? null : preMetaServer), preRootServer);
+
     enableServerShutdownHandler();
 
+    // handle other dead servers in SSH
+    status.setStatus("Submit log splitting work of non-meta region servers");
+    for (ServerName curServer : failedServers) {
+      this.serverManager.expireServer(curServer);
+    }
+
     // Update meta with new HRI if required. i.e migrate all HRI with HTD to
     // HRI with out HTD in meta and update the status in ROOT. This must happen
     // before we assign all user regions or else the assignment will fail.
@@ -658,22 +702,13 @@ protected void startCatalogJanitorChore() {
   }
 
   /**
-   * Override to change master's splitLogAfterStartup. Used testing
-   * @param mfs
-   */
-  protected void splitLogAfterStartup(final MasterFileSystem mfs) {
-    mfs.splitLogAfterStartup();
-  }
-
-  /**
-   * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,
-   * assign them.
+   * Check <code>-ROOT-</code> is assigned. If not, assign it.
+   * @param status MonitoredTask
    * @throws InterruptedException
    * @throws IOException
    * @throws KeeperException
-   * @return Count of regions we assigned.
    */
-  int assignRootAndMeta(MonitoredTask status)
+  private void assignRoot(MonitoredTask status)
   throws InterruptedException, IOException, KeeperException {
     int assigned = 0;
     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
@@ -704,17 +739,39 @@ int assignRootAndMeta(MonitoredTask status)
     LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
       ", location=" + catalogTracker.getRootLocation());
 
-    // Work on meta region
+    status.setStatus("ROOT assigned.");
+  }
+
+  /**
+   * Check <code>.META.</code> is assigned. If not, assign it.
+   * @param status MonitoredTask
+   * @param previousMetaServer ServerName of previous meta region server before current start up
+   * @param previousRootServer ServerName of previous root region server before current start up
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private void assignMeta(MonitoredTask status, ServerName previousMetaServer,
+      ServerName previousRootServer)
+      throws InterruptedException,
+      IOException, KeeperException {
+    int assigned = 0;
+    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
+
     status.setStatus("Assigning META region");
-    rit = this.assignmentManager.
-      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
+    boolean rit =
+        this.assignmentManager
+            .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
     boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
     if (!rit && !metaRegionLocation) {
       ServerName currentMetaServer =
-        this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
-      if (currentMetaServer != null
-          && !currentMetaServer.equals(currentRootServer)) {
-        splitLogAndExpireIfOnline(currentMetaServer);
+          (previousMetaServer != null) ? previousMetaServer : this.catalogTracker
+              .getMetaLocationOrReadLocationFromRoot();
+      if (currentMetaServer != null && !currentMetaServer.equals(previousRootServer)) {
+        fileSystemManager.splitAllLogs(currentMetaServer);
+        if (this.serverManager.isServerOnline(currentMetaServer)) {
+          this.serverManager.expireServer(currentMetaServer);
+        }
       }
       assignmentManager.assignMeta();
       enableSSHandWaitForMeta();
@@ -723,15 +780,14 @@ int assignRootAndMeta(MonitoredTask status)
       enableSSHandWaitForMeta();
       assigned++;
     } else {
-      // Region already assigned.  We didnt' assign it.  Add to in-memory state.
+      // Region already assigned. We didnt' assign it. Add to in-memory state.
       this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
         this.catalogTracker.getMetaLocation());
     }
     enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
-    LOG.info(".META. assigned=" + assigned + ", rit=" + rit +
-      ", location=" + catalogTracker.getMetaLocation());
-    status.setStatus("META and ROOT assigned.");
-    return assigned;
+    LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
+        + catalogTracker.getMetaLocation());
+    status.setStatus("META assigned.");
   }
 
   private void enableSSHandWaitForMeta() throws IOException,
@@ -744,7 +800,10 @@ private void enableSSHandWaitForMeta() throws IOException,
         .waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
   }
 
-  private void waitForRootAssignment() throws InterruptedException {
+  private void waitForRootAssignment() throws InterruptedException, IOException {
+    // Enable SSH for ROOT to prevent a newly assigned ROOT crashes again before global SSH is
+    // enabled
+    this.serverManager.enableSSHForRoot();
     this.catalogTracker.waitForRoot();
     // This guarantees that the transition has completed
     this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);
@@ -790,8 +849,7 @@ public boolean visit(Result r) throws IOException {
   }
 
   /**
-   * Split a server's log and expire it if we find it is one of the online
-   * servers.
+   * Expire a server if we find it is one of the online servers.
    * @param sn ServerName to check.
    * @throws IOException
    */
@@ -1563,6 +1621,7 @@ public Boolean call() throws InterruptedException,
           if (!becomeActiveMaster(status)) {
             return Boolean.FALSE;
           }
+          serverManager.disableSSHForRoot();
           serverShutdownHandlerEnabled = false;
           initialized = false;
           finishInitialization(status, true);
@@ -1661,12 +1720,23 @@ public void shutdown() {
     }
     if (this.assignmentManager != null) this.assignmentManager.shutdown();
     if (this.serverManager != null) this.serverManager.shutdownCluster();
+
     try {
       if (this.clusterStatusTracker != null){
         this.clusterStatusTracker.setClusterDown();
       }
     } catch (KeeperException e) {
-      LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      if (e instanceof KeeperException.SessionExpiredException) {
+        LOG.warn("ZK session expired. Retry a new connection...");
+        try {
+          this.zooKeeper.reconnectAfterExpiration();
+          this.clusterStatusTracker.setClusterDown();
+        } catch (Exception ex) {
+          LOG.error("Retry setClusterDown failed", ex);
+        }
+      } else {
+        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      }
     }
   }
 
@@ -1749,6 +1819,14 @@ public boolean shouldSplitMetaSeparately() {
     return this.shouldSplitMetaSeparately;
   }
 
+  /**
+   * Report whether this master has started initialization and is about to do meta region assignment
+   * @return true if master is in initialization & about to assign ROOT & META regions
+   */
+  public boolean isInitializationStartsMetaRegoinAssignment() {
+    return this.initializationBeforeMetaAssignment;
+  }
+
   @Override
   @Deprecated
   public void assign(final byte[] regionName, final boolean force)
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 7dd33b4e8cf6..acce4b559da3 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
@@ -211,30 +212,31 @@ public String getClusterId() {
   }
 
   /**
-   * Inspect the log directory to recover any log file without
-   * an active region server.
+   * Inspect the log directory to find dead servers which need log splitting
    */
-  void splitLogAfterStartup() {
+  Set<ServerName> getFailedServersFromLogFolders() {
     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
-        HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+      HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+    
+    Set<ServerName> serverNames = new HashSet<ServerName>();
     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
+
     do {
       if (master.isStopped()) {
-        LOG.warn("Master stopped while splitting logs");
+        LOG.warn("Master stopped while trying to get failed servers.");
         break;
       }
-      List<ServerName> serverNames = new ArrayList<ServerName>();
       try {
-        if (!this.fs.exists(logsDirPath)) return;
+        if (!this.fs.exists(logsDirPath)) return serverNames;
         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
         // Get online servers after getting log folders to avoid log folder deletion of newly
         // checked in region servers . see HBASE-5916
-        Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
-            .keySet();
+        Set<ServerName> onlineServers =
+            ((HMaster) master).getServerManager().getOnlineServers().keySet();
 
         if (logFolders == null || logFolders.length == 0) {
           LOG.debug("No log files to split, proceeding...");
-          return;
+          return serverNames;
         }
         for (FileStatus status : logFolders) {
           String sn = status.getPath().getName();
@@ -248,27 +250,19 @@ void splitLogAfterStartup() {
                 + "to a known region server, splitting");
             serverNames.add(serverName);
           } else {
-            LOG.info("Log folder " + status.getPath()
-                + " belongs to an existing region server");
+            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
           }
         }
-        if (services.shouldSplitMetaSeparately()) {
-          splitLog(serverNames, META_FILTER);
-          splitLog(serverNames, NON_META_FILTER);
-        } else {
-          splitAllLogs(serverNames);
-        }
         retrySplitting = false;
       } catch (IOException ioe) {
-        LOG.warn("Failed splitting of " + serverNames, ioe);
+        LOG.warn("Failed getting failed servers to be recovered.", ioe);
         if (!checkFileSystem()) {
           LOG.warn("Bad Filesystem, exiting");
           Runtime.getRuntime().halt(1);
         }
         try {
           if (retrySplitting) {
-            Thread.sleep(conf.getInt(
-              "hbase.hlog.split.failure.retry.interval", 30 * 1000));
+            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
           }
         } catch (InterruptedException e) {
           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
@@ -278,6 +272,8 @@ void splitLogAfterStartup() {
         }
       }
     } while (retrySplitting);
+
+    return serverNames;
   }
 
   public void splitLog(final ServerName serverName) throws IOException {
@@ -361,11 +357,24 @@ public void splitLog(final List<ServerName> serverNames, PathFilter filter) thro
       return;
     }
 
+    boolean lockAcquired = false;
     if (distributedLogSplitting) {
-      splitLogManager.handleDeadWorkers(serverNames);
-      splitTime = EnvironmentEdgeManager.currentTimeMillis();
-      splitLogSize = splitLogManager.splitLogDistributed(logDirs,filter);
-      splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
+      try {
+        if (!this.services.isServerShutdownHandlerEnabled()) {
+          // process one log splitting task at one time before SSH is enabled.
+          // because ROOT SSH and HMaster#assignMeta could both log split a same server
+          this.splitLogLock.lock();
+          lockAcquired = true;
+        }
+        splitLogManager.handleDeadWorkers(serverNames);
+        splitTime = EnvironmentEdgeManager.currentTimeMillis();
+        splitLogSize = splitLogManager.splitLogDistributed(logDirs, filter);
+        splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
+      } finally {
+        if (lockAcquired) {
+          this.splitLogLock.unlock();
+        }
+      }
     } else {
       for(Path logDir: logDirs){
         // splitLogLock ensures that dead region servers' logs are processed
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 039702007613..1b6d29e3ca05 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -118,6 +118,12 @@ public class ServerManager {
    */
   private Set<ServerName> deadNotExpiredServers = new HashSet<ServerName>();
 
+  /**
+   * Flag to enable SSH for ROOT region server. It's used in master initialization to enable SSH for
+   * ROOT before META assignment.
+   */
+  private boolean isSSHForRootEnabled = false;
+
   /**
    * Constructor.
    * @param master
@@ -373,7 +379,8 @@ void letRegionServersShutdown() {
    * shutdown processing.
    */
   public synchronized void expireServer(final ServerName serverName) {
-    if (!services.isServerShutdownHandlerEnabled()) {
+    boolean carryingRoot = services.getAssignmentManager().isCarryingRoot(serverName);
+    if (!services.isServerShutdownHandlerEnabled() && (!carryingRoot || !this.isSSHForRootEnabled)) {
       LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
           + "delay expiring server " + serverName);
       this.deadNotExpiredServers.add(serverName);
@@ -382,7 +389,6 @@ public synchronized void expireServer(final ServerName serverName) {
     if (!this.onlineServers.containsKey(serverName)) {
       LOG.warn("Received expiration of " + serverName +
         " but server is not currently online");
-      return;
     }
     if (this.deadservers.contains(serverName)) {
       // TODO: Can this happen?  It shouldn't be online in this case?
@@ -410,7 +416,6 @@ public synchronized void expireServer(final ServerName serverName) {
       return;
     }
 
-    boolean carryingRoot = services.getAssignmentManager().isCarryingRoot(serverName);
     boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
     if (carryingRoot || carryingMeta) {
       this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
@@ -440,6 +445,33 @@ synchronized void expireDeadNotExpiredServers() throws IOException {
     }
   }
 
+  /**
+   * Enable SSH for ROOT region server and expire ROOT which died during master's initialization. It
+   * will be called before Meta assignment.
+   * @throws IOException
+   */
+  void enableSSHForRoot() throws IOException {
+    if (this.isSSHForRootEnabled) {
+      return;
+    }
+    this.isSSHForRootEnabled = true;
+    Iterator<ServerName> serverIterator = deadNotExpiredServers.iterator();
+    while (serverIterator.hasNext()) {
+      ServerName curServerName = serverIterator.next();
+      if (services.getAssignmentManager().isCarryingRoot(curServerName)) {
+        expireServer(curServerName);
+        serverIterator.remove();
+      }
+    }
+  }
+
+  /**
+   * Reset flag isSSHForRootEnabled to false
+   */
+  void disableSSHForRoot() {
+    this.isSSHForRootEnabled = false;
+  }
+
   /*
    * Remove the server from the drain list.
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
index 198d6f42bc54..a241e2058424 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
@@ -84,6 +84,13 @@ public void process() throws IOException {
           LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
         }
       }
+      
+      if(!this.services.isServerShutdownHandlerEnabled()) {
+        // resubmit in case we're in master initialization and SSH hasn't been enabled yet.
+        this.services.getExecutorService().submit(this);
+        this.deadServers.add(serverName);
+        return;
+      }
   
       // Carrying meta?
       if (isCarryingMeta()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 2be3a84dcfd5..41efd27191da 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.zookeeper.KeeperException;
 
@@ -116,6 +117,10 @@ public String toString() {
   public void process() throws IOException {
     final ServerName serverName = this.serverName;
     try {
+      if (this.server.isStopped()) {
+        throw new IOException("Server is stopped");
+      }
+
       try {
         if (this.shouldSplitHlog) {
           LOG.info("Splitting logs for " + serverName);
@@ -200,9 +205,15 @@ public void process() throws IOException {
         if (!this.services.getAssignmentManager().isRegionAssigned(hri)) {
           if (!regionsToAssign.contains(hri)) {
             regionsToAssign.add(hri);
+            RegionState rit =
+                services.getAssignmentManager().getRegionsInTransition().get(hri.getEncodedName());
+            removeRITsOfRregionInDisablingOrDisabledTables(regionsToAssign, rit,
+              services.getAssignmentManager(), hri);
           }
         }
       }
+
+      // re-assign regions
       for (HRegionInfo hri : regionsToAssign) {
         this.services.getAssignmentManager().assign(hri, true);
       }
@@ -244,12 +255,13 @@ private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Res
       }
     }
 
+    AssignmentManager assignmentManager = this.services.getAssignmentManager();
     for (Map.Entry<HRegionInfo, Result> e : metaHRIs.entrySet()) {
-      RegionState rit = services.getAssignmentManager().getRegionsInTransition().get(
-          e.getKey().getEncodedName());
-      AssignmentManager assignmentManager = this.services.getAssignmentManager();
+      RegionState rit =
+          assignmentManager.getRegionsInTransition().get(e.getKey().getEncodedName());
+
       if (processDeadRegion(e.getKey(), e.getValue(), assignmentManager,
-          this.server.getCatalogTracker())) {
+        this.server.getCatalogTracker())) {
         ServerName addressFromAM = assignmentManager.getRegionServerOfRegion(e.getKey());
         if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()
             && !ritsGoingToServer.contains(e.getKey())) {
@@ -268,7 +280,7 @@ private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Res
               ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), e.getKey());
             } catch (KeeperException ke) {
               this.server.abort("Unexpected ZK exception deleting unassigned node " + e.getKey(),
-                  ke);
+                ke);
               return null;
             }
           }
@@ -291,27 +303,25 @@ private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Res
       // CLOSING state. Doing this will have no harm. The rit can be null if region server went
       // down during master startup. In that case If any znodes' exists for partially disabled 
       // table regions deleting them during startup only. See HBASE-8127. 
-      toAssign =
-          checkForDisablingOrDisabledTables(ritsGoingToServer, toAssign, rit, e.getKey(),
-            assignmentManager);
+      removeRITsOfRregionInDisablingOrDisabledTables(toAssign, rit, assignmentManager, e.getKey());
     }
+
     return toAssign;
   }
 
-  private List<HRegionInfo> checkForDisablingOrDisabledTables(Set<HRegionInfo> regionsFromRIT,
-      List<HRegionInfo> toAssign, RegionState rit, HRegionInfo hri,
-      AssignmentManager assignmentManager) {
-    boolean disabled =
-        assignmentManager.getZKTable().isDisablingOrDisabledTable(hri.getTableNameAsString());
-    if (disabled) {
-      // To avoid region assignment if table is in disabling or disabled state.
-      toAssign.remove(hri);
-      regionsFromRIT.remove(hri);
+  private void removeRITsOfRregionInDisablingOrDisabledTables(List<HRegionInfo> toAssign,
+      RegionState rit, AssignmentManager assignmentManager, HRegionInfo hri) {
+
+    if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(hri.getTableNameAsString())) {
+      return;
     }
-    if (rit != null && disabled) {
+
+    // To avoid region assignment if table is in disabling or disabled state.
+    toAssign.remove(hri);
+
+    if (rit != null) {
       assignmentManager.deleteNodeAndOfflineRegion(hri);
     }
-    return toAssign;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index f94e0f13866a..cf47aa85ed25 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -67,12 +67,12 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
-import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterAddressTracker;
 import org.apache.hadoop.hbase.NotServingRegionException;
@@ -173,7 +173,6 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.joda.time.field.MillisDurationField;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
@@ -1782,7 +1781,9 @@ public CatalogTracker getCatalogTracker() {
   @Override
   public void stop(final String msg) {
     try {
-      this.rsHost.preStop(msg);
+      if (this.rsHost != null) {
+        this.rsHost.preStop(msg);
+      }
       this.stopped = true;
       LOG.info("STOPPED: " + msg);
       // Wakes run() if it is sleeping
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index c6e607ef1334..4365f78e7172 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -45,7 +45,7 @@ public abstract class ZooKeeperNodeTracker extends ZooKeeperListener {
   /** Used to abort if a fatal error occurs */
   protected final Abortable abortable;
 
-  private boolean stopped = false;
+  private volatile boolean stopped = false;
 
   /**
    * Constructs a new ZK node tracker.
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 288bd6ccbd41..109d94eeae49 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.security.User;
@@ -452,7 +453,9 @@ public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
     while (!(mts = getMasterThreads()).isEmpty()
         && (System.currentTimeMillis() - start) < timeout) {
       for (JVMClusterUtil.MasterThread mt : mts) {
-        if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) {
+        ServerManager serverManager = mt.getMaster().getServerManager();
+        if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()
+            && !serverManager.areDeadServersInProgress()) {
           return true;
         }
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 823d7da933de..83bd4c1f6a17 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -19,22 +19,29 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_mgr_wait_for_zk_delete;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_final_transistion_failed;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_done;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_err;
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_resigned;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,7 +50,13 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -54,8 +67,9 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -82,15 +96,21 @@ public class TestDistributedLogSplitting {
   Configuration conf;
   HBaseTestingUtility TEST_UTIL;
 
+
   private void startCluster(int num_rs) throws Exception{
+    conf = HBaseConfiguration.create();
+    startCluster(NUM_MASTERS, num_rs, conf);
+  }
+
+  private void startCluster(int num_master, int num_rs, Configuration inConf) throws Exception {
     ZKSplitLog.Counters.resetCounters();
     LOG.info("Starting cluster");
-    conf = HBaseConfiguration.create();
+    this.conf = inConf;
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
     TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(NUM_MASTERS, num_rs);
+    TEST_UTIL.startMiniCluster(num_master, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
@@ -102,6 +122,10 @@ private void startCluster(int num_rs) throws Exception{
 
   @After
   public void after() throws Exception {
+    for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
+      mt.getMaster().abort("closing...", new Exception("Trace info"));
+    }
+
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -205,6 +229,89 @@ public void testRecoveredEdits() throws Exception {
     assertEquals(NUM_LOG_LINES, count);
   }
 
+  @Test(timeout = 300000)
+  public void testMasterStartsUpWithLogSplittingWork() throws Exception {
+    LOG.info("testMasterStartsUpWithLogSplittingWork");
+    Configuration curConf = HBaseConfiguration.create();
+    curConf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
+    startCluster(2, NUM_RS, curConf);
+
+    final int NUM_REGIONS_TO_CREATE = 40;
+    final int NUM_LOG_LINES = 1000;
+    // turn off load balancing to prevent regions from moving around otherwise
+    // they will consume recovered.edits
+    master.balanceSwitch(false);
+
+    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
+    HTable ht = installTable(zkw, "table", "f", NUM_REGIONS_TO_CREATE);
+
+    List<HRegionInfo> regions = null;
+    HRegionServer hrs = null;
+    for (int i = 0; i < NUM_RS; i++) {
+      boolean isCarryingMeta = false;
+      hrs = rsts.get(i).getRegionServer();
+      regions = hrs.getOnlineRegions();
+      for (HRegionInfo region : regions) {
+        if (region.isRootRegion() || region.isMetaRegion()) {
+          isCarryingMeta = true;
+          break;
+        }
+      }
+      if (isCarryingMeta) {
+        continue;
+      }
+      break;
+    }
+
+    LOG.info("#regions = " + regions.size());
+    Iterator<HRegionInfo> it = regions.iterator();
+    while (it.hasNext()) {
+      HRegionInfo region = it.next();
+      if (region.isMetaTable()) {
+        it.remove();
+      }
+    }
+    makeHLog(hrs.getWAL(), regions, "table", NUM_LOG_LINES, 100);
+
+    // abort master
+    abortMaster(cluster);
+
+    // abort RS
+    int numRS = cluster.getLiveRegionServerThreads().size();
+    LOG.info("Aborting region server: " + hrs.getServerName());
+    hrs.abort("testing");
+
+    // wait for the RS dies
+    long start = EnvironmentEdgeManager.currentTimeMillis();
+    while (cluster.getLiveRegionServerThreads().size() > (numRS - 1)) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue(false);
+      }
+      Thread.sleep(200);
+    }
+
+    Thread.sleep(2000);
+    LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
+    
+    startMasterTillNoDeadServers(cluster);
+    
+    start = EnvironmentEdgeManager.currentTimeMillis();
+    while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue("Timedout", false);
+      }
+      Thread.sleep(200);
+    }
+
+    LOG.info("Current Open Regions After Master Node Starts Up:"
+        + getAllOnlineRegions(cluster).size());
+
+    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+
+    ht.close();
+  }
+
   /**
    * The original intention of this test was to force an abort of a region
    * server and to make sure that the failure path in the region servers is
@@ -393,33 +500,40 @@ public void makeHLog(HLog log,
       List<HRegionInfo> hris, String tname,
       int num_edits, int edit_size) throws IOException {
 
+    // remove root and meta region
+    hris.remove(HRegionInfo.ROOT_REGIONINFO);
+    hris.remove(HRegionInfo.FIRST_META_REGIONINFO);
     byte[] table = Bytes.toBytes(tname);
     HTableDescriptor htd = new HTableDescriptor(tname);
     byte[] value = new byte[edit_size];
     for (int i = 0; i < edit_size; i++) {
-      value[i] = (byte)('a' + (i % 26));
+      value[i] = (byte) ('a' + (i % 26));
     }
     int n = hris.size();
     int[] counts = new int[n];
-    int j = 0;
     if (n > 0) {
       for (int i = 0; i < num_edits; i += 1) {
         WALEdit e = new WALEdit();
-        byte [] row = Bytes.toBytes("r" + Integer.toString(i));
-        byte [] family = Bytes.toBytes("f");
-        byte [] qualifier = Bytes.toBytes("c" + Integer.toString(i));
-        e.add(new KeyValue(row, family, qualifier,
-            System.currentTimeMillis(), value));
-        j++;
-        log.append(hris.get(j % n), table, e, System.currentTimeMillis(), htd);
-        counts[j % n] += 1;
+        HRegionInfo curRegionInfo = hris.get(i % n);
+        byte[] startRow = curRegionInfo.getStartKey();
+        if (startRow == null || startRow.length == 0) {
+          startRow = new byte[] { 0, 0, 0, 0, 1 };
+        }
+        byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
+        row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because
+                                             // HBaseTestingUtility.createMultiRegions use 5 bytes
+                                             // key
+        byte[] family = Bytes.toBytes("f");
+        byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
+        e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
+        log.append(curRegionInfo, table, e, System.currentTimeMillis(), htd);
+        counts[i % n] += 1;
       }
     }
     log.sync();
     log.close();
     for (int i = 0; i < n; i++) {
-      LOG.info("region " + hris.get(i).getRegionNameAsString() +
-          " has " + counts[i] + " edits");
+      LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
     }
     return;
   }
@@ -479,6 +593,30 @@ private void waitForCounter(AtomicLong ctr, long oldval, long newval,
     assertTrue(false);
   }
 
+  private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException {
+    for (MasterThread mt : cluster.getLiveMasterThreads()) {
+      if (mt.getMaster().isActiveMaster()) {
+        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
+        mt.join();
+        break;
+      }
+    }
+    LOG.debug("Master is aborted");
+  }
+
+  private void startMasterTillNoDeadServers(MiniHBaseCluster cluster)
+      throws IOException, InterruptedException {
+    cluster.startMaster();
+    HMaster master = cluster.getMaster();
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    ServerManager serverManager = master.getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
+      Thread.sleep(100);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index a75aeeb709a5..6d1492d97e59 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -611,7 +611,7 @@ public void testMasterFailoverWithMockedRIT() throws Exception {
    * </ul>
    * @throws Exception
    */
-  @Test (timeout=180000)
+  @Test(timeout = 180000)
   public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
 
     final int NUM_MASTERS = 1;
@@ -1030,6 +1030,141 @@ public boolean isAborted() {
     TEST_UTIL.shutdownMiniCluster();
   }
 
+  @Test(timeout = 180000)
+  public void testRSKilledWithMockedOpeningRITGoingToDeadRS() throws Exception {
+    final int NUM_MASTERS = 1;
+    final int NUM_RS = 2;
+
+    // Create config to use for this cluster
+    Configuration conf = HBaseConfiguration.create();
+    // Need to drop the timeout much lower
+    conf.setInt("hbase.master.assignment.timeoutmonitor.period", 10000);
+    conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 30000);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
+    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
+
+    // Create and start the cluster
+    HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    log("Cluster started");
+
+    // Create a ZKW to use in the test
+    ZooKeeperWatcher zkw =
+        new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "unittest", new Abortable() {
+
+          @Override
+          public void abort(String why, Throwable e) {
+            LOG.error("Fatal ZK Error: " + why, e);
+            org.junit.Assert.assertFalse("Fatal ZK error", true);
+          }
+
+          @Override
+          public boolean isAborted() {
+            return false;
+          }
+
+        });
+
+    // get all the master threads
+    List<MasterThread> masterThreads = cluster.getMasterThreads();
+    assertEquals(1, masterThreads.size());
+
+    // only one master thread, let's wait for it to be initialized
+    assertTrue(cluster.waitForActiveAndReadyMaster());
+    HMaster master = masterThreads.get(0).getMaster();
+    assertTrue(master.isActiveMaster());
+    assertTrue(master.isInitialized());
+
+    // disable load balancing on this master
+    master.balanceSwitch(false);
+
+    // create two tables in META, each with 30 regions
+    byte[] FAMILY = Bytes.toBytes("family");
+    byte[][] SPLIT_KEYS =
+        TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 15);
+
+    FileSystem filesystem = FileSystem.get(conf);
+    Path rootdir = filesystem.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
+
+    byte[] disabledTable = Bytes.toBytes("disabledTable");
+    HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
+    htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
+    // Write the .tableinfo
+    FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
+    HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
+    createRegion(hriDisabled, rootdir, conf, htdDisabled);
+
+    List<HRegionInfo> tableRegions =
+        TEST_UTIL.createMultiRegionsInMeta(TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
+
+    log("Regions in META have been created");
+
+    // at this point we only expect 2 regions to be assigned out (catalogs)
+    assertEquals(2, cluster.countServedRegions());
+
+    // The first RS will stay online
+    List<RegionServerThread> regionservers = cluster.getRegionServerThreads();
+    HRegionServer hrs = regionservers.get(0).getRegionServer();
+
+    // The second RS is going to be hard-killed
+    RegionServerThread hrsDeadThread = regionservers.get(1);
+    HRegionServer hrsDead = hrsDeadThread.getRegionServer();
+    ServerName deadServerName = hrsDead.getServerName();
+
+    // we'll need some regions to already be assigned out properly on live RS
+    List<HRegionInfo> assignedRegionsOnLiveRS = new ArrayList<HRegionInfo>();
+    assignedRegionsOnLiveRS.addAll(tableRegions.subList(0, 3));
+    tableRegions.removeAll(assignedRegionsOnLiveRS);
+
+    // now actually assign them
+    for (HRegionInfo hri : assignedRegionsOnLiveRS) {
+      master.assignmentManager.regionPlans.put(hri.getEncodedName(),
+        new RegionPlan(hri, null, hrs.getServerName()));
+      master.assignRegion(hri);
+    }
+
+    log("Waiting for assignment to finish");
+    ZKAssign.blockUntilNoRIT(zkw);
+    master.assignmentManager.waitUntilNoRegionsInTransition(60000);
+    log("Assignment completed");
+
+    // Due to master.assignRegion(hri) could fail to assign a region to a specified RS
+    // therefore, we need make sure that regions are in the expected RS
+    verifyRegionLocation(hrs, assignedRegionsOnLiveRS);
+
+    assertTrue(" Table must be enabled.", master.getAssignmentManager().getZKTable()
+        .isEnabledTable("disabledTable"));
+
+    assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
+      assignedRegionsOnLiveRS.size() >= 1);
+
+    // Disable the disabledTable in ZK
+    ZKTable zktable = master.assignmentManager.getZKTable();
+    zktable.setDisablingTable("disabledTable");
+
+    // RS was opening a region of disabled table then died
+    HRegionInfo region = assignedRegionsOnLiveRS.remove(0);
+    master.assignmentManager.regionOffline(region);
+    master.assignmentManager.regionsInTransition.put(region.getEncodedName(), new RegionState(
+        region, RegionState.State.OPENING, System.currentTimeMillis(), deadServerName));
+    ZKAssign.createNodeOffline(zkw, region, deadServerName);
+    ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
+
+    // Kill the RS that had a hard death
+    log("Killing RS " + deadServerName);
+    hrsDead.abort("Killing for unit test");
+    while (hrsDeadThread.isAlive()) {
+      Threads.sleep(10);
+    }
+    log("RS " + deadServerName + " killed");
+
+    log("Waiting for no more RIT");
+    ZKAssign.blockUntilNoRIT(zkw);
+    log("No more RIT in ZK");
+    assertTrue(master.assignmentManager.waitUntilNoRegionsInTransition(120000));
+  }
+
   /**
    * Verify regions are on the expected region server
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index 3d1da30d1227..1689238a056d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -44,12 +44,10 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.TestMasterFailover;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -103,25 +101,12 @@ public TestingMaster(Configuration conf) throws IOException,
       super(conf);
     }
 
-    @Override
-    protected void splitLogAfterStartup(MasterFileSystem mfs) {
-      super.splitLogAfterStartup(mfs);
-      logSplit = true;
-      // If "TestingMaster.sleep" is set, sleep after log split.
-      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
-        int duration = getConfiguration().getInt(
-            "TestingMaster.sleep.duration", 0);
-        Threads.sleep(duration);
-      }
-    }
-
-
     public boolean isLogSplitAfterStartup() {
       return logSplit;
     }
   }
 
-  @Test(timeout = 120000)
+  @Test(timeout = 180000)
   public void testCorrectnessWhenMasterFailOver() throws Exception {
     final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
     final byte[] FAMILY = Bytes.toBytes("family");
@@ -164,7 +149,7 @@ public void testCorrectnessWhenMasterFailOver() throws Exception {
     /* NO.1 .META. region correctness */
     // First abort master
     abortMaster(cluster);
-    TestingMaster master = startMasterAndWaitUntilLogSplit(cluster);
+    TestingMaster master = startMasterAndWaitTillMetaRegionAssignment(cluster);
 
     // Second kill meta server
     int metaServerNum = cluster.getServerWithMeta();
@@ -195,7 +180,7 @@ public void testCorrectnessWhenMasterFailOver() throws Exception {
     if (rootServerNum != metaServerNum) {
       // First abort master
       abortMaster(cluster);
-      master = startMasterAndWaitUntilLogSplit(cluster);
+      master = startMasterAndWaitTillMetaRegionAssignment(cluster);
 
       // Second kill meta server
       HRegionServer rootRS = cluster.getRegionServer(rootServerNum);
@@ -215,6 +200,7 @@ public void testCorrectnessWhenMasterFailOver() throws Exception {
       assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
     }
 
+
     /* NO.3 data region correctness */
     ServerManager serverManager = cluster.getMaster().getServerManager();
     while (serverManager.areDeadServersInProgress()) {
@@ -274,7 +260,7 @@ public void testMasterFailoverWhenDisablingTableRegionsInRITOnDeadRS() throws Ex
 
     // Stop the master
     abortMaster(cluster);
-    master = startMasterAndWaitUntilLogSplit(cluster);
+    master = startMasterAndWaitTillMetaRegionAssignment(cluster);
     deadRS.kill();
     deadRS.join();
     waitUntilMasterIsInitialized(master);
@@ -302,14 +288,12 @@ private void abortMaster(MiniHBaseCluster cluster)
     LOG.debug("Master is aborted");
   }
 
-  private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
+  private TestingMaster startMasterAndWaitTillMetaRegionAssignment(MiniHBaseCluster cluster)
       throws IOException, InterruptedException {
     TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
-    while (!master.isLogSplitAfterStartup()) {
+    while (!master.isInitializationStartsMetaRegoinAssignment()) {
       Thread.sleep(100);
     }
-    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
-        + master.isInitialized());
     return master;
   }
 
@@ -318,6 +302,9 @@ private void waitUntilMasterIsInitialized(HMaster master)
     while (!master.isInitialized()) {
       Thread.sleep(100);
     }
+    while (master.getServerManager().areDeadServersInProgress()) {
+      Thread.sleep(100);
+    }
     LOG.debug("master isInitialized");
   }
   

From 740000f3b185179a3430c6cb00464d7787bfaa87 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 11 Apr 2013 10:31:14 +0000
Subject: [PATCH 0925/1540] HBASE-7658 grant with an empty string as permission
 should throw an exception (addendum)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1466826 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/security.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/ruby/hbase/security.rb b/src/main/ruby/hbase/security.rb
index 778e20ec9a89..5041ec4dbb5b 100644
--- a/src/main/ruby/hbase/security.rb
+++ b/src/main/ruby/hbase/security.rb
@@ -38,7 +38,7 @@ def grant(user, permissions, table_name=nil, family=nil, qualifier=nil)
 
       # Verify that the specified permission is valid
       if (permissions == nil || permissions.length == 0)
-        raise(ArgumentError, "Ivalid permission: no actions associated with user")
+        raise(ArgumentError, "Invalid permission: no actions associated with user")
       end
 
       if (table_name != nil)

From 4e46fb890b2534b86786fe874ef9aebaaa02ef0e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 11 Apr 2013 22:28:50 +0000
Subject: [PATCH 0926/1540] HBASE-7929 Reapply hbase-7507 'Make memstore flush
 be able to retry after exception' to 0.94 branch. (Original patch by chunhui
 shen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1467121 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   | 11 ++++
 .../hadoop/hbase/regionserver/Store.java      | 55 ++++++++++++++++++-
 2 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 53254e530fe6..e7a4e6e9745f 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -478,6 +478,17 @@ public static enum Modify {
    */
   public static long DEFAULT_HBASE_CLIENT_PAUSE = 1000;
 
+  /**
+   * Parameter name for server pause value, used mostly as value to wait before
+   * running a retry of a failed operation.
+   */
+  public static String HBASE_SERVER_PAUSE = "hbase.server.pause";
+
+  /**
+   * Default value of {@link #HBASE_SERVER_PAUSE}.
+   */
+  public static int DEFAULT_HBASE_SERVER_PAUSE = 1000;
+
   /**
    * Parameter name for maximum retries, used as maximum for all retryable
    * operations such as fetching of the root region from root region server,
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index ab5e1d56fde5..6b2f800727d0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -173,6 +173,10 @@ public class Store extends SchemaConfigured implements HeapSize {
 
   private final Compactor compactor;
 
+  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
+  private static int flush_retries_number;
+  private static int pauseTime;
+
   /**
    * Constructor
    * @param basedir qualified path under which the region directory lives;
@@ -259,6 +263,17 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
     this.bytesPerChecksum = getBytesPerChecksum(conf);
     // Create a compaction tool instance
     this.compactor = new Compactor(this.conf);
+    if (Store.flush_retries_number == 0) {
+      Store.flush_retries_number = conf.getInt(
+          "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
+      Store.pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE,
+          HConstants.DEFAULT_HBASE_SERVER_PAUSE);
+      if (Store.flush_retries_number <= 0) {
+        throw new IllegalArgumentException(
+            "hbase.hstore.flush.retries.number must be > 0, not "
+                + Store.flush_retries_number);
+      }
+    }
   }
 
   /**
@@ -755,8 +770,43 @@ protected Path flushCache(final long logCacheFlushId,
     // If an exception happens flushing, we let it out without clearing
     // the memstore snapshot.  The old snapshot will be returned when we say
     // 'snapshot', the next time flush comes around.
-    return internalFlushCache(
-        snapshot, logCacheFlushId, snapshotTimeRangeTracker, flushedSize, status);
+    // Retry after catching exception when flushing, otherwise server will abort
+    // itself
+    IOException lastException = null;
+    for (int i = 0; i < Store.flush_retries_number; i++) {
+      try {
+        Path pathName = internalFlushCache(snapshot, logCacheFlushId,
+            snapshotTimeRangeTracker, flushedSize, status);
+        try {
+          // Path name is null if there is no entry to flush
+          if (pathName != null) {
+            validateStoreFile(pathName);
+          }
+          return pathName;
+        } catch (Exception e) {
+          LOG.warn("Failed validating store file " + pathName
+              + ", retring num=" + i, e);
+          if (e instanceof IOException) {
+            lastException = (IOException) e;
+          } else {
+            lastException = new IOException(e);
+          }
+        }
+      } catch (IOException e) {
+        LOG.warn("Failed flushing store file, retring num=" + i, e);
+        lastException = e;
+      }
+      if (lastException != null) {
+        try {
+          Thread.sleep(pauseTime);
+        } catch (InterruptedException e) {
+          IOException iie = new InterruptedIOException();
+          iie.initCause(e);
+          throw iie;
+        }
+      }
+    }
+    throw lastException;
   }
 
   /*
@@ -875,7 +925,6 @@ private StoreFile commitFile(final Path path,
     // Write-out finished successfully, move into the right spot
     String fileName = path.getName();
     Path dstPath = new Path(homedir, fileName);
-    validateStoreFile(path);
     String msg = "Renaming flushed file at " + path + " to " + dstPath;
     LOG.debug(msg);
     status.setStatus("Flushing " + this + ": " + msg);

From 460f6a7338c69c234ceaafd83fc810141d00093b Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 12 Apr 2013 02:14:36 +0000
Subject: [PATCH 0927/1540] HBASE-8303. Increase the test timeout to 60s when
 they are less than 20s

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1467157 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/TestZooKeeper.java  |  2 +-
 .../hadoop/hbase/client/TestSnapshotFromAdmin.java   |  2 +-
 .../hadoop/hbase/client/TestSnapshotsFromAdmin.java  |  2 +-
 .../hadoop/hbase/constraint/TestConstraint.java      |  2 +-
 .../errorhandling/TestTimeoutExceptionInjector.java  |  6 +++---
 .../apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java |  2 +-
 .../hadoop/hbase/master/TestAssignmentManager.java   | 10 +++++-----
 .../hbase/master/cleaner/TestSnapshotFromMaster.java |  2 +-
 .../hbase/master/handler/TestCreateTableHandler.java |  2 +-
 .../apache/hadoop/hbase/procedure/TestProcedure.java |  8 ++++----
 .../hbase/procedure/TestProcedureCoordinator.java    | 12 ++++++------
 .../hadoop/hbase/procedure/TestProcedureMember.java  | 12 ++++++------
 .../hbase/procedure/TestZKProcedureControllers.java  |  8 ++++----
 .../regionserver/TestSplitTransactionOnCluster.java  |  2 +-
 .../hbase/snapshot/TestFlushSnapshotFromClient.java  |  2 +-
 .../apache/hadoop/hbase/thrift/TestCallQueue.java    |  4 ++--
 .../org/apache/hadoop/hbase/util/TestThreads.java    |  2 +-
 17 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index fa537bc97d1c..308beba555a7 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -367,7 +367,7 @@ public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
    *  test differs from {@link #testMasterSessionExpired} because here
    *  the master znode will exist in ZK.
    */
-  @Test(timeout=20000)
+  @Test(timeout=60000)
   public void testMasterZKSessionRecoveryFailure() throws Exception {
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     HMaster m = cluster.getMaster();
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
index ced199e66688..0ee5b609cee2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
@@ -50,7 +50,7 @@ public class TestSnapshotFromAdmin {
    * passed from the server ensures the correct overall waiting for the snapshot to finish.
    * @throws Exception
    */
-  @Test(timeout = 10000)
+  @Test(timeout = 60000)
   public void testBackoffLogic() throws Exception {
     final int maxWaitTime = 7500;
     final int numRetries = 10;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
index 12c575398b99..20b5edea1a7e 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
@@ -50,7 +50,7 @@ public class TestSnapshotsFromAdmin {
    * passed from the server ensures the correct overall waiting for the snapshot to finish.
    * @throws Exception
    */
-  @Test(timeout = 10000)
+  @Test(timeout = 60000)
   public void testBackoffLogic() throws Exception {
     final int maxWaitTime = 7500;
     final int numRetries = 10;
diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
index 47f7e33427db..e0f2569fb846 100644
--- a/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
+++ b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
@@ -94,7 +94,7 @@ public void testConstraintPasses() throws Exception {
    * @throws Exception
    */
   @SuppressWarnings("unchecked")
-  @Test(timeout = 10000)
+  @Test(timeout = 60000)
   public void testConstraintFails() throws Exception {
 
     // create the table
diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
index ff72d9174371..641dbe042014 100644
--- a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
+++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
@@ -37,7 +37,7 @@ public class TestTimeoutExceptionInjector {
   /**
    * Test that a manually triggered timer fires an exception.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testTimerTrigger() {
     final long time = 10000000; // pick a value that is very far in the future
     ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
@@ -64,7 +64,7 @@ public void testTimerPassesOnErrorInfo() {
    * Demonstrate TimeoutExceptionInjector semantics -- completion means no more exceptions passed to
    * error listener.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testStartAfterComplete() throws InterruptedException {
     final long time = 10;
     ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
@@ -84,7 +84,7 @@ public void testStartAfterComplete() throws InterruptedException {
    * Demonstrate TimeoutExceptionInjector semantics -- triggering fires exception and completes
    * the timer.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testStartAfterTrigger() throws InterruptedException {
     final long time = 10;
     ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
index 797a26d948ad..d0eb78bac171 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java
@@ -74,7 +74,7 @@ public EnumDescriptorProto exchangeProto(EnumDescriptorProto arg) {
     }
   }
 
-  @Test(timeout=10000)
+  @Test(timeout=60000)
   public void testCalls() throws Exception {
     testCallsInternal(conf);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index aa29ca346f2a..e3ebeaa6a230 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -167,7 +167,7 @@ public void after() throws KeeperException {
    * @throws KeeperException
    * @throws InterruptedException
    */
-  @Test(timeout = 5000)
+  @Test(timeout = 60000)
   public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
       throws IOException, KeeperException, InterruptedException {
     AssignmentManagerWithExtrasForTesting am =
@@ -214,7 +214,7 @@ public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
     }
   }
 
-  @Test(timeout = 5000)
+  @Test(timeout = 60000)
   public void testBalanceOnMasterFailoverScenarioWithClosedNode()
       throws IOException, KeeperException, InterruptedException {
     AssignmentManagerWithExtrasForTesting am =
@@ -262,7 +262,7 @@ public void testBalanceOnMasterFailoverScenarioWithClosedNode()
     }
   }
 
-  @Test(timeout = 5000)
+  @Test(timeout = 60000)
   public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
       throws IOException, KeeperException, InterruptedException {
     AssignmentManagerWithExtrasForTesting am =
@@ -327,7 +327,7 @@ private void createRegionPlanAndBalance(final AssignmentManager am,
    * @throws KeeperException
    * @throws InterruptedException
    */
-  @Test(timeout = 10000)
+  @Test(timeout = 60000)
   public void testBalance()
   throws IOException, KeeperException, InterruptedException {
     // Create and startup an executor.  This is used by AssignmentManager
@@ -724,7 +724,7 @@ public void testUnassignWithSplitAtSameTime() throws KeeperException, IOExceptio
    * situation
    * @throws ServiceException
    */
-  @Test(timeout = 5000)
+  @Test(timeout = 60000)
   public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
       throws IOException, KeeperException, InterruptedException, ServiceException {
     final RecoverableZooKeeper recoverableZk = Mockito
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 56db35826a3a..d67429ffd528 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -168,7 +168,7 @@ public static void cleanupTest() throws Exception {
    * <li>If asking about a snapshot has hasn't occurred, you should get an error.</li>
    * </ol>
    */
-  @Test(timeout = 15000)
+  @Test(timeout = 60000)
   public void testIsDoneContract() throws Exception {
 
     String snapshotName = "asyncExpectedFailureTest";
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
index 6900352d714b..1f85eb226fe8 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
@@ -99,7 +99,7 @@ public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() t
 
   }
   
-  @Test (timeout=10000)
+  @Test (timeout=60000)
   public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
     byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated");
     final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
index 4a249ba55f5f..4026f394da5c 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
@@ -82,7 +82,7 @@ public void sendGlobalBarrierComplete() {
    * With a single member, verify ordered execution.  The Coordinator side is run in a separate
    * thread so we can only trigger from members and wait for particular state latches.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testSingleMember() throws Exception {
     // The member
     List<String> members =  new ArrayList<String>();
@@ -126,7 +126,7 @@ public void run() {
     verify(procspy, never()).receive(any(ForeignException.class));
   }
 
-  @Test(timeout=1000)
+  @Test(timeout=60000)
   public void testMultipleMember() throws Exception {
     // 2 members
     List<String> members =  new ArrayList<String>();
@@ -178,7 +178,7 @@ public void run() {
     verify(procspy, never()).receive(any(ForeignException.class));
   }
 
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testErrorPropagation() throws Exception {
     List<String> members =  new ArrayList<String>();
     members.add("member");
@@ -203,7 +203,7 @@ public void run() {
     verify(procspy).sendGlobalBarrierComplete();
   }
 
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testBarrieredErrorPropagation() throws Exception {
     List<String> members =  new ArrayList<String>();
     members.add("member");
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
index 6bd46cce1c72..f48aa23bd680 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
@@ -116,7 +116,7 @@ public void testThreadPoolSize() throws Exception {
   /**
    * Check handling a connection failure correctly if we get it during the acquiring phase
    */
-  @Test(timeout = 5000)
+  @Test(timeout = 60000)
   public void testUnreachableControllerDuringPrepare() throws Exception {
     coordinator = buildNewCoordinator();
     // setup the proc
@@ -147,7 +147,7 @@ public void testUnreachableControllerDuringPrepare() throws Exception {
   /**
    * Check handling a connection failure correctly if we get it during the barrier phase
    */
-  @Test(timeout = 5000)
+  @Test(timeout = 60000)
   public void testUnreachableControllerDuringCommit() throws Exception {
     coordinator = buildNewCoordinator();
 
@@ -177,17 +177,17 @@ public void testUnreachableControllerDuringCommit() throws Exception {
         anyListOf(String.class));
   }
 
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testNoCohort() throws Exception {
     runSimpleProcedure();
   }
 
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testSingleCohortOrchestration() throws Exception {
     runSimpleProcedure("one");
   }
 
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testMultipleCohortOrchestration() throws Exception {
     runSimpleProcedure("one", "two", "three", "four");
   }
@@ -203,7 +203,7 @@ public void runSimpleProcedure(String... members) throws Exception {
   /**
    * Test that if nodes join the barrier early we still correctly handle the progress
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testEarlyJoiningBarrier() throws Exception {
     final String[] cohort = new String[] { "one", "two", "three", "four" };
     coordinator = buildNewCoordinator();
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
index b4c1b27f3510..5af2e403f030 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
@@ -124,7 +124,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
   /**
    * Test the normal sub procedure execution case.
    */
-  @Test(timeout = 500)
+  @Test(timeout = 60000)
   public void testSimpleRun() throws Exception {
     member = buildCohortMember();
     EmptySubprocedure subproc = new EmptySubprocedure(member, mockListener);
@@ -155,7 +155,7 @@ public void testSimpleRun() throws Exception {
    * Make sure we call cleanup etc, when we have an exception during
    * {@link Subprocedure#acquireBarrier()}.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testMemberPrepareException() throws Exception {
     buildCohortMemberPair();
 
@@ -190,7 +190,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
   /**
    * Make sure we call cleanup etc, when we have an exception during prepare.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testSendMemberAcquiredCommsFailure() throws Exception {
     buildCohortMemberPair();
 
@@ -229,7 +229,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
    * is checked.  Thus, the {@link Subprocedure#prepare} should succeed but later get rolled back
    * via {@link Subprocedure#cleanup}.
    */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testCoordinatorAbort() throws Exception {
     buildCohortMemberPair();
 
@@ -274,7 +274,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
    * member.  Members are then responsible for reading its TX log.  This implementation actually
    * rolls back, and thus breaks the normal TX guarantees.
   */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testMemberCommitException() throws Exception {
     buildCohortMemberPair();
 
@@ -315,7 +315,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
    * member.  Members are then responsible for reading its TX log.  This implementation actually
    * rolls back, and thus breaks the normal TX guarantees.
   */
-  @Test(timeout = 1000)
+  @Test(timeout = 60000)
   public void testMemberCommitCommsFailure() throws Exception {
     buildCohortMemberPair();
     final TimeoutException oate = new TimeoutException("bogus timeout",1,2,0);
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
index 66e4e901cc23..745d75aec897 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
@@ -74,7 +74,7 @@ public static void cleanupTest() throws Exception {
    * Smaller test to just test the actuation on the cohort member
    * @throws Exception on failure
    */
-  @Test(timeout = 15000)
+  @Test(timeout = 60000)
   public void testSimpleZKCohortMemberController() throws Exception {
     ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
     final String operationName = "instanceTest";
@@ -137,7 +137,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
     assertEquals("Didn't delete commit node", -1, ZKUtil.checkExists(watcher, commit));
   }
 
-  @Test(timeout = 15000)
+  @Test(timeout = 60000)
   public void testZKCoordinatorControllerWithNoCohort() throws Exception {
     final String operationName = "no cohort controller test";
     final byte[] data = new byte[] { 1, 2, 3 };
@@ -146,7 +146,7 @@ public void testZKCoordinatorControllerWithNoCohort() throws Exception {
     runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data);
   }
 
-  @Test(timeout = 15000)
+  @Test(timeout = 60000)
   public void testZKCoordinatorControllerWithSingleMemberCohort() throws Exception {
     final String operationName = "single member controller test";
     final byte[] data = new byte[] { 1, 2, 3 };
@@ -155,7 +155,7 @@ public void testZKCoordinatorControllerWithSingleMemberCohort() throws Exception
     runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data, "cohort");
   }
 
-  @Test(timeout = 15000)
+  @Test(timeout = 60000)
   public void testZKCoordinatorControllerMultipleCohort() throws Exception {
     final String operationName = "multi member controller test";
     final byte[] data = new byte[] { 1, 2, 3 };
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index eea0221458ee..3c1c9448ec25 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -744,7 +744,7 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
     }
   }
   
-  @Test(timeout = 20000)
+  @Test(timeout = 60000)
   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
     final byte[] tableName = 
         Bytes.toBytes("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index d22fb73fa67e..8c406c1658dc 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -210,7 +210,7 @@ public void testSnapshotFailsOnNonExistantTable() throws Exception {
     }
   }
 
-  @Test(timeout = 15000)
+  @Test(timeout = 60000)
   public void testAsyncFlushSnapshot() throws Exception {
     HBaseAdmin admin = UTIL.getHBaseAdmin();
     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java
index 671991e45935..1965b0508bae 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java
@@ -76,7 +76,7 @@ public TestCallQueue(int elementsAdded, int elementsRemoved) {
               " elementsRemoved:" + elementsRemoved);
   }
 
-  @Test(timeout=3000)
+  @Test(timeout=60000)
   public void testPutTake() throws Exception {
     ThriftMetrics metrics = createMetrics();
     CallQueue callQueue = new CallQueue(
@@ -90,7 +90,7 @@ public void testPutTake() throws Exception {
     verifyMetrics(metrics, "timeInQueue_num_ops", elementsRemoved);
   }
 
-  @Test(timeout=3000)
+  @Test(timeout=60000)
   public void testOfferPoll() throws Exception {
     ThriftMetrics metrics = createMetrics();
     CallQueue callQueue = new CallQueue(
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java b/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java
index 3bd39afb1295..6c249c746d5e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java
@@ -35,7 +35,7 @@ public class TestThreads {
 
   private volatile boolean wasInterrupted;
 
-  @Test(timeout=6000)
+  @Test(timeout=60000)
   public void testSleepWithoutInterrupt() throws InterruptedException {
     Thread sleeper = new Thread(new Runnable() {
       @Override

From 74b32794724d3f163962583f1ec0af4ea8fd61bf Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 13 Apr 2013 17:05:35 +0000
Subject: [PATCH 0928/1540] HBASE-8096 [replication] NPE while replicating a
 log that is acquiring a new block from HDFS

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1467660 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationHLogReaderManager.java    |  6 +++++-
 .../replication/regionserver/ReplicationSource.java   | 11 ++++++++---
 .../hbase/replication/TestReplicationSmallTests.java  |  2 +-
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
index 06b4935caf27..98ff1c699665 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -67,7 +67,11 @@ public HLog.Reader openReader(Path path) throws IOException {
       this.reader = HLog.getReader(this.fs, path, this.conf);
       this.lastPath = path;
     } else {
-      this.reader.reset();
+      try {
+        this.reader.reset();
+      } catch (NullPointerException npe) {
+        throw new IOException("NPE resetting reader, likely HDFS-4380", npe);
+      }
     }
     return this.reader;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index d4dfbf602522..c00f0580ce12 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -613,9 +613,14 @@ protected boolean openReader(int sleepMultiplier) {
     } catch (IOException ioe) {
       LOG.warn(peerClusterZnode + " Got: ", ioe);
       this.reader = null;
-      // TODO Need a better way to determinate if a file is really gone but
-      // TODO without scanning all logs dir
-      if (sleepMultiplier == this.maxRetriesMultiplier) {
+      if (ioe.getCause() instanceof NullPointerException) {
+        // Workaround for race condition in HDFS-4380
+        // which throws a NPE if we open a file before any data node has the most recent block
+        // Just sleep and retry.  Will require re-reading compressed HLogs for compressionContext.
+        LOG.warn("Got NPE opening reader, will retry.");
+      } else if (sleepMultiplier == this.maxRetriesMultiplier) {
+        // TODO Need a better way to determine if a file is really gone but
+        // TODO without scanning all logs dir  
         LOG.warn("Waited too long for this file, considering dumping");
         return !processEndOfFile();
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index d8a0f5dcd8f9..c1f0a5bd9a25 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -428,7 +428,7 @@ public void loadTesting() throws Exception {
     Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH);
     scanner.close();
 
-    assertEquals(NB_ROWS_IN_BATCH *10, res.length);
+    assertEquals(NB_ROWS_IN_BIG_BATCH, res.length);
 
     scan = new Scan();
 

From 79f6af240fa088c91088c03aca05c48814480274 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sun, 14 Apr 2013 15:18:32 +0000
Subject: [PATCH 0929/1540] HBASE-1936 ClassLoader that loads from hdfs; useful
 adding filters to classpath without having to restart services

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1467789 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/Get.java   |  18 +-
 .../org/apache/hadoop/hbase/client/Scan.java  |  17 +-
 .../hadoop/hbase/filter/SkipFilter.java       |  15 +-
 .../hadoop/hbase/filter/WhileMatchFilter.java |  15 +-
 .../org/apache/hadoop/hbase/util/Classes.java |  57 +++++
 .../hadoop/hbase/util/DynamicClassLoader.java | 218 ++++++++++++++++++
 .../apache/hadoop/hbase/client/TestGet.java   |  61 +++++
 .../hbase/util/TestDynamicClassLoader.java    | 216 +++++++++++++++++
 8 files changed, 565 insertions(+), 52 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index 0fb68385cac7..ad9a058831a5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -19,14 +19,13 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Classes;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -248,6 +247,7 @@ public boolean getCacheBlocks() {
    * Method for retrieving the get's RowLock
    * @return RowLock
    */
+  @SuppressWarnings("deprecation")
   public RowLock getRowLock() {
     return new RowLock(this.row, this.lockId);
   }
@@ -400,7 +400,8 @@ public void readFields(final DataInput in)
     this.maxVersions = in.readInt();
     boolean hasFilter = in.readBoolean();
     if (hasFilter) {
-      this.filter = (Filter)createForName(Bytes.toString(Bytes.readByteArray(in)));
+      this.filter = Classes.createWritableForName(
+        Bytes.toString(Bytes.readByteArray(in)));
       this.filter.readFields(in);
     }
     this.cacheBlocks = in.readBoolean();
@@ -458,15 +459,4 @@ public void write(final DataOutput out)
     }
     writeAttributes(out);
   }
-
-  @SuppressWarnings("unchecked")
-  private Writable createForName(String className) {
-    try {
-      Class<? extends Writable> clazz =
-        (Class<? extends Writable>) Class.forName(className);
-      return WritableFactories.newInstance(clazz, new Configuration());
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException("Can't find class " + className);
-    }
-  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 0518719826f8..553bde3d2858 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -20,14 +20,13 @@
 
 package org.apache.hadoop.hbase.client;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
 import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Classes;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -574,17 +573,6 @@ public Map<String, Object> toMap(int maxCols) {
     return map;
   }
 
-  @SuppressWarnings("unchecked")
-  private Writable createForName(String className) {
-    try {
-      Class<? extends Writable> clazz =
-        (Class<? extends Writable>) Class.forName(className);
-      return WritableFactories.newInstance(clazz, new Configuration());
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException("Can't find class " + className);
-    }
-  }
-
   //Writable
   public void readFields(final DataInput in)
   throws IOException {
@@ -599,7 +587,8 @@ public void readFields(final DataInput in)
     this.caching = in.readInt();
     this.cacheBlocks = in.readBoolean();
     if(in.readBoolean()) {
-      this.filter = (Filter)createForName(Bytes.toString(Bytes.readByteArray(in)));
+      this.filter = Classes.createWritableForName(
+        Bytes.toString(Bytes.readByteArray(in)));
       this.filter.readFields(in);
     }
     this.tr = new TimeRange();
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
index c27ac175d303..5fe17dfe176c 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
@@ -21,11 +21,11 @@
 package org.apache.hadoop.hbase.filter;
 
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Classes;
 
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.List;
 
 /**
  * A wrapper filter that filters an entire row if any of the KeyValue checks do
@@ -91,17 +91,8 @@ public void write(DataOutput out) throws IOException {
   }
 
   public void readFields(DataInput in) throws IOException {
-    String className = in.readUTF();
-    try {
-      this.filter = (Filter)(Class.forName(className).newInstance());
-      this.filter.readFields(in);
-    } catch (InstantiationException e) {
-      throw new RuntimeException("Failed deserialize.", e);
-    } catch (IllegalAccessException e) {
-      throw new RuntimeException("Failed deserialize.", e);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException("Failed deserialize.", e);
-    }
+    this.filter = Classes.createForName(in.readUTF());
+    this.filter.readFields(in);
   }
 
   public boolean isFamilyEssential(byte[] name) {
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
index 591a247c1e1e..bed8d5834df8 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
@@ -21,11 +21,11 @@
 package org.apache.hadoop.hbase.filter;
 
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Classes;
 
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.List;
 
 /**
  * A wrapper filter that returns true from {@link #filterAllRemaining()} as soon
@@ -92,17 +92,8 @@ public void write(DataOutput out) throws IOException {
   }
 
   public void readFields(DataInput in) throws IOException {
-    String className = in.readUTF();
-    try {
-      this.filter = (Filter)(Class.forName(className).newInstance());
-      this.filter.readFields(in);
-    } catch (InstantiationException e) {
-      throw new RuntimeException("Failed deserialize.", e);
-    } catch (IllegalAccessException e) {
-      throw new RuntimeException("Failed deserialize.", e);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException("Failed deserialize.", e);
-    }
+    this.filter = Classes.createForName(in.readUTF());
+    this.filter.readFields(in);
   }
 
   public boolean isFamilyEssential(byte[] name) {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Classes.java b/src/main/java/org/apache/hadoop/hbase/util/Classes.java
index 2b353e737b35..beca9c2f0ddf 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Classes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Classes.java
@@ -20,11 +20,27 @@
 
 package org.apache.hadoop.hbase.util;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.io.WritableFactories;
+
 /**
  * Utilities for class manipulation.
  */
 public class Classes {
 
+  /**
+   * Dynamic class loader to load filter/comparators
+   */
+  private final static ClassLoader CLASS_LOADER;
+
+  static {
+    ClassLoader parent = Classes.class.getClassLoader();
+    Configuration conf = HBaseConfiguration.create();
+    CLASS_LOADER = new DynamicClassLoader(conf, parent);
+  }
+
   /**
    * Equivalent of {@link Class#forName(String)} which also returns classes for
    * primitives like <code>boolean</code>, etc.
@@ -61,6 +77,7 @@ public static Class<?> extendedForName(String className)
     return valueType;
   }
 
+  @SuppressWarnings("rawtypes")
   public static String stringify(Class[] classes) {
     StringBuilder buf = new StringBuilder();
     if (classes != null) {
@@ -75,4 +92,44 @@ public static String stringify(Class[] classes) {
     }
     return buf.toString();
   }
+
+  /**
+   * Used to dynamically load a filter class, and create a Writable filter.
+   * This filter class most likely extends Configurable.
+   *
+   * @param className the filter class name.
+   * @return a filter
+   */
+  @SuppressWarnings("unchecked")
+  public static Filter createWritableForName(String className) {
+    try {
+      Class<? extends Filter> clazz =
+        (Class<? extends Filter>) Class.forName(className, true, CLASS_LOADER);
+      return (Filter)WritableFactories.newInstance(clazz, new Configuration());
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException("Can't find class " + className);
+    }
+  }
+
+  /**
+   * This method is almost the same as #createWritableForName, except
+   * that this one doesn't expect the filter class to extends Configurable.
+   *
+   * @param className the filter class name.
+   * @return a filter
+   */
+  @SuppressWarnings("unchecked")
+  public static Filter createForName(String className) {
+    try {
+      Class<? extends Filter> clazz =
+        (Class<? extends Filter>)Class.forName(className, true, CLASS_LOADER);
+      return (Filter)clazz.newInstance();
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException("Can't find class " + className);
+    } catch (InstantiationException e) {
+      throw new RuntimeException("Couldn't instantiate " + className, e);
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException("No access to " + className, e);
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java b/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
new file mode 100644
index 000000000000..01132ae8f746
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * This is a class loader that can load classes dynamically from new
+ * jar files under a configured folder. It always uses its parent class
+ * loader to load a class at first. Only if its parent class loader
+ * can not load a class, we will try to load it using the logic here.
+ * <p>
+ * We can't unload a class already loaded. So we will use the existing
+ * jar files we already know to load any class which can't be loaded
+ * using the parent class loader. If we still can't load the class from
+ * the existing jar files, we will check if any new jar file is added,
+ * if so, we will load the new jar file and try to load the class again.
+ * If still failed, a class not found exception will be thrown.
+ * <p>
+ * Be careful in uploading new jar files and make sure all classes
+ * are consistent, otherwise, we may not be able to load your
+ * classes properly.
+ */
+@InterfaceAudience.Private
+public class DynamicClassLoader extends URLClassLoader {
+  private static final Log LOG =
+      LogFactory.getLog(DynamicClassLoader.class);
+
+  // Dynamic jars are put under ${hbase.local.dir}/dynamic/jars/
+  private static final String DYNAMIC_JARS_DIR = File.separator
+    + "dynamic" + File.separator + "jars" + File.separator;
+
+  /**
+   * Parent class loader used to load any class at first.
+   */
+  private final ClassLoader parent;
+
+  private File localDir;
+
+  // FileSystem of the remote path, set only if remoteDir != null
+  private FileSystem remoteDirFs;
+  private Path remoteDir;
+
+  // Last modified time of local jars
+  private HashMap<String, Long> jarModifiedTime;
+
+  /**
+   * Creates a DynamicClassLoader that can load classes dynamically
+   * from jar files under a specific folder.
+   *
+   * @param conf the configuration for the cluster.
+   * @param parent the parent ClassLoader to set.
+   */
+  public DynamicClassLoader(
+      final Configuration conf, final ClassLoader parent) {
+    super(new URL[]{}, parent);
+    this.parent = parent;
+
+    jarModifiedTime = new HashMap<String, Long>();
+    String localDirPath = conf.get("hbase.local.dir") + DYNAMIC_JARS_DIR;
+    localDir = new File(localDirPath);
+    if (!localDir.mkdirs() && !localDir.isDirectory()) {
+      throw new RuntimeException("Failed to create local dir " + localDir.getPath()
+        + ", DynamicClassLoader failed to init");
+    }
+
+    String remotePath = conf.get("hbase.dynamic.jars.dir");
+    if (remotePath == null || remotePath.equals(localDirPath)) {
+      remoteDir = null;  // ignore if it is the same as the local path
+    } else {
+      remoteDir = new Path(remotePath);
+      try {
+        remoteDirFs = remoteDir.getFileSystem(conf);
+      } catch (IOException ioe) {
+        LOG.warn("Failed to identify the fs of dir "
+          + remoteDir + ", ignored", ioe);
+        remoteDir = null;
+      }
+    }
+  }
+
+  @Override
+  public Class<?> loadClass(String name)
+      throws ClassNotFoundException {
+    try {
+      return parent.loadClass(name);
+    } catch (ClassNotFoundException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Class " + name + " not found - using dynamical class loader");
+      }
+
+      // Check whether the class has already been loaded:
+      Class<?> clasz = findLoadedClass(name);
+      if (clasz != null) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Class " + name + " already loaded");
+        }
+      }
+      else {
+        try {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Finding class: " + name);
+          }
+          clasz = findClass(name);
+        } catch (ClassNotFoundException cnfe) {
+          // Load new jar files if any
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Loading new jar files, if any");
+          }
+          loadNewJars();
+
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Finding class again: " + name);
+          }
+          clasz = findClass(name);
+        }
+      }
+      return clasz;
+    }
+  }
+
+  private synchronized void loadNewJars() {
+    // Refresh local jar file lists
+    for (File file: localDir.listFiles()) {
+      String fileName = file.getName();
+      if (jarModifiedTime.containsKey(fileName)) {
+        continue;
+      }
+      if (file.isFile() && fileName.endsWith(".jar")) {
+        jarModifiedTime.put(fileName, Long.valueOf(file.lastModified()));
+        try {
+          URL url = file.toURI().toURL();
+          addURL(url);
+        } catch (MalformedURLException mue) {
+          // This should not happen, just log it
+          LOG.warn("Failed to load new jar " + fileName, mue);
+        }
+      }
+    }
+
+    // Check remote files
+    FileStatus[] statuses = null;
+    if (remoteDir != null) {
+      try {
+        statuses = remoteDirFs.listStatus(remoteDir);
+      } catch (IOException ioe) {
+        LOG.warn("Failed to check remote dir status " + remoteDir, ioe);
+      }
+    }
+    if (statuses == null || statuses.length == 0) {
+      return; // no remote files at all
+    }
+
+    for (FileStatus status: statuses) {
+      if (status.isDir()) continue; // No recursive lookup
+      Path path = status.getPath();
+      String fileName = path.getName();
+      if (!fileName.endsWith(".jar")) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Ignored non-jar file " + fileName);
+        }
+        continue; // Ignore non-jar files
+      }
+      Long cachedLastModificationTime = jarModifiedTime.get(fileName);
+      if (cachedLastModificationTime != null) {
+        long lastModified = status.getModificationTime();
+        if (lastModified < cachedLastModificationTime.longValue()) {
+          // There could be some race, for example, someone uploads
+          // a new one right in the middle the old one is copied to
+          // local. We can check the size as well. But it is still
+          // not guaranteed. This should be rare. Most likely,
+          // we already have the latest one.
+          // If you are unlucky to hit this race issue, you have
+          // to touch the remote jar to update its last modified time
+          continue;
+        }
+      }
+      try {
+        // Copy it to local
+        File dst = new File(localDir, fileName);
+        remoteDirFs.copyToLocalFile(path, new Path(dst.getPath()));
+        jarModifiedTime.put(fileName, Long.valueOf(dst.lastModified()));
+        URL url = dst.toURI().toURL();
+        addURL(url);
+      } catch (IOException ioe) {
+        LOG.warn("Failed to load new jar " + fileName, ioe);
+      }
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index 93c59ef48997..1f584a517ebb 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -20,23 +20,55 @@
 
 package org.apache.hadoop.hbase.client;
 
+import static org.junit.Assert.fail;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.io.ByteStreams;
+import com.sun.org.apache.xml.internal.security.utils.Base64;
+
 // TODO: cover more test cases
 @Category(SmallTests.class)
 public class TestGet {
+
+  private static final String WRITABLE_GET =
+    "AgD//////////wAAAAEBD3Rlc3QuTW9ja0ZpbHRlcgEAAAAAAAAAAH//////////AQAAAAAAAAAA";
+
+  private static final String MOCK_FILTER_JAR =
+    "UEsDBBQACAgIACmBi0IAAAAAAAAAAAAAAAAJAAQATUVUQS1JTkYv/soAAAMAUEsHCAAAAAACAAAA" +
+    "AAAAAFBLAwQUAAgICAApgYtCAAAAAAAAAAAAAAAAFAAAAE1FVEEtSU5GL01BTklGRVNULk1G803M" +
+    "y0xLLS7RDUstKs7Mz7NSMNQz4OVyLkpNLElN0XWqBAmY6xnEG1gqaPgXJSbnpCo45xcV5BcllgCV" +
+    "a/Jy8XIBAFBLBwgxyqRbQwAAAEQAAABQSwMECgAACAAAbICLQgAAAAAAAAAAAAAAAAUAAAB0ZXN0" +
+    "L1BLAwQUAAgICAAcgItCAAAAAAAAAAAAAAAAFQAAAHRlc3QvTW9ja0ZpbHRlci5jbGFzc41Qy07C" +
+    "QBS9A4VKBZGHoO7cgQvHmLjCuPBBQlJloWE/tCMdLZ1mOlV/y5WJCz/AjzLeDqCRYOIs7uuce87N" +
+    "fHy+vQPAEezakCNQ1TzR9Ep6D30Raq5ssAh0pZpQFjMv4DRgvpQxDcYs4fTOcOiMeoYTAsUTEQl9" +
+    "SiDf6Y4IWOfS5w7koVSGAhTRwBURv06nY65u2TjEjborPRaOmBJZPx9aOhAJgZq7dE+PgKM48/uC" +
+    "hz4SWh33nj0yKiS9YJoNojjVvczYuXz2eKyFjBIb6gQaC9pg+I2gDVOTQwRXiBAoPCmh8Zb2b49h" +
+    "qhcmzVUAet/IVHkcL8bt6s/xBxkb9gA/B7KXxwo/BaONHcVMMBf2X2HtBYscOBiLZliCdYzlGQFz" +
+    "BTOBDagiaxNrC7uakTk2m4guS1SMRGsGziWyqgFN47xlsH+K1f4UaxuxbcPf+QJQSwcI8UIYqlEB" +
+    "AABeAgAAUEsBAhQAFAAICAgAKYGLQgAAAAACAAAAAAAAAAkABAAAAAAAAAAAAAAAAAAAAE1FVEEt" +
+    "SU5GL/7KAABQSwECFAAUAAgICAApgYtCMcqkW0MAAABEAAAAFAAAAAAAAAAAAAAAAAA9AAAATUVU" +
+    "QS1JTkYvTUFOSUZFU1QuTUZQSwECCgAKAAAIAABsgItCAAAAAAAAAAAAAAAABQAAAAAAAAAAAAAA" +
+    "AADCAAAAdGVzdC9QSwECFAAUAAgICAAcgItC8UIYqlEBAABeAgAAFQAAAAAAAAAAAAAAAADlAAAA" +
+    "dGVzdC9Nb2NrRmlsdGVyLmNsYXNzUEsFBgAAAAAEAAQA8wAAAHkCAAAAAA==";
+
   @Test
   public void testAttributesSerialization() throws IOException {
     Get get = new Get();
@@ -107,6 +139,35 @@ public void testGetAttributes() {
     Assert.assertNull(get.getAttributesMap().get("attribute1"));
   }
 
+  @Test
+  public void testDynamicFilter() throws Exception {
+    DataInput dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET));
+    Get get = new Get();
+    try {
+      get.readFields(dis);
+      fail("Should not be able to load the filter class");
+    } catch (RuntimeException re) {
+      String msg = re.getMessage();
+      Assert.assertTrue(msg != null
+        && msg.contains("Can't find class test.MockFilter"));
+    }
+
+    Configuration conf = HBaseConfiguration.create();
+    String localPath = conf.get("hbase.local.dir") + File.separator
+      + "dynamic" + File.separator + "jars" + File.separator;
+    File jarFile = new File(localPath, "MockFilter.jar");
+    jarFile.deleteOnExit();
+
+    FileOutputStream fos = new FileOutputStream(jarFile);
+    fos.write(Base64.decode(MOCK_FILTER_JAR));
+    fos.close();
+
+    dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET));
+    get.readFields(dis);
+    Assert.assertEquals("test.MockFilter",
+      get.getFilter().getClass().getName());
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java b/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
new file mode 100644
index 000000000000..ab5e788585e7
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
@@ -0,0 +1,216 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test TestDynamicClassLoader
+ */
+@Category(SmallTests.class)
+public class TestDynamicClassLoader {
+  private static final Log LOG = LogFactory.getLog(TestDynamicClassLoader.class);
+
+  private static final Configuration conf = HBaseConfiguration.create();
+
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  static {
+    conf.set("hbase.dynamic.jars.dir", TEST_UTIL.getDataTestDir().toString());
+  }
+
+  // generate jar file
+  private boolean createJarArchive(File archiveFile, File[] tobeJared) {
+    try {
+      byte buffer[] = new byte[4096];
+      // Open archive file
+      FileOutputStream stream = new FileOutputStream(archiveFile);
+      JarOutputStream out = new JarOutputStream(stream, new Manifest());
+
+      for (int i = 0; i < tobeJared.length; i++) {
+        if (tobeJared[i] == null || !tobeJared[i].exists()
+            || tobeJared[i].isDirectory()) {
+          continue;
+        }
+
+        // Add archive entry
+        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
+        jarAdd.setTime(tobeJared[i].lastModified());
+        out.putNextEntry(jarAdd);
+
+        // Write file to archive
+        FileInputStream in = new FileInputStream(tobeJared[i]);
+        while (true) {
+          int nRead = in.read(buffer, 0, buffer.length);
+          if (nRead <= 0)
+            break;
+          out.write(buffer, 0, nRead);
+        }
+        in.close();
+      }
+      out.close();
+      stream.close();
+      LOG.info("Adding classes to jar file completed");
+      return true;
+    } catch (Exception ex) {
+      LOG.error("Error: " + ex.getMessage());
+      return false;
+    }
+  }
+
+  private File buildJar(
+      String className, String folder) throws Exception {
+    String javaCode = "public class " + className + " {}";
+    Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
+    File srcDirPath = new File(srcDir.toString());
+    srcDirPath.mkdirs();
+    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
+    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
+    bw.write(javaCode);
+    bw.close();
+
+    // compile it by JavaCompiler
+    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+    ArrayList<String> srcFileNames = new ArrayList<String>();
+    srcFileNames.add(sourceCodeFile.toString());
+    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
+      null);
+    Iterable<? extends JavaFileObject> cu =
+      fm.getJavaFileObjects(sourceCodeFile);
+    List<String> options = new ArrayList<String>();
+    options.add("-classpath");
+    // only add hbase classes to classpath. This is a little bit tricky: assume
+    // the classpath is {hbaseSrc}/target/classes.
+    String currentDir = new File(".").getAbsolutePath();
+    String classpath =
+        currentDir + File.separator + "target"+ File.separator + "classes" +
+        System.getProperty("path.separator") + System.getProperty("java.class.path");
+    options.add(classpath);
+    LOG.debug("Setting classpath to: "+classpath);
+
+    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
+      options, null, cu);
+    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
+
+    // build a jar file by the classes files
+    String jarFileName = className + ".jar";
+    File jarFile = new File(folder, jarFileName);
+    if (!createJarArchive(jarFile,
+        new File[]{new File(srcDir.toString(), className + ".class")})){
+      assertTrue("Build jar file failed.", false);
+    }
+    return jarFile;
+  }
+
+  @Test
+  public void testLoadClassFromLocalPath() throws Exception {
+    ClassLoader parent = TestDynamicClassLoader.class.getClassLoader();
+    DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
+
+    String className = "TestLoadClassFromLocalPath";
+    try {
+      classLoader.loadClass(className);
+      fail("Should not be able to load class " + className);
+    } catch (ClassNotFoundException cnfe) {
+      // expected, move on
+    }
+
+    try {
+      buildJar(className, localDirPath());
+      classLoader.loadClass(className);
+    } catch (ClassNotFoundException cnfe) {
+      LOG.error("Should be able to load class " + className, cnfe);
+      fail(cnfe.getMessage());
+    } finally {
+      deleteClass(className);
+    }
+  }
+
+  @Test
+  public void testLoadClassFromAnotherPath() throws Exception {
+    ClassLoader parent = TestDynamicClassLoader.class.getClassLoader();
+    DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
+
+    String className = "TestLoadClassFromAnotherPath";
+    try {
+      classLoader.loadClass(className);
+      fail("Should not be able to load class " + className);
+    } catch (ClassNotFoundException cnfe) {
+      // expected, move on
+    }
+
+    try {
+      buildJar(className, TEST_UTIL.getDataTestDir().toString());
+      classLoader.loadClass(className);
+    } catch (ClassNotFoundException cnfe) {
+      LOG.error("Should be able to load class " + className, cnfe);
+      fail(cnfe.getMessage());
+    } finally {
+      deleteClass(className);
+    }
+  }
+
+  private String localDirPath() {
+    return conf.get("hbase.local.dir") + File.separator
+      + "dynamic" + File.separator + "jars" + File.separator;
+  }
+
+  private void deleteClass(String className) throws Exception {
+    String jarFileName = className + ".jar";
+    File file = new File(TEST_UTIL.getDataTestDir().toString(), jarFileName);
+    file.deleteOnExit();
+
+    file = new File(conf.get("hbase.dynamic.jars.dir"), jarFileName);
+    file.deleteOnExit();
+
+    file = new File(localDirPath(), jarFileName);
+    file.deleteOnExit();
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}

From 0a8b6cb7e128beadc433a82a4d3b1ac8f9627df5 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sun, 14 Apr 2013 15:49:14 +0000
Subject: [PATCH 0930/1540] HBASE-1936 ClassLoader that loads from hdfs; useful
 adding filters to classpath without having to restart services - ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1467793 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/client/TestGet.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index 1f584a517ebb..eb5d136c50a4 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -36,13 +36,13 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 import com.google.common.io.ByteStreams;
-import com.sun.org.apache.xml.internal.security.utils.Base64;
 
 // TODO: cover more test cases
 @Category(SmallTests.class)

From eccba9812eba36fe2fde7d94f2a97a5afa28f58a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 15 Apr 2013 18:45:16 +0000
Subject: [PATCH 0931/1540] HBASE-7801 Allow a deferred sync option per
 Mutation.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468179 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/Durability.java       |  65 +++++++
 .../apache/hadoop/hbase/client/Mutation.java  |  33 +++-
 .../hadoop/hbase/regionserver/HRegion.java    |  57 +++++-
 .../regionserver/wal/TestDurability.java      | 168 ++++++++++++++++++
 4 files changed, 313 insertions(+), 10 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/client/Durability.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Durability.java b/src/main/java/org/apache/hadoop/hbase/client/Durability.java
new file mode 100644
index 000000000000..82995a691357
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/client/Durability.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+/**
+ * Enum describing the durability guarantees for {@link Mutation}
+ * Note that the items must be sorted in order of increasing durability
+ */
+public enum Durability {
+  /**
+   * Use the column family's default setting to determine durability.
+   * This must remain the first option.
+   */
+  USE_DEFAULT,
+  /**
+   * Do not write the Mutation to the WAL
+   */
+  SKIP_WAL,
+  /**
+   * Write the Mutation to the WAL asynchronously
+   */
+  ASYNC_WAL,
+  /**
+   * Write the Mutation to the WAL synchronously.
+   * The data is flushed to the filesystem implementation, but not necessarily to disk.
+   * For HDFS this will flush the data to the designated number of DataNodes.
+   * See <a href="https://issues.apache.org/jira/browse/HADOOP-6313">HADOOP-6313<a/>
+   */
+  SYNC_WAL,
+  /**
+   * Write the Mutation to the WAL synchronously and force the entries to disk.
+   * (Note: this is currently not supported and will behave identical to {@link #SYNC_WAL})
+   * See <a href="https://issues.apache.org/jira/browse/HADOOP-6313">HADOOP-6313<a/>
+   */
+  FSYNC_WAL;
+
+  // efficiently translate ordinal back to items of this Enum
+  // (Enum.values()[ordinal] generates too much garbage)
+  public static Durability valueOf(int ordinal) {
+    switch (ordinal) {
+      case 0: return USE_DEFAULT;
+      case 1: return SKIP_WAL;
+      case 2: return ASYNC_WAL;
+      case 3: return SYNC_WAL;
+      case 4: return FSYNC_WAL;
+      default: throw new IllegalArgumentException("Unknown Durability Ordinal:"+ordinal);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 812b560d5df4..730dee50eaa0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -27,13 +27,17 @@
 import java.util.TreeMap;
 import java.util.UUID;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 
 public abstract class Mutation extends OperationWithAttributes implements Row {
+  private static final Log LOG = LogFactory.getLog(Mutation.class);
   // Attribute used in Mutations to indicate the originating cluster.
   private static final String CLUSTER_ID_ATTR = "_c.id_";
+  private static final String DURABILITY_ID_ATTR = "_dur_";
 
   protected byte [] row = null;
   protected long ts = HConstants.LATEST_TIMESTAMP;
@@ -110,6 +114,7 @@ public Map<String, Object> toMap(int maxCols) {
   }
 
   /**
+   * @deprecated Use {@link #getDurability()} instead.
    * @return true if edits should be applied to WAL, false if not
    */
   public boolean getWriteToWAL() {
@@ -119,10 +124,36 @@ public boolean getWriteToWAL() {
   /**
    * Set whether this Delete should be written to the WAL or not.
    * Not writing the WAL means you may lose edits on server crash.
+   * This method will reset any changes made via {@link #setDurability(Durability)} 
    * @param write true if edits should be written to WAL, false if not
+   * @deprecated Use {@link #setDurability(Durability)} instead.
    */
   public void setWriteToWAL(boolean write) {
-    this.writeToWAL = write;
+    setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
+  }
+
+  /**
+   * Set the durability for this mutation.
+   * Note that RegionServers prior to 0.94.7 will only honor {@link Durability#SKIP_WAL}.
+   * This method will reset any changes made via {@link #setWriteToWAL(boolean)} 
+   * @param d
+   */
+  public void setDurability(Durability d) {
+    setAttribute(DURABILITY_ID_ATTR, Bytes.toBytes(d.ordinal()));
+    this.writeToWAL = d != Durability.SKIP_WAL;
+  }
+
+  /** Get the current durability */
+  public Durability getDurability() {
+    byte[] attr = getAttribute(DURABILITY_ID_ATTR);
+    if (attr != null) {
+      try {
+        return Durability.valueOf(Bytes.toInt(attr));
+      } catch (IllegalArgumentException iax) {
+        LOG.warn("Invalid or unknown durability settting", iax);
+      }
+    }
+    return writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index bef10b2bccaf..e5e885178b66 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -84,6 +84,7 @@
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
@@ -2380,6 +2381,8 @@ private long doMiniBatchMutation(
       // ------------------------------------
       // STEP 4. Build WAL edit
       // ----------------------------------
+      Durability durability = Durability.USE_DEFAULT;
+
       for (int i = firstIndex; i < lastIndexExclusive; i++) {
         // Skip puts that were determined to be invalid during preprocessing
         if (batchOp.retCodeDetails[i].getOperationStatusCode()
@@ -2389,12 +2392,17 @@ private long doMiniBatchMutation(
         batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
 
         Mutation m = batchOp.operations[i].getFirst();
-        if (!m.getWriteToWAL()) {
+        Durability tmpDur = m.getDurability(); 
+        if (tmpDur.ordinal() > durability.ordinal()) {
+          durability = tmpDur;
+        }
+        if (tmpDur == Durability.SKIP_WAL) {
           if (m instanceof Put) {
             recordPutWithoutWal(m.getFamilyMap());
           }
           continue;
         }
+
         // Add WAL edits by CP
         WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
         if (fromCP != null) {
@@ -2429,7 +2437,7 @@ private long doMiniBatchMutation(
       // STEP 7. Sync wal.
       // -------------------------
       if (walEdit.size() > 0) {
-        syncOrDefer(txid);
+        syncOrDefer(txid, durability);
       }
       walSyncSuccessful = true;
       // calling the post CP hook for batch mutation
@@ -4856,6 +4864,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
 
       long now = EnvironmentEdgeManager.currentTimeMillis();
       byte[] byteNow = Bytes.toBytes(now);
+      Durability durability = Durability.USE_DEFAULT;
       try {
         // 5. Check mutations and apply edits to a single WALEdit
         for (Mutation m : mutations) {
@@ -4873,7 +4882,11 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
                 "Action must be Put or Delete. But was: "
                     + m.getClass().getName());
           }
-          if (m.getWriteToWAL()) {
+          Durability tmpDur = m.getDurability(); 
+          if (tmpDur.ordinal() > durability.ordinal()) {
+            durability = tmpDur;
+          }
+          if (tmpDur != Durability.SKIP_WAL) {
             addFamilyMapToWALEdit(m.getFamilyMap(), walEdit);
           }
         }
@@ -4904,7 +4917,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
 
         // 9. sync WAL if required
         if (walEdit.size() > 0) {
-          syncOrDefer(txid);
+          syncOrDefer(txid, durability);
         }
         walSyncSuccessful = true;
 
@@ -5122,7 +5135,8 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
         releaseRowLock(lid);
       }
       if (writeToWAL) {
-        syncOrDefer(txid); // sync the transaction log outside the rowlock
+        // sync the transaction log outside the rowlock
+        syncOrDefer(txid, append.getDurability());
       }
     } finally {
       closeRegionOperation();
@@ -5269,7 +5283,8 @@ public Result increment(Increment increment, Integer lockid,
         releaseRowLock(lid);
       }
       if (writeToWAL) {
-        syncOrDefer(txid); // sync the transaction log outside the rowlock
+        // sync the transaction log outside the rowlock
+        syncOrDefer(txid, Durability.USE_DEFAULT);
       }
     } finally {
       closeRegionOperation();
@@ -5366,7 +5381,8 @@ public long incrementColumnValue(byte [] row, byte [] family,
         releaseRowLock(lid);
       }
       if (writeToWAL) {
-        syncOrDefer(txid); // sync the transaction log outside the rowlock
+        // sync the transaction log outside the rowlock
+        syncOrDefer(txid, Durability.USE_DEFAULT);
       }
     } finally {
       closeRegionOperation();
@@ -5835,9 +5851,32 @@ private void lock(final Lock lock, final int multiplier)
    * @param txid should sync up to which transaction
    * @throws IOException If anything goes wrong with DFS
    */
-  private void syncOrDefer(long txid) throws IOException {
-    if (this.getRegionInfo().isMetaRegion() || !isDeferredLogSyncEnabled()) {
+  private void syncOrDefer(long txid, Durability durability) throws IOException {
+    if (this.getRegionInfo().isMetaRegion()) {
       this.log.sync(txid);
+    } else {
+      switch(durability) {
+      case USE_DEFAULT:
+        // do what CF defaults to
+        if (!isDeferredLogSyncEnabled()) {
+          this.log.sync(txid);
+        }
+        break;
+      case SKIP_WAL:
+        // nothing do to
+        break;
+      case ASYNC_WAL:
+        // defer the sync, unless we globally can't
+        if (this.deferredLogSyncDisabled) {
+          this.log.sync(txid);
+        }
+        break;
+      case SYNC_WAL:
+      case FSYNC_WAL:
+        // sync the WAL edit (SYNC and FSYNC treated the same for now)
+        this.log.sync(txid);
+        break;
+      }
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java
new file mode 100644
index 000000000000..53086364543f
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests for HLog write durability
+ */
+@Category(MediumTests.class)
+public class TestDurability {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static FileSystem FS;
+  private static MiniDFSCluster CLUSTER;
+  private static Configuration CONF;
+  private static final Path DIR = TEST_UTIL.getDataTestDir("TestDurability");
+
+  private static byte[] FAMILY = Bytes.toBytes("family");
+  private static byte[] ROW = Bytes.toBytes("row");
+  private static byte[] COL = Bytes.toBytes("col");
+
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    CONF = TEST_UTIL.getConfiguration();
+    CONF.setLong("hbase.regionserver.optionallogflushinterval", 500*1000);
+    TEST_UTIL.startMiniDFSCluster(1);
+
+    CLUSTER = TEST_UTIL.getDFSCluster();
+    FS = CLUSTER.getFileSystem();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testDurability() throws Exception {
+    HLog wal = new HLog(FS, new Path(DIR, "hlogdir"),
+        new Path(DIR, "hlogdir_archive"), CONF);
+    byte[] tableName = Bytes.toBytes("TestDurability");
+    HRegion region = createHRegion(tableName, "region", wal, false);
+    HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, true);
+
+    region.put(newPut(null));
+
+    verifyHLogCount(wal, 1);
+
+    // a put through the deferred table does not write to the wal immdiately
+    deferredRegion.put(newPut(null));
+    verifyHLogCount(wal, 1);
+    // but will after we sync the wal
+    wal.sync();
+    verifyHLogCount(wal, 2);
+
+    // a put through a deferred table will be sync with the put sync'ed put
+    deferredRegion.put(newPut(null));
+    verifyHLogCount(wal, 2);
+    region.put(newPut(null));
+    verifyHLogCount(wal, 4);
+
+    // a put through a deferred table will be sync with the put sync'ed put
+    deferredRegion.put(newPut(Durability.USE_DEFAULT));
+    verifyHLogCount(wal, 4);
+    region.put(newPut(Durability.USE_DEFAULT));
+    verifyHLogCount(wal, 6);
+
+    // SKIP_WAL never writes to the wal
+    region.put(newPut(Durability.SKIP_WAL));
+    deferredRegion.put(newPut(Durability.SKIP_WAL));
+    verifyHLogCount(wal, 6);
+    wal.sync();
+    verifyHLogCount(wal, 6);
+
+    // async overrides sync table default
+    region.put(newPut(Durability.ASYNC_WAL));
+    deferredRegion.put(newPut(Durability.ASYNC_WAL));
+    verifyHLogCount(wal, 6);
+    wal.sync();
+    verifyHLogCount(wal, 8);
+
+    // sync overrides async table default
+    region.put(newPut(Durability.SYNC_WAL));
+    deferredRegion.put(newPut(Durability.SYNC_WAL));
+    verifyHLogCount(wal, 10);
+
+    // fsync behaves like sync
+    region.put(newPut(Durability.FSYNC_WAL));
+    deferredRegion.put(newPut(Durability.FSYNC_WAL));
+    verifyHLogCount(wal, 12);
+  }
+
+  private Put[] newPut(Durability durability) {
+    Put p = new Put(ROW);
+    p.add(FAMILY, COL, COL);
+    if (durability != null) {
+      p.setDurability(durability);
+    }
+    return new Put[]{p};
+  }
+
+  private void verifyHLogCount(HLog log, int expected) throws Exception {
+    Path walPath = log.computeFilename();
+    HLog.Reader reader = HLog.getReader(FS, walPath, CONF);
+    int count = 0;
+    HLog.Entry entry = new HLog.Entry();
+    while (reader.next(entry) != null) count++;
+    reader.close();
+    assertEquals(expected, count);
+  }
+
+  // lifted from TestAtomicOperation
+  private HRegion createHRegion (byte [] tableName, String callingMethod, HLog log, boolean isDeferredLogFlush)
+    throws IOException {
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.setDeferredLogFlush(isDeferredLogFlush);
+      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+      htd.addFamily(hcd);
+      HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+      Path path = new Path(DIR + callingMethod);
+      if (FS.exists(path)) {
+        if (!FS.delete(path, true)) {
+          throw new IOException("Failed delete of " + path);
+        }
+      }
+      return HRegion.createHRegion(info, path, HBaseConfiguration.create(), htd, log);
+    }
+
+}

From db72ac3d23ed7a11e97d8763201d5123b56ae507 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 15 Apr 2013 20:14:09 +0000
Subject: [PATCH 0932/1540] HBASE-8326 mapreduce.TestTableInputFormatScan times
 out frequently (Nick Dimiduk)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468207 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TableMapReduceUtil.java   |  84 +++++---
 .../mapreduce/TestTableInputFormatScan1.java  |  99 ++++++++++
 .../mapreduce/TestTableInputFormatScan2.java  | 117 +++++++++++
 ...java => TestTableInputFormatScanBase.java} | 187 ++----------------
 4 files changed, 291 insertions(+), 196 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java
 rename src/test/java/org/apache/hadoop/hbase/mapreduce/{TestTableInputFormatScan.java => TestTableInputFormatScanBase.java} (64%)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index ceacb7182977..175a1eda7d33 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -30,9 +30,13 @@
 import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -535,32 +539,33 @@ public static void addDependencyJars(Job job) throws IOException {
    * the DistributedCache.
    */
   public static void addDependencyJars(Configuration conf,
-      Class... classes) throws IOException {
+      Class<?>... classes) throws IOException {
 
     FileSystem localFs = FileSystem.getLocal(conf);
-
     Set<String> jars = new HashSet<String>();
-
     // Add jars that are already in the tmpjars variable
-    jars.addAll( conf.getStringCollection("tmpjars") );
+    jars.addAll(conf.getStringCollection("tmpjars"));
+
+    // add jars as we find them to a map of contents jar name so that we can avoid
+    // creating new jars for classes that have already been packaged.
+    Map<String, String> packagedClasses = new HashMap<String, String>();
 
     // Add jars containing the specified classes
-    for (Class clazz : classes) {
+    for (Class<?> clazz : classes) {
       if (clazz == null) continue;
 
-      String pathStr = findOrCreateJar(clazz);
-      if (pathStr == null) {
+      Path path = findOrCreateJar(clazz, localFs, packagedClasses);
+      if (path == null) {
         LOG.warn("Could not find jar for class " + clazz +
                  " in order to ship it to the cluster.");
         continue;
       }
-      Path path = new Path(pathStr);
       if (!localFs.exists(path)) {
         LOG.warn("Could not validate jar file " + path + " for class "
                  + clazz);
         continue;
       }
-      jars.add(path.makeQualified(localFs).toString());
+      jars.add(path.toString());
     }
     if (jars.isEmpty()) return;
 
@@ -574,17 +579,22 @@ public static void addDependencyJars(Configuration conf,
    * a directory in the classpath, it creates a Jar on the fly with the
    * contents of the directory and returns the path to that Jar. If a Jar is
    * created, it is created in the system temporary directory. Otherwise,
-   * returns an existing jar that contains a class of the same name.
+   * returns an existing jar that contains a class of the same name. Maintains
+   * a mapping from jar contents to the tmp jar created.
    * @param my_class the class to find.
+   * @param fs the FileSystem with which to qualify the returned path.
+   * @param packagedClasses a map of class name to path.
    * @return a jar file that contains the class.
    * @throws IOException
    */
-  private static String findOrCreateJar(Class<?> my_class)
+  private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
+      Map<String, String> packagedClasses)
   throws IOException {
     // attempt to locate an existing jar for the class.
-    String jar = findContainingJar(my_class);
+    String jar = findContainingJar(my_class, packagedClasses);
     if (null == jar || jar.isEmpty()) {
       jar = getJar(my_class);
+      updateMap(jar, packagedClasses);
     }
 
     if (null == jar || jar.isEmpty()) {
@@ -592,23 +602,45 @@ private static String findOrCreateJar(Class<?> my_class)
     }
 
     LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
-    return jar;
+    return new Path(jar).makeQualified(fs);
   }
 
   /**
-   * Find a jar that contains a class of the same name, if any.
-   * It will return a jar file, even if that is not the first thing
-   * on the class path that has a class with the same name.
-   * 
-   * This is shamelessly copied from JobConf
-   * 
+   * Add entries to <code>packagedClasses</code> corresponding to class files
+   * contained in <code>jar</code>.
+   * @param jar The jar who's content to list.
+   * @param packagedClasses map[class -> jar]
+   */
+  private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
+    ZipFile zip = null;
+    try {
+      zip = new ZipFile(jar);
+      for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
+        ZipEntry entry = iter.nextElement();
+        if (entry.getName().endsWith("class")) {
+          packagedClasses.put(entry.getName(), jar);
+        }
+      }
+    } finally {
+      if (null != zip) zip.close();
+    }
+  }
+
+  /**
+   * Find a jar that contains a class of the same name, if any. It will return
+   * a jar file, even if that is not the first thing on the class path that
+   * has a class with the same name. Looks first on the classpath and then in
+   * the <code>packagedClasses</code> map.
    * @param my_class the class to find.
    * @return a jar file that contains the class, or null.
    * @throws IOException
    */
-  private static String findContainingJar(Class<?> my_class) throws IOException {
+  private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
+      throws IOException {
     ClassLoader loader = my_class.getClassLoader();
     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+
+    // first search the classpath
     for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
       URL url = itr.nextElement();
       if ("jar".equals(url.getProtocol())) {
@@ -627,14 +659,18 @@ private static String findContainingJar(Class<?> my_class) throws IOException {
         return toReturn.replaceAll("!.*$", "");
       }
     }
+
+    // now look in any jars we've packaged using JarFinder
+    for (Map.Entry<String, String> e : packagedClasses.entrySet()) {
+      if (e.getKey().equals(class_file)) return e.getValue();
+    }
     return null;
   }
 
   /**
-   * Invoke 'getJar' on a JarFinder implementation. Useful for some job configuration
-   * contexts (HBASE-8140) and also for testing on MRv2. First check if we have
-   * HADOOP-9426. Lacking that, fall back to the backport.
-   *
+   * Invoke 'getJar' on a JarFinder implementation. Useful for some job
+   * configuration contexts (HBASE-8140) and also for testing on MRv2. First
+   * check if we have HADOOP-9426. Lacking that, fall back to the backport.
    * @param my_class the class to find.
    * @return a jar file that contains the class, or null.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java
new file mode 100644
index 000000000000..77ea47a290e5
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java
@@ -0,0 +1,99 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * TestTableInputFormatScan part 1.
+ * @see TestTableInputFormatScanBase
+ */
+@Category(LargeTests.class)
+public class TestTableInputFormatScan1 extends TestTableInputFormatScanBase {
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToEmpty()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan(null, null, null);
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToAPP()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan(null, "app", "apo");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToBBA()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan(null, "bba", "baz");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToBBB()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan(null, "bbb", "bba");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToOPP()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan(null, "opp", "opo");
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java
new file mode 100644
index 000000000000..f35bbd112083
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java
@@ -0,0 +1,117 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * TestTableInputFormatScan part 2.
+ * @see TestTableInputFormatScanBase
+ */
+@Category(LargeTests.class)
+public class TestTableInputFormatScan2 extends TestTableInputFormatScanBase {
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanOBBToOPP()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan("obb", "opp", "opo");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanOBBToQPP()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan("obb", "qpp", "qpo");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanOPPToEmpty()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan("opp", null, "zzz");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanYYXToEmpty()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan("yyx", null, "zzz");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanYYYToEmpty()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan("yyy", null, "zzz");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   *
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanYZYToEmpty()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScan("yzy", null, "zzz");
+  }
+
+  @Test
+  public void testScanFromConfiguration()
+  throws IOException, InterruptedException, ClassNotFoundException {
+    testScanFromConfiguration("bba", "bbd", "bbc");
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
similarity index 64%
rename from src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan.java
rename to src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
index afead7d4a897..15ea4986388e 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -19,7 +18,9 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
-import java.io.File;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -27,10 +28,8 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -40,25 +39,23 @@
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
+ * <p>
  * Tests various scan start and stop row scenarios. This is set in a scan and
  * tested in a MapReduce job to see if that is handed over and done properly
  * too.
+ * </p>
+ * <p>
+ * This test is broken into two parts in order to side-step the test timeout
+ * period of 900, as documented in HBASE-8326.
+ * </p>
  */
-@Category(LargeTests.class)
-public class TestTableInputFormatScan {
+public abstract class TestTableInputFormatScanBase {
 
-  static final Log LOG = LogFactory.getLog(TestTableInputFormatScan.class);
+  static final Log LOG = LogFactory.getLog(TestTableInputFormatScanBase.class);
   static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
   static final byte[] TABLE_NAME = Bytes.toBytes("scantest");
@@ -165,155 +162,6 @@ protected void cleanup(Context context)
 
   }
 
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanEmptyToEmpty()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan(null, null, null);
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanEmptyToAPP()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan(null, "app", "apo");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanEmptyToBBA()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan(null, "bba", "baz");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanEmptyToBBB()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan(null, "bbb", "bba");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanEmptyToOPP()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan(null, "opp", "opo");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanOBBToOPP()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan("obb", "opp", "opo");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanOBBToQPP()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan("obb", "qpp", "qpo");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanOPPToEmpty()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan("opp", null, "zzz");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanYYXToEmpty()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan("yyx", null, "zzz");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanYYYToEmpty()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan("yyy", null, "zzz");
-  }
-
-  /**
-   * Tests a MR scan using specific start and stop rows.
-   *
-   * @throws IOException
-   * @throws ClassNotFoundException
-   * @throws InterruptedException
-   */
-  @Test
-  public void testScanYZYToEmpty()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScan("yzy", null, "zzz");
-  }
-
-  @Test
-  public void testScanFromConfiguration()
-  throws IOException, InterruptedException, ClassNotFoundException {
-    testScanFromConfiguration("bba", "bbd", "bbc");
-  }
-
   /**
    * Tests an MR Scan initialized from properties set in the Configuration.
    * 
@@ -321,7 +169,7 @@ public void testScanFromConfiguration()
    * @throws ClassNotFoundException
    * @throws InterruptedException
    */
-  private void testScanFromConfiguration(String start, String stop, String last)
+  protected void testScanFromConfiguration(String start, String stop, String last)
   throws IOException, InterruptedException, ClassNotFoundException {
     String jobName = "ScanFromConfig" + (start != null ? start.toUpperCase() : "Empty") +
       "To" + (stop != null ? stop.toUpperCase() : "Empty");
@@ -347,8 +195,7 @@ private void testScanFromConfiguration(String start, String stop, String last)
     job.setInputFormatClass(TableInputFormat.class);
     job.setNumReduceTasks(1);
     FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
-    job.waitForCompletion(true);
-    assertTrue(job.isComplete());
+    assertTrue(job.waitForCompletion(true));
   }
 
   /**
@@ -358,7 +205,7 @@ private void testScanFromConfiguration(String start, String stop, String last)
    * @throws ClassNotFoundException
    * @throws InterruptedException
    */
-  private void testScan(String start, String stop, String last)
+  protected void testScan(String start, String stop, String last)
   throws IOException, InterruptedException, ClassNotFoundException {
     String jobName = "Scan" + (start != null ? start.toUpperCase() : "Empty") +
       "To" + (stop != null ? stop.toUpperCase() : "Empty");
@@ -383,13 +230,9 @@ private void testScan(String start, String stop, String last)
     job.setNumReduceTasks(1); // one to get final "first" and "last" key
     FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
     LOG.info("Started " + job.getJobName());
-    job.waitForCompletion(true);
-    assertTrue(job.isComplete());
+    assertTrue(job.waitForCompletion(true));
     LOG.info("After map/reduce completion - job " + jobName);
   }
 
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
 

From 03bee0b45a04ae15eaaa80f69b669fb62532fed7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 15 Apr 2013 20:41:23 +0000
Subject: [PATCH 0933/1540] HBASE-8285 HBaseClient never recovers for single
 HTable.get() calls with no retries when regions move (Varun Sharma)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468218 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HConnection.java      |  6 ++++
 .../hbase/client/HConnectionManager.java      | 29 ++++++++++++++++---
 .../hadoop/hbase/client/ServerCallable.java   |  5 ++++
 3 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index f3bd9fa0c5e6..1774a8938142 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -162,6 +162,12 @@ public HRegionLocation locateRegion(final byte [] tableName,
    */
   public void clearRegionCache(final byte [] tableName);
 
+  /**
+   * Deletes cached locations for the specific region.
+   * @param location The location object for the region, to be purged from cache.
+   */
+  public void deleteCachedRegionLocation(final HRegionLocation location);
+
   /**
    * Find the location of the region of <i>tableName</i> that <i>row</i>
    * lives in, ignoring any value that might be in the cache.
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index a9a51f912dd3..80bf48cf903b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1172,11 +1172,10 @@ HRegionLocation getCachedLocation(final byte [] tableName,
      */
     void deleteCachedLocation(final byte [] tableName, final byte [] row) {
       synchronized (this.cachedRegionLocations) {
-        Map<byte[], HRegionLocation> tableLocations =
-            getTableLocations(tableName);
-        // start to examine the cache. we can only do cache actions
-        // if there's something in the cache for this table.
+        Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
         if (!tableLocations.isEmpty()) {
+          // start to examine the cache. we can only do cache actions
+          // if there's something in the cache for this table.
           HRegionLocation rl = getCachedLocation(tableName, row);
           if (rl != null) {
             tableLocations.remove(rl.getRegionInfo().getStartKey());
@@ -1191,6 +1190,28 @@ void deleteCachedLocation(final byte [] tableName, final byte [] row) {
       }
     }
 
+    @Override
+    public void deleteCachedRegionLocation(final HRegionLocation location) {
+      if (location == null) {
+        return;
+      }
+      synchronized (this.cachedRegionLocations) {
+        byte[] tableName = location.getRegionInfo().getTableName();
+        Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
+        if (!tableLocations.isEmpty()) {
+          // Delete if there's something in the cache for this region.
+          HRegionLocation removedLocation =
+              tableLocations.remove(location.getRegionInfo().getStartKey());
+          if (LOG.isDebugEnabled() && removedLocation != null) {
+            LOG.debug("Removed " +
+              location.getRegionInfo().getRegionNameAsString() +
+              " for tableName=" + Bytes.toString(tableName) +
+              " from cache");
+          }
+        }
+      }
+    }
+
     @Override
     public void clearCaches(String sn) {
       clearCachedLocationForServer(sn);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
index fcf43e755a26..f238eb04a24d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -174,6 +175,10 @@ public T withRetries()
           if (hrl != null) {
             getConnection().clearCaches(hrl.getHostnamePort());
           }
+        } else if (t instanceof NotServingRegionException && numRetries == 1) {
+          // Purge cache entries for this specific region from META cache
+          // since we don't call connect(true) when number of retries is 1.
+          getConnection().deleteCachedRegionLocation(location);
         }
         RetriesExhaustedException.ThrowableWithExtraContext qt =
           new RetriesExhaustedException.ThrowableWithExtraContext(t,

From 7384462bafc35c63fa5f3bdee7bd8b052e1b60a1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 15 Apr 2013 23:44:39 +0000
Subject: [PATCH 0934/1540] CHANGES.txt and pom.xml for 0.94.7RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468273 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 7f105ca39b22..18042c516c3b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,94 @@
 HBase Change Log
 
+Release 0.94.7 - 4/15/2013
+Sub-task
+
+    [HBASE-7615] - Add metrics for snapshots
+    [HBASE-7801] - Allow a deferred sync option per Mutation.
+    [HBASE-8210] - Backport the LoadTest portions of HBASE-7383
+    [HBASE-8316] - JoinedHeap for non essential column families should reseek instead of seek
+
+Bug
+
+    [HBASE-7401] - Remove warning message about running 'hbase migrate'
+    [HBASE-7658] - grant with an empty string as permission should throw an exception
+    [HBASE-7817] - Suggested JDWP debug options in hbase-env.sh are wrong
+    [HBASE-7824] - Improve master start up time when there is log splitting work
+    [HBASE-7925] - Back port HBASE-6881 into 0.94
+    [HBASE-7961] - truncate on disabled table should throw TableNotEnabledException.
+    [HBASE-8014] - Backport HBASE-6915 to 0.94.
+    [HBASE-8030] - znode path of online region servers is hard coded in rolling_restart.sh
+    [HBASE-8044] - split/flush/compact/major_compact from hbase shell does not work for region key with \x format
+    [HBASE-8081] - Backport HBASE-7213 (separate hlog for meta tables) to 0.94
+    [HBASE-8092] - bulk assignment in 0.94 doesn't handle ZK errors very well
+    [HBASE-8096] - [replication] NPE while replicating a log that is acquiring a new block from HDFS
+    [HBASE-8118] - TestTablePermission depends on the execution order
+    [HBASE-8125] - HBASE-7435 breaks BuiltInGzipDecompressor on Hadoop < 1.0.x
+    [HBASE-8127] - Region of a disabling or disabled table could be stuck in transition state when RS dies during Master initialization
+    [HBASE-8128] - HTable#put improvements
+    [HBASE-8131] - Create table handler needs to handle failure cases.
+    [HBASE-8142] - Sporadic TestZKProcedureControllers failures on trunk
+    [HBASE-8146] - IntegrationTestBigLinkedList does not work on distributed setup
+    [HBASE-8150] - server should not produce RAITE for already-opening region in 0.94 (because master retry logic handles this case poorly)
+    [HBASE-8151] - Decode memstoreTS in HFileReaderV2 only when necessary
+    [HBASE-8158] - Backport HBASE-8140 "TableMapReduceUtils#addDependencyJar fails when nested inside another MR job"
+    [HBASE-8160] - HMaster#move doesn't check if master initialized
+    [HBASE-8166] - Avoid writing the memstoreTS into HFiles when possible
+    [HBASE-8169] - TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS may fail due to regions randomly assigned to a RS
+    [HBASE-8170] - HbaseAdmin.createTable cannot handle creating three regions
+    [HBASE-8176] - Backport HBASE-5335 "Dynamic Schema Configurations" to 0.94
+    [HBASE-8179] - JSON formatting for cluster status is sort of broken
+    [HBASE-8188] - Avoid unnecessary row compare in StoreScanner
+    [HBASE-8192] - Logic errror causes infinite loop in HRegion.bulkLoadHFiles(List)
+    [HBASE-8207] - Replication could have data loss when machine name contains hyphen "-"
+    [HBASE-8208] - In some situations data is not replicated to slaves when deferredLogSync is enabled
+    [HBASE-8211] - Support for NN HA for 0.94
+    [HBASE-8212] - Introduce a new separator instead of hyphen('-') for renaming recovered queues' znodes
+    [HBASE-8213] - global authorization may lose efficacy
+    [HBASE-8215] - Removing existing .regioninfo in writeRegioninfoOnFilesystem
+    [HBASE-8222] - User class should implement equals() and hashCode()
+    [HBASE-8225] - [replication] minor code bug when registering ReplicationLogCleaner
+    [HBASE-8226] - HBaseTestingUtility#waitUntilAllRegionsAssigned won't return if it counts "too many" regions
+    [HBASE-8229] - Replication code logs like crazy if a target table cannot be found.
+    [HBASE-8230] - Possible NPE on regionserver abort if replication service has not been started
+    [HBASE-8231] - delete tests in table_tests.rb(TestShell) always running on empty table.
+    [HBASE-8232] - TestAccessController occasionally fails with IndexOutOfBoundsException
+    [HBASE-8246] - Backport HBASE-6318 to 0.94 where SplitLogWorker exits due to ConcurrentModificationException
+    [HBASE-8259] - Snapshot backport in 0.94.6 breaks rolling restarts
+    [HBASE-8266] - Master cannot start if TableNotFoundException is thrown while partial table recovery
+    [HBASE-8270] - Backport HBASE-8097 'MetaServerShutdownHandler may potentially keep bumping up DeadServer.numProcessing' to 0.94
+    [HBASE-8274] - Backport to 94: HBASE-7488 Implement HConnectionManager.locateRegions which is currently returning null
+    [HBASE-8276] - Backport hbase-6738 to 0.94 "Too aggressive task resubmission from the distributed log manager"
+    [HBASE-8285] - HBaseClient never recovers for single HTable.get() calls with no retries when regions move
+    [HBASE-8288] - HBaseFileSystem: Refactoring and correct semantics for createPath methods
+    [HBASE-8303] - Increse the test timeout to 60s when they are less than 20s
+    [HBASE-8313] - Add Bloom filter testing for HFileOutputFormat
+    [HBASE-8326] - mapreduce.TestTableInputFormatScan times out frequently
+
+Improvement
+
+    [HBASE-7599] - Port HBASE-6066 (low hanging read path improvements) to 0.94
+    [HBASE-8148] - Allow IPC to bind on a specific address
+    [HBASE-8152] - Avoid creating empty reference file when splitkey is outside the key range of a store file
+    [HBASE-8174] - Backport HBASE-8161(setting blocking file count on table level doesn't work) to 0.94
+    [HBASE-8198] - Backport HBASE-8063(Filter HFiles based on first/last key) into 0.94
+    [HBASE-8199] - Eliminate exception for ExportSnapshot against the null table snapshot (with no data in)
+    [HBASE-8209] - Improve LoadTest extensibility
+
+New Feature
+
+    [HBASE-1936] - ClassLoader that loads from hdfs; useful adding filters to classpath without having to restart services
+    [HBASE-7415] - [snapshots] Add task information to snapshot operation
+
+Task
+
+    [HBASE-7929] - Reapply hbase-7507 "Make memstore flush be able to retry after exception" to 0.94 branch.
+
+Test
+
+    [HBASE-8106] - Test to check replication log znodes move is done correctly
+
+
 Release 0.94.6 - 3/14/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 1f9dbbb4205d..2f898812fe8d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.6</version>
+  <version>0.94.7</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From a91eaeb1cc7890a824f22170415a1b567c6d4a68 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 16 Apr 2013 20:58:01 +0000
Subject: [PATCH 0935/1540] HBASE-8326 Addendum simplifies lookup in
 packagedClasses (Nick)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468607 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 175a1eda7d33..cac8dd47900c 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -660,11 +660,9 @@ private static String findContainingJar(Class<?> my_class, Map<String, String> p
       }
     }
 
-    // now look in any jars we've packaged using JarFinder
-    for (Map.Entry<String, String> e : packagedClasses.entrySet()) {
-      if (e.getKey().equals(class_file)) return e.getValue();
-    }
-    return null;
+    // now look in any jars we've packaged using JarFinder. Returns null when
+    // no jar is found.
+    return packagedClasses.get(class_file);
   }
 
   /**

From 95c3c582026f0d204fec074b950f948a0cd2ebae Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 16 Apr 2013 23:22:14 +0000
Subject: [PATCH 0936/1540] HBASE-8352 Rename '.snapshot' directory (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468671 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HConstants.java   |  8 ++++--
 .../master/snapshot/SnapshotManager.java      | 27 ++++++++++++++++---
 .../hbase/client/TestSnapshotFromAdmin.java   |  2 +-
 .../hbase/client/TestSnapshotsFromAdmin.java  |  2 +-
 .../hbase/snapshot/TestExportSnapshot.java    |  8 +++---
 .../TestSnapshotDescriptionUtils.java         |  5 ++--
 6 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e7a4e6e9745f..ac914545c9ee 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -688,8 +688,11 @@ public static enum Modify {
    * remaining snapshot constants; this is here to keep HConstants dependencies at a minimum and
    * uni-directional.
    */
-  public static final String SNAPSHOT_DIR_NAME = ".snapshot";
+  public static final String SNAPSHOT_DIR_NAME = ".hbase-snapshot";
 
+  /* Name of old snapshot directory. See HBASE-8352 for details on why it needs to be renamed */
+  public static final String OLD_SNAPSHOT_DIR_NAME = ".snapshot";
+  
   /** Temporary directory used for table creation and deletion */
   public static final String HBASE_TEMP_DIRECTORY = ".tmp";
 
@@ -697,7 +700,8 @@ public static enum Modify {
   public static final List<String> HBASE_NON_TABLE_DIRS =
     Collections.unmodifiableList(Arrays.asList(new String[] { HREGION_LOGDIR_NAME,
       HREGION_OLDLOGDIR_NAME, CORRUPT_DIR_NAME, SPLIT_LOGDIR_NAME,
-      HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY, SNAPSHOT_DIR_NAME, HBASE_TEMP_DIRECTORY }));
+      HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY, SNAPSHOT_DIR_NAME, HBASE_TEMP_DIRECTORY,
+      OLD_SNAPSHOT_DIR_NAME }));
 
   /** Directories that are not HBase user table directories */
   public static final List<String> HBASE_NON_USER_TABLE_DIRS =
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index f30210f5c070..adb37adb0f63 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -145,6 +145,7 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
     this.master = master;
     this.metricsMaster = metricsMaster;
 
+    this.rootDir = master.getMasterFileSystem().getRootDir();
     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
 
     // get the configuration for the coordinator
@@ -158,7 +159,6 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
     ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
         master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
     this.coordinator = new ProcedureCoordinator(comms, tpool);
-    this.rootDir = master.getMasterFileSystem().getRootDir();
     this.executorService = master.getExecutorService();
     resetTempDir();
   }
@@ -175,12 +175,12 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
     this.master = master;
     this.metricsMaster = metricsMaster;
 
+    this.rootDir = master.getMasterFileSystem().getRootDir();
     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
 
     this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
       SNAPSHOT_WAKE_MILLIS_DEFAULT);
     this.coordinator = coordinator;
-    this.rootDir = master.getMasterFileSystem().getRootDir();
     this.executorService = pool;
     resetTempDir();
   }
@@ -191,9 +191,20 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
    * @throws IOException File system exception
    */
   public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
+    return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
+  }
+  
+  /**
+   * Gets the list of all completed snapshots.
+   * @param snapshotDir snapshot directory
+   * @return list of SnapshotDescriptions
+   * @throws IOException File system exception
+   */
+  private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
     List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
     // first create the snapshot root path and check to see if it exists
-    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+
     FileSystem fs = master.getMasterFileSystem().getFileSystem();
 
     // if there are no snapshots, return an empty list
@@ -873,6 +884,15 @@ private void checkSnapshotSupport(final Configuration conf, final MasterFileSyst
     cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
     if (cleaners != null) Collections.addAll(logCleaners, cleaners);
 
+    // check if an older version of snapshot directory was present
+    Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
+    FileSystem fs = mfs.getFileSystem();
+    List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
+    if (ss != null && !ss.isEmpty()) {
+      LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
+      LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
+    }
+    
     // If the user has enabled the snapshot, we force the cleaners to be present
     // otherwise we still need to check if cleaners are enabled or not and verify
     // that there're no snapshot in the .snapshot folder.
@@ -909,7 +929,6 @@ private void checkSnapshotSupport(final Configuration conf, final MasterFileSyst
     if (!snapshotEnabled) {
       LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
       Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
-      FileSystem fs = mfs.getFileSystem();
       if (fs.exists(snapshotDir)) {
         FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
           new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
index 0ee5b609cee2..af9ad98e8d9a 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
@@ -116,7 +116,7 @@ public void testValidateSnapshotName() throws Exception {
     HBaseAdmin admin = new HBaseAdmin(mockConnection);
     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
     // check that invalid snapshot names fail
-    failSnapshotStart(admin, builder.setName(".snapshot").build());
+    failSnapshotStart(admin, builder.setName(HConstants.SNAPSHOT_DIR_NAME).build());
     failSnapshotStart(admin, builder.setName("-snapshot").build());
     failSnapshotStart(admin, builder.setName("snapshot fails").build());
     failSnapshotStart(admin, builder.setName("snap$hot").build());
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
index 20b5edea1a7e..2c1c85eabd1f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java
@@ -114,7 +114,7 @@ public void testValidateSnapshotName() throws IOException {
     HBaseAdmin admin = new HBaseAdmin(mockConnection);
     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
     // check that invalid snapshot names fail
-    failSnapshotStart(admin, builder.setName(".snapshot").build());
+    failSnapshotStart(admin, builder.setName(HConstants.SNAPSHOT_DIR_NAME).build());
     failSnapshotStart(admin, builder.setName("-snapshot").build());
     failSnapshotStart(admin, builder.setName("snapshot fails").build());
     failSnapshotStart(admin, builder.setName("snap$hot").build());
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index dfd0a7878e24..f5e0c8d67145 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
@@ -205,12 +206,12 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
     for (FileStatus fileStatus: rootFiles) {
       String name = fileStatus.getPath().getName();
       assertTrue(fileStatus.isDir());
-      assertTrue(name.equals(".snapshot") || name.equals(".archive"));
+      assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || name.equals(".archive"));
     }
 
     // compare the snapshot metadata and verify the hfiles
     final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
-    final Path snapshotDir = new Path(".snapshot", Bytes.toString(snapshotName));
+    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
     verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
         fs, new Path(copyDir, snapshotDir));
     verifyArchive(fs, copyDir, Bytes.toString(snapshotName));
@@ -233,7 +234,8 @@ private void verifySnapshot(final FileSystem fs1, final Path root1,
    */
   private void verifyArchive(final FileSystem fs, final Path rootDir, final String snapshotName)
       throws IOException {
-    final Path exportedSnapshot = new Path(rootDir, new Path(".snapshot", snapshotName));
+    final Path exportedSnapshot = new Path(rootDir,
+      new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
     final Path exportedArchive = new Path(rootDir, ".archive");
     LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
     SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
index 3c452f7eae98..0a32ca3db46c 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
@@ -88,7 +89,7 @@ public void testValidateMissingTableName() {
    */
   @Test
   public void testCompleteSnapshotWithNoSnapshotDirectoryFailure() throws Exception {
-    Path snapshotDir = new Path(root, ".snapshot");
+    Path snapshotDir = new Path(root, HConstants.SNAPSHOT_DIR_NAME);
     Path tmpDir = new Path(snapshotDir, ".tmp");
     Path workingDir = new Path(tmpDir, "not_a_snapshot");
     assertFalse("Already have working snapshot dir: " + workingDir
@@ -101,4 +102,4 @@ public void testCompleteSnapshotWithNoSnapshotDirectoryFailure() throws Exceptio
       LOG.info("Correctly failed to move non-existant directory: " + e.getMessage());
     }
   }
-}
\ No newline at end of file
+}

From d87752106d0f3dc456e6cd02ca4c4c5ac2df6eb3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 17 Apr 2013 00:15:48 +0000
Subject: [PATCH 0937/1540] CHANGES.txt 0.94.7RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468694 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 18042c516c3b..52009fb4ed36 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.7 - 4/15/2013
+Release 0.94.7 - 4/16/2013
 Sub-task
 
     [HBASE-7615] - Add metrics for snapshots
@@ -64,6 +64,7 @@ Bug
     [HBASE-8303] - Increse the test timeout to 60s when they are less than 20s
     [HBASE-8313] - Add Bloom filter testing for HFileOutputFormat
     [HBASE-8326] - mapreduce.TestTableInputFormatScan times out frequently
+    [HBASE-8352] - Rename '.snapshot' directory
 
 Improvement
 

From 03ae8b1a6ab28f0aaba6ed891383553e13c45277 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 17 Apr 2013 15:42:16 +0000
Subject: [PATCH 0938/1540] HBASE-7410 add snapshot/clone/restore/export docs
 to ref guide

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468977 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/ops_mgt.xml | 100 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)

diff --git a/src/docbkx/ops_mgt.xml b/src/docbkx/ops_mgt.xml
index a1bde700aabe..0009ab42bc09 100644
--- a/src/docbkx/ops_mgt.xml
+++ b/src/docbkx/ops_mgt.xml
@@ -704,6 +704,106 @@ false
       </para>
     </section>
   </section>  <!--  backup -->
+
+  <section xml:id="ops.snapshots">
+    <title>HBase Snapshots</title>
+    <para>HBase Snapshots allow you to take a snapshot of a table without too much impact on Region Servers.
+      Snapshot, Clone and restore operations don't involve data copying.
+      Also, Exporting the snapshot to another cluster doesn't have impact on the Region Servers.
+    </para>
+    <para>Prior to version 0.94.6, the only way to backup or to clone a table is to use CopyTable/ExportTable,
+      or to copy all the hfiles in HDFS after disabling the table.
+      The disadvantages of these methods are that you can degrade region server performance
+      (Copy/Export Table) or you need to disable the table, that means no reads or writes;
+      and this is usually unacceptable.
+    </para>
+    <section xml:id="ops.snapshots.configuration"><title>Configuration</title>
+      <para>To turn on the snapshot support just set the
+        <varname>hbase.snapshot.enabled</varname> property to true.
+        (Snapshots are enabled by default in 0.95+ and off by default in 0.94.6+)
+        <programlisting>
+  &lt;property>
+    &lt;name>hbase.snapshot.enabled&lt;/name>
+    &lt;value>true&lt;/value>
+  &lt;/property>
+        </programlisting>
+      </para>
+    </section>
+    <section xml:id="ops.snapshots.takeasnapshot"><title>Take a Snapshot</title>
+      <para>You can take a snapshot of a table regardless of whether it is enabled or disabled.
+        The snapshot operation doesn't involve any data copying.
+        <programlisting>
+    $ ./bin/hbase shell
+    hbase> snapshot 'myTable', 'myTableSnapshot-122112'
+        </programlisting>
+      </para>
+    </section>
+    <section xml:id="ops.snapshots.list"><title>Listing Snapshots</title>
+      <para>List all snapshots taken (by printing the names and relative information).
+        <programlisting>
+    $ ./bin/hbase shell
+    hbase> list_snapshots
+        </programlisting>
+      </para>
+    </section>
+    <section xml:id="ops.snapshots.delete"><title>Deleting Snapshots</title>
+      <para>You can remove a snapshot, and the files retained for that snapshot will be removed
+        if no longer needed.
+        <programlisting>
+    $ ./bin/hbase shell
+    hbase> delete_snapshot 'myTableSnapshot-122112'
+        </programlisting>
+      </para>
+    </section>
+    <section xml:id="ops.snapshots.clone"><title>Clone a table from snapshot</title>
+      <para>From a snapshot you can create a new table (clone operation) with the same data
+      that you had when the snapshot was taken.
+      The clone operation, doesn't involve data copies, and a change to the cloned table
+      doesn't impact the snapshot or the original table.
+        <programlisting>
+    $ ./bin/hbase shell
+    hbase> clone_snapshot 'myTableSnapshot-122112', 'myNewTestTable'
+        </programlisting>
+      </para>
+    </section>
+    <section xml:id="ops.snapshots.restore"><title>Restore a snapshot</title>
+      <para>The restore operation requires the table to be disabled, and the table will be
+      restored to the state at the time when the snapshot was taken,
+      changing both data and schema if required.
+        <programlisting>
+    $ ./bin/hbase shell
+    hbase> disable 'myTable'
+    hbase> restore_snapshot 'myTableSnapshot-122112'
+        </programlisting>
+      </para>
+      <note>
+        <para>Since Replication works at log level and snapshots at file-system level,
+      after a restore, the replicas will be in a different state from the master.
+      If you want to use restore, you need to stop replication and redo the bootstrap.
+        </para>
+      </note>
+      <para>In case of partial data-loss due to misbehaving client, instead of a full restore
+      that requires the table to be disabled, you can clone the table from the snapshot
+      and use a Map-Reduce job to copy the data that you need, from the clone to the main one.
+      </para>
+    </section>
+    <section xml:id="ops.snapshots.acls"><title>Snapshots operations and ACLs</title>
+    If you are using security with the AccessController Coprocessor (See <xref linkend="hbase.accesscontrol.configuration" />),
+    only a global administrator can take, clone, or restore a snapshot, and these actions do not capture the ACL rights.
+    This means that restoring a table preserves the ACL rights of the existing table,
+    while cloning a table creates a new table that has no ACL rights until the administrator adds them.
+    </section>
+    <section xml:id="ops.snapshots.export"><title>Export to another cluster</title>
+      <para>The ExportSnapshot tool copies all the data related to a snapshot (hfiles, logs, snapshot metadata) to another cluster.
+        The tool executes a Map-Reduce job, similar to distcp, to copy files between the two clusters,
+        and since it works at file-system level the hbase cluster does not have to be online.
+        <para>To copy a snapshot called MySnapshot to an HBase cluster srv2 (hdfs:///srv2:8082/hbase) using 16 mappers:
+<programlisting>$ bin/hbase class org.apache.hadoop.hbase.snapshot.ExportSnapshot -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase -mappers 16</programlisting>
+        </para>
+      </para>
+    </section>
+  </section>  <!--  snapshots -->
+
   <section xml:id="ops.capacity"><title>Capacity Planning</title>
     <section xml:id="ops.capacity.storage"><title>Storage</title>
       <para>A common question for HBase administrators is estimating how much storage will be required for an HBase cluster.

From 3c5b89083699075d4da8cd52eae019e02c7e8f23 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 17 Apr 2013 16:53:46 +0000
Subject: [PATCH 0939/1540] New CHANGES.txt

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1468999 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 52009fb4ed36..ad95b5939481 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.7 - 4/16/2013
+Release 0.94.7 - 4/17/2013
 Sub-task
 
     [HBASE-7615] - Add metrics for snapshots
@@ -68,6 +68,7 @@ Bug
 
 Improvement
 
+    [HBASE-7410] - [snapshots] add snapshot/clone/restore/export docs to ref guide
     [HBASE-7599] - Port HBASE-6066 (low hanging read path improvements) to 0.94
     [HBASE-8148] - Allow IPC to bind on a specific address
     [HBASE-8152] - Avoid creating empty reference file when splitkey is outside the key range of a store file

From 4dc4bd17218344a4d117e86fd777f1a2098cca9f Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Thu, 18 Apr 2013 20:52:40 +0000
Subject: [PATCH 0940/1540] HBASE-8350 enable ChaosMonkey to run commands as
 different users

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1469564 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/developer.xml                      |  8 ++++
 .../hadoop/hbase/HBaseClusterManager.java     | 43 ++++++++++---------
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/src/docbkx/developer.xml b/src/docbkx/developer.xml
index d23e276518af..19d89bc796b9 100644
--- a/src/docbkx/developer.xml
+++ b/src/docbkx/developer.xml
@@ -655,6 +655,14 @@ cluster uniformly, <code>IntegrationTestingUtility</code>, and <code>HBaseCluste
 and public client API's can be used.
 </para>
 
+<para>
+On a distributed cluster, integration tests that use ChaosMonkey or otherwise manipulate services thru cluster manager (e.g. restart regionservers) use SSH to do it.
+To run these, test process should be able to run commands on remote end, so ssh should be configured accordingly (for example, if HBase runs under hbase
+user in your cluster, you can set up passwordless ssh for that user and run the test also under it). To facilitate that, <code>hbase.it.clustermanager.ssh.user</code> and
+<code>hbase.it.clustermanager.ssh.opts</code> configuration settings can be used. The former is the remote user that cluster manager should use to perform ssh commands.
+The latter contains additional options that are passed to SSH (for example, "-i /tmp/my-key").
+</para>
+
 <section xml:id="maven.build.commands.integration.tests.mini">
 <title>Running integration tests against mini cluster</title>
 <para>HBase 0.92 added a <varname>verify</varname> maven target.
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
index b432e72a9128..e3fb77c0a11a 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
@@ -24,6 +24,7 @@
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClusterManager.CommandProvider.Operation;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.util.Shell;
@@ -37,16 +38,33 @@
  */
 @InterfaceAudience.Private
 public class HBaseClusterManager extends ClusterManager {
+  private String sshUserName;
+  private String sshOptions;
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    if (conf == null) {
+      // Configured gets passed null before real conf. Why? I don't know.
+      return;
+    }
+    sshUserName = conf.get("hbase.it.clustermanager.ssh.user", "");
+    String extraSshOptions = conf.get("hbase.it.clustermanager.ssh.opts", "");
+    sshOptions = System.getenv("HBASE_SSH_OPTS");
+    if (!extraSshOptions.isEmpty()) {
+      sshOptions = StringUtils.join(new Object[] { sshOptions, extraSshOptions }, " ");
+    }
+    LOG.info("Running with SSH user [" + sshUserName + "] and options [" + sshOptions + "]");
+  }
 
   /**
    * Executes commands over SSH
    */
-  static class RemoteShell extends Shell.ShellCommandExecutor {
+  protected class RemoteShell extends Shell.ShellCommandExecutor {
 
     private String hostname;
 
     private String sshCmd = "/usr/bin/ssh";
-    private String sshOptions = System.getenv("HBASE_SSH_OPTS"); //from conf/hbase-env.sh
 
     public RemoteShell(String hostname, String[] execString, File dir, Map<String, String> env,
         long timeout) {
@@ -70,11 +88,12 @@ public RemoteShell(String hostname, String[] execString) {
     }
 
     public String[] getExecString() {
+      String userAndHost = sshUserName.isEmpty() ? hostname : (sshUserName + "@" + hostname);
       return new String[] {
           "bash", "-c",
           StringUtils.join(new String[] { sshCmd,
-              sshOptions == null ? "" : sshOptions,
-              hostname,
+              (sshOptions == null) ? "" : sshOptions,
+              userAndHost,
               "\"" + StringUtils.join(super.getExecString(), " ") + "\""
           }, " ")};
     }
@@ -83,22 +102,6 @@ public String[] getExecString() {
     public void execute() throws IOException {
       super.execute();
     }
-
-    public void setSshCmd(String sshCmd) {
-      this.sshCmd = sshCmd;
-    }
-
-    public void setSshOptions(String sshOptions) {
-      this.sshOptions = sshOptions;
-    }
-
-    public String getSshCmd() {
-      return sshCmd;
-    }
-
-    public String getSshOptions() {
-      return sshOptions;
-    }
   }
 
   /**

From d6aad5da1b182431869263bd06d939bfa95f6b73 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 19 Apr 2013 21:33:44 +0000
Subject: [PATCH 0941/1540] HBASE-8377 IntegrationTestBigLinkedList calculates
 wrap for linked list size incorrectly

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1470050 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/test/IntegrationTestBigLinkedList.java     | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index be522185450e..8f2f86e852ff 100644
--- a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -322,7 +322,7 @@ protected void setup(Context context) throws IOException, InterruptedException {
         table = new HTable(conf, getTableName(conf));
         table.setAutoFlush(false);
         table.setWriteBufferSize(4 * 1024 * 1024);
-        numNodes = context.getConfiguration().getLong(GENERATOR_NUM_MAPPERS_KEY, 25000000);
+        numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
         if (numNodes < 25000000) {
           wrap = numNodes;
         }
@@ -374,8 +374,8 @@ private void persist(Context output, long count, long[] prev, long[] current, by
           Put put = new Put(Bytes.toBytes(current[i]));
           put.add(FAMILY_NAME, COLUMN_PREV, Bytes.toBytes(prev == null ? -1 : prev[i]));
 
-          if (count > 0) {
-            put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + 1));
+          if (count >= 0) {
+            put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i));
           }
           if (id != null) {
             put.add(FAMILY_NAME, COLUMN_CLIENT, id);
@@ -901,12 +901,18 @@ private static CINode getCINode(Result result, CINode node) {
     node.key = Bytes.toLong(result.getRow());
     if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) {
       node.prev = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_PREV));
+    } else {
+      node.prev = -1;
     }
     if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) {
       node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT));
+    } else {
+      node.count = -1;
     }
     if (result.containsColumn(FAMILY_NAME, COLUMN_CLIENT)) {
       node.client = Bytes.toString(result.getValue(FAMILY_NAME, COLUMN_CLIENT));
+    } else {
+      node.client = "";
     }
     return node;
   }

From ed9a0a5091ab719a31e0605a081ad8221314000e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 21 Apr 2013 00:27:35 +0000
Subject: [PATCH 0942/1540] HBASE-8354 Backport HBASE-7878 'recoverFileLease
 does not check return value of recoverLease' to 0.94 (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1470253 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/HLogSplitter.java  | 33 ++++++++++--
 .../apache/hadoop/hbase/util/FSHDFSUtils.java | 37 +++++++++-----
 .../hbase/regionserver/wal/TestHLog.java      | 27 +++++++---
 .../hbase/regionserver/wal/TestHLogSplit.java | 51 ++++++++++++-------
 4 files changed, 107 insertions(+), 41 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 71c5850b4b8f..57f75b79066f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -34,6 +34,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -177,6 +178,20 @@ public HLogSplitter(Configuration conf, Path rootDir, Path srcDir,
    */
   public List<Path> splitLog()
       throws IOException {
+    return splitLog((CountDownLatch) null);
+  }
+  
+  /**
+   * Split up a bunch of regionserver commit log files that are no longer being
+   * written to, into new files, one per region for region to replay on startup.
+   * Delete the old log files when finished.
+   *
+   * @param latch
+   * @throws IOException will throw if corrupted hlogs aren't tolerated
+   * @return the list of splits
+   */
+  public List<Path> splitLog(CountDownLatch latch)
+      throws IOException {
     Preconditions.checkState(!hasSplit,
         "An HLogSplitter instance may only be used once");
     hasSplit = true;
@@ -200,7 +215,7 @@ public List<Path> splitLog()
     }
     logAndReport("Splitting " + logfiles.length + " hlog(s) in "
     + srcDir.toString());
-    splits = splitLog(logfiles);
+    splits = splitLog(logfiles, latch);
 
     splitTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
     String msg = "hlog file splitting completed in " + splitTime +
@@ -260,7 +275,8 @@ Map<byte[], Long> getOutputCounts() {
    * After the process is complete, the log files are archived to a separate
    * directory.
    */
-  private List<Path> splitLog(final FileStatus[] logfiles) throws IOException {
+  private List<Path> splitLog(final FileStatus[] logfiles, CountDownLatch latch)
+      throws IOException {
     List<Path> processedLogs = new ArrayList<Path>();
     List<Path> corruptedLogs = new ArrayList<Path>();
     List<Path> splits = null;
@@ -307,10 +323,19 @@ private List<Path> splitLog(final FileStatus[] logfiles) throws IOException {
       }
       status.setStatus("Log splits complete. Checking for orphaned logs.");
       
-      if (fs.listStatus(srcDir).length > processedLogs.size()
+      if (latch != null) {
+        try {
+          latch.await();
+        } catch (InterruptedException ie) {
+          LOG.warn("wait for latch interrupted");
+          Thread.currentThread().interrupt();
+        }
+      }
+      FileStatus[] currFiles = fs.listStatus(srcDir);
+      if (currFiles.length > processedLogs.size()
           + corruptedLogs.size()) {
         throw new OrphanHLogAfterSplitException(
-            "Discovered orphan hlog after split. Maybe the "
+          "Discovered orphan hlog after split. Maybe the "
             + "HRegionServer was not dead when we started");
       }
     } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 4186f0ad54c0..6bc63c3d85b6 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -51,6 +51,8 @@ public class FSHDFSUtils extends FSUtils{
    */
   public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
 
+  public static final String TEST_TRIGGER_DFS_APPEND = "hbase.test.trigger.dfs.append";
+
   @Override
   public void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
   throws IOException{
@@ -68,26 +70,37 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
 
     // Trying recovery
     boolean recovered = false;
+    long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 300000);
+    // conf parameter passed from unit test, indicating whether fs.append() should be triggered
+    boolean triggerAppend = conf.getBoolean(TEST_TRIGGER_DFS_APPEND, false);
+    Exception ex = null;
     while (!recovered) {
       try {
         try {
-          if (fs instanceof DistributedFileSystem) {
-            DistributedFileSystem dfs = (DistributedFileSystem)fs;
-            DistributedFileSystem.class.getMethod("recoverLease",
-              new Class[] {Path.class}).invoke(dfs, p);
-          } else {
-            throw new Exception("Not a DistributedFileSystem");
+          DistributedFileSystem dfs = (DistributedFileSystem) fs;
+          if (triggerAppend) throw new IOException();
+          try {
+            recovered = (Boolean) DistributedFileSystem.class.getMethod(
+              "recoverLease", new Class[] { Path.class }).invoke(dfs, p);
+            if (!recovered) LOG.debug("recoverLease returned false");
+          } catch (InvocationTargetException ite) {
+            // function was properly called, but threw it's own exception
+            throw (IOException) ite.getCause();
           }
-        } catch (InvocationTargetException ite) {
-          // function was properly called, but threw it's own exception
-          throw (IOException) ite.getCause();
         } catch (Exception e) {
           LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
             ", trying fs.append instead");
+          ex = e;
+        }
+        if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) {
+          LOG.debug("trying fs.append for " + p + " with " + ex);
+          ex = null; // assume the following append() call would succeed
           FSDataOutputStream out = fs.append(p);
           out.close();
+          recovered = true;
+          LOG.debug("fs.append passed");
         }
-        recovered = true;
+        if (recovered) break;
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
         if (e instanceof AlreadyBeingCreatedException) {
@@ -111,9 +124,9 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
       }
       try {
         Thread.sleep(1000);
-      } catch (InterruptedException ex) {
+      } catch (InterruptedException ie) {
         InterruptedIOException iioe = new InterruptedIOException();
-        iioe.initCause(ex);
+        iioe.initCause(ie);
         throw iioe;
       }
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 60f77a025db6..c9bb3b247d2c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@@ -101,6 +102,7 @@ public static void setUpBeforeClass() throws Exception {
     // Make block sizes small.
     TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
     // needed for testAppendClose()
+    TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true);
     TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
     // quicker heartbeat interval for faster DN death notification
     TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000);
@@ -370,17 +372,29 @@ private void verifySplits(List<Path> splits, final int howmany)
     }
   }
   
-  // For this test to pass, requires:
-  // 1. HDFS-200 (append support)
-  // 2. HDFS-988 (SafeMode should freeze file operations
-  //              [FSNamesystem.nextGenerationStampForBlock])
-  // 3. HDFS-142 (on restart, maintain pendingCreates)
+  /*
+   * We pass different values to recoverFileLease() so that different code paths are covered
+   * 
+   * For this test to pass, requires:
+   * 1. HDFS-200 (append support)
+   * 2. HDFS-988 (SafeMode should freeze file operations
+   *              [FSNamesystem.nextGenerationStampForBlock])
+   * 3. HDFS-142 (on restart, maintain pendingCreates)
+   */
   @Test
   public void testAppendClose() throws Exception {
+    testAppendClose(true);
+    testAppendClose(false);
+  }
+
+  /*
+   * @param triggerDirectAppend whether to trigger direct call of fs.append()
+   */
+  public void testAppendClose(final boolean triggerDirectAppend) throws Exception {
     byte [] tableName = Bytes.toBytes(getName());
     HRegionInfo regioninfo = new HRegionInfo(tableName,
              HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
-    Path subdir = new Path(dir, "hlogdir");
+    Path subdir = new Path(dir, "hlogdir" + triggerDirectAppend);
     Path archdir = new Path(dir, "hlogdir_archive");
     HLog wal = new HLog(fs, subdir, archdir, conf);
     final int total = 20;
@@ -454,6 +468,7 @@ class RecoverLogThread extends Thread {
       public Exception exception = null;
       public void run() {
           try {
+            rlConf.setBoolean(FSHDFSUtils.TEST_TRIGGER_DFS_APPEND, triggerDirectAppend);
             FSUtils.getInstance(fs, rlConf)
               .recoverFileLease(recoveredFs, walPath, rlConf);
           } catch (IOException e) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 37366280e821..d3e3a53d6622 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -33,6 +33,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -119,12 +120,11 @@ static enum Corruptions {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().
-            setStrings("hbase.rootdir", hbaseDir.toString());
-    TEST_UTIL.getConfiguration().
-            setClass("hbase.regionserver.hlog.writer.impl",
-                InstrumentedSequenceFileLogWriter.class, HLog.Writer.class);
-
+    TEST_UTIL.getConfiguration().setStrings("hbase.rootdir", hbaseDir.toString());
+    TEST_UTIL.getConfiguration().setClass("hbase.regionserver.hlog.writer.impl",
+      InstrumentedSequenceFileLogWriter.class, HLog.Writer.class);
+    TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true);
+    TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
     TEST_UTIL.startMiniDFSCluster(2);
   }
 
@@ -189,11 +189,12 @@ public void testSplitFailsIfNewHLogGetsCreatedAfterSplitStarted()
 
     generateHLogs(-1);
 
+    CountDownLatch latch = new CountDownLatch(1);
     try {
-    (new ZombieNewLogWriterRegionServer(stop)).start();
-    HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf,
-        hbaseDir, hlogDir, oldLogDir, fs);
-    logSplitter.splitLog();
+      (new ZombieNewLogWriterRegionServer(latch, stop)).start();
+      HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, hbaseDir, hlogDir, oldLogDir,
+        fs);
+      logSplitter.splitLog(latch);
     } finally {
       stop.set(true);
     }
@@ -553,16 +554,23 @@ public void testSplitWillNotTouchLogsIfNewHLogGetsCreatedAfterSplitStarted()
     AtomicBoolean stop = new AtomicBoolean(false);
     generateHLogs(-1);
     fs.initialize(fs.getUri(), conf);
-    Thread zombie = new ZombieNewLogWriterRegionServer(stop);
+    CountDownLatch latch = new CountDownLatch(1);
+    Thread zombie = new ZombieNewLogWriterRegionServer(latch, stop);
 
+    List<Path> splits = null;
     try {
       zombie.start();
       try {
         HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf,
             hbaseDir, hlogDir, oldLogDir, fs);
-        logSplitter.splitLog();
-      } catch (IOException ex) {/* expected */}
-      int logFilesNumber = fs.listStatus(hlogDir).length;
+        splits = logSplitter.splitLog(latch);
+      } catch (IOException ex) {
+        /* expected */
+        LOG.warn("testSplitWillNotTouchLogsIfNewHLogGetsCreatedAfterSplitStarted", ex);
+      }
+      FileStatus[] files = fs.listStatus(hlogDir);
+      if (files == null) fail("no files in " + hlogDir + " with splits " + splits);
+      int logFilesNumber = files.length;
 
       assertEquals("Log files should not be archived if there's an extra file after split",
               NUM_WRITERS + 1, logFilesNumber);
@@ -959,8 +967,10 @@ public void run() {
    */
   class ZombieNewLogWriterRegionServer extends Thread {
     AtomicBoolean stop;
-    public ZombieNewLogWriterRegionServer(AtomicBoolean stop) {
+    CountDownLatch latch;
+    public ZombieNewLogWriterRegionServer(CountDownLatch latch, AtomicBoolean stop) {
       super("ZombieNewLogWriterRegionServer");
+      this.latch = latch;
       this.stop = stop;
     }
 
@@ -977,7 +987,7 @@ public void run() {
       try {
 
         while (!fs.exists(recoveredEdits) && !stop.get()) {
-          flushToConsole("Juliet: split not started, sleeping a bit...");
+          LOG.info("Juliet: split not started, sleeping a bit...");
           Threads.sleep(10);
         }
  
@@ -987,8 +997,10 @@ public void run() {
         appendEntry(writer, "juliet".getBytes(), ("juliet").getBytes(),
             ("r").getBytes(), FAMILY, QUALIFIER, VALUE, 0);
         writer.close();
-        flushToConsole("Juliet file creator: created file " + julietLog);
+        LOG.info("Juliet file creator: created file " + julietLog);
+        latch.countDown();
       } catch (IOException e1) {
+        LOG.error("Failed to create file " + julietLog, e1);
         assertTrue("Failed to create file " + julietLog, false);
       }
     }
@@ -1192,7 +1204,7 @@ private void generateHLogs(int writers, int entries, int leaveOpen) throws IOExc
       }
       if (i != leaveOpen) {
         writer[i].close();
-        flushToConsole("Closing writer " + i);
+        LOG.info("Closing writer " + i);
       }
     }
   }
@@ -1305,8 +1317,9 @@ public long appendEntry(HLog.Writer writer, byte[] table, byte[] region,
                           byte[] row, byte[] family, byte[] qualifier,
                           byte[] value, long seq)
           throws IOException {
-
+    LOG.info(Thread.currentThread().getName() + " append");
     writer.append(createTestEntry(table, region, row, family, qualifier, value, seq));
+    LOG.info(Thread.currentThread().getName() + " sync");
     writer.sync();
     return seq;
   }

From 5ad49036928ca7c2f549f77c8ad44edaacd057cd Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Mon, 22 Apr 2013 02:35:46 +0000
Subject: [PATCH 0943/1540] HBASE-6870 HTable#coprocessorExec always scan the
 whole table

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1470374 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTable.java    | 58 +++++++++----------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 52deff7ae9ef..c7c13f8040be 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -495,22 +495,42 @@ public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOExcep
    */
   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
     final byte [] endKey) throws IOException {
-    final boolean endKeyIsEndOfTable = Bytes.equals(endKey,
-                                                    HConstants.EMPTY_END_ROW);
+    return getKeysAndRegionsInRange(startKey, endKey, false).getSecond();
+    }
+
+  /**
+   * Get the corresponding start keys and regions for an arbitrary range of
+   * keys.
+   * <p>
+   * @param startKey Starting row in range, inclusive
+   * @param endKey Ending row in range
+   * @param includeEndKey true if endRow is inclusive, false if exclusive
+   * @return A pair of list of start keys and list of HRegionLocations that
+   *         contain the specified range
+   * @throws IOException if a remote or network exception occurs
+   */
+  private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
+      final byte[] startKey, final byte[] endKey, final boolean includeEndKey)
+      throws IOException {
+    final boolean endKeyIsEndOfTable = Bytes.equals(endKey,HConstants.EMPTY_END_ROW);
     if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
       throw new IllegalArgumentException(
         "Invalid range: " + Bytes.toStringBinary(startKey) +
         " > " + Bytes.toStringBinary(endKey));
     }
-    final List<HRegionLocation> regionList = new ArrayList<HRegionLocation>();
-    byte [] currentKey = startKey;
+    List<byte[]> keysInRange = new ArrayList<byte[]>();
+    List<HRegionLocation> regionsInRange = new ArrayList<HRegionLocation>();
+    byte[] currentKey = startKey;
     do {
       HRegionLocation regionLocation = getRegionLocation(currentKey, false);
-      regionList.add(regionLocation);
+      keysInRange.add(currentKey);
+      regionsInRange.add(regionLocation);
       currentKey = regionLocation.getRegionInfo().getEndKey();
-    } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) &&
-             (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
-    return regionList;
+    } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
+        && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0
+            || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0)));
+    return new Pair<List<byte[]>, List<HRegionLocation>>(keysInRange,
+        regionsInRange);
   }
 
   /**
@@ -1237,33 +1257,13 @@ public <T extends CoprocessorProtocol, R> void coprocessorExec(
 
   private List<byte[]> getStartKeysInRange(byte[] start, byte[] end)
   throws IOException {
-    Pair<byte[][],byte[][]> startEndKeys = getStartEndKeys();
-    byte[][] startKeys = startEndKeys.getFirst();
-    byte[][] endKeys = startEndKeys.getSecond();
-
     if (start == null) {
       start = HConstants.EMPTY_START_ROW;
     }
     if (end == null) {
       end = HConstants.EMPTY_END_ROW;
     }
-
-    List<byte[]> rangeKeys = new ArrayList<byte[]>();
-    for (int i=0; i<startKeys.length; i++) {
-      if (Bytes.compareTo(start, startKeys[i]) >= 0 ) {
-        if (Bytes.equals(endKeys[i], HConstants.EMPTY_END_ROW) ||
-            Bytes.compareTo(start, endKeys[i]) < 0) {
-          rangeKeys.add(start);
-        }
-      } else if (Bytes.equals(end, HConstants.EMPTY_END_ROW) ||
-          Bytes.compareTo(startKeys[i], end) <= 0) {
-        rangeKeys.add(startKeys[i]);
-      } else {
-        break; // past stop
-      }
-    }
-
-    return rangeKeys;
+    return getKeysAndRegionsInRange(start, end, true).getFirst();
   }
 
   public void setOperationTimeout(int operationTimeout) {

From fa8eda4539a3276400037861ea4c88114bbad0ff Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 23 Apr 2013 03:23:32 +0000
Subject: [PATCH 0944/1540] HBASE-8389 HBASE-8354 DDoSes Namenode with lease
 recovery requests (Varun and Ted)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1470800 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/FSHDFSUtils.java   | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 6bc63c3d85b6..0f1ea148a70d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -73,6 +73,9 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
     long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 300000);
     // conf parameter passed from unit test, indicating whether fs.append() should be triggered
     boolean triggerAppend = conf.getBoolean(TEST_TRIGGER_DFS_APPEND, false);
+    // retrying lease recovery may preempt pending lease recovery; default to waiting for 4 seconds
+    // after calling recoverLease
+    int waitingPeriod = conf.getInt("hbase.lease.recovery.waiting.period", 4000);
     Exception ex = null;
     while (!recovered) {
       try {
@@ -123,13 +126,15 @@ public void recoverFileLease(final FileSystem fs, final Path p, Configuration co
         }
       }
       try {
-        Thread.sleep(1000);
+        Thread.sleep(waitingPeriod);
       } catch (InterruptedException ie) {
         InterruptedIOException iioe = new InterruptedIOException();
         iioe.initCause(ie);
         throw iioe;
       }
+      // we keep original behavior without retrying lease recovery
+      break;
     }
-    LOG.info("Finished lease recover attempt for " + p);
+    LOG.info("Finished lease recovery attempt for " + p);
   }
 }

From 26f4aee0c0b4aabb5ab63e850bf855b52f07b7c4 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 24 Apr 2013 01:13:50 +0000
Subject: [PATCH 0945/1540] HBASE-8399
 TestTableInputFormatScan2#testScanFromConfiguration fails on hadoop2 profile

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471221 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
index 15ea4986388e..76596046ce72 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
@@ -195,6 +195,7 @@ protected void testScanFromConfiguration(String start, String stop, String last)
     job.setInputFormatClass(TableInputFormat.class);
     job.setNumReduceTasks(1);
     FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
+    TableMapReduceUtil.addDependencyJars(job);
     assertTrue(job.waitForCompletion(true));
   }
 

From a96d0e55a014f3efa3a9cb0a5583c9e2c92dd171 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 24 Apr 2013 02:03:01 +0000
Subject: [PATCH 0946/1540] HBASE-7921. TestHFileBlock.testGzipCompression
 should ignore the block checksum

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471227 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/hfile/TestHFileBlock.java | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index 02a241baf977..74402244c09d 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -202,6 +202,7 @@ static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
         HFile.DEFAULT_BYTES_PER_CHECKSUM);
     DataOutputStream dos = hbw.startWriting(blockType);
     writeTestBlockContents(dos);
+    dos.flush();
     byte[] headerAndData = hbw.getHeaderAndDataForTest();
     assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
     hbw.releaseCompressor();
@@ -244,14 +245,19 @@ public void testGzipCompression() throws IOException {
             + "\\x00"  // XFL (extra flags)
             // OS (0 = FAT filesystems, 3 = Unix). However, this field
             // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
+            // This appears to be a difference caused by the availability
+            // (and use) of the native GZ codec.
             + "\\x03"
             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
             + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"
-            + "\\xAB\\x85g\\x91"; //  4 byte checksum
+            + "\\x00\\x00\\x00\\x00"; //  4 byte checksum (ignored)
     final int correctGzipBlockLength = 95;
-    assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
-        correctGzipBlockLength));
+    final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength);
+    // We ignore the block checksum because createTestBlockStr can change the
+    // gzip header after the block is produced
+    assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4),
+      testBlockStr.substring(0, correctGzipBlockLength - 4));
   }
 
   @Test

From 732b289dd454f30c0b9c0f651da40e64b8d71702 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 24 Apr 2013 05:14:59 +0000
Subject: [PATCH 0947/1540] HBASE-8379 bin/graceful_stop.sh does not return the
 balancer to original state (Jean-Marc)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471253 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/graceful_stop.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh
index beac4b15b146..9a6244006495 100755
--- a/bin/graceful_stop.sh
+++ b/bin/graceful_stop.sh
@@ -69,8 +69,9 @@ fi
 hostname=$1
 filename="/tmp/$hostname"
 # Run the region mover script.
-echo "Disabling balancer!"
-echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell
+echo "Disabling balancer! (if required)"
+HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1`
+echo "Previous balancer state was $HBASE_BALANCER_STATE"
 echo "Unloading $hostname region(s)"
 HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname
 echo "Unloaded $hostname region(s)"
@@ -100,5 +101,10 @@ if [ "$restart" != "" ]; then
   fi
 fi
 
+if [ $HBASE_BALANCER_STATE != "false" ]; then
+  echo "Restoring balancer state to" $HBASE_BALANCER_STATE
+  echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null
+fi
+
 # Cleanup tmp files.
 trap "rm -f  "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT

From 7eb35afffc40995ccd6ad05022eeaa6c5be9e072 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 24 Apr 2013 09:34:40 +0000
Subject: [PATCH 0948/1540] HBASE-8413 Snapshot verify region will always fail
 if the HFile has been archived (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471323 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index 4a4457eeb7e9..db89e66441ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -222,7 +222,7 @@ private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) t
         String fileName = hfile.getPath().getName();
         Path file = new Path(realCfDir, fileName);
         Path archived = new Path(archivedCfDir, fileName);
-        if (!fs.exists(file) && !file.equals(archived)) {
+        if (!fs.exists(file) && !fs.exists(archived)) {
           throw new CorruptedSnapshotException("Can't find hfile: " + hfile.getPath()
               + " in the real (" + realCfDir + ") or archive (" + archivedCfDir
               + ") directory for the primary table.", snapshot);

From 9445cc729566066858b5b97894a68ecdf80cb872 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 24 Apr 2013 15:14:48 +0000
Subject: [PATCH 0949/1540] HBASE-8327 Consolidate class loaders

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471469 13f79535-47bb-0310-9956-ffa450edef68
---
 .../coprocessor/CoprocessorClassLoader.java   | 213 -----------
 .../hbase/coprocessor/CoprocessorHost.java    | 125 ++-----
 .../hadoop/hbase/util/ClassLoaderBase.java    |  79 ++++
 .../hbase/util/CoprocessorClassLoader.java    | 336 ++++++++++++++++++
 .../hadoop/hbase/util/DynamicClassLoader.java |  77 ++--
 .../apache/hadoop/hbase/client/TestGet.java   |   4 +-
 .../hbase/coprocessor/TestClassLoading.java   | 113 +-----
 .../hbase/util/ClassLoaderTestHelper.java     | 164 +++++++++
 .../hbase/util/TestDynamicClassLoader.java    | 125 +------
 9 files changed, 683 insertions(+), 553 deletions(-)
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
deleted file mode 100644
index 42d7dfe24d34..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.coprocessor;
-
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * ClassLoader used to load Coprocessor instances.
- * 
- * This ClassLoader always tries to load classes from the Coprocessor jar first 
- * before delegating to the parent ClassLoader, thus avoiding dependency 
- * conflicts between HBase's classpath and classes in the coprocessor's jar.  
- * Certain classes are exempt from being loaded by this ClassLoader because it 
- * would prevent them from being cast to the equivalent classes in the region 
- * server.  For example, the Coprocessor interface needs to be loaded by the 
- * region server's ClassLoader to prevent a ClassCastException when casting the 
- * coprocessor implementation.
- * 
- * This ClassLoader also handles resource loading.  In most cases this 
- * ClassLoader will attempt to load resources from the coprocessor jar first 
- * before delegating to the parent.  However, like in class loading, 
- * some resources need to be handled differently.  For all of the Hadoop 
- * default configurations (e.g. hbase-default.xml) we will check the parent 
- * ClassLoader first to prevent issues such as failing the HBase default 
- * configuration version check.
- */
-public class CoprocessorClassLoader extends URLClassLoader {
-  private static final Log LOG = 
-      LogFactory.getLog(CoprocessorClassLoader.class);
-  
-  /**
-   * If the class being loaded starts with any of these strings, we will skip
-   * trying to load it from the coprocessor jar and instead delegate 
-   * directly to the parent ClassLoader.
-   */
-  private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
-    // Java standard library:
-    "com.sun.",
-    "launcher.",
-    "java.",
-    "javax.",
-    "org.ietf",
-    "org.omg",
-    "org.w3c",
-    "org.xml",
-    "sunw.",
-    // logging
-    "org.apache.commons.logging",
-    "org.apache.log4j",
-    "com.hadoop",
-    // Hadoop/HBase/ZK:
-    "org.apache.hadoop",
-    "org.apache.zookeeper",
-  };
-  
-  /**
-   * If the resource being loaded matches any of these patterns, we will first 
-   * attempt to load the resource with the parent ClassLoader.  Only if the 
-   * resource is not found by the parent do we attempt to load it from the 
-   * coprocessor jar.
-   */
-  private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS = 
-      new Pattern[] {
-    Pattern.compile("^[^-]+-default\\.xml$")
-  };
-
-  /**
-   * Parent classloader used to load any class not matching the exemption list.
-   */
-  private final ClassLoader parent;
-
-  /**
-   * Creates a CoprocessorClassLoader that loads classes from the given paths.
-   * @param paths paths from which to load classes.
-   * @param parent the parent ClassLoader to set.
-   */
-  public CoprocessorClassLoader(List<URL> paths, ClassLoader parent) {
-    super(paths.toArray(new URL[]{}), parent);
-    this.parent = parent;
-    if (parent == null) {
-      throw new IllegalArgumentException("No parent classloader!");
-    }
-  }
-
-  @Override
-  synchronized public Class<?> loadClass(String name) 
-      throws ClassNotFoundException {
-    // Delegate to the parent immediately if this class is exempt
-    if (isClassExempt(name)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Skipping exempt class " + name + 
-            " - delegating directly to parent");
-      }
-      return parent.loadClass(name);
-    }
-
-    // Check whether the class has already been loaded:
-    Class<?> clasz = findLoadedClass(name);
-    if (clasz != null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Class " + name + " already loaded");
-      }
-    }
-    else {
-      try {
-        // Try to find this class using the URLs passed to this ClassLoader, 
-        // which includes the coprocessor jar
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Finding class: " + name);
-        }
-        clasz = findClass(name);
-      } catch (ClassNotFoundException e) {
-        // Class not found using this ClassLoader, so delegate to parent
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Class " + name + " not found - delegating to parent");
-        }
-        try {
-          clasz = parent.loadClass(name);
-        } catch (ClassNotFoundException e2) {
-          // Class not found in this ClassLoader or in the parent ClassLoader
-          // Log some debug output before rethrowing ClassNotFoundException
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Class " + name + " not found in parent loader");
-          }
-          throw e2;
-        }
-      }
-    }
-    
-    return clasz;
-  }
-  
-  @Override
-  synchronized public URL getResource(String name) {
-    URL resource = null;
-    boolean parentLoaded = false;
-    
-    // Delegate to the parent first if necessary
-    if (loadResourceUsingParentFirst(name)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Checking parent first for resource " + name);
-      }
-      resource = super.getResource(name);
-      parentLoaded = true;
-    }
-    
-    if (resource == null) {
-      // Try to find the resource in the coprocessor jar
-      resource = findResource(name);
-      if ((resource == null) && !parentLoaded) {
-        // Not found in the coprocessor jar and we haven't attempted to load 
-        // the resource in the parent yet; fall back to the parent
-        resource = super.getResource(name);
-      }
-    }
-
-    return resource;
-  }
-  
-  /**
-   * Determines whether the given class should be exempt from being loaded 
-   * by this ClassLoader.
-   * @param name the name of the class to test.
-   * @return true if the class should *not* be loaded by this ClassLoader; 
-   * false otherwise.
-   */
-  protected boolean isClassExempt(String name) {
-    for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
-      if (name.startsWith(exemptPrefix)) {
-        return true;
-      }
-    }
-    return false;
-  }
-  
-  /**
-   * Determines whether we should attempt to load the given resource using the  
-   * parent first before attempting to load the resource using this ClassLoader.
-   * @param name the name of the resource to test.
-   * @return true if we should attempt to load the resource using the parent 
-   * first; false if we should attempt to load the resource using this 
-   * ClassLoader first.
-   */
-  protected boolean loadResourceUsingParentFirst(String name) {
-    for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
-      if (resourcePattern.matcher(name).matches()) {
-        return true;
-      }
-    }
-    return false;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 0241e11be20f..838d05682dbc 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -20,35 +20,46 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
-import com.google.common.collect.MapMaker;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowLock;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
 import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.util.*;
-import java.util.concurrent.ConcurrentMap;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
 
 /**
  * Provides the common setup framework and runtime services for coprocessor
@@ -68,10 +79,6 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
   public static final String WAL_COPROCESSOR_CONF_KEY =
     "hbase.coprocessor.wal.classes";
 
-  //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
-  private static final String COPROCESSOR_JARS_DIR = File.separator
-      + "coprocessor" + File.separator + "jars" + File.separator;
-
   private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
   /** Ordered set of loaded coprocessors with lock */
   protected SortedSet<E> coprocessors =
@@ -81,15 +88,6 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
   protected String pathPrefix;
   protected volatile int loadSequence;
 
-  /*
-   * External classloaders cache keyed by external jar path.
-   * ClassLoader instance is stored as a weak-reference
-   * to allow GC'ing when no CoprocessorHost is using it
-   * (@see HBASE-7205)
-   */
-  static ConcurrentMap<Path, ClassLoader> classLoadersCache =
-      new MapMaker().concurrencyLevel(3).weakValues().makeMap();
-
   public CoprocessorHost() {
     pathPrefix = UUID.randomUUID().toString();
   }
@@ -185,75 +183,8 @@ public E load(Path path, String className, int priority,
         throw new IOException("No jar path specified for " + className);
       }
     } else {
-      // Have we already loaded the class, perhaps from an earlier region open
-      // for the same table?
-      cl = classLoadersCache.get(path);
-      if (cl != null){
-        LOG.debug("Found classloader "+ cl + "for "+path.toString());
-        try {
-          implClass = cl.loadClass(className);
-        } catch (ClassNotFoundException e) {
-          LOG.info("Class " + className + " needs to be loaded from a file - " +
-              path + ".");
-          // go ahead to load from file system.
-        }
-      }
-    }
-
-    // If not, load
-    if (implClass == null) {
-      if (path == null) {
-        throw new IOException("No jar path specified for " + className);
-      }
-      // copy the jar to the local filesystem
-      if (!path.toString().endsWith(".jar")) {
-        throw new IOException(path.toString() + ": not a jar file?");
-      }
-      FileSystem fs = path.getFileSystem(this.conf);
-      File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
-      parentDir.mkdirs();
-      File dst = new File(parentDir, "." + pathPrefix +
-          "." + className + "." + System.currentTimeMillis() + ".jar");
-      fs.copyToLocalFile(path, new Path(dst.toString()));
-      dst.deleteOnExit();
-
-      // TODO: code weaving goes here
-
-      // TODO: wrap heap allocations and enforce maximum usage limits
-
-      /* TODO: inject code into loop headers that monitors CPU use and
-         aborts runaway user code */
-
-      // load the jar and get the implementation main class
-      // NOTE: Path.toURL is deprecated (toURI instead) but the URLClassLoader
-      // unsurprisingly wants URLs, not URIs; so we will use the deprecated
-      // method which returns URLs for as long as it is available
-      List<URL> paths = new ArrayList<URL>();
-      URL url = dst.getCanonicalFile().toURL();
-      paths.add(url);
-
-      JarFile jarFile = new JarFile(dst.toString());
-      Enumeration<JarEntry> entries = jarFile.entries();
-      while (entries.hasMoreElements()) {
-        JarEntry entry = entries.nextElement();
-        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
-          File file = new File(parentDir, "." + pathPrefix +
-              "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
-          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
-          file.deleteOnExit();
-          paths.add(file.toURL());
-        }
-      }
-      jarFile.close();
-
-      cl = new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
-      // cache cp classloader as a weak value, will be GC'ed when no reference left
-      ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
-      if (prev != null) {
-        //lost update race, use already added classloader
-        cl = prev;
-      }
-
+      cl = CoprocessorClassLoader.getClassLoader(
+        path, getClass().getClassLoader(), pathPrefix, conf);
       try {
         implClass = cl.loadClass(className);
       } catch (ClassNotFoundException e) {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java b/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
new file mode 100644
index 000000000000..00a0f8e08cfa
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base class loader that defines couple shared constants used
+ * by sub-classes. It also defined method getClassLoadingLock for parallel
+ * class loading and JDK 1.6 support. This method (getClassLoadingLock)
+ * is similar to the same method in the base class Java ClassLoader
+ * introduced in JDK 1.7, but not in JDK 1.6.
+ */
+@InterfaceAudience.Private
+public class ClassLoaderBase extends URLClassLoader {
+
+  // Maps class name to the corresponding lock object
+  private final ConcurrentHashMap<String, Object> parallelLockMap
+    = new ConcurrentHashMap<String, Object>();
+
+  protected static final String DEFAULT_LOCAL_DIR = "/tmp/hbase-local-dir";
+  protected static final String LOCAL_DIR_KEY = "hbase.local.dir";
+
+  /**
+   * Parent class loader.
+   */
+  protected final ClassLoader parent;
+
+  /**
+   * Creates a DynamicClassLoader that can load classes dynamically
+   * from jar files under a specific folder.
+   *
+   * @param conf the configuration for the cluster.
+   * @param parent the parent ClassLoader to set.
+   */
+  public ClassLoaderBase(final ClassLoader parent) {
+    super(new URL[]{}, parent);
+    Preconditions.checkNotNull(parent, "No parent classloader!");
+    this.parent = parent;
+  }
+
+  /**
+   * Returns the lock object for class loading operations.
+   */
+  protected Object getClassLoadingLock(String className) {
+    Object lock = parallelLockMap.get(className);
+    if (lock != null) {
+      return lock;
+    }
+
+    Object newLock = new Object();
+    lock = parallelLockMap.putIfAbsent(className, newLock);
+    if (lock == null) {
+      lock = newLock;
+    }
+    return lock;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
new file mode 100644
index 000000000000..f4f9e9d5cd48
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.concurrent.ConcurrentMap;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * ClassLoader used to load classes for Coprocessor instances.
+ * <p>
+ * This ClassLoader always tries to load classes from the specified coprocessor
+ * jar first actually using URLClassLoader logic before delegating to the parent
+ * ClassLoader, thus avoiding dependency conflicts between HBase's classpath and
+ * classes in the coprocessor jar.
+ * <p>
+ * Certain classes are exempt from being loaded by this ClassLoader because it
+ * would prevent them from being cast to the equivalent classes in the region
+ * server.  For example, the Coprocessor interface needs to be loaded by the
+ * region server's ClassLoader to prevent a ClassCastException when casting the
+ * coprocessor implementation.
+ * <p>
+ * A HDFS path can be used to specify the coprocessor jar. In this case, the jar
+ * will be copied to local at first under some folder under ${hbase.local.dir}/jars/tmp/.
+ * The local copy will be removed automatically when the HBase server instance is
+ * stopped.
+ * <p>
+ * This ClassLoader also handles resource loading.  In most cases this
+ * ClassLoader will attempt to load resources from the coprocessor jar first
+ * before delegating to the parent.  However, like in class loading,
+ * some resources need to be handled differently.  For all of the Hadoop
+ * default configurations (e.g. hbase-default.xml) we will check the parent
+ * ClassLoader first to prevent issues such as failing the HBase default
+ * configuration version check.
+ */
+@InterfaceAudience.Private
+public class CoprocessorClassLoader extends ClassLoaderBase {
+  private static final Log LOG = LogFactory.getLog(CoprocessorClassLoader.class);
+
+  // A temporary place ${hbase.local.dir}/jars/tmp/ to store the local
+  // copy of the jar file and the libraries contained in the jar.
+  private static final String TMP_JARS_DIR = File.separator
+     + "jars" + File.separator + "tmp" + File.separator;
+
+  /**
+   * External class loaders cache keyed by external jar path.
+   * ClassLoader instance is stored as a weak-reference
+   * to allow GC'ing when it is not used
+   * (@see HBASE-7205)
+   */
+  private static final ConcurrentMap<Path, CoprocessorClassLoader> classLoadersCache =
+    new MapMaker().concurrencyLevel(3).weakValues().makeMap();
+
+  /**
+   * If the class being loaded starts with any of these strings, we will skip
+   * trying to load it from the coprocessor jar and instead delegate
+   * directly to the parent ClassLoader.
+   */
+  private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
+    // Java standard library:
+    "com.sun.",
+    "launcher.",
+    "java.",
+    "javax.",
+    "org.ietf",
+    "org.omg",
+    "org.w3c",
+    "org.xml",
+    "sunw.",
+    // logging
+    "org.apache.commons.logging",
+    "org.apache.log4j",
+    "com.hadoop",
+    // Hadoop/HBase/ZK:
+    "org.apache.hadoop",
+    "org.apache.zookeeper",
+  };
+
+  /**
+   * If the resource being loaded matches any of these patterns, we will first
+   * attempt to load the resource with the parent ClassLoader.  Only if the
+   * resource is not found by the parent do we attempt to load it from the coprocessor jar.
+   */
+  private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
+      new Pattern[] {
+    Pattern.compile("^[^-]+-default\\.xml$")
+  };
+
+  /**
+   * Creates a JarClassLoader that loads classes from the given paths.
+   */
+  private CoprocessorClassLoader(ClassLoader parent) {
+    super(parent);
+  }
+
+  private void init(Path path, String pathPrefix,
+      Configuration conf) throws IOException {
+    if (path == null) {
+      throw new IOException("The jar path is null");
+    }
+    if (!path.toString().endsWith(".jar")) {
+      throw new IOException(path.toString() + ": not a jar file?");
+    }
+
+    // Copy the jar to the local filesystem
+    String parentDirPath =
+      conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR;
+    File parentDir = new File(parentDirPath);
+    if (!parentDir.mkdirs() && !parentDir.isDirectory()) {
+      throw new RuntimeException("Failed to create local dir " + parentDir.getPath()
+        + ", CoprocessorClassLoader failed to init");
+    }
+
+    FileSystem fs = path.getFileSystem(conf);
+    File dst = new File(parentDir, "." + pathPrefix + "."
+      + path.getName() + "." + System.currentTimeMillis() + ".jar");
+    fs.copyToLocalFile(path, new Path(dst.toString()));
+    dst.deleteOnExit();
+
+    addURL(dst.getCanonicalFile().toURI().toURL());
+
+    JarFile jarFile = new JarFile(dst.toString());
+    try {
+      Enumeration<JarEntry> entries = jarFile.entries();
+      while (entries.hasMoreElements()) {
+        JarEntry entry = entries.nextElement();
+        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
+          File file = new File(parentDir, "." + pathPrefix + "." + path.getName()
+            + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
+          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
+          file.deleteOnExit();
+          addURL(file.toURI().toURL());
+        }
+      }
+    } finally {
+      jarFile.close();
+    }
+  }
+
+  // This method is used in unit test
+  public static CoprocessorClassLoader getIfCached(final Path path) {
+    if (path == null) return null; // No class loader for null path
+    return classLoadersCache.get(path);
+  }
+
+  // This method is used in unit test
+  public static Collection<? extends ClassLoader> getAllCached() {
+    return classLoadersCache.values();
+  }
+
+  // This method is used in unit test
+  public static void clearCache() {
+    classLoadersCache.clear();
+  }
+
+  /**
+   * Get a CoprocessorClassLoader for a coprocessor jar path from cache.
+   * If not in cache, create one.
+   *
+   * @param path the path to the coprocessor jar file to load classes from
+   * @param parent the parent class loader for exempted classes
+   * @param pathPrefix a prefix used in temp path name to store the jar file locally
+   * @param conf the configuration used to create the class loader, if needed
+   * @return a CoprocessorClassLoader for the coprocessor jar path
+   * @throws IOException
+   */
+  public static CoprocessorClassLoader getClassLoader(final Path path,
+      final ClassLoader parent, final String pathPrefix,
+      final Configuration conf) throws IOException {
+    CoprocessorClassLoader cl = getIfCached(path);
+    if (cl != null){
+      LOG.debug("Found classloader "+ cl + "for "+ path.toString());
+      return cl;
+    }
+
+    cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
+      @Override
+      public CoprocessorClassLoader run() {
+        return new CoprocessorClassLoader(parent);
+      }
+    });
+
+    cl.init(path, pathPrefix, conf);
+
+    // Cache class loader as a weak value, will be GC'ed when no reference left
+    CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
+    if (prev != null) {
+      // Lost update race, use already added class loader
+      cl = prev;
+    }
+    return cl;
+  }
+
+  @Override
+  public Class<?> loadClass(String name)
+      throws ClassNotFoundException {
+    // Delegate to the parent immediately if this class is exempt
+    if (isClassExempt(name)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Skipping exempt class " + name +
+            " - delegating directly to parent");
+      }
+      return parent.loadClass(name);
+    }
+
+    synchronized (getClassLoadingLock(name)) {
+      // Check whether the class has already been loaded:
+      Class<?> clasz = findLoadedClass(name);
+      if (clasz != null) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Class " + name + " already loaded");
+        }
+      }
+      else {
+        try {
+          // Try to find this class using the URLs passed to this ClassLoader
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Finding class: " + name);
+          }
+          clasz = findClass(name);
+        } catch (ClassNotFoundException e) {
+          // Class not found using this ClassLoader, so delegate to parent
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Class " + name + " not found - delegating to parent");
+          }
+          try {
+            clasz = parent.loadClass(name);
+          } catch (ClassNotFoundException e2) {
+            // Class not found in this ClassLoader or in the parent ClassLoader
+            // Log some debug output before re-throwing ClassNotFoundException
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Class " + name + " not found in parent loader");
+            }
+            throw e2;
+          }
+        }
+      }
+      return clasz;
+    }
+  }
+
+  @Override
+  public URL getResource(String name) {
+    URL resource = null;
+    boolean parentLoaded = false;
+
+    // Delegate to the parent first if necessary
+    if (loadResourceUsingParentFirst(name)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Checking parent first for resource " + name);
+      }
+      resource = super.getResource(name);
+      parentLoaded = true;
+    }
+
+    if (resource == null) {
+      synchronized (getClassLoadingLock(name)) {
+        // Try to find the resource in this jar
+        resource = findResource(name);
+        if ((resource == null) && !parentLoaded) {
+          // Not found in this jar and we haven't attempted to load
+          // the resource in the parent yet; fall back to the parent
+          resource = super.getResource(name);
+        }
+      }
+    }
+    return resource;
+  }
+
+  /**
+   * Determines whether the given class should be exempt from being loaded
+   * by this ClassLoader.
+   * @param name the name of the class to test.
+   * @return true if the class should *not* be loaded by this ClassLoader;
+   * false otherwise.
+   */
+  protected boolean isClassExempt(String name) {
+    for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
+      if (name.startsWith(exemptPrefix)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Determines whether we should attempt to load the given resource using the
+   * parent first before attempting to load the resource using this ClassLoader.
+   * @param name the name of the resource to test.
+   * @return true if we should attempt to load the resource using the parent
+   * first; false if we should attempt to load the resource using this
+   * ClassLoader first.
+   */
+  protected boolean loadResourceUsingParentFirst(String name) {
+    for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
+      if (resourcePattern.matcher(name).matches()) {
+        return true;
+      }
+    }
+    return false;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java b/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
index 01132ae8f746..b8a221fea9d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.util.HashMap;
 
 import org.apache.commons.logging.Log;
@@ -34,10 +33,17 @@
 
 /**
  * This is a class loader that can load classes dynamically from new
- * jar files under a configured folder. It always uses its parent class
- * loader to load a class at first. Only if its parent class loader
+ * jar files under a configured folder. The paths to the jar files are
+ * converted to URLs, and URLClassLoader logic is actually used to load
+ * classes. This class loader always uses its parent class loader
+ * to load a class at first. Only if its parent class loader
  * can not load a class, we will try to load it using the logic here.
  * <p>
+ * The configured folder can be a HDFS path. In this case, the jar files
+ * under that folder will be copied to local at first under ${hbase.local.dir}/jars/.
+ * The local copy will be updated if the remote copy is updated, according to its
+ * last modified timestamp.
+ * <p>
  * We can't unload a class already loaded. So we will use the existing
  * jar files we already know to load any class which can't be loaded
  * using the parent class loader. If we still can't load the class from
@@ -50,18 +56,15 @@
  * classes properly.
  */
 @InterfaceAudience.Private
-public class DynamicClassLoader extends URLClassLoader {
+public class DynamicClassLoader extends ClassLoaderBase {
   private static final Log LOG =
       LogFactory.getLog(DynamicClassLoader.class);
 
-  // Dynamic jars are put under ${hbase.local.dir}/dynamic/jars/
+  // Dynamic jars are put under ${hbase.local.dir}/jars/
   private static final String DYNAMIC_JARS_DIR = File.separator
-    + "dynamic" + File.separator + "jars" + File.separator;
+    + "jars" + File.separator;
 
-  /**
-   * Parent class loader used to load any class at first.
-   */
-  private final ClassLoader parent;
+  private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir";
 
   private File localDir;
 
@@ -81,18 +84,18 @@ public class DynamicClassLoader extends URLClassLoader {
    */
   public DynamicClassLoader(
       final Configuration conf, final ClassLoader parent) {
-    super(new URL[]{}, parent);
-    this.parent = parent;
+    super(parent);
 
     jarModifiedTime = new HashMap<String, Long>();
-    String localDirPath = conf.get("hbase.local.dir") + DYNAMIC_JARS_DIR;
+    String localDirPath = conf.get(
+      LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR;
     localDir = new File(localDirPath);
     if (!localDir.mkdirs() && !localDir.isDirectory()) {
       throw new RuntimeException("Failed to create local dir " + localDir.getPath()
         + ", DynamicClassLoader failed to init");
     }
 
-    String remotePath = conf.get("hbase.dynamic.jars.dir");
+    String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY);
     if (remotePath == null || remotePath.equals(localDirPath)) {
       remoteDir = null;  // ignore if it is the same as the local path
     } else {
@@ -117,33 +120,35 @@ public Class<?> loadClass(String name)
         LOG.debug("Class " + name + " not found - using dynamical class loader");
       }
 
-      // Check whether the class has already been loaded:
-      Class<?> clasz = findLoadedClass(name);
-      if (clasz != null) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Class " + name + " already loaded");
-        }
-      }
-      else {
-        try {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Finding class: " + name);
-          }
-          clasz = findClass(name);
-        } catch (ClassNotFoundException cnfe) {
-          // Load new jar files if any
+      synchronized (getClassLoadingLock(name)) {
+        // Check whether the class has already been loaded:
+        Class<?> clasz = findLoadedClass(name);
+        if (clasz != null) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Loading new jar files, if any");
+            LOG.debug("Class " + name + " already loaded");
           }
-          loadNewJars();
-
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Finding class again: " + name);
+        }
+        else {
+          try {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Finding class: " + name);
+            }
+            clasz = findClass(name);
+          } catch (ClassNotFoundException cnfe) {
+            // Load new jar files if any
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Loading new jar files, if any");
+            }
+            loadNewJars();
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Finding class again: " + name);
+            }
+            clasz = findClass(name);
           }
-          clasz = findClass(name);
         }
+        return clasz;
       }
-      return clasz;
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index eb5d136c50a4..46c3ff38ff2c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -153,8 +153,8 @@ public void testDynamicFilter() throws Exception {
     }
 
     Configuration conf = HBaseConfiguration.create();
-    String localPath = conf.get("hbase.local.dir") + File.separator
-      + "dynamic" + File.separator + "jars" + File.separator;
+    String localPath = conf.get("hbase.local.dir")
+      + File.separator + "jars" + File.separator;
     File jarFile = new File(localPath, "MockFilter.jar");
     jarFile.deleteOnExit();
 
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index ba03e068cde9..df92c622ffe9 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -26,11 +26,13 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+
+import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
+import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import javax.tools.*;
 import java.io.*;
 import java.util.*;
 import java.util.jar.*;
@@ -39,6 +41,7 @@
 import org.junit.experimental.categories.Category;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 
@@ -100,91 +103,11 @@ public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  // generate jar file
-  private boolean createJarArchive(File archiveFile, File[] tobeJared) {
-    try {
-      byte buffer[] = new byte[BUFFER_SIZE];
-      // Open archive file
-      FileOutputStream stream = new FileOutputStream(archiveFile);
-      JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
-      for (int i = 0; i < tobeJared.length; i++) {
-        if (tobeJared[i] == null || !tobeJared[i].exists()
-            || tobeJared[i].isDirectory()) {
-          continue;
-        }
-
-        // Add archive entry
-        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
-        jarAdd.setTime(tobeJared[i].lastModified());
-        out.putNextEntry(jarAdd);
-
-        // Write file to archive
-        FileInputStream in = new FileInputStream(tobeJared[i]);
-        while (true) {
-          int nRead = in.read(buffer, 0, buffer.length);
-          if (nRead <= 0)
-            break;
-          out.write(buffer, 0, nRead);
-        }
-        in.close();
-      }
-      out.close();
-      stream.close();
-      LOG.info("Adding classes to jar file completed");
-      return true;
-    } catch (Exception ex) {
-      LOG.error("Error: " + ex.getMessage());
-      return false;
-    }
-  }
-
-  private File buildCoprocessorJar(String className) throws Exception {
-    // compose a java source file.
-    String javaCode = "import org.apache.hadoop.hbase.coprocessor.*;" +
+  static File buildCoprocessorJar(String className) throws Exception {
+    String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
       "public class " + className + " extends BaseRegionObserver {}";
-    Path baseDir = TEST_UTIL.getDataTestDir();
-    Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
-    File srcDirPath = new File(srcDir.toString());
-    srcDirPath.mkdirs();
-    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
-    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
-    bw.write(javaCode);
-    bw.close();
-
-    // compile it by JavaCompiler
-    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
-    ArrayList<String> srcFileNames = new ArrayList<String>();
-    srcFileNames.add(sourceCodeFile.toString());
-    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
-      null);
-    Iterable<? extends JavaFileObject> cu =
-      fm.getJavaFileObjects(sourceCodeFile);
-    List<String> options = new ArrayList<String>();
-    options.add("-classpath");
-    // only add hbase classes to classpath. This is a little bit tricky: assume
-    // the classpath is {hbaseSrc}/target/classes.
-    String currentDir = new File(".").getAbsolutePath();
-    String classpath =
-        currentDir + File.separator + "target"+ File.separator + "classes" +
-        System.getProperty("path.separator") +
-        System.getProperty("surefire.test.class.path");
-    options.add(classpath);
-    LOG.debug("Setting classpath to: "+classpath);
-
-    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
-      options, null, cu);
-    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
-
-    // build a jar file by the classes files
-    String jarFileName = className + ".jar";
-    File jarFile = new File(baseDir.toString(), jarFileName);
-    if (!createJarArchive(jarFile,
-        new File[]{new File(srcDir.toString(), className + ".class")})){
-      assertTrue("Build jar file failed.", false);
-    }
-
-    return jarFile;
+    return ClassLoaderTestHelper.buildJar(
+      TEST_UTIL.getDataTestDir().toString(), className, code);
   }
 
   @Test
@@ -231,7 +154,7 @@ public void testClassLoadingFromHDFS() throws Exception {
       admin.disableTable(tableName);
       admin.deleteTable(tableName);
     }
-    CoprocessorHost.classLoadersCache.clear();
+    CoprocessorClassLoader.clearCache();
     byte[] startKey = {10, 63};
     byte[] endKey = {12, 43};
     admin.createTable(htd, startKey, endKey, 4);
@@ -278,22 +201,22 @@ public void testClassLoadingFromHDFS() throws Exception {
     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
     // check if CP classloaders are cached
-    assertTrue(jarFileOnHDFS1 + " was not cached",
-      CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS1));
-    assertTrue(jarFileOnHDFS2 + " was not cached",
-      CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS2));
+    assertNotNull(jarFileOnHDFS1 + " was not cached",
+      CoprocessorClassLoader.getIfCached(pathOnHDFS1));
+    assertNotNull(jarFileOnHDFS2 + " was not cached",
+      CoprocessorClassLoader.getIfCached(pathOnHDFS2));
     //two external jar used, should be one classloader per jar
     assertEquals("The number of cached classloaders should be equal to the number" +
       " of external jar files",
-      2, CoprocessorHost.classLoadersCache.size());
+      2, CoprocessorClassLoader.getAllCached().size());
     //check if region active classloaders are shared across all RS regions
     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
-        CoprocessorHost.classLoadersCache.values());
+      CoprocessorClassLoader.getAllCached());
     for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
-            + " ClassLoader Cache:" + externalClassLoaders
-            + " Region ClassLoaders:" + regionCP.getValue(),
-            externalClassLoaders.containsAll(regionCP.getValue()));
+        + " ClassLoader Cache:" + externalClassLoaders
+        + " Region ClassLoaders:" + regionCP.getValue(),
+        externalClassLoaders.containsAll(regionCP.getValue()));
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java b/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java
new file mode 100644
index 000000000000..55a3feb0d691
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Some utilities to help class loader testing
+ */
+public class ClassLoaderTestHelper {
+  private static final Log LOG = LogFactory.getLog(ClassLoaderTestHelper.class);
+
+  /**
+   * Jar a list of files into a jar archive.
+   *
+   * @param archiveFile the target jar archive
+   * @param tobejared a list of files to be jared
+   */
+  private static boolean createJarArchive(File archiveFile, File[] tobeJared) {
+    try {
+      byte buffer[] = new byte[4096];
+      // Open archive file
+      FileOutputStream stream = new FileOutputStream(archiveFile);
+      JarOutputStream out = new JarOutputStream(stream, new Manifest());
+
+      for (int i = 0; i < tobeJared.length; i++) {
+        if (tobeJared[i] == null || !tobeJared[i].exists()
+            || tobeJared[i].isDirectory()) {
+          continue;
+        }
+
+        // Add archive entry
+        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
+        jarAdd.setTime(tobeJared[i].lastModified());
+        out.putNextEntry(jarAdd);
+
+        // Write file to archive
+        FileInputStream in = new FileInputStream(tobeJared[i]);
+        while (true) {
+          int nRead = in.read(buffer, 0, buffer.length);
+          if (nRead <= 0)
+            break;
+          out.write(buffer, 0, nRead);
+        }
+        in.close();
+      }
+      out.close();
+      stream.close();
+      LOG.info("Adding classes to jar file completed");
+      return true;
+    } catch (Exception ex) {
+      LOG.error("Error: " + ex.getMessage());
+      return false;
+    }
+  }
+
+  /**
+   * Create a test jar for testing purpose for a given class
+   * name with specified code string: save the class to a file,
+   * compile it, and jar it up. If the code string passed in is
+   * null, a bare empty class will be created and used.
+   *
+   * @param testDir the folder under which to store the test class and jar
+   * @param className the test class name
+   * @param code the optional test class code, which can be null.
+   * If null, a bare empty class will be used
+   * @return the test jar file generated
+   */
+  public static File buildJar(String testDir,
+      String className, String code) throws Exception {
+    return buildJar(testDir, className, code, testDir);
+  }
+
+  /**
+   * Create a test jar for testing purpose for a given class
+   * name with specified code string.
+   *
+   * @param testDir the folder under which to store the test class
+   * @param className the test class name
+   * @param code the optional test class code, which can be null.
+   * If null, an empty class will be used
+   * @param folder the folder under which to store the generated jar
+   * @return the test jar file generated
+   */
+  public static File buildJar(String testDir,
+      String className, String code, String folder) throws Exception {
+    String javaCode = code != null ? code : "public class " + className + " {}";
+    Path srcDir = new Path(testDir, "src");
+    File srcDirPath = new File(srcDir.toString());
+    srcDirPath.mkdirs();
+    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
+    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
+    bw.write(javaCode);
+    bw.close();
+
+    // compile it by JavaCompiler
+    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+    ArrayList<String> srcFileNames = new ArrayList<String>();
+    srcFileNames.add(sourceCodeFile.toString());
+    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
+      null);
+    Iterable<? extends JavaFileObject> cu =
+      fm.getJavaFileObjects(sourceCodeFile);
+    List<String> options = new ArrayList<String>();
+    options.add("-classpath");
+    // only add hbase classes to classpath. This is a little bit tricky: assume
+    // the classpath is {hbaseSrc}/target/classes.
+    String currentDir = new File(".").getAbsolutePath();
+    String classpath = currentDir + File.separator + "target"+ File.separator
+      + "classes" + System.getProperty("path.separator")
+      + System.getProperty("java.class.path") + System.getProperty("path.separator")
+      + System.getProperty("surefire.test.class.path");
+    options.add(classpath);
+    LOG.debug("Setting classpath to: " + classpath);
+
+    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
+      options, null, cu);
+    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
+
+    // build a jar file by the classes files
+    String jarFileName = className + ".jar";
+    File jarFile = new File(folder, jarFileName);
+    if (!createJarArchive(jarFile,
+        new File[]{new File(srcDir.toString(), className + ".class")})){
+      assertTrue("Build jar file failed.", false);
+    }
+    return jarFile;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java b/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
index ab5e788585e7..de2f77ef7cc1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
@@ -18,29 +18,14 @@
  */
 package org.apache.hadoop.hbase.util;
 
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-
-import javax.tools.JavaCompiler;
-import javax.tools.JavaFileObject;
-import javax.tools.StandardJavaFileManager;
-import javax.tools.ToolProvider;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.SmallTests;
@@ -62,95 +47,13 @@ public class TestDynamicClassLoader {
     conf.set("hbase.dynamic.jars.dir", TEST_UTIL.getDataTestDir().toString());
   }
 
-  // generate jar file
-  private boolean createJarArchive(File archiveFile, File[] tobeJared) {
-    try {
-      byte buffer[] = new byte[4096];
-      // Open archive file
-      FileOutputStream stream = new FileOutputStream(archiveFile);
-      JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
-      for (int i = 0; i < tobeJared.length; i++) {
-        if (tobeJared[i] == null || !tobeJared[i].exists()
-            || tobeJared[i].isDirectory()) {
-          continue;
-        }
-
-        // Add archive entry
-        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
-        jarAdd.setTime(tobeJared[i].lastModified());
-        out.putNextEntry(jarAdd);
-
-        // Write file to archive
-        FileInputStream in = new FileInputStream(tobeJared[i]);
-        while (true) {
-          int nRead = in.read(buffer, 0, buffer.length);
-          if (nRead <= 0)
-            break;
-          out.write(buffer, 0, nRead);
-        }
-        in.close();
-      }
-      out.close();
-      stream.close();
-      LOG.info("Adding classes to jar file completed");
-      return true;
-    } catch (Exception ex) {
-      LOG.error("Error: " + ex.getMessage());
-      return false;
-    }
-  }
-
-  private File buildJar(
-      String className, String folder) throws Exception {
-    String javaCode = "public class " + className + " {}";
-    Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
-    File srcDirPath = new File(srcDir.toString());
-    srcDirPath.mkdirs();
-    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
-    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
-    bw.write(javaCode);
-    bw.close();
-
-    // compile it by JavaCompiler
-    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
-    ArrayList<String> srcFileNames = new ArrayList<String>();
-    srcFileNames.add(sourceCodeFile.toString());
-    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
-      null);
-    Iterable<? extends JavaFileObject> cu =
-      fm.getJavaFileObjects(sourceCodeFile);
-    List<String> options = new ArrayList<String>();
-    options.add("-classpath");
-    // only add hbase classes to classpath. This is a little bit tricky: assume
-    // the classpath is {hbaseSrc}/target/classes.
-    String currentDir = new File(".").getAbsolutePath();
-    String classpath =
-        currentDir + File.separator + "target"+ File.separator + "classes" +
-        System.getProperty("path.separator") + System.getProperty("java.class.path");
-    options.add(classpath);
-    LOG.debug("Setting classpath to: "+classpath);
-
-    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
-      options, null, cu);
-    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
-
-    // build a jar file by the classes files
-    String jarFileName = className + ".jar";
-    File jarFile = new File(folder, jarFileName);
-    if (!createJarArchive(jarFile,
-        new File[]{new File(srcDir.toString(), className + ".class")})){
-      assertTrue("Build jar file failed.", false);
-    }
-    return jarFile;
-  }
-
   @Test
   public void testLoadClassFromLocalPath() throws Exception {
     ClassLoader parent = TestDynamicClassLoader.class.getClassLoader();
     DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
 
     String className = "TestLoadClassFromLocalPath";
+    deleteClass(className);
     try {
       classLoader.loadClass(className);
       fail("Should not be able to load class " + className);
@@ -159,13 +62,12 @@ public void testLoadClassFromLocalPath() throws Exception {
     }
 
     try {
-      buildJar(className, localDirPath());
+      String folder = TEST_UTIL.getDataTestDir().toString();
+      ClassLoaderTestHelper.buildJar(folder, className, null, localDirPath());
       classLoader.loadClass(className);
     } catch (ClassNotFoundException cnfe) {
       LOG.error("Should be able to load class " + className, cnfe);
       fail(cnfe.getMessage());
-    } finally {
-      deleteClass(className);
     }
   }
 
@@ -175,6 +77,7 @@ public void testLoadClassFromAnotherPath() throws Exception {
     DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
 
     String className = "TestLoadClassFromAnotherPath";
+    deleteClass(className);
     try {
       classLoader.loadClass(className);
       fail("Should not be able to load class " + className);
@@ -183,31 +86,33 @@ public void testLoadClassFromAnotherPath() throws Exception {
     }
 
     try {
-      buildJar(className, TEST_UTIL.getDataTestDir().toString());
+      String folder = TEST_UTIL.getDataTestDir().toString();
+      ClassLoaderTestHelper.buildJar(folder, className, null);
       classLoader.loadClass(className);
     } catch (ClassNotFoundException cnfe) {
       LOG.error("Should be able to load class " + className, cnfe);
       fail(cnfe.getMessage());
-    } finally {
-      deleteClass(className);
     }
   }
 
   private String localDirPath() {
-    return conf.get("hbase.local.dir") + File.separator
-      + "dynamic" + File.separator + "jars" + File.separator;
+    return conf.get("hbase.local.dir")
+      + File.separator + "jars" + File.separator;
   }
 
   private void deleteClass(String className) throws Exception {
     String jarFileName = className + ".jar";
     File file = new File(TEST_UTIL.getDataTestDir().toString(), jarFileName);
-    file.deleteOnExit();
+    file.delete();
+    assertFalse("Should be deleted: " + file.getPath(), file.exists());
 
     file = new File(conf.get("hbase.dynamic.jars.dir"), jarFileName);
-    file.deleteOnExit();
+    file.delete();
+    assertFalse("Should be deleted: " + file.getPath(), file.exists());
 
     file = new File(localDirPath(), jarFileName);
-    file.deleteOnExit();
+    file.delete();
+    assertFalse("Should be deleted: " + file.getPath(), file.exists());
   }
 
   @org.junit.Rule

From bad6cc1b13eccf19b603112ec716250ce7c9cd09 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 24 Apr 2013 18:08:28 +0000
Subject: [PATCH 0950/1540] HBASE-8381 TestTableInputFormatScan on Hadoop 2
 fails because YARN kills our applications

This patch bumps up the vmem-pmem ratio so that yarn's resource checker does not kill jobs that exceed the vmem (virtual memory!) allocation limit.  Related to HBASE-6330.


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471570 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HBaseTestingUtility.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index dc9e0c9d8cff..3a76d89c9b2b 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1417,6 +1417,11 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
     }
     System.setProperty("hadoop.log.dir", logDir);
     c.set("mapred.output.dir", tmpDir);
+
+    // Tests were failing because this process used 6GB of virtual memory and was getting killed.
+    // we up the VM usable so that processes don't get killed.
+    conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
+
     mrCluster = new MiniMRCluster(servers,
       FileSystem.get(conf).getUri().toString(), 1);
     LOG.info("Mini mapreduce cluster started");
@@ -1743,7 +1748,6 @@ public boolean ensureSomeRegionServersAvailable(final int num)
     return startedServer;
   }
 
-
   /**
    * Make sure that at least the specified number of region servers
    * are running. We don't count the ones that are currently stopping or are

From 5e4107a5fc55515ba34356c2fb7bead63fa6447d Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 24 Apr 2013 22:51:47 +0000
Subject: [PATCH 0951/1540] HBASE-8383 Support lib/*jar inside coprocessor jar

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471756 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/ClassLoaderBase.java |  1 -
 .../hadoop/hbase/util/CoprocessorClassLoader.java     |  2 +-
 .../hadoop/hbase/coprocessor/TestClassLoading.java    | 11 ++++++++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java b/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
index 00a0f8e08cfa..0c470ad68484 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
@@ -51,7 +51,6 @@ public class ClassLoaderBase extends URLClassLoader {
    * Creates a DynamicClassLoader that can load classes dynamically
    * from jar files under a specific folder.
    *
-   * @param conf the configuration for the cluster.
    * @param parent the parent ClassLoader to set.
    */
   public ClassLoaderBase(final ClassLoader parent) {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index f4f9e9d5cd48..ee8ca3b26d79 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -158,7 +158,7 @@ private void init(Path path, String pathPrefix,
       Enumeration<JarEntry> entries = jarFile.entries();
       while (entries.hasMoreElements()) {
         JarEntry entry = entries.nextElement();
-        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
+        if (entry.getName().matches("[/]?lib/[^/]+\\.jar")) {
           File file = new File(parentDir, "." + pathPrefix + "." + path.getName()
             + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
           IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index df92c622ffe9..ba1b5287e99e 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -376,6 +376,15 @@ public void testHBase3810() throws Exception {
 
   @Test
   public void testClassLoadingFromLibDirInJar() throws Exception {
+    loadingClassFromLibDirInJar("/lib/");
+  }
+
+  @Test
+  public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
+    loadingClassFromLibDirInJar("lib/");
+  }
+
+  void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
     FileSystem fs = cluster.getFileSystem();
 
     File innerJarFile1 = buildCoprocessorJar(cpName1);
@@ -391,7 +400,7 @@ public void testClassLoadingFromLibDirInJar() throws Exception {
 
     for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) {
       // Add archive entry
-      JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName());
+      JarEntry jarAdd = new JarEntry(libPrefix + jarFile.getName());
       jarAdd.setTime(jarFile.lastModified());
       out.putNextEntry(jarAdd);
 

From df7dfe98867cd581b2dbf88642abd299c4d407b9 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 25 Apr 2013 00:28:51 +0000
Subject: [PATCH 0952/1540] HBASE-8427 Apache rat is incorrectly excluding test
 source files

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471794 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                         |  2 +-
 .../client/InstantSchemaChangeTestBase.java     |  0
 .../regionserver/NoOpScanPolicyObserver.java    | 17 +++++++++++++++++
 .../hbase/regionserver/TestHBase7051.java       | 17 +++++++++++++++++
 .../replication/TestReplicationSmallTests.java  | 17 +++++++++++++++++
 5 files changed, 52 insertions(+), 1 deletion(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java

diff --git a/pom.xml b/pom.xml
index 2f898812fe8d..a1ae0e2a541f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -517,7 +517,7 @@
               <exclude>**/.*</exclude>
               <exclude>**/*.tgz</exclude>
               <exclude>**/*.orig</exclude>
-              <exclude>**/test/**</exclude>
+              <exclude>test/**</exclude>
               <exclude>**/8e8ab58dcf39412da19833fcd8f687ac</exclude>
               <exclude>**/.git/**</exclude>
               <exclude>**/target/**</exclude>
diff --git a/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java b/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
index 668c04372c20..c933ade81c94 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
index a80ea5c864af..f5ce0b028c0f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.hadoop.hbase.regionserver;
 
 import static org.junit.Assert.assertEquals;
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index c1f0a5bd9a25..7da487a6db91 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.hadoop.hbase.replication;
 
 

From 18ed85e4d51bceb688064f3fa8532dc2c33ffbe9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 25 Apr 2013 01:26:51 +0000
Subject: [PATCH 0953/1540] CHANGES.txt for 0.94.7RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471802 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index ad95b5939481..fc956cb50518 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.7 - 4/17/2013
+Release 0.94.7 - 4/24/2013
 Sub-task
 
     [HBASE-7615] - Add metrics for snapshots
@@ -65,6 +65,7 @@ Bug
     [HBASE-8313] - Add Bloom filter testing for HFileOutputFormat
     [HBASE-8326] - mapreduce.TestTableInputFormatScan times out frequently
     [HBASE-8352] - Rename '.snapshot' directory
+    [HBASE-8427] - Apache Rat is incorrectly excluding test source files
 
 Improvement
 
@@ -89,6 +90,13 @@ Task
 Test
 
     [HBASE-8106] - Test to check replication log znodes move is done correctly
+    [HBASE-8260] - create generic integration test for trunk and 94 that is more deterministic, can be run for longer and is less aggressive
+
+
+Release 0.94.6.1 - 4/13/2013
+Bug
+
+    [HBASE-8259] - Snapshot backport in 0.94.6 breaks rolling restarts
 
 
 Release 0.94.6 - 3/14/2013

From 116a47f8bdba07d1cb9c508f7e05dcf0d8c21879 Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Thu, 25 Apr 2013 02:20:53 +0000
Subject: [PATCH 0954/1540] HBASE-8405 Add more custom options to how
 ClusterManager runs commands

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1471813 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/developer.xml                                  | 6 +++---
 .../java/org/apache/hadoop/hbase/HBaseClusterManager.java | 8 ++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/docbkx/developer.xml b/src/docbkx/developer.xml
index 19d89bc796b9..67b4b26e8705 100644
--- a/src/docbkx/developer.xml
+++ b/src/docbkx/developer.xml
@@ -658,9 +658,9 @@ and public client API's can be used.
 <para>
 On a distributed cluster, integration tests that use ChaosMonkey or otherwise manipulate services thru cluster manager (e.g. restart regionservers) use SSH to do it.
 To run these, test process should be able to run commands on remote end, so ssh should be configured accordingly (for example, if HBase runs under hbase
-user in your cluster, you can set up passwordless ssh for that user and run the test also under it). To facilitate that, <code>hbase.it.clustermanager.ssh.user</code> and
-<code>hbase.it.clustermanager.ssh.opts</code> configuration settings can be used. The former is the remote user that cluster manager should use to perform ssh commands.
-The latter contains additional options that are passed to SSH (for example, "-i /tmp/my-key").
+user in your cluster, you can set up passwordless ssh for that user and run the test also under it). To facilitate that, <code>hbase.it.clustermanager.ssh.user</code>, 
+<code>hbase.it.clustermanager.ssh.opts</code> and <code>hbase.it.clustermanager.ssh.beforeCommand</code> configuration settings can be used. "User" is the remote user that cluster manager should use to perform ssh commands.
+"Opts" contains additional options that are passed to SSH (for example, "-i /tmp/my-key"). "BeforeCommand" is the command that will be run immediately after SSH-ing to the server, before the actual command (for example, "su hbase").
 </para>
 
 <section xml:id="maven.build.commands.integration.tests.mini">
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
index e3fb77c0a11a..75222877a27e 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
@@ -40,6 +40,7 @@
 public class HBaseClusterManager extends ClusterManager {
   private String sshUserName;
   private String sshOptions;
+  private String sshBeforeCommand;
 
   @Override
   public void setConf(Configuration conf) {
@@ -54,6 +55,10 @@ public void setConf(Configuration conf) {
     if (!extraSshOptions.isEmpty()) {
       sshOptions = StringUtils.join(new Object[] { sshOptions, extraSshOptions }, " ");
     }
+    sshBeforeCommand =  conf.get("hbase.it.clustermanager.ssh.beforeCommand", "");
+    if (!sshBeforeCommand.isEmpty()) {
+      sshBeforeCommand += " && ";
+    }
     LOG.info("Running with SSH user [" + sshUserName + "] and options [" + sshOptions + "]");
   }
 
@@ -61,7 +66,6 @@ public void setConf(Configuration conf) {
    * Executes commands over SSH
    */
   protected class RemoteShell extends Shell.ShellCommandExecutor {
-
     private String hostname;
 
     private String sshCmd = "/usr/bin/ssh";
@@ -94,7 +98,7 @@ public String[] getExecString() {
           StringUtils.join(new String[] { sshCmd,
               (sshOptions == null) ? "" : sshOptions,
               userAndHost,
-              "\"" + StringUtils.join(super.getExecString(), " ") + "\""
+              "\"" + sshBeforeCommand + StringUtils.join(super.getExecString(), " ") + "\""
           }, " ")};
     }
 

From a3a678a227d171e4abba3ed13d5bd763d6510a25 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 25 Apr 2013 20:28:52 +0000
Subject: [PATCH 0955/1540] HBASE-8415 DisabledRegionSplitPolicy

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1475946 13f79535-47bb-0310-9956-ffa450edef68
---
 .../DisabledRegionSplitPolicy.java            | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java
new file mode 100644
index 000000000000..c5f40b334c1b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * A {@link RegionSplitPolicy} that disables region splits.
+ * This should be used with care, since it will disable automatic sharding.
+ * Most of the time, using {@link ConstantSizeRegionSplitPolicy} with a
+ * large region size (10GB, etc) is safer.
+ */
+public class DisabledRegionSplitPolicy extends RegionSplitPolicy {
+  @Override
+  protected boolean shouldSplit() {
+    return false;
+  }
+}

From 2786cd93e6e11b5ca041e68adbff359415705b6e Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 26 Apr 2013 02:19:44 +0000
Subject: [PATCH 0956/1540] HBASE-8345 Add all available resources in
 RootResource and VersionResource to rest RemoteAdmin (Aleksandr Shulman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1476028 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/client/RemoteAdmin.java | 270 ++++++++++++++++--
 .../hbase/rest/client/TestRemoteAdmin.java    |  89 +++++-
 2 files changed, 318 insertions(+), 41 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index f7c0394a32a2..a3d01973ee8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -20,14 +20,32 @@
 
 package org.apache.hadoop.hbase.rest.client;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+
 import org.apache.hadoop.conf.Configuration;
 
+import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.rest.Constants;
+import org.apache.hadoop.hbase.rest.ProtobufMessageHandler;
+import org.apache.hadoop.hbase.rest.VersionResource;
+import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel;
+import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel;
+import org.apache.hadoop.hbase.rest.model.TableListModel;
 import org.apache.hadoop.hbase.rest.model.TableSchemaModel;
+import org.apache.hadoop.hbase.rest.model.VersionModel;
+import org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus;
+import org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.mortbay.jetty.MimeTypes;
+import org.xml.sax.InputSource;
 
 public class RemoteAdmin {
 
@@ -37,8 +55,14 @@ public class RemoteAdmin {
   final int maxRetries;
   final long sleepTime;
 
+  // This unmarshaller is necessary for getting the /version/cluster resource.
+  // This resource does not support protobufs. Therefore this is necessary to
+  // request/interpret it as XML.
+  private static volatile Unmarshaller versionClusterUnmarshaller;
+
   /**
    * Constructor
+   * 
    * @param client
    * @param conf
    */
@@ -46,6 +70,16 @@ public RemoteAdmin(Client client, Configuration conf) {
     this(client, conf, null);
   }
 
+  static Unmarshaller getUnmarsheller() throws JAXBException {
+
+    if (versionClusterUnmarshaller == null) {
+
+      RemoteAdmin.versionClusterUnmarshaller = JAXBContext.newInstance(
+          StorageClusterVersionModel.class).createUnmarshaller();
+    }
+    return RemoteAdmin.versionClusterUnmarshaller;
+  }
+
   /**
    * Constructor
    * @param client
@@ -69,24 +103,156 @@ public boolean isTableAvailable(String tableName) throws IOException {
     return isTableAvailable(Bytes.toBytes(tableName));
   }
 
+  /**
+   * @return string representing the rest api's version
+   * @throws IOEXception
+   *           if the endpoint does not exist, there is a timeout, or some other
+   *           general failure mode
+   */
+  public VersionModel getRestVersion() throws IOException {
+
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    if (accessToken != null) {
+      path.append(accessToken);
+      path.append('/');
+    }
+
+    path.append("version/rest");
+
+    int code = 0;
+    for (int i = 0; i < maxRetries; i++) {
+      Response response = client.get(path.toString(),
+          Constants.MIMETYPE_PROTOBUF);
+      code = response.getCode();
+      switch (code) {
+      case 200:
+
+        VersionModel v = new VersionModel();
+        return (VersionModel) v.getObjectFromMessage(response.getBody());
+      case 404:
+        throw new IOException("REST version not found");
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+        }
+        break;
+      default:
+        throw new IOException("get request to " + path.toString()
+            + " returned " + code);
+      }
+    }
+    throw new IOException("get request to " + path.toString() + " timed out");
+  }
+
+  /**
+   * @return string representing the cluster's version
+   * @throws IOEXception if the endpoint does not exist, there is a timeout, or some other general failure mode
+   */
+  public StorageClusterStatusModel getClusterStatus() throws IOException {
+
+      StringBuilder path = new StringBuilder();
+      path.append('/');
+      if (accessToken !=null) {
+          path.append(accessToken);
+          path.append('/');
+      }
+
+    path.append("status/cluster");
+
+    int code = 0;
+    for (int i = 0; i < maxRetries; i++) {
+      Response response = client.get(path.toString(),
+          Constants.MIMETYPE_PROTOBUF);
+      code = response.getCode();
+      switch (code) {
+      case 200:
+        StorageClusterStatusModel s = new StorageClusterStatusModel();
+        return (StorageClusterStatusModel) s.getObjectFromMessage(response
+            .getBody());
+      case 404:
+        throw new IOException("Cluster version not found");
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+        }
+        break;
+      default:
+        throw new IOException("get request to " + path + " returned " + code);
+      }
+    }
+    throw new IOException("get request to " + path + " timed out");
+  }
+
+  /**
+   * @return string representing the cluster's version
+   * @throws IOEXception
+   *           if the endpoint does not exist, there is a timeout, or some other
+   *           general failure mode
+   */
+  public StorageClusterVersionModel getClusterVersion() throws IOException {
+
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    if (accessToken != null) {
+      path.append(accessToken);
+      path.append('/');
+    }
+
+    path.append("version/cluster");
+
+    int code = 0;
+    for (int i = 0; i < maxRetries; i++) {
+      Response response = client.get(path.toString(), Constants.MIMETYPE_XML);
+      code = response.getCode();
+      switch (code) {
+      case 200:
+        try {
+
+          return (StorageClusterVersionModel) getUnmarsheller().unmarshal(
+              new ByteArrayInputStream(response.getBody()));
+        } catch (JAXBException jaxbe) {
+
+          throw new IOException(
+              "Issue parsing StorageClusterVersionModel object in XML form: "
+                  + jaxbe.getLocalizedMessage());
+        }
+      case 404:
+        throw new IOException("Cluster version not found");
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+        }
+        break;
+      default:
+        throw new IOException(path.toString() + " request returned " + code);
+      }
+    }
+    throw new IOException("get request to " + path.toString()
+        + " request timed out");
+  }
+
   /**
    * @param tableName name of table to check
    * @return true if all regions of the table are available
    * @throws IOException if a remote or network exception occurs
    */
   public boolean isTableAvailable(byte[] tableName) throws IOException {
-    StringBuilder sb = new StringBuilder();
-    sb.append('/');
+    StringBuilder path = new StringBuilder();
+    path.append('/');
     if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
+      path.append(accessToken);
+      path.append('/');
     }
-    sb.append(Bytes.toStringBinary(tableName));
-    sb.append('/');
-    sb.append("exists");
+    path.append(Bytes.toStringBinary(tableName));
+    path.append('/');
+    path.append("exists");
     int code = 0;
     for (int i = 0; i < maxRetries; i++) {
-      Response response = client.get(sb.toString());
+      Response response = client.get(path.toString(), Constants.MIMETYPE_PROTOBUF);
       code = response.getCode();
       switch (code) {
       case 200:
@@ -99,10 +265,10 @@ public boolean isTableAvailable(byte[] tableName) throws IOException {
         } catch (InterruptedException e) { }
         break;
       default:
-        throw new IOException("exists request returned " + code);
+        throw new IOException("get request to " + path.toString() + " returned " + code);
       }
     }
-    throw new IOException("exists request timed out");
+    throw new IOException("get request to " + path.toString() + " timed out");
   }
 
   /**
@@ -113,18 +279,18 @@ public boolean isTableAvailable(byte[] tableName) throws IOException {
   public void createTable(HTableDescriptor desc)
       throws IOException {
     TableSchemaModel model = new TableSchemaModel(desc);
-    StringBuilder sb = new StringBuilder();
-    sb.append('/');
+    StringBuilder path = new StringBuilder();
+    path.append('/');
     if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
+      path.append(accessToken);
+      path.append('/');
     }
-    sb.append(Bytes.toStringBinary(desc.getName()));
-    sb.append('/');
-    sb.append("schema");
+    path.append(Bytes.toStringBinary(desc.getName()));
+    path.append('/');
+    path.append("schema");
     int code = 0;
     for (int i = 0; i < maxRetries; i++) {
-      Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF,
+      Response response = client.put(path.toString(), Constants.MIMETYPE_PROTOBUF,
         model.createProtobufOutput());
       code = response.getCode();
       switch (code) {
@@ -136,10 +302,10 @@ public void createTable(HTableDescriptor desc)
         } catch (InterruptedException e) { }
         break;
       default:
-        throw new IOException("create request returned " + code);
+        throw new IOException("create request to " + path.toString() + " returned " + code);
       }
     }
-    throw new IOException("create request timed out");
+    throw new IOException("create request to " + path.toString() + " timed out");
   }
 
   /**
@@ -157,18 +323,18 @@ public void deleteTable(final String tableName) throws IOException {
    * @throws IOException if a remote or network exception occurs
    */
   public void deleteTable(final byte [] tableName) throws IOException {
-    StringBuilder sb = new StringBuilder();
-    sb.append('/');
+    StringBuilder path = new StringBuilder();
+    path.append('/');
     if (accessToken != null) {
-      sb.append(accessToken);
-      sb.append('/');
+      path.append(accessToken);
+      path.append('/');
     }
-    sb.append(Bytes.toStringBinary(tableName));
-    sb.append('/');
-    sb.append("schema");
+    path.append(Bytes.toStringBinary(tableName));
+    path.append('/');
+    path.append("schema");
     int code = 0;
-    for (int i = 0; i < maxRetries; i++) { 
-      Response response = client.delete(sb.toString());
+    for (int i = 0; i < maxRetries; i++) {
+      Response response = client.delete(path.toString());
       code = response.getCode();
       switch (code) {
       case 200:
@@ -179,10 +345,52 @@ public void deleteTable(final byte [] tableName) throws IOException {
         } catch (InterruptedException e) { }
         break;
       default:
-        throw new IOException("delete request returned " + code);
+        throw new IOException("delete request to " + path.toString() + " returned " + code);
       }
     }
-    throw new IOException("delete request timed out");
+    throw new IOException("delete request to " + path.toString() + " timed out");
   }
 
+  /**
+   * @return string representing the cluster's version
+   * @throws IOEXception
+   *           if the endpoint does not exist, there is a timeout, or some other
+   *           general failure mode
+   */
+  public TableListModel getTableList() throws IOException {
+
+    StringBuilder path = new StringBuilder();
+    path.append('/');
+    if (accessToken != null) {
+      path.append(accessToken);
+      path.append('/');
+    }
+
+    int code = 0;
+    for (int i = 0; i < maxRetries; i++) {
+      // Response response = client.get(path.toString(),
+      // Constants.MIMETYPE_XML);
+      Response response = client.get(path.toString(),
+          Constants.MIMETYPE_PROTOBUF);
+      code = response.getCode();
+      switch (code) {
+      case 200:
+        TableListModel t = new TableListModel();
+        return (TableListModel) t.getObjectFromMessage(response.getBody());
+      case 404:
+        throw new IOException("Table list not found");
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+        }
+        break;
+      default:
+        throw new IOException("get request to " + path.toString()
+            + " request returned " + code);
+      }
+    }
+    throw new IOException("get request to " + path.toString()
+        + " request timed out");
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
index 5a648aa45adf..630a9d8d82c8 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
@@ -20,13 +20,22 @@
 
 package org.apache.hadoop.hbase.rest.client;
 
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.rest.HBaseRESTTestingUtility;
-import org.apache.hadoop.hbase.rest.client.Client;
+import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel;
+import org.apache.hadoop.hbase.rest.model.TableModel;
+import org.apache.hadoop.hbase.rest.model.VersionModel;
 import org.apache.hadoop.hbase.util.Bytes;
-
-import static org.junit.Assert.*;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -34,13 +43,15 @@
 
 @Category(MediumTests.class)
 public class TestRemoteAdmin {
-  private static final String TABLE_1 = "TestRemoteAdmin_Table_1";
-  private static final byte[] COLUMN_1 = Bytes.toBytes("a");
-  static final HTableDescriptor DESC_1 =  new HTableDescriptor(TABLE_1);
   private static final HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
-  private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
+  private static final HBaseRESTTestingUtility REST_TEST_UTIL =
     new HBaseRESTTestingUtility();
+  private static final String TABLE_1 = "TestRemoteAdmin_Table_1";
+  private static final String TABLE_2 = TABLE_1 + System.currentTimeMillis();
+  private static final byte[] COLUMN_1 = Bytes.toBytes("a");
+  static final HTableDescriptor DESC_1 =  new HTableDescriptor(TABLE_1);
+  static final HTableDescriptor DESC_2 =  new HTableDescriptor(TABLE_2);
   private static RemoteAdmin remoteAdmin;
 
   @BeforeClass
@@ -70,8 +81,66 @@ public void testCreateAnDeleteTable() throws Exception {
     assertFalse(remoteAdmin.isTableAvailable(TABLE_1));
   }
 
+  @Test
+  public void testGetRestVersion() throws Exception {
+
+    VersionModel RETURNED_REST_VERSION = remoteAdmin.getRestVersion();
+    System.out.print("Returned version is: " + RETURNED_REST_VERSION);
+
+    // Assert that it contains info about rest version, OS, JVM
+    assertTrue("Returned REST version did not contain info about rest.",
+        RETURNED_REST_VERSION.toString().contains("rest"));
+    assertTrue("Returned REST version did not contain info about the JVM.",
+        RETURNED_REST_VERSION.toString().contains("JVM"));
+    assertTrue("Returned REST version did not contain info about OS.",
+        RETURNED_REST_VERSION.toString().contains("OS"));
+  }
+
+  @Test
+  public void testClusterVersion() throws Exception {
+    // testing the /version/cluster endpoint
+    final String HBASE_VERSION = TEST_UTIL.getHBaseCluster().getClusterStatus()
+        .getHBaseVersion();
+    assertEquals("Cluster status from REST API did not match. ", HBASE_VERSION,
+        remoteAdmin.getClusterVersion().getVersion());
+  }
+
+  @Test
+  public void testClusterStatus() throws Exception {
+
+    ClusterStatus status = TEST_UTIL.getHBaseClusterInterface()
+        .getClusterStatus();
+    StorageClusterStatusModel returnedStatus = remoteAdmin.getClusterStatus();
+    assertEquals(
+        "Region count from cluster status and returned status did not match up. ",
+        status.getRegionsCount(), returnedStatus.getRegions());
+    assertEquals(
+        "Dead server count from cluster status and returned status did not match up. ",
+        status.getDeadServers(), returnedStatus.getDeadNodes().size());
+    assertEquals(
+        "Number of requests from cluster status and returned status did not match up. ",
+        status.getRequestsCount(), returnedStatus.getRequests());
+  }
+
+  @Test
+  public void testListTables() throws Exception {
+
+    remoteAdmin.createTable(DESC_2);
+    List<TableModel> tableList = remoteAdmin.getTableList().getTables();
+    System.out.println("List of tables is: ");
+    boolean found = false;
+    for (TableModel tm : tableList) {
+
+      if (tm.getName().equals(TABLE_2)) {
+        found = true;
+        break;
+      }
+    }
+    assertTrue("Table " + TABLE_2 + " was not found by get request to '/'",
+        found);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-

From 9f054e718524a253e47f8e1079f38c0c6b795933 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 28 Apr 2013 03:18:15 +0000
Subject: [PATCH 0957/1540] HBASE-8451 MetricsMBeanBase has concurrency issues
 in init (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1476715 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
index 75971715ad89..00d514c57e49 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
@@ -20,9 +20,9 @@
 package org.apache.hadoop.hbase.metrics;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import javax.management.AttributeNotFoundException;
 import javax.management.MBeanAttributeInfo;
@@ -55,8 +55,8 @@ public class MetricsMBeanBase extends MetricsDynamicMBeanBase {
   /** HBase MetricsBase implementations that MetricsDynamicMBeanBase does
    * not understand
    */
-  protected Map<String,MetricsBase> extendedAttributes =
-      new HashMap<String,MetricsBase>();
+  protected Map<String, MetricsBase> extendedAttributes =
+      new ConcurrentHashMap<String, MetricsBase>();
   protected MBeanInfo extendedInfo;
 
   protected MetricsMBeanBase( MetricsRegistry mr, String description ) {

From fa9422e8dc27208e3f4bb118203d94375a18e6e8 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 29 Apr 2013 22:11:46 +0000
Subject: [PATCH 0958/1540] HBASE-8453. TestImportExport failing again due to
 configuration issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1477375 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     |  4 ++
 .../hbase/mapreduce/TestImportExport.java     | 64 +++++++++++--------
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 3a76d89c9b2b..79166bb8da82 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1663,6 +1663,10 @@ public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
     this.dfsCluster = cluster;
   }
 
+  public MiniMRCluster getMRCluster() {
+    return mrCluster;
+  }
+
   public FileSystem getTestFileSystem() throws IOException {
     return HFileSystem.get(conf);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index c0cd2f156933..ff597c42d010 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -42,6 +43,7 @@
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -64,12 +66,11 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
-  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    cluster = UTIL.startMiniCluster();
+    UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -111,16 +112,16 @@ public void testSimpleCase() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
-
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -129,12 +130,13 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -158,12 +160,13 @@ public void testMetaExport() throws Exception {
     String EXPORT_TABLE = ".META.";
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
+        UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
   }
@@ -200,12 +203,13 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -224,12 +228,13 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -268,12 +273,13 @@ public void testWithFilter() throws Exception {
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
 
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
+        UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -287,12 +293,13 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
         "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -310,14 +317,15 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
         OUTPUT_DIR, "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
-    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
+    assertFalse("Job succeeded, but it had a non-instantiable filter!", job.isSuccessful());
 
     // cleanup
     exportTable.close();

From 7b0e785975d5144c787f479d9543bf40e8a3705d Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 29 Apr 2013 22:17:12 +0000
Subject: [PATCH 0959/1540] HBASE-8454. TestImportTsv failing due to
 configuration issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1477379 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/TestImportTsv.java    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index 77d044b944ca..0da6f0a07b1e 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -26,6 +26,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
@@ -289,7 +290,10 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
         LOG.info("set the hbaseAdmin");
         ImportTsv.createHbaseAdmin(conf);
       }
-      Job job = ImportTsv.createSubmittableJob(conf, args);
+
+      JobConf jobConf = htu1.getMRCluster().createJobConf();
+      HBaseConfiguration.merge(jobConf, conf);
+      Job job = ImportTsv.createSubmittableJob(jobConf, args);
       job.waitForCompletion(false);
       assertTrue(job.isSuccessful());
       

From 01976b08e89761559f076ebf1542c87cbeedd42a Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 30 Apr 2013 01:52:50 +0000
Subject: [PATCH 0960/1540] Revert HBASE-8453, HBASE-8454.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1477450 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     |  4 --
 .../hbase/mapreduce/TestImportExport.java     | 64 ++++++++-----------
 .../hadoop/hbase/mapreduce/TestImportTsv.java |  6 +-
 3 files changed, 29 insertions(+), 45 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 79166bb8da82..3a76d89c9b2b 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1663,10 +1663,6 @@ public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
     this.dfsCluster = cluster;
   }
 
-  public MiniMRCluster getMRCluster() {
-    return mrCluster;
-  }
-
   public FileSystem getTestFileSystem() throws IOException {
     return HFileSystem.get(conf);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index ff597c42d010..c0cd2f156933 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -43,7 +42,6 @@
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -66,11 +64,12 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
+  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    UTIL.startMiniCluster();
+    cluster = UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -112,16 +111,16 @@ public void testSimpleCase() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
+
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -130,13 +129,12 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -160,13 +158,12 @@ public void testMetaExport() throws Exception {
     String EXPORT_TABLE = ".META.";
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        UTIL.getConfiguration()), args);
+        cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
   }
@@ -203,13 +200,12 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -228,13 +224,12 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -273,13 +268,12 @@ public void testWithFilter() throws Exception {
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
 
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        UTIL.getConfiguration()), args);
+        cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -293,13 +287,12 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
         "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -317,15 +310,14 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
         OUTPUT_DIR, "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
-    assertFalse("Job succeeded, but it had a non-instantiable filter!", job.isSuccessful());
+    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
 
     // cleanup
     exportTable.close();
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index 0da6f0a07b1e..77d044b944ca 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -26,7 +26,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
@@ -290,10 +289,7 @@ private void doMROnTableTest(String inputFile, String family, String tableName,
         LOG.info("set the hbaseAdmin");
         ImportTsv.createHbaseAdmin(conf);
       }
-
-      JobConf jobConf = htu1.getMRCluster().createJobConf();
-      HBaseConfiguration.merge(jobConf, conf);
-      Job job = ImportTsv.createSubmittableJob(jobConf, args);
+      Job job = ImportTsv.createSubmittableJob(conf, args);
       job.waitForCompletion(false);
       assertTrue(job.isSuccessful());
       

From 5627d2d27d2cb5b4b49d7fda7b64b6d2a6d3ad7e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 2 May 2013 03:55:02 +0000
Subject: [PATCH 0961/1540] moving 0.94 to 0.94.8-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1478260 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index a1ae0e2a541f..1fe0e07b1578 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.7</version>
+  <version>0.94.8-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 6ec710c45aff28405529026eeb7dd0ad42a9e524 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 2 May 2013 08:46:58 +0000
Subject: [PATCH 0962/1540] HBASE-8455 Update ExportSnapshot to reflect changes
 in HBASE-7419

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1478300 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 17 +++--
 .../hbase/snapshot/TestExportSnapshot.java    | 72 ++++++++++++++++---
 2 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 62f2bcbaf388..46aea0f82434 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.HLogLink;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.snapshot.ExportSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
@@ -154,7 +155,7 @@ public void map(Text key, NullWritable value, Context context)
      */
     private Path getOutputPath(final Path inputPath) throws IOException {
       Path path;
-      if (HFileLink.isHFileLink(inputPath)) {
+      if (HFileLink.isHFileLink(inputPath) || StoreFile.isReference(inputPath)) {
         String family = inputPath.getParent().getName();
         String table = HFileLink.getReferencedTableName(inputPath.getName());
         String region = HFileLink.getReferencedRegionName(inputPath.getName());
@@ -183,10 +184,12 @@ private boolean copyFile(final Context context, final Path inputPath, final Path
         if (inputStat == null) return false;
 
         // Verify if the output file exists and is the same that we want to copy
-        FileStatus outputStat = getFileStatus(outputFs, outputPath);
-        if (outputStat != null && sameFile(inputStat, outputStat)) {
-          LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
-          return true;
+        if (outputFs.exists(outputPath)) {
+          FileStatus outputStat = outputFs.getFileStatus(outputPath);
+          if (sameFile(inputStat, outputStat)) {
+            LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+            return true;
+          }
         }
 
         context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
@@ -295,7 +298,7 @@ private boolean copyData(final Context context,
 
     private FSDataInputStream openSourceFile(final Path path) {
       try {
-        if (HFileLink.isHFileLink(path)) {
+        if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
           return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
         } else if (isHLogLinkPath(path)) {
           String serverName = path.getParent().getName();
@@ -311,7 +314,7 @@ private FSDataInputStream openSourceFile(final Path path) {
 
     private FileStatus getFileStatus(final FileSystem fs, final Path path) {
       try {
-        if (HFileLink.isHFileLink(path)) {
+        if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
           HFileLink link = new HFileLink(inputRoot, inputArchive, path);
           return link.getFileStatus(fs);
         } else if (isHLogLinkPath(path)) {
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index f5e0c8d67145..9dafa9a9e21b 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -36,6 +36,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -51,6 +53,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
 import org.apache.hadoop.mapreduce.Job;
@@ -110,18 +114,14 @@ public void setUp() throws Exception {
     admin.createTable(htd, null);
 
     // Take an empty snapshot
-    admin.disableTable(tableName);
     admin.snapshot(emptySnapshotName, tableName);
-    admin.enableTable(tableName);
 
     // Add some rows
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
     TEST_UTIL.loadTable(table, FAMILY);
 
     // take a snapshot
-    admin.disableTable(tableName);
     admin.snapshot(snapshotName, tableName);
-    admin.enableTable(tableName);
   }
 
   @After
@@ -183,6 +183,56 @@ public void testEmptyExportFileSystemState() throws Exception {
     testExportFileSystemState(tableName, emptySnapshotName, 1);
   }
 
+  /**
+   * Mock a snapshot with files in the archive dir,
+   * two regions, and one reference file.
+   */
+  @Test
+  public void testSnapshotWithRefsExportFileSystemState() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+
+    final byte[] tableWithRefsName = Bytes.toBytes("tableWithRefs");
+    final String snapshotName = "tableWithRefs";
+    final String TEST_FAMILY = Bytes.toString(FAMILY);
+    final String TEST_HFILE = "abc";
+
+    final SnapshotDescription sd = SnapshotDescription.newBuilder()
+        .setName(snapshotName).setTable(Bytes.toString(tableWithRefsName)).build();
+
+    FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+    HTableDescriptor htd = new HTableDescriptor(tableWithRefsName);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+    // First region, simple with one plain hfile.
+    HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+        conf, htd, null, true, true);
+    Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE);
+    FSDataOutputStream out = fs.create(storeFile);
+    out.write(Bytes.toBytes("Test Data"));
+    out.close();
+    r0.close();
+
+    // Second region, used to test the split case.
+    // This region contains a reference to the hfile in the first region.
+    HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+        conf, htd, null, true, true);
+    out = fs.create(new Path(new Path(r1.getRegionDir(), TEST_FAMILY),
+        storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName()));
+    out.write(Bytes.toBytes("Test Data"));
+    out.close();
+    r1.close();
+
+    Path tableDir = HTableDescriptor.getTableDir(archiveDir, tableWithRefsName);
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
+    SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
+
+    testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
+  }
+
   /**
    * Test ExportSnapshot
    */
@@ -214,7 +264,8 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
     final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
     verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
         fs, new Path(copyDir, snapshotDir));
-    verifyArchive(fs, copyDir, Bytes.toString(snapshotName));
+    verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
+    FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
 
     // Remove the exported dir
     fs.delete(copyDir, true);
@@ -232,11 +283,11 @@ private void verifySnapshot(final FileSystem fs1, final Path root1,
   /*
    * Verify if the files exists
    */
-  private void verifyArchive(final FileSystem fs, final Path rootDir, final String snapshotName)
-      throws IOException {
+  private void verifyArchive(final FileSystem fs, final Path rootDir,
+      final byte[] tableName, final String snapshotName) throws IOException {
     final Path exportedSnapshot = new Path(rootDir,
       new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
-    final Path exportedArchive = new Path(rootDir, ".archive");
+    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
     LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
     SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
         new SnapshotReferenceUtil.FileVisitor() {
@@ -258,9 +309,8 @@ public void logFile (final String server, final String logfile)
         }
 
         private void verifyNonEmptyFile(final Path path) throws IOException {
-          LOG.debug(path);
-          assertTrue(fs.exists(path));
-          assertTrue(fs.getFileStatus(path).getLen() > 0);
+          assertTrue(path + " should exist", fs.exists(path));
+          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
         }
     });
   }

From d0d91bf9c5422bdda8bba7bb8d765b0c6f7c68c0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 4 May 2013 04:06:55 +0000
Subject: [PATCH 0963/1540] HBASE-7122 Proper warning message when opening a
 log file with no entries (idle cluster) (Himanshu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1479037 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/ReplicationSource.java   | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index c00f0580ce12..d902010c1c3c 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -611,6 +611,7 @@ protected boolean openReader(int sleepMultiplier) {
         }
       }
     } catch (IOException ioe) {
+      if (ioe instanceof EOFException && isCurrentLogEmpty()) return true;
       LOG.warn(peerClusterZnode + " Got: ", ioe);
       this.reader = null;
       if (ioe.getCause() instanceof NullPointerException) {
@@ -628,6 +629,16 @@ protected boolean openReader(int sleepMultiplier) {
     return true;
   }
 
+  /*
+   * Checks whether the current log file is empty, and it is not a recovered queue. This is to
+   * handle scenario when in an idle cluster, there is no entry in the current log and we keep on
+   * trying to read the log file and get EOFEception. In case of a recovered queue the last log file
+   * may be empty, and we don't want to retry that.
+   */
+  private boolean isCurrentLogEmpty() {
+    return (this.repLogReader.getPosition() == 0 && !queueRecovered && queue.size() == 0);
+  }
+
   /**
    * Do the sleeping logic
    * @param msg Why we sleep

From 9da76b79026d745117ab128515412bac71b492cb Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Tue, 7 May 2013 01:47:29 +0000
Subject: [PATCH 0964/1540] HBASE-8405 Add more custom options to how
 ClusterManager runs commands REDO DIFFERENTLY

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1479745 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/developer.xml                      |  6 ++--
 .../hadoop/hbase/HBaseClusterManager.java     | 29 +++++++++----------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/src/docbkx/developer.xml b/src/docbkx/developer.xml
index 67b4b26e8705..854b6f6038fe 100644
--- a/src/docbkx/developer.xml
+++ b/src/docbkx/developer.xml
@@ -659,8 +659,10 @@ and public client API's can be used.
 On a distributed cluster, integration tests that use ChaosMonkey or otherwise manipulate services thru cluster manager (e.g. restart regionservers) use SSH to do it.
 To run these, test process should be able to run commands on remote end, so ssh should be configured accordingly (for example, if HBase runs under hbase
 user in your cluster, you can set up passwordless ssh for that user and run the test also under it). To facilitate that, <code>hbase.it.clustermanager.ssh.user</code>, 
-<code>hbase.it.clustermanager.ssh.opts</code> and <code>hbase.it.clustermanager.ssh.beforeCommand</code> configuration settings can be used. "User" is the remote user that cluster manager should use to perform ssh commands.
-"Opts" contains additional options that are passed to SSH (for example, "-i /tmp/my-key"). "BeforeCommand" is the command that will be run immediately after SSH-ing to the server, before the actual command (for example, "su hbase").
+<code>hbase.it.clustermanager.ssh.opts</code> and <code>hbase.it.clustermanager.ssh.cmd</code> configuration settings can be used. "User" is the remote user that cluster manager should use to perform ssh commands.
+"Opts" contains additional options that are passed to SSH (for example, "-i /tmp/my-key"). 
+Finally, if you have some custom environment setup, "cmd" is the override format for the entire tunnel (ssh) command. The default string is {<code>/usr/bin/ssh %1$s %2$s%3$s%4$s "%5$s"</code>} and is a good starting point. This is a standard Java format string with 5 arguments that is used to execute the remote command. The argument 1 (%1$s) is SSH options set the via opts setting or via environment variable, 2 is SSH user name, 3 is "@" if username is set or "" otherwise, 4 is the target host name, and 5 is the logical command to execute (that may include single quotes, so don't use them). For example, if you run the tests under non-hbase user and want to ssh as that user and change to hbase on remote machine, you can use {<code>/usr/bin/ssh %1$s %2$s%3$s%4$s "su hbase - -c \"%5$s\""</code>}. That way, to kill RS (for example) integration tests may run {<code>/usr/bin/ssh some-hostname "su hbase - -c \"ps aux | ... | kill ...\""</code>}.
+The command is logged in the test logs, so you can verify it is correct for your environment.
 </para>
 
 <section xml:id="maven.build.commands.integration.tests.mini">
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
index 75222877a27e..7209ddcb01d9 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
@@ -40,7 +40,13 @@
 public class HBaseClusterManager extends ClusterManager {
   private String sshUserName;
   private String sshOptions;
-  private String sshBeforeCommand;
+
+  /**
+   * The command format that is used to execute the remote command. Arguments:
+   * 1 SSH options, 2 user name , 3 "@" if username is set, 4 host, 5 original command.
+   */
+  private static final String DEFAULT_TUNNEL_CMD = "/usr/bin/ssh %1$s %2$s%3$s%4$s \"%5$s\"";
+  private String tunnelCmd;
 
   @Override
   public void setConf(Configuration conf) {
@@ -55,10 +61,8 @@ public void setConf(Configuration conf) {
     if (!extraSshOptions.isEmpty()) {
       sshOptions = StringUtils.join(new Object[] { sshOptions, extraSshOptions }, " ");
     }
-    sshBeforeCommand =  conf.get("hbase.it.clustermanager.ssh.beforeCommand", "");
-    if (!sshBeforeCommand.isEmpty()) {
-      sshBeforeCommand += " && ";
-    }
+    sshOptions = (sshOptions == null) ? "" : sshOptions;
+    tunnelCmd = conf.get("hbase.it.clustermanager.ssh.cmd", DEFAULT_TUNNEL_CMD);
     LOG.info("Running with SSH user [" + sshUserName + "] and options [" + sshOptions + "]");
   }
 
@@ -68,8 +72,6 @@ public void setConf(Configuration conf) {
   protected class RemoteShell extends Shell.ShellCommandExecutor {
     private String hostname;
 
-    private String sshCmd = "/usr/bin/ssh";
-
     public RemoteShell(String hostname, String[] execString, File dir, Map<String, String> env,
         long timeout) {
       super(execString, dir, env, timeout);
@@ -92,14 +94,11 @@ public RemoteShell(String hostname, String[] execString) {
     }
 
     public String[] getExecString() {
-      String userAndHost = sshUserName.isEmpty() ? hostname : (sshUserName + "@" + hostname);
-      return new String[] {
-          "bash", "-c",
-          StringUtils.join(new String[] { sshCmd,
-              (sshOptions == null) ? "" : sshOptions,
-              userAndHost,
-              "\"" + sshBeforeCommand + StringUtils.join(super.getExecString(), " ") + "\""
-          }, " ")};
+      String at = sshUserName.isEmpty() ? "" : "@";
+      String remoteCmd = StringUtils.join(super.getExecString(), " ");
+      String cmd = String.format(tunnelCmd, sshOptions, sshUserName, at, hostname, remoteCmd);
+      LOG.info("Executing full command [" + cmd + "]");
+      return new String[] { "/usr/bin/env", "bash", "-c", cmd };
     }
 
     @Override

From 4ef73742ca340ad3ff5dd288d21d22bfb90a6c17 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 7 May 2013 20:51:56 +0000
Subject: [PATCH 0965/1540] HBASE-8483 HConnectionManager can leak ZooKeeper
 connections when using deleteStaleConnection

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1480070 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 17 +++--
 .../apache/hadoop/hbase/client/TestHCM.java   | 62 +++++++++++++++++++
 2 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 80bf48cf903b..e813f7a6dac1 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1392,6 +1392,9 @@ public synchronized ZooKeeperWatcher getZooKeeperWatcher()
         throws ZooKeeperConnectionException {
       if(zooKeeper == null) {
         try {
+          if (this.closed) {
+            throw new IOException(toString() + " closed");
+          }
           this.zooKeeper = new ZooKeeperWatcher(conf, "hconnection", this);
         } catch(ZooKeeperConnectionException zce) {
           throw zce;
@@ -1813,13 +1816,15 @@ void internalClose() {
         this.rpcEngine.close();
       }
 
-      if (this.zooKeeper != null) {
-        LOG.info("Closed zookeeper sessionid=0x" +
-          Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
-        this.zooKeeper.close();
-        this.zooKeeper = null;
+      synchronized (this) {
+        if (this.zooKeeper != null) {
+          LOG.info("Closed zookeeper sessionid=0x" +
+            Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
+          this.zooKeeper.close();
+          this.zooKeeper = null;
+        }
+        this.closed = true;
       }
-      this.closed = true;
     }
 
     public void close() {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index d95ddcd35021..98a99e7033a8 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -23,7 +23,9 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -46,6 +48,7 @@
 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -61,6 +64,7 @@ public class TestHCM {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final byte[] TABLE_NAME = Bytes.toBytes("test");
   private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
+  private static final byte[] TABLE_NAME2 = Bytes.toBytes("test2");
   private static final byte[] FAM_NAM = Bytes.toBytes("f");
   private static final byte[] ROW = Bytes.toBytes("bbb");
 
@@ -326,6 +330,64 @@ public void testCreateConnection() throws Exception {
     assertTrue(c2 != c3);
   }
 
+  /**
+   * Tests that a closed connection does not have a live zookeeper
+   * @throws Exception
+   */
+  @Test
+  public void testDeleteForZKConnLeak() throws Exception {
+    TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
+    final Configuration config = TEST_UTIL.getConfiguration();
+
+    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
+      5, TimeUnit.SECONDS,
+      new SynchronousQueue<Runnable>(),
+      Threads.newDaemonThreadFactory("test-hcm-delete"));
+
+    pool.submit(new Runnable() {
+      @Override
+      public void run() {
+        while (!Thread.interrupted()) {
+          try {
+            HConnection conn = HConnectionManager.getConnection(config);
+            HConnectionManager.deleteStaleConnection(conn);
+          } catch (Exception e) {
+          }
+        }
+      }
+    });
+
+    // use connection multiple times
+    for (int i = 0; i < 10; i++) {
+      HConnection c1 = null;
+        try {
+          c1 = HConnectionManager.getConnection(config);
+          HTable table = new HTable(TABLE_NAME2, c1, pool);
+          table.close();
+        } catch (Exception e) {
+        } finally {
+          if (c1 != null) {
+            if (c1.isClosed()) {
+              // cannot use getZooKeeper as method instantiates watcher if null
+              Field zkwField = c1.getClass().getDeclaredField("zooKeeper");
+              zkwField.setAccessible(true);
+              Object watcher = zkwField.get(c1);
+
+              if (watcher != null) {
+                if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
+                  pool.shutdownNow();
+                  fail("Live zookeeper in closed connection");
+                }
+              }
+            }
+            c1.close();
+          }
+        }
+    }
+
+    pool.shutdownNow();
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From a6f3b31b8af5fb74831c79f27bd62d2ad13ab0a3 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 9 May 2013 10:45:32 +0000
Subject: [PATCH 0966/1540] HBASE-8446 Allow parallel snapshot of different
 tables

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1480586 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java   |  58 +--
 .../hadoop/hbase/master/SnapshotSentinel.java |  12 +
 .../master/snapshot/CloneSnapshotHandler.java |  10 +
 .../DisabledTableSnapshotHandler.java         |   2 +-
 .../snapshot/EnabledTableSnapshotHandler.java |   2 +-
 .../snapshot/RestoreSnapshotHandler.java      |  11 +
 .../master/snapshot/SnapshotManager.java      | 350 ++++++++++--------
 .../master/snapshot/TakeSnapshotHandler.java  |  19 +-
 .../cleaner/TestSnapshotFromMaster.java       |  10 +-
 .../master/snapshot/TestSnapshotManager.java  |  15 +-
 .../snapshot/TestFlushSnapshotFromClient.java |  69 ++--
 11 files changed, 335 insertions(+), 223 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 7cbcf279647c..1a2a14aeba3d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -192,7 +192,7 @@ public class HMaster extends HasThread
   private CatalogTracker catalogTracker;
   // Cluster status zk tracker and local setter
   private ClusterStatusTracker clusterStatusTracker;
-  
+
   // buffer for "fatal error" notices from region servers
   // in the cluster. This is only used for assisting
   // operations/debugging.
@@ -372,7 +372,7 @@ private static void stallIfBackupMaster(final Configuration c,
         "(Also watching cluster state node)");
       Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000));
     }
-    
+
   }
 
   /**
@@ -410,7 +410,7 @@ public void run() {
       }
     } catch (Throwable t) {
       // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
-      if (t instanceof NoClassDefFoundError && 
+      if (t instanceof NoClassDefFoundError &&
           t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
           // improved error message for this special case
           abort("HBase is having a problem with its Hadoop jars.  You may need to "
@@ -422,7 +422,7 @@ public void run() {
       }
     } finally {
       startupStatus.cleanup();
-      
+
       stopChores();
       // Wait for all the remaining region servers to report in IFF we were
       // running a cluster shutdown AND we were NOT aborting.
@@ -445,7 +445,7 @@ public void run() {
 
   /**
    * Try becoming active master.
-   * @param startupStatus 
+   * @param startupStatus
    * @return True if we could successfully become the active master.
    * @throws InterruptedException
    */
@@ -526,7 +526,7 @@ private void loop() {
    * <li>Ensure assignment of root and meta regions<li>
    * <li>Handle either fresh cluster start or master failover</li>
    * </ol>
-   * @param masterRecovery 
+   * @param masterRecovery
    *
    * @throws IOException
    * @throws InterruptedException
@@ -563,7 +563,7 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
 
     status.setStatus("Initializing ZK system trackers");
     initializeZKBasedSystemTrackers();
-    
+
     if (!masterRecovery) {
       // initialize master side coprocessors before we start handling requests
       status.setStatus("Initializing master coprocessors");
@@ -667,7 +667,7 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     // removing dead server with same hostname and port of rs which is trying to check in before
     // master initialization. See HBASE-5916.
     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
-    
+
     if (!masterRecovery) {
       if (this.cpHost != null) {
         // don't let cp initialization errors kill the master
@@ -679,11 +679,11 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
       }
     }
   }
-  
+
   /**
    * If ServerShutdownHandler is disabled, we enable it and expire those dead
    * but not expired servers.
-   * 
+   *
    * @throws IOException
    */
   private void enableServerShutdownHandler() throws IOException {
@@ -692,7 +692,7 @@ private void enableServerShutdownHandler() throws IOException {
       this.serverManager.expireDeadNotExpiredServers();
     }
   }
-  
+
   /**
    * Useful for testing purpose also where we have
    * master restart scenarios.
@@ -863,7 +863,7 @@ private void splitLogAndExpireIfOnline(final ServerName sn)
       fileSystemManager.splitMetaLog(sn);
       fileSystemManager.splitLog(sn);
     } else {
-      fileSystemManager.splitAllLogs(sn);  
+      fileSystemManager.splitAllLogs(sn);
     }
     serverManager.expireServer(sn);
   }
@@ -937,7 +937,7 @@ public ZooKeeperWatcher getZooKeeperWatcher() {
    *  need to install an unexpected exception handler.
    */
   private void startServiceThreads() throws IOException{
- 
+
    // Start the executor service pools
    this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
       conf.getInt("hbase.master.executor.openregion.threads", 5));
@@ -947,7 +947,7 @@ private void startServiceThreads() throws IOException{
       conf.getInt("hbase.master.executor.serverops.threads", 3));
    this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
       conf.getInt("hbase.master.executor.serverops.threads", 5));
-   
+
    // We depend on there being only one instance of this executor running
    // at a time.  To do concurrency, would need fencing of enable/disable of
    // tables.
@@ -1204,11 +1204,11 @@ public boolean switchBalancer(final boolean b, BalanceSwitchMode mode) {
         newValue = this.cpHost.preBalanceSwitch(newValue);
       }
       if (mode == BalanceSwitchMode.SYNC) {
-        synchronized (this.balancer) {        
+        synchronized (this.balancer) {
           this.balanceSwitch = newValue;
         }
       } else {
-        this.balanceSwitch = newValue;        
+        this.balanceSwitch = newValue;
       }
       LOG.info("BalanceSwitch=" + newValue);
       if (this.cpHost != null) {
@@ -1217,14 +1217,14 @@ public boolean switchBalancer(final boolean b, BalanceSwitchMode mode) {
     } catch (IOException ioe) {
       LOG.warn("Error flipping balance switch", ioe);
     }
-    return oldValue;    
+    return oldValue;
   }
-  
+
   @Override
   public boolean synchronousBalanceSwitch(final boolean b) {
     return switchBalancer(b, BalanceSwitchMode.SYNC);
   }
-  
+
   @Override
   public boolean balanceSwitch(final boolean b) {
     return switchBalancer(b, BalanceSwitchMode.ASYNC);
@@ -1257,10 +1257,10 @@ public void move(final byte[] encodedRegionName, final byte[] destServerName)
     } else {
       dest = new ServerName(Bytes.toString(destServerName));
     }
-    
+
     // Now we can do the move
     RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
-    
+
     try {
       checkInitialized();
       if (this.cpHost != null) {
@@ -1347,7 +1347,7 @@ public void deleteTable(final byte [] tableName) throws IOException {
    * @return Pair indicating the number of regions updated Pair.getFirst is the
    *         regions that are yet to be updated Pair.getSecond is the total number
    *         of regions of the table
-   * @throws IOException 
+   * @throws IOException
    */
   public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
   throws IOException {
@@ -1699,7 +1699,7 @@ public CatalogTracker getCatalogTracker() {
   public AssignmentManager getAssignmentManager() {
     return this.assignmentManager;
   }
-  
+
   public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
     return rsFatals;
   }
@@ -1774,13 +1774,13 @@ public boolean isStopped() {
   public boolean isAborted() {
     return this.abort;
   }
-  
+
   void checkInitialized() throws PleaseHoldException {
     if (!this.initialized) {
       throw new PleaseHoldException("Master is initializing");
     }
   }
-  
+
   /**
    * Report whether this master is currently the active master or not.
    * If not active master, we are parked on ZK waiting to become active.
@@ -1850,8 +1850,8 @@ public void assign(final byte [] regionName)throws IOException {
       cpHost.postAssign(pair.getFirst());
     }
   }
-  
-  
+
+
 
   public void assignRegion(HRegionInfo hri) {
     assignmentManager.assign(hri, true);
@@ -1882,7 +1882,7 @@ public void unassign(final byte [] regionName, final boolean force)
   }
 
   /**
-   * Get HTD array for given tables 
+   * Get HTD array for given tables
    * @param tableNames
    * @return HTableDescriptor[]
    */
@@ -2184,7 +2184,7 @@ public void restoreSnapshot(final HSnapshotDescription request) throws IOExcepti
    */
   @Override
   public boolean isRestoreSnapshotDone(final HSnapshotDescription request) throws IOException {
-    return !snapshotManager.isRestoringTable(request.getProto());
+    return snapshotManager.isRestoreDone(request.getProto());
   }
 }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java b/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
index bb2d7169c789..d621d7724d5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
@@ -36,6 +36,11 @@ public interface SnapshotSentinel {
    */
   public boolean isFinished();
 
+  /**
+   * @return -1 if the snapshot is in progress, otherwise the completion timestamp.
+   */
+  public long getCompletionTimestamp();
+
   /**
    * Actively cancel a running snapshot.
    * @param why Reason for cancellation.
@@ -54,4 +59,11 @@ public interface SnapshotSentinel {
    */
   public ForeignException getExceptionIfFailed();
 
+  /**
+   * Rethrow the exception returned by {@link SnapshotSentinel#getExceptionIfFailed}.
+   * If there is no exception this is a no-op.
+   *
+   * @throws ForeignException all exceptions from remote sources are procedure exceptions
+   */
+  public void rethrowExceptionIfFailed() throws ForeignException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
index eebf99d26b56..bd0349f71660 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -150,6 +150,11 @@ public boolean isFinished() {
     return this.stopped;
   }
 
+  @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
   @Override
   public SnapshotDescription getSnapshot() {
     return snapshot;
@@ -169,4 +174,9 @@ public void cancel(String why) {
   public ForeignException getExceptionIfFailed() {
     return this.monitor.getException();
   }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
index a804f07e277a..eee454846dc1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -63,7 +63,7 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
    * @throws IOException on unexpected error
    */
   public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
-      final MasterServices masterServices, final MasterMetrics metricsMaster) throws IOException {
+      final MasterServices masterServices, final MasterMetrics metricsMaster) {
     super(snapshot, masterServices, metricsMaster);
 
     // setup the timer
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
index c795855374f7..4bbdef3cbe30 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
@@ -50,7 +50,7 @@ public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
   private final ProcedureCoordinator coordinator;
 
   public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
-      final SnapshotManager manager, final MasterMetrics metricsMaster) throws IOException {
+      final SnapshotManager manager, final MasterMetrics metricsMaster) {
     super(snapshot, master, metricsMaster);
     this.coordinator = manager.getCoordinator();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index 0dfa2c0fcd66..1660972b316e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -158,6 +158,11 @@ public boolean isFinished() {
     return this.stopped;
   }
 
+  @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
   @Override
   public SnapshotDescription getSnapshot() {
     return snapshot;
@@ -174,7 +179,13 @@ public void cancel(String why) {
     this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
   }
 
+  @Override
   public ForeignException getExceptionIfFailed() {
     return this.monitor.getException();
   }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index adb37adb0f63..439989cff5b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -68,6 +68,7 @@
 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.zookeeper.KeeperException;
@@ -89,6 +90,19 @@ public class SnapshotManager implements Stoppable {
   /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
   private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
 
+  /**
+   * Wait time before removing a finished sentinel from the in-progress map
+   *
+   * NOTE: This is used as a safety auto cleanup.
+   * The snapshot and restore handlers map entries are removed when a user asks if a snapshot or
+   * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API flow.
+   * In case something fails on the client side and the snapshot/restore state is not reclaimed
+   * after a default timeout, the entry is removed from the in-progress map.
+   * At this point, if the user asks for the snapshot/restore status, the result will be
+   * snapshot done if exists or failed if it doesn't exists.
+   */
+  private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
+
   /** Enable or disable snapshot support */
   public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
 
@@ -110,10 +124,11 @@ public class SnapshotManager implements Stoppable {
   /** Name of the operation to use in the controller */
   public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
 
-  // TODO - enable having multiple snapshots with multiple monitors/threads
-  // this needs to be configuration based when running multiple snapshots is implemented
+  /** Conf key for # of threads used by the SnapshotManager thread pool */
+  private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
+
   /** number of current operations running on the master */
-  private static final int opThreads = 1;
+  private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
 
   private boolean stopped;
   private final long wakeFrequency;
@@ -124,18 +139,21 @@ public class SnapshotManager implements Stoppable {
   // Is snapshot feature enabled?
   private boolean isSnapshotSupported = false;
 
-  // A reference to a handler.  If the handler is non-null, then it is assumed that a snapshot is
-  // in progress currently
-  // TODO: this is a bad smell;  likely replace with a collection in the future.  Also this gets
-  // reset by every operation.
-  private TakeSnapshotHandler handler;
+  // Snapshot handlers map, with table name as key.
+  // The map is always accessed and modified under the object lock using synchronized.
+  // snapshotTable() will insert an Handler in the table.
+  // isSnapshotDone() will remove the handler requested if the operation is finished.
+  private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
+
+  // Restore Sentinels map, with table name as key.
+  // The map is always accessed and modified under the object lock using synchronized.
+  // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
+  // isRestoreDone() will remove the handler requested if the operation is finished.
+  private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
 
   private final Path rootDir;
   private final ExecutorService executorService;
 
-  // Restore Sentinels map, with table name as key
-  private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
-
   /**
    * Construct a snapshot manager.
    * @param master
@@ -152,6 +170,7 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
     Configuration conf = master.getConfiguration();
     this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
     long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+    int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
 
     // setup the default procedure coordinator
     String name = master.getServerName().toString();
@@ -193,7 +212,7 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
   public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
     return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
   }
-  
+
   /**
    * Gets the list of all completed snapshots.
    * @param snapshotDir snapshot directory
@@ -203,9 +222,8 @@ public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
   private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
     List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
     // first create the snapshot root path and check to see if it exists
-    if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
-
     FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
 
     // if there are no snapshots, return an empty list
     if (!fs.exists(snapshotDir)) {
@@ -290,27 +308,9 @@ public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotE
 
   }
 
-  /**
-   * Return the handler if it is currently running and has the same snapshot target name.
-   * @param snapshot
-   * @return null if doesn't match, else a live handler.
-   */
-  private synchronized TakeSnapshotHandler getTakeSnapshotHandler(SnapshotDescription snapshot) {
-    TakeSnapshotHandler h = this.handler;
-    if (h == null) {
-      return null;
-    }
-
-    if (!h.getSnapshot().getName().equals(snapshot.getName())) {
-      // specified snapshot is to the one currently running
-      return null;
-    }
-
-    return h;
-  }
-
   /**
    * Check if the specified snapshot is done
+   *
    * @param expected
    * @return true if snapshot is ready to be restored, false if it is still being taken.
    * @throws IOException IOException if error from HDFS or RPC
@@ -325,10 +325,20 @@ public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
 
     String ssString = SnapshotDescriptionUtils.toString(expected);
 
-    // check to see if the sentinel exists
-    TakeSnapshotHandler handler = getTakeSnapshotHandler(expected);
+    // check to see if the sentinel exists,
+    // and if the task is complete removes it from the in-progress snapshots map.
+    SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
     if (handler == null) {
-      // doesn't exist, check if it is already completely done.
+      // If there's no handler in the in-progress map, it means one of the following:
+      //   - someone has already requested the snapshot state
+      //   - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
+      //   - the snapshot was never requested
+      // In those cases returns to the user the "done state" if the snapshots exists on disk,
+      // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
       if (!isSnapshotCompleted(expected)) {
         throw new UnknownSnapshotException("Snapshot " + ssString
             + " is not currently running or one of the known completed snapshots.");
@@ -339,7 +349,7 @@ public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
 
     // pass on any failure we find in the sentinel
     try {
-      handler.rethrowException();
+      handler.rethrowExceptionIfFailed();
     } catch (ForeignException e) {
       // Give some procedure info on an exception.
       String status;
@@ -363,33 +373,20 @@ public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
     return false;
   }
 
-  /**
-   * Check to see if there are any snapshots in progress currently.  Currently we have a
-   * limitation only allowing a single snapshot attempt at a time.
-   * @return <tt>true</tt> if there any snapshots in progress, <tt>false</tt> otherwise
-   * @throws SnapshotCreationException if the snapshot failed
-   */
-  synchronized boolean isTakingSnapshot() throws SnapshotCreationException {
-    // TODO later when we handle multiple there would be a map with ssname to handler.
-    return handler != null && !handler.isFinished();
-  }
-
   /**
    * Check to see if the specified table has a snapshot in progress.  Currently we have a
-   * limitation only allowing a single snapshot attempt at a time.
+   * limitation only allowing a single snapshot per table at a time.
    * @param tableName name of the table being snapshotted.
    * @return <tt>true</tt> if there is a snapshot in progress on the specified table.
    */
-  private boolean isTakingSnapshot(final String tableName) {
-    if (handler != null && handler.getSnapshot().getTable().equals(tableName)) {
-      return !handler.isFinished();
-    }
-    return false;
+  synchronized boolean isTakingSnapshot(final String tableName) {
+    SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
+    return handler != null && !handler.isFinished();
   }
 
   /**
    * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
-   * aren't already running a snapshot.
+   * aren't already running a snapshot or restore on the requested table.
    * @param snapshot description of the snapshot we want to start
    * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
    */
@@ -399,19 +396,21 @@ private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
 
     // make sure we aren't already running a snapshot
-    if (isTakingSnapshot()) {
+    if (isTakingSnapshot(snapshot.getTable())) {
+      SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
       throw new SnapshotCreationException("Rejected taking "
           + SnapshotDescriptionUtils.toString(snapshot)
           + " because we are already running another snapshot "
-          + SnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+          + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
     }
 
     // make sure we aren't running a restore on the same table
     if (isRestoringTable(snapshot.getTable())) {
+      SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
       throw new SnapshotCreationException("Rejected taking "
           + SnapshotDescriptionUtils.toString(snapshot)
           + " because we are already have a restore in progress on the same snapshot "
-          + SnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+          + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
     }
 
     try {
@@ -432,32 +431,65 @@ private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
     }
   }
 
+  /**
+   * Take a snapshot of a disabled table.
+   * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
+   * @throws HBaseSnapshotException if the snapshot could not be started
+   */
+  private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // set the snapshot to be a disabled snapshot, since the client doesn't know about that
+    snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
+
+    // Take the snapshot of the disabled table
+    DisabledTableSnapshotHandler handler =
+        new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
+    snapshotTable(snapshot, handler);
+  }
+
   /**
    * Take a snapshot of an enabled table.
-   * <p>
-   * The thread limitation on the executorService's thread pool for snapshots ensures the
-   * snapshot won't be started if there is another snapshot already running. Does
-   * <b>not</b> check to see if another snapshot of the same name already exists.
    * @param snapshot description of the snapshot to take.
    * @throws HBaseSnapshotException if the snapshot could not be started
    */
   private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
       throws HBaseSnapshotException {
-    TakeSnapshotHandler handler;
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // Take the snapshot of the enabled table
+    EnabledTableSnapshotHandler handler =
+        new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+    snapshotTable(snapshot, handler);
+  }
+
+  /**
+   * Take a snapshot using the specified handler.
+   * On failure the snapshot temporary working directory is removed.
+   * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
+   *       snapshot request if the table is busy with another snapshot/restore operation.
+   * @param snapshot the snapshot description
+   * @param handler the snapshot handler
+   */
+  private synchronized void snapshotTable(SnapshotDescription snapshot,
+      final TakeSnapshotHandler handler) throws HBaseSnapshotException {
     try {
-      handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+      handler.prepare();
       this.executorService.submit(handler);
-      this.handler = handler;
-    } catch (IOException e) {
+      this.snapshotHandlers.put(snapshot.getTable(), handler);
+    } catch (Exception e) {
       // cleanup the working directory by trying to delete it from the fs.
       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
       try {
         if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
-          LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:"
-              + SnapshotDescriptionUtils.toString(snapshot));
+          LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+              SnapshotDescriptionUtils.toString(snapshot));
         }
       } catch (IOException e1) {
-        LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
             SnapshotDescriptionUtils.toString(snapshot));
       }
       // fail the snapshot
@@ -481,6 +513,9 @@ public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
 
     LOG.debug("No existing snapshot, attempting snapshot...");
 
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
     // check to see if the table exists
     HTableDescriptor desc = null;
     try {
@@ -508,9 +543,6 @@ public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
       cpHost.preSnapshot(snapshot, desc);
     }
 
-    // setup the snapshot
-    prepareToTakeSnapshot(snapshot);
-
     // if the table is enabled, then have the RS run actually the snapshot work
     AssignmentManager assignmentMgr = master.getAssignmentManager();
     if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
@@ -537,53 +569,22 @@ else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
     }
   }
 
-  /**
-   * Take a snapshot of a disabled table.
-   * <p>
-   * The thread limitation on the executorService's thread pool for snapshots ensures the
-   * snapshot won't be started if there is another snapshot already running. Does
-   * <b>not</b> check to see if another snapshot of the same name already exists.
-   * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
-   * @throws HBaseSnapshotException if the snapshot could not be started
-   */
-  private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
-      throws HBaseSnapshotException {
-
-    // set the snapshot to be a disabled snapshot, since the client doesn't know about that
-    snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
-
-    DisabledTableSnapshotHandler handler;
-    try {
-      handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
-      this.executorService.submit(handler);
-      this.handler = handler;
-    } catch (IOException e) {
-      // cleanup the working directory by trying to delete it from the fs.
-      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
-      try {
-        if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
-          LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
-              SnapshotDescriptionUtils.toString(snapshot));
-        }
-      } catch (IOException e1) {
-        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
-            SnapshotDescriptionUtils.toString(snapshot));
-      }
-      // fail the snapshot
-      throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
-    }
-  }
-
   /**
    * Set the handler for the current snapshot
    * <p>
    * Exposed for TESTING
+   * @param tableName
    * @param handler handler the master should use
    *
    * TODO get rid of this if possible, repackaging, modify tests.
    */
-  public synchronized void setSnapshotHandlerForTesting(TakeSnapshotHandler handler) {
-    this.handler = handler;
+  public synchronized void setSnapshotHandlerForTesting(final String tableName,
+      final SnapshotSentinel handler) {
+    if (handler != null) {
+      this.snapshotHandlers.put(tableName, handler);
+    } else {
+      this.snapshotHandlers.remove(tableName);
+    }
   }
 
   /**
@@ -595,7 +596,9 @@ ProcedureCoordinator getCoordinator() {
 
   /**
    * Check to see if the snapshot is one of the currently completed snapshots
-   * @param expected snapshot to check
+   * Returns true if the snapshot exists in the "completed snapshots folder".
+   *
+   * @param snapshot expected snapshot to check
    * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
    *         not stored
    * @throws IOException if the filesystem throws an unexpected exception,
@@ -619,7 +622,6 @@ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOExcep
    *
    * @param snapshot Snapshot Descriptor
    * @param hTableDescriptor Table Descriptor of the table to create
-   * @param waitTime timeout before considering the clone failed
    */
   synchronized void cloneSnapshot(final SnapshotDescription snapshot,
       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
@@ -639,7 +641,7 @@ synchronized void cloneSnapshot(final SnapshotDescription snapshot,
       CloneSnapshotHandler handler =
         new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
       this.executorService.submit(handler);
-      restoreHandlers.put(tableName, handler);
+      this.restoreHandlers.put(tableName, handler);
     } catch (Exception e) {
       String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
         " on table=" + tableName;
@@ -669,8 +671,8 @@ public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException
     HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
     String tableName = reqSnapshot.getTable();
 
-    // stop tracking completed restores
-    cleanupRestoreSentinels();
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
 
     // Execute the restore/clone operation
     if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
@@ -710,7 +712,6 @@ public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException
    *
    * @param snapshot Snapshot Descriptor
    * @param hTableDescriptor Table Descriptor
-   * @param waitTime timeout before considering the restore failed
    */
   private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
@@ -732,7 +733,8 @@ private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
       this.executorService.submit(handler);
       restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
     } catch (Exception e) {
-      String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(snapshot)  +
+      String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
+          snapshot)  +
           " on table=" + tableName;
       LOG.error(msg, e);
       throw new RestoreSnapshotException(msg, e);
@@ -745,79 +747,107 @@ private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    * @param tableName table under restore
    * @return <tt>true</tt> if there is a restore in progress of the specified table.
    */
-  private boolean isRestoringTable(final String tableName) {
-    SnapshotSentinel sentinel = restoreHandlers.get(tableName);
+  private synchronized boolean isRestoringTable(final String tableName) {
+    SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
     return(sentinel != null && !sentinel.isFinished());
   }
 
   /**
-   * Returns status of a restore request, specifically comparing source snapshot and target table
-   * names.  Throws exception if not a known snapshot.
+   * Returns the status of a restore operation.
+   * If the in-progress restore is failed throws the exception that caused the failure.
+   *
    * @param snapshot
-   * @return true if in progress, false if snapshot is completed.
-   * @throws UnknownSnapshotException if specified source snapshot does not exit.
-   * @throws IOException if there was some sort of IO failure
+   * @return false if in progress, true if restore is completed or not requested.
+   * @throws IOException if there was a failure during the restore
    */
-  public boolean isRestoringTable(final SnapshotDescription snapshot) throws IOException {
-    // check to see if the snapshot is already on the fs
-    if (!isSnapshotCompleted(snapshot)) {
-      throw new UnknownSnapshotException("Snapshot:" + snapshot.getName()
-          + " is not one of the known completed snapshots.");
-    }
+  public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
+    // check to see if the sentinel exists,
+    // and if the task is complete removes it from the in-progress restore map.
+    SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
 
-    SnapshotSentinel sentinel = getRestoreSnapshotSentinel(snapshot.getTable());
     if (sentinel == null) {
       // there is no sentinel so restore is not in progress.
-      return false;
-    }
-    if (!sentinel.getSnapshot().getName().equals(snapshot.getName())) {
-      // another handler is trying to restore to the table, but it isn't the same snapshot source.
-      return false;
+      return true;
     }
 
     LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
         + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
-    ForeignException e = sentinel.getExceptionIfFailed();
-    if (e != null) throw e;
+
+    // If the restore is failed, rethrow the exception
+    sentinel.rethrowExceptionIfFailed();
 
     // check to see if we are done
     if (sentinel.isFinished()) {
       LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
           " has completed. Notifying the client.");
-      return false;
+      return true;
     }
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
           SnapshotDescriptionUtils.toString(snapshot));
     }
-    return true;
+    return false;
   }
 
   /**
-   * Get the restore snapshot sentinel for the specified table
-   * @param tableName table under restore
-   * @return the restore snapshot handler
+   * Return the handler if it is currently live and has the same snapshot target name.
+   * The handler is removed from the sentinels map if completed.
+   * @param sentinels live handlers
+   * @param snapshot snapshot description
+   * @return null if doesn't match, else a live handler.
    */
-  private synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName) {
-    try {
-      return restoreHandlers.get(tableName);
-    } finally {
-      cleanupRestoreSentinels();
+  private synchronized SnapshotSentinel removeSentinelIfFinished(
+      final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot) {
+    SnapshotSentinel h = sentinels.get(snapshot.getTable());
+    if (h == null) {
+      return null;
     }
+
+    if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+      // specified snapshot is to the one currently running
+      return null;
+    }
+
+    // Remove from the "in-progress" list once completed
+    if (h.isFinished()) {
+      sentinels.remove(snapshot.getTable());
+    }
+
+    return h;
   }
 
   /**
-   * Scan the restore handlers and remove the finished ones.
+   * Removes "abandoned" snapshot/restore requests.
+   * As part of the HBaseAdmin snapshot/restore API the operation status is checked until completed,
+   * and the in-progress maps are cleaned up when the status of a completed task is requested.
+   * To avoid having sentinels staying around for long time if something client side is failed,
+   * each operation tries to clean up the in-progress maps sentinels finished from a long time.
    */
-  private synchronized void cleanupRestoreSentinels() {
-    Iterator<Map.Entry<String, SnapshotSentinel>> it = restoreHandlers.entrySet().iterator();
+  private void cleanupSentinels() {
+    cleanupSentinels(this.snapshotHandlers);
+    cleanupSentinels(this.restoreHandlers);
+  }
+
+  /**
+   * Remove the sentinels that are marked as finished and the completion time
+   * has exceeded the removal timeout.
+   * @param sentinels map of sentinels to clean
+   */
+  private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels) {
+    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+    Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
     while (it.hasNext()) {
-        Map.Entry<String, SnapshotSentinel> entry = it.next();
-        SnapshotSentinel sentinel = entry.getValue();
-        if (sentinel.isFinished()) {
-          it.remove();
-        }
+      Map.Entry<String, SnapshotSentinel> entry = it.next();
+      SnapshotSentinel sentinel = entry.getValue();
+      if (sentinel.isFinished() &&
+          (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
+      {
+        it.remove();
+      }
     }
   }
 
@@ -832,7 +862,9 @@ public void stop(String why) {
     // make sure we get stop
     this.stopped = true;
     // pass the stop onto take snapshot handlers
-    if (this.handler != null) this.handler.cancel(why);
+    for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
+      snapshotHandler.cancel(why);
+    }
 
     // pass the stop onto all the restore handlers
     for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
@@ -892,7 +924,7 @@ private void checkSnapshotSupport(final Configuration conf, final MasterFileSyst
       LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
       LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
     }
-    
+
     // If the user has enabled the snapshot, we force the cleaners to be present
     // otherwise we still need to check if cleaners are enabled or not and verify
     // that there're no snapshot in the .snapshot folder.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index acbc679da412..48ed9676244a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -84,7 +84,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
    * @throws IOException on unexpected error
    */
   public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
-      final MasterMetrics metricsMaster) throws IOException {
+      final MasterMetrics metricsMaster) {
     super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
     assert snapshot != null : "SnapshotDescription must not be nul1";
     assert masterServices != null : "MasterServices must not be nul1";
@@ -99,8 +99,6 @@ public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices ma
     this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
     this.monitor =  new ForeignExceptionDispatcher();
 
-    loadTableDescriptor(); // check that .tableinfo is present
-
     // prepare the verify
     this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
     // update the running tasks
@@ -119,6 +117,11 @@ private HTableDescriptor loadTableDescriptor()
     return htd;
   }
 
+  public TakeSnapshotHandler prepare() throws Exception {
+    loadTableDescriptor(); // check that .tableinfo is present
+    return this;
+  }
+
   /**
    * Execute the core common portions of taking a snapshot. The {@link #snapshotRegions(List)}
    * call should get implemented for each snapshot flavor.
@@ -227,6 +230,11 @@ public boolean isFinished() {
     return finished;
   }
 
+  @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
   @Override
   public SnapshotDescription getSnapshot() {
     return snapshot;
@@ -237,6 +245,11 @@ public ForeignException getExceptionIfFailed() {
     return monitor.getException();
   }
 
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
+
   @Override
   public void rethrowException() throws ForeignException {
     monitor.rethrowException();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index d67429ffd528..92a8bb856736 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -50,6 +50,7 @@
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.junit.After;
@@ -126,7 +127,7 @@ private static void setupConf(Configuration conf) {
   @Before
   public void setup() throws Exception {
     UTIL.createTable(TABLE_NAME, TEST_FAM);
-    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
+    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(STRING_TABLE_NAME, null);
   }
 
   @After
@@ -179,7 +180,7 @@ public void testIsDoneContract() throws Exception {
 
     // and that we get the same issue, even if we specify a name
     SnapshotDescription desc = SnapshotDescription.newBuilder()
-      .setName(snapshotName).build();
+      .setName(snapshotName).setTable(STRING_TABLE_NAME).build();
     SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
       UnknownSnapshotException.class);
 
@@ -188,8 +189,11 @@ public void testIsDoneContract() throws Exception {
     Mockito.when(mockHandler.getException()).thenReturn(null);
     Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
     Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+    Mockito.when(mockHandler.getCompletionTimestamp())
+      .thenReturn(EnvironmentEdgeManager.currentTimeMillis());
 
-    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
+    master.getSnapshotManagerForTesting()
+        .setSnapshotHandlerForTesting(STRING_TABLE_NAME, mockHandler);
 
     // if we do a lookup without a snapshot name, we should fail - you should always know your name
     SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
index ffd5672ec9ec..47c57bf3b63e 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
@@ -67,7 +67,8 @@ private SnapshotManager getNewManager() throws IOException, KeeperException {
     return getNewManager(UTIL.getConfiguration());
   }
 
-  private SnapshotManager getNewManager(final Configuration conf) throws IOException, KeeperException {
+  private SnapshotManager getNewManager(final Configuration conf)
+      throws IOException, KeeperException {
     Mockito.reset(services);
     Mockito.when(services.getConfiguration()).thenReturn(conf);
     Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
@@ -78,14 +79,18 @@ private SnapshotManager getNewManager(final Configuration conf) throws IOExcepti
 
   @Test
   public void testInProcess() throws KeeperException, IOException {
+    String tableName = "testTable";
     SnapshotManager manager = getNewManager();
     TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
-    assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
-    manager.setSnapshotHandlerForTesting(handler);
+    assertFalse("Manager is in process when there is no current handler",
+        manager.isTakingSnapshot(tableName));
+    manager.setSnapshotHandlerForTesting(tableName, handler);
     Mockito.when(handler.isFinished()).thenReturn(false);
-    assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot());
+    assertTrue("Manager isn't in process when handler is running",
+        manager.isTakingSnapshot(tableName));
     Mockito.when(handler.isFinished()).thenReturn(true);
-    assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
+    assertFalse("Manager is process when handler isn't running",
+        manager.isTakingSnapshot(tableName));
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index 8c406c1658dc..ded6c66cf877 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -251,11 +251,8 @@ public void testFlushCreateListDestroy() throws Exception {
     // load the table so we have some data
     UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
     // and wait until everything stabilizes
-    HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
-    List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
-    for (HRegion region : onlineRegions) {
-      region.waitForFlushesAndCompactions();
-    }
+    waitForTableToBeOnline(TABLE_NAME);
+
     String snapshotName = "flushSnapshotCreateListDestroy";
     // test creating the snapshot
     admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
@@ -305,32 +302,27 @@ public void testFlushCreateListDestroy() throws Exception {
   }
 
   /**
-   * Demonstrate that we reject snapshot requests if there is a snapshot currently running.
+   * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
+   * same table currently running and that concurrent snapshots on different tables can both
+   * succeed concurretly.
    */
   @Test(timeout=60000)
   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
-    int ssNum = 10;
+    final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
+    final byte[] TABLE2_NAME = Bytes.toBytes(STRING_TABLE2_NAME);
+
+    int ssNum = 20;
     HBaseAdmin admin = UTIL.getHBaseAdmin();
     // make sure we don't fail on listing snapshots
     SnapshotTestingUtils.assertNoSnapshots(admin);
+    // create second testing table
+    UTIL.createTable(TABLE2_NAME, TEST_FAM);
     // load the table so we have some data
     UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+    UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE2_NAME), TEST_FAM);
     // and wait until everything stabilizes
-    HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
-    List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
-    for (HRegion region : onlineRegions) {
-      region.waitForFlushesAndCompactions();
-    }
-
-    // build descriptions
-    SnapshotDescription[] descs = new SnapshotDescription[ssNum];
-    for (int i = 0; i < ssNum; i++) {
-      SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
-      builder.setTable(STRING_TABLE_NAME);
-      builder.setName("ss"+i);
-      builder.setType(SnapshotDescription.Type.FLUSH);
-      descs[i] = builder.build();
-    }
+    waitForTableToBeOnline(TABLE_NAME);
+    waitForTableToBeOnline(TABLE2_NAME);
 
     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
     // We'll have one of these per thread
@@ -355,6 +347,16 @@ public void run() {
       }
     };
 
+    // build descriptions
+    SnapshotDescription[] descs = new SnapshotDescription[ssNum];
+    for (int i = 0; i < ssNum; i++) {
+      SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+      builder.setTable((i % 2) == 0 ? STRING_TABLE_NAME : STRING_TABLE2_NAME);
+      builder.setName("ss"+i);
+      builder.setType(SnapshotDescription.Type.FLUSH);
+      descs[i] = builder.build();
+    }
+
     // kick each off its own thread
     for (int i=0 ; i < ssNum; i++) {
       new Thread(new SSRunnable(descs[i])).start();
@@ -390,13 +392,36 @@ public void run() {
     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
         " issued many requests", takenSize < ssNum && takenSize > 0);
+
+    // Verify that there's at least one snapshot per table
+    int t1SnapshotsCount = 0;
+    int t2SnapshotsCount = 0;
+    for (SnapshotDescription ss : taken) {
+      if (ss.getTable().equals(STRING_TABLE_NAME)) {
+        t1SnapshotsCount++;
+      } else if (ss.getTable().equals(STRING_TABLE2_NAME)) {
+        t2SnapshotsCount++;
+      }
+    }
+    assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
+    assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
+
     // delete snapshots so subsequent tests are clean.
     for (SnapshotDescription ss : taken) {
       admin.deleteSnapshot(ss.getName());
     }
+    UTIL.deleteTable(TABLE2_NAME);
   }
 
   private void logFSTree(Path root) throws IOException {
     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
   }
+
+  private void waitForTableToBeOnline(final byte[] tableName) throws IOException {
+    HRegionServer rs = UTIL.getRSForFirstRegionInTable(tableName);
+    List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
+    for (HRegion region : onlineRegions) {
+      region.waitForFlushesAndCompactions();
+    }
+  }
 }

From 147e90b2d09b66228b47e7704fcf3d132ed51c6f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 9 May 2013 23:14:05 +0000
Subject: [PATCH 0967/1540] HBASE-8509 ZKUtil#createWithParents won't set data
 during znode creation when parent folder doesn't exit (Jefferey and Ted)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1480836 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java |  2 +-
 .../apache/hadoop/hbase/TestZooKeeper.java    | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 041f4ae3ab52..a91d44154a17 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1166,7 +1166,7 @@ public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[]
       return;
     } catch(KeeperException.NoNodeException nne) {
       createWithParents(zkw, getParent(znode));
-      createWithParents(zkw, znode);
+      createWithParents(zkw, znode, data);
     } catch(InterruptedException ie) {
       zkw.interruptedException(ie);
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 308beba555a7..1ee0500568ea 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -221,6 +221,27 @@ public void testMultipleZK() {
     }
   }
 
+  /**
+   * Create a znode with data
+   * @throws Exception
+   */
+  @Test
+  public void testCreateWithParents() throws Exception {
+    ZooKeeperWatcher zkw =
+        new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
+          TestZooKeeper.class.getName(), null);
+    byte[] expectedData = new byte[] { 1, 2, 3 };
+    ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
+    byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
+    assertEquals(Bytes.equals(expectedData, data), true);
+    ZKUtil.deleteNodeRecursively(zkw, "/l1");
+
+    ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
+    data = ZKUtil.getData(zkw, "/testCreateWithParents");
+    assertEquals(Bytes.equals(expectedData, data), true);
+    ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
+  }
+
   /**
    * Create a bunch of znodes in a hierarchy, try deleting one that has childs
    * (it will fail), then delete it recursively, then delete the last znode

From 7be3057e923487705b1ae4ceccf897e366ac304a Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 10 May 2013 05:58:16 +0000
Subject: [PATCH 0968/1540] HBASE-8513 [0.94] Fix class files with CRLF endings

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1480887 13f79535-47bb-0310-9956-ffa450edef68
---
 .../BigDecimalColumnInterpreter.java          | 208 +++----
 .../hadoop/hbase/filter/FuzzyRowFilter.java   | 578 +++++++++---------
 .../hbase/filter/TestFuzzyRowFilter.java      | 408 ++++++-------
 .../zookeeper/TestRecoverableZooKeeper.java   | 246 ++++----
 4 files changed, 720 insertions(+), 720 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
index d1594428041b..592e7922145b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
@@ -1,104 +1,104 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client.coprocessor;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.RoundingMode;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * ColumnInterpreter for doing Aggregation's with BigDecimal columns. 
- * This class is required at the RegionServer also.
- *
- */
-public class BigDecimalColumnInterpreter implements ColumnInterpreter<BigDecimal, BigDecimal> {
-  private static final Log log = LogFactory.getLog(BigDecimalColumnInterpreter.class);
-
-  @Override
-  public void readFields(DataInput arg0) throws IOException {
-  }
-
-  @Override
-  public void write(DataOutput arg0) throws IOException {
-  }
-
-  @Override
-  public BigDecimal getValue(byte[] family, byte[] qualifier, KeyValue kv)
-      throws IOException {
-    if ((kv == null || kv.getValue() == null)) return null;
-    return Bytes.toBigDecimal(kv.getValue()).setScale(2, RoundingMode.HALF_EVEN);
-  }
-
-  @Override
-  public BigDecimal add(BigDecimal val1, BigDecimal val2) {
-    if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) {
-      return ((val1 == null) ? val2 : val1);
-    }
-    if (val1 == null) return null;
-    return val1.add(val2).setScale(2, RoundingMode.HALF_EVEN);
-  }
-
-  @Override
-  public BigDecimal getMaxValue() {
-    return BigDecimal.valueOf(Double.MAX_VALUE);
-  }
-
-  @Override
-  public BigDecimal getMinValue() {
-    return BigDecimal.valueOf(Double.MIN_VALUE);
-  }
-
-  @Override
-  public BigDecimal multiply(BigDecimal val1, BigDecimal val2) {
-    return (((val1 == null) || (val2 == null)) ? null : val1.multiply(val2).setScale(2,
-      RoundingMode.HALF_EVEN));
-  }
-
-  @Override
-  public BigDecimal increment(BigDecimal val) {
-    return ((val == null) ? null : val.add(BigDecimal.ONE));
-  }
-
-  @Override
-  public BigDecimal castToReturnType(BigDecimal val) {
-    return val;
-  }
-
-  @Override
-  public int compare(BigDecimal val1, BigDecimal val2) {
-    if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) {
-      return ((val1 == null) ? -1 : 1);
-    }
-    if (val1 == null) return 0;
-    return val1.compareTo(val2);
-  }
-
-  @Override
-  public double divideForAvg(BigDecimal val1, Long paramLong) {
-    return (((paramLong == null) || (val1 == null)) ? (Double.NaN) :
-      val1.doubleValue() / paramLong.doubleValue());
-  }
-}
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client.coprocessor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * ColumnInterpreter for doing Aggregation's with BigDecimal columns. 
+ * This class is required at the RegionServer also.
+ *
+ */
+public class BigDecimalColumnInterpreter implements ColumnInterpreter<BigDecimal, BigDecimal> {
+  private static final Log log = LogFactory.getLog(BigDecimalColumnInterpreter.class);
+
+  @Override
+  public void readFields(DataInput arg0) throws IOException {
+  }
+
+  @Override
+  public void write(DataOutput arg0) throws IOException {
+  }
+
+  @Override
+  public BigDecimal getValue(byte[] family, byte[] qualifier, KeyValue kv)
+      throws IOException {
+    if ((kv == null || kv.getValue() == null)) return null;
+    return Bytes.toBigDecimal(kv.getValue()).setScale(2, RoundingMode.HALF_EVEN);
+  }
+
+  @Override
+  public BigDecimal add(BigDecimal val1, BigDecimal val2) {
+    if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) {
+      return ((val1 == null) ? val2 : val1);
+    }
+    if (val1 == null) return null;
+    return val1.add(val2).setScale(2, RoundingMode.HALF_EVEN);
+  }
+
+  @Override
+  public BigDecimal getMaxValue() {
+    return BigDecimal.valueOf(Double.MAX_VALUE);
+  }
+
+  @Override
+  public BigDecimal getMinValue() {
+    return BigDecimal.valueOf(Double.MIN_VALUE);
+  }
+
+  @Override
+  public BigDecimal multiply(BigDecimal val1, BigDecimal val2) {
+    return (((val1 == null) || (val2 == null)) ? null : val1.multiply(val2).setScale(2,
+      RoundingMode.HALF_EVEN));
+  }
+
+  @Override
+  public BigDecimal increment(BigDecimal val) {
+    return ((val == null) ? null : val.add(BigDecimal.ONE));
+  }
+
+  @Override
+  public BigDecimal castToReturnType(BigDecimal val) {
+    return val;
+  }
+
+  @Override
+  public int compare(BigDecimal val1, BigDecimal val2) {
+    if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) {
+      return ((val1 == null) ? -1 : 1);
+    }
+    if (val1 == null) return 0;
+    return val1.compareTo(val2);
+  }
+
+  @Override
+  public double divideForAvg(BigDecimal val1, Long paramLong) {
+    return (((paramLong == null) || (val1 == null)) ? (Double.NaN) :
+      val1.doubleValue() / paramLong.doubleValue());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index ef8909a0021d..388e091ff4fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -1,289 +1,289 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.filter;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * Filters data based on fuzzy row key. Performs fast-forwards during scanning.
- * It takes pairs (row key, fuzzy info) to match row keys. Where fuzzy info is
- * a byte array with 0 or 1 as its values:
- * <ul>
- *   <li>
- *     0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position
- *         must match
- *   </li>
- *   <li>
- *     1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this
- *         position can be different from the one in provided row key
- *   </li>
- * </ul>
- *
- *
- * Example:
- * Let's assume row key format is userId_actionId_year_month. Length of userId is fixed
- * and is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively.
- *
- * Let's assume that we need to fetch all users that performed certain action (encoded as "99")
- * in Jan of any year. Then the pair (row key, fuzzy info) would be the following:
- * row key = "????_99_????_01" (one can use any value instead of "?")
- * fuzzy info = "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00"
- *
- * I.e. fuzzy info tells the matching mask is "????_99_????_01", where at ? can be any value.
- *
- */
-public class FuzzyRowFilter extends FilterBase {
-  private List<Pair<byte[], byte[]>> fuzzyKeysData;
-  private boolean done = false;
-
-  /**
-   * Used internally for reflection, do NOT use it directly
-   */
-  public FuzzyRowFilter() {
-  }
-
-  public FuzzyRowFilter(List<Pair<byte[], byte[]>> fuzzyKeysData) {
-    this.fuzzyKeysData = fuzzyKeysData;
-  }
-
-  // TODO: possible improvement: save which fuzzy row key to use when providing a hint
-  @Override
-  public ReturnCode filterKeyValue(KeyValue kv) {
-    byte[] rowKey = kv.getRow();
-    // assigning "worst" result first and looking for better options
-    SatisfiesCode bestOption = SatisfiesCode.NO_NEXT;
-    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
-      SatisfiesCode satisfiesCode =
-              satisfies(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond());
-      if (satisfiesCode == SatisfiesCode.YES) {
-        return ReturnCode.INCLUDE;
-      }
-
-      if (satisfiesCode == SatisfiesCode.NEXT_EXISTS) {
-        bestOption = SatisfiesCode.NEXT_EXISTS;
-      }
-    }
-
-    if (bestOption == SatisfiesCode.NEXT_EXISTS) {
-      return ReturnCode.SEEK_NEXT_USING_HINT;
-    }
-
-    // the only unhandled SatisfiesCode is NO_NEXT, i.e. we are done
-    done = true;
-    return ReturnCode.NEXT_ROW;
-  }
-
-  @Override
-  public KeyValue getNextKeyHint(KeyValue currentKV) {
-    byte[] rowKey = currentKV.getRow();
-    byte[] nextRowKey = null;
-    // Searching for the "smallest" row key that satisfies at least one fuzzy row key
-    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
-      byte[] nextRowKeyCandidate = getNextForFuzzyRule(rowKey,
-              fuzzyData.getFirst(), fuzzyData.getSecond());
-      if (nextRowKeyCandidate == null) {
-        continue;
-      }
-      if (nextRowKey == null || Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0) {
-        nextRowKey = nextRowKeyCandidate;
-      }
-    }
-
-    if (nextRowKey == null) {
-      // SHOULD NEVER happen
-      // TODO: is there a better way than throw exception? (stop the scanner?)
-      throw new IllegalStateException("No next row key that satisfies fuzzy exists when" +
-                                         " getNextKeyHint() is invoked." +
-                                         " Filter: " + this.toString() +
-                                         " currentKV: " + currentKV.toString());
-    }
-
-    return KeyValue.createFirstOnRow(nextRowKey);
-  }
-
-  @Override
-  public boolean filterAllRemaining() {
-    return done;
-  }
-
-  @Override
-  public void write(DataOutput dataOutput) throws IOException {
-    dataOutput.writeInt(this.fuzzyKeysData.size());
-    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
-      Bytes.writeByteArray(dataOutput, fuzzyData.getFirst());
-      Bytes.writeByteArray(dataOutput, fuzzyData.getSecond());
-    }
-  }
-
-  @Override
-  public void readFields(DataInput dataInput) throws IOException {
-    int count = dataInput.readInt();
-    this.fuzzyKeysData = new ArrayList<Pair<byte[], byte[]>>(count);
-    for (int i = 0; i < count; i++) {
-      byte[] keyBytes = Bytes.readByteArray(dataInput);
-      byte[] keyMeta = Bytes.readByteArray(dataInput);
-      this.fuzzyKeysData.add(new Pair<byte[], byte[]>(keyBytes, keyMeta));
-    }
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb = new StringBuilder();
-    sb.append("FuzzyRowFilter");
-    sb.append("{fuzzyKeysData=");
-    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
-      sb.append('{').append(Bytes.toStringBinary(fuzzyData.getFirst())).append(":");
-      sb.append(Bytes.toStringBinary(fuzzyData.getSecond())).append('}');
-    }
-    sb.append("}, ");
-    return sb.toString();
-  }
-
-  // Utility methods
-
-  static enum SatisfiesCode {
-    // row satisfies fuzzy rule
-    YES,
-    // row doesn't satisfy fuzzy rule, but there's possible greater row that does
-    NEXT_EXISTS,
-    // row doesn't satisfy fuzzy rule and there's no greater row that does
-    NO_NEXT
-  }
-
-  static SatisfiesCode satisfies(byte[] row,
-                                         byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
-    return satisfies(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
-  }
-
-  private static SatisfiesCode satisfies(byte[] row, int offset, int length,
-                                         byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
-    if (row == null) {
-      // do nothing, let scan to proceed
-      return SatisfiesCode.YES;
-    }
-
-    boolean nextRowKeyCandidateExists = false;
-
-    for (int i = 0; i < fuzzyKeyMeta.length && i < length; i++) {
-      // First, checking if this position is fixed and not equals the given one
-      boolean byteAtPositionFixed = fuzzyKeyMeta[i] == 0;
-      boolean fixedByteIncorrect = byteAtPositionFixed && fuzzyKeyBytes[i] != row[i + offset];
-      if (fixedByteIncorrect) {
-        // in this case there's another row that satisfies fuzzy rule and bigger than this row
-        if (nextRowKeyCandidateExists) {
-          return SatisfiesCode.NEXT_EXISTS;
-        }
-
-        // If this row byte is less than fixed then there's a byte array bigger than
-        // this row and which satisfies the fuzzy rule. Otherwise there's no such byte array:
-        // this row is simply bigger than any byte array that satisfies the fuzzy rule
-        boolean rowByteLessThanFixed = (row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF);
-        return  rowByteLessThanFixed ? SatisfiesCode.NEXT_EXISTS : SatisfiesCode.NO_NEXT;
-      }
-
-      // Second, checking if this position is not fixed and byte value is not the biggest. In this
-      // case there's a byte array bigger than this row and which satisfies the fuzzy rule. To get
-      // bigger byte array that satisfies the rule we need to just increase this byte
-      // (see the code of getNextForFuzzyRule below) by one.
-      // Note: if non-fixed byte is already at biggest value, this doesn't allow us to say there's
-      //       bigger one that satisfies the rule as it can't be increased.
-      if (fuzzyKeyMeta[i] == 1 && !isMax(fuzzyKeyBytes[i])) {
-        nextRowKeyCandidateExists = true;
-      }
-    }
-
-    return SatisfiesCode.YES;
-  }
-
-  private static boolean isMax(byte fuzzyKeyByte) {
-    return (fuzzyKeyByte & 0xFF) == 255;
-  }
-
-  static byte[] getNextForFuzzyRule(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
-    return getNextForFuzzyRule(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
-  }
-
-  /**
-   * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists,
-   *         null otherwise
-   */
-  private static byte[] getNextForFuzzyRule(byte[] row, int offset, int length,
-                                            byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
-    // To find out the next "smallest" byte array that satisfies fuzzy rule and "greater" than
-    // the given one we do the following:
-    // 1. setting values on all "fixed" positions to the values from fuzzyKeyBytes
-    // 2. if during the first step given row did not increase, then we increase the value at
-    //    the first "non-fixed" position (where it is not maximum already)
-
-    // It is easier to perform this by using fuzzyKeyBytes copy and setting "non-fixed" position
-    // values than otherwise.
-    byte[] result = Arrays.copyOf(fuzzyKeyBytes,
-                                  length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length);
-    int toInc = -1;
-
-    boolean increased = false;
-    for (int i = 0; i < result.length; i++) {
-      if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) {
-        result[i] = row[offset + i];
-        if (!isMax(row[i])) {
-          // this is "non-fixed" position and is not at max value, hence we can increase it
-          toInc = i;
-        }
-      } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == 0) {
-        if ((row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF)) {
-          // if setting value for any fixed position increased the original array,
-          // we are OK
-          increased = true;
-          break;
-        }
-        if ((row[i + offset] & 0xFF) > (fuzzyKeyBytes[i] & 0xFF)) {
-          // if setting value for any fixed position makes array "smaller", then just stop:
-          // in case we found some non-fixed position to increase we will do it, otherwise
-          // there's no "next" row key that satisfies fuzzy rule and "greater" than given row
-          break;
-        }
-      }
-    }
-
-    if (!increased) {
-      if (toInc < 0) {
-        return null;
-      }
-      result[toInc]++;
-
-      // Setting all "non-fixed" positions to zeroes to the right of the one we increased so
-      // that found "next" row key is the smallest possible
-      for (int i = toInc + 1; i < result.length; i++) {
-        if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) {
-          result[i] = 0;
-        }
-      }
-    }
-
-    return result;
-  }
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * Filters data based on fuzzy row key. Performs fast-forwards during scanning.
+ * It takes pairs (row key, fuzzy info) to match row keys. Where fuzzy info is
+ * a byte array with 0 or 1 as its values:
+ * <ul>
+ *   <li>
+ *     0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position
+ *         must match
+ *   </li>
+ *   <li>
+ *     1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this
+ *         position can be different from the one in provided row key
+ *   </li>
+ * </ul>
+ *
+ *
+ * Example:
+ * Let's assume row key format is userId_actionId_year_month. Length of userId is fixed
+ * and is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively.
+ *
+ * Let's assume that we need to fetch all users that performed certain action (encoded as "99")
+ * in Jan of any year. Then the pair (row key, fuzzy info) would be the following:
+ * row key = "????_99_????_01" (one can use any value instead of "?")
+ * fuzzy info = "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00"
+ *
+ * I.e. fuzzy info tells the matching mask is "????_99_????_01", where at ? can be any value.
+ *
+ */
+public class FuzzyRowFilter extends FilterBase {
+  private List<Pair<byte[], byte[]>> fuzzyKeysData;
+  private boolean done = false;
+
+  /**
+   * Used internally for reflection, do NOT use it directly
+   */
+  public FuzzyRowFilter() {
+  }
+
+  public FuzzyRowFilter(List<Pair<byte[], byte[]>> fuzzyKeysData) {
+    this.fuzzyKeysData = fuzzyKeysData;
+  }
+
+  // TODO: possible improvement: save which fuzzy row key to use when providing a hint
+  @Override
+  public ReturnCode filterKeyValue(KeyValue kv) {
+    byte[] rowKey = kv.getRow();
+    // assigning "worst" result first and looking for better options
+    SatisfiesCode bestOption = SatisfiesCode.NO_NEXT;
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      SatisfiesCode satisfiesCode =
+              satisfies(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond());
+      if (satisfiesCode == SatisfiesCode.YES) {
+        return ReturnCode.INCLUDE;
+      }
+
+      if (satisfiesCode == SatisfiesCode.NEXT_EXISTS) {
+        bestOption = SatisfiesCode.NEXT_EXISTS;
+      }
+    }
+
+    if (bestOption == SatisfiesCode.NEXT_EXISTS) {
+      return ReturnCode.SEEK_NEXT_USING_HINT;
+    }
+
+    // the only unhandled SatisfiesCode is NO_NEXT, i.e. we are done
+    done = true;
+    return ReturnCode.NEXT_ROW;
+  }
+
+  @Override
+  public KeyValue getNextKeyHint(KeyValue currentKV) {
+    byte[] rowKey = currentKV.getRow();
+    byte[] nextRowKey = null;
+    // Searching for the "smallest" row key that satisfies at least one fuzzy row key
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      byte[] nextRowKeyCandidate = getNextForFuzzyRule(rowKey,
+              fuzzyData.getFirst(), fuzzyData.getSecond());
+      if (nextRowKeyCandidate == null) {
+        continue;
+      }
+      if (nextRowKey == null || Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0) {
+        nextRowKey = nextRowKeyCandidate;
+      }
+    }
+
+    if (nextRowKey == null) {
+      // SHOULD NEVER happen
+      // TODO: is there a better way than throw exception? (stop the scanner?)
+      throw new IllegalStateException("No next row key that satisfies fuzzy exists when" +
+                                         " getNextKeyHint() is invoked." +
+                                         " Filter: " + this.toString() +
+                                         " currentKV: " + currentKV.toString());
+    }
+
+    return KeyValue.createFirstOnRow(nextRowKey);
+  }
+
+  @Override
+  public boolean filterAllRemaining() {
+    return done;
+  }
+
+  @Override
+  public void write(DataOutput dataOutput) throws IOException {
+    dataOutput.writeInt(this.fuzzyKeysData.size());
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      Bytes.writeByteArray(dataOutput, fuzzyData.getFirst());
+      Bytes.writeByteArray(dataOutput, fuzzyData.getSecond());
+    }
+  }
+
+  @Override
+  public void readFields(DataInput dataInput) throws IOException {
+    int count = dataInput.readInt();
+    this.fuzzyKeysData = new ArrayList<Pair<byte[], byte[]>>(count);
+    for (int i = 0; i < count; i++) {
+      byte[] keyBytes = Bytes.readByteArray(dataInput);
+      byte[] keyMeta = Bytes.readByteArray(dataInput);
+      this.fuzzyKeysData.add(new Pair<byte[], byte[]>(keyBytes, keyMeta));
+    }
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder();
+    sb.append("FuzzyRowFilter");
+    sb.append("{fuzzyKeysData=");
+    for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
+      sb.append('{').append(Bytes.toStringBinary(fuzzyData.getFirst())).append(":");
+      sb.append(Bytes.toStringBinary(fuzzyData.getSecond())).append('}');
+    }
+    sb.append("}, ");
+    return sb.toString();
+  }
+
+  // Utility methods
+
+  static enum SatisfiesCode {
+    // row satisfies fuzzy rule
+    YES,
+    // row doesn't satisfy fuzzy rule, but there's possible greater row that does
+    NEXT_EXISTS,
+    // row doesn't satisfy fuzzy rule and there's no greater row that does
+    NO_NEXT
+  }
+
+  static SatisfiesCode satisfies(byte[] row,
+                                         byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    return satisfies(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
+  }
+
+  private static SatisfiesCode satisfies(byte[] row, int offset, int length,
+                                         byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    if (row == null) {
+      // do nothing, let scan to proceed
+      return SatisfiesCode.YES;
+    }
+
+    boolean nextRowKeyCandidateExists = false;
+
+    for (int i = 0; i < fuzzyKeyMeta.length && i < length; i++) {
+      // First, checking if this position is fixed and not equals the given one
+      boolean byteAtPositionFixed = fuzzyKeyMeta[i] == 0;
+      boolean fixedByteIncorrect = byteAtPositionFixed && fuzzyKeyBytes[i] != row[i + offset];
+      if (fixedByteIncorrect) {
+        // in this case there's another row that satisfies fuzzy rule and bigger than this row
+        if (nextRowKeyCandidateExists) {
+          return SatisfiesCode.NEXT_EXISTS;
+        }
+
+        // If this row byte is less than fixed then there's a byte array bigger than
+        // this row and which satisfies the fuzzy rule. Otherwise there's no such byte array:
+        // this row is simply bigger than any byte array that satisfies the fuzzy rule
+        boolean rowByteLessThanFixed = (row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF);
+        return  rowByteLessThanFixed ? SatisfiesCode.NEXT_EXISTS : SatisfiesCode.NO_NEXT;
+      }
+
+      // Second, checking if this position is not fixed and byte value is not the biggest. In this
+      // case there's a byte array bigger than this row and which satisfies the fuzzy rule. To get
+      // bigger byte array that satisfies the rule we need to just increase this byte
+      // (see the code of getNextForFuzzyRule below) by one.
+      // Note: if non-fixed byte is already at biggest value, this doesn't allow us to say there's
+      //       bigger one that satisfies the rule as it can't be increased.
+      if (fuzzyKeyMeta[i] == 1 && !isMax(fuzzyKeyBytes[i])) {
+        nextRowKeyCandidateExists = true;
+      }
+    }
+
+    return SatisfiesCode.YES;
+  }
+
+  private static boolean isMax(byte fuzzyKeyByte) {
+    return (fuzzyKeyByte & 0xFF) == 255;
+  }
+
+  static byte[] getNextForFuzzyRule(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    return getNextForFuzzyRule(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
+  }
+
+  /**
+   * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists,
+   *         null otherwise
+   */
+  private static byte[] getNextForFuzzyRule(byte[] row, int offset, int length,
+                                            byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
+    // To find out the next "smallest" byte array that satisfies fuzzy rule and "greater" than
+    // the given one we do the following:
+    // 1. setting values on all "fixed" positions to the values from fuzzyKeyBytes
+    // 2. if during the first step given row did not increase, then we increase the value at
+    //    the first "non-fixed" position (where it is not maximum already)
+
+    // It is easier to perform this by using fuzzyKeyBytes copy and setting "non-fixed" position
+    // values than otherwise.
+    byte[] result = Arrays.copyOf(fuzzyKeyBytes,
+                                  length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length);
+    int toInc = -1;
+
+    boolean increased = false;
+    for (int i = 0; i < result.length; i++) {
+      if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) {
+        result[i] = row[offset + i];
+        if (!isMax(row[i])) {
+          // this is "non-fixed" position and is not at max value, hence we can increase it
+          toInc = i;
+        }
+      } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == 0) {
+        if ((row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF)) {
+          // if setting value for any fixed position increased the original array,
+          // we are OK
+          increased = true;
+          break;
+        }
+        if ((row[i + offset] & 0xFF) > (fuzzyKeyBytes[i] & 0xFF)) {
+          // if setting value for any fixed position makes array "smaller", then just stop:
+          // in case we found some non-fixed position to increase we will do it, otherwise
+          // there's no "next" row key that satisfies fuzzy rule and "greater" than given row
+          break;
+        }
+      }
+    }
+
+    if (!increased) {
+      if (toInc < 0) {
+        return null;
+      }
+      result[toInc]++;
+
+      // Setting all "non-fixed" positions to zeroes to the right of the one we increased so
+      // that found "next" row key is the smallest possible
+      for (int i = toInc + 1; i < result.length; i++) {
+        if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) {
+          result[i] = 0;
+        }
+      }
+    }
+
+    return result;
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
index 0b930657e955..4faca8216508 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
@@ -1,204 +1,204 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.filter;
-
-import org.apache.hadoop.hbase.SmallTests;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(SmallTests.class)
-public class TestFuzzyRowFilter {
-  @Test
-  public void testSatisfies() {
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
-            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 0, 0, 1}, // row to check
-                                     new byte[]{1, 0, 1}, // fuzzy row
-                                     new byte[]{0, 1, 0})); // mask
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES,
-            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 1, 0, 1},
-                                     new byte[]{1, 0, 1},
-                                     new byte[]{0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
-            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 2, 0, 1},
-                                     new byte[]{1, 0, 1},
-                                     new byte[]{0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
-            FuzzyRowFilter.satisfies(new byte[]{2, 3, 1, 1, 1},
-                                     new byte[]{1, 0, 1},
-                                     new byte[]{0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES,
-            FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 3, 3},
-                                     new byte[]{1, 2, 0, 3},
-                                     new byte[]{0, 0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
-            FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0}, // row to check
-                                     new byte[]{1, 2, 0, 3}, // fuzzy row
-                                     new byte[]{0, 0, 1, 0})); // mask
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
-            FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0},
-                                     new byte[]{1, (byte) 245, 0, 3},
-                                     new byte[]{0, 0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
-            FuzzyRowFilter.satisfies(new byte[]{1, (byte) 245, 1, 3, 0},
-                                     new byte[]{1, 1, 0, 3},
-                                     new byte[]{0, 0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
-            FuzzyRowFilter.satisfies(new byte[]{1, 3, 1, 3, 0},
-                                     new byte[]{1, 2, 0, 3},
-                                     new byte[]{0, 0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
-            FuzzyRowFilter.satisfies(new byte[]{2, 1, 1, 1, 0},
-                                     new byte[]{1, 2, 0, 3},
-                                     new byte[]{0, 0, 1, 0}));
-
-    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
-            FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 0, 1},
-                                     new byte[]{0, 1, 2},
-                                     new byte[]{1, 0, 0}));
-  }
-
-  @Test
-  public void testGetNextForFuzzyRule() {
-    assertNext(
-            new byte[]{0, 1, 2}, // fuzzy row
-            new byte[]{1, 0, 0}, // mask
-            new byte[]{1, 2, 1, 0, 1}, // current
-            new byte[]{2, 1, 2, 0, 0}); // expected next
-
-    assertNext(
-            new byte[]{0, 1, 2}, // fuzzy row
-            new byte[]{1, 0, 0}, // mask
-            new byte[]{1, 1, 2, 0, 1}, // current
-            new byte[]{1, 1, 2, 0, 2}); // expected next
-
-    assertNext(
-            new byte[]{0, 1, 0, 2, 0}, // fuzzy row
-            new byte[]{1, 0, 1, 0, 1}, // mask
-            new byte[]{1, 0, 2, 0, 1}, // current
-            new byte[]{1, 1, 0, 2, 0}); // expected next
-
-    assertNext(
-            new byte[]{1, 0, 1},
-            new byte[]{0, 1, 0},
-            new byte[]{1, (byte) 128, 2, 0, 1},
-            new byte[]{1, (byte) 129, 1, 0, 0});
-
-    assertNext(
-            new byte[]{0, 1, 0, 1},
-            new byte[]{1, 0, 1, 0},
-            new byte[]{5, 1, 0, 1},
-            new byte[]{5, 1, 1, 1});
-
-    assertNext(
-            new byte[]{0, 1, 0, 1},
-            new byte[]{1, 0, 1, 0},
-            new byte[]{5, 1, 0, 1, 1},
-            new byte[]{5, 1, 0, 1, 2});
-
-    assertNext(
-            new byte[]{0, 1, 0, 0}, // fuzzy row
-            new byte[]{1, 0, 1, 1}, // mask
-            new byte[]{5, 1, (byte) 255, 1}, // current
-            new byte[]{5, 1, (byte) 255, 2}); // expected next
-
-    assertNext(
-            new byte[]{0, 1, 0, 1}, // fuzzy row
-            new byte[]{1, 0, 1, 0}, // mask
-            new byte[]{5, 1, (byte) 255, 1}, // current
-            new byte[]{6, 1, 0, 1}); // expected next
-
-    assertNext(
-            new byte[]{0, 1, 0, 1}, // fuzzy row
-            new byte[]{1, 0, 1, 0}, // mask
-            new byte[]{5, 1, (byte) 255, 0}, // current
-            new byte[]{5, 1, (byte) 255, 1}); // expected next
-
-    assertNext(
-            new byte[]{5, 1, 1, 0},
-            new byte[]{0, 0, 1, 1},
-            new byte[]{5, 1, (byte) 255, 1},
-            new byte[]{5, 1, (byte) 255, 2});
-
-    assertNext(
-            new byte[]{1, 1, 1, 1},
-            new byte[]{0, 0, 1, 1},
-            new byte[]{1, 1, 2, 2},
-            new byte[]{1, 1, 2, 3});
-
-    assertNext(
-            new byte[]{1, 1, 1, 1},
-            new byte[]{0, 0, 1, 1},
-            new byte[]{1, 1, 3, 2},
-            new byte[]{1, 1, 3, 3});
-
-    assertNext(
-            new byte[]{1, 1, 1, 1},
-            new byte[]{1, 1, 1, 1},
-            new byte[]{1, 1, 2, 3},
-            new byte[]{1, 1, 2, 4});
-
-    assertNext(
-            new byte[]{1, 1, 1, 1},
-            new byte[]{1, 1, 1, 1},
-            new byte[]{1, 1, 3, 2},
-            new byte[]{1, 1, 3, 3});
-
-    assertNext(
-            new byte[]{1, 1, 0, 0},
-            new byte[]{0, 0, 1, 1},
-            new byte[]{0, 1, 3, 2},
-            new byte[]{1, 1, 0, 0});
-
-    // No next for this one
-    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
-            new byte[]{2, 3, 1, 1, 1}, // row to check
-            new byte[]{1, 0, 1}, // fuzzy row
-            new byte[]{0, 1, 0})); // mask
-    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
-            new byte[]{1, (byte) 245, 1, 3, 0},
-            new byte[]{1, 1, 0, 3},
-            new byte[]{0, 0, 1, 0}));
-    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
-            new byte[]{1, 3, 1, 3, 0},
-            new byte[]{1, 2, 0, 3},
-            new byte[]{0, 0, 1, 0}));
-    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
-            new byte[]{2, 1, 1, 1, 0},
-            new byte[]{1, 2, 0, 3},
-            new byte[]{0, 0, 1, 0}));
-  }
-
-  private void assertNext(byte[] fuzzyRow, byte[] mask, byte[] current, byte[] expected) {
-    byte[] nextForFuzzyRule = FuzzyRowFilter.getNextForFuzzyRule(current, fuzzyRow, mask);
-    Assert.assertArrayEquals(expected, nextForFuzzyRule);
-  }
-
-  @org.junit.Rule
-  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
-          new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestFuzzyRowFilter {
+  @Test
+  public void testSatisfies() {
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 0, 0, 1}, // row to check
+                                     new byte[]{1, 0, 1}, // fuzzy row
+                                     new byte[]{0, 1, 0})); // mask
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 1, 0, 1},
+                                     new byte[]{1, 0, 1},
+                                     new byte[]{0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 2, 0, 1},
+                                     new byte[]{1, 0, 1},
+                                     new byte[]{0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{2, 3, 1, 1, 1},
+                                     new byte[]{1, 0, 1},
+                                     new byte[]{0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES,
+            FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 3, 3},
+                                     new byte[]{1, 2, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0}, // row to check
+                                     new byte[]{1, 2, 0, 3}, // fuzzy row
+                                     new byte[]{0, 0, 1, 0})); // mask
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0},
+                                     new byte[]{1, (byte) 245, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{1, (byte) 245, 1, 3, 0},
+                                     new byte[]{1, 1, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{1, 3, 1, 3, 0},
+                                     new byte[]{1, 2, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT,
+            FuzzyRowFilter.satisfies(new byte[]{2, 1, 1, 1, 0},
+                                     new byte[]{1, 2, 0, 3},
+                                     new byte[]{0, 0, 1, 0}));
+
+    Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS,
+            FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 0, 1},
+                                     new byte[]{0, 1, 2},
+                                     new byte[]{1, 0, 0}));
+  }
+
+  @Test
+  public void testGetNextForFuzzyRule() {
+    assertNext(
+            new byte[]{0, 1, 2}, // fuzzy row
+            new byte[]{1, 0, 0}, // mask
+            new byte[]{1, 2, 1, 0, 1}, // current
+            new byte[]{2, 1, 2, 0, 0}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 2}, // fuzzy row
+            new byte[]{1, 0, 0}, // mask
+            new byte[]{1, 1, 2, 0, 1}, // current
+            new byte[]{1, 1, 2, 0, 2}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 0, 2, 0}, // fuzzy row
+            new byte[]{1, 0, 1, 0, 1}, // mask
+            new byte[]{1, 0, 2, 0, 1}, // current
+            new byte[]{1, 1, 0, 2, 0}); // expected next
+
+    assertNext(
+            new byte[]{1, 0, 1},
+            new byte[]{0, 1, 0},
+            new byte[]{1, (byte) 128, 2, 0, 1},
+            new byte[]{1, (byte) 129, 1, 0, 0});
+
+    assertNext(
+            new byte[]{0, 1, 0, 1},
+            new byte[]{1, 0, 1, 0},
+            new byte[]{5, 1, 0, 1},
+            new byte[]{5, 1, 1, 1});
+
+    assertNext(
+            new byte[]{0, 1, 0, 1},
+            new byte[]{1, 0, 1, 0},
+            new byte[]{5, 1, 0, 1, 1},
+            new byte[]{5, 1, 0, 1, 2});
+
+    assertNext(
+            new byte[]{0, 1, 0, 0}, // fuzzy row
+            new byte[]{1, 0, 1, 1}, // mask
+            new byte[]{5, 1, (byte) 255, 1}, // current
+            new byte[]{5, 1, (byte) 255, 2}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 0, 1}, // fuzzy row
+            new byte[]{1, 0, 1, 0}, // mask
+            new byte[]{5, 1, (byte) 255, 1}, // current
+            new byte[]{6, 1, 0, 1}); // expected next
+
+    assertNext(
+            new byte[]{0, 1, 0, 1}, // fuzzy row
+            new byte[]{1, 0, 1, 0}, // mask
+            new byte[]{5, 1, (byte) 255, 0}, // current
+            new byte[]{5, 1, (byte) 255, 1}); // expected next
+
+    assertNext(
+            new byte[]{5, 1, 1, 0},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{5, 1, (byte) 255, 1},
+            new byte[]{5, 1, (byte) 255, 2});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{1, 1, 2, 2},
+            new byte[]{1, 1, 2, 3});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{1, 1, 3, 2},
+            new byte[]{1, 1, 3, 3});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 2, 3},
+            new byte[]{1, 1, 2, 4});
+
+    assertNext(
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 1, 1},
+            new byte[]{1, 1, 3, 2},
+            new byte[]{1, 1, 3, 3});
+
+    assertNext(
+            new byte[]{1, 1, 0, 0},
+            new byte[]{0, 0, 1, 1},
+            new byte[]{0, 1, 3, 2},
+            new byte[]{1, 1, 0, 0});
+
+    // No next for this one
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{2, 3, 1, 1, 1}, // row to check
+            new byte[]{1, 0, 1}, // fuzzy row
+            new byte[]{0, 1, 0})); // mask
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{1, (byte) 245, 1, 3, 0},
+            new byte[]{1, 1, 0, 3},
+            new byte[]{0, 0, 1, 0}));
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{1, 3, 1, 3, 0},
+            new byte[]{1, 2, 0, 3},
+            new byte[]{0, 0, 1, 0}));
+    Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
+            new byte[]{2, 1, 1, 1, 0},
+            new byte[]{1, 2, 0, 3},
+            new byte[]{0, 0, 1, 0}));
+  }
+
+  private void assertNext(byte[] fuzzyRow, byte[] mask, byte[] current, byte[] expected) {
+    byte[] nextForFuzzyRule = FuzzyRowFilter.getNextForFuzzyRule(current, fuzzyRow, mask);
+    Assert.assertArrayEquals(expected, nextForFuzzyRule);
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+          new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
index a7b0c4b34c72..a69388469744 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
@@ -1,123 +1,123 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.zookeeper;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.util.Properties;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.Stat;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(MediumTests.class)
-public class TestRecoverableZooKeeper {
-
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
-  Abortable abortable = new Abortable() {
-    @Override
-    public void abort(String why, Throwable e) {
-
-    }
-
-    @Override
-    public boolean isAborted() {
-      return false;
-    }
-  };
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.startMiniZKCluster();
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniZKCluster();
-  }
-
-  @Test
-  public void testSetDataVersionMismatchInLoop() throws Exception {
-    String znode = "/hbase/unassigned/9af7cfc9b15910a0b3d714bf40a3248f";
-    Configuration conf = TEST_UTIL.getConfiguration();
-    Properties properties = ZKConfig.makeZKProps(conf);
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testSetDataVersionMismatchInLoop",
-        abortable, true);
-    String ensemble = ZKConfig.getZKQuorumServersString(properties);
-    RecoverableZooKeeper rzk = ZKUtil.connect(conf, ensemble, zkw);
-    rzk.create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-    rzk.setData(znode, "OPENING".getBytes(), 0);
-    Field zkField = RecoverableZooKeeper.class.getDeclaredField("zk");
-    zkField.setAccessible(true);
-    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
-    ZookeeperStub zkStub = new ZookeeperStub(ensemble, timeout, zkw);
-    zkStub.setThrowExceptionInNumOperations(1);
-    zkField.set(rzk, zkStub);
-    byte[] opened = "OPENED".getBytes();
-    rzk.setData(znode, opened, 1);
-    byte[] data = rzk.getData(znode, false, new Stat());
-    assertTrue(Bytes.equals(opened, data));
-  }
-
-  class ZookeeperStub extends ZooKeeper {
-
-    private int throwExceptionInNumOperations;
-
-    public ZookeeperStub(String connectString, int sessionTimeout, Watcher watcher)
-        throws IOException {
-      super(connectString, sessionTimeout, watcher);
-    }
-
-    public void setThrowExceptionInNumOperations(int throwExceptionInNumOperations) {
-      this.throwExceptionInNumOperations = throwExceptionInNumOperations;
-    }
-
-    private void checkThrowKeeperException() throws KeeperException {
-      if (throwExceptionInNumOperations == 1) {
-        throwExceptionInNumOperations = 0;
-        throw new KeeperException.ConnectionLossException();
-      }
-      if (throwExceptionInNumOperations > 0)
-        throwExceptionInNumOperations--;
-    }
-
-    @Override
-    public Stat setData(String path, byte[] data, int version) throws KeeperException,
-        InterruptedException {
-      Stat stat = super.setData(path, data, version);
-      checkThrowKeeperException();
-      return stat;
-    }
-  }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestRecoverableZooKeeper {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  Abortable abortable = new Abortable() {
+    @Override
+    public void abort(String why, Throwable e) {
+
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+  };
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  @Test
+  public void testSetDataVersionMismatchInLoop() throws Exception {
+    String znode = "/hbase/unassigned/9af7cfc9b15910a0b3d714bf40a3248f";
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Properties properties = ZKConfig.makeZKProps(conf);
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testSetDataVersionMismatchInLoop",
+        abortable, true);
+    String ensemble = ZKConfig.getZKQuorumServersString(properties);
+    RecoverableZooKeeper rzk = ZKUtil.connect(conf, ensemble, zkw);
+    rzk.create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+    rzk.setData(znode, "OPENING".getBytes(), 0);
+    Field zkField = RecoverableZooKeeper.class.getDeclaredField("zk");
+    zkField.setAccessible(true);
+    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
+    ZookeeperStub zkStub = new ZookeeperStub(ensemble, timeout, zkw);
+    zkStub.setThrowExceptionInNumOperations(1);
+    zkField.set(rzk, zkStub);
+    byte[] opened = "OPENED".getBytes();
+    rzk.setData(znode, opened, 1);
+    byte[] data = rzk.getData(znode, false, new Stat());
+    assertTrue(Bytes.equals(opened, data));
+  }
+
+  class ZookeeperStub extends ZooKeeper {
+
+    private int throwExceptionInNumOperations;
+
+    public ZookeeperStub(String connectString, int sessionTimeout, Watcher watcher)
+        throws IOException {
+      super(connectString, sessionTimeout, watcher);
+    }
+
+    public void setThrowExceptionInNumOperations(int throwExceptionInNumOperations) {
+      this.throwExceptionInNumOperations = throwExceptionInNumOperations;
+    }
+
+    private void checkThrowKeeperException() throws KeeperException {
+      if (throwExceptionInNumOperations == 1) {
+        throwExceptionInNumOperations = 0;
+        throw new KeeperException.ConnectionLossException();
+      }
+      if (throwExceptionInNumOperations > 0)
+        throwExceptionInNumOperations--;
+    }
+
+    @Override
+    public Stat setData(String path, byte[] data, int version) throws KeeperException,
+        InterruptedException {
+      Stat stat = super.setData(path, data, version);
+      checkThrowKeeperException();
+      return stat;
+    }
+  }
+}

From bfe346a496d983312939513fca7835a1bdc8d400 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 12 May 2013 15:12:11 +0000
Subject: [PATCH 0969/1540] HBASE-8530 Refine error message from ExportSnapshot
 when there is leftover snapshot in target cluster (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1481581 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 46aea0f82434..2455831b0ff5 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -622,7 +622,9 @@ public int run(String[] args) throws Exception {
 
     // Check if the snapshot already in-progress
     if (outputFs.exists(snapshotTmpDir)) {
-      System.err.println("A snapshot with the same name '" + snapshotName + "' is in-progress");
+      System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
+      System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
+      System.err.println("consider removing " + snapshotTmpDir + " before retrying export"); 
       return 1;
     }
 

From 8136a70b7c5bbb3ffbe0799a1f660e5627b23054 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 13 May 2013 21:33:21 +0000
Subject: [PATCH 0970/1540] HBASE-8367 LoadIncrementalHFiles does not return an
 error code nor throw Exception when failures occur due to timeouts (Brian
 Dougan)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1482113 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 448eca1ea02f..7366469a824c 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -290,6 +290,11 @@ public void doBulkLoad(Path hfofDir, final HTable table)
         LOG.error(err);
       }
     }
+    
+    if (queue != null && !queue.isEmpty()) {
+        throw new RuntimeException("Bulk load aborted with some files not yet loaded."
+          + "Please check log for more details.");
+    }
   }
 
   /**

From e4c383267d6c673174aad0dd3db05182346d49b9 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 14 May 2013 15:26:18 +0000
Subject: [PATCH 0971/1540] HBASE-8516 FSUtils.create() fail with ViewFS

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1482392 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HBaseFileSystem.java  |  5 +-
 .../org/apache/hadoop/hbase/io/FileLink.java  |  3 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   | 31 +------
 .../wal/SequenceFileLogWriter.java            |  9 ++-
 .../org/apache/hadoop/hbase/util/FSUtils.java | 81 +++++++++++++++++++
 5 files changed, 93 insertions(+), 36 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
index a1ae5b0f4bc6..1a864c2151f1 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 
 /**
@@ -205,8 +206,8 @@ public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
     boolean existsBefore = fs.exists(path);
     do {
       try {
-        return fs.create(path, perm, overwrite, fs.getConf().getInt("io.file.buffer.size", 4096),
-          fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+        return fs.create(path, perm, overwrite, FSUtils.getDefaultBufferSize(fs),
+          FSUtils.getDefaultReplication(fs, path), FSUtils.getDefaultBlockSize(fs, path), null);
       } catch (IOException ioe) {
         lastIOE = ioe;
         if (existsBefore && !overwrite) throw ioe;// a legitimate exception
diff --git a/src/main/java/org/apache/hadoop/hbase/io/FileLink.java b/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
index e04de85ccbed..dc910e6a0e29 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PositionedReadable;
 import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.hbase.util.FSUtils;
 
 /**
  * The FileLink is a sort of hardlink, that allows to access a file given a set of locations.
@@ -107,7 +108,7 @@ private static class FileLinkInputStream extends InputStream
 
     public FileLinkInputStream(final FileSystem fs, final FileLink fileLink)
         throws IOException {
-      this(fs, fileLink, fs.getConf().getInt("io.file.buffer.size", 4096));
+      this(fs, fileLink, FSUtils.getDefaultBufferSize(fs));
     }
 
     public FileLinkInputStream(final FileSystem fs, final FileLink fileLink, int bufferSize)
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 7fbdb6475ff1..b8d90a2a3aff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -406,7 +406,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
       }
     }
     this.blocksize = conf.getLong("hbase.regionserver.hlog.blocksize",
-        getDefaultBlockSize());
+        FSUtils.getDefaultBlockSize(this.fs, this.dir));
     // Roll at 95% of block size.
     float multi = conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f);
     this.logrollsize = (long)(this.blocksize * multi);
@@ -427,7 +427,7 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32);
     this.minTolerableReplication = conf.getInt(
         "hbase.regionserver.hlog.tolerable.lowreplication",
-        this.fs.getDefaultReplication());
+        FSUtils.getDefaultReplication(this.fs, this.dir));
     this.lowReplicationRollLimit = conf.getInt(
         "hbase.regionserver.hlog.lowreplication.rolllimit", 5);
     this.enabled = conf.getBoolean("hbase.regionserver.hlog.enabled", true);
@@ -460,33 +460,6 @@ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
     }
     coprocessorHost = new WALCoprocessorHost(this, conf);
   }
-  
-  // use reflection to search for getDefaultBlockSize(Path f)
-  // if the method doesn't exist, fall back to using getDefaultBlockSize()
-  private long getDefaultBlockSize() throws IOException {
-    Method m = null;
-    Class<? extends FileSystem> cls = this.fs.getClass();
-    try {
-      m = cls.getMethod("getDefaultBlockSize",
-          new Class<?>[] { Path.class });
-    } catch (NoSuchMethodException e) {
-      LOG.info("FileSystem doesn't support getDefaultBlockSize");
-    } catch (SecurityException e) {
-      LOG.info("Doesn't have access to getDefaultBlockSize on "
-          + "FileSystems", e);
-      m = null; // could happen on setAccessible()
-    }
-    if (null == m) {
-      return this.fs.getDefaultBlockSize();
-    } else {
-      try {
-        Object ret = m.invoke(this.fs, this.dir);
-        return ((Long)ret).longValue();
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
-    }
-  }
 
   /**
    * Find the 'getNumCurrentReplicas' on the passed <code>os</code> stream.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index fbe1ce5ca672..0c09c834d256 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
@@ -161,9 +162,9 @@ public void init(FileSystem fs, Path path, Configuration conf)
             Integer.valueOf(fs.getConf().getInt("io.file.buffer.size", 4096)),
             Short.valueOf((short)
               conf.getInt("hbase.regionserver.hlog.replication",
-              fs.getDefaultReplication())),
+              FSUtils.getDefaultReplication(fs, path))),
             Long.valueOf(conf.getLong("hbase.regionserver.hlog.blocksize",
-                fs.getDefaultBlockSize())),
+                FSUtils.getDefaultBlockSize(fs, path))),
             Boolean.valueOf(false) /*createParent*/,
             SequenceFile.CompressionType.NONE, new DefaultCodec(),
             createMetadata(conf, compress)
@@ -182,9 +183,9 @@ SequenceFile.CompressionType.NONE, new DefaultCodec(),
         HLog.getKeyClass(conf), WALEdit.class,
         fs.getConf().getInt("io.file.buffer.size", 4096),
         (short) conf.getInt("hbase.regionserver.hlog.replication",
-          fs.getDefaultReplication()),
+          FSUtils.getDefaultReplication(fs, path)),
         conf.getLong("hbase.regionserver.hlog.blocksize",
-          fs.getDefaultBlockSize()),
+          FSUtils.getDefaultBlockSize(fs, path)),
         SequenceFile.CompressionType.NONE,
         new DefaultCodec(),
         null,
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 8ae580868f6d..1bc3eea7d535 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -376,6 +376,87 @@ public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries
     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
   }
 
+  /**
+   * Return the number of bytes that large input files should be optimally
+   * be split into to minimize i/o time.
+   *
+   * use reflection to search for getDefaultBlockSize(Path f)
+   * if the method doesn't exist, fall back to using getDefaultBlockSize()
+   *
+   * @param fs filesystem object
+   * @return the default block size for the path's filesystem
+   * @throws IOException e
+   */
+  public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException {
+    Method m = null;
+    Class<? extends FileSystem> cls = fs.getClass();
+    try {
+      m = cls.getMethod("getDefaultBlockSize", new Class<?>[] { Path.class });
+    } catch (NoSuchMethodException e) {
+      LOG.info("FileSystem doesn't support getDefaultBlockSize");
+    } catch (SecurityException e) {
+      LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e);
+      m = null; // could happen on setAccessible()
+    }
+    if (m == null) {
+      return fs.getDefaultBlockSize();
+    } else {
+      try {
+        Object ret = m.invoke(fs, path);
+        return ((Long)ret).longValue();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
+  /*
+   * Get the default replication.
+   *
+   * use reflection to search for getDefaultReplication(Path f)
+   * if the method doesn't exist, fall back to using getDefaultReplication()
+   *
+   * @param fs filesystem object
+   * @param f path of file
+   * @return default replication for the path's filesystem
+   * @throws IOException e
+   */
+  public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException {
+    Method m = null;
+    Class<? extends FileSystem> cls = fs.getClass();
+    try {
+      m = cls.getMethod("getDefaultReplication", new Class<?>[] { Path.class });
+    } catch (NoSuchMethodException e) {
+      LOG.info("FileSystem doesn't support getDefaultReplication");
+    } catch (SecurityException e) {
+      LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e);
+      m = null; // could happen on setAccessible()
+    }
+    if (m == null) {
+      return fs.getDefaultReplication();
+    } else {
+      try {
+        Object ret = m.invoke(fs, path);
+        return ((Number)ret).shortValue();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
+  /**
+   * Returns the default buffer size to use during writes.
+   *
+   * The size of the buffer should probably be a multiple of hardware
+   * page size (4096 on Intel x86), and it determines how much data is
+   * buffered during read and write operations.
+   *
+   * @param fs filesystem object
+   * @return default buffer size to use during writes
+   */
+  public static int getDefaultBufferSize(final FileSystem fs) {
+    return fs.getConf().getInt("io.file.buffer.size", 4096);
+  }
 
   /**
    * Sets version of file system

From f0143683954338db2072c6c4b79b549d956a4de6 Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Wed, 15 May 2013 00:41:46 +0000
Subject: [PATCH 0972/1540] HBASE-8550 0.94 ChaosMonkey grep for master is too
 broad

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1482650 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HBaseClusterManager.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
index 7209ddcb01d9..bc3b3fd7f496 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
@@ -125,8 +125,12 @@ public String isRunningCommand(ServiceType service) {
     }
 
     protected String findPidCommand(ServiceType service) {
-      return String.format("ps aux | grep %s | grep -v grep | tr -s ' ' | cut -d ' ' -f2",
-          service);
+      String servicePathFilter = "";
+      if (service == ServiceType.HBASE_MASTER || service == ServiceType.HBASE_REGIONSERVER) {
+        servicePathFilter = " | grep hbase";
+      }
+      return String.format("ps ux | grep %s %s | grep -v grep | tr -s ' ' | cut -d ' ' -f2",
+          service, servicePathFilter);
     }
 
     public String signalCommand(ServiceType service, String signal) {

From 17ada13e060a3e1059a3bb6b5a8d655258bba40f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 15 May 2013 04:12:50 +0000
Subject: [PATCH 0973/1540] HBASE-8539 Double(or tripple ...) ZooKeeper
 listeners of the same type when Master recovers from ZK
 SessionExpiredException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1482671 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/HMaster.java    |  1 +
 .../hadoop/hbase/zookeeper/ZooKeeperWatcher.java   | 14 ++++++++++++++
 .../hbase/master/TestMasterZKSessionRecovery.java  |  4 ++++
 3 files changed, 19 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 1a2a14aeba3d..7dc1693221db 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1610,6 +1610,7 @@ public void abort(final String msg, final Throwable t) {
   private boolean tryRecoveringExpiredZKSession() throws InterruptedException,
       IOException, KeeperException, ExecutionException {
 
+    this.zooKeeper.unregisterAllListeners();
     this.zooKeeper.reconnectAfterExpiration();
 
     Callable<Boolean> callable = new Callable<Boolean> () {
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 0274db97311b..a43ff1688d1e 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -242,6 +242,20 @@ public void registerListenerFirst(ZooKeeperListener listener) {
     listeners.add(0, listener);
   }
 
+  /**
+   * Clean all existing listeners
+   */
+  public void unregisterAllListeners() {
+    listeners.clear();
+  }
+
+  /**
+   * @return The number of currently registered listeners
+   */
+  public int getNumberOfListeners() {
+    return listeners.size();
+  }
+
   /**
    * Get the connection to ZooKeeper.
    * @return connection reference to zookeeper
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
index b732cb5f88ad..d643e90f087e 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java
@@ -89,6 +89,8 @@ public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Except
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     cluster.startRegionServer();
     HMaster m = cluster.getMaster();
+    ZooKeeperWatcher zkw = m.getZooKeeperWatcher();
+    int expectedNumOfListeners = zkw.getNumberOfListeners();
     // now the cluster is up. So assign some regions.
     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
     byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
@@ -106,6 +108,8 @@ public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Except
     // The recovered master should not call retainAssignment, as it is not a
     // clean startup.
     assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
+    // number of listeners should be same as the value before master aborted
+    assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners());
   }
 
   static class MockLoadBalancer extends DefaultLoadBalancer {

From d340847a19b629e9ebde9d1a2581c6fc7ec0bb25 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 15 May 2013 06:23:23 +0000
Subject: [PATCH 0974/1540] HBASE-8508 improve unit-test coverage of package
 org.apache.hadoop.hbase.metrics.file (Ivan A. Veselovsky)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1482687 13f79535-47bb-0310-9956-ffa450edef68
---
 .../file/TestTimeStampingMetricsContext.java  | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java

diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java b/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
new file mode 100644
index 000000000000..0700508e87e8
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics.file;
+
+import java.io.File;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.SmallTests;
+
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test for TimeStampingMetricsContext functionality.
+ * (FQN class names are used to suppress javac warnings in imports.)
+ */
+@Category(SmallTests.class)
+@SuppressWarnings("deprecation")
+public class TestTimeStampingMetricsContext {
+
+  private static final int updatePeriodSeconds = 2;
+
+  private TimeStampingFileContext mc;
+
+  @Test
+  public void testFileUpdate() throws Exception {
+    final Date start = new Date();
+    final File metricOutFile = FileUtil.createLocalTempFile(
+      new File(FileUtils.getTempDirectory(),getClass().getName() + "-out-"), "", true);
+    assertTrue(metricOutFile.exists());
+    assertEquals(0L, metricOutFile.length());
+
+    mc = new TimeStampingFileContext();
+    org.apache.hadoop.metrics.ContextFactory cf
+      = org.apache.hadoop.metrics.ContextFactory.getFactory();
+    cf.setAttribute("test1.fileName", metricOutFile.getAbsolutePath());
+    cf.setAttribute("test1.period", Integer.toString(updatePeriodSeconds));
+    mc.init("test1", cf);
+
+    assertEquals("test1", mc.getContextName());
+
+    org.apache.hadoop.metrics.MetricsRecord r = mc.createRecord("testRecord");
+    r.setTag("testTag1", "testTagValue1");
+    r.setTag("testTag2", "testTagValue2");
+    r.setMetric("testMetric1", 1);
+    r.setMetric("testMetric2", 33);
+    r.update();
+
+    mc.startMonitoring();
+    assertTrue(mc.isMonitoring());
+
+    // wait 3/2 of the update period:
+    Thread.sleep((1000 * updatePeriodSeconds * 3)/2);
+
+    mc.stopMonitoring();
+    assertFalse(mc.isMonitoring());
+
+    mc.close();
+
+    Map<String, Collection<org.apache.hadoop.metrics.spi.OutputRecord>> m = mc.getAllRecords();
+    assertEquals(1, m.size());
+    Collection<org.apache.hadoop.metrics.spi.OutputRecord> outputRecords = m.get("testRecord");
+    assertNotNull(outputRecords);
+    assertEquals(1, outputRecords.size());
+    org.apache.hadoop.metrics.spi.OutputRecord outputRecord = outputRecords.iterator().next();
+    assertNotNull(outputRecord);
+
+    String outStr = FileUtils.readFileToString(metricOutFile);
+    assertTrue(outStr.length() > 0);
+    int pos = outStr.indexOf(" ");
+    String time = outStr.substring(0, pos);
+
+    DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+    Date date = df.parse(time);
+    assertTrue(date.after(start));
+    assertTrue(date.before(new Date()));
+
+    String reminder = outStr.substring(pos);
+    assertEquals(" test1.testRecord: testTag1=testTagValue1, testTag2=testTagValue2, testMetric1=1,"
+      +" testMetric2=33\n", reminder);
+  }
+
+}

From 668120cb0b9cab519db3f087616b6363bfb027c1 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 15 May 2013 07:19:10 +0000
Subject: [PATCH 0975/1540] HBASE-8547 Fix java.lang.RuntimeException: Cached
 an already cached block

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1482704 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/hfile/LruBlockCache.java | 13 ++++++++-----
 .../hadoop/hbase/io/hfile/TestLruBlockCache.java    |  4 ++--
 .../org/apache/hadoop/hbase/util/TestIdLock.java    |  7 ++++---
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 3812e9cafca0..c7e83fe82072 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -193,11 +193,11 @@ public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
     this(maxSize, blockSize, evictionThread,
         (int)Math.ceil(1.2*maxSize/blockSize),
-        DEFAULT_LOAD_FACTOR, 
+        DEFAULT_LOAD_FACTOR,
         DEFAULT_CONCURRENCY_LEVEL,
-        conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR), 
-        conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR), 
-        DEFAULT_SINGLE_FACTOR, 
+        conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR),
+        conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR),
+        DEFAULT_SINGLE_FACTOR,
         DEFAULT_MULTI_FACTOR,
         DEFAULT_MEMORY_FACTOR);
   }
@@ -276,7 +276,10 @@ public void setMaxSize(long maxSize) {
   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
     CachedBlock cb = map.get(cacheKey);
     if(cb != null) {
-      throw new RuntimeException("Cached an already cached block");
+      String msg = "Cached an already cached block: " + cacheKey + " cb:" + cb.getCacheKey();
+      msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
+      LOG.warn(msg);
+      assert false : msg;
     }
     cb = new CachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
     long newSize = updateSizeMetrics(cb, false);
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index 3a581ef7ffa4..7d8953168c35 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -27,7 +27,6 @@
 import java.util.Map;
 import java.util.Random;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.io.HeapSize;
@@ -149,8 +148,9 @@ public void testCacheSimple() throws Exception {
       try {
         cache.cacheBlock(block.cacheKey, block);
         assertTrue("Cache should not allow re-caching a block", false);
-      } catch(RuntimeException re) {
+      } catch(AssertionError re) {
         // expected
+        assertTrue(re.getMessage().contains("Cached an already cached block"));
       }
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java b/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java
index 478bfbdc54b6..403897466b2d 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.util;
 
+import static org.junit.Assert.assertTrue;
+
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.Callable;
@@ -28,12 +30,10 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
-import static org.junit.Assert.*;
-
 import org.apache.hadoop.hbase.MediumTests;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -106,6 +106,7 @@ public void testMultipleClients() throws Exception {
       idLock.assertMapEmpty();
     } finally {
       exec.shutdown();
+      exec.awaitTermination(5000, TimeUnit.MILLISECONDS);
     }
   }
 

From 376577c2441741ab6994d3a0bb7a49ea4a47fef2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 15 May 2013 19:23:45 +0000
Subject: [PATCH 0976/1540] HBASE-5930 Limits the amount of time an edit can
 live in the memstore. (Davaraj and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483022 13f79535-47bb-0310-9956-ffa450edef68
---
 security/src/test/resources/hbase-site.xml    |   8 --
 .../hbase/regionserver/FlushRequester.java    |   8 ++
 .../hadoop/hbase/regionserver/HRegion.java    |  34 +++++-
 .../hbase/regionserver/HRegionServer.java     |  48 ++++++++-
 .../hadoop/hbase/regionserver/MemStore.java   |  36 ++++++-
 .../hbase/regionserver/MemStoreFlusher.java   |  12 +++
 .../hadoop/hbase/regionserver/Store.java      |   7 ++
 src/main/resources/hbase-default.xml          |   8 ++
 .../hbase/regionserver/TestMemStore.java      | 101 +++++++++++++++++-
 .../hbase/regionserver/wal/TestWALReplay.java |   6 ++
 src/test/resources/hbase-site.xml             |   8 --
 11 files changed, 251 insertions(+), 25 deletions(-)

diff --git a/security/src/test/resources/hbase-site.xml b/security/src/test/resources/hbase-site.xml
index dcc7df2fe57b..cca8832b5a3d 100644
--- a/security/src/test/resources/hbase-site.xml
+++ b/security/src/test/resources/hbase-site.xml
@@ -96,14 +96,6 @@
     the master will notice a dead region server sooner. The default is 15 seconds.
     </description>
   </property>
-  <property>
-    <name>hbase.regionserver.optionalcacheflushinterval</name>
-    <value>1000</value>
-    <description>
-    Amount of time to wait since the last time a region was flushed before
-    invoking an optional cache flush. Default 60,000.
-    </description>
-  </property>
   <property>
     <name>hbase.regionserver.safemode</name>
     <value>false</value>
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
index b843c91f7c44..aecd9c383159 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
@@ -30,4 +30,12 @@ public interface FlushRequester {
    * @param region the HRegion requesting the cache flush
    */
   void requestFlush(HRegion region);
+
+  /**
+   * Tell the listener the cache needs to be flushed after a delay
+   *
+   * @param region the HRegion requesting the cache flush
+   * @param delay after how much time should the flush happen
+   */
+  void requestDelayedFlush(HRegion region, long delay);
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e5e885178b66..63bab9edb20b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -345,6 +345,7 @@ boolean isFlushRequested() {
   final RegionServerServices rsServices;
   private RegionServerAccounting rsAccounting;
   private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
+  private long flushCheckInterval;
   private long blockingMemStoreSize;
   final long threadWakeFrequency;
   // Used to guard closes
@@ -452,6 +453,8 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration confParam,
     else {
       this.conf = new CompoundConfiguration().add(confParam);
     }
+    this.flushCheckInterval = conf.getInt(MEMSTORE_PERIODIC_FLUSH_INTERVAL,
+        DEFAULT_CACHE_FLUSH_INTERVAL);
     this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
                     DEFAULT_ROWLOCK_WAIT_DURATION);
 
@@ -953,6 +956,12 @@ public List<StoreFile> close() throws IOException {
 
   private final Object closeLock = new Object();
 
+  /** Conf key for the periodic flush interval */
+  public static final String MEMSTORE_PERIODIC_FLUSH_INTERVAL = 
+      "hbase.regionserver.optionalcacheflushinterval";
+  /** Default interval for the memstore flush */
+  public static final int DEFAULT_CACHE_FLUSH_INTERVAL = 3600000;
+
   /**
    * Close down this HRegion.  Flush the cache unless abort parameter is true,
    * Shut down each HStore, don't service any more calls.
@@ -1474,6 +1483,29 @@ public boolean flushcache() throws IOException {
     }
   }
 
+  /**
+   * Should the memstore be flushed now
+   */
+  boolean shouldFlush() {
+    if (flushCheckInterval <= 0) { //disabled
+      return false;
+    }
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+    //if we flushed in the recent past, we don't need to do again now
+    if ((now - getLastFlushTime() < flushCheckInterval)) {
+      return false;
+    }
+    //since we didn't flush in the recent past, flush now if certain conditions
+    //are met. Return true on first such memstore hit.
+    for (Store s : this.getStores().values()) {
+      if (s.timeOfOldestEdit() < now - flushCheckInterval) {
+        // we have an old enough edit in the memstore, flush
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Flush the memstore.
    *
@@ -5421,7 +5453,7 @@ private void checkFamily(final byte [] family)
       ClassSize.OBJECT +
       ClassSize.ARRAY +
       36 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
-      (7 * Bytes.SIZEOF_LONG) +
+      (8 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index cf47aa85ed25..d1e55da12d10 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -31,7 +31,6 @@
 import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -210,7 +209,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   private HFileSystem fs;
   private boolean useHBaseChecksum; // verify hbase checksums?
   private Path rootDir;
-  private final Random rand = new Random();
+  private final Random rand;
 
   //RegionName vs current action in progress
   //true - if open region action in progress
@@ -291,6 +290,11 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
    */
   Chore compactionChecker;
 
+  /*
+   * Check for flushes
+   */
+  Chore periodicFlusher;
+
   // HLog and HLog roller. log is protected rather than private to avoid
   // eclipse warning when accessed by inner classes
   protected volatile HLog hlog;
@@ -443,6 +447,8 @@ public HRegionServer(Configuration conf)
     if (initialIsa.getAddress() == null) {
       throw new IllegalArgumentException("Failed resolve of " + initialIsa);
     }
+
+    this.rand = new Random(initialIsa.hashCode());
     this.rpcServer = HBaseRPC.getServer(this,
       new Class<?>[]{HRegionInterface.class, HBaseRPCErrorHandler.class,
         OnlineRegions.class},
@@ -697,6 +703,8 @@ private void initializeThreads() throws IOException {
     this.compactionChecker = new CompactionChecker(this,
       this.threadWakeFrequency * multiplier, this);
 
+    this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
+
     // Health checker thread.
     int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
       HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
@@ -1348,6 +1356,36 @@ protected void chore() {
     }
   }
 
+  class PeriodicMemstoreFlusher extends Chore {
+    final HRegionServer server;
+    final static int RANGE_OF_DELAY = 20000; //millisec
+    final static int MIN_DELAY_TIME = 3000; //millisec
+    public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) {
+      super(server.getServerName() + "-MemstoreFlusherChore", cacheFlushInterval, server);
+      this.server = server;
+    }
+
+    @Override
+    protected void chore() {
+      for (HRegion r : this.server.onlineRegions.values()) {
+        if (r == null)
+          continue;
+        if (r.shouldFlush()) {
+          FlushRequester requester = server.getFlushRequester();
+          if (requester != null) {
+            long randomDelay = rand.nextInt(RANGE_OF_DELAY) + MIN_DELAY_TIME;
+            LOG.info(getName() + " requesting flush for region " + r.getRegionNameAsString() + 
+                " after a delay of " + randomDelay);
+            //Throttle the flushes by putting a delay. If we don't throttle, and there
+            //is a balanced write-load on the regions in a table, we might end up 
+            //overwhelming the filesystem with too many flushes at once.
+            requester.requestDelayedFlush(r, randomDelay);
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Report the status of the server. A server is online once all the startup is
    * completed (setting up filesystem, starting service threads, etc.). This
@@ -1659,6 +1697,8 @@ private void startServiceThreads() throws IOException {
         uncaughtExceptionHandler);
     Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), n +
       ".compactionChecker", uncaughtExceptionHandler);
+    Threads.setDaemonThreadRunning(this.periodicFlusher.getThread(), n +
+        ".periodicFlusher", uncaughtExceptionHandler);
     if (this.healthCheckChore != null) {
       Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker",
           uncaughtExceptionHandler);
@@ -1739,7 +1779,8 @@ private boolean isHealthy() {
     // Verify that all threads are alive
     if (!(leases.isAlive()
         && cacheFlusher.isAlive() && hlogRoller.isAlive()
-        && this.compactionChecker.isAlive())) {
+        && this.compactionChecker.isAlive())
+        && this.periodicFlusher.isAlive()) {
       stop("One or more threads are no longer alive -- stop");
       return false;
     }
@@ -1916,6 +1957,7 @@ protected void kill() {
    */
   protected void join() {
     Threads.shutdown(this.compactionChecker.getThread());
+    Threads.shutdown(this.periodicFlusher.getThread());
     Threads.shutdown(this.cacheFlusher.getThread());
     if (this.healthCheckChore != null) {
       Threads.shutdown(this.healthCheckChore.getThread());
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 2f543739b9ae..113af807dd8d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -43,6 +43,7 @@
 import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 /**
  * The MemStore holds in-memory modifications to the Store.  Modifications
@@ -87,6 +88,9 @@ public class MemStore implements HeapSize {
   // Used to track own heapSize
   final AtomicLong size;
 
+  // Used to track when to flush
+  volatile long timeOfOldestEdit = Long.MAX_VALUE;
+
   TimeRangeTracker timeRangeTracker;
   TimeRangeTracker snapshotTimeRangeTracker;
 
@@ -158,6 +162,7 @@ void snapshot() {
           if (allocator != null) {
             this.allocator = new MemStoreLAB(conf);
           }
+          timeOfOldestEdit = Long.MAX_VALUE;
         }
       }
     } finally {
@@ -217,6 +222,28 @@ long add(final KeyValue kv) {
     }
   }
 
+  long timeOfOldestEdit() {
+    return timeOfOldestEdit;
+  }
+
+  private boolean addToKVSet(KeyValue e) {
+    boolean b = this.kvset.add(e);
+    setOldestEditTimeToNow();
+    return b;
+  }
+
+  private boolean removeFromKVSet(KeyValue e) {
+    boolean b = this.kvset.remove(e);
+    setOldestEditTimeToNow();
+    return b;
+  }
+
+  void setOldestEditTimeToNow() {
+    if (timeOfOldestEdit == Long.MAX_VALUE) {
+      timeOfOldestEdit = EnvironmentEdgeManager.currentTimeMillis();
+    }
+  }
+
   /**
    * Internal version of add() that doesn't clone KVs with the
    * allocator, and doesn't take the lock.
@@ -224,7 +251,7 @@ long add(final KeyValue kv) {
    * Callers should ensure they already have the read lock taken
    */
   private long internalAdd(final KeyValue toAdd) {
-    long s = heapSizeChange(toAdd, this.kvset.add(toAdd));
+    long s = heapSizeChange(toAdd, addToKVSet(toAdd));
     timeRangeTracker.includeTimestamp(toAdd);
     this.size.addAndGet(s);
     return s;
@@ -272,7 +299,7 @@ void rollback(final KeyValue kv) {
       // If the key is in the memstore, delete it. Update this.size.
       found = this.kvset.get(kv);
       if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
-        this.kvset.remove(kv);
+        removeFromKVSet(kv);
         long s = heapSizeChange(kv, true);
         this.size.addAndGet(-s);
       }
@@ -291,7 +318,7 @@ long delete(final KeyValue delete) {
     this.lock.readLock().lock();
     try {
       KeyValue toAdd = maybeCloneWithAllocator(delete);
-      s += heapSizeChange(toAdd, this.kvset.add(toAdd));
+      s += heapSizeChange(toAdd, addToKVSet(toAdd));
       timeRangeTracker.includeTimestamp(toAdd);
     } finally {
       this.lock.readLock().unlock();
@@ -588,6 +615,7 @@ private long upsert(KeyValue kv) {
           addedSize -= delta;
           this.size.addAndGet(-delta);
           it.remove();
+          setOldestEditTimeToNow();
         }
       } else {
         // past the column, done
@@ -899,7 +927,7 @@ public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns,
   }
 
   public final static long FIXED_OVERHEAD = ClassSize.align(
-      ClassSize.OBJECT + (11 * ClassSize.REFERENCE));
+      ClassSize.OBJECT + (11 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index a4a564e5f128..7f704d8916c1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -322,6 +322,18 @@ public void requestFlush(HRegion r) {
     }
   }
 
+  public void requestDelayedFlush(HRegion r, long delay) {
+    synchronized (regionsInQueue) {
+      if (!regionsInQueue.containsKey(r)) {
+        // This entry has some delay
+        FlushRegionEntry fqe = new FlushRegionEntry(r);
+        fqe.requeue(delay);
+        this.regionsInQueue.put(r, fqe);
+        this.flushQueue.add(fqe);
+      }
+    }
+  }
+
   public int getFlushQueueSize() {
     return flushQueue.size();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 6b2f800727d0..8542392b7120 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -512,6 +512,13 @@ protected long add(final KeyValue kv) {
     }
   }
 
+  /**
+   * When was the oldest edit done in the memstore
+   */
+  public long timeOfOldestEdit() {
+    return memstore.timeOfOldestEdit();
+  }
+
   /**
    * Adds a value to the memstore
    *
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index ea1db8a50284..d59457ac7230 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -352,6 +352,14 @@
     hbase.server.thread.wakefrequency milliseconds.
     </description>
   </property>
+  <property>
+    <name>hbase.regionserver.optionalcacheflushinterval</name>
+    <value>3600000</value>
+    <description>
+    Maximum amount of time an edit lives in memory before being automatically flushed.
+    Default 1 hour. Set it to 0 to disable automatic flushing.
+    </description>
+  </property>
   <property>
     <name>hbase.hregion.memstore.flush.size</name>
     <value>134217728</value>
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
index 62245f0c01de..ba89b7c09d89 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
@@ -27,6 +27,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
 import junit.framework.TestCase;
@@ -39,6 +40,8 @@
 import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdge;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
@@ -870,6 +873,99 @@ public void testUpsertMemstoreSize() throws Exception {
     assertEquals(newSize, this.memstore.size.get());
   }
 
+  ////////////////////////////////////
+  // Test for periodic memstore flushes 
+  // based on time of oldest edit
+  ////////////////////////////////////
+
+  /**
+   * Tests that the timeOfOldestEdit is updated correctly for the 
+   * various edit operations in memstore.
+   * @throws Exception
+   */
+  public void testUpdateToTimeOfOldestEdit() throws Exception {
+    try {
+      EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
+      EnvironmentEdgeManager.injectEdge(edge);
+      MemStore memstore = new MemStore();
+      long t = memstore.timeOfOldestEdit();
+      assertEquals(t, Long.MAX_VALUE);
+
+      // test the case that the timeOfOldestEdit is updated after a KV add
+      memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
+      t = memstore.timeOfOldestEdit();
+      assertTrue(t == 1234);
+      // snapshot() will reset timeOfOldestEdit. The method will also assert the 
+      // value is reset to Long.MAX_VALUE
+      t = runSnapshot(memstore);
+
+      // test the case that the timeOfOldestEdit is updated after a KV delete
+      memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
+      t = memstore.timeOfOldestEdit();
+      assertTrue(t == 1234);
+      t = runSnapshot(memstore);
+
+      // test the case that the timeOfOldestEdit is updated after a KV upsert
+      List<KeyValue> l = new ArrayList<KeyValue>();
+      KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
+      l.add(kv1);
+      memstore.upsert(l);
+      t = memstore.timeOfOldestEdit();
+      assertTrue(t == 1234);
+    } finally {
+      EnvironmentEdgeManager.reset();
+    }
+  }
+
+  /**
+   * Tests the HRegion.shouldFlush method - adds an edit in the memstore
+   * and checks that shouldFlush returns true, and another where it disables
+   * the periodic flush functionality and tests whether shouldFlush returns
+   * false. 
+   * @throws Exception
+   */
+  public void testShouldFlush() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
+    checkShouldFlush(conf, true);
+    // test disable flush
+    conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
+    checkShouldFlush(conf, false);
+  }
+
+  private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
+    try {
+      EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
+      EnvironmentEdgeManager.injectEdge(edge);
+      HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
+      HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
+
+      Map<byte[], Store> stores = region.getStores();
+      assertTrue(stores.size() == 1);
+
+      Store s = stores.entrySet().iterator().next().getValue();
+      edge.setCurrentTimeMillis(1234);
+      s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
+      edge.setCurrentTimeMillis(1234 + 100);
+      assertTrue(region.shouldFlush() == false);
+      edge.setCurrentTimeMillis(1234 + 10000);
+      assertTrue(region.shouldFlush() == expected);
+    } finally {
+      EnvironmentEdgeManager.reset();
+    }
+  }
+
+  private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
+    long t = 1234;
+    @Override
+    public long currentTimeMillis() {
+      return t; 
+    }
+    public void setCurrentTimeMillis(long t) {
+      this.t = t;
+    }
+  }
+
   /**   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
    * @param hmc Instance to add rows to.
    * @return How many rows we added.
@@ -898,14 +994,17 @@ private int addRows(final MemStore hmc, final long ts) {
     return ROW_COUNT;
   }
 
-  private void runSnapshot(final MemStore hmc) throws UnexpectedException {
+  private long runSnapshot(final MemStore hmc) throws UnexpectedException {
     // Save off old state.
     int oldHistorySize = hmc.getSnapshot().size();
     hmc.snapshot();
     KeyValueSkipListSet ss = hmc.getSnapshot();
     // Make some assertions about what just happened.
     assertTrue("History size has not increased", oldHistorySize < ss.size());
+    long t = memstore.timeOfOldestEdit();
+    assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
     hmc.clearSnapshot(ss);
+    return t;
   }
 
   private void isExpectedRowWithoutTimestamps(final int rowIndex,
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 9d4e7a7e10f7..8b1f3b75c2ee 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -762,6 +762,12 @@ public void requestFlush(HRegion region) {
         throw new RuntimeException("Exception flushing", e);
       }
     }
+
+    @Override
+    public void requestDelayedFlush(HRegion region, long when) {
+      // TODO Auto-generated method stub
+      
+    }
   }
 
   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
diff --git a/src/test/resources/hbase-site.xml b/src/test/resources/hbase-site.xml
index 84b561265927..b9a446862210 100644
--- a/src/test/resources/hbase-site.xml
+++ b/src/test/resources/hbase-site.xml
@@ -96,14 +96,6 @@
     the master will notice a dead region server sooner. The default is 15 seconds.
     </description>
   </property>
-  <property>
-    <name>hbase.regionserver.optionalcacheflushinterval</name>
-    <value>1000</value>
-    <description>
-    Amount of time to wait since the last time a region was flushed before
-    invoking an optional cache flush. Default 60,000.
-    </description>
-  </property>
   <property>
     <name>hbase.regionserver.safemode</name>
     <value>false</value>

From 538c7a7b2748ea2a2e9c355c7eecde5b1321ea96 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 15 May 2013 19:42:48 +0000
Subject: [PATCH 0977/1540] HBASE-8493 Backport HBASE-8422, 'Master won't go
 down', to 0.94 (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483045 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  2 +-
 .../apache/hadoop/hbase/master/HMaster.java   | 29 +++++++++++--
 .../hbase/master/TestMasterShutdown.java      | 41 +++++++++++++++++++
 3 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 32ccfff7f465..4d57a235b9d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2318,7 +2318,7 @@ private boolean isSplitOrSplitting(final String path) throws KeeperException {
   public void waitForAssignment(HRegionInfo regionInfo)
   throws InterruptedException {
     synchronized(regions) {
-      while(!regions.containsKey(regionInfo)) {
+      while (!this.master.isStopped() && !regions.containsKey(regionInfo)) {
         // We should receive a notification, but it's
         //  better to have a timeout to recheck the condition here:
         //  it lowers the impact of a race condition if any
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 7dc1693221db..32e87d2891b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -606,7 +606,7 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
 
     this.initializationBeforeMetaAssignment = true;
     // Make sure root assigned before proceeding.
-    assignRoot(status);
+    if (!assignRoot(status)) return;
 
     // SSH should enabled for ROOT before META region assignment
     // because META region assignment is depending on ROOT server online.
@@ -621,7 +621,7 @@ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
     }
 
     // Make sure meta assigned before proceeding.
-    assignMeta(status, ((masterRecovery) ? null : preMetaServer), preRootServer);
+    if (!assignMeta(status, ((masterRecovery) ? null : preMetaServer), preRootServer)) return;
 
     enableServerShutdownHandler();
 
@@ -708,7 +708,7 @@ protected void startCatalogJanitorChore() {
    * @throws IOException
    * @throws KeeperException
    */
-  private void assignRoot(MonitoredTask status)
+  private boolean assignRoot(MonitoredTask status)
   throws InterruptedException, IOException, KeeperException {
     int assigned = 0;
     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
@@ -724,9 +724,15 @@ private void assignRoot(MonitoredTask status)
       splitLogAndExpireIfOnline(currentRootServer);
       this.assignmentManager.assignRoot();
       waitForRootAssignment();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.ROOT_REGIONINFO) || this.stopped) {
+        return false;
+      }
       assigned++;
     } else if (rit && !rootRegionLocation) {
       waitForRootAssignment();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.ROOT_REGIONINFO) || this.stopped) {
+        return false;
+      }
       assigned++;
     } else {
       // Region already assigned. We didn't assign it. Add to in-memory state.
@@ -740,6 +746,7 @@ private void assignRoot(MonitoredTask status)
       ", location=" + catalogTracker.getRootLocation());
 
     status.setStatus("ROOT assigned.");
+    return true;
   }
 
   /**
@@ -751,7 +758,7 @@ private void assignRoot(MonitoredTask status)
    * @throws IOException
    * @throws KeeperException
    */
-  private void assignMeta(MonitoredTask status, ServerName previousMetaServer,
+  private boolean assignMeta(MonitoredTask status, ServerName previousMetaServer,
       ServerName previousRootServer)
       throws InterruptedException,
       IOException, KeeperException {
@@ -775,9 +782,17 @@ private void assignMeta(MonitoredTask status, ServerName previousMetaServer,
       }
       assignmentManager.assignMeta();
       enableSSHandWaitForMeta();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.FIRST_META_REGIONINFO)
+          || this.stopped) {
+        return false;
+      }
       assigned++;
     } else if (rit && !metaRegionLocation) {
       enableSSHandWaitForMeta();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.FIRST_META_REGIONINFO)
+          || this.stopped) {
+        return false;
+      }
       assigned++;
     } else {
       // Region already assigned. We didnt' assign it. Add to in-memory state.
@@ -788,6 +803,7 @@ private void assignMeta(MonitoredTask status, ServerName previousMetaServer,
     LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
         + catalogTracker.getMetaLocation());
     status.setStatus("META assigned.");
+    return true;
   }
 
   private void enableSSHandWaitForMeta() throws IOException,
@@ -1765,6 +1781,11 @@ public void stop(final String why) {
         this.activeMasterManager.clusterHasActiveMaster.notifyAll();
       }
     }
+    // If no region server is online then master may stuck waiting on -ROOT- and .META. to come on
+    // line. See HBASE-8422.
+    if (this.catalogTracker != null && this.serverManager.getOnlineServers().isEmpty()) {
+      this.catalogTracker.stop();
+    }
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
index 436b2c9eab48..01e6bd8e118f 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
@@ -92,6 +92,47 @@ public void testMasterShutdown() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
+  @Test//(timeout = 180000)
+  public void testMasterShutdownBeforeStartingAnyRegionServer() throws Exception {
+
+    final int NUM_MASTERS = 1;
+    final int NUM_RS = 0;
+
+    // Create config to use for this cluster
+    Configuration conf = HBaseConfiguration.create();
+
+    // Start the cluster
+    final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+    TEST_UTIL.startMiniDFSCluster(1);
+    TEST_UTIL.startMiniZKCluster();
+    TEST_UTIL.createRootDir();
+    final LocalHBaseCluster cluster =
+        new LocalHBaseCluster(conf, NUM_MASTERS, NUM_RS, HMaster.class,
+            MiniHBaseCluster.MiniHBaseClusterRegionServer.class);
+    final MasterThread master = cluster.getMasters().get(0);
+    master.start();
+    Thread shutdownThread = new Thread() {
+      public void run() {
+        try {
+          TEST_UTIL.getHBaseAdmin().shutdown();
+          cluster.waitOnMaster(0);
+        } catch (Exception e) {
+        }
+      };
+    };
+    shutdownThread.start();
+    master.join();
+    shutdownThread.join();
+
+    List<MasterThread> masterThreads = cluster.getMasters();
+    // make sure all the masters properly shutdown
+    assertEquals(0, masterThreads.size());
+
+    TEST_UTIL.shutdownMiniZKCluster();
+    TEST_UTIL.cleanupTestDir();
+    TEST_UTIL.shutdownMiniDFSCluster();
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 0e13da6138998664b500317ce63764ea197f3df4 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 15 May 2013 19:42:59 +0000
Subject: [PATCH 0978/1540] HBASE-8540 SnapshotFileCache logs too many times if
 snapshot dir doesn't exists

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483046 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/snapshot/SnapshotFileCache.java   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index 0e998f706463..13857416bcd1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -188,7 +188,9 @@ private synchronized void refreshCache() throws IOException {
     try {
       status = fs.getFileStatus(snapshotDir);
     } catch (FileNotFoundException e) {
-      LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
+      if (this.cache.size() > 0) {
+        LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
+      }
       return;
     }
     // if the snapshot directory wasn't modified since we last check, we are done
@@ -210,7 +212,9 @@ private synchronized void refreshCache() throws IOException {
     FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
     if (snapshots == null) {
       // remove all the remembered snapshots because we don't have any left
-      LOG.debug("No snapshots on-disk, cache empty");
+      if (LOG.isDebugEnabled() && this.snapshots.size() > 0) {
+        LOG.debug("No snapshots on-disk, cache empty");
+      }
       this.snapshots.clear();
       return;
     }

From c93ee447020e38d3d5118df12c7e16bdf2a5adbd Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 15 May 2013 21:46:53 +0000
Subject: [PATCH 0979/1540] HBASE-8355:
 BaseRegionObserver#pre(Compact|Flush|Store)ScannerOpen returns null (Andrew
 Purtell, Jesse Yates)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483103 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/BaseRegionObserver.java |   6 +-
 .../TestRegionObserverScannerOpenHook.java    | 309 ++++++++++++++++++
 2 files changed, 312 insertions(+), 3 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 81c590f380e7..b98bba9c10d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -79,7 +79,7 @@ public void postClose(ObserverContext<RegionCoprocessorEnvironment> e,
   public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final KeyValueScanner memstoreScanner, final InternalScanner s)
       throws IOException {
-    return null;
+    return s;
   }
 
   @Override
@@ -148,7 +148,7 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
   public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
       final long earliestPutTs, final InternalScanner s) throws IOException {
-    return null;
+    return s;
   }
 
   @Override
@@ -315,7 +315,7 @@ public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvir
   public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
       final KeyValueScanner s) throws IOException {
-    return null;
+    return s;
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
new file mode 100644
index 000000000000..a1c11950ab7f
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -0,0 +1,309 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestRegionObserverScannerOpenHook {
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  static final Path DIR = UTIL.getDataTestDir();
+
+  public static class NoDataFilter extends FilterBase {
+
+    @Override
+    public ReturnCode filterKeyValue(KeyValue ignored) {
+      return ReturnCode.SKIP;
+    }
+
+    @Override
+    public boolean filterAllRemaining() {
+      return true;
+    }
+
+    @Override
+    public boolean filterRow() {
+      return true;
+    }
+
+    @Override
+    public void readFields(DataInput arg0) throws IOException {
+      // noop
+    }
+
+    @Override
+    public void write(DataOutput arg0) throws IOException {
+      // noop
+    }
+  }
+
+  /**
+   * Do the same logic as the {@link BaseRegionObserver}. Needed since {@link BaseRegionObserver} is
+   * an abstract class.
+   */
+  public static class EmptyRegionObsever extends BaseRegionObserver {
+  }
+
+  /**
+   * Don't return any data from a scan by creating a custom {@link StoreScanner}.
+   */
+  public static class NoDataFromScan extends BaseRegionObserver {
+    @Override
+    public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
+        Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s)
+        throws IOException {
+      scan.setFilter(new NoDataFilter());
+      return new StoreScanner(store, store.getScanInfo(), scan, targetCols);
+    }
+  }
+
+  /**
+   * Don't allow any data in a flush by creating a custom {@link StoreScanner}.
+   */
+  public static class NoDataFromFlush extends BaseRegionObserver {
+    @Override
+    public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
+        Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
+      Scan scan = new Scan();
+      scan.setFilter(new NoDataFilter());
+      return new StoreScanner(store, store.getScanInfo(), scan,
+          Collections.singletonList(memstoreScanner), ScanType.MINOR_COMPACT, store.getHRegion()
+              .getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
+    }
+  }
+
+  /**
+   * Don't allow any data to be written out in the compaction by creating a custom
+   * {@link StoreScanner}.
+   */
+  public static class NoDataFromCompaction extends BaseRegionObserver {
+    @Override
+    public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
+        Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
+        long earliestPutTs, InternalScanner s) throws IOException {
+      Scan scan = new Scan();
+      scan.setFilter(new NoDataFilter());
+      return new StoreScanner(store, store.getScanInfo(), scan, scanners, ScanType.MINOR_COMPACT,
+          store.getHRegion().getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
+    }
+  }
+
+  HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
+      byte[]... families) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (byte[] family : families) {
+      htd.addFamily(new HColumnDescriptor(family));
+    }
+    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+    Path path = new Path(DIR + callingMethod);
+    HRegion r = HRegion.createHRegion(info, path, conf, htd);
+    // this following piece is a hack. currently a coprocessorHost
+    // is secretly loaded at OpenRegionHandler. we don't really
+    // start a region server here, so just manually create cphost
+    // and set it to region.
+    RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
+    r.setCoprocessorHost(host);
+    return r;
+  }
+
+  @Test
+  public void testRegionObserverScanTimeStacking() throws Exception {
+    byte[] ROW = Bytes.toBytes("testRow");
+    byte[] TABLE = Bytes.toBytes(getClass().getName());
+    byte[] A = Bytes.toBytes("A");
+    byte[][] FAMILIES = new byte[][] { A };
+
+    Configuration conf = HBaseConfiguration.create();
+    HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
+    RegionCoprocessorHost h = region.getCoprocessorHost();
+    h.load(NoDataFromScan.class, Coprocessor.PRIORITY_HIGHEST, conf);
+    h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
+
+    Put put = new Put(ROW);
+    put.add(A, A, A);
+    region.put(put);
+
+    Get get = new Get(ROW);
+    Result r = region.get(get);
+    assertNull(
+      "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
+          + r, r.list());
+  }
+
+  @Test
+  public void testRegionObserverFlushTimeStacking() throws Exception {
+    byte[] ROW = Bytes.toBytes("testRow");
+    byte[] TABLE = Bytes.toBytes(getClass().getName());
+    byte[] A = Bytes.toBytes("A");
+    byte[][] FAMILIES = new byte[][] { A };
+
+    Configuration conf = HBaseConfiguration.create();
+    HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
+    RegionCoprocessorHost h = region.getCoprocessorHost();
+    h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf);
+    h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
+
+    // put a row and flush it to disk
+    Put put = new Put(ROW);
+    put.add(A, A, A);
+    region.put(put);
+    region.flushcache();
+    Get get = new Get(ROW);
+    Result r = region.get(get);
+    assertNull(
+      "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
+          + r, r.list());
+  }
+
+  /**
+   * Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do
+   * the usual compaction mechanism on the region, rather than going through the backdoor to the
+   * region
+   */
+  @Test
+  @Category(MediumTests.class)
+  public void testRegionObserverCompactionTimeStacking() throws Exception {
+    // setup a mini cluster so we can do a real compaction on a region
+    Configuration conf = UTIL.getConfiguration();
+    conf.setInt("hbase.hstore.compaction.min", 2);
+    UTIL.startMiniCluster();
+    String tableName = "testRegionObserverCompactionTimeStacking";
+    byte[] ROW = Bytes.toBytes("testRow");
+    byte[] A = Bytes.toBytes("A");
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor(A));
+    desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null);
+    desc.addCoprocessor(NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST,
+      null);
+
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    admin.createTable(desc);
+
+    HTable table = new HTable(conf, desc.getName());
+
+    // put a row and flush it to disk
+    Put put = new Put(ROW);
+    put.add(A, A, A);
+    table.put(put);
+    table.flushCommits();
+
+    HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getName());
+    List<HRegion> regions = rs.getOnlineRegions(desc.getName());
+    assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
+    HRegion region = regions.get(0);
+    admin.flush(region.getRegionName());
+
+    // put another row and flush that too
+    put = new Put(Bytes.toBytes("anotherrow"));
+    put.add(A, A, A);
+    table.put(put);
+    table.flushCommits();
+    admin.flush(region.getRegionName());
+
+    // run a compaction, which normally would should get rid of the data
+    Store s = region.getStores().get(A);
+    CountDownLatch latch = new CountDownLatch(1);
+    WaitableCompactionRequest request = new WaitableCompactionRequest(region, s, latch);
+    rs.compactSplitThread.requestCompaction(region, s,
+      "compact for testRegionObserverCompactionTimeStacking", Store.PRIORITY_USER, request);
+    // wait for the compaction to complete
+    latch.await();
+
+    // check both rows to ensure that they aren't there
+    Get get = new Get(ROW);
+    Result r = table.get(get);
+    assertNull(
+      "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
+          + r, r.list());
+
+    get = new Get(Bytes.toBytes("anotherrow"));
+    r = table.get(get);
+    assertNull(
+      "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: "
+          + r, r.list());
+
+    table.close();
+    UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * A simple compaction on which you can wait for the passed in latch until the compaction finishes
+   * (either successfully or if it failed).
+   */
+  public static class WaitableCompactionRequest extends CompactionRequest {
+    private CountDownLatch done;
+
+    /**
+     * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+     * compaction before being used.
+     */
+    public WaitableCompactionRequest(HRegion region, Store store, CountDownLatch finished) {
+      super(region, store, Store.PRIORITY_USER);
+      this.done = finished;
+    }
+
+    @Override
+    public void finishRequest() {
+      super.finishRequest();
+      this.done.countDown();
+    }
+  }
+}
\ No newline at end of file

From ab26ead51a01dd374cc142fd87d6e0eeeda3e803 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 16 May 2013 18:06:26 +0000
Subject: [PATCH 0980/1540] HBASE-8505 References to split daughters should not
 be deleted separately from parent META entry

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483485 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/MetaEditor.java      |  20 --
 .../hadoop/hbase/client/MetaScanner.java      |  50 +++--
 .../hadoop/hbase/master/CatalogJanitor.java   |  40 ++--
 .../org/apache/hadoop/hbase/util/Bytes.java   |  33 +++-
 .../hadoop/hbase/client/TestMetaScanner.java  | 184 ++++++++++++++++--
 .../hbase/master/TestCatalogJanitor.java      |  91 ++++++++-
 6 files changed, 322 insertions(+), 96 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
index c7ce68433f31..94ffeb605092 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -21,7 +21,6 @@
 import java.io.InterruptedIOException;
 import java.net.ConnectException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -358,25 +357,6 @@ public static void mutateRegions(CatalogTracker catalogTracker,
     }
   }
 
-  /**
-   * Deletes daughters references in offlined split parent.
-   * @param catalogTracker
-   * @param parent Parent row we're to remove daughter reference from
-   * @throws NotAllMetaRegionsOnlineException
-   * @throws IOException
-   */
-  public static void deleteDaughtersReferencesInParent(CatalogTracker catalogTracker,
-      final HRegionInfo parent)
-  throws NotAllMetaRegionsOnlineException, IOException {
-    Delete delete = new Delete(parent.getRegionName());
-    delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
-    delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
-    deleteFromMetaTable(catalogTracker, delete);
-    LOG.info("Deleted daughters references, qualifier=" + Bytes.toStringBinary(HConstants.SPLITA_QUALIFIER) +
-      " and qualifier=" + Bytes.toStringBinary(HConstants.SPLITB_QUALIFIER) +
-      ", from parent " + parent.getRegionNameAsString());
-  }
-
   public static HRegionInfo getHRegionInfo(
       Result data) throws IOException {
     byte [] bytes =
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 1f1bc2b6d543..80531cc2251d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -403,32 +403,36 @@ public boolean processRow(Result rowResult) throws IOException {
          * seen by this scanner as well, so we block until they are added to the META table. Even
          * though we are waiting for META entries, ACID semantics in HBase indicates that this
          * scanner might not see the new rows. So we manually query the daughter rows */
-        HRegionInfo splitA = Writables.getHRegionInfo(rowResult.getValue(HConstants.CATALOG_FAMILY,
+        HRegionInfo splitA = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY,
             HConstants.SPLITA_QUALIFIER));
-        HRegionInfo splitB = Writables.getHRegionInfo(rowResult.getValue(HConstants.CATALOG_FAMILY,
+        HRegionInfo splitB = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY,
             HConstants.SPLITB_QUALIFIER));
 
         HTable metaTable = getMetaTable();
         long start = System.currentTimeMillis();
-        Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
-            splitA.getRegionName());
-        if (resultA != null) {
-          processRow(resultA);
-          daughterRegions.add(splitA.getRegionName());
-        } else {
-          throw new RegionOfflineException("Split daughter region " +
-              splitA.getRegionNameAsString() + " cannot be found in META.");
+        if (splitA != null) {
+          Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
+              splitA.getRegionName());
+          if (resultA != null) {
+            processRow(resultA);
+            daughterRegions.add(splitA.getRegionName());
+          } else {
+            throw new RegionOfflineException("Split daughter region " +
+                splitA.getRegionNameAsString() + " cannot be found in META.");
+          }
         }
         long rem = blockingTimeout - (System.currentTimeMillis() - start);
 
-        Result resultB = getRegionResultBlocking(metaTable, rem,
-            splitB.getRegionName());
-        if (resultB != null) {
-          processRow(resultB);
-          daughterRegions.add(splitB.getRegionName());
-        } else {
-          throw new RegionOfflineException("Split daughter region " +
-              splitB.getRegionNameAsString() + " cannot be found in META.");
+        if (splitB != null) {
+          Result resultB = getRegionResultBlocking(metaTable, rem,
+              splitB.getRegionName());
+          if (resultB != null) {
+            processRow(resultB);
+            daughterRegions.add(splitB.getRegionName());
+          } else {
+            throw new RegionOfflineException("Split daughter region " +
+                splitB.getRegionNameAsString() + " cannot be found in META.");
+          }
         }
       }
 
@@ -437,9 +441,7 @@ public boolean processRow(Result rowResult) throws IOException {
 
     private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] regionName)
         throws IOException {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName));
-      }
+      boolean logged = false;
       long start = System.currentTimeMillis();
       while (System.currentTimeMillis() - start < timeout) {
         Get get = new Get(regionName);
@@ -450,6 +452,12 @@ private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] re
           return result;
         }
         try {
+          if (!logged) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName));
+            }
+            logged = true;
+          }
           Thread.sleep(10);
         } catch (InterruptedException ex) {
           Thread.currentThread().interrupt();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 94ff5bc3f8b8..ef6dad0e501a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -41,7 +41,8 @@
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
-import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -107,9 +108,10 @@ Pair<Integer, Map<HRegionInfo, Result>> getSplitParents() throws IOException {
     final Map<HRegionInfo, Result> splitParents =
       new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
     // This visitor collects split parents and counts rows in the .META. table
-    MetaReader.Visitor visitor = new MetaReader.Visitor() {
+
+    MetaScannerVisitor visitor = new MetaScanner.BlockingMetaScannerVisitor(server.getConfiguration()) {
       @Override
-      public boolean visit(Result r) throws IOException {
+      public boolean processRowInternal(Result r) throws IOException {
         if (r == null || r.isEmpty()) return true;
         count.incrementAndGet();
         HRegionInfo info = getHRegionInfo(r);
@@ -119,8 +121,9 @@ public boolean visit(Result r) throws IOException {
         return true;
       }
     };
+
     // Run full scan of .META. catalog table passing in our custom visitor
-    MetaReader.fullScan(this.server.getCatalogTracker(), visitor);
+    MetaScanner.metaScan(server.getConfiguration(), visitor);
 
     return new Pair<Integer, Map<HRegionInfo, Result>>(count.get(), splitParents);
   }
@@ -164,6 +167,7 @@ int scan() throws IOException {
    * daughters.
    */
   static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
+    Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
     @Override
     public int compare(HRegionInfo left, HRegionInfo right) {
       // This comparator differs from the one HRegionInfo in that it sorts
@@ -178,19 +182,9 @@ public int compare(HRegionInfo left, HRegionInfo right) {
       result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
       if (result != 0) return result;
       // Compare end keys.
-      result = Bytes.compareTo(left.getEndKey(), right.getEndKey());
-      if (result != 0) {
-        if (left.getStartKey().length != 0
-                && left.getEndKey().length == 0) {
-            return -1;  // left is last region
-        }
-        if (right.getStartKey().length != 0
-                && right.getEndKey().length == 0) {
-            return 1;  // right is the last region
-        }
-        return -result; // Flip the result so parent comes first.
-      }
-      return result;
+      result = rowEndKeyComparator.compare(left.getEndKey(), right.getEndKey());
+
+      return -result; // Flip the result so parent comes first.
     }
   }
 
@@ -235,8 +229,6 @@ boolean cleanParent(final HRegionInfo parent, Result rowContent)
     if (hasNoReferences(a) && hasNoReferences(b)) {
       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
         " because daughter splits no longer hold references");
-      // wipe out daughter references from parent region in meta
-      removeDaughtersFromParent(parent);
 
       // This latter regionOffline should not be necessary but is done for now
       // until we let go of regionserver to master heartbeats.  See HBASE-3368.
@@ -278,16 +270,6 @@ private HRegionInfo getDaughterRegionInfo(final Result result,
     return Writables.getHRegionInfoOrNull(bytes);
   }
 
-  /**
-   * Remove mention of daughters from parent row.
-   * @param parent
-   * @throws IOException
-   */
-  private void removeDaughtersFromParent(final HRegionInfo parent)
-  throws IOException {
-    MetaEditor.deleteDaughtersReferencesInParent(this.server.getCatalogTracker(), parent);
-  }
-
   /**
    * Checks if a daughter region -- either splitA or splitB -- still holds
    * references to parent.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index 74b390a6b94f..6a4130865409 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -33,7 +33,6 @@
 import java.util.Comparator;
 import java.util.Iterator;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
@@ -123,6 +122,34 @@ public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
     }
   }
 
+  /**
+   * A {@link ByteArrayComparator} that treats the empty array as the largest value.
+   * This is useful for comparing row end keys for regions.
+   */
+  // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region
+  // boundaries. Thus semantically, we should treat empty byte array as the smallest value
+  // while comparing row keys, start keys etc; but as the largest value for comparing
+  // region boundaries for endKeys.
+  public static class RowEndKeyComparator extends ByteArrayComparator {
+    @Override
+    public int compare(byte[] left, byte[] right) {
+      return compare(left, 0, left.length, right, 0, right.length);
+    }
+    @Override
+    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+      if (b1 == b2 && s1 == s2 && l1 == l2) {
+        return 0;
+      }
+      if (l1 == 0) {
+        return l2; //0 or positive
+      }
+      if (l2 == 0) {
+        return -1;
+      }
+      return super.compare(b1, s1, l1, b2, s2, l2);
+    }
+  }
+
   /**
    * Pass this to TreeMaps where byte [] are keys.
    */
@@ -339,9 +366,9 @@ public static String toStringBinary(ByteBuffer buf) {
    */
   public static String toStringBinary(final byte [] b, int off, int len) {
     StringBuilder result = new StringBuilder();
-    for (int i = off; i < off + len ; ++i ) { 
+    for (int i = off; i < off + len ; ++i ) {
       int ch = b[i] & 0xFF;
-      if ( (ch >= '0' && ch <= '9')      
+      if ( (ch >= '0' && ch <= '9')
           || (ch >= 'A' && ch <= 'Z')
           || (ch >= 'a' && ch <= 'z')
           || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
index 55eac79beee8..914d7081bbcb 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
@@ -19,39 +19,55 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.Random;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.apache.hadoop.hbase.util.StoppableImplementation;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.util.StringUtils;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-
-import static org.mockito.Mockito.*;
-
 @Category(MediumTests.class)
 public class TestMetaScanner {
   final Log LOG = LogFactory.getLog(getClass());
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
+  public void setUp() throws Exception {
     TEST_UTIL.startMiniCluster(1);
   }
 
-  /**
-   * @throws java.lang.Exception
-   */
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
   @Test
   public void testMetaScanner() throws Exception {
     LOG.info("Starting testMetaScanner");
+    setUp();
     final byte[] TABLENAME = Bytes.toBytes("testMetaScanner");
     final byte[] FAMILY = Bytes.toBytes("family");
     TEST_UTIL.createTable(TABLENAME, FAMILY);
@@ -64,29 +80,29 @@ public void testMetaScanner() throws Exception {
           Bytes.toBytes("region_b")});
     // Make sure all the regions are deployed
     TEST_UTIL.countRows(table);
-    
-    MetaScanner.MetaScannerVisitor visitor = 
+
+    MetaScanner.MetaScannerVisitor visitor =
       mock(MetaScanner.MetaScannerVisitor.class);
     doReturn(true).when(visitor).processRow((Result)anyObject());
 
     // Scanning the entire table should give us three rows
     MetaScanner.metaScan(conf, visitor, TABLENAME);
     verify(visitor, times(3)).processRow((Result)anyObject());
-    
+
     // Scanning the table with a specified empty start row should also
     // give us three META rows
     reset(visitor);
     doReturn(true).when(visitor).processRow((Result)anyObject());
     MetaScanner.metaScan(conf, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
     verify(visitor, times(3)).processRow((Result)anyObject());
-    
+
     // Scanning the table starting in the middle should give us two rows:
     // region_a and region_b
     reset(visitor);
     doReturn(true).when(visitor).processRow((Result)anyObject());
     MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
     verify(visitor, times(2)).processRow((Result)anyObject());
-    
+
     // Scanning with a limit of 1 should only give us one row
     reset(visitor);
     doReturn(true).when(visitor).processRow((Result)anyObject());
@@ -95,6 +111,138 @@ public void testMetaScanner() throws Exception {
     table.close();
   }
 
+  @Test
+  public void testConcurrentMetaScannerAndCatalogJanitor() throws Throwable {
+    /* TEST PLAN: start with only one region in a table. Have a splitter
+     * thread  and metascanner threads that continously scan the meta table for regions.
+     * CatalogJanitor from master will run frequently to clean things up
+     */
+    TEST_UTIL.getConfiguration().setLong("hbase.catalogjanitor.interval", 500);
+    setUp();
+
+    final long runtime = 30 * 1000; //30 sec
+    LOG.info("Starting testConcurrentMetaScannerAndCatalogJanitor");
+    final byte[] TABLENAME = Bytes.toBytes("testConcurrentMetaScannerAndCatalogJanitor");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    TEST_UTIL.createTable(TABLENAME, FAMILY);
+    final CatalogTracker catalogTracker = mock(CatalogTracker.class);
+    when(catalogTracker.getConnection()).thenReturn(TEST_UTIL.getHBaseAdmin().getConnection());
+
+    class RegionMetaSplitter extends StoppableImplementation implements Runnable {
+      Random random = new Random();
+      Throwable ex = null;
+      @Override
+      public void run() {
+        while (!isStopped()) {
+          try {
+            List<HRegionInfo> regions = MetaScanner.listAllRegions(
+              TEST_UTIL.getConfiguration(), false);
+
+            //select a random region
+            HRegionInfo parent = regions.get(random.nextInt(regions.size()));
+            if (parent == null || !Bytes.equals(TABLENAME, parent.getTableName())) {
+              continue;
+            }
+
+            long startKey = 0, endKey = Long.MAX_VALUE;
+            byte[] start = parent.getStartKey();
+            byte[] end = parent.getEndKey();
+            if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
+              startKey = Bytes.toLong(parent.getStartKey());
+            }
+            if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
+              endKey = Bytes.toLong(parent.getEndKey());
+            }
+            if (startKey == endKey) {
+              continue;
+            }
+
+            long midKey = BigDecimal.valueOf(startKey).add(BigDecimal.valueOf(endKey))
+                .divideToIntegralValue(BigDecimal.valueOf(2)).longValue();
+
+            HRegionInfo splita = new HRegionInfo(TABLENAME,
+              start,
+              Bytes.toBytes(midKey));
+            HRegionInfo splitb = new HRegionInfo(TABLENAME,
+              Bytes.toBytes(midKey),
+              end);
+
+            MetaEditor.offlineParentInMeta(catalogTracker, parent, splita, splitb);
+            Threads.sleep(100);
+            MetaEditor.addDaughter(catalogTracker, splitb, null);
+            MetaEditor.addDaughter(catalogTracker, splita, null);
+
+            Threads.sleep(random.nextInt(200));
+          } catch (Throwable e) {
+            ex = e;
+            Assert.fail(StringUtils.stringifyException(e));
+          }
+        }
+      }
+      void rethrowExceptionIfAny() throws Throwable {
+        if (ex != null) { throw ex; }
+      }
+    }
+
+    class MetaScannerVerifier extends StoppableImplementation implements Runnable {
+      Random random = new Random();
+      Throwable ex = null;
+      @Override
+      public void run() {
+         while(!isStopped()) {
+           try {
+            NavigableMap<HRegionInfo, ServerName> regions =
+                MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), TABLENAME, false);
+
+            LOG.info("-------");
+            byte[] lastEndKey = HConstants.EMPTY_START_ROW;
+            for (HRegionInfo hri: regions.navigableKeySet()) {
+              long startKey = 0, endKey = Long.MAX_VALUE;
+              if (!Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())) {
+                startKey = Bytes.toLong(hri.getStartKey());
+              }
+              if (!Bytes.equals(HConstants.EMPTY_END_ROW, hri.getEndKey())) {
+                endKey = Bytes.toLong(hri.getEndKey());
+              }
+              LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri);
+              Assert.assertTrue(Bytes.equals(lastEndKey, hri.getStartKey()));
+              lastEndKey = hri.getEndKey();
+            }
+            Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW));
+            LOG.info("-------");
+            Threads.sleep(10 + random.nextInt(50));
+          } catch (Throwable e) {
+            ex = e;
+            Assert.fail(StringUtils.stringifyException(e));
+          }
+         }
+      }
+      void rethrowExceptionIfAny() throws Throwable {
+        if (ex != null) { throw ex; }
+      }
+    }
+
+    RegionMetaSplitter regionMetaSplitter = new RegionMetaSplitter();
+    MetaScannerVerifier metaScannerVerifier = new MetaScannerVerifier();
+
+    Thread regionMetaSplitterThread = new Thread(regionMetaSplitter);
+    Thread metaScannerVerifierThread = new Thread(metaScannerVerifier);
+
+    regionMetaSplitterThread.start();
+    metaScannerVerifierThread.start();
+
+    Threads.sleep(runtime);
+
+    regionMetaSplitter.stop("test finished");
+    metaScannerVerifier.stop("test finished");
+
+    regionMetaSplitterThread.join();
+    metaScannerVerifierThread.join();
+
+    regionMetaSplitter.rethrowExceptionIfAny();
+    metaScannerVerifier.rethrowExceptionIfAny();
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 0d9aaf32d96a..13a3c2724477 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -59,8 +59,8 @@
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
-import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -242,24 +242,24 @@ public TableDescriptors getTableDescriptors() {
         public HTableDescriptor remove(String tablename) throws IOException {
           return null;
         }
-        
+
         @Override
         public Map<String, HTableDescriptor> getAll() throws IOException {
           return null;
         }
-        
+
         @Override
         public HTableDescriptor get(byte[] tablename)
         throws IOException {
           return get(Bytes.toString(tablename));
         }
-        
+
         @Override
         public HTableDescriptor get(String tablename)
         throws IOException {
           return createHTableDescriptor();
         }
-        
+
         @Override
         public void add(HTableDescriptor htd) throws IOException {
         }
@@ -588,6 +588,87 @@ public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
     janitor.join();
   }
 
+  @Test
+  public void testSplitParentFirstComparator() {
+    SplitParentFirstComparator comp = new SplitParentFirstComparator();
+    final HTableDescriptor htd = createHTableDescriptor();
+
+    /*  Region splits:
+     *
+     *  rootRegion --- firstRegion --- firstRegiona
+     *              |               |- firstRegionb
+     *              |
+     *              |- lastRegion --- lastRegiona  --- lastRegionaa
+     *                             |                |- lastRegionab
+     *                             |- lastRegionb
+     *
+     *  rootRegion   :   []  - []
+     *  firstRegion  :   []  - bbb
+     *  lastRegion   :   bbb - []
+     *  firstRegiona :   []  - aaa
+     *  firstRegionb :   aaa - bbb
+     *  lastRegiona  :   bbb - ddd
+     *  lastRegionb  :   ddd - []
+     */
+
+    // root region
+    HRegionInfo rootRegion = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW,
+      HConstants.EMPTY_END_ROW, true);
+    HRegionInfo firstRegion = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW,
+      Bytes.toBytes("bbb"), true);
+    HRegionInfo lastRegion = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"),
+      HConstants.EMPTY_END_ROW, true);
+
+    assertTrue(comp.compare(rootRegion, rootRegion) == 0);
+    assertTrue(comp.compare(firstRegion, firstRegion) == 0);
+    assertTrue(comp.compare(lastRegion, lastRegion) == 0);
+    assertTrue(comp.compare(rootRegion, firstRegion) < 0);
+    assertTrue(comp.compare(rootRegion, lastRegion) < 0);
+    assertTrue(comp.compare(firstRegion, lastRegion) < 0);
+
+    //first region split into a, b
+    HRegionInfo firstRegiona = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW,
+      Bytes.toBytes("aaa"), true);
+    HRegionInfo firstRegionb = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
+      Bytes.toBytes("bbb"), true);
+    //last region split into a, b
+    HRegionInfo lastRegiona = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"),
+      Bytes.toBytes("ddd"), true);
+    HRegionInfo lastRegionb = new HRegionInfo(htd.getName(), Bytes.toBytes("ddd"),
+      HConstants.EMPTY_END_ROW, true);
+
+    assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
+    assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
+    assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
+    assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
+    assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
+    assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
+    assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
+
+    assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
+    assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
+    assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
+    assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
+    assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
+    assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
+    assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
+
+    assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
+    assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
+    assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
+    assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
+
+    HRegionInfo lastRegionaa = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"),
+      Bytes.toBytes("ccc"), false);
+    HRegionInfo lastRegionab = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"),
+      Bytes.toBytes("ddd"), false);
+
+    assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
+    assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
+    assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
+
+  }
+
   @Test
   public void testArchiveOldRegion() throws Exception {
     String table = "table";

From a3cfefca685def7a8dc520c9f7a3d0f65bff8193 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 17 May 2013 05:50:18 +0000
Subject: [PATCH 0981/1540] HBASE-8538 HBaseAdmin#isTableEnabled() should check
 table existence before checking zk state. (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483657 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HBaseAdmin.java  | 3 +++
 .../apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java   | 2 ++
 .../java/org/apache/hadoop/hbase/client/TestAdmin.java   | 9 +++++++++
 3 files changed, 14 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 3410b81f9a95..6f12fe33e61b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -934,6 +934,9 @@ public boolean isTableEnabled(byte[] tableName) throws IOException {
     if (!HTableDescriptor.isMetaTable(tableName)) {
       HTableDescriptor.isLegalTableName(tableName);
     }
+    if(!tableExists(tableName)){
+      throw new TableNotFoundException(Bytes.toString(tableName));
+    }
     return connection.isTableEnabled(tableName);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
index 1784f794374b..94a82642e8cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
@@ -66,6 +66,8 @@ public static boolean isDisabledTable(final ZooKeeperWatcher zkw,
   public static boolean isEnabledTable(final ZooKeeperWatcher zkw,
       final String tableName) throws KeeperException {
     TableState state = getTableState(zkw, tableName);
+    // If a table is ENABLED then we are removing table state znode in 0.92
+    // but in 0.94 we keep it in ENABLED state.
     return state == null || state == TableState.ENABLED;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 9ba1864b23f2..6c596d3346ca 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -255,6 +255,15 @@ public void testDisableAndEnableTable() throws IOException {
     ht.close();
   }
 
+  @Test
+  public void testIsEnabledOnUnknownTable() throws Exception {
+    try {
+      admin.isTableEnabled(Bytes.toBytes("unkownTable"));
+      fail("Test should fail if isTableEnabled called on unknown table.");
+    } catch (IOException e) {
+    }
+  }
+  
   @Test
   public void testDisableAndEnableTables() throws IOException {
     final byte [] row = Bytes.toBytes("row");

From 8a57e178ee844559c133e4cb568f1328f129465b Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 17 May 2013 13:40:05 +0000
Subject: [PATCH 0982/1540] HBASE-8563 Double count of read requests for Gets
 (Francis Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483796 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 63bab9edb20b..fd8229023307 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1815,8 +1815,6 @@ void prepareScanner(Scan scan) throws IOException {
   protected RegionScanner getScanner(Scan scan,
       List<KeyValueScanner> additionalScanners) throws IOException {
     startRegionOperation();
-    this.readRequestsCount.increment();
-    this.opMetrics.setReadRequestCountMetrics(this.readRequestsCount.get());
     try {
       // Verify families are all valid
       prepareScanner(scan);

From 4eb48a20439057e4b28f6685e88adac97de548bd Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 17 May 2013 21:07:51 +0000
Subject: [PATCH 0983/1540] HBASE-7210 Backport HBASE-6059 to 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1483992 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Compactor.java  |   5 +-
 .../hadoop/hbase/regionserver/HRegion.java    |  73 ++++++------
 .../hadoop/hbase/regionserver/Store.java      |   4 +
 .../hbase/regionserver/TestHRegion.java       |  25 +++-
 .../hadoop/hbase/regionserver/TestStore.java  |  16 ++-
 .../hbase/regionserver/wal/TestWALReplay.java | 107 +++++++++++++++++-
 6 files changed, 181 insertions(+), 49 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
index 50ef8b7d8395..7d5f77fc1772 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
@@ -191,7 +191,10 @@ StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOExcepti
         boolean hasMore;
         do {
           hasMore = scanner.next(kvs, compactionKVMax);
-          if (writer == null && !kvs.isEmpty()) {
+          // Create the writer even if no kv(Empty store file is also ok),
+          // because we need record the max seq id for the store file, see
+          // HBASE-6059
+          if (writer == null) {
             writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true,
                 maxMVCCReadpoint >= smallestReadPoint);
           }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index fd8229023307..57b7d0082c71 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -572,22 +572,13 @@ private long initializeRegionInternals(final CancelableProgressable reporter,
     cleanupTmpDir();
 
     // Load in all the HStores.
-    // Get minimum of the maxSeqId across all the store.
     //
     // Context: During replay we want to ensure that we do not lose any data. So, we
     // have to be conservative in how we replay logs. For each store, we calculate
-    // the maxSeqId up to which the store was flushed. But, since different stores
-    // could have a different maxSeqId, we choose the
-    // minimum across all the stores.
-    // This could potentially result in duplication of data for stores that are ahead
-    // of others. ColumnTrackers in the ScanQueryMatchers do the de-duplication, so we
-    // do not have to worry.
-    // TODO: If there is a store that was never flushed in a long time, we could replay
-    // a lot of data. Currently, this is not a problem because we flush all the stores at
-    // the same time. If we move to per-cf flushing, we might want to revisit this and send
-    // in a vector of maxSeqIds instead of sending in a single number, which has to be the
-    // min across all the max.
-    long minSeqId = -1;
+    // the maxSeqId up to which the store was flushed. And, skip the edits which
+    // is equal to or lower than maxSeqId for each store.
+    Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
+        Bytes.BYTES_COMPARATOR);
     long maxSeqId = -1;
     // initialized to -1 so that we pick up MemstoreTS from column families
     long maxMemstoreTS = -1;
@@ -617,9 +608,8 @@ public Store call() throws IOException {
 
           this.stores.put(store.getColumnFamilyName().getBytes(), store);
           long storeSeqId = store.getMaxSequenceId();
-          if (minSeqId == -1 || storeSeqId < minSeqId) {
-            minSeqId = storeSeqId;
-          }
+          maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
+              storeSeqId);
           if (maxSeqId == -1 || storeSeqId > maxSeqId) {
             maxSeqId = storeSeqId;
           }
@@ -639,7 +629,7 @@ public Store call() throws IOException {
     mvcc.initialize(maxMemstoreTS + 1);
     // Recover any edits if available.
     maxSeqId = Math.max(maxSeqId, replayRecoveredEditsIfAny(
-        this.regiondir, minSeqId, reporter, status));
+        this.regiondir, maxSeqIdInStores, reporter, status));
 
     status.setStatus("Cleaning up detritus from prior splits");
     // Get rid of any splits or merges that were lost in-progress.  Clean out
@@ -3094,8 +3084,8 @@ private boolean isFlushSize(final long size) {
    * make sense in a this single region context only -- until we online.
    *
    * @param regiondir
-   * @param minSeqId Any edit found in split editlogs needs to be in excess of
-   * this minSeqId to be applied, else its skipped.
+   * @param maxSeqIdInStores Any edit found in split editlogs needs to be in excess of
+   * the maxSeqId for the store to be applied, else its skipped.
    * @param reporter
    * @return the sequence id of the last edit added to this region out of the
    * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
@@ -3103,12 +3093,19 @@ private boolean isFlushSize(final long size) {
    * @throws IOException
    */
   protected long replayRecoveredEditsIfAny(final Path regiondir,
-      final long minSeqId, final CancelableProgressable reporter,
-      final MonitoredTask status)
+      Map<byte[], Long> maxSeqIdInStores,
+      final CancelableProgressable reporter, final MonitoredTask status)
       throws UnsupportedEncodingException, IOException {
-    long seqid = minSeqId;
+    long minSeqIdForTheRegion = -1;
+    for (Long maxSeqIdInStore : maxSeqIdInStores.values()) {
+      if (maxSeqIdInStore < minSeqIdForTheRegion || minSeqIdForTheRegion == -1) {
+        minSeqIdForTheRegion = maxSeqIdInStore;
+      }
+    }
+    long seqid = minSeqIdForTheRegion;
     NavigableSet<Path> files = HLog.getSplitEditFilesSorted(this.fs, regiondir);
     if (files == null || files.isEmpty()) return seqid;
+
     for (Path edits: files) {
       if (edits == null || !this.fs.exists(edits)) {
         LOG.warn("Null or non-existent edits file: " + edits);
@@ -3119,16 +3116,16 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
       long maxSeqId = Long.MAX_VALUE;
       String fileName = edits.getName();
       maxSeqId = Math.abs(Long.parseLong(fileName));
-      if (maxSeqId <= minSeqId) {
+      if (maxSeqId <= minSeqIdForTheRegion) {
         String msg = "Maximum sequenceid for this log is " + maxSeqId
-            + " and minimum sequenceid for the region is " + minSeqId
+            + " and minimum sequenceid for the region is " + minSeqIdForTheRegion
             + ", skipped the whole file, path=" + edits;
         LOG.debug(msg);
         continue;
       }
 
       try {
-        seqid = replayRecoveredEdits(edits, seqid, reporter);
+        seqid = replayRecoveredEdits(edits, maxSeqIdInStores, reporter);
       } catch (IOException e) {
         boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
         if (skipErrors) {
@@ -3145,7 +3142,7 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
         this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
       }
     }
-    if (seqid > minSeqId) {
+    if (seqid > minSeqIdForTheRegion) {
       // Then we added some edits to memory. Flush and cleanup split edit files.
       internalFlushcache(null, seqid, status);
     }
@@ -3162,18 +3159,17 @@ protected long replayRecoveredEditsIfAny(final Path regiondir,
 
   /*
    * @param edits File of recovered edits.
-   * @param minSeqId Minimum sequenceid found in a store file.  Edits in log
-   * must be larger than this to be replayed.
+   * @param maxSeqIdInStores Maximum sequenceid found in each store.  Edits in log
+   * must be larger than this to be replayed for each store.
    * @param reporter
    * @return the sequence id of the last edit added to this region out of the
    * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
    * @throws IOException
    */
   private long replayRecoveredEdits(final Path edits,
-      final long minSeqId, final CancelableProgressable reporter)
+      Map<byte[], Long> maxSeqIdInStores, final CancelableProgressable reporter)
     throws IOException {
-    String msg = "Replaying edits from " + edits + "; minSequenceid=" +
-      minSeqId + "; path=" + edits;
+    String msg = "Replaying edits from " + edits;
     LOG.info(msg);
     MonitoredTask status = TaskMonitor.get().createStatus(msg);
 
@@ -3181,7 +3177,7 @@ private long replayRecoveredEdits(final Path edits,
     HLog.Reader reader = null;
     try {
       reader = HLog.getReader(this.fs, edits, conf);
-      long currentEditSeqId = minSeqId;
+      long currentEditSeqId = -1;
       long firstSeqIdInLog = -1;
       long skippedEdits = 0;
       long editsCount = 0;
@@ -3240,12 +3236,6 @@ private long replayRecoveredEdits(final Path edits,
           if (firstSeqIdInLog == -1) {
             firstSeqIdInLog = key.getLogSeqNum();
           }
-          // Now, figure if we should skip this edit.
-          if (key.getLogSeqNum() <= currentEditSeqId) {
-            skippedEdits++;
-            continue;
-          }
-          currentEditSeqId = key.getLogSeqNum();
           boolean flush = false;
           for (KeyValue kv: val.getKeyValues()) {
             // Check this edit is for me. Also, guard against writing the special
@@ -3266,6 +3256,13 @@ private long replayRecoveredEdits(final Path edits,
               skippedEdits++;
               continue;
             }
+            // Now, figure if we should skip this edit.
+            if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getFamily()
+                .getName())) {
+              skippedEdits++;
+              continue;
+            }
+            currentEditSeqId = key.getLogSeqNum();
             // Once we are over the limit, restoreEdit will keep returning true to
             // flush -- but don't flush until we've played all the kvs that make up
             // the WALEdit.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 8542392b7120..a20892ee821f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1864,6 +1864,10 @@ private void rowAtOrBeforeFromStoreFile(final StoreFile f,
       LOG.warn("StoreFile " + f + " has a null Reader");
       return;
     }
+    if (r.getEntries() == 0) {
+      LOG.warn("StoreFile " + f + " is a empty store file");
+      return;
+    }
     // TODO: Cache these keys rather than make each time?
     byte [] fk = r.getFirstKey();
     if (fk == null) return;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index dec7fc557ed2..28419d7d5236 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -268,7 +268,13 @@ public void testSkipRecoveredEditsReplay() throws Exception {
         writer.close();
       }
       MonitoredTask status = TaskMonitor.get().createStatus(method);
-      long seqId = region.replayRecoveredEditsIfAny(regiondir, minSeqId-1, null, status);
+      Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
+          Bytes.BYTES_COMPARATOR);
+      for (Store store : region.getStores().values()) {
+        maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
+            minSeqId - 1);
+      }
+      long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
       assertEquals(maxSeqId, seqId);
       Get get = new Get(row);
       Result result = region.get(get, null);
@@ -314,7 +320,13 @@ public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
       }
       long recoverSeqId = 1030;
       MonitoredTask status = TaskMonitor.get().createStatus(method);
-      long seqId = region.replayRecoveredEditsIfAny(regiondir, recoverSeqId-1, null, status);
+      Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
+          Bytes.BYTES_COMPARATOR);
+      for (Store store : region.getStores().values()) {
+        maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
+            recoverSeqId - 1);
+      }
+      long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
       assertEquals(maxSeqId, seqId);
       Get get = new Get(row);
       Result result = region.get(get, null);
@@ -355,7 +367,14 @@ public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
           recoveredEditsDir, String.format("%019d", minSeqId-1));
       FSDataOutputStream dos=  fs.create(recoveredEdits);
       dos.close();
-      long seqId = region.replayRecoveredEditsIfAny(regiondir, minSeqId, null, null);
+      
+      Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
+        Bytes.BYTES_COMPARATOR);
+      for (Store store : region.getStores().values()) {
+        maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
+      }
+      long seqId = region.replayRecoveredEditsIfAny(regiondir,
+          maxSeqIdInStores, null, null);
       assertEquals(minSeqId, seqId);
     } finally {
       HRegion.closeHRegion(this.region);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 121d2778930c..94c1f1a4804d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -183,12 +183,18 @@ public void testDeleteExpiredStoreFiles() throws Exception {
     for (int i = 1; i <= storeFileNum; i++) {
       // verify the expired store file.
       CompactionRequest cr = this.store.requestCompaction();
-      assertEquals(1, cr.getFiles().size());
-      assertTrue(cr.getFiles().get(0).getReader().getMaxTimestamp() < 
-          (System.currentTimeMillis() - this.store.scanInfo.getTtl()));
-      // Verify that the expired the store has been deleted.
+      // the first is expired normally.
+      // If not the first compaction, there is another empty store file,
+      assertEquals(Math.min(i, 2), cr.getFiles().size());
+      for (int j = 0; i < cr.getFiles().size(); j++) {
+        assertTrue(cr.getFiles().get(j).getReader().getMaxTimestamp() < (System
+            .currentTimeMillis() - this.store.scanInfo.getTtl()));
+      }
+      // Verify that the expired store file is compacted to an empty store file.
       this.store.compact(cr);
-      assertEquals(storeFileNum - i, this.store.getStorefiles().size());
+      // It is an empty store file.
+      assertEquals(0, this.store.getStorefiles().get(0).getReader()
+          .getEntries());
 
       // Let the next store file expired.
       Thread.sleep(sleepTime);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 8b1f3b75c2ee..f95fa1f3e274 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -45,15 +45,24 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.regionserver.FlushRequester;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.regionserver.Store;
@@ -91,7 +100,7 @@ public static void setUpBeforeClass() throws Exception {
     conf.setBoolean("dfs.support.append", true);
     // The below config supported by 0.20-append and CDH3b2
     conf.setInt("dfs.client.block.recovery.retries", 2);
-    TEST_UTIL.startMiniDFSCluster(3);
+    TEST_UTIL.startMiniCluster(3);
     Path hbaseRootDir =
       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
     LOG.info("hbase.rootdir=" + hbaseRootDir);
@@ -100,7 +109,7 @@ public static void setUpBeforeClass() throws Exception {
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniDFSCluster();
+    TEST_UTIL.shutdownMiniCluster();
   }
 
   @Before
@@ -131,6 +140,100 @@ private void deleteDir(final Path p) throws IOException {
     }
   }
 
+  /**
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
+    final byte[] tableName = Bytes
+        .toBytes("testReplayEditsAfterRegionMovedWithMultiCF");
+    byte[] family1 = Bytes.toBytes("cf1");
+    byte[] family2 = Bytes.toBytes("cf2");
+    byte[] qualifier = Bytes.toBytes("q");
+    byte[] value = Bytes.toBytes("testV");
+    byte[][] familys = { family1, family2 };
+    TEST_UTIL.createTable(tableName, familys);
+    HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    Put put = new Put(Bytes.toBytes("r1"));
+    put.add(family1, qualifier, value);
+    htable.put(put);
+    ResultScanner resultScanner = htable.getScanner(new Scan());
+    int count = 0;
+    while (resultScanner.next() != null) {
+      count++;
+    }
+    resultScanner.close();
+    assertEquals(1, count);
+
+    MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
+    List<HRegion> regions = hbaseCluster.getRegions(tableName);
+    assertEquals(1, regions.size());
+
+    // move region to another regionserver
+    HRegion destRegion = regions.get(0);
+    int originServerNum = hbaseCluster
+        .getServerWith(destRegion.getRegionName());
+    assertTrue("Please start more than 1 regionserver", hbaseCluster
+        .getRegionServerThreads().size() > 1);
+    int destServerNum = 0;
+    while (destServerNum == originServerNum) {
+      destServerNum++;
+    }
+    HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
+    HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
+    // move region to destination regionserver
+    moveRegionAndWait(destRegion, destServer);
+
+    // delete the row
+    Delete del = new Delete(Bytes.toBytes("r1"));
+    htable.delete(del);
+    resultScanner = htable.getScanner(new Scan());
+    count = 0;
+    while (resultScanner.next() != null) {
+      count++;
+    }
+    resultScanner.close();
+    assertEquals(0, count);
+
+    // flush region and make major compaction
+    destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
+    // wait to complete major compaction
+    for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
+        .getStores().values()) {
+      store.triggerMajorCompaction();
+    }
+    destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
+
+    // move region to origin regionserver
+    moveRegionAndWait(destRegion, originServer);
+    // abort the origin regionserver
+    originServer.abort("testing");
+
+    // see what we get
+    Result result = htable.get(new Get(Bytes.toBytes("r1")));
+    if (result != null) {
+      assertTrue("Row is deleted, but we get" + result.toString(),
+          (result == null) || result.isEmpty());
+    }
+    resultScanner.close();
+  }
+
+  private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
+      throws InterruptedException, MasterNotRunningException,
+      ZooKeeperConnectionException, IOException {
+    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+    TEST_UTIL.getHBaseAdmin().move(
+        destRegion.getRegionInfo().getEncodedNameAsBytes(),
+        Bytes.toBytes(destServer.getServerName().getServerName()));
+    while (true) {
+      ServerName serverName = master.getAssignmentManager()
+          .getRegionServerOfRegion(destRegion.getRegionInfo());
+      if (serverName != null && serverName.equals(destServer.getServerName())) break;
+      Thread.sleep(10);
+    }
+  }
+
   /**
    * Tests for hbase-2727.
    * @throws Exception

From 17b0d6c8d96c019dbcd32871cd329d5e9ae43720 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Sat, 18 May 2013 00:12:14 +0000
Subject: [PATCH 0984/1540] HBASE-8547. Fix java.lang.RuntimeException: Cached
 an already cached block (Addendum2 and 3)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1484034 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/LruBlockCache.java  | 22 ++++++++++++++++---
 .../hbase/io/hfile/TestLruBlockCache.java     | 11 ----------
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index c7e83fe82072..6533c9333886 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumMap;
@@ -267,19 +268,25 @@ public void setMaxSize(long maxSize) {
   /**
    * Cache the block with the specified name and buffer.
    * <p>
-   * It is assumed this will NEVER be called on an already cached block.  If
-   * that is done, an exception will be thrown.
+   * It is assumed this will NOT be called on an already cached block. In rare cases (HBASE-8547)
+   * this can happen, for which we compare the buffer contents.
    * @param cacheKey block's cache key
    * @param buf block buffer
    * @param inMemory if block is in-memory
    */
+  @Override
   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
     CachedBlock cb = map.get(cacheKey);
     if(cb != null) {
+      // compare the contents, if they are not equal, we are in big trouble
+      if (compare(buf, cb.getBuffer()) != 0) {
+        throw new RuntimeException("Cached block contents differ, which should not have happened."
+          + "cacheKey:" + cacheKey);
+      }
       String msg = "Cached an already cached block: " + cacheKey + " cb:" + cb.getCacheKey();
       msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
       LOG.warn(msg);
-      assert false : msg;
+      return;
     }
     cb = new CachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
     long newSize = updateSizeMetrics(cb, false);
@@ -290,6 +297,15 @@ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory)
     }
   }
 
+  private int compare(Cacheable left, Cacheable right) {
+    ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
+    left.serialize(l);
+    ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
+    right.serialize(r);
+    return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
+      r.array(), r.arrayOffset(), r.limit());
+  }
+
   /**
    * Cache the block with the specified name and buffer.
    * <p>
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index 7d8953168c35..65af0d74ece8 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -143,17 +143,6 @@ public void testCacheSimple() throws Exception {
       assertEquals(buf.heapSize(), block.heapSize());
     }
 
-    // Re-add same blocks and ensure nothing has changed
-    for (CachedItem block : blocks) {
-      try {
-        cache.cacheBlock(block.cacheKey, block);
-        assertTrue("Cache should not allow re-caching a block", false);
-      } catch(AssertionError re) {
-        // expected
-        assertTrue(re.getMessage().contains("Cached an already cached block"));
-      }
-    }
-
     // Verify correctly calculated cache heap size
     assertEquals(expectedCacheSize, cache.heapSize());
 

From fe82075e63f17e5440eabbd39f9d4378bc951db2 Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Sat, 18 May 2013 01:41:53 +0000
Subject: [PATCH 0985/1540] HBASE-8539 ADDENDUM Double(or tripple ...)
 ZooKeeper listeners of the same type when Master recovers from ZK
 SessionExpiredException (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1484041 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/HMaster.java     | 13 +++++++++++++
 .../hadoop/hbase/zookeeper/ZooKeeperWatcher.java    |  7 +++++++
 2 files changed, 20 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 32e87d2891b7..8c62fc1b84b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -118,6 +118,7 @@
 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Text;
@@ -260,6 +261,9 @@ public class HMaster extends HasThread
   /** flag used in test cases in order to simulate RS failures during master initialization */
   private volatile boolean initializationBeforeMetaAssignment = false;
 
+  /** The following is used in master recovery scenario to re-register listeners */
+  private List<ZooKeeperListener> registeredZKListenersBeforeRecovery;
+
   /**
    * Initializes the HMaster. The steps are as follows:
    * <p>
@@ -391,6 +395,7 @@ public void run() {
     startupStatus.setDescription("Master startup");
     masterStartTime = System.currentTimeMillis();
     try {
+      this.registeredZKListenersBeforeRecovery = this.zooKeeper.getListeners();
       /*
        * Block on becoming the active master.
        *
@@ -1627,6 +1632,14 @@ private boolean tryRecoveringExpiredZKSession() throws InterruptedException,
       IOException, KeeperException, ExecutionException {
 
     this.zooKeeper.unregisterAllListeners();
+    // add back listeners which were registered before master initialization
+    // because they won't be added back in below Master re-initialization code
+    if (this.registeredZKListenersBeforeRecovery != null) {
+      for (ZooKeeperListener curListener : this.registeredZKListenersBeforeRecovery) {
+        this.zooKeeper.registerListener(curListener);
+      }
+    }
+
     this.zooKeeper.reconnectAfterExpiration();
 
     Callable<Boolean> callable = new Callable<Boolean> () {
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index a43ff1688d1e..e023692caad5 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -249,6 +249,13 @@ public void unregisterAllListeners() {
     listeners.clear();
   }
 
+  /**
+   * Get a copy of current registered listeners
+   */
+  public List<ZooKeeperListener> getListeners() {
+    return new ArrayList<ZooKeeperListener>(listeners);
+  }
+
   /**
    * @return The number of currently registered listeners
    */

From 50256d7035649b08a677e8f622dd49ac6a59b278 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 20 May 2013 15:45:08 +0000
Subject: [PATCH 0986/1540] HBASE-8067
 TestHFileArchiving.testArchiveOnTableDelete sometimes fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1484504 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/backup/TestHFileArchiving.java      | 36 ++++++++++---------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index 73912a703d38..7e28f6b1ef25 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -39,7 +39,6 @@
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -52,7 +51,6 @@
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -64,11 +62,8 @@
 @Category(MediumTests.class)
 public class TestHFileArchiving {
 
-  private static final String STRING_TABLE_NAME = "test_table";
-
   private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-  private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
   private static final byte[] TEST_FAM = Bytes.toBytes("fam");
 
   /**
@@ -96,18 +91,9 @@ private static void setupConf(Configuration conf) {
       ConstantSizeRegionSplitPolicy.class.getName());
   }
 
-  @Before
-  public void setup() throws Exception {
-    UTIL.createTable(TABLE_NAME, TEST_FAM);
-  }
-
   @After
   public void tearDown() throws Exception {
-    // cleanup the cluster if its up still
-    if (UTIL.getHBaseAdmin().tableExists(STRING_TABLE_NAME)) {
-      UTIL.deleteTable(TABLE_NAME);
-    }
-    // and cleanup the archive directory
+    // cleanup the archive directory
     try {
       clearArchiveDirectory();
     } catch (IOException e) {
@@ -126,6 +112,9 @@ public static void cleanupTest() throws Exception {
 
   @Test
   public void testRemovesRegionDirOnArchive() throws Exception {
+    byte[] TABLE_NAME = Bytes.toBytes("testRemovesRegionDirOnArchive");
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+
     final HBaseAdmin admin = UTIL.getHBaseAdmin();
 
     // get the current store files for the region
@@ -138,7 +127,7 @@ public void testRemovesRegionDirOnArchive() throws Exception {
     UTIL.loadRegion(region, TEST_FAM);
 
     // shutdown the table so we can manipulate the files
-    admin.disableTable(STRING_TABLE_NAME);
+    admin.disableTable(TABLE_NAME);
 
     FileSystem fs = UTIL.getTestFileSystem();
 
@@ -161,6 +150,8 @@ public void testRemovesRegionDirOnArchive() throws Exception {
 
     // then ensure the region's directory isn't present
     assertFalse(fs.exists(regionDir));
+
+    UTIL.deleteTable(TABLE_NAME);
   }
 
   /**
@@ -170,6 +161,9 @@ public void testRemovesRegionDirOnArchive() throws Exception {
    */
   @Test
   public void testDeleteRegionWithNoStoreFiles() throws Exception {
+    byte[] TABLE_NAME = Bytes.toBytes("testDeleteRegionWithNoStoreFiles");
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+
     // get the current store files for the region
     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
     // make sure we only have 1 region serving this table
@@ -209,10 +203,15 @@ public boolean accept(Path file) {
 
     // and check to make sure the region directoy got deleted
     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
+
+    UTIL.deleteTable(TABLE_NAME);
   }
 
   @Test
   public void testArchiveOnTableDelete() throws Exception {
+    byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableDelete");
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+
     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
     // make sure we only have 1 region serving this table
     assertEquals(1, servingRegions.size());
@@ -273,6 +272,9 @@ public void testArchiveOnTableDelete() throws Exception {
    */
   @Test
   public void testArchiveOnTableFamilyDelete() throws Exception {
+    byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableFamilyDelete");
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+
     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
     // make sure we only have 1 region serving this table
     assertEquals(1, servingRegions.size());
@@ -325,6 +327,8 @@ public void testArchiveOnTableFamilyDelete() throws Exception {
 
     assertTrue("Archived files are missing some of the store files!",
       archivedFiles.containsAll(storeFiles));
+
+    UTIL.deleteTable(TABLE_NAME);
   }
 
   /**

From fce85b342b209fa955db9049f2d036e4745c167c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 21 May 2013 05:37:48 +0000
Subject: [PATCH 0987/1540] HBASE-8282 User triggered flushes does not allow
 compaction to get triggered even if compaction criteria is met (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1484663 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java     |  2 +-
 .../hbase/regionserver/HRegionServer.java      | 18 +++++++++++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 57b7d0082c71..07a10df910d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1405,7 +1405,7 @@ public boolean compact(CompactionRequest cr)
    * <p>This method may block for some time, so it should not be called from a
    * time-sensitive thread.
    *
-   * @return true if cache was flushed
+   * @return true if the region needs compaction
    *
    * @throws IOException general io exceptions
    * @throws DroppedSnapshotException Thrown when replay of hlog is required
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index d1e55da12d10..b168855a66dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2449,7 +2449,10 @@ public void flushRegion(byte[] regionName)
       throw new IllegalArgumentException("No region : " + new String(regionName)
       + " available");
     }
-    region.flushcache();
+    boolean needsCompaction = region.flushcache();
+    if (needsCompaction) {
+      this.compactSplitThread.requestCompaction(region, "Compaction through user triggered flush");
+    }
   }
 
  /**
@@ -2462,7 +2465,13 @@ public void flushRegion(byte[] regionName, long ifOlderThanTS)
        throw new IllegalArgumentException("No region : " + new String(regionName)
        + " available");
      }
-     if (region.getLastFlushTime() < ifOlderThanTS) region.flushcache();
+     if (region.getLastFlushTime() < ifOlderThanTS) {
+      boolean needsCompaction = region.flushcache();
+      if (needsCompaction) {
+        this.compactSplitThread
+            .requestCompaction(region, "Compaction through user triggered flush");
+      }
+    }
    }
 
   /**
@@ -3247,7 +3256,10 @@ public void flushRegion(HRegionInfo regionInfo)
     checkOpen();
     LOG.info("Flushing " + regionInfo.getRegionNameAsString());
     HRegion region = getRegion(regionInfo.getRegionName());
-    region.flushcache();
+    boolean needsCompaction = region.flushcache();
+    if (needsCompaction) {
+      this.compactSplitThread.requestCompaction(region, "Compaction through user triggered flush");
+    }
   }
 
   @Override

From 5b21a55f67c117210168d94070e26d80a363f792 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 22 May 2013 18:01:17 +0000
Subject: [PATCH 0988/1540] HBASE-8525 Use sleep multilier when choosing sinks
 in ReplicationSource

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1485318 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSource.java        | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index d902010c1c3c..930924dfff50 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -171,9 +171,9 @@ public void init(final Configuration conf,
     for (int i = 0; i < this.replicationQueueNbCapacity; i++) {
       this.entriesArray[i] = new HLog.Entry();
     }
-    this.maxRetriesMultiplier =
-        this.conf.getInt("replication.source.maxretriesmultiplier", 10);
-    this.socketTimeoutMultiplier = maxRetriesMultiplier * maxRetriesMultiplier;
+    this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 10);
+    this.socketTimeoutMultiplier = this.conf.getInt("replication.source.socketTimeoutMultiplier",
+        maxRetriesMultiplier * maxRetriesMultiplier);
     this.queue =
         new PriorityBlockingQueue<Path>(
             conf.getInt("hbase.regionserver.maxlogs", 32),
@@ -521,14 +521,16 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
   }
 
   private void connectToPeers() {
+    int sleepMultiplier = 1;
+
     // Connect to peer cluster first, unless we have to stop
     while (this.isActive() && this.currentPeers.size() == 0) {
 
-      try {
-        chooseSinks();
-        Thread.sleep(this.sleepForRetries);
-      } catch (InterruptedException e) {
-        LOG.error("Interrupted while trying to connect to sinks", e);
+      chooseSinks();
+      if (this.isActive() && this.currentPeers.size() == 0) {
+        if (sleepForRetries("Waiting for peers", sleepMultiplier)) {
+          sleepMultiplier++;
+        }
       }
     }
   }

From a8f197b7e2321e6d3c67dc25665a60cbda820279 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 22 May 2013 20:38:18 +0000
Subject: [PATCH 0989/1540] CHANGES.txt, pom.xml for 0.94.8

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1485407 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index fc956cb50518..02dfcf88f12f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,70 @@
 HBase Change Log
 
+Release 0.94.8 - 5/22/2013
+Sub-task
+
+    [HBASE-8381] - TestTableInputFormatScan on Hadoop 2 fails because YARN kills our applications
+    [HBASE-8399] - TestTableInputFormatScan2#testScanFromConfiguration fails on hadoop2 profile
+
+Bug
+
+    [HBASE-7122] - Proper warning message when opening a log file with no entries (idle cluster)
+    [HBASE-7210] - Backport HBASE-6059 to 0.94
+    [HBASE-7921] - TestHFileBlock.testGzipCompression should ignore the block checksum
+    [HBASE-8282] - User triggered flushes does not allow compaction to get triggered even if compaction criteria is met
+    [HBASE-8327] - Consolidate class loaders
+    [HBASE-8354] - Backport HBASE-7878 'recoverFileLease does not check return value of recoverLease' to 0.94
+    [HBASE-8355] - BaseRegionObserver#pre(Compact|Flush|Store)ScannerOpen returns null
+    [HBASE-8377] - IntegrationTestBigLinkedList calculates wrap for linked list size incorrectly
+    [HBASE-8379] - bin/graceful_stop.sh does not return the balancer to original state
+    [HBASE-8385] - [SNAPSHOTS]: Restore fails to restore snapshot of a deleted table
+    [HBASE-8389] - HBASE-8354 forces Namenode into loop with lease recovery requests
+    [HBASE-8413] - Snapshot verify region will always fail if the HFile has been archived
+    [HBASE-8451] - MetricsMBeanBase has concurrency issues in init
+    [HBASE-8455] - Update ExportSnapshot to reflect changes in HBASE-7419
+    [HBASE-8464] - FastDiffEncoder - valueOffset calculation is incorrect
+    [HBASE-8483] - HConnectionManager can leak ZooKeeper connections when using deleteStaleConnection
+    [HBASE-8493] - Backport HBASE-8422, 'Master won't go down', to 0.94
+    [HBASE-8503] - Backport hbase-8483 "HConnectionManager can leak ZooKeeper connections when using deleteStaleConnection" to 0.94
+    [HBASE-8505] - References to split daughters should not be deleted separately from parent META entry
+    [HBASE-8509] - ZKUtil#createWithParents won't set data during znode creation when parent folder doesn't exit
+    [HBASE-8513] - [0.94] Fix class files with CRLF endings
+    [HBASE-8516] - FSUtils.create() fail with ViewFS
+    [HBASE-8525] - Use sleep multilier when choosing sinks in ReplicationSource
+    [HBASE-8530] - Refine error message from ExportSnapshot when there is leftover snapshot in target cluster
+    [HBASE-8538] - HBaseAdmin#isTableEnabled() should check table existence before checking zk state.
+    [HBASE-8539] - Double(or tripple ...) ZooKeeper listeners of the same type when Master recovers from ZK SessionExpiredException
+    [HBASE-8540] - SnapshotFileCache logs too many times if snapshot dir doesn't exists
+    [HBASE-8547] - Fix java.lang.RuntimeException: Cached an already cached block
+    [HBASE-8550] - 0.94 ChaosMonkey grep for master is too broad
+    [HBASE-8563] - Double count of read requests for Gets
+    [HBASE-8588] - [Documentation]: Add information about adding REST and Thrift API kerberos principals to HBase ACL table
+
+Improvement
+
+    [HBASE-5930] - Limits the amount of time an edit can live in the memstore.
+    [HBASE-6870] - HTable#coprocessorExec always scan the whole table
+    [HBASE-8345] - Add all available resources in o.a.h.h.rest.RootResource and VersionResource to o.a.h.h.rest.client.RemoteAdmin
+    [HBASE-8350] - enable ChaosMonkey to run commands as different users
+    [HBASE-8367] - LoadIncrementalHFiles does not return an error code nor throw Exception when failures occur due to timeouts
+    [HBASE-8383] - Support lib/*jar inside coprocessor jar
+    [HBASE-8405] - Add more custom options to how ClusterManager runs commands
+    [HBASE-8446] - Allow parallel snapshot of different tables
+
+New Feature
+
+    [HBASE-7965] - Port table locking to 0.94 (HBASE-7305, HBASE-7546, HBASE-7933)
+    [HBASE-8415] - DisabledRegionSplitPolicy
+
+Task
+
+    [HBASE-8574] - Add how to rename a table in the docbook
+
+Test
+
+    [HBASE-8508] - improve unit-test coverage of package org.apache.hadoop.hbase.metrics.file
+
+
 Release 0.94.7 - 4/24/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 1fe0e07b1578..221e8cd25d0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.8-SNAPSHOT</version>
+  <version>0.94.8</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 018daa0c064aa6e6b90c82b9c9940ca2c8ada304 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 22 May 2013 23:12:45 +0000
Subject: [PATCH 0990/1540] HBASE-8555 FilterList correctness may be affected
 by random ordering of sub-filter(list) (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1485492 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/FilterList.java       |  7 +-
 .../hadoop/hbase/filter/TestFilter.java       | 94 +++++++++++++++++++
 .../hadoop/hbase/filter/TestFilterList.java   | 12 +--
 3 files changed, 103 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index f65065ce8dbb..18629e157460 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -149,20 +149,21 @@ public void reset() {
 
   @Override
   public boolean filterRowKey(byte[] rowKey, int offset, int length) {
+    boolean flag = (this.operator == Operator.MUST_PASS_ONE) ? true : false;
     for (Filter filter : filters) {
       if (this.operator == Operator.MUST_PASS_ALL) {
         if (filter.filterAllRemaining() ||
             filter.filterRowKey(rowKey, offset, length)) {
-          return true;
+          flag = true;
         }
       } else if (this.operator == Operator.MUST_PASS_ONE) {
         if (!filter.filterAllRemaining() &&
             !filter.filterRowKey(rowKey, offset, length)) {
-          return false;
+          flag = false;
         }
       }
     }
-    return this.operator == Operator.MUST_PASS_ONE;
+    return flag;
   }
 
   @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index d5837006ca48..0c61aa25c954 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -1225,6 +1225,100 @@ public void testFilterListWithSingleColumnValueFilter() throws IOException {
     verifyScanFull(s, kvs);
   }
 
+  public void testNestedFilterListWithSCVF() throws IOException {
+    byte[] columnStatus = Bytes.toBytes("S");
+    HTableDescriptor htd = new HTableDescriptor(getName());
+    htd.addFamily(new HColumnDescriptor(FAMILIES[0]));
+    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+    HRegion testRegion = HRegion.createHRegion(info, testDir, conf, htd);
+    for(int i=0; i<10; i++) {
+      Put p = new Put(Bytes.toBytes("row" + i));
+      p.setWriteToWAL(false);
+      p.add(FAMILIES[0], columnStatus, Bytes.toBytes(i%2));
+      testRegion.put(p);
+    }
+    testRegion.flushcache();
+    // 1. got rows > "row4"
+    Filter rowFilter = new RowFilter(CompareOp.GREATER, new BinaryComparator(
+        Bytes.toBytes("row4")));
+    Scan s1 = new Scan();
+    s1.setFilter(rowFilter);
+    InternalScanner scanner = testRegion.getScanner(s1);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    int i = 5;
+    for (boolean done = true; done; i++) {
+      done = scanner.next(results);
+      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
+      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
+      results.clear();
+    }
+    // 2. got rows <= "row4" and S=
+    FilterList subFilterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
+    Filter subFilter1 = new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator(
+        Bytes.toBytes("row4")));
+    subFilterList.addFilter(subFilter1);
+    Filter subFilter2 = new SingleColumnValueFilter(FAMILIES[0], columnStatus, CompareOp.EQUAL,
+        Bytes.toBytes(0));
+    subFilterList.addFilter(subFilter2);
+    s1 = new Scan();
+    s1.setFilter(subFilterList);
+    scanner = testRegion.getScanner(s1);
+    results = new ArrayList<KeyValue>();
+    for (i=0; i<=4; i+=2) {
+      scanner.next(results);
+      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
+      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
+      results.clear();
+    }
+    Assert.assertFalse(scanner.next(results));
+    // 3. let's begin to verify nested filter list
+    // 3.1 add rowFilter, then add subFilterList
+    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+    filterList.addFilter(rowFilter);
+    filterList.addFilter(subFilterList);
+    s1 = new Scan();
+    s1.setFilter(filterList);
+    scanner = testRegion.getScanner(s1);
+    results = new ArrayList<KeyValue>();
+    for (i=0; i<=4; i+=2) {
+      scanner.next(results);
+      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
+      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
+      results.clear();
+    }
+    for (i=5; i<=9; i++) {
+      scanner.next(results);
+      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
+      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
+      results.clear();
+    }
+    Assert.assertFalse(scanner.next(results));
+    // 3.2 MAGIC here! add subFilterList first, then add rowFilter
+    filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+    filterList.addFilter(subFilterList);
+    filterList.addFilter(rowFilter);
+    s1 = new Scan();
+    s1.setFilter(filterList);
+    scanner = testRegion.getScanner(s1);
+    results = new ArrayList<KeyValue>();
+    for (i=0; i<=4; i+=2) {
+      scanner.next(results);
+      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
+      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
+      results.clear();
+    }
+    for (i=5; i<=9; i++) {
+      scanner.next(results);
+      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
+      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
+      results.clear();
+    }
+    Assert.assertFalse(scanner.next(results));
+    HLog hlog = testRegion.getLog();
+    testRegion.close();
+    hlog.closeAndDelete();
+  }
+
   public void testSingleColumnValueFilter() throws IOException {
 
     // From HBASE-1821
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index b3a1596954f6..5825796c856d 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -79,33 +79,31 @@ public void testMPONE() throws Exception {
     byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
     for (int i = 0; i < MAX_PAGES - 1; i++) {
       assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
-      assertFalse(filterMPONE.filterRow());
       KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
         Bytes.toBytes(i));
       assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
+      assertFalse(filterMPONE.filterRow());
     }
 
     /* Only pass PageFilter */
     rowkey = Bytes.toBytes("z");
     assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
-    assertFalse(filterMPONE.filterRow());
     KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
         Bytes.toBytes(0));
     assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
+    assertFalse(filterMPONE.filterRow());
 
-    /* PageFilter will fail now, but should pass because we match yyy */
+    /* reach MAX_PAGES already, should filter any rows */
     rowkey = Bytes.toBytes("yyy");
-    assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
-    assertFalse(filterMPONE.filterRow());
+    assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
     kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
         Bytes.toBytes(0));
-    assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
+    assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
 
     /* We should filter any row */
     rowkey = Bytes.toBytes("z");
     assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
     assertTrue(filterMPONE.filterAllRemaining());
-
   }
 
   /**

From 90cf27d6d8492920acb7cfd1835e5b1e2ed75518 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 23 May 2013 02:27:14 +0000
Subject: [PATCH 0991/1540] HBASE-8504 HTable.getRegionsInRange() should
 provide a non-cached API

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1485543 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HTable.java    | 39 +++++++++++++++++--
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index c7c13f8040be..b7e62e46d000 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -495,8 +495,23 @@ public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOExcep
    */
   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
     final byte [] endKey) throws IOException {
-    return getKeysAndRegionsInRange(startKey, endKey, false).getSecond();
-    }
+    return getRegionsInRange(startKey, endKey, false);
+  }
+
+  /**
+   * Get the corresponding regions for an arbitrary range of keys.
+   * <p>
+   * @param startKey Starting row in range, inclusive
+   * @param endKey Ending row in range, exclusive
+   * @param reload true to reload information or false to use cached information
+   * @return A list of HRegionLocations corresponding to the regions that
+   * contain the specified range
+   * @throws IOException if a remote or network exception occurs
+   */
+  public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
+      final byte [] endKey, final boolean reload) throws IOException {
+    return getKeysAndRegionsInRange(startKey, endKey, false, reload).getSecond();
+  }
 
   /**
    * Get the corresponding start keys and regions for an arbitrary range of
@@ -512,6 +527,24 @@ public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
   private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
       final byte[] startKey, final byte[] endKey, final boolean includeEndKey)
       throws IOException {
+    return getKeysAndRegionsInRange(startKey, endKey, includeEndKey, false);
+  }
+
+  /**
+   * Get the corresponding start keys and regions for an arbitrary range of
+   * keys.
+   * <p>
+   * @param startKey Starting row in range, inclusive
+   * @param endKey Ending row in range
+   * @param includeEndKey true if endRow is inclusive, false if exclusive
+   * @param reload true to reload information or false to use cached information
+   * @return A pair of list of start keys and list of HRegionLocations that
+   *         contain the specified range
+   * @throws IOException if a remote or network exception occurs
+   */
+  private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
+      final byte[] startKey, final byte[] endKey, final boolean includeEndKey,
+      final boolean reload) throws IOException {
     final boolean endKeyIsEndOfTable = Bytes.equals(endKey,HConstants.EMPTY_END_ROW);
     if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
       throw new IllegalArgumentException(
@@ -522,7 +555,7 @@ private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
     List<HRegionLocation> regionsInRange = new ArrayList<HRegionLocation>();
     byte[] currentKey = startKey;
     do {
-      HRegionLocation regionLocation = getRegionLocation(currentKey, false);
+      HRegionLocation regionLocation = getRegionLocation(currentKey, reload);
       keysInRange.add(currentKey);
       regionsInRange.add(regionLocation);
       currentKey = regionLocation.getRegionInfo().getEndKey();

From bf1e92d208b69849b7bee16efdf1d82689c5e253 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 23 May 2013 22:08:08 +0000
Subject: [PATCH 0992/1540] HBASE-8522 Archived hfiles and old hlogs may be
 deleted immediately by HFileCleaner, LogCleaner in HMaster (Shaohui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1485872 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HBaseFileSystem.java  | 12 ++++++
 .../hadoop/hbase/backup/HFileArchiver.java    |  4 +-
 .../hadoop/hbase/regionserver/wal/HLog.java   |  5 +--
 .../hbase/regionserver/wal/HLogSplitter.java  |  2 +-
 .../hadoop/hbase/TestHBaseFileSystem.java     | 38 +++++++++++++++++++
 5 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
index 1a864c2151f1..c2e04f336342 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 
@@ -252,4 +253,15 @@ protected static void sleepBeforeRetry(String msg, int sleepMultiplier) {
     LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
     Threads.sleep(baseSleepBeforeRetries * sleepMultiplier);
   }
+  
+  /**
+   * rename the src path to dest path and set the dest path's modify time to current timestamp
+   */
+  public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
+      throws IOException {
+    if (!renameDirForFileSystem(fs, src, dest)) return false;
+    // set the modify time for TimeToLive Cleaner
+    fs.setTimes(dest, EnvironmentEdgeManager.currentTimeMillis(), -1);
+    return true;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index 8a14b1116b80..c880ae87b461 100644
--- a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -386,7 +386,7 @@ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
 
       // move the archive file to the stamped backup
       Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, archiveFile, backedupArchiveFile)) {
+      if (!HBaseFileSystem.renameAndSetModifyTime(fs, archiveFile, backedupArchiveFile)) {
         LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
             + ", deleting existing file in favor of newer.");
         // try to delete the exisiting file, if we can't rename it
@@ -610,7 +610,7 @@ public File(FileSystem fs) {
     public boolean moveAndClose(Path dest) throws IOException {
       this.close();
       Path p = this.getPath();
-      return HBaseFileSystem.renameDirForFileSystem(fs, p, dest);
+      return HBaseFileSystem.renameAndSetModifyTime(fs, p, dest);
     }
 
     /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index b8d90a2a3aff..cb9783c8a42d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -904,7 +904,7 @@ private void archiveLogFile(final Path p, final Long seqno) throws IOException {
         i.preLogArchive(p, newPath);
       }
     }
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) {
+    if (!HBaseFileSystem.renameAndSetModifyTime(this.fs, p, newPath)) {
       throw new IOException("Unable to rename " + p + " to " + newPath);
     }
     // Tell our listeners that a log has been archived.
@@ -966,8 +966,7 @@ public void closeAndDelete() throws IOException {
           i.preLogArchive(file.getPath(), p);
         }
       }
-
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, file.getPath(), p)) {
+      if (!HBaseFileSystem.renameAndSetModifyTime(fs, file.getPath(), p)) {
         throw new IOException("Unable to rename " + file.getPath() + " to " + p);
       }
       // Tell our listeners that a log was archived.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 57f75b79066f..18708f19d3c1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -629,7 +629,7 @@ private static void archiveLogs(
     for (Path p : processedLogs) {
       Path newPath = HLog.getHLogArchivePath(oldLogDir, p);
       if (fs.exists(p)) {
-        if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) {
+        if (!HBaseFileSystem.renameAndSetModifyTime(fs, p, newPath)) {
           LOG.warn("Unable to move  " + p + " to " + newPath);
         } else {
           LOG.debug("Archived processed log " + p + " to " + newPath);
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
index 67602c5527ee..6fac0245116e 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
@@ -17,10 +17,12 @@
  */
 package org.apache.hadoop.hbase;
 
+import static org.junit.Assert.*;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.UUID;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,6 +33,9 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
 import org.apache.hadoop.util.Progressable;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -91,7 +96,40 @@ public void testNonIdempotentOpsWithRetries() throws IOException {
     assertTrue("Couldn't delete the directory", result);
     fs.delete(rootDir, true);
   }
+  
+  @Test
+  public void testRenameAndSetModifyTime() throws Exception {
+    assertTrue(FSUtils.isHDFS(conf));
+
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = TEST_UTIL.getDataTestDir("testArchiveFile");
+
+    String file = UUID.randomUUID().toString();
+    Path p = new Path(testDir, file);
+
+    FSDataOutputStream out = fs.create(p);
+    out.close();
+    assertTrue("The created file should be present", FSUtils.isExists(fs, p));
+
+    long expect = System.currentTimeMillis() + 1000;
+    assertFalse(expect == fs.getFileStatus(p).getModificationTime());
 
+    ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge();
+    mockEnv.setValue(expect);
+    EnvironmentEdgeManager.injectEdge(mockEnv);
+
+    String dstFile = UUID.randomUUID().toString();
+    Path dst = new Path(testDir , dstFile);
+
+    assertTrue(HBaseFileSystem.renameAndSetModifyTime(fs, p, dst));
+    assertFalse("The moved file should not be present", FSUtils.isExists(fs,
+                                                                         p));
+    assertTrue("The dst file should be present", FSUtils.isExists(fs, dst));
+
+    assertEquals(expect, fs.getFileStatus(dst).getModificationTime());
+  }
+
+  
   static class MockFileSystemForCreate extends MockFileSystem {
     @Override
     public boolean exists(Path path) {

From 951ba1bb6555c9aa11a6782739e3de45d2c13531 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 24 May 2013 23:05:53 +0000
Subject: [PATCH 0993/1540] HBASE-8603 Backport HBASE-6921 to 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1486249 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/ClassSize.java | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
index 4d6cdfbb8603..61c9d6728165 100755
--- a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.util;
 
+import java.util.concurrent.ConcurrentHashMap;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -141,10 +143,14 @@ public class ClassSize {
 
     TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + align(7 * REFERENCE));
 
-    STRING = align(OBJECT + ARRAY + REFERENCE + ((JDK7? 2: 3) * Bytes.SIZEOF_INT));
+    // STRING is different size in jdk6 and jdk7. Just use what we estimate as size rather than
+    // have a conditional on whether jdk7.
+    STRING = (int) estimateBase(String.class, false);
 
-    CONCURRENT_HASHMAP = align(((JDK7? 3: 2) * Bytes.SIZEOF_INT) + ARRAY +
-        (6 * REFERENCE) + OBJECT);
+    // CONCURRENT_HASHMAP is different size in jdk6 and jdk7; it looks like its different between
+    // 23.6-b03 and 23.0-b21. Just use what we estimate as size rather than have a conditional on
+    // whether jdk7.
+    CONCURRENT_HASHMAP = (int) estimateBase(ConcurrentHashMap.class, false);
 
     CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) +
         (2 * Bytes.SIZEOF_INT));

From 3278f60b870240a3a6dd47b79a0a4c22c0120b17 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 29 May 2013 04:08:42 +0000
Subject: [PATCH 0994/1540] HBASE-8609 Make the CopyTable support startRow,
 stopRow options (Shaohui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1487228 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/CopyTable.java     |  56 +++++--
 .../hadoop/hbase/mapreduce/TestCopyTable.java | 148 ++++++++++++++++++
 2 files changed, 194 insertions(+), 10 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index d8e9860c5249..025473b74d18 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -19,13 +19,18 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -36,18 +41,23 @@
  * It is also configurable with a start and time as well as a specification
  * of the region server implementation if different from the local cluster.
  */
-public class CopyTable {
+public class CopyTable extends Configured implements Tool {
 
   final static String NAME = "copytable";
   static long startTime = 0;
   static long endTime = 0;
   static int versions = -1;
   static String tableName = null;
+  static String startRow = null;
+  static String stopRow = null;
   static String newTableName = null;
   static String peerAddress = null;
   static String families = null;
   static boolean allCells = false;
-
+  
+  public CopyTable(Configuration conf) {
+    super(conf);
+  }
   /**
    * Sets up the actual job.
    *
@@ -75,6 +85,15 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
     if (versions >= 0) {
       scan.setMaxVersions(versions);
     }
+    
+    if (startRow != null) {
+      scan.setStartRow(Bytes.toBytes(startRow));
+    }
+    
+    if (stopRow != null) {
+      scan.setStopRow(Bytes.toBytes(stopRow));
+    }
+    
     if(families != null) {
       String[] fams = families.split(",");
       Map<String,String> cfRenameMap = new HashMap<String,String>();
@@ -117,6 +136,8 @@ private static void printUsage(final String errorMsg) {
     System.err.println(" rs.class     hbase.regionserver.class of the peer cluster");
     System.err.println("              specify if different from current cluster");
     System.err.println(" rs.impl      hbase.regionserver.impl of the peer cluster");
+    System.err.println(" startrow     the start row");
+    System.err.println(" stoprow      the stop row");
     System.err.println(" starttime    beginning of the time range (unixtime in millis)");
     System.err.println("              without endtime means from starttime to forever");
     System.err.println(" endtime      end of the time range.  Ignored if no starttime specified.");
@@ -156,7 +177,19 @@ private static boolean doCommandLine(final String[] args) {
           printUsage(null);
           return false;
         }
-
+        
+        final String startRowArgKey = "--startrow=";
+        if (cmd.startsWith(startRowArgKey)) {
+          startRow = cmd.substring(startRowArgKey.length());
+          continue;
+        }
+        
+        final String stopRowArgKey = "--stoprow=";
+        if (cmd.startsWith(stopRowArgKey)) {
+          stopRow = cmd.substring(stopRowArgKey.length());
+          continue;
+        }
+        
         final String startTimeArgKey = "--starttime=";
         if (cmd.startsWith(startTimeArgKey)) {
           startTime = Long.parseLong(cmd.substring(startTimeArgKey.length()));
@@ -229,12 +262,15 @@ private static boolean doCommandLine(final String[] args) {
    * @throws Exception When running the job fails.
    */
   public static void main(String[] args) throws Exception {
-    Configuration conf = HBaseConfiguration.create();
-    String[] otherArgs =
-      new GenericOptionsParser(conf, args).getRemainingArgs();
-    Job job = createSubmittableJob(conf, otherArgs);
-    if (job != null) {
-      System.exit(job.waitForCompletion(true) ? 0 : 1);
-    }
+    int ret = ToolRunner.run(new CopyTable(HBaseConfiguration.create()), args);
+    System.exit(ret);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
+    Job job = createSubmittableJob(getConf(), otherArgs);
+    if (job == null) return 1;
+    return job.waitForCompletion(true) ? 0 : 1;
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
new file mode 100644
index 000000000000..e93d781b02bd
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Basic test for the CopyTable M/R tool
+ */
+@Category(LargeTests.class)
+public class TestCopyTable {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static MiniHBaseCluster cluster;
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    cluster = TEST_UTIL.startMiniCluster(3);
+    TEST_UTIL.startMiniMapReduceCluster();
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Simple end-to-end test
+   * @throws Exception
+   */
+  @Test
+  public void testCopyTable() throws Exception {
+    final byte[] TABLENAME1 = Bytes.toBytes("testCopyTable1");
+    final byte[] TABLENAME2 = Bytes.toBytes("testCopyTable2");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[] COLUMN1 = Bytes.toBytes("c1");
+
+    HTable t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
+    HTable t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
+
+    // put rows into the first table
+    for (int i = 0; i < 10; i++) {
+      Put p = new Put(Bytes.toBytes("row" + i));
+      p.add(FAMILY, COLUMN1, COLUMN1);
+      t1.put(p);
+    }
+
+    CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
+
+    assertEquals(
+      0,
+      copy.run(new String[] { "--new.name=" + Bytes.toString(TABLENAME2),
+          Bytes.toString(TABLENAME1) }));
+
+    // verify the data was copied into table 2
+    for (int i = 0; i < 10; i++) {
+      Get g = new Get(Bytes.toBytes("row" + i));
+      Result r = t2.get(g);
+      assertEquals(1, r.size());
+      assertTrue(Bytes.equals(COLUMN1, r.raw()[0].getQualifier()));
+    }
+
+    t1.close();
+    t2.close();
+    TEST_UTIL.deleteTable(TABLENAME1);
+    TEST_UTIL.deleteTable(TABLENAME2);
+  }
+
+  @Test
+  public void testStartStopRow() throws Exception {
+    final byte[] TABLENAME1 = Bytes.toBytes("testStartStopRow1");
+    final byte[] TABLENAME2 = Bytes.toBytes("testStartStopRow2");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[] COLUMN1 = Bytes.toBytes("c1");
+    final byte[] ROW0 = Bytes.toBytes("row0");
+    final byte[] ROW1 = Bytes.toBytes("row1");
+    final byte[] ROW2 = Bytes.toBytes("row2");
+
+    HTable t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
+    HTable t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
+
+    // put rows into the first table
+    Put p = new Put(ROW0);
+    p.add(FAMILY, COLUMN1, COLUMN1);
+    t1.put(p);
+    p = new Put(ROW1);
+    p.add(FAMILY, COLUMN1, COLUMN1);
+    t1.put(p);
+    p = new Put(ROW2);
+    p.add(FAMILY, COLUMN1, COLUMN1);
+    t1.put(p);
+
+    CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
+    assertEquals(
+      0,
+      copy.run(new String[] { "--new.name=" + Bytes.toString(TABLENAME2), "--startrow=row1",
+          "--stoprow=row2", Bytes.toString(TABLENAME1) }));
+
+    // verify the data was copied into table 2
+    // row1 exist, row0, row2 do not exist
+    Get g = new Get(ROW1);
+    Result r = t2.get(g);
+    assertEquals(1, r.size());
+    assertTrue(Bytes.equals(COLUMN1, r.raw()[0].getQualifier()));
+    
+    g = new Get(ROW0);
+    r = t2.get(g);
+    assertEquals(0, r.size());
+    
+    g = new Get(ROW2);
+    r = t2.get(g);
+    assertEquals(0, r.size());
+    
+    t1.close();
+    t2.close();
+    TEST_UTIL.deleteTable(TABLENAME1);
+    TEST_UTIL.deleteTable(TABLENAME2);
+  }
+}

From 81d752cc550f79b972b0504b64b7b2bafc79952e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 29 May 2013 19:43:23 +0000
Subject: [PATCH 0995/1540] HBASE-8639 very poor performance of
 htable#getscanner in multithreaded environment (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1487612 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/ScannerCallable.java  | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
index 27101509eb2a..0c4677fdea74 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
@@ -52,6 +52,14 @@ public class ScannerCallable extends ServerCallable<Result[]> {
   private ScanMetrics scanMetrics;
   private boolean logScannerActivity = false;
   private int logCutOffLatency = 1000;
+  private static String myAddress;
+  static {
+    try {
+      myAddress = DNS.getDefaultHost("default", "default");
+    } catch (UnknownHostException uhe) {
+      LOG.error("cannot determine my address", uhe);
+    }
+  }
 
   // indicate if it is a remote server call
   private boolean isRegionServerRemote = true;
@@ -99,10 +107,8 @@ public void connect(boolean reload) throws IOException {
   /**
    * compare the local machine hostname with region server's hostname
    * to decide if hbase client connects to a remote region server
-   * @throws UnknownHostException.
    */
-  private void checkIfRegionServerIsRemote() throws UnknownHostException {
-    String myAddress = DNS.getDefaultHost("default", "default");
+  private void checkIfRegionServerIsRemote() {
     if (this.location.getHostname().equalsIgnoreCase(myAddress)) {
       isRegionServerRemote = false;
     } else {

From 1d879faba278150251f099b21e7d2b11259c0dcd Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 30 May 2013 21:57:09 +0000
Subject: [PATCH 0996/1540] HBASE-8655 Backport to 94 - HBASE-8346(Prefetching
 .META. rows in case only when useCache is set to true) (Himanshu and Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1488034 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index e813f7a6dac1..5af6021d209a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1006,23 +1006,26 @@ private HRegionLocation locateRegionInMeta(final byte [] parentTable,
           // region at the same time. The first will load the meta region and
           // the second will use the value that the first one found.
           synchronized (regionLockObject) {
-            // If the parent table is META, we may want to pre-fetch some
-            // region info into the global region cache for this table.
-            if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME) &&
-                (getRegionCachePrefetch(tableName)) )  {
-              prefetchRegionCache(tableName, row);
-            }
-
             // Check the cache again for a hit in case some other thread made the
-            // same query while we were waiting on the lock. If not supposed to
-            // be using the cache, delete any existing cached location so it won't
-            // interfere.
+            // same query while we were waiting on the lock.
             if (useCache) {
               location = getCachedLocation(tableName, row);
               if (location != null) {
                 return location;
               }
+              // If the parent table is META, we may want to pre-fetch some
+              // region info into the global region cache for this table.
+              if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME)
+                  && (getRegionCachePrefetch(tableName))) {
+                prefetchRegionCache(tableName, row);
+              }
+              location = getCachedLocation(tableName, row);
+              if (location != null) {
+                return location;
+              }
             } else {
+              // If we are not supposed to be using the cache, delete any existing cached location
+              // so it won't interfere.
               deleteCachedLocation(tableName, row);
             }
 

From 680743e3d1d56158dbb0f33dda97abc49762d616 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 3 Jun 2013 03:51:52 +0000
Subject: [PATCH 0997/1540] HBASE-8640 ServerName in master may not initialize
 with the configured ipc address of hbase.master.ipc.address

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1488837 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/HMaster.java | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 8c62fc1b84b5..1eff890517fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -284,23 +284,16 @@ public HMaster(final Configuration conf)
     // Set how many times to retry talking to another server over HConnection.
     HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
     // Server to handle client requests.
-    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
-      conf.get("hbase.master.dns.interface", "default"),
-      conf.get("hbase.master.dns.nameserver", "default")));
+    String hostname = conf.get("hbase.master.ipc.address",
+      Strings.domainNamePointerToHostName(DNS.getDefaultHost(
+        conf.get("hbase.master.dns.interface", "default"),
+        conf.get("hbase.master.dns.nameserver", "default"))));
     int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
     // Test that the hostname is reachable
     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
     if (initialIsa.getAddress() == null) {
       throw new IllegalArgumentException("Failed resolve of hostname " + initialIsa);
     }
-    // Verify that the bind address is reachable if set
-    String bindAddress = conf.get("hbase.master.ipc.address");
-    if (bindAddress != null) {
-      initialIsa = new InetSocketAddress(bindAddress, port);
-      if (initialIsa.getAddress() == null) {
-        throw new IllegalArgumentException("Failed resolve of bind address " + initialIsa);
-      }
-    }
     int numHandlers = conf.getInt("hbase.master.handler.count",
       conf.getInt("hbase.regionserver.handler.count", 25));
     this.rpcServer = HBaseRPC.getServer(this,
@@ -313,7 +306,7 @@ public HMaster(final Configuration conf)
         0); // this is a DNC w/o high priority handlers
     // Set our address.
     this.isa = this.rpcServer.getListenerAddress();
-    this.serverName = new ServerName(hostname,
+    this.serverName = new ServerName(this.isa.getHostName(),
       this.isa.getPort(), System.currentTimeMillis());
     this.rsFatals = new MemoryBoundedLogMessageBuffer(
         conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));

From f9902d702cb77baf10d971b3a5fcd840109efd9c Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Tue, 4 Jun 2013 02:16:41 +0000
Subject: [PATCH 0998/1540] HBASE-8671: Per-region WAL breaks CP backwards
 compatibility in 0.94 for non-enabled case

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1489274 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/RegionServerServices.java      | 3 +++
 .../apache/hadoop/hbase/util/MockRegionServerServices.java   | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index c3473c6a1225..5927af9ee526 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -41,6 +41,9 @@ public interface RegionServerServices extends OnlineRegions {
    * default (common) WAL */
   public HLog getWAL(HRegionInfo regionInfo) throws IOException;
 
+  /** @return get the default (common) WAL for the server*/
+  public HLog getWAL() throws IOException;
+
   /**
    * @return Implementation of {@link CompactionRequestor} or null.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
index 09c954959e6f..70a0e3d64711 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
@@ -86,6 +86,11 @@ public HLog getWAL(HRegionInfo regionInfo) throws IOException {
     return null;
   }
 
+  @Override
+  public HLog getWAL() throws IOException {
+    return null;
+  }
+
   @Override
   public RpcServer getRpcServer() {
     return null;

From 2693e05474a4cd3d752c975f2d181d86fb505a59 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 4 Jun 2013 18:55:59 +0000
Subject: [PATCH 0999/1540] move version forward

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1489563 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 221e8cd25d0d..61152caf7f43 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.8</version>
+  <version>0.94.9-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From dae6da49045c45e3568335551789ecd866e4196d Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Wed, 5 Jun 2013 03:14:41 +0000
Subject: [PATCH 1000/1540] HBASE-8590 [0.94] BlockingMetaScannerVisitor should
 check for parent meta entry while waiting for split daughter

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1489691 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/MetaScanner.java      | 62 ++++++++++++++-----
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 80531cc2251d..360de34844dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
+import org.apache.hadoop.hbase.errorhandling.TimeoutException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
 
@@ -411,27 +412,35 @@ public boolean processRow(Result rowResult) throws IOException {
         HTable metaTable = getMetaTable();
         long start = System.currentTimeMillis();
         if (splitA != null) {
-          Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
-              splitA.getRegionName());
-          if (resultA != null) {
-            processRow(resultA);
-            daughterRegions.add(splitA.getRegionName());
-          } else {
+          try {
+            Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
+              info.getRegionName(), splitA.getRegionName());
+            if (resultA != null) {
+              processRow(resultA);
+              daughterRegions.add(splitA.getRegionName());
+            }
+            // else parent is gone, so skip this daughter
+          } catch (TimeoutException e) {
             throw new RegionOfflineException("Split daughter region " +
-                splitA.getRegionNameAsString() + " cannot be found in META.");
+                splitA.getRegionNameAsString() + " cannot be found in META. Parent:" +
+                info.getRegionNameAsString());
           }
         }
         long rem = blockingTimeout - (System.currentTimeMillis() - start);
 
         if (splitB != null) {
-          Result resultB = getRegionResultBlocking(metaTable, rem,
-              splitB.getRegionName());
-          if (resultB != null) {
-            processRow(resultB);
-            daughterRegions.add(splitB.getRegionName());
-          } else {
+          try {
+            Result resultB = getRegionResultBlocking(metaTable, rem,
+              info.getRegionName(), splitB.getRegionName());
+            if (resultB != null) {
+              processRow(resultB);
+              daughterRegions.add(splitB.getRegionName());
+            }
+            // else parent is gone, so skip this daughter
+          } catch (TimeoutException e) {
             throw new RegionOfflineException("Split daughter region " +
-                splitB.getRegionNameAsString() + " cannot be found in META.");
+                splitB.getRegionNameAsString() + " cannot be found in META. Parent:" +
+                info.getRegionNameAsString());
           }
         }
       }
@@ -439,8 +448,15 @@ public boolean processRow(Result rowResult) throws IOException {
       return processRowInternal(rowResult);
     }
 
-    private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] regionName)
-        throws IOException {
+    /**
+     * Returns region Result by querying the META table for regionName. It will block until
+     * the region is found in META. It will also check for parent in META to make sure that
+     * if parent is deleted, we no longer have to wait, and should continue (HBASE-8590)
+     * @return Result object is daughter is found, or null if parent is gone from META
+     * @throws TimeoutException if timeout is reached
+     */
+    private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] parentRegionName, byte[] regionName)
+        throws IOException, TimeoutException {
       boolean logged = false;
       long start = System.currentTimeMillis();
       while (System.currentTimeMillis() - start < timeout) {
@@ -451,6 +467,17 @@ private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] re
         if (info != null) {
           return result;
         }
+
+        // check whether parent is still there, if not it means we do not need to wait
+        Get parentGet = new Get(parentRegionName);
+        Result parentResult = metaTable.get(parentGet);
+        HRegionInfo parentInfo = Writables.getHRegionInfoOrNull(
+            parentResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
+        if (parentInfo == null) {
+          // this means that parent is no more (catalog janitor or somebody else deleted it)
+          return null;
+        }
+
         try {
           if (!logged) {
             if (LOG.isDebugEnabled()) {
@@ -464,7 +491,8 @@ private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] re
           break;
         }
       }
-      return null;
+      throw new TimeoutException("getRegionResultBlocking", start, System.currentTimeMillis(),
+        timeout);
     }
   }
 

From c52a035e579021773370968c7c9298d98bec48e7 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 5 Jun 2013 23:10:55 +0000
Subject: [PATCH 1001/1540] HBASE-8684: Table Coprocessor can't access external
 HTable by default

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1490068 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/CompoundConfiguration.java   |  83 +++++++++++-
 .../regionserver/RegionCoprocessorHost.java   |   5 +-
 .../hbase/TestOpenTableInCoprocessor.java     | 122 ++++++++++++++++++
 .../TestCompoundConfiguration.java            |  49 +++++++
 4 files changed, 252 insertions(+), 7 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
index 0197a602b3e0..def1cb66e090 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
@@ -25,10 +25,13 @@
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
+import org.apache.commons.collections.iterators.UnmodifiableIterator;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -61,7 +64,7 @@ public CompoundConfiguration() {
 
   // Devs: these APIs are the same contract as their counterparts in
   // Configuration.java
-  private static interface ImmutableConfigMap {
+  private static interface ImmutableConfigMap extends Iterable<Map.Entry<String,String>> {
     String get(String key);
     String getRaw(String key);
     Class<?> getClassByName(String name) throws ClassNotFoundException;
@@ -114,6 +117,11 @@ public int size() {
       public String toString() {
         return c.toString();
       }
+
+      @Override
+      public Iterator<Entry<String, String>> iterator() {
+        return c.iterator();
+      }
     });
     return this;
   }
@@ -166,6 +174,55 @@ public int size() {
       public String toString() {
         return m.toString();
       }
+
+      @Override
+      public Iterator<Entry<String, String>> iterator() {
+        final Iterator<Entry<ImmutableBytesWritable, ImmutableBytesWritable>> entries = m
+            .entrySet().iterator();
+        return new Iterator<Entry<String, String>>() {
+
+          @Override
+          public boolean hasNext() {
+            return entries.hasNext();
+          }
+
+          @Override
+          public Entry<String, String> next() {
+            final Entry<ImmutableBytesWritable, ImmutableBytesWritable> e = entries.next();
+            return new Entry<String, String>() {
+
+              @Override
+              public String setValue(String value) {
+                throw new UnsupportedOperationException(
+                    "Cannot set value on entry from a CompoundConfiguration!");
+              }
+
+              @Override
+              public String getValue() {
+                ImmutableBytesWritable bytes = e.getValue();
+                // unlike regular configuration, ImmutableBytesWritableMaps can take a null value
+                if (bytes != null) {
+                  return Bytes.toString(bytes.get(), bytes.getOffset(), bytes.getLength());
+                }
+                return null;
+              }
+
+              @Override
+              public String getKey() {
+                ImmutableBytesWritable bytes = e.getKey();
+                return Bytes.toString(bytes.get(), bytes.getOffset(), bytes.getLength());
+              }
+            };
+          }
+
+          @Override
+          public void remove() {
+            throw new UnsupportedOperationException(
+                "Cannot remove an entry from a CompoundConfiguration iterator");
+          }
+        };
+
+      }
     });
     return this;
   }
@@ -227,6 +284,25 @@ public int size() {
     return ret;
   }
 
+  @Override
+  public Iterator<Map.Entry<String, String>> iterator() {
+    Map<String, String> ret = new HashMap<String, String>();
+
+    // add in reverse order so that oldest get overridden.
+    if (!configs.isEmpty()) {
+      for (int i = configs.size() - 1; i >= 0; i--) {
+        ImmutableConfigMap map = configs.get(i);
+        Iterator<Map.Entry<String, String>> iter = map.iterator();
+        while (iter.hasNext()) {
+          Map.Entry<String, String> entry = iter.next();
+          ret.put(entry.getKey(), entry.getValue());
+        }
+      }
+    }
+
+    return UnmodifiableIterator.decorate(ret.entrySet().iterator());
+  }
+
   /***************************************************************************
    * You should just ignore everything below this line unless there's a bug in
    * Configuration.java...
@@ -403,11 +479,6 @@ public void clear() {
     throw new UnsupportedOperationException("Immutable Configuration");
   }
 
-  @Override
-  public Iterator<Map.Entry<String, String>> iterator() {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-
   @Override
   public void set(String name, String value) {
     throw new UnsupportedOperationException("Immutable Configuration");
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 0da7b2d0fb0c..407ca240d6f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -182,7 +183,9 @@ void loadTableCoprocessors(final Configuration conf) {
             }
             if (cfgSpec != null) {
               cfgSpec = cfgSpec.substring(cfgSpec.indexOf('|') + 1);
-              Configuration newConf = new Configuration(conf);
+              // do an explicit deep copy of the passed configuration
+              Configuration newConf = new Configuration(false);
+              HBaseConfiguration.merge(newConf, conf);
               Matcher m = HConstants.CP_HTD_ATTR_VALUE_PARAM_PATTERN.matcher(cfgSpec);
               while (m.find()) {
                 newConf.set(m.group(1), m.group(2));
diff --git a/src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java
new file mode 100644
index 000000000000..c24d1169d0cb
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that a coprocessor can open a connection and write to another table, inside a hook.
+ */
+@Category(MediumTests.class)
+public class TestOpenTableInCoprocessor {
+
+  private static final byte[] otherTable = Bytes.toBytes("otherTable");
+  private static final byte[] family = new byte[] { 'f' };
+
+  private static boolean completed = false;
+
+  /**
+   * Custom coprocessor that just copies the write to another table.
+   */
+  public static class SendToOtherTableCoprocessor extends BaseRegionObserver {
+
+    @Override
+    public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
+        boolean writeToWAL) throws IOException {
+      HTableInterface table = e.getEnvironment().getTable(otherTable);
+      Put p = new Put(new byte[] { 'a' });
+      p.add(family, null, new byte[] { 'a' });
+      table.put(put);
+      table.flushCommits();
+      completed = true;
+      table.close();
+    }
+
+  }
+
+  @Test
+  public void testCoprocessorCanCreateConnectionToRemoteTable() throws Throwable {
+    HBaseTestingUtility UTIL = new HBaseTestingUtility();
+    HTableDescriptor primary = new HTableDescriptor("primary");
+    primary.addFamily(new HColumnDescriptor(family));
+    // add our coprocessor
+    primary.addCoprocessor(SendToOtherTableCoprocessor.class.getName());
+
+    HTableDescriptor other = new HTableDescriptor(otherTable);
+    other.addFamily(new HColumnDescriptor(family));
+    UTIL.startMiniCluster();
+
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    admin.createTable(primary);
+    admin.createTable(other);
+    admin.close();
+
+    HTable table = new HTable(UTIL.getConfiguration(), "primary");
+    Put p = new Put(new byte[] { 'a' });
+    p.add(family, null, new byte[] { 'a' });
+    table.put(p);
+    table.flushCommits();
+    table.close();
+
+    HTable target = new HTable(UTIL.getConfiguration(), otherTable);
+    assertTrue("Didn't complete update to target table!", completed);
+    assertEquals("Didn't find inserted row", 1, getKeyValueCount(target));
+    target.close();
+
+    UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Count the number of keyvalue in the table. Scans all possible versions
+   * @param table table to scan
+   * @return number of keyvalues over all rows in the table
+   * @throws IOException
+   */
+  private int getKeyValueCount(HTable table) throws IOException {
+    Scan scan = new Scan();
+    scan.setMaxVersions(Integer.MAX_VALUE - 1);
+
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      count += res.list().size();
+      System.out.println(count + ") " + res);
+    }
+    results.close();
+
+    return count;
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
index 12bc40b3fedc..cff9d8aab13f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
@@ -36,6 +36,7 @@
 @Category(SmallTests.class)
 public class TestCompoundConfiguration extends TestCase {
   private Configuration baseConf;
+  private int baseConfSize;
 
   @Override
   protected void setUp() throws Exception {
@@ -43,6 +44,7 @@ protected void setUp() throws Exception {
     baseConf.set("A", "1");
     baseConf.setInt("B", 2);
     baseConf.set("C", "3");
+    baseConfSize = baseConf.size();
   }
 
   @Test
@@ -81,6 +83,15 @@ public void testWithConfig() {
     assertEquals(4, compoundConf.getInt("D", 0));
     assertNull(compoundConf.get("E"));
     assertEquals(6, compoundConf.getInt("F", 6));
+
+    int cnt = 0;
+    for (Map.Entry<String,String> entry : compoundConf) {
+      cnt++;
+      if (entry.getKey().equals("B")) assertEquals("2b", entry.getValue());
+      else if (entry.getKey().equals("G")) assertEquals(null, entry.getValue());
+    }
+    // verify that entries from ImmutableConfigMap's are merged in the iterator's view
+    assertEquals(baseConfSize + 1, cnt);
   }
 
   private ImmutableBytesWritable strToIbw(String s) {
@@ -108,6 +119,44 @@ public void testWithIbwMap() {
     assertNull(compoundConf.get("E"));
     assertEquals(6, compoundConf.getInt("F", 6));
     assertNull(compoundConf.get("G"));
+
+    int cnt = 0;
+    for (Map.Entry<String,String> entry : compoundConf) {
+      cnt++;
+      if (entry.getKey().equals("B")) assertEquals("2b", entry.getValue());
+      else if (entry.getKey().equals("G")) assertEquals(null, entry.getValue());
+    }
+    // verify that entries from ImmutableConfigMap's are merged in the iterator's view
+    assertEquals(baseConfSize + 2, cnt);
+  }
+
+  @Test
+  public void testLaterConfigsOverrideEarlier() {
+    Configuration map1 = new Configuration(false);
+    map1.set("A", "2");
+    map1.set("D", "5");
+    Configuration map2 = new Configuration(false);
+    String newValueForA = "3", newValueForB = "4";
+    map2.set("A", newValueForA);
+    map2.set("B", newValueForB);
+
+    CompoundConfiguration compoundConf = new CompoundConfiguration()
+      .add(map1).add(baseConf);
+    assertEquals("1", compoundConf.get("A"));
+    assertEquals("5", compoundConf.get("D"));
+    compoundConf.add(map2);
+    assertEquals(newValueForA, compoundConf.get("A"));
+    assertEquals(newValueForB, compoundConf.get("B"));
+    assertEquals("5", compoundConf.get("D"));
+
+    int cnt = 0;
+    for (Map.Entry<String,String> entry : compoundConf) {
+      cnt++;
+      if (entry.getKey().equals("A")) assertEquals(newValueForA, entry.getValue());
+      else if (entry.getKey().equals("B")) assertEquals(newValueForB, entry.getValue());
+    }
+    // verify that entries from ImmutableConfigMap's are merged in the iterator's view
+    assertEquals(baseConfSize + 1, cnt);
   }
 
   @org.junit.Rule

From e7c75c9550c3d95741044352d25f37b6ab5b18c1 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 5 Jun 2013 23:21:53 +0000
Subject: [PATCH 1002/1540] HBASE-8684: Table Coprocessor can't access external
 HTable by default - ADDENDUM: moving test to correct package

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1490076 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/{ => coprocessor}/TestOpenTableInCoprocessor.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
 rename src/test/java/org/apache/hadoop/hbase/{ => coprocessor}/TestOpenTableInCoprocessor.java (94%)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
similarity index 94%
rename from src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java
rename to src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
index c24d1169d0cb..414bc25de852 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestOpenTableInCoprocessor.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
@@ -15,13 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.coprocessor;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;

From beaef04358c53699a4e5d11c2942f7e2696a8fb8 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 5 Jun 2013 23:31:26 +0000
Subject: [PATCH 1003/1540] HBASE-8636: Backport KeyValue Codec to 0.94
 (HBASE-7413)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1490079 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/codec/BaseDecoder.java       | 77 ++++++++++++++++
 .../hadoop/hbase/codec/BaseEncoder.java       | 47 ++++++++++
 .../org/apache/hadoop/hbase/codec/Codec.java  | 36 ++++++++
 .../hadoop/hbase/codec/CodecException.java    | 43 +++++++++
 .../apache/hadoop/hbase/codec/Decoder.java    | 60 +++++++++++++
 .../apache/hadoop/hbase/codec/Encoder.java    | 55 ++++++++++++
 .../hadoop/hbase/codec/KeyValueCodec.java     | 88 +++++++++++++++++++
 .../hadoop/hbase/regionserver/wal/HLog.java   |  9 --
 .../regionserver/wal/KeyValueCompression.java | 32 +++++++
 .../wal/SequenceFileLogReader.java            |  9 +-
 .../wal/SequenceFileLogWriter.java            |  8 +-
 .../hbase/regionserver/wal/WALEdit.java       | 74 +++++++++-------
 .../hbase/regionserver/wal/WALEditCodec.java  | 55 ++++++++++++
 .../ReplicationHLogReaderManager.java         |  5 +-
 .../wal/FaultySequenceFileLogReader.java      |  3 +-
 15 files changed, 556 insertions(+), 45 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/Codec.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/CodecException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/Decoder.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/Encoder.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java

diff --git a/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java b/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
new file mode 100644
index 000000000000..59d6aa897edb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.KeyValue;
+
+public abstract class BaseDecoder implements Decoder {
+  protected static final Log LOG = LogFactory.getLog(BaseDecoder.class);
+  protected final InputStream in;
+  private boolean hasNext = true;
+  protected KeyValue current = null;
+
+  public BaseDecoder(final InputStream in) {
+    this.in = in;
+  }
+
+  @Override
+  public boolean advance() throws IOException {
+    if (!this.hasNext) return this.hasNext;
+    if (this.in.available() == 0) {
+      this.hasNext = false;
+      return this.hasNext;
+    }
+    try {
+      this.current = parseCell();
+    } catch (IOException ioEx) {
+      rethrowEofException(ioEx);
+    }
+    return this.hasNext;
+  }
+
+  private void rethrowEofException(IOException ioEx) throws IOException {
+    boolean isEof = false;
+    try {
+      isEof = this.in.available() == 0;
+    } catch (Throwable t) {
+      LOG.trace("Error getting available for error message - ignoring", t);
+    }
+    if (!isEof) throw ioEx;
+    LOG.error("Partial cell read caused by EOF: " + ioEx);
+    EOFException eofEx = new EOFException("Partial cell read");
+    eofEx.initCause(ioEx);
+    throw eofEx;
+  }
+
+  /**
+   * @return extract a Cell
+   * @throws IOException
+   */
+  protected abstract KeyValue parseCell() throws IOException;
+
+  @Override
+  public KeyValue current() {
+    return this.current;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java b/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java
new file mode 100644
index 000000000000..2be2a6a6125c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+public abstract class BaseEncoder implements Encoder {
+  protected final OutputStream out;
+  // This encoder is 'done' once flush has been called.
+  protected boolean flushed = false;
+
+  public BaseEncoder(final OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public abstract void write(KeyValue cell) throws IOException;
+
+  protected void checkFlushed() throws CodecException {
+    if (this.flushed) throw new CodecException("Flushed; done");
+  }
+
+  @Override
+  public void flush() throws IOException {
+    if (this.flushed) return;
+    this.flushed = true;
+    this.out.flush();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/codec/Codec.java b/src/main/java/org/apache/hadoop/hbase/codec/Codec.java
new file mode 100644
index 000000000000..07936906766e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/Codec.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * Encoder/Decoder for KeyValue.
+ * <p>
+ * This is a backport of the Codec from HBase 0.96, but ignoring support for a Cell interface and
+ * sticking with the good, old {@link KeyValue}.
+ */
+public interface Codec {
+
+  Decoder getDecoder(InputStream is);
+
+  Encoder getEncoder(OutputStream os);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java b/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java
new file mode 100644
index 000000000000..8124d9e33781
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+
+import org.apache.hadoop.hbase.HBaseIOException;
+
+/**
+ * Thrown when problems in the codec whether setup or context.
+ */
+@SuppressWarnings("serial")
+public class CodecException extends HBaseIOException {
+  public CodecException() {
+    super();
+  }
+
+  public CodecException(String message) {
+    super(message);
+  }
+
+  public CodecException(Throwable t) {
+    super(t);
+  }
+
+  public CodecException(String message, Throwable t) {
+    super(message, t);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/codec/Decoder.java b/src/main/java/org/apache/hadoop/hbase/codec/Decoder.java
new file mode 100644
index 000000000000..ebeae482f44a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/Decoder.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * An interface for iterating through a sequence of KeyValues. Similar to Java's Iterator, but
+ * without the hasNext() or remove() methods. The hasNext() method is problematic because it may
+ * require actually loading the next object, which in turn requires storing the previous object
+ * somewhere.
+ * <p>
+ * The core data block decoder should be as fast as possible, so we push the complexity and
+ * performance expense of concurrently tracking multiple cells to layers above the {@link Decoder}.
+ * <p>
+ * The {@link #current()} method will return a reference to a the decodable type.
+ * <p/>
+ * Typical usage:
+ *
+ * <pre>
+ * while (scanner.next()) {
+ *   KeyValue kv = scanner.get();
+ *   // do something
+ * }
+ * </pre>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface Decoder {
+  /**
+   * @return the current object which may be mutable
+   */
+  KeyValue current();
+
+  /**
+   * Advance the scanner 1 object
+   * @return true if the next cell is found and {@link #current()} will return a valid object
+   * @throws IOException if there is an error reading the next entry
+   */
+  boolean advance() throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/codec/Encoder.java b/src/main/java/org/apache/hadoop/hbase/codec/Encoder.java
new file mode 100644
index 000000000000..9f2f211c56b2
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/Encoder.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.codec;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * Accepts a stream of KeyValues. This can be used to anywhere KeyValues need to be written out, but
+ * currently it is only used for serializing WALEdits. This could be backed by a List<KeyValue>, but
+ * more efficient implementations will append results to a byte[] to eliminate overhead, and
+ * possibly encode the underlying data further.
+ * <p>
+ * To read the data back, use a corresponding {@link Decoder}
+ * @see Decoder
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface Encoder {
+  /**
+   * Implementation must copy the entire state of the cell. If the written cell is modified
+   * immediately after the write method returns, the modifications must have absolutely no effect on
+   * the copy of the cell that was added in the write.
+   * @param cell cell to serialize
+   * @throws IOException
+   */
+  void write(KeyValue cell) throws IOException;
+
+  /**
+   * Let the implementation decide what to do.  Usually means writing accumulated data into a
+   * byte[] that can then be read from the implementation to be sent to disk, put in the block
+   * cache, or sent over the network.
+   * @throws IOException
+   */
+  void flush() throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java b/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java
new file mode 100644
index 000000000000..f5452598fc3e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * Codec that does KeyValue version 1 serialization.
+ * <p>Encodes by casting Cell to KeyValue and writing out the backing array with a length prefix.
+ * This is how KVs were serialized in Puts, Deletes and Results pre-0.96.  Its what would
+ * happen if you called the Writable#write KeyValue implementation.  This encoder will fail
+ * if the passed Cell is not an old-school pre-0.96 KeyValue.  Does not copy bytes writing.
+ * It just writes them direct to the passed stream.
+ * <p>If you wrote two KeyValues to this encoder, it would look like this in the stream:
+ * <pre>
+ * length-of-KeyValue1 // A java int with the length of KeyValue1 backing array
+ * KeyValue1 backing array filled with a KeyValue serialized in its particular format
+ * length-of-KeyValue2
+ * KeyValue2 backing array
+ * </pre>
+ */
+public class KeyValueCodec implements Codec {
+  public static class KeyValueEncoder extends BaseEncoder {
+    public KeyValueEncoder(final DataOutputStream out) {
+      super(out);
+    }
+
+    @Override
+    public void write(KeyValue kv) throws IOException {
+      checkFlushed();
+      kv.write((DataOutputStream) out);
+    }
+  }
+
+  public static class KeyValueDecoder extends BaseDecoder{
+    public KeyValueDecoder(final DataInputStream in) {
+      super(in);
+    }
+
+    @Override
+    protected KeyValue parseCell() throws IOException {
+      KeyValue kv = new KeyValue();
+      kv.readFields((DataInputStream) this.in);
+      return kv;
+    }
+  }
+
+  /**
+   * Implementation depends on {@link InputStream#available()}
+   * <p>
+   * Must be passed a {@link DataInputStream} so KeyValues can be derserialized with the usual
+   * Writable mechanisms
+   */
+  @Override
+  public Decoder getDecoder(InputStream is) {
+    return new KeyValueDecoder((DataInputStream) is);
+  }
+
+  /**
+   * Must be passed a {@link DataOutputStream} so KeyValues can be serialized using the usual
+   * Writable mechanisms
+   */
+  @Override
+  public Encoder getEncoder(OutputStream os) {
+    return new KeyValueEncoder((DataOutputStream) os);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index cb9783c8a42d..4e792b20dc2c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1739,15 +1739,6 @@ public HLogKey getKey() {
       return key;
     }
 
-    /**
-     * Set compression context for this entry.
-     * @param compressionContext Compression context
-     */
-    public void setCompressionContext(CompressionContext compressionContext) {
-      edit.setCompressionContext(compressionContext);
-      key.setCompressionContext(compressionContext);
-    }
-
     @Override
     public String toString() {
       return this.key + "=" + this.edit;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
index 0f9743f70a09..c026f6e7ec79 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
@@ -19,10 +19,14 @@
 package org.apache.hadoop.hbase.regionserver.wal;
 
 import java.io.DataInput;
+import java.io.DataInputStream;
 import java.io.DataOutput;
+import java.io.DataOutputStream;
 import java.io.IOException;
 
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.codec.BaseDecoder;
+import org.apache.hadoop.hbase.codec.BaseEncoder;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.WritableUtils;
 
@@ -123,4 +127,32 @@ public static void writeKV(final DataOutput out, KeyValue keyVal,
     int remainingLength = keyVal.getLength() + offset - (pos);
     out.write(backingArray, pos, remainingLength);
   }
+
+  static class CompressedKvEncoder extends BaseEncoder {
+    private final CompressionContext compression;
+
+    public CompressedKvEncoder(DataOutputStream out, CompressionContext compression) {
+      super(out);
+      this.compression = compression;
+    }
+
+    @Override
+    public void write(KeyValue kv) throws IOException {
+      KeyValueCompression.writeKV((DataOutputStream) out, kv, compression);
+    }
+  }
+
+  static class CompressedKvDecoder extends BaseDecoder {
+    private final CompressionContext compression;
+
+    public CompressedKvDecoder(DataInputStream in, CompressionContext compression) {
+      super(in);
+      this.compression = compression;
+    }
+
+    @Override
+    protected KeyValue parseCell() throws IOException {
+      return KeyValueCompression.readKV((DataInputStream) in, compression);
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index a56be623d004..c167ed2605fe 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -152,6 +152,7 @@ public long getPos() throws IOException {
   protected CompressionContext compressionContext = null;
 
   protected Class<? extends HLogKey> keyClass;
+  private WALEditCodec codec;
 
   /**
    * Default constructor.
@@ -175,6 +176,7 @@ public void init(FileSystem fs, Path path, Configuration conf)
     this.conf = conf;
     this.path = path;
     reader = new WALReader(fs, path, conf);
+
     this.fs = fs;
 
     // If compression is enabled, new dictionaries are created here.
@@ -190,6 +192,9 @@ public void init(FileSystem fs, Path path, Configuration conf)
         throw new IOException("Failed to initialize CompressionContext", e);
       }
     }
+
+    // setup the codec
+    this.codec = new WALEditCodec(compressionContext);
   }
 
   @Override
@@ -230,10 +235,12 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
       WALEdit val = new WALEdit();
       e = new HLog.Entry(key, val);
     }
+
     boolean b = false;
     try {
+      e.getEdit().setCodec(codec);
       if (compressionContext != null) {
-        e.setCompressionContext(compressionContext);
+        e.getKey().setCompressionContext(compressionContext);
       }
       b = this.reader.next(e.getKey(), e.getEdit());
     } catch (IOException ioe) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index 0c09c834d256..5a49c5252b6e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -75,6 +75,7 @@ public class SequenceFileLogWriter implements HLog.Writer {
 
   private Method syncFs = null;
   private Method hflush = null;
+  private WALEditCodec codec;
 
   /**
    * Default constructor.
@@ -193,7 +194,8 @@ SequenceFile.CompressionType.NONE, new DefaultCodec(),
     } else {
       LOG.debug("using new createWriter -- HADOOP-6840");
     }
-    
+
+    this.codec = new WALEditCodec(compressionContext);
     this.writer_out = getSequenceFilePrivateFSDataOutputStreamAccessible();
     this.syncFs = getSyncFs();
     this.hflush = getHFlush();
@@ -274,7 +276,9 @@ private FSDataOutputStream getSequenceFilePrivateFSDataOutputStreamAccessible()
 
   @Override
   public void append(HLog.Entry entry) throws IOException {
-    entry.setCompressionContext(compressionContext);
+    entry.getEdit().setCodec(this.codec);
+    entry.getKey().setCompressionContext(compressionContext);
+
     try {
       this.writer.append(entry.getKey(), entry.getEdit());
     } catch (NullPointerException npe) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index efd5a32cf68e..2b946f4c571b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -20,15 +20,19 @@
 package org.apache.hadoop.hbase.regionserver.wal;
 
 import java.io.DataInput;
+import java.io.DataInputStream;
 import java.io.DataOutput;
+import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 
-import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.codec.Decoder;
+import org.apache.hadoop.hbase.codec.Encoder;
+import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.Writable;
@@ -74,15 +78,28 @@ public class WALEdit implements Writable, HeapSize {
   private final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
   private NavigableMap<byte[], Integer> scopes;
 
-  private CompressionContext compressionContext;
+  // default to decoding uncompressed data - needed for replication, which enforces that
+  // uncompressed edits are sent across the wire. In the regular case (reading/writing WAL), the
+  // codec will be setup by the reader/writer class, not here.
+  private WALEditCodec codec = new WALEditCodec(null);
 
   public WALEdit() {
   }
 
-  public void setCompressionContext(final CompressionContext compressionContext) {
-    this.compressionContext = compressionContext;
+  /**
+   * {@link #setCodec(WALEditCodec)} must be called before calling this method.
+   * @param compression the {@link CompressionContext} for the underlying codec.
+   */
+  @SuppressWarnings("javadoc")
+  public void setCompressionContext(final CompressionContext compression) {
+    this.codec.setCompression(compression);
   }
 
+  public void setCodec(WALEditCodec codec) {
+    this.codec = codec;
+  }
+
+
   public void add(KeyValue kv) {
     this.kvs.add(kv);
   }
@@ -115,19 +132,22 @@ public void readFields(DataInput in) throws IOException {
     if (scopes != null) {
       scopes.clear();
     }
+    Decoder decoder = this.codec.getDecoder((DataInputStream) in);
     int versionOrLength = in.readInt();
+    int length = versionOrLength;
+
+    // make sure we get the real length
+    if (versionOrLength == VERSION_2) {
+      length = in.readInt();
+    }
+
+    // read in all the key values
+    for(int i=0; i< length && decoder.advance(); i++) {
+      kvs.add(decoder.current());
+    }
+
+    //its a new style WAL, so we need replication scopes too
     if (versionOrLength == VERSION_2) {
-      // this is new style HLog entry containing multiple KeyValues.
-      int numEdits = in.readInt();
-      for (int idx = 0; idx < numEdits; idx++) {
-        if (compressionContext != null) {
-          this.add(KeyValueCompression.readKV(in, compressionContext));
-        } else {
-          KeyValue kv = new KeyValue();
-          kv.readFields(in);
-          this.add(kv);
-    	  }
-      }
       int numFamilies = in.readInt();
       if (numFamilies > 0) {
         if (scopes == null) {
@@ -139,27 +159,20 @@ public void readFields(DataInput in) throws IOException {
           scopes.put(fam, scope);
         }
       }
-    } else {
-      // this is an old style HLog entry. The int that we just
-      // read is actually the length of a single KeyValue
-      KeyValue kv = new KeyValue();
-      kv.readFields(versionOrLength, in);
-      this.add(kv);
     }
-
   }
 
   public void write(DataOutput out) throws IOException {
+    Encoder kvEncoder = codec.getEncoder((DataOutputStream) out);
     out.writeInt(VERSION_2);
+
+    //write out the keyvalues
     out.writeInt(kvs.size());
-    // We interleave the two lists for code simplicity
-    for (KeyValue kv : kvs) {
-      if (compressionContext != null) {
-        KeyValueCompression.writeKV(out, kv, compressionContext);
-      } else{
-        kv.write(out);
-      }
+    for(KeyValue kv: kvs){
+      kvEncoder.write(kv);
     }
+    kvEncoder.flush();
+
     if (scopes == null) {
       out.writeInt(0);
     } else {
@@ -198,5 +211,4 @@ public String toString() {
     sb.append(">]");
     return sb.toString();
   }
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java
new file mode 100644
index 000000000000..1b9adaabf101
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.codec.Codec;
+import org.apache.hadoop.hbase.codec.Decoder;
+import org.apache.hadoop.hbase.codec.Encoder;
+import org.apache.hadoop.hbase.codec.KeyValueCodec;
+import org.apache.hadoop.hbase.regionserver.wal.KeyValueCompression.CompressedKvEncoder;
+
+public class WALEditCodec implements Codec {
+  private CompressionContext compression;
+
+  public WALEditCodec(CompressionContext compression) {
+    this.compression = compression;
+  }
+
+  public void setCompression(CompressionContext compression) {
+    this.compression = compression;
+  }
+
+  @Override
+  public Decoder getDecoder(InputStream is) {
+    return
+        (compression == null) ? new KeyValueCodec.KeyValueDecoder((DataInputStream) is)
+            : new KeyValueCompression.CompressedKvDecoder((DataInputStream) is, compression);
+  }
+
+  @Override
+  public Encoder getEncoder(OutputStream os) {
+    return
+        (compression == null) ? new KeyValueCodec.KeyValueEncoder((DataOutputStream) os)
+        : new CompressedKvEncoder((DataOutputStream) os, compression);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
index 98ff1c699665..c14ac284d46e 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec;
 
 import java.io.IOException;
 
@@ -36,6 +37,7 @@
 public class ReplicationHLogReaderManager {
 
   private static final Log LOG = LogFactory.getLog(ReplicationHLogReaderManager.class);
+  private final WALEditCodec nonCompressingCodec = new WALEditCodec(null);
   private final FileSystem fs;
   private final Configuration conf;
   private long position = 0;
@@ -93,7 +95,8 @@ public HLog.Entry readNextAndSetPosition(HLog.Entry[] entriesArray,
     this.position = this.reader.getPosition();
     // We need to set the CC to null else it will be compressed when sent to the sink
     if (entry != null) {
-      entry.setCompressionContext(null);
+      entry.getKey().setCompressionContext(null);
+      entry.getEdit().setCodec(nonCompressingCodec);
     }
     return entry;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
index 45c47270647c..6f6060f7b860 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
@@ -49,8 +49,9 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
         HLogKey key = HLog.newKey(conf);
         WALEdit val = new WALEdit();
         HLog.Entry e = new HLog.Entry(key, val);
+        e.getEdit().setCodec(new WALEditCodec(compressionContext));
         if (compressionContext != null) {
-          e.setCompressionContext(compressionContext);
+          e.getKey().setCompressionContext(compressionContext);
         }
         b = this.reader.next(e.getKey(), e.getEdit());
         nextQueue.offer(e);

From 38009b65bffee86f620c29775ec7732a286e2a3c Mon Sep 17 00:00:00 2001
From: eclark <eclark@unknown>
Date: Mon, 10 Jun 2013 18:04:28 +0000
Subject: [PATCH 1004/1540] HBASE-8283 Backport HBASE-7842 Add compaction
 policy that explores more storefile groups to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1491546 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Store.java      | 218 ++++++++++++++----
 1 file changed, 169 insertions(+), 49 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index a20892ee821f..b9ca9ed673f6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1553,9 +1553,6 @@ CompactSelection compactSelection(List<StoreFile> candidates, int priority)
 
     if (!majorcompaction &&
         !hasReferences(compactSelection.getFilesToCompact())) {
-      // we're doing a minor compaction, let's see what files are applicable
-      int start = 0;
-      double r = compactSelection.getCompactSelectionRatio();
 
       // remove bulk import files that request to be excluded from minors
       compactSelection.getFilesToCompact().removeAll(Collections2.filter(
@@ -1576,25 +1573,48 @@ public boolean apply(StoreFile input) {
         compactSelection.emptyFileList();
         return compactSelection;
       }
+      if (conf.getBoolean("hbase.hstore.useExploringCompation", false)) {
+        compactSelection = exploringCompactionSelection(compactSelection);
+      } else {
+        compactSelection = defaultCompactionSelection(compactSelection);
+      }
+    } else {
+      if(majorcompaction) {
+        if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
+          LOG.debug("Warning, compacting more than " + this.maxFilesToCompact +
+            " files, probably because of a user-requested major compaction");
+          if(priority != PRIORITY_USER) {
+            LOG.error("Compacting more than max files on a non user-requested compaction");
+          }
+        }
+      } else if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
+        // all files included in this compaction, up to max
+        int pastMax = compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
+        compactSelection.getFilesToCompact().subList(0, pastMax).clear();
+      }
+    }
+    return compactSelection;
+  }
 
-      /* TODO: add sorting + unit test back in when HBASE-2856 is fixed
-      // Sort files by size to correct when normal skew is altered by bulk load.
-      Collections.sort(filesToCompact, StoreFile.Comparators.FILE_SIZE);
-       */
+  private CompactSelection defaultCompactionSelection(CompactSelection compactSelection) {
+    // we're doing a minor compaction, let's see what files are applicable
+    int start = 0;
 
-      // get store file sizes for incremental compacting selection.
-      int countOfFiles = compactSelection.getFilesToCompact().size();
-      long [] fileSizes = new long[countOfFiles];
-      long [] sumSize = new long[countOfFiles];
-      for (int i = countOfFiles-1; i >= 0; --i) {
-        StoreFile file = compactSelection.getFilesToCompact().get(i);
-        fileSizes[i] = file.getReader().length();
-        // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
-        int tooFar = i + this.maxFilesToCompact - 1;
-        sumSize[i] = fileSizes[i]
-                   + ((i+1    < countOfFiles) ? sumSize[i+1]      : 0)
-                   - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
-      }
+    double r = compactSelection.getCompactSelectionRatio();
+
+    // get store file sizes for incremental compacting selection.
+    int countOfFiles = compactSelection.getFilesToCompact().size();
+    long [] fileSizes = new long[countOfFiles];
+    long [] sumSize = new long[countOfFiles];
+    for (int i = countOfFiles-1; i >= 0; --i) {
+      StoreFile file = compactSelection.getFilesToCompact().get(i);
+      fileSizes[i] = file.getReader().length();
+      // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
+      int tooFar = i + this.maxFilesToCompact - 1;
+      sumSize[i] = fileSizes[i]
+          + ((i+1    < countOfFiles) ? sumSize[i+1]      : 0)
+          - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
+    }
 
       /* Start at the oldest file and stop when you find the first file that
        * meets compaction criteria:
@@ -1609,43 +1629,143 @@ public boolean apply(StoreFile input) {
        * situation where we always compact [end-threshold,end).  Then, the
        * last file becomes an aggregate of the previous compactions.
        */
-      while(countOfFiles - start >= this.minFilesToCompact &&
-            fileSizes[start] >
-              Math.max(minCompactSize, (long)(sumSize[start+1] * r))) {
-        ++start;
-      }
-      int end = Math.min(countOfFiles, start + this.maxFilesToCompact);
-      long totalSize = fileSizes[start]
-                     + ((start+1 < countOfFiles) ? sumSize[start+1] : 0);
-      compactSelection = compactSelection.getSubList(start, end);
-
-      // if we don't have enough files to compact, just wait
-      if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Skipped compaction of " + this
+    while(countOfFiles - start >= this.minFilesToCompact &&
+        fileSizes[start] >
+            Math.max(minCompactSize, (long)(sumSize[start+1] * r))) {
+      ++start;
+    }
+    int end = Math.min(countOfFiles, start + this.maxFilesToCompact);
+    long totalSize = fileSizes[start]
+        + ((start+1 < countOfFiles) ? sumSize[start+1] : 0);
+    compactSelection = compactSelection.getSubList(start, end);
+
+    // if we don't have enough files to compact, just wait
+    if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Skipped compaction of " + this
             + ".  Only " + (end - start) + " file(s) of size "
             + StringUtils.humanReadableInt(totalSize)
             + " have met compaction criteria.");
-        }
-        compactSelection.emptyFileList();
-        return compactSelection;
       }
-    } else {
-      if(majorcompaction) {
-        if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
-          LOG.debug("Warning, compacting more than " + this.maxFilesToCompact +
-            " files, probably because of a user-requested major compaction");
-          if(priority != PRIORITY_USER) {
-            LOG.error("Compacting more than max files on a non user-requested compaction");
-          }
+      compactSelection.emptyFileList();
+      return compactSelection;
+    }
+    return compactSelection;
+  }
+
+  private CompactSelection exploringCompactionSelection(CompactSelection compactSelection) {
+
+    List<StoreFile> candidates = compactSelection.getFilesToCompact();
+    int futureFiles = filesCompacting.isEmpty() ? 0 : 1;
+    boolean mayBeStuck = (candidates.size() - filesCompacting.size() + futureFiles)
+        >= blockingStoreFileCount;
+    // Start off choosing nothing.
+    List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
+    List<StoreFile> smallest = new ArrayList<StoreFile>(0);
+    long bestSize = 0;
+    long smallestSize = Long.MAX_VALUE;
+    double r = compactSelection.getCompactSelectionRatio();
+
+    // Consider every starting place.
+    for (int startIndex = 0; startIndex < candidates.size(); startIndex++) {
+      // Consider every different sub list permutation in between start and end with min files.
+      for (int currentEnd = startIndex + minFilesToCompact - 1;
+           currentEnd < candidates.size(); currentEnd++) {
+        List<StoreFile> potentialMatchFiles = candidates.subList(startIndex, currentEnd + 1);
+
+        // Sanity checks
+        if (potentialMatchFiles.size() < minFilesToCompact) {
+          continue;
+        }
+        if (potentialMatchFiles.size() > maxFilesToCompact) {
+          continue;
+        }
+
+        // Compute the total size of files that will
+        // have to be read if this set of files is compacted.
+        long size = getCompactionSize(potentialMatchFiles);
+
+        // Store the smallest set of files.  This stored set of files will be used
+        // if it looks like the algorithm is stuck.
+        if (size < smallestSize) {
+          smallest = potentialMatchFiles;
+          smallestSize = size;
+        }
+
+        if (size >= minCompactSize
+            && !filesInRatio(potentialMatchFiles, r)) {
+          continue;
+        }
+
+        if (size > maxCompactSize) {
+          continue;
+        }
+
+        // Keep if this gets rid of more files.  Or the same number of files for less io.
+        if (potentialMatchFiles.size() > bestSelection.size()
+            || (potentialMatchFiles.size() == bestSelection.size() && size < bestSize)) {
+          bestSelection = potentialMatchFiles;
+          bestSize = size;
         }
-      } else if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
-        // all files included in this compaction, up to max
-        int pastMax = compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
-        compactSelection.getFilesToCompact().subList(0, pastMax).clear();
       }
     }
+
+    if (bestSelection.size() == 0 && mayBeStuck) {
+      smallest = new ArrayList<StoreFile>(smallest);
+      compactSelection.getFilesToCompact().clear();
+      compactSelection.getFilesToCompact().addAll(smallest);
+    } else {
+      bestSelection = new ArrayList<StoreFile>(bestSelection);
+      compactSelection.getFilesToCompact().clear();
+      compactSelection.getFilesToCompact().addAll(bestSelection);
+    }
+
     return compactSelection;
+
+  }
+
+  /**
+   * Check that all files satisfy the ratio
+   *
+   * @param files set of files to examine.
+   * @param currentRatio The raio
+   * @return if all files are in ratio.
+   */
+  private boolean filesInRatio(final List<StoreFile> files, final double currentRatio) {
+    if (files.size() < 2) {
+      return true;
+    }
+    long totalFileSize = 0;
+    for (int i = 0; i < files.size(); i++) {
+      totalFileSize += files.get(i).getReader().length();
+    }
+    for (int i = 0; i < files.size(); i++) {
+      long singleFileSize = files.get(i).getReader().length();
+      long sumAllOtherFilesize = totalFileSize - singleFileSize;
+
+      if ((singleFileSize > sumAllOtherFilesize * currentRatio)
+          && (sumAllOtherFilesize >= this.minCompactSize)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Get the number of bytes a proposed compaction would have to read.
+   *
+   * @param files Set of files in a proposed compaction.
+   * @return size in bytes.
+   */
+  private long getCompactionSize(final List<StoreFile> files) {
+    long size = 0;
+    if (files == null) {
+      return size;
+    }
+    for (StoreFile f : files) {
+      size += f.getReader().length();
+    }
+    return size;
   }
 
   /**

From 937b820386c668ca2fc359308ece52e2883f7e3d Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Mon, 10 Jun 2013 20:53:42 +0000
Subject: [PATCH 1005/1540] HBASE-8683 Add major compaction support in
 CompactionTool (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1491599 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/CompactionTool.java    | 48 ++++++++++++-------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
index e123c22761fe..04611cb89cbe 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -78,6 +78,7 @@ public class CompactionTool extends Configured implements Tool {
 
   private final static String CONF_TMP_DIR = "hbase.tmp.dir";
   private final static String CONF_COMPACT_ONCE = "hbase.compactiontool.compact.once";
+  private final static String CONF_COMPACT_MAJOR = "hbase.compactiontool.compact.major";
   private final static String CONF_DELETE_COMPACTED = "hbase.compactiontool.delete";
   private final static String CONF_COMPLETE_COMPACTION = "hbase.hstore.compaction.complete";
 
@@ -103,44 +104,45 @@ public CompactionWorker(final FileSystem fs, final Configuration conf) {
     /**
      * Execute the compaction on the specified path.
      *
-     * @param path Directory path on which run a
+     * @param path Directory path on which to run compaction.
      * @param compactOnce Execute just a single step of compaction.
+     * @param major Request major compaction.
      */
-    public void compact(final Path path, final boolean compactOnce) throws IOException {
+    public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException {
       if (isFamilyDir(fs, path)) {
         Path regionDir = path.getParent();
         Path tableDir = regionDir.getParent();
         HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
         HRegion region = loadRegion(fs, conf, htd, regionDir);
-        compactStoreFiles(region, path, compactOnce);
+        compactStoreFiles(region, path, compactOnce, major);
       } else if (isRegionDir(fs, path)) {
         Path tableDir = path.getParent();
         HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
-        compactRegion(htd, path, compactOnce);
+        compactRegion(htd, path, compactOnce, major);
       } else if (isTableDir(fs, path)) {
-        compactTable(path, compactOnce);
+        compactTable(path, compactOnce, major);
       } else {
         throw new IOException(
           "Specified path is not a table, region or family directory. path=" + path);
       }
     }
 
-    private void compactTable(final Path tableDir, final boolean compactOnce)
+    private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major)
         throws IOException {
       HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
       LOG.info("Compact table=" + htd.getNameAsString());
       for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
-        compactRegion(htd, regionDir, compactOnce);
+        compactRegion(htd, regionDir, compactOnce, major);
       }
     }
 
     private void compactRegion(final HTableDescriptor htd, final Path regionDir,
-        final boolean compactOnce) throws IOException {
+        final boolean compactOnce, final boolean major) throws IOException {
       HRegion region = loadRegion(fs, conf, htd, regionDir);
       LOG.info("Compact table=" + htd.getNameAsString() +
         " region=" + region.getRegionNameAsString());
       for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
-        compactStoreFiles(region, familyDir, compactOnce);
+        compactStoreFiles(region, familyDir, compactOnce, major);
       }
     }
 
@@ -150,13 +152,16 @@ private void compactRegion(final HTableDescriptor htd, final Path regionDir,
      * no more compactions are needed. Uses the Configuration settings provided.
      */
     private void compactStoreFiles(final HRegion region, final Path familyDir,
-        final boolean compactOnce) throws IOException {
+        final boolean compactOnce, final boolean major) throws IOException {
       LOG.info("Compact table=" + region.getTableDesc().getNameAsString() +
         " region=" + region.getRegionNameAsString() +
         " family=" + familyDir.getName());
       Store store = getStore(region, familyDir);
+      if (major) {
+        store.triggerMajorCompaction();
+      }
       do {
-        CompactionRequest cr = store.requestCompaction();
+        CompactionRequest cr = store.requestCompaction(Store.PRIORITY_USER, null);
         StoreFile storeFile = store.compact(cr);
         if (storeFile != null) {
           if (keepCompactedFiles && deleteCompacted) {
@@ -213,11 +218,13 @@ private static class CompactionMapper
       extends Mapper<LongWritable, Text, NullWritable, NullWritable> {
     private CompactionWorker compactor = null;
     private boolean compactOnce = false;
+    private boolean major = false;
 
     @Override
     public void setup(Context context) {
       Configuration conf = context.getConfiguration();
       compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false);
+      major = conf.getBoolean(CONF_COMPACT_MAJOR, false);
 
       try {
         FileSystem fs = FileSystem.get(conf);
@@ -231,7 +238,7 @@ public void setup(Context context) {
     public void map(LongWritable key, Text value, Context context)
         throws InterruptedException, IOException {
       Path path = new Path(value.toString());
-      this.compactor.compact(path, compactOnce);
+      this.compactor.compact(path, compactOnce, major);
     }
   }
 
@@ -343,9 +350,10 @@ public static void createInputFile(final FileSystem fs, final Path path,
    * Execute compaction, using a Map-Reduce job.
    */
   private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
-      final boolean compactOnce) throws Exception {
+      final boolean compactOnce, final boolean major) throws Exception {
     Configuration conf = getConf();
     conf.setBoolean(CONF_COMPACT_ONCE, compactOnce);
+    conf.setBoolean(CONF_COMPACT_MAJOR, major);
 
     Job job = new Job(conf);
     job.setJobName("CompactionTool");
@@ -379,10 +387,10 @@ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
    * Execute compaction, from this client, one path at the time.
    */
   private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
-      final boolean compactOnce) throws IOException {
+      final boolean compactOnce, final boolean major) throws IOException {
     CompactionWorker worker = new CompactionWorker(fs, getConf());
     for (Path path: toCompactDirs) {
-      worker.compact(path, compactOnce);
+      worker.compact(path, compactOnce, major);
     }
     return 0;
   }
@@ -391,6 +399,7 @@ private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
   public int run(String[] args) throws Exception {
     Set<Path> toCompactDirs = new HashSet<Path>();
     boolean compactOnce = false;
+    boolean major = false;
     boolean mapred = false;
 
     Configuration conf = getConf();
@@ -401,6 +410,8 @@ public int run(String[] args) throws Exception {
         String opt = args[i];
         if (opt.equals("-compactOnce")) {
           compactOnce = true;
+        } else if (opt.equals("-major")) {
+          major = true;
         } else if (opt.equals("-mapred")) {
           mapred = true;
         } else if (!opt.startsWith("-")) {
@@ -427,9 +438,9 @@ public int run(String[] args) throws Exception {
 
     // Execute compaction!
     if (mapred) {
-      return doMapReduce(fs, toCompactDirs, compactOnce);
+      return doMapReduce(fs, toCompactDirs, compactOnce, major);
     } else {
-      return doClient(fs, toCompactDirs, compactOnce);
+      return doClient(fs, toCompactDirs, compactOnce, major);
     }
   }
 
@@ -442,11 +453,12 @@ private void printUsage(final String message) {
       System.err.println(message);
     }
     System.err.println("Usage: java " + this.getClass().getName() + " \\");
-    System.err.println("  [-compactOnce] [-mapred] [-D<property=value>]* files...");
+    System.err.println("  [-compactOnce] [-major] [-mapred] [-D<property=value>]* files...");
     System.err.println();
     System.err.println("Options:");
     System.err.println(" mapred         Use MapReduce to run compaction.");
     System.err.println(" compactOnce    Execute just one compaction step. (default: while needed)");
+    System.err.println(" major          Trigger major compaction.");
     System.err.println();
     System.err.println("Note: -D properties will be applied to the conf used. ");
     System.err.println("For example: ");

From 64e72ca8405c97bfdc14156e41479434dd74051e Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Tue, 11 Jun 2013 22:26:19 +0000
Subject: [PATCH 1006/1540] HBASE-8724 [0.94] ExportSnapshot should not use
 hbase.tmp.dir as a staging dir on hdfs

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1491993 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/snapshot/ExportSnapshot.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 2455831b0ff5..602597c8fc50 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -79,13 +79,13 @@
 public final class ExportSnapshot extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
 
-  private static final String CONF_TMP_DIR = "hbase.tmp.dir";
   private static final String CONF_FILES_USER = "snapshot.export.files.attributes.user";
   private static final String CONF_FILES_GROUP = "snapshot.export.files.attributes.group";
   private static final String CONF_FILES_MODE = "snapshot.export.files.attributes.mode";
   private static final String CONF_CHECKSUM_VERIFY = "snapshot.export.checksum.verify";
   private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root";
   private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
+  private static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
 
   private static final String INPUT_FOLDER_PREFIX = "export-files.";
 
@@ -470,7 +470,8 @@ public int compare(Pair<Path, Long> a, Pair<Path, Long> b) {
   private static Path getInputFolderPath(final FileSystem fs, final Configuration conf)
       throws IOException, InterruptedException {
     String stagingName = "exportSnapshot-" + EnvironmentEdgeManager.currentTimeMillis();
-    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
+    Path stagingDir = new Path(conf.get(CONF_STAGING_ROOT, fs.getWorkingDirectory().toString())
+        , stagingName);
     fs.mkdirs(stagingDir);
     return new Path(stagingDir, INPUT_FOLDER_PREFIX +
       String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));

From 3018b9f228fd60d94fb31502345f55303becb9df Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 13 Jun 2013 02:51:22 +0000
Subject: [PATCH 1007/1540] HBASE-8702: Make WALEditCodec pluggable

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1492518 13f79535-47bb-0310-9956-ffa450edef68
---
 .../wal/SequenceFileLogReader.java            |  2 +-
 .../wal/SequenceFileLogWriter.java            |  3 +-
 .../hbase/regionserver/wal/WALEdit.java       |  2 +-
 .../hbase/regionserver/wal/WALEditCodec.java  | 48 ++++++++++++++-
 .../ReplicationHLogReaderManager.java         |  2 +-
 .../wal/FaultySequenceFileLogReader.java      |  5 +-
 .../wal/TestCustomWALEditCodec.java           | 61 +++++++++++++++++++
 7 files changed, 116 insertions(+), 7 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCustomWALEditCodec.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index c167ed2605fe..9420dde688dd 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -194,7 +194,7 @@ public void init(FileSystem fs, Path path, Configuration conf)
     }
 
     // setup the codec
-    this.codec = new WALEditCodec(compressionContext);
+    this.codec = WALEditCodec.create(conf, compressionContext);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index 5a49c5252b6e..e224658a0a4e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -195,7 +195,8 @@ SequenceFile.CompressionType.NONE, new DefaultCodec(),
       LOG.debug("using new createWriter -- HADOOP-6840");
     }
 
-    this.codec = new WALEditCodec(compressionContext);
+    // setup the WALEditCodec
+    this.codec = WALEditCodec.create(conf, compressionContext);
     this.writer_out = getSequenceFilePrivateFSDataOutputStreamAccessible();
     this.syncFs = getSyncFs();
     this.hflush = getHFlush();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index 2b946f4c571b..5ae50add9362 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -81,7 +81,7 @@ public class WALEdit implements Writable, HeapSize {
   // default to decoding uncompressed data - needed for replication, which enforces that
   // uncompressed edits are sent across the wire. In the regular case (reading/writing WAL), the
   // codec will be setup by the reader/writer class, not here.
-  private WALEditCodec codec = new WALEditCodec(null);
+  private WALEditCodec codec = new WALEditCodec();
 
   public WALEdit() {
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java
index 1b9adaabf101..375e5ed34a99 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEditCodec.java
@@ -19,9 +19,11 @@
 
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.codec.Codec;
 import org.apache.hadoop.hbase.codec.Decoder;
 import org.apache.hadoop.hbase.codec.Encoder;
@@ -29,10 +31,27 @@
 import org.apache.hadoop.hbase.regionserver.wal.KeyValueCompression.CompressedKvEncoder;
 
 public class WALEditCodec implements Codec {
+  /** Configuration key for a custom class to use when serializing the WALEdits to the HLog */
+  public static final String WAL_EDIT_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec";
+
   private CompressionContext compression;
 
-  public WALEditCodec(CompressionContext compression) {
-    this.compression = compression;
+  /**
+   * Nullary Constructor - all subclass must support this to load from configuration. Setup can be
+   * completed in the {@link #init} method.
+   * <p>
+   * This implementation defaults to having no compression on the resulting {@link Encoder}/
+   * {@link Decoder}, though it can be added via {@link #setCompression(CompressionContext)}
+   */
+  public WALEditCodec() {
+  }
+
+  /**
+   * Initialize <tt>this</tt> - called exactly once after the object is instantiated and before any
+   * other method in this class. By default, does nothing.
+   * @param conf {@link Configuration} from which to configure <tt>this</tt>
+   */
+  public void init(Configuration conf) {
   }
 
   public void setCompression(CompressionContext compression) {
@@ -52,4 +71,29 @@ public Encoder getEncoder(OutputStream os) {
         (compression == null) ? new KeyValueCodec.KeyValueEncoder((DataOutputStream) os)
         : new CompressedKvEncoder((DataOutputStream) os, compression);
   }
+
+  /**
+   * Create and setup a {@link WALEditCodec} from the {@link Configuration}, if one has been
+   * specified. Fully prepares the codec for use in serialization.
+   * @param conf {@link Configuration} to read for the user-specified codec. If none is specified,
+   *          uses a {@link WALEditCodec}.
+   * @param compressionContext compression to setup on the codec.
+   * @return a {@link WALEditCodec} ready for use.
+   * @throws IOException if the codec cannot be created
+   */
+  public static WALEditCodec create(Configuration conf, CompressionContext compressionContext)
+      throws IOException {
+    Class<? extends WALEditCodec> codecClazz = conf.getClass(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY,
+      WALEditCodec.class, WALEditCodec.class);
+    try {
+      WALEditCodec codec = codecClazz.newInstance();
+      codec.init(conf);
+      codec.setCompression(compressionContext);
+      return codec;
+    } catch (InstantiationException e) {
+      throw new IOException("Couldn't instantiate the configured WALEditCodec!", e);
+    } catch (IllegalAccessException e) {
+      throw new IOException("Couldn't instantiate the configured WALEditCodec!", e);
+    }
+  }
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
index c14ac284d46e..bd8f44d16cdc 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -37,7 +37,7 @@
 public class ReplicationHLogReaderManager {
 
   private static final Log LOG = LogFactory.getLog(ReplicationHLogReaderManager.class);
-  private final WALEditCodec nonCompressingCodec = new WALEditCodec(null);
+  private final WALEditCodec nonCompressingCodec = new WALEditCodec();
   private final FileSystem fs;
   private final Configuration conf;
   private long position = 0;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
index 6f6060f7b860..c12cac19ff57 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
@@ -39,6 +39,8 @@ FailureType getFailureType() {
     return FailureType.valueOf(conf.get("faultysequencefilelogreader.failuretype", "NONE"));
   }
 
+  WALEditCodec codec = new WALEditCodec();
+
   @Override
   public HLog.Entry next(HLog.Entry reuse) throws IOException {
     this.entryStart = this.reader.getPosition();
@@ -49,7 +51,8 @@ public HLog.Entry next(HLog.Entry reuse) throws IOException {
         HLogKey key = HLog.newKey(conf);
         WALEdit val = new WALEdit();
         HLog.Entry e = new HLog.Entry(key, val);
-        e.getEdit().setCodec(new WALEditCodec(compressionContext));
+        codec.setCompression(compressionContext);
+        e.getEdit().setCodec(codec);
         if (compressionContext != null) {
           e.getKey().setCompressionContext(compressionContext);
         }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCustomWALEditCodec.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCustomWALEditCodec.java
new file mode 100644
index 000000000000..dbc440d5451d
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCustomWALEditCodec.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that we can create, load, setup our own custom codec
+ */
+@Category(SmallTests.class)
+public class TestCustomWALEditCodec {
+
+  public static class CustomWALEditCodec extends WALEditCodec {
+    public boolean initialized = false;
+    public boolean compressionSet = false;
+
+    @Override
+    public void init(Configuration conf) {
+      this.initialized = true;
+    }
+
+    @Override
+    public void setCompression(CompressionContext compression) {
+      this.compressionSet = true;
+    }
+  }
+
+  /**
+   * Test that a custom WALEditCodec will be completely setup when it is instantiated via
+   * {@link WALEditCodec}
+   * @throws Exception on failure
+   */
+  @Test
+  public void testCreatePreparesCodec() throws Exception {
+    Configuration conf = new Configuration(false);
+    conf.setClass(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, CustomWALEditCodec.class, WALEditCodec.class);
+    CustomWALEditCodec codec = (CustomWALEditCodec) WALEditCodec.create(conf, null);
+    assertTrue("Custom codec didn't get initialized", codec.initialized);
+    assertTrue("Custom codec didn't have compression set", codec.compressionSet);
+  }
+}
\ No newline at end of file

From 614cb66805eacb436f4971f035d718742f4b8c1b Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 14 Jun 2013 22:30:19 +0000
Subject: [PATCH 1008/1540] HBASE-8742 HTableDescriptor Properties not
 preserved when cloning

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1493270 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java  | 4 +++-
 .../apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java  | 5 +++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index e9c17b146c39..697e884de5d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -476,10 +476,12 @@ private void setValue(final ImmutableBytesWritable key,
   }
 
   /*
+   * Setter for storing metadata as a (key, value) pair in {@link #values} map
+   *
    * @param key The key.
    * @param value The value.
    */
-  private void setValue(final ImmutableBytesWritable key,
+  public void setValue(final ImmutableBytesWritable key,
       final ImmutableBytesWritable value) {
     values.put(key, value);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 5b92060403dd..a81cca6a9c81 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -594,6 +595,10 @@ public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotT
     for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
       htd.addFamily(hcd);
     }
+    for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
+        snapshotTableDescriptor.getValues().entrySet()) {
+      htd.setValue(e.getKey(), e.getValue());
+    }
     return htd;
   }
 }

From e2de60f9017b8be2023586bd4e2caed4617b9b58 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 17 Jun 2013 07:58:23 +0000
Subject: [PATCH 1009/1540] HBASE-8749 Potential race condition between
 FSUtils.renameAndSetModifyTime() and HFile/LogCleaner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1493672 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
index c2e04f336342..fdbc343a3b83 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
@@ -259,9 +259,8 @@ protected static void sleepBeforeRetry(String msg, int sleepMultiplier) {
    */
   public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
       throws IOException {
-    if (!renameDirForFileSystem(fs, src, dest)) return false;
     // set the modify time for TimeToLive Cleaner
-    fs.setTimes(dest, EnvironmentEdgeManager.currentTimeMillis(), -1);
-    return true;
+    fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
+    return renameDirForFileSystem(fs, src, dest);
   }
 }

From a8269529ebf140f22711cacb9d3c9af4d710d492 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 18 Jun 2013 18:55:39 +0000
Subject: [PATCH 1010/1540] HBASE-8453 TestImportExport failing again due to
 configuration issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494257 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     |  4 ++
 .../hbase/mapreduce/TestImportExport.java     | 64 +++++++++++--------
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 3a76d89c9b2b..79166bb8da82 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1663,6 +1663,10 @@ public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
     this.dfsCluster = cluster;
   }
 
+  public MiniMRCluster getMRCluster() {
+    return mrCluster;
+  }
+
   public FileSystem getTestFileSystem() throws IOException {
     return HFileSystem.get(conf);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index c0cd2f156933..ff597c42d010 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -42,6 +43,7 @@
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -64,12 +66,11 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
-  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    cluster = UTIL.startMiniCluster();
+    UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -111,16 +112,16 @@ public void testSimpleCase() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
-
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -129,12 +130,13 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -158,12 +160,13 @@ public void testMetaExport() throws Exception {
     String EXPORT_TABLE = ".META.";
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
+        UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
   }
@@ -200,12 +203,13 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -224,12 +228,13 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -268,12 +273,13 @@ public void testWithFilter() throws Exception {
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
 
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
+        UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    JobConf jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    Job job = Export.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -287,12 +293,13 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
         "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -310,14 +317,15 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
         OUTPUT_DIR, "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    jobConf = UTIL.getMRCluster().createJobConf();
+    HBaseConfiguration.merge(jobConf, conf);
+    job = Import.createSubmittableJob(jobConf, args);
     job.waitForCompletion(false);
-    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
+    assertFalse("Job succeeded, but it had a non-instantiable filter!", job.isSuccessful());
 
     // cleanup
     exportTable.close();

From 9638034f5938fafce7e689962cdaa8c08d57eb39 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 18 Jun 2013 22:36:49 +0000
Subject: [PATCH 1011/1540] HBASE-8453 TestImportExport failing again due to
 configuration issues -- REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494349 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     |  4 --
 .../hbase/mapreduce/TestImportExport.java     | 64 ++++++++-----------
 2 files changed, 28 insertions(+), 40 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 79166bb8da82..3a76d89c9b2b 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1663,10 +1663,6 @@ public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
     this.dfsCluster = cluster;
   }
 
-  public MiniMRCluster getMRCluster() {
-    return mrCluster;
-  }
-
   public FileSystem getTestFileSystem() throws IOException {
     return HFileSystem.get(conf);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index ff597c42d010..c0cd2f156933 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -43,7 +42,6 @@
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -66,11 +64,12 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
+  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    UTIL.startMiniCluster();
+    cluster = UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -112,16 +111,16 @@ public void testSimpleCase() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
+
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -130,13 +129,12 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -160,13 +158,12 @@ public void testMetaExport() throws Exception {
     String EXPORT_TABLE = ".META.";
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        UTIL.getConfiguration()), args);
+        cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
   }
@@ -203,13 +200,12 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -228,13 +224,12 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -273,13 +268,12 @@ public void testWithFilter() throws Exception {
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
 
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        UTIL.getConfiguration()), args);
+        cluster.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    JobConf jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    Job job = Export.createSubmittableJob(jobConf, args);
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -293,13 +287,12 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
         "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
     assertTrue(job.isSuccessful());
 
@@ -317,15 +310,14 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
         OUTPUT_DIR, "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
-    jobConf = UTIL.getMRCluster().createJobConf();
-    HBaseConfiguration.merge(jobConf, conf);
-    job = Import.createSubmittableJob(jobConf, args);
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
-    assertFalse("Job succeeded, but it had a non-instantiable filter!", job.isSuccessful());
+    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
 
     // cleanup
     exportTable.close();

From 4c5da0e29c8a9528d7ce011b1f54aa0800030f3e Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Tue, 18 Jun 2013 22:51:35 +0000
Subject: [PATCH 1012/1540] HBASE-8700 IntegrationTestBigLinkedList can fail
 due to random number collision

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494359 13f79535-47bb-0310-9956-ffa450edef68
---
 .../test/IntegrationTestBigLinkedList.java    | 289 +++++++++++-------
 1 file changed, 177 insertions(+), 112 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 8f2f86e852ff..35bf09f74dc7 100644
--- a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -21,6 +21,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.io.StringWriter;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,10 +56,9 @@
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.mapreduce.TableMapper;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.VLongWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.Counters;
@@ -148,6 +148,7 @@
  */
 @Category(IntegrationTests.class)
 public class IntegrationTestBigLinkedList extends Configured implements Tool {
+  private static final byte[] NO_KEY = new byte[1];
 
   private static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
 
@@ -171,9 +172,23 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool {
   private static final String GENERATOR_NUM_MAPPERS_KEY
     = "IntegrationTestBigLinkedList.generator.map.tasks";
 
+  private static final String GENERATOR_WIDTH_KEY
+    = "IntegrationTestBigLinkedList.generator.width";
+
+  private static final String GENERATOR_WRAP_KEY
+    = "IntegrationTestBigLinkedList.generator.wrap";
+
+  protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster
+
+  private static final int WIDTH_DEFAULT = 1000000;
+  private static final int WRAP_DEFAULT = 25;
+
+  private static final int ROWKEY_LENGTH = 16;
+
   static class CINode {
-    long key;
-    long prev;
+    byte[] key;
+    byte[] prev;
+
     String client;
     long count;
   }
@@ -185,14 +200,11 @@ static class Generator extends Configured implements Tool {
 
     private static final Log LOG = LogFactory.getLog(Generator.class);
 
-    private static final int WIDTH = 1000000;
-    private static final int WRAP = WIDTH * 25;
-
     public static enum Counts {
       UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
     }
 
-    static class GeneratorInputFormat extends InputFormat<LongWritable,NullWritable> {
+    static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> {
       static class GeneratorInputSplit extends InputSplit implements Writable {
         @Override
         public long getLength() throws IOException, InterruptedException {
@@ -210,7 +222,7 @@ public void write(DataOutput arg0) throws IOException {
         }
       }
 
-      static class GeneratorRecordReader extends RecordReader<LongWritable,NullWritable> {
+      static class GeneratorRecordReader extends RecordReader<BytesWritable,NullWritable> {
         private long count;
         private long numNodes;
         private Random rand;
@@ -220,8 +232,10 @@ public void close() throws IOException {
         }
 
         @Override
-        public LongWritable getCurrentKey() throws IOException, InterruptedException {
-          return new LongWritable(Math.abs(rand.nextLong()));
+        public BytesWritable getCurrentKey() throws IOException, InterruptedException {
+          byte[] bytes = new byte[ROWKEY_LENGTH];
+          rand.nextBytes(bytes);
+          return new BytesWritable(bytes);
         }
 
         @Override
@@ -249,7 +263,7 @@ public boolean nextKeyValue() throws IOException, InterruptedException {
       }
 
       @Override
-      public RecordReader<LongWritable,NullWritable> createRecordReader(
+      public RecordReader<BytesWritable,NullWritable> createRecordReader(
           InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
         GeneratorRecordReader rr = new GeneratorRecordReader();
         rr.initialize(split, context);
@@ -303,18 +317,19 @@ protected boolean isSplitable(JobContext context, Path filename) {
      *             |___________________________|
      */
     static class GeneratorMapper
-      extends Mapper<LongWritable, NullWritable, NullWritable, NullWritable> {
+      extends Mapper<BytesWritable, NullWritable, NullWritable, NullWritable> {
       Random rand = new Random();
 
-      long[] first = null;
-      long[] prev = null;
-      long[] current = new long[WIDTH];
+      byte[][] first = null;
+      byte[][] prev = null;
+      byte[][] current = null;
       byte[] id;
       long count = 0;
       int i;
       HTable table;
       long numNodes;
-      long wrap = WRAP;
+      long wrap;
+      int width;
 
       protected void setup(Context context) throws IOException, InterruptedException {
         id = Bytes.toBytes(UUID.randomUUID().toString());
@@ -322,9 +337,14 @@ protected void setup(Context context) throws IOException, InterruptedException {
         table = new HTable(conf, getTableName(conf));
         table.setAutoFlush(false);
         table.setWriteBufferSize(4 * 1024 * 1024);
-        numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
-        if (numNodes < 25000000) {
-          wrap = numNodes;
+        this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT);
+        current = new byte[this.width][];
+        int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT);
+        this.wrap = (long)wrapMultiplier * width;
+        this.numNodes = context.getConfiguration().getLong(
+            GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT);
+        if (this.numNodes < this.wrap) {
+          this.wrap = this.numNodes;
         }
       };
 
@@ -333,17 +353,17 @@ protected void cleanup(Context context) throws IOException ,InterruptedException
       };
 
       @Override
-      protected void map(LongWritable key, NullWritable value, Context output) throws IOException {
-        current[i++] = Math.abs(key.get());
-
-        if (i == current.length) {
+      protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
+        current[i] = new byte[key.getLength()];
+        System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
+        if (++i == current.length) {
           persist(output, count, prev, current, id);
           i = 0;
 
           if (first == null)
             first = current;
           prev = current;
-          current = new long[WIDTH];
+          current = new byte[this.width][];
 
           count += current.length;
           output.setStatus("Count " + count);
@@ -361,18 +381,18 @@ protected void map(LongWritable key, NullWritable value, Context output) throws
         }
       }
 
-      private static void circularLeftShift(long[] first) {
-        long ez = first[0];
+      private static <T> void circularLeftShift(T[] first) {
+        T ez = first[0];
         for (int i = 0; i < first.length - 1; i++)
           first[i] = first[i + 1];
         first[first.length - 1] = ez;
       }
 
-      private void persist(Context output, long count, long[] prev, long[] current, byte[] id)
+      private void persist(Context output, long count, byte[][] prev, byte[][] current, byte[] id)
           throws IOException {
         for (int i = 0; i < current.length; i++) {
-          Put put = new Put(Bytes.toBytes(current[i]));
-          put.add(FAMILY_NAME, COLUMN_PREV, Bytes.toBytes(prev == null ? -1 : prev[i]));
+          Put put = new Put(current[i]);
+          put.add(FAMILY_NAME, COLUMN_PREV, prev == null ? NO_KEY : prev[i]);
 
           if (count >= 0) {
             put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i));
@@ -396,15 +416,18 @@ private void persist(Context output, long count, long[] prev, long[] current, by
     public int run(String[] args) throws Exception {
       if (args.length < 3) {
         System.out.println("Usage : " + Generator.class.getSimpleName() +
-            " <num mappers> <num nodes per map> <tmp output dir>");
-        System.out.println("   where <num nodes per map> should be a multiple of 25M");
+            " <num mappers> <num nodes per map> <tmp output dir> [<width> <wrap multiplier>]");
+        System.out.println("   where <num nodes per map> should be a multiple of " +
+            " width*wrap multiplier, 25M by default");
         return 0;
       }
 
       int numMappers = Integer.parseInt(args[0]);
       long numNodes = Long.parseLong(args[1]);
       Path tmpOutput = new Path(args[2]);
-      return run(numMappers, numNodes, tmpOutput);
+      Integer width = (args.length < 4) ? null : Integer.parseInt(args[3]);
+      Integer wrapMuplitplier = (args.length < 5) ? null : Integer.parseInt(args[4]);
+      return run(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
     }
 
     protected void createSchema() throws IOException {
@@ -418,8 +441,8 @@ protected void createSchema() throws IOException {
       admin.close();
     }
 
-    public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput)
-        throws Exception {
+    public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
+        Integer width, Integer wrapMuplitplier) throws Exception {
       LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
           + ", numNodes=" + numNodes);
       Job job = new Job(getConf());
@@ -429,11 +452,10 @@ public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput
       job.setJarByClass(getClass());
 
       job.setInputFormatClass(GeneratorInputFormat.class);
-      job.setOutputKeyClass(LongWritable.class);
+      job.setOutputKeyClass(BytesWritable.class);
       job.setOutputValueClass(NullWritable.class);
 
-      job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
-      job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+      setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
 
       job.setMapperClass(Mapper.class); //identity mapper
 
@@ -445,7 +467,8 @@ public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput
       return success ? 0 : 1;
     }
 
-    public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Exception {
+    public int runGenerator(int numMappers, long numNodes, Path tmpOutput,
+        Integer width, Integer wrapMuplitplier) throws Exception {
       LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
       createSchema();
 
@@ -460,8 +483,7 @@ public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Ex
       job.setOutputKeyClass(NullWritable.class);
       job.setOutputValueClass(NullWritable.class);
 
-      job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
-      job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+      setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
 
       job.setMapperClass(GeneratorMapper.class);
 
@@ -476,13 +498,13 @@ public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Ex
       return success ? 0 : 1;
     }
 
-    public int run(int numMappers, long numNodes, Path tmpOutput) throws Exception {
-      int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput);
+    public int run(int numMappers, long numNodes, Path tmpOutput,
+        Integer width, Integer wrapMuplitplier) throws Exception {
+      int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
       if (ret > 0) {
         return ret;
       }
-
-      return runGenerator(numMappers, numNodes, tmpOutput);
+      return runGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
     }
   }
 
@@ -493,72 +515,83 @@ public int run(int numMappers, long numNodes, Path tmpOutput) throws Exception {
   static class Verify extends Configured implements Tool {
 
     private static final Log LOG = LogFactory.getLog(Verify.class);
-    private static final VLongWritable DEF = new VLongWritable(-1);
+    private static final BytesWritable DEF = new BytesWritable(NO_KEY);
 
     private Job job;
 
-    public static class VerifyMapper extends TableMapper<LongWritable, VLongWritable> {
-      private LongWritable row = new LongWritable();
-      private LongWritable ref = new LongWritable();
-      private VLongWritable vrow = new VLongWritable();
+    public static class VerifyMapper extends TableMapper<BytesWritable, BytesWritable> {
+      private BytesWritable row = new BytesWritable();
+      private BytesWritable ref = new BytesWritable();
 
       @Override
       protected void map(ImmutableBytesWritable key, Result value, Context context)
           throws IOException ,InterruptedException {
-        row.set(Bytes.toLong(key.get()));
+        byte[] rowKey = key.get();
+        row.set(rowKey, 0, rowKey.length);
         context.write(row, DEF);
-
-        long prev = Bytes.toLong(value.getValue(FAMILY_NAME, COLUMN_PREV));
-        if (prev >= 0) {
-          ref.set(prev);
-          vrow.set(Bytes.toLong(key.get()));
-          context.write(ref, vrow);
+        byte[] prev = value.getValue(FAMILY_NAME, COLUMN_PREV);
+        if (prev != null && prev.length > 0) {
+          ref.set(prev, 0, prev.length);
+          context.write(ref, row);
+        } else {
+          LOG.warn(String.format("Prev is not set for: %s", Bytes.toStringBinary(rowKey)));
         }
       }
     }
 
     public static enum Counts {
-      UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+      UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT, EXTRAREFERENCES
     }
 
-    public static class VerifyReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
-      private ArrayList<Long> refs = new ArrayList<Long>();
+    public static class VerifyReducer extends Reducer<BytesWritable,BytesWritable,Text,Text> {
+      private ArrayList<byte[]> refs = new ArrayList<byte[]>();
 
-      public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context)
+      public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
           throws IOException, InterruptedException {
 
         int defCount = 0;
 
         refs.clear();
-        for (VLongWritable type : values) {
-          if (type.get() == -1) {
+        for (BytesWritable type : values) {
+          if (type.getLength() == DEF.getLength()) {
             defCount++;
           } else {
-            refs.add(type.get());
+            byte[] bytes = new byte[type.getLength()];
+            System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
+            refs.add(bytes);
           }
         }
 
         // TODO check for more than one def, should not happen
 
-        if (defCount == 0 && refs.size() > 0) {
-          // this is bad, found a node that is referenced but not defined. It must have been
-          //lost, emit some info about this node for debugging purposes.
-
-          StringBuilder sb = new StringBuilder();
+        StringBuilder refsSb = null;
+        String keyString = null;
+        if (defCount == 0 || refs.size() != 1) {
+          refsSb = new StringBuilder();
           String comma = "";
-          for (Long ref : refs) {
-            sb.append(comma);
+          for (byte[] ref : refs) {
+            refsSb.append(comma);
             comma = ",";
-            sb.append(String.format("%016x", ref));
+            refsSb.append(Bytes.toStringBinary(ref));
           }
+          byte[] bytes = new byte[key.getLength()];
+          keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength());
+        }
 
-          context.write(new Text(String.format("%016x", key.get())), new Text(sb.toString()));
+        if (defCount == 0 && refs.size() > 0) {
+          // this is bad, found a node that is referenced but not defined. It must have been
+          // lost, emit some info about this node for debugging purposes.
+          context.write(new Text(keyString), new Text(refsSb.toString()));
           context.getCounter(Counts.UNDEFINED).increment(1);
-
         } else if (defCount > 0 && refs.size() == 0) {
           // node is defined but not referenced
+          context.write(new Text(keyString), new Text("none"));
           context.getCounter(Counts.UNREFERENCED).increment(1);
         } else {
+          if (refs.size() > 1) {
+            context.write(new Text(keyString), new Text(refsSb.toString()));
+            context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
+          }
           // node is defined and referenced
           context.getCounter(Counts.REFERENCED).increment(1);
         }
@@ -599,7 +632,7 @@ public int run(Path outputDir, int numReducers) throws Exception {
       scan.setCacheBlocks(false);
 
       TableMapReduceUtil.initTableMapperJob(getTableName(getConf()), scan,
-          VerifyMapper.class, LongWritable.class, VLongWritable.class, job);
+          VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
 
       job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
 
@@ -622,6 +655,7 @@ public boolean verify(long expectedReferenced) throws Exception {
       Counter referenced = counters.findCounter(Counts.REFERENCED);
       Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
       Counter undefined = counters.findCounter(Counts.UNDEFINED);
+      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);
 
       boolean success = true;
       //assert
@@ -632,7 +666,9 @@ public boolean verify(long expectedReferenced) throws Exception {
       }
 
       if (unreferenced.getValue() > 0) {
-        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
+        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
+        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
+            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
         success = false;
       }
 
@@ -653,15 +689,15 @@ private static class Loop extends Configured implements Tool {
 
     private static final Log LOG = LogFactory.getLog(Loop.class);
 
-    protected void runGenerator(int numMappers, long numNodes, String outputDir) throws Exception {
+    protected void runGenerator(int numMappers, long numNodes,
+        String outputDir, Integer width, Integer wrapMuplitplier) throws Exception {
       Path outputPath = new Path(outputDir);
       UUID uuid = UUID.randomUUID(); //create a random UUID.
       Path generatorOutput = new Path(outputPath, uuid.toString());
 
       Generator generator = new Generator();
       generator.setConf(getConf());
-      int retCode = generator.run(numMappers, numNodes, generatorOutput);
-
+      int retCode = generator.run(numMappers, numNodes, generatorOutput, width, wrapMuplitplier);
       if (retCode > 0) {
         throw new RuntimeException("Generator failed with return code: " + retCode);
       }
@@ -690,10 +726,9 @@ protected void runVerify(String outputDir, int numReducers, long expectedNumNode
     @Override
     public int run(String[] args) throws Exception {
       if (args.length < 5) {
-        System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers>");
+        System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers> [<width> <wrap multiplier>]");
         return 1;
       }
-
       LOG.info("Running Loop with args:" + Arrays.deepToString(args));
 
       int numIterations = Integer.parseInt(args[0]);
@@ -701,6 +736,8 @@ public int run(String[] args) throws Exception {
       long numNodes = Long.parseLong(args[2]);
       String outputDir = args[3];
       int numReducers = Integer.parseInt(args[4]);
+      Integer width = (args.length < 6) ? null : Integer.parseInt(args[5]);
+      Integer wrapMuplitplier = (args.length < 7) ? null : Integer.parseInt(args[6]);
 
       long expectedNumNodes = 0;
 
@@ -708,9 +745,9 @@ public int run(String[] args) throws Exception {
         numIterations = Integer.MAX_VALUE; //run indefinitely (kind of)
       }
 
-      for (int i=0; i < numIterations; i++) {
+      for (int i = 0; i < numIterations; i++) {
         LOG.info("Starting iteration = " + i);
-        runGenerator(numMappers, numNodes, outputDir);
+        runGenerator(numMappers, numNodes, outputDir, width, wrapMuplitplier);
         expectedNumNodes += numMappers * numNodes;
 
         runVerify(outputDir, numReducers, expectedNumNodes);
@@ -751,10 +788,10 @@ public int run(String[] args) throws Exception {
       scan.setBatch(10000);
 
       if (cmd.hasOption("s"))
-        scan.setStartRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("s"), 16).longValue()));
+        scan.setStartRow(Bytes.toBytesBinary(cmd.getOptionValue("s")));
 
       if (cmd.hasOption("e"))
-        scan.setStopRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("e"), 16).longValue()));
+        scan.setStopRow(Bytes.toBytesBinary(cmd.getOptionValue("e")));
 
       int limit = 0;
       if (cmd.hasOption("l"))
@@ -769,7 +806,8 @@ public int run(String[] args) throws Exception {
       int count = 0;
       while (result != null && count++ < limit) {
         node = getCINode(result, node);
-        System.out.printf("%016x:%016x:%012d:%s\n", node.key, node.prev, node.count, node.client);
+        System.out.printf("%s:%s:%012d:%s\n", Bytes.toStringBinary(node.key),
+            Bytes.toStringBinary(node.prev), node.count, node.client);
         result = scanner.next();
       }
       scanner.close();
@@ -788,10 +826,10 @@ public int run(String[] args) throws Exception {
         System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
         return 0;
       }
-      long val = new BigInteger(args[0], 16).longValue();
+      byte[] val = Bytes.toBytesBinary(args[0]);
 
       org.apache.hadoop.hbase.client.Delete delete
-        = new org.apache.hadoop.hbase.client.Delete(Bytes.toBytes(val));
+        = new org.apache.hadoop.hbase.client.Delete(val);
 
       HTable table = new HTable(getConf(), getTableName(getConf()));
 
@@ -811,6 +849,8 @@ private static class Walker extends Configured implements Tool {
     public int run(String[] args) throws IOException {
       Options options = new Options();
       options.addOption("n", "num", true, "number of queries");
+      options.addOption("s", "start", true, "key to start at, binary string");
+      options.addOption("l", "logevery", true, "log every N queries");
 
       GnuParser parser = new GnuParser();
       CommandLine cmd = null;
@@ -831,30 +871,40 @@ public int run(String[] args) throws IOException {
       if (cmd.hasOption('n')) {
         maxQueries = Long.parseLong(cmd.getOptionValue("n"));
       }
-
-      HTable table = new HTable(getConf(), getTableName(getConf()));
-
       Random rand = new Random();
+      boolean isSpecificStart = cmd.hasOption('s');
+      byte[] startKey = isSpecificStart ? Bytes.toBytesBinary(cmd.getOptionValue('s')) : null;
+      int logEvery = cmd.hasOption('l') ? Integer.parseInt(cmd.getOptionValue('l')) : 1;
 
+      HTable table = new HTable(getConf(), getTableName(getConf()));
       long numQueries = 0;
-
-      while (numQueries < maxQueries) {
-        CINode node = findStartNode(rand, table);
+      // If isSpecificStart is set, only walk one list from that particular node.
+      // Note that in case of circular (or P-shaped) list it will walk forever, as is
+      // the case in normal run without startKey.
+      while (numQueries < maxQueries && (numQueries == 0 || !isSpecificStart)) {
+        if (!isSpecificStart) {
+          startKey = new byte[ROWKEY_LENGTH];
+          rand.nextBytes(startKey);
+        }
+        CINode node = findStartNode(table, startKey);
+        if (node == null && isSpecificStart) {
+          System.err.printf("Start node not found: %s \n", Bytes.toStringBinary(startKey));
+        }
         numQueries++;
-        while (node != null && node.prev >= 0 && numQueries < maxQueries) {
-          long prev = node.prev;
-
+        while (node != null && node.prev.length != NO_KEY.length && numQueries < maxQueries) {
+          byte[] prev = node.prev;
           long t1 = System.currentTimeMillis();
           node = getNode(prev, table, node);
           long t2 = System.currentTimeMillis();
-          System.out.printf("CQ %d %016x \n", t2 - t1, prev); //cold cache
-          numQueries++;
-
-          t1 = System.currentTimeMillis();
-          node = getNode(prev, table, node);
-          t2 = System.currentTimeMillis();
-          System.out.printf("HQ %d %016x \n", t2 - t1, prev); //hot cache
+          if (numQueries % logEvery == 0) {
+            System.out.printf("CQ %d: %d %s \n", numQueries, t2 - t1, Bytes.toStringBinary(prev));
+          }
           numQueries++;
+          if (node == null) {
+            System.err.printf("UNDEFINED NODE %s \n", Bytes.toStringBinary(prev));
+          } else if (node.prev.length == NO_KEY.length) {
+            System.err.printf("TERMINATING NODE %s \n", Bytes.toStringBinary(node.key));
+          }
         }
       }
 
@@ -862,9 +912,9 @@ public int run(String[] args) throws IOException {
       return 0;
     }
 
-    private static CINode findStartNode(Random rand, HTable table) throws IOException {
+    private static CINode findStartNode(HTable table, byte[] startKey) throws IOException {
       Scan scan = new Scan();
-      scan.setStartRow(Bytes.toBytes(Math.abs(rand.nextLong())));
+      scan.setStartRow(startKey);
       scan.setBatch(1);
       scan.addColumn(FAMILY_NAME, COLUMN_PREV);
 
@@ -876,7 +926,7 @@ private static CINode findStartNode(Random rand, HTable table) throws IOExceptio
 
       if ( result != null) {
         CINode node = getCINode(result, new CINode());
-        System.out.printf("FSR %d %016x\n", t2 - t1, node.key);
+        System.out.printf("FSR %d %s\n", t2 - t1, Bytes.toStringBinary(node.key));
         return node;
       }
 
@@ -885,8 +935,8 @@ private static CINode findStartNode(Random rand, HTable table) throws IOExceptio
       return null;
     }
 
-    private CINode getNode(long row, HTable table, CINode node) throws IOException {
-      Get get = new Get(Bytes.toBytes(row));
+    private CINode getNode(byte[] row, HTable table, CINode node) throws IOException {
+      Get get = new Get(row);
       get.addColumn(FAMILY_NAME, COLUMN_PREV);
       Result result = table.get(get);
       return getCINode(result, node);
@@ -898,11 +948,14 @@ private static byte[] getTableName(Configuration conf) {
   }
 
   private static CINode getCINode(Result result, CINode node) {
-    node.key = Bytes.toLong(result.getRow());
+    node.key = new byte[result.getRow().length];
+    System.arraycopy(result.getRow(), 0, node.key, 0, node.key.length);
     if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) {
-      node.prev = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_PREV));
+      byte[] value = result.getValue(FAMILY_NAME, COLUMN_PREV);
+      node.prev = new byte[value.length];
+      System.arraycopy(value, 0, node.prev, 0, node.prev.length);
     } else {
-      node.prev = -1;
+      node.prev = NO_KEY;
     }
     if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) {
       node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT));
@@ -1016,4 +1069,16 @@ public static void main(String[] args) throws Exception {
     int ret = ToolRunner.run(HBaseConfiguration.create(), new IntegrationTestBigLinkedList(), args);
     System.exit(ret);
   }
+  
+  private static void setJobConf(Job job, int numMappers, long numNodes,
+    Integer width, Integer wrapMuplitplier) {
+    job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
+    job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+    if (width != null) {
+      job.getConfiguration().setInt(GENERATOR_WIDTH_KEY, width.intValue());
+    }
+    if (wrapMuplitplier != null) {
+      job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMuplitplier.intValue());
+    }
+  }
 }

From f9583081f12e478b6489ea8458310970416de406 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 19 Jun 2013 04:07:48 +0000
Subject: [PATCH 1013/1540] HBASE-8685 [REST] Minor fix to XMLSchema.xsd and
 ScannerModel

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494445 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/rest/model/ScannerModel.java | 2 +-
 src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
index d76bdb18f535..1f6b1d768116 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
@@ -56,8 +56,8 @@
  * &lt;complexType name="Scanner"&gt;
  *   &lt;sequence>
  *     &lt;element name="column" type="base64Binary" minOccurs="0" maxOccurs="unbounded"/&gt;
+ *     &lt;element name="filter" type="string" minOccurs="0" maxOccurs="1"&gt;&lt;/element&gt;
  *   &lt;/sequence&gt;
- *   &lt;element name="filter" type="string" minOccurs="0" maxOccurs="1"&gt;&lt;/element&gt;
  *   &lt;attribute name="startRow" type="base64Binary"&gt;&lt;/attribute&gt;
  *   &lt;attribute name="endRow" type="base64Binary"&gt;&lt;/attribute&gt;
  *   &lt;attribute name="batch" type="int"&gt;&lt;/attribute&gt;
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd b/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
index c2df60ddaa7e..9577ce23eac5 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
@@ -98,7 +98,7 @@
     <complexType name="Cell">
         <sequence>
             <element name="value" maxOccurs="1" minOccurs="1">
-                <simpleType><restriction base="base64Binary">
+                <simpleType><restriction base="base64Binary" />
                 </simpleType>
             </element>
         </sequence>
@@ -111,8 +111,6 @@
     <complexType name="Scanner">
         <sequence>
             <element name="column" type="base64Binary" minOccurs="0" maxOccurs="unbounded"></element>
-        </sequence>
-        <sequence>
             <element name="filter" type="string" minOccurs="0" maxOccurs="1"></element>
         </sequence>
         <attribute name="startRow" type="base64Binary"></attribute>

From adfa0471cc06338b8515b2048e0bc49729ac0ff1 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 19 Jun 2013 05:29:31 +0000
Subject: [PATCH 1014/1540] HBASE-8762 Performance/operational penalty when
 calling HTable.get with a list of one Get

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494462 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HTable.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index b7e62e46d000..6375ad67a3d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -719,6 +719,9 @@ public Result call() throws IOException {
    */
   @Override
   public Result[] get(List<Get> gets) throws IOException {
+    if (gets.size() == 1) {
+      return new Result[]{get(gets.get(0))};
+    }
     try {
       Object [] r1 = batch((List)gets);
 

From 47655406d36fa7c5175e3bb43f1b3cecc20c5645 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 19 Jun 2013 08:25:44 +0000
Subject: [PATCH 1015/1540] HBASE-8767 Backport hbase-8001, avoid lazy seek
 (original patch by Raymond Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494494 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 8613a872fcb5..1c5cd38c9e9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -314,7 +314,7 @@ private boolean generalizedSeek(boolean isLazy, KeyValue seekKey,
       }
 
       boolean seekResult;
-      if (isLazy) {
+      if (isLazy && heap.size() > 0) {
         seekResult = scanner.requestSeek(seekKey, forward, useBloom);
       } else {
         seekResult = NonLazyKeyValueScanner.doRealSeek(

From 787e7776d1ad7eaf5ed8ed0c89cbde65e9697a46 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 19 Jun 2013 09:02:21 +0000
Subject: [PATCH 1016/1540] HBASE-8767 Revert, breaks some tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494520 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 1c5cd38c9e9c..8613a872fcb5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -314,7 +314,7 @@ private boolean generalizedSeek(boolean isLazy, KeyValue seekKey,
       }
 
       boolean seekResult;
-      if (isLazy && heap.size() > 0) {
+      if (isLazy) {
         seekResult = scanner.requestSeek(seekKey, forward, useBloom);
       } else {
         seekResult = NonLazyKeyValueScanner.doRealSeek(

From 1ec24ec1a72f6101f8f5e50f870a147abcbe4a24 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 19 Jun 2013 15:49:51 +0000
Subject: [PATCH 1017/1540] HBASE-8494. TestRemoteAdmin#testClusterStatus
 should not assume 'requests' does not change

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494666 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java   | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
index 630a9d8d82c8..78c5be6f17f1 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
@@ -117,9 +117,6 @@ public void testClusterStatus() throws Exception {
     assertEquals(
         "Dead server count from cluster status and returned status did not match up. ",
         status.getDeadServers(), returnedStatus.getDeadNodes().size());
-    assertEquals(
-        "Number of requests from cluster status and returned status did not match up. ",
-        status.getRequestsCount(), returnedStatus.getRequests());
   }
 
   @Test

From 408eb243ad51bbad593d83ad2cfd35cc0e90b38e Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Thu, 20 Jun 2013 06:40:58 +0000
Subject: [PATCH 1018/1540] Fix up RPC handling

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1494871 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ipc/SecureClient.java | 14 +++++++++++++-
 .../hadoop/hbase/security/HBaseSaslRpcClient.java | 14 +++++++++++---
 src/main/resources/hbase-default.xml              | 15 +++++++++++++++
 src/test/resources/hbase-site.xml                 |  4 ++++
 4 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index d11592874a78..5c21d93388b0 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -72,6 +72,10 @@ public class SecureClient extends HBaseClient {
   private static final Log LOG =
     LogFactory.getLog("org.apache.hadoop.ipc.SecureClient");
 
+  public static final String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY =
+      "hbase.ipc.client.fallback-to-simple-auth-allowed";
+  public static final boolean IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT = false;
+
   protected static Map<String,TokenSelector<? extends TokenIdentifier>> tokenHandlers =
       new HashMap<String,TokenSelector<? extends TokenIdentifier>>();
   static {
@@ -173,7 +177,7 @@ private synchronized boolean shouldAuthenticateOverKrb() throws IOException {
     private synchronized boolean setupSaslConnection(final InputStream in2,
         final OutputStream out2)
         throws IOException {
-      saslRpcClient = new HBaseSaslRpcClient(authMethod, token, serverPrincipal);
+      saslRpcClient = new HBaseSaslRpcClient(authMethod, token, serverPrincipal, fallbackAllowed);
       return saslRpcClient.saslConnect(in2, out2);
     }
 
@@ -451,6 +455,8 @@ protected synchronized void close() {
     }
   }
 
+  private final boolean fallbackAllowed;
+
   /**
    * Construct an IPC client whose values are of the given {@link org.apache.hadoop.io.Writable}
    * class.
@@ -461,6 +467,12 @@ protected synchronized void close() {
   public SecureClient(Class<? extends Writable> valueClass, Configuration conf,
       SocketFactory factory) {
     super(valueClass, conf, factory);
+    this.fallbackAllowed =
+      conf.getBoolean(IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
+        IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("fallbackAllowed=" + this.fallbackAllowed);
+    }
   }
 
   /**
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java b/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java
index 809097305b45..c1eb055b1907 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java
@@ -56,6 +56,7 @@ public class HBaseSaslRpcClient {
   public static final Log LOG = LogFactory.getLog(HBaseSaslRpcClient.class);
 
   private final SaslClient saslClient;
+  private final boolean fallbackAllowed;
 
   /**
    * Create a HBaseSaslRpcClient for an authentication method
@@ -66,8 +67,9 @@ public class HBaseSaslRpcClient {
    *          token to use if needed by the authentication method
    */
   public HBaseSaslRpcClient(AuthMethod method,
-      Token<? extends TokenIdentifier> token, String serverPrincipal)
-      throws IOException {
+      Token<? extends TokenIdentifier> token, String serverPrincipal,
+      boolean fallbackAllowed) throws IOException {
+    this.fallbackAllowed = fallbackAllowed;
     switch (method) {
     case DIGEST:
       if (LOG.isDebugEnabled())
@@ -148,8 +150,14 @@ public boolean saslConnect(InputStream inS, OutputStream outS)
         readStatus(inStream);
         int len = inStream.readInt();
         if (len == HBaseSaslRpcServer.SWITCH_TO_SIMPLE_AUTH) {
-          if (LOG.isDebugEnabled())
+          if (!fallbackAllowed) {
+            throw new IOException("Server asks us to fall back to SIMPLE auth,"
+              + " but this client is configured to only allow secure"
+              + " connections.");
+          }
+          if (LOG.isDebugEnabled()) {
             LOG.debug("Server asks us to fall back to simple auth.");
+          }
           saslClient.dispose();
           return false;
         }
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index d59457ac7230..e5b7764e8cd0 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -657,6 +657,21 @@
     </description>
   </property>
 
+   <property>
+    <name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
+    <value>false</value>
+    <description>
+      When a client is configured to attempt a secure connection, but
+      attempts to connect to an insecure server, that server may instruct the
+      client to switch to SASL SIMPLE (unsecure) authentication. This setting
+      controls whether or not the client will accept this instruction from the
+      server. When false (the default), the client will not allow the fallback
+      to SIMPLE authentication, and will abort the connection.
+
+      This setting is only used by the secure RPC engine.
+    </description>
+  </property>
+
   <property>
     <name>zookeeper.znode.acl.parent</name>
     <value>acl</value>
diff --git a/src/test/resources/hbase-site.xml b/src/test/resources/hbase-site.xml
index b9a446862210..67e6cf55a8e6 100644
--- a/src/test/resources/hbase-site.xml
+++ b/src/test/resources/hbase-site.xml
@@ -134,4 +134,8 @@
     version is X.X.X-SNAPSHOT"
     </description>
   </property>
+  <property>
+    <name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
+    <value>true</value>
+  </property>
 </configuration>

From bfc537da2acbca3771e75f92df941b92fddf7db8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 21 Jun 2013 22:24:43 +0000
Subject: [PATCH 1019/1540] HBASE-8743 upgrade hadoop-23 version to 0.23.7

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1495626 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 61152caf7f43..cae02cdc31b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1996,7 +1996,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>0.23.3</hadoop.version>
+        <hadoop.version>0.23.7</hadoop.version>
         <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencies>

From df71eddb92ecfc215cb0516af470af387b296b2c Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 24 Jun 2013 08:15:43 +0000
Subject: [PATCH 1020/1540] HBASE-8783 RSSnapshotManager.ZKProcedureMemberRpcs
 may be initialized with the wrong server name

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1495945 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/procedure/ProcedureMemberRpcs.java  |  2 +-
 .../procedure/ZKProcedureCoordinatorRpcs.java |  4 ++--
 .../procedure/ZKProcedureMemberRpcs.java      | 19 +++++++++----------
 .../hbase/procedure/ZKProcedureUtil.java      | 14 +++-----------
 .../snapshot/RegionServerSnapshotManager.java | 11 ++++++-----
 .../hbase/procedure/TestZKProcedure.java      | 10 +++++-----
 .../procedure/TestZKProcedureControllers.java | 14 ++++++--------
 7 files changed, 32 insertions(+), 42 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
index 72cec2bdb2e7..9dc95a12bcd4 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
@@ -35,7 +35,7 @@ public interface ProcedureMemberRpcs extends Closeable {
   /**
    * Initialize and start any threads or connections the member needs.
    */
-  public void start(ProcedureMember member);
+  public void start(final String memberName, final ProcedureMember member);
 
   /**
    * Each subprocedure is being executed on a member.  This is the identifier for the member.
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
index 4595335be912..3f5ae1a53ac1 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
@@ -164,7 +164,7 @@ final public boolean start(final ProcedureCoordinator coordinator) {
     this.coordinator = coordinator;
 
     try {
-      this.zkProc = new ZKProcedureUtil(watcher, procedureType, coordName) {
+      this.zkProc = new ZKProcedureUtil(watcher, procedureType) {
         @Override
         public void nodeCreated(String path) {
           if (!isInProcedurePath(path)) return;
@@ -191,7 +191,7 @@ public void nodeCreated(String path) {
       return false;
     }
 
-    LOG.debug("Starting the controller for procedure member:" + zkProc.getMemberName());
+    LOG.debug("Starting the controller for procedure member:" + coordName);
     return true;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index f5a552a88306..193ace1bebab 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -54,24 +54,23 @@
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
-
   private static final Log LOG = LogFactory.getLog(ZKProcedureMemberRpcs.class);
-  private final String memberName;
+
+  private final ZKProcedureUtil zkController;
 
   protected ProcedureMember member;
-  private ZKProcedureUtil zkController;
+  private String memberName;
 
   /**
-   * Must call {@link #start(ProcedureMember)} before this can be used.
+   * Must call {@link #start(String, ProcedureMember)} before this can be used.
    * @param watcher {@link ZooKeeperWatcher} to be owned by <tt>this</tt>. Closed via
    *          {@link #close()}.
    * @param procType name of the znode describing the procedure type
-   * @param memberName name of the member to join the procedure
    * @throws KeeperException if we can't reach zookeeper
    */
-  public ZKProcedureMemberRpcs(ZooKeeperWatcher watcher,
-      String procType, String memberName) throws KeeperException {
-    this.zkController = new ZKProcedureUtil(watcher, procType, memberName) {
+  public ZKProcedureMemberRpcs(final ZooKeeperWatcher watcher, final String procType)
+      throws KeeperException {
+    this.zkController = new ZKProcedureUtil(watcher, procType) {
       @Override
       public void nodeCreated(String path) {
         if (!isInProcedurePath(path)) {
@@ -113,7 +112,6 @@ public void nodeChildrenChanged(String path) {
         }
       }
     };
-    this.memberName = memberName;
   }
 
   public ZKProcedureUtil getZkController() {
@@ -335,9 +333,10 @@ protected void abort(String abortZNode) {
     }
   }
 
-  public void start(ProcedureMember listener) {
+  public void start(final String memberName, final ProcedureMember listener) {
     LOG.debug("Starting procedure member '" + this.memberName + "'");
     this.member = listener;
+    this.memberName = memberName;
     watchForAbortedProcedures();
     waitForNewProcedures();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
index 0a4dd4a237aa..56941f2781ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -64,8 +64,6 @@ public abstract class ZKProcedureUtil
   protected final String reachedZnode;
   protected final String abortZnode;
 
-  protected final String memberName;
-
   /**
    * Top-level watcher/controller for procedures across the cluster.
    * <p>
@@ -74,13 +72,11 @@ public abstract class ZKProcedureUtil
    * @param watcher watcher for the cluster ZK. Owned by <tt>this</tt> and closed via
    *          {@link #close()}
    * @param procDescription name of the znode describing the procedure to run
-   * @param memberName name of the member from which we are interacting with running procedures
    * @throws KeeperException when the procedure znodes cannot be created
    */
-  public ZKProcedureUtil(ZooKeeperWatcher watcher, String procDescription,
-      String memberName) throws KeeperException {
+  public ZKProcedureUtil(ZooKeeperWatcher watcher, String procDescription)
+      throws KeeperException {
     super(watcher);
-    this.memberName = memberName;
     // make sure we are listening for events
     watcher.registerListener(this);
     // setup paths for the zknodes used in procedures
@@ -127,10 +123,6 @@ public String getAcquiredBarrier() {
     return acquiredZnode;
   }
 
-  public String getMemberName() {
-    return memberName;
-  }
-
   /**
    * Get the full znode path for the node used by the coordinator to trigger a global barrier
    * acquire on each subprocedure.
@@ -189,7 +181,7 @@ boolean isAcquiredNode(String path) {
     return path.equals(acquiredZnode);
   }
 
-  
+
   /**
    * Is this in the procedure barrier acquired znode path
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
index 987a713ca289..019bf971f974 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -119,9 +119,8 @@ public RegionServerSnapshotManager(RegionServerServices rss)
       throws KeeperException {
     this.rss = rss;
     ZooKeeperWatcher zkw = rss.getZooKeeper();
-    String nodeName = rss.getServerName().toString();
     this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
-        SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, nodeName);
+        SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION);
 
     // read in the snapshot request configuration properties
     Configuration conf = rss.getConfiguration();
@@ -130,7 +129,8 @@ public RegionServerSnapshotManager(RegionServerServices rss)
     int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT);
 
     // create the actual snapshot procedure member
-    ThreadPoolExecutor pool = ProcedureMember.defaultPool(wakeMillis, keepAlive, opThreads, nodeName);
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(wakeMillis, keepAlive, opThreads, 
+      rss.getServerName().toString());
     this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder());
   }
 
@@ -138,7 +138,8 @@ public RegionServerSnapshotManager(RegionServerServices rss)
    * Start accepting snapshot requests.
    */
   public void start() {
-    this.memberRpcs.start(member);
+    LOG.debug("Start Snapshot Manager " + rss.getServerName().toString());
+    this.memberRpcs.start(rss.getServerName().toString(), member);
   }
 
   /**
@@ -283,7 +284,7 @@ static class SnapshotSubprocedurePool {
     boolean hasTasks() {
       return futures.size() != 0;
     }
-    
+
     /**
      * Submit a task to the pool.
      *
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
index a0fc5491fda3..fe555ad20ed1 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
@@ -144,11 +144,11 @@ public Procedure createProcedure(ForeignExceptionDispatcher fed, String procName
     // start each member
     for (String member : members) {
       ZooKeeperWatcher watcher = newZooKeeperWatcher();
-      ZKProcedureMemberRpcs comms = new ZKProcedureMemberRpcs(watcher, opDescription, member);
+      ZKProcedureMemberRpcs comms = new ZKProcedureMemberRpcs(watcher, opDescription);
       ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member);
       ProcedureMember procMember = new ProcedureMember(comms, pool2, subprocFactory);
       procMembers.add(new Pair<ProcedureMember, ZKProcedureMemberRpcs>(procMember, comms));
-      comms.start(procMember);
+      comms.start(member, procMember);
     }
 
     // setup mock member subprocedures
@@ -218,11 +218,11 @@ public void testMultiCohortWithMemberTimeoutDuringPrepare() throws Exception {
         expected.size());
     for (String member : expected) {
       ZooKeeperWatcher watcher = newZooKeeperWatcher();
-      ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(watcher, opDescription, member);
+      ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(watcher, opDescription);
       ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member);
       ProcedureMember mem = new ProcedureMember(controller, pool2, subprocFactory);
       members.add(new Pair<ProcedureMember, ZKProcedureMemberRpcs>(mem, controller));
-      controller.start(mem);
+      controller.start(member, mem);
     }
 
     // setup mock subprocedures
@@ -310,7 +310,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
     try {
       task.waitForCompleted();
     } catch (ForeignException fe) {
-      // this may get caught or may not 
+      // this may get caught or may not
     }
 
     // -------------
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
index 745d75aec897..6065cb6e0a1b 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java
@@ -88,7 +88,7 @@ public void testSimpleZKCohortMemberController() throws Exception {
 
     final ForeignExceptionDispatcher monitor = spy(new ForeignExceptionDispatcher());
     final ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(
-        watcher, "testSimple", COHORT_NODE_NAME);
+        watcher, "testSimple");
 
     // mock out cohort member callbacks
     final ProcedureMember member = Mockito
@@ -112,7 +112,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
     }).when(member).receivedReachedGlobalBarrier(operationName);
 
     // start running the listener
-    controller.start(member);
+    controller.start(COHORT_NODE_NAME, member);
 
     // set a prepare node from a 'coordinator'
     String prepare = ZKProcedureUtil.getAcquireBarrierNode(controller.getZkController(), operationName);
@@ -386,9 +386,8 @@ public Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> start(
 
       List<ZKProcedureMemberRpcs> cohortControllers = new ArrayList<ZKProcedureMemberRpcs>();
       for (String nodeName : expected) {
-        ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(
-            watcher, operationName, nodeName);
-        cc.start(member);
+        ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(watcher, operationName);
+        cc.start(nodeName, member);
         cohortControllers.add(cc);
       }
       return new Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>>(
@@ -411,9 +410,8 @@ public Pair<ZKProcedureCoordinatorRpcs, List<ZKProcedureMemberRpcs>> start(
       // make a cohort controller for each expected node
       List<ZKProcedureMemberRpcs> cohortControllers = new ArrayList<ZKProcedureMemberRpcs>();
       for (String nodeName : expected) {
-        ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(
-            watcher, operationName, nodeName);
-        cc.start(member);
+        ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(watcher, operationName);
+        cc.start(nodeName, member);
         cohortControllers.add(cc);
       }
 

From 413dec2e342ef50e6b227f5095304083d8e2f9e8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Jun 2013 20:42:11 +0000
Subject: [PATCH 1021/1540] HBASE-5083 Backup HMaster should have http infoport
 open with link to the active master (Cody Marcel)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496209 13f79535-47bb-0310-9956-ffa450edef68
---
 .../tmpl/master/BackupMasterStatusTmpl.jamon  | 77 +++++++++++++++++++
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  | 45 +++++++----
 .../hbase/master/ActiveMasterManager.java     | 27 ++++++-
 .../apache/hadoop/hbase/master/HMaster.java   | 41 ++++++----
 .../hbase/master/MasterStatusServlet.java     | 20 +++--
 .../hbase/master/TestMasterStatusServlet.java | 20 +++--
 6 files changed, 182 insertions(+), 48 deletions(-)
 create mode 100644 src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon

diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
new file mode 100644
index 000000000000..660f441c1d1a
--- /dev/null
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
@@ -0,0 +1,77 @@
+<%doc>
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+</%doc>
+<%args>
+HMaster master;
+</%args>
+<%import>
+java.util.*;
+org.apache.hadoop.hbase.util.Bytes;
+org.apache.hadoop.hbase.ServerName;
+org.apache.hadoop.hbase.ClusterStatus;
+org.apache.hadoop.hbase.master.HMaster;
+org.apache.hadoop.hbase.master.ServerManager;
+org.apache.hadoop.hbase.master.AssignmentManager;
+org.apache.hadoop.hbase.master.ActiveMasterManager;
+</%import>
+<%java>
+Collection<ServerName> masters;
+
+if (master.isActiveMaster()) {
+    ClusterStatus status = master.getClusterStatus();
+    masters = status.getBackupMasters();
+} else{
+    ServerName sn = master.getActiveMasterManager().getActiveMaster() ;
+    assert sn != null : "Failed to retreive master's ServerName!";
+
+    List<ServerName> serverNames = new ArrayList<ServerName>(1);
+    serverNames.add(sn);
+    masters = Collections.unmodifiableCollection(serverNames);
+}
+</%java>
+
+<%java>
+ServerName [] serverNames = masters.toArray(new ServerName[masters.size()]);
+</%java>
+<%if (!master.isActiveMaster()) %>
+    <h2>Master</h2>
+    <a href="<% serverNames[0].getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverNames[0].getHostname() %></a>
+<%else>
+    <h2>Backup Masters</h2>
+
+    <table class="table table-striped">
+    <tr>
+        <th>ServerName</th>
+        <th>Port</th>
+        <th>Start Time</th>
+    </tr>
+    <%java>
+    Arrays.sort(serverNames);
+    for (ServerName serverName: serverNames) {
+    </%java>
+    <tr>
+        <td><a href="<% serverName.getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverName.getHostname() %></a></td>
+        <td><% serverName.getPort() %></td>
+        <td><% new Date(serverName.getStartcode()) %></td>
+    </tr>
+    <%java>
+    }
+    </%java>
+    <tr><td>Total:<% (masters != null) ? masters.size() : 0 %></td>
+    </table>
+</%if>
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 097934e91b5c..3a28794c4949 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -28,6 +28,8 @@ Set<ServerName> deadServers = null;
 boolean showAppendWarning = false;
 String filter = "general";
 String format = "html";
+ServerManager serverManager = null;
+AssignmentManager assignmentManager = null;
 </%args>
 <%import>
 java.util.*;
@@ -36,6 +38,8 @@ org.apache.hadoop.hbase.util.Bytes;
 org.apache.hadoop.hbase.util.JvmVersion;
 org.apache.hadoop.hbase.util.FSUtils;
 org.apache.hadoop.hbase.master.HMaster;
+org.apache.hadoop.hbase.master.AssignmentManager;
+org.apache.hadoop.hbase.master.ServerManager;
 org.apache.hadoop.hbase.HConstants;
 org.apache.hadoop.hbase.HServerLoad;
 org.apache.hadoop.hbase.ServerName;
@@ -49,6 +53,11 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
   <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &>
   <%java return; %>
 </%if>
+<%java>
+ServerManager serverManager = master.getServerManager();
+AssignmentManager assignmentManager = master.getAssignmentManager();
+</%java>
+
 <?xml version="1.0" encoding="UTF-8" ?>
 <!-- Commenting out DOCTYPE so our blue outline shows on hadoop 0.20.205.0, etc.
      See tail of HBASE-2110 for explaination.
@@ -100,20 +109,23 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 <tr><td>Hadoop Version</td><td><% org.apache.hadoop.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.util.VersionInfo.getRevision() %></td><td>Hadoop version and revision</td></tr>
 <tr><td>Hadoop Compiled</td><td><% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %></td><td>When Hadoop version was compiled and by whom</td></tr>
 <tr><td>HBase Root Directory</td><td><% FSUtils.getRootDir(master.getConfiguration()).toString() %></td><td>Location of HBase home directory</td></tr>
-<tr><td>HBase Cluster ID</td><td><% master.getClusterId() != null ? master.getClusterId() : "Not set" %><td>Unique identifier generated for each HBase cluster</td></tr>
-<tr><td>Load average</td><td><% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %></td><td>Average number of regions per regionserver. Naive computation.</td></tr>
-<%if frags != null %>
-        <tr><td>Fragmentation</td><td><% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %></td><td>Overall fragmentation of all tables, including .META. and -ROOT-.</td></tr>
-</%if>
 <tr><td>Zookeeper Quorum</td><td><% master.getZooKeeperWatcher().getQuorum() %></td><td>Addresses of all registered ZK servers. For more, see <a href="/zk.jsp">zk dump</a>.</td></tr>
-<tr>
-  <td>
-   Coprocessors</td><td><% java.util.Arrays.toString(master.getCoprocessors()) %>
-  </td>
-  <td>Coprocessors currently loaded loaded by the master</td>
-</tr>
 <tr><td>HMaster Start Time</td><td><% new Date(master.getMasterStartTime()) %></td><td>Date stamp of when this HMaster was started</td></tr>
-<tr><td>HMaster Active Time</td><td><% new Date(master.getMasterActiveTime()) %></td><td>Date stamp of when this HMaster became active</td></tr>
+
+<%if master.isActiveMaster() %>
+  <tr><td>HMaster Active Time</td><td><% new Date(master.getMasterActiveTime()) %></td><td>Date stamp of when this HMaster became active</td></tr>
+  <tr><td>Load average</td><td><% StringUtils.limitDecimalTo2((serverManager != null) ? serverManager.getAverageLoad() : 0) %></td><td>Average number of regions per regionserver. Naive computation.</td></tr>
+  <tr><td>HBase Cluster ID</td><td><% master.getClusterId() != null ? master.getClusterId() : "Not set" %><td>Unique identifier generated for each HBase cluster</td></tr>
+  <%if frags != null %>
+          <tr><td>Fragmentation</td><td><% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %></td><td>Overall fragmentation of all tables, including .META. and -ROOT-.</td></tr>
+  </%if>
+  <tr>
+    <td>
+     Coprocessors</td><td><% java.util.Arrays.toString(master.getCoprocessors()) %>
+    </td>
+    <td>Coprocessors currently loaded loaded by the master</td>
+  </tr>
+</%if>
 </table>
 
 <& ../common/TaskMonitorTmpl; filter = filter &>
@@ -131,13 +143,16 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 <%if (deadServers != null) %>
 <& deadRegionServers &>
 </%if>
-
-<& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&>
+<%if (assignmentManager != null) %>
+<& AssignmentManagerStatusTmpl; assignmentManager=assignmentManager&>
+</%if>
+<%if (master != null) %>
+    <& BackupMasterStatusTmpl; master = master &>
+</%if>
 
 </body>
 </html>
 
-
 <%def catalogTables>
 <h2>Tables</h2>
 <table>
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index c1c4294b5cab..2d2e5867ad7f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -23,16 +23,14 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
-import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 
 /**
  * Handles everything on master-side related to master election.
@@ -47,7 +45,7 @@
  * #blockUntilBecomingActiveMaster() is called to wait until becoming
  * the active master of the cluster.
  */
-class ActiveMasterManager extends ZooKeeperListener {
+public class ActiveMasterManager extends ZooKeeperListener {
   private static final Log LOG = LogFactory.getLog(ActiveMasterManager.class);
 
   final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false);
@@ -242,4 +240,25 @@ public void stop() {
       LOG.error(this.watcher.prefix("Error deleting our own master address node"), e);
     }
   }
+
+  /**
+   * @return the ServerName for the current active master
+   */
+  public ServerName getActiveMaster() {
+    ServerName sn = null;
+    String msg;
+    try {
+      byte[] bytes = ZKUtil.getDataAndWatch(this.watcher, this.watcher.masterAddressZNode);
+      if (bytes == null) {
+        msg = "A master was detected, but went down before its address.";
+        LOG.info(msg);
+      } else {
+        sn = ServerName.parseVersionedServerName(bytes);
+      }
+    } catch (KeeperException e) {
+      msg = "Could not find active master";
+      LOG.info(msg);
+    }
+    return sn;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 1eff890517fa..f3bd95c679f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -100,9 +100,9 @@
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
@@ -130,6 +130,7 @@
 import com.google.common.collect.ClassToInstanceMap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.MutableClassToInstanceMap;
+import com.google.protobuf.ServiceException;
 
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
@@ -389,6 +390,18 @@ public void run() {
     masterStartTime = System.currentTimeMillis();
     try {
       this.registeredZKListenersBeforeRecovery = this.zooKeeper.getListeners();
+
+      // Put up info server.
+      int port = this.conf.getInt("hbase.master.info.port", 60010);
+      if (port >= 0) {
+        String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");
+        this.infoServer = new InfoServer(MASTER, a, port, false, this.conf);
+        this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class);
+        this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class);
+        this.infoServer.setAttribute(MASTER, this);
+        this.infoServer.start();
+      }
+
       /*
        * Block on becoming the active master.
        *
@@ -943,6 +956,10 @@ public ZooKeeperWatcher getZooKeeperWatcher() {
     return this.zooKeeper;
   }
 
+  public ActiveMasterManager getActiveMasterManager() {
+    return this.activeMasterManager;
+  }
+
   /*
    * Start up all services. If any of these threads gets an unhandled exception
    * then they just die with a logged message.  This should be fine because
@@ -982,17 +999,6 @@ this, conf, getMasterFileSystem().getFileSystem(),
         .getFileSystem(), archiveDir);
     Threads.setDaemonThreadRunning(hfileCleaner.getThread(), n + ".archivedHFileCleaner");
 
-   // Put up info server.
-   int port = this.conf.getInt("hbase.master.info.port", 60010);
-   if (port >= 0) {
-     String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");
-     this.infoServer = new InfoServer(MASTER, a, port, false, this.conf);
-     this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class);
-     this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class);
-     this.infoServer.setAttribute(MASTER, this);
-     this.infoServer.start();
-    }
-
    // Start the health checker
    if (this.healthCheckChore != null) {
      Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker");
@@ -1551,7 +1557,7 @@ public int compare(ServerName s1, ServerName s2) {
   }
 
   public String getClusterId() {
-    return fileSystemManager.getClusterId();
+    return (fileSystemManager == null) ? null : fileSystemManager.getClusterId();
   }
 
   /**
@@ -1583,9 +1589,12 @@ public long getMasterActiveTime() {
    * @return array of coprocessor SimpleNames.
    */
   public String[] getCoprocessors() {
-    Set<String> masterCoprocessors =
-        getCoprocessorHost().getCoprocessors();
-    return masterCoprocessors.toArray(new String[0]);
+    MasterCoprocessorHost cp = getCoprocessorHost();
+    String[] cpList = new String[0];
+    if (cp == null) return cpList;
+
+    Set<String> masterCoprocessors = cp.getCoprocessors();
+    return masterCoprocessors.toArray(cpList);
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
index 862db2e68260..5f37aa455988 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.master;
 
 import java.io.IOException;
-
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -34,8 +33,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
+import org.apache.hadoop.hbase.util.FSUtils;
 
 /**
  * The servlet responsible for rendering the index page of the
@@ -54,13 +53,19 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     
     Configuration conf = master.getConfiguration();
     HBaseAdmin admin = new HBaseAdmin(conf);    
-        
+    ServerName metaLocation = null;
+    List<ServerName> servers = null;
+    Set<ServerName> deadServers = null;
+
     Map<String, Integer> frags = getFragmentationInfo(master, conf);
     
     ServerName rootLocation = getRootLocationOrNull(master);
-    ServerName metaLocation = master.getCatalogTracker().getMetaLocation();
-    List<ServerName> servers = master.getServerManager().getOnlineServersList();
-    Set<ServerName> deadServers = master.getServerManager().getDeadServers();
+
+    if (master.isActiveMaster()) {
+      metaLocation = master.getCatalogTracker().getMetaLocation();
+      servers = master.getServerManager().getOnlineServersList();
+      deadServers = master.getServerManager().getDeadServers();
+    }
 
     response.setContentType("text/html");
     MasterStatusTmpl tmpl = new MasterStatusTmpl()
@@ -80,7 +85,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
 
   private ServerName getRootLocationOrNull(HMaster master) {
     try {
-      return master.getCatalogTracker().getRootLocation();
+      return (master.getCatalogTracker() == null) ? null : master.getCatalogTracker()
+          .getRootLocation();
     } catch (InterruptedException e) {
       LOG.warn("Unable to get root location", e);
       return null;
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
index be5dea526e94..b62185df62fe 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
@@ -19,7 +19,8 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.io.StringWriter;
@@ -31,15 +32,17 @@
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.ServerManager;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.hbase.tmpl.master.AssignmentManagerStatusTmpl;
 import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -73,6 +76,11 @@ public void setupBasicMocks() {
     Mockito.doReturn(FAKE_HOST).when(master).getServerName();
     Mockito.doReturn(conf).when(master).getConfiguration();
     
+    // Fake ActiveMasterManager
+    ActiveMasterManager amm = Mockito.mock(ActiveMasterManager.class);
+    Mockito.doReturn(amm).when(master).getActiveMasterManager();
+    Mockito.doReturn(FAKE_HOST).when(amm).getActiveMaster();
+
     // Fake serverManager
     ServerManager serverManager = Mockito.mock(ServerManager.class);
     Mockito.doReturn(1.0).when(serverManager).getAverageLoad();

From 32b7f3f4e19c6712d81f8499ba47dea96d78212b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Jun 2013 20:45:36 +0000
Subject: [PATCH 1022/1540] CHANGES and pom for 0.94.9RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496212 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 41 +++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 02dfcf88f12f..b3696f0db3f5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,46 @@
 HBase Change Log
 
+Release 0.94.9 - 6/24/2013
+Sub-task
+
+    [HBASE-8453] - TestImportExport failing again due to configuration issues
+
+Bug
+
+    [HBASE-8494] - TestRemoteAdmin#testClusterStatus should not assume 'requests' does not change
+    [HBASE-8522] - Archived hfiles and old hlogs may be deleted immediately by HFileCleaner, LogCleaner in HMaster
+    [HBASE-8555] - FilterList correctness may be affected by random ordering of sub-filter(list)
+    [HBASE-8590] - [0.94] BlockingMetaScannerVisitor should check for parent meta entry while waiting for split daughter
+    [HBASE-8639] - Poor performance of htable#getscanner in multithreaded environment due to DNS.getDefaultHost() being called in ScannerCallable#prepare()
+    [HBASE-8640] - ServerName in master may not initialize with the configured ipc address of hbase.master.ipc.address
+    [HBASE-8655] - Backport to 94 - HBASE-8346(Prefetching .META. rows in case only when useCache is set to true)
+    [HBASE-8671] - Per-region WAL breaks CP backwards compatibility in 0.94 for non-enabled case
+    [HBASE-8684] - Table Coprocessor can't access external HTable by default
+    [HBASE-8700] - IntegrationTestBigLinkedList can fail due to random number collision
+    [HBASE-8724] - [0.94] ExportSnapshot should not use hbase.tmp.dir as a staging dir on hdfs
+    [HBASE-8742] - HTableDescriptor Properties not preserved when cloning
+    [HBASE-8743] - upgrade hadoop-23 version to 0.23.7
+    [HBASE-8749] - Potential race condition between FSUtils.renameAndSetModifyTime() and HFile/LogCleaner
+    [HBASE-8762] - Performance/operational penalty when calling HTable.get with a list of one Get
+    [HBASE-8783] - RSSnapshotManager.ZKProcedureMemberRpcs may be initialized with the wrong server name
+
+Improvement
+
+    [HBASE-5083] - Backup HMaster should have http infoport open with link to the active master
+    [HBASE-8609] - Make the CopyTable support startRow, stopRow options
+    [HBASE-8636] - Backport KeyValue Codec to 0.94 (HBASE-7413)
+    [HBASE-8683] - Add major compaction support in CompactionTool
+    [HBASE-8702] - Make WALEditCodec pluggable
+
+New Feature
+
+    [HBASE-8504] - HTable.getRegionsInRange() should provide a non-cached API
+
+Task
+
+    [HBASE-8603] - Backport HBASE-6921 to 0.94
+
+
 Release 0.94.8 - 5/22/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index cae02cdc31b1..6b8cdaa71fe5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.9-SNAPSHOT</version>
+  <version>0.94.9</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 88c6cf088f7868274ef2d9ddedd552f669997cb5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Jun 2013 20:52:05 +0000
Subject: [PATCH 1023/1540] HBASE-8656 Rpc call may not be notified in
 SecureClient (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496216 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/SecureClient.java   | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 5c21d93388b0..e25085504dac 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -367,7 +367,11 @@ protected void receiveResponse() {
         if (LOG.isDebugEnabled())
           LOG.debug(getName() + " got value #" + id);
 
-        Call call = calls.remove(id);
+        // we first get the call by id, then remove it from call map after processed.
+        // If we remove the call here, thread waiting on the call can not be notified
+        // if any we encounter any exception in the 'try' block. Refer to 'receiveResponse'
+        // in org.apache.hadoop.hbase.ipc.HBaseClient.java
+        Call call = calls.get(id);
 
         int state = in.readInt();     // read call status
         if (LOG.isDebugEnabled()) {
@@ -390,10 +394,17 @@ protected void receiveResponse() {
                 .readString(in)));
           }
         } else if (state == Status.FATAL.state) {
+          RemoteException exception = new RemoteException(WritableUtils.readString(in),
+              WritableUtils.readString(in));
+          // the call will be removed from call map, we must set Exception here to notify
+          // the thread waited on the call
+          if (call != null) {
+            call.setException(exception);
+          }
           // Close the connection
-          markClosed(new RemoteException(WritableUtils.readString(in),
-                                         WritableUtils.readString(in)));
+          markClosed(exception);
         }
+        calls.remove(id);
       } catch (IOException e) {
         if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
           // Clean up open calls but don't treat this as a fatal condition,

From 86c8d139e2b352aaa7751514c147571405d3bc82 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Jun 2013 20:53:24 +0000
Subject: [PATCH 1024/1540] CHANGES one more for 0.94.9RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496217 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index b3696f0db3f5..88abf80bc9df 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,6 +14,7 @@ Bug
     [HBASE-8639] - Poor performance of htable#getscanner in multithreaded environment due to DNS.getDefaultHost() being called in ScannerCallable#prepare()
     [HBASE-8640] - ServerName in master may not initialize with the configured ipc address of hbase.master.ipc.address
     [HBASE-8655] - Backport to 94 - HBASE-8346(Prefetching .META. rows in case only when useCache is set to true)
+    [HBASE-8656] - Rpc call may not be notified in SecureClient
     [HBASE-8671] - Per-region WAL breaks CP backwards compatibility in 0.94 for non-enabled case
     [HBASE-8684] - Table Coprocessor can't access external HTable by default
     [HBASE-8700] - IntegrationTestBigLinkedList can fail due to random number collision

From d78ea9649c65925ffea989161737044f0a819d8d Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 24 Jun 2013 22:21:41 +0000
Subject: [PATCH 1025/1540] HBASE-8692. [AccessController] Restrict
 HTableDescriptor enumeration

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496240 13f79535-47bb-0310-9956-ffa450edef68
---
 .../security/access/AccessController.java     | 37 ++++++++
 .../security/access/TestAccessController.java | 91 +++++++++++++++++++
 .../hadoop/hbase/client/HBaseAdmin.java       | 14 +--
 .../hbase/client/HConnectionManager.java      | 11 +--
 .../hbase/coprocessor/BaseMasterObserver.java | 11 +++
 .../hbase/coprocessor/MasterObserver.java     | 20 ++++
 .../hadoop/hbase/ipc/HMasterInterface.java    | 19 +++-
 .../apache/hadoop/hbase/master/HMaster.java   | 57 ++++++++----
 .../hbase/master/MasterCoprocessorHost.java   | 41 +++++++++
 src/main/ruby/hbase/admin.rb                  | 11 +--
 .../hbase/coprocessor/TestMasterObserver.java | 38 +++++++-
 src/test/ruby/hbase/admin_test.rb             |  2 +-
 12 files changed, 301 insertions(+), 51 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 28dea867eee2..da1685a95ff3 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -35,6 +35,8 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
@@ -56,6 +58,7 @@
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.ProtocolSignature;
 import org.apache.hadoop.hbase.ipc.RequestContext;
+import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -1330,4 +1333,38 @@ public void preStopRegionServer(ObserverContext<RegionServerCoprocessorEnvironme
       throws IOException {
     requirePermission("stop", Permission.Action.ADMIN);
   }
+
+  @Override
+  public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+      List<String> tableNamesList, List<HTableDescriptor> descriptors) throws IOException {
+    // If the list is empty, this is a request for all table descriptors and requires GLOBAL
+    // ADMIN privs.
+    if (tableNamesList == null || tableNamesList.isEmpty()) {
+      requirePermission("getTableDescriptors", Permission.Action.ADMIN);
+    }
+    // Otherwise, if the requestor has ADMIN or CREATE privs for all listed tables, the
+    // request can be granted.
+    else {
+      MasterServices masterServices = ctx.getEnvironment().getMasterServices();
+      for (String tableName: tableNamesList) {
+        // Do not deny if the table does not exist
+        byte[] nameAsBytes = Bytes.toBytes(tableName);
+        try {
+          masterServices.checkTableModifiable(nameAsBytes);
+        } catch (TableNotFoundException ex) {
+          // Skip checks for a table that does not exist
+          continue;
+        } catch (TableNotDisabledException ex) {
+          // We don't care about this
+        }
+        requirePermission("getTableDescriptors", nameAsBytes, null, null,
+          Permission.Action.ADMIN, Permission.Action.CREATE);
+      }
+    }
+  }
+
+  @Override
+  public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+      List<HTableDescriptor> descriptors) throws IOException {
+  }
 }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 212c14f9342d..861aeb8d73a6 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -79,6 +79,7 @@
 import org.apache.hadoop.hbase.security.access.Permission.Action;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.ipc.RemoteException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -238,6 +239,9 @@ public void verifyDenied(User user, PrivilegedExceptionAction... actions) throws
           }
         }
         else {
+          if (e instanceof RemoteException) {
+            e = ((RemoteException)e).unwrapRemoteException();
+          }
           // For doBulkLoad calls AccessDeniedException
           // is buried in the stack trace
           Throwable ex = e;
@@ -1869,4 +1873,91 @@ public Object run() throws Exception {
     }
   }
 
+  @Test
+  public void testTableDescriptorsEnumeration() throws Exception {
+    User TABLE_ADMIN = User.createUserForTesting(conf, "UserA", new String[0]);
+
+    // Grant TABLE ADMIN privs on test table to UserA
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(
+          AccessControllerProtocol.class, TEST_TABLE);
+      protocol.grant(new UserPermission(Bytes.toBytes(TABLE_ADMIN.getShortName()),
+        TEST_TABLE, null, Permission.Action.ADMIN));
+    } finally {
+      acl.close();
+    }
+
+    PrivilegedExceptionAction listTablesAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+        try {
+          admin.listTables();
+        } finally {
+          admin.close();
+        }
+        return null;
+      }
+    };
+
+    PrivilegedExceptionAction getTableDescAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+        try {
+          admin.getTableDescriptor(TEST_TABLE);
+        } finally {
+          admin.close();
+        }
+        return null;
+      }
+    };
+
+    verifyAllowed(listTablesAction, SUPERUSER, USER_ADMIN);
+    verifyDenied(listTablesAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, TABLE_ADMIN);
+
+    verifyAllowed(getTableDescAction, SUPERUSER, USER_ADMIN, USER_CREATE, TABLE_ADMIN);
+    verifyDenied(getTableDescAction, USER_RW, USER_RO, USER_NONE);
+  }
+
+  @Test
+  public void testTableDeletion() throws Exception {
+    final User tableAdmin = User.createUserForTesting(conf, "TestUser", new String[0]);
+
+    // We need to create a new table here because we will be testing what
+    // happens when it is deleted
+    final byte[] tableName = Bytes.toBytes("testTableDeletion");
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    admin.createTable(htd);
+    TEST_UTIL.waitTableEnabled(tableName, 5000);
+
+    // Grant TABLE ADMIN privs
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(
+          AccessControllerProtocol.class, tableName);
+      protocol.grant(new UserPermission(Bytes.toBytes(tableAdmin.getShortName()),
+        tableName, null, Permission.Action.ADMIN));
+    } finally {
+      acl.close();
+    }
+
+    PrivilegedExceptionAction deleteTableAction = new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+        try {
+          admin.disableTable(tableName);
+          admin.deleteTable(tableName);
+        } finally {
+          admin.close();
+        }
+        return null;
+      }
+    };
+
+    verifyDenied(deleteTableAction, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(deleteTableAction, tableAdmin);
+  }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 6f12fe33e61b..2fb99106550e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -25,6 +25,7 @@
 import java.lang.reflect.Proxy;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.net.SocketTimeoutException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
@@ -564,6 +565,8 @@ public void deleteTable(final byte [] tableName) throws IOException {
     // Wait until all regions deleted
     HRegionInterface server =
       connection.getHRegionConnection(firstMetaServer.getHostname(), firstMetaServer.getPort());
+    List<String> tableNameAsList = new ArrayList<String>(1);
+    tableNameAsList.add(Bytes.toString(tableName));
     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
       long scannerId = -1L;
       try {
@@ -580,15 +583,8 @@ public void deleteTable(final byte [] tableName) throws IOException {
         // HMaster removes the table from its HTableDescriptors
         if (values == null) {
           tableExists = false;
-          HTableDescriptor[] htds = getMaster().getHTableDescriptors();
-          if (htds != null && htds.length > 0) {
-            for (HTableDescriptor htd: htds) {
-              if (Bytes.equals(tableName, htd.getName())) {
-                tableExists = true;
-                break;
-              }
-            }
-          }
+          HTableDescriptor[] htds = getMaster().getHTableDescriptors(tableNameAsList);
+          tableExists = (htds != null && htds.length > 0);
           if (!tableExists) {
             break;
           }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 5af6021d209a..2b1c8328b509 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1889,17 +1889,14 @@ public HTableDescriptor getHTableDescriptor(final byte[] tableName)
       if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
         return HTableDescriptor.META_TABLEDESC;
       }
+      List<String> tableNameList = new ArrayList<String>(1);
+      tableNameList.add(Bytes.toString(tableName));
       if (this.master == null) {
         this.master = getMaster();
       }
-
-      HTableDescriptor[] htds = master.getHTableDescriptors();
+      HTableDescriptor[] htds = master.getHTableDescriptors(tableNameList);
       if (htds != null && htds.length > 0) {
-        for (HTableDescriptor htd: htds) {
-          if (Bytes.equals(tableName, htd.getName())) {
-            return htd;
-          }
-        }
+        return htds[0];
       }
       throw new TableNotFoundException(Bytes.toString(tableName));
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
index 3b71ff879d18..21fe91e9b23c 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
+import java.util.List;
 
 public class BaseMasterObserver implements MasterObserver {
   @Override
@@ -232,4 +233,14 @@ public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment
   public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final SnapshotDescription snapshot) throws IOException {
   }
+
+  @Override
+  public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+      List<String> tableNamesList, List<HTableDescriptor> descriptors) throws IOException {
+  }
+
+  @Override
+  public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+      List<HTableDescriptor> descriptors) throws IOException {
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index 084279e829a0..c2aa3bb85009 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  * Defines coprocessor hooks for interacting with operations on the
@@ -386,4 +387,23 @@ void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    */
   void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final SnapshotDescription snapshot) throws IOException;
+
+  /**
+   * Called before a getTableDescriptors request has been processed.
+   * @param ctx the environment to interact with the framework and master
+   * @param tableNamesList the list of table names, or null if querying for all
+   * @param descriptors an empty list, can be filled with what to return if bypassing
+   * @throws IOException
+   */
+  void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+      List<String> tableNamesList, List<HTableDescriptor> descriptors) throws IOException;
+
+  /**
+   * Called after a getTableDescriptors request has been processed.
+   * @param ctx the environment to interact with the framework and master
+   * @param descriptors the list of descriptors about to be returned
+   * @throws IOException
+   */
+  void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+      List<HTableDescriptor> descriptors) throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index 5cd7a69eac82..bbb23fe565e2 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -166,10 +166,10 @@ public void modifyTable(byte[] tableName, HTableDescriptor htd)
   /**
    * Return cluster status.
    * @return status object
+   * @throws IOException
    */
   public ClusterStatus getClusterStatus();
 
-
   /**
    * Move the region <code>r</code> to <code>dest</code>.
    * @param encodedRegionName The encoded region name; i.e. the hash that makes
@@ -259,16 +259,29 @@ public void unassign(final byte [] regionName, final boolean force)
 
   /**
    * Get array of all HTDs.
+   * <p>
+   * NOTE: This interface up to and including 0.94.8 included a getHTableDescriptors which
+   * did not throw IOE. Adding and deleting checked exceptions declared as thrown by a method
+   * does not break binary compatibility, so rolling restart scenarios will work; however,
+   * it will break contract compatibility possibly requiring source code changes upon
+   * next recompilation.
    * @return array of HTableDescriptor
    */
-  public HTableDescriptor[] getHTableDescriptors();
+  public HTableDescriptor[] getHTableDescriptors() throws IOException;
 
   /**
    * Get array of HTDs for requested tables.
+   * <p>
+   * NOTE: This interface up to and including 0.94.8 included a getHTableDescriptors which
+   * did not throw IOE. Adding and deleting checked exceptions declared as thrown by a method
+   * does not break binary compatibility, so rolling restart scenarios will work; however,
+   * it will break contract compatibility possibly requiring source code changes upon
+   * next recompilation.
    * @param tableNames
    * @return array of HTableDescriptor
+   * @throws IOException 
    */
-  public HTableDescriptor[] getHTableDescriptors(List<String> tableNames);
+  public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) throws IOException;
 
   /**
    * Executes a single {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f3bd95c679f3..48d6f58c6296 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1923,20 +1923,34 @@ public void unassign(final byte [] regionName, final boolean force)
    * @param tableNames
    * @return HTableDescriptor[]
    */
-  public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
-    List<HTableDescriptor> list =
+  public HTableDescriptor[] getHTableDescriptors(List<String> tableNames)
+      throws IOException {
+    List<HTableDescriptor> descriptors =
       new ArrayList<HTableDescriptor>(tableNames.size());
-    for (String s: tableNames) {
-      HTableDescriptor htd = null;
-      try {
-        htd = this.tableDescriptors.get(s);
-      } catch (IOException e) {
-        LOG.warn("Failed getting descriptor for " + s, e);
+    
+    boolean bypass = false;
+    if (this.cpHost != null) {
+      bypass = this.cpHost.preGetTableDescriptors(tableNames, descriptors);
+    }
+
+    if (!bypass) {
+      for (String s: tableNames) {
+        HTableDescriptor htd = null;
+        try {
+          htd = this.tableDescriptors.get(s);
+        } catch (IOException e) {
+          LOG.warn("Failed getting descriptor for " + s, e);
+        }
+        if (htd == null) continue;
+        descriptors.add(htd);
       }
-      if (htd == null) continue;
-      list.add(htd);
     }
-    return list.toArray(new HTableDescriptor [] {});
+
+    if (this.cpHost != null) {
+      this.cpHost.postGetTableDescriptors(descriptors);
+    }
+
+    return descriptors.toArray(new HTableDescriptor [] {});
   }
 
   @Override
@@ -2014,16 +2028,21 @@ public ExecResult execCoprocessor(Exec call) throws IOException {
   /**
    * Get all table descriptors
    * @return All descriptors or null if none.
+   * @throws IOException
    */
-  public HTableDescriptor [] getHTableDescriptors() {
-    Map<String, HTableDescriptor> descriptors = null;
-    try {
-      descriptors = this.tableDescriptors.getAll();
-    } catch (IOException e) {
-      LOG.warn("Failed getting all descriptors", e);
+  public HTableDescriptor [] getHTableDescriptors() throws IOException {
+    List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
+    boolean bypass = false;
+    if (this.cpHost != null) {
+      bypass = this.cpHost.preGetTableDescriptors(null, descriptors);
+    }
+    if (!bypass) {
+      descriptors.addAll(this.tableDescriptors.getAll().values());
+    }
+    if (this.cpHost != null) {
+      this.cpHost.postGetTableDescriptors(descriptors);
     }
-    return descriptors == null?
-      null: descriptors.values().toArray(new HTableDescriptor [] {});
+    return descriptors.toArray(new HTableDescriptor [] {});
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index a158085acf41..936eb5a4d6d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  * Provides the coprocessor framework and environment for master oriented
@@ -772,4 +773,44 @@ public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOExce
       }
     }
   }
+
+  public boolean preGetTableDescriptors(final List<String> tableNamesList,
+      final List<HTableDescriptor> descriptors) throws IOException {
+    boolean bypass = false;
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preGetTableDescriptors(ctx,
+            tableNamesList, descriptors);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postGetTableDescriptors(List<HTableDescriptor> descriptors) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postGetTableDescriptors(ctx, descriptors);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
 }
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index a40a37d3aaf0..b42331ffbd1a 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -293,16 +293,7 @@ def move(encoded_region_name, server = nil)
     #----------------------------------------------------------------------------------------------
     # Returns table's structure description
     def describe(table_name)
-      tables = @admin.listTables.to_a
-      tables << org.apache.hadoop.hbase.HTableDescriptor::META_TABLEDESC
-      tables << org.apache.hadoop.hbase.HTableDescriptor::ROOT_TABLEDESC
-
-      tables.each do |t|
-        # Found the table
-        return t.to_s if t.getNameAsString == table_name
-      end
-
-      raise(ArgumentError, "Failed to find table named #{table_name}")
+      @admin.getTableDescriptor(table_name.to_java_bytes).to_s
     end
 
     #----------------------------------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index eec9151d2d28..ff412b38da4e 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -31,8 +31,6 @@
 import java.util.Map;
 import java.util.NavigableMap;
 
-import junit.framework.Assert;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -102,6 +100,8 @@ public static class CPMasterObserver implements MasterObserver {
     private boolean postRestoreSnapshotCalled;
     private boolean preDeleteSnapshotCalled;
     private boolean postDeleteSnapshotCalled;
+    private boolean preGetTableDescriptorsCalled;
+    private boolean postGetTableDescriptorsCalled;
 
     public void enableBypass(boolean bypass) {
       this.bypass = bypass;
@@ -142,6 +142,8 @@ public void resetStates() {
       postRestoreSnapshotCalled = false;
       preDeleteSnapshotCalled = false;
       postDeleteSnapshotCalled = false;
+      preGetTableDescriptorsCalled = false;
+      postGetTableDescriptorsCalled = false;
     }
 
     @Override
@@ -551,6 +553,22 @@ public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironmen
     public boolean wasDeleteSnapshotCalled() {
       return preDeleteSnapshotCalled && postDeleteSnapshotCalled;
     }
+
+    @Override
+    public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+        List<String> tableNamesList, List<HTableDescriptor> descriptors) throws IOException {
+      preGetTableDescriptorsCalled = true;
+    }
+
+    @Override
+    public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
+        List<HTableDescriptor> descriptors) throws IOException {
+      postGetTableDescriptorsCalled = true;
+    }
+
+    public boolean wasGetTableDescriptorsCalled() {
+      return preGetTableDescriptorsCalled && postGetTableDescriptorsCalled;
+    }
   }
 
   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
@@ -878,6 +896,22 @@ private void waitForRITtoBeZero(HMaster master) throws IOException {
     }
   }
 
+  @Test
+  public void testTableDescriptorsEnumeration() throws Exception {
+    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+
+    HMaster master = cluster.getMaster();
+    MasterCoprocessorHost host = master.getCoprocessorHost();
+    CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
+        CPMasterObserver.class.getName());
+    cp.resetStates();
+
+    master.getHTableDescriptors();
+
+    assertTrue("Coprocessor should be called on table descriptors request",
+      cp.wasGetTableDescriptorsCalled());
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/ruby/hbase/admin_test.rb b/src/test/ruby/hbase/admin_test.rb
index 97bf4433a2d2..ae67ba627bc2 100644
--- a/src/test/ruby/hbase/admin_test.rb
+++ b/src/test/ruby/hbase/admin_test.rb
@@ -161,7 +161,7 @@ def setup
     #-------------------------------------------------------------------------------
 
     define_test "describe should fail for non-existent tables" do
-      assert_raise(ArgumentError) do
+      assert_raise(NativeException) do
         admin.describe('.NOT.EXISTS.')
       end
     end

From 2fc28939001c25b83952ad5795dc40015a26397f Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Tue, 25 Jun 2013 01:08:34 +0000
Subject: [PATCH 1026/1540] HBASE-8776 tweak retry settings some more (on trunk
 and 0.94)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496325 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HConstants.java | 2 +-
 src/main/resources/hbase-default.xml                  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index ac914545c9ee..962529993d06 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -420,7 +420,7 @@ public enum OperationStatusCode {
    * This is a retry backoff multiplier table similar to the BSD TCP syn
    * backoff table, a bit more aggressive than simple exponential backoff.
    */
-  public static int RETRY_BACKOFF[] = { 1, 1, 1, 2, 2, 4, 4, 8, 16, 32 };
+  public static int RETRY_BACKOFF[] = { 1, 1, 1, 2, 2, 4, 4, 8, 16, 32, 64 };
 
   public static final String REGION_IMPL = "hbase.hregion.impl";
 
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index e5b7764e8cd0..328b748277ae 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -130,11 +130,11 @@
   </property>
   <property>
     <name>hbase.client.retries.number</name>
-    <value>10</value>
+    <value>14</value>
     <description>Maximum retries.  Used as maximum for all retryable
     operations such as fetching of the root region from root region
     server, getting a cell's value, starting a row update, etc.
-    Default: 10.
+    Default: 14.
     </description>
   </property> 
   <property>

From 5225eaf1b833aaca956a6ff4518f3530e40a11a2 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Tue, 25 Jun 2013 17:37:53 +0000
Subject: [PATCH 1027/1540] HBASE-8782 Thrift2 can not parse values when using
 framed transport (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1496553 13f79535-47bb-0310-9956-ffa450edef68
---
 .../thrift2/ThriftHBaseServiceHandler.java    | 84 +++++++++----------
 1 file changed, 42 insertions(+), 42 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index 2a8863f271de..11ea77784759 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -30,6 +30,7 @@
 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.resultFromHBase;
 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.resultsFromHBase;
 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift;
+import static org.apache.thrift.TBaseHelper.byteBufferToByteArray;
 
 import java.io.IOException;
 import java.lang.reflect.InvocationHandler;
@@ -62,8 +63,8 @@
 import org.apache.thrift.TException;
 
 /**
- * This class is a glue object that connects Thrift RPC calls to the HBase client API primarily defined in the
- * HTableInterface.
+ * This class is a glue object that connects Thrift RPC calls to the HBase client API primarily
+ * defined in the HTableInterface.
  */
 public class ThriftHBaseServiceHandler implements THBaseService.Iface {
 
@@ -74,46 +75,41 @@ public class ThriftHBaseServiceHandler implements THBaseService.Iface {
   // nextScannerId and scannerMap are used to manage scanner state
   // TODO: Cleanup thread for Scanners, Scanner id wrap
   private final AtomicInteger nextScannerId = new AtomicInteger(0);
-  private final Map<Integer, ResultScanner> scannerMap = new ConcurrentHashMap<Integer, ResultScanner>();
+  private final Map<Integer, ResultScanner> scannerMap =
+      new ConcurrentHashMap<Integer, ResultScanner>();
 
-  public static THBaseService.Iface newInstance(
-      Configuration conf, ThriftMetrics metrics) {
+  public static THBaseService.Iface newInstance(Configuration conf, ThriftMetrics metrics) {
     THBaseService.Iface handler = new ThriftHBaseServiceHandler(conf);
-    return (THBaseService.Iface) Proxy.newProxyInstance(
-        handler.getClass().getClassLoader(),
-        new Class[]{THBaseService.Iface.class},
-        new THBaseServiceMetricsProxy(handler, metrics));
+    return (THBaseService.Iface) Proxy.newProxyInstance(handler.getClass().getClassLoader(),
+      new Class[] { THBaseService.Iface.class }, new THBaseServiceMetricsProxy(handler, metrics));
   }
 
   private static class THBaseServiceMetricsProxy implements InvocationHandler {
     private final THBaseService.Iface handler;
     private final ThriftMetrics metrics;
 
-    private THBaseServiceMetricsProxy(
-        THBaseService.Iface handler, ThriftMetrics metrics) {
+    private THBaseServiceMetricsProxy(THBaseService.Iface handler, ThriftMetrics metrics) {
       this.handler = handler;
       this.metrics = metrics;
     }
 
     @Override
-    public Object invoke(Object proxy, Method m, Object[] args)
-        throws Throwable {
+    public Object invoke(Object proxy, Method m, Object[] args) throws Throwable {
       Object result;
       try {
         long start = now();
         result = m.invoke(handler, args);
-        int processTime = (int)(now() - start);
+        int processTime = (int) (now() - start);
         metrics.incMethodTime(m.getName(), processTime);
       } catch (InvocationTargetException e) {
         throw e.getTargetException();
       } catch (Exception e) {
-        throw new RuntimeException(
-            "unexpected invocation exception: " + e.getMessage());
+        throw new RuntimeException("unexpected invocation exception: " + e.getMessage());
       }
       return result;
     }
   }
-    
+
   private static long now() {
     return System.nanoTime();
   }
@@ -122,8 +118,8 @@ private static long now() {
     htablePool = new HTablePool(conf, Integer.MAX_VALUE);
   }
 
-  private HTableInterface getTable(byte[] tableName) {
-    return htablePool.getTable(tableName);
+  private HTableInterface getTable(ByteBuffer tableName) {
+    return htablePool.getTable(byteBufferToByteArray(tableName));
   }
 
   private void closeTable(HTableInterface table) throws TIOError {
@@ -142,7 +138,6 @@ private TIOError getTIOError(IOException e) {
 
   /**
    * Assigns a unique ID to the scanner and adds the mapping to an internal HashMap.
-   * 
    * @param scanner to add
    * @return Id for this Scanner
    */
@@ -154,7 +149,6 @@ private int addScanner(ResultScanner scanner) {
 
   /**
    * Returns the Scanner associated with the specified Id.
-   * 
    * @param id of the Scanner to get
    * @return a Scanner, or null if the Id is invalid
    */
@@ -164,7 +158,6 @@ private ResultScanner getScanner(int id) {
 
   /**
    * Removes the scanner associated with the specified ID from the internal HashMap.
-   * 
    * @param id of the Scanner to remove
    * @return the removed Scanner, or <code>null</code> if the Id is invalid
    */
@@ -174,7 +167,7 @@ protected ResultScanner removeScanner(int id) {
 
   @Override
   public boolean exists(ByteBuffer table, TGet get) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       return htable.exists(getFromThrift(get));
     } catch (IOException e) {
@@ -186,7 +179,7 @@ public boolean exists(ByteBuffer table, TGet get) throws TIOError, TException {
 
   @Override
   public TResult get(ByteBuffer table, TGet get) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       return resultFromHBase(htable.get(getFromThrift(get)));
     } catch (IOException e) {
@@ -198,7 +191,7 @@ public TResult get(ByteBuffer table, TGet get) throws TIOError, TException {
 
   @Override
   public List<TResult> getMultiple(ByteBuffer table, List<TGet> gets) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       return resultsFromHBase(htable.get(getsFromThrift(gets)));
     } catch (IOException e) {
@@ -210,7 +203,7 @@ public List<TResult> getMultiple(ByteBuffer table, List<TGet> gets) throws TIOEr
 
   @Override
   public void put(ByteBuffer table, TPut put) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       htable.put(putFromThrift(put));
     } catch (IOException e) {
@@ -221,11 +214,13 @@ public void put(ByteBuffer table, TPut put) throws TIOError, TException {
   }
 
   @Override
-  public boolean checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put)
-    throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+  public boolean checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family,
+      ByteBuffer qualifier, ByteBuffer value, TPut put) throws TIOError, TException {
+    HTableInterface htable = getTable(table);
     try {
-      return htable.checkAndPut(row.array(), family.array(), qualifier.array(), (value == null) ? null : value.array(), putFromThrift(put));
+      return htable.checkAndPut(byteBufferToByteArray(row), byteBufferToByteArray(family),
+        byteBufferToByteArray(qualifier), (value == null) ? null : byteBufferToByteArray(value),
+        putFromThrift(put));
     } catch (IOException e) {
       throw getTIOError(e);
     } finally {
@@ -235,7 +230,7 @@ public boolean checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family,
 
   @Override
   public void putMultiple(ByteBuffer table, List<TPut> puts) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       htable.put(putsFromThrift(puts));
     } catch (IOException e) {
@@ -247,7 +242,7 @@ public void putMultiple(ByteBuffer table, List<TPut> puts) throws TIOError, TExc
 
   @Override
   public void deleteSingle(ByteBuffer table, TDelete deleteSingle) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       htable.delete(deleteFromThrift(deleteSingle));
     } catch (IOException e) {
@@ -258,8 +253,9 @@ public void deleteSingle(ByteBuffer table, TDelete deleteSingle) throws TIOError
   }
 
   @Override
-  public List<TDelete> deleteMultiple(ByteBuffer table, List<TDelete> deletes) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+  public List<TDelete> deleteMultiple(ByteBuffer table, List<TDelete> deletes) throws TIOError,
+      TException {
+    HTableInterface htable = getTable(table);
     List<Delete> tempDeletes = deletesFromThrift(deletes);
     try {
       htable.delete(tempDeletes);
@@ -272,15 +268,18 @@ public List<TDelete> deleteMultiple(ByteBuffer table, List<TDelete> deletes) thr
   }
 
   @Override
-  public boolean checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value,
-      TDelete deleteSingle) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+  public boolean checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family,
+      ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle) throws TIOError, TException {
+    HTableInterface htable = getTable(table);
 
     try {
       if (value == null) {
-        return htable.checkAndDelete(row.array(), family.array(), qualifier.array(), null, deleteFromThrift(deleteSingle));
+        return htable.checkAndDelete(byteBufferToByteArray(row), byteBufferToByteArray(family),
+          byteBufferToByteArray(qualifier), null, deleteFromThrift(deleteSingle));
       } else {
-        return htable.checkAndDelete(row.array(), family.array(), qualifier.array(), value.array(), deleteFromThrift(deleteSingle));
+        return htable.checkAndDelete(byteBufferToByteArray(row), byteBufferToByteArray(family),
+          byteBufferToByteArray(qualifier), byteBufferToByteArray(value),
+          deleteFromThrift(deleteSingle));
       }
     } catch (IOException e) {
       throw getTIOError(e);
@@ -291,7 +290,7 @@ public boolean checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer famil
 
   @Override
   public TResult increment(ByteBuffer table, TIncrement increment) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     try {
       return resultFromHBase(htable.increment(incrementFromThrift(increment)));
     } catch (IOException e) {
@@ -303,7 +302,7 @@ public TResult increment(ByteBuffer table, TIncrement increment) throws TIOError
 
   @Override
   public int openScanner(ByteBuffer table, TScan scan) throws TIOError, TException {
-    HTableInterface htable = getTable(table.array());
+    HTableInterface htable = getTable(table);
     ResultScanner resultScanner = null;
     try {
       resultScanner = htable.getScanner(scanFromThrift(scan));
@@ -316,7 +315,8 @@ public int openScanner(ByteBuffer table, TScan scan) throws TIOError, TException
   }
 
   @Override
-  public List<TResult> getScannerRows(int scannerId, int numRows) throws TIOError, TIllegalArgument, TException {
+  public List<TResult> getScannerRows(int scannerId, int numRows) throws TIOError,
+      TIllegalArgument, TException {
     ResultScanner scanner = getScanner(scannerId);
     if (scanner == null) {
       TIllegalArgument ex = new TIllegalArgument();

From 51177d62744e31849f9028ed681d1dbaf721f45d Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 27 Jun 2013 05:06:29 +0000
Subject: [PATCH 1028/1540] HBASE-8811. REST service ignores misspelled 'check'
 parameter, causing unexpected mutations (Chip Salzenberg and Andrew Purtell)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1497211 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/rest/RowResource.java   |  4 +++-
 .../hadoop/hbase/rest/TestRowResource.java      | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index c348f365db06..5bdede75f248 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -181,7 +181,9 @@ Response update(final CellSetModel model, final boolean replace) {
     } else if (CHECK_DELETE.equalsIgnoreCase(check)) {
       return checkAndDelete(model);
     } else if (check != null && check.length() > 0) {
-      LOG.warn("Unknown check value: " + check + ", ignored");
+      return Response.status(Response.Status.BAD_REQUEST)
+        .type(MIMETYPE_TEXT).entity("Invalid check value '" + check + "'" + CRLF)
+        .build();
     }
 
     HTablePool pool = servlet.getTablePool();
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
index 2a0284e00f43..6cff8510ec05 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
@@ -667,6 +667,23 @@ public void testStartEndRowGetPutXML() throws IOException, JAXBException {
     }
   }
 
+  @Test
+  public void testInvalidCheckParam() throws IOException, JAXBException {
+    CellSetModel cellSetModel = new CellSetModel();
+    RowModel rowModel = new RowModel(ROW_1);
+    rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_1),
+      Bytes.toBytes(VALUE_1)));
+    cellSetModel.addRow(rowModel);
+    StringWriter writer = new StringWriter();
+    marshaller.marshal(cellSetModel, writer);
+
+    final String path = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1 + "?check=blah";
+
+    Response response = client.put(path, Constants.MIMETYPE_XML,
+      Bytes.toBytes(writer.toString()));
+    assertEquals(response.getCode(), 400);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 6aaddd239d24bf257290f62f0b96fc7cceaacf15 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 28 Jun 2013 20:22:27 +0000
Subject: [PATCH 1029/1540] HBASE-8829 Improve failed TestMetaScanner assert
 message so can see where/why failure

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1497910 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/TestMetaScanner.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
index 914d7081bbcb..d03987bfdce8 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
@@ -205,7 +205,8 @@ public void run() {
                 endKey = Bytes.toLong(hri.getEndKey());
               }
               LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri);
-              Assert.assertTrue(Bytes.equals(lastEndKey, hri.getStartKey()));
+              Assert.assertTrue("lastEndKey=" + Bytes.toString(lastEndKey) + ", startKey=" +
+                Bytes.toString(hri.getStartKey()), Bytes.equals(lastEndKey, hri.getStartKey()));
               lastEndKey = hri.getEndKey();
             }
             Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW));

From a96e85baac4e753cb9b8417dd1863a260e819272 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 1 Jul 2013 15:55:43 +0000
Subject: [PATCH 1030/1540] HBASE-8809 Include deletes in the scan (setRaw)
 method does not respect the time range or the filter

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1498532 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanQueryMatcher.java  |  6 ++-
 .../hbase/regionserver/TestKeepDeletes.java   | 54 ++++++++++++++++---
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 4b52159b59b4..967159fe0dbc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -296,7 +296,11 @@ public MatchCode match(KeyValue kv) throws IOException {
           || kv.getMemstoreTS() > maxReadPointToTrackVersions) {
         // always include or it is not time yet to check whether it is OK
         // to purge deltes or not
-        return MatchCode.INCLUDE;
+        if (!isUserScan) {
+          // if this is not a user scan (compaction), we can filter this deletemarker right here
+          // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking
+          return MatchCode.INCLUDE;
+        }
       } else if (keepDeletedCells) {
         if (timestamp < earliestPutTs) {
           // keeping delete rows, but there are no puts older than
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
index ae5da03e0e63..357f86695627 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
@@ -276,16 +276,16 @@ public void testRawScan() throws Exception {
     p.add(c0, c0, T3);
     region.put(p);
 
-    Delete d = new Delete(T1, ts+1, null);
-    region.delete(d, null, true);
+    Delete d = new Delete(T1, ts+1);
+    region.delete(d, true);
 
-    d = new Delete(T1, ts+2, null);
+    d = new Delete(T1, ts+2);
     d.deleteColumn(c0, c0, ts+2);
-    region.delete(d, null, true);
+    region.delete(d, true);
 
-    d = new Delete(T1, ts+3, null);
+    d = new Delete(T1, ts+3);
     d.deleteColumns(c0, c0, ts+3);
-    region.delete(d, null, true);
+    region.delete(d, true);
 
     Scan s = new Scan();
     s.setRaw(true);
@@ -293,12 +293,54 @@ public void testRawScan() throws Exception {
     InternalScanner scan = region.getScanner(s);
     List<KeyValue> kvs = new ArrayList<KeyValue>();
     scan.next(kvs);
+    assertEquals(8, kvs.size());
     assertTrue(kvs.get(0).isDeleteFamily());
     assertEquals(kvs.get(1).getValue(), T3);
     assertTrue(kvs.get(2).isDelete());
     assertTrue(kvs.get(3).isDeleteType());
     assertEquals(kvs.get(4).getValue(), T2);
     assertEquals(kvs.get(5).getValue(), T1);
+    // we have 3 CFs, so there are two more delete markers
+    assertTrue(kvs.get(6).isDeleteFamily());
+    assertTrue(kvs.get(7).isDeleteFamily());
+
+    // verify that raw scans honor the passed timerange
+    s = new Scan();
+    s.setRaw(true);
+    s.setMaxVersions();
+    s.setTimeRange(0, 1);
+    scan = region.getScanner(s);
+    kvs = new ArrayList<KeyValue>();
+    scan.next(kvs);
+    // nothing in this interval, not even delete markers
+    assertTrue(kvs.isEmpty());
+
+    // filter new delete markers
+    s = new Scan();
+    s.setRaw(true);
+    s.setMaxVersions();
+    s.setTimeRange(0, ts+2);
+    scan = region.getScanner(s);
+    kvs = new ArrayList<KeyValue>();
+    scan.next(kvs);
+    assertEquals(4, kvs.size());
+    assertTrue(kvs.get(0).isDeleteFamily());
+    assertEquals(kvs.get(1).getValue(), T1);
+    // we have 3 CFs
+    assertTrue(kvs.get(2).isDeleteFamily());
+    assertTrue(kvs.get(3).isDeleteFamily());
+
+    // filter old delete markers
+    s = new Scan();
+    s.setRaw(true);
+    s.setMaxVersions();
+    s.setTimeRange(ts+3, ts+5);
+    scan = region.getScanner(s);
+    kvs = new ArrayList<KeyValue>();
+    scan.next(kvs);
+    assertEquals(2, kvs.size());
+    assertEquals(kvs.get(0).getValue(), T3);
+    assertTrue(kvs.get(1).isDelete());
 
     region.close();
     region.getLog().closeAndDelete();

From c349c0e1db215415d06efdde458783a9e77b3428 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 1 Jul 2013 16:03:03 +0000
Subject: [PATCH 1031/1540] HBASE-8794 DependentColumnFilter.toString() throws
 NullPointerException (Stefan Seelmann)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1498537 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/filter/DependentColumnFilter.java   |  2 +-
 .../filter/TestDependentColumnFilter.java     | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
index c50c821a2d54..2d7dd3e02ec2 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
@@ -245,6 +245,6 @@ public String toString() {
         Bytes.toStringBinary(this.columnQualifier),
         this.dropDependentColumn,
         this.compareOp.name(),
-        Bytes.toStringBinary(this.comparator.getValue()));
+        this.comparator != null ? Bytes.toStringBinary(this.comparator.getValue()) : "null");
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
index 8e13a5b34ef0..09f83da29f60 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
@@ -243,6 +243,31 @@ public void testFilterDropping() throws Exception {
       assertEquals("check cell retention", 2, accepted.size());
   }
 
+  /**
+   * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
+   */
+  public void testToStringWithNullComparator() {
+    // Test constructor that implicitly sets a null comparator
+    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
+    assertNotNull(filter.toString());
+    assertTrue("check string contains 'null' as compatator is null",
+      filter.toString().contains("null"));
+
+    // Test constructor with explicit null comparator
+    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL, null);
+    assertNotNull(filter.toString());
+    assertTrue("check string contains 'null' as compatator is null",
+      filter.toString().contains("null"));
+  }
+
+  public void testToStringWithNonNullComparator() {
+    Filter filter =
+        new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL,
+            new BinaryComparator(MATCH_VAL));
+    assertNotNull(filter.toString());
+    assertTrue("check string contains comparator value", filter.toString().contains("match"));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From e1e64a0fa54da2f39d0cb2faa9a70b9228cd1b69 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Tue, 2 Jul 2013 12:17:25 +0000
Subject: [PATCH 1032/1540] HBASE-8774 Add BatchSize and Filter to Thrift2
 (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1498900 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift2/ThriftUtilities.java |  15 ++
 .../hbase/thrift2/generated/TDelete.java      |   4 +-
 .../hadoop/hbase/thrift2/generated/TGet.java  | 127 +++++++++-
 .../thrift2/generated/THBaseService.java      |  32 ++-
 .../hbase/thrift2/generated/TIncrement.java   |   4 +-
 .../hadoop/hbase/thrift2/generated/TPut.java  |   4 +-
 .../hbase/thrift2/generated/TResult.java      |   4 +-
 .../hadoop/hbase/thrift2/generated/TScan.java | 222 +++++++++++++++++-
 .../apache/hadoop/hbase/thrift2/hbase.thrift  |   5 +-
 .../TestThriftHBaseServiceHandler.java        |   3 +
 10 files changed, 383 insertions(+), 37 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
index b63ed74e933e..254de6990d48 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.thrift2.generated.*;
 
 import java.io.IOException;
@@ -71,6 +72,11 @@ public static Get getFromThrift(TGet in) throws IOException {
       }
     }
 
+    if (in.isSetFilterString()) {
+      ParseFilter parseFilter = new ParseFilter();
+      out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
+    }
+
     return out;
   }
 
@@ -322,6 +328,15 @@ public static Scan scanFromThrift(TScan in) throws IOException {
       out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
     }
 
+    if (in.isSetBatchSize()) {
+      out.setBatch(in.getBatchSize());
+    }
+
+    if (in.isSetFilterString()) {
+      ParseFilter parseFilter = new ParseFilter();
+      out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
+    }
+
     return out;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
index a732d7a0fa8c..f2eef0d19939 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
@@ -709,7 +709,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) thr
                 struct.columns = new ArrayList<TColumn>(_list24.size);
                 for (int _i25 = 0; _i25 < _list24.size; ++_i25)
                 {
-                  TColumn _elem26; // optional
+                  TColumn _elem26; // required
                   _elem26 = new TColumn();
                   _elem26.read(iprot);
                   struct.columns.add(_elem26);
@@ -860,7 +860,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro
           struct.columns = new ArrayList<TColumn>(_list29.size);
           for (int _i30 = 0; _i30 < _list29.size; ++_i30)
           {
-            TColumn _elem31; // optional
+            TColumn _elem31; // required
             _elem31 = new TColumn();
             _elem31.read(iprot);
             struct.columns.add(_elem31);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
index 14e6d51ca62c..f3b496951a36 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
@@ -39,8 +39,6 @@
  * 
  * If you specify a time range and a timestamp the range is ignored.
  * Timestamps on TColumns are ignored.
- * 
- * TODO: Filter, Locks
  */
 public class TGet implements org.apache.thrift.TBase<TGet, TGet._Fields>, java.io.Serializable, Cloneable {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGet");
@@ -50,6 +48,7 @@ public class TGet implements org.apache.thrift.TBase<TGet, TGet._Fields>, java.i
   private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
   private static final org.apache.thrift.protocol.TField TIME_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeRange", org.apache.thrift.protocol.TType.STRUCT, (short)4);
   private static final org.apache.thrift.protocol.TField MAX_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxVersions", org.apache.thrift.protocol.TType.I32, (short)5);
+  private static final org.apache.thrift.protocol.TField FILTER_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("filterString", org.apache.thrift.protocol.TType.STRING, (short)6);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -62,6 +61,7 @@ public class TGet implements org.apache.thrift.TBase<TGet, TGet._Fields>, java.i
   public long timestamp; // optional
   public TTimeRange timeRange; // optional
   public int maxVersions; // optional
+  public ByteBuffer filterString; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -69,7 +69,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     COLUMNS((short)2, "columns"),
     TIMESTAMP((short)3, "timestamp"),
     TIME_RANGE((short)4, "timeRange"),
-    MAX_VERSIONS((short)5, "maxVersions");
+    MAX_VERSIONS((short)5, "maxVersions"),
+    FILTER_STRING((short)6, "filterString");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -94,6 +95,8 @@ public static _Fields findByThriftId(int fieldId) {
           return TIME_RANGE;
         case 5: // MAX_VERSIONS
           return MAX_VERSIONS;
+        case 6: // FILTER_STRING
+          return FILTER_STRING;
         default:
           return null;
       }
@@ -137,7 +140,7 @@ public String getFieldName() {
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __MAXVERSIONS_ISSET_ID = 1;
   private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.TIME_RANGE,_Fields.MAX_VERSIONS};
+  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.TIME_RANGE,_Fields.MAX_VERSIONS,_Fields.FILTER_STRING};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -152,6 +155,8 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTimeRange.class)));
     tmpMap.put(_Fields.MAX_VERSIONS, new org.apache.thrift.meta_data.FieldMetaData("maxVersions", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+    tmpMap.put(_Fields.FILTER_STRING, new org.apache.thrift.meta_data.FieldMetaData("filterString", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGet.class, metaDataMap);
   }
@@ -188,6 +193,10 @@ public TGet(TGet other) {
       this.timeRange = new TTimeRange(other.timeRange);
     }
     this.maxVersions = other.maxVersions;
+    if (other.isSetFilterString()) {
+      this.filterString = org.apache.thrift.TBaseHelper.copyBinary(other.filterString);
+;
+    }
   }
 
   public TGet deepCopy() {
@@ -203,6 +212,7 @@ public void clear() {
     this.timeRange = null;
     setMaxVersionsIsSet(false);
     this.maxVersions = 0;
+    this.filterString = null;
   }
 
   public byte[] getRow() {
@@ -348,6 +358,40 @@ public void setMaxVersionsIsSet(boolean value) {
     __isset_bit_vector.set(__MAXVERSIONS_ISSET_ID, value);
   }
 
+  public byte[] getFilterString() {
+    setFilterString(org.apache.thrift.TBaseHelper.rightSize(filterString));
+    return filterString == null ? null : filterString.array();
+  }
+
+  public ByteBuffer bufferForFilterString() {
+    return filterString;
+  }
+
+  public TGet setFilterString(byte[] filterString) {
+    setFilterString(filterString == null ? (ByteBuffer)null : ByteBuffer.wrap(filterString));
+    return this;
+  }
+
+  public TGet setFilterString(ByteBuffer filterString) {
+    this.filterString = filterString;
+    return this;
+  }
+
+  public void unsetFilterString() {
+    this.filterString = null;
+  }
+
+  /** Returns true if field filterString is set (has been assigned a value) and false otherwise */
+  public boolean isSetFilterString() {
+    return this.filterString != null;
+  }
+
+  public void setFilterStringIsSet(boolean value) {
+    if (!value) {
+      this.filterString = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -390,6 +434,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case FILTER_STRING:
+      if (value == null) {
+        unsetFilterString();
+      } else {
+        setFilterString((ByteBuffer)value);
+      }
+      break;
+
     }
   }
 
@@ -410,6 +462,9 @@ public Object getFieldValue(_Fields field) {
     case MAX_VERSIONS:
       return Integer.valueOf(getMaxVersions());
 
+    case FILTER_STRING:
+      return getFilterString();
+
     }
     throw new IllegalStateException();
   }
@@ -431,6 +486,8 @@ public boolean isSet(_Fields field) {
       return isSetTimeRange();
     case MAX_VERSIONS:
       return isSetMaxVersions();
+    case FILTER_STRING:
+      return isSetFilterString();
     }
     throw new IllegalStateException();
   }
@@ -493,6 +550,15 @@ public boolean equals(TGet that) {
         return false;
     }
 
+    boolean this_present_filterString = true && this.isSetFilterString();
+    boolean that_present_filterString = true && that.isSetFilterString();
+    if (this_present_filterString || that_present_filterString) {
+      if (!(this_present_filterString && that_present_filterString))
+        return false;
+      if (!this.filterString.equals(that.filterString))
+        return false;
+    }
+
     return true;
   }
 
@@ -559,6 +625,16 @@ public int compareTo(TGet other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetFilterString()).compareTo(typedOther.isSetFilterString());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetFilterString()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.filterString, typedOther.filterString);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -618,6 +694,16 @@ public String toString() {
       sb.append(this.maxVersions);
       first = false;
     }
+    if (isSetFilterString()) {
+      if (!first) sb.append(", ");
+      sb.append("filterString:");
+      if (this.filterString == null) {
+        sb.append("null");
+      } else {
+        org.apache.thrift.TBaseHelper.toString(this.filterString, sb);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -680,7 +766,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TGet struct) throws
                 struct.columns = new ArrayList<TColumn>(_list8.size);
                 for (int _i9 = 0; _i9 < _list8.size; ++_i9)
                 {
-                  TColumn _elem10; // optional
+                  TColumn _elem10; // required
                   _elem10 = new TColumn();
                   _elem10.read(iprot);
                   struct.columns.add(_elem10);
@@ -717,6 +803,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TGet struct) throws
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 6: // FILTER_STRING
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.filterString = iprot.readBinary();
+              struct.setFilterStringIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -768,6 +862,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TGet struct) throw
         oprot.writeI32(struct.maxVersions);
         oprot.writeFieldEnd();
       }
+      if (struct.filterString != null) {
+        if (struct.isSetFilterString()) {
+          oprot.writeFieldBegin(FILTER_STRING_FIELD_DESC);
+          oprot.writeBinary(struct.filterString);
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -799,7 +900,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
       if (struct.isSetMaxVersions()) {
         optionals.set(3);
       }
-      oprot.writeBitSet(optionals, 4);
+      if (struct.isSetFilterString()) {
+        optionals.set(4);
+      }
+      oprot.writeBitSet(optionals, 5);
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
@@ -818,6 +922,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
       if (struct.isSetMaxVersions()) {
         oprot.writeI32(struct.maxVersions);
       }
+      if (struct.isSetFilterString()) {
+        oprot.writeBinary(struct.filterString);
+      }
     }
 
     @Override
@@ -825,14 +932,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
       TTupleProtocol iprot = (TTupleProtocol) prot;
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
+      BitSet incoming = iprot.readBitSet(5);
       if (incoming.get(0)) {
         {
           org.apache.thrift.protocol.TList _list13 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
           struct.columns = new ArrayList<TColumn>(_list13.size);
           for (int _i14 = 0; _i14 < _list13.size; ++_i14)
           {
-            TColumn _elem15; // optional
+            TColumn _elem15; // required
             _elem15 = new TColumn();
             _elem15.read(iprot);
             struct.columns.add(_elem15);
@@ -853,6 +960,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
         struct.maxVersions = iprot.readI32();
         struct.setMaxVersionsIsSet(true);
       }
+      if (incoming.get(4)) {
+        struct.filterString = iprot.readBinary();
+        struct.setFilterStringIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
index 56289987eae5..45b6e64cfafb 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
@@ -2207,6 +2207,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -3704,7 +3706,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_args st
                   struct.gets = new ArrayList<TGet>(_list48.size);
                   for (int _i49 = 0; _i49 < _list48.size; ++_i49)
                   {
-                    TGet _elem50; // optional
+                    TGet _elem50; // required
                     _elem50 = new TGet();
                     _elem50.read(iprot);
                     struct.gets.add(_elem50);
@@ -3785,7 +3787,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_args str
           struct.gets = new ArrayList<TGet>(_list53.size);
           for (int _i54 = 0; _i54 < _list53.size; ++_i54)
           {
-            TGet _elem55; // optional
+            TGet _elem55; // required
             _elem55 = new TGet();
             _elem55.read(iprot);
             struct.gets.add(_elem55);
@@ -4186,7 +4188,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_result
                   struct.success = new ArrayList<TResult>(_list56.size);
                   for (int _i57 = 0; _i57 < _list56.size; ++_i57)
                   {
-                    TResult _elem58; // optional
+                    TResult _elem58; // required
                     _elem58 = new TResult();
                     _elem58.read(iprot);
                     struct.success.add(_elem58);
@@ -4288,7 +4290,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_result s
             struct.success = new ArrayList<TResult>(_list61.size);
             for (int _i62 = 0; _i62 < _list61.size; ++_i62)
             {
-              TResult _elem63; // optional
+              TResult _elem63; // required
               _elem63 = new TResult();
               _elem63.read(iprot);
               struct.success.add(_elem63);
@@ -6447,6 +6449,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -7000,7 +7004,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, putMultiple_args st
                   struct.puts = new ArrayList<TPut>(_list64.size);
                   for (int _i65 = 0; _i65 < _list64.size; ++_i65)
                   {
-                    TPut _elem66; // optional
+                    TPut _elem66; // required
                     _elem66 = new TPut();
                     _elem66.read(iprot);
                     struct.puts.add(_elem66);
@@ -7081,7 +7085,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, putMultiple_args str
           struct.puts = new ArrayList<TPut>(_list69.size);
           for (int _i70 = 0; _i70 < _list69.size; ++_i70)
           {
-            TPut _elem71; // optional
+            TPut _elem71; // required
             _elem71 = new TPut();
             _elem71.read(iprot);
             struct.puts.add(_elem71);
@@ -8720,7 +8724,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_args
                   struct.deletes = new ArrayList<TDelete>(_list72.size);
                   for (int _i73 = 0; _i73 < _list72.size; ++_i73)
                   {
-                    TDelete _elem74; // optional
+                    TDelete _elem74; // required
                     _elem74 = new TDelete();
                     _elem74.read(iprot);
                     struct.deletes.add(_elem74);
@@ -8801,7 +8805,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
           struct.deletes = new ArrayList<TDelete>(_list77.size);
           for (int _i78 = 0; _i78 < _list77.size; ++_i78)
           {
-            TDelete _elem79; // optional
+            TDelete _elem79; // required
             _elem79 = new TDelete();
             _elem79.read(iprot);
             struct.deletes.add(_elem79);
@@ -9202,7 +9206,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_resu
                   struct.success = new ArrayList<TDelete>(_list80.size);
                   for (int _i81 = 0; _i81 < _list80.size; ++_i81)
                   {
-                    TDelete _elem82; // optional
+                    TDelete _elem82; // required
                     _elem82 = new TDelete();
                     _elem82.read(iprot);
                     struct.success.add(_elem82);
@@ -9304,7 +9308,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resul
             struct.success = new ArrayList<TDelete>(_list85.size);
             for (int _i86 = 0; _i86 < _list85.size; ++_i86)
             {
-              TDelete _elem87; // optional
+              TDelete _elem87; // required
               _elem87 = new TDelete();
               _elem87.read(iprot);
               struct.success.add(_elem87);
@@ -10629,6 +10633,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -12496,6 +12502,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -13558,7 +13566,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerRows_resu
                   struct.success = new ArrayList<TResult>(_list88.size);
                   for (int _i89 = 0; _i89 < _list88.size; ++_i89)
                   {
-                    TResult _elem90; // optional
+                    TResult _elem90; // required
                     _elem90 = new TResult();
                     _elem90.read(iprot);
                     struct.success.add(_elem90);
@@ -13680,7 +13688,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resul
             struct.success = new ArrayList<TResult>(_list93.size);
             for (int _i94 = 0; _i94 < _list93.size; ++_i94)
             {
-              TResult _elem95; // optional
+              TResult _elem95; // required
               _elem95 = new TResult();
               _elem95.read(iprot);
               struct.success.add(_elem95);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java
index 6b23b99c87de..c618f42ad4b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java
@@ -527,7 +527,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TIncrement struct)
                 struct.columns = new ArrayList<TColumnIncrement>(_list32.size);
                 for (int _i33 = 0; _i33 < _list32.size; ++_i33)
                 {
-                  TColumnIncrement _elem34; // optional
+                  TColumnIncrement _elem34; // required
                   _elem34 = new TColumnIncrement();
                   _elem34.read(iprot);
                   struct.columns.add(_elem34);
@@ -629,7 +629,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) t
         struct.columns = new ArrayList<TColumnIncrement>(_list37.size);
         for (int _i38 = 0; _i38 < _list37.size; ++_i38)
         {
-          TColumnIncrement _elem39; // optional
+          TColumnIncrement _elem39; // required
           _elem39 = new TColumnIncrement();
           _elem39.read(iprot);
           struct.columns.add(_elem39);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
index a93482440a41..33bd4458c18c 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
@@ -604,7 +604,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TPut struct) throws
                 struct.columnValues = new ArrayList<TColumnValue>(_list16.size);
                 for (int _i17 = 0; _i17 < _list16.size; ++_i17)
                 {
-                  TColumnValue _elem18; // optional
+                  TColumnValue _elem18; // required
                   _elem18 = new TColumnValue();
                   _elem18.read(iprot);
                   struct.columnValues.add(_elem18);
@@ -725,7 +725,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
         struct.columnValues = new ArrayList<TColumnValue>(_list21.size);
         for (int _i22 = 0; _i22 < _list21.size; ++_i22)
         {
-          TColumnValue _elem23; // optional
+          TColumnValue _elem23; // required
           _elem23 = new TColumnValue();
           _elem23.read(iprot);
           struct.columnValues.add(_elem23);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java
index f0fe2a0c583f..e91f1371d20b 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java
@@ -442,7 +442,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TResult struct) thr
                 struct.columnValues = new ArrayList<TColumnValue>(_list0.size);
                 for (int _i1 = 0; _i1 < _list0.size; ++_i1)
                 {
-                  TColumnValue _elem2; // optional
+                  TColumnValue _elem2; // required
                   _elem2 = new TColumnValue();
                   _elem2.read(iprot);
                   struct.columnValues.add(_elem2);
@@ -530,7 +530,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TResult struct) thro
         struct.columnValues = new ArrayList<TColumnValue>(_list5.size);
         for (int _i6 = 0; _i6 < _list5.size; ++_i6)
         {
-          TColumnValue _elem7; // optional
+          TColumnValue _elem7; // required
           _elem7 = new TColumnValue();
           _elem7.read(iprot);
           struct.columnValues.add(_elem7);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
index cbcdf910c88f..1935f01dcce4 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
@@ -40,6 +40,8 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
   private static final org.apache.thrift.protocol.TField CACHING_FIELD_DESC = new org.apache.thrift.protocol.TField("caching", org.apache.thrift.protocol.TType.I32, (short)4);
   private static final org.apache.thrift.protocol.TField MAX_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxVersions", org.apache.thrift.protocol.TType.I32, (short)5);
   private static final org.apache.thrift.protocol.TField TIME_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeRange", org.apache.thrift.protocol.TType.STRUCT, (short)6);
+  private static final org.apache.thrift.protocol.TField FILTER_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("filterString", org.apache.thrift.protocol.TType.STRING, (short)7);
+  private static final org.apache.thrift.protocol.TField BATCH_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("batchSize", org.apache.thrift.protocol.TType.I32, (short)8);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -53,6 +55,8 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
   public int caching; // optional
   public int maxVersions; // optional
   public TTimeRange timeRange; // optional
+  public ByteBuffer filterString; // optional
+  public int batchSize; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -61,7 +65,9 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     COLUMNS((short)3, "columns"),
     CACHING((short)4, "caching"),
     MAX_VERSIONS((short)5, "maxVersions"),
-    TIME_RANGE((short)6, "timeRange");
+    TIME_RANGE((short)6, "timeRange"),
+    FILTER_STRING((short)7, "filterString"),
+    BATCH_SIZE((short)8, "batchSize");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -88,6 +94,10 @@ public static _Fields findByThriftId(int fieldId) {
           return MAX_VERSIONS;
         case 6: // TIME_RANGE
           return TIME_RANGE;
+        case 7: // FILTER_STRING
+          return FILTER_STRING;
+        case 8: // BATCH_SIZE
+          return BATCH_SIZE;
         default:
           return null;
       }
@@ -130,8 +140,9 @@ public String getFieldName() {
   // isset id assignments
   private static final int __CACHING_ISSET_ID = 0;
   private static final int __MAXVERSIONS_ISSET_ID = 1;
-  private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE};
+  private static final int __BATCHSIZE_ISSET_ID = 2;
+  private BitSet __isset_bit_vector = new BitSet(3);
+  private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -148,6 +159,10 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
     tmpMap.put(_Fields.TIME_RANGE, new org.apache.thrift.meta_data.FieldMetaData("timeRange", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTimeRange.class)));
+    tmpMap.put(_Fields.FILTER_STRING, new org.apache.thrift.meta_data.FieldMetaData("filterString", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
+    tmpMap.put(_Fields.BATCH_SIZE, new org.apache.thrift.meta_data.FieldMetaData("batchSize", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap);
   }
@@ -183,6 +198,11 @@ public TScan(TScan other) {
     if (other.isSetTimeRange()) {
       this.timeRange = new TTimeRange(other.timeRange);
     }
+    if (other.isSetFilterString()) {
+      this.filterString = org.apache.thrift.TBaseHelper.copyBinary(other.filterString);
+;
+    }
+    this.batchSize = other.batchSize;
   }
 
   public TScan deepCopy() {
@@ -199,6 +219,9 @@ public void clear() {
     this.maxVersions = 1;
 
     this.timeRange = null;
+    this.filterString = null;
+    setBatchSizeIsSet(false);
+    this.batchSize = 0;
   }
 
   public byte[] getStartRow() {
@@ -378,6 +401,63 @@ public void setTimeRangeIsSet(boolean value) {
     }
   }
 
+  public byte[] getFilterString() {
+    setFilterString(org.apache.thrift.TBaseHelper.rightSize(filterString));
+    return filterString == null ? null : filterString.array();
+  }
+
+  public ByteBuffer bufferForFilterString() {
+    return filterString;
+  }
+
+  public TScan setFilterString(byte[] filterString) {
+    setFilterString(filterString == null ? (ByteBuffer)null : ByteBuffer.wrap(filterString));
+    return this;
+  }
+
+  public TScan setFilterString(ByteBuffer filterString) {
+    this.filterString = filterString;
+    return this;
+  }
+
+  public void unsetFilterString() {
+    this.filterString = null;
+  }
+
+  /** Returns true if field filterString is set (has been assigned a value) and false otherwise */
+  public boolean isSetFilterString() {
+    return this.filterString != null;
+  }
+
+  public void setFilterStringIsSet(boolean value) {
+    if (!value) {
+      this.filterString = null;
+    }
+  }
+
+  public int getBatchSize() {
+    return this.batchSize;
+  }
+
+  public TScan setBatchSize(int batchSize) {
+    this.batchSize = batchSize;
+    setBatchSizeIsSet(true);
+    return this;
+  }
+
+  public void unsetBatchSize() {
+    __isset_bit_vector.clear(__BATCHSIZE_ISSET_ID);
+  }
+
+  /** Returns true if field batchSize is set (has been assigned a value) and false otherwise */
+  public boolean isSetBatchSize() {
+    return __isset_bit_vector.get(__BATCHSIZE_ISSET_ID);
+  }
+
+  public void setBatchSizeIsSet(boolean value) {
+    __isset_bit_vector.set(__BATCHSIZE_ISSET_ID, value);
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case START_ROW:
@@ -428,6 +508,22 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case FILTER_STRING:
+      if (value == null) {
+        unsetFilterString();
+      } else {
+        setFilterString((ByteBuffer)value);
+      }
+      break;
+
+    case BATCH_SIZE:
+      if (value == null) {
+        unsetBatchSize();
+      } else {
+        setBatchSize((Integer)value);
+      }
+      break;
+
     }
   }
 
@@ -451,6 +547,12 @@ public Object getFieldValue(_Fields field) {
     case TIME_RANGE:
       return getTimeRange();
 
+    case FILTER_STRING:
+      return getFilterString();
+
+    case BATCH_SIZE:
+      return Integer.valueOf(getBatchSize());
+
     }
     throw new IllegalStateException();
   }
@@ -474,6 +576,10 @@ public boolean isSet(_Fields field) {
       return isSetMaxVersions();
     case TIME_RANGE:
       return isSetTimeRange();
+    case FILTER_STRING:
+      return isSetFilterString();
+    case BATCH_SIZE:
+      return isSetBatchSize();
     }
     throw new IllegalStateException();
   }
@@ -545,6 +651,24 @@ public boolean equals(TScan that) {
         return false;
     }
 
+    boolean this_present_filterString = true && this.isSetFilterString();
+    boolean that_present_filterString = true && that.isSetFilterString();
+    if (this_present_filterString || that_present_filterString) {
+      if (!(this_present_filterString && that_present_filterString))
+        return false;
+      if (!this.filterString.equals(that.filterString))
+        return false;
+    }
+
+    boolean this_present_batchSize = true && this.isSetBatchSize();
+    boolean that_present_batchSize = true && that.isSetBatchSize();
+    if (this_present_batchSize || that_present_batchSize) {
+      if (!(this_present_batchSize && that_present_batchSize))
+        return false;
+      if (this.batchSize != that.batchSize)
+        return false;
+    }
+
     return true;
   }
 
@@ -621,6 +745,26 @@ public int compareTo(TScan other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetFilterString()).compareTo(typedOther.isSetFilterString());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetFilterString()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.filterString, typedOther.filterString);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetBatchSize()).compareTo(typedOther.isSetBatchSize());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetBatchSize()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.batchSize, typedOther.batchSize);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -692,6 +836,22 @@ public String toString() {
       }
       first = false;
     }
+    if (isSetFilterString()) {
+      if (!first) sb.append(", ");
+      sb.append("filterString:");
+      if (this.filterString == null) {
+        sb.append("null");
+      } else {
+        org.apache.thrift.TBaseHelper.toString(this.filterString, sb);
+      }
+      first = false;
+    }
+    if (isSetBatchSize()) {
+      if (!first) sb.append(", ");
+      sb.append("batchSize:");
+      sb.append(this.batchSize);
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -759,7 +919,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
                 struct.columns = new ArrayList<TColumn>(_list40.size);
                 for (int _i41 = 0; _i41 < _list40.size; ++_i41)
                 {
-                  TColumn _elem42; // optional
+                  TColumn _elem42; // required
                   _elem42 = new TColumn();
                   _elem42.read(iprot);
                   struct.columns.add(_elem42);
@@ -796,6 +956,22 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 7: // FILTER_STRING
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.filterString = iprot.readBinary();
+              struct.setFilterStringIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 8: // BATCH_SIZE
+            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+              struct.batchSize = iprot.readI32();
+              struct.setBatchSizeIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -856,6 +1032,18 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) thro
           oprot.writeFieldEnd();
         }
       }
+      if (struct.filterString != null) {
+        if (struct.isSetFilterString()) {
+          oprot.writeFieldBegin(FILTER_STRING_FIELD_DESC);
+          oprot.writeBinary(struct.filterString);
+          oprot.writeFieldEnd();
+        }
+      }
+      if (struct.isSetBatchSize()) {
+        oprot.writeFieldBegin(BATCH_SIZE_FIELD_DESC);
+        oprot.writeI32(struct.batchSize);
+        oprot.writeFieldEnd();
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -892,7 +1080,13 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetTimeRange()) {
         optionals.set(5);
       }
-      oprot.writeBitSet(optionals, 6);
+      if (struct.isSetFilterString()) {
+        optionals.set(6);
+      }
+      if (struct.isSetBatchSize()) {
+        optionals.set(7);
+      }
+      oprot.writeBitSet(optionals, 8);
       if (struct.isSetStartRow()) {
         oprot.writeBinary(struct.startRow);
       }
@@ -917,12 +1111,18 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetTimeRange()) {
         struct.timeRange.write(oprot);
       }
+      if (struct.isSetFilterString()) {
+        oprot.writeBinary(struct.filterString);
+      }
+      if (struct.isSetBatchSize()) {
+        oprot.writeI32(struct.batchSize);
+      }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(6);
+      BitSet incoming = iprot.readBitSet(8);
       if (incoming.get(0)) {
         struct.startRow = iprot.readBinary();
         struct.setStartRowIsSet(true);
@@ -937,7 +1137,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
           struct.columns = new ArrayList<TColumn>(_list45.size);
           for (int _i46 = 0; _i46 < _list45.size; ++_i46)
           {
-            TColumn _elem47; // optional
+            TColumn _elem47; // required
             _elem47 = new TColumn();
             _elem47.read(iprot);
             struct.columns.add(_elem47);
@@ -958,6 +1158,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
         struct.timeRange.read(iprot);
         struct.setTimeRangeIsSet(true);
       }
+      if (incoming.get(6)) {
+        struct.filterString = iprot.readBinary();
+        struct.setFilterStringIsSet(true);
+      }
+      if (incoming.get(7)) {
+        struct.batchSize = iprot.readI32();
+        struct.setBatchSizeIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
index 5bb0f51cbd3c..5be6d12285ac 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
@@ -89,8 +89,6 @@ enum TDeleteType {
  *
  * If you specify a time range and a timestamp the range is ignored.
  * Timestamps on TColumns are ignored.
- *
- * TODO: Filter, Locks
  */
 struct TGet {
   1: required binary row,
@@ -100,6 +98,7 @@ struct TGet {
   4: optional TTimeRange timeRange,
 
   5: optional i32 maxVersions,
+  6: optional binary filterString
 }
 
 /**
@@ -174,6 +173,8 @@ struct TScan {
   4: optional i32 caching,
   5: optional i32 maxVersions=1,
   6: optional TTimeRange timeRange,
+  7: optional binary filterString,
+  8: optional i32 batchSize
 }
 
 //
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index b848ac732926..781e065ffe87 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -504,6 +504,8 @@ public void testScan() throws Exception {
     columns.add(column);
     scan.setColumns(columns);
     scan.setStartRow("testScan".getBytes());
+    // only get the key part
+    scan.setFilterString(ByteBuffer.wrap(("KeyOnlyFilter()").getBytes()));
 
     TColumnValue columnValue = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
         ByteBuffer.wrap(valueAname));
@@ -519,6 +521,7 @@ public void testScan() throws Exception {
     assertEquals(10, results.size());
     for (int i = 0; i < 10; i++) {
       assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
+      assertArrayEquals(("").getBytes(), results.get(i).getColumnValues().get(0).getValue());
     }
 
     results = handler.getScannerRows(scanId, 10);

From 586e2b099c0c9e18dc95409f25a381e49c9bde18 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 2 Jul 2013 14:21:35 +0000
Subject: [PATCH 1033/1540] HBASE-8767 Backport hbase-8001 and hbase-8012,
 avoid lazy seek (original patches by Raymond Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1498953 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/KeyValueHeap.java      |  3 +-
 .../hbase/regionserver/StoreFileScanner.java  | 14 +++++-----
 .../hbase/regionserver/TestStoreFile.java     | 28 +++++++++++++++++++
 3 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 8613a872fcb5..c15bf0f5ad42 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -314,7 +314,8 @@ private boolean generalizedSeek(boolean isLazy, KeyValue seekKey,
       }
 
       boolean seekResult;
-      if (isLazy) {
+      if (isLazy && heap.size() > 0) {
+        // If there is only one scanner left, we don't do lazy seek.
         seekResult = scanner.requestSeek(seekKey, forward, useBloom);
       } else {
         seekResult = NonLazyKeyValueScanner.doRealSeek(
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index d69fc9fdf5dc..83144fff3fb8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -52,11 +52,6 @@ public class StoreFileScanner implements KeyValueScanner {
 
   private boolean enforceMVCC = false;
 
-  //The variable, realSeekDone, may cheat on store file scanner for the
-  // multi-column bloom-filter optimization.
-  // So this flag shows whether this storeFileScanner could do a reseek.
-  private boolean isReseekable = false;
-
   private static final AtomicLong seekCount = new AtomicLong();
 
   private ScanQueryMatcher matcher;
@@ -147,7 +142,6 @@ public boolean seek(KeyValue key) throws IOException {
           return false;
         }
 
-        this.isReseekable = true;
         cur = hfs.getKeyValue();
 
         return skipKVsNewerThanReadpoint();
@@ -241,6 +235,12 @@ static boolean reseekAtOrAfter(HFileScanner s, KeyValue k)
     //This function is similar to seekAtOrAfter function
     int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength());
     if (result <= 0) {
+      // If up to now scanner is not seeked yet, this means passed KV is smaller
+      // than first KV in file, and it is the first time we seek on this file.
+      // So we also need to work from the start of file.
+      if (!s.isSeeked()) {
+        return  s.seekTo();
+      }
       return true;
     } else {
       // passed KV is larger than current KV in file, if there is a next
@@ -345,7 +345,7 @@ public void enforceSeek() throws IOException {
     if (realSeekDone)
       return;
 
-    if (delayedReseek && this.isReseekable) {
+    if (delayedReseek) {
       reseek(delayedSeekKV);
     } else {
       seek(delayedSeekKV);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index a8f2f84ab4d4..7c4860337947 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
@@ -608,6 +609,33 @@ public void testDeleteFamilyBloomFilter() throws Exception {
         + ", expected no more than " + maxFalsePos, falsePos <= maxFalsePos);
   }
 
+  /**
+   * Test for HBASE-8012
+   */
+  public void testReseek() throws Exception {
+    // write the file
+    Path f = new Path(ROOT_DIR, getName());
+
+    // Make a store file and write data to it.
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+         this.fs, 8 * 1024)
+            .withFilePath(f)
+            .build();
+
+    writeStoreFile(writer);
+    writer.close();
+
+    StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf, DataBlockEncoding.NONE);
+
+    // Now do reseek with empty KV to position to the beginning of the file
+
+    KeyValue k = KeyValue.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY);
+    StoreFileScanner s = reader.getStoreFileScanner(false, false);
+    s.reseek(k);
+
+    assertNotNull("Intial reseek should position at the beginning of the file", s.peek());
+  }
+
   public void testBloomTypes() throws Exception {
     float err = (float) 0.01;
     FileSystem fs = FileSystem.getLocal(conf);

From 4bfeeec41d1c8fdcdf98e4565d9d99a45277a8a7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 3 Jul 2013 06:13:08 +0000
Subject: [PATCH 1034/1540] HBASE-8814 Possible NPE in split if a region has
 empty store files

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1499221 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |  4 ++
 .../TestSplitTransactionOnCluster.java        | 64 +++++++++++++++++++
 2 files changed, 68 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index c73935eb47a6..759672f9206b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -953,6 +953,8 @@ static Path split(final FileSystem fs,
       //check if smaller than first key
       KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
       byte[] firstKey = f.createReader().getFirstKey();
+      // If firstKey is null means storefile is empty.
+      if (firstKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           firstKey, 0, firstKey.length) < 0) {
@@ -963,6 +965,8 @@ static Path split(final FileSystem fs,
       //check if larger than last key.
       KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
       byte[] lastKey = f.createReader().getLastKey();      
+      // If lastKey is null means storefile is empty.
+      if (lastKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           lastKey, 0, lastKey.length) > 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 3c1c9448ec25..25bce9bb70d9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -43,6 +43,9 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -779,6 +782,67 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
     }
   }
   
+  @Test(timeout = 180000)
+  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
+    Configuration conf = TESTING_UTIL.getConfiguration();
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
+    String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
+    HTableDescriptor htd = new HTableDescriptor(userTableName);
+    HColumnDescriptor hcd = new HColumnDescriptor("col");
+    htd.addFamily(hcd);
+    admin.createTable(htd);
+    ZKAssign.blockUntilNoRIT(zkw);
+    HTable table = new HTable(conf, userTableName);
+    try {
+      for (int i = 0; i <= 5; i++) {
+        String row = "row" + i;
+        Put p = new Put(row.getBytes());
+        String val = "Val" + i;
+        p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
+        table.put(p);
+        admin.flush(userTableName);
+        Delete d = new Delete(row.getBytes());
+        // Do a normal delete
+        table.delete(d);
+        admin.flush(userTableName);
+      }
+      admin.majorCompact(userTableName);
+      List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
+          .getMaster().getAssignmentManager()
+          .getRegionsOfTable(userTableName.getBytes());
+      HRegionInfo hRegionInfo = regionsOfTable.get(0);
+      Put p = new Put("row6".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      p = new Put("row7".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      p = new Put("row8".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      admin.flush(userTableName);
+      admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
+      regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+          .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
+
+      while (regionsOfTable.size() != 2) {
+        Thread.sleep(2000);
+        regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+            .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
+      }
+      assertEquals(2, regionsOfTable.size());
+      Scan s = new Scan();
+      ResultScanner scanner = table.getScanner(s);
+      int mainTableCount = 0;
+      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
+        mainTableCount++;
+      }
+      assertEquals(3, mainTableCount);
+    } finally {
+      table.close();
+    }
+  }
+
   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
       InterruptedException {
     Put p = new Put(Bytes.toBytes("row1"));

From 01a0d4206c0920e4232843c1546861f2297b1efe Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 3 Jul 2013 18:05:40 +0000
Subject: [PATCH 1035/1540] HBASE-8848 Backport hbase-6979 "recovered.edits
 file should not break distributed log splitting"

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1499499 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/HLogSplitter.java  | 18 +++++++++++--
 .../hbase/regionserver/wal/TestHLogSplit.java | 27 +++++++++++++++++++
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 18708f19d3c1..404caa4808fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -661,8 +661,8 @@ static Path getRegionSplitEditsPath(final FileSystem fs,
   throws IOException {
     Path tableDir = HTableDescriptor.getTableDir(rootDir, logEntry.getKey()
         .getTablename());
-    Path regiondir = HRegion.getRegionDir(tableDir,
-        Bytes.toString(logEntry.getKey().getEncodedRegionName()));
+    String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());
+    Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);
     Path dir = HLog.getRegionDirRecoveredEditsDir(regiondir);
 
     if (!fs.exists(regiondir)) {
@@ -671,6 +671,20 @@ static Path getRegionSplitEditsPath(final FileSystem fs,
           " already split so it's safe to discard those edits.");
       return null;
     }
+    if (fs.exists(dir) && fs.isFile(dir)) {
+      Path tmp = new Path("/tmp");
+      if (!fs.exists(tmp)) {
+        fs.mkdirs(tmp);
+      }
+      tmp = new Path(tmp,
+        HLog.RECOVERED_EDITS_DIR + "_" + encodedRegionName);
+      LOG.warn("Found existing old file: " + dir + ". It could be some "
+        + "leftover of an old installation. It should be a folder instead. "
+        + "So moving it to " + tmp);
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, dir, tmp)) {
+        LOG.warn("Failed to sideline old file " + dir);
+      }
+    }
     if (isCreate && !fs.exists(dir) && 
         !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) {
       LOG.warn("mkdir failed on " + dir);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index d3e3a53d6622..1d49cc8d8d9d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -200,6 +200,33 @@ public void testSplitFailsIfNewHLogGetsCreatedAfterSplitStarted()
     }
   }
 
+  /**
+   * Test old recovered edits file doesn't break HLogSplitter.
+   * This is useful in upgrading old instances.
+   */
+  @Test
+  public void testOldRecoveredEditsFileSidelined() throws IOException {
+    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    byte [] encoded = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes();
+    Path tdir = new Path(hbaseDir, Bytes.toString(HConstants.META_TABLE_NAME));
+    Path regiondir = new Path(tdir,
+        HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
+    fs.mkdirs(regiondir);
+    long now = System.currentTimeMillis();
+    HLog.Entry entry =
+        new HLog.Entry(new HLogKey(encoded,
+            HConstants.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
+      new WALEdit());
+    Path parent = HLog.getRegionDirRecoveredEditsDir(regiondir);
+    assertEquals(parent.getName(), HLog.RECOVERED_EDITS_DIR);
+    fs.createNewFile(parent); // create a recovered.edits file
+
+    Path p = HLogSplitter.getRegionSplitEditsPath(fs, entry, hbaseDir, true);
+    String parentOfParent = p.getParent().getParent().getName();
+    assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
+    HLog.createWriter(fs, p, conf).close();
+  }
+
   @Test
   public void testSplitPreservesEdits() throws IOException{
     final String REGION = "region__1";

From 3af55dbfb0e485f2ed0185155220bced2cdbce33 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 3 Jul 2013 21:28:21 +0000
Subject: [PATCH 1036/1540] HBASE-8814 Possible NPE in split if a region has
 empty store files; REVERT.  FAILING TEST

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1499564 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |  4 --
 .../TestSplitTransactionOnCluster.java        | 64 -------------------
 2 files changed, 68 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 759672f9206b..c73935eb47a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -953,8 +953,6 @@ static Path split(final FileSystem fs,
       //check if smaller than first key
       KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
       byte[] firstKey = f.createReader().getFirstKey();
-      // If firstKey is null means storefile is empty.
-      if (firstKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           firstKey, 0, firstKey.length) < 0) {
@@ -965,8 +963,6 @@ static Path split(final FileSystem fs,
       //check if larger than last key.
       KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
       byte[] lastKey = f.createReader().getLastKey();      
-      // If lastKey is null means storefile is empty.
-      if (lastKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           lastKey, 0, lastKey.length) > 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 25bce9bb70d9..3c1c9448ec25 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -43,9 +43,6 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -782,67 +779,6 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
     }
   }
   
-  @Test(timeout = 180000)
-  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
-    Configuration conf = TESTING_UTIL.getConfiguration();
-    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
-    String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
-    HTableDescriptor htd = new HTableDescriptor(userTableName);
-    HColumnDescriptor hcd = new HColumnDescriptor("col");
-    htd.addFamily(hcd);
-    admin.createTable(htd);
-    ZKAssign.blockUntilNoRIT(zkw);
-    HTable table = new HTable(conf, userTableName);
-    try {
-      for (int i = 0; i <= 5; i++) {
-        String row = "row" + i;
-        Put p = new Put(row.getBytes());
-        String val = "Val" + i;
-        p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
-        table.put(p);
-        admin.flush(userTableName);
-        Delete d = new Delete(row.getBytes());
-        // Do a normal delete
-        table.delete(d);
-        admin.flush(userTableName);
-      }
-      admin.majorCompact(userTableName);
-      List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
-          .getMaster().getAssignmentManager()
-          .getRegionsOfTable(userTableName.getBytes());
-      HRegionInfo hRegionInfo = regionsOfTable.get(0);
-      Put p = new Put("row6".getBytes());
-      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
-      table.put(p);
-      p = new Put("row7".getBytes());
-      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
-      table.put(p);
-      p = new Put("row8".getBytes());
-      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
-      table.put(p);
-      admin.flush(userTableName);
-      admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
-      regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
-          .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
-
-      while (regionsOfTable.size() != 2) {
-        Thread.sleep(2000);
-        regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
-            .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
-      }
-      assertEquals(2, regionsOfTable.size());
-      Scan s = new Scan();
-      ResultScanner scanner = table.getScanner(s);
-      int mainTableCount = 0;
-      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
-        mainTableCount++;
-      }
-      assertEquals(3, mainTableCount);
-    } finally {
-      table.close();
-    }
-  }
-
   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
       InterruptedException {
     Put p = new Put(Bytes.toBytes("row1"));

From 857ea21d154967605810906a971cb4e85916000b Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Thu, 4 Jul 2013 16:30:25 +0000
Subject: [PATCH 1037/1540] HBASE-8814 Possible NPE in split if a region has
 empty store files; REAPPLY WITH ADDENDUM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1499800 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |  4 ++
 .../TestSplitTransactionOnCluster.java        | 69 +++++++++++++++++++
 2 files changed, 73 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index c73935eb47a6..759672f9206b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -953,6 +953,8 @@ static Path split(final FileSystem fs,
       //check if smaller than first key
       KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
       byte[] firstKey = f.createReader().getFirstKey();
+      // If firstKey is null means storefile is empty.
+      if (firstKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           firstKey, 0, firstKey.length) < 0) {
@@ -963,6 +965,8 @@ static Path split(final FileSystem fs,
       //check if larger than last key.
       KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
       byte[] lastKey = f.createReader().getLastKey();      
+      // If lastKey is null means storefile is empty.
+      if (lastKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           lastKey, 0, lastKey.length) > 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 3c1c9448ec25..10c87ca3131f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -43,6 +43,9 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -746,6 +749,7 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
   
   @Test(timeout = 60000)
   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
     final byte[] tableName = 
         Bytes.toBytes("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
     HRegionServer regionServer = null;
@@ -774,11 +778,76 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
           Bytes.toString(tableName));
       assertEquals("The specified table should present.", true, tableExists);
     } finally {
+      if (regions != null) {
+        String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
+            .getEncodedName());
+        ZKUtil.deleteNodeFailSilent(zkw, node);
+      }
       cluster.getMaster().setCatalogJanitorEnabled(true);
       admin.close();
     }
   }
   
+  @Test(timeout = 180000)
+  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
+    Configuration conf = TESTING_UTIL.getConfiguration();
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
+    String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
+    HTableDescriptor htd = new HTableDescriptor(userTableName);
+    HColumnDescriptor hcd = new HColumnDescriptor("col");
+    htd.addFamily(hcd);
+    admin.createTable(htd);
+    HTable table = new HTable(conf, userTableName);
+    try {
+      for (int i = 0; i <= 5; i++) {
+        String row = "row" + i;
+        Put p = new Put(row.getBytes());
+        String val = "Val" + i;
+        p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
+        table.put(p);
+        admin.flush(userTableName);
+        Delete d = new Delete(row.getBytes());
+        // Do a normal delete
+        table.delete(d);
+        admin.flush(userTableName);
+      }
+      admin.majorCompact(userTableName);
+      List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
+          .getMaster().getAssignmentManager()
+          .getRegionsOfTable(userTableName.getBytes());
+      HRegionInfo hRegionInfo = regionsOfTable.get(0);
+      Put p = new Put("row6".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      p = new Put("row7".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      p = new Put("row8".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      admin.flush(userTableName);
+      admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
+      regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+          .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
+
+      while (regionsOfTable.size() != 2) {
+        Thread.sleep(2000);
+        regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+            .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
+      }
+      assertEquals(2, regionsOfTable.size());
+      Scan s = new Scan();
+      ResultScanner scanner = table.getScanner(s);
+      int mainTableCount = 0;
+      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
+        mainTableCount++;
+      }
+      assertEquals(3, mainTableCount);
+    } finally {
+      table.close();
+    }
+  }
+
   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
       InterruptedException {
     Put p = new Put(Bytes.toBytes("row1"));

From 280b3ef409720d881b66ab30e3dd4a4f59392443 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Fri, 5 Jul 2013 15:18:24 +0000
Subject: [PATCH 1038/1540] HBASE-8876 Addendum to HBASE-8774 Add BatchSize and
 Filter to Thrift2 - Add BatchSize Test

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500035 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestThriftHBaseServiceHandler.java        | 359 ++++++++++++------
 1 file changed, 237 insertions(+), 122 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 781e065ffe87..1e732b6e47a0 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -19,13 +19,6 @@
  */
 package org.apache.hadoop.hbase.thrift2;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -68,8 +61,12 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import static org.junit.Assert.*;
+import static java.nio.ByteBuffer.wrap;
+
 /**
- * Unit testing for ThriftServer.HBaseHandler, a part of the org.apache.hadoop.hbase.thrift2 package.
+ * Unit testing for ThriftServer.HBaseHandler, a part of the org.apache.hadoop.hbase.thrift2 
+ * package.
  */
 @Category(MediumTests.class)
 public class TestThriftHBaseServiceHandler {
@@ -91,7 +88,8 @@ public class TestThriftHBaseServiceHandler {
           .setMaxVersions(2)
   };
 
-  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA, List<TColumnValue> columnValuesB) {
+  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
+      List<TColumnValue> columnValuesB) {
     assertEquals(columnValuesA.size(), columnValuesB.size());
     Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
       @Override
@@ -141,17 +139,15 @@ private ThriftHBaseServiceHandler createHandler() {
   public void testExists() throws TIOError, TException {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testExists".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     assertFalse(handler.exists(table, get));
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer
-        .wrap(valueAname)));
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer
-        .wrap(valueBname)));
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
+    TPut put = new TPut(wrap(rowName), columnValues);
     put.setColumnValues(columnValues);
 
     handler.put(table, put);
@@ -163,20 +159,18 @@ public void testExists() throws TIOError, TException {
   public void testPutGet() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testPutGet".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer
-        .wrap(valueAname)));
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer
-        .wrap(valueBname)));
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
+    TPut put = new TPut(wrap(rowName), columnValues);
 
     put.setColumnValues(columnValues);
 
     handler.put(table, put);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
 
     TResult result = handler.get(table, get);
     assertArrayEquals(rowName, result.getRow());
@@ -187,24 +181,22 @@ public void testPutGet() throws Exception {
   @Test
   public void testPutGetMultiple() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
     byte[] rowName1 = "testPutGetMultiple1".getBytes();
     byte[] rowName2 = "testPutGetMultiple2".getBytes();
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer
-        .wrap(valueAname)));
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer
-        .wrap(valueBname)));
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
     List<TPut> puts = new ArrayList<TPut>();
-    puts.add(new TPut(ByteBuffer.wrap(rowName1), columnValues));
-    puts.add(new TPut(ByteBuffer.wrap(rowName2), columnValues));
+    puts.add(new TPut(wrap(rowName1), columnValues));
+    puts.add(new TPut(wrap(rowName2), columnValues));
 
     handler.putMultiple(table, puts);
 
     List<TGet> gets = new ArrayList<TGet>();
-    gets.add(new TGet(ByteBuffer.wrap(rowName1)));
-    gets.add(new TGet(ByteBuffer.wrap(rowName2)));
+    gets.add(new TGet(wrap(rowName1)));
+    gets.add(new TGet(wrap(rowName2)));
 
     List<TResult> results = handler.getMultiple(table, gets);
     assertEquals(2, results.size());
@@ -219,62 +211,60 @@ public void testPutGetMultiple() throws Exception {
   @Test
   public void testDeleteMultiple() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
     byte[] rowName1 = "testDeleteMultiple1".getBytes();
     byte[] rowName2 = "testDeleteMultiple2".getBytes();
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer
-        .wrap(valueAname)));
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer
-        .wrap(valueBname)));
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
     List<TPut> puts = new ArrayList<TPut>();
-    puts.add(new TPut(ByteBuffer.wrap(rowName1), columnValues));
-    puts.add(new TPut(ByteBuffer.wrap(rowName2), columnValues));
+    puts.add(new TPut(wrap(rowName1), columnValues));
+    puts.add(new TPut(wrap(rowName2), columnValues));
 
     handler.putMultiple(table, puts);
 
     List<TDelete> deletes = new ArrayList<TDelete>();
-    deletes.add(new TDelete(ByteBuffer.wrap(rowName1)));
-    deletes.add(new TDelete(ByteBuffer.wrap(rowName2)));
+    deletes.add(new TDelete(wrap(rowName1)));
+    deletes.add(new TDelete(wrap(rowName2)));
 
     List<TDelete> deleteResults = handler.deleteMultiple(table, deletes);
     // 0 means they were all successfully applies
     assertEquals(0, deleteResults.size());
 
-    assertFalse(handler.exists(table, new TGet(ByteBuffer.wrap(rowName1))));
-    assertFalse(handler.exists(table, new TGet(ByteBuffer.wrap(rowName2))));
+    assertFalse(handler.exists(table, new TGet(wrap(rowName1))));
+    assertFalse(handler.exists(table, new TGet(wrap(rowName2))));
   }
 
   @Test
   public void testDelete() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testDelete".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
-        ByteBuffer.wrap(valueAname));
-    TColumnValue columnValueB = new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname),
-        ByteBuffer.wrap(valueBname));
+    TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
+    TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
+      wrap(valueBname));
     columnValues.add(columnValueA);
     columnValues.add(columnValueB);
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    TPut put = new TPut(wrap(rowName), columnValues);
 
     put.setColumnValues(columnValues);
 
     handler.put(table, put);
 
-    TDelete delete = new TDelete(ByteBuffer.wrap(rowName));
+    TDelete delete = new TDelete(wrap(rowName));
     List<TColumn> deleteColumns = new ArrayList<TColumn>();
-    TColumn deleteColumn = new TColumn(ByteBuffer.wrap(familyAname));
+    TColumn deleteColumn = new TColumn(wrap(familyAname));
     deleteColumn.setQualifier(qualifierAname);
     deleteColumns.add(deleteColumn);
     delete.setColumns(deleteColumns);
 
     handler.deleteSingle(table, delete);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     TResult result = handler.get(table, get);
     assertArrayEquals(rowName, result.getRow());
     List<TColumnValue> returnedColumnValues = result.getColumnValues();
@@ -287,14 +277,14 @@ public void testDelete() throws Exception {
   public void testDeleteAllTimestamps() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testDeleteAllTimestamps".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
-        ByteBuffer.wrap(valueAname));
+    TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
     columnValueA.setTimestamp(System.currentTimeMillis() - 10);
     columnValues.add(columnValueA);
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    TPut put = new TPut(wrap(rowName), columnValues);
 
     put.setColumnValues(columnValues);
 
@@ -302,14 +292,14 @@ public void testDeleteAllTimestamps() throws Exception {
     columnValueA.setTimestamp(System.currentTimeMillis());
     handler.put(table, put);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     get.setMaxVersions(2);
     TResult result = handler.get(table, get);
     assertEquals(2, result.getColumnValuesSize());
 
-    TDelete delete = new TDelete(ByteBuffer.wrap(rowName));
+    TDelete delete = new TDelete(wrap(rowName));
     List<TColumn> deleteColumns = new ArrayList<TColumn>();
-    TColumn deleteColumn = new TColumn(ByteBuffer.wrap(familyAname));
+    TColumn deleteColumn = new TColumn(wrap(familyAname));
     deleteColumn.setQualifier(qualifierAname);
     deleteColumns.add(deleteColumn);
     delete.setColumns(deleteColumns);
@@ -317,7 +307,7 @@ public void testDeleteAllTimestamps() throws Exception {
 
     handler.deleteSingle(table, delete);
 
-    get = new TGet(ByteBuffer.wrap(rowName));
+    get = new TGet(wrap(rowName));
     result = handler.get(table, get);
     assertNull(result.getRow());
     assertEquals(0, result.getColumnValuesSize());
@@ -327,17 +317,17 @@ public void testDeleteAllTimestamps() throws Exception {
   public void testDeleteSingleTimestamp() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testDeleteSingleTimestamp".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     long timestamp1 = System.currentTimeMillis() - 10;
     long timestamp2 = System.currentTimeMillis();
     
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
-        ByteBuffer.wrap(valueAname));
+    TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
     columnValueA.setTimestamp(timestamp1);
     columnValues.add(columnValueA);
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    TPut put = new TPut(wrap(rowName), columnValues);
 
     put.setColumnValues(columnValues);
 
@@ -345,14 +335,14 @@ public void testDeleteSingleTimestamp() throws Exception {
     columnValueA.setTimestamp(timestamp2);
     handler.put(table, put);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     get.setMaxVersions(2);
     TResult result = handler.get(table, get);
     assertEquals(2, result.getColumnValuesSize());
 
-    TDelete delete = new TDelete(ByteBuffer.wrap(rowName));
+    TDelete delete = new TDelete(wrap(rowName));
     List<TColumn> deleteColumns = new ArrayList<TColumn>();
-    TColumn deleteColumn = new TColumn(ByteBuffer.wrap(familyAname));
+    TColumn deleteColumn = new TColumn(wrap(familyAname));
     deleteColumn.setQualifier(qualifierAname);
     deleteColumns.add(deleteColumn);
     delete.setColumns(deleteColumns);
@@ -360,7 +350,7 @@ public void testDeleteSingleTimestamp() throws Exception {
 
     handler.deleteSingle(table, delete);
 
-    get = new TGet(ByteBuffer.wrap(rowName));
+    get = new TGet(wrap(rowName));
     result = handler.get(table, get);
     assertArrayEquals(rowName, result.getRow());
     assertEquals(1, result.getColumnValuesSize());
@@ -372,21 +362,21 @@ public void testDeleteSingleTimestamp() throws Exception {
   public void testIncrement() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testIncrement".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer
-        .wrap(Bytes.toBytes(1L))));
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(Bytes.toBytes(1L))));
+    TPut put = new TPut(wrap(rowName), columnValues);
     put.setColumnValues(columnValues);
     handler.put(table, put);
 
     List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
-    incrementColumns.add(new TColumnIncrement(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname)));
-    TIncrement increment = new TIncrement(ByteBuffer.wrap(rowName), incrementColumns);
+    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
+    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
     handler.increment(table, increment);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     TResult result = handler.get(table, get);
 
     assertArrayEquals(rowName, result.getRow());
@@ -396,8 +386,8 @@ public void testIncrement() throws Exception {
   }
 
   /**
-   * check that checkAndPut fails if the cell does not exist, then put in the cell, then check that the checkAndPut
-   * succeeds.
+   * check that checkAndPut fails if the cell does not exist, then put in the cell, then check
+   * that the checkAndPut succeeds.
    * 
    * @throws Exception
    */
@@ -405,33 +395,33 @@ public void testIncrement() throws Exception {
   public void testCheckAndPut() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testCheckAndPut".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     List<TColumnValue> columnValuesA = new ArrayList<TColumnValue>();
-    TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
-        ByteBuffer.wrap(valueAname));
+    TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
     columnValuesA.add(columnValueA);
-    TPut putA = new TPut(ByteBuffer.wrap(rowName), columnValuesA);
+    TPut putA = new TPut(wrap(rowName), columnValuesA);
     putA.setColumnValues(columnValuesA);
 
     List<TColumnValue> columnValuesB = new ArrayList<TColumnValue>();
-    TColumnValue columnValueB = new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname),
-        ByteBuffer.wrap(valueBname));
+    TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
+      wrap(valueBname));
     columnValuesB.add(columnValueB);
-    TPut putB = new TPut(ByteBuffer.wrap(rowName), columnValuesB);
+    TPut putB = new TPut(wrap(rowName), columnValuesB);
     putB.setColumnValues(columnValuesB);
 
-    assertFalse(handler.checkAndPut(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname),
-        ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), putB));
+    assertFalse(handler.checkAndPut(table, wrap(rowName), wrap(familyAname),
+      wrap(qualifierAname), wrap(valueAname), putB));
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     TResult result = handler.get(table, get);
     assertEquals(0, result.getColumnValuesSize());
 
     handler.put(table, putA);
 
-    assertTrue(handler.checkAndPut(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname),
-        ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), putB));
+    assertTrue(handler.checkAndPut(table, wrap(rowName), wrap(familyAname),
+      wrap(qualifierAname), wrap(valueAname), putB));
 
     result = handler.get(table, get);
     assertArrayEquals(rowName, result.getRow());
@@ -443,8 +433,8 @@ public void testCheckAndPut() throws Exception {
   }
 
   /**
-   * check that checkAndDelete fails if the cell does not exist, then put in the cell, then check that the
-   * checkAndDelete succeeds.
+   * check that checkAndDelete fails if the cell does not exist, then put in the cell, then
+   * check that the checkAndDelete succeeds.
    * 
    * @throws Exception
    */
@@ -452,39 +442,39 @@ public void testCheckAndPut() throws Exception {
   public void testCheckAndDelete() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
     byte[] rowName = "testCheckAndDelete".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
     List<TColumnValue> columnValuesA = new ArrayList<TColumnValue>();
-    TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
-        ByteBuffer.wrap(valueAname));
+    TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
     columnValuesA.add(columnValueA);
-    TPut putA = new TPut(ByteBuffer.wrap(rowName), columnValuesA);
+    TPut putA = new TPut(wrap(rowName), columnValuesA);
     putA.setColumnValues(columnValuesA);
 
     List<TColumnValue> columnValuesB = new ArrayList<TColumnValue>();
-    TColumnValue columnValueB = new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname),
-        ByteBuffer.wrap(valueBname));
+    TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
+      wrap(valueBname));
     columnValuesB.add(columnValueB);
-    TPut putB = new TPut(ByteBuffer.wrap(rowName), columnValuesB);
+    TPut putB = new TPut(wrap(rowName), columnValuesB);
     putB.setColumnValues(columnValuesB);
 
     // put putB so that we know whether the row has been deleted or not
     handler.put(table, putB);
 
-    TDelete delete = new TDelete(ByteBuffer.wrap(rowName));
+    TDelete delete = new TDelete(wrap(rowName));
 
-    assertFalse(handler.checkAndDelete(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname),
-        ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), delete));
+    assertFalse(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
+      wrap(qualifierAname), wrap(valueAname), delete));
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     TResult result = handler.get(table, get);
     assertArrayEquals(rowName, result.getRow());
     assertTColumnValuesEqual(columnValuesB, result.getColumnValues());
 
     handler.put(table, putA);
 
-    assertTrue(handler.checkAndDelete(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname),
-        ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), delete));
+    assertTrue(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
+      wrap(qualifierAname), wrap(valueAname), delete));
 
     result = handler.get(table, get);
     assertFalse(result.isSetRow());
@@ -494,8 +484,19 @@ public void testCheckAndDelete() throws Exception {
   @Test
   public void testScan() throws Exception {
     ThriftHBaseServiceHandler handler = createHandler();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
+
+    // insert data
+    TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
+    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
+    columnValues.add(columnValue);
+    for (int i = 0; i < 10; i++) {
+      TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
+      handler.put(table, put);
+    }
 
+    // create scan instance
     TScan scan = new TScan();
     List<TColumn> columns = new ArrayList<TColumn>();
     TColumn column = new TColumn();
@@ -504,31 +505,75 @@ public void testScan() throws Exception {
     columns.add(column);
     scan.setColumns(columns);
     scan.setStartRow("testScan".getBytes());
-    // only get the key part
-    scan.setFilterString(ByteBuffer.wrap(("KeyOnlyFilter()").getBytes()));
+    scan.setStopRow("testScan\uffff".getBytes());
+
+    // get scanner and rows
+    int scanId = handler.openScanner(table, scan);
+    List<TResult> results = handler.getScannerRows(scanId, 10);
+    assertEquals(10, results.size());
+    for (int i = 0; i < 10; i++) {
+      // check if the rows are returned and in order
+      assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
+    }
+
+    // check that we are at the end of the scan
+    results = handler.getScannerRows(scanId, 10);
+    assertEquals(0, results.size());
 
-    TColumnValue columnValue = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname),
-        ByteBuffer.wrap(valueAname));
+    // close scanner and check that it was indeed closed
+    handler.closeScanner(scanId);
+    try {
+      handler.getScannerRows(scanId, 10);
+      fail("Scanner id should be invalid");
+    } catch (TIllegalArgument e) {
+    }
+  }
+
+  @Test
+  public void testScanWithFilter() throws Exception {
+    ThriftHBaseServiceHandler handler = createHandler();
+    ByteBuffer table = wrap(tableAname);
+
+    // insert data
+    TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+      wrap(valueAname));
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
     columnValues.add(columnValue);
     for (int i = 0; i < 10; i++) {
-      TPut put = new TPut(ByteBuffer.wrap(("testScan" + i).getBytes()), columnValues);
+      TPut put = new TPut(wrap(("testScanWithFilter" + i).getBytes()), columnValues);
       handler.put(table, put);
     }
 
+    // create scan instance with filter
+    TScan scan = new TScan();
+    List<TColumn> columns = new ArrayList<TColumn>();
+    TColumn column = new TColumn();
+    column.setFamily(familyAname);
+    column.setQualifier(qualifierAname);
+    columns.add(column);
+    scan.setColumns(columns);
+    scan.setStartRow("testScanWithFilter".getBytes());
+    scan.setStopRow("testScanWithFilter\uffff".getBytes());
+    // only get the key part
+    scan.setFilterString(wrap(("KeyOnlyFilter()").getBytes()));
+
+    // get scanner and rows
     int scanId = handler.openScanner(table, scan);
     List<TResult> results = handler.getScannerRows(scanId, 10);
     assertEquals(10, results.size());
     for (int i = 0; i < 10; i++) {
-      assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
-      assertArrayEquals(("").getBytes(), results.get(i).getColumnValues().get(0).getValue());
+      // check if the rows are returned and in order
+      assertArrayEquals(("testScanWithFilter" + i).getBytes(), results.get(i).getRow());
+      // check that the value is indeed stripped by the filter
+      assertEquals(0, results.get(i).getColumnValues().get(0).getValue().length);
     }
 
+    // check that we are at the end of the scan
     results = handler.getScannerRows(scanId, 10);
     assertEquals(0, results.size());
 
+    // close scanner and check that it was indeed closed
     handler.closeScanner(scanId);
-
     try {
       handler.getScannerRows(scanId, 10);
       fail("Scanner id should be invalid");
@@ -536,6 +581,80 @@ public void testScan() throws Exception {
     }
   }
 
+  /**
+   * Padding numbers to make comparison of sort order easier in a for loop
+   *
+   * @param n  The number to pad.
+   * @param pad  The length to pad up to.
+   * @return The padded number as a string.
+   */
+  private String pad(int n, byte pad) {
+    String res = Integer.toString(n);
+    while (res.length() < pad) res = "0" + res;
+    return res;
+  }
+
+  @Test
+  public void testScanWithBatchSize() throws Exception {
+    ThriftHBaseServiceHandler handler = createHandler();
+    ByteBuffer table = wrap(tableAname);
+
+    // insert data
+    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
+    for (int i = 0; i < 100; i++) {
+      String colNum = pad(i, (byte) 3);
+      TColumnValue columnValue = new TColumnValue(wrap(familyAname),
+        wrap(("col" + colNum).getBytes()), wrap(("val" + colNum).getBytes()));
+      columnValues.add(columnValue);
+    }
+    TPut put = new TPut(wrap(("testScanWithBatchSize").getBytes()), columnValues);
+    handler.put(table, put);
+
+    // create scan instance
+    TScan scan = new TScan();
+    List<TColumn> columns = new ArrayList<TColumn>();
+    TColumn column = new TColumn();
+    column.setFamily(familyAname);
+    columns.add(column);
+    scan.setColumns(columns);
+    scan.setStartRow("testScanWithBatchSize".getBytes());
+    scan.setStopRow("testScanWithBatchSize\uffff".getBytes());
+    // set batch size to 10 columns per call
+    scan.setBatchSize(10);
+
+    // get scanner
+    int scanId = handler.openScanner(table, scan);
+    List<TResult> results = null;
+    for (int i = 0; i < 10; i++) {
+      System.out.println("batch: " + i);
+      // get batch for single row (10x10 is what we expect)
+      results = handler.getScannerRows(scanId, 1);
+      assertEquals(1, results.size());
+      // check length of batch
+      List<TColumnValue> cols = results.get(0).getColumnValues();
+      assertEquals(10, cols.size());
+      // check if the columns are returned and in order
+      for (int y = 0; y < 10; y++) {
+        int colNum = y + (10 * i);
+        String colNumPad = pad(colNum, (byte) 3);
+        System.out.println("col" + colNumPad + ": " + new String(cols.get(y).getQualifier()));
+        assertArrayEquals(("col" + colNumPad).getBytes(), cols.get(y).getQualifier());
+      }
+    }
+
+    // check that we are at the end of the scan
+    results = handler.getScannerRows(scanId, 1);
+    assertEquals(0, results.size());
+
+    // close scanner and check that it was indeed closed
+    handler.closeScanner(scanId);
+    try {
+      handler.getScannerRows(scanId, 1);
+      fail("Scanner id should be invalid");
+    } catch (TIllegalArgument e) {
+    }
+  }
+
   @Test
   public void testMetrics() throws Exception {
     Configuration conf = UTIL.getConfiguration();
@@ -543,19 +662,15 @@ public void testMetrics() throws Exception {
     THBaseService.Iface handler =
         ThriftHBaseServiceHandler.newInstance(conf, metrics);
     byte[] rowName = "testMetrics".getBytes();
-    ByteBuffer table = ByteBuffer.wrap(tableAname);
+    ByteBuffer table = wrap(tableAname);
 
-    TGet get = new TGet(ByteBuffer.wrap(rowName));
+    TGet get = new TGet(wrap(rowName));
     assertFalse(handler.exists(table, get));
 
     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname),
-                                      ByteBuffer.wrap(qualifierAname),
-                                      ByteBuffer.wrap(valueAname)));
-    columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname),
-                                      ByteBuffer.wrap(qualifierBname),
-                                      ByteBuffer.wrap(valueBname)));
-    TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues);
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),  wrap(valueBname)));
+    TPut put = new TPut(wrap(rowName), columnValues);
     put.setColumnValues(columnValues);
 
     handler.put(table, put);

From 811770ec11a9fba9357bb65c19e1f72fe1fcbb58 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 5 Jul 2013 17:51:40 +0000
Subject: [PATCH 1039/1540] HBASE-8814 Possible NPE in split if a region has
 empty store files; REVERT AGAIN (2nd TIME)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500084 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |  4 --
 .../TestSplitTransactionOnCluster.java        | 69 -------------------
 2 files changed, 73 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 759672f9206b..c73935eb47a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -953,8 +953,6 @@ static Path split(final FileSystem fs,
       //check if smaller than first key
       KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
       byte[] firstKey = f.createReader().getFirstKey();
-      // If firstKey is null means storefile is empty.
-      if (firstKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           firstKey, 0, firstKey.length) < 0) {
@@ -965,8 +963,6 @@ static Path split(final FileSystem fs,
       //check if larger than last key.
       KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
       byte[] lastKey = f.createReader().getLastKey();      
-      // If lastKey is null means storefile is empty.
-      if (lastKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           lastKey, 0, lastKey.length) > 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 10c87ca3131f..3c1c9448ec25 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -43,9 +43,6 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -749,7 +746,6 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
   
   @Test(timeout = 60000)
   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
-    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
     final byte[] tableName = 
         Bytes.toBytes("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
     HRegionServer regionServer = null;
@@ -778,76 +774,11 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
           Bytes.toString(tableName));
       assertEquals("The specified table should present.", true, tableExists);
     } finally {
-      if (regions != null) {
-        String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
-            .getEncodedName());
-        ZKUtil.deleteNodeFailSilent(zkw, node);
-      }
       cluster.getMaster().setCatalogJanitorEnabled(true);
       admin.close();
     }
   }
   
-  @Test(timeout = 180000)
-  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
-    Configuration conf = TESTING_UTIL.getConfiguration();
-    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
-    String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
-    HTableDescriptor htd = new HTableDescriptor(userTableName);
-    HColumnDescriptor hcd = new HColumnDescriptor("col");
-    htd.addFamily(hcd);
-    admin.createTable(htd);
-    HTable table = new HTable(conf, userTableName);
-    try {
-      for (int i = 0; i <= 5; i++) {
-        String row = "row" + i;
-        Put p = new Put(row.getBytes());
-        String val = "Val" + i;
-        p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
-        table.put(p);
-        admin.flush(userTableName);
-        Delete d = new Delete(row.getBytes());
-        // Do a normal delete
-        table.delete(d);
-        admin.flush(userTableName);
-      }
-      admin.majorCompact(userTableName);
-      List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
-          .getMaster().getAssignmentManager()
-          .getRegionsOfTable(userTableName.getBytes());
-      HRegionInfo hRegionInfo = regionsOfTable.get(0);
-      Put p = new Put("row6".getBytes());
-      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
-      table.put(p);
-      p = new Put("row7".getBytes());
-      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
-      table.put(p);
-      p = new Put("row8".getBytes());
-      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
-      table.put(p);
-      admin.flush(userTableName);
-      admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
-      regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
-          .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
-
-      while (regionsOfTable.size() != 2) {
-        Thread.sleep(2000);
-        regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
-            .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
-      }
-      assertEquals(2, regionsOfTable.size());
-      Scan s = new Scan();
-      ResultScanner scanner = table.getScanner(s);
-      int mainTableCount = 0;
-      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
-        mainTableCount++;
-      }
-      assertEquals(3, mainTableCount);
-    } finally {
-      table.close();
-    }
-  }
-
   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
       InterruptedException {
     Put p = new Put(Bytes.toBytes("row1"));

From 423de6443e06a9eab64edeb7fb6b4d41afa778c4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 5 Jul 2013 21:07:45 +0000
Subject: [PATCH 1040/1540] HBASE-8814 Possible NPE in split if a region has
 empty store files; RESTORE FOR SECOND TIME

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500136 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFile.java  |  4 ++
 .../TestSplitTransactionOnCluster.java        | 69 +++++++++++++++++++
 2 files changed, 73 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index c73935eb47a6..759672f9206b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -953,6 +953,8 @@ static Path split(final FileSystem fs,
       //check if smaller than first key
       KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
       byte[] firstKey = f.createReader().getFirstKey();
+      // If firstKey is null means storefile is empty.
+      if (firstKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           firstKey, 0, firstKey.length) < 0) {
@@ -963,6 +965,8 @@ static Path split(final FileSystem fs,
       //check if larger than last key.
       KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
       byte[] lastKey = f.createReader().getLastKey();      
+      // If lastKey is null means storefile is empty.
+      if (lastKey == null) return null;
       if (f.getReader().getComparator().compare(splitKey.getBuffer(), 
           splitKey.getKeyOffset(), splitKey.getKeyLength(), 
           lastKey, 0, lastKey.length) > 0) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 3c1c9448ec25..10c87ca3131f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -43,6 +43,9 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -746,6 +749,7 @@ void createSplitDir(FileSystem fs, Path splitdir) throws IOException {
   
   @Test(timeout = 60000)
   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
     final byte[] tableName = 
         Bytes.toBytes("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
     HRegionServer regionServer = null;
@@ -774,11 +778,76 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
           Bytes.toString(tableName));
       assertEquals("The specified table should present.", true, tableExists);
     } finally {
+      if (regions != null) {
+        String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
+            .getEncodedName());
+        ZKUtil.deleteNodeFailSilent(zkw, node);
+      }
       cluster.getMaster().setCatalogJanitorEnabled(true);
       admin.close();
     }
   }
   
+  @Test(timeout = 180000)
+  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
+    Configuration conf = TESTING_UTIL.getConfiguration();
+    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
+    String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
+    HTableDescriptor htd = new HTableDescriptor(userTableName);
+    HColumnDescriptor hcd = new HColumnDescriptor("col");
+    htd.addFamily(hcd);
+    admin.createTable(htd);
+    HTable table = new HTable(conf, userTableName);
+    try {
+      for (int i = 0; i <= 5; i++) {
+        String row = "row" + i;
+        Put p = new Put(row.getBytes());
+        String val = "Val" + i;
+        p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
+        table.put(p);
+        admin.flush(userTableName);
+        Delete d = new Delete(row.getBytes());
+        // Do a normal delete
+        table.delete(d);
+        admin.flush(userTableName);
+      }
+      admin.majorCompact(userTableName);
+      List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
+          .getMaster().getAssignmentManager()
+          .getRegionsOfTable(userTableName.getBytes());
+      HRegionInfo hRegionInfo = regionsOfTable.get(0);
+      Put p = new Put("row6".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      p = new Put("row7".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      p = new Put("row8".getBytes());
+      p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
+      table.put(p);
+      admin.flush(userTableName);
+      admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
+      regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+          .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
+
+      while (regionsOfTable.size() != 2) {
+        Thread.sleep(2000);
+        regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+            .getAssignmentManager().getRegionsOfTable(userTableName.getBytes());
+      }
+      assertEquals(2, regionsOfTable.size());
+      Scan s = new Scan();
+      ResultScanner scanner = table.getScanner(s);
+      int mainTableCount = 0;
+      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
+        mainTableCount++;
+      }
+      assertEquals(3, mainTableCount);
+    } finally {
+      table.close();
+    }
+  }
+
   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
       InterruptedException {
     Put p = new Put(Bytes.toBytes("row1"));

From 7f0e0b2409be096f57672f0a3c36ac57832422c8 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 5 Jul 2013 21:51:28 +0000
Subject: [PATCH 1041/1540] HBASE-8881 TestGet failing in testDynamicFilter
 with AbstractMethodException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500143 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java | 1 +
 src/test/java/org/apache/hadoop/hbase/client/TestGet.java     | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 590e774f9c9c..f7329bf597e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -61,6 +61,7 @@ public HBaseConfiguration(final Configuration c) {
   }
 
   private static void checkDefaultsVersion(Configuration conf) {
+    if (true) return;
     if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
     String defaultsVersion = conf.get("hbase.defaults.for.version");
     String thisVersion = VersionInfo.getVersion();
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index 46c3ff38ff2c..a2418f280549 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -41,6 +41,7 @@
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mortbay.log.Log;
 
 import com.google.common.io.ByteStreams;
 
@@ -146,6 +147,8 @@ public void testDynamicFilter() throws Exception {
     try {
       get.readFields(dis);
       fail("Should not be able to load the filter class");
+    } catch (AbstractMethodError ame) {
+      Log.warn("Why has this started to show up recently? See note in HBASE-1936");
     } catch (RuntimeException re) {
       String msg = re.getMessage();
       Assert.assertTrue(msg != null

From e1ab2ec08e52d9193c432adb8197244eb64994d4 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 5 Jul 2013 21:52:40 +0000
Subject: [PATCH 1042/1540] HBASE-8881 TestGet failing in testDynamicFilter
 with AbstractMethodException; REMOVE OVERCOMMIT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500145 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index f7329bf597e3..590e774f9c9c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -61,7 +61,6 @@ public HBaseConfiguration(final Configuration c) {
   }
 
   private static void checkDefaultsVersion(Configuration conf) {
-    if (true) return;
     if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
     String defaultsVersion = conf.get("hbase.defaults.for.version");
     String thisVersion = VersionInfo.getVersion();

From 5c450ff8398ca401bd2ada8f1ae2c51d4ffaed2f Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 5 Jul 2013 22:58:22 +0000
Subject: [PATCH 1043/1540] HBASE-8881 TestGet failing in testDynamicFilter
 with AbstractMethodException; REMOVE AND DISABLE FAILING TEST

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500158 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/client/TestGet.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index a2418f280549..c2002b5b0211 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.Ignore;
 import org.junit.experimental.categories.Category;
 import org.mortbay.log.Log;
 
@@ -140,15 +141,13 @@ public void testGetAttributes() {
     Assert.assertNull(get.getAttributesMap().get("attribute1"));
   }
 
-  @Test
+  @Ignore @Test
   public void testDynamicFilter() throws Exception {
     DataInput dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET));
     Get get = new Get();
     try {
       get.readFields(dis);
       fail("Should not be able to load the filter class");
-    } catch (AbstractMethodError ame) {
-      Log.warn("Why has this started to show up recently? See note in HBASE-1936");
     } catch (RuntimeException re) {
       String msg = re.getMessage();
       Assert.assertTrue(msg != null

From b3b997d7186181764ef8449c1d3fd467a466a2ce Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Jul 2013 07:32:18 +0000
Subject: [PATCH 1044/1540] HBASE-8809 Addendum to correctly hand versions in
 raw scans aswell (Jesse and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500218 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ScanQueryMatcher.java  |  5 +-
 .../hbase/client/TestFromClientSide.java      | 76 +++++++++++++++++++
 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 967159fe0dbc..53e59ed0f403 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -158,7 +158,10 @@ public ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
     // seePastDeleteMarker: user initiated scans
     this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() && isUserScan;
 
-    int maxVersions = Math.min(scan.getMaxVersions(), scanInfo.getMaxVersions());
+    int maxVersions =
+        scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(),
+          scanInfo.getMaxVersions());
+
     // Single branch to deal with two types of reads (columns vs all in family)
     if (columns == null || columns.size() == 0) {
       // there is always a null column in the wildcard column query.
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index aa0a7f2a1ffb..c347096b6a14 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -50,6 +50,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -4860,6 +4861,81 @@ public void testJira6912() throws Exception {
     assertEquals(1, bar.length);
   }
 
+  @Test
+  public void testRawScanRespectsVersions() throws Exception {
+    byte[] TABLE = Bytes.toBytes("testRawScan");
+    HTable table = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY });
+    byte[] row = Bytes.toBytes("row");
+
+    // put the same row 4 times, with different values
+    Put p = new Put(row);
+    p.add(FAMILY, QUALIFIER, 10, VALUE);
+    table.put(p);
+    table.flushCommits();
+
+    p = new Put(row);
+    p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
+    table.put(p);
+    table.flushCommits();
+
+    p = new Put(row);
+    p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
+    table.put(p);
+    table.flushCommits();
+
+    p = new Put(row);
+    p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
+    table.put(p);
+    table.flushCommits();
+
+    int versions = 4;
+    Scan s = new Scan(row);
+    // get all the possible versions
+    s.setMaxVersions();
+    s.setRaw(true);
+
+    ResultScanner scanner = table.getScanner(s);
+    int count = 0;
+    for (Result r : scanner) {
+      assertEquals("Found an unexpected number of results for the row!", versions, r.list().size());
+      count++;
+    }
+    assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
+      count);
+    scanner.close();
+
+    // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
+    // number of versions
+    versions = 2;
+    s.setMaxVersions(versions);
+    scanner = table.getScanner(s);
+    count = 0;
+    for (Result r : scanner) {
+      assertEquals("Found an unexpected number of results for the row!", versions, r.list().size());
+      count++;
+    }
+    assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
+      count);
+    scanner.close();
+
+    // finally, if we turn off raw scanning, but max out the number of versions, we should go back
+    // to seeing just three
+    versions = 3;
+    s.setMaxVersions(versions);
+    scanner = table.getScanner(s);
+    count = 0;
+    for (Result r : scanner) {
+      assertEquals("Found an unexpected number of results for the row!", versions, r.list().size());
+      count++;
+    }
+    assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
+      count);
+    scanner.close();
+
+    table.close();
+    TEST_UTIL.deleteTable(TABLE);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 804cb569deae3498b5aa78089b18b51c0a006403 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 6 Jul 2013 08:53:49 +0000
Subject: [PATCH 1045/1540] HBASE-8809 test fix

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500231 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
index 357f86695627..85b35691a1a7 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
@@ -770,7 +770,8 @@ private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col,
   private int countDeleteMarkers(HRegion region) throws IOException {
     Scan s = new Scan();
     s.setRaw(true);
-    s.setMaxVersions();
+    // use max versions from the store(s)
+    s.setMaxVersions(region.getStores().values().iterator().next().getScanInfo().getMaxVersions());
     InternalScanner scan = region.getScanner(s);
     List<KeyValue> kvs = new ArrayList<KeyValue>();
     int res = 0;

From f0a494b7d9952eeef27002982739405848cef2fa Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sun, 7 Jul 2013 03:31:25 +0000
Subject: [PATCH 1046/1540] HBASE-8885 Fix and reenable TestGet
 failing#testDynamicFilter

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1500365 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/TestGet.java    | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index c2002b5b0211..3cf92cbd9e7b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -20,6 +20,7 @@
 
 package org.apache.hadoop.hbase.client;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
@@ -40,9 +41,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Assert;
 import org.junit.Test;
-import org.junit.Ignore;
 import org.junit.experimental.categories.Category;
-import org.mortbay.log.Log;
 
 import com.google.common.io.ByteStreams;
 
@@ -141,8 +140,15 @@ public void testGetAttributes() {
     Assert.assertNull(get.getAttributesMap().get("attribute1"));
   }
 
-  @Ignore @Test
+  @Test
   public void testDynamicFilter() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String localPath = conf.get("hbase.local.dir")
+      + File.separator + "jars" + File.separator;
+    File jarFile = new File(localPath, "MockFilter.jar");
+    jarFile.delete();
+    assertFalse("Should be deleted: " + jarFile.getPath(), jarFile.exists());
+
     DataInput dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET));
     Get get = new Get();
     try {
@@ -154,12 +160,6 @@ public void testDynamicFilter() throws Exception {
         && msg.contains("Can't find class test.MockFilter"));
     }
 
-    Configuration conf = HBaseConfiguration.create();
-    String localPath = conf.get("hbase.local.dir")
-      + File.separator + "jars" + File.separator;
-    File jarFile = new File(localPath, "MockFilter.jar");
-    jarFile.deleteOnExit();
-
     FileOutputStream fos = new FileOutputStream(jarFile);
     fos.write(Base64.decode(MOCK_FILTER_JAR));
     fos.close();

From 9250035a1c4c2264c8e866ec2c1cb57dd34506e9 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Tue, 9 Jul 2013 12:05:58 +0000
Subject: [PATCH 1047/1540] HBASE-8819 Port HBASE-5428 to Thrift 2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1501211 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift2/ThriftServer.java    | 21 +++++++++++++++++++
 .../TestThriftHBaseServiceHandler.java        | 13 ++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index 1b97c6399ae4..edabdb9739df 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -40,6 +40,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.thrift.CallQueue;
 import org.apache.hadoop.hbase.thrift.CallQueue.Call;
 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
@@ -192,6 +193,25 @@ private static TServer getTThreadPoolServer(TProtocolFactory protocolFactory, TH
     return new TThreadPoolServer(serverArgs);
   }
 
+  /**
+   * Adds the option to pre-load filters at startup.
+   *
+   * @param conf  The current configuration instance.
+   */
+  protected static void registerFilters(Configuration conf) {
+    String[] filters = conf.getStrings("hbase.thrift.filters");
+    if(filters != null) {
+      for(String filterClass: filters) {
+        String[] filterPart = filterClass.split(":");
+        if(filterPart.length != 2) {
+          log.warn("Invalid filter specification " + filterClass + " - skipping");
+        } else {
+          ParseFilter.registerFilter(filterPart[0], filterPart[1]);
+        }
+      }
+    }
+  }
+
   /**
    * Start up the Thrift2 server.
    * 
@@ -237,6 +257,7 @@ public static void main(String[] args) throws Exception {
 
       conf.set("hbase.regionserver.thrift.server.type", implType);
       conf.setInt("hbase.regionserver.thrift.port", listenPort);
+      registerFilters(conf);
 
       // Construct correct ProtocolFactory
       boolean compact = cmd.hasOption("compact");
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 1e732b6e47a0..589bafb58b54 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -25,6 +25,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -34,6 +35,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
@@ -626,7 +628,6 @@ public void testScanWithBatchSize() throws Exception {
     int scanId = handler.openScanner(table, scan);
     List<TResult> results = null;
     for (int i = 0; i < 10; i++) {
-      System.out.println("batch: " + i);
       // get batch for single row (10x10 is what we expect)
       results = handler.getScannerRows(scanId, 1);
       assertEquals(1, results.size());
@@ -637,7 +638,6 @@ public void testScanWithBatchSize() throws Exception {
       for (int y = 0; y < 10; y++) {
         int colNum = y + (10 * i);
         String colNumPad = pad(colNum, (byte) 3);
-        System.out.println("col" + colNumPad + ": " + new String(cols.get(y).getQualifier()));
         assertArrayEquals(("col" + colNumPad).getBytes(), cols.get(y).getQualifier());
       }
     }
@@ -655,6 +655,15 @@ public void testScanWithBatchSize() throws Exception {
     }
   }
 
+  @Test
+  public void testFilterRegistration() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+    conf.set("hbase.thrift.filters", "MyFilter:filterclass");
+    ThriftServer.registerFilters(conf);
+    Map<String, String> registeredFilters = ParseFilter.getAllFilters();
+    assertEquals("filterclass", registeredFilters.get("MyFilter"));
+  }
+
   @Test
   public void testMetrics() throws Exception {
     Configuration conf = UTIL.getConfiguration();

From b6afe0127c6169d90c4049dbd06835a750e85e9d Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Tue, 9 Jul 2013 18:30:28 +0000
Subject: [PATCH 1048/1540] HBASE-8678 Wrongly delete cells in some case which
 can not be deleted (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1501441 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/client/RemoteHTable.java  |  4 +---
 .../hbase/rest/client/TestRemoteTable.java      | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index e3be8e1a8970..527384dea52f 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -86,9 +86,7 @@ protected String buildRowSpec(final byte[] row, final Map familyMap,
     Set families = familyMap.entrySet();
     if (families != null) {
       Iterator i = familyMap.entrySet().iterator();
-      if (i.hasNext()) {
-        sb.append('/');
-      }
+      sb.append('/');
       while (i.hasNext()) {
         Map.Entry e = (Map.Entry)i.next();
         Collection quals = (Collection)e.getValue();
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index a3ab9ec19ef0..dad189f3b8bc 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -303,6 +303,7 @@ public void testPut() throws IOException {
     assertTrue(Bytes.equals(VALUE_2, value));
   }
 
+  @Test
   public void testDelete() throws IOException {
     Put put = new Put(ROW_3);
     put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
@@ -323,7 +324,21 @@ public void testDelete() throws IOException {
     Delete delete = new Delete(ROW_3);
     delete.deleteColumn(COLUMN_2, QUALIFIER_2);
     remoteTable.delete(delete);
-    
+
+    get = new Get(ROW_3);
+    get.addFamily(COLUMN_1);
+    get.addFamily(COLUMN_2);
+    result = remoteTable.get(get);
+    value1 = result.getValue(COLUMN_1, QUALIFIER_1);
+    value2 = result.getValue(COLUMN_2, QUALIFIER_2);
+    assertNotNull(value1);
+    assertTrue(Bytes.equals(VALUE_1, value1));
+    assertNull(value2);
+
+    delete = new Delete(ROW_3);
+    delete.setTimestamp(1L);
+    remoteTable.delete(delete);
+
     get = new Get(ROW_3);
     get.addFamily(COLUMN_1);
     get.addFamily(COLUMN_2);

From 4a534e30a066cf8fc15c6d8d4d81e2409858d693 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Wed, 10 Jul 2013 17:34:59 +0000
Subject: [PATCH 1049/1540] HBASE-8910  HMaster.abortNow shouldn't try to
 become a master again if it was stopped

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1501869 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 48d6f58c6296..288ea6d072f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1690,7 +1690,7 @@ public Boolean call() throws InterruptedException,
    * @return true if we should proceed with abort operation, false other wise.
    */
   private boolean abortNow(final String msg, final Throwable t) {
-    if (!this.isActiveMaster) {
+    if (!this.isActiveMaster || this.stopped) {
       return true;
     }
     if (t != null && t instanceof KeeperException.SessionExpiredException) {

From b8f04bfb932d9029e701adca56c8ee04dfa99478 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Wed, 10 Jul 2013 20:29:10 +0000
Subject: [PATCH 1050/1540] HBASE-8826 Ensure HBASE-8695 is covered in Thrift 2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1501970 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/thrift2/ThriftServer.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index edabdb9739df..ca57fa9a0b9d 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -260,14 +260,16 @@ public static void main(String[] args) throws Exception {
       registerFilters(conf);
 
       // Construct correct ProtocolFactory
-      boolean compact = cmd.hasOption("compact");
+      boolean compact = cmd.hasOption("compact") ||
+        conf.getBoolean("hbase.regionserver.thrift.compact", false);
       TProtocolFactory protocolFactory = getTProtocolFactory(compact);
       THBaseService.Iface handler =
           ThriftHBaseServiceHandler.newInstance(conf, metrics);
       THBaseService.Processor processor = new THBaseService.Processor(handler);
       conf.setBoolean("hbase.regionserver.thrift.compact", compact);
 
-      boolean framed = cmd.hasOption("framed") || nonblocking || hsha;
+      boolean framed = cmd.hasOption("framed") ||
+        conf.getBoolean("hbase.regionserver.thrift.framed", false) || nonblocking || hsha;
       TTransportFactory transportFactory = getTTransportFactory(framed);
       conf.setBoolean("hbase.regionserver.thrift.framed", framed);
 

From 790f0d4804912d8d431cdb45b8c5f986c00a7499 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Wed, 10 Jul 2013 20:47:36 +0000
Subject: [PATCH 1051/1540] HBASE-8921 [thrift2] Add GenericOptionsParser to
 Thrift 2 server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1501984 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/thrift2/ThriftServer.java   | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index ca57fa9a0b9d..b6235ac6a441 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.thrift2;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
@@ -46,6 +47,7 @@
 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
 import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
 import org.apache.hadoop.hbase.util.InfoServer;
+import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TCompactProtocol;
 import org.apache.thrift.protocol.TProtocolFactory;
@@ -104,9 +106,12 @@ private static Options getOptions() {
     return options;
   }
 
-  private static CommandLine parseArguments(Options options, String[] args) throws ParseException {
+  private static CommandLine parseArguments(Configuration conf, Options options, String[] args)
+      throws ParseException, IOException {
+    GenericOptionsParser genParser = new GenericOptionsParser(conf, args);
+    String[] remainingArgs = genParser.getRemainingArgs();
     CommandLineParser parser = new PosixParser();
-    return parser.parse(options, args);
+    return parser.parse(options, remainingArgs);
   }
 
   private static TProtocolFactory getTProtocolFactory(boolean isCompact) {
@@ -221,7 +226,8 @@ public static void main(String[] args) throws Exception {
     TServer server = null;
     Options options = getOptions();
     try {
-      CommandLine cmd = parseArguments(options, args);
+      Configuration conf = HBaseConfiguration.create();
+      CommandLine cmd = parseArguments(conf, options, args);
 
       /**
        * This is to please both bin/hbase and bin/hbase-daemon. hbase-daemon provides "start" and "stop" arguments hbase
@@ -244,7 +250,6 @@ public static void main(String[] args) throws Exception {
       boolean nonblocking = cmd.hasOption("nonblocking");
       boolean hsha = cmd.hasOption("hsha");
 
-      Configuration conf = HBaseConfiguration.create();
       ThriftMetrics metrics = new ThriftMetrics(
           listenPort, conf, THBaseService.Iface.class);
 

From ae9b21ccc8e94a32bc9760cfe146a07949c3f12b Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Thu, 11 Jul 2013 09:10:17 +0000
Subject: [PATCH 1052/1540] HBASE-8695 The HBase thrift service ignores XML
 configuration

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1502161 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/thrift/ThriftServer.java   | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
index 43da992e72b9..a2f244c67523 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
@@ -194,10 +194,12 @@ private void processOptions(final String[] args) throws Exception {
         conf, TBoundedThreadPoolServer.THREAD_KEEP_ALIVE_TIME_SEC_CONF_KEY);
 
     // Set general thrift server options
-    conf.setBoolean(
-        ThriftServerRunner.COMPACT_CONF_KEY, cmd.hasOption(COMPACT_OPTION));
-    conf.setBoolean(
-        ThriftServerRunner.FRAMED_CONF_KEY, cmd.hasOption(FRAMED_OPTION));
+    boolean compact = cmd.hasOption(COMPACT_OPTION) ||
+      conf.getBoolean(ThriftServerRunner.COMPACT_CONF_KEY, false);
+    conf.setBoolean(ThriftServerRunner.COMPACT_CONF_KEY, compact);
+    boolean framed = cmd.hasOption(FRAMED_OPTION) ||
+      conf.getBoolean(ThriftServerRunner.FRAMED_CONF_KEY, false);
+    conf.setBoolean(ThriftServerRunner.FRAMED_CONF_KEY, framed);
     if (cmd.hasOption(BIND_OPTION)) {
       conf.set(
           ThriftServerRunner.BIND_CONF_KEY, cmd.getOptionValue(BIND_OPTION));

From 4309655ab64fc497c0343d279f2f3625e9bbcbad Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Fri, 12 Jul 2013 03:25:07 +0000
Subject: [PATCH 1053/1540] HBASE-7770 minor integration test framework fixes
 (Francis Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1502417 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/DistributedHBaseCluster.java | 20 ++++++++++++++-----
 .../hadoop/hbase/HBaseTestingUtility.java     |  2 +-
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 53d4ce3e3f0b..b709613b8669 100644
--- a/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -225,7 +225,9 @@ public void restoreClusterStatus(ClusterStatus initial) throws IOException {
       //start backup masters
       for (ServerName backup : initial.getBackupMasters()) {
         //these are not started in backup mode, but we should already have an active master
-        startMaster(backup.getHostname());
+        if(!clusterManager.isRunning(ServiceType.HBASE_MASTER, backup.getHostname())) {
+          startMaster(backup.getHostname());
+        }
       }
     } else {
       //current master has not changed, match up backup masters
@@ -240,11 +242,15 @@ public void restoreClusterStatus(ClusterStatus initial) throws IOException {
       }
 
       for (String hostname : Sets.difference(initialBackups.keySet(), currentBackups.keySet())) {
-        startMaster(hostname);
+        if(!clusterManager.isRunning(ServiceType.HBASE_MASTER, hostname)) {
+          startMaster(hostname);
+        }
       }
 
       for (String hostname : Sets.difference(currentBackups.keySet(), initialBackups.keySet())) {
-        stopMaster(currentBackups.get(hostname));
+        if(clusterManager.isRunning(ServiceType.HBASE_MASTER, hostname)) {
+          stopMaster(currentBackups.get(hostname));
+        }
       }
     }
 
@@ -260,11 +266,15 @@ public void restoreClusterStatus(ClusterStatus initial) throws IOException {
     }
 
     for (String hostname : Sets.difference(initialServers.keySet(), currentServers.keySet())) {
-      startRegionServer(hostname);
+      if(!clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER, hostname)) {
+        startRegionServer(hostname);
+      }
     }
 
     for (String hostname : Sets.difference(currentServers.keySet(), initialServers.keySet())) {
-      stopRegionServer(currentServers.get(hostname));
+      if(clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER, hostname)) {
+        stopRegionServer(currentServers.get(hostname));
+      }
     }
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 3a76d89c9b2b..1214045b5664 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1266,7 +1266,7 @@ public int createMultiRegions(final Configuration c, final HTable table,
       Path tableDir = new Path(getDefaultRootDirPath().toString()
           + System.getProperty("file.separator") + htd.getNameAsString()
           + System.getProperty("file.separator") + regionToDeleteInFS);
-      getDFSCluster().getFileSystem().delete(tableDir);
+      FileSystem.get(c).delete(tableDir);
     }
     // flush cache of regions
     HConnection conn = table.getConnection();

From 13336fa288facf1487b1d808ef2e71dc63e2b9f5 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Fri, 12 Jul 2013 06:04:40 +0000
Subject: [PATCH 1054/1540] HBASE-8832 Ensure HBASE-4658 is supported by Thrift
 2 (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1502433 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift2/ThriftUtilities.java |  46 +++-
 .../hbase/thrift2/generated/TDelete.java      | 211 +++++++++++++++--
 .../hadoop/hbase/thrift2/generated/TGet.java  | 197 ++++++++++++++--
 .../thrift2/generated/THBaseService.java      | 216 +++++++++---------
 .../hadoop/hbase/thrift2/generated/TPut.java  | 211 +++++++++++++++--
 .../hadoop/hbase/thrift2/generated/TScan.java | 211 +++++++++++++++--
 .../apache/hadoop/hbase/thrift2/hbase.thrift  |  12 +-
 .../TestThriftHBaseServiceHandler.java        |  40 ++++
 8 files changed, 946 insertions(+), 198 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
index 254de6990d48..982b512fcb86 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
@@ -24,11 +24,14 @@
 import org.apache.hadoop.hbase.client.*;
 import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.thrift2.generated.*;
+import org.apache.hadoop.hbase.util.Bytes;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
 
+import static org.apache.hadoop.hbase.util.Bytes.getBytes;
+
 public class ThriftUtilities {
 
   private ThriftUtilities() {
@@ -60,6 +63,15 @@ public static Get getFromThrift(TGet in) throws IOException {
       out.setMaxVersions(in.getMaxVersions());
     }
 
+    if (in.isSetFilterString()) {
+      ParseFilter parseFilter = new ParseFilter();
+      out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
+    }
+
+    if (in.isSetAttributes()) {
+      addAttributes(out,in.getAttributes());
+    }
+
     if (!in.isSetColumns()) {
       return out;
     }
@@ -72,11 +84,6 @@ public static Get getFromThrift(TGet in) throws IOException {
       }
     }
 
-    if (in.isSetFilterString()) {
-      ParseFilter parseFilter = new ParseFilter();
-      out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
-    }
-
     return out;
   }
 
@@ -169,6 +176,10 @@ public static Put putFromThrift(TPut in) {
       }
     }
 
+    if (in.isSetAttributes()) {
+      addAttributes(out,in.getAttributes());
+    }
+
     return out;
   }
 
@@ -232,6 +243,11 @@ public static Delete deleteFromThrift(TDelete in) {
         out = new Delete(in.getRow());
       }
     }
+
+    if (in.isSetAttributes()) {
+      addAttributes(out,in.getAttributes());
+    }
+
     out.setWriteToWAL(in.isWriteToWal());
     return out;
   }
@@ -337,6 +353,10 @@ public static Scan scanFromThrift(TScan in) throws IOException {
       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
     }
 
+    if (in.isSetAttributes()) {
+      addAttributes(out,in.getAttributes());
+    }
+
     return out;
   }
 
@@ -345,7 +365,23 @@ public static Increment incrementFromThrift(TIncrement in) throws IOException {
     for (TColumnIncrement column : in.getColumns()) {
       out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
     }
+
     out.setWriteToWAL(in.isWriteToWal());
     return out;
   }
+
+  /**
+   * Adds all the attributes into the Operation object
+   */
+  private static void addAttributes(OperationWithAttributes op,
+                                    Map<ByteBuffer, ByteBuffer> attributes) {
+    if (attributes == null || attributes.size() == 0) {
+      return;
+    }
+    for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
+      String name = Bytes.toStringBinary(getBytes(entry.getKey()));
+      byte[] value =  getBytes(entry.getValue());
+      op.setAttribute(name, value);
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
index f2eef0d19939..ddf98d442897 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
@@ -58,6 +58,7 @@ public class TDelete implements org.apache.thrift.TBase<TDelete, TDelete._Fields
   private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
   private static final org.apache.thrift.protocol.TField DELETE_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("deleteType", org.apache.thrift.protocol.TType.I32, (short)4);
   private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)5);
+  private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)6);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -74,6 +75,7 @@ public class TDelete implements org.apache.thrift.TBase<TDelete, TDelete._Fields
    */
   public TDeleteType deleteType; // optional
   public boolean writeToWal; // optional
+  public Map<ByteBuffer,ByteBuffer> attributes; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -85,7 +87,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
      * @see TDeleteType
      */
     DELETE_TYPE((short)4, "deleteType"),
-    WRITE_TO_WAL((short)5, "writeToWal");
+    WRITE_TO_WAL((short)5, "writeToWal"),
+    ATTRIBUTES((short)6, "attributes");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -110,6 +113,8 @@ public static _Fields findByThriftId(int fieldId) {
           return DELETE_TYPE;
         case 5: // WRITE_TO_WAL
           return WRITE_TO_WAL;
+        case 6: // ATTRIBUTES
+          return ATTRIBUTES;
         default:
           return null;
       }
@@ -153,7 +158,7 @@ public String getFieldName() {
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __WRITETOWAL_ISSET_ID = 1;
   private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.WRITE_TO_WAL};
+  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.WRITE_TO_WAL,_Fields.ATTRIBUTES};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -168,6 +173,10 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TDeleteType.class)));
     tmpMap.put(_Fields.WRITE_TO_WAL, new org.apache.thrift.meta_data.FieldMetaData("writeToWal", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
+    tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true), 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDelete.class, metaDataMap);
   }
@@ -208,6 +217,23 @@ public TDelete(TDelete other) {
       this.deleteType = other.deleteType;
     }
     this.writeToWal = other.writeToWal;
+    if (other.isSetAttributes()) {
+      Map<ByteBuffer,ByteBuffer> __this__attributes = new HashMap<ByteBuffer,ByteBuffer>();
+      for (Map.Entry<ByteBuffer, ByteBuffer> other_element : other.attributes.entrySet()) {
+
+        ByteBuffer other_element_key = other_element.getKey();
+        ByteBuffer other_element_value = other_element.getValue();
+
+        ByteBuffer __this__attributes_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
+;
+
+        ByteBuffer __this__attributes_copy_value = org.apache.thrift.TBaseHelper.copyBinary(other_element_value);
+;
+
+        __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value);
+      }
+      this.attributes = __this__attributes;
+    }
   }
 
   public TDelete deepCopy() {
@@ -224,6 +250,7 @@ public void clear() {
 
     this.writeToWal = true;
 
+    this.attributes = null;
   }
 
   public byte[] getRow() {
@@ -377,6 +404,41 @@ public void setWriteToWalIsSet(boolean value) {
     __isset_bit_vector.set(__WRITETOWAL_ISSET_ID, value);
   }
 
+  public int getAttributesSize() {
+    return (this.attributes == null) ? 0 : this.attributes.size();
+  }
+
+  public void putToAttributes(ByteBuffer key, ByteBuffer val) {
+    if (this.attributes == null) {
+      this.attributes = new HashMap<ByteBuffer,ByteBuffer>();
+    }
+    this.attributes.put(key, val);
+  }
+
+  public Map<ByteBuffer,ByteBuffer> getAttributes() {
+    return this.attributes;
+  }
+
+  public TDelete setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
+    this.attributes = attributes;
+    return this;
+  }
+
+  public void unsetAttributes() {
+    this.attributes = null;
+  }
+
+  /** Returns true if field attributes is set (has been assigned a value) and false otherwise */
+  public boolean isSetAttributes() {
+    return this.attributes != null;
+  }
+
+  public void setAttributesIsSet(boolean value) {
+    if (!value) {
+      this.attributes = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -419,6 +481,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case ATTRIBUTES:
+      if (value == null) {
+        unsetAttributes();
+      } else {
+        setAttributes((Map<ByteBuffer,ByteBuffer>)value);
+      }
+      break;
+
     }
   }
 
@@ -439,6 +509,9 @@ public Object getFieldValue(_Fields field) {
     case WRITE_TO_WAL:
       return Boolean.valueOf(isWriteToWal());
 
+    case ATTRIBUTES:
+      return getAttributes();
+
     }
     throw new IllegalStateException();
   }
@@ -460,6 +533,8 @@ public boolean isSet(_Fields field) {
       return isSetDeleteType();
     case WRITE_TO_WAL:
       return isSetWriteToWal();
+    case ATTRIBUTES:
+      return isSetAttributes();
     }
     throw new IllegalStateException();
   }
@@ -522,6 +597,15 @@ public boolean equals(TDelete that) {
         return false;
     }
 
+    boolean this_present_attributes = true && this.isSetAttributes();
+    boolean that_present_attributes = true && that.isSetAttributes();
+    if (this_present_attributes || that_present_attributes) {
+      if (!(this_present_attributes && that_present_attributes))
+        return false;
+      if (!this.attributes.equals(that.attributes))
+        return false;
+    }
+
     return true;
   }
 
@@ -588,6 +672,16 @@ public int compareTo(TDelete other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetAttributes()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -647,6 +741,16 @@ public String toString() {
       sb.append(this.writeToWal);
       first = false;
     }
+    if (isSetAttributes()) {
+      if (!first) sb.append(", ");
+      sb.append("attributes:");
+      if (this.attributes == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.attributes);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -705,14 +809,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) thr
           case 2: // COLUMNS
             if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list24 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumn>(_list24.size);
-                for (int _i25 = 0; _i25 < _list24.size; ++_i25)
+                org.apache.thrift.protocol.TList _list44 = iprot.readListBegin();
+                struct.columns = new ArrayList<TColumn>(_list44.size);
+                for (int _i45 = 0; _i45 < _list44.size; ++_i45)
                 {
-                  TColumn _elem26; // required
-                  _elem26 = new TColumn();
-                  _elem26.read(iprot);
-                  struct.columns.add(_elem26);
+                  TColumn _elem46; // required
+                  _elem46 = new TColumn();
+                  _elem46.read(iprot);
+                  struct.columns.add(_elem46);
                 }
                 iprot.readListEnd();
               }
@@ -745,6 +849,26 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) thr
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 6: // ATTRIBUTES
+            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+              {
+                org.apache.thrift.protocol.TMap _map47 = iprot.readMapBegin();
+                struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map47.size);
+                for (int _i48 = 0; _i48 < _map47.size; ++_i48)
+                {
+                  ByteBuffer _key49; // required
+                  ByteBuffer _val50; // optional
+                  _key49 = iprot.readBinary();
+                  _val50 = iprot.readBinary();
+                  struct.attributes.put(_key49, _val50);
+                }
+                iprot.readMapEnd();
+              }
+              struct.setAttributesIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -770,9 +894,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TDelete struct) th
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-            for (TColumn _iter27 : struct.columns)
+            for (TColumn _iter51 : struct.columns)
             {
-              _iter27.write(oprot);
+              _iter51.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -796,6 +920,21 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TDelete struct) th
         oprot.writeBool(struct.writeToWal);
         oprot.writeFieldEnd();
       }
+      if (struct.attributes != null) {
+        if (struct.isSetAttributes()) {
+          oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
+          {
+            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter52 : struct.attributes.entrySet())
+            {
+              oprot.writeBinary(_iter52.getKey());
+              oprot.writeBinary(_iter52.getValue());
+            }
+            oprot.writeMapEnd();
+          }
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -827,13 +966,16 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thr
       if (struct.isSetWriteToWal()) {
         optionals.set(3);
       }
-      oprot.writeBitSet(optionals, 4);
+      if (struct.isSetAttributes()) {
+        optionals.set(4);
+      }
+      oprot.writeBitSet(optionals, 5);
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
-          for (TColumn _iter28 : struct.columns)
+          for (TColumn _iter53 : struct.columns)
           {
-            _iter28.write(oprot);
+            _iter53.write(oprot);
           }
         }
       }
@@ -846,6 +988,16 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thr
       if (struct.isSetWriteToWal()) {
         oprot.writeBool(struct.writeToWal);
       }
+      if (struct.isSetAttributes()) {
+        {
+          oprot.writeI32(struct.attributes.size());
+          for (Map.Entry<ByteBuffer, ByteBuffer> _iter54 : struct.attributes.entrySet())
+          {
+            oprot.writeBinary(_iter54.getKey());
+            oprot.writeBinary(_iter54.getValue());
+          }
+        }
+      }
     }
 
     @Override
@@ -853,17 +1005,17 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro
       TTupleProtocol iprot = (TTupleProtocol) prot;
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
+      BitSet incoming = iprot.readBitSet(5);
       if (incoming.get(0)) {
         {
-          org.apache.thrift.protocol.TList _list29 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.columns = new ArrayList<TColumn>(_list29.size);
-          for (int _i30 = 0; _i30 < _list29.size; ++_i30)
+          org.apache.thrift.protocol.TList _list55 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.columns = new ArrayList<TColumn>(_list55.size);
+          for (int _i56 = 0; _i56 < _list55.size; ++_i56)
           {
-            TColumn _elem31; // required
-            _elem31 = new TColumn();
-            _elem31.read(iprot);
-            struct.columns.add(_elem31);
+            TColumn _elem57; // required
+            _elem57 = new TColumn();
+            _elem57.read(iprot);
+            struct.columns.add(_elem57);
           }
         }
         struct.setColumnsIsSet(true);
@@ -880,6 +1032,21 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro
         struct.writeToWal = iprot.readBool();
         struct.setWriteToWalIsSet(true);
       }
+      if (incoming.get(4)) {
+        {
+          org.apache.thrift.protocol.TMap _map58 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+          struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map58.size);
+          for (int _i59 = 0; _i59 < _map58.size; ++_i59)
+          {
+            ByteBuffer _key60; // required
+            ByteBuffer _val61; // optional
+            _key60 = iprot.readBinary();
+            _val61 = iprot.readBinary();
+            struct.attributes.put(_key60, _val61);
+          }
+        }
+        struct.setAttributesIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
index f3b496951a36..fca79c324e2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
@@ -49,6 +49,7 @@ public class TGet implements org.apache.thrift.TBase<TGet, TGet._Fields>, java.i
   private static final org.apache.thrift.protocol.TField TIME_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeRange", org.apache.thrift.protocol.TType.STRUCT, (short)4);
   private static final org.apache.thrift.protocol.TField MAX_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxVersions", org.apache.thrift.protocol.TType.I32, (short)5);
   private static final org.apache.thrift.protocol.TField FILTER_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("filterString", org.apache.thrift.protocol.TType.STRING, (short)6);
+  private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)7);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -62,6 +63,7 @@ public class TGet implements org.apache.thrift.TBase<TGet, TGet._Fields>, java.i
   public TTimeRange timeRange; // optional
   public int maxVersions; // optional
   public ByteBuffer filterString; // optional
+  public Map<ByteBuffer,ByteBuffer> attributes; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -70,7 +72,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     TIMESTAMP((short)3, "timestamp"),
     TIME_RANGE((short)4, "timeRange"),
     MAX_VERSIONS((short)5, "maxVersions"),
-    FILTER_STRING((short)6, "filterString");
+    FILTER_STRING((short)6, "filterString"),
+    ATTRIBUTES((short)7, "attributes");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -97,6 +100,8 @@ public static _Fields findByThriftId(int fieldId) {
           return MAX_VERSIONS;
         case 6: // FILTER_STRING
           return FILTER_STRING;
+        case 7: // ATTRIBUTES
+          return ATTRIBUTES;
         default:
           return null;
       }
@@ -140,7 +145,7 @@ public String getFieldName() {
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __MAXVERSIONS_ISSET_ID = 1;
   private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.TIME_RANGE,_Fields.MAX_VERSIONS,_Fields.FILTER_STRING};
+  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.TIME_RANGE,_Fields.MAX_VERSIONS,_Fields.FILTER_STRING,_Fields.ATTRIBUTES};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -157,6 +162,10 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
     tmpMap.put(_Fields.FILTER_STRING, new org.apache.thrift.meta_data.FieldMetaData("filterString", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
+    tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true), 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGet.class, metaDataMap);
   }
@@ -197,6 +206,23 @@ public TGet(TGet other) {
       this.filterString = org.apache.thrift.TBaseHelper.copyBinary(other.filterString);
 ;
     }
+    if (other.isSetAttributes()) {
+      Map<ByteBuffer,ByteBuffer> __this__attributes = new HashMap<ByteBuffer,ByteBuffer>();
+      for (Map.Entry<ByteBuffer, ByteBuffer> other_element : other.attributes.entrySet()) {
+
+        ByteBuffer other_element_key = other_element.getKey();
+        ByteBuffer other_element_value = other_element.getValue();
+
+        ByteBuffer __this__attributes_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
+;
+
+        ByteBuffer __this__attributes_copy_value = org.apache.thrift.TBaseHelper.copyBinary(other_element_value);
+;
+
+        __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value);
+      }
+      this.attributes = __this__attributes;
+    }
   }
 
   public TGet deepCopy() {
@@ -213,6 +239,7 @@ public void clear() {
     setMaxVersionsIsSet(false);
     this.maxVersions = 0;
     this.filterString = null;
+    this.attributes = null;
   }
 
   public byte[] getRow() {
@@ -392,6 +419,41 @@ public void setFilterStringIsSet(boolean value) {
     }
   }
 
+  public int getAttributesSize() {
+    return (this.attributes == null) ? 0 : this.attributes.size();
+  }
+
+  public void putToAttributes(ByteBuffer key, ByteBuffer val) {
+    if (this.attributes == null) {
+      this.attributes = new HashMap<ByteBuffer,ByteBuffer>();
+    }
+    this.attributes.put(key, val);
+  }
+
+  public Map<ByteBuffer,ByteBuffer> getAttributes() {
+    return this.attributes;
+  }
+
+  public TGet setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
+    this.attributes = attributes;
+    return this;
+  }
+
+  public void unsetAttributes() {
+    this.attributes = null;
+  }
+
+  /** Returns true if field attributes is set (has been assigned a value) and false otherwise */
+  public boolean isSetAttributes() {
+    return this.attributes != null;
+  }
+
+  public void setAttributesIsSet(boolean value) {
+    if (!value) {
+      this.attributes = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -442,6 +504,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case ATTRIBUTES:
+      if (value == null) {
+        unsetAttributes();
+      } else {
+        setAttributes((Map<ByteBuffer,ByteBuffer>)value);
+      }
+      break;
+
     }
   }
 
@@ -465,6 +535,9 @@ public Object getFieldValue(_Fields field) {
     case FILTER_STRING:
       return getFilterString();
 
+    case ATTRIBUTES:
+      return getAttributes();
+
     }
     throw new IllegalStateException();
   }
@@ -488,6 +561,8 @@ public boolean isSet(_Fields field) {
       return isSetMaxVersions();
     case FILTER_STRING:
       return isSetFilterString();
+    case ATTRIBUTES:
+      return isSetAttributes();
     }
     throw new IllegalStateException();
   }
@@ -559,6 +634,15 @@ public boolean equals(TGet that) {
         return false;
     }
 
+    boolean this_present_attributes = true && this.isSetAttributes();
+    boolean that_present_attributes = true && that.isSetAttributes();
+    if (this_present_attributes || that_present_attributes) {
+      if (!(this_present_attributes && that_present_attributes))
+        return false;
+      if (!this.attributes.equals(that.attributes))
+        return false;
+    }
+
     return true;
   }
 
@@ -635,6 +719,16 @@ public int compareTo(TGet other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetAttributes()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -704,6 +798,16 @@ public String toString() {
       }
       first = false;
     }
+    if (isSetAttributes()) {
+      if (!first) sb.append(", ");
+      sb.append("attributes:");
+      if (this.attributes == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.attributes);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -811,6 +915,26 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TGet struct) throws
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 7: // ATTRIBUTES
+            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+              {
+                org.apache.thrift.protocol.TMap _map11 = iprot.readMapBegin();
+                struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map11.size);
+                for (int _i12 = 0; _i12 < _map11.size; ++_i12)
+                {
+                  ByteBuffer _key13; // required
+                  ByteBuffer _val14; // optional
+                  _key13 = iprot.readBinary();
+                  _val14 = iprot.readBinary();
+                  struct.attributes.put(_key13, _val14);
+                }
+                iprot.readMapEnd();
+              }
+              struct.setAttributesIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -836,9 +960,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TGet struct) throw
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-            for (TColumn _iter11 : struct.columns)
+            for (TColumn _iter15 : struct.columns)
             {
-              _iter11.write(oprot);
+              _iter15.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -869,6 +993,21 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TGet struct) throw
           oprot.writeFieldEnd();
         }
       }
+      if (struct.attributes != null) {
+        if (struct.isSetAttributes()) {
+          oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
+          {
+            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter16 : struct.attributes.entrySet())
+            {
+              oprot.writeBinary(_iter16.getKey());
+              oprot.writeBinary(_iter16.getValue());
+            }
+            oprot.writeMapEnd();
+          }
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -903,13 +1042,16 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
       if (struct.isSetFilterString()) {
         optionals.set(4);
       }
-      oprot.writeBitSet(optionals, 5);
+      if (struct.isSetAttributes()) {
+        optionals.set(5);
+      }
+      oprot.writeBitSet(optionals, 6);
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
-          for (TColumn _iter12 : struct.columns)
+          for (TColumn _iter17 : struct.columns)
           {
-            _iter12.write(oprot);
+            _iter17.write(oprot);
           }
         }
       }
@@ -925,6 +1067,16 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
       if (struct.isSetFilterString()) {
         oprot.writeBinary(struct.filterString);
       }
+      if (struct.isSetAttributes()) {
+        {
+          oprot.writeI32(struct.attributes.size());
+          for (Map.Entry<ByteBuffer, ByteBuffer> _iter18 : struct.attributes.entrySet())
+          {
+            oprot.writeBinary(_iter18.getKey());
+            oprot.writeBinary(_iter18.getValue());
+          }
+        }
+      }
     }
 
     @Override
@@ -932,17 +1084,17 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
       TTupleProtocol iprot = (TTupleProtocol) prot;
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
-      BitSet incoming = iprot.readBitSet(5);
+      BitSet incoming = iprot.readBitSet(6);
       if (incoming.get(0)) {
         {
-          org.apache.thrift.protocol.TList _list13 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.columns = new ArrayList<TColumn>(_list13.size);
-          for (int _i14 = 0; _i14 < _list13.size; ++_i14)
+          org.apache.thrift.protocol.TList _list19 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.columns = new ArrayList<TColumn>(_list19.size);
+          for (int _i20 = 0; _i20 < _list19.size; ++_i20)
           {
-            TColumn _elem15; // required
-            _elem15 = new TColumn();
-            _elem15.read(iprot);
-            struct.columns.add(_elem15);
+            TColumn _elem21; // required
+            _elem21 = new TColumn();
+            _elem21.read(iprot);
+            struct.columns.add(_elem21);
           }
         }
         struct.setColumnsIsSet(true);
@@ -964,6 +1116,21 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
         struct.filterString = iprot.readBinary();
         struct.setFilterStringIsSet(true);
       }
+      if (incoming.get(5)) {
+        {
+          org.apache.thrift.protocol.TMap _map22 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+          struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map22.size);
+          for (int _i23 = 0; _i23 < _map22.size; ++_i23)
+          {
+            ByteBuffer _key24; // required
+            ByteBuffer _val25; // optional
+            _key24 = iprot.readBinary();
+            _val25 = iprot.readBinary();
+            struct.attributes.put(_key24, _val25);
+          }
+        }
+        struct.setAttributesIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
index 45b6e64cfafb..9ee49623a75f 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
@@ -3702,14 +3702,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_args st
             case 2: // GETS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list48 = iprot.readListBegin();
-                  struct.gets = new ArrayList<TGet>(_list48.size);
-                  for (int _i49 = 0; _i49 < _list48.size; ++_i49)
+                  org.apache.thrift.protocol.TList _list88 = iprot.readListBegin();
+                  struct.gets = new ArrayList<TGet>(_list88.size);
+                  for (int _i89 = 0; _i89 < _list88.size; ++_i89)
                   {
-                    TGet _elem50; // required
-                    _elem50 = new TGet();
-                    _elem50.read(iprot);
-                    struct.gets.add(_elem50);
+                    TGet _elem90; // required
+                    _elem90 = new TGet();
+                    _elem90.read(iprot);
+                    struct.gets.add(_elem90);
                   }
                   iprot.readListEnd();
                 }
@@ -3742,9 +3742,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_args s
           oprot.writeFieldBegin(GETS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.gets.size()));
-            for (TGet _iter51 : struct.gets)
+            for (TGet _iter91 : struct.gets)
             {
-              _iter51.write(oprot);
+              _iter91.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -3770,9 +3770,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_args st
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.gets.size());
-          for (TGet _iter52 : struct.gets)
+          for (TGet _iter92 : struct.gets)
           {
-            _iter52.write(oprot);
+            _iter92.write(oprot);
           }
         }
       }
@@ -3783,14 +3783,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_args str
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list53 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.gets = new ArrayList<TGet>(_list53.size);
-          for (int _i54 = 0; _i54 < _list53.size; ++_i54)
+          org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.gets = new ArrayList<TGet>(_list93.size);
+          for (int _i94 = 0; _i94 < _list93.size; ++_i94)
           {
-            TGet _elem55; // required
-            _elem55 = new TGet();
-            _elem55.read(iprot);
-            struct.gets.add(_elem55);
+            TGet _elem95; // required
+            _elem95 = new TGet();
+            _elem95.read(iprot);
+            struct.gets.add(_elem95);
           }
         }
         struct.setGetsIsSet(true);
@@ -4184,14 +4184,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_result
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list56 = iprot.readListBegin();
-                  struct.success = new ArrayList<TResult>(_list56.size);
-                  for (int _i57 = 0; _i57 < _list56.size; ++_i57)
+                  org.apache.thrift.protocol.TList _list96 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list96.size);
+                  for (int _i97 = 0; _i97 < _list96.size; ++_i97)
                   {
-                    TResult _elem58; // required
-                    _elem58 = new TResult();
-                    _elem58.read(iprot);
-                    struct.success.add(_elem58);
+                    TResult _elem98; // required
+                    _elem98 = new TResult();
+                    _elem98.read(iprot);
+                    struct.success.add(_elem98);
                   }
                   iprot.readListEnd();
                 }
@@ -4228,9 +4228,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_result
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TResult _iter59 : struct.success)
+            for (TResult _iter99 : struct.success)
             {
-              _iter59.write(oprot);
+              _iter99.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -4269,9 +4269,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_result
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TResult _iter60 : struct.success)
+            for (TResult _iter100 : struct.success)
             {
-              _iter60.write(oprot);
+              _iter100.write(oprot);
             }
           }
         }
@@ -4286,14 +4286,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_result s
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list61 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TResult>(_list61.size);
-            for (int _i62 = 0; _i62 < _list61.size; ++_i62)
+            org.apache.thrift.protocol.TList _list101 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list101.size);
+            for (int _i102 = 0; _i102 < _list101.size; ++_i102)
             {
-              TResult _elem63; // required
-              _elem63 = new TResult();
-              _elem63.read(iprot);
-              struct.success.add(_elem63);
+              TResult _elem103; // required
+              _elem103 = new TResult();
+              _elem103.read(iprot);
+              struct.success.add(_elem103);
             }
           }
           struct.setSuccessIsSet(true);
@@ -7000,14 +7000,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, putMultiple_args st
             case 2: // PUTS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list64 = iprot.readListBegin();
-                  struct.puts = new ArrayList<TPut>(_list64.size);
-                  for (int _i65 = 0; _i65 < _list64.size; ++_i65)
+                  org.apache.thrift.protocol.TList _list104 = iprot.readListBegin();
+                  struct.puts = new ArrayList<TPut>(_list104.size);
+                  for (int _i105 = 0; _i105 < _list104.size; ++_i105)
                   {
-                    TPut _elem66; // required
-                    _elem66 = new TPut();
-                    _elem66.read(iprot);
-                    struct.puts.add(_elem66);
+                    TPut _elem106; // required
+                    _elem106 = new TPut();
+                    _elem106.read(iprot);
+                    struct.puts.add(_elem106);
                   }
                   iprot.readListEnd();
                 }
@@ -7040,9 +7040,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, putMultiple_args s
           oprot.writeFieldBegin(PUTS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.puts.size()));
-            for (TPut _iter67 : struct.puts)
+            for (TPut _iter107 : struct.puts)
             {
-              _iter67.write(oprot);
+              _iter107.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -7068,9 +7068,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, putMultiple_args st
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.puts.size());
-          for (TPut _iter68 : struct.puts)
+          for (TPut _iter108 : struct.puts)
           {
-            _iter68.write(oprot);
+            _iter108.write(oprot);
           }
         }
       }
@@ -7081,14 +7081,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, putMultiple_args str
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list69 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.puts = new ArrayList<TPut>(_list69.size);
-          for (int _i70 = 0; _i70 < _list69.size; ++_i70)
+          org.apache.thrift.protocol.TList _list109 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.puts = new ArrayList<TPut>(_list109.size);
+          for (int _i110 = 0; _i110 < _list109.size; ++_i110)
           {
-            TPut _elem71; // required
-            _elem71 = new TPut();
-            _elem71.read(iprot);
-            struct.puts.add(_elem71);
+            TPut _elem111; // required
+            _elem111 = new TPut();
+            _elem111.read(iprot);
+            struct.puts.add(_elem111);
           }
         }
         struct.setPutsIsSet(true);
@@ -8720,14 +8720,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_args
             case 2: // DELETES
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list72 = iprot.readListBegin();
-                  struct.deletes = new ArrayList<TDelete>(_list72.size);
-                  for (int _i73 = 0; _i73 < _list72.size; ++_i73)
+                  org.apache.thrift.protocol.TList _list112 = iprot.readListBegin();
+                  struct.deletes = new ArrayList<TDelete>(_list112.size);
+                  for (int _i113 = 0; _i113 < _list112.size; ++_i113)
                   {
-                    TDelete _elem74; // required
-                    _elem74 = new TDelete();
-                    _elem74.read(iprot);
-                    struct.deletes.add(_elem74);
+                    TDelete _elem114; // required
+                    _elem114 = new TDelete();
+                    _elem114.read(iprot);
+                    struct.deletes.add(_elem114);
                   }
                   iprot.readListEnd();
                 }
@@ -8760,9 +8760,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_arg
           oprot.writeFieldBegin(DELETES_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.deletes.size()));
-            for (TDelete _iter75 : struct.deletes)
+            for (TDelete _iter115 : struct.deletes)
             {
-              _iter75.write(oprot);
+              _iter115.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -8788,9 +8788,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.deletes.size());
-          for (TDelete _iter76 : struct.deletes)
+          for (TDelete _iter116 : struct.deletes)
           {
-            _iter76.write(oprot);
+            _iter116.write(oprot);
           }
         }
       }
@@ -8801,14 +8801,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list77 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.deletes = new ArrayList<TDelete>(_list77.size);
-          for (int _i78 = 0; _i78 < _list77.size; ++_i78)
+          org.apache.thrift.protocol.TList _list117 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.deletes = new ArrayList<TDelete>(_list117.size);
+          for (int _i118 = 0; _i118 < _list117.size; ++_i118)
           {
-            TDelete _elem79; // required
-            _elem79 = new TDelete();
-            _elem79.read(iprot);
-            struct.deletes.add(_elem79);
+            TDelete _elem119; // required
+            _elem119 = new TDelete();
+            _elem119.read(iprot);
+            struct.deletes.add(_elem119);
           }
         }
         struct.setDeletesIsSet(true);
@@ -9202,14 +9202,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list80 = iprot.readListBegin();
-                  struct.success = new ArrayList<TDelete>(_list80.size);
-                  for (int _i81 = 0; _i81 < _list80.size; ++_i81)
+                  org.apache.thrift.protocol.TList _list120 = iprot.readListBegin();
+                  struct.success = new ArrayList<TDelete>(_list120.size);
+                  for (int _i121 = 0; _i121 < _list120.size; ++_i121)
                   {
-                    TDelete _elem82; // required
-                    _elem82 = new TDelete();
-                    _elem82.read(iprot);
-                    struct.success.add(_elem82);
+                    TDelete _elem122; // required
+                    _elem122 = new TDelete();
+                    _elem122.read(iprot);
+                    struct.success.add(_elem122);
                   }
                   iprot.readListEnd();
                 }
@@ -9246,9 +9246,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TDelete _iter83 : struct.success)
+            for (TDelete _iter123 : struct.success)
             {
-              _iter83.write(oprot);
+              _iter123.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -9287,9 +9287,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TDelete _iter84 : struct.success)
+            for (TDelete _iter124 : struct.success)
             {
-              _iter84.write(oprot);
+              _iter124.write(oprot);
             }
           }
         }
@@ -9304,14 +9304,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resul
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list85 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TDelete>(_list85.size);
-            for (int _i86 = 0; _i86 < _list85.size; ++_i86)
+            org.apache.thrift.protocol.TList _list125 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TDelete>(_list125.size);
+            for (int _i126 = 0; _i126 < _list125.size; ++_i126)
             {
-              TDelete _elem87; // required
-              _elem87 = new TDelete();
-              _elem87.read(iprot);
-              struct.success.add(_elem87);
+              TDelete _elem127; // required
+              _elem127 = new TDelete();
+              _elem127.read(iprot);
+              struct.success.add(_elem127);
             }
           }
           struct.setSuccessIsSet(true);
@@ -13562,14 +13562,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerRows_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list88 = iprot.readListBegin();
-                  struct.success = new ArrayList<TResult>(_list88.size);
-                  for (int _i89 = 0; _i89 < _list88.size; ++_i89)
+                  org.apache.thrift.protocol.TList _list128 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list128.size);
+                  for (int _i129 = 0; _i129 < _list128.size; ++_i129)
                   {
-                    TResult _elem90; // required
-                    _elem90 = new TResult();
-                    _elem90.read(iprot);
-                    struct.success.add(_elem90);
+                    TResult _elem130; // required
+                    _elem130 = new TResult();
+                    _elem130.read(iprot);
+                    struct.success.add(_elem130);
                   }
                   iprot.readListEnd();
                 }
@@ -13615,9 +13615,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerRows_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TResult _iter91 : struct.success)
+            for (TResult _iter131 : struct.success)
             {
-              _iter91.write(oprot);
+              _iter131.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -13664,9 +13664,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TResult _iter92 : struct.success)
+            for (TResult _iter132 : struct.success)
             {
-              _iter92.write(oprot);
+              _iter132.write(oprot);
             }
           }
         }
@@ -13684,14 +13684,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resul
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TResult>(_list93.size);
-            for (int _i94 = 0; _i94 < _list93.size; ++_i94)
+            org.apache.thrift.protocol.TList _list133 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list133.size);
+            for (int _i134 = 0; _i134 < _list133.size; ++_i134)
             {
-              TResult _elem95; // required
-              _elem95 = new TResult();
-              _elem95.read(iprot);
-              struct.success.add(_elem95);
+              TResult _elem135; // required
+              _elem135 = new TResult();
+              _elem135.read(iprot);
+              struct.success.add(_elem135);
             }
           }
           struct.setSuccessIsSet(true);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
index 33bd4458c18c..b35d134ff316 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
@@ -45,6 +45,7 @@ public class TPut implements org.apache.thrift.TBase<TPut, TPut._Fields>, java.i
   private static final org.apache.thrift.protocol.TField COLUMN_VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("columnValues", org.apache.thrift.protocol.TType.LIST, (short)2);
   private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
   private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)4);
+  private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -56,13 +57,15 @@ public class TPut implements org.apache.thrift.TBase<TPut, TPut._Fields>, java.i
   public List<TColumnValue> columnValues; // required
   public long timestamp; // optional
   public boolean writeToWal; // optional
+  public Map<ByteBuffer,ByteBuffer> attributes; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     ROW((short)1, "row"),
     COLUMN_VALUES((short)2, "columnValues"),
     TIMESTAMP((short)3, "timestamp"),
-    WRITE_TO_WAL((short)4, "writeToWal");
+    WRITE_TO_WAL((short)4, "writeToWal"),
+    ATTRIBUTES((short)5, "attributes");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -85,6 +88,8 @@ public static _Fields findByThriftId(int fieldId) {
           return TIMESTAMP;
         case 4: // WRITE_TO_WAL
           return WRITE_TO_WAL;
+        case 5: // ATTRIBUTES
+          return ATTRIBUTES;
         default:
           return null;
       }
@@ -128,7 +133,7 @@ public String getFieldName() {
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __WRITETOWAL_ISSET_ID = 1;
   private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.TIMESTAMP,_Fields.WRITE_TO_WAL};
+  private _Fields optionals[] = {_Fields.TIMESTAMP,_Fields.WRITE_TO_WAL,_Fields.ATTRIBUTES};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -141,6 +146,10 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
     tmpMap.put(_Fields.WRITE_TO_WAL, new org.apache.thrift.meta_data.FieldMetaData("writeToWal", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
+    tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true), 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TPut.class, metaDataMap);
   }
@@ -178,6 +187,23 @@ public TPut(TPut other) {
     }
     this.timestamp = other.timestamp;
     this.writeToWal = other.writeToWal;
+    if (other.isSetAttributes()) {
+      Map<ByteBuffer,ByteBuffer> __this__attributes = new HashMap<ByteBuffer,ByteBuffer>();
+      for (Map.Entry<ByteBuffer, ByteBuffer> other_element : other.attributes.entrySet()) {
+
+        ByteBuffer other_element_key = other_element.getKey();
+        ByteBuffer other_element_value = other_element.getValue();
+
+        ByteBuffer __this__attributes_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
+;
+
+        ByteBuffer __this__attributes_copy_value = org.apache.thrift.TBaseHelper.copyBinary(other_element_value);
+;
+
+        __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value);
+      }
+      this.attributes = __this__attributes;
+    }
   }
 
   public TPut deepCopy() {
@@ -192,6 +218,7 @@ public void clear() {
     this.timestamp = 0;
     this.writeToWal = true;
 
+    this.attributes = null;
   }
 
   public byte[] getRow() {
@@ -313,6 +340,41 @@ public void setWriteToWalIsSet(boolean value) {
     __isset_bit_vector.set(__WRITETOWAL_ISSET_ID, value);
   }
 
+  public int getAttributesSize() {
+    return (this.attributes == null) ? 0 : this.attributes.size();
+  }
+
+  public void putToAttributes(ByteBuffer key, ByteBuffer val) {
+    if (this.attributes == null) {
+      this.attributes = new HashMap<ByteBuffer,ByteBuffer>();
+    }
+    this.attributes.put(key, val);
+  }
+
+  public Map<ByteBuffer,ByteBuffer> getAttributes() {
+    return this.attributes;
+  }
+
+  public TPut setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
+    this.attributes = attributes;
+    return this;
+  }
+
+  public void unsetAttributes() {
+    this.attributes = null;
+  }
+
+  /** Returns true if field attributes is set (has been assigned a value) and false otherwise */
+  public boolean isSetAttributes() {
+    return this.attributes != null;
+  }
+
+  public void setAttributesIsSet(boolean value) {
+    if (!value) {
+      this.attributes = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -347,6 +409,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case ATTRIBUTES:
+      if (value == null) {
+        unsetAttributes();
+      } else {
+        setAttributes((Map<ByteBuffer,ByteBuffer>)value);
+      }
+      break;
+
     }
   }
 
@@ -364,6 +434,9 @@ public Object getFieldValue(_Fields field) {
     case WRITE_TO_WAL:
       return Boolean.valueOf(isWriteToWal());
 
+    case ATTRIBUTES:
+      return getAttributes();
+
     }
     throw new IllegalStateException();
   }
@@ -383,6 +456,8 @@ public boolean isSet(_Fields field) {
       return isSetTimestamp();
     case WRITE_TO_WAL:
       return isSetWriteToWal();
+    case ATTRIBUTES:
+      return isSetAttributes();
     }
     throw new IllegalStateException();
   }
@@ -436,6 +511,15 @@ public boolean equals(TPut that) {
         return false;
     }
 
+    boolean this_present_attributes = true && this.isSetAttributes();
+    boolean that_present_attributes = true && that.isSetAttributes();
+    if (this_present_attributes || that_present_attributes) {
+      if (!(this_present_attributes && that_present_attributes))
+        return false;
+      if (!this.attributes.equals(that.attributes))
+        return false;
+    }
+
     return true;
   }
 
@@ -492,6 +576,16 @@ public int compareTo(TPut other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetAttributes()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -539,6 +633,16 @@ public String toString() {
       sb.append(this.writeToWal);
       first = false;
     }
+    if (isSetAttributes()) {
+      if (!first) sb.append(", ");
+      sb.append("attributes:");
+      if (this.attributes == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.attributes);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -600,14 +704,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TPut struct) throws
           case 2: // COLUMN_VALUES
             if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list16 = iprot.readListBegin();
-                struct.columnValues = new ArrayList<TColumnValue>(_list16.size);
-                for (int _i17 = 0; _i17 < _list16.size; ++_i17)
+                org.apache.thrift.protocol.TList _list26 = iprot.readListBegin();
+                struct.columnValues = new ArrayList<TColumnValue>(_list26.size);
+                for (int _i27 = 0; _i27 < _list26.size; ++_i27)
                 {
-                  TColumnValue _elem18; // required
-                  _elem18 = new TColumnValue();
-                  _elem18.read(iprot);
-                  struct.columnValues.add(_elem18);
+                  TColumnValue _elem28; // required
+                  _elem28 = new TColumnValue();
+                  _elem28.read(iprot);
+                  struct.columnValues.add(_elem28);
                 }
                 iprot.readListEnd();
               }
@@ -632,6 +736,26 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TPut struct) throws
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 5: // ATTRIBUTES
+            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+              {
+                org.apache.thrift.protocol.TMap _map29 = iprot.readMapBegin();
+                struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map29.size);
+                for (int _i30 = 0; _i30 < _map29.size; ++_i30)
+                {
+                  ByteBuffer _key31; // required
+                  ByteBuffer _val32; // optional
+                  _key31 = iprot.readBinary();
+                  _val32 = iprot.readBinary();
+                  struct.attributes.put(_key31, _val32);
+                }
+                iprot.readMapEnd();
+              }
+              struct.setAttributesIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -656,9 +780,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TPut struct) throw
         oprot.writeFieldBegin(COLUMN_VALUES_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columnValues.size()));
-          for (TColumnValue _iter19 : struct.columnValues)
+          for (TColumnValue _iter33 : struct.columnValues)
           {
-            _iter19.write(oprot);
+            _iter33.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -674,6 +798,21 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TPut struct) throw
         oprot.writeBool(struct.writeToWal);
         oprot.writeFieldEnd();
       }
+      if (struct.attributes != null) {
+        if (struct.isSetAttributes()) {
+          oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
+          {
+            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter34 : struct.attributes.entrySet())
+            {
+              oprot.writeBinary(_iter34.getKey());
+              oprot.writeBinary(_iter34.getValue());
+            }
+            oprot.writeMapEnd();
+          }
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -694,9 +833,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
       oprot.writeBinary(struct.row);
       {
         oprot.writeI32(struct.columnValues.size());
-        for (TColumnValue _iter20 : struct.columnValues)
+        for (TColumnValue _iter35 : struct.columnValues)
         {
-          _iter20.write(oprot);
+          _iter35.write(oprot);
         }
       }
       BitSet optionals = new BitSet();
@@ -706,13 +845,26 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
       if (struct.isSetWriteToWal()) {
         optionals.set(1);
       }
-      oprot.writeBitSet(optionals, 2);
+      if (struct.isSetAttributes()) {
+        optionals.set(2);
+      }
+      oprot.writeBitSet(optionals, 3);
       if (struct.isSetTimestamp()) {
         oprot.writeI64(struct.timestamp);
       }
       if (struct.isSetWriteToWal()) {
         oprot.writeBool(struct.writeToWal);
       }
+      if (struct.isSetAttributes()) {
+        {
+          oprot.writeI32(struct.attributes.size());
+          for (Map.Entry<ByteBuffer, ByteBuffer> _iter36 : struct.attributes.entrySet())
+          {
+            oprot.writeBinary(_iter36.getKey());
+            oprot.writeBinary(_iter36.getValue());
+          }
+        }
+      }
     }
 
     @Override
@@ -721,18 +873,18 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
       {
-        org.apache.thrift.protocol.TList _list21 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.columnValues = new ArrayList<TColumnValue>(_list21.size);
-        for (int _i22 = 0; _i22 < _list21.size; ++_i22)
+        org.apache.thrift.protocol.TList _list37 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+        struct.columnValues = new ArrayList<TColumnValue>(_list37.size);
+        for (int _i38 = 0; _i38 < _list37.size; ++_i38)
         {
-          TColumnValue _elem23; // required
-          _elem23 = new TColumnValue();
-          _elem23.read(iprot);
-          struct.columnValues.add(_elem23);
+          TColumnValue _elem39; // required
+          _elem39 = new TColumnValue();
+          _elem39.read(iprot);
+          struct.columnValues.add(_elem39);
         }
       }
       struct.setColumnValuesIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
+      BitSet incoming = iprot.readBitSet(3);
       if (incoming.get(0)) {
         struct.timestamp = iprot.readI64();
         struct.setTimestampIsSet(true);
@@ -741,6 +893,21 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
         struct.writeToWal = iprot.readBool();
         struct.setWriteToWalIsSet(true);
       }
+      if (incoming.get(2)) {
+        {
+          org.apache.thrift.protocol.TMap _map40 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+          struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map40.size);
+          for (int _i41 = 0; _i41 < _map40.size; ++_i41)
+          {
+            ByteBuffer _key42; // required
+            ByteBuffer _val43; // optional
+            _key42 = iprot.readBinary();
+            _val43 = iprot.readBinary();
+            struct.attributes.put(_key42, _val43);
+          }
+        }
+        struct.setAttributesIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
index 1935f01dcce4..4c58a8c84f66 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
@@ -42,6 +42,7 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
   private static final org.apache.thrift.protocol.TField TIME_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeRange", org.apache.thrift.protocol.TType.STRUCT, (short)6);
   private static final org.apache.thrift.protocol.TField FILTER_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("filterString", org.apache.thrift.protocol.TType.STRING, (short)7);
   private static final org.apache.thrift.protocol.TField BATCH_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("batchSize", org.apache.thrift.protocol.TType.I32, (short)8);
+  private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)9);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -57,6 +58,7 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
   public TTimeRange timeRange; // optional
   public ByteBuffer filterString; // optional
   public int batchSize; // optional
+  public Map<ByteBuffer,ByteBuffer> attributes; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -67,7 +69,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     MAX_VERSIONS((short)5, "maxVersions"),
     TIME_RANGE((short)6, "timeRange"),
     FILTER_STRING((short)7, "filterString"),
-    BATCH_SIZE((short)8, "batchSize");
+    BATCH_SIZE((short)8, "batchSize"),
+    ATTRIBUTES((short)9, "attributes");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -98,6 +101,8 @@ public static _Fields findByThriftId(int fieldId) {
           return FILTER_STRING;
         case 8: // BATCH_SIZE
           return BATCH_SIZE;
+        case 9: // ATTRIBUTES
+          return ATTRIBUTES;
         default:
           return null;
       }
@@ -142,7 +147,7 @@ public String getFieldName() {
   private static final int __MAXVERSIONS_ISSET_ID = 1;
   private static final int __BATCHSIZE_ISSET_ID = 2;
   private BitSet __isset_bit_vector = new BitSet(3);
-  private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE};
+  private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE,_Fields.ATTRIBUTES};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -163,6 +168,10 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
     tmpMap.put(_Fields.BATCH_SIZE, new org.apache.thrift.meta_data.FieldMetaData("batchSize", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+    tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true), 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap);
   }
@@ -203,6 +212,23 @@ public TScan(TScan other) {
 ;
     }
     this.batchSize = other.batchSize;
+    if (other.isSetAttributes()) {
+      Map<ByteBuffer,ByteBuffer> __this__attributes = new HashMap<ByteBuffer,ByteBuffer>();
+      for (Map.Entry<ByteBuffer, ByteBuffer> other_element : other.attributes.entrySet()) {
+
+        ByteBuffer other_element_key = other_element.getKey();
+        ByteBuffer other_element_value = other_element.getValue();
+
+        ByteBuffer __this__attributes_copy_key = org.apache.thrift.TBaseHelper.copyBinary(other_element_key);
+;
+
+        ByteBuffer __this__attributes_copy_value = org.apache.thrift.TBaseHelper.copyBinary(other_element_value);
+;
+
+        __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value);
+      }
+      this.attributes = __this__attributes;
+    }
   }
 
   public TScan deepCopy() {
@@ -222,6 +248,7 @@ public void clear() {
     this.filterString = null;
     setBatchSizeIsSet(false);
     this.batchSize = 0;
+    this.attributes = null;
   }
 
   public byte[] getStartRow() {
@@ -458,6 +485,41 @@ public void setBatchSizeIsSet(boolean value) {
     __isset_bit_vector.set(__BATCHSIZE_ISSET_ID, value);
   }
 
+  public int getAttributesSize() {
+    return (this.attributes == null) ? 0 : this.attributes.size();
+  }
+
+  public void putToAttributes(ByteBuffer key, ByteBuffer val) {
+    if (this.attributes == null) {
+      this.attributes = new HashMap<ByteBuffer,ByteBuffer>();
+    }
+    this.attributes.put(key, val);
+  }
+
+  public Map<ByteBuffer,ByteBuffer> getAttributes() {
+    return this.attributes;
+  }
+
+  public TScan setAttributes(Map<ByteBuffer,ByteBuffer> attributes) {
+    this.attributes = attributes;
+    return this;
+  }
+
+  public void unsetAttributes() {
+    this.attributes = null;
+  }
+
+  /** Returns true if field attributes is set (has been assigned a value) and false otherwise */
+  public boolean isSetAttributes() {
+    return this.attributes != null;
+  }
+
+  public void setAttributesIsSet(boolean value) {
+    if (!value) {
+      this.attributes = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case START_ROW:
@@ -524,6 +586,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case ATTRIBUTES:
+      if (value == null) {
+        unsetAttributes();
+      } else {
+        setAttributes((Map<ByteBuffer,ByteBuffer>)value);
+      }
+      break;
+
     }
   }
 
@@ -553,6 +623,9 @@ public Object getFieldValue(_Fields field) {
     case BATCH_SIZE:
       return Integer.valueOf(getBatchSize());
 
+    case ATTRIBUTES:
+      return getAttributes();
+
     }
     throw new IllegalStateException();
   }
@@ -580,6 +653,8 @@ public boolean isSet(_Fields field) {
       return isSetFilterString();
     case BATCH_SIZE:
       return isSetBatchSize();
+    case ATTRIBUTES:
+      return isSetAttributes();
     }
     throw new IllegalStateException();
   }
@@ -669,6 +744,15 @@ public boolean equals(TScan that) {
         return false;
     }
 
+    boolean this_present_attributes = true && this.isSetAttributes();
+    boolean that_present_attributes = true && that.isSetAttributes();
+    if (this_present_attributes || that_present_attributes) {
+      if (!(this_present_attributes && that_present_attributes))
+        return false;
+      if (!this.attributes.equals(that.attributes))
+        return false;
+    }
+
     return true;
   }
 
@@ -765,6 +849,16 @@ public int compareTo(TScan other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetAttributes()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -852,6 +946,16 @@ public String toString() {
       sb.append(this.batchSize);
       first = false;
     }
+    if (isSetAttributes()) {
+      if (!first) sb.append(", ");
+      sb.append("attributes:");
+      if (this.attributes == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.attributes);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -915,14 +1019,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
           case 3: // COLUMNS
             if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list40 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumn>(_list40.size);
-                for (int _i41 = 0; _i41 < _list40.size; ++_i41)
+                org.apache.thrift.protocol.TList _list70 = iprot.readListBegin();
+                struct.columns = new ArrayList<TColumn>(_list70.size);
+                for (int _i71 = 0; _i71 < _list70.size; ++_i71)
                 {
-                  TColumn _elem42; // required
-                  _elem42 = new TColumn();
-                  _elem42.read(iprot);
-                  struct.columns.add(_elem42);
+                  TColumn _elem72; // required
+                  _elem72 = new TColumn();
+                  _elem72.read(iprot);
+                  struct.columns.add(_elem72);
                 }
                 iprot.readListEnd();
               }
@@ -972,6 +1076,26 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 9: // ATTRIBUTES
+            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+              {
+                org.apache.thrift.protocol.TMap _map73 = iprot.readMapBegin();
+                struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map73.size);
+                for (int _i74 = 0; _i74 < _map73.size; ++_i74)
+                {
+                  ByteBuffer _key75; // required
+                  ByteBuffer _val76; // optional
+                  _key75 = iprot.readBinary();
+                  _val76 = iprot.readBinary();
+                  struct.attributes.put(_key75, _val76);
+                }
+                iprot.readMapEnd();
+              }
+              struct.setAttributesIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -1006,9 +1130,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) thro
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-            for (TColumn _iter43 : struct.columns)
+            for (TColumn _iter77 : struct.columns)
             {
-              _iter43.write(oprot);
+              _iter77.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -1044,6 +1168,21 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) thro
         oprot.writeI32(struct.batchSize);
         oprot.writeFieldEnd();
       }
+      if (struct.attributes != null) {
+        if (struct.isSetAttributes()) {
+          oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
+          {
+            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter78 : struct.attributes.entrySet())
+            {
+              oprot.writeBinary(_iter78.getKey());
+              oprot.writeBinary(_iter78.getValue());
+            }
+            oprot.writeMapEnd();
+          }
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -1086,7 +1225,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetBatchSize()) {
         optionals.set(7);
       }
-      oprot.writeBitSet(optionals, 8);
+      if (struct.isSetAttributes()) {
+        optionals.set(8);
+      }
+      oprot.writeBitSet(optionals, 9);
       if (struct.isSetStartRow()) {
         oprot.writeBinary(struct.startRow);
       }
@@ -1096,9 +1238,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
-          for (TColumn _iter44 : struct.columns)
+          for (TColumn _iter79 : struct.columns)
           {
-            _iter44.write(oprot);
+            _iter79.write(oprot);
           }
         }
       }
@@ -1117,12 +1259,22 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetBatchSize()) {
         oprot.writeI32(struct.batchSize);
       }
+      if (struct.isSetAttributes()) {
+        {
+          oprot.writeI32(struct.attributes.size());
+          for (Map.Entry<ByteBuffer, ByteBuffer> _iter80 : struct.attributes.entrySet())
+          {
+            oprot.writeBinary(_iter80.getKey());
+            oprot.writeBinary(_iter80.getValue());
+          }
+        }
+      }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(8);
+      BitSet incoming = iprot.readBitSet(9);
       if (incoming.get(0)) {
         struct.startRow = iprot.readBinary();
         struct.setStartRowIsSet(true);
@@ -1133,14 +1285,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
       }
       if (incoming.get(2)) {
         {
-          org.apache.thrift.protocol.TList _list45 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.columns = new ArrayList<TColumn>(_list45.size);
-          for (int _i46 = 0; _i46 < _list45.size; ++_i46)
+          org.apache.thrift.protocol.TList _list81 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.columns = new ArrayList<TColumn>(_list81.size);
+          for (int _i82 = 0; _i82 < _list81.size; ++_i82)
           {
-            TColumn _elem47; // required
-            _elem47 = new TColumn();
-            _elem47.read(iprot);
-            struct.columns.add(_elem47);
+            TColumn _elem83; // required
+            _elem83 = new TColumn();
+            _elem83.read(iprot);
+            struct.columns.add(_elem83);
           }
         }
         struct.setColumnsIsSet(true);
@@ -1166,6 +1318,21 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
         struct.batchSize = iprot.readI32();
         struct.setBatchSizeIsSet(true);
       }
+      if (incoming.get(8)) {
+        {
+          org.apache.thrift.protocol.TMap _map84 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+          struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map84.size);
+          for (int _i85 = 0; _i85 < _map84.size; ++_i85)
+          {
+            ByteBuffer _key86; // required
+            ByteBuffer _val87; // optional
+            _key86 = iprot.readBinary();
+            _val87 = iprot.readBinary();
+            struct.attributes.put(_key86, _val87);
+          }
+        }
+        struct.setAttributesIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
index 5be6d12285ac..b4dd11242bc9 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
@@ -98,7 +98,8 @@ struct TGet {
   4: optional TTimeRange timeRange,
 
   5: optional i32 maxVersions,
-  6: optional binary filterString
+  6: optional binary filterString,
+  7: optional map<binary, binary> attributes
 }
 
 /**
@@ -116,7 +117,8 @@ struct TPut {
   1: required binary row,
   2: required list<TColumnValue> columnValues
   3: optional i64 timestamp,
-  4: optional bool writeToWal = 1
+  4: optional bool writeToWal = 1,
+  5: optional map<binary, binary> attributes
 }
 
 /**
@@ -147,7 +149,8 @@ struct TDelete {
   2: optional list<TColumn> columns,
   3: optional i64 timestamp,
   4: optional TDeleteType deleteType = 1,
-  5: optional bool writeToWal = 1
+  5: optional bool writeToWal = 1,
+  6: optional map<binary, binary> attributes
 }
 
 /**
@@ -174,7 +177,8 @@ struct TScan {
   5: optional i32 maxVersions=1,
   6: optional TTimeRange timeRange,
   7: optional binary filterString,
-  8: optional i32 batchSize
+  8: optional i32 batchSize,
+  9: optional map<binary, binary> attributes
 }
 
 //
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 589bafb58b54..22b7be69802e 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -26,6 +26,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -35,6 +36,10 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
@@ -63,6 +68,10 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.getFromThrift;
+import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.putFromThrift;
+import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift;
+import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deleteFromThrift;
 import static org.junit.Assert.*;
 import static java.nio.ByteBuffer.wrap;
 
@@ -689,6 +698,37 @@ public void testMetrics() throws Exception {
     verifyMetrics(metrics, "put_num_ops", 1);
     verifyMetrics(metrics, "exists_num_ops", 2);
   }
+
+  @Test
+  public void testAttribute() throws Exception {
+    byte[] rowName = "testAttribute".getBytes();
+    byte[] attributeKey = "attribute1".getBytes();
+    byte[] attributeValue = "value1".getBytes();
+    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<ByteBuffer, ByteBuffer>();
+    attributes.put(wrap(attributeKey), wrap(attributeValue));
+
+    TGet tGet = new TGet(wrap(rowName));
+    tGet.setAttributes(attributes);
+    Get get = getFromThrift(tGet);
+    assertArrayEquals(get.getAttribute("attribute1"), attributeValue);
+
+    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+    TPut tPut = new TPut(wrap(rowName) , columnValues);
+    tPut.setAttributes(attributes);
+    Put put = putFromThrift(tPut);
+    assertArrayEquals(put.getAttribute("attribute1"), attributeValue);
+
+    TScan tScan = new TScan();
+    tScan.setAttributes(attributes);
+    Scan scan = scanFromThrift(tScan);
+    assertArrayEquals(scan.getAttribute("attribute1"), attributeValue);
+
+    TDelete tDelete = new TDelete(wrap(rowName));
+    tDelete.setAttributes(attributes);
+    Delete delete = deleteFromThrift(tDelete);
+    assertArrayEquals(delete.getAttribute("attribute1"), attributeValue);
+  }
  
   private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
     setupMetricsContext();

From eae147c87ab3fdfc1061040dbefdc7f329c324b6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 12 Jul 2013 17:26:45 +0000
Subject: [PATCH 1055/1540] HBASE-8934 Fix bunch of flaky tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1502628 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     | 24 ++++++---
 .../hbase/backup/TestHFileArchiving.java      | 39 +++++++-------
 .../apache/hadoop/hbase/client/TestAdmin.java | 52 +++++++++++++------
 .../hbase/client/TestScannerTimeout.java      | 17 ++++--
 ...nServerCoprocessorExceptionWithRemove.java | 13 +++--
 .../io/encoding/TestChangingEncoding.java     |  6 +--
 .../regionserver/TestCompactionState.java     | 12 ++---
 .../hbase/util/TestSizeBasedThrottler.java    |  6 +--
 8 files changed, 106 insertions(+), 63 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 1214045b5664..9332e89085d7 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -88,8 +88,8 @@
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.ZooKeeper;
 
 /**
@@ -842,6 +842,8 @@ public HTable createTable(byte[] tableName, byte[][] families,
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(getConfiguration(), tableName);
   }
 
@@ -861,6 +863,8 @@ public HTable createTable(byte[] tableName, byte[][] families,
       desc.addFamily(new HColumnDescriptor(family));
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(c, tableName);
   }
 
@@ -883,6 +887,8 @@ public HTable createTable(byte[] tableName, byte[][] families,
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(c, tableName);
   }
 
@@ -917,6 +923,8 @@ public HTable createTable(byte[] tableName, byte[][] families,
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(new Configuration(getConfiguration()), tableName);
   }
 
@@ -938,6 +946,8 @@ public HTable createTable(byte[] tableName, byte[][] families,
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(new Configuration(getConfiguration()), tableName);
   }
 
@@ -961,6 +971,8 @@ public HTable createTable(byte[] tableName, byte[][] families,
       i++;
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(new Configuration(getConfiguration()), tableName);
   }
 
@@ -1065,7 +1077,7 @@ public int loadRegion(final HRegion r, final byte[] f)
   throws IOException {
     return loadRegion(r, f, false);
   }
-  
+
   /**
    * Load region with rows from 'aaa' to 'zzz'.
    * @param r Region
@@ -1183,7 +1195,7 @@ public int createMultiRegions(HTable table, byte[] columnFamily)
    *
    * @param table  The table to use for the data.
    * @param columnFamily  The family to insert the data into.
-   * @param cleanupFS  True if a previous region should be remove from the FS  
+   * @param cleanupFS  True if a previous region should be remove from the FS
    * @return count of regions created.
    * @throws IOException When creating the regions fails.
    */
@@ -1220,7 +1232,7 @@ public int createMultiRegions(final Configuration c, final HTable table,
       final byte[] columnFamily, byte [][] startKeys) throws IOException {
     return createMultiRegions(c, table, columnFamily, startKeys, true);
   }
-  
+
   public int createMultiRegions(final Configuration c, final HTable table,
           final byte[] columnFamily, byte [][] startKeys, boolean cleanupFS)
   throws IOException {
@@ -2242,10 +2254,10 @@ public HRegion createTestRegion(String tableName, HColumnDescriptor hcd)
         HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
     return region;
   }
-  
+
   /**
    * Create region split keys between startkey and endKey
-   * 
+   *
    * @param startKey
    * @param endKey
    * @param numRegions the number of regions to be created. it has to be greater than 3.
diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index 7e28f6b1ef25..b5f277485902 100644
--- a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
@@ -247,9 +246,23 @@ public void testArchiveOnTableDelete() throws Exception {
     // then delete the table so the hfiles get archived
     UTIL.deleteTable(TABLE_NAME);
 
-    // then get the files in the archive directory.
+    assertArchiveFiles(fs, storeFiles, 30000);
+  }
+
+  private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
+    long end = System.currentTimeMillis() + timeout;
     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
-    List<String> archivedFiles = getAllFileNames(fs, archiveDir);
+    List<String> archivedFiles = new ArrayList<String>();
+
+    // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
+    // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
+    while (System.currentTimeMillis() < end) {
+      archivedFiles = getAllFileNames(fs, archiveDir);
+      if (archivedFiles.size() >= storeFiles.size()) {
+        break;
+      }
+    }
+
     Collections.sort(storeFiles);
     Collections.sort(archivedFiles);
 
@@ -310,23 +323,7 @@ public void testArchiveOnTableFamilyDelete() throws Exception {
     // then delete the table so the hfiles get archived
     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
 
-    // then get the files in the archive directory.
-    Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
-    List<String> archivedFiles = getAllFileNames(fs, archiveDir);
-    Collections.sort(storeFiles);
-    Collections.sort(archivedFiles);
-
-    LOG.debug("Store files:");
-    for (int i = 0; i < storeFiles.size(); i++) {
-      LOG.debug(i + " - " + storeFiles.get(i));
-    }
-    LOG.debug("Archive files:");
-    for (int i = 0; i < archivedFiles.size(); i++) {
-      LOG.debug(i + " - " + archivedFiles.get(i));
-    }
-
-    assertTrue("Archived files are missing some of the store files!",
-      archivedFiles.containsAll(storeFiles));
+    assertArchiveFiles(fs, storeFiles, 30000);
 
     UTIL.deleteTable(TABLE_NAME);
   }
@@ -423,7 +420,7 @@ private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<Stri
     return fileNames;
   }
 
-  private List<String> getRegionStoreFiles(final FileSystem fs, final Path regionDir) 
+  private List<String> getRegionStoreFiles(final FileSystem fs, final Path regionDir)
       throws IOException {
     List<String> storeFiles = getAllFileNames(fs, regionDir);
     // remove all the non-storefile named files for the region
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 6c596d3346ca..35653a8a05f2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -39,7 +39,21 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
@@ -48,12 +62,16 @@
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.wal.HLogUtilsForTests;
-import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 
@@ -263,7 +281,7 @@ public void testIsEnabledOnUnknownTable() throws Exception {
     } catch (IOException e) {
     }
   }
-  
+
   @Test
   public void testDisableAndEnableTables() throws IOException {
     final byte [] row = Bytes.toBytes("row");
@@ -604,7 +622,7 @@ public void testCreateTableNumberOfRegions() throws IOException, InterruptedExce
     assertEquals("Table should have 16 region", 16, regions.size());
     ht5.close();
   }
- 
+
   @Test
   public void testCreateTableWithRegions() throws IOException, InterruptedException {
 
@@ -776,8 +794,8 @@ public void testCreateTableWithRegions() throws IOException, InterruptedExceptio
     }
     ladmin.close();
   }
-  
-    
+
+
   @Test
   public void testCreateTableWithOnlyEmptyStartRow() throws IOException {
     byte[] tableName = Bytes.toBytes("testCreateTableWithOnlyEmptyStartRow");
@@ -791,7 +809,7 @@ public void testCreateTableWithOnlyEmptyStartRow() throws IOException {
     } catch (IllegalArgumentException e) {
     }
   }
-    
+
   @Test
   public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException {
     byte[] tableName = Bytes
@@ -1058,7 +1076,7 @@ public void testInvalidHColumnDescriptor() throws IOException {
      new HColumnDescriptor("/cfamily/name");
   }
 
-  @Test(timeout=36000)
+  @Test(timeout=300000)
   public void testEnableDisableAddColumnDeleteColumn() throws Exception {
     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
     byte [] tableName = Bytes.toBytes("testMasterAdmin");
@@ -1300,10 +1318,14 @@ public void testShouldCloseTheRegionBasedOnTheEncodedRegionName()
             .getServerName().getServerName());
       }
     }
-    Thread.sleep(1000);
-    onlineRegions = rs.getOnlineRegions();
+    boolean isInList = rs.getOnlineRegions().contains(info);
+    long timeout = System.currentTimeMillis() + 10000;
+    while ((System.currentTimeMillis() < timeout) && (isInList)) {
+      Thread.sleep(100);
+      isInList = rs.getOnlineRegions().contains(info);
+    }
     assertFalse("The region should not be present in online regions list.",
-        onlineRegions.contains(info));
+        isInList);
   }
 
   @Test
@@ -1348,7 +1370,7 @@ public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception {
     }
 
     boolean isInList = rs.getOnlineRegions().contains(info);
-    long timeout = System.currentTimeMillis() + 2000;
+    long timeout = System.currentTimeMillis() + 10000;
     while ((System.currentTimeMillis() < timeout) && (isInList)) {
       Thread.sleep(100);
       isInList = rs.getOnlineRegions().contains(info);
@@ -1634,7 +1656,7 @@ public void testGetRegion() throws Exception {
       ct.stop();
     }
   }
-  
+
   @Test
   public void testRootTableSplit() throws Exception {
     Scan s = new Scan();
@@ -1646,7 +1668,7 @@ public void testRootTableSplit() throws Exception {
     List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(HConstants.ROOT_TABLE_NAME);
     assertEquals("ROOT region should not be splitted.",1, regions.size());
   }
-  
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
index d4de8970b47f..4b7be5cec4b6 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
@@ -25,7 +25,9 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -147,7 +149,7 @@ public void test2772() throws Exception {
     LOG.info("END ************ test2772");
 
   }
-  
+
   /**
    * Test that scanner won't miss any rows if the region server it was reading
    * from failed. Before 3686, it would skip rows in the scan.
@@ -163,7 +165,14 @@ public void test3686a() throws Exception {
     scan.setCaching(SCANNER_CACHING);
     LOG.info("************ TEST3686A");
     MetaReader.fullScanMetaAndPrint(TEST_UTIL.getHBaseCluster().getMaster().getCatalogTracker());
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME);
+    // Set a very high timeout, we want to test what happens when a RS
+    // fails but the region is recovered before the lease times out.
+    // Since the RS is already created, this conf is client-side only for
+    // this new table
+    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+    conf.setInt(
+        HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, SCANNER_TIMEOUT*100);
+    HTable table = new HTable(conf, TABLE_NAME);
     LOG.info("START ************ TEST3686A---22");
 
     ResultScanner r = table.getScanner(scan);
@@ -183,7 +192,7 @@ public void test3686a() throws Exception {
     table.close();
     LOG.info("************ END TEST3686A");
   }
-  
+
   /**
    * Make sure that no rows are lost if the scanner timeout is longer on the
    * client than the server, and the scan times out on the server but not the
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
index d1916650b0bb..a857a82b5c4b 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
@@ -20,23 +20,26 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import static org.junit.Assert.*;
-
 /**
  * Tests unhandled exceptions thrown by coprocessors running on regionserver.
  * Expected result is that the master will remove the buggy coprocessor from
@@ -77,7 +80,7 @@ public static void teardownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  @Test(timeout=30000)
+  @Test(timeout=60000)
   public void testExceptionFromCoprocessorDuringPut()
       throws IOException {
     // Set watches on the zookeeper nodes for all of the regionservers in the
diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
index 77ce818bab9c..15b13cb17a45 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
@@ -67,7 +67,7 @@ public class TestChangingEncoding {
       new HBaseTestingUtility();
   private static final Configuration conf = TEST_UTIL.getConfiguration();
 
-  private static final int TIMEOUT_MS = 240000;
+  private static final int TIMEOUT_MS = 600000;
 
   private HBaseAdmin admin;
   private HColumnDescriptor hcd;
@@ -239,7 +239,7 @@ private void compactAndWait() throws IOException, InterruptedException {
     LOG.debug("Compacting table " + tableName);
     admin.majorCompact(tableName);
     HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
-    
+
     // Waiting for the compaction to start, at least .5s.
     final long maxWaitime = System.currentTimeMillis() + 500;
     boolean cont;
@@ -247,7 +247,7 @@ private void compactAndWait() throws IOException, InterruptedException {
       cont = rs.compactSplitThread.getCompactionQueueSize() == 0;
       Threads.sleep(1);
     } while (cont && System.currentTimeMillis() < maxWaitime);
-    
+
     while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
       Threads.sleep(5);
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
index 79594d730c65..96849b423a24 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
@@ -58,22 +58,22 @@ public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=600000)
   public void testMajorCompaction() throws IOException, InterruptedException {
     compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=600000)
   public void testMinorCompaction() throws IOException, InterruptedException {
     compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=600000)
   public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
     compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=600000)
   public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
     compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
   }
@@ -214,7 +214,7 @@ private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[]
     }
     return count;
   }
-  
+
   private static void loadData(final HTable ht, final byte[][] families,
       final int rows, final int flushes) throws IOException {
     List<Put> puts = new ArrayList<Put>(rows);
@@ -234,7 +234,7 @@ private static void loadData(final HTable ht, final byte[][] families,
       puts.clear();
     }
   }
-  
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
index e974b9a330c6..180ebae4150a 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java
@@ -100,7 +100,7 @@ public void testSmallIncreases(){
           15, // fail if throttler's value
               // exceeds 15
           1000, // use 1000 threads
-          500 // wait for 500ms
+          5000 // wait for 5 sec
           );
     }
   }
@@ -114,7 +114,7 @@ public void testBigIncreases() {
           4, // fail if throttler's value
              // exceeds 4
           1000, // use 1000 threads
-          500 // wait for 500ms
+          5000 // wait for 5 sec
           );
     }
   }
@@ -128,7 +128,7 @@ public void testIncreasesEqualToThreshold(){
           2, // fail if throttler's value
              // exceeds 2
           1000, // use 1000 threads
-          500 // wait for 500ms
+          5000 // wait for 5 sec
           );
     }
   }

From 6796dae06ee990b329d5974f5650b6d65ee1a530 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 12 Jul 2013 22:04:05 +0000
Subject: [PATCH 1056/1540] HBASE-8914 [0.94]
 TestRegionServerCoprocessorExceptionWithAbort is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1502709 13f79535-47bb-0310-9956-ffa450edef68
---
 ...onServerCoprocessorExceptionWithAbort.java | 116 +++++-------------
 1 file changed, 28 insertions(+), 88 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
index ae5b9fae167b..6d2fc33d3236 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -20,27 +19,28 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import static org.junit.Assert.fail;
+
 import java.io.IOException;
 
+import junit.framework.Assert;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import static org.junit.Assert.*;
-
 /**
  * Tests unhandled exceptions thrown by coprocessors running on a regionserver..
  * Expected result is that the regionserver will abort with an informative
@@ -49,49 +49,18 @@
  */
 @Category(MediumTests.class)
 public class TestRegionServerCoprocessorExceptionWithAbort {
-  static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
-
-  private class zkwAbortable implements Abortable {
-    @Override
-    public void abort(String why, Throwable e) {
-      throw new RuntimeException("Fatal ZK rs tracker error, why=", e);
-    }
-    @Override
-    public boolean isAborted() {
-      return false;
-    }
-  };
-
-  private class RSTracker extends ZooKeeperNodeTracker {
-    public boolean regionZKNodeWasDeleted = false;
-    public String rsNode;
-    private Thread mainThread;
-
-    public RSTracker(ZooKeeperWatcher zkw, String rsNode, Thread mainThread) {
-      super(zkw, rsNode, new zkwAbortable());
-      this.rsNode = rsNode;
-      this.mainThread = mainThread;
-    }
-
-    @Override
-    public synchronized void nodeDeleted(String path) {
-      if (path.equals(rsNode)) {
-        regionZKNodeWasDeleted = true;
-        mainThread.interrupt();
-      }
-    }
-  }
-  private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  static final int timeout = 30000;
+  static final Log LOG = LogFactory.getLog(TestRegionServerCoprocessorExceptionWithAbort.class);
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final String TABLE_NAME = "observed_table";
 
   @BeforeClass
   public static void setupBeforeClass() throws Exception {
     // set configure to indicate which cp should be loaded
     Configuration conf = TEST_UTIL.getConfiguration();
-    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
-        BuggyRegionObserver.class.getName());
+    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);  // Let's fail fast.
+    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, BuggyRegionObserver.class.getName());
     conf.set("hbase.coprocessor.abortonerror", "true");
-    TEST_UTIL.startMiniCluster(2);
+    TEST_UTIL.startMiniCluster(3);
   }
 
   @AfterClass
@@ -101,11 +70,11 @@ public static void teardownAfterClass() throws Exception {
 
   @Test
   public void testExceptionFromCoprocessorDuringPut()
-      throws IOException {
+    throws IOException, InterruptedException {
     // When we try to write to TEST_TABLE, the buggy coprocessor will
     // cause a NullPointerException, which will cause the regionserver (which
     // hosts the region we attempted to write to) to abort.
-    byte[] TEST_TABLE = Bytes.toBytes("observed_table");
+    byte[] TEST_TABLE = Bytes.toBytes(TABLE_NAME);
     byte[] TEST_FAMILY = Bytes.toBytes("aaa");
 
     HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
@@ -113,47 +82,20 @@ public void testExceptionFromCoprocessorDuringPut()
     TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
 
     // Note which regionServer will abort (after put is attempted).
-    final HRegionServer regionServer =
-        TEST_UTIL.getRSForFirstRegionInTable(TEST_TABLE);
-
-    // add watch so we can know when this regionserver aborted.
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
-        "unittest", new zkwAbortable());
+    final HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(TEST_TABLE);
 
-    RSTracker rsTracker = new RSTracker(zkw,
-        "/hbase/rs/"+regionServer.getServerName(), Thread.currentThread());
-    rsTracker.start();
-    zkw.registerListener(rsTracker);
+    final byte[] ROW = Bytes.toBytes("aaa");
+    Put put = new Put(ROW);
+    put.add(TEST_FAMILY, ROW, ROW);
 
-    boolean caughtInterruption = false;
+    Assert.assertFalse("The region server should be available", regionServer.isAborted());
     try {
-      final byte[] ROW = Bytes.toBytes("aaa");
-      Put put = new Put(ROW);
-      put.add(TEST_FAMILY, ROW, ROW);
       table.put(put);
-    } catch (IOException e) {
-      // Depending on exact timing of the threads involved, zkw's interruption
-      // might be caught here ...
-      if (e.getCause().getClass().equals(InterruptedException.class)) {
-	LOG.debug("caught interruption here (during put()).");
-        caughtInterruption = true;
-      } else {
-        fail("put() failed: " + e);
-      }
-    }
-    if (caughtInterruption == false) {
-      try {
-        Thread.sleep(timeout);
-        fail("RegionServer did not abort within 30 seconds.");
-      } catch (InterruptedException e) {
-        // .. or it might be caught here.
-	LOG.debug("caught interruption here (during sleep()).");
-        caughtInterruption = true;
-      }
+      fail("The put should have failed, as the coprocessor is buggy");
+    } catch (IOException ignored) {
+      // Expected.
     }
-    assertTrue("Main thread caught interruption.",caughtInterruption);
-    assertTrue("RegionServer aborted on coprocessor exception, as expected.",
-        rsTracker.regionZKNodeWasDeleted);
+    Assert.assertTrue("The region server should have aborted", regionServer.isAborted());
     table.close();
   }
 
@@ -162,11 +104,9 @@ public static class BuggyRegionObserver extends SimpleRegionObserver {
     public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
                        final Put put, final WALEdit edit,
                        final boolean writeToWAL) {
-      String tableName =
-          c.getEnvironment().getRegion().getRegionInfo().getTableNameAsString();
-      if (tableName.equals("observed_table")) {
-        Integer i = null;
-        i = i + 1;
+      String tableName = c.getEnvironment().getRegion().getRegionInfo().getTableNameAsString();
+      if (TABLE_NAME.equals(tableName)) {
+        throw new NullPointerException("Buggy coprocessor");
       }
     }
   }

From 89fac2b85d8ad92e372931bd043b83b738434ed9 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Sat, 13 Jul 2013 15:19:07 +0000
Subject: [PATCH 1057/1540] HBASE-8938 Thrift2 does not close scanner instance
 (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1502799 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index 11ea77784759..0089568cce3d 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -333,11 +333,17 @@ public List<TResult> getScannerRows(int scannerId, int numRows) throws TIOError,
 
   @Override
   public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, TException {
-    if (removeScanner(scannerId) == null) {
+    LOG.debug("scannerClose: id=" + scannerId);
+    ResultScanner scanner = getScanner(scannerId);
+    if (scanner == null) {
+      String message = "scanner ID is invalid";
+      LOG.warn(message);
       TIllegalArgument ex = new TIllegalArgument();
       ex.setMessage("Invalid scanner Id");
       throw ex;
     }
+    scanner.close();
+    removeScanner(scannerId);
   }
 
 }

From a35e7dcc57291746152f9b66f821fd11509ce8fb Mon Sep 17 00:00:00 2001
From: sershe <sershe@unknown>
Date: Mon, 15 Jul 2013 23:31:10 +0000
Subject: [PATCH 1058/1540] HBASE-8935 IntegrationTestBigLinkedList fails under
 load on 0.94 due to some scan issues - add logging

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1503522 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TableRecordReaderImpl.java      |  2 +-
 .../hadoop/hbase/regionserver/HRegionServer.java    | 13 +++++++++++--
 .../hbase/test/IntegrationTestBigLinkedList.java    | 10 ++++++++++
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index d91b555824ca..39c9b8ec8e8d 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -46,7 +46,7 @@ public class TableRecordReaderImpl {
   public static final String LOG_PER_ROW_COUNT
     = "hbase.mapreduce.log.scanner.rowcount";
 
-  static final Log LOG = LogFactory.getLog(TableRecordReader.class);
+  static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
 
   // HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
   private static final String HBASE_COUNTER_GROUP_NAME =
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b168855a66dc..31ea1805ee43 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2592,7 +2592,10 @@ public Result next(final long scannerId) throws IOException {
   public Result[] next(final long scannerId, int nbRows) throws IOException {
     String scannerName = String.valueOf(scannerId);
     RegionScanner s = this.scanners.get(scannerName);
-    if (s == null) throw new UnknownScannerException("Name: " + scannerName);
+    if (s == null) {
+      LOG.info("Client tried to access missing scanner " + scannerName);
+      throw new UnknownScannerException("Name: " + scannerName);
+    }
     try {
       checkOpen();
     } catch (IOException e) {
@@ -2610,7 +2613,13 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
     try {
       // Remove lease while its being processed in server; protects against case
       // where processing of request takes > lease expiration time.
-      lease = this.leases.removeLease(scannerName);
+      try {
+        lease = this.leases.removeLease(scannerName);
+      } catch (LeaseException le) {
+        // What it really means is that there's no such scanner.
+        LOG.info("Client tried to access missing scanner " + scannerName + " (no lease)");
+        throw new UnknownScannerException("No lease for " + scannerName + ": " + le.getMessage());
+      }
       List<Result> results = new ArrayList<Result>(nbRows);
       long currentScanResultSize = 0;
       List<KeyValue> values = new ArrayList<KeyValue>();
diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 35bf09f74dc7..8a397f5cbc56 100644
--- a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -52,9 +52,11 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ScannerCallable;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.NullWritable;
@@ -626,6 +628,8 @@ public int run(Path outputDir, int numReducers) throws Exception {
       job.setNumReduceTasks(numReducers);
       job.setJarByClass(getClass());
 
+      setJobScannerConf(job);
+
       Scan scan = new Scan();
       scan.addColumn(FAMILY_NAME, COLUMN_PREV);
       scan.setCaching(10000);
@@ -1081,4 +1085,10 @@ private static void setJobConf(Job job, int numMappers, long numNodes,
       job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMuplitplier.intValue());
     }
   }
+
+  private static void setJobScannerConf(Job job) {
+    // Make sure scanners log something useful to make debugging possible.
+    job.getConfiguration().setBoolean(ScannerCallable.LOG_SCANNER_ACTIVITY, true);
+    job.getConfiguration().setInt(TableRecordReaderImpl.LOG_PER_ROW_COUNT, 100000);
+  }
 }

From c2e24893ade96e9738f4b7a8fd85d3a81335912d Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 16 Jul 2013 00:38:22 +0000
Subject: [PATCH 1059/1540] HBASE-8908. Backport
 HBASE-8882+HBASE-8904+HBASE-8924 (An Integration Test to Test MTTR) to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1503530 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/AbstractHBaseTool.java  |   3 +-
 .../hbase/mttr/IntegrationTestMTTR.java       | 518 ++++++++++++++++++
 .../apache/hadoop/hbase/util/ChaosMonkey.java |  99 +++-
 3 files changed, 601 insertions(+), 19 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java

diff --git a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
index 814dd740ac50..293c3634eeaa 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -16,6 +16,7 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.io.IOException;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -76,7 +77,7 @@ public void setConf(Configuration conf) {
   }
 
   @Override
-  public final int run(String[] args) throws Exception {
+  public final int run(String[] args) throws IOException {
     if (conf == null) {
       LOG.error("Tool configuration is not initialized");
       throw new NullPointerException("conf");
diff --git a/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java b/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java
new file mode 100644
index 000000000000..a69dbbb35391
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java
@@ -0,0 +1,518 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mttr;
+
+import com.google.common.base.Objects;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.IntegrationTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChaosMonkey;
+import org.apache.hadoop.hbase.util.LoadTestTool;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * Integration test that should benchmark how fast HBase can recover from failures. This test starts
+ * different threads:
+ * <ol>
+ * <li>
+ * Load Test Tool.<br/>
+ * This runs so that all RegionServers will have some load and HLogs will be full.
+ * </li>
+ * <li>
+ * Scan thread.<br/>
+ * This thread runs a very short scan over and over again recording how log it takes to respond.
+ * The longest response is assumed to be the time it took to recover.
+ * </li>
+ * <li>
+ * Put thread.<br/>
+ * This thread just like the scan thread except it does a very small put.
+ * </li>
+ * <li>
+ * Admin thread. <br/>
+ * This thread will continually go to the master to try and get the cluster status.  Just like the
+ * put and scan threads, the time to respond is recorded.
+ * </li>
+ * <li>
+ * Chaos Monkey thread.<br/>
+ * This thread runs a ChaosMonkey.Action.
+ * </li>
+ * </ol>
+ * <p/>
+ * The ChaosMonkey actions currently run are:
+ * <ul>
+ * <li>Restart the RegionServer holding meta.</li>
+ * <li>Restart the RegionServer holding the table the scan and put threads are targeting.</li>
+ * <li>Move the Regions of the table used by the scan and put threads.</li>
+ * <li>Restart the master.</li>
+ * </ul>
+ * <p/>
+ * At the end of the test a log line is output on the INFO level containing the timing data that was
+ * collected.
+ */
+
+@Category(IntegrationTests.class)
+public class IntegrationTestMTTR {
+  /**
+   * Constants.
+   */
+  private static final byte[] FAMILY = Bytes.toBytes("d");
+  private static final Log LOG = LogFactory.getLog(IntegrationTestMTTR.class);
+  private static final long SLEEP_TIME = 60 * 1000l;
+
+  /**
+   * Configurable table names.
+   */
+  private static String tableName;
+  private static byte[] tableNameBytes;
+  private static String loadTableName;
+  private static byte[] loadTableNameBytes;
+
+  /**
+   * Util to get at the cluster.
+   */
+  private static IntegrationTestingUtility util;
+
+  /**
+   * Executor for test threads.
+   */
+  private static ExecutorService executorService;
+
+  /**
+   * All of the chaos monkey actions used.
+   */
+  private static ChaosMonkey.Action restartRSAction;
+  private static ChaosMonkey.Action restartMetaAction;
+  private static ChaosMonkey.Action moveRegionAction;
+  private static ChaosMonkey.Action restartMasterAction;
+
+  /**
+   * The load test tool used to create load and make sure that HLogs aren't empty.
+   */
+  private static LoadTestTool loadTool;
+
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    // Set up the integration test util
+    if (util == null) {
+      util = new IntegrationTestingUtility();
+    }
+
+    // Make sure there are three servers.
+    util.initializeCluster(3);
+
+    // Set up the load test tool.
+    loadTool = new LoadTestTool();
+    loadTool.setConf(util.getConfiguration());
+
+    // Create executor with enough threads to restart rs's,
+    // run scans, puts, admin ops and load test tool.
+    executorService = Executors.newFixedThreadPool(8);
+
+    // Set up the tables needed.
+    setupTables();
+
+    // Set up the actions.
+    setupActions();
+  }
+
+  private static void setupActions() throws IOException {
+    // Set up the action that will restart a region server holding a region from our table
+    // because this table should only have one region we should be good.
+    restartRSAction = new ChaosMonkey.RestartRsHoldingTable(SLEEP_TIME, tableName);
+
+    // Set up the action that will kill the region holding meta.
+    restartMetaAction = new ChaosMonkey.RestartRsHoldingMeta(SLEEP_TIME);
+
+    // Set up the action that will move the regions of our table.
+    moveRegionAction = new ChaosMonkey.MoveRegionsOfTable(SLEEP_TIME, tableName);
+
+    // Kill the master
+    restartMasterAction = new ChaosMonkey.RestartActiveMaster(1000);
+
+    // Give the action the access to the cluster.
+    ChaosMonkey.ActionContext actionContext = new ChaosMonkey.ActionContext(util);
+    restartRSAction.init(actionContext);
+    restartMetaAction.init(actionContext);
+    moveRegionAction.init(actionContext);
+    restartMasterAction.init(actionContext);
+  }
+
+  private static void setupTables() throws IOException {
+    // Get the table name.
+    tableName = util.getConfiguration()
+        .get("hbase.IntegrationTestMTTR.tableName", "IntegrationTestMTTR");
+    tableNameBytes = Bytes.toBytes(tableName);
+
+    loadTableName = util.getConfiguration()
+        .get("hbase.IntegrationTestMTTR.loadTableName", "IntegrationTestMTTRLoadTestTool");
+    loadTableNameBytes = Bytes.toBytes(loadTableName);
+
+    if (util.getHBaseAdmin().tableExists(tableNameBytes)) {
+      util.deleteTable(tableNameBytes);
+    }
+
+    if (util.getHBaseAdmin().tableExists(loadTableName)) {
+      util.deleteTable(loadTableNameBytes);
+    }
+
+    // Create the table.  If this fails then fail everything.
+    HTableDescriptor tableDescriptor = new HTableDescriptor(tableNameBytes);
+
+    // Make the max file size huge so that splits don't happen during the test.
+    tableDescriptor.setMaxFileSize(Long.MAX_VALUE);
+
+    HColumnDescriptor descriptor = new HColumnDescriptor(FAMILY);
+    descriptor.setMaxVersions(1);
+    tableDescriptor.addFamily(descriptor);
+    util.getHBaseAdmin().createTable(tableDescriptor);
+
+    // Setup the table for LoadTestTool
+    int ret = loadTool.run(new String[]{"-tn", loadTableName, "-init_only"});
+    assertEquals("Failed to initialize LoadTestTool", 0, ret);
+  }
+
+  @AfterClass
+  public static void after() throws IOException {
+    // Clean everything up.
+    util.restoreCluster();
+    util = null;
+
+    // Stop the threads so that we know everything is complete.
+    executorService.shutdown();
+    executorService = null;
+
+    // Clean up the actions.
+    moveRegionAction = null;
+    restartMetaAction = null;
+    restartRSAction = null;
+    restartMasterAction = null;
+
+    loadTool = null;
+  }
+
+  @Test
+  public void testRestartRsHoldingTable() throws Exception {
+    run(new ActionCallable(restartRSAction), "RestartRsHoldingTable");
+  }
+
+  @Test
+  public void testKillRsHoldingMeta() throws Exception {
+    run(new ActionCallable(restartMetaAction), "KillRsHoldingMeta");
+  }
+
+  @Test
+  public void testMoveRegion() throws Exception {
+    run(new ActionCallable(moveRegionAction), "MoveRegion");
+  }
+
+  @Test
+  public void testRestartMaster() throws Exception {
+    run(new ActionCallable(restartMasterAction), "RestartMaster");
+  }
+
+  public void run(Callable<Boolean> monkeyCallable, String testName) throws Exception {
+    int maxIters = util.getHBaseClusterInterface().isDistributedCluster() ? 10 : 3;
+
+    // Array to keep track of times.
+    ArrayList<TimingResult> resultPuts = new ArrayList<TimingResult>(maxIters);
+    ArrayList<TimingResult> resultScan = new ArrayList<TimingResult>(maxIters);
+    ArrayList<TimingResult> resultAdmin = new ArrayList<TimingResult>(maxIters);
+    long start = System.nanoTime();
+
+    // We're going to try this multiple times
+    for (int fullIterations = 0; fullIterations < maxIters; fullIterations++) {
+      // Create and start executing a callable that will kill the servers
+      Future<Boolean> monkeyFuture = executorService.submit(monkeyCallable);
+
+      // Pass that future to the timing Callables.
+      Future<TimingResult> putFuture = executorService.submit(new PutCallable(monkeyFuture));
+      Future<TimingResult> scanFuture = executorService.submit(new ScanCallable(monkeyFuture));
+      Future<TimingResult> adminFuture = executorService.submit(new AdminCallable(monkeyFuture));
+
+      Future<Boolean> loadFuture = executorService.submit(new LoadCallable(monkeyFuture));
+
+      monkeyFuture.get();
+      loadFuture.get();
+
+      // Get the values from the futures.
+      TimingResult putTime = putFuture.get();
+      TimingResult scanTime = scanFuture.get();
+      TimingResult adminTime = adminFuture.get();
+
+      // Store the times to display later.
+      resultPuts.add(putTime);
+      resultScan.add(scanTime);
+      resultAdmin.add(adminTime);
+
+      // Wait some time for everything to settle down.
+      Thread.sleep(5000l);
+    }
+
+    long runtimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS);
+
+    Objects.ToStringHelper helper = Objects.toStringHelper("MTTRResults")
+        .add("putResults", resultPuts)
+        .add("scanResults", resultScan)
+        .add("adminResults", resultAdmin)
+        .add("totalRuntimeMs", runtimeMs)
+        .add("name", testName);
+
+    // Log the info
+    LOG.info(helper.toString());
+  }
+
+  /**
+   * Class to store results of TimingCallable.
+   *
+   * Stores times and trace id.
+   */
+  private class TimingResult {
+    DescriptiveStatistics stats = new DescriptiveStatistics();
+
+    /**
+     * Add a result to this aggregate result.
+     * @param time Time in nanoseconds
+     * @param span Span.  To be kept if the time taken was over 1 second
+     */
+    public void addResult(long time) {
+      stats.addValue(TimeUnit.MILLISECONDS.convert(time, TimeUnit.NANOSECONDS));
+    }
+
+    public String toString() {
+      Objects.ToStringHelper helper = Objects.toStringHelper(this)
+          .add("numResults", stats.getN())
+          .add("minTime", stats.getMin())
+          .add("meanTime", stats.getMean())
+          .add("maxTime", stats.getMax())
+          .add("25th", stats.getPercentile(25))
+          .add("50th", stats.getPercentile(50))
+          .add("75th", stats.getPercentile(75))
+          .add("90th", stats.getPercentile(90))
+          .add("95th", stats.getPercentile(95))
+          .add("99th", stats.getPercentile(99))
+          .add("99.9th", stats.getPercentile(99.9))
+          .add("99.99th", stats.getPercentile(99.99));
+      return helper.toString();
+    }
+  }
+
+  /**
+   * Base class for actions that need to record the time needed to recover from a failure.
+   */
+  public abstract class TimingCallable implements Callable<TimingResult> {
+    protected final Future future;
+
+    public TimingCallable(Future f) {
+      future = f;
+    }
+
+    @Override
+    public TimingResult call() throws Exception {
+      TimingResult result = new TimingResult();
+      int numAfterDone = 0;
+      // Keep trying until the rs is back up and we've gotten a put through
+      while (numAfterDone < 10) {
+        long start = System.nanoTime();
+        try {
+          boolean actionResult = doAction();
+          if (actionResult && future.isDone()) {
+            numAfterDone ++;
+          }
+        } catch (Exception e) {
+          numAfterDone = 0;
+        }
+        result.addResult(System.nanoTime() - start);
+      }
+      return result;
+    }
+
+    protected abstract boolean doAction() throws Exception;
+
+    protected String getSpanName() {
+      return this.getClass().getSimpleName();
+    }
+  }
+
+  /**
+   * Callable that will keep putting small amounts of data into a table
+   * until  the future supplied returns.  It keeps track of the max time.
+   */
+  public class PutCallable extends TimingCallable {
+
+    private final HTable table;
+
+    public PutCallable(Future f) throws IOException {
+      super(f);
+      this.table = new HTable(util.getConfiguration(), tableNameBytes);
+    }
+
+    @Override
+    protected boolean doAction() throws Exception {
+      Put p = new Put(Bytes.toBytes(RandomStringUtils.randomAlphanumeric(5)));
+      p.add(FAMILY, Bytes.toBytes("\0"), Bytes.toBytes(RandomStringUtils.randomAscii(5)));
+      table.put(p);
+      table.flushCommits();
+      return true;
+    }
+
+    @Override
+    protected String getSpanName() {
+      return "MTTR Put Test";
+    }
+  }
+
+  /**
+   * Callable that will keep scanning for small amounts of data until the
+   * supplied future returns.  Returns the max time taken to scan.
+   */
+  public class ScanCallable extends TimingCallable {
+    private final HTable table;
+
+    public ScanCallable(Future f) throws IOException {
+      super(f);
+      this.table = new HTable(util.getConfiguration(), tableNameBytes);
+    }
+
+    @Override
+    protected boolean doAction() throws Exception {
+      ResultScanner rs = null;
+      try {
+      Scan s = new Scan();
+      s.setBatch(2);
+      s.addFamily(FAMILY);
+      s.setFilter(new KeyOnlyFilter());
+      s.setMaxVersions(1);
+
+      rs = table.getScanner(s);
+      Result result = rs.next();
+      return rs != null && result != null && result.size() > 0;
+      } finally {
+        if (rs != null) {
+          rs.close();
+        }
+      }
+    }
+    @Override
+    protected String getSpanName() {
+      return "MTTR Scan Test";
+    }
+  }
+
+  /**
+   * Callable that will keep going to the master for cluster status.  Returns the max time taken.
+   */
+  public class AdminCallable extends TimingCallable {
+
+    public AdminCallable(Future f) throws IOException {
+      super(f);
+    }
+
+    @Override
+    protected boolean doAction() throws Exception {
+      HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
+      ClusterStatus status = admin.getClusterStatus();
+      return status != null;
+    }
+
+    @Override
+    protected String getSpanName() {
+      return "MTTR Admin Test";
+    }
+  }
+
+
+  public class ActionCallable implements Callable<Boolean> {
+    private final ChaosMonkey.Action action;
+
+    public ActionCallable(ChaosMonkey.Action action) {
+      this.action = action;
+    }
+
+    @Override
+    public Boolean call() throws Exception {
+      this.action.perform();
+      return true;
+    }
+  }
+
+  /**
+   * Callable used to make sure the cluster has some load on it.
+   * This callable uses LoadTest tool to
+   */
+  public class LoadCallable implements Callable<Boolean> {
+
+    private final Future future;
+
+    public LoadCallable(Future f) {
+      future = f;
+    }
+
+    @Override
+    public Boolean call() throws Exception {
+      int colsPerKey = 10;
+      int recordSize = 500;
+      int numServers = util.getHBaseClusterInterface().getInitialClusterStatus().getServersSize();
+      int numKeys = numServers * 5000;
+      int writeThreads = 10;
+
+
+      // Loop until the chaos monkey future is done.
+      // But always go in just in case some action completes quickly
+      do {
+        int ret = loadTool.run(new String[]{
+            "-tn", loadTableName,
+            "-write", String.format("%d:%d:%d", colsPerKey, recordSize, writeThreads),
+            "-num_keys", String.valueOf(numKeys),
+            "-skip_init"
+        });
+        assertEquals("Load failed", 0, ret);
+      } while (!future.isDone());
+
+      return true;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index 266586f394dc..ff8cac1ae93e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -22,13 +22,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Random;
-import java.util.Set;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.logging.Log;
@@ -44,12 +42,12 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.protobuf.ServiceException;
 
 /**
  * A utility to injects faults in a running cluster.
@@ -113,10 +111,10 @@ private void setPoliciesByName(String... policies) {
   /**
    * Context for Action's
    */
-  private static class ActionContext {
+  public static class ActionContext {
     private IntegrationTestingUtility util;
 
-    ActionContext(IntegrationTestingUtility util) {
+    public ActionContext(IntegrationTestingUtility util) {
       this.util = util;
     }
 
@@ -143,7 +141,7 @@ public static class Action {
     protected ClusterStatus initialStatus;
     protected ServerName[] initialServers;
 
-    void init(ActionContext context) throws Exception {
+    public void init(ActionContext context) throws IOException {
       this.context = context;
       cluster = context.getHBaseCluster();
       initialStatus = cluster.getInitialClusterStatus();
@@ -151,7 +149,7 @@ void init(ActionContext context) throws Exception {
       initialServers = regionServers.toArray(new ServerName[regionServers.size()]);
     }
 
-    void perform() throws Exception { };
+    public void perform() throws Exception { };
 
     // TODO: perhaps these methods should be elsewhere?
     /** Returns current region servers */
@@ -204,12 +202,14 @@ void sleep(long sleepTime) {
     }
 
     void restartMaster(ServerName server, long sleepTime) throws IOException {
+      sleepTime = Math.max(sleepTime, 1000);
       killMaster(server);
       sleep(sleepTime);
       startMaster(server);
     }
 
     void restartRs(ServerName server, long sleepTime) throws IOException {
+      sleepTime = Math.max(sleepTime, 1000);
       killRs(server);
       sleep(sleepTime);
       startRs(server);
@@ -221,7 +221,7 @@ public RestartActiveMaster(long sleepTime) {
       super(sleepTime);
     }
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info("Performing action: Restart active master");
 
       ServerName master = cluster.getClusterStatus().getMaster();
@@ -235,7 +235,7 @@ public RestartRandomRs(long sleepTime) {
     }
 
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info("Performing action: Restart random region server");
       ServerName server = selectRandomItem(getCurrentServers());
 
@@ -243,12 +243,12 @@ void perform() throws Exception {
     }
   }
 
-  public static class RestartRsHoldingMeta extends RestartRandomRs {
+  public static class RestartRsHoldingMeta extends RestartActionBase {
     public RestartRsHoldingMeta(long sleepTime) {
       super(sleepTime);
     }
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info("Performing action: Restart region server holding META");
       ServerName server = cluster.getServerHoldingMeta();
       if (server == null) {
@@ -264,7 +264,7 @@ public RestartRsHoldingRoot(long sleepTime) {
       super(sleepTime);
     }
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info("Performing action: Restart region server holding ROOT");
       ServerName server = cluster.getServerHoldingMeta();
       if (server == null) {
@@ -275,6 +275,70 @@ void perform() throws Exception {
     }
   }
 
+  public static class RestartRsHoldingTable extends RestartActionBase {
+
+    private final String tableName;
+
+    public RestartRsHoldingTable(long sleepTime, String tableName) {
+      super(sleepTime);
+      this.tableName = tableName;
+    }
+
+    @Override
+    public void perform() throws Exception {
+      HTable table = null;
+      Collection<ServerName> serverNames;
+      try {
+        Configuration conf = context.getHaseIntegrationTestingUtility().getConfiguration();
+        table = new HTable(conf, tableName);
+        serverNames = table.getRegionLocations().values();
+      } catch (IOException e) {
+        LOG.debug("Error creating HTable used to get list of region locations.", e);
+        return;
+      } finally {
+        if (table != null) {
+          table.close();
+        }
+      }
+      Random random = new Random();
+      ServerName[] nameArray = serverNames.toArray(new ServerName[serverNames.size()]);
+      restartRs(nameArray[random.nextInt(nameArray.length)], sleepTime);
+    }
+  }
+
+  public static class MoveRegionsOfTable extends Action {
+    private final long sleepTime;
+    private final byte[] tableNameBytes;
+
+    public MoveRegionsOfTable(long sleepTime, String tableName) {
+      this.sleepTime = sleepTime;
+      this.tableNameBytes = Bytes.toBytes(tableName);
+    }
+
+    @Override
+    public void perform() throws Exception {
+      try {
+        HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
+        List<HRegionInfo> regions = admin.getTableRegions(tableNameBytes);
+        Collection<ServerName> serversList = admin.getClusterStatus().getServers();
+        ServerName[] servers = serversList.toArray(new ServerName[serversList.size()]);
+        Random random = new Random();
+        for (HRegionInfo regionInfo:regions) {
+          try {
+            byte[] destServerName =
+              Bytes.toBytes(servers[random.nextInt(servers.length)].getServerName());
+            admin.move(regionInfo.getRegionName(), destServerName);
+          } catch (Exception e) {
+            LOG.debug("Error moving region", e);
+          }
+        }
+        Thread.sleep(sleepTime);
+      } catch (Exception e) {
+        LOG.debug("Error performing MoveRegionsOfTable", e);
+      }
+    }
+  }
+
   /**
    * Restarts a ratio of the running regionservers at the same time
    */
@@ -287,7 +351,7 @@ public BatchRestartRs(long sleepTime, float ratio) {
     }
 
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info(String.format("Performing action: Batch restarting %d%% of region servers",
           (int)(ratio * 100)));
       List<ServerName> selectedServers = selectRandomItems(getCurrentServers(), ratio);
@@ -329,10 +393,10 @@ public RollingBatchRestartRs(long sleepTime, float ratio) {
     }
 
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
+      Random random = new Random();
       LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers",
           (int)(ratio * 100)));
-      Random random = new Random();
       List<ServerName> selectedServers = selectRandomItems(getCurrentServers(), ratio);
 
       Queue<ServerName> serversToBeKilled = new LinkedList<ServerName>(selectedServers);
@@ -379,7 +443,7 @@ public UnbalanceRegionsAction(double fractionOfRegions, double fractionOfServers
     }
 
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info("Unbalancing regions");
       ClusterStatus status = this.cluster.getClusterStatus();
       List<ServerName> victimServers = new LinkedList<ServerName>(status.getServers());
@@ -417,7 +481,7 @@ void perform() throws Exception {
 
   public static class ForceBalancerAction extends Action {
     @Override
-    void perform() throws Exception {
+    public void perform() throws Exception {
       LOG.info("Balancing regions");
       HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
       boolean result = admin.balancer();
@@ -514,7 +578,6 @@ public void init(PolicyContext context) throws Exception {
     }
   }
 
-
   /** A policy which performs a sequence of actions deterministically. */
   public static class DoActionsOncePolicy extends PeriodicPolicy {
     private List<Action> actions;

From 669b6f2d3fb113337a6b3aead2a080b111f2e1df Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 16 Jul 2013 21:42:18 +0000
Subject: [PATCH 1060/1540] BASE-8806 Row locks are acquired repeatedly in
 HRegion.doMiniBatchMutation for duplicate rows. (rahul gidwani and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1503909 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  53 ++++++---
 .../TestBatchHRegionLockingAndWrites.java     | 107 ++++++++++++++++++
 .../hbase/regionserver/TestHBase7051.java     |   3 +-
 3 files changed, 148 insertions(+), 15 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 07a10df910d8..b7629ca428e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -142,6 +142,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.common.collect.MutableClassToInstanceMap;
 
 /**
@@ -2241,6 +2242,8 @@ private long doMiniBatchMutation(
 
     /** Keep track of the locks we hold so we can release them in finally clause */
     List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
+    Set<HashedBytes> rowsAlreadyLocked = Sets.newHashSet();
+      
     // reference family maps directly so coprocessors can mutate them if desired
     Map<byte[],List<KeyValue>>[] familyMaps = new Map[batchOp.operations.length];
     // We try to set up a batch in the range [firstIndex,lastIndexExclusive)
@@ -2297,15 +2300,23 @@ private long doMiniBatchMutation(
         // If we haven't got any rows in our batch, we should block to
         // get the next one.
         boolean shouldBlock = numReadyToWrite == 0;
+        boolean failedToAcquire = false;
         Integer acquiredLockId = null;
+        HashedBytes currentRow = new HashedBytes(mutation.getRow());
         try {
-          acquiredLockId = getLock(providedLockId, mutation.getRow(),
-              shouldBlock);
+          if (providedLockId != null || !rowsAlreadyLocked.contains(currentRow)) {
+            acquiredLockId = getLock(providedLockId, currentRow, shouldBlock);
+            if (acquiredLockId == null) {
+              failedToAcquire = true;
+            } else if (providedLockId == null) {
+              rowsAlreadyLocked.add(currentRow);
+            }
+          } 
         } catch (IOException ioe) {
-          LOG.warn("Failed getting lock in batch put, row="
-                  + Bytes.toStringBinary(mutation.getRow()), ioe);
+          LOG.warn("Failed getting lock in batch put, row=" + currentRow, ioe);
+          failedToAcquire = true;
         }
-        if (acquiredLockId == null) {
+        if (failedToAcquire) {
           // We failed to grab another lock
           assert !shouldBlock : "Should never fail to get lock when blocking";
           break; // stop acquiring more rows for this batch
@@ -2452,6 +2463,7 @@ private long doMiniBatchMutation(
           releaseRowLock(toRelease);
         }
         acquiredLocks = null;
+        rowsAlreadyLocked = null;
       }
       // -------------------------
       // STEP 7. Sync wal.
@@ -3432,7 +3444,7 @@ public Integer obtainRowLock(final byte [] row) throws IOException {
     this.writeRequestsCount.increment();
     this.opMetrics.setWriteRequestCountMetrics( this.writeRequestsCount.get());
     try {
-      return internalObtainRowLock(row, true);
+      return internalObtainRowLock(new HashedBytes(row), true);
     } finally {
       closeRegionOperation();
     }
@@ -3444,12 +3456,11 @@ public Integer obtainRowLock(final byte [] row) throws IOException {
    *        Otherwise, just tries to obtain the lock and returns
    *        null if unavailable.
    */
-  private Integer internalObtainRowLock(final byte[] row, boolean waitForLock)
+  private Integer internalObtainRowLock(final HashedBytes rowKey, boolean waitForLock)
       throws IOException {
-    checkRow(row, "row lock");
+    checkRow(rowKey.getBytes(), "row lock");
     startRegionOperation();
     try {
-      HashedBytes rowKey = new HashedBytes(row);
       CountDownLatch rowLatch = new CountDownLatch(1);
 
       // loop until we acquire the row lock (unless !waitForLock)
@@ -3465,8 +3476,7 @@ private Integer internalObtainRowLock(final byte[] row, boolean waitForLock)
           try {
             if (!existingLatch.await(this.rowLockWaitDuration,
                             TimeUnit.MILLISECONDS)) {
-              throw new IOException("Timed out on getting lock for row="
-                  + Bytes.toStringBinary(row));
+              throw new IOException("Timed out on getting lock for row=" + rowKey);
             }
           } catch (InterruptedException ie) {
             // Empty
@@ -3540,12 +3550,27 @@ boolean isRowLocked(final Integer lockId) {
    */
   public Integer getLock(Integer lockid, byte [] row, boolean waitForLock)
   throws IOException {
-    Integer lid = null;
+    return getLock(lockid, new HashedBytes(row), waitForLock);
+  }
+
+  /**
+   * Returns existing row lock if found, otherwise
+   * obtains a new row lock and returns it.
+   * @param lockid requested by the user, or null if the user didn't already hold lock
+   * @param row the row to lock
+   * @param waitForLock if true, will block until the lock is available, otherwise will
+   * simply return null if it could not acquire the lock.
+   * @return lockid or null if waitForLock is false and the lock was unavailable.
+   */
+  protected Integer getLock(Integer lockid, HashedBytes row, boolean waitForLock)
+  throws IOException {
+    Integer lid;
     if (lockid == null) {
       lid = internalObtainRowLock(row, waitForLock);
     } else {
-      if (!isRowLocked(lockid)) {
-        throw new IOException("Invalid row lock");
+      HashedBytes rowFromLock = lockIds.get(lockid);
+      if (!row.equals(rowFromLock)) {
+        throw new IOException("Invalid row lock: LockId: " + lockid + " holds the lock for row: " + rowFromLock + " but wanted lock for row: " + row);
       }
       lid = lockid;
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java
new file mode 100644
index 000000000000..e20aab3121c0
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HashedBytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+
+@Category(SmallTests.class)
+public class TestBatchHRegionLockingAndWrites {
+  private static final String FAMILY = "a";
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testRedundantRowKeys() throws Exception {
+
+    final int batchSize = 100000;
+
+    String tableName = getClass().getSimpleName();
+    Configuration conf = HBaseConfiguration.create();
+    conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
+    MockHRegion region = (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName), tableName, conf, Bytes.toBytes("a"));
+
+    List<Pair<Mutation, Integer>> someBatch = Lists.newArrayList();
+    int i = 0;
+    while (i < batchSize) {
+      if (i % 2 == 0) {
+        someBatch.add(new Pair<Mutation, Integer>(new Put(Bytes.toBytes(0)), null));
+      } else {
+        someBatch.add(new Pair<Mutation, Integer>(new Put(Bytes.toBytes(1)), null));
+      }
+      i++;
+    }
+    long start = System.nanoTime();
+    region.batchMutate(someBatch.toArray(new Pair[0]));
+    long duration = System.nanoTime() - start;
+    System.out.println("Batch mutate took: " + duration + "ns");
+    assertEquals(2, region.getAcquiredLockCount());
+  }
+
+  @Test
+  public void testGettingTheLockMatchesMyRow() throws Exception {
+    MockHRegion region = getMockHRegion();
+    HashedBytes rowKey = new HashedBytes(Bytes.toBytes(1));
+    assertEquals(Integer.valueOf(2), region.getLock(null, rowKey, false));
+    assertEquals(Integer.valueOf(2), region.getLock(2, rowKey, false));
+  }
+
+  private MockHRegion getMockHRegion() throws IOException {
+    String tableName = getClass().getSimpleName();
+    Configuration conf = HBaseConfiguration.create();
+    conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
+    return (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName), tableName, conf, Bytes.toBytes(FAMILY));
+  }
+
+  private static class MockHRegion extends HRegion {
+    private int acqioredLockCount = 0;
+
+    public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf, final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) {
+      super(tableDir, log, fs, conf, regionInfo, htd, rsServices);
+    }
+
+    private int getAcquiredLockCount() {
+      return acqioredLockCount;
+    }
+
+    @Override
+    public Integer getLock(Integer lockid, HashedBytes row, boolean waitForLock) throws IOException {
+      acqioredLockCount++;
+      return super.getLock(lockid, row, waitForLock);
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
index f5ce0b028c0f..5636dd611845 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HashedBytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -192,7 +193,7 @@ else if (testStep == TestStep.CHECKANDPUT_STARTED) {
     }
 
     @Override
-    public Integer getLock(Integer lockid, byte[] row, boolean waitForLock) throws IOException {
+    public Integer getLock(Integer lockid, HashedBytes row, boolean waitForLock) throws IOException {
       if (testStep == TestStep.CHECKANDPUT_STARTED) {
         latch.countDown();
       }

From 74a2b7f4d6eab22c9e7df1e1924f6d9e5031e525 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Wed, 17 Jul 2013 18:20:00 +0000
Subject: [PATCH 1061/1540] HBASE-8969 Backport HBASE-8535+HBASE-8586
 TestHCM#testDeleteForZKConnLeak enhancement to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504215 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/TestHCM.java   | 63 ++++++++++++-------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 98a99e7033a8..d9e88673ca41 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -331,13 +331,21 @@ public void testCreateConnection() throws Exception {
   }
 
   /**
-   * Tests that a closed connection does not have a live zookeeper
+   * Tests that a destroyed connection does not have a live zookeeper.
+   * Below is timing based.  We put up a connection to a table and then close the connection while
+   * having a background thread running that is forcing close of the connection to try and
+   * provoke a close catastrophe; we are hoping for a car crash so we can see if we are leaking
+   * zk connections.
    * @throws Exception
    */
   @Test
   public void testDeleteForZKConnLeak() throws Exception {
     TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
-    final Configuration config = TEST_UTIL.getConfiguration();
+    final Configuration config = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+    config.setInt("zookeeper.recovery.retry", 1);
+    config.setInt("zookeeper.recovery.retry.intervalmill", 1000);
+    config.setInt("hbase.rpc.timeout", 2000);
+    config.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
 
     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
       5, TimeUnit.SECONDS,
@@ -350,41 +358,54 @@ public void run() {
         while (!Thread.interrupted()) {
           try {
             HConnection conn = HConnectionManager.getConnection(config);
+            LOG.info("Connection " + conn);
             HConnectionManager.deleteStaleConnection(conn);
+            LOG.info("Connection closed " + conn);
+            // TODO: This sleep time should be less than the time that it takes to open and close
+            // a table.  Ideally we would do a few runs first to measure.  For now this is
+            // timing based; hopefully we hit the bad condition.
+            Threads.sleep(10);
           } catch (Exception e) {
           }
         }
       }
     });
 
-    // use connection multiple times
-    for (int i = 0; i < 10; i++) {
+    // Use connection multiple times.
+    for (int i = 0; i < 30; i++) {
       HConnection c1 = null;
-        try {
-          c1 = HConnectionManager.getConnection(config);
-          HTable table = new HTable(TABLE_NAME2, c1, pool);
-          table.close();
-        } catch (Exception e) {
-        } finally {
-          if (c1 != null) {
-            if (c1.isClosed()) {
-              // cannot use getZooKeeper as method instantiates watcher if null
-              Field zkwField = c1.getClass().getDeclaredField("zooKeeper");
-              zkwField.setAccessible(true);
-              Object watcher = zkwField.get(c1);
-
-              if (watcher != null) {
-                if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
+      try {
+        c1 = HConnectionManager.getConnection(config);
+        LOG.info("HTable connection " + i + " " + c1);
+        HTable table = new HTable(TABLE_NAME2, c1, pool);
+        table.close();
+        LOG.info("HTable connection " + i + " closed " + c1);
+      } catch (Exception e) {
+        LOG.info("We actually want this to happen!!!!  So we can see if we are leaking zk", e);
+      } finally {
+        if (c1 != null) {
+          if (c1.isClosed()) {
+            // cannot use getZooKeeper as method instantiates watcher if null
+            Field zkwField = c1.getClass().getDeclaredField("zooKeeper");
+            zkwField.setAccessible(true);
+            Object watcher = zkwField.get(c1);
+
+            if (watcher != null) {
+              if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
+                // non-synchronized access to watcher; sleep and check again in case zk connection
+                // hasn't been cleaned up yet.
+                Thread.sleep(1000);
+                if (((ZooKeeperWatcher) watcher).getRecoverableZooKeeper().getState().isAlive()) {
                   pool.shutdownNow();
                   fail("Live zookeeper in closed connection");
                 }
               }
             }
-            c1.close();
           }
+          c1.close();
         }
+      }
     }
-
     pool.shutdownNow();
   }
 

From 75834f0262194bcfe21a88790a2a8b2d27c0fc3b Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 17 Jul 2013 18:53:27 +0000
Subject: [PATCH 1062/1540] HBASE-8971: Bump 0.94 version to 0.94.10-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504233 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 6b8cdaa71fe5..e427b0e620bd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.9</version>
+  <version>0.94.10-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 01ce487ae14e61cab84dc6f9f2e3434e0e29f7d8 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Wed, 17 Jul 2013 20:06:07 +0000
Subject: [PATCH 1063/1540] HBASE-8779 Add mutateRow method support to Thrift2
 (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504255 13f79535-47bb-0310-9956-ffa450edef68
---
 .../thrift2/ThriftHBaseServiceHandler.java    |   13 +
 .../hadoop/hbase/thrift2/ThriftUtilities.java |   21 +
 .../thrift2/generated/THBaseService.java      | 1147 +++++++++++++++--
 .../hbase/thrift2/generated/TMutation.java    |  357 +++++
 .../thrift2/generated/TRowMutations.java      |  539 ++++++++
 .../apache/hadoop/hbase/thrift2/hbase.thrift  |   27 +
 .../TestThriftHBaseServiceHandler.java        |   67 +-
 7 files changed, 2059 insertions(+), 112 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index 0089568cce3d..41805c87f683 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -30,6 +30,7 @@
 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.resultFromHBase;
 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.resultsFromHBase;
 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift;
+import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.rowMutationsFromThrift;
 import static org.apache.thrift.TBaseHelper.byteBufferToByteArray;
 
 import java.io.IOException;
@@ -60,6 +61,7 @@
 import org.apache.hadoop.hbase.thrift2.generated.TPut;
 import org.apache.hadoop.hbase.thrift2.generated.TResult;
 import org.apache.hadoop.hbase.thrift2.generated.TScan;
+import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
 import org.apache.thrift.TException;
 
 /**
@@ -346,4 +348,15 @@ public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, TExce
     removeScanner(scannerId);
   }
 
+  @Override
+  public void mutateRow(ByteBuffer table, TRowMutations rowMutations) throws TIOError, TException {
+    HTableInterface htable = getTable(table);
+    try {
+      htable.mutateRow(rowMutationsFromThrift(rowMutations));
+    } catch (IOException e) {
+      throw getTIOError(e);
+    } finally {
+      closeTable(htable);
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
index 982b512fcb86..b1373fbde177 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
@@ -315,6 +315,27 @@ public static List<TDelete> deletesFromHBase(List<Delete> in) {
     return out;
   }
 
+  /**
+   * Creates a {@link RowMutations} (HBase) from a {@link TRowMutations} (Thrift)
+   *
+   * @param in the <code>TRowMutations</code> to convert
+   *
+   * @return converted <code>RowMutations</code>
+   */
+  public static RowMutations rowMutationsFromThrift(TRowMutations in) throws IOException {
+    RowMutations out = new RowMutations(in.getRow());
+    List<TMutation> mutations = in.getMutations();
+    for (TMutation mutation : mutations) {
+      if (mutation.isSetPut()) {
+        out.add(putFromThrift(mutation.getPut()));
+      }
+      if (mutation.isSetDeleteSingle()) {
+        out.add(deleteFromThrift(mutation.getDeleteSingle()));
+      }
+    }
+    return out;
+  }
+
   public static Scan scanFromThrift(TScan in) throws IOException {
     Scan out = new Scan();
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
index 9ee49623a75f..ea35d07fac2d 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
@@ -128,9 +128,9 @@ public interface Iface {
     /**
      * Bulk commit a List of TDeletes to the table.
      * 
-     * This returns a list of TDeletes that were not
-     * executed. So if everything succeeds you'll
-     * receive an empty list.
+     * Throws a TIOError if any of the deletes fail.
+     * 
+     * Always returns an empty list for backwards compatibility.
      * 
      * @param table the table to delete from
      * 
@@ -194,6 +194,15 @@ public interface Iface {
      */
     public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, org.apache.thrift.TException;
 
+    /**
+     * mutateRow performs multiple mutations atomically on a single row.
+     * 
+     * @param table table to apply the mutations
+     * 
+     * @param rowMutations mutations to apply
+     */
+    public void mutateRow(ByteBuffer table, TRowMutations rowMutations) throws TIOError, org.apache.thrift.TException;
+
   }
 
   public interface AsyncIface {
@@ -224,6 +233,8 @@ public interface AsyncIface {
 
     public void closeScanner(int scannerId, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.closeScanner_call> resultHandler) throws org.apache.thrift.TException;
 
+    public void mutateRow(ByteBuffer table, TRowMutations rowMutations, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.mutateRow_call> resultHandler) throws org.apache.thrift.TException;
+
   }
 
   public static class Client extends org.apache.thrift.TServiceClient implements Iface {
@@ -598,6 +609,30 @@ public void recv_closeScanner() throws TIOError, TIllegalArgument, org.apache.th
       return;
     }
 
+    public void mutateRow(ByteBuffer table, TRowMutations rowMutations) throws TIOError, org.apache.thrift.TException
+    {
+      send_mutateRow(table, rowMutations);
+      recv_mutateRow();
+    }
+
+    public void send_mutateRow(ByteBuffer table, TRowMutations rowMutations) throws org.apache.thrift.TException
+    {
+      mutateRow_args args = new mutateRow_args();
+      args.setTable(table);
+      args.setRowMutations(rowMutations);
+      sendBase("mutateRow", args);
+    }
+
+    public void recv_mutateRow() throws TIOError, org.apache.thrift.TException
+    {
+      mutateRow_result result = new mutateRow_result();
+      receiveBase(result, "mutateRow");
+      if (result.io != null) {
+        throw result.io;
+      }
+      return;
+    }
+
   }
   public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
     public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
@@ -1092,6 +1127,41 @@ public void getResult() throws TIOError, TIllegalArgument, org.apache.thrift.TEx
       }
     }
 
+    public void mutateRow(ByteBuffer table, TRowMutations rowMutations, org.apache.thrift.async.AsyncMethodCallback<mutateRow_call> resultHandler) throws org.apache.thrift.TException {
+      checkReady();
+      mutateRow_call method_call = new mutateRow_call(table, rowMutations, resultHandler, this, ___protocolFactory, ___transport);
+      this.___currentMethod = method_call;
+      ___manager.call(method_call);
+    }
+
+    public static class mutateRow_call extends org.apache.thrift.async.TAsyncMethodCall {
+      private ByteBuffer table;
+      private TRowMutations rowMutations;
+      public mutateRow_call(ByteBuffer table, TRowMutations rowMutations, org.apache.thrift.async.AsyncMethodCallback<mutateRow_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+        super(client, protocolFactory, transport, resultHandler, false);
+        this.table = table;
+        this.rowMutations = rowMutations;
+      }
+
+      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("mutateRow", org.apache.thrift.protocol.TMessageType.CALL, 0));
+        mutateRow_args args = new mutateRow_args();
+        args.setTable(table);
+        args.setRowMutations(rowMutations);
+        args.write(prot);
+        prot.writeMessageEnd();
+      }
+
+      public void getResult() throws TIOError, org.apache.thrift.TException {
+        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+          throw new IllegalStateException("Method call not finished!");
+        }
+        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+        (new Client(prot)).recv_mutateRow();
+      }
+    }
+
   }
 
   public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
@@ -1118,6 +1188,7 @@ protected Processor(I iface, Map<String,  org.apache.thrift.ProcessFunction<I, ?
       processMap.put("openScanner", new openScanner());
       processMap.put("getScannerRows", new getScannerRows());
       processMap.put("closeScanner", new closeScanner());
+      processMap.put("mutateRow", new mutateRow());
       return processMap;
     }
 
@@ -1389,6 +1460,26 @@ protected closeScanner_result getResult(I iface, closeScanner_args args) throws
       }
     }
 
+    private static class mutateRow<I extends Iface> extends org.apache.thrift.ProcessFunction<I, mutateRow_args> {
+      public mutateRow() {
+        super("mutateRow");
+      }
+
+      protected mutateRow_args getEmptyArgsInstance() {
+        return new mutateRow_args();
+      }
+
+      protected mutateRow_result getResult(I iface, mutateRow_args args) throws org.apache.thrift.TException {
+        mutateRow_result result = new mutateRow_result();
+        try {
+          iface.mutateRow(args.table, args.rowMutations);
+        } catch (TIOError io) {
+          result.io = io;
+        }
+        return result;
+      }
+    }
+
   }
 
   public static class exists_args implements org.apache.thrift.TBase<exists_args, exists_args._Fields>, java.io.Serializable, Cloneable   {
@@ -3702,14 +3793,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_args st
             case 2: // GETS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list88 = iprot.readListBegin();
-                  struct.gets = new ArrayList<TGet>(_list88.size);
-                  for (int _i89 = 0; _i89 < _list88.size; ++_i89)
+                  org.apache.thrift.protocol.TList _list106 = iprot.readListBegin();
+                  struct.gets = new ArrayList<TGet>(_list106.size);
+                  for (int _i107 = 0; _i107 < _list106.size; ++_i107)
                   {
-                    TGet _elem90; // required
-                    _elem90 = new TGet();
-                    _elem90.read(iprot);
-                    struct.gets.add(_elem90);
+                    TGet _elem108; // required
+                    _elem108 = new TGet();
+                    _elem108.read(iprot);
+                    struct.gets.add(_elem108);
                   }
                   iprot.readListEnd();
                 }
@@ -3742,9 +3833,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_args s
           oprot.writeFieldBegin(GETS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.gets.size()));
-            for (TGet _iter91 : struct.gets)
+            for (TGet _iter109 : struct.gets)
             {
-              _iter91.write(oprot);
+              _iter109.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -3770,9 +3861,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_args st
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.gets.size());
-          for (TGet _iter92 : struct.gets)
+          for (TGet _iter110 : struct.gets)
           {
-            _iter92.write(oprot);
+            _iter110.write(oprot);
           }
         }
       }
@@ -3783,14 +3874,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_args str
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.gets = new ArrayList<TGet>(_list93.size);
-          for (int _i94 = 0; _i94 < _list93.size; ++_i94)
+          org.apache.thrift.protocol.TList _list111 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.gets = new ArrayList<TGet>(_list111.size);
+          for (int _i112 = 0; _i112 < _list111.size; ++_i112)
           {
-            TGet _elem95; // required
-            _elem95 = new TGet();
-            _elem95.read(iprot);
-            struct.gets.add(_elem95);
+            TGet _elem113; // required
+            _elem113 = new TGet();
+            _elem113.read(iprot);
+            struct.gets.add(_elem113);
           }
         }
         struct.setGetsIsSet(true);
@@ -4184,14 +4275,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_result
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list96 = iprot.readListBegin();
-                  struct.success = new ArrayList<TResult>(_list96.size);
-                  for (int _i97 = 0; _i97 < _list96.size; ++_i97)
+                  org.apache.thrift.protocol.TList _list114 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list114.size);
+                  for (int _i115 = 0; _i115 < _list114.size; ++_i115)
                   {
-                    TResult _elem98; // required
-                    _elem98 = new TResult();
-                    _elem98.read(iprot);
-                    struct.success.add(_elem98);
+                    TResult _elem116; // required
+                    _elem116 = new TResult();
+                    _elem116.read(iprot);
+                    struct.success.add(_elem116);
                   }
                   iprot.readListEnd();
                 }
@@ -4228,9 +4319,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_result
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TResult _iter99 : struct.success)
+            for (TResult _iter117 : struct.success)
             {
-              _iter99.write(oprot);
+              _iter117.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -4269,9 +4360,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_result
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TResult _iter100 : struct.success)
+            for (TResult _iter118 : struct.success)
             {
-              _iter100.write(oprot);
+              _iter118.write(oprot);
             }
           }
         }
@@ -4286,14 +4377,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_result s
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list101 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TResult>(_list101.size);
-            for (int _i102 = 0; _i102 < _list101.size; ++_i102)
+            org.apache.thrift.protocol.TList _list119 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list119.size);
+            for (int _i120 = 0; _i120 < _list119.size; ++_i120)
             {
-              TResult _elem103; // required
-              _elem103 = new TResult();
-              _elem103.read(iprot);
-              struct.success.add(_elem103);
+              TResult _elem121; // required
+              _elem121 = new TResult();
+              _elem121.read(iprot);
+              struct.success.add(_elem121);
             }
           }
           struct.setSuccessIsSet(true);
@@ -7000,14 +7091,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, putMultiple_args st
             case 2: // PUTS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list104 = iprot.readListBegin();
-                  struct.puts = new ArrayList<TPut>(_list104.size);
-                  for (int _i105 = 0; _i105 < _list104.size; ++_i105)
+                  org.apache.thrift.protocol.TList _list122 = iprot.readListBegin();
+                  struct.puts = new ArrayList<TPut>(_list122.size);
+                  for (int _i123 = 0; _i123 < _list122.size; ++_i123)
                   {
-                    TPut _elem106; // required
-                    _elem106 = new TPut();
-                    _elem106.read(iprot);
-                    struct.puts.add(_elem106);
+                    TPut _elem124; // required
+                    _elem124 = new TPut();
+                    _elem124.read(iprot);
+                    struct.puts.add(_elem124);
                   }
                   iprot.readListEnd();
                 }
@@ -7040,9 +7131,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, putMultiple_args s
           oprot.writeFieldBegin(PUTS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.puts.size()));
-            for (TPut _iter107 : struct.puts)
+            for (TPut _iter125 : struct.puts)
             {
-              _iter107.write(oprot);
+              _iter125.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -7068,9 +7159,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, putMultiple_args st
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.puts.size());
-          for (TPut _iter108 : struct.puts)
+          for (TPut _iter126 : struct.puts)
           {
-            _iter108.write(oprot);
+            _iter126.write(oprot);
           }
         }
       }
@@ -7081,14 +7172,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, putMultiple_args str
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list109 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.puts = new ArrayList<TPut>(_list109.size);
-          for (int _i110 = 0; _i110 < _list109.size; ++_i110)
+          org.apache.thrift.protocol.TList _list127 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.puts = new ArrayList<TPut>(_list127.size);
+          for (int _i128 = 0; _i128 < _list127.size; ++_i128)
           {
-            TPut _elem111; // required
-            _elem111 = new TPut();
-            _elem111.read(iprot);
-            struct.puts.add(_elem111);
+            TPut _elem129; // required
+            _elem129 = new TPut();
+            _elem129.read(iprot);
+            struct.puts.add(_elem129);
           }
         }
         struct.setPutsIsSet(true);
@@ -8720,14 +8811,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_args
             case 2: // DELETES
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list112 = iprot.readListBegin();
-                  struct.deletes = new ArrayList<TDelete>(_list112.size);
-                  for (int _i113 = 0; _i113 < _list112.size; ++_i113)
+                  org.apache.thrift.protocol.TList _list130 = iprot.readListBegin();
+                  struct.deletes = new ArrayList<TDelete>(_list130.size);
+                  for (int _i131 = 0; _i131 < _list130.size; ++_i131)
                   {
-                    TDelete _elem114; // required
-                    _elem114 = new TDelete();
-                    _elem114.read(iprot);
-                    struct.deletes.add(_elem114);
+                    TDelete _elem132; // required
+                    _elem132 = new TDelete();
+                    _elem132.read(iprot);
+                    struct.deletes.add(_elem132);
                   }
                   iprot.readListEnd();
                 }
@@ -8760,9 +8851,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_arg
           oprot.writeFieldBegin(DELETES_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.deletes.size()));
-            for (TDelete _iter115 : struct.deletes)
+            for (TDelete _iter133 : struct.deletes)
             {
-              _iter115.write(oprot);
+              _iter133.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -8788,9 +8879,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.deletes.size());
-          for (TDelete _iter116 : struct.deletes)
+          for (TDelete _iter134 : struct.deletes)
           {
-            _iter116.write(oprot);
+            _iter134.write(oprot);
           }
         }
       }
@@ -8801,14 +8892,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list117 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.deletes = new ArrayList<TDelete>(_list117.size);
-          for (int _i118 = 0; _i118 < _list117.size; ++_i118)
+          org.apache.thrift.protocol.TList _list135 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.deletes = new ArrayList<TDelete>(_list135.size);
+          for (int _i136 = 0; _i136 < _list135.size; ++_i136)
           {
-            TDelete _elem119; // required
-            _elem119 = new TDelete();
-            _elem119.read(iprot);
-            struct.deletes.add(_elem119);
+            TDelete _elem137; // required
+            _elem137 = new TDelete();
+            _elem137.read(iprot);
+            struct.deletes.add(_elem137);
           }
         }
         struct.setDeletesIsSet(true);
@@ -9202,14 +9293,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list120 = iprot.readListBegin();
-                  struct.success = new ArrayList<TDelete>(_list120.size);
-                  for (int _i121 = 0; _i121 < _list120.size; ++_i121)
+                  org.apache.thrift.protocol.TList _list138 = iprot.readListBegin();
+                  struct.success = new ArrayList<TDelete>(_list138.size);
+                  for (int _i139 = 0; _i139 < _list138.size; ++_i139)
                   {
-                    TDelete _elem122; // required
-                    _elem122 = new TDelete();
-                    _elem122.read(iprot);
-                    struct.success.add(_elem122);
+                    TDelete _elem140; // required
+                    _elem140 = new TDelete();
+                    _elem140.read(iprot);
+                    struct.success.add(_elem140);
                   }
                   iprot.readListEnd();
                 }
@@ -9246,9 +9337,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TDelete _iter123 : struct.success)
+            for (TDelete _iter141 : struct.success)
             {
-              _iter123.write(oprot);
+              _iter141.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -9287,9 +9378,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TDelete _iter124 : struct.success)
+            for (TDelete _iter142 : struct.success)
             {
-              _iter124.write(oprot);
+              _iter142.write(oprot);
             }
           }
         }
@@ -9304,14 +9395,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resul
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list125 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TDelete>(_list125.size);
-            for (int _i126 = 0; _i126 < _list125.size; ++_i126)
+            org.apache.thrift.protocol.TList _list143 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TDelete>(_list143.size);
+            for (int _i144 = 0; _i144 < _list143.size; ++_i144)
             {
-              TDelete _elem127; // required
-              _elem127 = new TDelete();
-              _elem127.read(iprot);
-              struct.success.add(_elem127);
+              TDelete _elem145; // required
+              _elem145 = new TDelete();
+              _elem145.read(iprot);
+              struct.success.add(_elem145);
             }
           }
           struct.setSuccessIsSet(true);
@@ -13562,14 +13653,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerRows_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list128 = iprot.readListBegin();
-                  struct.success = new ArrayList<TResult>(_list128.size);
-                  for (int _i129 = 0; _i129 < _list128.size; ++_i129)
+                  org.apache.thrift.protocol.TList _list146 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list146.size);
+                  for (int _i147 = 0; _i147 < _list146.size; ++_i147)
                   {
-                    TResult _elem130; // required
-                    _elem130 = new TResult();
-                    _elem130.read(iprot);
-                    struct.success.add(_elem130);
+                    TResult _elem148; // required
+                    _elem148 = new TResult();
+                    _elem148.read(iprot);
+                    struct.success.add(_elem148);
                   }
                   iprot.readListEnd();
                 }
@@ -13615,9 +13706,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerRows_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TResult _iter131 : struct.success)
+            for (TResult _iter149 : struct.success)
             {
-              _iter131.write(oprot);
+              _iter149.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -13664,9 +13755,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TResult _iter132 : struct.success)
+            for (TResult _iter150 : struct.success)
             {
-              _iter132.write(oprot);
+              _iter150.write(oprot);
             }
           }
         }
@@ -13684,14 +13775,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resul
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list133 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TResult>(_list133.size);
-            for (int _i134 = 0; _i134 < _list133.size; ++_i134)
+            org.apache.thrift.protocol.TList _list151 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list151.size);
+            for (int _i152 = 0; _i152 < _list151.size; ++_i152)
             {
-              TResult _elem135; // required
-              _elem135 = new TResult();
-              _elem135.read(iprot);
-              struct.success.add(_elem135);
+              TResult _elem153; // required
+              _elem153 = new TResult();
+              _elem153.read(iprot);
+              struct.success.add(_elem153);
             }
           }
           struct.setSuccessIsSet(true);
@@ -14538,4 +14629,838 @@ public void read(org.apache.thrift.protocol.TProtocol prot, closeScanner_result
 
   }
 
+  public static class mutateRow_args implements org.apache.thrift.TBase<mutateRow_args, mutateRow_args._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRow_args");
+
+    private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1);
+    private static final org.apache.thrift.protocol.TField ROW_MUTATIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("rowMutations", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new mutateRow_argsStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new mutateRow_argsTupleSchemeFactory());
+    }
+
+    /**
+     * table to apply the mutations
+     */
+    public ByteBuffer table; // required
+    /**
+     * mutations to apply
+     */
+    public TRowMutations rowMutations; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      /**
+       * table to apply the mutations
+       */
+      TABLE((short)1, "table"),
+      /**
+       * mutations to apply
+       */
+      ROW_MUTATIONS((short)2, "rowMutations");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // TABLE
+            return TABLE;
+          case 2: // ROW_MUTATIONS
+            return ROW_MUTATIONS;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING          , true)));
+      tmpMap.put(_Fields.ROW_MUTATIONS, new org.apache.thrift.meta_data.FieldMetaData("rowMutations", org.apache.thrift.TFieldRequirementType.REQUIRED, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowMutations.class)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRow_args.class, metaDataMap);
+    }
+
+    public mutateRow_args() {
+    }
+
+    public mutateRow_args(
+      ByteBuffer table,
+      TRowMutations rowMutations)
+    {
+      this();
+      this.table = table;
+      this.rowMutations = rowMutations;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public mutateRow_args(mutateRow_args other) {
+      if (other.isSetTable()) {
+        this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table);
+;
+      }
+      if (other.isSetRowMutations()) {
+        this.rowMutations = new TRowMutations(other.rowMutations);
+      }
+    }
+
+    public mutateRow_args deepCopy() {
+      return new mutateRow_args(this);
+    }
+
+    @Override
+    public void clear() {
+      this.table = null;
+      this.rowMutations = null;
+    }
+
+    /**
+     * table to apply the mutations
+     */
+    public byte[] getTable() {
+      setTable(org.apache.thrift.TBaseHelper.rightSize(table));
+      return table == null ? null : table.array();
+    }
+
+    public ByteBuffer bufferForTable() {
+      return table;
+    }
+
+    /**
+     * table to apply the mutations
+     */
+    public mutateRow_args setTable(byte[] table) {
+      setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table));
+      return this;
+    }
+
+    public mutateRow_args setTable(ByteBuffer table) {
+      this.table = table;
+      return this;
+    }
+
+    public void unsetTable() {
+      this.table = null;
+    }
+
+    /** Returns true if field table is set (has been assigned a value) and false otherwise */
+    public boolean isSetTable() {
+      return this.table != null;
+    }
+
+    public void setTableIsSet(boolean value) {
+      if (!value) {
+        this.table = null;
+      }
+    }
+
+    /**
+     * mutations to apply
+     */
+    public TRowMutations getRowMutations() {
+      return this.rowMutations;
+    }
+
+    /**
+     * mutations to apply
+     */
+    public mutateRow_args setRowMutations(TRowMutations rowMutations) {
+      this.rowMutations = rowMutations;
+      return this;
+    }
+
+    public void unsetRowMutations() {
+      this.rowMutations = null;
+    }
+
+    /** Returns true if field rowMutations is set (has been assigned a value) and false otherwise */
+    public boolean isSetRowMutations() {
+      return this.rowMutations != null;
+    }
+
+    public void setRowMutationsIsSet(boolean value) {
+      if (!value) {
+        this.rowMutations = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case TABLE:
+        if (value == null) {
+          unsetTable();
+        } else {
+          setTable((ByteBuffer)value);
+        }
+        break;
+
+      case ROW_MUTATIONS:
+        if (value == null) {
+          unsetRowMutations();
+        } else {
+          setRowMutations((TRowMutations)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case TABLE:
+        return getTable();
+
+      case ROW_MUTATIONS:
+        return getRowMutations();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case TABLE:
+        return isSetTable();
+      case ROW_MUTATIONS:
+        return isSetRowMutations();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof mutateRow_args)
+        return this.equals((mutateRow_args)that);
+      return false;
+    }
+
+    public boolean equals(mutateRow_args that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_table = true && this.isSetTable();
+      boolean that_present_table = true && that.isSetTable();
+      if (this_present_table || that_present_table) {
+        if (!(this_present_table && that_present_table))
+          return false;
+        if (!this.table.equals(that.table))
+          return false;
+      }
+
+      boolean this_present_rowMutations = true && this.isSetRowMutations();
+      boolean that_present_rowMutations = true && that.isSetRowMutations();
+      if (this_present_rowMutations || that_present_rowMutations) {
+        if (!(this_present_rowMutations && that_present_rowMutations))
+          return false;
+        if (!this.rowMutations.equals(that.rowMutations))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(mutateRow_args other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      mutateRow_args typedOther = (mutateRow_args)other;
+
+      lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetTable()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      lastComparison = Boolean.valueOf(isSetRowMutations()).compareTo(typedOther.isSetRowMutations());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetRowMutations()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rowMutations, typedOther.rowMutations);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("mutateRow_args(");
+      boolean first = true;
+
+      sb.append("table:");
+      if (this.table == null) {
+        sb.append("null");
+      } else {
+        org.apache.thrift.TBaseHelper.toString(this.table, sb);
+      }
+      first = false;
+      if (!first) sb.append(", ");
+      sb.append("rowMutations:");
+      if (this.rowMutations == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.rowMutations);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+      if (table == null) {
+        throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString());
+      }
+      if (rowMutations == null) {
+        throw new org.apache.thrift.protocol.TProtocolException("Required field 'rowMutations' was not present! Struct: " + toString());
+      }
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class mutateRow_argsStandardSchemeFactory implements SchemeFactory {
+      public mutateRow_argsStandardScheme getScheme() {
+        return new mutateRow_argsStandardScheme();
+      }
+    }
+
+    private static class mutateRow_argsStandardScheme extends StandardScheme<mutateRow_args> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_args struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // TABLE
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+                struct.table = iprot.readBinary();
+                struct.setTableIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            case 2: // ROW_MUTATIONS
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.rowMutations = new TRowMutations();
+                struct.rowMutations.read(iprot);
+                struct.setRowMutationsIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRow_args struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.table != null) {
+          oprot.writeFieldBegin(TABLE_FIELD_DESC);
+          oprot.writeBinary(struct.table);
+          oprot.writeFieldEnd();
+        }
+        if (struct.rowMutations != null) {
+          oprot.writeFieldBegin(ROW_MUTATIONS_FIELD_DESC);
+          struct.rowMutations.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class mutateRow_argsTupleSchemeFactory implements SchemeFactory {
+      public mutateRow_argsTupleScheme getScheme() {
+        return new mutateRow_argsTupleScheme();
+      }
+    }
+
+    private static class mutateRow_argsTupleScheme extends TupleScheme<mutateRow_args> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, mutateRow_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        oprot.writeBinary(struct.table);
+        struct.rowMutations.write(oprot);
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        struct.table = iprot.readBinary();
+        struct.setTableIsSet(true);
+        struct.rowMutations = new TRowMutations();
+        struct.rowMutations.read(iprot);
+        struct.setRowMutationsIsSet(true);
+      }
+    }
+
+  }
+
+  public static class mutateRow_result implements org.apache.thrift.TBase<mutateRow_result, mutateRow_result._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRow_result");
+
+    private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new mutateRow_resultStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new mutateRow_resultTupleSchemeFactory());
+    }
+
+    public TIOError io; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      IO((short)1, "io");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // IO
+            return IO;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRow_result.class, metaDataMap);
+    }
+
+    public mutateRow_result() {
+    }
+
+    public mutateRow_result(
+      TIOError io)
+    {
+      this();
+      this.io = io;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public mutateRow_result(mutateRow_result other) {
+      if (other.isSetIo()) {
+        this.io = new TIOError(other.io);
+      }
+    }
+
+    public mutateRow_result deepCopy() {
+      return new mutateRow_result(this);
+    }
+
+    @Override
+    public void clear() {
+      this.io = null;
+    }
+
+    public TIOError getIo() {
+      return this.io;
+    }
+
+    public mutateRow_result setIo(TIOError io) {
+      this.io = io;
+      return this;
+    }
+
+    public void unsetIo() {
+      this.io = null;
+    }
+
+    /** Returns true if field io is set (has been assigned a value) and false otherwise */
+    public boolean isSetIo() {
+      return this.io != null;
+    }
+
+    public void setIoIsSet(boolean value) {
+      if (!value) {
+        this.io = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case IO:
+        if (value == null) {
+          unsetIo();
+        } else {
+          setIo((TIOError)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case IO:
+        return getIo();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case IO:
+        return isSetIo();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof mutateRow_result)
+        return this.equals((mutateRow_result)that);
+      return false;
+    }
+
+    public boolean equals(mutateRow_result that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_io = true && this.isSetIo();
+      boolean that_present_io = true && that.isSetIo();
+      if (this_present_io || that_present_io) {
+        if (!(this_present_io && that_present_io))
+          return false;
+        if (!this.io.equals(that.io))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(mutateRow_result other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      mutateRow_result typedOther = (mutateRow_result)other;
+
+      lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetIo()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+      }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("mutateRow_result(");
+      boolean first = true;
+
+      sb.append("io:");
+      if (this.io == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.io);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class mutateRow_resultStandardSchemeFactory implements SchemeFactory {
+      public mutateRow_resultStandardScheme getScheme() {
+        return new mutateRow_resultStandardScheme();
+      }
+    }
+
+    private static class mutateRow_resultStandardScheme extends StandardScheme<mutateRow_result> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_result struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // IO
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.io = new TIOError();
+                struct.io.read(iprot);
+                struct.setIoIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRow_result struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.io != null) {
+          oprot.writeFieldBegin(IO_FIELD_DESC);
+          struct.io.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class mutateRow_resultTupleSchemeFactory implements SchemeFactory {
+      public mutateRow_resultTupleScheme getScheme() {
+        return new mutateRow_resultTupleScheme();
+      }
+    }
+
+    private static class mutateRow_resultTupleScheme extends TupleScheme<mutateRow_result> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, mutateRow_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        BitSet optionals = new BitSet();
+        if (struct.isSetIo()) {
+          optionals.set(0);
+        }
+        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetIo()) {
+          struct.io.write(oprot);
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        BitSet incoming = iprot.readBitSet(1);
+        if (incoming.get(0)) {
+          struct.io = new TIOError();
+          struct.io.read(iprot);
+          struct.setIoIsSet(true);
+        }
+      }
+    }
+
+  }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java
new file mode 100644
index 000000000000..5c409528600a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java
@@ -0,0 +1,357 @@
+/**
+ * Autogenerated by Thrift Compiler (0.8.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hbase.thrift2.generated;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Atomic mutation for the specified row. It can be either Put or Delete.
+ */
+public class TMutation extends org.apache.thrift.TUnion<TMutation, TMutation._Fields> {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TMutation");
+  private static final org.apache.thrift.protocol.TField PUT_FIELD_DESC = new org.apache.thrift.protocol.TField("put", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+  private static final org.apache.thrift.protocol.TField DELETE_SINGLE_FIELD_DESC = new org.apache.thrift.protocol.TField("deleteSingle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+    PUT((short)1, "put"),
+    DELETE_SINGLE((short)2, "deleteSingle");
+
+    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+    static {
+      for (_Fields field : EnumSet.allOf(_Fields.class)) {
+        byName.put(field.getFieldName(), field);
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, or null if its not found.
+     */
+    public static _Fields findByThriftId(int fieldId) {
+      switch(fieldId) {
+        case 1: // PUT
+          return PUT;
+        case 2: // DELETE_SINGLE
+          return DELETE_SINGLE;
+        default:
+          return null;
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, throwing an exception
+     * if it is not found.
+     */
+    public static _Fields findByThriftIdOrThrow(int fieldId) {
+      _Fields fields = findByThriftId(fieldId);
+      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+      return fields;
+    }
+
+    /**
+     * Find the _Fields constant that matches name, or null if its not found.
+     */
+    public static _Fields findByName(String name) {
+      return byName.get(name);
+    }
+
+    private final short _thriftId;
+    private final String _fieldName;
+
+    _Fields(short thriftId, String fieldName) {
+      _thriftId = thriftId;
+      _fieldName = fieldName;
+    }
+
+    public short getThriftFieldId() {
+      return _thriftId;
+    }
+
+    public String getFieldName() {
+      return _fieldName;
+    }
+  }
+
+  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+  static {
+    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+    tmpMap.put(_Fields.PUT, new org.apache.thrift.meta_data.FieldMetaData("put", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TPut.class)));
+    tmpMap.put(_Fields.DELETE_SINGLE, new org.apache.thrift.meta_data.FieldMetaData("deleteSingle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDelete.class)));
+    metaDataMap = Collections.unmodifiableMap(tmpMap);
+    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TMutation.class, metaDataMap);
+  }
+
+  public TMutation() {
+    super();
+  }
+
+  public TMutation(_Fields setField, Object value) {
+    super(setField, value);
+  }
+
+  public TMutation(TMutation other) {
+    super(other);
+  }
+  public TMutation deepCopy() {
+    return new TMutation(this);
+  }
+
+  public static TMutation put(TPut value) {
+    TMutation x = new TMutation();
+    x.setPut(value);
+    return x;
+  }
+
+  public static TMutation deleteSingle(TDelete value) {
+    TMutation x = new TMutation();
+    x.setDeleteSingle(value);
+    return x;
+  }
+
+
+  @Override
+  protected void checkType(_Fields setField, Object value) throws ClassCastException {
+    switch (setField) {
+      case PUT:
+        if (value instanceof TPut) {
+          break;
+        }
+        throw new ClassCastException("Was expecting value of type TPut for field 'put', but got " + value.getClass().getSimpleName());
+      case DELETE_SINGLE:
+        if (value instanceof TDelete) {
+          break;
+        }
+        throw new ClassCastException("Was expecting value of type TDelete for field 'deleteSingle', but got " + value.getClass().getSimpleName());
+      default:
+        throw new IllegalArgumentException("Unknown field id " + setField);
+    }
+  }
+
+  @Override
+  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
+    _Fields setField = _Fields.findByThriftId(field.id);
+    if (setField != null) {
+      switch (setField) {
+        case PUT:
+          if (field.type == PUT_FIELD_DESC.type) {
+            TPut put;
+            put = new TPut();
+            put.read(iprot);
+            return put;
+          } else {
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+            return null;
+          }
+        case DELETE_SINGLE:
+          if (field.type == DELETE_SINGLE_FIELD_DESC.type) {
+            TDelete deleteSingle;
+            deleteSingle = new TDelete();
+            deleteSingle.read(iprot);
+            return deleteSingle;
+          } else {
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+            return null;
+          }
+        default:
+          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
+      }
+    } else {
+      return null;
+    }
+  }
+
+  @Override
+  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+    switch (setField_) {
+      case PUT:
+        TPut put = (TPut)value_;
+        put.write(oprot);
+        return;
+      case DELETE_SINGLE:
+        TDelete deleteSingle = (TDelete)value_;
+        deleteSingle.write(oprot);
+        return;
+      default:
+        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
+    }
+  }
+
+  @Override
+  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
+    _Fields setField = _Fields.findByThriftId(fieldID);
+    if (setField != null) {
+      switch (setField) {
+        case PUT:
+          TPut put;
+          put = new TPut();
+          put.read(iprot);
+          return put;
+        case DELETE_SINGLE:
+          TDelete deleteSingle;
+          deleteSingle = new TDelete();
+          deleteSingle.read(iprot);
+          return deleteSingle;
+        default:
+          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
+      }
+    } else {
+      return null;
+    }
+  }
+
+  @Override
+  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+    switch (setField_) {
+      case PUT:
+        TPut put = (TPut)value_;
+        put.write(oprot);
+        return;
+      case DELETE_SINGLE:
+        TDelete deleteSingle = (TDelete)value_;
+        deleteSingle.write(oprot);
+        return;
+      default:
+        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
+    }
+  }
+
+  @Override
+  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
+    switch (setField) {
+      case PUT:
+        return PUT_FIELD_DESC;
+      case DELETE_SINGLE:
+        return DELETE_SINGLE_FIELD_DESC;
+      default:
+        throw new IllegalArgumentException("Unknown field id " + setField);
+    }
+  }
+
+  @Override
+  protected org.apache.thrift.protocol.TStruct getStructDesc() {
+    return STRUCT_DESC;
+  }
+
+  @Override
+  protected _Fields enumForId(short id) {
+    return _Fields.findByThriftIdOrThrow(id);
+  }
+
+  public _Fields fieldForId(int fieldId) {
+    return _Fields.findByThriftId(fieldId);
+  }
+
+
+  public TPut getPut() {
+    if (getSetField() == _Fields.PUT) {
+      return (TPut)getFieldValue();
+    } else {
+      throw new RuntimeException("Cannot get field 'put' because union is currently set to " + getFieldDesc(getSetField()).name);
+    }
+  }
+
+  public void setPut(TPut value) {
+    if (value == null) throw new NullPointerException();
+    setField_ = _Fields.PUT;
+    value_ = value;
+  }
+
+  public TDelete getDeleteSingle() {
+    if (getSetField() == _Fields.DELETE_SINGLE) {
+      return (TDelete)getFieldValue();
+    } else {
+      throw new RuntimeException("Cannot get field 'deleteSingle' because union is currently set to " + getFieldDesc(getSetField()).name);
+    }
+  }
+
+  public void setDeleteSingle(TDelete value) {
+    if (value == null) throw new NullPointerException();
+    setField_ = _Fields.DELETE_SINGLE;
+    value_ = value;
+  }
+
+  public boolean isSetPut() {
+    return setField_ == _Fields.PUT;
+  }
+
+
+  public boolean isSetDeleteSingle() {
+    return setField_ == _Fields.DELETE_SINGLE;
+  }
+
+
+  public boolean equals(Object other) {
+    if (other instanceof TMutation) {
+      return equals((TMutation)other);
+    } else {
+      return false;
+    }
+  }
+
+  public boolean equals(TMutation other) {
+    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
+  }
+
+  @Override
+  public int compareTo(TMutation other) {
+    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
+    if (lastComparison == 0) {
+      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
+    }
+    return lastComparison;
+  }
+
+
+  /**
+   * If you'd like this to perform more respectably, use the hashcode generator option.
+   */
+  @Override
+  public int hashCode() {
+    return 0;
+  }
+
+  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+    try {
+      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+
+  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+    try {
+      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java
new file mode 100644
index 000000000000..44e66789206d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java
@@ -0,0 +1,539 @@
+/**
+ * Autogenerated by Thrift Compiler (0.8.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hbase.thrift2.generated;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A TRowMutations object is used to apply a number of Mutations to a single row.
+ */
+public class TRowMutations implements org.apache.thrift.TBase<TRowMutations, TRowMutations._Fields>, java.io.Serializable, Cloneable {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRowMutations");
+
+  private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1);
+  private static final org.apache.thrift.protocol.TField MUTATIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("mutations", org.apache.thrift.protocol.TType.LIST, (short)2);
+
+  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+  static {
+    schemes.put(StandardScheme.class, new TRowMutationsStandardSchemeFactory());
+    schemes.put(TupleScheme.class, new TRowMutationsTupleSchemeFactory());
+  }
+
+  public ByteBuffer row; // required
+  public List<TMutation> mutations; // required
+
+  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+    ROW((short)1, "row"),
+    MUTATIONS((short)2, "mutations");
+
+    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+    static {
+      for (_Fields field : EnumSet.allOf(_Fields.class)) {
+        byName.put(field.getFieldName(), field);
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, or null if its not found.
+     */
+    public static _Fields findByThriftId(int fieldId) {
+      switch(fieldId) {
+        case 1: // ROW
+          return ROW;
+        case 2: // MUTATIONS
+          return MUTATIONS;
+        default:
+          return null;
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, throwing an exception
+     * if it is not found.
+     */
+    public static _Fields findByThriftIdOrThrow(int fieldId) {
+      _Fields fields = findByThriftId(fieldId);
+      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+      return fields;
+    }
+
+    /**
+     * Find the _Fields constant that matches name, or null if its not found.
+     */
+    public static _Fields findByName(String name) {
+      return byName.get(name);
+    }
+
+    private final short _thriftId;
+    private final String _fieldName;
+
+    _Fields(short thriftId, String fieldName) {
+      _thriftId = thriftId;
+      _fieldName = fieldName;
+    }
+
+    public short getThriftFieldId() {
+      return _thriftId;
+    }
+
+    public String getFieldName() {
+      return _fieldName;
+    }
+  }
+
+  // isset id assignments
+  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+  static {
+    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+    tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
+    tmpMap.put(_Fields.MUTATIONS, new org.apache.thrift.meta_data.FieldMetaData("mutations", org.apache.thrift.TFieldRequirementType.REQUIRED, 
+        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TMutation.class))));
+    metaDataMap = Collections.unmodifiableMap(tmpMap);
+    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRowMutations.class, metaDataMap);
+  }
+
+  public TRowMutations() {
+  }
+
+  public TRowMutations(
+    ByteBuffer row,
+    List<TMutation> mutations)
+  {
+    this();
+    this.row = row;
+    this.mutations = mutations;
+  }
+
+  /**
+   * Performs a deep copy on <i>other</i>.
+   */
+  public TRowMutations(TRowMutations other) {
+    if (other.isSetRow()) {
+      this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row);
+;
+    }
+    if (other.isSetMutations()) {
+      List<TMutation> __this__mutations = new ArrayList<TMutation>();
+      for (TMutation other_element : other.mutations) {
+        __this__mutations.add(new TMutation(other_element));
+      }
+      this.mutations = __this__mutations;
+    }
+  }
+
+  public TRowMutations deepCopy() {
+    return new TRowMutations(this);
+  }
+
+  @Override
+  public void clear() {
+    this.row = null;
+    this.mutations = null;
+  }
+
+  public byte[] getRow() {
+    setRow(org.apache.thrift.TBaseHelper.rightSize(row));
+    return row == null ? null : row.array();
+  }
+
+  public ByteBuffer bufferForRow() {
+    return row;
+  }
+
+  public TRowMutations setRow(byte[] row) {
+    setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row));
+    return this;
+  }
+
+  public TRowMutations setRow(ByteBuffer row) {
+    this.row = row;
+    return this;
+  }
+
+  public void unsetRow() {
+    this.row = null;
+  }
+
+  /** Returns true if field row is set (has been assigned a value) and false otherwise */
+  public boolean isSetRow() {
+    return this.row != null;
+  }
+
+  public void setRowIsSet(boolean value) {
+    if (!value) {
+      this.row = null;
+    }
+  }
+
+  public int getMutationsSize() {
+    return (this.mutations == null) ? 0 : this.mutations.size();
+  }
+
+  public java.util.Iterator<TMutation> getMutationsIterator() {
+    return (this.mutations == null) ? null : this.mutations.iterator();
+  }
+
+  public void addToMutations(TMutation elem) {
+    if (this.mutations == null) {
+      this.mutations = new ArrayList<TMutation>();
+    }
+    this.mutations.add(elem);
+  }
+
+  public List<TMutation> getMutations() {
+    return this.mutations;
+  }
+
+  public TRowMutations setMutations(List<TMutation> mutations) {
+    this.mutations = mutations;
+    return this;
+  }
+
+  public void unsetMutations() {
+    this.mutations = null;
+  }
+
+  /** Returns true if field mutations is set (has been assigned a value) and false otherwise */
+  public boolean isSetMutations() {
+    return this.mutations != null;
+  }
+
+  public void setMutationsIsSet(boolean value) {
+    if (!value) {
+      this.mutations = null;
+    }
+  }
+
+  public void setFieldValue(_Fields field, Object value) {
+    switch (field) {
+    case ROW:
+      if (value == null) {
+        unsetRow();
+      } else {
+        setRow((ByteBuffer)value);
+      }
+      break;
+
+    case MUTATIONS:
+      if (value == null) {
+        unsetMutations();
+      } else {
+        setMutations((List<TMutation>)value);
+      }
+      break;
+
+    }
+  }
+
+  public Object getFieldValue(_Fields field) {
+    switch (field) {
+    case ROW:
+      return getRow();
+
+    case MUTATIONS:
+      return getMutations();
+
+    }
+    throw new IllegalStateException();
+  }
+
+  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+  public boolean isSet(_Fields field) {
+    if (field == null) {
+      throw new IllegalArgumentException();
+    }
+
+    switch (field) {
+    case ROW:
+      return isSetRow();
+    case MUTATIONS:
+      return isSetMutations();
+    }
+    throw new IllegalStateException();
+  }
+
+  @Override
+  public boolean equals(Object that) {
+    if (that == null)
+      return false;
+    if (that instanceof TRowMutations)
+      return this.equals((TRowMutations)that);
+    return false;
+  }
+
+  public boolean equals(TRowMutations that) {
+    if (that == null)
+      return false;
+
+    boolean this_present_row = true && this.isSetRow();
+    boolean that_present_row = true && that.isSetRow();
+    if (this_present_row || that_present_row) {
+      if (!(this_present_row && that_present_row))
+        return false;
+      if (!this.row.equals(that.row))
+        return false;
+    }
+
+    boolean this_present_mutations = true && this.isSetMutations();
+    boolean that_present_mutations = true && that.isSetMutations();
+    if (this_present_mutations || that_present_mutations) {
+      if (!(this_present_mutations && that_present_mutations))
+        return false;
+      if (!this.mutations.equals(that.mutations))
+        return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return 0;
+  }
+
+  public int compareTo(TRowMutations other) {
+    if (!getClass().equals(other.getClass())) {
+      return getClass().getName().compareTo(other.getClass().getName());
+    }
+
+    int lastComparison = 0;
+    TRowMutations typedOther = (TRowMutations)other;
+
+    lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetRow()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetMutations()).compareTo(typedOther.isSetMutations());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetMutations()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mutations, typedOther.mutations);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    return 0;
+  }
+
+  public _Fields fieldForId(int fieldId) {
+    return _Fields.findByThriftId(fieldId);
+  }
+
+  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+  }
+
+  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder("TRowMutations(");
+    boolean first = true;
+
+    sb.append("row:");
+    if (this.row == null) {
+      sb.append("null");
+    } else {
+      org.apache.thrift.TBaseHelper.toString(this.row, sb);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("mutations:");
+    if (this.mutations == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.mutations);
+    }
+    first = false;
+    sb.append(")");
+    return sb.toString();
+  }
+
+  public void validate() throws org.apache.thrift.TException {
+    // check for required fields
+    if (row == null) {
+      throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString());
+    }
+    if (mutations == null) {
+      throw new org.apache.thrift.protocol.TProtocolException("Required field 'mutations' was not present! Struct: " + toString());
+    }
+  }
+
+  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+    try {
+      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+    try {
+      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private static class TRowMutationsStandardSchemeFactory implements SchemeFactory {
+    public TRowMutationsStandardScheme getScheme() {
+      return new TRowMutationsStandardScheme();
+    }
+  }
+
+  private static class TRowMutationsStandardScheme extends StandardScheme<TRowMutations> {
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot, TRowMutations struct) throws org.apache.thrift.TException {
+      org.apache.thrift.protocol.TField schemeField;
+      iprot.readStructBegin();
+      while (true)
+      {
+        schemeField = iprot.readFieldBegin();
+        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+          break;
+        }
+        switch (schemeField.id) {
+          case 1: // ROW
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.row = iprot.readBinary();
+              struct.setRowIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 2: // MUTATIONS
+            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+              {
+                org.apache.thrift.protocol.TList _list98 = iprot.readListBegin();
+                struct.mutations = new ArrayList<TMutation>(_list98.size);
+                for (int _i99 = 0; _i99 < _list98.size; ++_i99)
+                {
+                  TMutation _elem100; // required
+                  _elem100 = new TMutation();
+                  _elem100.read(iprot);
+                  struct.mutations.add(_elem100);
+                }
+                iprot.readListEnd();
+              }
+              struct.setMutationsIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          default:
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+        }
+        iprot.readFieldEnd();
+      }
+      iprot.readStructEnd();
+
+      // check for required fields of primitive type, which can't be checked in the validate method
+      struct.validate();
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot, TRowMutations struct) throws org.apache.thrift.TException {
+      struct.validate();
+
+      oprot.writeStructBegin(STRUCT_DESC);
+      if (struct.row != null) {
+        oprot.writeFieldBegin(ROW_FIELD_DESC);
+        oprot.writeBinary(struct.row);
+        oprot.writeFieldEnd();
+      }
+      if (struct.mutations != null) {
+        oprot.writeFieldBegin(MUTATIONS_FIELD_DESC);
+        {
+          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size()));
+          for (TMutation _iter101 : struct.mutations)
+          {
+            _iter101.write(oprot);
+          }
+          oprot.writeListEnd();
+        }
+        oprot.writeFieldEnd();
+      }
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+
+  }
+
+  private static class TRowMutationsTupleSchemeFactory implements SchemeFactory {
+    public TRowMutationsTupleScheme getScheme() {
+      return new TRowMutationsTupleScheme();
+    }
+  }
+
+  private static class TRowMutationsTupleScheme extends TupleScheme<TRowMutations> {
+
+    @Override
+    public void write(org.apache.thrift.protocol.TProtocol prot, TRowMutations struct) throws org.apache.thrift.TException {
+      TTupleProtocol oprot = (TTupleProtocol) prot;
+      oprot.writeBinary(struct.row);
+      {
+        oprot.writeI32(struct.mutations.size());
+        for (TMutation _iter102 : struct.mutations)
+        {
+          _iter102.write(oprot);
+        }
+      }
+    }
+
+    @Override
+    public void read(org.apache.thrift.protocol.TProtocol prot, TRowMutations struct) throws org.apache.thrift.TException {
+      TTupleProtocol iprot = (TTupleProtocol) prot;
+      struct.row = iprot.readBinary();
+      struct.setRowIsSet(true);
+      {
+        org.apache.thrift.protocol.TList _list103 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+        struct.mutations = new ArrayList<TMutation>(_list103.size);
+        for (int _i104 = 0; _i104 < _list103.size; ++_i104)
+        {
+          TMutation _elem105; // required
+          _elem105 = new TMutation();
+          _elem105.read(iprot);
+          struct.mutations.add(_elem105);
+        }
+      }
+      struct.setMutationsIsSet(true);
+    }
+  }
+
+}
+
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
index b4dd11242bc9..11e77c8322a5 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
@@ -181,6 +181,22 @@ struct TScan {
   9: optional map<binary, binary> attributes
 }
 
+/**
+ * Atomic mutation for the specified row. It can be either Put or Delete.
+ */
+union TMutation {
+  1: optional TPut put,
+  2: optional TDelete deleteSingle,
+}
+
+/**
+ * A TRowMutations object is used to apply a number of Mutations to a single row.
+ */
+struct TRowMutations {
+  1: required binary row
+  2: required list<TMutation> mutations
+}
+
 //
 // Exceptions
 //
@@ -414,4 +430,15 @@ service THBaseService {
     2: TIllegalArgument ia
   )
 
+  /**
+   * mutateRow performs multiple mutations atomically on a single row.
+  */
+  void mutateRow(
+  /** table to apply the mutations */
+    1: required binary table,
+
+    /** mutations to apply */
+    2: required TRowMutations rowMutations
+  ) throws (1: TIOError io)
+
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 22b7be69802e..b9c1ed9672f7 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -55,6 +55,8 @@
 import org.apache.hadoop.hbase.thrift2.generated.TPut;
 import org.apache.hadoop.hbase.thrift2.generated.TResult;
 import org.apache.hadoop.hbase.thrift2.generated.TScan;
+import org.apache.hadoop.hbase.thrift2.generated.TMutation;
+import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.MetricsContext;
@@ -729,7 +731,70 @@ public void testAttribute() throws Exception {
     Delete delete = deleteFromThrift(tDelete);
     assertArrayEquals(delete.getAttribute("attribute1"), attributeValue);
   }
- 
+
+  /**
+   * Put valueA to a row, make sure put has happened, then create a mutation object to put valueB
+   * and delete ValueA, then check that the row value is only valueB.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testMutateRow() throws Exception {
+    ThriftHBaseServiceHandler handler = createHandler();
+    byte[] rowName = "testMutateRow".getBytes();
+    ByteBuffer table = wrap(tableAname);
+
+    List<TColumnValue> columnValuesA = new ArrayList<TColumnValue>();
+    TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
+        wrap(valueAname));
+    columnValuesA.add(columnValueA);
+    TPut putA = new TPut(wrap(rowName), columnValuesA);
+    putA.setColumnValues(columnValuesA);
+
+    handler.put(table,putA);
+
+    TGet get = new TGet(wrap(rowName));
+    TResult result = handler.get(table, get);
+    assertArrayEquals(rowName, result.getRow());
+    List<TColumnValue> returnedColumnValues = result.getColumnValues();
+
+    List<TColumnValue> expectedColumnValues = new ArrayList<TColumnValue>();
+    expectedColumnValues.add(columnValueA);
+    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
+
+    List<TColumnValue> columnValuesB = new ArrayList<TColumnValue>();
+    TColumnValue columnValueB = new TColumnValue(wrap(familyAname), wrap(qualifierBname),
+        wrap(valueBname));
+    columnValuesB.add(columnValueB);
+    TPut putB = new TPut(wrap(rowName), columnValuesB);
+    putB.setColumnValues(columnValuesB);
+
+    TDelete delete = new TDelete(wrap(rowName));
+    List<TColumn> deleteColumns = new ArrayList<TColumn>();
+    TColumn deleteColumn = new TColumn(wrap(familyAname));
+    deleteColumn.setQualifier(qualifierAname);
+    deleteColumns.add(deleteColumn);
+    delete.setColumns(deleteColumns);
+
+    List<TMutation> mutations = new ArrayList<TMutation>();
+    TMutation mutationA = TMutation.put(putB);
+    mutations.add(mutationA);
+
+    TMutation mutationB = TMutation.deleteSingle(delete);
+    mutations.add(mutationB);
+
+    TRowMutations tRowMutations = new TRowMutations(wrap(rowName),mutations);
+    handler.mutateRow(table,tRowMutations);
+
+    result = handler.get(table, get);
+    assertArrayEquals(rowName, result.getRow());
+    returnedColumnValues = result.getColumnValues();
+
+    expectedColumnValues = new ArrayList<TColumnValue>();
+    expectedColumnValues.add(columnValueB);
+    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
+  }
+
   private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
     setupMetricsContext();
     return new ThriftMetrics(Integer.parseInt(ThriftServer.DEFAULT_LISTEN_PORT),

From 79e66f03778124e8ecc0aeaef54f6e2b981133a4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 17 Jul 2013 22:39:20 +0000
Subject: [PATCH 1064/1540] HBASE-8945 Backport to 0.94: HBASE-7952 Remove
 update() and Improve ExplicitColumnTracker performance. (orginal path by
 Raymond Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504313 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ColumnTracker.java     | 19 ++---
 .../regionserver/ExplicitColumnTracker.java   | 75 ++++++-------------
 .../ScanWildcardColumnTracker.java            |  7 --
 .../TestExplicitColumnTracker.java            |  2 +-
 4 files changed, 35 insertions(+), 68 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
index 2eeaab18b405..693e7341a6dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
@@ -31,12 +31,18 @@
  * Currently there are two different types of Store/Family-level queries.
  * <ul><li>{@link ExplicitColumnTracker} is used when the query specifies
  * one or more column qualifiers to return in the family.
+ * <ul><li>{@link ScanWildcardColumnTracker} is used when no columns are
+ * explicitly specified.
  * <p>
- * This class is utilized by {@link ScanQueryMatcher} through two methods:
+ * This class is utilized by {@link ScanQueryMatcher} mainly through two methods:
  * <ul><li>{@link #checkColumn} is called when a Put satisfies all other
- * conditions of the query.  This method returns a {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} to define
- * what action should be taken.
- * <li>{@link #update} is called at the end of every StoreFile or memstore.
+ * conditions of the query.
+ * <ul><li>{@link #getNextRowOrNextColumn} is called whenever ScanQueryMatcher
+ * believes that the current column should be skipped (by timestamp, filter etc.)
+ * <p>
+ * These two methods returns a 
+ * {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode}
+ * to define what action should be taken.
  * <p>
  * This class is NOT thread-safe as queries are never multi-threaded
  */
@@ -59,11 +65,6 @@ public ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset,
       int length, long ttl, byte type, boolean ignoreCount)
       throws IOException;
 
-  /**
-   * Updates internal variables in between files
-   */
-  public void update();
-
   /**
    * Resets the Matcher
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index 49375cf1cc13..d265a2725332 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -39,11 +39,15 @@
  * between rows.
  *
  * <p>
- * This class is utilized by {@link ScanQueryMatcher} through two methods:
+ * This class is utilized by {@link ScanQueryMatcher} mainly through two methods:
  * <ul><li>{@link #checkColumn} is called when a Put satisfies all other
- * conditions of the query.  This method returns a {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} to define
- * what action should be taken.
- * <li>{@link #update} is called at the end of every StoreFile or memstore.
+ * conditions of the query.
+ * <ul><li>{@link #getNextRowOrNextColumn} is called whenever ScanQueryMatcher
+ * believes that the current column should be skipped (by timestamp, filter etc.)
+ * <p>
+ * These two methods returns a 
+ * {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode}
+ * to define what action should be taken.
  * <p>
  * This class is NOT thread-safe as queries are never multi-threaded
  */
@@ -58,7 +62,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
   * column have been returned.
   */
   private final List<ColumnCount> columns;
-  private final List<ColumnCount> columnsToReuse;
   private int index;
   private ColumnCount column;
   /** Keeps track of the latest timestamp included for current column.
@@ -81,9 +84,8 @@ public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
     this.minVersions = minVersions;
     this.oldestStamp = oldestUnexpiredTS;
     this.columns = new ArrayList<ColumnCount>(columns.size());
-    this.columnsToReuse = new ArrayList<ColumnCount>(columns.size());
     for(byte [] column : columns) {
-      this.columnsToReuse.add(new ColumnCount(column));
+      this.columns.add(new ColumnCount(column));
     }
     reset();
   }
@@ -92,7 +94,7 @@ public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
    * Done when there are no more columns to match against.
    */
   public boolean done() {
-    return this.columns.size() == 0;
+    return this.index >= this.columns.size();
   }
 
   public ColumnCount getColumnHint() {
@@ -110,7 +112,7 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
     assert !KeyValue.isDelete(type);
     do {
       // No more columns left, we are done with this query
-      if(this.columns.size() == 0) {
+      if(done()) {
         return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
       }
 
@@ -136,14 +138,9 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
         int count = this.column.increment();
         if(count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
           // Done with versions for this column
-          // Note: because we are done with this column, and are removing
-          // it from columns, we don't do a ++this.index. The index stays
-          // the same but the columns have shifted within the array such
-          // that index now points to the next column we are interested in.
-          this.columns.remove(this.index);
-
+          ++this.index;
           resetTS();
-          if (this.columns.size() == this.index) {
+          if (done()) {
             // We have served all the requested columns.
             this.column = null;
             return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
@@ -172,7 +169,8 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
       // of interest. Advance the ExplicitColumnTracker state to next
       // column of interest, and check again.
       if (ret <= -1) {
-        if (++this.index >= this.columns.size()) {
+        ++this.index;
+        if (done()) {
           // No more to match, do not include, done with this row.
           return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
         }
@@ -182,24 +180,13 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
     } while(true);
   }
 
-  /**
-   * Called at the end of every StoreFile or memstore.
-   */
-  public void update() {
-    if(this.columns.size() != 0) {
-      this.index = 0;
-      this.column = this.columns.get(this.index);
-    } else {
-      this.index = -1;
-      this.column = null;
-    }
-  }
-
   // Called between every row.
   public void reset() {
-    buildColumnList();
     this.index = 0;
     this.column = this.columns.get(this.index);
+    for(ColumnCount col : this.columns) {
+      col.setCount(0);
+    }
     resetTS();
   }
 
@@ -219,14 +206,6 @@ private boolean isExpired(long timestamp) {
     return timestamp < oldestStamp;
   }
 
-  private void buildColumnList() {
-    this.columns.clear();
-    this.columns.addAll(this.columnsToReuse);
-    for(ColumnCount col : this.columns) {
-      col.setCount(0);
-    }
-  }
-
   /**
    * This method is used to inform the column tracker that we are done with
    * this column. We may get this information from external filters or
@@ -241,24 +220,18 @@ public void doneWithColumn(byte [] bytes, int offset, int length) {
       int compare = Bytes.compareTo(column.getBuffer(), column.getOffset(),
           column.getLength(), bytes, offset, length);
       resetTS();
-      if (compare == 0) {
-        this.columns.remove(this.index);
-        if (this.columns.size() == this.index) {
+      if (compare <= 0) {
+        ++this.index;
+        if (done()) {
           // Will not hit any more columns in this storefile
           this.column = null;
         } else {
           this.column = this.columns.get(this.index);
         }
-        return;
-      } else if ( compare <= -1) {
-        if(++this.index != this.columns.size()) {
-          this.column = this.columns.get(this.index);
-        } else {
-          this.column = null;
-        }
-      } else {
-        return;
+        if (compare <= -1)
+          continue;
       }
+      return;
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
index 1402455668e7..569d0d34f6e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
@@ -140,13 +140,6 @@ private MatchCode checkVersion(byte type, long timestamp) {
 
   }
 
-  @Override
-  public void update() {
-    // no-op, shouldn't even be called
-    throw new UnsupportedOperationException(
-        "ScanWildcardColumnTracker.update should never be called!");
-  }
-
   @Override
   public void reset() {
     columnBuffer = null;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index 246b14581ba6..5993f1fce735 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -172,7 +172,7 @@ public void testStackOverflow() throws IOException{
       explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
-    explicit.update();
+    explicit.reset();
 
     for (int i = 1; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);

From 594860eb3ba6b00adf523f85e38233ce4b4ac682 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Jul 2013 00:35:35 +0000
Subject: [PATCH 1065/1540] HBASE-8968  Avoid call to zk in
 HRegionServer#getMaster() to log the master address which already read from
 the zk. (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504335 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 31ea1805ee43..7fda3acebe1a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2029,8 +2029,7 @@ private ServerName getMaster() {
       masterIsa =
         new InetSocketAddress(masterServerName.getHostname(), masterServerName.getPort());
 
-      LOG.info("Attempting connect to Master server at " +
-        this.masterAddressManager.getMasterAddress());
+      LOG.info("Attempting connect to Master server at " + masterServerName);
       try {
         // Do initial RPC setup. The final argument indicates that the RPC
         // should retry indefinitely.

From 237cd1c88b0349258e454a6bb999f3bf3490bb6f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Jul 2013 03:13:21 +0000
Subject: [PATCH 1066/1540] HBASE-8967 Duplicate call to snapshotManager.stop()
 in HRegionServer (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504351 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/procedure/ZKProcedureMemberRpcs.java  |  4 ++--
 .../hadoop/hbase/regionserver/HRegionServer.java       | 10 ++--------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index 193ace1bebab..89669051a5fb 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -334,7 +334,7 @@ protected void abort(String abortZNode) {
   }
 
   public void start(final String memberName, final ProcedureMember listener) {
-    LOG.debug("Starting procedure member '" + this.memberName + "'");
+    LOG.debug("Starting procedure member '" + memberName + "'");
     this.member = listener;
     this.memberName = memberName;
     watchForAbortedProcedures();
@@ -346,4 +346,4 @@ public void close() throws IOException {
     zkController.close();
   }
 
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7fda3acebe1a..ead66cbe0700 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -835,8 +835,9 @@ public void run() {
       this.healthCheckChore.interrupt();
     }
 
+    // Stop the snapshot handler, forcefully killing all running tasks
     try {
-      if (snapshotManager != null) snapshotManager.stop(this.abortRequested);
+      if (snapshotManager != null) snapshotManager.stop(this.abortRequested || this.killed);
     } catch (IOException e) {
       LOG.warn("Failed to close snapshot handler cleanly", e);
     }
@@ -857,13 +858,6 @@ public void run() {
     // handlers are stuck waiting on meta or root.
     if (this.catalogTracker != null) this.catalogTracker.stop();
 
-    // stop the snapshot handler, forcefully killing all running tasks
-    try {
-      if (snapshotManager != null) snapshotManager.stop(this.abortRequested || this.killed);
-    } catch (IOException e) {
-      LOG.warn("Failed to close snapshot handler cleanly", e);
-    }
-
     // Closing the compactSplit thread before closing meta regions
     if (!this.killed && containsMetaTableRegions()) {
       if (!abortRequested || this.fsOk) {

From b59a9c182f1becb1c658ba9ae2aa16c894978f37 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 18 Jul 2013 19:01:23 +0000
Subject: [PATCH 1067/1540] HBASE-8928: Make ChaosMonkey & LoadTest tools
 extensible (Gautam Soni)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504602 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/ChaosMonkey.java | 10 +--
 .../hadoop/hbase/util/LoadTestTool.java       | 90 +++++++++----------
 2 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index ff8cac1ae93e..f3248a34c71e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -118,11 +118,11 @@ public ActionContext(IntegrationTestingUtility util) {
       this.util = util;
     }
 
-    IntegrationTestingUtility getHaseIntegrationTestingUtility() {
+    public IntegrationTestingUtility getHaseIntegrationTestingUtility() {
       return util;
     }
 
-    HBaseCluster getHBaseCluster() {
+    public HBaseCluster getHBaseCluster() {
       return util.getHBaseClusterInterface();
     }
   }
@@ -494,8 +494,8 @@ public void perform() throws Exception {
   /**
    * A context for a Policy
    */
-  private static class PolicyContext extends ActionContext {
-    PolicyContext(IntegrationTestingUtility util) {
+  public static class PolicyContext extends ActionContext {
+    public PolicyContext(IntegrationTestingUtility util) {
       super(util);
     }
   }
@@ -504,7 +504,7 @@ private static class PolicyContext extends ActionContext {
    * A policy to introduce chaos to the cluster
    */
   public static abstract class Policy extends StoppableImplementation implements Runnable {
-    PolicyContext context;
+    protected PolicyContext context;
     public void init(PolicyContext context) throws Exception {
       this.context = context;
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index 3761242ac80f..cbcd24752100 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -42,33 +42,33 @@ public class LoadTestTool extends AbstractHBaseTool {
   private static final Log LOG = LogFactory.getLog(LoadTestTool.class);
 
   /** Table name for the test */
-  private byte[] tableName;
+  protected byte[] tableName;
 
   /** Table name to use of not overridden on the command line */
-  private static final String DEFAULT_TABLE_NAME = "cluster_test";
+  protected static final String DEFAULT_TABLE_NAME = "cluster_test";
 
   /** Column family used by the test */
-  static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf");
+  protected static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf");
 
   /** Column families used by the test */
-  static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY };
+  protected static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY };
 
   /** The number of reader/writer threads if not specified */
-  private static final int DEFAULT_NUM_THREADS = 20;
+  protected static final int DEFAULT_NUM_THREADS = 20;
 
   /** Usage string for the load option */
-  private static final String OPT_USAGE_LOAD =
+  protected static final String OPT_USAGE_LOAD =
       "<avg_cols_per_key>:<avg_data_size>" +
       "[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
 
   /** Usa\ge string for the read option */
-  private static final String OPT_USAGE_READ =
+  protected static final String OPT_USAGE_READ =
       "<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
 
-  private static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
+  protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
       Arrays.toString(StoreFile.BloomType.values());
 
-  private static final String OPT_USAGE_COMPRESSION = "Compression type, " +
+  protected static final String OPT_USAGE_COMPRESSION = "Compression type, " +
       "one of " + Arrays.toString(Compression.Algorithm.values());
 
   public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
@@ -86,54 +86,54 @@ public class LoadTestTool extends AbstractHBaseTool {
       "If this is specified, data blocks will only be encoded in block " +
       "cache but not on disk";
 
-  private static final String OPT_KEY_WINDOW = "key_window";
-  private static final String OPT_WRITE = "write";
-  private static final String OPT_MAX_READ_ERRORS = "max_read_errors";
-  private static final String OPT_MULTIPUT = "multiput";
-  private static final String OPT_NUM_KEYS = "num_keys";
-  private static final String OPT_READ = "read";
-  private static final String OPT_START_KEY = "start_key";
-  private static final String OPT_TABLE_NAME = "tn";
-  private static final String OPT_ZK_QUORUM = "zk";
-  private static final String OPT_SKIP_INIT = "skip_init";
-  private static final String OPT_INIT_ONLY = "init_only";
+  protected static final String OPT_KEY_WINDOW = "key_window";
+  protected static final String OPT_WRITE = "write";
+  protected static final String OPT_MAX_READ_ERRORS = "max_read_errors";
+  protected static final String OPT_MULTIPUT = "multiput";
+  protected static final String OPT_NUM_KEYS = "num_keys";
+  protected static final String OPT_READ = "read";
+  protected static final String OPT_START_KEY = "start_key";
+  protected static final String OPT_TABLE_NAME = "tn";
+  protected static final String OPT_ZK_QUORUM = "zk";
+  protected static final String OPT_SKIP_INIT = "skip_init";
+  protected static final String OPT_INIT_ONLY = "init_only";
 
-  private static final long DEFAULT_START_KEY = 0;
+  protected static final long DEFAULT_START_KEY = 0;
 
   /** This will be removed as we factor out the dependency on command line */
-  private CommandLine cmd;
+  protected CommandLine cmd;
 
-  private MultiThreadedWriter writerThreads = null;
-  private MultiThreadedReader readerThreads = null;
+  protected MultiThreadedWriter writerThreads = null;
+  protected MultiThreadedReader readerThreads = null;
 
-  private long startKey, endKey;
+  protected long startKey, endKey;
 
-  private boolean isWrite, isRead;
+  protected boolean isWrite, isRead;
 
   // Column family options
-  private DataBlockEncoding dataBlockEncodingAlgo;
-  private boolean encodeInCacheOnly;
-  private Compression.Algorithm compressAlgo;
-  private StoreFile.BloomType bloomType;
+  protected DataBlockEncoding dataBlockEncodingAlgo;
+  protected boolean encodeInCacheOnly;
+  protected Compression.Algorithm compressAlgo;
+  protected StoreFile.BloomType bloomType;
 
   // Writer options
-  private int numWriterThreads = DEFAULT_NUM_THREADS;
-  private int minColsPerKey, maxColsPerKey;
-  private int minColDataSize, maxColDataSize;
-  private boolean isMultiPut;
+  protected int numWriterThreads = DEFAULT_NUM_THREADS;
+  protected int minColsPerKey, maxColsPerKey;
+  protected int minColDataSize, maxColDataSize;
+  protected boolean isMultiPut;
 
   // Reader options
-  private int numReaderThreads = DEFAULT_NUM_THREADS;
-  private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
-  private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
-  private int verifyPercent;
+  protected int numReaderThreads = DEFAULT_NUM_THREADS;
+  protected int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
+  protected int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
+  protected int verifyPercent;
 
   // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
-  //       console tool itself should only be used from console.
-  private boolean isSkipInit = false;
-  private boolean isInitOnly = false;
+  // console tool itself should only be used from console.
+  protected boolean isSkipInit = false;
+  protected boolean isInitOnly = false;
 
-  private String[] splitColonSeparated(String option,
+  protected String[] splitColonSeparated(String option,
       int minNumCols, int maxNumCols) {
     String optVal = cmd.getOptionValue(option);
     String[] cols = optVal.split(":");
@@ -146,7 +146,7 @@ private String[] splitColonSeparated(String option,
     return cols;
   }
 
-  private int getNumThreads(String numThreadsStr) {
+  protected int getNumThreads(String numThreadsStr) {
     return parseInt(numThreadsStr, 1, Short.MAX_VALUE);
   }
 
@@ -154,7 +154,7 @@ private int getNumThreads(String numThreadsStr) {
    * Apply column family options such as Bloom filters, compression, and data
    * block encoding.
    */
-  private void applyColumnFamilyOptions(byte[] tableName,
+  protected void applyColumnFamilyOptions(byte[] tableName,
       byte[][] columnFamilies) throws IOException {
     HBaseAdmin admin = new HBaseAdmin(conf);
     HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
@@ -301,7 +301,7 @@ protected void processOptions(CommandLine cmd) {
     }
   }
 
-  private void parseColumnFamilyOptions(CommandLine cmd) {
+  protected void parseColumnFamilyOptions(CommandLine cmd) {
     String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
     dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null :
         DataBlockEncoding.valueOf(dataBlockEncodingStr);

From c61eb6fc83419efb8673fadd2907834e0d303ca1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Jul 2013 20:18:06 +0000
Subject: [PATCH 1068/1540] HBASE-8847 Filter.transform() always applies
 unconditionally, even when combined in a FilterList (Christophe Taton)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504631 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/filter/Filter.java    | 22 ++++----
 .../hadoop/hbase/filter/FilterList.java       | 56 ++++++++++++++++---
 .../hadoop/hbase/filter/TestFilterList.java   | 43 ++++++++++++++
 3 files changed, 104 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
index ac1079f82bf4..3265b380ba05 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
@@ -27,17 +27,19 @@
 
 /**
  * Interface for row and column filters directly applied within the regionserver.
+ *
  * A filter can expect the following call sequence:
- *<ul>
- * <li>{@link #reset()}</li>
- * <li>{@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.</li>
- * <li>{@link #filterRowKey(byte[],int,int)} -> true to drop this row,
- * if false, we will also call</li>
- * <li>{@link #filterKeyValue(KeyValue)} -> true to drop this key/value</li>
- * <li>{@link #filterRow(List)} -> allows directmodification of the final list to be submitted
- * <li>{@link #filterRow()} -> last chance to drop entire row based on the sequence of
- * filterValue() calls. Eg: filter a row if it doesn't contain a specified column.
- * </li>
+ * <ul>
+ *   <li> {@link #reset()} : reset the filter state before filtering a new row. </li>
+ *   <li> {@link #filterAllRemaining()}: true means row scan is over; false means keep going. </li>
+ *   <li> {@link #filterRowKey(byte[],int,int)}: true means drop this row; false means include.</li>
+ *   <li> {@link #filterKeyValue(KeyValue)}: decides whether to include or exclude this KeyValue.
+ *        See {@link ReturnCode}. </li>
+ *   <li> {@link #transform(KeyValue)}: if the KeyValue is included, let the filter transform the
+ *        KeyValue. </li>
+ *   <li> {@link #filterRow(List)}: allows direct modification of the final list to be submitted
+ *   <li> {@link #filterRow()}: last chance to drop entire row based on the sequence of
+ *        filter calls. Eg: filter a row if it doesn't contain a specified column. </li>
  * </ul>
  *
  * Filter instances are created one per region/scan.  This interface replaces
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 18629e157460..fe3df8c6930d 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -35,9 +35,18 @@
 /**
  * Implementation of {@link Filter} that represents an ordered List of Filters
  * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL}
- * (<code>!AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>!OR</code>).
+ * (<code>AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>OR</code>).
  * Since you can use Filter Lists as children of Filter Lists, you can create a
  * hierarchy of filters to be evaluated.
+ *
+ * <br/>
+ * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does
+ * not include the KeyValue.
+ *
+ * <br/>
+ * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated.
+ *
+ * <br/>
  * Defaults to {@link Operator#MUST_PASS_ALL}.
  * <p>TODO: Fix creation of Configuration on serialization and deserialization.
  */
@@ -55,6 +64,18 @@ public static enum Operator {
   private Operator operator = Operator.MUST_PASS_ALL;
   private List<Filter> filters = new ArrayList<Filter>();
 
+  /** Reference KeyValue used by {@link #transform(KeyValue)} for validation purpose. */
+  private KeyValue referenceKV = null;
+
+  /**
+   * When filtering a given KeyValue in {@link #filterKeyValue(KeyValue)},
+   * this stores the transformed KeyValue to be returned by {@link #transform(KeyValue)}.
+   *
+   * Individual filters transformation are applied only when the filter includes the KeyValue.
+   * Transformations are composed in the order specified by {@link #filters}.
+   */
+  private KeyValue transformedKV = null;
+
   /**
    * Default constructor, filters nothing. Required though for RPC
    * deserialization.
@@ -184,15 +205,21 @@ public boolean filterAllRemaining() {
 
   @Override
   public KeyValue transform(KeyValue v) {
-    KeyValue current = v;
-    for (Filter filter : filters) {
-      current = filter.transform(current);
-    }
-    return current;
+    // transform() is expected to follow an inclusive filterKeyValue() immediately:
+    if (!v.equals(this.referenceKV)) {
+      throw new IllegalStateException(
+          "Reference KeyValue: " + this.referenceKV + " does not match: " + v);
+     }
+    return this.transformedKV;
   }
 
   @Override
   public ReturnCode filterKeyValue(KeyValue v) {
+    this.referenceKV = v;
+
+    // Accumulates successive transformation of every filter that includes the KeyValue:
+    KeyValue transformed = v;
+
     ReturnCode rc = operator == Operator.MUST_PASS_ONE?
         ReturnCode.SKIP: ReturnCode.INCLUDE;
     for (Filter filter : filters) {
@@ -206,6 +233,7 @@ public ReturnCode filterKeyValue(KeyValue v) {
         case INCLUDE_AND_NEXT_COL:
           rc = ReturnCode.INCLUDE_AND_NEXT_COL;
         case INCLUDE:
+          transformed = filter.transform(transformed);
           continue;
         default:
           return code;
@@ -220,16 +248,30 @@ public ReturnCode filterKeyValue(KeyValue v) {
           if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) {
             rc = ReturnCode.INCLUDE;
           }
+          transformed = filter.transform(transformed);
           break;
         case INCLUDE_AND_NEXT_COL:
           rc = ReturnCode.INCLUDE_AND_NEXT_COL;
+          transformed = filter.transform(transformed);
           // must continue here to evaluate all filters
+          break;
         case NEXT_ROW:
+          break;
         case SKIP:
-          // continue;
+          break;
+        case NEXT_COL:
+          break;
+        case SEEK_NEXT_USING_HINT:
+          break;
+        default:
+          throw new IllegalStateException("Received code is not valid.");
         }
       }
     }
+
+    // Save the transformed KeyValue for transform():
+    this.transformedKV = transformed;
+
     return rc;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 5825796c856d..9adb80eb0740 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -34,10 +34,13 @@
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FilterList.Operator;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.collect.Lists;
+
 /**
  * Tests filter sets
  *
@@ -394,6 +397,46 @@ public void write(DataOutput arg0) throws IOException {}
         minKeyValue));
   }
 
+  /**
+   * Tests the behavior of transform() in a hierarchical filter.
+   *
+   * transform() only applies after a filterKeyValue() whose return-code includes the KeyValue.
+   * Lazy evaluation of AND
+   */
+  public void testTransformMPO() throws Exception {
+    // Apply the following filter:
+    //     (family=fam AND qualifier=qual1 AND KeyOnlyFilter)
+    //  OR (family=fam AND qualifier=qual2)
+    final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
+        new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
+            new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
+            new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
+            new KeyOnlyFilter())),
+        new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
+            new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
+            new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));
+
+    final KeyValue kvQual1 = new KeyValue(
+        Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
+    final KeyValue kvQual2 = new KeyValue(
+        Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
+    final KeyValue kvQual3 = new KeyValue(
+        Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));
+
+    // Value for fam:qual1 should be stripped:
+    assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual1));
+    final KeyValue transformedQual1 = flist.transform(kvQual1);
+    assertEquals(0, transformedQual1.getValue().length);
+
+    // Value for fam:qual2 should not be stripped:
+    assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual2));
+    final KeyValue transformedQual2 = flist.transform(kvQual2);
+    assertEquals("value", Bytes.toString(transformedQual2.getValue()));
+
+    // Other keys should be skipped:
+    assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3));
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From e935ae27a718d8cd90036685fc0b262fec7b8015 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Jul 2013 21:01:58 +0000
Subject: [PATCH 1069/1540] HBASE-8988 Reuse the thread pool in the shell to
 not run out of threads (Jean-Daniel Cryans)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504659 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/table.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index ed4113228565..ed8437ca88d1 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -26,9 +26,15 @@ module Hbase
   class Table
     include HBaseConstants
     attr_reader :table
+    @@thread_pool = nil
 
     def initialize(configuration, table_name, formatter)
-      @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name)
+      if @@thread_pool then
+        @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name.to_java_bytes, @@thread_pool)
+      else
+        @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name)
+        @@thread_pool = @table.getPool()
+      end
     end
 
     #----------------------------------------------------------------------------------------------

From c9c35f916151645cc97a9d85c7c9849b190d28e5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 18 Jul 2013 21:11:14 +0000
Subject: [PATCH 1070/1540] HBASE-8599 HLogs in ZK are not cleaned up when
 replication lag is minimal (Varun Sharma)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504665 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSource.java       |  2 ++
 .../ReplicationSourceManager.java             | 28 +++++++++++++------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 930924dfff50..ccd39850d63d 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -352,6 +352,8 @@ public void run() {
           sleepMultiplier++;
         }
         continue;
+      } else if (oldPath != null && !oldPath.getName().equals(getCurrentPath().getName())) {
+        this.manager.cleanOldLogs(getCurrentPath().getName(), this.peerId, this.queueRecovered);
       }
       boolean currentWALisBeingWrittenTo = false;
       //For WAL files we own (rather than recovered), take a snapshot of whether the
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 51d0e61928f5..c0ee8b14eb7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -161,17 +161,29 @@ public void logPositionAndCleanOldLogs(Path log, String id, long position,
     if (holdLogInZK) {
      return;
     }
+    cleanOldLogs(key, id, queueRecovered);
+  }
+
+  /**
+   * Cleans a log file and all older files from ZK. Called when we are sure that a
+   * log file is closed and has no more entries.
+   * @param key Path to the log
+   * @param id id of the peer cluster
+   * @param queueRecovered Whether this is a recovered queue
+   */
+  public void cleanOldLogs(String key,
+                           String id,
+                           boolean queueRecovered) {
     synchronized (this.hlogsById) {
       SortedSet<String> hlogs = this.hlogsById.get(id);
-      if (!queueRecovered && hlogs.first() != key) {
-        SortedSet<String> hlogSet = hlogs.headSet(key);
-        LOG.info("Removing " + hlogSet.size() +
-            " logs in the list: " + hlogSet);
-        for (String hlog : hlogSet) {
-          this.zkHelper.removeLogFromList(hlog, id);
-        }
-        hlogSet.clear();
+      if (queueRecovered || hlogs.first().equals(key)) {
+        return;
+      }
+      SortedSet<String> hlogSet = hlogs.headSet(key);
+      for (String hlog : hlogSet) {
+        this.zkHelper.removeLogFromList(hlog, id);
       }
+      hlogSet.clear();
     }
   }
 

From 53666783c7473fdcb952616631c3df604567ed8c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 19 Jul 2013 16:54:37 +0000
Subject: [PATCH 1071/1540] BASE-8432 a table with unbalanced regions will
 balance indefinitely with the
 'org.apache.hadoop.hbase.master.DefaultLoadBalancer' (Wang Qiang and Ted)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504936 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/master/DefaultLoadBalancer.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
index 82a33e0a3a3c..acc7d4e9d053 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
@@ -330,8 +330,8 @@ public List<RegionPlan> balanceCluster(
       if (regionsToPut == 0)
       {
         regionsToPut = 1;
-        maxToTake--;
       }
+      maxToTake -= regionsToPut;
       underloadedServers.put(server.getKey().getServerName(), regionsToPut);
     }
     // number of servers that get new regions
@@ -439,6 +439,10 @@ public List<RegionPlan> balanceCluster(
       for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server :
         serversByLoad.entrySet()) {
         int regionCount = server.getKey().getLoad();
+        BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey().getServerName());
+        if(balanceInfo != null) {
+          regionCount += balanceInfo.getNumRegionsAdded();
+        }
         if(regionCount >= max) {
           break;
         }

From f0a5caeeea1a26a039aec2e6f941636082aa37a2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 19 Jul 2013 20:13:15 +0000
Subject: [PATCH 1072/1540] CHANGES.txt and pom.xml in preparation for
 0.94.10RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1504994 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 88abf80bc9df..c6de9d84682c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,57 @@
 HBase Change Log
 
+Release 0.94.10 - 7/19/2013
+Sub-task
+
+    [HBASE-8774] - Add BatchSize and Filter to Thrift2
+    [HBASE-8819] - Port HBASE-5428 to Thrift 2
+    [HBASE-8826] - Ensure HBASE-8695 is covered in Thrift 2
+    [HBASE-8832] - Ensure HBASE-4658 is supported by Thrift 2
+    [HBASE-8876] - Addendum to HBASE-8774 Add BatchSize and Filter to Thrift2 - Add BatchSize Test
+    [HBASE-8938] - Thrift2 does not close scanner instance
+
+Bug
+
+    [HBASE-8432] - a table with unbalanced regions will balance indefinitely with the 'org.apache.hadoop.hbase.master.DefaultLoadBalancer'
+    [HBASE-8678] - Wrongly delete cells in some case which can not be deleted
+    [HBASE-8695] - The HBase thrift service ignores XML configuration
+    [HBASE-8776] - tweak retry settings some more (on trunk and 0.94)
+    [HBASE-8782] - Thrift2 can not parse values when using framed transport
+    [HBASE-8794] - DependentColumnFilter.toString() throws NullPointerException
+    [HBASE-8811] - REST service ignores misspelled "check=" parameter, causing unexpected mutations
+    [HBASE-8814] - Possible NPE in split if a region has empty store files.
+    [HBASE-8858] - Backport hbase-6979 "recovered.edits file should not break distributed log splitting"
+    [HBASE-8910] - HMaster.abortNow shouldn't try to become a master again if it was stopped
+    [HBASE-8967] - Duplicate call to snapshotManager.stop() in HRegionServer
+    [HBASE-8968] - Avoid call to zk in HRegionServer#getMaster() to log the master address which already read from the zk.
+    [HBASE-8971] - Bump 0.94 version to 0.94.10-SNAPSHOT
+    [HBASE-8988] - Reuse the thread pool in the shell to not run out of threads
+
+Improvement
+
+    [HBASE-8599] - HLogs in ZK are not cleaned up when replication lag is minimal
+    [HBASE-8767] - Backport hbase-8001 and hbase-8012, avoid lazy seek
+    [HBASE-8806] - Row locks are acquired repeatedly in HRegion.doMiniBatchMutation for duplicate rows.
+    [HBASE-8809] - Include deletes in the scan (setRaw) method does not respect the time range or the filter
+    [HBASE-8847] - Filter.transform() always applies unconditionally, even when combined in a FilterList
+    [HBASE-8908] - Backport HBASE-8882 and HBASE-8904 (An Integration Test to Test MTTR) to 0.94
+    [HBASE-8921] - [thrift2] Add GenericOptionsParser to Thrift 2 server
+    [HBASE-8945] - Backport to 0.94: HBASE-7952 Remove update() and Improve ExplicitColumnTracker performance.
+
+Task
+
+    [HBASE-8829] - Improve failed TestMetaScanner assert message so can see where/why failure
+
+Test
+
+    [HBASE-7770] - minor integration test framework fixes
+    [HBASE-8885] - Fix and reenable TestGet failing#testDynamicFilter
+    [HBASE-8914] - [0.94] TestRegionServerCoprocessorExceptionWithAbort is flaky
+    [HBASE-8928] - Make ChaosMonkey & LoadTest tools extensible, to allow addition of more actions and policies.
+    [HBASE-8934] - Fix bunch of flaky tests
+    [HBASE-8969] - Backport HBASE-8535+HBASE-8586 TestHCM#testDeleteForZKConnLeak enhancement to 0.94
+
+
 Release 0.94.9 - 6/24/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index e427b0e620bd..160d2ac2fc0a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.10-SNAPSHOT</version>
+  <version>0.94.10</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From d375167e58764f25bebd28fc423793be6479b558 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Sun, 21 Jul 2013 08:01:50 +0000
Subject: [PATCH 1073/1540] HBASE-8946 Add a new function to Thrift 2 to open
 scanner, get results and close scanner (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1505292 13f79535-47bb-0310-9956-ffa450edef68
---
 .../thrift2/ThriftHBaseServiceHandler.java    |   20 +
 .../thrift2/generated/THBaseService.java      | 1436 +++++++++++++++--
 .../apache/hadoop/hbase/thrift2/hbase.thrift  |   26 +-
 .../TestThriftHBaseServiceHandler.java        |   56 +
 4 files changed, 1419 insertions(+), 119 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index 41805c87f683..c9f838ca9629 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -348,6 +348,26 @@ public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, TExce
     removeScanner(scannerId);
   }
 
+  @Override
+  public List<TResult> getScannerResults(ByteBuffer table, TScan scan, int numRows)
+      throws TIOError, TException {
+    HTableInterface htable = getTable(table);
+    List<TResult> results = null;
+    ResultScanner scanner = null;
+    try {
+      scanner = htable.getScanner(scanFromThrift(scan));
+      results = resultsFromHBase(scanner.next(numRows));
+    } catch (IOException e) {
+      throw getTIOError(e);
+    } finally {
+      if (scanner != null) {
+        scanner.close();
+      }
+      closeTable(htable);
+    }
+    return results;
+  }
+
   @Override
   public void mutateRow(ByteBuffer table, TRowMutations rowMutations) throws TIOError, TException {
     HTableInterface htable = getTable(table);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
index ea35d07fac2d..4a80126230da 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
@@ -128,9 +128,9 @@ public interface Iface {
     /**
      * Bulk commit a List of TDeletes to the table.
      * 
-     * Throws a TIOError if any of the deletes fail.
-     * 
-     * Always returns an empty list for backwards compatibility.
+     * This returns a list of TDeletes that were not
+     * executed. So if everything succeeds you'll
+     * receive an empty list.
      * 
      * @param table the table to delete from
      * 
@@ -185,10 +185,9 @@ public interface Iface {
     public List<TResult> getScannerRows(int scannerId, int numRows) throws TIOError, TIllegalArgument, org.apache.thrift.TException;
 
     /**
-     * Closes the scanner. Should be called if you need to close
-     * the Scanner before all results are read.
-     * 
-     * Exhausted scanners are closed automatically.
+     * Closes the scanner. Should be called to free server side resources timely.
+     * Typically close once the scanner is not needed anymore, i.e. after looping
+     * over it to get all the required rows.
      * 
      * @param scannerId the Id of the Scanner to close *
      */
@@ -203,6 +202,20 @@ public interface Iface {
      */
     public void mutateRow(ByteBuffer table, TRowMutations rowMutations) throws TIOError, org.apache.thrift.TException;
 
+    /**
+     * Get results for the provided TScan object.
+     * This helper function opens a scanner, get the results and close the scanner.
+     * 
+     * @return between zero and numRows TResults
+     * 
+     * @param table the table to get the Scanner for
+     * 
+     * @param scan the scan object to get a Scanner for
+     * 
+     * @param numRows number of rows to return
+     */
+    public List<TResult> getScannerResults(ByteBuffer table, TScan scan, int numRows) throws TIOError, org.apache.thrift.TException;
+
   }
 
   public interface AsyncIface {
@@ -235,6 +248,8 @@ public interface AsyncIface {
 
     public void mutateRow(ByteBuffer table, TRowMutations rowMutations, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.mutateRow_call> resultHandler) throws org.apache.thrift.TException;
 
+    public void getScannerResults(ByteBuffer table, TScan scan, int numRows, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.getScannerResults_call> resultHandler) throws org.apache.thrift.TException;
+
   }
 
   public static class Client extends org.apache.thrift.TServiceClient implements Iface {
@@ -633,6 +648,34 @@ public void recv_mutateRow() throws TIOError, org.apache.thrift.TException
       return;
     }
 
+    public List<TResult> getScannerResults(ByteBuffer table, TScan scan, int numRows) throws TIOError, org.apache.thrift.TException
+    {
+      send_getScannerResults(table, scan, numRows);
+      return recv_getScannerResults();
+    }
+
+    public void send_getScannerResults(ByteBuffer table, TScan scan, int numRows) throws org.apache.thrift.TException
+    {
+      getScannerResults_args args = new getScannerResults_args();
+      args.setTable(table);
+      args.setScan(scan);
+      args.setNumRows(numRows);
+      sendBase("getScannerResults", args);
+    }
+
+    public List<TResult> recv_getScannerResults() throws TIOError, org.apache.thrift.TException
+    {
+      getScannerResults_result result = new getScannerResults_result();
+      receiveBase(result, "getScannerResults");
+      if (result.isSetSuccess()) {
+        return result.success;
+      }
+      if (result.io != null) {
+        throw result.io;
+      }
+      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getScannerResults failed: unknown result");
+    }
+
   }
   public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
     public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
@@ -1162,6 +1205,44 @@ public void getResult() throws TIOError, org.apache.thrift.TException {
       }
     }
 
+    public void getScannerResults(ByteBuffer table, TScan scan, int numRows, org.apache.thrift.async.AsyncMethodCallback<getScannerResults_call> resultHandler) throws org.apache.thrift.TException {
+      checkReady();
+      getScannerResults_call method_call = new getScannerResults_call(table, scan, numRows, resultHandler, this, ___protocolFactory, ___transport);
+      this.___currentMethod = method_call;
+      ___manager.call(method_call);
+    }
+
+    public static class getScannerResults_call extends org.apache.thrift.async.TAsyncMethodCall {
+      private ByteBuffer table;
+      private TScan scan;
+      private int numRows;
+      public getScannerResults_call(ByteBuffer table, TScan scan, int numRows, org.apache.thrift.async.AsyncMethodCallback<getScannerResults_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+        super(client, protocolFactory, transport, resultHandler, false);
+        this.table = table;
+        this.scan = scan;
+        this.numRows = numRows;
+      }
+
+      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getScannerResults", org.apache.thrift.protocol.TMessageType.CALL, 0));
+        getScannerResults_args args = new getScannerResults_args();
+        args.setTable(table);
+        args.setScan(scan);
+        args.setNumRows(numRows);
+        args.write(prot);
+        prot.writeMessageEnd();
+      }
+
+      public List<TResult> getResult() throws TIOError, org.apache.thrift.TException {
+        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+          throw new IllegalStateException("Method call not finished!");
+        }
+        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+        return (new Client(prot)).recv_getScannerResults();
+      }
+    }
+
   }
 
   public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
@@ -1189,6 +1270,7 @@ protected Processor(I iface, Map<String,  org.apache.thrift.ProcessFunction<I, ?
       processMap.put("getScannerRows", new getScannerRows());
       processMap.put("closeScanner", new closeScanner());
       processMap.put("mutateRow", new mutateRow());
+      processMap.put("getScannerResults", new getScannerResults());
       return processMap;
     }
 
@@ -1480,6 +1562,26 @@ protected mutateRow_result getResult(I iface, mutateRow_args args) throws org.ap
       }
     }
 
+    private static class getScannerResults<I extends Iface> extends org.apache.thrift.ProcessFunction<I, getScannerResults_args> {
+      public getScannerResults() {
+        super("getScannerResults");
+      }
+
+      protected getScannerResults_args getEmptyArgsInstance() {
+        return new getScannerResults_args();
+      }
+
+      protected getScannerResults_result getResult(I iface, getScannerResults_args args) throws org.apache.thrift.TException {
+        getScannerResults_result result = new getScannerResults_result();
+        try {
+          result.success = iface.getScannerResults(args.table, args.scan, args.numRows);
+        } catch (TIOError io) {
+          result.io = io;
+        }
+        return result;
+      }
+    }
+
   }
 
   public static class exists_args implements org.apache.thrift.TBase<exists_args, exists_args._Fields>, java.io.Serializable, Cloneable   {
@@ -3793,14 +3895,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_args st
             case 2: // GETS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list106 = iprot.readListBegin();
-                  struct.gets = new ArrayList<TGet>(_list106.size);
-                  for (int _i107 = 0; _i107 < _list106.size; ++_i107)
+                  org.apache.thrift.protocol.TList _list96 = iprot.readListBegin();
+                  struct.gets = new ArrayList<TGet>(_list96.size);
+                  for (int _i97 = 0; _i97 < _list96.size; ++_i97)
                   {
-                    TGet _elem108; // required
-                    _elem108 = new TGet();
-                    _elem108.read(iprot);
-                    struct.gets.add(_elem108);
+                    TGet _elem98; // required
+                    _elem98 = new TGet();
+                    _elem98.read(iprot);
+                    struct.gets.add(_elem98);
                   }
                   iprot.readListEnd();
                 }
@@ -3833,9 +3935,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_args s
           oprot.writeFieldBegin(GETS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.gets.size()));
-            for (TGet _iter109 : struct.gets)
+            for (TGet _iter99 : struct.gets)
             {
-              _iter109.write(oprot);
+              _iter99.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -3861,9 +3963,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_args st
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.gets.size());
-          for (TGet _iter110 : struct.gets)
+          for (TGet _iter100 : struct.gets)
           {
-            _iter110.write(oprot);
+            _iter100.write(oprot);
           }
         }
       }
@@ -3874,14 +3976,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_args str
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list111 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.gets = new ArrayList<TGet>(_list111.size);
-          for (int _i112 = 0; _i112 < _list111.size; ++_i112)
+          org.apache.thrift.protocol.TList _list101 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.gets = new ArrayList<TGet>(_list101.size);
+          for (int _i102 = 0; _i102 < _list101.size; ++_i102)
           {
-            TGet _elem113; // required
-            _elem113 = new TGet();
-            _elem113.read(iprot);
-            struct.gets.add(_elem113);
+            TGet _elem103; // required
+            _elem103 = new TGet();
+            _elem103.read(iprot);
+            struct.gets.add(_elem103);
           }
         }
         struct.setGetsIsSet(true);
@@ -4275,14 +4377,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_result
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list114 = iprot.readListBegin();
-                  struct.success = new ArrayList<TResult>(_list114.size);
-                  for (int _i115 = 0; _i115 < _list114.size; ++_i115)
+                  org.apache.thrift.protocol.TList _list104 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list104.size);
+                  for (int _i105 = 0; _i105 < _list104.size; ++_i105)
                   {
-                    TResult _elem116; // required
-                    _elem116 = new TResult();
-                    _elem116.read(iprot);
-                    struct.success.add(_elem116);
+                    TResult _elem106; // required
+                    _elem106 = new TResult();
+                    _elem106.read(iprot);
+                    struct.success.add(_elem106);
                   }
                   iprot.readListEnd();
                 }
@@ -4319,9 +4421,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_result
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TResult _iter117 : struct.success)
+            for (TResult _iter107 : struct.success)
             {
-              _iter117.write(oprot);
+              _iter107.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -4360,9 +4462,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_result
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TResult _iter118 : struct.success)
+            for (TResult _iter108 : struct.success)
             {
-              _iter118.write(oprot);
+              _iter108.write(oprot);
             }
           }
         }
@@ -4377,14 +4479,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_result s
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list119 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TResult>(_list119.size);
-            for (int _i120 = 0; _i120 < _list119.size; ++_i120)
+            org.apache.thrift.protocol.TList _list109 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list109.size);
+            for (int _i110 = 0; _i110 < _list109.size; ++_i110)
             {
-              TResult _elem121; // required
-              _elem121 = new TResult();
-              _elem121.read(iprot);
-              struct.success.add(_elem121);
+              TResult _elem111; // required
+              _elem111 = new TResult();
+              _elem111.read(iprot);
+              struct.success.add(_elem111);
             }
           }
           struct.setSuccessIsSet(true);
@@ -7091,14 +7193,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, putMultiple_args st
             case 2: // PUTS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list122 = iprot.readListBegin();
-                  struct.puts = new ArrayList<TPut>(_list122.size);
-                  for (int _i123 = 0; _i123 < _list122.size; ++_i123)
+                  org.apache.thrift.protocol.TList _list112 = iprot.readListBegin();
+                  struct.puts = new ArrayList<TPut>(_list112.size);
+                  for (int _i113 = 0; _i113 < _list112.size; ++_i113)
                   {
-                    TPut _elem124; // required
-                    _elem124 = new TPut();
-                    _elem124.read(iprot);
-                    struct.puts.add(_elem124);
+                    TPut _elem114; // required
+                    _elem114 = new TPut();
+                    _elem114.read(iprot);
+                    struct.puts.add(_elem114);
                   }
                   iprot.readListEnd();
                 }
@@ -7131,9 +7233,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, putMultiple_args s
           oprot.writeFieldBegin(PUTS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.puts.size()));
-            for (TPut _iter125 : struct.puts)
+            for (TPut _iter115 : struct.puts)
             {
-              _iter125.write(oprot);
+              _iter115.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -7159,9 +7261,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, putMultiple_args st
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.puts.size());
-          for (TPut _iter126 : struct.puts)
+          for (TPut _iter116 : struct.puts)
           {
-            _iter126.write(oprot);
+            _iter116.write(oprot);
           }
         }
       }
@@ -7172,14 +7274,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, putMultiple_args str
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list127 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.puts = new ArrayList<TPut>(_list127.size);
-          for (int _i128 = 0; _i128 < _list127.size; ++_i128)
+          org.apache.thrift.protocol.TList _list117 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.puts = new ArrayList<TPut>(_list117.size);
+          for (int _i118 = 0; _i118 < _list117.size; ++_i118)
           {
-            TPut _elem129; // required
-            _elem129 = new TPut();
-            _elem129.read(iprot);
-            struct.puts.add(_elem129);
+            TPut _elem119; // required
+            _elem119 = new TPut();
+            _elem119.read(iprot);
+            struct.puts.add(_elem119);
           }
         }
         struct.setPutsIsSet(true);
@@ -8811,14 +8913,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_args
             case 2: // DELETES
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list130 = iprot.readListBegin();
-                  struct.deletes = new ArrayList<TDelete>(_list130.size);
-                  for (int _i131 = 0; _i131 < _list130.size; ++_i131)
+                  org.apache.thrift.protocol.TList _list120 = iprot.readListBegin();
+                  struct.deletes = new ArrayList<TDelete>(_list120.size);
+                  for (int _i121 = 0; _i121 < _list120.size; ++_i121)
                   {
-                    TDelete _elem132; // required
-                    _elem132 = new TDelete();
-                    _elem132.read(iprot);
-                    struct.deletes.add(_elem132);
+                    TDelete _elem122; // required
+                    _elem122 = new TDelete();
+                    _elem122.read(iprot);
+                    struct.deletes.add(_elem122);
                   }
                   iprot.readListEnd();
                 }
@@ -8851,9 +8953,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_arg
           oprot.writeFieldBegin(DELETES_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.deletes.size()));
-            for (TDelete _iter133 : struct.deletes)
+            for (TDelete _iter123 : struct.deletes)
             {
-              _iter133.write(oprot);
+              _iter123.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -8879,9 +8981,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
         oprot.writeBinary(struct.table);
         {
           oprot.writeI32(struct.deletes.size());
-          for (TDelete _iter134 : struct.deletes)
+          for (TDelete _iter124 : struct.deletes)
           {
-            _iter134.write(oprot);
+            _iter124.write(oprot);
           }
         }
       }
@@ -8892,14 +8994,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args
         struct.table = iprot.readBinary();
         struct.setTableIsSet(true);
         {
-          org.apache.thrift.protocol.TList _list135 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.deletes = new ArrayList<TDelete>(_list135.size);
-          for (int _i136 = 0; _i136 < _list135.size; ++_i136)
+          org.apache.thrift.protocol.TList _list125 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.deletes = new ArrayList<TDelete>(_list125.size);
+          for (int _i126 = 0; _i126 < _list125.size; ++_i126)
           {
-            TDelete _elem137; // required
-            _elem137 = new TDelete();
-            _elem137.read(iprot);
-            struct.deletes.add(_elem137);
+            TDelete _elem127; // required
+            _elem127 = new TDelete();
+            _elem127.read(iprot);
+            struct.deletes.add(_elem127);
           }
         }
         struct.setDeletesIsSet(true);
@@ -9293,14 +9395,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list138 = iprot.readListBegin();
-                  struct.success = new ArrayList<TDelete>(_list138.size);
-                  for (int _i139 = 0; _i139 < _list138.size; ++_i139)
+                  org.apache.thrift.protocol.TList _list128 = iprot.readListBegin();
+                  struct.success = new ArrayList<TDelete>(_list128.size);
+                  for (int _i129 = 0; _i129 < _list128.size; ++_i129)
                   {
-                    TDelete _elem140; // required
-                    _elem140 = new TDelete();
-                    _elem140.read(iprot);
-                    struct.success.add(_elem140);
+                    TDelete _elem130; // required
+                    _elem130 = new TDelete();
+                    _elem130.read(iprot);
+                    struct.success.add(_elem130);
                   }
                   iprot.readListEnd();
                 }
@@ -9337,9 +9439,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TDelete _iter141 : struct.success)
+            for (TDelete _iter131 : struct.success)
             {
-              _iter141.write(oprot);
+              _iter131.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -9378,9 +9480,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TDelete _iter142 : struct.success)
+            for (TDelete _iter132 : struct.success)
             {
-              _iter142.write(oprot);
+              _iter132.write(oprot);
             }
           }
         }
@@ -9395,14 +9497,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_resul
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list143 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TDelete>(_list143.size);
-            for (int _i144 = 0; _i144 < _list143.size; ++_i144)
+            org.apache.thrift.protocol.TList _list133 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TDelete>(_list133.size);
+            for (int _i134 = 0; _i134 < _list133.size; ++_i134)
             {
-              TDelete _elem145; // required
-              _elem145 = new TDelete();
-              _elem145.read(iprot);
-              struct.success.add(_elem145);
+              TDelete _elem135; // required
+              _elem135 = new TDelete();
+              _elem135.read(iprot);
+              struct.success.add(_elem135);
             }
           }
           struct.setSuccessIsSet(true);
@@ -13653,14 +13755,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerRows_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list146 = iprot.readListBegin();
-                  struct.success = new ArrayList<TResult>(_list146.size);
-                  for (int _i147 = 0; _i147 < _list146.size; ++_i147)
+                  org.apache.thrift.protocol.TList _list136 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list136.size);
+                  for (int _i137 = 0; _i137 < _list136.size; ++_i137)
                   {
-                    TResult _elem148; // required
-                    _elem148 = new TResult();
-                    _elem148.read(iprot);
-                    struct.success.add(_elem148);
+                    TResult _elem138; // required
+                    _elem138 = new TResult();
+                    _elem138.read(iprot);
+                    struct.success.add(_elem138);
                   }
                   iprot.readListEnd();
                 }
@@ -13706,9 +13808,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerRows_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TResult _iter149 : struct.success)
+            for (TResult _iter139 : struct.success)
             {
-              _iter149.write(oprot);
+              _iter139.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -13755,9 +13857,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TResult _iter150 : struct.success)
+            for (TResult _iter140 : struct.success)
             {
-              _iter150.write(oprot);
+              _iter140.write(oprot);
             }
           }
         }
@@ -13775,14 +13877,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getScannerRows_resul
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list151 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TResult>(_list151.size);
-            for (int _i152 = 0; _i152 < _list151.size; ++_i152)
+            org.apache.thrift.protocol.TList _list141 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list141.size);
+            for (int _i142 = 0; _i142 < _list141.size; ++_i142)
             {
-              TResult _elem153; // required
-              _elem153 = new TResult();
-              _elem153.read(iprot);
-              struct.success.add(_elem153);
+              TResult _elem143; // required
+              _elem143 = new TResult();
+              _elem143.read(iprot);
+              struct.success.add(_elem143);
             }
           }
           struct.setSuccessIsSet(true);
@@ -15463,4 +15565,1108 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_result str
 
   }
 
+  public static class getScannerResults_args implements org.apache.thrift.TBase<getScannerResults_args, getScannerResults_args._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getScannerResults_args");
+
+    private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1);
+    private static final org.apache.thrift.protocol.TField SCAN_FIELD_DESC = new org.apache.thrift.protocol.TField("scan", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+    private static final org.apache.thrift.protocol.TField NUM_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("numRows", org.apache.thrift.protocol.TType.I32, (short)3);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new getScannerResults_argsStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new getScannerResults_argsTupleSchemeFactory());
+    }
+
+    /**
+     * the table to get the Scanner for
+     */
+    public ByteBuffer table; // required
+    /**
+     * the scan object to get a Scanner for
+     */
+    public TScan scan; // required
+    /**
+     * number of rows to return
+     */
+    public int numRows; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      /**
+       * the table to get the Scanner for
+       */
+      TABLE((short)1, "table"),
+      /**
+       * the scan object to get a Scanner for
+       */
+      SCAN((short)2, "scan"),
+      /**
+       * number of rows to return
+       */
+      NUM_ROWS((short)3, "numRows");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // TABLE
+            return TABLE;
+          case 2: // SCAN
+            return SCAN;
+          case 3: // NUM_ROWS
+            return NUM_ROWS;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    private static final int __NUMROWS_ISSET_ID = 0;
+    private BitSet __isset_bit_vector = new BitSet(1);
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING          , true)));
+      tmpMap.put(_Fields.SCAN, new org.apache.thrift.meta_data.FieldMetaData("scan", org.apache.thrift.TFieldRequirementType.REQUIRED, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TScan.class)));
+      tmpMap.put(_Fields.NUM_ROWS, new org.apache.thrift.meta_data.FieldMetaData("numRows", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getScannerResults_args.class, metaDataMap);
+    }
+
+    public getScannerResults_args() {
+      this.numRows = 1;
+
+    }
+
+    public getScannerResults_args(
+      ByteBuffer table,
+      TScan scan,
+      int numRows)
+    {
+      this();
+      this.table = table;
+      this.scan = scan;
+      this.numRows = numRows;
+      setNumRowsIsSet(true);
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public getScannerResults_args(getScannerResults_args other) {
+      __isset_bit_vector.clear();
+      __isset_bit_vector.or(other.__isset_bit_vector);
+      if (other.isSetTable()) {
+        this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table);
+;
+      }
+      if (other.isSetScan()) {
+        this.scan = new TScan(other.scan);
+      }
+      this.numRows = other.numRows;
+    }
+
+    public getScannerResults_args deepCopy() {
+      return new getScannerResults_args(this);
+    }
+
+    @Override
+    public void clear() {
+      this.table = null;
+      this.scan = null;
+      this.numRows = 1;
+
+    }
+
+    /**
+     * the table to get the Scanner for
+     */
+    public byte[] getTable() {
+      setTable(org.apache.thrift.TBaseHelper.rightSize(table));
+      return table == null ? null : table.array();
+    }
+
+    public ByteBuffer bufferForTable() {
+      return table;
+    }
+
+    /**
+     * the table to get the Scanner for
+     */
+    public getScannerResults_args setTable(byte[] table) {
+      setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table));
+      return this;
+    }
+
+    public getScannerResults_args setTable(ByteBuffer table) {
+      this.table = table;
+      return this;
+    }
+
+    public void unsetTable() {
+      this.table = null;
+    }
+
+    /** Returns true if field table is set (has been assigned a value) and false otherwise */
+    public boolean isSetTable() {
+      return this.table != null;
+    }
+
+    public void setTableIsSet(boolean value) {
+      if (!value) {
+        this.table = null;
+      }
+    }
+
+    /**
+     * the scan object to get a Scanner for
+     */
+    public TScan getScan() {
+      return this.scan;
+    }
+
+    /**
+     * the scan object to get a Scanner for
+     */
+    public getScannerResults_args setScan(TScan scan) {
+      this.scan = scan;
+      return this;
+    }
+
+    public void unsetScan() {
+      this.scan = null;
+    }
+
+    /** Returns true if field scan is set (has been assigned a value) and false otherwise */
+    public boolean isSetScan() {
+      return this.scan != null;
+    }
+
+    public void setScanIsSet(boolean value) {
+      if (!value) {
+        this.scan = null;
+      }
+    }
+
+    /**
+     * number of rows to return
+     */
+    public int getNumRows() {
+      return this.numRows;
+    }
+
+    /**
+     * number of rows to return
+     */
+    public getScannerResults_args setNumRows(int numRows) {
+      this.numRows = numRows;
+      setNumRowsIsSet(true);
+      return this;
+    }
+
+    public void unsetNumRows() {
+      __isset_bit_vector.clear(__NUMROWS_ISSET_ID);
+    }
+
+    /** Returns true if field numRows is set (has been assigned a value) and false otherwise */
+    public boolean isSetNumRows() {
+      return __isset_bit_vector.get(__NUMROWS_ISSET_ID);
+    }
+
+    public void setNumRowsIsSet(boolean value) {
+      __isset_bit_vector.set(__NUMROWS_ISSET_ID, value);
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case TABLE:
+        if (value == null) {
+          unsetTable();
+        } else {
+          setTable((ByteBuffer)value);
+        }
+        break;
+
+      case SCAN:
+        if (value == null) {
+          unsetScan();
+        } else {
+          setScan((TScan)value);
+        }
+        break;
+
+      case NUM_ROWS:
+        if (value == null) {
+          unsetNumRows();
+        } else {
+          setNumRows((Integer)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case TABLE:
+        return getTable();
+
+      case SCAN:
+        return getScan();
+
+      case NUM_ROWS:
+        return Integer.valueOf(getNumRows());
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case TABLE:
+        return isSetTable();
+      case SCAN:
+        return isSetScan();
+      case NUM_ROWS:
+        return isSetNumRows();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof getScannerResults_args)
+        return this.equals((getScannerResults_args)that);
+      return false;
+    }
+
+    public boolean equals(getScannerResults_args that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_table = true && this.isSetTable();
+      boolean that_present_table = true && that.isSetTable();
+      if (this_present_table || that_present_table) {
+        if (!(this_present_table && that_present_table))
+          return false;
+        if (!this.table.equals(that.table))
+          return false;
+      }
+
+      boolean this_present_scan = true && this.isSetScan();
+      boolean that_present_scan = true && that.isSetScan();
+      if (this_present_scan || that_present_scan) {
+        if (!(this_present_scan && that_present_scan))
+          return false;
+        if (!this.scan.equals(that.scan))
+          return false;
+      }
+
+      boolean this_present_numRows = true;
+      boolean that_present_numRows = true;
+      if (this_present_numRows || that_present_numRows) {
+        if (!(this_present_numRows && that_present_numRows))
+          return false;
+        if (this.numRows != that.numRows)
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(getScannerResults_args other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      getScannerResults_args typedOther = (getScannerResults_args)other;
+
+      lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetTable()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      lastComparison = Boolean.valueOf(isSetScan()).compareTo(typedOther.isSetScan());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetScan()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.scan, typedOther.scan);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      lastComparison = Boolean.valueOf(isSetNumRows()).compareTo(typedOther.isSetNumRows());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetNumRows()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.numRows, typedOther.numRows);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("getScannerResults_args(");
+      boolean first = true;
+
+      sb.append("table:");
+      if (this.table == null) {
+        sb.append("null");
+      } else {
+        org.apache.thrift.TBaseHelper.toString(this.table, sb);
+      }
+      first = false;
+      if (!first) sb.append(", ");
+      sb.append("scan:");
+      if (this.scan == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.scan);
+      }
+      first = false;
+      if (!first) sb.append(", ");
+      sb.append("numRows:");
+      sb.append(this.numRows);
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+      if (table == null) {
+        throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString());
+      }
+      if (scan == null) {
+        throw new org.apache.thrift.protocol.TProtocolException("Required field 'scan' was not present! Struct: " + toString());
+      }
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class getScannerResults_argsStandardSchemeFactory implements SchemeFactory {
+      public getScannerResults_argsStandardScheme getScheme() {
+        return new getScannerResults_argsStandardScheme();
+      }
+    }
+
+    private static class getScannerResults_argsStandardScheme extends StandardScheme<getScannerResults_args> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerResults_args struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 1: // TABLE
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+                struct.table = iprot.readBinary();
+                struct.setTableIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            case 2: // SCAN
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.scan = new TScan();
+                struct.scan.read(iprot);
+                struct.setScanIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            case 3: // NUM_ROWS
+              if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+                struct.numRows = iprot.readI32();
+                struct.setNumRowsIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerResults_args struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.table != null) {
+          oprot.writeFieldBegin(TABLE_FIELD_DESC);
+          oprot.writeBinary(struct.table);
+          oprot.writeFieldEnd();
+        }
+        if (struct.scan != null) {
+          oprot.writeFieldBegin(SCAN_FIELD_DESC);
+          struct.scan.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldBegin(NUM_ROWS_FIELD_DESC);
+        oprot.writeI32(struct.numRows);
+        oprot.writeFieldEnd();
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class getScannerResults_argsTupleSchemeFactory implements SchemeFactory {
+      public getScannerResults_argsTupleScheme getScheme() {
+        return new getScannerResults_argsTupleScheme();
+      }
+    }
+
+    private static class getScannerResults_argsTupleScheme extends TupleScheme<getScannerResults_args> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, getScannerResults_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        oprot.writeBinary(struct.table);
+        struct.scan.write(oprot);
+        BitSet optionals = new BitSet();
+        if (struct.isSetNumRows()) {
+          optionals.set(0);
+        }
+        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetNumRows()) {
+          oprot.writeI32(struct.numRows);
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, getScannerResults_args struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        struct.table = iprot.readBinary();
+        struct.setTableIsSet(true);
+        struct.scan = new TScan();
+        struct.scan.read(iprot);
+        struct.setScanIsSet(true);
+        BitSet incoming = iprot.readBitSet(1);
+        if (incoming.get(0)) {
+          struct.numRows = iprot.readI32();
+          struct.setNumRowsIsSet(true);
+        }
+      }
+    }
+
+  }
+
+  public static class getScannerResults_result implements org.apache.thrift.TBase<getScannerResults_result, getScannerResults_result._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getScannerResults_result");
+
+    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0);
+    private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+    static {
+      schemes.put(StandardScheme.class, new getScannerResults_resultStandardSchemeFactory());
+      schemes.put(TupleScheme.class, new getScannerResults_resultTupleSchemeFactory());
+    }
+
+    public List<TResult> success; // required
+    public TIOError io; // required
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      SUCCESS((short)0, "success"),
+      IO((short)1, "io");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 0: // SUCCESS
+            return SUCCESS;
+          case 1: // IO
+            return IO;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+              new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResult.class))));
+      tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getScannerResults_result.class, metaDataMap);
+    }
+
+    public getScannerResults_result() {
+    }
+
+    public getScannerResults_result(
+      List<TResult> success,
+      TIOError io)
+    {
+      this();
+      this.success = success;
+      this.io = io;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public getScannerResults_result(getScannerResults_result other) {
+      if (other.isSetSuccess()) {
+        List<TResult> __this__success = new ArrayList<TResult>();
+        for (TResult other_element : other.success) {
+          __this__success.add(new TResult(other_element));
+        }
+        this.success = __this__success;
+      }
+      if (other.isSetIo()) {
+        this.io = new TIOError(other.io);
+      }
+    }
+
+    public getScannerResults_result deepCopy() {
+      return new getScannerResults_result(this);
+    }
+
+    @Override
+    public void clear() {
+      this.success = null;
+      this.io = null;
+    }
+
+    public int getSuccessSize() {
+      return (this.success == null) ? 0 : this.success.size();
+    }
+
+    public java.util.Iterator<TResult> getSuccessIterator() {
+      return (this.success == null) ? null : this.success.iterator();
+    }
+
+    public void addToSuccess(TResult elem) {
+      if (this.success == null) {
+        this.success = new ArrayList<TResult>();
+      }
+      this.success.add(elem);
+    }
+
+    public List<TResult> getSuccess() {
+      return this.success;
+    }
+
+    public getScannerResults_result setSuccess(List<TResult> success) {
+      this.success = success;
+      return this;
+    }
+
+    public void unsetSuccess() {
+      this.success = null;
+    }
+
+    /** Returns true if field success is set (has been assigned a value) and false otherwise */
+    public boolean isSetSuccess() {
+      return this.success != null;
+    }
+
+    public void setSuccessIsSet(boolean value) {
+      if (!value) {
+        this.success = null;
+      }
+    }
+
+    public TIOError getIo() {
+      return this.io;
+    }
+
+    public getScannerResults_result setIo(TIOError io) {
+      this.io = io;
+      return this;
+    }
+
+    public void unsetIo() {
+      this.io = null;
+    }
+
+    /** Returns true if field io is set (has been assigned a value) and false otherwise */
+    public boolean isSetIo() {
+      return this.io != null;
+    }
+
+    public void setIoIsSet(boolean value) {
+      if (!value) {
+        this.io = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case SUCCESS:
+        if (value == null) {
+          unsetSuccess();
+        } else {
+          setSuccess((List<TResult>)value);
+        }
+        break;
+
+      case IO:
+        if (value == null) {
+          unsetIo();
+        } else {
+          setIo((TIOError)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case SUCCESS:
+        return getSuccess();
+
+      case IO:
+        return getIo();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case SUCCESS:
+        return isSetSuccess();
+      case IO:
+        return isSetIo();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof getScannerResults_result)
+        return this.equals((getScannerResults_result)that);
+      return false;
+    }
+
+    public boolean equals(getScannerResults_result that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_success = true && this.isSetSuccess();
+      boolean that_present_success = true && that.isSetSuccess();
+      if (this_present_success || that_present_success) {
+        if (!(this_present_success && that_present_success))
+          return false;
+        if (!this.success.equals(that.success))
+          return false;
+      }
+
+      boolean this_present_io = true && this.isSetIo();
+      boolean that_present_io = true && that.isSetIo();
+      if (this_present_io || that_present_io) {
+        if (!(this_present_io && that_present_io))
+          return false;
+        if (!this.io.equals(that.io))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(getScannerResults_result other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      getScannerResults_result typedOther = (getScannerResults_result)other;
+
+      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetSuccess()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetIo()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+      }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("getScannerResults_result(");
+      boolean first = true;
+
+      sb.append("success:");
+      if (this.success == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.success);
+      }
+      first = false;
+      if (!first) sb.append(", ");
+      sb.append("io:");
+      if (this.io == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.io);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private static class getScannerResults_resultStandardSchemeFactory implements SchemeFactory {
+      public getScannerResults_resultStandardScheme getScheme() {
+        return new getScannerResults_resultStandardScheme();
+      }
+    }
+
+    private static class getScannerResults_resultStandardScheme extends StandardScheme<getScannerResults_result> {
+
+      public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerResults_result struct) throws org.apache.thrift.TException {
+        org.apache.thrift.protocol.TField schemeField;
+        iprot.readStructBegin();
+        while (true)
+        {
+          schemeField = iprot.readFieldBegin();
+          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+            break;
+          }
+          switch (schemeField.id) {
+            case 0: // SUCCESS
+              if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+                {
+                  org.apache.thrift.protocol.TList _list144 = iprot.readListBegin();
+                  struct.success = new ArrayList<TResult>(_list144.size);
+                  for (int _i145 = 0; _i145 < _list144.size; ++_i145)
+                  {
+                    TResult _elem146; // required
+                    _elem146 = new TResult();
+                    _elem146.read(iprot);
+                    struct.success.add(_elem146);
+                  }
+                  iprot.readListEnd();
+                }
+                struct.setSuccessIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            case 1: // IO
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.io = new TIOError();
+                struct.io.read(iprot);
+                struct.setIoIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
+            default:
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+          }
+          iprot.readFieldEnd();
+        }
+        iprot.readStructEnd();
+
+        // check for required fields of primitive type, which can't be checked in the validate method
+        struct.validate();
+      }
+
+      public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerResults_result struct) throws org.apache.thrift.TException {
+        struct.validate();
+
+        oprot.writeStructBegin(STRUCT_DESC);
+        if (struct.success != null) {
+          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+          {
+            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
+            for (TResult _iter147 : struct.success)
+            {
+              _iter147.write(oprot);
+            }
+            oprot.writeListEnd();
+          }
+          oprot.writeFieldEnd();
+        }
+        if (struct.io != null) {
+          oprot.writeFieldBegin(IO_FIELD_DESC);
+          struct.io.write(oprot);
+          oprot.writeFieldEnd();
+        }
+        oprot.writeFieldStop();
+        oprot.writeStructEnd();
+      }
+
+    }
+
+    private static class getScannerResults_resultTupleSchemeFactory implements SchemeFactory {
+      public getScannerResults_resultTupleScheme getScheme() {
+        return new getScannerResults_resultTupleScheme();
+      }
+    }
+
+    private static class getScannerResults_resultTupleScheme extends TupleScheme<getScannerResults_result> {
+
+      @Override
+      public void write(org.apache.thrift.protocol.TProtocol prot, getScannerResults_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol oprot = (TTupleProtocol) prot;
+        BitSet optionals = new BitSet();
+        if (struct.isSetSuccess()) {
+          optionals.set(0);
+        }
+        if (struct.isSetIo()) {
+          optionals.set(1);
+        }
+        oprot.writeBitSet(optionals, 2);
+        if (struct.isSetSuccess()) {
+          {
+            oprot.writeI32(struct.success.size());
+            for (TResult _iter148 : struct.success)
+            {
+              _iter148.write(oprot);
+            }
+          }
+        }
+        if (struct.isSetIo()) {
+          struct.io.write(oprot);
+        }
+      }
+
+      @Override
+      public void read(org.apache.thrift.protocol.TProtocol prot, getScannerResults_result struct) throws org.apache.thrift.TException {
+        TTupleProtocol iprot = (TTupleProtocol) prot;
+        BitSet incoming = iprot.readBitSet(2);
+        if (incoming.get(0)) {
+          {
+            org.apache.thrift.protocol.TList _list149 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TResult>(_list149.size);
+            for (int _i150 = 0; _i150 < _list149.size; ++_i150)
+            {
+              TResult _elem151; // required
+              _elem151 = new TResult();
+              _elem151.read(iprot);
+              struct.success.add(_elem151);
+            }
+          }
+          struct.setSuccessIsSet(true);
+        }
+        if (incoming.get(1)) {
+          struct.io = new TIOError();
+          struct.io.read(iprot);
+          struct.setIoIsSet(true);
+        }
+      }
+    }
+
+  }
+
 }
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
index 11e77c8322a5..ca3d3d6f02c0 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
@@ -415,10 +415,9 @@ service THBaseService {
   )
 
   /**
-   * Closes the scanner. Should be called if you need to close
-   * the Scanner before all results are read.
-   *
-   * Exhausted scanners are closed automatically.
+   * Closes the scanner. Should be called to free server side resources timely.
+   * Typically close once the scanner is not needed anymore, i.e. after looping
+   * over it to get all the required rows.
    */
   void closeScanner(
     /** the Id of the Scanner to close **/
@@ -441,4 +440,23 @@ service THBaseService {
     2: required TRowMutations rowMutations
   ) throws (1: TIOError io)
 
+  /**
+   * Get results for the provided TScan object.
+   * This helper function opens a scanner, get the results and close the scanner.
+   *
+   * @return between zero and numRows TResults
+   */
+  list<TResult> getScannerResults(
+    /** the table to get the Scanner for */
+    1: required binary table,
+
+    /** the scan object to get a Scanner for */
+    2: required TScan scan,
+
+    /** number of rows to return */
+    3: i32 numRows = 1
+  ) throws (
+    1: TIOError io
+  )
+
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index b9c1ed9672f7..4278de9212f0 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -665,6 +665,62 @@ public void testScanWithBatchSize() throws Exception {
     } catch (TIllegalArgument e) {
     }
   }
+  @Test
+  public void testGetScannerResults() throws Exception {
+    ThriftHBaseServiceHandler handler = createHandler();
+    ByteBuffer table = wrap(tableAname);
+
+    // insert data
+    TColumnValue columnValue =
+        new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
+    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
+    columnValues.add(columnValue);
+    for (int i = 0; i < 20; i++) {
+      TPut put =
+          new TPut(wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()), columnValues);
+      handler.put(table, put);
+    }
+
+    // create scan instance
+    TScan scan = new TScan();
+    List<TColumn> columns = new ArrayList<TColumn>();
+    TColumn column = new TColumn();
+    column.setFamily(familyAname);
+    column.setQualifier(qualifierAname);
+    columns.add(column);
+    scan.setColumns(columns);
+    scan.setStartRow("testGetScannerResults".getBytes());
+
+    // get 5 rows and check the returned results
+    scan.setStopRow("testGetScannerResults05".getBytes());
+    List<TResult> results = handler.getScannerResults(table, scan, 5);
+    assertEquals(5, results.size());
+    for (int i = 0; i < 5; i++) {
+      // check if the rows are returned and in order
+      assertArrayEquals(("testGetScannerResults" + pad(i, (byte) 2)).getBytes(), results.get(i)
+          .getRow());
+    }
+
+    // get 10 rows and check the returned results
+    scan.setStopRow("testGetScannerResults10".getBytes());
+    results = handler.getScannerResults(table, scan, 10);
+    assertEquals(10, results.size());
+    for (int i = 0; i < 10; i++) {
+      // check if the rows are returned and in order
+      assertArrayEquals(("testGetScannerResults" + pad(i, (byte) 2)).getBytes(), results.get(i)
+          .getRow());
+    }
+
+    // get 20 rows and check the returned results
+    scan.setStopRow("testGetScannerResults20".getBytes());
+    results = handler.getScannerResults(table, scan, 20);
+    assertEquals(20, results.size());
+    for (int i = 0; i < 20; i++) {
+      // check if the rows are returned and in order
+      assertArrayEquals(("testGetScannerResults" + pad(i, (byte) 2)).getBytes(), results.get(i)
+          .getRow());
+    }
+  }
 
   @Test
   public void testFilterRegistration() throws Exception {

From 5e9351245041b2ac619cc111846d513504163994 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Sun, 21 Jul 2013 09:31:58 +0000
Subject: [PATCH 1074/1540] HBASE-8948 Bound table pool size in Thrift 2 server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1505323 13f79535-47bb-0310-9956-ffa450edef68
---
 .../thrift2/ThriftHBaseServiceHandler.java    |  3 +-
 src/main/resources/hbase-default.xml          | 33 ++++++++++++-------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index c9f838ca9629..0fa9830be4ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -117,7 +117,8 @@ private static long now() {
   }
 
   ThriftHBaseServiceHandler(Configuration conf) {
-    htablePool = new HTablePool(conf, Integer.MAX_VALUE);
+    int maxPoolSize = conf.getInt("hbase.thrift.htablepool.size.max", 1000);
+    htablePool = new HTablePool(conf, maxPoolSize);
   }
 
   private HTableInterface getTable(ByteBuffer tableName) {
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 328b748277ae..365e58726f7d 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -880,25 +880,34 @@
     </description>
   </property>
   <property>
-     <name>hbase.thrift.maxQueuedRequests</name>
-     <value>1000</value>
-     <description>
+    <name>hbase.thrift.maxQueuedRequests</name>
+    <value>1000</value>
+    <description>
      The maximum number of pending Thrift connections waiting in the queue. If
      there are no idle threads in the pool, the server queues requests. Only
      when the queue overflows, new threads are added, up to
      hbase.thrift.maxQueuedRequests threads.
-     </description>
-   </property>
-     <property>
-     <name>hbase.offheapcache.percentage</name>
-     <value>0</value>
-     <description>
+    </description>
+  </property>
+  <property>
+    <name>hbase.thrift.htablepool.size.max</name>
+    <value>1000</value>
+    <description>The upper bound for the table pool used in the Thrift gateways server.
+      Since this is per table name, we assume a single table and so with 1000 default
+      worker threads max this is set to a matching number. For other workloads this number
+      can be adjusted as needed.
+    </description>
+  </property>
+  <property>
+    <name>hbase.offheapcache.percentage</name>
+    <value>0</value>
+    <description>
      The amount of off heap space to be allocated towards the experimental
      off heap cache. If you desire the cache to be disabled, simply set this
      value to 0.
-     </description>
-   </property>
-   <property>
+    </description>
+  </property>
+  <property>
     <name>hbase.data.umask.enable</name>
     <value>false</value>
     <description>Enable, if true, that file permissions should be assigned

From ae050587242d9a06b5791fef2e8155ad698aff09 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 22 Jul 2013 20:34:28 +0000
Subject: [PATCH 1075/1540] HBASE-8995 Add hadoop-1.2 profile (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1505796 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)

diff --git a/pom.xml b/pom.xml
index 160d2ac2fc0a..e609b62cea7d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1804,6 +1804,87 @@
       </build>
     </profile>
 
+    <!--
+      profile for building against Hadoop 1.2. Activate using:
+       mvn -Dhadoop.profile=1.2
+    -->
+    <profile>
+      <id>hadoop-1.2</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>1.2</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>1.2.0</hadoop.version>
+        <slf4j.version>1.4.3</slf4j.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-core</artifactId>
+          <version>${hadoop.version}</version>
+          <optional>true</optional>
+          <exclusions>
+            <exclusion>
+              <groupId>hsqldb</groupId>
+              <artifactId>hsqldb</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>net.sf.kosmosfs</groupId>
+              <artifactId>kfs</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.eclipse.jdt</groupId>
+              <artifactId>core</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>net.java.dev.jets3t</groupId>
+              <artifactId>jets3t</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>oro</groupId>
+              <artifactId>oro</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-test</artifactId>
+          <version>${hadoop.version}</version>
+          <optional>true</optional>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-resource</id>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>src/test/resources</directory>
+                      <includes>
+                        <include>hbase-site.xml</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!--
       profile for building against Hadoop 0.22.0. Activate using:
        mvn -Dhadoop.profile=22

From 7b2a7eb46c47a0aba1bc5e2760b19119cf75fc4d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 23 Jul 2013 17:38:15 +0000
Subject: [PATCH 1076/1540] HBASE-9019 Port HBASE-8690: Reduce unnecessary
 getFileStatus hdfs calls in TTL hfile and hlog cleanners to 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1506176 13f79535-47bb-0310-9956-ffa450edef68
---
 .../cleaner/BaseLogCleanerDelegate.java       | 13 +++---
 .../hbase/master/cleaner/CleanerChore.java    | 11 ++---
 .../master/cleaner/FileCleanerDelegate.java   |  8 ++--
 .../master/cleaner/HFileLinkCleaner.java      |  5 ++-
 .../cleaner/TimeToLiveHFileCleaner.java       | 41 +++----------------
 .../master/cleaner/TimeToLiveLogCleaner.java  | 35 +++++++---------
 .../master/snapshot/SnapshotHFileCleaner.java |  9 ++--
 .../master/snapshot/SnapshotLogCleaner.java   |  9 ++--
 .../master/ReplicationLogCleaner.java         |  7 ++--
 .../master/cleaner/TestCleanerChore.java      | 19 +++++----
 .../master/cleaner/TestHFileCleaner.java      |  2 +-
 .../snapshot/TestSnapshotHFileCleaner.java    |  4 +-
 .../snapshot/TestSnapshotLogCleaner.java      |  4 +-
 .../CheckedArchivingHFileCleaner.java         |  4 +-
 14 files changed, 72 insertions(+), 99 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
index d62d347d4868..3b85c145455e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.master.cleaner;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.BaseConfigurable;
 
@@ -40,17 +41,17 @@
 public abstract class BaseLogCleanerDelegate extends BaseConfigurable implements FileCleanerDelegate {
 
   @Override
-  public boolean isFileDeletable(Path file) {
-    return isLogDeletable(file);
+  public boolean isFileDeletable(FileStatus fStat) {
+    return isLogDeletable(fStat);
   }
 
   /**
    * Should the master delete the log or keep it?
    * <p>
-   * Implementing classes should override {@link #isFileDeletable(Path)} instead.
-   * @param filePath full path to log.
+   * Implementing classes should override {@link #isFileDeletable(FileStatus)} instead.
+   * @param fStat file status of the file
    * @return true if the log is deletable, false if not
    */
   @Deprecated
-  public abstract boolean isLogDeletable(Path filePath);
-}
\ No newline at end of file
+  public abstract boolean isLogDeletable(FileStatus fStat);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 0fa53c045389..7d214604e3af 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -123,7 +123,7 @@ protected void chore() {
       for (FileStatus file : files) {
         try {
           if (file.isDir()) checkAndDeleteDirectory(file.getPath());
-          else checkAndDelete(file.getPath());
+          else checkAndDelete(file);
         } catch (IOException e) {
           e = RemoteExceptionHandler.checkIOException(e);
           LOG.warn("Error while cleaning the logs", e);
@@ -174,7 +174,7 @@ public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
         }
       }
       // otherwise we can just check the file
-      else if (!checkAndDelete(path)) {
+      else if (!checkAndDelete(child)) {
         canDeleteThis = false;
       }
     }
@@ -200,11 +200,12 @@ else if (!checkAndDelete(path)) {
   /**
    * Run the given file through each of the cleaners to see if it should be deleted, deleting it if
    * necessary.
-   * @param filePath path of the file to check (and possibly delete)
+   * @param fStat path of the file to check (and possibly delete)
    * @throws IOException if cann't delete a file because of a filesystem issue
    * @throws IllegalArgumentException if the file is a directory and has children
    */
-  private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
+  private boolean checkAndDelete(FileStatus fStat) throws IOException, IllegalArgumentException {
+    Path filePath = fStat.getPath();
     // first check to see if the path is valid
     if (!validate(filePath)) {
       LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
@@ -222,7 +223,7 @@ private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumen
         return false;
       }
 
-      if (!cleaner.isFileDeletable(filePath)) {
+      if (!cleaner.isFileDeletable(fStat)) {
         // this file is not deletable, then we are done
         if (LOG.isTraceEnabled()) {
           LOG.trace(filePath + " is not deletable according to:" + cleaner);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
index 4cf503881c79..c03278195d70 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
@@ -19,7 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hbase.Stoppable;
 
 /**
@@ -32,8 +32,8 @@ public interface FileCleanerDelegate extends Configurable, Stoppable {
 
   /**
    * Should the master delete the file or keep it?
-   * @param file full path to the file to check
+   * @param fStat file status of the file to check
    * @return <tt>true</tt> if the file is deletable, <tt>false</tt> if not
    */
-  public boolean isFileDeletable(Path file);
-}
\ No newline at end of file
+  public boolean isFileDeletable(FileStatus fStat);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
index 4da1b9b6fe72..4c39f7353b32 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -48,9 +49,9 @@ public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
   private FileSystem fs = null;
 
   @Override
-  public synchronized boolean isFileDeletable(Path filePath) {
+  public synchronized boolean isFileDeletable(FileStatus fStat) {
     if (this.fs == null) return false;
-
+    Path filePath = fStat.getPath();
     // HFile Link is always deletable
     if (HFileLink.isHFileLink(filePath)) return true;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
index 82231afbe4f0..00d33f676202 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
@@ -17,15 +17,11 @@
  */
 package org.apache.hadoop.hbase.master.cleaner;
 
-import java.io.IOException;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 /**
@@ -41,7 +37,6 @@ public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate {
   private static final long DEFAULT_TTL = 60000 * 5;
   // Configured time a hfile can be kept after it was moved to the archive
   private long ttl;
-  private FileSystem fs;
 
   @Override
   public void setConf(Configuration conf) {
@@ -50,45 +45,19 @@ public void setConf(Configuration conf) {
   }
 
   @Override
-  public boolean isFileDeletable(Path filePath) {
-    if (!instantiateFS()) {
-      return false;
-    }
-    long time = 0;
+  public boolean isFileDeletable(FileStatus fStat) {
     long currentTime = EnvironmentEdgeManager.currentTimeMillis();
-    try {
-      FileStatus fStat = fs.getFileStatus(filePath);
-      time = fStat.getModificationTime();
-    } catch (IOException e) {
-      LOG.error("Unable to get modification time of file " + filePath.getName()
-          + ", not deleting it.", e);
-      return false;
-    }
+		long time = fStat.getModificationTime();
     long life = currentTime - time;
     if (LOG.isTraceEnabled()) {
       LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: "
           + time);
     }
     if (life < 0) {
-      LOG.warn("Found a log (" + filePath + ") newer than current time (" + currentTime + " < "
-          + time + "), probably a clock skew");
+      LOG.warn("Found a hfile (" + fStat.getPath() + ") newer than current time (" + currentTime
+         + " < " + time + "), probably a clock skew");
       return false;
     }
     return life > ttl;
   }
-
-  /**
-   * setup the filesystem, if it hasn't been already
-   */
-  private synchronized boolean instantiateFS() {
-    if (this.fs == null) {
-      try {
-        this.fs = FileSystem.get(this.getConf());
-      } catch (IOException e) {
-        LOG.error("Couldn't instantiate the file system, not deleting file, just incase");
-        return false;
-      }
-    }
-    return true;
-  }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
index 6043a8a2847d..66b04230ea20 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
@@ -17,14 +17,12 @@
  */
 package org.apache.hadoop.hbase.master.cleaner;
 
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 /**
  * Log cleaner that uses the timestamp of the hlog to determine if it should
@@ -38,21 +36,18 @@ public class TimeToLiveLogCleaner extends BaseLogCleanerDelegate {
   private boolean stopped = false;
 
   @Override
-  public boolean isLogDeletable(Path filePath) {
-    long time = 0;
-    long currentTime = System.currentTimeMillis();
-    try {
-      FileStatus fStat = filePath.getFileSystem(this.getConf()).getFileStatus(filePath);
-      time = fStat.getModificationTime();
-    } catch (IOException e) {
-      LOG.error("Unable to get modification time of file " + filePath.getName() +
-      ", not deleting it.", e);
-      return false;
-    }
+  public boolean isLogDeletable(FileStatus fStat) {
+    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+    long time = fStat.getModificationTime();
     long life = currentTime - time;
+    
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Log life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: "
+          + time);
+    }
     if (life < 0) {
-      LOG.warn("Found a log newer than current time, " +
-          "probably a clock skew");
+      LOG.warn("Found a log (" + fStat.getPath() + ") newer than current time (" + currentTime
+          + " < " + time + "), probably a clock skew");
       return false;
     }
     return life > ttl;
@@ -74,4 +69,4 @@ public void stop(String why) {
   public boolean isStopped() {
     return this.stopped;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
index 7b75fd217211..e82ca1608c45 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
@@ -54,12 +55,12 @@ public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
   private SnapshotFileCache cache;
 
   @Override
-  public synchronized boolean isFileDeletable(Path filePath) {
+  public synchronized boolean isFileDeletable(FileStatus fStat) {
     try {
-      return !cache.contains(filePath.getName());
+      return !cache.contains(fStat.getPath().getName());
     } catch (IOException e) {
-      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
-        e);
+      LOG.error("Exception while checking if:" + fStat.getPath()
+          + " was valid, keeping it just in case.", e);
       return false;
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
index 12af8c429909..c1edd6f7f0db 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
@@ -53,13 +54,13 @@ public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
   private SnapshotFileCache cache;
 
   @Override
-  public synchronized boolean isLogDeletable(Path filePath) {
+  public synchronized boolean isLogDeletable(FileStatus fStat) {
     try {
       if (null == cache) return false;
-      return !cache.contains(filePath.getName());
+      return !cache.contains(fStat.getPath().getName());
     } catch (IOException e) {
-      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
-        e);
+      LOG.error("Exception while checking if:" + fStat.getPath()
+        + " was valid, keeping it just in case.", e);
       return false;
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index ee8941bed885..caf77b948a61 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -22,6 +22,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HConstants;
@@ -53,7 +54,7 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abo
   public ReplicationLogCleaner() {}
 
   @Override
-  public boolean isLogDeletable(Path filePath) {
+  public boolean isLogDeletable(FileStatus fStat) {
 
     try {
       if (!zkHelper.getReplication()) {
@@ -69,7 +70,7 @@ public boolean isLogDeletable(Path filePath) {
     if (this.getConf() == null) {
       return true;
     }
-    String log = filePath.getName();
+    String log = fStat.getPath().getName();
     // If we saw the hlog previously, let's consider it's still used
     // At some point in the future we will refresh the list and it will be gone
     if (this.hlogs.contains(log)) {
@@ -170,4 +171,4 @@ public void abort(String why, Throwable e) {
   public boolean isAborted() {
     return this.aborted;
   }
-}
\ No newline at end of file
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
index 3ce4de300415..17c7109c4329 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -137,13 +138,15 @@ public void testDoesNotCheckDirectories() throws Exception {
     Path parent = new Path(testDir, "parent");
     Path file = new Path(parent, "someFile");
     fs.mkdirs(parent);
+    assertTrue("Test parent didn't get created.", fs.exists(parent));
     // touch a new file
     fs.create(file).close();
     assertTrue("Test file didn't get created.", fs.exists(file));
-
+    
+    FileStatus fStat = fs.getFileStatus(parent);
     chore.chore();
     // make sure we never checked the directory
-    Mockito.verify(spy, Mockito.never()).isFileDeletable(parent);
+    Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat);
     Mockito.reset(spy);
   }
 
@@ -210,7 +213,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable {
         FSUtils.logFileSystemState(fs, testDir, LOG);
         return (Boolean) invocation.callRealMethod();
       }
-    }).when(spy).isFileDeletable(Mockito.any(Path.class));
+    }).when(spy).isFileDeletable(Mockito.any(FileStatus.class));
 
     // run the chore
     chore.chore();
@@ -219,7 +222,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable {
     assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
     assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
     assertFalse("Original file unexpectedly retained", fs.exists(file));
-    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
+    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
     Mockito.reset(spy);
   }
   
@@ -268,7 +271,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable {
         FSUtils.logFileSystemState(fs, testDir, LOG);
         return (Boolean) invocation.callRealMethod();
       }
-    }).when(spy).isFileDeletable(Mockito.any(Path.class));
+    }).when(spy).isFileDeletable(Mockito.any(FileStatus.class));
 
     // attempt to delete the directory, which
     if (chore.checkAndDeleteDirectory(parent)) {
@@ -280,7 +283,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable {
     assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
     assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
     assertFalse("Original file unexpectedly retained", fs.exists(file));
-    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
+    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
   }
 
   private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
@@ -299,14 +302,14 @@ protected boolean validate(Path file) {
 
   public static class AlwaysDelete extends BaseHFileCleanerDelegate {
     @Override
-    public boolean isFileDeletable(Path file) {
+    public boolean isFileDeletable(FileStatus fStat) {
       return true;
     }
   }
 
   public static class NeverDelete extends BaseHFileCleanerDelegate {
     @Override
-    public boolean isFileDeletable(Path file) {
+    public boolean isFileDeletable(FileStatus fStat) {
       return false;
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
index 23920997d92f..b49cbc2826e7 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
@@ -75,7 +75,7 @@ public void testTTLCleaner() throws IOException, InterruptedException {
     conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100);
     cleaner.setConf(conf);
     assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs)
-        + " with create time:" + createTime, cleaner.isFileDeletable(file));
+        + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file)));
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
index 03847c4e5bf4..0b6f1cd6ca7f 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
@@ -84,6 +84,6 @@ public void testFindsSnapshotFilesWhenCleaning() throws IOException {
     fs.createNewFile(new Path(archivedHfileDir, hfile));
 
     // make sure that the file isn't deletable
-    assertFalse(cleaner.isFileDeletable(new Path(hfile)));
+    assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
   }
-}
\ No newline at end of file
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
index f0f2ebd620ca..2c2a5aa0eb67 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
@@ -80,6 +80,6 @@ public void testFindsSnapshotFilesWhenCleaning() throws IOException {
     fs.create(logFile);
 
     // make sure that the file isn't deletable
-    assertFalse(cleaner.isFileDeletable(logFile));
+    assertFalse(cleaner.isFileDeletable(fs.getFileStatus(logFile)));
   }
-}
\ No newline at end of file
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
index efc9cb654d02..c5d9cc9cbfca 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
 
 /**
@@ -31,7 +31,7 @@ public class CheckedArchivingHFileCleaner extends BaseHFileCleanerDelegate {
   private static boolean checked;
 
   @Override
-  public boolean isFileDeletable(Path file) {
+  public boolean isFileDeletable(FileStatus fStat) {
     checked = true;
     return true;
   }

From 98e3c340248a88ad178c85b673a079ae84a2ef85 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 24 Jul 2013 23:28:05 +0000
Subject: [PATCH 1077/1540] Roll version to 0.94.11-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1506780 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index e609b62cea7d..1739c12b25b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.10</version>
+  <version>0.94.11-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From d89714b84d24c950ab845621370cb51fb87cab75 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 24 Jul 2013 23:40:11 +0000
Subject: [PATCH 1078/1540] HBASE-9026 RestartRsHoldingRoot action in
 org.apache.hadoop.hbase.util.ChaosMonkey restarting the server holding .META.
 instead of -ROOT- (Screenivasulu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1506781 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index f3248a34c71e..f3300082f9be 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -266,7 +266,7 @@ public RestartRsHoldingRoot(long sleepTime) {
     @Override
     public void perform() throws Exception {
       LOG.info("Performing action: Restart region server holding ROOT");
-      ServerName server = cluster.getServerHoldingMeta();
+      ServerName server = cluster.getServerHoldingRoot();
       if (server == null) {
         LOG.warn("No server is holding -ROOT- right now.");
         return;

From 61107bec966f4b3b2f355679fa8c62d8fca99aa5 Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Thu, 25 Jul 2013 17:38:37 +0000
Subject: [PATCH 1079/1540] HBASE-8947 Thrift 2 : Replace "bool writeToWAL"
 with "TDurability durability" (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1507074 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/thrift2/ThriftUtilities.java |  23 ++-
 .../hbase/thrift2/generated/TDelete.java      | 132 ++++++++++++++++--
 .../hadoop/hbase/thrift2/generated/TPut.java  | 132 ++++++++++++++++--
 .../apache/hadoop/hbase/thrift2/hbase.thrift  |  25 +++-
 .../TestThriftHBaseServiceHandler.java        |  84 +++++++++++
 5 files changed, 374 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
index b1373fbde177..1e64b4717d34 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
@@ -165,7 +165,11 @@ public static Put putFromThrift(TPut in) {
       out = new Put(in.getRow());
     }
 
-    out.setWriteToWAL(in.isWriteToWal());
+    if (in.isSetDurability()) {
+      out.setDurability(durabilityFromThrift(in.getDurability()));
+    } else if (in.isSetWriteToWal()) {
+      out.setWriteToWAL(in.isWriteToWal());
+    }
 
     for (TColumnValue columnValue : in.getColumnValues()) {
       if (columnValue.isSetTimestamp()) {
@@ -248,7 +252,12 @@ public static Delete deleteFromThrift(TDelete in) {
       addAttributes(out,in.getAttributes());
     }
 
-    out.setWriteToWAL(in.isWriteToWal());
+    if (in.isSetDurability()) {
+      out.setDurability(durabilityFromThrift(in.getDurability()));
+    } else if (in.isSetWriteToWal()) {
+      out.setWriteToWAL(in.isWriteToWal());
+    }
+
     return out;
   }
 
@@ -405,4 +414,14 @@ private static void addAttributes(OperationWithAttributes op,
       op.setAttribute(name, value);
     }
   }
+
+  private static Durability durabilityFromThrift(TDurability tDurability) {
+    switch (tDurability.getValue()) {
+      case 1: return Durability.SKIP_WAL;
+      case 2: return Durability.ASYNC_WAL;
+      case 3: return Durability.SYNC_WAL;
+      case 4: return Durability.FSYNC_WAL;
+      default: return null;
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
index ddf98d442897..0560a4936fa2 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
@@ -59,6 +59,7 @@ public class TDelete implements org.apache.thrift.TBase<TDelete, TDelete._Fields
   private static final org.apache.thrift.protocol.TField DELETE_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("deleteType", org.apache.thrift.protocol.TType.I32, (short)4);
   private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)5);
   private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)6);
+  private static final org.apache.thrift.protocol.TField DURABILITY_FIELD_DESC = new org.apache.thrift.protocol.TField("durability", org.apache.thrift.protocol.TType.I32, (short)7);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -76,6 +77,11 @@ public class TDelete implements org.apache.thrift.TBase<TDelete, TDelete._Fields
   public TDeleteType deleteType; // optional
   public boolean writeToWal; // optional
   public Map<ByteBuffer,ByteBuffer> attributes; // optional
+  /**
+   * 
+   * @see TDurability
+   */
+  public TDurability durability; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -88,7 +94,12 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
      */
     DELETE_TYPE((short)4, "deleteType"),
     WRITE_TO_WAL((short)5, "writeToWal"),
-    ATTRIBUTES((short)6, "attributes");
+    ATTRIBUTES((short)6, "attributes"),
+    /**
+     * 
+     * @see TDurability
+     */
+    DURABILITY((short)7, "durability");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -115,6 +126,8 @@ public static _Fields findByThriftId(int fieldId) {
           return WRITE_TO_WAL;
         case 6: // ATTRIBUTES
           return ATTRIBUTES;
+        case 7: // DURABILITY
+          return DURABILITY;
         default:
           return null;
       }
@@ -158,7 +171,7 @@ public String getFieldName() {
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __WRITETOWAL_ISSET_ID = 1;
   private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.WRITE_TO_WAL,_Fields.ATTRIBUTES};
+  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.WRITE_TO_WAL,_Fields.ATTRIBUTES,_Fields.DURABILITY};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -177,6 +190,8 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
             new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true), 
             new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
+    tmpMap.put(_Fields.DURABILITY, new org.apache.thrift.meta_data.FieldMetaData("durability", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TDurability.class)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDelete.class, metaDataMap);
   }
@@ -184,8 +199,6 @@ public String getFieldName() {
   public TDelete() {
     this.deleteType = org.apache.hadoop.hbase.thrift2.generated.TDeleteType.DELETE_COLUMNS;
 
-    this.writeToWal = true;
-
   }
 
   public TDelete(
@@ -234,6 +247,9 @@ public TDelete(TDelete other) {
       }
       this.attributes = __this__attributes;
     }
+    if (other.isSetDurability()) {
+      this.durability = other.durability;
+    }
   }
 
   public TDelete deepCopy() {
@@ -248,9 +264,10 @@ public void clear() {
     this.timestamp = 0;
     this.deleteType = org.apache.hadoop.hbase.thrift2.generated.TDeleteType.DELETE_COLUMNS;
 
-    this.writeToWal = true;
-
+    setWriteToWalIsSet(false);
+    this.writeToWal = false;
     this.attributes = null;
+    this.durability = null;
   }
 
   public byte[] getRow() {
@@ -439,6 +456,38 @@ public void setAttributesIsSet(boolean value) {
     }
   }
 
+  /**
+   * 
+   * @see TDurability
+   */
+  public TDurability getDurability() {
+    return this.durability;
+  }
+
+  /**
+   * 
+   * @see TDurability
+   */
+  public TDelete setDurability(TDurability durability) {
+    this.durability = durability;
+    return this;
+  }
+
+  public void unsetDurability() {
+    this.durability = null;
+  }
+
+  /** Returns true if field durability is set (has been assigned a value) and false otherwise */
+  public boolean isSetDurability() {
+    return this.durability != null;
+  }
+
+  public void setDurabilityIsSet(boolean value) {
+    if (!value) {
+      this.durability = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -489,6 +538,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case DURABILITY:
+      if (value == null) {
+        unsetDurability();
+      } else {
+        setDurability((TDurability)value);
+      }
+      break;
+
     }
   }
 
@@ -512,6 +569,9 @@ public Object getFieldValue(_Fields field) {
     case ATTRIBUTES:
       return getAttributes();
 
+    case DURABILITY:
+      return getDurability();
+
     }
     throw new IllegalStateException();
   }
@@ -535,6 +595,8 @@ public boolean isSet(_Fields field) {
       return isSetWriteToWal();
     case ATTRIBUTES:
       return isSetAttributes();
+    case DURABILITY:
+      return isSetDurability();
     }
     throw new IllegalStateException();
   }
@@ -606,6 +668,15 @@ public boolean equals(TDelete that) {
         return false;
     }
 
+    boolean this_present_durability = true && this.isSetDurability();
+    boolean that_present_durability = true && that.isSetDurability();
+    if (this_present_durability || that_present_durability) {
+      if (!(this_present_durability && that_present_durability))
+        return false;
+      if (!this.durability.equals(that.durability))
+        return false;
+    }
+
     return true;
   }
 
@@ -682,6 +753,16 @@ public int compareTo(TDelete other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetDurability()).compareTo(typedOther.isSetDurability());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetDurability()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.durability, typedOther.durability);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -751,6 +832,16 @@ public String toString() {
       }
       first = false;
     }
+    if (isSetDurability()) {
+      if (!first) sb.append(", ");
+      sb.append("durability:");
+      if (this.durability == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.durability);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -869,6 +960,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) thr
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 7: // DURABILITY
+            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+              struct.durability = TDurability.findByValue(iprot.readI32());
+              struct.setDurabilityIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -935,6 +1034,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TDelete struct) th
           oprot.writeFieldEnd();
         }
       }
+      if (struct.durability != null) {
+        if (struct.isSetDurability()) {
+          oprot.writeFieldBegin(DURABILITY_FIELD_DESC);
+          oprot.writeI32(struct.durability.getValue());
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -969,7 +1075,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thr
       if (struct.isSetAttributes()) {
         optionals.set(4);
       }
-      oprot.writeBitSet(optionals, 5);
+      if (struct.isSetDurability()) {
+        optionals.set(5);
+      }
+      oprot.writeBitSet(optionals, 6);
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
@@ -998,6 +1107,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thr
           }
         }
       }
+      if (struct.isSetDurability()) {
+        oprot.writeI32(struct.durability.getValue());
+      }
     }
 
     @Override
@@ -1005,7 +1117,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro
       TTupleProtocol iprot = (TTupleProtocol) prot;
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
-      BitSet incoming = iprot.readBitSet(5);
+      BitSet incoming = iprot.readBitSet(6);
       if (incoming.get(0)) {
         {
           org.apache.thrift.protocol.TList _list55 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
@@ -1047,6 +1159,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro
         }
         struct.setAttributesIsSet(true);
       }
+      if (incoming.get(5)) {
+        struct.durability = TDurability.findByValue(iprot.readI32());
+        struct.setDurabilityIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
index b35d134ff316..1cf071db37b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
@@ -46,6 +46,7 @@ public class TPut implements org.apache.thrift.TBase<TPut, TPut._Fields>, java.i
   private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3);
   private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)4);
   private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5);
+  private static final org.apache.thrift.protocol.TField DURABILITY_FIELD_DESC = new org.apache.thrift.protocol.TField("durability", org.apache.thrift.protocol.TType.I32, (short)6);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -58,6 +59,11 @@ public class TPut implements org.apache.thrift.TBase<TPut, TPut._Fields>, java.i
   public long timestamp; // optional
   public boolean writeToWal; // optional
   public Map<ByteBuffer,ByteBuffer> attributes; // optional
+  /**
+   * 
+   * @see TDurability
+   */
+  public TDurability durability; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -65,7 +71,12 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     COLUMN_VALUES((short)2, "columnValues"),
     TIMESTAMP((short)3, "timestamp"),
     WRITE_TO_WAL((short)4, "writeToWal"),
-    ATTRIBUTES((short)5, "attributes");
+    ATTRIBUTES((short)5, "attributes"),
+    /**
+     * 
+     * @see TDurability
+     */
+    DURABILITY((short)6, "durability");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -90,6 +101,8 @@ public static _Fields findByThriftId(int fieldId) {
           return WRITE_TO_WAL;
         case 5: // ATTRIBUTES
           return ATTRIBUTES;
+        case 6: // DURABILITY
+          return DURABILITY;
         default:
           return null;
       }
@@ -133,7 +146,7 @@ public String getFieldName() {
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __WRITETOWAL_ISSET_ID = 1;
   private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.TIMESTAMP,_Fields.WRITE_TO_WAL,_Fields.ATTRIBUTES};
+  private _Fields optionals[] = {_Fields.TIMESTAMP,_Fields.WRITE_TO_WAL,_Fields.ATTRIBUTES,_Fields.DURABILITY};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -150,13 +163,13 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
             new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true), 
             new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
+    tmpMap.put(_Fields.DURABILITY, new org.apache.thrift.meta_data.FieldMetaData("durability", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TDurability.class)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TPut.class, metaDataMap);
   }
 
   public TPut() {
-    this.writeToWal = true;
-
   }
 
   public TPut(
@@ -204,6 +217,9 @@ public TPut(TPut other) {
       }
       this.attributes = __this__attributes;
     }
+    if (other.isSetDurability()) {
+      this.durability = other.durability;
+    }
   }
 
   public TPut deepCopy() {
@@ -216,9 +232,10 @@ public void clear() {
     this.columnValues = null;
     setTimestampIsSet(false);
     this.timestamp = 0;
-    this.writeToWal = true;
-
+    setWriteToWalIsSet(false);
+    this.writeToWal = false;
     this.attributes = null;
+    this.durability = null;
   }
 
   public byte[] getRow() {
@@ -375,6 +392,38 @@ public void setAttributesIsSet(boolean value) {
     }
   }
 
+  /**
+   * 
+   * @see TDurability
+   */
+  public TDurability getDurability() {
+    return this.durability;
+  }
+
+  /**
+   * 
+   * @see TDurability
+   */
+  public TPut setDurability(TDurability durability) {
+    this.durability = durability;
+    return this;
+  }
+
+  public void unsetDurability() {
+    this.durability = null;
+  }
+
+  /** Returns true if field durability is set (has been assigned a value) and false otherwise */
+  public boolean isSetDurability() {
+    return this.durability != null;
+  }
+
+  public void setDurabilityIsSet(boolean value) {
+    if (!value) {
+      this.durability = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -417,6 +466,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case DURABILITY:
+      if (value == null) {
+        unsetDurability();
+      } else {
+        setDurability((TDurability)value);
+      }
+      break;
+
     }
   }
 
@@ -437,6 +494,9 @@ public Object getFieldValue(_Fields field) {
     case ATTRIBUTES:
       return getAttributes();
 
+    case DURABILITY:
+      return getDurability();
+
     }
     throw new IllegalStateException();
   }
@@ -458,6 +518,8 @@ public boolean isSet(_Fields field) {
       return isSetWriteToWal();
     case ATTRIBUTES:
       return isSetAttributes();
+    case DURABILITY:
+      return isSetDurability();
     }
     throw new IllegalStateException();
   }
@@ -520,6 +582,15 @@ public boolean equals(TPut that) {
         return false;
     }
 
+    boolean this_present_durability = true && this.isSetDurability();
+    boolean that_present_durability = true && that.isSetDurability();
+    if (this_present_durability || that_present_durability) {
+      if (!(this_present_durability && that_present_durability))
+        return false;
+      if (!this.durability.equals(that.durability))
+        return false;
+    }
+
     return true;
   }
 
@@ -586,6 +657,16 @@ public int compareTo(TPut other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetDurability()).compareTo(typedOther.isSetDurability());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetDurability()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.durability, typedOther.durability);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -643,6 +724,16 @@ public String toString() {
       }
       first = false;
     }
+    if (isSetDurability()) {
+      if (!first) sb.append(", ");
+      sb.append("durability:");
+      if (this.durability == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.durability);
+      }
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -756,6 +847,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TPut struct) throws
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 6: // DURABILITY
+            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+              struct.durability = TDurability.findByValue(iprot.readI32());
+              struct.setDurabilityIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -813,6 +912,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TPut struct) throw
           oprot.writeFieldEnd();
         }
       }
+      if (struct.durability != null) {
+        if (struct.isSetDurability()) {
+          oprot.writeFieldBegin(DURABILITY_FIELD_DESC);
+          oprot.writeI32(struct.durability.getValue());
+          oprot.writeFieldEnd();
+        }
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -848,7 +954,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
       if (struct.isSetAttributes()) {
         optionals.set(2);
       }
-      oprot.writeBitSet(optionals, 3);
+      if (struct.isSetDurability()) {
+        optionals.set(3);
+      }
+      oprot.writeBitSet(optionals, 4);
       if (struct.isSetTimestamp()) {
         oprot.writeI64(struct.timestamp);
       }
@@ -865,6 +974,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
           }
         }
       }
+      if (struct.isSetDurability()) {
+        oprot.writeI32(struct.durability.getValue());
+      }
     }
 
     @Override
@@ -884,7 +996,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
         }
       }
       struct.setColumnValuesIsSet(true);
-      BitSet incoming = iprot.readBitSet(3);
+      BitSet incoming = iprot.readBitSet(4);
       if (incoming.get(0)) {
         struct.timestamp = iprot.readI64();
         struct.setTimestampIsSet(true);
@@ -908,6 +1020,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
         }
         struct.setAttributesIsSet(true);
       }
+      if (incoming.get(3)) {
+        struct.durability = TDurability.findByValue(iprot.readI32());
+        struct.setDurabilityIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
index ca3d3d6f02c0..502cea925aa5 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
@@ -77,6 +77,21 @@ enum TDeleteType {
   DELETE_COLUMNS = 1
 }
 
+/**
+ * Specify Durability:
+ *  - SKIP_WAL means do not write the Mutation to the WAL.
+ *  - ASYNC_WAL means write the Mutation to the WAL asynchronously,
+ *  - SYNC_WAL means write the Mutation to the WAL synchronously,
+ *  - FSYNC_WAL means Write the Mutation to the WAL synchronously and force the entries to disk.
+ */
+
+enum TDurability {
+  SKIP_WAL = 1,
+  ASYNC_WAL = 2,
+  SYNC_WAL = 3,
+  FSYNC_WAL = 4
+}
+
 /**
  * Used to perform Get operations on a single row.
  *
@@ -117,8 +132,9 @@ struct TPut {
   1: required binary row,
   2: required list<TColumnValue> columnValues
   3: optional i64 timestamp,
-  4: optional bool writeToWal = 1,
-  5: optional map<binary, binary> attributes
+  4: optional bool writeToWal,
+  5: optional map<binary, binary> attributes,
+  6: optional TDurability durability
 }
 
 /**
@@ -149,8 +165,9 @@ struct TDelete {
   2: optional list<TColumn> columns,
   3: optional i64 timestamp,
   4: optional TDeleteType deleteType = 1,
-  5: optional bool writeToWal = 1,
-  6: optional map<binary, binary> attributes
+  5: optional bool writeToWal,
+  6: optional map<binary, binary> attributes,
+  7: optional TDurability durability
 }
 
 /**
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 4278de9212f0..3578d10e4063 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.filter.ParseFilter;
 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
@@ -57,6 +58,7 @@
 import org.apache.hadoop.hbase.thrift2.generated.TScan;
 import org.apache.hadoop.hbase.thrift2.generated.TMutation;
 import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
+import org.apache.hadoop.hbase.thrift2.generated.TDurability;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.MetricsContext;
@@ -851,6 +853,88 @@ public void testMutateRow() throws Exception {
     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
   }
 
+  /**
+   * Create TPut, TDelete , TIncrement objects, set durability then call ThriftUtility
+   * functions to get Put , Delete and Increment respectively. Use getDurability to make sure
+   * the returned objects have the appropriate durability setting.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testDurability() throws Exception {
+    byte[] rowName = "testDurability".getBytes();
+    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
+    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
+
+    List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
+    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
+
+    TDelete tDelete = new TDelete(wrap(rowName));
+
+    //if not setting writeToWal, check for default value
+    Delete delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.USE_DEFAULT);
+
+    //if setting writeToWal to true, durability should be CF default
+    tDelete.setWriteToWal(true);
+    delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.USE_DEFAULT);
+
+    //if setting writeToWal to false, durability should be SKIP_WAL
+    tDelete.setWriteToWal(false);
+    delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.SKIP_WAL);
+
+
+    tDelete.setDurability(TDurability.SKIP_WAL);
+    delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.SKIP_WAL);
+
+    tDelete.setDurability(TDurability.ASYNC_WAL);
+    delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.ASYNC_WAL);
+
+    tDelete.setDurability(TDurability.SYNC_WAL);
+    delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.SYNC_WAL);
+
+    tDelete.setDurability(TDurability.FSYNC_WAL);
+    delete = deleteFromThrift(tDelete);
+    assertEquals(delete.getDurability(), Durability.FSYNC_WAL);
+
+    TPut tPut = new TPut(wrap(rowName), columnValues);
+
+    //if not setting writeToWal, check for default value
+    Put put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.USE_DEFAULT);
+
+    //if setting writeToWal to true, durability should be CF default
+    tPut.setWriteToWal(true);
+    put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.USE_DEFAULT);
+
+    //if setting writeToWal to false, durability should be SKIP_WAL
+    tPut.setWriteToWal(false);
+    put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.SKIP_WAL);
+
+    tPut.setDurability(TDurability.SKIP_WAL);
+    put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.SKIP_WAL);
+
+    tPut.setDurability(TDurability.ASYNC_WAL);
+    put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.ASYNC_WAL);
+
+    tPut.setDurability(TDurability.SYNC_WAL);
+    put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.SYNC_WAL);
+
+    tPut.setDurability(TDurability.FSYNC_WAL);
+    put = putFromThrift(tPut);
+    assertEquals(put.getDurability(), Durability.FSYNC_WAL);
+  }
+
   private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
     setupMetricsContext();
     return new ThriftMetrics(Integer.parseInt(ThriftServer.DEFAULT_LISTEN_PORT),

From a16982c772ba8977748270112c54189346a204bc Mon Sep 17 00:00:00 2001
From: Lars George <larsgeorge@apache.org>
Date: Thu, 25 Jul 2013 19:29:24 +0000
Subject: [PATCH 1080/1540] HBASE-8947 Thrift 2 : Replace "bool writeToWAL"
 with "TDurability durability" (Hamed Madani)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1507102 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/thrift2/generated/TDurability.java  | 58 +++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java
new file mode 100644
index 000000000000..cffe84c40245
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java
@@ -0,0 +1,58 @@
+/**
+ * Autogenerated by Thrift Compiler (0.8.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hbase.thrift2.generated;
+
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.thrift.TEnum;
+
+/**
+ * Specify Durability:
+ *  - SKIP_WAL means do not write the Mutation to the WAL.
+ *  - ASYNC_WAL means write the Mutation to the WAL asynchronously,
+ *  - SYNC_WAL means write the Mutation to the WAL synchronously,
+ *  - FSYNC_WAL means Write the Mutation to the WAL synchronously and force the entries to disk.
+ */
+public enum TDurability implements org.apache.thrift.TEnum {
+  SKIP_WAL(1),
+  ASYNC_WAL(2),
+  SYNC_WAL(3),
+  FSYNC_WAL(4);
+
+  private final int value;
+
+  private TDurability(int value) {
+    this.value = value;
+  }
+
+  /**
+   * Get the integer value of this enum value, as defined in the Thrift IDL.
+   */
+  public int getValue() {
+    return value;
+  }
+
+  /**
+   * Find a the enum type by its integer value, as defined in the Thrift IDL.
+   * @return null if the value is not found.
+   */
+  public static TDurability findByValue(int value) { 
+    switch (value) {
+      case 1:
+        return SKIP_WAL;
+      case 2:
+        return ASYNC_WAL;
+      case 3:
+        return SYNC_WAL;
+      case 4:
+        return FSYNC_WAL;
+      default:
+        return null;
+    }
+  }
+}

From e741aab5bca3ff7b21363eac3757cca10cbabbcf Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 26 Jul 2013 22:58:59 +0000
Subject: [PATCH 1081/1540] HBASE-9048 HCM throws NullPointerException under
 load

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1507504 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/HConnectionManager.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 2b1c8328b509..d95209cf098e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1235,7 +1235,9 @@ private void clearCachedLocationForServer(final String server) {
         for (Map<byte[], HRegionLocation> tableLocations :
           cachedRegionLocations.values()) {
           for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
-            if (e.getValue().getHostnamePort().equals(server)) {
+            HRegionLocation value = e.getValue();
+            if (value != null
+                && value.getHostnamePort().equals(server)) {
               tableLocations.remove(e.getKey());
               deletedSomething = true;
             }

From 4bdfaaf45672ac05e7ca52730adc0ed4f31ae18e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 28 Jul 2013 06:04:02 +0000
Subject: [PATCH 1082/1540] HBASE-8698 potential thread creation in
 MetaScanner.metaScan

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1507768 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       |   9 +-
 .../hbase/client/HConnectionManager.java      |  10 +-
 .../apache/hadoop/hbase/client/HTable.java    |   8 +-
 .../hadoop/hbase/client/MetaScanner.java      | 203 ++++++++----------
 .../hadoop/hbase/rest/RegionsResource.java    |   2 +-
 .../apache/hadoop/hbase/util/HBaseFsck.java   |   2 +-
 .../hadoop/hbase/client/TestMetaScanner.java  |   4 +-
 .../hbase/master/TestRestartCluster.java      |   4 +-
 .../TestEndToEndSplitTransaction.java         |   7 +-
 9 files changed, 113 insertions(+), 136 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 2fb99106550e..d5300862f750 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -103,6 +103,7 @@ public class HBaseAdmin implements Abortable, Closeable {
   private boolean aborted;
 
   private static volatile boolean synchronousBalanceSwitchSupported = true;
+  private final boolean cleanupConnectionOnClose; // close the connection in close()
 
   /**
    * Constructor
@@ -119,6 +120,7 @@ public HBaseAdmin(Configuration c)
     this.numRetries = this.conf.getInt("hbase.client.retries.number", 10);
     this.retryLongerMultiplier = this.conf.getInt(
         "hbase.client.retries.longer.multiplier", 10);
+    this.cleanupConnectionOnClose = true;
 
     int tries = 0;
     while ( true ){
@@ -165,6 +167,7 @@ public HBaseAdmin(HConnection connection)
       throws MasterNotRunningException, ZooKeeperConnectionException {
     this.conf = connection.getConfiguration();
     this.connection = connection;
+    this.cleanupConnectionOnClose = false;
 
     this.pause = this.conf.getLong("hbase.client.pause", 1000);
     this.numRetries = this.conf.getInt("hbase.client.retries.number", 10);
@@ -454,7 +457,7 @@ public boolean processRow(Result rowResult) throws IOException {
             return true;
           }
         };
-        MetaScanner.metaScan(conf, visitor, desc.getName());
+        MetaScanner.metaScan(conf, connection, visitor, desc.getName());
         if (actualRegCount.get() != numRegs) {
           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
             throw new RegionOfflineException("Only " + actualRegCount.get() +
@@ -1675,7 +1678,7 @@ public boolean processRow(Result data) throws IOException {
         }
       };
 
-      MetaScanner.metaScan(conf, visitor);
+      MetaScanner.metaScan(conf, connection, visitor, null);
       pair = result.get();
     }
     return pair;
@@ -1802,7 +1805,7 @@ public List<HRegionInfo> getTableRegions(final byte[] tableName)
   }
 
   public void close() throws IOException {
-    if (this.connection != null) {
+    if (cleanupConnectionOnClose && this.connection != null) {
       this.connection.close();
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index d95209cf098e..345426f2f275 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -802,7 +802,7 @@ public boolean processRow(Result row) throws IOException {
           return true;
         }
       };
-      MetaScanner.metaScan(conf, visitor);
+      MetaScanner.metaScan(conf, this, visitor, null);
       return available.get() && (regionCount.get() > 0);
     }
 
@@ -842,8 +842,8 @@ public List<HRegionLocation> locateRegions(final byte[] tableName)
     @Override
     public List<HRegionLocation> locateRegions(final byte[] tableName, final boolean useCache,
         final boolean offlined) throws IOException {
-      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
-        offlined);
+      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, this,
+          tableName, offlined);
       final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
       for (HRegionInfo regionInfo : regions.keySet()) {
         locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
@@ -955,8 +955,8 @@ public boolean processRow(Result result) throws IOException {
       };
       try {
         // pre-fetch certain number of regions info at region cache.
-        MetaScanner.metaScan(conf, visitor, tableName, row,
-            this.prefetchRegionLimit);
+        MetaScanner.metaScan(conf, this, visitor, tableName, row,
+            this.prefetchRegionLimit, HConstants.META_TABLE_NAME);
       } catch (IOException e) {
         LOG.warn("Encountered problems when prefetch META table: ", e);
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 6375ad67a3d9..b5ee9ea62dbf 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -25,7 +25,6 @@
 import java.io.IOException;
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Collections;
@@ -33,10 +32,8 @@
 import java.util.TreeMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -210,9 +207,6 @@ public HTable(Configuration conf, final byte[] tableName, final ExecutorService
    */
   public HTable(final byte[] tableName, final HConnection connection, 
       final ExecutorService pool) throws IOException {
-    if (pool == null || pool.isShutdown()) {
-      throw new IllegalArgumentException("Pool is null or shut down.");
-    }
     if (connection == null || connection.isClosed()) {
       throw new IllegalArgumentException("Connection is null or closed.");
     }
@@ -481,7 +475,7 @@ public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
    * @throws IOException if a remote or network exception occurs
    */
   public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOException {
-    return MetaScanner.allTableRegions(getConfiguration(), getTableName(), false);
+    return MetaScanner.allTableRegions(getConfiguration(), this.connection, getTableName(), false);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 360de34844dc..4c02d6ffda52 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -35,7 +35,6 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
 import org.apache.hadoop.hbase.errorhandling.TimeoutException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
@@ -65,23 +64,23 @@ public class MetaScanner {
   public static void metaScan(Configuration configuration,
       MetaScannerVisitor visitor)
   throws IOException {
-    metaScan(configuration, visitor, null);
+    metaScan(configuration, null, visitor, null);
   }
 
   /**
-   * Scans the meta table and calls a visitor on each RowResult. Uses a table
-   * name to locate meta regions.
+   * Scans the meta table and calls a visitor on each RowResult and uses a empty
+   * start row value as table name.
    *
-   * @param configuration config
-   * @param visitor visitor object
-   * @param userTableName User table name in meta table to start scan at.  Pass
-   * null if not interested in a particular table.
+   * @param configuration conf
+   * @param connection connection to be used internally (null not allowed)
+   * @param visitor A custom visitor
    * @throws IOException e
    */
-  public static void metaScan(Configuration configuration,
+  public static void metaScan(Configuration configuration, HConnection connection,
       MetaScannerVisitor visitor, byte [] userTableName)
   throws IOException {
-    metaScan(configuration, visitor, userTableName, null, Integer.MAX_VALUE);
+    metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
+        HConstants.META_TABLE_NAME);
   }
 
   /**
@@ -103,7 +102,7 @@ public static void metaScan(Configuration configuration,
       MetaScannerVisitor visitor, byte [] userTableName, byte[] row,
       int rowLimit)
   throws IOException {
-    metaScan(configuration, visitor, userTableName, row, rowLimit,
+    metaScan(configuration, null, visitor, userTableName, row, rowLimit,
       HConstants.META_TABLE_NAME);
   }
 
@@ -113,6 +112,7 @@ public static void metaScan(Configuration configuration,
    * <code>rowLimit</code> of rows.
    *
    * @param configuration HBase configuration.
+   * @param connection connection to be used internally (null not allowed)
    * @param visitor Visitor object. Closes the visitor before returning.
    * @param tableName User table name in meta table to start scan at.  Pass
    * null if not interested in a particular table.
@@ -123,42 +123,28 @@ public static void metaScan(Configuration configuration,
    * @param metaTableName Meta table to scan, root or meta.
    * @throws IOException e
    */
-  public static void metaScan(Configuration configuration,
-      final MetaScannerVisitor visitor, final byte[] tableName,
-      final byte[] row, final int rowLimit, final byte[] metaTableName)
-      throws IOException {
-    try {
-      HConnectionManager.execute(new HConnectable<Void>(configuration) {
-        @Override
-        public Void connect(HConnection connection) throws IOException {
-          metaScan(conf, connection, visitor, tableName, row, rowLimit,
-              metaTableName);
-          return null;
-        }
-      });
-    } finally {
-      visitor.close();
-    }
-  }
-
-  private static void metaScan(Configuration configuration, HConnection connection,
+  public static void metaScan(Configuration configuration, HConnection connection,
       MetaScannerVisitor visitor, byte [] tableName, byte[] row,
       int rowLimit, final byte [] metaTableName)
   throws IOException {
-    int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
-
-    // if row is not null, we want to use the startKey of the row's region as
-    // the startRow for the meta scan.
-    byte[] startRow;
-    if (row != null) {
-      // Scan starting at a particular row in a particular table
-      assert tableName != null;
-      byte[] searchRow =
-        HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
-          false);
-      HTable metaTable = null;
-      try {
-        metaTable = new HTable(configuration, HConstants.META_TABLE_NAME);
+    HTable metaTable = null;
+    try {
+      if (connection == null) {
+        metaTable = new HTable(configuration, HConstants.META_TABLE_NAME, null);
+      } else {
+        metaTable = new HTable(HConstants.META_TABLE_NAME, connection, null);
+      }
+      int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
+
+      // if row is not null, we want to use the startKey of the row's region as
+      // the startRow for the meta scan.
+      byte[] startRow;
+      if (row != null) {
+        // Scan starting at a particular row in a particular table
+        assert tableName != null;
+        byte[] searchRow =
+          HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
+            false);
         Result startRowResult = metaTable.getRowOrBefore(searchRow,
             HConstants.CATALOG_FAMILY);
         if (startRowResult == null) {
@@ -176,80 +162,72 @@ private static void metaScan(Configuration configuration, HConnection connection
         byte[] rowBefore = regionInfo.getStartKey();
         startRow = HRegionInfo.createRegionName(tableName, rowBefore,
             HConstants.ZEROES, false);
-      } finally {
-        if (metaTable != null) {
-          metaTable.close();
-        }
+      } else if (tableName == null || tableName.length == 0) {
+        // Full META scan
+        startRow = HConstants.EMPTY_START_ROW;
+      } else {
+        // Scan META for an entire table
+        startRow = HRegionInfo.createRegionName(
+            tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
       }
-    } else if (tableName == null || tableName.length == 0) {
-      // Full META scan
-      startRow = HConstants.EMPTY_START_ROW;
-    } else {
-      // Scan META for an entire table
-      startRow = HRegionInfo.createRegionName(
-          tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
-    }
 
-    // Scan over each meta region
-    ScannerCallable callable;
-    int rows = Math.min(rowLimit, configuration.getInt(
-        HConstants.HBASE_META_SCANNER_CACHING,
-        HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
-    do {
-      final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Scanning " + Bytes.toString(metaTableName) +
-          " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
-          rowUpperLimit + " rows using " + connection.toString());
-      }
-      callable = new ScannerCallable(connection, metaTableName, scan, null);
-      // Open scanner
-      callable.withRetries();
-
-      int processedRows = 0;
-      try {
-        callable.setCaching(rows);
-        done: do {
-          if (processedRows >= rowUpperLimit) {
-            break;
-          }
-          //we have all the rows here
-          Result [] rrs = callable.withRetries();
-          if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
-            break; //exit completely
-          }
-          for (Result rr : rrs) {
+      // Scan over each meta region
+      ScannerCallable callable;
+      int rows = Math.min(rowLimit, configuration.getInt(
+          HConstants.HBASE_META_SCANNER_CACHING,
+          HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
+      do {
+        final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Scanning " + Bytes.toString(metaTableName) +
+            " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
+            rowUpperLimit + " rows using " + metaTable.getConnection().toString());
+        }
+        callable = new ScannerCallable(metaTable.getConnection(), metaTableName, scan, null);
+        // Open scanner
+        callable.withRetries();
+
+        int processedRows = 0;
+        try {
+          callable.setCaching(rows);
+          done: do {
             if (processedRows >= rowUpperLimit) {
-              break done;
+              break;
             }
-            if (!visitor.processRow(rr))
-              break done; //exit completely
-            processedRows++;
-          }
-          //here, we didn't break anywhere. Check if we have more rows
-        } while(true);
-        // Advance the startRow to the end key of the current region
-        startRow = callable.getHRegionInfo().getEndKey();
-      } finally {
-        // Close scanner
-        callable.setClose();
-        callable.withRetries();
+            //we have all the rows here
+            Result [] rrs = callable.withRetries();
+            if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
+              break; //exit completely
+            }
+            for (Result rr : rrs) {
+              if (processedRows >= rowUpperLimit) {
+                break done;
+              }
+              if (!visitor.processRow(rr))
+                break done; //exit completely
+              processedRows++;
+            }
+            //here, we didn't break anywhere. Check if we have more rows
+          } while(true);
+          // Advance the startRow to the end key of the current region
+          startRow = callable.getHRegionInfo().getEndKey();
+        } finally {
+          // Close scanner
+          callable.setClose();
+          callable.withRetries();
+        }
+      } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
+    } finally {
+      visitor.close();
+      if (metaTable != null) {
+        metaTable.close();
       }
-    } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
-  }
-
-  /**
-   * Lists all of the regions currently in META.
-   * @param conf
-   * @return List of all user-space regions.
-   * @throws IOException
-   */
-  public static List<HRegionInfo> listAllRegions(Configuration conf)
-  throws IOException {
-    return listAllRegions(conf, true);
+    }
   }
 
   /**
+   * Used in tests.
+   *
    * Lists all of the regions currently in META.
    * @param conf
    * @param offlined True if we are to include offlined regions, false and we'll
@@ -286,13 +264,14 @@ public boolean processRowInternal(Result result) throws IOException {
   /**
    * Lists all of the table regions currently in META.
    * @param conf
+   * @param connection connection to be used internally (null to create a new connection)
    * @param offlined True if we are to include offlined regions, false and we'll
    * leave out offlined regions from returned list.
    * @return Map of all user-space regions to servers
    * @throws IOException
    */
   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
-      final byte [] tablename, final boolean offlined) throws IOException {
+      HConnection connection, final byte[] tablename, final boolean offlined) throws IOException {
     final NavigableMap<HRegionInfo, ServerName> regions =
       new TreeMap<HRegionInfo, ServerName>();
     MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) {
@@ -321,7 +300,7 @@ public boolean processRowInternal(Result rowResult) throws IOException {
         return true;
       }
     };
-    metaScan(conf, visitor, tablename);
+    metaScan(conf, connection, visitor, tablename);
     return regions;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index 8d4b621f4013..35c0aa6f987a 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -76,7 +76,7 @@ public Response get(final @Context UriInfo uriInfo) {
       String tableName = tableResource.getName();
       TableInfoModel model = new TableInfoModel(tableName);
       Map<HRegionInfo,ServerName> regions = MetaScanner.allTableRegions(
-        servlet.getConfiguration(), Bytes.toBytes(tableName), false);
+        servlet.getConfiguration(), null, Bytes.toBytes(tableName), false);
       for (Map.Entry<HRegionInfo,ServerName> e: regions.entrySet()) {
         HRegionInfo hri = e.getKey();
         ServerName addr = e.getValue();
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 052ae95e774d..6ebe9e758ca6 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2593,7 +2593,7 @@ public boolean processRow(Result result) throws IOException {
     };
 
     // Scan -ROOT- to pick up META regions
-    MetaScanner.metaScan(getConf(), visitor, null, null,
+    MetaScanner.metaScan(getConf(), null, visitor, null, null,
       Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME);
 
     if (!checkMetaOnly) {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
index d03987bfdce8..3d7912591193 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
@@ -86,7 +86,7 @@ public void testMetaScanner() throws Exception {
     doReturn(true).when(visitor).processRow((Result)anyObject());
 
     // Scanning the entire table should give us three rows
-    MetaScanner.metaScan(conf, visitor, TABLENAME);
+    MetaScanner.metaScan(conf, null, visitor, TABLENAME);
     verify(visitor, times(3)).processRow((Result)anyObject());
 
     // Scanning the table with a specified empty start row should also
@@ -192,7 +192,7 @@ public void run() {
          while(!isStopped()) {
            try {
             NavigableMap<HRegionInfo, ServerName> regions =
-                MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), TABLENAME, false);
+                MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), null, TABLENAME, false);
 
             LOG.info("-------");
             byte[] lastEndKey = HConstants.EMPTY_START_ROW;
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index 3b8b6c6a8cb1..7703be3d5beb 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -102,7 +102,7 @@ public void testClusterRestart() throws Exception {
     }
 
     List<HRegionInfo> allRegions =
-      MetaScanner.listAllRegions(UTIL.getConfiguration());
+      MetaScanner.listAllRegions(UTIL.getConfiguration(), true);
     assertEquals(3, allRegions.size());
 
     LOG.info("\n\nShutting down cluster");
@@ -118,7 +118,7 @@ public void testClusterRestart() throws Exception {
     // Otherwise we're reusing an HConnection that has gone stale because
     // the shutdown of the cluster also called shut of the connection.
     allRegions = MetaScanner.
-      listAllRegions(new Configuration(UTIL.getConfiguration()));
+      listAllRegions(new Configuration(UTIL.getConfiguration()), true);
     assertEquals(3, allRegions.size());
 
     LOG.info("\n\nWaiting for tables to be available");
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 4e62a1fa6bf8..bf0d0a6a9f2d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -214,7 +214,8 @@ public void run() {
       try {
         Random random = new Random();
         for (int i=0; i< 5; i++) {
-          NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName, false);
+          NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
+              tableName, false);
           if (regions.size() == 0) {
             continue;
           }
@@ -286,8 +287,8 @@ static class RegionChecker extends Chore {
     void verifyRegionsUsingMetaScanner() throws Exception {
 
       //MetaScanner.allTableRegions()
-      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
-          false);
+      NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
+          tableName, false);
       verifyTableRegions(regions.keySet());
 
       //MetaScanner.listAllRegions()

From b6cfdf8fff284d5e6751e34e808a480bae5f88e6 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sun, 28 Jul 2013 12:17:39 +0000
Subject: [PATCH 1083/1540] HBASE-9060 ExportSnapshot job fails if target path
 contains percentage character (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1507792 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/snapshot/ExportSnapshot.java  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 602597c8fc50..ab8656cc60d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -253,7 +253,7 @@ private boolean copyData(final Context context,
         final Path outputPath, final FSDataOutputStream out,
         final long inputFileSize) {
       final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
-                                   " (%.3f%%) from " + inputPath + " to " + outputPath;
+                                   " (%.3f%%)";
 
       try {
         byte[] buffer = new byte[BUFFER_SIZE];
@@ -270,7 +270,8 @@ private boolean copyData(final Context context,
             context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
             context.setStatus(String.format(statusMessage,
                               StringUtils.humanReadableInt(totalBytesWritten),
-                              reportBytes/(float)inputFileSize));
+                              totalBytesWritten/(float)inputFileSize) +
+                              " from " + inputPath + " to " + outputPath);
             reportBytes = 0;
           }
         }
@@ -278,7 +279,8 @@ private boolean copyData(final Context context,
         context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
         context.setStatus(String.format(statusMessage,
                           StringUtils.humanReadableInt(totalBytesWritten),
-                          reportBytes/(float)inputFileSize));
+                          totalBytesWritten/(float)inputFileSize) +
+                          " from " + inputPath + " to " + outputPath);
 
         // Verify that the written size match
         if (totalBytesWritten != inputFileSize) {

From 75af0e79a06b0f53f67af180ae104d6efc31628a Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 29 Jul 2013 00:53:17 +0000
Subject: [PATCH 1084/1540] HBASE-9050 HBaseClient#call could hang

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1507878 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index c5b76c95e375..bf0a6efba67e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -977,8 +977,11 @@ public Writable call(Writable param, InetSocketAddress addr,
     //noinspection SynchronizationOnLocalVariableOrMethodParameter
     synchronized (call) {
       while (!call.done) {
+        if (connection.shouldCloseConnection.get()) {
+          throw new IOException("Unexpected closed connection");
+        }
         try {
-          call.wait();                           // wait for the result
+          call.wait(1000);                       // wait for the result
         } catch (InterruptedException ignored) {
           // save the fact that we were interrupted
           interrupted = true;

From b363d035f77d05a1ee903cf4a2a6b5b5299d8a15 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 29 Jul 2013 17:25:01 +0000
Subject: [PATCH 1085/1540] HBASE-6826 [WINDOWS] TestFromClientSide failures
 (Enis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1508143 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestFromClientSide.java      | 23 +++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index c347096b6a14..f21583a1b8d0 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -90,6 +90,7 @@
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.junit.After;
@@ -4180,7 +4181,7 @@ public void testAppend() throws Exception {
     assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0])));
     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
   }
- 
+
   @Test
   public void testIncrementWithDeletes() throws Exception {
     LOG.info("Starting testIncrementWithDeletes");
@@ -4351,6 +4352,10 @@ public void run() {
       Threads.sleep(1);
     }
 
+    //ensure that ThreadPoolExecutor knows that threads are finished.
+    while (pool.getCompletedTaskCount() < 2) {
+      Threads.sleep(1);
+    }
     // Now let's simulate adding a RS meaning that we'll go up to three
     // concurrent threads. The pool should not grow larger than three.
     pool.submit(tasks.get(2));
@@ -4375,14 +4380,15 @@ public void testClientPoolRoundRobin() throws IOException {
     HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY },
         conf, Integer.MAX_VALUE);
     table.setAutoFlush(true);
-    Put put = new Put(ROW);
-    put.add(FAMILY, QUALIFIER, VALUE);
 
+    final long ts = EnvironmentEdgeManager.currentTimeMillis();
     Get get = new Get(ROW);
     get.addColumn(FAMILY, QUALIFIER);
     get.setMaxVersions();
 
     for (int versions = 1; versions <= numVersions; versions++) {
+      Put put = new Put(ROW);
+      put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
       table.put(put);
 
       Result result = table.get(get);
@@ -4412,14 +4418,15 @@ public void testClientPoolThreadLocal() throws IOException {
     final HTable table = TEST_UTIL.createTable(tableName,
         new byte[][] { FAMILY }, conf);
     table.setAutoFlush(true);
-    final Put put = new Put(ROW);
-    put.add(FAMILY, QUALIFIER, VALUE);
 
+    final long ts = EnvironmentEdgeManager.currentTimeMillis();
     final Get get = new Get(ROW);
     get.addColumn(FAMILY, QUALIFIER);
     get.setMaxVersions();
 
     for (int versions = 1; versions <= numVersions; versions++) {
+      Put put = new Put(ROW);
+      put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
       table.put(put);
 
       Result result = table.get(get);
@@ -4444,6 +4451,8 @@ public void testClientPoolThreadLocal() throws IOException {
         @Override
         public Void call() {
           try {
+            Put put = new Put(ROW);
+            put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
             table.put(put);
 
             Result result = table.get(get);
@@ -4572,7 +4581,7 @@ public void testScanMetrics() throws Exception {
     assertEquals("Did not access all the regions in the table", numOfRegions,
         scanMetrics.countOfRegions.getCurrentIntervalValue());
 
-    // now, test that the metrics are still collected even if you don't call close, but do 
+    // now, test that the metrics are still collected even if you don't call close, but do
     // run past the end of all the records
     Scan scanWithoutClose = new Scan();
     scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
@@ -4775,7 +4784,7 @@ public void testNonCachedGetRegionLocation() throws Exception {
     assertNotNull(addrAfter);
     assertTrue(addrAfter.getPort() != addrCache.getPort());
     assertEquals(addrAfter.getPort(), addrNoCache.getPort());
-  }  
+  }
 
   @Test
   /**

From f0a2d8c0b58b46b7a12db3285f106fd44fe86487 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 29 Jul 2013 18:53:49 +0000
Subject: [PATCH 1086/1540] HBASE-9032 Result.getBytes() returns null if backed
 by KeyValue array

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1508176 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/Result.java    | 17 ++++++++++++++
 .../hadoop/hbase/client/TestResult.java       | 23 ++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java
index bf8bda31ec5b..05f528c4787b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.WritableWithSize;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Writable;
 
 /**
@@ -401,6 +402,22 @@ public NavigableMap<byte[], byte[]> getFamilyMap(byte [] family) {
    * @return pointer to raw binary of Result
    */
   public ImmutableBytesWritable getBytes() {
+    if (this.bytes == null && this.kvs != null) {
+      int totalLen = 0;
+      for(KeyValue kv : kvs) {
+        totalLen += kv.getLength() + Bytes.SIZEOF_INT;
+      }
+      DataOutputBuffer out = new DataOutputBuffer(totalLen);
+      try {
+        for(KeyValue kv : kvs) {
+          kv.write(out);
+        }
+        out.close();
+      } catch (IOException e) {
+        throw new RuntimeException("IOException in Result.getBytes()", e);
+      }
+      this.bytes = new ImmutableBytesWritable(out.getData(), 0, out.getLength());
+    }
     return this.bytes;
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
index f9e29c267e37..685ff2e2308c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
@@ -21,8 +21,10 @@
 package org.apache.hadoop.hbase.client;
 
 import junit.framework.TestCase;
+
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
 
@@ -122,8 +124,27 @@ public void testCompareResults() throws Exception {
     }
   }
 
+  /**
+   * Verify that Result.getBytes(...) behaves correctly.
+   */
+  public void testResultGetBytes() throws Exception {
+    byte [] value1 = Bytes.toBytes("value1");
+    byte [] qual = Bytes.toBytes("qual");
+
+    KeyValue kv1 = new KeyValue(row, family, qual, value);
+    KeyValue kv2 = new KeyValue(row, family, qual, value1);
+
+    Result r1 = new Result(new KeyValue[] {kv1, kv2});
+
+    ImmutableBytesWritable bytes = r1.getBytes();
+    assertNotNull(bytes);
+
+    Result r2 = new Result(bytes);
+    // no exception thrown
+    Result.compareResults(r1, r2);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
 }
-

From e1bebe6af1570ba5572e02985e5f597f66708bf8 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Mon, 29 Jul 2013 23:15:56 +0000
Subject: [PATCH 1087/1540] HBASE-7826  Improve Hbase Thrift v1 to return
 results in sorted order

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1508231 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/thrift/ThriftServerRunner.java      |  58 +-
 .../hadoop/hbase/thrift/ThriftUtilities.java  |  44 +-
 .../hbase/thrift/generated/TColumn.java       | 497 ++++++++++++++++++
 .../hbase/thrift/generated/TRowResult.java    | 231 ++++++--
 .../hadoop/hbase/thrift/generated/TScan.java  | 137 ++++-
 .../apache/hadoop/hbase/thrift/Hbase.thrift   |  14 +-
 .../hadoop/hbase/thrift/TestThriftServer.java |  41 ++
 7 files changed, 937 insertions(+), 85 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index cf170e4ab75d..de2a81918e97 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -381,6 +381,25 @@ private InetAddress getBindAddress(Configuration conf)
     return InetAddress.getByName(bindAddressStr);
   }
 
+  protected static class ResultScannerWrapper {
+
+    private final ResultScanner scanner; 
+    private final boolean sortColumns; 
+    public ResultScannerWrapper(ResultScanner resultScanner,
+                                boolean sortResultColumns) {
+      scanner = resultScanner; 
+      sortColumns = sortResultColumns;  
+   } 
+
+    public ResultScanner getScanner() {
+      return scanner; 
+    }
+
+    public boolean isColumnSorted() {
+      return sortColumns; 
+    }
+  }
+
   /**
    * The HBaseHandler is a glue object that connects Thrift RPC calls to the
    * HBase client API primarily defined in the HBaseAdmin and HTable objects.
@@ -392,7 +411,7 @@ public static class HBaseHandler implements Hbase.Iface {
 
     // nextScannerId and scannerMap are used to manage scanner state
     protected int nextScannerId = 0;
-    protected HashMap<Integer, ResultScanner> scannerMap = null;
+    protected HashMap<Integer, ResultScannerWrapper> scannerMap = null;
     private ThriftMetrics metrics = null;
 
     private static ThreadLocal<Map<String, HTable>> threadLocalTables =
@@ -452,9 +471,10 @@ public HTable getTable(final ByteBuffer tableName) throws IOException {
      * @param scanner
      * @return integer scanner id
      */
-    protected synchronized int addScanner(ResultScanner scanner) {
+    protected synchronized int addScanner(ResultScanner scanner,boolean sortColumns) {
       int id = nextScannerId++;
-      scannerMap.put(id, scanner);
+      ResultScannerWrapper resultScannerWrapper = new ResultScannerWrapper(scanner, sortColumns);
+      scannerMap.put(id, resultScannerWrapper);
       return id;
     }
 
@@ -464,7 +484,7 @@ protected synchronized int addScanner(ResultScanner scanner) {
      * @param id
      * @return a Scanner, or null if ID was invalid.
      */
-    protected synchronized ResultScanner getScanner(int id) {
+    protected synchronized ResultScannerWrapper getScanner(int id) {
       return scannerMap.get(id);
     }
 
@@ -475,7 +495,7 @@ protected synchronized ResultScanner getScanner(int id) {
      * @param id
      * @return a Scanner, or null if ID was invalid.
      */
-    protected synchronized ResultScanner removeScanner(int id) {
+    protected synchronized ResultScannerWrapper removeScanner(int id) {
       return scannerMap.remove(id);
     }
 
@@ -491,7 +511,7 @@ protected HBaseHandler()
     protected HBaseHandler(final Configuration c) throws IOException {
       this.conf = c;
       admin = new HBaseAdmin(conf);
-      scannerMap = new HashMap<Integer, ResultScanner>();
+      scannerMap = new HashMap<Integer, ResultScannerWrapper>();
       this.coalescer = new IncrementCoalescer(this);
     }
 
@@ -1086,13 +1106,13 @@ protected long atomicIncrement(ByteBuffer tableName, ByteBuffer row,
 
     public void scannerClose(int id) throws IOError, IllegalArgument {
       LOG.debug("scannerClose: id=" + id);
-      ResultScanner scanner = getScanner(id);
-      if (scanner == null) {
+      ResultScannerWrapper resultScannerWrapper = getScanner(id);
+      if (resultScannerWrapper == null) {
         String message = "scanner ID is invalid";
         LOG.warn(message);
         throw new IllegalArgument("scanner ID is invalid");
       }
-      scanner.close();
+      resultScannerWrapper.getScanner().close();
       removeScanner(id);
     }
 
@@ -1100,8 +1120,8 @@ public void scannerClose(int id) throws IOError, IllegalArgument {
     public List<TRowResult> scannerGetList(int id,int nbRows)
         throws IllegalArgument, IOError {
       LOG.debug("scannerGetList: id=" + id);
-      ResultScanner scanner = getScanner(id);
-      if (null == scanner) {
+      ResultScannerWrapper resultScannerWrapper = getScanner(id);
+      if (null == resultScannerWrapper) {
         String message = "scanner ID is invalid";
         LOG.warn(message);
         throw new IllegalArgument("scanner ID is invalid");
@@ -1109,7 +1129,7 @@ public List<TRowResult> scannerGetList(int id,int nbRows)
 
       Result [] results = null;
       try {
-        results = scanner.next(nbRows);
+        results = resultScannerWrapper.getScanner().next(nbRows);
         if (null == results) {
           return new ArrayList<TRowResult>();
         }
@@ -1117,7 +1137,7 @@ public List<TRowResult> scannerGetList(int id,int nbRows)
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
       }
-      return ThriftUtilities.rowResultFromHBase(results);
+      return ThriftUtilities.rowResultFromHBase(results, resultScannerWrapper.isColumnSorted());
     }
 
     @Override
@@ -1159,7 +1179,7 @@ public int scannerOpenWithScan(ByteBuffer tableName, TScan tScan,
           scan.setFilter(
               parseFilter.parseFilterString(tScan.getFilterString()));
         }
-        return addScanner(table.getScanner(scan));
+        return addScanner(table.getScanner(scan), tScan.sortColumns);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
@@ -1184,7 +1204,7 @@ public int scannerOpen(ByteBuffer tableName, ByteBuffer startRow,
             }
           }
         }
-        return addScanner(table.getScanner(scan));
+        return addScanner(table.getScanner(scan), false);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
@@ -1210,7 +1230,7 @@ public int scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow,
             }
           }
         }
-        return addScanner(table.getScanner(scan));
+        return addScanner(table.getScanner(scan), false);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
@@ -1240,7 +1260,7 @@ public int scannerOpenWithPrefix(ByteBuffer tableName,
             }
           }
         }
-        return addScanner(table.getScanner(scan));
+        return addScanner(table.getScanner(scan), false);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
@@ -1266,7 +1286,7 @@ public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow,
             }
           }
         }
-        return addScanner(table.getScanner(scan));
+        return addScanner(table.getScanner(scan), false);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
@@ -1294,7 +1314,7 @@ public int scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow,
           }
         }
         scan.setTimeRange(Long.MIN_VALUE, timestamp);
-        return addScanner(table.getScanner(scan));
+        return addScanner(table.getScanner(scan), false);
       } catch (IOException e) {
         LOG.warn(e.getMessage(), e);
         throw new IOError(e.getMessage());
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
index 8313aa8f875e..f853facd47ca 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.hbase.thrift.generated.IllegalArgument;
 import org.apache.hadoop.hbase.thrift.generated.TCell;
 import org.apache.hadoop.hbase.thrift.generated.TIncrement;
+import org.apache.hadoop.hbase.thrift.generated.TColumn;
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -130,9 +131,15 @@ static public List<TCell> cellFromHBase(KeyValue[] in) {
    *
    * @param in
    *          Hbase RowResult object
+   * @param sortColumns
+   *          This boolean dictates if row data is returned in a sorted order
+   *          sortColumns = True will set TRowResult's sortedColumns member
+   *                        which is an ArrayList of TColumn struct
+   *          sortColumns = False will set TRowResult's columns member which is
+   *                        a map of columnName and TCell struct
    * @return Thrift TRowResult array
    */
-  static public List<TRowResult> rowResultFromHBase(Result[] in) {
+  static public List<TRowResult> rowResultFromHBase(Result[] in, boolean sortColumns) {
     List<TRowResult> results = new ArrayList<TRowResult>();
     for ( Result result_ : in) {
         if(result_ == null || result_.isEmpty()) {
@@ -140,18 +147,41 @@ static public List<TRowResult> rowResultFromHBase(Result[] in) {
         }
         TRowResult result = new TRowResult();
         result.row = ByteBuffer.wrap(result_.getRow());
-        result.columns = new TreeMap<ByteBuffer, TCell>();
-        for(KeyValue kv : result_.raw()) {
-          result.columns.put(
-              ByteBuffer.wrap(KeyValue.makeColumn(kv.getFamily(),
-                  kv.getQualifier())),
-              new TCell(ByteBuffer.wrap(kv.getValue()), kv.getTimestamp()));
+        if (sortColumns) {
+          result.sortedColumns = new ArrayList<TColumn>();
+          for (KeyValue kv : result_.raw()) {
+            result.sortedColumns.add(new TColumn(
+                ByteBuffer.wrap(KeyValue.makeColumn(kv.getFamily(),
+                    kv.getQualifier())),
+                new TCell(ByteBuffer.wrap(kv.getValue()), kv.getTimestamp())));
+          }
+        } else {
+          result.columns = new TreeMap<ByteBuffer, TCell>();
+          for( KeyValue kv : result_.raw()) {
+            result.columns.put(
+                ByteBuffer.wrap(KeyValue.makeColumn(kv.getFamily(),
+                    kv.getQualifier())),
+                new TCell(ByteBuffer.wrap(kv.getValue()), kv.getTimestamp()));
+          }
         }
       results.add(result);
     }
     return results;
   }
 
+  /**
+   * This utility method creates a list of Thrift TRowResult "struct" based on
+   * an array of Hbase RowResult objects. The empty list is returned if the input is
+   * null.
+   *
+   * @param in
+   *          Array of Hbase RowResult objects
+   * @return Thrift TRowResult array
+   */
+  static public List<TRowResult> rowResultFromHBase(Result[] in) {
+    return rowResultFromHBase(in, false);
+  }
+
   static public List<TRowResult> rowResultFromHBase(Result in) {
     Result [] result = { in };
     return rowResultFromHBase(result);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java
new file mode 100644
index 000000000000..cb978e82508c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java
@@ -0,0 +1,497 @@
+/**
+ * Autogenerated by Thrift Compiler (0.8.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hbase.thrift.generated;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Holds column name and the cell.
+ */
+public class TColumn implements org.apache.thrift.TBase<TColumn, TColumn._Fields>, java.io.Serializable, Cloneable {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumn");
+
+  private static final org.apache.thrift.protocol.TField COLUMN_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("columnName", org.apache.thrift.protocol.TType.STRING, (short)1);
+  private static final org.apache.thrift.protocol.TField CELL_FIELD_DESC = new org.apache.thrift.protocol.TField("cell", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+  static {
+    schemes.put(StandardScheme.class, new TColumnStandardSchemeFactory());
+    schemes.put(TupleScheme.class, new TColumnTupleSchemeFactory());
+  }
+
+  public ByteBuffer columnName; // required
+  public TCell cell; // required
+
+  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+    COLUMN_NAME((short)1, "columnName"),
+    CELL((short)2, "cell");
+
+    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+    static {
+      for (_Fields field : EnumSet.allOf(_Fields.class)) {
+        byName.put(field.getFieldName(), field);
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, or null if its not found.
+     */
+    public static _Fields findByThriftId(int fieldId) {
+      switch(fieldId) {
+        case 1: // COLUMN_NAME
+          return COLUMN_NAME;
+        case 2: // CELL
+          return CELL;
+        default:
+          return null;
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, throwing an exception
+     * if it is not found.
+     */
+    public static _Fields findByThriftIdOrThrow(int fieldId) {
+      _Fields fields = findByThriftId(fieldId);
+      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+      return fields;
+    }
+
+    /**
+     * Find the _Fields constant that matches name, or null if its not found.
+     */
+    public static _Fields findByName(String name) {
+      return byName.get(name);
+    }
+
+    private final short _thriftId;
+    private final String _fieldName;
+
+    _Fields(short thriftId, String fieldName) {
+      _thriftId = thriftId;
+      _fieldName = fieldName;
+    }
+
+    public short getThriftFieldId() {
+      return _thriftId;
+    }
+
+    public String getFieldName() {
+      return _fieldName;
+    }
+  }
+
+  // isset id assignments
+  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+  static {
+    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+    tmpMap.put(_Fields.COLUMN_NAME, new org.apache.thrift.meta_data.FieldMetaData("columnName", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "Text")));
+    tmpMap.put(_Fields.CELL, new org.apache.thrift.meta_data.FieldMetaData("cell", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class)));
+    metaDataMap = Collections.unmodifiableMap(tmpMap);
+    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumn.class, metaDataMap);
+  }
+
+  public TColumn() {
+  }
+
+  public TColumn(
+    ByteBuffer columnName,
+    TCell cell)
+  {
+    this();
+    this.columnName = columnName;
+    this.cell = cell;
+  }
+
+  /**
+   * Performs a deep copy on <i>other</i>.
+   */
+  public TColumn(TColumn other) {
+    if (other.isSetColumnName()) {
+      this.columnName = other.columnName;
+    }
+    if (other.isSetCell()) {
+      this.cell = new TCell(other.cell);
+    }
+  }
+
+  public TColumn deepCopy() {
+    return new TColumn(this);
+  }
+
+  @Override
+  public void clear() {
+    this.columnName = null;
+    this.cell = null;
+  }
+
+  public byte[] getColumnName() {
+    setColumnName(org.apache.thrift.TBaseHelper.rightSize(columnName));
+    return columnName == null ? null : columnName.array();
+  }
+
+  public ByteBuffer bufferForColumnName() {
+    return columnName;
+  }
+
+  public TColumn setColumnName(byte[] columnName) {
+    setColumnName(columnName == null ? (ByteBuffer)null : ByteBuffer.wrap(columnName));
+    return this;
+  }
+
+  public TColumn setColumnName(ByteBuffer columnName) {
+    this.columnName = columnName;
+    return this;
+  }
+
+  public void unsetColumnName() {
+    this.columnName = null;
+  }
+
+  /** Returns true if field columnName is set (has been assigned a value) and false otherwise */
+  public boolean isSetColumnName() {
+    return this.columnName != null;
+  }
+
+  public void setColumnNameIsSet(boolean value) {
+    if (!value) {
+      this.columnName = null;
+    }
+  }
+
+  public TCell getCell() {
+    return this.cell;
+  }
+
+  public TColumn setCell(TCell cell) {
+    this.cell = cell;
+    return this;
+  }
+
+  public void unsetCell() {
+    this.cell = null;
+  }
+
+  /** Returns true if field cell is set (has been assigned a value) and false otherwise */
+  public boolean isSetCell() {
+    return this.cell != null;
+  }
+
+  public void setCellIsSet(boolean value) {
+    if (!value) {
+      this.cell = null;
+    }
+  }
+
+  public void setFieldValue(_Fields field, Object value) {
+    switch (field) {
+    case COLUMN_NAME:
+      if (value == null) {
+        unsetColumnName();
+      } else {
+        setColumnName((ByteBuffer)value);
+      }
+      break;
+
+    case CELL:
+      if (value == null) {
+        unsetCell();
+      } else {
+        setCell((TCell)value);
+      }
+      break;
+
+    }
+  }
+
+  public Object getFieldValue(_Fields field) {
+    switch (field) {
+    case COLUMN_NAME:
+      return getColumnName();
+
+    case CELL:
+      return getCell();
+
+    }
+    throw new IllegalStateException();
+  }
+
+  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+  public boolean isSet(_Fields field) {
+    if (field == null) {
+      throw new IllegalArgumentException();
+    }
+
+    switch (field) {
+    case COLUMN_NAME:
+      return isSetColumnName();
+    case CELL:
+      return isSetCell();
+    }
+    throw new IllegalStateException();
+  }
+
+  @Override
+  public boolean equals(Object that) {
+    if (that == null)
+      return false;
+    if (that instanceof TColumn)
+      return this.equals((TColumn)that);
+    return false;
+  }
+
+  public boolean equals(TColumn that) {
+    if (that == null)
+      return false;
+
+    boolean this_present_columnName = true && this.isSetColumnName();
+    boolean that_present_columnName = true && that.isSetColumnName();
+    if (this_present_columnName || that_present_columnName) {
+      if (!(this_present_columnName && that_present_columnName))
+        return false;
+      if (!this.columnName.equals(that.columnName))
+        return false;
+    }
+
+    boolean this_present_cell = true && this.isSetCell();
+    boolean that_present_cell = true && that.isSetCell();
+    if (this_present_cell || that_present_cell) {
+      if (!(this_present_cell && that_present_cell))
+        return false;
+      if (!this.cell.equals(that.cell))
+        return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return 0;
+  }
+
+  public int compareTo(TColumn other) {
+    if (!getClass().equals(other.getClass())) {
+      return getClass().getName().compareTo(other.getClass().getName());
+    }
+
+    int lastComparison = 0;
+    TColumn typedOther = (TColumn)other;
+
+    lastComparison = Boolean.valueOf(isSetColumnName()).compareTo(typedOther.isSetColumnName());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetColumnName()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnName, typedOther.columnName);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetCell()).compareTo(typedOther.isSetCell());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetCell()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cell, typedOther.cell);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    return 0;
+  }
+
+  public _Fields fieldForId(int fieldId) {
+    return _Fields.findByThriftId(fieldId);
+  }
+
+  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+  }
+
+  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder("TColumn(");
+    boolean first = true;
+
+    sb.append("columnName:");
+    if (this.columnName == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.columnName);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("cell:");
+    if (this.cell == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.cell);
+    }
+    first = false;
+    sb.append(")");
+    return sb.toString();
+  }
+
+  public void validate() throws org.apache.thrift.TException {
+    // check for required fields
+  }
+
+  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+    try {
+      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+    try {
+      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private static class TColumnStandardSchemeFactory implements SchemeFactory {
+    public TColumnStandardScheme getScheme() {
+      return new TColumnStandardScheme();
+    }
+  }
+
+  private static class TColumnStandardScheme extends StandardScheme<TColumn> {
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot, TColumn struct) throws org.apache.thrift.TException {
+      org.apache.thrift.protocol.TField schemeField;
+      iprot.readStructBegin();
+      while (true)
+      {
+        schemeField = iprot.readFieldBegin();
+        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+          break;
+        }
+        switch (schemeField.id) {
+          case 1: // COLUMN_NAME
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.columnName = iprot.readBinary();
+              struct.setColumnNameIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 2: // CELL
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+              struct.cell = new TCell();
+              struct.cell.read(iprot);
+              struct.setCellIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          default:
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+        }
+        iprot.readFieldEnd();
+      }
+      iprot.readStructEnd();
+
+      // check for required fields of primitive type, which can't be checked in the validate method
+      struct.validate();
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot, TColumn struct) throws org.apache.thrift.TException {
+      struct.validate();
+
+      oprot.writeStructBegin(STRUCT_DESC);
+      if (struct.columnName != null) {
+        oprot.writeFieldBegin(COLUMN_NAME_FIELD_DESC);
+        oprot.writeBinary(struct.columnName);
+        oprot.writeFieldEnd();
+      }
+      if (struct.cell != null) {
+        oprot.writeFieldBegin(CELL_FIELD_DESC);
+        struct.cell.write(oprot);
+        oprot.writeFieldEnd();
+      }
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+
+  }
+
+  private static class TColumnTupleSchemeFactory implements SchemeFactory {
+    public TColumnTupleScheme getScheme() {
+      return new TColumnTupleScheme();
+    }
+  }
+
+  private static class TColumnTupleScheme extends TupleScheme<TColumn> {
+
+    @Override
+    public void write(org.apache.thrift.protocol.TProtocol prot, TColumn struct) throws org.apache.thrift.TException {
+      TTupleProtocol oprot = (TTupleProtocol) prot;
+      BitSet optionals = new BitSet();
+      if (struct.isSetColumnName()) {
+        optionals.set(0);
+      }
+      if (struct.isSetCell()) {
+        optionals.set(1);
+      }
+      oprot.writeBitSet(optionals, 2);
+      if (struct.isSetColumnName()) {
+        oprot.writeBinary(struct.columnName);
+      }
+      if (struct.isSetCell()) {
+        struct.cell.write(oprot);
+      }
+    }
+
+    @Override
+    public void read(org.apache.thrift.protocol.TProtocol prot, TColumn struct) throws org.apache.thrift.TException {
+      TTupleProtocol iprot = (TTupleProtocol) prot;
+      BitSet incoming = iprot.readBitSet(2);
+      if (incoming.get(0)) {
+        struct.columnName = iprot.readBinary();
+        struct.setColumnNameIsSet(true);
+      }
+      if (incoming.get(1)) {
+        struct.cell = new TCell();
+        struct.cell.read(iprot);
+        struct.setCellIsSet(true);
+      }
+    }
+  }
+
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
index 44772899c4f3..763ab099f921 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
@@ -35,6 +35,7 @@ public class TRowResult implements org.apache.thrift.TBase<TRowResult, TRowResul
 
   private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1);
   private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.MAP, (short)2);
+  private static final org.apache.thrift.protocol.TField SORTED_COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("sortedColumns", org.apache.thrift.protocol.TType.LIST, (short)3);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -43,12 +44,14 @@ public class TRowResult implements org.apache.thrift.TBase<TRowResult, TRowResul
   }
 
   public ByteBuffer row; // required
-  public Map<ByteBuffer,TCell> columns; // required
+  public Map<ByteBuffer,TCell> columns; // optional
+  public List<TColumn> sortedColumns; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     ROW((short)1, "row"),
-    COLUMNS((short)2, "columns");
+    COLUMNS((short)2, "columns"),
+    SORTED_COLUMNS((short)3, "sortedColumns");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -67,6 +70,8 @@ public static _Fields findByThriftId(int fieldId) {
           return ROW;
         case 2: // COLUMNS
           return COLUMNS;
+        case 3: // SORTED_COLUMNS
+          return SORTED_COLUMNS;
         default:
           return null;
       }
@@ -107,15 +112,19 @@ public String getFieldName() {
   }
 
   // isset id assignments
+  private _Fields optionals[] = {_Fields.COLUMNS,_Fields.SORTED_COLUMNS};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
     tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "Text")));
-    tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+    tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
             new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , "Text"), 
             new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class))));
+    tmpMap.put(_Fields.SORTED_COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("sortedColumns", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumn.class))));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRowResult.class, metaDataMap);
   }
@@ -124,12 +133,10 @@ public TRowResult() {
   }
 
   public TRowResult(
-    ByteBuffer row,
-    Map<ByteBuffer,TCell> columns)
+    ByteBuffer row)
   {
     this();
     this.row = row;
-    this.columns = columns;
   }
 
   /**
@@ -154,6 +161,13 @@ public TRowResult(TRowResult other) {
       }
       this.columns = __this__columns;
     }
+    if (other.isSetSortedColumns()) {
+      List<TColumn> __this__sortedColumns = new ArrayList<TColumn>();
+      for (TColumn other_element : other.sortedColumns) {
+        __this__sortedColumns.add(new TColumn(other_element));
+      }
+      this.sortedColumns = __this__sortedColumns;
+    }
   }
 
   public TRowResult deepCopy() {
@@ -164,6 +178,7 @@ public TRowResult deepCopy() {
   public void clear() {
     this.row = null;
     this.columns = null;
+    this.sortedColumns = null;
   }
 
   public byte[] getRow() {
@@ -235,6 +250,45 @@ public void setColumnsIsSet(boolean value) {
     }
   }
 
+  public int getSortedColumnsSize() {
+    return (this.sortedColumns == null) ? 0 : this.sortedColumns.size();
+  }
+
+  public java.util.Iterator<TColumn> getSortedColumnsIterator() {
+    return (this.sortedColumns == null) ? null : this.sortedColumns.iterator();
+  }
+
+  public void addToSortedColumns(TColumn elem) {
+    if (this.sortedColumns == null) {
+      this.sortedColumns = new ArrayList<TColumn>();
+    }
+    this.sortedColumns.add(elem);
+  }
+
+  public List<TColumn> getSortedColumns() {
+    return this.sortedColumns;
+  }
+
+  public TRowResult setSortedColumns(List<TColumn> sortedColumns) {
+    this.sortedColumns = sortedColumns;
+    return this;
+  }
+
+  public void unsetSortedColumns() {
+    this.sortedColumns = null;
+  }
+
+  /** Returns true if field sortedColumns is set (has been assigned a value) and false otherwise */
+  public boolean isSetSortedColumns() {
+    return this.sortedColumns != null;
+  }
+
+  public void setSortedColumnsIsSet(boolean value) {
+    if (!value) {
+      this.sortedColumns = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case ROW:
@@ -253,6 +307,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case SORTED_COLUMNS:
+      if (value == null) {
+        unsetSortedColumns();
+      } else {
+        setSortedColumns((List<TColumn>)value);
+      }
+      break;
+
     }
   }
 
@@ -264,6 +326,9 @@ public Object getFieldValue(_Fields field) {
     case COLUMNS:
       return getColumns();
 
+    case SORTED_COLUMNS:
+      return getSortedColumns();
+
     }
     throw new IllegalStateException();
   }
@@ -279,6 +344,8 @@ public boolean isSet(_Fields field) {
       return isSetRow();
     case COLUMNS:
       return isSetColumns();
+    case SORTED_COLUMNS:
+      return isSetSortedColumns();
     }
     throw new IllegalStateException();
   }
@@ -314,6 +381,15 @@ public boolean equals(TRowResult that) {
         return false;
     }
 
+    boolean this_present_sortedColumns = true && this.isSetSortedColumns();
+    boolean that_present_sortedColumns = true && that.isSetSortedColumns();
+    if (this_present_sortedColumns || that_present_sortedColumns) {
+      if (!(this_present_sortedColumns && that_present_sortedColumns))
+        return false;
+      if (!this.sortedColumns.equals(that.sortedColumns))
+        return false;
+    }
+
     return true;
   }
 
@@ -350,6 +426,16 @@ public int compareTo(TRowResult other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetSortedColumns()).compareTo(typedOther.isSetSortedColumns());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetSortedColumns()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sortedColumns, typedOther.sortedColumns);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -377,14 +463,26 @@ public String toString() {
       sb.append(this.row);
     }
     first = false;
-    if (!first) sb.append(", ");
-    sb.append("columns:");
-    if (this.columns == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.columns);
+    if (isSetColumns()) {
+      if (!first) sb.append(", ");
+      sb.append("columns:");
+      if (this.columns == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.columns);
+      }
+      first = false;
+    }
+    if (isSetSortedColumns()) {
+      if (!first) sb.append(", ");
+      sb.append("sortedColumns:");
+      if (this.sortedColumns == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.sortedColumns);
+      }
+      first = false;
     }
-    first = false;
     sb.append(")");
     return sb.toString();
   }
@@ -443,7 +541,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowResult struct)
                 for (int _i9 = 0; _i9 < _map8.size; ++_i9)
                 {
                   ByteBuffer _key10; // required
-                  TCell _val11; // optional
+                  TCell _val11; // required
                   _key10 = iprot.readBinary();
                   _val11 = new TCell();
                   _val11.read(iprot);
@@ -456,6 +554,25 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowResult struct)
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 3: // SORTED_COLUMNS
+            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+              {
+                org.apache.thrift.protocol.TList _list12 = iprot.readListBegin();
+                struct.sortedColumns = new ArrayList<TColumn>(_list12.size);
+                for (int _i13 = 0; _i13 < _list12.size; ++_i13)
+                {
+                  TColumn _elem14; // required
+                  _elem14 = new TColumn();
+                  _elem14.read(iprot);
+                  struct.sortedColumns.add(_elem14);
+                }
+                iprot.readListEnd();
+              }
+              struct.setSortedColumnsIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -477,17 +594,33 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TRowResult struct)
         oprot.writeFieldEnd();
       }
       if (struct.columns != null) {
-        oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-          for (Map.Entry<ByteBuffer, TCell> _iter12 : struct.columns.entrySet())
+        if (struct.isSetColumns()) {
+          oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
-            oprot.writeBinary(_iter12.getKey());
-            _iter12.getValue().write(oprot);
+            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
+            for (Map.Entry<ByteBuffer, TCell> _iter15 : struct.columns.entrySet())
+            {
+              oprot.writeBinary(_iter15.getKey());
+              _iter15.getValue().write(oprot);
+            }
+            oprot.writeMapEnd();
           }
-          oprot.writeMapEnd();
+          oprot.writeFieldEnd();
+        }
+      }
+      if (struct.sortedColumns != null) {
+        if (struct.isSetSortedColumns()) {
+          oprot.writeFieldBegin(SORTED_COLUMNS_FIELD_DESC);
+          {
+            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.sortedColumns.size()));
+            for (TColumn _iter16 : struct.sortedColumns)
+            {
+              _iter16.write(oprot);
+            }
+            oprot.writeListEnd();
+          }
+          oprot.writeFieldEnd();
         }
-        oprot.writeFieldEnd();
       }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
@@ -513,17 +646,29 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRowResult struct)
       if (struct.isSetColumns()) {
         optionals.set(1);
       }
-      oprot.writeBitSet(optionals, 2);
+      if (struct.isSetSortedColumns()) {
+        optionals.set(2);
+      }
+      oprot.writeBitSet(optionals, 3);
       if (struct.isSetRow()) {
         oprot.writeBinary(struct.row);
       }
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
-          for (Map.Entry<ByteBuffer, TCell> _iter13 : struct.columns.entrySet())
+          for (Map.Entry<ByteBuffer, TCell> _iter17 : struct.columns.entrySet())
           {
-            oprot.writeBinary(_iter13.getKey());
-            _iter13.getValue().write(oprot);
+            oprot.writeBinary(_iter17.getKey());
+            _iter17.getValue().write(oprot);
+          }
+        }
+      }
+      if (struct.isSetSortedColumns()) {
+        {
+          oprot.writeI32(struct.sortedColumns.size());
+          for (TColumn _iter18 : struct.sortedColumns)
+          {
+            _iter18.write(oprot);
           }
         }
       }
@@ -532,27 +677,41 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRowResult struct)
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, TRowResult struct) throws org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(2);
+      BitSet incoming = iprot.readBitSet(3);
       if (incoming.get(0)) {
         struct.row = iprot.readBinary();
         struct.setRowIsSet(true);
       }
       if (incoming.get(1)) {
         {
-          org.apache.thrift.protocol.TMap _map14 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.columns = new HashMap<ByteBuffer,TCell>(2*_map14.size);
-          for (int _i15 = 0; _i15 < _map14.size; ++_i15)
+          org.apache.thrift.protocol.TMap _map19 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.columns = new HashMap<ByteBuffer,TCell>(2*_map19.size);
+          for (int _i20 = 0; _i20 < _map19.size; ++_i20)
           {
-            ByteBuffer _key16; // required
-            TCell _val17; // required
-            _key16 = iprot.readBinary();
-            _val17 = new TCell();
-            _val17.read(iprot);
-            struct.columns.put(_key16, _val17);
+            ByteBuffer _key21; // required
+            TCell _val22; // required
+            _key21 = iprot.readBinary();
+            _val22 = new TCell();
+            _val22.read(iprot);
+            struct.columns.put(_key21, _val22);
           }
         }
         struct.setColumnsIsSet(true);
       }
+      if (incoming.get(2)) {
+        {
+          org.apache.thrift.protocol.TList _list23 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.sortedColumns = new ArrayList<TColumn>(_list23.size);
+          for (int _i24 = 0; _i24 < _list23.size; ++_i24)
+          {
+            TColumn _elem25; // required
+            _elem25 = new TColumn();
+            _elem25.read(iprot);
+            struct.sortedColumns.add(_elem25);
+          }
+        }
+        struct.setSortedColumnsIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java
index f7cc05d69ddc..0dce8555b5e9 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java
@@ -39,6 +39,7 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
   private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)4);
   private static final org.apache.thrift.protocol.TField CACHING_FIELD_DESC = new org.apache.thrift.protocol.TField("caching", org.apache.thrift.protocol.TType.I32, (short)5);
   private static final org.apache.thrift.protocol.TField FILTER_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("filterString", org.apache.thrift.protocol.TType.STRING, (short)6);
+  private static final org.apache.thrift.protocol.TField SORT_COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("sortColumns", org.apache.thrift.protocol.TType.BOOL, (short)7);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -52,6 +53,7 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
   public List<ByteBuffer> columns; // optional
   public int caching; // optional
   public ByteBuffer filterString; // optional
+  public boolean sortColumns; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -60,7 +62,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     TIMESTAMP((short)3, "timestamp"),
     COLUMNS((short)4, "columns"),
     CACHING((short)5, "caching"),
-    FILTER_STRING((short)6, "filterString");
+    FILTER_STRING((short)6, "filterString"),
+    SORT_COLUMNS((short)7, "sortColumns");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -87,6 +90,8 @@ public static _Fields findByThriftId(int fieldId) {
           return CACHING;
         case 6: // FILTER_STRING
           return FILTER_STRING;
+        case 7: // SORT_COLUMNS
+          return SORT_COLUMNS;
         default:
           return null;
       }
@@ -129,8 +134,9 @@ public String getFieldName() {
   // isset id assignments
   private static final int __TIMESTAMP_ISSET_ID = 0;
   private static final int __CACHING_ISSET_ID = 1;
-  private BitSet __isset_bit_vector = new BitSet(2);
-  private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.TIMESTAMP,_Fields.COLUMNS,_Fields.CACHING,_Fields.FILTER_STRING};
+  private static final int __SORTCOLUMNS_ISSET_ID = 2;
+  private BitSet __isset_bit_vector = new BitSet(3);
+  private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.TIMESTAMP,_Fields.COLUMNS,_Fields.CACHING,_Fields.FILTER_STRING,_Fields.SORT_COLUMNS};
   public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -147,6 +153,8 @@ public String getFieldName() {
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
     tmpMap.put(_Fields.FILTER_STRING, new org.apache.thrift.meta_data.FieldMetaData("filterString", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "Text")));
+    tmpMap.put(_Fields.SORT_COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("sortColumns", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap);
   }
@@ -178,6 +186,7 @@ public TScan(TScan other) {
     if (other.isSetFilterString()) {
       this.filterString = other.filterString;
     }
+    this.sortColumns = other.sortColumns;
   }
 
   public TScan deepCopy() {
@@ -194,6 +203,8 @@ public void clear() {
     setCachingIsSet(false);
     this.caching = 0;
     this.filterString = null;
+    setSortColumnsIsSet(false);
+    this.sortColumns = false;
   }
 
   public byte[] getStartRow() {
@@ -383,6 +394,29 @@ public void setFilterStringIsSet(boolean value) {
     }
   }
 
+  public boolean isSortColumns() {
+    return this.sortColumns;
+  }
+
+  public TScan setSortColumns(boolean sortColumns) {
+    this.sortColumns = sortColumns;
+    setSortColumnsIsSet(true);
+    return this;
+  }
+
+  public void unsetSortColumns() {
+    __isset_bit_vector.clear(__SORTCOLUMNS_ISSET_ID);
+  }
+
+  /** Returns true if field sortColumns is set (has been assigned a value) and false otherwise */
+  public boolean isSetSortColumns() {
+    return __isset_bit_vector.get(__SORTCOLUMNS_ISSET_ID);
+  }
+
+  public void setSortColumnsIsSet(boolean value) {
+    __isset_bit_vector.set(__SORTCOLUMNS_ISSET_ID, value);
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case START_ROW:
@@ -433,6 +467,14 @@ public void setFieldValue(_Fields field, Object value) {
       }
       break;
 
+    case SORT_COLUMNS:
+      if (value == null) {
+        unsetSortColumns();
+      } else {
+        setSortColumns((Boolean)value);
+      }
+      break;
+
     }
   }
 
@@ -456,6 +498,9 @@ public Object getFieldValue(_Fields field) {
     case FILTER_STRING:
       return getFilterString();
 
+    case SORT_COLUMNS:
+      return Boolean.valueOf(isSortColumns());
+
     }
     throw new IllegalStateException();
   }
@@ -479,6 +524,8 @@ public boolean isSet(_Fields field) {
       return isSetCaching();
     case FILTER_STRING:
       return isSetFilterString();
+    case SORT_COLUMNS:
+      return isSetSortColumns();
     }
     throw new IllegalStateException();
   }
@@ -550,6 +597,15 @@ public boolean equals(TScan that) {
         return false;
     }
 
+    boolean this_present_sortColumns = true && this.isSetSortColumns();
+    boolean that_present_sortColumns = true && that.isSetSortColumns();
+    if (this_present_sortColumns || that_present_sortColumns) {
+      if (!(this_present_sortColumns && that_present_sortColumns))
+        return false;
+      if (this.sortColumns != that.sortColumns)
+        return false;
+    }
+
     return true;
   }
 
@@ -626,6 +682,16 @@ public int compareTo(TScan other) {
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetSortColumns()).compareTo(typedOther.isSetSortColumns());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetSortColumns()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sortColumns, typedOther.sortColumns);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -697,6 +763,12 @@ public String toString() {
       }
       first = false;
     }
+    if (isSetSortColumns()) {
+      if (!first) sb.append(", ");
+      sb.append("sortColumns:");
+      sb.append(this.sortColumns);
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -768,13 +840,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
           case 4: // COLUMNS
             if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list18 = iprot.readListBegin();
-                struct.columns = new ArrayList<ByteBuffer>(_list18.size);
-                for (int _i19 = 0; _i19 < _list18.size; ++_i19)
+                org.apache.thrift.protocol.TList _list26 = iprot.readListBegin();
+                struct.columns = new ArrayList<ByteBuffer>(_list26.size);
+                for (int _i27 = 0; _i27 < _list26.size; ++_i27)
                 {
-                  ByteBuffer _elem20; // optional
-                  _elem20 = iprot.readBinary();
-                  struct.columns.add(_elem20);
+                  ByteBuffer _elem28; // required
+                  _elem28 = iprot.readBinary();
+                  struct.columns.add(_elem28);
                 }
                 iprot.readListEnd();
               }
@@ -799,6 +871,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 7: // SORT_COLUMNS
+            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
+              struct.sortColumns = iprot.readBool();
+              struct.setSortColumnsIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -838,9 +918,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) thro
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter21 : struct.columns)
+            for (ByteBuffer _iter29 : struct.columns)
             {
-              oprot.writeBinary(_iter21);
+              oprot.writeBinary(_iter29);
             }
             oprot.writeListEnd();
           }
@@ -859,6 +939,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) thro
           oprot.writeFieldEnd();
         }
       }
+      if (struct.isSetSortColumns()) {
+        oprot.writeFieldBegin(SORT_COLUMNS_FIELD_DESC);
+        oprot.writeBool(struct.sortColumns);
+        oprot.writeFieldEnd();
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -895,7 +980,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetFilterString()) {
         optionals.set(5);
       }
-      oprot.writeBitSet(optionals, 6);
+      if (struct.isSetSortColumns()) {
+        optionals.set(6);
+      }
+      oprot.writeBitSet(optionals, 7);
       if (struct.isSetStartRow()) {
         oprot.writeBinary(struct.startRow);
       }
@@ -908,9 +996,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetColumns()) {
         {
           oprot.writeI32(struct.columns.size());
-          for (ByteBuffer _iter22 : struct.columns)
+          for (ByteBuffer _iter30 : struct.columns)
           {
-            oprot.writeBinary(_iter22);
+            oprot.writeBinary(_iter30);
           }
         }
       }
@@ -920,12 +1008,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throw
       if (struct.isSetFilterString()) {
         oprot.writeBinary(struct.filterString);
       }
+      if (struct.isSetSortColumns()) {
+        oprot.writeBool(struct.sortColumns);
+      }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(6);
+      BitSet incoming = iprot.readBitSet(7);
       if (incoming.get(0)) {
         struct.startRow = iprot.readBinary();
         struct.setStartRowIsSet(true);
@@ -940,13 +1031,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
       }
       if (incoming.get(3)) {
         {
-          org.apache.thrift.protocol.TList _list23 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.columns = new ArrayList<ByteBuffer>(_list23.size);
-          for (int _i24 = 0; _i24 < _list23.size; ++_i24)
+          org.apache.thrift.protocol.TList _list31 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+          struct.columns = new ArrayList<ByteBuffer>(_list31.size);
+          for (int _i32 = 0; _i32 < _list31.size; ++_i32)
           {
-            ByteBuffer _elem25; // optional
-            _elem25 = iprot.readBinary();
-            struct.columns.add(_elem25);
+            ByteBuffer _elem33; // required
+            _elem33 = iprot.readBinary();
+            struct.columns.add(_elem33);
           }
         }
         struct.setColumnsIsSet(true);
@@ -959,6 +1050,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
         struct.filterString = iprot.readBinary();
         struct.setFilterStringIsSet(true);
       }
+      if (incoming.get(6)) {
+        struct.sortColumns = iprot.readBool();
+        struct.setSortColumnsIsSet(true);
+      }
     }
   }
 
diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
index 4b81b1e58531..890339d5d9b9 100644
--- a/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
+++ b/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
@@ -121,12 +121,21 @@ struct TIncrement {
   4:i64  ammount
 }
 
+/**
+ * Holds column name and the cell.
+ */
+struct TColumn {
+  1:Text columnName,
+  2:TCell cell
+ }
+
 /**
  * Holds row name and then a map of columns to cells. 
  */
 struct TRowResult {
   1:Text row,
-  2:map<Text, TCell> columns
+  2:optional map<Text, TCell> columns,
+  3:optional list<TColumn> sortedColumns 
 }
 
 /**
@@ -138,7 +147,8 @@ struct TScan {
   3:optional i64 timestamp,
   4:optional list<Text> columns,
   5:optional i32 caching,
-  6:optional Text filterString
+  6:optional Text filterString,
+  7:optional bool sortColumns
 }
 
 //
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 81409c4f5b0e..0cd4620a839a 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
 import org.apache.hadoop.hbase.thrift.generated.TIncrement;
 import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler;
+import org.apache.hadoop.hbase.thrift.generated.TScan;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.MetricsContext;
@@ -462,6 +463,46 @@ public void doTestTableScanners() throws Exception {
     assertEquals(rowResult4a.columns.size(), 1);
     assertEquals(rowResult4a.columns.get(columnBname).value, valueBname);
 
+    // Test scanner using a TScan object once with sortColumns False and once with sortColumns true
+    TScan scanNoSortColumns = new TScan();
+    scanNoSortColumns.setStartRow(rowAname);
+    scanNoSortColumns.setStopRow(rowBname);
+
+    int scanner5 = handler.scannerOpenWithScan(tableAname , scanNoSortColumns, null);
+    TRowResult rowResult5 = handler.scannerGet(scanner5).get(0);
+    assertEquals(rowResult5.columns.size(), 1);
+    assertEquals(rowResult5.columns.get(columnBname).value, valueCname);
+
+    TScan scanSortColumns = new TScan();
+    scanSortColumns.setStartRow(rowAname);
+    scanSortColumns.setStopRow(rowBname);
+    scanSortColumns = scanSortColumns.setSortColumns(true);
+
+    int scanner6 = handler.scannerOpenWithScan(tableAname ,scanSortColumns, null);
+    TRowResult rowResult6 = handler.scannerGet(scanner6).get(0);
+    assertEquals(rowResult6.sortedColumns.size(), 1);
+    assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname);
+
+    List<Mutation> rowBmutations = new ArrayList<Mutation>();
+    for (int i = 0; i < 20; i++) {
+      rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true));
+    }
+    ByteBuffer rowC = asByteBuffer("rowC");
+    handler.mutateRow(tableAname, rowC, rowBmutations, null);
+
+    TScan scanSortMultiColumns = new TScan();
+    scanSortMultiColumns.setStartRow(rowC);
+    scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true);
+    int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null);
+    TRowResult rowResult7 = handler.scannerGet(scanner7).get(0);
+
+    ByteBuffer smallerColumn = asByteBuffer("columnA:");
+    for (int i = 0; i < 20; i++) {
+      ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName;
+      assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0);
+      smallerColumn = currentColumn;
+    }
+
     // Teardown
     handler.disableTable(tableAname);
     handler.deleteTable(tableAname);

From c38f5c93a53205a9265d1f7d89aa4f81d0a1138a Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 30 Jul 2013 17:42:12 +0000
Subject: [PATCH 1088/1540] HBASE-9080 Retain assignment should be used when
 re-enabling table(s) (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1508546 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/handler/EnableTableHandler.java    | 52 +++++++------------
 .../apache/hadoop/hbase/client/TestAdmin.java | 42 ++++-----------
 2 files changed, 29 insertions(+), 65 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
index 7e348107507c..7f549648470d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -52,7 +52,7 @@ public class EnableTableHandler extends EventHandler {
   private final String tableNameStr;
   private final AssignmentManager assignmentManager;
   private final CatalogTracker ct;
-  private boolean retainAssignment = false;
+  private boolean skipTableStateCheck = false;
 
   public EnableTableHandler(Server server, byte [] tableName,
       CatalogTracker catalogTracker, AssignmentManager assignmentManager,
@@ -63,12 +63,12 @@ public EnableTableHandler(Server server, byte [] tableName,
     this.tableNameStr = Bytes.toString(tableName);
     this.ct = catalogTracker;
     this.assignmentManager = assignmentManager;
-    this.retainAssignment = skipTableStateCheck;
+    this.skipTableStateCheck = skipTableStateCheck;
     // Check if table exists
     if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
-      // retainAssignment is true only during recovery. In normal case it is
+      // skipTableStateCheck is true only during recovery. In normal case it is
       // false
-      if (!this.retainAssignment) {
+      if (!this.skipTableStateCheck) {
         throw new TableNotFoundException(tableNameStr);
       }
       try {
@@ -142,8 +142,7 @@ private void handleEnableTable() throws IOException, KeeperException, Interrupte
     }
     LOG.info("Table has " + countOfRegionsInTable + " regions of which " +
       regionsCount + " are offline.");
-    BulkEnabler bd = new BulkEnabler(this.server, regions, countOfRegionsInTable,
-        this.retainAssignment);
+    BulkEnabler bd = new BulkEnabler(this.server, regions, countOfRegionsInTable, true);
     try {
       if (bd.bulkAssign()) {
         done = true;
@@ -174,7 +173,7 @@ private List<HRegionInfo> regionsToAssignWithServerName(
     for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
       HRegionInfo hri = regionLocation.getFirst();
       ServerName sn = regionLocation.getSecond();
-      if (this.retainAssignment) {
+      if (this.skipTableStateCheck) {
         // Region may be available in enablingTableRegions during master startup only.
         if (enablingTableRegions != null && enablingTableRegions.contains(hri)) {
           regions.add(hri);
@@ -186,6 +185,9 @@ private List<HRegionInfo> regionsToAssignWithServerName(
         continue;
       } else {
         regions.add(hri);
+        if (sn != null && serverManager.isServerOnline(sn)) {
+          this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn));
+        }
       }
     }
     return regions;
@@ -198,44 +200,26 @@ class BulkEnabler extends BulkAssigner {
     private final List<HRegionInfo> regions;
     // Count of regions in table at time this assign was launched.
     private final int countOfRegionsInTable;
-    private final boolean retainAssignment;
 
     BulkEnabler(final Server server, final List<HRegionInfo> regions,
         final int countOfRegionsInTable,final boolean retainAssignment) {
       super(server);
       this.regions = regions;
       this.countOfRegionsInTable = countOfRegionsInTable;
-      this.retainAssignment = retainAssignment;
     }
 
     @Override
     protected void populatePool(ExecutorService pool) throws IOException {
-      boolean roundRobinAssignment = this.server.getConfiguration().getBoolean(
-          "hbase.master.enabletable.roundrobin", false);
-
-      if (retainAssignment || !roundRobinAssignment) {
-        for (HRegionInfo region : regions) {
-          if (assignmentManager.isRegionInTransition(region) != null) {
-            continue;
-          }
-          final HRegionInfo hri = region;
-          pool.execute(new Runnable() {
-            public void run() {
-              if (retainAssignment) {
-                assignmentManager.assign(hri, true, false, false);
-              } else {
-                assignmentManager.assign(hri, true);
-              }
-            }
-          });
-        }
-      } else {
-        try {
-          assignmentManager.assignUserRegionsToOnlineServers(regions);
-        } catch (InterruptedException e) {
-          LOG.warn("Assignment was interrupted");
-          Thread.currentThread().interrupt();
+      for (HRegionInfo region : regions) {
+        if (assignmentManager.isRegionInTransition(region) != null) {
+          continue;
         }
+        final HRegionInfo hri = region;
+        pool.execute(new Runnable() {
+          public void run() {
+            assignmentManager.assign(hri, true, false, false);
+          }
+        });
       }
     }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 35653a8a05f2..ce068cd21f47 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -856,12 +856,12 @@ public void testForceSplit() throws Exception {
   }
 
   /**
-   * Test round-robin assignment on enableTable.
+   * Test retain assignment on enableTable.
    *
    * @throws IOException
    */
   @Test
-  public void testEnableTableRoundRobinAssignment() throws IOException {
+  public void testEnableTableRetainAssignment() throws IOException, InterruptedException {
     byte[] tableName = Bytes.toBytes("testEnableTableAssignment");
     byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
         new byte[] { 3, 3, 3 }, new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 },
@@ -872,42 +872,22 @@ public void testEnableTableRoundRobinAssignment() throws IOException {
     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
     admin.createTable(desc, splitKeys);
     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    Map<HRegionInfo, HServerAddress> regions = ht.getRegionsInfo();
+    Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
     assertEquals("Tried to create " + expectedRegions + " regions "
         + "but only found " + regions.size(), expectedRegions, regions.size());
     // Disable table.
     admin.disableTable(tableName);
-    // Enable table, use round-robin assignment to assign regions.
+    // Enable table, use retain assignment to assign regions.
     admin.enableTable(tableName);
+    Map<HRegionInfo, ServerName> regions2 = ht.getRegionLocations();
 
     // Check the assignment.
-    HTable metaTable = new HTable(TEST_UTIL.getConfiguration(),
-        HConstants.META_TABLE_NAME);
-    List<HRegionInfo> regionInfos = admin.getTableRegions(tableName);
-    Map<String, Integer> serverMap = new HashMap<String, Integer>();
-    for (int i = 0, j = regionInfos.size(); i < j; i++) {
-      HRegionInfo hri = regionInfos.get(i);
-      Get get = new Get(hri.getRegionName());
-      Result result = metaTable.get(get);
-      String server = Bytes.toString(result.getValue(HConstants.CATALOG_FAMILY,
-          HConstants.SERVER_QUALIFIER));
-      Integer regioncount = serverMap.get(server);
-      if (regioncount == null) {
-        regioncount = 0;
-      }
-      regioncount++;
-      serverMap.put(server, regioncount);
-    }
-    List<Map.Entry<String, Integer>> entryList = new ArrayList<Map.Entry<String, Integer>>(
-        serverMap.entrySet());
-    Collections.sort(entryList, new Comparator<Map.Entry<String, Integer>>() {
-      public int compare(Map.Entry<String, Integer> oa,
-          Map.Entry<String, Integer> ob) {
-        return (oa.getValue() - ob.getValue());
-      }
-    });
-    assertTrue(entryList.size() == 3);
-    assertTrue((entryList.get(2).getValue() - entryList.get(0).getValue()) < 2);
+    assertEquals(regions.size(), regions2.size());
+    for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
+      ServerName sn1 = regions2.get(entry.getKey());
+      ServerName sn2 = entry.getValue();
+      assertEquals(sn1, sn2);
+    }
   }
 
   /**

From 309ce9df57ac9408eb787ef8ae828c055c24ad1e Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 31 Jul 2013 11:11:09 +0000
Subject: [PATCH 1089/1540] HBASE-9090 cleanup snapshot tests setup/teardown
 code

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1508811 13f79535-47bb-0310-9956-ffa450edef68
---
 .../client/TestCloneSnapshotFromClient.java   |  82 +++-----------
 .../client/TestRestoreSnapshotFromClient.java |  76 +++----------
 .../hbase/client/TestSnapshotFromClient.java  |   8 +-
 .../cleaner/TestSnapshotFromMaster.java       |  18 +--
 .../hbase/snapshot/SnapshotTestingUtils.java  | 106 ++++++++++++++++++
 .../hbase/snapshot/TestExportSnapshot.java    |   7 +-
 .../snapshot/TestFlushSnapshotFromClient.java |   8 +-
 .../TestRestoreFlushSnapshotFromClient.java   | 101 +++++------------
 8 files changed, 180 insertions(+), 226 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
index 66a8b3de8e62..841ab7400f26 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
@@ -32,8 +32,8 @@
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -96,7 +96,7 @@ public void setup() throws Exception {
     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
 
     // create Table and disable it
-    createTable(tableName, FAMILY);
+    SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
     admin.disableTable(tableName);
 
     // take an empty snapshot
@@ -106,7 +106,7 @@ public void setup() throws Exception {
     try {
       // enable table and insert data
       admin.enableTable(tableName);
-      loadData(table, 500, FAMILY);
+      SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
       snapshot0Rows = TEST_UTIL.countRows(table);
       admin.disableTable(tableName);
 
@@ -115,7 +115,7 @@ public void setup() throws Exception {
 
       // enable table and insert more data
       admin.enableTable(tableName);
-      loadData(table, 500, FAMILY);
+      SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
       snapshot1Rows = TEST_UTIL.countRows(table);
       admin.disableTable(tableName);
 
@@ -134,13 +134,8 @@ public void tearDown() throws Exception {
     if (admin.tableExists(tableName)) {
       TEST_UTIL.deleteTable(tableName);
     }
-    admin.deleteSnapshot(snapshotName0);
-    admin.deleteSnapshot(snapshotName1);
-
-    // Ensure the archiver to be empty
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
-    mfs.getFileSystem().delete(
-      new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+    SnapshotTestingUtils.deleteAllSnapshots(admin);
+    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
   }
 
   @Test(expected=SnapshotDoesNotExistException.class)
@@ -162,10 +157,9 @@ private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName
       int snapshotRows) throws IOException, InterruptedException {
     // create a new table from snapshot
     admin.cloneSnapshot(snapshotName, tableName);
-    verifyRowCount(tableName, snapshotRows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshotRows);
 
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
+    TEST_UTIL.deleteTable(tableName);
   }
 
   /**
@@ -176,7 +170,7 @@ public void testCloneLinksAfterDelete() throws IOException, InterruptedException
     // Clone a table from the first snapshot
     byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
     admin.cloneSnapshot(snapshotName0, clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
 
     // Take a snapshot of this cloned table.
     admin.disableTable(clonedTableName);
@@ -185,85 +179,45 @@ public void testCloneLinksAfterDelete() throws IOException, InterruptedException
     // Clone the snapshot of the cloned table
     byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
     admin.cloneSnapshot(snapshotName2, clonedTableName2);
-    verifyRowCount(clonedTableName2, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
     admin.disableTable(clonedTableName2);
 
     // Remove the original table
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
+    TEST_UTIL.deleteTable(tableName);
     waitCleanerRun();
 
     // Verify the first cloned table
     admin.enableTable(clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
 
     // Verify the second cloned table
     admin.enableTable(clonedTableName2);
-    verifyRowCount(clonedTableName2, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
     admin.disableTable(clonedTableName2);
 
     // Delete the first cloned table
-    admin.disableTable(clonedTableName);
-    admin.deleteTable(clonedTableName);
+    TEST_UTIL.deleteTable(clonedTableName);
     waitCleanerRun();
 
     // Verify the second cloned table
     admin.enableTable(clonedTableName2);
-    verifyRowCount(clonedTableName2, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
 
     // Clone a new table from cloned
     byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
     admin.cloneSnapshot(snapshotName2, clonedTableName3);
-    verifyRowCount(clonedTableName3, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName3, snapshot0Rows);
 
     // Delete the cloned tables
-    admin.disableTable(clonedTableName2);
-    admin.deleteTable(clonedTableName2);
-    admin.disableTable(clonedTableName3);
-    admin.deleteTable(clonedTableName3);
+    TEST_UTIL.deleteTable(clonedTableName2);
+    TEST_UTIL.deleteTable(clonedTableName3);
     admin.deleteSnapshot(snapshotName2);
   }
 
   // ==========================================================================
   //  Helpers
   // ==========================================================================
-  private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    for (byte[] family: families) {
-      HColumnDescriptor hcd = new HColumnDescriptor(family);
-      htd.addFamily(hcd);
-    }
-    byte[][] splitKeys = new byte[16][];
-    byte[] hex = Bytes.toBytes("0123456789abcdef");
-    for (int i = 0; i < 16; ++i) {
-      splitKeys[i] = new byte[] { hex[i] };
-    }
-    admin.createTable(htd, splitKeys);
-  }
-
-  public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
-    byte[] qualifier = Bytes.toBytes("q");
-    table.setAutoFlush(false);
-    while (rows-- > 0) {
-      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
-      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
-      Put put = new Put(key);
-      put.setWriteToWAL(false);
-      for (byte[] family: families) {
-        put.add(family, qualifier, value);
-      }
-      table.put(put);
-    }
-    table.flushCommits();
-  }
-
   private void waitCleanerRun() throws InterruptedException {
     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
   }
-
-  private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    assertEquals(expectedRows, TEST_UTIL.countRows(table));
-    table.close();
-  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index f3af219fb749..2fec47ce6b3c 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -35,9 +35,9 @@
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -100,7 +100,7 @@ public void setup() throws Exception {
     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
 
     // create Table and disable it
-    createTable(tableName, FAMILY);
+    SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
     admin.disableTable(tableName);
 
     // take an empty snapshot
@@ -110,7 +110,7 @@ public void setup() throws Exception {
     try {
       // enable table and insert data
       admin.enableTable(tableName);
-      loadData(table, 500, FAMILY);
+      SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
       snapshot0Rows = TEST_UTIL.countRows(table);
       admin.disableTable(tableName);
 
@@ -119,7 +119,7 @@ public void setup() throws Exception {
 
       // enable table and insert more data
       admin.enableTable(tableName);
-      loadData(table, 500, FAMILY);
+      SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
       snapshot1Rows = TEST_UTIL.countRows(table);
       admin.disableTable(tableName);
 
@@ -135,43 +135,36 @@ public void setup() throws Exception {
 
   @After
   public void tearDown() throws Exception {
-    if (admin.tableExists(tableName)) {
-      TEST_UTIL.deleteTable(tableName);
-    }
-    admin.deleteSnapshot(snapshotName0);
-    admin.deleteSnapshot(snapshotName1);
-
-    // Ensure the archiver to be empty
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
-    mfs.getFileSystem().delete(
-      new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+    TEST_UTIL.deleteTable(tableName);
+    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
+    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
   }
 
   @Test
   public void testRestoreSnapshot() throws IOException {
-    verifyRowCount(tableName, snapshot1Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
 
     // Restore from snapshot-0
     admin.disableTable(tableName);
     admin.restoreSnapshot(snapshotName0);
     admin.enableTable(tableName);
-    verifyRowCount(tableName, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
 
     // Restore from emptySnapshot
     admin.disableTable(tableName);
     admin.restoreSnapshot(emptySnapshot);
     admin.enableTable(tableName);
-    verifyRowCount(tableName, 0);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
 
     // Restore from snapshot-1
     admin.disableTable(tableName);
     admin.restoreSnapshot(snapshotName1);
     admin.enableTable(tableName);
-    verifyRowCount(tableName, snapshot1Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
   }
 
   @Test
-  public void testRestoreSchemaChange() throws IOException {
+  public void testRestoreSchemaChange() throws Exception {
     byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
 
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
@@ -183,7 +176,7 @@ public void testRestoreSchemaChange() throws IOException {
     assertEquals(2, table.getTableDescriptor().getFamilies().size());
     HTableDescriptor htd = admin.getTableDescriptor(tableName);
     assertEquals(2, htd.getFamilies().size());
-    loadData(table, 500, TEST_FAMILY2);
+    SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, TEST_FAMILY2);
     long snapshot2Rows = snapshot1Rows + 500;
     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
@@ -230,51 +223,20 @@ public void testRestoreSchemaChange() throws IOException {
   public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
     admin.cloneSnapshot(snapshotName0, clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
     admin.disableTable(clonedTableName);
     admin.snapshot(snapshotName2, clonedTableName);
     admin.deleteTable(clonedTableName);
     waitCleanerRun();
 
     admin.cloneSnapshot(snapshotName2, clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
-    admin.disableTable(clonedTableName);
-    admin.deleteTable(clonedTableName);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
+    TEST_UTIL.deleteTable(clonedTableName);
   }
 
   // ==========================================================================
   //  Helpers
   // ==========================================================================
-  private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    for (byte[] family: families) {
-      HColumnDescriptor hcd = new HColumnDescriptor(family);
-      htd.addFamily(hcd);
-    }
-    byte[][] splitKeys = new byte[16][];
-    byte[] hex = Bytes.toBytes("0123456789abcdef");
-    for (int i = 0; i < 16; ++i) {
-      splitKeys[i] = new byte[] { hex[i] };
-    }
-    admin.createTable(htd, splitKeys);
-  }
-
-  public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
-    byte[] qualifier = Bytes.toBytes("q");
-    table.setAutoFlush(false);
-    while (rows-- > 0) {
-      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
-      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
-      Put put = new Put(key);
-      put.setWriteToWAL(false);
-      for (byte[] family: families) {
-        put.add(family, qualifier, value);
-      }
-      table.put(put);
-    }
-    table.flushCommits();
-  }
-
   private void waitCleanerRun() throws InterruptedException {
     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
   }
@@ -290,10 +252,4 @@ private Set<String> getFamiliesFromFS(final byte[] tableName) throws IOException
     }
     return families;
   }
-
-  private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    assertEquals(expectedRows, TEST_UTIL.countRows(table));
-    table.close();
-  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index e1b5ff8e322b..c8ff1cd9a011 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -101,12 +101,8 @@ public void setup() throws Exception {
   @After
   public void tearDown() throws Exception {
     UTIL.deleteTable(TABLE_NAME);
-    // and cleanup the archive directory
-    try {
-      UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
-    } catch (IOException e) {
-      LOG.warn("Failure to delete archive directory", e);
-    }
+    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
   }
 
   @AfterClass
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 92a8bb856736..37e40cdc2f44 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -133,22 +133,8 @@ public void setup() throws Exception {
   @After
   public void tearDown() throws Exception {
     UTIL.deleteTable(TABLE_NAME);
-
-    // delete the archive directory, if its exists
-    if (fs.exists(archiveDir)) {
-      if (!fs.delete(archiveDir, true)) {
-        throw new IOException("Couldn't delete archive directory (" + archiveDir
-            + " for an unknown reason");
-      }
-    }
-
-    // delete the snapshot directory, if its exists
-    if (fs.exists(snapshots)) {
-      if (!fs.delete(snapshots, true)) {
-        throw new IOException("Couldn't delete snapshots directory (" + snapshots
-            + " for an unknown reason");
-      }
-    }
+    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
   }
 
   @AfterClass
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index b04fb55f54f7..cb9aa33c524c 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -34,19 +34,27 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
 import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.Assert;
 
 /**
@@ -252,4 +260,102 @@ public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families, Fi
     }
     return files;
   }
+
+  // ==========================================================================
+  //  Table Helpers
+  // ==========================================================================
+  public static void waitForTableToBeOnline(final HBaseTestingUtility util, final byte[] tableName)
+      throws IOException, InterruptedException {
+    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
+    List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
+    for (HRegion region : onlineRegions) {
+      region.waitForFlushesAndCompactions();
+    }
+    util.getHBaseAdmin().isTableAvailable(tableName);
+  }
+
+  public static void createTable(final HBaseTestingUtility util, final byte[] tableName,
+      final byte[]... families) throws IOException, InterruptedException {
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    for (byte[] family: families) {
+      HColumnDescriptor hcd = new HColumnDescriptor(family);
+      htd.addFamily(hcd);
+    }
+    byte[][] splitKeys = new byte[14][];
+    byte[] hex = Bytes.toBytes("123456789abcde");
+    for (int i = 0; i < splitKeys.length; ++i) {
+      splitKeys[i] = new byte[] { hex[i] };
+    }
+    util.getHBaseAdmin().createTable(htd, splitKeys);
+    waitForTableToBeOnline(util, tableName);
+    assertEquals(15, util.getHBaseAdmin().getTableRegions(tableName).size());
+  }
+
+  public static void loadData(final HBaseTestingUtility util, final byte[] tableName, int rows,
+      byte[]... families) throws IOException, InterruptedException {
+    loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
+  }
+
+  public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
+      byte[]... families) throws IOException, InterruptedException {
+    table.setAutoFlush(false);
+
+    // Ensure one row per region
+    assertTrue(rows >= 16);
+    for (byte k0: Bytes.toBytes("0123456789abcdef")) {
+      byte[] k = new byte[] { k0 };
+      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
+      byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
+      putData(table, families, key, value);
+      rows--;
+    }
+
+    // Add other extra rows. more rows, more files
+    while (rows-- > 0) {
+      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
+      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
+      putData(table, families, key, value);
+    }
+    table.flushCommits();
+
+    waitForTableToBeOnline(util, table.getTableName());
+  }
+
+  private static void putData(final HTable table, final byte[][] families,
+      final byte[] key, final byte[] value) throws IOException {
+    byte[] q = Bytes.toBytes("q");
+    Put put = new Put(key);
+    put.setDurability(Durability.SKIP_WAL);
+    for (byte[] family: families) {
+      put.add(family, q, value);
+    }
+    table.put(put);
+  }
+
+  public static void deleteAllSnapshots(final HBaseAdmin admin)
+      throws IOException {
+    // Delete all the snapshots
+    for (SnapshotDescription snapshot: admin.listSnapshots()) {
+      admin.deleteSnapshot(snapshot.getName());
+    }
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+  }
+
+  public static void deleteArchiveDirectory(final HBaseTestingUtility util)
+      throws IOException {
+    // Ensure the archiver to be empty
+    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
+    mfs.getFileSystem().delete(archiveDir, true);
+  }
+
+  public static void verifyRowCount(final HBaseTestingUtility util, final byte[] tableName,
+      long expectedRows) throws IOException {
+    HTable table = new HTable(util.getConfiguration(), tableName);
+    try {
+      assertEquals(expectedRows, util.countRows(table));
+    } finally {
+      table.close();
+    }
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index 9dafa9a9e21b..a666c5df624c 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -126,10 +126,9 @@ public void setUp() throws Exception {
 
   @After
   public void tearDown() throws Exception {
-    admin.disableTable(tableName);
-    admin.deleteSnapshot(snapshotName);
-    admin.deleteSnapshot(emptySnapshotName);
-    admin.deleteTable(tableName);
+    TEST_UTIL.deleteTable(tableName);
+    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
+    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
     admin.close();
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index ded6c66cf877..5dfd7ebe10a0 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -112,12 +112,8 @@ public void setup() throws Exception {
   @After
   public void tearDown() throws Exception {
     UTIL.deleteTable(TABLE_NAME);
-    // and cleanup the archive directory
-    try {
-      UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
-    } catch (IOException e) {
-      LOG.warn("Failure to delete archive directory", e);
-    }
+    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
   }
 
   @AfterClass
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
index 498d50885aad..519086e44f3c 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
@@ -18,8 +18,10 @@
 package org.apache.hadoop.hbase.snapshot;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -56,9 +58,10 @@
 public class TestRestoreFlushSnapshotFromClient {
   final Log LOG = LogFactory.getLog(getClass());
 
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
 
   private final byte[] FAMILY = Bytes.toBytes("cf");
+  private static final byte[] TEST_QUAL = Bytes.toBytes("q");
 
   private byte[] snapshotName0;
   private byte[] snapshotName1;
@@ -70,22 +73,22 @@ public class TestRestoreFlushSnapshotFromClient {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
-    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
-    TEST_UTIL.getConfiguration().setBoolean(
+    UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
+    UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+    UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+    UTIL.getConfiguration().setBoolean(
         "hbase.master.enabletable.roundrobin", true);
 
     // Enable snapshot
-    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
 
-    TEST_UTIL.startMiniCluster(3);
+    UTIL.startMiniCluster(3);
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
+    UTIL.shutdownMiniCluster();
   }
 
   /**
@@ -95,7 +98,7 @@ public static void tearDownAfterClass() throws Exception {
    */
   @Before
   public void setup() throws Exception {
-    this.admin = TEST_UTIL.getHBaseAdmin();
+    this.admin = UTIL.getHBaseAdmin();
 
     long tid = System.currentTimeMillis();
     tableName = Bytes.toBytes("testtb-" + tid);
@@ -104,11 +107,11 @@ public void setup() throws Exception {
     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
 
     // create Table and disable it
-    createTable(tableName, FAMILY);
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY);
+    HTable table = new HTable(UTIL.getConfiguration(), tableName);
     try {
-      loadData(table, 500, FAMILY);
-      snapshot0Rows = TEST_UTIL.countRows(table);
+      SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
+      snapshot0Rows = UTIL.countRows(table);
       LOG.info("=== before snapshot with 500 rows");
       logFSTree();
 
@@ -120,8 +123,8 @@ public void setup() throws Exception {
       logFSTree();
 
       // insert more data
-      loadData(table, 500, FAMILY);
-      snapshot1Rows = TEST_UTIL.countRows(table);
+      SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
+      snapshot1Rows = UTIL.countRows(table);
       LOG.info("=== before snapshot with 1000 rows");
       logFSTree();
 
@@ -137,14 +140,8 @@ public void setup() throws Exception {
 
   @After
   public void tearDown() throws Exception {
-    TEST_UTIL.deleteTable(tableName);
-    admin.deleteSnapshot(snapshotName0);
-    admin.deleteSnapshot(snapshotName1);
-
-    // Ensure the archiver to be empty
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
-    mfs.getFileSystem().delete(
-      new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
   }
 
   @Test
@@ -154,7 +151,7 @@ public void testTakeFlushSnapshot() throws IOException {
 
   @Test
   public void testRestoreSnapshot() throws IOException {
-    verifyRowCount(tableName, snapshot1Rows);
+    SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
 
     // Restore from snapshot-0
     admin.disableTable(tableName);
@@ -163,13 +160,13 @@ public void testRestoreSnapshot() throws IOException {
     admin.enableTable(tableName);
     LOG.info("=== after restore with 500 row snapshot");
     logFSTree();
-    verifyRowCount(tableName, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows);
 
     // Restore from snapshot-1
     admin.disableTable(tableName);
     admin.restoreSnapshot(snapshotName1);
     admin.enableTable(tableName);
-    verifyRowCount(tableName, snapshot1Rows);
+    SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
   }
 
   @Test(expected=SnapshotDoesNotExistException.class)
@@ -190,65 +187,29 @@ private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName
       int snapshotRows) throws IOException, InterruptedException {
     // create a new table from snapshot
     admin.cloneSnapshot(snapshotName, tableName);
-    verifyRowCount(tableName, snapshotRows);
+    SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshotRows);
 
-    TEST_UTIL.deleteTable(tableName);
+    UTIL.deleteTable(tableName);
   }
 
   @Test
   public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
     admin.cloneSnapshot(snapshotName0, clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
+    SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
     admin.snapshot(Bytes.toString(snapshotName2), Bytes.toString(clonedTableName), SnapshotDescription.Type.FLUSH);
-    TEST_UTIL.deleteTable(clonedTableName);
+    UTIL.deleteTable(clonedTableName);
 
     admin.cloneSnapshot(snapshotName2, clonedTableName);
-    verifyRowCount(clonedTableName, snapshot0Rows);
-    TEST_UTIL.deleteTable(clonedTableName);
+    SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
+    UTIL.deleteTable(clonedTableName);
   }
 
   // ==========================================================================
   //  Helpers
   // ==========================================================================
-  private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    for (byte[] family: families) {
-      HColumnDescriptor hcd = new HColumnDescriptor(family);
-      htd.addFamily(hcd);
-    }
-    byte[][] splitKeys = new byte[16][];
-    byte[] hex = Bytes.toBytes("0123456789abcdef");
-    for (int i = 0; i < 16; ++i) {
-      splitKeys[i] = new byte[] { hex[i] };
-    }
-    admin.createTable(htd, splitKeys);
-  }
-
-  public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
-    byte[] qualifier = Bytes.toBytes("q");
-    table.setAutoFlush(false);
-    while (rows-- > 0) {
-      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
-      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
-      Put put = new Put(key);
-      put.setWriteToWAL(false);
-      for (byte[] family: families) {
-        put.add(family, qualifier, value);
-      }
-      table.put(put);
-    }
-    table.flushCommits();
-  }
-
   private void logFSTree() throws IOException {
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
     FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
   }
-
-  private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    assertEquals(expectedRows, TEST_UTIL.countRows(table));
-    table.close();
-  }
 }

From bcf89528dc06fa1b63e587283e1bb8a7988aa8d1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 2 Aug 2013 01:40:53 +0000
Subject: [PATCH 1090/1540] HBASE-9029 Backport HBASE-8706 Some improvement in
 snapshot to 0.94 (Jerry He, original patch by Matteo)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509512 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/snapshot/SnapshotManager.java      | 19 +++++----
 .../master/snapshot/TakeSnapshotHandler.java  |  3 +-
 .../hbase/procedure/ProcedureCoordinator.java | 42 +++++++++++++++++--
 .../hbase/procedure/ProcedureMember.java      | 27 ++++++++++--
 .../hadoop/hbase/procedure/Subprocedure.java  |  1 -
 .../snapshot/RegionServerSnapshotManager.java |  8 ++--
 .../procedure/TestProcedureCoordinator.java   |  2 +-
 .../hbase/procedure/TestProcedureMember.java  |  4 +-
 .../hbase/procedure/TestZKProcedure.java      |  8 ++--
 9 files changed, 84 insertions(+), 30 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 439989cff5b9..eeffb96844a7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -113,7 +113,7 @@ public class SnapshotManager implements Stoppable {
   private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
 
   /** By default, check to see if the snapshot is complete (ms) */
-  private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 5000;
+  private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
 
   /**
    * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
@@ -131,7 +131,6 @@ public class SnapshotManager implements Stoppable {
   private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
 
   private boolean stopped;
-  private final long wakeFrequency;
   private final MasterServices master;  // Needed by TableEventHandlers
   private final MasterMetrics metricsMaster;
   private final ProcedureCoordinator coordinator;
@@ -168,16 +167,16 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
 
     // get the configuration for the coordinator
     Configuration conf = master.getConfiguration();
-    this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
-    long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+    long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
+    long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
     int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
 
     // setup the default procedure coordinator
     String name = master.getServerName().toString();
-    ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, keepAliveTime, opThreads, wakeFrequency);
+    ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
     ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
         master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
-    this.coordinator = new ProcedureCoordinator(comms, tpool);
+    this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
     this.executorService = master.getExecutorService();
     resetTempDir();
   }
@@ -197,8 +196,6 @@ public SnapshotManager(final MasterServices master, final MasterMetrics metricsM
     this.rootDir = master.getMasterFileSystem().getRootDir();
     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
 
-    this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
-      SNAPSHOT_WAKE_MILLIS_DEFAULT);
     this.coordinator = coordinator;
     this.executorService = pool;
     resetTempDir();
@@ -870,6 +867,12 @@ public void stop(String why) {
     for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
       restoreHandler.cancel(why);
     }
+
+    try {
+      coordinator.close();
+    } catch (IOException e) {
+      LOG.error("stop ProcedureCoordinator error", e);
+    }
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 48ed9676244a..e8d9abb07d9d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -97,7 +97,7 @@ public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices ma
     this.rootDir = this.master.getMasterFileSystem().getRootDir();
     this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
     this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
-    this.monitor =  new ForeignExceptionDispatcher();
+    this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
 
     // prepare the verify
     this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
@@ -147,6 +147,7 @@ public void process() {
 
       // run the snapshot
       snapshotRegions(regionsAndLocations);
+      monitor.rethrowException();
 
       // extract each pair to separate lists
       Set<String> serverNames = new HashSet<String>();
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
index f9c62590a276..dbeac4fb3ae7 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
@@ -51,11 +51,14 @@
 public class ProcedureCoordinator {
   private static final Log LOG = LogFactory.getLog(ProcedureCoordinator.class);
 
+  final static long KEEP_ALIVE_MILLIS_DEFAULT = 5000;
   final static long TIMEOUT_MILLIS_DEFAULT = 60000;
   final static long WAKE_MILLIS_DEFAULT = 500;
 
   private final ProcedureCoordinatorRpcs rpcs;
   private final ExecutorService pool;
+  private final long wakeTimeMillis;
+  private final long timeoutMillis;
 
   // Running procedure table.  Maps procedure name to running procedure reference
   private final ConcurrentMap<String, Procedure> procedures =
@@ -71,6 +74,23 @@ public class ProcedureCoordinator {
    * @param pool Used for executing procedures.
    */
   public ProcedureCoordinator(ProcedureCoordinatorRpcs rpcs, ThreadPoolExecutor pool) {
+    this(rpcs, pool, TIMEOUT_MILLIS_DEFAULT, WAKE_MILLIS_DEFAULT);
+  }
+
+  /**
+   * Create and start a ProcedureCoordinator.
+   *
+   * The rpc object registers the ProcedureCoordinator and starts any threads in
+   * this constructor.
+   *
+   * @param rpcs
+   * @param pool Used for executing procedures.
+   * @param timeoutMillis
+   */
+  public ProcedureCoordinator(ProcedureCoordinatorRpcs rpcs, ThreadPoolExecutor pool,
+      long timeoutMillis, long wakeTimeMillis) {
+    this.timeoutMillis = timeoutMillis;
+    this.wakeTimeMillis = wakeTimeMillis;
     this.rpcs = rpcs;
     this.pool = pool;
     this.rpcs.start(this);
@@ -78,10 +98,24 @@ public ProcedureCoordinator(ProcedureCoordinatorRpcs rpcs, ThreadPoolExecutor po
 
   /**
    * Default thread pool for the procedure
+   *
+   * @param coordName
+   * @param opThreads the maximum number of threads to allow in the pool
+   */
+  public static ThreadPoolExecutor defaultPool(String coordName, int opThreads) {
+    return defaultPool(coordName, opThreads, KEEP_ALIVE_MILLIS_DEFAULT);
+  }
+
+  /**
+   * Default thread pool for the procedure
+   *
+   * @param coordName
+   * @param opThreads the maximum number of threads to allow in the pool
+   * @param keepAliveMillis the maximum time (ms) that excess idle threads will wait for new tasks
    */
-  public static ThreadPoolExecutor defaultPool(String coordName, long keepAliveTime, int opThreads,
-      long wakeFrequency) {
-    return new ThreadPoolExecutor(1, opThreads, keepAliveTime, TimeUnit.SECONDS,
+  public static ThreadPoolExecutor defaultPool(String coordName, int opThreads,
+      long keepAliveMillis) {
+    return new ThreadPoolExecutor(1, opThreads, keepAliveMillis, TimeUnit.MILLISECONDS,
         new SynchronousQueue<Runnable>(),
         new DaemonThreadFactory("(" + coordName + ")-proc-coordinator-pool"));
   }
@@ -194,7 +228,7 @@ public void abortProcedure(String procName, ForeignException reason) {
   Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
       List<String> expectedMembers) {
     // build the procedure
-    return new Procedure(this, fed, WAKE_MILLIS_DEFAULT, TIMEOUT_MILLIS_DEFAULT,
+    return new Procedure(this, fed, wakeTimeMillis, timeoutMillis,
         procName, procArgs, expectedMembers);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
index 876f5c4a7063..ffc38e95ef48 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
@@ -51,6 +51,8 @@
 public class ProcedureMember implements Closeable {
   private static final Log LOG = LogFactory.getLog(ProcedureMember.class);
 
+  final static long KEEP_ALIVE_MILLIS_DEFAULT = 5000;
+
   private final SubprocedureFactory builder;
   private final ProcedureMemberRpcs rpcs;
 
@@ -72,9 +74,26 @@ public ProcedureMember(ProcedureMemberRpcs rpcs, ThreadPoolExecutor pool,
     this.builder = factory;
   }
 
-  public static ThreadPoolExecutor defaultPool(long wakeFrequency, long keepAlive,
-      int procThreads, String memberName) {
-    return new ThreadPoolExecutor(1, procThreads, keepAlive, TimeUnit.SECONDS,
+  /**
+   * Default thread pool for the procedure
+   *
+   * @param memberName
+   * @param procThreads the maximum number of threads to allow in the pool
+   */
+  public static ThreadPoolExecutor defaultPool(String memberName, int procThreads) {
+    return defaultPool(memberName, procThreads, KEEP_ALIVE_MILLIS_DEFAULT);
+  }
+
+  /**
+   * Default thread pool for the procedure
+   *
+   * @param memberName
+   * @param procThreads the maximum number of threads to allow in the pool
+   * @param keepAliveMillis the maximum time (ms) that excess idle threads will wait for new tasks
+   */
+  public static ThreadPoolExecutor defaultPool(String memberName, int procThreads,
+      long keepAliveMillis) {
+    return new ThreadPoolExecutor(1, procThreads, keepAliveMillis, TimeUnit.MILLISECONDS,
         new SynchronousQueue<Runnable>(),
         new DaemonThreadFactory("member: '" + memberName + "' subprocedure-pool"));
   }
@@ -85,7 +104,7 @@ public static ThreadPoolExecutor defaultPool(long wakeFrequency, long keepAlive,
    * @return reference to the Procedure member's rpcs object
    */
   ProcedureMemberRpcs getRpcs() {
-     return rpcs;
+    return rpcs;
   }
 
 
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java b/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
index ea669a0520c1..43787aaa6303 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
@@ -198,7 +198,6 @@ final public Void call() {
       } else {
         msg = "Subprocedure '" + barrierName + "' failed!";
       }
-      LOG.error(msg , e);
       cancel(msg, e);
 
       LOG.debug("Subprocedure '" + barrierName + "' running cleanup.");
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
index 019bf971f974..60e23dfbcfb5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -124,13 +124,12 @@ public RegionServerSnapshotManager(RegionServerServices rss)
 
     // read in the snapshot request configuration properties
     Configuration conf = rss.getConfiguration();
-    long wakeMillis = conf.getLong(SNAPSHOT_REQUEST_WAKE_MILLIS_KEY, SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT);
     long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
     int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT);
 
     // create the actual snapshot procedure member
-    ThreadPoolExecutor pool = ProcedureMember.defaultPool(wakeMillis, keepAlive, opThreads, 
-      rss.getServerName().toString());
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(),
+      opThreads, keepAlive);
     this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder());
   }
 
@@ -192,7 +191,7 @@ public Subprocedure buildSubprocedure(SnapshotDescription snapshot) {
 
     LOG.debug("Launching subprocedure for snapshot " + snapshot.getName() + " from table "
         + snapshot.getTable());
-    ForeignExceptionDispatcher exnDispatcher = new ForeignExceptionDispatcher();
+    ForeignExceptionDispatcher exnDispatcher = new ForeignExceptionDispatcher(snapshot.getName());
     Configuration conf = rss.getConfiguration();
     long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY,
         SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
@@ -356,7 +355,6 @@ void cancelTasks() throws InterruptedException {
       }
 
       // evict remaining tasks and futures from taskPool.
-      LOG.debug(taskPool);
       while (!futures.isEmpty()) {
         // block to remove cancelled futures;
         LOG.warn("Removing cancelled elements from taskPool");
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
index f48aa23bd680..d0b62b29bc29 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
@@ -86,7 +86,7 @@ public void resetTest() throws IOException {
   }
 
   private ProcedureCoordinator buildNewCoordinator() {
-    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(nodeName, POOL_KEEP_ALIVE, 1, WAKE_FREQUENCY);
+    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(nodeName, 1, POOL_KEEP_ALIVE);
     return spy(new ProcedureCoordinator(controller, pool));
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
index 5af2e403f030..29ca89c23b74 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java
@@ -88,7 +88,7 @@ public void resetTest() {
    */
   private ProcedureMember buildCohortMember() {
     String name = "node";
-    ThreadPoolExecutor pool = ProcedureMember.defaultPool(WAKE_FREQUENCY, POOL_KEEP_ALIVE, 1, name);
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(name, 1, POOL_KEEP_ALIVE);
     return new ProcedureMember(mockMemberComms, pool, mockBuilder);
   }
 
@@ -98,7 +98,7 @@ private ProcedureMember buildCohortMember() {
   private void buildCohortMemberPair() throws IOException {
     dispatcher = new ForeignExceptionDispatcher();
     String name = "node";
-    ThreadPoolExecutor pool = ProcedureMember.defaultPool(WAKE_FREQUENCY, POOL_KEEP_ALIVE, 1, name);
+    ThreadPoolExecutor pool = ProcedureMember.defaultPool(name, 1, POOL_KEEP_ALIVE);
     member = new ProcedureMember(mockMemberComms, pool, mockBuilder);
     when(mockMemberComms.getMemberName()).thenReturn("membername"); // needed for generating exception
     Subprocedure subproc = new EmptySubprocedure(member, dispatcher);
diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
index fe555ad20ed1..36e0855c0fe8 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
@@ -127,7 +127,7 @@ private void runCommit(String... members) throws Exception {
     // start running the controller
     ZKProcedureCoordinatorRpcs coordinatorComms = new ZKProcedureCoordinatorRpcs(
         coordZkw, opDescription, COORDINATOR_NODE_NAME);
-    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, KEEP_ALIVE, POOL_SIZE, WAKE_FREQUENCY);
+    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, POOL_SIZE, KEEP_ALIVE);
     ProcedureCoordinator coordinator = new ProcedureCoordinator(coordinatorComms, pool) {
       @Override
       public Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
@@ -145,7 +145,7 @@ public Procedure createProcedure(ForeignExceptionDispatcher fed, String procName
     for (String member : members) {
       ZooKeeperWatcher watcher = newZooKeeperWatcher();
       ZKProcedureMemberRpcs comms = new ZKProcedureMemberRpcs(watcher, opDescription);
-      ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member);
+      ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(member, 1, KEEP_ALIVE);
       ProcedureMember procMember = new ProcedureMember(comms, pool2, subprocFactory);
       procMembers.add(new Pair<ProcedureMember, ZKProcedureMemberRpcs>(procMember, comms));
       comms.start(member, procMember);
@@ -209,7 +209,7 @@ public void testMultiCohortWithMemberTimeoutDuringPrepare() throws Exception {
     ZooKeeperWatcher coordinatorWatcher = newZooKeeperWatcher();
     ZKProcedureCoordinatorRpcs coordinatorController = new ZKProcedureCoordinatorRpcs(
         coordinatorWatcher, opDescription, COORDINATOR_NODE_NAME);
-    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, KEEP_ALIVE, POOL_SIZE, WAKE_FREQUENCY);
+    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, POOL_SIZE, KEEP_ALIVE);
     ProcedureCoordinator coordinator = spy(new ProcedureCoordinator(coordinatorController, pool));
 
     // start a member for each node
@@ -219,7 +219,7 @@ public void testMultiCohortWithMemberTimeoutDuringPrepare() throws Exception {
     for (String member : expected) {
       ZooKeeperWatcher watcher = newZooKeeperWatcher();
       ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(watcher, opDescription);
-      ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member);
+      ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(member, 1, KEEP_ALIVE);
       ProcedureMember mem = new ProcedureMember(controller, pool2, subprocFactory);
       members.add(new Pair<ProcedureMember, ZKProcedureMemberRpcs>(mem, controller));
       controller.start(member, mem);

From fb93645d8d947edca4702a041efd6d68f706e8b1 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 2 Aug 2013 04:22:07 +0000
Subject: [PATCH 1091/1540] HBASE-8670 [0.94] Backport HBASE-8449,HBASE-8204
 and HBASE-8699 to 0.94 (Refactor recoverLease retries and pauses)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509532 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/FSHDFSUtils.java | 233 ++++++++++++------
 src/main/resources/hbase-default.xml          |  17 ++
 .../hbase/regionserver/wal/TestHLog.java      |  46 ++--
 .../hadoop/hbase/util/TestFSHDFSUtils.java    | 105 ++++++++
 4 files changed, 295 insertions(+), 106 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index 0f1ea148a70d..d7a5f7297963 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -21,17 +21,14 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
 
 
@@ -42,99 +39,175 @@ public class FSHDFSUtils extends FSUtils{
   private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
 
   /**
-   * Lease timeout constant, sourced from HDFS upstream.
-   * The upstream constant is defined in a private interface, so we
-   * can't reuse for compatibility reasons.
-   * NOTE: On versions earlier than Hadoop 0.23, the constant is in
-   * o.a.h.hdfs.protocol.FSConstants, while for 0.23 and above it is
-   * in o.a.h.hdfs.protocol.HdfsConstants cause of HDFS-1620.
+   * Recover the lease from HDFS, retrying multiple times.
    */
-  public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
-
-  public static final String TEST_TRIGGER_DFS_APPEND = "hbase.test.trigger.dfs.append";
-
   @Override
-  public void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
-  throws IOException{
+  public void recoverFileLease(final FileSystem fs, final Path p,
+      Configuration conf)
+  throws IOException {
     if (!isAppendSupported(conf)) {
       LOG.warn("Running on HDFS without append enabled may result in data loss");
       return;
     }
     // lease recovery not needed for local file system case.
-    // currently, local file system doesn't implement append either.
-    if (!(fs instanceof DistributedFileSystem)) {
-      return;
-    }
-    LOG.info("Recovering file " + p);
-    long startWaiting = System.currentTimeMillis();
+    if (!(fs instanceof DistributedFileSystem)) return;
+    recoverDFSFileLease((DistributedFileSystem)fs, p, conf);
+  }
 
-    // Trying recovery
+  /*
+   * Run the dfs recover lease. recoverLease is asynchronous. It returns:
+   *    -false when it starts the lease recovery (i.e. lease recovery not *yet* done)
+   *    - true when the lease recovery has succeeded or the file is closed.
+   * But, we have to be careful.  Each time we call recoverLease, it starts the recover lease
+   * process over from the beginning.  We could put ourselves in a situation where we are
+   * doing nothing but starting a recovery, interrupting it to start again, and so on.
+   * The findings over in HBASE-8354 have it that the namenode will try to recover the lease
+   * on the file's primary node.  If all is well, it should return near immediately.  But,
+   * as is common, it is the very primary node that has crashed and so the namenode will be
+   * stuck waiting on a socket timeout before it will ask another datanode to start the
+   * recovery. It does not help if we call recoverLease in the meantime and in particular,
+   * subsequent to the socket timeout, a recoverLease invocation will cause us to start
+   * over from square one (possibly waiting on socket timeout against primary node).  So,
+   * in the below, we do the following:
+   * 1. Call recoverLease.
+   * 2. If it returns true, break.
+   * 3. If it returns false, wait a few seconds and then call it again.
+   * 4. If it returns true, break.
+   * 5. If it returns false, wait for what we think the datanode socket timeout is
+   * (configurable) and then try again.
+   * 6. If it returns true, break.
+   * 7. If it returns false, repeat starting at step 5. above.
+   *
+   * If HDFS-4525 is available, call it every second and we might be able to exit early.
+   */
+  boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
+      final Configuration conf)
+  throws IOException {
+    LOG.info("Recovering lease on dfs file " + p);
+    long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
+    // Default is 15 minutes. It's huge, but the idea is that if we have a major issue, HDFS
+    // usually needs 10 minutes before marking the nodes as dead. So we're putting ourselves
+    // beyond that limit 'to be safe'.
+    long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
+    // This setting should be what the cluster dfs heartbeat is set to.
+    long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 3000);
+    // This should be set to how long it'll take for us to timeout against primary datanode if it
+    // is dead.  We set it to 61 seconds, 1 second than the default READ_TIMEOUT in HDFS, the
+    // default value for DFS_CLIENT_SOCKET_TIMEOUT_KEY.
+    long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
+
+    Method isFileClosedMeth = null;
+    // whether we need to look for isFileClosed method
+    boolean findIsFileClosedMeth = true;
     boolean recovered = false;
-    long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 300000);
-    // conf parameter passed from unit test, indicating whether fs.append() should be triggered
-    boolean triggerAppend = conf.getBoolean(TEST_TRIGGER_DFS_APPEND, false);
-    // retrying lease recovery may preempt pending lease recovery; default to waiting for 4 seconds
-    // after calling recoverLease
-    int waitingPeriod = conf.getInt("hbase.lease.recovery.waiting.period", 4000);
-    Exception ex = null;
-    while (!recovered) {
+    // We break the loop if we succeed the lease recovery, timeout, or we throw an exception.
+    for (int nbAttempt = 0; !recovered; nbAttempt++) {
+      recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
+      if (recovered) break;
+      if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) break;
       try {
-        try {
-          DistributedFileSystem dfs = (DistributedFileSystem) fs;
-          if (triggerAppend) throw new IOException();
-          try {
-            recovered = (Boolean) DistributedFileSystem.class.getMethod(
-              "recoverLease", new Class[] { Path.class }).invoke(dfs, p);
-            if (!recovered) LOG.debug("recoverLease returned false");
-          } catch (InvocationTargetException ite) {
-            // function was properly called, but threw it's own exception
-            throw (IOException) ite.getCause();
-          }
-        } catch (Exception e) {
-          LOG.debug("Failed fs.recoverLease invocation, " + e.toString() +
-            ", trying fs.append instead");
-          ex = e;
-        }
-        if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) {
-          LOG.debug("trying fs.append for " + p + " with " + ex);
-          ex = null; // assume the following append() call would succeed
-          FSDataOutputStream out = fs.append(p);
-          out.close();
-          recovered = true;
-          LOG.debug("fs.append passed");
-        }
-        if (recovered) break;
-      } catch (IOException e) {
-        e = RemoteExceptionHandler.checkIOException(e);
-        if (e instanceof AlreadyBeingCreatedException) {
-          // We expect that we'll get this message while the lease is still
-          // within its soft limit, but if we get it past that, it means
-          // that the RS is holding onto the file even though it lost its
-          // znode. We could potentially abort after some time here.
-          long waitedFor = System.currentTimeMillis() - startWaiting;
-          if (waitedFor > LEASE_SOFTLIMIT_PERIOD) {
-            LOG.warn("Waited " + waitedFor + "ms for lease recovery on " + p +
-              ":" + e.getMessage());
-          }
-        } else if (e instanceof LeaseExpiredException &&
-            e.getMessage().contains("File does not exist")) {
-          // This exception comes out instead of FNFE, fix it
-          throw new FileNotFoundException(
-              "The given HLog wasn't found at " + p.toString());
+        // On the first time through wait the short 'firstPause'.
+        if (nbAttempt == 0) {
+          Thread.sleep(firstPause);
         } else {
-          throw new IOException("Failed to open " + p + " for append", e);
+          // Cycle here until subsequentPause elapses.  While spinning, check isFileClosed if
+          // available (should be in hadoop 2.0.5... not in hadoop 1 though.
+          long localStartWaiting = EnvironmentEdgeManager.currentTimeMillis();
+          while ((EnvironmentEdgeManager.currentTimeMillis() - localStartWaiting) <
+              subsequentPause) {
+            Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
+            if (findIsFileClosedMeth) {
+               try {
+                 isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
+                   new Class[]{ Path.class });
+               } catch (NoSuchMethodException nsme) {
+                 LOG.debug("isFileClosed not available");
+               } finally {
+                 findIsFileClosedMeth = false;
+               }
+             }
+             if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
+               recovered = true;
+               break;
+             }
+          }
         }
-      }
-      try {
-        Thread.sleep(waitingPeriod);
       } catch (InterruptedException ie) {
         InterruptedIOException iioe = new InterruptedIOException();
         iioe.initCause(ie);
         throw iioe;
       }
-      // we keep original behavior without retrying lease recovery
-      break;
     }
-    LOG.info("Finished lease recovery attempt for " + p);
+    return recovered;
+  }
+
+  boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
+      final int nbAttempt, final Path p, final long startWaiting) {
+    if (recoveryTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
+      LOG.warn("Cannot recoverLease after trying for " +
+        conf.getInt("hbase.lease.recovery.timeout", 900000) +
+        "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
+        getLogMessageDetail(nbAttempt, p, startWaiting));
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Try to recover the lease.
+   * @param dfs
+   * @param nbAttempt
+   * @param p
+   * @param startWaiting
+   * @return True if dfs#recoverLease came by true.
+   * @throws FileNotFoundException
+   */
+  boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
+      final long startWaiting)
+  throws FileNotFoundException {
+    boolean recovered = false;
+    try {
+      recovered = dfs.recoverLease(p);
+      LOG.info("recoverLease=" + recovered + ", " +
+        getLogMessageDetail(nbAttempt, p, startWaiting));
+    } catch (IOException e) {
+      if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
+        // This exception comes out instead of FNFE, fix it
+        throw new FileNotFoundException("The given HLog wasn't found at " + p);
+      } else if (e instanceof FileNotFoundException) {
+        throw (FileNotFoundException)e;
+      }
+      LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
+    }
+    return recovered;
+  }
+
+  /**
+   * @param nbAttempt
+   * @param p
+   * @param startWaiting
+   * @return Detail to append to any log message around lease recovering.
+   */
+  private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
+    return "attempt=" + nbAttempt + " on file=" + p + " after " +
+      (EnvironmentEdgeManager.currentTimeMillis() - startWaiting) + "ms";
+  }
+
+  /**
+   * Call HDFS-4525 isFileClosed if it is available.
+   * @param dfs
+   * @param m
+   * @param p
+   * @return True if file is closed.
+   */
+  private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
+    try {
+      return (Boolean) m.invoke(dfs, p);
+    } catch (SecurityException e) {
+      LOG.warn("No access", e);
+    } catch (Exception e) {
+      LOG.warn("Failed invocation for " + p.toString(), e);
+    }
+    return false;
   }
 }
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 365e58726f7d..709bcb8f2663 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -973,4 +973,21 @@
         is 2.
     </description>
   </property>
+  <property>
+    <name>hbase.lease.recovery.timeout</name>
+    <value>900000</value>
+    <description>
+      How long we wait on dfs lease recovery in total before giving up.
+    </description>
+  </property>
+  <property>
+    <name>hbase.lease.recovery.dfs.timeout</name>
+    <value>61000</value>
+    <description>
+        How long between dfs recover lease invocations. Should be just larger than how long
+        it takes the namenode to timeout trying to reach a datanode; usually
+        dfs.socket.timeout.  If HBase asked hdfs its cluster configs, we would not need
+        this config.  See the end of HBASE-8389 for more.
+    </description>
+  </property>
 </configuration>
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index c9bb3b247d2c..6077078986ce 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -20,8 +20,8 @@
 package org.apache.hadoop.hbase.regionserver.wal;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
@@ -39,21 +39,25 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver;
+import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -198,7 +202,7 @@ public void testSplit() throws IOException {
       }
       log.close();
       HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf,
-          hbaseDir, logdir, this.oldLogDir, this.fs);
+          hbaseDir, logdir, oldLogDir, fs);
       List<Path> splits =
         logSplitter.splitLog();
       verifySplits(splits, howmany);
@@ -371,10 +375,10 @@ private void verifySplits(List<Path> splits, final int howmany)
       }
     }
   }
-  
+
   /*
    * We pass different values to recoverFileLease() so that different code paths are covered
-   * 
+   *
    * For this test to pass, requires:
    * 1. HDFS-200 (append support)
    * 2. HDFS-988 (SafeMode should freeze file operations
@@ -383,18 +387,10 @@ private void verifySplits(List<Path> splits, final int howmany)
    */
   @Test
   public void testAppendClose() throws Exception {
-    testAppendClose(true);
-    testAppendClose(false);
-  }
-
-  /*
-   * @param triggerDirectAppend whether to trigger direct call of fs.append()
-   */
-  public void testAppendClose(final boolean triggerDirectAppend) throws Exception {
     byte [] tableName = Bytes.toBytes(getName());
     HRegionInfo regioninfo = new HRegionInfo(tableName,
              HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
-    Path subdir = new Path(dir, "hlogdir" + triggerDirectAppend);
+    Path subdir = new Path(dir, "hlogdir");
     Path archdir = new Path(dir, "hlogdir_archive");
     HLog wal = new HLog(fs, subdir, archdir, conf);
     final int total = 20;
@@ -411,7 +407,7 @@ public void testAppendClose(final boolean triggerDirectAppend) throws Exception
     wal.sync();
      int namenodePort = cluster.getNameNodePort();
     final Path walPath = wal.computeFilename();
-    
+
 
     // Stop the cluster.  (ensure restart since we're sharing MiniDFSCluster)
     try {
@@ -452,23 +448,21 @@ public void testAppendClose(final boolean triggerDirectAppend) throws Exception
     Method setLeasePeriod = cluster.getClass()
       .getDeclaredMethod("setLeasePeriod", new Class[]{Long.TYPE, Long.TYPE});
     setLeasePeriod.setAccessible(true);
-    setLeasePeriod.invoke(cluster,
-                          new Object[]{new Long(1000), new Long(1000)});
+    setLeasePeriod.invoke(cluster, 1000L, 1000L);
     try {
       Thread.sleep(1000);
     } catch (InterruptedException e) {
       LOG.info(e);
     }
-    
+
     // Now try recovering the log, like the HMaster would do
     final FileSystem recoveredFs = fs;
     final Configuration rlConf = conf;
-    
+
     class RecoverLogThread extends Thread {
       public Exception exception = null;
       public void run() {
           try {
-            rlConf.setBoolean(FSHDFSUtils.TEST_TRIGGER_DFS_APPEND, triggerDirectAppend);
             FSUtils.getInstance(fs, rlConf)
               .recoverFileLease(recoveredFs, walPath, rlConf);
           } catch (IOException e) {
@@ -805,7 +799,7 @@ public void postLogArchive(Path oldFile, Path newFile) {
     @Override
     public void logRollRequested() {
       // TODO Auto-generated method stub
-      
+
     }
 
     @Override
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
new file mode 100644
index 000000000000..4c7bc97a3940
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test our recoverLease loop against mocked up filesystem.
+ */
+@Category(MediumTests.class)
+public class TestFSHDFSUtils {
+  private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
+  static {
+    Configuration conf = HTU.getConfiguration();
+    conf.setInt("hbase.lease.recovery.dfs.timeout", 1000);
+    conf.setInt("hbase.lease.recovery.first.pause", 10);
+    conf.setInt("hbase.lease.recovery.pause", 10);
+  };
+  private FSHDFSUtils fsHDFSUtils = new FSHDFSUtils();
+  private static Path FILE = new Path(HTU.getDataTestDir(), "file.txt");
+  long startTime = -1;
+
+  @Before
+  public void setup() {
+    this.startTime = EnvironmentEdgeManager.currentTimeMillis();
+  }
+
+  /**
+   * Test recover lease eventually succeeding.
+   * @throws IOException
+   */
+  @Test (timeout = 30000)
+  public void testRecoverLease() throws IOException {
+    DistributedFileSystem dfs = Mockito.mock(DistributedFileSystem.class);
+    // Fail four times and pass on the fifth.
+    Mockito.when(dfs.recoverLease(FILE)).
+      thenReturn(false).thenReturn(false).thenReturn(false).thenReturn(false).thenReturn(true);
+    assertTrue(this.fsHDFSUtils.recoverDFSFileLease(dfs, FILE, HTU.getConfiguration()));
+    Mockito.verify(dfs, Mockito.times(5)).recoverLease(FILE);
+    // Make sure we waited at least hbase.lease.recovery.dfs.timeout * 3 (the first two
+    // invocations will happen pretty fast... the we fall into the longer wait loop).
+    assertTrue((EnvironmentEdgeManager.currentTimeMillis() - this.startTime) >
+      (3 * HTU.getConfiguration().getInt("hbase.lease.recovery.dfs.timeout", 61000)));
+  }
+
+  /**
+   * Test that isFileClosed makes us recover lease faster.
+   * @throws IOException
+   */
+  @Test (timeout = 30000)
+  public void testIsFileClosed() throws IOException {
+    // Make this time long so it is plain we broke out because of the isFileClosed invocation.
+    HTU.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 100000);
+    IsFileClosedDistributedFileSystem dfs = Mockito.mock(IsFileClosedDistributedFileSystem.class);
+    // Now make it so we fail the first two times -- the two fast invocations, then we fall into
+    // the long loop during which we will call isFileClosed.... the next invocation should
+    // therefore return true if we are to break the loop.
+    Mockito.when(dfs.recoverLease(FILE)).
+      thenReturn(false).thenReturn(false).thenReturn(true);
+    Mockito.when(dfs.isFileClosed(FILE)).thenReturn(true);
+    assertTrue(this.fsHDFSUtils.recoverDFSFileLease(dfs, FILE, HTU.getConfiguration()));
+    Mockito.verify(dfs, Mockito.times(2)).recoverLease(FILE);
+    Mockito.verify(dfs, Mockito.times(1)).isFileClosed(FILE);
+  }
+
+  /**
+   * Version of DFS that has HDFS-4525 in it.
+   */
+  class IsFileClosedDistributedFileSystem extends DistributedFileSystem {
+    /**
+     * Close status of a file. Copied over from HDFS-4525
+     * @return true if file is already closed
+     **/
+    public boolean isFileClosed(Path f) throws IOException{
+      return false;
+    }
+  }
+}
\ No newline at end of file

From 566fa53fc8972785daf3950b517eee040da8a3c2 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 2 Aug 2013 04:28:47 +0000
Subject: [PATCH 1092/1540] HBASE-9107 [0.94] Backport HBASE-6950
 TestAcidGuarantees system test now flushes too aggressively to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509533 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/TestAcidGuarantees.java      | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
index 227d24387452..041bb8795bd8 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
@@ -240,12 +240,20 @@ private void gotFailure(byte[] expected, Result res) {
     }
   }
 
-
   public void runTestAtomicity(long millisToRun,
       int numWriters,
       int numGetters,
       int numScanners,
       int numUniqueRows) throws Exception {
+    runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false);
+  }
+
+  public void runTestAtomicity(long millisToRun,
+      int numWriters,
+      int numGetters,
+      int numScanners,
+      int numUniqueRows,
+      final boolean systemTest) throws Exception {
     createTableIfMissing();
     TestContext ctx = new TestContext(util.getConfiguration());
     
@@ -266,6 +274,15 @@ public void runTestAtomicity(long millisToRun,
       HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
       public void doAnAction() throws Exception {
         admin.flush(TABLE_NAME);
+        // Flushing has been a source of ACID violations previously (see HBASE-2856), so ideally,
+        // we would flush as often as possible.  On a running cluster, this isn't practical:
+        // (1) we will cause a lot of load due to all the flushing and compacting
+        // (2) we cannot change the flushing/compacting related Configuration options to try to
+        // alleviate this
+        // (3) it is an unrealistic workload, since no one would actually flush that often.
+        // Therefore, let's flush every minute to have more flushes than usual, but not overload
+        // the running cluster.
+        if (systemTest) Thread.sleep(60000);
       }
     });
 
@@ -355,7 +372,7 @@ public int run(String[] arg0) throws Exception {
     int numGetters = c.getInt("numGetters", 2);
     int numScanners = c.getInt("numScanners", 2);
     int numUniqueRows = c.getInt("numUniqueRows", 3);
-    runTestAtomicity(millis, numWriters, numGetters, numScanners, numUniqueRows);
+    runTestAtomicity(millis, numWriters, numGetters, numScanners, numUniqueRows, true);
     return 0;
   }
 

From d300b5e4a717a4002e7117b1c850bb717174fc0e Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 2 Aug 2013 04:33:41 +0000
Subject: [PATCH 1093/1540] HBASE-9106 Do not fail TestAcidGuarantees for
 exceptions on table flush

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509534 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/TestAcidGuarantees.java      | 41 +++++++++++--------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
index 041bb8795bd8..ace876a9e61d 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
@@ -48,7 +49,7 @@
 /**
  * Test case that uses multiple threads to read and write multifamily rows
  * into a table, verifying that reads never see partially-complete writes.
- * 
+ *
  * This can run as a junit test, or with a main() function which runs against
  * a real cluster (eg for testing with failures, region movement, etc)
  */
@@ -88,7 +89,7 @@ public TestAcidGuarantees() {
       ConstantSizeRegionSplitPolicy.class.getName());
     util = new HBaseTestingUtility(conf);
   }
-  
+
   /**
    * Thread that does random full-row writes into a table.
    */
@@ -99,7 +100,7 @@ public static class AtomicityWriter extends RepeatingTestThread {
     byte targetFamilies[][];
     HTable table;
     AtomicLong numWritten = new AtomicLong();
-    
+
     public AtomicityWriter(TestContext ctx, byte targetRows[][],
                            byte targetFamilies[][]) throws IOException {
       super(ctx);
@@ -110,7 +111,7 @@ public AtomicityWriter(TestContext ctx, byte targetRows[][],
     public void doAnAction() throws Exception {
       // Pick a random row to write into
       byte[] targetRow = targetRows[rand.nextInt(targetRows.length)];
-      Put p = new Put(targetRow); 
+      Put p = new Put(targetRow);
       rand.nextBytes(data);
 
       for (byte[] family : targetFamilies) {
@@ -123,7 +124,7 @@ public void doAnAction() throws Exception {
       numWritten.getAndIncrement();
     }
   }
-  
+
   /**
    * Thread that does single-row reads in a table, looking for partially
    * completed rows.
@@ -153,7 +154,7 @@ public void doAnAction() throws Exception {
         // ignore this action
         return;
       }
-      
+
       for (byte[] family : targetFamilies) {
         for (int i = 0; i < NUM_COLS_TO_CHECK; i++) {
           byte qualifier[] = Bytes.toBytes("col" + i);
@@ -182,7 +183,7 @@ private void gotFailure(byte[] expected, Result res) {
       throw new RuntimeException(msg.toString());
     }
   }
-  
+
   /**
    * Thread that does full scans of the table looking for any partially completed
    * rows.
@@ -206,10 +207,10 @@ public void doAnAction() throws Exception {
         s.addFamily(family);
       }
       ResultScanner scanner = table.getScanner(s);
-      
+
       for (Result res : scanner) {
         byte[] gotValue = null;
-  
+
         for (byte[] family : targetFamilies) {
           for (int i = 0; i < NUM_COLS_TO_CHECK; i++) {
             byte qualifier[] = Bytes.toBytes("col" + i);
@@ -256,12 +257,12 @@ public void runTestAtomicity(long millisToRun,
       final boolean systemTest) throws Exception {
     createTableIfMissing();
     TestContext ctx = new TestContext(util.getConfiguration());
-    
+
     byte rows[][] = new byte[numUniqueRows][];
     for (int i = 0; i < numUniqueRows; i++) {
       rows[i] = Bytes.toBytes("test_row_" + i);
     }
-    
+
     List<AtomicityWriter> writers = Lists.newArrayList();
     for (int i = 0; i < numWriters; i++) {
       AtomicityWriter writer = new AtomicityWriter(
@@ -273,7 +274,11 @@ public void runTestAtomicity(long millisToRun,
     ctx.addThread(new RepeatingTestThread(ctx) {
       HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
       public void doAnAction() throws Exception {
-        admin.flush(TABLE_NAME);
+        try {
+          admin.flush(TABLE_NAME);
+        } catch(IOException ioe) {
+          LOG.warn("Ignoring exception while flushing: " + StringUtils.stringifyException(ioe));
+        }
         // Flushing has been a source of ACID violations previously (see HBASE-2856), so ideally,
         // we would flush as often as possible.  On a running cluster, this isn't practical:
         // (1) we will cause a lot of load due to all the flushing and compacting
@@ -293,18 +298,18 @@ public void doAnAction() throws Exception {
       getters.add(getter);
       ctx.addThread(getter);
     }
-    
+
     List<AtomicScanReader> scanners = Lists.newArrayList();
     for (int i = 0; i < numScanners; i++) {
       AtomicScanReader scanner = new AtomicScanReader(ctx, FAMILIES);
       scanners.add(scanner);
       ctx.addThread(scanner);
     }
-    
+
     ctx.startThreads();
     ctx.waitFor(millisToRun);
     ctx.stop();
-    
+
     LOG.info("Finished test. Writers:");
     for (AtomicityWriter writer : writers) {
       LOG.info("  wrote " + writer.numWritten.get());
@@ -327,7 +332,7 @@ public void testGetAtomicity() throws Exception {
       runTestAtomicity(20000, 5, 5, 0, 3);
     } finally {
       util.shutdownMiniCluster();
-    }    
+    }
   }
 
   @Test
@@ -337,7 +342,7 @@ public void testScanAtomicity() throws Exception {
       runTestAtomicity(20000, 5, 0, 5, 3);
     } finally {
       util.shutdownMiniCluster();
-    }    
+    }
   }
 
   @Test
@@ -347,7 +352,7 @@ public void testMixedAtomicity() throws Exception {
       runTestAtomicity(20000, 5, 2, 2, 3);
     } finally {
       util.shutdownMiniCluster();
-    }    
+    }
   }
 
   ////////////////////////////////////////////////////////////////////////////

From cea837fa8fb2931e27defa44ea676f6c8154a2fb Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 2 Aug 2013 04:43:36 +0000
Subject: [PATCH 1094/1540] HBASE-9085 Integration Tests fails because of bug
 in teardown phase where the cluster state is not being restored properly.
 (gautam)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509540 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/DistributedHBaseCluster.java | 29 +++++++++++++------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index b709613b8669..7aa53803a936 100644
--- a/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -203,25 +203,25 @@ public void restoreClusterStatus(ClusterStatus initial) throws IOException {
 
     //check whether current master has changed
     if (!ServerName.isSameHostnameAndPort(initial.getMaster(), current.getMaster())) {
+      LOG.info("Initial active master : " + initial.getMaster().getHostname()
+          + " has changed to : " + current.getMaster().getHostname());
+      // If initial master is stopped, start it, before restoring the state.
+      // It will come up as a backup master, if there is already an active master.
+      if (!clusterManager.isRunning(ServiceType.HBASE_MASTER, initial.getMaster().getHostname())) {
+        startMaster(initial.getMaster().getHostname());
+      }
+
       //master has changed, we would like to undo this.
       //1. Kill the current backups
       //2. Stop current master
-      //3. Start a master at the initial hostname (if not already running as backup)
-      //4. Start backup masters
-      boolean foundOldMaster = false;
+      //3. Start backup masters
       for (ServerName currentBackup : current.getBackupMasters()) {
         if (!ServerName.isSameHostnameAndPort(currentBackup, initial.getMaster())) {
           stopMaster(currentBackup);
-        } else {
-          foundOldMaster = true;
         }
       }
       stopMaster(current.getMaster());
-      if (foundOldMaster) { //if initial master is not running as a backup
-        startMaster(initial.getMaster().getHostname());
-      }
       waitForActiveAndReadyMaster(); //wait so that active master takes over
-
       //start backup masters
       for (ServerName backup : initial.getBackupMasters()) {
         //these are not started in backup mode, but we should already have an active master
@@ -276,5 +276,16 @@ public void restoreClusterStatus(ClusterStatus initial) throws IOException {
         stopRegionServer(currentServers.get(hostname));
       }
     }
+    // While restoring above, if the HBase Master which was initially the Active one, was down
+    // and the restore put the cluster back to Initial configuration, HAdmin instance will need
+    // to refresh its connections (otherwise it will return incorrect information) or we can
+    // point it to new instance.
+    try {
+      admin.close();
+    } catch (IOException ioe) {
+      LOG.info("While closing the old connection", ioe);
+    }
+    this.admin = new HBaseAdmin(conf);
+    LOG.info("Added new HBaseAdmin");
   }
 }

From c5d91288dcf72abcf6030c77657aa1108c005698 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 2 Aug 2013 07:28:03 +0000
Subject: [PATCH 1095/1540] HBASE-9075 [0.94] Backport HBASE-5760 Unit tests
 should write only under /target to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509572 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     |  9 +++-
 .../coprocessor/TestCoprocessorInterface.java | 14 +++----
 .../TestRegionObserverInterface.java          | 17 ++++----
 .../TestRegionObserverStacking.java           | 20 +++++----
 .../mapreduce/TestHFileOutputFormat.java      |  7 +++-
 .../hbase/regionserver/TestHRegionInfo.java   | 42 ++++---------------
 6 files changed, 48 insertions(+), 61 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 9332e89085d7..868e8cf64642 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1420,6 +1420,9 @@ public void startMiniMapReduceCluster() throws IOException {
    */
   public void startMiniMapReduceCluster(final int servers) throws IOException {
     LOG.info("Starting mini mapreduce cluster...");
+    if (dataTestDir == null) {
+      setupDataTestDir();
+    }
     // These are needed for the new and improved Map/Reduce framework
     Configuration c = getConfiguration();
     String logDir = c.get("hadoop.log.dir");
@@ -1434,8 +1437,10 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
     // we up the VM usable so that processes don't get killed.
     conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
 
-    mrCluster = new MiniMRCluster(servers,
-      FileSystem.get(conf).getUri().toString(), 1);
+    mrCluster = new MiniMRCluster(0, 0, servers,
+      FileSystem.get(conf).getUri().toString(), 1, null, null, null, new JobConf(conf));
+    mrCluster.getJobTrackerRunner().getJobTracker().getConf().set("mapred.local.dir",
+      conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
     LOG.info("Mini mapreduce cluster started");
     JobConf mrClusterJobConf = mrCluster.createJobConf();
     c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 88becfc89008..c03be922d297 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -20,6 +20,8 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import static org.mockito.Mockito.when;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -45,14 +47,12 @@
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
-import static org.mockito.Mockito.when;
-
 @Category(SmallTests.class)
 public class TestCoprocessorInterface extends HBaseTestCase {
   static final Log LOG = LogFactory.getLog(TestCoprocessorInterface.class);
-  static final String DIR = "test/build/data/TestCoprocessorInterface/";
   private static final HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
+  static final Path DIR = TEST_UTIL.getDataTestDir();
 
   private static class CustomScanner implements RegionScanner {
 
@@ -66,9 +66,9 @@ public CustomScanner(RegionScanner delegate) {
     public boolean next(List<KeyValue> results) throws IOException {
       return delegate.next(results);
     }
-    
+
     @Override
-    public boolean next(List<KeyValue> results, String metric) 
+    public boolean next(List<KeyValue> results, String metric)
         throws IOException {
       return delegate.next(results, metric);
     }
@@ -77,9 +77,9 @@ public boolean next(List<KeyValue> results, String metric)
     public boolean next(List<KeyValue> result, int limit) throws IOException {
       return delegate.next(result, limit);
     }
-    
+
     @Override
-    public boolean next(List<KeyValue> result, int limit, String metric) 
+    public boolean next(List<KeyValue> result, int limit, String metric)
         throws IOException {
       return delegate.next(result, limit, metric);
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index bba0365b64f8..9d8f0fce2176 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -20,6 +20,11 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -44,18 +49,14 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
-
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import static org.junit.Assert.*;
-
 @Category(MediumTests.class)
 public class TestRegionObserverInterface {
   static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
-  static final String DIR = "test/build/data/TestRegionObserver/";
 
   public static final byte[] TEST_TABLE = Bytes.toBytes("TestTable");
   public final static byte[] A = Bytes.toBytes("a");
@@ -304,21 +305,21 @@ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment>
         public boolean next(List<KeyValue> results) throws IOException {
           return next(results, -1);
         }
-        
+
         @Override
-        public boolean next(List<KeyValue> results, String metric) 
+        public boolean next(List<KeyValue> results, String metric)
             throws IOException {
           return next(results, -1, metric);
         }
 
         @Override
-        public boolean next(List<KeyValue> results, int limit) 
+        public boolean next(List<KeyValue> results, int limit)
             throws IOException{
           return next(results, limit, null);
         }
 
         @Override
-        public boolean next(List<KeyValue> results, int limit, String metric) 
+        public boolean next(List<KeyValue> results, int limit, String metric)
             throws IOException {
           List<KeyValue> internalResults = new ArrayList<KeyValue>();
           boolean hasMore;
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
index 002d611a4394..ffa5de4a8697 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
@@ -21,24 +21,30 @@
 package org.apache.hadoop.hbase.coprocessor;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Map;
+
+import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
-
-import junit.framework.TestCase;
 import org.junit.experimental.categories.Category;
 
 @Category(SmallTests.class)
 public class TestRegionObserverStacking extends TestCase {
-  static final String DIR = "test/build/data/TestRegionObserverStacking/";
+  private static HBaseTestingUtility TEST_UTIL
+    = new HBaseTestingUtility();
+  static final Path DIR = TEST_UTIL.getDataTestDir();
 
   public static class ObserverA extends BaseRegionObserver {
     long id;
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 5bf8bbd73af7..2bf0c1b21ef4 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -275,7 +275,7 @@ public void test_TIMERANGE() throws Exception {
       // verify that the file has the proper FileInfo.
       writer.close(context);
 
-      // the generated file lives 1 directory down from the attempt directory 
+      // the generated file lives 1 directory down from the attempt directory
       // and is the only file, e.g.
       // _attempt__0000_r_000000_0/b/1979617994050536795
       FileSystem fs = FileSystem.get(conf);
@@ -344,7 +344,8 @@ public void testWritingPEData() throws Exception {
 
   @Test
   public void testJobConfiguration() throws Exception {
-    Job job = new Job();
+    Job job = new Job(util.getConfiguration());
+    job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration"));
     HTable table = Mockito.mock(HTable.class);
     setupMockStartKeys(table);
     HFileOutputFormat.configureIncrementalLoad(job, table);
@@ -469,6 +470,7 @@ private void runIncrementalPELoad(
       Configuration conf, HTable table, Path outDir)
   throws Exception {
     Job job = new Job(conf, "testLocalMRIncrementalLoad");
+    job.setWorkingDirectory(util.getDataTestDir("runIncrementalPELoad"));
     setupRandomGeneratorMapper(job);
     HFileOutputFormat.configureIncrementalLoad(job, table);
     FileOutputFormat.setOutputPath(job, outDir);
@@ -583,6 +585,7 @@ public void testColumnFamilySettings() throws Exception {
       // pollutes the GZip codec pool with an incompatible compressor.
       conf.set("io.seqfile.compression.type", "NONE");
       Job job = new Job(conf, "testLocalMRIncrementalLoad");
+      job.setWorkingDirectory(util.getDataTestDir("testColumnFamilyCompression"));
       setupRandomGeneratorMapper(job);
       HFileOutputFormat.configureIncrementalLoad(job, table);
       FileOutputFormat.setOutputPath(job, dir);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
index 6dfba41ca292..17538750e717 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
@@ -21,16 +21,13 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -60,32 +57,7 @@ public void testCreateHRegionInfoName() throws Exception {
                  + id + "." + md5HashInHex + ".",
                  nameStr);
   }
-  
-  @Test
-  public void testGetSetOfHTD() throws IOException {
-    HBaseTestingUtility HTU = new HBaseTestingUtility();
-        final String tablename = "testGetSetOfHTD";
-
-    // Delete the temporary table directory that might still be there from the
-    // previous test run.
-    FSTableDescriptors.deleteTableDescriptorIfExists(tablename,
-        HTU.getConfiguration());
 
-    HTableDescriptor htd = new HTableDescriptor(tablename);
-    FSTableDescriptors.createTableDescriptor(htd, HTU.getConfiguration());
-    HRegionInfo hri = new HRegionInfo(Bytes.toBytes("testGetSetOfHTD"),
-        HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
-    HTableDescriptor htd2 = hri.getTableDesc();
-    assertTrue(htd.equals(htd2));
-    final String key = "SOME_KEY";
-    assertNull(htd.getValue(key));
-    final String value = "VALUE";
-    htd.setValue(key, value);
-    hri.setTableDesc(htd);
-    HTableDescriptor htd3 = hri.getTableDesc();
-    assertTrue(htd.equals(htd3));
-  }
-  
   @Test
   public void testContainsRange() {
     HTableDescriptor tableDesc = new HTableDescriptor("testtable");
@@ -105,7 +77,7 @@ public void testContainsRange() {
     assertFalse(hri.containsRange(Bytes.toBytes("g"), Bytes.toBytes("g")));
     // Single row range entirely outside
     assertFalse(hri.containsRange(Bytes.toBytes("z"), Bytes.toBytes("z")));
-    
+
     // Degenerate range
     try {
       hri.containsRange(Bytes.toBytes("z"), Bytes.toBytes("a"));
@@ -134,14 +106,14 @@ public void testMetaTables() {
   public void testComparator() {
     byte[] tablename = Bytes.toBytes("comparatorTablename");
     byte[] empty = new byte[0];
-    HRegionInfo older = new HRegionInfo(tablename, empty, empty, false, 0L); 
-    HRegionInfo newer = new HRegionInfo(tablename, empty, empty, false, 1L); 
+    HRegionInfo older = new HRegionInfo(tablename, empty, empty, false, 0L);
+    HRegionInfo newer = new HRegionInfo(tablename, empty, empty, false, 1L);
     assertTrue(older.compareTo(newer) < 0);
     assertTrue(newer.compareTo(older) > 0);
     assertTrue(older.compareTo(older) == 0);
     assertTrue(newer.compareTo(newer) == 0);
   }
-  
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From c0e6df8ddcc5c945a96dd00e9024c77beb400457 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 2 Aug 2013 21:41:01 +0000
Subject: [PATCH 1096/1540] HBASE-9075 [0.94] Backport HBASE-5760 Unit tests
 should write only under /target to 0.94 (addendum patch to fix Hadoop2 build)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509873 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HBaseTestingUtility.java     |  9 ++++-
 .../hbase/mapreduce/MapreduceTestingShim.java | 40 ++++++++++++++++++-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 868e8cf64642..de3e132cce4f 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -62,6 +62,7 @@
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -1439,8 +1440,14 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
 
     mrCluster = new MiniMRCluster(0, 0, servers,
       FileSystem.get(conf).getUri().toString(), 1, null, null, null, new JobConf(conf));
-    mrCluster.getJobTrackerRunner().getJobTracker().getConf().set("mapred.local.dir",
+
+    JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
+    if (jobConf == null) {
+      jobConf = mrCluster.createJobConf();
+    }
+    jobConf.set("mapred.local.dir",
       conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
+
     LOG.info("Mini mapreduce cluster started");
     JobConf mrClusterJobConf = mrCluster.createJobConf();
     c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
index 76de6a39c804..74327922264e 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
@@ -19,9 +19,12 @@
 
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
@@ -29,11 +32,12 @@
 /**
  * This class provides shims for HBase to interact with the Hadoop 1.0.x and the
  * Hadoop 0.23.x series.
- * 
+ *
  * NOTE: No testing done against 0.22.x, or 0.21.x.
  */
 abstract public class MapreduceTestingShim {
   private static MapreduceTestingShim instance;
+  private static Class[] emptyParam = new Class[] {};
 
   static {
     try {
@@ -49,11 +53,17 @@ abstract public class MapreduceTestingShim {
   abstract public JobContext newJobContext(Configuration jobConf)
       throws IOException;
 
+  abstract public JobConf obtainJobConf(MiniMRCluster cluster);
+
   public static JobContext createJobContext(Configuration jobConf)
       throws IOException {
     return instance.newJobContext(jobConf);
   }
 
+  public static JobConf getJobConf(MiniMRCluster cluster) {
+    return instance.obtainJobConf(cluster);
+  }
+
   private static class MapreduceV1Shim extends MapreduceTestingShim {
     public JobContext newJobContext(Configuration jobConf) throws IOException {
       // Implementing:
@@ -68,6 +78,22 @@ public JobContext newJobContext(Configuration jobConf) throws IOException {
             "Failed to instantiate new JobContext(jobConf, new JobID())", e);
       }
     }
+    public JobConf obtainJobConf(MiniMRCluster cluster) {
+      if (cluster == null) return null;
+      try {
+        Object runner = cluster.getJobTrackerRunner();
+        Method meth = runner.getClass().getDeclaredMethod("getJobTracker", emptyParam);
+        Object tracker = meth.invoke(runner, new Object []{});
+        Method m = tracker.getClass().getDeclaredMethod("getConf", emptyParam);
+        return (JobConf) m.invoke(tracker, new Object []{});
+      } catch (NoSuchMethodException nsme) {
+        return null;
+      } catch (InvocationTargetException ite) {
+        return null;
+      } catch (IllegalAccessException iae) {
+        return null;
+      }
+    }
   };
 
   private static class MapreduceV2Shim extends MapreduceTestingShim {
@@ -83,6 +109,18 @@ public JobContext newJobContext(Configuration jobConf) {
             "Failed to return from Job.getInstance(jobConf)");
       }
     }
+    public JobConf obtainJobConf(MiniMRCluster cluster) {
+      try {
+        Method meth = MiniMRCluster.class.getMethod("getJobTrackerConf", emptyParam);
+        return (JobConf) meth.invoke(cluster, new Object []{});
+      } catch (NoSuchMethodException nsme) {
+        return null;
+      } catch (InvocationTargetException ite) {
+        return null;
+      } catch (IllegalAccessException iae) {
+        return null;
+      }
+    }
   };
 
 }

From dfb7778d1acb25679be169cbfcd1bfda69a903c3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 3 Aug 2013 00:41:42 +0000
Subject: [PATCH 1097/1540] HBASE-9115 HTableInterface.append operation may
 overwrites values (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509921 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/Append.java  | 8 +++++++-
 .../apache/hadoop/hbase/client/TestFromClientSide.java    | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java
index ce435c840ee6..f11aac9e06da 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -88,7 +88,13 @@ public Append add(byte [] family, byte [] qualifier, byte [] value) {
     if(list == null) {
       list = new ArrayList<KeyValue>();
     }
-    list.add(new KeyValue(
+    // find where the new entry should be placed in the List
+    int idx = 0;
+    for (KeyValue kv : list) {
+      if (Bytes.compareTo(qualifier, kv.getQualifier()) < 0) break;
+      idx ++;
+    }
+    list.add(idx, new KeyValue(
         this.row, family, qualifier, this.ts, KeyValue.Type.Put, value));
     familyMap.put(family, list);
     return this;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index f21583a1b8d0..0cc5d778b407 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4166,7 +4166,7 @@ public void testAppend() throws Exception {
     byte[] v1 = Bytes.toBytes("42");
     byte[] v2 = Bytes.toBytes("23");
     byte [][] QUALIFIERS = new byte [][] {
-        Bytes.toBytes("a"), Bytes.toBytes("b")
+        Bytes.toBytes("b"), Bytes.toBytes("a")
     };
     Append a = new Append(ROW);
     a.add(FAMILY, QUALIFIERS[0], v1);

From 06ac688493f8349102811ae7dfd1a116f2a3d771 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 3 Aug 2013 00:44:33 +0000
Subject: [PATCH 1098/1540] HBASE-9087 Handlers being blocked during reads
 (Elliott)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509922 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/Store.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index b9ca9ed673f6..d1e7e2289fac 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -27,9 +27,11 @@
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.Random;
+import java.util.Set;
 import java.util.SortedSet;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
@@ -158,8 +160,8 @@ public class Store extends SchemaConfigured implements HeapSize {
   List<StoreFile> filesCompacting = Lists.newArrayList();
 
   // All access must be synchronized.
-  private final CopyOnWriteArraySet<ChangedReadersObserver> changedReaderObservers =
-    new CopyOnWriteArraySet<ChangedReadersObserver>();
+  private final Set<ChangedReadersObserver> changedReaderObservers =
+      Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>());
 
   private final int blocksize;
   private HFileDataBlockEncoder dataBlockEncoder;

From 087ae389f109ae43e84ef566532f2cd2cc05eca5 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 3 Aug 2013 00:46:45 +0000
Subject: [PATCH 1099/1540] HBASE-8949
 hbase.mapreduce.hfileoutputformat.blocksize should configure with blocksize
 of a table (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1509923 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/HFileOutputFormat.java    | 41 ++++++++++++++++---
 src/main/resources/hbase-default.xml          | 13 ------
 2 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
index acb7d73f5631..028b220379fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
@@ -82,6 +82,7 @@ public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable,
   private static final String BLOOM_TYPE_CONF_KEY = "hbase.hfileoutputformat.families.bloomtype";
   private static final String DATABLOCK_ENCODING_CONF_KEY = 
      "hbase.mapreduce.hfileoutputformat.datablock.encoding";
+  private static final String BLOCK_SIZE_CONF_KEY = "hbase.mapreduce.hfileoutputformat.blocksize";
 
   public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
   throws IOException, InterruptedException {
@@ -93,8 +94,6 @@ public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final Task
     // These configs. are from hbase-*.xml
     final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
         HConstants.DEFAULT_MAX_FILE_SIZE);
-    final int blocksize = conf.getInt("hbase.mapreduce.hfileoutputformat.blocksize",
-        HFile.DEFAULT_BLOCKSIZE);
     // Invented config.  Add to hbase-*.xml if other than default compression.
     final String defaultCompression = conf.get("hfile.compression",
         Compression.Algorithm.NONE.getName());
@@ -104,7 +103,8 @@ public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final Task
     // create a map from column family to the compression algorithm
     final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf);
     final Map<byte[], String> bloomTypeMap = createFamilyBloomMap(conf);
-    
+    final Map<byte[], String> blockSizeMap = createFamilyBlockSizeMap(conf);
+
     String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
     final HFileDataBlockEncoder encoder;
     if (dataBlockEncodingStr == null) {
@@ -201,9 +201,12 @@ private WriterLength getNewWriter(byte[] family, Configuration conf)
         if (bloomTypeStr != null) {
           bloomType = BloomType.valueOf(bloomTypeStr);
         }
+        String blockSizeString = blockSizeMap.get(family);
+        int blockSize = blockSizeString == null ? HFile.DEFAULT_BLOCKSIZE
+            : Integer.parseInt(blockSizeString);
         Configuration tempConf = new Configuration(conf);
         tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
-        wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blocksize)
+        wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blockSize)
             .withOutputDir(familydir)
             .withCompression(AbstractHFileWriter.compressionByName(compression))
             .withBloomType(bloomType)
@@ -374,6 +377,7 @@ public static void configureIncrementalLoad(Job job, HTable table)
     // Set compression algorithms based on column families
     configureCompression(table, conf);
     configureBloomType(table, conf);
+    configureBlockSize(table, conf);
     
     TableMapReduceUtil.addDependencyJars(job);
     LOG.info("Incremental table output configured.");
@@ -417,7 +421,11 @@ static Map<byte[], String> createFamilyCompressionMap(Configuration conf) {
   private static Map<byte[], String> createFamilyBloomMap(Configuration conf) {
     return createFamilyConfValueMap(conf, BLOOM_TYPE_CONF_KEY);
   }
-  
+
+  private static Map<byte[], String> createFamilyBlockSizeMap(Configuration conf) {
+    return createFamilyConfValueMap(conf, BLOCK_SIZE_CONF_KEY);
+  }
+
   /**
    * Run inside the task to deserialize column family to given conf value map.
    * 
@@ -474,6 +482,29 @@ static void configureCompression(HTable table, Configuration conf) throws IOExce
     conf.set(COMPRESSION_CONF_KEY, compressionConfigValue.toString());
   }
   
+  private static void configureBlockSize(HTable table, Configuration conf) throws IOException {
+    StringBuilder blockSizeConfigValue = new StringBuilder();
+    HTableDescriptor tableDescriptor = table.getTableDescriptor();
+    if (tableDescriptor == null) {
+      // could happen with mock table instance
+      return;
+    }
+    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
+    int i = 0;
+    for (HColumnDescriptor familyDescriptor : families) {
+      if (i++ > 0) {
+        blockSizeConfigValue.append('&');
+      }
+      blockSizeConfigValue.append(URLEncoder.encode(
+          familyDescriptor.getNameAsString(), "UTF-8"));
+      blockSizeConfigValue.append('=');
+      blockSizeConfigValue.append(URLEncoder.encode(
+          String.valueOf(familyDescriptor.getBlocksize()), "UTF-8"));
+    }
+    // Get rid of the last ampersand
+    conf.set(BLOCK_SIZE_CONF_KEY, blockSizeConfigValue.toString());
+  }
+
   /**
    * Serialize column family to bloom type map to configuration.
    * Invoked while configuring the MR job for incremental load.
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 709bcb8f2663..df0c96f4d5a9 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -459,19 +459,6 @@
     Set to 0 to disable automated major compactions.
     </description>
   </property>
-  <property>
-    <name>hbase.mapreduce.hfileoutputformat.blocksize</name>
-    <value>65536</value>
-    <description>The mapreduce HFileOutputFormat writes storefiles/hfiles.
-    This is the minimum hfile blocksize to emit.  Usually in hbase, writing
-    hfiles, the blocksize is gotten from the table schema (HColumnDescriptor)
-    but in the mapreduce outputformat context, we don't have access to the
-    schema so get blocksize from Configuration.  The smaller you make
-    the blocksize, the bigger your index and the less you fetch on a
-    random-access.  Set the blocksize down if you have small cells and want
-    faster random-access of individual cells.
-    </description>
-  </property>
   <property>
     <name>hfile.block.cache.size</name>
     <value>0.25</value>

From 89945bf9e7fc63c5afe8c754db9d38a864514e30 Mon Sep 17 00:00:00 2001
From: jeffreyz <jeffreyz@unknown>
Date: Sat, 3 Aug 2013 18:50:30 +0000
Subject: [PATCH 1100/1540] hbase-8816: Add support of loading multiple tables
 into LoadTestTool

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1510058 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/AbstractHBaseTool.java  |  11 +-
 .../hadoop/hbase/util/LoadTestTool.java       | 141 +++++++++++++++++-
 2 files changed, 142 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
index 293c3634eeaa..c59a307b4fcd 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -39,8 +39,8 @@
  */
 public abstract class AbstractHBaseTool implements Tool {
 
-  private static final int EXIT_SUCCESS = 0;
-  private static final int EXIT_FAILURE = 1;
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
 
   private static final String HELP_OPTION = "help";
 
@@ -51,6 +51,8 @@ public abstract class AbstractHBaseTool implements Tool {
   protected Configuration conf = null;
 
   private static final Set<String> requiredOptions = new TreeSet<String>();
+  
+  protected String[] cmdLineArgs = null;
 
   /**
    * Override this to add command-line options using {@link #addOptWithArg}
@@ -87,6 +89,7 @@ public final int run(String[] args) throws IOException {
     try {
       // parse the command line arguments
       cmd = parseArgs(args);
+      cmdLineArgs = args;
     } catch (ParseException e) {
       LOG.error("Error when parsing command-line arguemnts", e);
       printUsage();
@@ -121,14 +124,14 @@ private boolean sanityCheckOptions(CommandLine cmd) {
     return success;
   }
 
-  private CommandLine parseArgs(String[] args) throws ParseException {
+  protected CommandLine parseArgs(String[] args) throws ParseException {
     options.addOption(HELP_OPTION, false, "Show usage");
     addOptions();
     CommandLineParser parser = new BasicParser();
     return parser.parse(options, args);
   }
 
-  private void printUsage() {
+  protected void printUsage() {
     HelpFormatter helpFormatter = new HelpFormatter();
     helpFormatter.setWidth(80);
     String usageHeader = "Options:";
diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index cbcd24752100..d6e963bc2f3f 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -17,11 +17,17 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -31,6 +37,7 @@
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * A command-line utility that reads, writes, and verifies data. Unlike
@@ -97,6 +104,7 @@ public class LoadTestTool extends AbstractHBaseTool {
   protected static final String OPT_ZK_QUORUM = "zk";
   protected static final String OPT_SKIP_INIT = "skip_init";
   protected static final String OPT_INIT_ONLY = "init_only";
+  private static final String NUM_TABLES = "num_tables";
 
   protected static final long DEFAULT_START_KEY = 0;
 
@@ -123,10 +131,12 @@ public class LoadTestTool extends AbstractHBaseTool {
   protected boolean isMultiPut;
 
   // Reader options
-  protected int numReaderThreads = DEFAULT_NUM_THREADS;
-  protected int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
-  protected int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
-  protected int verifyPercent;
+  private int numReaderThreads = DEFAULT_NUM_THREADS;
+  private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
+  private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
+  private int verifyPercent;
+ 
+  private int numTables = 1;
 
   // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
   // console tool itself should only be used from console.
@@ -214,6 +224,11 @@ protected void addOptions() {
         DEFAULT_START_KEY + ".");
     addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table "
         + "already exists");
+    
+    addOptWithArg(NUM_TABLES,
+      "A positive integer number. When a number n is speicfied, load test "
+          + "tool  will load n table parallely. -tn parameter value becomes "
+          + "table name prefix. Each table name is in format <tn>_1...<tn>_n");
   }
 
   @Override
@@ -299,6 +314,11 @@ protected void processOptions(CommandLine cmd) {
       System.out.println("Percent of keys to verify: " + verifyPercent);
       System.out.println("Reader threads: " + numReaderThreads);
     }
+    
+    numTables = 1;
+    if(cmd.hasOption(NUM_TABLES)) {
+      numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
+    }
   }
 
   protected void parseColumnFamilyOptions(CommandLine cmd) {
@@ -327,6 +347,14 @@ public void initTestTable() throws IOException {
 
   @Override
   protected int doWork() throws IOException {
+    if (numTables > 1) {
+      return parallelLoadTables();
+    } else {
+      return loadTable();
+    }
+  }
+
+  protected int loadTable() throws IOException {
     if (cmd.hasOption(OPT_ZK_QUORUM)) {
       conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
     }
@@ -387,11 +415,112 @@ protected int doWork() throws IOException {
       success = success && readerThreads.getNumReadErrors() == 0
           && readerThreads.getNumReadFailures() == 0;
     }
-    return success ? 0 : 1;
+    return success ? EXIT_SUCCESS : this.EXIT_FAILURE;
   }
-
+  
   public static void main(String[] args) {
     new LoadTestTool().doStaticMain(args);
   }
 
+  /**
+   * When NUM_TABLES is specified, the function starts multiple worker threads 
+   * which individually start a LoadTestTool instance to load a table. Each 
+   * table name is in format <tn>_<index>. For example, "-tn test -num_tables 2"
+   * , table names will be "test_1", "test_2"
+   * 
+   * @throws IOException
+   */
+  private int parallelLoadTables() 
+      throws IOException {
+    // create new command args
+    String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME);
+    String[] newArgs = null;
+    if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) {
+      newArgs = new String[cmdLineArgs.length + 2];
+      newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME;
+      for (int i = 0; i < cmdLineArgs.length; i++) {
+        newArgs[i + 2] = cmdLineArgs[i];
+      }
+    } else {
+      newArgs = cmdLineArgs;
+    }
+
+    int tableNameValueIndex = -1;
+    for (int j = 0; j < newArgs.length; j++) {
+      if (newArgs[j].endsWith(OPT_TABLE_NAME)) {
+        tableNameValueIndex = j + 1;
+      } else if (newArgs[j].endsWith(NUM_TABLES)) {
+        // change NUM_TABLES to 1 so that each worker loads one table
+        newArgs[j + 1] = "1"; 
+      }
+    }
+
+    // starting to load multiple tables
+    List<WorkerThread> workers = new ArrayList<WorkerThread>();
+    for (int i = 0; i < numTables; i++) {
+      String[] workerArgs = newArgs.clone();
+      workerArgs[tableNameValueIndex] = tableName + "_" + (i+1);
+      WorkerThread worker = new WorkerThread(i, workerArgs);
+      workers.add(worker);
+      LOG.info(worker + " starting");
+      worker.start();
+    }
+
+    // wait for all workers finish
+    LOG.info("Waiting for worker threads to finish");
+    for (WorkerThread t : workers) {
+      try {
+        t.join();
+      } catch (InterruptedException ie) {
+        IOException iie = new InterruptedIOException();
+        iie.initCause(ie);
+        throw iie;
+      }
+      checkForErrors();
+    }
+    
+    return EXIT_SUCCESS;
+  }
+
+  // If an exception is thrown by one of worker threads, it will be
+  // stored here.
+  protected AtomicReference<Throwable> thrown = new AtomicReference<Throwable>();
+
+  private void workerThreadError(Throwable t) {
+    thrown.compareAndSet(null, t);
+  }
+
+  /**
+   * Check for errors in the writer threads. If any is found, rethrow it.
+   */
+  private void checkForErrors() throws IOException {
+    Throwable thrown = this.thrown.get();
+    if (thrown == null) return;
+    if (thrown instanceof IOException) {
+      throw (IOException) thrown;
+    } else {
+      throw new RuntimeException(thrown);
+    }
+  }
+
+  class WorkerThread extends Thread {
+    private String[] workerArgs;
+
+    WorkerThread(int i, String[] args) {
+      super("WorkerThread-" + i);
+      workerArgs = args;
+    }
+
+    public void run() {
+      try {
+        int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs);
+        if (ret != 0) {
+          throw new RuntimeException("LoadTestTool exit with non-zero return code.");
+        }
+      } catch (Exception ex) {
+        LOG.error("Error in worker thread", ex);
+        workerThreadError(ex);
+      }
+    }
+  }
 }

From 4ca4a3a3278886e174c3693f98e8fee868961ab4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 4 Aug 2013 01:52:56 +0000
Subject: [PATCH 1101/1540] HBASE-9120 ClassFinder logs errors that are not
 (Sergey)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1510088 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/ClassFinder.java     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/ClassFinder.java b/src/test/java/org/apache/hadoop/hbase/ClassFinder.java
index 2461534d65b8..afc4aa6f7da4 100644
--- a/src/test/java/org/apache/hadoop/hbase/ClassFinder.java
+++ b/src/test/java/org/apache/hadoop/hbase/ClassFinder.java
@@ -120,7 +120,7 @@ private Set<Class<?>> findClassesFromJar(String jarFileName,
       if (!proceedOnExceptions) {
         throw ioEx;
       }
-      LOG.error("Failed to look for classes in " + jarFileName + ": " + ioEx);
+      LOG.warn("Failed to look for classes in " + jarFileName + ": " + ioEx);
     }
 
     Set<Class<?>> classes = new HashSet<Class<?>>();
@@ -132,7 +132,7 @@ private Set<Class<?>> findClassesFromJar(String jarFileName,
         if (!proceedOnExceptions) {
           throw ioEx;
         }
-        LOG.error("Failed to get next entry from " + jarFileName + ": " + ioEx);
+        LOG.warn("Failed to get next entry from " + jarFileName + ": " + ioEx);
         break;
       }
       if (entry == null) {
@@ -156,7 +156,7 @@ private Set<Class<?>> findClassesFromJar(String jarFileName,
       Class<?> c = makeClass(className, proceedOnExceptions);
       if (c != null) {
         if (!classes.add(c)) {
-          LOG.error("Ignoring duplicate class " + className);
+          LOG.warn("Ignoring duplicate class " + className);
         }
       }
     }
@@ -167,13 +167,13 @@ private Set<Class<?>> findClassesFromFiles(File baseDirectory, String packageNam
       boolean proceedOnExceptions) throws ClassNotFoundException, LinkageError {
     Set<Class<?>> classes = new HashSet<Class<?>>();
     if (!baseDirectory.exists()) {
-      LOG.error("Failed to find " + baseDirectory.getAbsolutePath());
+      LOG.warn("Failed to find " + baseDirectory.getAbsolutePath());
       return classes;
     }
 
     File[] files = baseDirectory.listFiles(this.fileFilter);
     if (files == null) {
-      LOG.error("Failed to get files from " + baseDirectory.getAbsolutePath());
+      LOG.warn("Failed to get files from " + baseDirectory.getAbsolutePath());
       return classes;
     }
 
@@ -188,7 +188,7 @@ private Set<Class<?>> findClassesFromFiles(File baseDirectory, String packageNam
         Class<?> c = makeClass(className, proceedOnExceptions);
         if (c != null) {
           if (!classes.add(c)) {
-            LOG.error("Ignoring duplicate class " + className);
+            LOG.warn("Ignoring duplicate class " + className);
           }
         }
       }
@@ -205,12 +205,12 @@ private Class<?> makeClass(String className, boolean proceedOnExceptions)
       if (!proceedOnExceptions) {
         throw classNotFoundEx;
       }
-      LOG.error("Failed to instantiate or check " + className + ": " + classNotFoundEx);
+      LOG.info("Failed to instantiate or check " + className + ": " + classNotFoundEx);
     } catch (LinkageError linkageEx) {
       if (!proceedOnExceptions) {
         throw linkageEx;
       }
-      LOG.error("Failed to instantiate or check " + className + ": " + linkageEx);
+      LOG.info("Failed to instantiate or check " + className + ": " + linkageEx);
     }
     return null;
   }

From dc795cca2032aafef0ad5571fe2b21fcf3e70ec1 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 4 Aug 2013 23:30:54 +0000
Subject: [PATCH 1102/1540] HBASE-9115 Addendum (Ted and Lars)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1510358 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/Append.java  | 8 +-------
 .../org/apache/hadoop/hbase/regionserver/HRegion.java     | 1 +
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java
index f11aac9e06da..ce435c840ee6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -88,13 +88,7 @@ public Append add(byte [] family, byte [] qualifier, byte [] value) {
     if(list == null) {
       list = new ArrayList<KeyValue>();
     }
-    // find where the new entry should be placed in the List
-    int idx = 0;
-    for (KeyValue kv : list) {
-      if (Bytes.compareTo(qualifier, kv.getQualifier()) < 0) break;
-      idx ++;
-    }
-    list.add(idx, new KeyValue(
+    list.add(new KeyValue(
         this.row, family, qualifier, this.ts, KeyValue.Type.Put, value));
     familyMap.put(family, list);
     return this;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b7629ca428e3..9dc9fde86b2d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5094,6 +5094,7 @@ public Result append(Append append, Integer lockid, boolean writeToWAL)
             .entrySet()) {
 
           Store store = stores.get(family.getKey());
+          Collections.sort(family.getValue(), store.getComparator());
           List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());
 
           // Get previous values for all columns in this family

From 09ccdd36f8b53bc59bdf68e8d1a11acc574035d3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 6 Aug 2013 16:19:51 +0000
Subject: [PATCH 1103/1540] HBASE-9123 Use table dir modtime to avoid scanning
 table dir to check cached table descriptor in 0.94 (Dave Latham)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1511018 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/FSTableDescriptors.java | 65 ++++++++++++++-----
 1 file changed, 50 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 6b4c64b1f3d3..11e532120605 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -80,20 +80,27 @@ public class FSTableDescriptors implements TableDescriptors {
     new ConcurrentHashMap<String, TableDescriptorModtime>();
 
   /**
-   * Data structure to hold modification time and table descriptor.
+   * Data structure to cache a table descriptor, the time it was modified,
+   * and the time the table directory was modified.
    */
   static class TableDescriptorModtime {
     private final HTableDescriptor descriptor;
     private final long modtime;
+    private final long dirmodtime;
 
-    TableDescriptorModtime(final long modtime, final HTableDescriptor htd) {
+    TableDescriptorModtime(final long modtime, final long dirmodtime, final HTableDescriptor htd) {
       this.descriptor = htd;
       this.modtime = modtime;
+      this.dirmodtime = dirmodtime;
     }
 
     long getModtime() {
       return this.modtime;
     }
+    
+    long getDirModtime() {
+      return this.dirmodtime;
+    }
 
     HTableDescriptor getTableDescriptor() {
       return this.descriptor;
@@ -153,15 +160,28 @@ public HTableDescriptor get(final String tablename)
 
     if (cachedtdm != null) {
       // Check mod time has not changed (this is trip to NN).
-      if (getTableInfoModtime(this.fs, this.rootdir, tablename) <= cachedtdm.getModtime()) {
+      // First check directory modtime as it doesn't require a scan of the full table directory
+      long tableDirModtime = getTableDirModtime(fs, this.rootdir, tablename);
+      boolean cachehit = false;
+      if (tableDirModtime <= cachedtdm.getDirModtime()) {
+        // table dir not changed since our cached entry
+        cachehit = true;
+      } else if (getTableInfoModtime(this.fs, this.rootdir, tablename) <= cachedtdm.getModtime()) {
+        // the table dir has changed (perhaps a region split) but the info file itself has not
+        // so the cached descriptor is good, we just need to update the entry
+        this.cache.put(tablename, new TableDescriptorModtime(cachedtdm.getModtime(),
+            tableDirModtime, cachedtdm.getTableDescriptor()));
+        cachehit = true;
+      }  // else table info file has been changed, need to read it 
+      if (cachehit) {
         cachehits++;
         return cachedtdm.getTableDescriptor();
       }
-    }
+   }
     
     TableDescriptorModtime tdmt = null;
     try {
-      tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename);
+      tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename, true);
     } catch (NullPointerException e) {
       LOG.debug("Exception during readTableDecriptor. Current table name = "
           + tablename, e);
@@ -215,8 +235,10 @@ public void add(HTableDescriptor htd) throws IOException {
       throw new NotImplementedException();
     }
     if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd);
-    long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString());
-    this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd));
+    String tableName = htd.getNameAsString();
+    long modtime = getTableInfoModtime(this.fs, this.rootdir, tableName);
+    long dirmodtime = getTableDirModtime(this.fs, this.rootdir, tableName);
+    this.cache.put(tableName, new TableDescriptorModtime(modtime, dirmodtime, htd));
   }
 
   @Override
@@ -355,6 +377,14 @@ static Path getTableInfoFileName(final Path tabledir, final int sequenceid) {
       TABLEINFO_NAME + "." + formatTableInfoSequenceId(sequenceid));
   }
 
+  static long getTableDirModtime(final FileSystem fs, final Path rootdir,
+      final String tableName)
+  throws IOException {
+    Path tabledir = FSUtils.getTablePath(rootdir, tableName);
+    FileStatus status = fs.getFileStatus(tabledir);
+    return status == null? 0: status.getModificationTime();
+  }
+  
   /**
    * @param fs
    * @param rootdir
@@ -384,7 +414,7 @@ public static HTableDescriptor getTableDescriptor(FileSystem fs,
      HTableDescriptor htd = null;
      try {
        TableDescriptorModtime tdmt =
-         getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName));
+         getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName), false);
        htd = tdmt == null ? null : tdmt.getTableDescriptor();
      } catch (NullPointerException e) {
        LOG.debug("Exception during readTableDecriptor. Current table name = "
@@ -395,21 +425,22 @@ public static HTableDescriptor getTableDescriptor(FileSystem fs,
 
   static HTableDescriptor getTableDescriptor(FileSystem fs,
       Path hbaseRootDir, String tableName) throws NullPointerException, IOException {
-    TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, hbaseRootDir, tableName);
+    TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, hbaseRootDir, tableName, false);
     return tdmt == null ? null : tdmt.getTableDescriptor();
   }
 
   static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs,
-      Path hbaseRootDir, String tableName) throws NullPointerException, IOException{
+      Path hbaseRootDir, String tableName, boolean readDirModtime)
+  throws NullPointerException, IOException{
     // ignore both -ROOT- and .META. tables
     if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0
         || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) {
       return null;
     }
-    return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
+    return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName), readDirModtime);
   }
 
-  static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir)
+  static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir, boolean readDirModtime)
   throws NullPointerException, IOException {
     if (tableDir == null) throw new NullPointerException();
     FileStatus status = getTableInfoPath(fs, tableDir);
@@ -425,12 +456,16 @@ static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tabl
     } finally {
       fsDataInputStream.close();
     }
-    return new TableDescriptorModtime(status.getModificationTime(), hTableDescriptor);
+    long dirModtime = 0;
+    if (readDirModtime) {
+      dirModtime = fs.getFileStatus(tableDir).getModificationTime();
+    }
+    return new TableDescriptorModtime(status.getModificationTime(), dirModtime, hTableDescriptor);
   }
-
+  
   public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
   throws IOException, NullPointerException {
-    TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, tableDir);
+    TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, tableDir, false);
     return tdmt == null? null: tdmt.getTableDescriptor();
   }
  

From c54d7052c9605347467eb76ec4d79b641ab7948a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 7 Aug 2013 23:58:28 +0000
Subject: [PATCH 1104/1540] HBASE-6580 Deprecate HTablePool in favor of
 HConnection.getTable(...)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1511545 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/book.xml                           |  13 +-
 .../hadoop/hbase/client/HConnection.java      |  54 ++++++++
 .../hbase/client/HConnectionManager.java      | 123 ++++++++++++++++--
 .../hadoop/hbase/client/HTablePool.java       |   1 +
 .../client/HConnectionTestingUtility.java     |   2 +-
 .../hbase/client/TestFromClientSide.java      |   7 +-
 .../apache/hadoop/hbase/client/TestHCM.java   |  71 ++++++++--
 .../hadoop/hbase/client/TestHConnection.java  |   2 +-
 8 files changed, 242 insertions(+), 31 deletions(-)

diff --git a/src/docbkx/book.xml b/src/docbkx/book.xml
index 4e19d912eb63..65c170a5326a 100644
--- a/src/docbkx/book.xml
+++ b/src/docbkx/book.xml
@@ -1516,11 +1516,14 @@ HTable table2 = new HTable(conf2, "myTable");</programlisting>
             </para>
             <para>
                 Another solution is to precreate an <classname>HConnection</classname> using
-                <programlisting>HConnectionManager.createConnection(Configuration)</programlisting> as
-                well as an <classname>ExecutorService</classname>; then use the
-                <programlisting>HTable(byte[], HConnection, ExecutorService)</programlisting>
-                constructor to create <classname>HTable</classname> instances on demand.
-                This construction is very lightweight and resources are controlled/shared if you go this route.
+                <programlisting>// Create a connection to the cluster.
+HConnection connection = HConnectionManager.createConnection(Configuration);
+HTableInterface table = connection.getTable("myTable");
+// use table as needed, the table returned is lightweight
+table.close();
+// use the connection for other access to the cluster
+connection.close();</programlisting>
+                Constructing HTableInterface implementation is very lightweight and resources are controlled/shared if you go this route.
             </para>
           </section>
    	  </section>
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index 1774a8938142..616beeadfba7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -63,6 +63,60 @@ public interface HConnection extends Abortable, Closeable {
    */
   public Configuration getConfiguration();
 
+  /**
+   * Retrieve an HTableInterface implementation for access to a table.
+   * The returned HTableInterface is not thread safe, a new instance should
+   * be created for each using thread.
+   * This is a lightweight operation, pooling or caching of the returned HTableInterface
+   * is neither required nor desired.
+   * Note that the HConnection needs to be unmanaged
+   * (created with {@link HConnectionManager#createConnection(Configuration)}).
+   * @param tableName
+   * @return an HTable to use for interactions with this table
+   */
+  public HTableInterface getTable(String tableName) throws IOException;
+
+  /**
+   * Retrieve an HTableInterface implementation for access to a table.
+   * The returned HTableInterface is not thread safe, a new instance should
+   * be created for each using thread.
+   * This is a lightweight operation, pooling or caching of the returned HTableInterface
+   * is neither required nor desired.
+   * Note that the HConnection needs to be unmanaged
+   * (created with {@link HConnectionManager#createConnection(Configuration)}).
+   * @param tableName
+   * @return an HTable to use for interactions with this table
+   */
+  public HTableInterface getTable(byte[] tableName) throws IOException;
+
+  /**
+   * Retrieve an HTableInterface implementation for access to a table.
+   * The returned HTableInterface is not thread safe, a new instance should
+   * be created for each using thread.
+   * This is a lightweight operation, pooling or caching of the returned HTableInterface
+   * is neither required nor desired.
+   * Note that the HConnection needs to be unmanaged
+   * (created with {@link HConnectionManager#createConnection(Configuration)}).
+   * @param tableName
+   * @param pool The thread pool to use for batch operations, null to use a default pool.
+   * @return an HTable to use for interactions with this table
+   */
+  public HTableInterface getTable(String tableName, ExecutorService pool)  throws IOException;
+
+  /**
+   * Retrieve an HTableInterface implementation for access to a table.
+   * The returned HTableInterface is not thread safe, a new instance should
+   * be created for each using thread.
+   * This is a lightweight operation, pooling or caching of the returned HTableInterface
+   * is neither required nor desired.
+   * Note that the HConnection needs to be unmanaged
+   * (created with {@link HConnectionManager#createConnection(Configuration)}).
+   * @param tableName
+   * @param pool The thread pool to use for batch operations, null to use a default pool.
+   * @return an HTable to use for interactions with this table
+   */
+  public HTableInterface getTable(byte[] tableName, ExecutorService pool)  throws IOException;
+
   /**
    * Retrieve ZooKeeperWatcher used by this connection.
    * @return ZooKeeperWatcher handle being used by the connection.
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 345426f2f275..3e79cad02fa9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -41,6 +41,9 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -75,6 +78,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.SoftValueSortedMap;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
@@ -85,8 +89,22 @@
 import org.apache.zookeeper.KeeperException;
 
 /**
- * A non-instantiable class that manages {@link HConnection}s.
- * This class has a static Map of {@link HConnection} instances keyed by
+ * A non-instantiable class that manages creation of {@link HConnection}s.
+ * <p>The simplest way to use this class is by using {@link #createConnection(Configuration)}.
+ * This creates a new {@link HConnection} that is managed by the caller.
+ * From this {@link HConnection} {@link HTableInterface} implementations are retrieved 
+ * with {@link HConnection#getTable(byte[])}. Example:
+ * <pre>
+ * {@code
+ * HConnection connection = HConnectionManager.createConnection(config);
+ * HTableInterface table = connection.getTable("table1");
+ * // use the table as needed, for a single operation and a single thread
+ * table.close();
+ * connection.close();
+ * }
+ * </pre>
+ * <p>The following logic and API will be removed in the future:
+ * <p>This class has a static Map of {@link HConnection} instances keyed by
  * {@link Configuration}; all invocations of {@link #getConnection(Configuration)}
  * that pass the same {@link Configuration} instance will be returned the same
  * {@link  HConnection} instance (Adding properties to a Configuration
@@ -182,11 +200,11 @@ public static HConnection getConnection(Configuration conf)
     synchronized (HBASE_INSTANCES) {
       HConnectionImplementation connection = HBASE_INSTANCES.get(connectionKey);
       if (connection == null) {
-        connection = new HConnectionImplementation(conf, true);
+        connection = new HConnectionImplementation(conf, true, null);
         HBASE_INSTANCES.put(connectionKey, connection);
       } else if (connection.isClosed()) {
         HConnectionManager.deleteConnection(connectionKey, true);
-        connection = new HConnectionImplementation(conf, true);
+        connection = new HConnectionImplementation(conf, true, null);
         HBASE_INSTANCES.put(connectionKey, connection);
       }
       connection.incCount();
@@ -197,16 +215,31 @@ public static HConnection getConnection(Configuration conf)
   /**
    * Create a new HConnection instance using the passed <code>conf</code>
    * instance.
-   * Note: This bypasses the usual HConnection life cycle management!
-   * Use this with caution, the caller is responsible for closing the
-   * created connection.
+   * Note: This bypasses the usual HConnection life cycle management.
+   * The caller is responsible for calling {@link HConnection#close()}
+   * on the returned connection instance.
+   *
+   * This is the recommended way to create HConnections.
+   * {@code
+   * HConnection connection = HConnectionManager.createConnection(conf);
+   * HTableInterface table = connection.getTable("mytable");
+   * table.get(...);
+   * ...
+   * table.close();
+   * connection.close();
+   * }
    * @param conf configuration
    * @return HConnection object for <code>conf</code>
    * @throws ZooKeeperConnectionException
    */
   public static HConnection createConnection(Configuration conf)
   throws ZooKeeperConnectionException {
-    return new HConnectionImplementation(conf, false);
+    return new HConnectionImplementation(conf, false, null);
+  }
+
+  public static HConnection createConnection(Configuration conf, ExecutorService pool)
+      throws IOException {
+    return new HConnectionImplementation(conf, false, pool);
   }
 
   /**
@@ -235,6 +268,7 @@ public static void deleteConnection(Configuration conf, boolean stopProxy) {
    *          configuration whose identity is used to find {@link HConnection}
    *          instance.
    */
+  @Deprecated
   public static void deleteConnection(Configuration conf) {
     deleteConnection(new HConnectionKey(conf), false);
   }
@@ -264,6 +298,7 @@ public static void deleteAllConnections(boolean stopProxy) {
    * Delete information for all connections.
    * @throws IOException
    */
+  @Deprecated
   public static void deleteAllConnections() {
     synchronized (HBASE_INSTANCES) {
       Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
@@ -275,6 +310,7 @@ public static void deleteAllConnections() {
     }
   }
 
+  @Deprecated
   private static void deleteConnection(HConnection connection, boolean staleConnection) {
     synchronized (HBASE_INSTANCES) {
       for (Entry<HConnectionKey, HConnectionImplementation> connectionEntry : HBASE_INSTANCES
@@ -287,6 +323,7 @@ private static void deleteConnection(HConnection connection, boolean staleConnec
     }
   }
 
+  @Deprecated
   private static void deleteConnection(HConnectionKey connectionKey,
       boolean staleConnection) {
     synchronized (HBASE_INSTANCES) {
@@ -533,6 +570,11 @@ static class HConnectionImplementation implements HConnection, Closeable {
 	
     private final Object resetLock = new Object();
 
+    // thread executor shared by all HTableInterface instances created
+    // by this connection
+    private volatile ExecutorService batchPool = null;
+    private volatile boolean cleanupPool = false;
+
     private final Configuration conf;
 
     private RpcEngine rpcEngine;
@@ -573,9 +615,10 @@ static class HConnectionImplementation implements HConnection, Closeable {
      * @param conf Configuration object
      */
     @SuppressWarnings("unchecked")
-    public HConnectionImplementation(Configuration conf, boolean managed)
+    public HConnectionImplementation(Configuration conf, boolean managed, ExecutorService pool)
     throws ZooKeeperConnectionException {
       this.conf = conf;
+      this.batchPool = pool;
       this.managed = managed;
       String serverClassName = conf.get(HConstants.REGION_SERVER_CLASS,
         HConstants.DEFAULT_REGION_SERVER_CLASS);
@@ -605,6 +648,67 @@ public HConnectionImplementation(Configuration conf, boolean managed)
       this.resetting = false;
     }
 
+    @Override
+    public HTableInterface getTable(String tableName) throws IOException {
+      return getTable(Bytes.toBytes(tableName));
+    }
+
+    @Override
+    public HTableInterface getTable(byte[] tableName) throws IOException {
+      return getTable(tableName, getBatchPool());
+    }
+
+    @Override
+    public HTableInterface getTable(String tableName, ExecutorService pool) throws IOException {
+      return getTable(Bytes.toBytes(tableName), pool);
+    }
+
+    @Override
+    public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {
+      if (managed) {
+        throw new IOException("The connection has to be unmanaged.");
+      }
+      return new HTable(tableName, this, pool);
+    }
+
+    private ExecutorService getBatchPool() {
+      if (batchPool == null) {
+        // shared HTable thread executor not yet initialized
+        synchronized (this) {
+          if (batchPool == null) {
+            int maxThreads = conf.getInt("hbase.hconnection.threads.max", Integer.MAX_VALUE);
+            if (maxThreads == 0) {
+              maxThreads = Runtime.getRuntime().availableProcessors();
+            }
+            long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60);
+            this.batchPool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(),
+                maxThreads, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+                Threads.newDaemonThreadFactory("hbase-connection-shared-executor"));
+            ((ThreadPoolExecutor) this.batchPool).allowCoreThreadTimeOut(true);
+          }
+          this.cleanupPool = true;
+        }
+      }
+      return this.batchPool;
+    }
+
+    protected ExecutorService getCurrentBatchPool() {
+      return batchPool;
+    }
+
+    private void shutdownBatchPool() {
+      if (this.cleanupPool && this.batchPool != null && !this.batchPool.isShutdown()) {
+        this.batchPool.shutdown();
+        try {
+          if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
+            this.batchPool.shutdownNow();
+          }
+        } catch (InterruptedException e) {
+          this.batchPool.shutdownNow();
+        }
+      }
+    }
+
     private synchronized void ensureZookeeperTrackers()
         throws ZooKeeperConnectionException {
       // initialize zookeeper and master address manager
@@ -1814,6 +1918,7 @@ void internalClose() {
       if (this.closed) {
         return;
       }
+      shutdownBatchPool();
       master = null;
 
       this.servers.clear();
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index d9615acfdb5b..01e6bbd0c2ee 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -56,6 +56,7 @@
  * <p>
  * Pool will manage its own connections to the cluster. See
  * {@link HConnectionManager}.
+ * @deprecated Use {@link HConnection#getTable(String)} instead.
  */
 public class HTablePool implements Closeable {
   private final PoolMap<String, HTableInterface> tables;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
index e34d8bcdd4fa..fdac02d575a6 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
@@ -129,7 +129,7 @@ public static HConnection getSpiedConnection(final Configuration conf)
       HConnectionImplementation connection =
         HConnectionManager.HBASE_INSTANCES.get(connectionKey);
       if (connection == null) {
-        connection = Mockito.spy(new HConnectionImplementation(conf, true));
+        connection = Mockito.spy(new HConnectionImplementation(conf, true, null));
         HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection);
       }
       return connection;
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 0cc5d778b407..a38b8785ac90 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -3901,12 +3901,7 @@ public void testListTables() throws IOException, InterruptedException {
   HTable createUnmangedHConnectionHTable(final byte [] tableName) throws IOException {
     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
-    ExecutorService pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE,
-      60, TimeUnit.SECONDS,
-      new SynchronousQueue<Runnable>(),
-      Threads.newDaemonThreadFactory("test-from-client-table"));
-    ((ThreadPoolExecutor)pool).allowCoreThreadTimeOut(true);
-    return new HTable(tableName, conn, pool);
+    return (HTable)conn.getTable(tableName);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index d9e88673ca41..f6343c50ac58 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -25,12 +25,14 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -129,6 +131,61 @@ private static int getHConnectionManagerCacheSize(){
     return HConnectionTestingUtility.getConnectionCount();
   }
   
+  @Test
+  public void testClusterConnection() throws IOException {
+    ThreadPoolExecutor otherPool = new ThreadPoolExecutor(1, 1, 5, TimeUnit.SECONDS,
+        new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("test-hcm"));
+
+    HConnection con1 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
+    HConnection con2 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration(), otherPool);
+    // make sure the internally created ExecutorService is the one passed
+    assertTrue(otherPool == ((HConnectionImplementation) con2).getCurrentBatchPool());
+
+    String tableName = "testClusterConnection";
+    TEST_UTIL.createTable(tableName.getBytes(), FAM_NAM).close();
+    HTable t = (HTable) con1.getTable(tableName, otherPool);
+    // make sure passing a pool to the getTable does not trigger creation of an
+    // internal pool
+    assertNull("Internal Thread pool should be null",
+        ((HConnectionImplementation) con1).getCurrentBatchPool());
+    // table should use the pool passed
+    assertTrue(otherPool == t.getPool());
+    t.close();
+
+    t = (HTable) con2.getTable(tableName);
+    // table should use the connectin's internal pool
+    assertTrue(otherPool == t.getPool());
+    t.close();
+
+    t = (HTable) con2.getTable(Bytes.toBytes(tableName));
+    // try other API too
+    assertTrue(otherPool == t.getPool());
+    t.close();
+
+    t = (HTable) con1.getTable(tableName);
+    ExecutorService pool = ((HConnectionImplementation) con1).getCurrentBatchPool();
+    // make sure an internal pool was created
+    assertNotNull("An internal Thread pool should have been created", pool);
+    // and that the table is using it
+    assertTrue(t.getPool() == pool);
+    t.close();
+
+    t = (HTable) con1.getTable(tableName);
+    // still using the *same* internal pool
+    assertTrue(t.getPool() == pool);
+    t.close();
+
+    con1.close();
+    // if the pool was created on demand it should be closed upon connectin
+    // close
+    assertTrue(pool.isShutdown());
+
+    con2.close();
+    // if the pool is passed, it is not closed
+    assertFalse(otherPool.isShutdown());
+    otherPool.shutdownNow();
+  }
+  
   @Test
   public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
     // Save off current HConnections
@@ -180,20 +237,16 @@ public void testRegionCaching() throws Exception{
   public void testConnectionManagement() throws Exception{
     TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
-    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
-        60, TimeUnit.SECONDS,
-        new SynchronousQueue<Runnable>(),
-        Threads.newDaemonThreadFactory("test-hcm-table"));
 
-    HTable table = new HTable(TABLE_NAME1, conn, pool);
+    HTableInterface table = conn.getTable(TABLE_NAME1);
     table.close();
     assertFalse(conn.isClosed());
-    assertFalse(pool.isShutdown());
-    table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME1, pool);
+    assertFalse(((HTable)table).getPool().isShutdown());
+    table = conn.getTable(TABLE_NAME1);
     table.close();
-    assertFalse(pool.isShutdown());
+    assertFalse(((HTable)table).getPool().isShutdown());
     conn.close();
-    pool.shutdownNow();
+    assertTrue(((HTable)table).getPool().isShutdown());
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java b/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java
index 470616e231de..ab0cc09c3013 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java
@@ -83,7 +83,7 @@ public void doAnAction() throws IOException {
 
   class HConnectionRaceTester extends HConnectionImplementation {
     public HConnectionRaceTester(Configuration configuration, boolean managed) throws ZooKeeperConnectionException {
-      super(configuration, managed);
+      super(configuration, managed, null);
     }
 
     /**

From bcd9dca21100a0c5e491c16ac3606e3cbb58823d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 8 Aug 2013 20:29:27 +0000
Subject: [PATCH 1105/1540] HBASE-9079 FilterList getNextKeyHint skips rows
 that should be included in the results (Viral Bajaria and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1512021 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/FilterList.java       |  23 ++-
 .../hadoop/hbase/filter/TestFilterList.java   |  36 ++--
 .../TestFuzzyRowAndColumnRangeFilter.java     | 170 ++++++++++++++++++
 3 files changed, 210 insertions(+), 19 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index fe3df8c6930d..26abd98617fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -63,6 +63,7 @@ public static enum Operator {
   private static final int MAX_LOG_FILTERS = 5;
   private Operator operator = Operator.MUST_PASS_ALL;
   private List<Filter> filters = new ArrayList<Filter>();
+  private Filter seekHintFilter = null;
 
   /** Reference KeyValue used by {@link #transform(KeyValue)} for validation purpose. */
   private KeyValue referenceKV = null;
@@ -166,6 +167,7 @@ public void reset() {
     for (Filter filter : filters) {
       filter.reset();
     }
+    seekHintFilter = null;
   }
 
   @Override
@@ -235,6 +237,9 @@ public ReturnCode filterKeyValue(KeyValue v) {
         case INCLUDE:
           transformed = filter.transform(transformed);
           continue;
+        case SEEK_NEXT_USING_HINT:
+          seekHintFilter = filter;
+          return code;
         default:
           return code;
         }
@@ -243,7 +248,8 @@ public ReturnCode filterKeyValue(KeyValue v) {
           continue;
         }
 
-        switch (filter.filterKeyValue(v)) {
+        ReturnCode code = filter.filterKeyValue(v);
+        switch (code) {
         case INCLUDE:
           if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) {
             rc = ReturnCode.INCLUDE;
@@ -332,9 +338,14 @@ public void write(final DataOutput out) throws IOException {
   @Override
   public KeyValue getNextKeyHint(KeyValue currentKV) {
     KeyValue keyHint = null;
+    if (operator == Operator.MUST_PASS_ALL) {
+      keyHint = seekHintFilter.getNextKeyHint(currentKV);
+      return keyHint;
+    }
+
     for (Filter filter : filters) {
       KeyValue curKeyHint = filter.getNextKeyHint(currentKV);
-      if (curKeyHint == null && operator == Operator.MUST_PASS_ONE) {
+      if (curKeyHint == null) {
         // If we ever don't have a hint and this is must-pass-one, then no hint
         return null;
       }
@@ -344,13 +355,7 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
           keyHint = curKeyHint;
           continue;
         }
-        // There is an existing hint
-        if (operator == Operator.MUST_PASS_ALL &&
-            KeyValue.COMPARATOR.compare(keyHint, curKeyHint) < 0) {
-          // If all conditions must pass, we can keep the max hint
-          keyHint = curKeyHint;
-        } else if (operator == Operator.MUST_PASS_ONE &&
-            KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) {
+        if (KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) {
           // If any condition can pass, we need to keep the min hint
           keyHint = curKeyHint;
         }
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 9adb80eb0740..49dc9699bfbe 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
 import org.apache.hadoop.hbase.filter.FilterList.Operator;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
@@ -319,6 +320,11 @@ public void write(DataOutput arg0) throws IOException {}
     };
 
     Filter filterMinHint = new FilterBase() {
+      @Override
+      public ReturnCode filterKeyValue(KeyValue ignored) {
+        return ReturnCode.SEEK_NEXT_USING_HINT;
+      }
+
       @Override
       public KeyValue getNextKeyHint(KeyValue currentKV) {
         return minKeyValue;
@@ -332,6 +338,11 @@ public void write(DataOutput arg0) throws IOException {}
     };
 
     Filter filterMaxHint = new FilterBase() {
+      @Override
+      public ReturnCode filterKeyValue(KeyValue ignored) {
+        return ReturnCode.SEEK_NEXT_USING_HINT;
+      }
+
       @Override
       public KeyValue getNextKeyHint(KeyValue currentKV) {
         return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null);
@@ -369,30 +380,35 @@ public void write(DataOutput arg0) throws IOException {}
 
     // MUST PASS ALL
 
-    // Should take the max if given two hints
+    // Should take the first hint
     filterList = new FilterList(Operator.MUST_PASS_ALL,
         Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
+    filterList.filterKeyValue(null);
+    assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
+        minKeyValue));
+
+    filterList = new FilterList(Operator.MUST_PASS_ALL,
+        Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } ));
+    filterList.filterKeyValue(null);
     assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
         maxKeyValue));
 
-    // Should have max hint even if a filter has no hint
+    // Should have first hint even if a filter has no hint
     filterList = new FilterList(Operator.MUST_PASS_ALL,
         Arrays.asList(
-            new Filter [] { filterMinHint, filterMaxHint, filterNoHint } ));
+            new Filter [] { filterNoHint, filterMinHint, filterMaxHint } ));
+    filterList.filterKeyValue(null);
     assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
-        maxKeyValue));
+        minKeyValue));
     filterList = new FilterList(Operator.MUST_PASS_ALL,
         Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
+    filterList.filterKeyValue(null);
     assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
         maxKeyValue));
+    filterList.filterKeyValue(null);
     filterList = new FilterList(Operator.MUST_PASS_ALL,
         Arrays.asList(new Filter [] { filterNoHint, filterMinHint } ));
-    assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
-        minKeyValue));
-
-    // Should give min hint if its the only one
-    filterList = new FilterList(Operator.MUST_PASS_ALL,
-        Arrays.asList(new Filter [] { filterNoHint, filterMinHint } ));
+    filterList.filterKeyValue(null);
     assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
         minKeyValue));
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java
new file mode 100644
index 000000000000..991d3904143a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java
@@ -0,0 +1,170 @@
+/**
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *  under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ */
+@Category(MediumTests.class)
+public class TestFuzzyRowAndColumnRangeFilter {
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final Log LOG = LogFactory.getLog(this.getClass());
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @Before
+  public void setUp() throws Exception {
+    // Nothing to do.
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @After
+  public void tearDown() throws Exception {
+    // Nothing to do.
+  }
+
+  @Test
+  public void Test() throws Exception {
+    String cf = "f";
+    String table = "TestFuzzyAndColumnRangeFilterClient";
+    HTable ht = TEST_UTIL.createTable(Bytes.toBytes(table),
+            Bytes.toBytes(cf), Integer.MAX_VALUE);
+
+    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
+    // 4 byte qualifier
+    // 4 byte value
+
+    for (int i1 = 0; i1 < 2; i1++) {
+      for (int i2 = 0; i2 < 5; i2++) {
+        byte[] rk = new byte[10];
+
+        ChannelBuffer buf = ChannelBuffers.wrappedBuffer(rk);
+        buf.clear();
+        buf.writeShort((short) 2);
+        buf.writeInt(i1);
+        buf.writeInt(i2);
+
+        for (int c = 0; c < 5; c++) {
+          byte[] cq = new byte[4];
+          Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
+
+          Put p = new Put(rk);
+          p.setDurability(Durability.SKIP_WAL);
+          p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+          ht.put(p);
+          LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
+                  + Bytes.toStringBinary(cq));
+        }
+      }
+    }
+
+    TEST_UTIL.flush();
+
+    // test passes
+    runTest(ht, 0, 10);
+
+    // test fails
+    runTest(ht, 1, 8);
+  }
+
+  private void runTest(HTable hTable, int cqStart, int expectedSize) throws IOException {
+    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
+    byte[] fuzzyKey = new byte[10];
+    ChannelBuffer buf = ChannelBuffers.wrappedBuffer(fuzzyKey);
+    buf.clear();
+    buf.writeShort((short) 2);
+    for (int i = 0; i < 4; i++)
+      buf.writeByte((short)63);
+    buf.writeInt((short)1);
+
+    byte[] mask = new byte[] {0 , 0, 1, 1, 1, 1, 0, 0, 0, 0};
+
+    Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
+    FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(Lists.newArrayList(pair));
+    ColumnRangeFilter columnRangeFilter = new ColumnRangeFilter(Bytes.toBytes(cqStart), true
+            , Bytes.toBytes(4), true);
+    //regular test
+    runScanner(hTable, expectedSize, fuzzyRowFilter, columnRangeFilter);
+    //reverse filter order test
+    runScanner(hTable, expectedSize, columnRangeFilter, fuzzyRowFilter);
+  }
+
+  private void runScanner(HTable hTable, int expectedSize, Filter... filters) throws IOException {
+    String cf = "f";
+    Scan scan = new Scan();
+    scan.addFamily(cf.getBytes());
+    FilterList filterList = new FilterList(filters);
+    scan.setFilter(filterList);
+
+    ResultScanner scanner = hTable.getScanner(scan);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    Result result;
+    long timeBeforeScan = System.currentTimeMillis();
+    while ((result = scanner.next()) != null) {
+      for (KeyValue kv : result.list()) {
+        LOG.info("Got rk: " + Bytes.toStringBinary(kv.getRow()) + " cq: "
+                + Bytes.toStringBinary(kv.getQualifier()));
+        results.add(kv);
+      }
+    }
+    long scanTime = System.currentTimeMillis() - timeBeforeScan;
+    scanner.close();
+
+    LOG.info("scan time = " + scanTime + "ms");
+    LOG.info("found " + results.size() + " results");
+
+    assertEquals(expectedSize, results.size());
+  }
+}
\ No newline at end of file

From e7657c930bc50dfebad95a5fdc057668571d73b2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 8 Aug 2013 23:08:07 +0000
Subject: [PATCH 1106/1540] HBASE-9158 Serious bug in cyclic replication

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1512093 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSink.java         | 69 ++++++++-----------
 .../replication/TestMasterReplication.java    | 23 ++++++-
 2 files changed, 52 insertions(+), 40 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 41b19ff7fa5f..1bec9684312d 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -28,25 +28,22 @@
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Row;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.Stoppable;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.util.UUID;
 
 /**
  * This class is responsible for replicating the edits coming
@@ -68,7 +65,6 @@ public class ReplicationSink {
   // Name of the HDFS directory that contains the temporary rep logs
   public static final String REPLICATION_LOG_DIR = ".replogs";
   private final Configuration conf;
-  private final ExecutorService sharedThreadPool;
   private final HConnection sharedHtableCon;
   private final ReplicationSinkMetrics metrics;
 
@@ -84,11 +80,6 @@ public ReplicationSink(Configuration conf, Stoppable stopper)
     this.conf = HBaseConfiguration.create(conf);
     decorateConf();
     this.sharedHtableCon = HConnectionManager.createConnection(this.conf);
-    this.sharedThreadPool = new ThreadPoolExecutor(1, 
-        conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE), 
-        conf.getLong("hbase.htable.threads.keepalivetime", 60), TimeUnit.SECONDS,
-        new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("hbase-repl"));
-    ((ThreadPoolExecutor)this.sharedThreadPool).allowCoreThreadTimeOut(true);
     this.metrics = new ReplicationSinkMetrics();
   }
 
@@ -119,9 +110,9 @@ public void replicateEntries(HLog.Entry[] entries)
     // to the same table.
     try {
       long totalReplicated = 0;
-      // Map of table => list of Rows, we only want to flushCommits once per
-      // invocation of this method per table.
-      Map<byte[], List<Row>> rows = new TreeMap<byte[], List<Row>>(Bytes.BYTES_COMPARATOR);
+      // Map of table => list of Rows, grouped by cluster id, we only want to flushCommits once per
+      // invocation of this method per table and cluster id.
+      Map<byte[], Map<UUID,List<Row>>> rowMap = new TreeMap<byte[], Map<UUID,List<Row>>>(Bytes.BYTES_COMPARATOR);
       for (HLog.Entry entry : entries) {
         WALEdit edit = entry.getEdit();
         byte[] table = entry.getKey().getTablename();
@@ -131,14 +122,15 @@ public void replicateEntries(HLog.Entry[] entries)
         List<KeyValue> kvs = edit.getKeyValues();
         for (KeyValue kv : kvs) {
           if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
+            UUID clusterId = entry.getKey().getClusterId();
             if (kv.isDelete()) {
               del = new Delete(kv.getRow());
-              del.setClusterId(entry.getKey().getClusterId());
-              addToMultiMap(rows, table, del);
+              del.setClusterId(clusterId);
+              addToHashMultiMap(rowMap, table, clusterId, del);
             } else {
               put = new Put(kv.getRow());
-              put.setClusterId(entry.getKey().getClusterId());
-              addToMultiMap(rows, table, put);
+              put.setClusterId(clusterId);
+              addToHashMultiMap(rowMap, table, clusterId, put);
             }
           }
           if (kv.isDelete()) {
@@ -150,8 +142,8 @@ public void replicateEntries(HLog.Entry[] entries)
         }
         totalReplicated++;
       }
-      for(byte [] table : rows.keySet()) {
-        batch(table, rows.get(table));
+      for(Map.Entry<byte[], Map<UUID, List<Row>>> entry : rowMap.entrySet()) {
+        batch(entry.getKey(), entry.getValue().values());
       }
       this.metrics.setAgeOfLastAppliedOp(
           entries[entries.length-1].getKey().getWriteTime());
@@ -167,15 +159,21 @@ public void replicateEntries(HLog.Entry[] entries)
    * Simple helper to a map from key to (a list of) values
    * TODO: Make a general utility method
    * @param map
-   * @param key
+   * @param key1
+   * @param key2
    * @param value
    * @return
    */
-  private <K, V> List<V> addToMultiMap(Map<K, List<V>> map, K key, V value) {
-    List<V> values = map.get(key);
+  private <K1, K2, V> List<V> addToHashMultiMap(Map<K1, Map<K2,List<V>>> map, K1 key1, K2 key2, V value) {
+    Map<K2,List<V>> innerMap = map.get(key1);
+    if (innerMap == null) {
+      innerMap = new HashMap<K2, List<V>>();
+      map.put(key1, innerMap);
+    }
+    List<V> values = innerMap.get(key2);
     if (values == null) {
       values = new ArrayList<V>();
-      map.put(key, values);
+      innerMap.put(key2, values);
     }
     values.add(value);
     return values;
@@ -185,15 +183,6 @@ private <K, V> List<V> addToMultiMap(Map<K, List<V>> map, K key, V value) {
    * stop the thread pool executor. It is called when the regionserver is stopped.
    */
   public void stopReplicationSinkServices() {
-    try {
-      this.sharedThreadPool.shutdown();
-      if (!this.sharedThreadPool.awaitTermination(60000, TimeUnit.MILLISECONDS)) {
-        this.sharedThreadPool.shutdownNow();
-      }
-    } catch (InterruptedException e) {
-      LOG.warn("Interrupted while closing the table pool", e); // ignoring it as we are closing.
-      Thread.currentThread().interrupt();
-    }
     try {
       this.sharedHtableCon.close();
     } catch (IOException e) {
@@ -207,15 +196,17 @@ public void stopReplicationSinkServices() {
    * @param rows list of actions
    * @throws IOException
    */
-  private void batch(byte[] tableName, List<Row> rows) throws IOException {
-    if (rows.isEmpty()) {
+  private void batch(byte[] tableName, Collection<List<Row>> allRows) throws IOException {
+    if (allRows.isEmpty()) {
       return;
     }
     HTableInterface table = null;
     try {
-      table = new HTable(tableName, this.sharedHtableCon, this.sharedThreadPool);
-      table.batch(rows);
-      this.metrics.appliedOpsRate.inc(rows.size());
+      table = this.sharedHtableCon.getTable(tableName);
+      for (List<Row> rows : allRows) {
+        table.batch(rows);
+        this.metrics.appliedOpsRate.inc(rows.size());
+      }
     } catch (InterruptedException ix) {
       throw new IOException(ix);
     } finally {
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
index 6a0c2ccbbb51..0bbef220d0d6 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
@@ -73,6 +73,8 @@ public class TestMasterReplication {
   private static final byte[] row = Bytes.toBytes("row");
   private static final byte[] row1 = Bytes.toBytes("row1");
   private static final byte[] row2 = Bytes.toBytes("row2");
+  private static final byte[] row3 = Bytes.toBytes("row3");
+  private static final byte[] row4 = Bytes.toBytes("row4");
   private static final byte[] noRepfamName = Bytes.toBytes("norep");
 
   private static final byte[] count = Bytes.toBytes("count");
@@ -178,6 +180,21 @@ public void testCyclicReplication() throws Exception {
     assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete));
     assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete));
     assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete));
+
+    // Test HBASE-9158
+    admin2.disablePeer("1");
+    // we now have an edit that was replicated into cluster originating from cluster 1
+    putAndWait(row3, famName, htable1, htable2);
+    // now add a local edit to cluster 2
+    Put put = new Put(row4);
+    put.add(famName, row4, row4);
+    htable2.put(put);
+    // reenable replication from cluster 2 to cluster 3
+    admin2.enablePeer("1");
+    // without HBASE-9158 the edit for row4 would have been marked with cluster 1's id
+    // and hence not replicated to cluster 1
+    wait(row4, htable1);
+    
     utility3.shutdownMiniCluster();
     utility2.shutdownMiniCluster();
     utility1.shutdownMiniCluster();
@@ -271,6 +288,10 @@ private void putAndWait(byte[] row, byte[] fam, HTable source, HTable target)
     put.add(fam, row, row);
     source.put(put);
 
+    wait(row, target);
+  }
+
+  private void wait(byte[] row, HTable target) throws Exception {
     Get get = new Get(row);
     for (int i = 0; i < NB_RETRIES; i++) {
       if (i==NB_RETRIES-1) {
@@ -284,7 +305,7 @@ private void putAndWait(byte[] row, byte[] fam, HTable source, HTable target)
         assertArrayEquals(res.value(), row);
         break;
       }
-    }
+    }    
   }
 
   /**

From 840ea0b627e5ba3581c0e59b9aa3d13469434bbb Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Fri, 9 Aug 2013 19:20:09 +0000
Subject: [PATCH 1107/1540] HBASE-9097: Set HBASE_CLASSPATH before rest of the
 classpath

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1512457 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/hbase b/bin/hbase
index 15e109bada41..caa498b8856f 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -186,9 +186,9 @@ for f in $HBASE_HOME/lib/*.jar; do
   CLASSPATH=${CLASSPATH}:$f;
 done
 
-# Add user-specified CLASSPATH last
+# Add user-specified CLASSPATH first
 if [ "$HBASE_CLASSPATH" != "" ]; then
-  CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH}
+  CLASSPATH=${HBASE_CLASSPATH}:${CLASSPATH}
 fi
 
 # default log directory & file

From 966983e6b4ee6b3e3d81680f965187d411786be2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 12 Aug 2013 23:54:36 +0000
Subject: [PATCH 1108/1540] HBASE-9146 TestHTablePool hangs when run as part of
 runMediumTests profile (Rajesh Venkatachalam)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513301 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/TestHTablePool.java   | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
index 74f10e57195c..be033512952b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
@@ -21,9 +21,6 @@
 
 import java.io.IOException;
 
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.PoolMap.PoolType;
@@ -39,22 +36,23 @@
 @Suite.SuiteClasses({TestHTablePool.TestHTableReusablePool.class, TestHTablePool.TestHTableThreadLocalPool.class})
 @Category(MediumTests.class)
 public class TestHTablePool {
-	private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-	private final static byte[] TABLENAME = Bytes.toBytes("TestHTablePool");
+  private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static byte[] TABLENAME = Bytes.toBytes("TestHTablePool");
+
+  public abstract static class TestHTablePoolType {
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
-			TEST_UTIL.startMiniCluster(1);
-			TEST_UTIL.createTable(TABLENAME, HConstants.CATALOG_FAMILY);
-		}
+      TEST_UTIL.startMiniCluster(1);
+      TEST_UTIL.createTable(TABLENAME, HConstants.CATALOG_FAMILY);
+    }
 
     @AfterClass
-		public static void tearDownAfterClass() throws Exception {
-			TEST_UTIL.shutdownMiniCluster();
-		}
+    public static void tearDownAfterClass() throws Exception {
+      TEST_UTIL.shutdownMiniCluster();
+    }
 
-	public abstract static class TestHTablePoolType extends TestCase {
-		protected abstract PoolType getPoolType();
+    protected abstract PoolType getPoolType();
 
 		@Test
 		public void testTableWithStringName() throws Exception {

From bc7f0000d29d591e34a3e953f4266655de2d07b3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Aug 2013 00:02:29 +0000
Subject: [PATCH 1109/1540] HBASE-9189 
 IncreasingToUpperBoundRegionSplitPolicy.shouldSplit() should check all the
 stores before returning. (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513304 13f79535-47bb-0310-9956-ffa450edef68
---
 .../IncreasingToUpperBoundRegionSplitPolicy.java         | 1 -
 .../hadoop/hbase/regionserver/RegionSplitPolicy.java     | 9 ++++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
index 0c9f15195ffa..327553d9b545 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
@@ -80,7 +80,6 @@ protected boolean shouldSplit() {
           " size=" + size + ", sizeToCheck=" + sizeToCheck +
           ", regionsWithCommonTable=" + tableRegionsCount);
         foundABigStore = true;
-        break;
       }
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
index a526544e31f6..952c2af69b82 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
@@ -42,7 +42,7 @@ public abstract class RegionSplitPolicy extends Configured {
    * The region configured for this split policy.
    */
   protected HRegion region;
-  
+
   /**
    * Upon construction, this method will be called with the region
    * to be governed. It will be called once and only once.
@@ -52,7 +52,7 @@ protected void configureForRegion(HRegion region) {
         this.region == null,
         "Policy already configured for region {}",
         this.region);
-    
+
     this.region = region;
   }
 
@@ -83,7 +83,7 @@ protected byte[] getSplitPoint() {
         largestStoreSize = storeSize;
       }
     }
-    
+
     return splitPointFromLargestStore;
   }
 
@@ -97,14 +97,13 @@ protected byte[] getSplitPoint() {
    */
   public static RegionSplitPolicy create(HRegion region,
       Configuration conf) throws IOException {
-    
     Class<? extends RegionSplitPolicy> clazz = getSplitPolicyClass(
         region.getTableDesc(), conf);
     RegionSplitPolicy policy = ReflectionUtils.newInstance(clazz, conf);
     policy.configureForRegion(region);
     return policy;
   }
-  
+
   static Class<? extends RegionSplitPolicy> getSplitPolicyClass(
       HTableDescriptor htd, Configuration conf) throws IOException {
     String className = htd.getRegionSplitPolicyClassName();

From 9e3dfcd080ad2fe8b9ce70db2e0589e2a7002699 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Aug 2013 03:53:36 +0000
Subject: [PATCH 1110/1540] HBASE-8663  a HBase Shell command to list the
 tables replicated from current cluster (Demai Ni)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513333 13f79535-47bb-0310-9956-ffa450edef68
---
 .../client/replication/ReplicationAdmin.java  | 48 +++++++++++++++
 src/main/ruby/hbase/replication_admin.rb      |  6 ++
 src/main/ruby/shell.rb                        |  1 +
 .../TestReplicationSmallTests.java            | 61 +++++++++++++++++++
 4 files changed, 116 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index b4efc3fb62f5..ac3bc81040ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -22,6 +22,10 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.lang.Integer;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
@@ -30,6 +34,8 @@
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 
 /**
  * <p>
@@ -66,6 +72,15 @@
  */
 public class ReplicationAdmin implements Closeable {
 
+  public static final String TNAME = "tableName";
+  public static final String CFNAME = "columnFamlyName";
+
+  // only Global for now, can add other type
+  // such as, 1) no global replication, or 2) the table is replicated to this cluster, etc.
+  public static final String REPLICATIONTYPE = "replicationType";
+  public static final String REPLICATIONGLOBAL = Integer
+      .toString(HConstants.REPLICATION_SCOPE_GLOBAL);
+      
   private final ReplicationZookeeper replicationZk;
   private final HConnection connection;
 
@@ -199,4 +214,37 @@ public void close() throws IOException {
       this.connection.close();
     }
   }
+  
+  /**
+   * Find all column families that are replicated from this cluster
+   * @return the full list of the replicated column families of this cluster as:
+   *        tableName, family name, replicationType
+   *
+   * Currently replicationType is Global. In the future, more replication
+   * types may be extended here. For example
+   *  1) the replication may only apply to selected peers instead of all peers
+   *  2) the replicationType may indicate the host Cluster servers as Slave
+   *     for the table:columnFam.         
+   */
+  public List<HashMap<String, String>> listReplicated() throws IOException {
+    List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();
+    HTableDescriptor[] tables = this.connection.listTables();
+  
+    for (HTableDescriptor table : tables) {
+      HColumnDescriptor[] columns = table.getColumnFamilies();
+      String tableName = table.getNameAsString();
+      for (HColumnDescriptor column : columns) {
+        if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
+          // At this moment, the columfam is replicated to all peers
+          HashMap<String, String> replicationEntry = new HashMap<String, String>();
+          replicationEntry.put(TNAME, tableName);
+          replicationEntry.put(CFNAME, column.getNameAsString());
+          replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
+          replicationColFams.add(replicationEntry);
+        }
+      }
+    }
+ 
+    return replicationColFams;
+  } 
 }
diff --git a/src/main/ruby/hbase/replication_admin.rb b/src/main/ruby/hbase/replication_admin.rb
index f694f5f678d7..99f0479b0e6f 100644
--- a/src/main/ruby/hbase/replication_admin.rb
+++ b/src/main/ruby/hbase/replication_admin.rb
@@ -43,6 +43,12 @@ def remove_peer(id)
       @replication_admin.removePeer(id)
     end
 
+    #---------------------------------------------------------------------------------------------
+    # Show replcated tables/column families, and their ReplicationType
+    def list_replicated_tables
+       @replication_admin.listReplicated()
+    end
+
     #----------------------------------------------------------------------------------------------
     # List all peer clusters
     def list_peers
diff --git a/src/main/ruby/shell.rb b/src/main/ruby/shell.rb
index d1ec550d1797..1330b7e834b0 100644
--- a/src/main/ruby/shell.rb
+++ b/src/main/ruby/shell.rb
@@ -286,6 +286,7 @@ def help_footer
     disable_peer
     start_replication
     stop_replication
+    list_replicated_tables
   ]
 )
 
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 7da487a6db91..1b47ab676b18 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -17,12 +17,18 @@
  */
 package org.apache.hadoop.hbase.replication;
 
+import java.util.HashMap;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
 import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -35,6 +41,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
 
 @Category(LargeTests.class)
 public class TestReplicationSmallTests extends TestReplicationBase {
@@ -527,4 +534,58 @@ public void testVerifyRepJob() throws Exception {
         findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
   }
 
+    
+    /**
+     * Test for HBASE-8663
+     * Create two new Tables with colfamilies enabled for replication then run
+     * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note:
+     * TestReplicationAdmin is a better place for this testing but it would need mocks.
+     * @throws Exception
+     */
+    @Test(timeout = 300000)
+    public void testVerifyListReplicatedTable() throws Exception {
+      LOG.info("testVerifyListReplicatedTable");
+   
+      final String tName = "VerifyListReplicated_";
+      final String colFam = "cf1";
+      final int numOfTables = 3;
+  
+      HBaseAdmin hadmin = new HBaseAdmin(conf1);
+  
+      // Create Tables
+      for (int i = 0; i < numOfTables; i++) {
+        HTableDescriptor ht = new HTableDescriptor(tName + i);
+        HColumnDescriptor cfd = new HColumnDescriptor(colFam);
+        cfd.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+        ht.addFamily(cfd);
+        hadmin.createTable(ht);
+      }
+  
+      // verify the result
+      List<HashMap<String, String>> replicationColFams = admin.listReplicated();
+      int[] match = new int[numOfTables]; // array of 3 with init value of zero
+  
+      for (int i = 0; i < replicationColFams.size(); i++) {
+        HashMap<String, String> replicationEntry = replicationColFams.get(i);
+        String tn = replicationEntry.get(ReplicationAdmin.TNAME);
+        if ((tn.startsWith(tName)) && replicationEntry.get(ReplicationAdmin.CFNAME).equals(colFam)) {
+          int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit
+          match[m]++; // should only increase once
+        }
+      }
+  
+      // check the matching result
+      for (int i = 0; i < match.length; i++) {
+        assertTrue("listReplicated() does not match table " + i, (match[i] == 1));
+      }
+  
+      // drop tables
+      for (int i = 0; i < numOfTables; i++) {
+        String ht = tName + i;
+        hadmin.disableTable(ht);
+        hadmin.deleteTable(ht);
+      }
+  
+      hadmin.close();
+    }  
 }

From f5c74c9b2687f82f199ba5674cfa25066dced8db Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Aug 2013 04:09:50 +0000
Subject: [PATCH 1111/1540] HBASE-8663 Forgot new file

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513339 13f79535-47bb-0310-9956-ffa450edef68
---
 .../shell/commands/list_replicated_tables.rb  | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 src/main/ruby/shell/commands/list_replicated_tables.rb

diff --git a/src/main/ruby/shell/commands/list_replicated_tables.rb b/src/main/ruby/shell/commands/list_replicated_tables.rb
new file mode 100644
index 000000000000..b1494b8d3a50
--- /dev/null
+++ b/src/main/ruby/shell/commands/list_replicated_tables.rb
@@ -0,0 +1,52 @@
+#
+# Copyright The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class ListReplicatedTables< Command
+      def help
+        return <<-EOF
+List all the tables and column families replicated from this cluster
+
+  hbase> list_replicated_tables
+  hbase> list_replicated_tables 'abc.*'
+EOF
+      end
+
+      def command(regex = ".*")
+        now = Time.now
+
+        formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ])
+        list = replication_admin.list_replicated_tables
+        regex = /#{regex}/ unless regex.is_a?(Regexp)
+        list = list.select {|s| regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))}
+        list.each do |e|
+          if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL
+             replicateType = "GLOBAL"
+          else
+             replicateType = "unknown"
+          end
+          formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME) + ":" + e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME), replicateType], true, [32])
+        end
+        formatter.footer(now)
+      end
+    end
+  end
+end

From adc99e1d2885c8d40a0c74dcf79b7d491eee8c43 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Aug 2013 04:13:04 +0000
Subject: [PATCH 1112/1540] HBASE-9139 Independent timeout configuration for
 rpc channel between cluster nodes (Julian Zhou)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513341 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HConstants.java  | 10 ++++++++++
 .../hadoop/hbase/regionserver/HRegionServer.java  |  4 ++--
 src/main/resources/hbase-default.xml              | 15 +++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 962529993d06..fa14c3707547 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -569,6 +569,16 @@ public static enum Modify {
    */
   public static int DEFAULT_HBASE_RPC_TIMEOUT = 60000;
 
+  /**
+   * timeout for short operation RPC
+   */
+  public static String HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY = "hbase.rpc.shortoperation.timeout";
+
+  /**
+   * Default value of {@link #HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY}
+   */
+  public static int DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT = 10000;
+
   /*
    * cluster replication constants.
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ead66cbe0700..fa7146ff999f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -429,8 +429,8 @@ public HRegionServer(Configuration conf)
       "hbase.regionserver.numregionstoreport", 10);
 
     this.rpcTimeout = conf.getInt(
-      HConstants.HBASE_RPC_TIMEOUT_KEY,
-      HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+      HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY,
+      HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT);
 
     this.abortRequested = false;
     this.stopped = false;
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index df0c96f4d5a9..fd8a1fe07ef2 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -542,6 +542,21 @@
     used for client / server RPC call marshalling.
     </description>
   </property>
+  <property>
+    <name>hbase.rpc.timeout</name>
+    <value>60000</value>
+    <description>This is for the RPC layer to define how long HBase client applications
+        take for a remote call to time out. It uses pings to check connections
+        but will eventually throw a TimeoutException.</description>
+  </property>
+  <property>
+    <name>hbase.rpc.shortoperation.timeout</name>
+    <value>10000</value>
+    <description>This is another version of "hbase.rpc.timeout". For those RPC operation
+        within cluster, we rely on this configuration to set a short timeout limitation
+        for short operation. For example, short rpc timeout for region server's trying
+        to report to active master can benefit quicker master failover process.</description>
+  </property>
 
   <!-- The following properties configure authentication information for
        HBase processes when using Kerberos security.  There are no default

From dde5d400659e3861f37b723e376e5657654b22be Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Aug 2013 23:11:20 +0000
Subject: [PATCH 1113/1540] HBASE-9200 HFilePrettyPrinter finds incorrect
 largest row (Shrijeet Paliwal)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513686 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index b68e2af66e17..6086408796dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -423,6 +423,7 @@ private void collectRow() {
 
       if (curRowBytes > maxRowBytes && prevKV != null) {
         biggestRow = prevKV.getRow();
+        maxRowBytes = curRowBytes;
       }
 
       curRowBytes = 0;

From 197e56d572937f0c8ffaf71a854fe9c41021d83b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 Aug 2013 23:47:40 +0000
Subject: [PATCH 1114/1540] CHANGES.txt and pom.xml for 0.94.11RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513696 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index c6de9d84682c..f0400eeff9d6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,61 @@
 HBase Change Log
 
+Release 0.94.11 - 8/13/2013
+Sub-task
+
+    [HBASE-8779] - Add mutateRow method support to Thrift2
+    [HBASE-8946] - Add a new function to Thrift 2 to open scanner, get results and close scanner
+    [HBASE-8947] - Thrift 2 : Replace "bool writeToWAL" with "TDurability durability"
+    [HBASE-8948] - Bound table pool size in Thrift 2 server
+
+Bug
+
+    [HBASE-6826] - [WINDOWS] TestFromClientSide failures
+    [HBASE-8067] - TestHFileArchiving.testArchiveOnTableDelete sometimes fails
+    [HBASE-8670] - [0.94] Backport HBASE-8449,HBASE-8204 and HBASE-8699 to 0.94 (Refactor recoverLease retries and pauses)
+    [HBASE-8698] - potential thread creation in MetaScanner.metaScan
+    [HBASE-8935] - IntegrationTestBigLinkedList fails under load on 0.94 due to some scan issues - add logging
+    [HBASE-8949] - hbase.mapreduce.hfileoutputformat.blocksize should configure with blocksize of a table
+    [HBASE-9026] - RestartRsHoldingRoot action in org.apache.hadoop.hbase.util.ChaosMonkey restarting the server holding .META. instead of -ROOT-
+    [HBASE-9032] - Result.getBytes() returns null if backed by KeyValue array
+    [HBASE-9048] - HCM throws NullPointerException under load
+    [HBASE-9050] - HBaseClient#call could hang
+    [HBASE-9060] - ExportSnapshot job fails if target path contains percentage character
+    [HBASE-9079] - FilterList getNextKeyHint skips rows that should be included in the results
+    [HBASE-9080] - Retain assignment should be used when re-enabling table(s)
+    [HBASE-9085] - Integration Tests fails because of bug in teardown phase where the cluster state is not being restored properly.
+    [HBASE-9087] - Handlers being blocked during reads
+    [HBASE-9097] - Set HBASE_CLASSPATH before rest of the classpath
+    [HBASE-9115] - HTableInterface.append operation may overwrites values
+    [HBASE-9120] - ClassFinder logs errors that are not
+    [HBASE-9146] - TestHTablePool hangs when run as part of runMediumTests profile
+    [HBASE-9158] - Serious bug in cyclic replication
+    [HBASE-9189] - IncreasingToUpperBoundRegionSplitPolicy.shouldSplit() should check all the stores before returning.
+    [HBASE-9200] - HFilePrettyPrinter finds incorrect largest row
+
+Improvement
+
+    [HBASE-6580] - Deprecate HTablePool in favor of HConnection.getTable(...)
+    [HBASE-8995] - Add hadoop-1.2 profile
+    [HBASE-9019] - Port HBASE-8690: Reduce unnecessary getFileStatus hdfs calls in TTL hfile and hlog cleanners to 0.94
+    [HBASE-9029] - Backport HBASE-8706 Some improvement in snapshot to 0.94
+    [HBASE-9132] - Use table dir modtime to avoid scanning table dir to check cached table descriptor in 0.94
+    [HBASE-9139] - Independent timeout configuration for rpc channel between cluster nodes
+
+New Feature
+
+    [HBASE-7826] - Improve Hbase Thrift v1 to return results in sorted order
+    [HBASE-8663] - a HBase Shell command to list the tables replicated from current cluster
+
+Test
+
+    [HBASE-8816] - Add support of loading multiple tables into LoadTestTool
+    [HBASE-9075] - [0.94] Backport HBASE-5760 Unit tests should write only under /target to 0.94
+    [HBASE-9090] - cleanup snapshot tests setup/teardown code
+    [HBASE-9106] - Do not fail TestAcidGuarantees for exceptions on table flush
+    [HBASE-9107] - [0.94] Backport HBASE-6950 TestAcidGuarantees system test now flushes too aggressively to 0.94
+
+
 Release 0.94.10 - 7/19/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 1739c12b25b3..d22ad2c70955 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.11-SNAPSHOT</version>
+  <version>0.94.11</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 1b62085a01ff3924aca796e994950c6d4b70cbea Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 14 Aug 2013 06:23:53 +0000
Subject: [PATCH 1115/1540] HBASE-9182. Allow non-admin users to list all table
 names

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1513738 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       | 36 +++++++++++++++++--
 .../hadoop/hbase/client/HConnection.java      |  6 ++++
 .../hbase/client/HConnectionManager.java      |  8 +++++
 .../hadoop/hbase/ipc/HMasterInterface.java    |  7 ++++
 .../apache/hadoop/hbase/master/HMaster.java   | 22 +++++++++++-
 src/main/ruby/hbase/admin.rb                  |  4 +--
 6 files changed, 78 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index d5300862f750..bce395dc32ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -78,8 +78,6 @@
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.util.StringUtils;
 
-import com.google.protobuf.ServiceException;
-
 /**
  * Provides an interface to manage HBase database table metadata + general
  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
@@ -313,6 +311,40 @@ public HTableDescriptor[] listTables(String regex) throws IOException {
     return listTables(Pattern.compile(regex));
   }
 
+  /**
+   * List all of the names of userspace tables.
+   * @return the list of table names
+   * @throws IOException if a remote or network exception occurs
+   */
+  public String[] getTableNames() throws IOException {
+    return this.connection.getTableNames();
+  }
+
+  /**
+   * List all of the names of userspace tables matching the given pattern
+   * @param pattern The compiled regular expression to match against
+   * @return the list of table names
+   * @throws IOException if a remote or network exception occurs
+   */
+  public String[] getTableNames(Pattern pattern) throws IOException {
+    List<String> matched = new ArrayList<String>();
+    for (String name : this.connection.getTableNames()) {
+      if (pattern.matcher(name).matches()) {
+        matched.add(name);
+      }
+    }
+    return matched.toArray(new String[matched.size()]);
+  }
+
+  /**
+   * List all of the names of userspace tables matching the given regex
+   * @param regex The regular expression to match against
+   * @return the list of table names
+   * @throws IOException if a remote or network exception occurs
+   */
+  public String[] getTableNames(String regex) throws IOException {
+    return getTableNames(Pattern.compile(regex));
+  }
 
   /**
    * Method for getting the tableDescriptor
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index 616beeadfba7..da8078c45c53 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -451,6 +451,12 @@ public void prewarmRegionCache(final byte[] tableName,
   public HTableDescriptor[] getHTableDescriptors(List<String> tableNames)
   throws IOException;
 
+  /**
+   * @return string[] table names
+   * @throws IOException if a remote or network exception occurs
+   */
+  public String[] getTableNames() throws IOException;
+
   /**
    * @return true if this connection is closed
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 3e79cad02fa9..2ae35d2390fd 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1987,6 +1987,14 @@ public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) throws I
       return master.getHTableDescriptors(tableNames);
     }
 
+    @Override
+    public String[] getTableNames() throws IOException {
+      if (this.master == null) {
+        this.master = getMaster();
+      }
+      return master.getTableNames();
+    }
+
     public HTableDescriptor getHTableDescriptor(final byte[] tableName)
     throws IOException {
       if (tableName == null || tableName.length == 0) return null;
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index bbb23fe565e2..3dc6ba067b6e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -318,4 +318,11 @@ public void restoreSnapshot(final HSnapshotDescription request)
 
   public boolean isRestoreSnapshotDone(final HSnapshotDescription request)
     throws IOException;
+
+  /**
+   * Return all table names.
+   * @return the list of table names
+   * @throws IOException if an error occurred while getting the list of tables
+   */
+  public String[] getTableNames() throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 288ea6d072f4..9b1ad919506f 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -26,9 +26,11 @@
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -2242,5 +2244,23 @@ public void restoreSnapshot(final HSnapshotDescription request) throws IOExcepti
   public boolean isRestoreSnapshotDone(final HSnapshotDescription request) throws IOException {
     return snapshotManager.isRestoreDone(request.getProto());
   }
-}
 
+  /**
+   * Return all table names.
+   * @return the list of table names
+   * @throws IOException if an error occurred while getting the list of tables
+   */
+  @Override
+  public String[] getTableNames() throws IOException {
+    // Anyone is allowed to see the names of tables, so there is no coprocessor
+    // hook nor AccessController interception necessary
+    Collection<HTableDescriptor> descriptors = tableDescriptors.getAll().values();
+    Iterator<HTableDescriptor> iter = descriptors.iterator();
+    String names[] = new String[descriptors.size()];
+    int i = 0;
+    while (iter.hasNext()) {
+      names[i++] = iter.next().getNameAsString();
+    }
+    return names;
+  }
+}
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index b42331ffbd1a..103bbfe3489e 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -40,8 +40,8 @@ def initialize(configuration, formatter)
 
     #----------------------------------------------------------------------------------------------
     # Returns a list of tables in hbase
-    def list
-      @admin.listTables.map { |t| t.getNameAsString }
+    def list(regex = ".*")
+      @admin.getTableNames(regex)
     end
 
     #----------------------------------------------------------------------------------------------

From bfb05cdca89925e7655967206f71d14d6fefca00 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 15 Aug 2013 17:42:46 +0000
Subject: [PATCH 1116/1540] HBASE-9231 Multipage book is generated to the wrong
 location

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1514397 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d22ad2c70955..9a4b398822aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -632,7 +632,7 @@
           <sectionAutolabelMaxDepth>100</sectionAutolabelMaxDepth>
           <sectionAutolabel>true</sectionAutolabel>
           <sectionLabelIncludesComponentLabel>true</sectionLabelIncludesComponentLabel>
-          <targetDirectory>${basedir}/target/site/book/</targetDirectory>
+          <targetDirectory>${basedir}/target/site/</targetDirectory>
           <htmlStylesheet>../css/freebsd_docbook.css</htmlStylesheet>
           <htmlCustomization>src/docbkx/customization.xsl</htmlCustomization>
           <imgSrcPath>../images/</imgSrcPath>

From 9526c6807f141881e1c06a4934254d5b9369c16e Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Thu, 15 Aug 2013 20:10:47 +0000
Subject: [PATCH 1117/1540] HBASE-9207 An Offline SplitParent Region can be
 assigned breaking split references

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1514464 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 40 ++++++-------
 .../apache/hadoop/hbase/master/HMaster.java   |  9 ++-
 .../hadoop/hbase/util/ModifyRegionUtils.java  | 24 --------
 .../handler/TestCreateTableHandler.java       | 57 ++++++++++++++-----
 .../TestSplitTransactionOnCluster.java        |  8 ++-
 5 files changed, 73 insertions(+), 65 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 4d57a235b9d1..9afa95536de8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -22,7 +22,6 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -1418,6 +1417,9 @@ public void assign(HRegionInfo region, boolean setOfflineInZK,
         region.getRegionNameAsString());
       return;
     }
+    if (isAssigningSplitParentRegion(region)) {
+      return;
+    }
     RegionState state = addToRegionsInTransition(region,
         hijack);
     synchronized (state) {
@@ -1846,6 +1848,15 @@ private void assign(final HRegionInfo region, final RegionState state,
     }
   }
 
+  private static boolean isAssigningSplitParentRegion(final HRegionInfo region) {
+    if (region.isSplitParent()) {
+      LOG.info("Skipping assign of " + region.getRegionNameAsString()
+        + ", already split, or still splitting");
+      return true;
+    }
+    return false;
+  }
+
   private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) {
     String tableName = region.getTableNameAsString();
     boolean disabled = this.zkTable.isDisabledTable(tableName);
@@ -2526,28 +2537,6 @@ protected long getTimeoutOnRIT() {
     }
   }
 
-  /**
-   * Bulk user region assigner.
-   * If failed assign, lets timeout in RIT do cleanup.
-   */
-  static class GeneralBulkAssigner extends StartupBulkAssigner {
-    GeneralBulkAssigner(final Server server,
-        final Map<ServerName, List<HRegionInfo>> bulkPlan,
-        final AssignmentManager am) {
-      super(server, bulkPlan, am);
-    }
-
-    @Override
-    protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
-      return new UncaughtExceptionHandler() {
-        @Override
-        public void uncaughtException(Thread t, Throwable e) {
-          LOG.warn("Assigning regions in " + t.getName(), e);
-        }
-      };
-    }
-  }
-
   /**
    * Manage bulk assigning to a server.
    */
@@ -2558,6 +2547,11 @@ static class SingleServerBulkAssigner implements Runnable {
 
     SingleServerBulkAssigner(final ServerName regionserver,
         final List<HRegionInfo> regions, final AssignmentManager am) {
+      for (Iterator<HRegionInfo> it = regions.iterator(); it.hasNext(); ) {
+        if (isAssigningSplitParentRegion(it.next())) {
+          it.remove();
+        }
+      }
       this.regionserver = regionserver;
       this.regions = regions;
       this.assignmentManager = am;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9b1ad919506f..f2e3520fd27b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -119,6 +119,7 @@
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -844,7 +845,7 @@ private void enableCatalogTables(String catalogTableName) {
     }
   }
 
-  void fixupDaughters(final MonitoredTask status) throws IOException {
+  void fixupDaughters(final MonitoredTask status) throws IOException, KeeperException {
     final Map<HRegionInfo, Result> offlineSplitParents =
       new HashMap<HRegionInfo, Result>();
     // This visitor collects offline split parents in the .META. table
@@ -868,8 +869,12 @@ public boolean visit(Result r) throws IOException {
     // Now work on our list of found parents. See if any we can clean up.
     int fixups = 0;
     for (Map.Entry<HRegionInfo, Result> e : offlineSplitParents.entrySet()) {
-      fixups += ServerShutdownHandler.fixupDaughters(
+      String node = ZKAssign.getNodeName(zooKeeper, e.getKey().getEncodedName());
+      byte[] data = ZKUtil.getData(zooKeeper, node);
+      if (data == null) { // otherwise, splitting is still going on, skip it
+        fixups += ServerShutdownHandler.fixupDaughters(
           e.getValue(), assignmentManager, catalogTracker);
+      }
     }
     if (fixups != 0) {
       LOG.info("Scanned the catalog and fixed up " + fixups +
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index ba72227587b8..3744f634172e 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -22,7 +22,6 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
@@ -35,18 +34,11 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.catalog.MetaEditor;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 
 /**
@@ -157,20 +149,4 @@ public Thread newThread(Runnable r) {
         });
     return regionOpenAndInitThreadPool;
   }
-
-  /**
-   * Trigger immediate assignment of the regions in round-robin fashion
-   *
-   * @param assignmentManager
-   * @param regions
-   */
-  public static void assignRegions(final AssignmentManager assignmentManager,
-      final List<HRegionInfo> regions) throws IOException {
-    try {
-      assignmentManager.assignUserRegionsToOnlineServers(regions);
-    } catch (InterruptedException ie) {
-      LOG.error("Caught " + ie + " during round-robin assignment");
-      throw new InterruptedIOException(ie.getMessage());
-    }
-  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
index 1f85eb226fe8..c101623bfb73 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.master.handler;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -26,6 +27,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -40,16 +42,11 @@
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
-import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.ServerManager;
-import org.apache.hadoop.hbase.master.TestMaster;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZKTable;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -57,7 +54,6 @@
 public class TestCreateTableHandler {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class);
-  private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler");
   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
   private static boolean throwException = false;
 
@@ -73,10 +69,11 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() throws Exception {
+  public void testCreateTableCalledTwiceAndFirstOneInProgress() throws Exception {
+    final byte[] tableName = Bytes.toBytes("testCreateTableCalledTwiceAndFirstOneInProgress");
     final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     final HMaster m = cluster.getMaster();
-    final HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+    final HTableDescriptor desc = new HTableDescriptor(tableName);
     desc.addFamily(new HColumnDescriptor(FAMILYNAME));
     final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null,
         null) };
@@ -91,14 +88,47 @@ public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() t
         m.getCatalogTracker(), m.getAssignmentManager());
     handler1.process();
     for (int i = 0; i < 100; i++) {
-      if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(TABLENAME)) {
+      if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) {
         Thread.sleep(200);
       }
     }
-    assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME));
+    assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName));
+  }
+
+  @Test (timeout=300000)
+  public void testCreateTableWithSplitRegion() throws Exception {
+    final byte[] tableName = Bytes.toBytes("testCreateTableWithSplitRegion");
+    final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    final HMaster m = cluster.getMaster();
+    final HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor(FAMILYNAME));
+    byte[] splitPoint = Bytes.toBytes("split-point");
+    long ts = System.currentTimeMillis();
+    HRegionInfo d1 = new HRegionInfo(tableName, null, splitPoint, false, ts);
+    HRegionInfo d2 = new HRegionInfo(tableName, splitPoint, null, false, ts + 1);
+    HRegionInfo parent = new HRegionInfo(tableName, null, null, true, ts + 2);
+    parent.setOffline(true);
 
+    Path tempdir = m.getMasterFileSystem().getTempDir();
+    FileSystem fs = m.getMasterFileSystem().getFileSystem();
+    Path tempTableDir = FSUtils.getTablePath(tempdir, tableName);
+    fs.delete(tempTableDir, true); // Clean up temp table dir if exists
+
+    final HRegionInfo[] hRegionInfos = new HRegionInfo[] {d1, d2, parent};
+    CreateTableHandler handler = new CreateTableHandler(m, m.getMasterFileSystem(),
+      m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos,
+      m.getCatalogTracker(), m.getAssignmentManager());
+    handler.process();
+    for (int i = 0; i < 200; i++) {
+      if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) {
+        Thread.sleep(300);
+      }
+    }
+    assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName));
+    List<HRegionInfo> regions = m.getAssignmentManager().getRegionsOfTable(tableName);
+    assertFalse("Split parent should not be assigned", regions.contains(parent));
   }
-  
+
   @Test (timeout=60000)
   public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
     byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated");
@@ -115,9 +145,8 @@ public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
     handler.process();
     abortAndStartNewMaster(cluster);
     assertTrue(cluster.getLiveMasterThreads().size() == 1);
-    
   }
-  
+
   private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException {
     cluster.abortMaster(0);
     cluster.waitOnMaster(0);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 10c87ca3131f..ddc43090de94 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -73,6 +73,7 @@
  * is tests against a bare {@link HRegion}.
  */
 @Category(LargeTests.class)
+@SuppressWarnings("deprecation")
 public class TestSplitTransactionOnCluster {
   private static final Log LOG =
     LogFactory.getLog(TestSplitTransactionOnCluster.class);
@@ -791,7 +792,6 @@ public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Excep
   @Test(timeout = 180000)
   public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
     Configuration conf = TESTING_UTIL.getConfiguration();
-    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
     String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
     HTableDescriptor htd = new HTableDescriptor(userTableName);
     HColumnDescriptor hcd = new HColumnDescriptor("col");
@@ -1084,14 +1084,18 @@ protected void startCatalogJanitorChore() {
       LOG.debug("Customised master executed.");
     }
   }
-  
+
   private static class TransitionToSplittingFailedException extends IOException {
+    private static final long serialVersionUID = 7025885032995944524L;
+
     public TransitionToSplittingFailedException() {
       super();
     }
   }
 
   private static class SplittingNodeCreationFailedException  extends IOException {
+    private static final long serialVersionUID = 1652404976265623004L;
+
     public SplittingNodeCreationFailedException () {
       super();
     }

From caf29f212ea3647067ea18462d79ea1b454f0a90 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 16 Aug 2013 21:55:30 +0000
Subject: [PATCH 1118/1540] HBASE-9233 isTableAvailable() may be stuck if an
 offline parent was never assigned

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1514904 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HConnectionManager.java | 2 +-
 .../hadoop/hbase/master/handler/TestCreateTableHandler.java     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 2ae35d2390fd..8b9f0cd1aee3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -892,7 +892,7 @@ public boolean processRow(Result row) throws IOException {
           byte[] value = row.getValue(HConstants.CATALOG_FAMILY,
               HConstants.REGIONINFO_QUALIFIER);
           HRegionInfo info = Writables.getHRegionInfoOrNull(value);
-          if (info != null) {
+          if (info != null && !info.isSplitParent()) {
             if (Bytes.equals(tableName, info.getTableName())) {
               value = row.getValue(HConstants.CATALOG_FAMILY,
                   HConstants.SERVER_QUALIFIER);
diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
index c101623bfb73..938d9e0e9d8c 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
@@ -125,6 +125,7 @@ public void testCreateTableWithSplitRegion() throws Exception {
       }
     }
     assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName));
+    assertTrue(TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName));
     List<HRegionInfo> regions = m.getAssignmentManager().getRegionsOfTable(tableName);
     assertFalse("Split parent should not be assigned", regions.contains(parent));
   }

From 8325367c1c17638463838befd0f1ef646ca31907 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sat, 17 Aug 2013 00:40:09 +0000
Subject: [PATCH 1119/1540] HBASE-9252 HConnectionManager#getZooKeeperWatcher()
 should be deprecated in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1514928 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/client/HConnectionManager.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 8b9f0cd1aee3..f8503c37301f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1497,6 +1497,7 @@ HRegionInterface getHRegionConnection(final String hostname, final int port,
      * @returns zookeeper reference
      * @throws ZooKeeperConnectionException if there's a problem connecting to zk
      */
+    @Deprecated
     public synchronized ZooKeeperWatcher getZooKeeperWatcher()
         throws ZooKeeperConnectionException {
       if(zooKeeper == null) {

From b227d97f0b6869e5c4dac072832a6cb7689a117d Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Sun, 18 Aug 2013 15:42:30 +0000
Subject: [PATCH 1120/1540] HBASE-9256 HBaseClient#setupIOStreams should handle
 all exceptions

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1515147 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/ipc/HBaseClient.java     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index bf0a6efba67e..2bdcd1b60fd1 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -445,8 +445,14 @@ protected synchronized void setupIOstreams()
 
         // start the receiver thread after the socket connection has been set up
         start();
-      } catch (IOException e) {
+      } catch (Throwable t) {
         failedServers.addToFailedServers(remoteId.address);
+        IOException e;
+        if (t instanceof IOException) {
+          e = (IOException)t;
+        } else {
+          e = new IOException("Could not set up IO Streams", t);
+        }
         markClosed(e);
         close();
 

From 279c2c3ac07b9f57f239c136a294ef621c7bda77 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 19 Aug 2013 18:45:49 +0000
Subject: [PATCH 1121/1540] HBASE-9243. Add more useful statistics in the HFile
 tool (Alexandre Normand)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1515553 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/io/hfile/HFilePrettyPrinter.java    | 93 ++++++++++++-------
 1 file changed, 59 insertions(+), 34 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index 6086408796dc..1cf23df8ddda 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -20,11 +20,18 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
+import java.io.ByteArrayOutputStream;
 import java.io.DataInput;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.SortedMap;
+
+import com.yammer.metrics.core.*;
+import com.yammer.metrics.reporting.ConsoleReporter;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -369,32 +376,14 @@ private void printMeta(HFile.Reader reader, Map<byte[], byte[]> fileInfo)
     }
   }
 
-  private static class LongStats {
-    private long min = Long.MAX_VALUE;
-    private long max = Long.MIN_VALUE;
-    private long sum = 0;
-    private long count = 0;
-
-    void collect(long d) {
-      if (d < min) min = d;
-      if (d > max) max = d;
-      sum += d;
-      count++;
-    }
-
-    public String toString() {
-      return "count: " + count +
-        "\tmin: " + min +
-        "\tmax: " + max +
-        "\tmean: " + ((double)sum/count);
-    }
-  }
-
   private static class KeyValueStatsCollector {
-    LongStats keyLen = new LongStats();
-    LongStats valLen = new LongStats();
-    LongStats rowSizeBytes = new LongStats();
-    LongStats rowSizeCols = new LongStats();
+    private final MetricsRegistry metricsRegistry = new MetricsRegistry();
+    private final ByteArrayOutputStream metricsOutput = new ByteArrayOutputStream();
+    private final SimpleReporter simpleReporter = new SimpleReporter(metricsRegistry, new PrintStream(metricsOutput));
+    Histogram keyLen = metricsRegistry.newHistogram(HFilePrettyPrinter.class, "Key length");
+    Histogram valLen = metricsRegistry.newHistogram(HFilePrettyPrinter.class, "Val length");
+    Histogram rowSizeBytes = metricsRegistry.newHistogram(HFilePrettyPrinter.class, "Row size (bytes)");
+    Histogram rowSizeCols = metricsRegistry.newHistogram(HFilePrettyPrinter.class, "Row size (columns)");
 
     long curRowBytes = 0;
     long curRowCols = 0;
@@ -403,23 +392,25 @@ private static class KeyValueStatsCollector {
 
     private KeyValue prevKV = null;
     private long maxRowBytes = 0;
+    private long curRowKeyLength;
 
     public void collect(KeyValue kv) {
-      keyLen.collect(kv.getKeyLength());
-      valLen.collect(kv.getValueLength());
+      valLen.update(kv.getValueLength());
       if (prevKV != null &&
           KeyValue.COMPARATOR.compareRows(prevKV, kv) != 0) {
         // new row
         collectRow();
       }
       curRowBytes += kv.getLength();
+      curRowKeyLength = kv.getKeyLength();
       curRowCols++;
       prevKV = kv;
     }
 
     private void collectRow() {
-      rowSizeBytes.collect(curRowBytes);
-      rowSizeCols.collect(curRowCols);
+      rowSizeBytes.update(curRowBytes);
+      rowSizeCols.update(curRowCols);
+      keyLen.update(curRowKeyLength);
 
       if (curRowBytes > maxRowBytes && prevKV != null) {
         biggestRow = prevKV.getRow();
@@ -441,12 +432,46 @@ public String toString() {
       if (prevKV == null)
         return "no data available for statistics";
 
+      // Dump the metrics to the output stream
+      simpleReporter.shutdown();
+      simpleReporter.run();
+      metricsRegistry.shutdown();
+
       return
-        "Key length: " + keyLen + "\n" +
-        "Val length: " + valLen + "\n" +
-        "Row size (bytes): " + rowSizeBytes + "\n" +
-        "Row size (columns): " + rowSizeCols + "\n" +
-        "Key of biggest row: " + Bytes.toStringBinary(biggestRow);
+              metricsOutput.toString() +
+                      "Key of biggest row: " + Bytes.toStringBinary(biggestRow);
+    }
+  }
+
+  private static class SimpleReporter extends ConsoleReporter {
+    private final PrintStream out;
+
+    public SimpleReporter(MetricsRegistry metricsRegistry, PrintStream out) {
+      super(metricsRegistry, out, MetricPredicate.ALL);
+      this.out = out;
+    }
+
+    @Override
+    public void run() {
+      for (Map.Entry<String, SortedMap<MetricName, Metric>> entry : getMetricsRegistry().groupedMetrics(
+              MetricPredicate.ALL).entrySet()) {
+        try {
+          for (Map.Entry<MetricName, Metric> subEntry : entry.getValue().entrySet()) {
+            out.print("   " + subEntry.getKey().getName());
+            out.println(':');
+
+            subEntry.getValue().processWith(this, subEntry.getKey(), out);
+          }
+        } catch (Exception e) {
+          e.printStackTrace(out);
+        }
+      }
+    }
+
+    @Override
+    public void processHistogram(MetricName name, Histogram histogram, PrintStream stream) {
+      super.processHistogram(name, histogram, stream);
+      stream.printf(Locale.getDefault(), "             count = %d\n", histogram.count());
     }
   }
 }

From 6c9530f0ea2488e22ee6e0f77ff4c16dada66578 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 20 Aug 2013 00:03:58 +0000
Subject: [PATCH 1122/1540] HBASE-9266 Javadoc: Document that
 HBaseAdmin.flush(...) is synchronous

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1515676 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index bce395dc32ec..6bdebd3f7a52 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1231,7 +1231,7 @@ public void closeRegion(final ServerName sn, final HRegionInfo hri)
 
   /**
    * Flush a table or an individual region.
-   * Asynchronous operation.
+   * Synchronous operation.
    *
    * @param tableNameOrRegionName table or region to flush
    * @throws IOException if a remote or network exception occurs
@@ -1244,7 +1244,7 @@ public void flush(final String tableNameOrRegionName)
 
   /**
    * Flush a table or an individual region.
-   * Asynchronous operation.
+   * Synchronous operation.
    *
    * @param tableNameOrRegionName table or region to flush
    * @throws IOException if a remote or network exception occurs

From dfc9fc00abb959c17f66f7dabb5f4ef15312e088 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Tue, 20 Aug 2013 14:07:33 +0000
Subject: [PATCH 1123/1540] HBASE-9195 Fix TestFDHDFSUtils against java7 test
 re-ordering (Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1515828 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
index 4c7bc97a3940..7d477bad2677 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
@@ -39,7 +39,6 @@ public class TestFSHDFSUtils {
   private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
   static {
     Configuration conf = HTU.getConfiguration();
-    conf.setInt("hbase.lease.recovery.dfs.timeout", 1000);
     conf.setInt("hbase.lease.recovery.first.pause", 10);
     conf.setInt("hbase.lease.recovery.pause", 10);
   };
@@ -58,6 +57,7 @@ public void setup() {
    */
   @Test (timeout = 30000)
   public void testRecoverLease() throws IOException {
+    HTU.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 1000);
     DistributedFileSystem dfs = Mockito.mock(DistributedFileSystem.class);
     // Fail four times and pass on the fifth.
     Mockito.when(dfs.recoverLease(FILE)).

From 35d419bf78be34d4ec1bf67bc949e1c70389b08e Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 20 Aug 2013 16:55:46 +0000
Subject: [PATCH 1124/1540] HBASE-9167 ServerCallable retries just once if
 timeout is not integer.max

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1515874 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ServerCallable.java   | 62 +++++++++++++------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
index f238eb04a24d..14d1290738f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.ipc.RemoteException;
 
 /**
@@ -56,7 +57,9 @@ public abstract class ServerCallable<T> implements Callable<T> {
   protected HRegionLocation location;
   protected HRegionInterface server;
   protected int callTimeout;
+  protected long globalStartTime;
   protected long startTime, endTime;
+  protected final static int MIN_RPC_TIMEOUT = 2000;
 
   /**
    * @param connection Connection to use.
@@ -109,27 +112,20 @@ public byte[] getRegionName() {
   }
 
   public void beforeCall() {
-    HBaseRPC.setRpcTimeout(this.callTimeout);
-    this.startTime = System.currentTimeMillis();
+    this.startTime = EnvironmentEdgeManager.currentTimeMillis();
+    int remaining = (int)(callTimeout - (this.startTime - this.globalStartTime));
+    if (remaining < MIN_RPC_TIMEOUT) {
+      // If there is no time left, we're trying anyway. It's too late.
+      // 0 means no timeout, and it's not the intent here. So we secure both cases by
+      // resetting to the minimum.
+      remaining = MIN_RPC_TIMEOUT;
+    }
+    HBaseRPC.setRpcTimeout(remaining);
   }
 
   public void afterCall() {
     HBaseRPC.resetRpcTimeout();
-    this.endTime = System.currentTimeMillis();
-  }
-
-  public void shouldRetry(Throwable throwable) throws IOException {
-    if (this.callTimeout != HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT)
-      if (throwable instanceof SocketTimeoutException
-          || (this.endTime - this.startTime > this.callTimeout)) {
-        throw (SocketTimeoutException) (SocketTimeoutException) new SocketTimeoutException(
-            "Call to access row '" + Bytes.toString(row) + "' on table '"
-                + Bytes.toString(tableName)
-                + "' failed on socket timeout exception: " + throwable)
-            .initCause(throwable);
-      } else {
-        this.callTimeout = ((int) (this.endTime - this.startTime));
-      }
+    this.endTime = EnvironmentEdgeManager.currentTimeMillis();
   }
 
   /**
@@ -157,13 +153,15 @@ public T withRetries()
       HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
     List<RetriesExhaustedException.ThrowableWithExtraContext> exceptions =
       new ArrayList<RetriesExhaustedException.ThrowableWithExtraContext>();
+    globalStartTime = EnvironmentEdgeManager.currentTimeMillis();
+    long expectedSleep = 0;
+
     for (int tries = 0; tries < numRetries; tries++) {
       try {
         beforeCall();
         connect(tries != 0);
         return call();
       } catch (Throwable t) {
-        shouldRetry(t);
         t = translateException(t);
         if (t instanceof SocketTimeoutException ||
             t instanceof ConnectException ||
@@ -182,16 +180,30 @@ public T withRetries()
         }
         RetriesExhaustedException.ThrowableWithExtraContext qt =
           new RetriesExhaustedException.ThrowableWithExtraContext(t,
-            System.currentTimeMillis(), toString());
+            EnvironmentEdgeManager.currentTimeMillis(), toString());
         exceptions.add(qt);
         if (tries == numRetries - 1) {
           throw new RetriesExhaustedException(tries, exceptions);
         }
+        // If the server is dead, we need to wait a little before retrying, to give
+        //  a chance to the regions to be
+        // tries hasn't been bumped up yet so we use "tries + 1" to get right pause time
+        expectedSleep = ConnectionUtils.getPauseTime(pause, tries + 1);
+
+        // If, after the planned sleep, there won't be enough time left, we stop now.
+        long duration = singleCallDuration(expectedSleep);
+        if (duration > this.callTimeout) {
+          throw (SocketTimeoutException) new SocketTimeoutException(
+            "Call to access row '" + Bytes.toString(row) + "' on table '"
+              + Bytes.toString(tableName) + "' failed on timeout. "
+              + " callTimeout=" + this.callTimeout + ", callDuration="
+              + duration).initCause(t);
+        }
       } finally {
         afterCall();
       }
       try {
-        Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
+        Thread.sleep(expectedSleep);
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
         throw new IOException("Giving up after tries=" + tries, e);
@@ -209,6 +221,7 @@ public T withRetries()
    */
   public T withoutRetries()
   throws IOException, RuntimeException {
+    globalStartTime = EnvironmentEdgeManager.currentTimeMillis();
     try {
       beforeCall();
       connect(false);
@@ -225,6 +238,15 @@ public T withoutRetries()
     }
   }
 
+  /**
+   * @param expectedSleep
+   * @return Calculate how long a single call took
+   */
+  private long singleCallDuration(final long expectedSleep) {
+    return (EnvironmentEdgeManager.currentTimeMillis() - this.globalStartTime)
+      + MIN_RPC_TIMEOUT + expectedSleep;
+  }
+
   private static Throwable translateException(Throwable t) throws IOException {
     if (t instanceof UndeclaredThrowableException) {
       t = t.getCause();

From 4d080e10d82f17aebd14e2ec085660bd0d07b2f4 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 20 Aug 2013 20:49:38 +0000
Subject: [PATCH 1125/1540] HBASE-8760 possible loss of data in snapshot taken
 after region split

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1515967 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/HFileLink.java |  41 ++++++-
 .../master/handler/CreateTableHandler.java    |  10 +-
 .../master/snapshot/CloneSnapshotHandler.java |  12 +-
 .../DisabledTableSnapshotHandler.java         |  24 +---
 .../snapshot/EnabledTableSnapshotHandler.java |  25 +++-
 .../snapshot/MasterSnapshotVerifier.java      | 116 ++++++++++--------
 .../snapshot/RestoreSnapshotHandler.java      |   1 +
 .../master/snapshot/TakeSnapshotHandler.java  |  38 +++++-
 .../hadoop/hbase/regionserver/HRegion.java    |  20 ++-
 .../hadoop/hbase/regionserver/StoreFile.java  |   8 ++
 .../hadoop/hbase/snapshot/ExportSnapshot.java |   4 +-
 .../hbase/snapshot/RestoreSnapshotHelper.java |  99 +++++++++++++--
 .../hadoop/hbase/snapshot/SnapshotInfo.java   |   3 +-
 13 files changed, 303 insertions(+), 98 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
index 59bf57834048..749273812838 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -108,6 +108,37 @@ public HFileLink(final Path rootDir, final Path archiveDir, final Path path) {
     setLocations(originPath, tempPath, archivePath);
   }
 
+  /**
+   * Create an HFileLink relative path for the table/region/family/hfile location
+   * @param table Table name
+   * @param region Region Name
+   * @param family Family Name
+   * @param hfile HFile Name
+   * @return the relative Path to open the specified table/region/family/hfile link
+   */
+  public static Path createPath(final String table, final String region,
+      final String family, final String hfile) {
+    if (HFileLink.isHFileLink(hfile)) {
+      return new Path(family, hfile);
+    }
+    return new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
+  }
+
+  /**
+   * Create an HFileLink instance from table/region/family/hfile location
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param table Table name
+   * @param region Region Name
+   * @param family Family Name
+   * @param hfile HFile Name
+   * @return Link to the file with the specified table/region/family/hfile location
+   * @throws IOException on unexpected error.
+   */
+  public static HFileLink create(final Configuration conf, final String table,
+      final String region, final String family, final String hfile) throws IOException {
+    return new HFileLink(conf, createPath(table, region, family, hfile));
+  }
+
   /**
    * @return the origin path of the hfile.
    */
@@ -130,7 +161,6 @@ public static boolean isHFileLink(final Path path) {
     return isHFileLink(path.getName());
   }
 
-
   /**
    * @param fileName File name to check.
    * @return True if the path is a HFileLink.
@@ -208,6 +238,15 @@ public static String getReferencedTableName(final String fileName) {
     return(m.group(1));
   }
 
+  /**
+   * Returns true if the HFileLink exists
+   */
+  public boolean exists(final FileSystem fs) throws IOException {
+    return fs.exists(this.originPath) ||
+           fs.exists(this.tempPath) ||
+           fs.exists(this.archivePath);
+  }
+
   /**
    * Create a new HFileLink name
    *
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
index 2e2ebfbb6149..92d28c31229b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -188,7 +188,7 @@ private void handleCreateTable(String tableName) throws IOException, KeeperExcep
     
     if (regionInfos != null && regionInfos.size() > 0) {
       // 4. Add regions to META
-      MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
+      addRegionsToMeta(this.catalogTracker, regionInfos);
 
       // 5. Trigger immediate assignment of the regions in round-robin fashion
       List<ServerName> servers = serverManager.getOnlineServersList();
@@ -225,4 +225,12 @@ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
     return ModifyRegionUtils.createRegions(conf, tableRootDir,
         hTableDescriptor, newRegions, null);
   }
+
+  /**
+   * Add the specified set of regions to the META table.
+   */
+  protected void addRegionsToMeta(final CatalogTracker ct, final List<HRegionInfo> regionInfos)
+      throws IOException {
+    MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
index bd0349f71660..58fbf1d76e0e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.master.MasterServices;
@@ -67,6 +68,8 @@ public class CloneSnapshotHandler extends CreateTableHandler implements Snapshot
   private final MasterMetrics metricsMaster;
   private final MonitoredTask status;
 
+  private RestoreSnapshotHelper.RestoreMetaChanges metaChanges;
+
   private volatile boolean stopped = false;
 
   public CloneSnapshotHandler(final MasterServices masterServices,
@@ -106,7 +109,7 @@ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
       RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
           snapshot, snapshotDir, hTableDescriptor, tableDir, monitor, status);
-      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+      metaChanges = restoreHelper.restoreHdfsRegions();
 
       // Clone operation should not have stuff to restore or remove
       Preconditions.checkArgument(!metaChanges.hasRegionsToRestore(),
@@ -132,6 +135,13 @@ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
     }
   }
 
+  @Override
+  protected void addRegionsToMeta(final CatalogTracker ct, final List<HRegionInfo> regionInfos)
+      throws IOException {
+    super.addRegionsToMeta(ct, regionInfos);
+    metaChanges.updateMetaParentRegions(ct, regionInfos);
+  }
+
   @Override
   protected void completed(final Throwable exception) {
     this.stopped = true;
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
index eee454846dc1..b5720c0699ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -37,8 +37,6 @@
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
-import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
 import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
@@ -81,36 +79,16 @@ public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocati
       // 1. get all the regions hosting this table.
 
       // extract each pair to separate lists
-      Set<String> serverNames = new HashSet<String>();
       Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
       for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
         regions.add(p.getFirst());
-        serverNames.add(p.getSecond().toString());
       }
 
       // 2. for each region, write all the info to disk
       LOG.info("Starting to write region info and WALs for regions for offline snapshot:"
           + SnapshotDescriptionUtils.toString(snapshot));
       for (HRegionInfo regionInfo : regions) {
-        // 2.1 copy the regionInfo files to the snapshot
-        Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
-          regionInfo.getEncodedName());
-        HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
-        // check for error for each region
-        monitor.rethrowException();
-
-        // 2.2 for each region, copy over its recovered.edits directory
-        Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
-        new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
-        monitor.rethrowException();
-        status.setStatus("Completed copying recovered edits for offline snapshot of table: "
-            + snapshot.getTable());
-
-        // 2.3 reference all the files in the region
-        new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
-        monitor.rethrowException();
-        status.setStatus("Completed referencing HFiles for offline snapshot of table: " +
-          snapshot.getTable());
+        snapshotDisabledRegion(regionInfo);
       }
 
       // 3. write the table info to disk
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
index 4bbdef3cbe30..3d7828576f23 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
@@ -33,7 +34,9 @@
 import org.apache.hadoop.hbase.procedure.Procedure;
 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.Pair;
 
 import com.google.common.collect.Lists;
@@ -65,10 +68,14 @@ public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices
    */
   @Override
   protected void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regions)
-      throws HBaseSnapshotException {
+      throws HBaseSnapshotException, IOException {
     Set<String> regionServers = new HashSet<String>(regions.size());
     for (Pair<HRegionInfo, ServerName> region : regions) {
-      regionServers.add(region.getSecond().toString());
+      if (region != null && region.getFirst() != null && region.getSecond() != null) {
+        HRegionInfo hri = region.getFirst();
+        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) continue;
+        regionServers.add(region.getSecond().toString());
+      }
     }
 
     // start the snapshot on the RS
@@ -85,7 +92,19 @@ protected void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regions)
       // wait for the snapshot to complete.  A timer thread is kicked off that should cancel this
       // if it takes too long.
       proc.waitForCompleted();
-      LOG.info("Done waiting - snapshot for " + this.snapshot.getName() + " finished!");
+      LOG.info("Done waiting - online snapshot for " + this.snapshot.getName());
+
+      // Take the offline regions as disabled
+      Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+      for (Pair<HRegionInfo, ServerName> region : regions) {
+        HRegionInfo regionInfo = region.getFirst();
+        if (regionInfo.isOffline() && (regionInfo.isSplit() || regionInfo.isSplitParent())) {
+          if (!fs.exists(new Path(snapshotDir, regionInfo.getEncodedName()))) {
+            LOG.info("Take disabled snapshot of offline region=" + regionInfo);
+            snapshotDisabledRegion(regionInfo);
+          }
+        }
+      }
     } catch (InterruptedException e) {
       ForeignException ee =
           new ForeignException("Interrupted while waiting for snapshot to finish", e);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index db89e66441ff..d4ce7c01533e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -21,6 +21,8 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -32,6 +34,7 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
@@ -39,10 +42,12 @@
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
 import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 
 /**
@@ -76,6 +81,7 @@
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public final class MasterSnapshotVerifier {
+  private static final Log LOG = LogFactory.getLog(MasterSnapshotVerifier.class);
 
   private SnapshotDescription snapshot;
   private FileSystem fs;
@@ -143,10 +149,28 @@ private void verifyTableInfo(Path snapshotDir) throws IOException {
   private void verifyRegions(Path snapshotDir) throws IOException {
     List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
       Bytes.toBytes(tableName));
+
+    Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
+    if (snapshotRegions == null) {
+      String msg = "Snapshot " + SnapshotDescriptionUtils.toString(snapshot) + " looks empty";
+      LOG.error(msg);
+      throw new CorruptedSnapshotException(msg);
+    }
+
+    if (snapshotRegions.size() != regions.size()) {
+      String msg = "Regions moved during the snapshot '" + 
+                   SnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
+                   regions.size() + " snapshotted=" + snapshotRegions.size();
+      LOG.error(msg);
+      throw new CorruptedSnapshotException(msg);
+    }
+
     for (HRegionInfo region : regions) {
-      // if offline split parent, skip it
-      if (region.isOffline() && (region.isSplit() || region.isSplitParent())) {
-        continue;
+      if (!snapshotRegions.contains(region.getEncodedName())) {
+        // could happen due to a move or split race.
+        String msg = "No region directory found for region:" + region;
+        LOG.error(msg);
+        throw new CorruptedSnapshotException(msg, snapshot);
       }
 
       verifyRegion(fs, snapshotDir, region);
@@ -159,20 +183,18 @@ private void verifyRegions(Path snapshotDir) throws IOException {
    * @param snapshotDir snapshot directory to check
    * @param region the region to check
    */
-  private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) throws IOException {
+  private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region)
+      throws IOException {
     // make sure we have region in the snapshot
     Path regionDir = new Path(snapshotDir, region.getEncodedName());
-    if (!fs.exists(regionDir)) {
-      // could happen due to a move or split race.
-      throw new CorruptedSnapshotException("No region directory found for region:" + region,
-          snapshot);
-    }
+
     // make sure we have the region info in the snapshot
     Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
     // make sure the file exists
     if (!fs.exists(regionInfo)) {
       throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
     }
+
     FSDataInputStream in = fs.open(regionInfo);
     HRegionInfo found = new HRegionInfo();
     try {
@@ -188,46 +210,44 @@ private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) t
     // make sure we have the expected recovered edits files
     TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
 
-    // check for the existance of each hfile
-    PathFilter familiesDirs = new FSUtils.FamilyDirFilter(fs);
-    FileStatus[] columnFamilies = FSUtils.listStatus(fs, regionDir, familiesDirs);
-    // should we do some checking here to make sure the cfs are correct?
-    if (columnFamilies == null) return;
-
-    // setup the suffixes for the snapshot directories
-    Path tableNameSuffix = new Path(tableName);
-    Path regionNameSuffix = new Path(tableNameSuffix, region.getEncodedName());
-
-    // get the potential real paths
-    Path archivedRegion = new Path(HFileArchiveUtil.getArchivePath(services.getConfiguration()),
-        regionNameSuffix);
-    Path realRegion = new Path(rootDir, regionNameSuffix);
-
-    // loop through each cf and check we can find each of the hfiles
-    for (FileStatus cf : columnFamilies) {
-      FileStatus[] hfiles = FSUtils.listStatus(fs, cf.getPath(), null);
-      // should we check if there should be hfiles?
-      if (hfiles == null || hfiles.length == 0) continue;
-
-      Path realCfDir = new Path(realRegion, cf.getPath().getName());
-      Path archivedCfDir = new Path(archivedRegion, cf.getPath().getName());
-      for (FileStatus hfile : hfiles) {
-        // make sure the name is correct
-        if (!StoreFile.validateStoreFileName(hfile.getPath().getName())) {
-          throw new CorruptedSnapshotException("HFile: " + hfile.getPath()
-              + " is not a valid hfile name.", snapshot);
-        }
-
-        // check to see if hfile is present in the real table
-        String fileName = hfile.getPath().getName();
-        Path file = new Path(realCfDir, fileName);
-        Path archived = new Path(archivedCfDir, fileName);
-        if (!fs.exists(file) && !fs.exists(archived)) {
-          throw new CorruptedSnapshotException("Can't find hfile: " + hfile.getPath()
-              + " in the real (" + realCfDir + ") or archive (" + archivedCfDir
-              + ") directory for the primary table.", snapshot);
-        }
+    // make sure we have all the expected store files
+    SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile(final String regionNameSuffix, final String family,
+          final String hfileName) throws IOException {
+        verifyStoreFile(snapshotDir, region, family, hfileName);
       }
+    });
+  }
+
+  private void verifyStoreFile(final Path snapshotDir, final HRegionInfo regionInfo,
+      final String family, final String fileName) throws IOException {
+    Path refPath = null;
+    if (StoreFile.isReference(fileName)) {
+      // If is a reference file check if the parent file is present in the snapshot
+      Path snapshotHFilePath = new Path(new Path(
+          new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName);
+      refPath = StoreFile.getReferredToFile(snapshotHFilePath);
+      if (!fs.exists(refPath)) {
+        throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot);
+      }
+    }
+
+    Path linkPath;
+    if (refPath != null && HFileLink.isHFileLink(refPath)) {
+      linkPath = new Path(family, refPath.getName());
+    } else if (HFileLink.isHFileLink(fileName)) {
+      linkPath = new Path(family, fileName);
+    } else {
+      linkPath = new Path(family, HFileLink.createHFileLinkName(tableName,
+        regionInfo.getEncodedName(), fileName));
+    }
+
+    // check if the linked file exists (in the archive, or in the table dir)
+    HFileLink link = new HFileLink(services.getConfiguration(), linkPath);
+    if (!link.exists(fs)) {
+      throw new CorruptedSnapshotException("Can't find hfile: " + fileName
+          + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath()
+          + ") directory for the primary table.", snapshot);
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index 1660972b316e..f39a45ecf627 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -125,6 +125,7 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
       if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
       List<HRegionInfo> hrisToRemove = metaChanges.getRegionsToRemove();
       MetaEditor.mutateRegions(catalogTracker, hrisToRemove, hris);
+      metaChanges.updateMetaParentRegions(catalogTracker, hris);
 
       // At this point the restore is complete. Next step is enabling the table.
       LOG.info("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " on table=" +
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index e8d9abb07d9d..75ff129f4851 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -44,9 +44,13 @@
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
+import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.zookeeper.KeeperException;
@@ -143,7 +147,7 @@ public void process() {
 
       List<Pair<HRegionInfo, ServerName>> regionsAndLocations =
           MetaReader.getTableRegionsAndLocations(this.server.getCatalogTracker(),
-            Bytes.toBytes(snapshot.getTable()), true);
+            Bytes.toBytes(snapshot.getTable()), false);
 
       // run the snapshot
       snapshotRegions(regionsAndLocations);
@@ -152,7 +156,11 @@ public void process() {
       // extract each pair to separate lists
       Set<String> serverNames = new HashSet<String>();
       for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
-        serverNames.add(p.getSecond().toString());
+        if (p != null && p.getFirst() != null && p.getSecond() != null) {
+          HRegionInfo hri = p.getFirst();
+          if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) continue;
+          serverNames.add(p.getSecond().toString());
+        }
       }
 
       // verify the snapshot is valid
@@ -209,6 +217,32 @@ public void completeSnapshot(Path snapshotDir, Path workingDir, FileSystem fs)
     finished = true;
   }
 
+  /**
+   * Take a snapshot of the specified disabled region
+   */
+  protected void snapshotDisabledRegion(final HRegionInfo regionInfo)
+      throws IOException {
+    // 1 copy the regionInfo files to the snapshot
+    Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
+      regionInfo.getEncodedName());
+    HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
+    // check for error for each region
+    monitor.rethrowException();
+
+    // 2 for each region, copy over its recovered.edits directory
+    Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+    new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
+    monitor.rethrowException();
+    status.setStatus("Completed copying recovered edits for offline snapshot of table: "
+        + snapshot.getTable());
+
+    // 3 reference all the files in the region
+    new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+    monitor.rethrowException();
+    status.setStatus("Completed referencing HFiles for offline snapshot of table: " +
+      snapshot.getTable());
+  }
+
   /**
    * Snapshot the specified regions
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9dc9fde86b2d..b244f38e77ce 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2737,14 +2737,22 @@ public void addRegionToSnapshot(SnapshotDescription desc,
         if (exnSnare != null) {
           exnSnare.rethrowException();
         }
-        Path file = storeFiles.get(i).getPath();
-        // create "reference" to this store file.  It is intentionally an empty file -- all
-        // necessary infomration is captured by its fs location and filename.  This allows us to
-        // only figure out what needs to be done via a single nn operation (instead of having to
-        // open and read the files as well).
+        StoreFile storeFile = storeFiles.get(i);
+        Path file = storeFile.getPath();
+
         LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
         Path referenceFile = new Path(dstStoreDir, file.getName());
-        boolean success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
+        boolean success = true;
+        if (storeFile.isReference()) {
+          // write the Reference object to the snapshot
+          storeFile.getReference().write(fs, referenceFile);
+        } else {
+          // create "reference" to this store file.  It is intentionally an empty file -- all
+          // necessary information is captured by its fs location and filename.  This allows us to
+          // only figure out what needs to be done via a single nn operation (instead of having to
+          // open and read the files as well).
+          success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
+        }
         if (!success) {
           throw new IOException("Failed to create reference file:" + referenceFile);
         }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 759672f9206b..37e2a4c8e93c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -321,6 +321,14 @@ boolean isReference() {
     return this.reference != null;
   }
 
+  /**
+   * @return the Reference object associated to this StoreFile.
+   *         null if the StoreFile is not a reference.
+   */
+  Reference getReference() {
+    return this.reference;
+  }
+
   /**
    * @return <tt>true</tt> if this StoreFile is an HFileLink
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index ab8656cc60d1..f14cd4eb4d54 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -388,7 +388,7 @@ private List<Pair<Path, Long>> getSnapshotFiles(final FileSystem fs, final Path
       new SnapshotReferenceUtil.FileVisitor() {
         public void storeFile (final String region, final String family, final String hfile)
             throws IOException {
-          Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
+          Path path = HFileLink.createPath(table, region, family, hfile);
           long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
           files.add(new Pair<Path, Long>(path, size));
         }
@@ -627,7 +627,7 @@ public int run(String[] args) throws Exception {
     if (outputFs.exists(snapshotTmpDir)) {
       System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
       System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
-      System.err.println("consider removing " + snapshotTmpDir + " before retrying export"); 
+      System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
       return 1;
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index a81cca6a9c81..a5dd2e50b689 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -41,6 +41,8 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
@@ -52,6 +54,7 @@
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.io.IOUtils;
 
 /**
@@ -102,6 +105,9 @@ public class RestoreSnapshotHelper {
   private final Map<byte[], byte[]> regionsMap =
         new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
 
+  private final Map<String, Pair<String, String> > parentsMap =
+      new HashMap<String, Pair<String, String> >();
+
   private final ForeignExceptionDispatcher monitor;
   private final MonitoredTask status;
 
@@ -141,7 +147,7 @@ public RestoreMetaChanges restoreHdfsRegions() throws IOException {
       return null;
     }
 
-    RestoreMetaChanges metaChanges = new RestoreMetaChanges();
+    RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
 
     // Identify which region are still available and which not.
     // NOTE: we rely upon the region name as: "table name, start key, end key"
@@ -205,10 +211,16 @@ public RestoreMetaChanges restoreHdfsRegions() throws IOException {
    * Describe the set of operations needed to update META after restore.
    */
   public static class RestoreMetaChanges {
+    private final Map<String, Pair<String, String> > parentsMap;
+
     private List<HRegionInfo> regionsToRestore = null;
     private List<HRegionInfo> regionsToRemove = null;
     private List<HRegionInfo> regionsToAdd = null;
 
+    RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
+      this.parentsMap = parentsMap;
+    }
+
     /**
      * @return true if there're new regions
      */
@@ -280,6 +292,50 @@ void addRegionToRestore(final HRegionInfo hri) {
       }
       regionsToRestore.add(hri);
     }
+
+    public void updateMetaParentRegions(final CatalogTracker catalogTracker,
+        final List<HRegionInfo> regionInfos) throws IOException {
+      if (regionInfos == null || parentsMap.isEmpty()) return;
+
+      // Extract region names and offlined regions
+      Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
+      List<HRegionInfo> parentRegions = new LinkedList();
+      for (HRegionInfo regionInfo: regionInfos) {
+        if (regionInfo.isSplitParent()) {
+          parentRegions.add(regionInfo);
+        } else {
+          regionsByName.put(regionInfo.getEncodedName(), regionInfo);
+        }
+      }
+
+      // Update Offline parents
+      for (HRegionInfo regionInfo: parentRegions) {
+        Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
+
+        // TODO-REMOVE-ME: HConnectionManager.isTableAvailable() is checking the SERVER_QUALIFIER
+        // also on offline regions, so to keep the compatibility with older clients we must add
+        // a location to this region even if it will never be assigned. (See HBASE-9233)
+        MetaEditor.updateRegionLocation(catalogTracker, regionInfo,
+                                        catalogTracker.getMetaLocation());
+
+        if (daughters == null) {
+          // The snapshot contains an unreferenced region.
+          // It will be removed by the CatalogJanitor.
+          LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
+          continue;
+        }
+
+        // One side of the split is already compacted
+        if (daughters.getSecond() == null) {
+          daughters.setSecond(daughters.getFirst());
+        }
+
+        LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
+        MetaEditor.offlineParentInMeta(catalogTracker, regionInfo,
+            regionsByName.get(daughters.getFirst()),
+            regionsByName.get(daughters.getSecond()));
+      }
+    }
   }
 
   /**
@@ -492,9 +548,10 @@ private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo
   private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
       final String hfileName) throws IOException {
     // Extract the referred information (hfile name and parent region)
-    String tableName = snapshotDesc.getTable();
-    Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(tableName,
-        regionInfo.getEncodedName()), familyDir.getName()), hfileName));
+    String snapshotTable = snapshotDesc.getTable();
+    Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(
+        snapshotTable, regionInfo.getEncodedName()), familyDir.getName()),
+        hfileName));
     String snapshotRegionName = refPath.getParent().getParent().getName();
     String fileName = refPath.getName();
 
@@ -503,18 +560,40 @@ private void restoreReferenceFile(final Path familyDir, final HRegionInfo region
     if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
 
     // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
+    Path linkPath = null;
     String refLink = fileName;
     if (!HFileLink.isHFileLink(fileName)) {
-      refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
+      refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
+      linkPath = new Path(familyDir,
+        HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
     }
+
     Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
 
     // Create the new reference
-    Path linkPath = new Path(familyDir,
-      HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
-    InputStream in = new HFileLink(conf, linkPath).open(fs);
+    InputStream in;
+    if (linkPath != null) {
+      in = new HFileLink(conf, linkPath).open(fs);
+    } else {
+      linkPath = new Path(new Path(HRegion.getRegionDir(snapshotDir, regionInfo.getEncodedName()),
+                      familyDir.getName()), hfileName);
+      in = fs.open(linkPath);
+    }
     OutputStream out = fs.create(outPath);
     IOUtils.copyBytes(in, out, conf);
+
+    // Add the daughter region to the map
+    String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
+    LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
+    synchronized (parentsMap) {
+      Pair<String, String> daughters = parentsMap.get(clonedRegionName);
+      if (daughters == null) {
+        daughters = new Pair<String, String>(regionName, null);
+        parentsMap.put(clonedRegionName, daughters);
+      } else if (!regionName.equals(daughters.getFirst())) {
+        daughters.setSecond(regionName);
+      }
+    }
   }
 
   /**
@@ -526,9 +605,11 @@ private void restoreReferenceFile(final Path familyDir, final HRegionInfo region
    * @return the new HRegion instance
    */
   public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
-    return new HRegionInfo(tableDesc.getName(),
+    HRegionInfo regionInfo = new HRegionInfo(tableDesc.getName(),
                       snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
                       snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
+    regionInfo.setOffline(snapshotRegionInfo.isOffline());
+    return regionInfo;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index 7caad2b251d7..b3c4b80e54fe 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -188,8 +188,7 @@ public long getLogsSize() {
     FileInfo addStoreFile(final String region, final String family, final String hfile)
           throws IOException {
       String table = this.snapshot.getTable();
-      Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
-      HFileLink link = new HFileLink(conf, path);
+      HFileLink link = HFileLink.create(conf, table, region, family, hfile);
       boolean inArchive = false;
       long size = -1;
       try {

From 3148ce2c8ca883ef9c41960d9ac705f5cd4e44c8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 20 Aug 2013 23:36:33 +0000
Subject: [PATCH 1126/1540] roll release to 0.94.12-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1516028 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 9a4b398822aa..9bd36b495d8b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.11</version>
+  <version>0.94.12-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 3c80d2657599b4c8e9c00fd25344aceba2bb36c7 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 21 Aug 2013 01:00:54 +0000
Subject: [PATCH 1127/1540] HBASE-9277. REST should use getTableNames to list
 tables

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1516047 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/rest/RootResource.java    | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
index e1e6d72203f8..e26575a69879 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
@@ -35,7 +35,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.rest.model.TableListModel;
 import org.apache.hadoop.hbase.rest.model.TableModel;
 
@@ -60,9 +59,9 @@ public RootResource() throws IOException {
 
   private final TableListModel getTableList() throws IOException {
     TableListModel tableList = new TableListModel();
-    HTableDescriptor[] list = servlet.getAdmin().listTables();
-    for (HTableDescriptor htd: list) {
-      tableList.add(new TableModel(htd.getNameAsString()));
+    String[] tableNames = servlet.getAdmin().getTableNames();
+    for (String name: tableNames) {
+      tableList.add(new TableModel(name));
     }
     return tableList;
   }

From 792800ebbbe2add812625959cf33891e06ebb1f9 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 21 Aug 2013 01:23:04 +0000
Subject: [PATCH 1128/1540] HBASE-9279. Thrift should use getTableNames to list
 tables

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1516055 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/thrift/ThriftServerRunner.java    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index de2a81918e97..4aaa5657a939 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -573,10 +573,10 @@ public void majorCompact(ByteBuffer tableNameOrRegionName) throws IOError {
     @Override
     public List<ByteBuffer> getTableNames() throws IOError {
       try {
-        HTableDescriptor[] tables = this.admin.listTables();
-        ArrayList<ByteBuffer> list = new ArrayList<ByteBuffer>(tables.length);
-        for (int i = 0; i < tables.length; i++) {
-          list.add(ByteBuffer.wrap(tables[i].getName()));
+        String[] tableNames = this.admin.getTableNames();
+        ArrayList<ByteBuffer> list = new ArrayList<ByteBuffer>(tableNames.length);
+        for (int i = 0; i < tableNames.length; i++) {
+          list.add(ByteBuffer.wrap(Bytes.toBytes(tableNames[i])));
         }
         return list;
       } catch (IOException e) {

From 48633dd609193985dc4e6d74179081c15fec3b26 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Wed, 21 Aug 2013 20:21:10 +0000
Subject: [PATCH 1129/1540] HBASE-9287 TestCatalogTracker depends on the
 execution order

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1516290 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/catalog/TestCatalogTracker.java     | 303 ++++++++----------
 1 file changed, 139 insertions(+), 164 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
index 5071a89237f3..39ef3a0e3d2a 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
@@ -98,6 +98,17 @@ public boolean isAborted()  {
   }
 
   @After public void after() {
+    try {
+      // Clean out root location or later tests will be confused... they presume
+      // start fresh in zk.
+      RootLocationEditor.deleteRootLocation(this.watcher);
+    } catch (KeeperException e) {
+      LOG.warn("Unable to delete root location", e);
+    }
+
+    // Clear out our doctored connection or could mess up subsequent tests.
+    HConnectionManager.deleteConnection(UTIL.getConfiguration());
+
     this.watcher.close();
   }
 
@@ -119,14 +130,9 @@ private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
   throws IOException, InterruptedException, KeeperException {
     HConnection connection = Mockito.mock(HConnection.class);
     constructAndStartCatalogTracker(connection);
-    try {
-      RootLocationEditor.setRootLocation(this.watcher,
-        new ServerName("example.com", 1234, System.currentTimeMillis()));
-    } finally {
-      // Clean out root location or later tests will be confused... they presume
-      // start fresh in zk.
-      RootLocationEditor.deleteRootLocation(this.watcher);
-    }
+
+    RootLocationEditor.setRootLocation(this.watcher,
+      new ServerName("example.com", 1234, System.currentTimeMillis()));
   }
 
   /**
@@ -138,33 +144,30 @@ private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
   throws IOException, InterruptedException {
     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
     HConnection connection = mockConnection(implementation);
-    try {
-      final CatalogTracker ct = constructAndStartCatalogTracker(connection);
-      ServerName hsa = ct.getRootLocation();
-      Assert.assertNull(hsa);
-      ServerName meta = ct.getMetaLocation();
-      Assert.assertNull(meta);
-      Thread t = new Thread() {
-        @Override
-        public void run() {
-          try {
-            ct.waitForMeta();
-          } catch (InterruptedException e) {
-            throw new RuntimeException("Interrupted", e);
-          }
+
+    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+    ServerName hsa = ct.getRootLocation();
+    Assert.assertNull(hsa);
+    ServerName meta = ct.getMetaLocation();
+    Assert.assertNull(meta);
+    Thread t = new Thread() {
+      @Override
+      public void run() {
+        try {
+          ct.waitForMeta();
+        } catch (InterruptedException e) {
+          throw new RuntimeException("Interrupted", e);
         }
-      };
-      t.start();
-      while (!t.isAlive())
-        Threads.sleep(1);
+      }
+    };
+    t.start();
+    while (!t.isAlive())
       Threads.sleep(1);
-      assertTrue(t.isAlive());
-      ct.stop();
-      // Join the thread... should exit shortly.
-      t.join();
-    } finally {
-      HConnectionManager.deleteConnection(UTIL.getConfiguration());
-    }
+    Threads.sleep(1);
+    assertTrue(t.isAlive());
+    ct.stop();
+    // Join the thread... should exit shortly.
+    t.join();
   }
 
   /**
@@ -181,63 +184,58 @@ public void testServerNotRunningIOException()
     // Mock an HRegionInterface.
     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
     HConnection connection = mockConnection(implementation);
+
+    // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
+    // the first time.  'Succeed' the second time we are called.
+    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
+      thenThrow(new IOException("Server not running, aborting")).
+      thenReturn(new HRegionInfo());
+
+    // After we encounter the above 'Server not running', we should catch the
+    // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
+    // get new meta location.  Return something so this 'get' succeeds
+    // (here we mock up getRegionServerWithRetries, the wrapper around
+    // the actual get).
+
+    // TODO: Refactor.  This method has been moved out of HConnection.
+    // It works for now but has been deprecated.
+    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
+      thenReturn(getMetaTableRowResult());
+
+    // Now start up the catalogtracker with our doctored Connection.
+    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
     try {
-      // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
-      // the first time.  'Succeed' the second time we are called.
-      Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
-        thenThrow(new IOException("Server not running, aborting")).
-        thenReturn(new HRegionInfo());
-
-      // After we encounter the above 'Server not running', we should catch the
-      // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
-      // get new meta location.  Return something so this 'get' succeeds
-      // (here we mock up getRegionServerWithRetries, the wrapper around
-      // the actual get).
-
-      // TODO: Refactor.  This method has been moved out of HConnection.
-      // It works for now but has been deprecated.
-      Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
-        thenReturn(getMetaTableRowResult());
-
-      // Now start up the catalogtracker with our doctored Connection.
-      final CatalogTracker ct = constructAndStartCatalogTracker(connection);
-      try {
-        // Set a location for root and meta.
-        RootLocationEditor.setRootLocation(this.watcher, SN);
-        ct.setMetaLocation(SN);
-        // Call the method that HBASE-4288 calls.  It will try and verify the
-        // meta location and will fail on first attempt then go into a long wait.
-        // So, do this in a thread and then reset meta location to break it out
-        // of its wait after a bit of time.
-        final AtomicBoolean metaSet = new AtomicBoolean(false);
-        final CountDownLatch latch = new CountDownLatch(1);
-        Thread t = new Thread() {
-          @Override
-          public void run() {
-            try {
-              latch.countDown();
-              metaSet.set(ct.waitForMeta(100000) !=  null);
-            } catch (Exception e) {
-              throw new RuntimeException(e);
-            }
+      // Set a location for root and meta.
+      RootLocationEditor.setRootLocation(this.watcher, SN);
+      ct.setMetaLocation(SN);
+      // Call the method that HBASE-4288 calls.  It will try and verify the
+      // meta location and will fail on first attempt then go into a long wait.
+      // So, do this in a thread and then reset meta location to break it out
+      // of its wait after a bit of time.
+      final AtomicBoolean metaSet = new AtomicBoolean(false);
+      final CountDownLatch latch = new CountDownLatch(1);
+      Thread t = new Thread() {
+        @Override
+        public void run() {
+          try {
+            latch.countDown();
+            metaSet.set(ct.waitForMeta(100000) !=  null);
+          } catch (Exception e) {
+            throw new RuntimeException(e);
           }
-        };
-        t.start();
-        latch.await();
-        Threads.sleep(1);
-        // Now reset the meta as though it were redeployed.
-        ct.setMetaLocation(SN);
-        t.join();
-        Assert.assertTrue(metaSet.get());
-      } finally {
-        // Clean out root and meta locations or later tests will be confused...
-        // they presume start fresh in zk.
-        ct.resetMetaLocation();
-        RootLocationEditor.deleteRootLocation(this.watcher);
-      }
+        }
+      };
+      t.start();
+      latch.await();
+      Threads.sleep(1);
+      // Now reset the meta as though it were redeployed.
+      ct.setMetaLocation(SN);
+      t.join();
+      Assert.assertTrue(metaSet.get());
     } finally {
-      // Clear out our doctored connection or could mess up subsequent tests.
-      HConnectionManager.deleteConnection(UTIL.getConfiguration());
+      // Clean out root and meta locations or later tests will be confused...
+      // they presume start fresh in zk.
+      ct.resetMetaLocation();
     }
   }
 
@@ -246,26 +244,16 @@ private void testVerifyMetaRegionLocationWithException(Exception ex)
     // Mock an HRegionInterface.
     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
     HConnection connection = mockConnection(implementation);
-    try {
-      // If a 'get' is called on mocked interface, throw connection refused.
-      Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).
-        thenThrow(ex);
-      // Now start up the catalogtracker with our doctored Connection.
-      final CatalogTracker ct = constructAndStartCatalogTracker(connection);
-      try {
-        RootLocationEditor.setRootLocation(this.watcher, SN);
-        long timeout = UTIL.getConfiguration().
-          getLong("hbase.catalog.verification.timeout", 1000);
-        Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
-      } finally {
-        // Clean out root location or later tests will be confused... they
-        // presume start fresh in zk.
-        RootLocationEditor.deleteRootLocation(this.watcher);
-      }
-    } finally {
-      // Clear out our doctored connection or could mess up subsequent tests.
-      HConnectionManager.deleteConnection(UTIL.getConfiguration());
-    }
+
+    // If a 'get' is called on mocked interface, throw connection refused.
+    Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).
+      thenThrow(ex);
+    // Now start up the catalogtracker with our doctored Connection.
+    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+    RootLocationEditor.setRootLocation(this.watcher, SN);
+    long timeout = UTIL.getConfiguration().
+      getLong("hbase.catalog.verification.timeout", 1000);
+    Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
   }
 
   /**
@@ -319,15 +307,9 @@ public void testVerifyRootRegionLocationFails()
       Mockito.anyInt(), Mockito.anyBoolean())).
       thenReturn(implementation);
     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
-    try {
-      RootLocationEditor.setRootLocation(this.watcher,
-        new ServerName("example.com", 1234, System.currentTimeMillis()));
-      Assert.assertFalse(ct.verifyRootRegionLocation(100));
-    } finally {
-      // Clean out root location or later tests will be confused... they presume
-      // start fresh in zk.
-      RootLocationEditor.deleteRootLocation(this.watcher);
-    }
+    RootLocationEditor.setRootLocation(this.watcher,
+      new ServerName("example.com", 1234, System.currentTimeMillis()));
+    Assert.assertFalse(ct.verifyRootRegionLocation(100));
   }
 
   @Test (expected = NotAllMetaRegionsOnlineException.class)
@@ -343,12 +325,8 @@ public void testTimeoutWaitForMeta()
   throws IOException, InterruptedException {
     HConnection connection =
       HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
-    try {
-      final CatalogTracker ct = constructAndStartCatalogTracker(connection);
-      ct.waitForMeta(100);
-    } finally {
-      HConnectionManager.deleteConnection(UTIL.getConfiguration());
-    }
+    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+    ct.waitForMeta(100);
   }
 
   /**
@@ -397,48 +375,45 @@ private ServerName setRootLocation() throws KeeperException {
     // Mock an HRegionInterface.
     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
     HConnection connection = mockConnection(implementation);
-    try {
-      // Now the ct is up... set into the mocks some answers that make it look
-      // like things have been getting assigned. Make it so we'll return a
-      // location (no matter what the Get is). Same for getHRegionInfo -- always
-      // just return the meta region.
-      final Result result = getMetaTableRowResult();
-
-      // TODO: Refactor.  This method has been moved out of HConnection.
-      // It works for now but has been deprecated.
-      Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
-        thenReturn(result);
-      Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
-        thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
-      final CatalogTracker ct = constructAndStartCatalogTracker(connection);
-      ServerName hsa = ct.getMetaLocation();
-      Assert.assertNull(hsa);
-
-      // Now test waiting on meta location getting set.
-      Thread t = new WaitOnMetaThread(ct) {
-        @Override
-        void doWaiting() throws InterruptedException {
-          this.ct.waitForMeta();
-        }
-      };
-      startWaitAliveThenWaitItLives(t, 1000);
-
-      // This should trigger wake up of meta wait (Its the removal of the meta
-      // region unassigned node that triggers catalogtrackers that a meta has
-      // been assigned).
-      String node = ct.getMetaNodeTracker().getNode();
-      ZKUtil.createAndFailSilent(this.watcher, node);
-      MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
-      ZKUtil.deleteNode(this.watcher, node);
-      // Go get the new meta location. waitForMeta gets and verifies meta.
-      Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
-      // Join the thread... should exit shortly.
-      t.join();
-      // Now meta is available.
-      Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
-    } finally {
-      HConnectionManager.deleteConnection(UTIL.getConfiguration());
-    }
+
+    // Now the ct is up... set into the mocks some answers that make it look
+    // like things have been getting assigned. Make it so we'll return a
+    // location (no matter what the Get is). Same for getHRegionInfo -- always
+    // just return the meta region.
+    final Result result = getMetaTableRowResult();
+
+    // TODO: Refactor.  This method has been moved out of HConnection.
+    // It works for now but has been deprecated.
+    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
+      thenReturn(result);
+    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
+      thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
+    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+    ServerName hsa = ct.getMetaLocation();
+    Assert.assertNull(hsa);
+
+    // Now test waiting on meta location getting set.
+    Thread t = new WaitOnMetaThread(ct) {
+      @Override
+      void doWaiting() throws InterruptedException {
+        this.ct.waitForMeta();
+      }
+    };
+    startWaitAliveThenWaitItLives(t, 1000);
+
+    // This should trigger wake up of meta wait (Its the removal of the meta
+    // region unassigned node that triggers catalogtrackers that a meta has
+    // been assigned).
+    String node = ct.getMetaNodeTracker().getNode();
+    ZKUtil.createAndFailSilent(this.watcher, node);
+    MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
+    ZKUtil.deleteNode(this.watcher, node);
+    // Go get the new meta location. waitForMeta gets and verifies meta.
+    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
+    // Join the thread... should exit shortly.
+    t.join();
+    // Now meta is available.
+    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
   }
 
   /**

From 51d732baaac6106f680e52db1311139469e14d3c Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 23 Aug 2013 20:30:27 +0000
Subject: [PATCH 1130/1540] HBASE-9303 Snapshot restore of table which splits
 after snapshot was taken encounters 'Region is not online'

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1517021 13f79535-47bb-0310-9956-ffa450edef68
---
 .../snapshot/RestoreSnapshotHandler.java      | 52 +++++++++++++++++--
 1 file changed, 48 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index f39a45ecf627..fcb7067730b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.master.snapshot;
 
 import java.io.IOException;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CancellationException;
 
@@ -34,6 +35,7 @@
 import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
@@ -118,13 +120,40 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
           snapshot, snapshotDir, hTableDescriptor, tableDir, monitor, status);
       RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
 
-      // 3. Applies changes to .META.
+      // 3. Forces all the RegionStates to be offline
+      //
+      // The AssignmentManager keeps all the region states around
+      // with no possibility to remove them, until the master is restarted.
+      // This means that a region marked as SPLIT before the restore will never be assigned again.
+      // To avoid having all states around all the regions are switched to the OFFLINE state,
+      // which is the same state that the regions will be after a delete table.
+      forceRegionsOffline(metaChanges);
+      forceRegionsOffline(metaChanges);
+
+      // 4. Applies changes to .META.
+
+      // 4.1 Removes the current set of regions from META
+      //
+      // By removing also the regions to restore (the ones present both in the snapshot
+      // and in the current state) we ensure that no extra fields are present in META
+      // e.g. with a simple add addRegionToMeta() the splitA and splitB attributes
+      // not overwritten/removed, so you end up with old informations
+      // that are not correct after the restore.
+      List<HRegionInfo> hrisToRemove = new LinkedList<HRegionInfo>();
+      if (metaChanges.hasRegionsToRemove()) hrisToRemove.addAll(metaChanges.getRegionsToRemove());
+      if (metaChanges.hasRegionsToRestore()) hrisToRemove.addAll(metaChanges.getRegionsToRestore());
+      MetaEditor.deleteRegions(catalogTracker, hrisToRemove);
+
+      // 4.2 Add the new set of regions to META
+      //
+      // At this point the old regions are no longer present in META.
+      // and the set of regions present in the snapshot will be written to META.
+      // All the information in META are coming from the .regioninfo of each region present
+      // in the snapshot folder.
       hris.clear();
-      status.setStatus("Preparing to restore each region");
       if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
       if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
-      List<HRegionInfo> hrisToRemove = metaChanges.getRegionsToRemove();
-      MetaEditor.mutateRegions(catalogTracker, hrisToRemove, hris);
+      MetaEditor.addRegionsToMeta(catalogTracker, hris);
       metaChanges.updateMetaParentRegions(catalogTracker, hris);
 
       // At this point the restore is complete. Next step is enabling the table.
@@ -141,6 +170,21 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
     }
   }
 
+  private void forceRegionsOffline(final RestoreSnapshotHelper.RestoreMetaChanges metaChanges) {
+    forceRegionsOffline(metaChanges.getRegionsToAdd());
+    forceRegionsOffline(metaChanges.getRegionsToRestore());
+    forceRegionsOffline(metaChanges.getRegionsToRemove());
+  }
+
+  private void forceRegionsOffline(final List<HRegionInfo> hris) {
+    AssignmentManager am = this.masterServices.getAssignmentManager();
+    if (hris != null) {
+      for (HRegionInfo hri: hris) {
+        am.regionOffline(hri);
+      }
+    }
+  }
+
   @Override
   protected void completed(final Throwable exception) {
     this.stopped = true;

From 4d9ad65693d3caa33a4ca54698285c0808f6a16b Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 26 Aug 2013 17:08:43 +0000
Subject: [PATCH 1131/1540] HBASE-9329 SnapshotManager should check for
 directory existance before throwing a warning (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1517609 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/snapshot/SnapshotManager.java       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index eeffb96844a7..fbca2c3fc9b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -263,8 +263,10 @@ private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws
   void resetTempDir() throws IOException {
     // cleanup any existing snapshots.
     Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
-    if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
-      LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
+    if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
+      if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
+        LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
+      }
     }
   }
 

From 8b075ad7f8d109e29beb19fbffcc3176717eeff7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 27 Aug 2013 19:57:04 +0000
Subject: [PATCH 1132/1540] HBASE-9344 RegionServer not shutting down upon
 KeeperException in open region

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1517959 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/handler/OpenRegionHandler.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 1fbf8f0ef95d..791848130623 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -239,6 +239,9 @@ public void run() {
       try {
         this.services.postOpenDeployTasks(this.region,
           this.server.getCatalogTracker(), false);
+      } catch (KeeperException e) {
+        server.abort("Exception running postOpenDeployTasks; region=" +
+            this.region.getRegionInfo().getEncodedName(), e);
       } catch (Exception e) {
         LOG.warn("Exception running postOpenDeployTasks; region=" +
           this.region.getRegionInfo().getEncodedName(), e);

From 5f54bf55267e83348f93512909a44232e0621480 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 27 Aug 2013 23:12:44 +0000
Subject: [PATCH 1133/1540] HBASE-9356 [0.94] SecureServer.INSECURE_VERSIONS is
 declared incorrectly

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1518028 13f79535-47bb-0310-9956-ffa450edef68
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index 3448d997f790..848b94abf0ad 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -86,7 +86,7 @@ public abstract class SecureServer extends HBaseServer {
   // 3 : Introduce the protocol into the RPC connection header
   // 4 : Introduced SASL security layer
   public static final byte CURRENT_VERSION = 4;
-  public static final Set<Byte> INSECURE_VERSIONS = ImmutableSet.of((byte) 3);
+  public static final Set<Integer> INSECURE_VERSIONS = ImmutableSet.of(3);
 
   public static final Log LOG = LogFactory.getLog(SecureServer.class);
   private static final Log AUDITLOG = LogFactory.getLog("SecurityLogger." +

From 634cdae54647dc005d600544c5b7395df250e319 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 28 Aug 2013 22:58:55 +0000
Subject: [PATCH 1134/1540] HBASE-7709 Infinite loop possible in Master/Master
 replication (Vasu Mariyala)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1518410 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/Mutation.java  |  38 ++
 .../hadoop/hbase/regionserver/HRegion.java    |   3 +
 .../hbase/regionserver/wal/WALEdit.java       | 113 +++--
 .../replication/regionserver/Replication.java |   9 +-
 .../regionserver/ReplicationSink.java         |  21 +-
 .../regionserver/ReplicationSource.java       |  19 +-
 .../replication/TestMasterReplication.java    | 446 +++++++++++-------
 7 files changed, 418 insertions(+), 231 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 730dee50eaa0..32c9e6f98a8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -33,12 +33,20 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+
 public abstract class Mutation extends OperationWithAttributes implements Row {
   private static final Log LOG = LogFactory.getLog(Mutation.class);
   // Attribute used in Mutations to indicate the originating cluster.
   private static final String CLUSTER_ID_ATTR = "_c.id_";
   private static final String DURABILITY_ID_ATTR = "_dur_";
 
+  /**
+   * The attribute for storing the list of clusters that have consumed the change.
+   */
+  private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
   protected byte [] row = null;
   protected long ts = HConstants.LATEST_TIMESTAMP;
   protected long lockId = -1L;
@@ -242,6 +250,36 @@ public UUID getClusterId() {
     return new UUID(Bytes.toLong(attr,0), Bytes.toLong(attr, Bytes.SIZEOF_LONG));
   }
 
+  /**
+   * Marks that the clusters with the given clusterIds have consumed the mutation
+   * @param clusterIds of the clusters that have consumed the mutation
+   */
+  public void setClusterIds(List<UUID> clusterIds) {
+    ByteArrayDataOutput out = ByteStreams.newDataOutput();
+    out.writeInt(clusterIds.size());
+    for (UUID clusterId : clusterIds) {
+      out.writeLong(clusterId.getMostSignificantBits());
+      out.writeLong(clusterId.getLeastSignificantBits());
+    }
+    setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
+  }
+
+  /**
+   * @return the set of cluster Ids that have consumed the mutation
+   */
+  public List<UUID> getClusterIds() {
+    List<UUID> clusterIds = new ArrayList<UUID>();
+    byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
+    if(bytes != null) {
+      ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
+      int numClusters = in.readInt();
+      for(int i=0; i<numClusters; i++){
+        clusterIds.add(new UUID(in.readLong(), in.readLong()));
+      }
+    }
+    return clusterIds;
+  }
+
   /**
    * @return the total number of KeyValues
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b244f38e77ce..c6358bdce74b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1995,6 +1995,7 @@ private void internalDelete(Delete delete, UUID clusterId,
         // bunch up all edits across all column families into a
         // single WALEdit.
         addFamilyMapToWALEdit(familyMap, walEdit);
+        walEdit.addClusterIds(delete.getClusterIds());
         this.log.append(regionInfo, this.htableDescriptor.getName(),
             walEdit, clusterId, now, this.htableDescriptor);
       }
@@ -2448,6 +2449,7 @@ private long doMiniBatchMutation(
       // STEP 5. Append the edit to WAL. Do not sync wal.
       // -------------------------
       Mutation first = batchOp.operations[firstIndex].getFirst();
+      walEdit.addClusterIds(first.getClusterIds());
       txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
                walEdit, first.getClusterId(), now, this.htableDescriptor);
 
@@ -2904,6 +2906,7 @@ private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOE
       // will contain uncommitted transactions.
       if (writeToWAL) {
         addFamilyMapToWALEdit(familyMap, walEdit);
+        walEdit.addClusterIds(put.getClusterIds());
         this.log.append(regionInfo, this.htableDescriptor.getName(),
             walEdit, clusterId, now, this.htableDescriptor);
       } else {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index 5ae50add9362..0d244b496b40 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -28,6 +28,7 @@
 import java.util.List;
 import java.util.NavigableMap;
 import java.util.TreeMap;
+import java.util.UUID;
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.codec.Decoder;
@@ -73,10 +74,29 @@
  */
 public class WALEdit implements Writable, HeapSize {
 
+  /*
+   * The cluster id of the cluster which has consumed the change represented by this class is
+   * prefixed with the value of this variable while storing in the scopes variable. This is to
+   * ensure that the cluster ids don't interfere with the column family replication settings stored
+   * in the scopes. The value is chosen to start with period as the column families can't start with
+   * it.
+   */
+  private static final String PREFIX_CLUSTER_KEY = ".";
   private final int VERSION_2 = -1;
 
   private final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
-  private NavigableMap<byte[], Integer> scopes;
+
+  /**
+   * This variable contains the information of the column family replication settings and contains
+   * the clusters that have already consumed the change represented by the object. This overloading
+   * of scopes with the consumed clusterids was introduced while porting the fix for HBASE-7709 back
+   * to 0.94 release. However, this overloading has been removed in the newer releases(0.95.2+). To
+   * check/change the column family settings, please use the getFromScope and putIntoScope methods
+   * and for marking/checking if a cluster has consumed the change, please use addCluster,
+   * addClusters and getClusters methods.
+   */
+  private final NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(
+      Bytes.BYTES_COMPARATOR);
 
   // default to decoding uncompressed data - needed for replication, which enforces that
   // uncompressed edits are sent across the wire. In the regular case (reading/writing WAL), the
@@ -116,22 +136,58 @@ public List<KeyValue> getKeyValues() {
     return kvs;
   }
 
-  public NavigableMap<byte[], Integer> getScopes() {
-    return scopes;
+  public Integer getFromScope(byte[] key) {
+    return scopes.get(key);
   }
 
+  public void putIntoScope(byte[] key, Integer value) {
+    scopes.put(key, value);
+  }
 
-  public void setScopes (NavigableMap<byte[], Integer> scopes) {
-    // We currently process the map outside of WALEdit,
-    // TODO revisit when replication is part of core
-    this.scopes = scopes;
+  public boolean hasKeyInScope(byte[] key) {
+    return scopes.containsKey(key);
+  }
+
+  /**
+   * @return true if the cluster with the given clusterId has consumed the change.
+   */
+  public boolean hasClusterId(UUID clusterId) {
+    return hasKeyInScope(Bytes.toBytes(PREFIX_CLUSTER_KEY + clusterId.toString()));
+  }
+
+  /**
+   * Marks that the cluster with the given clusterId has consumed the change.
+   */
+  public void addClusterId(UUID clusterId) {
+    scopes.put(Bytes.toBytes(PREFIX_CLUSTER_KEY + clusterId.toString()), 1);
+  }
+
+  /**
+   * Marks that the clusters with the given clusterIds have consumed the change.
+   */
+  public void addClusterIds(List<UUID> clusterIds) {
+    for (UUID clusterId : clusterIds) {
+      addClusterId(clusterId);
+    }
+  }
+
+  /**
+   * @return the set of cluster Ids that have consumed the change.
+   */
+  public List<UUID> getClusterIds() {
+    List<UUID> clusterIds = new ArrayList<UUID>();
+    for (byte[] keyBytes : scopes.keySet()) {
+      String key = Bytes.toString(keyBytes);
+      if (key.startsWith(PREFIX_CLUSTER_KEY)) {
+        clusterIds.add(UUID.fromString(key.substring(PREFIX_CLUSTER_KEY.length())));
+      }
+    }
+    return clusterIds;
   }
 
   public void readFields(DataInput in) throws IOException {
     kvs.clear();
-    if (scopes != null) {
-      scopes.clear();
-    }
+    scopes.clear();
     Decoder decoder = this.codec.getDecoder((DataInputStream) in);
     int versionOrLength = in.readInt();
     int length = versionOrLength;
@@ -148,15 +204,12 @@ public void readFields(DataInput in) throws IOException {
 
     //its a new style WAL, so we need replication scopes too
     if (versionOrLength == VERSION_2) {
-      int numFamilies = in.readInt();
-      if (numFamilies > 0) {
-        if (scopes == null) {
-          scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
-        }
-        for (int i = 0; i < numFamilies; i++) {
-          byte[] fam = Bytes.readByteArray(in);
+      int numEntries = in.readInt();
+      if (numEntries > 0) {
+        for (int i = 0; i < numEntries; i++) {
+          byte[] key = Bytes.readByteArray(in);
           int scope = in.readInt();
-          scopes.put(fam, scope);
+          scopes.put(key, scope);
         }
       }
     }
@@ -173,14 +226,10 @@ public void write(DataOutput out) throws IOException {
     }
     kvEncoder.flush();
 
-    if (scopes == null) {
-      out.writeInt(0);
-    } else {
-      out.writeInt(scopes.size());
-      for (byte[] key : scopes.keySet()) {
-        Bytes.writeByteArray(out, key);
-        out.writeInt(scopes.get(key));
-      }
+    out.writeInt(scopes.size());
+    for (byte[] key : scopes.keySet()) {
+      Bytes.writeByteArray(out, key);
+      out.writeInt(scopes.get(key));
     }
   }
 
@@ -189,11 +238,9 @@ public long heapSize() {
     for (KeyValue kv : kvs) {
       ret += kv.heapSize();
     }
-    if (scopes != null) {
-      ret += ClassSize.TREEMAP;
-      ret += ClassSize.align(scopes.size() * ClassSize.MAP_ENTRY);
-      // TODO this isn't quite right, need help here
-    }
+    ret += ClassSize.TREEMAP;
+    ret += ClassSize.align(scopes.size() * ClassSize.MAP_ENTRY);
+    // TODO this isn't quite right, need help here
     return ret;
   }
 
@@ -205,9 +252,7 @@ public String toString() {
       sb.append(kv.toString());
       sb.append("; ");
     }
-    if (scopes != null) {
-      sb.append(" scopes: " + scopes.toString());
-    }
+    sb.append(" scopes: " + scopes.toString());
     sb.append(">]");
     return sb.toString();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 509112a28911..13a80bfb7014 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -172,20 +172,15 @@ public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
   @Override
   public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey,
                                        WALEdit logEdit) {
-    NavigableMap<byte[], Integer> scopes =
-        new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
     byte[] family;
     for (KeyValue kv : logEdit.getKeyValues()) {
       family = kv.getFamily();
       int scope = htd.getFamily(family).getScope();
       if (scope != REPLICATION_SCOPE_LOCAL &&
-          !scopes.containsKey(family)) {
-        scopes.put(family, scope);
+          !logEdit.hasKeyInScope(family)) {
+        logEdit.putIntoScope(family, scope);
       }
     }
-    if (!scopes.isEmpty()) {
-      logEdit.setScopes(scopes);
-    }
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 1bec9684312d..ad67367d8abe 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -110,9 +110,11 @@ public void replicateEntries(HLog.Entry[] entries)
     // to the same table.
     try {
       long totalReplicated = 0;
-      // Map of table => list of Rows, grouped by cluster id, we only want to flushCommits once per
-      // invocation of this method per table and cluster id.
-      Map<byte[], Map<UUID,List<Row>>> rowMap = new TreeMap<byte[], Map<UUID,List<Row>>>(Bytes.BYTES_COMPARATOR);
+      // Map of table => list of Rows, grouped by clusters that consumed the change, we only want to
+      // flushCommits once per
+      // invocation of this method per table and clusters that have consumed the change.
+      Map<byte[], Map<List<UUID>, List<Row>>> rowMap =
+          new TreeMap<byte[], Map<List<UUID>, List<Row>>>(Bytes.BYTES_COMPARATOR);
       for (HLog.Entry entry : entries) {
         WALEdit edit = entry.getEdit();
         byte[] table = entry.getKey().getTablename();
@@ -123,14 +125,19 @@ public void replicateEntries(HLog.Entry[] entries)
         for (KeyValue kv : kvs) {
           if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
             UUID clusterId = entry.getKey().getClusterId();
+            List<UUID> clusterIds = edit.getClusterIds();
             if (kv.isDelete()) {
               del = new Delete(kv.getRow());
               del.setClusterId(clusterId);
-              addToHashMultiMap(rowMap, table, clusterId, del);
+              del.setClusterIds(clusterIds);
+              clusterIds.add(clusterId);
+              addToHashMultiMap(rowMap, table, clusterIds, del);
             } else {
               put = new Put(kv.getRow());
               put.setClusterId(clusterId);
-              addToHashMultiMap(rowMap, table, clusterId, put);
+              put.setClusterIds(clusterIds);
+              clusterIds.add(clusterId);
+              addToHashMultiMap(rowMap, table, clusterIds, put);
             }
           }
           if (kv.isDelete()) {
@@ -142,7 +149,7 @@ public void replicateEntries(HLog.Entry[] entries)
         }
         totalReplicated++;
       }
-      for(Map.Entry<byte[], Map<UUID, List<Row>>> entry : rowMap.entrySet()) {
+      for(Map.Entry<byte[], Map<List<UUID>, List<Row>>> entry : rowMap.entrySet()) {
         batch(entry.getKey(), entry.getValue().values());
       }
       this.metrics.setAgeOfLastAppliedOp(
@@ -162,7 +169,7 @@ public void replicateEntries(HLog.Entry[] entries)
    * @param key1
    * @param key2
    * @param value
-   * @return
+   * @return the list of values for the combination of key1 and key2
    */
   private <K1, K2, V> List<V> addToHashMultiMap(Map<K1, Map<K2,List<V>>> map, K1 key1, K2 key2, V value) {
     Map<K2,List<V>> innerMap = map.get(key1);
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index ccd39850d63d..00db9e4a8f2a 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -30,7 +30,6 @@
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
-import java.util.NavigableMap;
 import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
@@ -59,7 +58,6 @@
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.zookeeper.KeeperException;
 
@@ -477,8 +475,14 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
       seenEntries++;
       // Remove all KVs that should not be replicated
       HLogKey logKey = entry.getKey();
-      // don't replicate if the log entries originated in the peer
-      if (!logKey.getClusterId().equals(peerClusterId)) {
+      List<UUID> consumedClusterIds = edit.getClusterIds();
+      // This cluster id has been added to resolve the scenario of A -> B -> A where A has old
+      // point release and B has the new point release which has the fix HBASE-7709. A change on
+      // cluster A would infinitely replicate to
+      // cluster B if we don't add the original cluster id to the set.
+      consumedClusterIds.add(logKey.getClusterId());
+      // don't replicate if the log entries if it has not already been replicated
+      if (!consumedClusterIds.contains(peerClusterId)) {
         removeNonReplicableEdits(edit);
         // Don't replicate catalog entries, if the WALEdit wasn't
         // containing anything to replicate and if we're currently not set to replicate
@@ -491,6 +495,8 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
           // This is *only* place where a cluster id other than the default is set.
           if (HConstants.DEFAULT_CLUSTER_ID == logKey.getClusterId()) {
             logKey.setClusterId(this.clusterId);
+          } else if (logKey.getClusterId() != this.clusterId) {
+            edit.addClusterId(clusterId);
           }
           currentNbOperations += countDistinctRowKeys(edit);
           currentNbEntries++;
@@ -664,13 +670,12 @@ protected boolean sleepForRetries(String msg, int sleepMultiplier) {
    * @param edit The KV to check for replication
    */
   protected void removeNonReplicableEdits(WALEdit edit) {
-    NavigableMap<byte[], Integer> scopes = edit.getScopes();
     List<KeyValue> kvs = edit.getKeyValues();
     for (int i = edit.size()-1; i >= 0; i--) {
       KeyValue kv = kvs.get(i);
       // The scope will be null or empty if
       // there's nothing to replicate in that WALEdit
-      if (scopes == null || !scopes.containsKey(kv.getFamily())) {
+      if (!edit.hasKeyInScope(kv.getFamily())) {
         kvs.remove(i);
       }
     }
@@ -928,4 +933,4 @@ private long getTS(Path p) {
       return Long.parseLong(parts[parts.length-1]);
     }
   }
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
index 0bbef220d0d6..69a8f2f26511 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
@@ -23,8 +23,10 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -55,15 +57,11 @@ public class TestMasterReplication {
 
   private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
 
-  private Configuration conf1;
-  private Configuration conf2;
-  private Configuration conf3;
+  private Configuration baseConfiguration;
 
-  private HBaseTestingUtility utility1;
-  private HBaseTestingUtility utility2;
-  private HBaseTestingUtility utility3;
-  
-  private MiniZooKeeperCluster miniZK; 
+  private HBaseTestingUtility[] utilities;
+  private Configuration[] configurations;
+  private MiniZooKeeperCluster miniZK;
 
   private static final long SLEEP_TIME = 500;
   private static final int NB_RETRIES = 10;
@@ -85,44 +83,21 @@ public class TestMasterReplication {
 
   @Before
   public void setUp() throws Exception {
-    conf1 = HBaseConfiguration.create();
-    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
+    baseConfiguration = HBaseConfiguration.create();
     // smaller block size and capacity to trigger more operations
     // and test them
-    conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
-    conf1.setInt("replication.source.size.capacity", 1024);
-    conf1.setLong("replication.source.sleepforretries", 100);
-    conf1.setInt("hbase.regionserver.maxlogs", 10);
-    conf1.setLong("hbase.master.logcleaner.ttl", 10);
-    conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
-    conf1.setBoolean("dfs.support.append", true);
-    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
-    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
+    baseConfiguration.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
+    baseConfiguration.setInt("replication.source.size.capacity", 1024);
+    baseConfiguration.setLong("replication.source.sleepforretries", 100);
+    baseConfiguration.setInt("hbase.regionserver.maxlogs", 10);
+    baseConfiguration.setLong("hbase.master.logcleaner.ttl", 10);
+    baseConfiguration.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
+    baseConfiguration.setBoolean("dfs.support.append", true);
+    baseConfiguration.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
+    baseConfiguration.setStrings(
+        CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
         CoprocessorCounter.class.getName());
 
-    utility1 = new HBaseTestingUtility(conf1);
-    utility1.startMiniZKCluster();
-    miniZK = utility1.getZkCluster();
-    // By setting the mini ZK cluster through this method, even though this is
-    // already utility1's mini ZK cluster, we are telling utility1 not to shut
-    // the mini ZK cluster when we shut down the HBase cluster.
-    utility1.setZkCluster(miniZK);
-    new ZooKeeperWatcher(conf1, "cluster1", null, true);
-
-    conf2 = new Configuration(conf1);
-    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
-
-    utility2 = new HBaseTestingUtility(conf2);
-    utility2.setZkCluster(miniZK);
-    new ZooKeeperWatcher(conf2, "cluster2", null, true);
-
-    conf3 = new Configuration(conf1);
-    conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
-
-    utility3 = new HBaseTestingUtility(conf3);
-    utility3.setZkCluster(miniZK);
-    new ZooKeeperWatcher(conf3, "cluster3", null, true);
-
     table = new HTableDescriptor(tableName);
     HColumnDescriptor fam = new HColumnDescriptor(famName);
     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
@@ -131,181 +106,300 @@ public void setUp() throws Exception {
     table.addFamily(fam);
   }
 
-  @After
-  public void tearDown() throws IOException {
-    miniZK.shutdown();
+  /**
+   * It tests the replication scenario involving 0 -> 1 -> 0. It does it by
+   * adding and deleting a row to a table in each cluster, checking if it's
+   * replicated. It also tests that the puts and deletes are not replicated back
+   * to the originating cluster.
+   */
+  @Test(timeout = 300000)
+  public void testCyclicReplication1() throws Exception {
+    LOG.info("testSimplePutDelete");
+    int numClusters = 2;
+    HTable[] htables = null;
+    try {
+      startMiniClusters(numClusters);
+      createTableOnClusters(table);
+
+      htables = getHTablesOnClusters(tableName);
+
+      // Test the replication scenarios of 0 -> 1 -> 0
+      addPeer("1", 0, 1);
+      addPeer("1", 1, 0);
+
+      int[] expectedCounts = new int[] { 2, 2 };
+
+      // add rows to both clusters,
+      // make sure they are both replication
+      putAndWait(row, famName, htables[0], htables[1]);
+      putAndWait(row1, famName, htables[1], htables[0]);
+      validateCounts(htables, put, expectedCounts);
+
+      deleteAndWait(row, htables[0], htables[1]);
+      deleteAndWait(row1, htables[1], htables[0]);
+      validateCounts(htables, delete, expectedCounts);
+    } finally {
+      close(htables);
+      shutDownMiniClusters();
+    }
   }
 
-  @Test(timeout=300000)
-  public void testCyclicReplication() throws Exception {
-    LOG.info("testCyclicReplication");
-    utility1.startMiniCluster();
-    utility2.startMiniCluster();
-    utility3.startMiniCluster();
-    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
-    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
-    ReplicationAdmin admin3 = new ReplicationAdmin(conf3);
-
-    new HBaseAdmin(conf1).createTable(table);
-    new HBaseAdmin(conf2).createTable(table);
-    new HBaseAdmin(conf3).createTable(table);
-    HTable htable1 = new HTable(conf1, tableName);
-    htable1.setWriteBufferSize(1024);
-    HTable htable2 = new HTable(conf2, tableName);
-    htable2.setWriteBufferSize(1024);
-    HTable htable3 = new HTable(conf3, tableName);
-    htable3.setWriteBufferSize(1024);
-    
-    admin1.addPeer("1", utility2.getClusterKey());
-    admin2.addPeer("1", utility3.getClusterKey());
-    admin3.addPeer("1", utility1.getClusterKey());
-
-    // put "row" and wait 'til it got around
-    putAndWait(row, famName, htable1, htable3);
-    // it should have passed through table2
-    check(row,famName,htable2);
-
-    putAndWait(row1, famName, htable2, htable1);
-    check(row,famName,htable3);
-    putAndWait(row2, famName, htable3, htable2);
-    check(row,famName,htable1);
-    
-    deleteAndWait(row,htable1,htable3);
-    deleteAndWait(row1,htable2,htable1);
-    deleteAndWait(row2,htable3,htable2);
-
-    assertEquals("Puts were replicated back ", 3, getCount(htable1, put));
-    assertEquals("Puts were replicated back ", 3, getCount(htable2, put));
-    assertEquals("Puts were replicated back ", 3, getCount(htable3, put));
-    assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete));
-    assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete));
-    assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete));
-
-    // Test HBASE-9158
-    admin2.disablePeer("1");
-    // we now have an edit that was replicated into cluster originating from cluster 1
-    putAndWait(row3, famName, htable1, htable2);
-    // now add a local edit to cluster 2
-    Put put = new Put(row4);
-    put.add(famName, row4, row4);
-    htable2.put(put);
-    // reenable replication from cluster 2 to cluster 3
-    admin2.enablePeer("1");
-    // without HBASE-9158 the edit for row4 would have been marked with cluster 1's id
-    // and hence not replicated to cluster 1
-    wait(row4, htable1);
-    
-    utility3.shutdownMiniCluster();
-    utility2.shutdownMiniCluster();
-    utility1.shutdownMiniCluster();
+  /**
+   * Tests the cyclic replication scenario of 0 -> 1 -> 2 -> 0 by adding and
+   * deleting rows to a table in each clusters and ensuring that the each of
+   * these clusters get the appropriate mutations. It also tests the grouping
+   * scenario where a cluster needs to replicate the edits originating from
+   * itself and also the edits that it received using replication from a
+   * different cluster. The scenario is explained in HBASE-9158
+   */
+  @Test(timeout = 300000)
+  public void testCyclicReplication2() throws Exception {
+    LOG.info("testCyclicReplication1");
+    int numClusters = 3;
+    HTable[] htables = null;
+    try {
+      startMiniClusters(numClusters);
+      createTableOnClusters(table);
+
+      // Test the replication scenario of 0 -> 1 -> 2 -> 0
+      addPeer("1", 0, 1);
+      addPeer("1", 1, 2);
+      addPeer("1", 2, 0);
+
+      htables = getHTablesOnClusters(tableName);
+
+      // put "row" and wait 'til it got around
+      putAndWait(row, famName, htables[0], htables[2]);
+      putAndWait(row1, famName, htables[1], htables[0]);
+      putAndWait(row2, famName, htables[2], htables[1]);
+
+      deleteAndWait(row, htables[0], htables[2]);
+      deleteAndWait(row1, htables[1], htables[0]);
+      deleteAndWait(row2, htables[2], htables[1]);
+
+      int[] expectedCounts = new int[] { 3, 3, 3 };
+      validateCounts(htables, put, expectedCounts);
+      validateCounts(htables, delete, expectedCounts);
+
+      // Test HBASE-9158
+      disablePeer("1", 2);
+      // we now have an edit that was replicated into cluster originating from
+      // cluster 0
+      putAndWait(row3, famName, htables[0], htables[1]);
+      // now add a local edit to cluster 1
+      htables[1].put(new Put(row4).add(famName, row4, row4));
+      // re-enable replication from cluster 2 to cluster 0
+      enablePeer("1", 2);
+      // without HBASE-9158 the edit for row4 would have been marked with
+      // cluster 0's id
+      // and hence not replicated to cluster 0
+      wait(row4, htables[0], true);
+    } finally {
+      close(htables);
+      shutDownMiniClusters();
+    }
   }
 
   /**
-   * Add a row to a table in each cluster, check it's replicated,
-   * delete it, check's gone
-   * Also check the puts and deletes are not replicated back to
-   * the originating cluster.
+   * Tests cyclic replication scenario of 0 -> 1 -> 2 -> 1.
    */
-  @Test(timeout=300000)
-  public void testSimplePutDelete() throws Exception {
-    LOG.info("testSimplePutDelete");
-    utility1.startMiniCluster();
-    utility2.startMiniCluster();
+  @Test(timeout = 300000)
+  public void testCyclicReplication3() throws Exception {
+    LOG.info("testCyclicReplication2");
+    int numClusters = 3;
+    HTable[] htables = null;
+    try {
+      startMiniClusters(numClusters);
+      createTableOnClusters(table);
+
+      // Test the replication scenario of 0 -> 1 -> 2 -> 1
+      addPeer("1", 0, 1);
+      addPeer("1", 1, 2);
+      addPeer("1", 2, 1);
+
+      htables = getHTablesOnClusters(tableName);
+
+      // put "row" and wait 'til it got around
+      putAndWait(row, famName, htables[0], htables[2]);
+      putAndWait(row1, famName, htables[1], htables[2]);
+      putAndWait(row2, famName, htables[2], htables[1]);
+
+      deleteAndWait(row, htables[0], htables[2]);
+      deleteAndWait(row1, htables[1], htables[2]);
+      deleteAndWait(row2, htables[2], htables[1]);
+
+      int[] expectedCounts = new int[] { 1, 3, 3 };
+      validateCounts(htables, put, expectedCounts);
+      validateCounts(htables, delete, expectedCounts);
+    } finally {
+      close(htables);
+      shutDownMiniClusters();
+    }
+  }
 
-    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
-    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
+  @After
+  public void tearDown() throws IOException {
+    configurations = null;
+    utilities = null;
+  }
 
-    new HBaseAdmin(conf1).createTable(table);
-    new HBaseAdmin(conf2).createTable(table);
-    HTable htable1 = new HTable(conf1, tableName);
-    htable1.setWriteBufferSize(1024);
-    HTable htable2 = new HTable(conf2, tableName);
-    htable2.setWriteBufferSize(1024);
+  @SuppressWarnings("resource")
+  private void startMiniClusters(int numClusters) throws Exception {
+    Random random = new Random();
+    utilities = new HBaseTestingUtility[numClusters];
+    configurations = new Configuration[numClusters];
+    for (int i = 0; i < numClusters; i++) {
+      Configuration conf = new Configuration(baseConfiguration);
+      conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/" + i + random.nextInt());
+      HBaseTestingUtility utility = new HBaseTestingUtility(conf);
+      if (i == 0) {
+        utility.startMiniZKCluster();
+        miniZK = utility.getZkCluster();
+      } else {
+        utility.setZkCluster(miniZK);
+      }
+      utility.startMiniCluster();
+      utilities[i] = utility;
+      configurations[i] = conf;
+      new ZooKeeperWatcher(conf, "cluster" + i, null, true);
+    }
+  }
 
-    // set M-M
-    admin1.addPeer("1", utility2.getClusterKey());
-    admin2.addPeer("1", utility1.getClusterKey());
+  private void shutDownMiniClusters() throws Exception {
+    int numClusters = utilities.length;
+    for (int i = numClusters - 1; i >= 0; i--) {
+      if (utilities[i] != null) {
+        utilities[i].shutdownMiniCluster();
+      }
+    }
+    miniZK.shutdown();
+  }
 
-    // add rows to both clusters,
-    // make sure they are both replication
-    putAndWait(row, famName, htable1, htable2);
-    putAndWait(row1, famName, htable2, htable1);
+  private void createTableOnClusters(HTableDescriptor table) throws Exception {
+    int numClusters = configurations.length;
+    for (int i = 0; i < numClusters; i++) {
+      HBaseAdmin hbaseAdmin = null;
+      try {
+        hbaseAdmin = new HBaseAdmin(configurations[i]);
+        hbaseAdmin.createTable(table);
+      } finally {
+        close(hbaseAdmin);
+      }
+    }
+  }
 
-    // make sure "row" did not get replicated back.
-    assertEquals("Puts were replicated back ", 2, getCount(htable1, put));
+  private void addPeer(String id, int masterClusterNumber,
+      int slaveClusterNumber) throws Exception {
+    ReplicationAdmin replicationAdmin = null;
+    try {
+      replicationAdmin = new ReplicationAdmin(
+          configurations[masterClusterNumber]);
+      replicationAdmin.addPeer(id,
+          utilities[slaveClusterNumber].getClusterKey());
+    } finally {
+      close(replicationAdmin);
+    }
+  }
 
-    // delete "row" and wait
-    deleteAndWait(row, htable1, htable2);
+  private void disablePeer(String id, int masterClusterNumber) throws Exception {
+    ReplicationAdmin replicationAdmin = null;
+    try {
+      replicationAdmin = new ReplicationAdmin(
+          configurations[masterClusterNumber]);
+      replicationAdmin.disablePeer(id);
+    } finally {
+      close(replicationAdmin);
+    }
+  }
 
-    // make the 2nd cluster replicated back
-    assertEquals("Puts were replicated back ", 2, getCount(htable2, put));
+  private void enablePeer(String id, int masterClusterNumber) throws Exception {
+    ReplicationAdmin replicationAdmin = null;
+    try {
+      replicationAdmin = new ReplicationAdmin(
+          configurations[masterClusterNumber]);
+      replicationAdmin.enablePeer(id);
+    } finally {
+      close(replicationAdmin);
+    }
+  }
 
-    deleteAndWait(row1, htable2, htable1);
+  private void close(Closeable... closeables) {
+    try {
+      if (closeables != null) {
+        for (Closeable closeable : closeables) {
+          closeable.close();
+        }
+      }
+    } catch (Exception e) {
+      LOG.warn("Exception occured while closing the object:", e);
+    }
+  }
 
-    assertEquals("Deletes were replicated back ", 2, getCount(htable1, delete));
-    utility2.shutdownMiniCluster();
-    utility1.shutdownMiniCluster();
+  @SuppressWarnings("resource")
+  private HTable[] getHTablesOnClusters(byte[] tableName) throws Exception {
+    int numClusters = utilities.length;
+    HTable[] htables = new HTable[numClusters];
+    for (int i = 0; i < numClusters; i++) {
+      HTable htable = new HTable(configurations[i], tableName);
+      htable.setWriteBufferSize(1024);
+      htables[i] = htable;
+    }
+    return htables;
   }
 
-  private int getCount(HTable t, byte[] type)  throws IOException {
+  private void validateCounts(HTable[] htables, byte[] type,
+      int[] expectedCounts) throws IOException {
+    for (int i = 0; i < htables.length; i++) {
+      assertEquals(Bytes.toString(type) + " were replicated back ",
+          expectedCounts[i], getCount(htables[i], type));
+    }
+  }
+
+  private int getCount(HTable t, byte[] type) throws IOException {
     Get test = new Get(row);
-    test.setAttribute("count", new byte[]{});
+    test.setAttribute("count", new byte[] {});
     Result res = t.get(test);
     return Bytes.toInt(res.getValue(count, type));
   }
 
   private void deleteAndWait(byte[] row, HTable source, HTable target)
-  throws Exception {
+      throws Exception {
     Delete del = new Delete(row);
     source.delete(del);
-
-    Get get = new Get(row);
-    for (int i = 0; i < NB_RETRIES; i++) {
-      if (i==NB_RETRIES-1) {
-        fail("Waited too much time for del replication");
-      }
-      Result res = target.get(get);
-      if (res.size() >= 1) {
-        LOG.info("Row not deleted");
-        Thread.sleep(SLEEP_TIME);
-      } else {
-        break;
-      }
-    }
-  }
-
-  private void check(byte[] row, byte[] fam, HTable t) throws IOException {
-    Get get = new Get(row);
-    Result res = t.get(get);
-    if (res.size() == 0) {
-      fail("Row is missing");
-    }
+    wait(row, target, true);
   }
 
   private void putAndWait(byte[] row, byte[] fam, HTable source, HTable target)
-  throws Exception {
+      throws Exception {
     Put put = new Put(row);
     put.add(fam, row, row);
     source.put(put);
-
-    wait(row, target);
+    wait(row, target, false);
   }
 
-  private void wait(byte[] row, HTable target) throws Exception {
+  private void wait(byte[] row, HTable target, boolean isDeleted)
+      throws Exception {
     Get get = new Get(row);
     for (int i = 0; i < NB_RETRIES; i++) {
-      if (i==NB_RETRIES-1) {
-        fail("Waited too much time for put replication");
+      if (i == NB_RETRIES - 1) {
+        fail("Waited too much time for replication. Row:" + Bytes.toString(row)
+            + ". IsDeleteReplication:" + isDeleted);
       }
       Result res = target.get(get);
-      if (res.size() == 0) {
-        LOG.info("Row not available");
+      boolean sleep = isDeleted ? res.size() > 0 : res.size() == 0;
+      if (sleep) {
+        LOG.info("Waiting for more time for replication. Row:"
+            + Bytes.toString(row) + ". IsDeleteReplication:" + isDeleted);
         Thread.sleep(SLEEP_TIME);
       } else {
-        assertArrayEquals(res.value(), row);
+        if (!isDeleted) {
+          assertArrayEquals(res.value(), row);
+        }
+        LOG.info("Obtained row:"
+            + Bytes.toString(row) + ". IsDeleteReplication:" + isDeleted);
         break;
       }
-    }    
+    }
   }
 
   /**

From 400a8f7e4a1f5cb2fc3c981ede2e88119ca40b80 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Thu, 29 Aug 2013 00:37:18 +0000
Subject: [PATCH 1135/1540] HBASE-9326 ServerName is created using
 getLocalSocketAddress, breaks binding to the wildcard            address.
 Revert HBASE-8640

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1518432 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/HMaster.java    | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f2e3520fd27b..50e7989426d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -288,16 +288,23 @@ public HMaster(final Configuration conf)
     // Set how many times to retry talking to another server over HConnection.
     HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
     // Server to handle client requests.
-    String hostname = conf.get("hbase.master.ipc.address",
-      Strings.domainNamePointerToHostName(DNS.getDefaultHost(
-        conf.get("hbase.master.dns.interface", "default"),
-        conf.get("hbase.master.dns.nameserver", "default"))));
+    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
+      conf.get("hbase.master.dns.interface", "default"),
+      conf.get("hbase.master.dns.nameserver", "default")));
     int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
     // Test that the hostname is reachable
     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
     if (initialIsa.getAddress() == null) {
       throw new IllegalArgumentException("Failed resolve of hostname " + initialIsa);
     }
+    // Verify that the bind address is reachable if set
+    String bindAddress = conf.get("hbase.master.ipc.address");
+    if (bindAddress != null) {
+      initialIsa = new InetSocketAddress(bindAddress, port);
+      if (initialIsa.getAddress() == null) {
+        throw new IllegalArgumentException("Failed resolve of bind address " + initialIsa);
+      }
+    }
     int numHandlers = conf.getInt("hbase.master.handler.count",
       conf.getInt("hbase.regionserver.handler.count", 25));
     this.rpcServer = HBaseRPC.getServer(this,
@@ -310,7 +317,8 @@ public HMaster(final Configuration conf)
         0); // this is a DNC w/o high priority handlers
     // Set our address.
     this.isa = this.rpcServer.getListenerAddress();
-    this.serverName = new ServerName(this.isa.getHostName(),
+    // We don't want to pass isa's hostname here since it could be 0.0.0.0
+    this.serverName = new ServerName(hostname,
       this.isa.getPort(), System.currentTimeMillis());
     this.rsFatals = new MemoryBoundedLogMessageBuffer(
         conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));

From 7044e00e293a6b8ebc5bbc131c6c2cda8925fefe Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 30 Aug 2013 16:13:08 +0000
Subject: [PATCH 1136/1540] HBASE-7954 Fix the retrying logic of memstore
 flushes to avoid extra sleep

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1519011 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index d1e7e2289fac..9ee4a0809d45 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -805,7 +805,7 @@ protected Path flushCache(final long logCacheFlushId,
         LOG.warn("Failed flushing store file, retring num=" + i, e);
         lastException = e;
       }
-      if (lastException != null) {
+      if (lastException != null && i < (flush_retries_number - 1)) {
         try {
           Thread.sleep(pauseTime);
         } catch (InterruptedException e) {

From 78a92ef8a6d52bb40bed208c79f1b8f2922ee1ac Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 3 Sep 2013 17:48:48 +0000
Subject: [PATCH 1137/1540] HBASE-9397 Snapshots with the same name are allowed
 to proceed concurrently (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1519767 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/snapshot/SnapshotManager.java      | 30 +++++++++++++++++--
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index fbca2c3fc9b7..b4f3fbb6bcff 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -372,6 +372,29 @@ public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
     return false;
   }
 
+  /**
+   * Check to see if there is a snapshot in progress with the same name or on the same table.
+   * Currently we have a limitation only allowing a single snapshot per table at a time. Also we
+   * don't allow snapshot with the same name.
+   * @param snapshot description of the snapshot being checked.
+   * @return <tt>true</tt> if there is a snapshot in progress with the same name or on the same
+   *         table.
+   */
+  synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
+    if (isTakingSnapshot(snapshot.getTable())) {
+      return true;
+    }
+    Iterator<Map.Entry<String, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
+    while (it.hasNext()) {
+      Map.Entry<String, SnapshotSentinel> entry = it.next();
+      SnapshotSentinel sentinel = entry.getValue();
+      if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Check to see if the specified table has a snapshot in progress.  Currently we have a
    * limitation only allowing a single snapshot per table at a time.
@@ -393,14 +416,15 @@ private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
       throws HBaseSnapshotException {
     FileSystem fs = master.getMasterFileSystem().getFileSystem();
     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
-
     // make sure we aren't already running a snapshot
-    if (isTakingSnapshot(snapshot.getTable())) {
+    if (isTakingSnapshot(snapshot)) {
       SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
       throw new SnapshotCreationException("Rejected taking "
           + SnapshotDescriptionUtils.toString(snapshot)
           + " because we are already running another snapshot "
-          + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
+          + (handler != null ? ("on the same table " +
+              SnapshotDescriptionUtils.toString(handler.getSnapshot()))
+              : "with the same name"), snapshot);
     }
 
     // make sure we aren't running a restore on the same table

From d0cce3f48d4b448d09850a78a8499a32db8f9d28 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Tue, 3 Sep 2013 22:33:36 +0000
Subject: [PATCH 1138/1540] HBASE-9415  In rpcServer, replicationQueue is
 initialized with the max queue size instead of the max queue lenght

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1519873 13f79535-47bb-0310-9956-ffa450edef68
---
 conf/hbase-site.xml                                        | 1 -
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml
index af4c30095216..c516ac729114 100644
--- a/conf/hbase-site.xml
+++ b/conf/hbase-site.xml
@@ -2,7 +2,6 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 8813a009944e..9291734bc10d 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -1553,7 +1553,7 @@ protected HBaseServer(String bindAddress, int port,
     this.numOfReplicationHandlers = 
       conf.getInt("hbase.regionserver.replication.handler.count", 3);
     if (numOfReplicationHandlers > 0) {
-      this.replicationQueue = new LinkedBlockingQueue<Call>(maxQueueSize);
+      this.replicationQueue = new LinkedBlockingQueue<Call>(maxQueueLength);
     }
     // Start the listener here and let it bind to the port
     listener = new Listener();

From de186bdb7b73fe3bd9d487ef9a2c23859f0cdceb Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Tue, 3 Sep 2013 22:45:08 +0000
Subject: [PATCH 1139/1540] HBASE-9415  In rpcServer, replicationQueue is
 initialized with the max queue size instead of the max queue lenght - fix bad
 commit

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1519876 13f79535-47bb-0310-9956-ffa450edef68
---
 conf/hbase-site.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml
index c516ac729114..af4c30095216 100644
--- a/conf/hbase-site.xml
+++ b/conf/hbase-site.xml
@@ -2,6 +2,7 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file

From 519cf4bc382e6f9e7717862c04f245f431dfc1c2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 4 Sep 2013 18:41:20 +0000
Subject: [PATCH 1140/1540] HBASE-9428 Regex filters are at least an order of
 magnitude slower since 0.94.3

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520101 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/filter/RegexStringComparator.java  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
index 210de0d3b86a..912d127b7b17 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
@@ -30,6 +30,7 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
+import java.util.Arrays;
 import java.util.regex.Pattern;
 
 /**
@@ -91,10 +92,12 @@ public void setCharset(final Charset charset) {
 
   @Override
   public int compareTo(byte[] value, int offset, int length) {
+    // See HBASE-9428. Make a copy of the relevant part of the byte[],
+    // or the JDK will copy the entire byte[] during String decode
+    byte[] tmp = Arrays.copyOfRange(value, offset, offset+length);
     // Use find() for subsequence match instead of matches() (full sequence
     // match) to adhere to the principle of least surprise.
-    return pattern.matcher(new String(value, offset, length, charset)).find() ? 0
-        : 1;
+    return pattern.matcher(new String(tmp, charset)).find() ? 0 : 1;
   }
 
   @Override

From e5a3f27a8cdc9eaae6ce5b97d25163e9f826562b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 4 Sep 2013 21:03:46 +0000
Subject: [PATCH 1141/1540] HBASE-9432 Backport HBASE-8781 to 0.94 (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520143 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java b/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
index 0cd521377b67..1ed73d16df3c 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
@@ -66,7 +66,7 @@ public ImmutableBytesWritable(byte[] bytes) {
    * @param ibw the value to set this ImmutableBytesWritable to.
    */
   public ImmutableBytesWritable(final ImmutableBytesWritable ibw) {
-    this(ibw.get(), 0, ibw.getSize());
+    this(ibw.get(), ibw.getOffset(), ibw.getSize());
   }
 
   /**

From 744a253a40340af2cd427dd73a78c7c2bfa6d00b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 4 Sep 2013 22:17:23 +0000
Subject: [PATCH 1142/1540] HBASE-9429 Add back
 MetaScanner.allTableRegions(Configuration conf,byte[] tablename,boolean
 offlined) method

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520155 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/MetaScanner.java  | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 4c02d6ffda52..be48fc6d8e08 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -261,6 +261,22 @@ public boolean processRowInternal(Result result) throws IOException {
     return regions;
   }
 
+  /**
+   * Lists all of the table regions currently in META.
+   * @param conf
+   * @param connection connection to be used internally (null to create a new connection)
+   * @param offlined True if we are to include offlined regions, false and we'll
+   * leave out offlined regions from returned list.
+   * @return Map of all user-space regions to servers
+   * @throws IOException
+   * @Deprecated Use {@link #allTableRegions(Configuration, HConnection, byte[], boolean)}
+   * instead
+   */
+  @Deprecated
+  public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
+      final byte[] tablename, final boolean offlined) throws IOException {
+    return allTableRegions(conf, null, tablename, offlined);
+  }
   /**
    * Lists all of the table regions currently in META.
    * @param conf

From 41b9c58f967f789c19d6926976d3ea0eb40a8a7c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 4 Sep 2013 23:23:51 +0000
Subject: [PATCH 1143/1540] HBASE-8930 Filter evaluates KVs outside requested
 columns (Vasu Mariyala)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520171 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ColumnTracker.java     |  39 +++-
 .../regionserver/ExplicitColumnTracker.java   |  60 +++---
 .../hbase/regionserver/ScanQueryMatcher.java  |  94 ++++++---
 .../ScanWildcardColumnTracker.java            |  19 +-
 .../filter/TestInvocationRecordFilter.java    | 192 ++++++++++++++++++
 .../TestExplicitColumnTracker.java            |   6 +-
 .../TestScanWildcardColumnTracker.java        |   6 +-
 7 files changed, 335 insertions(+), 81 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
index 693e7341a6dc..c192e9e96317 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
@@ -47,24 +47,45 @@
  * This class is NOT thread-safe as queries are never multi-threaded
  */
 public interface ColumnTracker {
+
   /**
-   * Keeps track of the number of versions for the columns asked for
+   * Checks if the column is present in the list of requested columns by returning the match code
+   * instance. It does not check against the number of versions for the columns asked for. To do the
+   * version check, one has to call {@link #checkVersions(byte[], int, int, long, byte, boolean)}
+   * method based on the return type (INCLUDE) of this method. The values that can be returned by
+   * this method are {@link MatchCode#INCLUDE}, {@link MatchCode#SEEK_NEXT_COL} and
+   * {@link MatchCode#SEEK_NEXT_ROW}.
    * @param bytes
    * @param offset
    * @param length
-   * @param ttl The timeToLive to enforce.
    * @param type The type of the KeyValue
-   * @param ignoreCount indicates if the KV needs to be excluded while counting
-   *   (used during compactions. We only count KV's that are older than all the
-   *   scanners' read points.)
    * @return The match code instance.
-   * @throws IOException in case there is an internal consistency problem
-   *      caused by a data corruption.
+   * @throws IOException in case there is an internal consistency problem caused by a data
+   *           corruption.
    */
-  public ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset,
-      int length, long ttl, byte type, boolean ignoreCount)
+  ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
       throws IOException;
 
+  /**
+   * Keeps track of the number of versions for the columns asked for. It assumes that the user has
+   * already checked if the keyvalue needs to be included by calling the
+   * {@link #checkColumn(byte[], int, int, byte)} method. The enum values returned by this method
+   * are {@link MatchCode#SKIP}, {@link MatchCode#INCLUDE},
+   * {@link MatchCode#INCLUDE_AND_SEEK_NEXT_COL} and {@link MatchCode#INCLUDE_AND_SEEK_NEXT_ROW}.
+   * Implementations which include all the columns could just return {@link MatchCode#INCLUDE} in
+   * the {@link #checkColumn(byte[], int, int, byte)} method and perform all the operations in this
+   * checkVersions method.
+   * @param type the type of the key value (Put/Delete)
+   * @param ttl The timeToLive to enforce.
+   * @param ignoreCount indicates if the KV needs to be excluded while counting (used during
+   *          compactions. We only count KV's that are older than all the scanners' read points.)
+   * @return the scan query matcher match code instance
+   * @throws IOException in case there is an internal consistency problem caused by a data
+   *           corruption.
+   */
+  ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long ttl,
+      byte type, boolean ignoreCount) throws IOException;
+
   /**
    * Resets the Matcher
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index d265a2725332..9590b318657b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -106,7 +106,7 @@ public ColumnCount getColumnHint() {
    */
   @Override
   public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
-      int length, long timestamp, byte type, boolean ignoreCount) {
+      int length, byte type) {
     // delete markers should never be passed to an
     // *Explicit*ColumnTracker
     assert !KeyValue.isDelete(type);
@@ -125,34 +125,9 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
       int ret = Bytes.compareTo(column.getBuffer(), column.getOffset(),
           column.getLength(), bytes, offset, length);
 
-      // Column Matches. If it is not a duplicate key, increment the version count
-      // and include.
+      // Column Matches. Return include code. The caller would call checkVersions
+      // to limit the number of versions.
       if(ret == 0) {
-        if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
-
-        //If column matches, check if it is a duplicate timestamp
-        if (sameAsPreviousTS(timestamp)) {
-          //If duplicate, skip this Key
-          return ScanQueryMatcher.MatchCode.SKIP;
-        }
-        int count = this.column.increment();
-        if(count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
-          // Done with versions for this column
-          ++this.index;
-          resetTS();
-          if (done()) {
-            // We have served all the requested columns.
-            this.column = null;
-            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
-          } else {
-            // We are done with current column; advance to next column
-            // of interest.
-            this.column = this.columns.get(this.index);
-            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
-          }
-        } else {
-          setTS(timestamp);
-        }
         return ScanQueryMatcher.MatchCode.INCLUDE;
       }
 
@@ -180,6 +155,35 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
     } while(true);
   }
 
+  @Override
+  public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length,
+      long timestamp, byte type, boolean ignoreCount) {
+    assert !KeyValue.isDelete(type);
+    if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+    // Check if it is a duplicate timestamp
+    if (sameAsPreviousTS(timestamp)) {
+      // If duplicate, skip this Key
+      return ScanQueryMatcher.MatchCode.SKIP;
+    }
+    int count = this.column.increment();
+    if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
+      // Done with versions for this column
+      ++this.index;
+      resetTS();
+      if (done()) {
+        // We have served all the requested columns.
+        this.column = null;
+        return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
+      }
+      // We are done with current column; advance to next column
+      // of interest.
+      this.column = this.columns.get(this.index);
+      return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+    }
+    setTS(timestamp);
+    return ScanQueryMatcher.MatchCode.INCLUDE;
+  }
+
   // Called between every row.
   public void reset() {
     this.index = 0;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 53e59ed0f403..ba11abd98c30 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -340,42 +340,60 @@ public MatchCode match(KeyValue kv) throws IOException {
       return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
     }
 
-    /**
-     * Filters should be checked before checking column trackers. If we do
-     * otherwise, as was previously being done, ColumnTracker may increment its
-     * counter for even that KV which may be discarded later on by Filter. This
-     * would lead to incorrect results in certain cases.
-     */
-    ReturnCode filterResponse = ReturnCode.SKIP;
-    if (filter != null) {
-      filterResponse = filter.filterKeyValue(kv);
-      if (filterResponse == ReturnCode.SKIP) {
-        return MatchCode.SKIP;
-      } else if (filterResponse == ReturnCode.NEXT_COL) {
-        return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
-      } else if (filterResponse == ReturnCode.NEXT_ROW) {
-        stickyNextRow = true;
-        return MatchCode.SEEK_NEXT_ROW;
-      } else if (filterResponse == ReturnCode.SEEK_NEXT_USING_HINT) {
-        return MatchCode.SEEK_NEXT_USING_HINT;
+    // STEP 1: Check if the column is part of the requested columns
+    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength, type);
+    if (colChecker == MatchCode.INCLUDE) {
+      ReturnCode filterResponse = ReturnCode.SKIP;
+      // STEP 2: Yes, the column is part of the requested columns. Check if filter is present
+      if (filter != null) {
+        // STEP 3: Filter the key value and return if it filters out
+        filterResponse = filter.filterKeyValue(kv);
+        switch (filterResponse) {
+        case SKIP:
+          return MatchCode.SKIP;
+        case NEXT_COL:
+          return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+        case NEXT_ROW:
+          stickyNextRow = true;
+          return MatchCode.SEEK_NEXT_ROW;
+        case SEEK_NEXT_USING_HINT:
+          return MatchCode.SEEK_NEXT_USING_HINT;
+        default:
+          //It means it is either include or include and seek next
+          break;
+        }
       }
+      /*
+       * STEP 4: Reaching this step means the column is part of the requested columns and either
+       * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response.
+       * Now check the number of versions needed. This method call returns SKIP, INCLUDE,
+       * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL.
+       *
+       * FilterResponse            ColumnChecker               Desired behavior
+       * INCLUDE                   SKIP                        row has already been included, SKIP.
+       * INCLUDE                   INCLUDE                     INCLUDE
+       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
+       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
+       * INCLUDE_AND_SEEK_NEXT_COL SKIP                        row has already been included, SKIP.
+       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE                     INCLUDE_AND_SEEK_NEXT_COL
+       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
+       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
+       *
+       * In all the above scenarios, we return the column checker return value except for
+       * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE)
+       */
+      colChecker =
+          columns.checkVersions(bytes, offset, qualLength, timestamp, type,
+            kv.getMemstoreTS() > maxReadPointToTrackVersions);
+      //Optimize with stickyNextRow
+      stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
+      return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
+          colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
+          : colChecker;
     }
-
-    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength,
-        timestamp, type, kv.getMemstoreTS() > maxReadPointToTrackVersions);
-    /*
-     * According to current implementation, colChecker can only be
-     * SEEK_NEXT_COL, SEEK_NEXT_ROW, SKIP or INCLUDE. Therefore, always return
-     * the MatchCode. If it is SEEK_NEXT_ROW, also set stickyNextRow.
-     */
-    if (colChecker == MatchCode.SEEK_NEXT_ROW) {
-      stickyNextRow = true;
-    } else if (filter != null && colChecker == MatchCode.INCLUDE &&
-               filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL) {
-      return MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
-    }
+    stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
+        : stickyNextRow;
     return colChecker;
-
   }
 
   public boolean moreRowsMayExistAfter(KeyValue kv) {
@@ -454,6 +472,16 @@ public KeyValue getKeyForNextRow(KeyValue kv) {
         null, 0, 0);
   }
 
+  // Used only for testing purposes
+  static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, int length,
+      long ttl, byte type, boolean ignoreCount) throws IOException {
+    MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
+    if (matchCode == MatchCode.INCLUDE) {
+      return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
+    }
+    return matchCode;
+  }
+
   /**
    * {@link #match} return codes.  These instruct the scanner moving through
    * memstores and StoreFiles what to do with the current KeyValue.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
index 569d0d34f6e6..53dd45517a15 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
@@ -61,13 +61,22 @@ public ScanWildcardColumnTracker(int minVersion, int maxVersion,
   /**
    * {@inheritDoc}
    * This receives puts *and* deletes.
-   * Deletes do not count as a version, but rather take the version
-   * of the previous put (so eventually all but the last can be reclaimed).
    */
   @Override
-  public MatchCode checkColumn(byte[] bytes, int offset, int length,
-      long timestamp, byte type, boolean ignoreCount) throws IOException {
-    
+  public MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
+      throws IOException {
+    return MatchCode.INCLUDE;
+  }
+
+  /**
+   * {@inheritDoc}
+   * This receives puts *and* deletes. Deletes do not count as a version, but rather
+   * take the version of the previous put (so eventually all but the last can be reclaimed).
+   */
+  @Override
+  public MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type,
+      boolean ignoreCount) throws IOException {
+
     if (columnBuffer == null) {
       // first iteration.
       resetBuffer(bytes, offset, length);
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
new file mode 100644
index 000000000000..b52efede8701
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
@@ -0,0 +1,192 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the invocation logic of the filters. A filter must be invoked only for
+ * the columns that are requested for.
+ */
+@Category(SmallTests.class)
+public class TestInvocationRecordFilter {
+
+  private static final byte[] TABLE_NAME_BYTES = Bytes
+      .toBytes("invocationrecord");
+  private static final byte[] FAMILY_NAME_BYTES = Bytes.toBytes("cf");
+
+  private static final byte[] ROW_BYTES = Bytes.toBytes("row");
+  private static final String QUALIFIER_PREFIX = "qualifier";
+  private static final String VALUE_PREFIX = "value";
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private HRegion region;
+
+  @Before
+  public void setUp() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME_BYTES);
+    htd.addFamily(new HColumnDescriptor(FAMILY_NAME_BYTES));
+    HRegionInfo info = new HRegionInfo(TABLE_NAME_BYTES, null, null, false);
+    this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
+        TEST_UTIL.getConfiguration(), htd);
+
+    Put put = new Put(ROW_BYTES);
+    for (int i = 0; i < 10; i += 2) {
+      // puts 0, 2, 4, 6 and 8
+      put.add(FAMILY_NAME_BYTES, Bytes.toBytes(QUALIFIER_PREFIX + i), i,
+          Bytes.toBytes(VALUE_PREFIX + i));
+    }
+    this.region.put(put);
+    this.region.flushcache();
+  }
+
+  @Test
+  public void testFilterInvocation() throws Exception {
+    List<Integer> selectQualifiers = new ArrayList<Integer>();
+    List<Integer> expectedQualifiers = new ArrayList<Integer>();
+
+    selectQualifiers.add(-1);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.clear();
+
+    selectQualifiers.add(0);
+    expectedQualifiers.add(0);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(3);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(4);
+    expectedQualifiers.add(4);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(5);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(8);
+    expectedQualifiers.add(8);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+  }
+
+  public void verifyInvocationResults(Integer[] selectQualifiers,
+      Integer[] expectedQualifiers) throws Exception {
+    Get get = new Get(ROW_BYTES);
+    for (int i = 0; i < selectQualifiers.length; i++) {
+      get.addColumn(FAMILY_NAME_BYTES,
+          Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
+    }
+
+    get.setFilter(new InvocationRecordFilter());
+
+    List<KeyValue> expectedValues = new ArrayList<KeyValue>();
+    for (int i = 0; i < expectedQualifiers.length; i++) {
+      expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes
+          .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
+          expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX
+              + expectedQualifiers[i])));
+    }
+
+    Scan scan = new Scan(get);
+    List<KeyValue> actualValues = new ArrayList<KeyValue>();
+    List<KeyValue> temp = new ArrayList<KeyValue>();
+    InternalScanner scanner = this.region.getScanner(scan);
+    while (scanner.next(temp)) {
+      actualValues.addAll(temp);
+      temp.clear();
+    }
+    actualValues.addAll(temp);
+    Assert.assertTrue("Actual values " + actualValues
+        + " differ from the expected values:" + expectedValues,
+        expectedValues.equals(actualValues));
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    HLog hlog = region.getLog();
+    region.close();
+    hlog.closeAndDelete();
+  }
+
+  /**
+   * Filter which gives the list of keyvalues for which the filter is invoked.
+   */
+  private static class InvocationRecordFilter extends FilterBase {
+
+    private List<KeyValue> visitedKeyValues = new ArrayList<KeyValue>();
+
+    public void reset() {
+      visitedKeyValues.clear();
+    }
+
+    public ReturnCode filterKeyValue(KeyValue ignored) {
+      visitedKeyValues.add(ignored);
+      return ReturnCode.INCLUDE;
+    }
+
+    public void filterRow(List<KeyValue> kvs) {
+      kvs.clear();
+      kvs.addAll(visitedKeyValues);
+    }
+
+    public boolean hasFilterRow() {
+      return true;
+    }
+
+    @Override
+    public void readFields(DataInput arg0) throws IOException {
+      //do nothing
+    }
+
+    @Override
+    public void write(DataOutput arg0) throws IOException {
+      //do nothing
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index 5993f1fce735..c69a6bee92da 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -56,7 +56,7 @@ private void runTest(int maxVersions,
     long timestamp = 0;
     //"Match"
     for(byte [] col : scannerColumns){
-      result.add(exp.checkColumn(col, 0, col.length, ++timestamp,
+      result.add(ScanQueryMatcher.checkColumn(exp, col, 0, col.length, ++timestamp,
           KeyValue.Type.Put.getCode(), false));
     }
 
@@ -169,14 +169,14 @@ public void testStackOverflow() throws IOException{
         Long.MIN_VALUE);
     for (int i = 0; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
-      explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+      ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
     explicit.reset();
 
     for (int i = 1; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
-      explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+      ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
index 086a4082ff0b..2a76c1e93bb9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
@@ -55,7 +55,7 @@ public void testCheckColumn_Ok() throws IOException {
     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<MatchCode>();
 
     for(byte [] qualifier : qualifiers) {
-      ScanQueryMatcher.MatchCode mc = tracker.checkColumn(qualifier, 0,
+      ScanQueryMatcher.MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0,
           qualifier.length, 1, KeyValue.Type.Put.getCode(), false);
       actual.add(mc);
     }
@@ -88,7 +88,7 @@ public void testCheckColumn_EnforceVersions() throws IOException {
 
     long timestamp = 0;
     for(byte [] qualifier : qualifiers) {
-      MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length,
+      MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length,
           ++timestamp, KeyValue.Type.Put.getCode(), false);
       actual.add(mc);
     }
@@ -112,7 +112,7 @@ public void DisabledTestCheckColumn_WrongOrder() {
 
     try {
       for(byte [] qualifier : qualifiers) {
-        tracker.checkColumn(qualifier, 0, qualifier.length, 1,
+        ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1,
             KeyValue.Type.Put.getCode(), false);
       }
     } catch (Exception e) {

From ca9d14195585f6d3ca3eb8f574641dd75755dcf3 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 5 Sep 2013 23:05:38 +0000
Subject: [PATCH 1144/1540] HBASE-9314. Dropping a table always prints a
 TableInfoMissingException in the master log

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520457 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/util/FSTableDescriptors.java | 56 ++++++++++++++++---
 .../apache/hadoop/hbase/util/HBaseFsck.java   |  2 +-
 2 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 11e532120605..eadd30cd22ca 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -44,7 +44,6 @@
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableInfoMissingException;
 
-
 /**
  * Implementation of {@link TableDescriptors} that reads descriptors from the
  * passed filesystem.  It expects descriptors to be in a file under the
@@ -190,11 +189,7 @@ public HTableDescriptor get(final String tablename)
           + tablename, ioe);
     }
     
-    if (tdmt == null) {
-      LOG.warn("The following folder is in HBase's root directory and " +
-        "doesn't contain a table descriptor, " +
-        "do consider deleting it: " + tablename);
-    } else {
+    if (tdmt != null) {
       this.cache.put(tablename, tdmt);
     }
     return tdmt == null ? null : tdmt.getTableDescriptor();
@@ -400,6 +395,44 @@ static long getTableInfoModtime(final FileSystem fs, final Path rootdir,
     return status == null? 0: status.getModificationTime();
   }
 
+  /**
+   * Returns the latest table descriptor for the given table directly from the file system
+   * if it exists, bypassing the local cache.
+   * Returns null if it's not found.
+   */
+  public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs,
+      Path hbaseRootDir, String tableName) throws IOException {
+    // ignore both -ROOT- and .META. tables
+    if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0
+        || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) {
+      return null;
+    }
+    Path tableDir = FSUtils.getTablePath(hbaseRootDir, tableName);
+    return getTableDescriptorFromFs(fs, tableDir);
+  }
+
+  /**
+   * Returns the latest table descriptor for the table located at the given directory
+   * directly from the file system if it exists.
+   * @throws TableInfoMissingException if there is no descriptor
+   */
+  public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
+  throws IOException {
+    FileStatus status = getTableInfoPath(fs, tableDir);
+    if (status == null) {
+      throw new TableInfoMissingException("No table descriptor file under " + tableDir);
+    }
+    FSDataInputStream fsDataInputStream = fs.open(status.getPath());
+    HTableDescriptor hTableDescriptor = null;
+    try {
+      hTableDescriptor = new HTableDescriptor();
+      hTableDescriptor.readFields(fsDataInputStream);
+    } finally {
+      fsDataInputStream.close();
+    }
+    return hTableDescriptor;
+  }
+
   /**
    * Get HTD from HDFS.
    * @param fs
@@ -440,13 +473,20 @@ static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs,
     return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName), readDirModtime);
   }
 
+  /**
+   * @param fs filesystem
+   * @param tableDir path to table directory
+   * @param readDirModtime true if dirmodtime should be read also
+   * @return TableDescriptorModtime or null if no table descriptor was found
+   * at the specified path
+   * @throws IOException
+   */
   static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir, boolean readDirModtime)
   throws NullPointerException, IOException {
     if (tableDir == null) throw new NullPointerException();
     FileStatus status = getTableInfoPath(fs, tableDir);
     if (status == null) {
-      throw new TableInfoMissingException("No .tableinfo file under "
-          + tableDir.toUri());
+      return null;
     }
     FSDataInputStream fsDataInputStream = fs.open(status.getPath());
     HTableDescriptor hTableDescriptor = null;
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 6ebe9e758ca6..789db91df442 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -793,7 +793,7 @@ private SortedMap<String, TableInfo> loadHdfsRegionInfos() throws IOException, I
         tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
-              FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(getConf()),
+              FSTableDescriptors.getTableDescriptorFromFs(hbaseRoot.getFileSystem(getConf()),
               hbaseRoot, tableName);
           modTInfo.htds.add(htd);
         } catch (IOException ioe) {

From b9a7fee84168a8aa28d09c979bffc399a22dafd4 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 6 Sep 2013 03:58:54 +0000
Subject: [PATCH 1145/1540] HBASE-9448. [0.94] Shell needs to fall back after
 HBASE-9182 if talking to older servers

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520488 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/ruby/hbase/admin.rb | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 103bbfe3489e..4ad3d88c7453 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -41,7 +41,19 @@ def initialize(configuration, formatter)
     #----------------------------------------------------------------------------------------------
     # Returns a list of tables in hbase
     def list(regex = ".*")
-      @admin.getTableNames(regex)
+      begin
+        # Use the old listTables API first for compatibility with older servers
+        @admin.listTables(regex).map { |t| t.getNameAsString }
+      rescue => e
+        # listTables failed, try the new unprivileged getTableNames API if the cause was
+        # an AccessDeniedException
+        if e.cause.kind_of? org.apache.hadoop.ipc.RemoteException and e.cause.unwrapRemoteException().kind_of? org.apache.hadoop.hbase.security.AccessDeniedException
+          @admin.getTableNames(regex)
+        else
+          # Not an access control failure, re-raise
+          raise e
+        end
+      end
     end
 
     #----------------------------------------------------------------------------------------------

From 9c238759b029a0a2ee7deba6832ac60e656ee353 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 6 Sep 2013 17:59:25 +0000
Subject: [PATCH 1146/1540] HBASE-9286 [0.94] ageOfLastShippedOp replication
 metric doesn't update if the slave regionserver is stalled (Alex Newman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520646 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSourceMetrics.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
index 85b42579eb0e..705a6dde55f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
@@ -111,6 +111,7 @@ public void refreshAgeOfLastShippedOp() {
   @Override
   public void doUpdates(MetricsContext metricsContext) {
     synchronized (this) {
+      refreshAgeOfLastShippedOp();
       this.shippedOpsRate.pushMetric(this.metricsRecord);
       this.shippedBatchesRate.pushMetric(this.metricsRecord);
       this.logEditsReadRate.pushMetric(this.metricsRecord);

From 7f1282beb290227c09df79051fc0e39db4544456 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 6 Sep 2013 23:44:18 +0000
Subject: [PATCH 1147/1540] HBASE-9455 Port HBASE-7113 'TestGzipFilter is flaky
 with jdk1.7' to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1520715 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/rest/TestGzipFilter.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java b/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java
index 570adddb51bd..4db967f13c22 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java
@@ -116,6 +116,8 @@ public void testGzipFilter() throws Exception {
     is.read(value, 0, VALUE_1.length);
     assertTrue(Bytes.equals(value, VALUE_1));
     is.close();
+
+    testScannerResultCodes();
   }
 
   @Test
@@ -133,8 +135,7 @@ public void testErrorNotGzipped() throws Exception {
     assertTrue(contentEncoding == null || !contentEncoding.contains("gzip"));
   }
 
-  @Test
-  public void testScannerResultCodes() throws Exception {
+  void testScannerResultCodes() throws Exception {
     Header[] headers = new Header[3];
     headers[0] = new Header("Content-Type", Constants.MIMETYPE_XML);
     headers[1] = new Header("Accept", Constants.MIMETYPE_JSON);

From 6905ddc2e1d80c9168261d799e270c50635040d8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 9 Sep 2013 17:40:10 +0000
Subject: [PATCH 1148/1540] HBASE-8930 REVERT due to test issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1521219 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ColumnTracker.java     |  39 +---
 .../regionserver/ExplicitColumnTracker.java   |  60 +++---
 .../hbase/regionserver/ScanQueryMatcher.java  |  94 +++------
 .../ScanWildcardColumnTracker.java            |  19 +-
 .../filter/TestInvocationRecordFilter.java    | 192 ------------------
 .../TestExplicitColumnTracker.java            |   6 +-
 .../TestScanWildcardColumnTracker.java        |   6 +-
 7 files changed, 81 insertions(+), 335 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
index c192e9e96317..693e7341a6dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
@@ -47,45 +47,24 @@
  * This class is NOT thread-safe as queries are never multi-threaded
  */
 public interface ColumnTracker {
-
   /**
-   * Checks if the column is present in the list of requested columns by returning the match code
-   * instance. It does not check against the number of versions for the columns asked for. To do the
-   * version check, one has to call {@link #checkVersions(byte[], int, int, long, byte, boolean)}
-   * method based on the return type (INCLUDE) of this method. The values that can be returned by
-   * this method are {@link MatchCode#INCLUDE}, {@link MatchCode#SEEK_NEXT_COL} and
-   * {@link MatchCode#SEEK_NEXT_ROW}.
+   * Keeps track of the number of versions for the columns asked for
    * @param bytes
    * @param offset
    * @param length
+   * @param ttl The timeToLive to enforce.
    * @param type The type of the KeyValue
+   * @param ignoreCount indicates if the KV needs to be excluded while counting
+   *   (used during compactions. We only count KV's that are older than all the
+   *   scanners' read points.)
    * @return The match code instance.
-   * @throws IOException in case there is an internal consistency problem caused by a data
-   *           corruption.
+   * @throws IOException in case there is an internal consistency problem
+   *      caused by a data corruption.
    */
-  ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
+  public ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset,
+      int length, long ttl, byte type, boolean ignoreCount)
       throws IOException;
 
-  /**
-   * Keeps track of the number of versions for the columns asked for. It assumes that the user has
-   * already checked if the keyvalue needs to be included by calling the
-   * {@link #checkColumn(byte[], int, int, byte)} method. The enum values returned by this method
-   * are {@link MatchCode#SKIP}, {@link MatchCode#INCLUDE},
-   * {@link MatchCode#INCLUDE_AND_SEEK_NEXT_COL} and {@link MatchCode#INCLUDE_AND_SEEK_NEXT_ROW}.
-   * Implementations which include all the columns could just return {@link MatchCode#INCLUDE} in
-   * the {@link #checkColumn(byte[], int, int, byte)} method and perform all the operations in this
-   * checkVersions method.
-   * @param type the type of the key value (Put/Delete)
-   * @param ttl The timeToLive to enforce.
-   * @param ignoreCount indicates if the KV needs to be excluded while counting (used during
-   *          compactions. We only count KV's that are older than all the scanners' read points.)
-   * @return the scan query matcher match code instance
-   * @throws IOException in case there is an internal consistency problem caused by a data
-   *           corruption.
-   */
-  ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long ttl,
-      byte type, boolean ignoreCount) throws IOException;
-
   /**
    * Resets the Matcher
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index 9590b318657b..d265a2725332 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -106,7 +106,7 @@ public ColumnCount getColumnHint() {
    */
   @Override
   public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
-      int length, byte type) {
+      int length, long timestamp, byte type, boolean ignoreCount) {
     // delete markers should never be passed to an
     // *Explicit*ColumnTracker
     assert !KeyValue.isDelete(type);
@@ -125,9 +125,34 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
       int ret = Bytes.compareTo(column.getBuffer(), column.getOffset(),
           column.getLength(), bytes, offset, length);
 
-      // Column Matches. Return include code. The caller would call checkVersions
-      // to limit the number of versions.
+      // Column Matches. If it is not a duplicate key, increment the version count
+      // and include.
       if(ret == 0) {
+        if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+
+        //If column matches, check if it is a duplicate timestamp
+        if (sameAsPreviousTS(timestamp)) {
+          //If duplicate, skip this Key
+          return ScanQueryMatcher.MatchCode.SKIP;
+        }
+        int count = this.column.increment();
+        if(count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
+          // Done with versions for this column
+          ++this.index;
+          resetTS();
+          if (done()) {
+            // We have served all the requested columns.
+            this.column = null;
+            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
+          } else {
+            // We are done with current column; advance to next column
+            // of interest.
+            this.column = this.columns.get(this.index);
+            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+          }
+        } else {
+          setTS(timestamp);
+        }
         return ScanQueryMatcher.MatchCode.INCLUDE;
       }
 
@@ -155,35 +180,6 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
     } while(true);
   }
 
-  @Override
-  public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length,
-      long timestamp, byte type, boolean ignoreCount) {
-    assert !KeyValue.isDelete(type);
-    if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
-    // Check if it is a duplicate timestamp
-    if (sameAsPreviousTS(timestamp)) {
-      // If duplicate, skip this Key
-      return ScanQueryMatcher.MatchCode.SKIP;
-    }
-    int count = this.column.increment();
-    if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
-      // Done with versions for this column
-      ++this.index;
-      resetTS();
-      if (done()) {
-        // We have served all the requested columns.
-        this.column = null;
-        return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
-      }
-      // We are done with current column; advance to next column
-      // of interest.
-      this.column = this.columns.get(this.index);
-      return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
-    }
-    setTS(timestamp);
-    return ScanQueryMatcher.MatchCode.INCLUDE;
-  }
-
   // Called between every row.
   public void reset() {
     this.index = 0;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index ba11abd98c30..53e59ed0f403 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -340,60 +340,42 @@ public MatchCode match(KeyValue kv) throws IOException {
       return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
     }
 
-    // STEP 1: Check if the column is part of the requested columns
-    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength, type);
-    if (colChecker == MatchCode.INCLUDE) {
-      ReturnCode filterResponse = ReturnCode.SKIP;
-      // STEP 2: Yes, the column is part of the requested columns. Check if filter is present
-      if (filter != null) {
-        // STEP 3: Filter the key value and return if it filters out
-        filterResponse = filter.filterKeyValue(kv);
-        switch (filterResponse) {
-        case SKIP:
-          return MatchCode.SKIP;
-        case NEXT_COL:
-          return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
-        case NEXT_ROW:
-          stickyNextRow = true;
-          return MatchCode.SEEK_NEXT_ROW;
-        case SEEK_NEXT_USING_HINT:
-          return MatchCode.SEEK_NEXT_USING_HINT;
-        default:
-          //It means it is either include or include and seek next
-          break;
-        }
+    /**
+     * Filters should be checked before checking column trackers. If we do
+     * otherwise, as was previously being done, ColumnTracker may increment its
+     * counter for even that KV which may be discarded later on by Filter. This
+     * would lead to incorrect results in certain cases.
+     */
+    ReturnCode filterResponse = ReturnCode.SKIP;
+    if (filter != null) {
+      filterResponse = filter.filterKeyValue(kv);
+      if (filterResponse == ReturnCode.SKIP) {
+        return MatchCode.SKIP;
+      } else if (filterResponse == ReturnCode.NEXT_COL) {
+        return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+      } else if (filterResponse == ReturnCode.NEXT_ROW) {
+        stickyNextRow = true;
+        return MatchCode.SEEK_NEXT_ROW;
+      } else if (filterResponse == ReturnCode.SEEK_NEXT_USING_HINT) {
+        return MatchCode.SEEK_NEXT_USING_HINT;
       }
-      /*
-       * STEP 4: Reaching this step means the column is part of the requested columns and either
-       * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response.
-       * Now check the number of versions needed. This method call returns SKIP, INCLUDE,
-       * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL.
-       *
-       * FilterResponse            ColumnChecker               Desired behavior
-       * INCLUDE                   SKIP                        row has already been included, SKIP.
-       * INCLUDE                   INCLUDE                     INCLUDE
-       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
-       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
-       * INCLUDE_AND_SEEK_NEXT_COL SKIP                        row has already been included, SKIP.
-       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE                     INCLUDE_AND_SEEK_NEXT_COL
-       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
-       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
-       *
-       * In all the above scenarios, we return the column checker return value except for
-       * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE)
-       */
-      colChecker =
-          columns.checkVersions(bytes, offset, qualLength, timestamp, type,
-            kv.getMemstoreTS() > maxReadPointToTrackVersions);
-      //Optimize with stickyNextRow
-      stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
-      return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
-          colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
-          : colChecker;
     }
-    stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
-        : stickyNextRow;
+
+    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength,
+        timestamp, type, kv.getMemstoreTS() > maxReadPointToTrackVersions);
+    /*
+     * According to current implementation, colChecker can only be
+     * SEEK_NEXT_COL, SEEK_NEXT_ROW, SKIP or INCLUDE. Therefore, always return
+     * the MatchCode. If it is SEEK_NEXT_ROW, also set stickyNextRow.
+     */
+    if (colChecker == MatchCode.SEEK_NEXT_ROW) {
+      stickyNextRow = true;
+    } else if (filter != null && colChecker == MatchCode.INCLUDE &&
+               filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL) {
+      return MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+    }
     return colChecker;
+
   }
 
   public boolean moreRowsMayExistAfter(KeyValue kv) {
@@ -472,16 +454,6 @@ public KeyValue getKeyForNextRow(KeyValue kv) {
         null, 0, 0);
   }
 
-  // Used only for testing purposes
-  static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, int length,
-      long ttl, byte type, boolean ignoreCount) throws IOException {
-    MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
-    if (matchCode == MatchCode.INCLUDE) {
-      return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
-    }
-    return matchCode;
-  }
-
   /**
    * {@link #match} return codes.  These instruct the scanner moving through
    * memstores and StoreFiles what to do with the current KeyValue.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
index 53dd45517a15..569d0d34f6e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
@@ -61,22 +61,13 @@ public ScanWildcardColumnTracker(int minVersion, int maxVersion,
   /**
    * {@inheritDoc}
    * This receives puts *and* deletes.
+   * Deletes do not count as a version, but rather take the version
+   * of the previous put (so eventually all but the last can be reclaimed).
    */
   @Override
-  public MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
-      throws IOException {
-    return MatchCode.INCLUDE;
-  }
-
-  /**
-   * {@inheritDoc}
-   * This receives puts *and* deletes. Deletes do not count as a version, but rather
-   * take the version of the previous put (so eventually all but the last can be reclaimed).
-   */
-  @Override
-  public MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type,
-      boolean ignoreCount) throws IOException {
-
+  public MatchCode checkColumn(byte[] bytes, int offset, int length,
+      long timestamp, byte type, boolean ignoreCount) throws IOException {
+    
     if (columnBuffer == null) {
       // first iteration.
       resetBuffer(bytes, offset, length);
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
deleted file mode 100644
index b52efede8701..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.filter;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test the invocation logic of the filters. A filter must be invoked only for
- * the columns that are requested for.
- */
-@Category(SmallTests.class)
-public class TestInvocationRecordFilter {
-
-  private static final byte[] TABLE_NAME_BYTES = Bytes
-      .toBytes("invocationrecord");
-  private static final byte[] FAMILY_NAME_BYTES = Bytes.toBytes("cf");
-
-  private static final byte[] ROW_BYTES = Bytes.toBytes("row");
-  private static final String QUALIFIER_PREFIX = "qualifier";
-  private static final String VALUE_PREFIX = "value";
-
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private HRegion region;
-
-  @Before
-  public void setUp() throws Exception {
-    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME_BYTES);
-    htd.addFamily(new HColumnDescriptor(FAMILY_NAME_BYTES));
-    HRegionInfo info = new HRegionInfo(TABLE_NAME_BYTES, null, null, false);
-    this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
-        TEST_UTIL.getConfiguration(), htd);
-
-    Put put = new Put(ROW_BYTES);
-    for (int i = 0; i < 10; i += 2) {
-      // puts 0, 2, 4, 6 and 8
-      put.add(FAMILY_NAME_BYTES, Bytes.toBytes(QUALIFIER_PREFIX + i), i,
-          Bytes.toBytes(VALUE_PREFIX + i));
-    }
-    this.region.put(put);
-    this.region.flushcache();
-  }
-
-  @Test
-  public void testFilterInvocation() throws Exception {
-    List<Integer> selectQualifiers = new ArrayList<Integer>();
-    List<Integer> expectedQualifiers = new ArrayList<Integer>();
-
-    selectQualifiers.add(-1);
-    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
-        expectedQualifiers.toArray(new Integer[0]));
-
-    selectQualifiers.clear();
-
-    selectQualifiers.add(0);
-    expectedQualifiers.add(0);
-    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
-        expectedQualifiers.toArray(new Integer[0]));
-
-    selectQualifiers.add(3);
-    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
-        expectedQualifiers.toArray(new Integer[0]));
-
-    selectQualifiers.add(4);
-    expectedQualifiers.add(4);
-    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
-        expectedQualifiers.toArray(new Integer[0]));
-
-    selectQualifiers.add(5);
-    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
-        expectedQualifiers.toArray(new Integer[0]));
-
-    selectQualifiers.add(8);
-    expectedQualifiers.add(8);
-    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
-        expectedQualifiers.toArray(new Integer[0]));
-  }
-
-  public void verifyInvocationResults(Integer[] selectQualifiers,
-      Integer[] expectedQualifiers) throws Exception {
-    Get get = new Get(ROW_BYTES);
-    for (int i = 0; i < selectQualifiers.length; i++) {
-      get.addColumn(FAMILY_NAME_BYTES,
-          Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
-    }
-
-    get.setFilter(new InvocationRecordFilter());
-
-    List<KeyValue> expectedValues = new ArrayList<KeyValue>();
-    for (int i = 0; i < expectedQualifiers.length; i++) {
-      expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes
-          .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
-          expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX
-              + expectedQualifiers[i])));
-    }
-
-    Scan scan = new Scan(get);
-    List<KeyValue> actualValues = new ArrayList<KeyValue>();
-    List<KeyValue> temp = new ArrayList<KeyValue>();
-    InternalScanner scanner = this.region.getScanner(scan);
-    while (scanner.next(temp)) {
-      actualValues.addAll(temp);
-      temp.clear();
-    }
-    actualValues.addAll(temp);
-    Assert.assertTrue("Actual values " + actualValues
-        + " differ from the expected values:" + expectedValues,
-        expectedValues.equals(actualValues));
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    HLog hlog = region.getLog();
-    region.close();
-    hlog.closeAndDelete();
-  }
-
-  /**
-   * Filter which gives the list of keyvalues for which the filter is invoked.
-   */
-  private static class InvocationRecordFilter extends FilterBase {
-
-    private List<KeyValue> visitedKeyValues = new ArrayList<KeyValue>();
-
-    public void reset() {
-      visitedKeyValues.clear();
-    }
-
-    public ReturnCode filterKeyValue(KeyValue ignored) {
-      visitedKeyValues.add(ignored);
-      return ReturnCode.INCLUDE;
-    }
-
-    public void filterRow(List<KeyValue> kvs) {
-      kvs.clear();
-      kvs.addAll(visitedKeyValues);
-    }
-
-    public boolean hasFilterRow() {
-      return true;
-    }
-
-    @Override
-    public void readFields(DataInput arg0) throws IOException {
-      //do nothing
-    }
-
-    @Override
-    public void write(DataOutput arg0) throws IOException {
-      //do nothing
-    }
-  }
-}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index c69a6bee92da..5993f1fce735 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -56,7 +56,7 @@ private void runTest(int maxVersions,
     long timestamp = 0;
     //"Match"
     for(byte [] col : scannerColumns){
-      result.add(ScanQueryMatcher.checkColumn(exp, col, 0, col.length, ++timestamp,
+      result.add(exp.checkColumn(col, 0, col.length, ++timestamp,
           KeyValue.Type.Put.getCode(), false));
     }
 
@@ -169,14 +169,14 @@ public void testStackOverflow() throws IOException{
         Long.MIN_VALUE);
     for (int i = 0; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
-      ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+      explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
     explicit.reset();
 
     for (int i = 1; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
-      ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+      explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
index 2a76c1e93bb9..086a4082ff0b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
@@ -55,7 +55,7 @@ public void testCheckColumn_Ok() throws IOException {
     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<MatchCode>();
 
     for(byte [] qualifier : qualifiers) {
-      ScanQueryMatcher.MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0,
+      ScanQueryMatcher.MatchCode mc = tracker.checkColumn(qualifier, 0,
           qualifier.length, 1, KeyValue.Type.Put.getCode(), false);
       actual.add(mc);
     }
@@ -88,7 +88,7 @@ public void testCheckColumn_EnforceVersions() throws IOException {
 
     long timestamp = 0;
     for(byte [] qualifier : qualifiers) {
-      MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length,
+      MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length,
           ++timestamp, KeyValue.Type.Put.getCode(), false);
       actual.add(mc);
     }
@@ -112,7 +112,7 @@ public void DisabledTestCheckColumn_WrongOrder() {
 
     try {
       for(byte [] qualifier : qualifiers) {
-        ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1,
+        tracker.checkColumn(qualifier, 0, qualifier.length, 1,
             KeyValue.Type.Put.getCode(), false);
       }
     } catch (Exception e) {

From e6291fe23c7f44f3994abe8feb8592573b86c786 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 9 Sep 2013 18:25:24 +0000
Subject: [PATCH 1149/1540] HBASE-9301 Default hbase.dynamic.jars.dir to
 hbase.rootdir/jars (Vasu Mariyala)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1521228 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/resources/hbase-default.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index fd8a1fe07ef2..28f346dd9e88 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -992,4 +992,14 @@
         this config.  See the end of HBASE-8389 for more.
     </description>
   </property>
+  <property>
+  	<name>hbase.dynamic.jars.dir</name>
+  	<value>${hbase.rootdir}/.lib</value>
+	<description>
+	The directory from which the custom filter/co-processor jars can be loaded
+	dynamically by the region server without the need to restart. However,
+	an already loaded filter/co-processor class would not be un-loaded. See
+	HBASE-1936 for more details.
+    </description>
+  </property>
 </configuration>

From e15deb5cf1c9d4b637aef1774511db6cc354cc02 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 10 Sep 2013 04:58:57 +0000
Subject: [PATCH 1150/1540] HBASE-8930 REAPPLY with testfix

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1521356 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/ColumnTracker.java     |  39 +++-
 .../regionserver/ExplicitColumnTracker.java   |  60 +++---
 .../hbase/regionserver/ScanQueryMatcher.java  |  94 ++++++---
 .../ScanWildcardColumnTracker.java            |  19 +-
 .../filter/TestInvocationRecordFilter.java    | 192 ++++++++++++++++++
 .../TestExplicitColumnTracker.java            |   6 +-
 .../TestScanWildcardColumnTracker.java        |   6 +-
 7 files changed, 335 insertions(+), 81 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
index 693e7341a6dc..c192e9e96317 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
@@ -47,24 +47,45 @@
  * This class is NOT thread-safe as queries are never multi-threaded
  */
 public interface ColumnTracker {
+
   /**
-   * Keeps track of the number of versions for the columns asked for
+   * Checks if the column is present in the list of requested columns by returning the match code
+   * instance. It does not check against the number of versions for the columns asked for. To do the
+   * version check, one has to call {@link #checkVersions(byte[], int, int, long, byte, boolean)}
+   * method based on the return type (INCLUDE) of this method. The values that can be returned by
+   * this method are {@link MatchCode#INCLUDE}, {@link MatchCode#SEEK_NEXT_COL} and
+   * {@link MatchCode#SEEK_NEXT_ROW}.
    * @param bytes
    * @param offset
    * @param length
-   * @param ttl The timeToLive to enforce.
    * @param type The type of the KeyValue
-   * @param ignoreCount indicates if the KV needs to be excluded while counting
-   *   (used during compactions. We only count KV's that are older than all the
-   *   scanners' read points.)
    * @return The match code instance.
-   * @throws IOException in case there is an internal consistency problem
-   *      caused by a data corruption.
+   * @throws IOException in case there is an internal consistency problem caused by a data
+   *           corruption.
    */
-  public ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset,
-      int length, long ttl, byte type, boolean ignoreCount)
+  ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
       throws IOException;
 
+  /**
+   * Keeps track of the number of versions for the columns asked for. It assumes that the user has
+   * already checked if the keyvalue needs to be included by calling the
+   * {@link #checkColumn(byte[], int, int, byte)} method. The enum values returned by this method
+   * are {@link MatchCode#SKIP}, {@link MatchCode#INCLUDE},
+   * {@link MatchCode#INCLUDE_AND_SEEK_NEXT_COL} and {@link MatchCode#INCLUDE_AND_SEEK_NEXT_ROW}.
+   * Implementations which include all the columns could just return {@link MatchCode#INCLUDE} in
+   * the {@link #checkColumn(byte[], int, int, byte)} method and perform all the operations in this
+   * checkVersions method.
+   * @param type the type of the key value (Put/Delete)
+   * @param ttl The timeToLive to enforce.
+   * @param ignoreCount indicates if the KV needs to be excluded while counting (used during
+   *          compactions. We only count KV's that are older than all the scanners' read points.)
+   * @return the scan query matcher match code instance
+   * @throws IOException in case there is an internal consistency problem caused by a data
+   *           corruption.
+   */
+  ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long ttl,
+      byte type, boolean ignoreCount) throws IOException;
+
   /**
    * Resets the Matcher
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index d265a2725332..9590b318657b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -106,7 +106,7 @@ public ColumnCount getColumnHint() {
    */
   @Override
   public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
-      int length, long timestamp, byte type, boolean ignoreCount) {
+      int length, byte type) {
     // delete markers should never be passed to an
     // *Explicit*ColumnTracker
     assert !KeyValue.isDelete(type);
@@ -125,34 +125,9 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
       int ret = Bytes.compareTo(column.getBuffer(), column.getOffset(),
           column.getLength(), bytes, offset, length);
 
-      // Column Matches. If it is not a duplicate key, increment the version count
-      // and include.
+      // Column Matches. Return include code. The caller would call checkVersions
+      // to limit the number of versions.
       if(ret == 0) {
-        if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
-
-        //If column matches, check if it is a duplicate timestamp
-        if (sameAsPreviousTS(timestamp)) {
-          //If duplicate, skip this Key
-          return ScanQueryMatcher.MatchCode.SKIP;
-        }
-        int count = this.column.increment();
-        if(count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
-          // Done with versions for this column
-          ++this.index;
-          resetTS();
-          if (done()) {
-            // We have served all the requested columns.
-            this.column = null;
-            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
-          } else {
-            // We are done with current column; advance to next column
-            // of interest.
-            this.column = this.columns.get(this.index);
-            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
-          }
-        } else {
-          setTS(timestamp);
-        }
         return ScanQueryMatcher.MatchCode.INCLUDE;
       }
 
@@ -180,6 +155,35 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
     } while(true);
   }
 
+  @Override
+  public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length,
+      long timestamp, byte type, boolean ignoreCount) {
+    assert !KeyValue.isDelete(type);
+    if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+    // Check if it is a duplicate timestamp
+    if (sameAsPreviousTS(timestamp)) {
+      // If duplicate, skip this Key
+      return ScanQueryMatcher.MatchCode.SKIP;
+    }
+    int count = this.column.increment();
+    if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
+      // Done with versions for this column
+      ++this.index;
+      resetTS();
+      if (done()) {
+        // We have served all the requested columns.
+        this.column = null;
+        return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
+      }
+      // We are done with current column; advance to next column
+      // of interest.
+      this.column = this.columns.get(this.index);
+      return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+    }
+    setTS(timestamp);
+    return ScanQueryMatcher.MatchCode.INCLUDE;
+  }
+
   // Called between every row.
   public void reset() {
     this.index = 0;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 53e59ed0f403..ba11abd98c30 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -340,42 +340,60 @@ public MatchCode match(KeyValue kv) throws IOException {
       return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
     }
 
-    /**
-     * Filters should be checked before checking column trackers. If we do
-     * otherwise, as was previously being done, ColumnTracker may increment its
-     * counter for even that KV which may be discarded later on by Filter. This
-     * would lead to incorrect results in certain cases.
-     */
-    ReturnCode filterResponse = ReturnCode.SKIP;
-    if (filter != null) {
-      filterResponse = filter.filterKeyValue(kv);
-      if (filterResponse == ReturnCode.SKIP) {
-        return MatchCode.SKIP;
-      } else if (filterResponse == ReturnCode.NEXT_COL) {
-        return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
-      } else if (filterResponse == ReturnCode.NEXT_ROW) {
-        stickyNextRow = true;
-        return MatchCode.SEEK_NEXT_ROW;
-      } else if (filterResponse == ReturnCode.SEEK_NEXT_USING_HINT) {
-        return MatchCode.SEEK_NEXT_USING_HINT;
+    // STEP 1: Check if the column is part of the requested columns
+    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength, type);
+    if (colChecker == MatchCode.INCLUDE) {
+      ReturnCode filterResponse = ReturnCode.SKIP;
+      // STEP 2: Yes, the column is part of the requested columns. Check if filter is present
+      if (filter != null) {
+        // STEP 3: Filter the key value and return if it filters out
+        filterResponse = filter.filterKeyValue(kv);
+        switch (filterResponse) {
+        case SKIP:
+          return MatchCode.SKIP;
+        case NEXT_COL:
+          return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+        case NEXT_ROW:
+          stickyNextRow = true;
+          return MatchCode.SEEK_NEXT_ROW;
+        case SEEK_NEXT_USING_HINT:
+          return MatchCode.SEEK_NEXT_USING_HINT;
+        default:
+          //It means it is either include or include and seek next
+          break;
+        }
       }
+      /*
+       * STEP 4: Reaching this step means the column is part of the requested columns and either
+       * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response.
+       * Now check the number of versions needed. This method call returns SKIP, INCLUDE,
+       * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL.
+       *
+       * FilterResponse            ColumnChecker               Desired behavior
+       * INCLUDE                   SKIP                        row has already been included, SKIP.
+       * INCLUDE                   INCLUDE                     INCLUDE
+       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
+       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
+       * INCLUDE_AND_SEEK_NEXT_COL SKIP                        row has already been included, SKIP.
+       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE                     INCLUDE_AND_SEEK_NEXT_COL
+       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
+       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
+       *
+       * In all the above scenarios, we return the column checker return value except for
+       * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE)
+       */
+      colChecker =
+          columns.checkVersions(bytes, offset, qualLength, timestamp, type,
+            kv.getMemstoreTS() > maxReadPointToTrackVersions);
+      //Optimize with stickyNextRow
+      stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
+      return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
+          colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
+          : colChecker;
     }
-
-    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength,
-        timestamp, type, kv.getMemstoreTS() > maxReadPointToTrackVersions);
-    /*
-     * According to current implementation, colChecker can only be
-     * SEEK_NEXT_COL, SEEK_NEXT_ROW, SKIP or INCLUDE. Therefore, always return
-     * the MatchCode. If it is SEEK_NEXT_ROW, also set stickyNextRow.
-     */
-    if (colChecker == MatchCode.SEEK_NEXT_ROW) {
-      stickyNextRow = true;
-    } else if (filter != null && colChecker == MatchCode.INCLUDE &&
-               filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL) {
-      return MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
-    }
+    stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
+        : stickyNextRow;
     return colChecker;
-
   }
 
   public boolean moreRowsMayExistAfter(KeyValue kv) {
@@ -454,6 +472,16 @@ public KeyValue getKeyForNextRow(KeyValue kv) {
         null, 0, 0);
   }
 
+  // Used only for testing purposes
+  static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, int length,
+      long ttl, byte type, boolean ignoreCount) throws IOException {
+    MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
+    if (matchCode == MatchCode.INCLUDE) {
+      return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
+    }
+    return matchCode;
+  }
+
   /**
    * {@link #match} return codes.  These instruct the scanner moving through
    * memstores and StoreFiles what to do with the current KeyValue.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
index 569d0d34f6e6..53dd45517a15 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
@@ -61,13 +61,22 @@ public ScanWildcardColumnTracker(int minVersion, int maxVersion,
   /**
    * {@inheritDoc}
    * This receives puts *and* deletes.
-   * Deletes do not count as a version, but rather take the version
-   * of the previous put (so eventually all but the last can be reclaimed).
    */
   @Override
-  public MatchCode checkColumn(byte[] bytes, int offset, int length,
-      long timestamp, byte type, boolean ignoreCount) throws IOException {
-    
+  public MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
+      throws IOException {
+    return MatchCode.INCLUDE;
+  }
+
+  /**
+   * {@inheritDoc}
+   * This receives puts *and* deletes. Deletes do not count as a version, but rather
+   * take the version of the previous put (so eventually all but the last can be reclaimed).
+   */
+  @Override
+  public MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type,
+      boolean ignoreCount) throws IOException {
+
     if (columnBuffer == null) {
       // first iteration.
       resetBuffer(bytes, offset, length);
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
new file mode 100644
index 000000000000..ee2eb8c3b6da
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
@@ -0,0 +1,192 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the invocation logic of the filters. A filter must be invoked only for
+ * the columns that are requested for.
+ */
+@Category(SmallTests.class)
+public class TestInvocationRecordFilter {
+
+  private static final byte[] TABLE_NAME_BYTES = Bytes
+      .toBytes("invocationrecord");
+  private static final byte[] FAMILY_NAME_BYTES = Bytes.toBytes("mycf");
+
+  private static final byte[] ROW_BYTES = Bytes.toBytes("row");
+  private static final String QUALIFIER_PREFIX = "qualifier";
+  private static final String VALUE_PREFIX = "value";
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private HRegion region;
+
+  @Before
+  public void setUp() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME_BYTES);
+    htd.addFamily(new HColumnDescriptor(FAMILY_NAME_BYTES));
+    HRegionInfo info = new HRegionInfo(TABLE_NAME_BYTES, null, null, false);
+    this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
+        TEST_UTIL.getConfiguration(), htd);
+
+    Put put = new Put(ROW_BYTES);
+    for (int i = 0; i < 10; i += 2) {
+      // puts 0, 2, 4, 6 and 8
+      put.add(FAMILY_NAME_BYTES, Bytes.toBytes(QUALIFIER_PREFIX + i), i,
+          Bytes.toBytes(VALUE_PREFIX + i));
+    }
+    this.region.put(put);
+    this.region.flushcache();
+  }
+
+  @Test
+  public void testFilterInvocation() throws Exception {
+    List<Integer> selectQualifiers = new ArrayList<Integer>();
+    List<Integer> expectedQualifiers = new ArrayList<Integer>();
+
+    selectQualifiers.add(-1);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.clear();
+
+    selectQualifiers.add(0);
+    expectedQualifiers.add(0);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(3);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(4);
+    expectedQualifiers.add(4);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(5);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+
+    selectQualifiers.add(8);
+    expectedQualifiers.add(8);
+    verifyInvocationResults(selectQualifiers.toArray(new Integer[0]),
+        expectedQualifiers.toArray(new Integer[0]));
+  }
+
+  public void verifyInvocationResults(Integer[] selectQualifiers,
+      Integer[] expectedQualifiers) throws Exception {
+    Get get = new Get(ROW_BYTES);
+    for (int i = 0; i < selectQualifiers.length; i++) {
+      get.addColumn(FAMILY_NAME_BYTES,
+          Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
+    }
+
+    get.setFilter(new InvocationRecordFilter());
+
+    List<KeyValue> expectedValues = new ArrayList<KeyValue>();
+    for (int i = 0; i < expectedQualifiers.length; i++) {
+      expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes
+          .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
+          expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX
+              + expectedQualifiers[i])));
+    }
+
+    Scan scan = new Scan(get);
+    List<KeyValue> actualValues = new ArrayList<KeyValue>();
+    List<KeyValue> temp = new ArrayList<KeyValue>();
+    InternalScanner scanner = this.region.getScanner(scan);
+    while (scanner.next(temp)) {
+      actualValues.addAll(temp);
+      temp.clear();
+    }
+    actualValues.addAll(temp);
+    Assert.assertTrue("Actual values " + actualValues
+        + " differ from the expected values:" + expectedValues,
+        expectedValues.equals(actualValues));
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    HLog hlog = region.getLog();
+    region.close();
+    hlog.closeAndDelete();
+  }
+
+  /**
+   * Filter which gives the list of keyvalues for which the filter is invoked.
+   */
+  private static class InvocationRecordFilter extends FilterBase {
+
+    private List<KeyValue> visitedKeyValues = new ArrayList<KeyValue>();
+
+    public void reset() {
+      visitedKeyValues.clear();
+    }
+
+    public ReturnCode filterKeyValue(KeyValue ignored) {
+      visitedKeyValues.add(ignored);
+      return ReturnCode.INCLUDE;
+    }
+
+    public void filterRow(List<KeyValue> kvs) {
+      kvs.clear();
+      kvs.addAll(visitedKeyValues);
+    }
+
+    public boolean hasFilterRow() {
+      return true;
+    }
+
+    @Override
+    public void readFields(DataInput arg0) throws IOException {
+      //do nothing
+    }
+
+    @Override
+    public void write(DataOutput arg0) throws IOException {
+      //do nothing
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index 5993f1fce735..c69a6bee92da 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -56,7 +56,7 @@ private void runTest(int maxVersions,
     long timestamp = 0;
     //"Match"
     for(byte [] col : scannerColumns){
-      result.add(exp.checkColumn(col, 0, col.length, ++timestamp,
+      result.add(ScanQueryMatcher.checkColumn(exp, col, 0, col.length, ++timestamp,
           KeyValue.Type.Put.getCode(), false));
     }
 
@@ -169,14 +169,14 @@ public void testStackOverflow() throws IOException{
         Long.MIN_VALUE);
     for (int i = 0; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
-      explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+      ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
     explicit.reset();
 
     for (int i = 1; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
-      explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+      ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
           false);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
index 086a4082ff0b..2a76c1e93bb9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
@@ -55,7 +55,7 @@ public void testCheckColumn_Ok() throws IOException {
     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<MatchCode>();
 
     for(byte [] qualifier : qualifiers) {
-      ScanQueryMatcher.MatchCode mc = tracker.checkColumn(qualifier, 0,
+      ScanQueryMatcher.MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0,
           qualifier.length, 1, KeyValue.Type.Put.getCode(), false);
       actual.add(mc);
     }
@@ -88,7 +88,7 @@ public void testCheckColumn_EnforceVersions() throws IOException {
 
     long timestamp = 0;
     for(byte [] qualifier : qualifiers) {
-      MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length,
+      MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length,
           ++timestamp, KeyValue.Type.Put.getCode(), false);
       actual.add(mc);
     }
@@ -112,7 +112,7 @@ public void DisabledTestCheckColumn_WrongOrder() {
 
     try {
       for(byte [] qualifier : qualifiers) {
-        tracker.checkColumn(qualifier, 0, qualifier.length, 1,
+        ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1,
             KeyValue.Type.Put.getCode(), false);
       }
     } catch (Exception e) {

From 80759212d6fc0c2f790dc87ca5d6bbd9759462d6 Mon Sep 17 00:00:00 2001
From: Gary Helmling <garyh@apache.org>
Date: Tue, 10 Sep 2013 21:09:35 +0000
Subject: [PATCH 1151/1540] Update garyh affiliation

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1521644 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9bd36b495d8b..4c45209a4ef5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,8 +129,8 @@
       <name>Gary Helmling</name>
       <email>garyh@apache.org</email>
       <timezone>-8</timezone>
-      <organization>Twitter</organization>
-      <organizationUrl>http://www.twitter.com</organizationUrl>
+      <organization>Continuuity</organization>
+      <organizationUrl>http://www.continuuity.com</organizationUrl>
     </developer>
     <developer>
       <id>jdcryans</id>

From a8c3669ede371fd994393bcf398480044ac6a667 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 10 Sep 2013 21:46:33 +0000
Subject: [PATCH 1152/1540] Backport HBASE- 9208 - ReplicationLogCleaner slow
 at large scale (Dave Latham)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1521662 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HBaseFileSystem.java  |  14 +-
 .../cleaner/BaseFileCleanerDelegate.java      |  49 +++++
 .../cleaner/BaseHFileCleanerDelegate.java     |   4 +-
 .../cleaner/BaseLogCleanerDelegate.java       |  10 +-
 .../hbase/master/cleaner/CleanerChore.java    | 197 ++++++++++--------
 .../master/cleaner/FileCleanerDelegate.java   |   8 +-
 .../master/cleaner/HFileLinkCleaner.java      |   5 -
 .../master/ReplicationLogCleaner.java         |  83 ++++----
 .../org/apache/hadoop/hbase/util/FSUtils.java |   5 +-
 9 files changed, 219 insertions(+), 156 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
index fdbc343a3b83..a061018d1249 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
@@ -56,22 +56,22 @@ public abstract class HBaseFileSystem {
   }
 
   /**
-   * Deletes a file. Assumes the user has already checked for this directory existence.
+   * Deletes a file. Assumes the user has already checked for this file's existence.
    * @param fs
-   * @param dir
-   * @return true if the directory is deleted.
+   * @param file
+   * @return true if the file is deleted.
    * @throws IOException
    */
-  public static boolean deleteFileFromFileSystem(FileSystem fs, Path dir)
+  public static boolean deleteFileFromFileSystem(FileSystem fs, Path file)
       throws IOException {
     IOException lastIOE = null;
     int i = 0;
     do {
       try {
-        return fs.delete(dir, false);
+        return fs.delete(file, false);
       } catch (IOException ioe) {
         lastIOE = ioe;
-        if (!fs.exists(dir)) return true;
+        if (!fs.exists(file)) return true;
         // dir is there, retry deleting after some time.
         sleepBeforeRetry("Delete File", i + 1);
       }
@@ -81,7 +81,7 @@ public static boolean deleteFileFromFileSystem(FileSystem fs, Path dir)
   
   
   /**
-   * Deletes a directory. Assumes the user has already checked for this directory existence.
+   * Deletes a directory. Assumes the user has already checked for this directory's existence.
    * @param fs
    * @param dir
    * @return true if the directory is deleted.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java
new file mode 100644
index 000000000000..10492919e2af
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hbase.BaseConfigurable;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
+/**
+ * Base class for file cleaners which allows subclasses to implement a simple
+ * isFileDeletable method (which used to be the FileCleanerDelegate contract).
+ */
+public abstract class BaseFileCleanerDelegate extends BaseConfigurable
+implements FileCleanerDelegate {
+
+  @Override
+  public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
+    return Iterables.filter(files, new Predicate<FileStatus>() {
+      @Override
+      public boolean apply(FileStatus file) {
+        return isFileDeletable(file);
+      }});
+  }
+
+  /**
+   * Should the master delete the file or keep it?
+   * @param fStat file status of the file to check
+   * @return <tt>true</tt> if the file is deletable, <tt>false</tt> if not
+   */
+  protected abstract boolean isFileDeletable(FileStatus fStat);
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
index ed890ac98928..1a2aad98df84 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hbase.master.cleaner;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.BaseConfigurable;
 
 /**
  * Base class for the hfile cleaning function inside the master. By default, only the
@@ -36,8 +35,7 @@
  * provide a default constructor.
  */
 @InterfaceAudience.Private
-public abstract class BaseHFileCleanerDelegate extends BaseConfigurable implements
-    FileCleanerDelegate {
+public abstract class BaseHFileCleanerDelegate extends BaseFileCleanerDelegate {
 
   private boolean stopped = false;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
index 3b85c145455e..847a2a710826 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
@@ -19,8 +19,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.BaseConfigurable;
 
 /**
  * Base class for the log cleaning function inside the master. By default, two
@@ -38,7 +36,7 @@
  * provide a default constructor.
  */
 @InterfaceAudience.Private
-public abstract class BaseLogCleanerDelegate extends BaseConfigurable implements FileCleanerDelegate {
+public abstract class BaseLogCleanerDelegate extends BaseFileCleanerDelegate {
 
   @Override
   public boolean isFileDeletable(FileStatus fStat) {
@@ -50,8 +48,10 @@ public boolean isFileDeletable(FileStatus fStat) {
    * <p>
    * Implementing classes should override {@link #isFileDeletable(FileStatus)} instead.
    * @param fStat file status of the file
-   * @return true if the log is deletable, false if not
+   * @return true if the log is deletable, false (default) if not
    */
   @Deprecated
-  public abstract boolean isLogDeletable(FileStatus fStat);
+  public boolean isLogDeletable(FileStatus fStat) {
+    return false;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 7d214604e3af..847ea9900237 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -33,6 +33,11 @@
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.util.FSUtils;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
 /**
  * Abstract Cleaner that uses a chain of delegates to clean a directory of files
  * @param <T> Cleaner delegate class that is dynamically loaded from configuration
@@ -74,7 +79,7 @@ public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Confi
   protected abstract boolean validate(Path file);
 
   /**
-   * Instanitate and initialize all the file cleaners set in the configuration
+   * Instantiate and initialize all the file cleaners set in the configuration
    * @param confKey key to get the file cleaner classes from the configuration
    */
   private void initCleanerChain(String confKey) {
@@ -98,7 +103,7 @@ private void initCleanerChain(String confKey) {
    * @param conf
    * @return the new instance
    */
-  public T newFileCleaner(String className, Configuration conf) {
+  private T newFileCleaner(String className, Configuration conf) {
     try {
       Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
         FileCleanerDelegate.class);
@@ -116,131 +121,153 @@ public T newFileCleaner(String className, Configuration conf) {
   @Override
   protected void chore() {
     try {
-      FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
-      // if the path (file or directory) doesn't exist, then we can just return
-      if (files == null) return;
-      // loop over the found files and see if they should be deleted
-      for (FileStatus file : files) {
-        try {
-          if (file.isDir()) checkAndDeleteDirectory(file.getPath());
-          else checkAndDelete(file);
-        } catch (IOException e) {
-          e = RemoteExceptionHandler.checkIOException(e);
-          LOG.warn("Error while cleaning the logs", e);
-        }
-      }
+      FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir);
+      checkAndDeleteEntries(files);
     } catch (IOException e) {
-      LOG.warn("Failed to get status of:" + oldFileDir);
+      e = RemoteExceptionHandler.checkIOException(e);
+      LOG.warn("Error while cleaning the logs", e);
     }
-
   }
 
+  /**
+   * Loop over the given directory entries, and check whether they can be deleted.
+   * If an entry is itself a directory it will be recursively checked and deleted itself iff
+   * all subentries are deleted (and no new subentries are added in the mean time)
+   *
+   * @param entries directory entries to check
+   * @return true if all entries were successfully deleted
+   */
+  private boolean checkAndDeleteEntries(FileStatus[] entries) {
+    if (entries == null) {
+      return true;
+    }
+    boolean allEntriesDeleted = true;
+    List<FileStatus> files = Lists.newArrayListWithCapacity(entries.length);
+    for (FileStatus child : entries) {
+      Path path = child.getPath();
+      if (child.isDir()) {
+        // for each subdirectory delete it and all entries if possible
+        if (!checkAndDeleteDirectory(path)) {
+          allEntriesDeleted = false;
+        }
+      } else {
+        // collect all files to attempt to delete in one batch
+        files.add(child);
+      }
+    }
+    if (!checkAndDeleteFiles(files)) {
+      allEntriesDeleted = false;
+    }
+    return allEntriesDeleted;
+  }
+  
   /**
    * Attempt to delete a directory and all files under that directory. Each child file is passed
-   * through the delegates to see if it can be deleted. If the directory has not children when the
+   * through the delegates to see if it can be deleted. If the directory has no children when the
    * cleaners have finished it is deleted.
    * <p>
    * If new children files are added between checks of the directory, the directory will <b>not</b>
    * be deleted.
-   * @param toCheck directory to check
+   * @param dir directory to check
    * @return <tt>true</tt> if the directory was deleted, <tt>false</tt> otherwise.
-   * @throws IOException if there is an unexpected filesystem error
    */
-  public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
+  @VisibleForTesting boolean checkAndDeleteDirectory(Path dir) {
     if (LOG.isTraceEnabled()) {
-      LOG.trace("Checking directory: " + toCheck);
+      LOG.trace("Checking directory: " + dir);
     }
-    FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
-    // if the directory doesn't exist, then we are done
-    if (children == null) {
-      try {
-        return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
-      } catch (IOException e) {
-        if (LOG.isTraceEnabled()) {
-          LOG.trace("Couldn't delete directory: " + toCheck, e);
-        }
-      }
-      // couldn't delete w/o exception, so we can't return success.
-      return false;
-    }     
 
-    boolean canDeleteThis = true;
-    for (FileStatus child : children) {
-      Path path = child.getPath();
-      // attempt to delete all the files under the directory
-      if (child.isDir()) {
-        if (!checkAndDeleteDirectory(path)) {
-          canDeleteThis = false;
-        }
-      }
-      // otherwise we can just check the file
-      else if (!checkAndDelete(child)) {
-        canDeleteThis = false;
-      }
+    try {
+      FileStatus[] children = FSUtils.listStatus(fs, dir);
+      boolean allChildrenDeleted = checkAndDeleteEntries(children);
+  
+      // if the directory still has children, we can't delete it, so we are done
+      if (!allChildrenDeleted) return false;
+    } catch (IOException e) {
+      e = RemoteExceptionHandler.checkIOException(e);
+      LOG.warn("Error while listing directory: " + dir, e);
+      // couldn't list directory, so don't try to delete, and don't return success
+      return false;
     }
 
-    // if the directory has children, we can't delete it, so we are done
-    if (!canDeleteThis) return false;
-
     // otherwise, all the children (that we know about) have been deleted, so we should try to
     // delete this directory. However, don't do so recursively so we don't delete files that have
     // been added since we last checked.
     try {
-      return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
+      return HBaseFileSystem.deleteFileFromFileSystem(fs, dir);
     } catch (IOException e) {
       if (LOG.isTraceEnabled()) {
-        LOG.trace("Couldn't delete directory: " + toCheck, e);
+        LOG.trace("Couldn't delete directory: " + dir, e);
       }
+      // couldn't delete w/o exception, so we can't return success.
+      return false;
     }
-
-    // couldn't delete w/o exception, so we can't return success.
-    return false;
   }
 
   /**
-   * Run the given file through each of the cleaners to see if it should be deleted, deleting it if
+   * Run the given files through each of the cleaners to see if it should be deleted, deleting it if
    * necessary.
-   * @param fStat path of the file to check (and possibly delete)
-   * @throws IOException if cann't delete a file because of a filesystem issue
-   * @throws IllegalArgumentException if the file is a directory and has children
+   * @param files List of FileStatus for the files to check (and possibly delete)
+   * @return true iff successfully deleted all files
    */
-  private boolean checkAndDelete(FileStatus fStat) throws IOException, IllegalArgumentException {
-    Path filePath = fStat.getPath();
+  private boolean checkAndDeleteFiles(List<FileStatus> files) {
     // first check to see if the path is valid
-    if (!validate(filePath)) {
-      LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
-      boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, filePath);
-      if (!success) LOG.warn("Attempted to delete:" + filePath
-          + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
-
-      return success;
+    List<FileStatus> validFiles = Lists.newArrayListWithCapacity(files.size());
+    List<FileStatus> invalidFiles = Lists.newArrayList();
+    for (FileStatus file : files) {
+      if (validate(file.getPath())) {
+        validFiles.add(file);
+      } else {
+        LOG.warn("Found a wrongly formatted file: " + file.getPath() + " - will delete it.");
+        invalidFiles.add(file);
+      }
     }
-    // check each of the cleaners for the file
+
+    Iterable<FileStatus> deletableValidFiles = validFiles;
+    // check each of the cleaners for the valid files
     for (T cleaner : cleanersChain) {
       if (cleaner.isStopped() || this.stopper.isStopped()) {
-        LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
+        LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any more files in:"
             + this.oldFileDir);
         return false;
       }
 
-      if (!cleaner.isFileDeletable(fStat)) {
-        // this file is not deletable, then we are done
-        if (LOG.isTraceEnabled()) {
-          LOG.trace(filePath + " is not deletable according to:" + cleaner);
+      Iterable<FileStatus> filteredFiles = cleaner.getDeletableFiles(deletableValidFiles);
+      
+      // trace which cleaner is holding on to each file
+      if (LOG.isTraceEnabled()) {
+        ImmutableSet<FileStatus> filteredFileSet = ImmutableSet.copyOf(filteredFiles);
+        for (FileStatus file : deletableValidFiles) {
+          if (!filteredFileSet.contains(file)) {
+            LOG.trace(file.getPath() + " is not deletable according to:" + cleaner);
+          }
         }
-        return false;
       }
+      
+      deletableValidFiles = filteredFiles;
     }
-    // delete this file if it passes all the cleaners
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("Removing:" + filePath + " from archive");
-    }
-    boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, filePath);
-    if (!success) {
-      LOG.warn("Attempted to delete:" + filePath
-          + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
+    
+    Iterable<FileStatus> filesToDelete = Iterables.concat(invalidFiles, deletableValidFiles);
+    int deletedFileCount = 0;
+    for (FileStatus file : filesToDelete) {
+      Path filePath = file.getPath();
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Removing: " + filePath + " from archive");
+      }
+      try {
+        boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, filePath);
+        if (success) {
+          deletedFileCount++;
+        } else {
+          LOG.warn("Attempted to delete:" + filePath
+              + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
+        }
+      } catch (IOException e) {
+        e = RemoteExceptionHandler.checkIOException(e);
+        LOG.warn("Error while deleting: " + filePath, e);
+      }
     }
-    return success;
+
+    return deletedFileCount == files.size();
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
index c03278195d70..ab706334047c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
@@ -31,9 +31,9 @@
 public interface FileCleanerDelegate extends Configurable, Stoppable {
 
   /**
-   * Should the master delete the file or keep it?
-   * @param fStat file status of the file to check
-   * @return <tt>true</tt> if the file is deletable, <tt>false</tt> if not
+   * Determines which of the given files are safe to delete
+   * @param files files to check for deletion
+   * @return files that are ok to delete according to this cleaner
    */
-  public boolean isFileDeletable(FileStatus fStat);
+  Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files);
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
index 4c39f7353b32..47af39ff071c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
@@ -18,8 +18,6 @@
 package org.apache.hadoop.hbase.master.cleaner;
 
 import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -28,11 +26,8 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
 
 /**
  * HFileLink cleaner that determines if a hfile should be deleted.
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index caf77b948a61..203407f96296 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -19,11 +19,14 @@
  */
 package org.apache.hadoop.hbase.replication.master;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -32,10 +35,11 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
 
 /**
  * Implementation of a log cleaner that checks if a log is still scheduled for
@@ -44,7 +48,6 @@
 public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abortable {
   private static final Log LOG = LogFactory.getLog(ReplicationLogCleaner.class);
   private ReplicationZookeeper zkHelper;
-  private Set<String> hlogs = new HashSet<String>();
   private boolean stopped = false;
   private boolean aborted;
 
@@ -54,51 +57,49 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abo
   public ReplicationLogCleaner() {}
 
   @Override
-  public boolean isLogDeletable(FileStatus fStat) {
-
+  public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
     try {
       if (!zkHelper.getReplication()) {
-        return false;
+        return ImmutableList.of();
       }
     } catch (KeeperException e) {
       abort("Cannot get the state of replication", e);
-      return false;
+      return ImmutableList.of();
     }
 
-    // all members of this class are null if replication is disabled, and we
-    // return true since false would render the LogsCleaner useless
+   // all members of this class are null if replication is disabled, 
+   // so we cannot filter the files
     if (this.getConf() == null) {
-      return true;
-    }
-    String log = fStat.getPath().getName();
-    // If we saw the hlog previously, let's consider it's still used
-    // At some point in the future we will refresh the list and it will be gone
-    if (this.hlogs.contains(log)) {
-      return false;
+      return files;
     }
-
-    // Let's see it's still there
-    // This solution makes every miss very expensive to process since we
-    // almost completely refresh the cache each time
-    return !refreshHLogsAndSearch(log);
+    
+    final Set<String> hlogs = loadHLogsFromQueues();
+    return Iterables.filter(files, new Predicate<FileStatus>() {
+      @Override
+      public boolean apply(FileStatus file) {
+        String hlog = file.getPath().getName();
+        boolean logInReplicationQueue = hlogs.contains(hlog);
+        if (LOG.isDebugEnabled()) {
+          if (logInReplicationQueue) {
+            LOG.debug("Found log in ZK, keeping: " + hlog);
+          } else {
+            LOG.debug("Didn't find this log in ZK, deleting: " + hlog);
+          }
+        }
+       return !logInReplicationQueue;
+      }});
   }
 
   /**
-   * Search through all the hlogs we have in ZK to refresh the cache
-   * If a log is specified and found, then we early out and return true
-   * @param searchedLog log we are searching for, pass null to cache everything
-   *                    that's in zookeeper.
-   * @return false until a specified log is found.
+   * Load all hlogs in all replication queues from ZK
    */
-  private boolean refreshHLogsAndSearch(String searchedLog) {
-    this.hlogs.clear();
-    final boolean lookForLog = searchedLog != null;
+  private Set<String> loadHLogsFromQueues() {
     List<String> rss = zkHelper.getListOfReplicators();
     if (rss == null) {
-      LOG.debug("Didn't find any region server that replicates, deleting: " +
-          searchedLog);
-      return false;
+      LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
+      return ImmutableSet.of();
     }
+    Set<String> hlogs = Sets.newHashSet();
     for (String rs: rss) {
       List<String> listOfPeers = zkHelper.getListPeersForRS(rs);
       // if rs just died, this will be null
@@ -108,23 +109,18 @@ private boolean refreshHLogsAndSearch(String searchedLog) {
       for (String id : listOfPeers) {
         List<String> peersHlogs = zkHelper.getListHLogsForPeerForRS(rs, id);
         if (peersHlogs != null) {
-          this.hlogs.addAll(peersHlogs);
-        }
-        // early exit if we found the log
-        if(lookForLog && this.hlogs.contains(searchedLog)) {
-          LOG.debug("Found log in ZK, keeping: " + searchedLog);
-          return true;
+          hlogs.addAll(peersHlogs);
         }
       }
     }
-    LOG.debug("Didn't find this log in ZK, deleting: " + searchedLog);
-    return false;
+    return hlogs;
   }
 
   @Override
   public void setConf(Configuration config) {
     // If replication is disabled, keep all members null
     if (!config.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
+      LOG.warn("Not configured - allowing all hlogs to be deleted");
       return;
     }
     // Make my own Configuration.  Then I'll have my own connection to zk that
@@ -139,7 +135,6 @@ public void setConf(Configuration config) {
     } catch (IOException e) {
       LOG.error("Error while configuring " + this.getClass().getName(), e);
     }
-    refreshHLogsAndSearch(null);
   }
 
 
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 1bc3eea7d535..75f85b04954c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -56,7 +56,6 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 
@@ -1262,7 +1261,7 @@ public static Map<String, Path> getTableStoreFilePathMap(
    * @param fs file system
    * @param dir directory
    * @param filter path filter
-   * @return null if tabledir doesn't exist, otherwise FileStatus array
+   * @return null if dir is empty or doesn't exist, otherwise FileStatus array
    */
   public static FileStatus [] listStatus(final FileSystem fs,
       final Path dir, final PathFilter filter) throws IOException {
@@ -1283,7 +1282,7 @@ public static Map<String, Path> getTableStoreFilePathMap(
    *
    * @param fs file system
    * @param dir directory
-   * @return null if tabledir doesn't exist, otherwise FileStatus array
+   * @return null if dir is empty or doesn't exist, otherwise FileStatus array
    */
   public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
     return listStatus(fs, dir, null);

From 35b052cd0e0e791c57d75acbec6a94380831b903 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 11 Sep 2013 02:07:33 +0000
Subject: [PATCH 1153/1540] pom.xml and CHANGES.txt for 0.94.12RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1521717 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index f0400eeff9d6..fa8c85a9319b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,55 @@
 HBase Change Log
 
+Release 0.94.12 - 9/10/2013
+Sub-task
+
+    [HBASE-9277] - REST should use listTableNames to list tables
+    [HBASE-9279] - Thrift should use listTableNames to list tables
+
+Bug
+
+    [HBASE-7709] - Infinite loop possible in Master/Master replication
+    [HBASE-7954] - Fix the retrying logic of memstore flushes to avoid extra sleep
+    [HBASE-8760] - possible loss of data in snapshot taken after region split
+    [HBASE-8930] - Filter evaluates KVs outside requested columns
+    [HBASE-9167] - ServerCallable retries just once if timeout is not integer.max
+    [HBASE-9182] - Allow non-admin users to list all table names
+    [HBASE-9195] - Fix TestFSHDFSUtils against java7 test re-ordering
+    [HBASE-9207] - An Offline SplitParent Region can be assigned breaking split references
+    [HBASE-9231] - Multipage book is generated to the wrong location
+    [HBASE-9233] - isTableAvailable() may be stuck if an offline parent was never assigned
+    [HBASE-9252] - HConnectionManager#getZooKeeperWatcher() should be deprecated in 0.94
+    [HBASE-9256] - HBaseClient#setupIOStreams should handle all exceptions
+    [HBASE-9266] - Javadoc: Document that HBaseAdmin.flush(...) is synchronous
+    [HBASE-9286] - [0.94] ageOfLastShippedOp replication metric doesn't update if the slave regionserver is stalled
+    [HBASE-9301] - Default hbase.dynamic.jars.dir to hbase.rootdir/jars
+    [HBASE-9303] - Snapshot restore of table which splits after snapshot was taken encounters 'Region is not online'
+    [HBASE-9326] - ServerName is created using getLocalSocketAddress, breaks binding to the wildcard address. Revert HBASE-8640
+    [HBASE-9329] - SnapshotManager should check for directory existance before throwing a warning.
+    [HBASE-9344] - RegionServer not shutting down upon KeeperException in open region
+    [HBASE-9356] - [0.94] SecureServer.INSECURE_VERSIONS is declared incorrectly
+    [HBASE-9397] - Snapshots with the same name are allowed to proceed concurrently
+    [HBASE-9415] - In rpcServer, replicationQueue is initialized with the max queue size instead of the max queue lenght
+    [HBASE-9428] - Regex filters are at least an order of magnitude slower since 0.94.3
+    [HBASE-9429] - Add back MetaScanner.allTableRegions(Configuration conf,byte[] tablename,boolean offlined) method
+    [HBASE-9432] - Backport HBASE-8781 to 0.94
+    [HBASE-9448] - [0.94] Shell needs to fall back after HBASE-9182 if talking to older servers
+    [HBASE-9455] - Port HBASE-7113 'TestGzipFilter is flaky with jdk1.7' to 0.94
+
+Improvement
+
+    [HBASE-9243] - Add more useful statistics in the HFile tool
+    [HBASE-9314] - Dropping a table always prints a TableInfoMissingException in the master log
+
+Task
+
+    [HBASE-9377] - Backport HBASE- 9208 "ReplicationLogCleaner slow at large scale"
+
+Test
+
+    [HBASE-9287] - TestCatalogTracker depends on the execution order
+
+
 Release 0.94.11 - 8/13/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 4c45209a4ef5..5f63b5b0dd05 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.12-SNAPSHOT</version>
+  <version>0.94.12</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 40d63222795a9d00b7013050f9b54bed520a35ba Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 11 Sep 2013 19:44:13 +0000
Subject: [PATCH 1154/1540] HBASE-9462 HBaseAdmin#isTableEnabled() should throw
 exception for non-existent table

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1522008 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/HBaseAdmin.java    | 15 ++++++++++++---
 .../org/apache/hadoop/hbase/client/TestAdmin.java | 15 +++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 6bdebd3f7a52..5a3536c71a70 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -948,6 +948,16 @@ public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
     return failed.toArray(new HTableDescriptor[failed.size()]);
   }
 
+  /*
+   * Checks whether table exists. If not, throws TableNotFoundException
+   * @param tableName
+   */
+  private void checkTableExistence(byte[] tableName) throws IOException {
+    if (!tableExists(tableName)) {
+      throw new TableNotFoundException(Bytes.toString(tableName));
+    }
+  }
+
   /**
    * @param tableName name of table to check
    * @return true if table is on-line
@@ -965,9 +975,7 @@ public boolean isTableEnabled(byte[] tableName) throws IOException {
     if (!HTableDescriptor.isMetaTable(tableName)) {
       HTableDescriptor.isLegalTableName(tableName);
     }
-    if(!tableExists(tableName)){
-      throw new TableNotFoundException(Bytes.toString(tableName));
-    }
+    checkTableExistence(tableName);
     return connection.isTableEnabled(tableName);
   }
 
@@ -989,6 +997,7 @@ public boolean isTableDisabled(byte[] tableName) throws IOException {
     if (!HTableDescriptor.isMetaTable(tableName)) {
       HTableDescriptor.isLegalTableName(tableName);
     }
+    checkTableExistence(tableName);
     return connection.isTableDisabled(tableName);
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index ce068cd21f47..71dd59f532e3 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -1649,6 +1649,21 @@ public void testRootTableSplit() throws Exception {
     assertEquals("ROOT region should not be splitted.",1, regions.size());
   }
 
+  @Test
+  public void testIsEnabledOrDisabledOnUnknownTable() throws Exception {
+    try {
+      admin.isTableEnabled(Bytes.toBytes("unkownTable"));
+      fail("Test should fail if isTableEnabled called on unknown table.");
+    } catch (IOException e) {
+    }
+
+    try {
+      admin.isTableDisabled(Bytes.toBytes("unkownTable"));
+      fail("Test should fail if isTableDisabled called on unknown table.");
+    } catch (IOException e) {
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 85e2c12adc4a36f5e5c756ea3cb5e88a18a2c804 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 11 Sep 2013 21:04:47 +0000
Subject: [PATCH 1155/1540] ASE-9468 Previous active master can still serves
 RPC request when it is trying recovering expired zk session (Feng Honghua)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1522033 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 5 ++++-
 src/main/resources/hbase-default.xml                      | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 50e7989426d7..f975a34b75bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1708,7 +1708,10 @@ private boolean abortNow(final String msg, final Throwable t) {
     if (!this.isActiveMaster || this.stopped) {
       return true;
     }
-    if (t != null && t instanceof KeeperException.SessionExpiredException) {
+
+    boolean failFast = conf.getBoolean("fail.fast.expired.active.master", false);
+    if (t != null && t instanceof KeeperException.SessionExpiredException
+        && !failFast) {
       try {
         LOG.info("Primary Master trying to recover from ZooKeeper session " +
             "expiry.");
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 28f346dd9e88..6c71817c0594 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -272,6 +272,12 @@
       master for communication and display purposes.
     </description>
   </property>
+  <property>
+    <name>fail.fast.expired.active.master</name>
+    <value>false</value>
+    <description>If abort immediately for the expired master without trying
+      to recover its zk session.</description>
+  </property>
   <property>
     <name>hbase.master.dns.interface</name>
     <value>default</value>

From 11f1037a8d4caa1d4be2f2b449f330d300d5b9de Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 11 Sep 2013 21:09:05 +0000
Subject: [PATCH 1156/1540] HBASE-9482 Do not enforce secure Hadoop for secure
 HBase (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1522036 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/security/User.java   | 11 ++++-------
 src/main/resources/hbase-default.xml                  |  8 ++++++++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index 298e3ab3a931..a0cacc08b0b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -234,15 +234,12 @@ public static boolean isSecurityEnabled() {
   }
 
   /**
-   * Returns whether or not secure authentication is enabled for HBase.  Note that
-   * HBase security requires HDFS security to provide any guarantees, so this requires that
-   * both <code>hbase.security.authentication</code> and <code>hadoop.security.authentication</code>
-   * are set to <code>kerberos</code>.
+   * Returns whether or not secure authentication is enabled for HBase. Note that
+   * HBase security requires HDFS security to provide any guarantees, so it is
+   * recommended that secure HBase should run on secure HDFS.
    */
   public static boolean isHBaseSecurityEnabled(Configuration conf) {
-    return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY)) &&
-        "kerberos".equalsIgnoreCase(
-            conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION));
+    return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY));
   }
 
   /* Concrete implementations */
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 6c71817c0594..41a68e7fc1a1 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -1008,4 +1008,12 @@
 	HBASE-1936 for more details.
     </description>
   </property>
+  <property>
+    <name>hbase.security.authentication</name>
+    <value>simple</value>
+    <description>
+        Controls whether or not secure authentication is enabled for HBase.
+        Possible values are 'simple' (no authentication), and 'kerberos'.
+     </description>
+   </property>
 </configuration>

From 4327ff27d83b474a81ef17609a8c4f0246da269e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 11 Sep 2013 21:26:07 +0000
Subject: [PATCH 1157/1540] [0.94] Backport HBASE-9309 The links in the backup
 masters template are bad

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1522054 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
index 660f441c1d1a..1dabe274d255 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
@@ -50,7 +50,7 @@ ServerName [] serverNames = masters.toArray(new ServerName[masters.size()]);
 </%java>
 <%if (!master.isActiveMaster()) %>
     <h2>Master</h2>
-    <a href="<% serverNames[0].getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverNames[0].getHostname() %></a>
+    <a href="http://<% serverNames[0].getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverNames[0].getHostname() %></a>
 <%else>
     <h2>Backup Masters</h2>
 
@@ -65,7 +65,7 @@ ServerName [] serverNames = masters.toArray(new ServerName[masters.size()]);
     for (ServerName serverName: serverNames) {
     </%java>
     <tr>
-        <td><a href="<% serverName.getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverName.getHostname() %></a></td>
+        <td><a href="http://<% serverName.getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverName.getHostname() %></a></td>
         <td><% serverName.getPort() %></td>
         <td><% new Date(serverName.getStartcode()) %></td>
     </tr>

From da37f491ec67ccfdca8512c0e91a86b2a063d840 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Sep 2013 06:29:16 +0000
Subject: [PATCH 1158/1540] HBASE-9566 Add back WALEdit#get/setScopes method
 (Jesse)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524304 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/WALEdit.java       | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index 0d244b496b40..eb4a59f4164f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -140,6 +140,26 @@ public Integer getFromScope(byte[] key) {
     return scopes.get(key);
   }
 
+  /**
+   * @return the underlying replication scope map
+   * @deprecated use {@link #getFromScope(byte[])} instead
+   */
+  @Deprecated
+  public NavigableMap<byte[], Integer> getScopes() {
+    return scopes;
+  }
+
+  /**
+   * @param scopes set all the replication scope information. Must be non-<tt>null</tt>
+   * @deprecated use {@link #putIntoScope(byte[], Integer)} instead. This completely overrides any
+   *             existing scopes
+   */
+  @Deprecated
+  public void setScopes(NavigableMap<byte[], Integer> scopes) {
+    this.scopes.clear();
+    this.scopes.putAll(scopes);
+  }
+
   public void putIntoScope(byte[] key, Integer value) {
     scopes.put(key, value);
   }

From 69e56e387dd91463a64d12f8dcaa6c054736dd36 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Wed, 18 Sep 2013 18:37:40 +0000
Subject: [PATCH 1159/1540] HBASE-9534: Short-Circuit Coprocessor HTable access
 when on the same server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524522 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/CoprocessorHConnection.java  | 124 ++++++++++++++++++
 .../apache/hadoop/hbase/client/HTable.java    |  32 ++++-
 .../hbase/coprocessor/CoprocessorHost.java    |  29 +++-
 3 files changed, 177 insertions(+), 8 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java b/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
new file mode 100644
index 000000000000..f5d684b5858a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Connection to an HTable from within a Coprocessor. Can do some nice tricks since we know we are
+ * on a regionserver.
+ * <p>
+ * This shouldn't be used by usual HBase clients - its merely in this package to maintain visibility
+ * considerations for the {@link HConnectionImplementation}.
+ */
+@SuppressWarnings("javadoc")
+public class CoprocessorHConnection extends HConnectionManager.HConnectionImplementation {
+
+  /**
+   * Create an unmanaged {@link HConnection} based on the environment in which we are running the
+   * coprocessor. The {@link HConnection} must be externally cleaned up (we bypass the usual HTable
+   * cleanup mechanisms since we own everything).
+   * @param env environment hosting the {@link HConnection}
+   * @return an unmanaged {@link HConnection}.
+   * @throws IOException if we cannot create the basic connection
+   */
+  @SuppressWarnings("resource")
+  public static HConnection getConnectionForEnvironment(CoprocessorEnvironment env)
+      throws IOException {
+    Configuration conf = env.getConfiguration();
+    HConnection connection = null;
+    // this bit is a little hacky - we need to reach kind far into the internals. However, since we
+    // are in a coprocessor (which is part of the internals), this is more ok.
+    if (env instanceof RegionCoprocessorEnvironment) {
+      RegionCoprocessorEnvironment e = (RegionCoprocessorEnvironment) env;
+      RegionServerServices services = e.getRegionServerServices();
+      if (services instanceof HRegionServer) {
+        connection = new CoprocessorHConnection(conf, (HRegionServer) services);
+      }
+    }
+    // didn't create the custom HConnection, so just create the usual connection. Saves us some conf
+    // lookups, but no network accesses or anything else with excessive overhead.
+    if (connection == null) {
+      connection = HConnectionManager.createConnection(conf);
+    }
+    return connection;
+  }
+
+  private ServerName serverName;
+  private HRegionServer server;
+
+  CoprocessorHConnection(Configuration conf, HRegionServer server) throws IOException {
+    super(conf, false, null);
+    this.server = server;
+    this.serverName = server.getServerName();
+  }
+
+  public HRegionInterface getHRegionConnection(String hostname, int port) throws IOException {
+    // if we aren't talking to the local HRegionServer, then do the usual thing
+    if (!this.serverName.getHostname().equals(hostname) || !(this.serverName.getPort() == port)) {
+      return super.getHRegionConnection(hostname, port);
+    }
+
+    // in the usual HConnectionImplementation we would check a cache for the server. However,
+    // because we can just return the actual server, we don't need to do anything special.
+    return this.server;
+  }
+
+  @Deprecated
+  public HRegionInterface getHRegionConnection(HServerAddress regionServer) throws IOException {
+    throw new UnsupportedOperationException(
+        "Coprocessors only find tables via #getHRegionConnection(String, int)");
+  }
+
+  @Deprecated
+  public HRegionInterface getHRegionConnection(HServerAddress regionServer, boolean getMaster)
+      throws IOException {
+    throw new UnsupportedOperationException(
+        "Coprocessors only find tables via #getHRegionConnection(String, int)");
+  }
+
+  @Deprecated
+  public HRegionInterface getHRegionConnection(String hostname, int port, boolean getMaster)
+      throws IOException {
+    throw new UnsupportedOperationException(
+        "Coprocessors only find tables via #getHRegionConnection(String, int)");
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index b5ee9ea62dbf..0ce498556dea 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -151,6 +151,31 @@ public HTable(Configuration conf, final byte [] tableName)
     this.connection = HConnectionManager.getConnection(conf);
     this.configuration = conf;
 
+    this.pool = getDefaultExecutor(conf);
+    this.finishSetup();
+  }
+
+  /**
+   * Creates an object to access a HBase table. Shares zookeeper connection and other resources with
+   * other HTable instances created with the same <code>connection</code> instance. Use this
+   * constructor when the HConnection instance is externally managed (does not close the connection
+   * on {@link #close()}).
+   * @param tableName Name of the table.
+   * @param connection @param connection HConnection to be used.
+   * @throws IOException if a remote or network exception occurs
+   */
+  public HTable(byte[] tableName, HConnection connection) throws IOException {
+    this.tableName = tableName;
+    this.cleanupPoolOnClose = true;
+    this.cleanupConnectionOnClose = false;
+    this.connection = connection;
+    this.configuration = connection.getConfiguration();
+
+    this.pool = getDefaultExecutor(this.configuration);
+    this.finishSetup();
+  }
+
+  private static ThreadPoolExecutor getDefaultExecutor(Configuration conf) {
     int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE);
     if (maxThreads == 0) {
       maxThreads = 1; // is there a better default?
@@ -161,13 +186,12 @@ public HTable(Configuration conf, final byte [] tableName)
     // if it is necessary and will grow unbounded. This could be bad but in HCM
     // we only create as many Runnables as there are region servers. It means
     // it also scales when new region servers are added.
-    this.pool = new ThreadPoolExecutor(1, maxThreads,
+    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads,
         keepAliveTime, TimeUnit.SECONDS,
         new SynchronousQueue<Runnable>(),
         Threads.newDaemonThreadFactory("hbase-table"));
-    ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
-
-    this.finishSetup();
+    pool.allowCoreThreadTimeOut(true);
+    return pool;
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 838d05682dbc..198b26e33489 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -42,8 +42,10 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.CoprocessorHConnection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Increment;
@@ -60,6 +62,7 @@
 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
 import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.io.MultipleIOException;
 
 /**
  * Provides the common setup framework and runtime services for coprocessor
@@ -358,15 +361,33 @@ class HTableWrapper implements HTableInterface {
 
       private byte[] tableName;
       private HTable table;
+      private HConnection connection;
 
-      public HTableWrapper(byte[] tableName) throws IOException {
+      public HTableWrapper(byte[] tableName, HConnection connection) throws IOException {
         this.tableName = tableName;
-        this.table = new HTable(conf, tableName);
+        this.table = new HTable(tableName, connection);
+        this.connection = connection;
         openTables.add(this);
       }
 
       void internalClose() throws IOException {
-        table.close();
+        List<IOException> exceptions = new ArrayList<IOException>(2);
+        try {
+          table.close();
+        } catch (IOException e) {
+          exceptions.add(e);
+        }
+        try {
+          // have to self-manage our connection, as per the HTable contract
+          if (this.connection != null) {
+            this.connection.close();
+          }
+        } catch (IOException e) {
+          exceptions.add(e);
+        }
+        if (!exceptions.isEmpty()) {
+          throw MultipleIOException.createIOException(exceptions);
+        }
       }
 
       public Configuration getConfiguration() {
@@ -659,7 +680,7 @@ public Configuration getConfiguration() {
      */
     @Override
     public HTableInterface getTable(byte[] tableName) throws IOException {
-      return new HTableWrapper(tableName);
+      return new HTableWrapper(tableName, CoprocessorHConnection.getConnectionForEnvironment(this));
     }
   }
 

From 632ec7ace7a9bf1ef225819ef3a15b53212c7694 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Sep 2013 19:16:00 +0000
Subject: [PATCH 1160/1540] CHANGES.txt for 0.94.12RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524534 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index fa8c85a9319b..e9ea5b250fc2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -35,6 +35,11 @@ Bug
     [HBASE-9432] - Backport HBASE-8781 to 0.94
     [HBASE-9448] - [0.94] Shell needs to fall back after HBASE-9182 if talking to older servers
     [HBASE-9455] - Port HBASE-7113 'TestGzipFilter is flaky with jdk1.7' to 0.94
+    [HBASE-9468] - Previous active master can still serves RPC request when it is trying recovering expired zk session
+    [HBASE-9482] - Do not enforce secure Hadoop for secure HBase
+    [HBASE-9506] - [0.94] Backport HBASE-9309 The links in the backup masters template are bad
+    [HBASE-9534] - Short-Circuit Coprocessor HTable access when on the same server
+    [HBASE-9566] - Add back WALEdit#get/setScopes method
 
 Improvement
 

From 13690f4cb942b9e9167e702b9aa399b57b00a874 Mon Sep 17 00:00:00 2001
From: ndimiduk <ndimiduk@unknown>
Date: Wed, 18 Sep 2013 23:07:18 +0000
Subject: [PATCH 1161/1540] HBASE-9153: Introduce/update a script to generate
 jdiff reports

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524599 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase_jdiff_acrossSingularityTemplate.xml |  54 ++++
 .../hbase_jdiff_afterSingularityTemplate.xml  |  61 +++++
 dev-support/hbase_jdiff_template.xml          |  48 ++++
 dev-support/jdiffHBasePublicAPI.sh            | 232 ++++++++++++++++++
 dev-support/jdiffHBasePublicAPI_common.sh     |  71 ++++++
 5 files changed, 466 insertions(+)
 create mode 100644 dev-support/hbase_jdiff_acrossSingularityTemplate.xml
 create mode 100644 dev-support/hbase_jdiff_afterSingularityTemplate.xml
 create mode 100644 dev-support/hbase_jdiff_template.xml
 create mode 100644 dev-support/jdiffHBasePublicAPI.sh
 create mode 100644 dev-support/jdiffHBasePublicAPI_common.sh

diff --git a/dev-support/hbase_jdiff_acrossSingularityTemplate.xml b/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
new file mode 100644
index 000000000000..4c70eae91a4f
--- /dev/null
+++ b/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
+
+<description>
+Creates a report in the directory "hbase_jdiff_report-p-PREVIOUS_BRANCH-c-CURRENT_BRANCH" of the default jdiff report folder.
+This defaults to /tmp/jdiff but can optionally be specified by export $JDIFF_WORKING_DIRECTORY.
+</description>
+
+<target name="jdiff_report" depends="">
+  <!-- This variable is used by the Ant JDiff task. Set it to the
+       location where the jdiff package was unpacked -->
+  <property name="JDIFF_HOME" value="JDIFF_HOME_NAME" />
+  <property name="OLD_BRANCH" value="OLD_BRANCH_NAME"/>
+  <property name="NEW_BRANCH" value="NEW_BRANCH_NAME"/>
+
+  <taskdef name="jdiff"
+           classname="jdiff.JDiffAntTask"
+           classpath="${JDIFF_HOME}/antjdiff.jar" />
+  <jdiff destdir="hbase_jdiff_report"
+         verbose="off"
+         stats="on">
+    <old name="V1">
+      <!-- This is the location of your old source files -->
+      <dirset dir="${OLD_BRANCH}/src/main/java" includes="**" />
+    </old>
+    <new name="V2">
+      <!-- This is the location of your new source files -->
+      <dirset dir="${NEW_BRANCH}/hbase-server/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-client/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-common/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-protocol/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-it/src/test/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-hadoop-compat/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-hadoop2-compat/src/main/java" includes="**" />
+    </new>
+  </jdiff>
+</target>
+
+</project>
+
diff --git a/dev-support/hbase_jdiff_afterSingularityTemplate.xml b/dev-support/hbase_jdiff_afterSingularityTemplate.xml
new file mode 100644
index 000000000000..81adad017621
--- /dev/null
+++ b/dev-support/hbase_jdiff_afterSingularityTemplate.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
+
+<description>
+Creates a report in the directory "hbase_jdiff_report-p-PREVIOUS_BRANCH-c-CURRENT_BRANCH" of the default jdiff report folder.
+This defaults to /tmp/jdiff but can optionally be specified by export $JDIFF_WORKING_DIRECTORY.
+</description>
+
+<target name="jdiff_report" depends="">
+  <!-- This variable is used by the Ant JDiff task. Set it to the
+       location where the jdiff package was unpacked -->
+  <property name="JDIFF_HOME" value="JDIFF_HOME_NAME" />
+  <property name="OLD_BRANCH" value="OLD_BRANCH_NAME"/>
+  <property name="NEW_BRANCH" value="NEW_BRANCH_NAME"/>
+
+  <taskdef name="jdiff"
+           classname="jdiff.JDiffAntTask"
+           classpath="${JDIFF_HOME}/antjdiff.jar" />
+  <jdiff destdir="hbase_jdiff_report"
+         verbose="off"
+         stats="on">
+    <old name="V1">
+      <!-- This is the location of your old source files -->
+      <dirset dir="${OLD_BRANCH}/hbase-server/src/main/java" includes="**" />
+      <dirset dir="${OLD_BRANCH}/hbase-client/src/main/java" includes="**" />
+      <dirset dir="${OLD_BRANCH}/hbase-common/src/main/java" includes="**" />
+      <dirset dir="${OLD_BRANCH}/hbase-protocol/src/main/java" includes="**" />
+      <dirset dir="${OLD_BRANCH}/hbase-it/src/test/java" includes="**" />
+      <dirset dir="${OLD_BRANCH}/hbase-hadoop-compat/src/main/java" includes="**" />
+      <dirset dir="${OLD_BRANCH}/hbase-hadoop2-compat/src/main/java" includes="**" />
+
+    </old>
+    <new name="V2">
+      <!-- This is the location of your new source files -->
+      <dirset dir="${NEW_BRANCH}/hbase-server/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-client/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-common/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-protocol/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-it/src/test/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-hadoop-compat/src/main/java" includes="**" />
+      <dirset dir="${NEW_BRANCH}/hbase-hadoop2-compat/src/main/java" includes="**" />
+    </new>
+  </jdiff>
+</target>
+
+</project>
+
diff --git a/dev-support/hbase_jdiff_template.xml b/dev-support/hbase_jdiff_template.xml
new file mode 100644
index 000000000000..48b1ab50a687
--- /dev/null
+++ b/dev-support/hbase_jdiff_template.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
+
+<description>
+Creates a report in the directory "hbase_jdiff_report-p-PREVIOUS_BRANCH-c-CURRENT_BRANCH" of the default jdiff report folder.
+This defaults to /tmp/jdiff but can optionally be specified by export $JDIFF_WORKING_DIRECTORY.
+</description>
+
+<target name="jdiff_report" depends="">
+  <!-- This variable is used by the Ant JDiff task. Set it to the
+       location where the jdiff package was unpacked -->
+  <property name="JDIFF_HOME" value="JDIFF_HOME_NAME" />
+  <property name="OLD_BRANCH" value="OLD_BRANCH_NAME"/>
+  <property name="NEW_BRANCH" value="NEW_BRANCH_NAME"/>
+
+  <taskdef name="jdiff"
+           classname="jdiff.JDiffAntTask"
+           classpath="${JDIFF_HOME}/antjdiff.jar" />
+  <jdiff destdir="hbase_jdiff_report"
+         verbose="off"
+         stats="on">
+    <old name="V1">
+      <!-- This is the location of your old source files -->
+      <dirset dir="${OLD_BRANCH}/src/main/java" includes="**" />
+    </old>
+    <new name="V2">
+      <!-- This is the location of your new source files -->
+      <dirset dir="${NEW_BRANCH}/src/main/java" includes="**" />
+    </new>
+  </jdiff>
+</target>
+
+</project>
+
diff --git a/dev-support/jdiffHBasePublicAPI.sh b/dev-support/jdiffHBasePublicAPI.sh
new file mode 100644
index 000000000000..15d116d8d4d2
--- /dev/null
+++ b/dev-support/jdiffHBasePublicAPI.sh
@@ -0,0 +1,232 @@
+#!/bin/bash
+set -e
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+################################################ ABOUT JDIFF #######################################################
+#
+#   What is JDiff? JDiff is a tool for comparing the public APIs of two separate Java codebases. Like diff, it will
+#   give additions, changes, and removals. It will output an HTML report with the information.
+#   To learn more, visit http://javadiff.sourceforge.net/.
+#   JDiff is licensed under LGPL.
+
+############################################# QUICK-START EXAMPLE ##################################################
+#
+#   Suppose we wanted to see the API diffs between HBase 0.92 and HBase 0.94. We could use this tool like so:
+#   > ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.92 https://github.com/apache/hbase.git 0.94
+#
+#   This would generate a report in the local folder /tmp/jdiff/hbase_jdiff_report-p-0.92-c-0.94/
+#   To view the report, simply examine /tmp/jdiff/hbase_jdiff_report-p-0.92-c-0.94/changes.html in your choice of
+#   browser.
+#
+#   Note that this works because 0.92 and 0.94 have the source directory structure that is specified in the
+#   hbase_jdiff_template.xml file. To compare 0.95 to 0.96, which have the post-singularity structure,  two other
+#   template files (included) are used. The formats are autoated and is all taken care of automatically by the script.
+#
+#   On a local machine, JDiff reports have taken ~20-30 minutes to run. On Jenkins, it has taken over 35 minutes
+#   in some cases. Your mileage may vary. Trunk and 0.95 take more time than 0.92 and 0.94.
+#
+#
+############################################ SPECIFYING A LOCAL REPO ###############################################
+#
+#   The JDiff tool also works with local code. Instead of specifying a repo and a branch, you can specifying the
+#   absolute path of the ./hbase folder and a name for code (e.g. experimental_94).
+#
+#   A local repo can be specified for none, one, or both of the sources.
+#
+############################################### EXAMPLE USE CASES ##################################################
+#
+#   Example 1: Generate a report to check if potential change doesn't break API compatibility with Apache HBase 0.94
+#
+#   In this case, you could compare the version you are using against a repo branch where your changes are.
+#   >  ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.94 https://github.com/MY_REPO/hbase.git 0.94
+#
+#   Example 2: Generate a report to check if two branches of the same repo have any public API incompatibilities
+#   >  ./jdiffHBasePublicAPI.sh https://github.com/MY_REPO/hbase.git $BRANCH_1  \
+#   >   https://github.com/MY_REPO/hbase.git $BRANCH_2
+#
+#   Example 3: Have Example 1 done in a special directory in the user's home folder
+#
+#   >  export JDIFF_WORKING_DIRECTORY=~/jdiff_reports
+#   >  ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.94 https://github.com/MY_REPO/hbase.git 0.94
+#
+#   Example 4: Check the API diff of a local change against an existing repo branch.
+#   >  ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.95 /home/aleks/exp_hbase/hbase experiment_95
+#
+#   Example 5: Compare two local repos for public API changes
+#   >  ./jdiffHBasePublicAPI.sh /home/aleks/stable_hbase/hbase stable_95 /home/aleks/exp_hbase/hbase experiment_95
+#
+#
+############################################# READING A JDIFF REPORT ###############################################
+#
+#   The purpose of the JDiff report is show things that have changed between two versions of the public API. A user
+#   would use this report to determine if committing a change would cause existing API clients to break. To do so,
+#   there are specific things that one should look for in the report.
+#
+#   1. Identify the classes that constitute the public API. An example in 0.94 might be all classes in
+#      org.apache.hadoop.hbase.client.*
+#   2. After identifying those classes, go through each one and look for offending changes.
+#      Those may include, but are not limited to:
+#      1. Removed methods
+#      2. Changed methods (including changes in return type and exception types)
+#      3. Methods added to interfaces
+#      4. Changed class inheritence information (may in innocuous but definitely worth validating)
+#      5. Removed or renamed public static member variables and constants
+#      6. Removed or renamed packages
+#      7. Class moved to a different package
+
+########################################### SETTING THE JDIFF WORKING DIRECTORY ####################################
+#
+#   By default, the working environment of jdiff is /tmp/jdiff. However, sometimes it is nice to have it place reports
+#   and temp files elsewhere. In that case, please export JDIFF_WORKING_DIRECTORY into the bash environment and this
+#   script will pick that up and use it.
+#
+
+scriptDirectory=$(dirname ${BASH_SOURCE[0]})
+x=`echo $scriptDirectory | sed "s{\.{{g"`
+DEV_SUPPORT_HOME="`pwd`$x"
+. $scriptDirectory/jdiffHBasePublicAPI_common.sh
+
+EXPECTED_ARGS=4
+
+if [[ "$#" -ne "$EXPECTED_ARGS" ]]; then
+  echo "This tool expects $EXPECTED_ARGS arguments, but received $#. Please check your command and try again.";
+  echo "Usage: $0 <repoUrl or local source directory 1> <branch or source name 1> <repoUrl or local source directory 2> <branch or source name 2>"
+  exit 1;
+fi
+
+echo "JDiff evaluation beginning:";
+isGitRepo $1
+FIRST_SOURCE_TYPE=$INPUT_FORMAT;
+isGitRepo $3
+SECOND_SOURCE_TYPE=$INPUT_FORMAT;
+
+PREVIOUS_BRANCH=$2  ## We will still call it a branch even if it's not from a git repo.
+CURRENT_BRANCH=$4
+
+echo "We are going to compare source 1 which is a $FIRST_SOURCE_TYPE and source 2, which is a $SECOND_SOURCE_TYPE"
+
+
+# Check that if either source is from a git repo, that the name is reasonable.
+if [[ "$FIRST_SOURCE_TYPE" = "git_repo" ]]; then
+
+  git check-ref-format --branch $2
+fi
+
+if [[ "$SECOND_SOURCE_TYPE" = "git_repo" ]]; then
+
+  git check-ref-format --branch $4
+fi
+
+#If the JDIFF_WORKING_DIRECTORY is set, then we will output the report there. Otherwise, to the default location
+if [[ "$JDIFF_WORKING_DIRECTORY" = "" ]]; then
+
+  echo "JDIFF_WORKING_DIRECTORY not set. That's not an issue. We will default it to ./jidff"
+  JDIFF_WORKING_DIRECTORY=/tmp/jdiff
+else
+  echo "JDIFF_WORKING_DIRECTORY set to $JDIFF_WORKING_DIRECTORY";
+fi
+mkdir -p $JDIFF_WORKING_DIRECTORY
+
+# We will need this to reference the template we want to use
+cd $JDIFF_WORKING_DIRECTORY
+scenario_template_name=hbase_jdiff_p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH.xml
+
+
+# Pull down JDiff tool and unpack it
+if [ ! -d jdiff-1.1.1-with-incompatible-option ]; then
+  curl -O http://cloud.github.com/downloads/tomwhite/jdiff/jdiff-1.1.1-with-incompatible-option.zip
+  unzip jdiff-1.1.1-with-incompatible-option.zip
+fi
+
+JDIFF_HOME=`pwd`/jdiff-1.1.1-with-incompatible-option
+cd $JDIFF_WORKING_DIRECTORY
+
+# Pull down sources if necessary. Note that references to previous change are prefaced with p- in order to avoid collission of branch names
+if [[ "$FIRST_SOURCE_TYPE" = "git_repo" ]]; then
+
+  PREVIOUS_REPO=$1
+  rm -rf p-$PREVIOUS_BRANCH
+  mkdir -p p-$PREVIOUS_BRANCH
+  cd p-$PREVIOUS_BRANCH
+  git clone --depth 1 $PREVIOUS_REPO && cd hbase && git checkout origin/$PREVIOUS_BRANCH
+  cd $JDIFF_WORKING_DIRECTORY
+  HBASE_1_HOME=`pwd`/p-$PREVIOUS_BRANCH/hbase
+else
+  HBASE_1_HOME=$1
+fi
+
+echo "HBASE_1_HOME set to $HBASE_1_HOME"
+echo "In HBASE_1_HOME, we have"
+ls -la $HBASE_1_HOME
+
+if [[ "$SECOND_SOURCE_TYPE" = "git_repo" ]]; then
+  CURRENT_REPO=$3
+  rm -rf $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
+  mkdir -p $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
+  cd $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
+  git clone --depth 1 $CURRENT_REPO && cd hbase && git checkout origin/$CURRENT_BRANCH
+  cd $JDIFF_WORKING_DIRECTORY
+  HBASE_2_HOME=`pwd`/c-$CURRENT_BRANCH/hbase
+else
+  HBASE_2_HOME=$3
+fi
+
+echo "HBASE_2_HOME set to $HBASE_2_HOME"
+echo "In HBASE_2_HOME, we have"
+ls -la $HBASE_2_HOME
+
+# Next step is to pull down the proper template based on the directory structure
+isNewFormat $HBASE_1_HOME
+export P_FORMAT=$BRANCH_FORMAT
+
+isNewFormat $HBASE_2_HOME
+export C_FORMAT=$BRANCH_FORMAT
+
+if  [[ "$C_FORMAT" = "new" ]]; then
+
+  if [[ "$P_FORMAT" = "new" ]]; then
+    templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_afterSingularityTemplate.xml
+    echo "Previous format is of the new style. We'll be using template $templateFile";
+  else
+    templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_acrossSingularityTemplate.xml
+    echo "Previous format is of the old style. We'll be using template $templateFile";
+  fi
+
+else
+  templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_template.xml
+  echo "Both formats are using the 94 and earlier style directory format. We'll be using template $templateFile"
+fi
+
+cp $templateFile $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+
+### Configure the jdiff script
+
+### Note that PREVIOUS_BRANCH and CURRENT_BRANCH will be the absolute locations of the source.
+echo "Configuring the jdiff script"
+sed -i "s]hbase_jdiff_report]hbase_jdiff_report-p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+sed -i "s]JDIFF_HOME_NAME]$JDIFF_HOME]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+sed -i "s]OLD_BRANCH_NAME]$HBASE_1_HOME]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+sed -i "s]NEW_BRANCH_NAME]$HBASE_2_HOME]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+
+sed -i "s]V1]$PREVIOUS_BRANCH]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+sed -i "s]V2]$CURRENT_BRANCH]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+
+sed -i "s]JDIFF_FOLDER]$JDIFF_WORKING_DIRECTORY]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
+
+echo "Running jdiff";
+ls -la $JDIFF_WORKING_DIRECTORY;
+ant -f $JDIFF_WORKING_DIRECTORY/$scenario_template_name;
+
+echo "jdiff operation complete. Report placed into $JDIFF_WORKING_DIRECTORY/hbase_jdiff_report-p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH/changes.html";
+
diff --git a/dev-support/jdiffHBasePublicAPI_common.sh b/dev-support/jdiffHBasePublicAPI_common.sh
new file mode 100644
index 000000000000..8c7b53ff9fa7
--- /dev/null
+++ b/dev-support/jdiffHBasePublicAPI_common.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+##########################################################################################################################
+#
+### Purpose: To describe whether the directory specified has the old directory format or the new directory format
+### Usage: This function takes one argument: The directory in question
+### It will set the temporary variable BRANCH_FORMAT. This variable can change with every call, so it is up to the user to
+### store it into something else as soon as the function exists
+### Example:
+### > isNewFormat ./myDevDir/testing/branch/hbase
+isNewFormat() {
+
+  echo "Determining if directory $1 is of the 0.94 and before OR 0.95 and after versions";
+  if [[ "$1" = "" ]]; then
+    echo "Directory not specified. Exiting";
+  fi
+  echo "First, check that $1 exists";
+  if [[ -d $1 ]]; then
+    echo "Directory $1 exists"
+  else
+    echo "Directory $1 does not exist. Exiting";
+    exit 1;
+  fi
+
+  if [[ -d "$1/hbase-server" ]]; then
+
+    echo "The directory $1/hbase-server exists so this is of the new format";
+    export BRANCH_FORMAT=new;
+
+  else
+    echo "The directory $1/hbase-server does not exist. Therefore, this is of the old format";
+    export BRANCH_FORMAT=old;
+  fi
+}
+
+### Purpose: To describe whether the argument specified is a git repo or a local directory
+### Usage: This function takes one argument: The directory in question
+### It will set the temporary variable INPUT_FORMAT. This variable can change with every call, so it is up to the user to
+### store it into something else as soon as the function exists
+### Example:
+### > isGitRepo ./myDevDir/testing/branch/hbase
+
+isGitRepo() {
+
+  echo "Determining if this is a local directory or a git repo.";
+  if [[ "$1" = "" ]]; then
+    echo "No value specified for repo or directory. Exiting."
+    exit 1;
+  fi
+
+  if [[ `echo $1 | grep 'http://'` || `echo $1 | grep 'https://'` || `echo $1 | grep 'git://'`  ]]; then
+    echo "Looks like $1 is a git repo";
+    export INPUT_FORMAT=git_repo
+  else
+    echo "$1 is a local directory";
+    export INPUT_FORMAT=local_directory
+  fi
+
+
+}

From 0c946042d7b52b365950fb5fa868508fe44fd67d Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 19 Sep 2013 20:32:39 +0000
Subject: [PATCH 1162/1540] HBASE-9584: Short-Circuit Coprocessor doesn't
 correctly lookup table when on server

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524827 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/CoprocessorHConnection.java  |  60 ++++-----
 .../client/TestCoprocessorHConnection.java    | 115 ++++++++++++++++++
 2 files changed, 138 insertions(+), 37 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorHConnection.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java b/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
index f5d684b5858a..a56036653b35 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
@@ -18,29 +18,16 @@
 package org.apache.hadoop.hbase.client;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
+import java.net.InetSocketAddress;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
 /**
  * Connection to an HTable from within a Coprocessor. Can do some nice tricks since we know we are
@@ -91,34 +78,33 @@ public static HConnection getConnectionForEnvironment(CoprocessorEnvironment env
     this.serverName = server.getServerName();
   }
 
-  public HRegionInterface getHRegionConnection(String hostname, int port) throws IOException {
+  @Override
+  HRegionInterface getHRegionConnection(final String hostname, final int port,
+      final InetSocketAddress isa, final boolean master) throws IOException {
+    // check to see where the server is running
+    // need this isa stuff here since its what the HConnectionManager is doing too
+    boolean isRemote = false;
+    if (isa != null) {
+      isRemote = checkRemote(isa.getHostName(), isa.getPort());
+    } else {
+      isRemote = checkRemote(hostname, port);
+    }
     // if we aren't talking to the local HRegionServer, then do the usual thing
-    if (!this.serverName.getHostname().equals(hostname) || !(this.serverName.getPort() == port)) {
-      return super.getHRegionConnection(hostname, port);
+    if (isRemote) {
+      return super.getHRegionConnection(hostname, port, isa, master);
     }
 
-    // in the usual HConnectionImplementation we would check a cache for the server. However,
-    // because we can just return the actual server, we don't need to do anything special.
+    // local access, so just pass the actual server, rather than a proxy
     return this.server;
   }
 
-  @Deprecated
-  public HRegionInterface getHRegionConnection(HServerAddress regionServer) throws IOException {
-    throw new UnsupportedOperationException(
-        "Coprocessors only find tables via #getHRegionConnection(String, int)");
-  }
-
-  @Deprecated
-  public HRegionInterface getHRegionConnection(HServerAddress regionServer, boolean getMaster)
-      throws IOException {
-    throw new UnsupportedOperationException(
-        "Coprocessors only find tables via #getHRegionConnection(String, int)");
-  }
-
-  @Deprecated
-  public HRegionInterface getHRegionConnection(String hostname, int port, boolean getMaster)
-      throws IOException {
-    throw new UnsupportedOperationException(
-        "Coprocessors only find tables via #getHRegionConnection(String, int)");
+  /**
+   * Check that the hostname and port map the the server on which we are currently running
+   * @param hostName hostname to check
+   * @param port port to check
+   * @return <tt>true</tt> the connection is <b>not</b> currently running on the given host and port
+   */
+  private boolean checkRemote(String hostName, int port) {
+    return !(this.serverName.getHostname().equals(hostName) && this.serverName.getPort() == port);
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorHConnection.java b/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorHConnection.java
new file mode 100644
index 000000000000..535c5b268bf2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorHConnection.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+@Category(MediumTests.class)
+public class TestCoprocessorHConnection {
+
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void shutdownCluster() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Ensure that if the HRegion we are looking up isn't on this server (and not in the cache), that
+   * we still correctly look it up.
+   * @throws Exception on failure
+   */
+  @Test
+  public void testNonServerLocalLookup() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+    // make a fake server that should never be called
+    HRegionServer server = Mockito.mock(HRegionServer.class);
+    ServerName name = new ServerName("not.a.server.hostname", 12345, -1L);
+    Mockito.when(server.getServerName()).thenReturn(name);
+    CoprocessorHConnection connection = new CoprocessorHConnection(conf, server);
+
+    // make sure we get the mock server when doing a direct lookup
+    assertEquals("Didn't get the mock server from the connection", server,
+      connection.getHRegionConnection(name.getHostname(), name.getPort()));
+
+    // create a table that exists
+    byte[] tableName = Bytes.toBytes("testNonServerLocalLookup");
+    byte[] family = Bytes.toBytes("family");
+    UTIL.createTable(tableName, family);
+
+    // if we can write to the table correctly, then our connection is doing the right thing
+    HTable table = new HTable(tableName, connection);
+    Put p = new Put(Bytes.toBytes("row"));
+    p.add(family, null, null);
+    table.put(p);
+    table.flushCommits();
+
+
+    // cleanup
+    table.close();
+    connection.close();
+  }
+
+  @Test
+  public void testLocalServerLookup() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+    // get a real rs
+    HRegionServer server =
+        UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0).getRegionServer();
+    // fake the connection to look like we are living on that server
+    CoprocessorHConnection connection = new CoprocessorHConnection(conf, server);
+
+    // create a table that exists
+    byte[] tableName = Bytes.toBytes("testLocalServerLookup");
+    byte[] family = Bytes.toBytes("family");
+    UTIL.createTable(tableName, family);
+
+    // if we can write to the table correctly, then our connection is doing the right thing
+    HTable table = new HTable(tableName, connection);
+    Put p = new Put(Bytes.toBytes("row"));
+    p.add(family, null, null);
+    table.put(p);
+    table.flushCommits();
+
+    //make sure we get the actual server when doing a direct lookup
+    ServerName name = server.getServerName();
+    assertEquals("Didn't get the expected server from the connection", server,
+      connection.getHRegionConnection(name.getHostname(), name.getPort()));
+
+    // cleanup
+    table.close();
+    connection.close();
+  }
+}
\ No newline at end of file

From ca3d4f5868fd09dc14b419c83e1b21ecc4c3a931 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 19 Sep 2013 21:47:44 +0000
Subject: [PATCH 1163/1540] CHANGES.txt for 0.94.12RC2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524850 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index e9ea5b250fc2..d6dac9ff638f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.12 - 9/10/2013
+Release 0.94.12 - 9/19/2013
 Sub-task
 
     [HBASE-9277] - REST should use listTableNames to list tables
@@ -40,6 +40,7 @@ Bug
     [HBASE-9506] - [0.94] Backport HBASE-9309 The links in the backup masters template are bad
     [HBASE-9534] - Short-Circuit Coprocessor HTable access when on the same server
     [HBASE-9566] - Add back WALEdit#get/setScopes method
+    [HBASE-9584] - Short-Circuit Coprocessor doesn't correctly lookup table when on server
 
 Improvement
 

From 86fb1203df38d6dc593213a20212a7e6ddbefbc0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 19 Sep 2013 23:30:01 +0000
Subject: [PATCH 1164/1540] CHANGES.txt for 0.94.12RC2 take2

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1524861 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index d6dac9ff638f..d6185dcbccee 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -49,6 +49,7 @@ Improvement
 
 Task
 
+    [HBASE-9153] - Introduce/update a script to generate jdiff reports
     [HBASE-9377] - Backport HBASE- 9208 "ReplicationLogCleaner slow at large scale"
 
 Test

From c903aa045ededae2ef56b3707d50d47e2f48d373 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sun, 22 Sep 2013 12:22:08 +0000
Subject: [PATCH 1165/1540] HBASE-9607 Data loss after snapshot restore into
 cloned table

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1525351 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/snapshot/RestoreSnapshotHelper.java   | 14 +++++++-------
 .../client/TestRestoreSnapshotFromClient.java   | 17 ++++++++++++++++-
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index a5dd2e50b689..a43c4b808d1a 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -385,13 +385,6 @@ private void restoreRegion(HRegionInfo regionInfo) throws IOException {
           }
         }
 
-        // Restore Missing files
-        for (String hfileName: hfilesToAdd) {
-          LOG.trace("Adding HFileLink " + hfileName +
-            " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
-          restoreStoreFile(familyDir, regionInfo, hfileName);
-        }
-
         // Remove hfiles not present in the snapshot
         for (String hfileName: familyFiles) {
           Path hfile = new Path(familyDir, hfileName);
@@ -399,6 +392,13 @@ private void restoreRegion(HRegionInfo regionInfo) throws IOException {
             " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
           HFileArchiver.archiveStoreFile(fs, regionInfo, conf, tableDir, family, hfile);
         }
+
+        // Restore Missing files
+        for (String hfileName: hfilesToAdd) {
+          LOG.trace("Adding HFileLink " + hfileName +
+            " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
+          restoreStoreFile(familyDir, regionInfo, hfileName);
+        }
       } else {
         // Family doesn't exists in the snapshot
         LOG.trace("Removing family=" + Bytes.toString(family) +
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 2fec47ce6b3c..324a176e02c1 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -220,7 +220,7 @@ public void testRestoreSchemaChange() throws Exception {
   }
 
   @Test
-  public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
+  public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
     admin.cloneSnapshot(snapshotName0, clonedTableName);
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
@@ -234,6 +234,21 @@ public void testRestoreSnapshotOfCloned() throws IOException, InterruptedExcepti
     TEST_UTIL.deleteTable(clonedTableName);
   }
 
+  @Test
+  public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
+    TEST_UTIL.deleteTable(tableName);
+    waitCleanerRun();
+
+    admin.cloneSnapshot(snapshotName0, tableName);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
+    waitCleanerRun();
+
+    admin.disableTable(tableName);
+    admin.restoreSnapshot(snapshotName0);
+    admin.enableTable(tableName);
+    SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
+  }
+
   // ==========================================================================
   //  Helpers
   // ==========================================================================

From b99f8febab19b6672c92e450b6e575567442f261 Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Mon, 23 Sep 2013 12:56:28 +0000
Subject: [PATCH 1166/1540] HBASE-9430 Memstore heapSize calculation -
 DEEP_OVERHEAD is incorrect

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1525572 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/KeyValueSkipListSet.java     |  2 +-
 .../hadoop/hbase/regionserver/MemStore.java   |  4 +--
 .../apache/hadoop/hbase/util/ClassSize.java   | 10 ++++++
 .../apache/hadoop/hbase/io/TestHeapSize.java  | 32 +++++++++++++++----
 4 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
index 51df1ee2a034..20feacf8d6fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
@@ -43,7 +43,7 @@
  * has same attributes as ConcurrentSkipListSet: e.g. tolerant of concurrent
  * get and set and won't throw ConcurrentModificationException when iterating.
  */
-class KeyValueSkipListSet implements NavigableSet<KeyValue> {
+public class KeyValueSkipListSet implements NavigableSet<KeyValue> {
   private final ConcurrentNavigableMap<KeyValue, KeyValue> delegatee;
 
   KeyValueSkipListSet(final KeyValue.KVComparator c) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 113af807dd8d..22f933543fe2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -931,8 +931,8 @@ public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns,
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
-      ClassSize.COPYONWRITE_ARRAYSET + ClassSize.COPYONWRITE_ARRAYLIST +
-      (2 * ClassSize.CONCURRENT_SKIPLISTMAP));
+      (2 * ClassSize.TIMERANGE_TRACKER) +
+      (2 * ClassSize.KEYVALUE_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP));
 
   /** Used for readability when we don't store memstore timestamp in HFile */
   public static final boolean NO_PERSISTENT_TS = false;
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
index 61c9d6728165..95313ad6ecb8 100755
--- a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
@@ -99,6 +99,12 @@ public class ClassSize {
   /** Overhead for CopyOnWriteArrayList */
   public static final int COPYONWRITE_ARRAYLIST;
 
+  /** Overhead for TimeRangeTracker */
+  public static final int TIMERANGE_TRACKER;
+
+  /** Overhead for KeyValueSkipListSet */
+  public static final int KEYVALUE_SKIPLIST_SET;
+
   /* Are we running on jdk7? */
   private static final boolean JDK7;
   static {
@@ -175,6 +181,10 @@ public class ClassSize {
     COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE);
 
     COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY);
+
+    TIMERANGE_TRACKER = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2);
+
+    KEYVALUE_SKIPLIST_SET = align(OBJECT + REFERENCE);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
index a3c9ae9f5cde..dbd4ab7dc555 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
@@ -44,8 +44,10 @@
 import org.apache.hadoop.hbase.io.hfile.CachedBlock;
 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet;
 import org.apache.hadoop.hbase.regionserver.MemStore;
 import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -207,7 +209,23 @@ public void testNativeSizes() throws IOException {
       assertEquals(expected, actual);
     }
 
+    // TimeRangeTracker
+    cl = TimeRangeTracker.class;
+    expected = ClassSize.estimateBase(cl, false);
+    actual = ClassSize.TIMERANGE_TRACKER;
+    if (expected != actual) {
+      ClassSize.estimateBase(cl, true);
+      assertEquals(expected, actual);
+    }
 
+    // KeyValueSkipListSet
+    cl = KeyValueSkipListSet.class;
+    expected = ClassSize.estimateBase(cl, false);
+    actual = ClassSize.KEYVALUE_SKIPLIST_SET;
+    if (expected != actual) {
+      ClassSize.estimateBase(cl, true);
+      assertEquals(expected, actual);
+    }
   }
 
   /**
@@ -282,17 +300,19 @@ public void testSizes() throws IOException {
     expected = ClassSize.estimateBase(cl, false);
     expected += ClassSize.estimateBase(ReentrantReadWriteLock.class, false);
     expected += ClassSize.estimateBase(AtomicLong.class, false);
-    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
-    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
-    expected += ClassSize.estimateBase(CopyOnWriteArraySet.class, false);
-    expected += ClassSize.estimateBase(CopyOnWriteArrayList.class, false);
+    expected += (2 * ClassSize.estimateBase(KeyValueSkipListSet.class, false));
+    expected += (2 * ClassSize.estimateBase(ConcurrentSkipListMap.class, false));
+    expected += (2 * ClassSize.estimateBase(TimeRangeTracker.class, false));
     if(expected != actual) {
       ClassSize.estimateBase(cl, true);
       ClassSize.estimateBase(ReentrantReadWriteLock.class, true);
       ClassSize.estimateBase(AtomicLong.class, true);
+      ClassSize.estimateBase(KeyValueSkipListSet.class, true);
+      ClassSize.estimateBase(KeyValueSkipListSet.class, true);
+      ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
       ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
-      ClassSize.estimateBase(CopyOnWriteArraySet.class, true);
-      ClassSize.estimateBase(CopyOnWriteArrayList.class, true);
+      ClassSize.estimateBase(TimeRangeTracker.class, true);
+      ClassSize.estimateBase(TimeRangeTracker.class, true);
       assertEquals(expected, actual);
     }
 

From 77468488ab2a9e68ee48c5d225f4e4a19f3e9987 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 25 Sep 2013 14:43:47 +0000
Subject: [PATCH 1167/1540] HBASE-9651 Backport HBASE-3890 'Scheduled tasks in
 distributed log splitting not in sync with ZK' to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1526194 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/SplitLogManager.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 3c9fcc5b75a1..161a7310389c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -907,7 +907,7 @@ static class Task {
     volatile long last_update;
     volatile int last_version;
     volatile String cur_worker_name;
-    TaskBatch batch;
+    volatile TaskBatch batch;
     volatile TerminationStatus status;
     volatile int incarnation;
     volatile int unforcedResubmits;
@@ -1196,7 +1196,7 @@ public void processResult(int rc, String path, Object ctx) {
           }
           return;
         } else {
-        LOG.debug(path +
+        LOG.info(path +
             " does not exist. Either was created but deleted behind our" +
             " back by another pending delete OR was deleted" +
             " in earlier retry rounds. zkretries = " + (Long) ctx);

From d05feccbc1ae47a0c0a8424d0c81de1e4ce25303 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 26 Sep 2013 02:17:09 +0000
Subject: [PATCH 1168/1540] HBASE-9649 HFilePrettyPrinter should not throw a
 NPE if FirstKey or LastKey is null (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1526353 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/KeyValue.java         | 3 +++
 .../apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java    | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 39d1f095b7f1..915ad74d265c 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -713,6 +713,9 @@ public String toString() {
    * @return Key as a String.
    */
   public static String keyToString(final byte [] k) {
+    if (k == null) {
+      return "";
+    }
     return keyToString(k, 0, k.length);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index 1cf23df8ddda..fead56cb84c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -344,7 +344,11 @@ private void printMeta(HFile.Reader reader, Map<byte[], byte[]> fileInfo)
       }
     }
 
-    System.out.println("Mid-key: " + Bytes.toStringBinary(reader.midkey()));
+    try {
+      System.out.println("Mid-key: " + Bytes.toStringBinary(reader.midkey()));
+    } catch (Exception e) {
+      System.out.println("Unable to retrieve the midkey");
+    }
 
     // Printing general bloom information
     DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();

From 4a75e5d3f9056438844373c4fa50aea7210b78e7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 27 Sep 2013 05:50:52 +0000
Subject: [PATCH 1169/1540] roll release version to 0.94.13-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1526796 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 5f63b5b0dd05..ea8810954219 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.12</version>
+  <version>0.94.13-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From acd7b540047ed804a2225235d1672273cbc8ecb4 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 27 Sep 2013 18:27:30 +0000
Subject: [PATCH 1170/1540] HBASE-9548 Cleanup SnapshotTestingUtils

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1527021 13f79535-47bb-0310-9956-ffa450edef68
---
 .../cleaner/TestSnapshotFromMaster.java       |  20 +-
 .../hbase/snapshot/SnapshotTestingUtils.java  | 266 ++++++++++++------
 2 files changed, 195 insertions(+), 91 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 37e40cdc2f44..ffb59e569201 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -304,19 +304,19 @@ public void testSnapshotHFileArchiving() throws Exception {
 
     // get the snapshot files for the table
     Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-    FileStatus[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+    Path[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
     // check that the files in the archive contain the ones that we need for the snapshot
     LOG.debug("Have snapshot hfiles:");
-    for (FileStatus file : snapshotHFiles) {
-      LOG.debug(file.getPath());
+    for (Path file : snapshotHFiles) {
+      LOG.debug(file);
     }
     // get the archived files for the table
     Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
 
     // and make sure that there is a proper subset
-    for (FileStatus file : snapshotHFiles) {
-      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file.getPath(),
-        files.contains(file.getPath().getName()));
+    for (Path file : snapshotHFiles) {
+      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file,
+        files.contains(file.getName()));
     }
 
     // delete the existing snapshot
@@ -350,12 +350,12 @@ public void testSnapshotHFileArchiving() throws Exception {
   private final Collection<String> getArchivedHFiles(Path archiveDir, Path rootDir,
       FileSystem fs, String tableName) throws IOException {
     Path tableArchive = new Path(archiveDir, tableName);
-    FileStatus[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
+    Path[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
     List<String> files = new ArrayList<String>(archivedHFiles.length);
     LOG.debug("Have archived hfiles: " + tableArchive);
-    for (FileStatus file : archivedHFiles) {
-      LOG.debug(file.getPath());
-      files.add(file.getPath().getName());
+    for (Path file : archivedHFiles) {
+      LOG.debug(file);
+      files.add(file.getName());
     }
     // sort the archived files
 
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index cb9aa33c524c..1a2dc89a78c3 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.snapshot;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -26,6 +27,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
+import java.util.HashSet;
 import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
@@ -39,6 +41,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableNotEnabledException;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
@@ -48,12 +51,10 @@
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
-import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.Assert;
 
@@ -73,12 +74,34 @@ public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
   }
 
   /**
-   * Make sure that there is only one snapshot returned from the master and its name and table match
-   * the passed in parameters.
+   * Make sure that there is only one snapshot returned from the master and its
+   * name and table match the passed in parameters.
    */
-  public static void assertOneSnapshotThatMatches(HBaseAdmin admin, HSnapshotDescription snapshot)
+  public static List<SnapshotDescription> assertExistsMatchingSnapshot(
+      HBaseAdmin admin, String snapshotName, String tableName)
       throws IOException {
-    assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
+    // list the snapshot
+    List<SnapshotDescription> snapshots = admin.listSnapshots();
+
+    List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
+    for (SnapshotDescription sd : snapshots) {
+      if (snapshotName.equals(sd.getName()) &&
+          tableName.equals(sd.getTable())) {
+        returnedSnapshots.add(sd);
+      }
+    }
+
+    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
+    return returnedSnapshots;
+  }
+
+  /**
+   * Make sure that there is only one snapshot returned from the master
+   */
+  public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
+      HSnapshotDescription snapshot) throws IOException {
+    assertOneSnapshotThatMatches(admin, snapshot.getName(),
+        snapshot.getTable());
   }
 
   /**
@@ -91,11 +114,12 @@ public static void assertOneSnapshotThatMatches(HBaseAdmin admin, SnapshotDescri
   }
 
   /**
-   * Make sure that there is only one snapshot returned from the master and its name and table match
-   * the passed in parameters.
+   * Make sure that there is only one snapshot returned from the master and its
+   * name and table match the passed in parameters.
    */
-  public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
-      String snapshotName, String tableName) throws IOException {
+  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
+      HBaseAdmin admin, String snapshotName, String tableName)
+      throws IOException {
     // list the snapshot
     List<SnapshotDescription> snapshots = admin.listSnapshots();
 
@@ -107,47 +131,122 @@ public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin
   }
 
   /**
-   * Make sure that there is only one snapshot returned from the master and its name and table match
-   * the passed in parameters.
+   * Make sure that there is only one snapshot returned from the master and its
+   * name and table match the passed in parameters.
+   */
+  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
+      HBaseAdmin admin, byte[] snapshot, byte[] tableName) throws IOException {
+    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
+        Bytes.toString(tableName));
+  }
+
+  /**
+   * Confirm that the snapshot contains references to all the files that should
+   * be in the snapshot.
    */
-  public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
-      byte[] snapshot, byte[] tableName) throws IOException {
-    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), Bytes.toString(tableName));
+  public static void confirmSnapshotValid(
+      SnapshotDescription snapshotDescriptor, byte[] tableName,
+      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
+      boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+      throws IOException {
+    ArrayList nonEmptyTestFamilies = new ArrayList(1);
+    nonEmptyTestFamilies.add(testFamily);
+    confirmSnapshotValid(snapshotDescriptor, Bytes.toString(tableName),
+      nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
+      logsDir, snapshotServers);
   }
 
   /**
-   * Confirm that the snapshot contains references to all the files that should be in the snapshot
+   * Confirm that the snapshot has no references files but only metadata.
    */
-  public static void confirmSnapshotValid(SnapshotDescription snapshotDescriptor,
-      byte[] tableName, byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
-      boolean requireLogs, Path logsDir, Set<String> snapshotServers) throws IOException {
-    Path snapshotDir = SnapshotDescriptionUtils
-        .getCompletedSnapshotDir(snapshotDescriptor, rootDir);
+  public static void confirmEmptySnapshotValid(
+      SnapshotDescription snapshotDescriptor, byte[] tableName,
+      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
+      boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+      throws IOException {
+    ArrayList emptyTestFamilies = new ArrayList(1);
+    emptyTestFamilies.add(testFamily);
+    confirmSnapshotValid(snapshotDescriptor, Bytes.toString(tableName),
+      null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
+      logsDir, snapshotServers);
+  }
+
+  /**
+   * Confirm that the snapshot contains references to all the files that should
+   * be in the snapshot. This method also perform some redundant check like
+   * the existence of the snapshotinfo or the regioninfo which are done always
+   * by the MasterSnapshotVerifier, at the end of the snapshot operation.
+   */
+  public static void confirmSnapshotValid(
+      SnapshotDescription snapshotDescriptor, String tableName,
+      List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
+      Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
+      Path logsDir, Set<String> snapshotServers) throws IOException {
+    // check snapshot dir
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+        snapshotDescriptor, rootDir);
     assertTrue(fs.exists(snapshotDir));
+
+    // check snapshot info
     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
     assertTrue(fs.exists(snapshotinfo));
+
     // check the logs dir
     if (requireLogs) {
-      TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers,
-        snapshotDescriptor, new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME));
+      TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
+          snapshotServers, snapshotDescriptor, new Path(snapshotDir,
+              HConstants.HREGION_LOGDIR_NAME));
     }
+
     // check the table info
-    HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, tableName);
-    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+    HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
+    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
     assertEquals(desc, snapshotDesc);
 
+    // Extract regions and families with store files
+    final Set<String> snapshotRegions = new HashSet<String>();
+    final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+    FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile(final String region, final String family, final String hfileName)
+          throws IOException {
+        snapshotRegions.add(region);
+        snapshotFamilies.add(Bytes.toBytes(family));
+      }
+    });
+
+    // Verify that there are store files in the specified families
+    if (nonEmptyTestFamilies != null) {
+      for (final byte[] familyName: nonEmptyTestFamilies) {
+        assertTrue(snapshotFamilies.contains(familyName));
+      }
+    }
+
+    // Verify that there are no store files in the specified families
+    if (emptyTestFamilies != null) {
+      for (final byte[] familyName: emptyTestFamilies) {
+        assertFalse(snapshotFamilies.contains(familyName));
+      }
+    }
+
+    // Avoid checking regions if the request is for an empty snapshot
+    if ((nonEmptyTestFamilies == null || nonEmptyTestFamilies.size() == 0) &&
+        (emptyTestFamilies != null && emptyTestFamilies.size() > 0)) {
+      assertEquals(0, snapshotRegions.size());
+      return;
+    }
+
     // check the region snapshot for all the regions
-    List<HRegionInfo> regions = admin.getTableRegions(tableName);
+    List<HRegionInfo> regions = admin.getTableRegions(Bytes.toBytes(tableName));
+    assertEquals(regions.size(), snapshotRegions.size());
+
+    // Verify Regions
     for (HRegionInfo info : regions) {
       String regionName = info.getEncodedName();
+      assertTrue(snapshotRegions.contains(regionName));
+
       Path regionDir = new Path(snapshotDir, regionName);
       HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
       assertEquals(info, snapshotRegionInfo);
-      // check to make sure we have the family
-      Path familyDir = new Path(regionDir, Bytes.toString(testFamily));
-      assertTrue("Expected to find: " + familyDir + ", but it doesn't exist", fs.exists(familyDir));
-      // make sure we have some files references
-      assertTrue(fs.listStatus(familyDir).length > 0);
     }
   }
 
@@ -172,11 +271,13 @@ public static void waitForSnapshotToComplete(HMaster master, HSnapshotDescriptio
     }
   }
 
-  public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName) throws IOException {
+  public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
+      throws IOException {
     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
   }
 
-  public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName) throws IOException {
+  public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
+      throws IOException {
     // delete the taken snapshot
     admin.deleteSnapshot(snapshotName);
     assertNoSnapshots(admin);
@@ -202,63 +303,66 @@ public static void expectSnapshotDoneException(HMaster master, HSnapshotDescript
 
   /**
    * List all the HFiles in the given table
-   * @param fs FileSystem where the table lives
+   *
+   * @param fs: FileSystem where the table lives
    * @param tableDir directory of the table
    * @return array of the current HFiles in the table (could be a zero-length array)
    * @throws IOException on unexecpted error reading the FS
    */
-  public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir) throws IOException {
-    // setup the filters we will need based on the filesystem
-    PathFilter regionFilter = new FSUtils.RegionDirFilter(fs);
-    PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs);
-    final PathFilter fileFilter = new PathFilter() {
-      @Override
-      public boolean accept(Path file) {
-        try {
-          return fs.isFile(file);
-        } catch (IOException e) {
-          return false;
-        }
+  public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
+      throws IOException {
+    final ArrayList<Path> hfiles = new ArrayList<Path>();
+    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile(final String region, final String family, final String hfileName)
+          throws IOException {
+        hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
       }
-    };
-
-    FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter);
-    // if no regions, then we are done
-    if (regionDirs == null || regionDirs.length == 0) return new FileStatus[0];
-
-    // go through each of the regions, and add al the hfiles under each family
-    List<FileStatus> regionFiles = new ArrayList<FileStatus>(regionDirs.length);
-    for (FileStatus regionDir : regionDirs) {
-      FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(), familyFilter);
-      // if no families, then we are done again
-      if (fams == null || fams.length == 0) continue;
-      // add all the hfiles under the family
-      regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs, fileFilter));
-    }
-    FileStatus[] files = new FileStatus[regionFiles.size()];
-    regionFiles.toArray(files);
-    return files;
+    });
+    return hfiles.toArray(new Path[hfiles.size()]);
   }
 
   /**
-   * Get all the hfiles in the region, under the passed set of families
-   * @param families all the family directories under the region
-   * @param fs filesystem where the families live
-   * @param fileFilter filter to only include files
-   * @return collection of all the hfiles under all the passed in families (non-null)
-   * @throws IOException on unexecpted error reading the FS
+   * Take a snapshot of the specified table and verify that the given family is
+   * not empty. Note that this will leave the table disabled
+   * in the case of an offline snapshot.
+   */
+  public static void createSnapshotAndValidate(HBaseAdmin admin,
+      String tableName, String familyName, String snapshotNameString,
+      Path rootDir, FileSystem fs, boolean onlineSnapshot)
+      throws Exception {
+    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
+    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
+    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
+                              snapshotNameString, rootDir, fs, onlineSnapshot);
+  }
+
+  /**
+   * Take a snapshot of the specified table and verify the given families.
+   * Note that this will leave the table disabled in the case of an offline snapshot.
    */
-  public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families, FileSystem fs,
-      PathFilter fileFilter) throws IOException {
-    Set<FileStatus> files = new TreeSet<FileStatus>();
-    for (FileStatus family : families) {
-      // get all the hfiles in the family
-      FileStatus[] hfiles = FSUtils.listStatus(fs, family.getPath(), fileFilter);
-      // if no hfiles, then we are done with this family
-      if (hfiles == null || hfiles.length == 0) continue;
-      files.addAll(Arrays.asList(hfiles));
+  public static void createSnapshotAndValidate(HBaseAdmin admin,
+      String tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
+      String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
+        throws Exception {
+    if (!onlineSnapshot) {
+      try {
+        admin.disableTable(tableName);
+      } catch (TableNotEnabledException tne) {
+        LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
+            "already disabled.");
+      }
     }
-    return files;
+    admin.snapshot(snapshotNameString, tableName);
+
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
+      snapshotNameString, tableName);
+    if (snapshots == null || snapshots.size() != 1) {
+      Assert.fail("Incorrect number of snapshots for table " + tableName);
+    }
+
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
+      emptyFamilyNames, rootDir, admin, fs, false,
+      new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
   }
 
   // ==========================================================================

From 4c43a5a30293aa336287aece67103ec2d48d63d1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 2 Oct 2013 00:03:54 +0000
Subject: [PATCH 1171/1540] HBASE-9504 Backport HBASE-1212 to 0.94 - merge tool
 expects regions all have different sequence ids (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1528280 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegion.java  | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c6358bdce74b..0abdeddce659 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4670,16 +4670,6 @@ public static HRegion merge(HRegion a, HRegion b)
       // Because we compacted the source regions we should have no more than two
       // HStoreFiles per family and there will be no reference store
       List<StoreFile> srcFiles = es.getValue();
-      if (srcFiles.size() == 2) {
-        long seqA = srcFiles.get(0).getMaxSequenceId();
-        long seqB = srcFiles.get(1).getMaxSequenceId();
-        if (seqA == seqB) {
-          // Can't have same sequenceid since on open of a store, this is what
-          // distingushes the files (see the map of stores how its keyed by
-          // sequenceid).
-          throw new IOException("Files have same sequenceid: " + seqA);
-        }
-      }
       for (StoreFile hsf: srcFiles) {
         StoreFile.rename(fs, hsf.getPath(),
           StoreFile.getUniqueFile(fs, Store.getStoreHomedir(tableDir,

From 49de0f913342290650ea609647211b1893c34b87 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 2 Oct 2013 19:06:15 +0000
Subject: [PATCH 1172/1540] HBASE-8521 Cells cannot be overwritten with bulk
 loaded HFiles (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1528592 13f79535-47bb-0310-9956-ffa450edef68
---
 .../coprocessor/SecureBulkLoadClient.java     | 14 ++++--
 .../hadoop/hbase/ipc/HRegionInterface.java    | 15 ++++++
 .../mapreduce/LoadIncrementalHFiles.java      | 30 +++++------
 .../hadoop/hbase/regionserver/HRegion.java    | 49 ++++++++++++++----
 .../hbase/regionserver/HRegionServer.java     | 13 ++++-
 .../hadoop/hbase/regionserver/Store.java      | 19 +++----
 .../hadoop/hbase/regionserver/StoreFile.java  | 47 +++++++++++++----
 .../hadoop/hbase/regionserver/wal/HLog.java   |  2 +-
 .../mapreduce/TestLoadIncrementalHFiles.java  | 50 +++++++++++++++++++
 .../hbase/regionserver/TestCompaction.java    |  2 +-
 .../TestHRegionServerBulkLoad.java            |  2 +-
 .../hbase/regionserver/TestStoreFile.java     | 12 ++---
 .../hbase/regionserver/wal/TestWALReplay.java |  2 +-
 13 files changed, 194 insertions(+), 63 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
index a5218331676c..cbfcd5740e33 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
@@ -70,11 +70,17 @@ public void cleanupBulkLoad(String bulkToken) throws IOException {
     }
   }
 
-  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
-                         Token<?> userToken, String bulkToken) throws IOException {
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, Token<?> userToken,
+      String bulkToken) throws IOException {
+    return bulkLoadHFiles(familyPaths, userToken, bulkToken, false);
+  }
+
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, Token<?> userToken,
+      String bulkToken, boolean assignSeqNum) throws IOException {
     try {
-      return (Boolean)Methods.call(protocolClazz, proxy, "bulkLoadHFiles",
-          new Class[]{List.class, Token.class, String.class},new Object[]{familyPaths, userToken, bulkToken});
+      return (Boolean) Methods.call(protocolClazz, proxy, "bulkLoadHFiles", new Class[] {
+          List.class, Token.class, String.class, Boolean.class },
+        new Object[] { familyPaths, userToken, bulkToken, assignSeqNum });
     } catch (Exception e) {
       throw new IOException("Failed to bulkLoadHFiles", e);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index 9886b3a0dc09..9854f31fbf22 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -392,6 +392,21 @@ public void unlockRow(final byte [] regionName, final long lockId)
   public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, byte[] regionName)
   throws IOException;
 
+  /**
+   * Atomically bulk load multiple HFiles (say from different column families)
+   * into an open region.
+   * 
+   * @param familyPaths List of (family, hfile path) pairs
+   * @param regionName name of region to load hfiles into
+   * @param assignSeqNum should we assign sequence numbers
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if fails unrecoverably
+   */
+   public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, byte[] regionName,
+       boolean assignSeqNum)
+  throws IOException;
+
+
   // Master methods
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 7366469a824c..ca7bbc95f67c 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -93,17 +93,17 @@
 public class LoadIncrementalHFiles extends Configured implements Tool {
 
   private static Log LOG = LogFactory.getLog(LoadIncrementalHFiles.class);
-  private static final int  TABLE_CREATE_MAX_RETRIES = 20;
-  private static final long TABLE_CREATE_SLEEP = 60000;
   static AtomicLong regionCount = new AtomicLong(0);
   private HBaseAdmin hbAdmin;
   private Configuration cfg;
 
   public static String NAME = "completebulkload";
+  public static String ASSIGN_SEQ_IDS = "hbase.mapreduce.bulkload.assign.sequenceNumbers";
 
   private boolean useSecure;
   private Token<?> userToken;
   private String bulkToken;
+  private final boolean assignSeqIds;
 
   //package private for testing
   LoadIncrementalHFiles(Configuration conf, Boolean useSecure) throws Exception {
@@ -112,6 +112,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
     this.hbAdmin = new HBaseAdmin(conf);
     //added simple for testing
     this.useSecure = useSecure != null ? useSecure : User.isHBaseSecurityEnabled(conf);
+    this.assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, false);
   }
 
   public LoadIncrementalHFiles(Configuration conf) throws Exception {
@@ -290,7 +291,7 @@ public void doBulkLoad(Path hfofDir, final HTable table)
         LOG.error(err);
       }
     }
-    
+
     if (queue != null && !queue.isEmpty()) {
         throw new RuntimeException("Bulk load aborted with some files not yet loaded."
           + "Please check log for more details.");
@@ -360,7 +361,7 @@ private Multimap<ByteBuffer, LoadQueueItem> groupOrSplitPhase(final HTable table
     Set<Future<List<LoadQueueItem>>> splittingFutures = new HashSet<Future<List<LoadQueueItem>>>();
     while (!queue.isEmpty()) {
       final LoadQueueItem item = queue.remove();
-      
+
       final Callable<List<LoadQueueItem>> call = new Callable<List<LoadQueueItem>>() {
         public List<LoadQueueItem> call() throws Exception {
           List<LoadQueueItem> splits = groupOrSplit(regionGroups, item, table, startEndKeys);
@@ -524,11 +525,11 @@ public Boolean call() throws Exception {
               + Bytes.toStringBinary(row));
           byte[] regionName = location.getRegionInfo().getRegionName();
           if(!useSecure) {
-            success = server.bulkLoadHFiles(famPaths, regionName);
+             success = server.bulkLoadHFiles(famPaths, regionName, assignSeqIds);
           } else {
             HTable table = new HTable(conn.getConfiguration(), tableName);
             secureClient = new SecureBulkLoadClient(table, location.getRegionInfo().getStartKey());
-            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken);
+            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken, assignSeqIds);
           }
           return success;
         } finally {
@@ -653,7 +654,7 @@ private static boolean shouldCopyHFileMetaKey(byte[] key) {
   private boolean doesTableExist(String tableName) throws Exception {
     return hbAdmin.tableExists(tableName);
   }
-  
+
   /*
    * Infers region boundaries for a new table.
    * Parameter:
@@ -671,16 +672,15 @@ public static byte[][] inferBoundaries(TreeMap<byte[], Integer> bdryMap) {
     int runningValue = 0;
     byte[] currStartKey = null;
     boolean firstBoundary = true;
-    
+
     for (Map.Entry<byte[], Integer> item: bdryMap.entrySet()) {
       if (runningValue == 0) currStartKey = item.getKey();
       runningValue += item.getValue();
       if (runningValue == 0) {
         if (!firstBoundary) keysArray.add(currStartKey);
         firstBoundary = false;
-      } 
+      }
     }
-    
     return keysArray.toArray(new byte[0][0]);
   }
  
@@ -709,7 +709,7 @@ private void createTable(String tableName, String dirPath) throws Exception {
     // Build a set of keys
     byte[][] keys = null;
     TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
-    
+
     for (FileStatus stat : familyDirStatuses) {
       if (!stat.isDir()) {
         LOG.warn("Skipping non-directory " + stat.getPath());
@@ -719,10 +719,10 @@ private void createTable(String tableName, String dirPath) throws Exception {
       // Skip _logs, etc
       if (familyDir.getName().startsWith("_")) continue;
       byte[] family = familyDir.getName().getBytes();
-     
+
       hcd = new HColumnDescriptor(family);
       htd.addFamily(hcd);
-      
+
       Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
       for (Path hfile : hfiles) {
         if (hfile.getName().startsWith("_")) continue;
@@ -742,7 +742,7 @@ private void createTable(String tableName, String dirPath) throws Exception {
           LOG.info("Trying to figure out region boundaries hfile=" + hfile +
             " first=" + Bytes.toStringBinary(first) +
             " last="  + Bytes.toStringBinary(last));
-          
+
           // To eventually infer start key-end key boundaries
           Integer value = map.containsKey(first)?(Integer)map.get(first):0;
           map.put(first, value+1);
@@ -754,7 +754,7 @@ private void createTable(String tableName, String dirPath) throws Exception {
         }
       }
     }
-    
+
     keys = LoadIncrementalHFiles.inferBoundaries(map);
     this.hbAdmin.createTable(htd,keys);
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 0abdeddce659..97036310d500 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -494,7 +494,7 @@ public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration confParam,
     // When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled.
     this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval",
         1 * 1000) <= 0;
-    
+
     if (rsServices != null) {
       this.rsAccounting = this.rsServices.getRegionServerAccounting();
       // don't initialize coprocessors if not running within a regionserver
@@ -608,11 +608,13 @@ public Store call() throws IOException {
           Store store = future.get();
 
           this.stores.put(store.getColumnFamilyName().getBytes(), store);
-          long storeSeqId = store.getMaxSequenceId();
-          maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
-              storeSeqId);
-          if (maxSeqId == -1 || storeSeqId > maxSeqId) {
-            maxSeqId = storeSeqId;
+          // Do not include bulk loaded files when determining seqIdForReplay
+          long storeSeqIdForReplay = store.getMaxSequenceId(false);
+          maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), storeSeqIdForReplay);
+          // Include bulk loaded files when determining seqIdForAssignment
+          long storeSeqIdForAssignment = store.getMaxSequenceId(true);
+          if (maxSeqId == -1 || storeSeqIdForAssignment > maxSeqId) {
+            maxSeqId = storeSeqIdForAssignment;
           }
           long maxStoreMemstoreTS = store.getMaxMemstoreTS();
           if (maxStoreMemstoreTS > maxMemstoreTS) {
@@ -2244,7 +2246,7 @@ private long doMiniBatchMutation(
     /** Keep track of the locks we hold so we can release them in finally clause */
     List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
     Set<HashedBytes> rowsAlreadyLocked = Sets.newHashSet();
-      
+
     // reference family maps directly so coprocessors can mutate them if desired
     Map<byte[],List<KeyValue>>[] familyMaps = new Map[batchOp.operations.length];
     // We try to set up a batch in the range [firstIndex,lastIndexExclusive)
@@ -3619,7 +3621,20 @@ private static boolean hasMultipleColumnFamilies(
    * @throws IOException if failed unrecoverably.
    */
   public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths) throws IOException {
-    return bulkLoadHFiles(familyPaths, null);
+    return bulkLoadHFiles(familyPaths, false);
+  }
+
+  /**
+   * Attempts to atomically load a group of hfiles. This is critical for loading rows with multiple
+   * column families atomically.
+   * @param familyPaths List of Pair<byte[] column family, String hfilePath> * @param assignSeqNum
+   *          should we assign sequence numbers
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if failed unrecoverably.
+   */
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, boolean assignSeqId)
+      throws IOException {
+    return bulkLoadHFiles(familyPaths, null, assignSeqId);
   }
 
   /**
@@ -3634,6 +3649,20 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths) throws IOE
    */
   public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
       BulkLoadListener bulkLoadListener) throws IOException {
+    return bulkLoadHFiles(familyPaths, bulkLoadListener, false);
+  }
+
+  /**
+   * Attempts to atomically load a group of hfiles. This is critical for loading rows with multiple
+   * column families atomically.
+   * @param familyPaths List of Pair<byte[] column family, String hfilePath>
+   * @param bulkLoadListener Internal hooks enabling massaging/preparation of a file about to be
+   *          bulk loaded * @param assignSeqNum should we assign sequence numbers
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if failed unrecoverably.
+   */
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
+      BulkLoadListener bulkLoadListener, boolean assignSeqId) throws IOException {
     Preconditions.checkNotNull(familyPaths);
     // we need writeLock for multi-family bulk load
     startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));
@@ -3642,7 +3671,7 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
       this.opMetrics.setWriteRequestCountMetrics( this.writeRequestsCount.get());
 
       // There possibly was a split that happend between when the split keys
-      // were gathered and before the HReiogn's write lock was taken.  We need
+      // were gathered and before the HRegion's write lock was taken.  We need
       // to validate the HFile region before attempting to bulk load all of them
       List<IOException> ioes = new ArrayList<IOException>();
       List<Pair<byte[], String>> failures = new ArrayList<Pair<byte[], String>>();
@@ -3697,7 +3726,7 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
           if(bulkLoadListener != null) {
             finalPath = bulkLoadListener.prepareBulkLoad(familyName, path);
           }
-          store.bulkLoadHFile(finalPath);
+          store.bulkLoadHFile(finalPath, assignSeqId ? this.log.obtainSeqNum() : -1);
           if(bulkLoadListener != null) {
             bulkLoadListener.doneBulkLoad(familyName, path);
           }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index fa7146ff999f..530e51aff670 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2919,6 +2919,17 @@ public void unlockRow(byte[] regionName, long lockId) throws IOException {
   @Override
   public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
       byte[] regionName) throws IOException {
+    return bulkLoadHFiles(familyPaths, regionName, false);
+  }
+
+  /**
+   * Atomically bulk load several HFiles into an open region
+   * @return true if successful, false is failed but recoverably (no action)
+   * @throws IOException if failed unrecoverably
+   */
+  @Override
+  public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
+      byte[] regionName, boolean assignSeqNum) throws IOException {
     checkOpen();
     HRegion region = getRegion(regionName);
     boolean bypass = false;
@@ -2927,7 +2938,7 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
     }
     boolean loaded = false;
     if (!bypass) {
-      loaded = region.bulkLoadHFiles(familyPaths);
+      loaded = region.bulkLoadHFiles(familyPaths, assignSeqNum);
     }
     if (region.getCoprocessorHost() != null) {
       loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 9ee4a0809d45..768482144549 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -347,8 +347,8 @@ public HColumnDescriptor getFamily() {
   /**
    * @return The maximum sequence id in all store files.
    */
-  long getMaxSequenceId() {
-    return StoreFile.getMaxSequenceIdInList(this.getStorefiles());
+  long getMaxSequenceId(boolean includeBulkFiles) {
+    return StoreFile.getMaxSequenceIdInList(this.getStorefiles(), includeBulkFiles);
   }
 
   /**
@@ -628,7 +628,7 @@ void assertBulkLoadHFileOk(Path srcPath) throws IOException {
    * ranges of values in the HFile fit within the stores assigned region.
    * (assertBulkLoadHFileOk checks this)
    */
-  void bulkLoadHFile(String srcPathStr) throws IOException {
+  public void bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
     Path srcPath = new Path(srcPathStr);
 
     // Move the file if it's on another filesystem
@@ -647,7 +647,8 @@ void bulkLoadHFile(String srcPathStr) throws IOException {
       srcPath = tmpPath;
     }
 
-    Path dstPath = StoreFile.getRandomFilename(fs, homedir);
+    Path dstPath =
+        StoreFile.getRandomFilename(fs, homedir, (seqNum == -1) ? null : "_SeqId_" + seqNum + "_");
     LOG.debug("Renaming bulk load file " + srcPath + " to " + dstPath);
     StoreFile.rename(fs, srcPath, dstPath);
 
@@ -1138,7 +1139,7 @@ StoreFile compact(CompactionRequest cr) throws IOException {
     }
 
     // Max-sequenceID is the last key in the files we're compacting
-    long maxId = StoreFile.getMaxSequenceIdInList(filesToCompact);
+    long maxId = StoreFile.getMaxSequenceIdInList(filesToCompact, true);
 
     // Ready to go. Have list of files to compact.
     LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
@@ -1206,10 +1207,10 @@ public void compactRecentForTesting(int N) throws IOException {
         }
 
         filesToCompact = filesToCompact.subList(count - N, count);
-        maxId = StoreFile.getMaxSequenceIdInList(filesToCompact);
+        maxId = StoreFile.getMaxSequenceIdInList(filesToCompact, true);
         isMajor = (filesToCompact.size() == storefiles.size());
         filesCompacting.addAll(filesToCompact);
-        Collections.sort(filesCompacting, StoreFile.Comparators.FLUSH_TIME);
+        Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
       }
     } finally {
       this.lock.readLock().unlock();
@@ -1426,7 +1427,7 @@ public CompactionRequest requestCompaction(int priority, CompactionRequest reque
               filesToCompact, filesCompacting);
         }
         filesCompacting.addAll(filesToCompact.getFilesToCompact());
-        Collections.sort(filesCompacting, StoreFile.Comparators.FLUSH_TIME);
+        Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
 
         // major compaction iff all StoreFiles are included
         boolean isMajor = (filesToCompact.getFilesToCompact().size() == this.storefiles.size());
@@ -1896,7 +1897,7 @@ StoreFile completeCompaction(final Collection<StoreFile> compactedFiles,
   }
 
   public ImmutableList<StoreFile> sortAndClone(List<StoreFile> storeFiles) {
-    Collections.sort(storeFiles, StoreFile.Comparators.FLUSH_TIME);
+    Collections.sort(storeFiles, StoreFile.Comparators.SEQ_ID);
     ImmutableList<StoreFile> newList = ImmutableList.copyOf(storeFiles);
     return newList;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 37e2a4c8e93c..cf686e741b74 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -439,10 +439,11 @@ public static long getMaxMemstoreTSInList(Collection<StoreFile> sfs) {
    * @return 0 if no non-bulk-load files are provided or, this is Store that
    * does not yet have any store files.
    */
-  public static long getMaxSequenceIdInList(Collection<StoreFile> sfs) {
+  public static long getMaxSequenceIdInList(Collection<StoreFile> sfs, 
+      boolean includeBulkLoadedFiles) {
     long max = 0;
     for (StoreFile sf : sfs) {
-      if (!sf.isBulkLoadResult()) {
+      if (includeBulkLoadedFiles || !sf.isBulkLoadResult()) {
         max = Math.max(max, sf.getMaxSequenceId());
       }
     }
@@ -582,6 +583,24 @@ private Reader open() throws IOException {
         }
       }
     }
+
+    if (isBulkLoadResult()) {
+      // generate the sequenceId from the fileName
+      // fileName is of the form <randomName>_SeqId_<id-when-loaded>_
+      String fileName = this.path.getName();
+      int startPos = fileName.indexOf("SeqId_");
+      if (startPos != -1) {
+        this.sequenceid =
+            Long.parseLong(fileName.substring(startPos + 6, fileName.indexOf('_', startPos + 6)));
+        // Handle reference files as done above.
+        if (isReference()) {
+          if (Reference.isTopFileRegion(this.reference.getFileRegion())) {
+            this.sequenceid += 1;
+          }
+        }
+      }
+    }
+
     this.reader.setSequenceID(this.sequenceid);
 
     b = metadataMap.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
@@ -1859,29 +1878,35 @@ abstract static class Comparators {
      * Comparator that compares based on the flush time of
      * the StoreFiles. All bulk loads are placed before all non-
      * bulk loads, and then all files are sorted by sequence ID.
-     * If there are ties, the path name is used as a tie-breaker.
+     * Comparator that compares based on the Sequence Ids of the
+     * the StoreFiles. Bulk loads that did not request a seq ID
+     * are given a seq id of -1; thus, they are placed before all non-
+     * bulk loads, and bulk loads with sequence Id. Among these files,
+     * the bulkLoadTime is used to determine the ordering.
+     * If there are ties, the path name is used as a tie-breaker. 
      */
-    static final Comparator<StoreFile> FLUSH_TIME =
+    static final Comparator<StoreFile> SEQ_ID =
       Ordering.compound(ImmutableList.of(
-          Ordering.natural().onResultOf(new GetBulkTime()),
           Ordering.natural().onResultOf(new GetSeqId()),
+          Ordering.natural().onResultOf(new GetBulkTime()),
           Ordering.natural().onResultOf(new GetPathName())
       ));
 
-    private static class GetBulkTime implements Function<StoreFile, Long> {
+    private static class GetSeqId implements Function<StoreFile, Long> {
       @Override
       public Long apply(StoreFile sf) {
-        if (!sf.isBulkLoadResult()) return Long.MAX_VALUE;
-        return sf.getBulkLoadTimestamp();
+        return sf.getMaxSequenceId();
       }
     }
-    private static class GetSeqId implements Function<StoreFile, Long> {
+
+    private static class GetBulkTime implements Function<StoreFile, Long> {
       @Override
       public Long apply(StoreFile sf) {
-        if (sf.isBulkLoadResult()) return -1L;
-        return sf.getMaxSequenceId();
+        if (!sf.isBulkLoadResult()) return Long.MAX_VALUE;
+        return sf.getBulkLoadTimestamp();
       }
     }
+
     private static class GetPathName implements Function<StoreFile, String> {
       @Override
       public String apply(StoreFile sf) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 4e792b20dc2c..a4f156ff9617 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1514,7 +1514,7 @@ int getNumEntries() {
   /**
    * Obtain a log sequence number.
    */
-  private long obtainSeqNum() {
+  public long obtainSeqNum() {
     return this.logSeqNum.incrementAndGet();
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 05abd3a76d10..e89727fc652f 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -24,6 +24,7 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
@@ -32,10 +33,12 @@
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.*;
@@ -157,6 +160,53 @@ private void runTest(String testName, BloomType bloomType,
     assertEquals(expectedRows, util.countRows(table));
   }
 
+  private void
+      verifyAssignedSequenceNumber(String testName, byte[][][] hfileRanges, boolean nonZero)
+          throws Exception {
+    Path dir = util.getDataTestDir(testName);
+    FileSystem fs = util.getTestFileSystem();
+    dir = dir.makeQualified(fs);
+    Path familyDir = new Path(dir, Bytes.toString(FAMILY));
+
+    int hfileIdx = 0;
+    for (byte[][] range : hfileRanges) {
+      byte[] from = range[0];
+      byte[] to = range[1];
+      createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + hfileIdx++), FAMILY,
+        QUALIFIER, from, to, 1000);
+    }
+
+    final byte[] TABLE = Bytes.toBytes("mytable_" + testName);
+
+    HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
+    HTableDescriptor htd = new HTableDescriptor(TABLE);
+    HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
+    htd.addFamily(familyDesc);
+    admin.createTable(htd, SPLIT_KEYS);
+
+    HTable table = new HTable(util.getConfiguration(), TABLE);
+    util.waitTableAvailable(TABLE, 30000);
+    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
+
+    // Do a dummy put to increase the hlog sequence number
+    Put put = new Put(Bytes.toBytes("row"));
+    put.add(FAMILY, QUALIFIER, Bytes.toBytes("value"));
+    table.put(put);
+
+    loader.doBulkLoad(dir, table);
+
+    // Get the store files
+    List<StoreFile> files =
+        util.getHBaseCluster().getRegions(TABLE).get(0).getStore(FAMILY).getStorefiles();
+    for (StoreFile file : files) {
+      // the sequenceId gets initialized during createReader
+      file.createReader();
+
+      if (nonZero) assertTrue(file.getMaxSequenceId() > 0);
+      else assertTrue(file.getMaxSequenceId() == -1);
+    }
+  }
+
   /**
    * Test loading into a column family that does not exist.
    */
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 4971a86852cc..1660f9f03f10 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -594,7 +594,7 @@ public void testCompactionWithCorruptResult() throws Exception {
     Store store = r.getStore(COLUMN_FAMILY);
 
     List<StoreFile> storeFiles = store.getStorefiles();
-    long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
+    long maxId = StoreFile.getMaxSequenceIdInList(storeFiles, true);
     Compactor tool = new Compactor(this.conf);
 
     StoreFile.Writer compactedFile = tool.compactForTesting(store, this.conf, storeFiles, false,
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
index a1bf73b6b6e4..6a5d7482c4c5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
@@ -145,7 +145,7 @@ public Void call() throws Exception {
           LOG.debug("Going to connect to server " + location + " for row "
               + Bytes.toStringBinary(row));
           byte[] regionName = location.getRegionInfo().getRegionName();
-          server.bulkLoadHFiles(famPaths, regionName);
+          server.bulkLoadHFiles(famPaths, regionName, true);
           return null;
         }
       }.withRetries();
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 7c4860337947..34116c71c4d5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -778,8 +778,8 @@ public void testBloomEdgeCases() throws Exception {
     fs.delete(f, true);
   }
 
-  public void testFlushTimeComparator() {
-    assertOrdering(StoreFile.Comparators.FLUSH_TIME,
+  public void testSeqIdComparator() {
+    assertOrdering(StoreFile.Comparators.SEQ_ID,
         mockStoreFile(true, 1000, -1, "/foo/123"),
         mockStoreFile(true, 1000, -1, "/foo/126"),
         mockStoreFile(true, 2000, -1, "/foo/126"),
@@ -810,13 +810,7 @@ private StoreFile mockStoreFile(boolean bulkLoad, long bulkTimestamp,
     StoreFile mock = Mockito.mock(StoreFile.class);
     Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult();
     Mockito.doReturn(bulkTimestamp).when(mock).getBulkLoadTimestamp();
-    if (bulkLoad) {
-      // Bulk load files will throw if you ask for their sequence ID
-      Mockito.doThrow(new IllegalAccessError("bulk load"))
-        .when(mock).getMaxSequenceId();
-    } else {
-      Mockito.doReturn(seqId).when(mock).getMaxSequenceId();
-    }
+    Mockito.doReturn(seqId).when(mock).getMaxSequenceId();
     Mockito.doReturn(new Path(path)).when(mock).getPath();
     String name = "mock storefile, bulkLoad=" + bulkLoad +
       " bulkTimestamp=" + bulkTimestamp +
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index f95fa1f3e274..030afa538dbf 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -326,7 +326,7 @@ public void testRegionMadeOfBulkLoadedFilesOnly()
     writer.close();
     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
     hfs.add(Pair.newPair(family, f.toString()));
-    region.bulkLoadHFiles(hfs);
+    region.bulkLoadHFiles(hfs, true);
     // Add an edit so something in the WAL
     region.put((new Put(row)).add(family, family, family));
     wal.sync();

From f2d10f4e9ea00f098db21e6d2aee6d6dfab2a722 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 3 Oct 2013 00:10:02 +0000
Subject: [PATCH 1173/1540] HBASE-8521 Addendum - test fix (Jean-Marc
 Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1528692 13f79535-47bb-0310-9956-ffa450edef68
---
 .../mapreduce/TestLoadIncrementalHFilesSplitRecovery.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
index 0ac498071ff5..eaf4aec72b31 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
@@ -273,8 +273,8 @@ private HConnection getMockedConnection(final Configuration conf)
     Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())).
       thenReturn(loc);
     HRegionInterface hri = Mockito.mock(HRegionInterface.class);
-    Mockito.when(hri.bulkLoadHFiles(Mockito.anyList(), (byte [])Mockito.any())).
-      thenThrow(new IOException("injecting bulk load error"));
+    Mockito.when(hri.bulkLoadHFiles(Mockito.anyList(), (byte [])Mockito.any(),
+      Mockito.anyBoolean())).thenThrow(new IOException("injecting bulk load error"));
     Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
       thenReturn(hri);
     return c;

From a4b1fa081f2ec53d2fd493deb377771eab09a816 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 7 Oct 2013 20:57:37 +0000
Subject: [PATCH 1174/1540] HBASE-9711 Improve HBASE-9428 - avoid copying bytes
 for RegexFilter unless necessary

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1530061 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/RegexStringComparator.java  | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
index 912d127b7b17..98c693e2aee5 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
@@ -92,12 +92,17 @@ public void setCharset(final Charset charset) {
 
   @Override
   public int compareTo(byte[] value, int offset, int length) {
-    // See HBASE-9428. Make a copy of the relevant part of the byte[],
-    // or the JDK will copy the entire byte[] during String decode
-    byte[] tmp = Arrays.copyOfRange(value, offset, offset+length);
     // Use find() for subsequence match instead of matches() (full sequence
     // match) to adhere to the principle of least surprise.
-    return pattern.matcher(new String(tmp, charset)).find() ? 0 : 1;
+    String tmp;
+    if (length < value.length / 2) {
+      // See HBASE-9428. Make a copy of the relevant part of the byte[],
+      // or the JDK will copy the entire byte[] during String decode
+      tmp = new String(Arrays.copyOfRange(value, offset, offset + length), charset);
+    } else {
+      tmp = new String(value, offset, length, charset);
+    }
+    return pattern.matcher(tmp).find() ? 0 : 1;
   }
 
   @Override

From 09ba5d62a7d2836231f58f80ccb50445e6263367 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 9 Oct 2013 19:07:19 +0000
Subject: [PATCH 1175/1540] HBASE=9732 Static AtomicLong updated in
 StoreFileScanner every (re)seek

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1530766 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/StoreFileScanner.java      | 9 ++++++---
 .../hadoop/hbase/regionserver/TestSeekOptimizations.java | 2 ++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index 83144fff3fb8..a51db1eb3911 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -52,7 +52,7 @@ public class StoreFileScanner implements KeyValueScanner {
 
   private boolean enforceMVCC = false;
 
-  private static final AtomicLong seekCount = new AtomicLong();
+  private static AtomicLong seekCount;
 
   private ScanQueryMatcher matcher;
 
@@ -133,7 +133,7 @@ public KeyValue next() throws IOException {
   }
 
   public boolean seek(KeyValue key) throws IOException {
-    seekCount.incrementAndGet();
+    if (seekCount != null) seekCount.incrementAndGet();
 
     try {
       try {
@@ -154,7 +154,7 @@ public boolean seek(KeyValue key) throws IOException {
   }
 
   public boolean reseek(KeyValue key) throws IOException {
-    seekCount.incrementAndGet();
+    if (seekCount != null) seekCount.incrementAndGet();
 
     try {
       try {
@@ -366,6 +366,9 @@ public boolean isFileScanner() {
   static final long getSeekCount() {
     return seekCount.get();
   }
+  static final void instrument() {
+    seekCount = new AtomicLong();
+  }
 
   @Override
   public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns, long oldestUnexpiredTS) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
index 2a092e7f8833..345b10a6cfb5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
@@ -142,6 +142,8 @@ public void setUp() {
 
   @Test
   public void testMultipleTimestampRanges() throws IOException {
+    // enable seek counting
+    StoreFileScanner.instrument();
     region = TEST_UTIL.createTestRegion(TestSeekOptimizations.class.getName(),
         new HColumnDescriptor(FAMILY)
             .setCompressionType(comprAlgo)

From db6392a70ee035828cb199f33a60993fe2fb56e7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 9 Oct 2013 19:10:01 +0000
Subject: [PATCH 1176/1540] HBASE=9731 updatesBlockedSeconds RegionServer
 metric should not be a histogram

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1530769 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 6 ++----
 .../hbase/regionserver/metrics/RegionServerMetrics.java     | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 530e51aff670..308e5b5d33e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1605,11 +1605,9 @@ protected void metrics() {
         .getCompactionQueueSize());
     this.metrics.flushQueueSize.set(cacheFlusher
         .getFlushQueueSize());
-    this.metrics.updatesBlockedSeconds.update(updatesBlockedMs > 0 ? 
-        updatesBlockedMs/1000: 0);
+    this.metrics.updatesBlockedSeconds.set(updatesBlockedMs/1000);
     final long updatesBlockedMsHigherWater = cacheFlusher.getUpdatesBlockedMsHighWater().get();
-    this.metrics.updatesBlockedSecondsHighWater.update(updatesBlockedMsHigherWater > 0 ? 
-        updatesBlockedMsHigherWater/1000: 0);
+    this.metrics.updatesBlockedSecondsHighWater.set(updatesBlockedMsHigherWater/1000);
 
     BlockCache blockCache = cacheConfig.getBlockCache();
     if (blockCache != null) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index e6590f5ba3d9..9557c54fe1f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -300,13 +300,13 @@ public class RegionServerMetrics implements Updater {
   /**
    * time blocked on lack of resources
    */
-  public final MetricsHistogram updatesBlockedSeconds = new MetricsHistogram(
+  public final MetricsLongValue updatesBlockedSeconds = new MetricsLongValue(
       "updatesBlockedSeconds", registry);
 
   /**
    * time blocked on memstoreHW
    */
-  public final MetricsHistogram updatesBlockedSecondsHighWater = new MetricsHistogram(
+  public final MetricsLongValue updatesBlockedSecondsHighWater = new MetricsLongValue(
       "updatesBlockedSecondsHighWater",registry);
 
   public RegionServerMetrics() {

From 8aec79e00c5a52a01c3523e1a2abf8b6c1524872 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 12 Oct 2013 22:35:39 +0000
Subject: [PATCH 1177/1540] HBASE-9745 Append HBASE_CLASSPATH to end of Java
 classpath and use another env var for prefix (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1531604 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/bin/hbase b/bin/hbase
index caa498b8856f..841c68c6e349 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -31,11 +31,14 @@
 #
 #   HBASE_CLASSPATH  Extra Java CLASSPATH entries.
 #
+#   HBASE_CLASSPATH_PREFIX Extra Java CLASSPATH entries that should be
+#                    prefixed to the system classpath.
+#
 #   HBASE_HEAPSIZE   The maximum amount of heap to use, in MB. 
 #                    Default is 1000.
 #
 #   HBASE_LIBRARY_PATH  HBase additions to JAVA_LIBRARY_PATH for adding
-#                    native libaries.
+#                    native libraries.
 #
 #   HBASE_OPTS       Extra Java runtime options.
 #
@@ -186,11 +189,6 @@ for f in $HBASE_HOME/lib/*.jar; do
   CLASSPATH=${CLASSPATH}:$f;
 done
 
-# Add user-specified CLASSPATH first
-if [ "$HBASE_CLASSPATH" != "" ]; then
-  CLASSPATH=${HBASE_CLASSPATH}:${CLASSPATH}
-fi
-
 # default log directory & file
 if [ "$HBASE_LOG_DIR" = "" ]; then
   HBASE_LOG_DIR="$HBASE_HOME/logs"
@@ -240,6 +238,16 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then
   fi
 fi
 
+# Add user-specified CLASSPATH last
+if [ "$HBASE_CLASSPATH" != "" ]; then
+  CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH}
+fi
+
+# Add user-specified CLASSPATH prefix first
+if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then
+  CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH}
+fi
+
 # cygwin path translation
 if $cygwin; then
   JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`

From 81977f3337fc92976f979f7eeb46665eb26f3038 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 14 Oct 2013 03:59:44 +0000
Subject: [PATCH 1178/1540] HBASE-9751 Excessive readpoints checks in
 StoreFileScanner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1531792 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/hfile/AbstractHFileReader.java    |  5 +++++
 .../java/org/apache/hadoop/hbase/io/hfile/HFile.java  |  2 ++
 .../apache/hadoop/hbase/io/hfile/HFileReaderV2.java   |  5 +++++
 .../apache/hadoop/hbase/regionserver/StoreFile.java   |  2 +-
 .../hadoop/hbase/regionserver/StoreFileScanner.java   | 11 +++++++----
 5 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
index 5573b661e60b..3c9aab0da1db 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
@@ -354,4 +354,9 @@ public Path getPath() {
   public DataBlockEncoding getEncodingOnDisk() {
     return dataBlockEncoder.getEncodingOnDisk();
   }
+
+  @Override
+  public boolean hasMVCCInfo() {
+    return true;
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 002a69a85930..e32af408ac69 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -522,6 +522,8 @@ ByteBuffer getMetaBlock(String metaBlockName,
     void close(boolean evictOnClose) throws IOException;
 
     DataBlockEncoding getEncodingOnDisk();
+
+    boolean hasMVCCInfo();
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index f5f1f9b61fdf..5725d04a8bf8 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -371,6 +371,11 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
     }
   }
 
+  @Override
+  public boolean hasMVCCInfo() {
+    return includesMemstoreTS && decodeMemstoreTS;
+  }
+
   /**
    * Compares the actual type of a block retrieved from cache or disk with its
    * expected type and throws an exception in case of a mismatch. Expected
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index cf686e741b74..540ee1553d9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -1424,7 +1424,7 @@ public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
                                                boolean isCompaction) {
       return new StoreFileScanner(this,
                                  getScanner(cacheBlocks, pread,
-                                            isCompaction), !isCompaction);
+                                            isCompaction), !isCompaction, reader.hasMVCCInfo());
     }
 
     /**
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index a51db1eb3911..b0556309a7ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -51,6 +51,7 @@ public class StoreFileScanner implements KeyValueScanner {
   private KeyValue delayedSeekKV;
 
   private boolean enforceMVCC = false;
+  private boolean hasMVCCInfo = false;
 
   private static AtomicLong seekCount;
 
@@ -60,10 +61,11 @@ public class StoreFileScanner implements KeyValueScanner {
    * Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner}
    * @param hfs HFile scanner
    */
-  public StoreFileScanner(StoreFile.Reader reader, HFileScanner hfs, boolean useMVCC) {
+  public StoreFileScanner(StoreFile.Reader reader, HFileScanner hfs, boolean useMVCC, boolean hasMVCC) {
     this.reader = reader;
     this.hfs = hfs;
     this.enforceMVCC = useMVCC;
+    this.hasMVCCInfo = hasMVCC;
   }
 
   /**
@@ -124,7 +126,8 @@ public KeyValue next() throws IOException {
       if (cur != null) {
         hfs.next();
         cur = hfs.getKeyValue();
-        skipKVsNewerThanReadpoint();
+        if (hasMVCCInfo)
+          skipKVsNewerThanReadpoint();
       }
     } catch(IOException e) {
       throw new IOException("Could not iterate " + this, e);
@@ -144,7 +147,7 @@ public boolean seek(KeyValue key) throws IOException {
 
         cur = hfs.getKeyValue();
 
-        return skipKVsNewerThanReadpoint();
+        return !hasMVCCInfo ? true : skipKVsNewerThanReadpoint();
       } finally {
         realSeekDone = true;
       }
@@ -164,7 +167,7 @@ public boolean reseek(KeyValue key) throws IOException {
         }
         cur = hfs.getKeyValue();
 
-        return skipKVsNewerThanReadpoint();
+        return !hasMVCCInfo ? true : skipKVsNewerThanReadpoint();
       } finally {
         realSeekDone = true;
       }

From df2800c8b596667c2f055f3da4604f1a9251d506 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Oct 2013 17:12:12 +0000
Subject: [PATCH 1179/1540] HBASE-9753 Excessive readpoint checks in
 MemstoreScanner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1532442 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/MemStore.java    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 22f933543fe2..12e5bfd75c42 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -668,7 +668,7 @@ List<KeyValueScanner> getScanners() {
     this.lock.readLock().lock();
     try {
       return Collections.<KeyValueScanner>singletonList(
-          new MemStoreScanner());
+          new MemStoreScanner(MultiVersionConsistencyControl.getThreadReadPoint()));
     } finally {
       this.lock.readLock().unlock();
     }
@@ -716,6 +716,7 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
 
     // the pre-calculated KeyValue to be returned by peek() or next()
     private KeyValue theNext;
+		private long readPoint;
 
     /*
     Some notes...
@@ -738,16 +739,15 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
       the adds to kvset in the MemStoreScanner.
     */
 
-    MemStoreScanner() {
+    MemStoreScanner(long readPoint) {
       super();
 
+      this.readPoint = readPoint;
       kvsetAtCreation = kvset;
       snapshotAtCreation = snapshot;
     }
 
     private KeyValue getNext(Iterator<KeyValue> it) {
-      long readPoint = MultiVersionConsistencyControl.getThreadReadPoint();
-
       KeyValue v = null;
       try {
         while (it.hasNext()) {

From a1b272037dcc176ca5677f2d7c9bb57fe0f6fc38 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Oct 2013 17:26:23 +0000
Subject: [PATCH 1180/1540] HBASE-9753 Addendum removes tab

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1532450 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/MemStore.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 12e5bfd75c42..c5007d963c7b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -716,7 +716,7 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
 
     // the pre-calculated KeyValue to be returned by peek() or next()
     private KeyValue theNext;
-		private long readPoint;
+    private long readPoint;
 
     /*
     Some notes...

From c03bfb0a97ad48b25daa3f39071421c42d2e9b71 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 17 Oct 2013 17:55:02 +0000
Subject: [PATCH 1181/1540] HBASE-9789 Change logging for Coprocessor exec call
 to trace

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1533178 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/master/HMaster.java     | 4 ++--
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java    | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f975a34b75bc..2fef4ffd3c14 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1999,8 +1999,8 @@ public ExecResult execCoprocessor(Exec call) throws IOException {
     Class<? extends CoprocessorProtocol> protocol = call.getProtocol();
     if (protocol == null) {
       String protocolName = call.getProtocolName();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Received dynamic protocol exec call with protocolName " + protocolName);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Received dynamic protocol exec call with protocolName " + protocolName);
       }
       // detect the actual protocol class
       protocol  = protocolHandlerNames.get(protocolName);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 97036310d500..19003084bf0f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5604,8 +5604,8 @@ public ExecResult exec(Exec call)
     Class<? extends CoprocessorProtocol> protocol = call.getProtocol();
     if (protocol == null) {
       String protocolName = call.getProtocolName();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Received dynamic protocol exec call with protocolName " + protocolName);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Received dynamic protocol exec call with protocolName " + protocolName);
       }
       // detect the actual protocol class
       protocol  = protocolHandlerNames.get(protocolName);

From bcc5439d40dcd1b4e9a23940e80f8c7afbe29ea2 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 17 Oct 2013 18:10:16 +0000
Subject: [PATCH 1182/1540] HBASE-9749: Custom threadpool for Coprocessor
 obtained HTables

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1533185 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/CoprocessorEnvironment.java  | 10 +-
 .../apache/hadoop/hbase/client/HTable.java    |  2 +-
 .../hbase/coprocessor/CoprocessorHost.java    | 20 +++-
 .../TestOpenTableInCoprocessor.java           | 95 ++++++++++++++++---
 4 files changed, 110 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java
index dbb5cc94631d..09bca1153bc1 100644
--- a/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java
+++ b/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java
@@ -17,6 +17,7 @@
 package org.apache.hadoop.hbase;
 
 import java.io.IOException;
+import java.util.concurrent.ExecutorService;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
@@ -49,4 +50,11 @@ public interface CoprocessorEnvironment {
    * @throws IOException
    */
   public HTableInterface getTable(byte[] tableName) throws IOException;
-}
+
+  /**
+   * @return an interface for accessing the given table using the passed executor to run batch
+   *         operations
+   * @throws IOException
+   */
+  public HTableInterface getTable(byte[] tableName, ExecutorService service) throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 0ce498556dea..298fc43674e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -175,7 +175,7 @@ public HTable(byte[] tableName, HConnection connection) throws IOException {
     this.finishSetup();
   }
 
-  private static ThreadPoolExecutor getDefaultExecutor(Configuration conf) {
+  public static ThreadPoolExecutor getDefaultExecutor(Configuration conf) {
     int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE);
     if (maxThreads == 0) {
       maxThreads = 1; // is there a better default?
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 198b26e33489..4b7ea3c31362 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -31,6 +31,7 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.UUID;
+import java.util.concurrent.ExecutorService;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -363,9 +364,10 @@ class HTableWrapper implements HTableInterface {
       private HTable table;
       private HConnection connection;
 
-      public HTableWrapper(byte[] tableName, HConnection connection) throws IOException {
+      public HTableWrapper(byte[] tableName, HConnection connection, ExecutorService executor)
+          throws IOException {
         this.tableName = tableName;
-        this.table = new HTable(tableName, connection);
+        this.table = new HTable(tableName, connection, executor);
         this.connection = connection;
         openTables.add(this);
       }
@@ -680,7 +682,19 @@ public Configuration getConfiguration() {
      */
     @Override
     public HTableInterface getTable(byte[] tableName) throws IOException {
-      return new HTableWrapper(tableName, CoprocessorHConnection.getConnectionForEnvironment(this));
+      return this.getTable(tableName, HTable.getDefaultExecutor(getConfiguration()));
+    }
+
+    /**
+     * Open a table from within the Coprocessor environment
+     * @param tableName the table name
+     * @return an interface for manipulating the table
+     * @exception java.io.IOException Exception
+     */
+    @Override
+    public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {
+      return new HTableWrapper(tableName, CoprocessorHConnection.getConnectionForEnvironment(this),
+          pool);
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
index 414bc25de852..e73d7adcd6ab 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
@@ -21,6 +21,11 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -33,11 +38,12 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -48,9 +54,10 @@
 public class TestOpenTableInCoprocessor {
 
   private static final byte[] otherTable = Bytes.toBytes("otherTable");
+  private static final byte[] primaryTable = Bytes.toBytes("primary");
   private static final byte[] family = new byte[] { 'f' };
 
-  private static boolean completed = false;
+  private static boolean [] completed = new boolean[1];
 
   /**
    * Custom coprocessor that just copies the write to another table.
@@ -65,28 +72,93 @@ public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WAL
       p.add(family, null, new byte[] { 'a' });
       table.put(put);
       table.flushCommits();
-      completed = true;
+      completed[0] = true;
       table.close();
     }
 
   }
 
+  private static boolean []  completedWithPool = new boolean [1] ;
+
+  public static class CustomThreadPoolCoprocessor extends BaseRegionObserver {
+
+    /**
+     * Get a pool that has only ever one thread. A second action added to the pool (running
+     * concurrently), will cause an exception.
+     * @return
+     */
+    private ExecutorService getPool() {
+      int maxThreads = 1;
+      long keepAliveTime = 60;
+      ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime,
+          TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+          Threads.newDaemonThreadFactory("hbase-table"));
+      pool.allowCoreThreadTimeOut(true);
+      return pool;
+    }
+
+    @Override
+    public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
+        boolean writeToWAL) throws IOException {
+      HTableInterface table = e.getEnvironment().getTable(otherTable, getPool());
+      Put p = new Put(new byte[] { 'a' });
+      p.add(family, null, new byte[] { 'a' });
+      try {
+        table.batch(Collections.singletonList(put));
+      } catch (InterruptedException e1) {
+        throw new IOException(e1);
+      }
+      completedWithPool[0] = true;
+      table.close();
+    }
+  }
+
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    UTIL.startMiniCluster();
+  }
+
+  @After
+  public void cleanupTestTable() throws Exception {
+    UTIL.getHBaseAdmin().disableTable(primaryTable);
+    UTIL.getHBaseAdmin().deleteTable(primaryTable);
+
+    UTIL.getHBaseAdmin().disableTable(otherTable);
+    UTIL.getHBaseAdmin().deleteTable(otherTable);
+
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws Exception{
+    UTIL.shutdownMiniCluster();
+  }
+
   @Test
   public void testCoprocessorCanCreateConnectionToRemoteTable() throws Throwable {
-    HBaseTestingUtility UTIL = new HBaseTestingUtility();
-    HTableDescriptor primary = new HTableDescriptor("primary");
+    runCoprocessorConnectionToRemoteTable(SendToOtherTableCoprocessor.class, completed);
+  }
+
+  @Test
+  public void testCoprocessorCanCreateConnectionToRemoteTableWithCustomPool() throws Throwable {
+    runCoprocessorConnectionToRemoteTable(CustomThreadPoolCoprocessor.class, completedWithPool);
+  }
+
+  private void runCoprocessorConnectionToRemoteTable(Class<? extends BaseRegionObserver> clazz,
+      boolean[] completeCheck) throws Throwable {
+    HTableDescriptor primary = new HTableDescriptor(primaryTable);
     primary.addFamily(new HColumnDescriptor(family));
     // add our coprocessor
-    primary.addCoprocessor(SendToOtherTableCoprocessor.class.getName());
+    primary.addCoprocessor(clazz.getName());
 
     HTableDescriptor other = new HTableDescriptor(otherTable);
     other.addFamily(new HColumnDescriptor(family));
-    UTIL.startMiniCluster();
+
 
     HBaseAdmin admin = UTIL.getHBaseAdmin();
     admin.createTable(primary);
     admin.createTable(other);
-    admin.close();
 
     HTable table = new HTable(UTIL.getConfiguration(), "primary");
     Put p = new Put(new byte[] { 'a' });
@@ -96,11 +168,10 @@ public void testCoprocessorCanCreateConnectionToRemoteTable() throws Throwable {
     table.close();
 
     HTable target = new HTable(UTIL.getConfiguration(), otherTable);
-    assertTrue("Didn't complete update to target table!", completed);
+    assertTrue("Didn't complete update to target table!", completeCheck[0]);
     assertEquals("Didn't find inserted row", 1, getKeyValueCount(target));
     target.close();
 
-    UTIL.shutdownMiniCluster();
   }
 
   /**

From e7dfc21d4f6b36d06ce0d9e73e5951db941e51a8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 17 Oct 2013 20:10:10 +0000
Subject: [PATCH 1183/1540] HBASE-9747 PrefixFilter with OR condition gives
 wrong results (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1533238 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/PrefixFilter.java     | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
index e3c3d39c83b4..59e3ad3337dd 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
@@ -20,15 +20,13 @@
 
 package org.apache.hadoop.hbase.filter;
 
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-
+import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.io.DataInput;
-import java.util.List;
 import java.util.ArrayList;
 
+import org.apache.hadoop.hbase.util.Bytes;
+
 import com.google.common.base.Preconditions;
 
 /**
@@ -37,6 +35,7 @@
 public class PrefixFilter extends FilterBase {
   protected byte [] prefix = null;
   protected boolean passedPrefix = false;
+  protected boolean filterRow = true;
 
   public PrefixFilter(final byte [] prefix) {
     this.prefix = prefix;
@@ -63,7 +62,16 @@ public boolean filterRowKey(byte[] buffer, int offset, int length) {
     if(cmp > 0) {
       passedPrefix = true;
     }
-    return cmp != 0;
+    filterRow = (cmp != 0);
+    return filterRow;
+  }
+
+  public boolean filterRow() {
+    return filterRow;
+  }
+
+  public void reset() {
+    filterRow = true;
   }
 
   public boolean filterAllRemaining() {

From b4df85d6b0502c354c0b9c4b63ab31ba1da0475b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 18 Oct 2013 19:37:37 +0000
Subject: [PATCH 1184/1540] HBASE-9747 added test (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1533605 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/TestFilter.java       | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index 0c61aa25c954..c09bba00a487 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -28,10 +28,12 @@
 import java.util.List;
 
 import junit.framework.Assert;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -1225,6 +1227,52 @@ public void testFilterListWithSingleColumnValueFilter() throws IOException {
     verifyScanFull(s, kvs);
   }
 
+  // HBASE-9747
+  public void testFilterListWithPrefixFilter() throws IOException {
+    byte[] family = Bytes.toBytes("f1");
+    byte[] qualifier = Bytes.toBytes("q1");
+    HTableDescriptor htd = new HTableDescriptor(getName());
+    htd.addFamily(new HColumnDescriptor(family));
+    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
+    HRegion testRegion = HRegion.createHRegion(info, testDir, conf, htd);
+
+    for(int i=0; i<5; i++) {
+      Put p = new Put(Bytes.toBytes((char)('a'+i) + "row"));
+      p.setDurability(Durability.SKIP_WAL);
+      p.add(family, qualifier, Bytes.toBytes(String.valueOf(111+i)));
+      testRegion.put(p);
+    }
+    testRegion.flushcache();
+
+    // rows starting with "b"
+    PrefixFilter pf = new PrefixFilter(new byte[] {'b'}) ;
+    // rows with value of column 'q1' set to '113'
+    SingleColumnValueFilter scvf = new SingleColumnValueFilter(
+        family, qualifier, CompareOp.EQUAL, Bytes.toBytes("113"));
+    // combine these two with OR in a FilterList
+    FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf);
+
+    Scan s1 = new Scan();
+    s1.setFilter(filterList);
+    InternalScanner scanner = testRegion.getScanner(s1);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    int resultCount = 0;
+    while(scanner.next(results)) {
+      resultCount++;
+      byte[] row = results.get(0).getRow();
+      LOG.debug("Found row: " + Bytes.toStringBinary(row));
+      Assert.assertTrue(Bytes.equals(row, Bytes.toBytes("brow"))
+          || Bytes.equals(row, Bytes.toBytes("crow")));
+      results.clear();
+    }
+    Assert.assertEquals(2, resultCount);
+    scanner.close();
+
+    HLog hlog = testRegion.getLog();
+    testRegion.close();
+    hlog.closeAndDelete();
+  }
+
   public void testNestedFilterListWithSCVF() throws IOException {
     byte[] columnStatus = Bytes.toBytes("S");
     HTableDescriptor htd = new HTableDescriptor(getName());

From 9545bdfe92013b727d62e049a43cec226dbb5b6b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 20 Oct 2013 20:43:42 +0000
Subject: [PATCH 1185/1540] HBase Rest Server - DELETE scanner operation is a
 no-op (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1533976 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/ScannerResultGenerator.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
index e068c2e4edc3..4cf2d7beec0e 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
@@ -99,6 +99,10 @@ public String getID() {
   }
 
   public void close() {
+    if (scanner != null) {
+      scanner.close();
+      scanner = null;
+    }
   }
 
   public boolean hasNext() {

From 770eeecdf3a6758aeb3eaca6d048caaed2398a6b Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Mon, 21 Oct 2013 19:59:47 +0000
Subject: [PATCH 1186/1540] HBASE-9716. LoadTestTool should provide default min
 and max settings to the data generator

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1534355 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index d6e963bc2f3f..1cb656df6a04 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -60,6 +60,9 @@ public class LoadTestTool extends AbstractHBaseTool {
   /** Column families used by the test */
   protected static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY };
 
+  /** The default data size if not specified */
+  protected static final int DEFAULT_DATA_SIZE = 64;
+
   /** The number of reader/writer threads if not specified */
   protected static final int DEFAULT_NUM_THREADS = 20;
 
@@ -127,7 +130,7 @@ public class LoadTestTool extends AbstractHBaseTool {
   // Writer options
   protected int numWriterThreads = DEFAULT_NUM_THREADS;
   protected int minColsPerKey, maxColsPerKey;
-  protected int minColDataSize, maxColDataSize;
+  protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE;
   protected boolean isMultiPut;
 
   // Reader options

From 39fa0d405631abb003a9cbd2336e67a439e938cc Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 21 Oct 2013 20:31:47 +0000
Subject: [PATCH 1187/1540] HBASE-9783 o.a.h.h.r.HRegion.mutateRow() with
 non-existent CF cause NPE (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1534365 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    |  4 +++-
 .../hbase/regionserver/TestHRegion.java       | 21 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 19003084bf0f..450ea8c5c944 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4923,6 +4923,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
 
       long txid = 0;
       boolean walSyncSuccessful = false;
+      boolean memstoreUpdated = false;
       boolean locked = false;
 
       // 2. acquire the row lock(s)
@@ -4982,6 +4983,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
 
         // 7. apply to memstore
         long addedSize = 0;
+        memstoreUpdated = true;
         for (Mutation m : mutations) {
           addedSize += applyFamilyMapToMemstore(m.getFamilyMap(), w);
         }
@@ -5021,7 +5023,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
         }
       } finally {
         // 12. clean up if needed
-        if (!walSyncSuccessful) {
+        if (memstoreUpdated && !walSyncSuccessful) {
           int kvsRolledback = 0;
           for (Mutation m : mutations) {
             for (Map.Entry<byte[], List<KeyValue>> e : m.getFamilyMap()
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 28419d7d5236..6e6f6aa69c5a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -63,6 +63,7 @@
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -1013,6 +1014,26 @@ public void testCheckAndPut_wrongRowInPut() throws IOException {
     }
   }
 
+  public void testmutateRowsWithLocks_wrongCF() throws IOException {
+    this.region = initHRegion(tableName, this.getName(), conf, fam1, fam2);
+    try {
+      Put put = new Put(row2);
+      put.add(fam3, qual1, value1);
+      RowMutations rm = new RowMutations(row2);
+      rm.add(put);
+      try {
+        region.mutateRow(rm);
+        fail();
+      } catch (DoNotRetryIOException expected) {
+        // expected exception.
+        LOG.debug("Caught expected exception: " + expected.getMessage());
+      }
+    } finally {
+      HRegion.closeHRegion(this.region);
+      this.region = null;
+    }
+  }
+  
   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
     byte [] tableName = Bytes.toBytes("testtable");
     byte [] row1 = Bytes.toBytes("row1");

From c3b22255709c03c45a45c65c98b277728c4bc858 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 22 Oct 2013 04:59:19 +0000
Subject: [PATCH 1188/1540] HBASE-9488 Improve performance for small scan
 (chunhui shen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1534494 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ClientScanner.java    |  27 ++-
 .../hbase/client/ClientSmallScanner.java      | 210 ++++++++++++++++++
 .../apache/hadoop/hbase/client/HTable.java    |   4 +
 .../org/apache/hadoop/hbase/client/Scan.java  |  34 +++
 .../hadoop/hbase/client/ServerCallable.java   |  11 +
 .../hadoop/hbase/ipc/HRegionInterface.java    |  13 ++
 .../hbase/regionserver/HRegionServer.java     |  73 ++++--
 .../hadoop/hbase/regionserver/Store.java      |   4 +-
 .../hbase/regionserver/StoreScanner.java      |   7 +-
 .../hbase/client/TestFromClientSide.java      |  37 +++
 10 files changed, 388 insertions(+), 32 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 0fbabe5ded5c..9aa4112feb4f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -42,19 +42,19 @@
  */
 public class ClientScanner extends AbstractClientScanner {
     private final Log LOG = LogFactory.getLog(this.getClass());
-    private Scan scan;
-    private boolean closed = false;
+    protected Scan scan;
+    protected boolean closed = false;
     // Current region scanner is against.  Gets cleared if current region goes
     // wonky: e.g. if it splits on us.
-    private HRegionInfo currentRegion = null;
+    protected HRegionInfo currentRegion = null;
     private ScannerCallable callable = null;
-    private final LinkedList<Result> cache = new LinkedList<Result>();
-    private final int caching;
-    private long lastNext;
+    protected final LinkedList<Result> cache = new LinkedList<Result>();
+    protected final int caching;
+    protected long lastNext;
     // Keep lastResult returned successfully in case we have to reset scanner.
-    private Result lastResult = null;
-    private ScanMetrics scanMetrics = null;
-    private final long maxScannerResultSize;
+    protected Result lastResult = null;
+    protected ScanMetrics scanMetrics = null;
+    protected final long maxScannerResultSize;
     private final HConnection connection;
     private final byte[] tableName;
     private final int scannerTimeout;
@@ -116,6 +116,11 @@ public ClientScanner(final Configuration conf, final Scan scan,
         this.caching = conf.getInt("hbase.client.scanner.caching", 1);
       }
 
+      // initialize the scanner
+      initializeScannerInConstruction();
+    }
+    
+    protected void initializeScannerInConstruction() throws IOException{
       // initialize the scanner
       nextScanner(this.caching, false);
     }
@@ -137,7 +142,7 @@ protected long getTimestamp() {
     }
 
     // returns true if the passed region endKey
-    private boolean checkScanStopRow(final byte [] endKey) {
+    protected boolean checkScanStopRow(final byte [] endKey) {
       if (this.scan.getStopRow().length > 0) {
         // there is a stop row, check to see if we are past it.
         byte [] stopRow = scan.getStopRow();
@@ -234,7 +239,7 @@ protected ScannerCallable getScannerCallable(byte [] localStartKey,
      *
      * scan.setAttribute(SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE))
      */
-    private void writeScanMetrics() throws IOException {
+    protected void writeScanMetrics() throws IOException {
       if (this.scanMetrics == null) {
         return;
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
new file mode 100644
index 000000000000..adbcd8ba4a23
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
@@ -0,0 +1,210 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.protobuf.ServiceException;
+
+/**
+ * Client scanner for small scan. Generally, only one RPC is called to fetch the
+ * scan results, unless the results cross multiple regions or the row count of
+ * results excess the caching.
+ * 
+ * For small scan, it will get better performance than {@link ClientScanner}
+ */
+public class ClientSmallScanner extends ClientScanner {
+  private final Log LOG = LogFactory.getLog(this.getClass());
+  private ServerCallable<Result[]> smallScanCallable = null;
+  // When fetching results from server, skip the first result if it has the same
+  // row with this one
+  private byte[] skipRowOfFirstResult = null;
+
+  /**
+   * Create a new ClientSmallScanner for the specified table. An HConnection
+   * will be retrieved using the passed Configuration. Note that the passed
+   * {@link Scan} 's start row maybe changed.
+   * 
+   * @param conf The {@link Configuration} to use.
+   * @param scan {@link Scan} to use in this scanner
+   * @param tableName The table that we wish to rangeGet
+   * @throws IOException
+   */
+  public ClientSmallScanner(final Configuration conf, final Scan scan,
+      final byte[] tableName) throws IOException {
+    this(conf, scan, tableName, HConnectionManager.getConnection(conf));
+  }
+
+  /**
+   * Create a new ClientSmallScanner for the specified table. An HConnection
+   * will be retrieved using the passed Configuration. Note that the passed
+   * {@link Scan} 's start row maybe changed.
+   * @param conf
+   * @param scan
+   * @param tableName
+   * @param connection
+   * @throws IOException
+   */
+  public ClientSmallScanner(final Configuration conf, final Scan scan,
+      final byte[] tableName, HConnection connection) throws IOException {
+    super(conf, scan, tableName, connection);
+  }
+
+  @Override
+  protected void initializeScannerInConstruction() throws IOException {
+    // No need to initialize the scanner when constructing instance, do it when
+    // calling next(). Do nothing here.
+  }
+
+  /**
+   * Gets a scanner for following scan. Move to next region or continue from the
+   * last result or start from the start row.
+   * @param nbRows
+   * @param done true if Server-side says we're done scanning.
+   * @param currentRegionDone true if scan is over on current region
+   * @return true if has next scanner
+   * @throws IOException
+   */
+  private boolean nextScanner(int nbRows, final boolean done,
+      boolean currentRegionDone) throws IOException {
+    // Where to start the next getter
+    byte[] localStartKey;
+    int cacheNum = nbRows;
+    skipRowOfFirstResult = null;
+    // if we're at end of table, close and return false to stop iterating
+    if (this.currentRegion != null && currentRegionDone) {
+      byte[] endKey = this.currentRegion.getEndKey();
+      if (endKey == null || Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)
+          || checkScanStopRow(endKey) || done) {
+        close();
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Finished with small scan at " + this.currentRegion);
+        }
+        return false;
+      }
+      localStartKey = endKey;
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Finished with region " + this.currentRegion);
+      }
+    } else if (this.lastResult != null) {
+      localStartKey = this.lastResult.getRow();
+      skipRowOfFirstResult = this.lastResult.getRow();
+      cacheNum++;
+    } else {
+      localStartKey = this.scan.getStartRow();
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Advancing internal small scanner to startKey at '"
+          + Bytes.toStringBinary(localStartKey) + "'");
+    }
+    smallScanCallable = getSmallScanCallable(localStartKey, cacheNum);
+    if (this.scanMetrics != null && skipRowOfFirstResult == null) {
+      this.scanMetrics.countOfRegions.inc();
+    }
+    return true;
+  }
+
+  private ServerCallable<Result[]> getSmallScanCallable(byte[] localStartKey,
+      final int nbRows) {
+    this.scan.setStartRow(localStartKey);
+    ServerCallable<Result[]> callable = new ServerCallable<Result[]>(
+        getConnection(), getTableName(), scan.getStartRow()) {
+      public Result[] call() throws IOException {
+        return server.scan(location.getRegionInfo().getRegionName(), scan,
+            nbRows);
+      }
+    };
+    return callable;
+  }
+
+  @Override
+  public Result next() throws IOException {
+    // If the scanner is closed and there's nothing left in the cache, next is a
+    // no-op.
+    if (cache.size() == 0 && this.closed) {
+      return null;
+    }
+    if (cache.size() == 0) {
+      Result[] values = null;
+      long remainingResultSize = maxScannerResultSize;
+      int countdown = this.caching;
+      boolean currentRegionDone = false;
+      // Values == null means server-side filter has determined we must STOP
+      while (remainingResultSize > 0 && countdown > 0
+          && nextScanner(countdown, values == null, currentRegionDone)) {
+        // Server returns a null values if scanning is to stop. Else,
+        // returns an empty array if scanning is to go on and we've just
+        // exhausted current region.
+        values = smallScanCallable.withRetries();
+        this.currentRegion = smallScanCallable.getHRegionInfo();
+        long currentTime = System.currentTimeMillis();
+        if (this.scanMetrics != null) {
+          this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime
+              - lastNext);
+        }
+        lastNext = currentTime;
+        if (values != null && values.length > 0) {
+          for (int i = 0; i < values.length; i++) {
+            Result rs = values[i];
+            if (i == 0 && this.skipRowOfFirstResult != null
+                && Bytes.equals(skipRowOfFirstResult, rs.getRow())) {
+              // Skip the first result
+              continue;
+            }
+            cache.add(rs);
+            for (KeyValue kv : rs.raw()) {
+              remainingResultSize -= kv.heapSize();
+            }
+            countdown--;
+            this.lastResult = rs;
+          }
+        }
+        currentRegionDone = countdown > 0;
+      }
+    }
+
+    if (cache.size() > 0) {
+      return cache.poll();
+    }
+    // if we exhausted this scanner before calling close, write out the scan
+    // metrics
+    writeScanMetrics();
+    return null;
+  }
+
+  @Override
+  public void close() {
+    closed = true;
+    try {
+      writeScanMetrics();
+    } catch (IOException e) {
+      // As ClientScanner#close, we don't want the scanner close() method to
+      // throw.
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 298fc43674e8..309732f4f41a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -695,6 +695,10 @@ public ResultScanner getScanner(final Scan scan) throws IOException {
     if (scan.getCaching() <= 0) {
       scan.setCaching(getScannerCaching());
     }
+    if (scan.isSmall()) {
+      return new ClientSmallScanner(getConfiguration(), scan, getTableName(),
+          this.connection);
+    }
     return new ClientScanner(getConfiguration(), scan, getTableName(),
         this.connection);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 553bde3d2858..17a110acf297 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -83,6 +83,7 @@ public class Scan extends OperationWithAttributes implements Writable {
   private static final String RAW_ATTR = "_raw_";
   private static final String ONDEMAND_ATTR = "_ondemand_";
   private static final String ISOLATION_LEVEL = "_isolationlevel_";
+  private static final String SMALL_ATTR = "_small_";
 
   private static final byte SCAN_VERSION = (byte)2;
   private byte [] startRow = HConstants.EMPTY_START_ROW;
@@ -490,6 +491,39 @@ public boolean doLoadColumnFamiliesOnDemand() {
     return attr == null ? false : Bytes.toBoolean(attr);
   }
 
+  /**
+   * Set whether this scan is a small scan
+   * <p>
+   * Small scan should use pread and big scan can use seek + read
+   * 
+   * seek + read is fast but can cause two problem (1) resource contention (2)
+   * cause too much network io
+   * 
+   * [89-fb] Using pread for non-compaction read request
+   * https://issues.apache.org/jira/browse/HBASE-7266
+   * 
+   * On the other hand, if setting it true, we would do
+   * openScanner,next,closeScanner in one RPC call. It means the better
+   * performance for small scan. [HBASE-9488].
+   * 
+   * Generally, if the scan range is within one data block(64KB), it could be
+   * considered as a small scan.
+   * 
+   * @param small
+   */
+  public void setSmall(boolean small) {
+    setAttribute(SMALL_ATTR, Bytes.toBytes(small));
+  }
+
+  /**
+   * Get whether this scan is a small scan
+   * @return true if small scan
+   */
+  public boolean isSmall() {
+    byte[] attr = getAttribute(SMALL_ATTR);
+    return attr == null ? false : Bytes.toBoolean(attr);
+  }
+
   /**
    * Compile the table and column family (i.e. schema) information
    * into a String. Useful for parsing and aggregation by debugging,
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
index 14d1290738f4..1dd6943e78fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
@@ -259,4 +260,14 @@ private static Throwable translateException(Throwable t) throws IOException {
     }
     return t;
   }
+
+  /**
+   * @return the HRegionInfo for the current region
+   */
+  public HRegionInfo getHRegionInfo() {
+    if (this.location == null) {
+      return null;
+    }
+    return this.location.getRegionInfo();
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index 9854f31fbf22..014ee9f4e180 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -662,4 +662,17 @@ public boolean checkAndDelete(final byte[] regionName, final byte[] row,
 
   @Override
   public void stop(String why);
+
+  /**
+   * Perform scan operation.
+   * @param regionName name of region to get from
+   * @param Scan scan operation
+   * @param numberOfRows the maximum number of rows to fetch
+   * @return Array of Results;array is empty if done with this region and null
+   *         if we are NOT to go to the next region (happens when a filter rules
+   *         that the scan is done).
+   * @throws IOException e
+   */
+  public Result[] scan(byte[] regionName, Scan scan, int numberOfRows)
+      throws IOException;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 308e5b5d33e4..31ba0742f7c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2521,8 +2521,15 @@ public boolean checkAndDelete(final byte[] regionName, final byte[] row,
   //
   // remote scanner interface
   //
-
+  
   public long openScanner(byte[] regionName, Scan scan) throws IOException {
+    RegionScanner s = internalOpenScanner(regionName, scan);
+    long scannerId = addScanner(s);
+    return scannerId;
+  }
+
+  private RegionScanner internalOpenScanner(byte[] regionName, Scan scan)
+      throws IOException {
     checkOpen();
     NullPointerException npe = null;
     if (regionName == null) {
@@ -2557,7 +2564,7 @@ public long openScanner(byte[] regionName, Scan scan) throws IOException {
           s = savedScanner;
         }
       }
-      return addScanner(s);
+      return s;
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t, "Failed openScanner"));
     }
@@ -2587,16 +2594,23 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
       LOG.info("Client tried to access missing scanner " + scannerName);
       throw new UnknownScannerException("Name: " + scannerName);
     }
+    return internalNext(s, nbRows, scannerName);
+  }
+
+  private Result[] internalNext(final RegionScanner s, int nbRows,
+      String scannerName) throws IOException {
     try {
       checkOpen();
     } catch (IOException e) {
       // If checkOpen failed, server not running or filesystem gone,
       // cancel this lease; filesystem is gone or we're closing or something.
-      try {
-        this.leases.cancelLease(scannerName);
-      } catch (LeaseException le) {
-        LOG.info("Server shutting down and client tried to access missing scanner " +
-          scannerName);
+      if (scannerName != null) {
+        try {
+          this.leases.cancelLease(scannerName);
+        } catch (LeaseException le) {
+          LOG.info("Server shutting down and client tried to access missing scanner "
+              + scannerName);
+        }
       }
       throw e;
     }
@@ -2605,7 +2619,9 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
       // Remove lease while its being processed in server; protects against case
       // where processing of request takes > lease expiration time.
       try {
-        lease = this.leases.removeLease(scannerName);
+        if (scannerName != null) {
+          lease = this.leases.removeLease(scannerName);
+        }
       } catch (LeaseException le) {
         // What it really means is that there's no such scanner.
         LOG.info("Client tried to access missing scanner " + scannerName + " (no lease)");
@@ -2675,25 +2691,30 @@ public Result[] next(final long scannerId, int nbRows) throws IOException {
       return s.isFilterDone() && results.isEmpty() ? null
           : results.toArray(new Result[0]);
     } catch (Throwable t) {
-      if (t instanceof NotServingRegionException) {
+      if (t instanceof NotServingRegionException && scannerName != null) {
         this.scanners.remove(scannerName);
       }
       throw convertThrowableToIOE(cleanup(t));
     } finally {
       // We're done. On way out readd the above removed lease.  Adding resets
       // expiration time on lease.
-      if (this.scanners.containsKey(scannerName)) {
+      if (scannerName != null && this.scanners.containsKey(scannerName)) {
         if (lease != null) this.leases.addLease(lease);
       }
     }
   }
 
   public void close(final long scannerId) throws IOException {
+    String scannerName = String.valueOf(scannerId);
+    RegionScanner s = scanners.get(scannerName);
+    internalCloseScanner(s, scannerName);
+  }
+
+  private void internalCloseScanner(final RegionScanner s, String scannerName)
+      throws IOException {
     try {
       checkOpen();
       requestCount.incrementAndGet();
-      String scannerName = String.valueOf(scannerId);
-      RegionScanner s = scanners.get(scannerName);
 
       HRegion region = null;
       if (s != null) {
@@ -2705,14 +2726,18 @@ public void close(final long scannerId) throws IOException {
           }
         }
       }
-
-      s = scanners.remove(scannerName);
-      if (s != null) {
-        s.close();
-        this.leases.cancelLease(scannerName);
+      RegionScanner toCloseScanner = s;
+      if (scannerName != null) {
+        toCloseScanner = scanners.remove(scannerName);
+      }
+      if (toCloseScanner != null) {
+        toCloseScanner.close();
+        if (scannerName != null) {
+          this.leases.cancelLease(scannerName);
+        }
 
         if (region != null && region.getCoprocessorHost() != null) {
-          region.getCoprocessorHost().postScannerClose(s);
+          region.getCoprocessorHost().postScannerClose(toCloseScanner);
         }
       }
     } catch (Throwable t) {
@@ -2720,6 +2745,18 @@ public void close(final long scannerId) throws IOException {
     }
   }
 
+  @Override
+  public Result[] scan(byte[] regionName, Scan scan, int numberOfRows)
+      throws IOException {
+    RegionScanner s = internalOpenScanner(regionName, scan);
+    try {
+      Result[] results = internalNext(s, numberOfRows, null);
+      return results;
+    } finally {
+      internalCloseScanner(s, null);
+    }
+  }
+
   /**
    * Instantiated as a scanner lease. If the lease times out, the scanner is
    * closed
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 768482144549..04c12c5276ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1060,7 +1060,7 @@ private void notifyChangedReadersObservers() throws IOException {
    * @return all scanners for this store
    */
   protected List<KeyValueScanner> getScanners(boolean cacheBlocks,
-      boolean isGet,
+      boolean usePread,
       boolean isCompaction,
       ScanQueryMatcher matcher) throws IOException {
     List<StoreFile> storeFiles;
@@ -1079,7 +1079,7 @@ protected List<KeyValueScanner> getScanners(boolean cacheBlocks,
     // but now we get them in ascending order, which I think is
     // actually more correct, since memstore get put at the end.
     List<StoreFileScanner> sfScanners = StoreFileScanner
-      .getScannersForStoreFiles(storeFiles, cacheBlocks, isGet, isCompaction, matcher);
+      .getScannersForStoreFiles(storeFiles, cacheBlocks, usePread, isCompaction, matcher);
     List<KeyValueScanner> scanners =
       new ArrayList<KeyValueScanner>(sfScanners.size()+1);
     scanners.addAll(sfScanners);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index db2bd8d274f0..100b71801a67 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -73,6 +73,9 @@ public class StoreScanner extends NonLazyKeyValueScanner
   // if heap == null and lastTop != null, you need to reseek given the key below
   private KeyValue lastTop = null;
 
+  // A flag whether use pread for scan
+  private boolean scanUsePread = false;
+
   /** An internal constructor. */
   private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
       final NavigableSet<byte[]> columns, long ttl, int minVersions) {
@@ -91,6 +94,7 @@ private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
     // for multi-row (non-"get") scans because this is not done in
     // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
     useRowColBloom = numCol > 1 || (!isGet && numCol == 1);
+    this.scanUsePread = scan.isSmall();
   }
 
   /**
@@ -218,7 +222,8 @@ private void initializeMetricNames() {
    */
   private List<KeyValueScanner> getScannersNoCompaction() throws IOException {
     final boolean isCompaction = false;
-    return selectScannersFrom(store.getScanners(cacheBlocks, isGet,
+    boolean usePread = isGet || scanUsePread;
+    return selectScannersFrom(store.getScanners(cacheBlocks, usePread,
         isCompaction, matcher));
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index a38b8785ac90..084fdaeb8368 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4940,6 +4940,43 @@ public void testRawScanRespectsVersions() throws Exception {
     TEST_UTIL.deleteTable(TABLE);
   }
 
+  @Test
+  public void testSmallScan() throws Exception {
+    // Test Initialization.
+    byte[] TABLE = Bytes.toBytes("testSmallScan");
+    HTable table = TEST_UTIL.createTable(TABLE, FAMILY);
+
+    // Insert one row each region
+    int insertNum = 10;
+    for (int i = 0; i < 10; i++) {
+      Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
+      put.add(FAMILY, QUALIFIER, VALUE);
+      table.put(put);
+    }
+
+    // nomal scan
+    ResultScanner scanner = table.getScanner(new Scan());
+    int count = 0;
+    for (Result r : scanner) {
+      assertTrue(!r.isEmpty());
+      count++;
+    }
+    assertEquals(insertNum, count);
+
+    // small scan
+    Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+    scan.setSmall(true);
+    scan.setCaching(2);
+    scanner = table.getScanner(scan);
+    count = 0;
+    for (Result r : scanner) {
+      assertTrue(!r.isEmpty());
+      count++;
+    }
+    assertEquals(insertNum, count);
+
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 6c5ffadab7b3ed85a441a459169daff5ef5a258c Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 22 Oct 2013 23:45:21 +0000
Subject: [PATCH 1189/1540] HBASE-9737 Corrupt HFile cause resource leak
 leading to Region Server OOM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1534855 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/hfile/HFile.java   | 44 +++++++++++++------
 1 file changed, 31 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index e32af408ac69..89924640d9ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -549,19 +549,37 @@ private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
     FixedFileTrailer trailer = null;
     try {
       trailer = FixedFileTrailer.readFromStream(fsdis, size);
-    } catch (IllegalArgumentException iae) {
-      throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, iae);
-    }
-    switch (trailer.getMajorVersion()) {
-    case 1:
-      return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
-          cacheConf);
-    case 2:
-      return new HFileReaderV2(path, trailer, fsdis, fsdisNoFsChecksum,
-          size, closeIStream,
-          cacheConf, preferredEncodingInCache, hfs);
-    default:
-      throw new CorruptHFileException("Invalid HFile version " + trailer.getMajorVersion());
+      switch (trailer.getMajorVersion()) {
+      case 1:
+        return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
+            cacheConf);
+      case 2:
+        return new HFileReaderV2(path, trailer, fsdis, fsdisNoFsChecksum,
+            size, closeIStream,
+            cacheConf, preferredEncodingInCache, hfs);
+      default:
+        throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
+      }
+    } catch (Throwable t) {
+      if (closeIStream) {
+        try {
+          if (fsdis != fsdisNoFsChecksum && fsdisNoFsChecksum != null) {
+            fsdisNoFsChecksum.close();
+            fsdisNoFsChecksum = null;
+          }
+        } catch (Throwable t2) {
+          LOG.warn("Error closing fsdisNoFsChecksum FSDataInputStream", t2);
+        }
+        try {
+          if (fsdis != null) {
+            fsdis.close();
+            fsdis = null;
+          }
+        } catch (Throwable t2) {
+          LOG.warn("Error closing fsdis FSDataInputStream", t2);
+        }
+      }
+      throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, t);
     }
   }
 

From a4b7d8bc3b74fa872eb5320f66c22bb65f07053d Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 24 Oct 2013 17:07:55 +0000
Subject: [PATCH 1190/1540] HBASE-9819 Backport HBASE-8372 'Provide mutability
 to CompoundConfiguration' to 0.94 (Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1535442 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/CompoundConfiguration.java   | 417 ++++++++----------
 .../TestCompoundConfiguration.java            |  71 ++-
 2 files changed, 243 insertions(+), 245 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
index def1cb66e090..89c9331f6059 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
@@ -19,12 +19,10 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import java.io.DataInput;
 import java.io.DataOutput;
-import java.io.IOException;
+import java.io.IOException; 
 import java.io.OutputStream;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -36,7 +34,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.util.StringUtils;
 
 /**
  * Do a shallow merge of multiple KV configuration pools. This is a very useful
@@ -53,9 +50,15 @@
  * This class is package private because we expect significant refactoring here
  * on the HBase side when certain HDFS changes are added & ubiquitous. Will
  * revisit expanding access at that point.
+ *
+ * WARNING: The values set in the CompoundConfiguration are do not handle Property variable
+ * substitution.  However, if they are set in the underlying configuration substitutions are
+ * done.
  */
 @InterfaceAudience.Private
 class CompoundConfiguration extends Configuration {
+	private Configuration mutableConf = null;
+
   /**
    * Default Constructor. Initializes empty configuration
    */
@@ -78,51 +81,73 @@ private static interface ImmutableConfigMap extends Iterable<Map.Entry<String,St
    * These initial APIs actually required original thought
    ***************************************************************************/
 
+  static class ImmutableConfWrapper implements  ImmutableConfigMap {
+    Configuration c;
+    
+    ImmutableConfWrapper(Configuration conf) {
+      c = conf;
+    }
+
+    @Override
+    public Iterator<Map.Entry<String,String>> iterator() {
+      return c.iterator();
+    }
+    
+    @Override
+    public String get(String key) {
+      return c.get(key);
+    }
+
+    @Override
+    public String getRaw(String key) {
+      return c.getRaw(key);
+    }
+
+    @Override
+    public Class<?> getClassByName(String name)
+        throws ClassNotFoundException {
+      return c.getClassByName(name);
+    }
+
+    @Override
+    public int size() {
+      return c.size();
+    }
+
+    @Override
+    public String toString() {
+      return c.toString();
+    }
+  }
+
+  /**
+   * If set has been called, it will create a mutableConf.  This converts the mutableConf to an
+   * immutable one and resets it to allow a new mutable conf.  This is used when a new map or
+   * conf is added to the compound configuration to preserve proper override semantics.
+   */
+  void freezeMutableConf() {
+    if (mutableConf == null) {
+      // do nothing if there is no current mutableConf
+      return;
+    }
+
+    this.configs.add(0, new ImmutableConfWrapper(mutableConf));
+    mutableConf = null;
+  }
+
   /**
    * Add Hadoop Configuration object to config list
    * @param conf configuration object
    * @return this, for builder pattern
    */
   public CompoundConfiguration add(final Configuration conf) {
+    freezeMutableConf();
     if (conf instanceof CompoundConfiguration) {
       this.configs.addAll(0, ((CompoundConfiguration) conf).configs);
       return this;
     }
     // put new config at the front of the list (top priority)
-    this.configs.add(0, new ImmutableConfigMap() {
-      Configuration c = conf;
-
-      @Override
-      public String get(String key) {
-        return c.get(key);
-      }
-
-      @Override
-      public String getRaw(String key) {
-        return c.getRaw(key);
-      }
-
-      @Override
-      public Class<?> getClassByName(String name)
-          throws ClassNotFoundException {
-        return c.getClassByName(name);
-      }
-
-      @Override
-      public int size() {
-        return c.size();
-      }
-
-      @Override
-      public String toString() {
-        return c.toString();
-      }
-
-      @Override
-      public Iterator<Entry<String, String>> iterator() {
-        return c.iterator();
-      }
-    });
+    this.configs.add(0, new ImmutableConfWrapper(conf));
     return this;
   }
 
@@ -137,6 +162,8 @@ public Iterator<Entry<String, String>> iterator() {
    */
   public CompoundConfiguration add(
       final Map<ImmutableBytesWritable, ImmutableBytesWritable> map) {
+    freezeMutableConf();
+
     // put new map at the front of the list (top priority)
     this.configs.add(0, new ImmutableConfigMap() {
       Map<ImmutableBytesWritable, ImmutableBytesWritable> m = map;
@@ -227,6 +254,54 @@ public void remove() {
     return this;
   }
 
+  /**
+   * Add String map to config list. This map is generally created by HTableDescriptor
+   * or HColumnDescriptor, but can be abstractly used. The added configuration
+   * overrides the previous ones if there are name collisions.
+   *
+   * @return this, for builder pattern
+   */
+  public CompoundConfiguration addStringMap(final Map<String, String> map) {
+    freezeMutableConf();
+
+    // put new map at the front of the list (top priority)
+    this.configs.add(0, new ImmutableConfigMap() {
+      Map<String, String> m = map;
+
+      @Override
+      public Iterator<Map.Entry<String,String>> iterator() {
+        return map.entrySet().iterator();
+      }
+
+      @Override
+      public String get(String key) {
+        return m.get(key);
+      }
+
+      @Override
+      public String getRaw(String key) {
+        return get(key);
+      }
+
+      @Override
+      public Class<?> getClassByName(String name)
+      throws ClassNotFoundException {
+        return null;
+      }
+
+      @Override
+      public int size() {
+        return m.size();
+      }
+
+      @Override
+      public String toString() {
+        return m.toString();
+      }
+    });
+    return this;
+  }
+
   @Override
   public String toString() {
     StringBuffer sb = new StringBuffer();
@@ -239,6 +314,13 @@ public String toString() {
 
   @Override
   public String get(String key) {
+    if (mutableConf != null) {
+      String value = mutableConf.get(key);
+      if (value != null) {
+        return value;
+      }
+    }
+
     for (ImmutableConfigMap m : this.configs) {
       String value = m.get(key);
       if (value != null) {
@@ -250,6 +332,13 @@ public String get(String key) {
 
   @Override
   public String getRaw(String key) {
+    if (mutableConf != null) {
+      String value = mutableConf.getRaw(key);
+      if (value != null) {
+        return value;
+      }
+    }
+
     for (ImmutableConfigMap m : this.configs) {
       String value = m.getRaw(key);
       if (value != null) {
@@ -261,6 +350,13 @@ public String getRaw(String key) {
 
   @Override
   public Class<?> getClassByName(String name) throws ClassNotFoundException {
+    if (mutableConf != null) {
+      Class<?> value = mutableConf.getClassByName(name);
+      if (value != null) {
+        return value;
+      }
+    }
+
     for (ImmutableConfigMap m : this.configs) {
       try {
         Class<?> value = m.getClassByName(name);
@@ -275,9 +371,14 @@ public Class<?> getClassByName(String name) throws ClassNotFoundException {
     throw new ClassNotFoundException();
   }
 
+  // TODO: This method overestimates the number of configuration settings -- if a value is masked
+  // by an overriding config or map, it will be counted multiple times.
   @Override
   public int size() {
     int ret = 0;
+    if (mutableConf != null) {
+      ret += mutableConf.size();
+    }
     for (ImmutableConfigMap m : this.configs) {
       ret += m.size();
     }
@@ -299,28 +400,36 @@ public Iterator<Map.Entry<String, String>> iterator() {
         }
       }
     }
+    // add mutations to this CompoundConfiguration last.
+    if (mutableConf != null) {
+      Iterator<Map.Entry<String, String>> miter = mutableConf.iterator();
+      while (miter.hasNext()) {
+        Map.Entry<String, String> entry = miter.next();
+        ret.put(entry.getKey(), entry.getValue());
+      }
+    }
 
     return UnmodifiableIterator.decorate(ret.entrySet().iterator());
   }
 
-  /***************************************************************************
-   * You should just ignore everything below this line unless there's a bug in
-   * Configuration.java...
+  /**
+   * Get the value of the <code>name</code>. If the key is deprecated,
+   * it returns the value of the first key which replaces the deprecated key
+   * and is not null.
+   * If no such property exists,
+   * then <code>defaultValue</code> is returned.
+
+   * The CompooundConfiguration does not do property substitution.  To do so we need
+   * Configuration.getProps to be protected or package visible.  Though in hadoop2 it is
+   * protected, in hadoop1 the method is private and not accessible.
    *
-   * Below get APIs are directly copied from Configuration.java Oh, how I wish
-   * this wasn't so! A tragically-sad example of why you use interfaces instead
-   * of inheritance.
+   * All of the get* methods call this overridden get method.
    *
-   * Why the duplication? We basically need to override Configuration.getProps
-   * or we'd need protected access to Configuration.properties so we can modify
-   * that pointer. There are a bunch of functions in the base Configuration that
-   * call getProps() and we need to use our derived version instead of the base
-   * version. We need to make a generic implementation that works across all
-   * HDFS versions. We should modify Configuration.properties in HDFS 1.0 to be
-   * protected, but we still need to have this code until that patch makes it to
-   * all the HDFS versions we support.
-   ***************************************************************************/
-
+   * @param name property name.
+   * @param defaultValue default value.
+   * @return property value, or <code>defaultValue</code> if the property
+   *         doesn't exist.
+   **/
   @Override
   public String get(String name, String defaultValue) {
     String ret = get(name);
@@ -328,203 +437,23 @@ public String get(String name, String defaultValue) {
   }
 
   @Override
-  public int getInt(String name, int defaultValue) {
-    String valueString = get(name);
-    if (valueString == null)
-      return defaultValue;
-    try {
-      String hexString = getHexDigits(valueString);
-      if (hexString != null) {
-        return Integer.parseInt(hexString, 16);
-      }
-      return Integer.parseInt(valueString);
-    } catch (NumberFormatException e) {
-      return defaultValue;
-    }
-  }
-
-  @Override
-  public long getLong(String name, long defaultValue) {
-    String valueString = get(name);
-    if (valueString == null)
-      return defaultValue;
-    try {
-      String hexString = getHexDigits(valueString);
-      if (hexString != null) {
-        return Long.parseLong(hexString, 16);
-      }
-      return Long.parseLong(valueString);
-    } catch (NumberFormatException e) {
-      return defaultValue;
-    }
-  }
-
-  protected String getHexDigits(String value) {
-    boolean negative = false;
-    String str = value;
-    String hexString = null;
-    if (value.startsWith("-")) {
-      negative = true;
-      str = value.substring(1);
-    }
-    if (str.startsWith("0x") || str.startsWith("0X")) {
-      hexString = str.substring(2);
-      if (negative) {
-        hexString = "-" + hexString;
-      }
-      return hexString;
-    }
-    return null;
-  }
-
-  @Override
-  public float getFloat(String name, float defaultValue) {
-    String valueString = get(name);
-    if (valueString == null)
-      return defaultValue;
-    try {
-      return Float.parseFloat(valueString);
-    } catch (NumberFormatException e) {
-      return defaultValue;
-    }
-  }
-
-  @Override
-  public boolean getBoolean(String name, boolean defaultValue) {
-    String valueString = get(name);
-    if ("true".equals(valueString))
-      return true;
-    else if ("false".equals(valueString))
-      return false;
-    else return defaultValue;
-  }
-
-  @Override
-  public IntegerRanges getRange(String name, String defaultValue) {
-    return new IntegerRanges(get(name, defaultValue));
-  }
-
-  @Override
-  public Collection<String> getStringCollection(String name) {
-    String valueString = get(name);
-    return StringUtils.getStringCollection(valueString);
-  }
-
-  @Override
-  public String[] getStrings(String name) {
-    String valueString = get(name);
-    return StringUtils.getStrings(valueString);
-  }
-
-  @Override
-  public String[] getStrings(String name, String... defaultValue) {
-    String valueString = get(name);
-    if (valueString == null) {
-      return defaultValue;
-    } else {
-      return StringUtils.getStrings(valueString);
-    }
-  }
-
-  @Override
-  public Class<?>[] getClasses(String name, Class<?>... defaultValue) {
-    String[] classnames = getStrings(name);
-    if (classnames == null)
-      return defaultValue;
-    try {
-      Class<?>[] classes = new Class<?>[classnames.length];
-      for (int i = 0; i < classnames.length; i++) {
-        classes[i] = getClassByName(classnames[i]);
-      }
-      return classes;
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public Class<?> getClass(String name, Class<?> defaultValue) {
-    String valueString = get(name);
-    if (valueString == null)
-      return defaultValue;
-    try {
-      return getClassByName(valueString);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public <U> Class<? extends U> getClass(String name,
-      Class<? extends U> defaultValue, Class<U> xface) {
-    try {
-      Class<?> theClass = getClass(name, defaultValue);
-      if (theClass != null && !xface.isAssignableFrom(theClass))
-        throw new RuntimeException(theClass + " not " + xface.getName());
-      else if (theClass != null)
-        return theClass.asSubclass(xface);
-      else
-        return null;
-    } catch (Exception e) {
-      throw new RuntimeException(e);
+  public void set(String name, String value) {
+    if (mutableConf == null) {
+      // not thread safe
+      mutableConf = new Configuration(false); // an empty configuration
     }
+    mutableConf.set(name,  value);
   }
-
-  /*******************************************************************
-   * This class is immutable. Quickly abort any attempts to alter it *
-   *******************************************************************/
-
+  
+  /***********************************************************************************************
+   * These methods are unsupported, and no code using CompoundConfiguration depend upon them.
+   * Quickly abort upon any attempts to use them.
+   **********************************************************************************************/
   @Override
   public void clear() {
     throw new UnsupportedOperationException("Immutable Configuration");
   }
 
-  @Override
-  public void set(String name, String value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setIfUnset(String name, String value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setInt(String name, int value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setLong(String name, long value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setFloat(String name, float value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setBoolean(String name, boolean value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setBooleanIfUnset(String name, boolean value) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setStrings(String name, String... values) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setClass(String name, Class<?> theClass, Class<?> xface) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-  @Override
-  public void setClassLoader(ClassLoader classLoader) {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    throw new UnsupportedOperationException("Immutable Configuration");
-  }
-
   @Override
   public void write(DataOutput out) throws IOException {
     throw new UnsupportedOperationException("Immutable Configuration");
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
index cff9d8aab13f..e147eb49e4f6 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
@@ -50,7 +50,7 @@ protected void setUp() throws Exception {
   @Test
   public void testBasicFunctionality() throws ClassNotFoundException {
     CompoundConfiguration compoundConf = new CompoundConfiguration()
-        .add(baseConf);
+        .add(baseConf); 
     assertEquals("1", compoundConf.get("A"));
     assertEquals(2, compoundConf.getInt("B", 0));
     assertEquals(3, compoundConf.getInt("C", 0));
@@ -66,6 +66,29 @@ public void testBasicFunctionality() throws ClassNotFoundException {
     }
   }
 
+  @Test
+  public void testPut() {
+    CompoundConfiguration compoundConf = new CompoundConfiguration()
+      .add(baseConf);
+    assertEquals("1", compoundConf.get("A"));
+    assertEquals(2, compoundConf.getInt("B", 0));
+    assertEquals(3, compoundConf.getInt("C", 0));
+    assertEquals(0, compoundConf.getInt("D", 0));
+
+    compoundConf.set("A", "1337");
+    compoundConf.set("string", "stringvalue");
+    assertEquals(1337, compoundConf.getInt("A", 0));
+    assertEquals("stringvalue", compoundConf.get("string"));
+
+    // we didn't modify the base conf
+    assertEquals("1", baseConf.get("A"));
+    assertNull(baseConf.get("string"));
+
+    // adding to the base shows up in the compound
+    baseConf.set("setInParent", "fromParent");
+    assertEquals("fromParent", compoundConf.get("setInParent"));
+  }
+
   @Test
   public void testWithConfig() {
     Configuration conf = new Configuration();
@@ -128,6 +151,52 @@ public void testWithIbwMap() {
     }
     // verify that entries from ImmutableConfigMap's are merged in the iterator's view
     assertEquals(baseConfSize + 2, cnt);
+
+    // Verify that adding map after compound configuration is modified overrides properly
+    CompoundConfiguration conf2 = new CompoundConfiguration();
+    conf2.set("X", "modification");
+    conf2.set("D", "not4");
+    assertEquals("modification", conf2.get("X"));
+    assertEquals("not4", conf2.get("D"));
+    conf2.add(map);
+    assertEquals("4", conf2.get("D")); // map overrides
+  }
+
+  @Test
+  public void testWithStringMap() {
+    Map<String, String> map = new HashMap<String, String>();
+    map.put("B", "2b");
+    map.put("C", "33");
+    map.put("D", "4");
+    // unlike config, note that IBW Maps can accept null values
+    map.put("G", null);
+
+    CompoundConfiguration compoundConf = new CompoundConfiguration().addStringMap(map);
+    assertEquals("2b", compoundConf.get("B"));
+    assertEquals(33, compoundConf.getInt("C", 0));
+    assertEquals("4", compoundConf.get("D"));
+    assertEquals(4, compoundConf.getInt("D", 0));
+    assertNull(compoundConf.get("E"));
+    assertEquals(6, compoundConf.getInt("F", 6));
+    assertNull(compoundConf.get("G"));
+
+    int cnt = 0;
+    for (Map.Entry<String,String> entry : compoundConf) {
+      cnt++;
+      if (entry.getKey().equals("B")) assertEquals("2b", entry.getValue());
+      else if (entry.getKey().equals("G")) assertEquals(null, entry.getValue());
+    }
+    // verify that entries from ImmutableConfigMap's are merged in the iterator's view
+    assertEquals(4, cnt);
+    
+    // Verify that adding map after compound configuration is modified overrides properly
+    CompoundConfiguration conf2 = new CompoundConfiguration();
+    conf2.set("X", "modification");
+    conf2.set("D", "not4");
+    assertEquals("modification", conf2.get("X"));
+    assertEquals("not4", conf2.get("D"));
+    conf2.addStringMap(map);
+    assertEquals("4", conf2.get("D")); // map overrides
   }
 
   @Test

From 07d903aafecd3e5ce5635536f57dd8c2628954ed Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 24 Oct 2013 17:43:56 +0000
Subject: [PATCH 1191/1540] HBASE-8397 improve unit-test coverage of package
 org.apache.hadoop.hbase.master.metrics (0.94)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1535452 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/metrics/MasterMetrics.java   |   5 +-
 .../master/metrics/TestMasterStatistics.java  | 116 ++++++++++++++++++
 2 files changed, 118 insertions(+), 3 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
index ccb8a38eb15d..36800b818511 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
@@ -30,7 +30,6 @@
 import org.apache.hadoop.metrics.MetricsUtil;
 import org.apache.hadoop.metrics.Updater;
 import org.apache.hadoop.metrics.jvm.JvmMetrics;
-import org.apache.hadoop.metrics.util.MetricsLongValue;
 import org.apache.hadoop.metrics.util.MetricsRegistry;
 
 
@@ -90,7 +89,7 @@ public MasterMetrics(final String name) {
 
     // get custom attributes
     try {
-      Object m = 
+      Object m =
         ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
       if (m instanceof String) {
         this.extendedPeriod = Long.parseLong((String) m)*1000;
@@ -135,7 +134,7 @@ public void doUpdates(MetricsContext unused) {
   public void resetAllMinMax() {
     // Nothing to do
   }
-  
+
   /**
    * Record a single instance of a split
    * @param time time that the split took
diff --git a/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java b/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java
new file mode 100644
index 000000000000..ef3add97dae1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java
@@ -0,0 +1,116 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.metrics;
+
+import static org.junit.Assert.*;
+
+import java.lang.management.ManagementFactory;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests {@link MasterMetrics} and access to it through the
+ * {@link MasterStatistics} management bean.
+ */
+@Category(SmallTests.class)
+public class TestMasterStatistics {
+
+  @Test
+  public void testMasterStatistics() throws Exception {
+    // No timer updates started here since NullContext is
+    // instantiated by default for HBase:
+    MasterMetrics masterMetrics = new MasterMetrics("foo");
+    try {
+      final MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+      final ObjectName objectName = new ObjectName(
+          "hadoop:name=MasterStatistics,service=Master");
+
+      masterMetrics.doUpdates(null);
+
+      masterMetrics.resetAllMinMax();
+
+      masterMetrics.incrementRequests(10);
+      Thread.sleep(1001);
+
+      masterMetrics.addSnapshot(1L);
+      masterMetrics.addSnapshotClone(2L);
+      masterMetrics.addSnapshotRestore(3L);
+
+      // 3 times added split, average = (5+3+4)/3 = 4
+      masterMetrics.addSplit(4L, 5L);
+      masterMetrics.addSplit(2L, 3L);
+      masterMetrics.addSplit(13L, 4L);
+
+      masterMetrics.doUpdates(null);
+
+      final float f = masterMetrics.getRequests();
+      // f = 10/T, where T >= 1 sec. So, we assert that 0 < f <= 10:
+      if (f <= 0.0f || f > 10.0f) {
+        fail("Unexpected rate value: " + f);
+      }
+      Object attribute = server.getAttribute(objectName, "cluster_requests");
+      float f2 = ((Float) attribute).floatValue();
+      assertEquals("The value obtained through bean server should be equal to the one " +
+          "obtained directly.", f, f2, 1e-4);
+
+      // NB: these 3 metrics are not pushed upon masterMetrics.doUpdates(),
+      // so they always return null:
+      attribute = server.getAttribute(objectName, "snapshotTimeNumOps");
+      assertEquals(Integer.valueOf(0), attribute);
+      attribute = server.getAttribute(objectName, "snapshotRestoreTimeNumOps");
+      assertEquals(Integer.valueOf(0), attribute);
+      attribute = server.getAttribute(objectName, "snapshotCloneTimeNumOps");
+      assertEquals(Integer.valueOf(0), attribute);
+
+      attribute = server.getAttribute(objectName, "splitSizeNumOps");
+      assertEquals(Integer.valueOf(3), attribute);
+      attribute = server.getAttribute(objectName, "splitSizeAvgTime");
+      assertEquals(Long.valueOf(4), attribute);
+    } finally {
+      masterMetrics.shutdown();
+    }
+  }
+
+  @Test
+  public void testHBaseInfoBean() throws Exception {
+    MasterMetrics masterMetrics = new MasterMetrics("foo");
+    try {
+      final MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+      // Test Info bean:
+      final ObjectName objectName2 = new ObjectName(
+          "hadoop:name=Info,service=HBase");
+      Object attribute;
+      attribute = server.getAttribute(objectName2, "revision");
+      assertNotNull(attribute);
+      attribute = server.getAttribute(objectName2, "version");
+      assertNotNull(attribute);
+      attribute = server.getAttribute(objectName2, "hdfsUrl");
+      assertNotNull(attribute);
+      attribute = server.getAttribute(objectName2, "user");
+      assertNotNull(attribute);
+    } finally {
+      masterMetrics.shutdown();
+    }
+  }
+}

From 9c123e61d7e48908e0ed063a22d5dc4d559f6ccc Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 24 Oct 2013 19:15:13 +0000
Subject: [PATCH 1192/1540] HBASE-8397 Revert due to test failure in secure
 build

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1535503 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/metrics/MasterMetrics.java   |   5 +-
 .../master/metrics/TestMasterStatistics.java  | 116 ------------------
 2 files changed, 3 insertions(+), 118 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java

diff --git a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
index 36800b818511..ccb8a38eb15d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.metrics.MetricsUtil;
 import org.apache.hadoop.metrics.Updater;
 import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.hadoop.metrics.util.MetricsLongValue;
 import org.apache.hadoop.metrics.util.MetricsRegistry;
 
 
@@ -89,7 +90,7 @@ public MasterMetrics(final String name) {
 
     // get custom attributes
     try {
-      Object m =
+      Object m = 
         ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
       if (m instanceof String) {
         this.extendedPeriod = Long.parseLong((String) m)*1000;
@@ -134,7 +135,7 @@ public void doUpdates(MetricsContext unused) {
   public void resetAllMinMax() {
     // Nothing to do
   }
-
+  
   /**
    * Record a single instance of a split
    * @param time time that the split took
diff --git a/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java b/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java
deleted file mode 100644
index ef3add97dae1..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.metrics;
-
-import static org.junit.Assert.*;
-
-import java.lang.management.ManagementFactory;
-
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-import org.apache.hadoop.hbase.SmallTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Tests {@link MasterMetrics} and access to it through the
- * {@link MasterStatistics} management bean.
- */
-@Category(SmallTests.class)
-public class TestMasterStatistics {
-
-  @Test
-  public void testMasterStatistics() throws Exception {
-    // No timer updates started here since NullContext is
-    // instantiated by default for HBase:
-    MasterMetrics masterMetrics = new MasterMetrics("foo");
-    try {
-      final MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-      final ObjectName objectName = new ObjectName(
-          "hadoop:name=MasterStatistics,service=Master");
-
-      masterMetrics.doUpdates(null);
-
-      masterMetrics.resetAllMinMax();
-
-      masterMetrics.incrementRequests(10);
-      Thread.sleep(1001);
-
-      masterMetrics.addSnapshot(1L);
-      masterMetrics.addSnapshotClone(2L);
-      masterMetrics.addSnapshotRestore(3L);
-
-      // 3 times added split, average = (5+3+4)/3 = 4
-      masterMetrics.addSplit(4L, 5L);
-      masterMetrics.addSplit(2L, 3L);
-      masterMetrics.addSplit(13L, 4L);
-
-      masterMetrics.doUpdates(null);
-
-      final float f = masterMetrics.getRequests();
-      // f = 10/T, where T >= 1 sec. So, we assert that 0 < f <= 10:
-      if (f <= 0.0f || f > 10.0f) {
-        fail("Unexpected rate value: " + f);
-      }
-      Object attribute = server.getAttribute(objectName, "cluster_requests");
-      float f2 = ((Float) attribute).floatValue();
-      assertEquals("The value obtained through bean server should be equal to the one " +
-          "obtained directly.", f, f2, 1e-4);
-
-      // NB: these 3 metrics are not pushed upon masterMetrics.doUpdates(),
-      // so they always return null:
-      attribute = server.getAttribute(objectName, "snapshotTimeNumOps");
-      assertEquals(Integer.valueOf(0), attribute);
-      attribute = server.getAttribute(objectName, "snapshotRestoreTimeNumOps");
-      assertEquals(Integer.valueOf(0), attribute);
-      attribute = server.getAttribute(objectName, "snapshotCloneTimeNumOps");
-      assertEquals(Integer.valueOf(0), attribute);
-
-      attribute = server.getAttribute(objectName, "splitSizeNumOps");
-      assertEquals(Integer.valueOf(3), attribute);
-      attribute = server.getAttribute(objectName, "splitSizeAvgTime");
-      assertEquals(Long.valueOf(4), attribute);
-    } finally {
-      masterMetrics.shutdown();
-    }
-  }
-
-  @Test
-  public void testHBaseInfoBean() throws Exception {
-    MasterMetrics masterMetrics = new MasterMetrics("foo");
-    try {
-      final MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-      // Test Info bean:
-      final ObjectName objectName2 = new ObjectName(
-          "hadoop:name=Info,service=HBase");
-      Object attribute;
-      attribute = server.getAttribute(objectName2, "revision");
-      assertNotNull(attribute);
-      attribute = server.getAttribute(objectName2, "version");
-      assertNotNull(attribute);
-      attribute = server.getAttribute(objectName2, "hdfsUrl");
-      assertNotNull(attribute);
-      attribute = server.getAttribute(objectName2, "user");
-      assertNotNull(attribute);
-    } finally {
-      masterMetrics.shutdown();
-    }
-  }
-}

From a3ccde5f229061268153359adcc585e9f4fefc73 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 25 Oct 2013 01:45:19 +0000
Subject: [PATCH 1193/1540] HBASE-7600 TestAdmin.testCreateBadTables is failing
 occasionally (Liu Shaohui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1535606 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/zookeeper/ZKTable.java   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 6aa502798b1d..cd95e0692102 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -170,7 +170,7 @@ public void setEnablingTable(final String tableName)
   public boolean checkAndSetEnablingTable(final String tableName)
     throws KeeperException {
     synchronized (this.cache) {
-      if (isEnablingTable(tableName)) {
+      if (isEnablingOrEnabledTable(tableName)) {
         return false;
       }
       setTableState(tableName, TableState.ENABLING);
@@ -291,6 +291,12 @@ public boolean isDisablingOrDisabledTable(final String tableName) {
     }
   }
 
+  public boolean isEnablingOrEnabledTable(final String tableName) {
+    synchronized (this.cache) {
+      return isEnablingTable(tableName) || isEnabledTable(tableName);
+    }
+  }
+
   public boolean isEnabledOrDisablingTable(final String tableName) {
     synchronized (this.cache) {
       return isEnabledTable(tableName) || isDisablingTable(tableName);

From e5f5751bff1218886669fa2809625da9afb09e08 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 25 Oct 2013 16:58:39 +0000
Subject: [PATCH 1194/1540] HBASE-9807 block encoder unnecessarily copies the
 key for each reseek

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1535781 13f79535-47bb-0310-9956-ffa450edef68
---
 .../io/encoding/BufferedDataBlockEncoder.java |  5 ++++
 .../hbase/io/encoding/DataBlockEncoder.java   | 10 +++++++
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  | 26 ++++++++++++++++---
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 1dedbbdc59f3..cef789c902af 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -124,6 +124,11 @@ public BufferedEncodedSeeker(RawComparator<byte[]> comparator) {
       }
     }
 
+    @Override
+    public int compareKey(RawComparator<byte[]> comparator, byte[] key, int offset, int length) {
+      return comparator.compare(key,  offset, length, current.keyBuffer, 0, current.keyLength);
+    }
+
     @Override
     public void setCurrentBuffer(ByteBuffer buffer) {
       currentBuffer = buffer;
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
index b2f2319fe138..bcb3b6cf2eae 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
@@ -156,5 +156,15 @@ public static interface EncodedSeeker {
      */
     public int seekToKeyInBlock(byte[] key, int offset, int length,
         boolean seekBefore);
+
+    /**
+     * Compare the given key against the current key
+     * @param comparator
+     * @param key
+     * @param offset
+     * @param length
+     * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
+     */
+    public int compareKey(RawComparator<byte []> comparator, byte[] key, int offset, int length);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 5725d04a8bf8..09fd912b8896 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.IdLock;
+import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.WritableUtils;
 
 /**
@@ -514,9 +515,7 @@ public int seekTo(byte[] key, int offset, int length) throws IOException {
     public int reseekTo(byte[] key, int offset, int length) throws IOException {
       int compared;
       if (isSeeked()) {
-        ByteBuffer bb = getKey();
-        compared = reader.getComparator().compare(key, offset,
-            length, bb.array(), bb.arrayOffset(), bb.limit());
+        compared = compareKey(reader.getComparator(), key, offset, length);
         if (compared < 1) {
           // If the required key is less than or equal to current key, then
           // don't do anything.
@@ -608,6 +607,16 @@ protected HFileBlock readNextDataBlock() throws IOException {
 
       return curBlock;
     }
+    /**
+     * Compare the given key against the current key
+     * @param comparator
+     * @param key
+     * @param offset
+     * @param length
+     * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
+     */
+    public abstract int compareKey(RawComparator<byte[]> comparator, byte[] key, int offset,
+        int length);
   }
 
   /**
@@ -646,6 +655,12 @@ public ByteBuffer getKey() {
               + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
     }
 
+    @Override
+    public int compareKey(RawComparator<byte []> comparator, byte[] key, int offset, int length) {
+      return comparator.compare(key, offset, length, blockBuffer.array(), blockBuffer.arrayOffset()
+          + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
+      }
+
     @Override
     public ByteBuffer getValue() {
       assertSeeked();
@@ -1041,6 +1056,11 @@ public ByteBuffer getKey() {
       return seeker.getKeyDeepCopy();
     }
 
+    @Override
+    public int compareKey(RawComparator<byte []> comparator, byte[] key, int offset, int length) {
+      return seeker.compareKey(comparator, key, offset, length);
+    }
+
     @Override
     public ByteBuffer getValue() {
       assertValidSeek();

From 74d9a33fddf1b703ef84365c007f5ae830ec4c95 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Sat, 26 Oct 2013 00:42:51 +0000
Subject: [PATCH 1195/1540] HBASE-9715. Backport -in_memory option support for
 LoadTestTool

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1535903 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/LoadTestTool.java   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index 1cb656df6a04..dfefe9de5bf5 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -24,7 +24,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.ParseException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -86,6 +85,10 @@ public class LoadTestTool extends AbstractHBaseTool {
         + "compression) to use for data blocks in the test column family, "
         + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
 
+  public static final String OPT_INMEMORY = "in_memory";
+  public static final String OPT_USAGE_IN_MEMORY = "Tries to keep the HFiles of the CF " +
+      "inmemory as far as possible.  Not guaranteed that reads are always served from inmemory";
+
   private static final String OPT_BLOOM = "bloom";
   private static final String OPT_COMPRESSION = "compression";
   public static final String OPT_DATA_BLOCK_ENCODING =
@@ -126,6 +129,7 @@ public class LoadTestTool extends AbstractHBaseTool {
   protected boolean encodeInCacheOnly;
   protected Compression.Algorithm compressAlgo;
   protected StoreFile.BloomType bloomType;
+  private boolean inMemoryCF;
 
   // Writer options
   protected int numWriterThreads = DEFAULT_NUM_THREADS;
@@ -189,6 +193,9 @@ protected void applyColumnFamilyOptions(byte[] tableName,
         columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
         columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
       }
+      if (inMemoryCF) {
+        columnDesc.setInMemory(inMemoryCF);
+      }
       if (isNewCf) {
         admin.addColumn(tableName, columnDesc);
       } else {
@@ -220,6 +227,7 @@ protected void addOptions() {
     addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " +
         "separate puts for every column in a row");
     addOptNoArg(OPT_ENCODE_IN_CACHE_ONLY, OPT_ENCODE_IN_CACHE_ONLY_USAGE);
+    addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY);
 
     addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
     addOptWithArg(OPT_START_KEY, "The first key to read/write " +
@@ -340,6 +348,8 @@ protected void parseColumnFamilyOptions(CommandLine cmd) {
     String bloomStr = cmd.getOptionValue(OPT_BLOOM);
     bloomType = bloomStr == null ? null :
         StoreFile.BloomType.valueOf(bloomStr);
+
+    inMemoryCF = cmd.hasOption(OPT_INMEMORY);
   }
 
   public void initTestTable() throws IOException {

From eebfdfe271507dbd280e17d198e27f24f7962299 Mon Sep 17 00:00:00 2001
From: Devaraj Das <ddas@apache.org>
Date: Mon, 28 Oct 2013 20:13:20 +0000
Subject: [PATCH 1196/1540] HBASE-8553. improve unit-test coverage of package
 org.apache.hadoop.hbase.mapreduce.hadoopbackport. Contributed by Ivan A.
 Veselovsky.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536502 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoopbackport/InputSampler.java          |  26 ++-
 .../hadoopbackport/TestInputSampler.java      | 165 ++++++++++++++
 .../hadoopbackport/TestInputSamplerTool.java  | 207 ++++++++++++++++++
 .../TestTotalOrderPartitioner.java            | 205 +++++++++++++++++
 4 files changed, 592 insertions(+), 11 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSampler.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSamplerTool.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestTotalOrderPartitioner.java

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
index 89c342837c7c..a556709a50d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
@@ -79,7 +79,7 @@ public InputSampler(Configuration conf) {
   }
 
   /**
-   * Interface to sample using an 
+   * Interface to sample using an
    * {@link org.apache.hadoop.mapreduce.InputFormat}.
    */
   public interface Sampler<K,V> {
@@ -87,7 +87,7 @@ public interface Sampler<K,V> {
      * For a given job, collect and return a subset of the keys from the
      * input data.
      */
-    K[] getSample(InputFormat<K,V> inf, Job job) 
+    K[] getSample(InputFormat<K,V> inf, Job job)
     throws IOException, InterruptedException;
   }
 
@@ -125,7 +125,8 @@ public SplitSampler(int numSamples, int maxSplitsSampled) {
      * From each split sampled, take the first numSamples / numSplits records.
      */
     @SuppressWarnings("unchecked") // ArrayList::toArray doesn't preserve type
-    public K[] getSample(InputFormat<K,V> inf, Job job) 
+    @Override
+    public K[] getSample(InputFormat<K,V> inf, Job job)
         throws IOException, InterruptedException {
       List<InputSplit> splits = inf.getSplits(job);
       ArrayList<K> samples = new ArrayList<K>(numSamples);
@@ -160,7 +161,7 @@ public K[] getSample(InputFormat<K,V> inf, Job job)
    * here when we should be using native hadoop TotalOrderPartitioner).
    * @param job
    * @return Context
-   * @throws IOException 
+   * @throws IOException
    */
   public static TaskAttemptContext getTaskAttemptContext(final Job job)
   throws IOException {
@@ -218,7 +219,8 @@ public RandomSampler(double freq, int numSamples, int maxSplitsSampled) {
      * the quota of keys from that split is satisfied.
      */
     @SuppressWarnings("unchecked") // ArrayList::toArray doesn't preserve type
-    public K[] getSample(InputFormat<K,V> inf, Job job) 
+    @Override
+    public K[] getSample(InputFormat<K,V> inf, Job job)
         throws IOException, InterruptedException {
       List<InputSplit> splits = inf.getSplits(job);
       ArrayList<K> samples = new ArrayList<K>(numSamples);
@@ -302,7 +304,8 @@ public IntervalSampler(double freq, int maxSplitsSampled) {
      * frequency.
      */
     @SuppressWarnings("unchecked") // ArrayList::toArray doesn't preserve type
-    public K[] getSample(InputFormat<K,V> inf, Job job) 
+    @Override
+    public K[] getSample(InputFormat<K,V> inf, Job job)
         throws IOException, InterruptedException {
       List<InputSplit> splits = inf.getSplits(job);
       ArrayList<K> samples = new ArrayList<K>();
@@ -335,10 +338,10 @@ public K[] getSample(InputFormat<K,V> inf, Job job)
    * returned from {@link TotalOrderPartitioner#getPartitionFile}.
    */
   @SuppressWarnings("unchecked") // getInputFormat, getOutputKeyComparator
-  public static <K,V> void writePartitionFile(Job job, Sampler<K,V> sampler) 
+  public static <K,V> void writePartitionFile(Job job, Sampler<K,V> sampler)
       throws IOException, ClassNotFoundException, InterruptedException {
     Configuration conf = job.getConfiguration();
-    final InputFormat inf = 
+    final InputFormat inf =
         ReflectionUtils.newInstance(job.getInputFormatClass(), conf);
     int numPartitions = job.getNumReduceTasks();
     K[] samples = sampler.getSample(inf, job);
@@ -351,7 +354,7 @@ public static <K,V> void writePartitionFile(Job job, Sampler<K,V> sampler)
     if (fs.exists(dst)) {
       fs.delete(dst, false);
     }
-    SequenceFile.Writer writer = SequenceFile.createWriter(fs, 
+    SequenceFile.Writer writer = SequenceFile.createWriter(fs,
       conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
     NullWritable nullValue = NullWritable.get();
     float stepSize = samples.length / (float) numPartitions;
@@ -371,6 +374,7 @@ public static <K,V> void writePartitionFile(Job job, Sampler<K,V> sampler)
    * Driver for InputSampler from the command line.
    * Configures a JobConf instance and calls {@link #writePartitionFile}.
    */
+  @Override
   public int run(String[] args) throws Exception {
     Job job = new Job(getConf());
     ArrayList<String> otherArgs = new ArrayList<String>();
@@ -426,8 +430,8 @@ public int run(String[] args) throws Exception {
     }
 
     Path outf = new Path(otherArgs.remove(otherArgs.size() - 1));
-    TotalOrderPartitioner.setPartitionFile(getConf(), outf);
-    for (String s : otherArgs) {
+    TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), outf);
+    for (String s: otherArgs) {
       FileInputFormat.addInputPath(job, new Path(s));
     }
     InputSampler.<K,V>writePartitionFile(job, sampler);
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSampler.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSampler.java
new file mode 100644
index 000000000000..65b6702efd65
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSampler.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * The test is ported from Hadoop branch-0.23 with very small changes.
+ */
+@Category(SmallTests.class)
+public class TestInputSampler {
+
+  static class SequentialSplit extends InputSplit {
+    private int i;
+    SequentialSplit(int i) {
+      this.i = i;
+    }
+    @Override
+    public long getLength() { return 0; }
+    @Override
+    public String[] getLocations() { return new String[0]; }
+    public int getInit() { return i; }
+  }
+
+  static class TestInputSamplerIF
+      extends InputFormat<IntWritable,NullWritable> {
+
+    final int maxDepth;
+    final ArrayList<InputSplit> splits = new ArrayList<InputSplit>();
+
+    TestInputSamplerIF(int maxDepth, int numSplits, int... splitInit) {
+      this.maxDepth = maxDepth;
+      assert splitInit.length == numSplits;
+      for (int i = 0; i < numSplits; ++i) {
+        splits.add(new SequentialSplit(splitInit[i]));
+      }
+    }
+
+    @Override
+    public List<InputSplit> getSplits(JobContext context)
+        throws IOException, InterruptedException {
+      return splits;
+    }
+
+    @Override
+    public RecordReader<IntWritable,NullWritable> createRecordReader(
+        final InputSplit split, TaskAttemptContext context)
+        throws IOException, InterruptedException {
+      return new RecordReader<IntWritable,NullWritable>() {
+        private int maxVal;
+        private final IntWritable i = new IntWritable();
+        @Override
+        public void initialize(InputSplit split, TaskAttemptContext context)
+            throws IOException, InterruptedException {
+          i.set(((SequentialSplit)split).getInit() - 1);
+          maxVal = i.get() + maxDepth + 1;
+        }
+        @Override
+        public boolean nextKeyValue() {
+          i.set(i.get() + 1);
+          return i.get() < maxVal;
+        }
+        @Override
+        public IntWritable getCurrentKey() { return i; }
+        @Override
+        public NullWritable getCurrentValue() { return NullWritable.get(); }
+        @Override
+        public float getProgress() { return 1.0f; }
+        @Override
+        public void close() { }
+      };
+    }
+
+  }
+
+  /**
+   * Verify SplitSampler contract, that an equal number of records are taken
+   * from the first splits.
+   */
+  @Test
+  @SuppressWarnings("unchecked") // IntWritable comparator not typesafe
+  public void testSplitSampler() throws Exception {
+    final int TOT_SPLITS = 15;
+    final int NUM_SPLITS = 5;
+    final int STEP_SAMPLE = 5;
+    final int NUM_SAMPLES = NUM_SPLITS * STEP_SAMPLE;
+    InputSampler.Sampler<IntWritable,NullWritable> sampler =
+      new InputSampler.SplitSampler<IntWritable,NullWritable>(
+          NUM_SAMPLES, NUM_SPLITS);
+    int inits[] = new int[TOT_SPLITS];
+    for (int i = 0; i < TOT_SPLITS; ++i) {
+      inits[i] = i * STEP_SAMPLE;
+    }
+    Job ignored = new Job();//Job.getInstance();
+    Object[] samples = sampler.getSample(
+        new TestInputSamplerIF(100000, TOT_SPLITS, inits), ignored);
+    assertEquals(NUM_SAMPLES, samples.length);
+    Arrays.sort(samples, new IntWritable.Comparator());
+    for (int i = 0; i < NUM_SAMPLES; ++i) {
+      assertEquals(i, ((IntWritable)samples[i]).get());
+    }
+  }
+
+  /**
+   * Verify IntervalSampler contract, that samples are taken at regular
+   * intervals from the given splits.
+   */
+  @Test
+  @SuppressWarnings("unchecked") // IntWritable comparator not typesafe
+  public void testIntervalSampler() throws Exception {
+    final int TOT_SPLITS = 16;
+    final int PER_SPLIT_SAMPLE = 4;
+    final int NUM_SAMPLES = TOT_SPLITS * PER_SPLIT_SAMPLE;
+    final double FREQ = 1.0 / TOT_SPLITS;
+    InputSampler.Sampler<IntWritable,NullWritable> sampler =
+      new InputSampler.IntervalSampler<IntWritable,NullWritable>(
+          FREQ, NUM_SAMPLES);
+    int inits[] = new int[TOT_SPLITS];
+    for (int i = 0; i < TOT_SPLITS; ++i) {
+      inits[i] = i;
+    }
+    Job ignored = new Job();
+    Object[] samples = sampler.getSample(new TestInputSamplerIF(
+          NUM_SAMPLES, TOT_SPLITS, inits), ignored);
+    assertEquals(NUM_SAMPLES, samples.length);
+    Arrays.sort(samples, new IntWritable.Comparator());
+    for (int i = 0; i < NUM_SAMPLES; ++i) {
+      assertEquals(i, ((IntWritable)samples[i]).get());
+    }
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSamplerTool.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSamplerTool.java
new file mode 100644
index 000000000000..f90c20447c48
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestInputSamplerTool.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Tool;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests {@link InputSampler} as a {@link Tool}.
+ */
+@Category(SmallTests.class)
+public class TestInputSamplerTool {
+
+  private static final int NUM_REDUCES = 4;
+
+  private static final String input1Str =
+     "2\n"
+    +"...5\n"
+    +"......8\n";
+  private static final String input2Str =
+     "2\n"
+    +".3\n"
+    +"..4\n"
+    +"...5\n"
+    +"....6\n"
+    +".....7\n"
+    +"......8\n"
+    +".......9\n";
+
+  private static File tempDir;
+  private static String input1, input2, output;
+
+  @BeforeClass
+  public static void beforeClass() throws IOException {
+    tempDir = FileUtil.createLocalTempFile(
+      new File(FileUtils.getTempDirectory(), TestInputSamplerTool.class.getName() + "-tmp-"),
+      "", false);
+    tempDir.delete();
+    tempDir.mkdirs();
+    assertTrue(tempDir.exists());
+    assertTrue(tempDir.isDirectory());
+    // define files:
+    input1 = tempDir.getAbsolutePath() + "/input1";
+    input2 = tempDir.getAbsolutePath() + "/input2";
+    output = tempDir.getAbsolutePath() + "/output";
+    // create 2 input files:
+    IOUtils.copy(new StringReader(input1Str), new FileOutputStream(input1));
+    IOUtils.copy(new StringReader(input2Str), new FileOutputStream(input2));
+  }
+
+  @AfterClass
+  public static void afterClass() throws IOException {
+    final File td = tempDir;
+    if (td != null && td.exists()) {
+      FileUtil.fullyDelete(tempDir);
+    }
+  }
+
+  @Test
+  public void testIncorrectParameters() throws Exception {
+    Tool tool = new InputSampler<Object,Object>(new Configuration());
+
+    int result = tool.run(new String[] { "-r" });
+    assertTrue(result != 0);
+
+    result = tool.run(new String[] { "-r", "not-a-number" });
+    assertTrue(result != 0);
+
+    // more than one reducer is required:
+    result = tool.run(new String[] { "-r", "1" });
+    assertTrue(result != 0);
+
+    try {
+      result = tool.run(new String[] { "-inFormat", "java.lang.Object" });
+      fail("ClassCastException expected");
+    } catch (ClassCastException cce) {
+      // expected
+    }
+
+    try {
+      result = tool.run(new String[] { "-keyClass", "java.lang.Object" });
+      fail("ClassCastException expected");
+    } catch (ClassCastException cce) {
+      // expected
+    }
+
+    result = tool.run(new String[] { "-splitSample", "1", });
+    assertTrue(result != 0);
+
+    result = tool.run(new String[] { "-splitRandom", "1.0", "2", "xxx" });
+    assertTrue(result != 0);
+
+    result = tool.run(new String[] { "-splitInterval", "yyy", "5" });
+    assertTrue(result != 0);
+
+    // not enough subsequent arguments:
+    result = tool.run(new String[] { "-r", "2", "-splitInterval", "11.0f", "0", "input" });
+    assertTrue(result != 0);
+  }
+
+  @Test
+  public void testSplitSample() throws Exception {
+    Tool tool = new InputSampler<Object,Object>(new Configuration());
+    int result = tool.run(new String[] { "-r", Integer.toString(NUM_REDUCES),
+        "-splitSample", "10", "100",
+        input1, input2, output });
+    assertEquals(0, result);
+
+    Object[] partitions = readPartitions(output);
+    assertArrayEquals(
+        new LongWritable[] { new LongWritable(2L), new LongWritable(7L), new LongWritable(20L),},
+        partitions);
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testSplitRamdom() throws Exception {
+    Tool tool = new InputSampler<Object,Object>(new Configuration());
+    int result = tool.run(new String[] { "-r", Integer.toString(NUM_REDUCES),
+        // Use 0.999 probability to reduce the flakiness of the test because
+        // the test will fail if the number of samples is less than (number of reduces + 1).
+        "-splitRandom", "0.999f", "20", "100",
+        input1, input2, output });
+    assertEquals(0, result);
+    Object[] partitions = readPartitions(output);
+    // must be 3 split points since NUM_REDUCES = 4:
+    assertEquals(3, partitions.length);
+    // check that the partition array is sorted:
+    Object[] sortedPartitions = Arrays.copyOf(partitions, partitions.length);
+    Arrays.sort(sortedPartitions, new LongWritable.Comparator());
+    assertArrayEquals(sortedPartitions, partitions);
+  }
+
+  @Test
+  public void testSplitInterval() throws Exception {
+    Tool tool = new InputSampler<Object,Object>(new Configuration());
+    int result = tool.run(new String[] { "-r", Integer.toString(NUM_REDUCES),
+        "-splitInterval", "0.5f", "0",
+        input1, input2, output });
+    assertEquals(0, result);
+    Object[] partitions = readPartitions(output);
+    assertArrayEquals(new LongWritable[] { new LongWritable(7L), new LongWritable(9L),
+      new LongWritable(35L),}, partitions);
+  }
+
+  private Object[] readPartitions(String filePath) throws Exception {
+    Configuration conf = new Configuration();
+    TotalOrderPartitioner.setPartitionFile(conf, new Path(filePath));
+    Object[] partitions = readPartitions(FileSystem.getLocal(conf), new Path(filePath),
+      LongWritable.class, conf);
+    return partitions;
+  }
+
+  private Object[] readPartitions(FileSystem fs, Path p, Class<?> keyClass,
+      Configuration conf) throws IOException {
+    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
+    ArrayList<Object> parts = new ArrayList<Object>();
+    Writable key = (Writable)ReflectionUtils.newInstance(keyClass, conf);
+    NullWritable value = NullWritable.get();
+    while (reader.next(key, value)) {
+      parts.add(key);
+      key = (Writable)ReflectionUtils.newInstance(keyClass, conf);
+    }
+    reader.close();
+    return parts.toArray((Object[])Array.newInstance(keyClass, parts.size()));
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestTotalOrderPartitioner.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestTotalOrderPartitioner.java
new file mode 100644
index 000000000000..7cb1f99d65c5
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestTotalOrderPartitioner.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.experimental.categories.Category;
+
+/**
+ * The test is ported from Hadoop branch-0.23 with very small changes.
+ */
+@Category(SmallTests.class)
+public class TestTotalOrderPartitioner extends TestCase {
+
+  private static final Text[] splitStrings = new Text[] {
+    // -inf            // 0
+    new Text("aabbb"), // 1
+    new Text("babbb"), // 2
+    new Text("daddd"), // 3
+    new Text("dddee"), // 4
+    new Text("ddhee"), // 5
+    new Text("dingo"), // 6
+    new Text("hijjj"), // 7
+    new Text("n"),     // 8
+    new Text("yak"),   // 9
+  };
+
+  static class Check<T> {
+    T data;
+    int part;
+    Check(T data, int part) {
+      this.data = data;
+      this.part = part;
+    }
+  }
+
+  private static final ArrayList<Check<Text>> testStrings =
+    new ArrayList<Check<Text>>();
+  static {
+    testStrings.add(new Check<Text>(new Text("aaaaa"), 0));
+    testStrings.add(new Check<Text>(new Text("aaabb"), 0));
+    testStrings.add(new Check<Text>(new Text("aabbb"), 1));
+    testStrings.add(new Check<Text>(new Text("aaaaa"), 0));
+    testStrings.add(new Check<Text>(new Text("babbb"), 2));
+    testStrings.add(new Check<Text>(new Text("baabb"), 1));
+    testStrings.add(new Check<Text>(new Text("yai"), 8));
+    testStrings.add(new Check<Text>(new Text("yak"), 9));
+    testStrings.add(new Check<Text>(new Text("z"), 9));
+    testStrings.add(new Check<Text>(new Text("ddngo"), 5));
+    testStrings.add(new Check<Text>(new Text("hi"), 6));
+  };
+
+  private static <T extends WritableComparable<?>> Path writePartitionFile(
+      String testname, Configuration conf, T[] splits) throws IOException {
+    final FileSystem fs = FileSystem.getLocal(conf);
+    final Path testdir = new Path(System.getProperty("test.build.data", "/tmp")
+                                 ).makeQualified(fs);
+    Path p = new Path(testdir, testname + "/_partition.lst");
+    TotalOrderPartitioner.setPartitionFile(conf, p);
+    conf.setInt("mapreduce.job.reduces", splits.length + 1);
+    SequenceFile.Writer w = null;
+    try {
+      w = SequenceFile.createWriter(fs, conf, p,
+          splits[0].getClass(), NullWritable.class,
+          SequenceFile.CompressionType.NONE);
+      for (int i = 0; i < splits.length; ++i) {
+        w.append(splits[i], NullWritable.get());
+      }
+    } finally {
+      if (null != w)
+        w.close();
+    }
+    return p;
+  }
+
+  public void testTotalOrderMemCmp() throws Exception {
+    TotalOrderPartitioner<Text,NullWritable> partitioner =
+      new TotalOrderPartitioner<Text,NullWritable>();
+
+    // Need to use old JobConf-based variant here:
+    JobConf conf = new JobConf();
+    conf.setMapOutputKeyClass(Text.class);
+    conf.setNumReduceTasks(splitStrings.length + 1);
+
+    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
+        "totalordermemcmp", conf, splitStrings);
+    try {
+      partitioner.setConf(conf);
+      NullWritable nw = NullWritable.get();
+      for (Check<Text> chk : testStrings) {
+        assertEquals(chk.data.toString(), chk.part,
+            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
+      }
+    } finally {
+      p.getFileSystem(conf).delete(p, true);
+    }
+  }
+
+  public void testTotalOrderBinarySearch() throws Exception {
+    TotalOrderPartitioner<Text,NullWritable> partitioner =
+      new TotalOrderPartitioner<Text,NullWritable>();
+    JobConf conf = new JobConf();
+    conf.setMapOutputKeyClass(Text.class);
+    conf.setNumReduceTasks(splitStrings.length + 1);
+
+    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
+        "totalorderbinarysearch", conf, splitStrings);
+    conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
+    try {
+      partitioner.setConf(conf);
+      NullWritable nw = NullWritable.get();
+      for (Check<Text> chk : testStrings) {
+        assertEquals(chk.data.toString(), chk.part,
+            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
+      }
+    } finally {
+      p.getFileSystem(conf).delete(p, true);
+    }
+  }
+
+  public static class ReverseStringComparator implements RawComparator<Text> {
+    @Override
+    public int compare(Text a, Text b) {
+      return -a.compareTo(b);
+    }
+    @Override
+    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+      int n1 = WritableUtils.decodeVIntSize(b1[s1]);
+      int n2 = WritableUtils.decodeVIntSize(b2[s2]);
+      return -1 * WritableComparator.compareBytes(b1, s1+n1, l1-n1,
+                                                  b2, s2+n2, l2-n2);
+    }
+  }
+
+  public void testTotalOrderCustomComparator() throws Exception {
+    TotalOrderPartitioner<Text,NullWritable> partitioner =
+      new TotalOrderPartitioner<Text,NullWritable>();
+
+    final JobConf conf = new JobConf();
+    conf.setMapOutputKeyClass(Text.class);
+    conf.setNumReduceTasks(splitStrings.length + 1);
+
+    Text[] revSplitStrings = Arrays.copyOf(splitStrings, splitStrings.length);
+    Arrays.sort(revSplitStrings, new ReverseStringComparator());
+    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
+        "totalordercustomcomparator", conf, revSplitStrings);
+    conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
+    conf.setOutputKeyComparatorClass(ReverseStringComparator.class);
+
+    ArrayList<Check<Text>> revCheck = new ArrayList<Check<Text>>();
+    revCheck.add(new Check<Text>(new Text("aaaaa"), 9));
+    revCheck.add(new Check<Text>(new Text("aaabb"), 9));
+    revCheck.add(new Check<Text>(new Text("aabbb"), 9));
+    revCheck.add(new Check<Text>(new Text("aaaaa"), 9));
+    revCheck.add(new Check<Text>(new Text("babbb"), 8));
+    revCheck.add(new Check<Text>(new Text("baabb"), 8));
+    revCheck.add(new Check<Text>(new Text("yai"), 1));
+    revCheck.add(new Check<Text>(new Text("yak"), 1));
+    revCheck.add(new Check<Text>(new Text("z"), 0));
+    revCheck.add(new Check<Text>(new Text("ddngo"), 4));
+    revCheck.add(new Check<Text>(new Text("hi"), 3));
+    try {
+      partitioner.setConf(conf);
+      NullWritable nw = NullWritable.get();
+      for (Check<Text> chk : revCheck) {
+        assertEquals(chk.data.toString(), chk.part,
+            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
+      }
+    } finally {
+      p.getFileSystem(conf).delete(p, true);
+    }
+  }
+}

From 6f387248d8fbea17c4d41fa253e67606742d4008 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 28 Oct 2013 20:41:21 +0000
Subject: [PATCH 1197/1540] HBASE-9851 TestHBaseFsck.testQuarantineMissingHFile
 is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536510 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/TestHBaseFsck.java  | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 84d8db89f160..20a413332f8a 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -356,7 +356,9 @@ void deleteTable(String tablename) throws IOException {
     HBaseAdmin admin = new HBaseAdmin(conf);
     admin.getConnection().clearRegionCache();
     byte[] tbytes = Bytes.toBytes(tablename);
-    admin.disableTableAsync(tbytes);
+    if (admin.isTableEnabled(tbytes)) {
+      admin.disableTableAsync(tbytes);
+    }
     while (!admin.isTableDisabled(tbytes)) {
       try {
         Thread.sleep(250);
@@ -1616,7 +1618,7 @@ Path getFlushedHFile(FileSystem fs, String table) throws IOException {
   /**
    * This creates a table and then corrupts an hfile.  Hbck should quarantine the file.
    */
-  @Test(timeout=120000)
+  @Test(timeout=180000)
   public void testQuarantineCorruptHFile() throws Exception {
     String table = name.getMethodName();
     try {
@@ -1697,7 +1699,7 @@ private void doQuarantineTest(String table, HBaseFsck hbck, int check, int corru
    * This creates a table and simulates the race situation where a concurrent compaction or split
    * has removed an hfile after the corruption checker learned about it.
    */
-  @Test(timeout=120000)
+  @Test(timeout=180000)
   public void testQuarantineMissingHFile() throws Exception {
     String table = name.getMethodName();
     ExecutorService exec = new ScheduledThreadPoolExecutor(10);
@@ -1724,7 +1726,7 @@ protected void checkHFile(Path p) throws IOException {
    * This creates a table and simulates the race situation where a concurrent compaction or split
    * has removed an colfam dir before the corruption checker got to it.
    */
-  @Test(timeout=120000)
+  @Test(timeout=180000)
   public void testQuarantineMissingFamdir() throws Exception {
     String table = name.getMethodName();
     ExecutorService exec = new ScheduledThreadPoolExecutor(10);
@@ -1751,7 +1753,7 @@ protected void checkColFamDir(Path p) throws IOException {
    * This creates a table and simulates the race situation where a concurrent compaction or split
    * has removed a region dir before the corruption checker got to it.
    */
-  @Test(timeout=120000)
+  @Test(timeout=180000)
   public void testQuarantineMissingRegionDir() throws Exception {
     String table = name.getMethodName();
     ExecutorService exec = new ScheduledThreadPoolExecutor(10);

From cd845a6683ad86297f75d8ca9253d57524ba4709 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 29 Oct 2013 00:01:18 +0000
Subject: [PATCH 1198/1540] HBASE-9852 TestRpcMetrics.testCustomMetrics is
 flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536563 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestRpcMetrics.java    | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java
index 0a90f0f48cae..38e5b3413424 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java
@@ -20,10 +20,11 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,6 +32,7 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.ipc.HBaseRpcMetrics;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.MetricsContext;
 import org.apache.hadoop.metrics.MetricsUtil;
@@ -41,8 +43,6 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import static org.junit.Assert.*;
-
 @Category(MediumTests.class)
 public class TestRpcMetrics {
   /**
@@ -123,12 +123,15 @@ public void testCustomMetrics() throws Exception {
     TestRegionServer rs = new TestRegionServer(TEST_UTIL.getConfiguration());
     rs.incTest(5);
 
-    // wait for metrics context update
-    Thread.sleep(1000);
-
     String metricName = HBaseRpcMetrics.getMetricName(TestMetrics.class, "test");
-    assertTrue("Metric should have set incremented for "+metricName,
-        wasSet(metricName + "_num_ops"));
+    long start = EnvironmentEdgeManager.currentTimeMillis();
+    while (!wasSet(metricName + "_num_ops")) {
+      if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
+        assertTrue("Metric should have set incremented for "+metricName,
+          wasSet(metricName + "_num_ops"));
+      }
+      Thread.sleep(200);
+    }
   }
 
   public boolean wasSet(String name) {

From fad52d51bce21e6f260861d5c8e02e50dfbce7bb Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 29 Oct 2013 05:31:06 +0000
Subject: [PATCH 1199/1540] HBASE-9847 HConnectionImplementation does not
 connect to new active master (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536591 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 20 ++++---------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index f8503c37301f..a7b025b49239 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1972,28 +1972,19 @@ protected void finalize() throws Throwable {
     }
 
     public HTableDescriptor[] listTables() throws IOException {
-      if (this.master == null) {
-        this.master = getMaster();
-      }
-      HTableDescriptor[] htd = master.getHTableDescriptors();
+      HTableDescriptor[] htd = getMaster().getHTableDescriptors();
       return htd;
     }
 
     public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) throws IOException {
       if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
       if (tableNames == null || tableNames.size() == 0) return null;
-      if (this.master == null) {
-        this.master = getMaster();
-      }
-      return master.getHTableDescriptors(tableNames);
+      return getMaster().getHTableDescriptors(tableNames);
     }
 
     @Override
     public String[] getTableNames() throws IOException {
-      if (this.master == null) {
-        this.master = getMaster();
-      }
-      return master.getTableNames();
+      return getMaster().getTableNames();
     }
 
     public HTableDescriptor getHTableDescriptor(final byte[] tableName)
@@ -2007,10 +1998,7 @@ public HTableDescriptor getHTableDescriptor(final byte[] tableName)
       }
       List<String> tableNameList = new ArrayList<String>(1);
       tableNameList.add(Bytes.toString(tableName));
-      if (this.master == null) {
-        this.master = getMaster();
-      }
-      HTableDescriptor[] htds = master.getHTableDescriptors(tableNameList);
+      HTableDescriptor[] htds = getHTableDescriptors(tableNameList);
       if (htds != null && htds.length > 0) {
         return htds[0];
       }

From 3b7e6d123c910ece06ad435873b7433d08be2009 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 29 Oct 2013 05:45:27 +0000
Subject: [PATCH 1200/1540] HBASE-9842 Backport HBASE-9593 and HBASE-8667 to
 0.94 (rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536592 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/HBaseClient.java  |   8 ++
 .../hbase/regionserver/HRegionServer.java     |   7 +-
 .../TestRSKilledWhenInitializing.java         | 135 ++++++++++++++++++
 3 files changed, 148 insertions(+), 2 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 2bdcd1b60fd1..2dce646ea8ef 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -87,6 +87,7 @@ public class HBaseClient {
   protected final boolean tcpKeepAlive; // if T then use keepalives
   protected int pingInterval; // how often sends ping to the server in msecs
   protected int socketTimeout; // socket timeout
+  protected final InetSocketAddress bindAddress; // address to bind to the client socket
   protected FailedServers failedServers;
 
   protected final SocketFactory socketFactory;           // how to create sockets
@@ -386,6 +387,7 @@ protected synchronized void setupConnection() throws IOException {
           this.socket = socketFactory.createSocket();
           this.socket.setTcpNoDelay(tcpNoDelay);
           this.socket.setKeepAlive(tcpKeepAlive);
+          if (bindAddress != null) this.socket.bind(bindAddress);
           // connection time out is 20s
           NetUtils.connect(this.socket, remoteId.getAddress(),
               getSocketTimeout(conf));
@@ -870,6 +872,12 @@ public HBaseClient(Class<? extends Writable> valueClass, Configuration conf,
     this.clusterId = conf.get(HConstants.CLUSTER_ID, "default");
     this.connections = new PoolMap<ConnectionId, Connection>(
         getPoolType(conf), getPoolSize(conf));
+    String hostName = this.conf.get("hbase.regionserver.rpc.client.socket.bind.address");
+    if (hostName != null) {
+      this.bindAddress = new InetSocketAddress(hostName, 0);
+    } else {
+      this.bindAddress = null;
+    }
     this.failedServers = new FailedServers(conf);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 31ba0742f7c3..0e793bc6aef2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -466,6 +466,8 @@ public HRegionServer(Configuration conf)
     this.rpcServer.setQosFunction(new QosFunction());
     this.startcode = System.currentTimeMillis();
 
+    conf.set("hbase.regionserver.rpc.client.socket.bind.address", this.isa.getHostName());
+
     // login the zookeeper client principal (if using security)
     ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
       "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
@@ -738,6 +740,9 @@ public void run() {
     }
 
     try {
+      // Set our ephemeral znode up in zookeeper now we have a name.
+      createMyEphemeralNode();
+   
       // Try and register with the Master; tell it we are here.  Break if
       // server is stopped or the clusterup flag is down or hdfs went wacky.
       while (keepLooping()) {
@@ -1080,8 +1085,6 @@ protected void handleReportForDutyResponse(final MapWritable c)
         this.conf.set("mapred.task.id", "hb_rs_" +
           this.serverNameFromMasterPOV.toString());
       }
-      // Set our ephemeral znode up in zookeeper now we have a name.
-      createMyEphemeralNode();
 
       // Master sent us hbase.rootdir to use. Should be fully qualified
       // path with file system specification included. Set 'fs.defaultFS'
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
new file mode 100644
index 000000000000..cb097d2de20f
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
@@ -0,0 +1,135 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Writable;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests region server termination during startup.
+ */
+@Category(LargeTests.class)
+public class TestRSKilledWhenInitializing {
+  private static boolean masterActive = false;
+  
+  private static AtomicBoolean firstRS = new AtomicBoolean(true);
+
+  /**
+   * Test verifies whether a region server is removing from online servers list in master if it went
+   * down after registering with master.
+   * @throws Exception
+   */
+  @Test(timeout = 180000)
+  public void testRSTermnationAfterRegisteringToMasterBeforeCreatingEphemeralNod() throws Exception {
+
+    final int NUM_MASTERS = 1;
+    final int NUM_RS = 2;
+    firstRS.set(true);
+    // Create config to use for this cluster
+    Configuration conf = HBaseConfiguration.create();
+
+    // Start the cluster
+    final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+    TEST_UTIL.startMiniDFSCluster(3);
+    TEST_UTIL.startMiniZKCluster();
+    TEST_UTIL.createRootDir();
+    final LocalHBaseCluster cluster =
+        new LocalHBaseCluster(conf, NUM_MASTERS, NUM_RS, HMaster.class, MockedRegionServer.class);
+    final MasterThread master = cluster.getMasters().get(0);
+    master.start();
+    try {
+      long startTime = System.currentTimeMillis();
+      while (!master.getMaster().isActiveMaster()) {
+        try {
+          Thread.sleep(100);
+        } catch (InterruptedException ignored) {
+        }
+        if (System.currentTimeMillis() > startTime + 30000) {
+          throw new RuntimeException("Master not active after 30 seconds");
+        }
+      }
+      masterActive = true;
+      cluster.getRegionServers().get(0).start();
+      cluster.getRegionServers().get(1).start();
+      Thread.sleep(10000);
+      List<ServerName> onlineServersList =
+          master.getMaster().getServerManager().getOnlineServersList();
+      while (onlineServersList.size()!=1) {
+        Thread.sleep(100);
+        onlineServersList = master.getMaster().getServerManager().getOnlineServersList();
+      }
+      assertEquals(onlineServersList.size(), 1);
+    } finally {
+      masterActive = false;
+      firstRS.set(true);
+      TEST_UTIL.shutdownMiniZKCluster();
+      TEST_UTIL.cleanupTestDir();
+      TEST_UTIL.shutdownMiniDFSCluster();
+    }
+  }
+
+  public static class MockedRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
+
+    public MockedRegionServer(Configuration conf) throws IOException, InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    protected void handleReportForDutyResponse(final MapWritable c) throws IOException {
+      if (firstRS.getAndSet(false)) {
+        for (Map.Entry<Writable, Writable> e : c.entrySet()) {
+          String key = e.getKey().toString();
+          // The hostname the master sees us as.
+          if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
+            String hostnameFromMasterPOV = e.getValue().toString();
+            assertEquals(super.getRpcServer().getListenerAddress().getHostName(),
+              hostnameFromMasterPOV);
+          }
+        }
+        while (!masterActive) {
+          Threads.sleep(100);
+        }
+        super.kill();
+      } else {
+        super.handleReportForDutyResponse(c);
+      }
+    }
+  }
+}

From 9a6bfd811f3496bbdb95e511eca8426d9633c4e2 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Tue, 29 Oct 2013 23:34:36 +0000
Subject: [PATCH 1201/1540] HBASE-9221: Provide interface for getting a User in
 the client

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536930 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/ipc/SecureClient.java |  14 ++-
 .../hbase/ipc/SecureConnectionHeader.java     |  11 +-
 .../hadoop/hbase/ipc/SecureRpcEngine.java     |   9 +-
 .../apache/hadoop/hbase/ipc/SecureServer.java |  12 +-
 .../security/access/AccessController.java     |  11 +-
 .../access/SecureBulkLoadEndpoint.java        |   5 +-
 .../security/access/TableAuthManager.java     |   4 +-
 .../TestSecureLoadIncrementalHFiles.java      |   4 +-
 ...ureLoadIncrementalHFilesSplitRecovery.java |   5 +-
 .../hbase/client/HConnectionManager.java      |   6 +-
 .../hadoop/hbase/client/UserProvider.java     | 112 ++++++++++++++++++
 .../hadoop/hbase/ipc/WritableRpcEngine.java   |   5 +-
 .../hbase/mapred/TableMapReduceUtil.java      |   6 +-
 .../mapreduce/LoadIncrementalHFiles.java      |  14 +--
 .../hbase/mapreduce/TableMapReduceUtil.java   |   9 +-
 .../apache/hadoop/hbase/master/HMaster.java   |   4 +-
 .../hbase/regionserver/HRegionServer.java     |   4 +-
 .../org/apache/hadoop/hbase/rest/Main.java    |   6 +-
 .../apache/hadoop/hbase/security/User.java    |   4 +-
 .../hadoop/hbase/thrift/ThriftServer.java     |  20 ++--
 .../apache/hadoop/hbase/util/HBaseFsck.java   |   9 +-
 ...SecurityEnabledUserProviderForTesting.java |  41 +++++++
 .../mapreduce/TestLoadIncrementalHFiles.java  |  20 +++-
 ...estLoadIncrementalHFilesSplitRecovery.java |  13 +-
 .../hbase/master/TestMasterStatusServlet.java |   1 +
 25 files changed, 277 insertions(+), 72 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/client/UserProvider.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index e25085504dac..1f670df5dc40 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -21,6 +21,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.HBaseSaslRpcClient;
 import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.AuthMethod;
 import org.apache.hadoop.hbase.security.KerberosInfo;
@@ -102,7 +103,7 @@ public SecureConnection(ConnectionId remoteId) throws IOException {
 
       User ticket = remoteId.getTicket();
       Class<?> protocol = remoteId.getProtocol();
-      this.useSasl = User.isHBaseSecurityEnabled(conf);
+      this.useSasl = userProvider.isHBaseSecurityEnabled();
       if (useSasl && protocol != null) {
         TokenInfo tokenInfo = protocol.getAnnotation(TokenInfo.class);
         if (tokenInfo != null) {
@@ -276,7 +277,7 @@ protected synchronized void setupIOstreams()
             if (authMethod == AuthMethod.KERBEROS) {
               UserGroupInformation ugi = ticket.getUGI();
               if (ugi != null && ugi.getRealUser() != null) {
-                ticket = User.create(ugi.getRealUser());
+                ticket = userProvider.create(ugi.getRealUser());
               }
             }
             boolean continueSasl = false;
@@ -467,6 +468,7 @@ protected synchronized void close() {
   }
 
   private final boolean fallbackAllowed;
+  private UserProvider userProvider;
 
   /**
    * Construct an IPC client whose values are of the given {@link org.apache.hadoop.io.Writable}
@@ -476,7 +478,7 @@ protected synchronized void close() {
    * @param factory socket factory
    */
   public SecureClient(Class<? extends Writable> valueClass, Configuration conf,
-      SocketFactory factory) {
+      SocketFactory factory, UserProvider provider) {
     super(valueClass, conf, factory);
     this.fallbackAllowed =
       conf.getBoolean(IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
@@ -484,6 +486,7 @@ public SecureClient(Class<? extends Writable> valueClass, Configuration conf,
     if (LOG.isDebugEnabled()) {
       LOG.debug("fallbackAllowed=" + this.fallbackAllowed);
     }
+    this.userProvider = provider;
   }
 
   /**
@@ -491,8 +494,9 @@ public SecureClient(Class<? extends Writable> valueClass, Configuration conf,
    * @param valueClass value class
    * @param conf configuration
    */
-  public SecureClient(Class<? extends Writable> valueClass, Configuration conf) {
-    this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
+  public SecureClient(Class<? extends Writable> valueClass, Configuration conf,
+      UserProvider provider) {
+    this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf), provider);
   }
 
   /**
diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java
index 506082151997..cfcaf53f0ac8 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java
@@ -21,9 +21,12 @@
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.AuthMethod;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
 
 /**
@@ -61,14 +64,16 @@ public void readFields(DataInput in) throws IOException {
     if (ugiUsernamePresent) {
       String username = in.readUTF();
       boolean realUserNamePresent = in.readBoolean();
+      Configuration conf = HBaseConfiguration.create();
+      UserProvider provider = UserProvider.instantiate(conf);
       if (realUserNamePresent) {
         String realUserName = in.readUTF();
         UserGroupInformation realUserUgi =
             UserGroupInformation.createRemoteUser(realUserName);
-        user = User.create(
+        user = provider.create(
             UserGroupInformation.createProxyUser(username, realUserUgi));
       } else {
-        user = User.create(UserGroupInformation.createRemoteUser(username));
+        user = provider.create(UserGroupInformation.createRemoteUser(username));
       }
     } else {
       user = null;
diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
index fd505d71b598..5e6e0e727c63 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java
@@ -21,6 +21,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.security.HBasePolicyProvider;
@@ -58,18 +59,20 @@ public class SecureRpcEngine implements RpcEngine {
 
   private Configuration conf;
   private SecureClient client;
+  private UserProvider provider;
 
   @Override
   public void setConf(Configuration config) {
     this.conf = config;
-    if (User.isHBaseSecurityEnabled(conf)) {
+    this.provider = UserProvider.instantiate(config);
+    if (provider.isHBaseSecurityEnabled()) {
       HBaseSaslRpcServer.init(conf);
     }
     // check for an already created client
     if (this.client != null) {
       this.client.stop();
     }
-    this.client = new SecureClient(HbaseObjectWritable.class, conf);
+    this.client = new SecureClient(HbaseObjectWritable.class, conf, provider);
   }
 
   @Override
@@ -136,7 +139,7 @@ public <T extends VersionedProtocol> T getProxy(
     T proxy =
         (T) Proxy.newProxyInstance(
             protocol.getClassLoader(), new Class[] { protocol },
-            new Invoker(this.client, protocol, addr, User.getCurrent(),
+          new Invoker(this.client, protocol, addr, provider.getCurrent(),
                 HBaseRPC.getRpcTimeout(rpcTimeout)));
     /*
      * TODO: checking protocol version only needs to be done once when we setup a new
diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index 848b94abf0ad..e6301250946a 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -22,6 +22,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.io.WritableWithSize;
 import org.apache.hadoop.hbase.security.HBaseSaslRpcServer;
@@ -97,6 +98,7 @@ public abstract class SecureServer extends HBaseServer {
 
   protected SecretManager<TokenIdentifier> secretManager;
   protected ServiceAuthorizationManager authManager;
+  private UserProvider userProvider;
 
   protected class SecureCall extends HBaseServer.Call {
     public SecureCall(int id, Writable param, Connection connection,
@@ -252,9 +254,9 @@ private User getAuthorizedUgi(String authorizedId)
               "Can't retrieve username from tokenIdentifier.");
         }
         ugi.addTokenIdentifier(tokenId);
-        return User.create(ugi);
+        return userProvider.create(ugi);
       } else {
-        return User.create(UserGroupInformation.createRemoteUser(authorizedId));
+        return userProvider.create(UserGroupInformation.createRemoteUser(authorizedId));
       }
     }
 
@@ -540,7 +542,8 @@ private void processHeader(byte[] buf) throws IOException {
             // for simple auth or kerberos auth
             // The user is the real user. Now we create a proxy user
             UserGroupInformation realUser = ticket.getUGI();
-            ticket = User.create(
+            ticket =
+                userProvider.create(
                 UserGroupInformation.createProxyUser(protocolUser.getName(),
                     realUser));
             // Now the user is a proxy user, set Authentication method Proxy.
@@ -690,7 +693,8 @@ protected SecureServer(String bindAddress, int port,
         conf, serverName, highPriorityLevel);
     this.authorize =
       conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false);
-    this.isSecurityEnabled = User.isHBaseSecurityEnabled(this.conf);
+    this.userProvider = UserProvider.instantiate(this.conf);
+    this.isSecurityEnabled = userProvider.isHBaseSecurityEnabled();
 
     if (isSecurityEnabled) {
       HBaseSaslRpcServer.init(conf);
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index da1685a95ff3..6f56edcf63c0 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
@@ -202,9 +203,10 @@ public static AuthResult deny(String request, String reason, User user,
   private Map<InternalScanner,String> scannerOwners =
       new MapMaker().weakKeys().makeMap();
 
+  private UserProvider userProvider;
+
   void initialize(RegionCoprocessorEnvironment e) throws IOException {
     final HRegion region = e.getRegion();
-
     Map<byte[],ListMultimap<String,TablePermission>> tables =
         AccessControlLists.loadAll(region);
     // For each table, write out the table's permissions to the respective
@@ -377,7 +379,7 @@ private User getActiveUser() throws IOException {
     User user = RequestContext.getRequestUser();
     if (!RequestContext.isInRequestContext()) {
       // for non-rpc handling, fallback to system user
-      user = User.getCurrent();
+      user = userProvider.getCurrent();
     }
 
     return user;
@@ -543,6 +545,9 @@ public void start(CoprocessorEnvironment env) throws IOException {
       zk = regionEnv.getRegionServerServices().getZooKeeper();
     }
 
+    // set the user provider
+    this.userProvider = UserProvider.instantiate(env.getConfiguration());
+
     // If zk is null or IOException while obtaining auth manager,
     // throw RuntimeException so that the coprocessor is unloaded.
     if (zk != null) {
@@ -1304,7 +1309,7 @@ public void preUnlockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte
   }
 
   private void isSystemOrSuperUser(Configuration conf) throws IOException {
-    User user = User.getCurrent();
+    User user = userProvider.getCurrent();
     if (user == null) {
       throw new IOException("Unable to obtain the current user, "
           + "authorization checks for internal operations will not work correctly!");
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
index e422710b72fd..46a707a93135 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.ipc.RequestContext;
@@ -101,6 +102,7 @@ public class SecureBulkLoadEndpoint extends BaseEndpointCoprocessor
 
   private RegionCoprocessorEnvironment env;
 
+  private UserProvider provider;
 
   @Override
   public void start(CoprocessorEnvironment env) {
@@ -110,6 +112,7 @@ public void start(CoprocessorEnvironment env) {
     random = new SecureRandom();
     conf = env.getConfiguration();
     baseStagingDir = getBaseStagingDir(conf);
+    this.provider = UserProvider.instantiate(conf);
 
     try {
       fs = FileSystem.get(conf);
@@ -152,7 +155,7 @@ public boolean bulkLoadHFiles(final List<Pair<byte[], String>> familyPaths,
     final UserGroupInformation ugi = user.getUGI();
     if(userToken != null) {
       ugi.addToken(userToken);
-    } else if(User.isSecurityEnabled()) {
+    } else if (provider.isHadoopSecurityEnabled()) {
       //we allow this to pass through in "simple" security mode
       //for mini cluster testing
       throw new DoNotRetryIOException("User token cannot be null");
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
index ba73f4efb43e..3e3257cd6e72 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -117,7 +118,8 @@ private TableAuthManager(ZooKeeperWatcher watcher, Configuration conf)
    * from the {@code hbase.superuser} configuration key.
    */
   private PermissionCache<Permission> initGlobal(Configuration conf) throws IOException {
-    User user = User.getCurrent();
+    UserProvider userProvider = UserProvider.instantiate(conf);
+    User user = userProvider.getCurrent();
     if (user == null) {
       throw new IOException("Unable to obtain the current user, " +
           "authorization checks for internal operations will not work correctly!");
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
index 03ec4c571833..e950685f6ce6 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.access.AccessControlLists;
 import org.apache.hadoop.hbase.security.access.SecureTestUtil;
 
@@ -42,9 +43,10 @@ public class TestSecureLoadIncrementalHFiles extends  TestLoadIncrementalHFiles{
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    useSecure = true;
     // setup configuration
     SecureTestUtil.enableSecurity(util.getConfiguration());
+    UserProvider.setUserProviderForTesting(util.getConfiguration(),
+      HadoopSecurityEnabledUserProviderForTesting.class);
 
     util.startMiniCluster();
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
index e8593b2cdce2..a808966ba604 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.access.AccessControlLists;
 import org.apache.hadoop.hbase.security.access.SecureTestUtil;
 
@@ -46,11 +47,11 @@ public class TestSecureLoadIncrementalHFilesSplitRecovery extends TestLoadIncrem
   //make sure they are in sync
   @BeforeClass
   public static void setupCluster() throws Exception {
-    useSecure = true;
     util = new HBaseTestingUtility();
     // setup configuration
     SecureTestUtil.enableSecurity(util.getConfiguration());
-
+    UserProvider.setUserProviderForTesting(util.getConfiguration(),
+      HadoopSecurityEnabledUserProviderForTesting.class);
     util.startMiniCluster();
 
     // Wait for the ACL table to become available
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index a7b025b49239..30a56befc952 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -470,10 +470,8 @@ public HConnectionKey(Configuration conf) {
       this.properties = Collections.unmodifiableMap(m);
 
       try {
-        User currentUser = User.getCurrent();
-        if (currentUser != null) {
-          username = currentUser.getName();
-        }
+        UserProvider provider = UserProvider.instantiate(conf);
+        username = provider.getCurrentUserName();
       } catch (IOException ioe) {
         LOG.warn("Error obtaining current user, skipping username in HConnectionKey",
             ioe);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/UserProvider.java b/src/main/java/org/apache/hadoop/hbase/client/UserProvider.java
new file mode 100644
index 000000000000..7150f9c22b53
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/client/UserProvider.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.BaseConfigurable;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Provide an instance of a user. Allows custom {@link User} creation.
+ */
+public class UserProvider extends BaseConfigurable {
+
+  private static final String USER_PROVIDER_CONF_KEY = "hbase.client.userprovider.class";
+
+  /**
+   * Instantiate the {@link UserProvider} specified in the configuration and set the passed
+   * configuration via {@link UserProvider#setConf(Configuration)}
+   * @param conf to read and set on the created {@link UserProvider}
+   * @return a {@link UserProvider} ready for use.
+   */
+  public static UserProvider instantiate(Configuration conf){
+    Class<? extends UserProvider> clazz =
+        conf.getClass(USER_PROVIDER_CONF_KEY, UserProvider.class, UserProvider.class);
+    return ReflectionUtils.newInstance(clazz, conf);
+  }
+
+  /**
+   * Set the {@link UserProvider} in the given configuration that should be instantiated
+   * @param conf to update
+   * @param provider class of the provider to set
+   */
+  public static void setUserProviderForTesting(Configuration conf,
+      Class<? extends UserProvider> provider) {
+    conf.set(USER_PROVIDER_CONF_KEY, provider.getName());
+  }
+
+  /**
+   * @return the userName for the current logged-in user.
+   * @throws IOException if the underlying user cannot be obtained
+   */
+  public String getCurrentUserName() throws IOException {
+    User user = getCurrent();
+    return user == null ? null : user.getName();
+  }
+
+  /**
+   * @return <tt>true</tt> if security is enabled, <tt>false</tt> otherwise
+   */
+  public boolean isHBaseSecurityEnabled() {
+    return User.isHBaseSecurityEnabled(this.getConf());
+  }
+
+  /**
+   * @return the current user within the current execution context
+   * @throws IOException if the user cannot be loaded
+   */
+  public User getCurrent() throws IOException {
+    return User.getCurrent();
+  }
+
+  public User create(UserGroupInformation ugi) {
+    return User.create(ugi);
+  }
+
+  /**
+   * Log in the current process using the given configuration keys for the credential file and login
+   * principal.
+   * <p>
+   * <strong>This is only applicable when running on secure Hadoop</strong> -- see
+   * org.apache.hadoop.security.SecurityUtil#login(Configuration,String,String,String). On regular
+   * Hadoop (without security features), this will safely be ignored.
+   * </p>
+   * @param conf The configuration data to use
+   * @param fileConfKey Property key used to configure path to the credential file
+   * @param principalConfKey Property key used to configure login principal
+   * @param localhost Current hostname to use in any credentials
+   * @throws IOException underlying exception from SecurityUtil.login() call
+   */
+  public void login(String fileConfKey, String principalConfKey, String localhost)
+      throws IOException {
+    User.login(getConf(), fileConfKey, principalConfKey, localhost);
+  }
+
+  /**
+   * @return whether or not Kerberos authentication is configured for Hadoop. For non-secure Hadoop,
+   *         this always returns <code>false</code>. For secure Hadoop, it will return the value
+   *         from {@code UserGroupInformation.isSecurityEnabled()}.
+   */
+  public boolean isHadoopSecurityEnabled() {
+    return User.isSecurityEnabled();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index a93fa4997d8d..e8ca2f6887dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -37,6 +37,7 @@
 
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Operation;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -97,6 +98,7 @@ public Object invoke(Object proxy, Method method, Object[] args)
 
   private Configuration conf;
   private HBaseClient client;
+  private UserProvider userProvider;
 
   @Override
   public void setConf(Configuration config) {
@@ -106,6 +108,7 @@ public void setConf(Configuration config) {
       this.client.stop();
     }
     this.client = new HBaseClient(HbaseObjectWritable.class, conf);
+    this.userProvider = UserProvider.instantiate(config);
   }
 
   @Override
@@ -127,7 +130,7 @@ public <T extends VersionedProtocol> T getProxy(
     T proxy =
           (T) Proxy.newProxyInstance(
               protocol.getClassLoader(), new Class[] { protocol },
-              new Invoker(client, protocol, addr, User.getCurrent(), conf,
+              new Invoker(client, protocol, addr, userProvider.getCurrent(), conf,
                   HBaseRPC.getRpcTimeout(rpcTimeout)));
 
     /*
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index bd098796c02e..6aa03261bd45 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.io.Writable;
@@ -169,9 +170,10 @@ public static void initTableReduceJob(String table,
   }
 
   public static void initCredentials(JobConf job) throws IOException {
-    if (User.isHBaseSecurityEnabled(job)) {
+    UserProvider provider = UserProvider.instantiate(job);
+    if (provider.isHBaseSecurityEnabled()) {
       try {
-        User.getCurrent().obtainAuthTokenForJob(job);
+        provider.getCurrent().obtainAuthTokenForJob(job);
       } catch (InterruptedException ie) {
         ie.printStackTrace();
         Thread.interrupted();
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index ca7bbc95f67c..7b459abd2eaa 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -60,6 +60,7 @@
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.ServerCallable;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.coprocessor.SecureBulkLoadClient;
 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
 import org.apache.hadoop.hbase.io.Reference;
@@ -74,7 +75,6 @@
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
-import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.security.token.Token;
@@ -104,21 +104,19 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
   private Token<?> userToken;
   private String bulkToken;
   private final boolean assignSeqIds;
+  private UserProvider userProvider;
 
   //package private for testing
-  LoadIncrementalHFiles(Configuration conf, Boolean useSecure) throws Exception {
+  public LoadIncrementalHFiles(Configuration conf) throws Exception {
     super(conf);
     this.cfg = conf;
     this.hbAdmin = new HBaseAdmin(conf);
     //added simple for testing
-    this.useSecure = useSecure != null ? useSecure : User.isHBaseSecurityEnabled(conf);
+    this.userProvider = UserProvider.instantiate(conf);
+    this.useSecure = userProvider.isHBaseSecurityEnabled();
     this.assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, false);
   }
 
-  public LoadIncrementalHFiles(Configuration conf) throws Exception {
-    this(conf, null);
-  }
-
   private void usage() {
     System.err.println("usage: " + NAME +
         " /path/to/hfileoutputformat-output " +
@@ -232,7 +230,7 @@ public void doBulkLoad(Path hfofDir, final HTable table)
       if(useSecure) {
         //This condition is here for unit testing
         //Since delegation token doesn't work in mini cluster
-        if(User.isSecurityEnabled()) {
+        if (userProvider.isHadoopSecurityEnabled()) {
           FileSystem fs = FileSystem.get(cfg);
           userToken = fs.getDelegationToken("renewer");
         }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index cac8dd47900c..00e03eba0028 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -47,6 +47,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
 import org.apache.hadoop.hbase.security.User;
@@ -284,11 +285,13 @@ public static void initTableMapperJob(List<Scan> scans,
   }
 
   public static void initCredentials(Job job) throws IOException {
-    if (User.isHBaseSecurityEnabled(job.getConfiguration())) {
+    UserProvider provider = UserProvider.instantiate(job.getConfiguration());
+    if (provider.isHBaseSecurityEnabled()) {
       try {
         // init credentials for remote cluster
         String quorumAddress = job.getConfiguration().get(
             TableOutputFormat.QUORUM_ADDRESS);
+        User user = provider.getCurrent();
         if (quorumAddress != null) {
           String[] parts = ZKUtil.transformClusterKey(quorumAddress);
           Configuration peerConf = HBaseConfiguration.create(job
@@ -296,10 +299,10 @@ public static void initCredentials(Job job) throws IOException {
           peerConf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
           peerConf.set("hbase.zookeeper.client.port", parts[1]);
           peerConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
-          User.getCurrent().obtainAuthTokenForJob(peerConf, job);
+          user.obtainAuthTokenForJob(peerConf, job);
         }
         
-        User.getCurrent().obtainAuthTokenForJob(job.getConfiguration(), job);
+        user.obtainAuthTokenForJob(job.getConfiguration(), job);
       } catch (InterruptedException ie) {
         LOG.info("Interrupted obtaining user authentication token");
         Thread.interrupted();
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 2fef4ffd3c14..6be1beab259d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -67,6 +67,7 @@
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
 import org.apache.hadoop.hbase.client.Result;
@@ -328,7 +329,8 @@ public HMaster(final Configuration conf)
       "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
 
     // initialize server principal (if using secure Hadoop)
-    User.login(conf, "hbase.master.keytab.file",
+    UserProvider provider = UserProvider.instantiate(conf);
+    provider.login("hbase.master.keytab.file",
       "hbase.master.kerberos.principal", this.isa.getHostName());
 
     // set the thread name now we have an address
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 0e793bc6aef2..7a9c751096ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -101,6 +101,7 @@
 import org.apache.hadoop.hbase.client.RowLock;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.client.coprocessor.Exec;
 import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@@ -473,7 +474,8 @@ public HRegionServer(Configuration conf)
       "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
 
     // login the server principal (if using secure Hadoop)
-    User.login(this.conf, "hbase.regionserver.keytab.file",
+    UserProvider provider = UserProvider.instantiate(conf);
+    provider.login("hbase.regionserver.keytab.file",
       "hbase.regionserver.kerberos.principal", this.isa.getHostName());
     regionServerAccounting = new RegionServerAccounting();
     cacheConfig = new CacheConfig(conf);
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index ca3edc5ad547..306abf48ec46 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -30,6 +30,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.rest.filter.GzipFilter;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.InfoServer;
@@ -82,11 +83,12 @@ public static void main(String[] args) throws Exception {
     VersionInfo.logVersion();
     Configuration conf = HBaseConfiguration.create();
     // login the server principal (if using secure Hadoop)
-    if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
+    UserProvider provider = UserProvider.instantiate(conf);
+    if (provider.isHadoopSecurityEnabled() && provider.isHBaseSecurityEnabled()) {
       String machineName = Strings.domainNamePointerToHostName(
         DNS.getDefaultHost(conf.get("hbase.rest.dns.interface", "default"),
           conf.get("hbase.rest.dns.nameserver", "default")));
-      User.login(conf, "hbase.rest.keytab.file", "hbase.rest.kerberos.principal",
+      provider.login("hbase.rest.keytab.file", "hbase.rest.kerberos.principal",
         machineName);
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index a0cacc08b0b8..207536bd3c14 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -403,7 +403,7 @@ public static boolean isSecurityEnabled() {
    * {@link org.apache.hadoop.security.UserGroupInformation} for secure Hadoop
    * 0.20 and versions 0.21 and above.
    */
-  private static class SecureHadoopUser extends User {
+  public static class SecureHadoopUser extends User {
     private String shortName;
 
     private SecureHadoopUser() throws IOException {
@@ -419,7 +419,7 @@ private SecureHadoopUser() throws IOException {
       }
     }
 
-    private SecureHadoopUser(UserGroupInformation ugi) {
+    public SecureHadoopUser(UserGroupInformation ugi) {
       this.ugi = ugi;
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
index a2f244c67523..3f4790ba45fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
@@ -30,6 +30,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.thrift.ThriftServerRunner.ImplType;
 import org.apache.hadoop.hbase.util.InfoServer;
@@ -89,14 +90,17 @@ private static void printUsageAndExit(Options options, int exitCode)
    */
    void doMain(final String[] args) throws Exception {
      processOptions(args);
-     // login the server principal (if using secure Hadoop)
-     if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) {
-       String machineName = Strings.domainNamePointerToHostName(
-         DNS.getDefaultHost(conf.get("hbase.thrift.dns.interface", "default"),
-           conf.get("hbase.thrift.dns.nameserver", "default")));
-       User.login(conf, "hbase.thrift.keytab.file",
-           "hbase.thrift.kerberos.principal", machineName);
-     }
+    // login the server principal (if using secure Hadoop)
+    UserProvider provider = UserProvider.instantiate(conf);
+    if (provider.isHadoopSecurityEnabled() && provider.isHBaseSecurityEnabled()) {
+      String machineName =
+          Strings.domainNamePointerToHostName(DNS.getDefaultHost(
+            conf.get("hbase.thrift.dns.interface", "default"),
+            conf.get("hbase.thrift.dns.nameserver", "default")));
+
+      provider.login("hbase.thrift.keytab.file", 
+        "hbase.thrift.kerberos.principal", machineName);
+    }
      serverRunner = new ThriftServerRunner(conf);
 
      // Put up info server.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 789db91df442..09791edbef13 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -79,6 +79,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -1411,9 +1412,11 @@ private void preCheckPermission() throws IOException, AccessControlException {
       return;
     }
 
-    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
-    FileSystem fs = hbaseDir.getFileSystem(getConf());
-    User user = User.getCurrent();
+    Configuration conf = getConf();
+    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = hbaseDir.getFileSystem(conf);
+    UserProvider provider = UserProvider.instantiate(conf);
+    User user = provider.getCurrent();
     FileStatus[] files = fs.listStatus(hbaseDir);
     for (FileStatus file : files) {
       try {
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java
new file mode 100644
index 000000000000..a693755f4113
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.hbase.client.UserProvider;
+
+/**
+ * A {@link UserProvider} that always says hadoop security is enabled, regardless of the underlying
+ * configuration. HBase security is <i>not enabled</i> as this is used to determine if SASL is used
+ * to do the authentication, which requires a Kerberos ticket (which we currently don't have in
+ * tests).
+ * <p>
+ * This should only be used for <b>TESTING</b>.
+ */
+public class HadoopSecurityEnabledUserProviderForTesting extends UserProvider {
+
+  @Override
+  public boolean isHBaseSecurityEnabled() {
+    return false;
+  }
+
+  @Override
+  public boolean isHadoopSecurityEnabled() {
+    return true;
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index e89727fc652f..02afd4eb1042 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -30,10 +30,15 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -41,7 +46,9 @@
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.*;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 /**
@@ -64,8 +71,6 @@ public class TestLoadIncrementalHFiles {
     Compression.Algorithm.NONE.getName();
 
   static HBaseTestingUtility util = new HBaseTestingUtility();
-  //used by secure subclass
-  static boolean useSecure = false;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
@@ -154,7 +159,8 @@ private void runTest(String testName, BloomType bloomType,
 
     HTable table = new HTable(util.getConfiguration(), TABLE);
     util.waitTableAvailable(TABLE, 30000);
-    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), useSecure);
+
+    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
     loader.doBulkLoad(dir, table);
 
     assertEquals(expectedRows, util.countRows(table));
@@ -239,7 +245,9 @@ public void testNonexistentColumnFamilyLoad() throws Exception {
 
     HTable table = new HTable(util.getConfiguration(), TABLE);
     util.waitTableAvailable(TABLE, 30000);
-    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), false);
+    // make sure we go back to the usual user provider
+    UserProvider.setUserProviderForTesting(util.getConfiguration(), UserProvider.class);
+    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
     try {
       loader.doBulkLoad(dir, table);
       assertTrue("Loading into table with non-existent family should have failed", false);
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
index eaf4aec72b31..bb67ebe36b8a 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
@@ -47,7 +47,6 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.ServerCallable;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
@@ -69,8 +68,6 @@ public class TestLoadIncrementalHFilesSplitRecovery {
   final static Log LOG = LogFactory.getLog(TestHRegionServerBulkLoad.class);
 
   static HBaseTestingUtility util;
-  //used by secure subclass
-  static boolean useSecure = false;
 
   final static int NUM_CFS = 10;
   final static byte[] QUAL = Bytes.toBytes("qual");
@@ -137,7 +134,7 @@ private Path buildBulkFiles(String table, int value) throws Exception {
    */
   private void populateTable(String table, int value) throws Exception {
     // create HFiles for different column families
-    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration(), useSecure);
+    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration());
     Path bulk1 = buildBulkFiles(table, value);
     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
     lih.doBulkLoad(bulk1, t);
@@ -229,7 +226,7 @@ public void testBulkLoadPhaseFailure() throws Exception {
     final AtomicInteger attmptedCalls = new AtomicInteger();
     final AtomicInteger failedCalls = new AtomicInteger();
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration(), useSecure) {
+        util.getConfiguration()) {
 
       protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
           byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis)
@@ -297,7 +294,7 @@ public void testSplitWhileBulkLoadPhase() throws Exception {
     // files to fail when attempt to atomically import.  This is recoverable.
     final AtomicInteger attemptedCalls = new AtomicInteger();
     LoadIncrementalHFiles lih2 = new LoadIncrementalHFiles(
-        util.getConfiguration(), useSecure) {
+        util.getConfiguration()) {
 
       protected void bulkLoadPhase(final HTable htable, final HConnection conn,
           ExecutorService pool, Deque<LoadQueueItem> queue,
@@ -338,7 +335,7 @@ public void testGroupOrSplitPresplit() throws Exception {
 
     final AtomicInteger countedLqis= new AtomicInteger();
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration(), useSecure) {
+        util.getConfiguration()) {
       protected List<LoadQueueItem> groupOrSplit(
           Multimap<ByteBuffer, LoadQueueItem> regionGroups,
           final LoadQueueItem item, final HTable htable,
@@ -370,7 +367,7 @@ public void testGroupOrSplitFailure() throws Exception {
     setupTable(table, 10);
 
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration(), useSecure) {
+        util.getConfiguration()) {
       int i = 0;
 
       protected List<LoadQueueItem> groupOrSplit(
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
index b62185df62fe..885cea6bd3fe 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
@@ -102,6 +102,7 @@ public void setupBasicMocks() {
 
     // Mock admin
     admin = Mockito.mock(HBaseAdmin.class); 
+    Mockito.when(admin.getConfiguration()).thenReturn(conf);
   }
   
 

From 08a34c5a2f61aebeebb0961a25b4e0735417f478 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Oct 2013 00:04:10 +0000
Subject: [PATCH 1202/1540] CHANGES.txt, pom.xml for 0.94.13

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1536938 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index d6185dcbccee..1265d6fcaa40 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,52 @@
 HBase Change Log
 
+Release 0.94.13 - 10/29/2013
+Sub-task
+
+    [HBASE-9711] - Improve HBASE-9428 - avoid copying bytes for RegexFilter unless necessary
+
+Bug
+
+    [HBASE-7600] - TestAdmin.testCreateBadTables is failing occasionally
+    [HBASE-8521] - Cells cannot be overwritten with bulk loaded HFiles
+    [HBASE-9430] - Memstore heapSize calculation - DEEP_OVERHEAD is incorrect
+    [HBASE-9504] - Backport HBASE-1212 to 0.94
+    [HBASE-9548] - Cleanup SnapshotTestingUtils
+    [HBASE-9607] - Data loss after snapshot restore into cloned table
+    [HBASE-9649] - HFilePrettyPrinter should not throw a NPE if FirstKey or LastKey is null.
+    [HBASE-9651] - Backport HBASE-3890 'Scheduled tasks in distributed log splitting not in sync with ZK' to 0.94
+    [HBASE-9727] - HBase Rest Server - DELETE scanner operation is a no-op
+    [HBASE-9731] - updatesBlockedSeconds RegionServer metric should not be a histogram
+    [HBASE-9732] - Static AtomicLong updated in StoreFileScanner every (re)seek
+    [HBASE-9737] - Corrupt HFile cause resource leak leading to Region Server OOM
+    [HBASE-9745] - Append HBASE_CLASSPATH to end of Java classpath and use another env var for prefix
+    [HBASE-9747] - PrefixFilter with OR condition gives wrong results
+    [HBASE-9751] - Excessive readpoints checks in StoreFileScanner
+    [HBASE-9753] - Excessive readpoint checks in MemstoreScanner
+    [HBASE-9783] - o.a.h.h.r.HRegion.mutateRow() with non-existent CF cause NPE
+    [HBASE-9789] - Change logging for Coprocessor exec call to trace
+    [HBASE-9807] - block encoder unnecessarily copies the key for each reseek
+    [HBASE-9842] - Backport HBASE-9593 and HBASE-8667 to 0.94
+    [HBASE-9847] - HConnectionImplementation does not connect to new active master
+
+Improvement
+
+    [HBASE-9221] - Provide interface for getting a User in the client
+    [HBASE-9488] - Improve performance for small scan
+    [HBASE-9716] - LoadTestTool should provide default min and max settings to the data generator
+    [HBASE-9749] - Custom threadpool for Coprocessor obtained HTables
+
+Task
+
+    [HBASE-9819] - Backport HBASE-8372 'Provide mutability to CompoundConfiguration' to 0.94
+
+Test
+
+    [HBASE-8553] - improve unit-test coverage of package org.apache.hadoop.hbase.mapreduce.hadoopbackport
+    [HBASE-9851] - TestHBaseFsck.testQuarantineMissingHFile is flaky
+    [HBASE-9852] - TestRpcMetrics.testCustomMetrics is flaky
+
+
 Release 0.94.12 - 9/19/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index ea8810954219..dda0e0254897 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.13-SNAPSHOT</version>
+  <version>0.94.13</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 4f9b424c08dc207156ad38b9f42ecca8eddd9fb5 Mon Sep 17 00:00:00 2001
From: jeffreyz <jeffreyz@unknown>
Date: Wed, 30 Oct 2013 17:45:32 +0000
Subject: [PATCH 1203/1540] HBASE-8552: fix coverage
 org.apache.hadoop.hbase.rest.filter(by Andrey Klochkov)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1537199 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/rest/TestGZIPResponseWrapper.java   | 142 ++++++++++++++++++
 1 file changed, 142 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java

diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java b/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java
new file mode 100644
index 000000000000..5063a8f643a6
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java
@@ -0,0 +1,142 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest;
+
+import java.io.IOException;
+
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.rest.filter.GZIPResponseStream;
+import org.apache.hadoop.hbase.rest.filter.GZIPResponseWrapper;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.mockito.Mockito.*;
+import static org.junit.Assert.*;
+
+@Category(MediumTests.class)
+public class TestGZIPResponseWrapper {
+
+  /**
+   * headers function should be called in response except header "content-length"
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testHeader() throws IOException {
+
+    HttpServletResponse response = mock(HttpServletResponse.class);
+
+    GZIPResponseWrapper test = new GZIPResponseWrapper(response);
+    test.setStatus(200);
+    verify(response).setStatus(200);
+    test.addHeader("header", "header value");
+    verify(response).addHeader("header", "header value");
+    test.addHeader("content-length", "header value2");
+    verify(response, never()).addHeader("content-length", "header value");
+
+    test.setIntHeader("header", 5);
+    verify(response).setIntHeader("header", 5);
+    test.setIntHeader("content-length", 4);
+    verify(response, never()).setIntHeader("content-length", 4);
+
+    test.setHeader("set-header", "new value");
+    verify(response).setHeader("set-header", "new value");
+    test.setHeader("content-length", "content length value");
+    verify(response, never()).setHeader("content-length", "content length value");
+
+    test.sendRedirect("location");
+    verify(response).sendRedirect("location");
+    
+    test.flushBuffer();
+    verify(response).flushBuffer();
+
+  }
+
+  @Test
+  public void testResetBuffer() throws IOException {
+    HttpServletResponse response = mock(HttpServletResponse.class);
+    when(response.isCommitted()).thenReturn(false);
+    ServletOutputStream out = mock(ServletOutputStream.class);
+    when(response.getOutputStream()).thenReturn(out);
+    GZIPResponseWrapper test = new GZIPResponseWrapper(response);
+
+    ServletOutputStream servletOutput = test.getOutputStream();
+    assertEquals(org.apache.hadoop.hbase.rest.filter.GZIPResponseStream.class,
+        servletOutput.getClass());
+    test.resetBuffer();
+    verify(response).setHeader("Content-Encoding", null);
+
+    when(response.isCommitted()).thenReturn(true);
+    servletOutput = test.getOutputStream();
+    assertEquals(out.getClass(), servletOutput.getClass());
+    assertNotNull(test.getWriter());
+
+  }
+
+  @Test
+  public void testReset() throws IOException {
+    HttpServletResponse response = mock(HttpServletResponse.class);
+    when(response.isCommitted()).thenReturn(false);
+    ServletOutputStream out = mock(ServletOutputStream.class);
+    when(response.getOutputStream()).thenReturn(out);
+    GZIPResponseWrapper test = new GZIPResponseWrapper(response);
+
+    ServletOutputStream servletOutput = test.getOutputStream();
+    assertEquals(org.apache.hadoop.hbase.rest.filter.GZIPResponseStream.class,
+        servletOutput.getClass());
+    test.reset();
+    verify(response).setHeader("Content-Encoding", null);
+
+    when(response.isCommitted()).thenReturn(true);
+    servletOutput = test.getOutputStream();
+    assertEquals(out.getClass(), servletOutput.getClass());
+  }
+
+  @Test
+  public void testSendError() throws IOException {
+    HttpServletResponse response = mock(HttpServletResponse.class);
+    GZIPResponseWrapper test = new GZIPResponseWrapper(response);
+
+    test.sendError(404);
+    verify(response).sendError(404);
+
+    test.sendError(404, "error message");
+    verify(response).sendError(404, "error message");
+
+  }
+
+  @Test
+  public void testGZIPResponseStream() throws IOException {
+    HttpServletResponse httpResponce = mock(HttpServletResponse.class);
+    ServletOutputStream out = mock(ServletOutputStream.class);
+
+    when(httpResponce.getOutputStream()).thenReturn(out);
+    GZIPResponseStream test = new GZIPResponseStream(httpResponce);
+    verify(httpResponce).addHeader("Content-Encoding", "gzip");
+
+    test.close();
+
+    test.resetBuffer();
+    verify(httpResponce).setHeader("Content-Encoding", null);
+  }
+}

From 662ea4f0d376b9e5b13133a27713f1b445dd32d9 Mon Sep 17 00:00:00 2001
From: jeffreyz <jeffreyz@unknown>
Date: Wed, 30 Oct 2013 18:00:22 +0000
Subject: [PATCH 1204/1540] HBASE-8557: fix coverage
 org.apache.hadoop.hbase.rest.metrics(by Aleksey Gorshkov)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1537213 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/TestRESTMetrics.java    | 90 +++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java

diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java
new file mode 100644
index 000000000000..2ea49b4dc91f
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java
@@ -0,0 +1,90 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hbase.rest;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.rest.metrics.RESTMetrics;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import static org.junit.Assert.*;
+
+/**
+ * Test RESTMetrics class
+ */
+@Category(SmallTests.class)
+public class TestRESTMetrics {
+
+  @Test
+  public void testRESTMetrics() throws InterruptedException {
+    long timeout = 2000;
+    RESTMetrics test = new RESTMetrics();
+    int incrementSucessfulGet = 20000;
+    int incrementSucessfulDelete = 3000000;
+    int incrementSucessfulPut = 3000000;
+    int incrementRequest = incrementSucessfulGet + incrementSucessfulDelete + incrementSucessfulPut;
+
+    int incrementFailedGetRequests = 100;
+    int incrementFailedDeleteRequests = 30;
+    int incrementFailedPutRequests = 2;
+
+    long start1 = System.currentTimeMillis();
+    test.doUpdates(null);
+
+    // started value
+    assertEquals(0, test.getRequests(), 0.01);
+    assertEquals(0, test.getSucessfulDeleteCount(), 0.01);
+    assertEquals(0, test.getSucessfulPutCount(), 0.01);
+    assertEquals(0, test.getSucessfulGetCount(), 0.01);
+    assertEquals(0, test.getFailedDeleteCount(), 0.01);
+    assertEquals(0, test.getFailedGetCount(), 0.01);
+    assertEquals(0, test.getFailedPutCount(), 0.01);
+
+    // sleep some seconds
+    Thread.sleep(timeout);
+    test.incrementRequests(incrementRequest);
+    test.incrementSucessfulGetRequests(incrementSucessfulGet);
+    test.incrementSucessfulDeleteRequests(incrementSucessfulDelete);
+    test.incrementSucessfulPutRequests(incrementSucessfulPut);
+    test.incrementFailedGetRequests(incrementFailedGetRequests);
+    test.incrementFailedDeleteRequests(incrementFailedDeleteRequests);
+    test.incrementFailedPutRequests(incrementFailedPutRequests);
+
+    test.doUpdates(null);
+
+    // The maximum time for stability test
+    long tmax = System.currentTimeMillis() - start1;
+
+    testData(tmax, timeout, test.getRequests(), incrementRequest);
+    testData(tmax, timeout, test.getSucessfulGetCount(), incrementSucessfulGet);
+    testData(tmax, timeout, test.getSucessfulDeleteCount(), incrementSucessfulDelete);
+    testData(tmax, timeout, test.getSucessfulPutCount(), incrementSucessfulPut);
+    testData(tmax, timeout, test.getFailedGetCount(), incrementFailedGetRequests);
+    testData(tmax, timeout, test.getFailedDeleteCount(), incrementFailedDeleteRequests);
+    testData(tmax, timeout, test.getFailedPutCount(), incrementFailedPutRequests);
+
+    test.shutdown();
+  }
+
+  // test minimum and maximum speed
+  private void testData(double tmax, long tmin, float value, double requests) {
+    assertTrue((requests / tmax) * 1000 <= value);
+    assertTrue((requests / tmin) * 1000 >= value);
+
+  }
+}

From 4e4954ef1e977cf2dda39b21b878ed8fef719ea2 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Thu, 31 Oct 2013 10:55:30 +0000
Subject: [PATCH 1205/1540] HBASE-8556 fix coverage
 org.apache.hadoop.hbase.metrics.histogram (Aleksey Gorshkov)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1537437 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/metrics/TestMetricsHistogram.java   | 111 ++++++++++--------
 1 file changed, 61 insertions(+), 50 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
index da0e87edfdfd..0d853fa487cd 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java
@@ -14,20 +14,27 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- */
-
+ */ 
 package org.apache.hadoop.hbase.metrics;
 
-import java.util.Arrays;
+import static org.mockito.Matchers.anyFloat;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
 import java.util.Random;
 
-import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
 import org.apache.hadoop.hbase.SmallTests;
-import com.yammer.metrics.stats.Snapshot;
+import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+import org.apache.hadoop.metrics.MetricsRecord;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.yammer.metrics.stats.Snapshot;
+
+@SuppressWarnings("deprecation")
 @Category(SmallTests.class)
 public class TestMetricsHistogram {
 
@@ -42,60 +49,64 @@ public void testBasicUniform() {
     Assert.assertEquals(100, h.getCount());
     Assert.assertEquals(0, h.getMin());
     Assert.assertEquals(99, h.getMax());
+    Assert.assertEquals(49.5d, h.getMean(), 0.01);
   }
 
-  private static int safeIndex(int i, int len) {
-    if (i < len && i>= 0) {
-      return i;
-    } else if (i >= len) {
-      return len - 1; 
-    } else {
-      return 0;
-    }    
-  }
-  
   @Test
-  public void testRandom() {
-    final Random r = new Random();
+  public void testSnapshotPercentiles() {
     final MetricsHistogram h = new MetricsHistogram("testHistogram", null);
-
-    final long[] data = new long[1000];
-
-    for (int i = 0; i < data.length; i++) {
-      data[i] = (long) (r.nextGaussian() * 10000.0);
-      h.update(data[i]);
-    }
+    final long[] data = genRandomData(h);
 
     final Snapshot s = h.getSnapshot();
-    Arrays.sort(data);
-
-    // as long as the histogram chooses an item with index N+/-slop, accept it
-    final int slop = 20;
 
-    // make sure the median, 75th percentile and 95th percentile are good
-    final int medianIndex = data.length / 2;
-    final long minAcceptableMedian = data[safeIndex(medianIndex - slop, 
-        data.length)];
-    final long maxAcceptableMedian = data[safeIndex(medianIndex + slop, 
-        data.length)];
-    Assert.assertTrue(s.getMedian() >= minAcceptableMedian 
-        && s.getMedian() <= maxAcceptableMedian);
+    assertPercentile(data, 50, s.getMedian());
+    assertPercentile(data, 75, s.get75thPercentile());
+    assertPercentile(data, 95, s.get95thPercentile());
+    assertPercentile(data, 98, s.get98thPercentile());
+    assertPercentile(data, 99, s.get99thPercentile());
+    assertPercentile(data, 99.9, s.get999thPercentile());
+  }
 
-    final int seventyFifthIndex = (int) (data.length * 0.75);
-    final long minAcceptableseventyFifth = data[safeIndex(seventyFifthIndex 
-        - slop, data.length)];
-    final long maxAcceptableseventyFifth = data[safeIndex(seventyFifthIndex 
-        + slop, data.length)];
-    Assert.assertTrue(s.get75thPercentile() >= minAcceptableseventyFifth 
-        && s.get75thPercentile() <= maxAcceptableseventyFifth);
+  @Test
+  public void testPushMetric() {
+    final MetricsHistogram h = new MetricsHistogram("testHistogram", null);
+    genRandomData(h);
+
+    MetricsRecord mr = mock(MetricsRecord.class);
+    h.pushMetric(mr);
+    
+    verify(mr).setMetric("testHistogram_num_ops", 10000L);
+    verify(mr).setMetric(eq("testHistogram_min"), anyLong());
+    verify(mr).setMetric(eq("testHistogram_max"), anyLong());
+    verify(mr).setMetric(eq("testHistogram_mean"), anyFloat());
+    verify(mr).setMetric(eq("testHistogram_std_dev"), anyFloat());
+    verify(mr).setMetric(eq("testHistogram_median"), anyFloat());
+    verify(mr).setMetric(eq("testHistogram_75th_percentile"), anyFloat());
+    verify(mr).setMetric(eq("testHistogram_95th_percentile"), anyFloat());
+    verify(mr).setMetric(eq("testHistogram_99th_percentile"), anyFloat());    
+  }
 
-    final int ninetyFifthIndex = (int) (data.length * 0.95);
-    final long minAcceptableninetyFifth = data[safeIndex(ninetyFifthIndex 
-        - slop, data.length)];
-    final long maxAcceptableninetyFifth = data[safeIndex(ninetyFifthIndex 
-        + slop, data.length)];
-    Assert.assertTrue(s.get95thPercentile() >= minAcceptableninetyFifth 
-        && s.get95thPercentile() <= maxAcceptableninetyFifth);
+  private void assertPercentile(long[] data, double percentile, double value) {
+    int count = 0;
+    for (long v : data) {
+      if (v < value) {
+        count++;
+      }
+    }
+    Assert.assertEquals("Wrong " + percentile + " percentile", 
+        (int)(percentile / 100), count / data.length);
+  }
+  
+  private long[] genRandomData(final MetricsHistogram h) {
+    final Random r = new Random();
+    final long[] data = new long[10000];
 
+    for (int i = 0; i < data.length; i++) {
+      data[i] = (long) (r.nextGaussian() * 10000);
+      h.update(data[i]);
+    }
+    
+    return data;
   }
+  
 }

From cff2dce66fd32b6883d08e45a18378d59dfcd061 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 31 Oct 2013 19:58:40 +0000
Subject: [PATCH 1206/1540] HBASE-8397 improve unit-test coverage of package
 org.apache.hadoop.hbase.master.metrics (0.94)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1537607 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/metrics/TestMasterStatistics.java  | 145 ++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java

diff --git a/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java b/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java
new file mode 100644
index 000000000000..77dd9c2309c1
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/metrics/TestMasterStatistics.java
@@ -0,0 +1,145 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.metrics;
+
+import static org.junit.Assert.*;
+
+import java.lang.management.ManagementFactory;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests {@link MasterMetrics} and access to it through the
+ * {@link MasterStatistics} management bean.
+ * 
+ * Note: this test must always be run in separate fork (process)
+ * because it changes static contents of metrics subsystem and 
+ * is affected itself by that static contents. For that reason 
+ * the test put into {@link MediumTests} Category.   
+ */
+@Category(MediumTests.class)
+public class TestMasterStatistics {
+
+  @Before
+  @SuppressWarnings("deprecation")
+  public void ensureNullContext() throws Exception {
+    // Clean up the factory attributes to instantiate the NullContext,
+    // regardless if the resource "/hadoop-metrics.properties" is present 
+    // in the class-path: 
+    org.apache.hadoop.metrics.ContextFactory factory = 
+        org.apache.hadoop.metrics.ContextFactory.getFactory();
+    String[] attributeNames = factory.getAttributeNames();
+    for (String attributeName: attributeNames) {
+      factory.removeAttribute(attributeName);
+    }
+    // ensure the attributes are cleaned up:
+    attributeNames = factory.getAttributeNames();
+    assertEquals(0, attributeNames.length);
+    // Get the "hbase" context and ensure it is NullContext:  
+    org.apache.hadoop.metrics.MetricsContext context 
+      = org.apache.hadoop.metrics.MetricsUtil.getContext("hbase");
+    assertTrue(context instanceof org.apache.hadoop.metrics.spi.NullContext);
+    assertTrue(!context.isMonitoring());
+  }
+  
+  @Test
+  public void testMasterStatistics() throws Exception {
+    // No timer updates started here since NullContext is used, see #ensureNullContext().
+    // (NullContext never starts the updater thread).
+    MasterMetrics masterMetrics = new MasterMetrics("foo");
+    
+    try {
+      final MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+      final ObjectName objectName = new ObjectName(
+          "hadoop:name=MasterStatistics,service=Master");
+
+      masterMetrics.doUpdates(null);
+
+      masterMetrics.resetAllMinMax();
+
+      masterMetrics.incrementRequests(10);
+      Thread.sleep(1001);
+
+      masterMetrics.addSnapshot(1L);
+      masterMetrics.addSnapshotClone(2L);
+      masterMetrics.addSnapshotRestore(3L);
+
+      // 3 times added split, average = (5+3+4)/3 = 4
+      masterMetrics.addSplit(4L, 5L);
+      masterMetrics.addSplit(2L, 3L);
+      masterMetrics.addSplit(13L, 4L);
+
+      masterMetrics.doUpdates(null);
+
+      final float f = masterMetrics.getRequests();
+      // f = 10/T, where T >= 1 sec. So, we assert that 0 < f <= 10:
+      if (f <= 0.0f || f > 10.0f) {
+        fail("Unexpected rate value: " + f);
+      }
+      Object attribute = server.getAttribute(objectName, "cluster_requests");
+      float f2 = ((Float) attribute).floatValue();
+      assertEquals("The value obtained through bean server should be equal to the one " +
+          "obtained directly.", f, f2, 1e-4);
+
+      // NB: these 3 metrics are not pushed upon masterMetrics.doUpdates(),
+      // so they always return null:
+      attribute = server.getAttribute(objectName, "snapshotTimeNumOps");
+      assertEquals(Integer.valueOf(0), attribute);
+      attribute = server.getAttribute(objectName, "snapshotRestoreTimeNumOps");
+      assertEquals(Integer.valueOf(0), attribute);
+      attribute = server.getAttribute(objectName, "snapshotCloneTimeNumOps");
+      assertEquals(Integer.valueOf(0), attribute);
+
+      attribute = server.getAttribute(objectName, "splitSizeNumOps");
+      assertEquals(Integer.valueOf(3), attribute);
+      attribute = server.getAttribute(objectName, "splitSizeAvgTime");
+      assertEquals(Long.valueOf(4), attribute);
+    } finally {
+      masterMetrics.shutdown();
+    }
+  }
+
+  @Test
+  public void testHBaseInfoBean() throws Exception {
+    MasterMetrics masterMetrics = new MasterMetrics("foo");
+    try {
+      final MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+      // Test Info bean:
+      final ObjectName objectName2 = new ObjectName(
+          "hadoop:name=Info,service=HBase");
+      Object attribute;
+      attribute = server.getAttribute(objectName2, "revision");
+      assertNotNull(attribute);
+      attribute = server.getAttribute(objectName2, "version");
+      assertNotNull(attribute);
+      attribute = server.getAttribute(objectName2, "hdfsUrl");
+      assertNotNull(attribute);
+      attribute = server.getAttribute(objectName2, "user");
+      assertNotNull(attribute);
+    } finally {
+      masterMetrics.shutdown();
+    }
+  }
+}

From 3a1ee21bfb68ee33195bd5b27abb0ab2a913c504 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 31 Oct 2013 20:30:35 +0000
Subject: [PATCH 1207/1540] HBASE-8543 fix coverage
 org.apache.hadoop.hbase.rest.client

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1537626 13f79535-47bb-0310-9956-ffa450edef68
---
 .../rest/client/TestRemoteAdminRetries.java   | 164 +++++++++++++++
 .../rest/client/TestRemoteHTableRetries.java  | 193 +++++++++++++++++
 .../hbase/rest/client/TestRemoteTable.java    | 194 +++++++++++++++---
 3 files changed, 518 insertions(+), 33 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java

diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java
new file mode 100644
index 000000000000..cb801e3cf9da
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest.client;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests {@link RemoteAdmin} retries.
+ */
+@Category(SmallTests.class)
+public class TestRemoteAdminRetries {
+
+  private static final int SLEEP_TIME = 50;
+  private static final int RETRIES = 3;
+  private static final long MAX_TIME = SLEEP_TIME * (RETRIES - 1);
+  
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  
+  private RemoteAdmin remoteAdmin;
+  private Client client;
+
+  @Before
+  public void setup() throws Exception {
+    client = mock(Client.class);
+    Response response = new Response(509);
+    when(client.get(anyString(), anyString())).thenReturn(response);
+    when(client.delete(anyString())).thenReturn(response);
+    when(client.put(anyString(), anyString(), any(byte[].class))).thenReturn(response);
+    when(client.post(anyString(), anyString(), any(byte[].class))).thenReturn(response);
+    Configuration configuration = TEST_UTIL.getConfiguration();
+
+    configuration.setInt("hbase.rest.client.max.retries", RETRIES);
+    configuration.setInt("hbase.rest.client.sleep", SLEEP_TIME);
+
+    remoteAdmin = new RemoteAdmin(client, TEST_UTIL.getConfiguration(), "MyTable");
+  }
+
+  @Test
+  public void testFailingGetRestVersion() throws Exception  {
+    testTimedOutGetCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.getRestVersion();
+      }
+    });
+  }
+  
+  @Test
+  public void testFailingGetClusterStatus() throws Exception  {
+    testTimedOutGetCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.getClusterStatus();
+      }
+    });
+  }
+
+  @Test
+  public void testFailingGetClusterVersion() throws Exception {
+    testTimedOutGetCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.getClusterVersion();
+      }
+    });
+  }
+
+  @Test
+  public void testFailingGetTableAvailable() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.isTableAvailable(Bytes.toBytes("TestTable"));
+      }
+    });
+  }
+
+  @Test
+  public void testFailingCreateTable() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.createTable(new HTableDescriptor(Bytes.toBytes("TestTable")));
+      }
+    });
+    verify(client, times(RETRIES)).put(anyString(), anyString(), any(byte[].class));
+  }
+
+  @Test
+  public void testFailingDeleteTable() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.deleteTable("TestTable");
+      }
+    });
+    verify(client, times(RETRIES)).delete(anyString());
+  }
+
+  @Test
+  public void testFailingGetTableList() throws Exception {
+    testTimedOutGetCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteAdmin.getTableList();
+      }
+    });
+  }
+  
+  private void testTimedOutGetCall(CallExecutor callExecutor) throws Exception {
+    testTimedOutCall(callExecutor);
+    verify(client, times(RETRIES)).get(anyString(), anyString());
+  }
+  
+  private void testTimedOutCall(CallExecutor callExecutor) throws Exception {
+    long start = System.currentTimeMillis();
+    try {
+      callExecutor.run();
+      fail("should be timeout exception!");
+    } catch (IOException e) {
+      assertTrue(Pattern.matches(".*MyTable.*timed out", e.toString()));
+    }
+    assertTrue((System.currentTimeMillis() - start) > MAX_TIME);
+  }
+
+  private static interface CallExecutor {
+    void run() throws Exception;
+  }
+  
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java
new file mode 100644
index 000000000000..547dfab8101a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest.client;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test RemoteHTable retries.
+ */
+@Category(SmallTests.class)
+public class TestRemoteHTableRetries {
+
+  private static final int SLEEP_TIME = 50;
+  private static final int RETRIES = 3;
+  private static final long MAX_TIME = SLEEP_TIME * (RETRIES - 1);
+
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  
+  private static final byte[] ROW_1 = Bytes.toBytes("testrow1");
+  private static final byte[] COLUMN_1 = Bytes.toBytes("a");
+  private static final byte[] QUALIFIER_1 = Bytes.toBytes("1");
+  private static final byte[] VALUE_1 = Bytes.toBytes("testvalue1");
+
+  private Client client;
+  private RemoteHTable remoteTable;
+  
+  @Before
+  public void setup() throws Exception {
+    client = mock(Client.class);
+    Response response = new Response(509);
+    when(client.get(anyString(), anyString())).thenReturn(response);
+    when(client.delete(anyString())).thenReturn(response);
+    when(client.put(anyString(), anyString(), any(byte[].class))).thenReturn(
+        response);
+    when(client.post(anyString(), anyString(), any(byte[].class))).thenReturn(
+        response);
+
+    Configuration configuration = TEST_UTIL.getConfiguration();
+    configuration.setInt("hbase.rest.client.max.retries", RETRIES);
+    configuration.setInt("hbase.rest.client.sleep", SLEEP_TIME);
+
+    remoteTable = new RemoteHTable(client, TEST_UTIL.getConfiguration(),
+        "MyTable");
+  }
+
+  @After
+  public void tearDownAfterClass() throws Exception {
+    remoteTable.close();
+  }
+  
+  @Test
+  public void testDelete() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        Delete delete = new Delete(Bytes.toBytes("delete"));
+        remoteTable.delete(delete);
+      }
+    });
+    verify(client, times(RETRIES)).delete(anyString());
+  }
+  
+  @Test
+  public void testGet() throws Exception {
+    testTimedOutGetCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteTable.get(new Get(Bytes.toBytes("Get")));
+      }
+    });
+  }
+
+  @Test
+  public void testSingleRowPut() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteTable.put(new Put(Bytes.toBytes("Row")));
+      }
+    });
+    verify(client, times(RETRIES)).put(anyString(), anyString(), any(byte[].class));
+  }
+  
+  @Test
+  public void testMultiRowPut() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        Put[] puts = { new Put(Bytes.toBytes("Row1")),
+            new Put(Bytes.toBytes("Row2")) };
+        remoteTable.put(Arrays.asList(puts));
+      }
+    });
+    verify(client, times(RETRIES)).put(anyString(), anyString(), any(byte[].class));
+  }
+
+  @Test
+  public void testGetScanner() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        remoteTable.getScanner(new Scan());
+      }
+    });
+    verify(client, times(RETRIES)).post(anyString(), anyString(), any(byte[].class));
+  }
+  
+  @Test
+  public void testCheckAndPut() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        Put put = new Put(ROW_1);
+        put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+        remoteTable.checkAndPut(ROW_1, COLUMN_1, QUALIFIER_1, VALUE_1, put );
+      }
+    });
+    verify(client, times(RETRIES)).put(anyString(), anyString(), any(byte[].class));
+  }
+
+  @Test
+  public void testCheckAndDelete() throws Exception {
+    testTimedOutCall(new CallExecutor() {
+      @Override
+      public void run() throws Exception {
+        Put put = new Put(ROW_1);
+        put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+        Delete delete= new Delete(ROW_1);
+        remoteTable.checkAndDelete(ROW_1, COLUMN_1, QUALIFIER_1,  VALUE_1, delete );
+      }
+    });
+  }
+  
+  private void testTimedOutGetCall(CallExecutor callExecutor) throws Exception {
+    testTimedOutCall(callExecutor);
+    verify(client, times(RETRIES)).get(anyString(), anyString());
+  }
+  
+  private void testTimedOutCall(CallExecutor callExecutor) throws Exception {
+    long start = System.currentTimeMillis();
+    try {
+      callExecutor.run();
+      fail("should be timeout exception!");
+    } catch (IOException e) {
+      assertTrue(Pattern.matches(".*request timed out", e.toString()));
+    }
+    assertTrue((System.currentTimeMillis() - start) > MAX_TIME);
+  }
+
+  private static interface CallExecutor {
+    void run() throws Exception;
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index dad189f3b8bc..376947dff323 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -20,13 +20,24 @@
 
 package org.apache.hadoop.hbase.rest.client;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
+import org.apache.commons.httpclient.Header;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -36,20 +47,16 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.rest.HBaseRESTTestingUtility;
-import org.apache.hadoop.hbase.rest.client.Client;
-import org.apache.hadoop.hbase.rest.client.Cluster;
-import org.apache.hadoop.hbase.rest.client.RemoteHTable;
 import org.apache.hadoop.hbase.util.Bytes;
-
-import static org.junit.Assert.*;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 @Category(MediumTests.class)
 public class TestRemoteTable {
-  private static final Log LOG = LogFactory.getLog(TestRemoteTable.class);
   private static final String TABLE = "TestRemoteTable";
   private static final byte[] ROW_1 = Bytes.toBytes("testrow1");
   private static final byte[] ROW_2 = Bytes.toBytes("testrow2");
@@ -70,24 +77,29 @@ public class TestRemoteTable {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
     new HBaseRESTTestingUtility();
-  private static RemoteHTable remoteTable;
+  private RemoteHTable remoteTable;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.startMiniCluster();
     REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration());
+  }
+
+  @Before
+  public void before() throws Exception  {
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
-    LOG.info("Admin Connection=" + admin.getConnection() + ", " + 
-      admin.getConnection().getZooKeeperWatcher());
-    if (!admin.tableExists(TABLE)) {
-      HTableDescriptor htd = new HTableDescriptor(TABLE);
-      htd.addFamily(new HColumnDescriptor(COLUMN_1));
-      htd.addFamily(new HColumnDescriptor(COLUMN_2));
-      htd.addFamily(new HColumnDescriptor(COLUMN_3));
-      admin.createTable(htd);
-      HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
-      LOG.info("Table connection=" + table.getConnection() + ", " +
-        admin.getConnection().getZooKeeperWatcher());
+    if (admin.tableExists(TABLE)) {
+      if (admin.isTableEnabled(TABLE)) admin.disableTable(TABLE);
+      admin.deleteTable(TABLE);
+    }
+    HTableDescriptor htd = new HTableDescriptor(TABLE);
+    htd.addFamily(new HColumnDescriptor(COLUMN_1).setMaxVersions(3));
+    htd.addFamily(new HColumnDescriptor(COLUMN_2).setMaxVersions(3));
+    htd.addFamily(new HColumnDescriptor(COLUMN_3).setMaxVersions(3));
+    admin.createTable(htd);
+    HTable table = null;
+    try {
+      table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
       Put put = new Put(ROW_1);
       put.add(COLUMN_1, QUALIFIER_1, TS_2, VALUE_1);
       table.put(put);
@@ -97,25 +109,36 @@ public static void setUpBeforeClass() throws Exception {
       put.add(COLUMN_2, QUALIFIER_2, TS_2, VALUE_2);
       table.put(put);
       table.flushCommits();
+    } finally {
+      if (null != table) table.close();
     }
     remoteTable = new RemoteHTable(
       new Client(new Cluster().add("localhost", 
           REST_TEST_UTIL.getServletPort())),
         TEST_UTIL.getConfiguration(), TABLE);
   }
-
+  
+  @After
+  public void after() throws Exception {
+    remoteTable.close();
+  }
+  
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    remoteTable.close();
     REST_TEST_UTIL.shutdownServletContainer();
     TEST_UTIL.shutdownMiniCluster();
   }
 
   @Test
   public void testGetTableDescriptor() throws IOException {
-    HTableDescriptor local = new HTable(TEST_UTIL.getConfiguration(),
-      TABLE).getTableDescriptor();
-    assertEquals(remoteTable.getTableDescriptor(), local);
+    HTable table = null;
+    try {
+      table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
+      HTableDescriptor local = table.getTableDescriptor();
+      assertEquals(remoteTable.getTableDescriptor(), local);
+    } finally {
+      if (null != table) table.close();
+    }
   }
 
   @Test
@@ -301,6 +324,8 @@ public void testPut() throws IOException {
     value = result.getValue(COLUMN_2, QUALIFIER_2);
     assertNotNull(value);
     assertTrue(Bytes.equals(VALUE_2, value));
+
+    assertTrue(Bytes.equals(Bytes.toBytes("TestRemoteTable"), remoteTable.getTableName()));
   }
 
   @Test
@@ -362,6 +387,7 @@ public void testDelete() throws IOException {
     assertNull(value2);
   }
 
+  @Test
   public void testScanner() throws IOException {
     List<Put> puts = new ArrayList<Put>();
     Put put = new Put(ROW_1);
@@ -385,21 +411,123 @@ public void testScanner() throws IOException {
     assertEquals(1, results.length);
     assertTrue(Bytes.equals(ROW_1, results[0].getRow()));
 
-    results = scanner.next(3);
+    Result result = scanner.next();
+    assertNotNull(result);
+    assertTrue(Bytes.equals(ROW_2, result.getRow()));
+
+    results = scanner.next(2);
     assertNotNull(results);
-    assertEquals(3, results.length);
-    assertTrue(Bytes.equals(ROW_2, results[0].getRow()));
-    assertTrue(Bytes.equals(ROW_3, results[1].getRow()));
-    assertTrue(Bytes.equals(ROW_4, results[2].getRow()));
+    assertEquals(2, results.length);
+    assertTrue(Bytes.equals(ROW_3, results[0].getRow()));
+    assertTrue(Bytes.equals(ROW_4, results[1].getRow()));
 
     results = scanner.next(1);
     assertNull(results);
+    scanner.close();
 
+    scanner = remoteTable.getScanner(COLUMN_1);
+    results = scanner.next(4);
+    assertNotNull(results);
+    assertEquals(4, results.length);
+    assertTrue(Bytes.equals(ROW_1, results[0].getRow()));
+    assertTrue(Bytes.equals(ROW_2, results[1].getRow()));
+    assertTrue(Bytes.equals(ROW_3, results[2].getRow()));
+    assertTrue(Bytes.equals(ROW_4, results[3].getRow()));
+
+    scanner.close();
+
+    scanner = remoteTable.getScanner(COLUMN_1, QUALIFIER_1);
+    results = scanner.next(4);
+    assertNotNull(results);
+    assertEquals(4, results.length);
+    assertTrue(Bytes.equals(ROW_1, results[0].getRow()));
+    assertTrue(Bytes.equals(ROW_2, results[1].getRow()));
+    assertTrue(Bytes.equals(ROW_3, results[2].getRow()));
+    assertTrue(Bytes.equals(ROW_4, results[3].getRow()));
     scanner.close();
+    assertTrue(remoteTable.isAutoFlush());
+  }
+
+  @Test
+  public void testCheckAndDelete() throws IOException {
+    Get get = new Get(ROW_1);
+    Result result = remoteTable.get(get);
+    byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1);
+    byte[] value2 = result.getValue(COLUMN_2, QUALIFIER_2);
+    assertNotNull(value1);
+    assertTrue(Bytes.equals(VALUE_1, value1));
+    assertNull(value2);
+    assertTrue(remoteTable.exists(get));
+    assertEquals(1, remoteTable.get(Collections.singletonList(get)).length);
+    Delete delete = new Delete(ROW_1);
+
+    remoteTable.checkAndDelete(ROW_1, COLUMN_1, QUALIFIER_1, VALUE_1, delete);
+    assertFalse(remoteTable.exists(get));
+
+    Put put = new Put(ROW_1);
+    put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+    remoteTable.put(put);
+
+    assertTrue(remoteTable.checkAndPut(ROW_1, COLUMN_1, QUALIFIER_1, VALUE_1, put));
+    assertFalse(remoteTable.checkAndPut(ROW_1, COLUMN_1, QUALIFIER_1, VALUE_2, put));
+  }
+
+  /**
+   * Test RemoteHable.Scanner.iterator method  
+   */
+  @Test
+  public void testIteratorScaner() throws IOException {
+    List<Put> puts = new ArrayList<Put>();
+    Put put = new Put(ROW_1);
+    put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+    puts.add(put);
+    put = new Put(ROW_2);
+    put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+    puts.add(put);
+    put = new Put(ROW_3);
+    put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+    puts.add(put);
+    put = new Put(ROW_4);
+    put.add(COLUMN_1, QUALIFIER_1, VALUE_1);
+    puts.add(put);
+    remoteTable.put(puts);
+
+    ResultScanner scanner = remoteTable.getScanner(new Scan());
+    Iterator<Result> iterator = scanner.iterator();
+    assertTrue(iterator.hasNext());
+    int counter = 0;
+    while (iterator.hasNext()) {
+      iterator.next();
+      counter++;
+    }
+    assertEquals(4, counter);
+  }
+
+  /**
+   * Test a some methods of class Response.
+   */
+  @Test
+  public void testResponse(){
+    Response response = new Response(200);
+    assertEquals(200, response.getCode());
+    Header[] headers = new Header[2];
+    headers[0] = new Header("header1", "value1");
+    headers[1] = new Header("header2", "value2");
+    response = new Response(200, headers);
+    assertEquals("value1", response.getHeader("header1"));
+    assertFalse(response.hasBody());
+    response.setCode(404);
+    assertEquals(404, response.getCode());
+    headers = new Header[2];
+    headers[0] = new Header("header1", "value1.1");
+    headers[1] = new Header("header2", "value2");
+    response.setHeaders(headers);
+    assertEquals("value1.1", response.getHeader("header1"));
+    response.setBody(Bytes.toBytes("body"));
+    assertTrue(response.hasBody());    
   }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
-
+}
\ No newline at end of file

From b55a01f9b3f2e6d98a6d2507ce86d031ca71a71f Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 31 Oct 2013 22:19:38 +0000
Subject: [PATCH 1208/1540] HBASE-8559 increase unit-test coverage of package
 org.apache.hadoop.hbase.coprocessor

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1537739 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/coprocessor/TestHTableWrapper.java  | 323 ++++++++++++++++++
 1 file changed, 323 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/TestHTableWrapper.java

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestHTableWrapper.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestHTableWrapper.java
new file mode 100644
index 000000000000..fd83023116e2
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestHTableWrapper.java
@@ -0,0 +1,323 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests class {@link org.apache.hadoop.hbase.coprocessor.CoprocessorHost.Environment.HTableWrapper}
+ * by invoking its methods and briefly asserting the result is reasonable.
+ */
+@Category(MediumTests.class)
+public class TestHTableWrapper {
+
+  private static final HBaseTestingUtility util = new HBaseTestingUtility();
+
+  private static final byte[] TEST_TABLE = Bytes.toBytes("test");
+  private static final byte[] TEST_FAMILY = Bytes.toBytes("f1");
+
+  private static final byte[] ROW_A = Bytes.toBytes("aaa");
+  private static final byte[] ROW_B = Bytes.toBytes("bbb");
+  private static final byte[] ROW_C = Bytes.toBytes("ccc");
+  private static final byte[] ROW_D = Bytes.toBytes("ddd");
+  private static final byte[] ROW_E = Bytes.toBytes("eee");
+
+  private static final byte[] qualifierCol1 = Bytes.toBytes("col1");
+
+  private static final byte[] bytes1 = Bytes.toBytes(1);
+  private static final byte[] bytes2 = Bytes.toBytes(2);
+  private static final byte[] bytes3 = Bytes.toBytes(3);
+  private static final byte[] bytes4 = Bytes.toBytes(4);
+  private static final byte[] bytes5 = Bytes.toBytes(5);
+
+  static class DummyRegionObserver extends BaseRegionObserver {
+  }
+
+  private HTableInterface hTableInterface;
+  private HTable table;
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    util.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    util.shutdownMiniCluster();
+  }
+
+  @Before
+  public void before() throws Exception {
+    table = util.createTable(TEST_TABLE, TEST_FAMILY);
+
+    Put puta = new Put(ROW_A);
+    puta.add(TEST_FAMILY, qualifierCol1, bytes1);
+    table.put(puta);
+
+    Put putb = new Put(ROW_B);
+    putb.add(TEST_FAMILY, qualifierCol1, bytes2);
+    table.put(putb);
+
+    Put putc = new Put(ROW_C);
+    putc.add(TEST_FAMILY, qualifierCol1, bytes3);
+    table.put(putc);
+  }
+
+  @After
+  public void after() throws Exception {
+    try {
+      if (table != null) {
+        table.close();
+      }
+    } finally {
+      util.deleteTable(TEST_TABLE);
+    }
+  }
+
+  @Test
+  public void testHTableInterfaceMethods() throws Exception {
+    Configuration conf = util.getConfiguration();
+    MasterCoprocessorHost cpHost = util.getMiniHBaseCluster().getMaster().getCoprocessorHost();
+    Class<?> implClazz = DummyRegionObserver.class;
+    cpHost.load(implClazz, Coprocessor.PRIORITY_HIGHEST, conf);
+    CoprocessorEnvironment env = cpHost.findCoprocessorEnvironment(implClazz.getName());
+    assertEquals(Coprocessor.VERSION, env.getVersion());
+    assertEquals(VersionInfo.getVersion(), env.getHBaseVersion());
+    hTableInterface = env.getTable(TEST_TABLE);
+    checkHTableInterfaceMethods();
+    cpHost.shutdown(env);
+  }
+
+  private void checkHTableInterfaceMethods() throws Exception {
+    checkConf();
+    checkNameAndDescriptor();
+    checkAutoFlush();
+    checkBufferSize();
+    checkExists();
+    checkGetRowOrBefore();
+    checkAppend();
+    checkPutsAndDeletes();
+    checkCheckAndPut();
+    checkCheckAndDelete();
+    checkIncrementColumnValue();
+    checkIncrement();
+    checkBatch();
+    checkMutateRow();
+    checkResultScanner();
+
+    hTableInterface.flushCommits();
+    hTableInterface.close();
+  }
+
+  private void checkConf() {
+    Configuration confExpected = util.getConfiguration();
+    Configuration confActual = hTableInterface.getConfiguration();
+    assertTrue(confExpected == confActual);
+  }
+
+  private void checkNameAndDescriptor() throws IOException {
+    assertArrayEquals(TEST_TABLE, hTableInterface.getTableName());
+    assertEquals(table.getTableDescriptor(), hTableInterface.getTableDescriptor());
+  }
+
+  private void checkAutoFlush() {
+    boolean initialAutoFlush = hTableInterface.isAutoFlush();
+    hTableInterface.setAutoFlush(false);
+    assertFalse(hTableInterface.isAutoFlush());
+    hTableInterface.setAutoFlush(true, true);
+    assertTrue(hTableInterface.isAutoFlush());
+    hTableInterface.setAutoFlush(initialAutoFlush);
+  }
+
+  private void checkBufferSize() throws IOException {
+    long initialWriteBufferSize = hTableInterface.getWriteBufferSize();
+    hTableInterface.setWriteBufferSize(12345L);
+    assertEquals(12345L, hTableInterface.getWriteBufferSize());
+    hTableInterface.setWriteBufferSize(initialWriteBufferSize);
+  }
+
+  private void checkExists() throws IOException {
+    boolean ex = hTableInterface.exists(new Get(ROW_A).addColumn(TEST_FAMILY, qualifierCol1));
+    assertTrue(ex);
+  }
+
+  @SuppressWarnings("deprecation")
+  private void checkGetRowOrBefore() throws IOException {
+    Result rowOrBeforeResult = hTableInterface.getRowOrBefore(ROW_A, TEST_FAMILY);
+    assertArrayEquals(ROW_A, rowOrBeforeResult.getRow());
+  }
+
+  private void checkAppend() throws IOException {
+    final byte[] appendValue = Bytes.toBytes("append");
+    Append append = new Append(qualifierCol1).add(TEST_FAMILY, qualifierCol1, appendValue);
+    Result appendResult = hTableInterface.append(append);
+    byte[] appendedRow = appendResult.getRow();
+    checkRowValue(appendedRow, appendValue);
+  }
+
+  private void checkPutsAndDeletes() throws IOException {
+    // put:
+    Put putD = new Put(ROW_D).add(TEST_FAMILY, qualifierCol1, bytes2);
+    hTableInterface.put(putD);
+    checkRowValue(ROW_D, bytes2);
+
+    // delete:
+    Delete delete = new Delete(ROW_D);
+    hTableInterface.delete(delete);
+    checkRowValue(ROW_D, null);
+
+    // multiple puts:
+    Put[] puts = new Put[] { new Put(ROW_D).add(TEST_FAMILY, qualifierCol1, bytes2),
+        new Put(ROW_E).add(TEST_FAMILY, qualifierCol1, bytes3) };
+    hTableInterface.put(Arrays.asList(puts));
+    checkRowsValues(new byte[][] { ROW_D, ROW_E }, new byte[][] { bytes2, bytes3 });
+
+    // multiple deletes:
+    Delete[] deletes = new Delete[] { new Delete(ROW_D), new Delete(ROW_E) };
+    hTableInterface.delete(new ArrayList<Delete>(Arrays.asList(deletes)));
+    checkRowsValues(new byte[][] { ROW_D, ROW_E }, new byte[][] { null, null });
+  }
+
+  private void checkCheckAndPut() throws IOException {
+    Put putC = new Put(ROW_C).add(TEST_FAMILY, qualifierCol1, bytes5);
+    assertFalse(hTableInterface.checkAndPut(ROW_C, TEST_FAMILY, qualifierCol1, /* expect */
+        bytes4, putC/* newValue */));
+    assertTrue(hTableInterface.checkAndPut(ROW_C, TEST_FAMILY, qualifierCol1, /* expect */
+        bytes3, putC/* newValue */));
+    checkRowValue(ROW_C, bytes5);
+  }
+
+  private void checkCheckAndDelete() throws IOException {
+    Delete delete = new Delete(ROW_C);
+    assertFalse(hTableInterface.checkAndDelete(ROW_C, TEST_FAMILY, qualifierCol1, bytes4, delete));
+    assertTrue(hTableInterface.checkAndDelete(ROW_C, TEST_FAMILY, qualifierCol1, bytes5, delete));
+    checkRowValue(ROW_C, null);
+  }
+
+  private void checkIncrementColumnValue() throws IOException {
+    hTableInterface.put(new Put(ROW_A).add(TEST_FAMILY, qualifierCol1, Bytes.toBytes(1L)));
+    checkRowValue(ROW_A, Bytes.toBytes(1L));
+
+    final long newVal = hTableInterface
+        .incrementColumnValue(ROW_A, TEST_FAMILY, qualifierCol1, 10L);
+    assertEquals(11L, newVal);
+    checkRowValue(ROW_A, Bytes.toBytes(11L));
+  }
+
+  private void checkIncrement() throws IOException {
+    hTableInterface.increment(new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, -15L));
+    checkRowValue(ROW_A, Bytes.toBytes(-4L));
+  }
+
+  private void checkBatch() throws IOException, InterruptedException {
+    Object[] results1 = hTableInterface.batch(Arrays.asList(new Row[] {
+        new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 2L),
+        new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 2L) }));
+    assertEquals(2, results1.length);
+    for (Object r2 : results1) {
+      assertTrue(r2 instanceof Result);
+    }
+    checkRowValue(ROW_A, Bytes.toBytes(0L));
+    Object[] results2 = new Result[2];
+    hTableInterface.batch(
+        Arrays.asList(new Row[] { new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 2L),
+            new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 2L) }), results2);
+    for (Object r2 : results2) {
+      assertTrue(r2 instanceof Result);
+    }
+    checkRowValue(ROW_A, Bytes.toBytes(4L));
+  }
+
+  private void checkMutateRow() throws IOException {
+    Put put = new Put(ROW_A).add(TEST_FAMILY, qualifierCol1, bytes1);
+    RowMutations rowMutations = new RowMutations(ROW_A);
+    rowMutations.add(put);
+    hTableInterface.mutateRow(rowMutations);
+    checkRowValue(ROW_A, bytes1);
+  }
+
+  private void checkResultScanner() throws IOException {
+    ResultScanner resultScanner = hTableInterface.getScanner(TEST_FAMILY);
+    Result[] results = resultScanner.next(10);
+    assertEquals(3, results.length);
+
+    resultScanner = hTableInterface.getScanner(TEST_FAMILY, qualifierCol1);
+    results = resultScanner.next(10);
+    assertEquals(3, results.length);
+
+    resultScanner = hTableInterface.getScanner(new Scan(ROW_A, ROW_C));
+    results = resultScanner.next(10);
+    assertEquals(2, results.length);
+  }
+
+  private void checkRowValue(byte[] row, byte[] expectedValue) throws IOException {
+    Get get = new Get(row).addColumn(TEST_FAMILY, qualifierCol1);
+    Result result = hTableInterface.get(get);
+    byte[] actualValue = result.getValue(TEST_FAMILY, qualifierCol1);
+    assertArrayEquals(expectedValue, actualValue);
+  }
+
+  private void checkRowsValues(byte[][] rows, byte[][] expectedValues) throws IOException {
+    if (rows.length != expectedValues.length) {
+      throw new IllegalArgumentException();
+    }
+    Get[] gets = new Get[rows.length];
+    for (int i = 0; i < gets.length; i++) {
+      gets[i] = new Get(rows[i]).addColumn(TEST_FAMILY, qualifierCol1);
+    }
+    Result[] results = hTableInterface.get(Arrays.asList(gets));
+    for (int i = 0; i < expectedValues.length; i++) {
+      byte[] actualValue = results[i].getValue(TEST_FAMILY, qualifierCol1);
+      assertArrayEquals(expectedValues[i], actualValue);
+    }
+  }
+
+}

From e97a20f8b1297b63e326d0f494a8395ae8773d3d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 4 Nov 2013 01:27:02 +0000
Subject: [PATCH 1209/1540] HBASE-8942 DFS errors during a read operation
 (get/scan), may cause write outliers (Amitanand Aiyer)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1538484 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Store.java      | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 04c12c5276ec..03dc653c2d86 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -2200,20 +2200,15 @@ boolean getForceMajorCompaction() {
    * @throws IOException
    */
   public KeyValueScanner getScanner(Scan scan,
-      final NavigableSet<byte []> targetCols) throws IOException {
-    lock.readLock().lock();
-    try {
-      KeyValueScanner scanner = null;
-      if (getHRegion().getCoprocessorHost() != null) {
-        scanner = getHRegion().getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
-      }
-      if (scanner == null) {
-        scanner = new StoreScanner(this, getScanInfo(), scan, targetCols);
-      }
-      return scanner;
-    } finally {
-      lock.readLock().unlock();
+    final NavigableSet<byte []> targetCols) throws IOException {
+    KeyValueScanner scanner = null;
+    if (getHRegion().getCoprocessorHost() != null) {
+      scanner = getHRegion().getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
+    }
+    if (scanner == null) {
+      scanner = new StoreScanner(this, getScanInfo(), scan, targetCols);
     }
+    return scanner;
   }
 
   @Override

From f14f42ec5b6324f37ba1bcc0401d4066fb9f1f88 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 5 Nov 2013 05:31:13 +0000
Subject: [PATCH 1210/1540] HBASE-8942 DFS errors during a read operation
 (get/scan), may cause write outliers; REVERT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1538869 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Store.java      | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 03dc653c2d86..04c12c5276ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -2200,15 +2200,20 @@ boolean getForceMajorCompaction() {
    * @throws IOException
    */
   public KeyValueScanner getScanner(Scan scan,
-    final NavigableSet<byte []> targetCols) throws IOException {
-    KeyValueScanner scanner = null;
-    if (getHRegion().getCoprocessorHost() != null) {
-      scanner = getHRegion().getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
-    }
-    if (scanner == null) {
-      scanner = new StoreScanner(this, getScanInfo(), scan, targetCols);
+      final NavigableSet<byte []> targetCols) throws IOException {
+    lock.readLock().lock();
+    try {
+      KeyValueScanner scanner = null;
+      if (getHRegion().getCoprocessorHost() != null) {
+        scanner = getHRegion().getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
+      }
+      if (scanner == null) {
+        scanner = new StoreScanner(this, getScanInfo(), scan, targetCols);
+      }
+      return scanner;
+    } finally {
+      lock.readLock().unlock();
     }
-    return scanner;
   }
 
   @Override

From edf689e91d41b8cd8d9c083eb245744ce3838fac Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 8 Nov 2013 03:09:58 +0000
Subject: [PATCH 1211/1540] HBASE-9906 Restore snapshot fails to restore the
 meta edits sporadically

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1539910 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/catalog/MetaEditor.java      | 19 +++++++++++++++++++
 .../snapshot/RestoreSnapshotHandler.java      |  3 +--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
index 94ffeb605092..0d75532fab7a 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
 
 /**
@@ -357,6 +358,24 @@ public static void mutateRegions(CatalogTracker catalogTracker,
     }
   }
 
+  /**
+   * Overwrites the specified regions from hbase:meta
+   * @param catalogTracker
+   * @param regionInfos list of regions to be added to META
+   * @throws IOException
+   */
+  public static void overwriteRegions(CatalogTracker catalogTracker,
+      List<HRegionInfo> regionInfos) throws IOException {
+    deleteRegions(catalogTracker, regionInfos);
+    // Why sleep? This is the easiest way to ensure that the previous deletes does not
+    // eclipse the following puts, that might happen in the same ts from the server.
+    // See HBASE-9906, and HBASE-9879. Once either HBASE-9879, HBASE-8770 is fixed,
+    // or HBASE-9905 is fixed and meta uses seqIds, we do not need the sleep.
+    Threads.sleep(20);
+    addRegionsToMeta(catalogTracker, regionInfos);
+    LOG.info("Overwritten " + regionInfos);
+  }
+
   public static HRegionInfo getHRegionInfo(
       Result data) throws IOException {
     byte [] bytes =
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index fcb7067730b5..837c47048bb1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -141,7 +141,6 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
       // that are not correct after the restore.
       List<HRegionInfo> hrisToRemove = new LinkedList<HRegionInfo>();
       if (metaChanges.hasRegionsToRemove()) hrisToRemove.addAll(metaChanges.getRegionsToRemove());
-      if (metaChanges.hasRegionsToRestore()) hrisToRemove.addAll(metaChanges.getRegionsToRestore());
       MetaEditor.deleteRegions(catalogTracker, hrisToRemove);
 
       // 4.2 Add the new set of regions to META
@@ -152,8 +151,8 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
       // in the snapshot folder.
       hris.clear();
       if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
-      if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
       MetaEditor.addRegionsToMeta(catalogTracker, hris);
+      MetaEditor.overwriteRegions(catalogTracker, metaChanges.getRegionsToRestore());
       metaChanges.updateMetaParentRegions(catalogTracker, hris);
 
       // At this point the restore is complete. Next step is enabling the table.

From 3001ee9fd53e4c7cdd2baef958fd85090c2606ef Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 8 Nov 2013 07:29:07 +0000
Subject: [PATCH 1212/1540] HBASE-9915 Performance: isSeeked() in
 EncodedScannerV2 always returns false

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1539936 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java   | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 09fd912b8896..62a6746239dd 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -972,6 +972,11 @@ private void setDataBlockEncoder(DataBlockEncoder dataBlockEncoder) {
           includesMemstoreTS);
     }
 
+    @Override
+    public boolean isSeeked(){
+      return this.block != null;
+    }
+
     /**
      * Updates the current block to be the given {@link HFileBlock}. Seeks to
      * the the first key/value pair.
@@ -998,6 +1003,9 @@ private void updateCurrentBlock(HFileBlock newBlock) {
 
       seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
       blockFetches++;
+
+      // Reset the next indexed key
+      this.nextIndexedKey = null;
     }
 
     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {

From 0221e28b701ff1bc55986eee0acaf887ce9c5d85 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sat, 9 Nov 2013 01:44:13 +0000
Subject: [PATCH 1213/1540] HBASE-9890 MR jobs are not working if started by a
 delegated user

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1540241 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapred/TableMapReduceUtil.java      | 39 ++++++++++++++-
 .../mapreduce/LoadIncrementalHFiles.java      | 13 ++++-
 .../hbase/mapreduce/TableMapReduceUtil.java   | 48 +++++++++++++++++--
 .../apache/hadoop/hbase/security/User.java    | 20 ++++++++
 4 files changed, 113 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index 6aa03261bd45..0c7d47da0c9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -21,12 +21,15 @@
 
 import java.io.IOException;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.zookeeper.ClusterId;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.FileInputFormat;
@@ -35,6 +38,8 @@
 import org.apache.hadoop.mapred.OutputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.security.token.Token;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * Utility for {@link TableMap} and {@link TableReduce}
@@ -171,9 +176,24 @@ public static void initTableReduceJob(String table,
 
   public static void initCredentials(JobConf job) throws IOException {
     UserProvider provider = UserProvider.instantiate(job);
+
+    if (provider.isHadoopSecurityEnabled()) {
+      // propagate delegation related props from launcher job to MR job
+      if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
+        job.set("mapreduce.job.credentials.binary",
+                System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
+      }
+    }
+
     if (provider.isHBaseSecurityEnabled()) {
       try {
-        provider.getCurrent().obtainAuthTokenForJob(job);
+        User user = provider.getCurrent();
+        Token<?> authToken = getAuthToken(job, user);
+        if (authToken == null) {
+          user.obtainAuthTokenForJob(job);
+        } else {
+          job.getCredentials().addToken(authToken.getService(), authToken);
+        }
       } catch (InterruptedException ie) {
         ie.printStackTrace();
         Thread.interrupted();
@@ -181,6 +201,23 @@ public static void initCredentials(JobConf job) throws IOException {
     }
   }
 
+  /**
+   * Get the authentication token of the user for the cluster specified in the configuration
+   * @return null if the user does not have the token, otherwise the auth token for the cluster.
+   */
+  private static Token<?> getAuthToken(Configuration conf, User user)
+      throws IOException, InterruptedException {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null);
+    try {
+      String clusterId = ClusterId.readClusterIdZNode(zkw);
+      return user.getToken("HBASE_AUTH_TOKEN", clusterId);
+    } catch (KeeperException e) {
+      throw new IOException(e);
+    } finally {
+      zkw.close();
+    }
+  }
+
   /**
    * Ensures that the given number of reduce tasks for the given job
    * configuration does not exceed the number of regions for the given table.
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 7b459abd2eaa..217d87d93d35 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -101,6 +101,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
   public static String ASSIGN_SEQ_IDS = "hbase.mapreduce.bulkload.assign.sequenceNumbers";
 
   private boolean useSecure;
+  private boolean hasForwardedToken;
   private Token<?> userToken;
   private String bulkToken;
   private final boolean assignSeqIds;
@@ -232,7 +233,15 @@ public void doBulkLoad(Path hfofDir, final HTable table)
         //Since delegation token doesn't work in mini cluster
         if (userProvider.isHadoopSecurityEnabled()) {
           FileSystem fs = FileSystem.get(cfg);
-          userToken = fs.getDelegationToken("renewer");
+          userToken = userProvider.getCurrent().getToken("HDFS_DELEGATION_TOKEN",
+                                                         fs.getCanonicalServiceName());
+          if (userToken == null) {
+            hasForwardedToken = false;
+            userToken = fs.getDelegationToken("renewer");
+          } else {
+            hasForwardedToken = true;
+            LOG.info("Use the existing token: " + userToken);
+          }
         }
         bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getTableName());
       }
@@ -266,7 +275,7 @@ public void doBulkLoad(Path hfofDir, final HTable table)
 
     } finally {
       if(useSecure) {
-        if(userToken != null) {
+        if(userToken != null && !hasForwardedToken) {
           try {
             userToken.cancel(cfg);
           } catch (Exception e) {
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 00e03eba0028..954d0a16fad5 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -53,12 +53,16 @@
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.security.token.Token;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * Utility for {@link TableMapper} and {@link TableReducer}
@@ -284,8 +288,17 @@ public static void initTableMapperJob(List<Scan> scans,
     }
   }
 
-  public static void initCredentials(Job job) throws IOException {
+public static void initCredentials(Job job) throws IOException {
     UserProvider provider = UserProvider.instantiate(job.getConfiguration());
+
+    if (provider.isHadoopSecurityEnabled()) {
+      // propagate delegation related props from launcher job to MR job
+      if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
+        job.getConfiguration().set("mapreduce.job.credentials.binary",
+                                   System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
+      }
+    }
+
     if (provider.isHBaseSecurityEnabled()) {
       try {
         // init credentials for remote cluster
@@ -299,10 +312,10 @@ public static void initCredentials(Job job) throws IOException {
           peerConf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
           peerConf.set("hbase.zookeeper.client.port", parts[1]);
           peerConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
-          user.obtainAuthTokenForJob(peerConf, job);
+          obtainAuthTokenForJob(job, peerConf, user);
         }
-        
-        user.obtainAuthTokenForJob(job.getConfiguration(), job);
+
+        obtainAuthTokenForJob(job, job.getConfiguration(), user);
       } catch (InterruptedException ie) {
         LOG.info("Interrupted obtaining user authentication token");
         Thread.interrupted();
@@ -310,6 +323,33 @@ public static void initCredentials(Job job) throws IOException {
     }
   }
 
+  private static void obtainAuthTokenForJob(Job job, Configuration conf, User user)
+      throws IOException, InterruptedException {
+    Token<?> authToken = getAuthToken(conf, user);
+    if (authToken == null) {
+      user.obtainAuthTokenForJob(conf, job);
+    } else {
+      job.getCredentials().addToken(authToken.getService(), authToken);
+    }
+  }
+
+  /**
+   * Get the authentication token of the user for the cluster specified in the configuration
+   * @return null if the user does not have the token, otherwise the auth token for the cluster.
+   */
+  private static Token<?> getAuthToken(Configuration conf, User user)
+      throws IOException, InterruptedException {
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null);
+    try {
+      String clusterId = ClusterId.readClusterIdZNode(zkw);
+      return user.getToken("HBASE_AUTH_TOKEN", clusterId);
+    } catch (KeeperException e) {
+      throw new IOException(e);
+    } finally {
+      zkw.close();
+    }
+  }
+
   /**
    * Writes the given scan into a Base64 encoded string.
    *
diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index 207536bd3c14..c0eb3a51a863 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hbase.util.Methods;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.UserGroupInformation;
 
 import java.io.IOException;
@@ -127,6 +128,25 @@ public abstract void obtainAuthTokenForJob(Configuration conf, Job job)
   public abstract void obtainAuthTokenForJob(JobConf job)
       throws IOException, InterruptedException;
 
+  /**
+   * Returns the Token of the specified kind associated with this user,
+   * or null if the Token is not present.
+   *
+   * @param kind the kind of token
+   * @param service service on which the token is supposed to be used
+   * @return the token of the specified kind.
+   */
+  public Token<?> getToken(String kind, String service) throws IOException {
+    for (Token<?> token: ugi.getTokens()) {
+      if (token.getKind().toString().equals(kind) &&
+          (service != null && token.getService().toString().equals(service)))
+      {
+        return token;
+      }
+    }
+    return null;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {

From 4012942970b27a69ae6337ab7eca3fd8dc63dec0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 9 Nov 2013 06:38:27 +0000
Subject: [PATCH 1214/1540] roll version to 0.94.14-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1540261 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index dda0e0254897..2e3b67ce0e5c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.13</version>
+  <version>0.94.14-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 99668889c8e96aaae34375e9001e4209e1d6d1e8 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 10 Nov 2013 04:04:24 +0000
Subject: [PATCH 1215/1540] HBASE-9809 RegionTooBusyException should provide
 region name which was too busy

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1540442 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 450ea8c5c944..4b731b40a809 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5920,7 +5920,11 @@ private void lock(final Lock lock, final int multiplier)
         busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
       if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
         throw new RegionTooBusyException(
-          "failed to get a lock in " + waitTime + "ms");
+          "failed to get a lock in " + waitTime + " ms. " +
+            "regionName=" + (this.getRegionInfo() == null ? "unknown" :
+            this.getRegionInfo().getRegionNameAsString()) +
+            ", server=" + (this.getRegionServerServices() == null ? "unknown" :
+            this.getRegionServerServices().getServerName()));
       }
     } catch (InterruptedException ie) {
       LOG.info("Interrupted while waiting for a lock");

From 3a8db682ca60f2a2b6d70b316c3b8d7fa8b5a31e Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Tue, 12 Nov 2013 01:33:51 +0000
Subject: [PATCH 1216/1540] HBASE-9952 Snapshot restore may fail due to
 NullPointerException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1540912 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index 837c47048bb1..fcd8e2418f49 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -152,7 +152,9 @@ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
       hris.clear();
       if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
       MetaEditor.addRegionsToMeta(catalogTracker, hris);
-      MetaEditor.overwriteRegions(catalogTracker, metaChanges.getRegionsToRestore());
+      if (metaChanges.hasRegionsToRestore()) {
+        MetaEditor.overwriteRegions(catalogTracker, metaChanges.getRegionsToRestore());
+      }
       metaChanges.updateMetaParentRegions(catalogTracker, hris);
 
       // At this point the restore is complete. Next step is enabling the table.

From de9b1c90c8286e5799560ccd7b9a97230a22bd01 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 12 Nov 2013 05:01:03 +0000
Subject: [PATCH 1217/1540] HBASE-9850 Issues with UI for table compact/split
 operation completion. After split/compaction operation using UI, the page is
 not automatically redirecting back using IE8/Firefox.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1540948 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/resources/hbase-webapps/master/snapshot.jsp | 6 +++++-
 src/main/resources/hbase-webapps/master/table.jsp    | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/main/resources/hbase-webapps/master/snapshot.jsp b/src/main/resources/hbase-webapps/master/snapshot.jsp
index ab007e90c47b..f77c26d101bf 100644
--- a/src/main/resources/hbase-webapps/master/snapshot.jsp
+++ b/src/main/resources/hbase-webapps/master/snapshot.jsp
@@ -65,7 +65,11 @@
 <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
 <% if (isActionResultPage) { %>
   <title>HBase Master: <%= master.getServerName() %></title>
-  <meta http-equiv="refresh" content="5,javascript:history.back()" />
+  <script type="text/javascript">
+  <!--
+  setTimeout("history.back()",5000);
+  -->
+  </script>
 <% } else { %>
   <title>Snapshot: <%= snapshotName %></title>
 <% } %>
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index a2e20103f3a2..52b2950a3087 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -70,7 +70,11 @@
 %>
 <head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
 <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
-<meta http-equiv="refresh" content="5,javascript:history.back()" />
+  <script type="text/javascript">
+  <!--
+  setTimeout("history.back()",5000);
+  -->
+  </script>
 </head>
 <body>
 <a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo.png" alt="HBase Logo" title="HBase Logo" /></a>

From c6eef2d2a14a33bfdb8bda8b3e7c6079d42f2e53 Mon Sep 17 00:00:00 2001
From: rajeshbabu <rajeshbabu@unknown>
Date: Tue, 12 Nov 2013 15:47:47 +0000
Subject: [PATCH 1218/1540] HBASE-9902 Region Server is starting normally even
 if clock skew is more than default 30 seconds(or any configured). ->
 Regionserver node time is greater than master node time(Kashif)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541115 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/ServerManager.java    |  2 +-
 .../hbase/master/TestClockSkewDetection.java  | 35 ++++++++++++++-----
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 1b6d29e3ca05..1391b6b409f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -226,7 +226,7 @@ void checkAlreadySameHostPort(final ServerName serverName)
    */
   private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
   throws ClockOutOfSyncException {
-    long skew = System.currentTimeMillis() - serverCurrentTime;
+    long skew = Math.abs(System.currentTimeMillis() - serverCurrentTime);
     if (skew > maxSkew) {
       String message = "Server " + serverName + " has been " +
         "rejected; Reported time is too far out of sync with master.  " +
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
index 7899bf095526..e1a07c8a3cfd 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
@@ -89,19 +89,38 @@ public void stop(String why) {
     long warningSkew = c.getLong("hbase.master.warningclockskew", 1000);
 
     try {
+      // Master Time > Region Server Time
+      LOG.debug("Test: Master Time > Region Server Time");
       LOG.debug("regionServerStartup 2");
       InetAddress ia2 = InetAddress.getLocalHost();
       sm.regionServerStartup(ia2, 1235, -1, System.currentTimeMillis() - maxSkew * 2);
-      fail("HMaster should have thrown an ClockOutOfSyncException but didn't.");
-    } catch(ClockOutOfSyncException e) {
-      //we want an exception
-      LOG.info("Recieved expected exception: "+e);
+      fail("HMaster should have thrown a ClockOutOfSyncException but didn't.");
+    } catch (ClockOutOfSyncException e) {
+      // we want an exception
+      LOG.info("Recieved expected exception: " + e);
     }
-    
+
+    try {
+      // Master Time < Region Server Time
+      LOG.debug("Test: Master Time < Region Server Time");
+      LOG.debug("regionServerStartup 3");
+      InetAddress ia3 = InetAddress.getLocalHost();
+      sm.regionServerStartup(ia3, 1236, -1, System.currentTimeMillis() + maxSkew * 2);
+      fail("HMaster should have thrown a ClockOutOfSyncException but didn't.");
+    } catch (ClockOutOfSyncException e) {
+      // we want an exception
+      LOG.info("Recieved expected exception: " + e);
+    }
+
+    // make sure values above warning threshold but below max threshold don't kill
+    LOG.debug("regionServerStartup 4");
+    InetAddress ia4 = InetAddress.getLocalHost();
+    sm.regionServerStartup(ia4, 1237, -1, System.currentTimeMillis() - warningSkew * 2);
+
     // make sure values above warning threshold but below max threshold don't kill
-    LOG.debug("regionServerStartup 3");
-    InetAddress ia3 = InetAddress.getLocalHost();
-    sm.regionServerStartup(ia3, 1236, -1, System.currentTimeMillis() - warningSkew * 2);
+    LOG.debug("regionServerStartup 5");
+    InetAddress ia5 = InetAddress.getLocalHost();
+    sm.regionServerStartup(ia5, 1238, -1, System.currentTimeMillis() + warningSkew * 2);
     
   }
 

From 13734d0b759dac729769df58b93c60765538b7f4 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 12 Nov 2013 22:14:32 +0000
Subject: [PATCH 1219/1540] HBASE-9865 Reused WALEdits in replication may cause
 RegionServers to go OOM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541269 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/WALEdit.java       |  7 ++-
 .../ReplicationHLogReaderManager.java         |  7 +--
 .../regionserver/ReplicationSource.java       | 55 ++++++++-----------
 3 files changed, 29 insertions(+), 40 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index eb4a59f4164f..bd3e33ffd7f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -84,7 +84,7 @@ public class WALEdit implements Writable, HeapSize {
   private static final String PREFIX_CLUSTER_KEY = ".";
   private final int VERSION_2 = -1;
 
-  private final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
+  private final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1);
 
   /**
    * This variable contains the information of the column family replication settings and contains
@@ -132,7 +132,7 @@ public int size() {
     return kvs.size();
   }
 
-  public List<KeyValue> getKeyValues() {
+  public ArrayList<KeyValue> getKeyValues() {
     return kvs;
   }
 
@@ -218,6 +218,7 @@ public void readFields(DataInput in) throws IOException {
     }
 
     // read in all the key values
+    kvs.ensureCapacity(length);
     for(int i=0; i< length && decoder.advance(); i++) {
       kvs.add(decoder.current());
     }
@@ -254,7 +255,7 @@ public void write(DataOutput out) throws IOException {
   }
 
   public long heapSize() {
-    long ret = 0;
+    long ret = ClassSize.ARRAYLIST;
     for (KeyValue kv : kvs) {
       ret += kv.heapSize();
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
index bd8f44d16cdc..ed8424592ae8 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java
@@ -80,14 +80,11 @@ public HLog.Reader openReader(Path path) throws IOException {
 
   /**
    * Get the next entry, returned and also added in the array
-   * @param entriesArray
-   * @param currentNbEntries
    * @return a new entry or null
    * @throws IOException
    */
-  public HLog.Entry readNextAndSetPosition(HLog.Entry[] entriesArray,
-                                           int currentNbEntries) throws IOException {
-    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
+  public HLog.Entry readNextAndSetPosition() throws IOException {
+    HLog.Entry entry = this.reader.next();
     // Store the position so that in the future the reader can start
     // reading from here. If the above call to next() throws an
     // exception, the position won't be changed and retry will happen
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 00db9e4a8f2a..c72a1fd47f35 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -25,7 +25,6 @@
 import java.net.ConnectException;
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -79,8 +78,6 @@ public class ReplicationSource extends Thread
   private static final Log LOG = LogFactory.getLog(ReplicationSource.class);
   // Queue of logs to process
   private PriorityBlockingQueue<Path> queue;
-  // container of entries to replicate
-  private HLog.Entry[] entriesArray;
   private HConnection conn;
   // Helper class for zookeeper
   private ReplicationZookeeper zkHelper;
@@ -127,8 +124,6 @@ public class ReplicationSource extends Thread
   private int maxRetriesMultiplier;
   // Socket timeouts require even bolder actions since we don't want to DDOS
   private int socketTimeoutMultiplier;
-  // Current number of entries that we need to replicate
-  private int currentNbEntries = 0;
   // Current number of operations (Put/Delete) that we need to replicate
   private int currentNbOperations = 0;
   // Current size of data we need to replicate
@@ -165,10 +160,6 @@ public void init(final Configuration conf,
         this.conf.getLong("replication.source.size.capacity", 1024*1024*64);
     this.replicationQueueNbCapacity =
         this.conf.getInt("replication.source.nb.capacity", 25000);
-    this.entriesArray = new HLog.Entry[this.replicationQueueNbCapacity];
-    for (int i = 0; i < this.replicationQueueNbCapacity; i++) {
-      this.entriesArray[i] = new HLog.Entry();
-    }
     this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 10);
     this.socketTimeoutMultiplier = this.conf.getInt("replication.source.socketTimeoutMultiplier",
         maxRetriesMultiplier * maxRetriesMultiplier);
@@ -382,10 +373,10 @@ public void run() {
 
       boolean gotIOE = false;
       currentNbOperations = 0;
-      currentNbEntries = 0;
+      List<HLog.Entry> entries = new ArrayList<HLog.Entry>(1);
       currentSize = 0;
       try {
-        if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) {
+        if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo, entries)) {
           continue;
         }
       } catch (IOException ioe) {
@@ -404,11 +395,6 @@ public void run() {
             } catch (IOException e) {
               LOG.warn(peerClusterZnode + " Got while getting file size: ", e);
             }
-          } else if (currentNbEntries != 0) {
-            LOG.warn(peerClusterZnode + " Got EOF while reading, " +
-                "looks like this file is broken? " + currentPath);
-            considerDumping = true;
-            currentNbEntries = 0;
           }
 
           if (considerDumping &&
@@ -430,7 +416,7 @@ public void run() {
       // If we didn't get anything to replicate, or if we hit a IOE,
       // wait a bit and retry.
       // But if we need to stop, don't bother sleeping
-      if (this.isActive() && (gotIOE || currentNbEntries == 0)) {
+      if (this.isActive() && (gotIOE || entries.isEmpty())) {
         if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
               this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
@@ -442,8 +428,7 @@ public void run() {
         continue;
       }
       sleepMultiplier = 1;
-      shipEdits(currentWALisBeingWrittenTo);
-
+      shipEdits(currentWALisBeingWrittenTo, entries);
     }
     if (this.conn != null) {
       try {
@@ -459,16 +444,17 @@ public void run() {
    * Read all the entries from the current log files and retain those
    * that need to be replicated. Else, process the end of the current file.
    * @param currentWALisBeingWrittenTo is the current WAL being written to
+   * @param entries resulting entries to be replicated
    * @return true if we got nothing and went to the next file, false if we got
    * entries
    * @throws IOException
    */
-  protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo)
+  protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo, List<HLog.Entry> entries)
       throws IOException{
     long seenEntries = 0;
     this.repLogReader.seek();
     HLog.Entry entry =
-        this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
+        this.repLogReader.readNextAndSetPosition();
     while (entry != null) {
       WALEdit edit = entry.getEdit();
       this.metrics.logEditsReadRate.inc(1);
@@ -499,7 +485,7 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
             edit.addClusterId(clusterId);
           }
           currentNbOperations += countDistinctRowKeys(edit);
-          currentNbEntries++;
+          entries.add(entry);
           currentSize += entry.getEdit().heapSize();
         } else {
           this.metrics.logEditsFilteredRate.inc(1);
@@ -507,11 +493,11 @@ protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingW
       }
       // Stop if too many entries or too big
       if (currentSize >= this.replicationQueueSizeCapacity ||
-          currentNbEntries >= this.replicationQueueNbCapacity) {
+          entries.size() >= this.replicationQueueNbCapacity) {
         break;
       }
       try {
-        entry = this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
+        entry = this.repLogReader.readNextAndSetPosition();
       } catch (IOException ie) {
         LOG.debug("Break on IOE: " + ie.getMessage());
         break;
@@ -670,8 +656,9 @@ protected boolean sleepForRetries(String msg, int sleepMultiplier) {
    * @param edit The KV to check for replication
    */
   protected void removeNonReplicableEdits(WALEdit edit) {
-    List<KeyValue> kvs = edit.getKeyValues();
-    for (int i = edit.size()-1; i >= 0; i--) {
+    ArrayList<KeyValue> kvs = edit.getKeyValues();
+    int size = edit.size();
+    for (int i = size-1; i >= 0; i--) {
       KeyValue kv = kvs.get(i);
       // The scope will be null or empty if
       // there's nothing to replicate in that WALEdit
@@ -679,6 +666,9 @@ protected void removeNonReplicableEdits(WALEdit edit) {
         kvs.remove(i);
       }
     }
+    if (edit.size() < size/2) {
+      kvs.trimToSize();
+    }
   }
 
   /**
@@ -704,9 +694,9 @@ private int countDistinctRowKeys(WALEdit edit) {
    * @param currentWALisBeingWrittenTo was the current WAL being (seemingly) 
    * written to when this method was called
    */
-  protected void shipEdits(boolean currentWALisBeingWrittenTo) {
+  protected void shipEdits(boolean currentWALisBeingWrittenTo, List<HLog.Entry> entries) {
     int sleepMultiplier = 1;
-    if (this.currentNbEntries == 0) {
+    if (entries.isEmpty()) {
       LOG.warn("Was given 0 edits to ship");
       return;
     }
@@ -719,19 +709,20 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo) {
       }
       try {
         HRegionInterface rrs = getRS();
-        LOG.debug("Replicating " + currentNbEntries);
-        rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries));
+        LOG.debug("Replicating " + entries.size());
+        // can't avoid the copy here, the replicateLogEntries RPC require an HLog.Entry[]
+        rrs.replicateLogEntries(entries.toArray(new HLog.Entry[entries.size()]));
         if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
               this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
           this.lastLoggedPosition = this.repLogReader.getPosition();
         }
-        this.totalReplicatedEdits += currentNbEntries;
+        this.totalReplicatedEdits += entries.size();
         this.metrics.shippedBatchesRate.inc(1);
         this.metrics.shippedOpsRate.inc(
             this.currentNbOperations);
         this.metrics.setAgeOfLastShippedOp(
-            this.entriesArray[currentNbEntries-1].getKey().getWriteTime());
+            entries.get(entries.size()-1).getKey().getWriteTime());
         LOG.debug("Replicated in total: " + this.totalReplicatedEdits);
         break;
 

From 0e5fe2aa77ce77aab3f7324b93374b42f57e9c52 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 12 Nov 2013 23:31:46 +0000
Subject: [PATCH 1220/1540] HBASE-9956 Remove keyLength cache from KeyValue

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541314 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     | 29 ++++---------------
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |  3 +-
 .../hbase/regionserver/ScanQueryMatcher.java  |  9 +++---
 3 files changed, 11 insertions(+), 30 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 915ad74d265c..0716ff27398c 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -64,7 +64,7 @@
  * be < <code>Integer.MAX_SIZE</code>.
  * The column does not contain the family/qualifier delimiter, {@link #COLUMN_FAMILY_DELIMITER}
  */
-public class KeyValue implements Writable, HeapSize {
+public class KeyValue implements Writable, HeapSize, Cloneable {
   static final Log LOG = LogFactory.getLog(KeyValue.class);
   // TODO: Group Key-only comparators and operations into a Key class, just
   // for neatness sake, if can figure what to call it.
@@ -278,21 +278,6 @@ public KeyValue(final byte [] bytes, final int offset, final int length) {
     this.length = length;
   }
 
-  /**
-   * Creates a KeyValue from the specified byte array, starting at offset,
-   * for length <code>length</code>, and a known <code>keyLength</code>.
-   * @param bytes byte array
-   * @param offset offset to start of the KeyValue
-   * @param length length of the KeyValue
-   * @param keyLength length of the key portion of the KeyValue
-   */
-  public KeyValue(final byte [] bytes, final int offset, final int length, final int keyLength) {
-    this.bytes = bytes;
-    this.offset = offset;
-    this.length = length;
-    this.keyLength = keyLength;
-  }
-
   /** Constructors that build a new backing byte array from fields */
 
   /**
@@ -632,6 +617,7 @@ static byte[] createEmptyByteArray(final int rlength, int flength,
 
   // Needed doing 'contains' on List.  Only compares the key portion, not the
   // value.
+  @Override
   public boolean equals(Object other) {
     if (!(other instanceof KeyValue)) {
       return false;
@@ -643,6 +629,7 @@ public boolean equals(Object other) {
       kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength());
   }
 
+  @Override
   public int hashCode() {
     byte[] b = getBuffer();
     int start = getOffset(), end = getOffset() + getLength();
@@ -834,13 +821,8 @@ public String getKeyString() {
   /**
    * @return Length of key portion.
    */
-  private int keyLength = 0;
-
   public int getKeyLength() {
-    if (keyLength == 0) {
-      keyLength = Bytes.toInt(this.bytes, this.offset);
-    }
-    return keyLength;
+    return Bytes.toInt(this.bytes, this.offset);
   }
 
   /**
@@ -2265,7 +2247,7 @@ int compareTimestamps(final long ltimestamp, final long rtimestamp) {
   public long heapSize() {
     return ClassSize.align(ClassSize.OBJECT + ClassSize.REFERENCE
         + ClassSize.align(ClassSize.ARRAY) + ClassSize.align(length)
-        + (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_LONG);
+        + (2 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_LONG);
   }
 
   // this overload assumes that the length bytes have already been read,
@@ -2274,7 +2256,6 @@ public long heapSize() {
   public void readFields(int length, final DataInput in) throws IOException {
     this.length = length;
     this.offset = 0;
-    this.keyLength = 0;
     this.bytes = new byte[this.length];
     in.readFully(this.bytes, 0, this.length);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 62a6746239dd..ad90663d0b51 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -638,8 +638,7 @@ public KeyValue getKeyValue() {
 
       KeyValue ret = new KeyValue(blockBuffer.array(),
           blockBuffer.arrayOffset() + blockBuffer.position(),
-          KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen,
-          currKeyLen);
+          KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen);
       if (this.reader.shouldIncludeMemstoreTS()) {
         ret.setMemstoreTS(currMemstoreTS);
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index ba11abd98c30..d50baf403899 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -219,11 +219,12 @@ public MatchCode match(KeyValue kv) throws IOException {
 
     byte [] bytes = kv.getBuffer();
     int offset = kv.getOffset();
-    int initialOffset = offset;
 
     int keyLength = Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
     offset += KeyValue.ROW_OFFSET;
 
+    int initialOffset = offset;
+
     short rowLength = Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT);
     offset += Bytes.SIZEOF_SHORT;
 
@@ -254,10 +255,10 @@ public MatchCode match(KeyValue kv) throws IOException {
     byte familyLength = bytes [offset];
     offset += familyLength + 1;
 
-    int qualLength = keyLength + KeyValue.ROW_OFFSET -
+    int qualLength = keyLength -
       (offset - initialOffset) - KeyValue.TIMESTAMP_TYPE_SIZE;
 
-    long timestamp = kv.getTimestamp();
+    long timestamp = Bytes.toLong(bytes, initialOffset + keyLength - KeyValue.TIMESTAMP_TYPE_SIZE);
     // check for early out based on timestamp alone
     if (columns.isDone(timestamp)) {
         return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
@@ -276,7 +277,7 @@ public MatchCode match(KeyValue kv) throws IOException {
      * 7. Delete marker need to be version counted together with puts
      *    they affect
      */
-    byte type = kv.getType();
+    byte type = bytes[initialOffset + keyLength - 1];
     if (kv.isDelete()) {
       if (!keepDeletedCells) {
         // first ignore delete markers if the scanner can do so, and the

From 5c33adfe339856c0397d0f3aca225ad3b44559e0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 12 Nov 2013 23:49:19 +0000
Subject: [PATCH 1221/1540] HBASE-9894 remove the inappropriate assert
 statement in Store.getSplitPoint() (Liang Xie)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541321 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 04c12c5276ec..22cde936b1f3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -2119,7 +2119,6 @@ public byte[] getSplitPoint() {
       for (StoreFile sf : storefiles) {
         if (sf.isReference()) {
           // Should already be enforced since we return false in this case
-          assert false : "getSplitPoint() called on a region that can't split!";
           return null;
         }
 

From 06383f84f86e1f89093ddc5f6d0da57616bdf161 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 12 Nov 2013 23:56:37 +0000
Subject: [PATCH 1222/1540] HBASE-9138 getHaseIntegrationTestingUtility() is
 misspelled

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541325 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/util/ChaosMonkey.java | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index f3300082f9be..ef9496408ee1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -118,7 +118,7 @@ public ActionContext(IntegrationTestingUtility util) {
       this.util = util;
     }
 
-    public IntegrationTestingUtility getHaseIntegrationTestingUtility() {
+    public IntegrationTestingUtility getHBaseIntegrationTestingUtility() {
       return util;
     }
 
@@ -289,7 +289,7 @@ public void perform() throws Exception {
       HTable table = null;
       Collection<ServerName> serverNames;
       try {
-        Configuration conf = context.getHaseIntegrationTestingUtility().getConfiguration();
+        Configuration conf = context.getHBaseIntegrationTestingUtility().getConfiguration();
         table = new HTable(conf, tableName);
         serverNames = table.getRegionLocations().values();
       } catch (IOException e) {
@@ -318,7 +318,7 @@ public MoveRegionsOfTable(long sleepTime, String tableName) {
     @Override
     public void perform() throws Exception {
       try {
-        HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
+        HBaseAdmin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin();
         List<HRegionInfo> regions = admin.getTableRegions(tableNameBytes);
         Collection<ServerName> serversList = admin.getClusterStatus().getServers();
         ServerName[] servers = serversList.toArray(new ServerName[serversList.size()]);
@@ -471,7 +471,7 @@ public void perform() throws Exception {
 
       LOG.info("Moving " + victimRegions.size() + " regions from " + victimServers.size()
           + " servers to " + targetServers.size() + " different servers");
-      HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
+      HBaseAdmin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin();
       for (byte[] victimRegion : victimRegions) {
         int targetIx = random.nextInt(targetServers.size());
         admin.move(victimRegion, targetServers.get(targetIx));
@@ -483,7 +483,7 @@ public static class ForceBalancerAction extends Action {
     @Override
     public void perform() throws Exception {
       LOG.info("Balancing regions");
-      HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin();
+      HBaseAdmin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin();
       boolean result = admin.balancer();
       if (!result) {
         LOG.error("Balancer didn't succeed");

From 38f92174d20e9062b027360105ab5d0085ac4856 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 13 Nov 2013 18:08:08 +0000
Subject: [PATCH 1223/1540] HBASE-9849 [REST] Forbidden schema delete in read
 only mode (Julian Zhou)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541642 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/rest/SchemaResource.java     |  5 ++++
 .../hadoop/hbase/rest/TestSchemaResource.java | 26 ++++++++++++-------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index 864b2a617ec0..e6bbfb91f537 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -238,6 +238,11 @@ public Response delete(final @Context UriInfo uriInfo) {
       LOG.debug("DELETE " + uriInfo.getAbsolutePath());
     }
     servlet.getMetrics().incrementRequests(1);
+    if (servlet.isReadOnly()) {
+      return Response.status(Response.Status.FORBIDDEN).type(MIMETYPE_TEXT)
+          .entity("Forbidden" + CRLF).build();
+    }
+
     try {
       HBaseAdmin admin = servlet.getAdmin();
       boolean success = false;
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
index 1d2ed5dca491..838798771521 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
@@ -115,14 +115,17 @@ public void testTableCreateAndDeleteXML() throws IOException, JAXBException {
     model = fromXML(response.getBody());
     TestTableSchemaModel.checkModel(model, TABLE1);
 
-    // delete the table
-    client.delete(schemaPath);
-
-    // make sure HBase concurs
-    assertFalse(admin.tableExists(TABLE1));
+    // test delete schema operation is forbidden in read-only mode
+    response = client.delete(schemaPath);
+    assertEquals(response.getCode(), 403);
 
     // return read-only setting back to default
     conf.set("hbase.rest.readonly", "false");
+
+    // delete the table and make sure HBase concurs
+    response = client.delete(schemaPath);
+    assertEquals(response.getCode(), 200);
+    assertFalse(admin.tableExists(TABLE1));
   }
 
   @Test
@@ -163,14 +166,17 @@ public void testTableCreateAndDeletePB() throws IOException, JAXBException {
     model.getObjectFromMessage(response.getBody());
     TestTableSchemaModel.checkModel(model, TABLE2);
 
-    // delete the table
-    client.delete(schemaPath);
-
-    // make sure HBase concurs
-    assertFalse(admin.tableExists(TABLE2));
+    // test delete schema operation is forbidden in read-only mode
+    response = client.delete(schemaPath);
+    assertEquals(response.getCode(), 403);
 
     // return read-only setting back to default
     conf.set("hbase.rest.readonly", "false");
+
+    // delete the table and make sure HBase concurs
+    response = client.delete(schemaPath);
+    assertEquals(response.getCode(), 200);
+    assertFalse(admin.tableExists(TABLE2));
   }
 
   @org.junit.Rule

From abfb7b5bc4ae13f7c60a6210c53978009283cf49 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Nov 2013 04:07:58 +0000
Subject: [PATCH 1224/1540] HBASE-4654 [replication] Add a check to make sure
 we don't replicate to ourselves (Demai Ni)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1541806 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index c72a1fd47f35..de628255877d 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -306,6 +306,12 @@ public void run() {
     // resetting to 1 to reuse later
     sleepMultiplier = 1;
 
+    // In rare case, zookeeper setting may be messed up. That leads to the incorrect
+    // peerClusterId value, which is the same as the source clusterId
+    if (clusterId.equals(peerClusterId)) {
+      this.terminate("ClusterId " + clusterId + " is replicating to itself: peerClusterId "
+          + peerClusterId);
+    }
     LOG.info("Replicating "+clusterId + " -> " + peerClusterId);
 
     // If this is recovered, the queue is already full and the first log

From aa76269570b236c0f94182f955a9a7449b44ec94 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 14 Nov 2013 20:09:39 +0000
Subject: [PATCH 1225/1540] HBASE-9834: Minimize byte[] copies for 'smart'
 clients

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542052 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/Append.java    | 66 +++++++++----------
 .../apache/hadoop/hbase/client/Delete.java    | 36 ++--------
 .../apache/hadoop/hbase/client/Mutation.java  | 53 ++++++++++++++-
 .../org/apache/hadoop/hbase/client/Put.java   | 46 ++-----------
 4 files changed, 98 insertions(+), 103 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java
index ce435c840ee6..bd77b7c77968 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -23,11 +23,9 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Writable;
 
 /**
  * Performs Append operations on a single row.
@@ -105,23 +103,8 @@ public void readFields(final DataInput in)
     this.ts = in.readLong();
     this.lockId = in.readLong();
     this.writeToWAL = in.readBoolean();
-    int numFamilies = in.readInt();
     if (!this.familyMap.isEmpty()) this.familyMap.clear();
-    for(int i=0;i<numFamilies;i++) {
-      byte [] family = Bytes.readByteArray(in);
-      int numKeys = in.readInt();
-      List<KeyValue> keys = new ArrayList<KeyValue>(numKeys);
-      int totalLen = in.readInt();
-      byte [] buf = new byte[totalLen];
-      int offset = 0;
-      for (int j = 0; j < numKeys; j++) {
-        int keyLength = in.readInt();
-        in.readFully(buf, offset, keyLength);
-        keys.add(new KeyValue(buf, offset, keyLength));
-        offset += keyLength;
-      }
-      this.familyMap.put(family, keys);
-    }
+    readFamilyMap(in);
     readAttributes(in);
   }
 
@@ -133,21 +116,36 @@ public void write(final DataOutput out)
     out.writeLong(this.ts);
     out.writeLong(this.lockId);
     out.writeBoolean(this.writeToWAL);
-    out.writeInt(familyMap.size());
-    for (Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
-      Bytes.writeByteArray(out, entry.getKey());
-      List<KeyValue> keys = entry.getValue();
-      out.writeInt(keys.size());
-      int totalLen = 0;
-      for(KeyValue kv : keys) {
-        totalLen += kv.getLength();
-      }
-      out.writeInt(totalLen);
-      for(KeyValue kv : keys) {
-        out.writeInt(kv.getLength());
-        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
-      }
-    }
+    writeFamilyMap(out);
     writeAttributes(out);
   }
-}
+
+  /**
+   * Add the specified {@link KeyValue} to this operation.
+   * @param kv whose value should be to appended to the specified column
+   * @return <tt?this</tt>
+   * @throws IllegalArgumentException if the row or type does not match <tt>this</tt>
+   */
+  public Append add(KeyValue kv) {
+    if(!(kv.getType() == KeyValue.Type.Put.getCode())){
+      throw new IllegalArgumentException("Added type " + KeyValue.Type.codeToType(kv.getType())
+          + ", but appends can only be of type " + KeyValue.Type.Put + ". Rowkey:"
+          + Bytes.toStringBinary(kv.getRow()));
+    }
+    
+    if (!kv.matchingRow(row)) {
+      throw new IllegalArgumentException("The row in the recently added KeyValue "
+          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
+          + Bytes.toStringBinary(this.row));
+    }
+
+    byte[] family = kv.getFamily();
+    List<KeyValue> list = familyMap.get(family);
+    if (list == null) {
+      list = new ArrayList<KeyValue>();
+      familyMap.put(family, list);
+    }
+    list.add(kv);
+    return this;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 4472dcb41b69..78a43d7f9f72 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -137,22 +137,19 @@ public Delete(final Delete d) {
   }
 
   /**
-   * Advanced use only.
-   * Add an existing delete marker to this Delete object.
-   * @param kv An existing KeyValue of type "delete".
+   * Advanced use only. Add an existing delete marker to this Delete object.
+   * @param kv An existing 'delete' tpye KeyValue - can be family, column, or point delete
    * @return this for invocation chaining
    * @throws IOException
    */
   public Delete addDeleteMarker(KeyValue kv) throws IOException {
-    if (!kv.isDelete()) {
+    if (!(kv.isDelete() || kv.isDeleteColumnOrFamily())) {
       throw new IOException("The recently added KeyValue is not of type "
           + "delete. Rowkey: " + Bytes.toStringBinary(this.row));
     }
-    if (Bytes.compareTo(this.row, 0, row.length, kv.getBuffer(),
-        kv.getRowOffset(), kv.getRowLength()) != 0) {
+    if (!kv.matchingRow(row)) {
       throw new IOException("The row in the recently added KeyValue "
-          + Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
-              kv.getRowLength()) + " doesn't match the original one "
+          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
           + Bytes.toStringBinary(this.row));
     }
     byte [] family = kv.getFamily();
@@ -293,18 +290,7 @@ public void readFields(final DataInput in) throws IOException {
       this.writeToWAL = in.readBoolean();
     }
     this.familyMap.clear();
-    int numFamilies = in.readInt();
-    for(int i=0;i<numFamilies;i++) {
-      byte [] family = Bytes.readByteArray(in);
-      int numColumns = in.readInt();
-      List<KeyValue> list = new ArrayList<KeyValue>(numColumns);
-      for(int j=0;j<numColumns;j++) {
-    	KeyValue kv = new KeyValue();
-    	kv.readFields(in);
-    	list.add(kv);
-      }
-      this.familyMap.put(family, list);
-    }
+    readFamilyMap(in);
     if (version > 1) {
       readAttributes(in);
     }
@@ -316,15 +302,7 @@ public void write(final DataOutput out) throws IOException {
     out.writeLong(this.ts);
     out.writeLong(this.lockId);
     out.writeBoolean(this.writeToWAL);
-    out.writeInt(familyMap.size());
-    for(Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
-      Bytes.writeByteArray(out, entry.getKey());
-      List<KeyValue> list = entry.getValue();
-      out.writeInt(list.size());
-      for(KeyValue kv : list) {
-        kv.write(out);
-      }
-    }
+    writeFamilyMap(out);
     writeAttributes(out);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 32c9e6f98a8e..2f00a4884f55 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -20,6 +20,9 @@
 
 package org.apache.hadoop.hbase.client;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -297,4 +300,52 @@ public int size() {
   public int numFamilies() {
     return familyMap.size();
   }
-}
+
+  /**
+   * Helper method to read in the family map that was written via
+   * {@link #writeFamilyMap(DataOutput)}
+   * @param in to read from
+   * @throws IOException if there was an error reading
+   */
+  protected void readFamilyMap(DataInput in) throws IOException {
+    int numFamilies = in.readInt();
+    for (int i = 0; i < numFamilies; i++) {
+      byte[] family = Bytes.readByteArray(in);
+      int numKeys = in.readInt();
+      List<KeyValue> keys = new ArrayList<KeyValue>(numKeys);
+      int totalLen = in.readInt();
+      byte[] buf = new byte[totalLen];
+      int offset = 0;
+      for (int j = 0; j < numKeys; j++) {
+        int keyLength = in.readInt();
+        in.readFully(buf, offset, keyLength);
+        keys.add(new KeyValue(buf, offset, keyLength));
+        offset += keyLength;
+      }
+      this.familyMap.put(family, keys);
+    }
+  }
+
+  /**
+   * Helper method to write out the family map. Can be read in via {@link #readFamilyMap(DataInput)}
+   * .
+   * @param out to write to
+   * @throws IOException if there was an error writing
+   */
+  protected void writeFamilyMap(DataOutput out) throws IOException {
+    out.writeInt(familyMap.size());
+    for (Map.Entry<byte[], List<KeyValue>> entry : familyMap.entrySet()) {
+      Bytes.writeByteArray(out, entry.getKey());
+      List<KeyValue> keys = entry.getValue();
+      out.writeInt(keys.size());
+      int totalLen = 0;
+      for (KeyValue kv : keys) {
+        totalLen += kv.getLength();
+      }
+      out.writeInt(totalLen);
+      for (KeyValue kv : keys) {
+        kv.write(out);
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index 4596e57584b0..19c0cdb9655f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -157,13 +157,10 @@ public Put add(KeyValue kv) throws IOException{
     byte [] family = kv.getFamily();
     List<KeyValue> list = getKeyValueList(family);
     //Checking that the row of the kv is the same as the put
-    int res = Bytes.compareTo(this.row, 0, row.length,
-        kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
-    if(res != 0) {
-      throw new IOException("The row in the recently added KeyValue " +
-          Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
-        kv.getRowLength()) + " doesn't match the original one " +
-        Bytes.toStringBinary(this.row));
+    if (!kv.matchingRow(row)) {
+      throw new IOException("The row in the recently added KeyValue "
+          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
+          + Bytes.toStringBinary(this.row));
     }
     list.add(kv);
     familyMap.put(family, list);
@@ -177,7 +174,7 @@ public Put add(KeyValue kv) throws IOException{
    */
   private KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts,
       byte[] value) {
-  return  new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put,
+    return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put,
       value);
   }
 
@@ -375,23 +372,8 @@ public void readFields(final DataInput in)
     this.ts = in.readLong();
     this.lockId = in.readLong();
     this.writeToWAL = in.readBoolean();
-    int numFamilies = in.readInt();
     if (!this.familyMap.isEmpty()) this.familyMap.clear();
-    for(int i=0;i<numFamilies;i++) {
-      byte [] family = Bytes.readByteArray(in);
-      int numKeys = in.readInt();
-      List<KeyValue> keys = new ArrayList<KeyValue>(numKeys);
-      int totalLen = in.readInt();
-      byte [] buf = new byte[totalLen];
-      int offset = 0;
-      for (int j = 0; j < numKeys; j++) {
-        int keyLength = in.readInt();
-        in.readFully(buf, offset, keyLength);
-        keys.add(new KeyValue(buf, offset, keyLength));
-        offset += keyLength;
-      }
-      this.familyMap.put(family, keys);
-    }
+    readFamilyMap(in);
     if (version > 1) {
       readAttributes(in);
     }
@@ -404,21 +386,7 @@ public void write(final DataOutput out)
     out.writeLong(this.ts);
     out.writeLong(this.lockId);
     out.writeBoolean(this.writeToWAL);
-    out.writeInt(familyMap.size());
-    for (Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
-      Bytes.writeByteArray(out, entry.getKey());
-      List<KeyValue> keys = entry.getValue();
-      out.writeInt(keys.size());
-      int totalLen = 0;
-      for(KeyValue kv : keys) {
-        totalLen += kv.getLength();
-      }
-      out.writeInt(totalLen);
-      for(KeyValue kv : keys) {
-        out.writeInt(kv.getLength());
-        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
-      }
-    }
+    writeFamilyMap(out);
     writeAttributes(out);
   }
 }

From 74c726e05ed16909606fc453076ab2ba7fe11785 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Nov 2013 22:06:34 +0000
Subject: [PATCH 1226/1540] HBASE-9963 Remove the ReentrantReadWriteLock in the
 MemStore (Nicolas Liochon)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542104 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/MemStore.java   | 242 +++++++-----------
 .../hadoop/hbase/regionserver/Store.java      |   8 +-
 .../apache/hadoop/hbase/io/TestHeapSize.java  |   2 -
 3 files changed, 102 insertions(+), 150 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index c5007d963c7b..e73f453f0d80 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -30,7 +30,6 @@
 import java.util.NavigableSet;
 import java.util.SortedSet;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -51,6 +50,10 @@
  * to snapshot and is cleared.  We continue to serve edits out of new memstore
  * and backing snapshot until flusher reports in that the flush succeeded. At
  * this point we let the snapshot go.
+ * <p>
+ * The MemStore functions should not be called in parallel. Callers should hold
+ * write and read locks. This is done in {@link Store}.
+ * </p>
  * TODO: Adjust size of the memstore when we remove items because they have
  * been deleted.
  * TODO: With new KVSLS, need to make sure we update HeapSize with difference
@@ -75,8 +78,6 @@ public class MemStore implements HeapSize {
   // Snapshot of memstore.  Made for flusher.
   volatile KeyValueSkipListSet snapshot;
 
-  final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-
   final KeyValue.KVComparator comparator;
 
   // Used comparing versions -- same r/c and ts but different type.
@@ -143,30 +144,25 @@ void dump() {
    * To get the snapshot made by this method, use {@link #getSnapshot()}
    */
   void snapshot() {
-    this.lock.writeLock().lock();
-    try {
-      // If snapshot currently has entries, then flusher failed or didn't call
-      // cleanup.  Log a warning.
-      if (!this.snapshot.isEmpty()) {
-        LOG.warn("Snapshot called again without clearing previous. " +
-          "Doing nothing. Another ongoing flush or did we fail last attempt?");
-      } else {
-        if (!this.kvset.isEmpty()) {
-          this.snapshot = this.kvset;
-          this.kvset = new KeyValueSkipListSet(this.comparator);
-          this.snapshotTimeRangeTracker = this.timeRangeTracker;
-          this.timeRangeTracker = new TimeRangeTracker();
-          // Reset heap to not include any keys
-          this.size.set(DEEP_OVERHEAD);
-          // Reset allocator so we get a fresh buffer for the new memstore
-          if (allocator != null) {
-            this.allocator = new MemStoreLAB(conf);
-          }
-          timeOfOldestEdit = Long.MAX_VALUE;
+    // If snapshot currently has entries, then flusher failed or didn't call
+    // cleanup.  Log a warning.
+    if (!this.snapshot.isEmpty()) {
+      LOG.warn("Snapshot called again without clearing previous. " +
+        "Doing nothing. Another ongoing flush or did we fail last attempt?");
+    } else {
+      if (!this.kvset.isEmpty()) {
+        this.snapshot = this.kvset;
+        this.kvset = new KeyValueSkipListSet(this.comparator);
+        this.snapshotTimeRangeTracker = this.timeRangeTracker;
+        this.timeRangeTracker = new TimeRangeTracker();
+        // Reset heap to not include any keys
+        this.size.set(DEEP_OVERHEAD);
+        // Reset allocator so we get a fresh buffer for the new memstore
+        if (allocator != null) {
+          this.allocator = new MemStoreLAB(conf);
         }
+        timeOfOldestEdit = Long.MAX_VALUE;
       }
-    } finally {
-      this.lock.writeLock().unlock();
     }
   }
 
@@ -190,20 +186,15 @@ KeyValueSkipListSet getSnapshot() {
    */
   void clearSnapshot(final SortedSet<KeyValue> ss)
   throws UnexpectedException {
-    this.lock.writeLock().lock();
-    try {
-      if (this.snapshot != ss) {
-        throw new UnexpectedException("Current snapshot is " +
-          this.snapshot + ", was passed " + ss);
-      }
-      // OK. Passed in snapshot is same as current snapshot.  If not-empty,
-      // create a new snapshot and let the old one go.
-      if (!ss.isEmpty()) {
-        this.snapshot = new KeyValueSkipListSet(this.comparator);
-        this.snapshotTimeRangeTracker = new TimeRangeTracker();
-      }
-    } finally {
-      this.lock.writeLock().unlock();
+    if (this.snapshot != ss) {
+      throw new UnexpectedException("Current snapshot is " +
+        this.snapshot + ", was passed " + ss);
+    }
+    // OK. Passed in snapshot is same as current snapshot.  If not-empty,
+    // create a new snapshot and let the old one go.
+    if (!ss.isEmpty()) {
+      this.snapshot = new KeyValueSkipListSet(this.comparator);
+      this.snapshotTimeRangeTracker = new TimeRangeTracker();
     }
   }
 
@@ -213,13 +204,8 @@ void clearSnapshot(final SortedSet<KeyValue> ss)
    * @return approximate size of the passed key and value.
    */
   long add(final KeyValue kv) {
-    this.lock.readLock().lock();
-    try {
-      KeyValue toAdd = maybeCloneWithAllocator(kv);
-      return internalAdd(toAdd);
-    } finally {
-      this.lock.readLock().unlock();
-    }
+    KeyValue toAdd = maybeCloneWithAllocator(kv);
+    return internalAdd(toAdd);
   }
 
   long timeOfOldestEdit() {
@@ -285,26 +271,21 @@ private KeyValue maybeCloneWithAllocator(KeyValue kv) {
    * @param kv
    */
   void rollback(final KeyValue kv) {
-    this.lock.readLock().lock();
-    try {
-      // If the key is in the snapshot, delete it. We should not update
-      // this.size, because that tracks the size of only the memstore and
-      // not the snapshot. The flush of this snapshot to disk has not
-      // yet started because Store.flush() waits for all rwcc transactions to
-      // commit before starting the flush to disk.
-      KeyValue found = this.snapshot.get(kv);
-      if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
-        this.snapshot.remove(kv);
-      }
-      // If the key is in the memstore, delete it. Update this.size.
-      found = this.kvset.get(kv);
-      if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
-        removeFromKVSet(kv);
-        long s = heapSizeChange(kv, true);
-        this.size.addAndGet(-s);
-      }
-    } finally {
-      this.lock.readLock().unlock();
+    // If the key is in the snapshot, delete it. We should not update
+    // this.size, because that tracks the size of only the memstore and
+    // not the snapshot. The flush of this snapshot to disk has not
+    // yet started because Store.flush() waits for all rwcc transactions to
+    // commit before starting the flush to disk.
+    KeyValue found = this.snapshot.get(kv);
+    if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
+      this.snapshot.remove(kv);
+    }
+    // If the key is in the memstore, delete it. Update this.size.
+    found = this.kvset.get(kv);
+    if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
+      removeFromKVSet(kv);
+      long s = heapSizeChange(kv, true);
+      this.size.addAndGet(-s);
     }
   }
 
@@ -314,15 +295,9 @@ void rollback(final KeyValue kv) {
    * @return approximate size of the passed key and value.
    */
   long delete(final KeyValue delete) {
-    long s = 0;
-    this.lock.readLock().lock();
-    try {
-      KeyValue toAdd = maybeCloneWithAllocator(delete);
-      s += heapSizeChange(toAdd, addToKVSet(toAdd));
-      timeRangeTracker.includeTimestamp(toAdd);
-    } finally {
-      this.lock.readLock().unlock();
-    }
+    KeyValue toAdd = maybeCloneWithAllocator(delete);
+    long s = heapSizeChange(toAdd, addToKVSet(toAdd));
+    timeRangeTracker.includeTimestamp(toAdd);
     this.size.addAndGet(s);
     return s;
   }
@@ -333,12 +308,7 @@ long delete(final KeyValue delete) {
    * @return Next row or null if none found.
    */
   KeyValue getNextRow(final KeyValue kv) {
-    this.lock.readLock().lock();
-    try {
-      return getLowest(getNextRow(kv, this.kvset), getNextRow(kv, this.snapshot));
-    } finally {
-      this.lock.readLock().unlock();
-    }
+    return getLowest(getNextRow(kv, this.kvset), getNextRow(kv, this.snapshot));
   }
 
   /*
@@ -382,13 +352,8 @@ private KeyValue getNextRow(final KeyValue key,
    * @param state column/delete tracking state
    */
   void getRowKeyAtOrBefore(final GetClosestRowBeforeTracker state) {
-    this.lock.readLock().lock();
-    try {
-      getRowKeyAtOrBefore(kvset, state);
-      getRowKeyAtOrBefore(snapshot, state);
-    } finally {
-      this.lock.readLock().unlock();
-    }
+    getRowKeyAtOrBefore(kvset, state);
+    getRowKeyAtOrBefore(snapshot, state);
   }
 
   /*
@@ -480,54 +445,49 @@ public long updateColumnValue(byte[] row,
                                 byte[] qualifier,
                                 long newValue,
                                 long now) {
-   this.lock.readLock().lock();
-    try {
-      KeyValue firstKv = KeyValue.createFirstOnRow(
-          row, family, qualifier);
-      // Is there a KeyValue in 'snapshot' with the same TS? If so, upgrade the timestamp a bit.
-      SortedSet<KeyValue> snSs = snapshot.tailSet(firstKv);
-      if (!snSs.isEmpty()) {
-        KeyValue snKv = snSs.first();
-        // is there a matching KV in the snapshot?
-        if (snKv.matchingRow(firstKv) && snKv.matchingQualifier(firstKv)) {
-          if (snKv.getTimestamp() == now) {
-            // poop,
-            now += 1;
-          }
+    KeyValue firstKv = KeyValue.createFirstOnRow(
+        row, family, qualifier);
+    // Is there a KeyValue in 'snapshot' with the same TS? If so, upgrade the timestamp a bit.
+    SortedSet<KeyValue> snSs = snapshot.tailSet(firstKv);
+    if (!snSs.isEmpty()) {
+      KeyValue snKv = snSs.first();
+      // is there a matching KV in the snapshot?
+      if (snKv.matchingRow(firstKv) && snKv.matchingQualifier(firstKv)) {
+        if (snKv.getTimestamp() == now) {
+          // poop,
+          now += 1;
         }
       }
+    }
 
-      // logic here: the new ts MUST be at least 'now'. But it could be larger if necessary.
-      // But the timestamp should also be max(now, mostRecentTsInMemstore)
-
-      // so we cant add the new KV w/o knowing what's there already, but we also
-      // want to take this chance to delete some kvs. So two loops (sad)
+    // logic here: the new ts MUST be at least 'now'. But it could be larger if necessary.
+    // But the timestamp should also be max(now, mostRecentTsInMemstore)
 
-      SortedSet<KeyValue> ss = kvset.tailSet(firstKv);
-      Iterator<KeyValue> it = ss.iterator();
-      while ( it.hasNext() ) {
-        KeyValue kv = it.next();
+    // so we cant add the new KV w/o knowing what's there already, but we also
+    // want to take this chance to delete some kvs. So two loops (sad)
 
-        // if this isnt the row we are interested in, then bail:
-        if (!kv.matchingColumn(family,qualifier) || !kv.matchingRow(firstKv) ) {
-          break; // rows dont match, bail.
-        }
+    SortedSet<KeyValue> ss = kvset.tailSet(firstKv);
+    Iterator<KeyValue> it = ss.iterator();
+    while ( it.hasNext() ) {
+      KeyValue kv = it.next();
 
-        // if the qualifier matches and it's a put, just RM it out of the kvset.
-        if (kv.getType() == KeyValue.Type.Put.getCode() &&
-            kv.getTimestamp() > now && firstKv.matchingQualifier(kv)) {
-          now = kv.getTimestamp();
-        }
+      // if this isnt the row we are interested in, then bail:
+      if (!kv.matchingColumn(family,qualifier) || !kv.matchingRow(firstKv) ) {
+        break; // rows dont match, bail.
       }
 
-      // create or update (upsert) a new KeyValue with
-      // 'now' and a 0 memstoreTS == immediately visible
-      return upsert(Arrays.asList(
-          new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue)))
-      );
-    } finally {
-      this.lock.readLock().unlock();
+      // if the qualifier matches and it's a put, just RM it out of the kvset.
+      if (kv.getType() == KeyValue.Type.Put.getCode() &&
+          kv.getTimestamp() > now && firstKv.matchingQualifier(kv)) {
+        now = kv.getTimestamp();
+      }
     }
+
+    // create or update (upsert) a new KeyValue with
+    // 'now' and a 0 memstoreTS == immediately visible
+    return upsert(Arrays.asList(
+        new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue)))
+    );
   }
 
   /**
@@ -548,17 +508,12 @@ public long updateColumnValue(byte[] row,
    * @return change in memstore size
    */
   public long upsert(List<KeyValue> kvs) {
-   this.lock.readLock().lock();
-    try {
-      long size = 0;
-      for (KeyValue kv : kvs) {
-        kv.setMemstoreTS(0);
-        size += upsert(kv);
-      }
-      return size;
-    } finally {
-      this.lock.readLock().unlock();
+    long size = 0;
+    for (KeyValue kv : kvs) {
+      kv.setMemstoreTS(0);
+      size += upsert(kv);
     }
+    return size;
   }
 
   /**
@@ -665,13 +620,8 @@ private Member memberOfPreviousRow(NavigableSet<KeyValue> set,
    * @return scanner on memstore and snapshot in this order.
    */
   List<KeyValueScanner> getScanners() {
-    this.lock.readLock().lock();
-    try {
-      return Collections.<KeyValueScanner>singletonList(
-          new MemStoreScanner(MultiVersionConsistencyControl.getThreadReadPoint()));
-    } finally {
-      this.lock.readLock().unlock();
-    }
+    return Collections.<KeyValueScanner>singletonList(
+        new MemStoreScanner(MultiVersionConsistencyControl.getThreadReadPoint()));
   }
 
   /**
@@ -716,7 +666,7 @@ protected class MemStoreScanner extends NonLazyKeyValueScanner {
 
     // the pre-calculated KeyValue to be returned by peek() or next()
     private KeyValue theNext;
-    private long readPoint;
+    private final long readPoint;
 
     /*
     Some notes...
@@ -930,7 +880,7 @@ public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns,
       ClassSize.OBJECT + (11 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
-      ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
+      ClassSize.ATOMIC_LONG +
       (2 * ClassSize.TIMERANGE_TRACKER) +
       (2 * ClassSize.KEYVALUE_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP));
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 22cde936b1f3..389a3556d214 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -32,7 +32,6 @@
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.Future;
@@ -758,7 +757,12 @@ public Void call() throws IOException {
    * {@link #flushCache(long, SortedSet<KeyValue>)} so it has some work to do.
    */
   void snapshot() {
-    this.memstore.snapshot();
+    this.lock.writeLock().lock();
+    try {
+      this.memstore.snapshot();
+    } finally {
+      this.lock.writeLock().unlock();
+    }
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
index dbd4ab7dc555..4ed6aa55b184 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
@@ -298,14 +298,12 @@ public void testSizes() throws IOException {
     // MemStore Deep Overhead
     actual = MemStore.DEEP_OVERHEAD;
     expected = ClassSize.estimateBase(cl, false);
-    expected += ClassSize.estimateBase(ReentrantReadWriteLock.class, false);
     expected += ClassSize.estimateBase(AtomicLong.class, false);
     expected += (2 * ClassSize.estimateBase(KeyValueSkipListSet.class, false));
     expected += (2 * ClassSize.estimateBase(ConcurrentSkipListMap.class, false));
     expected += (2 * ClassSize.estimateBase(TimeRangeTracker.class, false));
     if(expected != actual) {
       ClassSize.estimateBase(cl, true);
-      ClassSize.estimateBase(ReentrantReadWriteLock.class, true);
       ClassSize.estimateBase(AtomicLong.class, true);
       ClassSize.estimateBase(KeyValueSkipListSet.class, true);
       ClassSize.estimateBase(KeyValueSkipListSet.class, true);

From 89ff9ce65286e062fb74f69bd98c80838305730e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 14 Nov 2013 23:25:22 +0000
Subject: [PATCH 1227/1540] HBASE-9799 Change Hadoop 1.2 dependency to 1.2.1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542117 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 2e3b67ce0e5c..292a8e321041 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1817,7 +1817,7 @@
         </property>
       </activation>
       <properties>
-        <hadoop.version>1.2.0</hadoop.version>
+        <hadoop.version>1.2.1</hadoop.version>
         <slf4j.version>1.4.3</slf4j.version>
       </properties>
       <dependencies>

From c9d4d2c9b5e30ae79d0ec7782709eaa917600f9a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 15 Nov 2013 18:17:44 +0000
Subject: [PATCH 1228/1540] HBASE-9963 Fix TestHeapSize

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542358 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/MemStore.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index e73f453f0d80..6b51d74bfdbc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -877,7 +877,7 @@ public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns,
   }
 
   public final static long FIXED_OVERHEAD = ClassSize.align(
-      ClassSize.OBJECT + (11 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
+      ClassSize.OBJECT + (10 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.ATOMIC_LONG +

From 8a6291a89613e54d18870530829f2d6ff9672ec6 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 15 Nov 2013 18:29:50 +0000
Subject: [PATCH 1229/1540] HBASE-9971 Port part of HBASE-9958 to 0.94 - change
 lock scope in locateRegion (original patch by Nicolas Liochon)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542360 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/HConnectionManager.java      | 57 +++++++++----------
 1 file changed, 28 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 30a56befc952..adf629b3e640 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -73,7 +73,6 @@
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.ipc.RpcEngine;
-import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -1104,38 +1103,38 @@ private HRegionLocation locateRegionInMeta(final byte [] parentTable,
             getHRegionConnection(metaLocation.getHostname(), metaLocation.getPort());
 
           Result regionInfoRow = null;
-          // This block guards against two threads trying to load the meta
-          // region at the same time. The first will load the meta region and
-          // the second will use the value that the first one found.
-          synchronized (regionLockObject) {
-            // Check the cache again for a hit in case some other thread made the
-            // same query while we were waiting on the lock.
-            if (useCache) {
-              location = getCachedLocation(tableName, row);
-              if (location != null) {
-                return location;
-              }
-              // If the parent table is META, we may want to pre-fetch some
-              // region info into the global region cache for this table.
-              if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME)
-                  && (getRegionCachePrefetch(tableName))) {
+          if (useCache) {
+            if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME)
+                && (getRegionCachePrefetch(tableName))) {
+              // This block guards against two threads trying to load the meta
+              // region at the same time. The first will load the meta region and
+              // the second will use the value that the first one found.
+              synchronized (regionLockObject) {
+                // Check the cache again for a hit in case some other thread made the
+                // same query while we were waiting on the lock.
+                location = getCachedLocation(tableName, row);
+                if (location != null) {
+                  return location;
+                }
+                // If the parent table is META, we may want to pre-fetch some
+                // region info into the global region cache for this table.
                 prefetchRegionCache(tableName, row);
               }
-              location = getCachedLocation(tableName, row);
-              if (location != null) {
-                return location;
-              }
-            } else {
-              // If we are not supposed to be using the cache, delete any existing cached location
-              // so it won't interfere.
-              deleteCachedLocation(tableName, row);
             }
-
-            // Query the root or meta region for the location of the meta region
-            regionInfoRow = server.getClosestRowBefore(
-            metaLocation.getRegionInfo().getRegionName(), metaKey,
-            HConstants.CATALOG_FAMILY);
+            location = getCachedLocation(tableName, row);
+            if (location != null) {
+              return location;
+            }
+          } else {
+            // If we are not supposed to be using the cache, delete any existing cached location
+            // so it won't interfere.
+            deleteCachedLocation(tableName, row);
           }
+
+          // Query the root or meta region for the location of the meta region
+          regionInfoRow = server.getClosestRowBefore(
+          metaLocation.getRegionInfo().getRegionName(), metaKey,
+          HConstants.CATALOG_FAMILY);
           if (regionInfoRow == null) {
             throw new TableNotFoundException(Bytes.toString(tableName));
           }

From c09fd6b4ee5bd7758ccc88c36685b362321ad18f Mon Sep 17 00:00:00 2001
From: ndimiduk <ndimiduk@unknown>
Date: Fri, 15 Nov 2013 21:54:00 +0000
Subject: [PATCH 1230/1540] HBASE-9165 [mapreduce] Modularize building
 dependency jars

Separate adding HBase and dependencies from adding other job dependencies, and
expose it as a separate method that other projects can use (for PIG-3285,
HIVE-2055).


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542414 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapred/TableMapReduceUtil.java      |  7 ++-
 .../hbase/mapreduce/TableMapReduceUtil.java   | 46 +++++++++++++------
 .../hbase/mapreduce/TestTableMapReduce.java   | 10 ++--
 3 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index 0c7d47da0c9a..13a304383428 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -294,14 +294,13 @@ public static void setScannerCaching(JobConf job, int batchSize) {
   }
 
   /**
-   * @see org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#addDependencyJars(Job)
+   * @see org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)
    */
   public static void addDependencyJars(JobConf job) throws IOException {
+    org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
     org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
       job,
-      org.apache.zookeeper.ZooKeeper.class,
-      com.google.common.base.Function.class,
-      com.google.protobuf.Message.class,
+      // when making changes here, consider also mapreduce.TableMapReduceUtil
       job.getMapOutputKeyClass(),
       job.getMapOutputValueClass(),
       job.getOutputKeyClass(),
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 954d0a16fad5..df1f1608e074 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -551,26 +551,45 @@ public static void setScannerCaching(Job job, int batchSize) {
     job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
   }
 
+  /**
+   * Add HBase and its dependencies (only) to the job configuration.
+   * <p>
+   * This is intended as a low-level API, facilitating code reuse between this
+   * class and its mapred counterpart. It also of use to extenral tools that
+   * need to build a MapReduce job that interacts with HBase but want
+   * fine-grained control over the jars shipped to the cluster.
+   * </p>
+   * @param conf The Configuration object to extend with dependencies.
+   * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
+   * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
+   */
+  public static void addHBaseDependencyJars(Configuration conf) throws IOException {
+    addDependencyJars(conf,
+      org.apache.zookeeper.ZooKeeper.class,
+      com.google.protobuf.Message.class,
+      com.google.common.base.Function.class,
+      com.google.common.collect.ImmutableSet.class,
+      org.apache.hadoop.hbase.util.Bytes.class); //one class from hbase.jar
+  }
+
   /**
    * Add the HBase dependency jars as well as jars for any of the configured
    * job classes to the job configuration, so that JobClient will ship them
    * to the cluster and add them to the DistributedCache.
    */
   public static void addDependencyJars(Job job) throws IOException {
+    addHBaseDependencyJars(job.getConfiguration());
     try {
       addDependencyJars(job.getConfiguration(),
-          org.apache.zookeeper.ZooKeeper.class,
-          com.google.protobuf.Message.class,
-          com.google.common.collect.ImmutableSet.class,
-          org.apache.hadoop.hbase.util.Bytes.class, //one class from hbase.jar
-          job.getMapOutputKeyClass(),
-          job.getMapOutputValueClass(),
-          job.getInputFormatClass(),
-          job.getOutputKeyClass(),
-          job.getOutputValueClass(),
-          job.getOutputFormatClass(),
-          job.getPartitionerClass(),
-          job.getCombinerClass());
+        // when making changes here, consider also mapred.TableMapReduceUtil
+        job.getMapOutputKeyClass(),
+        job.getMapOutputValueClass(),
+        job.getInputFormatClass(),
+        job.getOutputKeyClass(),
+        job.getOutputValueClass(),
+        job.getOutputFormatClass(),
+        job.getPartitionerClass(),
+        job.getCombinerClass());
     } catch (ClassNotFoundException e) {
       throw new IOException(e);
     }    
@@ -612,8 +631,7 @@ public static void addDependencyJars(Configuration conf,
     }
     if (jars.isEmpty()) return;
 
-    conf.set("tmpjars",
-             StringUtils.arrayToString(jars.toArray(new String[0])));
+    conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index b351444cbeb0..96ca6cfd8c1d 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -263,19 +263,15 @@ private void verifyAttempt(final HTable table) throws IOException, NullPointerEx
   /**
    * Test that we add tmpjars correctly including the ZK jar.
    */
+  @Test
   public void testAddDependencyJars() throws Exception {
     Job job = new Job();
     TableMapReduceUtil.addDependencyJars(job);
     String tmpjars = job.getConfiguration().get("tmpjars");
 
-    System.err.println("tmpjars: " + tmpjars);
+    assertTrue(tmpjars.contains("hbase"));
     assertTrue(tmpjars.contains("zookeeper"));
-    assertFalse(tmpjars.contains("guava"));
-
-    System.err.println("appending guava jar");
-    TableMapReduceUtil.addDependencyJars(job.getConfiguration(), 
-        com.google.common.base.Function.class);
-    tmpjars = job.getConfiguration().get("tmpjars");
+    assertTrue(tmpjars.contains("protobuf"));
     assertTrue(tmpjars.contains("guava"));
   }
 

From d714d6b5cdda8c9ac99032ae5b6caba27200d08b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 16 Nov 2013 01:06:45 +0000
Subject: [PATCH 1231/1540] HBASE-9834 Revert, due to wire compatibility issues

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542440 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/Append.java    | 66 ++++++++++---------
 .../apache/hadoop/hbase/client/Delete.java    | 36 ++++++++--
 .../apache/hadoop/hbase/client/Mutation.java  | 53 +--------------
 .../org/apache/hadoop/hbase/client/Put.java   | 46 +++++++++++--
 4 files changed, 103 insertions(+), 98 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java
index bd77b7c77968..ce435c840ee6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -23,9 +23,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
 
 /**
  * Performs Append operations on a single row.
@@ -103,8 +105,23 @@ public void readFields(final DataInput in)
     this.ts = in.readLong();
     this.lockId = in.readLong();
     this.writeToWAL = in.readBoolean();
+    int numFamilies = in.readInt();
     if (!this.familyMap.isEmpty()) this.familyMap.clear();
-    readFamilyMap(in);
+    for(int i=0;i<numFamilies;i++) {
+      byte [] family = Bytes.readByteArray(in);
+      int numKeys = in.readInt();
+      List<KeyValue> keys = new ArrayList<KeyValue>(numKeys);
+      int totalLen = in.readInt();
+      byte [] buf = new byte[totalLen];
+      int offset = 0;
+      for (int j = 0; j < numKeys; j++) {
+        int keyLength = in.readInt();
+        in.readFully(buf, offset, keyLength);
+        keys.add(new KeyValue(buf, offset, keyLength));
+        offset += keyLength;
+      }
+      this.familyMap.put(family, keys);
+    }
     readAttributes(in);
   }
 
@@ -116,36 +133,21 @@ public void write(final DataOutput out)
     out.writeLong(this.ts);
     out.writeLong(this.lockId);
     out.writeBoolean(this.writeToWAL);
-    writeFamilyMap(out);
-    writeAttributes(out);
-  }
-
-  /**
-   * Add the specified {@link KeyValue} to this operation.
-   * @param kv whose value should be to appended to the specified column
-   * @return <tt?this</tt>
-   * @throws IllegalArgumentException if the row or type does not match <tt>this</tt>
-   */
-  public Append add(KeyValue kv) {
-    if(!(kv.getType() == KeyValue.Type.Put.getCode())){
-      throw new IllegalArgumentException("Added type " + KeyValue.Type.codeToType(kv.getType())
-          + ", but appends can only be of type " + KeyValue.Type.Put + ". Rowkey:"
-          + Bytes.toStringBinary(kv.getRow()));
-    }
-    
-    if (!kv.matchingRow(row)) {
-      throw new IllegalArgumentException("The row in the recently added KeyValue "
-          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
-          + Bytes.toStringBinary(this.row));
+    out.writeInt(familyMap.size());
+    for (Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
+      Bytes.writeByteArray(out, entry.getKey());
+      List<KeyValue> keys = entry.getValue();
+      out.writeInt(keys.size());
+      int totalLen = 0;
+      for(KeyValue kv : keys) {
+        totalLen += kv.getLength();
+      }
+      out.writeInt(totalLen);
+      for(KeyValue kv : keys) {
+        out.writeInt(kv.getLength());
+        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
+      }
     }
-
-    byte[] family = kv.getFamily();
-    List<KeyValue> list = familyMap.get(family);
-    if (list == null) {
-      list = new ArrayList<KeyValue>();
-      familyMap.put(family, list);
-    }
-    list.add(kv);
-    return this;
+    writeAttributes(out);
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 78a43d7f9f72..4472dcb41b69 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -137,19 +137,22 @@ public Delete(final Delete d) {
   }
 
   /**
-   * Advanced use only. Add an existing delete marker to this Delete object.
-   * @param kv An existing 'delete' tpye KeyValue - can be family, column, or point delete
+   * Advanced use only.
+   * Add an existing delete marker to this Delete object.
+   * @param kv An existing KeyValue of type "delete".
    * @return this for invocation chaining
    * @throws IOException
    */
   public Delete addDeleteMarker(KeyValue kv) throws IOException {
-    if (!(kv.isDelete() || kv.isDeleteColumnOrFamily())) {
+    if (!kv.isDelete()) {
       throw new IOException("The recently added KeyValue is not of type "
           + "delete. Rowkey: " + Bytes.toStringBinary(this.row));
     }
-    if (!kv.matchingRow(row)) {
+    if (Bytes.compareTo(this.row, 0, row.length, kv.getBuffer(),
+        kv.getRowOffset(), kv.getRowLength()) != 0) {
       throw new IOException("The row in the recently added KeyValue "
-          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
+          + Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
+              kv.getRowLength()) + " doesn't match the original one "
           + Bytes.toStringBinary(this.row));
     }
     byte [] family = kv.getFamily();
@@ -290,7 +293,18 @@ public void readFields(final DataInput in) throws IOException {
       this.writeToWAL = in.readBoolean();
     }
     this.familyMap.clear();
-    readFamilyMap(in);
+    int numFamilies = in.readInt();
+    for(int i=0;i<numFamilies;i++) {
+      byte [] family = Bytes.readByteArray(in);
+      int numColumns = in.readInt();
+      List<KeyValue> list = new ArrayList<KeyValue>(numColumns);
+      for(int j=0;j<numColumns;j++) {
+    	KeyValue kv = new KeyValue();
+    	kv.readFields(in);
+    	list.add(kv);
+      }
+      this.familyMap.put(family, list);
+    }
     if (version > 1) {
       readAttributes(in);
     }
@@ -302,7 +316,15 @@ public void write(final DataOutput out) throws IOException {
     out.writeLong(this.ts);
     out.writeLong(this.lockId);
     out.writeBoolean(this.writeToWAL);
-    writeFamilyMap(out);
+    out.writeInt(familyMap.size());
+    for(Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
+      Bytes.writeByteArray(out, entry.getKey());
+      List<KeyValue> list = entry.getValue();
+      out.writeInt(list.size());
+      for(KeyValue kv : list) {
+        kv.write(out);
+      }
+    }
     writeAttributes(out);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 2f00a4884f55..32c9e6f98a8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -20,9 +20,6 @@
 
 package org.apache.hadoop.hbase.client;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -300,52 +297,4 @@ public int size() {
   public int numFamilies() {
     return familyMap.size();
   }
-
-  /**
-   * Helper method to read in the family map that was written via
-   * {@link #writeFamilyMap(DataOutput)}
-   * @param in to read from
-   * @throws IOException if there was an error reading
-   */
-  protected void readFamilyMap(DataInput in) throws IOException {
-    int numFamilies = in.readInt();
-    for (int i = 0; i < numFamilies; i++) {
-      byte[] family = Bytes.readByteArray(in);
-      int numKeys = in.readInt();
-      List<KeyValue> keys = new ArrayList<KeyValue>(numKeys);
-      int totalLen = in.readInt();
-      byte[] buf = new byte[totalLen];
-      int offset = 0;
-      for (int j = 0; j < numKeys; j++) {
-        int keyLength = in.readInt();
-        in.readFully(buf, offset, keyLength);
-        keys.add(new KeyValue(buf, offset, keyLength));
-        offset += keyLength;
-      }
-      this.familyMap.put(family, keys);
-    }
-  }
-
-  /**
-   * Helper method to write out the family map. Can be read in via {@link #readFamilyMap(DataInput)}
-   * .
-   * @param out to write to
-   * @throws IOException if there was an error writing
-   */
-  protected void writeFamilyMap(DataOutput out) throws IOException {
-    out.writeInt(familyMap.size());
-    for (Map.Entry<byte[], List<KeyValue>> entry : familyMap.entrySet()) {
-      Bytes.writeByteArray(out, entry.getKey());
-      List<KeyValue> keys = entry.getValue();
-      out.writeInt(keys.size());
-      int totalLen = 0;
-      for (KeyValue kv : keys) {
-        totalLen += kv.getLength();
-      }
-      out.writeInt(totalLen);
-      for (KeyValue kv : keys) {
-        kv.write(out);
-      }
-    }
-  }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index 19c0cdb9655f..4596e57584b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -157,10 +157,13 @@ public Put add(KeyValue kv) throws IOException{
     byte [] family = kv.getFamily();
     List<KeyValue> list = getKeyValueList(family);
     //Checking that the row of the kv is the same as the put
-    if (!kv.matchingRow(row)) {
-      throw new IOException("The row in the recently added KeyValue "
-          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
-          + Bytes.toStringBinary(this.row));
+    int res = Bytes.compareTo(this.row, 0, row.length,
+        kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
+    if(res != 0) {
+      throw new IOException("The row in the recently added KeyValue " +
+          Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
+        kv.getRowLength()) + " doesn't match the original one " +
+        Bytes.toStringBinary(this.row));
     }
     list.add(kv);
     familyMap.put(family, list);
@@ -174,7 +177,7 @@ public Put add(KeyValue kv) throws IOException{
    */
   private KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts,
       byte[] value) {
-    return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put,
+  return  new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put,
       value);
   }
 
@@ -372,8 +375,23 @@ public void readFields(final DataInput in)
     this.ts = in.readLong();
     this.lockId = in.readLong();
     this.writeToWAL = in.readBoolean();
+    int numFamilies = in.readInt();
     if (!this.familyMap.isEmpty()) this.familyMap.clear();
-    readFamilyMap(in);
+    for(int i=0;i<numFamilies;i++) {
+      byte [] family = Bytes.readByteArray(in);
+      int numKeys = in.readInt();
+      List<KeyValue> keys = new ArrayList<KeyValue>(numKeys);
+      int totalLen = in.readInt();
+      byte [] buf = new byte[totalLen];
+      int offset = 0;
+      for (int j = 0; j < numKeys; j++) {
+        int keyLength = in.readInt();
+        in.readFully(buf, offset, keyLength);
+        keys.add(new KeyValue(buf, offset, keyLength));
+        offset += keyLength;
+      }
+      this.familyMap.put(family, keys);
+    }
     if (version > 1) {
       readAttributes(in);
     }
@@ -386,7 +404,21 @@ public void write(final DataOutput out)
     out.writeLong(this.ts);
     out.writeLong(this.lockId);
     out.writeBoolean(this.writeToWAL);
-    writeFamilyMap(out);
+    out.writeInt(familyMap.size());
+    for (Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
+      Bytes.writeByteArray(out, entry.getKey());
+      List<KeyValue> keys = entry.getValue();
+      out.writeInt(keys.size());
+      int totalLen = 0;
+      for(KeyValue kv : keys) {
+        totalLen += kv.getLength();
+      }
+      out.writeInt(totalLen);
+      for(KeyValue kv : keys) {
+        out.writeInt(kv.getLength());
+        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
+      }
+    }
     writeAttributes(out);
   }
 }

From fd9a70e6c750cbbd8db4f15eb189d1d707e2aa50 Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Sat, 16 Nov 2013 03:46:33 +0000
Subject: [PATCH 1232/1540] HBASE-9970 HBase BulkLoad, table is creating with
 the timestamp key also as a column to the table. (Sreenivasulu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542462 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index 4a61da77eee9..3e0b3b4f0edd 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -301,7 +301,8 @@ private static void createTable(Configuration conf, String tableName)
     String columns[] = conf.getStrings(COLUMNS_CONF_KEY);
     Set<String> cfSet = new HashSet<String>();
     for (String aColumn : columns) {
-      if (TsvParser.ROWKEY_COLUMN_SPEC.equals(aColumn)) continue;
+      if (TsvParser.ROWKEY_COLUMN_SPEC.equals(aColumn)
+          || TsvParser.TIMESTAMPKEY_COLUMN_SPEC.equals(aColumn)) continue;
       // we are only concerned with the first one (in case this is a cf:cq)
       cfSet.add(aColumn.split(":", 2)[0]);
     }

From a8e1d20164849d429180bacd37abc1950c7f4acb Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Sat, 16 Nov 2013 07:39:57 +0000
Subject: [PATCH 1233/1540] HBASE-9975 Not starting ReplicationSink when using
 custom implementation for the ReplicationSink

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542470 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegionServer.java      | 11 +++++++----
 .../replication/regionserver/ReplicationSink.java     |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7a9c751096ea..affdb9b815fe 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1713,10 +1713,13 @@ private void startServiceThreads() throws IOException {
     if (this.replicationSourceHandler == this.replicationSinkHandler &&
         this.replicationSourceHandler != null) {
       this.replicationSourceHandler.startReplicationService();
-    } else if (this.replicationSourceHandler != null) {
-      this.replicationSourceHandler.startReplicationService();
-    } else if (this.replicationSinkHandler != null) {
-      this.replicationSinkHandler.startReplicationService();
+    } else {
+      if (this.replicationSourceHandler != null) {
+        this.replicationSourceHandler.startReplicationService();
+      }
+      if (this.replicationSinkHandler != null) {
+        this.replicationSinkHandler.startReplicationService();
+      }
     }
 
     // Start Server.  This service is like leases in that it internally runs
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index ad67367d8abe..141cd1464c29 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -203,7 +203,7 @@ public void stopReplicationSinkServices() {
    * @param rows list of actions
    * @throws IOException
    */
-  private void batch(byte[] tableName, Collection<List<Row>> allRows) throws IOException {
+  protected void batch(byte[] tableName, Collection<List<Row>> allRows) throws IOException {
     if (allRows.isEmpty()) {
       return;
     }

From a38854cc68a735890f310188d352121255469e0c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 17 Nov 2013 04:14:12 +0000
Subject: [PATCH 1234/1540] HBASE-9165 Addendum, fix TestTableMapreduce

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542642 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index 96ca6cfd8c1d..e26825f5100e 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -269,7 +269,6 @@ public void testAddDependencyJars() throws Exception {
     TableMapReduceUtil.addDependencyJars(job);
     String tmpjars = job.getConfiguration().get("tmpjars");
 
-    assertTrue(tmpjars.contains("hbase"));
     assertTrue(tmpjars.contains("zookeeper"));
     assertTrue(tmpjars.contains("protobuf"));
     assertTrue(tmpjars.contains("guava"));

From b4e9220e37880612f62b278f9f89ea1dd9f5a1f5 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Sun, 17 Nov 2013 04:49:30 +0000
Subject: [PATCH 1235/1540] HBASE-9834: Minimize byte[] copies for 'smart'
 clients (again)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542645 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/client/Append.java    | 35 ++++++++++++++++---
 .../apache/hadoop/hbase/client/Delete.java    | 19 +++++-----
 .../org/apache/hadoop/hbase/client/Put.java   | 14 +++-----
 3 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java
index ce435c840ee6..12a4aefa76fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -27,7 +27,6 @@
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Writable;
 
 /**
  * Performs Append operations on a single row.
@@ -144,10 +143,38 @@ public void write(final DataOutput out)
       }
       out.writeInt(totalLen);
       for(KeyValue kv : keys) {
-        out.writeInt(kv.getLength());
-        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
+        kv.write(out);
       }
     }
     writeAttributes(out);
   }
-}
+
+  /**
+   * Add the specified {@link KeyValue} to this operation.
+   * @param kv whose value should be to appended to the specified column
+   * @return <tt?this</tt>
+   * @throws IllegalArgumentException if the row or type does not match <tt>this</tt>
+   */
+  public Append add(KeyValue kv) {
+    if(!(kv.getType() == KeyValue.Type.Put.getCode())){
+      throw new IllegalArgumentException("Added type " + KeyValue.Type.codeToType(kv.getType())
+          + ", but appends can only be of type " + KeyValue.Type.Put + ". Rowkey:"
+          + Bytes.toStringBinary(kv.getRow()));
+    }
+
+    if (!kv.matchingRow(row)) {
+      throw new IllegalArgumentException("The row in the recently added KeyValue "
+          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
+          + Bytes.toStringBinary(this.row));
+    }
+
+    byte[] family = kv.getFamily();
+    List<KeyValue> list = familyMap.get(family);
+    if (list == null) {
+      list = new ArrayList<KeyValue>();
+      familyMap.put(family, list);
+    }
+    list.add(kv);
+    return this;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 4472dcb41b69..c147f2fbf7bd 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -137,22 +137,19 @@ public Delete(final Delete d) {
   }
 
   /**
-   * Advanced use only.
-   * Add an existing delete marker to this Delete object.
-   * @param kv An existing KeyValue of type "delete".
+   * Advanced use only. Add an existing delete marker to this Delete object.
+   * @param kv An existing 'delete' tpye KeyValue - can be family, column, or point delete
    * @return this for invocation chaining
    * @throws IOException
    */
   public Delete addDeleteMarker(KeyValue kv) throws IOException {
-    if (!kv.isDelete()) {
+    if (!(kv.isDelete() || kv.isDeleteColumnOrFamily())) {
       throw new IOException("The recently added KeyValue is not of type "
           + "delete. Rowkey: " + Bytes.toStringBinary(this.row));
     }
-    if (Bytes.compareTo(this.row, 0, row.length, kv.getBuffer(),
-        kv.getRowOffset(), kv.getRowLength()) != 0) {
+    if (!kv.matchingRow(row)) {
       throw new IOException("The row in the recently added KeyValue "
-          + Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
-              kv.getRowLength()) + " doesn't match the original one "
+          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
           + Bytes.toStringBinary(this.row));
     }
     byte [] family = kv.getFamily();
@@ -299,9 +296,9 @@ public void readFields(final DataInput in) throws IOException {
       int numColumns = in.readInt();
       List<KeyValue> list = new ArrayList<KeyValue>(numColumns);
       for(int j=0;j<numColumns;j++) {
-    	KeyValue kv = new KeyValue();
-    	kv.readFields(in);
-    	list.add(kv);
+       KeyValue kv = new KeyValue();
+       kv.readFields(in);
+       list.add(kv);
       }
       this.familyMap.put(family, list);
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index 4596e57584b0..e8ac0a8bdcb0 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -157,13 +157,10 @@ public Put add(KeyValue kv) throws IOException{
     byte [] family = kv.getFamily();
     List<KeyValue> list = getKeyValueList(family);
     //Checking that the row of the kv is the same as the put
-    int res = Bytes.compareTo(this.row, 0, row.length,
-        kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
-    if(res != 0) {
-      throw new IOException("The row in the recently added KeyValue " +
-          Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
-        kv.getRowLength()) + " doesn't match the original one " +
-        Bytes.toStringBinary(this.row));
+    if (!kv.matchingRow(row)) {
+      throw new IOException("The row in the recently added KeyValue "
+          + Bytes.toStringBinary(kv.getRow()) + " doesn't match the original one "
+          + Bytes.toStringBinary(this.row));
     }
     list.add(kv);
     familyMap.put(family, list);
@@ -415,8 +412,7 @@ public void write(final DataOutput out)
       }
       out.writeInt(totalLen);
       for(KeyValue kv : keys) {
-        out.writeInt(kv.getLength());
-        out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
+        kv.write(out);
       }
     }
     writeAttributes(out);

From d5e71907549696c6086659bdd90ba974c4799730 Mon Sep 17 00:00:00 2001
From: ndimiduk <ndimiduk@unknown>
Date: Sun, 17 Nov 2013 20:59:10 +0000
Subject: [PATCH 1236/1540] HBASE-8438 Extend bin/hbase to print a "mapreduce
 classpath"

$ ./bin/hbase mapredcp --help
Usage: hbase mapredcp [-Dtmpjars=...]
  Construct a CLASSPATH containing dependency jars required to run a mapreduce
  job. By default, includes any jars detected by TableMapReduceUtils. Provide
  additional entries by specifying a comma-separated list in tmpjars.

$ ./bin/hbase mapredcp | tr ':' '\n'
/private/tmp/hbase-0.94.14-SNAPSHOT/hbase-0.94.14-SNAPSHOT.jar
/private/tmp/hbase-0.94.14-SNAPSHOT/lib/protobuf-java-2.4.0a.jar
/private/tmp/hbase-0.94.14-SNAPSHOT/lib/zookeeper-3.4.5.jar
/private/tmp/hbase-0.94.14-SNAPSHOT/lib/guava-11.0.2.jar
/private/tmp/hbase-0.94.14-SNAPSHOT/lib/hadoop-core-1.0.4.jar


git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542816 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/hbase                                     |  4 +-
 .../hbase/mapreduce/TableMapReduceUtil.java   | 24 +++++++
 .../MapreduceDependencyClasspathTool.java     | 72 +++++++++++++++++++
 3 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java

diff --git a/bin/hbase b/bin/hbase
index 841c68c6e349..adf4dae92ad3 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -94,6 +94,7 @@ if [ $# = 0 ]; then
   echo ""
   echo "PACKAGE MANAGEMENT"
   echo "  classpath        dump hbase CLASSPATH"
+  echo "  mapredcp         dump CLASSPATH entries required by mapreduce"
   echo "  version          print the version"
   echo ""
   echo " or"
@@ -325,7 +326,8 @@ elif [ "$COMMAND" = "zookeeper" ] ; then
   if [ "$1" != "stop" ] ; then
     HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS"
   fi
-
+elif [ "$COMMAND" = "mapredcp" ] ; then
+  CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool'
 elif [ "$COMMAND" = "classpath" ] ; then
   echo $CLASSPATH
   exit 0
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index df1f1608e074..2014e2b3edf8 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -23,6 +23,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -572,6 +573,29 @@ public static void addHBaseDependencyJars(Configuration conf) throws IOException
       org.apache.hadoop.hbase.util.Bytes.class); //one class from hbase.jar
   }
 
+   /**
+    * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
+    * Also exposed to shell scripts via `bin/hbase mapredcp`.
+    */
+  public static String buildDependencyClasspath(Configuration conf) {
+    if (conf == null) {
+      throw new IllegalArgumentException("Must provide a configuration object.");
+    }
+    Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
+    if (paths.size() == 0) {
+      throw new IllegalArgumentException("Configuration contains no tmpjars.");
+    }
+    StringBuilder sb = new StringBuilder();
+    for (String s : paths) {
+      // entries can take the form 'file:/path/to/file.jar'.
+      int idx = s.indexOf(":");
+      if (idx != -1) s = s.substring(idx + 1);
+      if (sb.length() > 0) sb.append(File.pathSeparator);
+      sb.append(s);
+    }
+    return sb.toString();
+  }
+
   /**
    * Add the HBase dependency jars as well as jars for any of the configured
    * job classes to the job configuration, so that JobClient will ship them
diff --git a/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java b/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java
new file mode 100644
index 000000000000..171b34b3a880
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * Generate a classpath string containing any jars required by mapreduce jobs. Specify
+ * additional values by providing a comma-separated list of paths via -Dtmpjars.
+ */
+public class MapreduceDependencyClasspathTool implements Tool {
+
+  private Configuration conf;
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length > 0) {
+      System.err.println("Usage: hbase mapredcp [-Dtmpjars=...]");
+      System.err.println("  Construct a CLASSPATH containing dependency jars required to run a mapreduce");
+      System.err.println("  job. By default, includes any jars detected by TableMapReduceUtils. Provide");
+      System.err.println("  additional entries by specifying a comma-separated list in tmpjars.");
+      return 0;
+    }
+
+    Job job = new Job(getConf());
+    TableMapReduceUtil.addDependencyJars(job);
+    System.out.println(TableMapReduceUtil.buildDependencyClasspath(job.getConfiguration()));
+    return 0;
+  }
+
+  public static void main(String[] argv) throws Exception {
+    // Silence the usual noise. This is probably fragile...
+    Logger logger = Logger.getLogger("org.apache.hadoop.hbase");
+    if (logger != null) {
+      logger.setLevel(Level.WARN);
+    }
+    System.exit(ToolRunner.run(
+      HBaseConfiguration.create(), new MapreduceDependencyClasspathTool(), argv));
+  }
+}

From f5c74347c5d61be5a49364f2658a0caf221509cf Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 17 Nov 2013 21:48:21 +0000
Subject: [PATCH 1237/1540] CHANGES.txt and pom.xml in preparation for
 0.94.14RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1542837 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 43 +++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 1265d6fcaa40..a2ff2b96e65d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,48 @@
 HBase Change Log
 
+Release 0.94.14 - 11/17/2013
+Sub-task
+
+    [HBASE-9165] - Improvements to addDependencyJars
+
+Bug
+
+    [HBASE-9138] - getHaseIntegrationTestingUtility() is misspelled
+    [HBASE-9799] - Change Hadoop 1.2 dependency to 1.2.1
+    [HBASE-9809] - RegionTooBusyException should provide region name which was too busy
+    [HBASE-9834] - Minimize byte[] copies for 'smart' clients
+    [HBASE-9849] - [REST] Forbidden schema delete in read only mode
+    [HBASE-9850] - Issues with UI for table compact/split operation completion. After split/compaction operation using UI, the page is not automatically redirecting back using IE8/Firefox.
+    [HBASE-9865] - Reused WALEdits in replication may cause RegionServers to go OOM
+    [HBASE-9872] - ModifyTable does not modify the attributes of a newly modified/changed ColumnDescriptor
+    [HBASE-9890] - MR jobs are not working if started by a delegated user
+    [HBASE-9902] - Region Server is starting normally even if clock skew is more than default 30 seconds(or any configured). -> Regionserver node time is greater than master node time
+    [HBASE-9906] - Restore snapshot fails to restore the meta edits sporadically
+    [HBASE-9915] - Performance: isSeeked() in EncodedScannerV2 always returns false
+    [HBASE-9952] - Snapshot restore may fail due to NullPointerException
+    [HBASE-9956] - Remove keyLength cache from KeyValue
+    [HBASE-9970] - HBase BulkLoad, table is creating with the timestamp key also as a column to the table.
+    [HBASE-9971] - Port part of HBASE-9958 to 0.94 - change lock scope in locateRegion
+    [HBASE-9975] - Not starting ReplicationSink when using custom implementation for the ReplicationSink.
+
+Improvement
+
+    [HBASE-4654] - [replication] Add a check to make sure we don't replicate to ourselves
+    [HBASE-8438] - Extend bin/hbase to print a "mapreduce classpath"
+    [HBASE-9715] - Backport -in_memory option support for LoadTestTool from trunk
+    [HBASE-9894] - remove the inappropriate assert statement in Store.getSplitPoint()
+    [HBASE-9963] - Remove the ReentrantReadWriteLock in the MemStore
+
+Test
+
+    [HBASE-8397] - improve unit-test coverage of package org.apache.hadoop.hbase.master.metrics (0.94)
+    [HBASE-8543] - fix coverage org.apache.hadoop.hbase.rest.client
+    [HBASE-8552] - fix coverage org.apache.hadoop.hbase.rest.filter
+    [HBASE-8556] - fix coverage org.apache.hadoop.hbase.metrics.histogram
+    [HBASE-8557] - fix coverage org.apache.hadoop.hbase.rest.metrics
+    [HBASE-8559] - increase unit-test coverage of package org.apache.hadoop.hbase.coprocessor
+
+
 Release 0.94.13 - 10/29/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 292a8e321041..3047153b7d10 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.14-SNAPSHOT</version>
+  <version>0.94.14</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 39299e24ae520d077b693168b8f897b53a73f8ed Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 18 Nov 2013 20:21:28 +0000
Subject: [PATCH 1238/1540] HBASE-9831 'hbasefsck.numthreads' property isn't
 passed to hbck via cmdline -D option (Takeshi Miao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1543139 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 09791edbef13..5cfc95469dab 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -96,7 +96,6 @@
 import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -252,8 +251,7 @@ public HBaseFsck(Configuration conf) throws MasterNotRunningException,
     super(conf);
     errors = getErrorReporter(conf);
 
-    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
-    executor = new ScheduledThreadPoolExecutor(numThreads);
+    initialPoolNumThreads();
   }
 
   /**
@@ -284,6 +282,18 @@ public void connect() throws IOException {
     connection = admin.getConnection();
   }
 
+  /**
+   * Initial numThreads for {@link #executor}
+   */
+  private void initialPoolNumThreads() {
+    if (executor != null) {
+      executor.shutdown();
+    }
+
+    int numThreads = getConf().getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
+    executor = new ScheduledThreadPoolExecutor(numThreads);
+  }
+
   /**
    * Get deployed regions according to the region servers.
    */
@@ -3486,6 +3496,9 @@ public static void main(String[] args) throws Exception {
 
   @Override
   public int run(String[] args) throws Exception {
+    // reset the numThreads due to user may set it via generic options
+    initialPoolNumThreads();
+    
     exec(executor, args);
     return getRetCode();
   }

From d01dcdc1cb1791a14a93db3d282e6c2c74f4555c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 18 Nov 2013 23:02:23 +0000
Subject: [PATCH 1239/1540] HBASE-9993 0.94: HBASE-9865 breaks coprocessor
 compatibility with WALEdit.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1543220 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java | 2 +-
 .../hbase/replication/regionserver/ReplicationSource.java      | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index bd3e33ffd7f5..eb6c10c4176b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -132,7 +132,7 @@ public int size() {
     return kvs.size();
   }
 
-  public ArrayList<KeyValue> getKeyValues() {
+  public List<KeyValue> getKeyValues() {
     return kvs;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index de628255877d..58e38c42b6e0 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -662,7 +662,8 @@ protected boolean sleepForRetries(String msg, int sleepMultiplier) {
    * @param edit The KV to check for replication
    */
   protected void removeNonReplicableEdits(WALEdit edit) {
-    ArrayList<KeyValue> kvs = edit.getKeyValues();
+    // for backward compatibility WALEdit returns a List
+    ArrayList<KeyValue> kvs = (ArrayList<KeyValue>)edit.getKeyValues();
     int size = edit.size();
     for (int i = size-1; i >= 0; i--) {
       KeyValue kv = kvs.get(i);

From b280d1ed25d2e58ce442f2fbf4be03791df1323c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 18 Nov 2013 23:05:08 +0000
Subject: [PATCH 1240/1540] CHANGES.txt for 0.94.14RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1543221 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index a2ff2b96e65d..20403b035df5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
 HBase Change Log
 
-Release 0.94.14 - 11/17/2013
+Release 0.94.14 - 11/18/2013
 Sub-task
 
     [HBASE-9165] - Improvements to addDependencyJars
@@ -24,12 +24,14 @@ Bug
     [HBASE-9970] - HBase BulkLoad, table is creating with the timestamp key also as a column to the table.
     [HBASE-9971] - Port part of HBASE-9958 to 0.94 - change lock scope in locateRegion
     [HBASE-9975] - Not starting ReplicationSink when using custom implementation for the ReplicationSink.
+    [HBASE-9993] - 0.94: HBASE-9865 breaks coprocessor compatibility with WALEdit.
 
 Improvement
 
     [HBASE-4654] - [replication] Add a check to make sure we don't replicate to ourselves
     [HBASE-8438] - Extend bin/hbase to print a "mapreduce classpath"
     [HBASE-9715] - Backport -in_memory option support for LoadTestTool from trunk
+    [HBASE-9831] - 'hbasefsck.numthreads' property isn't passed to hbck via cmdline -D option
     [HBASE-9894] - remove the inappropriate assert statement in Store.getSplitPoint()
     [HBASE-9963] - Remove the ReentrantReadWriteLock in the MemStore
 

From 891e99362f0756a669f88f08eb01e12c2eb52a7e Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Tue, 19 Nov 2013 17:11:29 +0000
Subject: [PATCH 1241/1540] HBASE-9995 Not stopping ReplicationSink when using
 custom implementation for the ReplicationSink

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1543503 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegionServer.java      | 11 +++++++----
 .../replication/regionserver/ReplicationSink.java     |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index affdb9b815fe..9efbbf6c9753 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1975,10 +1975,13 @@ protected void join() {
     if (this.replicationSourceHandler != null &&
         this.replicationSourceHandler == this.replicationSinkHandler) {
       this.replicationSourceHandler.stopReplicationService();
-    } else if (this.replicationSourceHandler != null) {
-      this.replicationSourceHandler.stopReplicationService();
-    } else if (this.replicationSinkHandler != null) {
-      this.replicationSinkHandler.stopReplicationService();
+    } else {
+      if (this.replicationSourceHandler != null) {
+        this.replicationSourceHandler.stopReplicationService();
+      }
+      if (this.replicationSinkHandler != null) {
+        this.replicationSinkHandler.stopReplicationService();
+      }
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 141cd1464c29..b11db60db74b 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -200,7 +200,7 @@ public void stopReplicationSinkServices() {
   /**
    * Do the changes and handle the pool
    * @param tableName table to insert into
-   * @param rows list of actions
+   * @param allRows list of actions
    * @throws IOException
    */
   protected void batch(byte[] tableName, Collection<List<Row>> allRows) throws IOException {

From 76935742ddf3ea0269a9f2e4f384b5f2ae131864 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Wed, 20 Nov 2013 20:30:05 +0000
Subject: [PATCH 1242/1540] HBASE-10001 Add a coprocessor to help testing the
 performances without taking into account the disk i/o

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1543936 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/tool/WriteSinkCoprocessor.java      | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/tool/WriteSinkCoprocessor.java

diff --git a/src/main/java/org/apache/hadoop/hbase/tool/WriteSinkCoprocessor.java b/src/main/java/org/apache/hadoop/hbase/tool/WriteSinkCoprocessor.java
new file mode 100644
index 000000000000..dba1f2e74d62
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/tool/WriteSinkCoprocessor.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.tool;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.regionserver.OperationStatus;
+import org.apache.hadoop.hbase.util.Pair;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This coprocessor 'shallows' all the writes. It allows to test a pure
+ * write workload, going through all the communication layers.
+ * The reads will work as well, but they as we never write, they will always always
+ * return an empty structure. The WAL is also skipped.
+ * Obviously, the region will never be split automatically. It's up to the user
+ * to split and move it.
+ * <p/>
+ * For a table created like this:
+ * create 'usertable', {NAME => 'f1', VERSIONS => 1}
+ * <p/>
+ * You can then add the coprocessor with this command:
+ * alter 'usertable', METHOD => 'table_att', 'coprocessor'=>'|org.apache.hadoop.hbase.tool.WriteSinkCoprocessor|'
+ * <p/>
+ * And then
+ * put 'usertable', 'f1', 'f1', 'f1'
+ * <p/>
+ * scan 'usertable'
+ * Will return:
+ * 0 row(s) in 0.0050 seconds
+ * <p/>
+ */
+public class WriteSinkCoprocessor extends BaseRegionObserver {
+  private static final Log LOG = LogFactory.getLog(WriteSinkCoprocessor.class);
+  private final AtomicLong ops = new AtomicLong();
+  private String regionName;
+
+  @Override
+  public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
+    regionName = e.getEnvironment().getRegion().getRegionNameAsString();
+  }
+
+
+  @Override
+  public void preBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
+                    final MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp)
+      throws IOException {
+    if (ops.incrementAndGet() % 20000 == 0) {
+      LOG.info("Wrote " + ops.get() + " times in region " + regionName);
+    }
+
+    for (int i = 0; i < miniBatchOp.size(); i++) {
+      miniBatchOp.setOperationStatus(i,
+          new OperationStatus(HConstants.OperationStatusCode.SUCCESS));
+    }
+    c.bypass();
+  }
+}

From db68e2f58fa0c4c58788d6551c95db2dab25471a Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Thu, 21 Nov 2013 10:27:54 +0000
Subject: [PATCH 1243/1540] HBASE-10014 HRegion#doMiniBatchMutation rollbacks
 the memstore even if there is nothing to rollback.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1544090 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4b731b40a809..0e958a6db903 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2240,7 +2240,7 @@ private long doMiniBatchMutation(
 
     MultiVersionConsistencyControl.WriteEntry w = null;
     long txid = 0;
-    boolean walSyncSuccessful = false;
+    boolean doRollBackMemstore = false;
     boolean locked = false;
 
     /** Keep track of the locks we hold so we can release them in finally clause */
@@ -2409,6 +2409,7 @@ private long doMiniBatchMutation(
             != OperationStatusCode.NOT_RUN) {
           continue;
         }
+        doRollBackMemstore = true;
         addedSize += applyFamilyMapToMemstore(familyMaps[i], w);
       }
 
@@ -2475,7 +2476,7 @@ private long doMiniBatchMutation(
       if (walEdit.size() > 0) {
         syncOrDefer(txid, durability);
       }
-      walSyncSuccessful = true;
+      doRollBackMemstore = false;
       // calling the post CP hook for batch mutation
       if (coprocessorHost != null) {
         MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp = 
@@ -2516,7 +2517,7 @@ private long doMiniBatchMutation(
     } finally {
 
       // if the wal sync was unsuccessful, remove keys from memstore
-      if (!walSyncSuccessful) {
+      if (doRollBackMemstore) {
         rollbackMemstore(batchOp, familyMaps, firstIndex, lastIndexExclusive);
       }
       if (w != null) mvcc.completeMemstoreInsert(w);

From bd7938ceab1932c9acabc3691a527bb93fb3a1a9 Mon Sep 17 00:00:00 2001
From: nkeywal <nkeywal@unknown>
Date: Fri, 22 Nov 2013 21:21:20 +0000
Subject: [PATCH 1244/1540] HBASE-10007 PerformanceEvaluation: Add sampling and
 latency collection to randomRead test (Nick Dimiduk)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1544685 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/PerformanceEvaluation.java   | 245 ++++++++++++------
 1 file changed, 171 insertions(+), 74 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 16c7653506c6..9284b4e01629 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -21,9 +21,12 @@
 
 import java.io.DataInput;
 import java.io.DataOutput;
+import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
-import java.io.File;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.text.DecimalFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
@@ -98,20 +101,20 @@
 public class PerformanceEvaluation {
   protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
 
-  private static final int ROW_LENGTH = 1000;
-  private static final int ONE_GB = 1024 * 1024 * 1000;
-  private static final int ROWS_PER_GB = ONE_GB / ROW_LENGTH;
-
   public static final byte[] TABLE_NAME = Bytes.toBytes("TestTable");
   public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
   public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data");
+  public static final int VALUE_LENGTH = 1000;
+  public static final int ROW_LENGTH = 26;
 
-  protected static final HTableDescriptor TABLE_DESCRIPTOR;
-  static {
-    TABLE_DESCRIPTOR = new HTableDescriptor(TABLE_NAME);
-    TABLE_DESCRIPTOR.addFamily(new HColumnDescriptor(FAMILY_NAME));
-  }
+  private static final int ONE_GB = 1024 * 1024 * 1000;
+  private static final int ROWS_PER_GB = ONE_GB / VALUE_LENGTH;
+  private static final DecimalFormat FMT = new DecimalFormat("0.##");
+  private static final MathContext CXT = MathContext.DECIMAL64;
+  private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
+  private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
 
+  protected static HTableDescriptor TABLE_DESCRIPTOR;
   protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
 
   volatile Configuration conf;
@@ -119,21 +122,24 @@ public class PerformanceEvaluation {
   private boolean nomapred = false;
   private int N = 1;
   private int R = ROWS_PER_GB;
+  private float sampleRate = 1.0f;
   private boolean flushCommits = true;
+  private boolean reportLatency = false;
   private boolean writeToWAL = true;
   private int presplitRegions = 0;
 
   private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
-  /**
-   * Regex to parse lines in input file passed to mapreduce task.
-   */
+
+  /** Regex to parse lines in input file passed to mapreduce task. */
   public static final Pattern LINE_PATTERN =
     Pattern.compile("startRow=(\\d+),\\s+" +
         "perClientRunRows=(\\d+),\\s+" +
         "totalRows=(\\d+),\\s+" +
+        "sampleRate=([-+]?[0-9]*\\.?[0-9]+),\\s+" +
         "clients=(\\d+),\\s+" +
         "flushCommits=(\\w+),\\s+" +
-        "writeToWAL=(\\w+)");
+        "writeToWAL=(\\w+),\\s+" +
+        "reportLatency=(\\w+)");
 
   /**
    * Enum for map metrics.  Keep it out here rather than inside in the Map
@@ -143,8 +149,8 @@ protected static enum Counter {
     /** elapsed time */
     ELAPSED_TIME,
     /** number of rows */
-    ROWS}
-
+    ROWS
+  }
 
   /**
    * Constructor
@@ -206,27 +212,24 @@ public static class PeInputSplit extends InputSplit implements Writable {
     private int startRow = 0;
     private int rows = 0;
     private int totalRows = 0;
+    private float sampleRate = 1.0f;
     private int clients = 0;
     private boolean flushCommits = false;
     private boolean writeToWAL = true;
+    private boolean reportLatency = false;
 
-    public PeInputSplit() {
-      this.startRow = 0;
-      this.rows = 0;
-      this.totalRows = 0;
-      this.clients = 0;
-      this.flushCommits = false;
-      this.writeToWAL = true;
-    }
+    public PeInputSplit() {}
 
-    public PeInputSplit(int startRow, int rows, int totalRows, int clients,
-        boolean flushCommits, boolean writeToWAL) {
+    public PeInputSplit(int startRow, int rows, int totalRows, float sampleRate,
+        int clients, boolean flushCommits, boolean writeToWAL, boolean reportLatency) {
       this.startRow = startRow;
       this.rows = rows;
       this.totalRows = totalRows;
+      this.sampleRate = sampleRate;
       this.clients = clients;
       this.flushCommits = flushCommits;
       this.writeToWAL = writeToWAL;
+      this.reportLatency = reportLatency;
     }
 
     @Override
@@ -234,9 +237,11 @@ public void readFields(DataInput in) throws IOException {
       this.startRow = in.readInt();
       this.rows = in.readInt();
       this.totalRows = in.readInt();
+      this.sampleRate = in.readFloat();
       this.clients = in.readInt();
       this.flushCommits = in.readBoolean();
       this.writeToWAL = in.readBoolean();
+      this.reportLatency = in.readBoolean();
     }
 
     @Override
@@ -244,9 +249,11 @@ public void write(DataOutput out) throws IOException {
       out.writeInt(startRow);
       out.writeInt(rows);
       out.writeInt(totalRows);
+      out.writeFloat(sampleRate);
       out.writeInt(clients);
       out.writeBoolean(flushCommits);
       out.writeBoolean(writeToWAL);
+      out.writeBoolean(reportLatency);
     }
 
     @Override
@@ -271,6 +278,10 @@ public int getTotalRows() {
       return totalRows;
     }
 
+    public float getSampleRate() {
+      return sampleRate;
+    }
+
     public int getClients() {
       return clients;
     }
@@ -282,6 +293,10 @@ public boolean isFlushCommits() {
     public boolean isWriteToWAL() {
       return writeToWAL;
     }
+
+    public boolean isReportLatency() {
+      return reportLatency;
+    }
   }
 
   /**
@@ -312,21 +327,25 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
             int startRow = Integer.parseInt(m.group(1));
             int rows = Integer.parseInt(m.group(2));
             int totalRows = Integer.parseInt(m.group(3));
-            int clients = Integer.parseInt(m.group(4));
-            boolean flushCommits = Boolean.parseBoolean(m.group(5));
-            boolean writeToWAL = Boolean.parseBoolean(m.group(6));
+            float sampleRate = Float.parseFloat(m.group(4));
+            int clients = Integer.parseInt(m.group(5));
+            boolean flushCommits = Boolean.parseBoolean(m.group(6));
+            boolean writeToWAL = Boolean.parseBoolean(m.group(7));
+            boolean reportLatency = Boolean.parseBoolean(m.group(8));
 
             LOG.debug("split["+ splitList.size() + "] " +
                      " startRow=" + startRow +
                      " rows=" + rows +
                      " totalRows=" + totalRows +
+                     " sampleRate=" + sampleRate +
                      " clients=" + clients +
                      " flushCommits=" + flushCommits +
-                     " writeToWAL=" + writeToWAL);
+                     " writeToWAL=" + writeToWAL +
+                     " reportLatency=" + reportLatency);
 
             PeInputSplit newSplit =
-              new PeInputSplit(startRow, rows, totalRows, clients,
-                flushCommits, writeToWAL);
+              new PeInputSplit(startRow, rows, totalRows, sampleRate, clients,
+                flushCommits, writeToWAL, reportLatency);
             splitList.add(newSplit);
           }
         }
@@ -363,7 +382,7 @@ public boolean nextKeyValue() throws IOException, InterruptedException {
         }
 
         key = NullWritable.get();
-        value = (PeInputSplit)split;
+        value = split;
 
         readOver = true;
         return true;
@@ -446,9 +465,9 @@ public void setStatus(String msg) {
 
       // Evaluation task
       long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(),
-                                  value.getRows(), value.getTotalRows(),
-                                  value.isFlushCommits(), value.isWriteToWAL(),
-                                  status);
+        value.getRows(), value.getTotalRows(), value.getSampleRate(),
+        value.isFlushCommits(), value.isWriteToWAL(), value.isReportLatency(),
+        status);
       // Collect how much time the thing took. Report as map output and
       // to the ELAPSED_TIME counter.
       context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -487,11 +506,15 @@ private boolean checkTable(HBaseAdmin admin) throws IOException {
         LOG.info("Table " + tableDescriptor + " created");
       }
     }
-    boolean tableExists = admin.tableExists(tableDescriptor.getName());
-    return tableExists;
+    return admin.tableExists(tableDescriptor.getName());
   }
 
   protected HTableDescriptor getTableDescriptor() {
+    if (TABLE_DESCRIPTOR == null) {
+      TABLE_DESCRIPTOR = new HTableDescriptor(TABLE_NAME);
+      HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+      TABLE_DESCRIPTOR.addFamily(family);
+    }
     return TABLE_DESCRIPTOR;
   }
 
@@ -546,8 +569,8 @@ public void run() {
           int index = Integer.parseInt(getName());
           try {
             long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
-               perClientRows, R,
-                flushCommits, writeToWAL, new Status() {
+               perClientRows, R, sampleRate, flushCommits, writeToWAL,
+               reportLatency, new Status() {
                   public void setStatus(final String msg) throws IOException {
                     LOG.info("client-" + getName() + " " + msg);
                   }
@@ -641,9 +664,11 @@ private Path writeInputFile(final Configuration c) throws IOException {
           String s = "startRow=" + ((j * perClientRows) + (i * (perClientRows/10))) +
           ", perClientRunRows=" + (perClientRows / 10) +
           ", totalRows=" + this.R +
+          ", sampleRate=" + this.sampleRate +
           ", clients=" + this.N +
           ", flushCommits=" + this.flushCommits +
-          ", writeToWAL=" + this.writeToWAL;
+          ", writeToWAL=" + this.writeToWAL +
+          ", reportLatency=" + this.reportLatency;
           int hash = h.hash(Bytes.toBytes(s));
           m.put(hash, s);
         }
@@ -692,20 +717,24 @@ static class TestOptions {
     private int startRow;
     private int perClientRunRows;
     private int totalRows;
+    private float sampleRate;
     private byte[] tableName;
     private boolean flushCommits;
     private boolean writeToWAL = true;
+    private boolean reportLatency;
 
-    TestOptions() {
-    }
+    TestOptions() {}
 
-    TestOptions(int startRow, int perClientRunRows, int totalRows, byte[] tableName, boolean flushCommits, boolean writeToWAL) {
+    TestOptions(int startRow, int perClientRunRows, int totalRows, float sampleRate,
+        byte[] tableName, boolean flushCommits, boolean writeToWAL, boolean reportLatency) {
       this.startRow = startRow;
       this.perClientRunRows = perClientRunRows;
       this.totalRows = totalRows;
+      this.sampleRate = sampleRate;
       this.tableName = tableName;
       this.flushCommits = flushCommits;
       this.writeToWAL = writeToWAL;
+      this.reportLatency = reportLatency;
     }
 
     public int getStartRow() {
@@ -720,6 +749,10 @@ public int getTotalRows() {
       return totalRows;
     }
 
+    public float getSampleRate() {
+      return sampleRate;
+    }
+
     public byte[] getTableName() {
       return tableName;
     }
@@ -731,6 +764,10 @@ public boolean isFlushCommits() {
     public boolean isWriteToWAL() {
       return writeToWAL;
     }
+
+    public boolean isReportLatency() {
+      return reportLatency;
+    }
   }
 
   /*
@@ -750,6 +787,7 @@ private static long nextRandomSeed() {
     protected final int startRow;
     protected final int perClientRunRows;
     protected final int totalRows;
+    protected final float sampleRate;
     private final Status status;
     protected byte[] tableName;
     protected HBaseAdmin admin;
@@ -757,6 +795,7 @@ private static long nextRandomSeed() {
     protected volatile Configuration conf;
     protected boolean flushCommits;
     protected boolean writeToWAL;
+    protected boolean reportlatency;
 
     /**
      * Note that all subclasses of this class must provide a public contructor
@@ -767,12 +806,14 @@ private static long nextRandomSeed() {
       this.startRow = options.getStartRow();
       this.perClientRunRows = options.getPerClientRunRows();
       this.totalRows = options.getTotalRows();
+      this.sampleRate = options.getSampleRate();
       this.status = status;
       this.tableName = options.getTableName();
       this.table = null;
       this.conf = conf;
       this.flushCommits = options.isFlushCommits();
       this.writeToWAL = options.isWriteToWAL();
+      this.reportlatency = options.isReportLatency();
     }
 
     private String generateStatus(final int sr, final int i, final int lr) {
@@ -781,7 +822,7 @@ private String generateStatus(final int sr, final int i, final int lr) {
 
     protected int getReportingPeriod() {
       int period = this.perClientRunRows / 10;
-      return period == 0? this.perClientRunRows: period;
+      return period == 0 ? this.perClientRunRows : period;
     }
 
     void testSetup() throws IOException {
@@ -850,17 +891,14 @@ void testRow(final int i) throws IOException {
       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
       scan.setFilter(new WhileMatchFilter(new PageFilter(120)));
       ResultScanner s = this.table.getScanner(scan);
-      //int count = 0;
-      for (Result rr = null; (rr = s.next()) != null;) {
-        // LOG.info("" + count++ + " " + rr.toString());
-      }
+      for (Result rr; (rr = s.next()) != null;) ;
       s.close();
     }
 
     @Override
     protected int getReportingPeriod() {
       int period = this.perClientRunRows / 100;
-      return period == 0? this.perClientRunRows: period;
+      return period == 0 ? this.perClientRunRows : period;
     }
 
   }
@@ -878,7 +916,7 @@ void testRow(final int i) throws IOException {
       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
       ResultScanner s = this.table.getScanner(scan);
       int count = 0;
-      for (Result rr = null; (rr = s.next()) != null;) {
+      for (Result rr; (rr = s.next()) != null;) {
         count++;
       }
 
@@ -951,23 +989,49 @@ protected Pair<byte[], byte[]> getStartAndStopRow() {
   }
 
   static class RandomReadTest extends Test {
+    private final int everyN;
+    private final boolean reportLatency;
+    private final float[] times;
+    int idx = 0;
+
     RandomReadTest(Configuration conf, TestOptions options, Status status) {
       super(conf, options, status);
+      everyN = (int) (this.totalRows / (this.totalRows * this.sampleRate));
+      LOG.info("Sampling 1 every " + everyN + " out of " + perClientRunRows + " total rows.");
+      this.reportLatency = options.isReportLatency();
+      if (this.reportLatency) {
+        times = new float[(int) Math.ceil(this.perClientRunRows * this.sampleRate)];
+      } else {
+        times = null;
+      }
     }
 
     @Override
     void testRow(final int i) throws IOException {
-      Get get = new Get(getRandomRow(this.rand, this.totalRows));
-      get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
-      this.table.get(get);
+      if (i % everyN == 0) {
+        Get get = new Get(getRandomRow(this.rand, this.totalRows));
+        get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+        long start = System.nanoTime();
+        this.table.get(get);
+        if (this.reportLatency) {
+          times[idx++] = (float) ((System.nanoTime() - start) / 1000000.0);
+        }
+      }
     }
 
     @Override
     protected int getReportingPeriod() {
       int period = this.perClientRunRows / 100;
-      return period == 0? this.perClientRunRows: period;
+      return period == 0 ? this.perClientRunRows : period;
     }
 
+    @Override
+    protected void testTakedown() throws IOException {
+      super.testTakedown();
+      if (this.reportLatency) {
+        LOG.info("randomRead latency log (ms): " + Arrays.toString(times));
+      }
+    }
   }
 
   static class RandomWriteTest extends Test {
@@ -1030,7 +1094,6 @@ void testRow(final int i) throws IOException {
       get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
       table.get(get);
     }
-
   }
 
   static class SequentialWriteTest extends Test {
@@ -1046,7 +1109,6 @@ void testRow(final int i) throws IOException {
       put.setWriteToWAL(writeToWAL);
       table.put(put);
     }
-
   }
 
   static class FilteredScanTest extends Test {
@@ -1082,14 +1144,31 @@ protected Scan constructScan(byte[] valuePrefix) throws IOException {
     }
   }
 
+  /**
+   * Compute a throughput rate in MB/s.
+   * @param rows Number of records consumed.
+   * @param timeMs Time taken in milliseconds.
+   * @return String value with label, ie '123.76 MB/s'
+   */
+  private static String calculateMbps(int rows, long timeMs) {
+    // MB/s = ((totalRows * ROW_SIZE_BYTES) / totalTimeMS)
+    //        * 1000 MS_PER_SEC / (1024 * 1024) BYTES_PER_MB
+    BigDecimal rowSize =
+      BigDecimal.valueOf(VALUE_LENGTH + VALUE_LENGTH + FAMILY_NAME.length + QUALIFIER_NAME.length);
+    BigDecimal mbps = BigDecimal.valueOf(rows).multiply(rowSize, CXT)
+      .divide(BigDecimal.valueOf(timeMs), CXT).multiply(MS_PER_SEC, CXT)
+      .divide(BYTES_PER_MB, CXT);
+    return FMT.format(mbps) + " MB/s";
+  }
+
   /*
    * Format passed integer.
    * @param number
-   * @return Returns zero-prefixed 10-byte wide decimal version of passed
+   * @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of passed
    * number (Does absolute in case number is negative).
    */
   public static byte [] format(final int number) {
-    byte [] b = new byte[10];
+    byte [] b = new byte[ROW_LENGTH];
     int d = Math.abs(number);
     for (int i = b.length - 1; i >= 0; i--) {
       b[i] = (byte)((d % 10) + '0');
@@ -1105,7 +1184,7 @@ protected Scan constructScan(byte[] valuePrefix) throws IOException {
    * @return Generated random value to insert into a table cell.
    */
   public static byte[] generateValue(final Random r) {
-    byte [] b = new byte [ROW_LENGTH];
+    byte [] b = new byte [VALUE_LENGTH];
     r.nextBytes(b);
     return b;
   }
@@ -1115,17 +1194,17 @@ public static byte[] generateValue(final Random r) {
   }
 
   long runOneClient(final Class<? extends Test> cmd, final int startRow,
-                    final int perClientRunRows, final int totalRows,
-                    boolean flushCommits, boolean writeToWAL,
-                    final Status status)
+      final int perClientRunRows, final int totalRows, final float sampleRate,
+      boolean flushCommits, boolean writeToWAL, boolean reportLatency,
+      final Status status)
   throws IOException {
     status.setStatus("Start " + cmd + " at offset " + startRow + " for " +
       perClientRunRows + " rows");
     long totalElapsedTime = 0;
 
     Test t = null;
-    TestOptions options = new TestOptions(startRow, perClientRunRows,
-        totalRows, getTableDescriptor().getName(), flushCommits, writeToWAL);
+    TestOptions options = new TestOptions(startRow, perClientRunRows, totalRows,
+      sampleRate, getTableDescriptor().getName(), flushCommits, writeToWAL, reportLatency);
     try {
       Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
           Configuration.class, TestOptions.class, Status.class);
@@ -1141,11 +1220,12 @@ long runOneClient(final Class<? extends Test> cmd, final int startRow,
     totalElapsedTime = t.test();
 
     status.setStatus("Finished " + cmd + " in " + totalElapsedTime +
-      "ms at offset " + startRow + " for " + perClientRunRows + " rows");
+      "ms at offset " + startRow + " for " + perClientRunRows + " rows" +
+      " (" + calculateMbps((int)(perClientRunRows * sampleRate), totalElapsedTime) + ")");
     return totalElapsedTime;
   }
 
-  private void runNIsOne(final Class<? extends Test> cmd) {
+  private void runNIsOne(final Class<? extends Test> cmd) throws IOException {
     Status status = new Status() {
       public void setStatus(String msg) throws IOException {
         LOG.info(msg);
@@ -1156,10 +1236,12 @@ public void setStatus(String msg) throws IOException {
     try {
       admin = new HBaseAdmin(this.conf);
       checkTable(admin);
-      runOneClient(cmd, 0, this.R, this.R, this.flushCommits, this.writeToWAL,
-        status);
+      runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
+        this.writeToWAL, this.writeToWAL, status);
     } catch (Exception e) {
       LOG.error("Failed", e);
+    } finally {
+      if (admin != null) admin.close();
     }
   }
 
@@ -1219,9 +1301,15 @@ protected void printUsage(final String message) {
     System.err.println(" nomapred        Run multiple clients using threads " +
       "(rather than use mapreduce)");
     System.err.println(" rows            Rows each client runs. Default: One million");
-    System.err.println(" flushCommits    Used to determine if the test should flush the table.  Default: false");
+    System.err.println(" sampleRate      Execute test on a sample of total " +
+      "rows. Only supported by randomRead. Default: 1.0");
+    System.err.println(" flushCommits    Used to determine if the test should flush the table. " +
+      "Default: false");
     System.err.println(" writeToWAL      Set writeToWAL on puts. Default: True");
-    System.err.println(" presplit        Create presplit table. Recommended for accurate perf analysis (see guide).  Default: disabled");
+    System.err.println(" presplit        Create presplit table. Recommended for accurate perf " +
+      "analysis (see guide).  Default: disabled");
+    System.err.println(" latency         Set to report operation latencies. " +
+      "Currently only supported by randomRead test. Default: False");
     System.err.println();
     System.err.println("Command:");
     for (CmdDescriptor command : commands.values()) {
@@ -1286,6 +1374,12 @@ public int doCommandLine(final String[] args) {
           continue;
         }
 
+        final String sampleRate = "--sampleRate=";
+        if (cmd.startsWith(sampleRate)) {
+          this.sampleRate = Float.parseFloat(cmd.substring(sampleRate.length()));
+          continue;
+        }
+
         final String flushCommits = "--flushCommits=";
         if (cmd.startsWith(flushCommits)) {
           this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
@@ -1304,6 +1398,12 @@ public int doCommandLine(final String[] args) {
           continue;
         }
 
+        final String latency = "--latency";
+        if (cmd.startsWith(latency)) {
+          this.reportLatency = true;
+          continue;
+        }
+
         Class<? extends Test> cmdClass = determineCommandClass(cmd);
         if (cmdClass != null) {
           getArgs(i + 1, args);
@@ -1327,9 +1427,6 @@ private Class<? extends Test> determineCommandClass(String cmd) {
     return descriptor != null ? descriptor.getCmdClass() : null;
   }
 
-  /**
-   * @param args
-   */
   public static void main(final String[] args) {
     Configuration c = HBaseConfiguration.create();
     System.exit(new PerformanceEvaluation(c).doCommandLine(args));

From 99c4b9760591883afc6e91e758b7faf2f0bc2fb7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 26 Nov 2013 05:59:10 +0000
Subject: [PATCH 1245/1540] roll version to 0.94.15-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1545550 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 3047153b7d10..b6eb308f1266 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.14</version>
+  <version>0.94.15-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From a553d07fcff6b1f0affd7f7361bbd0a76ee70606 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Tue, 26 Nov 2013 17:13:58 +0000
Subject: [PATCH 1246/1540] HBASE-10026 HBaseAdmin#createTable could fail if
 region splits too fast

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1545740 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 5a3536c71a70..88090175b534 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -490,7 +490,7 @@ public boolean processRow(Result rowResult) throws IOException {
           }
         };
         MetaScanner.metaScan(conf, connection, visitor, desc.getName());
-        if (actualRegCount.get() != numRegs) {
+        if (actualRegCount.get() < numRegs) {
           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
             throw new RegionOfflineException("Only " + actualRegCount.get() +
               " of " + numRegs + " regions are online; retries exhausted.");

From fd0d0ca0cde7e6a67a7e1e20ac3ca9d967377896 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 26 Nov 2013 21:08:20 +0000
Subject: [PATCH 1247/1540] HBASE-10015 Replace intrinsic locking with explicit
 locks in StoreScanner

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1545840 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/StoreScanner.java      | 51 +++++++++++++++----
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 100b71801a67..6cb9711db049 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -24,6 +24,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.NavigableSet;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -75,6 +76,7 @@ public class StoreScanner extends NonLazyKeyValueScanner
 
   // A flag whether use pread for scan
   private boolean scanUsePread = false;
+  private ReentrantLock lock = new ReentrantLock();
 
   /** An internal constructor. */
   private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
@@ -267,11 +269,17 @@ private List<KeyValueScanner> selectScannersFrom(
   }
 
   @Override
-  public synchronized KeyValue peek() {
+  public KeyValue peek() {
+    lock.lock();
+    try {
+
     if (this.heap == null) {
       return this.lastTop;
     }
     return this.heap.peek();
+    } finally {
+      lock.unlock();
+    }
   }
 
   @Override
@@ -281,7 +289,9 @@ public KeyValue next() {
   }
 
   @Override
-  public synchronized void close() {
+  public void close() {
+    lock.lock();
+    try {
     if (this.closing) return;
     this.closing = true;
     // under test, we dont have a this.store
@@ -291,10 +301,15 @@ public synchronized void close() {
       this.heap.close();
     this.heap = null; // CLOSED!
     this.lastTop = null; // If both are null, we are closed.
+    } finally {
+      lock.unlock();
+    }
   }
 
   @Override
-  public synchronized boolean seek(KeyValue key) throws IOException {
+  public boolean seek(KeyValue key) throws IOException {
+    lock.lock();
+    try {
     if (this.heap == null) {
 
       List<KeyValueScanner> scanners = getScannersNoCompaction();
@@ -303,6 +318,9 @@ public synchronized boolean seek(KeyValue key) throws IOException {
     }
 
     return this.heap.seek(key);
+  } finally {
+    lock.unlock();
+  }
   }
 
   /**
@@ -312,7 +330,7 @@ public synchronized boolean seek(KeyValue key) throws IOException {
    * @return true if there are more rows, false if scanner is done
    */
   @Override
-  public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException {
+  public boolean next(List<KeyValue> outResult, int limit) throws IOException {
     return next(outResult, limit, null);
   }
 
@@ -323,8 +341,10 @@ public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOE
    * @return true if there are more rows, false if scanner is done
    */
   @Override
-  public synchronized boolean next(List<KeyValue> outResult, int limit,
+  public boolean next(List<KeyValue> outResult, int limit,
       String metric) throws IOException {
+    lock.lock();
+    try {
 
     if (checkReseek()) {
       return true;
@@ -447,22 +467,27 @@ public synchronized boolean next(List<KeyValue> outResult, int limit,
     // No more keys
     close();
     return false;
+    } finally {
+      lock.unlock();
+    }
   }
 
   @Override
-  public synchronized boolean next(List<KeyValue> outResult) throws IOException {
+  public boolean next(List<KeyValue> outResult) throws IOException {
     return next(outResult, -1, null);
   }
 
   @Override
-  public synchronized boolean next(List<KeyValue> outResult, String metric)
+  public boolean next(List<KeyValue> outResult, String metric)
       throws IOException {
     return next(outResult, -1, metric);
   }
 
   // Implementation of ChangedReadersObserver
   @Override
-  public synchronized void updateReaders() throws IOException {
+  public void updateReaders() throws IOException {
+    lock.lock();
+    try {
     if (this.closing) return;
 
     // All public synchronized API calls will call 'checkReseek' which will cause
@@ -482,6 +507,9 @@ public synchronized void updateReaders() throws IOException {
     this.heap = null; // the re-seeks could be slow (access HDFS) free up memory ASAP
 
     // Let the next() call handle re-creating and seeking
+    } finally {
+      lock.unlock();
+    }
   }
 
   /**
@@ -539,7 +567,9 @@ private void resetScannerStack(KeyValue lastTopKey) throws IOException {
   }
 
   @Override
-  public synchronized boolean reseek(KeyValue kv) throws IOException {
+  public boolean reseek(KeyValue kv) throws IOException {
+    lock.lock();
+    try {
     //Heap will not be null, if this is called from next() which.
     //If called from RegionScanner.reseek(...) make sure the scanner
     //stack is reset if needed.
@@ -549,6 +579,9 @@ public synchronized boolean reseek(KeyValue kv) throws IOException {
     } else {
       return heap.reseek(kv);
     }
+    } finally {
+      lock.unlock();
+    }
   }
 
   @Override

From 71dfc9a95e6debef0d04768b2291eb7b58a84eb7 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 27 Nov 2013 16:34:52 +0000
Subject: [PATCH 1248/1540] HBASE-10046 Unmonitored HBase service could
 accumulate Status objects and OOM

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1546098 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/monitoring/TaskMonitor.java    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
index 942b7c112589..d9c4d4bf99cc 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
@@ -72,8 +72,9 @@ public synchronized MonitoredTask createStatus(String description) {
         new Class<?>[] { MonitoredTask.class },
         new PassthroughInvocationHandler<MonitoredTask>(stat));
     TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
-    synchronized (this) {
-      tasks.add(pair);
+    tasks.add(pair);
+    if (tasks.size() > MAX_TASKS) {
+      purgeExpiredTasks();
     }
     return proxy;
   }
@@ -86,8 +87,9 @@ public synchronized MonitoredRPCHandler createRPCStatus(String description) {
         new Class<?>[] { MonitoredRPCHandler.class },
         new PassthroughInvocationHandler<MonitoredRPCHandler>(stat));
     TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
-    synchronized (this) {
-      tasks.add(pair);
+    tasks.add(pair);
+    if (tasks.size() > MAX_TASKS) {
+      purgeExpiredTasks();
     }
     return proxy;
   }

From 499a3a061d5ca037caf5aeb21343afe5538c64aa Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 28 Nov 2013 18:00:08 +0000
Subject: [PATCH 1249/1540] HBASE-10049 Small improvments in region_mover.rb

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1546417 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/region_mover.rb | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 5c6bea0ae1b2..d2bbfe3ff84c 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -75,6 +75,17 @@ def getTable(config, name)
   return $TABLES[key]
 end
 
+def closeTables()
+  if not $TABLES
+    return
+  end
+
+  $LOG.info("Close all tables")
+  $TABLES.each do |name, table|
+    $TABLES.delete(name)
+    table.close()
+  end
+end
 
 # Returns true if passed region is still on 'original' when we look at .META.
 def isSameServer(admin, r, original)
@@ -137,7 +148,9 @@ def isSuccessfulScan(admin, r)
     # But if no exception, presume scanning is working.
   ensure
     scanner.close()
-    table.close()
+    # Do not close the htable. It is cached in $TABLES and 
+    # may be reused in moving another region of same table. 
+    # table.close()
   end
 end
 
@@ -150,6 +163,7 @@ def move(admin, r, newServer, original)
   retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5)
   count = 0
   same = true
+  start = Time.now
   while count < retries and same
     if count > 0
       $LOG.info("Retry " + count.to_s + " of maximum " + retries.to_s)
@@ -174,6 +188,8 @@ def move(admin, r, newServer, original)
   raise RuntimeError, "Region stuck on #{original}, newserver=#{newServer}" if same
   # Assert can Scan from new location.
   isSuccessfulScan(admin, r)
+  $LOG.info("Moved region " + r.getRegionNameAsString() + " cost: " + 
+    java.lang.String.format("%.3f", (Time.now - start)))
 end
 
 # Return the hostname portion of a servername (all up to first ',')
@@ -333,8 +349,9 @@ def unloadRegions(options, hostname)
     for r in rs
       # Get a random server to move the region to.
       server = servers[rand(servers.length)]
-      $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s +
-        " of " + rs.length.to_s + ") to server=" + server);
+      $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + 
+        count.to_s + " of " + rs.length.to_s + ") from server=" + 
+        servername + " to server=" + server);
       count = count + 1
       # Assert we can scan region in its current location
       isSuccessfulScan(admin, r)
@@ -375,6 +392,8 @@ def loadRegions(options, hostname)
   end
   $LOG.info("Moving " + regions.size().to_s + " regions to " + servername)
   count = 0
+  # sleep 20s to make sure the rs finished initialization.
+  sleep 20
   for r in regions
     exists = false
     begin
@@ -391,8 +410,9 @@ def loadRegions(options, hostname)
         " of " + regions.length.to_s + ") already on target server=" + servername) 
       next
     end
-    $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s +
-      " of " + regions.length.to_s + ") to server=" + servername);
+    $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + 
+      count.to_s + " of " + regions.length.to_s + ") from server=" + 
+      currentServer + " to server=" + servername);
     move(admin, r, servername, currentServer)
   end
 end
@@ -473,3 +493,5 @@ def getFilename(options, targetServer)
     puts optparse
     exit 3
 end
+
+closeTables()

From 0678603c88771cf94147edc6e20ff3992f29e558 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 28 Nov 2013 18:38:42 +0000
Subject: [PATCH 1250/1540] HBASE-10057 TestRestoreFlushSnapshotFromClient and
 TestRestoreSnapshotFromClient fail to finish occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1546443 13f79535-47bb-0310-9956-ffa450edef68
---
 .../client/TestRestoreSnapshotFromClient.java | 46 +++++++-------
 .../TestRestoreFlushSnapshotFromClient.java   | 63 +++++++------------
 2 files changed, 45 insertions(+), 64 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 324a176e02c1..1da0e94f7850 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
@@ -107,30 +106,27 @@ public void setup() throws Exception {
     admin.snapshot(emptySnapshot, tableName);
 
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    try {
-      // enable table and insert data
-      admin.enableTable(tableName);
-      SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
-      snapshot0Rows = TEST_UTIL.countRows(table);
-      admin.disableTable(tableName);
-
-      // take a snapshot
-      admin.snapshot(snapshotName0, tableName);
-
-      // enable table and insert more data
-      admin.enableTable(tableName);
-      SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
-      snapshot1Rows = TEST_UTIL.countRows(table);
-      admin.disableTable(tableName);
-
-      // take a snapshot of the updated table
-      admin.snapshot(snapshotName1, tableName);
-
-      // re-enable table
-      admin.enableTable(tableName);
-    } finally {
-      table.close();
-    }
+    // enable table and insert data
+    admin.enableTable(tableName);
+    SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
+    snapshot0Rows = TEST_UTIL.countRows(table);
+    admin.disableTable(tableName);
+
+    // take a snapshot
+    admin.snapshot(snapshotName0, tableName);
+
+    // enable table and insert more data
+    admin.enableTable(tableName);
+    SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
+    snapshot1Rows = TEST_UTIL.countRows(table);
+    admin.disableTable(tableName);
+
+    // take a snapshot of the updated table
+    admin.snapshot(snapshotName1, tableName);
+
+    // re-enable table
+    admin.enableTable(tableName);
+    table.close();
   }
 
   @After
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
index 519086e44f3c..7eadad9aee92 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
@@ -17,30 +17,19 @@
  */
 package org.apache.hadoop.hbase.snapshot;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.io.IOException;
-import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -61,7 +50,6 @@ public class TestRestoreFlushSnapshotFromClient {
   private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
 
   private final byte[] FAMILY = Bytes.toBytes("cf");
-  private static final byte[] TEST_QUAL = Bytes.toBytes("q");
 
   private byte[] snapshotName0;
   private byte[] snapshotName1;
@@ -109,33 +97,30 @@ public void setup() throws Exception {
     // create Table and disable it
     SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY);
     HTable table = new HTable(UTIL.getConfiguration(), tableName);
-    try {
-      SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
-      snapshot0Rows = UTIL.countRows(table);
-      LOG.info("=== before snapshot with 500 rows");
-      logFSTree();
-
-      // take a snapshot
-      admin.snapshot(Bytes.toString(snapshotName0), Bytes.toString(tableName),
-          SnapshotDescription.Type.FLUSH);
-
-      LOG.info("=== after snapshot with 500 rows");
-      logFSTree();
-
-      // insert more data
-      SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
-      snapshot1Rows = UTIL.countRows(table);
-      LOG.info("=== before snapshot with 1000 rows");
-      logFSTree();
-
-      // take a snapshot of the updated table
-      admin.snapshot(Bytes.toString(snapshotName1), Bytes.toString(tableName),
-          SnapshotDescription.Type.FLUSH);
-      LOG.info("=== after snapshot with 1000 rows");
-      logFSTree();
-    } finally {
-      table.close();
-    }
+    SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
+    snapshot0Rows = UTIL.countRows(table);
+    LOG.info("=== before snapshot with 500 rows");
+    logFSTree();
+
+    // take a snapshot
+    admin.snapshot(Bytes.toString(snapshotName0), Bytes.toString(tableName),
+        SnapshotDescription.Type.FLUSH);
+
+    LOG.info("=== after snapshot with 500 rows");
+    logFSTree();
+
+    // insert more data
+    SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
+    snapshot1Rows = UTIL.countRows(table);
+    LOG.info("=== before snapshot with 1000 rows");
+    logFSTree();
+
+    // take a snapshot of the updated table
+    admin.snapshot(Bytes.toString(snapshotName1), Bytes.toString(tableName),
+        SnapshotDescription.Type.FLUSH);
+    LOG.info("=== after snapshot with 1000 rows");
+    logFSTree();
+    table.close();
   }
 
   @After

From 56aaf9af20a0c2cd8d7c7baeafe02edaf3dc8d68 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 29 Nov 2013 23:03:57 +0000
Subject: [PATCH 1251/1540] HBASE-10058 Test for HBASE-9915 (avoid reading
 index blocks)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1546659 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestBlocksScanned.java | 64 ++++++++++++-------
 1 file changed, 42 insertions(+), 22 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
index 19c690f6c560..932a8fbe9ff5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
@@ -39,7 +40,6 @@
 @SuppressWarnings("deprecation")
 @Category(SmallTests.class)
 public class TestBlocksScanned extends HBaseTestCase {
-  private static byte [] TABLE = Bytes.toBytes("TestBlocksScanned");
   private static byte [] FAMILY = Bytes.toBytes("family");
   private static byte [] COL = Bytes.toBytes("col");
   private static byte [] START_KEY = Bytes.toBytes("aaa");
@@ -47,34 +47,54 @@ public class TestBlocksScanned extends HBaseTestCase {
   private static int BLOCK_SIZE = 70;
 
   private static HBaseTestingUtility TEST_UTIL = null;
-  private static HTableDescriptor TESTTABLEDESC = null;
-
-   @Override
-   public void setUp() throws Exception {
-     super.setUp();
-     SchemaMetrics.setUseTableNameInTest(true);
-     TEST_UTIL = new HBaseTestingUtility();
-     TESTTABLEDESC = new HTableDescriptor(TABLE);
-
-     TESTTABLEDESC.addFamily(
-         new HColumnDescriptor(FAMILY)
-         .setMaxVersions(10)
-         .setBlockCacheEnabled(true)
-         .setBlocksize(BLOCK_SIZE)
-         .setCompressionType(Compression.Algorithm.NONE)
-     );
-   }
-
-   @Test
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    SchemaMetrics.setUseTableNameInTest(true);
+    TEST_UTIL = new HBaseTestingUtility();
+  }
+
+  @Test
   public void testBlocksScanned() throws Exception {
-    HRegion r = createNewHRegion(TESTTABLEDESC, START_KEY, END_KEY,
+    byte [] tableName = Bytes.toBytes("TestBlocksScanned");
+    HTableDescriptor table = new HTableDescriptor(tableName);
+
+    table.addFamily(
+        new HColumnDescriptor(FAMILY)
+        .setMaxVersions(10)
+        .setBlockCacheEnabled(true)
+        .setBlocksize(BLOCK_SIZE)
+        .setCompressionType(Compression.Algorithm.NONE)
+        );
+    _testBlocksScanned(table);
+  }
+
+  @Test
+  public void testBlocksScannedWithEncoding() throws Exception {
+    byte [] tableName = Bytes.toBytes("TestBlocksScannedWithEncoding");
+    HTableDescriptor table = new HTableDescriptor(tableName);
+
+    table.addFamily(
+        new HColumnDescriptor(FAMILY)
+        .setMaxVersions(10)
+        .setBlockCacheEnabled(true)
+        .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF)
+        .setBlocksize(BLOCK_SIZE)
+        .setCompressionType(Compression.Algorithm.NONE)
+        );
+    _testBlocksScanned(table);
+  }
+
+  private void _testBlocksScanned(HTableDescriptor table) throws Exception {
+    HRegion r = createNewHRegion(table, START_KEY, END_KEY,
         TEST_UTIL.getConfiguration());
     addContent(r, FAMILY, COL);
     r.flushcache();
 
     // Get the per-cf metrics
     SchemaMetrics schemaMetrics =
-      SchemaMetrics.getInstance(Bytes.toString(TABLE), Bytes.toString(FAMILY));
+      SchemaMetrics.getInstance(Bytes.toString(table.getName()), Bytes.toString(FAMILY));
     Map<String, Long> schemaMetricSnapshot = SchemaMetrics.getMetricsSnapshot();
 
     // Do simple test of getting one row only first.

From 4d9771baab9ccf1ab4847c388d4748b79196b91b Mon Sep 17 00:00:00 2001
From: ndimiduk <ndimiduk@unknown>
Date: Tue, 3 Dec 2013 16:39:51 +0000
Subject: [PATCH 1252/1540] Optional setBatch for CopyTable to copy large rows
 in batches

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1547471 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/TableInputFormat.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
index 27abad569ce4..58bd78c7928e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
@@ -65,6 +65,8 @@ public class TableInputFormat extends TableInputFormatBase
   public static final String SCAN_CACHEBLOCKS = "hbase.mapreduce.scan.cacheblocks";
   /** The number of rows for caching that will be passed to scanners. */
   public static final String SCAN_CACHEDROWS = "hbase.mapreduce.scan.cachedrows";
+  /** Set the maximum number of values to return for each call to next(). */
+  public static final String SCAN_BATCHSIZE = "hbase.mapreduce.scan.batchsize";
 
   /** The configuration. */
   private Configuration conf = null;
@@ -144,6 +146,10 @@ public void setConf(Configuration configuration) {
           scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
         }
 
+        if (conf.get(SCAN_BATCHSIZE) != null) {
+          scan.setBatch(Integer.parseInt(conf.get(SCAN_BATCHSIZE)));
+        }
+
         // false by default, full table scans generate too much BC churn
         scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
       } catch (Exception e) {

From c325123083fd407ce398160419a2f47ed72b56d3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 4 Dec 2013 04:29:41 +0000
Subject: [PATCH 1253/1540] HBASE-9986 Incorporate HTTPS support for HBase
 (0.94 port) (Aditya Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1547706 13f79535-47bb-0310-9956-ffa450edef68
---
 .../tmpl/master/BackupMasterStatusTmpl.jamon  |  4 ++--
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  |  2 +-
 .../tmpl/regionserver/RSStatusTmpl.jamon      |  2 +-
 .../resources/hbase-webapps/master/table.jsp  | 19 +++++++++----------
 4 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
index 1dabe274d255..ca8308b5e3fc 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon
@@ -50,7 +50,7 @@ ServerName [] serverNames = masters.toArray(new ServerName[masters.size()]);
 </%java>
 <%if (!master.isActiveMaster()) %>
     <h2>Master</h2>
-    <a href="http://<% serverNames[0].getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverNames[0].getHostname() %></a>
+    <a href="//<% serverNames[0].getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverNames[0].getHostname() %></a>
 <%else>
     <h2>Backup Masters</h2>
 
@@ -65,7 +65,7 @@ ServerName [] serverNames = masters.toArray(new ServerName[masters.size()]);
     for (ServerName serverName: serverNames) {
     </%java>
     <tr>
-        <td><a href="http://<% serverName.getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverName.getHostname() %></a></td>
+        <td><a href="//<% serverName.getHostname() %>:<% master.getConfiguration().getInt("hbase.master.info.port", 60010) %>/master-status" target="_blank"><% serverName.getHostname() %></a></td>
         <td><% serverName.getPort() %></td>
         <td><% new Date(serverName.getStartcode()) %></td>
     </tr>
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 3a28794c4949..8683be6ecc8c 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -256,7 +256,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
        // TODO: this is incorrect since this conf might differ from RS to RS
        // or be set to 0 to get ephemeral ports
        int infoPort = master.getConfiguration().getInt("hbase.regionserver.info.port", 60030);
-       String url = "http://" + serverName.getHostname() + ":" + infoPort + "/";
+       String url = "//" + serverName.getHostname() + ":" + infoPort + "/";
        HServerLoad hsl = master.getServerManager().getLoad(serverName);
        String loadStr = hsl == null? "-": hsl.toString();
        if (hsl != null) {
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
index ae762048caec..ee17852b1085 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
@@ -105,7 +105,7 @@ No hbase.master.info.port found
 <%else>
 <%java>
 String host = regionServer.getMasterAddressManager().getMasterAddress().getHostname() + ":" + masterInfoPort;
-String url = "http://" + host + "/";
+String url = "//" + host + "/";
 </%java>
 <a href="<% url %>"><% host %></a>
 </%if>
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 52b2950a3087..f44cbfb275b2 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -118,7 +118,7 @@
 %>
 <%= tableHeader %>
 <%
-  String url = "http://" + rl.getHostname() + ":" + infoPort + "/";
+  String url = "//" + rl.getHostname() + ":" + infoPort + "/";
 %>
 <tr>
   <td><%= tableName %></td>
@@ -137,7 +137,7 @@
   HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
   ServerName metaLocation = master.getCatalogTracker().waitForMeta(1);
   for (int i = 0; i < 1; i++) {
-    String url = "http://" + metaLocation.getHostname() + ":" + infoPort + "/";
+    String url = "//" + metaLocation.getHostname() + ":" + infoPort + "/";
 %>
 <tr>
   <td><%= meta.getRegionNameAsString() %></td>
@@ -183,7 +183,7 @@
     ServerName addr = hriEntry.getValue();
     long req = 0;
 
-    String urlRegionServer = null;
+    String regionServer = null;
 
     if (addr != null) {
       HServerLoad sl = master.getServerManager().getLoad(addr);
@@ -193,21 +193,20 @@
           req = map.get(regionInfo.getRegionName()).getRequestsCount();
         }
         // This port might be wrong if RS actually ended up using something else.
-        urlRegionServer =
-            "http://" + addr.getHostname().toString() + ":" + infoPort + "/";
-        Integer i = regDistribution.get(urlRegionServer);
+        regionServer = addr.getHostname().toString() + ":" + infoPort;
+        Integer i = regDistribution.get(regionServer);
         if (null == i) i = new Integer(0);
-        regDistribution.put(urlRegionServer, i+1);
+        regDistribution.put(regionServer, i+1);
       }
     }
 %>
 <tr>
   <td><%= Bytes.toStringBinary(regionInfo.getRegionName())%></td>
   <%
-  if (urlRegionServer != null) {
+  if (regionServer != null) {
   %>
   <td>
-    <a href="<%= urlRegionServer %>"><%= addr.getHostname().toString() + ":" + infoPort %></a>
+    <a href="<%= "//" + regionServer + "/" %>"><%= regionServer %></a>
   </td>
   <%
   } else {
@@ -228,7 +227,7 @@
   for (Map.Entry<String, Integer> rdEntry : regDistribution.entrySet()) {
 %>
 <tr>
-  <td><%= rdEntry.getKey()%></td>
+  <td><a href="<%= "//" + rdEntry.getKey() + "/" %>"><%= rdEntry.getKey() %></a></td>
   <td><%= rdEntry.getValue()%></td>
 </tr>
 <% } %>

From 8b0a56d2c10a7ebc94dade103fd4dde26e7f5f7c Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 4 Dec 2013 21:08:51 +0000
Subject: [PATCH 1254/1540] HBASE-9485 TableOutputCommitter should implement
 recovery if we don't want jobs to start from 0 on RM restart

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1547917 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/TableOutputCommitter.java      | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
index 5289da7e0dc7..90f066af404b 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
@@ -55,4 +55,12 @@ public void setupJob(JobContext arg0) throws IOException {
   public void setupTask(TaskAttemptContext arg0) throws IOException {
   }
 
+  public boolean isRecoverySupported() {
+    return true;
+  }
+
+  public void recoverTask(TaskAttemptContext taskContext)
+  throws IOException
+  {
+  }
 }

From 89ef938c3d81307d2cb547c565497df5463aa8cc Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Fri, 6 Dec 2013 08:30:27 +0000
Subject: [PATCH 1255/1540] HBASE-10064. AggregateClient.validateParameters can
 throw NPE (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1548416 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/client/Scan.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 17a110acf297..9a1821134016 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -272,7 +272,7 @@ public Scan setTimeStamp(long timestamp) {
    * @return this
    */
   public Scan setStartRow(byte [] startRow) {
-    this.startRow = startRow;
+    this.startRow = startRow == null ? HConstants.EMPTY_START_ROW : startRow;
     return this;
   }
 
@@ -283,7 +283,7 @@ public Scan setStartRow(byte [] startRow) {
    * @return this
    */
   public Scan setStopRow(byte [] stopRow) {
-    this.stopRow = stopRow;
+    this.stopRow = stopRow == null ? HConstants.EMPTY_END_ROW : stopRow;
     return this;
   }
 

From c496bdf98ba746dde0390bf5c466d18a131d0585 Mon Sep 17 00:00:00 2001
From: ndimiduk <ndimiduk@unknown>
Date: Fri, 6 Dec 2013 23:13:54 +0000
Subject: [PATCH 1256/1540] HBASE-10061 TableMapReduceUtil.findOrCreateJar
 calls updateMap(null, ) resulting in thrown NPE (Amit Sela)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1548750 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 2014e2b3edf8..cdca97ce5120 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -683,7 +683,7 @@ private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
     }
 
     if (null == jar || jar.isEmpty()) {
-      throw new IOException("Cannot locate resource for class " + my_class.getName());
+      return null;
     }
 
     LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
@@ -697,6 +697,9 @@ private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
    * @param packagedClasses map[class -> jar]
    */
   private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
+    if (null == jar || jar.isEmpty()) {
+      return;
+    }
     ZipFile zip = null;
     try {
       zip = new ZipFile(jar);

From 0ac7f744c56560c232a9c2a130281a6e96260c12 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 7 Dec 2013 07:26:17 +0000
Subject: [PATCH 1257/1540] HBASE-10093 Unregister ReplicationSource metric
 bean when the replication source thread is terminated (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1548802 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/ReplicationSource.java      | 1 +
 .../regionserver/ReplicationSourceMetrics.java           | 9 ++++++++-
 .../replication/regionserver/ReplicationStatistics.java  | 6 ++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 58e38c42b6e0..8cceadb69f5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -443,6 +443,7 @@ public void run() {
         LOG.debug("Attempt to close connection failed", e);
       }
     }
+    metrics.stopReportMetrics();
     LOG.debug("Source exiting " + peerId);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
index 705a6dde55f4..da0905c5dfe7 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
@@ -38,6 +38,7 @@
 public class ReplicationSourceMetrics implements Updater {
   private final MetricsRecord metricsRecord;
   private MetricsRegistry registry = new MetricsRegistry();
+  private ReplicationStatistics replicationStatistics;
 
   /** Rate of shipped operations by the source */
   public final MetricsRate shippedOpsRate =
@@ -87,7 +88,7 @@ public ReplicationSourceMetrics(String id) {
       id = "CAN'T ENCODE UTF8";
     }
     // export for JMX
-    new ReplicationStatistics(this.registry, "ReplicationSource for " + id);
+    replicationStatistics = new ReplicationStatistics(this.registry, "ReplicationSource for " + id);
   }
 
   /**
@@ -121,4 +122,10 @@ public void doUpdates(MetricsContext metricsContext) {
     }
     this.metricsRecord.update();
   }
+  
+  public void stopReportMetrics() {
+    if (this.replicationStatistics != null) {
+      this.replicationStatistics.unRegisterMBean();
+    }
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java
index 54ca3df75ffe..d496e5fb828a 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java
@@ -42,4 +42,10 @@ public ReplicationStatistics(MetricsRegistry registry, String name) {
     super(registry, name);
     mbeanName = MBeanUtil.registerMBean("Replication", name, this);
   }
+  
+  public void unRegisterMBean() {
+    if (mbeanName != null) {
+      MBeanUtil.unregisterMBean(mbeanName);
+    }
+  }
 }

From 5df010663326be9e09cf3201318de60f42fc89f6 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 9 Dec 2013 05:02:29 +0000
Subject: [PATCH 1258/1540] HBASE-10010 eliminate the put latency spike on the
 new log file beginning

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1549385 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/wal/HLog.java     | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index a4f156ff9617..b711677a6265 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -625,6 +625,13 @@ OutputStream getOutputStream() {
       FSDataOutputStream nextHdfsOut = null;
       if (nextWriter instanceof SequenceFileLogWriter) {
         nextHdfsOut = ((SequenceFileLogWriter)nextWriter).getWriterFSDataOutputStream();
+        // perform the costly sync before we get the lock to roll writers.
+        try {
+          nextWriter.sync();
+        } catch (IOException e) {
+          // optimization failed, no need to abort here.
+          LOG.warn("pre-sync failed", e);
+        }
       }
 
       synchronized (updateLock) {

From d24751d8ca28e5186908ea34b65134f43f93d6c2 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 10 Dec 2013 11:46:14 +0000
Subject: [PATCH 1259/1540] HBASE-10111 Verify that a snapshot is not corrupted
 before restoring it

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1549820 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/snapshot/SnapshotManager.java      |  4 +++
 .../hbase/snapshot/SnapshotReferenceUtil.java | 29 +++++++++++++++
 .../client/TestRestoreSnapshotFromClient.java | 18 ++++++++++
 .../hbase/snapshot/SnapshotTestingUtils.java  | 36 +++++++++++++++++++
 4 files changed, 87 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index b4f3fbb6bcff..cd1fd5f85b6d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -65,6 +65,7 @@
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -697,6 +698,9 @@ public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException
     // stop tracking "abandoned" handlers
     cleanupSentinels();
 
+    // Verify snapshot validity
+    SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, snapshotDir, fsSnapshot);
+
     // Execute the restore/clone operation
     if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
       if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
index eee205792a56..f8c8cdd38973 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.snapshot;
 
 import java.io.IOException;
+import java.io.FileNotFoundException;
 import java.util.HashSet;
 import java.util.TreeMap;
 import java.util.LinkedList;
@@ -29,6 +30,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileSystem;
@@ -37,6 +39,7 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.FSVisitor;
@@ -155,6 +158,32 @@ public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
     FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
   }
 
+  /**
+   * Verify the validity of the snapshot
+   *
+   * @param conf The current {@link Configuration} instance.
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
+   * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify
+   * @throws CorruptedSnapshotException if the snapshot is corrupted
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void verifySnapshot(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
+    final String table = snapshotDesc.getTable();
+    visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile (final String region, final String family, final String hfile)
+          throws IOException {
+        HFileLink link = HFileLink.create(conf, table, region, family, hfile);
+        try {
+          link.getFileStatus(fs);
+        } catch (FileNotFoundException e) {
+          throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
+        }
+      }
+    });
+  }
+
   /**
    * Returns the set of region names available in the snapshot.
    *
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 1da0e94f7850..04cb2f8d2323 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.client;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
@@ -34,6 +35,7 @@
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -245,6 +247,22 @@ public void testCloneAndRestoreSnapshot() throws IOException, InterruptedExcepti
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
   }
 
+  @Test
+  public void testCorruptedSnapshot() throws IOException, InterruptedException {
+    SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
+    byte[] cloneName = Bytes.toBytes("corruptedClone-" + System.currentTimeMillis());
+    try {
+      admin.cloneSnapshot(snapshotName0, cloneName);
+      fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
+    } catch (CorruptedSnapshotException e) {
+      // Got the expected corruption exception.
+      // check for no references of the cloned table.
+      assertFalse(admin.tableExists(cloneName));
+    } catch (Exception e) {
+      fail("Expected CorruptedSnapshotException got: " + e);
+    }
+  }
+
   // ==========================================================================
   //  Helpers
   // ==========================================================================
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 1a2dc89a78c3..16803460a848 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -46,11 +46,13 @@
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -365,6 +367,40 @@ public static void createSnapshotAndValidate(HBaseAdmin admin,
       new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
   }
 
+  /**
+   * Corrupt the specified snapshot by deleting some files.
+   *
+   * @param util {@link HBaseTestingUtility}
+   * @param snapshotName name of the snapshot to corrupt
+   * @return array of the corrupted HFiles
+   * @throws IOException on unexecpted error reading the FS
+   */
+  public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
+      throws IOException {
+    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
+    final FileSystem fs = mfs.getFileSystem();
+
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
+                                                                        mfs.getRootDir());
+    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    final String table = snapshotDesc.getTable();
+
+    final ArrayList corruptedFiles = new ArrayList();
+    SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile (final String region, final String family, final String hfile)
+          throws IOException {
+        HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
+        if (corruptedFiles.size() % 2 == 0) {
+          fs.delete(link.getAvailablePath(fs));
+          corruptedFiles.add(hfile);
+        }
+      }
+    });
+
+    assertTrue(corruptedFiles.size() > 0);
+    return corruptedFiles;
+  }
+
   // ==========================================================================
   //  Table Helpers
   // ==========================================================================

From b8a79bba9be22135dbb6f052bfd8a5b7808cc4d2 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Tue, 10 Dec 2013 18:20:56 +0000
Subject: [PATCH 1260/1540] HBASE-10112. REST query params for maxVersions and
 maxValues are not parsed (Jean-Marc Spaggiari)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1549916 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java b/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
index 93d0ed1adeb4..033d685d40cc 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
@@ -245,7 +245,7 @@ private int parseQueryParams(final String path, int i) {
       case 'm': {
         StringBuilder sb = new StringBuilder();
         while (j <= query.length()) {
-          c = query.charAt(i);
+          c = query.charAt(j);
           if (c < '0' || c > '9') {
             j--;
             break;
@@ -257,7 +257,7 @@ private int parseQueryParams(final String path, int i) {
       case 'n': {
         StringBuilder sb = new StringBuilder();
         while (j <= query.length()) {
-          c = query.charAt(i);
+          c = query.charAt(j);
           if (c < '0' || c > '9') {
             j--;
             break;

From 6a439ee54819c9009847b0da3b3758aca6039edd Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 10 Dec 2013 18:21:06 +0000
Subject: [PATCH 1261/1540] HBASE-10117 Avoid synchronization in
 HRegionScannerImpl.isFilterDone

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1549917 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegion.java     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 0e958a6db903..f082b8ee08fb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3794,7 +3794,7 @@ class RegionScannerImpl implements RegionScanner {
     // KeyValue indicating that limit is reached when scanning
     private final KeyValue KV_LIMIT = new KeyValue();
     private final byte [] stopRow;
-    private Filter filter;
+    private final Filter filter;
     private int batch;
     private int isScan;
     private boolean filterClosed = false;
@@ -3924,7 +3924,7 @@ public boolean nextRaw(List<KeyValue> outResults, int limit,
         outResults.addAll(tmpList);
       }
       resetFilters();
-      if (isFilterDone()) {
+      if (isFilterDoneInternal()) {
         return false;
       }
       return returnResult;
@@ -3987,6 +3987,10 @@ private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int l
      * @return True if a filter rules the scanner is over, done.
      */
     public synchronized boolean isFilterDone() {
+      return isFilterDoneInternal();
+    }
+
+    private boolean isFilterDoneInternal() {
       return this.filter != null && this.filter.filterAllRemaining();
     }
 

From 66f6e86597c6ba4f1fba48a8e2dbfc541d53621a Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 11 Dec 2013 02:27:47 +0000
Subject: [PATCH 1262/1540] HBASE-10119. Allow HBase coprocessors to clean up
 when they fail (Benoit Sigoure)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1550030 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/coprocessor/CoprocessorHost.java  | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 4b7ea3c31362..c41d63c15514 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -751,8 +751,14 @@ protected void handleCoprocessorThrowable(final CoprocessorEnvironment env,
       LOG.error("Removing coprocessor '" + env.toString() + "' from " +
           "environment because it threw:  " + e,e);
       coprocessors.remove(env);
+      try {
+        shutdown(env);
+      } catch (Exception x) {
+        LOG.error("Uncaught exception when shutting down coprocessor '"
+            + env.toString() + "'", x);
+      }
       throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
-          "' threw: '" + e + "' and has been removed" + "from the active " +
+          "' threw: '" + e + "' and has been removed from the active " +
           "coprocessor set.", e);
     }
   }

From c8ae3418fc8de6a104d506a9b33497cfc56629ce Mon Sep 17 00:00:00 2001
From: ndimiduk <ndimiduk@unknown>
Date: Wed, 11 Dec 2013 03:16:16 +0000
Subject: [PATCH 1263/1540] HBASE-10120 start-hbase.sh doesn't respect --config
 in non-distributed mode

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1550041 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/start-hbase.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/start-hbase.sh b/bin/start-hbase.sh
index 6f3cd9085d94..ca95f49ee631 100755
--- a/bin/start-hbase.sh
+++ b/bin/start-hbase.sh
@@ -44,7 +44,7 @@ distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBa
 
 if [ "$distMode" == 'false' ] 
 then
-  "$bin"/hbase-daemon.sh start master
+  "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master
 else
   "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
   "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 

From bc37d486a4905d7f3f3b40a77ce5399187c87912 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Thu, 12 Dec 2013 22:53:09 +0000
Subject: [PATCH 1264/1540] HBASE-10106: Remove some unnecessary code from
 TestOpenTableInCoprocessor (Benoit Sigoure)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1550567 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
index e73d7adcd6ab..c0d4a50c4ef9 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestOpenTableInCoprocessor.java
@@ -68,8 +68,6 @@ public static class SendToOtherTableCoprocessor extends BaseRegionObserver {
     public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
         boolean writeToWAL) throws IOException {
       HTableInterface table = e.getEnvironment().getTable(otherTable);
-      Put p = new Put(new byte[] { 'a' });
-      p.add(family, null, new byte[] { 'a' });
       table.put(put);
       table.flushCommits();
       completed[0] = true;
@@ -194,4 +192,4 @@ private int getKeyValueCount(HTable table) throws IOException {
 
     return count;
   }
-}
\ No newline at end of file
+}

From 973098a0c124edc835b83c9e422e71affd9000a5 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 13 Dec 2013 19:03:07 +0000
Subject: [PATCH 1265/1540] HBASE-10089 Metrics intern table names cause
 eventual permgen OOM in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1550809 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/metrics/SchemaConfigured.java   | 16 +++++++---------
 .../regionserver/metrics/SchemaMetrics.java      |  2 +-
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
index 74939518fb90..05d808994bef 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
@@ -123,10 +123,8 @@ public SchemaConfigured(Configuration conf, Path path) {
           // This is probably a compaction or flush output file. We will set
           // the real CF name later.
           cfName = null;
-        } else {
-          cfName = cfName.intern();
         }
-        tableName = splits[splits.length - 4].intern();
+        tableName = splits[splits.length - 4];
         return;
       }
     }
@@ -153,13 +151,13 @@ public SchemaConfigured(Path path) {
   public SchemaConfigured(Configuration conf, String tableName, String cfName)
   {
     this(conf);
-    this.tableName = tableName != null ? tableName.intern() : tableName;
-    this.cfName = cfName != null ? cfName.intern() : cfName;
+    this.tableName = tableName;
+    this.cfName = cfName;
   }
 
   public SchemaConfigured(SchemaAware that) {
-    tableName = that.getTableName().intern();
-    cfName = that.getColumnFamilyName().intern();
+    tableName = that.getTableName();
+    cfName = that.getColumnFamilyName();
     schemaMetrics = that.getSchemaMetrics();
   }
 
@@ -210,8 +208,8 @@ public void passSchemaMetricsTo(SchemaConfigured target) {
           target.schemaConfAsJSON());
     }
 
-    target.tableName = tableName.intern();
-    target.cfName = cfName.intern();
+    target.tableName = tableName;
+    target.cfName = cfName;
     target.schemaMetrics = schemaMetrics;
     target.schemaConfigurationChanged();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 62d3f8e6db9c..844560f9c24b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -295,7 +295,7 @@ private SchemaMetrics(final String tableName, final String cfName) {
           sb.append(metricType);
 
           int i = getBlockMetricIndex(blockCategory, isCompaction, metricType);
-          blockMetricNames[i] = sb.toString().intern();
+          blockMetricNames[i] = sb.toString();
           blockMetricTimeVarying[i] = metricType.timeVarying;
         }
       }

From 65c9eba3448fd7b1be7e8767383eb23ef302ff7c Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Sun, 15 Dec 2013 03:03:14 +0000
Subject: [PATCH 1266/1540] HBASE-10048 Add hlog number metric in regionserver

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1550997 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     |  2 ++
 .../metrics/RegionServerMetrics.java          | 18 +++++++++++
 .../hadoop/hbase/regionserver/wal/HLog.java   | 32 +++++++++++++++----
 .../apache/hadoop/hbase/client/TestAdmin.java |  2 +-
 .../regionserver/wal/HLogUtilsForTests.java   |  3 --
 .../hbase/regionserver/wal/TestHLog.java      | 10 +++---
 .../regionserver/wal/TestLogRolling.java      |  2 +-
 7 files changed, 52 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9efbbf6c9753..39b158c25a06 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1593,6 +1593,8 @@ protected void metrics() {
 
     this.metrics.stores.set(stores);
     this.metrics.storefiles.set(storefiles);
+    this.metrics.hlogFileCount.set(this.hlog.getNumLogFiles());
+    this.metrics.hlogFileSizeMB.set(this.hlog.getNumLogFileSize() /(1024 * 1024));
     this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
     this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024)));
     this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index 9557c54fe1f7..1a4d978d55c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -146,6 +146,18 @@ public class RegionServerMetrics implements Updater {
   public final MetricsIntValue storefiles = 
       new MetricsIntValue("storefiles", registry);
 
+  /**
+   * Count of hlogfiles
+   */
+  public final MetricsIntValue hlogFileCount = 
+      new MetricsIntValue("hlogFileCount", registry);
+  
+  /**
+   * the total size of hlog files in MB
+   */
+  public final MetricsLongValue hlogFileSizeMB = 
+      new MetricsLongValue("hlogFileSizeMB", registry);
+  
   /**
    * Count of read requests
    */
@@ -363,6 +375,8 @@ public void doUpdates(MetricsContext caller) {
 
       this.stores.pushMetric(this.metricsRecord);
       this.storefiles.pushMetric(this.metricsRecord);
+      this.hlogFileCount.pushMetric(this.metricsRecord);
+      this.hlogFileSizeMB.pushMetric(this.metricsRecord);
       this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
       this.rootIndexSizeKB.pushMetric(this.metricsRecord);
       this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
@@ -525,6 +539,10 @@ public String toString() {
       Integer.valueOf(this.regions.get()));
     sb = Strings.appendKeyValue(sb, "numberOfStores",
       Integer.valueOf(this.stores.get()));
+    sb = Strings.appendKeyValue(sb, this.hlogFileCount.getName(),
+      Integer.valueOf(this.hlogFileCount.get()));
+    sb = Strings.appendKeyValue(sb, this.hlogFileSizeMB.getName(),
+      Long.valueOf(this.hlogFileSizeMB.get()));
     sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
       Integer.valueOf(this.storefiles.get()));
     sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(),
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index b711677a6265..980e1e56b2ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -229,6 +229,12 @@ public interface Writer {
   // of the default Hdfs block size.
   private final long logrollsize;
 
+  // size of current log 
+  private long curLogSize = 0;
+
+  // The total size of hlog
+  private AtomicLong totalLogSize = new AtomicLong(0);
+  
   // This lock prevents starting a log roll during a cache flush.
   // synchronized is insufficient because a cache flush spans two method calls.
   private final Lock cacheFlushLock = new ReentrantLock();
@@ -640,11 +646,15 @@ OutputStream getOutputStream() {
         this.writer = nextWriter;
         this.hdfs_out = nextHdfsOut;
 
+        long oldFileLen = 0;
+        if (oldFile != null) {
+          oldFileLen = this.fs.getFileStatus(oldFile).getLen();
+          this.totalLogSize.addAndGet(oldFileLen);
+        }
         LOG.info((oldFile != null?
             "Roll " + FSUtils.getPath(oldFile) + ", entries=" +
             this.numEntries.get() +
-            ", filesize=" +
-            this.fs.getFileStatus(oldFile).getLen() + ". ": "") +
+            ", filesize=" + oldFileLen + ". ": "") +
           " for " + FSUtils.getPath(newPath));
         this.numEntries.set(0);
       }
@@ -663,7 +673,9 @@ OutputStream getOutputStream() {
           // flushed (and removed from the lastSeqWritten map). Means can
           // remove all but currently open log file.
           for (Map.Entry<Long, Path> e : this.outputfiles.entrySet()) {
-            archiveLogFile(e.getValue(), e.getKey());
+            Path path = e.getValue();
+            this.totalLogSize.addAndGet(-this.fs.getFileStatus(path).getLen());
+            archiveLogFile(path, e.getKey());
           }
           this.outputfiles.clear();
         } else {
@@ -1360,7 +1372,8 @@ private void syncer(long txid) throws IOException {
       if (!this.logRollRunning) {
         checkLowReplication();
         try {
-          if (tempWriter.getLength() > this.logrollsize) {
+          curLogSize = tempWriter.getLength();
+          if (curLogSize > this.logrollsize) {
             requestLogRoll();
           }
         } catch (IOException x) {
@@ -1525,9 +1538,14 @@ public long obtainSeqNum() {
     return this.logSeqNum.incrementAndGet();
   }
 
-  /** @return the number of log files in use */
-  int getNumLogFiles() {
-    return outputfiles.size();
+  /** @return the number of log files in use, including current one */
+  public int getNumLogFiles() {
+    return outputfiles.size() + 1;
+  }
+  
+  /** @return the total size of log files in use, including current one */
+  public long getNumLogFileSize() {
+    return totalLogSize.get() + curLogSize;
   }
 
   private byte[] getSnapshotName(byte[] encodedRegionName) {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 71dd59f532e3..7f72c0d262d8 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -1503,7 +1503,7 @@ public void testHLogRollWriting() throws Exception {
     int count = HLogUtilsForTests.getNumLogFiles(regionServer.getWAL());
     LOG.info("after flushing all regions and rolling logs there are " +
         count + " log files");
-    assertTrue(("actual count: " + count), count <= 2);
+    assertTrue(("actual count: " + count), count <= 3);
   }
 
   private void setUpforLogRolling() {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
index 33a6b6b5d16b..0ec54f32b11f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
@@ -19,9 +19,6 @@
  */
 package org.apache.hadoop.hbase.regionserver.wal;
 
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-
 /**
  * An Utility testcase that returns the number of log files that
  * were rolled to be accessed from outside packages.
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 6077078986ce..96ac8a602f54 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -697,12 +697,12 @@ public void testLogCleaning() throws Exception {
       // Before HBASE-3198 it used to delete it
       addEdits(log, hri, tableName, 1);
       log.rollWriter();
-      assertEquals(1, log.getNumLogFiles());
+      assertEquals(2, log.getNumLogFiles());
 
       // See if there's anything wrong with more than 1 edit
       addEdits(log, hri, tableName, 2);
       log.rollWriter();
-      assertEquals(2, log.getNumLogFiles());
+      assertEquals(3, log.getNumLogFiles());
 
       // Now mix edits from 2 regions, still no flushing
       addEdits(log, hri, tableName, 1);
@@ -710,14 +710,14 @@ public void testLogCleaning() throws Exception {
       addEdits(log, hri, tableName, 1);
       addEdits(log, hri2, tableName2, 1);
       log.rollWriter();
-      assertEquals(3, log.getNumLogFiles());
+      assertEquals(4, log.getNumLogFiles());
 
       // Flush the first region, we expect to see the first two files getting
       // archived
       long seqId = log.startCacheFlush(hri.getEncodedNameAsBytes());
       log.completeCacheFlush(hri.getEncodedNameAsBytes(), tableName, seqId, false);
       log.rollWriter();
-      assertEquals(2, log.getNumLogFiles());
+      assertEquals(3, log.getNumLogFiles());
 
       // Flush the second region, which removes all the remaining output files
       // since the oldest was completely flushed and the two others only contain
@@ -725,7 +725,7 @@ public void testLogCleaning() throws Exception {
       seqId = log.startCacheFlush(hri2.getEncodedNameAsBytes());
       log.completeCacheFlush(hri2.getEncodedNameAsBytes(), tableName2, seqId, false);
       log.rollWriter();
-      assertEquals(0, log.getNumLogFiles());
+      assertEquals(1, log.getNumLogFiles());
     } finally {
       if (log != null) log.closeAndDelete();
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
index a42c801e30bc..ce57070ead4d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
@@ -209,7 +209,7 @@ public void testLogRolling() throws FailedLogCloseException, IOException {
       int count = log.getNumLogFiles();
       LOG.info("after flushing all regions and rolling logs there are " +
           log.getNumLogFiles() + " log files");
-      assertTrue(("actual count: " + count), count <= 2);
+      assertTrue(("actual count: " + count), count <= 3);
   }
 
   private static String getName() {

From f2677748a3f293c6ef74adb1922934de278adee4 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 16 Dec 2013 17:20:47 +0000
Subject: [PATCH 1267/1540] HBASE-9927 ReplicationLogCleaner#stop() calls
 HConnectionManager#deleteConnection() unnecessarily

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551273 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/replication/master/ReplicationLogCleaner.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 203407f96296..edc8bc9a9500 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -146,8 +146,6 @@ public void stop(String why) {
       LOG.info("Stopping " + this.zkHelper.getZookeeperWatcher());
       this.zkHelper.getZookeeperWatcher().close();
     }
-    // Not sure why we're deleting a connection that we never acquired or used
-    HConnectionManager.deleteConnection(this.getConf());
   }
 
   @Override

From f89534971b15fdc4348a951e4a0716df3118e4fa Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 17 Dec 2013 05:41:55 +0000
Subject: [PATCH 1268/1540] HBASE-9047 Tool to handle finishing replication
 when the cluster is offline (Demai Ni)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551461 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSource.java       |  20 +
 .../ReplicationSourceManager.java             |   8 +
 .../regionserver/ReplicationSyncUp.java       | 174 +++++++++
 .../TestReplicationSyncUpTool.java            | 361 ++++++++++++++++++
 4 files changed, 563 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 8cceadb69f5e..a46fe5e8607a 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -589,6 +589,26 @@ protected boolean openReader(int sleepMultiplier) {
               }
             }
           }
+          // In the case of disaster/recovery, HMaster may be shutdown/crashed before flush data
+          // from .logs to .oldlogs. Loop into .logs folders and check whether a match exists
+          if (stopper instanceof ReplicationSyncUp.DummyServer) {
+            FileStatus[] rss = fs.listStatus(manager.getLogDir());
+            for (FileStatus rs : rss) {
+              Path p = rs.getPath();
+              FileStatus[] logs = fs.listStatus(p);
+              for (FileStatus log : logs) {
+                p = new Path(p, log.getPath().getName());
+                if (p.getName().equals(currentPath.getName())) {
+                  currentPath = p;
+                  LOG.info("Log " + this.currentPath + " exists under " + manager.getLogDir());
+                  // Open the log at the new location
+                  this.openReader(sleepMultiplier);
+                  return true;
+                }
+              }
+            }
+          }
+
           // TODO What happens if the log was missing from every single location?
           // Although we need to check a couple of times as the log could have
           // been moved by the master between the checks
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index c0ee8b14eb7d..185597eed244 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -274,6 +274,14 @@ public List<ReplicationSourceInterface> getSources() {
     return this.sources;
   }
 
+  /**
+   * Get a list of all the old sources of this rs
+   * @return list of all old sources
+   */
+  public List<ReplicationSourceInterface> getOldSources() {
+    return this.oldsources;
+  }
+
   void preLogRoll(Path newLog) throws IOException {
     if (!this.replicating.get()) {
       LOG.warn("Replication stopped, won't add new log");
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
new file mode 100644
index 000000000000..3cfc1c28fe1b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * In a scenario of Replication based Disaster/Recovery, when hbase
+ * Master-Cluster crashes, this tool is used to sync-up the delta from Master to
+ * Slave using the info from Zookeeper. The tool will run on Master-Cluser, and
+ * assume ZK, Filesystem and NetWork still available after hbase crashes
+ *
+ * hbase org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp
+ */
+
+public class ReplicationSyncUp extends Configured implements Tool {
+
+  static final Log LOG = LogFactory.getLog(ReplicationSyncUp.class.getName());
+
+  private static Configuration conf;
+
+  private static final long SLEEP_TIME = 10000;
+
+  // although the tool is designed to be run on command line
+  // this api is provided for executing the tool through another app
+  public static void setConfigure(Configuration config) {
+    conf = config;
+  }
+
+  /**
+   * Main program
+   * @param args
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    if (conf == null) conf = HBaseConfiguration.create();
+    int ret = ToolRunner.run(conf, new ReplicationSyncUp(), args);
+    System.exit(ret);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    Replication replication;
+    ReplicationSourceManager manager;
+    FileSystem fs;
+    Path oldLogDir, logDir, rootDir;
+    ZooKeeperWatcher zkw;
+
+    Abortable abortable = new Abortable() {
+      @Override
+      public void abort(String why, Throwable e) {
+      }
+
+      @Override
+      public boolean isAborted() {
+        return false;
+      }
+    };
+
+    zkw =
+        new ZooKeeperWatcher(conf, "syncupReplication" + System.currentTimeMillis(), abortable,
+            true);
+
+    rootDir = FSUtils.getRootDir(conf);
+    fs = FileSystem.get(conf);
+    oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+
+    System.out.println("Start Replication Server start");
+    replication = new Replication(new DummyServer(zkw), fs, logDir, oldLogDir);
+    manager = replication.getReplicationManager();
+    manager.init();
+
+    try {
+      int numberOfOldSource = 1; // default wait once
+      while (numberOfOldSource > 0) {
+        Thread.sleep(SLEEP_TIME);
+        numberOfOldSource = manager.getOldSources().size();
+      }
+    } catch (InterruptedException e) {
+      System.err.println("didn't wait long enough:" + e);
+      return (-1);
+    }
+
+    manager.join();
+
+    return (0);
+  }
+
+  static class DummyServer implements Server {
+    String hostname;
+    ZooKeeperWatcher zkw;
+
+    DummyServer(ZooKeeperWatcher zkw) {
+      // an unique name in case the first run fails
+      hostname = System.currentTimeMillis() + ".SyncUpTool.replication.org";
+      this.zkw = zkw;
+    }
+
+    DummyServer(String hostname) {
+      this.hostname = hostname;
+    }
+
+    @Override
+    public Configuration getConfiguration() {
+      return conf;
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      return zkw;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+
+    @Override
+    public ServerName getServerName() {
+      return new ServerName(hostname, 1234, 1L);
+    }
+
+    @Override
+    public void abort(String why, Throwable e) {
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {
+    }
+
+    @Override
+    public boolean isStopped() {
+      return false;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
new file mode 100644
index 000000000000..a505452c1235
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestReplicationSyncUpTool extends TestReplicationBase {
+
+  private static final Log LOG = LogFactory.getLog(TestReplicationSyncUpTool.class);
+
+  private static final byte[] t1_su = Bytes.toBytes("t1_syncup");
+  private static final byte[] t2_su = Bytes.toBytes("t2_syncup");
+
+  private static final byte[] famName = Bytes.toBytes("cf1");
+  private static final byte[] qualName = Bytes.toBytes("q1");
+
+  private static final byte[] noRepfamName = Bytes.toBytes("norep");
+
+  private HTableDescriptor t1_syncupSource, t1_syncupTarget;
+  private HTableDescriptor t2_syncupSource, t2_syncupTarget;
+
+  private HTable ht1Source, ht2Source, ht1TargetAtPeer1, ht2TargetAtPeer1;
+  private int rowCount_ht1Source, rowCount_ht2Source, rowCount_ht1TargetAtPeer1,
+      rowCount_ht2TargetAtPeer1;
+
+  @Before
+  public void setUp() throws Exception {
+
+    HColumnDescriptor fam;
+
+    t1_syncupSource = new HTableDescriptor(t1_su);
+    fam = new HColumnDescriptor(famName);
+    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+    t1_syncupSource.addFamily(fam);
+    fam = new HColumnDescriptor(noRepfamName);
+    t1_syncupSource.addFamily(fam);
+
+    t1_syncupTarget = new HTableDescriptor(t1_su);
+    fam = new HColumnDescriptor(famName);
+    t1_syncupTarget.addFamily(fam);
+    fam = new HColumnDescriptor(noRepfamName);
+    t1_syncupTarget.addFamily(fam);
+
+    t2_syncupSource = new HTableDescriptor(t2_su);
+    fam = new HColumnDescriptor(famName);
+    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+    t2_syncupSource.addFamily(fam);
+    fam = new HColumnDescriptor(noRepfamName);
+    t2_syncupSource.addFamily(fam);
+
+    t2_syncupTarget = new HTableDescriptor(t2_su);
+    fam = new HColumnDescriptor(famName);
+    t2_syncupTarget.addFamily(fam);
+    fam = new HColumnDescriptor(noRepfamName);
+    t2_syncupTarget.addFamily(fam);
+
+  }
+
+  /**
+   * Add a row to a table in each cluster, check it's replicated, delete it,
+   * check's gone Also check the puts and deletes are not replicated back to
+   * the originating cluster.
+   */
+  @Test(timeout = 300000)
+  public void testSyncUpTool() throws Exception {
+
+    /**
+     * Set up Replication: on Master and one Slave
+     * Table: t1_syncup and t2_syncup
+     * columnfamily:
+     *    'cf1'  : replicated
+     *    'norep': not replicated
+     */
+    setupReplication();
+
+    /**
+     * at Master:
+     * t1_syncup: put 100 rows into cf1, and 1 rows into norep
+     * t2_syncup: put 200 rows into cf1, and 1 rows into norep
+     *
+     * verify correctly replicated to slave
+     */
+    putAndReplicateRows();
+
+    /**
+     * Verify delete works
+     *
+     * step 1: stop hbase on Slave
+     *
+     * step 2: at Master:
+     *  t1_syncup: delete 50 rows  from cf1
+     *  t2_syncup: delete 100 rows from cf1
+     *  no change on 'norep'
+     *
+     * step 3: stop hbase on master, restart hbase on Slave
+     *
+     * step 4: verify Slave still have the rows before delete
+     *      t1_syncup: 100 rows from cf1
+     *      t2_syncup: 200 rows from cf1
+     *
+     * step 5: run syncup tool on Master
+     *
+     * step 6: verify that delete show up on Slave
+     *      t1_syncup: 50 rows from cf1
+     *      t2_syncup: 100 rows from cf1
+     *
+     * verify correctly replicated to Slave
+     */
+    mimicSyncUpAfterDelete();
+
+    /**
+     * Verify put works
+     *
+     * step 1: stop hbase on Slave
+     *
+     * step 2: at Master:
+     *  t1_syncup: put 100 rows  from cf1
+     *  t2_syncup: put 200 rows  from cf1
+     *  and put another row on 'norep'
+     *  ATTN: put to 'cf1' will overwrite existing rows, so end count will
+     *        be 100 and 200 respectively
+     *      put to 'norep' will add a new row.
+     *
+     * step 3: stop hbase on master, restart hbase on Slave
+     *
+     * step 4: verify Slave still has the rows before put
+     *      t1_syncup: 50 rows from cf1
+     *      t2_syncup: 100 rows from cf1
+     *
+     * step 5: run syncup tool on Master
+     *
+     * step 6: verify that put show up on Slave
+     *         and 'norep' does not
+     *      t1_syncup: 100 rows from cf1
+     *      t2_syncup: 200 rows from cf1
+     *
+     * verify correctly replicated to Slave
+     */
+    mimicSyncUpAfterPut();
+
+  }
+
+  private void setupReplication() throws Exception {
+    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
+    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
+
+    HBaseAdmin ha = new HBaseAdmin(conf1);
+    ha.createTable(t1_syncupSource);
+    ha.createTable(t2_syncupSource);
+    ha.close();
+
+    ha = new HBaseAdmin(conf2);
+    ha.createTable(t1_syncupTarget);
+    ha.createTable(t2_syncupTarget);
+    ha.close();
+
+    // Get HTable from Master
+    ht1Source = new HTable(conf1, t1_su);
+    ht1Source.setWriteBufferSize(1024);
+    ht2Source = new HTable(conf1, t2_su);
+    ht1Source.setWriteBufferSize(1024);
+
+    // Get HTable from Peer1
+    ht1TargetAtPeer1 = new HTable(conf2, t1_su);
+    ht1TargetAtPeer1.setWriteBufferSize(1024);
+    ht2TargetAtPeer1 = new HTable(conf2, t2_su);
+    ht2TargetAtPeer1.setWriteBufferSize(1024);
+
+    /**
+     * set M-S : Master: utility1 Slave1: utility2
+     */
+    admin1.addPeer("1", utility2.getClusterKey());
+
+    admin1.close();
+    admin2.close();
+  }
+
+  private void putAndReplicateRows() throws Exception {
+    // add rows to Master cluster,
+    Put p;
+
+    // 100 + 1 row to t1_syncup
+    for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+      p = new Put(Bytes.toBytes("row" + i));
+      p.add(famName, qualName, Bytes.toBytes("val" + i));
+      ht1Source.put(p);
+    }
+    p = new Put(Bytes.toBytes("row" + 9999));
+    p.add(noRepfamName, qualName, Bytes.toBytes("val" + 9999));
+    ht1Source.put(p);
+
+    // 200 + 1 row to t2_syncup
+    for (int i = 0; i < NB_ROWS_IN_BATCH * 2; i++) {
+      p = new Put(Bytes.toBytes("row" + i));
+      p.add(famName, qualName, Bytes.toBytes("val" + i));
+      ht2Source.put(p);
+    }
+    p = new Put(Bytes.toBytes("row" + 9999));
+    p.add(noRepfamName, qualName, Bytes.toBytes("val" + 9999));
+    ht2Source.put(p);
+
+    // ensure replication completed
+    Thread.sleep(SLEEP_TIME);
+
+    rowCount_ht1Source = utility1.countRows(ht1Source);
+    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    assertEquals("t1_syncup has 101 rows on source, and 100 on slave1", rowCount_ht1Source - 1,
+      rowCount_ht1TargetAtPeer1);
+
+    rowCount_ht2Source = utility1.countRows(ht2Source);
+    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    assertEquals("t2_syncup has 201 rows on source, and 200 on slave1", rowCount_ht2Source - 1,
+      rowCount_ht2TargetAtPeer1);
+
+  }
+
+  private void mimicSyncUpAfterDelete() throws Exception {
+    utility2.shutdownMiniHBaseCluster();
+
+    List<Delete> list = new ArrayList<Delete>();
+    // delete half of the rows
+    for (int i = 0; i < NB_ROWS_IN_BATCH / 2; i++) {
+      String rowKey = "row" + i;
+      Delete del = new Delete(rowKey.getBytes());
+      list.add(del);
+    }
+    ht1Source.delete(list);
+
+    for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+      String rowKey = "row" + i;
+      Delete del = new Delete(rowKey.getBytes());
+      list.add(del);
+    }
+    ht2Source.delete(list);
+
+    rowCount_ht1Source = utility1.countRows(ht1Source);
+    assertEquals("t1_syncup has 51 rows on source, after remove 50 of the replicated colfam", 51,
+      rowCount_ht1Source);
+
+    rowCount_ht2Source = utility1.countRows(ht2Source);
+    assertEquals("t2_syncup has 101 rows on source, after remove 100 of the replicated colfam",
+      101, rowCount_ht2Source);
+
+    utility1.shutdownMiniHBaseCluster();
+    utility2.restartHBaseCluster(1);
+
+    Thread.sleep(SLEEP_TIME);
+
+    // before sync up
+    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    assertEquals("@Peer1 t1_syncup should still have 100 rows", 100, rowCount_ht1TargetAtPeer1);
+    assertEquals("@Peer1 t2_syncup should still have 200 rows", 200, rowCount_ht2TargetAtPeer1);
+
+    // After sync up
+    syncUp(utility1);
+    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    assertEquals("@Peer1 t1_syncup should be sync up and have 50 rows", 50,
+      rowCount_ht1TargetAtPeer1);
+    assertEquals("@Peer1 t2_syncup should be sync up and have 100 rows", 100,
+      rowCount_ht2TargetAtPeer1);
+
+  }
+
+  private void mimicSyncUpAfterPut() throws Exception {
+    utility1.restartHBaseCluster(1);
+    utility2.shutdownMiniHBaseCluster();
+
+    Put p;
+    // another 100 + 1 row to t1_syncup
+    // we should see 100 + 2 rows now
+    for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+      p = new Put(Bytes.toBytes("row" + i));
+      p.add(famName, qualName, Bytes.toBytes("val" + i));
+      ht1Source.put(p);
+    }
+    p = new Put(Bytes.toBytes("row" + 9998));
+    p.add(noRepfamName, qualName, Bytes.toBytes("val" + 9998));
+    ht1Source.put(p);
+
+    // another 200 + 1 row to t1_syncup
+    // we should see 200 + 2 rows now
+    for (int i = 0; i < NB_ROWS_IN_BATCH * 2; i++) {
+      p = new Put(Bytes.toBytes("row" + i));
+      p.add(famName, qualName, Bytes.toBytes("val" + i));
+      ht2Source.put(p);
+    }
+    p = new Put(Bytes.toBytes("row" + 9998));
+    p.add(noRepfamName, qualName, Bytes.toBytes("val" + 9998));
+    ht2Source.put(p);
+
+    rowCount_ht1Source = utility1.countRows(ht1Source);
+    assertEquals("t1_syncup has 102 rows on source", 102, rowCount_ht1Source);
+    rowCount_ht2Source = utility1.countRows(ht2Source);
+    assertEquals("t2_syncup has 202 rows on source", 202, rowCount_ht2Source);
+
+    utility1.shutdownMiniHBaseCluster();
+    utility2.restartHBaseCluster(1);
+
+    Thread.sleep(SLEEP_TIME);
+
+    // before sync up
+    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    assertEquals("@Peer1 t1_syncup should be NOT sync up and have 50 rows", 50,
+      rowCount_ht1TargetAtPeer1);
+    assertEquals("@Peer1 t2_syncup should be NOT sync up and have 100 rows", 100,
+      rowCount_ht2TargetAtPeer1);
+
+    // after syun up
+    syncUp(utility1);
+    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    assertEquals("@Peer1 t1_syncup should be sync up and have 100 rows", 100,
+      rowCount_ht1TargetAtPeer1);
+    assertEquals("@Peer1 t2_syncup should be sync up and have 200 rows", 200,
+      rowCount_ht2TargetAtPeer1);
+
+  }
+
+  private void syncUp(HBaseTestingUtility ut) throws Exception {
+    ReplicationSyncUp.setConfigure(ut.getConfiguration());
+    String[] arguments = new String[] { null };
+    new ReplicationSyncUp().run(arguments);
+  }
+
+}

From 930f22fc94f7ffbf8987635f05967e43981df54d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 17 Dec 2013 05:55:27 +0000
Subject: [PATCH 1269/1540] HBASE-7886 [replication] hlog zk node will not be
 deleted if client roll hlog (JD)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551466 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSource.java  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index a46fe5e8607a..3831bba54c09 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -347,8 +347,6 @@ public void run() {
           sleepMultiplier++;
         }
         continue;
-      } else if (oldPath != null && !oldPath.getName().equals(getCurrentPath().getName())) {
-        this.manager.cleanOldLogs(getCurrentPath().getName(), this.peerId, this.queueRecovered);
       }
       boolean currentWALisBeingWrittenTo = false;
       //For WAL files we own (rather than recovered), take a snapshot of whether the
@@ -545,6 +543,11 @@ protected boolean getNextPath() {
       if (this.currentPath == null) {
         this.currentPath = queue.poll(this.sleepForRetries, TimeUnit.MILLISECONDS);
         this.metrics.sizeOfLogQueue.set(queue.size());
+        if (this.currentPath != null) {
+          this.manager.cleanOldLogs(this.currentPath.getName(),
+              this.peerId,
+              this.queueRecovered);
+        }
       }
     } catch (InterruptedException e) {
       LOG.warn("Interrupted while reading edits", e);

From 04513a7d2da933c1de635d429fbc18583011c4cc Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 17 Dec 2013 19:10:04 +0000
Subject: [PATCH 1270/1540] HBASE-10181 HBaseObjectWritable.readObject catches
 DoNotRetryIOException and wraps it back in a regular IOException

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551657 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/io/HbaseObjectWritable.java    | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
index 317601e36313..430981b73539 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -668,7 +669,7 @@ public static Object readObject(DataInput in,
         instance = tryInstantiateProtobuf(declaredClass, in);
       } catch (ClassNotFoundException e) {
         LOG.error("Can't find class " + className, e);
-        throw new IOException("Can't find class " + className, e);
+        throw new DoNotRetryIOException("Can't find class " + className, e);
       }
     } else {                                      // Writable or Serializable
       Class instanceClass = null;
@@ -679,7 +680,7 @@ public static Object readObject(DataInput in,
           instanceClass = getClassByName(conf, className);
         } catch (ClassNotFoundException e) {
           LOG.error("Can't find class " + className, e);
-          throw new IOException("Can't find class " + className, e);
+          throw new DoNotRetryIOException("Can't find class " + className, e);
         }
       } else {
         instanceClass = CODE_TO_CLASS.get(b);
@@ -688,6 +689,9 @@ public static Object readObject(DataInput in,
         Writable writable = WritableFactories.newInstance(instanceClass, conf);
         try {
           writable.readFields(in);
+        } catch (IOException io) {
+          LOG.error("Error in readFields", io);
+          throw io;
         } catch (Exception e) {
           LOG.error("Error in readFields", e);
           throw new IOException("Error in readFields" , e);
@@ -709,7 +713,7 @@ public static Object readObject(DataInput in,
           instance = ois.readObject();
         } catch (ClassNotFoundException e) {
           LOG.error("Class not found when attempting to deserialize object", e);
-          throw new IOException("Class not found when attempting to " +
+          throw new DoNotRetryIOException("Class not found when attempting to " +
               "deserialize object", e);
         } finally {
           if(bis!=null) bis.close();

From a6614d91381fe26ef6e2e6ee19418aca8625f409 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Dec 2013 00:54:41 +0000
Subject: [PATCH 1271/1540] HBASE-10189 Intermittent TestReplicationSyncUpTool
 failure (LarsH and Demai Ni)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551785 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestReplicationSyncUpTool.java            | 56 +++++++++++--------
 1 file changed, 32 insertions(+), 24 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index a505452c1235..7d553ac70e16 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -22,8 +22,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -39,8 +37,6 @@
 @Category(LargeTests.class)
 public class TestReplicationSyncUpTool extends TestReplicationBase {
 
-  private static final Log LOG = LogFactory.getLog(TestReplicationSyncUpTool.class);
-
   private static final byte[] t1_su = Bytes.toBytes("t1_syncup");
   private static final byte[] t2_su = Bytes.toBytes("t2_syncup");
 
@@ -53,8 +49,6 @@ public class TestReplicationSyncUpTool extends TestReplicationBase {
   private HTableDescriptor t2_syncupSource, t2_syncupTarget;
 
   private HTable ht1Source, ht2Source, ht1TargetAtPeer1, ht2TargetAtPeer1;
-  private int rowCount_ht1Source, rowCount_ht2Source, rowCount_ht1TargetAtPeer1,
-      rowCount_ht2TargetAtPeer1;
 
   @Before
   public void setUp() throws Exception {
@@ -234,17 +228,31 @@ private void putAndReplicateRows() throws Exception {
 
     // ensure replication completed
     Thread.sleep(SLEEP_TIME);
+    int rowCount_ht1Source = utility1.countRows(ht1Source);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      int rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+      if (i==NB_RETRIES-1) {
+        assertEquals("t1_syncup has 101 rows on source, and 100 on slave1", rowCount_ht1Source - 1,
+            rowCount_ht1TargetAtPeer1);
+      }
+      if (rowCount_ht1Source - 1 == rowCount_ht1TargetAtPeer1) {
+        break;
+      }
+      Thread.sleep(SLEEP_TIME);
+    }
 
-    rowCount_ht1Source = utility1.countRows(ht1Source);
-    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
-    assertEquals("t1_syncup has 101 rows on source, and 100 on slave1", rowCount_ht1Source - 1,
-      rowCount_ht1TargetAtPeer1);
-
-    rowCount_ht2Source = utility1.countRows(ht2Source);
-    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
-    assertEquals("t2_syncup has 201 rows on source, and 200 on slave1", rowCount_ht2Source - 1,
-      rowCount_ht2TargetAtPeer1);
-
+    int rowCount_ht2Source = utility1.countRows(ht2Source);
+    for (int i = 0; i < NB_RETRIES; i++) {
+      int rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+      if (i==NB_RETRIES-1) {
+        assertEquals("t2_syncup has 201 rows on source, and 200 on slave1", rowCount_ht2Source - 1,
+            rowCount_ht2TargetAtPeer1);
+      }
+      if (rowCount_ht2Source - 1 == rowCount_ht2TargetAtPeer1) {
+        break;
+      }
+      Thread.sleep(SLEEP_TIME);
+    }
   }
 
   private void mimicSyncUpAfterDelete() throws Exception {
@@ -266,11 +274,11 @@ private void mimicSyncUpAfterDelete() throws Exception {
     }
     ht2Source.delete(list);
 
-    rowCount_ht1Source = utility1.countRows(ht1Source);
+    int rowCount_ht1Source = utility1.countRows(ht1Source);
     assertEquals("t1_syncup has 51 rows on source, after remove 50 of the replicated colfam", 51,
       rowCount_ht1Source);
 
-    rowCount_ht2Source = utility1.countRows(ht2Source);
+    int rowCount_ht2Source = utility1.countRows(ht2Source);
     assertEquals("t2_syncup has 101 rows on source, after remove 100 of the replicated colfam",
       101, rowCount_ht2Source);
 
@@ -280,8 +288,8 @@ private void mimicSyncUpAfterDelete() throws Exception {
     Thread.sleep(SLEEP_TIME);
 
     // before sync up
-    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
-    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    int rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    int rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
     assertEquals("@Peer1 t1_syncup should still have 100 rows", 100, rowCount_ht1TargetAtPeer1);
     assertEquals("@Peer1 t2_syncup should still have 200 rows", 200, rowCount_ht2TargetAtPeer1);
 
@@ -323,9 +331,9 @@ private void mimicSyncUpAfterPut() throws Exception {
     p.add(noRepfamName, qualName, Bytes.toBytes("val" + 9998));
     ht2Source.put(p);
 
-    rowCount_ht1Source = utility1.countRows(ht1Source);
+    int rowCount_ht1Source = utility1.countRows(ht1Source);
     assertEquals("t1_syncup has 102 rows on source", 102, rowCount_ht1Source);
-    rowCount_ht2Source = utility1.countRows(ht2Source);
+    int rowCount_ht2Source = utility1.countRows(ht2Source);
     assertEquals("t2_syncup has 202 rows on source", 202, rowCount_ht2Source);
 
     utility1.shutdownMiniHBaseCluster();
@@ -334,8 +342,8 @@ private void mimicSyncUpAfterPut() throws Exception {
     Thread.sleep(SLEEP_TIME);
 
     // before sync up
-    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
-    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+    int rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+    int rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
     assertEquals("@Peer1 t1_syncup should be NOT sync up and have 50 rows", 50,
       rowCount_ht1TargetAtPeer1);
     assertEquals("@Peer1 t2_syncup should be NOT sync up and have 100 rows", 100,

From 980db8b6685e4f27a7c3ab5c4422ea5ac8fc10a9 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 18 Dec 2013 01:43:29 +0000
Subject: [PATCH 1272/1540] HBASE-10179. HRegionServer underreports
 readRequestCounts by 1 under certain conditions (Perry Trolard)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551800 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 39b158c25a06..a4737701c4b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2670,7 +2670,7 @@ private Result[] internalNext(final RegionScanner s, int nbRows,
         int i = 0;
         synchronized(s) {
           for (; i < nbRows
-              && currentScanResultSize < maxScannerResultSize; i++) {
+              && currentScanResultSize < maxScannerResultSize; ) {
             // Collect values to be returned here
             boolean moreRows = s.nextRaw(values, SchemaMetrics.METRIC_NEXTSIZE);
             if (!values.isEmpty()) {
@@ -2680,6 +2680,7 @@ private Result[] internalNext(final RegionScanner s, int nbRows,
                 }
               }
               results.add(new Result(values));
+              i++;
             }
             if (!moreRows) {
               break;

From f0899192d85387e8ee93f0177641c89b3b2e907d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 18 Dec 2013 03:52:12 +0000
Subject: [PATCH 1273/1540] CHANGES.txt and pom.xml for 0.94.15RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1551828 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 20403b035df5..5a8859e0e8af 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,52 @@
 HBase Change Log
 
+Release 0.94.15 - 12/17/2013
+Bug
+
+    [HBASE-7886] - [replication] hlog zk node will not be deleted if client roll hlog
+    [HBASE-9485] - TableOutputCommitter should implement recovery if we don't want jobs to start from 0 on RM restart
+    [HBASE-9995] - Not stopping ReplicationSink when using custom implementation for the ReplicationSink
+    [HBASE-10014] - HRegion#doMiniBatchMutation rollbacks the memstore even if there is nothing to rollback.
+    [HBASE-10015] - Replace intrinsic locking with explicit locks in StoreScanner
+    [HBASE-10026] - HBaseAdmin#createTable could fail if region splits too fast
+    [HBASE-10046] - Unmonitored HBase service could accumulate Status objects and OOM
+    [HBASE-10057] - TestRestoreFlushSnapshotFromClient and TestRestoreSnapshotFromClient fail to finish occasionally
+    [HBASE-10061] - TableMapReduceUtil.findOrCreateJar calls updateMap(null, ) resulting in thrown NPE
+    [HBASE-10064] - AggregateClient.validateParameters can throw NPE
+    [HBASE-10089] - Metrics intern table names cause eventual permgen OOM in 0.94
+    [HBASE-10111] - Verify that a snapshot is not corrupted before restoring it
+    [HBASE-10112] - Hbase rest query params for maxVersions and maxValues are not parsed
+    [HBASE-10117] - Avoid synchronization in HRegionScannerImpl.isFilterDone
+    [HBASE-10120] - start-hbase.sh doesn't respect --config in non-distributed mode
+    [HBASE-10179] - HRegionServer underreports readRequestCounts by 1 under certain conditions
+    [HBASE-10181] - HBaseObjectWritable.readObject catches DoNotRetryIOException and wraps it back in a regular IOException
+
+Improvement
+
+    [HBASE-9931] - Optional setBatch for CopyTable to copy large rows in batches
+    [HBASE-10001] - Add a coprocessor to help testing the performances without taking into account the i/o
+    [HBASE-10007] - PerformanceEvaluation: Add sampling and latency collection to randomRead test
+    [HBASE-10010] - eliminate the put latency spike on the new log file beginning
+    [HBASE-10048] - Add hlog number metric in regionserver
+    [HBASE-10049] - Small improvments in region_mover.rb
+    [HBASE-10093] - Unregister ReplicationSource metric bean when the replication source thread is terminated
+
+New Feature
+
+    [HBASE-9047] - Tool to handle finishing replication when the cluster is offline
+    [HBASE-10119] - Allow HBase coprocessors to clean up when they fail
+
+Task
+
+    [HBASE-9927] - ReplicationLogCleaner#stop() calls HConnectionManager#deleteConnection() unnecessarily
+    [HBASE-9986] - Incorporate HTTPS support for HBase (0.94 port)
+
+Test
+
+    [HBASE-10058] - Test for HBASE-9915 (avoid reading index blocks)
+    [HBASE-10189] - Intermittent TestReplicationSyncUpTool failure
+
+
 Release 0.94.14 - 11/18/2013
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index b6eb308f1266..dfa01934a2c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.15-SNAPSHOT</version>
+  <version>0.94.15</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 968d233fc1c294fe51937165907b3491d00b1f9e Mon Sep 17 00:00:00 2001
From: ramkrishna <ramkrishna@unknown>
Date: Fri, 20 Dec 2013 16:22:38 +0000
Subject: [PATCH 1274/1540] HBASE-10193 - Cleanup HRegion if one of the store
 fails to open at region initialization (Aditya Kishore

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1552718 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/HRegion.java   | 13 +++++++++++++
 .../org/apache/hadoop/hbase/regionserver/Store.java |  8 +++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f082b8ee08fb..7586f989f22d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -602,6 +602,7 @@ public Store call() throws IOException {
           }
         });
       }
+      boolean allStoresOpened = false;
       try {
         for (int i = 0; i < htableDescriptor.getFamilies().size(); i++) {
           Future<Store> future = completionService.take();
@@ -621,12 +622,24 @@ public Store call() throws IOException {
             maxMemstoreTS = maxStoreMemstoreTS;
           }
         }
+        allStoresOpened = true;
       } catch (InterruptedException e) {
         throw new IOException(e);
       } catch (ExecutionException e) {
         throw new IOException(e.getCause());
       } finally {
         storeOpenerThreadPool.shutdownNow();
+        if (!allStoresOpened) {
+          // something went wrong, close all opened stores
+          LOG.error("Could not initialize all stores for the region=" + this);
+          for (Store store : this.stores.values()) {
+            try {
+              store.close();
+            } catch (IOException e) { 
+              LOG.warn(e.getMessage());
+            }
+          }
+        }
       }
     }
     mvcc.initialize(maxMemstoreTS + 1);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 389a3556d214..a71c84d9e785 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -487,11 +487,13 @@ public StoreFile call() throws IOException {
     }
     if (ioe != null) {
       // close StoreFile readers
-      try {
-        for (StoreFile file : results) {
+      for (StoreFile file : results) {
+        try {
           if (file != null) file.closeReader(true);
+        } catch (IOException e) { 
+          LOG.warn(e.getMessage());
         }
-      } catch (IOException e) { }
+      }
       throw ioe;
     }
 

From f25ffaf13422a49c3d5398eaee01aec969a11a95 Mon Sep 17 00:00:00 2001
From: liangxie <liangxie@unknown>
Date: Sat, 21 Dec 2013 08:42:45 +0000
Subject: [PATCH 1275/1540] HBASE-8558 Add timeout limit for HBaseClient
 dataOutputStream

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1552888 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 2dce646ea8ef..8d5cff7c55ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -438,8 +438,8 @@ protected synchronized void setupIOstreams()
         setupConnection();
         this.in = new DataInputStream(new BufferedInputStream
             (new PingInputStream(NetUtils.getInputStream(socket))));
-        this.out = new DataOutputStream
-            (new BufferedOutputStream(NetUtils.getOutputStream(socket)));
+        this.out = new DataOutputStream(new BufferedOutputStream(
+            NetUtils.getOutputStream(socket, pingInterval)));
         writeHeader();
 
         // update last activity time

From 5890705c7bf0b0596ece7ca74f891198ae47036a Mon Sep 17 00:00:00 2001
From: anoopsamjohn <anoopsamjohn@unknown>
Date: Mon, 23 Dec 2013 06:46:41 +0000
Subject: [PATCH 1276/1540] HBASE-9346 HBCK should provide an option to check
 if regions boundaries are the same in META and in stores. (Jean Marc)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1553078 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 124 +++++++++++++++++-
 1 file changed, 123 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 5cfc95469dab..8ec65524ccc1 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -88,6 +88,7 @@
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
@@ -201,6 +202,7 @@ public class HBaseFsck extends Configured implements Tool {
   private boolean rerun = false; // if we tried to fix something, rerun hbck
   private static boolean summary = false; // if we want to print less output
   private boolean checkMetaOnly = false;
+  private boolean checkRegionBoundaries = false;
   private boolean ignorePreCheckPermission = false; // if pre-check permission
 
   /*********
@@ -451,6 +453,10 @@ public int onlineHbck() throws IOException, KeeperException, InterruptedExceptio
       admin.setBalancerRunning(oldBalancer, false);
     }
 
+    if (checkRegionBoundaries) {
+      checkRegionBoundaries();
+    }
+
     offlineReferenceFileRepair();
 
     // Print table summary
@@ -458,6 +464,112 @@ public int onlineHbck() throws IOException, KeeperException, InterruptedExceptio
     return errors.summarize();
   }
 
+    public static byte[] keyOnly (byte[] b) {
+        if (b == null)
+          return b;
+        int rowlength = Bytes.toShort(b, 0);
+        byte[] result = new byte[rowlength];
+        System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
+        return result;
+      }
+    
+      private static class RegionBoundariesInformation {
+        public byte [] regionName;
+        public byte [] metaFirstKey;
+        public byte [] metaLastKey;
+        public byte [] storesFirstKey;
+        public byte [] storesLastKey;
+        public String toString () {
+          return "regionName=" + Bytes.toStringBinary(regionName) +
+                 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
+                 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
+                 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
+                 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
+        }
+      }
+    
+      public void checkRegionBoundaries() {
+        try {
+          ByteArrayComparator comparator = new ByteArrayComparator();
+          List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
+          final RegionBoundariesInformation currentRegionBoundariesInformation =
+              new RegionBoundariesInformation();
+          for (HRegionInfo regionInfo : regions) {
+            currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
+            // For each region, get the start and stop key from the META and compare them to the
+            // same information from the Stores.
+            Path path = new Path(getConf().get(HConstants.HBASE_DIR) + "/"
+                + Bytes.toString(regionInfo.getTableName()) + "/"
+                + regionInfo.getEncodedName() + "/");
+            FileSystem fs = path.getFileSystem(getConf());
+            FileStatus[] files = fs.listStatus(path);
+            // For all the column families in this region...
+            byte[] storeFirstKey = null;
+            byte[] storeLastKey = null;
+            for (FileStatus file : files) {
+              String fileName = file.getPath().toString();
+              fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
+              if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
+                FileStatus[] storeFiles = fs.listStatus(file.getPath());
+                // For all the stores in this column family.
+                for (FileStatus storeFile : storeFiles) {
+                  HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
+                      getConf()));
+                  if ((reader.getFirstKey() != null)
+                      && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
+                          reader.getFirstKey()) > 0))) {
+                    storeFirstKey = reader.getFirstKey();
+                  }
+                  if ((reader.getLastKey() != null)
+                      && ((storeLastKey == null) || (comparator.compare(storeLastKey,
+                          reader.getLastKey())) < 0)) {
+                    storeLastKey = reader.getLastKey();
+                  }
+                  reader.close();
+                }
+              }
+            }
+            currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
+            currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
+            currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
+            currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
+            if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
+              currentRegionBoundariesInformation.metaFirstKey = null;
+            if (currentRegionBoundariesInformation.metaLastKey.length == 0)
+              currentRegionBoundariesInformation.metaLastKey = null;
+    
+            // For a region to be correct, we need the META start key to be smaller or equal to the
+            // smallest start key from all the stores, and the start key from the next META entry to
+            // be bigger than the last key from all the current stores. First region start key is null;
+            // Last region end key is null; some regions can be empty and not have any store.
+    
+            boolean valid = true;
+            // Checking start key.
+            if ((currentRegionBoundariesInformation.storesFirstKey != null)
+                && (currentRegionBoundariesInformation.metaFirstKey != null)) {
+              valid = valid
+                  && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
+                    currentRegionBoundariesInformation.metaFirstKey) >= 0;
+            }
+            // Checking stop key.
+            if ((currentRegionBoundariesInformation.storesLastKey != null)
+                && (currentRegionBoundariesInformation.metaLastKey != null)) {
+              valid = valid
+                  && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
+                    currentRegionBoundariesInformation.metaLastKey) < 0;
+            }
+            if (!valid) {
+              errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
+                tablesInfo.get(Bytes.toString(regionInfo.getTableName())));
+              LOG.warn("Region's boundaries not alligned between stores and META for:");
+              LOG.warn(currentRegionBoundariesInformation);
+            }
+          }
+        } catch (IOException e) {
+          LOG.error(e);
+        }
+      }
+    
   /**
    * Iterates through the list of all orphan/invalid regiondirs.
    */
@@ -2898,7 +3010,7 @@ public static enum ERROR_CODE {
       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
-      WRONG_USAGE
+      WRONG_USAGE, BOUNDARIES_ERROR
     }
     public void clear();
     public void report(String message);
@@ -3415,6 +3527,13 @@ public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
     this.hfcc = hfcc;
   }
 
+  /**
+   * Set region boundaries check mode.
+   */
+  void setRegionBoundariesCheck() {
+    checkRegionBoundaries = true;
+  }
+
   public void setRetCode(int code) {
     this.retcode = code;
   }
@@ -3457,6 +3576,7 @@ protected HBaseFsck printUsageAndExit() {
     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
+    out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
 
     out.println("");
     out.println("  Datafile Repair options: (expert features, use with caution!)");
@@ -3639,6 +3759,8 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
         setSummary();
       } else if (cmd.equals("-metaonly")) {
         setCheckMetaOnly();
+      } else if (cmd.equals("-boundaries")) {
+        setRegionBoundariesCheck();
       } else if (cmd.startsWith("-")) {
         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
         return printUsageAndExit();

From 13a19f5593ce46d94d95b4cff8e1e9abd1b61acd Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 23 Dec 2013 18:36:02 +0000
Subject: [PATCH 1277/1540] HBASE-10225 Bug in calls to
 RegionObsever.postScannerFilterRow (Anoop)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1553170 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/coprocessor/BaseRegionObserver.java      | 5 +++--
 .../apache/hadoop/hbase/coprocessor/RegionObserver.java   | 5 ++++-
 .../org/apache/hadoop/hbase/regionserver/HRegion.java     | 3 ++-
 .../hadoop/hbase/regionserver/RegionCoprocessorHost.java  | 8 +++++---
 4 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index b98bba9c10d4..ed925f9c1d27 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -340,10 +340,11 @@ public boolean postScannerNext(final ObserverContext<RegionCoprocessorEnvironmen
 
   @Override
   public boolean postScannerFilterRow(final ObserverContext<RegionCoprocessorEnvironment> e,
-      final InternalScanner s, final byte[] currentRow, final boolean hasMore) throws IOException {
+      final InternalScanner s, final byte[] currentRow, final int offset, final short length,
+      final boolean hasMore) throws IOException {
     return hasMore;
   }
-  
+
   @Override
   public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> e,
       final InternalScanner s) throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 93ff2528ba94..f11b4edf8d35 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -819,12 +819,15 @@ boolean postScannerNext(final ObserverContext<RegionCoprocessorEnvironment> c,
    * @param c the environment provided by the region server
    * @param s the scanner
    * @param currentRow The current rowkey which got filtered out
+   * @param offset offset to rowkey
+   * @param length length of rowkey
    * @param hasMore the 'has more' indication
    * @return whether more rows are available for the scanner or not
    * @throws IOException
    */
   boolean postScannerFilterRow(final ObserverContext<RegionCoprocessorEnvironment> c,
-      final InternalScanner s, final byte[] currentRow, final boolean hasMore) throws IOException;
+      final InternalScanner s, final byte[] currentRow, final int offset, final short length,
+      final boolean hasMore) throws IOException;
   
   /**
    * Called before the client closes a scanner.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 7586f989f22d..86b958c3f5fd 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4154,7 +4154,8 @@ protected boolean nextRow(byte [] currentRow, int offset, short length) throws I
       resetFilters();
       // Calling the hook in CP which allows it to do a fast forward
       if (this.region.getCoprocessorHost() != null) {
-        return this.region.getCoprocessorHost().postScannerFilterRow(this, currentRow);
+        return this.region.getCoprocessorHost().postScannerFilterRow(this, currentRow, offset,
+            length);
       }
       return true;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 407ca240d6f0..6efb8dcd1492 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1390,11 +1390,13 @@ public boolean postScannerNext(final InternalScanner s,
    * filter.
    * @param s the scanner
    * @param currentRow The current rowkey which got filtered out
+   * @param offset offset to rowkey
+   * @param length length of rowkey
    * @return whether more rows are available for the scanner or not
    * @throws IOException
    */
-  public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow)
-      throws IOException {
+  public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow, int offset,
+      short length) throws IOException {
     boolean hasMore = true; // By default assume more rows there.
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env : coprocessors) {
@@ -1402,7 +1404,7 @@ public boolean postScannerFilterRow(final InternalScanner s, final byte[] curren
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
           hasMore = ((RegionObserver) env.getInstance()).postScannerFilterRow(ctx, s, currentRow,
-              hasMore);
+              offset, length, hasMore);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env, e);
         }

From d2984358d449505a6b98deb701fba14ca5dfb2bf Mon Sep 17 00:00:00 2001
From: zjushch <zjushch@unknown>
Date: Tue, 24 Dec 2013 05:57:02 +0000
Subject: [PATCH 1278/1540] HBASE-10214 Regionserver shutdown improperly and
 leaves the dir in .old not deleted(binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1553246 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a4737701c4b0..030b12f91508 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -146,7 +146,6 @@
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CompressionTest;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -927,6 +926,10 @@ private boolean areAllUserRegionsOffline() {
 
   void tryRegionServerReport()
   throws IOException {
+    if (!keepLooping() && hbaseMaster == null) {
+      // the current server is stopping
+      return;
+    }
     HServerLoad hsl = buildServerLoad();
     // Why we do this?
     this.requestCount.set(0);

From 00ce6b03f7eee625e95fa6cbd1a91cda2d074026 Mon Sep 17 00:00:00 2001
From: rajeshbabu <rajeshbabu@unknown>
Date: Tue, 24 Dec 2013 18:08:42 +0000
Subject: [PATCH 1279/1540] HBASE-10215 TableNotFoundException should be thrown
 after removing stale znode in ETH

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1553311 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       | 11 +++++--
 .../master/handler/EnableTableHandler.java    |  1 +
 .../hbase/master/TestAssignmentManager.java   | 31 +++++++++++++++++++
 3 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 9afa95536de8..7e1b538239b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2810,8 +2810,15 @@ private void recoverTableInEnablingState(Set<String> enablingTables,
             + " to ENABLED state.");
         // enableTable in sync way during master startup,
         // no need to invoke coprocessor
-        new EnableTableHandler(this.master, tableName.getBytes(),
-            catalogTracker, this, true).process();
+        EnableTableHandler eth = null;
+        try {
+          eth =
+              new EnableTableHandler(this.master, tableName.getBytes(), catalogTracker, this, true);
+        } catch (TableNotFoundException e) {
+          LOG.warn("Table " + tableName + " not found in .META. to recover.");
+          continue;
+        }
+        if (eth != null) eth.process();
       }
     }
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
index 7f549648470d..42d49175eae9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -73,6 +73,7 @@ public EnableTableHandler(Server server, byte [] tableName,
       }
       try {
         this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true);
+        throw new TableNotFoundException(tableNameStr);
       } catch (KeeperException e) {
         // TODO : Use HBCK to clear such nodes
         LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index e3ebeaa6a230..9ff184a37e53 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -959,6 +959,37 @@ public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
     }
   }
 
+  /**
+   * Test verifies whether stale znodes of unknown tables as for the hbase:meta will be removed or
+   * not.
+   * @throws KeeperException
+   * @throws IOException
+   * @throws Exception
+   */
+  @Test
+  public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
+      throws KeeperException, IOException, Exception {
+    List<ServerName> destServers = new ArrayList<ServerName>(1);
+    destServers.add(SERVERNAME_A);
+    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(destServers);
+    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
+    HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
+    Server server = new HMaster(HTU.getConfiguration());
+    Whitebox.setInternalState(server, "serverManager", this.serverManager);
+    AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
+        this.serverManager);
+    try {
+      String tableName = "dummyTable";
+      am.enablingTables.put(tableName, null);
+      // set table in enabling state.
+      am.getZKTable().setEnablingTable(tableName);
+      am.joinCluster();
+      assertFalse("Table should not be present in zookeeper.",
+        am.getZKTable().isTablePresent(tableName));
+    } finally {
+    }
+  }
+
   /**
    * Test verifies whether all the enabling table regions assigned only once during master startup.
    * 

From d947ae4a82482fb3fb40d9d8b87cd88518ffb10e Mon Sep 17 00:00:00 2001
From: liangxie <liangxie@unknown>
Date: Thu, 26 Dec 2013 05:04:46 +0000
Subject: [PATCH 1280/1540] HBASE-8558 Addendum for handling SecureClient

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1553452 13f79535-47bb-0310-9956-ffa450edef68
---
 .../src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 1f670df5dc40..9599a6e909e9 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -268,7 +268,7 @@ protected synchronized void setupIOstreams()
         while (true) {
           setupConnection();
           InputStream inStream = NetUtils.getInputStream(socket);
-          OutputStream outStream = NetUtils.getOutputStream(socket);
+          OutputStream outStream = NetUtils.getOutputStream(socket, pingInterval);
           writeRpcHeader(outStream);
           if (useSasl) {
             final InputStream in2 = inStream;

From 82b1d9a50aeff9da6e8ad25ecfcdd5cd8471746c Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 27 Dec 2013 18:20:03 +0000
Subject: [PATCH 1281/1540] HBASE-7226 HRegion.checkAndMutate uses incorrect
 comparison result for <, <=, > and >=

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1553714 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegion.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 86b958c3f5fd..19f54c175f92 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2659,10 +2659,10 @@ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
               kv.getValueOffset(), kv.getValueLength());
           switch (compareOp) {
           case LESS:
-            matches = compareResult <= 0;
+            matches = compareResult < 0;
             break;
           case LESS_OR_EQUAL:
-            matches = compareResult < 0;
+            matches = compareResult <= 0;
             break;
           case EQUAL:
             matches = compareResult == 0;
@@ -2671,10 +2671,10 @@ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
             matches = compareResult != 0;
             break;
           case GREATER_OR_EQUAL:
-            matches = compareResult > 0;
+            matches = compareResult >= 0;
             break;
           case GREATER:
-            matches = compareResult >= 0;
+            matches = compareResult > 0;
             break;
           default:
             throw new RuntimeException("Unknown Compare op " + compareOp.name());

From 9b6caeb9fe8cfe17973f1a45193f0c82634f5a62 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 30 Dec 2013 05:46:19 +0000
Subject: [PATCH 1282/1540] HBASE-10257 [0.94] Master aborts due to assignment
 race

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1554144 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/AssignmentManager.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 7e1b538239b0..c8284f645704 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1886,8 +1886,7 @@ int setOfflineInZooKeeper(final RegionState state, boolean hijack,
     // OFFLINE. 
     if (!hijack && !state.isClosed() && !state.isOffline()) {
       if (!regionAlreadyInTransitionException ) {
-        String msg = "Unexpected state : " + state + " .. Cannot transit it to OFFLINE.";
-        this.master.abort(msg, new IllegalStateException(msg));
+        LOG.warn("Unexpected state : " + state + " .. Cannot transit it to OFFLINE.");
         return -1;
       } 
       LOG.debug("Unexpected state : " + state

From e21031a0a5cf7935ef750550e20b5d6bec3cf42b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 30 Dec 2013 05:50:20 +0000
Subject: [PATCH 1283/1540] HBASE-10250 [0.94] TestHLog fails occasionally

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1554146 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/wal/TestHLog.java     | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 96ac8a602f54..5496eaff51c4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.lang.reflect.Method;
+import java.net.BindException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -436,7 +437,17 @@ public void testAppendClose() throws Exception {
       // the idle time threshold configured in the conf above
       Thread.sleep(2000);
 
-      cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
+      cluster = null;
+      // retry a few times if the port is not freed, yet.
+      for (int i = 0; i < 30; i++) {
+        try {
+          cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
+          break;
+        } catch (BindException e) {
+          LOG.info("Sleeping.  BindException bringing up new cluster");
+          Thread.sleep(1000);
+        }
+      }
       TEST_UTIL.setDFSCluster(cluster);
       cluster.waitActive();
       fs = cluster.getFileSystem();

From e12ab86ca84967db714edd7bdb03a840e997dfee Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 30 Dec 2013 07:54:05 +0000
Subject: [PATCH 1284/1540] 0.94.16-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1554163 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index dfa01934a2c2..b840bb619dd1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.15</version>
+  <version>0.94.16-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 474af2252b150d8b6a39715a688f848dfc5f82ce Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 2 Jan 2014 19:02:12 +0000
Subject: [PATCH 1285/1540] HBASE-10259 [0.94] Upgrade JUnit to 4.11

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1554879 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                                    | 2 +-
 src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java   | 7 ++++++-
 .../java/org/apache/hadoop/hbase/client/TestAdmin.java     | 6 ++----
 .../org/apache/hadoop/hbase/client/TestMultiParallel.java  | 7 ++++++-
 .../apache/hadoop/hbase/master/TestAssignmentManager.java  | 3 +++
 .../apache/hadoop/hbase/master/TestSplitLogManager.java    | 4 +++-
 6 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/pom.xml b/pom.xml
index b840bb619dd1..d1fc35ae2ced 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1030,7 +1030,7 @@
     <jetty.jspapi.version>6.1.14</jetty.jspapi.version>
     <jersey.version>1.8</jersey.version>
     <jruby.version>1.6.5</jruby.version>
-    <junit.version>4.10-HBASE-1</junit.version>
+    <junit.version>4.11</junit.version>
     <slf4j.version>1.4.3</slf4j.version>
     <log4j.version>1.2.16</log4j.version>
     <mockito-all.version>1.8.5</mockito-all.version>
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 1ee0500568ea..1359ac1dee60 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -367,8 +367,13 @@ public void testCreateSilentIsReallySilent() throws InterruptedException,
         }
       }
     }
-    zk.close();
     ZKUtil.createAndFailSilent(zk2, aclZnode);
+
+    // reset /'s ACL for tests that follow
+    zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
+    zk.addAuthInfo("digest", "hbase:rox".getBytes());
+    zk.setACL("/", ZooDefs.Ids.OPEN_ACL_UNSAFE, -1);
+    zk.close();
  }
   
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index 7f72c0d262d8..c44543372bbc 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -27,8 +27,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -380,8 +378,6 @@ public void testHColumnValidName() {
   @Test
   public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
     final byte [] tableName = Bytes.toBytes("changeTableSchemaOnline");
-    TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
-        "hbase.online.schema.update.enable", true);
     HTableDescriptor [] tables = admin.listTables();
     int numTables = tables.length;
     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
@@ -500,6 +496,8 @@ public void testShouldFailOnlineSchemaUpdateIfOnlineSchemaIsNotEnabled()
       expectedException = true;
     }
     assertTrue("Online schema update should not happen.", expectedException);
+    TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
+        "hbase.online.schema.update.enable", true);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
index ae5ff1f80c2f..57135b2e4b8b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
@@ -34,11 +34,14 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.runners.MethodSorters;
 
 import static org.junit.Assert.*;
 
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
 @Category(MediumTests.class)
 public class TestMultiParallel {
   private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
@@ -290,7 +293,9 @@ private void doTestFlushCommits(boolean doAbort) throws Exception {
   }
 
   @Test (timeout=300000)
-  public void testBatchWithPut() throws Exception {
+  // FIXME: sort test lexicographically to the end.
+  // otherwise clashed with testFlushCommitsWithAbort
+  public void testZBatchWithPut() throws Exception {
     LOG.info("test=testBatchWithPut");
     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 9ff184a37e53..2f9779f607b2 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -71,8 +71,10 @@
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.runners.MethodSorters;
 import org.mockito.Mockito;
 import org.mockito.internal.util.reflection.Whitebox;
 
@@ -82,6 +84,7 @@
 /**
  * Test {@link AssignmentManager}
  */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
 @Category(MediumTests.class)
 public class TestAssignmentManager {
   private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index cb1d3e8b2204..ff87e3cfb9c1 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -27,7 +27,6 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import junit.framework.Assert;
-
 import static org.junit.Assert.*;
 
 import org.apache.commons.logging.Log;
@@ -50,10 +49,13 @@
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.runners.MethodSorters;
 import org.mockito.Mockito;
 
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
 @Category(MediumTests.class)
 public class TestSplitLogManager {
   private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class);

From 94d3a42dfc874a201c0b41de2f6cfa3a5083e348 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 2 Jan 2014 21:52:06 +0000
Subject: [PATCH 1286/1540] HBASE-8912 [0.94] AssignmentManager throws
 IllegalStateException from PENDING_OPEN to OFFLINE

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1554933 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java    | 3 ++-
 .../hadoop/hbase/regionserver/handler/OpenRegionHandler.java   | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 030b12f91508..c809f61a523c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3977,7 +3977,8 @@ public RegionServerCoprocessorHost getCoprocessorHost(){
 
   @Override
   public boolean removeFromRegionsInTransition(final HRegionInfo hri) {
-    return this.regionsInTransitionInRS.remove(hri.getEncodedNameAsBytes());
+    Boolean res = this.regionsInTransitionInRS.remove(hri.getEncodedNameAsBytes());
+    return res != null && res.booleanValue();
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 791848130623..d7edc300c559 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -100,6 +100,7 @@ public void process() throws IOException {
       // processing needs to do a close as part of cleanup.
       region = openRegion();
       if (region == null) {
+        this.rsServices.removeFromRegionsInTransition(this.regionInfo);
         tryTransitionToFailedOpen(regionInfo);
         transitionToFailedOpen = true;
         return;
@@ -112,6 +113,7 @@ public void process() throws IOException {
       }
       if (failed || this.server.isStopped() ||
           this.rsServices.isStopping()) {
+        this.rsServices.removeFromRegionsInTransition(this.regionInfo);
         cleanupFailedOpen(region);
         tryTransitionToFailedOpen(regionInfo);
         transitionToFailedOpen = true;

From f98e55fc980665f0ad8d4befe4e225820d1195c2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 2 Jan 2014 22:00:16 +0000
Subject: [PATCH 1287/1540] HBASE-10249 Intermittent TestReplicationSyncUpTool
 failure (Demai Ni)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1554937 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestReplicationSyncUpTool.java            | 76 +++++++++++++++----
 1 file changed, 60 insertions(+), 16 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index 7d553ac70e16..2541ef3850c7 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -22,6 +22,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -37,6 +39,8 @@
 @Category(LargeTests.class)
 public class TestReplicationSyncUpTool extends TestReplicationBase {
 
+  private static final Log LOG = LogFactory.getLog(TestReplicationSyncUpTool.class);
+
   private static final byte[] t1_su = Bytes.toBytes("t1_syncup");
   private static final byte[] t2_su = Bytes.toBytes("t2_syncup");
 
@@ -294,14 +298,34 @@ private void mimicSyncUpAfterDelete() throws Exception {
     assertEquals("@Peer1 t2_syncup should still have 200 rows", 200, rowCount_ht2TargetAtPeer1);
 
     // After sync up
-    syncUp(utility1);
-    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
-    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
-    assertEquals("@Peer1 t1_syncup should be sync up and have 50 rows", 50,
-      rowCount_ht1TargetAtPeer1);
-    assertEquals("@Peer1 t2_syncup should be sync up and have 100 rows", 100,
-      rowCount_ht2TargetAtPeer1);
-
+    for (int i = 0; i < NB_RETRIES; i++) {
+      syncUp(utility1);
+      rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+      rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+      if (i == NB_RETRIES - 1) {
+        if (rowCount_ht1TargetAtPeer1 != 50 || rowCount_ht2TargetAtPeer1 != 100) {
+          // syncUP still failed. Let's look at the source in case anything wrong there
+          utility1.restartHBaseCluster(1);
+          rowCount_ht1Source = utility1.countRows(ht1Source);
+          LOG.debug("t1_syncup should have 51 rows at source, and it is " + rowCount_ht1Source);
+          rowCount_ht2Source = utility1.countRows(ht2Source);
+          LOG.debug("t2_syncup should have 101 rows at source, and it is " + rowCount_ht2Source);
+        }
+        assertEquals("@Peer1 t1_syncup should be sync up and have 50 rows", 50,
+          rowCount_ht1TargetAtPeer1);
+        assertEquals("@Peer1 t2_syncup should be sync up and have 100 rows", 100,
+          rowCount_ht2TargetAtPeer1);
+      }
+      if (rowCount_ht1TargetAtPeer1 == 50 && rowCount_ht2TargetAtPeer1 == 100) {
+        LOG.info("SyncUpAfterDelete succeeded at retry = " + i);
+        break;
+      } else {
+        LOG.debug("SyncUpAfterDelete failed at retry = " + i + ", with rowCount_ht1TargetPeer1 ="
+            + rowCount_ht1TargetAtPeer1 + " and rowCount_ht2TargetAtPeer1 ="
+            + rowCount_ht2TargetAtPeer1);
+      }
+      Thread.sleep(SLEEP_TIME);
+    }
   }
 
   private void mimicSyncUpAfterPut() throws Exception {
@@ -350,14 +374,34 @@ private void mimicSyncUpAfterPut() throws Exception {
       rowCount_ht2TargetAtPeer1);
 
     // after syun up
-    syncUp(utility1);
-    rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
-    rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
-    assertEquals("@Peer1 t1_syncup should be sync up and have 100 rows", 100,
-      rowCount_ht1TargetAtPeer1);
-    assertEquals("@Peer1 t2_syncup should be sync up and have 200 rows", 200,
-      rowCount_ht2TargetAtPeer1);
-
+    for (int i = 0; i < NB_RETRIES; i++) {
+      syncUp(utility1);
+      rowCount_ht1TargetAtPeer1 = utility2.countRows(ht1TargetAtPeer1);
+      rowCount_ht2TargetAtPeer1 = utility2.countRows(ht2TargetAtPeer1);
+      if (i == NB_RETRIES - 1) {
+        if (rowCount_ht1TargetAtPeer1 != 100 || rowCount_ht2TargetAtPeer1 != 200) {
+          // syncUP still failed. Let's look at the source in case anything wrong there
+          utility1.restartHBaseCluster(1);
+          rowCount_ht1Source = utility1.countRows(ht1Source);
+          LOG.debug("t1_syncup should have 102 rows at source, and it is " + rowCount_ht1Source);
+          rowCount_ht2Source = utility1.countRows(ht2Source);
+          LOG.debug("t2_syncup should have 202 rows at source, and it is " + rowCount_ht2Source);
+        }
+        assertEquals("@Peer1 t1_syncup should be sync up and have 100 rows", 100,
+          rowCount_ht1TargetAtPeer1);
+        assertEquals("@Peer1 t2_syncup should be sync up and have 200 rows", 200,
+          rowCount_ht2TargetAtPeer1);
+      }
+      if (rowCount_ht1TargetAtPeer1 == 100 && rowCount_ht2TargetAtPeer1 == 200) {
+        LOG.info("SyncUpAfterPut succeeded at retry = " + i);
+        break;
+      } else {
+        LOG.debug("SyncUpAfterPut failed at retry = " + i + ", with rowCount_ht1TargetPeer1 ="
+            + rowCount_ht1TargetAtPeer1 + " and rowCount_ht2TargetAtPeer1 ="
+            + rowCount_ht2TargetAtPeer1);
+      }
+      Thread.sleep(SLEEP_TIME);
+    }
   }
 
   private void syncUp(HBaseTestingUtility ut) throws Exception {

From f145ddb085e80343ef7ca59e46791a0efdc62850 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 4 Jan 2014 04:49:29 +0000
Subject: [PATCH 1288/1540] HBASE-10279 TestStore.testDeleteExpiredStoreFiles
 is flaky

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1555321 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/Store.java    |  2 +-
 .../apache/hadoop/hbase/regionserver/TestStore.java    | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index a71c84d9e785..5ad34b08f3c7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1316,7 +1316,7 @@ private boolean isMajorCompaction(final List<StoreFile> filesToCompact) throws I
     }
     // TODO: Use better method for determining stamp of last major (HBASE-2990)
     long lowTimestamp = getLowestTimestamp(filesToCompact);
-    long now = System.currentTimeMillis();
+    long now = EnvironmentEdgeManager.currentTimeMillis();
     if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) {
       // Major compaction time has elapsed.
       if (filesToCompact.size() == 1) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 94c1f1a4804d..e39795ceff2c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -151,6 +151,8 @@ private void init(String methodName, Configuration conf,
   }
 
   public void testDeleteExpiredStoreFiles() throws Exception {
+    ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
+    EnvironmentEdgeManagerTestHelper.injectEdge(mee);
     int storeFileNum = 4;
     int ttl = 4;
     
@@ -172,8 +174,10 @@ public void testDeleteExpiredStoreFiles() throws Exception {
       this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
       this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
       flush(i);
-      Thread.sleep(sleepTime);
+      mee.incValue(sleepTime);
     }
+    // move time forward a bit more, so that the first file is expired
+    mee.incValue(1);
 
     // Verify the total number of store files
     assertEquals(storeFileNum, this.store.getStorefiles().size());
@@ -187,7 +191,7 @@ public void testDeleteExpiredStoreFiles() throws Exception {
       // If not the first compaction, there is another empty store file,
       assertEquals(Math.min(i, 2), cr.getFiles().size());
       for (int j = 0; i < cr.getFiles().size(); j++) {
-        assertTrue(cr.getFiles().get(j).getReader().getMaxTimestamp() < (System
+        assertTrue(cr.getFiles().get(j).getReader().getMaxTimestamp() < (EnvironmentEdgeManager
             .currentTimeMillis() - this.store.scanInfo.getTtl()));
       }
       // Verify that the expired store file is compacted to an empty store file.
@@ -197,7 +201,7 @@ public void testDeleteExpiredStoreFiles() throws Exception {
           .getEntries());
 
       // Let the next store file expired.
-      Thread.sleep(sleepTime);
+      mee.incValue(sleepTime);
     }
   }
 

From 75ba57242cb246391912355d75f9a349864dfa8d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 6 Jan 2014 18:22:29 +0000
Subject: [PATCH 1289/1540] HBASE-10272 Cluster becomes nonoperational if the
 node hosting the active Master AND ROOT/META table goes offline (Aditya
 Kishore)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1555960 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/catalog/CatalogTracker.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index 58c61ee1703d..bf606288200d 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
+import org.apache.hadoop.hbase.ipc.HBaseClient.FailedServerException;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -518,6 +519,10 @@ private HRegionInterface getCachedConnection(ServerName sn)
       LOG.debug("Exception connecting to " + sn);
     } catch (UnknownHostException e) {
       LOG.debug("Unknown host exception connecting to  " + sn);
+    } catch (FailedServerException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Server " + sn + " is in failed server list.");
+      }
     } catch (IOException ioe) {
       Throwable cause = ioe.getCause();
       if (ioe instanceof ConnectException) {

From 81eb1f901f49a383055b5b1933e61e61b1a631f9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 6 Jan 2014 18:24:32 +0000
Subject: [PATCH 1290/1540] HBASE-10284 Build broken with svn 1.8

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1555961 13f79535-47bb-0310-9956-ffa450edef68
---
 src/saveVersion.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/saveVersion.sh b/src/saveVersion.sh
index baae4e27793b..1a078f848111 100755
--- a/src/saveVersion.sh
+++ b/src/saveVersion.sh
@@ -27,7 +27,7 @@ date=`date`
 cwd=`pwd`
 if [ -d .svn ]; then
   revision=`svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'`
-  url=`svn info | sed -n -e 's/URL: \(.*\)/\1/p'`
+  url=`svn info | sed -n -e 's/^URL: \(.*\)/\1/p'`
 elif [ -d .git ]; then
   revision=`git log -1 --pretty=format:"%H"`
   hostname=`hostname`

From 3274bb8ef5805a0e5d4b3bd8d54f2ee406761e12 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 6 Jan 2014 18:32:31 +0000
Subject: [PATCH 1291/1540] HBASE-10273 AssignmentManager.regions and
 AssignmentManager.servers are not always updated in tandem (Feng Honghua)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1555966 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/master/AssignmentManager.java    | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index c8284f645704..aba6371fd0c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1766,6 +1766,7 @@ private void assign(final HRegionInfo region, final RegionState state,
           }
           synchronized (this.regions) {
             this.regions.put(plan.getRegionInfo(), plan.getDestination());
+            addToServers(plan.getDestination(), plan.getRegionInfo());
           }
         }
         break;
@@ -2250,7 +2251,13 @@ public void unassign(HRegionInfo region, boolean force) {
             }
             // Remove from the regionsMap
             synchronized (this.regions) {
-              this.regions.remove(region);
+              ServerName sn = this.regions.remove(region);
+              if (sn != null) {
+                Set<HRegionInfo> serverRegions = this.servers.get(sn);
+                if (serverRegions == null || !serverRegions.remove(region)) {
+                  LOG.warn("No " + region + " on " + sn);
+                }
+              }
             }
             deleteClosingOrClosedNode(region);
           }

From af2bd50f734ca91f5a2066f477216a19ee7c5095 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Mon, 6 Jan 2014 20:48:43 +0000
Subject: [PATCH 1292/1540] HBASE-10078 Dynamic Filter - Not using
 DynamicClassLoader when using FilterList

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1556027 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/FilterList.java       |  2 +-
 .../hadoop/hbase/io/HbaseObjectWritable.java  | 54 ++++++++++++++
 .../org/apache/hadoop/hbase/util/Classes.java | 21 ++++--
 .../apache/hadoop/hbase/client/TestGet.java   | 72 ++++++++++++++-----
 4 files changed, 123 insertions(+), 26 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 26abd98617fc..b9189c3dc326 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -321,7 +321,7 @@ public void readFields(final DataInput in) throws IOException {
     if (size > 0) {
       filters = new ArrayList<Filter>(size);
       for (int i = 0; i < size; i++) {
-        Filter filter = (Filter)HbaseObjectWritable.readObject(in, conf);
+        Filter filter = HbaseObjectWritable.readFilter(in, conf);
         filters.add(filter);
       }
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
index 430981b73539..149350de2bab 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -72,6 +72,7 @@
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.DependentColumnFilter;
+import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
 import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
@@ -93,6 +94,7 @@
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Classes;
 import org.apache.hadoop.hbase.util.ProtoUtil;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.ObjectWritable;
@@ -728,6 +730,58 @@ public static Object readObject(DataInput in,
     return instance;
   }
 
+  /**
+   * Read a {@link Filter} which is written as a {@link Writable}
+   * or a {@link Filter} directly. A custom filter class may be loaded
+   * dynamically.
+   *
+   * @param in
+   * @param conf
+   * @return the filter
+   * @throws IOException
+   */
+  @SuppressWarnings("unchecked")
+  public static Filter readFilter(
+      DataInput in, Configuration conf) throws IOException {
+    Class<?> instanceClass = null;
+    int b = (byte)WritableUtils.readVInt(in);
+    if (b != NOT_ENCODED) {
+      instanceClass = CODE_TO_CLASS.get(b);
+      if (instanceClass == Writable.class) {
+        // In case Writable, the actual type code follows
+        b = (byte)WritableUtils.readVInt(in);
+        if (b != NOT_ENCODED) {
+          instanceClass = CODE_TO_CLASS.get(b);
+        }
+      }
+    }
+    if (b == NOT_ENCODED) {
+      String className = Text.readString(in);
+      try {
+        instanceClass = (Class<? extends Filter>)getClassByName(conf, className);
+      } catch (ClassNotFoundException cnfe) {
+        try {
+          instanceClass = Classes.getFilterClassByName(className);
+        } catch (ClassNotFoundException e) {
+          LOG.error("Can't find class " + className, e);
+          throw new DoNotRetryIOException("Can't find class " + className, e);
+        }
+      }
+    }
+    Filter filter = (Filter)WritableFactories.newInstance(
+      (Class<? extends Filter>)instanceClass, conf);
+    try {
+      filter.readFields(in);
+    } catch (IOException io) {
+      LOG.error("Error in readFields", io);
+      throw io;
+    } catch (Exception e) {
+      LOG.error("Error in readFields", e);
+      throw new IOException("Error in readFields" , e);
+    }
+    return filter;
+  }
+
   /**
    * Try to instantiate a protocol buffer of the given message class
    * from the given input stream.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Classes.java b/src/main/java/org/apache/hadoop/hbase/util/Classes.java
index beca9c2f0ddf..f4f5bffb79ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Classes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Classes.java
@@ -93,6 +93,19 @@ public static String stringify(Class[] classes) {
     return buf.toString();
   }
 
+  /**
+   * Used to dynamically load a filter class.
+   *
+   * @param className the filter class name
+   * @return a filter class
+   * @throws ClassNotFoundException if couldn't find the class
+   */
+  @SuppressWarnings("unchecked")
+  public static Class<? extends Filter> getFilterClassByName(
+      String className) throws ClassNotFoundException {
+    return (Class<? extends Filter>) Class.forName(className, true, CLASS_LOADER);
+  }
+
   /**
    * Used to dynamically load a filter class, and create a Writable filter.
    * This filter class most likely extends Configurable.
@@ -100,11 +113,9 @@ public static String stringify(Class[] classes) {
    * @param className the filter class name.
    * @return a filter
    */
-  @SuppressWarnings("unchecked")
   public static Filter createWritableForName(String className) {
     try {
-      Class<? extends Filter> clazz =
-        (Class<? extends Filter>) Class.forName(className, true, CLASS_LOADER);
+      Class<? extends Filter> clazz = getFilterClassByName(className);
       return (Filter)WritableFactories.newInstance(clazz, new Configuration());
     } catch (ClassNotFoundException e) {
       throw new RuntimeException("Can't find class " + className);
@@ -118,11 +129,9 @@ public static Filter createWritableForName(String className) {
    * @param className the filter class name.
    * @return a filter
    */
-  @SuppressWarnings("unchecked")
   public static Filter createForName(String className) {
     try {
-      Class<? extends Filter> clazz =
-        (Class<? extends Filter>)Class.forName(className, true, CLASS_LOADER);
+      Class<? extends Filter> clazz = getFilterClassByName(className);
       return (Filter)clazz.newInstance();
     } catch (ClassNotFoundException e) {
       throw new RuntimeException("Can't find class " + className);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index 3cf92cbd9e7b..d434ccc4d431 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -20,7 +20,9 @@
 
 package org.apache.hadoop.hbase.client;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
@@ -33,10 +35,14 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Assert;
@@ -52,23 +58,35 @@ public class TestGet {
   private static final String WRITABLE_GET =
     "AgD//////////wAAAAEBD3Rlc3QuTW9ja0ZpbHRlcgEAAAAAAAAAAH//////////AQAAAAAAAAAA";
 
+  private static final String WRITABLE_GET_WITH_FILTER_LIST =
+    "AgD//////////wAAAAEBKW9yZy5hcGFjaGUuaGFkb29wLmhiYXNlLmZpbHRlci5GaWx0ZXJMaXN0" +
+    "AAAAAAMOAA90ZXN0Lk1vY2tGaWx0ZXIOAA1teS5Nb2NrRmlsdGVyDkYAAQAAAAAAAAAAf///////" +
+    "//8BAAAAAAAAAAA=";
+
   private static final String MOCK_FILTER_JAR =
-    "UEsDBBQACAgIACmBi0IAAAAAAAAAAAAAAAAJAAQATUVUQS1JTkYv/soAAAMAUEsHCAAAAAACAAAA" +
-    "AAAAAFBLAwQUAAgICAApgYtCAAAAAAAAAAAAAAAAFAAAAE1FVEEtSU5GL01BTklGRVNULk1G803M" +
+    "UEsDBBQACAgIADRQI0QAAAAAAAAAAAAAAAAJAAQATUVUQS1JTkYv/soAAAMAUEsHCAAAAAACAAAA" +
+    "AAAAAFBLAwQUAAgICAA0UCNEAAAAAAAAAAAAAAAAFAAAAE1FVEEtSU5GL01BTklGRVNULk1G803M" +
     "y0xLLS7RDUstKs7Mz7NSMNQz4OVyLkpNLElN0XWqBAmY6xnEG1gqaPgXJSbnpCo45xcV5BcllgCV" +
-    "a/Jy8XIBAFBLBwgxyqRbQwAAAEQAAABQSwMECgAACAAAbICLQgAAAAAAAAAAAAAAAAUAAAB0ZXN0" +
-    "L1BLAwQUAAgICAAcgItCAAAAAAAAAAAAAAAAFQAAAHRlc3QvTW9ja0ZpbHRlci5jbGFzc41Qy07C" +
-    "QBS9A4VKBZGHoO7cgQvHmLjCuPBBQlJloWE/tCMdLZ1mOlV/y5WJCz/AjzLeDqCRYOIs7uuce87N" +
-    "fHy+vQPAEezakCNQ1TzR9Ep6D30Raq5ssAh0pZpQFjMv4DRgvpQxDcYs4fTOcOiMeoYTAsUTEQl9" +
-    "SiDf6Y4IWOfS5w7koVSGAhTRwBURv06nY65u2TjEjborPRaOmBJZPx9aOhAJgZq7dE+PgKM48/uC" +
-    "hz4SWh33nj0yKiS9YJoNojjVvczYuXz2eKyFjBIb6gQaC9pg+I2gDVOTQwRXiBAoPCmh8Zb2b49h" +
-    "qhcmzVUAet/IVHkcL8bt6s/xBxkb9gA/B7KXxwo/BaONHcVMMBf2X2HtBYscOBiLZliCdYzlGQFz" +
-    "BTOBDagiaxNrC7uakTk2m4guS1SMRGsGziWyqgFN47xlsH+K1f4UaxuxbcPf+QJQSwcI8UIYqlEB" +
-    "AABeAgAAUEsBAhQAFAAICAgAKYGLQgAAAAACAAAAAAAAAAkABAAAAAAAAAAAAAAAAAAAAE1FVEEt" +
-    "SU5GL/7KAABQSwECFAAUAAgICAApgYtCMcqkW0MAAABEAAAAFAAAAAAAAAAAAAAAAAA9AAAATUVU" +
-    "QS1JTkYvTUFOSUZFU1QuTUZQSwECCgAKAAAIAABsgItCAAAAAAAAAAAAAAAABQAAAAAAAAAAAAAA" +
-    "AADCAAAAdGVzdC9QSwECFAAUAAgICAAcgItC8UIYqlEBAABeAgAAFQAAAAAAAAAAAAAAAADlAAAA" +
-    "dGVzdC9Nb2NrRmlsdGVyLmNsYXNzUEsFBgAAAAAEAAQA8wAAAHkCAAAAAA==";
+    "a/Jy8XIBAFBLBwgxyqRbQwAAAEQAAABQSwMEFAAICAgAcIsiRAAAAAAAAAAAAAAAABMAAABteS9N" +
+    "b2NrRmlsdGVyLmNsYXNzjVDLTsJAFL1DC5UKIg9B3bkDF4wxcYVx4YOEBGWhYT+0Ix0tnWaY+vgs" +
+    "VyYu/AA/yng7gAaDibO4r3PuOTfz8fn2DgCHsOtAhkBx8kwvpXffFaHmygGbQEuqMWUx8wJOA+ZL" +
+    "GdNgxKac3hoOnVFPcUIgdywioU8IWM3WkIB9Jn3uggX5AmQhR6DUFxG/SiYjrm7YKMSNSl96LBwy" +
+    "JdJ+PrR1IKYpe+maDgFXceZ3BQ99hOvN/h17YFRIes4060VxojuprXvx5PFYCxlNHagQqC5ovcE3" +
+    "giZMjQ8QXCFCIPuohMZLGsseg0QvTGqrAPS+lonyOF6M26Wf49spG/YAvwbSZ2GFX4LRwY5iJpiz" +
+    "+6+w9oJFBlyMOTPMwzrGwoyAuYiZwAaUkLWJtY1d2cgcmU1Ef0sUjUR9Bs4l0qoKNeO8ZbB/ipX/" +
+    "FGsYsW3D3/kCUEsHCEYmW6RQAQAAWgIAAFBLAwQUAAgICABuiyJEAAAAAAAAAAAAAAAAFQAAAHRl" +
+    "c3QvTW9ja0ZpbHRlci5jbGFzc41Qy07CQBS9A4VKBZGHoO7cgQvHmLjCuPBBQlJloWE/tCMdLZ1m" +
+    "OlV/y5WJCz/AjzLeDqCRYOIs7uuce87NfHy+vQPAEezakCNQ1TzR9Ep6D30Raq5ssAh0pZpQFjMv" +
+    "4DRgvpQxDcYs4fTOcOiMeoYTAsUTEQl9SiDf6Y4IWOfS5w7koVSGAhTRwBURv06nY65u2TjEjbor" +
+    "PRaOmBJZPx9aOhAJgZq7dE+PgKM48/uChz4SWh33nj0yKiS9YJoNojjVvczYuXz2eKyFjBIb6gQa" +
+    "C9pg+I2gDVOTQwRXiBAoPCmh8Zb2b49hqhcmzVUAet/IVHkcL8bt6s/xBxkb9gA/B7KXxwo/BaON" +
+    "HcVMMBf2X2HtBYscOBiLZliCdYzlGQFzBTOBDagiaxNrC7uakTk2m4guS1SMRGsGziWyqgFN47xl" +
+    "sH+K1f4UaxuxbcPf+QJQSwcI8UIYqlEBAABeAgAAUEsBAhQAFAAICAgANFAjRAAAAAACAAAAAAAA" +
+    "AAkABAAAAAAAAAAAAAAAAAAAAE1FVEEtSU5GL/7KAABQSwECFAAUAAgICAA0UCNEMcqkW0MAAABE" +
+    "AAAAFAAAAAAAAAAAAAAAAAA9AAAATUVUQS1JTkYvTUFOSUZFU1QuTUZQSwECFAAUAAgICABwiyJE" +
+    "RiZbpFABAABaAgAAEwAAAAAAAAAAAAAAAADCAAAAbXkvTW9ja0ZpbHRlci5jbGFzc1BLAQIUABQA" +
+    "CAgIAG6LIkTxQhiqUQEAAF4CAAAVAAAAAAAAAAAAAAAAAFMCAAB0ZXN0L01vY2tGaWx0ZXIuY2xh" +
+    "c3NQSwUGAAAAAAQABAABAQAA5wMAAAAA";
 
   @Test
   public void testAttributesSerialization() throws IOException {
@@ -156,8 +174,15 @@ public void testDynamicFilter() throws Exception {
       fail("Should not be able to load the filter class");
     } catch (RuntimeException re) {
       String msg = re.getMessage();
-      Assert.assertTrue(msg != null
-        && msg.contains("Can't find class test.MockFilter"));
+      assertTrue(msg != null && msg.contains("Can't find class test.MockFilter"));
+    }
+
+    dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET_WITH_FILTER_LIST));
+    try {
+      get.readFields(dis);
+      fail("Should not be able to load the filter class");
+    } catch (IOException ioe) {
+      assertTrue(ioe.getCause() instanceof ClassNotFoundException);
     }
 
     FileOutputStream fos = new FileOutputStream(jarFile);
@@ -166,8 +191,17 @@ public void testDynamicFilter() throws Exception {
 
     dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET));
     get.readFields(dis);
-    Assert.assertEquals("test.MockFilter",
-      get.getFilter().getClass().getName());
+    assertEquals("test.MockFilter", get.getFilter().getClass().getName());
+
+    get = new Get();
+    dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET_WITH_FILTER_LIST));
+    get.readFields(dis);
+    assertTrue(get.getFilter() instanceof FilterList);
+    List<Filter> filters = ((FilterList)get.getFilter()).getFilters();
+    assertEquals(3, filters.size());
+    assertEquals("test.MockFilter", filters.get(0).getClass().getName());
+    assertEquals("my.MockFilter", filters.get(1).getClass().getName());
+    assertTrue(filters.get(2) instanceof KeyOnlyFilter);
   }
 
   @org.junit.Rule

From c621140225dfeb087a9ad5b38689704efbda1416 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 6 Jan 2014 23:13:05 +0000
Subject: [PATCH 1293/1540] HBASE-10286 Revert HBASE-9593, breaks RS wildcard
 addresses

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1556061 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     |   5 +-
 .../TestRSKilledWhenInitializing.java         | 135 ------------------
 2 files changed, 2 insertions(+), 138 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c809f61a523c..9b918e2fe0d1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -741,9 +741,6 @@ public void run() {
     }
 
     try {
-      // Set our ephemeral znode up in zookeeper now we have a name.
-      createMyEphemeralNode();
-   
       // Try and register with the Master; tell it we are here.  Break if
       // server is stopped or the clusterup flag is down or hdfs went wacky.
       while (keepLooping()) {
@@ -1090,6 +1087,8 @@ protected void handleReportForDutyResponse(final MapWritable c)
         this.conf.set("mapred.task.id", "hb_rs_" +
           this.serverNameFromMasterPOV.toString());
       }
+      // Set our ephemeral znode up in zookeeper now we have a name.
+      createMyEphemeralNode();
 
       // Master sent us hbase.rootdir to use. Should be fully qualified
       // path with file system specification included. Set 'fs.defaultFS'
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
deleted file mode 100644
index cb097d2de20f..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.LocalHBaseCluster;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
-import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.io.Writable;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Tests region server termination during startup.
- */
-@Category(LargeTests.class)
-public class TestRSKilledWhenInitializing {
-  private static boolean masterActive = false;
-  
-  private static AtomicBoolean firstRS = new AtomicBoolean(true);
-
-  /**
-   * Test verifies whether a region server is removing from online servers list in master if it went
-   * down after registering with master.
-   * @throws Exception
-   */
-  @Test(timeout = 180000)
-  public void testRSTermnationAfterRegisteringToMasterBeforeCreatingEphemeralNod() throws Exception {
-
-    final int NUM_MASTERS = 1;
-    final int NUM_RS = 2;
-    firstRS.set(true);
-    // Create config to use for this cluster
-    Configuration conf = HBaseConfiguration.create();
-
-    // Start the cluster
-    final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniDFSCluster(3);
-    TEST_UTIL.startMiniZKCluster();
-    TEST_UTIL.createRootDir();
-    final LocalHBaseCluster cluster =
-        new LocalHBaseCluster(conf, NUM_MASTERS, NUM_RS, HMaster.class, MockedRegionServer.class);
-    final MasterThread master = cluster.getMasters().get(0);
-    master.start();
-    try {
-      long startTime = System.currentTimeMillis();
-      while (!master.getMaster().isActiveMaster()) {
-        try {
-          Thread.sleep(100);
-        } catch (InterruptedException ignored) {
-        }
-        if (System.currentTimeMillis() > startTime + 30000) {
-          throw new RuntimeException("Master not active after 30 seconds");
-        }
-      }
-      masterActive = true;
-      cluster.getRegionServers().get(0).start();
-      cluster.getRegionServers().get(1).start();
-      Thread.sleep(10000);
-      List<ServerName> onlineServersList =
-          master.getMaster().getServerManager().getOnlineServersList();
-      while (onlineServersList.size()!=1) {
-        Thread.sleep(100);
-        onlineServersList = master.getMaster().getServerManager().getOnlineServersList();
-      }
-      assertEquals(onlineServersList.size(), 1);
-    } finally {
-      masterActive = false;
-      firstRS.set(true);
-      TEST_UTIL.shutdownMiniZKCluster();
-      TEST_UTIL.cleanupTestDir();
-      TEST_UTIL.shutdownMiniDFSCluster();
-    }
-  }
-
-  public static class MockedRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
-
-    public MockedRegionServer(Configuration conf) throws IOException, InterruptedException {
-      super(conf);
-    }
-
-    @Override
-    protected void handleReportForDutyResponse(final MapWritable c) throws IOException {
-      if (firstRS.getAndSet(false)) {
-        for (Map.Entry<Writable, Writable> e : c.entrySet()) {
-          String key = e.getKey().toString();
-          // The hostname the master sees us as.
-          if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
-            String hostnameFromMasterPOV = e.getValue().toString();
-            assertEquals(super.getRpcServer().getListenerAddress().getHostName(),
-              hostnameFromMasterPOV);
-          }
-        }
-        while (!masterActive) {
-          Threads.sleep(100);
-        }
-        super.kill();
-      } else {
-        super.handleReportForDutyResponse(c);
-      }
-    }
-  }
-}

From 15cd724443af5bdb499f4cb409f984dfa44821d2 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 7 Jan 2014 23:01:35 +0000
Subject: [PATCH 1294/1540] HBASE-10286 All for configurable policies in
 ChaosMonkey (Cody Marcel)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1556395 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/ChaosMonkey.java | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
index ef9496408ee1..8c6ebda0084e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
@@ -37,8 +37,8 @@
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerLoad;
-import org.apache.hadoop.hbase.IntegrationTestingUtility;
 import org.apache.hadoop.hbase.IntegrationTestDataIngestWithChaosMonkey;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -81,6 +81,14 @@ public class ChaosMonkey extends AbstractHBaseTool implements Stoppable {
 
   final IntegrationTestingUtility util;
 
+  /**
+   * Construct a new ChaosMonkey
+   * @param util the HBaseIntegrationTestingUtility already configured
+   */
+  public ChaosMonkey(IntegrationTestingUtility util) {
+	  this.util = util;
+  }
+  
   /**
    * Construct a new ChaosMonkey
    * @param util the HBaseIntegrationTestingUtility already configured
@@ -108,6 +116,13 @@ private void setPoliciesByName(String... policies) {
     }
   }
 
+  private void setPolicies(Policy... policies) {
+    this.policies = new Policy[policies.length];
+    for (int i = 0; i < policies.length; i++) {
+      this.policies[i] = policies[i];
+    }
+  }
+
   /**
    * Context for Action's
    */
@@ -786,6 +801,9 @@ protected void processOptions(CommandLine cmd) {
     String[] policies = cmd.getOptionValues("policy");
     if (policies != null) {
       setPoliciesByName(policies);
+    } else {
+      // Set a default policy if none is provided
+      setPolicies(NAMED_POLICIES.get(EVERY_MINUTE_RANDOM_ACTION_POLICY));
     }
   }
 
@@ -802,7 +820,7 @@ public static void main(String[] args) throws Exception {
     IntegrationTestingUtility util = new IntegrationTestingUtility(conf);
     util.initializeCluster(1);
 
-    ChaosMonkey monkey = new ChaosMonkey(util, EVERY_MINUTE_RANDOM_ACTION_POLICY);
+    ChaosMonkey monkey = new ChaosMonkey(util);
     int ret = ToolRunner.run(conf, monkey, args);
     System.exit(ret);
   }

From 9b00861b466449b49dc4240cdbc23e802a65fb42 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Thu, 9 Jan 2014 19:02:17 +0000
Subject: [PATCH 1295/1540] HBASE-10268. TestSplitLogWorker occasionally fails

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1556921 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/TestSplitLogWorker.java      | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index 6fa56dbf78c1..5d4b78279d6a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -56,6 +56,7 @@ public class TestSplitLogWorker {
   static {
     Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
   }
+  private static final int WAIT_TIME = 15000;
   private final static HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
   private ZooKeeperWatcher zkw;
@@ -126,7 +127,7 @@ public Status exec(String name, CancelableProgressable p) {
 
   };
 
-  @Test
+  @Test(timeout=60000)
   public void testAcquireTaskAtStartup() throws Exception {
     LOG.info("testAcquireTaskAtStartup");
     ZKSplitLog.Counters.resetCounters();
@@ -139,7 +140,7 @@ public void testAcquireTaskAtStartup() throws Exception {
       "rs", neverEndingTask);
     slw.start();
     try {
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, WAIT_TIME);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tatas")), "rs"));
     } finally {
@@ -151,14 +152,14 @@ private void stopSplitLogWorker(final SplitLogWorker slw)
   throws InterruptedException {
     if (slw != null) {
       slw.stop();
-      slw.worker.join(3000);
+      slw.worker.join(WAIT_TIME);
       if (slw.worker.isAlive()) {
         assertTrue(("Could not stop the worker thread slw=" + slw) == null);
       }
     }
   }
 
-  @Test
+  @Test(timeout=60000)
   public void testRaceForTask() throws Exception {
     LOG.info("testRaceForTask");
     ZKSplitLog.Counters.resetCounters();
@@ -174,10 +175,10 @@ public void testRaceForTask() throws Exception {
     slw1.start();
     slw2.start();
     try {
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, WAIT_TIME);
       // Assert that either the tot_wkr_failed_to_grab_task_owned count was set of if
       // not it, that we fell through to the next counter in line and it was set.
-      assertTrue(waitForCounterBoolean(tot_wkr_failed_to_grab_task_owned, 0, 1, 1500) ||
+      assertTrue(waitForCounterBoolean(tot_wkr_failed_to_grab_task_owned, 0, 1, WAIT_TIME) ||
         tot_wkr_failed_to_grab_task_lost_race.get() == 1);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "trft")), "svr1") ||
@@ -189,7 +190,7 @@ public void testRaceForTask() throws Exception {
     }
   }
 
-  @Test
+  @Test(timeout=60000)
   public void testPreemptTask() throws Exception {
     LOG.info("testPreemptTask");
     ZKSplitLog.Counters.resetCounters();
@@ -206,20 +207,20 @@ public void testPreemptTask() throws Exception {
         TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, WAIT_TIME);
       assertEquals(1, slw.taskReadySeq);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tpt_task")), "tpt_svr"));
 
       ZKUtil.setData(zkw, ZKSplitLog.getEncodedNodeName(zkw, "tpt_task"),
         TaskState.TASK_UNASSIGNED.get("manager"));
-      waitForCounter(tot_wkr_preempt_task, 0, 1, 1500);
+      waitForCounter(tot_wkr_preempt_task, 0, 1, WAIT_TIME);
     } finally {
       stopSplitLogWorker(slw);
     }
   }
 
-  @Test
+  @Test(timeout=60000)
   public void testMultipleTasks() throws Exception {
     LOG.info("testMultipleTasks");
     ZKSplitLog.Counters.resetCounters();
@@ -234,7 +235,7 @@ public void testMultipleTasks() throws Exception {
         TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
         CreateMode.PERSISTENT);
 
-      waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
+      waitForCounter(tot_wkr_task_acquired, 0, 1, WAIT_TIME);
       // now the worker is busy doing the above task
 
       // create another task
@@ -245,9 +246,9 @@ public void testMultipleTasks() throws Exception {
       // preempt the first task, have it owned by another worker
       ZKUtil.setData(zkw, ZKSplitLog.getEncodedNodeName(zkw, "tmt_task"),
         TaskState.TASK_OWNED.get("another-worker"));
-      waitForCounter(tot_wkr_preempt_task, 0, 1, 1500);
+      waitForCounter(tot_wkr_preempt_task, 0, 1, WAIT_TIME);
 
-      waitForCounter(tot_wkr_task_acquired, 1, 2, 1500);
+      waitForCounter(tot_wkr_task_acquired, 1, 2, WAIT_TIME);
       assertEquals(2, slw.taskReadySeq);
       assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw,
         ZKSplitLog.getEncodedNodeName(zkw, "tmt_task_2")), "tmt_svr"));
@@ -256,7 +257,7 @@ public void testMultipleTasks() throws Exception {
     }
   }
 
-  @Test
+  @Test(timeout=60000)
   public void testRescan() throws Exception {
     LOG.info("testRescan");
     ZKSplitLog.Counters.resetCounters();
@@ -271,12 +272,12 @@ public void testRescan() throws Exception {
       TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
       CreateMode.PERSISTENT);
 
-    waitForCounter(tot_wkr_task_acquired, 0, 1, 1500);
+    waitForCounter(tot_wkr_task_acquired, 0, 1, WAIT_TIME);
     // now the worker is busy doing the above task
 
     // preempt the task, have it owned by another worker
     ZKUtil.setData(zkw, task, TaskState.TASK_UNASSIGNED.get("manager"));
-    waitForCounter(tot_wkr_preempt_task, 0, 1, 1500);
+    waitForCounter(tot_wkr_preempt_task, 0, 1, WAIT_TIME);
 
     // create a RESCAN node
     String rescan = ZKSplitLog.getEncodedNodeName(zkw, "RESCAN");
@@ -284,13 +285,13 @@ public void testRescan() throws Exception {
       TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE,
       CreateMode.PERSISTENT_SEQUENTIAL);
 
-    waitForCounter(tot_wkr_task_acquired, 1, 2, 1500);
+    waitForCounter(tot_wkr_task_acquired, 1, 2, WAIT_TIME);
     // RESCAN node might not have been processed if the worker became busy
     // with the above task. preempt the task again so that now the RESCAN
     // node is processed
     ZKUtil.setData(zkw, task, TaskState.TASK_UNASSIGNED.get("manager"));
-    waitForCounter(tot_wkr_preempt_task, 1, 2, 1500);
-    waitForCounter(tot_wkr_task_acquired_rescan, 0, 1, 1500);
+    waitForCounter(tot_wkr_preempt_task, 1, 2, WAIT_TIME);
+    waitForCounter(tot_wkr_task_acquired_rescan, 0, 1, WAIT_TIME);
 
     List<String> nodes = ZKUtil.listChildrenNoWatch(zkw, zkw.splitLogZNode);
     LOG.debug(nodes);

From 0d58e2d42390d5a0dfbcb8d7a7be2ef277f16e65 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 10 Jan 2014 00:51:43 +0000
Subject: [PATCH 1296/1540] HBASE-10282
 TestMultiParallel.testFlushCommitsNoAbort fails frequently in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1557003 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/client/TestMultiParallel.java  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
index 57135b2e4b8b..404ccffc4430 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
@@ -277,17 +277,16 @@ private void doTestFlushCommits(boolean doAbort) throws Exception {
     List<JVMClusterUtil.RegionServerThread> liveRSs =
       UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
     int count = 0;
+    int regionCount = 0;
     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
       count++;
+      regionCount += t.getRegionServer().getOnlineRegions().size();
       LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
     }
     LOG.info("Count=" + count);
     Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
         (doAbort ? (liveRScount - 1) : liveRScount), count);
-    for (JVMClusterUtil.RegionServerThread t: liveRSs) {
-      int regions = t.getRegionServer().getOnlineRegions().size();
-      Assert.assertTrue("Count of regions=" + regions, regions > 10);
-    }
+    Assert.assertTrue("Count of regions=" + regionCount, regionCount >= 25);
     table.close();
     LOG.info("done");
   }

From ec8ab0aad5c17503c1e6da0a02af6665f53f1ea3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 10 Jan 2014 01:52:20 +0000
Subject: [PATCH 1297/1540] HBASE-10306 Backport HBASE-6820 to 0.94,
 MiniZookeeperCluster should ensure that ZKDatabase is closed upon shutdown()
 (chendihao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1557009 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/zookeeper/MiniZooKeeperCluster.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
index 2551e6868b26..6587eb9bc02d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
@@ -197,7 +197,9 @@ public int startup(File baseDir, int numZooKeeperServers) throws IOException,
 
   private void recreateDir(File dir) throws IOException {
     if (dir.exists()) {
-      FileUtil.fullyDelete(dir);
+      if(!FileUtil.fullyDelete(dir)) {
+        throw new IOException("Could not delete zk base directory: " + dir);
+      }
     }
     try {
       dir.mkdirs();
@@ -213,6 +215,7 @@ public void shutdown() throws IOException {
     if (!started) {
       return;
     }
+
     // shut down all the zk servers
     for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
       NIOServerCnxnFactory standaloneServerFactory =
@@ -224,6 +227,10 @@ public void shutdown() throws IOException {
         throw new IOException("Waiting for shutdown of standalone server");
       }
     }
+    for (ZooKeeperServer zkServer: zooKeeperServers) {
+      //explicitly close ZKDatabase since ZookeeperServer does not close them
+      zkServer.getZKDatabase().close();
+    }
 
     // clear everything
     started = false;

From 829c241a0908b08931b1ee4f1fc397bea8d669d9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 10 Jan 2014 20:06:14 +0000
Subject: [PATCH 1298/1540] Release notes and version change for 0.94.16RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1557240 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 34 ++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 5a8859e0e8af..266c3e69b5dc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,38 @@
 HBase Change Log
+Release 0.94.16 - 01/10/2014
+Sub-task
+
+    [HBASE-10257] - [0.94] Master aborts due to assignment race
+
+Bug
+
+    [HBASE-7226] - HRegion.checkAndMutate uses incorrect comparison result for <, <=, > and >=
+    [HBASE-8558] - Add timeout limit for HBaseClient dataOutputStream
+    [HBASE-8912] - [0.94] AssignmentManager throws IllegalStateException from PENDING_OPEN to OFFLINE
+    [HBASE-9346] - HBCK should provide an option to check if regions boundaries are the same in META and in stores.
+    [HBASE-10078] - Dynamic Filter - Not using DynamicClassLoader when using FilterList
+    [HBASE-10193] - Cleanup HRegion if one of the store fails to open at region initialization
+    [HBASE-10214] - Regionserver shutdown improperly and leaves the dir in .old not deleted
+    [HBASE-10215] - TableNotFoundException should be thrown after removing stale znode in ETH
+    [HBASE-10225] - Bug in calls to RegionObsever.postScannerFilterRow
+    [HBASE-10250] - [0.94] TestHLog fails occasionally
+    [HBASE-10268] - TestSplitLogWorker occasionally fails
+    [HBASE-10272] - Cluster becomes nonoperational if the node hosting the active Master AND ROOT/META table goes offline
+    [HBASE-10273] - AssignmentManager.regions and AssignmentManager.servers are not always updated in tandem
+    [HBASE-10279] - TestStore.testDeleteExpiredStoreFiles is flaky
+    [HBASE-10281] - TestMultiParallel.testFlushCommitsNoAbort fails frequently in 0.94
+    [HBASE-10284] - Build broken with svn 1.8
+    [HBASE-10286] - Revert HBASE-9593, breaks RS wildcard addresses
+    [HBASE-10306] - Backport HBASE-6820 to 0.94, MiniZookeeperCluster should ensure that ZKDatabase is closed upon shutdown()
+
+Improvement
+
+    [HBASE-10285] - All for configurable policies in ChaosMonkey
+
+Test
+
+    [HBASE-10259] - [0.94] Upgrade JUnit to 4.11
+
 
 Release 0.94.15 - 12/17/2013
 Bug
diff --git a/pom.xml b/pom.xml
index d1fc35ae2ced..7bb45e111043 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.16-SNAPSHOT</version>
+  <version>0.94.16</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 53384e31e991944d65afb45b281872f5a5efa19d Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Tue, 14 Jan 2014 03:38:16 +0000
Subject: [PATCH 1299/1540] HBASE-10274 MiniZookeeperCluster should close
 ZKDatabase when shutdown ZooKeeperServers (chendihao via enis)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1557923 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
index 6587eb9bc02d..92ff81b800bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
@@ -262,6 +262,8 @@ public int killCurrentActiveZooKeeperServer() throws IOException,
     if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
       throw new IOException("Waiting for shutdown of standalone server");
     }
+    
+    zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close();
 
     // remove the current active zk server
     standaloneServerFactoryList.remove(activeZKServerIndex);
@@ -303,6 +305,8 @@ public void killOneBackupZooKeeperServer() throws IOException,
     if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
       throw new IOException("Waiting for shutdown of standalone server");
     }
+    
+    zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close();
 
     // remove this backup zk server
     standaloneServerFactoryList.remove(backupZKServerIndex);

From 19cf85909e3867c6c83ae38024c26e9018cf2a65 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 14 Jan 2014 06:54:02 +0000
Subject: [PATCH 1300/1540] HBASE-10320 Avoid ArrayList.iterator()
 ExplicitColumnTracker

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1557951 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ExplicitColumnTracker.java   | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index 9590b318657b..419d17f8a138 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -19,8 +19,6 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.NavigableSet;
 
 import org.apache.hadoop.hbase.HConstants;
@@ -61,7 +59,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
   * Each ColumnCount instance also tracks how many versions of the requested
   * column have been returned.
   */
-  private final List<ColumnCount> columns;
+  private final ColumnCount[] columns;
   private int index;
   private ColumnCount column;
   /** Keeps track of the latest timestamp included for current column.
@@ -83,9 +81,10 @@ public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
     this.maxVersions = maxVersions;
     this.minVersions = minVersions;
     this.oldestStamp = oldestUnexpiredTS;
-    this.columns = new ArrayList<ColumnCount>(columns.size());
+    this.columns = new ColumnCount[columns.size()];
+    int i=0;
     for(byte [] column : columns) {
-      this.columns.add(new ColumnCount(column));
+      this.columns[i++] = new ColumnCount(column);
     }
     reset();
   }
@@ -94,7 +93,7 @@ public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
    * Done when there are no more columns to match against.
    */
   public boolean done() {
-    return this.index >= this.columns.size();
+    return this.index >= columns.length;
   }
 
   public ColumnCount getColumnHint() {
@@ -150,7 +149,7 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
           return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
         }
         // This is the recursive case.
-        this.column = this.columns.get(this.index);
+        this.column = this.columns[this.index];
       }
     } while(true);
   }
@@ -177,7 +176,7 @@ public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int le
       }
       // We are done with current column; advance to next column
       // of interest.
-      this.column = this.columns.get(this.index);
+      this.column = this.columns[this.index];
       return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
     }
     setTS(timestamp);
@@ -187,7 +186,7 @@ public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int le
   // Called between every row.
   public void reset() {
     this.index = 0;
-    this.column = this.columns.get(this.index);
+    this.column = this.columns[this.index];
     for(ColumnCount col : this.columns) {
       col.setCount(0);
     }
@@ -230,7 +229,7 @@ public void doneWithColumn(byte [] bytes, int offset, int length) {
           // Will not hit any more columns in this storefile
           this.column = null;
         } else {
-          this.column = this.columns.get(this.index);
+          this.column = this.columns[this.index];
         }
         if (compare <= -1)
           continue;

From 86e4472c21ea2f1ba9f9b3d8fe13a42d88b68426 Mon Sep 17 00:00:00 2001
From: liangxie <liangxie@unknown>
Date: Wed, 15 Jan 2014 06:09:23 +0000
Subject: [PATCH 1301/1540] HBASE-10335 AuthFailedException in zookeeper may
 block replication forever (Liu Shaohui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1558304 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/replication/ReplicationZookeeper.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index a7aa59edf22b..5c234d5adedb 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -48,6 +48,7 @@
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.AuthFailedException;
 import org.apache.zookeeper.KeeperException.ConnectionLossException;
 import org.apache.zookeeper.KeeperException.SessionExpiredException;
 
@@ -242,6 +243,9 @@ public List<ServerName> getSlavesAddresses(String peerClusterId) {
     try {
       addresses = fetchSlavesAddresses(peer.getZkw());
     } catch (KeeperException ke) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Fetch salves addresses failed.", ke);
+      }
       reconnectPeer(ke, peer);
       addresses = Collections.emptyList();
     }
@@ -878,8 +882,8 @@ public UUID getUUIDForCluster(ZooKeeperWatcher zkw) throws KeeperException {
   }
 
   private void reconnectPeer(KeeperException ke, ReplicationPeer peer) {
-    if (ke instanceof ConnectionLossException
-      || ke instanceof SessionExpiredException) {
+    if (ke instanceof ConnectionLossException || ke instanceof SessionExpiredException
+        || ke instanceof AuthFailedException) {
       LOG.warn(
         "Lost the ZooKeeper connection for peer " + peer.getClusterKey(),
         ke);

From 9b6d6d629b14720c48ee690524d6df53e3e0baef Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Tue, 21 Jan 2014 21:30:12 +0000
Subject: [PATCH 1302/1540] HBASE-10249 TestReplicationSyncUpTool fails because
 failover takes too long

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1560198 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/ReplicationZookeeper.java    | 14 ++++++++++----
 .../regionserver/ReplicationSourceManager.java     |  4 ++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 5c234d5adedb..6436f0b9ae3c 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -631,6 +631,16 @@ public List<String> getListHLogsForPeerForRS(String rs, String id) {
     return result;
   }
 
+  /**
+   * Checks if the provided znode is the same as this region server's
+   * @param znode to check
+   * @return if this is this rs's znode
+   */
+  public boolean isThisOurZnode(String znode) {
+    String otherRs = ZKUtil.joinZNode(this.rsZNode, znode);
+    return otherRs.equals(rsServerNameZnode);
+  }
+
   /**
    * Try to set a lock in another server's znode.
    * @param znode the server names of the other server
@@ -639,10 +649,6 @@ public List<String> getListHLogsForPeerForRS(String rs, String id) {
   public boolean lockOtherRS(String znode) {
     try {
       String parent = ZKUtil.joinZNode(this.rsZNode, znode);
-      if (parent.equals(rsServerNameZnode)) {
-        LOG.warn("Won't lock because this is us, we're dead!");
-        return false;
-      }
       String p = ZKUtil.joinZNode(parent, RS_LOCK_ZNODE);
       ZKUtil.createAndWatch(this.zookeeper, p, Bytes.toBytes(rsServerNameZnode));
     } catch (KeeperException e) {
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 185597eed244..86c16d14f479 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -594,6 +594,10 @@ public NodeFailoverWorker(String rsZnode) {
 
     @Override
     public void run() {
+      // We could end up checking if this is us
+      if (zkHelper.isThisOurZnode(this.rsZnode)) {
+        return;
+      }
       // Wait a bit before transferring the queues, we may be shutting down.
       // This sleep may not be enough in some cases.
       try {

From 14ca9196e62d31017ad0c9bfbc4f526f155189ea Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 23 Jan 2014 03:23:42 +0000
Subject: [PATCH 1303/1540] HBASE-10400 [hbck] Continue if region dir missing
 on region merge attempt

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1560579 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/util/HBaseFsck.java  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8ec65524ccc1..d250a48aee8f 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -1958,7 +1959,20 @@ public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOEx
 
     // rename the contained into the container.
     FileSystem fs = targetRegionDir.getFileSystem(getConf());
-    FileStatus[] dirs = fs.listStatus(contained.getHdfsRegionDir());
+    FileStatus[] dirs = null;
+    try { 
+      dirs = fs.listStatus(contained.getHdfsRegionDir());
+    } catch (FileNotFoundException fnfe) {
+      // region we are attempting to merge in is not present!  Since this is a merge, there is
+      // no harm skipping this region if it does not exist.
+      if (!fs.exists(contained.getHdfsRegionDir())) {
+        LOG.warn("HDFS region dir " + contained.getHdfsRegionDir() + " is missing. " +
+            "Assuming already sidelined or moved.");
+      } else {
+        sidelineRegionDir(fs, contained);
+      }
+      return fileMoves;
+    }
 
     if (dirs == null) {
       if (!fs.exists(contained.getHdfsRegionDir())) {

From 2f0f665ec001c06003c9a3dff7f160d2a39af451 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 23 Jan 2014 17:42:00 +0000
Subject: [PATCH 1304/1540] move to 0.94.17-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1560767 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 7bb45e111043..d5524de297fc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.16</version>
+  <version>0.94.17-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 648c068d472ee231f4ed85fb990c5a8f2bbe80d5 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Thu, 23 Jan 2014 18:10:56 +0000
Subject: [PATCH 1305/1540] HBASE-10401 [hbck] perform overlap group merges in
 parallel

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1560776 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 111 ++++++++++++++----
 1 file changed, 89 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index d250a48aee8f..08cf82a08492 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -171,7 +171,8 @@ public class HBaseFsck extends Configured implements Tool {
   private HConnection connection;
   private HBaseAdmin admin;
   private HTable meta;
-  protected ExecutorService executor; // threads to retrieve data from regionservers
+  // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
+  protected ExecutorService executor;
   private long startMillis = System.currentTimeMillis();
   private HFileCorruptionChecker hfcc;
   private int retcode = 0;
@@ -1953,8 +1954,8 @@ SortedMap<String, TableInfo> checkIntegrity() throws IOException {
    */
   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
     int fileMoves = 0;
-
-    LOG.debug("Contained region dir after close and pause");
+    String thread = Thread.currentThread().getName();
+    LOG.debug("[" + thread + "] Contained region dir after close and pause");
     debugLsr(contained.getHdfsRegionDir());
 
     // rename the contained into the container.
@@ -1966,8 +1967,8 @@ public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOEx
       // region we are attempting to merge in is not present!  Since this is a merge, there is
       // no harm skipping this region if it does not exist.
       if (!fs.exists(contained.getHdfsRegionDir())) {
-        LOG.warn("HDFS region dir " + contained.getHdfsRegionDir() + " is missing. " +
-            "Assuming already sidelined or moved.");
+        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() 
+            + " is missing. Assuming already sidelined or moved.");
       } else {
         sidelineRegionDir(fs, contained);
       }
@@ -1976,7 +1977,8 @@ public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOEx
 
     if (dirs == null) {
       if (!fs.exists(contained.getHdfsRegionDir())) {
-        LOG.warn("HDFS region dir " + contained.getHdfsRegionDir() + " already sidelined.");
+        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() 
+            + " already sidelined.");
       } else {
         sidelineRegionDir(fs, contained);
       }
@@ -1997,7 +1999,7 @@ public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOEx
         continue;
       }
 
-      LOG.info("Moving files from " + src + " into containing region " + dst);
+      LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
       // FileSystem.rename is inconsistent with directories -- if the
       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
       // it moves the src into the dst dir resulting in (foo/a/b).  If
@@ -2008,19 +2010,37 @@ public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOEx
           fileMoves++;
         }
       }
-      LOG.debug("Sideline directory contents:");
+      LOG.debug("[" + thread + "] Sideline directory contents:");
       debugLsr(targetRegionDir);
     }
 
     // if all success.
     sidelineRegionDir(fs, contained);
-    LOG.info("Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
+    LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
         getSidelineDir());
     debugLsr(contained.getHdfsRegionDir());
 
     return fileMoves;
   }
 
+
+  static class WorkItemOverlapMerge implements Callable<Void> {
+    private TableIntegrityErrorHandler handler;
+    Collection<HbckInfo> overlapgroup;
+    
+    WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
+      this.handler = handler;
+      this.overlapgroup = overlapgroup;
+    }
+    
+    @Override
+    public Void call() throws Exception {
+      handler.handleOverlapGroup(overlapgroup);
+      return null;
+    }
+  };
+  
+  
   /**
    * Maintain information about a particular table.
    */
@@ -2246,6 +2266,8 @@ public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) thr
        * Cases:
        * - Clean regions that overlap
        * - Only .oldlogs regions (can't find start/stop range, or figure out)
+       * 
+       * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
        */
       @Override
       public void handleOverlapGroup(Collection<HbckInfo> overlap)
@@ -2273,7 +2295,8 @@ public void handleOverlapGroup(Collection<HbckInfo> overlap)
 
       void mergeOverlaps(Collection<HbckInfo> overlap)
           throws IOException {
-        LOG.info("== Merging regions into one region: "
+        String thread = Thread.currentThread().getName();
+        LOG.info("== [" + thread + "] Merging regions into one region: "
           + Joiner.on(",").join(overlap));
         // get the min / max range and close all concerned regions
         Pair<byte[], byte[]> range = null;
@@ -2291,25 +2314,25 @@ void mergeOverlaps(Collection<HbckInfo> overlap)
             }
           }
           // need to close files so delete can happen.
-          LOG.debug("Closing region before moving data around: " +  hi);
-          LOG.debug("Contained region dir before close");
+          LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
+          LOG.debug("[" + thread + "] Contained region dir before close");
           debugLsr(hi.getHdfsRegionDir());
           try {
-            LOG.info("Closing region: " + hi);
+            LOG.info("[" + thread + "] Closing region: " + hi);
             closeRegion(hi);
           } catch (IOException ioe) {
-            LOG.warn("Was unable to close region " + hi
+            LOG.warn("[" + thread + "] Was unable to close region " + hi
               + ".  Just continuing... ", ioe);
           } catch (InterruptedException e) {
-            LOG.warn("Was unable to close region " + hi
+            LOG.warn("[" + thread + "] Was unable to close region " + hi
               + ".  Just continuing... ", e);
           }
 
           try {
-            LOG.info("Offlining region: " + hi);
+            LOG.info("[" + thread + "] Offlining region: " + hi);
             offline(hi.getRegionName());
           } catch (IOException ioe) {
-            LOG.warn("Unable to offline region from master: " + hi
+            LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
               + ".  Just continuing... ", ioe);
           }
         }
@@ -2320,7 +2343,7 @@ void mergeOverlaps(Collection<HbckInfo> overlap)
         HRegionInfo newRegion = new HRegionInfo(htd.getName(), range.getFirst(),
             range.getSecond());
         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
-        LOG.info("Created new empty container region: " +
+        LOG.info("[" + thread + "] Created new empty container region: " +
             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
         debugLsr(region.getRegionDir());
 
@@ -2328,7 +2351,7 @@ void mergeOverlaps(Collection<HbckInfo> overlap)
         boolean didFix= false;
         Path target = region.getRegionDir();
         for (HbckInfo contained : overlap) {
-          LOG.info("Merging " + contained  + " into " + target );
+          LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
           int merges = mergeRegionDirs(target, contained);
           if (merges > 0) {
             didFix = true;
@@ -2477,9 +2500,21 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc
       if (prevKey != null) {
         handler.handleRegionEndKeyNotEmpty(prevKey);
       }
-      
-      for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
-        handler.handleOverlapGroup(overlap);
+
+      // TODO fold this into the TableIntegrityHandler
+      if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
+        LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
+            " false to run serially.");
+        boolean ok = handleOverlapsParallel(handler, prevKey);
+        if (!ok) {
+          return false;
+        }
+      } else {
+        LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
+            " true to run in parallel.");
+        for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
+          handler.handleOverlapGroup(overlap);
+        }
       }
 
       if (details) {
@@ -2503,6 +2538,38 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc
       return errors.getErrorList().size() == originalErrorsCount;
     }
 
+    private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
+        throws IOException {
+      // we parallelize overlap handler for the case we have lots of groups to fix.  We can
+      // safely assume each group is independent. 
+      List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
+      List<Future<Void>> rets;
+      for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
+        // 
+        merges.add(new WorkItemOverlapMerge(overlap, handler));
+      }
+      try {
+        rets = executor.invokeAll(merges);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+        LOG.error("Overlap merges were interrupted", e);
+        return false;
+      }
+      for(int i=0; i<merges.size(); i++) {
+        WorkItemOverlapMerge work = merges.get(i);
+        Future<Void> f = rets.get(i);
+        try {
+          f.get();
+        } catch(ExecutionException e) {
+          LOG.warn("Failed to merge overlap group" + work, e.getCause());
+        } catch (InterruptedException e) {
+          LOG.error("Waiting for overlap merges was interrupted", e);
+          return false;
+        }
+      }
+      return true;
+    }
+
     /**
      * This dumps data in a visually reasonable way for visual debugging
      * 

From 5175446888ed11d6b6a51c387508253e99c8bcc4 Mon Sep 17 00:00:00 2001
From: jyates <jyates@unknown>
Date: Sat, 25 Jan 2014 01:20:35 +0000
Subject: [PATCH 1306/1540] HBASE-10383 Secure Bulk Load for 'completebulkload'
 fails for version 0.94.15

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1561243 13f79535-47bb-0310-9956-ffa450edef68
---
 .../access/SecureBulkLoadEndpoint.java        | 14 ++++++-
 .../access/SecureBulkLoadProtocol.java        |  3 +-
 .../TestSecureLoadIncrementalHFiles.java      |  5 +--
 ...ureLoadIncrementalHFilesSplitRecovery.java |  3 +-
 .../coprocessor/SecureBulkLoadClient.java     |  2 +-
 .../mapreduce/LoadIncrementalHFiles.java      | 17 ++++++--
 ...SecurityEnabledUserProviderForTesting.java | 41 -------------------
 .../mapreduce/TestLoadIncrementalHFiles.java  | 10 +++--
 ...estLoadIncrementalHFilesSplitRecovery.java | 19 +++++----
 9 files changed, 49 insertions(+), 65 deletions(-)
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
index 46a707a93135..946a97ed2647 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
@@ -150,7 +150,7 @@ public void cleanupBulkLoad(String bulkToken) throws IOException {
 
   @Override
   public boolean bulkLoadHFiles(final List<Pair<byte[], String>> familyPaths,
-                                final Token<?> userToken, final String bulkToken) throws IOException {
+                                final Token<?> userToken, final String bulkToken, boolean assignSeqNum) throws IOException {
     User user = getActiveUser();
     final UserGroupInformation ugi = user.getUGI();
     if(userToken != null) {
@@ -167,6 +167,7 @@ public boolean bulkLoadHFiles(final List<Pair<byte[], String>> familyPaths,
       bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
     }
     boolean loaded = false;
+    final IOException[] es = new IOException[1];
     if (!bypass) {
       loaded = ugi.doAs(new PrivilegedAction<Boolean>() {
         @Override
@@ -189,13 +190,22 @@ public Boolean run() {
             //To enable access prior to staging
             return env.getRegion().bulkLoadHFiles(familyPaths,
                 new SecureBulkLoadListener(fs, bulkToken));
-          } catch (Exception e) {
+          }
+          catch(DoNotRetryIOException e){
+            es[0] = e;
+          }
+          catch (Exception e) {
             LOG.error("Failed to complete bulk load", e);
           }
           return false;
         }
       });
     }
+
+    if (es[0] != null) {
+      throw es[0];
+    }
+
     if (region.getCoprocessorHost() != null) {
       loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
     }
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
index e381d3c0472c..4f58a465954e 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
@@ -58,10 +58,11 @@ public interface SecureBulkLoadProtocol extends CoprocessorProtocol {
    * @param familyPaths column family to HFile path pairs
    * @param userToken requesting user's HDFS delegation token
    * @param bulkToken
+   * @param assignSeqId
    * @return
    * @throws IOException
    */
   boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
-                         Token<?> userToken, String bulkToken) throws IOException;
+                         Token<?> userToken, String bulkToken, boolean assignSeqNum) throws IOException;
 
 }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
index e950685f6ce6..f3187d590463 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
@@ -20,10 +20,8 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import org.apache.hadoop.hbase.LargeTests;
-import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.security.access.AccessControlLists;
 import org.apache.hadoop.hbase.security.access.SecureTestUtil;
-
 import org.junit.BeforeClass;
 import org.junit.experimental.categories.Category;
 
@@ -43,10 +41,9 @@ public class TestSecureLoadIncrementalHFiles extends  TestLoadIncrementalHFiles{
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
+    useSecureHBaseOverride = true;
     // setup configuration
     SecureTestUtil.enableSecurity(util.getConfiguration());
-    UserProvider.setUserProviderForTesting(util.getConfiguration(),
-      HadoopSecurityEnabledUserProviderForTesting.class);
 
     util.startMiniCluster();
 
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
index a808966ba604..7eadedd65278 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
@@ -47,11 +47,10 @@ public class TestSecureLoadIncrementalHFilesSplitRecovery extends TestLoadIncrem
   //make sure they are in sync
   @BeforeClass
   public static void setupCluster() throws Exception {
+    useSecureHBaseOverride = true;
     util = new HBaseTestingUtility();
     // setup configuration
     SecureTestUtil.enableSecurity(util.getConfiguration());
-    UserProvider.setUserProviderForTesting(util.getConfiguration(),
-      HadoopSecurityEnabledUserProviderForTesting.class);
     util.startMiniCluster();
 
     // Wait for the ACL table to become available
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
index cbfcd5740e33..5489d03c7dee 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java
@@ -79,7 +79,7 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, Token<?> u
       String bulkToken, boolean assignSeqNum) throws IOException {
     try {
       return (Boolean) Methods.call(protocolClazz, proxy, "bulkLoadHFiles", new Class[] {
-          List.class, Token.class, String.class, Boolean.class },
+          List.class, Token.class, String.class, boolean.class },
         new Object[] { familyPaths, userToken, bulkToken, assignSeqNum });
     } catch (Exception e) {
       throw new IOException("Failed to bulkLoadHFiles", e);
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 217d87d93d35..633e5faa24c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -75,6 +75,7 @@
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.security.token.Token;
@@ -107,14 +108,17 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
   private final boolean assignSeqIds;
   private UserProvider userProvider;
 
-  //package private for testing
   public LoadIncrementalHFiles(Configuration conf) throws Exception {
+    this(conf, false);
+  }
+
+  public LoadIncrementalHFiles(Configuration conf, boolean useSecureHBaseOverride) throws Exception {
     super(conf);
     this.cfg = conf;
     this.hbAdmin = new HBaseAdmin(conf);
     //added simple for testing
     this.userProvider = UserProvider.instantiate(conf);
-    this.useSecure = userProvider.isHBaseSecurityEnabled();
+    this.useSecure = useSecureHBaseOverride || userProvider.isHBaseSecurityEnabled();
     this.assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, false);
   }
 
@@ -571,7 +575,14 @@ public Boolean call() throws Exception {
 
     try {
       List<LoadQueueItem> toRetry = new ArrayList<LoadQueueItem>();
-      boolean success = svrCallable.withRetries();
+      boolean success;
+      // secure client wraps the result in another layer of callables, which does its own retrying -
+      // we shouldn't rety again here as well.
+      if (useSecure) {
+        success = svrCallable.withoutRetries();
+      } else {
+        success = svrCallable.withRetries();
+      }
       if (!success) {
         LOG.warn("Attempt to bulk load region containing "
             + Bytes.toStringBinary(first) + " into table "
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java
deleted file mode 100644
index a693755f4113..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/HadoopSecurityEnabledUserProviderForTesting.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mapreduce;
-
-import org.apache.hadoop.hbase.client.UserProvider;
-
-/**
- * A {@link UserProvider} that always says hadoop security is enabled, regardless of the underlying
- * configuration. HBase security is <i>not enabled</i> as this is used to determine if SASL is used
- * to do the authentication, which requires a Kerberos ticket (which we currently don't have in
- * tests).
- * <p>
- * This should only be used for <b>TESTING</b>.
- */
-public class HadoopSecurityEnabledUserProviderForTesting extends UserProvider {
-
-  @Override
-  public boolean isHBaseSecurityEnabled() {
-    return false;
-  }
-
-  @Override
-  public boolean isHadoopSecurityEnabled() {
-    return true;
-  }
-}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 02afd4eb1042..b6506b66aef0 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -58,6 +58,7 @@
  */
 @Category(LargeTests.class)
 public class TestLoadIncrementalHFiles {
+  protected static boolean useSecureHBaseOverride = false;
   private static final byte[] QUALIFIER = Bytes.toBytes("myqual");
   private static final byte[] FAMILY = Bytes.toBytes("myfam");
 
@@ -160,7 +161,8 @@ private void runTest(String testName, BloomType bloomType,
     HTable table = new HTable(util.getConfiguration(), TABLE);
     util.waitTableAvailable(TABLE, 30000);
 
-    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
+    LoadIncrementalHFiles loader =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride);
     loader.doBulkLoad(dir, table);
 
     assertEquals(expectedRows, util.countRows(table));
@@ -192,7 +194,8 @@ private void runTest(String testName, BloomType bloomType,
 
     HTable table = new HTable(util.getConfiguration(), TABLE);
     util.waitTableAvailable(TABLE, 30000);
-    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
+    LoadIncrementalHFiles loader =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride);
 
     // Do a dummy put to increase the hlog sequence number
     Put put = new Put(Bytes.toBytes("row"));
@@ -247,7 +250,8 @@ public void testNonexistentColumnFamilyLoad() throws Exception {
     util.waitTableAvailable(TABLE, 30000);
     // make sure we go back to the usual user provider
     UserProvider.setUserProviderForTesting(util.getConfiguration(), UserProvider.class);
-    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
+    LoadIncrementalHFiles loader =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride);
     try {
       loader.doBulkLoad(dir, table);
       assertTrue("Loading into table with non-existent family should have failed", false);
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
index bb67ebe36b8a..391fd37f4df6 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
@@ -72,6 +72,8 @@ public class TestLoadIncrementalHFilesSplitRecovery {
   final static int NUM_CFS = 10;
   final static byte[] QUAL = Bytes.toBytes("qual");
   final static int ROWCOUNT = 100;
+  
+  protected static boolean useSecureHBaseOverride = false;
 
   private final static byte[][] families = new byte[NUM_CFS][];
   static {
@@ -134,7 +136,8 @@ private Path buildBulkFiles(String table, int value) throws Exception {
    */
   private void populateTable(String table, int value) throws Exception {
     // create HFiles for different column families
-    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration());
+    LoadIncrementalHFiles lih =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride);
     Path bulk1 = buildBulkFiles(table, value);
     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
     lih.doBulkLoad(bulk1, t);
@@ -226,7 +229,7 @@ public void testBulkLoadPhaseFailure() throws Exception {
     final AtomicInteger attmptedCalls = new AtomicInteger();
     final AtomicInteger failedCalls = new AtomicInteger();
     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+        util.getConfiguration(), useSecureHBaseOverride) {
 
       protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
           byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis)
@@ -293,8 +296,8 @@ public void testSplitWhileBulkLoadPhase() throws Exception {
     // Now let's cause trouble.  This will occur after checks and cause bulk
     // files to fail when attempt to atomically import.  This is recoverable.
     final AtomicInteger attemptedCalls = new AtomicInteger();
-    LoadIncrementalHFiles lih2 = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+    LoadIncrementalHFiles lih2 =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
 
       protected void bulkLoadPhase(final HTable htable, final HConnection conn,
           ExecutorService pool, Deque<LoadQueueItem> queue,
@@ -334,8 +337,8 @@ public void testGroupOrSplitPresplit() throws Exception {
     forceSplit(table);
 
     final AtomicInteger countedLqis= new AtomicInteger();
-    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+    LoadIncrementalHFiles lih =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
       protected List<LoadQueueItem> groupOrSplit(
           Multimap<ByteBuffer, LoadQueueItem> regionGroups,
           final LoadQueueItem item, final HTable htable,
@@ -366,8 +369,8 @@ public void testGroupOrSplitFailure() throws Exception {
     String table = "groupOrSplitFailure";
     setupTable(table, 10);
 
-    LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
-        util.getConfiguration()) {
+    LoadIncrementalHFiles lih =
+        new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
       int i = 0;
 
       protected List<LoadQueueItem> groupOrSplit(

From 25985cefb3391155cd863aee1d43c7e349e42bc3 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Mon, 27 Jan 2014 17:00:56 +0000
Subject: [PATCH 1307/1540] HBASE-10423 Report back the message of split or
 rollback failure to the master (Harsh J)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1561749 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/SplitRequest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
index 9ca424f7ce70..11acee5c3238 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
@@ -89,7 +89,7 @@ public void run() {
             parent.getRegionNameAsString() + " -- aborting server";
           // If failed rollback, kill this server to avoid having a hole in table.
           LOG.info(msg, ee);
-          this.server.abort(msg);
+          this.server.abort(msg + " -- Cause: " + ee.getMessage());
         }
         return;
       }

From aa9a9d26382c010a9246cc8e4f47a7718b1e840c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 27 Jan 2014 19:49:55 +0000
Subject: [PATCH 1308/1540] HBASE-10371 Compaction creates empty hfile, then
 selects this file for compaction and creates empty hfile and over again.
 (binlijin)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1561810 13f79535-47bb-0310-9956-ffa450edef68
---
 .../compactions/CompactSelection.java         |  5 ++
 .../regionserver/TestCompactSelection.java    | 49 ++++++++++++++++++-
 2 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java
index 26338e17eefd..9b335785db71 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java
@@ -112,6 +112,11 @@ public CompactSelection selectExpiredStoreFilesToCompact(
     }
 
     if (hasExpiredStoreFiles) {
+      if (expiredStoreFiles.size() == 1
+          && expiredStoreFiles.get(0).getReader().getEntries() == 0) {
+        // If just one empty store file, do not select for compaction.
+        return expiredSFSelection;
+      }
       expiredSFSelection = new CompactSelection(conf, expiredStoreFiles);
     }
     return expiredSFSelection;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
index 451ada65d489..f996dda5b227 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
@@ -33,15 +33,20 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.SmallTests;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.experimental.categories.Category;
 
 import com.google.common.collect.Lists;
-import org.junit.experimental.categories.Category;
 
 @Category(SmallTests.class)
 public class TestCompactSelection extends TestCase {
@@ -100,6 +105,8 @@ public void setUp() throws Exception {
   static class MockStoreFile extends StoreFile {
     long length = 0;
     boolean isRef = false;
+    TimeRangeTracker timeRangeTracker;
+    long entryCount;
 
     MockStoreFile(long length, boolean isRef) throws IOException {
       super(TEST_UTIL.getTestFileSystem(), TEST_FILE,
@@ -124,14 +131,35 @@ boolean isReference() {
       return this.isRef;
     }
 
+    void setTimeRangeTracker(TimeRangeTracker timeRangeTracker) {
+      this.timeRangeTracker = timeRangeTracker;
+    }
+
+    void setEntries(long entryCount) {
+      this.entryCount = entryCount;
+    }
+
     @Override
     public StoreFile.Reader getReader() {
       final long len = this.length;
+      final TimeRangeTracker timeRange = this.timeRangeTracker;
+      final long entries = this.entryCount;
       return new StoreFile.Reader() {
         @Override
         public long length() {
           return len;
         }
+
+        @Override
+        public long getMaxTimestamp() {
+          return timeRange == null ? Long.MAX_VALUE
+              : timeRange.maximumTimestamp;
+        }
+
+        @Override
+        public long getEntries() {
+          return entries;
+        }
       };
     }
   }
@@ -282,6 +310,23 @@ public void testOffPeakCompactionRatio() throws IOException {
     compactEquals(sfCreate(999,50,12,12, 1), 12, 12, 1);
   }
 
+  public void testCompactionEmptyHFile() throws IOException {
+    // Do not compact empty store file
+    List<StoreFile> candidates = sfCreate(0);
+    for (StoreFile file : candidates) {
+      if (file instanceof MockStoreFile) {
+        MockStoreFile mockFile = (MockStoreFile) file;
+        mockFile.setTimeRangeTracker(new TimeRangeTracker(-1, -1));
+        mockFile.setEntries(0);
+      }
+    }
+    // Test Default compactions
+    CompactSelection compactSelection = new CompactSelection(conf, candidates);
+    CompactSelection result = compactSelection
+        .selectExpiredStoreFilesToCompact(600L);
+    assertTrue(result == null);
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 06fe63f1362079e6a918376e10c0ad86422b6f47 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 3 Feb 2014 05:39:55 +0000
Subject: [PATCH 1309/1540] HBASE-10448 ZKUtil create and watch methods don't
 set watch in some cases. (Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1563783 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index a91d44154a17..98e53ac722ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -960,20 +960,21 @@ private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
       String znode, byte [] data)
   throws KeeperException {
+    boolean ret = true;
     try {
       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
           CreateMode.EPHEMERAL);
     } catch (KeeperException.NodeExistsException nee) {
-      if(!watchAndCheckExists(zkw, znode)) {
-        // It did exist but now it doesn't, try again
-        return createEphemeralNodeAndWatch(zkw, znode, data);
-      }
-      return false;
+      ret = false;
     } catch (InterruptedException e) {
       LOG.info("Interrupted", e);
       Thread.currentThread().interrupt();
     }
-    return true;
+    if(!watchAndCheckExists(zkw, znode)) {
+      // It did exist but now it doesn't, try again
+      return createEphemeralNodeAndWatch(zkw, znode, data);
+    }
+    return ret;
   }
 
   /**
@@ -999,22 +1000,23 @@ public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
   public static boolean createNodeIfNotExistsAndWatch(
       ZooKeeperWatcher zkw, String znode, byte [] data)
   throws KeeperException {
+    boolean ret = true;
     try {
       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
           CreateMode.PERSISTENT);
     } catch (KeeperException.NodeExistsException nee) {
-      try {
-        zkw.getRecoverableZooKeeper().exists(znode, zkw);
-      } catch (InterruptedException e) {
-        zkw.interruptedException(e);
-        return false;
-      }
+      ret = false;
+    } catch (InterruptedException e) {
+      zkw.interruptedException(e);
       return false;
+    }
+    try {
+      zkw.getRecoverableZooKeeper().exists(znode, zkw);
     } catch (InterruptedException e) {
       zkw.interruptedException(e);
       return false;
     }
-    return true;
+    return ret;
   }
 
   /**

From 248d2e521bfbd6010baf3d88722977826fea90c5 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 3 Feb 2014 22:25:40 +0000
Subject: [PATCH 1310/1540] HBASE-7963 HBase VerifyReplication not working when
 security enabled

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1564109 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TableMapReduceUtil.java   | 28 ++++++++++-
 .../replication/VerifyReplication.java        | 48 ++++++++++++++-----
 2 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index cdca97ce5120..3d1db62e5387 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -289,7 +289,7 @@ public static void initTableMapperJob(List<Scan> scans,
     }
   }
 
-public static void initCredentials(Job job) throws IOException {
+  public static void initCredentials(Job job) throws IOException {
     UserProvider provider = UserProvider.instantiate(job.getConfiguration());
 
     if (provider.isHadoopSecurityEnabled()) {
@@ -323,6 +323,32 @@ public static void initCredentials(Job job) throws IOException {
       }
     }
   }
+  
+  /**
+   * Obtain an authentication token, for the specified cluster, on behalf of the current user
+   * and add it to the credentials for the given map reduce job.
+   *
+   * The quorumAddress is the key to the ZK ensemble, which contains:
+   * hbase.zookeeper.quorum, hbase.zookeeper.client.port and zookeeper.znode.parent
+   *
+   * @param job The job that requires the permission.
+   * @param quorumAddress string that contains the 3 required configuratins
+   * @throws IOException When the authentication token cannot be obtained.
+   */
+  public static void initCredentialsForCluster(Job job, String quorumAddress)
+      throws IOException {
+    UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
+    if (userProvider.isHBaseSecurityEnabled()) {
+      try {
+        Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
+        ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
+        obtainAuthTokenForJob(job, peerConf, userProvider.getCurrent());
+      } catch (InterruptedException e) {
+        LOG.info("Interrupted obtaining user authentication token");
+        Thread.interrupted();
+      }
+    }
+  }
 
   private static void obtainAuthTokenForJob(Job job, Configuration conf, User user)
       throws IOException, InterruptedException {
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index b7d540b0c4bb..8ad86d38e778 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hbase.replication.ReplicationPeer;
 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.apache.zookeeper.KeeperException;
@@ -110,17 +111,13 @@ public void map(ImmutableBytesWritable row, final Result value,
         HConnectionManager.execute(new HConnectable<Void>(conf) {
           @Override
           public Void connect(HConnection conn) throws IOException {
-            try {
-              ReplicationZookeeper zk = new ReplicationZookeeper(conn, conf,
-                  conn.getZooKeeperWatcher());
-              ReplicationPeer peer = zk.getPeer(conf.get(NAME+".peerId"));
-              HTable replicatedTable = new HTable(peer.getConfiguration(),
-                  conf.get(NAME+".tableName"));
-              scan.setStartRow(value.getRow());
-              replicatedScanner = replicatedTable.getScanner(scan);
-            } catch (KeeperException e) {
-              throw new IOException("Got a ZK exception", e);
-            }
+            String zkClusterKey = conf.get(NAME + ".peerQuorumAddress");
+            Configuration peerConf = HBaseConfiguration.create(conf);
+            ZKUtil.applyClusterKeyToConf(peerConf, zkClusterKey);
+
+            HTable replicatedTable = new HTable(peerConf, conf.get(NAME + ".tableName"));
+            scan.setStartRow(value.getRow());
+            replicatedScanner = replicatedTable.getScanner(scan);
             return null;
           }
         });
@@ -143,6 +140,26 @@ protected void cleanup(Context context) {
     }
   }
 
+  private static String getPeerQuorumAddress(final Configuration conf) throws IOException {
+    HConnection conn = HConnectionManager.getConnection(conf);
+    try {
+      ReplicationZookeeper zk = new ReplicationZookeeper(conn, conf,
+          conn.getZooKeeperWatcher());
+
+      ReplicationPeer peer = zk.getPeer(peerId);
+      if (peer == null) {
+        throw new IOException("Couldn't get peer conf!");
+      }
+
+      Configuration peerConf = peer.getConfiguration();
+      return ZKUtil.getZooKeeperClusterKey(peerConf);
+    } catch (KeeperException e) {
+      throw new IOException("Got a ZK exception", e);
+    } finally {
+      conn.close();
+    }
+  }
+
   /**
    * Sets up the actual job.
    *
@@ -185,6 +202,11 @@ public Void connect(HConnection conn) throws IOException {
     if (families != null) {
       conf.set(NAME+".families", families);
     }
+
+    String peerQuorumAddress = getPeerQuorumAddress(conf);
+    conf.set(NAME + ".peerQuorumAddress", peerQuorumAddress);
+    LOG.info("Peer Quorum Address: " + peerQuorumAddress);
+
     Job job = new Job(conf, NAME + "_" + tableName);
     job.setJarByClass(VerifyReplication.class);
 
@@ -201,6 +223,10 @@ public Void connect(HConnection conn) throws IOException {
     }
     TableMapReduceUtil.initTableMapperJob(tableName, scan,
         Verifier.class, null, null, job);
+
+    // Obtain the auth token from peer cluster
+    TableMapReduceUtil.initCredentialsForCluster(job, peerQuorumAddress);
+
     job.setOutputFormatClass(NullOutputFormat.class);
     job.setNumReduceTasks(0);
     return job;

From 356b7bbed1c3253591618d60714a2f85edf1b38e Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 4 Feb 2014 10:38:21 +0000
Subject: [PATCH 1311/1540] HBASE-10319 HLog should roll periodically to allow
 DN decommission to eventually complete

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1564249 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/LogRoller.java  |   5 +-
 .../regionserver/wal/TestLogRollPeriod.java   | 169 ++++++++++++++++++
 2 files changed, 172 insertions(+), 2 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 01ae295c818e..acdd62f043c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -91,8 +91,9 @@ public void run() {
       rollLock.lock(); // FindBugs UL_UNRELEASED_LOCK_EXCEPTION_PATH
       try {
         this.lastrolltime = now;
-        // This is array of actual region names.
-        byte [][] regionsToFlush = getWAL().rollWriter(rollLog.get());
+        // Force the roll if the logroll.period is elapsed or if a roll was requested.
+        // The returned value is an array of actual region names.
+        byte [][] regionsToFlush = getWAL().rollWriter(periodic || rollLog.get());
         if (regionsToFlush != null) {
           for (byte [] r: regionsToFlush) scheduleFlush(r);
         }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
new file mode 100644
index 000000000000..28dbdef933de
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests that verifies that the log is forced to be rolled every "hbase.regionserver.logroll.period"
+ */
+@Category(MediumTests.class)
+public class TestLogRollPeriod {
+  private static final Log LOG = LogFactory.getLog(TestLogRolling.class);
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final static long LOG_ROLL_PERIOD = 4000;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // disable the ui
+    TEST_UTIL.getConfiguration().setInt("hbase.regionsever.info.port", -1);
+
+    TEST_UTIL.getConfiguration().setLong("hbase.regionserver.logroll.period", LOG_ROLL_PERIOD);
+
+    TEST_UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Tests that the LogRoller perform the roll even if there are no edits
+   */
+  @Test
+  public void testNoEdits() throws Exception {
+    final byte[] tableName = Bytes.toBytes("TestLogRollPeriodNoEdits");
+
+    TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
+    try {
+      HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+      try {
+        HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(tableName);
+        HLog log = server.getWAL();
+        checkMinLogRolls(log, 5);
+      } finally {
+        table.close();
+      }
+    } finally {
+      TEST_UTIL.deleteTable(tableName);
+    }
+  }
+
+  /**
+   * Tests that the LogRoller perform the roll with some data in the log
+   */
+  @Test(timeout=60000)
+  public void testWithEdits() throws Exception {
+    final byte[] tableName = Bytes.toBytes("TestLogRollPeriodWithEdits");
+    final byte[] family = Bytes.toBytes("cf");
+
+    TEST_UTIL.createTable(tableName, family);
+    try {
+      HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(tableName);
+      HLog log = server.getWAL();
+      final HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+
+      Thread writerThread = new Thread("writer") {
+        @Override
+        public void run() {
+          try {
+            long row = 0;
+            while (!interrupted()) {
+              Put p = new Put(Bytes.toBytes(String.format("row%d", row)));
+              p.add(family, Bytes.toBytes("col"), Bytes.toBytes(row));
+              table.put(p);
+              row++;
+
+              Thread.sleep(LOG_ROLL_PERIOD / 16);
+            }
+          } catch (Exception e) {
+            LOG.warn(e);
+          } 
+        }
+      };
+
+      try {
+        writerThread.start();
+        checkMinLogRolls(log, 5);
+      } finally {
+        writerThread.interrupt();
+        writerThread.join();
+        table.close();
+      }  
+    } finally {
+      TEST_UTIL.deleteTable(tableName);
+    }
+  }
+
+  private void checkMinLogRolls(final HLog log, final int minRolls)
+      throws Exception {
+    final List<Path> paths = new ArrayList<Path>();
+    log.registerWALActionsListener(new WALActionsListener() {
+      @Override
+      public void preLogRoll(Path oldFile, Path newFile)  {}
+      @Override
+      public void postLogRoll(Path oldFile, Path newFile) {
+        LOG.debug("postLogRoll: oldFile="+oldFile+" newFile="+newFile);
+        paths.add(newFile);
+      }
+      @Override
+      public void preLogArchive(Path oldFile, Path newFile) {}
+      @Override
+      public void postLogArchive(Path oldFile, Path newFile) {}
+      @Override
+      public void logRollRequested() {}
+      @Override
+      public void logCloseRequested() {}
+      @Override
+      public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) {}
+      @Override
+      public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) {}
+    });
+
+    // Sleep until we should get at least min-LogRoll events
+    Thread.sleep((minRolls + 1) * LOG_ROLL_PERIOD);
+    assertFalse(paths.size() < minRolls);
+  }
+}

From 0be4a3ad1fdf03ddcf2e0efbe55dcce6a988cb75 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 4 Feb 2014 19:20:32 +0000
Subject: [PATCH 1312/1540] HBASE-10457 Print corrupted file information in
 SnapshotInfo tool without -file option (Bharath Vissapragada)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1564429 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/SnapshotInfo.java   | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index b3c4b80e54fe..81e2bbf67a1a 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -265,6 +265,7 @@ public int run(String[] args) throws IOException, InterruptedException {
           snapshotName = args[++i];
         } else if (cmd.equals("-files")) {
           showFiles = true;
+          showStats = true;
         } else if (cmd.equals("-stats")) {
           showStats = true;
         } else if (cmd.equals("-schema")) {
@@ -298,7 +299,7 @@ public int run(String[] args) throws IOException, InterruptedException {
 
     printInfo();
     if (showSchema) printSchema();
-    if (showFiles || showStats) printFiles(showFiles);
+    printFiles(showFiles, showStats);
 
     return 0;
   }
@@ -349,7 +350,7 @@ private void printSchema() {
    * Collect the hfiles and logs statistics of the snapshot and
    * dump the file list if requested and the collected information.
    */
-  private void printFiles(final boolean showFiles) throws IOException {
+  private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
     if (showFiles) {
       System.out.println("Snapshot Files");
       System.out.println("----------------------------------------");
@@ -404,15 +405,17 @@ public void logFile (final String server, final String logfile)
       System.out.println("**************************************************************");
     }
 
-    System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
-      stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
-      StringUtils.humanReadableInt(stats.getStoreFilesSize()),
-      stats.getSharedStoreFilePercentage(),
-      StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
-    );
-    System.out.printf("%d Logs, total size %s%n",
-      stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
-    System.out.println();
+    if (showStats) {
+      System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
+        stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
+        StringUtils.humanReadableInt(stats.getStoreFilesSize()),
+        stats.getSharedStoreFilePercentage(),
+        StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
+      );
+      System.out.printf("%d Logs, total size %s%n",
+        stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
+      System.out.println();
+    }
   }
 
   private void printUsageAndExit() {

From c7a15664e9b123f9f467d287acdc45426ffbc7bf Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 5 Feb 2014 21:24:23 +0000
Subject: [PATCH 1313/1540] HBASE-10473 Add utility for adorning http Context

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1564936 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/rest/Main.java    |  4 +-
 .../hadoop/hbase/util/HttpServerUtil.java     | 52 +++++++++++++++++++
 .../apache/hadoop/hbase/util/InfoServer.java  |  1 +
 .../hbase/rest/HBaseRESTTestingUtility.java   |  2 +
 4 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java

diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index 306abf48ec46..d17ac14cb7a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.UserProvider;
+import org.apache.hadoop.hbase.util.HttpServerUtil;
 import org.apache.hadoop.hbase.rest.filter.GzipFilter;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.InfoServer;
@@ -198,6 +199,7 @@ public static void main(String[] args) throws Exception {
     context.addServlet(shPojoMap, "/status/cluster");
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
+    HttpServerUtil.constrainHttpMethods(context);
 
     // Put up info server.
     int port = conf.getInt("hbase.rest.info.port", 8085);
@@ -213,4 +215,4 @@ public static void main(String[] args) throws Exception {
     server.start();
     server.join();
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java b/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java
new file mode 100644
index 000000000000..742d1a4e54c8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import org.mortbay.jetty.security.Constraint;
+import org.mortbay.jetty.security.ConstraintMapping;
+import org.mortbay.jetty.security.SecurityHandler;
+import org.mortbay.jetty.servlet.Context;
+
+/**
+ * HttpServer utility.
+ */
+public class HttpServerUtil {
+  /**
+   * Add constraints to a Jetty Context to disallow undesirable Http methods.
+   * @param context The context to modify
+   */
+  public static void constrainHttpMethods(Context context) {
+    Constraint c = new Constraint();
+    c.setAuthenticate(true);
+
+    ConstraintMapping cmt = new ConstraintMapping();
+    cmt.setConstraint(c);
+    cmt.setMethod("TRACE");
+    cmt.setPathSpec("/*");
+
+    ConstraintMapping cmo = new ConstraintMapping();
+    cmo.setConstraint(c);
+    cmo.setMethod("OPTIONS");
+    cmo.setPathSpec("/*");
+
+    SecurityHandler sh = new SecurityHandler();
+    sh.setConstraintMappings(new ConstraintMapping[]{ cmt, cmo });
+
+    context.addHandler(sh);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
index a1698daf252b..11e77dc52afd 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
@@ -90,6 +90,7 @@ private void fixupLogsServletLocation() {
           logsContextPath);
       logContext.setResourceBase(logDir);
       logContext.addServlet(DefaultServlet.class, "/");
+      HttpServerUtil.constrainHttpMethods(logContext);
       defaultContexts.put(logContext, true);
     }
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
index 6b723be2f1db..b4579edb68db 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
@@ -27,6 +27,7 @@
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.servlet.Context;
 import org.mortbay.jetty.servlet.ServletHolder;
+import org.apache.hadoop.hbase.util.HttpServerUtil;
 
 import com.sun.jersey.spi.container.servlet.ServletContainer;
 
@@ -68,6 +69,7 @@ public void startServletContainer(Configuration conf) throws Exception {
     Context context = new Context(server, "/", Context.SESSIONS);
     context.addServlet(sh, "/*");
     context.addFilter(GzipFilter.class, "/*", 0);
+    HttpServerUtil.constrainHttpMethods(context);
       // start the server
     server.start();
       // get the port

From 3f7f3ef6ad1a981d65f5665c0860098cc02a411c Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 5 Feb 2014 21:52:52 +0000
Subject: [PATCH 1314/1540] HBASE-10470 Import generates huge log file while
 importing large amounts of data. (Vasu Mariyala)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1564950 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/mapreduce/Import.java     | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index e5a797f81f93..1c9062d65965 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -247,13 +247,12 @@ private static KeyValue filterKv(KeyValue kv) {
     // apply the filter and skip this kv if the filter doesn't apply
     if (filter != null) {
       Filter.ReturnCode code = filter.filterKeyValue(kv);
-      System.out.println("Filter returned:" + code);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Filter returned:" + code + " for the key value:" + kv);
+      }
       // if its not an accept type, then skip this kv
       if (!(code.equals(Filter.ReturnCode.INCLUDE) || code
           .equals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL))) {
-        if (LOG.isDebugEnabled()) {
-          System.out.println("Skipping key: " + kv + " from filter decision: " + code);
-        }
         return null;
       }
     }

From 25b58d42c17e02ffcad2add82a59d6fe0915deed Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 7 Feb 2014 01:26:57 +0000
Subject: [PATCH 1315/1540] HBASE-10363 [0.94] TestInputSampler and
 TestInputSamplerTool fail under hadoop 2.0/23 profiles.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1565511 13f79535-47bb-0310-9956-ffa450edef68
---
 .../mapreduce/hadoopbackport/InputSampler.java      | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
index a556709a50d2..c998c1a6fa69 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
@@ -165,14 +165,21 @@ public K[] getSample(InputFormat<K,V> inf, Job job)
    */
   public static TaskAttemptContext getTaskAttemptContext(final Job job)
   throws IOException {
-    Constructor<TaskAttemptContext> c;
+    Constructor<?> c;
     try {
-      c = TaskAttemptContext.class.getConstructor(Configuration.class, TaskAttemptID.class);
+      if (TaskAttemptContext.class.isInterface()) {
+        // Hadoop 2.x
+        Class<?> clazz = Class.forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl");
+        c = clazz.getConstructor(Configuration.class, TaskAttemptID.class);
+      } else {
+        // Hadoop 1.x
+        c = TaskAttemptContext.class.getConstructor(Configuration.class, TaskAttemptID.class);
+      }
     } catch (Exception e) {
       throw new IOException("Failed getting constructor", e);
     }
     try {
-      return c.newInstance(job.getConfiguration(), new TaskAttemptID());
+      return (TaskAttemptContext)c.newInstance(job.getConfiguration(), new TaskAttemptID());
     } catch (Exception e) {
       throw new IOException("Failed creating instance", e);
     }

From 2488ab928a0cc9d4ae46b900f32b2307b6e80594 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 7 Feb 2014 19:53:57 +0000
Subject: [PATCH 1316/1540] HBASE-10480 TestLogRollPeriod#testWithEdits may
 fail due to insufficient waiting

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1565776 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/wal/TestLogRollPeriod.java      | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
index 28dbdef933de..a7d9ccb90a46 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
@@ -163,7 +163,17 @@ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEd
     });
 
     // Sleep until we should get at least min-LogRoll events
+    long wtime = System.currentTimeMillis();
     Thread.sleep((minRolls + 1) * LOG_ROLL_PERIOD);
+    // Do some extra sleep in case the machine is slow,
+    // and the log-roll is not triggered exactly on LOG_ROLL_PERIOD.
+    final int NUM_RETRIES = 1 + 8 * (minRolls - paths.size());
+    for (int retry = 0; paths.size() < minRolls && retry < NUM_RETRIES; ++retry) {
+      Thread.sleep(LOG_ROLL_PERIOD / 4);
+    }
+    wtime = System.currentTimeMillis() - wtime;
+    LOG.info(String.format("got %d rolls after %dms (%dms each) - expected at least %d rolls",
+                           paths.size(), wtime, wtime / paths.size(), minRolls));
     assertFalse(paths.size() < minRolls);
   }
 }

From 70573a27e2e72726a159fc7a5d87145a3b88ff39 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 7 Feb 2014 22:06:29 +0000
Subject: [PATCH 1317/1540] HBASE-10340 [BACKPORT] HBASE-9892 Add info port to
 ServerName to support multi instances in a node

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1565828 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  |   5 +-
 .../org/apache/hadoop/hbase/HConstants.java   |   3 +
 .../hadoop/hbase/LocalHBaseCluster.java       |   2 +
 .../apache/hadoop/hbase/master/HMaster.java   |  10 +
 .../hbase/master/HMasterCommandLine.java      |   2 +-
 .../hbase/protobuf/generated/HBaseProtos.java | 391 +++++++++++++++++-
 .../hbase/regionserver/HRegionServer.java     |  25 +-
 .../hbase/regionserver/RSDumpServlet.java     |   7 +
 .../hbase/regionserver/RSStatusServlet.java   |   7 +
 .../hbase/zookeeper/RegionServerTracker.java  |  34 +-
 src/main/protobuf/hbase.proto                 |   7 +
 .../resources/hbase-webapps/master/table.jsp  |  29 +-
 12 files changed, 482 insertions(+), 40 deletions(-)

diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 8683be6ecc8c..6af4025ec7f9 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -253,9 +253,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
    ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]);
      Arrays.sort(serverNames);
      for (ServerName serverName: serverNames) {
-       // TODO: this is incorrect since this conf might differ from RS to RS
-       // or be set to 0 to get ephemeral ports
-       int infoPort = master.getConfiguration().getInt("hbase.regionserver.info.port", 60030);
+       int infoPort = master.getRegionServerInfoPort(serverName);
        String url = "//" + serverName.getHostname() + ":" + infoPort + "/";
        HServerLoad hsl = master.getServerManager().getLoad(serverName);
        String loadStr = hsl == null? "-": hsl.toString();
@@ -286,7 +284,6 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
    ServerName [] deadServerNames = deadServers.toArray(new ServerName[deadServers.size()]);
      Arrays.sort(deadServerNames);
      for (ServerName deadServerName: deadServerNames) {
-       int infoPort = master.getConfiguration().getInt("hbase.regionserver.info.port", 60030);
 </%java>
 <tr><td><% deadServerName %></td></tr>
 <%java>
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index fa14c3707547..6821850820ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -167,6 +167,9 @@ public enum OperationStatusCode {
   /** Parameter name for port region server listens on. */
   public static final String REGIONSERVER_PORT = "hbase.regionserver.port";
 
+  /** Parameter name for port region server's info server listens on. */
+  public static final String REGIONSERVER_INFO_PORT = "hbase.regionserver.info.port";
+  
   /** Default port region server listens on. */
   public static final int DEFAULT_REGIONSERVER_PORT = 60020;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index fe4897a685d6..315309f94e67 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -140,6 +140,8 @@ public LocalHBaseCluster(final Configuration conf, final int noMasters,
     // clash over default ports.
     conf.set(HConstants.MASTER_PORT, "0");
     conf.set(HConstants.REGIONSERVER_PORT, "0");
+    conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
+
     this.masterClass = (Class<? extends HMaster>)
       conf.getClass(HConstants.MASTER_IMPL, masterClass);
     // Start the HMasters.
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 6be1beab259d..ec18ae937d7c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -100,6 +100,7 @@
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
@@ -1602,6 +1603,15 @@ public long getMasterActiveTime() {
     return masterActiveTime;
   }
 
+  public int getRegionServerInfoPort(final ServerName sn) {
+    RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
+    if (info == null || info.getInfoPort() == 0) {
+      return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
+        HConstants.DEFAULT_REGIONSERVER_INFOPORT);
+    }
+    return info.getInfoPort();
+  }
+  
   /**
    * @return array of coprocessor SimpleNames.
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index e6fcce54cc5e..70287bf8f9b6 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -143,7 +143,7 @@ private int startMaster() {
                  Integer.toString(clientPort));
         // Need to have the zk cluster shutdown when master is shutdown.
         // Run a subclass that does the zk cluster shutdown on its way out.
-        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1,
+        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 3,
                                                           LocalHMaster.class, HRegionServer.class);
         ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
         cluster.startup();
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
index 84abf0151d7f..e0ccb1c24a3b 100644
--- a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
@@ -803,11 +803,387 @@ public Builder clearVersion() {
     // @@protoc_insertion_point(class_scope:SnapshotDescription)
   }
   
+  public interface RegionServerInfoOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int32 infoPort = 1;
+    boolean hasInfoPort();
+    int getInfoPort();
+  }
+  public static final class RegionServerInfo extends
+      com.google.protobuf.GeneratedMessage
+      implements RegionServerInfoOrBuilder {
+    // Use RegionServerInfo.newBuilder() to construct.
+    private RegionServerInfo(Builder builder) {
+      super(builder);
+    }
+    private RegionServerInfo(boolean noInit) {}
+    
+    private static final RegionServerInfo defaultInstance;
+    public static RegionServerInfo getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public RegionServerInfo getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_RegionServerInfo_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_RegionServerInfo_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int32 infoPort = 1;
+    public static final int INFOPORT_FIELD_NUMBER = 1;
+    private int infoPort_;
+    public boolean hasInfoPort() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public int getInfoPort() {
+      return infoPort_;
+    }
+    
+    private void initFields() {
+      infoPort_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(1, infoPort_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(1, infoPort_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo other = (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo) obj;
+      
+      boolean result = true;
+      result = result && (hasInfoPort() == other.hasInfoPort());
+      if (hasInfoPort()) {
+        result = result && (getInfoPort()
+            == other.getInfoPort());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasInfoPort()) {
+        hash = (37 * hash) + INFOPORT_FIELD_NUMBER;
+        hash = (53 * hash) + getInfoPort();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfoOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_RegionServerInfo_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_RegionServerInfo_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        infoPort_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo build() {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo result = new org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.infoPort_ = infoPort_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo.getDefaultInstance()) return this;
+        if (other.hasInfoPort()) {
+          setInfoPort(other.getInfoPort());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              infoPort_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int32 infoPort = 1;
+      private int infoPort_ ;
+      public boolean hasInfoPort() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public int getInfoPort() {
+        return infoPort_;
+      }
+      public Builder setInfoPort(int value) {
+        bitField0_ |= 0x00000001;
+        infoPort_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearInfoPort() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        infoPort_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:RegionServerInfo)
+    }
+    
+    static {
+      defaultInstance = new RegionServerInfo(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:RegionServerInfo)
+  }
+  
   private static com.google.protobuf.Descriptors.Descriptor
     internal_static_SnapshotDescription_descriptor;
   private static
     com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_SnapshotDescription_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_RegionServerInfo_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_RegionServerInfo_fieldAccessorTable;
   
   public static com.google.protobuf.Descriptors.FileDescriptor
       getDescriptor() {
@@ -821,9 +1197,10 @@ public Builder clearVersion() {
       "\004name\030\001 \002(\t\022\r\n\005table\030\002 \001(\t\022\027\n\014creationTi" +
       "me\030\003 \001(\003:\0010\022.\n\004type\030\004 \001(\0162\031.SnapshotDesc" +
       "ription.Type:\005FLUSH\022\017\n\007version\030\005 \001(\005\"\037\n\004" +
-      "Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001B>\n*org.apa" +
-      "che.hadoop.hbase.protobuf.generatedB\013HBa" +
-      "seProtosH\001\240\001\001"
+      "Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001\"$\n\020RegionS" +
+      "erverInfo\022\020\n\010infoPort\030\001 \001(\005B>\n*org.apach" +
+      "e.hadoop.hbase.protobuf.generatedB\013HBase" +
+      "ProtosH\001\240\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -838,6 +1215,14 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors(
               new java.lang.String[] { "Name", "Table", "CreationTime", "Type", "Version", },
               org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.class,
               org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Builder.class);
+          internal_static_RegionServerInfo_descriptor =
+            getDescriptor().getMessageTypes().get(1);
+          internal_static_RegionServerInfo_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_RegionServerInfo_descriptor,
+              new java.lang.String[] { "InfoPort", },
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo.class,
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo.Builder.class);
           return null;
         }
       };
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9b918e2fe0d1..b580bdbf60ae 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -128,6 +128,8 @@
 import org.apache.hadoop.hbase.ipc.RpcEngine;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
@@ -356,8 +358,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
    */
   private ServerName serverNameFromMasterPOV;
 
-  // Port we put up the webui on.
-  private int webuiport = -1;
+  // region server static info like info port
+  private RegionServerInfo.Builder rsInfo;
 
   /**
    * This servers startcode.
@@ -483,6 +485,10 @@ public void uncaughtException(Thread t, Throwable e) {
         abort("Uncaught exception in service thread " + t.getName(), e);
       }
     };
+    this.rsInfo = RegionServerInfo.newBuilder();
+    // Put up the webui.  Webui may come up on port other than configured if
+    // that port is occupied. Adjust serverInfo if this is the case.
+    this.rsInfo.setInfoPort(putUpWebUI());
   }
 
   /** Handle all the snapshot requests to this server */
@@ -1126,9 +1132,10 @@ private String getMyEphemeralNodePath() {
     return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
   }
 
-  private void createMyEphemeralNode() throws KeeperException {
-    ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper, getMyEphemeralNodePath(),
-      HConstants.EMPTY_BYTE_ARRAY);
+  private void createMyEphemeralNode() throws KeeperException, IOException {
+    byte[] data = ProtobufUtil.prependPBMagic(rsInfo.build().toByteArray());
+    ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper,
+      getMyEphemeralNodePath(), data);
   }
 
   private void deleteMyEphemeralNode() throws KeeperException {
@@ -1710,10 +1717,6 @@ private void startServiceThreads() throws IOException {
     this.leases.setName(n + ".leaseChecker");
     this.leases.start();
 
-    // Put up the webui.  Webui may come up on port other than configured if
-    // that port is occupied. Adjust serverInfo if this is the case.
-    this.webuiport = putUpWebUI();
-
     if (this.replicationSourceHandler == this.replicationSinkHandler &&
         this.replicationSourceHandler != null) {
       this.replicationSourceHandler.startReplicationService();
@@ -1769,7 +1772,7 @@ private int putUpWebUI() throws IOException {
         port++;
       }
     }
-    return port;
+    return this.infoServer.getPort();
   }
 
   /*
@@ -3790,7 +3793,7 @@ public long incrementColumnValue(byte[] regionName, byte[] row,
   public HServerInfo getHServerInfo() throws IOException {
     checkOpen();
     return new HServerInfo(new HServerAddress(this.isa),
-      this.startcode, this.webuiport);
+      this.startcode, this.rsInfo.getInfoPort());
   }
 
   @SuppressWarnings("unchecked")
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
index a4ec6d389601..979cd5225716 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
@@ -50,6 +50,13 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     assert hrsconf != null : "No RS conf in context";
 
     response.setContentType("text/plain");
+ 
+    if (!hrs.isOnline()) {
+      response.getWriter().write("The RegionServer is initializing!");
+      response.getWriter().close();
+      return;
+    }
+
     OutputStream os = response.getOutputStream();
     PrintWriter out = new PrintWriter(os);
     
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
index 7521cd4c6e1f..824c79dcc177 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
@@ -40,6 +40,13 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
     assert hrs != null : "No RS in context!";
     
     resp.setContentType("text/html");
+    
+    if (!hrs.isOnline()) {
+      resp.getWriter().write("The RegionServer is initializing!");
+      resp.getWriter().close();
+      return;
+    }
+    
     RSStatusTmpl tmpl = new RSStatusTmpl();
     if (req.getParameter("format") != null)
       tmpl.setFormat(req.getParameter("format"));
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
index b3b21303f64e..b89306c625f8 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
@@ -22,14 +22,17 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
+import java.util.NavigableMap;
+import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -44,7 +47,8 @@
  */
 public class RegionServerTracker extends ZooKeeperListener {
   private static final Log LOG = LogFactory.getLog(RegionServerTracker.class);
-  private NavigableSet<ServerName> regionServers = new TreeSet<ServerName>();
+  private NavigableMap<ServerName, RegionServerInfo> regionServers = 
+		  new TreeMap<ServerName, RegionServerInfo>();
   private ServerManager serverManager;
   private Abortable abortable;
 
@@ -75,7 +79,23 @@ private void add(final List<String> servers) throws IOException {
       this.regionServers.clear();
       for (String n: servers) {
         ServerName sn = ServerName.parseServerName(ZKUtil.getNodeName(n));
-        this.regionServers.add(sn);
+        if (regionServers.get(sn) == null) {
+          RegionServerInfo.Builder rsInfoBuilder = RegionServerInfo.newBuilder();
+          try {
+            String nodePath = ZKUtil.joinZNode(watcher.rsZNode, n);
+            byte[] data = ZKUtil.getData(watcher, nodePath);
+            LOG.info("Rs node: " + nodePath + " data: " + Bytes.toString(data));
+            if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) {
+              int magicLen = ProtobufUtil.lengthOfPBMagic();
+              rsInfoBuilder.mergeFrom(data, magicLen, data.length - magicLen);
+            }
+          } catch (KeeperException e) {
+            LOG.warn("Get Rs info port from ephemeral node", e);
+          } catch (IOException e) {
+            LOG.warn("Illegal data from ephemeral node", e);
+          }
+          this.regionServers.put(sn, rsInfoBuilder.build());
+        }
       }
     }
   }
@@ -118,13 +138,17 @@ public void nodeChildrenChanged(String path) {
     }
   }
 
+  public RegionServerInfo getRegionServerInfo(final ServerName sn) {
+    return regionServers.get(sn);
+  }
+  
   /**
    * Gets the online servers.
    * @return list of online servers
    */
   public List<ServerName> getOnlineServers() {
     synchronized (this.regionServers) {
-      return new ArrayList<ServerName>(this.regionServers);
+      return new ArrayList<ServerName>(this.regionServers.keySet());
     }
   }
 }
diff --git a/src/main/protobuf/hbase.proto b/src/main/protobuf/hbase.proto
index 93d4232c699e..c3c97ad6d7ea 100644
--- a/src/main/protobuf/hbase.proto
+++ b/src/main/protobuf/hbase.proto
@@ -37,3 +37,10 @@ message SnapshotDescription {
   optional Type type = 4 [default = FLUSH];
   optional int32 version = 5;
 }
+
+/**
+ * Description of the region server info
+ */
+message RegionServerInfo {
+  optional int32 infoPort = 1;
+}
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index f44cbfb275b2..f5c85b57429a 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -18,7 +18,7 @@
  */
 --%>
 <%@ page contentType="text/html;charset=UTF-8"
-  import="java.util.HashMap"
+  import="java.util.TreeMap"
   import="org.apache.hadoop.io.Writable"
   import="org.apache.hadoop.conf.Configuration"
   import="org.apache.hadoop.hbase.client.HTable"
@@ -50,9 +50,6 @@
   if (showFragmentation) {
       frags = FSUtils.getTableFragmentation(master);
   }
-  // HARDCODED FOR NOW TODO: FIX GET FROM ZK
-  // This port might be wrong if RS actually ended up using something else.
-  int infoPort = conf.getInt("hbase.regionserver.info.port", 60030);
 %>
 
 <?xml version="1.0" encoding="UTF-8" ?>
@@ -118,7 +115,7 @@
 %>
 <%= tableHeader %>
 <%
-  String url = "//" + rl.getHostname() + ":" + infoPort + "/";
+  String url = "//" + rl.getHostname() + ":" + master.getRegionServerInfoPort(rl) + "/";
 %>
 <tr>
   <td><%= tableName %></td>
@@ -137,11 +134,11 @@
   HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
   ServerName metaLocation = master.getCatalogTracker().waitForMeta(1);
   for (int i = 0; i < 1; i++) {
-    String url = "//" + metaLocation.getHostname() + ":" + infoPort + "/";
+    String url = "//" + metaLocation.getHostname() + ":" + master.getRegionServerInfoPort(metaLocation) + "/";
 %>
 <tr>
   <td><%= meta.getRegionNameAsString() %></td>
-    <td><a href="<%= url %>"><%= metaLocation.getHostname().toString() + ":" + infoPort %></a></td>
+    <td><a href="<%= url %>"><%= metaLocation.getHostname().toString() + ":" + metaLocation.getPort() %></a></td>
     <td>-</td><td><%= Bytes.toString(meta.getStartKey()) %></td><td><%= Bytes.toString(meta.getEndKey()) %></td>
 </tr>
 <%  } %>
@@ -173,7 +170,7 @@
 <%  } %>
 </table>
 <%
-  Map<String, Integer> regDistribution = new HashMap<String, Integer>();
+  Map<ServerName, Integer> regDistribution = new TreeMap<ServerName, Integer>();
   Map<HRegionInfo, ServerName> regions = table.getRegionLocations();
   if(regions != null && regions.size() > 0) { %>
 <%=     tableHeader %>
@@ -192,21 +189,19 @@
         if (map.containsKey(regionInfo.getRegionName())) {
           req = map.get(regionInfo.getRegionName()).getRequestsCount();
         }
-        // This port might be wrong if RS actually ended up using something else.
-        regionServer = addr.getHostname().toString() + ":" + infoPort;
-        Integer i = regDistribution.get(regionServer);
+        Integer i = regDistribution.get(addr);
         if (null == i) i = new Integer(0);
-        regDistribution.put(regionServer, i+1);
+        regDistribution.put(addr, i+1);
       }
     }
 %>
 <tr>
   <td><%= Bytes.toStringBinary(regionInfo.getRegionName())%></td>
   <%
-  if (regionServer != null) {
+  if (addr != null) {
   %>
   <td>
-    <a href="<%= "//" + regionServer + "/" %>"><%= regionServer %></a>
+    <a href="<%= "//" + addr.getHostname() + ":" + master.getRegionServerInfoPort(addr) + "/" %>"><%= addr %></a>
   </td>
   <%
   } else {
@@ -224,10 +219,12 @@
 <h2>Regions by Region Server</h2>
 <table><tr><th>Region Server</th><th>Region Count</th></tr>
 <%
-  for (Map.Entry<String, Integer> rdEntry : regDistribution.entrySet()) {
+  for (Map.Entry<ServerName, Integer> rdEntry : regDistribution.entrySet()) {
+      ServerName addr = rdEntry.getKey();
+      String url = "//" + addr.getHostname() + ":" + master.getRegionServerInfoPort(addr) + "/";
 %>
 <tr>
-  <td><a href="<%= "//" + rdEntry.getKey() + "/" %>"><%= rdEntry.getKey() %></a></td>
+  <td><a href="<%= "//" + url %>"><%= rdEntry.getKey() %></a></td>
   <td><%= rdEntry.getValue()%></td>
 </tr>
 <% } %>

From 7783810f16bc31f3b61575f1d844115753cbd6d0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 10 Feb 2014 01:00:26 +0000
Subject: [PATCH 1318/1540] HBASE-10489 TestImportExport fails in 0.94 with
 Hadoop2.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1566484 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TestImportExport.java     | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index c0cd2f156933..569364c85c1f 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -31,7 +31,6 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
@@ -64,13 +63,13 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
-  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    cluster = UTIL.startMiniCluster();
+    UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
+    UTIL.getConfiguration().set("mapred.job.tracker", "local");
   }
 
   @AfterClass
@@ -110,8 +109,7 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR,
         "1000"
     };
-
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -129,7 +127,7 @@ public void testSimpleCase() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -158,7 +156,7 @@ public void testMetaExport() throws Exception {
     String EXPORT_TABLE = ".META.";
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
+        UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -187,7 +185,7 @@ public void testWithDeletes() throws Exception {
     p.add(FAMILYA, QUAL, now+4, QUAL);
     t.put(p);
 
-    Delete d = new Delete(ROW1, now+3, null);
+    Delete d = new Delete(ROW1, now+3);
     t.delete(d);
     d = new Delete(ROW1);
     d.deleteColumns(FAMILYA, QUAL, now+2);
@@ -200,7 +198,7 @@ public void testWithDeletes() throws Exception {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -224,7 +222,7 @@ public void testWithDeletes() throws Exception {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -268,7 +266,7 @@ public void testWithFilter() throws Exception {
     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
 
     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
+        UTIL.getConfiguration()), args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -287,7 +285,7 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
         "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -310,7 +308,7 @@ public void testWithFilter() throws Exception {
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
         OUTPUT_DIR, "1000" };
 
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 

From 1ca5d7a87572f008848a9be0cfbcf1e44eafa3bc Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Mon, 10 Feb 2014 23:36:19 +0000
Subject: [PATCH 1319/1540] HBASE-10482 ReplicationSyncUp doesn't clean up its
 ZK, needed for tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1566855 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/replication/regionserver/ReplicationSyncUp.java      | 1 +
 .../hadoop/hbase/replication/TestReplicationSyncUpTool.java    | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 3cfc1c28fe1b..351bc218bbc7 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -115,6 +115,7 @@ public boolean isAborted() {
     }
 
     manager.join();
+    zkw.close();
 
     return (0);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index 2541ef3850c7..978c436ca500 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -207,6 +207,7 @@ private void setupReplication() throws Exception {
   }
 
   private void putAndReplicateRows() throws Exception {
+    LOG.debug("putAndReplicateRows");
     // add rows to Master cluster,
     Put p;
 
@@ -260,6 +261,7 @@ private void putAndReplicateRows() throws Exception {
   }
 
   private void mimicSyncUpAfterDelete() throws Exception {
+    LOG.debug("mimicSyncUpAfterDelete");
     utility2.shutdownMiniHBaseCluster();
 
     List<Delete> list = new ArrayList<Delete>();
@@ -329,6 +331,7 @@ private void mimicSyncUpAfterDelete() throws Exception {
   }
 
   private void mimicSyncUpAfterPut() throws Exception {
+    LOG.debug("mimicSyncUpAfterPut");
     utility1.restartHBaseCluster(1);
     utility2.shutdownMiniHBaseCluster();
 

From fbe636a54d631f69b4db7c257ea3a0232db9c2d1 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 11 Feb 2014 22:32:36 +0000
Subject: [PATCH 1320/1540] HBASE-10493 InclusiveStopFilter#filterKeyValue()
 should perform filtering on row key

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1567426 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/filter/InclusiveStopFilter.java     |  6 +++
 .../hadoop/hbase/filter/TestFilterList.java   | 39 +++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
index afa31c57e4f3..0f8740b90094 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
@@ -53,6 +53,12 @@ public byte[] getStopRowKey() {
     return this.stopRowKey;
   }
 
+  @Override
+  public ReturnCode filterKeyValue(KeyValue v) {
+    if (done) return ReturnCode.NEXT_ROW;
+    return ReturnCode.INCLUDE;
+  }
+
   public boolean filterRowKey(byte[] buffer, int offset, int length) {
     if (buffer == null) {
       //noinspection RedundantIfStatement
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 49dc9699bfbe..968250904abc 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -19,6 +19,8 @@
  */
 package org.apache.hadoop.hbase.filter;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInput;
@@ -205,6 +207,43 @@ public void testOrdering() throws Exception {
     }
   }
 
+  public static class AlwaysNextColFilter extends FilterBase {
+    public AlwaysNextColFilter() {
+      super();
+    }
+    @Override
+    public ReturnCode filterKeyValue(KeyValue v) {
+      return ReturnCode.NEXT_COL;
+    }
+    @Override
+    public void readFields(DataInput arg0) throws IOException {}
+
+    @Override
+    public void write(DataOutput arg0) throws IOException {}
+  }
+
+  /**
+   * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters
+   * we expect to get the same result as the inclusive stop result.
+   * @throws Exception
+   */
+  public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
+    byte[] r1 = Bytes.toBytes("Row1");
+    byte[] r11 = Bytes.toBytes("Row11");
+    byte[] r2 = Bytes.toBytes("Row2");
+  
+    FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+    flist.addFilter(new AlwaysNextColFilter());
+    flist.addFilter(new InclusiveStopFilter(r1));
+    flist.filterRowKey(r1, 0, r1.length);
+    assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
+    assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
+
+    flist.reset();
+    flist.filterRowKey(r2, 0, r2.length);
+    assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
+  }
+
   /**
    * Test serialization
    * @throws Exception

From e115b5125a4cfa0b2fa82c458a3d3380799d7661 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 11 Feb 2014 23:34:23 +0000
Subject: [PATCH 1321/1540] HBASE-10485 PrefixFilter#filterKeyValue() should
 perform filtering on row key (Ted Yu and LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1567461 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/PrefixFilter.java     |  6 ++++
 .../hadoop/hbase/filter/TestFilterList.java   | 36 +++++++++++++++++--
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
index 59e3ad3337dd..4e93925f6890 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import com.google.common.base.Preconditions;
@@ -66,6 +67,11 @@ public boolean filterRowKey(byte[] buffer, int offset, int length) {
     return filterRow;
   }
 
+  @Override
+  public ReturnCode filterKeyValue(KeyValue ignored) {
+    return filterRow ? ReturnCode.NEXT_ROW : ReturnCode.INCLUDE;
+  }
+
   public boolean filterRow() {
     return filterRow;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 968250904abc..7a25661817c9 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -207,7 +207,7 @@ public void testOrdering() throws Exception {
     }
   }
 
-  public static class AlwaysNextColFilter extends FilterBase {
+  private static class AlwaysNextColFilter extends FilterBase {
     public AlwaysNextColFilter() {
       super();
     }
@@ -231,7 +231,7 @@ public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
     byte[] r1 = Bytes.toBytes("Row1");
     byte[] r11 = Bytes.toBytes("Row11");
     byte[] r2 = Bytes.toBytes("Row2");
-  
+
     FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
     flist.addFilter(new AlwaysNextColFilter());
     flist.addFilter(new InclusiveStopFilter(r1));
@@ -244,6 +244,38 @@ public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
     assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
   }
 
+  /**
+   * When we do a "MUST_PASS_ONE" (a logical 'OR') of the above two filters
+   * we expect to get the same result as the 'prefix' only result.
+   * @throws Exception
+   */
+  public void testFilterListTwoFiltersMustPassOne() throws Exception {
+  byte[] r1 = Bytes.toBytes("Row1");
+    byte[] r11 = Bytes.toBytes("Row11");
+    byte[] r2 = Bytes.toBytes("Row2");
+
+    FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+    flist.addFilter(new PrefixFilter(r1));
+    flist.filterRowKey(r1, 0, r1.length);
+    assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
+    assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
+
+    flist.reset();
+    flist.filterRowKey(r2, 0, r2.length);
+    assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
+
+    flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+    flist.addFilter(new AlwaysNextColFilter());
+    flist.addFilter(new PrefixFilter(r1));
+    flist.filterRowKey(r1, 0, r1.length);
+    assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
+    assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
+
+    flist.reset();
+    flist.filterRowKey(r2, 0, r2.length);
+    assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
+  }
+
   /**
    * Test serialization
    * @throws Exception

From b3190c676cb80eb877c4bb567ad2ef1f77f51654 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 11 Feb 2014 23:59:39 +0000
Subject: [PATCH 1322/1540] HBASE-10481 API Compatibility JDiff script does not
 properly handle arguments in reverse order (Aleksandr Shulman)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1567472 13f79535-47bb-0310-9956-ffa450edef68
---
 dev-support/jdiffHBasePublicAPI.sh | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/dev-support/jdiffHBasePublicAPI.sh b/dev-support/jdiffHBasePublicAPI.sh
index 15d116d8d4d2..b6adbbfed5bc 100644
--- a/dev-support/jdiffHBasePublicAPI.sh
+++ b/dev-support/jdiffHBasePublicAPI.sh
@@ -67,6 +67,11 @@ set -e
 #   >  ./jdiffHBasePublicAPI.sh /home/aleks/stable_hbase/hbase stable_95 /home/aleks/exp_hbase/hbase experiment_95
 #
 #
+################################################## NOTE ON USAGE ###################################################
+#
+#   1. When using this tool, please specify the initial version first and the current version second. The semantics
+#      do not make sense otherwise. For example: jdiff 94 95 is good. jdiff 95 94 is bad
+#
 ############################################# READING A JDIFF REPORT ###############################################
 #
 #   The purpose of the JDiff report is show things that have changed between two versions of the public API. A user
@@ -131,8 +136,8 @@ fi
 #If the JDIFF_WORKING_DIRECTORY is set, then we will output the report there. Otherwise, to the default location
 if [[ "$JDIFF_WORKING_DIRECTORY" = "" ]]; then
 
-  echo "JDIFF_WORKING_DIRECTORY not set. That's not an issue. We will default it to ./jidff"
   JDIFF_WORKING_DIRECTORY=/tmp/jdiff
+  echo "JDIFF_WORKING_DIRECTORY not set. That's not an issue. We will default it to $JDIFF_WORKING_DIRECTORY."
 else
   echo "JDIFF_WORKING_DIRECTORY set to $JDIFF_WORKING_DIRECTORY";
 fi
@@ -204,6 +209,13 @@ if  [[ "$C_FORMAT" = "new" ]]; then
   fi
 
 else
+
+  if [[ "P_FORMAT" != "old" ]]; then
+    echo "When using this tool, please specify the initial version first and the current version second. They should be in ascending chronological order.
+          The semantics do not make sense otherwise. For example: jdiff 94 95 is good. jdiff 95 94 is bad."
+    echo "Exiting the script."
+    exit 5;
+  fi
   templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_template.xml
   echo "Both formats are using the 94 and earlier style directory format. We'll be using template $templateFile"
 fi

From 08d82b93c09f107dd77ebedd177ad8761312e897 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 12 Feb 2014 04:41:47 +0000
Subject: [PATCH 1323/1540] HBASE-10505 Import.filterKv does not call
 Filter.filterRowKey.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1567522 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/mapreduce/Import.java | 66 ++++++++++---------
 1 file changed, 36 insertions(+), 30 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 1c9062d65965..433eaca7aa69 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -88,12 +88,14 @@ public void map(ImmutableBytesWritable row, Result value,
       Context context)
     throws IOException {
       try {
-        for (KeyValue kv : value.raw()) {
-          kv = filterKv(kv);
-          // skip if we filtered it out
-          if (kv == null) continue;
-
-          context.write(row, convertKv(kv, cfRenameMap));
+        if (filter == null || !filter.filterRowKey(row.get(), row.getOffset(), row.getLength())) {
+          for (KeyValue kv : value.raw()) {
+            kv = filterKv(kv);
+            // skip if we filtered it out
+            if (kv == null) continue;
+  
+            context.write(row, convertKv(kv, cfRenameMap));
+          }
         }
       } catch (InterruptedException e) {
         e.printStackTrace();
@@ -138,32 +140,34 @@ private void writeResult(ImmutableBytesWritable key, Result result, Context cont
     throws IOException, InterruptedException {
       Put put = null;
       Delete delete = null;
-      for (KeyValue kv : result.raw()) {
-        kv = filterKv(kv);
-        // skip if we filter it out
-        if (kv == null) continue;
-
-        kv = convertKv(kv, cfRenameMap);
-        // Deletes and Puts are gathered and written when finished
-        if (kv.isDelete()) {
-          if (delete == null) {
-            delete = new Delete(key.get());
-          }
-          delete.addDeleteMarker(kv);
-        } else {
-          if (put == null) { 
-            put = new Put(key.get());
+      if (filter == null || !filter.filterRowKey(key.get(), key.getOffset(), key.getLength())) {
+        for (KeyValue kv : result.raw()) {
+          kv = filterKv(kv);
+          // skip if we filter it out
+          if (kv == null) continue;
+  
+          kv = convertKv(kv, cfRenameMap);
+          // Deletes and Puts are gathered and written when finished
+          if (kv.isDelete()) {
+            if (delete == null) {
+              delete = new Delete(key.get());
+            }
+            delete.addDeleteMarker(kv);
+          } else {
+            if (put == null) { 
+              put = new Put(key.get());
+            }
+            put.add(kv);
           }
-          put.add(kv);
         }
-      }
-      if (put != null) {
-        put.setClusterId(clusterId);
-        context.write(key, put);
-      }
-      if (delete != null) {
-        delete.setClusterId(clusterId);
-        context.write(key, delete);
+        if (put != null) {
+          put.setClusterId(clusterId);
+          context.write(key, put);
+        }
+        if (delete != null) {
+          delete.setClusterId(clusterId);
+          context.write(key, delete);
+        }
       }
     }
 
@@ -427,6 +431,8 @@ private static void usage(final String errorMsg) {
     System.err.println("  -D" + FILTER_ARGS_CONF_KEY + "=<comma separated list of args for filter");
     System.err.println(" NOTE: The filter will be applied BEFORE doing key renames via the "
         + CF_RENAME_PROP + " property. Futher, filters will only use the"
+        + " Filter#filterRowKey(byte[] buffer, int offset, int length) method to identify "
+        + " whether the current row needs to be ignored completely for processing and "
         + "Filter#filterKeyValue(KeyValue) method to determine if the KeyValue should be added;"
         + " Filter.ReturnCode#INCLUDE and #INCLUDE_AND_NEXT_COL will be considered as including "
         + "the KeyValue.");

From 5fb607236f55648509804e93c330e45da8eada02 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 12 Feb 2014 17:02:35 +0000
Subject: [PATCH 1324/1540] HBASE-10508 Backport HBASE-10365 'HBaseFsck should
 clean up connection properly when repair is completed' to 0.94 and 0.96

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1567678 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 83 ++++++++++---------
 1 file changed, 44 insertions(+), 39 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 08cf82a08492..8d08f0eb73e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -97,6 +97,7 @@
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
@@ -3863,49 +3864,53 @@ public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperExceptio
     // do the real work of hbck
     connect();
 
-    // if corrupt file mode is on, first fix them since they may be opened later
-    if (checkCorruptHFiles || sidelineCorruptHFiles) {
-      LOG.info("Checking all hfiles for corruption");
-      HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
-      setHFileCorruptionChecker(hfcc); // so we can get result
-      Collection<String> tables = getIncludedTables();
-      Collection<Path> tableDirs = new ArrayList<Path>();
-      Path rootdir = FSUtils.getRootDir(getConf());
-      if (tables.size() > 0) {
-        for (String t : tables) {
-          tableDirs.add(FSUtils.getTablePath(rootdir, t));
+    try {
+      // if corrupt file mode is on, first fix them since they may be opened later
+      if (checkCorruptHFiles || sidelineCorruptHFiles) {
+        LOG.info("Checking all hfiles for corruption");
+        HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
+        setHFileCorruptionChecker(hfcc); // so we can get result
+        Collection<String> tables = getIncludedTables();
+        Collection<Path> tableDirs = new ArrayList<Path>();
+        Path rootdir = FSUtils.getRootDir(getConf());
+        if (tables.size() > 0) {
+          for (String t : tables) {
+            tableDirs.add(FSUtils.getTablePath(rootdir, t));
+          }
+        } else {
+          tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
         }
-      } else {
-        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
+        hfcc.checkTables(tableDirs);
+        hfcc.report(errors);
       }
-      hfcc.checkTables(tableDirs);
-      hfcc.report(errors);
-    }
 
-    // check and fix table integrity, region consistency.
-    int code = onlineHbck();
-    setRetCode(code);
-    // If we have changed the HBase state it is better to run hbck again
-    // to see if we haven't broken something else in the process.
-    // We run it only once more because otherwise we can easily fall into
-    // an infinite loop.
-    if (shouldRerun()) {
-      try {
-        LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
-        Thread.sleep(sleepBeforeRerun);
-      } catch (InterruptedException ie) {
-        return this;
-      }
-      // Just report
-      setFixAssignments(false);
-      setFixMeta(false);
-      setFixHdfsHoles(false);
-      setFixHdfsOverlaps(false);
-      setFixVersionFile(false);
-      setFixTableOrphans(false);
-      errors.resetErrors();
-      code = onlineHbck();
+      // check and fix table integrity, region consistency.
+      int code = onlineHbck();
       setRetCode(code);
+      // If we have changed the HBase state it is better to run hbck again
+      // to see if we haven't broken something else in the process.
+      // We run it only once more because otherwise we can easily fall into
+      // an infinite loop.
+      if (shouldRerun()) {
+        try {
+          LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
+          Thread.sleep(sleepBeforeRerun);
+        } catch (InterruptedException ie) {
+          return this;
+        }
+        // Just report
+        setFixAssignments(false);
+        setFixMeta(false);
+        setFixHdfsHoles(false);
+        setFixHdfsOverlaps(false);
+        setFixVersionFile(false);
+        setFixTableOrphans(false);
+        errors.resetErrors();
+        code = onlineHbck();
+        setRetCode(code);
+      }
+    } finally {
+      IOUtils.cleanup(null, connection, meta, admin);
     }
     return this;
   }

From f6ac10f248b513231381945eeddb50fcf1533c1a Mon Sep 17 00:00:00 2001
From: liangxie <liangxie@unknown>
Date: Thu, 13 Feb 2014 03:47:23 +0000
Subject: [PATCH 1325/1540] HBASE-10506 Fail-fast if client connection is lost
 before the real call be executed in RPC layer

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1567845 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 9291734bc10d..159388698ad1 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -1397,6 +1397,12 @@ public void run() {
           status.pause("Waiting for a call");
           Call call = myCallQueue.take(); // pop the queue; maybe blocked here
           updateCallQueueLenMetrics(myCallQueue);
+          if (!call.connection.channel.isOpen()) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(Thread.currentThread().getName() + ": skipped " + call);
+            }
+            continue;
+          }
           status.setStatus("Setting up call");
           status.setConnection(call.connection.getHostAddress(), 
               call.connection.getRemotePort());

From cf49bb0895a849319c045f98b2714388de16a100 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 13 Feb 2014 22:59:22 +0000
Subject: [PATCH 1326/1540] CHANGES.txt and pom.xml for 0.94.17RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1568095 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 40 ++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 266c3e69b5dc..a97d4252952b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,44 @@
 HBase Change Log
+Release 0.94.17 - 02/13/2014
+Bug
+
+    [HBASE-7963] - HBase VerifyReplication not working when security enabled
+    [HBASE-10249] - TestReplicationSyncUpTool fails because failover takes too long
+    [HBASE-10274] - MiniZookeeperCluster should close ZKDatabase when shutdown ZooKeeperServers
+    [HBASE-10319] - HLog should roll periodically to allow DN decommission to eventually complete.
+    [HBASE-10320] - Avoid ArrayList.iterator() ExplicitColumnTracker
+    [HBASE-10335] - AuthFailedException in zookeeper may block replication forever
+    [HBASE-10340] - [BACKPORT] HBASE-9892 Add info port to ServerName to support multi instances in a node
+    [HBASE-10363] - [0.94] TestInputSampler and TestInputSamplerTool fail under hadoop 2.0/23 profiles.
+    [HBASE-10371] - Compaction creates empty hfile, then selects this file for compaction and creates empty hfile and over again
+    [HBASE-10383] - Secure Bulk Load for 'completebulkload' fails for version 0.94.15
+    [HBASE-10400] - [hbck] Continue if region dir missing on region merge attempt
+    [HBASE-10401] - [hbck] perform overlap group merges in parallel
+    [HBASE-10448] - ZKUtil create and watch methods don't set watch in some cases
+    [HBASE-10470] - Import generates huge log file while importing large amounts of data
+    [HBASE-10481] - API Compatibility JDiff script does not properly handle arguments in reverse order
+    [HBASE-10482] - ReplicationSyncUp doesn't clean up its ZK, needed for tests
+    [HBASE-10485] - PrefixFilter#filterKeyValue() should perform filtering on row key
+    [HBASE-10489] - TestImportExport fails in 0.94 with Hadoop2
+    [HBASE-10493] - InclusiveStopFilter#filterKeyValue() should perform filtering on row key
+    [HBASE-10505] - Import.filterKv does not call Filter.filterRowKey
+    [HBASE-10506] - Fail-fast if client connection is lost before the real call be executed in RPC layer
+    [HBASE-10508] - Backport HBASE-10365 'HBaseFsck should clean up connection properly when repair is completed' to 0.94 and 0.96
+
+Improvement
+
+    [HBASE-10423] - Report back the message of split or rollback failure to the master
+    [HBASE-10457] - Print corrupted file information in SnapshotInfo tool without -file option
+
+Task
+
+    [HBASE-10473] - Add utility for adorning http Context
+
+Test
+
+    [HBASE-10480] - TestLogRollPeriod#testWithEdits may fail due to insufficient waiting
+
+
 Release 0.94.16 - 01/10/2014
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index d5524de297fc..078bd7acb5f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.17-SNAPSHOT</version>
+  <version>0.94.17</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 23050039e4f4d63fb044043a7f86fe9715fd33a2 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sun, 16 Feb 2014 02:03:09 +0000
Subject: [PATCH 1327/1540] HBASE-10545 RS Hangs waiting on region to close on
 shutdown; has to timeout before can go down

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1568710 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/HRegionServer.java     | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b580bdbf60ae..ac6aadb54687 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -779,13 +779,14 @@ public void run() {
           } else if (this.stopping) {
             boolean allUserRegionsOffline = areAllUserRegionsOffline();
             if (allUserRegionsOffline) {
-              // Set stopped if no requests since last time we went around the loop.
-              // The remaining meta regions will be closed on our way out.
-              if (oldRequestCount == this.requestCount.get()) {
+              // Set stopped if no more write requests tp meta tables
+              // since last time we went around the loop.  Any open
+              // meta regions will be closed on our way out.
+              if (oldRequestCount == getWriteRequestCount()) {
                 stop("Stopped; only catalog regions remaining online");
                 break;
               }
-              oldRequestCount = this.requestCount.get();
+              oldRequestCount = getWriteRequestCount();
             } else {
               // Make sure all regions have been closed -- some regions may
               // have not got it because we were splitting at the time of
@@ -927,6 +928,17 @@ private boolean areAllUserRegionsOffline() {
     return allUserRegionsOffline;
   }
 
+  /**
+   * @return Current write count for all online regions.
+   */
+  private long getWriteRequestCount() {
+    int writeCount = 0;
+    for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
+      writeCount += e.getValue().getWriteRequestsCount();
+    }
+    return writeCount;
+  }
+
   void tryRegionServerReport()
   throws IOException {
     if (!keepLooping() && hbaseMaster == null) {

From 3d46d5481faff361f9cbf1872b1d2d6fc07ce7b8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 16 Feb 2014 20:34:03 +0000
Subject: [PATCH 1328/1540] HBASE-10551 Change local mode back to one RS thread
 by default

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1568822 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/HMasterCommandLine.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index 70287bf8f9b6..e6fcce54cc5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -143,7 +143,7 @@ private int startMaster() {
                  Integer.toString(clientPort));
         // Need to have the zk cluster shutdown when master is shutdown.
         // Run a subclass that does the zk cluster shutdown on its way out.
-        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 3,
+        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1,
                                                           LocalHMaster.class, HRegionServer.class);
         ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
         cluster.startup();

From 941e9cbb445539ad619fa96a8f791cb90df83a6a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 17 Feb 2014 22:24:20 +0000
Subject: [PATCH 1329/1540] HBASE-10212 New rpc metric: number of active
 handler. (Chao Shi)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569122 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java     | 2 ++
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java    | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
index bc897d970123..c6cf6b31fd2a 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
@@ -100,6 +100,8 @@ public HBaseRpcMetrics(String hostName, String port) {
       new MetricsTimeVaryingRate("RpcSlowResponse", registry);
   public final MetricsIntValue replicationCallQueueLen =
     new MetricsIntValue("replicationCallQueueLen", registry);
+  public final MetricsIntValue activeRpcCount =
+      new MetricsIntValue("activeRpcCount", registry);
 
   private void initMethods(Class<? extends VersionedProtocol> protocol) {
     for (Method m : protocol.getDeclaredMethods()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 159388698ad1..d0d472f3975a 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -222,6 +222,7 @@ public static String getRemoteAddress() {
   protected BlockingQueue<Call> callQueue; // queued calls
   protected final Counter callQueueSize = new Counter();
   protected BlockingQueue<Call> priorityCallQueue;
+  private final Counter activeRpcCount = new Counter();
 
   protected int highPriorityLevel;  // what level a high priority call is at
 
@@ -1417,6 +1418,7 @@ public void run() {
 
           CurCall.set(call);
           try {
+            activeRpcCount.increment();
             if (!started)
               throw new ServerNotRunningYetException("Server is not running yet");
 
@@ -1439,6 +1441,8 @@ public void run() {
             // Must always clear the request context to avoid leaking
             // credentials between requests.
             RequestContext.clear();
+            activeRpcCount.decrement();
+            rpcMetrics.activeRpcCount.set((int) activeRpcCount.get());
           }
           CurCall.set(null);
           callQueueSize.add(call.getSize() * -1);

From 20c4b8a68511b9739687fd602cce40b9908ca3c8 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 17 Feb 2014 22:28:10 +0000
Subject: [PATCH 1330/1540] HBASE-10555 Backport HBASE-8519 to 0.94, Backup
 master will never come up if primary master dies during initialization
 (Jingcheng Du, original patch by Jerry He)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569123 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/ActiveMasterManager.java | 16 +++++++++++++---
 .../org/apache/hadoop/hbase/master/HMaster.java  |  3 +--
 .../hbase/master/TestActiveMasterManager.java    |  9 ++++-----
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 2d2e5867ad7f..198224aa0d5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -49,6 +49,7 @@ public class ActiveMasterManager extends ZooKeeperListener {
   private static final Log LOG = LogFactory.getLog(ActiveMasterManager.class);
 
   final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false);
+  final AtomicBoolean clusterShutDown = new AtomicBoolean(false);
 
   private final ServerName sn;
   private final Server master;
@@ -73,6 +74,16 @@ public void nodeCreated(String path) {
 
   @Override
   public void nodeDeleted(String path) {
+    // We need to keep track of the cluster's shutdown status while
+    // we wait on the current master. We consider that, if the cluster
+    // was already in a "shutdown" state when we started, that this master
+    // is part of a new cluster that was started shortly after the old cluster
+    // shut down, so that state is now irrelevant. This means that the shutdown
+    // state must be set while we wait on the active master in order
+    // to shutdown this master. See HBASE-8519.
+    if (path.equals(watcher.clusterStateZNode) && !master.isStopped()) {
+      clusterShutDown.set(true);
+    }
     if(path.equals(watcher.masterAddressZNode) && !master.isStopped()) {
       handleMasterNodeChange();
     }
@@ -125,8 +136,7 @@ private void handleMasterNodeChange() {
    * master was running or if some other problem (zookeeper, stop flag has been
    * set on this Master)
    */
-  boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
-    ClusterStatusTracker clusterStatusTracker) {
+  boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus) {
     while (true) {
       startupStatus.setStatus("Trying to register in ZK as active master");
       // Try to become the active master, watch if there is another master.
@@ -199,7 +209,7 @@ boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
             LOG.debug("Interrupted waiting for master to die", e);
           }
         }
-        if (!clusterStatusTracker.isClusterUp()) {
+        if (clusterShutDown.get()) {
           this.master.stop("Cluster went down before this master became active");
         }
         if (this.master.isStopped()) {
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index ec18ae937d7c..82fd1f65d3e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -488,8 +488,7 @@ private boolean becomeActiveMaster(MonitoredTask startupStatus)
     // to check if the cluster should be shutdown.
     this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
     this.clusterStatusTracker.start();
-    return this.activeMasterManager.blockUntilBecomingActiveMaster(startupStatus,
-        this.clusterStatusTracker);
+    return this.activeMasterManager.blockUntilBecomingActiveMaster(startupStatus);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
index 05f6b1aec7ad..0f5f41c69405 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -84,7 +84,7 @@ public static void tearDownAfterClass() throws Exception {
     MonitoredTask status = Mockito.mock(MonitoredTask.class);
     clusterStatusTracker.setClusterUp();
 
-    activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker);
+    activeMasterManager.blockUntilBecomingActiveMaster(status);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, master);
 
@@ -93,7 +93,7 @@ public static void tearDownAfterClass() throws Exception {
     ActiveMasterManager secondActiveMasterManager =
       secondDummyMaster.getActiveMasterManager();
     assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
-    activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker);
+    activeMasterManager.blockUntilBecomingActiveMaster(status);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, master);
   }
@@ -129,7 +129,7 @@ public void testActiveMasterManagerFromZK() throws Exception {
       ms1.getClusterStatusTracker();
     clusterStatusTracker.setClusterUp();
     activeMasterManager.blockUntilBecomingActiveMaster(
-        Mockito.mock(MonitoredTask.class),clusterStatusTracker);
+        Mockito.mock(MonitoredTask.class));
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, firstMasterAddress);
 
@@ -211,8 +211,7 @@ public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
     @Override
     public void run() {
       manager.blockUntilBecomingActiveMaster(
-          Mockito.mock(MonitoredTask.class),
-          this.dummyMaster.getClusterStatusTracker());
+          Mockito.mock(MonitoredTask.class));
       LOG.info("Second master has become the active master!");
       isActiveMaster = true;
     }

From fe0b317d2925d91a91f52bd3c6e70246f008380e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 17 Feb 2014 22:35:48 +0000
Subject: [PATCH 1331/1540] HBASE-10552
 HFilePerformanceEvaluation.GaussianRandomReadBenchmark fails sometimes.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569129 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/HFilePerformanceEvaluation.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
index a3b7ed99851b..47a1d7e8e201 100644
--- a/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
@@ -355,7 +355,8 @@ void doRow(int i) throws Exception {
     private byte [] getGaussianRandomRowBytes() {
       int r = (int) randomData.nextGaussian((double)totalRows / 2.0,
           (double)totalRows / 10.0);
-      return format(r);
+      // make sure r falls into [0,totalRows)
+      return format(Math.min(totalRows, Math.max(r,0)));
     }
   }
 

From 978e5ae4c650a3b3bc794226d9917284755cbf65 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 18 Feb 2014 02:23:54 +0000
Subject: [PATCH 1332/1540] HBASE-10562 Fix TestMultiTableInputFormat for
 Hadoop 2 in 0.94.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569165 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index de3e132cce4f..f56aba1986a1 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1453,6 +1453,7 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
     c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
     /* this for mrv2 support */
     conf.set("mapreduce.framework.name", "yarn");
+    conf.setBoolean("yarn.is.minicluster", true);
     String rmAdress = mrClusterJobConf.get("yarn.resourcemanager.address");
     if (rmAdress != null) {
       conf.set("yarn.resourcemanager.address", rmAdress);

From 4052cd87da7c78e768658c22057468c634a220e7 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 18 Feb 2014 19:12:58 +0000
Subject: [PATCH 1333/1540] HBASE-10539 HRegion.addAndGetGlobalMemstoreSize
 returns previous size

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569480 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 19f54c175f92..48a6d728f7d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -779,7 +779,7 @@ public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     if (this.rsAccounting != null) {
       rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
     }
-    return this.memstoreSize.getAndAdd(memStoreSize);
+    return this.memstoreSize.addAndGet(memStoreSize);
   }
 
   /*

From 92146eba534a45386614fd2bb7da85b3e61001a7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 18 Feb 2014 19:17:01 +0000
Subject: [PATCH 1334/1540] HBASE-10546 Two scanner objects are open for each
 hbase map task but only one scanner object is closed. (Vasu Mariyala)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569487 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/TableRecordReaderImpl.java       | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 39c9b8ec8e8d..0afc47aa6ef5 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -77,6 +77,12 @@ public void restart(byte[] firstRow) throws IOException {
     currentScan.setStartRow(firstRow);
     currentScan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE,
       Bytes.toBytes(Boolean.TRUE));
+    if (this.scanner != null) {
+      if (logScannerActivity) {
+        LOG.info("Closing the previously opened scanner object.");
+      }
+      this.scanner.close();
+    }
     this.scanner = this.htable.getScanner(currentScan);
     if (logScannerActivity) {
       LOG.info("Current scan=" + currentScan.toString());

From d00cfd04549f7bcd3adae9bf58069825a7627b7f Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 18 Feb 2014 19:41:51 +0000
Subject: [PATCH 1335/1540] HBASE-10501 Improve
 IncreasingToUpperBoundRegionSplitPolicy to avoid too many regions.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569504 13f79535-47bb-0310-9956-ffa450edef68
---
 ...creasingToUpperBoundRegionSplitPolicy.java | 27 +++++++++++--------
 .../regionserver/TestRegionSplitPolicy.java   | 16 +++++------
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
index 327553d9b545..b4f0c3ac1f32 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
@@ -29,30 +29,34 @@
 
 /**
  * Split size is the number of regions that are on this server that all are
- * of the same table, squared, times the region flush size OR the maximum
+ * of the same table, cubed, times 2x the region flush size OR the maximum
  * region split size, whichever is smaller.  For example, if the flush size
- * is 128M, then on first flush we will split which will make two regions
- * that will split when their size is 2 * 2 * 128M = 512M.  If one of these
+ * is 128M, then after two flushes (256MB) we will split which will make two regions
+ * that will split when their size is 2^3 * 128M*2 = 2048M.  If one of these
  * regions splits, then there are three regions and now the split size is
- * 3 * 3 * 128M =  1152M, and so on until we reach the configured
+ * 3^3 * 128M*2 =  6912M, and so on until we reach the configured
  * maximum filesize and then from there on out, we'll use that.
  */
 public class IncreasingToUpperBoundRegionSplitPolicy
 extends ConstantSizeRegionSplitPolicy {
   static final Log LOG =
     LogFactory.getLog(IncreasingToUpperBoundRegionSplitPolicy.class);
-  private long flushSize;
+  private long initialSize;
 
   @Override
   protected void configureForRegion(HRegion region) {
     super.configureForRegion(region);
     Configuration conf = getConf();
+    this.initialSize = conf.getLong("hbase.increasing.policy.initial.size", -1);
+    if (this.initialSize > 0) {
+      return;
+    }
     HTableDescriptor desc = region.getTableDesc();
     if (desc != null) {
-      this.flushSize = desc.getMemStoreFlushSize();
+      this.initialSize = 2*desc.getMemStoreFlushSize();
     }
-    if (this.flushSize <= 0) {
-      this.flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
+    if (this.initialSize <= 0) {
+      this.initialSize = 2*conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
         HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
     }
   }
@@ -90,10 +94,11 @@ protected boolean shouldSplit() {
    * @return Region max size or <code>count of regions squared * flushsize, which ever is
    * smaller; guard against there being zero regions on this server.
    */
-  long getSizeToCheck(final int tableRegionsCount) {
-    return tableRegionsCount == 0? getDesiredMaxFileSize():
+  protected long getSizeToCheck(final int tableRegionsCount) {
+    // safety check for 100 to avoid numerical overflow in extreme cases
+    return tableRegionsCount == 0 || tableRegionsCount > 100 ? getDesiredMaxFileSize():
       Math.min(getDesiredMaxFileSize(),
-        this.flushSize * (tableRegionsCount * tableRegionsCount));
+        this.initialSize * tableRegionsCount * tableRegionsCount * tableRegionsCount);
   }
 
   /**
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
index 69e6704cfecc..4f170347436c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
@@ -75,9 +75,9 @@ public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
     // Set max size for this 'table'.
     long maxSplitSize = 1024L;
     htd.setMaxFileSize(maxSplitSize);
-    // Set flush size to 1/4.  IncreasingToUpperBoundRegionSplitPolicy
-    // grows by the square of the number of regions times flushsize each time.
-    long flushSize = maxSplitSize/4;
+    // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
+    // grows by the cube of the number of regions times flushsize each time.
+    long flushSize = maxSplitSize/8;
     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
     htd.setMemStoreFlushSize(flushSize);
     // If RegionServerService with no regions in it -- 'online regions' == 0 --
@@ -100,18 +100,18 @@ public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
     // Now test that we increase our split size as online regions for a table
     // grows. With one region, split size should be flushsize.
     regions.add(mockRegion);
-    Mockito.doReturn(flushSize/2).when(mockStore).getSize();
-    // Should not split since store is 1/2 flush size.
+    Mockito.doReturn(flushSize).when(mockStore).getSize();
+    // Should not split since store is flush size.
     assertFalse(policy.shouldSplit());
-    // Set size of store to be > flush size and we should split
-    Mockito.doReturn(flushSize + 1).when(mockStore).getSize();
+    // Set size of store to be > 2*flush size and we should split
+    Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
     assertTrue(policy.shouldSplit());
     // Add another region to the 'online regions' on this server and we should
     // now be no longer be splittable since split size has gone up.
     regions.add(mockRegion);
     assertFalse(policy.shouldSplit());
     // Quadruple (2 squared) the store size and make sure its just over; verify it'll split
-    Mockito.doReturn((flushSize * 2 * 2) + 1).when(mockStore).getSize();
+    Mockito.doReturn((flushSize * 2 * 2 * 2) + 1).when(mockStore).getSize();
     assertTrue(policy.shouldSplit());
 
     // Finally assert that even if loads of regions, we'll split at max size

From 55578c2d1d3bf02872e8eb1a5078e0b1667acb55 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 18 Feb 2014 19:46:00 +0000
Subject: [PATCH 1336/1540] CHANGES.txt for 0.94.17 RC1

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1569508 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index a97d4252952b..49af10057961 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
 HBase Change Log
-Release 0.94.17 - 02/13/2014
+Release 0.94.17 - 02/18/2014
 Bug
 
     [HBASE-7963] - HBase VerifyReplication not working when security enabled
@@ -21,12 +21,21 @@ Bug
     [HBASE-10485] - PrefixFilter#filterKeyValue() should perform filtering on row key
     [HBASE-10489] - TestImportExport fails in 0.94 with Hadoop2
     [HBASE-10493] - InclusiveStopFilter#filterKeyValue() should perform filtering on row key
+    [HBASE-10501] - Improve IncreasingToUpperBoundRegionSplitPolicy to avoid too many regions
     [HBASE-10505] - Import.filterKv does not call Filter.filterRowKey
     [HBASE-10506] - Fail-fast if client connection is lost before the real call be executed in RPC layer
     [HBASE-10508] - Backport HBASE-10365 'HBaseFsck should clean up connection properly when repair is completed' to 0.94 and 0.96
+    [HBASE-10539] - HRegion.addAndGetGlobalMemstoreSize returns previous size
+    [HBASE-10545] - RS Hangs waiting on region to close on shutdown; has to timeout before can go down
+    [HBASE-10546] - Two scanner objects are open for each hbase map task but only one scanner object is closed
+    [HBASE-10551] - Change local mode back to one RS thread by default
+    [HBASE-10552] - HFilePerformanceEvaluation.GaussianRandomReadBenchmark fails sometimes.
+    [HBASE-10555] - Backport HBASE-8519 to 0.94, Backup master will never come up if primary master dies during initialization
+    [HBASE-10562] - Fix TestMultiTableInputFormat for Hadoop 2 in 0.94
 
 Improvement
 
+    [HBASE-10212] - New rpc metric: number of active handler
     [HBASE-10423] - Report back the message of split or rollback failure to the master
     [HBASE-10457] - Print corrupted file information in SnapshotInfo tool without -file option
 

From 04cd457359f6c106be1b41a90c8af3ac98432c29 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 22 Feb 2014 00:06:31 +0000
Subject: [PATCH 1337/1540] HBASE-10583 backport HBASE-8402 to 0.94 -
 ScanMetrics depends on number of rpc calls to the server. (Liu Shaohui and
 Himanshu Vashishtha)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1570756 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/ClientScanner.java    | 20 ++++++++++---------
 .../hbase/client/TestFromClientSide.java      | 13 ++++++++++++
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 9aa4112feb4f..68751125e12b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -58,7 +58,8 @@ public class ClientScanner extends AbstractClientScanner {
     private final HConnection connection;
     private final byte[] tableName;
     private final int scannerTimeout;
-
+    private boolean scanMetricsPublished = false;
+    
     /**
      * Create a new ClientScanner for the specified table. An HConnection will be
      * retrieved using the passed Configuration.
@@ -240,12 +241,13 @@ protected ScannerCallable getScannerCallable(byte [] localStartKey,
      * scan.setAttribute(SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE))
      */
     protected void writeScanMetrics() throws IOException {
-      if (this.scanMetrics == null) {
+      if (this.scanMetrics == null || scanMetricsPublished) {
         return;
       }
       final DataOutputBuffer d = new DataOutputBuffer();
       scanMetrics.write(d);
       scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, d.getData());
+      scanMetricsPublished = true;
     }
 
     public Result next() throws IOException {
@@ -362,6 +364,13 @@ public Result next() throws IOException {
     }
 
     public void close() {
+      if (!scanMetricsPublished){
+        try {
+          writeScanMetrics();
+        } catch (IOException e) {
+        }
+      }
+      
       if (callable != null) {
         callable.setClose();
         try {
@@ -371,13 +380,6 @@ public void close() {
           // have since decided that it's not nice for a scanner's close to
           // throw exceptions. Chances are it was just an UnknownScanner
           // exception due to lease time out.
-        } finally {
-          // we want to output the scan metrics even if an error occurred on close
-          try {
-            writeScanMetrics();
-          } catch (IOException e) {
-            // As above, we still don't want the scanner close() method to throw.
-          }
         }
         callable = null;
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 084fdaeb8368..36d57f851fd7 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -4576,6 +4576,19 @@ public void testScanMetrics() throws Exception {
     assertEquals("Did not access all the regions in the table", numOfRegions,
         scanMetrics.countOfRegions.getCurrentIntervalValue());
 
+    // set caching to 100
+    scan = new Scan();
+    scan.setCaching(100);
+    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+    scanner = ht.getScanner(scan);
+    for (Result result : scanner.next(numRecords - 1)) {
+    }
+    scanner.close();
+    
+    scanMetrics = getScanMetrics(scan);
+    assertEquals("Did not access all the regions in the table", numOfRegions,
+        scanMetrics.countOfRegions.getCurrentIntervalValue());
+    
     // now, test that the metrics are still collected even if you don't call close, but do
     // run past the end of all the records
     Scan scanWithoutClose = new Scan();

From 395d515a7be0d913b37ba49101e29b2340df2f88 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 24 Feb 2014 05:10:03 +0000
Subject: [PATCH 1338/1540] HBASE-10594 Speed up TestRestoreSnapshotFromClient
 a bit.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1571144 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestRestoreSnapshotFromClient.java   | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 04cb2f8d2323..68768eeb003f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -121,13 +121,6 @@ public void setup() throws Exception {
     admin.enableTable(tableName);
     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
     snapshot1Rows = TEST_UTIL.countRows(table);
-    admin.disableTable(tableName);
-
-    // take a snapshot of the updated table
-    admin.snapshot(snapshotName1, tableName);
-
-    // re-enable table
-    admin.enableTable(tableName);
     table.close();
   }
 
@@ -141,9 +134,9 @@ public void tearDown() throws Exception {
   @Test
   public void testRestoreSnapshot() throws IOException {
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
-
-    // Restore from snapshot-0
     admin.disableTable(tableName);
+    admin.snapshot(snapshotName1, tableName);
+    // Restore from snapshot-0
     admin.restoreSnapshot(snapshotName0);
     admin.enableTable(tableName);
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);

From 13227db7cb7df4f2f395ee91fbb187c74464bdea Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 26 Feb 2014 04:34:04 +0000
Subject: [PATCH 1339/1540] HBASE-10614 Master could not be stopped (Jingcheng
 Du)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1571918 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/catalog/MetaReader.java   | 4 ++--
 .../hadoop/hbase/master/handler/ServerShutdownHandler.java | 7 +++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index cbb6d483bec9..c47166da8dfd 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -194,11 +194,11 @@ public static void fullScan(CatalogTracker catalogTracker,
   private static HTable getHTable(final CatalogTracker catalogTracker,
       final byte [] tableName)
   throws IOException {
-    // Passing the CatalogTracker's connection configuration ensures this
+    // Passing the CatalogTracker's connection ensures this
     // HTable instance uses the CatalogTracker's connection.
     org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
     if (c == null) throw new NullPointerException("No connection");
-    return new HTable(catalogTracker.getConnection().getConfiguration(), tableName);
+    return new HTable(tableName, c);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 41efd27191da..329404528c75 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -181,8 +181,11 @@ public void process() throws IOException {
       while (!this.server.isStopped()) {
         try {
           this.server.getCatalogTracker().waitForMeta();
-          hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
-            this.serverName);
+          // Skip getting user regions if the server is stopped.
+          if (!this.server.isStopped()) {
+            hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
+                this.serverName);
+          }
           break;
         } catch (InterruptedException e) {
           Thread.currentThread().interrupt();

From fbfb11664a264cb327d6caf84c82f0ff14a39915 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 26 Feb 2014 17:08:02 +0000
Subject: [PATCH 1340/1540] HBASE-9914 Port fix for HBASE-9836 'Intermittent
 TestRegionObserverScannerOpenHook#testRegionObserverCompactionTimeStacking
 failure' to 0.94 (Takeshi Miao)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1572166 13f79535-47bb-0310-9956-ffa450edef68
---
 .../TestRegionObserverScannerOpenHook.java    | 60 +++++++++++--------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index a1c11950ab7f..1fba781f05d9 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -31,6 +31,7 @@
 import java.util.concurrent.CountDownLatch;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -54,10 +55,12 @@
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreScanner;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -146,6 +149,7 @@ public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEn
     }
   }
 
+
   HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
       byte[]... families) throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
@@ -213,6 +217,31 @@ public void testRegionObserverFlushTimeStacking() throws Exception {
           + r, r.list());
   }
 
+  /*
+   * Custom HRegion which uses CountDownLatch to signal the completion of compaction
+   */
+  public static class CompactionCompletionNotifyingRegion extends HRegion {
+    private static volatile CountDownLatch compactionStateChangeLatch = null;
+
+    public CompactionCompletionNotifyingRegion(Path tableDir, HLog log, FileSystem fs,
+        Configuration confParam, HRegionInfo info, HTableDescriptor htd,
+        RegionServerServices rsServices) {
+      super(tableDir, log, fs, confParam, info, htd, rsServices);
+    }
+
+    public CountDownLatch getCompactionStateChangeLatch() {
+      if (compactionStateChangeLatch == null) compactionStateChangeLatch = new CountDownLatch(1);
+      return compactionStateChangeLatch;
+    }
+
+    @Override
+    public boolean compact(CompactionRequest cr) throws IOException {
+      boolean ret = super.compact(cr);
+      if (ret) compactionStateChangeLatch.countDown();
+      return ret;
+    }
+  }
+
   /**
    * Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do
    * the usual compaction mechanism on the region, rather than going through the backdoor to the
@@ -223,6 +252,7 @@ public void testRegionObserverFlushTimeStacking() throws Exception {
   public void testRegionObserverCompactionTimeStacking() throws Exception {
     // setup a mini cluster so we can do a real compaction on a region
     Configuration conf = UTIL.getConfiguration();
+    conf.setClass(HConstants.REGION_IMPL, CompactionCompletionNotifyingRegion.class, HRegion.class);
     conf.setInt("hbase.hstore.compaction.min", 2);
     UTIL.startMiniCluster();
     String tableName = "testRegionObserverCompactionTimeStacking";
@@ -250,6 +280,8 @@ public void testRegionObserverCompactionTimeStacking() throws Exception {
     assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
     HRegion region = regions.get(0);
     admin.flush(region.getRegionName());
+    CountDownLatch latch = ((CompactionCompletionNotifyingRegion)region)
+        .getCompactionStateChangeLatch();
 
     // put another row and flush that too
     put = new Put(Bytes.toBytes("anotherrow"));
@@ -260,8 +292,7 @@ public void testRegionObserverCompactionTimeStacking() throws Exception {
 
     // run a compaction, which normally would should get rid of the data
     Store s = region.getStores().get(A);
-    CountDownLatch latch = new CountDownLatch(1);
-    WaitableCompactionRequest request = new WaitableCompactionRequest(region, s, latch);
+    CompactionRequest request = new CompactionRequest(region, s, Store.PRIORITY_USER);
     rs.compactSplitThread.requestCompaction(region, s,
       "compact for testRegionObserverCompactionTimeStacking", Store.PRIORITY_USER, request);
     // wait for the compaction to complete
@@ -283,27 +314,4 @@ public void testRegionObserverCompactionTimeStacking() throws Exception {
     table.close();
     UTIL.shutdownMiniCluster();
   }
-
-  /**
-   * A simple compaction on which you can wait for the passed in latch until the compaction finishes
-   * (either successfully or if it failed).
-   */
-  public static class WaitableCompactionRequest extends CompactionRequest {
-    private CountDownLatch done;
-
-    /**
-     * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
-     * compaction before being used.
-     */
-    public WaitableCompactionRequest(HRegion region, Store store, CountDownLatch finished) {
-      super(region, store, Store.PRIORITY_USER);
-      this.done = finished;
-    }
-
-    @Override
-    public void finishRequest() {
-      super.finishRequest();
-      this.done.countDown();
-    }
-  }
-}
\ No newline at end of file
+}

From 5221dcfc00ea06fe73b5249a97102b645a62e7ee Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Wed, 26 Feb 2014 23:12:59 +0000
Subject: [PATCH 1341/1540] HBASE-10598 Written data can not be read out
 because MemStore#timeRangeTracker might be updated concurrently (cuijianwei)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1572334 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/TimeRangeTracker.java       | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
index ec028d1d1a84..aebb75377a29 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
@@ -95,7 +95,7 @@ public void includeTimestamp(final byte[] key) {
    * If required, update the current TimestampRange to include timestamp
    * @param timestamp the timestamp value to include
    */
-  private void includeTimestamp(final long timestamp) {
+  private synchronized void includeTimestamp(final long timestamp) {
     if (maximumTimestamp == -1) {
       minimumTimestamp = timestamp;
       maximumTimestamp = timestamp;
@@ -114,7 +114,7 @@ else if (maximumTimestamp < timestamp) {
    * @param tr TimeRange
    * @return True if there is overlap, false otherwise
    */
-  public boolean includesTimeRange(final TimeRange tr) {
+  public synchronized boolean includesTimeRange(final TimeRange tr) {
     return (this.minimumTimestamp < tr.getMax() &&
         this.maximumTimestamp >= tr.getMin());
   }
@@ -122,14 +122,14 @@ public boolean includesTimeRange(final TimeRange tr) {
   /**
    * @return the minimumTimestamp
    */
-  public long getMinimumTimestamp() {
+  public synchronized long getMinimumTimestamp() {
     return minimumTimestamp;
   }
 
   /**
    * @return the maximumTimestamp
    */
-  public long getMaximumTimestamp() {
+  public synchronized long getMaximumTimestamp() {
     return maximumTimestamp;
   }
 

From 1634af5ed1464e2206513c5c7542c37f86a8ef66 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 27 Feb 2014 05:32:19 +0000
Subject: [PATCH 1342/1540] HBASE-10594 Revert due to test instability

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1572436 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestRestoreSnapshotFromClient.java   | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 68768eeb003f..04cb2f8d2323 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -121,6 +121,13 @@ public void setup() throws Exception {
     admin.enableTable(tableName);
     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
     snapshot1Rows = TEST_UTIL.countRows(table);
+    admin.disableTable(tableName);
+
+    // take a snapshot of the updated table
+    admin.snapshot(snapshotName1, tableName);
+
+    // re-enable table
+    admin.enableTable(tableName);
     table.close();
   }
 
@@ -134,9 +141,9 @@ public void tearDown() throws Exception {
   @Test
   public void testRestoreSnapshot() throws IOException {
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
-    admin.disableTable(tableName);
-    admin.snapshot(snapshotName1, tableName);
+
     // Restore from snapshot-0
+    admin.disableTable(tableName);
     admin.restoreSnapshot(snapshotName0);
     admin.enableTable(tableName);
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);

From f769b6b96339527763cf2de837e3ff2f422e3f1a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 27 Feb 2014 05:50:22 +0000
Subject: [PATCH 1343/1540] HBASE-10575 ReplicationSource thread can't be
 terminated if it runs into the loop to contact peer's zk ensemble and fails
 continuously (Feng Honghua & LarsH)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1572441 13f79535-47bb-0310-9956-ffa450edef68
---
 .../replication/regionserver/ReplicationSource.java  | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 3831bba54c09..6d6ea7ba895d 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -289,20 +289,20 @@ public void enqueueLog(Path log) {
   @Override
   public void run() {
     connectToPeers();
-    // We were stopped while looping to connect to sinks, just abort
-    if (!this.isActive()) {
-      return;
-    }
     int sleepMultiplier = 1;
     // delay this until we are in an asynchronous thread
-    while (this.peerClusterId == null) {
+    while (this.isActive() && this.peerClusterId == null) {
       this.peerClusterId = zkHelper.getPeerUUID(this.peerId);
-      if (this.peerClusterId == null) {
+      if (this.isActive() && this.peerClusterId == null) {
         if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) {
           sleepMultiplier++;
         }
       }
     }
+    // We were stopped while looping to connect to sinks, just abort
+    if (!this.isActive()) {
+      return;
+    }
     // resetting to 1 to reuse later
     sleepMultiplier = 1;
 

From 97a7e0f53c3fd4fd532fe9af918b4262452fe36a Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Fri, 28 Feb 2014 09:20:34 +0000
Subject: [PATCH 1344/1540] HBASE-10631 Avoid extra seek on FileLink open

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1572892 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/io/FileLink.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/FileLink.java b/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
index dc910e6a0e29..d9f3f6454b79 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
@@ -287,7 +287,7 @@ private FSDataInputStream tryOpen() throws IOException {
         if (path.equals(currentPath)) continue;
         try {
           in = fs.open(path, bufferSize);
-          in.seek(pos);
+          if (pos != 0) in.seek(pos);
           assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos;
           if (LOG.isTraceEnabled()) {
             if (currentPath != null) {

From 085852fe081bf149f6a8a319194bb4c44eb70432 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 28 Feb 2014 17:35:14 +0000
Subject: [PATCH 1345/1540] HBASE-10627 A logic mistake in HRegionServer
 isHealthy (Shaohui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1573016 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ac6aadb54687..00636f404ae6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1798,8 +1798,8 @@ private boolean isHealthy() {
     // Verify that all threads are alive
     if (!(leases.isAlive()
         && cacheFlusher.isAlive() && hlogRoller.isAlive()
-        && this.compactionChecker.isAlive())
-        && this.periodicFlusher.isAlive()) {
+        && this.compactionChecker.isAlive()
+        && this.periodicFlusher.isAlive())) {
       stop("One or more threads are no longer alive -- stop");
       return false;
     }

From b7c338734bd62436f512e2c4f28eff8e2f098e4d Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 4 Mar 2014 09:28:48 +0000
Subject: [PATCH 1346/1540] HBASE-9708 Improve Snapshot Name Error Message
 (Esteban Gutierrez)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1573962 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/HTableDescriptor.java | 20 ++++++++++++-------
 .../hadoop/hbase/client/HBaseAdmin.java       |  2 +-
 .../snapshot/SnapshotDescriptionUtils.java    |  2 +-
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 697e884de5d8..edea63152a74 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -383,6 +383,10 @@ public static boolean isMetaTable(final byte [] tableName) {
   // A non-capture group so that this can be embedded.
   public static final String VALID_USER_TABLE_REGEX = "(?:[a-zA-Z_0-9][a-zA-Z_0-9.-]*)";
 
+  public static byte [] isLegalTableName(final byte [] tableName) {
+    return isLegalTableName(tableName, false);
+  }
+
   /**
    * Check passed byte buffer, "tableName", is legal user-space table name.
    * @return Returns passed <code>tableName</code> param
@@ -391,23 +395,25 @@ public static boolean isMetaTable(final byte [] tableName) {
    * that is made of other than 'word' characters or underscores: i.e.
    * <code>[a-zA-Z_0-9].
    */
-  public static byte [] isLegalTableName(final byte [] tableName) {
+  public static byte [] isLegalTableName(final byte [] tableName, boolean isSnapshot) {
     if (tableName == null || tableName.length <= 0) {
       throw new IllegalArgumentException("Name is null or empty");
     }
     if (tableName[0] == '.' || tableName[0] == '-') {
       throw new IllegalArgumentException("Illegal first character <" + tableName[0] +
-          "> at 0. User-space table names can only start with 'word " +
-          "characters': i.e. [a-zA-Z_0-9]: " + Bytes.toString(tableName));
+          "> at 0. " + (isSnapshot ? "snapshot" : "User-space table") +
+          " can only start with 'word characters': i.e. [a-zA-Z_0-9]: " +
+          Bytes.toString(tableName));
     }
     for (int i = 0; i < tableName.length; i++) {
-      if (Character.isLetterOrDigit(tableName[i]) || tableName[i] == '_' || 
-    		  tableName[i] == '-' || tableName[i] == '.') {
+      if (Character.isLetterOrDigit(tableName[i]) || tableName[i] == '_' ||
+          tableName[i] == '-' || tableName[i] == '.') {
         continue;
       }
       throw new IllegalArgumentException("Illegal character <" + tableName[i] +
-        "> at " + i + ". User-space table names can only contain " +
-        "'word characters': i.e. [a-zA-Z_0-9-.]: " + Bytes.toString(tableName));
+        "> at " + i + ". " + (isSnapshot ? "snapshot" : "User-space table") +
+        " can only contain 'word characters': i.e. [a-zA-Z_0-9-.]: " +
+        Bytes.toString(tableName));
     }
     return tableName;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 88090175b534..75106bc0a8a2 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2336,7 +2336,7 @@ public List<SnapshotDescription> listSnapshots() throws IOException {
    */
   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
     // make sure the snapshot is possibly valid
-    HTableDescriptor.isLegalTableName(snapshotName);
+    HTableDescriptor.isLegalTableName(snapshotName, true);
     // do the delete
     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
       .setName(Bytes.toString(snapshotName)).build();
diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index e04876871ee1..95ee2879c842 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -137,7 +137,7 @@ public static void assertSnapshotRequestIsValid(SnapshotDescription snapshot)
       throw new IllegalArgumentException(".META. and -ROOT- snapshots are not allowed");
     }
     // make sure the snapshot name is valid
-    HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getName()));
+    HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getName()), true);
     // make sure the table name is valid
     HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getTable()));
   }

From ea38f37435eb26e8d103bc0e2fde8f3f221278b0 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 4 Mar 2014 10:17:06 +0000
Subject: [PATCH 1347/1540] HBASE-10537 Let the ExportSnapshot mapper fail and
 retry on error

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574016 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 75 ++++++++++++++-----
 .../hbase/snapshot/TestExportSnapshot.java    | 47 +++++++++++-
 2 files changed, 98 insertions(+), 24 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index f14cd4eb4d54..357d690c9893 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -25,6 +25,7 @@
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -87,6 +88,9 @@ public final class ExportSnapshot extends Configured implements Tool {
   private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
   private static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
 
+  static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
+  static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
+
   private static final String INPUT_FOLDER_PREFIX = "export-files.";
 
   // Export Map-Reduce Counters, to keep track of the progress
@@ -96,6 +100,9 @@ private static class ExportMapper extends Mapper<Text, NullWritable, NullWritabl
     final static int REPORT_SIZE = 1 * 1024 * 1024;
     final static int BUFFER_SIZE = 64 * 1024;
 
+    private boolean testFailures;
+    private Random random;
+
     private boolean verifyChecksum;
     private String filesGroup;
     private String filesUser;
@@ -123,6 +130,8 @@ public void setup(Context context) {
       inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
       outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
 
+      testFailures = conf.getBoolean(CONF_TEST_FAILURE, false);
+
       try {
         inputFs = FileSystem.get(inputRoot.toUri(), conf);
       } catch (IOException e) {
@@ -143,9 +152,8 @@ public void map(Text key, NullWritable value, Context context)
       Path outputPath = getOutputPath(inputPath);
 
       LOG.info("copy file input=" + inputPath + " output=" + outputPath);
-      if (copyFile(context, inputPath, outputPath)) {
-        LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
-      }
+      copyFile(context, inputPath, outputPath);
+      LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
     }
 
     /**
@@ -170,25 +178,55 @@ private Path getOutputPath(final Path inputPath) throws IOException {
       return new Path(outputArchive, path);
     }
 
-    private boolean copyFile(final Context context, final Path inputPath, final Path outputPath)
+    /*
+     * Used by TestExportSnapshot to simulate a failure
+     */
+    private void injectTestFailure(final Context context, final Path inputPath)
         throws IOException {
+      if (testFailures) {
+        if (context.getConfiguration().getBoolean(CONF_TEST_RETRY, false)) {
+          if (random == null) {
+            random = new Random();
+          }
+
+          // FLAKY-TEST-WARN: lower is better, we can get some runs without the
+          // retry, but at least we reduce the number of test failures due to
+          // this test exception from the same map task.
+          if (random.nextFloat() < 0.03) {
+            throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath
+                                  + " time=" + System.currentTimeMillis());
+          }
+        } else {
+          context.getCounter(Counter.COPY_FAILED).increment(1);
+          throw new IOException("TEST FAILURE: Unable to copy input=" + inputPath);
+        }
+      }
+    }
+
+    private void copyFile(final Context context, final Path inputPath, final Path outputPath)
+        throws IOException {
+      injectTestFailure(context, inputPath);
+
       FSDataInputStream in = openSourceFile(inputPath);
       if (in == null) {
         context.getCounter(Counter.MISSING_FILES).increment(1);
-        return false;
+        throw new FileNotFoundException(inputPath.toString());
       }
 
       try {
         // Verify if the input file exists
         FileStatus inputStat = getFileStatus(inputFs, inputPath);
-        if (inputStat == null) return false;
+        if (inputStat == null) {
+          context.getCounter(Counter.MISSING_FILES).increment(1);
+          throw new FileNotFoundException(inputPath.toString());
+        }
 
         // Verify if the output file exists and is the same that we want to copy
         if (outputFs.exists(outputPath)) {
           FileStatus outputStat = outputFs.getFileStatus(outputPath);
           if (sameFile(inputStat, outputStat)) {
             LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
-            return true;
+            return;
           }
         }
 
@@ -198,14 +236,13 @@ private boolean copyFile(final Context context, final Path inputPath, final Path
         outputFs.mkdirs(outputPath.getParent());
         FSDataOutputStream out = outputFs.create(outputPath, true);
         try {
-          if (!copyData(context, inputPath, in, outputPath, out, inputStat.getLen()))
-            return false;
+          copyData(context, inputPath, in, outputPath, out, inputStat.getLen());
         } finally {
           out.close();
         }
 
         // Preserve attributes
-        return preserveAttributes(outputPath, inputStat);
+        preserveAttributes(outputPath, inputStat);
       } finally {
         in.close();
       }
@@ -248,10 +285,11 @@ private boolean preserveAttributes(final Path path, final FileStatus refStat) {
       return true;
     }
 
-    private boolean copyData(final Context context,
+    private void copyData(final Context context,
         final Path inputPath, final FSDataInputStream in,
         final Path outputPath, final FSDataOutputStream out,
-        final long inputFileSize) {
+        final long inputFileSize)
+        throws IOException {
       final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
                                    " (%.3f%%)";
 
@@ -284,17 +322,14 @@ private boolean copyData(final Context context,
 
         // Verify that the written size match
         if (totalBytesWritten != inputFileSize) {
-          LOG.error("number of bytes copied not matching copied=" + totalBytesWritten +
-                    " expected=" + inputFileSize + " for file=" + inputPath);
-          context.getCounter(Counter.COPY_FAILED).increment(1);
-          return false;
+          String msg = "number of bytes copied not matching copied=" + totalBytesWritten +
+                       " expected=" + inputFileSize + " for file=" + inputPath;
+          throw new IOException(msg);
         }
-
-        return true;
       } catch (IOException e) {
         LOG.error("Error copying " + inputPath + " to " + outputPath, e);
         context.getCounter(Counter.COPY_FAILED).increment(1);
-        return false;
+        throw e;
       }
     }
 
@@ -666,9 +701,9 @@ public int run(String[] args) throws Exception {
                            snapshotTmpDir + " to=" + outputSnapshotDir);
         return 1;
       }
-
       return 0;
     } catch (Exception e) {
+      LOG.error("Snapshot export failed", e);
       System.err.println("Snapshot export failed!");
       e.printStackTrace(System.err);
       outputFs.delete(outputSnapshotDir, true);
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index a666c5df624c..0b3724a3e13d 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -88,6 +88,8 @@ public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
     TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
     TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
+    TEST_UTIL.getConfiguration().setInt("mapreduce.map.max.attempts", 10);
+    TEST_UTIL.getConfiguration().setInt("mapred.map.max.attempts", 10);
     TEST_UTIL.startMiniCluster(3);
   }
 
@@ -109,16 +111,14 @@ public void setUp() throws Exception {
     emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid);
 
     // create Table
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    htd.addFamily(new HColumnDescriptor(FAMILY));
-    admin.createTable(htd, null);
+    SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
 
     // Take an empty snapshot
     admin.snapshot(emptySnapshotName, tableName);
 
     // Add some rows
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    TEST_UTIL.loadTable(table, FAMILY);
+    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 1000, FAMILY);
 
     // take a snapshot
     admin.snapshot(snapshotName, tableName);
@@ -132,6 +132,7 @@ public void tearDown() throws Exception {
     admin.close();
   }
 
+
   /**
    * Verfy the result of getBalanceSplits() method.
    * The result are groups of files, used as input list for the "export" mappers.
@@ -270,6 +271,44 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
     fs.delete(copyDir, true);
   }
 
+  /**
+   * Check that ExportSnapshot will return a failure if something fails.
+   */
+  @Test
+  public void testExportFailure() throws Exception {
+    assertEquals(1, runExportAndInjectFailures(snapshotName, false));
+  }
+
+  /**
+   * Check that ExportSnapshot will succede if something fails but the retry succede.
+   */
+  @Test
+  public void testExportRetry() throws Exception {
+    assertEquals(0, runExportAndInjectFailures(snapshotName, true));
+  }
+
+  /*
+   * Execute the ExportSnapshot job injecting failures
+   */
+  private int runExportAndInjectFailures(final byte[] snapshotName, boolean retry)
+      throws Exception {
+    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+    URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
+    FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
+    copyDir = copyDir.makeQualified(fs);
+
+    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+    conf.setBoolean(ExportSnapshot.CONF_TEST_FAILURE, true);
+    conf.setBoolean(ExportSnapshot.CONF_TEST_RETRY, retry);
+
+    // Export Snapshot
+    int res = ExportSnapshot.innerMain(conf, new String[] {
+      "-snapshot", Bytes.toString(snapshotName),
+      "-copy-to", copyDir.toString()
+    });
+    return res;
+  }
+
   /*
    * verify if the snapshot folder on file-system 1 match the one on file-system 2
    */

From bc8ebd49dc9d84a18419705f935217a75be923ab Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 4 Mar 2014 10:25:39 +0000
Subject: [PATCH 1348/1540] HBASE-10567 Add overwrite manifest option to
 ExportSnapshot

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574017 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 25 ++++++++----
 .../hbase/snapshot/TestExportSnapshot.java    | 40 ++++++++++++++-----
 2 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 357d690c9893..dac9662fc2ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -596,6 +596,7 @@ private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
   public int run(String[] args) throws Exception {
     boolean verifyChecksum = true;
     String snapshotName = null;
+    boolean overwrite = false;
     String filesGroup = null;
     String filesUser = null;
     Path outputRoot = null;
@@ -620,6 +621,8 @@ public int run(String[] args) throws Exception {
           filesGroup = args[++i];
         } else if (cmd.equals("-chmod")) {
           filesMode = Integer.parseInt(args[++i], 8);
+        } else if (cmd.equals("-overwrite")) {
+          overwrite = true;
         } else if (cmd.equals("-h") || cmd.equals("--help")) {
           printUsageAndExit();
         } else {
@@ -653,13 +656,20 @@ public int run(String[] args) throws Exception {
 
     // Check if the snapshot already exists
     if (outputFs.exists(outputSnapshotDir)) {
-      System.err.println("The snapshot '" + snapshotName +
-        "' already exists in the destination: " + outputSnapshotDir);
-      return 1;
+      if (overwrite) {
+        if (!outputFs.delete(outputSnapshotDir, true)) {
+          System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
+          return 1;
+        }
+      } else {
+        System.err.println("The snapshot '" + snapshotName +
+          "' already exists in the destination: " + outputSnapshotDir);
+        return 1;
+      }
     }
 
     // Check if the snapshot already in-progress
-    if (outputFs.exists(snapshotTmpDir)) {
+    if (!overwrite && outputFs.exists(snapshotTmpDir)) {
       System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
       System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
       System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
@@ -673,7 +683,7 @@ public int run(String[] args) throws Exception {
     // The snapshot references must be copied before the hfiles otherwise the cleaner
     // will remove them because they are unreferenced.
     try {
-      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, overwrite, conf);
     } catch (IOException e) {
       System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
         " to=" + snapshotTmpDir);
@@ -719,14 +729,15 @@ private void printUsageAndExit() {
     System.err.println("  -snapshot NAME          Snapshot to restore.");
     System.err.println("  -copy-to NAME           Remote destination hdfs://");
     System.err.println("  -no-checksum-verify     Do not verify checksum.");
+    System.err.println("  -overwrite              Rewrite the snapshot manifest if already exists");
     System.err.println("  -chuser USERNAME        Change the owner of the files to the specified one.");
     System.err.println("  -chgroup GROUP          Change the group of the files to the specified one.");
     System.err.println("  -chmod MODE             Change the permission of the files to the specified one.");
     System.err.println("  -mappers                Number of mappers to use during the copy (mapreduce.job.maps).");
     System.err.println();
     System.err.println("Examples:");
-    System.err.println("  hbase " + getClass() + " \\");
-    System.err.println("    -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\");
+    System.err.println("  hbase " + getClass().getName() + " \\");
+    System.err.println("    -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
     System.err.println("    -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
     System.exit(1);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index 0b3724a3e13d..5bd165c817d5 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -183,6 +183,14 @@ public void testEmptyExportFileSystemState() throws Exception {
     testExportFileSystemState(tableName, emptySnapshotName, 1);
   }
 
+  @Test
+  public void testConsecutiveExports() throws Exception {
+    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+    testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
+    testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
+    removeExportDir(copyDir);
+  }
+
   /**
    * Mock a snapshot with files in the archive dir,
    * two regions, and one reference file.
@@ -233,21 +241,32 @@ public void testSnapshotWithRefsExportFileSystemState() throws Exception {
     testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
   }
 
+  private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
+      int filesExpected) throws Exception {
+    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+    testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
+    removeExportDir(copyDir);
+  }
+
   /**
    * Test ExportSnapshot
    */
   private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
-      int filesExpected) throws Exception {
-    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+      int filesExpected, Path copyDir, boolean overwrite) throws Exception {
     URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
     FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
     copyDir = copyDir.makeQualified(fs);
 
+    List<String> opts = new ArrayList<String>();
+    opts.add("-snapshot");
+    opts.add(Bytes.toString(snapshotName));
+    opts.add("-copy-to");
+    opts.add(copyDir.toString());
+    if (overwrite) opts.add("-overwrite");
+
     // Export Snapshot
-    int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
-      "-snapshot", Bytes.toString(snapshotName),
-      "-copy-to", copyDir.toString()
-    });
+    int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(),
+        opts.toArray(new String[opts.size()]));
     assertEquals(0, res);
 
     // Verify File-System state
@@ -266,9 +285,6 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
         fs, new Path(copyDir, snapshotDir));
     verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
     FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
-
-    // Remove the exported dir
-    fs.delete(copyDir, true);
   }
 
   /**
@@ -370,4 +386,10 @@ private Set<String> listFiles(final FileSystem fs, final Path root, final Path d
     }
     return files;
   }
+
+  private void removeExportDir(final Path path) throws IOException {
+    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+    FSUtils.logFileSystemState(fs, path, LOG);
+    fs.delete(path, true);
+  }
 }

From c63118c25b8cf7bd9ff7909b085310a8c644cb00 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Tue, 4 Mar 2014 11:21:11 +0000
Subject: [PATCH 1349/1540] HBASE-10622 Improve log and Exceptions in Export
 Snapshot

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574034 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 190 +++++++++++-------
 .../hbase/snapshot/TestExportSnapshot.java    |   8 -
 2 files changed, 115 insertions(+), 83 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index dac9662fc2ad..46741e70c440 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -29,7 +29,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -87,6 +86,8 @@ public final class ExportSnapshot extends Configured implements Tool {
   private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root";
   private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
   private static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
+  private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
+  private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
 
   static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
   static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
@@ -94,7 +95,7 @@ public final class ExportSnapshot extends Configured implements Tool {
   private static final String INPUT_FOLDER_PREFIX = "export-files.";
 
   // Export Map-Reduce Counters, to keep track of the progress
-  public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED };
+  public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED, FILES_COPIED };
 
   private static class ExportMapper extends Mapper<Text, NullWritable, NullWritable, NullWritable> {
     final static int REPORT_SIZE = 1 * 1024 * 1024;
@@ -107,6 +108,7 @@ private static class ExportMapper extends Mapper<Text, NullWritable, NullWritabl
     private String filesGroup;
     private String filesUser;
     private short filesMode;
+    private int bufferSize;
 
     private FileSystem outputFs;
     private Path outputArchive;
@@ -117,7 +119,7 @@ private static class ExportMapper extends Mapper<Text, NullWritable, NullWritabl
     private Path inputRoot;
 
     @Override
-    public void setup(Context context) {
+    public void setup(Context context) throws IOException {
       Configuration conf = context.getConfiguration();
       verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
 
@@ -135,14 +137,19 @@ public void setup(Context context) {
       try {
         inputFs = FileSystem.get(inputRoot.toUri(), conf);
       } catch (IOException e) {
-        throw new RuntimeException("Could not get the input FileSystem with root=" + inputRoot, e);
+        throw new IOException("Could not get the input FileSystem with root=" + inputRoot, e);
       }
 
       try {
         outputFs = FileSystem.get(outputRoot.toUri(), conf);
       } catch (IOException e) {
-        throw new RuntimeException("Could not get the output FileSystem with root="+ outputRoot, e);
+        throw new IOException("Could not get the output FileSystem with root="+ outputRoot, e);
       }
+
+      // Use the default block size of the outputFs if bigger
+      int defaultBlockSize = Math.max((int) outputFs.getDefaultBlockSize(), BUFFER_SIZE);
+      bufferSize = conf.getInt(CONF_BUFFER_SIZE, defaultBlockSize);
+      LOG.info("Using bufferSize=" + StringUtils.humanReadableInt(bufferSize));
     }
 
     @Override
@@ -153,7 +160,6 @@ public void map(Text key, NullWritable value, Context context)
 
       LOG.info("copy file input=" + inputPath + " output=" + outputPath);
       copyFile(context, inputPath, outputPath);
-      LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
     }
 
     /**
@@ -207,29 +213,20 @@ private void copyFile(final Context context, final Path inputPath, final Path ou
         throws IOException {
       injectTestFailure(context, inputPath);
 
-      FSDataInputStream in = openSourceFile(inputPath);
-      if (in == null) {
-        context.getCounter(Counter.MISSING_FILES).increment(1);
-        throw new FileNotFoundException(inputPath.toString());
-      }
+      // Get the file information
+      FileStatus inputStat = getSourceFileStatus(context, inputPath);
 
-      try {
-        // Verify if the input file exists
-        FileStatus inputStat = getFileStatus(inputFs, inputPath);
-        if (inputStat == null) {
-          context.getCounter(Counter.MISSING_FILES).increment(1);
-          throw new FileNotFoundException(inputPath.toString());
-        }
-
-        // Verify if the output file exists and is the same that we want to copy
-        if (outputFs.exists(outputPath)) {
-          FileStatus outputStat = outputFs.getFileStatus(outputPath);
-          if (sameFile(inputStat, outputStat)) {
-            LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
-            return;
-          }
+      // Verify if the output file exists and is the same that we want to copy
+      if (outputFs.exists(outputPath)) {
+        FileStatus outputStat = outputFs.getFileStatus(outputPath);
+        if (outputStat != null && sameFile(inputStat, outputStat)) {
+          LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+          return;
         }
+      }
 
+      FSDataInputStream in = openSourceFile(context, inputPath);
+      try {
         context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
 
         // Ensure that the output folder is there and copy the file
@@ -241,15 +238,22 @@ private void copyFile(final Context context, final Path inputPath, final Path ou
           out.close();
         }
 
-        // Preserve attributes
-        preserveAttributes(outputPath, inputStat);
+        // Try to Preserve attributes
+        if (!preserveAttributes(outputPath, inputStat)) {
+          LOG.warn("You may have to run manually chown on: " + outputPath);
+        }
       } finally {
         in.close();
       }
     }
 
     /**
-     * Preserve the files attribute selected by the user copying them from the source file
+     * Try to Preserve the files attribute selected by the user copying them from the source file
+     * This is only required when you are exporting as a different user than "hbase" or on a system
+     * that doesn't have the "hbase" user.
+     *
+     * This is not considered a blocking failure since the user can force a chmod with the user
+     * that knows is available on the system.
      */
     private boolean preserveAttributes(final Path path, final FileStatus refStat) {
       FileStatus stat;
@@ -267,38 +271,47 @@ private boolean preserveAttributes(final Path path, final FileStatus refStat) {
           outputFs.setPermission(path, refStat.getPermission());
         }
       } catch (IOException e) {
-        LOG.error("Unable to set the permission for file=" + path, e);
+        LOG.warn("Unable to set the permission for file="+ stat.getPath() +": "+ e.getMessage());
         return false;
       }
 
-      try {
-        String user = (filesUser != null) ? filesUser : refStat.getOwner();
-        String group = (filesGroup != null) ? filesGroup : refStat.getGroup();
-        if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
-          outputFs.setOwner(path, user, group);
+      String user = stringIsNotEmpty(filesUser) ? filesUser : refStat.getOwner();
+      String group = stringIsNotEmpty(filesGroup) ? filesGroup : refStat.getGroup();
+      if (stringIsNotEmpty(user) || stringIsNotEmpty(group)) {
+        try {
+          if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
+            outputFs.setOwner(path, user, group);
+          }
+        } catch (IOException e) {
+          LOG.warn("Unable to set the owner/group for file="+ stat.getPath() +": "+ e.getMessage());
+          LOG.warn("The user/group may not exist on the destination cluster: user=" +
+                   user + " group=" + group);
+          return false;
         }
-      } catch (IOException e) {
-        LOG.error("Unable to set the owner/group for file=" + path, e);
-        return false;
       }
 
       return true;
     }
 
+    private boolean stringIsNotEmpty(final String str) {
+      return str != null && str.length() > 0;
+    }
+
     private void copyData(final Context context,
         final Path inputPath, final FSDataInputStream in,
         final Path outputPath, final FSDataOutputStream out,
         final long inputFileSize)
         throws IOException {
       final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
-                                   " (%.3f%%)";
+                                   " (%.1f%%)";
 
       try {
-        byte[] buffer = new byte[BUFFER_SIZE];
+        byte[] buffer = new byte[bufferSize];
         long totalBytesWritten = 0;
         int reportBytes = 0;
         int bytesRead;
 
+        long stime = System.currentTimeMillis();
         while ((bytesRead = in.read(buffer)) > 0) {
           out.write(buffer, 0, bytesRead);
           totalBytesWritten += bytesRead;
@@ -308,16 +321,17 @@ private void copyData(final Context context,
             context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
             context.setStatus(String.format(statusMessage,
                               StringUtils.humanReadableInt(totalBytesWritten),
-                              totalBytesWritten/(float)inputFileSize) +
+                              (totalBytesWritten/(float)inputFileSize) * 100.0f) +
                               " from " + inputPath + " to " + outputPath);
             reportBytes = 0;
           }
         }
+        long etime = System.currentTimeMillis();
 
         context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
         context.setStatus(String.format(statusMessage,
                           StringUtils.humanReadableInt(totalBytesWritten),
-                          totalBytesWritten/(float)inputFileSize) +
+                          (totalBytesWritten/(float)inputFileSize) * 100.0f) +
                           " from " + inputPath + " to " + outputPath);
 
         // Verify that the written size match
@@ -326,6 +340,13 @@ private void copyData(final Context context,
                        " expected=" + inputFileSize + " for file=" + inputPath;
           throw new IOException(msg);
         }
+
+        LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
+        LOG.info("size=" + totalBytesWritten +
+            " (" + StringUtils.humanReadableInt(totalBytesWritten) + ")" +
+            " time=" + StringUtils.formatTimeDiff(etime, stime) +
+            String.format(" %.3fM/sec", (totalBytesWritten / ((etime - stime)/1000.0))/1048576.0));
+        context.getCounter(Counter.FILES_COPIED).increment(1);
       } catch (IOException e) {
         LOG.error("Error copying " + inputPath + " to " + outputPath, e);
         context.getCounter(Counter.COPY_FAILED).increment(1);
@@ -333,7 +354,12 @@ private void copyData(final Context context,
       }
     }
 
-    private FSDataInputStream openSourceFile(final Path path) {
+    /**
+     * Try to open the "source" file.
+     * Throws an IOException if the communication with the inputFs fail or 
+     * if the file is not found.
+     */
+    private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException {
       try {
         if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
           return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
@@ -344,25 +370,30 @@ private FSDataInputStream openSourceFile(final Path path) {
         }
         return inputFs.open(path);
       } catch (IOException e) {
+        context.getCounter(Counter.MISSING_FILES).increment(1);
         LOG.error("Unable to open source file=" + path, e);
-        return null;
+        throw e;
       }
     }
 
-    private FileStatus getFileStatus(final FileSystem fs, final Path path) {
+    private FileStatus getSourceFileStatus(Context context, final Path path) throws IOException {
       try {
         if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
           HFileLink link = new HFileLink(inputRoot, inputArchive, path);
-          return link.getFileStatus(fs);
+          return link.getFileStatus(inputFs);
         } else if (isHLogLinkPath(path)) {
           String serverName = path.getParent().getName();
           String logName = path.getName();
-          return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs);
+          return new HLogLink(inputRoot, serverName, logName).getFileStatus(inputFs);
         }
-        return fs.getFileStatus(path);
+        return inputFs.getFileStatus(path);
+      } catch (FileNotFoundException e) {
+        context.getCounter(Counter.MISSING_FILES).increment(1);
+        LOG.error("Unable to get the status for source file=" + path, e);
+        throw e;
       } catch (IOException e) {
-        LOG.warn("Unable to get the status for file=" + path);
-        return null;
+        LOG.error("Unable to get the status for source file=" + path, e);
+        throw e;
       }
     }
 
@@ -554,7 +585,7 @@ private static Path[] createInputFiles(final Configuration conf,
   /**
    * Run Map-Reduce Job to perform the files copy.
    */
-  private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
+  private void runCopyJob(final Path inputRoot, final Path outputRoot,
       final List<Pair<Path, Long>> snapshotFiles, final boolean verifyChecksum,
       final String filesUser, final String filesGroup, final int filesMode,
       final int mappers) throws IOException, InterruptedException, ClassNotFoundException {
@@ -585,7 +616,12 @@ private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
       SequenceFileInputFormat.addInputPath(job, path);
     }
 
-    return job.waitForCompletion(true);
+    // Run the MR Job
+    if (!job.waitForCompletion(true)) {
+      // TODO: Replace the fixed string with job.getStatus().getFailureInfo()
+      // when it will be available on all the supported versions.
+      throw new ExportSnapshotException("Copy Files Map-Reduce Job failed");
+    }
   }
 
   /**
@@ -593,7 +629,7 @@ private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
    * @return 0 on success, and != 0 upon failure.
    */
   @Override
-  public int run(String[] args) throws Exception {
+  public int run(String[] args) throws IOException {
     boolean verifyChecksum = true;
     String snapshotName = null;
     boolean overwrite = false;
@@ -601,7 +637,7 @@ public int run(String[] args) throws Exception {
     String filesUser = null;
     Path outputRoot = null;
     int filesMode = 0;
-    int mappers = getConf().getInt("mapreduce.job.maps", 1);
+    int mappers = 0;
 
     // Process command line args
     for (int i = 0; i < args.length; i++) {
@@ -669,26 +705,35 @@ public int run(String[] args) throws Exception {
     }
 
     // Check if the snapshot already in-progress
-    if (!overwrite && outputFs.exists(snapshotTmpDir)) {
-      System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
-      System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
-      System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
-      return 1;
+    if (outputFs.exists(snapshotTmpDir)) {
+      if (overwrite) {
+        if (!outputFs.delete(snapshotTmpDir, true)) {
+          System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
+          return 1;
+        }
+      } else {
+        System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
+        System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
+        System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
+        return 1;
+      }
     }
 
     // Step 0 - Extract snapshot files to copy
     final List<Pair<Path, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
+    if (mappers == 0 && files.size() > 0) {
+      mappers = 1 + (files.size() / conf.getInt(CONF_MAP_GROUP, 10));
+      mappers = Math.min(mappers, files.size());
+    }
 
     // Step 1 - Copy fs1:/.snapshot/<snapshot> to  fs2:/.snapshot/.tmp/<snapshot>
     // The snapshot references must be copied before the hfiles otherwise the cleaner
     // will remove them because they are unreferenced.
     try {
-      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, overwrite, conf);
+      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
     } catch (IOException e) {
-      System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
-        " to=" + snapshotTmpDir);
-      e.printStackTrace(System.err);
-      return 1;
+      throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
+        snapshotDir + " to=" + snapshotTmpDir);
     }
 
     // Step 2 - Start MR Job to copy files
@@ -698,24 +743,19 @@ public int run(String[] args) throws Exception {
       if (files.size() == 0) {
         LOG.warn("There are 0 store file to be copied. There may be no data in the table.");
       } else {
-        if (!runCopyJob(inputRoot, outputRoot, files, verifyChecksum,
-            filesUser, filesGroup, filesMode, mappers)) {
-          throw new ExportSnapshotException("Snapshot export failed!");
-        }
+        runCopyJob(inputRoot, outputRoot, files, verifyChecksum,
+                   filesUser, filesGroup, filesMode, mappers);
       }
 
       // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
       if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
-        System.err.println("Snapshot export failed!");
-        System.err.println("Unable to rename snapshot directory from=" +
-                           snapshotTmpDir + " to=" + outputSnapshotDir);
-        return 1;
+        throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
+          snapshotTmpDir + " to=" + outputSnapshotDir);
       }
       return 0;
     } catch (Exception e) {
       LOG.error("Snapshot export failed", e);
-      System.err.println("Snapshot export failed!");
-      e.printStackTrace(System.err);
+      outputFs.delete(snapshotTmpDir, true);
       outputFs.delete(outputSnapshotDir, true);
       return 1;
     }
@@ -754,6 +794,6 @@ static int innerMain(final Configuration conf, final String [] args) throws Exce
   }
 
   public static void main(String[] args) throws Exception {
-     System.exit(innerMain(HBaseConfiguration.create(), args));
+    System.exit(innerMain(HBaseConfiguration.create(), args));
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index 5bd165c817d5..ed777fed3f27 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -295,14 +295,6 @@ public void testExportFailure() throws Exception {
     assertEquals(1, runExportAndInjectFailures(snapshotName, false));
   }
 
-  /**
-   * Check that ExportSnapshot will succede if something fails but the retry succede.
-   */
-  @Test
-  public void testExportRetry() throws Exception {
-    assertEquals(0, runExportAndInjectFailures(snapshotName, true));
-  }
-
   /*
    * Execute the ExportSnapshot job injecting failures
    */

From a11781eae8ff86094f721948cb064377057d3876 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 4 Mar 2014 17:57:40 +0000
Subject: [PATCH 1350/1540] HBASE-10624 Fix 2 new findbugs warnings introduced
 by HBASE-10598

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574161 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/TimeRangeTracker.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
index aebb75377a29..37358b39b983 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
@@ -133,18 +133,18 @@ public synchronized long getMaximumTimestamp() {
     return maximumTimestamp;
   }
 
-  public void write(final DataOutput out) throws IOException {
+  public synchronized void write(final DataOutput out) throws IOException {
     out.writeLong(minimumTimestamp);
     out.writeLong(maximumTimestamp);
   }
 
-  public void readFields(final DataInput in) throws IOException {
+  public synchronized void readFields(final DataInput in) throws IOException {
     this.minimumTimestamp = in.readLong();
     this.maximumTimestamp = in.readLong();
   }
 
   @Override
-  public String toString() {
+  public synchronized String toString() {
     return "[" + minimumTimestamp + "," + maximumTimestamp + "]";
   }
 

From 1d5bc869596986d1d95a6711afb85d400fe1786e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 5 Mar 2014 18:18:47 +0000
Subject: [PATCH 1351/1540] HBASE-10594 Speed up TestRestoreSnapshotFromClient.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574599 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/client/TestRestoreSnapshotFromClient.java  | 11 ++---------
 .../hadoop/hbase/snapshot/SnapshotTestingUtils.java  | 12 ++++++------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 04cb2f8d2323..68768eeb003f 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -121,13 +121,6 @@ public void setup() throws Exception {
     admin.enableTable(tableName);
     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
     snapshot1Rows = TEST_UTIL.countRows(table);
-    admin.disableTable(tableName);
-
-    // take a snapshot of the updated table
-    admin.snapshot(snapshotName1, tableName);
-
-    // re-enable table
-    admin.enableTable(tableName);
     table.close();
   }
 
@@ -141,9 +134,9 @@ public void tearDown() throws Exception {
   @Test
   public void testRestoreSnapshot() throws IOException {
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
-
-    // Restore from snapshot-0
     admin.disableTable(tableName);
+    admin.snapshot(snapshotName1, tableName);
+    // Restore from snapshot-0
     admin.restoreSnapshot(snapshotName0);
     admin.enableTable(tableName);
     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 16803460a848..2613f9ad1c13 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -66,6 +66,7 @@
 public class SnapshotTestingUtils {
 
   private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
+  private static byte[] KEYS = Bytes.toBytes("0123456789");
 
   /**
    * Assert that we don't have any snapshots lists
@@ -421,14 +422,13 @@ public static void createTable(final HBaseTestingUtility util, final byte[] tabl
       HColumnDescriptor hcd = new HColumnDescriptor(family);
       htd.addFamily(hcd);
     }
-    byte[][] splitKeys = new byte[14][];
-    byte[] hex = Bytes.toBytes("123456789abcde");
+    byte[][] splitKeys = new byte[KEYS.length-2][];
     for (int i = 0; i < splitKeys.length; ++i) {
-      splitKeys[i] = new byte[] { hex[i] };
+      splitKeys[i] = new byte[] { KEYS[i+1] };
     }
     util.getHBaseAdmin().createTable(htd, splitKeys);
     waitForTableToBeOnline(util, tableName);
-    assertEquals(15, util.getHBaseAdmin().getTableRegions(tableName).size());
+    assertEquals(KEYS.length-1, util.getHBaseAdmin().getTableRegions(tableName).size());
   }
 
   public static void loadData(final HBaseTestingUtility util, final byte[] tableName, int rows,
@@ -441,8 +441,8 @@ public static void loadData(final HBaseTestingUtility util, final HTable table,
     table.setAutoFlush(false);
 
     // Ensure one row per region
-    assertTrue(rows >= 16);
-    for (byte k0: Bytes.toBytes("0123456789abcdef")) {
+    assertTrue(rows >= KEYS.length);
+    for (byte k0: KEYS) {
       byte[] k = new byte[] { k0 };
       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));

From 22f9c941620db3db422faacd353879e2b59fa37f Mon Sep 17 00:00:00 2001
From: rajeshbabu <rajeshbabu@unknown>
Date: Wed, 5 Mar 2014 18:23:40 +0000
Subject: [PATCH 1352/1540] HBASE-10669 [hbck tool] Usage is wrong for hbck
 tool for -sidelineCorruptHfiles option(Deepak Sharma)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574606 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8d08f0eb73e1..e028b56decf7 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -3663,7 +3663,7 @@ protected HBaseFsck printUsageAndExit() {
     out.println("");
     out.println("  Datafile Repair options: (expert features, use with caution!)");
     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
-    out.println("   -sidelineCorruptHfiles  Quarantine corrupted HFiles.  implies -checkCorruptHfiles");
+    out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
 
     out.println("");
     out.println("  Metadata Repair shortcuts");

From 126f2083ec9fcac26e95902182074ee2a042a08b Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 6 Mar 2014 10:04:03 +0000
Subject: [PATCH 1353/1540] HBASE-10682 region_mover.rb throws "can't convert
 nil into String" for regions moved

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1574825 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/region_mover.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index d2bbfe3ff84c..9bd9d86d8f29 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -412,7 +412,7 @@ def loadRegions(options, hostname)
     end
     $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + 
       count.to_s + " of " + regions.length.to_s + ") from server=" + 
-      currentServer + " to server=" + servername);
+      currentServer.to_s + " to server=" + servername.to_s);
     move(admin, r, servername, currentServer)
   end
 end

From dbfcc276c357443aab32ce9aa24766a739043608 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 8 Mar 2014 00:37:39 +0000
Subject: [PATCH 1354/1540] HBASE-8604 improve reporting of incorrect peer
 address in replication

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1575456 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 98e53ac722ab..6247813d7b65 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -393,7 +393,7 @@ public static void applyClusterKeyToConf(Configuration conf, String key)
   public static String[] transformClusterKey(String key) throws IOException {
     String[] parts = key.split(":");
     if (parts.length != 3) {
-      throw new IOException("Cluster key invalid, the format should be:" +
+      throw new IOException("Cluster key passed " + key + " is invalid, the format should be:" +
           HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
           + HConstants.ZOOKEEPER_ZNODE_PARENT);
     }

From 719232e16765bd6a7c332addc77397913a8658a9 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Mon, 10 Mar 2014 16:05:48 +0000
Subject: [PATCH 1355/1540] HBASE-10712 Backport HBASE-8304 to 0.94 and 0.96
 (Haosdent)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1575985 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/Store.java      |  9 +-
 .../apache/hadoop/hbase/util/FSHDFSUtils.java | 82 +++++++++++++++++++
 .../hadoop/hbase/util/TestFSHDFSUtils.java    | 53 +++++++++++-
 3 files changed, 136 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 5ad34b08f3c7..d42d752ee6d0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -72,12 +72,7 @@
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.ChecksumType;
-import org.apache.hadoop.hbase.util.ClassSize;
-import org.apache.hadoop.hbase.util.CollectionBackedScanner;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.*;
 import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.base.Preconditions;
@@ -639,7 +634,7 @@ public void bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
     //equals() includes UGI instance as part of the comparison
     //and won't work when doing SecureBulkLoad
     //TODO deal with viewFS
-    if (!srcFs.getUri().equals(desFs.getUri())) {
+    if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) {
       LOG.info("File " + srcPath + " on different filesystem than " +
           "destination store - moving to this filesystem.");
       Path tmpPath = getTmpPath();
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
index d7a5f7297963..1e12f9b1db04 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
@@ -22,7 +22,13 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
+import com.google.common.collect.Sets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -37,6 +43,82 @@
  */
 public class FSHDFSUtils extends FSUtils{
   private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
+  private static Class dfsUtilClazz;
+  private static Method getNNAddressesMethod;
+
+  /**
+   * @param fs
+   * @param conf
+   * @return A set containing all namenode addresses of fs
+   */
+  private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,
+                                                      Configuration conf) {
+    Set<InetSocketAddress> addresses = new HashSet<InetSocketAddress>();
+    String serviceName = fs.getCanonicalServiceName();
+
+    if (serviceName.startsWith("ha-hdfs")) {
+      try {
+        if (dfsUtilClazz == null) {
+          dfsUtilClazz = Class.forName("org.apache.hadoop.hdfs.DFSUtil");
+        }
+        if (getNNAddressesMethod == null) {
+          getNNAddressesMethod =
+                  dfsUtilClazz.getMethod("getNNServiceRpcAddresses", Configuration.class);
+        }
+
+        Map<String, Map<String, InetSocketAddress>> addressMap =
+                (Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod
+                        .invoke(null, conf);
+        for (Map.Entry<String, Map<String, InetSocketAddress>> entry : addressMap.entrySet()) {
+          Map<String, InetSocketAddress> nnMap = entry.getValue();
+          for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
+            InetSocketAddress addr = e2.getValue();
+            addresses.add(addr);
+          }
+        }
+      } catch (Exception e) {
+        LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);
+      }
+    } else {
+      URI uri = fs.getUri();
+      InetSocketAddress addr = new InetSocketAddress(uri.getHost(), uri.getPort());
+      addresses.add(addr);
+    }
+
+    return addresses;
+  }
+
+  /**
+   * @param conf the Configuration of HBase
+   * @param srcFs
+   * @param desFs
+   * @return Whether srcFs and desFs are on same hdfs or not
+   */
+  public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {
+    // By getCanonicalServiceName, we could make sure both srcFs and desFs
+    // show a unified format which contains scheme, host and port.
+    String srcServiceName = srcFs.getCanonicalServiceName();
+    String desServiceName = desFs.getCanonicalServiceName();
+
+    if (srcServiceName == null || desServiceName == null) {
+      return false;
+    }
+    if (srcServiceName.equals(desServiceName)) {
+      return true;
+    }
+    if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {
+      //If one serviceName is an HA format while the other is a non-HA format,
+      // maybe they refer to the same FileSystem.
+      //For example, srcFs is "ha-hdfs://nameservices" and desFs is "hdfs://activeNamenode:port"
+      Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);
+      Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);
+      if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {
+        return true;
+      }
+    }
+
+    return false;
+  }
 
   /**
    * Recover the lease from HDFS, retrying multiple times.
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
index 7d477bad2677..dbe0a09a34d2 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
@@ -21,8 +21,12 @@
 
 import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -36,6 +40,7 @@
  */
 @Category(MediumTests.class)
 public class TestFSHDFSUtils {
+  private static final Log LOG = LogFactory.getLog(TestFSHDFSUtils.class);
   private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
   static {
     Configuration conf = HTU.getConfiguration();
@@ -90,6 +95,52 @@ public void testIsFileClosed() throws IOException {
     Mockito.verify(dfs, Mockito.times(1)).isFileClosed(FILE);
   }
 
+  @Test
+  public void testIsSameHdfs() throws IOException {
+    try {
+      Class dfsUtilClazz = Class.forName("org.apache.hadoop.hdfs.DFSUtil");
+      dfsUtilClazz.getMethod("getNNServiceRpcAddresses", Configuration.class);
+    } catch (Exception e) {
+      LOG.info("Skip testIsSameHdfs test case because of the no-HA hadoop version.");
+      return;
+    }
+
+    Configuration conf = HBaseConfiguration.create();
+    Path srcPath = new Path("hdfs://localhost:8020/");
+    Path desPath = new Path("hdfs://127.0.0.1/");
+    FileSystem srcFs = srcPath.getFileSystem(conf);
+    FileSystem desFs = desPath.getFileSystem(conf);
+
+    assertTrue(FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
+
+    desPath = new Path("hdfs://127.0.0.1:8070/");
+    desFs = desPath.getFileSystem(conf);
+    assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
+
+    desPath = new Path("hdfs://127.0.1.1:8020/");
+    desFs = desPath.getFileSystem(conf);
+    assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
+
+    conf.set("fs.defaultFS", "hdfs://haosong-hadoop");
+    conf.set("dfs.nameservices", "haosong-hadoop");
+    conf.set("dfs.federation.nameservices", "haosong-hadoop");
+    conf.set("dfs.ha.namenodes.haosong-hadoop", "nn1,nn2");
+    conf.set("dfs.client.failover.proxy.provider.haosong-hadoop",
+            "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
+
+    conf.set("dfs.namenode.rpc-address.haosong-hadoop.nn1", "127.0.0.1:8020");
+    conf.set("dfs.namenode.rpc-address.haosong-hadoop.nn2", "127.10.2.1:8000");
+    desPath = new Path("/");
+    desFs = desPath.getFileSystem(conf);
+    assertTrue(FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
+
+    conf.set("dfs.namenode.rpc-address.haosong-hadoop.nn1", "127.10.2.1:8020");
+    conf.set("dfs.namenode.rpc-address.haosong-hadoop.nn2", "127.0.0.1:8000");
+    desPath = new Path("/");
+    desFs = desPath.getFileSystem(conf);
+    assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
+  }
+
   /**
    * Version of DFS that has HDFS-4525 in it.
    */
@@ -102,4 +153,4 @@ public boolean isFileClosed(Path f) throws IOException{
       return false;
     }
   }
-}
\ No newline at end of file
+}

From 7f3161a61ad086a7996cc0c78f43e21ede8521ba Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Mon, 10 Mar 2014 23:44:03 +0000
Subject: [PATCH 1356/1540] HBASE-10716 [Configuration]:
 hbase.regionserver.region.split.policy should be part of hbase-default.xml
 (Srikanth Srungarapu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1576133 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/resources/hbase-default.xml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 41a68e7fc1a1..711406f0ddb5 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -563,6 +563,15 @@
         for short operation. For example, short rpc timeout for region server's trying
         to report to active master can benefit quicker master failover process.</description>
   </property>
+  <property>
+    <name>hbase.regionserver.region.split.policy</name>
+    <value>org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy</value>
+    <description>
+      A split policy determines when a region should be split. The various other split policies that
+      are available currently are ConstantSizeRegionSplitPolicy, DisabledRegionSplitPolicy, 
+      DelimitedKeyPrefixRegionSplitPolicy, KeyPrefixRegionSplitPolicy etc.  
+    </description>
+  </property>
 
   <!-- The following properties configure authentication information for
        HBase processes when using Kerberos security.  There are no default

From 330a8e1fc38c20f43ef997a4de79940808ef3c04 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 11 Mar 2014 18:28:18 +0000
Subject: [PATCH 1357/1540] HBASE-9778 Add hint to ExplicitColumnTracker to
 avoid seeking

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1576456 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/performance.xml                    | 15 +++++
 .../org/apache/hadoop/hbase/client/Scan.java  | 18 +++++
 .../regionserver/ExplicitColumnTracker.java   | 18 ++++-
 .../hbase/regionserver/ScanQueryMatcher.java  |  5 +-
 .../TestExplicitColumnTracker.java            | 66 +++++++++++++++++--
 .../hbase/regionserver/TestQueryMatcher.java  | 51 ++++++++++----
 6 files changed, 149 insertions(+), 24 deletions(-)

diff --git a/src/docbkx/performance.xml b/src/docbkx/performance.xml
index b65dab02fc01..463bbc3db2ef 100644
--- a/src/docbkx/performance.xml
+++ b/src/docbkx/performance.xml
@@ -441,6 +441,21 @@ Deferred log flush can be configured on tables via <link
       in the input scan because attribute over-selection is a non-trivial performance penalty over large datasets.
       </para>
     </section>
+    <section xml:id="perf.hbase.client.seek">
+      <title>Avoid scan seeks</title>
+      <para>When columns are selected explicitly with <code>scan.addColumn</code>, HBase will schedule seek operations to seek between the
+      selected columns. When rows have few columns and each column has only a few versions this can be inefficient. A seek operation is generally
+      slower if does not seek at least past 5-10 columns/versions or 512-1024 bytes.</para>
+      <para>In order to opportunistically look ahead a few columns/versions to see if the next column/version can be found that
+      way before a seek operation is scheduled, a new attribute <code>Scan.HINT_LOOKAHEAD</code> can be set the on Scan object. The following code instructs the
+      RegionServer to attempt two iterations of next before a seek is scheduled:<programlisting>
+Scan scan = new Scan();
+scan.addColumn(...);
+scan.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
+table.getScanner(scan);
+</programlisting></para>
+	</section>
+	</section>
     <section xml:id="perf.hbase.mr.input">
         <title>MapReduce - Input Splits</title>
         <para>For MapReduce jobs that use HBase tables as a source, if there a pattern where the "slow" map tasks seem to
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 9a1821134016..4ec4a85937d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -83,7 +83,25 @@ public class Scan extends OperationWithAttributes implements Writable {
   private static final String RAW_ATTR = "_raw_";
   private static final String ONDEMAND_ATTR = "_ondemand_";
   private static final String ISOLATION_LEVEL = "_isolationlevel_";
+
+  /** Scan Hints */
   private static final String SMALL_ATTR = "_small_";
+  /**
+   * EXPERT ONLY.
+   * An integer (not long) indicating to the scanner logic how many times we attempt to retrieve the
+   * next KV before we schedule a reseek.
+   * The right value depends on the size of the average KV. A reseek is more efficient when
+   * it can skip 5-10 KVs or 512B-1KB, or when the next KV is likely found in another HFile block.
+   * Setting this only has any effect when columns were added with
+   * {@link #addColumn(byte[], byte[])}
+   * <pre>{@code
+   * Scan s = new Scan(...);
+   * s.addColumn(...);
+   * s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
+   * }</pre>
+   * Default is 0 (always reseek).
+   */
+  public static final String HINT_LOOKAHEAD = "_look_ahead_";
 
   private static final byte SCAN_VERSION = (byte)2;
   private byte [] startRow = HConstants.EMPTY_START_ROW;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index 419d17f8a138..f953f7ae7943 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -54,6 +54,10 @@ public class ExplicitColumnTracker implements ColumnTracker {
   private final int maxVersions;
   private final int minVersions;
 
+  // hint for the tracker about how many KVs we will attempt to search via next()
+  // before we schedule a (re)seek operation
+  private final int lookAhead; 
+
  /**
   * Contains the list of columns that the ExplicitColumnTracker is tracking.
   * Each ColumnCount instance also tracks how many versions of the requested
@@ -66,6 +70,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
    * Used to eliminate duplicates. */
   private long latestTSOfCurrentColumn;
   private long oldestStamp;
+  private int skipCount;
 
   /**
    * Default constructor.
@@ -74,12 +79,14 @@ public class ExplicitColumnTracker implements ColumnTracker {
    * @param maxVersions maximum versions to return per column
    * @param oldestUnexpiredTS the oldest timestamp we are interested in,
    *  based on TTL 
-   * @param ttl The timeToLive to enforce
+   * @param lookAhead number of KeyValues to look ahead via next before
+   *  (re)seeking
    */
   public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
-      int maxVersions, long oldestUnexpiredTS) {
+      int maxVersions, long oldestUnexpiredTS, int lookAhead) {
     this.maxVersions = maxVersions;
     this.minVersions = minVersions;
+    this.lookAhead = lookAhead;
     this.oldestStamp = oldestUnexpiredTS;
     this.columns = new ColumnCount[columns.size()];
     int i=0;
@@ -135,7 +142,8 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
       if (ret > 0) {
         // The current KV is smaller than the column the ExplicitColumnTracker
         // is interested in, so seek to that column of interest.
-        return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
+        return this.skipCount++ < this.lookAhead ? ScanQueryMatcher.MatchCode.SKIP
+            : ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
       }
 
       // The current KV is bigger than the column the ExplicitColumnTracker
@@ -144,6 +152,7 @@ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
       // column of interest, and check again.
       if (ret <= -1) {
         ++this.index;
+        this.skipCount = 0;
         if (done()) {
           // No more to match, do not include, done with this row.
           return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
@@ -168,6 +177,7 @@ public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int le
     if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
       // Done with versions for this column
       ++this.index;
+      this.skipCount = 0;
       resetTS();
       if (done()) {
         // We have served all the requested columns.
@@ -186,6 +196,7 @@ public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int le
   // Called between every row.
   public void reset() {
     this.index = 0;
+    this.skipCount = 0;
     this.column = this.columns[this.index];
     for(ColumnCount col : this.columns) {
       col.setCount(0);
@@ -225,6 +236,7 @@ public void doneWithColumn(byte [] bytes, int offset, int length) {
       resetTS();
       if (compare <= 0) {
         ++this.index;
+        this.skipCount = 0;
         if (done()) {
           // Will not hit any more columns in this storefile
           this.column = null;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index d50baf403899..567c5289edad 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -176,8 +176,9 @@ public ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
 
       // We can share the ExplicitColumnTracker, diff is we reset
       // between rows, not between storefiles.
-      this.columns = new ExplicitColumnTracker(columns,
-          scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
+      byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD);
+      this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
+          oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr));
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index c69a6bee92da..71febf3a0d9d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -45,9 +45,9 @@ public class TestExplicitColumnTracker extends HBaseTestCase {
   private void runTest(int maxVersions,
                        TreeSet<byte[]> trackColumns,
                        List<byte[]> scannerColumns,
-                       List<MatchCode> expected) throws IOException {
+                       List<MatchCode> expected, int lookAhead) throws IOException {
     ColumnTracker exp = new ExplicitColumnTracker(
-      trackColumns, 0, maxVersions, Long.MIN_VALUE);
+      trackColumns, 0, maxVersions, Long.MIN_VALUE, lookAhead);
 
 
     //Initialize result
@@ -96,7 +96,7 @@ public void testGet_SingleVersion() throws IOException{
     scanner.add(col4);
     scanner.add(col5);
 
-    runTest(maxVersions, columns, scanner, expected);
+    runTest(maxVersions, columns, scanner, expected, 0);
   }
 
   public void testGet_MultiVersion() throws IOException{
@@ -151,9 +151,63 @@ public void testGet_MultiVersion() throws IOException{
     scanner.add(col5);
 
     //Initialize result
-    runTest(maxVersions, columns, scanner, expected);
+    runTest(maxVersions, columns, scanner, expected, 0);
   }
 
+  public void testGet_MultiVersionWithLookAhead() throws IOException{
+    if(PRINT){
+      System.out.println("\nMultiVersion");
+    }
+
+    //Create tracker
+    TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+    //Looking for every other
+    columns.add(col2);
+    columns.add(col4);
+
+    List<ScanQueryMatcher.MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE);                   // col2; 1st version
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2; 2nd version
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE);                   // col4; 1st version
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4; 2nd version
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+    int maxVersions = 2;
+
+    //Create "Scanner"
+    List<byte[]> scanner = new ArrayList<byte[]>();
+    scanner.add(col1);
+    scanner.add(col1);
+    scanner.add(col1);
+    scanner.add(col2);
+    scanner.add(col2);
+    scanner.add(col2);
+    scanner.add(col3);
+    scanner.add(col3);
+    scanner.add(col3);
+    scanner.add(col4);
+    scanner.add(col4);
+    scanner.add(col4);
+    scanner.add(col5);
+    scanner.add(col5);
+    scanner.add(col5);
+
+    //Initialize result
+    runTest(maxVersions, columns, scanner, expected, 2);
+  }
 
   /**
    * hbase-2259
@@ -166,7 +220,7 @@ public void testStackOverflow() throws IOException{
     }
 
     ColumnTracker explicit = new ExplicitColumnTracker(columns, 0, maxVersions,
-        Long.MIN_VALUE);
+        Long.MIN_VALUE, 0);
     for (int i = 0; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
       ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
@@ -194,7 +248,7 @@ public void testInfiniteLoop() throws IOException {
       new ScanQueryMatcher.MatchCode[] {
         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL });
-    runTest(1, columns, scanner, expected);
+    runTest(1, columns, scanner, expected, 0);
   }
 
   @org.junit.Rule
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index 956572c115c9..f5f427305f9a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -83,22 +83,11 @@ public void setUp() throws Exception {
 
   }
 
-  public void testMatch_ExplicitColumns()
-  throws IOException {
+  private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
     //Moving up from the Tracker by using Gets and List<KeyValue> instead
     //of just byte []
 
-    //Expected result
-    List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
-    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
-    expected.add(ScanQueryMatcher.MatchCode.DONE);
-
-    // 2,4,5
-    
+    // 2,4,5    
     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new Store.ScanInfo(fam2,
         0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2),
         EnvironmentEdgeManager.currentTimeMillis() - ttl);
@@ -130,6 +119,42 @@ public void testMatch_ExplicitColumns()
     }
   }
 
+  public void testMatch_ExplicitColumns()
+  throws IOException {
+    //Moving up from the Tracker by using Gets and List<KeyValue> instead
+    //of just byte []
+
+    //Expected result
+    List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.DONE);
+
+    _testMatch_ExplicitColumns(scan, expected);
+  }
+
+  public void testMatch_ExplicitColumnsWithLookAhead()
+  throws IOException {
+    //Moving up from the Tracker by using Gets and List<KeyValue> instead
+    //of just byte []
+
+    //Expected result
+    List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.DONE);
+
+    Scan s = new Scan(scan);
+    s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
+    _testMatch_ExplicitColumns(s, expected);
+  }
+
 
   public void testMatch_Wildcard()
   throws IOException {

From ca8b80433e9929ecb5edca238d5a5cbe2a348d99 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 11 Mar 2014 20:22:19 +0000
Subject: [PATCH 1358/1540] HBASE-9778 Addendum; fix documentation markup

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1576491 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/performance.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/docbkx/performance.xml b/src/docbkx/performance.xml
index 463bbc3db2ef..8fb9559bd9f9 100644
--- a/src/docbkx/performance.xml
+++ b/src/docbkx/performance.xml
@@ -455,7 +455,6 @@ scan.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
 table.getScanner(scan);
 </programlisting></para>
 	</section>
-	</section>
     <section xml:id="perf.hbase.mr.input">
         <title>MapReduce - Input Splits</title>
         <para>For MapReduce jobs that use HBase tables as a source, if there a pattern where the "slow" map tasks seem to

From a72cf164936636f7d8cbf462eaa26339d56d3ed5 Mon Sep 17 00:00:00 2001
From: Andrew Kyle Purtell <apurtell@apache.org>
Date: Wed, 12 Mar 2014 15:42:52 +0000
Subject: [PATCH 1359/1540] HBASE-10718 TestHLogSplit fails when it sets a KV
 size to be negative (Esteban Gutierrez)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1576789 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/KeyValue.java     |  5 ++
 .../hadoop/hbase/TestSerialization.java       | 47 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 0716ff27398c..69412a217c9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -2254,6 +2254,11 @@ public long heapSize() {
   // and it expects the length of the KeyValue to be explicitly passed
   // to it.
   public void readFields(int length, final DataInput in) throws IOException {
+
+    if (length <= 0) {
+      throw new IOException("Failed read " + length + " bytes, stream corrupt?");
+    }
+
     this.length = length;
     this.offset = 0;
     this.bytes = new byte[this.length];
diff --git a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
index 50cb9d472400..bd7afd08ee98 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
@@ -22,7 +22,9 @@
 
 import static org.junit.Assert.*;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -89,6 +91,51 @@ public void testHServerLoadVersioning() throws IOException {
     assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
   }
 
+  @Test public void testCreateKeyValueInvalidNegativeLength() {
+
+    KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),       // 51 bytes
+                                 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
+
+    KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),       // 49 bytes
+                                 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
+
+    KeyValue kv_i = new KeyValue();
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+
+    long l = 0;
+    try {
+      kv_0.write(dos);
+      l  = dos.size();
+      kv_1.write(dos);
+      l += dos.size();
+      assertEquals(151L, l);
+    } catch (IOException e) {
+      fail("Unexpected IOException" + e.getMessage());
+    }
+
+    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+    DataInputStream dis = new DataInputStream(bais);
+
+    try {
+      kv_i.readFields(dis);
+      assertTrue(kv_0.equals(kv_1));
+    } catch (Exception e) {
+      fail("Unexpected Exception" + e.getMessage());
+    }
+
+    // length -1
+    try {
+      // even if we have a good kv now in dis we will just pass length with -1 for simplicity
+      kv_i.readFields(-1, dis);
+      fail("Expected corrupt stream");
+    } catch (Exception e) {
+      assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
+    }
+
+  }
+
   @SuppressWarnings("unchecked")
   @Test public void testHbaseMapWritable() throws Exception {
     HbaseMapWritable<byte [], byte []> hmw =

From cf485612d6482998e5ebc3922b7d0e70d07f4f23 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jmhsieh@apache.org>
Date: Wed, 12 Mar 2014 22:16:25 +0000
Subject: [PATCH 1360/1540] HBASE-10731 Fix environment variables typos in
 scripts (Albert Chu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1576955 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/master-backup.sh   | 4 ++--
 bin/regionservers.sh   | 4 ++--
 bin/rolling-restart.sh | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/master-backup.sh b/bin/master-backup.sh
index 114757c7633b..a79c19e61c5e 100755
--- a/bin/master-backup.sh
+++ b/bin/master-backup.sh
@@ -28,8 +28,8 @@
 #     Default is ${HBASE_CONF_DIR}/backup-masters
 #   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
 #   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
-#   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
-#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
+#   HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   HBASE_SSH_OPTS Options passed to ssh when running remote commands.
 #
 # Modelled after $HADOOP_HOME/bin/slaves.sh.
 
diff --git a/bin/regionservers.sh b/bin/regionservers.sh
index 9759f2b00c41..a86322de9b0c 100755
--- a/bin/regionservers.sh
+++ b/bin/regionservers.sh
@@ -28,8 +28,8 @@
 #     Default is ${HADOOP_CONF_DIR}/regionservers
 #   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
 #   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
-#   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
-#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
+#   HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   HBASE_SSH_OPTS Options passed to ssh when running remote commands.
 #
 # Modelled after $HADOOP_HOME/bin/slaves.sh.
 
diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 23f8d325e81f..11a28c1fee15 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -28,9 +28,9 @@
 #     Default is ${HADOOP_CONF_DIR}/regionservers
 #   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
 #   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
-#   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
-#   HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. 
-#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
+#   HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. 
+#   HBASE_SSH_OPTS Options passed to ssh when running remote commands.
 #
 # Modelled after $HADOOP_HOME/bin/slaves.sh.
 

From 04fdcca56ef5af95119b20a1c81b4608edf8f8de Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 14 Mar 2014 01:04:59 +0000
Subject: [PATCH 1361/1540] HBASE-10722 [0.94]
 HRegion.computeHDFSBlocksDistribution does not account for links and
 reference files.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577404 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 48a6d728f7d7..17b0617ed86a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -104,10 +104,13 @@
 import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseServer;
@@ -736,15 +739,30 @@ public HDFSBlocksDistribution getHDFSBlocksDistribution() {
    * @param tableDescriptor HTableDescriptor of the table
    * @param regionEncodedName encoded name of the region
    * @return The HDFS blocks distribution for the given region.
- * @throws IOException
+   * @throws IOException
    */
   static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
     Configuration conf, HTableDescriptor tableDescriptor,
     String regionEncodedName) throws IOException {
-    HDFSBlocksDistribution hdfsBlocksDistribution =
-      new HDFSBlocksDistribution();
     Path tablePath = FSUtils.getTablePath(FSUtils.getRootDir(conf),
       tableDescriptor.getName());
+    return computeHDFSBlocksDistribution(conf, tableDescriptor, regionEncodedName, tablePath);
+  }
+
+  /**
+   * This is a helper function to compute HDFS block distribution on demand
+   * @param conf configuration
+   * @param tableDescriptor HTableDescriptor of the table
+   * @param regionEncodedName encoded name of the region
+   * @param tablePath The table's directory
+   * @return The HDFS blocks distribution for the given region.
+   * @throws IOException
+   */
+  static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
+    Configuration conf, HTableDescriptor tableDescriptor,
+    String regionEncodedName, Path tablePath) throws IOException {
+    HDFSBlocksDistribution hdfsBlocksDistribution =
+      new HDFSBlocksDistribution();
     FileSystem fs = tablePath.getFileSystem(conf);
 
     for (HColumnDescriptor family: tableDescriptor.getFamilies()) {
@@ -756,6 +774,17 @@ static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
       hfilesStatus = fs.listStatus(storeHomeDir);
 
       for (FileStatus hfileStatus : hfilesStatus) {
+        Path p = hfileStatus.getPath();
+        if (HFileLink.isHFileLink(p)) {
+          hfileStatus = new HFileLink(conf, p).getFileStatus(fs);
+        } else if (StoreFile.isReference(p)) {
+          p = StoreFile.getReferredToFile(p);
+          if (HFileLink.isHFileLink(p)) {
+            hfileStatus = new HFileLink(conf, p).getFileStatus(fs);
+          } else {
+            hfileStatus = fs.getFileStatus(p);
+          }
+        }
         HDFSBlocksDistribution storeFileBlocksDistribution =
           FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0,
           hfileStatus.getLen());

From 66fc295799348249b1acebfaacd9d0b59ceb97fa Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 14 Mar 2014 02:27:52 +0000
Subject: [PATCH 1362/1540] HBASE-10738 AssignmentManager should shut down
 executors on stop

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577407 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/master/AssignmentManager.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index aba6371fd0c8..312a211b6562 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -3639,6 +3639,7 @@ public void write(DataOutput out) throws IOException {
   }
   
   public void stop() {
+    shutdown(); // Stop executor service, etc
     this.timeoutMonitor.interrupt();
     this.timerUpdater.interrupt();
   }

From fcb6071d224814f37a8e941c59afc340d2bd3f7e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 14 Mar 2014 03:58:03 +0000
Subject: [PATCH 1363/1540] HBASE-10642 Add M/R over snapshots to 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577420 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/HDFSBlocksDistribution.java  |  48 +-
 .../hbase/mapreduce/TableMapReduceUtil.java   |  62 +++
 .../mapreduce/TableSnapshotInputFormat.java   | 527 ++++++++++++++++++
 .../hadoop/hbase/HBaseTestingUtility.java     |  59 +-
 .../TestTableSnapshotInputFormat.java         | 300 ++++++++++
 .../TestTableSnapshotInputFormatScan.java     | 211 +++++++
 6 files changed, 1186 insertions(+), 21 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java

diff --git a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
index 6682d172a53c..d0d1dd0d33fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
+++ b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
@@ -36,7 +36,7 @@
 public class HDFSBlocksDistribution {
   private Map<String,HostAndWeight> hostAndWeights = null;
   private long uniqueBlocksTotalWeight = 0;
-    
+
   /**
    * Stores the hostname and weight for that hostname.
    *
@@ -56,7 +56,7 @@ public static class HostAndWeight {
      * Constructor
      * @param host the host name
      * @param weight the weight
-     */    
+     */
     public HostAndWeight(String host, long weight) {
       this.host = host;
       this.weight = weight;
@@ -65,28 +65,28 @@ public HostAndWeight(String host, long weight) {
     /**
      * add weight
      * @param weight the weight
-     */        
+     */
     public void addWeight(long weight) {
       this.weight += weight;
     }
 
     /**
      * @return the host name
-     */            
+     */
     public String getHost() {
       return host;
     }
 
     /**
      * @return the weight
-     */                
+     */
     public long getWeight() {
       return weight;
     }
 
     /**
      * comparator used to sort hosts based on weight
-     */                
+     */
     public static class WeightComparator implements Comparator<HostAndWeight> {
       @Override
       public int compare(HostAndWeight l, HostAndWeight r) {
@@ -97,7 +97,7 @@ public int compare(HostAndWeight l, HostAndWeight r) {
       }
     }
   }
-  
+
   /**
    * Constructor
    */
@@ -135,12 +135,12 @@ public void addHostsAndBlockWeight(String[] hosts, long weight) {
   /**
    * add some weight to the total unique weight
    * @param weight the weight
-   */        
+   */
   private void addUniqueWeight(long weight) {
     uniqueBlocksTotalWeight += weight;
   }
-  
-  
+
+
   /**
    * add some weight to a specific host
    * @param host the host name
@@ -184,14 +184,14 @@ public long getWeight(String host) {
     }
     return weight;
   }
-  
+
   /**
    * @return the sum of all unique blocks' weight
    */
   public long getUniqueBlocksTotalWeight() {
     return uniqueBlocksTotalWeight;
   }
-  
+
   /**
    * return the locality index of a given host
    * @param host the host name
@@ -205,8 +205,8 @@ public float getBlockLocalityIndex(String host) {
     }
     return localityIndex;
   }
-  
-  
+
+
   /**
    * This will add the distribution from input to this object
    * @param otherBlocksDistribution the other hdfs blocks distribution
@@ -221,19 +221,27 @@ public void add(HDFSBlocksDistribution otherBlocksDistribution) {
     }
     addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight());
   }
-  
+
   /**
    * return the sorted list of hosts in terms of their weights
    */
   public List<String> getTopHosts() {
-    NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
-      new HostAndWeight.WeightComparator());
-    orderedHosts.addAll(this.hostAndWeights.values());
-    List<String> topHosts = new ArrayList<String>(orderedHosts.size());
-    for(HostAndWeight haw : orderedHosts.descendingSet()) {
+    HostAndWeight[] hostAndWeights = getTopHostsWithWeights();
+    List<String> topHosts = new ArrayList<String>(hostAndWeights.length);
+    for(HostAndWeight haw : hostAndWeights) {
       topHosts.add(haw.getHost());
     }
     return topHosts;
   }
 
+  /**
+   * return the sorted list of hosts in terms of their weights
+   */
+  public HostAndWeight[] getTopHostsWithWeights() {
+    NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
+      new HostAndWeight.WeightComparator());
+    orderedHosts.addAll(this.hostAndWeights.values());
+    return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]);
+  }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 3d1db62e5387..462bc90bbebd 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.security.token.Token;
 import org.apache.zookeeper.KeeperException;
+import org.cliffc.high_scale_lib.Counter;
 
 /**
  * Utility for {@link TableMapper} and {@link TableReducer}
@@ -289,6 +290,67 @@ public static void initTableMapperJob(List<Scan> scans,
     }
   }
 
+  /**
+   * Sets up the job for reading from a table snapshot. It bypasses hbase
+   * servers and read directly from snapshot files.
+   * 
+   * @param snapshotName
+   *          The name of the snapshot (of a table) to read from.
+   * @param scan
+   *          The scan instance with the columns, time range etc.
+   * @param mapper
+   *          The mapper class to use.
+   * @param outputKeyClass
+   *          The class of the output key.
+   * @param outputValueClass
+   *          The class of the output value.
+   * @param job
+   *          The current job to adjust. Make sure the passed job is carrying
+   *          all necessary HBase configuration.
+   * @param addDependencyJars
+   *          upload HBase jars and jars for any of the configured job classes
+   *          via the distributed cache (tmpjars).
+   * @param tableRootDir
+   *          The directory where the temp table will be created
+   * @throws IOException
+   *           When setting up the details fails.
+   * @see TableSnapshotInputFormat
+   */
+  public static void initTableSnapshotMapperJob(String snapshotName, Scan scan,
+      Class<? extends TableMapper> mapper, Class<?> outputKeyClass, Class<?> outputValueClass,
+      Job job, boolean addDependencyJars, Path tableRootDir) throws IOException {
+
+    TableSnapshotInputFormat.setInput(job, snapshotName, tableRootDir);
+
+    Configuration conf = job.getConfiguration();
+
+    job.setInputFormatClass(TableSnapshotInputFormat.class);
+    if (outputValueClass != null) {
+      job.setMapOutputValueClass(outputValueClass);
+    }
+    if (outputKeyClass != null) {
+      job.setMapOutputKeyClass(outputKeyClass);
+    }
+    job.setMapperClass(mapper);
+    conf.set(TableInputFormat.SCAN, convertScanToString(scan));
+
+    /*
+     * Enable a basic on-heap cache for these jobs. Any BlockCache implementation based on
+     * direct memory will likely cause the map tasks to OOM when opening the region. This
+     * is done here instead of in TableSnapshotRegionRecordReader in case an advanced user
+     * wants to override this behavior in their job.
+     */
+    job.getConfiguration().setFloat(
+      HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+    job.getConfiguration().setFloat("hbase.offheapcache.percentage", 0f);
+
+    if (addDependencyJars) {
+      TableMapReduceUtil.addDependencyJars(job);
+    }
+    // We would need even more libraries that hbase-server depends on
+    TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Counter.class);
+  }
+
   public static void initCredentials(Job job) throws IOException {
     UserProvider provider = UserProvider.instantiate(job.getConfiguration());
 
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
new file mode 100644
index 000000000000..8e009e680b5d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
@@ -0,0 +1,527 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.IsolationLevel;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * TableSnapshotInputFormat allows a MapReduce job to run over a table snapshot. The
+ * job bypasses HBase servers, and directly accesses the underlying files
+ * (hfile, recovered edits, hlogs, etc) directly to provide maximum performance.
+ * The snapshot is not required to be restored or cloned. This also allows to
+ * run the mapreduce job from an online or offline hbase cluster. The snapshot
+ * files can be exported by using the ExportSnapshot tool, to a pure-hdfs
+ * cluster, and this InputFormat can be used to run the mapreduce job directly
+ * over the snapshot files.
+ * <p>
+ * Usage is similar to TableInputFormat.
+ * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job,
+ * boolean, Path)} can be used to configure the job.
+ * 
+ * <pre>
+ * {
+ *   &#064;code
+ *   Job job = new Job(conf);
+ *   Scan scan = new Scan();
+ *   TableMapReduceUtil.initSnapshotMapperJob(snapshotName, scan,
+ *       MyTableMapper.class, MyMapKeyOutput.class,
+ *       MyMapOutputValueWritable.class, job, true, tmpDir);
+ * }
+ * </pre>
+ * <p>
+ * Internally, this input format restores the snapshot into the given tmp
+ * directory. Similar to {@link TableInputFormat} an {@link InputSplit} is created per region.
+ * The region is opened for reading from each RecordReader. An internal
+ * RegionScanner is used to execute the Scan obtained from the user.
+ * <p>
+ * <p>
+ * HBase owns all the data and snapshot files on the filesystem. Only the HBase user can read from
+ * snapshot files and data files. HBase also enforces security because all the requests are handled
+ * by the server layer, and the user cannot read from the data files directly. 
+ * To read from snapshot files directly from the file system, the user who is running the MR job 
+ * must have sufficient permissions to access snapshot and reference files. 
+ * This means that to run mapreduce over snapshot files, the MR job has to be run as the HBase 
+ * user or the user must have group or other priviledges in the filesystem (See HBASE-8369). 
+ * Note that, given other users access to read from snapshot/data files will completely circumvent 
+ * the access control enforced by HBase.
+ */
+public final class TableSnapshotInputFormat extends
+    InputFormat<ImmutableBytesWritable, Result> {
+  // TODO: Snapshots files are owned in fs by the hbase user. There is no
+  // easy way to delegate access.
+
+  private static final String SNAPSHOT_NAME_KEY = "hbase.mr.snapshot.input.name";
+  private static final String TABLE_DIR_KEY = "hbase.mr.snapshot.input.table.dir";
+
+  /** See {@link #getBestLocations(Configuration, HDFSBlocksDistribution)} */
+  private static final String LOCALITY_CUTOFF_MULTIPLIER = 
+      "hbase.tablesnapshotinputformat.locality.cutoff.multiplier";
+  private static final float DEFAULT_LOCALITY_CUTOFF_MULTIPLIER = 0.8f;
+
+  /**
+   * Snapshot region split.
+   */
+  public static final class TableSnapshotRegionSplit extends InputSplit implements
+      Writable {
+    private String regionName;
+    private String[] locations;
+
+    /**
+     * Constructor for serialization.
+     */
+    public TableSnapshotRegionSplit() {
+    }
+
+    /**
+     * Constructor.
+     * 
+     * @param regionName
+     *          Region name
+     * @param locationList
+     *          List of nodes with the region's HDFS blocks, in descending order
+     *          of weight
+     */
+    public TableSnapshotRegionSplit(final String regionName,
+        final List<String> locationList) {
+      this.regionName = regionName;
+
+      // only use the top node
+      List<String> list = locationList.size() > 1 ? locationList.subList(0, 1)
+          : locationList;
+      this.locations = list.toArray(new String[list.size()]);
+    }
+
+    @Override
+    public long getLength() throws IOException, InterruptedException {
+      return locations.length;
+    }
+
+    @Override
+    public String[] getLocations() throws IOException, InterruptedException {
+      return locations;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      regionName = Text.readString(in);
+      int locLength = in.readInt();
+      locations = new String[locLength];
+      for (int i = 0; i < locLength; i++) {
+        locations[i] = Text.readString(in);
+      }
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      Text.writeString(out, regionName);
+      out.writeInt(locations.length);
+      for (String l : locations) {
+        Text.writeString(out, l);
+      }
+    }
+  }
+
+  /**
+   * Snapshot region record reader.
+   */
+  public static final class TableSnapshotRegionRecordReader extends
+      RecordReader<ImmutableBytesWritable, Result> {
+    static final Log LOG = LogFactory.getLog(TableSnapshotRegionRecordReader.class);
+
+    // HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
+    private static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
+
+    private TableSnapshotRegionSplit split;
+    private HRegion region;
+    private Scan scan;
+    private RegionScanner scanner;
+    private List<KeyValue> values;
+    private Result result = null;
+    private ImmutableBytesWritable row = null;
+    private boolean more;
+    private ScanMetrics scanMetrics = null;
+    private TaskAttemptContext context = null;
+    private Method getCounter = null;
+
+    @Override
+    public void initialize(final InputSplit aSplit,
+        final TaskAttemptContext context) throws IOException,
+        InterruptedException {
+      Configuration conf = context.getConfiguration();
+      this.split = (TableSnapshotRegionSplit) aSplit;
+
+      Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+      FileSystem fs = rootDir.getFileSystem(conf);
+
+      String snapshotName = getSnapshotName(conf);
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+          snapshotName, rootDir);
+
+      // load region descriptor
+      String regionName = this.split.regionName;
+      Path regionDir = new Path(snapshotDir, regionName);
+      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+
+      // create scan
+      scan = TableMapReduceUtil.convertStringToScan(conf.get(TableInputFormat.SCAN));
+      // region is immutable, this should be fine, otherwise we have to set the
+      // thread read point...
+      scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
+      scan.setCacheBlocks(false);
+
+      // load table descriptor
+      HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs,
+          snapshotDir);
+      Path tableDir = new Path(conf.get(TABLE_DIR_KEY));
+
+      // open region from the snapshot directory
+      this.region = openRegion(tableDir, fs, conf, hri, htd);
+
+      // create region scanner
+      this.scanner = region.getScanner(scan);
+      values = new ArrayList<KeyValue>();
+      this.more = true;
+      this.scanMetrics = new ScanMetrics();
+
+      if (context != null) {
+        this.context = context;
+        getCounter = retrieveGetCounterWithStringsParams(context);
+      }
+      region.startRegionOperation();
+    }
+
+    private HRegion openRegion(final Path tableDir, final FileSystem fs,
+        final Configuration conf, final HRegionInfo hri,
+        final HTableDescriptor htd) throws IOException {
+      HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, hri, htd, null);
+      r.initialize(null);
+      return r;
+    }
+
+    @Override
+    public boolean nextKeyValue() throws IOException, InterruptedException {
+      values.clear();
+      // RegionScanner.next() has a different contract than
+      // RecordReader.nextKeyValue(). Scanner
+      // indicates no value read by returning empty results. Returns boolean
+      // indicates if more
+      // rows exist AFTER this one
+      if (!more) {
+        updateCounters();
+        return false;
+      }
+      more = scanner.nextRaw(values, scan.getBatch(), null);
+      if (values.isEmpty()) {
+        // we are done
+        updateCounters();
+        return false;
+      }
+      for (KeyValue kv : values) {
+        this.scanMetrics.countOfBytesInResults.inc(kv.getLength());
+      }
+      this.result = new Result(values);
+      if (this.row == null) {
+        this.row = new ImmutableBytesWritable();
+      }
+      this.row.set(result.getRow());
+
+      return true;
+    }
+
+    @Override
+    public ImmutableBytesWritable getCurrentKey() throws IOException,
+        InterruptedException {
+      return row;
+    }
+
+    @Override
+    public Result getCurrentValue() throws IOException, InterruptedException {
+      return result;
+    }
+
+    @Override
+    public float getProgress() throws IOException, InterruptedException {
+      return 0;
+    }
+
+    @Override
+    public void close() throws IOException {
+      try {
+        if (this.scanner != null) {
+          this.scanner.close();
+        }
+      } finally {
+        if (region != null) {
+          region.closeRegionOperation();
+          region.close(true);
+        }
+      }
+    }
+
+    /**
+     * If hbase runs on new version of mapreduce, RecordReader has access to
+     * counters thus can update counters based on scanMetrics.
+     * If hbase runs on old version of mapreduce, it won't be able to get
+     * access to counters and TableRecorderReader can't update counter values.
+     * @throws IOException
+     */
+    private void updateCounters() throws IOException {
+      // we can get access to counters only if hbase uses new mapreduce APIs
+      if (this.getCounter == null) {
+        return;
+      }
+
+      MetricsTimeVaryingLong[] mlvs =
+        scanMetrics.getMetricsTimeVaryingLongArray();
+
+      try {
+        for (MetricsTimeVaryingLong mlv : mlvs) {
+          Counter ct = (Counter)this.getCounter.invoke(context,
+            HBASE_COUNTER_GROUP_NAME, mlv.getName());
+          ct.increment(mlv.getCurrentIntervalValue());
+        }
+      } catch (Exception e) {
+        LOG.debug("can't update counter." + StringUtils.stringifyException(e));
+      }
+    }
+
+   /**
+     * In new mapreduce APIs, TaskAttemptContext has two getCounter methods
+     * Check if getCounter(String, String) method is available.
+     * @return The getCounter method or null if not available.
+     * @throws IOException
+     */
+    private Method retrieveGetCounterWithStringsParams(TaskAttemptContext context)
+    throws IOException {
+      Method m = null;
+      try {
+        m = context.getClass().getMethod("getCounter",
+          new Class [] {String.class, String.class});
+      } catch (SecurityException e) {
+        throw new IOException("Failed test for getCounter", e);
+      } catch (NoSuchMethodException e) {
+        // Ignore
+      }
+      return m;
+    }
+
+  }
+
+  @Override
+  public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
+      final InputSplit split, final TaskAttemptContext context)
+      throws IOException {
+    return new TableSnapshotRegionRecordReader();
+  }
+
+  @Override
+  public List<InputSplit> getSplits(final JobContext job) throws IOException,
+      InterruptedException {
+    Configuration conf = job.getConfiguration();
+    String snapshotName = getSnapshotName(job.getConfiguration());
+
+    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = rootDir.getFileSystem(conf);
+
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+        snapshotName, rootDir);
+
+    Set<String> snapshotRegionNames = SnapshotReferenceUtil
+        .getSnapshotRegionNames(fs, snapshotDir);
+    if (snapshotRegionNames == null) {
+      throw new IllegalArgumentException("Snapshot is empty");
+    }
+
+    // load table descriptor
+    HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs,
+        snapshotDir);
+
+    Scan scan = TableMapReduceUtil.convertStringToScan(conf.get(TableInputFormat.SCAN));
+    Path tableDir = new Path(conf.get(TABLE_DIR_KEY));
+
+    List<InputSplit> splits = new ArrayList<InputSplit>(
+        snapshotRegionNames.size());
+    for (String regionName : snapshotRegionNames) {
+      // load region descriptor
+      Path regionDir = new Path(snapshotDir, regionName);
+      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+
+      if (keyRangesOverlap(scan.getStartRow(), scan.getStopRow(),
+          hri.getStartKey(), hri.getEndKey())) {
+        // compute HDFS locations from snapshot files (which will get the locations for
+        // referred hfiles)
+        List<String> hosts = getBestLocations(conf,
+          HRegion.computeHDFSBlocksDistribution(conf, htd, hri.getEncodedName(), tableDir));
+
+        int len = Math.min(3, hosts.size());
+        hosts = hosts.subList(0, len);
+        splits.add(new TableSnapshotRegionSplit(regionName, hosts));
+      }
+    }
+
+    return splits;
+  }
+
+  private boolean keyRangesOverlap(final byte[] start1, final byte[] end1,
+      final byte[] start2, final byte[] end2) {
+    return (end2.length == 0 || start1.length == 0 || Bytes.compareTo(start1,
+        end2) < 0)
+        && (end1.length == 0 || start2.length == 0 || Bytes.compareTo(start2,
+            end1) < 0);
+  }
+
+  /**
+   * This computes the locations to be passed from the InputSplit. MR/Yarn schedulers does not take
+   * weights into account, thus will treat every location passed from the input split as equal. We
+   * do not want to blindly pass all the locations, since we are creating one split per region, and
+   * the region's blocks are all distributed throughout the cluster unless favorite node assignment
+   * is used. On the expected stable case, only one location will contain most of the blocks as 
+   * local.
+   * On the other hand, in favored node assignment, 3 nodes will contain highly local blocks. Here
+   * we are doing a simple heuristic, where we will pass all hosts which have at least 80%
+   * (hbase.tablesnapshotinputformat.locality.cutoff.multiplier) as much block locality as the top
+   * host with the best locality.
+   */
+  @VisibleForTesting
+  List<String> getBestLocations(Configuration conf, HDFSBlocksDistribution blockDistribution) {
+    List<String> locations = new ArrayList<String>(3);
+
+    HostAndWeight[] hostAndWeights = blockDistribution.getTopHostsWithWeights();
+
+    if (hostAndWeights.length == 0) {
+      return locations;
+    }
+
+    HostAndWeight topHost = hostAndWeights[0];
+    locations.add(topHost.getHost());
+
+    // Heuristic: filter all hosts which have at least cutoffMultiplier % of block locality
+    double cutoffMultiplier
+      = conf.getFloat(LOCALITY_CUTOFF_MULTIPLIER, DEFAULT_LOCALITY_CUTOFF_MULTIPLIER);
+
+    double filterWeight = topHost.getWeight() * cutoffMultiplier;
+
+    for (int i = 1; i < hostAndWeights.length; i++) {
+      if (hostAndWeights[i].getWeight() >= filterWeight) {
+        locations.add(hostAndWeights[i].getHost());
+      } else {
+        break;
+      }
+    }
+
+    return locations;
+  }
+
+  /**
+   * Set job input.
+   * 
+   * @param job
+   *          The job
+   * @param snapshotName
+   *          The snapshot name
+   * @param restoreDir
+   *          The directory where the temp table will be created
+   * @throws IOException
+   *           on error
+   */
+  public static void setInput(final Job job, final String snapshotName,
+      final Path restoreDir) throws IOException {
+    Configuration conf = job.getConfiguration();
+    conf.set(SNAPSHOT_NAME_KEY, snapshotName);
+
+    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = rootDir.getFileSystem(conf);
+
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+        snapshotName, rootDir);
+    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils
+        .readSnapshotInfo(fs, snapshotDir);
+
+    // load table descriptor
+    HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs,
+        snapshotDir);
+
+    Path tableDir = new Path(restoreDir, htd.getNameAsString());
+    conf.set(TABLE_DIR_KEY, tableDir.toString());
+
+    MonitoredTask status = TaskMonitor.get().createStatus(
+        "Restoring  snapshot '" + snapshotName + "' to directory " + tableDir);
+    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
+
+    RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
+        snapshotDesc, snapshotDir, htd, tableDir, monitor, status);
+    helper.restoreHdfsRegions();
+  }
+
+  private static String getSnapshotName(final Configuration conf) {
+    String snapshotName = conf.get(SNAPSHOT_NAME_KEY);
+    if (snapshotName == null) {
+      throw new IllegalArgumentException("Snapshot name must be provided");
+    }
+    return snapshotName;
+  }
+}
+
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index f56aba1986a1..d55b7b5b495c 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1045,6 +1045,18 @@ public int loadTable(final HTable t, final byte[] f) throws IOException {
    * @throws IOException
    */
   public int loadTable(final HTable t, final byte[][] f) throws IOException {
+    return loadTable(t, f, null);
+  }
+
+  /**
+   * Load table of multiple column families with rows from 'aaa' to 'zzz'.
+   * @param t Table
+   * @param f Array of Families to load
+   * @param value the values of the KVs. If null is passed, the row key is used as value
+   * @return Count of rows loaded.
+   * @throws IOException
+   */
+  public int loadTable(final HTable t, final byte[][] f, byte[] value) throws IOException {
     t.setAutoFlush(false);
     byte[] k = new byte[3];
     int rowCount = 0;
@@ -1056,7 +1068,7 @@ public int loadTable(final HTable t, final byte[][] f) throws IOException {
           k[2] = b3;
           Put put = new Put(k);
           for (int i = 0; i < f.length; i++) {
-            put.add(f[i], null, k);
+            put.add(f[i], null, value != null ? value : k);
           }
           t.put(put);
           rowCount++;
@@ -1067,6 +1079,51 @@ public int loadTable(final HTable t, final byte[][] f) throws IOException {
     return rowCount;
   }
 
+  /** A tracker for tracking and validating table rows
+   * generated with {@link HBaseTestingUtility#loadTable(HTable, byte[])}
+   */
+  public static class SeenRowTracker {
+    int dim = 'z' - 'a' + 1;
+    int[][][] seenRows = new int[dim][dim][dim]; //count of how many times the row is seen
+    byte[] startRow;
+    byte[] stopRow;
+
+    public SeenRowTracker(byte[] startRow, byte[] stopRow) {
+      this.startRow = startRow;
+      this.stopRow = stopRow;
+    }
+
+    int i(byte b) {
+      return b - 'a';
+    }
+
+    public void addRow(byte[] row) {
+      seenRows[i(row[0])][i(row[1])][i(row[2])]++;
+    }
+
+    /** Validate that all the rows between startRow and stopRow are seen exactly once, and
+     * all other rows none
+     */
+    public void validate() {
+      for (byte b1 = 'a'; b1 <= 'z'; b1++) {
+        for (byte b2 = 'a'; b2 <= 'z'; b2++) {
+          for (byte b3 = 'a'; b3 <= 'z'; b3++) {
+            int count = seenRows[i(b1)][i(b2)][i(b3)];
+            int expectedCount = 0;
+            if (Bytes.compareTo(new byte[] {b1,b2,b3}, startRow) >= 0
+                && Bytes.compareTo(new byte[] {b1,b2,b3}, stopRow) < 0) {
+              expectedCount = 1;
+            }
+            if (count != expectedCount) {
+              String row = new String(new byte[] {b1,b2,b3});
+              throw new RuntimeException("Row:" + row + " has a seen count of " + count + " instead of " + expectedCount);
+            }
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Load region with rows from 'aaa' to 'zzz'.
    * @param r Region
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
new file mode 100644
index 000000000000..2aac1734f4c5
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestTableSnapshotInputFormat {
+
+  private final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final int NUM_REGION_SERVERS = 2;
+  private static final byte[][] FAMILIES = {Bytes.toBytes("f1"), Bytes.toBytes("f2")};
+  public static byte[] bbb = Bytes.toBytes("bbb");
+  public static byte[] yyy = Bytes.toBytes("yyy");
+
+  public void setupCluster() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster(NUM_REGION_SERVERS);
+  }
+
+  public void tearDownCluster() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  private static void setupConf(Configuration conf) {
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  public static enum TestTableSnapshotCounters {
+    VALIDATION_ERROR
+  }
+
+  public static class TestTableSnapshotMapper
+    extends TableMapper<ImmutableBytesWritable, NullWritable> {
+    @Override
+    protected void map(ImmutableBytesWritable key, Result value,
+        Context context) throws IOException, InterruptedException {
+      // Validate a single row coming from the snapshot, and emit the row key
+      verifyRowFromMap(key, value);
+      context.write(key, NullWritable.get());
+    }
+  }
+
+  public static class TestTableSnapshotReducer
+    extends Reducer<ImmutableBytesWritable, NullWritable, NullWritable, NullWritable> {
+    HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
+    @Override
+    protected void reduce(ImmutableBytesWritable key, Iterable<NullWritable> values,
+       Context context) throws IOException, InterruptedException {
+      rowTracker.addRow(key.get());
+    }
+
+    @Override
+    protected void cleanup(Context context) throws IOException,
+        InterruptedException {
+      rowTracker.validate();
+    }
+  }
+
+  public static void createTableAndSnapshot(HBaseTestingUtility util, byte[] tableName,
+      String snapshotName, int numRegions)
+      throws Exception {
+    try {
+      util.deleteTable(tableName);
+    } catch(Exception ex) {
+      // ignore
+    }
+
+    if (numRegions > 1) {
+      util.createTable(tableName, FAMILIES, 1, bbb, yyy, numRegions);
+    } else {
+      util.createTable(tableName, FAMILIES);
+    }
+    HBaseAdmin admin = util.getHBaseAdmin();
+
+    // put some stuff in the table
+    HTable table = new HTable(util.getConfiguration(), tableName);
+    util.loadTable(table, FAMILIES);
+
+    Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR));
+    FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
+
+    SnapshotTestingUtils.createSnapshotAndValidate(admin, Bytes.toString(tableName),
+        Arrays.asList(FAMILIES), null, snapshotName, rootDir, fs, true);
+
+    // load different values
+    byte[] value = Bytes.toBytes("after_snapshot_value");
+    util.loadTable(table, FAMILIES, value);
+
+    // cause flush to create new files in the region
+    admin.flush(tableName);
+    table.close();
+  }
+
+  @Test
+  public void testWithMockedMapReduceSingleRegion() throws Exception {
+    testWithMockedMapReduce(UTIL, "testWithMockedMapReduceSingleRegion", 1, 1);
+  }
+
+  @Test
+  public void testWithMockedMapReduceMultiRegion() throws Exception {
+    testWithMockedMapReduce(UTIL, "testWithMockedMapReduceMultiRegion", 10, 8);
+  }
+
+  public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName, int numRegions, int expectedNumSplits)
+      throws Exception {
+    setupCluster();
+    byte[] tableName = Bytes.toBytes("testWithMockedMapReduce");
+    try {
+      createTableAndSnapshot(util, tableName, snapshotName, numRegions);
+
+      Job job = new Job(util.getConfiguration());
+      Path tmpTableDir = util.getDataTestDir(snapshotName);
+      Scan scan = new Scan(bbb, yyy); // limit the scan
+
+      TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
+          scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+          NullWritable.class, job, false, tmpTableDir);
+
+      verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, bbb, yyy);
+
+    } finally {
+      util.getHBaseAdmin().deleteSnapshot(snapshotName);
+      util.deleteTable(tableName);
+      tearDownCluster();
+    }
+  }
+
+  private void verifyWithMockedMapReduce(Job job, int numRegions, int expectedNumSplits,
+      byte[] startRow, byte[] stopRow)
+      throws IOException, InterruptedException {
+    TableSnapshotInputFormat tsif = new TableSnapshotInputFormat();
+    List<InputSplit> splits = tsif.getSplits(job);
+
+    Assert.assertEquals(expectedNumSplits, splits.size());
+
+    HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
+
+    for (int i = 0; i < splits.size(); i++) {
+      // validate input split
+      InputSplit split = splits.get(i);
+      Assert.assertTrue(split instanceof TableSnapshotRegionSplit);
+
+      // validate record reader
+      TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
+      when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
+      RecordReader<ImmutableBytesWritable, Result> rr = tsif.createRecordReader(split, taskAttemptContext);
+      rr.initialize(split, taskAttemptContext);
+
+      // validate we can read all the data back
+      while (rr.nextKeyValue()) {
+        byte[] row = rr.getCurrentKey().get();
+        verifyRowFromMap(rr.getCurrentKey(), rr.getCurrentValue());
+        rowTracker.addRow(row);
+      }
+
+      rr.close();
+    }
+
+    // validate all rows are seen
+    rowTracker.validate();
+  }
+
+  public static void verifyRowFromMap(ImmutableBytesWritable key, Result result) throws IOException {
+    byte[] row = key.get();
+    for (KeyValue kv : result.list()) {
+      //assert that all Cells in the Result have the same key
+     Assert.assertEquals(0, Bytes.compareTo(row, 0, row.length,
+         kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()));
+    }
+
+    for (int j = 0; j < FAMILIES.length; j++) {
+      byte[] actual = result.getValue(FAMILIES[j], null);
+      Assert.assertArrayEquals("Row in snapshot does not match, expected:" + Bytes.toString(row)
+          + " ,actual:" + Bytes.toString(actual), row, actual);
+    }
+  }
+
+  @Test
+  public void testWithMapReduceSingleRegion() throws Exception {
+    testWithMapReduce(UTIL, "testWithMapReduceSingleRegion", 1, 1, false);
+  }
+
+  @Test
+  public void testWithMapReduceMultiRegion() throws Exception {
+    testWithMapReduce(UTIL, "testWithMapReduceMultiRegion", 10, 8, false);
+  }
+
+  @Test
+  // run the MR job while HBase is offline
+  public void testWithMapReduceAndOfflineHBaseMultiRegion() throws Exception {
+    testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 8, true);
+  }
+
+  private void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
+      int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
+    setupCluster();
+    util.startMiniMapReduceCluster();
+    try {
+      Path tableDir = util.getDataTestDir(snapshotName);
+      byte[] tableName = Bytes.toBytes("testWithMapReduce");
+      doTestWithMapReduce(util, tableName, snapshotName, tableDir, numRegions,
+        expectedNumSplits, shutdownCluster);
+    } finally {
+      util.shutdownMiniMapReduceCluster();
+      tearDownCluster();
+    }
+  }
+
+  // this is also called by the IntegrationTestTableSnapshotInputFormat
+  public static void doTestWithMapReduce(HBaseTestingUtility util, byte[] tableName,
+      String snapshotName, Path tableDir, int numRegions, int expectedNumSplits, boolean shutdownCluster)
+          throws Exception {
+
+    //create the table and snapshot
+    createTableAndSnapshot(util, tableName, snapshotName, numRegions);
+
+    if (shutdownCluster) {
+      util.shutdownMiniHBaseCluster();
+    }
+
+    try {
+      // create the job
+      Job job = new Job(util.getConfiguration());
+      Scan scan = new Scan(bbb, yyy); // limit the scan
+
+      job.setJarByClass(util.getClass());
+      TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TestTableSnapshotInputFormat.class);
+
+      TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
+        scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+        NullWritable.class, job, true, tableDir);
+
+      job.setReducerClass(TestTableSnapshotInputFormat.TestTableSnapshotReducer.class);
+      job.setNumReduceTasks(1);
+      job.setOutputFormatClass(NullOutputFormat.class);
+
+      Assert.assertTrue(job.waitForCompletion(true));
+    } finally {
+      if (!shutdownCluster) {
+        util.getHBaseAdmin().deleteSnapshot(snapshotName);
+        util.deleteTable(tableName);
+      }
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java
new file mode 100644
index 000000000000..69c988f39874
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * <p>
+ * Tests scanning a snapshot. Tests various scan start and stop row scenarios.
+ * This is set in a scan and tested in a MapReduce job to see if that is handed
+ * over and done properly too.
+ * </p>
+ */
+@Category(LargeTests.class)
+public class TestTableSnapshotInputFormatScan {
+
+  static final Log LOG = LogFactory.getLog(TestTableSnapshotInputFormatScan.class);
+  static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  static final byte[] TABLE_NAME = Bytes.toBytes("scantest");
+  static final byte[] SNAPSHOT_NAME = Bytes.toBytes("scantest_snaphot");
+  static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
+  static final String KEY_STARTROW = "startRow";
+  static final String KEY_LASTROW = "stpRow";
+
+  private static HTable table = null;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // config snapshot support
+    TEST_UTIL.getConfiguration().setBoolean(
+        SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+    TEST_UTIL.getConfiguration().setBoolean(
+        "hbase.master.enabletable.roundrobin", true);
+
+    // switch TIF to log at DEBUG level
+    TEST_UTIL.enableDebug(TableSnapshotInputFormat.class);
+
+    // start mini hbase cluster
+    TEST_UTIL.startMiniCluster(3);
+
+    // create and fill table
+    table = TEST_UTIL.createTable(TABLE_NAME, INPUT_FAMILY);
+    TEST_UTIL.createMultiRegions(table, INPUT_FAMILY);
+    TEST_UTIL.loadTable(table, INPUT_FAMILY);
+    TEST_UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
+    TEST_UTIL.getHBaseAdmin().snapshot(SNAPSHOT_NAME, TABLE_NAME);
+    TEST_UTIL.getHBaseAdmin().enableTable(TABLE_NAME);
+
+    // start MR cluster
+    TEST_UTIL.startMiniMapReduceCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   * 
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToEmpty() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, null, null);
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   * 
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToAPP() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, "app", "apo");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   * 
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToBBA() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, "bba", "baz");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   * 
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToBBB() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, "bbb", "bba");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   * 
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testScanEmptyToOPP() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    testScan(null, "opp", "opo");
+  }
+
+  /**
+   * Tests a MR scan using specific start and stop rows.
+   * 
+   * @throws IOException
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   */
+  protected void testScan(String start, String stop, String last)
+      throws IOException, InterruptedException, ClassNotFoundException {
+    String jobName = "Scan" + (start != null ? start.toUpperCase() : "Empty")
+        + "To" + (stop != null ? stop.toUpperCase() : "Empty");
+    LOG.info("Before map/reduce startup - job " + jobName);
+    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
+    Scan scan = new Scan();
+    scan.addFamily(INPUT_FAMILY);
+    if (start != null) {
+      scan.setStartRow(Bytes.toBytes(start));
+    }
+    c.set(KEY_STARTROW, start != null ? start : "");
+    if (stop != null) {
+      scan.setStopRow(Bytes.toBytes(stop));
+    }
+    c.set(KEY_LASTROW, last != null ? last : "");
+    LOG.info("scan before: " + scan);
+    Job job = new Job(c, jobName);
+
+    FileSystem fs = FileSystem.get(c);
+    Path tmpDir = new Path("/" + UUID.randomUUID());
+    fs.mkdirs(tmpDir);
+    try {
+      TableMapReduceUtil.initTableSnapshotMapperJob(Bytes.toString(SNAPSHOT_NAME),
+          scan, TestTableInputFormatScanBase.ScanMapper.class,
+          ImmutableBytesWritable.class, ImmutableBytesWritable.class, job,
+          false, tmpDir);
+      job.setReducerClass(TestTableInputFormatScanBase.ScanReducer.class);
+      job.setNumReduceTasks(1); // one to get final "first" and "last" key
+      FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
+      LOG.info("Started " + job.getJobName());
+      assertTrue(job.waitForCompletion(true));
+      LOG.info("After map/reduce completion - job " + jobName);
+    } finally {
+      fs.delete(tmpDir, true);
+    }
+
+  }
+
+}

From 22ad2dcf4a12970be8cbf2489630f1f1e56ec4d5 Mon Sep 17 00:00:00 2001
From: jxiang <jxiang@unknown>
Date: Fri, 14 Mar 2014 15:59:53 +0000
Subject: [PATCH 1364/1540] HBASE-10745 Access ShutdownHook#fsShutdownHooks
 should be synchronized

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577591 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/ShutdownHook.java | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
index 5f16535dcc37..483c4b9e5018 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
@@ -191,12 +191,12 @@ private static Runnable suppressHdfsShutdownHook(final FileSystem fs) {
       if (hdfsClientFinalizer == null) {
         throw new RuntimeException("Client finalizer is null, can't suppress!");
       }
-      if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) &&
-          !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) {
-        throw new RuntimeException("Failed suppression of fs shutdown hook: " +
-          hdfsClientFinalizer);
-      }
       synchronized (fsShutdownHooks) {
+        if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) &&
+            !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) {
+          throw new RuntimeException("Failed suppression of fs shutdown hook: " +
+            hdfsClientFinalizer);
+        }
         Integer refs = fsShutdownHooks.get(hdfsClientFinalizer);
         fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1);
       }

From e5c5fb16f62bb6bd50cd6ef982e8bb987a797324 Mon Sep 17 00:00:00 2001
From: rajeshbabu <rajeshbabu@unknown>
Date: Fri, 14 Mar 2014 18:51:37 +0000
Subject: [PATCH 1365/1540] HBASE-10549 When there is a hole,
 LoadIncrementalHFiles will hang in an infinite loop.(yuanxinen)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577659 13f79535-47bb-0310-9956-ffa450edef68
---
 .../mapreduce/LoadIncrementalHFiles.java      | 24 ++++++
 ...estLoadIncrementalHFilesSplitRecovery.java | 85 +++++++++++++++++++
 2 files changed, 109 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 633e5faa24c6..5536a58d67a9 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -487,6 +487,30 @@ protected List<LoadQueueItem> groupOrSplit(Multimap<ByteBuffer, LoadQueueItem> r
       idx = -(idx + 1) - 1;
     }
     final int indexForCallable = idx;
+    
+    /**
+     * we can consider there is a region hole in following conditions. 1) if idx < 0,then first
+     * region info is lost. 2) if the endkey of a region is not equal to the startkey of the next
+     * region. 3) if the endkey of the last region is not empty.
+     */
+    if (indexForCallable < 0) {
+      throw new IOException("The first region info for table "
+          + Bytes.toString(table.getTableName())
+          + " cann't be found in .META..Please use hbck tool to fix it first.");
+    } else if ((indexForCallable == startEndKeys.getFirst().length - 1)
+        && !Bytes.equals(startEndKeys.getSecond()[indexForCallable], HConstants.EMPTY_BYTE_ARRAY)) {
+      throw new IOException("The last region info for table "
+          + Bytes.toString(table.getTableName())
+          + " cann't be found in .META..Please use hbck tool to fix it first.");
+    } else if (indexForCallable + 1 < startEndKeys.getFirst().length
+        && !(Bytes.compareTo(startEndKeys.getSecond()[indexForCallable],
+          startEndKeys.getFirst()[indexForCallable + 1]) == 0)) {
+      throw new IOException("The endkey of one region for table "
+          + Bytes.toString(table.getTableName())
+          + " is not equal to the startkey of the next region in .META.."
+          + "Please use hbck tool to fix it first.");
+    }
+    
     boolean lastKeyInRange =
       Bytes.compareTo(last, startEndKeys.getSecond()[idx]) < 0 ||
       Bytes.equals(startEndKeys.getSecond()[idx], HConstants.EMPTY_BYTE_ARRAY);
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
index 391fd37f4df6..73d549af0f3b 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java
@@ -37,11 +37,15 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
@@ -123,6 +127,29 @@ private void setupTable(String table, int cfs) throws IOException {
     }
   }
 
+  /**
+   * Creates a table with given table name,specified number of column families<br>
+   * and splitkeys if the table does not already exist.
+   * @param table
+   * @param cfs
+   * @param SPLIT_KEYS
+   */
+  private void setupTableWithSplitkeys(String table, int cfs, byte[][] SPLIT_KEYS)
+      throws IOException {
+    try {
+      LOG.info("Creating table " + table);
+      HTableDescriptor htd = new HTableDescriptor(table);
+      for (int i = 0; i < cfs; i++) {
+        htd.addFamily(new HColumnDescriptor(family(i)));
+      }
+
+      util.getHBaseAdmin().createTable(htd, SPLIT_KEYS);
+    } catch (TableExistsException tee) {
+      LOG.info("Table " + table + " already exists");
+    }
+  }
+
+  
   private Path buildBulkFiles(String table, int value) throws Exception {
     Path dir = util.getDataTestDir(table);
     Path bulk1 = new Path(dir, table+value);
@@ -393,6 +420,64 @@ protected List<LoadQueueItem> groupOrSplit(
 
     fail("doBulkLoad should have thrown an exception");
   }
+  
+  @Test
+  public void testGroupOrSplitWhenRegionHoleExistsInMeta() throws Exception {
+    String tableName = "testGroupOrSplitWhenRegionHoleExistsInMeta";
+    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000100") };
+
+    setupTableWithSplitkeys(tableName, 10, SPLIT_KEYS);
+    HTable table = new HTable(util.getConfiguration(), Bytes.toBytes(tableName));
+    Path dir = buildBulkFiles(tableName, 2);
+
+    final AtomicInteger countedLqis = new AtomicInteger();
+    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
+      util.getConfiguration()) {
+      
+    protected List<LoadQueueItem> groupOrSplit(
+        Multimap<ByteBuffer, LoadQueueItem> regionGroups,
+        final LoadQueueItem item, final HTable htable,
+        final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
+      List<LoadQueueItem> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
+      if (lqis != null) {
+        countedLqis.addAndGet(lqis.size());
+      }
+      return lqis;
+    }
+  };
+
+    // do bulkload when there is no region hole in hbase:meta.
+    try {
+      loader.doBulkLoad(dir, table);
+    } catch (Exception e) {
+      LOG.error("exeception=", e);
+    }
+    // check if all the data are loaded into the table.
+    this.assertExpectedTable(tableName, ROWCOUNT, 2);
+
+    dir = buildBulkFiles(tableName, 3);
+
+    // Mess it up by leaving a hole in the hbase:meta
+    CatalogTracker ct = new CatalogTracker(util.getConfiguration());
+    List<HRegionInfo> regionInfos = MetaReader.getTableRegions(ct, Bytes.toBytes(tableName));
+    for (HRegionInfo regionInfo : regionInfos) {
+      if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
+        MetaEditor.deleteRegion(ct, regionInfo);
+        break;
+      }
+    }
+
+    try {
+      loader.doBulkLoad(dir, table);
+    } catch (Exception e) {
+      LOG.error("exeception=", e);
+      assertTrue("IOException expected", e instanceof IOException);
+    }
+
+    table.close();
+
+    this.assertExpectedTable(tableName, ROWCOUNT, 2);
+  }
 
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =

From 76aa400b1ba3820e3c6970f18e5e9548cd4523d5 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 14 Mar 2014 20:36:39 +0000
Subject: [PATCH 1366/1540] HBASE-10514 Forward port HBASE-10466, possible data
 loss when failed flushes

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577694 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 50 +++++++++++++++----
 .../hadoop/hbase/regionserver/MemStore.java   | 18 ++++++-
 .../hadoop/hbase/regionserver/Store.java      |  9 +++-
 3 files changed, 63 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 17b0617ed86a..e6ad4906154b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1042,22 +1042,25 @@ private List<StoreFile> doClose(
     }
 
     status.setStatus("Disabling compacts and flushes for region");
-    boolean wasFlushing = false;
     synchronized (writestate) {
       // Disable compacting and flushing by background threads for this
       // region.
       writestate.writesEnabled = false;
-      wasFlushing = writestate.flushing;
       LOG.debug("Closing " + this + ": disabling compactions & flushes");
       waitForFlushesAndCompactions();
     }
     // If we were not just flushing, is it worth doing a preflush...one
     // that will clear out of the bulk of the memstore before we put up
     // the close flag?
-    if (!abort && !wasFlushing && worthPreFlushing()) {
+    if (!abort && worthPreFlushing()) {
       status.setStatus("Pre-flushing region before close");
       LOG.info("Running close preflush of " + this.getRegionNameAsString());
-      internalFlushcache(status);
+      try {
+        internalFlushcache(status);
+      } catch (IOException ioe) {
+        // Failed to flush the region. Keep going.
+        status.setStatus("Failed pre-flush " + this + "; " + ioe.getMessage());
+      }
     }
 
     this.closing.set(true);
@@ -1073,7 +1076,30 @@ private List<StoreFile> doClose(
       LOG.debug("Updates disabled for region " + this);
       // Don't flush the cache if we are aborting
       if (!abort) {
-        internalFlushcache(status);
+        int flushCount = 0;
+        while (this.getMemstoreSize().get() > 0) {
+          try {
+            if (flushCount++ > 0) {
+              int actualFlushes = flushCount - 1;
+              if (actualFlushes > 5) {
+                // If we tried 5 times and are unable to clear memory, abort
+                // so we do not lose data
+                throw new DroppedSnapshotException("Failed clearing memory after " +
+                  actualFlushes + " attempts on region: " + Bytes.toStringBinary(getRegionName()));
+              } 
+              LOG.info("Running extra flush, " + actualFlushes +
+                " (carrying snapshot?) " + this);
+            }
+            internalFlushcache(status);
+          } catch (IOException ioe) {
+            status.setStatus("Failed flush " + this + ", putting online again");
+            synchronized (writestate) {
+              writestate.writesEnabled = true;
+            }
+            // Have to throw to upper layers.  I can't abort server from here.
+            throw ioe;
+          }
+        }
       }
 
       List<StoreFile> result = new ArrayList<StoreFile>();
@@ -1088,6 +1114,7 @@ private List<StoreFile> doClose(
 
         // close each store in parallel
         for (final Store store : stores.values()) {
+          assert abort? true: store.getFlushableSize() == 0;
           completionService
               .submit(new Callable<ImmutableList<StoreFile>>() {
                 public ImmutableList<StoreFile> call() throws IOException {
@@ -1111,7 +1138,7 @@ public ImmutableList<StoreFile> call() throws IOException {
         }
       }
       this.closed.set(true);
-
+      if (memstoreSize.get() != 0) LOG.error("Memstore size is " + memstoreSize.get());
       if (coprocessorHost != null) {
         status.setStatus("Running coprocessor post-close hooks");
         this.coprocessorHost.postClose(abort);
@@ -1629,7 +1656,7 @@ protected boolean internalFlushcache(
     status.setStatus("Obtaining lock to block concurrent updates");
     // block waiting for the lock for internal flush
     this.updatesLock.writeLock().lock();
-    long flushsize = this.memstoreSize.get();
+    long totalFlushableSize = 0;
     status.setStatus("Preparing to flush by snapshotting stores");
     List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(stores.size());
     try {
@@ -1642,6 +1669,7 @@ protected boolean internalFlushcache(
       completeSequenceId = this.getCompleteCacheFlushSequenceId(sequenceId);
 
       for (Store s : stores.values()) {
+        totalFlushableSize += s.getFlushableSize();
         storeFlushers.add(s.getStoreFlusher(completeSequenceId));
       }
 
@@ -1653,7 +1681,7 @@ protected boolean internalFlushcache(
       this.updatesLock.writeLock().unlock();
     }
     String s = "Finished snapshotting " + this +
-      ", commencing wait for mvcc, flushsize=" + flushsize;
+      ", syncing WAL and waiting on mvcc, flushsize=" + totalFlushableSize;
     status.setStatus(s);
     LOG.debug(s);
 
@@ -1699,7 +1727,7 @@ protected boolean internalFlushcache(
       storeFlushers.clear();
 
       // Set down the memstore size by amount of flush.
-      this.addAndGetGlobalMemstoreSize(-flushsize);
+      this.addAndGetGlobalMemstoreSize(-totalFlushableSize);
     } catch (Throwable t) {
       // An exception here means that the snapshot was not persisted.
       // The hlog needs to be replayed so its content is restored to memstore.
@@ -1739,7 +1767,7 @@ protected boolean internalFlushcache(
     long time = EnvironmentEdgeManager.currentTimeMillis() - startTime;
     long memstoresize = this.memstoreSize.get();
     String msg = "Finished memstore flush of ~" +
-      StringUtils.humanReadableInt(flushsize) + "/" + flushsize +
+      StringUtils.humanReadableInt(totalFlushableSize) + "/" + totalFlushableSize +
       ", currentsize=" +
       StringUtils.humanReadableInt(memstoresize) + "/" + memstoresize +
       " for region " + this + " in " + time + "ms, sequenceid=" + sequenceId +
@@ -1747,7 +1775,7 @@ protected boolean internalFlushcache(
       ((wal == null)? "; wal=null": "");
     LOG.info(msg);
     status.setStatus(msg);
-    this.recentFlushes.add(new Pair<Long,Long>(time/1000, flushsize));
+    this.recentFlushes.add(new Pair<Long,Long>(time/1000, totalFlushableSize));
 
     return compactionRequested;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 6b51d74bfdbc..cb920a956f18 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -88,6 +88,7 @@ public class MemStore implements HeapSize {
 
   // Used to track own heapSize
   final AtomicLong size;
+  private volatile long snapshotSize;
 
   // Used to track when to flush
   volatile long timeOfOldestEdit = Long.MAX_VALUE;
@@ -122,6 +123,7 @@ public MemStore(final Configuration conf,
     timeRangeTracker = new TimeRangeTracker();
     snapshotTimeRangeTracker = new TimeRangeTracker();
     this.size = new AtomicLong(DEEP_OVERHEAD);
+    this.snapshotSize = 0;
     if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) {
       this.allocator = new MemStoreLAB(conf);
     } else {
@@ -151,6 +153,7 @@ void snapshot() {
         "Doing nothing. Another ongoing flush or did we fail last attempt?");
     } else {
       if (!this.kvset.isEmpty()) {
+        this.snapshotSize = keySize();
         this.snapshot = this.kvset;
         this.kvset = new KeyValueSkipListSet(this.comparator);
         this.snapshotTimeRangeTracker = this.timeRangeTracker;
@@ -178,6 +181,18 @@ KeyValueSkipListSet getSnapshot() {
     return this.snapshot;
   }
 
+  /**
+   * On flush, how much memory we will clear.
+   * Flush will first clear out the data in snapshot if any (It will take a second flush
+   * invocation to clear the current Cell set). If snapshot is empty, current
+   * Cell set will be flushed.
+   *
+   * @return size of data that is going to be flushed
+   */
+  long getFlushableSize() {
+    return this.snapshotSize > 0 ? this.snapshotSize : keySize();
+  }
+
   /**
    * The passed snapshot was successfully persisted; it can be let go.
    * @param ss The snapshot to clean out.
@@ -196,6 +211,7 @@ void clearSnapshot(final SortedSet<KeyValue> ss)
       this.snapshot = new KeyValueSkipListSet(this.comparator);
       this.snapshotTimeRangeTracker = new TimeRangeTracker();
     }
+    this.snapshotSize = 0;
   }
 
   /**
@@ -877,7 +893,7 @@ public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns,
   }
 
   public final static long FIXED_OVERHEAD = ClassSize.align(
-      ClassSize.OBJECT + (10 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
+      ClassSize.OBJECT + (10 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG));
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.ATOMIC_LONG +
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index d42d752ee6d0..111a7001aae3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -363,6 +363,11 @@ public static Path getStoreHomedir(final Path tabledir,
      return getStoreHomedir(tabledir, encodedName, Bytes.toString(family));
    }
 
+  public long getFlushableSize() {
+    return this.memstore.getFlushableSize();
+  }
+
+
   /**
    * @param tabledir
    * @param encodedName Encoded region name.
@@ -796,7 +801,7 @@ protected Path flushCache(final long logCacheFlushId,
           return pathName;
         } catch (Exception e) {
           LOG.warn("Failed validating store file " + pathName
-              + ", retring num=" + i, e);
+              + ", retrying num=" + i, e);
           if (e instanceof IOException) {
             lastException = (IOException) e;
           } else {
@@ -804,7 +809,7 @@ protected Path flushCache(final long logCacheFlushId,
           }
         }
       } catch (IOException e) {
-        LOG.warn("Failed flushing store file, retring num=" + i, e);
+        LOG.warn("Failed flushing store file, retrying num=" + i, e);
         lastException = e;
       }
       if (lastException != null && i < (flush_retries_number - 1)) {

From a3b0f0fb864cdba28772edbc23c1e48633c4e44d Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 14 Mar 2014 20:38:50 +0000
Subject: [PATCH 1367/1540] HBASE-19751 TestHRegion testWritesWhileScanning
 occasional fail since HBASE-10514 went in

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577699 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/TestHRegion.java    | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 6e6f6aa69c5a..0f9449785e88 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3260,7 +3260,14 @@ public void testFlushCacheWhileScanning() throws IOException, InterruptedExcepti
       flushThread.join();
       flushThread.checkNoError();
     } finally {
-      HRegion.closeHRegion(this.region);
+      try {
+        HRegion.closeHRegion(this.region);
+      } catch (DroppedSnapshotException dse) {
+        // We could get this on way out because we interrupt the background flusher and it could
+        // fail anywhere causing a DSE over in the background flusher... only it is not properly
+        // dealt with so could still be memory hanging out when we get to here -- memory we can't
+        // flush because the accounting is 'off' since original DSE.
+      }
       this.region = null;
     }
   }

From 1fd7f3397faa351df387648005ac4739a5b540a7 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 14 Mar 2014 21:28:59 +0000
Subject: [PATCH 1368/1540] CHANGES.txt and pom.xml for 0.94.18RC0

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577727 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 37 +++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 49af10057961..065f8e667fb3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,41 @@
 HBase Change Log
+Release 0.94.18 - 03/14/2014
+Bug
+
+    [HBASE-9708] - Improve Snapshot Name Error Message
+    [HBASE-9778] - Add hint to ExplicitColumnTracker to avoid seeking
+    [HBASE-10514] - Forward port HBASE-10466, possible data loss when failed flushes
+    [HBASE-10549] - When there is a hole, LoadIncrementalHFiles will hang in an infinite loop.
+    [HBASE-10575] - ReplicationSource thread can't be terminated if it runs into the loop to contact peer's zk ensemble and fails continuously
+    [HBASE-10583] - backport HBASE-8402 to 0.94 - ScanMetrics depends on number of rpc calls to the server.
+    [HBASE-10594] - Speed up TestRestoreSnapshotFromClient
+    [HBASE-10598] - Written data can not be read out because MemStore#timeRangeTracker might be updated concurrently
+    [HBASE-10614] - Master could not be stopped
+    [HBASE-10622] - Improve log and Exceptions in Export Snapshot
+    [HBASE-10624] - Fix 2 new findbugs warnings introduced by HBASE-10598
+    [HBASE-10627] - A logic mistake in HRegionServer isHealthy
+    [HBASE-10631] - Avoid extra seek on FileLink open
+    [HBASE-10642] - Add M/R over snapshots to 0.94
+    [HBASE-10669] - [hbck tool] Usage is wrong for hbck tool for -sidelineCorruptHfiles option
+    [HBASE-10682] - region_mover.rb throws "can't convert nil into String" for regions moved
+    [HBASE-10712] - Backport HBASE-8304 to 0.94 and 0.96
+    [HBASE-10716] - [Configuration]: hbase.regionserver.region.split.policy should be part of hbase-default.xml
+    [HBASE-10718] - TestHLogSplit fails when it sets a KV size to be negative
+    [HBASE-10722] - [0.94] HRegion.computeHDFSBlocksDistribution does not account for links and reference files.
+    [HBASE-10731] - Fix environment variables typos in scripts
+    [HBASE-10738] - AssignmentManager should shut down executors on stop
+    [HBASE-10745] - Access ShutdownHook#fsShutdownHooks should be synchronized
+    [HBASE-10751] - TestHRegion testWritesWhileScanning occasional fail since HBASE-10514 went in
+
+Improvement
+
+    [HBASE-8604] - improve reporting of incorrect peer address in replication
+
+Test
+
+    [HBASE-9914] - Port fix for HBASE-9836 'Intermittent TestRegionObserverScannerOpenHook#testRegionObserverCompactionTimeStacking failure' to 0.94
+
+
 Release 0.94.17 - 02/18/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index 078bd7acb5f6..d53e36fe1bc9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.17</version>
+  <version>0.94.18</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 2faa2015c126f0645afa05c3e7f63018415f9edc Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 15 Mar 2014 04:28:14 +0000
Subject: [PATCH 1369/1540] HBASE-10514 Addendum; fixing missing import

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577782 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/TestHRegion.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 0f9449785e88..9c17cd926fb0 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HBaseTestingUtility;

From 3e054d4f8b78af0734fee6007ae809d100f41466 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Sat, 15 Mar 2014 04:30:20 +0000
Subject: [PATCH 1370/1540] HBASE-10751 TestHRegion testWritesWhileScanning
 occasional fail since HBASE-10514 went in

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577784 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/TestHRegion.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 9c17cd926fb0..e71d2f573937 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.HBaseConfiguration;

From cdb12941c45c4012b11d94c7ded164906c1e4595 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 15 Mar 2014 04:39:28 +0000
Subject: [PATCH 1371/1540] HBASE-10715 Addendum; fixing double import

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1577786 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/TestHRegion.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index e71d2f573937..0c9c4004cff4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -42,7 +42,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HBaseTestingUtility;

From 072dd14dd202bd86bbf17fa48864f2b3311d8f4c Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Mon, 17 Mar 2014 16:16:14 +0000
Subject: [PATCH 1372/1540] HBASE-10766 SnapshotCleaner allows to delete
 referenced files (Bharath Vissapragada)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1578453 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/snapshot/SnapshotFileCache.java    | 22 +++++++---
 .../snapshot/TestSnapshotFileCache.java       | 41 ++++++++++++++-----
 2 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index 13857416bcd1..ec14aa16b1b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -183,18 +183,29 @@ public synchronized boolean contains(String fileName) throws IOException {
   }
 
   private synchronized void refreshCache() throws IOException {
-    // get the status of the snapshots directory
-    FileStatus status;
+    // get the status of the snapshots directory and <snapshot dir>/.tmp
+    FileStatus dirStatus, tempStatus;
     try {
-      status = fs.getFileStatus(snapshotDir);
+      dirStatus = fs.getFileStatus(snapshotDir);
     } catch (FileNotFoundException e) {
       if (this.cache.size() > 0) {
         LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
       }
       return;
     }
+
+    try {
+      Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+      tempStatus = fs.getFileStatus(snapshotTmpDir);
+    } catch (FileNotFoundException e) {
+      tempStatus = dirStatus;
+    }
+
     // if the snapshot directory wasn't modified since we last check, we are done
-    if (status.getModificationTime() <= lastModifiedTime) return;
+    if (dirStatus.getModificationTime() <= lastModifiedTime &&
+        tempStatus.getModificationTime() <= lastModifiedTime) {
+      return;
+    }
 
     // directory was modified, so we need to reload our cache
     // there could be a slight race here where we miss the cache, check the directory modification
@@ -202,7 +213,8 @@ private synchronized void refreshCache() throws IOException {
     // However, snapshot directories are only created once, so this isn't an issue.
 
     // 1. update the modified time
-    this.lastModifiedTime = status.getModificationTime();
+    this.lastModifiedTime = Math.min(dirStatus.getModificationTime(),
+                                     tempStatus.getModificationTime());
 
     // 2.clear the cache
     this.cache.clear();
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
index a6e627cde729..409f697f9441 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
@@ -87,8 +86,8 @@ public void testLoadAndDelete() throws Exception {
     Path file2 = new Path(family, "file2");
 
     // create two hfiles under the snapshot
-    fs.create(file1);
-    fs.create(file2);
+    fs.createNewFile(file1);
+    fs.createNewFile(file2);
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
@@ -138,7 +137,7 @@ public void testLoadsTmpDir() throws Exception {
     Path region = new Path(snapshot, "7e91021");
     Path family = new Path(region, "fam");
     Path file1 = new Path(family, "file1");
-    fs.create(file1);
+    fs.createNewFile(file1);
 
     // create an 'in progress' snapshot
     SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
@@ -146,7 +145,7 @@ public void testLoadsTmpDir() throws Exception {
     region = new Path(snapshot, "7e91021");
     family = new Path(region, "fam");
     Path file2 = new Path(family, "file2");
-    fs.create(file2);
+    fs.createNewFile(file2);
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
@@ -173,12 +172,12 @@ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
     Path region = new Path(snapshot, "7e91021");
     Path family = new Path(region, "fam");
     Path file1 = new Path(family, "file1");
-    fs.create(file1);
+    fs.createNewFile(file1);
 
     // and another file in the logs directory
     Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
     Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
-    fs.create(log);
+    fs.createNewFile(log);
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
@@ -203,8 +202,8 @@ public void testReloadModifiedDirectory() throws IOException {
     Path file2 = new Path(family, "file2");
 
     // create two hfiles under the snapshot
-    fs.create(file1);
-    fs.create(file2);
+    fs.createNewFile(file1);
+    fs.createNewFile(file2);
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
@@ -213,12 +212,34 @@ public void testReloadModifiedDirectory() throws IOException {
     // now delete the snapshot and add a file with a different name
     fs.delete(snapshot, true);
     Path file3 = new Path(family, "new_file");
-    fs.create(file3);
+    fs.createNewFile(file3);
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
     assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
   }
 
+  @Test
+  public void testSnapshotTempDirReload() throws IOException {
+    long period = Long.MAX_VALUE;
+    // This doesn't refresh cache until we invoke it explicitly
+    Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
+        SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    // Add a new snapshot
+    Path snapshot1 = new Path(snapshotDir, "snapshot1");
+    Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
+    fs.createNewFile(file1);
+    assertTrue(cache.contains(file1.getName()));
+
+    // Add another snapshot
+    Path snapshot2 = new Path(snapshotDir, "snapshot2");
+    Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
+    fs.createNewFile(file2);
+    assertTrue(cache.contains(file2.getName()));
+  }
+
   class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
     public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
       Collection<String> files =  new HashSet<String>();

From ebbb3acac168ce95fb398f22a8386d9a98be63e7 Mon Sep 17 00:00:00 2001
From: liangxie <liangxie@unknown>
Date: Thu, 20 Mar 2014 03:25:46 +0000
Subject: [PATCH 1373/1540] HBASE-10782 Hadoop2 MR tests fail occasionally
 because of mapreduce.jobhistory.address is no set in job conf (Shaohui Liu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1579523 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/HBaseTestingUtility.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index d55b7b5b495c..694c278688ea 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1515,6 +1515,10 @@ public void startMiniMapReduceCluster(final int servers) throws IOException {
     if (rmAdress != null) {
       conf.set("yarn.resourcemanager.address", rmAdress);
     }
+    String historyAddress = jobConf.get("mapreduce.jobhistory.address");
+    if (historyAddress != null) {
+      conf.set("mapreduce.jobhistory.address", historyAddress);
+    }
     String schedulerAdress =
       mrClusterJobConf.get("yarn.resourcemanager.scheduler.address");
     if (schedulerAdress != null) {

From be833481d39703abacac467e38b46757a736d3ad Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 22 Mar 2014 04:06:41 +0000
Subject: [PATCH 1374/1540] HBASE-10805 Speed up KeyValueHeap.next() a bit

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1580146 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/regionserver/KeyValueHeap.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index c15bf0f5ad42..3941843b90a3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -99,8 +99,8 @@ public KeyValue next()  throws IOException {
       this.current = pollRealKV();
     } else {
       KeyValueScanner topScanner = this.heap.peek();
-      if (topScanner == null ||
-          this.comparator.compare(kvNext, topScanner.peek()) >= 0) {
+      // no need to add current back to the heap if it is the only scanner left
+      if (topScanner != null && this.comparator.compare(kvNext, topScanner.peek()) >= 0) {
         this.heap.add(this.current);
         this.current = pollRealKV();
       }

From 105a249b27393a0b40e0653a10428cb94f6466f3 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 25 Mar 2014 04:58:15 +0000
Subject: [PATCH 1375/1540] 0.94.19-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1581230 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d53e36fe1bc9..2844bb27fe78 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.18</version>
+  <version>0.94.19-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 4c63f9219c1f5d9307eb44be950bdfdf69f4e880 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 31 Mar 2014 22:36:42 +0000
Subject: [PATCH 1376/1540] HBASE-10847 0.94: drop non-secure builds, make
 security the default.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1583480 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml                                       | 48 +++++++++----------
 .../hbase/client/TestFromClientSide.java      |  4 +-
 ...TestFromClientSideWithSecureRpcEngine.java | 42 ++++++++++++++++
 3 files changed, 69 insertions(+), 25 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithSecureRpcEngine.java

diff --git a/pom.xml b/pom.xml
index 2844bb27fe78..6ec107368a9a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -942,6 +942,28 @@
         <artifactId>build-helper-maven-plugin</artifactId>
         <version>1.5</version>
         <executions>
+          <execution>
+            <id>add-source</id>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>${project.basedir}/security/src/main/java</source>
+              </sources>
+            </configuration>
+          </execution>
+          <execution>
+            <id>add-test-source</id>
+            <goals>
+              <goal>add-test-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>${project.basedir}/security/src/test/java</source>
+              </sources>
+            </configuration>
+          </execution>
           <execution>
             <id>jspcSource-packageInfo-Avro-source</id>
             <phase>generate-sources</phase>
@@ -1665,9 +1687,9 @@
       </build>
     </profile>
 
-    <!-- profile for building against Hadoop 0.20+security-->
+    <!-- profile only used for testing, run's all tests with the SecureRpcEngine -->
     <profile>
-      <id>security</id>
+      <id>security-test</id>
       <properties>
         <hadoop.version>1.0.4</hadoop.version>
       </properties>
@@ -1678,28 +1700,6 @@
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>build-helper-maven-plugin</artifactId>
             <executions>
-              <execution>
-                <id>add-source</id>
-                <goals>
-                  <goal>add-source</goal>
-                </goals>
-                <configuration>
-                  <sources>
-                    <source>${project.basedir}/security/src/main/java</source>
-                  </sources>
-                </configuration>
-              </execution>
-              <execution>
-                <id>add-test-source</id>
-                <goals>
-                  <goal>add-test-source</goal>
-                </goals>
-                <configuration>
-                  <sources>
-                    <source>${project.basedir}/security/src/test/java</source>
-                  </sources>
-                </configuration>
-              </execution>
               <execution>
                 <id>add-test-resource</id>
                 <goals>
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 36d57f851fd7..f9167283aaab 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -47,7 +47,6 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.lang.ArrayUtils;
@@ -85,6 +84,7 @@
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.ipc.SecureRpcEngine;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
@@ -123,6 +123,8 @@ public class TestFromClientSide {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     Configuration conf = TEST_UTIL.getConfiguration();
+    // force the rpc engine to the non-secure one in order to get coverage
+    conf.set("hbase.rpc.engine", "org.apache.hadoop.hbase.ipc.WritableRpcEngine");
     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
         MultiRowMutationEndpoint.class.getName());
     // We need more than one region server in this test
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithSecureRpcEngine.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithSecureRpcEngine.java
new file mode 100644
index 000000000000..7fd8f9a8fb9e
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithSecureRpcEngine.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
+import org.apache.hadoop.hbase.ipc.SecureRpcEngine;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test all client operations with {@link SecureRpcEngine}
+ */
+@Category(LargeTests.class)
+public class TestFromClientSideWithSecureRpcEngine extends TestFromClientSide {
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.set("hbase.rpc.engine", SecureRpcEngine.class.getName());
+    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+      MultiRowMutationEndpoint.class.getName());
+    // We need more than one region server in this test
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+}

From 883e3a1e08894fcfd3dab6dfc161ff60621e5069 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Thu, 3 Apr 2014 20:16:03 +0000
Subject: [PATCH 1377/1540] HBASE-10848 Filter SingleColumnValueFilter combined
 with NullComparator does not work (Fabien)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1584375 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/filter/NullComparator.java   |  2 +-
 .../hbase/filter/TestNullComparator.java      | 73 +++++++++++++++++++
 .../filter/TestSingleColumnValueFilter.java   | 23 +++++-
 3 files changed, 96 insertions(+), 2 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java
index 45eb4778037d..9a7f770d4b1b 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java
@@ -38,6 +38,6 @@ public int compareTo(byte[] value) {
 
   @Override
   public int compareTo(byte[] value, int offset, int length) {
-    throw new UnsupportedOperationException();
+    return compareTo(value);
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java b/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java
new file mode 100644
index 000000000000..62639b8349e9
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java
@@ -0,0 +1,73 @@
+/**
+ * 
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *  under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestNullComparator {
+
+  @Test
+  public void testNullValue()
+  {
+    // given
+    byte[] value = null;
+    NullComparator comparator = new NullComparator();
+
+    // when
+    int comp1 = comparator.compareTo(value);
+    int comp2 = comparator.compareTo(value, 5, 15);
+
+    // then
+    Assert.assertEquals(0, comp1);
+    Assert.assertEquals(0, comp2);
+  }
+
+  @Test
+  public void testNonNullValue() {
+    // given
+    byte[] value = new byte[] { 0, 1, 2, 3, 4, 5 };
+    NullComparator comparator = new NullComparator();
+
+    // when
+    int comp1 = comparator.compareTo(value);
+    int comp2 = comparator.compareTo(value, 1, 3);
+
+    // then
+    Assert.assertEquals(1, comp1);
+    Assert.assertEquals(1, comp2);
+  }
+
+  @Test
+  public void testEmptyValue() {
+    // given
+    byte[] value = new byte[] { 0 };
+    NullComparator comparator = new NullComparator();
+
+    // when
+    int comp1 = comparator.compareTo(value);
+    int comp2 = comparator.compareTo(value, 1, 3);
+
+    // then
+    Assert.assertEquals(1, comp1);
+    Assert.assertEquals(1, comp2);
+  }
+
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
index 2a0751c587ee..fca32a73d470 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
@@ -52,6 +52,7 @@ public class TestSingleColumnValueFilter extends TestCase {
   private static final String QUICK_REGEX = ".+quick.+";
 
   Filter basicFilter;
+  Filter nullFilter;
   Filter substrFilter;
   Filter regexFilter;
 
@@ -59,6 +60,7 @@ public class TestSingleColumnValueFilter extends TestCase {
   protected void setUp() throws Exception {
     super.setUp();
     basicFilter = basicFilterNew();
+    nullFilter = nullFilterNew();
     substrFilter = substrFilterNew();
     regexFilter = regexFilterNew();
   }
@@ -68,6 +70,11 @@ private Filter basicFilterNew() {
       CompareOp.GREATER_OR_EQUAL, VAL_2);
   }
 
+  private Filter nullFilterNew() {
+    return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOp.NOT_EQUAL,
+        new NullComparator());
+  }
+
   private Filter substrFilterNew() {
     return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER,
       CompareOp.EQUAL,
@@ -105,6 +112,17 @@ private void basicFilterTests(SingleColumnValueFilter filter)
     assertFalse("basicFilterNotNull", filter.filterRow());
   }
 
+  private void nullFilterTests(Filter filter) throws Exception {
+    ((SingleColumnValueFilter) filter).setFilterIfMissing(true);
+    KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1);
+    assertTrue("null1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
+    assertFalse("null1FilterRow", filter.filterRow());
+    filter.reset();
+    kv = new KeyValue(ROW, COLUMN_FAMILY, Bytes.toBytes("qual2"), FULLSTRING_2);
+    assertTrue("null2", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
+    assertTrue("null2FilterRow", filter.filterRow());
+  }
+
   private void substrFilterTests(Filter filter)
       throws Exception {
     KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER,
@@ -154,7 +172,8 @@ private Filter serializationTest(Filter filter)
    * @throws Exception
    */
   public void testStop() throws Exception {
-    basicFilterTests((SingleColumnValueFilter)basicFilter);
+    basicFilterTests((SingleColumnValueFilter) basicFilter);
+    nullFilterTests(nullFilter);
     substrFilterTests(substrFilter);
     regexFilterTests(regexFilter);
   }
@@ -166,6 +185,8 @@ public void testStop() throws Exception {
   public void testSerialization() throws Exception {
     Filter newFilter = serializationTest(basicFilter);
     basicFilterTests((SingleColumnValueFilter)newFilter);
+    newFilter = serializationTest(nullFilter);
+    nullFilterTests(newFilter);
     newFilter = serializationTest(substrFilter);
     substrFilterTests(newFilter);
     newFilter = serializationTest(regexFilter);

From 703bc52e0df33b7e968673d30f0216c5a2bced75 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 4 Apr 2014 04:30:25 +0000
Subject: [PATCH 1378/1540] HBASE-10118 Major compact keeps deletes with future
 timestamps. (Liu Shaohui)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1584514 13f79535-47bb-0310-9956-ffa450edef68
---
 src/docbkx/book.xml                           |  9 +++
 .../hbase/regionserver/ScanQueryMatcher.java  |  8 ++-
 .../hbase/client/TestFromClientSide.java      | 66 +++++++++++++++++++
 3 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/src/docbkx/book.xml b/src/docbkx/book.xml
index 65c170a5326a..05133bad2df8 100644
--- a/src/docbkx/book.xml
+++ b/src/docbkx/book.xml
@@ -485,6 +485,15 @@ htable.put(put);
           up on the user mailing list.</para>
           <para>Also see <xref linkend="keyvalue"/> for more information on the internal KeyValue format.
           </para>
+          <para>Delete markers are purged during the major compaction of store, 
+          unless the KEEP_DELETED_CELLS is set in the column family. In some 
+          scenarios, users want to keep the deletes for a time and you can set the 
+          delete TTL: hbase.hstore.time.to.purge.deletes in the configuration. 
+          If this delete TTL is not set or set to 0, all delete markers including those 
+          with future timestamp are purged during the later major compaction. 
+          Otherwise, a delete marker is kept until the major compaction after 
+          marker's timestamp + delete TTL. 
+          </para>
         </section>
        </section>
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 567c5289edad..fec739d591a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -296,9 +296,11 @@ public MatchCode match(KeyValue kv) throws IOException {
         }
         // Can't early out now, because DelFam come before any other keys
       }
-      if (retainDeletesInOutput
-          || (!isUserScan && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes)
-          || kv.getMemstoreTS() > maxReadPointToTrackVersions) {
+      if ((!isUserScan)
+          && timeToPurgeDeletes > 0
+          && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes) {
+        return MatchCode.INCLUDE;
+      } else if (retainDeletesInOutput || kv.getMemstoreTS() > maxReadPointToTrackVersions) {
         // always include or it is not time yet to check whether it is OK
         // to purge deltes or not
         if (!isUserScan) {
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index f9167283aaab..aceaeedfa07a 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -223,6 +223,72 @@ public void testKeepDeletedCells() throws Exception {
      h.close();
    }
 
+  /**
+   * Basic client side validation of HBASE-10118
+   */
+  @Test
+  public void testPurgeFutureDeletes() throws Exception {
+    final byte[] TABLENAME = Bytes.toBytes("testPurgeFutureDeletes");
+    final byte[] ROW = Bytes.toBytes("row");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[] COLUMN = Bytes.toBytes("column");
+    final byte[] VALUE = Bytes.toBytes("value");
+
+    HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
+
+    // future timestamp
+    long ts = System.currentTimeMillis() * 2;
+    Put put = new Put(ROW, ts);
+    put.add(FAMILY, COLUMN, VALUE);
+    table.put(put);
+
+    Get get = new Get(ROW);
+    Result result = table.get(get);
+    assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
+
+    Delete del = new Delete(ROW);
+    del.deleteColumn(FAMILY, COLUMN, ts);
+    table.delete(del);
+
+    get = new Get(ROW);
+    result = table.get(get);
+    assertNull(result.getValue(FAMILY, COLUMN));
+
+    // major compaction, purged future deletes
+    TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
+    TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
+
+    // waiting for the major compaction to complete
+    int sleepTime = 0;
+    while (true) {
+      Thread.sleep(200);
+      sleepTime += 200;
+
+      Scan s = new Scan();
+      s.setRaw(true);
+      ResultScanner scanner = table.getScanner(s);
+      if (scanner.next() == null) {
+        scanner.close();
+        break;
+      }
+      scanner.close();
+
+      if (sleepTime > 6000) {
+        throw new IOException("Major compaction time is larger than 6000s");
+      }
+    }
+
+    put = new Put(ROW, ts);
+    put.add(FAMILY, COLUMN, VALUE);
+    table.put(put);
+
+    get = new Get(ROW);
+    result = table.get(get);
+    assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
+
+    table.close();
+  }
+
    /**
    * HBASE-2468 use case 1 and 2: region info de/serialization
    */

From c865efc090daefcf2173eb8c6418f56455cd81db Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Fri, 11 Apr 2014 15:51:54 +0000
Subject: [PATCH 1379/1540] HBASE-10921 Port HBASE-10323 'Auto detect data
 block encoding in HFileOutputFormat' to 0.94 (Kashif)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1586701 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/HFileOutputFormat.java    | 247 ++++++++++++----
 .../mapreduce/TestHFileOutputFormat.java      | 278 ++++++++++++++++--
 2 files changed, 438 insertions(+), 87 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
index 028b220379fc..f67afedc33e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
@@ -54,6 +54,7 @@
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
@@ -67,22 +68,39 @@
 import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Writes HFiles. Passed KeyValues must arrive in order.
  * Currently, can only write files to a single column family at a
  * time.  Multiple column families requires coordinating keys cross family.
  * Writes current time as the sequence id for the file. Sets the major compacted
- * attribute on created hfiles. Calling write(null,null) will forceably roll
+ * attribute on created hfiles. Calling write(null,null) will forcibly roll
  * all HFiles being written.
  * @see KeyValueSortReducer
  */
 public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable, KeyValue> {
   static Log LOG = LogFactory.getLog(HFileOutputFormat.class);
-  static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression";
-  private static final String BLOOM_TYPE_CONF_KEY = "hbase.hfileoutputformat.families.bloomtype";
-  private static final String DATABLOCK_ENCODING_CONF_KEY = 
-     "hbase.mapreduce.hfileoutputformat.datablock.encoding";
-  private static final String BLOCK_SIZE_CONF_KEY = "hbase.mapreduce.hfileoutputformat.blocksize";
+  
+  // The following constants are private since these are used by
+  // HFileOutputFormat to internally transfer data between job setup and
+  // reducer run using conf.
+  // These should not be changed by the client.
+  private static final String COMPRESSION_FAMILIES_CONF_KEY =
+      "hbase.hfileoutputformat.families.compression";
+  private static final String BLOOM_TYPE_FAMILIES_CONF_KEY =
+      "hbase.hfileoutputformat.families.bloomtype";
+  private static final String BLOCK_SIZE_FAMILIES_CONF_KEY =
+      "hbase.mapreduce.hfileoutputformat.blocksize";
+  private static final String DATABLOCK_ENCODING_FAMILIES_CONF_KEY =
+      "hbase.mapreduce.hfileoutputformat.families.datablock.encoding";
+
+  // This constant is public since the client can modify this when setting
+  // up their conf object and thus refer to this symbol.
+  // It is present for backwards compatibility reasons. Use it only to
+  // override the auto-detection of datablock encoding.
+  public static final String DATABLOCK_ENCODING_OVERRIDE_CONF_KEY =
+      "hbase.mapreduce.hfileoutputformat.datablock.encoding";
 
   public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
   throws IOException, InterruptedException {
@@ -95,30 +113,27 @@ public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final Task
     final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
         HConstants.DEFAULT_MAX_FILE_SIZE);
     // Invented config.  Add to hbase-*.xml if other than default compression.
-    final String defaultCompression = conf.get("hfile.compression",
+    final String defaultCompressionStr = conf.get("hfile.compression",
         Compression.Algorithm.NONE.getName());
+    final Algorithm defaultCompression =
+        AbstractHFileWriter.compressionByName(defaultCompressionStr);
+
     final boolean compactionExclude = conf.getBoolean(
         "hbase.mapreduce.hfileoutputformat.compaction.exclude", false);
 
     // create a map from column family to the compression algorithm
-    final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf);
-    final Map<byte[], String> bloomTypeMap = createFamilyBloomMap(conf);
-    final Map<byte[], String> blockSizeMap = createFamilyBlockSizeMap(conf);
-
-    String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
-    final HFileDataBlockEncoder encoder;
-    if (dataBlockEncodingStr == null) {
-      encoder = NoOpDataBlockEncoder.INSTANCE;
+    final Map<byte[], Algorithm> compressionMap = createFamilyCompressionMap(conf);
+    final Map<byte[], BloomType> bloomTypeMap = createFamilyBloomTypeMap(conf);
+    final Map<byte[], Integer> blockSizeMap = createFamilyBlockSizeMap(conf);
+    
+    String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY);
+    final Map<byte[], HFileDataBlockEncoder> datablockEncodingMap =
+        createFamilyDataBlockEncodingMap(conf);
+    final HFileDataBlockEncoder overriddenEncoder;
+    if (dataBlockEncodingStr != null) {
+      overriddenEncoder = getDataBlockEncoderFromString(dataBlockEncodingStr);
     } else {
-      try {
-        encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding
-            .valueOf(dataBlockEncodingStr));
-      } catch (IllegalArgumentException ex) {
-        throw new RuntimeException(
-            "Invalid data block encoding type configured for the param "
-                + DATABLOCK_ENCODING_CONF_KEY + " : "
-                + dataBlockEncodingStr);
-      }
+      overriddenEncoder = null;
     }
 
     return new RecordWriter<ImmutableBytesWritable, KeyValue>() {
@@ -194,21 +209,21 @@ private WriterLength getNewWriter(byte[] family, Configuration conf)
           throws IOException {
         WriterLength wl = new WriterLength();
         Path familydir = new Path(outputdir, Bytes.toString(family));
-        String compression = compressionMap.get(family);
+        Algorithm compression = compressionMap.get(family);
         compression = compression == null ? defaultCompression : compression;
-        String bloomTypeStr = bloomTypeMap.get(family);
-        BloomType bloomType = BloomType.NONE;
-        if (bloomTypeStr != null) {
-          bloomType = BloomType.valueOf(bloomTypeStr);
-        }
-        String blockSizeString = blockSizeMap.get(family);
-        int blockSize = blockSizeString == null ? HFile.DEFAULT_BLOCKSIZE
-            : Integer.parseInt(blockSizeString);
+        BloomType bloomType = bloomTypeMap.get(family);
+        bloomType = bloomType == null ? BloomType.NONE : bloomType;
+        Integer blockSize = blockSizeMap.get(family);
+        blockSize = blockSize == null ? HFile.DEFAULT_BLOCKSIZE : blockSize;
+        HFileDataBlockEncoder encoder = overriddenEncoder;
+        encoder = encoder == null ? datablockEncodingMap.get(family) : encoder;
+        encoder = encoder == null ? NoOpDataBlockEncoder.INSTANCE : encoder;
+
         Configuration tempConf = new Configuration(conf);
         tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
         wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blockSize)
             .withOutputDir(familydir)
-            .withCompression(AbstractHFileWriter.compressionByName(compression))
+            .withCompression(compression)
             .withBloomType(bloomType)
             .withComparator(KeyValue.COMPARATOR)
             .withDataBlockEncoder(encoder)
@@ -378,6 +393,7 @@ public static void configureIncrementalLoad(Job job, HTable table)
     configureCompression(table, conf);
     configureBloomType(table, conf);
     configureBlockSize(table, conf);
+    configureDataBlockEncoding(table, conf);
     
     TableMapReduceUtil.addDependencyJars(job);
     LOG.info("Incremental table output configured.");
@@ -405,32 +421,95 @@ private static Class<? extends Partitioner> getTotalOrderPartitionerClass()
   }
 
   /**
-   * Run inside the task to deserialize column family to compression algorithm
-   * map from the
-   * configuration.
-   *
-   * Package-private for unit tests only.
+   * Runs inside the task to deserialize column family to compression algorithm
+   * map from the configuration.
    *
    * @return a map from column family to the name of the configured compression
    *         algorithm
    */
-  static Map<byte[], String> createFamilyCompressionMap(Configuration conf) {
-    return createFamilyConfValueMap(conf, COMPRESSION_CONF_KEY);
+  @VisibleForTesting
+  static Map<byte[], Algorithm> createFamilyCompressionMap(Configuration conf) {
+    Map<byte[], String> stringMap = createFamilyConfValueMap(conf, COMPRESSION_FAMILIES_CONF_KEY);
+    Map<byte[], Algorithm> compressionMap = new TreeMap<byte[], Algorithm>(Bytes.BYTES_COMPARATOR);
+    for (Map.Entry<byte[], String> e : stringMap.entrySet()) {
+      Algorithm algorithm = AbstractHFileWriter.compressionByName(e.getValue());
+      compressionMap.put(e.getKey(), algorithm);
+    }
+    return compressionMap;
   }
 
-  private static Map<byte[], String> createFamilyBloomMap(Configuration conf) {
-    return createFamilyConfValueMap(conf, BLOOM_TYPE_CONF_KEY);
+  /**
+   * Runs inside the task to deserialize column family to bloom type
+   * map from the configuration.
+   *
+   * @param conf to read the serialized values from
+   * @return a map from column family to the configured bloom type
+   */
+  @VisibleForTesting
+  static Map<byte[], BloomType> createFamilyBloomTypeMap(Configuration conf) {
+    Map<byte[], String> stringMap = createFamilyConfValueMap(conf, BLOOM_TYPE_FAMILIES_CONF_KEY);
+    Map<byte[], BloomType> bloomTypeMap = new TreeMap<byte[], BloomType>(Bytes.BYTES_COMPARATOR);
+    for (Map.Entry<byte[], String> e : stringMap.entrySet()) {
+      BloomType bloomType = BloomType.valueOf(e.getValue());
+      bloomTypeMap.put(e.getKey(), bloomType);
+    }
+    return bloomTypeMap;
   }
 
-  private static Map<byte[], String> createFamilyBlockSizeMap(Configuration conf) {
-    return createFamilyConfValueMap(conf, BLOCK_SIZE_CONF_KEY);
+  /**
+   * Runs inside the task to deserialize column family to block size
+   * map from the configuration.
+   *
+   * @param conf to read the serialized values from
+   * @return a map from column family to the configured block size
+   */
+  @VisibleForTesting
+  static Map<byte[], Integer> createFamilyBlockSizeMap(Configuration conf) {
+    Map<byte[], String> stringMap = createFamilyConfValueMap(conf, BLOCK_SIZE_FAMILIES_CONF_KEY);
+    Map<byte[], Integer> blockSizeMap = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
+    for (Map.Entry<byte[], String> e : stringMap.entrySet()) {
+      Integer blockSize = Integer.parseInt(e.getValue());
+      blockSizeMap.put(e.getKey(), blockSize);
+    }
+    return blockSizeMap;
+  }
+  
+  /**
+   * Runs inside the task to deserialize column family to data block encoding type map from the
+   * configuration.
+   * 
+   * @param conf to read the serialized values from
+   * @return a map from column family to HFileDataBlockEncoder for the configured data block type
+   *         for the family
+   */
+  @VisibleForTesting
+  static Map<byte[], HFileDataBlockEncoder> createFamilyDataBlockEncodingMap(Configuration conf) {
+    Map<byte[], String> stringMap =
+        createFamilyConfValueMap(conf, DATABLOCK_ENCODING_FAMILIES_CONF_KEY);
+    Map<byte[], HFileDataBlockEncoder> encoderMap =
+        new TreeMap<byte[], HFileDataBlockEncoder>(Bytes.BYTES_COMPARATOR);
+    for (Map.Entry<byte[], String> e : stringMap.entrySet()) {
+      encoderMap.put(e.getKey(), getDataBlockEncoderFromString(e.getValue()));
+    }
+    return encoderMap;
+  }
+
+  private static HFileDataBlockEncoder getDataBlockEncoderFromString(String dataBlockEncodingStr) {
+    HFileDataBlockEncoder encoder;
+    try {
+      encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding.valueOf(dataBlockEncodingStr));
+    } catch (IllegalArgumentException ex) {
+      throw new RuntimeException("Invalid data block encoding type configured for the param "
+          + DATABLOCK_ENCODING_FAMILIES_CONF_KEY + " : " + dataBlockEncodingStr);
+    }
+    return encoder;
   }
 
   /**
    * Run inside the task to deserialize column family to given conf value map.
    * 
-   * @param conf
-   * @param confName
+   * @param conf to read the serialized values from
+   * @param confName conf key to read from the configuration
    * @return a map of column family to the given configuration value
    */
   private static Map<byte[], String> createFamilyConfValueMap(Configuration conf, String confName) {
@@ -455,12 +534,13 @@ private static Map<byte[], String> createFamilyConfValueMap(Configuration conf,
   /**
    * Serialize column family to compression algorithm map to configuration.
    * Invoked while configuring the MR job for incremental load.
-   *
-   * Package-private for unit tests only.
-   *
-   * @throws IOException
-   *           on failure to read column family descriptors
+   * 
+   * @param table to read the properties from
+   * @param conf to persist serialized values into
+   * @throws IOException 
+   *            on failure to read column family descriptors
    */
+  @VisibleForTesting
   static void configureCompression(HTable table, Configuration conf) throws IOException {
     StringBuilder compressionConfigValue = new StringBuilder();
     HTableDescriptor tableDescriptor = table.getTableDescriptor();
@@ -479,10 +559,20 @@ static void configureCompression(HTable table, Configuration conf) throws IOExce
       compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getCompression().getName(), "UTF-8"));
     }
     // Get rid of the last ampersand
-    conf.set(COMPRESSION_CONF_KEY, compressionConfigValue.toString());
+    conf.set(COMPRESSION_FAMILIES_CONF_KEY, compressionConfigValue.toString());
   }
   
-  private static void configureBlockSize(HTable table, Configuration conf) throws IOException {
+  /**
+   * Serialize column family to block size map to configuration.
+   * Invoked while configuring the MR job for incremental load.
+   * 
+   * @param table to read the properties from
+   * @param conf to persist serialized values into
+   * @throws IOException 
+   *            on failure to read column family descriptors
+   */
+  @VisibleForTesting
+  static void configureBlockSize(HTable table, Configuration conf) throws IOException {
     StringBuilder blockSizeConfigValue = new StringBuilder();
     HTableDescriptor tableDescriptor = table.getTableDescriptor();
     if (tableDescriptor == null) {
@@ -502,16 +592,19 @@ private static void configureBlockSize(HTable table, Configuration conf) throws
           String.valueOf(familyDescriptor.getBlocksize()), "UTF-8"));
     }
     // Get rid of the last ampersand
-    conf.set(BLOCK_SIZE_CONF_KEY, blockSizeConfigValue.toString());
+    conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString());
   }
 
   /**
    * Serialize column family to bloom type map to configuration.
    * Invoked while configuring the MR job for incremental load.
-   *
-   * @throws IOException
-   *           on failure to read column family descriptors
+   * 
+   * @param table to read the properties from
+   * @param conf to persist serialized values into
+   * @throws IOException 
+   *            on failure to read column family descriptors
    */
+  @VisibleForTesting
   static void configureBloomType(HTable table, Configuration conf) throws IOException {
     HTableDescriptor tableDescriptor = table.getTableDescriptor();
     if (tableDescriptor == null) {
@@ -533,6 +626,42 @@ static void configureBloomType(HTable table, Configuration conf) throws IOExcept
       }
       bloomTypeConfigValue.append(URLEncoder.encode(bloomType, "UTF-8"));
     }
-    conf.set(BLOOM_TYPE_CONF_KEY, bloomTypeConfigValue.toString());
+    conf.set(BLOOM_TYPE_FAMILIES_CONF_KEY, bloomTypeConfigValue.toString());
   }
+
+  /**
+   * Serialize column family to data block encoding map to configuration.
+   * Invoked while configuring the MR job for incremental load.
+   * 
+   * @param table to read the properties from
+   * @param conf to persist serialized values into
+   * @throws IOException 
+   *            on failure to read column family descriptors
+   */
+  @VisibleForTesting
+  static void configureDataBlockEncoding(HTable table, Configuration conf) throws IOException {
+    HTableDescriptor tableDescriptor = table.getTableDescriptor();
+    if (tableDescriptor == null) {
+      // could happen with mock table instance
+      return;
+    }
+    StringBuilder dataBlockEncodingConfigValue = new StringBuilder();
+    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
+    int i = 0;
+    for (HColumnDescriptor familyDescriptor : families) {
+      if (i++ > 0) {
+        dataBlockEncodingConfigValue.append('&');
+      }
+      dataBlockEncodingConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(),
+        "UTF-8"));
+      dataBlockEncodingConfigValue.append('=');
+      DataBlockEncoding encoding = familyDescriptor.getDataBlockEncoding();
+      if (encoding == null) {
+        encoding = DataBlockEncoding.NONE;
+      }
+      dataBlockEncodingConfigValue.append(URLEncoder.encode(encoding.toString(), "UTF-8"));
+    }
+    conf.set(DATABLOCK_ENCODING_FAMILIES_CONF_KEY, dataBlockEncodingConfigValue.toString());
+  }
+
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 2bf0c1b21ef4..565670537b70 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -30,11 +30,9 @@
 import java.lang.reflect.Constructor;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.TreeSet;
 import java.util.concurrent.Callable;
 import java.util.Random;
 
@@ -53,14 +51,17 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
@@ -77,8 +78,6 @@
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
-import com.google.common.collect.Lists;
-
 /**
  * Simple test for {@link KeyValueSortReducer} and {@link HFileOutputFormat}.
  * Sets up and runs a mapreduce job that writes hfile output.
@@ -484,36 +483,40 @@ private void runIncrementalPELoad(
   }
 
   /**
-   * Test for
-   * {@link HFileOutputFormat#createFamilyCompressionMap(Configuration)}. Tests
-   * that the compression map is correctly deserialized from configuration
+   * Test for {@link HFileOutputFormat#configureCompression(HTable,
+   * Configuration)} and {@link HFileOutputFormat#createFamilyCompressionMap
+   * (Configuration)}.
+   * Tests that the compression map is correctly serialized into
+   * and deserialized from configuration
    *
    * @throws IOException
    */
   @Test
-  public void testCreateFamilyCompressionMap() throws IOException {
+  public void testSerializeDeserializeFamilyCompressionMap() throws IOException {
     for (int numCfs = 0; numCfs <= 3; numCfs++) {
       Configuration conf = new Configuration(this.util.getConfiguration());
-      Map<String, Compression.Algorithm> familyToCompression = getMockColumnFamilies(numCfs);
+      Map<String, Compression.Algorithm> familyToCompression =
+          getMockColumnFamiliesForCompression(numCfs);
       HTable table = Mockito.mock(HTable.class);
-      setupMockColumnFamilies(table, familyToCompression);
+      setupMockColumnFamiliesForCompression(table, familyToCompression);
       HFileOutputFormat.configureCompression(table, conf);
 
       // read back family specific compression setting from the configuration
-      Map<byte[], String> retrievedFamilyToCompressionMap = HFileOutputFormat.createFamilyCompressionMap(conf);
+      Map<byte[], Algorithm> retrievedFamilyToCompressionMap = HFileOutputFormat
+          .createFamilyCompressionMap(conf);
 
       // test that we have a value for all column families that matches with the
       // used mock values
       for (Entry<String, Algorithm> entry : familyToCompression.entrySet()) {
-        assertEquals("Compression configuration incorrect for column family:" + entry.getKey(), entry.getValue()
-                     .getName(), retrievedFamilyToCompressionMap.get(entry.getKey().getBytes()));
+        assertEquals("Compression configuration incorrect for column family:"
+            + entry.getKey(), entry.getValue(),
+            retrievedFamilyToCompressionMap.get(entry.getKey().getBytes()));
       }
     }
   }
 
-  private void setupMockColumnFamilies(HTable table,
-    Map<String, Compression.Algorithm> familyToCompression) throws IOException
-  {
+  private void setupMockColumnFamiliesForCompression(HTable table,
+      Map<String, Compression.Algorithm> familyToCompression) throws IOException {
     HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME);
     for (Entry<String, Compression.Algorithm> entry : familyToCompression.entrySet()) {
       mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey())
@@ -525,21 +528,11 @@ private void setupMockColumnFamilies(HTable table,
     Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor();
   }
 
-  private void setupMockStartKeys(HTable table) throws IOException {
-    byte[][] mockKeys = new byte[][] {
-        HConstants.EMPTY_BYTE_ARRAY,
-        Bytes.toBytes("aaa"),
-        Bytes.toBytes("ggg"),
-        Bytes.toBytes("zzz")
-    };
-    Mockito.doReturn(mockKeys).when(table).getStartKeys();
-  }
-
   /**
    * @return a map from column family names to compression algorithms for
    *         testing column family compression. Column family names have special characters
    */
-  private Map<String, Compression.Algorithm> getMockColumnFamilies(int numCfs) {
+  private Map<String, Compression.Algorithm> getMockColumnFamiliesForCompression (int numCfs) {
     Map<String, Compression.Algorithm> familyToCompression = new HashMap<String, Compression.Algorithm>();
     // use column family names having special characters
     if (numCfs-- > 0) {
@@ -557,6 +550,235 @@ private Map<String, Compression.Algorithm> getMockColumnFamilies(int numCfs) {
     return familyToCompression;
   }
 
+  /**
+   * Test for {@link HFileOutputFormat#configureBloomType(HTable,
+   * Configuration)} and {@link HFileOutputFormat#createFamilyBloomTypeMap
+   * (Configuration)}.
+   * Tests that the compression map is correctly serialized into
+   * and deserialized from configuration
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testSerializeDeserializeFamilyBloomTypeMap() throws IOException {
+    for (int numCfs = 0; numCfs <= 2; numCfs++) {
+      Configuration conf = new Configuration(this.util.getConfiguration());
+      Map<String, BloomType> familyToBloomType =
+          getMockColumnFamiliesForBloomType(numCfs);
+      HTable table = Mockito.mock(HTable.class);
+      setupMockColumnFamiliesForBloomType(table,
+          familyToBloomType);
+      HFileOutputFormat.configureBloomType(table, conf);
+
+      // read back family specific bloom type settings from the configuration
+      Map<byte[], BloomType> retrievedFamilyToBloomTypeMap =
+          HFileOutputFormat
+              .createFamilyBloomTypeMap(conf);
+
+      // test that we have a value for all column families that matches with the
+      // used mock values
+      for (Entry<String, BloomType> entry : familyToBloomType.entrySet()) {
+        assertEquals("BloomType configuration incorrect for column family:" 
+            + entry.getKey(), entry.getValue(), 
+            retrievedFamilyToBloomTypeMap.get(entry.getKey().getBytes()));
+      }
+    }
+  }
+
+  private void setupMockColumnFamiliesForBloomType(HTable table,
+      Map<String, BloomType> familyToDataBlockEncoding) throws IOException {
+    HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME);
+    for (Entry<String, BloomType> entry : familyToDataBlockEncoding.entrySet()) {
+      mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey())
+          .setMaxVersions(1)
+          .setBloomFilterType(entry.getValue())
+          .setBlockCacheEnabled(false)
+          .setTimeToLive(0));
+    }
+    Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor();
+  }
+
+  /**
+   * @return a map from column family names to compression algorithms for
+   *         testing column family compression. Column family names have special characters
+   */
+  private Map<String, BloomType>
+  getMockColumnFamiliesForBloomType (int numCfs) {
+    Map<String, BloomType> familyToBloomType =
+        new HashMap<String, BloomType>();
+    // use column family names having special characters
+    if (numCfs-- > 0) {
+      familyToBloomType.put("Family1!@#!@#&", BloomType.ROW);
+    }
+    if (numCfs-- > 0) {
+      familyToBloomType.put("Family2=asdads&!AASD",
+          BloomType.ROWCOL);
+    }
+    if (numCfs-- > 0) {
+      familyToBloomType.put("Family3", BloomType.NONE);
+    }
+    return familyToBloomType;
+  }
+
+  /**
+   * Test for {@link HFileOutputFormat#configureBlockSize(HTable,
+   * Configuration)} and {@link HFileOutputFormat#createFamilyBlockSizeMap
+   * (Configuration)}.
+   * Tests that the compression map is correctly serialized into
+   * and deserialized from configuration
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testSerializeDeserializeFamilyBlockSizeMap() throws IOException {
+    for (int numCfs = 0; numCfs <= 3; numCfs++) {
+      Configuration conf = new Configuration(this.util.getConfiguration());
+      Map<String, Integer> familyToBlockSize =
+          getMockColumnFamiliesForBlockSize(numCfs);
+      HTable table = Mockito.mock(HTable.class);
+      setupMockColumnFamiliesForBlockSize(table,
+          familyToBlockSize);
+      HFileOutputFormat.configureBlockSize(table, conf);
+
+      // read back family specific data block size from the configuration
+      Map<byte[], Integer> retrievedFamilyToBlockSizeMap =
+          HFileOutputFormat
+              .createFamilyBlockSizeMap(conf);
+
+      // test that we have a value for all column families that matches with the
+      // used mock values
+      for (Entry<String, Integer> entry : familyToBlockSize.entrySet()) {
+        assertEquals("BlockSize configuration incorrect for column family:"
+            + entry.getKey(), entry.getValue(), 
+            retrievedFamilyToBlockSizeMap.get(entry.getKey().getBytes()));
+      }
+    }
+  }
+
+  private void setupMockColumnFamiliesForBlockSize(HTable table,
+      Map<String, Integer> familyToDataBlockEncoding) throws IOException {
+    HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME);
+    for (Entry<String, Integer> entry : familyToDataBlockEncoding.entrySet()) {
+      mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey())
+          .setMaxVersions(1)
+          .setBlocksize(entry.getValue())
+          .setBlockCacheEnabled(false)
+          .setTimeToLive(0));
+    }
+    Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor();
+  }
+
+  /**
+   * @return a map from column family names to compression algorithms for
+   *         testing column family compression. Column family names have special characters
+   */
+  private Map<String, Integer>
+  getMockColumnFamiliesForBlockSize (int numCfs) {
+    Map<String, Integer> familyToBlockSize =
+        new HashMap<String, Integer>();
+    // use column family names having special characters
+    if (numCfs-- > 0) {
+      familyToBlockSize.put("Family1!@#!@#&", 1234);
+    }
+    if (numCfs-- > 0) {
+      familyToBlockSize.put("Family2=asdads&!AASD",
+          Integer.MAX_VALUE);
+    }
+    if (numCfs-- > 0) {
+      familyToBlockSize.put("Family2=asdads&!AASD",
+          Integer.MAX_VALUE);
+    }
+    if (numCfs-- > 0) {
+      familyToBlockSize.put("Family3", 0);
+    }
+    return familyToBlockSize;
+  }
+
+    /**
+   * Test for {@link HFileOutputFormat#configureDataBlockEncoding(HTable,
+   * Configuration)} and {@link HFileOutputFormat#createFamilyDataBlockEncodingMap
+   * (Configuration)}.
+   * Tests that the compression map is correctly serialized into
+   * and deserialized from configuration
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testSerializeDeserializeFamilyDataBlockEncodingMap() throws IOException {
+    for (int numCfs = 0; numCfs <= 3; numCfs++) {
+      Configuration conf = new Configuration(this.util.getConfiguration());
+      Map<String, DataBlockEncoding> familyToDataBlockEncoding =
+          getMockColumnFamiliesForDataBlockEncoding(numCfs);
+      HTable table = Mockito.mock(HTable.class);
+      setupMockColumnFamiliesForDataBlockEncoding(table,
+          familyToDataBlockEncoding);
+      HFileOutputFormat.configureDataBlockEncoding(table, conf);
+
+      // read back family specific data block encoding settings from the configuration
+      Map<byte[], HFileDataBlockEncoder> retrievedFamilyToDataBlockEncodingMap =
+          HFileOutputFormat
+              .createFamilyDataBlockEncodingMap(conf);
+
+      // test that we have a value for all column families that matches with the
+      // used mock values
+      for (Entry<String, DataBlockEncoding> entry : familyToDataBlockEncoding.entrySet()) {
+        assertEquals("DataBlockEncoding configuration incorrect for column family:"
+            + entry.getKey(), entry.getValue(),
+            retrievedFamilyToDataBlockEncodingMap.get(entry.getKey().getBytes
+                ()).getEncodingOnDisk());
+      }
+    }
+  }
+
+  private void setupMockColumnFamiliesForDataBlockEncoding(HTable table,
+      Map<String, DataBlockEncoding> familyToDataBlockEncoding) throws IOException {
+    HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME);
+    for (Entry<String, DataBlockEncoding> entry : familyToDataBlockEncoding.entrySet()) {
+      mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey())
+          .setMaxVersions(1)
+          .setDataBlockEncoding(entry.getValue())
+          .setBlockCacheEnabled(false)
+          .setTimeToLive(0));
+    }
+    Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor();
+  }
+
+  /**
+   * @return a map from column family names to compression algorithms for
+   *         testing column family compression. Column family names have special characters
+   */
+  private Map<String, DataBlockEncoding>
+      getMockColumnFamiliesForDataBlockEncoding (int numCfs) {
+    Map<String, DataBlockEncoding> familyToDataBlockEncoding =
+        new HashMap<String, DataBlockEncoding>();
+    // use column family names having special characters
+    if (numCfs-- > 0) {
+      familyToDataBlockEncoding.put("Family1!@#!@#&", DataBlockEncoding.DIFF);
+    }
+    if (numCfs-- > 0) {
+      familyToDataBlockEncoding.put("Family2=asdads&!AASD",
+          DataBlockEncoding.FAST_DIFF);
+    }
+    if (numCfs-- > 0) {
+      familyToDataBlockEncoding.put("Family2=asdads&!AASD",
+          DataBlockEncoding.PREFIX);
+    }
+    if (numCfs-- > 0) {
+      familyToDataBlockEncoding.put("Family3", DataBlockEncoding.NONE);
+    }
+    return familyToDataBlockEncoding;
+  }
+
+  private void setupMockStartKeys(HTable table) throws IOException {
+    byte[][] mockKeys = new byte[][] {
+        HConstants.EMPTY_BYTE_ARRAY,
+        Bytes.toBytes("aaa"),
+        Bytes.toBytes("ggg"),
+        Bytes.toBytes("zzz")
+    };
+    Mockito.doReturn(mockKeys).when(table).getStartKeys();
+  }
+
   /**
    * Test that {@link HFileOutputFormat} RecordWriter uses compression and
    * bloom filter settings from the column family descriptor
@@ -618,7 +840,7 @@ public void testColumnFamilySettings() throws Exception {
         if (bloomFilter == null) bloomFilter = Bytes.toBytes("NONE");
         assertEquals("Incorrect bloom filter used for column family " + familyStr +
           "(reader: " + reader + ")",
-          hcd.getBloomFilterType(), StoreFile.BloomType.valueOf(Bytes.toString(bloomFilter)));
+          hcd.getBloomFilterType(), BloomType.valueOf(Bytes.toString(bloomFilter)));
         assertEquals("Incorrect compression used for column family " + familyStr +
           "(reader: " + reader + ")", hcd.getCompression(), reader.getCompressionAlgorithm());
       }

From f828448c337e63342018375fad6a020e60c67101 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 12 Apr 2014 23:18:32 +0000
Subject: [PATCH 1380/1540] HBASE-10807 -ROOT- still stale in table.jsp if it
 moved. (Esteban Gutierrez)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1586907 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/catalog/CatalogTracker.java    | 2 +-
 src/main/resources/hbase-webapps/master/table.jsp               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index bf606288200d..dcdd53c58efd 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -303,7 +303,7 @@ public void waitForRoot()
    * @throws NotAllMetaRegionsOnlineException if root not available before
    * timeout
    */
-  ServerName waitForRoot(final long timeout)
+  public ServerName waitForRoot(final long timeout)
   throws InterruptedException, NotAllMetaRegionsOnlineException {
     ServerName sn = rootRegionTracker.waitRootRegionLocation(timeout);
     if (sn == null) {
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index f5c85b57429a..3556ed5ec822 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -43,7 +43,7 @@
   String tableName = request.getParameter("name");
   HTable table = new HTable(conf, tableName);
   String tableHeader = "<h2>Table Regions</h2><table><tr><th>Name</th><th>Region Server</th><th>Start Key</th><th>End Key</th><th>Requests</th></tr>";
-  ServerName rl = master.getCatalogTracker().getRootLocation();
+  ServerName rl = master.getCatalogTracker().waitForRoot(1);
   boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false);
   boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false);
   Map<String, Integer> frags = null;

From d0cb64b880319daca22a0dd0a3e053e65ea69d4d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 14 Apr 2014 05:10:46 +0000
Subject: [PATCH 1381/1540] HBASE-10969 TestDistributedLogSplitting fails
 frequently in 0.94.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587151 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/master/TestDistributedLogSplitting.java   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 83bd4c1f6a17..fe0e6482a686 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -122,10 +122,11 @@ private void startCluster(int num_master, int num_rs, Configuration inConf) thro
 
   @After
   public void after() throws Exception {
-    for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
-      mt.getMaster().abort("closing...", new Exception("Trace info"));
+    if (TEST_UTIL.getHBaseCluster() != null) {
+      for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
+        mt.getMaster().abort("closing...", new Exception("Trace info"));
+      }
     }
-
     TEST_UTIL.shutdownMiniCluster();
   }
 

From 3af6521768c96969ae47db6f677455295c1eefe6 Mon Sep 17 00:00:00 2001
From: rajeshbabu <rajeshbabu@unknown>
Date: Mon, 14 Apr 2014 19:50:27 +0000
Subject: [PATCH 1382/1540] HBASE-10533 commands.rb is giving wrong error
 messages on exceptions(rajeshbabu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587297 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/client/HBaseAdmin.java       |  2 +-
 src/main/ruby/shell/commands.rb               | 38 +++++++++++++++----
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 75106bc0a8a2..678e02a66755 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2262,7 +2262,7 @@ public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
   public void cloneSnapshot(final String snapshotName, final String tableName)
       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
     if (tableExists(tableName)) {
-      throw new TableExistsException("Table '" + tableName + " already exists");
+      throw new TableExistsException(tableName);
     }
     internalRestoreSnapshot(snapshotName, tableName);
     waitUntilTableIsEnabled(Bytes.toBytes(tableName));
diff --git a/src/main/ruby/shell/commands.rb b/src/main/ruby/shell/commands.rb
index af6df3344008..20e6aafa6a5a 100644
--- a/src/main/ruby/shell/commands.rb
+++ b/src/main/ruby/shell/commands.rb
@@ -72,13 +72,37 @@ def format_simple_command
 
       def translate_hbase_exceptions(*args)
         yield
-      rescue org.apache.hadoop.hbase.TableNotFoundException
-        raise "Unknown table #{args.first}!"
-      rescue org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException
-        valid_cols = table(args.first).get_all_columns.map { |c| c + '*' }
-        raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}"
-      rescue org.apache.hadoop.hbase.TableExistsException
-        raise "Table already exists: #{args.first}!"
+      rescue => e
+        raise e unless e.respond_to?(:cause) && e.cause != nil
+        
+        # Get the special java exception which will be handled
+        cause = e.cause
+        if cause.kind_of?(org.apache.hadoop.hbase.TableNotFoundException) then
+          str = java.lang.String.new("#{cause}")
+          raise "Unknown table #{str}!"
+        end
+        if cause.kind_of?(org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException) then
+          exceptions = cause.getCauses
+          exceptions.each do |exception|
+            if exception.kind_of?(org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException) then
+              valid_cols = table(args.first).get_all_columns.map { |c| c + '*' }
+              raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}"
+            end
+          end
+        end
+        if cause.kind_of?(org.apache.hadoop.hbase.TableExistsException) then
+          str = java.lang.String.new("#{cause}")
+          strs = str.split("\n")
+          if strs.size > 0 then
+            s = strs[0].split(' ');
+            if(s.size > 1)
+              raise "Table already exists: #{s[1]}!"
+            end
+              raise "Table already exists: #{strs[0]}!"
+          end
+        end
+        # Throw the other exception which hasn't been handled above       
+        raise e
       end
     end
   end

From a6b0e568511f883231b2194e46253f6e9a4fb1b0 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 15 Apr 2014 03:59:53 +0000
Subject: [PATCH 1383/1540] HBASE-10982
 TestZKProcedure.testMultiCohortWithMemberTimeoutDuringPrepare fails
 frequently in 0.94.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587398 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/procedure/TestZKProcedure.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
index 36e0855c0fe8..6e098c0e45de 100644
--- a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
+++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
@@ -21,6 +21,7 @@
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyListOf;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atMost;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
@@ -316,7 +317,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
     // -------------
     // verification
     // -------------
-    waitAndVerifyProc(coordinatorTask, once, never(), once, once, true);
+    // always expect prepared, never committed, and possible to have cleanup and finish (racy since
+    // error case)
+    waitAndVerifyProc(coordinatorTask, once, never(), once, atMost(1), true);
     verifyCohortSuccessful(expected, subprocFactory, cohortTasks, once, never(), once,
       once, true);
 

From 271f745b8514938a0d3341e60ffe23b91fd2dd08 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Tue, 15 Apr 2014 06:18:05 +0000
Subject: [PATCH 1384/1540] HBASE-10966 RowCounter misinterprets column names
 that have colons in their qualifier

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587445 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/hadoop/hbase/mapreduce/RowCounter.java | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
index 5ebe71232086..ce2135fb1262 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import java.io.IOException;
+import org.apache.commons.lang.StringUtils;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -33,6 +34,7 @@
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
 
+
 /**
  * A job with a just a map phase to count rows. Map outputs table rows IF the
  * input row has columns that have content.
@@ -119,11 +121,12 @@ public static Job createSubmittableJob(Configuration conf, String[] args)
     scan.setFilter(new FirstKeyOnlyFilter());
     if (sb.length() > 0) {
       for (String columnName : sb.toString().trim().split(" ")) {
-        String [] fields = columnName.split(":");
-        if(fields.length == 1) {
-          scan.addFamily(Bytes.toBytes(fields[0]));
+        String family = StringUtils.substringBefore(columnName, ":");
+        String qualifier = StringUtils.substringAfter(columnName, ":");
+        if (StringUtils.isBlank(qualifier)) {
+          scan.addFamily(Bytes.toBytes(family));
         } else {
-          scan.addColumn(Bytes.toBytes(fields[0]), Bytes.toBytes(fields[1]));
+          scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
         }
       }
     }

From fdc97c03e873be948958cb933fcd9c2f6986b54d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 15 Apr 2014 07:16:16 +0000
Subject: [PATCH 1385/1540] HBASE-10969 Addendum - avoid races in dfs/zk
 cluster start/stop

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587458 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/TestDistributedLogSplitting.java   | 40 +++++++++++++++++--
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index fe0e6482a686..d397b6431fa0 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -70,13 +70,19 @@
 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.KeeperException;
 import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -94,8 +100,25 @@ public class TestDistributedLogSplitting {
   MiniHBaseCluster cluster;
   HMaster master;
   Configuration conf;
-  HBaseTestingUtility TEST_UTIL;
+  static HBaseTestingUtility TEST_UTIL;
+  static Configuration originalConf;
+  static MiniDFSCluster dfsCluster;
+  static MiniZooKeeperCluster zkCluster;
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    TEST_UTIL = new HBaseTestingUtility(HBaseConfiguration.create());
+    dfsCluster = TEST_UTIL.startMiniDFSCluster(1);
+    zkCluster = TEST_UTIL.startMiniZKCluster();
+    originalConf = TEST_UTIL.getConfiguration();
+  }
 
+  @AfterClass
+  public static void tearDown() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+    TEST_UTIL.shutdownMiniDFSCluster();
+    TEST_UTIL.shutdownMiniHBaseCluster();
+  }
 
   private void startCluster(int num_rs) throws Exception{
     conf = HBaseConfiguration.create();
@@ -109,8 +132,11 @@ private void startCluster(int num_master, int num_rs, Configuration inConf) thro
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
+    TEST_UTIL.shutdownMiniHBaseCluster();
     TEST_UTIL = new HBaseTestingUtility(conf);
-    TEST_UTIL.startMiniCluster(num_master, num_rs);
+    TEST_UTIL.setDFSCluster(dfsCluster);
+    TEST_UTIL.setZkCluster(zkCluster);
+    TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
@@ -120,6 +146,12 @@ private void startCluster(int num_master, int num_rs, Configuration inConf) thro
     }
   }
 
+  @Before
+  public void before() throws Exception {
+    // refresh configuration
+    conf = HBaseConfiguration.create(originalConf);
+  }
+  
   @After
   public void after() throws Exception {
     if (TEST_UTIL.getHBaseCluster() != null) {
@@ -127,7 +159,9 @@ public void after() throws Exception {
         mt.getMaster().abort("closing...", new Exception("Trace info"));
       }
     }
-    TEST_UTIL.shutdownMiniCluster();
+    TEST_UTIL.shutdownMiniHBaseCluster();
+    TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true);
+    ZKUtil.deleteNodeRecursively(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL), "/hbase");
   }
 
   @Test (timeout=300000)

From 8aa894f37cbabd119229ca19fa16d299b2d0ea9e Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 15 Apr 2014 16:15:15 +0000
Subject: [PATCH 1386/1540] HBASE-10845 Memstore snapshot size isn't updated in
 DefaultMemStore#rollback()

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587628 13f79535-47bb-0310-9956-ffa450edef68
---
 .../java/org/apache/hadoop/hbase/regionserver/MemStore.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index cb920a956f18..8b8a5b6be111 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -295,6 +295,8 @@ void rollback(final KeyValue kv) {
     KeyValue found = this.snapshot.get(kv);
     if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
       this.snapshot.remove(kv);
+      long sz = heapSizeChange(kv, true);
+      this.snapshotSize -= sz;
     }
     // If the key is in the memstore, delete it. Update this.size.
     found = this.kvset.get(kv);

From 9e8b960a94b18124530aa3e80048fc3181e535c9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 15 Apr 2014 19:15:20 +0000
Subject: [PATCH 1387/1540] HBASE-10987 Increase timeout in
 TestZKLeaderManager.testLeaderSelection.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587696 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java
index 43046809e257..8df742cc9e1d 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java
@@ -204,8 +204,8 @@ public void testLeaderSelection() throws Exception {
   private MockLeader getCurrentLeader() throws Exception {
     MockLeader currentLeader = null;
     outer:
-    // wait up to 2 secs for initial leader
-    for (int i = 0; i < 20; i++) {
+    // wait up to 10 secs for initial leader
+    for (int i = 0; i < 100; i++) {
       for (int j = 0; j < CANDIDATES.length; j++) {
         if (CANDIDATES[j].isMaster()) {
           // should only be one leader

From 4f34eb7485b0ae99462d3aa3fc91c67f2489741d Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 15 Apr 2014 22:41:39 +0000
Subject: [PATCH 1388/1540] HBASE-10989 TestAccessController needs better
 timeout.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587741 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/TestAccessController.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 861aeb8d73a6..d2b52d949e3f 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -1845,11 +1845,11 @@ public Object run() throws Exception {
 
       final int RETRIES_LIMIT = 10;
       int retries = 0;
-      while (newRs.getOnlineRegions().size() < 1 && retries < RETRIES_LIMIT) {
+      while (newRs.getOnlineRegions(TEST_TABLE2).size() < 1 && retries < RETRIES_LIMIT) {
         LOG.debug("Waiting for region to be opened. Already retried " + retries
             + " times.");
         try {
-          Thread.sleep(200);
+          Thread.sleep(1000);
         } catch (InterruptedException e) {
         }
         retries++;

From 83e238027b783774471e5e0e525ebf04c0040455 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Tue, 15 Apr 2014 22:52:47 +0000
Subject: [PATCH 1389/1540] HBASE-10312 Flooding the cluster with
 administrative actions leads to collapse

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587745 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/monitoring/TaskMonitor.java  | 23 ++++---------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
index d9c4d4bf99cc..dbdadccd4177 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
@@ -28,6 +28,7 @@
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.commons.collections.buffer.CircularFifoBuffer;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -50,8 +51,7 @@ public class TaskMonitor {
   static final int MAX_TASKS = 1000;
   
   private static TaskMonitor instance;
-  private List<TaskAndWeakRefPair> tasks =
-    Lists.newArrayList();
+  private CircularFifoBuffer tasks = new CircularFifoBuffer(MAX_TASKS);
 
   /**
    * Get singleton instance.
@@ -73,9 +73,6 @@ public synchronized MonitoredTask createStatus(String description) {
         new PassthroughInvocationHandler<MonitoredTask>(stat));
     TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
     tasks.add(pair);
-    if (tasks.size() > MAX_TASKS) {
-      purgeExpiredTasks();
-    }
     return proxy;
   }
 
@@ -88,15 +85,10 @@ public synchronized MonitoredRPCHandler createRPCStatus(String description) {
         new PassthroughInvocationHandler<MonitoredRPCHandler>(stat));
     TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
     tasks.add(pair);
-    if (tasks.size() > MAX_TASKS) {
-      purgeExpiredTasks();
-    }
     return proxy;
   }
 
   private synchronized void purgeExpiredTasks() {
-    int size = 0;
-    
     for (Iterator<TaskAndWeakRefPair> it = tasks.iterator();
          it.hasNext();) {
       TaskAndWeakRefPair pair = it.next();
@@ -113,15 +105,8 @@ private synchronized void purgeExpiredTasks() {
       
       if (canPurge(stat)) {
         it.remove();
-      } else {
-        size++;
       }
     }
-    
-    if (size > MAX_TASKS) {
-      LOG.warn("Too many actions in action monitor! Purging some.");
-      tasks = tasks.subList(size - MAX_TASKS, size);
-    }
   }
 
   /**
@@ -132,7 +117,9 @@ private synchronized void purgeExpiredTasks() {
   public synchronized List<MonitoredTask> getTasks() {
     purgeExpiredTasks();
     ArrayList<MonitoredTask> ret = Lists.newArrayListWithCapacity(tasks.size());
-    for (TaskAndWeakRefPair pair : tasks) {
+    for (Iterator<TaskAndWeakRefPair> it = tasks.iterator();
+         it.hasNext();) {
+      TaskAndWeakRefPair pair = it.next();
       MonitoredTask t = pair.get();
       ret.add(t.clone());
     }

From 01a81e124543fa25892da470b8718498d7925bf5 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Wed, 16 Apr 2014 00:14:48 +0000
Subject: [PATCH 1390/1540] HBASE-10991 Port HBASE-10639 'Unload script
 displays wrong counts (off by one) when unloading regions' to 0.94 (Srikanth
 Srungarapu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1587762 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/region_mover.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 9bd9d86d8f29..58761635dd12 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -350,7 +350,7 @@ def unloadRegions(options, hostname)
       # Get a random server to move the region to.
       server = servers[rand(servers.length)]
       $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + 
-        count.to_s + " of " + rs.length.to_s + ") from server=" + 
+        (count + 1).to_s + " of " + rs.length.to_s + ") from server=" +
         servername + " to server=" + server);
       count = count + 1
       # Assert we can scan region in its current location
@@ -410,8 +410,8 @@ def loadRegions(options, hostname)
         " of " + regions.length.to_s + ") already on target server=" + servername) 
       next
     end
-    $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + 
-      count.to_s + " of " + regions.length.to_s + ") from server=" + 
+    $LOG.info("Moving region " + r.getRegionNameAsString() + " (" +
+      (count + 1).to_s + " of " + regions.length.to_s + ") from server=" +
       currentServer.to_s + " to server=" + servername.to_s);
     move(admin, r, servername, currentServer)
   end

From 42c839a50627b9a1d7c85972f9738284218d31ba Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 16 Apr 2014 18:20:04 +0000
Subject: [PATCH 1391/1540] HBASE-10996 TestTableSnapshotInputFormatScan fails
 frequently on 0.94.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588012 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/mapreduce/TestTableSnapshotInputFormatScan.java      | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java
index 69c988f39874..08c4dcb612aa 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormatScan.java
@@ -67,9 +67,6 @@ public static void setUpBeforeClass() throws Exception {
     // config snapshot support
     TEST_UTIL.getConfiguration().setBoolean(
         SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
-    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
     TEST_UTIL.getConfiguration().setBoolean(
         "hbase.master.enabletable.roundrobin", true);
 

From f3c9b92eb166f1ad929f9a0d791c933278cd8678 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 16 Apr 2014 18:22:41 +0000
Subject: [PATCH 1392/1540] HBASE-10988 Properly wait for server in
 TestThriftServerCmdLine.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588015 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index 516f3215a635..c83bbee816ea 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -164,7 +164,11 @@ public void testRunThriftServer() throws Exception {
 
     thriftServer = new ThriftServer(TEST_UTIL.getConfiguration());
     startCmdLineThread(args.toArray(new String[0]));
-    Threads.sleepWithoutInterrupt(2000);
+    // wait up to 10s for the server to start
+    for (int i = 0; i < 100
+        && (thriftServer.serverRunner == null || thriftServer.serverRunner.tserver == null); i++) {
+      Thread.sleep(100);
+    }
 
     Class<? extends TServer> expectedClass = implType != null ?
         implType.serverClass : TBoundedThreadPoolServer.class;

From b5d07373ffc37847470d3123edee195ef2692861 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Thu, 17 Apr 2014 04:07:09 +0000
Subject: [PATCH 1393/1540] HBASE-11010 TestChangingEncoding is unnecessarily
 slow.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588128 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/io/encoding/TestChangingEncoding.java      | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
index 15b13cb17a45..7742b4eb8f57 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
@@ -134,14 +135,17 @@ static void writeTestDataBatch(Configuration conf, String tableName,
       int batchId) throws Exception {
     LOG.debug("Writing test data batch " + batchId);
     HTable table = new HTable(conf, tableName);
+    table.setAutoFlush(false);
     for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
       Put put = new Put(getRowKey(batchId, i));
       for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
         put.add(CF_BYTES, getQualifier(j),
             getValue(batchId, i, j));
-        table.put(put);
       }
+      put.setDurability(Durability.SKIP_WAL);
+      table.put(put);
     }
+    table.flushCommits();
     table.close();
   }
 

From d67f591003a9b6d4078830e1387d9650af74e0cc Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 17 Apr 2014 15:15:08 +0000
Subject: [PATCH 1394/1540] HBASE-11003 ExportSnapshot is using the wrong fs
 when staging dir is not in fs.defaultFS

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588280 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java     | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 46741e70c440..b3ec3f28d59c 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -535,11 +535,16 @@ public int compare(Pair<Path, Long> a, Pair<Path, Long> b) {
     return fileGroups;
   }
 
-  private static Path getInputFolderPath(final FileSystem fs, final Configuration conf)
+  private static Path getInputFolderPath(final Configuration conf)
       throws IOException, InterruptedException {
     String stagingName = "exportSnapshot-" + EnvironmentEdgeManager.currentTimeMillis();
-    Path stagingDir = new Path(conf.get(CONF_STAGING_ROOT, fs.getWorkingDirectory().toString())
-        , stagingName);
+    String stagingDirPath = conf.get(CONF_STAGING_ROOT);
+    if (stagingDirPath == null) {
+      stagingDirPath = FileSystem.get(conf).getWorkingDirectory().toString();
+    }
+
+    Path stagingDir = new Path(stagingDirPath, stagingName);
+    FileSystem fs = stagingDir.getFileSystem(conf);
     fs.mkdirs(stagingDir);
     return new Path(stagingDir, INPUT_FOLDER_PREFIX +
       String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
@@ -554,8 +559,8 @@ private static Path getInputFolderPath(final FileSystem fs, final Configuration
   private static Path[] createInputFiles(final Configuration conf,
       final List<Pair<Path, Long>> snapshotFiles, int mappers)
       throws IOException, InterruptedException {
-    FileSystem fs = FileSystem.get(conf);
-    Path inputFolderPath = getInputFolderPath(fs, conf);
+    Path inputFolderPath = getInputFolderPath(conf);
+    FileSystem fs = inputFolderPath.getFileSystem(conf);
     LOG.debug("Input folder location: " + inputFolderPath);
 
     List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);

From 44c15bada4a46827d2773690a7e83cd4cc49d860 Mon Sep 17 00:00:00 2001
From: Michael Stack <stack@apache.org>
Date: Fri, 18 Apr 2014 00:20:41 +0000
Subject: [PATCH 1395/1540] HBASE-11017
 TestHRegionBusyWait.testWritesWhileScanning fails frequently in 0.94

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588392 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/regionserver/TestHRegion.java    | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 0c9c4004cff4..55c2d026acb4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3403,7 +3403,14 @@ public void testWritesWhileScanning()
       flushThread.join();
       flushThread.checkNoError();
     } finally {
-      HRegion.closeHRegion(this.region);
+      try {
+        HRegion.closeHRegion(this.region);
+      } catch (DroppedSnapshotException dse) {
+        // We could get this on way out because we interrupt the background flusher and it could
+        // fail anywhere causing a DSE over in the background flusher... only it is not properly
+        // dealt with so could still be memory hanging out when we get to here -- memory we can't
+        // flush because the accounting is 'off' since original DSE.
+      }
       this.region = null;
     }
   }

From cf34ad21b9a6b4611b9c403ff3f5aece2c08747b Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 18 Apr 2014 00:35:17 +0000
Subject: [PATCH 1396/1540] HBASE-11022 Increase timeout for
 TestHBaseFsck.testSplitDaughtersNotInMeta.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588393 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 20a413332f8a..4ec70cf7eb9a 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1319,7 +1319,7 @@ public void testValidLingeringSplitParent() throws Exception {
    * Split crashed after write to META finished for the parent region, but
    * failed to write daughters (pre HBASE-7721 codebase)
    */
-  @Test
+  @Test(timeout=75000)
   public void testSplitDaughtersNotInMeta() throws Exception {
     String table = "testSplitdaughtersNotInMeta";
     HTable meta = null;

From 7bec7a8188a5000d5a718aa2ef640648ab9080f1 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 18 Apr 2014 03:54:47 +0000
Subject: [PATCH 1397/1540] HBASE-11024
 TestSecureLoadIncrementalHFilesSplitRecovery should wait longer for ACL
 table.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588408 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java | 2 +-
 .../mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
index f3187d590463..10aea8089b0b 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java
@@ -48,7 +48,7 @@ public static void setUpBeforeClass() throws Exception {
     util.startMiniCluster();
 
     // Wait for the ACL table to become available
-    util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+    util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 30000);
   }
 
 }
diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
index 7eadedd65278..38ac80899111 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java
@@ -54,7 +54,7 @@ public static void setupCluster() throws Exception {
     util.startMiniCluster();
 
     // Wait for the ACL table to become available
-    util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+    util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 30000);
   }
 
   //Disabling this test as it does not work in secure mode

From 9602dcfaafae81f8590848c766f40bf0b1d1afcb Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 18 Apr 2014 17:52:00 +0000
Subject: [PATCH 1398/1540] HBASE-11029 Increase wait in
 TestSplitTransactionOnCluster.split.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588530 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/regionserver/TestSplitTransactionOnCluster.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index ddc43090de94..ec0cfb8f6673 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -937,7 +937,7 @@ private void split(final HRegionInfo hri, final HRegionServer server,
       final int regionCount)
   throws IOException, InterruptedException {
     this.admin.split(hri.getRegionNameAsString());
-    for (int i=0; server.getOnlineRegions().size() <= regionCount && i<100; i++) {
+    for (int i=0; server.getOnlineRegions().size() <= regionCount && i<300; i++) {
       LOG.debug("Waiting on region to split");
       Thread.sleep(100);
     }

From f476e3d54c0787bed8a74697c57ce5a37c570f82 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Fri, 18 Apr 2014 18:48:12 +0000
Subject: [PATCH 1399/1540] HBASE-10988; Addendum: increase wait to 20s.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588541 13f79535-47bb-0310-9956-ffa450edef68
---
 .../apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
index c83bbee816ea..29857dddecea 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java
@@ -164,8 +164,8 @@ public void testRunThriftServer() throws Exception {
 
     thriftServer = new ThriftServer(TEST_UTIL.getConfiguration());
     startCmdLineThread(args.toArray(new String[0]));
-    // wait up to 10s for the server to start
-    for (int i = 0; i < 100
+    // wait up to 20s for the server to start
+    for (int i = 0; i < 200
         && (thriftServer.serverRunner == null || thriftServer.serverRunner.tserver == null); i++) {
       Thread.sleep(100);
     }

From 28c7dd69ca6ae3430efc10a15bff40c939e7145a Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 19 Apr 2014 05:16:17 +0000
Subject: [PATCH 1400/1540] HBASE-11037 Race condition in
 TestZKBasedOpenCloseRegion.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588609 13f79535-47bb-0310-9956-ffa450edef68
---
 .../master/TestZKBasedOpenCloseRegion.java    | 72 +------------------
 1 file changed, 2 insertions(+), 70 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
index 516c02864ac8..8ddb3511acb4 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
@@ -28,6 +28,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -213,40 +214,6 @@ public void afterProcess(EventHandler event) {
     }
   }
 
-  public static class CloseRegionEventListener implements EventHandlerListener {
-    private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
-    String regionToClose;
-    AtomicBoolean closeEventProcessed;
-
-    public CloseRegionEventListener(String regionToClose,
-        AtomicBoolean closeEventProcessed) {
-      this.regionToClose = regionToClose;
-      this.closeEventProcessed = closeEventProcessed;
-    }
-
-    @Override
-    public void afterProcess(EventHandler event) {
-      LOG.info("afterProcess(" + event + ")");
-      if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) {
-        LOG.info("Finished processing CLOSE REGION");
-        TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
-        if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
-          LOG.info("Setting closeEventProcessed flag");
-          closeEventProcessed.set(true);
-        } else {
-          LOG.info("Region to close didn't match");
-        }
-      }
-    }
-
-    @Override
-    public void beforeProcess(EventHandler event) {
-      if(event.getEventType() == EventType.M_RS_CLOSE_REGION) {
-        LOG.info("Received CLOSE RPC and beginning to process it");
-      }
-    }
-  }
-
   /**
    * This test shows how a region won't be able to be assigned to a RS
    * if it's already "processing" it.
@@ -308,31 +275,6 @@ public void testRSAlreadyProcessingRegion() throws Exception {
 
   }
 
-  @Test (timeout=300000) public void testCloseRegion()
-  throws Exception {
-    LOG.info("Running testCloseRegion");
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-    LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
-
-    int rsIdx = 0;
-    HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
-    HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
-    LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
-
-    AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
-    EventHandlerListener listener =
-      new CloseRegionEventListener(hri.getRegionNameAsString(),
-          closeEventProcessed);
-    cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener);
-
-    cluster.getMaster().assignmentManager.unassign(hri);
-
-    while (!closeEventProcessed.get()) {
-      Threads.sleep(100);
-    }
-    LOG.info("Done with testCloseRegion");
-  }
-
   /**
    * If region open fails with IOException in openRegion() while doing tableDescriptors.get()
    * the region should not add into regionsInTransitionInRS map
@@ -410,7 +352,7 @@ private static int addToEachStartKey(final int expected) throws IOException {
       // If start key, add 'aaa'.
       byte [] row = getStartKey(hri);
       Put p = new Put(row);
-      p.setWriteToWAL(false);
+      p.setDurability(Durability.SKIP_WAL);
       p.add(getTestFamily(), getTestQualifier(), row);
       t.put(p);
       rows++;
@@ -435,16 +377,6 @@ private static int addToEachStartKey(final int expected) throws IOException {
     return getTestFamily();
   }
 
-  public static void main(String args[]) throws Exception {
-    TestZKBasedOpenCloseRegion.beforeAllTests();
-
-    TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
-    test.setup();
-    test.testCloseRegion();
-
-    TestZKBasedOpenCloseRegion.afterAllTests();
-  }
-
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

From 5728be9af89b35f0e88c8f382e597255cf7bbcca Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sat, 19 Apr 2014 05:30:12 +0000
Subject: [PATCH 1401/1540] HBASE-11030 HBaseTestingUtility.getMiniHBaseCluster
 should be able to return null.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588616 13f79535-47bb-0310-9956-ffa450edef68
---
 src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 694c278688ea..c2bcd8004346 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -695,7 +695,7 @@ public void restartHBaseCluster(int servers) throws IOException, InterruptedExce
    * @see #startMiniCluster()
    */
   public MiniHBaseCluster getMiniHBaseCluster() {
-    if (this.hbaseCluster instanceof MiniHBaseCluster) {
+    if (this.hbaseCluster == null || this.hbaseCluster instanceof MiniHBaseCluster) {
       return (MiniHBaseCluster)this.hbaseCluster;
     }
     throw new RuntimeException(hbaseCluster + " not an instance of " +

From daf38c888ded7b2cc1a00af9abb1637f4786c5d9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 20 Apr 2014 01:58:06 +0000
Subject: [PATCH 1402/1540] HBASE-11040 TestAccessController,
 TestAccessControllerFilter, and TestTablePermissions need to wait longer to
 ACL table.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588716 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/security/access/TestAccessControlFilter.java   | 2 +-
 .../hadoop/hbase/security/access/TestAccessController.java      | 2 +-
 .../hadoop/hbase/security/access/TestTablePermissions.java      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
index bd423f18a8cb..7f7ef55ec5ee 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
@@ -70,7 +70,7 @@ public static void setupBeforeClass() throws Exception {
     conf.set("hbase.superuser", conf.get("hbase.superuser", "") +
         String.format(",%s.hfs.0,%s.hfs.1,%s.hfs.2", baseuser, baseuser, baseuser));
     TEST_UTIL.startMiniCluster();
-    TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+    TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 30000);
 
     ADMIN = User.createUserForTesting(conf, "admin", new String[]{"supergroup"});
     READER = User.createUserForTesting(conf, "reader", new String[0]);
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index d2b52d949e3f..c50a59957db3 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -145,7 +145,7 @@ public static void setupBeforeClass() throws Exception {
       Coprocessor.PRIORITY_HIGHEST, 1, conf);
 
     // Wait for the ACL table to become available
-    TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+    TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 30000);
 
     // create a set of test users
     SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
index 0410e97a79f5..a3a0f8a9610e 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java
@@ -91,7 +91,7 @@ public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
 
     // Wait for the ACL table to become available
-    UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000);
+    UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 30000);
 
     ZKW = new ZooKeeperWatcher(UTIL.getConfiguration(),
       "TestTablePermissions", ABORTABLE);

From 22dda7fe340e0f4769e4426b2e37f16b553ec89e Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Sun, 20 Apr 2014 23:55:17 +0000
Subject: [PATCH 1403/1540] HBASE-11042 TestForceCacheImportantBlocks OOMs
 occasionally in 0.94.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588841 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hfile/TestForceCacheImportantBlocks.java   | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
index 5f8214e0db1e..15b7d13367e0 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
@@ -70,28 +70,30 @@ public class TestForceCacheImportantBlocks {
   /** Extremely small block size, so that we can get some index blocks */
   private static final int BLOCK_SIZE = 256;
   
-  private static final Algorithm COMPRESSION_ALGORITHM =
-      Compression.Algorithm.GZ;
   private static final BloomType BLOOM_TYPE = BloomType.ROW;
 
   private final int hfileVersion;
   private final boolean cfCacheEnabled;
+  private final Algorithm compressionAlgorithm;
 
   @Parameters
   public static Collection<Object[]> parameters() {
     // HFile versions
     return Arrays.asList(new Object[][] {
-        new Object[] { new Integer(1), false },
-        new Object[] { new Integer(1), true },
-        new Object[] { new Integer(2), false },
-        new Object[] { new Integer(2), true }
+        new Object[] { new Integer(1), false, Compression.Algorithm.NONE },
+        new Object[] { new Integer(1), true, Compression.Algorithm.NONE },
+        new Object[] { new Integer(2), false, Compression.Algorithm.NONE },
+        new Object[] { new Integer(2), true, Compression.Algorithm.NONE },
+        new Object[] { new Integer(2), false, Compression.Algorithm.GZ },
+        new Object[] { new Integer(2), true, Compression.Algorithm.GZ }
     });
   }
 
   public TestForceCacheImportantBlocks(int hfileVersion,
-      boolean cfCacheEnabled) {
+      boolean cfCacheEnabled, Algorithm compression) {
     this.hfileVersion = hfileVersion;
     this.cfCacheEnabled = cfCacheEnabled;
+    this.compressionAlgorithm = compression;
     TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY,
         hfileVersion);
   }
@@ -106,7 +108,7 @@ public void testCacheBlocks() throws IOException {
     HColumnDescriptor hcd =
         new HColumnDescriptor(Bytes.toBytes(CF))
             .setMaxVersions(MAX_VERSIONS)
-            .setCompressionType(COMPRESSION_ALGORITHM)
+            .setCompressionType(compressionAlgorithm)
             .setBloomFilterType(BLOOM_TYPE);
     hcd.setBlocksize(BLOCK_SIZE);
     hcd.setBlockCacheEnabled(cfCacheEnabled);

From a14502015ec444b4c133b6b410c9a2fd376d66f9 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Mon, 21 Apr 2014 22:55:12 +0000
Subject: [PATCH 1404/1540] CHANGES.txt and pom.xml for 0.94.19RC0.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1588996 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt | 40 ++++++++++++++++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 065f8e667fb3..1941db77d432 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,44 @@
 HBase Change Log
+Release 0.94.19 - 04/21/2014
+Bug
+
+    [HBASE-10118] - Major compact keeps deletes with future timestamps
+    [HBASE-10312] - Flooding the cluster with administrative actions leads to collapse
+    [HBASE-10533] - commands.rb is giving wrong error messages on exceptions
+    [HBASE-10766] - SnapshotCleaner allows to delete referenced files
+    [HBASE-10805] - Speed up KeyValueHeap.next() a bit
+    [HBASE-10807] - -ROOT- still stale in table.jsp if it moved
+    [HBASE-10845] - Memstore snapshot size isn't updated in DefaultMemStore#rollback()
+    [HBASE-10847] - 0.94: drop non-secure builds, make security the default
+    [HBASE-10848] - Filter SingleColumnValueFilter combined with NullComparator does not work
+    [HBASE-10966] - RowCounter misinterprets column names that have colons in their qualifier
+    [HBASE-10991] - Port HBASE-10639 'Unload script displays wrong counts (off by one) when unloading regions' to 0.94
+    [HBASE-11003] - ExportSnapshot is using the wrong fs when staging dir is not in fs.defaultFS
+    [HBASE-11030] - HBaseTestingUtility.getMiniHBaseCluster should be able to return null
+
+Task
+
+    [HBASE-10921] - Port HBASE-10323 'Auto detect data block encoding in HFileOutputFormat' to 0.94 / 0.96
+
+Test
+
+    [HBASE-10782] - Hadoop2 MR tests fail occasionally because of mapreduce.jobhistory.address is no set in job conf
+    [HBASE-10969] - TestDistributedLogSplitting fails frequently in 0.94.
+    [HBASE-10982] - TestZKProcedure.testMultiCohortWithMemberTimeoutDuringPrepare fails frequently in 0.94
+    [HBASE-10987] - Increase timeout in TestZKLeaderManager.testLeaderSelection
+    [HBASE-10988] - Properly wait for server in TestThriftServerCmdLine
+    [HBASE-10989] - TestAccessController needs better timeout
+    [HBASE-10996] - TestTableSnapshotInputFormatScan fails frequently on 0.94
+    [HBASE-11010] - TestChangingEncoding is unnecessarily slow
+    [HBASE-11017] - TestHRegionBusyWait.testWritesWhileScanning fails frequently in 0.94
+    [HBASE-11022] - Increase timeout for TestHBaseFsck.testSplitDaughtersNotInMeta
+    [HBASE-11024] - TestSecureLoadIncrementalHFilesSplitRecovery should wait longer for ACL table
+    [HBASE-11029] - Increase wait in TestSplitTransactionOnCluster.split
+    [HBASE-11037] - Race condition in TestZKBasedOpenCloseRegion
+    [HBASE-11040] - TestAccessController, TestAccessControllerFilter, and TestTablePermissions need to wait longer to ACL table
+    [HBASE-11042] - TestForceCacheImportantBlocks OOMs occasionally in 0.94
+
+
 Release 0.94.18 - 03/14/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index 6ec107368a9a..7b88b5b45d34 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.19-SNAPSHOT</version>
+  <version>0.94.19</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 0d6d06559b2f440d8e6a5ef4ca2de34edf01c024 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 30 Apr 2014 20:43:52 +0000
Subject: [PATCH 1405/1540] roll version to 0.94.20-SNAPSHOT

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1591477 13f79535-47bb-0310-9956-ffa450edef68
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 7b88b5b45d34..4fb6dfe6a636 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.19</version>
+  <version>0.94.20-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 8ca23f089434cadcd665ef60946890926ad5894e Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Wed, 30 Apr 2014 22:05:39 +0000
Subject: [PATCH 1406/1540] HBASE-11008 Align bulk load, flush, and compact to
 require Action.CREATE

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1591494 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/security/access/AccessController.java      |  8 +++++---
 .../hbase/security/access/TestAccessController.java  | 12 +++++++-----
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 6f56edcf63c0..3b6869da021a 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -830,7 +830,8 @@ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
 
   @Override
   public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
-    requirePermission("flush", getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+    requirePermission("flush", getTableName(e.getEnvironment()), null, null, Action.ADMIN,
+        Action.CREATE);
   }
 
   @Override
@@ -841,7 +842,8 @@ public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOE
   @Override
   public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
       final Store store, final InternalScanner scanner) throws IOException {
-    requirePermission("compact", getTableName(e.getEnvironment()), null, null, Action.ADMIN);
+    requirePermission("compact", getTableName(e.getEnvironment()), null, null, Action.ADMIN,
+        Action.CREATE);
     return scanner;
   }
 
@@ -1091,7 +1093,7 @@ public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
     for(Pair<byte[],String> el : familyPaths) {
       cfs.add(el.getFirst());
     }
-    requirePermission("bulkLoadHFile", Permission.Action.WRITE, ctx.getEnvironment(), cfs);
+    requirePermission("bulkLoadHFile", Permission.Action.CREATE, ctx.getEnvironment(), cfs);
   }
 
   private AuthResult hasSomeAccess(RegionCoprocessorEnvironment e, String request, Action action) throws IOException {
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index c50a59957db3..d696acbfdfd2 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -546,8 +546,8 @@ public Object run() throws Exception {
       }
     };
 
-    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
-    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE);
+    verifyDenied(action, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -559,8 +559,8 @@ public Object run() throws Exception {
       }
     };
 
-    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER);
-    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE);
+    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE);
+    verifyDenied(action, USER_RW, USER_RO, USER_NONE);
   }
 
   @Test
@@ -742,7 +742,8 @@ public Object run() throws Exception {
         return null;
       }
     };
-    verifyWrite(bulkLoadAction);
+    verifyAllowed(bulkLoadAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE);
+    verifyDenied(bulkLoadAction, USER_RW, USER_RO, USER_NONE);
 
     // Reinit after the bulk upload
     TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE);
@@ -805,6 +806,7 @@ private void bulkLoadHFile(
       HTable table = new HTable(conf, tableName);
       try {
         TEST_UTIL.waitTableAvailable(tableName, 30000);
+        conf.setBoolean("hbase.mapreduce.bulkload.assign.sequenceNumbers", true);
         LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
         loader.doBulkLoad(loadPath, table);
       } finally {

From b9576d9bcc094f6b9fe41deea44d792a0cbbfb15 Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Wed, 30 Apr 2014 22:07:26 +0000
Subject: [PATCH 1407/1540] HBASE-10958 [dataloss] Bulk loading with seqids can
 prevent some log entries from being replayed

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1591495 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/HRegion.java    | 138 +++++++++++++++---
 .../hbase/regionserver/HRegionServer.java     |   6 +-
 .../hbase/regionserver/MemStoreFlusher.java   |   2 +-
 .../hadoop/hbase/regionserver/Store.java      |   8 +-
 .../hadoop/hbase/regionserver/StoreFile.java  |   7 +-
 .../mapreduce/TestLoadIncrementalHFiles.java  |  47 +-----
 .../hbase/regionserver/TestCompaction.java    |   2 +-
 .../hbase/regionserver/TestHRegion.java       |  38 ++++-
 .../hbase/regionserver/wal/TestWALReplay.java |  96 ++++++++++--
 .../hadoop/hbase/util/HFileTestUtil.java      |  63 ++++++++
 10 files changed, 317 insertions(+), 90 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e6ad4906154b..134edaa782bb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -341,6 +341,81 @@ boolean isFlushRequested() {
         ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN);
   }
 
+  /**
+   * Objects from this class are created when flushing to describe all the different states that
+   * that method ends up in. The Result enum describes those states. The sequence id should only
+   * be specified if the flush was successful, and the failure message should only be speficied
+   * if it didn't flush.
+   */
+  public static class FlushResult {
+    enum Result {
+      FLUSHED_NO_COMPACTION_NEEDED,
+      FLUSHED_COMPACTION_NEEDED,
+      // Special case where a flush didn't run because there's nothing in the memstores. Used when
+      // bulk loading to know when we can still load even if a flush didn't happen.
+      CANNOT_FLUSH_MEMSTORE_EMPTY,
+      CANNOT_FLUSH
+      // Be careful adding more to this enum, look at the below methods to make sure
+    }
+
+    final Result result;
+    final String failureReason;
+    final long flushSequenceId;
+
+    /**
+     * Convenience constructor to use when the flush is successful, the failure message is set to
+     * null.
+     * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED.
+     * @param flushSequenceId Generated sequence id that comes right after the edits in the
+     *                        memstores.
+     */
+    FlushResult(Result result, long flushSequenceId) {
+      this(result, flushSequenceId, null);
+      assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result
+          .FLUSHED_COMPACTION_NEEDED;
+    }
+
+    /**
+     * Convenience constructor to use when we cannot flush.
+     * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH.
+     * @param failureReason Reason why we couldn't flush.
+     */
+    FlushResult(Result result, String failureReason) {
+      this(result, -1, failureReason);
+      assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH;
+    }
+
+    /**
+     * Constructor with all the parameters.
+     * @param result Any of the Result.
+     * @param flushSequenceId Generated sequence id if the memstores were flushed else -1.
+     * @param failureReason Reason why we couldn't flush, or null.
+     */
+    FlushResult(Result result, long flushSequenceId, String failureReason) {
+      this.result = result;
+      this.flushSequenceId = flushSequenceId;
+      this.failureReason = failureReason;
+    }
+
+    /**
+     * Convenience method, the equivalent of checking if result is
+     * FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_NO_COMPACTION_NEEDED.
+     * @return true if the memstores were flushed, else false.
+     */
+    public boolean isFlushSucceeded() {
+      return result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result
+          .FLUSHED_COMPACTION_NEEDED;
+    }
+
+    /**
+     * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED.
+     * @return True if the flush requested a compaction, else false (doesn't even mean it flushed).
+     */
+    public boolean isCompactionNeeded() {
+      return result == Result.FLUSHED_COMPACTION_NEEDED;
+    }
+  }
+
   final WriteState writestate = new WriteState();
 
   long memstoreFlushSize;
@@ -610,15 +685,12 @@ public Store call() throws IOException {
         for (int i = 0; i < htableDescriptor.getFamilies().size(); i++) {
           Future<Store> future = completionService.take();
           Store store = future.get();
-
           this.stores.put(store.getColumnFamilyName().getBytes(), store);
-          // Do not include bulk loaded files when determining seqIdForReplay
-          long storeSeqIdForReplay = store.getMaxSequenceId(false);
+
+          long storeSeqIdForReplay = store.getMaxSequenceId();
           maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), storeSeqIdForReplay);
-          // Include bulk loaded files when determining seqIdForAssignment
-          long storeSeqIdForAssignment = store.getMaxSequenceId(true);
-          if (maxSeqId == -1 || storeSeqIdForAssignment > maxSeqId) {
-            maxSeqId = storeSeqIdForAssignment;
+          if (maxSeqId == -1 || storeSeqIdForReplay > maxSeqId) {
+            maxSeqId = storeSeqIdForReplay;
           }
           long maxStoreMemstoreTS = store.getMaxMemstoreTS();
           if (maxStoreMemstoreTS > maxMemstoreTS) {
@@ -1483,11 +1555,12 @@ public boolean compact(CompactionRequest cr)
    * @throws DroppedSnapshotException Thrown when replay of hlog is required
    * because a Snapshot was not properly persisted.
    */
-  public boolean flushcache() throws IOException {
+  public FlushResult flushcache() throws IOException {
     // fail-fast instead of waiting on the lock
     if (this.closing.get()) {
-      LOG.debug("Skipping flush on " + this + " because closing");
-      return false;
+      String msg = "Skipping flush on " + this + " because closing";
+      LOG.debug(msg);
+      return new FlushResult(FlushResult.Result.CANNOT_FLUSH, msg);
     }
     MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
     status.setStatus("Acquiring readlock on region");
@@ -1495,9 +1568,10 @@ public boolean flushcache() throws IOException {
     lock.readLock().lock();
     try {
       if (this.closed.get()) {
-        LOG.debug("Skipping flush on " + this + " because closed");
-        status.abort("Skipped: closed");
-        return false;
+        String msg = "Skipping flush on " + this + " because closed";
+        LOG.debug(msg);
+        status.abort(msg);
+        return new FlushResult(FlushResult.Result.CANNOT_FLUSH, msg);
       }
       if (coprocessorHost != null) {
         status.setStatus("Running coprocessor pre-flush hooks");
@@ -1516,14 +1590,15 @@ public boolean flushcache() throws IOException {
                 + ", flushing=" + writestate.flushing + ", writesEnabled="
                 + writestate.writesEnabled);
           }
-          status.abort("Not flushing since "
+          String msg = "Not flushing since "
               + (writestate.flushing ? "already flushing"
-                  : "writes not enabled"));
-          return false;
+              : "writes not enabled");
+          status.abort(msg);
+          return new FlushResult(FlushResult.Result.CANNOT_FLUSH, msg);
         }
       }
       try {
-        boolean result = internalFlushcache(status);
+        FlushResult fs = internalFlushcache(status);
 
         if (coprocessorHost != null) {
           status.setStatus("Running post-flush coprocessor hooks");
@@ -1531,7 +1606,7 @@ public boolean flushcache() throws IOException {
         }
 
         status.markComplete("Flush successful");
-        return result;
+        return fs;
       } finally {
         synchronized (writestate) {
           writestate.flushing = false;
@@ -1603,7 +1678,7 @@ boolean shouldFlush() {
    * @throws DroppedSnapshotException Thrown when replay of hlog is required
    * because a Snapshot was not properly persisted.
    */
-  protected boolean internalFlushcache(MonitoredTask status)
+  protected FlushResult internalFlushcache(MonitoredTask status)
       throws IOException {
     return internalFlushcache(this.log, -1, status);
   }
@@ -1617,7 +1692,7 @@ protected boolean internalFlushcache(MonitoredTask status)
    * @throws IOException
    * @see #internalFlushcache(MonitoredTask)
    */
-  protected boolean internalFlushcache(
+  protected FlushResult internalFlushcache(
       final HLog wal, final long myseqid, MonitoredTask status)
   throws IOException {
     if (this.rsServices != null && this.rsServices.isAborted()) {
@@ -1630,7 +1705,7 @@ protected boolean internalFlushcache(
     this.lastFlushTime = startTime;
     // If nothing to flush, return and avoid logging start/stop flush.
     if (this.memstoreSize.get() <= 0) {
-      return false;
+      return new FlushResult(FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, "Nothing to flush");
     }
     if (LOG.isDebugEnabled()) {
       LOG.debug("Started memstore flush for " + this +
@@ -1777,7 +1852,8 @@ protected boolean internalFlushcache(
     status.setStatus(msg);
     this.recentFlushes.add(new Pair<Long,Long>(time/1000, totalFlushableSize));
 
-    return compactionRequested;
+    return new FlushResult(compactionRequested ? FlushResult.Result.FLUSHED_COMPACTION_NEEDED :
+        FlushResult.Result.FLUSHED_NO_COMPACTION_NEEDED, sequenceId);
   }
 
    /**
@@ -3788,6 +3864,22 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
         return false;
       }
 
+      long seqId = -1;
+      // We need to assign a sequential ID that's in between two memstores in order to preserve
+      // the guarantee that all the edits lower than the highest sequential ID from all the
+      // HFiles are flushed on disk. See HBASE-10958.
+      if (assignSeqId) {
+        FlushResult fs = this.flushcache();
+        if (fs.isFlushSucceeded()) {
+          seqId = fs.flushSequenceId;
+        } else if (fs.result == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {
+          seqId = this.log.obtainSeqNum();
+        } else {
+          throw new IOException("Could not bulk load with an assigned sequential ID because the " +
+              "flush didn't run. Reason for not flushing: " + fs.failureReason);
+        }
+      }
+
       for (Pair<byte[], String> p : familyPaths) {
         byte[] familyName = p.getFirst();
         String path = p.getSecond();
@@ -3797,7 +3889,7 @@ public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
           if(bulkLoadListener != null) {
             finalPath = bulkLoadListener.prepareBulkLoad(familyName, path);
           }
-          store.bulkLoadHFile(finalPath, assignSeqId ? this.log.obtainSeqNum() : -1);
+          store.bulkLoadHFile(finalPath, seqId);
           if(bulkLoadListener != null) {
             bulkLoadListener.doneBulkLoad(familyName, path);
           }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 00636f404ae6..b1c338758253 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2470,7 +2470,7 @@ public void flushRegion(byte[] regionName)
       throw new IllegalArgumentException("No region : " + new String(regionName)
       + " available");
     }
-    boolean needsCompaction = region.flushcache();
+    boolean needsCompaction = region.flushcache().isCompactionNeeded();
     if (needsCompaction) {
       this.compactSplitThread.requestCompaction(region, "Compaction through user triggered flush");
     }
@@ -2487,7 +2487,7 @@ public void flushRegion(byte[] regionName, long ifOlderThanTS)
        + " available");
      }
      if (region.getLastFlushTime() < ifOlderThanTS) {
-      boolean needsCompaction = region.flushcache();
+      boolean needsCompaction = region.flushcache().isCompactionNeeded();
       if (needsCompaction) {
         this.compactSplitThread
             .requestCompaction(region, "Compaction through user triggered flush");
@@ -3335,7 +3335,7 @@ public void flushRegion(HRegionInfo regionInfo)
     checkOpen();
     LOG.info("Flushing " + regionInfo.getRegionNameAsString());
     HRegion region = getRegion(regionInfo.getRegionName());
-    boolean needsCompaction = region.flushcache();
+    boolean needsCompaction = region.flushcache().isCompactionNeeded();
     if (needsCompaction) {
       this.compactSplitThread.requestCompaction(region, "Compaction through user triggered flush");
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index 7f704d8916c1..04f89421b1c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -417,7 +417,7 @@ private boolean flushRegion(final HRegion region, final boolean emergencyFlush)
      lock.lock();
     }
     try {
-      boolean shouldCompact = region.flushcache();
+      boolean shouldCompact = region.flushcache().isCompactionNeeded();
       // We just want to check the size
       boolean shouldSplit = region.checkSplit() != null;
       if (shouldSplit) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 111a7001aae3..915777fae784 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -341,8 +341,8 @@ public HColumnDescriptor getFamily() {
   /**
    * @return The maximum sequence id in all store files.
    */
-  long getMaxSequenceId(boolean includeBulkFiles) {
-    return StoreFile.getMaxSequenceIdInList(this.getStorefiles(), includeBulkFiles);
+  long getMaxSequenceId() {
+    return StoreFile.getMaxSequenceIdInList(this.getStorefiles());
   }
 
   /**
@@ -1145,7 +1145,7 @@ StoreFile compact(CompactionRequest cr) throws IOException {
     }
 
     // Max-sequenceID is the last key in the files we're compacting
-    long maxId = StoreFile.getMaxSequenceIdInList(filesToCompact, true);
+    long maxId = StoreFile.getMaxSequenceIdInList(filesToCompact);
 
     // Ready to go. Have list of files to compact.
     LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
@@ -1213,7 +1213,7 @@ public void compactRecentForTesting(int N) throws IOException {
         }
 
         filesToCompact = filesToCompact.subList(count - N, count);
-        maxId = StoreFile.getMaxSequenceIdInList(filesToCompact, true);
+        maxId = StoreFile.getMaxSequenceIdInList(filesToCompact);
         isMajor = (filesToCompact.size() == storefiles.size());
         filesCompacting.addAll(filesToCompact);
         Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 540ee1553d9a..233f843ac364 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -439,13 +439,10 @@ public static long getMaxMemstoreTSInList(Collection<StoreFile> sfs) {
    * @return 0 if no non-bulk-load files are provided or, this is Store that
    * does not yet have any store files.
    */
-  public static long getMaxSequenceIdInList(Collection<StoreFile> sfs, 
-      boolean includeBulkLoadedFiles) {
+  public static long getMaxSequenceIdInList(Collection<StoreFile> sfs) {
     long max = 0;
     for (StoreFile sf : sfs) {
-      if (includeBulkLoadedFiles || !sf.isBulkLoadResult()) {
-        max = Math.max(max, sf.getMaxSequenceId());
-      }
+      max = Math.max(max, sf.getMaxSequenceId());
     }
     return max;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index b6506b66aef0..1e390baa9f11 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -33,19 +33,18 @@
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.UserProvider;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HFileTestUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -67,10 +66,6 @@ public class TestLoadIncrementalHFiles {
     Bytes.toBytes("ppp")
   };
 
-  public static int BLOCKSIZE = 64*1024;
-  public static String COMPRESSION =
-    Compression.Algorithm.NONE.getName();
-
   static HBaseTestingUtility util = new HBaseTestingUtility();
 
   @BeforeClass
@@ -144,7 +139,7 @@ private void runTest(String testName, BloomType bloomType,
     for (byte[][] range : hfileRanges) {
       byte[] from = range[0];
       byte[] to = range[1];
-      createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
+      HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
           + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
     }
     int expectedRows = hfileIdx * 1000;
@@ -180,8 +175,8 @@ private void runTest(String testName, BloomType bloomType,
     for (byte[][] range : hfileRanges) {
       byte[] from = range[0];
       byte[] to = range[1];
-      createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + hfileIdx++), FAMILY,
-        QUALIFIER, from, to, 1000);
+      HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir,
+          "hfile_" + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
     }
 
     final byte[] TABLE = Bytes.toBytes("mytable_" + testName);
@@ -236,7 +231,7 @@ public void testNonexistentColumnFamilyLoad() throws Exception {
     for (byte[][] range : hfileRanges) {
       byte[] from = range[0];
       byte[] to = range[1];
-      createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
+      HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
           + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
     }
 
@@ -268,7 +263,7 @@ public void testSplitStoreFile() throws IOException {
     FileSystem fs = util.getTestFileSystem();
     Path testIn = new Path(dir, "testhfile");
     HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
-    createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
+    HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
         Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
 
     Path bottomOut = new Path(dir, "bottom.out");
@@ -301,36 +296,6 @@ private int verifyHFile(Path p) throws IOException {
     return count;
   }
 
-
-  /**
-   * Create an HFile with the given number of rows between a given
-   * start key and end key.
-   * TODO put me in an HFileTestUtil or something?
-   */
-  static void createHFile(
-      Configuration conf,
-      FileSystem fs, Path path,
-      byte[] family, byte[] qualifier,
-      byte[] startKey, byte[] endKey, int numRows) throws IOException
-  {
-    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
-        .withPath(fs, path)
-        .withBlockSize(BLOCKSIZE)
-        .withCompression(COMPRESSION)
-        .withComparator(KeyValue.KEY_COMPARATOR)
-        .create();
-    long now = System.currentTimeMillis();
-    try {
-      // subtract 2 since iterateOnSplits doesn't include boundary keys
-      for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows-2)) {
-        KeyValue kv = new KeyValue(key, family, qualifier, now, key);
-        writer.append(kv);
-      }
-    } finally {
-      writer.close();
-    }
-  }
-
   private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) {
     Integer value = map.containsKey(first)?(Integer)map.get(first):0;
     map.put(first, value+1);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 1660f9f03f10..4971a86852cc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -594,7 +594,7 @@ public void testCompactionWithCorruptResult() throws Exception {
     Store store = r.getStore(COLUMN_FAMILY);
 
     List<StoreFile> storeFiles = store.getStorefiles();
-    long maxId = StoreFile.getMaxSequenceIdInList(storeFiles, true);
+    long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
     Compactor tool = new Compactor(this.conf);
 
     StoreFile.Writer compactedFile = tool.compactForTesting(store, this.conf, storeFiles, false,
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 55c2d026acb4..c459180bd3d0 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3540,7 +3540,7 @@ public void testWritesWhileGetting()
     	private int flushesSinceCompact = 0;
     	private final int maxFlushesSinceCompact = 20;
         public void doAnAction() throws Exception {
-          if (region.flushcache()) {
+          if (region.flushcache().isCompactionNeeded()) {
             ++flushesSinceCompact;
           }
           // Compact regularly to avoid creating too many files and exceeding the ulimit.
@@ -4263,6 +4263,42 @@ private void assertScan(final HRegion r, final byte [] fs,
     }
   }
 
+  /**
+   * Test that we get the expected flush results back
+   * @throws IOException
+   */
+  @Test
+  public void testFlushResult() throws IOException {
+    String method = "testFlushResult";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Bytes.toBytes("family");
+
+    this.region = initHRegion(tableName, method, conf, family);
+
+    // empty memstore, flush doesn't run
+    HRegion.FlushResult fr = region.flushcache();
+    assertFalse(fr.isFlushSucceeded());
+    assertFalse(fr.isCompactionNeeded());
+
+    // Flush enough files to get up to the threshold, doesn't need compactions
+    for (int i = 0; i < 3; i++) {
+      Put put = new Put(tableName).add(family, family, tableName);
+      region.put(put);
+      fr = region.flushcache();
+      assertTrue(fr.isFlushSucceeded());
+      assertFalse(fr.isCompactionNeeded());
+    }
+
+    // Two flushes after the threshold, compactions are needed
+    for (int i = 0; i < 2; i++) {
+      Put put = new Put(tableName).add(family, family, tableName);
+      region.put(put);
+      fr = region.flushcache();
+      assertTrue(fr.isFlushSucceeded());
+      assertTrue(fr.isCompactionNeeded());
+    }
+  }
+
   private Configuration initSplit() {
     Configuration conf = HBaseConfiguration.create(this.conf);
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 030afa538dbf..150c31a168e2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -57,7 +57,6 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.regionserver.FlushRequester;
@@ -71,6 +70,7 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdge;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.HFileTestUtil;
 import org.apache.hadoop.hbase.util.Pair;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -306,7 +306,7 @@ public void test2727() throws Exception {
   public void testRegionMadeOfBulkLoadedFilesOnly()
   throws IOException, SecurityException, IllegalArgumentException,
       NoSuchFieldException, IllegalAccessException, InterruptedException {
-    final String tableNameStr = "testReplayEditsWrittenViaHRegion";
+    final String tableNameStr = "testRegionMadeOfBulkLoadedFilesOnly";
     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
     final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
     deleteDir(basedir);
@@ -317,19 +317,22 @@ public void testRegionMadeOfBulkLoadedFilesOnly()
     region2.getLog().closeAndDelete();
     HLog wal = createWAL(this.conf);
     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
-    Path f =  new Path(basedir, "hfile");
-    HFile.Writer writer =
-      HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create();
+
     byte [] family = htd.getFamilies().iterator().next().getName();
-    byte [] row = Bytes.toBytes(tableNameStr);
-    writer.append(new KeyValue(row, family, family, row));
-    writer.close();
+    Path f =  new Path(basedir, "hfile");
+    HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
+        Bytes.toBytes("z"), 10);
     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
     hfs.add(Pair.newPair(family, f.toString()));
     region.bulkLoadHFiles(hfs, true);
+
     // Add an edit so something in the WAL
+    byte [] row = Bytes.toBytes(tableNameStr);
     region.put((new Put(row)).add(family, family, family));
     wal.sync();
+    final int rowsInsertedCount = 11;
+
+    assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
 
     // Now 'crash' the region by stealing its wal
     final Configuration newConf = HBaseConfiguration.create(this.conf);
@@ -343,6 +346,77 @@ public Object run() throws Exception {
           newConf, hri, htd, null);
         long seqid2 = region2.initialize();
         assertTrue(seqid2 > -1);
+        assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
+
+        // I can't close wal1.  Its been appropriated when we split.
+        region2.close();
+        wal2.closeAndDelete();
+        return null;
+      }
+    });
+  }
+
+  /**
+   * HRegion test case that is made of a major compacted HFile (created with three bulk loaded
+   * files) and an edit in the memstore.
+   * This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries
+   * from being replayed"
+   * @throws IOException
+   * @throws IllegalAccessException
+   * @throws NoSuchFieldException
+   * @throws IllegalArgumentException
+   * @throws SecurityException
+   */
+  @Test
+  public void testCompactedBulkLoadedFiles()
+      throws IOException, SecurityException, IllegalArgumentException,
+      NoSuchFieldException, IllegalAccessException, InterruptedException {
+    final String tableNameStr = "testCompactedBulkLoadedFiles";
+    final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
+    final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
+    deleteDir(basedir);
+    final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
+    HRegion region2 = HRegion.createHRegion(hri,
+        hbaseRootDir, this.conf, htd);
+    region2.close();
+    region2.getLog().closeAndDelete();
+    HLog wal = createWAL(this.conf);
+    HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
+
+    // Add an edit so something in the WAL
+    byte [] row = Bytes.toBytes(tableNameStr);
+    byte [] family = htd.getFamilies().iterator().next().getName();
+    region.put((new Put(row)).add(family, family, family));
+    wal.sync();
+
+    List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
+    for (int i = 0; i < 3; i++) {
+      Path f = new Path(basedir, "hfile"+i);
+      HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
+          Bytes.toBytes(i + "50"), 10);
+      hfs.add(Pair.newPair(family, f.toString()));
+    }
+    region.bulkLoadHFiles(hfs, true);
+    final int rowsInsertedCount = 31;
+    assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
+
+    // major compact to turn all the bulk loaded files into one normal file
+    region.compactStores(true);
+    assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
+
+    // Now 'crash' the region by stealing its wal
+    final Configuration newConf = HBaseConfiguration.create(this.conf);
+    User user = HBaseTestingUtility.getDifferentUser(newConf,
+        tableNameStr);
+    user.runAs(new PrivilegedExceptionAction() {
+      public Object run() throws Exception {
+        runWALSplit(newConf);
+        HLog wal2 = createWAL(newConf);
+        HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
+            newConf, hri, htd, null);
+        long seqid2 = region2.initialize();
+        assertTrue(seqid2 > -1);
+        assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
 
         // I can't close wal1.  Its been appropriated when we split.
         region2.close();
@@ -736,14 +810,14 @@ public Object run() throws Exception {
         try {
           final HRegion region =
               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
-            protected boolean internalFlushcache(
+            protected FlushResult internalFlushcache(
                 final HLog wal, final long myseqid, MonitoredTask status)
             throws IOException {
               LOG.info("InternalFlushCache Invoked");
-              boolean b = super.internalFlushcache(wal, myseqid,
+              FlushResult fs = super.internalFlushcache(wal, myseqid,
                   Mockito.mock(MonitoredTask.class));
               flushcount.incrementAndGet();
-              return b;
+              return fs;
             };
           };
           long seqid = region.initialize();
diff --git a/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java b/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java
new file mode 100644
index 000000000000..6bc2866660d9
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java
@@ -0,0 +1,63 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+
+import java.io.IOException;
+
+/**
+ * Utility class for HFile-related testing.
+ */
+public class HFileTestUtil {
+
+  /**
+   * Create an HFile with the given number of rows between a given
+   * start key and end key.
+   */
+  public static void createHFile(
+      Configuration configuration,
+      FileSystem fs, Path path,
+      byte[] family, byte[] qualifier,
+      byte[] startKey, byte[] endKey, int numRows) throws IOException
+  {
+    HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration))
+        .withPath(fs, path)
+        .withComparator(KeyValue.KEY_COMPARATOR)
+        .create();
+    long now = System.currentTimeMillis();
+    try {
+      // subtract 2 since iterateOnSplits doesn't include boundary keys
+      for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows-2)) {
+        KeyValue kv = new KeyValue(key, family, qualifier, now, key);
+        writer.append(kv);
+      }
+    } finally {
+      writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
+          Bytes.toBytes(System.currentTimeMillis()));
+      writer.close();
+    }
+  }
+}

From 3c3594832456e0cfbd2911596facac623724d8a9 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 8 May 2014 17:24:33 +0000
Subject: [PATCH 1408/1540] HBASE-11119 Update ExportSnapShot to optionally not
 use a tmp file on external file system (Ted Malaska)

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1593340 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 43 ++++++++++++-------
 .../hbase/snapshot/TestExportSnapshot.java    |  6 +++
 2 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index b3ec3f28d59c..0fcc48cb2f4e 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -88,6 +88,7 @@ public final class ExportSnapshot extends Configured implements Tool {
   private static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
   private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
   private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
+  protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
 
   static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
   static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
@@ -691,9 +692,12 @@ public int run(String[] args) throws IOException {
     FileSystem inputFs = FileSystem.get(conf);
     FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
 
+    boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
+
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
     Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
     Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+    Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
 
     // Check if the snapshot already exists
     if (outputFs.exists(outputSnapshotDir)) {
@@ -710,17 +714,20 @@ public int run(String[] args) throws IOException {
     }
 
     // Check if the snapshot already in-progress
-    if (outputFs.exists(snapshotTmpDir)) {
-      if (overwrite) {
-        if (!outputFs.delete(snapshotTmpDir, true)) {
-          System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
+    if (!skipTmp) {
+      // Check if the snapshot already in-progress
+      if (outputFs.exists(snapshotTmpDir)) {
+        if (overwrite) {
+          if (!outputFs.delete(snapshotTmpDir, true)) {
+            System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
+            return 1;
+          }
+        } else {
+          System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
+          System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
+          System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
           return 1;
         }
-      } else {
-        System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
-        System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
-        System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
-        return 1;
       }
     }
 
@@ -735,10 +742,11 @@ public int run(String[] args) throws IOException {
     // The snapshot references must be copied before the hfiles otherwise the cleaner
     // will remove them because they are unreferenced.
     try {
-      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+      LOG.info("Copy Snapshot Manifest");
+      FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
     } catch (IOException e) {
       throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
-        snapshotDir + " to=" + snapshotTmpDir);
+        snapshotDir + " to=" + initialOutputSnapshotDir, e);
     }
 
     // Step 2 - Start MR Job to copy files
@@ -753,14 +761,19 @@ public int run(String[] args) throws IOException {
       }
 
       // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
-      if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
-        throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
-          snapshotTmpDir + " to=" + outputSnapshotDir);
+      if (!skipTmp) {
+        // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
+        if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
+          throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
+            snapshotTmpDir + " to=" + outputSnapshotDir);
+        }
       }
       return 0;
     } catch (Exception e) {
       LOG.error("Snapshot export failed", e);
-      outputFs.delete(snapshotTmpDir, true);
+      if (!skipTmp) {
+        outputFs.delete(snapshotTmpDir, true);
+      }
       outputFs.delete(outputSnapshotDir, true);
       return 1;
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index ed777fed3f27..4d3a75d43845 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -178,6 +178,12 @@ public void testExportFileSystemState() throws Exception {
     testExportFileSystemState(tableName, snapshotName, 2);
   }
 
+  @Test
+  public void testExportFileSystemStateWithSkipTmp() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
+    testExportFileSystemState(tableName, snapshotName, 2);
+  }
+
   @Test
   public void testEmptyExportFileSystemState() throws Exception {
     testExportFileSystemState(tableName, emptySnapshotName, 1);

From d2d0ad01e051d2ec937addf2cec6303ac07c9d66 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Sun, 11 May 2014 11:40:09 +0000
Subject: [PATCH 1409/1540] HBASE-11128 Add -target option to ExportSnapshot to
 export with a different name

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1593778 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/ExportSnapshot.java | 29 +++++++--
 .../hbase/snapshot/TestExportSnapshot.java    | 59 ++++++++++++++-----
 2 files changed, 69 insertions(+), 19 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 0fcc48cb2f4e..2cd8011f4ce7 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -357,7 +357,7 @@ private void copyData(final Context context,
 
     /**
      * Try to open the "source" file.
-     * Throws an IOException if the communication with the inputFs fail or 
+     * Throws an IOException if the communication with the inputFs fail or
      * if the file is not found.
      */
     private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException {
@@ -638,6 +638,7 @@ private void runCopyJob(final Path inputRoot, final Path outputRoot,
   public int run(String[] args) throws IOException {
     boolean verifyChecksum = true;
     String snapshotName = null;
+    String targetName = null;
     boolean overwrite = false;
     String filesGroup = null;
     String filesUser = null;
@@ -651,6 +652,8 @@ public int run(String[] args) throws IOException {
       try {
         if (cmd.equals("-snapshot")) {
           snapshotName = args[++i];
+        } else if (cmd.equals("-target")) {
+          targetName = args[++i];
         } else if (cmd.equals("-copy-to")) {
           outputRoot = new Path(args[++i]);
         } else if (cmd.equals("-no-checksum-verify")) {
@@ -687,6 +690,10 @@ public int run(String[] args) throws IOException {
       printUsageAndExit();
     }
 
+    if (targetName == null) {
+      targetName = snapshotName;
+    }
+
     Configuration conf = getConf();
     Path inputRoot = FSUtils.getRootDir(conf);
     FileSystem inputFs = FileSystem.get(conf);
@@ -695,8 +702,8 @@ public int run(String[] args) throws IOException {
     boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
 
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
-    Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
-    Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+    Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
+    Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
     Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
 
     // Check if the snapshot already exists
@@ -707,7 +714,7 @@ public int run(String[] args) throws IOException {
           return 1;
         }
       } else {
-        System.err.println("The snapshot '" + snapshotName +
+        System.err.println("The snapshot '" + targetName +
           "' already exists in the destination: " + outputSnapshotDir);
         return 1;
       }
@@ -723,7 +730,7 @@ public int run(String[] args) throws IOException {
             return 1;
           }
         } else {
-          System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
+          System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");
           System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
           System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
           return 1;
@@ -749,6 +756,16 @@ public int run(String[] args) throws IOException {
         snapshotDir + " to=" + initialOutputSnapshotDir, e);
     }
 
+    // Write a new .snapshotinfo if the target name is different from the source name
+    if (!targetName.equals(snapshotName)) {
+      SnapshotDescription snapshotDesc =
+        SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)
+          .toBuilder()
+          .setName(targetName)
+          .build();
+      SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, snapshotTmpDir, outputFs);
+    }
+
     // Step 2 - Start MR Job to copy files
     // The snapshot references must be copied before the files otherwise the files gets removed
     // by the HFileArchiver, since they have no references.
@@ -768,6 +785,8 @@ public int run(String[] args) throws IOException {
             snapshotTmpDir + " to=" + outputSnapshotDir);
         }
       }
+
+      LOG.info("Export Completed: " + targetName);
       return 0;
     } catch (Exception e) {
       LOG.error("Snapshot export failed", e);
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index 4d3a75d43845..111833a44f57 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -118,7 +118,7 @@ public void setUp() throws Exception {
 
     // Add some rows
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 1000, FAMILY);
+    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
 
     // take a snapshot
     admin.snapshot(snapshotName, tableName);
@@ -175,28 +175,34 @@ public void testBalanceSplit() throws Exception {
    */
   @Test
   public void testExportFileSystemState() throws Exception {
-    testExportFileSystemState(tableName, snapshotName, 2);
+    testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
   }
 
   @Test
   public void testExportFileSystemStateWithSkipTmp() throws Exception {
     TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
-    testExportFileSystemState(tableName, snapshotName, 2);
+    testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
   }
 
   @Test
   public void testEmptyExportFileSystemState() throws Exception {
-    testExportFileSystemState(tableName, emptySnapshotName, 1);
+    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 1);
   }
 
   @Test
   public void testConsecutiveExports() throws Exception {
-    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
-    testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
-    testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
+    Path copyDir = getLocalDestinationDir();
+    testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, false);
+    testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, true);
     removeExportDir(copyDir);
   }
 
+  @Test
+  public void testExportWithTargetName() throws Exception {
+    final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
+    testExportFileSystemState(tableName, snapshotName, targetName, 2);
+  }
+
   /**
    * Mock a snapshot with files in the archive dir,
    * two regions, and one reference file.
@@ -244,13 +250,14 @@ public void testSnapshotWithRefsExportFileSystemState() throws Exception {
     FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
     SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
 
-    testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
+    byte[] name = Bytes.toBytes(snapshotName);
+    testExportFileSystemState(tableWithRefsName, name, name, 2);
   }
 
   private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
-      int filesExpected) throws Exception {
-    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
-    testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
+      final byte[] targetName, int filesExpected) throws Exception {
+    Path copyDir = getHdfsDestinationDir();
+    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false);
     removeExportDir(copyDir);
   }
 
@@ -258,7 +265,8 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
    * Test ExportSnapshot
    */
   private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
-      int filesExpected, Path copyDir, boolean overwrite) throws Exception {
+      final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite)
+      throws Exception {
     URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
     FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
     copyDir = copyDir.makeQualified(fs);
@@ -268,6 +276,10 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
     opts.add(Bytes.toString(snapshotName));
     opts.add("-copy-to");
     opts.add(copyDir.toString());
+    if (targetName != snapshotName) {
+      opts.add("-target");
+      opts.add(Bytes.toString(targetName));
+    }
     if (overwrite) opts.add("-overwrite");
 
     // Export Snapshot
@@ -287,9 +299,10 @@ private void testExportFileSystemState(final byte[] tableName, final byte[] snap
     // compare the snapshot metadata and verify the hfiles
     final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
     final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
+    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
     verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
-        fs, new Path(copyDir, snapshotDir));
-    verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
+        fs, new Path(copyDir, targetDir));
+    verifyArchive(fs, copyDir, tableName, Bytes.toString(targetName));
     FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
   }
 
@@ -365,6 +378,11 @@ private void verifyNonEmptyFile(final Path path) throws IOException {
           assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
         }
     });
+
+    // Verify Snapshot description
+    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
+    assertTrue(desc.getName().equals(snapshotName));
+    assertTrue(desc.getTable().equals(Bytes.toString(tableName)));
   }
 
   private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
@@ -385,6 +403,19 @@ private Set<String> listFiles(final FileSystem fs, final Path root, final Path d
     return files;
   }
 
+  private Path getHdfsDestinationDir() {
+    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
+    LOG.info("HDFS export destination path: " + path);
+    return path;
+  }
+
+  private Path getLocalDestinationDir() {
+    Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis());
+    LOG.info("Local export destination path: " + path);
+    return path;
+  }
+
   private void removeExportDir(final Path path) throws IOException {
     FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
     FSUtils.logFileSystemState(fs, path, LOG);

From 4416d0b85ec6808f328cc9f2283ed5ced291a174 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Tue, 13 May 2014 06:15:47 +0000
Subject: [PATCH 1410/1540] HBASE-10936 Add zeroByte encoding test.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1594139 13f79535-47bb-0310-9956-ffa450edef68
---
 .../io/encoding/TestDataBlockEncoders.java    | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
index f7b73977518e..607a19246f39 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
@@ -120,6 +120,26 @@ public void testNegativeTimestamps() throws IOException {
             includesMemstoreTS));
   }
 
+  /**
+   * Test KeyValues with negative timestamp.
+   * @throws IOException On test failure.
+   */
+  @Test
+  public void testZeroByte() throws IOException {
+    List<KeyValue> kvList = new ArrayList<KeyValue>();
+    byte[] row = Bytes.toBytes("abcd");
+    byte[] family = new byte[] { 'f' };
+    byte[] qualifier0 = new byte[] { 'b' };
+    byte[] qualifier1 = new byte[] { 'c' };
+    byte[] value0 = new byte[] { 'd' };
+    byte[] value1 = new byte[] { 0x00 };
+    kvList.add(new KeyValue(row, family, qualifier0, 0, Type.Put, value0));
+    kvList.add(new KeyValue(row, family, qualifier1, 0, Type.Put, value1));
+    testEncodersOnDataset(
+        RedundantKVGenerator.convertKvToByteBuffer(kvList,
+            includesMemstoreTS));
+  }
+
   /**
    * Test whether compression -> decompression gives the consistent results on
    * pseudorandom sample.

From 164c114f938a518f485a25f2c0cbf51b7a718c91 Mon Sep 17 00:00:00 2001
From: larsh <larsh@unknown>
Date: Wed, 14 May 2014 06:17:32 +0000
Subject: [PATCH 1411/1540] HBASE-11143 Improve replication metrics.

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1594471 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/ReplicationSource.java            | 14 +++++++++-----
 .../regionserver/ReplicationSourceMetrics.java     |  5 +++++
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 6d6ea7ba895d..2916ba782126 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -426,6 +426,11 @@ public void run() {
               this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
           this.lastLoggedPosition = this.repLogReader.getPosition();
         }
+        if (!gotIOE) {
+          // if there was nothing to ship and it's not an error
+          // set "ageOfLastShippedOp" to <now> to indicate that we're current
+          this.metrics.setAgeOfLastShippedOp(System.currentTimeMillis());
+        }
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
           sleepMultiplier++;
         }
@@ -740,7 +745,7 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo, List<HLog.Entry> en
       }
       try {
         HRegionInterface rrs = getRS();
-        LOG.debug("Replicating " + entries.size());
+        LOG.debug("Replicating " + entries.size() + ", " + this.currentSize + " bytes");
         // can't avoid the copy here, the replicateLogEntries RPC require an HLog.Entry[]
         rrs.replicateLogEntries(entries.toArray(new HLog.Entry[entries.size()]));
         if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
@@ -750,10 +755,9 @@ protected void shipEdits(boolean currentWALisBeingWrittenTo, List<HLog.Entry> en
         }
         this.totalReplicatedEdits += entries.size();
         this.metrics.shippedBatchesRate.inc(1);
-        this.metrics.shippedOpsRate.inc(
-            this.currentNbOperations);
-        this.metrics.setAgeOfLastShippedOp(
-            entries.get(entries.size()-1).getKey().getWriteTime());
+        this.metrics.shippedKBRate.inc(this.currentSize/1024);
+        this.metrics.shippedOpsRate.inc(this.currentNbOperations);
+        this.metrics.setAgeOfLastShippedOp(entries.get(entries.size()-1).getKey().getWriteTime());
         LOG.debug("Replicated in total: " + this.totalReplicatedEdits);
         break;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
index da0905c5dfe7..eee59a0eeaea 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
@@ -44,6 +44,10 @@ public class ReplicationSourceMetrics implements Updater {
   public final MetricsRate shippedOpsRate =
       new MetricsRate("shippedOpsRate", registry);
 
+  /** Rate of shipped bytes (in KB) by the source */
+  public final MetricsRate shippedKBRate =
+      new MetricsRate("shippedBytesRate", registry);
+
   /** Rate of shipped batches by the source */
   public final MetricsRate shippedBatchesRate =
       new MetricsRate("shippedBatchesRate", registry);
@@ -115,6 +119,7 @@ public void doUpdates(MetricsContext metricsContext) {
       refreshAgeOfLastShippedOp();
       this.shippedOpsRate.pushMetric(this.metricsRecord);
       this.shippedBatchesRate.pushMetric(this.metricsRecord);
+      this.shippedKBRate.pushMetric(this.metricsRecord);
       this.logEditsReadRate.pushMetric(this.metricsRecord);
       this.logEditsFilteredRate.pushMetric(this.metricsRecord);
       this.ageOfLastShippedOp.pushMetric(this.metricsRecord);

From cf354484795454def7010c9e23f5ad9c66880f35 Mon Sep 17 00:00:00 2001
From: mbertozzi <mbertozzi@unknown>
Date: Thu, 15 May 2014 10:39:05 +0000
Subject: [PATCH 1412/1540] HBASE-11134 Add a -list-snapshots option to
 SnapshotInfo

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1594857 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/snapshot/SnapshotInfo.java   | 54 +++++++++++++++++--
 .../org/apache/hadoop/hbase/util/FSUtils.java |  5 ++
 2 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index 81e2bbf67a1a..b8ba593e7097 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -20,15 +20,19 @@
 
 import java.io.IOException;
 import java.io.FileNotFoundException;
+import java.net.URI;
 import java.text.SimpleDateFormat;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -252,6 +256,8 @@ FileInfo addLogFile(final String server, final String logfile) throws IOExceptio
 
   @Override
   public int run(String[] args) throws IOException, InterruptedException {
+    final Configuration conf = getConf();
+    boolean listSnapshots = false;
     String snapshotName = null;
     boolean showSchema = false;
     boolean showFiles = false;
@@ -270,6 +276,13 @@ public int run(String[] args) throws IOException, InterruptedException {
           showStats = true;
         } else if (cmd.equals("-schema")) {
           showSchema = true;
+        } else if (cmd.equals("-remote-dir")) {
+          Path sourceDir = new Path(args[++i]);
+          URI defaultFs = sourceDir.getFileSystem(conf).getUri();
+          FSUtils.setFsDefault(conf, new Path(defaultFs));
+          FSUtils.setRootDir(conf, sourceDir);
+        } else if (cmd.equals("-list-snapshots")) {
+          listSnapshots = true;
         } else if (cmd.equals("-h") || cmd.equals("--help")) {
           printUsageAndExit();
         } else {
@@ -281,15 +294,28 @@ public int run(String[] args) throws IOException, InterruptedException {
       }
     }
 
+    // List Available Snapshots
+    if (listSnapshots) {
+      SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+      System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
+      for (SnapshotDescription desc: getSnapshotList(conf)) {
+        System.out.printf("%-20s | %20s | %s%n",
+                          desc.getName(),
+                          df.format(new Date(desc.getCreationTime())),
+                          desc.getTable());
+      }
+      return 0;
+    }
+
     if (snapshotName == null) {
       System.err.println("Missing snapshot name!");
       printUsageAndExit();
       return 1;
     }
 
-    Configuration conf = getConf();
-    fs = FileSystem.get(conf);
     rootDir = FSUtils.getRootDir(conf);
+    fs = FileSystem.get(rootDir.toUri(), conf);
+    LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
 
     // Load snapshot information
     if (!loadSnapshotInfo(snapshotName)) {
@@ -422,6 +448,8 @@ private void printUsageAndExit() {
     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
     System.err.println(" where [options] are:");
     System.err.println("  -h|-help                Show this help and exit.");
+    System.err.println("  -remote-dir             Root directory that contains the snapshots.");
+    System.err.println("  -list-snapshots         List all the available snapshots and exit.");
     System.err.println("  -snapshot NAME          Snapshot to examine.");
     System.err.println("  -files                  Files and logs list.");
     System.err.println("  -stats                  Files and logs stats.");
@@ -442,7 +470,7 @@ private void printUsageAndExit() {
   public static SnapshotStats getSnapshotStats(final Configuration conf,
       final SnapshotDescription snapshot) throws IOException {
     Path rootDir = FSUtils.getRootDir(conf);
-    FileSystem fs = FileSystem.get(conf);
+    FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
@@ -463,6 +491,26 @@ public void logFile (final String server, final String logfile) throws IOExcepti
     return stats;
   }
 
+  /**
+   * Returns the list of available snapshots in the specified location
+   * @param conf the {@link Configuration} to use
+   * @return the list of snapshots
+   */
+  public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
+      throws IOException {
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    FileStatus[] snapshots = fs.listStatus(snapshotDir,
+      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+    List<SnapshotDescription> snapshotLists =
+      new ArrayList<SnapshotDescription>(snapshots.length);
+    for (FileStatus snapshotDirStat: snapshots) {
+      snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
+    }
+    return snapshotLists;
+  }
+
   /**
    * The guts of the {@link #main} method.
    * Call this method to avoid the {@link #main(String[])} System.exit.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 75f85b04954c..fe30f4c1b1d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -671,6 +671,11 @@ public static void setRootDir(final Configuration c, final Path root) throws IOE
     c.set(HConstants.HBASE_DIR, root.toString());
   }
 
+  public static void setFsDefault(final Configuration c, final Path root) throws IOException {
+    c.set("fs.defaultFS", root.toString());    // for hadoop 0.21+
+    c.set("fs.default.name", root.toString()); // for hadoop 0.20
+  }
+
   /**
    * Checks if root region exists
    *

From 7495b6a5ca110691e321f5175426e93c970ceb0d Mon Sep 17 00:00:00 2001
From: Jean-Daniel Cryans <jdcryans@apache.org>
Date: Fri, 16 May 2014 17:00:10 +0000
Subject: [PATCH 1413/1540] HBASE-11188 "Inconsistent configuration" for
 SchemaMetrics is always shown

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.94@1595266 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hadoop/hbase/regionserver/metrics/SchemaMetrics.java      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 844560f9c24b..991e71ad780c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -586,10 +586,10 @@ public void updateBloomMetrics(boolean isInBloom) {
   public static void configureGlobally(Configuration conf) {
     if (conf != null) {
       final boolean useTableNameNew =
-          conf.getBoolean(SHOW_TABLE_NAME_CONF_KEY, false);
+          conf.getBoolean(SHOW_TABLE_NAME_CONF_KEY, true);
       setUseTableName(useTableNameNew);
     } else {
-      setUseTableName(false);
+      setUseTableName(true);
     }
   }
 

From 71d197c15f17bc23c77d811459f7ea8dfb4a551a Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 22 May 2014 10:56:33 -0700
Subject: [PATCH 1414/1540] HBASE-11110 Ability to load FilterList class is
 dependent on context classloader. (Gabriel Reid)

---
 .../hadoop/hbase/filter/FilterList.java       | 22 ++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index b9189c3dc326..44baf394357f 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -59,7 +59,23 @@ public static enum Operator {
     MUST_PASS_ONE
   }
 
-  private static final Configuration conf = HBaseConfiguration.create();
+  private static final Configuration CONF;
+  static {
+    // We don't know which thread will load this class, so we don't know what
+    // the state of the context classloader will be when this class is loaded.
+    // HBaseConfiguration.create is dependent on the state of the context
+    // classloader of the current thread, so we set it to be the classloader
+    // that was used to load the Filter class to guarantee the consistent
+    // ability to load this class from any thread
+    ClassLoader saveCtxCl = Thread.currentThread().getContextClassLoader();
+    try {
+      Thread.currentThread().setContextClassLoader(
+          Filter.class.getClassLoader());
+      CONF = HBaseConfiguration.create();
+    } finally {
+      Thread.currentThread().setContextClassLoader(saveCtxCl);
+    }
+  }
   private static final int MAX_LOG_FILTERS = 5;
   private Operator operator = Operator.MUST_PASS_ALL;
   private List<Filter> filters = new ArrayList<Filter>();
@@ -321,7 +337,7 @@ public void readFields(final DataInput in) throws IOException {
     if (size > 0) {
       filters = new ArrayList<Filter>(size);
       for (int i = 0; i < size; i++) {
-        Filter filter = HbaseObjectWritable.readFilter(in, conf);
+        Filter filter = HbaseObjectWritable.readFilter(in, CONF);
         filters.add(filter);
       }
     }
@@ -331,7 +347,7 @@ public void write(final DataOutput out) throws IOException {
     out.writeByte(operator.ordinal());
     out.writeInt(filters.size());
     for (Filter filter : filters) {
-      HbaseObjectWritable.writeObject(out, filter, Writable.class, conf);
+      HbaseObjectWritable.writeObject(out, filter, Writable.class, CONF);
     }
   }
 

From c9c01e6eb12451bf3eb4a0044fd3f496c9c1974f Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 22 May 2014 11:44:51 -0700
Subject: [PATCH 1415/1540] HBASE-11225 Backport fix for HBASE-10417 index is
 not incremented in PutSortReducer#reduce() (Gustavo Anatoly)

---
 .../java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index e76df8c19e27..da621af28d16 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -70,7 +70,7 @@ protected void reduce(
       int index = 0;
       for (KeyValue kv : map) {
         context.write(row, kv);
-        if (index > 0 && index % 100 == 0)
+        if (++index % 100 == 0)
           context.setStatus("Wrote " + index);
       }
 

From ab9234e71d82bff1a13a921e91a9cfe42abda143 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 22 May 2014 11:52:18 -0700
Subject: [PATCH 1416/1540] HBASE-11212 Fix increment index in
 KeyValueSortReducer. (Gustavo Anatoly)

---
 .../org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
index 1f1567ed8643..abc356f85f11 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
@@ -44,7 +44,7 @@ protected void reduce(ImmutableBytesWritable row, java.lang.Iterable<KeyValue> k
     int index = 0;
     for (KeyValue kv: map) {
       context.write(row, kv);
-      if (index > 0 && index % 100 == 0) context.setStatus("Wrote " + index);
+      if (++index % 100 == 0) context.setStatus("Wrote " + index);
     }
   }
 }

From 9264a40cf894918a7a9c3b3d5a907f8feb7285a0 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <lhofhansl@yahoo.com>
Date: Thu, 22 May 2014 20:52:38 -0700
Subject: [PATCH 1417/1540] pom.xml, CHANGES.txt for 0.94.20RC0

---
 CHANGES.txt | 22 ++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 1941db77d432..51571e66b7c7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,26 @@
 HBase Change Log
+Release 0.94.20 - 05/22/2014
+Sub-task
+
+    [HBASE-10936] - Add zeroByte encoding test
+
+Bug
+
+    [HBASE-10958] - [dataloss] Bulk loading with seqids can prevent some log entries from being replayed
+    [HBASE-11110] - Ability to load FilterList class is dependent on context classloader
+    [HBASE-11143] - Improve replication metrics
+    [HBASE-11188] - "Inconsistent configuration" for SchemaMetrics is always shown
+    [HBASE-11212] - Fix increment index in KeyValueSortReducer
+    [HBASE-11225] - Backport fix for HBASE-10417 'index is not incremented in PutSortReducer#reduce()'
+
+Improvement
+
+    [HBASE-11008] - Align bulk load, flush, and compact to require Action.CREATE
+    [HBASE-11119] - Update ExportSnapShot to optionally not use a tmp file on external file system
+    [HBASE-11128] - Add -target option to ExportSnapshot to export with a different name
+    [HBASE-11134] - Add a -list-snapshots option to SnapshotInfo
+
+
 Release 0.94.19 - 04/21/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index 4fb6dfe6a636..5498b638086d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.20-SNAPSHOT</version>
+  <version>0.94.20</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From b2a2e060982634cfae4bec2665f2b63e1f680ff1 Mon Sep 17 00:00:00 2001
From: Chunhui Shen <zjusch@163.com>
Date: Fri, 23 May 2014 14:46:17 +0800
Subject: [PATCH 1418/1540] HBASE-11234 FastDiffDeltaEncoder#getFirstKeyInBlock
 returns wrong result

---
 .../apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java   | 2 +-
 .../apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index 0ca02c8d9a45..7959e0e3c9e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -389,7 +389,7 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
     ByteBufferUtils.readCompressedInt(block); // commonLength
     int pos = block.position();
     block.reset();
-    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
+    return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength).slice();
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
index ac63ead63142..3e4416fb6c0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -154,7 +154,7 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
     }
     int pos = block.position();
     block.reset();
-    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
+    return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength).slice();
   }
 
   @Override

From c8fb799b1010df9c340641acb2c90f752bba776a Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 22 May 2014 23:59:35 -0700
Subject: [PATCH 1419/1540] updated CHANGES.txt for 0.94.20RC0

---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 51571e66b7c7..ad0abf22749f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,6 +12,7 @@ Bug
     [HBASE-11188] - "Inconsistent configuration" for SchemaMetrics is always shown
     [HBASE-11212] - Fix increment index in KeyValueSortReducer
     [HBASE-11225] - Backport fix for HBASE-10417 'index is not incremented in PutSortReducer#reduce()'
+    [HBASE-11234] - FastDiffDeltaEncoder#getFirstKeyInBlock returns wrong result
 
 Improvement
 

From 5f4537f06c76044ee3aa6d9e28378c0534f33d51 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 23 May 2014 12:35:12 -0700
Subject: [PATCH 1420/1540] Revert "HBASE-11234
 FastDiffDeltaEncoder#getFirstKeyInBlock returns wrong result" due to test
 issues

This reverts commit b2a2e060982634cfae4bec2665f2b63e1f680ff1.
---
 .../apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java   | 2 +-
 .../apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index 7959e0e3c9e8..0ca02c8d9a45 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -389,7 +389,7 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
     ByteBufferUtils.readCompressedInt(block); // commonLength
     int pos = block.position();
     block.reset();
-    return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength).slice();
+    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
index 3e4416fb6c0b..ac63ead63142 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -154,7 +154,7 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
     }
     int pos = block.position();
     block.reset();
-    return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength).slice();
+    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
   }
 
   @Override

From e2eefd60eb66b907223f2316e56fe67d238d48d0 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 23 May 2014 12:35:46 -0700
Subject: [PATCH 1421/1540] Revert "updated CHANGES.txt for 0.94.20RC0", remove
 a jira again

This reverts commit c8fb799b1010df9c340641acb2c90f752bba776a.
---
 CHANGES.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index ad0abf22749f..51571e66b7c7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,7 +12,6 @@ Bug
     [HBASE-11188] - "Inconsistent configuration" for SchemaMetrics is always shown
     [HBASE-11212] - Fix increment index in KeyValueSortReducer
     [HBASE-11225] - Backport fix for HBASE-10417 'index is not incremented in PutSortReducer#reduce()'
-    [HBASE-11234] - FastDiffDeltaEncoder#getFirstKeyInBlock returns wrong result
 
 Improvement
 

From 35408b5d2a1ec8f50471d6899e366801b1cd2992 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 23 May 2014 21:50:41 -0700
Subject: [PATCH 1422/1540] HBASE-11247 [0.94] update maven-site-plugin to 3.3.

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 5498b638086d..7a5bdf42b54d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -383,7 +383,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-site-plugin</artifactId>
-          <version>3.2</version>
+          <version>3.3</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>

From 09c60d770f2869ca315910ba0f9a5ee9797b1edc Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 23 May 2014 21:55:43 -0700
Subject: [PATCH 1423/1540] Sigh... One more addition to CHANGES.txt for RC0

---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 51571e66b7c7..2b3f8b6ac160 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
 HBase Change Log
-Release 0.94.20 - 05/22/2014
+Release 0.94.20 - 05/23/2014
 Sub-task
 
     [HBASE-10936] - Add zeroByte encoding test
@@ -12,6 +12,7 @@ Bug
     [HBASE-11188] - "Inconsistent configuration" for SchemaMetrics is always shown
     [HBASE-11212] - Fix increment index in KeyValueSortReducer
     [HBASE-11225] - Backport fix for HBASE-10417 'index is not incremented in PutSortReducer#reduce()'
+    [HBASE-11247] - [0.94] update maven-site-plugin to 3.3
 
 Improvement
 

From a616bb42c651ed7ab79f25c55d8f6baaea585676 Mon Sep 17 00:00:00 2001
From: Ted Yu <tedyu@apache.org>
Date: Tue, 27 May 2014 14:43:59 +0000
Subject: [PATCH 1424/1540] HBASE-11234 FastDiffDeltaEncoder#getFirstKeyInBlock
 returns wrong result (Chunhui)

---
 .../apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java   | 2 +-
 .../apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index 0ca02c8d9a45..7959e0e3c9e8 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -389,7 +389,7 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
     ByteBufferUtils.readCompressedInt(block); // commonLength
     int pos = block.position();
     block.reset();
-    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
+    return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength).slice();
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
index ac63ead63142..3e4416fb6c0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -154,7 +154,7 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
     }
     int pos = block.position();
     block.reset();
-    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
+    return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength).slice();
   }
 
   @Override

From 6ed3d0770202a497e9884d0347efa46121dc2808 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Tue, 27 May 2014 09:51:08 -0700
Subject: [PATCH 1425/1540] HBASE-11096 stop method of Master and RegionServer
 coprocessor is not invoked (Qiang Tian)

---
 .../hbase/coprocessor/CoprocessorHost.java    |   3 +
 .../hbase/master/MasterCoprocessorHost.java   |   4 +
 .../hbase/regionserver/HRegionServer.java     |  20 +--
 .../RegionServerCoprocessorHost.java          |   2 +
 .../coprocessor/TestCoprocessorStop.java      | 125 ++++++++++++++++++
 5 files changed, 145 insertions(+), 9 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorStop.java

diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index c41d63c15514..24d7519cffd8 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -260,6 +260,9 @@ public abstract E createEnvironment(Class<?> implClass, Coprocessor instance,
 
   public void shutdown(CoprocessorEnvironment e) {
     if (e instanceof Environment) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Stop coprocessor " + e.getInstance().getClass().getName());
+      }
       ((Environment)e).shutdown();
     } else {
       LOG.warn("Shutdown called on unknown environment: "+
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 936eb5a4d6d2..bfc59831e66e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -595,6 +595,8 @@ void preShutdown() throws IOException {
           break;
         }
       }
+      // invoke coprocessor stop method
+      shutdown(env);
     }
   }
 
@@ -612,6 +614,8 @@ void preStopMaster() throws IOException {
           break;
         }
       }
+      // invoke coprocessor stop method
+      shutdown(env);
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b1c338758253..36fa0a3151ad 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1840,16 +1840,18 @@ public CatalogTracker getCatalogTracker() {
 
   @Override
   public void stop(final String msg) {
-    try {
-      if (this.rsHost != null) {
-        this.rsHost.preStop(msg);
+    if (!this.stopped) {
+      try {
+        if (this.rsHost != null) {
+          this.rsHost.preStop(msg);
+        }
+        this.stopped = true;
+        LOG.info("STOPPED: " + msg);
+        // Wakes run() if it is sleeping
+        sleeper.skipSleepCycle();
+      } catch (IOException exp) {
+        LOG.warn("The region server did not stop", exp);
       }
-      this.stopped = true;
-      LOG.info("STOPPED: " + msg);
-      // Wakes run() if it is sleeping
-      sleeper.skipSleepCycle();
-    } catch (IOException exp) {
-      LOG.warn("The region server did not stop", exp);
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
index 2302f9e9698b..f7dafad7de5a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -58,6 +58,8 @@ public void preStop(String message) throws IOException {
           break;
         }
       }
+      // invoke coprocessor stop method
+      shutdown(env);
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorStop.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorStop.java
new file mode 100644
index 000000000000..68009863828b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorStop.java
@@ -0,0 +1,125 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Tests for master and regionserver coprocessor stop method
+ *
+ */
+@Category(MediumTests.class)
+public class TestCoprocessorStop {
+  private static final Log LOG = LogFactory.getLog(TestCoprocessorStop.class);
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final String MASTER_FILE =
+                              "master" + System.currentTimeMillis();
+  private static final String REGIONSERVER_FILE =
+                              "regionserver" + System.currentTimeMillis();
+
+  public static class FooCoprocessor implements Coprocessor {
+    @Override
+    public void start(CoprocessorEnvironment env) throws IOException {
+      String where = null;
+
+      if (env instanceof MasterCoprocessorEnvironment) {
+        // if running on HMaster
+        where = "master";
+      } else if (env instanceof RegionServerCoprocessorEnvironment) {
+        where = "regionserver";
+      } else if (env instanceof RegionCoprocessorEnvironment) {
+        LOG.error("on RegionCoprocessorEnvironment!!");
+      }
+      LOG.info("start coprocessor on " + where);
+    }
+
+    @Override
+    public void stop(CoprocessorEnvironment env) throws IOException {
+      String fileName = null;
+
+      if (env instanceof MasterCoprocessorEnvironment) {
+        // if running on HMaster
+        fileName = MASTER_FILE;
+      } else if (env instanceof RegionServerCoprocessorEnvironment) {
+        fileName = REGIONSERVER_FILE;
+      } else if (env instanceof RegionCoprocessorEnvironment) {
+        LOG.error("on RegionCoprocessorEnvironment!!");
+      }
+
+      Configuration conf = UTIL.getConfiguration();
+      Path resultFile = new Path(UTIL.getDataTestDir(), fileName);
+      FileSystem fs = FileSystem.get(conf);
+
+      boolean result = fs.createNewFile(resultFile);
+      LOG.info("create file " + resultFile + " return rc " + result);
+    }
+  }
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+
+    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+      FooCoprocessor.class.getName());
+    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
+      FooCoprocessor.class.getName());
+
+    UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testStopped() throws Exception {
+    //shutdown hbase only. then check flag file.
+    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+    LOG.info("shutdown hbase cluster...");
+    cluster.shutdown();
+    LOG.info("wait for the hbase cluster shutdown...");
+    cluster.waitUntilShutDown();
+
+    Configuration conf = UTIL.getConfiguration();
+    FileSystem fs = FileSystem.get(conf);
+
+    Path resultFile = new Path(UTIL.getDataTestDir(), MASTER_FILE);
+    assertTrue("Master flag file should have been created",fs.exists(resultFile));
+
+    resultFile = new Path(UTIL.getDataTestDir(), REGIONSERVER_FILE);
+    assertTrue("RegionServer flag file should have been created",fs.exists(resultFile));
+  }
+}

From 0328fcd33f0e9d07c307f39db84ef4c6740cbd27 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Tue, 27 May 2014 10:25:33 -0700
Subject: [PATCH 1426/1540] HBASE-10692 The Multi TableMap job don't support
 the security HBase cluster (Liu Shaohui)

---
 .../org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 462bc90bbebd..a6b2eb05ba42 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -288,6 +288,7 @@ public static void initTableMapperJob(List<Scan> scans,
     if (addDependencyJars) {
       addDependencyJars(job);
     }
+    initCredentials(job);
   }
 
   /**

From c8c9249bad4ccf46817434ff2595089f2bd0c7b1 Mon Sep 17 00:00:00 2001
From: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Date: Mon, 2 Jun 2014 08:40:12 +0100
Subject: [PATCH 1427/1540] HBASE-10935 support snapshot policy where flush
 memstore can be skipped to prevent production cluster freeze (Tianying Chang)

---
 .../hadoop/hbase/client/HBaseAdmin.java       | 22 ++++++++
 .../hbase/protobuf/generated/HBaseProtos.java | 17 ++++---
 .../snapshot/FlushSnapshotSubprocedure.java   | 27 ++++++++--
 .../snapshot/RegionServerSnapshotManager.java | 12 +++++
 src/main/protobuf/hbase.proto                 |  1 +
 src/main/ruby/hbase.rb                        |  1 +
 src/main/ruby/hbase/admin.rb                  | 15 +++++-
 src/main/ruby/shell/commands/snapshot.rb      |  5 +-
 .../snapshot/TestFlushSnapshotFromClient.java | 51 +++++++++++++++++++
 9 files changed, 137 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 678e02a66755..030e6c6456ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2014,6 +2014,28 @@ public void snapshot(final String snapshotName, final String tableName) throws I
     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
   }
 
+  /**
+   * Create snapshot for the given table of given flush type.
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will fail with
+   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link HTableDescriptor#isLegalTableName(byte[])}.
+   * @param snapshotName name of the snapshot to be created
+   * @param tableName name of the table for which snapshot is created
+   * @param flushType if the snapshot should be taken without flush memstore first
+   * @throws IOException if a remote or network exception occurs
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+  public void snapshot(final byte[] snapshotName, final byte[] tableName,
+                       final SnapshotDescription.Type flushType) throws
+      IOException, SnapshotCreationException, IllegalArgumentException {
+      snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
+  }
+
   /**
    * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
    * taken. If the table is disabled, an offline snapshot is taken.
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
index e0ccb1c24a3b..e3068daed7b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
@@ -63,10 +63,12 @@ public enum Type
         implements com.google.protobuf.ProtocolMessageEnum {
       DISABLED(0, 0),
       FLUSH(1, 1),
+      SKIPFLUSH(2, 2),
       ;
       
       public static final int DISABLED_VALUE = 0;
       public static final int FLUSH_VALUE = 1;
+      public static final int SKIPFLUSH_VALUE = 2;
       
       
       public final int getNumber() { return value; }
@@ -75,6 +77,7 @@ public static Type valueOf(int value) {
         switch (value) {
           case 0: return DISABLED;
           case 1: return FLUSH;
+          case 2: return SKIPFLUSH;
           default: return null;
         }
       }
@@ -105,7 +108,7 @@ public Type findValueByNumber(int number) {
       }
       
       private static final Type[] VALUES = {
-        DISABLED, FLUSH, 
+        DISABLED, FLUSH, SKIPFLUSH, 
       };
       
       public static Type valueOf(
@@ -1193,14 +1196,14 @@ public Builder clearInfoPort() {
       descriptor;
   static {
     java.lang.String[] descriptorData = {
-      "\n\013hbase.proto\"\255\001\n\023SnapshotDescription\022\014\n" +
+      "\n\013hbase.proto\"\274\001\n\023SnapshotDescription\022\014\n" +
       "\004name\030\001 \002(\t\022\r\n\005table\030\002 \001(\t\022\027\n\014creationTi" +
       "me\030\003 \001(\003:\0010\022.\n\004type\030\004 \001(\0162\031.SnapshotDesc" +
-      "ription.Type:\005FLUSH\022\017\n\007version\030\005 \001(\005\"\037\n\004" +
-      "Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001\"$\n\020RegionS" +
-      "erverInfo\022\020\n\010infoPort\030\001 \001(\005B>\n*org.apach" +
-      "e.hadoop.hbase.protobuf.generatedB\013HBase" +
-      "ProtosH\001\240\001\001"
+      "ription.Type:\005FLUSH\022\017\n\007version\030\005 \001(\005\".\n\004" +
+      "Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001\022\r\n\tSKIPFLU" +
+      "SH\020\002\"$\n\020RegionServerInfo\022\020\n\010infoPort\030\001 \001" +
+      "(\005B>\n*org.apache.hadoop.hbase.protobuf.g" +
+      "eneratedB\013HBaseProtosH\001\240\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
index b5d194ce9483..9917ba955cd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
@@ -49,12 +49,18 @@ public class FlushSnapshotSubprocedure extends Subprocedure {
   private final SnapshotDescription snapshot;
   private final SnapshotSubprocedurePool taskManager;
 
+  private boolean snapshotSkipFlush = false;
+
   public FlushSnapshotSubprocedure(ProcedureMember member,
       ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout,
       List<HRegion> regions, SnapshotDescription snapshot,
       SnapshotSubprocedurePool taskManager) {
     super(member, snapshot.getName(), errorListener, wakeFrequency, timeout);
     this.snapshot = snapshot;
+    if (this.snapshot.getType() == SnapshotDescription.Type.SKIPFLUSH) {
+      snapshotSkipFlush = true;
+    }
+
     this.regions = regions;
     this.taskManager = taskManager;
   }
@@ -78,10 +84,25 @@ public Void call() throws Exception {
       LOG.debug("Starting region operation on " + region);
       region.startRegionOperation();
       try {
-        LOG.debug("Flush Snapshotting region " + region.toString() + " started...");
-        region.flushcache();
+        if (snapshotSkipFlush) {
+        /*
+         * This is to take an online-snapshot without force a coordinated flush to prevent pause
+         * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure
+         * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior can be
+         * turned on/off based on the flush type.
+         * To minimized the code change, class name is not changed.
+         */
+          LOG.debug("take snapshot without flush memstore first");
+        } else {
+          LOG.debug("Flush Snapshotting region " + region.toString() + " started...");
+          region.flushcache();
+        }
         region.addRegionToSnapshot(snapshot, monitor);
-        LOG.debug("... Flush Snapshotting region " + region.toString() + " completed.");
+        if (snapshotSkipFlush) {
+          LOG.debug("... SkipFlush Snapshotting region " + region.toString() + " completed.");
+        } else {
+          LOG.debug("... Flush Snapshotting region " + region.toString() + " completed.");
+        }
       } finally {
         LOG.debug("Closing region operation on " + region);
         region.closeRegionOperation();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
index 60e23dfbcfb5..4eab185df912 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -204,6 +204,18 @@ public Subprocedure buildSubprocedure(SnapshotDescription snapshot) {
         new SnapshotSubprocedurePool(rss.getServerName().toString(), conf);
       return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis,
           timeoutMillis, involvedRegions, snapshot, taskManager);
+      case SKIPFLUSH:
+        /*
+         * This is to take an online-snapshot without force a coordinated flush to prevent pause
+         * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure
+         * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior can be
+         * turned on/off based on the flush type.
+         * To minimized the code change, class name is not changed.
+         */
+        SnapshotSubprocedurePool taskManager2 =
+            new SnapshotSubprocedurePool(rss.getServerName().toString(), conf);
+        return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis,
+            timeoutMillis, involvedRegions, snapshot, taskManager2);
     default:
       throw new UnsupportedOperationException("Unrecognized snapshot type:" + snapshot.getType());
     }
diff --git a/src/main/protobuf/hbase.proto b/src/main/protobuf/hbase.proto
index c3c97ad6d7ea..0cdeaeb5c2ad 100644
--- a/src/main/protobuf/hbase.proto
+++ b/src/main/protobuf/hbase.proto
@@ -33,6 +33,7 @@ message SnapshotDescription {
   enum Type {
     DISABLED = 0;
     FLUSH = 1;
+    SKIPFLUSH = 2;
   }
   optional Type type = 4 [default = FLUSH];
   optional int32 version = 5;
diff --git a/src/main/ruby/hbase.rb b/src/main/ruby/hbase.rb
index 4d97cd09482f..fbcdc006aef8 100644
--- a/src/main/ruby/hbase.rb
+++ b/src/main/ruby/hbase.rb
@@ -57,6 +57,7 @@ module HBaseConstants
   SPLITS_FILE = 'SPLITS_FILE'
   SPLITALGO = 'SPLITALGO'
   NUMREGIONS = 'NUMREGIONS'
+  SKIP_FLUSH = 'SKIP_FLUSH'
 
   # Load constants from hbase java API
   def self.promote_constants(constants)
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 4ad3d88c7453..3ccb7e3f3d68 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -21,6 +21,7 @@
 include Java
 java_import org.apache.hadoop.hbase.util.Pair
 java_import org.apache.hadoop.hbase.util.RegionSplitter
+java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription
 
 # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
 
@@ -636,8 +637,18 @@ def online(region_name, on_off)
 
     #----------------------------------------------------------------------------------------------
     # Take a snapshot of specified table
-    def snapshot(table, snapshot_name)
-      @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+    def snapshot(table, snapshot_name, *args)
+      if args.empty?
+         @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+      else
+         args.each do |arg|
+            if arg[SKIP_FLUSH] == true
+              @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes, SnapshotDescription::Type::SKIPFLUSH)
+            else
+               @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) 
+           end
+         end
+      end
     end
 
     #----------------------------------------------------------------------------------------------
diff --git a/src/main/ruby/shell/commands/snapshot.rb b/src/main/ruby/shell/commands/snapshot.rb
index 1c4ecfec0e03..026e5622b187 100644
--- a/src/main/ruby/shell/commands/snapshot.rb
+++ b/src/main/ruby/shell/commands/snapshot.rb
@@ -24,12 +24,13 @@ def help
 Take a snapshot of specified table. Examples:
 
   hbase> snapshot 'sourceTable', 'snapshotName'
+  hbase> snapshot 'sourceTable', 'snapshotName', {SKIP_FLUSH => true}
 EOF
       end
 
-      def command(table, snapshot_name)
+      def command(table, snapshot_name, *args)
         format_simple_command do
-          admin.snapshot(table, snapshot_name)
+          admin.snapshot(table, snapshot_name, *args)
         end
       end
     end
diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index 5dfd7ebe10a0..c7a265f18a54 100644
--- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -125,6 +125,57 @@ public static void cleanupTest() throws Exception {
     }
   }
 
+  /**
+   * Test snapshotting a table that is online without flushing
+   * @throws Exception
+   */
+  @Test
+  public void testSkipFlushTableSnapshot() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // put some stuff in the table
+    HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
+    UTIL.loadTable(table, TEST_FAM);
+
+    // get the name of all the regionservers hosting the snapshotted table
+    Set<String> snapshotServers = new HashSet<String>();
+    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
+    for (RegionServerThread server : servers) {
+      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
+        snapshotServers.add(server.getRegionServer().getServerName().toString());
+      }
+    }
+
+    LOG.debug("FS state before snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+        FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    // take a snapshot of the enabled table
+    String snapshotString = "skipFlushTableSnapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotString);
+    admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
+    LOG.debug("Snapshot completed.");
+
+    // make sure we have the snapshot
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+        snapshot, TABLE_NAME);
+
+    // make sure its a valid snapshot
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    LOG.debug("FS state after snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+        FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+        admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+
+    admin.deleteSnapshot(snapshot);
+    snapshots = admin.listSnapshots();
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+  }
   /**
    * Test simple flush snapshotting a table that is online
    * @throws Exception

From 9498adde2a190b46de2289e67ced4b2f73a8baeb Mon Sep 17 00:00:00 2001
From: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Date: Mon, 9 Jun 2014 22:25:33 +0100
Subject: [PATCH 1428/1540] HBASE-8495 Change ownership of the directory to
 bulk load

---
 .../apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 5536a58d67a9..0f476d03d82d 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -46,6 +46,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
@@ -430,6 +431,10 @@ protected List<LoadQueueItem> splitStoreFile(final LoadQueueItem item,
     splitStoreFile(getConf(), hfilePath, familyDesc, splitKey,
         botOut, topOut);
 
+    FileSystem fs = tmpDir.getFileSystem(getConf());
+    fs.setPermission(tmpDir, FsPermission.valueOf("-rwxrwxrwx"));
+    fs.setPermission(botOut, FsPermission.valueOf("-rwxrwxrwx"));
+
     // Add these back at the *front* of the queue, so there's a lower
     // chance that the region will just split again before we get there.
     List<LoadQueueItem> lqis = new ArrayList<LoadQueueItem>(2);

From a228e925e7a80ae479700aa40b43ab2e002d4e7c Mon Sep 17 00:00:00 2001
From: Jimmy Xiang <jxiang@cloudera.com>
Date: Fri, 13 Jun 2014 09:13:37 -0700
Subject: [PATCH 1429/1540] HBASE-10871 Indefinite OPEN/CLOSE wait on busy
 RegionServers (Esteban)

---
 .../hbase/master/AssignmentManager.java       | 23 +++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 312a211b6562..87553175b0a2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1681,6 +1681,8 @@ private void assign(final HRegionInfo region, final RegionState state,
       boolean hijack) {
     boolean regionAlreadyInTransitionException = false;
     boolean serverNotRunningYet = false;
+    boolean socketTimeoutException = false;
+
     long maxRegionServerStartupWaitTime = -1;
     for (int i = 0; i < this.maximumAssignmentAttempts; i++) {
       int versionOfOfflineNode = -1;
@@ -1776,6 +1778,8 @@ private void assign(final HRegionInfo region, final RegionState state,
         }
         regionAlreadyInTransitionException = false;
         serverNotRunningYet = false;
+        socketTimeoutException = false;
+
         if (t instanceof RegionAlreadyInTransitionException) {
           regionAlreadyInTransitionException = true;
           if (LOG.isDebugEnabled()) {
@@ -1812,16 +1816,26 @@ private void assign(final HRegionInfo region, final RegionState state,
               + region.getRegionNameAsString()
               + ", but the region might already be opened on "
               + plan.getDestination() + ".", t);
-          return;
+          socketTimeoutException = true;
+          try {
+            Thread.sleep(100);
+            i--; // reset the try count
+          } catch (InterruptedException ie) {
+            LOG.warn("Failed to assign " + state.getRegion().getRegionNameAsString()
+                + " since interrupted", ie);
+            Thread.currentThread().interrupt();
+            return;
+          }
         }
         LOG.warn("Failed assignment of "
           + state.getRegion().getRegionNameAsString()
           + " to "
           + plan.getDestination()
           + ", trying to assign "
-          + (regionAlreadyInTransitionException || serverNotRunningYet
+          + (regionAlreadyInTransitionException || serverNotRunningYet || socketTimeoutException
             ? "to the same region server because of "
-            + "RegionAlreadyInTransitionException/ServerNotRunningYetException;"
+            + "RegionAlreadyInTransitionException/ServerNotRunningYetException/"
+            + "SocketTimeoutException;"
             : "elsewhere instead; ")
           + "retry=" + i, t);
         // Clean out plan we failed execute and one that doesn't look like it'll
@@ -1832,7 +1846,8 @@ private void assign(final HRegionInfo region, final RegionState state,
         // RS may cause double assignments. In case of RegionAlreadyInTransitionException
         // reassigning to same RS.
         RegionPlan newPlan = plan;
-        if (!regionAlreadyInTransitionException && !serverNotRunningYet) {
+        if (!regionAlreadyInTransitionException
+            && !serverNotRunningYet && !socketTimeoutException) {
           // Force a new plan and reassign. Will return null if no servers.
           // The new plan could be the same as the existing plan since we don't
           // exclude the server of the original plan, which should not be

From 286c16c329fc148036244d092bb7d370a53a3dbf Mon Sep 17 00:00:00 2001
From: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Date: Tue, 17 Jun 2014 11:43:19 +0100
Subject: [PATCH 1430/1540] HBASE-11341 ZKProcedureCoordinatorRpcs should
 respond only to members

---
 .../hbase/procedure/ZKProcedureUtil.java      | 19 +++++++++++++++++--
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java |  2 +-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
index 56941f2781ed..9839c729d574 100644
--- a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -186,7 +186,8 @@ boolean isAcquiredNode(String path) {
    * Is this in the procedure barrier acquired znode path
    */
   boolean isAcquiredPathNode(String path) {
-    return path.startsWith(this.acquiredZnode) && !path.equals(acquiredZnode);
+    return path.startsWith(this.acquiredZnode) && !path.equals(acquiredZnode) &&
+      isMemberNode(path, acquiredZnode);
   }
 
   /**
@@ -200,9 +201,23 @@ boolean isReachedNode(String path) {
    * Is this in the procedure barrier reached znode path
    */
   boolean isReachedPathNode(String path) {
-    return path.startsWith(this.reachedZnode) && !path.equals(reachedZnode);
+    return path.startsWith(this.reachedZnode) && !path.equals(reachedZnode) &&
+      isMemberNode(path, reachedZnode);
   }
 
+  /*
+   * Returns true if the specified path is a member of the "statePath"
+   *      /hbase/<ProcName>/<state>/<instance>/member
+   *      |------ state path -----|
+   *      |------------------ path ------------------|
+   */
+  private boolean isMemberNode(final String path, final String statePath) {
+    int count = 0;
+    for (int i = statePath.length(); i < path.length(); ++i) {
+      count += (path.charAt(i) == ZKUtil.ZNODE_PATH_SEPARATOR) ? 1 : 0;
+    }
+    return count == 2;
+  }
 
   /**
    * Is this in the procedure barrier abort znode path
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 6247813d7b65..9e03bd08313f 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -82,7 +82,7 @@ public class ZKUtil {
   private static final Log LOG = LogFactory.getLog(ZKUtil.class);
 
   // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
-  private static final char ZNODE_PATH_SEPARATOR = '/';
+  public static final char ZNODE_PATH_SEPARATOR = '/';
   private static int zkDumpConnectionTimeOut;
 
   /**

From ce722679a0a9096d253230ee0f61b9fff9c3638e Mon Sep 17 00:00:00 2001
From: Ted Yu <tedyu@apache.org>
Date: Wed, 18 Jun 2014 16:08:20 +0000
Subject: [PATCH 1431/1540] HBASE-11052 Sending random data crashes thrift
 service (Adrian Muraru)

---
 .../thrift/TBoundedThreadPoolServer.java      | 10 +++++++++-
 .../hbase/thrift/ThriftServerRunner.java      |  9 +++++++--
 .../hadoop/hbase/thrift2/ThriftServer.java    |  9 ++++++---
 src/main/resources/hbase-default.xml          | 20 +++++++++++++++++++
 4 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
index c7e104b5a87c..dd79aca8d0eb 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
@@ -284,7 +284,15 @@ public void run() {
         outputProtocol = outputProtocolFactory_.getProtocol(outputTransport);
         // we check stopped_ first to make sure we're not supposed to be shutting
         // down. this is necessary for graceful shutdown.
-        while (!stopped && processor.process(inputProtocol, outputProtocol)) {}
+        while (!stopped && processor.process(inputProtocol, outputProtocol)) {
+           // message limit is reset for every request
+           // see THRIFT-601, working on thrift 0.8.0
+           // NOTE that THRIFT-820 breaks this again in thrift 0.9.0
+           // and TFramedTransport needs to be used from this version onwards
+           // to avoid the buffer overflow
+           inputProtocol = inputProtocolFactory_.getProtocol(inputTransport);
+           outputProtocol = outputProtocolFactory_.getProtocol(outputTransport);
+        }
       } catch (TTransportException ttx) {
         // Assume the client died and continue silently
       } catch (TException tx) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 4aaa5657a939..f8317d34c018 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -115,6 +115,9 @@ public class ThriftServerRunner implements Runnable {
   static final String COMPACT_CONF_KEY = "hbase.regionserver.thrift.compact";
   static final String FRAMED_CONF_KEY = "hbase.regionserver.thrift.framed";
   static final String PORT_CONF_KEY = "hbase.regionserver.thrift.port";
+  //The max length of an individual thrift message and frames in MB
+  static final String MAX_MESSAGE_LENGTH_CONF_KEY = "hbase.regionserver.thrift.binary.max_message_length_in_mb";
+  static final String MAX_FRAME_SIZE_CONF_KEY = "hbase.regionserver.thrift.framed.max_frame_size_in_mb";
   static final String COALESCE_INC_KEY = "hbase.regionserver.thrift.coalesceIncrement";
 
   private static final String DEFAULT_BIND_ADDR = "0.0.0.0";
@@ -269,7 +272,8 @@ private void setupServer() throws Exception {
       protocolFactory = new TCompactProtocol.Factory();
     } else {
       LOG.debug("Using binary protocol");
-      protocolFactory = new TBinaryProtocol.Factory();
+      int maxMessageLength = conf.getInt(MAX_MESSAGE_LENGTH_CONF_KEY, 2) * 1024 * 1024;
+      protocolFactory = new TBinaryProtocol.Factory(false, true, maxMessageLength);
     }
 
     Hbase.Processor<Hbase.Iface> processor =
@@ -279,7 +283,8 @@ private void setupServer() throws Exception {
     // Construct correct TransportFactory
     TTransportFactory transportFactory;
     if (conf.getBoolean(FRAMED_CONF_KEY, false) || implType.isAlwaysFramed) {
-      transportFactory = new TFramedTransport.Factory();
+      int maxFrameSize = conf.getInt(MAX_FRAME_SIZE_CONF_KEY, 2)  * 1024 * 1024;
+      transportFactory = new TFramedTransport.Factory(maxFrameSize);
       LOG.debug("Using framed transport");
     } else {
       transportFactory = new TTransportFactory();
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index b6235ac6a441..e2831f7ae705 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -74,6 +74,8 @@ public class ThriftServer {
   private static final Log log = LogFactory.getLog(ThriftServer.class);
 
   public static final String DEFAULT_LISTEN_PORT = "9090";
+  //The max length of a message frame in MB
+  static final String MAX_FRAME_SIZE_CONF_KEY = "hbase.regionserver.thrift.framed.max_frame_size_in_mb";
 
   public ThriftServer() {
   }
@@ -124,10 +126,10 @@ private static TProtocolFactory getTProtocolFactory(boolean isCompact) {
     }
   }
 
-  private static TTransportFactory getTTransportFactory(boolean framed) {
+  private static TTransportFactory getTTransportFactory(boolean framed, int maxFrameSize) {
     if (framed) {
       log.debug("Using framed transport");
-      return new TFramedTransport.Factory();
+      return new TFramedTransport.Factory(maxFrameSize);
     } else {
       return new TTransportFactory();
     }
@@ -275,7 +277,8 @@ public static void main(String[] args) throws Exception {
 
       boolean framed = cmd.hasOption("framed") ||
         conf.getBoolean("hbase.regionserver.thrift.framed", false) || nonblocking || hsha;
-      TTransportFactory transportFactory = getTTransportFactory(framed);
+      TTransportFactory transportFactory = getTTransportFactory(framed,
+        conf.getInt(MAX_FRAME_SIZE_CONF_KEY, 2)  * 1024 * 1024);
       conf.setBoolean("hbase.regionserver.thrift.framed", framed);
 
       // TODO: Remove once HBASE-2155 is resolved
diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml
index 711406f0ddb5..ecf6252352cb 100644
--- a/src/main/resources/hbase-default.xml
+++ b/src/main/resources/hbase-default.xml
@@ -879,6 +879,26 @@
     (You will have to restart your cluster after setting it).
     </description>
   </property>
+  <property>
+    <name>hbase.regionserver.thrift.framed</name>
+    <value>false</value>
+    <description>Use Thrift TFramedTransport on the server side.
+    This is the recommended transport for thrift servers and requires a similar setting
+    on the client side.
+    Setting this to false will select the default transport,
+    vulnerable to DoS when malformed requests are issued due to THRIFT-601.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.thrift.framed.max_frame_size_in_mb</name>
+    <value>2</value>
+    <description>Default frame size when using framed transport</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.thrift.compact</name>
+    <value>false</value>
+    <description>Use Thrift TCompactProtocol binary serialization protocol.</description>
+  </property>
   <property>
     <name>hbase.thrift.minWorkerThreads</name>
     <value>16</value>

From 7ed89e68ddce274757e2788834a0ae667c655fe2 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 27 Jun 2014 07:00:05 -0700
Subject: [PATCH 1432/1540] HBASE-11414 Backport to 0.94: HBASE-7711 rowlock
 release problem with thread interruptions in batchMutate. (Ted Yu)

---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 134edaa782bb..31959b44d4ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3639,7 +3639,10 @@ private Integer internalObtainRowLock(final HashedBytes rowKey, boolean waitForL
               throw new IOException("Timed out on getting lock for row=" + rowKey);
             }
           } catch (InterruptedException ie) {
-            // Empty
+            LOG.warn("internalObtainRowLock interrupted for row=" + rowKey);
+            InterruptedIOException iie = new InterruptedIOException();
+            iie.initCause(ie);
+            throw iie;
           }
         }
       }

From 83b4a1ee9b9a2fa4c7ae1739259e041cabe8edc2 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 27 Jun 2014 07:08:42 -0700
Subject: [PATCH 1433/1540] pom.xml and CHANGES.txt for 0.94.21RC0

---
 CHANGES.txt | 20 ++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 2b3f8b6ac160..503f9749ee42 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,24 @@
 HBase Change Log
+Release 0.94.21 - 06/27/2014
+Bug
+
+    [HBASE-10692] - The Multi TableMap job don't support the security HBase cluster
+    [HBASE-11052] - Sending random data crashes thrift service
+    [HBASE-11096] - stop method of Master and RegionServer coprocessor is not invoked
+    [HBASE-11234] - FastDiffDeltaEncoder#getFirstKeyInBlock returns wrong result
+    [HBASE-11341] - ZKProcedureCoordinatorRpcs should respond only to members
+    [HBASE-11414] - Backport to 0.94: HBASE-7711 rowlock release problem with thread interruptions in batchMutate
+
+Improvement
+
+    [HBASE-8495] - Change ownership of the directory to bulk load
+    [HBASE-10871] - Indefinite OPEN/CLOSE wait on busy RegionServers
+
+New Feature
+
+    [HBASE-10935] - support snapshot policy where flush memstore can be skipped to prevent production cluster freeze
+
+
 Release 0.94.20 - 05/23/2014
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 7a5bdf42b54d..b114fe3e275d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.20</version>
+  <version>0.94.21</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 0cc71fb2e7e18b436bd6d01cabf6f0c4076d202d Mon Sep 17 00:00:00 2001
From: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Date: Wed, 2 Jul 2014 18:23:57 +0200
Subject: [PATCH 1434/1540] HBASE-11450 Improve file size info in SnapshotInfo
 tool

---
 .../hadoop/hbase/snapshot/SnapshotInfo.java   | 31 ++++++++++++++-----
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index b8ba593e7097..fc416cdb4e49 100644
--- a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -100,6 +100,12 @@ public boolean isMissing() {
       public long getSize() {
         return this.size;
       }
+
+      String getStateToString() {
+        if (isMissing()) return "NOT FOUND";
+        if (inArchive()) return "archive";
+        return null;
+      }
     }
 
     private int hfileArchiveCount = 0;
@@ -247,6 +253,7 @@ FileInfo addLogFile(final String server, final String logfile) throws IOExceptio
     }
   }
 
+  private boolean printSizeInBytes = false;
   private FileSystem fs;
   private Path rootDir;
 
@@ -283,6 +290,8 @@ public int run(String[] args) throws IOException, InterruptedException {
           FSUtils.setRootDir(conf, sourceDir);
         } else if (cmd.equals("-list-snapshots")) {
           listSnapshots = true;
+        } else if (cmd.equals("-size-in-bytes")) {
+          printSizeInBytes = true;
         } else if (cmd.equals("-h") || cmd.equals("--help")) {
           printUsageAndExit();
         } else {
@@ -392,10 +401,11 @@ public void storeFile (final String region, final String family, final String hf
           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
 
           if (showFiles) {
+            String state = info.getStateToString();
             System.out.printf("%8s %s/%s/%s/%s %s%n",
-              (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
+              (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
               table, region, family, hfile,
-              (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
+              state == null ? "" : "(" + state + ")");
           }
         }
 
@@ -405,7 +415,7 @@ public void recoveredEdits (final String region, final String logfile)
 
           if (showFiles) {
             System.out.printf("%8s recovered.edits %s on region %s%n",
-              StringUtils.humanReadableInt(info.getSize()), logfile, region);
+              fileSizeToString(info.getSize()), logfile, region);
           }
         }
 
@@ -414,10 +424,11 @@ public void logFile (final String server, final String logfile)
           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
 
           if (showFiles) {
+            String state = info.getStateToString();
             System.out.printf("%8s log %s on server %s %s%n",
-              (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
+              (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
               logfile, server,
-              (info.isMissing() ? "(NOT FOUND)" : ""));
+              state == null ? "" : "(" + state + ")");
           }
         }
     });
@@ -434,16 +445,20 @@ public void logFile (final String server, final String logfile)
     if (showStats) {
       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
-        StringUtils.humanReadableInt(stats.getStoreFilesSize()),
+        fileSizeToString(stats.getStoreFilesSize()),
         stats.getSharedStoreFilePercentage(),
-        StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
+        fileSizeToString(stats.getSharedStoreFilesSize())
       );
       System.out.printf("%d Logs, total size %s%n",
-        stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
+        stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
       System.out.println();
     }
   }
 
+  private String fileSizeToString(long size) {
+    return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
+  }
+
   private void printUsageAndExit() {
     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
     System.err.println(" where [options] are:");

From 4b2938f19314dc6cf658cae0fbc76f77063c8fd6 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 3 Jul 2014 09:41:04 -0700
Subject: [PATCH 1435/1540] move version to 0.94.22-SNAPSHOT

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index b114fe3e275d..14cbc26ab30d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.21</version>
+  <version>0.94.22-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 24f150ebbc132baf99022cbdd1b28fa7842522f7 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Thu, 10 Jul 2014 17:47:04 -0700
Subject: [PATCH 1436/1540] HBASE-11496 HBASE-9745 broke cygwin CLASSPATH
 translation (Dave Latham)

---
 bin/hbase | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/bin/hbase b/bin/hbase
index adf4dae92ad3..ec6c23c6d114 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -198,13 +198,6 @@ if [ "$HBASE_LOGFILE" = "" ]; then
   HBASE_LOGFILE='hbase.log'
 fi
 
-# cygwin path translation
-if $cygwin; then
-  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-  HBASE_HOME=`cygpath -d "$HBASE_HOME"`
-  HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"`
-fi
-
 function append_path() {
   if [ -z "$1" ]; then
     echo $2
@@ -226,6 +219,23 @@ if [ -f ${HADOOP_IN_PATH} ]; then
   CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`)
 fi
 
+# Add user-specified CLASSPATH last
+if [ "$HBASE_CLASSPATH" != "" ]; then
+  CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH}
+fi
+
+# Add user-specified CLASSPATH prefix first
+if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then
+  CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH}
+fi
+
+# cygwin path translation
+if $cygwin; then
+  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
+  HBASE_HOME=`cygpath -d "$HBASE_HOME"`
+  HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"`
+fi
+
 if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then
   if [ -z $JAVA_PLATFORM ]; then
     JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
@@ -239,16 +249,6 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then
   fi
 fi
 
-# Add user-specified CLASSPATH last
-if [ "$HBASE_CLASSPATH" != "" ]; then
-  CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH}
-fi
-
-# Add user-specified CLASSPATH prefix first
-if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then
-  CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH}
-fi
-
 # cygwin path translation
 if $cygwin; then
   JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`

From 24b3584a1735b5a0fcd0f25a5d3903b31901b227 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Fri, 11 Jul 2014 13:11:00 -0700
Subject: [PATCH 1437/1540] HBASE-11479 SecureConnection can't be closed when
 SecureClient is stopping (cuijianwei)

---
 .../java/org/apache/hadoop/hbase/ipc/SecureClient.java     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 9599a6e909e9..38ad5aa58bdd 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -293,8 +293,11 @@ public Boolean run() throws IOException {
               if (rand == null) {
                 rand = new Random();
               }
-              handleSaslConnectionFailure(numRetries++, MAX_RETRIES, ex, rand,
-                   ticket);
+              try {
+                handleSaslConnectionFailure(numRetries++, MAX_RETRIES, ex, rand, ticket);
+              } catch (InterruptedException e) {
+                throw new IOException(e);
+              }
               continue;
             }
             if (continueSasl) {

From 4453e9cf6b6306a6c49359f1fe87263a72ef1c30 Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Tue, 15 Jul 2014 10:15:48 -0700
Subject: [PATCH 1438/1540] HBASE-11480 ClientScanner might not close the
 HConnection created in construction (cuijianwei)

---
 src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 68751125e12b..be93d586043e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -70,6 +70,7 @@ public class ClientScanner extends AbstractClientScanner {
      * @param tableName The table that we wish to scan
      * @throws IOException
      */
+    @Deprecated
     public ClientScanner(final Configuration conf, final Scan scan,
         final byte[] tableName) throws IOException {
       this(conf, scan, tableName, HConnectionManager.getConnection(conf));

From 225561cc6d6f69fe948b908bb305a73faab9c336 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Wed, 16 Jul 2014 17:11:54 -0700
Subject: [PATCH 1439/1540] HBASE-2217 VM OPTS for shell only

---
 bin/hbase | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/hbase b/bin/hbase
index ec6c23c6d114..851698507e6e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -279,6 +279,7 @@ if [ "$COMMAND" = "shell" ] ; then
     CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH"
     HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib"
   fi
+  HBASE_OPTS="$HBASE_OPTS $HBASE_SHELL_OPTS"
   CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb"
 elif [ "$COMMAND" = "hbck" ] ; then
   CLASS='org.apache.hadoop.hbase.util.HBaseFsck'

From 354dd40c8ee7363eb6200f7d08a83966f860ba58 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Mon, 21 Jul 2014 10:05:00 -0700
Subject: [PATCH 1440/1540] HBASE-7910 Dont use reflection for security (Mike
 Drob and Andrew Purtell)

---
 .../apache/hadoop/hbase/security/User.java    | 335 ++----------------
 1 file changed, 27 insertions(+), 308 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index c0eb3a51a863..96e0a395904d 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -20,23 +19,18 @@
 
 package org.apache.hadoop.hbase.security;
 
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.util.Methods;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.UserGroupInformation;
-
 import java.io.IOException;
-import java.lang.reflect.Constructor;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 
-import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.Methods;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 
 /**
  * Wrapper to abstract out usage of user and group information in HBase.
@@ -53,21 +47,6 @@ public abstract class User {
   public static final String HBASE_SECURITY_CONF_KEY =
       "hbase.security.authentication";
 
-  /**
-   * Flag to differentiate between API-incompatible changes to
-   * {@link org.apache.hadoop.security.UserGroupInformation} between vanilla
-   * Hadoop 0.20.x and secure Hadoop 0.20+.
-   */
-  private static boolean IS_SECURE_HADOOP = true;
-  static {
-    try {
-      UserGroupInformation.class.getMethod("isSecurityEnabled");
-    } catch (NoSuchMethodException nsme) {
-      IS_SECURE_HADOOP = false;
-    }
-  }
-  private static Log LOG = LogFactory.getLog(User.class);
-
   protected UserGroupInformation ugi;
 
   public UserGroupInformation getUGI() {
@@ -157,12 +136,12 @@ public boolean equals(Object o) {
     }
     return ugi.equals(((User) o).ugi);
   }
-  
+
   @Override
   public int hashCode() {
     return ugi.hashCode();
   }
-  
+
   @Override
   public String toString() {
     return ugi.toString();
@@ -172,12 +151,7 @@ public String toString() {
    * Returns the {@code User} instance within current execution context.
    */
   public static User getCurrent() throws IOException {
-    User user;
-    if (IS_SECURE_HADOOP) {
-      user = new SecureHadoopUser();
-    } else {
-      user = new HadoopUser();
-    }
+    User user = new SecureHadoopUser();
     if (user.getUGI() == null) {
       return null;
     }
@@ -193,11 +167,7 @@ public static User create(UserGroupInformation ugi) {
     if (ugi == null) {
       return null;
     }
-
-    if (IS_SECURE_HADOOP) {
-      return new SecureHadoopUser(ugi);
-    }
-    return new HadoopUser(ugi);
+    return new SecureHadoopUser(ugi);
   }
 
   /**
@@ -208,10 +178,7 @@ public static User create(UserGroupInformation ugi) {
    */
   public static User createUserForTesting(Configuration conf,
       String name, String[] groups) {
-    if (IS_SECURE_HADOOP) {
-      return SecureHadoopUser.createUserForTesting(conf, name, groups);
-    }
-    return HadoopUser.createUserForTesting(conf, name, groups);
+    return SecureHadoopUser.createUserForTesting(conf, name, groups);
   }
 
   /**
@@ -232,11 +199,7 @@ public static User createUserForTesting(Configuration conf,
    */
   public static void login(Configuration conf, String fileConfKey,
       String principalConfKey, String localhost) throws IOException {
-    if (IS_SECURE_HADOOP) {
-      SecureHadoopUser.login(conf, fileConfKey, principalConfKey, localhost);
-    } else {
-      HadoopUser.login(conf, fileConfKey, principalConfKey, localhost);
-    }
+    SecureHadoopUser.login(conf, fileConfKey, principalConfKey, localhost);
   }
 
   /**
@@ -246,11 +209,7 @@ public static void login(Configuration conf, String fileConfKey,
    * {@code UserGroupInformation.isSecurityEnabled()}.
    */
   public static boolean isSecurityEnabled() {
-    if (IS_SECURE_HADOOP) {
-      return SecureHadoopUser.isSecurityEnabled();
-    } else {
-      return HadoopUser.isSecurityEnabled();
-    }
+    return SecureHadoopUser.isSecurityEnabled();
   }
 
   /**
@@ -264,237 +223,50 @@ public static boolean isHBaseSecurityEnabled(Configuration conf) {
 
   /* Concrete implementations */
 
-  /**
-   * Bridges {@link User} calls to invocations of the appropriate methods
-   * in {@link org.apache.hadoop.security.UserGroupInformation} in regular
-   * Hadoop 0.20 (ASF Hadoop and other versions without the backported security
-   * features).
-   */
-  private static class HadoopUser extends User {
-
-    private HadoopUser() {
-      try {
-        ugi = (UserGroupInformation) callStatic("getCurrentUGI");
-        if (ugi == null) {
-          // Secure Hadoop UGI will perform an implicit login if the current
-          // user is null.  Emulate the same behavior here for consistency
-          Configuration conf = HBaseConfiguration.create();
-          ugi = (UserGroupInformation) callStatic("login",
-              new Class[]{ Configuration.class }, new Object[]{ conf });
-          if (ugi != null) {
-            callStatic("setCurrentUser",
-                new Class[]{ UserGroupInformation.class }, new Object[]{ ugi });
-          }
-        }
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception HadoopUser<init>");
-      }
-    }
-
-    private HadoopUser(UserGroupInformation ugi) {
-      this.ugi = ugi;
-    }
-
-    @Override
-    public String getShortName() {
-      return ugi != null ? ugi.getUserName() : null;
-    }
-
-    @Override
-    public <T> T runAs(PrivilegedAction<T> action) {
-      T result = null;
-      UserGroupInformation previous = null;
-      try {
-        previous = (UserGroupInformation) callStatic("getCurrentUGI");
-        try {
-          if (ugi != null) {
-            callStatic("setCurrentUser", new Class[]{UserGroupInformation.class},
-                new Object[]{ugi});
-          }
-          result = action.run();
-        } finally {
-          callStatic("setCurrentUser", new Class[]{UserGroupInformation.class},
-              new Object[]{previous});
-        }
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception in runAs()");
-      }
-      return result;
-    }
-
-    @Override
-    public <T> T runAs(PrivilegedExceptionAction<T> action)
-        throws IOException, InterruptedException {
-      T result = null;
-      try {
-        UserGroupInformation previous =
-            (UserGroupInformation) callStatic("getCurrentUGI");
-        try {
-          if (ugi != null) {
-            callStatic("setCurrentUGI", new Class[]{UserGroupInformation.class},
-                new Object[]{ugi});
-          }
-          result = action.run();
-        } finally {
-          callStatic("setCurrentUGI", new Class[]{UserGroupInformation.class},
-              new Object[]{previous});
-        }
-      } catch (Exception e) {
-        if (e instanceof IOException) {
-          throw (IOException)e;
-        } else if (e instanceof InterruptedException) {
-          throw (InterruptedException)e;
-        } else if (e instanceof RuntimeException) {
-          throw (RuntimeException)e;
-        } else {
-          throw new UndeclaredThrowableException(e, "Unknown exception in runAs()");
-        }
-      }
-      return result;
-    }
-
-    @Override
-    public void obtainAuthTokenForJob(Configuration conf, Job job)
-        throws IOException, InterruptedException {
-      // this is a no-op.  token creation is only supported for kerberos
-      // authenticated clients
-    }
-
-    @Override
-    public void obtainAuthTokenForJob(JobConf job)
-        throws IOException, InterruptedException {
-      // this is a no-op.  token creation is only supported for kerberos
-      // authenticated clients
-    }
-
-    /** @see User#createUserForTesting(org.apache.hadoop.conf.Configuration, String, String[]) */
-    public static User createUserForTesting(Configuration conf,
-        String name, String[] groups) {
-      try {
-        Class c = Class.forName("org.apache.hadoop.security.UnixUserGroupInformation");
-        Constructor constructor = c.getConstructor(String.class, String[].class);
-        if (constructor == null) {
-          throw new NullPointerException(
-             );
-        }
-        UserGroupInformation newUser =
-            (UserGroupInformation)constructor.newInstance(name, groups);
-        // set user in configuration -- hack for regular hadoop
-        conf.set("hadoop.job.ugi", newUser.toString());
-        return new HadoopUser(newUser);
-      } catch (ClassNotFoundException cnfe) {
-        throw new RuntimeException(
-            "UnixUserGroupInformation not found, is this secure Hadoop?", cnfe);
-      } catch (NoSuchMethodException nsme) {
-        throw new RuntimeException(
-            "No valid constructor found for UnixUserGroupInformation!", nsme);
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception instantiating new UnixUserGroupInformation");
-      }
-    }
-
-    /**
-     * No-op since we're running on a version of Hadoop that doesn't support
-     * logins.
-     * @see User#login(org.apache.hadoop.conf.Configuration, String, String, String)
-     */
-    public static void login(Configuration conf, String fileConfKey,
-        String principalConfKey, String localhost) throws IOException {
-      LOG.info("Skipping login, not running on secure Hadoop");
-    }
-
-    /** Always returns {@code false}. */
-    public static boolean isSecurityEnabled() {
-      return false;
-    }
-  }
-
   /**
    * Bridges {@code User} invocations to underlying calls to
    * {@link org.apache.hadoop.security.UserGroupInformation} for secure Hadoop
    * 0.20 and versions 0.21 and above.
    */
-  public static class SecureHadoopUser extends User {
+  private static class SecureHadoopUser extends User {
     private String shortName;
 
     private SecureHadoopUser() throws IOException {
-      try {
-        ugi = (UserGroupInformation) callStatic("getCurrentUser");
-      } catch (IOException ioe) {
-        throw ioe;
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception getting current secure user");
-      }
+      ugi = UserGroupInformation.getCurrentUser();
     }
 
-    public SecureHadoopUser(UserGroupInformation ugi) {
+    private SecureHadoopUser(UserGroupInformation ugi) {
       this.ugi = ugi;
     }
 
     @Override
     public String getShortName() {
       if (shortName != null) return shortName;
-
       try {
-        shortName = (String)call(ugi, "getShortUserName", null, null);
+        shortName = ugi.getShortUserName();
         return shortName;
-      } catch (RuntimeException re) {
-        throw re;
       } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected error getting user short name");
+        throw new RuntimeException("Unexpected error getting user short name",
+          e);
       }
     }
 
     @Override
     public <T> T runAs(PrivilegedAction<T> action) {
-      try {
-        return (T) call(ugi, "doAs", new Class[]{PrivilegedAction.class},
-            new Object[]{action});
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception in runAs()");
-      }
+      return ugi.doAs(action);
     }
 
     @Override
     public <T> T runAs(PrivilegedExceptionAction<T> action)
         throws IOException, InterruptedException {
-      try {
-        return (T) call(ugi, "doAs",
-            new Class[]{PrivilegedExceptionAction.class},
-            new Object[]{action});
-      } catch (IOException ioe) {
-        throw ioe;
-      } catch (InterruptedException ie) {
-        throw ie;
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception in runAs(PrivilegedExceptionAction)");
-      }
+      return ugi.doAs(action);
     }
 
     @Override
     public void obtainAuthTokenForJob(Configuration conf, Job job)
         throws IOException, InterruptedException {
       try {
-        Class c = Class.forName(
+        Class<?> c = Class.forName(
             "org.apache.hadoop.hbase.security.token.TokenUtil");
         Methods.call(c, null, "obtainTokenForJob",
             new Class[]{Configuration.class, UserGroupInformation.class,
@@ -519,7 +291,7 @@ public void obtainAuthTokenForJob(Configuration conf, Job job)
     public void obtainAuthTokenForJob(JobConf job)
         throws IOException, InterruptedException {
       try {
-        Class c = Class.forName(
+        Class<?> c = Class.forName(
             "org.apache.hadoop.hbase.security.token.TokenUtil");
         Methods.call(c, null, "obtainTokenForJob",
             new Class[]{JobConf.class, UserGroupInformation.class},
@@ -542,18 +314,7 @@ public void obtainAuthTokenForJob(JobConf job)
     /** @see User#createUserForTesting(org.apache.hadoop.conf.Configuration, String, String[]) */
     public static User createUserForTesting(Configuration conf,
         String name, String[] groups) {
-      try {
-        return new SecureHadoopUser(
-            (UserGroupInformation)callStatic("createUserForTesting",
-                new Class[]{String.class, String[].class},
-                new Object[]{name, groups})
-        );
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Error creating secure test user");
-      }
+      return new SecureHadoopUser(UserGroupInformation.createUserForTesting(name, groups));
     }
 
     /**
@@ -571,26 +332,7 @@ public static User createUserForTesting(Configuration conf,
     public static void login(Configuration conf, String fileConfKey,
         String principalConfKey, String localhost) throws IOException {
       if (isSecurityEnabled()) {
-        // check for SecurityUtil class
-        try {
-          Class c = Class.forName("org.apache.hadoop.security.SecurityUtil");
-          Class[] types = new Class[]{
-              Configuration.class, String.class, String.class, String.class };
-          Object[] args = new Object[]{
-              conf, fileConfKey, principalConfKey, localhost };
-          Methods.call(c, null, "login", types, args);
-        } catch (ClassNotFoundException cnfe) {
-          throw new RuntimeException("Unable to login using " +
-              "org.apache.hadoop.security.SecurityUtil.login(). SecurityUtil class " +
-              "was not found!  Is this a version of secure Hadoop?", cnfe);
-        } catch (IOException ioe) {
-          throw ioe;
-        } catch (RuntimeException re) {
-          throw re;
-        } catch (Exception e) {
-          throw new UndeclaredThrowableException(e,
-              "Unhandled exception in User.login()");
-        }
+        SecurityUtil.login(conf, fileConfKey, principalConfKey, localhost);
       }
     }
 
@@ -598,30 +340,7 @@ public static void login(Configuration conf, String fileConfKey,
      * Returns the result of {@code UserGroupInformation.isSecurityEnabled()}.
      */
     public static boolean isSecurityEnabled() {
-      try {
-        return (Boolean)callStatic("isSecurityEnabled");
-      } catch (RuntimeException re) {
-        throw re;
-      } catch (Exception e) {
-        throw new UndeclaredThrowableException(e,
-            "Unexpected exception calling UserGroupInformation.isSecurityEnabled()");
-      }
+      return UserGroupInformation.isSecurityEnabled();
     }
   }
-
-  /* Reflection helper methods */
-  private static Object callStatic(String methodName) throws Exception {
-    return call(null, methodName, null, null);
-  }
-
-  private static Object callStatic(String methodName, Class[] types,
-      Object[] args) throws Exception {
-    return call(null, methodName, types, args);
-  }
-
-  private static Object call(UserGroupInformation instance, String methodName,
-      Class[] types, Object[] args) throws Exception {
-    return Methods.call(UserGroupInformation.class, instance, methodName, types,
-        args);
-  }
 }

From 8cee70ff8b9b49c8942629b143867b8308560529 Mon Sep 17 00:00:00 2001
From: Jimmy Xiang <jxiang@cloudera.com>
Date: Wed, 23 Jul 2014 08:09:17 -0700
Subject: [PATCH 1441/1540] HBASE-11565 Stale connection could stay for a while

---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 8d5cff7c55ed..a74abdb3bc90 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -628,7 +628,12 @@ protected void sendParam(Call call) {
           out.flush();
         }
       } catch(IOException e) {
-        markClosed(e);
+        synchronized (this) {
+          if (!shouldCloseConnection.get()) {
+            markClosed(e);
+            interrupt();
+          }
+        }
       } finally {
         //the buffer is just an in-memory buffer, but it is still polite to
         // close early

From 7f4aee1f4b9097a9d7333b320acbbf167a086b02 Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Wed, 23 Jul 2014 11:49:05 -0700
Subject: [PATCH 1442/1540] HBASE-10645 Fix wrapping of Requests Counts
 Regionserver level metrics (deepankar)

---
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 36fa0a3151ad..e6622a001f3a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1527,8 +1527,8 @@ protected void metrics() {
     int stores = 0;
     int storefiles = 0;
     long memstoreSize = 0;
-    int readRequestsCount = 0;
-    int writeRequestsCount = 0;
+    long readRequestsCount = 0;
+    long writeRequestsCount = 0;
     long storefileIndexSize = 0;
     HDFSBlocksDistribution hdfsBlocksDistribution =
       new HDFSBlocksDistribution();

From 197d3657c7d3e35f8e8fd9a868aa9eadaac82065 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 28 Jul 2014 11:00:12 -0700
Subject: [PATCH 1443/1540] SnapshotFileCache causes too many cache refreshes.
 (churro morales)

---
 .../master/snapshot/SnapshotFileCache.java    |  92 +++++++-------
 .../master/snapshot/SnapshotHFileCleaner.java |  15 ++-
 .../master/snapshot/SnapshotLogCleaner.java   |  12 +-
 .../snapshot/TestSnapshotFileCache.java       | 114 +++++++++++++++---
 4 files changed, 162 insertions(+), 71 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index ec14aa16b1b4..a7379f6c7083 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -19,14 +19,10 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.*;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -39,13 +35,14 @@
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.FSUtils;
 
+
 /**
  * Intelligently keep track of all the files for all the snapshots.
  * <p>
  * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
  * miss the directory modification time is used to ensure that we don't rescan directories that we
  * already have in cache. We only check the modification times of the snapshot directories
- * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
+ * (/hbase/.hbase-snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
  * <p>
  * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
  * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
@@ -63,8 +60,8 @@
  * This allows you to only cache files under, for instance, all the logs in the .logs directory or
  * all the files under all the regions.
  * <p>
- * <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
- * snapshots and will attempt to cache files from those snapshots as well.
+ * <tt>this</tt> also considers all running snapshots (those under /hbase/.hbase-snapshot/.tmp) as valid
+ * snapshots but will not attempt to cache files from that directory.
  * <p>
  * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
  */
@@ -165,26 +162,40 @@ public void triggerCacheRefreshForTesting() {
    * at that point, cache will still think the file system contains that file and return
    * <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
    * on the filesystem, we will never find it and always return <tt>false</tt>.
-   * @param fileName file to check
-   * @return <tt>false</tt> if the file is not referenced in any current or running snapshot,
-   *         <tt>true</tt> if the file is in the cache.
+   * @param files file to check, NOTE: Relies that files are loaded from hdfs before method is called (NOT LAZY)
+   * @return <tt>unReferencedFiles</tt> the collection of files that do not have snapshot references
    * @throws IOException if there is an unexpected error reaching the filesystem.
    */
   // XXX this is inefficient to synchronize on the method, when what we really need to guard against
   // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
   // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
-  public synchronized boolean contains(String fileName) throws IOException {
-    if (this.cache.contains(fileName)) return true;
-
+  public synchronized Iterable<FileStatus> getUnreferencedFiles(Iterable<FileStatus> files) throws IOException {
+    List<FileStatus> unReferencedFiles = Lists.newArrayList();
+    List<String> snapshotsInProgress = null;
+    boolean refreshed = false;
+    for (FileStatus file : files) {
+      String fileName = file.getPath().getName();
+      if (!refreshed && !cache.contains(fileName)) {
     refreshCache();
-
-    // then check again
-    return this.cache.contains(fileName);
+        refreshed = true;
+      }
+      if (cache.contains(fileName)) {
+        continue;
+      }
+      if (snapshotsInProgress == null) {
+        snapshotsInProgress = getSnapshotsInProgress();
+      }
+      if (snapshotsInProgress.contains(fileName)) {
+        continue;
+      }
+      unReferencedFiles.add(file);
+    }
+    return unReferencedFiles;
   }
 
   private synchronized void refreshCache() throws IOException {
-    // get the status of the snapshots directory and <snapshot dir>/.tmp
-    FileStatus dirStatus, tempStatus;
+    // get the status of the snapshots directory 
+    FileStatus dirStatus;
     try {
       dirStatus = fs.getFileStatus(snapshotDir);
     } catch (FileNotFoundException e) {
@@ -194,16 +205,7 @@ private synchronized void refreshCache() throws IOException {
       return;
     }
 
-    try {
-      Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
-      tempStatus = fs.getFileStatus(snapshotTmpDir);
-    } catch (FileNotFoundException e) {
-      tempStatus = dirStatus;
-    }
-
-    // if the snapshot directory wasn't modified since we last check, we are done
-    if (dirStatus.getModificationTime() <= lastModifiedTime &&
-        tempStatus.getModificationTime() <= lastModifiedTime) {
+    if (dirStatus.getModificationTime() <= lastModifiedTime) {
       return;
     }
 
@@ -213,8 +215,7 @@ private synchronized void refreshCache() throws IOException {
     // However, snapshot directories are only created once, so this isn't an issue.
 
     // 1. update the modified time
-    this.lastModifiedTime = Math.min(dirStatus.getModificationTime(),
-                                     tempStatus.getModificationTime());
+    this.lastModifiedTime = dirStatus.getModificationTime();
 
     // 2.clear the cache
     this.cache.clear();
@@ -234,15 +235,7 @@ private synchronized void refreshCache() throws IOException {
     // 3.1 iterate through the on-disk snapshots
     for (FileStatus snapshot : snapshots) {
       String name = snapshot.getPath().getName();
-      // its the tmp dir
-      if (name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
-        // only add those files to the cache, but not to the known snapshots
-        FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
-        if (running == null) continue;
-        for (FileStatus run : running) {
-          this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
-        }
-      } else {
+      if (!name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
         SnapshotDirectoryInfo files = this.snapshots.remove(name);
         // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
         // the latter could occur where I create a snapshot, then delete it, and then make a new
@@ -264,6 +257,20 @@ private synchronized void refreshCache() throws IOException {
     this.snapshots.putAll(known);
   }
 
+  @VisibleForTesting List<String> getSnapshotsInProgress() throws IOException {
+    List<String> snapshotInProgress = Lists.newArrayList();
+    // only add those files to the cache, but not to the known snapshots
+    Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+    // only add those files to the cache, but not to the known snapshots
+    FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
+    if (running != null) {
+      for (FileStatus run : running) {
+        snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+      }
+    }
+    return snapshotInProgress;
+  }
+
   /**
    * Simple helper task that just periodically attempts to refresh the cache
    */
@@ -321,4 +328,5 @@ public boolean hasBeenModified(long mtime) {
       return this.lastModified < mtime;
     }
   }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
index e82ca1608c45..163bc317f7d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -55,14 +56,18 @@ public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
   private SnapshotFileCache cache;
 
   @Override
-  public synchronized boolean isFileDeletable(FileStatus fStat) {
+  public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
     try {
-      return !cache.contains(fStat.getPath().getName());
+      return cache.getUnreferencedFiles(files);
     } catch (IOException e) {
-      LOG.error("Exception while checking if:" + fStat.getPath()
-          + " was valid, keeping it just in case.", e);
-      return false;
+      LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
+      return Collections.emptyList();
     }
+    }
+
+  @Override
+  protected boolean isFileDeletable(FileStatus fStat) {
+    return false;
   }
 
   @Override
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
index c1edd6f7f0db..7da64c561ea4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -54,14 +55,13 @@ public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
   private SnapshotFileCache cache;
 
   @Override
-  public synchronized boolean isLogDeletable(FileStatus fStat) {
+  public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
+    if (null == cache) return Collections.emptyList();
     try {
-      if (null == cache) return false;
-      return !cache.contains(fStat.getPath().getName());
+      return cache.getUnreferencedFiles(files);
     } catch (IOException e) {
-      LOG.error("Exception while checking if:" + fStat.getPath()
-        + " was valid, keeping it just in case.", e);
-      return false;
+      LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
+      return Collections.emptyList();
     }
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
index 409f697f9441..8bf4e96cbc6b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
@@ -17,15 +17,19 @@
  */
 package org.apache.hadoop.hbase.master.snapshot;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ObjectArrays;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -85,6 +89,7 @@ public void testLoadAndDelete() throws Exception {
     Path file1 = new Path(family, "file1");
     Path file2 = new Path(family, "file2");
 
+
     // create two hfiles under the snapshot
     fs.createNewFile(file1);
     fs.createNewFile(file2);
@@ -92,10 +97,13 @@ public void testLoadAndDelete() throws Exception {
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
     // then make sure the cache finds them
-    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+    Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
+            Arrays.asList(FSUtils.listStatus(fs, family))
+    );
+    assertFalse("Cache didn't find:" + file1, Iterables.contains(nonSnapshotFiles, file1));
+    assertFalse("Cache didn't find:" + file2, Iterables.contains(nonSnapshotFiles, file2));
     String not = "file-shouldn't-be-found";
-    assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
+    assertFalse("Cache found '" + not + "', but it shouldn't have.", Iterables.contains(nonSnapshotFiles, not));
 
     // make sure we get a little bit of separation in the modification times
     // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
@@ -110,20 +118,79 @@ public void testLoadAndDelete() throws Exception {
 
 
     LOG.debug("Checking to see if file is deleted.");
-    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+    nonSnapshotFiles = cache.getUnreferencedFiles(
+            nonSnapshotFiles
+    );
+    
+    assertFalse("Cache didn't find:" + file1, Iterables.contains(nonSnapshotFiles, file1));
+    assertFalse("Cache didn't find:" + file2, Iterables.contains(nonSnapshotFiles, file2));
 
     // then trigger a refresh
     cache.triggerCacheRefreshForTesting();
+
+    nonSnapshotFiles = cache.getUnreferencedFiles(
+            nonSnapshotFiles
+    );
     // and not it shouldn't find those files
     assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
-      cache.contains(file1.getName()));
+            Iterables.contains(nonSnapshotFiles, file1));
     assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
-      cache.contains(file2.getName()));
+            Iterables.contains(nonSnapshotFiles, file2));
 
     fs.delete(snapshotDir, true);
   }
 
+  @Test
+  public void testWeNeverCacheTmpDirAndLoadIt() throws Exception {
+
+    final AtomicInteger count = new AtomicInteger(0);
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+            "test-snapshot-file-cache-refresh", new SnapshotFiles()) {
+      @Override
+      List<String> getSnapshotsInProgress() throws IOException {
+        List<String> result = super.getSnapshotsInProgress();
+        count.incrementAndGet();
+        return result;
+      }
+    };
+
+    // create a file in a 'completed' snapshot
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    fs.createNewFile(file1);
+
+    FileStatus[] completedFiles = FSUtils.listStatus(fs, family);
+
+    // create an 'in progress' snapshot
+    SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
+    snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+    region = new Path(snapshot, "7e91021");
+    family = new Path(region, "fam");
+    Path file2 = new Path(family, "file2");
+    fs.createNewFile(file2);
+    cache.triggerCacheRefreshForTesting();
+
+    Iterable<FileStatus> deletableFiles = cache.getUnreferencedFiles(Arrays.asList(
+            ObjectArrays.concat(completedFiles, FSUtils.listStatus(fs, family), FileStatus.class))
+    );
+    assertTrue(Iterables.isEmpty(deletableFiles));
+    assertEquals(1, count.get()); // we check the tmp directory
+
+    Path file3 = new Path(family, "file3");
+    fs.create(file3);
+    deletableFiles = cache.getUnreferencedFiles(Arrays.asList(
+            ObjectArrays.concat(completedFiles, FSUtils.listStatus(fs, family), FileStatus.class))
+    );
+    assertTrue(Iterables.isEmpty(deletableFiles));
+    assertEquals(2, count.get()); // we check the tmp directory
+
+  }
+
   @Test
   public void testLoadsTmpDir() throws Exception {
     // don't refresh the cache unless we tell it to
@@ -150,8 +217,11 @@ public void testLoadsTmpDir() throws Exception {
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
     // then make sure the cache finds both files
-    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+    Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
+            Arrays.asList(FSUtils.listStatus(fs, family))
+    );
+    assertFalse("Cache didn't find:" + file1, Iterables.contains(nonSnapshotFiles, file1));
+    assertFalse("Cache didn't find:" + file2, Iterables.contains(nonSnapshotFiles, file2));
   }
 
   @Test
@@ -181,10 +251,13 @@ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
+    Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
+            Arrays.asList(FSUtils.listStatus(fs, family))
+    );    
     // then make sure the cache only finds the log files
     assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
-      cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
+            Iterables.contains(nonSnapshotFiles, file1));
+    assertFalse("Cache didn't find:" + log, Iterables.contains(nonSnapshotFiles, log));
   }
 
   @Test
@@ -207,7 +280,10 @@ public void testReloadModifiedDirectory() throws IOException {
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
-    assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
+    Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
+            Arrays.asList(FSUtils.listStatus(fs, family))
+    );  
+    assertFalse("Cache didn't find " + file1, Iterables.contains(nonSnapshotFiles, file1));
 
     // now delete the snapshot and add a file with a different name
     fs.delete(snapshot, true);
@@ -215,13 +291,15 @@ public void testReloadModifiedDirectory() throws IOException {
     fs.createNewFile(file3);
 
     FSUtils.logFileSystemState(fs, rootDir, LOG);
-    assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
+    nonSnapshotFiles = cache.getUnreferencedFiles(
+            Arrays.asList(FSUtils.listStatus(fs, family))
+    );
+    assertFalse("Cache didn't find new file:" + file3, Iterables.contains(nonSnapshotFiles, file3));
   }
 
   @Test
   public void testSnapshotTempDirReload() throws IOException {
     long period = Long.MAX_VALUE;
-    // This doesn't refresh cache until we invoke it explicitly
     Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
         SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
@@ -231,13 +309,13 @@ public void testSnapshotTempDirReload() throws IOException {
     Path snapshot1 = new Path(snapshotDir, "snapshot1");
     Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
     fs.createNewFile(file1);
-    assertTrue(cache.contains(file1.getName()));
+    assertTrue(cache.getSnapshotsInProgress().contains(file1.getName()));
 
     // Add another snapshot
     Path snapshot2 = new Path(snapshotDir, "snapshot2");
     Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
     fs.createNewFile(file2);
-    assertTrue(cache.contains(file2.getName()));
+    assertTrue(cache.getSnapshotsInProgress().contains((file2.getName())));
   }
 
   class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {

From 812be0ca43f8ea0b6ce8e78249b170afa04b5767 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 31 Jul 2014 16:03:12 -0700
Subject: [PATCH 1444/1540] [0.94] port HBASE-11217 Race between
 SplitLogManager task creation + TimeoutMonitor. (LarsH - original patch by
 Enis)

---
 .../hadoop/hbase/master/SplitLogManager.java  | 43 +++++++++++-------
 .../hbase/master/TestSplitLogManager.java     | 45 -------------------
 2 files changed, 28 insertions(+), 60 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 161a7310389c..31981e7f5057 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -111,7 +111,7 @@ public class SplitLogManager extends ZooKeeperListener {
   private long resubmit_threshold;
   private long timeout;
   private long unassignedTimeout;
-  private long lastNodeCreateTime = Long.MAX_VALUE;
+  private long lastTaskCreateTime = Long.MAX_VALUE;
   public boolean ignoreZKDeleteForTesting = false;
 
   private final ConcurrentMap<String, Task> tasks =
@@ -229,7 +229,7 @@ private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws I
    * @throws IOException
    *             if there was an error while splitting any log file
    * @return cumulative size of the logfiles split
-   * @throws IOException 
+   * @throws IOException
    */
   public long splitLogDistributed(final Path logDir) throws IOException {
     List<Path> logDirs = new ArrayList<Path>();
@@ -307,7 +307,7 @@ public long splitLogDistributed(final List<Path> logDirs, PathFilter filter)
       } catch (IOException ioe) {
         FileStatus[] files = fs.listStatus(logDir);
         if (files != null && files.length > 0) {
-          LOG.warn("returning success without actually splitting and " + 
+          LOG.warn("returning success without actually splitting and " +
               "deleting all the log files in path " + logDir);
         } else {
           LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);
@@ -325,7 +325,7 @@ public long splitLogDistributed(final List<Path> logDirs, PathFilter filter)
 
   /**
    * Add a task entry to splitlog znode if it is not already there.
-   * 
+   *
    * @param taskname the path of the log to be split
    * @param batch the batch this task belongs to
    * @return true if a new entry is created, false if it is already there.
@@ -333,6 +333,7 @@ public long splitLogDistributed(final List<Path> logDirs, PathFilter filter)
   boolean enqueueSplitTask(String taskname, TaskBatch batch) {
     tot_mgr_log_split_start.incrementAndGet();
     String path = ZKSplitLog.getEncodedNodeName(watcher, taskname);
+    lastTaskCreateTime = EnvironmentEdgeManager.currentTimeMillis();
     Task oldtask = createTaskIfAbsent(path, batch);
     if (oldtask == null) {
       // publish the task in zk
@@ -461,7 +462,6 @@ private void createNode(String path, Long retry_count) {
   }
 
   private void createNodeSuccess(String path) {
-    lastNodeCreateTime = EnvironmentEdgeManager.currentTimeMillis();
     LOG.debug("put up splitlog task at znode " + path);
     getDataSetWatch(path, zkretries);
   }
@@ -476,7 +476,7 @@ private void createNodeFailure(String path) {
   private void getDataSetWatch(String path, Long retry_count) {
     this.watcher.getRecoverableZooKeeper().getZooKeeper().
         getData(path, this.watcher,
-        new GetDataAsyncCallback(), retry_count);
+        new GetDataAsyncCallback(true), retry_count);
     tot_mgr_get_data_queued.incrementAndGet();
   }
 
@@ -484,7 +484,7 @@ private void tryGetDataSetWatch(String path) {
     // A negative retry count will lead to ignoring all error processing.
     this.watcher.getRecoverableZooKeeper().getZooKeeper().
         getData(path, this.watcher,
-        new GetDataAsyncCallback(), new Long(-1) /* retry count */);
+        new GetDataAsyncCallback(false), new Long(-1) /* retry count */);
     tot_mgr_get_data_queued.incrementAndGet();
   }
 
@@ -726,7 +726,7 @@ private void deleteNodeFailure(String path) {
   /**
    * signal the workers that a task was resubmitted by creating the
    * RESCAN node.
-   * @throws KeeperException 
+   * @throws KeeperException
    */
   private void createRescanNode(long retries) {
     // The RESCAN node will be deleted almost immediately by the
@@ -736,6 +736,7 @@ private void createRescanNode(long retries) {
     // might miss the watch-trigger that creation of RESCAN node provides.
     // Since the TimeoutMonitor will keep resubmitting UNASSIGNED tasks
     // therefore this behavior is safe.
+    lastTaskCreateTime = EnvironmentEdgeManager.currentTimeMillis();
     this.watcher.getRecoverableZooKeeper().getZooKeeper().
       create(ZKSplitLog.getRescanNode(watcher),
         TaskState.TASK_DONE.get(serverName), Ids.OPEN_ACL_UNSAFE,
@@ -744,7 +745,6 @@ private void createRescanNode(long retries) {
   }
 
   private void createRescanSuccess(String path) {
-    lastNodeCreateTime = EnvironmentEdgeManager.currentTimeMillis();
     tot_mgr_rescan.incrementAndGet();
     getDataSetWatch(path, zkretries);
   }
@@ -1047,7 +1047,7 @@ protected void chore() {
       // master should spawn both a manager and a worker thread to guarantee
       // that there is always one worker in the system
       if (tot > 0 && !found_assigned_task &&
-          ((EnvironmentEdgeManager.currentTimeMillis() - lastNodeCreateTime) >
+          ((EnvironmentEdgeManager.currentTimeMillis() - lastTaskCreateTime) >
           unassignedTimeout)) {
         for (Map.Entry<String, Task> e : tasks.entrySet()) {
           String path = e.getKey();
@@ -1126,6 +1126,17 @@ public void processResult(int rc, String path, Object ctx, String name) {
    */
   class GetDataAsyncCallback implements AsyncCallback.DataCallback {
     private final Log LOG = LogFactory.getLog(GetDataAsyncCallback.class);
+    private boolean completeTaskOnNoNode;
+
+    /**
+     * @param completeTaskOnNoNode Complete the task if the znode cannot be found.
+     * Since in-memory task creation and znode creation are not atomic, there might be
+     * a race where there is a task in memory but the znode is not created yet (TimeoutMonitor).
+     * In this case completeTaskOnNoNode should be set to false. See HBASE-11217.
+     */
+    public GetDataAsyncCallback(boolean completeTaskOnNoNode) {
+      this.completeTaskOnNoNode = completeTaskOnNoNode;
+    }
 
     @Override
     public void processResult(int rc, String path, Object ctx, byte[] data,
@@ -1137,11 +1148,13 @@ public void processResult(int rc, String path, Object ctx, byte[] data,
         }
         if (rc == KeeperException.Code.NONODE.intValue()) {
           tot_mgr_get_data_nonode.incrementAndGet();
-          // The task znode has been deleted. Must be some pending delete
-          // that deleted the task. Assume success because a task-znode is
-          // is only deleted after TaskFinisher is successful.
           LOG.warn("task znode " + path + " vanished.");
-          getDataSetWatchSuccess(path, null, Integer.MIN_VALUE);
+          if (completeTaskOnNoNode) {
+            // The task znode has been deleted. Must be some pending delete
+            // that deleted the task. Assume success because a task-znode is
+            // is only deleted after TaskFinisher is successful.
+            getDataSetWatchSuccess(path, null, Integer.MIN_VALUE);
+          }
           return;
         }
         Long retry_count = (Long) ctx;
@@ -1284,7 +1297,7 @@ enum TerminationStatus {
     TerminationStatus(String msg) {
       statusMsg = msg;
     }
-    
+
     @Override
     public String toString() {
       return statusMsg;
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index ff87e3cfb9c1..a60bc9533bfa 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -439,51 +439,6 @@ public void testEmptyLogDir() throws Exception {
     assertFalse(fs.exists(emptyLogDirPath));
   }
 
-  @Test(timeout=45000)
-  public void testVanishingTaskZNode() throws Exception {
-    LOG.info("testVanishingTaskZNode");
-    conf.setInt("hbase.splitlog.manager.unassigned.timeout", 0);
-    conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 1000);
-    slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null);
-    slm.finishInitialization();
-    FileSystem fs = TEST_UTIL.getTestFileSystem();
-    final Path logDir = new Path(fs.getWorkingDirectory(),
-        UUID.randomUUID().toString());
-    fs.mkdirs(logDir);
-    Thread thread = null;
-    try {
-      Path logFile = new Path(logDir, UUID.randomUUID().toString());
-      fs.createNewFile(logFile);
-      thread = new Thread() {
-        public void run() {
-          try {
-            // this call will block because there are no SplitLogWorkers,
-            // until the task znode is deleted below. Then the call will
-            // complete successfully, assuming the log is split.
-            slm.splitLogDistributed(logDir);
-          } catch (Exception e) {
-            LOG.warn("splitLogDistributed failed", e);
-          }
-        }
-      };
-      thread.start();
-      waitForCounter(tot_mgr_node_create_result, 0, 1, 10000);
-      String znode = ZKSplitLog.getEncodedNodeName(zkw, logFile.toString());
-      // remove the task znode, to finish the distributed log splitting
-      ZKUtil.deleteNode(zkw, znode);
-      waitForCounter(tot_mgr_get_data_nonode, 0, 1, 30000);
-      waitForCounter(tot_mgr_log_split_batch_success, 0, 1, to/2);
-      assertTrue(fs.exists(logFile));
-    } finally {
-      if (thread != null) {
-        // interrupt the thread in case the test fails in the middle.
-        // it has no effect if the thread is already terminated.
-        thread.interrupt();
-      }
-      fs.delete(logDir, true);
-    }
-  }
-
   @Test
   public void testWorkerCrash() throws Exception {
     conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLog.DEFAULT_MAX_RESUBMIT);

From 8bf1a4d175a546b328319941a78b6eb0a8213013 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 31 Jul 2014 16:55:06 -0700
Subject: [PATCH 1445/1540] mutateRowsWithLocks might require
 updatesLock.readLock with waitTime=0. (cuijianwei)

---
 src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 31959b44d4ff..eaa02d126062 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5110,7 +5110,7 @@ public void mutateRowsWithLocks(Collection<Mutation> mutations,
       }
 
       // 3. acquire the region lock
-      lock(this.updatesLock.readLock(), acquiredLocks.size());
+      lock(this.updatesLock.readLock(), acquiredLocks.size() == 0 ? 1 : acquiredLocks.size());
       locked = true;
 
       // 4. Get a mvcc write number

From b3b224ddca3530139c0685fd5c52e37368b428df Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 31 Jul 2014 16:59:35 -0700
Subject: [PATCH 1446/1540] CHANGES.txt pom.xml for 0.94.22RC0

---
 CHANGES.txt | 21 +++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 503f9749ee42..1fafc1e59fa7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,25 @@
 HBase Change Log
+Release 0.94.22 - 07/31/2014
+Bug
+
+    [HBASE-10645] - Fix wrapping of Requests Counts Regionserver level metrics
+    [HBASE-11360] - SnapshotFileCache causes too many cache refreshes
+    [HBASE-11479] - SecureConnection can't be closed when SecureClient is stopping because InterruptedException won't be caught in SecureClient#setupIOstreams()
+    [HBASE-11496] - HBASE-9745 broke cygwin CLASSPATH translation
+    [HBASE-11552] - Read/Write requests count metric value is too short
+    [HBASE-11565] - Stale connection could stay for a while
+    [HBASE-11633] - [0.94] port HBASE-11217 Race between SplitLogManager task creation + TimeoutMonitor
+
+Improvement
+
+    [HBASE-2217] - VM OPTS for shell only
+    [HBASE-7910] - Dont use reflection for security
+    [HBASE-11444] - Remove use of reflection for User#getShortName
+    [HBASE-11450] - Improve file size info in SnapshotInfo tool
+    [HBASE-11480] - ClientScanner might not close the HConnection created in construction
+    [HBASE-11623] - mutateRowsWithLocks might require updatesLock.readLock with waitTime=0
+
+
 Release 0.94.21 - 06/27/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index 14cbc26ab30d..95970a88a52a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.22-SNAPSHOT</version>
+  <version>0.94.22</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From cce7a9770cb8353ccd9faab9fb5cb44ef155101e Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 1 Aug 2014 21:01:55 -0700
Subject: [PATCH 1447/1540] HBASE-11652 Port HBASE-3270 and HBASE-11650 to 0.94
 - create cluster id and version file in a tmp location and move it into
 place. (LarsH, original patches by Andrew Purtell)

---
 .../org/apache/hadoop/hbase/util/FSUtils.java  | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index fe30f4c1b1d3..24dde3011db6 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -470,13 +470,17 @@ public static int getDefaultBufferSize(final FileSystem fs) {
   public static void setVersion(FileSystem fs, Path rootdir, String version,
       int wait, int retries) throws IOException {
     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
+    Path tmpFile = new Path(new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY), HConstants.VERSION_FILE_NAME);
     while (true) {
       try {
-        FSDataOutputStream s = fs.create(versionFile);
+        FSDataOutputStream s = fs.create(tmpFile);
         s.writeUTF(version);
+        s.close();
+        if (!fs.rename(tmpFile, versionFile)) {
+          throw new IOException("Unable to move temp version file to " + versionFile);
+        }
         LOG.debug("Created version file at " + rootdir.toString() +
             " set its version at:" + version);
-        s.close();
         return;
       } catch (IOException e) {
         if (retries > 0) {
@@ -567,14 +571,18 @@ public static String getClusterId(FileSystem fs, Path rootdir)
    */
   public static void setClusterId(FileSystem fs, Path rootdir, String clusterId,
       int wait) throws IOException {
+    Path idFfile = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
+    Path tmpFile = new Path(new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY), HConstants.CLUSTER_ID_FILE_NAME);
     while (true) {
       try {
-        Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
-        FSDataOutputStream s = fs.create(filePath);
+        FSDataOutputStream s = fs.create(tmpFile);
         s.writeUTF(clusterId);
         s.close();
+        if (!fs.rename(tmpFile, idFfile)) {
+          throw new IOException("Unable to move temp version file to " + idFfile);
+        }
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Created cluster ID file at " + filePath.toString() +
+          LOG.debug("Created cluster ID file at " + idFfile.toString() +
               " with ID: " + clusterId);
         }
         return;

From e127626d3bfcf647910086f809cbdae2afacb1e5 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 5 Aug 2014 10:33:36 -0700
Subject: [PATCH 1448/1540] HBASE-11641
 TestDistributedLogSplitting.testMasterStartsUpWithLogSplittingWork fails
 frequently.

---
 .../hadoop/hbase/master/TestDistributedLogSplitting.java       | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index d397b6431fa0..d4237fc0bc21 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -132,11 +132,10 @@ private void startCluster(int num_master, int num_rs, Configuration inConf) thro
     conf.getLong("hbase.splitlog.max.resubmit", 0);
     // Make the failure test faster
     conf.setInt("zookeeper.recovery.retry", 0);
-    TEST_UTIL.shutdownMiniHBaseCluster();
     TEST_UTIL = new HBaseTestingUtility(conf);
     TEST_UTIL.setDFSCluster(dfsCluster);
     TEST_UTIL.setZkCluster(zkCluster);
-    TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, num_rs);
+    TEST_UTIL.startMiniHBaseCluster(num_master, num_rs);
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();

From c51e19afec1925de7769c24e0dbd2bfcf90a184e Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 6 Aug 2014 11:36:01 -0700
Subject: [PATCH 1449/1540] HBASE-11667 Comment ClientScanner logic for NSREs.

---
 .../java/org/apache/hadoop/hbase/client/ClientScanner.java | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index be93d586043e..7e72d57a9783 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -304,7 +304,14 @@ public Result next() throws IOException {
             // Else, its signal from depths of ScannerCallable that we got an
             // NSRE on a next and that we need to reset the scanner.
             if (this.lastResult != null) {
+              // The region has moved. We need to open a brand new scanner at
+              // the new location.
+              // Reset the startRow to the row we've seen last so that the new
+              // scanner starts at the correct row. Otherwise we may see previously
+              // returned rows again.
+              // (ScannerCallable by now has "relocated" the correct region)
               this.scan.setStartRow(this.lastResult.getRow());
+
               // Skip first row returned.  We already let it out on previous
               // invocation.
               skipFirst = true;

From 46c26949ca5aa7b1a34f06778b058f1e3ad72364 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Thu, 7 Aug 2014 15:20:02 -0700
Subject: [PATCH 1450/1540] HBASE-11690 Backport HBASE-5934 (Add the ability
 for Performance Evaluation to set the table compression) to 0.94

---
 .../hadoop/hbase/PerformanceEvaluation.java   | 42 +++++++++++++++++--
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 9284b4e01629..37ad8e217944 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -59,6 +59,7 @@
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -101,6 +102,7 @@
 public class PerformanceEvaluation {
   protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
 
+  public static final byte[] COMPRESSION = Bytes.toBytes("NONE");
   public static final byte[] TABLE_NAME = Bytes.toBytes("TestTable");
   public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
   public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data");
@@ -122,6 +124,8 @@ public class PerformanceEvaluation {
   private boolean nomapred = false;
   private int N = 1;
   private int R = ROWS_PER_GB;
+  private byte[] tableName = TABLE_NAME;
+  private Compression.Algorithm compression = Compression.Algorithm.NONE;
   private float sampleRate = 1.0f;
   private boolean flushCommits = true;
   private boolean reportLatency = false;
@@ -511,8 +515,9 @@ private boolean checkTable(HBaseAdmin admin) throws IOException {
 
   protected HTableDescriptor getTableDescriptor() {
     if (TABLE_DESCRIPTOR == null) {
-      TABLE_DESCRIPTOR = new HTableDescriptor(TABLE_NAME);
+      TABLE_DESCRIPTOR = new HTableDescriptor(tableName);
       HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+      family.setCompressionType(compression);
       TABLE_DESCRIPTOR.addFamily(family);
     }
     return TABLE_DESCRIPTOR;
@@ -1185,7 +1190,23 @@ private static String calculateMbps(int rows, long timeMs) {
    */
   public static byte[] generateValue(final Random r) {
     byte [] b = new byte [VALUE_LENGTH];
-    r.nextBytes(b);
+    int i = 0;
+
+    for(i = 0; i < (ROW_LENGTH-8); i += 8) {
+      b[i] = (byte) (65 + r.nextInt(26));
+      b[i+1] = b[i];
+      b[i+2] = b[i];
+      b[i+3] = b[i];
+      b[i+4] = b[i];
+      b[i+5] = b[i];
+      b[i+6] = b[i];
+      b[i+7] = b[i];
+    }
+
+    byte a = (byte) (65 + r.nextInt(26));
+    for(; i < ROW_LENGTH; i++) {
+      b[i] = a;
+    }
     return b;
   }
 
@@ -1294,13 +1315,16 @@ protected void printUsage(final String message) {
       System.err.println(message);
     }
     System.err.println("Usage: java " + this.getClass().getName() + " \\");
-    System.err.println("  [--miniCluster] [--nomapred] [--rows=ROWS] <command> <nclients>");
+    System.err.println("  [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\");
+    System.err.println("  [--compress=TYPE] <command> <nclients>");
     System.err.println();
     System.err.println("Options:");
     System.err.println(" miniCluster     Run the test on an HBaseMiniCluster");
     System.err.println(" nomapred        Run multiple clients using threads " +
       "(rather than use mapreduce)");
     System.err.println(" rows            Rows each client runs. Default: One million");
+    System.err.println(" table           Alternate table name. Default: 'TestTable'");
+    System.err.println(" compress        Compression type to use (GZ, LZO, ...). Default: 'NONE'");
     System.err.println(" sampleRate      Execute test on a sample of total " +
       "rows. Only supported by randomRead. Default: 1.0");
     System.err.println(" flushCommits    Used to determine if the test should flush the table. " +
@@ -1380,6 +1404,18 @@ public int doCommandLine(final String[] args) {
           continue;
         }
 
+        final String table = "--table=";
+        if (cmd.startsWith(table)) {
+          this.tableName = Bytes.toBytes(cmd.substring(table.length()));
+          continue;
+        }
+
+        final String compress = "--compress=";
+        if (cmd.startsWith(compress)) {
+          this.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length()));
+          continue;
+        }
+
         final String flushCommits = "--flushCommits=";
         if (cmd.startsWith(flushCommits)) {
           this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));

From 9a8142d11f39f37a24486f77ac989a483643a1ba Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Thu, 7 Aug 2014 15:22:42 -0700
Subject: [PATCH 1451/1540] HBASE-11691 Backport HBASE-7156 (Add Data Block
 Encoding and -D opts to Performance Evaluation) to 0.94

---
 .../hadoop/hbase/PerformanceEvaluation.java   | 56 ++++++++++++-------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 37ad8e217944..cca82103bade 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -42,6 +42,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -59,6 +60,7 @@
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -82,6 +84,8 @@
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
 import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * Script used evaluating HBase performance and scalability.  Runs a HBase
@@ -99,7 +103,7 @@
  * <p>If number of clients > 1, we start up a MapReduce job. Each map task
  * runs an individual client. Each client does about 1GB of data.
  */
-public class PerformanceEvaluation {
+public class PerformanceEvaluation extends Configured implements Tool {
   protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
 
   public static final byte[] COMPRESSION = Bytes.toBytes("NONE");
@@ -119,13 +123,13 @@ public class PerformanceEvaluation {
   protected static HTableDescriptor TABLE_DESCRIPTOR;
   protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
 
-  volatile Configuration conf;
   private boolean miniCluster = false;
   private boolean nomapred = false;
   private int N = 1;
   private int R = ROWS_PER_GB;
   private byte[] tableName = TABLE_NAME;
   private Compression.Algorithm compression = Compression.Algorithm.NONE;
+  private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
   private float sampleRate = 1.0f;
   private boolean flushCommits = true;
   private boolean reportLatency = false;
@@ -158,10 +162,10 @@ protected static enum Counter {
 
   /**
    * Constructor
-   * @param c Configuration object
+   * @param conf Configuration object
    */
-  public PerformanceEvaluation(final Configuration c) {
-    this.conf = c;
+  public PerformanceEvaluation(final Configuration conf) {
+    super(conf);
 
     addCommandDescriptor(RandomReadTest.class, "randomRead",
         "Run random read test");
@@ -517,6 +521,7 @@ protected HTableDescriptor getTableDescriptor() {
     if (TABLE_DESCRIPTOR == null) {
       TABLE_DESCRIPTOR = new HTableDescriptor(tableName);
       HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+      family.setDataBlockEncoding(blockEncoding);
       family.setCompressionType(compression);
       TABLE_DESCRIPTOR.addFamily(family);
     }
@@ -549,7 +554,7 @@ protected  byte[][] getSplits() {
    */
   private void runNIsMoreThanOne(final Class<? extends Test> cmd)
   throws IOException, InterruptedException, ClassNotFoundException {
-    checkTable(new HBaseAdmin(conf));
+    checkTable(new HBaseAdmin(getConf()));
     if (this.nomapred) {
       doMultipleClients(cmd);
     } else {
@@ -570,7 +575,7 @@ private void doMultipleClients(final Class<? extends Test> cmd) throws IOExcepti
         @Override
         public void run() {
           super.run();
-          PerformanceEvaluation pe = new PerformanceEvaluation(conf);
+          PerformanceEvaluation pe = new PerformanceEvaluation(getConf());
           int index = Integer.parseInt(getName());
           try {
             long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
@@ -612,10 +617,11 @@ public void setStatus(final String msg) throws IOException {
    */
   private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
         InterruptedException, ClassNotFoundException {
-    Path inputDir = writeInputFile(this.conf);
-    this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
-    this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
-    Job job = new Job(this.conf);
+    Configuration conf = getConf();
+    Path inputDir = writeInputFile(conf);
+    conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
+    conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
+    Job job = new Job(conf);
     job.setJarByClass(PerformanceEvaluation.class);
     job.setJobName("HBase Performance Evaluation");
 
@@ -1229,7 +1235,7 @@ long runOneClient(final Class<? extends Test> cmd, final int startRow,
     try {
       Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
           Configuration.class, TestOptions.class, Status.class);
-      t = constructor.newInstance(this.conf, options, status);
+      t = constructor.newInstance(getConf(), options, status);
     } catch (NoSuchMethodException e) {
       throw new IllegalArgumentException("Invalid command class: " +
           cmd.getName() + ".  It does not provide a constructor as described by" +
@@ -1255,7 +1261,7 @@ public void setStatus(String msg) throws IOException {
 
     HBaseAdmin admin = null;
     try {
-      admin = new HBaseAdmin(this.conf);
+      admin = new HBaseAdmin(getConf());
       checkTable(admin);
       runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
         this.writeToWAL, this.writeToWAL, status);
@@ -1271,6 +1277,7 @@ private void runTest(final Class<? extends Test> cmd) throws IOException,
     MiniHBaseCluster hbaseMiniCluster = null;
     MiniDFSCluster dfsCluster = null;
     MiniZooKeeperCluster zooKeeperCluster = null;
+    Configuration conf = getConf();
     if (this.miniCluster) {
       dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
       zooKeeperCluster = new MiniZooKeeperCluster();
@@ -1285,7 +1292,7 @@ private void runTest(final Class<? extends Test> cmd) throws IOException,
       conf.set(HConstants.HBASE_DIR, parentdir.toString());
       fs.mkdirs(parentdir);
       FSUtils.setVersion(fs, parentdir);
-      hbaseMiniCluster = new MiniHBaseCluster(this.conf, N);
+      hbaseMiniCluster = new MiniHBaseCluster(conf, N);
     }
 
     try {
@@ -1316,7 +1323,7 @@ protected void printUsage(final String message) {
     }
     System.err.println("Usage: java " + this.getClass().getName() + " \\");
     System.err.println("  [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\");
-    System.err.println("  [--compress=TYPE] <command> <nclients>");
+    System.err.println("  [--compress=TYPE] [--blockEncoding=TYPE] [-D<property=value>]* <command> <nclients>");
     System.err.println();
     System.err.println("Options:");
     System.err.println(" miniCluster     Run the test on an HBaseMiniCluster");
@@ -1335,6 +1342,11 @@ protected void printUsage(final String message) {
     System.err.println(" latency         Set to report operation latencies. " +
       "Currently only supported by randomRead test. Default: False");
     System.err.println();
+    System.err.println(" Note: -D properties will be applied to the conf used. ");
+    System.err.println("  For example: ");
+    System.err.println("   -Dmapred.output.compress=true");
+    System.err.println("   -Dmapreduce.task.timeout=60000");
+    System.err.println();
     System.err.println("Command:");
     for (CmdDescriptor command : commands.values()) {
       System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
@@ -1362,7 +1374,7 @@ private void getArgs(final int start, final String[] args) {
     this.R = this.R * N;
   }
 
-  public int doCommandLine(final String[] args) {
+  public int run(String[] args) throws Exception {
     // Process command-line args. TODO: Better cmd-line processing
     // (but hopefully something not as painful as cli options).
     int errCode = -1;
@@ -1416,6 +1428,12 @@ public int doCommandLine(final String[] args) {
           continue;
         }
 
+        final String blockEncoding = "--blockEncoding=";
+        if (cmd.startsWith(blockEncoding)) {
+          this.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
+          continue;
+        }
+
         final String flushCommits = "--flushCommits=";
         if (cmd.startsWith(flushCommits)) {
           this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
@@ -1463,8 +1481,8 @@ private Class<? extends Test> determineCommandClass(String cmd) {
     return descriptor != null ? descriptor.getCmdClass() : null;
   }
 
-  public static void main(final String[] args) {
-    Configuration c = HBaseConfiguration.create();
-    System.exit(new PerformanceEvaluation(c).doCommandLine(args));
+  public static void main(final String[] args) throws Exception {
+    int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
+    System.exit(res);
   }
 }

From 18ce81375d875461431c8868600301a6304f48b1 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Thu, 7 Aug 2014 15:25:05 -0700
Subject: [PATCH 1452/1540] HBASE-11693 Backport HBASE-11026 (Provide option to
 filter out all rows in PerformanceEvaluation tool) to 0.94

---
 .../hadoop/hbase/PerformanceEvaluation.java   | 93 +++++++++++++++----
 .../hadoop/hbase/filter/FilterAllFilter.java  | 49 ++++++++++
 2 files changed, 123 insertions(+), 19 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java

diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index cca82103bade..6d73e45ba30b 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -54,9 +54,10 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterAllFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.PageFilter;
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
@@ -132,8 +133,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
   private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
   private float sampleRate = 1.0f;
   private boolean flushCommits = true;
-  private boolean reportLatency = false;
   private boolean writeToWAL = true;
+  private boolean reportLatency = false;
+  private boolean filterAll = false;
   private int presplitRegions = 0;
 
   private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
@@ -147,7 +149,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
         "clients=(\\d+),\\s+" +
         "flushCommits=(\\w+),\\s+" +
         "writeToWAL=(\\w+),\\s+" +
-        "reportLatency=(\\w+)");
+        "reportLatency=(\\w+),\\s+" +
+        "filterAll=(\\w+)");
 
   /**
    * Enum for map metrics.  Keep it out here rather than inside in the Map
@@ -225,11 +228,13 @@ public static class PeInputSplit extends InputSplit implements Writable {
     private boolean flushCommits = false;
     private boolean writeToWAL = true;
     private boolean reportLatency = false;
+    private boolean filterAll = false;
 
     public PeInputSplit() {}
 
     public PeInputSplit(int startRow, int rows, int totalRows, float sampleRate,
-        int clients, boolean flushCommits, boolean writeToWAL, boolean reportLatency) {
+        int clients, boolean flushCommits, boolean writeToWAL, boolean reportLatency,
+        boolean filterAll) {
       this.startRow = startRow;
       this.rows = rows;
       this.totalRows = totalRows;
@@ -238,6 +243,7 @@ public PeInputSplit(int startRow, int rows, int totalRows, float sampleRate,
       this.flushCommits = flushCommits;
       this.writeToWAL = writeToWAL;
       this.reportLatency = reportLatency;
+      this.filterAll = filterAll;
     }
 
     @Override
@@ -250,6 +256,7 @@ public void readFields(DataInput in) throws IOException {
       this.flushCommits = in.readBoolean();
       this.writeToWAL = in.readBoolean();
       this.reportLatency = in.readBoolean();
+      this.filterAll = in.readBoolean();
     }
 
     @Override
@@ -262,6 +269,7 @@ public void write(DataOutput out) throws IOException {
       out.writeBoolean(flushCommits);
       out.writeBoolean(writeToWAL);
       out.writeBoolean(reportLatency);
+      out.writeBoolean(filterAll);
     }
 
     @Override
@@ -305,6 +313,10 @@ public boolean isWriteToWAL() {
     public boolean isReportLatency() {
       return reportLatency;
     }
+
+    public boolean isFilterAll() {
+      return filterAll;
+    }
   }
 
   /**
@@ -340,6 +352,7 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
             boolean flushCommits = Boolean.parseBoolean(m.group(6));
             boolean writeToWAL = Boolean.parseBoolean(m.group(7));
             boolean reportLatency = Boolean.parseBoolean(m.group(8));
+            boolean filterAll = Boolean.parseBoolean(m.group(9));
 
             LOG.debug("split["+ splitList.size() + "] " +
                      " startRow=" + startRow +
@@ -349,11 +362,12 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
                      " clients=" + clients +
                      " flushCommits=" + flushCommits +
                      " writeToWAL=" + writeToWAL +
-                     " reportLatency=" + reportLatency);
+                     " reportLatency=" + reportLatency +
+                     " filterAll=" + filterAll);
 
             PeInputSplit newSplit =
               new PeInputSplit(startRow, rows, totalRows, sampleRate, clients,
-                flushCommits, writeToWAL, reportLatency);
+                flushCommits, writeToWAL, reportLatency, filterAll);
             splitList.add(newSplit);
           }
         }
@@ -475,7 +489,7 @@ public void setStatus(String msg) {
       long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(),
         value.getRows(), value.getTotalRows(), value.getSampleRate(),
         value.isFlushCommits(), value.isWriteToWAL(), value.isReportLatency(),
-        status);
+        value.isFilterAll(), status);
       // Collect how much time the thing took. Report as map output and
       // to the ELAPSED_TIME counter.
       context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -580,7 +594,7 @@ public void run() {
           try {
             long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
                perClientRows, R, sampleRate, flushCommits, writeToWAL,
-               reportLatency, new Status() {
+               reportLatency, filterAll, new Status() {
                   public void setStatus(final String msg) throws IOException {
                     LOG.info("client-" + getName() + " " + msg);
                   }
@@ -679,7 +693,8 @@ private Path writeInputFile(final Configuration c) throws IOException {
           ", clients=" + this.N +
           ", flushCommits=" + this.flushCommits +
           ", writeToWAL=" + this.writeToWAL +
-          ", reportLatency=" + this.reportLatency;
+          ", reportLatency=" + this.reportLatency +
+          ", filterAll=" + this.filterAll;
           int hash = h.hash(Bytes.toBytes(s));
           m.put(hash, s);
         }
@@ -733,11 +748,13 @@ static class TestOptions {
     private boolean flushCommits;
     private boolean writeToWAL = true;
     private boolean reportLatency;
+    private boolean filterAll;
 
     TestOptions() {}
 
     TestOptions(int startRow, int perClientRunRows, int totalRows, float sampleRate,
-        byte[] tableName, boolean flushCommits, boolean writeToWAL, boolean reportLatency) {
+        byte[] tableName, boolean flushCommits, boolean writeToWAL, boolean reportLatency,
+        boolean filterAll) {
       this.startRow = startRow;
       this.perClientRunRows = perClientRunRows;
       this.totalRows = totalRows;
@@ -746,6 +763,7 @@ static class TestOptions {
       this.flushCommits = flushCommits;
       this.writeToWAL = writeToWAL;
       this.reportLatency = reportLatency;
+      this.filterAll = filterAll;
     }
 
     public int getStartRow() {
@@ -779,6 +797,10 @@ public boolean isWriteToWAL() {
     public boolean isReportLatency() {
       return reportLatency;
     }
+
+    public boolean isFilterAll() {
+      return filterAll;
+    }
   }
 
   /*
@@ -807,6 +829,7 @@ private static long nextRandomSeed() {
     protected boolean flushCommits;
     protected boolean writeToWAL;
     protected boolean reportlatency;
+    protected boolean filterAll;
 
     /**
      * Note that all subclasses of this class must provide a public contructor
@@ -825,6 +848,7 @@ private static long nextRandomSeed() {
       this.flushCommits = options.isFlushCommits();
       this.writeToWAL = options.isWriteToWAL();
       this.reportlatency = options.isReportLatency();
+      this.filterAll = options.isFilterAll();
     }
 
     private String generateStatus(final int sr, final int i, final int lr) {
@@ -900,7 +924,12 @@ static class RandomSeekScanTest extends Test {
     void testRow(final int i) throws IOException {
       Scan scan = new Scan(getRandomRow(this.rand, this.totalRows));
       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
-      scan.setFilter(new WhileMatchFilter(new PageFilter(120)));
+      FilterList list = new FilterList();
+      if (this.filterAll) {
+        list.addFilter(new FilterAllFilter());
+      }
+      list.addFilter(new WhileMatchFilter(new PageFilter(120)));
+      scan.setFilter(list);
       ResultScanner s = this.table.getScanner(scan);
       for (Result rr; (rr = s.next()) != null;) ;
       s.close();
@@ -925,6 +954,9 @@ void testRow(final int i) throws IOException {
       Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
       Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+      if (this.filterAll) {
+        scan.setFilter(new FilterAllFilter());
+      }
       ResultScanner s = this.table.getScanner(scan);
       int count = 0;
       for (Result rr; (rr = s.next()) != null;) {
@@ -1022,6 +1054,9 @@ void testRow(final int i) throws IOException {
       if (i % everyN == 0) {
         Get get = new Get(getRandomRow(this.rand, this.totalRows));
         get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+        if (this.filterAll) {
+          get.setFilter(new FilterAllFilter());
+        }
         long start = System.nanoTime();
         this.table.get(get);
         if (this.reportLatency) {
@@ -1087,6 +1122,9 @@ void testRow(final int i) throws IOException {
       if (this.testScanner == null) {
         Scan scan = new Scan(format(this.startRow));
         scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+        if (this.filterAll) {
+          scan.setFilter(new FilterAllFilter());
+        }
         this.testScanner = table.getScanner(scan);
       }
       testScanner.next();
@@ -1103,6 +1141,9 @@ static class SequentialReadTest extends Test {
     void testRow(final int i) throws IOException {
       Get get = new Get(format(i));
       get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+      if (this.filterAll) {
+        get.setFilter(new FilterAllFilter());
+      }
       table.get(get);
     }
   }
@@ -1144,13 +1185,17 @@ void testRow(int i) throws IOException {
     }
 
     protected Scan constructScan(byte[] valuePrefix) throws IOException {
-      Filter filter = new SingleColumnValueFilter(
-          FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL,
-          new BinaryComparator(valuePrefix)
-      );
       Scan scan = new Scan();
       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
-      scan.setFilter(filter);
+      FilterList list = new FilterList();
+      list.addFilter(new SingleColumnValueFilter(
+          FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL,
+          new BinaryComparator(valuePrefix)
+      ));
+      if (this.filterAll) {
+        list.addFilter(new FilterAllFilter());
+      }
+      scan.setFilter(list);
       return scan;
     }
   }
@@ -1223,7 +1268,7 @@ public static byte[] generateValue(final Random r) {
   long runOneClient(final Class<? extends Test> cmd, final int startRow,
       final int perClientRunRows, final int totalRows, final float sampleRate,
       boolean flushCommits, boolean writeToWAL, boolean reportLatency,
-      final Status status)
+      boolean filterAll, final Status status)
   throws IOException {
     status.setStatus("Start " + cmd + " at offset " + startRow + " for " +
       perClientRunRows + " rows");
@@ -1231,7 +1276,8 @@ long runOneClient(final Class<? extends Test> cmd, final int startRow,
 
     Test t = null;
     TestOptions options = new TestOptions(startRow, perClientRunRows, totalRows,
-      sampleRate, getTableDescriptor().getName(), flushCommits, writeToWAL, reportLatency);
+      sampleRate, getTableDescriptor().getName(), flushCommits, writeToWAL, reportLatency,
+      filterAll);
     try {
       Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
           Configuration.class, TestOptions.class, Status.class);
@@ -1264,7 +1310,7 @@ public void setStatus(String msg) throws IOException {
       admin = new HBaseAdmin(getConf());
       checkTable(admin);
       runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
-        this.writeToWAL, this.writeToWAL, status);
+        this.writeToWAL, this.reportLatency, this.filterAll, status);
     } catch (Exception e) {
       LOG.error("Failed", e);
     } finally {
@@ -1339,6 +1385,9 @@ protected void printUsage(final String message) {
     System.err.println(" writeToWAL      Set writeToWAL on puts. Default: True");
     System.err.println(" presplit        Create presplit table. Recommended for accurate perf " +
       "analysis (see guide).  Default: disabled");
+    System.err.println(" filterAll       Helps to filter out all the rows on the server side"
+        + " there by not returning any thing back to the client.  Helps to check the server side"
+        + " performance.  Uses FilterAllFilter internally. ");
     System.err.println(" latency         Set to report operation latencies. " +
       "Currently only supported by randomRead test. Default: False");
     System.err.println();
@@ -1458,6 +1507,12 @@ public int run(String[] args) throws Exception {
           continue;
         }
 
+        final String filterOutAll = "--filterAll";
+        if (cmd.startsWith(filterOutAll)) {
+          this.filterAll = true;
+          continue;
+        }
+
         Class<? extends Test> cmdClass = determineCommandClass(cmd);
         if (cmdClass != null) {
           getArgs(i + 1, args);
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java
new file mode 100644
index 000000000000..e883bd0849ea
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+public class FilterAllFilter extends FilterBase {
+
+  public FilterAllFilter() {
+  }
+
+  @Override
+  public ReturnCode filterKeyValue(KeyValue ignored) {
+    return ReturnCode.SKIP;
+  }
+
+  @Override
+  public boolean filterRow() {
+    return true;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+  }
+
+}
\ No newline at end of file

From dfcf767ae21548f65851044440381dd72e4cccf2 Mon Sep 17 00:00:00 2001
From: Jimmy Xiang <jxiang@cloudera.com>
Date: Thu, 14 Aug 2014 09:00:45 -0700
Subject: [PATCH 1453/1540] HBASE-10834 Better error messaging on issuing grant
 commands in non-authz mode (Srikanth Srungarapu)

---
 src/main/ruby/hbase/security.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/ruby/hbase/security.rb b/src/main/ruby/hbase/security.rb
index 5041ec4dbb5b..ce452d186a87 100644
--- a/src/main/ruby/hbase/security.rb
+++ b/src/main/ruby/hbase/security.rb
@@ -182,6 +182,8 @@ def security_available?()
       rescue NameError
         raise(ArgumentError, "DISABLED: Security features are not available in this build of HBase")
       end
+      raise(ArgumentError, "Command not supported as authorization is turned off ") \
+        unless exists?(org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME)
     end
 
   end

From 0ea4b86b07b32d46b23f4f35de032370d64dd021 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 16 Aug 2014 21:51:51 -0700
Subject: [PATCH 1454/1540] HBASE-11767 [0.94] Unnecessary garbage produced by
 schema metrics during scanning.

---
 .../hadoop/hbase/regionserver/HRegionServer.java       |  4 ++--
 .../apache/hadoop/hbase/regionserver/StoreScanner.java | 10 ++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e6622a001f3a..ed278c75e69f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2679,7 +2679,7 @@ private Result[] internalNext(final RegionScanner s, int nbRows,
         }
         if (bypass != null) {
           return s.isFilterDone() && results.isEmpty() ? null
-              : results.toArray(new Result[0]);
+              : results.toArray(new Result[results.size()]);
         }
       }
 
@@ -2722,7 +2722,7 @@ private Result[] internalNext(final RegionScanner s, int nbRows,
       // and wants to tell the client to stop the scan. This is done by passing
       // a null result.
       return s.isFilterDone() && results.isEmpty() ? null
-          : results.toArray(new Result[0]);
+          : results.toArray(new Result[results.size()]);
     } catch (Throwable t) {
       if (t instanceof NotServingRegionException && scannerName != null) {
         this.scanners.remove(scannerName);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 6cb9711db049..c4ee384ef208 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -53,6 +53,7 @@ public class StoreScanner extends NonLazyKeyValueScanner
 
 
   private String metricNamePrefix;
+  private String metricNamePrefixNext;
   // Used to indicate that the scanner has closed (see HBASE-1107)
   // Doesnt need to be volatile because it's always accessed via synchronized methods
   private boolean closing = false;
@@ -455,8 +456,13 @@ public boolean next(List<KeyValue> outResult, int limit,
       }
     } finally {
       if (cumulativeMetric > 0 && metric != null) {
-        RegionMetricsStorage.incrNumericMetric(this.metricNamePrefix + metric,
-            cumulativeMetric);
+        // OK to use identity here
+        if (metric == SchemaMetrics.METRIC_NEXTSIZE) {
+          if (metricNamePrefixNext == null) metricNamePrefixNext = metricNamePrefix + metric;
+          RegionMetricsStorage.incrNumericMetric(metricNamePrefixNext, cumulativeMetric);
+        } else {
+          RegionMetricsStorage.incrNumericMetric(metricNamePrefix + metric, cumulativeMetric);
+        }
       }
     }
 

From d8e2eb9697000e447602e3ef7e60fa88fe01ce4a Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 16 Aug 2014 22:05:04 -0700
Subject: [PATCH 1455/1540] forward version to .23-SNAPSHOT

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 95970a88a52a..a7213e7804db 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.22</version>
+  <version>0.94.23-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 817c2aac1fee86c98c402abced5c6736fbdc2540 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Mon, 18 Aug 2014 14:50:49 -0700
Subject: [PATCH 1456/1540] HBASE-11754 [Shell] Record table property
 SPLITS_FILE in descriptor (chendihao)

---
 src/main/ruby/hbase/admin.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 3ccb7e3f3d68..df2c69d2b7bc 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -400,6 +400,7 @@ def alter(table_name, wait = true, *args)
               puts "Updating all regions with the new schema..."
               alter_status(table_name)
             end
+            htd.setValue(SPLITS_FILE, arg[SPLITS_FILE])
           end
           next
         end

From 44492624d4b8a6cf1ce1c7ba595f3a3447f9f536 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 22 Aug 2014 21:42:33 -0700
Subject: [PATCH 1457/1540] HBASE-11323 Add MultiRowMutation tests. (Liu
 Shaohui)

---
 .../TestMultiRowMutationProtocol.java         | 211 ++++++++++++++++++
 1 file changed, 211 insertions(+)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java

diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java
new file mode 100644
index 000000000000..8afdb07fa980
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * A test class to cover multi row mutations protocol
+ */
+@Category(MediumTests.class)
+public class TestMultiRowMutationProtocol {
+
+  private static final byte[] TEST_TABLE = Bytes.toBytes("TestTable");
+  private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
+  private static final byte[] INVALID_FAMILY = Bytes.toBytes("InvalidFamily");
+  private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
+  private static byte[] ROW = Bytes.toBytes("testRow");
+  
+  private static final int ROWSIZE = 20;
+  private static final int rowSeperator1 = 5;
+  private static final int rowSeperator2 = 12;
+  private static byte[][] ROWS = makeN(ROW, ROWSIZE);
+
+  private static HBaseTestingUtility util = new HBaseTestingUtility();
+  private static MiniHBaseCluster cluster = null;
+  
+  private HTable table = null;
+  
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    // set configure to indicate which cp should be loaded
+    Configuration conf = util.getConfiguration();
+    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+      "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint");
+
+    util.startMiniCluster(2);
+    cluster = util.getMiniHBaseCluster();
+
+    HTable table = util.createTable(TEST_TABLE, TEST_FAMILY);
+    util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY,
+                            new byte[][] { HConstants.EMPTY_BYTE_ARRAY,
+                                ROWS[rowSeperator1], ROWS[rowSeperator2] });
+
+    for (int i = 0; i < ROWSIZE; i++) {
+      Put put = new Put(ROWS[i]);
+      put.setWriteToWAL(false);
+      put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
+      table.put(put);
+    }
+
+    // sleep here is an ugly hack to allow region transitions to finish
+    long timeout = System.currentTimeMillis() + (15 * 1000);
+    while ((System.currentTimeMillis() < timeout) &&
+      (table.getRegionsInfo().size() != 3)) {
+      Thread.sleep(250);
+    }
+    table.close();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    util.shutdownMiniCluster();
+  }
+  
+  @Before
+  public void setup() throws IOException {
+    table = new HTable(util.getConfiguration(), TEST_TABLE);
+    for (int i = 0; i < ROWSIZE; i++) {
+      Put put = new Put(ROWS[i]);
+      put.setWriteToWAL(false);
+      put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
+      table.put(put);
+    }
+  }
+  
+  @After
+  public void tearDown() throws IOException {
+    table.close();
+  }
+  
+  @Test
+  public void testMultiRowMutations() throws IOException {
+    List<Mutation> mutations = new ArrayList<Mutation>();
+
+    Put put = new Put(ROWS[1]);
+    put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(2 * 1));
+    mutations.add(put);
+    Delete del = new Delete(ROWS[3]);
+    del.deleteColumns(TEST_FAMILY, TEST_QUALIFIER);
+    mutations.add(del);
+    
+    MultiRowMutationProtocol p =
+        table.coprocessorProxy(MultiRowMutationProtocol.class, mutations.get(0).getRow());
+    try {
+      p.mutateRows(mutations);
+    } catch (IOException e) {
+      Assert.assertTrue(false);
+    }
+    
+    Get get = new Get(ROWS[1]);
+    get.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    Result result = table.get(get);
+    Assert.assertEquals(2, Bytes.toInt(result.getValue(TEST_FAMILY, TEST_QUALIFIER)));
+    
+    
+    get = new Get(ROWS[3]);
+    get.addColumn(TEST_FAMILY, TEST_QUALIFIER);
+    result = table.get(get);
+    Assert.assertNull(result.getValue(TEST_FAMILY, TEST_QUALIFIER));
+  }
+  
+  @Test
+  public void testMultiRowMutationsAcrossRegions() throws IOException {
+    List<Mutation> mutations = new ArrayList<Mutation>();
+
+    Put put = new Put(ROWS[1]);
+    put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(2 * 1));
+    mutations.add(put);
+    Delete del = new Delete(ROWS[7]);
+    del.deleteColumns(TEST_FAMILY, TEST_QUALIFIER);
+    mutations.add(del);
+    
+    MultiRowMutationProtocol p =
+        table.coprocessorProxy(MultiRowMutationProtocol.class, mutations.get(0).getRow());
+    try {
+      p.mutateRows(mutations);
+      Assert.assertTrue(false);
+    } catch (IOException e) {
+    }
+  }
+  
+  @Test
+  public void testInvalidFamiliy() throws IOException {
+    List<Mutation> invalids = new ArrayList<Mutation>();
+    Put put = new Put(ROWS[1]);
+    put.add(INVALID_FAMILY, TEST_QUALIFIER, Bytes.toBytes(2 * 1));
+    invalids.add(put);
+    
+    MultiRowMutationProtocol p =
+        table.coprocessorProxy(MultiRowMutationProtocol.class, ROWS[1]);
+    try {
+      p.mutateRows(invalids);
+      Assert.assertTrue(false);
+    } catch (IOException e) {
+    }
+    
+    List<Mutation> valids = new ArrayList<Mutation>();
+    put = new Put(ROWS[1]);
+    put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(2 * 1));
+    valids.add(put);
+    try {
+      p.mutateRows(valids);
+    } catch (IOException e) {
+      Assert.assertTrue(false);
+    }
+  }
+
+  /**
+   * an infrastructure method to prepare rows for the testtable.
+   * @param base
+   * @param n
+   * @return
+   */
+  private static byte[][] makeN(byte[] base, int n) {
+    byte[][] ret = new byte[n][];
+    for (int i = 0; i < n; i++) {
+      ret[i] = Bytes.add(base, Bytes.toBytes(i));
+    }
+    return ret;
+  }
+}

From 6283ff920057260ed6ae69aa0d1a084ac8ac6c74 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 25 Aug 2014 12:00:55 -0700
Subject: [PATCH 1458/1540] HBASE-11536 Puts of region location to Meta may be
 out of order which causes inconsistent of region location. (Liu Shaohui)

---
 .../java/org/apache/hadoop/hbase/catalog/MetaEditor.java  | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
index 0d75532fab7a..5f59fd754504 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
@@ -406,9 +407,12 @@ private static Put addRegionInfo(final Put p, final HRegionInfo hri)
   }
 
   private static Put addLocation(final Put p, final ServerName sn) {
-    p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+    // using regionserver's local time as the timestamp of Put.
+    // See: HBASE-11536
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+    p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, now,
       Bytes.toBytes(sn.getHostAndPort()));
-    p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
+    p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, now,
       Bytes.toBytes(sn.getStartcode()));
     return p;
   }

From 2928e64474029693d2a7a9c960fcf8170650ff70 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 25 Aug 2014 17:02:24 -0700
Subject: [PATCH 1459/1540] HBASE-9746 RegionServer can't start when
 replication tries to replicate to an unknown host.

---
 .../hbase/zookeeper/RecoverableZooKeeper.java | 75 ++++++++++++-------
 .../hadoop/hbase/zookeeper/ZKConfig.java      |  8 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java     |  4 +-
 3 files changed, 50 insertions(+), 37 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index ba5db2d76fb8..3cb6d2d1e2f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -95,7 +95,6 @@ public class RecoverableZooKeeper {
   public RecoverableZooKeeper(String quorumServers, int sessionTimeout,
       Watcher watcher, int maxRetries, int retryIntervalMillis) 
   throws IOException {
-    this.zk = new ZooKeeper(quorumServers, sessionTimeout, watcher);
     this.retryCounterFactory =
       new RetryCounterFactory(maxRetries, retryIntervalMillis);
 
@@ -106,15 +105,35 @@ public RecoverableZooKeeper(String quorumServers, int sessionTimeout,
     this.watcher = watcher;
     this.sessionTimeout = sessionTimeout;
     this.quorumServers = quorumServers;
+    try {checkZk();} catch (Exception x) {/* ignore */}
+  }
+
+  /**
+   * Try to create a Zookeeper connection. Turns any exception encountered into a
+   * {@link KeeperException.OperationTimeoutException} so it can retried.
+   * @return The created Zookeeper connection object
+   * @throws KeeperException
+   */
+  protected ZooKeeper checkZk() throws KeeperException {
+    if (this.zk == null) {
+      try {
+        this.zk = new ZooKeeper(quorumServers, sessionTimeout, watcher);
+      } catch (Exception uhe) {
+        LOG.warn("Unable to create ZooKeeper Connection", uhe);
+        throw new KeeperException.OperationTimeoutException();
+      }
+    }
+    return zk;
   }
 
   public void reconnectAfterExpiration()
-        throws IOException, InterruptedException {
-    LOG.info("Closing dead ZooKeeper connection, session" +
-      " was: 0x"+Long.toHexString(zk.getSessionId()));
-    zk.close();
-    this.zk = new ZooKeeper(this.quorumServers,
-      this.sessionTimeout, this.watcher);
+        throws IOException, KeeperException, InterruptedException {
+    if (zk != null) {
+      LOG.info("Closing dead ZooKeeper connection, session" +
+        " was: 0x"+Long.toHexString(zk.getSessionId()));
+      zk.close();
+    }
+    checkZk();
     LOG.info("Recreated a ZooKeeper, session" +
       " is: 0x"+Long.toHexString(zk.getSessionId()));
   }
@@ -130,7 +149,7 @@ public void delete(String path, int version)
     boolean isRetry = false; // False for first attempt, true for all retries.
     while (true) {
       try {
-        zk.delete(path, version);
+        checkZk().delete(path, version);
         return;
       } catch (KeeperException e) {
         switch (e.code()) {
@@ -169,7 +188,7 @@ public Stat exists(String path, Watcher watcher)
     RetryCounter retryCounter = retryCounterFactory.create();
     while (true) {
       try {
-        return zk.exists(path, watcher);
+        return checkZk().exists(path, watcher);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -196,7 +215,7 @@ public Stat exists(String path, boolean watch)
     RetryCounter retryCounter = retryCounterFactory.create();
     while (true) {
       try {
-        return zk.exists(path, watch);
+        return checkZk().exists(path, watch);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -233,7 +252,7 @@ public List<String> getChildren(String path, Watcher watcher)
     RetryCounter retryCounter = retryCounterFactory.create();
     while (true) {
       try {
-        return zk.getChildren(path, watcher);
+        return checkZk().getChildren(path, watcher);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -260,7 +279,7 @@ public List<String> getChildren(String path, boolean watch)
     RetryCounter retryCounter = retryCounterFactory.create();
     while (true) {
       try {
-        return zk.getChildren(path, watch);
+        return checkZk().getChildren(path, watch);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -287,7 +306,7 @@ public byte[] getData(String path, Watcher watcher, Stat stat)
     RetryCounter retryCounter = retryCounterFactory.create();
     while (true) {
       try {
-        byte[] revData = zk.getData(path, watcher, stat);       
+        byte[] revData = checkZk().getData(path, watcher, stat);       
         return this.removeMetaData(revData);
       } catch (KeeperException e) {
         switch (e.code()) {
@@ -315,7 +334,7 @@ public byte[] getData(String path, boolean watch, Stat stat)
     RetryCounter retryCounter = retryCounterFactory.create();
     while (true) {
       try {
-        byte[] revData = zk.getData(path, watch, stat);
+        byte[] revData = checkZk().getData(path, watch, stat);
         return this.removeMetaData(revData);
       } catch (KeeperException e) {
         switch (e.code()) {
@@ -346,7 +365,7 @@ public Stat setData(String path, byte[] data, int version)
     byte[] newData = appendMetaData(data);
     while (true) {
       try {
-        return zk.setData(path, newData, version);
+        return checkZk().setData(path, newData, version);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -358,7 +377,7 @@ public Stat setData(String path, byte[] data, int version)
             // try to verify whether the previous setData success or not
             try{
               Stat stat = new Stat();
-              byte[] revData = zk.getData(path, false, stat);
+              byte[] revData = checkZk().getData(path, false, stat);
               if (Bytes.equals(revData, newData)) {
                 // the bad version is caused by previous successful setData
                 return stat;
@@ -418,7 +437,7 @@ private String createNonSequential(String path, byte[] data, List<ACL> acl,
     boolean isRetry = false; // False for first attempt, true for all retries.
     while (true) {
       try {
-        return zk.create(path, data, acl, createMode);
+        return checkZk().create(path, data, acl, createMode);
       } catch (KeeperException e) {
         switch (e.code()) {
           case NODEEXISTS:
@@ -426,7 +445,7 @@ private String createNonSequential(String path, byte[] data, List<ACL> acl,
               // If the connection was lost, there is still a possibility that
               // we have successfully created the node at our previous attempt,
               // so we read the node and compare. 
-              byte[] currentData = zk.getData(path, false, null);
+              byte[] currentData = checkZk().getData(path, false, null);
               if (currentData != null &&
                   Bytes.compareTo(currentData, data) == 0) { 
                 // We successfully created a non-sequential node
@@ -473,7 +492,7 @@ private String createSequential(String path, byte[] data,
           }
         }
         first = false;
-        return zk.create(newPath, data, acl, createMode);
+        return checkZk().create(newPath, data, acl, createMode);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -528,7 +547,7 @@ public List<OpResult> multi(Iterable<Op> ops)
     Iterable<Op> multiOps = prepareZKMulti(ops);
     while (true) {
       try {
-        return zk.multi(multiOps);
+        return checkZk().multi(multiOps);
       } catch (KeeperException e) {
         switch (e.code()) {
           case CONNECTIONLOSS:
@@ -553,11 +572,11 @@ private String findPreviousSequentialNode(String path)
     String parent = path.substring(0, lastSlashIdx);
     String nodePrefix = path.substring(lastSlashIdx+1);
 
-    List<String> nodes = zk.getChildren(parent, false);
+    List<String> nodes = checkZk().getChildren(parent, false);
     List<String> matching = filterByPrefix(nodes, nodePrefix);
     for (String node : matching) {
       String nodePath = parent + "/" + node;
-      Stat stat = zk.exists(nodePath, false);
+      Stat stat = checkZk().exists(nodePath, false);
       if (stat != null) {
         return nodePath;
       }
@@ -602,15 +621,15 @@ private byte[] appendMetaData(byte[] data) {
   }
 
   public long getSessionId() {
-    return zk.getSessionId();
+    return zk == null ? null : zk.getSessionId();
   }
 
   public void close() throws InterruptedException {
-    zk.close();
+    if (zk != null) zk.close();
   }
 
   public States getState() {
-    return zk.getState();
+    return zk == null ? null : zk.getState();
   }
 
   public ZooKeeper getZooKeeper() {
@@ -618,11 +637,11 @@ public ZooKeeper getZooKeeper() {
   }
 
   public byte[] getSessionPasswd() {
-    return zk.getSessionPasswd();
+    return zk == null ? null : zk.getSessionPasswd();
   }
 
-  public void sync(String path, AsyncCallback.VoidCallback cb, Object ctx) {
-    this.zk.sync(path, null, null);
+  public void sync(String path, AsyncCallback.VoidCallback cb, Object ctx) throws KeeperException {
+    checkZk().sync(path, null, null);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
index 4cc9c016d12f..9161b8583d38 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
@@ -196,13 +196,7 @@ public static String getZKQuorumServersString(Properties properties) {
       else if (key.startsWith("server.")) {
         String host = value.substring(0, value.indexOf(':'));
         servers.add(host);
-        try {
-          //noinspection ResultOfMethodCallIgnored
-          InetAddress.getByName(host);
-          anyValid = true;
-        } catch (UnknownHostException e) {
-          LOG.warn(StringUtils.stringifyException(e));
-        }
+        anyValid = true;
       }
     }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index e023692caad5..81081d56a9ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -271,7 +271,7 @@ public RecoverableZooKeeper getRecoverableZooKeeper() {
     return recoverableZooKeeper;
   }
 
-  public void reconnectAfterExpiration() throws IOException, InterruptedException {
+  public void reconnectAfterExpiration() throws IOException, KeeperException, InterruptedException {
     recoverableZooKeeper.reconnectAfterExpiration();
   }
 
@@ -399,7 +399,7 @@ private void connectionEvent(WatchedEvent event) {
    * previously read version and data.  We want to ensure that the version read
    * is up-to-date from when we begin the operation.
    */
-  public void sync(String path) {
+  public void sync(String path) throws KeeperException {
     this.recoverableZooKeeper.sync(path, null, null);
   }
 

From 59b35056c84fb8b1ea5de1654e43686f93f3f86b Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 26 Aug 2014 11:01:27 -0700
Subject: [PATCH 1460/1540] HBASE-9746 Addendum.

---
 .../hadoop/hbase/zookeeper/RecoverableZooKeeper.java      | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 3cb6d2d1e2f4..ef95622caa3d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -110,7 +110,7 @@ public RecoverableZooKeeper(String quorumServers, int sessionTimeout,
 
   /**
    * Try to create a Zookeeper connection. Turns any exception encountered into a
-   * {@link KeeperException.OperationTimeoutException} so it can retried.
+   * KeeperException.OperationTimeoutException so it can retried.
    * @return The created Zookeeper connection object
    * @throws KeeperException
    */
@@ -118,8 +118,8 @@ protected ZooKeeper checkZk() throws KeeperException {
     if (this.zk == null) {
       try {
         this.zk = new ZooKeeper(quorumServers, sessionTimeout, watcher);
-      } catch (Exception uhe) {
-        LOG.warn("Unable to create ZooKeeper Connection", uhe);
+      } catch (IOException ex) {
+        LOG.warn("Unable to create ZooKeeper Connection", ex);
         throw new KeeperException.OperationTimeoutException();
       }
     }
@@ -132,6 +132,8 @@ public void reconnectAfterExpiration()
       LOG.info("Closing dead ZooKeeper connection, session" +
         " was: 0x"+Long.toHexString(zk.getSessionId()));
       zk.close();
+      // reset the Zookeeper connection
+      zk = null;
     }
     checkZk();
     LOG.info("Recreated a ZooKeeper, session" +

From f42302b28aceaab773b15f234aa8718fff7eea3c Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 26 Aug 2014 12:30:24 -0700
Subject: [PATCH 1461/1540] CHANGES.txt, pom.xml for 0.94.23RC0

---
 CHANGES.txt | 23 +++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 1fafc1e59fa7..cd1485b780eb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,27 @@
 HBase Change Log
+Release 0.94.23 - 08/26/2014
+Bug
+
+    [HBASE-9746] - RegionServer can't start when replication tries to replicate to an unknown host
+    [HBASE-10834] - Better error messaging on issuing grant commands in non-authz mode
+    [HBASE-11232] - Add MultiRowMutation tests.
+    [HBASE-11536] - Puts of region location to Meta may be out of order which causes inconsistent of region location
+    [HBASE-11641] - TestDistributedLogSplitting.testMasterStartsUpWithLogSplittingWork fails frequently
+    [HBASE-11652] - Port HBASE-3270 and HBASE-11650 to 0.94 - create cluster id and version file in a tmp location and move it into place
+    [HBASE-11767] - [0.94] Unnecessary garbage produced by schema metrics during scanning
+
+Improvement
+
+    [HBASE-11667] - Comment ClientScanner logic for NSREs.
+    [HBASE-11754] - [Shell] Record table property SPLITS_FILE in descriptor
+
+Task
+
+    [HBASE-11690] - Backport HBASE-5934 (Add the ability for Performance Evaluation to set the table compression) to 0.94
+    [HBASE-11691] - Backport HBASE-7156 (Add Data Block Encoding and -D opts to Performance Evaluation) to 0.94
+    [HBASE-11693] - Backport HBASE-11026 (Provide option to filter out all rows in PerformanceEvaluation tool) to 0.94
+
+
 Release 0.94.22 - 07/31/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index a7213e7804db..5c3382c743ac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.23-SNAPSHOT</version>
+  <version>0.94.23</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 1130af64af49e87019a94411957533c19206b837 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 5 Sep 2014 10:24:35 -0700
Subject: [PATCH 1462/1540] Start 0.94.24 snapshot

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 5c3382c743ac..aab0590d5d75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.23</version>
+  <version>0.94.24-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 045f41888ac51130b8696498ac51704eb9395046 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Tue, 9 Sep 2014 18:52:19 -0700
Subject: [PATCH 1463/1540] HBASE-11923 Potential race condition in
 RecoverableZookeeper.checkZk() (Lars Hofhansl)

---
 .../apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index ef95622caa3d..93ba93007716 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -70,7 +70,7 @@
 public class RecoverableZooKeeper {
   private static final Log LOG = LogFactory.getLog(RecoverableZooKeeper.class);
   // the actual ZooKeeper client instance
-  private volatile ZooKeeper zk;
+  private ZooKeeper zk;
   private final RetryCounterFactory retryCounterFactory;
   // An identifier of this process in the cluster
   private final String identifier;
@@ -114,7 +114,7 @@ public RecoverableZooKeeper(String quorumServers, int sessionTimeout,
    * @return The created Zookeeper connection object
    * @throws KeeperException
    */
-  protected ZooKeeper checkZk() throws KeeperException {
+  protected synchronized ZooKeeper checkZk() throws KeeperException {
     if (this.zk == null) {
       try {
         this.zk = new ZooKeeper(quorumServers, sessionTimeout, watcher);
@@ -126,7 +126,7 @@ protected ZooKeeper checkZk() throws KeeperException {
     return zk;
   }
 
-  public void reconnectAfterExpiration()
+  public synchronized void reconnectAfterExpiration()
         throws IOException, KeeperException, InterruptedException {
     if (zk != null) {
       LOG.info("Closing dead ZooKeeper connection, session" +

From b69f83995dc2d62b2395402bf373a4b5f1ddb501 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 12 Sep 2014 23:50:59 -0700
Subject: [PATCH 1464/1540] HBASE-11963 Synchronize peer cluster replication
 connection attempts. (Sukumar Maddineni)

---
 .../replication/ReplicationZookeeper.java     | 39 ++++++++++++-------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 6436f0b9ae3c..c1c2b581a28f 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -238,18 +238,23 @@ public List<ServerName> getSlavesAddresses(String peerClusterId) {
     if (peer == null) {
       return Collections.emptyList();
     }
-    
-    List<ServerName> addresses;
-    try {
-      addresses = fetchSlavesAddresses(peer.getZkw());
-    } catch (KeeperException ke) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Fetch salves addresses failed.", ke);
+    // Synchronize peer cluster connection attempts to avoid races and rate
+    // limit connections when multiple replication sources try to connect to
+    // the peer cluster. If the peer cluster is down we can get out of control
+    // over time.
+    synchronized (peer) {
+      List<ServerName> addresses;
+      try {
+        addresses = fetchSlavesAddresses(peer.getZkw());
+      } catch (KeeperException ke) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Fetch salves addresses failed.", ke);
+        }
+        reconnectPeer(ke, peer);
+        addresses = Collections.emptyList();
       }
-      reconnectPeer(ke, peer);
-      addresses = Collections.emptyList();
+      peer.setRegionServers(addresses);
     }
-    peer.setRegionServers(addresses);
     return peer.getRegionServers();
   }
 
@@ -869,10 +874,16 @@ public long getHLogRepPosition(String peerId, String hlog)
   public UUID getPeerUUID(String peerId) {
     ReplicationPeer peer = getPeerClusters().get(peerId);
     UUID peerUUID = null;
-    try {
-      peerUUID = getUUIDForCluster(peer.getZkw());
-    } catch (KeeperException ke) {
-      reconnectPeer(ke, peer);
+    // Synchronize peer cluster connection attempts to avoid races and rate
+    // limit connections when multiple replication sources try to connect to
+    // the peer cluster. If the peer cluster is down we can get out of control
+    // over time.
+    synchronized (peer) {
+      try {
+        peerUUID = getUUIDForCluster(peer.getZkw());
+      } catch (KeeperException ke) {
+        reconnectPeer(ke, peer);
+      }
     }
     return peerUUID;
   }

From 017bb0f6a60703f73b2867170a1132de4f55a9e6 Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Thu, 18 Sep 2014 17:00:53 -0700
Subject: [PATCH 1465/1540] HBASE-12019 hbase-daemon.sh overwrite
 HBASE_ROOT_LOGGER and HBASE_SECURITY_LOGGER variables (Sebastien Barrier)

Fixed conflict.
Conflicts:
	bin/hbase-daemon.sh
---
 bin/hbase-daemon.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/bin/hbase-daemon.sh b/bin/hbase-daemon.sh
index 6f680905cdad..f54142223e33 100755
--- a/bin/hbase-daemon.sh
+++ b/bin/hbase-daemon.sh
@@ -116,9 +116,17 @@ fi
 JAVA=$JAVA_HOME/bin/java
 export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME
 export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log
-export HBASE_ROOT_LOGGER="INFO,DRFA"
-export HBASE_SECURITY_LOGGER="INFO,DRFAS"
-logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out  
+
+if [ -z "${HBASE_ROOT_LOGGER}" ]; then
+export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"}
+fi
+
+if [ -z "${HBASE_SECURITY_LOGGER}" ]; then 
+export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"}
+fi
+
+logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out
+
 loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc
 loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}"
 pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid

From cbc08ceed5f0b1bab4adc0aa37dc00495c249634 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 19 Sep 2014 10:11:33 -0700
Subject: [PATCH 1466/1540] HBASE-12020 String formatting on each RPC Invoke.
 (Vladimir Rodionov)

---
 .../java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java b/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
index b8b290c89caf..8a3a45f39ac3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
@@ -78,8 +78,10 @@ public ExecResult call() throws Exception {
           };
       ExecResult result = callable.withRetries();
       this.regionName = result.getRegionName();
-      LOG.debug("Result is region="+ Bytes.toStringBinary(regionName) +
+      if(LOG.isDebugEnabled()){
+        LOG.debug("Result is region="+ Bytes.toStringBinary(regionName) +
           ", value="+result.getValue());
+      }
       return result.getValue();
     }
 
@@ -89,4 +91,4 @@ public ExecResult call() throws Exception {
   public byte[] getRegionName() {
     return regionName;
   }
-}
\ No newline at end of file
+}

From 0058e7aacee0a10d020590ce2c2485c53f54e270 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 19 Sep 2014 10:14:08 -0700
Subject: [PATCH 1467/1540] HBASE-12022 Payloads on Failure attempt to
 serialize the byte[] into strings. (Vladimir Rodionov)

---
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index d0d472f3975a..261ce4a62d7f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -1434,7 +1434,9 @@ public void run() {
             value = call(call.connection.protocol, call.param, call.timestamp, 
                 status);
           } catch (Throwable e) {
-            LOG.debug(getName()+", call "+call+": error: " + e, e);
+            if(LOG.isDebugEnabled()){
+              LOG.debug(getName()+", call "+call+": error: " + e, e);
+            }
             errorClass = e.getClass().getName();
             error = StringUtils.stringifyException(e);
           } finally {

From 8f9faabf579c02476acb791c145f34baf49ac8f5 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 19 Sep 2014 11:05:02 -0700
Subject: [PATCH 1468/1540] HBASE-11957 Backport to 0.94 HBASE-5974 Scanner
 retry behavior with RPC timeout on next() seems incorrect. (Liu Shaohui
 original patch by Anoop Sam John)

---
 .../CallSequenceOutOfOrderException.java      |  35 ++++++
 .../hadoop/hbase/client/ClientScanner.java    |   6 +-
 .../hadoop/hbase/client/ScannerCallable.java  |  32 ++++-
 .../hadoop/hbase/ipc/HRegionInterface.java    |  12 ++
 .../hbase/regionserver/HRegionServer.java     |  53 +++++---
 .../regionserver/RegionScannerHolder.java     |  44 +++++++
 .../hadoop/hbase/util/JVMClusterUtil.java     |   5 +-
 .../client/TestClientScannerRPCTimeout.java   | 115 ++++++++++++++++++
 8 files changed, 281 insertions(+), 21 deletions(-)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/CallSequenceOutOfOrderException.java
 create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerHolder.java
 create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java

diff --git a/src/main/java/org/apache/hadoop/hbase/CallSequenceOutOfOrderException.java b/src/main/java/org/apache/hadoop/hbase/CallSequenceOutOfOrderException.java
new file mode 100644
index 000000000000..d3a77beb9dc5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/CallSequenceOutOfOrderException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+/**
+ * Thrown by a region server while doing scan related next() calls. Both client and server maintain a
+ * callSequence and if they do not match, RS will throw this exception.
+ */
+public class CallSequenceOutOfOrderException extends DoNotRetryIOException {
+
+  private static final long serialVersionUID = 1565946556907760065L;
+
+  public CallSequenceOutOfOrderException() {
+    super();
+  }
+
+  public CallSequenceOutOfOrderException(String msg) {
+    super(msg);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 7e72d57a9783..1301ffb8c176 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.CallSequenceOutOfOrderException;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
@@ -296,8 +297,9 @@ public Result next() throws IOException {
               }
             } else {
               Throwable cause = e.getCause();
-              if (cause == null || (!(cause instanceof NotServingRegionException)
-                  && !(cause instanceof RegionServerStoppedException))) {
+              if ((cause == null || (!(cause instanceof NotServingRegionException)
+                  && !(cause instanceof RegionServerStoppedException)))
+                  && !(e instanceof CallSequenceOutOfOrderException)) {
                 throw e;
               }
             }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
index 0c4677fdea74..8662db0e9e8f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
@@ -21,10 +21,12 @@
 import java.io.IOException;
 import java.net.UnknownHostException;
 
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.CallSequenceOutOfOrderException;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
@@ -63,6 +65,8 @@ public class ScannerCallable extends ServerCallable<Result[]> {
 
   // indicate if it is a remote server call
   private boolean isRegionServerRemote = true;
+  private long callSeq = 0;
+  private boolean useCallSeq = true;
 
   /**
    * @param connection which connection
@@ -129,7 +133,33 @@ private void checkIfRegionServerIsRemote() {
       try {
         incRPCcallsMetrics();
         long timestamp = System.currentTimeMillis();
-        rrs = server.next(scannerId, caching);
+        if (useCallSeq) {
+          try {
+            rrs = server.next(scannerId, caching, callSeq);
+            // increment the callSeq which will be getting used for the next time next() call to
+            // the RS.In case of a timeout this increment should not happen so that the next
+            // trial also will be done with the same callSeq.
+            callSeq++;
+          } catch (IOException ioe) {
+            // TODO This is an ugly way of checking. Any other ways?
+            if (ioe instanceof RemoteException
+                && ExceptionUtils.getStackTrace(ioe).contains("java.lang.NoSuchMethodException")) {
+              // This will happen when we use a latest version of the client but still running with
+              // old region server. At server side there is no implementation for the seq number
+              // based scanning. Set the useCallSeq to false.
+              LOG.warn("Seq number based scan API not present at RS side! Trying with API: "
+                  + "next(scannerId, caching). Consider upgrading version at RS "
+                  + location.getHostnamePort());
+              useCallSeq = false;
+              rrs = server.next(scannerId, caching);
+            } else {
+              // Throw it back so that will get handled by the below original catch blocks;
+              throw ioe;
+            }
+          }
+        } else {
+          rrs = server.next(scannerId, caching);
+        }
         if (logScannerActivity) {
           long now = System.currentTimeMillis();
           if (now - timestamp > logCutOffLatency) {
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index 014ee9f4e180..8eae0ec6fe20 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -327,6 +327,18 @@ public long openScanner(final byte [] regionName, final Scan scan)
    */
   public Result [] next(long scannerId, int numberOfRows) throws IOException;
 
+  /**
+   * Get the next set of values
+   * @param scannerId clientId passed to openScanner
+   * @param numberOfRows the number of rows to fetch
+   * @param callSeq the number which represents the sequence used by client scanner
+   * @return Array of Results (map of values); array is empty if done with this
+   * region and null if we are NOT to go to the next region (happens when a
+   * filter rules that the scan is done).
+   * @throws IOException e
+   */
+  public Result[] next(long scannerId, int caching, long callSeq) throws IOException;
+
   /**
    * Close a scanner
    *
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ed278c75e69f..eadc9e8022d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -59,6 +59,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CallSequenceOutOfOrderException;
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ClockOutOfSyncException;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -312,8 +313,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
   // flag set after we're done setting up server threads (used for testing)
   protected volatile boolean isOnline;
 
-  final Map<String, RegionScanner> scanners =
-    new ConcurrentHashMap<String, RegionScanner>();
+  final Map<String, RegionScannerHolder> scanners =
+    new ConcurrentHashMap<String, RegionScannerHolder>();
 
   // zookeeper connection and watcher
   private ZooKeeperWatcher zooKeeper;
@@ -569,8 +570,8 @@ public Integer apply(Writable from) {
           return HConstants.NORMAL_QOS;
         }
         String scannerIdString = Long.toString(scannerId);
-        RegionScanner scanner = scanners.get(scannerIdString);
-        if (scanner != null && scanner.getRegionInfo().isMetaTable()) {
+        RegionScannerHolder holder = scanners.get(scannerIdString);
+        if (holder != null && holder.getScanner().getRegionInfo().isMetaRegion()) {
           // LOG.debug("High priority scanner request: " + scannerId);
           return HConstants.HIGH_QOS;
         }
@@ -1063,9 +1064,9 @@ private void closeWAL(final boolean delete) {
   private void closeAllScanners() {
     // Close any outstanding scanners. Means they'll get an UnknownScanner
     // exception next time they come in.
-    for (Map.Entry<String, RegionScanner> e : this.scanners.entrySet()) {
+    for (Map.Entry<String, RegionScannerHolder> e : this.scanners.entrySet()) {
       try {
-        e.getValue().close();
+        e.getValue().getScanner().close();
       } catch (IOException ioe) {
         LOG.warn("Closing scanner " + e.getKey(), ioe);
       }
@@ -2606,7 +2607,7 @@ protected long addScanner(RegionScanner s) throws LeaseStillHeldException {
     long scannerId = -1L;
     scannerId = rand.nextLong();
     String scannerName = String.valueOf(scannerId);
-    scanners.put(scannerName, s);
+    scanners.put(scannerName, new RegionScannerHolder(s));
     this.leases.createLease(scannerName, new ScannerListener(scannerName));
     return scannerId;
   }
@@ -2619,14 +2620,28 @@ public Result next(final long scannerId) throws IOException {
     return res[0];
   }
 
-  public Result[] next(final long scannerId, int nbRows) throws IOException {
+   public Result[] next(final long scannerId, int nbRows) throws IOException {
+    return next(scannerId, nbRows, -1);
+  }
+
+  public Result[] next(final long scannerId, int nbRows, long callSeq) throws IOException {
     String scannerName = String.valueOf(scannerId);
-    RegionScanner s = this.scanners.get(scannerName);
-    if (s == null) {
+    RegionScannerHolder holder = this.scanners.get(scannerName);
+    if (holder == null) {
       LOG.info("Client tried to access missing scanner " + scannerName);
       throw new UnknownScannerException("Name: " + scannerName);
     }
-    return internalNext(s, nbRows, scannerName);
+    // if callSeq does not match throw Exception straight away. This needs to be performed even
+    // before checking of Lease.
+    // Old next() APIs which do not take callSeq will pass it as -1 and for that no
+    // need to match the callSeq from client and the one in server.
+    if (callSeq != -1 && callSeq != holder.getCallSeq()) {
+      throw new CallSequenceOutOfOrderException("Expected seq: " + holder.getCallSeq()
+          + " But the seq got from client: " + callSeq);
+     }
+    // Increment the callSeq value which is the next expected from client.
+    holder.incrCallSeq();
+    return internalNext(holder.getScanner(), nbRows, scannerName);
   }
 
   private Result[] internalNext(final RegionScanner s, int nbRows,
@@ -2739,8 +2754,9 @@ private Result[] internalNext(final RegionScanner s, int nbRows,
 
   public void close(final long scannerId) throws IOException {
     String scannerName = String.valueOf(scannerId);
-    RegionScanner s = scanners.get(scannerName);
-    internalCloseScanner(s, scannerName);
+    RegionScannerHolder holder = this.scanners.get(scannerName);
+    if (holder == null) throw new UnknownScannerException("Name: " + scannerName);
+    internalCloseScanner(holder.getScanner(), scannerName);
   }
 
   private void internalCloseScanner(final RegionScanner s, String scannerName)
@@ -2748,7 +2764,6 @@ private void internalCloseScanner(final RegionScanner s, String scannerName)
     try {
       checkOpen();
       requestCount.incrementAndGet();
-
       HRegion region = null;
       if (s != null) {
         // call coprocessor.
@@ -2761,7 +2776,10 @@ private void internalCloseScanner(final RegionScanner s, String scannerName)
       }
       RegionScanner toCloseScanner = s;
       if (scannerName != null) {
-        toCloseScanner = scanners.remove(scannerName);
+        RegionScannerHolder holder = scanners.remove(scannerName);
+        if (holder!= null) {
+          toCloseScanner = holder.getScanner();
+        }
       }
       if (toCloseScanner != null) {
         toCloseScanner.close();
@@ -2802,8 +2820,9 @@ private class ScannerListener implements LeaseListener {
     }
 
     public void leaseExpired() {
-      RegionScanner s = scanners.remove(this.scannerName);
-      if (s != null) {
+      RegionScannerHolder holder = scanners.remove(this.scannerName);
+      if (holder != null) {
+        RegionScanner s = holder.getScanner();
         LOG.info("Scanner " + this.scannerName + " lease expired on region "
             + s.getRegionInfo().getRegionNameAsString());
         try {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerHolder.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerHolder.java
new file mode 100644
index 000000000000..50c690d7eead
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerHolder.java
@@ -0,0 +1,44 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * Holder class which holds the RegionScanner and callSequence together.
+ */
+public class RegionScannerHolder {
+  private RegionScanner s;
+  private long callSeq = 0L;
+
+  public RegionScannerHolder(RegionScanner s) {
+    this.s = s;
+  }
+
+  public RegionScanner getScanner() {
+    return s;
+  }
+
+  public long getCallSeq() {
+    return callSeq;
+  }
+
+  public void incrCallSeq() {
+    callSeq++;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
index b079f2e28608..c990178e28e9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
@@ -81,7 +82,9 @@ public static JVMClusterUtil.RegionServerThread createRegionServerThread(
   throws IOException {
     HRegionServer server;
     try {
-      server = hrsc.getConstructor(Configuration.class).newInstance(c);
+      Constructor<? extends HRegionServer> ctor = hrsc.getConstructor(Configuration.class);
+      ctor.setAccessible(true);
+      server = ctor.newInstance(c);
     } catch (InvocationTargetException ite) {
       Throwable target = ite.getTargetException();
       throw new RuntimeException("Failed construction of RegionServer: " +
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java b/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java
new file mode 100644
index 000000000000..38b558cd848c
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the scenario where a next() call, while scanning, timeout at client side and getting retried.
+ * This scenario should not result in some data being skipped at RS side.
+ */
+@Category(MediumTests.class)
+public class TestClientScannerRPCTimeout {
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final byte[] FAMILY = Bytes.toBytes("testFamily");
+  private static final byte[] QUALIFIER = Bytes.toBytes("testQualifier");
+  private static final byte[] VALUE = Bytes.toBytes("testValue");
+  private static final int SLAVES = 1;
+  private static final int rpcTimeout = 5 * 1000;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout);
+    conf.setStrings(HConstants.REGION_SERVER_IMPL, RegionServerWithScanTimeout.class.getName());
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testScannerNextRPCTimesout() throws Exception {
+    byte[] TABLE = Bytes.toBytes("testScannerNextRPCTimesout");
+    HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
+    putToTable(ht, "row-1");
+    putToTable(ht, "row-2");
+    RegionServerWithScanTimeout.seqNoToSleepOn = 1;
+    Scan scan = new Scan();
+    scan.setCaching(1);
+    ResultScanner scanner = ht.getScanner(scan);
+    Result result = scanner.next();
+    assertNotNull("Expected not null result", result);
+    result = scanner.next();
+    assertNotNull("Expected not null result", result);
+    scanner.close();
+  }
+
+  private void putToTable(HTable ht, String rowkey) throws IOException {
+    Put put = new Put(rowkey.getBytes());
+    put.add(FAMILY, QUALIFIER, VALUE);
+    ht.put(put);
+  }
+
+  private static class RegionServerWithScanTimeout extends MiniHBaseClusterRegionServer {
+    private long tableScannerId;
+    private boolean slept;
+    private static long seqNoToSleepOn = -1;
+
+    public RegionServerWithScanTimeout(Configuration conf) throws IOException,
+        InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    public long openScanner(byte[] regionName, Scan scan) throws IOException {
+      long scannerId = super.openScanner(regionName, scan);
+      if (!getRegionInfo(regionName).isMetaTable()) {
+        tableScannerId = scannerId;
+      }
+      return scannerId;
+    }
+
+    @Override
+    public Result[] next(long scannerId, int nbRows, long callSeq) throws IOException {
+      if (!slept && this.tableScannerId == scannerId && seqNoToSleepOn == callSeq) {
+        try {
+          Thread.sleep(rpcTimeout + 500);
+        } catch (InterruptedException e) {
+        }
+        slept = true;
+      }
+      return super.next(scannerId, nbRows, callSeq);
+    }
+  }
+}

From 6bf1641094e5ba91e78e151a3d2dc6fa099ebc74 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 19 Sep 2014 12:01:37 -0700
Subject: [PATCH 1469/1540] HBASE-12023 HRegion.applyFamilyMapToMemstore
 creates too many iterator objects. (Vladimir Rodionov)

---
 .../java/org/apache/hadoop/hbase/regionserver/HRegion.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index eaa02d126062..bf3a1933b27a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -39,6 +39,7 @@
 import java.util.NavigableMap;
 import java.util.NavigableSet;
 import java.util.Random;
+import java.util.RandomAccess;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
@@ -3108,9 +3109,11 @@ private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
       for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
         byte[] family = e.getKey();
         List<KeyValue> edits = e.getValue();
-
+        assert edits instanceof RandomAccess;
         Store store = getStore(family);
-        for (KeyValue kv: edits) {
+        int listSize = edits.size();
+        for (int i=0; i< listSize; i++) {
+          KeyValue kv = edits.get(i);
           kv.setMemstoreTS(localizedWriteEntry.getWriteNumber());
           size += store.add(kv);
         }

From a6dbb04593cbafa672910f66ad218d6928f92265 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Tue, 23 Sep 2014 13:12:05 -0700
Subject: [PATCH 1470/1540] Amend HBASE-12023 HRegion.applyFamilyMapToMemstore
 creates too many iterator objects; More cases (Vladimir Rodionov)

---
 .../hadoop/hbase/regionserver/HRegion.java    | 26 ++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index bf3a1933b27a..5da00dc9704d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2068,9 +2068,11 @@ private void prepareDeleteTimestamps(Map<byte[], List<KeyValue>> familyMap, byte
 
       byte[] family = e.getKey();
       List<KeyValue> kvs = e.getValue();
+      assert kvs instanceof RandomAccess;
       Map<byte[], Integer> kvCount = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
-
-      for (KeyValue kv: kvs) {
+      int listSize = kvs.size();
+      for (int i=0; i < listSize; i++) {
+        KeyValue kv = kvs.get(i);
         //  Check if time is LATEST, change to time of most recent addition if so
         //  This is expensive.
         if (kv.isLatestTimestamp() && kv.isDeleteType()) {
@@ -2920,7 +2922,10 @@ private void updateKVTimestamps(
       final Iterable<List<KeyValue>> keyLists, final byte[] now) {
     for (List<KeyValue> keys: keyLists) {
       if (keys == null) continue;
-      for (KeyValue key : keys) {
+      assert keys instanceof RandomAccess;
+      int listSize = keys.size();
+      for (int i=0; i < listSize; i++) {
+        KeyValue key = keys.get(i);
         key.updateLatestStamp(now);
       }
     }
@@ -3181,7 +3186,10 @@ private void checkTimestamps(final Map<byte[], List<KeyValue>> familyMap,
     }
     long maxTs = now + timestampSlop;
     for (List<KeyValue> kvs : familyMap.values()) {
-      for (KeyValue kv : kvs) {
+      assert kvs instanceof RandomAccess;
+      int listSize = kvs.size();
+      for (int i=0; i < listSize; i++) {
+        KeyValue kv = kvs.get(i);
         // see if the user-side TS is out of range. latest = server-side
         if (!kv.isLatestTimestamp() && kv.getTimestamp() > maxTs) {
           throw new DoNotRetryIOException("Timestamp for KV out of range "
@@ -3200,7 +3208,10 @@ private void checkTimestamps(final Map<byte[], List<KeyValue>> familyMap,
   private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap,
       WALEdit walEdit) {
     for (List<KeyValue> edits : familyMap.values()) {
-      for (KeyValue kv : edits) {
+      assert edits instanceof RandomAccess;
+      int listSize = edits.size();
+      for (int i=0; i < listSize; i++) {
+        KeyValue kv = edits.get(i);
         walEdit.add(kv);
       }
     }
@@ -6069,7 +6080,10 @@ private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
 
     long putSize = 0;
     for (List<KeyValue> edits : familyMap.values()) {
-      for (KeyValue kv : edits) {
+      assert edits instanceof RandomAccess;
+      int listSize = edits.size();
+      for (int i=0; i < listSize; i++) {
+        KeyValue kv = edits.get(i);
         putSize += kv.getKeyLength() + kv.getValueLength();
       }
     }

From c0d9ac9a0977ae02be51bf23b61826c1b6523021 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Wed, 24 Sep 2014 11:12:21 -0700
Subject: [PATCH 1471/1540] HBASE-12077 FilterLists create many ArrayList
 objects per row (Lars Hofhansl)

Amending-Author: Andrew Purtell <apurtell@apache.org>
---
 .../hadoop/hbase/filter/FilterList.java       | 43 +++++++++++++------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 44baf394357f..d676655f33b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -180,8 +180,9 @@ public void addFilter(Filter filter) {
 
   @Override
   public void reset() {
-    for (Filter filter : filters) {
-      filter.reset();
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      filters.get(i).reset();
     }
     seekHintFilter = null;
   }
@@ -189,7 +190,9 @@ public void reset() {
   @Override
   public boolean filterRowKey(byte[] rowKey, int offset, int length) {
     boolean flag = (this.operator == Operator.MUST_PASS_ONE) ? true : false;
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       if (this.operator == Operator.MUST_PASS_ALL) {
         if (filter.filterAllRemaining() ||
             filter.filterRowKey(rowKey, offset, length)) {
@@ -207,7 +210,9 @@ public boolean filterRowKey(byte[] rowKey, int offset, int length) {
 
   @Override
   public boolean filterAllRemaining() {
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       if (filter.filterAllRemaining()) {
         if (operator == Operator.MUST_PASS_ALL) {
           return true;
@@ -240,7 +245,9 @@ public ReturnCode filterKeyValue(KeyValue v) {
 
     ReturnCode rc = operator == Operator.MUST_PASS_ONE?
         ReturnCode.SKIP: ReturnCode.INCLUDE;
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       if (operator == Operator.MUST_PASS_ALL) {
         if (filter.filterAllRemaining()) {
           return ReturnCode.NEXT_ROW;
@@ -299,15 +306,19 @@ public ReturnCode filterKeyValue(KeyValue v) {
 
   @Override
   public void filterRow(List<KeyValue> kvs) {
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       filter.filterRow(kvs);
     }
   }
 
   @Override
   public boolean hasFilterRow() {
-    for (Filter filter : filters) {
-      if(filter.hasFilterRow()) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
+      if (filter.hasFilterRow()) {
     	return true;
       }
     }
@@ -316,7 +327,9 @@ public boolean hasFilterRow() {
 
   @Override
   public boolean filterRow() {
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       if (operator == Operator.MUST_PASS_ALL) {
         if (filter.filterRow()) {
           return true;
@@ -346,7 +359,9 @@ public void readFields(final DataInput in) throws IOException {
   public void write(final DataOutput out) throws IOException {
     out.writeByte(operator.ordinal());
     out.writeInt(filters.size());
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       HbaseObjectWritable.writeObject(out, filter, Writable.class, CONF);
     }
   }
@@ -359,7 +374,9 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
       return keyHint;
     }
 
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       KeyValue curKeyHint = filter.getNextKeyHint(currentKV);
       if (curKeyHint == null) {
         // If we ever don't have a hint and this is must-pass-one, then no hint
@@ -381,7 +398,9 @@ public KeyValue getNextKeyHint(KeyValue currentKV) {
   }
 
   public boolean isFamilyEssential(byte[] name) {
-    for (Filter filter : filters) {
+    int listSize = filters.size();
+    for (int i=0; i < listSize; i++) {
+      Filter filter = filters.get(i);
       if (FilterBase.isFamilyEssential(filter, name)) {
         return true;
       }

From b65849bcde31a0c0c4c8eda0cc3678d441c9cd1c Mon Sep 17 00:00:00 2001
From: Ted Yu <tedyu@apache.org>
Date: Thu, 25 Sep 2014 16:46:20 +0000
Subject: [PATCH 1472/1540] HBASE-11405 Multiple invocations of hbck in
 parallel disables balancer permanently (Sean Busbey and bharath v)

---
 .../apache/hadoop/hbase/util/HBaseFsck.java   | 86 ++++++++++++++++++-
 .../hadoop/hbase/HBaseTestingUtility.java     | 59 ++++++++++++-
 .../hadoop/hbase/util/TestHBaseFsck.java      | 50 +++++++++++
 3 files changed, 191 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index e028b56decf7..469c98ec753c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.net.InetAddress;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -43,6 +44,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
@@ -50,10 +52,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -97,7 +101,9 @@
 import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
@@ -163,6 +169,8 @@ public class HBaseFsck extends Configured implements Tool {
   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
   private static final String TO_BE_LOADED = "to_be_loaded";
+  private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
+
 
   /**********************
    * Internal resources
@@ -177,6 +185,12 @@ public class HBaseFsck extends Configured implements Tool {
   private long startMillis = System.currentTimeMillis();
   private HFileCorruptionChecker hfcc;
   private int retcode = 0;
+  private Path HBCK_LOCK_PATH;
+  private FSDataOutputStream hbckOutFd;
+  // This lock is to prevent cleanup of balancer resources twice between
+  // ShutdownHook and the main code. We cleanup only if the connect() is
+  // successful
+  private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
 
   /***********
    * Options
@@ -276,11 +290,79 @@ public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunni
     this.executor = exec;
   }
 
+  /**
+   * This method maintains a lock using a file. If the creation fails we return null
+   *
+   * @return FSDataOutputStream object corresponding to the newly opened lock file
+   * @throws IOException
+   */
+  private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
+    try {
+      FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
+      FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
+          HConstants.DATA_FILE_UMASK_KEY);
+      Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
+      fs.mkdirs(tmpDir);
+      HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
+      final FSDataOutputStream out = FSUtils.create(fs, HBCK_LOCK_PATH, defaultPerms, false);
+      out.writeBytes(InetAddress.getLocalHost().toString());
+      out.flush();
+      return out;
+    } catch (IOException exception) {
+      RemoteException e = null;
+      if (exception instanceof RemoteException) {
+        e = (RemoteException)exception;
+      } else if (exception.getCause() instanceof RemoteException) {
+        e = (RemoteException)(exception.getCause());
+      }
+      if(null != e && AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
+        return null;
+      }
+      throw exception;
+    }
+  }
+
+  private void unlockHbck() {
+    if(hbckLockCleanup.compareAndSet(true, false)){
+      IOUtils.closeStream(hbckOutFd);
+      try{
+        FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
+      } catch(IOException ioe) {
+        LOG.warn("Failed to delete " + HBCK_LOCK_PATH);
+        LOG.debug(ioe);
+      }
+    }
+  }
+
   /**
    * To repair region consistency, one must call connect() in order to repair
    * online state.
    */
   public void connect() throws IOException {
+
+    // Check if another instance of balancer is running
+    hbckOutFd = checkAndMarkRunningHbck();
+    if (hbckOutFd == null) {
+      setRetCode(-1);
+      LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
+		      " no other instance is running, delete the lock file " +
+		      HBCK_LOCK_PATH + " and rerun the tool]");
+      throw new IOException("Duplicate hbck - Abort");
+    }
+
+    // Make sure to cleanup the lock
+    hbckLockCleanup.set(true);
+
+    // Add a shutdown hook to this thread, incase user tries to
+    // kill the hbck with a ctrl-c, we want to cleanup the lock so that
+    // it is available for further calls
+    Runtime.getRuntime().addShutdownHook(new Thread() {
+      public void run() {
+          unlockHbck();
+      }
+    });
+    LOG.debug("Launching hbck");
+
     admin = new HBaseAdmin(getConf());
     meta = new HTable(getConf(), HConstants.META_TABLE_NAME);
     status = admin.getMaster().getClusterStatus();
@@ -462,6 +544,9 @@ public int onlineHbck() throws IOException, KeeperException, InterruptedExceptio
 
     offlineReferenceFileRepair();
 
+    // Remove the hbck lock
+    unlockHbck();
+
     // Print table summary
     printTableSummary(tablesInfo);
     return errors.summarize();
@@ -3691,7 +3776,6 @@ public static void main(String[] args) throws Exception {
     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
     conf.set("fs.defaultFS", defaultFs.toString());     // for hadoop 0.21+
     conf.set("fs.default.name", defaultFs.toString());  // for hadoop 0.20
-
     int ret = ToolRunner.run(new HBaseFsck(conf), args);
     System.exit(ret);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index c2bcd8004346..fc548261b34b 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -25,7 +25,9 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.UnknownHostException;
@@ -88,6 +90,7 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.apache.zookeeper.WatchedEvent;
@@ -199,12 +202,18 @@ public HBaseTestingUtility(Configuration conf) {
 
     // a hbase checksum verification failure will cause unit tests to fail
     ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
-    setHDFSClientRetryProperty();
+    setHDFSClientRetry(1);
   }
 
-  private void setHDFSClientRetryProperty() {
-    this.conf.setInt("hdfs.client.retries.number", 1);
+  /**
+   * Controls how many attempts we will make in the face of failures in HDFS.
+   */
+  public void setHDFSClientRetry(final int retries) {
+    this.conf.setInt("hdfs.client.retries.number", retries);
     HBaseFileSystem.setRetryCounts(conf);
+    if (0 == retries) {
+      makeDFSClientNonRetrying();
+    }
   }
 
   /**
@@ -1924,6 +1933,50 @@ public static void setMaxRecoveryErrorCount(final OutputStream stream,
     }
   }
 
+  void makeDFSClientNonRetrying() {
+    if (null == this.dfsCluster) {
+      LOG.debug("dfsCluster has not started, can't make client non-retrying.");
+      return;
+    }
+    try {
+      final FileSystem filesystem = this.dfsCluster.getFileSystem();
+      if (!(filesystem instanceof DistributedFileSystem)) {
+        LOG.debug("dfsCluster is not backed by a DistributedFileSystem, can't make client non-retrying.");
+        return;
+      }
+      // rely on FileSystem.CACHE to alter how we talk via DFSClient
+      final DistributedFileSystem fs = (DistributedFileSystem)filesystem;
+      // retrieve the backing DFSClient instance
+      final Field dfsField = fs.getClass().getDeclaredField("dfs");
+      dfsField.setAccessible(true);
+      final Class<?> dfsClazz = dfsField.getType();
+      final DFSClient dfs = DFSClient.class.cast(dfsField.get(fs));
+
+      // expose the method for creating direct RPC connections.
+      final Method createRPCNamenode = dfsClazz.getDeclaredMethod("createRPCNamenode", InetSocketAddress.class, Configuration.class, UserGroupInformation.class);
+      createRPCNamenode.setAccessible(true);
+
+      // grab the DFSClient instance's backing connection information
+      final Field nnField = dfsClazz.getDeclaredField("nnAddress");
+      nnField.setAccessible(true);
+      final InetSocketAddress nnAddress = InetSocketAddress.class.cast(nnField.get(dfs));
+      final Field confField = dfsClazz.getDeclaredField("conf");
+      confField.setAccessible(true);
+      final Configuration conf = Configuration.class.cast(confField.get(dfs));
+      final Field ugiField = dfsClazz.getDeclaredField("ugi");
+      ugiField.setAccessible(true);
+      final UserGroupInformation ugi = UserGroupInformation.class.cast(ugiField.get(dfs));
+
+      // replace the proxy for the namenode rpc with a direct instance
+      final Field namenodeField = dfsClazz.getDeclaredField("namenode");
+      namenodeField.setAccessible(true);
+      namenodeField.set(dfs, createRPCNamenode.invoke(null, nnAddress, conf, ugi));
+      LOG.debug("Set DSFClient namenode to bare RPC");
+    } catch (Exception exception) {
+      LOG.info("Could not alter DFSClient to be non-retrying.", exception);
+    }
+  }
+
   /**
    * Wait until all regions for a table in .META. have a non-empty
    * info:server, up to 60 seconds.  This means all regions have been deployed,
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 4ec70cf7eb9a..3e95a5e6fdc1 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -35,8 +35,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
@@ -116,6 +120,7 @@ public class TestHBaseFsck {
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.getConfiguration().setBoolean("hbase.master.distributed.log.splitting", false);
     TEST_UTIL.startMiniCluster(3);
+    TEST_UTIL.setHDFSClientRetry(0);
   }
 
   @AfterClass
@@ -538,6 +543,51 @@ ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
     return null;
   }
 
+  /**
+   * This test makes sure that parallel instances of Hbck is disabled.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testParallelHbck() throws Exception {
+    final ExecutorService service;
+    final Future<HBaseFsck> hbck1,hbck2;
+
+    class RunHbck implements Callable<HBaseFsck>{
+      boolean fail = true;
+      public HBaseFsck call(){
+        try{
+          return doFsck(conf, false);
+        } catch(Exception e){
+          if (e.getMessage().contains("Duplicate hbck")) {
+            fail = false;
+          } else {
+            LOG.fatal("hbck failed.", e);
+          }
+        }
+        // If we reach here, then an exception was caught
+        if (fail) fail();
+        return null;
+      }
+    }
+    service = Executors.newFixedThreadPool(2);
+    hbck1 = service.submit(new RunHbck());
+    hbck2 = service.submit(new RunHbck());
+    service.shutdown();
+    //wait for 15 seconds, for both hbck calls finish
+    service.awaitTermination(15, TimeUnit.SECONDS);
+    HBaseFsck h1 = hbck1.get();
+    HBaseFsck h2 = hbck2.get();
+    // Make sure only one of the calls was successful
+    assert(h1 == null || h2 == null);
+    if (h1 != null) {
+      assert(h1.getRetCode() >= 0);
+    }
+    if (h2 != null) {
+      assert(h2.getRetCode() >= 0);
+    }
+  }
+
   /**
    * This create and fixes a bad table with regions that have a duplicate
    * start key

From 7e4d7f04e14c5197949caeacd289748be9b1bf5b Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 26 Sep 2014 21:12:51 -0700
Subject: [PATCH 1473/1540] HBASE-12103 Backport HFileV1Detector to 0.94.
 (Jeffrey Zhong)

---
 .../hadoop/hbase/util/HFileV1Detector.java    | 412 ++++++++++++++++++
 1 file changed, 412 insertions(+)
 create mode 100644 src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java b/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
new file mode 100644
index 000000000000..34c79454233d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
@@ -0,0 +1,412 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.io.FileLink;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Tool to detect presence of any HFileV1 in the given directory. It prints all such regions which
+ * have such files.
+ * <p>
+ * To print the help section of the tool:
+ * <ul>
+ * <li>./bin/hbase org.apache.hadoop.hbase.util.HFileV1Detector --h or,
+ * <li>java -cp `hbase classpath` org.apache.hadoop.hbase.util.HFileV1Detector --h
+ * </ul>
+ * It also supports -h, --help, -help options.
+ * </p>
+ */
+public class HFileV1Detector extends Configured implements Tool {
+  private FileSystem fs;
+  private static final Log LOG = LogFactory.getLog(HFileV1Detector.class);
+  private static final int DEFAULT_NUM_OF_THREADS = 10;
+  /**
+   * Pre-namespace archive directory
+   */
+  private static final String PRE_NS_DOT_ARCHIVE = ".archive";
+  /**
+   * Pre-namespace tmp directory
+   */
+  private static final String PRE_NS_DOT_TMP = ".tmp";
+  private int numOfThreads;
+  /**
+   * directory to start the processing.
+   */
+  private Path targetDirPath;
+  /**
+   * executor for processing regions.
+   */
+  private ExecutorService exec;
+
+  /**
+   * Keeps record of processed tables.
+   */
+  private final Set<Path> processedTables = new HashSet<Path>();
+  /**
+   * set of corrupted HFiles (with undetermined major version)
+   */
+  private final Set<Path> corruptedHFiles = Collections
+      .newSetFromMap(new ConcurrentHashMap<Path, Boolean>());
+  /**
+   * set of HfileV1;
+   */
+  private final Set<Path> hFileV1Set = Collections
+      .newSetFromMap(new ConcurrentHashMap<Path, Boolean>());
+
+  private Options options = new Options();
+
+  private Path defaultRootDir = null;
+  public HFileV1Detector() {
+    Option pathOption = new Option("p", "path", true, "Path to a table, or hbase installation");
+    pathOption.setRequired(false);
+    options.addOption(pathOption);
+    Option threadOption = new Option("n", "numberOfThreads", true,
+        "Number of threads to use while processing HFiles.");
+    threadOption.setRequired(false);
+    options.addOption(threadOption);
+    options.addOption("h", "help", false, "Help");
+  }
+
+  private boolean parseOption(String[] args) throws ParseException, IOException {
+    if (args.length == 0) {
+      return true; // no args will process with default values.
+    }
+    CommandLineParser parser = new GnuParser();
+    CommandLine cmd = parser.parse(options, args);
+    if (cmd.hasOption("h")) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp("HFileV1Detector", options, true);
+      System.out
+          .println("In case no option is provided, it processes hbase.rootdir using 10 threads.");
+      System.out.println("Example:");
+      System.out.println(" To detect any HFileV1 in a given hbase installation '/myhbase':");
+      System.out.println(" $ $HBASE_HOME/bin/hbase " + this.getClass().getName() + " -p /myhbase");
+      System.out.println();
+      return false;
+    }
+
+    if (cmd.hasOption("p")) {
+      this.targetDirPath = new Path(FSUtils.getRootDir(getConf()), cmd.getOptionValue("p"));
+    }
+    try {
+      if (cmd.hasOption("n")) {
+        int n = Integer.parseInt(cmd.getOptionValue("n"));
+        if (n < 0 || n > 100) {
+          LOG.warn("Please use a positive number <= 100 for number of threads."
+              + " Continuing with default value " + DEFAULT_NUM_OF_THREADS);
+          return true;
+        }
+        this.numOfThreads = n;
+      }
+    } catch (NumberFormatException nfe) {
+      LOG.error("Please select a valid number for threads");
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Checks for HFileV1.
+   * @return 0 when no HFileV1 is present.
+   *         1 when a HFileV1 is present or, when there is a file with corrupt major version
+   *          (neither V1 nor V2).
+   *        -1 in case of any error/exception
+   */
+  @Override
+  public int run(String args[]) throws IOException, ParseException {
+    Path root = new Path(FSUtils.getRootDir(getConf()).toUri());
+    getConf().set("fs.defaultFS", root.toString());    // for hadoop 0.21+
+    fs = FileSystem.get(getConf());
+    numOfThreads = DEFAULT_NUM_OF_THREADS;
+    targetDirPath = FSUtils.getRootDir(getConf());
+    if (!parseOption(args)) {
+      System.exit(-1);
+    }
+    this.exec = Executors.newFixedThreadPool(numOfThreads);
+    try {
+      return processResult(checkForV1Files(targetDirPath));
+    } catch (Exception e) {
+      LOG.error(e);
+    } finally {
+      exec.shutdown();
+      fs.close();
+    }
+    return -1;
+  }
+
+  private int processResult(Set<Path> regionsWithHFileV1) {
+    LOG.info("Result: \n");
+    printSet(processedTables, "Tables Processed: ");
+
+    int count = hFileV1Set.size();
+    LOG.info("Count of HFileV1: " + count);
+    if (count > 0) printSet(hFileV1Set, "HFileV1:");
+
+    count = corruptedHFiles.size();
+    LOG.info("Count of corrupted files: " + count);
+    if (count > 0) printSet(corruptedHFiles, "Corrupted Files: ");
+
+    count = regionsWithHFileV1.size();
+    LOG.info("Count of Regions with HFileV1: " + count);
+    if (count > 0) printSet(regionsWithHFileV1, "Regions to Major Compact: ");
+
+    return (hFileV1Set.isEmpty() && corruptedHFiles.isEmpty()) ? 0 : 1;
+  }
+
+  private void printSet(Set<Path> result, String msg) {
+    LOG.info(msg);
+    for (Path p : result) {
+      LOG.info(p);
+    }
+  }
+
+  /**
+   * Takes a directory path, and lists out any HFileV1, if present.
+   * @param targetDir directory to start looking for HFilev1.
+   * @return set of Regions that have HFileV1
+   * @throws IOException
+   */
+  private Set<Path> checkForV1Files(Path targetDir) throws IOException {
+    LOG.info("Target dir is: " + targetDir);
+    if (!fs.exists(targetDir)) {
+      throw new IOException("The given path does not exist: " + targetDir);
+    }
+    if (isTableDir(fs, targetDir)) {
+      processedTables.add(targetDir);
+      return processTable(targetDir);
+    }
+    Set<Path> regionsWithHFileV1 = new HashSet<Path>();
+    FileStatus[] fsStats = fs.listStatus(targetDir);
+    for (FileStatus fsStat : fsStats) {
+      if (isTableDir(fs, fsStat.getPath()) && !isRootTable(fsStat.getPath())) {
+        processedTables.add(fsStat.getPath());
+        // look for regions and find out any v1 file.
+        regionsWithHFileV1.addAll(processTable(fsStat.getPath()));
+      } else {
+        LOG.info("Ignoring path: " + fsStat.getPath());
+      }
+    }
+    return regionsWithHFileV1;
+  }
+
+  /**
+   * Ignore ROOT table as it doesn't exist in 0.96.
+   * @param path
+   */
+  private boolean isRootTable(Path path) {
+    if (path != null && path.toString().endsWith("-ROOT-")) return true;
+    return false;
+  }
+
+  /**
+   * Find out regions in the table which have HFileV1.
+   * @param tableDir
+   * @return the set of regions containing HFile v1.
+   * @throws IOException
+   */
+  private Set<Path> processTable(Path tableDir) throws IOException {
+    // list out the regions and then process each file in it.
+    LOG.debug("processing table: " + tableDir);
+    List<Future<Path>> regionLevelResults = new ArrayList<Future<Path>>();
+    Set<Path> regionsWithHFileV1 = new HashSet<Path>();
+
+    FileStatus[] fsStats = fs.listStatus(tableDir);
+    for (FileStatus fsStat : fsStats) {
+      // process each region
+      if (isRegionDir(fs, fsStat.getPath())) {
+        regionLevelResults.add(processRegion(fsStat.getPath()));
+      }
+    }
+    for (Future<Path> f : regionLevelResults) {
+      try {
+        if (f.get() != null) {
+          regionsWithHFileV1.add(f.get());
+        }
+      } catch (InterruptedException e) {
+        LOG.error(e);
+      } catch (ExecutionException e) {
+        LOG.error(e); // might be a bad hfile. We print it at the end.
+      }
+    }
+    return regionsWithHFileV1;
+  }
+
+  /**
+   * Each region is processed by a separate handler. If a HRegion has a hfileV1, its path is
+   * returned as the future result, otherwise, a null value is returned.
+   * @param regionDir Region to process.
+   * @return corresponding Future object.
+   */
+  private Future<Path> processRegion(final Path regionDir) {
+    LOG.debug("processing region: " + regionDir);
+    Callable<Path> regionCallable = new Callable<Path>() {
+      @Override
+      public Path call() throws Exception {
+        for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
+          FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir);
+          if (storeFiles == null || storeFiles.length == 0) continue;
+          for (FileStatus storeFile : storeFiles) {
+            Path storeFilePath = storeFile.getPath();
+            FSDataInputStream fsdis = null;
+            long lenToRead = 0;
+            try {
+              // check whether this path is a reference.
+              if (StoreFile.isReference(storeFilePath)) continue;
+              // check whether this path is a HFileLink.
+              else if (HFileLink.isHFileLink(storeFilePath)) {
+                FileLink fLink = getFileLinkWithPreNSPath(storeFilePath);
+                fsdis = fLink.open(fs);
+                lenToRead = fLink.getFileStatus(fs).getLen();
+              } else {
+                // a regular hfile
+                fsdis = fs.open(storeFilePath);
+                lenToRead = storeFile.getLen();
+              }
+              int majorVersion = computeMajorVersion(fsdis, lenToRead);
+              if (majorVersion == 1) {
+                hFileV1Set.add(storeFilePath);
+                // return this region path, as it needs to be compacted.
+                return regionDir;
+              }
+              if (majorVersion > 2 || majorVersion < 1) throw new IllegalArgumentException(
+                  "Incorrect major version: " + majorVersion);
+            } catch (Exception iae) {
+              corruptedHFiles.add(storeFilePath);
+              LOG.error("Got exception while reading trailer for file: "+ storeFilePath, iae);
+            } finally {
+              if (fsdis != null) fsdis.close();
+            }
+          }
+        }
+        return null;
+      }
+
+      private int computeMajorVersion(FSDataInputStream istream, long fileSize)
+       throws IOException {
+        //read up the last int of the file. Major version is in the last 3 bytes.
+        long seekPoint = fileSize - Bytes.SIZEOF_INT;
+        if (seekPoint < 0)
+          throw new IllegalArgumentException("File too small, no major version found");
+
+        // Read the version from the last int of the file.
+        istream.seek(seekPoint);
+        int version = istream.readInt();
+        // Extract and return the major version
+        return version & 0x00ffffff;
+      }
+    };
+    Future<Path> f = exec.submit(regionCallable);
+    return f;
+  }
+
+  /**
+   * Creates a FileLink which adds pre-namespace paths in its list of available paths. This is used
+   * when reading a snapshot file in a pre-namespace file layout, for example, while upgrading.
+   * @param storeFilePath
+   * @return a FileLink which could read from pre-namespace paths.
+   * @throws IOException
+   */
+  public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException {
+    HFileLink link = new HFileLink(getConf(), storeFilePath);
+    List<Path> pathsToProcess = getPreNSPathsForHFileLink(link);
+    pathsToProcess.addAll(Arrays.asList(link.getLocations()));
+    return new FileLink(pathsToProcess);
+  }
+
+  private List<Path> getPreNSPathsForHFileLink(HFileLink fileLink) throws IOException {
+    List<Path> p = new ArrayList<Path>();
+    String relativeTablePath = removeDefaultNSPath(fileLink.getOriginPath());
+    p.add(getPreNSPath(PRE_NS_DOT_ARCHIVE, relativeTablePath));
+    p.add(getPreNSPath(PRE_NS_DOT_TMP, relativeTablePath));
+    p.add(getPreNSPath(null, relativeTablePath));
+    return p;
+  }
+  
+  /**
+   * Removes the prefix of defaultNamespace from the path.
+   * @param originPath
+ * @throws IOException 
+   */
+  private String removeDefaultNSPath(Path originalPath) throws IOException {
+    if (defaultRootDir == null) {
+      defaultRootDir = FSUtils.getRootDir(getConf());
+    }
+    String pathStr = originalPath.toString();
+    if (!pathStr.startsWith(defaultRootDir.toString())) return pathStr;
+    return pathStr.substring(defaultRootDir.toString().length() + 1);
+  }
+
+  private Path getPreNSPath(String prefix, String relativeTablePath) throws IOException {
+    String relativePath = (prefix == null ? relativeTablePath : prefix + Path.SEPARATOR
+        + relativeTablePath);
+    return new Path(FSUtils.getRootDir(getConf()), relativePath);
+  }
+
+  private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
+    // check for old format, of having /table/.tableinfo; hbase:meta doesn't has .tableinfo,
+    // include it.
+    if (fs.isFile(path)) return false;
+    return (FSTableDescriptors.getTableInfoPath(fs, path) != null)
+        || path.toString().endsWith(".META.");
+  }
+
+  private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
+    if (fs.isFile(path)) return false;
+    Path regionInfo = new Path(path, ".regioninfo");
+    return fs.exists(regionInfo);
+
+  }
+
+  public static void main(String args[]) throws Exception {
+    System.exit(ToolRunner.run(HBaseConfiguration.create(), new HFileV1Detector(), args));
+  }
+
+}

From 66cfcbe1532261f42524e8e02e762007ef0796a3 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 26 Sep 2014 22:46:20 -0700
Subject: [PATCH 1474/1540] HBASE-11957 Addendum; fix
 TestMetaReaderEditorNoCluster

---
 .../hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
index 222b847b951c..5e78c247128e 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
@@ -125,7 +125,7 @@ public void testRideOverServerNotRunning() throws IOException, InterruptedExcept
         HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
         Bytes.toBytes(sn.getStartcode())));
       final Result [] result = new Result [] {new Result(kvs)};
-      Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
+      Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).
         thenReturn(result).
         thenReturn(null);
 

From b4e1be5b4095ef796961fc9194bd46a77ee224c0 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 27 Sep 2014 19:50:48 -0700
Subject: [PATCH 1475/1540] HBASE-12090 Bytes: more Unsafe, more Faster.
 (Vladimir Rodionov)

---
 .../org/apache/hadoop/hbase/util/Bytes.java   | 285 ++++++++++++++----
 1 file changed, 219 insertions(+), 66 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index 6a4130865409..b548e5e79d46 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -41,9 +41,10 @@
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.io.WritableUtils;
 
-import sun.misc.Unsafe;
-
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.util.Bytes.LexicographicalComparerHolder.UnsafeComparer;
+
+import sun.misc.Unsafe;
 
 /**
  * Utility class that handles byte arrays, conversions to/from other types,
@@ -520,12 +521,16 @@ public static long toLong(byte[] bytes, int offset, final int length) {
     if (length != SIZEOF_LONG || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
     }
-    long l = 0;
-    for(int i = offset; i < offset + length; i++) {
-      l <<= 8;
-      l ^= bytes[i] & 0xFF;
+    if (UnsafeComparer.isAvailable()) {
+      return toLongUnsafe(bytes, offset);
+    } else {
+      long l = 0;
+      for(int i = offset; i < offset + length; i++) {
+        l <<= 8;
+        l ^= bytes[i] & 0xFF;
+      }
+      return l;
     }
-    return l;
   }
 
   private static IllegalArgumentException
@@ -557,11 +562,32 @@ public static int putLong(byte[] bytes, int offset, long val) {
       throw new IllegalArgumentException("Not enough room to put a long at"
           + " offset " + offset + " in a " + bytes.length + " byte array");
     }
-    for(int i = offset + 7; i > offset; i--) {
-      bytes[i] = (byte) val;
-      val >>>= 8;
+    if (UnsafeComparer.isAvailable()) {
+      return putLongUnsafe(bytes, offset, val);
+    } else {
+      for(int i = offset + 7; i > offset; i--) {
+        bytes[i] = (byte) val;
+        val >>>= 8;
+      }
+      bytes[offset] = (byte) val;
+      return offset + SIZEOF_LONG;
+    }
+  }
+
+  /**
+   * Put a long value out to the specified byte array position (Unsafe).
+   * @param bytes the byte array
+   * @param offset position in the array
+   * @param val long to write out
+   * @return incremented offset
+   */
+  public static int putLongUnsafe(byte[] bytes, int offset, long val)
+  {
+    if (UnsafeComparer.littleEndian) {
+      val = Long.reverseBytes(val);
     }
-    bytes[offset] = (byte) val;
+    UnsafeComparer.theUnsafe.putLong(bytes, (long) offset +
+      UnsafeComparer.BYTE_ARRAY_BASE_OFFSET , val);
     return offset + SIZEOF_LONG;
   }
 
@@ -689,12 +715,64 @@ public static int toInt(byte[] bytes, int offset, final int length) {
     if (length != SIZEOF_INT || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
     }
-    int n = 0;
-    for(int i = offset; i < (offset + length); i++) {
-      n <<= 8;
-      n ^= bytes[i] & 0xFF;
+    if (UnsafeComparer.isAvailable()) {
+      return toIntUnsafe(bytes, offset);
+    } else {
+      int n = 0;
+      for(int i = offset; i < (offset + length); i++) {
+        n <<= 8;
+        n ^= bytes[i] & 0xFF;
+      }
+      return n;
+    }
+  }
+
+  /**
+   * Converts a byte array to an int value (Unsafe version)
+   * @param bytes byte array
+   * @param offset offset into array
+   * @return the int value
+   */
+  public static int toIntUnsafe(byte[] bytes, int offset) {
+    if (UnsafeComparer.littleEndian) {
+      return Integer.reverseBytes(UnsafeComparer.theUnsafe.getInt(bytes,
+        (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
+    } else {
+      return UnsafeComparer.theUnsafe.getInt(bytes,
+        (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
+    }
+  }
+
+  /**
+   * Converts a byte array to an short value (Unsafe version)
+   * @param bytes byte array
+   * @param offset offset into array
+   * @return the short value
+   */
+  public static short toShortUnsafe(byte[] bytes, int offset) {
+    if (UnsafeComparer.littleEndian) {
+      return Short.reverseBytes(UnsafeComparer.theUnsafe.getShort(bytes,
+        (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
+    } else {
+      return UnsafeComparer.theUnsafe.getShort(bytes,
+        (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
+    }
+  }
+
+  /**
+   * Converts a byte array to an long value (Unsafe version)
+   * @param bytes byte array
+   * @param offset offset into array
+   * @return the long value
+   */
+  public static long toLongUnsafe(byte[] bytes, int offset) {
+    if (UnsafeComparer.littleEndian) {
+      return Long.reverseBytes(UnsafeComparer.theUnsafe.getLong(bytes,
+        (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
+    } else {
+      return UnsafeComparer.theUnsafe.getLong(bytes,
+        (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
     }
-    return n;
   }
 
   /**
@@ -711,11 +789,32 @@ public static int putInt(byte[] bytes, int offset, int val) {
       throw new IllegalArgumentException("Not enough room to put an int at"
           + " offset " + offset + " in a " + bytes.length + " byte array");
     }
-    for(int i= offset + 3; i > offset; i--) {
-      bytes[i] = (byte) val;
-      val >>>= 8;
+    if (UnsafeComparer.isAvailable()) {
+      return putIntUnsafe(bytes, offset, val);
+    } else {
+      for(int i= offset + 3; i > offset; i--) {
+        bytes[i] = (byte) val;
+        val >>>= 8;
+      }
+      bytes[offset] = (byte) val;
+      return offset + SIZEOF_INT;
     }
-    bytes[offset] = (byte) val;
+  }
+
+  /**
+   * Put an int value out to the specified byte array position (Unsafe).
+   * @param bytes the byte array
+   * @param offset position in the array
+   * @param val int to write out
+   * @return incremented offset
+   */
+  public static int putIntUnsafe(byte[] bytes, int offset, int val)
+  {
+    if (UnsafeComparer.littleEndian) {
+      val = Integer.reverseBytes(val);
+    }
+    UnsafeComparer.theUnsafe.putInt(bytes, (long) offset +
+      UnsafeComparer.BYTE_ARRAY_BASE_OFFSET , val);
     return offset + SIZEOF_INT;
   }
 
@@ -738,7 +837,7 @@ public static byte[] toBytes(short val) {
    * @return the short value
    */
   public static short toShort(byte[] bytes) {
-    return toShort(bytes, 0, SIZEOF_SHORT);
+    return toShortUnsafe(bytes, 0);
   }
 
   /**
@@ -764,11 +863,15 @@ public static short toShort(byte[] bytes, int offset, final int length) {
     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
     }
-    short n = 0;
-    n ^= bytes[offset] & 0xFF;
-    n <<= 8;
-    n ^= bytes[offset+1] & 0xFF;
-    return n;
+    if (UnsafeComparer.isAvailable()) {
+      return toShortUnsafe(bytes, offset);
+    } else {
+      short n = 0;
+      n ^= bytes[offset] & 0xFF;
+      n <<= 8;
+      n ^= bytes[offset+1] & 0xFF;
+      return n;
+   }
   }
 
   /**
@@ -799,9 +902,30 @@ public static int putShort(byte[] bytes, int offset, short val) {
       throw new IllegalArgumentException("Not enough room to put a short at"
           + " offset " + offset + " in a " + bytes.length + " byte array");
     }
-    bytes[offset+1] = (byte) val;
-    val >>= 8;
-    bytes[offset] = (byte) val;
+    if (UnsafeComparer.isAvailable()) {
+      return putShortUnsafe(bytes, offset, val);
+    } else {
+      bytes[offset+1] = (byte) val;
+      val >>= 8;
+      bytes[offset] = (byte) val;
+      return offset + SIZEOF_SHORT;
+    }
+  }
+
+  /**
+   * Put a short value out to the specified byte array position (Unsafe).
+   * @param bytes the byte array
+   * @param offset position in the array
+   * @param val short to write out
+   * @return incremented offset
+   */
+  public static int putShortUnsafe(byte[] bytes, int offset, short val)
+  {
+    if (UnsafeComparer.littleEndian) {
+      val = Short.reverseBytes(val);
+    }
+    UnsafeComparer.theUnsafe.putShort(bytes, (long) offset +
+      UnsafeComparer.BYTE_ARRAY_BASE_OFFSET , val);
     return offset + SIZEOF_SHORT;
   }
 
@@ -1085,12 +1209,37 @@ public Object run() {
 
       /**
        * Returns true if x1 is less than x2, when both values are treated as
-       * unsigned.
+       * unsigned long.
        */
-      static boolean lessThanUnsigned(long x1, long x2) {
+      static boolean lessThanUnsignedLong(long x1, long x2) {
         return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
       }
 
+      /**
+       * Returns true if x1 is less than x2, when both values are treated as
+       * unsigned int.
+       */
+      static boolean lessThanUnsignedInt(int x1, int x2) {
+        return (x1 & 0xffffffffL) < (x2 & 0xffffffffL);
+      }
+
+      /**
+       * Returns true if x1 is less than x2, when both values are treated as
+       * unsigned short.
+       */
+      static boolean lessThanUnsignedShort(short x1, short x2) {
+        return (x1 & 0xffff) < (x2 & 0xffff);
+      }
+
+      /**
+       * Checks if Unsafe is available
+       * @return true, if available, false - otherwise
+       */
+      public static boolean isAvailable()
+      {
+        return theUnsafe != null;
+      }
+
       /**
        * Lexicographically compare two arrays.
        *
@@ -1105,16 +1254,17 @@ static boolean lessThanUnsigned(long x1, long x2) {
       @Override
       public int compareTo(byte[] buffer1, int offset1, int length1,
           byte[] buffer2, int offset2, int length2) {
+
         // Short circuit equal case
         if (buffer1 == buffer2 &&
             offset1 == offset2 &&
             length1 == length2) {
           return 0;
         }
-        int minLength = Math.min(length1, length2);
-        int minWords = minLength / SIZEOF_LONG;
-        int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
-        int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
+        final int minLength = Math.min(length1, length2);
+        final int minWords = minLength / SIZEOF_LONG;
+        final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
+        final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
 
         /*
          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
@@ -1125,40 +1275,43 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
           long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
           long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
           long diff = lw ^ rw;
-
+          if(littleEndian){
+            lw = Long.reverseBytes(lw);
+            rw = Long.reverseBytes(rw);
+          }
           if (diff != 0) {
-            if (!littleEndian) {
-              return lessThanUnsigned(lw, rw) ? -1 : 1;
-            }
-
-            // Use binary search
-            int n = 0;
-            int y;
-            int x = (int) diff;
-            if (x == 0) {
-              x = (int) (diff >>> 32);
-              n = 32;
-            }
-
-            y = x << 16;
-            if (y == 0) {
-              n += 16;
-            } else {
-              x = y;
-            }
-
-            y = x << 8;
-            if (y == 0) {
-              n += 8;
-            }
-            return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+              return lessThanUnsignedLong(lw, rw) ? -1 : 1;
           }
         }
-
-        // The epilogue to cover the last (minLength % 8) elements.
-        for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
-          int a = (buffer1[offset1 + i] & 0xff);
-          int b = (buffer2[offset2 + i] & 0xff);
+        int offset = minWords * SIZEOF_LONG;
+
+        if (minLength - offset >= SIZEOF_INT) {
+          int il = theUnsafe.getInt(buffer1, offset1Adj + offset);
+          int ir = theUnsafe.getInt(buffer2, offset2Adj + offset);
+          if(littleEndian){
+            il = Integer.reverseBytes(il);
+            ir = Integer.reverseBytes(ir);
+          }
+          if(il != ir){
+            return lessThanUnsignedInt(il, ir) ? -1: 1;
+          }
+           offset += SIZEOF_INT;
+        }
+        if (minLength - offset >= SIZEOF_SHORT) {
+          short sl = theUnsafe.getShort(buffer1, offset1Adj + offset);
+          short sr = theUnsafe.getShort(buffer2, offset2Adj + offset);
+          if(littleEndian){
+            sl = Short.reverseBytes(sl);
+            sr = Short.reverseBytes(sr);
+          }
+          if(sl != sr){
+            return lessThanUnsignedShort(sl, sr) ? -1: 1;
+          }
+          offset += SIZEOF_SHORT;
+        }
+        if (minLength - offset == 1) {
+          int a = (buffer1[(int)(offset1 + offset)] & 0xff);
+          int b = (buffer2[(int)(offset2 + offset)] & 0xff);
           if (a != b) {
             return a - b;
           }

From 9f5c397e27c79124366041b3a93b49aa85abb2be Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 27 Sep 2014 21:02:37 -0700
Subject: [PATCH 1476/1540] HBASE-11957 addendum 2; fix TestAssignmentManager

---
 .../apache/hadoop/hbase/master/TestAssignmentManager.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 2f9779f607b2..6d1b589014c9 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -597,7 +597,7 @@ private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager a
     Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
       thenReturn(System.currentTimeMillis());
     // Return a good result first and then return null to indicate end of scan
-    Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
+    Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).
       thenReturn(new Result [] {r}, (Result [])null);
 
     // Get a connection w/ mocked up common methods.
@@ -835,14 +835,14 @@ private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final
     Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
       thenReturn(System.currentTimeMillis());
    if (enabling) {
-      Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
+      Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(result, result, result,
           (Result[]) null);
       // If a get, return the above result too for REGIONINFO_2
       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
           getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
     } else {
       // Return good result 'r' first and then return null to indicate end of scan
-      Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
+      Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(new Result[] { r });
       // If a get, return the above result too for REGIONINFO
       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
     }

From baccf6c9d434132cc027fc9ed28d06aefc25db77 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 27 Sep 2014 21:09:04 -0700
Subject: [PATCH 1477/1540] HBASE-11957 addendum 2; fix TestAssignmentManager

---
 src/main/java/org/apache/hadoop/hbase/util/Bytes.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index b548e5e79d46..6a4c25076281 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -837,7 +837,7 @@ public static byte[] toBytes(short val) {
    * @return the short value
    */
   public static short toShort(byte[] bytes) {
-    return toShortUnsafe(bytes, 0);
+    return toShort(bytes, 0, SIZEOF_SHORT);
   }
 
   /**

From ec6ea58df0361cb3f8bb62ed8a55240032957b8c Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sun, 28 Sep 2014 13:07:57 -0700
Subject: [PATCH 1478/1540] HBASE-12113 Backport to 0.94: HBASE-5525 Truncate
 and preserve region boundaries option. (Sean Busbey, original patch by Kevin
 Odell)

---
 src/main/ruby/hbase/admin.rb                  | 19 ++++++++++
 src/main/ruby/shell.rb                        |  1 +
 .../ruby/shell/commands/truncate_preserve.rb  | 38 +++++++++++++++++++
 3 files changed, 58 insertions(+)
 create mode 100644 src/main/ruby/shell/commands/truncate_preserve.rb

diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index df2c69d2b7bc..266f1ffc89d9 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -21,6 +21,7 @@
 include Java
 java_import org.apache.hadoop.hbase.util.Pair
 java_import org.apache.hadoop.hbase.util.RegionSplitter
+java_import org.apache.hadoop.hbase.util.Bytes
 java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription
 
 # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
@@ -325,6 +326,24 @@ def truncate(table_name, conf = @conf)
       @admin.createTable(table_description)
     end
 
+    #----------------------------------------------------------------------------------------------
+    # Truncates table while maintaing region boundaries (deletes all records by recreating the table)
+    def truncate_preserve(table_name, conf = @conf)
+      h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name)
+      splits = h_table.getRegionLocations().keys().map{|i| Bytes.toString(i.getStartKey)}.delete_if{|k| k == ""}.to_java :String
+      splits = org.apache.hadoop.hbase.util.Bytes.toByteArrays(splits)
+      table_description = h_table.getTableDescriptor()
+      yield 'Disabling table...' if block_given?
+      disable(table_name)
+
+      yield 'Dropping table...' if block_given?
+      drop(table_name)
+
+      yield 'Creating table with region boundaries...' if block_given?
+      @admin.createTable(table_description, splits)
+    end
+
+    #----------------------------------------------------------------------------------------------
     # Check the status of alter command (number of regions reopened)
     def alter_status(table_name)
       # Table name should be a string
diff --git a/src/main/ruby/shell.rb b/src/main/ruby/shell.rb
index 1330b7e834b0..29b08f6ead02 100644
--- a/src/main/ruby/shell.rb
+++ b/src/main/ruby/shell.rb
@@ -251,6 +251,7 @@ def help_footer
     put
     scan
     truncate
+    truncate_preserve
   ]
 )
 
diff --git a/src/main/ruby/shell/commands/truncate_preserve.rb b/src/main/ruby/shell/commands/truncate_preserve.rb
new file mode 100644
index 000000000000..918b23289a9f
--- /dev/null
+++ b/src/main/ruby/shell/commands/truncate_preserve.rb
@@ -0,0 +1,38 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class TruncatePreserve < Command
+      def help
+        return <<-EOF
+  Disables, drops and recreates the specified table while still maintaing the previous region boundaries.
+EOF
+      end
+
+      def command(table)
+        format_simple_command do
+          puts "Truncating '#{table}' table (it may take a while):"
+          admin.truncate_preserve(table) { |log| puts " - #{log}" }
+        end
+      end
+
+    end
+  end
+end

From e8e7d71ada4db6b1ca9dc53d50dfc0bb59155103 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sun, 28 Sep 2014 13:12:24 -0700
Subject: [PATCH 1479/1540] CHANGES.txt, pom.xml for 0.94.24RC0

---
 CHANGES.txt | 26 ++++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index cd1485b780eb..ab23bc269846 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,30 @@
 HBase Change Log
+Release 0.94.24 - 09/28/2014
+Sub-task
+
+    [HBASE-11923] - Potential race condition in RecoverableZookeeper.checkZk()
+    [HBASE-11963] - Synchronize peer cluster replication connection attempts
+    [HBASE-12023] - HRegion.applyFamilyMapToMemstore creates too many iterator objects.
+    [HBASE-12077] - FilterLists create many ArrayList$Itr objects per row.
+
+Bug
+
+    [HBASE-11405] - Multiple invocations of hbck in parallel disables balancer permanently
+    [HBASE-11957] - Backport to 0.94 HBASE-5974 Scanner retry behavior with RPC timeout on next() seems incorrect
+    [HBASE-12019] - hbase-daemon.sh overwrite HBASE_ROOT_LOGGER and HBASE_SECURITY_LOGGER variables
+    [HBASE-12020] - String formatting on each RPC Invoke
+    [HBASE-12022] - Payloads on Failure attempt to serialize the byte[] into strings.
+
+Improvement
+
+    [HBASE-12090] - Bytes: more Unsafe, more Faster
+
+Task
+
+    [HBASE-12103] - Backport HFileV1Detector to 0.94
+    [HBASE-12113] - Backport to 0.94: HBASE-5525 Truncate and preserve region boundaries option
+
+
 Release 0.94.23 - 08/26/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index aab0590d5d75..d90f121d3bf2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.24-SNAPSHOT</version>
+  <version>0.94.24</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From ffd3c383abb863c29b00b2e016d8116d7f9c2c1a Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 29 Sep 2014 11:34:56 -0700
Subject: [PATCH 1480/1540] HBASE-12114 Meta table cache hashing may access the
 wrong table. (He Liangliang)

---
 .../hbase/client/HConnectionManager.java      | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index adf629b3e640..cacc535c7d4a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -75,6 +75,7 @@
 import org.apache.hadoop.hbase.ipc.RpcEngine;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HashedBytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.SoftValueSortedMap;
 import org.apache.hadoop.hbase.util.Threads;
@@ -584,12 +585,11 @@ static class HConnectionImplementation implements HConnection, Closeable {
       new ConcurrentHashMap<String, String>();
 
     /**
-     * Map of table to table {@link HRegionLocation}s.  The table key is made
-     * by doing a {@link Bytes#mapKey(byte[])} of the table's name.
+     * Map of table to table {@link HRegionLocation}s.
      */
-    private final Map<Integer, SoftValueSortedMap<byte [], HRegionLocation>>
+    private final Map<HashedBytes, SoftValueSortedMap<byte [], HRegionLocation>>
       cachedRegionLocations =
-        new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();
+        new HashMap<HashedBytes, SoftValueSortedMap<byte [], HRegionLocation>>();
 
     // The presence of a server in the map implies it's likely that there is an
     // entry in cachedRegionLocations that map to this server; but the absence
@@ -600,8 +600,8 @@ static class HConnectionImplementation implements HConnection, Closeable {
 
     // region cache prefetch is enabled by default. this set contains all
     // tables whose region cache prefetch are disabled.
-    private final Set<Integer> regionCachePrefetchDisabledTables =
-      new CopyOnWriteArraySet<Integer>();
+    private final Set<HashedBytes> regionCachePrefetchDisabledTables =
+      new CopyOnWriteArraySet<HashedBytes>();
 
     private int refCount;
 
@@ -1358,7 +1358,7 @@ private void clearCachedLocationForServer(final String server) {
     private SoftValueSortedMap<byte [], HRegionLocation> getTableLocations(
         final byte [] tableName) {
       // find the map of cached locations for this table
-      Integer key = Bytes.mapKey(tableName);
+      HashedBytes key = new HashedBytes(tableName);
       SoftValueSortedMap<byte [], HRegionLocation> result;
       synchronized (this.cachedRegionLocations) {
         result = this.cachedRegionLocations.get(key);
@@ -1383,7 +1383,7 @@ public void clearRegionCache() {
     @Override
     public void clearRegionCache(final byte [] tableName) {
       synchronized (this.cachedRegionLocations) {
-        this.cachedRegionLocations.remove(Bytes.mapKey(tableName));
+        this.cachedRegionLocations.remove(new HashedBytes(tableName));
       }
     }
 
@@ -1792,10 +1792,9 @@ public <R> void processBatchCallback(
      * from a unit test.
      */
     int getNumberOfCachedRegionLocations(final byte[] tableName) {
-      Integer key = Bytes.mapKey(tableName);
       synchronized (this.cachedRegionLocations) {
         Map<byte[], HRegionLocation> tableLocs =
-          this.cachedRegionLocations.get(key);
+          this.cachedRegionLocations.get(new HashedBytes(tableName));
 
         if (tableLocs == null) {
           return 0;
@@ -1819,15 +1818,15 @@ boolean isRegionCached(final byte[] tableName, final byte[] row) {
     public void setRegionCachePrefetch(final byte[] tableName,
         final boolean enable) {
       if (!enable) {
-        regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName));
+        regionCachePrefetchDisabledTables.add(new HashedBytes(tableName));
       }
       else {
-        regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName));
+        regionCachePrefetchDisabledTables.remove(new HashedBytes(tableName));
       }
     }
 
     public boolean getRegionCachePrefetch(final byte[] tableName) {
-      return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName));
+      return !regionCachePrefetchDisabledTables.contains(new HashedBytes(tableName));
     }
 
     @Override

From a6d8e41cde8c93ed326d1ee95c40720d4530e6da Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 29 Sep 2014 11:37:17 -0700
Subject: [PATCH 1481/1540] CHANGES.txt for 0.94.24RC1

---
 CHANGES.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index ab23bc269846..805e94f415a8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
 HBase Change Log
-Release 0.94.24 - 09/28/2014
+Release 0.94.24 - 09/29/2014
 Sub-task
 
     [HBASE-11923] - Potential race condition in RecoverableZookeeper.checkZk()
@@ -14,6 +14,7 @@ Bug
     [HBASE-12019] - hbase-daemon.sh overwrite HBASE_ROOT_LOGGER and HBASE_SECURITY_LOGGER variables
     [HBASE-12020] - String formatting on each RPC Invoke
     [HBASE-12022] - Payloads on Failure attempt to serialize the byte[] into strings.
+    [HBASE-12114] - Meta table cache hashing may access the wrong table
 
 Improvement
 

From 68ed793a1d89226f8fb1ee1ee618c1e7779d46cd Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 29 Sep 2014 16:04:05 -0700
Subject: [PATCH 1482/1540] HBASE-12019 Addendum; set correct appender

---
 bin/hbase-daemon.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/hbase-daemon.sh b/bin/hbase-daemon.sh
index f54142223e33..9c49732bb29c 100755
--- a/bin/hbase-daemon.sh
+++ b/bin/hbase-daemon.sh
@@ -118,11 +118,11 @@ export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME
 export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log
 
 if [ -z "${HBASE_ROOT_LOGGER}" ]; then
-export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"}
+export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,DRFA"}
 fi
 
 if [ -z "${HBASE_SECURITY_LOGGER}" ]; then 
-export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"}
+export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,DRFAS"}
 fi
 
 logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out

From 14eb97c8d92a6e20119975cf8d240bb321c6ee03 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Wed, 1 Oct 2014 14:53:03 -0700
Subject: [PATCH 1483/1540] HBASE-12065 Import tool is not restoring multiple
 DeleteFamily markers of a row (Maddineni Sukumar)

---
 .../apache/hadoop/hbase/mapreduce/Import.java | 14 ++-
 .../hbase/mapreduce/TestImportExport.java     | 97 ++++++++++++++++++-
 2 files changed, 107 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 433eaca7aa69..d1ceaff45c01 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -148,7 +148,19 @@ private void writeResult(ImmutableBytesWritable key, Result result, Context cont
   
           kv = convertKv(kv, cfRenameMap);
           // Deletes and Puts are gathered and written when finished
-          if (kv.isDelete()) {
+          /*
+           * If there are sequence of mutations and tombstones in an Export, and after Import the same
+           * sequence should be restored as it is. If we combine all Delete tombstones into single
+           * request then there is chance of ignoring few DeleteFamily tombstones, because if we
+           * submit multiple DeleteFamily tombstones in single Delete request then we are maintaining
+           * only newest in hbase table and ignoring other. Check - HBASE-12065
+           */
+          if (kv.isDeleteFamily()) {
+            Delete deleteFamily = new Delete(key.get());
+            deleteFamily.addDeleteMarker(kv);
+            deleteFamily.setClusterId(clusterId);
+            context.write(key, deleteFamily);
+          } else if (kv.isDelete()) {
             if (delete == null) {
               delete = new Delete(key.get());
             }
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index 569364c85c1f..d77b9a420e75 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -17,9 +17,7 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 
@@ -321,6 +319,99 @@ public void testWithFilter() throws Exception {
     exportTable.close();
     importTable.close();
   }
+  
+  
+  @Test
+  public void testWithMultipleDeleteFamilyMarkersOfSameRowSameFamily() throws Exception {
+    String EXPORT_TABLE = "exportWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
+    HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILYA)
+        .setMaxVersions(5)
+        .setKeepDeletedCells(true)
+    );
+    UTIL.getHBaseAdmin().createTable(desc);
+    HTable exportT = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
+
+    //Add first version of QUAL
+    Put p = new Put(ROW1);
+    p.add(FAMILYA, QUAL, now, QUAL);
+    exportT.put(p);
+
+    //Add Delete family marker
+    Delete d = new Delete(ROW1, now+3);
+    exportT.delete(d);
+
+    //Add second version of QUAL
+    p = new Put(ROW1);
+    p.add(FAMILYA, QUAL, now+5, "s".getBytes());
+    exportT.put(p);
+
+    //Add second Delete family marker
+    d = new Delete(ROW1, now+7);
+    exportT.delete(d);
+
+    String[] args = new String[] {
+        "-D" + Export.RAW_SCAN + "=true",
+        EXPORT_TABLE,
+        OUTPUT_DIR,
+        "1000", // max number of key versions per key to export
+    };
+
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
+        UTIL.getConfiguration()), args);
+    Configuration conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+
+    String IMPORT_TABLE = "importWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
+    desc = new HTableDescriptor(IMPORT_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILYA)
+        .setMaxVersions(5)
+        .setKeepDeletedCells(true)
+    );
+    UTIL.getHBaseAdmin().createTable(desc);
+    
+    HTable importT = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
+    args = new String[] {
+        IMPORT_TABLE,
+        OUTPUT_DIR
+    };
+
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+
+    Scan s = new Scan();
+    s.setMaxVersions();
+    s.setRaw(true);
+    
+    ResultScanner importedTScanner = importT.getScanner(s);
+    Result importedTResult = importedTScanner.next();
+    
+    ResultScanner exportedTScanner = exportT.getScanner(s);
+    Result  exportedTResult =  exportedTScanner.next();
+    try
+    {
+      Result.compareResults(exportedTResult, importedTResult);
+    }
+    catch (Exception e) {
+      fail("Original and imported tables data comparision failed with error:"+e.getMessage());
+    }
+    finally
+    {
+      exportT.close();
+      importT.close();
+    }
+  }
 
   /**
    * Count the number of keyvalues in the specified table for the given timerange

From 02bc5141470bd04040603e70cfbba6c190371d6f Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 4 Oct 2014 11:39:05 -0700
Subject: [PATCH 1484/1540] HBASE-12171 Backport: PerformanceEvaluation:
 getSplits doesn't provide right splits. (Jean-Marc Spaggiari)

---
 .../org/apache/hadoop/hbase/PerformanceEvaluation.java   | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 6d73e45ba30b..1c2ae3b49466 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -551,10 +551,11 @@ protected  byte[][] getSplits() {
     if (this.presplitRegions == 0)
       return new byte [0][];
 
-    byte[][] splits = new byte[this.presplitRegions][];
-    int jump = this.R  / this.presplitRegions;
-    for (int i=0; i <this.presplitRegions; i++) {
-      int rowkey = jump * i;
+    int numSplitPoints = this.presplitRegions - 1;
+    byte[][] splits = new byte[numSplitPoints][];
+    int jump = this.R / this.presplitRegions;
+    for (int i = 0; i < numSplitPoints; i++) {
+      int rowkey = jump * (1 + i);
       splits[i] = format(rowkey);
     }
     return splits;

From fbf7b65303cceb672ab74ee66be268d4e2c19c65 Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Sat, 4 Oct 2014 15:35:57 -0700
Subject: [PATCH 1485/1540] HBASE-12146 RegionServerTracker should escape data
 in log messages

---
 .../apache/hadoop/hbase/zookeeper/RegionServerTracker.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
index b89306c625f8..d8f0491869aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
@@ -47,7 +47,7 @@
  */
 public class RegionServerTracker extends ZooKeeperListener {
   private static final Log LOG = LogFactory.getLog(RegionServerTracker.class);
-  private NavigableMap<ServerName, RegionServerInfo> regionServers = 
+  private NavigableMap<ServerName, RegionServerInfo> regionServers =
 		  new TreeMap<ServerName, RegionServerInfo>();
   private ServerManager serverManager;
   private Abortable abortable;
@@ -84,7 +84,7 @@ private void add(final List<String> servers) throws IOException {
           try {
             String nodePath = ZKUtil.joinZNode(watcher.rsZNode, n);
             byte[] data = ZKUtil.getData(watcher, nodePath);
-            LOG.info("Rs node: " + nodePath + " data: " + Bytes.toString(data));
+            LOG.info("Added tracking of RS " + nodePath);
             if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) {
               int magicLen = ProtobufUtil.lengthOfPBMagic();
               rsInfoBuilder.mergeFrom(data, magicLen, data.length - magicLen);
@@ -141,7 +141,7 @@ public void nodeChildrenChanged(String path) {
   public RegionServerInfo getRegionServerInfo(final ServerName sn) {
     return regionServers.get(sn);
   }
-  
+
   /**
    * Gets the online servers.
    * @return list of online servers

From f8bc9fd0cdcb484dfb1decf75bbc0ddaf0d67388 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Mon, 6 Oct 2014 23:25:37 -0700
Subject: [PATCH 1486/1540] HBASE-12039 Lower log level for
 TableNotFoundException log message when throwing (stack)

---
 .../java/org/apache/hadoop/hbase/client/HConnectionManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index cacc535c7d4a..7b688698f4c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1059,7 +1059,7 @@ public boolean processRow(Result result) throws IOException {
         MetaScanner.metaScan(conf, this, visitor, tableName, row,
             this.prefetchRegionLimit, HConstants.META_TABLE_NAME);
       } catch (IOException e) {
-        LOG.warn("Encountered problems when prefetch META table: ", e);
+        // ignore during prefetch
       }
     }
 

From 958edae673781e7fdee8bb405b87676ebecb4f45 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 10 Oct 2014 22:29:14 -0700
Subject: [PATCH 1487/1540] 0.94.25-SNAPSHOT

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d90f121d3bf2..7e8fe443059f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.24</version>
+  <version>0.94.25-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 45eb18ba5eda6c66e0b53377b9722b062cd3ee3c Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Sun, 12 Oct 2014 18:28:32 -0700
Subject: [PATCH 1488/1540] HBASE-12235 Backport to 0.94: HBASE-9002
 TestDistributedLogSplitting.testRecoverdEdits should test correct region
 (Sean Busbey)

---
 .../hadoop/hbase/master/TestDistributedLogSplitting.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index d4237fc0bc21..51e4eba94cf4 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -226,9 +226,16 @@ public void testRecoveredEdits() throws Exception {
     List<HRegionInfo> regions = null;
     HRegionServer hrs = null;
     for (int i = 0; i < NUM_RS; i++) {
+      boolean foundRS = false;
       hrs = rsts.get(i).getRegionServer();
       regions = hrs.getOnlineRegions();
-      if (regions.size() != 0) break;
+      for (HRegionInfo region : regions) {
+        if (region.getTableNameAsString().equalsIgnoreCase("table")) {
+          foundRS = true;
+          break;
+        }
+      }
+      if (foundRS) break;
     }
     final Path logDir = new Path(rootdir, HLog.getHLogDirectoryName(hrs
         .getServerName().toString()));

From 23c71579caf4884aaa6bea894e647c643afbc64c Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Wed, 29 Oct 2014 14:57:31 -0700
Subject: [PATCH 1489/1540] HBASE-12376 HBaseAdmin leaks ZK connections if
 failure starting watchers (ConnectionLossException)

---
 .../hadoop/hbase/catalog/CatalogTracker.java  | 24 ++++++++---
 .../hadoop/hbase/client/HBaseAdmin.java       | 34 +++++++++++++--
 .../apache/hadoop/hbase/client/TestAdmin.java | 43 +++++++++++++++++++
 3 files changed, 91 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
index dcdd53c58efd..912da55042bd 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -26,6 +26,8 @@
 import java.net.UnknownHostException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -48,7 +50,7 @@
 /**
  * Tracks the availability of the catalog tables <code>-ROOT-</code> and
  * <code>.META.</code>.
- * 
+ *
  * This class is "read-only" in that the locations of the catalog tables cannot
  * be explicitly set.  Instead, ZooKeeper is used to learn of the availability
  * and location of <code>-ROOT-</code>.  <code>-ROOT-</code> is used to learn of
@@ -65,7 +67,7 @@ public class CatalogTracker {
   // servers when they needed to know of root and meta movement but also by
   // client-side (inside in HTable) so rather than figure root and meta
   // locations on fault, the client would instead get notifications out of zk.
-  // 
+  //
   // But this original intent is frustrated by the fact that this class has to
   // read an hbase table, the -ROOT- table, to figure out the .META. region
   // location which means we depend on an HConnection.  HConnection will do
@@ -151,7 +153,7 @@ public CatalogTracker(final Configuration conf) throws IOException {
    * @param abortable If fatal exception we'll call abort on this.  May be null.
    * If it is we'll use the Connection associated with the passed
    * {@link Configuration} as our Abortable.
-   * @throws IOException 
+   * @throws IOException
    */
   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
       Abortable abortable)
@@ -205,7 +207,7 @@ public void nodeDeleted(String path) {
    * Determines current availability of catalog tables and ensures all further
    * transitions of either region are tracked.
    * @throws IOException
-   * @throws InterruptedException 
+   * @throws InterruptedException
    */
   public void start() throws IOException, InterruptedException {
     LOG.debug("Starting catalog tracker " + this);
@@ -219,6 +221,14 @@ public void start() throws IOException, InterruptedException {
     }
   }
 
+  /**
+   * @return True if we are stopped. Call only after start else indeterminate answer.
+   */
+  @VisibleForTesting
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
   /**
    * Stop working.
    * Interrupts any ongoing waits.
@@ -253,7 +263,7 @@ public void stop() {
    * not currently available.
    * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
    * if none available
-   * @throws InterruptedException 
+   * @throws InterruptedException
    */
   public ServerName getRootLocation() throws InterruptedException {
     return this.rootRegionTracker.getRootRegionLocation();
@@ -535,7 +545,7 @@ private HRegionInterface getCachedConnection(ServerName sn)
       } else {
         throw ioe;
       }
-      
+
     }
     return protocol;
   }
@@ -595,7 +605,7 @@ private boolean verifyRegionLocation(HRegionInterface hostingServer,
    * the internal call to {@link #waitForRootServerConnection(long)}.
    * @return True if the <code>-ROOT-</code> location is healthy.
    * @throws IOException
-   * @throws InterruptedException 
+   * @throws InterruptedException
    */
   public boolean verifyRootRegionLocation(final long timeout)
   throws InterruptedException, IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 030e6c6456ac..c30b676ad7af 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -33,6 +33,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -182,21 +184,47 @@ public HBaseAdmin(HConnection connection)
    * @throws IOException
    * @see #cleanupCatalogTracker(CatalogTracker)
    */
-  private synchronized CatalogTracker getCatalogTracker()
+  @VisibleForTesting
+  synchronized CatalogTracker getCatalogTracker()
   throws ZooKeeperConnectionException, IOException {
+    boolean succeeded = false;
     CatalogTracker ct = null;
     try {
       ct = new CatalogTracker(this.conf);
-      ct.start();
+      startCatalogTracker(ct);
+      succeeded = true;
     } catch (InterruptedException e) {
       // Let it out as an IOE for now until we redo all so tolerate IEs
       Thread.currentThread().interrupt();
       throw new IOException("Interrupted", e);
+    } finally {
+      // If we did not succeed but created a catalogtracker, clean it up. CT has a ZK instance
+      // in it and we'll leak if we don't do the 'stop'.
+      if (!succeeded && ct != null) {
+        try {
+          ct.stop();
+        } catch (RuntimeException re) {
+          LOG.error("Failed to clean up HBase's internal catalog tracker after a failed initialization. " +
+            "We may have leaked network connections to ZooKeeper; they won't be cleaned up until " +
+            "the JVM exits. If you see a large number of stale connections to ZooKeeper this is likely " +
+            "the cause. The following exception details will be needed for assistance from the " +
+            "HBase community.", re);
+        }
+        ct = null;
+      }
     }
     return ct;
   }
 
-  private void cleanupCatalogTracker(final CatalogTracker ct) {
+  @VisibleForTesting
+  CatalogTracker startCatalogTracker(final CatalogTracker ct)
+  throws IOException, InterruptedException {
+    ct.start();
+    return ct;
+  }
+
+  @VisibleForTesting
+  void cleanupCatalogTracker(final CatalogTracker ct) {
     ct.stop();
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index c44543372bbc..f2495508d7c3 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -24,6 +24,7 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import org.apache.zookeeper.KeeperException;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,6 +34,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -109,6 +111,47 @@ public void setUp() throws Exception {
   public void tearDown() throws Exception {
   }
 
+  @Test (timeout=300000)
+  public void testFailedCatalogTrackerGetCleansUpProperly()
+  throws IOException {
+    // An HBaseAdmin that we can make fail when it goes to get catalogtracker.
+    final AtomicBoolean fail = new AtomicBoolean(false);
+    final AtomicReference<CatalogTracker> internalCt = new AtomicReference<CatalogTracker>();
+    HBaseAdmin doctoredAdmin = new HBaseAdmin(this.admin.getConfiguration()) {
+      @Override
+      protected CatalogTracker startCatalogTracker(CatalogTracker ct)
+      throws IOException, InterruptedException {
+        internalCt.set(ct);
+        super.startCatalogTracker(ct);
+        if (fail.get()) {
+          throw new IOException("Intentional test fail",
+            new KeeperException.ConnectionLossException());
+        }
+        return ct;
+      }
+    };
+    try {
+      CatalogTracker ct = doctoredAdmin.getCatalogTracker();
+      assertFalse(ct.isStopped());
+      doctoredAdmin.cleanupCatalogTracker(ct);
+      assertTrue(ct.isStopped());
+      // Now have mess with our doctored admin and make the start of catalog tracker 'fail'.
+      fail.set(true);
+      boolean expectedException = false;
+      try {
+        doctoredAdmin.getCatalogTracker();
+      } catch (IOException ioe) {
+        assertTrue(ioe.getCause() instanceof KeeperException.ConnectionLossException);
+        expectedException = true;
+      }
+      if (!expectedException) fail("Didn't get expected exception!");
+      // Assert that the internally made ct was properly shutdown.
+      assertTrue("Internal CatalogTracker not closed down", internalCt.get().isStopped());
+    } finally {
+      doctoredAdmin.close();
+    }
+  }
+
   @Test
   public void testSplitFlushCompactUnknownTable() throws InterruptedException {
     final String unknowntable = "fubar";

From 7864744e05058ac0173ac055e9ade0dd0d7fbedd Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Wed, 29 Oct 2014 22:05:45 -0700
Subject: [PATCH 1490/1540] HBASE-12336 RegionServer failed to shutdown for
 NodeFailoverWorker thread (Liu Shaohui)

---
 .../hbase/replication/regionserver/ReplicationSourceManager.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 86c16d14f479..bee69e6116c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -138,6 +138,7 @@ public ReplicationSourceManager(final ReplicationZookeeper zkHelper,
         new LinkedBlockingQueue<Runnable>());
     ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
     tfb.setNameFormat("ReplicationExecutor-%d");
+    tfb.setDaemon(true);
     this.executor.setThreadFactory(tfb.build());
     this.rand = new Random();
   }

From f0a8640f0ae3c5750da826e4ab5b847ad1b0ae34 Mon Sep 17 00:00:00 2001
From: Sean Busbey <busbey@apache.org>
Date: Thu, 30 Oct 2014 11:28:35 -0500
Subject: [PATCH 1491/1540] HBASE-12381 use the Maven Enforcer Plugin to check
 maven and java versions.

Signed-off-by: stack <stack@apache.org>

Conflicts:
	pom.xml
---
 pom.xml | 46 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 7e8fe443059f..e965e78454c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -591,6 +591,44 @@
           <outputDirectory>${project.build.directory}/generated-sources/java</outputDirectory>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <!-- version set by parent -->
+        <configuration>
+          <rules>
+            <!-- The earliest maven version we verify builds for via ASF Jenkins -->
+            <requireMavenVersion>
+              <version>[${maven.min.version},)</version>
+              <message>Maven is out of date.
+  HBase requires at least version ${maven.min.version} of Maven to properly build from source.
+  You appear to be using an older version. You can use either "mvn -version" or
+  "mvn enforcer:display-info" to verify what version is active.
+  See the reference guide on building for more information: http://hbase.apache.org/book.html#build
+              </message>
+            </requireMavenVersion>
+            <!-- The earliest JVM version we verify builds for via ASF Jenkins -->
+            <requireJavaVersion>
+              <version>[${java.min.version},)</version>
+              <message>Java is out of date.
+  HBase requirs at least version ${java.min.version} of the JDK to properly build from source.
+  You appear to be using an older version. You can use either "mvn -version" or
+  "mvn enforcer:display-info" to verify what version is active.
+  See the reference guide on building for more information: http://hbase.apache.org/book.html#build
+              </message>
+            </requireJavaVersion>
+          </rules>
+        </configuration>
+        <executions>
+          <execution>
+            <id>enforce</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <!-- parent-module only plugins -->
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>xml-maven-plugin</artifactId>
@@ -1028,11 +1066,11 @@
     <maven.build.timestamp.format>
       yyyy-MM-dd'T'HH:mm
     </maven.build.timestamp.format>
-    <buildDate>
-      ${maven.build.timestamp}
-    </buildDate>
     <compileSource>1.6</compileSource>
-
+    <buildDate>${maven.build.timestamp}</buildDate>
+    <!-- Build dependencies -->
+    <maven.min.version>3.0.3</maven.min.version>
+    <java.min.version>${compileSource}</java.min.version>
     <!-- Dependencies -->
     <avro.version>1.5.3</avro.version>
     <commons-cli.version>1.2</commons-cli.version>

From 4160f8b83e5eceb33fe718fa115a61c466222abf Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Thu, 6 Nov 2014 10:56:04 -0800
Subject: [PATCH 1492/1540]  HBASE-12272 Generate Thrift code using a maven
 profile

---
 pom.xml                                           |  3 ++-
 .../org/apache/hadoop/hbase/thrift2/package.html  |  6 +-----
 .../org/apache/hadoop/hbase/thrift/package.html   | 15 ++++++---------
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/pom.xml b/pom.xml
index e965e78454c8..4be9ad6a30c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 <!--
@@ -1096,6 +1096,7 @@
     <mockito-all.version>1.8.5</mockito-all.version>
     <protobuf.version>2.4.0a</protobuf.version>
     <stax-api.version>1.0.1</stax-api.version>
+    <thrift.path>thrift</thrift.path>
     <thrift.version>0.8.0</thrift.version>
     <zookeeper.version>3.4.5</zookeeper.version>
     <hadoop-snappy.version>0.0.1-SNAPSHOT</hadoop-snappy.version>
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/package.html b/src/main/java/org/apache/hadoop/hbase/thrift2/package.html
index 3d25e4d382ca..5ab9bc41ceb2 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/package.html
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/package.html
@@ -91,11 +91,7 @@ <h3><a name="details">Details</a></h3>
 <p>HBase currently uses version 0.8.0 of Apache Thrift.</p>
 <p>The files were generated by running the commands under the hbase checkout dir:
 <pre>
-  thrift -strict --gen java:hashcode ./src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
-  # Move the generated files into place their expected location under hbase
-  mv gen-java/org/apache/hadoop/hbase/thrift2/generated/* src/main/java/org/apache/hadoop/hbase/thrift2/generated/
-  # Remove the gen-java file made by thrift
-  rm -rf gen-java
+   mvn compile -Pcompile-thrift
 </pre>
 
 <p>The 'thrift' binary is the Thrift compiler, and it is distributed separately from HBase
diff --git a/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html b/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html
index b82a98a6736e..eb4a1dcf7002 100644
--- a/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html
+++ b/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html
@@ -33,9 +33,8 @@ <h2><a name="whatisthrift">What is Thrift?</a></h2>
 that work efficiently and seamlessly between C++, Java, Python, PHP, Ruby,
 Erlang, Perl, Haskell, C#, Cocoa, Smalltalk, and OCaml."</blockquote>
 
-<h2><a name="description">Description</a></h2>
-
-<p><i>Important note:</i> We tried to deprecate this Thrift interface and replace it
+<h2>Important note</h2>
+<p><b><u>We tried to deprecate this Thrift interface and replace it
 with the Interface defined over in the thrift2 package only this package will not die.
 Folks keep adding to it and fixing it up so its around for another while until someone
 takes command and drives this package out of existence replacing it w/ an Interface that
@@ -48,7 +47,9 @@ <h2><a name="description">Description</a></h2>
 {@link org.apache.hadoop.hbase.thrift.ThriftServer}. The generated interfaces,
 types, and RPC utility files reside in the
 {@link org.apache.hadoop.hbase.thrift.generated} package.
-</p>
+</u></b></p>
+  
+<h2><a name="description">Description</a></h2>
 
 <p>To start ThriftServer, use:
 <pre>
@@ -93,11 +94,7 @@ <h3><a name="details">Details</a></h3>
 
 <p>The files were generated by running the commands under the hbase checkout dir:
 <pre>
-  thrift -strict --gen java:hashcode ./src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
-  # Move the generated files into place their expected location under hbase
-  mv gen-java/org/apache/hadoop/hbase/thrift/generated/* src/main/java/org/apache/hadoop/hbase/thrift/generated/
-  # Remove the gen-java file made by thrift
-  rm -rf gen-java
+  mvn compile -Pcompile-thrift
 </pre>
 
 <p>The 'thrift' binary is the Thrift compiler, and it is distributed as a part

From eed062217cfc483fdd7146880abf69afa71798b6 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Fri, 31 Oct 2014 13:53:28 -0700
Subject: [PATCH 1493/1540] Reduce the effective scope of CREATE and ADMIN
 permissions

---
 .../security/access/AccessController.java     | 130 ++++++++++------
 .../security/access/TestAccessController.java | 147 ++++++++++++++++++
 .../apache/hadoop/hbase/security/User.java    |  21 +++
 3 files changed, 251 insertions(+), 47 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 3b6869da021a..6161382bd628 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -16,6 +16,7 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -286,20 +287,6 @@ AuthResult permissionGranted(String request, User user, TablePermission.Action p
       return AuthResult.deny(request, "No user associated with request!", null, permRequest, tableName);
     }
 
-    // Users with CREATE/ADMIN rights need to modify .META. and _acl_ table
-    // e.g. When a new table is created a new entry in .META. is added,
-    // so the user need to be allowed to write on it.
-    // e.g. When a table is removed an entry is removed from .META. and _acl_
-    // and the user need to be allowed to write on both tables.
-    if (permRequest == TablePermission.Action.WRITE &&
-       (hri.isRootRegion() || hri.isMetaRegion() ||
-        Bytes.equals(tableName, AccessControlLists.ACL_GLOBAL_NAME)) &&
-       (authManager.authorize(user, Permission.Action.CREATE) ||
-        authManager.authorize(user, Permission.Action.ADMIN)))
-    {
-       return AuthResult.allow(request, "Table permission granted", user, permRequest, tableName);
-    }
-
     // 2. check for the table-level, if successful we can short-circuit
     if (authManager.authorize(user, tableName, (byte[])null, permRequest)) {
       return AuthResult.allow(request, "Table permission granted", user, permRequest, tableName);
@@ -573,14 +560,20 @@ public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
 
   @Override
   public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+      final HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
     if (!AccessControlLists.isAclTable(desc)) {
-      String owner = desc.getOwnerString();
-      // default the table owner to current user, if not specified.
-      if (owner == null) owner = getActiveUser().getShortName();
-      UserPermission userperm = new UserPermission(Bytes.toBytes(owner), desc.getName(), null,
-          Action.values());
-      AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(), userperm);
+      final Configuration conf = c.getEnvironment().getConfiguration();
+      final String owner = (desc.getOwnerString() != null) ? desc.getOwnerString() : 
+        getActiveUser().getShortName();
+      User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+        @Override
+        public Void run() throws Exception {
+          UserPermission userperm = new UserPermission(Bytes.toBytes(owner), desc.getName(), null,
+              Action.values());
+          AccessControlLists.addUserPermission(conf, userperm);
+          return null;
+        }
+      });
     }
   }
 
@@ -592,8 +585,15 @@ public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c, byte
 
   @Override
   public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName) throws IOException {
-    AccessControlLists.removeTablePermissions(c.getEnvironment().getConfiguration(), tableName);
+      final byte[] tableName) throws IOException {
+    final Configuration conf = c.getEnvironment().getConfiguration();
+    User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        AccessControlLists.removeTablePermissions(conf, tableName);
+        return null;
+      }
+    });
   }
 
   @Override
@@ -604,13 +604,19 @@ public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c, byte
 
   @Override
   public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, HTableDescriptor htd) throws IOException {
-    String owner = htd.getOwnerString();
-    // default the table owner to current user, if not specified.
-    if (owner == null) owner = getActiveUser().getShortName();
-    UserPermission userperm = new UserPermission(Bytes.toBytes(owner), htd.getName(), null,
-        Action.values());
-    AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(), userperm);
+      byte[] tableName, final HTableDescriptor htd) throws IOException {
+    final Configuration conf = c.getEnvironment().getConfiguration();
+    final String owner = (htd.getOwnerString() != null) ? htd.getOwnerString() : 
+      getActiveUser().getShortName();
+    User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        UserPermission userperm = new UserPermission(Bytes.toBytes(owner), htd.getName(), null,
+            Action.values());
+        AccessControlLists.addUserPermission(conf, userperm);
+        return null;
+      }
+    });
   }
 
   @Override
@@ -641,9 +647,15 @@ public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c, byt
 
   @Override
   public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c,
-      byte[] tableName, byte[] col) throws IOException {
-    AccessControlLists.removeTablePermissions(c.getEnvironment().getConfiguration(),
-                                              tableName, col);
+      final byte[] tableName, final byte[] col) throws IOException {
+    final Configuration conf = c.getEnvironment().getConfiguration();
+    User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        AccessControlLists.removeTablePermissions(conf, tableName, col);
+        return null;
+      }
+    });
   }
 
   @Override
@@ -1098,16 +1110,24 @@ public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
 
   private AuthResult hasSomeAccess(RegionCoprocessorEnvironment e, String request, Action action) throws IOException {
     User requestUser = getActiveUser();
-    byte[] tableName = e.getRegion().getTableDesc().getName();
+    final byte[] tableName = e.getRegion().getTableDesc().getName();
     AuthResult authResult = permissionGranted(request, requestUser,
         action, e, Collections.EMPTY_MAP);
     if (!authResult.isAllowed()) {
-      for(UserPermission userPerm:
-          AccessControlLists.getUserPermissions(regionEnv.getConfiguration(), tableName)) {
-        for(Permission.Action userAction: userPerm.getActions()) {
-          if(userAction.equals(action)) {
+      final Configuration conf = e.getConfiguration();
+      // hasSomeAccess is called from bulkload pre hooks
+      List<UserPermission> perms =
+        User.runAsLoginUser(new PrivilegedExceptionAction<List<UserPermission>>() {
+          @Override
+          public List<UserPermission> run() throws Exception {
+            return AccessControlLists.getUserPermissions(conf, tableName);
+          }
+        });
+      for (UserPermission userPerm: perms) {
+        for (Action userAction: userPerm.getActions()) {
+          if (userAction.equals(action)) {
             return AuthResult.allow(request, "Access allowed", requestUser,
-                action, tableName);
+              action, tableName);
           }
         }
       }
@@ -1152,7 +1172,7 @@ public void preCleanupBulkLoad(RegionCoprocessorEnvironment e) throws IOExceptio
    * This will be restricted by both client side and endpoint implementations.
    */
   @Override
-  public void grant(UserPermission perm) throws IOException {
+  public void grant(final UserPermission perm) throws IOException {
     // verify it's only running at .acl.
     if (aclRegion) {
       if (LOG.isDebugEnabled()) {
@@ -1161,7 +1181,14 @@ public void grant(UserPermission perm) throws IOException {
 
       requirePermission("grant", perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN);
 
-      AccessControlLists.addUserPermission(regionEnv.getConfiguration(), perm);
+      User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+        @Override
+        public Void run() throws Exception {
+          AccessControlLists.addUserPermission(regionEnv.getConfiguration(), perm);
+          return null;
+        }
+      });
+
       if (AUDITLOG.isTraceEnabled()) {
         // audit log should store permission changes in addition to auth results
         AUDITLOG.trace("Granted permission " + perm.toString());
@@ -1182,7 +1209,7 @@ public void grant(byte[] user, TablePermission permission)
   }
 
   @Override
-  public void revoke(UserPermission perm) throws IOException {
+  public void revoke(final UserPermission perm) throws IOException {
     // only allowed to be called on _acl_ region
     if (aclRegion) {
       if (LOG.isDebugEnabled()) {
@@ -1192,7 +1219,14 @@ public void revoke(UserPermission perm) throws IOException {
       requirePermission("revoke", perm.getTable(), perm.getFamily(),
                         perm.getQualifier(), Action.ADMIN);
 
-      AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), perm);
+      User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+        @Override
+        public Void run() throws Exception {
+          AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), perm);
+          return null;
+        }
+      });
+
       if (AUDITLOG.isTraceEnabled()) {
         // audit log should record all permission changes
         AUDITLOG.trace("Revoked permission " + perm.toString());
@@ -1217,10 +1251,12 @@ public List<UserPermission> getUserPermissions(final byte[] tableName) throws IO
     // only allowed to be called on _acl_ region
     if (aclRegion) {
       requirePermission("userPermissions", tableName, null, null, Action.ADMIN);
-
-      List<UserPermission> perms = AccessControlLists.getUserPermissions(
-        regionEnv.getConfiguration(), tableName);
-      return perms;
+      return User.runAsLoginUser(new PrivilegedExceptionAction<List<UserPermission>>() {
+        @Override
+        public List<UserPermission> run() throws Exception {
+          return AccessControlLists.getUserPermissions(regionEnv.getConfiguration(), tableName);
+        }
+      });
     } else {
       throw new CoprocessorException(AccessController.class, "This method "
           + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table.");
diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index d696acbfdfd2..688e30d9d835 100644
--- a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.security.access;
 
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -1962,4 +1963,150 @@ public Object run() throws Exception {
     verifyAllowed(deleteTableAction, tableAdmin);
   }
 
+  @Test
+  public void testCreateWithCorrectOwner() throws Exception {
+    final byte[] tableName = Bytes.toBytes("testCreateWithCorrectOwner");
+    
+    // Create a test user
+    User testUser = User.createUserForTesting(TEST_UTIL.getConfiguration(), "TestUser",
+      new String[0]);
+
+    // Grant the test user the ability to create tables
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(
+          AccessControllerProtocol.class, AccessControlLists.ACL_TABLE_NAME);
+      protocol.grant(new UserPermission(Bytes.toBytes(testUser.getShortName()),
+        Permission.Action.CREATE));
+    } finally {
+      acl.close();
+    }
+
+    verifyAllowed(new PrivilegedExceptionAction() {
+      @Override
+      public Object run() throws Exception {
+        HTableDescriptor desc = new HTableDescriptor(tableName);
+        desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
+        HBaseAdmin admin = new HBaseAdmin(conf);
+        try {
+          admin.createTable(desc);
+        } finally {
+          admin.close();
+        }
+        return null;
+      }
+    }, testUser);
+    TEST_UTIL.waitTableEnabled(tableName, 5000);
+
+    // Verify that owner permissions have been granted to the test user on the
+    // table just created
+    List<TablePermission> perms = AccessControlLists.getTablePermissions(conf, tableName)
+       .get(testUser.getShortName());
+    assertNotNull(perms);
+    assertFalse(perms.isEmpty());
+    // Should be RWXCA
+    assertTrue(perms.get(0).implies(Permission.Action.READ));
+    assertTrue(perms.get(0).implies(Permission.Action.WRITE));
+    assertTrue(perms.get(0).implies(Permission.Action.EXEC));
+    assertTrue(perms.get(0).implies(Permission.Action.CREATE));
+    assertTrue(perms.get(0).implies(Permission.Action.ADMIN));
+  }
+
+  @Test
+  public void testACLTableAccess() throws Exception {
+    final Configuration conf = TEST_UTIL.getConfiguration();
+
+    final byte[] tableName = Bytes.toBytes("testACLTableAccess");
+    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    admin.createTable(htd);
+    TEST_UTIL.waitTableEnabled(tableName, 5000);
+
+    // Global users
+    User globalRead = User.createUserForTesting(conf, "globalRead", new String[0]);
+    User globalWrite = User.createUserForTesting(conf, "globalWrite", new String[0]);
+    User globalCreate = User.createUserForTesting(conf, "globalCreate", new String[0]);
+    User globalAdmin = User.createUserForTesting(conf, "globalAdmin", new String[0]);
+    
+    // Table users
+    User tableRead = User.createUserForTesting(conf, "tableRead", new String[0]);
+    User tableWrite = User.createUserForTesting(conf, "tableWrite", new String[0]);
+    User tableCreate = User.createUserForTesting(conf, "tableCreate", new String[0]);
+    User tableAdmin = User.createUserForTesting(conf, "tableAdmin", new String[0]);
+
+    // Set up grants
+    HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+    try {
+      AccessControllerProtocol protocol = acl.coprocessorProxy(
+        AccessControllerProtocol.class, AccessControlLists.ACL_TABLE_NAME);
+      protocol.grant(new UserPermission(Bytes.toBytes(globalRead.getShortName()), Action.READ));
+      protocol.grant(new UserPermission(Bytes.toBytes(globalWrite.getShortName()), Action.WRITE));
+      protocol.grant(new UserPermission(Bytes.toBytes(globalCreate.getShortName()),
+        Action.CREATE));
+      protocol.grant(new UserPermission(Bytes.toBytes(globalAdmin.getShortName()),
+        Action.ADMIN));
+      protocol.grant(new UserPermission(Bytes.toBytes(tableRead.getShortName()), tableName,
+        null, Action.READ));
+      protocol.grant(new UserPermission(Bytes.toBytes(tableWrite.getShortName()), tableName,
+        null, Action.WRITE));
+      protocol.grant(new UserPermission(Bytes.toBytes(tableCreate.getShortName()), tableName,
+        null, Action.CREATE));
+      protocol.grant(new UserPermission(Bytes.toBytes(tableAdmin.getShortName()), tableName,
+        null, Action.ADMIN));
+    } finally {
+      acl.close();
+    }
+
+    // Write tests
+
+    PrivilegedExceptionAction writeAction = new PrivilegedExceptionAction() {
+      @Override
+      public Object run() throws Exception {
+        HTable t = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+        try {
+          t.put(new Put(Bytes.toBytes("test")).add(AccessControlLists.ACL_LIST_FAMILY,
+            Bytes.toBytes("q"), Bytes.toBytes("value")));
+          return null;
+        } finally {
+          t.close();
+        }
+      }
+    };
+
+    // All writes to ACL table denied except for GLOBAL WRITE permission and superuser
+
+    verifyDenied(writeAction, globalAdmin, globalCreate, globalRead);
+    verifyDenied(writeAction, tableAdmin, tableCreate, tableRead, tableWrite);
+    verifyAllowed(writeAction, SUPERUSER, globalWrite);
+
+    // Read tests
+
+    PrivilegedExceptionAction scanAction = new PrivilegedExceptionAction() {
+      @Override
+      public Object run() throws Exception {
+        HTable t = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
+        try {
+          ResultScanner s = t.getScanner(new Scan());
+          try {
+            for (Result r = s.next(); r != null; r = s.next()) {
+              // do nothing
+            }
+          } finally {
+            s.close();
+          }
+          return null;
+        } finally {
+          t.close();
+        }
+      }
+    };
+
+    // All reads from ACL table denied except for GLOBAL READ and superuser
+
+    verifyDenied(scanAction, globalAdmin, globalCreate, globalWrite);
+    verifyDenied(scanAction, tableCreate, tableAdmin, tableRead, tableWrite);
+    verifyAllowed(scanAction, SUPERUSER, globalRead);
+  }
+
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java
index 96e0a395904d..f19e7fb7c3fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/security/User.java
+++ b/src/main/java/org/apache/hadoop/hbase/security/User.java
@@ -158,6 +158,27 @@ public static User getCurrent() throws IOException {
     return user;
   }
 
+  /**
+   * Executes the given action as the login user
+   * @param action
+   * @return
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @SuppressWarnings({ "rawtypes", "unchecked" })
+  public static <T> T runAsLoginUser(PrivilegedExceptionAction<T> action) throws IOException {
+    return doAsUser(UserGroupInformation.getLoginUser(), action);
+  }
+
+  private static <T> T doAsUser(UserGroupInformation ugi,
+      PrivilegedExceptionAction<T> action) throws IOException {
+    try {
+      return ugi.doAs(action);
+    } catch (InterruptedException ie) {
+      throw new IOException(ie);
+    }
+  }
+
   /**
    * Wraps an underlying {@code UserGroupInformation} instance.
    * @param ugi The base Hadoop user

From 7aad0ed8de3dbca78da77fd6c85a1e940cf59ed8 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 7 Nov 2014 11:50:53 -0800
Subject: [PATCH 1494/1540] pom.xml, CHANGES.txt for 0.94.25

---
 CHANGES.txt | 20 ++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 805e94f415a8..e4dd4037f1d2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,24 @@
 HBase Change Log
+Release 0.94.25 - 11/7/2014
+Bug
+
+    [HBASE-12039] - Lower log level for TableNotFoundException log message when throwing
+    [HBASE-12065] - Import tool is not restoring multiple DeleteFamily markers of a row
+    [HBASE-12146] - RegionServerTracker should escape data in log messages
+    [HBASE-12171] - Backport: PerformanceEvaluation: getSplits doesn't provide right splits.
+    [HBASE-12336] - RegionServer failed to shutdown for NodeFailoverWorker thread
+    [HBASE-12376] - HBaseAdmin leaks ZK connections if failure starting watchers (ConnectionLossException)
+
+Improvement
+
+    [HBASE-12272] - Generate Thrift code through maven
+
+Task
+
+    [HBASE-12235] - Backport to 0.94: HBASE-9002 TestDistributedLogSplitting.testRecoverdEdits should test correct region
+    [HBASE-12381] - Add maven enforcer rules for build assumptions
+
+
 Release 0.94.24 - 09/29/2014
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 4be9ad6a30c6..2105f1b2a2dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.25-SNAPSHOT</version>
+  <version>0.94.25</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 7ddd9e4c45dadb788e1a8c6f5bb9f8e591ba9a73 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Tue, 11 Nov 2014 15:56:28 -0800
Subject: [PATCH 1495/1540] HBASE-12279 Generated thrift files were generated
 with the wrong parameters (Niels Basjes)

---
 .../hbase/thrift/generated/AlreadyExists.java |   10 +-
 .../hbase/thrift/generated/BatchMutation.java |   19 +-
 .../thrift/generated/ColumnDescriptor.java    |   50 +-
 .../hadoop/hbase/thrift/generated/Hbase.java  | 3762 +++++++++++------
 .../hbase/thrift/generated/IOError.java       |   10 +-
 .../thrift/generated/IllegalArgument.java     |   10 +-
 .../hbase/thrift/generated/Mutation.java      |   25 +-
 .../hadoop/hbase/thrift/generated/TCell.java  |   15 +-
 .../hbase/thrift/generated/TColumn.java       |   15 +-
 .../hbase/thrift/generated/TIncrement.java    |   25 +-
 .../hbase/thrift/generated/TRegionInfo.java   |   40 +-
 .../hbase/thrift/generated/TRowResult.java    |   20 +-
 .../hadoop/hbase/thrift/generated/TScan.java  |   40 +-
 .../hbase/thrift2/generated/TColumn.java      |   20 +-
 .../thrift2/generated/TColumnIncrement.java   |   20 +-
 .../hbase/thrift2/generated/TColumnValue.java |   25 +-
 .../hbase/thrift2/generated/TDelete.java      |   44 +-
 .../hadoop/hbase/thrift2/generated/TGet.java  |   44 +-
 .../thrift2/generated/THBaseService.java      |  456 +-
 .../hbase/thrift2/generated/TIOError.java     |   10 +-
 .../thrift2/generated/TIllegalArgument.java   |   10 +-
 .../hbase/thrift2/generated/TIncrement.java   |   56 +-
 .../hbase/thrift2/generated/TMutation.java    |   19 +-
 .../hadoop/hbase/thrift2/generated/TPut.java  |   39 +-
 .../hbase/thrift2/generated/TResult.java      |   15 +-
 .../thrift2/generated/TRowMutations.java      |   51 +-
 .../hadoop/hbase/thrift2/generated/TScan.java |   54 +-
 .../hbase/thrift2/generated/TTimeRange.java   |   15 +-
 28 files changed, 3523 insertions(+), 1396 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java
index a5b81f5b8501..abeac63729b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -226,7 +227,14 @@ public boolean equals(AlreadyExists that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_message = true && (isSetMessage());
+    builder.append(present_message);
+    if (present_message)
+      builder.append(message);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(AlreadyExists other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java
index d5df9407756c..d535baf13d42 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -314,7 +315,19 @@ public boolean equals(BatchMutation that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_mutations = true && (isSetMutations());
+    builder.append(present_mutations);
+    if (present_mutations)
+      builder.append(mutations);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(BatchMutation other) {
@@ -437,7 +450,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, BatchMutation struc
                 struct.mutations = new ArrayList<Mutation>(_list0.size);
                 for (int _i1 = 0; _i1 < _list0.size; ++_i1)
                 {
-                  Mutation _elem2; // optional
+                  Mutation _elem2; // required
                   _elem2 = new Mutation();
                   _elem2.read(iprot);
                   struct.mutations.add(_elem2);
@@ -534,7 +547,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, BatchMutation struct
           struct.mutations = new ArrayList<Mutation>(_list5.size);
           for (int _i6 = 0; _i6 < _list5.size; ++_i6)
           {
-            Mutation _elem7; // optional
+            Mutation _elem7; // required
             _elem7 = new Mutation();
             _elem7.read(iprot);
             struct.mutations.add(_elem7);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java
index 4ce85e7ed5e6..047de4a784bf 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -730,7 +731,54 @@ public boolean equals(ColumnDescriptor that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_name = true && (isSetName());
+    builder.append(present_name);
+    if (present_name)
+      builder.append(name);
+
+    boolean present_maxVersions = true;
+    builder.append(present_maxVersions);
+    if (present_maxVersions)
+      builder.append(maxVersions);
+
+    boolean present_compression = true && (isSetCompression());
+    builder.append(present_compression);
+    if (present_compression)
+      builder.append(compression);
+
+    boolean present_inMemory = true;
+    builder.append(present_inMemory);
+    if (present_inMemory)
+      builder.append(inMemory);
+
+    boolean present_bloomFilterType = true && (isSetBloomFilterType());
+    builder.append(present_bloomFilterType);
+    if (present_bloomFilterType)
+      builder.append(bloomFilterType);
+
+    boolean present_bloomFilterVectorSize = true;
+    builder.append(present_bloomFilterVectorSize);
+    if (present_bloomFilterVectorSize)
+      builder.append(bloomFilterVectorSize);
+
+    boolean present_bloomFilterNbHashes = true;
+    builder.append(present_bloomFilterNbHashes);
+    if (present_bloomFilterNbHashes)
+      builder.append(bloomFilterNbHashes);
+
+    boolean present_blockCacheEnabled = true;
+    builder.append(present_blockCacheEnabled);
+    if (present_blockCacheEnabled)
+      builder.append(blockCacheEnabled);
+
+    boolean present_timeToLive = true;
+    builder.append(present_timeToLive);
+    if (present_timeToLive)
+      builder.append(timeToLive);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(ColumnDescriptor other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java
index e9ca236327fd..3048c83bef07 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -4733,7 +4734,14 @@ public boolean equals(enableTable_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(enableTable_args other) {
@@ -5086,7 +5094,14 @@ public boolean equals(enableTable_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(enableTable_result other) {
@@ -5463,7 +5478,14 @@ public boolean equals(disableTable_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(disableTable_args other) {
@@ -5816,7 +5838,14 @@ public boolean equals(disableTable_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(disableTable_result other) {
@@ -6193,7 +6222,14 @@ public boolean equals(isTableEnabled_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(isTableEnabled_args other) {
@@ -6608,7 +6644,19 @@ public boolean equals(isTableEnabled_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(isTableEnabled_result other) {
@@ -6688,8 +6736,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -7010,7 +7056,14 @@ public boolean equals(compact_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableNameOrRegionName = true && (isSetTableNameOrRegionName());
+      builder.append(present_tableNameOrRegionName);
+      if (present_tableNameOrRegionName)
+        builder.append(tableNameOrRegionName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(compact_args other) {
@@ -7363,7 +7416,14 @@ public boolean equals(compact_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(compact_result other) {
@@ -7728,7 +7788,14 @@ public boolean equals(majorCompact_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableNameOrRegionName = true && (isSetTableNameOrRegionName());
+      builder.append(present_tableNameOrRegionName);
+      if (present_tableNameOrRegionName)
+        builder.append(tableNameOrRegionName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(majorCompact_args other) {
@@ -8081,7 +8148,14 @@ public boolean equals(majorCompact_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(majorCompact_result other) {
@@ -8371,7 +8445,9 @@ public boolean equals(getTableNames_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getTableNames_args other) {
@@ -8760,7 +8836,19 @@ public boolean equals(getTableNames_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getTableNames_result other) {
@@ -8871,13 +8959,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getTableNames_resul
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list26 = iprot.readListBegin();
-                  struct.success = new ArrayList<ByteBuffer>(_list26.size);
-                  for (int _i27 = 0; _i27 < _list26.size; ++_i27)
+                  org.apache.thrift.protocol.TList _list34 = iprot.readListBegin();
+                  struct.success = new ArrayList<ByteBuffer>(_list34.size);
+                  for (int _i35 = 0; _i35 < _list34.size; ++_i35)
                   {
-                    ByteBuffer _elem28; // optional
-                    _elem28 = iprot.readBinary();
-                    struct.success.add(_elem28);
+                    ByteBuffer _elem36; // required
+                    _elem36 = iprot.readBinary();
+                    struct.success.add(_elem36);
                   }
                   iprot.readListEnd();
                 }
@@ -8914,9 +9002,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getTableNames_resu
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.success.size()));
-            for (ByteBuffer _iter29 : struct.success)
+            for (ByteBuffer _iter37 : struct.success)
             {
-              oprot.writeBinary(_iter29);
+              oprot.writeBinary(_iter37);
             }
             oprot.writeListEnd();
           }
@@ -8955,9 +9043,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getTableNames_resul
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (ByteBuffer _iter30 : struct.success)
+            for (ByteBuffer _iter38 : struct.success)
             {
-              oprot.writeBinary(_iter30);
+              oprot.writeBinary(_iter38);
             }
           }
         }
@@ -8972,13 +9060,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getTableNames_result
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list31 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.success = new ArrayList<ByteBuffer>(_list31.size);
-            for (int _i32 = 0; _i32 < _list31.size; ++_i32)
+            org.apache.thrift.protocol.TList _list39 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.success = new ArrayList<ByteBuffer>(_list39.size);
+            for (int _i40 = 0; _i40 < _list39.size; ++_i40)
             {
-              ByteBuffer _elem33; // optional
-              _elem33 = iprot.readBinary();
-              struct.success.add(_elem33);
+              ByteBuffer _elem41; // required
+              _elem41 = iprot.readBinary();
+              struct.success.add(_elem41);
             }
           }
           struct.setSuccessIsSet(true);
@@ -9210,7 +9298,14 @@ public boolean equals(getColumnDescriptors_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getColumnDescriptors_args other) {
@@ -9647,7 +9742,19 @@ public boolean equals(getColumnDescriptors_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getColumnDescriptors_result other) {
@@ -9758,16 +9865,16 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getColumnDescriptor
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map34 = iprot.readMapBegin();
-                  struct.success = new HashMap<ByteBuffer,ColumnDescriptor>(2*_map34.size);
-                  for (int _i35 = 0; _i35 < _map34.size; ++_i35)
+                  org.apache.thrift.protocol.TMap _map42 = iprot.readMapBegin();
+                  struct.success = new HashMap<ByteBuffer,ColumnDescriptor>(2*_map42.size);
+                  for (int _i43 = 0; _i43 < _map42.size; ++_i43)
                   {
-                    ByteBuffer _key36; // required
-                    ColumnDescriptor _val37; // optional
-                    _key36 = iprot.readBinary();
-                    _val37 = new ColumnDescriptor();
-                    _val37.read(iprot);
-                    struct.success.put(_key36, _val37);
+                    ByteBuffer _key44; // required
+                    ColumnDescriptor _val45; // required
+                    _key44 = iprot.readBinary();
+                    _val45 = new ColumnDescriptor();
+                    _val45.read(iprot);
+                    struct.success.put(_key44, _val45);
                   }
                   iprot.readMapEnd();
                 }
@@ -9804,10 +9911,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getColumnDescripto
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (Map.Entry<ByteBuffer, ColumnDescriptor> _iter38 : struct.success.entrySet())
+            for (Map.Entry<ByteBuffer, ColumnDescriptor> _iter46 : struct.success.entrySet())
             {
-              oprot.writeBinary(_iter38.getKey());
-              _iter38.getValue().write(oprot);
+              oprot.writeBinary(_iter46.getKey());
+              _iter46.getValue().write(oprot);
             }
             oprot.writeMapEnd();
           }
@@ -9846,10 +9953,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptor
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (Map.Entry<ByteBuffer, ColumnDescriptor> _iter39 : struct.success.entrySet())
+            for (Map.Entry<ByteBuffer, ColumnDescriptor> _iter47 : struct.success.entrySet())
             {
-              oprot.writeBinary(_iter39.getKey());
-              _iter39.getValue().write(oprot);
+              oprot.writeBinary(_iter47.getKey());
+              _iter47.getValue().write(oprot);
             }
           }
         }
@@ -9864,16 +9971,16 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptors
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TMap _map40 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new HashMap<ByteBuffer,ColumnDescriptor>(2*_map40.size);
-            for (int _i41 = 0; _i41 < _map40.size; ++_i41)
+            org.apache.thrift.protocol.TMap _map48 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new HashMap<ByteBuffer,ColumnDescriptor>(2*_map48.size);
+            for (int _i49 = 0; _i49 < _map48.size; ++_i49)
             {
-              ByteBuffer _key42; // required
-              ColumnDescriptor _val43; // optional
-              _key42 = iprot.readBinary();
-              _val43 = new ColumnDescriptor();
-              _val43.read(iprot);
-              struct.success.put(_key42, _val43);
+              ByteBuffer _key50; // required
+              ColumnDescriptor _val51; // required
+              _key50 = iprot.readBinary();
+              _val51 = new ColumnDescriptor();
+              _val51.read(iprot);
+              struct.success.put(_key50, _val51);
             }
           }
           struct.setSuccessIsSet(true);
@@ -10105,7 +10212,14 @@ public boolean equals(getTableRegions_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getTableRegions_args other) {
@@ -10537,7 +10651,19 @@ public boolean equals(getTableRegions_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getTableRegions_result other) {
@@ -10648,14 +10774,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getTableRegions_res
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list44 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRegionInfo>(_list44.size);
-                  for (int _i45 = 0; _i45 < _list44.size; ++_i45)
+                  org.apache.thrift.protocol.TList _list52 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRegionInfo>(_list52.size);
+                  for (int _i53 = 0; _i53 < _list52.size; ++_i53)
                   {
-                    TRegionInfo _elem46; // optional
-                    _elem46 = new TRegionInfo();
-                    _elem46.read(iprot);
-                    struct.success.add(_elem46);
+                    TRegionInfo _elem54; // required
+                    _elem54 = new TRegionInfo();
+                    _elem54.read(iprot);
+                    struct.success.add(_elem54);
                   }
                   iprot.readListEnd();
                 }
@@ -10692,9 +10818,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getTableRegions_re
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRegionInfo _iter47 : struct.success)
+            for (TRegionInfo _iter55 : struct.success)
             {
-              _iter47.write(oprot);
+              _iter55.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -10733,9 +10859,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getTableRegions_res
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRegionInfo _iter48 : struct.success)
+            for (TRegionInfo _iter56 : struct.success)
             {
-              _iter48.write(oprot);
+              _iter56.write(oprot);
             }
           }
         }
@@ -10750,14 +10876,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getTableRegions_resu
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list49 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRegionInfo>(_list49.size);
-            for (int _i50 = 0; _i50 < _list49.size; ++_i50)
+            org.apache.thrift.protocol.TList _list57 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRegionInfo>(_list57.size);
+            for (int _i58 = 0; _i58 < _list57.size; ++_i58)
             {
-              TRegionInfo _elem51; // optional
-              _elem51 = new TRegionInfo();
-              _elem51.read(iprot);
-              struct.success.add(_elem51);
+              TRegionInfo _elem59; // required
+              _elem59 = new TRegionInfo();
+              _elem59.read(iprot);
+              struct.success.add(_elem59);
             }
           }
           struct.setSuccessIsSet(true);
@@ -11080,7 +11206,19 @@ public boolean equals(createTable_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_columnFamilies = true && (isSetColumnFamilies());
+      builder.append(present_columnFamilies);
+      if (present_columnFamilies)
+        builder.append(columnFamilies);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(createTable_args other) {
@@ -11199,14 +11337,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, createTable_args st
             case 2: // COLUMN_FAMILIES
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list52 = iprot.readListBegin();
-                  struct.columnFamilies = new ArrayList<ColumnDescriptor>(_list52.size);
-                  for (int _i53 = 0; _i53 < _list52.size; ++_i53)
+                  org.apache.thrift.protocol.TList _list60 = iprot.readListBegin();
+                  struct.columnFamilies = new ArrayList<ColumnDescriptor>(_list60.size);
+                  for (int _i61 = 0; _i61 < _list60.size; ++_i61)
                   {
-                    ColumnDescriptor _elem54; // optional
-                    _elem54 = new ColumnDescriptor();
-                    _elem54.read(iprot);
-                    struct.columnFamilies.add(_elem54);
+                    ColumnDescriptor _elem62; // required
+                    _elem62 = new ColumnDescriptor();
+                    _elem62.read(iprot);
+                    struct.columnFamilies.add(_elem62);
                   }
                   iprot.readListEnd();
                 }
@@ -11239,9 +11377,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, createTable_args s
           oprot.writeFieldBegin(COLUMN_FAMILIES_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columnFamilies.size()));
-            for (ColumnDescriptor _iter55 : struct.columnFamilies)
+            for (ColumnDescriptor _iter63 : struct.columnFamilies)
             {
-              _iter55.write(oprot);
+              _iter63.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -11278,9 +11416,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, createTable_args st
         if (struct.isSetColumnFamilies()) {
           {
             oprot.writeI32(struct.columnFamilies.size());
-            for (ColumnDescriptor _iter56 : struct.columnFamilies)
+            for (ColumnDescriptor _iter64 : struct.columnFamilies)
             {
-              _iter56.write(oprot);
+              _iter64.write(oprot);
             }
           }
         }
@@ -11296,14 +11434,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, createTable_args str
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list57 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.columnFamilies = new ArrayList<ColumnDescriptor>(_list57.size);
-            for (int _i58 = 0; _i58 < _list57.size; ++_i58)
+            org.apache.thrift.protocol.TList _list65 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.columnFamilies = new ArrayList<ColumnDescriptor>(_list65.size);
+            for (int _i66 = 0; _i66 < _list65.size; ++_i66)
             {
-              ColumnDescriptor _elem59; // optional
-              _elem59 = new ColumnDescriptor();
-              _elem59.read(iprot);
-              struct.columnFamilies.add(_elem59);
+              ColumnDescriptor _elem67; // required
+              _elem67 = new ColumnDescriptor();
+              _elem67.read(iprot);
+              struct.columnFamilies.add(_elem67);
             }
           }
           struct.setColumnFamiliesIsSet(true);
@@ -11626,7 +11764,24 @@ public boolean equals(createTable_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      boolean present_exist = true && (isSetExist());
+      builder.append(present_exist);
+      if (present_exist)
+        builder.append(exist);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(createTable_result other) {
@@ -12089,7 +12244,14 @@ public boolean equals(deleteTable_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteTable_args other) {
@@ -12442,7 +12604,14 @@ public boolean equals(deleteTable_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteTable_result other) {
@@ -13077,7 +13246,29 @@ public boolean equals(get_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_column = true && (isSetColumn());
+      builder.append(present_column);
+      if (present_column)
+        builder.append(column);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(get_args other) {
@@ -13248,15 +13439,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, get_args struct) th
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map60 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map60.size);
-                  for (int _i61 = 0; _i61 < _map60.size; ++_i61)
+                  org.apache.thrift.protocol.TMap _map68 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map68.size);
+                  for (int _i69 = 0; _i69 < _map68.size; ++_i69)
                   {
-                    ByteBuffer _key62; // required
-                    ByteBuffer _val63; // optional
-                    _key62 = iprot.readBinary();
-                    _val63 = iprot.readBinary();
-                    struct.attributes.put(_key62, _val63);
+                    ByteBuffer _key70; // required
+                    ByteBuffer _val71; // required
+                    _key70 = iprot.readBinary();
+                    _val71 = iprot.readBinary();
+                    struct.attributes.put(_key70, _val71);
                   }
                   iprot.readMapEnd();
                 }
@@ -13299,10 +13490,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, get_args struct) t
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter64 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter72 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter64.getKey());
-              oprot.writeBinary(_iter64.getValue());
+              oprot.writeBinary(_iter72.getKey());
+              oprot.writeBinary(_iter72.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -13351,10 +13542,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, get_args struct) th
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter65 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter73 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter65.getKey());
-              oprot.writeBinary(_iter65.getValue());
+              oprot.writeBinary(_iter73.getKey());
+              oprot.writeBinary(_iter73.getValue());
             }
           }
         }
@@ -13378,15 +13569,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, get_args struct) thr
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map66 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map66.size);
-            for (int _i67 = 0; _i67 < _map66.size; ++_i67)
+            org.apache.thrift.protocol.TMap _map74 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map74.size);
+            for (int _i75 = 0; _i75 < _map74.size; ++_i75)
             {
-              ByteBuffer _key68; // required
-              ByteBuffer _val69; // optional
-              _key68 = iprot.readBinary();
-              _val69 = iprot.readBinary();
-              struct.attributes.put(_key68, _val69);
+              ByteBuffer _key76; // required
+              ByteBuffer _val77; // required
+              _key76 = iprot.readBinary();
+              _val77 = iprot.readBinary();
+              struct.attributes.put(_key76, _val77);
             }
           }
           struct.setAttributesIsSet(true);
@@ -13670,7 +13861,19 @@ public boolean equals(get_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(get_result other) {
@@ -13781,14 +13984,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, get_result struct)
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list70 = iprot.readListBegin();
-                  struct.success = new ArrayList<TCell>(_list70.size);
-                  for (int _i71 = 0; _i71 < _list70.size; ++_i71)
+                  org.apache.thrift.protocol.TList _list78 = iprot.readListBegin();
+                  struct.success = new ArrayList<TCell>(_list78.size);
+                  for (int _i79 = 0; _i79 < _list78.size; ++_i79)
                   {
-                    TCell _elem72; // optional
-                    _elem72 = new TCell();
-                    _elem72.read(iprot);
-                    struct.success.add(_elem72);
+                    TCell _elem80; // required
+                    _elem80 = new TCell();
+                    _elem80.read(iprot);
+                    struct.success.add(_elem80);
                   }
                   iprot.readListEnd();
                 }
@@ -13825,9 +14028,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, get_result struct)
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TCell _iter73 : struct.success)
+            for (TCell _iter81 : struct.success)
             {
-              _iter73.write(oprot);
+              _iter81.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -13866,9 +14069,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, get_result struct)
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TCell _iter74 : struct.success)
+            for (TCell _iter82 : struct.success)
             {
-              _iter74.write(oprot);
+              _iter82.write(oprot);
             }
           }
         }
@@ -13883,14 +14086,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, get_result struct) t
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list75 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TCell>(_list75.size);
-            for (int _i76 = 0; _i76 < _list75.size; ++_i76)
+            org.apache.thrift.protocol.TList _list83 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TCell>(_list83.size);
+            for (int _i84 = 0; _i84 < _list83.size; ++_i84)
             {
-              TCell _elem77; // optional
-              _elem77 = new TCell();
-              _elem77.read(iprot);
-              struct.success.add(_elem77);
+              TCell _elem85; // required
+              _elem85 = new TCell();
+              _elem85.read(iprot);
+              struct.success.add(_elem85);
             }
           }
           struct.setSuccessIsSet(true);
@@ -14454,7 +14657,34 @@ public boolean equals(getVer_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_column = true && (isSetColumn());
+      builder.append(present_column);
+      if (present_column)
+        builder.append(column);
+
+      boolean present_numVersions = true;
+      builder.append(present_numVersions);
+      if (present_numVersions)
+        builder.append(numVersions);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getVer_args other) {
@@ -14649,15 +14879,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getVer_args struct)
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map78 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map78.size);
-                  for (int _i79 = 0; _i79 < _map78.size; ++_i79)
+                  org.apache.thrift.protocol.TMap _map86 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map86.size);
+                  for (int _i87 = 0; _i87 < _map86.size; ++_i87)
                   {
-                    ByteBuffer _key80; // required
-                    ByteBuffer _val81; // optional
-                    _key80 = iprot.readBinary();
-                    _val81 = iprot.readBinary();
-                    struct.attributes.put(_key80, _val81);
+                    ByteBuffer _key88; // required
+                    ByteBuffer _val89; // required
+                    _key88 = iprot.readBinary();
+                    _val89 = iprot.readBinary();
+                    struct.attributes.put(_key88, _val89);
                   }
                   iprot.readMapEnd();
                 }
@@ -14703,10 +14933,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getVer_args struct
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter82 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter90 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter82.getKey());
-              oprot.writeBinary(_iter82.getValue());
+              oprot.writeBinary(_iter90.getKey());
+              oprot.writeBinary(_iter90.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -14761,10 +14991,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getVer_args struct)
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter83 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter91 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter83.getKey());
-              oprot.writeBinary(_iter83.getValue());
+              oprot.writeBinary(_iter91.getKey());
+              oprot.writeBinary(_iter91.getValue());
             }
           }
         }
@@ -14792,15 +15022,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getVer_args struct)
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map84 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map84.size);
-            for (int _i85 = 0; _i85 < _map84.size; ++_i85)
+            org.apache.thrift.protocol.TMap _map92 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map92.size);
+            for (int _i93 = 0; _i93 < _map92.size; ++_i93)
             {
-              ByteBuffer _key86; // required
-              ByteBuffer _val87; // optional
-              _key86 = iprot.readBinary();
-              _val87 = iprot.readBinary();
-              struct.attributes.put(_key86, _val87);
+              ByteBuffer _key94; // required
+              ByteBuffer _val95; // required
+              _key94 = iprot.readBinary();
+              _val95 = iprot.readBinary();
+              struct.attributes.put(_key94, _val95);
             }
           }
           struct.setAttributesIsSet(true);
@@ -15084,7 +15314,19 @@ public boolean equals(getVer_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getVer_result other) {
@@ -15195,14 +15437,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getVer_result struc
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list88 = iprot.readListBegin();
-                  struct.success = new ArrayList<TCell>(_list88.size);
-                  for (int _i89 = 0; _i89 < _list88.size; ++_i89)
+                  org.apache.thrift.protocol.TList _list96 = iprot.readListBegin();
+                  struct.success = new ArrayList<TCell>(_list96.size);
+                  for (int _i97 = 0; _i97 < _list96.size; ++_i97)
                   {
-                    TCell _elem90; // optional
-                    _elem90 = new TCell();
-                    _elem90.read(iprot);
-                    struct.success.add(_elem90);
+                    TCell _elem98; // required
+                    _elem98 = new TCell();
+                    _elem98.read(iprot);
+                    struct.success.add(_elem98);
                   }
                   iprot.readListEnd();
                 }
@@ -15239,9 +15481,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getVer_result stru
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TCell _iter91 : struct.success)
+            for (TCell _iter99 : struct.success)
             {
-              _iter91.write(oprot);
+              _iter99.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -15280,9 +15522,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getVer_result struc
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TCell _iter92 : struct.success)
+            for (TCell _iter100 : struct.success)
             {
-              _iter92.write(oprot);
+              _iter100.write(oprot);
             }
           }
         }
@@ -15297,14 +15539,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getVer_result struct
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TCell>(_list93.size);
-            for (int _i94 = 0; _i94 < _list93.size; ++_i94)
+            org.apache.thrift.protocol.TList _list101 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TCell>(_list101.size);
+            for (int _i102 = 0; _i102 < _list101.size; ++_i102)
             {
-              TCell _elem95; // optional
-              _elem95 = new TCell();
-              _elem95.read(iprot);
-              struct.success.add(_elem95);
+              TCell _elem103; // required
+              _elem103 = new TCell();
+              _elem103.read(iprot);
+              struct.success.add(_elem103);
             }
           }
           struct.setSuccessIsSet(true);
@@ -15939,7 +16181,39 @@ public boolean equals(getVerTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_column = true && (isSetColumn());
+      builder.append(present_column);
+      if (present_column)
+        builder.append(column);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_numVersions = true;
+      builder.append(present_numVersions);
+      if (present_numVersions)
+        builder.append(numVersions);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getVerTs_args other) {
@@ -16156,15 +16430,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getVerTs_args struc
             case 6: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map96 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map96.size);
-                  for (int _i97 = 0; _i97 < _map96.size; ++_i97)
+                  org.apache.thrift.protocol.TMap _map104 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map104.size);
+                  for (int _i105 = 0; _i105 < _map104.size; ++_i105)
                   {
-                    ByteBuffer _key98; // required
-                    ByteBuffer _val99; // optional
-                    _key98 = iprot.readBinary();
-                    _val99 = iprot.readBinary();
-                    struct.attributes.put(_key98, _val99);
+                    ByteBuffer _key106; // required
+                    ByteBuffer _val107; // required
+                    _key106 = iprot.readBinary();
+                    _val107 = iprot.readBinary();
+                    struct.attributes.put(_key106, _val107);
                   }
                   iprot.readMapEnd();
                 }
@@ -16213,10 +16487,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getVerTs_args stru
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter100 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter108 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter100.getKey());
-              oprot.writeBinary(_iter100.getValue());
+              oprot.writeBinary(_iter108.getKey());
+              oprot.writeBinary(_iter108.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -16277,10 +16551,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getVerTs_args struc
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter101 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter109 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter101.getKey());
-              oprot.writeBinary(_iter101.getValue());
+              oprot.writeBinary(_iter109.getKey());
+              oprot.writeBinary(_iter109.getValue());
             }
           }
         }
@@ -16312,15 +16586,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getVerTs_args struct
         }
         if (incoming.get(5)) {
           {
-            org.apache.thrift.protocol.TMap _map102 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map102.size);
-            for (int _i103 = 0; _i103 < _map102.size; ++_i103)
+            org.apache.thrift.protocol.TMap _map110 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map110.size);
+            for (int _i111 = 0; _i111 < _map110.size; ++_i111)
             {
-              ByteBuffer _key104; // required
-              ByteBuffer _val105; // optional
-              _key104 = iprot.readBinary();
-              _val105 = iprot.readBinary();
-              struct.attributes.put(_key104, _val105);
+              ByteBuffer _key112; // required
+              ByteBuffer _val113; // required
+              _key112 = iprot.readBinary();
+              _val113 = iprot.readBinary();
+              struct.attributes.put(_key112, _val113);
             }
           }
           struct.setAttributesIsSet(true);
@@ -16604,7 +16878,19 @@ public boolean equals(getVerTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getVerTs_result other) {
@@ -16715,14 +17001,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getVerTs_result str
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list106 = iprot.readListBegin();
-                  struct.success = new ArrayList<TCell>(_list106.size);
-                  for (int _i107 = 0; _i107 < _list106.size; ++_i107)
+                  org.apache.thrift.protocol.TList _list114 = iprot.readListBegin();
+                  struct.success = new ArrayList<TCell>(_list114.size);
+                  for (int _i115 = 0; _i115 < _list114.size; ++_i115)
                   {
-                    TCell _elem108; // optional
-                    _elem108 = new TCell();
-                    _elem108.read(iprot);
-                    struct.success.add(_elem108);
+                    TCell _elem116; // required
+                    _elem116 = new TCell();
+                    _elem116.read(iprot);
+                    struct.success.add(_elem116);
                   }
                   iprot.readListEnd();
                 }
@@ -16759,9 +17045,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getVerTs_result st
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TCell _iter109 : struct.success)
+            for (TCell _iter117 : struct.success)
             {
-              _iter109.write(oprot);
+              _iter117.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -16800,9 +17086,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getVerTs_result str
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TCell _iter110 : struct.success)
+            for (TCell _iter118 : struct.success)
             {
-              _iter110.write(oprot);
+              _iter118.write(oprot);
             }
           }
         }
@@ -16817,14 +17103,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getVerTs_result stru
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list111 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TCell>(_list111.size);
-            for (int _i112 = 0; _i112 < _list111.size; ++_i112)
+            org.apache.thrift.protocol.TList _list119 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TCell>(_list119.size);
+            for (int _i120 = 0; _i120 < _list119.size; ++_i120)
             {
-              TCell _elem113; // optional
-              _elem113 = new TCell();
-              _elem113.read(iprot);
-              struct.success.add(_elem113);
+              TCell _elem121; // required
+              _elem121 = new TCell();
+              _elem121.read(iprot);
+              struct.success.add(_elem121);
             }
           }
           struct.setSuccessIsSet(true);
@@ -17233,7 +17519,24 @@ public boolean equals(getRow_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRow_args other) {
@@ -17378,15 +17681,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRow_args struct)
             case 3: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map114 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map114.size);
-                  for (int _i115 = 0; _i115 < _map114.size; ++_i115)
+                  org.apache.thrift.protocol.TMap _map122 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map122.size);
+                  for (int _i123 = 0; _i123 < _map122.size; ++_i123)
                   {
-                    ByteBuffer _key116; // required
-                    ByteBuffer _val117; // optional
-                    _key116 = iprot.readBinary();
-                    _val117 = iprot.readBinary();
-                    struct.attributes.put(_key116, _val117);
+                    ByteBuffer _key124; // required
+                    ByteBuffer _val125; // required
+                    _key124 = iprot.readBinary();
+                    _val125 = iprot.readBinary();
+                    struct.attributes.put(_key124, _val125);
                   }
                   iprot.readMapEnd();
                 }
@@ -17424,10 +17727,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRow_args struct
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter118 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter126 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter118.getKey());
-              oprot.writeBinary(_iter118.getValue());
+              oprot.writeBinary(_iter126.getKey());
+              oprot.writeBinary(_iter126.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -17470,10 +17773,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRow_args struct)
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter119 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter127 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter119.getKey());
-              oprot.writeBinary(_iter119.getValue());
+              oprot.writeBinary(_iter127.getKey());
+              oprot.writeBinary(_iter127.getValue());
             }
           }
         }
@@ -17493,15 +17796,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRow_args struct)
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TMap _map120 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map120.size);
-            for (int _i121 = 0; _i121 < _map120.size; ++_i121)
+            org.apache.thrift.protocol.TMap _map128 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map128.size);
+            for (int _i129 = 0; _i129 < _map128.size; ++_i129)
             {
-              ByteBuffer _key122; // required
-              ByteBuffer _val123; // optional
-              _key122 = iprot.readBinary();
-              _val123 = iprot.readBinary();
-              struct.attributes.put(_key122, _val123);
+              ByteBuffer _key130; // required
+              ByteBuffer _val131; // required
+              _key130 = iprot.readBinary();
+              _val131 = iprot.readBinary();
+              struct.attributes.put(_key130, _val131);
             }
           }
           struct.setAttributesIsSet(true);
@@ -17785,7 +18088,19 @@ public boolean equals(getRow_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRow_result other) {
@@ -17896,14 +18211,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRow_result struc
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list124 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list124.size);
-                  for (int _i125 = 0; _i125 < _list124.size; ++_i125)
+                  org.apache.thrift.protocol.TList _list132 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list132.size);
+                  for (int _i133 = 0; _i133 < _list132.size; ++_i133)
                   {
-                    TRowResult _elem126; // optional
-                    _elem126 = new TRowResult();
-                    _elem126.read(iprot);
-                    struct.success.add(_elem126);
+                    TRowResult _elem134; // required
+                    _elem134 = new TRowResult();
+                    _elem134.read(iprot);
+                    struct.success.add(_elem134);
                   }
                   iprot.readListEnd();
                 }
@@ -17940,9 +18255,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRow_result stru
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter127 : struct.success)
+            for (TRowResult _iter135 : struct.success)
             {
-              _iter127.write(oprot);
+              _iter135.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -17981,9 +18296,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRow_result struc
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter128 : struct.success)
+            for (TRowResult _iter136 : struct.success)
             {
-              _iter128.write(oprot);
+              _iter136.write(oprot);
             }
           }
         }
@@ -17998,14 +18313,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRow_result struct
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list129 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list129.size);
-            for (int _i130 = 0; _i130 < _list129.size; ++_i130)
+            org.apache.thrift.protocol.TList _list137 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list137.size);
+            for (int _i138 = 0; _i138 < _list137.size; ++_i138)
             {
-              TRowResult _elem131; // optional
-              _elem131 = new TRowResult();
-              _elem131.read(iprot);
-              struct.success.add(_elem131);
+              TRowResult _elem139; // required
+              _elem139 = new TRowResult();
+              _elem139.read(iprot);
+              struct.success.add(_elem139);
             }
           }
           struct.setSuccessIsSet(true);
@@ -18505,7 +18820,29 @@ public boolean equals(getRowWithColumns_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowWithColumns_args other) {
@@ -18668,13 +19005,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumns_a
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list132 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list132.size);
-                  for (int _i133 = 0; _i133 < _list132.size; ++_i133)
+                  org.apache.thrift.protocol.TList _list140 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list140.size);
+                  for (int _i141 = 0; _i141 < _list140.size; ++_i141)
                   {
-                    ByteBuffer _elem134; // optional
-                    _elem134 = iprot.readBinary();
-                    struct.columns.add(_elem134);
+                    ByteBuffer _elem142; // required
+                    _elem142 = iprot.readBinary();
+                    struct.columns.add(_elem142);
                   }
                   iprot.readListEnd();
                 }
@@ -18686,15 +19023,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumns_a
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map135 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map135.size);
-                  for (int _i136 = 0; _i136 < _map135.size; ++_i136)
+                  org.apache.thrift.protocol.TMap _map143 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map143.size);
+                  for (int _i144 = 0; _i144 < _map143.size; ++_i144)
                   {
-                    ByteBuffer _key137; // required
-                    ByteBuffer _val138; // optional
-                    _key137 = iprot.readBinary();
-                    _val138 = iprot.readBinary();
-                    struct.attributes.put(_key137, _val138);
+                    ByteBuffer _key145; // required
+                    ByteBuffer _val146; // required
+                    _key145 = iprot.readBinary();
+                    _val146 = iprot.readBinary();
+                    struct.attributes.put(_key145, _val146);
                   }
                   iprot.readMapEnd();
                 }
@@ -18732,9 +19069,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumns_
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter139 : struct.columns)
+            for (ByteBuffer _iter147 : struct.columns)
             {
-              oprot.writeBinary(_iter139);
+              oprot.writeBinary(_iter147);
             }
             oprot.writeListEnd();
           }
@@ -18744,10 +19081,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumns_
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter140 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter148 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter140.getKey());
-              oprot.writeBinary(_iter140.getValue());
+              oprot.writeBinary(_iter148.getKey());
+              oprot.writeBinary(_iter148.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -18793,19 +19130,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_a
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter141 : struct.columns)
+            for (ByteBuffer _iter149 : struct.columns)
             {
-              oprot.writeBinary(_iter141);
+              oprot.writeBinary(_iter149);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter142 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter150 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter142.getKey());
-              oprot.writeBinary(_iter142.getValue());
+              oprot.writeBinary(_iter150.getKey());
+              oprot.writeBinary(_iter150.getValue());
             }
           }
         }
@@ -18825,28 +19162,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_ar
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list143 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list143.size);
-            for (int _i144 = 0; _i144 < _list143.size; ++_i144)
+            org.apache.thrift.protocol.TList _list151 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list151.size);
+            for (int _i152 = 0; _i152 < _list151.size; ++_i152)
             {
-              ByteBuffer _elem145; // optional
-              _elem145 = iprot.readBinary();
-              struct.columns.add(_elem145);
+              ByteBuffer _elem153; // required
+              _elem153 = iprot.readBinary();
+              struct.columns.add(_elem153);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map146 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map146.size);
-            for (int _i147 = 0; _i147 < _map146.size; ++_i147)
+            org.apache.thrift.protocol.TMap _map154 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map154.size);
+            for (int _i155 = 0; _i155 < _map154.size; ++_i155)
             {
-              ByteBuffer _key148; // required
-              ByteBuffer _val149; // optional
-              _key148 = iprot.readBinary();
-              _val149 = iprot.readBinary();
-              struct.attributes.put(_key148, _val149);
+              ByteBuffer _key156; // required
+              ByteBuffer _val157; // required
+              _key156 = iprot.readBinary();
+              _val157 = iprot.readBinary();
+              struct.attributes.put(_key156, _val157);
             }
           }
           struct.setAttributesIsSet(true);
@@ -19130,7 +19467,19 @@ public boolean equals(getRowWithColumns_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowWithColumns_result other) {
@@ -19241,14 +19590,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumns_r
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list150 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list150.size);
-                  for (int _i151 = 0; _i151 < _list150.size; ++_i151)
+                  org.apache.thrift.protocol.TList _list158 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list158.size);
+                  for (int _i159 = 0; _i159 < _list158.size; ++_i159)
                   {
-                    TRowResult _elem152; // optional
-                    _elem152 = new TRowResult();
-                    _elem152.read(iprot);
-                    struct.success.add(_elem152);
+                    TRowResult _elem160; // required
+                    _elem160 = new TRowResult();
+                    _elem160.read(iprot);
+                    struct.success.add(_elem160);
                   }
                   iprot.readListEnd();
                 }
@@ -19285,9 +19634,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumns_
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter153 : struct.success)
+            for (TRowResult _iter161 : struct.success)
             {
-              _iter153.write(oprot);
+              _iter161.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -19326,9 +19675,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_r
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter154 : struct.success)
+            for (TRowResult _iter162 : struct.success)
             {
-              _iter154.write(oprot);
+              _iter162.write(oprot);
             }
           }
         }
@@ -19343,14 +19692,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_re
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list155 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list155.size);
-            for (int _i156 = 0; _i156 < _list155.size; ++_i156)
+            org.apache.thrift.protocol.TList _list163 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list163.size);
+            for (int _i164 = 0; _i164 < _list163.size; ++_i164)
             {
-              TRowResult _elem157; // optional
-              _elem157 = new TRowResult();
-              _elem157.read(iprot);
-              struct.success.add(_elem157);
+              TRowResult _elem165; // required
+              _elem165 = new TRowResult();
+              _elem165.read(iprot);
+              struct.success.add(_elem165);
             }
           }
           struct.setSuccessIsSet(true);
@@ -19833,7 +20182,29 @@ public boolean equals(getRowTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowTs_args other) {
@@ -20002,15 +20373,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowTs_args struc
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map158 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map158.size);
-                  for (int _i159 = 0; _i159 < _map158.size; ++_i159)
+                  org.apache.thrift.protocol.TMap _map166 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map166.size);
+                  for (int _i167 = 0; _i167 < _map166.size; ++_i167)
                   {
-                    ByteBuffer _key160; // required
-                    ByteBuffer _val161; // optional
-                    _key160 = iprot.readBinary();
-                    _val161 = iprot.readBinary();
-                    struct.attributes.put(_key160, _val161);
+                    ByteBuffer _key168; // required
+                    ByteBuffer _val169; // required
+                    _key168 = iprot.readBinary();
+                    _val169 = iprot.readBinary();
+                    struct.attributes.put(_key168, _val169);
                   }
                   iprot.readMapEnd();
                 }
@@ -20051,10 +20422,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowTs_args stru
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter162 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter170 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter162.getKey());
-              oprot.writeBinary(_iter162.getValue());
+              oprot.writeBinary(_iter170.getKey());
+              oprot.writeBinary(_iter170.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -20103,10 +20474,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowTs_args struc
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter163 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter171 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter163.getKey());
-              oprot.writeBinary(_iter163.getValue());
+              oprot.writeBinary(_iter171.getKey());
+              oprot.writeBinary(_iter171.getValue());
             }
           }
         }
@@ -20130,15 +20501,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowTs_args struct
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map164 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map164.size);
-            for (int _i165 = 0; _i165 < _map164.size; ++_i165)
+            org.apache.thrift.protocol.TMap _map172 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map172.size);
+            for (int _i173 = 0; _i173 < _map172.size; ++_i173)
             {
-              ByteBuffer _key166; // required
-              ByteBuffer _val167; // optional
-              _key166 = iprot.readBinary();
-              _val167 = iprot.readBinary();
-              struct.attributes.put(_key166, _val167);
+              ByteBuffer _key174; // required
+              ByteBuffer _val175; // required
+              _key174 = iprot.readBinary();
+              _val175 = iprot.readBinary();
+              struct.attributes.put(_key174, _val175);
             }
           }
           struct.setAttributesIsSet(true);
@@ -20422,7 +20793,19 @@ public boolean equals(getRowTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowTs_result other) {
@@ -20533,14 +20916,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowTs_result str
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list168 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list168.size);
-                  for (int _i169 = 0; _i169 < _list168.size; ++_i169)
+                  org.apache.thrift.protocol.TList _list176 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list176.size);
+                  for (int _i177 = 0; _i177 < _list176.size; ++_i177)
                   {
-                    TRowResult _elem170; // optional
-                    _elem170 = new TRowResult();
-                    _elem170.read(iprot);
-                    struct.success.add(_elem170);
+                    TRowResult _elem178; // required
+                    _elem178 = new TRowResult();
+                    _elem178.read(iprot);
+                    struct.success.add(_elem178);
                   }
                   iprot.readListEnd();
                 }
@@ -20577,9 +20960,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowTs_result st
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter171 : struct.success)
+            for (TRowResult _iter179 : struct.success)
             {
-              _iter171.write(oprot);
+              _iter179.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -20618,9 +21001,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowTs_result str
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter172 : struct.success)
+            for (TRowResult _iter180 : struct.success)
             {
-              _iter172.write(oprot);
+              _iter180.write(oprot);
             }
           }
         }
@@ -20635,14 +21018,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowTs_result stru
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list173 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list173.size);
-            for (int _i174 = 0; _i174 < _list173.size; ++_i174)
+            org.apache.thrift.protocol.TList _list181 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list181.size);
+            for (int _i182 = 0; _i182 < _list181.size; ++_i182)
             {
-              TRowResult _elem175; // optional
-              _elem175 = new TRowResult();
-              _elem175.read(iprot);
-              struct.success.add(_elem175);
+              TRowResult _elem183; // required
+              _elem183 = new TRowResult();
+              _elem183.read(iprot);
+              struct.success.add(_elem183);
             }
           }
           struct.setSuccessIsSet(true);
@@ -21204,7 +21587,34 @@ public boolean equals(getRowWithColumnsTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowWithColumnsTs_args other) {
@@ -21383,13 +21793,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumnsTs
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list176 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list176.size);
-                  for (int _i177 = 0; _i177 < _list176.size; ++_i177)
+                  org.apache.thrift.protocol.TList _list184 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list184.size);
+                  for (int _i185 = 0; _i185 < _list184.size; ++_i185)
                   {
-                    ByteBuffer _elem178; // optional
-                    _elem178 = iprot.readBinary();
-                    struct.columns.add(_elem178);
+                    ByteBuffer _elem186; // required
+                    _elem186 = iprot.readBinary();
+                    struct.columns.add(_elem186);
                   }
                   iprot.readListEnd();
                 }
@@ -21409,15 +21819,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumnsTs
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map179 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map179.size);
-                  for (int _i180 = 0; _i180 < _map179.size; ++_i180)
+                  org.apache.thrift.protocol.TMap _map187 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map187.size);
+                  for (int _i188 = 0; _i188 < _map187.size; ++_i188)
                   {
-                    ByteBuffer _key181; // required
-                    ByteBuffer _val182; // optional
-                    _key181 = iprot.readBinary();
-                    _val182 = iprot.readBinary();
-                    struct.attributes.put(_key181, _val182);
+                    ByteBuffer _key189; // required
+                    ByteBuffer _val190; // required
+                    _key189 = iprot.readBinary();
+                    _val190 = iprot.readBinary();
+                    struct.attributes.put(_key189, _val190);
                   }
                   iprot.readMapEnd();
                 }
@@ -21455,9 +21865,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumnsT
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter183 : struct.columns)
+            for (ByteBuffer _iter191 : struct.columns)
             {
-              oprot.writeBinary(_iter183);
+              oprot.writeBinary(_iter191);
             }
             oprot.writeListEnd();
           }
@@ -21470,10 +21880,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumnsT
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter184 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter192 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter184.getKey());
-              oprot.writeBinary(_iter184.getValue());
+              oprot.writeBinary(_iter192.getKey());
+              oprot.writeBinary(_iter192.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -21522,9 +21932,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter185 : struct.columns)
+            for (ByteBuffer _iter193 : struct.columns)
             {
-              oprot.writeBinary(_iter185);
+              oprot.writeBinary(_iter193);
             }
           }
         }
@@ -21534,10 +21944,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter186 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter194 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter186.getKey());
-              oprot.writeBinary(_iter186.getValue());
+              oprot.writeBinary(_iter194.getKey());
+              oprot.writeBinary(_iter194.getValue());
             }
           }
         }
@@ -21557,13 +21967,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list187 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list187.size);
-            for (int _i188 = 0; _i188 < _list187.size; ++_i188)
+            org.apache.thrift.protocol.TList _list195 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list195.size);
+            for (int _i196 = 0; _i196 < _list195.size; ++_i196)
             {
-              ByteBuffer _elem189; // optional
-              _elem189 = iprot.readBinary();
-              struct.columns.add(_elem189);
+              ByteBuffer _elem197; // required
+              _elem197 = iprot.readBinary();
+              struct.columns.add(_elem197);
             }
           }
           struct.setColumnsIsSet(true);
@@ -21574,15 +21984,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map190 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map190.size);
-            for (int _i191 = 0; _i191 < _map190.size; ++_i191)
+            org.apache.thrift.protocol.TMap _map198 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map198.size);
+            for (int _i199 = 0; _i199 < _map198.size; ++_i199)
             {
-              ByteBuffer _key192; // required
-              ByteBuffer _val193; // optional
-              _key192 = iprot.readBinary();
-              _val193 = iprot.readBinary();
-              struct.attributes.put(_key192, _val193);
+              ByteBuffer _key200; // required
+              ByteBuffer _val201; // required
+              _key200 = iprot.readBinary();
+              _val201 = iprot.readBinary();
+              struct.attributes.put(_key200, _val201);
             }
           }
           struct.setAttributesIsSet(true);
@@ -21866,7 +22276,19 @@ public boolean equals(getRowWithColumnsTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowWithColumnsTs_result other) {
@@ -21977,14 +22399,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumnsTs
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list194 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list194.size);
-                  for (int _i195 = 0; _i195 < _list194.size; ++_i195)
+                  org.apache.thrift.protocol.TList _list202 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list202.size);
+                  for (int _i203 = 0; _i203 < _list202.size; ++_i203)
                   {
-                    TRowResult _elem196; // optional
-                    _elem196 = new TRowResult();
-                    _elem196.read(iprot);
-                    struct.success.add(_elem196);
+                    TRowResult _elem204; // required
+                    _elem204 = new TRowResult();
+                    _elem204.read(iprot);
+                    struct.success.add(_elem204);
                   }
                   iprot.readListEnd();
                 }
@@ -22021,9 +22443,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumnsT
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter197 : struct.success)
+            for (TRowResult _iter205 : struct.success)
             {
-              _iter197.write(oprot);
+              _iter205.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -22062,9 +22484,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter198 : struct.success)
+            for (TRowResult _iter206 : struct.success)
             {
-              _iter198.write(oprot);
+              _iter206.write(oprot);
             }
           }
         }
@@ -22079,14 +22501,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list199 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list199.size);
-            for (int _i200 = 0; _i200 < _list199.size; ++_i200)
+            org.apache.thrift.protocol.TList _list207 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list207.size);
+            for (int _i208 = 0; _i208 < _list207.size; ++_i208)
             {
-              TRowResult _elem201; // optional
-              _elem201 = new TRowResult();
-              _elem201.read(iprot);
-              struct.success.add(_elem201);
+              TRowResult _elem209; // required
+              _elem209 = new TRowResult();
+              _elem209.read(iprot);
+              struct.success.add(_elem209);
             }
           }
           struct.setSuccessIsSet(true);
@@ -22505,7 +22927,24 @@ public boolean equals(getRows_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_rows = true && (isSetRows());
+      builder.append(present_rows);
+      if (present_rows)
+        builder.append(rows);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRows_args other) {
@@ -22642,13 +23081,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRows_args struct
             case 2: // ROWS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list202 = iprot.readListBegin();
-                  struct.rows = new ArrayList<ByteBuffer>(_list202.size);
-                  for (int _i203 = 0; _i203 < _list202.size; ++_i203)
+                  org.apache.thrift.protocol.TList _list210 = iprot.readListBegin();
+                  struct.rows = new ArrayList<ByteBuffer>(_list210.size);
+                  for (int _i211 = 0; _i211 < _list210.size; ++_i211)
                   {
-                    ByteBuffer _elem204; // optional
-                    _elem204 = iprot.readBinary();
-                    struct.rows.add(_elem204);
+                    ByteBuffer _elem212; // required
+                    _elem212 = iprot.readBinary();
+                    struct.rows.add(_elem212);
                   }
                   iprot.readListEnd();
                 }
@@ -22660,15 +23099,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRows_args struct
             case 3: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map205 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map205.size);
-                  for (int _i206 = 0; _i206 < _map205.size; ++_i206)
+                  org.apache.thrift.protocol.TMap _map213 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map213.size);
+                  for (int _i214 = 0; _i214 < _map213.size; ++_i214)
                   {
-                    ByteBuffer _key207; // required
-                    ByteBuffer _val208; // optional
-                    _key207 = iprot.readBinary();
-                    _val208 = iprot.readBinary();
-                    struct.attributes.put(_key207, _val208);
+                    ByteBuffer _key215; // required
+                    ByteBuffer _val216; // required
+                    _key215 = iprot.readBinary();
+                    _val216 = iprot.readBinary();
+                    struct.attributes.put(_key215, _val216);
                   }
                   iprot.readMapEnd();
                 }
@@ -22701,9 +23140,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRows_args struc
           oprot.writeFieldBegin(ROWS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size()));
-            for (ByteBuffer _iter209 : struct.rows)
+            for (ByteBuffer _iter217 : struct.rows)
             {
-              oprot.writeBinary(_iter209);
+              oprot.writeBinary(_iter217);
             }
             oprot.writeListEnd();
           }
@@ -22713,10 +23152,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRows_args struc
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter210 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter218 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter210.getKey());
-              oprot.writeBinary(_iter210.getValue());
+              oprot.writeBinary(_iter218.getKey());
+              oprot.writeBinary(_iter218.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -22756,19 +23195,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRows_args struct
         if (struct.isSetRows()) {
           {
             oprot.writeI32(struct.rows.size());
-            for (ByteBuffer _iter211 : struct.rows)
+            for (ByteBuffer _iter219 : struct.rows)
             {
-              oprot.writeBinary(_iter211);
+              oprot.writeBinary(_iter219);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter212 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter220 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter212.getKey());
-              oprot.writeBinary(_iter212.getValue());
+              oprot.writeBinary(_iter220.getKey());
+              oprot.writeBinary(_iter220.getValue());
             }
           }
         }
@@ -22784,28 +23223,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRows_args struct)
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list213 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.rows = new ArrayList<ByteBuffer>(_list213.size);
-            for (int _i214 = 0; _i214 < _list213.size; ++_i214)
+            org.apache.thrift.protocol.TList _list221 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.rows = new ArrayList<ByteBuffer>(_list221.size);
+            for (int _i222 = 0; _i222 < _list221.size; ++_i222)
             {
-              ByteBuffer _elem215; // optional
-              _elem215 = iprot.readBinary();
-              struct.rows.add(_elem215);
+              ByteBuffer _elem223; // required
+              _elem223 = iprot.readBinary();
+              struct.rows.add(_elem223);
             }
           }
           struct.setRowsIsSet(true);
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TMap _map216 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map216.size);
-            for (int _i217 = 0; _i217 < _map216.size; ++_i217)
+            org.apache.thrift.protocol.TMap _map224 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map224.size);
+            for (int _i225 = 0; _i225 < _map224.size; ++_i225)
             {
-              ByteBuffer _key218; // required
-              ByteBuffer _val219; // optional
-              _key218 = iprot.readBinary();
-              _val219 = iprot.readBinary();
-              struct.attributes.put(_key218, _val219);
+              ByteBuffer _key226; // required
+              ByteBuffer _val227; // required
+              _key226 = iprot.readBinary();
+              _val227 = iprot.readBinary();
+              struct.attributes.put(_key226, _val227);
             }
           }
           struct.setAttributesIsSet(true);
@@ -23089,7 +23528,19 @@ public boolean equals(getRows_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRows_result other) {
@@ -23200,14 +23651,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRows_result stru
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list220 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list220.size);
-                  for (int _i221 = 0; _i221 < _list220.size; ++_i221)
+                  org.apache.thrift.protocol.TList _list228 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list228.size);
+                  for (int _i229 = 0; _i229 < _list228.size; ++_i229)
                   {
-                    TRowResult _elem222; // optional
-                    _elem222 = new TRowResult();
-                    _elem222.read(iprot);
-                    struct.success.add(_elem222);
+                    TRowResult _elem230; // required
+                    _elem230 = new TRowResult();
+                    _elem230.read(iprot);
+                    struct.success.add(_elem230);
                   }
                   iprot.readListEnd();
                 }
@@ -23244,9 +23695,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRows_result str
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter223 : struct.success)
+            for (TRowResult _iter231 : struct.success)
             {
-              _iter223.write(oprot);
+              _iter231.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -23285,9 +23736,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRows_result stru
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter224 : struct.success)
+            for (TRowResult _iter232 : struct.success)
             {
-              _iter224.write(oprot);
+              _iter232.write(oprot);
             }
           }
         }
@@ -23302,14 +23753,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRows_result struc
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list225 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list225.size);
-            for (int _i226 = 0; _i226 < _list225.size; ++_i226)
+            org.apache.thrift.protocol.TList _list233 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list233.size);
+            for (int _i234 = 0; _i234 < _list233.size; ++_i234)
             {
-              TRowResult _elem227; // optional
-              _elem227 = new TRowResult();
-              _elem227.read(iprot);
-              struct.success.add(_elem227);
+              TRowResult _elem235; // required
+              _elem235 = new TRowResult();
+              _elem235.read(iprot);
+              struct.success.add(_elem235);
             }
           }
           struct.setSuccessIsSet(true);
@@ -23819,7 +24270,29 @@ public boolean equals(getRowsWithColumns_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_rows = true && (isSetRows());
+      builder.append(present_rows);
+      if (present_rows)
+        builder.append(rows);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowsWithColumns_args other) {
@@ -23974,13 +24447,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_
             case 2: // ROWS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list228 = iprot.readListBegin();
-                  struct.rows = new ArrayList<ByteBuffer>(_list228.size);
-                  for (int _i229 = 0; _i229 < _list228.size; ++_i229)
+                  org.apache.thrift.protocol.TList _list236 = iprot.readListBegin();
+                  struct.rows = new ArrayList<ByteBuffer>(_list236.size);
+                  for (int _i237 = 0; _i237 < _list236.size; ++_i237)
                   {
-                    ByteBuffer _elem230; // optional
-                    _elem230 = iprot.readBinary();
-                    struct.rows.add(_elem230);
+                    ByteBuffer _elem238; // required
+                    _elem238 = iprot.readBinary();
+                    struct.rows.add(_elem238);
                   }
                   iprot.readListEnd();
                 }
@@ -23992,13 +24465,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list231 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list231.size);
-                  for (int _i232 = 0; _i232 < _list231.size; ++_i232)
+                  org.apache.thrift.protocol.TList _list239 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list239.size);
+                  for (int _i240 = 0; _i240 < _list239.size; ++_i240)
                   {
-                    ByteBuffer _elem233; // optional
-                    _elem233 = iprot.readBinary();
-                    struct.columns.add(_elem233);
+                    ByteBuffer _elem241; // required
+                    _elem241 = iprot.readBinary();
+                    struct.columns.add(_elem241);
                   }
                   iprot.readListEnd();
                 }
@@ -24010,15 +24483,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map234 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map234.size);
-                  for (int _i235 = 0; _i235 < _map234.size; ++_i235)
+                  org.apache.thrift.protocol.TMap _map242 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map242.size);
+                  for (int _i243 = 0; _i243 < _map242.size; ++_i243)
                   {
-                    ByteBuffer _key236; // required
-                    ByteBuffer _val237; // optional
-                    _key236 = iprot.readBinary();
-                    _val237 = iprot.readBinary();
-                    struct.attributes.put(_key236, _val237);
+                    ByteBuffer _key244; // required
+                    ByteBuffer _val245; // required
+                    _key244 = iprot.readBinary();
+                    _val245 = iprot.readBinary();
+                    struct.attributes.put(_key244, _val245);
                   }
                   iprot.readMapEnd();
                 }
@@ -24051,9 +24524,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(ROWS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size()));
-            for (ByteBuffer _iter238 : struct.rows)
+            for (ByteBuffer _iter246 : struct.rows)
             {
-              oprot.writeBinary(_iter238);
+              oprot.writeBinary(_iter246);
             }
             oprot.writeListEnd();
           }
@@ -24063,9 +24536,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter239 : struct.columns)
+            for (ByteBuffer _iter247 : struct.columns)
             {
-              oprot.writeBinary(_iter239);
+              oprot.writeBinary(_iter247);
             }
             oprot.writeListEnd();
           }
@@ -24075,10 +24548,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter240 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter248 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter240.getKey());
-              oprot.writeBinary(_iter240.getValue());
+              oprot.writeBinary(_iter248.getKey());
+              oprot.writeBinary(_iter248.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -24121,28 +24594,28 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_
         if (struct.isSetRows()) {
           {
             oprot.writeI32(struct.rows.size());
-            for (ByteBuffer _iter241 : struct.rows)
+            for (ByteBuffer _iter249 : struct.rows)
             {
-              oprot.writeBinary(_iter241);
+              oprot.writeBinary(_iter249);
             }
           }
         }
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter242 : struct.columns)
+            for (ByteBuffer _iter250 : struct.columns)
             {
-              oprot.writeBinary(_iter242);
+              oprot.writeBinary(_iter250);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter243 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter251 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter243.getKey());
-              oprot.writeBinary(_iter243.getValue());
+              oprot.writeBinary(_iter251.getKey());
+              oprot.writeBinary(_iter251.getValue());
             }
           }
         }
@@ -24158,41 +24631,41 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_a
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list244 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.rows = new ArrayList<ByteBuffer>(_list244.size);
-            for (int _i245 = 0; _i245 < _list244.size; ++_i245)
+            org.apache.thrift.protocol.TList _list252 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.rows = new ArrayList<ByteBuffer>(_list252.size);
+            for (int _i253 = 0; _i253 < _list252.size; ++_i253)
             {
-              ByteBuffer _elem246; // optional
-              _elem246 = iprot.readBinary();
-              struct.rows.add(_elem246);
+              ByteBuffer _elem254; // required
+              _elem254 = iprot.readBinary();
+              struct.rows.add(_elem254);
             }
           }
           struct.setRowsIsSet(true);
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list247 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list247.size);
-            for (int _i248 = 0; _i248 < _list247.size; ++_i248)
+            org.apache.thrift.protocol.TList _list255 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list255.size);
+            for (int _i256 = 0; _i256 < _list255.size; ++_i256)
             {
-              ByteBuffer _elem249; // optional
-              _elem249 = iprot.readBinary();
-              struct.columns.add(_elem249);
+              ByteBuffer _elem257; // required
+              _elem257 = iprot.readBinary();
+              struct.columns.add(_elem257);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map250 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map250.size);
-            for (int _i251 = 0; _i251 < _map250.size; ++_i251)
+            org.apache.thrift.protocol.TMap _map258 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map258.size);
+            for (int _i259 = 0; _i259 < _map258.size; ++_i259)
             {
-              ByteBuffer _key252; // required
-              ByteBuffer _val253; // optional
-              _key252 = iprot.readBinary();
-              _val253 = iprot.readBinary();
-              struct.attributes.put(_key252, _val253);
+              ByteBuffer _key260; // required
+              ByteBuffer _val261; // required
+              _key260 = iprot.readBinary();
+              _val261 = iprot.readBinary();
+              struct.attributes.put(_key260, _val261);
             }
           }
           struct.setAttributesIsSet(true);
@@ -24476,7 +24949,19 @@ public boolean equals(getRowsWithColumns_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowsWithColumns_result other) {
@@ -24587,14 +25072,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list254 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list254.size);
-                  for (int _i255 = 0; _i255 < _list254.size; ++_i255)
+                  org.apache.thrift.protocol.TList _list262 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list262.size);
+                  for (int _i263 = 0; _i263 < _list262.size; ++_i263)
                   {
-                    TRowResult _elem256; // optional
-                    _elem256 = new TRowResult();
-                    _elem256.read(iprot);
-                    struct.success.add(_elem256);
+                    TRowResult _elem264; // required
+                    _elem264 = new TRowResult();
+                    _elem264.read(iprot);
+                    struct.success.add(_elem264);
                   }
                   iprot.readListEnd();
                 }
@@ -24631,9 +25116,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter257 : struct.success)
+            for (TRowResult _iter265 : struct.success)
             {
-              _iter257.write(oprot);
+              _iter265.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -24672,9 +25157,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter258 : struct.success)
+            for (TRowResult _iter266 : struct.success)
             {
-              _iter258.write(oprot);
+              _iter266.write(oprot);
             }
           }
         }
@@ -24689,14 +25174,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_r
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list259 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list259.size);
-            for (int _i260 = 0; _i260 < _list259.size; ++_i260)
+            org.apache.thrift.protocol.TList _list267 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list267.size);
+            for (int _i268 = 0; _i268 < _list267.size; ++_i268)
             {
-              TRowResult _elem261; // optional
-              _elem261 = new TRowResult();
-              _elem261.read(iprot);
-              struct.success.add(_elem261);
+              TRowResult _elem269; // required
+              _elem269 = new TRowResult();
+              _elem269.read(iprot);
+              struct.success.add(_elem269);
             }
           }
           struct.setSuccessIsSet(true);
@@ -25189,7 +25674,29 @@ public boolean equals(getRowsTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_rows = true && (isSetRows());
+      builder.append(present_rows);
+      if (present_rows)
+        builder.append(rows);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowsTs_args other) {
@@ -25342,13 +25849,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsTs_args stru
             case 2: // ROWS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list262 = iprot.readListBegin();
-                  struct.rows = new ArrayList<ByteBuffer>(_list262.size);
-                  for (int _i263 = 0; _i263 < _list262.size; ++_i263)
+                  org.apache.thrift.protocol.TList _list270 = iprot.readListBegin();
+                  struct.rows = new ArrayList<ByteBuffer>(_list270.size);
+                  for (int _i271 = 0; _i271 < _list270.size; ++_i271)
                   {
-                    ByteBuffer _elem264; // optional
-                    _elem264 = iprot.readBinary();
-                    struct.rows.add(_elem264);
+                    ByteBuffer _elem272; // required
+                    _elem272 = iprot.readBinary();
+                    struct.rows.add(_elem272);
                   }
                   iprot.readListEnd();
                 }
@@ -25368,15 +25875,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsTs_args stru
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map265 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map265.size);
-                  for (int _i266 = 0; _i266 < _map265.size; ++_i266)
+                  org.apache.thrift.protocol.TMap _map273 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map273.size);
+                  for (int _i274 = 0; _i274 < _map273.size; ++_i274)
                   {
-                    ByteBuffer _key267; // required
-                    ByteBuffer _val268; // optional
-                    _key267 = iprot.readBinary();
-                    _val268 = iprot.readBinary();
-                    struct.attributes.put(_key267, _val268);
+                    ByteBuffer _key275; // required
+                    ByteBuffer _val276; // required
+                    _key275 = iprot.readBinary();
+                    _val276 = iprot.readBinary();
+                    struct.attributes.put(_key275, _val276);
                   }
                   iprot.readMapEnd();
                 }
@@ -25409,9 +25916,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsTs_args str
           oprot.writeFieldBegin(ROWS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size()));
-            for (ByteBuffer _iter269 : struct.rows)
+            for (ByteBuffer _iter277 : struct.rows)
             {
-              oprot.writeBinary(_iter269);
+              oprot.writeBinary(_iter277);
             }
             oprot.writeListEnd();
           }
@@ -25424,10 +25931,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsTs_args str
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter270 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter278 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter270.getKey());
-              oprot.writeBinary(_iter270.getValue());
+              oprot.writeBinary(_iter278.getKey());
+              oprot.writeBinary(_iter278.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -25470,9 +25977,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args stru
         if (struct.isSetRows()) {
           {
             oprot.writeI32(struct.rows.size());
-            for (ByteBuffer _iter271 : struct.rows)
+            for (ByteBuffer _iter279 : struct.rows)
             {
-              oprot.writeBinary(_iter271);
+              oprot.writeBinary(_iter279);
             }
           }
         }
@@ -25482,10 +25989,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args stru
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter272 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter280 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter272.getKey());
-              oprot.writeBinary(_iter272.getValue());
+              oprot.writeBinary(_iter280.getKey());
+              oprot.writeBinary(_iter280.getValue());
             }
           }
         }
@@ -25501,13 +26008,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args struc
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list273 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.rows = new ArrayList<ByteBuffer>(_list273.size);
-            for (int _i274 = 0; _i274 < _list273.size; ++_i274)
+            org.apache.thrift.protocol.TList _list281 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.rows = new ArrayList<ByteBuffer>(_list281.size);
+            for (int _i282 = 0; _i282 < _list281.size; ++_i282)
             {
-              ByteBuffer _elem275; // optional
-              _elem275 = iprot.readBinary();
-              struct.rows.add(_elem275);
+              ByteBuffer _elem283; // required
+              _elem283 = iprot.readBinary();
+              struct.rows.add(_elem283);
             }
           }
           struct.setRowsIsSet(true);
@@ -25518,15 +26025,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args struc
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map276 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map276.size);
-            for (int _i277 = 0; _i277 < _map276.size; ++_i277)
+            org.apache.thrift.protocol.TMap _map284 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map284.size);
+            for (int _i285 = 0; _i285 < _map284.size; ++_i285)
             {
-              ByteBuffer _key278; // required
-              ByteBuffer _val279; // optional
-              _key278 = iprot.readBinary();
-              _val279 = iprot.readBinary();
-              struct.attributes.put(_key278, _val279);
+              ByteBuffer _key286; // required
+              ByteBuffer _val287; // required
+              _key286 = iprot.readBinary();
+              _val287 = iprot.readBinary();
+              struct.attributes.put(_key286, _val287);
             }
           }
           struct.setAttributesIsSet(true);
@@ -25810,7 +26317,19 @@ public boolean equals(getRowsTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowsTs_result other) {
@@ -25921,14 +26440,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsTs_result st
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list280 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list280.size);
-                  for (int _i281 = 0; _i281 < _list280.size; ++_i281)
+                  org.apache.thrift.protocol.TList _list288 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list288.size);
+                  for (int _i289 = 0; _i289 < _list288.size; ++_i289)
                   {
-                    TRowResult _elem282; // optional
-                    _elem282 = new TRowResult();
-                    _elem282.read(iprot);
-                    struct.success.add(_elem282);
+                    TRowResult _elem290; // required
+                    _elem290 = new TRowResult();
+                    _elem290.read(iprot);
+                    struct.success.add(_elem290);
                   }
                   iprot.readListEnd();
                 }
@@ -25965,9 +26484,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsTs_result s
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter283 : struct.success)
+            for (TRowResult _iter291 : struct.success)
             {
-              _iter283.write(oprot);
+              _iter291.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -26006,9 +26525,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsTs_result st
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter284 : struct.success)
+            for (TRowResult _iter292 : struct.success)
             {
-              _iter284.write(oprot);
+              _iter292.write(oprot);
             }
           }
         }
@@ -26023,14 +26542,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsTs_result str
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list285 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list285.size);
-            for (int _i286 = 0; _i286 < _list285.size; ++_i286)
+            org.apache.thrift.protocol.TList _list293 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list293.size);
+            for (int _i294 = 0; _i294 < _list293.size; ++_i294)
             {
-              TRowResult _elem287; // optional
-              _elem287 = new TRowResult();
-              _elem287.read(iprot);
-              struct.success.add(_elem287);
+              TRowResult _elem295; // required
+              _elem295 = new TRowResult();
+              _elem295.read(iprot);
+              struct.success.add(_elem295);
             }
           }
           struct.setSuccessIsSet(true);
@@ -26602,7 +27121,34 @@ public boolean equals(getRowsWithColumnsTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_rows = true && (isSetRows());
+      builder.append(present_rows);
+      if (present_rows)
+        builder.append(rows);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowsWithColumnsTs_args other) {
@@ -26773,13 +27319,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsT
             case 2: // ROWS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list288 = iprot.readListBegin();
-                  struct.rows = new ArrayList<ByteBuffer>(_list288.size);
-                  for (int _i289 = 0; _i289 < _list288.size; ++_i289)
+                  org.apache.thrift.protocol.TList _list296 = iprot.readListBegin();
+                  struct.rows = new ArrayList<ByteBuffer>(_list296.size);
+                  for (int _i297 = 0; _i297 < _list296.size; ++_i297)
                   {
-                    ByteBuffer _elem290; // optional
-                    _elem290 = iprot.readBinary();
-                    struct.rows.add(_elem290);
+                    ByteBuffer _elem298; // required
+                    _elem298 = iprot.readBinary();
+                    struct.rows.add(_elem298);
                   }
                   iprot.readListEnd();
                 }
@@ -26791,13 +27337,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsT
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list291 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list291.size);
-                  for (int _i292 = 0; _i292 < _list291.size; ++_i292)
+                  org.apache.thrift.protocol.TList _list299 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list299.size);
+                  for (int _i300 = 0; _i300 < _list299.size; ++_i300)
                   {
-                    ByteBuffer _elem293; // optional
-                    _elem293 = iprot.readBinary();
-                    struct.columns.add(_elem293);
+                    ByteBuffer _elem301; // required
+                    _elem301 = iprot.readBinary();
+                    struct.columns.add(_elem301);
                   }
                   iprot.readListEnd();
                 }
@@ -26817,15 +27363,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsT
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map294 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map294.size);
-                  for (int _i295 = 0; _i295 < _map294.size; ++_i295)
+                  org.apache.thrift.protocol.TMap _map302 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map302.size);
+                  for (int _i303 = 0; _i303 < _map302.size; ++_i303)
                   {
-                    ByteBuffer _key296; // required
-                    ByteBuffer _val297; // optional
-                    _key296 = iprot.readBinary();
-                    _val297 = iprot.readBinary();
-                    struct.attributes.put(_key296, _val297);
+                    ByteBuffer _key304; // required
+                    ByteBuffer _val305; // required
+                    _key304 = iprot.readBinary();
+                    _val305 = iprot.readBinary();
+                    struct.attributes.put(_key304, _val305);
                   }
                   iprot.readMapEnd();
                 }
@@ -26858,9 +27404,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(ROWS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size()));
-            for (ByteBuffer _iter298 : struct.rows)
+            for (ByteBuffer _iter306 : struct.rows)
             {
-              oprot.writeBinary(_iter298);
+              oprot.writeBinary(_iter306);
             }
             oprot.writeListEnd();
           }
@@ -26870,9 +27416,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter299 : struct.columns)
+            for (ByteBuffer _iter307 : struct.columns)
             {
-              oprot.writeBinary(_iter299);
+              oprot.writeBinary(_iter307);
             }
             oprot.writeListEnd();
           }
@@ -26885,10 +27431,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter300 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter308 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter300.getKey());
-              oprot.writeBinary(_iter300.getValue());
+              oprot.writeBinary(_iter308.getKey());
+              oprot.writeBinary(_iter308.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -26934,18 +27480,18 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsT
         if (struct.isSetRows()) {
           {
             oprot.writeI32(struct.rows.size());
-            for (ByteBuffer _iter301 : struct.rows)
+            for (ByteBuffer _iter309 : struct.rows)
             {
-              oprot.writeBinary(_iter301);
+              oprot.writeBinary(_iter309);
             }
           }
         }
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter302 : struct.columns)
+            for (ByteBuffer _iter310 : struct.columns)
             {
-              oprot.writeBinary(_iter302);
+              oprot.writeBinary(_iter310);
             }
           }
         }
@@ -26955,10 +27501,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsT
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter303 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter311 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter303.getKey());
-              oprot.writeBinary(_iter303.getValue());
+              oprot.writeBinary(_iter311.getKey());
+              oprot.writeBinary(_iter311.getValue());
             }
           }
         }
@@ -26974,26 +27520,26 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list304 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.rows = new ArrayList<ByteBuffer>(_list304.size);
-            for (int _i305 = 0; _i305 < _list304.size; ++_i305)
+            org.apache.thrift.protocol.TList _list312 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.rows = new ArrayList<ByteBuffer>(_list312.size);
+            for (int _i313 = 0; _i313 < _list312.size; ++_i313)
             {
-              ByteBuffer _elem306; // optional
-              _elem306 = iprot.readBinary();
-              struct.rows.add(_elem306);
+              ByteBuffer _elem314; // required
+              _elem314 = iprot.readBinary();
+              struct.rows.add(_elem314);
             }
           }
           struct.setRowsIsSet(true);
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list307 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list307.size);
-            for (int _i308 = 0; _i308 < _list307.size; ++_i308)
+            org.apache.thrift.protocol.TList _list315 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list315.size);
+            for (int _i316 = 0; _i316 < _list315.size; ++_i316)
             {
-              ByteBuffer _elem309; // optional
-              _elem309 = iprot.readBinary();
-              struct.columns.add(_elem309);
+              ByteBuffer _elem317; // required
+              _elem317 = iprot.readBinary();
+              struct.columns.add(_elem317);
             }
           }
           struct.setColumnsIsSet(true);
@@ -27004,15 +27550,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map310 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map310.size);
-            for (int _i311 = 0; _i311 < _map310.size; ++_i311)
+            org.apache.thrift.protocol.TMap _map318 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map318.size);
+            for (int _i319 = 0; _i319 < _map318.size; ++_i319)
             {
-              ByteBuffer _key312; // required
-              ByteBuffer _val313; // optional
-              _key312 = iprot.readBinary();
-              _val313 = iprot.readBinary();
-              struct.attributes.put(_key312, _val313);
+              ByteBuffer _key320; // required
+              ByteBuffer _val321; // required
+              _key320 = iprot.readBinary();
+              _val321 = iprot.readBinary();
+              struct.attributes.put(_key320, _val321);
             }
           }
           struct.setAttributesIsSet(true);
@@ -27296,7 +27842,19 @@ public boolean equals(getRowsWithColumnsTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowsWithColumnsTs_result other) {
@@ -27407,14 +27965,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsT
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list314 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list314.size);
-                  for (int _i315 = 0; _i315 < _list314.size; ++_i315)
+                  org.apache.thrift.protocol.TList _list322 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list322.size);
+                  for (int _i323 = 0; _i323 < _list322.size; ++_i323)
                   {
-                    TRowResult _elem316; // optional
-                    _elem316 = new TRowResult();
-                    _elem316.read(iprot);
-                    struct.success.add(_elem316);
+                    TRowResult _elem324; // required
+                    _elem324 = new TRowResult();
+                    _elem324.read(iprot);
+                    struct.success.add(_elem324);
                   }
                   iprot.readListEnd();
                 }
@@ -27451,9 +28009,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter317 : struct.success)
+            for (TRowResult _iter325 : struct.success)
             {
-              _iter317.write(oprot);
+              _iter325.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -27492,9 +28050,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsT
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter318 : struct.success)
+            for (TRowResult _iter326 : struct.success)
             {
-              _iter318.write(oprot);
+              _iter326.write(oprot);
             }
           }
         }
@@ -27509,14 +28067,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list319 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list319.size);
-            for (int _i320 = 0; _i320 < _list319.size; ++_i320)
+            org.apache.thrift.protocol.TList _list327 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list327.size);
+            for (int _i328 = 0; _i328 < _list327.size; ++_i328)
             {
-              TRowResult _elem321; // optional
-              _elem321 = new TRowResult();
-              _elem321.read(iprot);
-              struct.success.add(_elem321);
+              TRowResult _elem329; // required
+              _elem329 = new TRowResult();
+              _elem329.read(iprot);
+              struct.success.add(_elem329);
             }
           }
           struct.setSuccessIsSet(true);
@@ -28016,7 +28574,29 @@ public boolean equals(mutateRow_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_mutations = true && (isSetMutations());
+      builder.append(present_mutations);
+      if (present_mutations)
+        builder.append(mutations);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRow_args other) {
@@ -28179,14 +28759,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_args stru
             case 3: // MUTATIONS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list322 = iprot.readListBegin();
-                  struct.mutations = new ArrayList<Mutation>(_list322.size);
-                  for (int _i323 = 0; _i323 < _list322.size; ++_i323)
+                  org.apache.thrift.protocol.TList _list330 = iprot.readListBegin();
+                  struct.mutations = new ArrayList<Mutation>(_list330.size);
+                  for (int _i331 = 0; _i331 < _list330.size; ++_i331)
                   {
-                    Mutation _elem324; // optional
-                    _elem324 = new Mutation();
-                    _elem324.read(iprot);
-                    struct.mutations.add(_elem324);
+                    Mutation _elem332; // required
+                    _elem332 = new Mutation();
+                    _elem332.read(iprot);
+                    struct.mutations.add(_elem332);
                   }
                   iprot.readListEnd();
                 }
@@ -28198,15 +28778,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_args stru
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map325 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map325.size);
-                  for (int _i326 = 0; _i326 < _map325.size; ++_i326)
+                  org.apache.thrift.protocol.TMap _map333 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map333.size);
+                  for (int _i334 = 0; _i334 < _map333.size; ++_i334)
                   {
-                    ByteBuffer _key327; // required
-                    ByteBuffer _val328; // optional
-                    _key327 = iprot.readBinary();
-                    _val328 = iprot.readBinary();
-                    struct.attributes.put(_key327, _val328);
+                    ByteBuffer _key335; // required
+                    ByteBuffer _val336; // required
+                    _key335 = iprot.readBinary();
+                    _val336 = iprot.readBinary();
+                    struct.attributes.put(_key335, _val336);
                   }
                   iprot.readMapEnd();
                 }
@@ -28244,9 +28824,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRow_args str
           oprot.writeFieldBegin(MUTATIONS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size()));
-            for (Mutation _iter329 : struct.mutations)
+            for (Mutation _iter337 : struct.mutations)
             {
-              _iter329.write(oprot);
+              _iter337.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -28256,10 +28836,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRow_args str
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter330 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter338 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter330.getKey());
-              oprot.writeBinary(_iter330.getValue());
+              oprot.writeBinary(_iter338.getKey());
+              oprot.writeBinary(_iter338.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -28305,19 +28885,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, mutateRow_args stru
         if (struct.isSetMutations()) {
           {
             oprot.writeI32(struct.mutations.size());
-            for (Mutation _iter331 : struct.mutations)
+            for (Mutation _iter339 : struct.mutations)
             {
-              _iter331.write(oprot);
+              _iter339.write(oprot);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter332 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter340 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter332.getKey());
-              oprot.writeBinary(_iter332.getValue());
+              oprot.writeBinary(_iter340.getKey());
+              oprot.writeBinary(_iter340.getValue());
             }
           }
         }
@@ -28337,29 +28917,29 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_args struc
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list333 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.mutations = new ArrayList<Mutation>(_list333.size);
-            for (int _i334 = 0; _i334 < _list333.size; ++_i334)
+            org.apache.thrift.protocol.TList _list341 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.mutations = new ArrayList<Mutation>(_list341.size);
+            for (int _i342 = 0; _i342 < _list341.size; ++_i342)
             {
-              Mutation _elem335; // optional
-              _elem335 = new Mutation();
-              _elem335.read(iprot);
-              struct.mutations.add(_elem335);
+              Mutation _elem343; // required
+              _elem343 = new Mutation();
+              _elem343.read(iprot);
+              struct.mutations.add(_elem343);
             }
           }
           struct.setMutationsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map336 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map336.size);
-            for (int _i337 = 0; _i337 < _map336.size; ++_i337)
+            org.apache.thrift.protocol.TMap _map344 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map344.size);
+            for (int _i345 = 0; _i345 < _map344.size; ++_i345)
             {
-              ByteBuffer _key338; // required
-              ByteBuffer _val339; // optional
-              _key338 = iprot.readBinary();
-              _val339 = iprot.readBinary();
-              struct.attributes.put(_key338, _val339);
+              ByteBuffer _key346; // required
+              ByteBuffer _val347; // required
+              _key346 = iprot.readBinary();
+              _val347 = iprot.readBinary();
+              struct.attributes.put(_key346, _val347);
             }
           }
           struct.setAttributesIsSet(true);
@@ -28623,7 +29203,19 @@ public boolean equals(mutateRow_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRow_result other) {
@@ -29385,7 +29977,34 @@ public boolean equals(mutateRowTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_mutations = true && (isSetMutations());
+      builder.append(present_mutations);
+      if (present_mutations)
+        builder.append(mutations);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRowTs_args other) {
@@ -29519,8 +30138,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -29564,14 +30181,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowTs_args st
             case 3: // MUTATIONS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list340 = iprot.readListBegin();
-                  struct.mutations = new ArrayList<Mutation>(_list340.size);
-                  for (int _i341 = 0; _i341 < _list340.size; ++_i341)
+                  org.apache.thrift.protocol.TList _list348 = iprot.readListBegin();
+                  struct.mutations = new ArrayList<Mutation>(_list348.size);
+                  for (int _i349 = 0; _i349 < _list348.size; ++_i349)
                   {
-                    Mutation _elem342; // optional
-                    _elem342 = new Mutation();
-                    _elem342.read(iprot);
-                    struct.mutations.add(_elem342);
+                    Mutation _elem350; // required
+                    _elem350 = new Mutation();
+                    _elem350.read(iprot);
+                    struct.mutations.add(_elem350);
                   }
                   iprot.readListEnd();
                 }
@@ -29591,15 +30208,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowTs_args st
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map343 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map343.size);
-                  for (int _i344 = 0; _i344 < _map343.size; ++_i344)
+                  org.apache.thrift.protocol.TMap _map351 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map351.size);
+                  for (int _i352 = 0; _i352 < _map351.size; ++_i352)
                   {
-                    ByteBuffer _key345; // required
-                    ByteBuffer _val346; // optional
-                    _key345 = iprot.readBinary();
-                    _val346 = iprot.readBinary();
-                    struct.attributes.put(_key345, _val346);
+                    ByteBuffer _key353; // required
+                    ByteBuffer _val354; // required
+                    _key353 = iprot.readBinary();
+                    _val354 = iprot.readBinary();
+                    struct.attributes.put(_key353, _val354);
                   }
                   iprot.readMapEnd();
                 }
@@ -29637,9 +30254,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowTs_args s
           oprot.writeFieldBegin(MUTATIONS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size()));
-            for (Mutation _iter347 : struct.mutations)
+            for (Mutation _iter355 : struct.mutations)
             {
-              _iter347.write(oprot);
+              _iter355.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -29652,10 +30269,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowTs_args s
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter348 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter356 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter348.getKey());
-              oprot.writeBinary(_iter348.getValue());
+              oprot.writeBinary(_iter356.getKey());
+              oprot.writeBinary(_iter356.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -29704,9 +30321,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args st
         if (struct.isSetMutations()) {
           {
             oprot.writeI32(struct.mutations.size());
-            for (Mutation _iter349 : struct.mutations)
+            for (Mutation _iter357 : struct.mutations)
             {
-              _iter349.write(oprot);
+              _iter357.write(oprot);
             }
           }
         }
@@ -29716,10 +30333,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args st
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter350 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter358 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter350.getKey());
-              oprot.writeBinary(_iter350.getValue());
+              oprot.writeBinary(_iter358.getKey());
+              oprot.writeBinary(_iter358.getValue());
             }
           }
         }
@@ -29739,14 +30356,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args str
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list351 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.mutations = new ArrayList<Mutation>(_list351.size);
-            for (int _i352 = 0; _i352 < _list351.size; ++_i352)
+            org.apache.thrift.protocol.TList _list359 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.mutations = new ArrayList<Mutation>(_list359.size);
+            for (int _i360 = 0; _i360 < _list359.size; ++_i360)
             {
-              Mutation _elem353; // optional
-              _elem353 = new Mutation();
-              _elem353.read(iprot);
-              struct.mutations.add(_elem353);
+              Mutation _elem361; // required
+              _elem361 = new Mutation();
+              _elem361.read(iprot);
+              struct.mutations.add(_elem361);
             }
           }
           struct.setMutationsIsSet(true);
@@ -29757,15 +30374,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args str
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map354 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map354.size);
-            for (int _i355 = 0; _i355 < _map354.size; ++_i355)
+            org.apache.thrift.protocol.TMap _map362 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map362.size);
+            for (int _i363 = 0; _i363 < _map362.size; ++_i363)
             {
-              ByteBuffer _key356; // required
-              ByteBuffer _val357; // optional
-              _key356 = iprot.readBinary();
-              _val357 = iprot.readBinary();
-              struct.attributes.put(_key356, _val357);
+              ByteBuffer _key364; // required
+              ByteBuffer _val365; // required
+              _key364 = iprot.readBinary();
+              _val365 = iprot.readBinary();
+              struct.attributes.put(_key364, _val365);
             }
           }
           struct.setAttributesIsSet(true);
@@ -30029,7 +30646,19 @@ public boolean equals(mutateRowTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRowTs_result other) {
@@ -30636,7 +31265,24 @@ public boolean equals(mutateRows_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_rowBatches = true && (isSetRowBatches());
+      builder.append(present_rowBatches);
+      if (present_rowBatches)
+        builder.append(rowBatches);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRows_args other) {
@@ -30773,14 +31419,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRows_args str
             case 2: // ROW_BATCHES
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list358 = iprot.readListBegin();
-                  struct.rowBatches = new ArrayList<BatchMutation>(_list358.size);
-                  for (int _i359 = 0; _i359 < _list358.size; ++_i359)
+                  org.apache.thrift.protocol.TList _list366 = iprot.readListBegin();
+                  struct.rowBatches = new ArrayList<BatchMutation>(_list366.size);
+                  for (int _i367 = 0; _i367 < _list366.size; ++_i367)
                   {
-                    BatchMutation _elem360; // optional
-                    _elem360 = new BatchMutation();
-                    _elem360.read(iprot);
-                    struct.rowBatches.add(_elem360);
+                    BatchMutation _elem368; // required
+                    _elem368 = new BatchMutation();
+                    _elem368.read(iprot);
+                    struct.rowBatches.add(_elem368);
                   }
                   iprot.readListEnd();
                 }
@@ -30792,15 +31438,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRows_args str
             case 3: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map361 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map361.size);
-                  for (int _i362 = 0; _i362 < _map361.size; ++_i362)
+                  org.apache.thrift.protocol.TMap _map369 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map369.size);
+                  for (int _i370 = 0; _i370 < _map369.size; ++_i370)
                   {
-                    ByteBuffer _key363; // required
-                    ByteBuffer _val364; // optional
-                    _key363 = iprot.readBinary();
-                    _val364 = iprot.readBinary();
-                    struct.attributes.put(_key363, _val364);
+                    ByteBuffer _key371; // required
+                    ByteBuffer _val372; // required
+                    _key371 = iprot.readBinary();
+                    _val372 = iprot.readBinary();
+                    struct.attributes.put(_key371, _val372);
                   }
                   iprot.readMapEnd();
                 }
@@ -30833,9 +31479,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRows_args st
           oprot.writeFieldBegin(ROW_BATCHES_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rowBatches.size()));
-            for (BatchMutation _iter365 : struct.rowBatches)
+            for (BatchMutation _iter373 : struct.rowBatches)
             {
-              _iter365.write(oprot);
+              _iter373.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -30845,10 +31491,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRows_args st
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter366 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter374 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter366.getKey());
-              oprot.writeBinary(_iter366.getValue());
+              oprot.writeBinary(_iter374.getKey());
+              oprot.writeBinary(_iter374.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -30888,19 +31534,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, mutateRows_args str
         if (struct.isSetRowBatches()) {
           {
             oprot.writeI32(struct.rowBatches.size());
-            for (BatchMutation _iter367 : struct.rowBatches)
+            for (BatchMutation _iter375 : struct.rowBatches)
             {
-              _iter367.write(oprot);
+              _iter375.write(oprot);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter368 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter376 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter368.getKey());
-              oprot.writeBinary(_iter368.getValue());
+              oprot.writeBinary(_iter376.getKey());
+              oprot.writeBinary(_iter376.getValue());
             }
           }
         }
@@ -30916,29 +31562,29 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRows_args stru
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list369 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.rowBatches = new ArrayList<BatchMutation>(_list369.size);
-            for (int _i370 = 0; _i370 < _list369.size; ++_i370)
+            org.apache.thrift.protocol.TList _list377 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.rowBatches = new ArrayList<BatchMutation>(_list377.size);
+            for (int _i378 = 0; _i378 < _list377.size; ++_i378)
             {
-              BatchMutation _elem371; // optional
-              _elem371 = new BatchMutation();
-              _elem371.read(iprot);
-              struct.rowBatches.add(_elem371);
+              BatchMutation _elem379; // required
+              _elem379 = new BatchMutation();
+              _elem379.read(iprot);
+              struct.rowBatches.add(_elem379);
             }
           }
           struct.setRowBatchesIsSet(true);
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TMap _map372 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map372.size);
-            for (int _i373 = 0; _i373 < _map372.size; ++_i373)
+            org.apache.thrift.protocol.TMap _map380 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map380.size);
+            for (int _i381 = 0; _i381 < _map380.size; ++_i381)
             {
-              ByteBuffer _key374; // required
-              ByteBuffer _val375; // optional
-              _key374 = iprot.readBinary();
-              _val375 = iprot.readBinary();
-              struct.attributes.put(_key374, _val375);
+              ByteBuffer _key382; // required
+              ByteBuffer _val383; // required
+              _key382 = iprot.readBinary();
+              _val383 = iprot.readBinary();
+              struct.attributes.put(_key382, _val383);
             }
           }
           struct.setAttributesIsSet(true);
@@ -31202,7 +31848,19 @@ public boolean equals(mutateRows_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRows_result other) {
@@ -31883,7 +32541,29 @@ public boolean equals(mutateRowsTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_rowBatches = true && (isSetRowBatches());
+      builder.append(present_rowBatches);
+      if (present_rowBatches)
+        builder.append(rowBatches);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRowsTs_args other) {
@@ -32034,14 +32714,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowsTs_args s
             case 2: // ROW_BATCHES
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list376 = iprot.readListBegin();
-                  struct.rowBatches = new ArrayList<BatchMutation>(_list376.size);
-                  for (int _i377 = 0; _i377 < _list376.size; ++_i377)
+                  org.apache.thrift.protocol.TList _list384 = iprot.readListBegin();
+                  struct.rowBatches = new ArrayList<BatchMutation>(_list384.size);
+                  for (int _i385 = 0; _i385 < _list384.size; ++_i385)
                   {
-                    BatchMutation _elem378; // optional
-                    _elem378 = new BatchMutation();
-                    _elem378.read(iprot);
-                    struct.rowBatches.add(_elem378);
+                    BatchMutation _elem386; // required
+                    _elem386 = new BatchMutation();
+                    _elem386.read(iprot);
+                    struct.rowBatches.add(_elem386);
                   }
                   iprot.readListEnd();
                 }
@@ -32061,15 +32741,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowsTs_args s
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map379 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map379.size);
-                  for (int _i380 = 0; _i380 < _map379.size; ++_i380)
+                  org.apache.thrift.protocol.TMap _map387 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map387.size);
+                  for (int _i388 = 0; _i388 < _map387.size; ++_i388)
                   {
-                    ByteBuffer _key381; // required
-                    ByteBuffer _val382; // optional
-                    _key381 = iprot.readBinary();
-                    _val382 = iprot.readBinary();
-                    struct.attributes.put(_key381, _val382);
+                    ByteBuffer _key389; // required
+                    ByteBuffer _val390; // required
+                    _key389 = iprot.readBinary();
+                    _val390 = iprot.readBinary();
+                    struct.attributes.put(_key389, _val390);
                   }
                   iprot.readMapEnd();
                 }
@@ -32102,9 +32782,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowsTs_args
           oprot.writeFieldBegin(ROW_BATCHES_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rowBatches.size()));
-            for (BatchMutation _iter383 : struct.rowBatches)
+            for (BatchMutation _iter391 : struct.rowBatches)
             {
-              _iter383.write(oprot);
+              _iter391.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -32117,10 +32797,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowsTs_args
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter384 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter392 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter384.getKey());
-              oprot.writeBinary(_iter384.getValue());
+              oprot.writeBinary(_iter392.getKey());
+              oprot.writeBinary(_iter392.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -32163,9 +32843,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args s
         if (struct.isSetRowBatches()) {
           {
             oprot.writeI32(struct.rowBatches.size());
-            for (BatchMutation _iter385 : struct.rowBatches)
+            for (BatchMutation _iter393 : struct.rowBatches)
             {
-              _iter385.write(oprot);
+              _iter393.write(oprot);
             }
           }
         }
@@ -32175,10 +32855,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args s
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter386 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter394 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter386.getKey());
-              oprot.writeBinary(_iter386.getValue());
+              oprot.writeBinary(_iter394.getKey());
+              oprot.writeBinary(_iter394.getValue());
             }
           }
         }
@@ -32194,14 +32874,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args st
         }
         if (incoming.get(1)) {
           {
-            org.apache.thrift.protocol.TList _list387 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.rowBatches = new ArrayList<BatchMutation>(_list387.size);
-            for (int _i388 = 0; _i388 < _list387.size; ++_i388)
+            org.apache.thrift.protocol.TList _list395 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.rowBatches = new ArrayList<BatchMutation>(_list395.size);
+            for (int _i396 = 0; _i396 < _list395.size; ++_i396)
             {
-              BatchMutation _elem389; // optional
-              _elem389 = new BatchMutation();
-              _elem389.read(iprot);
-              struct.rowBatches.add(_elem389);
+              BatchMutation _elem397; // required
+              _elem397 = new BatchMutation();
+              _elem397.read(iprot);
+              struct.rowBatches.add(_elem397);
             }
           }
           struct.setRowBatchesIsSet(true);
@@ -32212,15 +32892,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args st
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map390 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map390.size);
-            for (int _i391 = 0; _i391 < _map390.size; ++_i391)
+            org.apache.thrift.protocol.TMap _map398 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map398.size);
+            for (int _i399 = 0; _i399 < _map398.size; ++_i399)
             {
-              ByteBuffer _key392; // required
-              ByteBuffer _val393; // optional
-              _key392 = iprot.readBinary();
-              _val393 = iprot.readBinary();
-              struct.attributes.put(_key392, _val393);
+              ByteBuffer _key400; // required
+              ByteBuffer _val401; // required
+              _key400 = iprot.readBinary();
+              _val401 = iprot.readBinary();
+              struct.attributes.put(_key400, _val401);
             }
           }
           struct.setAttributesIsSet(true);
@@ -32484,7 +33164,19 @@ public boolean equals(mutateRowsTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRowsTs_result other) {
@@ -33140,7 +33832,29 @@ public boolean equals(atomicIncrement_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_column = true && (isSetColumn());
+      builder.append(present_column);
+      if (present_column)
+        builder.append(column);
+
+      boolean present_value = true;
+      builder.append(present_value);
+      if (present_value)
+        builder.append(value);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(atomicIncrement_args other) {
@@ -33733,7 +34447,24 @@ public boolean equals(atomicIncrement_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(atomicIncrement_result other) {
@@ -33831,8 +34562,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -34448,7 +35177,29 @@ public boolean equals(deleteAll_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_column = true && (isSetColumn());
+      builder.append(present_column);
+      if (present_column)
+        builder.append(column);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAll_args other) {
@@ -34619,15 +35370,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAll_args stru
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map394 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map394.size);
-                  for (int _i395 = 0; _i395 < _map394.size; ++_i395)
+                  org.apache.thrift.protocol.TMap _map402 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map402.size);
+                  for (int _i403 = 0; _i403 < _map402.size; ++_i403)
                   {
-                    ByteBuffer _key396; // required
-                    ByteBuffer _val397; // optional
-                    _key396 = iprot.readBinary();
-                    _val397 = iprot.readBinary();
-                    struct.attributes.put(_key396, _val397);
+                    ByteBuffer _key404; // required
+                    ByteBuffer _val405; // required
+                    _key404 = iprot.readBinary();
+                    _val405 = iprot.readBinary();
+                    struct.attributes.put(_key404, _val405);
                   }
                   iprot.readMapEnd();
                 }
@@ -34670,10 +35421,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAll_args str
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter398 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter406 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter398.getKey());
-              oprot.writeBinary(_iter398.getValue());
+              oprot.writeBinary(_iter406.getKey());
+              oprot.writeBinary(_iter406.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -34722,10 +35473,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteAll_args stru
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter399 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter407 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter399.getKey());
-              oprot.writeBinary(_iter399.getValue());
+              oprot.writeBinary(_iter407.getKey());
+              oprot.writeBinary(_iter407.getValue());
             }
           }
         }
@@ -34749,15 +35500,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAll_args struc
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map400 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map400.size);
-            for (int _i401 = 0; _i401 < _map400.size; ++_i401)
+            org.apache.thrift.protocol.TMap _map408 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map408.size);
+            for (int _i409 = 0; _i409 < _map408.size; ++_i409)
             {
-              ByteBuffer _key402; // required
-              ByteBuffer _val403; // optional
-              _key402 = iprot.readBinary();
-              _val403 = iprot.readBinary();
-              struct.attributes.put(_key402, _val403);
+              ByteBuffer _key410; // required
+              ByteBuffer _val411; // required
+              _key410 = iprot.readBinary();
+              _val411 = iprot.readBinary();
+              struct.attributes.put(_key410, _val411);
             }
           }
           struct.setAttributesIsSet(true);
@@ -34962,7 +35713,14 @@ public boolean equals(deleteAll_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAll_result other) {
@@ -35671,7 +36429,34 @@ public boolean equals(deleteAllTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_column = true && (isSetColumn());
+      builder.append(present_column);
+      if (present_column)
+        builder.append(column);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAllTs_args other) {
@@ -35866,15 +36651,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllTs_args st
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map404 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map404.size);
-                  for (int _i405 = 0; _i405 < _map404.size; ++_i405)
+                  org.apache.thrift.protocol.TMap _map412 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map412.size);
+                  for (int _i413 = 0; _i413 < _map412.size; ++_i413)
                   {
-                    ByteBuffer _key406; // required
-                    ByteBuffer _val407; // optional
-                    _key406 = iprot.readBinary();
-                    _val407 = iprot.readBinary();
-                    struct.attributes.put(_key406, _val407);
+                    ByteBuffer _key414; // required
+                    ByteBuffer _val415; // required
+                    _key414 = iprot.readBinary();
+                    _val415 = iprot.readBinary();
+                    struct.attributes.put(_key414, _val415);
                   }
                   iprot.readMapEnd();
                 }
@@ -35920,10 +36705,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllTs_args s
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter408 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter416 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter408.getKey());
-              oprot.writeBinary(_iter408.getValue());
+              oprot.writeBinary(_iter416.getKey());
+              oprot.writeBinary(_iter416.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -35978,10 +36763,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_args st
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter409 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter417 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter409.getKey());
-              oprot.writeBinary(_iter409.getValue());
+              oprot.writeBinary(_iter417.getKey());
+              oprot.writeBinary(_iter417.getValue());
             }
           }
         }
@@ -36009,15 +36794,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_args str
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map410 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map410.size);
-            for (int _i411 = 0; _i411 < _map410.size; ++_i411)
+            org.apache.thrift.protocol.TMap _map418 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map418.size);
+            for (int _i419 = 0; _i419 < _map418.size; ++_i419)
             {
-              ByteBuffer _key412; // required
-              ByteBuffer _val413; // optional
-              _key412 = iprot.readBinary();
-              _val413 = iprot.readBinary();
-              struct.attributes.put(_key412, _val413);
+              ByteBuffer _key420; // required
+              ByteBuffer _val421; // required
+              _key420 = iprot.readBinary();
+              _val421 = iprot.readBinary();
+              struct.attributes.put(_key420, _val421);
             }
           }
           struct.setAttributesIsSet(true);
@@ -36222,7 +37007,14 @@ public boolean equals(deleteAllTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAllTs_result other) {
@@ -36776,7 +37568,24 @@ public boolean equals(deleteAllRow_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAllRow_args other) {
@@ -36921,15 +37730,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRow_args s
             case 3: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map414 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map414.size);
-                  for (int _i415 = 0; _i415 < _map414.size; ++_i415)
+                  org.apache.thrift.protocol.TMap _map422 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map422.size);
+                  for (int _i423 = 0; _i423 < _map422.size; ++_i423)
                   {
-                    ByteBuffer _key416; // required
-                    ByteBuffer _val417; // optional
-                    _key416 = iprot.readBinary();
-                    _val417 = iprot.readBinary();
-                    struct.attributes.put(_key416, _val417);
+                    ByteBuffer _key424; // required
+                    ByteBuffer _val425; // required
+                    _key424 = iprot.readBinary();
+                    _val425 = iprot.readBinary();
+                    struct.attributes.put(_key424, _val425);
                   }
                   iprot.readMapEnd();
                 }
@@ -36967,10 +37776,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRow_args
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter418 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter426 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter418.getKey());
-              oprot.writeBinary(_iter418.getValue());
+              oprot.writeBinary(_iter426.getKey());
+              oprot.writeBinary(_iter426.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -37013,10 +37822,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_args s
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter419 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter427 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter419.getKey());
-              oprot.writeBinary(_iter419.getValue());
+              oprot.writeBinary(_iter427.getKey());
+              oprot.writeBinary(_iter427.getValue());
             }
           }
         }
@@ -37036,15 +37845,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_args st
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TMap _map420 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map420.size);
-            for (int _i421 = 0; _i421 < _map420.size; ++_i421)
+            org.apache.thrift.protocol.TMap _map428 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map428.size);
+            for (int _i429 = 0; _i429 < _map428.size; ++_i429)
             {
-              ByteBuffer _key422; // required
-              ByteBuffer _val423; // optional
-              _key422 = iprot.readBinary();
-              _val423 = iprot.readBinary();
-              struct.attributes.put(_key422, _val423);
+              ByteBuffer _key430; // required
+              ByteBuffer _val431; // required
+              _key430 = iprot.readBinary();
+              _val431 = iprot.readBinary();
+              struct.attributes.put(_key430, _val431);
             }
           }
           struct.setAttributesIsSet(true);
@@ -37249,7 +38058,14 @@ public boolean equals(deleteAllRow_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAllRow_result other) {
@@ -37616,7 +38432,14 @@ public boolean equals(increment_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_increment = true && (isSetIncrement());
+      builder.append(present_increment);
+      if (present_increment)
+        builder.append(increment);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(increment_args other) {
@@ -37971,7 +38794,14 @@ public boolean equals(increment_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(increment_result other) {
@@ -38358,7 +39188,14 @@ public boolean equals(incrementRows_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_increments = true && (isSetIncrements());
+      builder.append(present_increments);
+      if (present_increments)
+        builder.append(increments);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(incrementRows_args other) {
@@ -38451,14 +39288,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, incrementRows_args
             case 1: // INCREMENTS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list424 = iprot.readListBegin();
-                  struct.increments = new ArrayList<TIncrement>(_list424.size);
-                  for (int _i425 = 0; _i425 < _list424.size; ++_i425)
+                  org.apache.thrift.protocol.TList _list432 = iprot.readListBegin();
+                  struct.increments = new ArrayList<TIncrement>(_list432.size);
+                  for (int _i433 = 0; _i433 < _list432.size; ++_i433)
                   {
-                    TIncrement _elem426; // optional
-                    _elem426 = new TIncrement();
-                    _elem426.read(iprot);
-                    struct.increments.add(_elem426);
+                    TIncrement _elem434; // required
+                    _elem434 = new TIncrement();
+                    _elem434.read(iprot);
+                    struct.increments.add(_elem434);
                   }
                   iprot.readListEnd();
                 }
@@ -38486,9 +39323,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, incrementRows_args
           oprot.writeFieldBegin(INCREMENTS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.increments.size()));
-            for (TIncrement _iter427 : struct.increments)
+            for (TIncrement _iter435 : struct.increments)
             {
-              _iter427.write(oprot);
+              _iter435.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -38519,9 +39356,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, incrementRows_args
         if (struct.isSetIncrements()) {
           {
             oprot.writeI32(struct.increments.size());
-            for (TIncrement _iter428 : struct.increments)
+            for (TIncrement _iter436 : struct.increments)
             {
-              _iter428.write(oprot);
+              _iter436.write(oprot);
             }
           }
         }
@@ -38533,14 +39370,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, incrementRows_args s
         BitSet incoming = iprot.readBitSet(1);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list429 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.increments = new ArrayList<TIncrement>(_list429.size);
-            for (int _i430 = 0; _i430 < _list429.size; ++_i430)
+            org.apache.thrift.protocol.TList _list437 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.increments = new ArrayList<TIncrement>(_list437.size);
+            for (int _i438 = 0; _i438 < _list437.size; ++_i438)
             {
-              TIncrement _elem431; // optional
-              _elem431 = new TIncrement();
-              _elem431.read(iprot);
-              struct.increments.add(_elem431);
+              TIncrement _elem439; // required
+              _elem439 = new TIncrement();
+              _elem439.read(iprot);
+              struct.increments.add(_elem439);
             }
           }
           struct.setIncrementsIsSet(true);
@@ -38745,7 +39582,14 @@ public boolean equals(incrementRows_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(incrementRows_result other) {
@@ -39373,7 +40217,29 @@ public boolean equals(deleteAllRowTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAllRowTs_args other) {
@@ -39542,15 +40408,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRowTs_args
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map432 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map432.size);
-                  for (int _i433 = 0; _i433 < _map432.size; ++_i433)
+                  org.apache.thrift.protocol.TMap _map440 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map440.size);
+                  for (int _i441 = 0; _i441 < _map440.size; ++_i441)
                   {
-                    ByteBuffer _key434; // required
-                    ByteBuffer _val435; // optional
-                    _key434 = iprot.readBinary();
-                    _val435 = iprot.readBinary();
-                    struct.attributes.put(_key434, _val435);
+                    ByteBuffer _key442; // required
+                    ByteBuffer _val443; // required
+                    _key442 = iprot.readBinary();
+                    _val443 = iprot.readBinary();
+                    struct.attributes.put(_key442, _val443);
                   }
                   iprot.readMapEnd();
                 }
@@ -39591,10 +40457,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRowTs_arg
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter436 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter444 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter436.getKey());
-              oprot.writeBinary(_iter436.getValue());
+              oprot.writeBinary(_iter444.getKey());
+              oprot.writeBinary(_iter444.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -39643,10 +40509,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_args
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter437 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter445 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter437.getKey());
-              oprot.writeBinary(_iter437.getValue());
+              oprot.writeBinary(_iter445.getKey());
+              oprot.writeBinary(_iter445.getValue());
             }
           }
         }
@@ -39670,15 +40536,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_args
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map438 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map438.size);
-            for (int _i439 = 0; _i439 < _map438.size; ++_i439)
+            org.apache.thrift.protocol.TMap _map446 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map446.size);
+            for (int _i447 = 0; _i447 < _map446.size; ++_i447)
             {
-              ByteBuffer _key440; // required
-              ByteBuffer _val441; // optional
-              _key440 = iprot.readBinary();
-              _val441 = iprot.readBinary();
-              struct.attributes.put(_key440, _val441);
+              ByteBuffer _key448; // required
+              ByteBuffer _val449; // required
+              _key448 = iprot.readBinary();
+              _val449 = iprot.readBinary();
+              struct.attributes.put(_key448, _val449);
             }
           }
           struct.setAttributesIsSet(true);
@@ -39883,7 +40749,14 @@ public boolean equals(deleteAllRowTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteAllRowTs_result other) {
@@ -40427,7 +41300,24 @@ public boolean equals(scannerOpenWithScan_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_scan = true && (isSetScan());
+      builder.append(present_scan);
+      if (present_scan)
+        builder.append(scan);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithScan_args other) {
@@ -40573,15 +41463,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithScan
             case 3: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map442 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map442.size);
-                  for (int _i443 = 0; _i443 < _map442.size; ++_i443)
+                  org.apache.thrift.protocol.TMap _map450 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map450.size);
+                  for (int _i451 = 0; _i451 < _map450.size; ++_i451)
                   {
-                    ByteBuffer _key444; // required
-                    ByteBuffer _val445; // optional
-                    _key444 = iprot.readBinary();
-                    _val445 = iprot.readBinary();
-                    struct.attributes.put(_key444, _val445);
+                    ByteBuffer _key452; // required
+                    ByteBuffer _val453; // required
+                    _key452 = iprot.readBinary();
+                    _val453 = iprot.readBinary();
+                    struct.attributes.put(_key452, _val453);
                   }
                   iprot.readMapEnd();
                 }
@@ -40619,10 +41509,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSca
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter446 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter454 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter446.getKey());
-              oprot.writeBinary(_iter446.getValue());
+              oprot.writeBinary(_iter454.getKey());
+              oprot.writeBinary(_iter454.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -40665,10 +41555,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter447 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter455 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter447.getKey());
-              oprot.writeBinary(_iter447.getValue());
+              oprot.writeBinary(_iter455.getKey());
+              oprot.writeBinary(_iter455.getValue());
             }
           }
         }
@@ -40689,15 +41579,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan_
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TMap _map448 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map448.size);
-            for (int _i449 = 0; _i449 < _map448.size; ++_i449)
+            org.apache.thrift.protocol.TMap _map456 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map456.size);
+            for (int _i457 = 0; _i457 < _map456.size; ++_i457)
             {
-              ByteBuffer _key450; // required
-              ByteBuffer _val451; // optional
-              _key450 = iprot.readBinary();
-              _val451 = iprot.readBinary();
-              struct.attributes.put(_key450, _val451);
+              ByteBuffer _key458; // required
+              ByteBuffer _val459; // required
+              _key458 = iprot.readBinary();
+              _val459 = iprot.readBinary();
+              struct.attributes.put(_key458, _val459);
             }
           }
           struct.setAttributesIsSet(true);
@@ -40964,7 +41854,19 @@ public boolean equals(scannerOpenWithScan_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithScan_result other) {
@@ -41044,8 +41946,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -41658,7 +42558,29 @@ public boolean equals(scannerOpen_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_startRow = true && (isSetStartRow());
+      builder.append(present_startRow);
+      if (present_startRow)
+        builder.append(startRow);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpen_args other) {
@@ -41821,13 +42743,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpen_args st
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list452 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list452.size);
-                  for (int _i453 = 0; _i453 < _list452.size; ++_i453)
+                  org.apache.thrift.protocol.TList _list460 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list460.size);
+                  for (int _i461 = 0; _i461 < _list460.size; ++_i461)
                   {
-                    ByteBuffer _elem454; // optional
-                    _elem454 = iprot.readBinary();
-                    struct.columns.add(_elem454);
+                    ByteBuffer _elem462; // required
+                    _elem462 = iprot.readBinary();
+                    struct.columns.add(_elem462);
                   }
                   iprot.readListEnd();
                 }
@@ -41839,15 +42761,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpen_args st
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map455 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map455.size);
-                  for (int _i456 = 0; _i456 < _map455.size; ++_i456)
+                  org.apache.thrift.protocol.TMap _map463 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map463.size);
+                  for (int _i464 = 0; _i464 < _map463.size; ++_i464)
                   {
-                    ByteBuffer _key457; // required
-                    ByteBuffer _val458; // optional
-                    _key457 = iprot.readBinary();
-                    _val458 = iprot.readBinary();
-                    struct.attributes.put(_key457, _val458);
+                    ByteBuffer _key465; // required
+                    ByteBuffer _val466; // required
+                    _key465 = iprot.readBinary();
+                    _val466 = iprot.readBinary();
+                    struct.attributes.put(_key465, _val466);
                   }
                   iprot.readMapEnd();
                 }
@@ -41885,9 +42807,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpen_args s
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter459 : struct.columns)
+            for (ByteBuffer _iter467 : struct.columns)
             {
-              oprot.writeBinary(_iter459);
+              oprot.writeBinary(_iter467);
             }
             oprot.writeListEnd();
           }
@@ -41897,10 +42819,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpen_args s
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter460 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter468 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter460.getKey());
-              oprot.writeBinary(_iter460.getValue());
+              oprot.writeBinary(_iter468.getKey());
+              oprot.writeBinary(_iter468.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -41946,19 +42868,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpen_args st
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter461 : struct.columns)
+            for (ByteBuffer _iter469 : struct.columns)
             {
-              oprot.writeBinary(_iter461);
+              oprot.writeBinary(_iter469);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter462 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter470 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter462.getKey());
-              oprot.writeBinary(_iter462.getValue());
+              oprot.writeBinary(_iter470.getKey());
+              oprot.writeBinary(_iter470.getValue());
             }
           }
         }
@@ -41978,28 +42900,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpen_args str
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list463 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list463.size);
-            for (int _i464 = 0; _i464 < _list463.size; ++_i464)
+            org.apache.thrift.protocol.TList _list471 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list471.size);
+            for (int _i472 = 0; _i472 < _list471.size; ++_i472)
             {
-              ByteBuffer _elem465; // optional
-              _elem465 = iprot.readBinary();
-              struct.columns.add(_elem465);
+              ByteBuffer _elem473; // required
+              _elem473 = iprot.readBinary();
+              struct.columns.add(_elem473);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map466 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map466.size);
-            for (int _i467 = 0; _i467 < _map466.size; ++_i467)
+            org.apache.thrift.protocol.TMap _map474 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map474.size);
+            for (int _i475 = 0; _i475 < _map474.size; ++_i475)
             {
-              ByteBuffer _key468; // required
-              ByteBuffer _val469; // optional
-              _key468 = iprot.readBinary();
-              _val469 = iprot.readBinary();
-              struct.attributes.put(_key468, _val469);
+              ByteBuffer _key476; // required
+              ByteBuffer _val477; // required
+              _key476 = iprot.readBinary();
+              _val477 = iprot.readBinary();
+              struct.attributes.put(_key476, _val477);
             }
           }
           struct.setAttributesIsSet(true);
@@ -42266,7 +43188,19 @@ public boolean equals(scannerOpen_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpen_result other) {
@@ -42346,8 +43280,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -43045,7 +43977,34 @@ public boolean equals(scannerOpenWithStop_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_startRow = true && (isSetStartRow());
+      builder.append(present_startRow);
+      if (present_startRow)
+        builder.append(startRow);
+
+      boolean present_stopRow = true && (isSetStopRow());
+      builder.append(present_stopRow);
+      if (present_stopRow)
+        builder.append(stopRow);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithStop_args other) {
@@ -43234,13 +44193,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 4: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list470 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list470.size);
-                  for (int _i471 = 0; _i471 < _list470.size; ++_i471)
+                  org.apache.thrift.protocol.TList _list478 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list478.size);
+                  for (int _i479 = 0; _i479 < _list478.size; ++_i479)
                   {
-                    ByteBuffer _elem472; // optional
-                    _elem472 = iprot.readBinary();
-                    struct.columns.add(_elem472);
+                    ByteBuffer _elem480; // required
+                    _elem480 = iprot.readBinary();
+                    struct.columns.add(_elem480);
                   }
                   iprot.readListEnd();
                 }
@@ -43252,15 +44211,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map473 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map473.size);
-                  for (int _i474 = 0; _i474 < _map473.size; ++_i474)
+                  org.apache.thrift.protocol.TMap _map481 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map481.size);
+                  for (int _i482 = 0; _i482 < _map481.size; ++_i482)
                   {
-                    ByteBuffer _key475; // required
-                    ByteBuffer _val476; // optional
-                    _key475 = iprot.readBinary();
-                    _val476 = iprot.readBinary();
-                    struct.attributes.put(_key475, _val476);
+                    ByteBuffer _key483; // required
+                    ByteBuffer _val484; // required
+                    _key483 = iprot.readBinary();
+                    _val484 = iprot.readBinary();
+                    struct.attributes.put(_key483, _val484);
                   }
                   iprot.readMapEnd();
                 }
@@ -43303,9 +44262,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter477 : struct.columns)
+            for (ByteBuffer _iter485 : struct.columns)
             {
-              oprot.writeBinary(_iter477);
+              oprot.writeBinary(_iter485);
             }
             oprot.writeListEnd();
           }
@@ -43315,10 +44274,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter478 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter486 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter478.getKey());
-              oprot.writeBinary(_iter478.getValue());
+              oprot.writeBinary(_iter486.getKey());
+              oprot.writeBinary(_iter486.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -43370,19 +44329,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter479 : struct.columns)
+            for (ByteBuffer _iter487 : struct.columns)
             {
-              oprot.writeBinary(_iter479);
+              oprot.writeBinary(_iter487);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter480 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter488 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter480.getKey());
-              oprot.writeBinary(_iter480.getValue());
+              oprot.writeBinary(_iter488.getKey());
+              oprot.writeBinary(_iter488.getValue());
             }
           }
         }
@@ -43406,28 +44365,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop_
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TList _list481 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list481.size);
-            for (int _i482 = 0; _i482 < _list481.size; ++_i482)
+            org.apache.thrift.protocol.TList _list489 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list489.size);
+            for (int _i490 = 0; _i490 < _list489.size; ++_i490)
             {
-              ByteBuffer _elem483; // optional
-              _elem483 = iprot.readBinary();
-              struct.columns.add(_elem483);
+              ByteBuffer _elem491; // required
+              _elem491 = iprot.readBinary();
+              struct.columns.add(_elem491);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map484 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map484.size);
-            for (int _i485 = 0; _i485 < _map484.size; ++_i485)
+            org.apache.thrift.protocol.TMap _map492 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map492.size);
+            for (int _i493 = 0; _i493 < _map492.size; ++_i493)
             {
-              ByteBuffer _key486; // required
-              ByteBuffer _val487; // optional
-              _key486 = iprot.readBinary();
-              _val487 = iprot.readBinary();
-              struct.attributes.put(_key486, _val487);
+              ByteBuffer _key494; // required
+              ByteBuffer _val495; // required
+              _key494 = iprot.readBinary();
+              _val495 = iprot.readBinary();
+              struct.attributes.put(_key494, _val495);
             }
           }
           struct.setAttributesIsSet(true);
@@ -43694,7 +44653,19 @@ public boolean equals(scannerOpenWithStop_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithStop_result other) {
@@ -43774,8 +44745,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -44376,7 +45345,29 @@ public boolean equals(scannerOpenWithPrefix_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_startAndPrefix = true && (isSetStartAndPrefix());
+      builder.append(present_startAndPrefix);
+      if (present_startAndPrefix)
+        builder.append(startAndPrefix);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithPrefix_args other) {
@@ -44539,13 +45530,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithPref
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list488 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list488.size);
-                  for (int _i489 = 0; _i489 < _list488.size; ++_i489)
+                  org.apache.thrift.protocol.TList _list496 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list496.size);
+                  for (int _i497 = 0; _i497 < _list496.size; ++_i497)
                   {
-                    ByteBuffer _elem490; // optional
-                    _elem490 = iprot.readBinary();
-                    struct.columns.add(_elem490);
+                    ByteBuffer _elem498; // required
+                    _elem498 = iprot.readBinary();
+                    struct.columns.add(_elem498);
                   }
                   iprot.readListEnd();
                 }
@@ -44557,15 +45548,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithPref
             case 4: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map491 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map491.size);
-                  for (int _i492 = 0; _i492 < _map491.size; ++_i492)
+                  org.apache.thrift.protocol.TMap _map499 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map499.size);
+                  for (int _i500 = 0; _i500 < _map499.size; ++_i500)
                   {
-                    ByteBuffer _key493; // required
-                    ByteBuffer _val494; // optional
-                    _key493 = iprot.readBinary();
-                    _val494 = iprot.readBinary();
-                    struct.attributes.put(_key493, _val494);
+                    ByteBuffer _key501; // required
+                    ByteBuffer _val502; // required
+                    _key501 = iprot.readBinary();
+                    _val502 = iprot.readBinary();
+                    struct.attributes.put(_key501, _val502);
                   }
                   iprot.readMapEnd();
                 }
@@ -44603,9 +45594,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithPre
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter495 : struct.columns)
+            for (ByteBuffer _iter503 : struct.columns)
             {
-              oprot.writeBinary(_iter495);
+              oprot.writeBinary(_iter503);
             }
             oprot.writeListEnd();
           }
@@ -44615,10 +45606,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithPre
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter496 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter504 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter496.getKey());
-              oprot.writeBinary(_iter496.getValue());
+              oprot.writeBinary(_iter504.getKey());
+              oprot.writeBinary(_iter504.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -44664,19 +45655,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPref
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter497 : struct.columns)
+            for (ByteBuffer _iter505 : struct.columns)
             {
-              oprot.writeBinary(_iter497);
+              oprot.writeBinary(_iter505);
             }
           }
         }
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter498 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter506 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter498.getKey());
-              oprot.writeBinary(_iter498.getValue());
+              oprot.writeBinary(_iter506.getKey());
+              oprot.writeBinary(_iter506.getValue());
             }
           }
         }
@@ -44696,28 +45687,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPrefi
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list499 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list499.size);
-            for (int _i500 = 0; _i500 < _list499.size; ++_i500)
+            org.apache.thrift.protocol.TList _list507 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list507.size);
+            for (int _i508 = 0; _i508 < _list507.size; ++_i508)
             {
-              ByteBuffer _elem501; // optional
-              _elem501 = iprot.readBinary();
-              struct.columns.add(_elem501);
+              ByteBuffer _elem509; // required
+              _elem509 = iprot.readBinary();
+              struct.columns.add(_elem509);
             }
           }
           struct.setColumnsIsSet(true);
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TMap _map502 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map502.size);
-            for (int _i503 = 0; _i503 < _map502.size; ++_i503)
+            org.apache.thrift.protocol.TMap _map510 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map510.size);
+            for (int _i511 = 0; _i511 < _map510.size; ++_i511)
             {
-              ByteBuffer _key504; // required
-              ByteBuffer _val505; // optional
-              _key504 = iprot.readBinary();
-              _val505 = iprot.readBinary();
-              struct.attributes.put(_key504, _val505);
+              ByteBuffer _key512; // required
+              ByteBuffer _val513; // required
+              _key512 = iprot.readBinary();
+              _val513 = iprot.readBinary();
+              struct.attributes.put(_key512, _val513);
             }
           }
           struct.setAttributesIsSet(true);
@@ -44984,7 +45975,19 @@ public boolean equals(scannerOpenWithPrefix_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithPrefix_result other) {
@@ -45064,8 +46067,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -45752,7 +46753,34 @@ public boolean equals(scannerOpenTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_startRow = true && (isSetStartRow());
+      builder.append(present_startRow);
+      if (present_startRow)
+        builder.append(startRow);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenTs_args other) {
@@ -45931,13 +46959,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenTs_args
             case 3: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list506 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list506.size);
-                  for (int _i507 = 0; _i507 < _list506.size; ++_i507)
+                  org.apache.thrift.protocol.TList _list514 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list514.size);
+                  for (int _i515 = 0; _i515 < _list514.size; ++_i515)
                   {
-                    ByteBuffer _elem508; // optional
-                    _elem508 = iprot.readBinary();
-                    struct.columns.add(_elem508);
+                    ByteBuffer _elem516; // required
+                    _elem516 = iprot.readBinary();
+                    struct.columns.add(_elem516);
                   }
                   iprot.readListEnd();
                 }
@@ -45957,15 +46985,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenTs_args
             case 5: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map509 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map509.size);
-                  for (int _i510 = 0; _i510 < _map509.size; ++_i510)
+                  org.apache.thrift.protocol.TMap _map517 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map517.size);
+                  for (int _i518 = 0; _i518 < _map517.size; ++_i518)
                   {
-                    ByteBuffer _key511; // required
-                    ByteBuffer _val512; // optional
-                    _key511 = iprot.readBinary();
-                    _val512 = iprot.readBinary();
-                    struct.attributes.put(_key511, _val512);
+                    ByteBuffer _key519; // required
+                    ByteBuffer _val520; // required
+                    _key519 = iprot.readBinary();
+                    _val520 = iprot.readBinary();
+                    struct.attributes.put(_key519, _val520);
                   }
                   iprot.readMapEnd();
                 }
@@ -46003,9 +47031,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenTs_args
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter513 : struct.columns)
+            for (ByteBuffer _iter521 : struct.columns)
             {
-              oprot.writeBinary(_iter513);
+              oprot.writeBinary(_iter521);
             }
             oprot.writeListEnd();
           }
@@ -46018,10 +47046,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenTs_args
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter514 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter522 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter514.getKey());
-              oprot.writeBinary(_iter514.getValue());
+              oprot.writeBinary(_iter522.getKey());
+              oprot.writeBinary(_iter522.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -46070,9 +47098,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter515 : struct.columns)
+            for (ByteBuffer _iter523 : struct.columns)
             {
-              oprot.writeBinary(_iter515);
+              oprot.writeBinary(_iter523);
             }
           }
         }
@@ -46082,10 +47110,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter516 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter524 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter516.getKey());
-              oprot.writeBinary(_iter516.getValue());
+              oprot.writeBinary(_iter524.getKey());
+              oprot.writeBinary(_iter524.getValue());
             }
           }
         }
@@ -46105,13 +47133,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args s
         }
         if (incoming.get(2)) {
           {
-            org.apache.thrift.protocol.TList _list517 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list517.size);
-            for (int _i518 = 0; _i518 < _list517.size; ++_i518)
+            org.apache.thrift.protocol.TList _list525 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list525.size);
+            for (int _i526 = 0; _i526 < _list525.size; ++_i526)
             {
-              ByteBuffer _elem519; // optional
-              _elem519 = iprot.readBinary();
-              struct.columns.add(_elem519);
+              ByteBuffer _elem527; // required
+              _elem527 = iprot.readBinary();
+              struct.columns.add(_elem527);
             }
           }
           struct.setColumnsIsSet(true);
@@ -46122,15 +47150,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args s
         }
         if (incoming.get(4)) {
           {
-            org.apache.thrift.protocol.TMap _map520 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map520.size);
-            for (int _i521 = 0; _i521 < _map520.size; ++_i521)
+            org.apache.thrift.protocol.TMap _map528 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map528.size);
+            for (int _i529 = 0; _i529 < _map528.size; ++_i529)
             {
-              ByteBuffer _key522; // required
-              ByteBuffer _val523; // optional
-              _key522 = iprot.readBinary();
-              _val523 = iprot.readBinary();
-              struct.attributes.put(_key522, _val523);
+              ByteBuffer _key530; // required
+              ByteBuffer _val531; // required
+              _key530 = iprot.readBinary();
+              _val531 = iprot.readBinary();
+              struct.attributes.put(_key530, _val531);
             }
           }
           struct.setAttributesIsSet(true);
@@ -46397,7 +47425,19 @@ public boolean equals(scannerOpenTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenTs_result other) {
@@ -46477,8 +47517,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -47250,7 +48288,39 @@ public boolean equals(scannerOpenWithStopTs_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_startRow = true && (isSetStartRow());
+      builder.append(present_startRow);
+      if (present_startRow)
+        builder.append(startRow);
+
+      boolean present_stopRow = true && (isSetStopRow());
+      builder.append(present_stopRow);
+      if (present_stopRow)
+        builder.append(stopRow);
+
+      boolean present_columns = true && (isSetColumns());
+      builder.append(present_columns);
+      if (present_columns)
+        builder.append(columns);
+
+      boolean present_timestamp = true;
+      builder.append(present_timestamp);
+      if (present_timestamp)
+        builder.append(timestamp);
+
+      boolean present_attributes = true && (isSetAttributes());
+      builder.append(present_attributes);
+      if (present_attributes)
+        builder.append(attributes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithStopTs_args other) {
@@ -47453,13 +48523,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 4: // COLUMNS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list524 = iprot.readListBegin();
-                  struct.columns = new ArrayList<ByteBuffer>(_list524.size);
-                  for (int _i525 = 0; _i525 < _list524.size; ++_i525)
+                  org.apache.thrift.protocol.TList _list532 = iprot.readListBegin();
+                  struct.columns = new ArrayList<ByteBuffer>(_list532.size);
+                  for (int _i533 = 0; _i533 < _list532.size; ++_i533)
                   {
-                    ByteBuffer _elem526; // optional
-                    _elem526 = iprot.readBinary();
-                    struct.columns.add(_elem526);
+                    ByteBuffer _elem534; // required
+                    _elem534 = iprot.readBinary();
+                    struct.columns.add(_elem534);
                   }
                   iprot.readListEnd();
                 }
@@ -47479,15 +48549,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop
             case 6: // ATTRIBUTES
               if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
                 {
-                  org.apache.thrift.protocol.TMap _map527 = iprot.readMapBegin();
-                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map527.size);
-                  for (int _i528 = 0; _i528 < _map527.size; ++_i528)
+                  org.apache.thrift.protocol.TMap _map535 = iprot.readMapBegin();
+                  struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map535.size);
+                  for (int _i536 = 0; _i536 < _map535.size; ++_i536)
                   {
-                    ByteBuffer _key529; // required
-                    ByteBuffer _val530; // optional
-                    _key529 = iprot.readBinary();
-                    _val530 = iprot.readBinary();
-                    struct.attributes.put(_key529, _val530);
+                    ByteBuffer _key537; // required
+                    ByteBuffer _val538; // required
+                    _key537 = iprot.readBinary();
+                    _val538 = iprot.readBinary();
+                    struct.attributes.put(_key537, _val538);
                   }
                   iprot.readMapEnd();
                 }
@@ -47530,9 +48600,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size()));
-            for (ByteBuffer _iter531 : struct.columns)
+            for (ByteBuffer _iter539 : struct.columns)
             {
-              oprot.writeBinary(_iter531);
+              oprot.writeBinary(_iter539);
             }
             oprot.writeListEnd();
           }
@@ -47545,10 +48615,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithSto
           oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC);
           {
             oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size()));
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter532 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter540 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter532.getKey());
-              oprot.writeBinary(_iter532.getValue());
+              oprot.writeBinary(_iter540.getKey());
+              oprot.writeBinary(_iter540.getValue());
             }
             oprot.writeMapEnd();
           }
@@ -47603,9 +48673,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop
         if (struct.isSetColumns()) {
           {
             oprot.writeI32(struct.columns.size());
-            for (ByteBuffer _iter533 : struct.columns)
+            for (ByteBuffer _iter541 : struct.columns)
             {
-              oprot.writeBinary(_iter533);
+              oprot.writeBinary(_iter541);
             }
           }
         }
@@ -47615,10 +48685,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop
         if (struct.isSetAttributes()) {
           {
             oprot.writeI32(struct.attributes.size());
-            for (Map.Entry<ByteBuffer, ByteBuffer> _iter534 : struct.attributes.entrySet())
+            for (Map.Entry<ByteBuffer, ByteBuffer> _iter542 : struct.attributes.entrySet())
             {
-              oprot.writeBinary(_iter534.getKey());
-              oprot.writeBinary(_iter534.getValue());
+              oprot.writeBinary(_iter542.getKey());
+              oprot.writeBinary(_iter542.getValue());
             }
           }
         }
@@ -47642,13 +48712,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopT
         }
         if (incoming.get(3)) {
           {
-            org.apache.thrift.protocol.TList _list535 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.columns = new ArrayList<ByteBuffer>(_list535.size);
-            for (int _i536 = 0; _i536 < _list535.size; ++_i536)
+            org.apache.thrift.protocol.TList _list543 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.columns = new ArrayList<ByteBuffer>(_list543.size);
+            for (int _i544 = 0; _i544 < _list543.size; ++_i544)
             {
-              ByteBuffer _elem537; // optional
-              _elem537 = iprot.readBinary();
-              struct.columns.add(_elem537);
+              ByteBuffer _elem545; // required
+              _elem545 = iprot.readBinary();
+              struct.columns.add(_elem545);
             }
           }
           struct.setColumnsIsSet(true);
@@ -47659,15 +48729,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopT
         }
         if (incoming.get(5)) {
           {
-            org.apache.thrift.protocol.TMap _map538 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map538.size);
-            for (int _i539 = 0; _i539 < _map538.size; ++_i539)
+            org.apache.thrift.protocol.TMap _map546 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+            struct.attributes = new HashMap<ByteBuffer,ByteBuffer>(2*_map546.size);
+            for (int _i547 = 0; _i547 < _map546.size; ++_i547)
             {
-              ByteBuffer _key540; // required
-              ByteBuffer _val541; // optional
-              _key540 = iprot.readBinary();
-              _val541 = iprot.readBinary();
-              struct.attributes.put(_key540, _val541);
+              ByteBuffer _key548; // required
+              ByteBuffer _val549; // required
+              _key548 = iprot.readBinary();
+              _val549 = iprot.readBinary();
+              struct.attributes.put(_key548, _val549);
             }
           }
           struct.setAttributesIsSet(true);
@@ -47934,7 +49004,19 @@ public boolean equals(scannerOpenWithStopTs_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerOpenWithStopTs_result other) {
@@ -48014,8 +49096,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -48341,7 +49421,14 @@ public boolean equals(scannerGet_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_id = true;
+      builder.append(present_id);
+      if (present_id)
+        builder.append(id);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerGet_args other) {
@@ -48828,7 +49915,24 @@ public boolean equals(scannerGet_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerGet_result other) {
@@ -48957,14 +50061,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGet_result s
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list542 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list542.size);
-                  for (int _i543 = 0; _i543 < _list542.size; ++_i543)
+                  org.apache.thrift.protocol.TList _list550 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list550.size);
+                  for (int _i551 = 0; _i551 < _list550.size; ++_i551)
                   {
-                    TRowResult _elem544; // optional
-                    _elem544 = new TRowResult();
-                    _elem544.read(iprot);
-                    struct.success.add(_elem544);
+                    TRowResult _elem552; // required
+                    _elem552 = new TRowResult();
+                    _elem552.read(iprot);
+                    struct.success.add(_elem552);
                   }
                   iprot.readListEnd();
                 }
@@ -49010,9 +50114,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGet_result
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter545 : struct.success)
+            for (TRowResult _iter553 : struct.success)
             {
-              _iter545.write(oprot);
+              _iter553.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -49059,9 +50163,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerGet_result s
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter546 : struct.success)
+            for (TRowResult _iter554 : struct.success)
             {
-              _iter546.write(oprot);
+              _iter554.write(oprot);
             }
           }
         }
@@ -49079,14 +50183,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerGet_result st
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list547 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list547.size);
-            for (int _i548 = 0; _i548 < _list547.size; ++_i548)
+            org.apache.thrift.protocol.TList _list555 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list555.size);
+            for (int _i556 = 0; _i556 < _list555.size; ++_i556)
             {
-              TRowResult _elem549; // optional
-              _elem549 = new TRowResult();
-              _elem549.read(iprot);
-              struct.success.add(_elem549);
+              TRowResult _elem557; // required
+              _elem557 = new TRowResult();
+              _elem557.read(iprot);
+              struct.success.add(_elem557);
             }
           }
           struct.setSuccessIsSet(true);
@@ -49387,7 +50491,19 @@ public boolean equals(scannerGetList_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_id = true;
+      builder.append(present_id);
+      if (present_id)
+        builder.append(id);
+
+      boolean present_nbRows = true;
+      builder.append(present_nbRows);
+      if (present_nbRows)
+        builder.append(nbRows);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerGetList_args other) {
@@ -49463,6 +50579,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -49907,7 +51025,24 @@ public boolean equals(scannerGetList_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerGetList_result other) {
@@ -50036,14 +51171,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGetList_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list550 = iprot.readListBegin();
-                  struct.success = new ArrayList<TRowResult>(_list550.size);
-                  for (int _i551 = 0; _i551 < _list550.size; ++_i551)
+                  org.apache.thrift.protocol.TList _list558 = iprot.readListBegin();
+                  struct.success = new ArrayList<TRowResult>(_list558.size);
+                  for (int _i559 = 0; _i559 < _list558.size; ++_i559)
                   {
-                    TRowResult _elem552; // optional
-                    _elem552 = new TRowResult();
-                    _elem552.read(iprot);
-                    struct.success.add(_elem552);
+                    TRowResult _elem560; // required
+                    _elem560 = new TRowResult();
+                    _elem560.read(iprot);
+                    struct.success.add(_elem560);
                   }
                   iprot.readListEnd();
                 }
@@ -50089,9 +51224,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGetList_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TRowResult _iter553 : struct.success)
+            for (TRowResult _iter561 : struct.success)
             {
-              _iter553.write(oprot);
+              _iter561.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -50138,9 +51273,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, scannerGetList_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TRowResult _iter554 : struct.success)
+            for (TRowResult _iter562 : struct.success)
             {
-              _iter554.write(oprot);
+              _iter562.write(oprot);
             }
           }
         }
@@ -50158,14 +51293,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, scannerGetList_resul
         BitSet incoming = iprot.readBitSet(3);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list555 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TRowResult>(_list555.size);
-            for (int _i556 = 0; _i556 < _list555.size; ++_i556)
+            org.apache.thrift.protocol.TList _list563 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TRowResult>(_list563.size);
+            for (int _i564 = 0; _i564 < _list563.size; ++_i564)
             {
-              TRowResult _elem557; // optional
-              _elem557 = new TRowResult();
-              _elem557.read(iprot);
-              struct.success.add(_elem557);
+              TRowResult _elem565; // required
+              _elem565 = new TRowResult();
+              _elem565.read(iprot);
+              struct.success.add(_elem565);
             }
           }
           struct.setSuccessIsSet(true);
@@ -50395,7 +51530,14 @@ public boolean equals(scannerClose_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_id = true;
+      builder.append(present_id);
+      if (present_id)
+        builder.append(id);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerClose_args other) {
@@ -50803,7 +51945,19 @@ public boolean equals(scannerClose_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(scannerClose_result other) {
@@ -51385,7 +52539,24 @@ public boolean equals(getRowOrBefore_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_tableName = true && (isSetTableName());
+      builder.append(present_tableName);
+      if (present_tableName)
+        builder.append(tableName);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_family = true && (isSetFamily());
+      builder.append(present_family);
+      if (present_family)
+        builder.append(family);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowOrBefore_args other) {
@@ -51899,7 +53070,19 @@ public boolean equals(getRowOrBefore_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRowOrBefore_result other) {
@@ -52010,14 +53193,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRowOrBefore_resu
             case 0: // SUCCESS
               if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
                 {
-                  org.apache.thrift.protocol.TList _list558 = iprot.readListBegin();
-                  struct.success = new ArrayList<TCell>(_list558.size);
-                  for (int _i559 = 0; _i559 < _list558.size; ++_i559)
+                  org.apache.thrift.protocol.TList _list566 = iprot.readListBegin();
+                  struct.success = new ArrayList<TCell>(_list566.size);
+                  for (int _i567 = 0; _i567 < _list566.size; ++_i567)
                   {
-                    TCell _elem560; // optional
-                    _elem560 = new TCell();
-                    _elem560.read(iprot);
-                    struct.success.add(_elem560);
+                    TCell _elem568; // required
+                    _elem568 = new TCell();
+                    _elem568.read(iprot);
+                    struct.success.add(_elem568);
                   }
                   iprot.readListEnd();
                 }
@@ -52054,9 +53237,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRowOrBefore_res
           oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
           {
             oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
-            for (TCell _iter561 : struct.success)
+            for (TCell _iter569 : struct.success)
             {
-              _iter561.write(oprot);
+              _iter569.write(oprot);
             }
             oprot.writeListEnd();
           }
@@ -52095,9 +53278,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_resu
         if (struct.isSetSuccess()) {
           {
             oprot.writeI32(struct.success.size());
-            for (TCell _iter562 : struct.success)
+            for (TCell _iter570 : struct.success)
             {
-              _iter562.write(oprot);
+              _iter570.write(oprot);
             }
           }
         }
@@ -52112,14 +53295,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_resul
         BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           {
-            org.apache.thrift.protocol.TList _list563 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-            struct.success = new ArrayList<TCell>(_list563.size);
-            for (int _i564 = 0; _i564 < _list563.size; ++_i564)
+            org.apache.thrift.protocol.TList _list571 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+            struct.success = new ArrayList<TCell>(_list571.size);
+            for (int _i572 = 0; _i572 < _list571.size; ++_i572)
             {
-              TCell _elem565; // optional
-              _elem565 = new TCell();
-              _elem565.read(iprot);
-              struct.success.add(_elem565);
+              TCell _elem573; // required
+              _elem573 = new TCell();
+              _elem573.read(iprot);
+              struct.success.add(_elem573);
             }
           }
           struct.setSuccessIsSet(true);
@@ -52351,7 +53534,14 @@ public boolean equals(getRegionInfo_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRegionInfo_args other) {
@@ -52763,7 +53953,19 @@ public boolean equals(getRegionInfo_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getRegionInfo_result other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java
index 11e31e3fd473..08ba49eb1995 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -227,7 +228,14 @@ public boolean equals(IOError that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_message = true && (isSetMessage());
+    builder.append(present_message);
+    if (present_message)
+      builder.append(message);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(IOError other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java
index ede215fe4abb..46db10735249 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -226,7 +227,14 @@ public boolean equals(IllegalArgument that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_message = true && (isSetMessage());
+    builder.append(present_message);
+    if (present_message)
+      builder.append(message);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(IllegalArgument other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java
index ef1817f4cb03..3d9394ec6326 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -429,7 +430,29 @@ public boolean equals(Mutation that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_isDelete = true;
+    builder.append(present_isDelete);
+    if (present_isDelete)
+      builder.append(isDelete);
+
+    boolean present_column = true && (isSetColumn());
+    builder.append(present_column);
+    if (present_column)
+      builder.append(column);
+
+    boolean present_value = true && (isSetValue());
+    builder.append(present_value);
+    if (present_value)
+      builder.append(value);
+
+    boolean present_writeToWAL = true;
+    builder.append(present_writeToWAL);
+    if (present_writeToWAL)
+      builder.append(writeToWAL);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(Mutation other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java
index 6ee8ca7977a7..c4361a87f717 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -300,7 +301,19 @@ public boolean equals(TCell that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_value = true && (isSetValue());
+    builder.append(present_value);
+    if (present_value)
+      builder.append(value);
+
+    boolean present_timestamp = true;
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TCell other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java
index cb978e82508c..0a6dadfb9a6d 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -294,7 +295,19 @@ public boolean equals(TColumn that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_columnName = true && (isSetColumnName());
+    builder.append(present_columnName);
+    if (present_columnName)
+      builder.append(columnName);
+
+    boolean present_cell = true && (isSetCell());
+    builder.append(present_cell);
+    if (present_cell)
+      builder.append(cell);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TColumn other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java
index 6d24aa08c042..13f2f3e1c3a4 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -436,7 +437,29 @@ public boolean equals(TIncrement that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_table = true && (isSetTable());
+    builder.append(present_table);
+    if (present_table)
+      builder.append(table);
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_column = true && (isSetColumn());
+    builder.append(present_column);
+    if (present_column)
+      builder.append(column);
+
+    boolean present_ammount = true;
+    builder.append(present_ammount);
+    if (present_ammount)
+      builder.append(ammount);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TIncrement other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java
index ed251e83ea50..d23cc7051c2c 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -622,7 +623,44 @@ public boolean equals(TRegionInfo that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_startKey = true && (isSetStartKey());
+    builder.append(present_startKey);
+    if (present_startKey)
+      builder.append(startKey);
+
+    boolean present_endKey = true && (isSetEndKey());
+    builder.append(present_endKey);
+    if (present_endKey)
+      builder.append(endKey);
+
+    boolean present_id = true;
+    builder.append(present_id);
+    if (present_id)
+      builder.append(id);
+
+    boolean present_name = true && (isSetName());
+    builder.append(present_name);
+    if (present_name)
+      builder.append(name);
+
+    boolean present_version = true;
+    builder.append(present_version);
+    if (present_version)
+      builder.append(version);
+
+    boolean present_serverName = true && (isSetServerName());
+    builder.append(present_serverName);
+    if (present_serverName)
+      builder.append(serverName);
+
+    boolean present_port = true;
+    builder.append(present_port);
+    if (present_port)
+      builder.append(port);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TRegionInfo other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
index 763ab099f921..8c9a4471b446 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -395,7 +396,24 @@ public boolean equals(TRowResult that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_columns = true && (isSetColumns());
+    builder.append(present_columns);
+    if (present_columns)
+      builder.append(columns);
+
+    boolean present_sortedColumns = true && (isSetSortedColumns());
+    builder.append(present_sortedColumns);
+    if (present_sortedColumns)
+      builder.append(sortedColumns);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TRowResult other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java
index 0dce8555b5e9..7afed3a9579b 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -611,7 +612,44 @@ public boolean equals(TScan that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_startRow = true && (isSetStartRow());
+    builder.append(present_startRow);
+    if (present_startRow)
+      builder.append(startRow);
+
+    boolean present_stopRow = true && (isSetStopRow());
+    builder.append(present_stopRow);
+    if (present_stopRow)
+      builder.append(stopRow);
+
+    boolean present_timestamp = true && (isSetTimestamp());
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    boolean present_columns = true && (isSetColumns());
+    builder.append(present_columns);
+    if (present_columns)
+      builder.append(columns);
+
+    boolean present_caching = true && (isSetCaching());
+    builder.append(present_caching);
+    if (present_caching)
+      builder.append(caching);
+
+    boolean present_filterString = true && (isSetFilterString());
+    builder.append(present_filterString);
+    if (present_filterString)
+      builder.append(filterString);
+
+    boolean present_sortColumns = true && (isSetSortColumns());
+    builder.append(present_sortColumns);
+    if (present_sortColumns)
+      builder.append(sortColumns);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TScan other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java
index ba11ecefb53c..19e1a08b7f68 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -366,7 +367,24 @@ public boolean equals(TColumn that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_family = true && (isSetFamily());
+    builder.append(present_family);
+    if (present_family)
+      builder.append(family);
+
+    boolean present_qualifier = true && (isSetQualifier());
+    builder.append(present_qualifier);
+    if (present_qualifier)
+      builder.append(qualifier);
+
+    boolean present_timestamp = true && (isSetTimestamp());
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TColumn other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java
index 636225004ae8..2b2a7b1003b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -368,7 +369,24 @@ public boolean equals(TColumnIncrement that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_family = true && (isSetFamily());
+    builder.append(present_family);
+    if (present_family)
+      builder.append(family);
+
+    boolean present_qualifier = true && (isSetQualifier());
+    builder.append(present_qualifier);
+    if (present_qualifier)
+      builder.append(qualifier);
+
+    boolean present_amount = true && (isSetAmount());
+    builder.append(present_amount);
+    if (present_amount)
+      builder.append(amount);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TColumnIncrement other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java
index 39e5d2a8911b..614536441913 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -436,7 +437,29 @@ public boolean equals(TColumnValue that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_family = true && (isSetFamily());
+    builder.append(present_family);
+    if (present_family)
+      builder.append(family);
+
+    boolean present_qualifier = true && (isSetQualifier());
+    builder.append(present_qualifier);
+    if (present_qualifier)
+      builder.append(qualifier);
+
+    boolean present_value = true && (isSetValue());
+    builder.append(present_value);
+    if (present_value)
+      builder.append(value);
+
+    boolean present_timestamp = true && (isSetTimestamp());
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TColumnValue other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
index 0560a4936fa2..a34f0825dc67 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -682,7 +683,44 @@ public boolean equals(TDelete that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_columns = true && (isSetColumns());
+    builder.append(present_columns);
+    if (present_columns)
+      builder.append(columns);
+
+    boolean present_timestamp = true && (isSetTimestamp());
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    boolean present_deleteType = true && (isSetDeleteType());
+    builder.append(present_deleteType);
+    if (present_deleteType)
+      builder.append(deleteType.getValue());
+
+    boolean present_writeToWal = true && (isSetWriteToWal());
+    builder.append(present_writeToWal);
+    if (present_writeToWal)
+      builder.append(writeToWal);
+
+    boolean present_attributes = true && (isSetAttributes());
+    builder.append(present_attributes);
+    if (present_attributes)
+      builder.append(attributes);
+
+    boolean present_durability = true && (isSetDurability());
+    builder.append(present_durability);
+    if (present_durability)
+      builder.append(durability.getValue());
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TDelete other) {
@@ -948,7 +986,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) thr
                 for (int _i48 = 0; _i48 < _map47.size; ++_i48)
                 {
                   ByteBuffer _key49; // required
-                  ByteBuffer _val50; // optional
+                  ByteBuffer _val50; // required
                   _key49 = iprot.readBinary();
                   _val50 = iprot.readBinary();
                   struct.attributes.put(_key49, _val50);
@@ -1151,7 +1189,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro
           for (int _i59 = 0; _i59 < _map58.size; ++_i59)
           {
             ByteBuffer _key60; // required
-            ByteBuffer _val61; // optional
+            ByteBuffer _val61; // required
             _key60 = iprot.readBinary();
             _val61 = iprot.readBinary();
             struct.attributes.put(_key60, _val61);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
index fca79c324e2f..530b7da528db 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -648,7 +649,44 @@ public boolean equals(TGet that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_columns = true && (isSetColumns());
+    builder.append(present_columns);
+    if (present_columns)
+      builder.append(columns);
+
+    boolean present_timestamp = true && (isSetTimestamp());
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    boolean present_timeRange = true && (isSetTimeRange());
+    builder.append(present_timeRange);
+    if (present_timeRange)
+      builder.append(timeRange);
+
+    boolean present_maxVersions = true && (isSetMaxVersions());
+    builder.append(present_maxVersions);
+    if (present_maxVersions)
+      builder.append(maxVersions);
+
+    boolean present_filterString = true && (isSetFilterString());
+    builder.append(present_filterString);
+    if (present_filterString)
+      builder.append(filterString);
+
+    boolean present_attributes = true && (isSetAttributes());
+    builder.append(present_attributes);
+    if (present_attributes)
+      builder.append(attributes);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TGet other) {
@@ -923,7 +961,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TGet struct) throws
                 for (int _i12 = 0; _i12 < _map11.size; ++_i12)
                 {
                   ByteBuffer _key13; // required
-                  ByteBuffer _val14; // optional
+                  ByteBuffer _val14; // required
                   _key13 = iprot.readBinary();
                   _val14 = iprot.readBinary();
                   struct.attributes.put(_key13, _val14);
@@ -1123,7 +1161,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws
           for (int _i23 = 0; _i23 < _map22.size; ++_i23)
           {
             ByteBuffer _key24; // required
-            ByteBuffer _val25; // optional
+            ByteBuffer _val25; // required
             _key24 = iprot.readBinary();
             _val25 = iprot.readBinary();
             struct.attributes.put(_key24, _val25);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
index 4a80126230da..fdd60747daaf 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -1873,7 +1874,19 @@ public boolean equals(exists_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_get = true && (isSetGet());
+      builder.append(present_get);
+      if (present_get)
+        builder.append(get);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(exists_args other) {
@@ -2320,7 +2333,19 @@ public boolean equals(exists_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(exists_result other) {
@@ -2400,8 +2425,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -2806,7 +2829,19 @@ public boolean equals(get_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_get = true && (isSetGet());
+      builder.append(present_get);
+      if (present_get)
+        builder.append(get);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(get_args other) {
@@ -3250,7 +3285,19 @@ public boolean equals(get_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(get_result other) {
@@ -3770,7 +3817,19 @@ public boolean equals(getMultiple_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_gets = true && (isSetGets());
+      builder.append(present_gets);
+      if (present_gets)
+        builder.append(gets);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getMultiple_args other) {
@@ -4266,7 +4325,19 @@ public boolean equals(getMultiple_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getMultiple_result other) {
@@ -4790,7 +4861,19 @@ public boolean equals(put_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_put = true && (isSetPut());
+      builder.append(present_put);
+      if (present_put)
+        builder.append(put);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(put_args other) {
@@ -5175,7 +5258,14 @@ public boolean equals(put_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(put_result other) {
@@ -5960,7 +6050,39 @@ public boolean equals(checkAndPut_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_family = true && (isSetFamily());
+      builder.append(present_family);
+      if (present_family)
+        builder.append(family);
+
+      boolean present_qualifier = true && (isSetQualifier());
+      builder.append(present_qualifier);
+      if (present_qualifier)
+        builder.append(qualifier);
+
+      boolean present_value = true && (isSetValue());
+      builder.append(present_value);
+      if (present_value)
+        builder.append(value);
+
+      boolean present_put = true && (isSetPut());
+      builder.append(present_put);
+      if (present_put)
+        builder.append(put);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(checkAndPut_args other) {
@@ -6562,7 +6684,19 @@ public boolean equals(checkAndPut_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(checkAndPut_result other) {
@@ -6642,8 +6776,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -7068,7 +7200,19 @@ public boolean equals(putMultiple_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_puts = true && (isSetPuts());
+      builder.append(present_puts);
+      if (present_puts)
+        builder.append(puts);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(putMultiple_args other) {
@@ -7485,7 +7629,14 @@ public boolean equals(putMultiple_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(putMultiple_result other) {
@@ -7934,7 +8085,19 @@ public boolean equals(deleteSingle_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_deleteSingle = true && (isSetDeleteSingle());
+      builder.append(present_deleteSingle);
+      if (present_deleteSingle)
+        builder.append(deleteSingle);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteSingle_args other) {
@@ -8319,7 +8482,14 @@ public boolean equals(deleteSingle_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteSingle_result other) {
@@ -8788,7 +8958,19 @@ public boolean equals(deleteMultiple_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_deletes = true && (isSetDeletes());
+      builder.append(present_deletes);
+      if (present_deletes)
+        builder.append(deletes);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteMultiple_args other) {
@@ -9284,7 +9466,19 @@ public boolean equals(deleteMultiple_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(deleteMultiple_result other) {
@@ -10144,7 +10338,39 @@ public boolean equals(checkAndDelete_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_row = true && (isSetRow());
+      builder.append(present_row);
+      if (present_row)
+        builder.append(row);
+
+      boolean present_family = true && (isSetFamily());
+      builder.append(present_family);
+      if (present_family)
+        builder.append(family);
+
+      boolean present_qualifier = true && (isSetQualifier());
+      builder.append(present_qualifier);
+      if (present_qualifier)
+        builder.append(qualifier);
+
+      boolean present_value = true && (isSetValue());
+      builder.append(present_value);
+      if (present_value)
+        builder.append(value);
+
+      boolean present_deleteSingle = true && (isSetDeleteSingle());
+      builder.append(present_deleteSingle);
+      if (present_deleteSingle)
+        builder.append(deleteSingle);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(checkAndDelete_args other) {
@@ -10746,7 +10972,19 @@ public boolean equals(checkAndDelete_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(checkAndDelete_result other) {
@@ -10826,8 +11064,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -11232,7 +11468,19 @@ public boolean equals(increment_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_increment = true && (isSetIncrement());
+      builder.append(present_increment);
+      if (present_increment)
+        builder.append(increment);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(increment_args other) {
@@ -11676,7 +11924,19 @@ public boolean equals(increment_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(increment_result other) {
@@ -12168,7 +12428,19 @@ public boolean equals(openScanner_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_scan = true && (isSetScan());
+      builder.append(present_scan);
+      if (present_scan)
+        builder.append(scan);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(openScanner_args other) {
@@ -12615,7 +12887,19 @@ public boolean equals(openScanner_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true;
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(openScanner_result other) {
@@ -12695,8 +12979,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -13095,7 +13377,19 @@ public boolean equals(getScannerRows_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_scannerId = true;
+      builder.append(present_scannerId);
+      if (present_scannerId)
+        builder.append(scannerId);
+
+      boolean present_numRows = true;
+      builder.append(present_numRows);
+      if (present_numRows)
+        builder.append(numRows);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getScannerRows_args other) {
@@ -13626,7 +13920,24 @@ public boolean equals(getScannerRows_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getScannerRows_result other) {
@@ -14114,7 +14425,14 @@ public boolean equals(closeScanner_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_scannerId = true;
+      builder.append(present_scannerId);
+      if (present_scannerId)
+        builder.append(scannerId);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(closeScanner_args other) {
@@ -14177,8 +14495,6 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
       try {
-        // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -14528,7 +14844,19 @@ public boolean equals(closeScanner_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      boolean present_ia = true && (isSetIa());
+      builder.append(present_ia);
+      if (present_ia)
+        builder.append(ia);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(closeScanner_result other) {
@@ -15020,7 +15348,19 @@ public boolean equals(mutateRow_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_rowMutations = true && (isSetRowMutations());
+      builder.append(present_rowMutations);
+      if (present_rowMutations)
+        builder.append(rowMutations);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRow_args other) {
@@ -15405,7 +15745,14 @@ public boolean equals(mutateRow_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(mutateRow_result other) {
@@ -15930,7 +16277,24 @@ public boolean equals(getScannerResults_args that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_table = true && (isSetTable());
+      builder.append(present_table);
+      if (present_table)
+        builder.append(table);
+
+      boolean present_scan = true && (isSetScan());
+      builder.append(present_scan);
+      if (present_scan)
+        builder.append(scan);
+
+      boolean present_numRows = true;
+      builder.append(present_numRows);
+      if (present_numRows)
+        builder.append(numRows);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getScannerResults_args other) {
@@ -16434,7 +16798,19 @@ public boolean equals(getScannerResults_result that) {
 
     @Override
     public int hashCode() {
-      return 0;
+      HashCodeBuilder builder = new HashCodeBuilder();
+
+      boolean present_success = true && (isSetSuccess());
+      builder.append(present_success);
+      if (present_success)
+        builder.append(success);
+
+      boolean present_io = true && (isSetIo());
+      builder.append(present_io);
+      if (present_io)
+        builder.append(io);
+
+      return builder.toHashCode();
     }
 
     public int compareTo(getScannerResults_result other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java
index 19f7546b8720..1d7c33194841 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -221,7 +222,14 @@ public boolean equals(TIOError that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_message = true && (isSetMessage());
+    builder.append(present_message);
+    if (present_message)
+      builder.append(message);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TIOError other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java
index b5395a07322c..9bb425db8a57 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -220,7 +221,14 @@ public boolean equals(TIllegalArgument that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_message = true && (isSetMessage());
+    builder.append(present_message);
+    if (present_message)
+      builder.append(message);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TIllegalArgument other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java
index c618f42ad4b3..fdcbb02cacb9 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -380,7 +381,24 @@ public boolean equals(TIncrement that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_columns = true && (isSetColumns());
+    builder.append(present_columns);
+    if (present_columns)
+      builder.append(columns);
+
+    boolean present_writeToWal = true && (isSetWriteToWal());
+    builder.append(present_writeToWal);
+    if (present_writeToWal)
+      builder.append(writeToWal);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TIncrement other) {
@@ -523,14 +541,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TIncrement struct)
           case 2: // COLUMNS
             if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list32 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumnIncrement>(_list32.size);
-                for (int _i33 = 0; _i33 < _list32.size; ++_i33)
+                org.apache.thrift.protocol.TList _list62 = iprot.readListBegin();
+                struct.columns = new ArrayList<TColumnIncrement>(_list62.size);
+                for (int _i63 = 0; _i63 < _list62.size; ++_i63)
                 {
-                  TColumnIncrement _elem34; // required
-                  _elem34 = new TColumnIncrement();
-                  _elem34.read(iprot);
-                  struct.columns.add(_elem34);
+                  TColumnIncrement _elem64; // required
+                  _elem64 = new TColumnIncrement();
+                  _elem64.read(iprot);
+                  struct.columns.add(_elem64);
                 }
                 iprot.readListEnd();
               }
@@ -571,9 +589,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TIncrement struct)
         oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-          for (TColumnIncrement _iter35 : struct.columns)
+          for (TColumnIncrement _iter65 : struct.columns)
           {
-            _iter35.write(oprot);
+            _iter65.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -604,9 +622,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TIncrement struct)
       oprot.writeBinary(struct.row);
       {
         oprot.writeI32(struct.columns.size());
-        for (TColumnIncrement _iter36 : struct.columns)
+        for (TColumnIncrement _iter66 : struct.columns)
         {
-          _iter36.write(oprot);
+          _iter66.write(oprot);
         }
       }
       BitSet optionals = new BitSet();
@@ -625,14 +643,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) t
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
       {
-        org.apache.thrift.protocol.TList _list37 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.columns = new ArrayList<TColumnIncrement>(_list37.size);
-        for (int _i38 = 0; _i38 < _list37.size; ++_i38)
+        org.apache.thrift.protocol.TList _list67 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+        struct.columns = new ArrayList<TColumnIncrement>(_list67.size);
+        for (int _i68 = 0; _i68 < _list67.size; ++_i68)
         {
-          TColumnIncrement _elem39; // required
-          _elem39 = new TColumnIncrement();
-          _elem39.read(iprot);
-          struct.columns.add(_elem39);
+          TColumnIncrement _elem69; // required
+          _elem69 = new TColumnIncrement();
+          _elem69.read(iprot);
+          struct.columns.add(_elem69);
         }
       }
       struct.setColumnsIsSet(true);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java
index 5c409528600a..2edfdf9cf664 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -328,14 +329,22 @@ public int compareTo(TMutation other) {
   }
 
 
-  /**
-   * If you'd like this to perform more respectably, use the hashcode generator option.
-   */
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder hcb = new HashCodeBuilder();
+    hcb.append(this.getClass().getName());
+    org.apache.thrift.TFieldIdEnum setField = getSetField();
+    if (setField != null) {
+      hcb.append(setField.getThriftFieldId());
+      Object value = getFieldValue();
+      if (value instanceof org.apache.thrift.TEnum) {
+        hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue());
+      } else {
+        hcb.append(value);
+      }
+    }
+    return hcb.toHashCode();
   }
-
   private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
     try {
       write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
index 1cf071db37b9..91da6e17510a 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -596,7 +597,39 @@ public boolean equals(TPut that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_columnValues = true && (isSetColumnValues());
+    builder.append(present_columnValues);
+    if (present_columnValues)
+      builder.append(columnValues);
+
+    boolean present_timestamp = true && (isSetTimestamp());
+    builder.append(present_timestamp);
+    if (present_timestamp)
+      builder.append(timestamp);
+
+    boolean present_writeToWal = true && (isSetWriteToWal());
+    builder.append(present_writeToWal);
+    if (present_writeToWal)
+      builder.append(writeToWal);
+
+    boolean present_attributes = true && (isSetAttributes());
+    builder.append(present_attributes);
+    if (present_attributes)
+      builder.append(attributes);
+
+    boolean present_durability = true && (isSetDurability());
+    builder.append(present_durability);
+    if (present_durability)
+      builder.append(durability.getValue());
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TPut other) {
@@ -835,7 +868,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TPut struct) throws
                 for (int _i30 = 0; _i30 < _map29.size; ++_i30)
                 {
                   ByteBuffer _key31; // required
-                  ByteBuffer _val32; // optional
+                  ByteBuffer _val32; // required
                   _key31 = iprot.readBinary();
                   _val32 = iprot.readBinary();
                   struct.attributes.put(_key31, _val32);
@@ -1012,7 +1045,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws
           for (int _i41 = 0; _i41 < _map40.size; ++_i41)
           {
             ByteBuffer _key42; // required
-            ByteBuffer _val43; // optional
+            ByteBuffer _val43; // required
             _key42 = iprot.readBinary();
             _val43 = iprot.readBinary();
             struct.attributes.put(_key42, _val43);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java
index e91f1371d20b..0f72e02a3fff 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -314,7 +315,19 @@ public boolean equals(TResult that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_columnValues = true && (isSetColumnValues());
+    builder.append(present_columnValues);
+    if (present_columnValues)
+      builder.append(columnValues);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TResult other) {
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java
index 44e66789206d..165e8c36e149 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -315,7 +316,19 @@ public boolean equals(TRowMutations that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_row = true && (isSetRow());
+    builder.append(present_row);
+    if (present_row)
+      builder.append(row);
+
+    boolean present_mutations = true && (isSetMutations());
+    builder.append(present_mutations);
+    if (present_mutations)
+      builder.append(mutations);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TRowMutations other) {
@@ -440,14 +453,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowMutations struc
           case 2: // MUTATIONS
             if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list98 = iprot.readListBegin();
-                struct.mutations = new ArrayList<TMutation>(_list98.size);
-                for (int _i99 = 0; _i99 < _list98.size; ++_i99)
+                org.apache.thrift.protocol.TList _list88 = iprot.readListBegin();
+                struct.mutations = new ArrayList<TMutation>(_list88.size);
+                for (int _i89 = 0; _i89 < _list88.size; ++_i89)
                 {
-                  TMutation _elem100; // required
-                  _elem100 = new TMutation();
-                  _elem100.read(iprot);
-                  struct.mutations.add(_elem100);
+                  TMutation _elem90; // required
+                  _elem90 = new TMutation();
+                  _elem90.read(iprot);
+                  struct.mutations.add(_elem90);
                 }
                 iprot.readListEnd();
               }
@@ -480,9 +493,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TRowMutations stru
         oprot.writeFieldBegin(MUTATIONS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size()));
-          for (TMutation _iter101 : struct.mutations)
+          for (TMutation _iter91 : struct.mutations)
           {
-            _iter101.write(oprot);
+            _iter91.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -508,9 +521,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRowMutations struc
       oprot.writeBinary(struct.row);
       {
         oprot.writeI32(struct.mutations.size());
-        for (TMutation _iter102 : struct.mutations)
+        for (TMutation _iter92 : struct.mutations)
         {
-          _iter102.write(oprot);
+          _iter92.write(oprot);
         }
       }
     }
@@ -521,14 +534,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TRowMutations struct
       struct.row = iprot.readBinary();
       struct.setRowIsSet(true);
       {
-        org.apache.thrift.protocol.TList _list103 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.mutations = new ArrayList<TMutation>(_list103.size);
-        for (int _i104 = 0; _i104 < _list103.size; ++_i104)
+        org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+        struct.mutations = new ArrayList<TMutation>(_list93.size);
+        for (int _i94 = 0; _i94 < _list93.size; ++_i94)
         {
-          TMutation _elem105; // required
-          _elem105 = new TMutation();
-          _elem105.read(iprot);
-          struct.mutations.add(_elem105);
+          TMutation _elem95; // required
+          _elem95 = new TMutation();
+          _elem95.read(iprot);
+          struct.mutations.add(_elem95);
         }
       }
       struct.setMutationsIsSet(true);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
index 4c58a8c84f66..3bca929382e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -758,7 +759,54 @@ public boolean equals(TScan that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_startRow = true && (isSetStartRow());
+    builder.append(present_startRow);
+    if (present_startRow)
+      builder.append(startRow);
+
+    boolean present_stopRow = true && (isSetStopRow());
+    builder.append(present_stopRow);
+    if (present_stopRow)
+      builder.append(stopRow);
+
+    boolean present_columns = true && (isSetColumns());
+    builder.append(present_columns);
+    if (present_columns)
+      builder.append(columns);
+
+    boolean present_caching = true && (isSetCaching());
+    builder.append(present_caching);
+    if (present_caching)
+      builder.append(caching);
+
+    boolean present_maxVersions = true && (isSetMaxVersions());
+    builder.append(present_maxVersions);
+    if (present_maxVersions)
+      builder.append(maxVersions);
+
+    boolean present_timeRange = true && (isSetTimeRange());
+    builder.append(present_timeRange);
+    if (present_timeRange)
+      builder.append(timeRange);
+
+    boolean present_filterString = true && (isSetFilterString());
+    builder.append(present_filterString);
+    if (present_filterString)
+      builder.append(filterString);
+
+    boolean present_batchSize = true && (isSetBatchSize());
+    builder.append(present_batchSize);
+    if (present_batchSize)
+      builder.append(batchSize);
+
+    boolean present_attributes = true && (isSetAttributes());
+    builder.append(present_attributes);
+    if (present_attributes)
+      builder.append(attributes);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TScan other) {
@@ -1084,7 +1132,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throw
                 for (int _i74 = 0; _i74 < _map73.size; ++_i74)
                 {
                   ByteBuffer _key75; // required
-                  ByteBuffer _val76; // optional
+                  ByteBuffer _val76; // required
                   _key75 = iprot.readBinary();
                   _val76 = iprot.readBinary();
                   struct.attributes.put(_key75, _val76);
@@ -1325,7 +1373,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws
           for (int _i85 = 0; _i85 < _map84.size; ++_i85)
           {
             ByteBuffer _key86; // required
-            ByteBuffer _val87; // optional
+            ByteBuffer _val87; // required
             _key86 = iprot.readBinary();
             _val87 = iprot.readBinary();
             struct.attributes.put(_key86, _val87);
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java
index 98058ea7c2d6..7d51039ea973 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java
@@ -6,6 +6,7 @@
  */
 package org.apache.hadoop.hbase.thrift2.generated;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.thrift.scheme.IScheme;
 import org.apache.thrift.scheme.SchemeFactory;
 import org.apache.thrift.scheme.StandardScheme;
@@ -284,7 +285,19 @@ public boolean equals(TTimeRange that) {
 
   @Override
   public int hashCode() {
-    return 0;
+    HashCodeBuilder builder = new HashCodeBuilder();
+
+    boolean present_minStamp = true;
+    builder.append(present_minStamp);
+    if (present_minStamp)
+      builder.append(minStamp);
+
+    boolean present_maxStamp = true;
+    builder.append(present_maxStamp);
+    if (present_maxStamp)
+      builder.append(maxStamp);
+
+    return builder.toHashCode();
   }
 
   public int compareTo(TTimeRange other) {

From 65b56212fa4ac9f3064b923331ce21d4b520a16c Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Sat, 29 Nov 2014 09:41:42 -0800
Subject: [PATCH 1496/1540] HBASE-12491 TableMapReduceUtil.findContainingJar()
 NPE

---
 .../hbase/mapreduce/TableMapReduceUtil.java   | 35 ++++++++++---------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index a6b2eb05ba42..5ea583177b62 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -815,25 +815,28 @@ private static void updateMap(String jar, Map<String, String> packagedClasses) t
   private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
       throws IOException {
     ClassLoader loader = my_class.getClassLoader();
+
     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
 
-    // first search the classpath
-    for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
-      URL url = itr.nextElement();
-      if ("jar".equals(url.getProtocol())) {
-        String toReturn = url.getPath();
-        if (toReturn.startsWith("file:")) {
-          toReturn = toReturn.substring("file:".length());
+    if (loader != null) {
+      // first search the classpath
+      for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
+        URL url = itr.nextElement();
+        if ("jar".equals(url.getProtocol())) {
+          String toReturn = url.getPath();
+          if (toReturn.startsWith("file:")) {
+            toReturn = toReturn.substring("file:".length());
+          }
+          // URLDecoder is a misnamed class, since it actually decodes
+          // x-www-form-urlencoded MIME type rather than actual
+          // URL encoding (which the file path has). Therefore it would
+          // decode +s to ' 's which is incorrect (spaces are actually
+          // either unencoded or encoded as "%20"). Replace +s first, so
+          // that they are kept sacred during the decoding process.
+          toReturn = toReturn.replaceAll("\\+", "%2B");
+          toReturn = URLDecoder.decode(toReturn, "UTF-8");
+          return toReturn.replaceAll("!.*$", "");
         }
-        // URLDecoder is a misnamed class, since it actually decodes
-        // x-www-form-urlencoded MIME type rather than actual
-        // URL encoding (which the file path has). Therefore it would
-        // decode +s to ' 's which is incorrect (spaces are actually
-        // either unencoded or encoded as "%20"). Replace +s first, so
-        // that they are kept sacred during the decoding process.
-        toReturn = toReturn.replaceAll("\\+", "%2B");
-        toReturn = URLDecoder.decode(toReturn, "UTF-8");
-        return toReturn.replaceAll("!.*$", "");
       }
     }
 

From aafed16af960afad614427513cacdbbabd98426d Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Thu, 4 Dec 2014 16:43:33 -0800
Subject: [PATCH 1497/1540] HBASE-12635 Delete acl notify znode of table after
 the table is deleted (Liu Shaohui)

---
 .../security/access/AccessController.java      |  1 +
 .../security/access/ZKPermissionWatcher.java   | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 6161382bd628..62faa691784d 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -656,6 +656,7 @@ public Void run() throws Exception {
         return null;
       }
     });
+    this.authManager.getZKPermissionWatcher().deleteTableACLNode(tableName);
   }
 
   @Override
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
index 4870bb19651f..e9cf8975968f 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
@@ -159,4 +159,22 @@ public void writeToZookeeper(byte[] tableName, byte[] parmsData) {
       watcher.abort("Failed writing node "+zkNode+" to zookeeper", e);
     }
   }
+
+  /***
+   * Delete the acl notify node of table
+   * @param tableName
+   */
+  public void deleteTableACLNode(final byte[] tableName) {
+    String zkNode = ZKUtil.joinZNode(watcher.baseZNode, ACL_NODE);
+    zkNode = ZKUtil.joinZNode(zkNode, Bytes.toString(tableName));
+
+    try {
+      ZKUtil.deleteNode(watcher, zkNode);
+    } catch (KeeperException.NoNodeException e) {
+      LOG.warn("No acl notify node of table '" + tableName + "'");
+    } catch (KeeperException e) {
+      LOG.error("Failed deleting acl node of table '" + tableName + "'", e);
+      watcher.abort("Failed deleting node " + zkNode, e);
+    }
+  }
 }

From 6e6dc4bd81695b7aa5ce283019d059c0570cb32e Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 11 Dec 2014 13:23:20 -0800
Subject: [PATCH 1498/1540] HBASE-12657 The Region is not being split and far
 exceeds the desired maximum size. (Vladimir Rodionov)

---
 .../java/org/apache/hadoop/hbase/regionserver/Store.java    | 6 ++++--
 .../hadoop/hbase/regionserver/TestCompactSelection.java     | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 915777fae784..46b4e528a017 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1598,8 +1598,10 @@ public boolean apply(StoreFile input) {
         }
       } else if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
         // all files included in this compaction, up to max
-        int pastMax = compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
-        compactSelection.getFilesToCompact().subList(0, pastMax).clear();
+        int excess = compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
+        LOG.debug("Too many admissible files. Excluding " + excess
+          + " files from compaction candidates");
+        candidates.subList(this.maxFilesToCompact, candidates.size()).clear();
       }
     }
     return compactSelection;
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
index f996dda5b227..b37053aeba8c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
@@ -265,7 +265,7 @@ public void testCompactionRatio() throws IOException {
     assertEquals(maxFiles,
         store.compactSelection(sfCreate(true, 7,6,5,4,3,2,1)).getFilesToCompact().size());
     // reference compaction
-    compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
+    compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
     
     // empty case
     compactEquals(new ArrayList<StoreFile>() /* empty */);

From 5e587d193db03e1ff99d06c80a4bdf9fea6d0b73 Mon Sep 17 00:00:00 2001
From: Jonathan M Hsieh <jon@cloudera.com>
Date: Mon, 15 Dec 2014 11:26:55 -0800
Subject: [PATCH 1499/1540] HBASE-12692 NPE from SnapshotManager#stop (Ashish
 Singhi)

---
 .../apache/hadoop/hbase/master/snapshot/SnapshotManager.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index cd1fd5f85b6d..226ffb1372a7 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -899,7 +899,9 @@ public void stop(String why) {
     }
 
     try {
-      coordinator.close();
+      if (coordinator != null) {
+        coordinator.close();
+      }
     } catch (IOException e) {
       LOG.error("stop ProcedureCoordinator error", e);
     }

From a9114543cd48c7e230dabd31f45a086f7b7a5d6a Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 16 Dec 2014 13:51:35 -0800
Subject: [PATCH 1500/1540] pom.xml and CHANGES.txt for 0.94.26RC0

---
 CHANGES.txt | 9 +++++++++
 pom.xml     | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index e4dd4037f1d2..eb9734f90654 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,13 @@
 HBase Change Log
+Release 0.94.26 - 12/16/2014
+Bug
+
+    [HBASE-12279] - Generated thrift files were generated with the wrong parameters
+    [HBASE-12491] - TableMapReduceUtil.findContainingJar() NPE
+    [HBASE-12635] - Delete acl notify znode of table after the table is deleted
+    [HBASE-12657] - The Region is not being split and far exceeds the desired maximum size.
+    [HBASE-12692] - NPE from SnapshotManager#stop
+
 Release 0.94.25 - 11/7/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index 2105f1b2a2dc..3a72dbab6b5f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.25</version>
+  <version>0.94.26</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 20c83b53cac6d45966f8a2e8b7f596a5b4639adb Mon Sep 17 00:00:00 2001
From: tedyu <yuzhihong@gmail.com>
Date: Thu, 18 Dec 2014 05:15:24 -0800
Subject: [PATCH 1501/1540] HBASE-12223 MultiTableInputFormatBase.getSplits is
 too slow (Yuanbo Peng)

---
 .../mapreduce/MultiTableInputFormatBase.java  | 97 +++++++++++--------
 1 file changed, 59 insertions(+), 38 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
index 76a1632bc192..7160eac2e780 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -37,6 +37,9 @@
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
 /**
  * A base for {@link MultiTableInputFormat}s. Receives a list of
  * {@link Scan} instances that define the input tables and
@@ -105,59 +108,77 @@ public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
    * @throws IOException When creating the list of splits fails.
    * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)
    */
+
+
   @Override
   public List<InputSplit> getSplits(JobContext context) throws IOException {
     if (scans.isEmpty()) {
       throw new IOException("No scans were provided.");
     }
-    List<InputSplit> splits = new ArrayList<InputSplit>();
 
+    Map<String, List<Scan>> tableMaps = new HashMap<String, List<Scan>>();
     for (Scan scan : scans) {
       byte[] tableName = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
-      if (tableName == null) 
+      if (tableName == null)
         throw new IOException("A scan object did not have a table name");
-      HTable table = new HTable(context.getConfiguration(), tableName);
-      Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
-      if (keys == null || keys.getFirst() == null ||
-          keys.getFirst().length == 0) {
-        throw new IOException("Expecting at least one region for table : "
-            + Bytes.toString(tableName));
+      String tableNameStr = Bytes.toString(tableName);
+
+      List<Scan> scanList = tableMaps.get(tableNameStr);
+
+      if (scanList == null) {
+        scanList = new ArrayList<Scan>();
+        tableMaps.put(tableNameStr, scanList);
       }
-      int count = 0;
-      
-      byte[] startRow = scan.getStartRow();
-      byte[] stopRow = scan.getStopRow();
-
-      for (int i = 0; i < keys.getFirst().length; i++) {
-        if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
-          continue;
+      scanList.add(scan);
+    }
+
+    List<InputSplit> splits = new ArrayList<InputSplit>();
+    Iterator iter = tableMaps.entrySet().iterator();
+    while (iter.hasNext()) {
+      Map.Entry<String, List<Scan>> entry = (Map.Entry<String, List<Scan>>) iter.next();
+      String tableNameStr = entry.getKey();
+      List<Scan> scanList = entry.getValue();
+      HTable table = new HTable(context.getConfiguration(), tableNameStr);
+      Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
+      for (Scan scan : scanList) {
+        if (keys == null || keys.getFirst() == null ||
+                keys.getFirst().length == 0) {
+          throw new IOException("Expecting at least one region for table : "
+                  + tableNameStr);
         }
-        String regionLocation =
-            table.getRegionLocation(keys.getFirst()[i], false).getHostname();
-        
-        // determine if the given start and stop keys fall into the range
-        if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
-            Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
-            (stopRow.length == 0 ||
-                Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
-          byte[] splitStart =
-              startRow.length == 0 ||
-                  Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
-                  .getFirst()[i] : startRow;
-          byte[] splitStop =
-              (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
-                  stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
-                  .getSecond()[i] : stopRow;
-          InputSplit split =
-              new TableSplit(tableName, scan, splitStart,
-                  splitStop, regionLocation);
-          splits.add(split);
-          if (LOG.isDebugEnabled())
-            LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+        int count = 0;
+        byte[] startRow = scan.getStartRow();
+        byte[] stopRow = scan.getStopRow();
+        for (int i = 0; i < keys.getFirst().length; i++) {
+          if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+            continue;
+          }
+
+          // determine if the given start and stop keys fall into the range
+          if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+                  Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+                  (stopRow.length == 0 || Bytes.compareTo(stopRow,
+                  keys.getFirst()[i]) > 0)) {
+            byte[] splitStart = startRow.length == 0 ||
+                    Bytes.compareTo(keys.getFirst()[i],
+                    startRow) >= 0 ? keys.getFirst()[i] : startRow;
+            byte[] splitStop = (stopRow.length == 0 ||
+                    Bytes.compareTo(keys.getSecond()[i],
+                    stopRow) <= 0) && keys.getSecond()[i].length > 0 ?
+                    keys.getSecond()[i] : stopRow;
+            String regionLocation = table.getRegionLocation(
+                    keys.getFirst()[i], false).getHostname();
+            InputSplit split = new TableSplit(Bytes.toBytes(tableNameStr), scan,
+                    splitStart, splitStop, regionLocation);
+            splits.add(split);
+            if (LOG.isDebugEnabled())
+              LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+          }
         }
       }
       table.close();
     }
+
     return splits;
   }
 

From 597d57b9efc4aa149adaab982c14e50eabd00ca0 Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Fri, 19 Dec 2014 14:09:25 -0800
Subject: [PATCH 1502/1540] HBASE-12720 Make InternalScan LimitedPrivate
 (Vladimir Rodionov)

---
 .../hadoop/hbase/regionserver/InternalScan.java | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
index db2e02d80a81..000c109eac18 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
@@ -19,11 +19,13 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.IOException;
+
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Scan;
 
 /**
- * Special internal-only scanner, currently used for increment operations to
+ * Special scanner, currently used for increment operations to
  * allow additional server-side arguments for Scan operations.
  * <p>
  * Rather than adding new options/parameters to the public Scan API, this new
@@ -33,7 +35,7 @@
  * {@link #checkOnlyMemStore()} or to only read from StoreFiles with
  * {@link #checkOnlyStoreFiles()}.
  */
-class InternalScan extends Scan {
+public class InternalScan extends Scan {
   private boolean memOnly = false;
   private boolean filesOnly = false;
 
@@ -43,7 +45,16 @@ class InternalScan extends Scan {
   public InternalScan(Get get) {
     super(get);
   }
-
+  
+  /**
+   * @param scan - original scan
+   * @throws IOException 
+   */
+  public InternalScan(Scan scan) 
+      throws IOException 
+  {
+    super(scan);
+  }
   /**
    * StoreFiles will not be scanned. Only MemStore will be scanned.
    */

From d93ea946146321fd7357ace83b9a0b0558fe605f Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Tue, 23 Dec 2014 12:40:22 -0800
Subject: [PATCH 1503/1540] HBASE-12739 Avoid too large identifier of
 ZooKeeperWatcher (Liu Shaohui)

---
 .../apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java   | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 81081d56a9ec..bca28e81a6c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -56,6 +56,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable {
 
   // Identifier for this watcher (for logging only).  It is made of the prefix
   // passed on construction and the zookeeper sessionid.
+  private String prefix;
   private String identifier;
 
   // zookeeper quorum
@@ -147,7 +148,8 @@ public ZooKeeperWatcher(Configuration conf, String descriptor,
     this.quorum = ZKConfig.getZKQuorumServersString(conf);
     // Identifier will get the sessionid appended later below down when we
     // handle the syncconnect event.
-    this.identifier = descriptor;
+    this.prefix = descriptor;
+    this.identifier = descriptor + "0x0";
     this.abortable = abortable;
     setNodeNames(conf);
     this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, descriptor);
@@ -365,7 +367,7 @@ private void connectionEvent(WatchedEvent event) {
             this.constructorCaller);
           throw new NullPointerException("ZK is null");
         }
-        this.identifier = this.identifier + "-0x" +
+        this.identifier = this.prefix + "-0x" +
           Long.toHexString(this.recoverableZooKeeper.getSessionId());
         // Update our identifier.  Otherwise ignore.
         LOG.debug(this.identifier + " connected");
@@ -459,7 +461,7 @@ public Configuration getConfiguration() {
   public void abort(String why, Throwable e) {
     this.abortable.abort(why, e);
   }
-  
+
   @Override
   public boolean isAborted() {
     return this.abortable.isAborted();

From 403d4eb4d295d22611699fea8acd055ea9df8ed0 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 29 Dec 2014 20:44:06 -0800
Subject: [PATCH 1504/1540] HBASE-12776 SpliTransaction: Log number of files to
 be split.

---
 .../org/apache/hadoop/hbase/regionserver/SplitTransaction.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 113e9d927db3..356110d28f7d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -599,6 +599,7 @@ private void splitStoreFiles(final Path splitdir,
       // no file needs to be splitted.
       return;
     }
+    LOG.info("Preparing to split " + nbFiles + " storefiles for region " + this.parent);
     ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
     builder.setNameFormat("StoreFileSplitter-%1$d");
     ThreadFactory factory = builder.build();

From d12bf9654184f3dd1e2ba86ff89bb758b1138f77 Mon Sep 17 00:00:00 2001
From: Esteban Gutierrez <esteban@cloudera.com>
Date: Tue, 30 Dec 2014 23:26:05 -0800
Subject: [PATCH 1505/1540] HBASE-5835: Catch and handle
 NotServingRegionException when close region attempt fails

Signed-off-by: stack <stack@apache.org>
---
 .../java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index 81ad866105ff..03f8a1cc9271 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -147,7 +147,11 @@ public static void closeRegionSilentlyAndWait(HBaseAdmin admin,
     HConnection connection = admin.getConnection();
     HRegionInterface rs = connection.getHRegionConnection(server.getHostname(),
         server.getPort());
-    rs.closeRegion(region, false);
+    try {
+      rs.closeRegion(region, false);
+    } catch (IOException ioe) {
+      LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), ioe);
+    }
     long timeout = admin.getConfiguration()
       .getLong("hbase.hbck.close.timeout", 120000);
     long expiration = timeout + System.currentTimeMillis();

From 5cdc9890ecfa24066a51941d014f4547d92cc806 Mon Sep 17 00:00:00 2001
From: tedyu <yuzhihong@gmail.com>
Date: Thu, 8 Jan 2015 15:12:47 -0800
Subject: [PATCH 1506/1540] HBASE-10528 DefaultBalancer selects plans to move
 regions onto draining nodes (Churro Morales)

---
 .../hbase/master/AssignmentManager.java       |  2 +
 .../hbase/master/TestAssignmentManager.java   | 68 +++++++++++++------
 2 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 87553175b0a2..31d8014cebdf 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -3431,6 +3431,7 @@ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
       }
     }
     Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
+    List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
     // Take care of servers w/o assignments.
     for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
       for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
@@ -3438,6 +3439,7 @@ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
           map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
         }
       }
+      map.keySet().removeAll(drainingServers);
     }
     return result;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 6d1b589014c9..e56068464691 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -24,12 +24,10 @@
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -92,6 +90,8 @@ public class TestAssignmentManager {
     new ServerName("example.org", 1234, 5678);
   private static final ServerName SERVERNAME_B =
     new ServerName("example.org", 0, 5678);
+  private static final ServerName SERVERNAME_C =
+      new ServerName("example.org", 123, 5678);
   private static final HRegionInfo REGIONINFO =
     new HRegionInfo(Bytes.toBytes("t"),
       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
@@ -134,24 +134,23 @@ public void before() throws ZooKeeperConnectionException, IOException {
 
     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
     // make it so if close or open, we return 'success'.
-    this.serverManager = Mockito.mock(ServerManager.class);
-    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
-    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
+    this.serverManager = mockManager(SERVERNAME_A, SERVERNAME_B);
+  }
+
+  private ServerManager mockManager(ServerName... servers) throws IOException {
+    ServerManager serverManager = Mockito.mock(ServerManager.class);
     final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
-    onlineServers.put(SERVERNAME_B, new HServerLoad());
-    onlineServers.put(SERVERNAME_A, new HServerLoad());
-    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
+    for (ServerName server : servers) {
+      Mockito.when(serverManager.isServerOnline(server)).thenReturn(true);
+      onlineServers.put(server, new HServerLoad());
+      Mockito.when(serverManager.sendRegionClose(server, REGIONINFO, -1)).thenReturn(true);
+      Mockito.when(serverManager.sendRegionOpen(server, REGIONINFO, -1)).
+          thenReturn(RegionOpeningState.OPENED);
+    }
+    Mockito.when(serverManager.getOnlineServersList()).thenReturn(
         new ArrayList<ServerName>(onlineServers.keySet()));
-    Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
-    Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
-      thenReturn(true);
-    Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
-      thenReturn(true);
-    // Ditto on open.
-    Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
-      thenReturn(RegionOpeningState.OPENED);
-    Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
-    thenReturn(RegionOpeningState.OPENED);
+    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
+    return serverManager;
   }
 
   @After
@@ -397,6 +396,35 @@ public void testBalance()
     }
   }
 
+  @Test
+  public void testGettingAssignmentsExcludesDrainingServers() throws Exception {
+    List<ServerName> availableServers =
+        Lists.newArrayList(SERVERNAME_A, SERVERNAME_B, SERVERNAME_C);
+    ServerManager serverManager = mockManager(availableServers.toArray(new ServerName[0]));
+
+
+    ExecutorService executor = startupMasterExecutor("testAssignmentsWithRSInDraining");
+    CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
+
+    Mockito.when(serverManager.getDrainingServersList()).thenReturn(
+        Lists.newArrayList(SERVERNAME_C));
+    AssignmentManager am = new AssignmentManager(this.server, serverManager, ct, balancer, executor);
+
+    for (ServerName availableServer : availableServers) {
+      HRegionInfo info = Mockito.mock(HRegionInfo.class);
+      Mockito.when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString());
+      am.regionOnline(info, availableServer);
+    }
+
+    Map<String, Map<ServerName, List<HRegionInfo>>> result = am.getAssignmentsByTable();
+    for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
+      System.out.println(map.keySet());
+      assertFalse(map.containsKey(SERVERNAME_C));
+    }
+  }
+
   /**
    * Run a simple server shutdown handler.
    * @throws KeeperException

From 72d17265c19e2c0bba4bcb4a101ad099e2ae658e Mon Sep 17 00:00:00 2001
From: tedyu <yuzhihong@gmail.com>
Date: Mon, 12 Jan 2015 18:53:50 -0800
Subject: [PATCH 1507/1540] HBASE-10528 Revert due to test failure

---
 .../hbase/master/AssignmentManager.java       |  2 -
 .../hbase/master/TestAssignmentManager.java   | 68 ++++++-------------
 2 files changed, 20 insertions(+), 50 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 31d8014cebdf..87553175b0a2 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -3431,7 +3431,6 @@ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
       }
     }
     Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
-    List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
     // Take care of servers w/o assignments.
     for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
       for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
@@ -3439,7 +3438,6 @@ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
           map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
         }
       }
-      map.keySet().removeAll(drainingServers);
     }
     return result;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index e56068464691..6d1b589014c9 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -24,10 +24,12 @@
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import com.google.common.collect.Lists;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -90,8 +92,6 @@ public class TestAssignmentManager {
     new ServerName("example.org", 1234, 5678);
   private static final ServerName SERVERNAME_B =
     new ServerName("example.org", 0, 5678);
-  private static final ServerName SERVERNAME_C =
-      new ServerName("example.org", 123, 5678);
   private static final HRegionInfo REGIONINFO =
     new HRegionInfo(Bytes.toBytes("t"),
       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
@@ -134,23 +134,24 @@ public void before() throws ZooKeeperConnectionException, IOException {
 
     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
     // make it so if close or open, we return 'success'.
-    this.serverManager = mockManager(SERVERNAME_A, SERVERNAME_B);
-  }
-
-  private ServerManager mockManager(ServerName... servers) throws IOException {
-    ServerManager serverManager = Mockito.mock(ServerManager.class);
+    this.serverManager = Mockito.mock(ServerManager.class);
+    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
+    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
     final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
-    for (ServerName server : servers) {
-      Mockito.when(serverManager.isServerOnline(server)).thenReturn(true);
-      onlineServers.put(server, new HServerLoad());
-      Mockito.when(serverManager.sendRegionClose(server, REGIONINFO, -1)).thenReturn(true);
-      Mockito.when(serverManager.sendRegionOpen(server, REGIONINFO, -1)).
-          thenReturn(RegionOpeningState.OPENED);
-    }
-    Mockito.when(serverManager.getOnlineServersList()).thenReturn(
+    onlineServers.put(SERVERNAME_B, new HServerLoad());
+    onlineServers.put(SERVERNAME_A, new HServerLoad());
+    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
         new ArrayList<ServerName>(onlineServers.keySet()));
-    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
-    return serverManager;
+    Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
+    Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
+      thenReturn(true);
+    Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
+      thenReturn(true);
+    // Ditto on open.
+    Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
+      thenReturn(RegionOpeningState.OPENED);
+    Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
+    thenReturn(RegionOpeningState.OPENED);
   }
 
   @After
@@ -396,35 +397,6 @@ public void testBalance()
     }
   }
 
-  @Test
-  public void testGettingAssignmentsExcludesDrainingServers() throws Exception {
-    List<ServerName> availableServers =
-        Lists.newArrayList(SERVERNAME_A, SERVERNAME_B, SERVERNAME_C);
-    ServerManager serverManager = mockManager(availableServers.toArray(new ServerName[0]));
-
-
-    ExecutorService executor = startupMasterExecutor("testAssignmentsWithRSInDraining");
-    CatalogTracker ct = Mockito.mock(CatalogTracker.class);
-
-    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
-
-    Mockito.when(serverManager.getDrainingServersList()).thenReturn(
-        Lists.newArrayList(SERVERNAME_C));
-    AssignmentManager am = new AssignmentManager(this.server, serverManager, ct, balancer, executor);
-
-    for (ServerName availableServer : availableServers) {
-      HRegionInfo info = Mockito.mock(HRegionInfo.class);
-      Mockito.when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString());
-      am.regionOnline(info, availableServer);
-    }
-
-    Map<String, Map<ServerName, List<HRegionInfo>>> result = am.getAssignmentsByTable();
-    for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
-      System.out.println(map.keySet());
-      assertFalse(map.containsKey(SERVERNAME_C));
-    }
-  }
-
   /**
    * Run a simple server shutdown handler.
    * @throws KeeperException

From eef4ad69a5325cd83b3391bea5a3eb327cb24be5 Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Mon, 12 Jan 2015 19:09:02 -0800
Subject: [PATCH 1508/1540] HBASE-12801 Failed to truncate a table while
 maintaing binary region boundaries (Liu Shaohui)

---
 .../java/org/apache/hadoop/hbase/util/Bytes.java    | 13 +++++++++++++
 src/main/ruby/hbase/admin.rb                        |  4 ++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index 6a4c25076281..3fa1317a6d97 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -1648,6 +1648,19 @@ public static int hashCode(byte[] bytes, int offset, int length) {
     return result;
   }
 
+
+  /**
+   * @param t operands
+   * @return Array of binary byte arrays made from passed array of binary strings
+   */
+  public static byte [][] toBinaryByteArrays(final String [] t) {
+    byte [][] result = new byte[t.length][];
+    for (int i = 0; i < t.length; i++) {
+      result[i] = Bytes.toBytesBinary(t[i]);
+    }
+    return result;
+  }
+
   /**
    * @param column operand
    * @return A byte array of a byte array where first and only entry is
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 266f1ffc89d9..367485bd0ff5 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -330,8 +330,8 @@ def truncate(table_name, conf = @conf)
     # Truncates table while maintaing region boundaries (deletes all records by recreating the table)
     def truncate_preserve(table_name, conf = @conf)
       h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name)
-      splits = h_table.getRegionLocations().keys().map{|i| Bytes.toString(i.getStartKey)}.delete_if{|k| k == ""}.to_java :String
-      splits = org.apache.hadoop.hbase.util.Bytes.toByteArrays(splits)
+      splits = h_table.getRegionLocations().keys().map{|i| Bytes.toStringBinary(i.getStartKey)}.delete_if{|k| k == ""}.to_java :String
+      splits = org.apache.hadoop.hbase.util.Bytes.toBinaryByteArrays(splits)
       table_description = h_table.getTableDescriptor()
       yield 'Disabling table...' if block_given?
       disable(table_name)

From 1f0f62ced826189bf9dbfcee2247d89b4d90b0e3 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 15 Jan 2015 09:57:57 -0800
Subject: [PATCH 1509/1540] HBASE-10528 DefaultBalancer selects plans to move
 regions onto draining nodes. (churro morales)

---
 .../hbase/master/AssignmentManager.java       |  2 +
 .../hbase/master/TestAssignmentManager.java   | 68 +++++++++++++------
 2 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 87553175b0a2..31d8014cebdf 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -3431,6 +3431,7 @@ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
       }
     }
     Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
+    List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
     // Take care of servers w/o assignments.
     for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
       for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
@@ -3438,6 +3439,7 @@ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
           map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
         }
       }
+      map.keySet().removeAll(drainingServers);
     }
     return result;
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 6d1b589014c9..e56068464691 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -24,12 +24,10 @@
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -92,6 +90,8 @@ public class TestAssignmentManager {
     new ServerName("example.org", 1234, 5678);
   private static final ServerName SERVERNAME_B =
     new ServerName("example.org", 0, 5678);
+  private static final ServerName SERVERNAME_C =
+      new ServerName("example.org", 123, 5678);
   private static final HRegionInfo REGIONINFO =
     new HRegionInfo(Bytes.toBytes("t"),
       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
@@ -134,24 +134,23 @@ public void before() throws ZooKeeperConnectionException, IOException {
 
     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
     // make it so if close or open, we return 'success'.
-    this.serverManager = Mockito.mock(ServerManager.class);
-    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
-    Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
+    this.serverManager = mockManager(SERVERNAME_A, SERVERNAME_B);
+  }
+
+  private ServerManager mockManager(ServerName... servers) throws IOException {
+    ServerManager serverManager = Mockito.mock(ServerManager.class);
     final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
-    onlineServers.put(SERVERNAME_B, new HServerLoad());
-    onlineServers.put(SERVERNAME_A, new HServerLoad());
-    Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
+    for (ServerName server : servers) {
+      Mockito.when(serverManager.isServerOnline(server)).thenReturn(true);
+      onlineServers.put(server, new HServerLoad());
+      Mockito.when(serverManager.sendRegionClose(server, REGIONINFO, -1)).thenReturn(true);
+      Mockito.when(serverManager.sendRegionOpen(server, REGIONINFO, -1)).
+          thenReturn(RegionOpeningState.OPENED);
+    }
+    Mockito.when(serverManager.getOnlineServersList()).thenReturn(
         new ArrayList<ServerName>(onlineServers.keySet()));
-    Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
-    Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
-      thenReturn(true);
-    Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
-      thenReturn(true);
-    // Ditto on open.
-    Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
-      thenReturn(RegionOpeningState.OPENED);
-    Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
-    thenReturn(RegionOpeningState.OPENED);
+    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
+    return serverManager;
   }
 
   @After
@@ -397,6 +396,35 @@ public void testBalance()
     }
   }
 
+  @Test
+  public void testGettingAssignmentsExcludesDrainingServers() throws Exception {
+    List<ServerName> availableServers =
+        Lists.newArrayList(SERVERNAME_A, SERVERNAME_B, SERVERNAME_C);
+    ServerManager serverManager = mockManager(availableServers.toArray(new ServerName[0]));
+
+
+    ExecutorService executor = startupMasterExecutor("testAssignmentsWithRSInDraining");
+    CatalogTracker ct = Mockito.mock(CatalogTracker.class);
+
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
+
+    Mockito.when(serverManager.getDrainingServersList()).thenReturn(
+        Lists.newArrayList(SERVERNAME_C));
+    AssignmentManager am = new AssignmentManager(this.server, serverManager, ct, balancer, executor);
+
+    for (ServerName availableServer : availableServers) {
+      HRegionInfo info = Mockito.mock(HRegionInfo.class);
+      Mockito.when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString());
+      am.regionOnline(info, availableServer);
+    }
+
+    Map<String, Map<ServerName, List<HRegionInfo>>> result = am.getAssignmentsByTable();
+    for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
+      System.out.println(map.keySet());
+      assertFalse(map.containsKey(SERVERNAME_C));
+    }
+  }
+
   /**
    * Run a simple server shutdown handler.
    * @throws KeeperException

From a0b77382477de4a3117709884bc3c5ee7c5edb2e Mon Sep 17 00:00:00 2001
From: Enis Soztutar <enis@apache.org>
Date: Thu, 17 Jul 2014 17:35:14 -0700
Subject: [PATCH 1510/1540] Commented out smart -p0 handling in
 dev-support/smart-apply-patch.sh until we fix our jenkins env

---
 dev-support/smart-apply-patch.sh | 54 +++++++++++++++++---------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
index 74a7128fb51c..cf1a7b06c319 100755
--- a/dev-support/smart-apply-patch.sh
+++ b/dev-support/smart-apply-patch.sh
@@ -46,32 +46,36 @@ if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
   # is adding new files and they would apply anywhere. So try to guess the
   # correct place to put those files.
 
-  TMP2=/tmp/tmp.paths.2.$$
-  TOCLEAN="$TOCLEAN $TMP2"
+#  NOTE 2014/07/17:
+#  Temporarily disabling below check since our jenkins boxes seems to be not defaulting to bash 
+#  causing below checks to fail. Once it is fixed, we can revert the commit and enable this again.
 
-  grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
-
-  #first off check that all of the files do not exist
-  FOUND_ANY=0
-  for CHECK_FILE in $(cat $TMP2)
-  do
-    if [[ -f $CHECK_FILE ]]; then
-      FOUND_ANY=1
-    fi
-  done
-
-  if [[ "$FOUND_ANY" = "0" ]]; then
-    #all of the files are new files so we have to guess where the correct place to put it is.
-
-    # if all of the lines start with a/ or b/, then this is a git patch that
-    # was generated without --no-prefix
-    if ! grep -qv '^a/\|^b/' $TMP2 ; then
-      echo Looks like this is a git patch. Stripping a/ and b/ prefixes
-      echo and incrementing PLEVEL
-      PLEVEL=$[$PLEVEL + 1]
-      sed -i -e 's,^[ab]/,,' $TMP2
-    fi
-  fi
+#  TMP2=/tmp/tmp.paths.2.$$
+#  TOCLEAN="$TOCLEAN $TMP2"
+#
+#  grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
+#
+#  #first off check that all of the files do not exist
+#  FOUND_ANY=0
+#  for CHECK_FILE in $(cat $TMP2)
+#  do
+#    if [[ -f $CHECK_FILE ]]; then
+#      FOUND_ANY=1
+#    fi
+#  done
+#
+#  if [[ "$FOUND_ANY" = "0" ]]; then
+#    #all of the files are new files so we have to guess where the correct place to put it is.
+#
+#    # if all of the lines start with a/ or b/, then this is a git patch that
+#    # was generated without --no-prefix
+#    if ! grep -qv '^a/\|^b/' $TMP2 ; then
+#      echo Looks like this is a git patch. Stripping a/ and b/ prefixes
+#      echo and incrementing PLEVEL
+#      PLEVEL=$[$PLEVEL + 1]
+#      sed -i -e 's,^[ab]/,,' $TMP2
+#    fi
+#  fi
 elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
   PLEVEL=1
 elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then

From 25d4d266f3c07e2af3663b8e806a0bb6b015522e Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Sat, 7 Feb 2015 23:39:08 -0800
Subject: [PATCH 1511/1540] HBASE-12968 [0.94]SecureServer should not ignore
 CallQueueSize. (hongyu bi)

---
 .../apache/hadoop/hbase/ipc/SecureServer.java  | 18 ++++++++++++++++--
 .../apache/hadoop/hbase/ipc/HBaseServer.java   |  4 ++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
index e6301250946a..c5fe8c1640c7 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java
@@ -612,15 +612,29 @@ protected void processData(byte[] buf) throws  IOException, InterruptedException
       DataInputStream dis =
         new DataInputStream(new ByteArrayInputStream(buf));
       int id = dis.readInt();                    // try to read an id
+      long callSize = buf.length;
 
       if (LOG.isTraceEnabled()) {
         LOG.trace(" got #" + id);
       }
 
+      // Enforcing the call queue size, this triggers a retry in the client
+      if ((callSize + callQueueSize.get()) > maxQueueSize) {
+        final SecureCall callTooBig =
+          new SecureCall(id, null, this, responder, callSize);
+        ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
+        setupResponse(responseBuffer, callTooBig, Status.FATAL, null,
+            IOException.class.getName(),
+            "Call queue is full, is ipc.server.max.callqueue.size too small?");
+        responder.doRespond(callTooBig);
+        return;
+      }
+
       Writable param = ReflectionUtils.newInstance(paramClass, conf);           // read param
       param.readFields(dis);
 
-      SecureCall call = new SecureCall(id, param, this, responder, buf.length);
+      SecureCall call = new SecureCall(id, param, this, responder, callSize);
+      callQueueSize.add(callSize);
 
       if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
         priorityCallQueue.put(call);
@@ -757,4 +771,4 @@ public void authorize(User user,
           protocol, getConf(), addr);
     }
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 261ce4a62d7f..417e2a248218 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -204,7 +204,7 @@ public static String getRemoteAddress() {
   protected Configuration conf;
 
   private int maxQueueLength;
-  private int maxQueueSize;
+  protected int maxQueueSize;
   protected int socketSendBufferSize;
   protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm
   protected final boolean tcpKeepAlive; // if T then use keepalives
@@ -1606,7 +1606,7 @@ protected Connection getConnection(SocketChannel channel, long time) {
    * @param error error message, if the call failed
    * @throws IOException
    */
-  private void setupResponse(ByteArrayOutputStream response,
+  protected void setupResponse(ByteArrayOutputStream response,
                              Call call, Status status,
                              Writable rv, String errorClass, String error)
   throws IOException {

From 0dcb39828586a43a9df0419c35e779e51bf229ac Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Wed, 11 Feb 2015 15:03:49 -0800
Subject: [PATCH 1512/1540] HBASE-13020 Add 'patchprocess/*' to RAT excludes on
 all branches

---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 3a72dbab6b5f..4b547c30bc46 100644
--- a/pom.xml
+++ b/pom.xml
@@ -535,6 +535,7 @@
               <!--  exclude source control files -->
               <exclude>.git/**</exclude>
               <exclude>.svn/**</exclude>
+              <exclude>**/patchprocess/**</exclude>
             </excludes>
           </configuration>
 	</plugin>

From 71be2f03606ba4a34df7a7605114c8451cf77adb Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Fri, 13 Feb 2015 12:35:26 -0800
Subject: [PATCH 1513/1540] HBASE-13039 Add patchprocess/* to .gitignore to fix
 builds of branches ( Adrey Stepachev)

Conflicts:
	.gitignore
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 225dd761a7b8..ded7f5cd729d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@
 *.iml
 *.orig
 *~
+patchprocess/

From ec8cfe14f1a558b5da1d1987d53cccf3ce0bd5f4 Mon Sep 17 00:00:00 2001
From: Sean Busbey <busbey@apache.org>
Date: Sat, 28 Feb 2015 19:58:30 -0600
Subject: [PATCH 1514/1540] HBASE-13131 ReplicationAdmin must clean up
 connections if constructor fails.

Conflicts:
	hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
---
 .../hbase/client/replication/ReplicationAdmin.java  | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index ac3bc81040ef..2a0f0caf2a36 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -99,8 +99,17 @@ public ReplicationAdmin(Configuration conf) throws IOException {
     ZooKeeperWatcher zkw = this.connection.getZooKeeperWatcher();
     try {
       this.replicationZk = new ReplicationZookeeper(this.connection, conf, zkw);
-    } catch (KeeperException e) {
-      throw new IOException("Unable setup the ZooKeeper connection", e);
+    } catch (Exception exception) {
+      if (connection != null) {
+        connection.close();
+      }
+      if (exception instanceof IOException) {
+        throw (IOException) exception;
+      } else if (exception instanceof RuntimeException) {
+        throw (RuntimeException) exception;
+      } else {
+        throw new IOException("Unable setup the ZooKeeper connection", exception);
+      }
     }
   }
 

From 9e6b65226a59aa13d470b6d1aaaae83eb7035ce9 Mon Sep 17 00:00:00 2001
From: Gustavo Anatoly <gustavoanatoly@gmail.com>
Date: Mon, 16 Mar 2015 23:45:01 -0300
Subject: [PATCH 1515/1540] Specify bash for local-regionservers.sh and
 local-master-backup.sh

Signed-off-by: Sean Busbey <busbey@apache.org>

Conflicts:
	src/main/asciidoc/_chapters/configuration.adoc
---
 bin/local-master-backup.sh | 2 +-
 bin/local-regionservers.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/local-master-backup.sh b/bin/local-master-backup.sh
index 2c0a4c02c76d..fc2efef1ce61 100755
--- a/bin/local-master-backup.sh
+++ b/bin/local-master-backup.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #/**
 # * Copyright 2007 The Apache Software Foundation
 # *
diff --git a/bin/local-regionservers.sh b/bin/local-regionservers.sh
index a4d5a1d93211..31e90cc37084 100755
--- a/bin/local-regionservers.sh
+++ b/bin/local-regionservers.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #/**
 # * Copyright 2007 The Apache Software Foundation
 # *

From ff01debea064aeeab3d980f8b5666d46d88e9f10 Mon Sep 17 00:00:00 2001
From: Sean Busbey <busbey@apache.org>
Date: Tue, 17 Mar 2015 12:52:42 -0500
Subject: [PATCH 1516/1540] Revert "Specify bash for local-regionservers.sh and
 local-master-backup.sh"

This reverts commit 9e6b65226a59aa13d470b6d1aaaae83eb7035ce9.
---
 bin/local-master-backup.sh | 2 +-
 bin/local-regionservers.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/local-master-backup.sh b/bin/local-master-backup.sh
index fc2efef1ce61..2c0a4c02c76d 100755
--- a/bin/local-master-backup.sh
+++ b/bin/local-master-backup.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #/**
 # * Copyright 2007 The Apache Software Foundation
 # *
diff --git a/bin/local-regionservers.sh b/bin/local-regionservers.sh
index 31e90cc37084..a4d5a1d93211 100755
--- a/bin/local-regionservers.sh
+++ b/bin/local-regionservers.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #/**
 # * Copyright 2007 The Apache Software Foundation
 # *

From 36c8f075f0b4c9e49f2fa3e2c7a80e0a921ac5a7 Mon Sep 17 00:00:00 2001
From: Gustavo Anatoly <gustavoanatoly@gmail.com>
Date: Mon, 16 Mar 2015 23:45:01 -0300
Subject: [PATCH 1517/1540] HBASE-13229 Specify bash for local-regionservers.sh
 and local-master-backup.sh

Signed-off-by: Sean Busbey <busbey@apache.org>

Conflicts:
	src/main/asciidoc/_chapters/configuration.adoc
---
 bin/local-master-backup.sh | 2 +-
 bin/local-regionservers.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/local-master-backup.sh b/bin/local-master-backup.sh
index 2c0a4c02c76d..fc2efef1ce61 100755
--- a/bin/local-master-backup.sh
+++ b/bin/local-master-backup.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #/**
 # * Copyright 2007 The Apache Software Foundation
 # *
diff --git a/bin/local-regionservers.sh b/bin/local-regionservers.sh
index a4d5a1d93211..31e90cc37084 100755
--- a/bin/local-regionservers.sh
+++ b/bin/local-regionservers.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #/**
 # * Copyright 2007 The Apache Software Foundation
 # *

From 459fc00ed8ae048efca737c90559a1b0e47bf300 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 18 Mar 2015 13:56:43 -0700
Subject: [PATCH 1518/1540] HBASE-11195 Potentially improve block locality
 during major compaction for old regions. (churro morales)

---
 .../hadoop/hbase/regionserver/Store.java      | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 46b4e528a017..1fdbdac316ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -106,7 +106,7 @@
  */
 public class Store extends SchemaConfigured implements HeapSize {
   static final Log LOG = LogFactory.getLog(Store.class);
-  
+
   public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";
   public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;
 
@@ -124,6 +124,7 @@ public class Store extends SchemaConfigured implements HeapSize {
   private final int maxFilesToCompact;
   private final long minCompactSize;
   private final long maxCompactSize;
+  private final float minStoreFileLocalitySkipCompact;
   private long lastCompactSize = 0;
   volatile boolean forceMajor = false;
   /* how many bytes to write between status checks */
@@ -241,6 +242,8 @@ protected Store(Path basedir, HRegion region, HColumnDescriptor family,
       this.region.memstoreFlushSize);
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
+    this.minStoreFileLocalitySkipCompact
+      = conf.getFloat("hbase.hstore.min.locality.to.skip.major.compact", 0f);
 
     this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
     
@@ -1326,17 +1329,31 @@ private boolean isMajorCompaction(final List<StoreFile> filesToCompact) throws I
             (sf.getReader().timeRangeTracker == null) ?
                 Long.MIN_VALUE :
                 now - sf.getReader().timeRangeTracker.minimumTimestamp;
-        if (sf.isMajorCompaction() &&
-            (this.ttl == HConstants.FOREVER || oldest < this.ttl)) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Skipping major compaction of " + this +
-                " because one (major) compacted file only and oldestTime " +
-                oldest + "ms is < ttl=" + this.ttl);
+        if (sf.isMajorCompaction() && (this.ttl == HConstants.FOREVER || oldest < this.ttl)) {
+          // if there is only one old store file, only compact if dfs blocks are not local.
+          float blockLocalityIndex = sf.getHDFSBlockDistribution().getBlockLocalityIndex(
+              region.getRegionServerServices().getServerName().getHostname()
+          );
+          if (blockLocalityIndex < minStoreFileLocalitySkipCompact) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Major compaction triggered on only store " + this +
+                      "; to make hdfs blocks local, current locality: " + blockLocalityIndex
+              );
+            }
+            result = true;
+          } else {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Skipping major compaction of " + this +
+                  " because one (major) compacted file only and oldestTime " +
+                  oldest + "ms is < ttl=" + this.ttl);
+            }
           }
         } else if (this.ttl != HConstants.FOREVER && oldest > this.ttl) {
-          LOG.debug("Major compaction triggered on store " + this +
-            ", because keyvalues outdated; time since last major compaction " +
-            (now - lowTimestamp) + "ms");
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Major compaction triggered on store " + this +
+                    ", because keyvalues outdated; time since last major compaction " +
+                    (now - lowTimestamp) + "ms");
+          }
           result = true;
         }
       } else {

From fa615825001d7979402999b206ce882d22489b07 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 18 Mar 2015 14:10:25 -0700
Subject: [PATCH 1519/1540] move to 0.94.27-SNAPSHOT

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 4b547c30bc46..9b73151ea263 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.26</version>
+  <version>0.94.27-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 71ce281a28aeba0a5a75b53ef7ffc8dd20566e6c Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 18 Mar 2015 14:18:31 -0700
Subject: [PATCH 1520/1540] pom and CHANGES for 0.94.27

---
 CHANGES.txt | 25 +++++++++++++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index eb9734f90654..2d6f60c24de3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,29 @@
 HBase Change Log
+Release 0.94.27 - 03/18/2015
+Sub-task
+
+    [HBASE-12776] - SpliTransaction: Log number of files to be split
+
+Bug
+
+    [HBASE-10528] - DefaultBalancer selects plans to move regions onto draining nodes
+    [HBASE-12792] - [backport] HBASE-5835: Catch and handle NotServingRegionException when close region attempt fails
+    [HBASE-12968] - [0.94]SecureServer should not ignore CallQueueSize
+    [HBASE-13039] - Add patchprocess/* to .gitignore to fix builds of branches
+    [HBASE-13131] - ReplicationAdmin leaks connections if there's an error in the constructor
+    [HBASE-13229] - Specify bash for local-regionservers.sh and local-master-backup.sh
+
+Improvement
+
+    [HBASE-11195] - Potentially improve block locality during major compaction for old regions
+    [HBASE-12223] - MultiTableInputFormatBase.getSplits is too slow
+    [HBASE-12720] - Make InternalScan LimitedPrivate
+
+Task
+
+    [HBASE-13020] - Add 'patchprocess/*' to RAT excludes on all branches
+
+
 Release 0.94.26 - 12/16/2014
 Bug
 
diff --git a/pom.xml b/pom.xml
index 9b73151ea263..f1683ceef357 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.27-SNAPSHOT</version>
+  <version>0.94.27</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 260f2137bdb8b4ae839f5cc285509f34e31a006b Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 18 Mar 2015 15:50:16 -0700
Subject: [PATCH 1521/1540] HBASE-11195 Addendum for TestHeapSize.

---
 src/main/java/org/apache/hadoop/hbase/regionserver/Store.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 1fdbdac316ed..7424788941f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -2487,7 +2487,7 @@ public CacheConfig getCacheConfig() {
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
           + (17 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG)
-          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN + Bytes.SIZEOF_FLOAT);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK

From fb434617716493eac82b55180b0bbd653beb90bf Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 18 Mar 2015 23:12:31 -0700
Subject: [PATCH 1522/1540] missed one misassigned issue in CHANGES.txt

---
 CHANGES.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index 2d6f60c24de3..3d368f11361e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@ Bug
 
     [HBASE-10528] - DefaultBalancer selects plans to move regions onto draining nodes
     [HBASE-12792] - [backport] HBASE-5835: Catch and handle NotServingRegionException when close region attempt fails
+    [HBASE-12801] - Failed to truncate a table while maintaing binary region boundaries
     [HBASE-12968] - [0.94]SecureServer should not ignore CallQueueSize
     [HBASE-13039] - Add patchprocess/* to .gitignore to fix builds of branches
     [HBASE-13131] - ReplicationAdmin leaks connections if there's an error in the constructor

From ca254ed51514d2f14fc1f751778fb82df423aab8 Mon Sep 17 00:00:00 2001
From: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Date: Fri, 15 May 2015 19:09:40 +0100
Subject: [PATCH 1523/1540] HBASE-13651 Handle StoreFileScanner
 FileNotFoundExceptin

---
 .../hadoop/hbase/regionserver/HRegion.java    |  37 ++-
 .../hbase/regionserver/HRegionServer.java     |   4 +
 .../hadoop/hbase/regionserver/Store.java      |   1 +
 .../hbase/regionserver/StoreFileScanner.java  |   7 +
 .../TestCorruptedRegionStoreFile.java         | 244 ++++++++++++++++++
 5 files changed, 285 insertions(+), 8 deletions(-)
 create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 5da00dc9704d..8f2037644ad0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4024,7 +4024,13 @@ public HRegionInfo getRegionInfo() {
       for (Map.Entry<byte[], NavigableSet<byte[]>> entry :
           scan.getFamilyMap().entrySet()) {
         Store store = stores.get(entry.getKey());
-        KeyValueScanner scanner = store.getScanner(scan, entry.getValue());
+        KeyValueScanner scanner;
+        try {
+          scanner = store.getScanner(scan, entry.getValue());
+        } catch (FileNotFoundException e) {
+          abortRegionServer(e.getMessage());
+          throw new NotServingRegionException(regionInfo.getRegionNameAsString() + " is closing");
+        }
         if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand()
           || FilterBase.isFamilyEssential(this.filter, entry.getKey())) {
           scanners.add(scanner);
@@ -4152,13 +4158,18 @@ private void populateFromJoinedHeap(List<KeyValue> results, int limit, String me
     private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int limit,
         byte[] currentRow, int offset, short length, String metric) throws IOException {
       KeyValue nextKv;
-      do {
-        heap.next(results, limit - results.size(), metric);
-        if (limit > 0 && results.size() == limit) {
-          return KV_LIMIT;
-        }
-        nextKv = heap.peek();
-      } while (nextKv != null && nextKv.matchingRow(currentRow, offset, length));
+      try {
+        do {
+          heap.next(results, limit - results.size(), metric);
+          if (limit > 0 && results.size() == limit) {
+            return KV_LIMIT;
+          }
+          nextKv = heap.peek();
+        } while (nextKv != null && nextKv.matchingRow(currentRow, offset, length));
+      } catch (FileNotFoundException e) {
+        abortRegionServer(e.getMessage());
+        throw new NotServingRegionException(regionInfo.getRegionNameAsString() + " is closing");
+      }
       return nextKv;
     }
 
@@ -4368,6 +4379,9 @@ public synchronized boolean reseek(byte[] row) throws IOException {
         if (this.joinedHeap != null) {
           result = this.joinedHeap.requestSeek(kv, true, true) || result;
         }
+      } catch (FileNotFoundException e) {
+        abortRegionServer(e.getMessage());
+        throw new NotServingRegionException(regionInfo.getRegionNameAsString() + " is closing");
       } finally {
         closeRegionOperation();
       }
@@ -6034,6 +6048,13 @@ public void closeRegionOperation(){
     lock.readLock().unlock();
   }
 
+  public void abortRegionServer(String msg) throws IOException {
+    RegionServerServices rs = getRegionServerServices();
+    if (rs instanceof HRegionServer) {
+      ((HRegionServer)rs).abort(msg);
+    }
+  }
+
   /**
    * This method needs to be called before any public call that reads or
    * modifies stores in bulk. It has to be called just before a try.
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index eadc9e8022d4..5499135fd468 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3446,6 +3446,10 @@ public Configuration getConfiguration() {
     return conf;
   }
 
+  public CacheConfig getCacheConfig() {
+    return cacheConfig;
+  }
+
   /** @return the write lock for the server */
   ReentrantReadWriteLock.WriteLock getWriteLock() {
     return lock.writeLock();
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 7424788941f0..3bd9f1173842 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index b0556309a7ab..1594a0aa9aee 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -20,6 +20,7 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -129,6 +130,8 @@ public KeyValue next() throws IOException {
         if (hasMVCCInfo)
           skipKVsNewerThanReadpoint();
       }
+    } catch (FileNotFoundException e) {
+      throw e;
     } catch(IOException e) {
       throw new IOException("Could not iterate " + this, e);
     }
@@ -151,6 +154,8 @@ public boolean seek(KeyValue key) throws IOException {
       } finally {
         realSeekDone = true;
       }
+    } catch (FileNotFoundException e) {
+      throw e;
     } catch (IOException ioe) {
       throw new IOException("Could not seek " + this + " to key " + key, ioe);
     }
@@ -171,6 +176,8 @@ public boolean reseek(KeyValue key) throws IOException {
       } finally {
         realSeekDone = true;
       }
+    } catch (FileNotFoundException e) {
+      throw e;
     } catch (IOException ioe) {
       throw new IOException("Could not reseek " + this + " to key " + key,
           ioe);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
new file mode 100644
index 000000000000..c4e0a7dae902
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSVisitor;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestCorruptedRegionStoreFile {
+  private static final Log LOG = LogFactory.getLog(TestCorruptedRegionStoreFile.class);
+
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  private static final String FAMILY_NAME_STR = "f";
+  private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
+
+  private static final int NUM_FILES = 25;
+  private static final int ROW_PER_FILE = 2000;
+  private static final int NUM_ROWS = NUM_FILES * ROW_PER_FILE;
+
+  private final ArrayList<Path> storeFiles = new ArrayList<Path>();
+  private Path tableDir;
+  private int rowCount;
+
+  private static void setupConf(Configuration conf) {
+    conf.setLong("hbase.hstore.compaction.min", 20);
+    conf.setLong("hbase.hstore.compaction.max", 39);
+    conf.setLong("hbase.hstore.blockingStoreFiles", 40);
+  }
+
+  private void setupTable(final String tableName) throws Exception {
+    // load the table
+    HTable table = UTIL.createTable(Bytes.toBytes(tableName), FAMILY_NAME);
+    try {
+      rowCount = 0;
+      byte[] value = new byte[1024];
+      byte[] q = Bytes.toBytes("q");
+      while (rowCount < NUM_ROWS) {
+        Put put = new Put(Bytes.toBytes(String.format("%010d", rowCount)));
+        put.setDurability(Durability.SKIP_WAL);
+        put.add(FAMILY_NAME, q, value);
+        table.put(put);
+
+        if ((rowCount++ % ROW_PER_FILE) == 0) {
+          // flush it
+          UTIL.getHBaseAdmin().flush(tableName);
+        }
+      }
+    } finally {
+      UTIL.getHBaseAdmin().flush(tableName);
+      table.close();
+    }
+
+    assertEquals(NUM_ROWS, rowCount);
+
+    // get the store file paths
+    storeFiles.clear();
+    tableDir = FSUtils.getTablePath(getRootDir(), tableName);
+    FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {
+      @Override
+      public void storeFile(final String region, final String family, final String hfile)
+          throws IOException {
+        HFileLink link = HFileLink.create(UTIL.getConfiguration(), tableName, region, family, hfile);
+        storeFiles.add(link.getOriginPath());
+      }
+    });
+    assertTrue("expected at least 1 store file", storeFiles.size() > 0);
+    LOG.info("store-files: " + storeFiles);
+  }
+
+  @Before
+  public void setup() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster(2, 3);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    try {
+      UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      LOG.warn("failure shutting down cluster", e);
+    }
+  }
+
+  @Test(timeout=90000)
+  public void testLosingFileDuringScan() throws Exception {
+    final String tableName = "testLosingFileDuringScan";
+    setupTable(tableName);
+    assertEquals(rowCount, fullScanAndCount(tableName));
+
+    final FileSystem fs = getFileSystem();
+    final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
+
+    // try to query with the missing file
+    int count = fullScanAndCount(tableName, new ScanInjector() {
+      private boolean hasFile = true;
+
+      @Override
+      public void beforeScanNext(HTable table) throws Exception {
+        // move the path away (now the region is corrupted)
+        if (hasFile) {
+          fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
+          LOG.info("Move file to local");
+          evictHFileCache(storeFiles.get(0));
+          hasFile = false;
+        }
+      }
+    });
+    assertTrue("expected one file lost: rowCount=" + count, count >= (NUM_ROWS - ROW_PER_FILE));
+  }
+
+  @Test(timeout=90000)
+  public void testLosingFileAfterScannerInit() throws Exception {
+    final String tableName = "testLosingFileAfterScannerInit";
+    setupTable(tableName);
+    assertEquals(rowCount, fullScanAndCount(tableName));
+
+    final FileSystem fs = getFileSystem();
+    final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
+
+    // try to query with the missing file
+    int count = fullScanAndCount(tableName, new ScanInjector() {
+      private boolean hasFile = true;
+
+      @Override
+      public void beforeScan(HTable table, Scan scan) throws Exception {
+        // move the path away (now the region is corrupted)
+        if (hasFile) {
+          fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
+          LOG.info("Move file to local");
+          evictHFileCache(storeFiles.get(0));
+          hasFile = false;
+        }
+      }
+    });
+    assertTrue("expected one file lost: rowCount=" + count, count >= (NUM_ROWS - ROW_PER_FILE));
+  }
+
+  // ==========================================================================
+  //  Helpers
+  // ==========================================================================
+  private FileSystem getFileSystem() {
+    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+  }
+
+  private Path getRootDir() {
+    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+  }
+
+  private void evictHFileCache(final Path hfile) throws Exception {
+    for (RegionServerThread rst: UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
+      HRegionServer rs = rst.getRegionServer();
+      rs.getCacheConfig().getBlockCache().evictBlocksByHfileName(hfile.getName());
+    }
+    Thread.sleep(6000);
+  }
+
+  private int fullScanAndCount(final String tableName) throws Exception {
+    return fullScanAndCount(tableName, new ScanInjector());
+  }
+
+  private int fullScanAndCount(final String tableName, final ScanInjector injector)
+      throws Exception {
+    HTable table = new HTable(UTIL.getConfiguration(), tableName);
+    int count = 0;
+    try {
+      Scan scan = new Scan();
+      scan.setCaching(1);
+      scan.setCacheBlocks(false);
+      injector.beforeScan(table, scan);
+      ResultScanner scanner = table.getScanner(scan);
+      try {
+        while (true) {
+          injector.beforeScanNext(table);
+          Result result = scanner.next();
+          injector.afterScanNext(table, result);
+          if (result == null) break;
+          if ((count++ % 1000) == 0) {
+            LOG.debug("scan next " + count);
+          }
+        }
+      } finally {
+        scanner.close();
+        injector.afterScan(table);
+      }
+    } finally {
+      table.close();
+    }
+    return count;
+  }
+
+  private class ScanInjector {
+    protected void beforeScan(HTable table, Scan scan) throws Exception {}
+    protected void beforeScanNext(HTable table) throws Exception {}
+    protected void afterScanNext(HTable table, Result result) throws Exception {}
+    protected void afterScan(HTable table) throws Exception {}
+  }
+}

From e5a433802c486abae89aa6e4bb1159bbff4d3d9f Mon Sep 17 00:00:00 2001
From: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Date: Fri, 15 May 2015 20:38:18 +0100
Subject: [PATCH 1524/1540] HBASE-13651 Handle StoreFileScanner
 FileNotFoundException (addendum)

---
 .../hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
index c4e0a7dae902..c35a056dfa36 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -51,6 +52,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+@Category(LargeTests.class)
 public class TestCorruptedRegionStoreFile {
   private static final Log LOG = LogFactory.getLog(TestCorruptedRegionStoreFile.class);
 

From c93953894a5224534b00675df8b1f8bd804ee938 Mon Sep 17 00:00:00 2001
From: Matt Warhaftig <mwarhaftig@gmail.com>
Date: Sun, 24 May 2015 18:49:25 -0400
Subject: [PATCH 1525/1540] HBASE-13344 - Add max Java version check for pom
 dependencies.

Signed-off-by: Sean Busbey <busbey@apache.org>

Conflicts:
	pom.xml
---
 pom.xml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/pom.xml b/pom.xml
index f1683ceef357..53cb95ddfcaf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -595,6 +595,13 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>extra-enforcer-rules</artifactId>
+            <version>${extra.enforcer.version}</version>
+          </dependency>
+        </dependencies>
         <!-- version set by parent -->
         <configuration>
           <rules>
@@ -618,6 +625,16 @@
   See the reference guide on building for more information: http://hbase.apache.org/book.html#build
               </message>
             </requireJavaVersion>
+            <enforceBytecodeVersion>
+              <maxJdkVersion>${compileSource}</maxJdkVersion>
+              <message>HBase has unsupported dependencies.
+  HBase requires that all dependencies be compiled with version ${compileSource} or earlier
+  of the JDK to properly build from source.  You appear to be using a newer dependency. You can use
+  either "mvn -version" or "mvn enforcer:display-info" to verify what version is active.
+  Non-release builds can temporarily build with a newer JDK version by setting the
+  'compileSource' property (eg. mvn -DcompileSource=1.8 clean package).
+              </message>
+            </enforceBytecodeVersion>
           </rules>
         </configuration>
         <executions>
@@ -1132,6 +1149,7 @@
     <surefire.secondPartGroups>org.apache.hadoop.hbase.MediumTests</surefire.secondPartGroups>
     <test.output.tofile>true</test.output.tofile>
     <surefire.timeout>900</surefire.timeout>
+    <extra.enforcer.version>1.0-beta-3</extra.enforcer.version>
   </properties>
 
   <!-- Sorted by groups of dependencies then groupId and artifactId -->

From c9e2d36b2a313f36005e3de064baa284ff288b2b Mon Sep 17 00:00:00 2001
From: Matt Warhaftig <mwarhaftig@gmail.com>
Date: Sun, 31 May 2015 11:17:23 -0400
Subject: [PATCH 1526/1540] HBASE-13344 Addendum - Move max Java version check
 to release.

Amending-Author: Sean Busbey <busbey@apache.org>
Signed-off-by: Sean Busbey <busbey@apache.org>
---
 pom.xml | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/pom.xml b/pom.xml
index 53cb95ddfcaf..0a6276bd69de 100644
--- a/pom.xml
+++ b/pom.xml
@@ -595,6 +595,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>
+        <version>1.0.1</version>
         <dependencies>
           <dependency>
             <groupId>org.codehaus.mojo</groupId>
@@ -625,16 +626,6 @@
   See the reference guide on building for more information: http://hbase.apache.org/book.html#build
               </message>
             </requireJavaVersion>
-            <enforceBytecodeVersion>
-              <maxJdkVersion>${compileSource}</maxJdkVersion>
-              <message>HBase has unsupported dependencies.
-  HBase requires that all dependencies be compiled with version ${compileSource} or earlier
-  of the JDK to properly build from source.  You appear to be using a newer dependency. You can use
-  either "mvn -version" or "mvn enforcer:display-info" to verify what version is active.
-  Non-release builds can temporarily build with a newer JDK version by setting the
-  'compileSource' property (eg. mvn -DcompileSource=1.8 clean package).
-              </message>
-            </enforceBytecodeVersion>
           </rules>
         </configuration>
         <executions>
@@ -1601,6 +1592,24 @@
               </execution>
             </executions>
           </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <configuration>
+              <rules>
+                <enforceBytecodeVersion>
+                <maxJdkVersion>${compileSource}</maxJdkVersion>
+                <message>HBase has unsupported dependencies.
+  HBase requires that all dependencies be compiled with version ${compileSource} or earlier
+  of the JDK to properly build from source.  You appear to be using a newer dependency. You can use
+  either "mvn -version" or "mvn enforcer:display-info" to verify what version is active.
+  Non-release builds can temporarily build with a newer JDK version by setting the
+  'compileSource' property (eg. mvn -DcompileSource=1.8 clean package).
+                </message>
+              </enforceBytecodeVersion>
+            </rules>
+          </configuration>
+          </plugin>
           <plugin>
             <artifactId>maven-javadoc-plugin</artifactId>
             <version>2.6.1</version>

From b2fb5b7849966d0c481f416cc847eaada80bafda Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 16 Jun 2015 13:03:46 -0700
Subject: [PATCH 1527/1540] HBASE-13454 SecureClient#setupIOStreams should
 handle all Exceptions. (cuijianwei)

---
 .../java/org/apache/hadoop/hbase/ipc/SecureClient.java   | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 38ad5aa58bdd..721e4628b61b 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -325,7 +325,14 @@ public Boolean run() throws IOException {
           start();
           return;
         }
-      } catch (IOException e) {
+      } catch (Throwable t) {
+        failedServers.addToFailedServers(remoteId.address);
+        IOException e;
+        if (t instanceof IOException) {
+          e = (IOException)t;
+        } else {
+          e = new IOException("Could not set up Secure IO Streams", t);
+        }
         markClosed(e);
         close();
 

From 4a1d46487cd83bb6f62b6367f688fbb44bbc6f82 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 16 Jun 2015 13:06:50 -0700
Subject: [PATCH 1528/1540] HBASE-12921 Port HBASE-5356 'region_mover.rb can
 hang if table region it belongs to is deleted' to 0.94. (Liu Shaohui)

---
 bin/region_mover.rb | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 58761635dd12..dc28331deef8 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -90,8 +90,7 @@ def closeTables()
 # Returns true if passed region is still on 'original' when we look at .META.
 def isSameServer(admin, r, original)
   server = getServerNameForRegion(admin, r)
-  return false unless server
-  return true unless original
+  return false unless server and original
   return server == original
 end
 
@@ -105,6 +104,7 @@ def abort(why, e)
 # Get servername that is up in .META.; this is hostname + port + startcode comma-delimited.
 # Can return nil
 def getServerNameForRegion(admin, r)
+  return nil unless admin.isTableEnabled(r.getTableName)
   if r.isRootRegion()
     # Hack
     tracker = org.apache.hadoop.hbase.zookeeper.RootRegionTracker.new(admin.getConnection().getZooKeeperWatcher(), RubyAbortable.new())
@@ -127,6 +127,7 @@ def getServerNameForRegion(admin, r)
   g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER)
   g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER)
   result = table.get(g)
+  return nil unless result
   server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER)
   startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER)
   return nil unless server
@@ -140,8 +141,15 @@ def isSuccessfulScan(admin, r)
   scan.setBatch(1)
   scan.setCaching(1)
   scan.setFilter(FirstKeyOnlyFilter.new()) 
-  table = getTable(admin.getConfiguration(), r.getTableName()) 
-  scanner = table.getScanner(scan)
+  begin
+    table = getTable(admin.getConfiguration(), r.getTableName())
+    scanner = table.getScanner(scan)
+  rescue org.apache.hadoop.hbase.TableNotFoundException,
+      org.apache.hadoop.hbase.TableNotEnabledException => e
+    $LOG.warn("Region " + r.getEncodedName() + " belongs to recently " +
+      "deleted/disabled table. Skipping... " + e.message)
+    return
+  end
   begin
     results = scanner.next() 
     # We might scan into next region, this might be an empty table.
@@ -171,7 +179,8 @@ def move(admin, r, newServer, original)
     count = count + 1
     begin
       admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer))
-    rescue java.lang.reflect.UndeclaredThrowableException => e
+    rescue java.lang.reflect.UndeclaredThrowableException,
+        org.apache.hadoop.hbase.UnknownRegionException => e
       $LOG.info("Exception moving "  + r.getEncodedName() +
         "; split/moved? Continuing: " + e)
       return
@@ -342,6 +351,8 @@ def unloadRegions(options, hostname)
   movedRegions = java.util.ArrayList.new()
   while true
     rs = getRegions(config, servername)
+    # Remove those already tried to move
+    rs.removeAll(movedRegions)
     break if rs.length == 0
     count = 0
     $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername +

From 6c74c79b2828ca1f507e92c42596f89cd0120b3c Mon Sep 17 00:00:00 2001
From: Misty Stanley-Jones <mstanleyjones@cloudera.com>
Date: Tue, 3 Nov 2015 11:44:18 +1000
Subject: [PATCH 1529/1540] HBASE-14747 Update pom.xml to build Javadocs and
 xref reports for HBase 0.94

---
 pom.xml | 248 +++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 193 insertions(+), 55 deletions(-)

diff --git a/pom.xml b/pom.xml
index 0a6276bd69de..fa45138bb90b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1100,6 +1100,7 @@
     <jersey.version>1.8</jersey.version>
     <jruby.version>1.6.5</jruby.version>
     <junit.version>4.11</junit.version>
+    <hamcrest.version>1.3</hamcrest.version>
     <slf4j.version>1.4.3</slf4j.version>
     <log4j.version>1.2.16</log4j.version>
     <mockito-all.version>1.8.5</mockito-all.version>
@@ -2606,76 +2607,213 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
 
       </plugin>
       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>
-        <version>2.6.1</version>
-        <configuration>
-          <docfilessubdirs>true</docfilessubdirs>
-        </configuration>
+        <version>2.10.3</version>
         <reportSets>
+          <!-- Dev API -->
+          <reportSet>
+            <id>devapi</id>
+            <reports>
+              <report>aggregate</report>
+              <report>test-aggregate</report>
+            </reports>
+            <configuration>
+              <destDir>devapidocs</destDir>
+              <name>Developer API</name>
+              <description>The full HBase API, including private and unstable APIs</description>
+              <sourceFileExcludes>
+                <exclude>**/generated/*</exclude>
+                <exclude>**/protobuf/*</exclude>
+                <exclude>**/*.scala</exclude>
+              </sourceFileExcludes>
+              <excludePackageNames>*.generated.master:*.generated:org.apache.hadoop.hbase.tmpl.common:com.google.protobuf:org.apache.hadoop.hbase.spark</excludePackageNames>
+              <quiet>true</quiet>
+              <linksource>true</linksource>
+              <sourcetab>2</sourcetab>
+              <validateLinks>true</validateLinks>
+              <fixClassComment>true</fixClassComment>
+              <fixFieldComment>true</fixFieldComment>
+              <fixMethodComment>true</fixMethodComment>
+              <fixTags>all</fixTags>
+              <notimestamp>true</notimestamp>
+              <!-- Pass some options straight to the javadoc executable since it is easier -->
+              <additionalJOption>-J-Xmx2G</additionalJOption>
+              <!-- JDK8 javadoc requires test scope transitive dependencies due to our custom doclet -->
+              <additionalDependencies>
+                <additionalDependency>
+                  <groupId>org.mockito</groupId>
+                  <artifactId>mockito-all</artifactId>
+                  <version>${mockito-all.version}</version>
+                </additionalDependency>
+                <additionalDependency>
+                  <groupId>org.hamcrest</groupId>
+                  <artifactId>hamcrest-core</artifactId>
+                  <version>${hamcrest.version}</version>
+                </additionalDependency>
+              </additionalDependencies>
+              <inherited>false</inherited>
+            </configuration>
+          </reportSet>
+          <reportSet>
+            <id>testdevapi</id>
+            <reports>
+              <report>test-aggregate</report>
+            </reports>
+            <configuration>
+              <destDir>testdevapidocs</destDir>
+              <name>Developer API</name>
+              <description>The full HBase API, including private and unstable APIs</description>
+              <sourceFileExcludes>
+                <exclude>**/generated/*</exclude>
+                <exclude>**/protobuf/*</exclude>
+                <exclude>**/*.scala</exclude>
+              </sourceFileExcludes>
+              <excludePackageNames>*.generated.master:*.generated:org.apache.hadoop.hbase.tmpl.common:com.google.protobuf:org.apache.hadoop.hbase.spark</excludePackageNames>
+              <quiet>true</quiet>
+              <linksource>true</linksource>
+              <sourcetab>2</sourcetab>
+              <validateLinks>true</validateLinks>
+              <fixClassComment>true</fixClassComment>
+              <fixFieldComment>true</fixFieldComment>
+              <fixMethodComment>true</fixMethodComment>
+              <fixTags>all</fixTags>
+              <notimestamp>true</notimestamp>
+              <!-- Pass some options straight to the javadoc executable since it is easier -->
+              <additionalJOption>-J-Xmx2G</additionalJOption>
+              <!-- JDK8 javadoc requires test scope transitive dependencies due to our custom doclet -->
+              <additionalDependencies>
+                <additionalDependency>
+                  <groupId>org.mockito</groupId>
+                  <artifactId>mockito-all</artifactId>
+                  <version>${mockito-all.version}</version>
+                </additionalDependency>
+                <additionalDependency>
+                  <groupId>org.hamcrest</groupId>
+                  <artifactId>hamcrest-core</artifactId>
+                  <version>${hamcrest.version}</version>
+                </additionalDependency>
+              </additionalDependencies>
+              <inherited>false</inherited>
+            </configuration>
+          </reportSet>
+
+          <!-- User API -->
           <reportSet>
-            <id>default</id>
+            <id>userapi</id>
             <reports>
-              <report>javadoc</report>
+              <report>aggregate</report>
             </reports>
+            <configuration>
+              <destDir>apidocs</destDir>
+              <name>User API</name>
+              <description>The HBase Application Programmer's API</description>
+              <excludePackageNames>
+                org.apache.hadoop.hbase.backup*:org.apache.hadoop.hbase.catalog:org.apache.hadoop.hbase.client.coprocessor:org.apache.hadoop.hbase.client.metrics:org.apache.hadoop.hbase.codec*:org.apache.hadoop.hbase.constraint:org.apache.hadoop.hbase.coprocessor.*:org.apache.hadoop.hbase.executor:org.apache.hadoop.hbase.fs:*.generated.*:org.apache.hadoop.hbase.io.hfile.*:org.apache.hadoop.hbase.mapreduce.hadoopbackport:org.apache.hadoop.hbase.mapreduce.replication:org.apache.hadoop.hbase.master.*:org.apache.hadoop.hbase.metrics*:org.apache.hadoop.hbase.migration:org.apache.hadoop.hbase.monitoring:org.apache.hadoop.hbase.p*:org.apache.hadoop.hbase.regionserver.compactions:org.apache.hadoop.hbase.regionserver.handler:org.apache.hadoop.hbase.regionserver.snapshot:org.apache.hadoop.hbase.replication.*:org.apache.hadoop.hbase.rest.filter:org.apache.hadoop.hbase.rest.model:org.apache.hadoop.hbase.rest.p*:org.apache.hadoop.hbase.security.*:org.apache.hadoop.hbase.thrift*:org.apache.hadoop.hbase.tmpl.*:org.apache.hadoop.hbase.tool:org.apache.hadoop.hbase.trace:org.apache.hadoop.hbase.util.byterange*:org.apache.hadoop.hbase.util.test:org.apache.hadoop.hbase.util.vint:org.apache.hadoop.hbase.zookeeper.lock:org.apache.hadoop.metrics2*
+              </excludePackageNames>
+              <!-- switch on dependency-driven aggregation -->
+              <includeDependencySources>false</includeDependencySources>
+              <dependencySourceIncludes>
+                <!-- include ONLY dependencies I control -->
+                <dependencySourceInclude>org.apache.hbase:hbase-annotations</dependencySourceInclude>
+                </dependencySourceIncludes>
+              <outputDirectory>${project.reporting.outputDirectory}/devapidocs</outputDirectory>
+              <name>Developer API</name>
+              <description>The full HBase API, including private and unstable APIs</description>
+              <sourceFilesExclude>**/generated/*</sourceFilesExclude>
+              <excludePackageNames>org.apache.hadoop.hbase.generated.master:org.apache.hadoop.hbase.protobuf.generated:org.apache.hadoop.hbase.tmpl.common</excludePackageNames>
+              <quiet>true</quiet>
+              <linksource>true</linksource>
+              <sourcetab>2</sourcetab>
+              <validateLinks>true</validateLinks>
+              <fixClassComment>true</fixClassComment>
+              <fixFieldComment>true</fixFieldComment>
+              <fixMethodComment>true</fixMethodComment>
+              <fixTags>all</fixTags>
+              <notimestamp>true</notimestamp>
+              <!-- Pass some options straight to the javadoc executable since it is easier -->
+              <additionalJOption>-J-Xmx2G</additionalJOption>
+              <!-- JDK8 javadoc requires test scope transitive dependencies due to our custom doclet -->
+              <additionalDependencies>
+                <additionalDependency>
+                  <groupId>org.mockito</groupId>
+                  <artifactId>mockito-all</artifactId>
+                  <version>${mockito-all.version}</version>
+                </additionalDependency>
+                <additionalDependency>
+                  <groupId>org.hamcrest</groupId>
+                  <artifactId>hamcrest-core</artifactId>
+                  <version>${hamcrest.version}</version>
+                </additionalDependency>
+              </additionalDependencies>
+              <inherited>false</inherited>
+            </configuration>
+          </reportSet>
+          <reportSet>
+            <id>testuserapi</id>
+            <reports>
+              <report>test-aggregate</report>
+            </reports>
+            <configuration>
+              <destDir>testapidocs</destDir>
+              <name>User API</name>
+              <description>The HBase Application Programmer's API</description>
+              <excludePackageNames>
+                org.apache.hadoop.hbase.backup*:org.apache.hadoop.hbase.catalog:org.apache.hadoop.hbase.client.coprocessor:org.apache.hadoop.hbase.client.metrics:org.apache.hadoop.hbase.codec*:org.apache.hadoop.hbase.constraint:org.apache.hadoop.hbase.coprocessor.*:org.apache.hadoop.hbase.executor:org.apache.hadoop.hbase.fs:*.generated.*:org.apache.hadoop.hbase.io.hfile.*:org.apache.hadoop.hbase.mapreduce.hadoopbackport:org.apache.hadoop.hbase.mapreduce.replication:org.apache.hadoop.hbase.master.*:org.apache.hadoop.hbase.metrics*:org.apache.hadoop.hbase.migration:org.apache.hadoop.hbase.monitoring:org.apache.hadoop.hbase.p*:org.apache.hadoop.hbase.regionserver.compactions:org.apache.hadoop.hbase.regionserver.handler:org.apache.hadoop.hbase.regionserver.snapshot:org.apache.hadoop.hbase.replication.*:org.apache.hadoop.hbase.rest.filter:org.apache.hadoop.hbase.rest.model:org.apache.hadoop.hbase.rest.p*:org.apache.hadoop.hbase.security.*:org.apache.hadoop.hbase.thrift*:org.apache.hadoop.hbase.tmpl.*:org.apache.hadoop.hbase.tool:org.apache.hadoop.hbase.trace:org.apache.hadoop.hbase.util.byterange*:org.apache.hadoop.hbase.util.test:org.apache.hadoop.hbase.util.vint:org.apache.hadoop.hbase.zookeeper.lock:org.apache.hadoop.metrics2*
+              </excludePackageNames>
+              <!-- switch on dependency-driven aggregation -->
+              <includeDependencySources>false</includeDependencySources>
+              <dependencySourceIncludes>
+                <!-- include ONLY dependencies I control -->
+                <dependencySourceInclude>org.apache.hbase:hbase-annotations</dependencySourceInclude>
+              </dependencySourceIncludes>
+              <outputDirectory>${project.reporting.outputDirectory}/devapidocs</outputDirectory>
+              <name>Developer API</name>
+              <description>The full HBase API, including private and unstable APIs</description>
+              <sourceFilesExclude>**/generated/*</sourceFilesExclude>
+              <excludePackageNames>org.apache.hadoop.hbase.generated.master:org.apache.hadoop.hbase.protobuf.generated:org.apache.hadoop.hbase.tmpl.common</excludePackageNames>
+              <quiet>true</quiet>
+              <linksource>true</linksource>
+              <sourcetab>2</sourcetab>
+              <validateLinks>true</validateLinks>
+              <fixClassComment>true</fixClassComment>
+              <fixFieldComment>true</fixFieldComment>
+              <fixMethodComment>true</fixMethodComment>
+              <fixTags>all</fixTags>
+              <notimestamp>true</notimestamp>
+              <!-- Pass some options straight to the javadoc executable since it is easier -->
+              <additionalJOption>-J-Xmx2G</additionalJOption>
+              <!-- JDK8 javadoc requires test scope transitive dependencies due to our custom doclet -->
+              <additionalDependencies>
+                <additionalDependency>
+                  <groupId>org.mockito</groupId>
+                  <artifactId>mockito-all</artifactId>
+                  <version>${mockito-all.version}</version>
+                </additionalDependency>
+                <additionalDependency>
+                  <groupId>org.hamcrest</groupId>
+                  <artifactId>hamcrest-core</artifactId>
+                  <version>${hamcrest.version}</version>
+                </additionalDependency>
+              </additionalDependencies>
+              <inherited>false</inherited>
+            </configuration>
           </reportSet>
         </reportSets>
-        <!--
-          This is probably not needed, given the smallness of the HBase source code, but left here in case
-          <minmemory>128m</minmemory>
-          <maxmemory>1024m</maxmemory>
-        -->
       </plugin>
-      <!--Disabled for now.
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>cobertura-maven-plugin</artifactId>
-        <version>2.3</version>
-      </plugin>
-      -->
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jxr-plugin</artifactId>
-        <version>2.1</version>
-      </plugin>
-      <!-- Disabled for now
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>jdepend-maven-plugin</artifactId>
-        <version>2.0-beta-2</version>
-      </plugin>
-      <plugin>
-        <artifactId>maven-changes-plugin</artifactId>
         <version>2.3</version>
         <configuration>
-          <issueLinkTemplate>%URL%/browse/%ISSUE%</issueLinkTemplate>
+          <aggregate>true</aggregate>
+          <test-aggregate>true</test-aggregate>
+          <linkJavadoc>true</linkJavadoc>
+          <javadocDir>${project.reporting.outputDirectory}/devapidocs</javadocDir>
+          <testJavadocDir>${project.reporting.outputDirectory}/testdevapidocs</testJavadocDir>
+          <destDir>${project.reporting.outputDirectory}/xref</destDir>
         </configuration>
-        <reportSets>
-          <reportSet>
-            <reports>
-              <report>changes-report</report>
-            </reports>
-          </reportSet>
-        </reportSets>
-      </plugin>
-      <plugin>
-        <groupId>com.atlassian.maven.plugins</groupId>
-        <artifactId>maven-clover2-plugin</artifactId>
-        <version>2.6.3</version>
-      </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>taglist-maven-plugin</artifactId>
-        <version>2.4</version>
       </plugin>
-      <plugin>
-        <artifactId>maven-surefire-report-plugin</artifactId>
-        <version>2.5</version>
-      </plugin>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <version>2.1</version>
-      </plugin>
-      -->
       <plugin>
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>

From 509131608210166b664a46c827a9e99f9873d18a Mon Sep 17 00:00:00 2001
From: Andrew Purtell <apurtell@apache.org>
Date: Tue, 17 Nov 2015 12:28:51 -0800
Subject: [PATCH 1530/1540] HBASE-14799 Commons-collections object
 deserialization remote command execution vulnerability

Signed-off-by: Andrew Purtell <apurtell@apache.org>
---
 pom.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/pom.xml b/pom.xml
index fa45138bb90b..47da6a9f9ae6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1084,6 +1084,8 @@
     <avro.version>1.5.3</avro.version>
     <commons-cli.version>1.2</commons-cli.version>
     <commons-codec.version>1.4</commons-codec.version>
+    <!-- Do not use versions earlier than 3.2.2 due to a security vulnerability -->
+    <commons-collections.version>3.2.2</commons-collections.version>
     <commons-httpclient.version>3.1</commons-httpclient.version><!-- pretty outdated -->
     <commons-io.version>2.1</commons-io.version>
     <commons-lang.version>2.5</commons-lang.version>
@@ -1187,6 +1189,11 @@
       <artifactId>commons-codec</artifactId>
       <version>${commons-codec.version}</version>
     </dependency>
+    <dependency>
+      <groupId>commons-collections</groupId>
+      <artifactId>commons-collections</artifactId>
+      <version>${commons-collections.version}</version>
+    </dependency>
     <dependency>
       <groupId>commons-httpclient</groupId>
       <artifactId>commons-httpclient</artifactId>

From 0f35a32ab123ee299f4aaaea02b4ba2d2b43cff2 Mon Sep 17 00:00:00 2001
From: Misty Stanley-Jones <mstanleyjones@cloudera.com>
Date: Tue, 15 Dec 2015 10:30:04 -0800
Subject: [PATCH 1531/1540] HBASE-14830 Make encoding classes
 @InterfaceAudience.Private

---
 .../hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java      | 3 +++
 .../apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java   | 3 +++
 .../apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java  | 3 +++
 .../apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java | 3 +++
 4 files changed, 12 insertions(+)

diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
index 548985b6c6ca..e9bebba97d67 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
@@ -25,10 +25,13 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+
 /**
  * Just copy data, do not do any kind of compression. Use for comparison and
  * benchmarking.
  */
+@InterfaceAudience.Private
 public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
   @Override
   public void compressKeyValues(DataOutputStream out,
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
index 7e518186ac19..4124055826cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
@@ -26,6 +26,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+
 /**
  * Compress using:
  * - store size of common prefix
@@ -47,6 +49,7 @@
  * - 1 byte:    type (only if FLAG_SAME_TYPE is not set in the flag)
  * - ... bytes: value
  */
+@InterfaceAudience.Private
 public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
   static final int FLAG_SAME_KEY_LENGTH = 1;
   static final int FLAG_SAME_VALUE_LENGTH = 1 << 1;
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index 7959e0e3c9e8..534906e3a456 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -27,6 +27,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+
 /**
  * Encoder similar to {@link DiffKeyDeltaEncoder} but supposedly faster.
  *
@@ -52,6 +54,7 @@
  * - ... bytes: value (only if FLAG_SAME_VALUE is not set in the flag)
  *
  */
+@InterfaceAudience.Private
 public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
   final int MASK_TIMESTAMP_LENGTH = (1 << 0) | (1 << 1) | (1 << 2);
   final int SHIFT_TIMESTAMP_LENGTH = 0;
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
index 3e4416fb6c0b..9e3806d1a77d 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -26,6 +26,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+
 /**
  * Compress key by storing size of common prefix with previous KeyValue
  * and storing raw size of rest.
@@ -40,6 +42,7 @@
  * In a worst case compressed KeyValue will be three bytes longer than original.
  *
  */
+@InterfaceAudience.Private
 public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
 
   private int addKV(int prevKeyOffset, DataOutputStream out,

From 56befcb90cdcddfcf4a4eb43166ab0750976ee3e Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 29 Dec 2015 23:28:02 -0800
Subject: [PATCH 1532/1540] 0.94.28-SNAPSHOT

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 47da6a9f9ae6..19c7ed52c8b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.27</version>
+  <version>0.94.28-SNAPSHOT</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From 30b0c98245b16a8d8e44b94d24aa3f5f35977b8e Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Wed, 30 Dec 2015 14:09:34 -0800
Subject: [PATCH 1533/1540] HBASE-15054 Allow 0.94 to compile with JDK8.

---
 .../main/java/org/apache/hadoop/hbase/ipc/SecureClient.java   | 2 +-
 src/main/java/org/apache/hadoop/hbase/client/HTablePool.java  | 2 +-
 src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java    | 2 +-
 .../hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java   | 2 +-
 src/main/java/org/apache/hadoop/hbase/util/PoolMap.java       | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
index 721e4628b61b..70fe4b72595d 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java
@@ -444,7 +444,7 @@ protected synchronized void close() {
       // release the resources
       // first thing to do;take the connection out of the connection list
       synchronized (connections) {
-        connections.remove(remoteId, this);
+        connections.removeValue(remoteId, this);
       }
 
       // close the streams and therefore the socket
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 01e6bbd0c2ee..05016e5127f3 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -255,7 +255,7 @@ private void returnTable(HTableInterface table) throws IOException {
     String tableName = Bytes.toString(table.getTableName());
     if (tables.size(tableName) >= maxSize) {
       // release table instance since we're not reusing it
-      this.tables.remove(tableName, table);
+      this.tables.removeValue(tableName, table);
       this.tableFactory.releaseHTableInterface(table);
       return;
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index a74abdb3bc90..5c33cc688efb 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -721,7 +721,7 @@ protected synchronized void close() {
       // release the resources
       // first thing to do;take the connection out of the connection list
       synchronized (connections) {
-        connections.remove(remoteId, this);
+        connections.removeValue(remoteId, this);
       }
 
       // close the streams and therefore the socket
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
index c998c1a6fa69..586af677a358 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
@@ -351,7 +351,7 @@ public static <K,V> void writePartitionFile(Job job, Sampler<K,V> sampler)
     final InputFormat inf =
         ReflectionUtils.newInstance(job.getInputFormatClass(), conf);
     int numPartitions = job.getNumReduceTasks();
-    K[] samples = sampler.getSample(inf, job);
+    K[] samples = (K[])sampler.getSample(inf, job);
     LOG.info("Using " + samples.length + " samples");
     RawComparator<K> comparator =
       (RawComparator<K>) job.getSortComparator();
diff --git a/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
index 1956e6b8ae5e..1b6e1c825c91 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
@@ -86,12 +86,12 @@ public V put(K key, V value) {
   public V remove(Object key) {
     Pool<V> pool = pools.remove(key);
     if (pool != null) {
-      remove((K) key, pool.get());
+      removeValue((K) key, pool.get());
     }
     return null;
   }
 
-  public boolean remove(K key, V value) {
+  public boolean removeValue(K key, V value) {
     Pool<V> pool = pools.get(key);
     boolean res = false;
     if (pool != null) {

From dd5660444a2587840bf982559d5b48f8a8e74e3c Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 4 Jan 2016 21:36:19 -0800
Subject: [PATCH 1534/1540] HBASE-15059 Allow 0.94 to compile against Hadoop
 2.7.x.

---
 dev-support/build-proto.sh                    |  37 ++++++
 pom.xml                                       |  98 ++++++++++++++++
 .../hbase/master/MasterDumpServlet.java       |   5 +-
 .../metrics/file/TimeStampingFileContext.java | 111 ------------------
 .../hbase/regionserver/RSDumpServlet.java     |   5 +-
 .../org/apache/hadoop/hbase/util/Threads.java |  49 +++++++-
 .../file/TestTimeStampingMetricsContext.java  | 107 -----------------
 7 files changed, 191 insertions(+), 221 deletions(-)
 create mode 100644 dev-support/build-proto.sh
 delete mode 100644 src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
 delete mode 100644 src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java

diff --git a/dev-support/build-proto.sh b/dev-support/build-proto.sh
new file mode 100644
index 000000000000..96380f862dfa
--- /dev/null
+++ b/dev-support/build-proto.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# script to run protoc to generate protocol buf files.
+# usage: ./build-proto.sh
+#
+
+which protoc
+if [ $? != 0 ] ; then
+  echo "Must have protoc compiler in your path to generate code"
+  exit 1
+fi
+
+HBASE_DIR=`dirname $0`/..
+PROTO_DIR=$HBASE_DIR/src/main/protobuf
+JAVA_DIR=$HBASE_DIR/src/main/java
+
+set -x
+for f in $PROTO_DIR/*.proto ; do
+  protoc -I$PROTO_DIR --java_out=$JAVA_DIR $f
+done
+
diff --git a/pom.xml b/pom.xml
index 19c7ed52c8b2..6507012b3bee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2385,6 +2385,104 @@
       </build>
     </profile>
 
+    <!--
+      profile for building against Hadoop 2.7. Activate using:
+       mvn -Dhadoop.profile=2.7
+
+      *** NOTE:
+       You MUST ensure that the protobuf Java classes are generated with protoc 2.5.0.
+       Do that by calling  dev-support/build-proto.sh  with protoc 2.5.0 in your path
+       before building HBase; that will regenerate the Java classes.
+       (HBase will not work with Hadoop 2.6+ with the older classes generated with protoc 2.4.1)
+
+       The necessary runtime dependencies for protobuf and commons.io are pulled in automatically.
+
+      Example:
+       $ protoc - -version
+       check that the output indicates 2.5.0
+
+       $ dev-support/build-proto.sh
+       $ mvn -Dhadoop.profile=2.7 ...
+    -->
+    <profile>
+      <id>hadoop-2.7</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>2.7</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>2.7.1</hadoop.version>
+        <slf4j.version>1.6.1</slf4j.version>
+        <protobuf.version>2.5.0</protobuf.version>
+        <commons-io.version>2.4</commons-io.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-annotations</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <!-- test deps for hadoop-2.0 profile -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>compile</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-resource</id>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>src/test/resources</directory>
+                      <includes>
+                        <include>hbase-site.xml</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>create-mrapp-generated-classpath</id>
+                <phase>generate-test-resources</phase>
+                <goals>
+                  <goal>build-classpath</goal>
+                </goals>
+                <configuration>
+                  <!-- needed to run the unit test for DS to generate
+                  the required classpath that is required in the env
+                  of the launch container in the mini mr/yarn cluster
+                  -->
+                  <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
 
     <!-- profiles for the tests
          See as well the properties of the project for the values
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
index 876eda4137c7..608979a30188 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.Date;
 import java.util.Map;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.hbase.monitoring.LogMonitoring;
 import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class MasterDumpServlet extends StateDumpServlet {
@@ -79,7 +81,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     
     out.println("\n\nStacks:");
     out.println(LINE);
-    ReflectionUtils.printThreadInfo(out, "");
+    PrintStream ps = new PrintStream(os, false, "UTF-8");
+    Threads.printThreadInfo(ps, "");
     
     out.println("\n\nMaster configuration:");
     out.println(LINE);
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java b/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
deleted file mode 100644
index 000e0d3d976a..000000000000
--- a/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Copyright 2008 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.metrics.file;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.file.FileContext;
-import org.apache.hadoop.metrics.spi.OutputRecord;
-
-/**
- * Add timestamp to {@link org.apache.hadoop.metrics.file.FileContext#emitRecord(String, String, OutputRecord)}.
- */
-public class TimeStampingFileContext extends FileContext {
-  // Copies bunch of FileContext here because writer and file are private in
-  // superclass.
-  private File file = null;
-  private PrintWriter writer = null;
-  private final SimpleDateFormat sdf;
-
-  public TimeStampingFileContext() {
-    super();
-    this.sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-  }
-
-  @Override
-  public void init(String contextName, ContextFactory factory) {
-    super.init(contextName, factory);
-    String fileName = getAttribute(FILE_NAME_PROPERTY);
-    if (fileName != null) {
-      file = new File(fileName);
-    }
-  }
-
-  @Override
-  public void startMonitoring() throws IOException {
-    if (file == null) {
-      writer = new PrintWriter(new BufferedOutputStream(System.out));
-    } else {
-      writer = new PrintWriter(new FileWriter(file, true));
-    }
-    super.startMonitoring();
-  }
-
-  @Override
-  public void stopMonitoring() {
-    super.stopMonitoring();
-    if (writer != null) {
-      writer.close();
-      writer = null;
-    }
-  }
-
-  private synchronized String iso8601() {
-    return this.sdf.format(new Date());
-  }
-
-  @Override
-  public void emitRecord(String contextName, String recordName,
-      OutputRecord outRec) {
-    writer.print(iso8601());
-    writer.print(" ");
-    writer.print(contextName);
-    writer.print(".");
-    writer.print(recordName);
-    String separator = ": ";
-    for (String tagName : outRec.getTagNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(tagName);
-      writer.print("=");
-      writer.print(outRec.getTag(tagName));
-    }
-    for (String metricName : outRec.getMetricNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(metricName);
-      writer.print("=");
-      writer.print(outRec.getMetric(metricName));
-    }
-    writer.println();
-  }
-
-  @Override
-  public void flush() {
-    writer.flush();
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
index 979cd5225716..2df824b02922 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.Date;
 
@@ -31,6 +32,7 @@
 import org.apache.hadoop.hbase.monitoring.LogMonitoring;
 import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class RSDumpServlet extends StateDumpServlet {
@@ -77,7 +79,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     
     out.println("\n\nStacks:");
     out.println(LINE);
-    ReflectionUtils.printThreadInfo(out, "");
+    PrintStream ps = new PrintStream(os, false, "UTF-8");
+    Threads.printThreadInfo(ps, "");
     
     out.println("\n\nRS Configuration:");
     out.println(LINE);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index a5a4d1311916..5a51bccbb389 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -19,8 +19,11 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -116,7 +119,7 @@ public static void threadDumpingIsAlive(final Thread t)
     while (t.isAlive()) {
       t.join(60 * 1000);
       if (t.isAlive()) {
-        ReflectionUtils.printThreadInfo(new PrintWriter(System.out),
+        printThreadInfo(System.out,
             "Automatic Stack Trace every 60 seconds waiting on " +
             t.getName());
       }
@@ -229,4 +232,48 @@ public Thread newThread(Runnable r) {
 
     };
   }
+
+  private static Method printThreadInfoMethod = null;
+  private static boolean printThreadInfoMethodWithPrintStream = true;
+
+  /**
+   * Print all of the thread's information and stack traces. Wrapper around Hadoop's method.
+   *
+   * @param stream the stream to
+   * @param title a string title for the stack trace
+   */
+  public static void printThreadInfo(PrintStream stream, String title) {
+
+    if (printThreadInfoMethod == null) {
+      try {
+        // Hadoop 2.7+ declares printThreadInfo(PrintStream, String)
+        printThreadInfoMethod = ReflectionUtils.class.getMethod("printThreadInfo",
+          PrintStream.class, String.class);
+      } catch (NoSuchMethodException e) {
+        // Hadoop 2.6 and earlier declares printThreadInfo(PrintWriter, String)
+        printThreadInfoMethodWithPrintStream = false;
+        try {
+          printThreadInfoMethod = ReflectionUtils.class.getMethod("printThreadInfo",
+            PrintWriter.class, String.class);
+        } catch (NoSuchMethodException e1) {
+          throw new RuntimeException("Cannot find method. Check hadoop jars linked", e1);
+        }
+      }
+      printThreadInfoMethod.setAccessible(true);
+    }
+
+    try {
+      if (printThreadInfoMethodWithPrintStream) {
+        printThreadInfoMethod.invoke(null, stream, title);
+      } else {
+        printThreadInfoMethod.invoke(null, new PrintWriter(stream), title);
+      }
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException(e.getCause());
+    } catch (IllegalArgumentException e) {
+      throw new RuntimeException(e.getCause());
+    } catch (InvocationTargetException e) {
+      throw new RuntimeException(e.getCause());
+    }
+  }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java b/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
deleted file mode 100644
index 0700508e87e8..000000000000
--- a/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.metrics.file;
-
-import java.io.File;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Map;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.SmallTests;
-
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-import static org.junit.Assert.*;
-
-/**
- * Test for TimeStampingMetricsContext functionality.
- * (FQN class names are used to suppress javac warnings in imports.)
- */
-@Category(SmallTests.class)
-@SuppressWarnings("deprecation")
-public class TestTimeStampingMetricsContext {
-
-  private static final int updatePeriodSeconds = 2;
-
-  private TimeStampingFileContext mc;
-
-  @Test
-  public void testFileUpdate() throws Exception {
-    final Date start = new Date();
-    final File metricOutFile = FileUtil.createLocalTempFile(
-      new File(FileUtils.getTempDirectory(),getClass().getName() + "-out-"), "", true);
-    assertTrue(metricOutFile.exists());
-    assertEquals(0L, metricOutFile.length());
-
-    mc = new TimeStampingFileContext();
-    org.apache.hadoop.metrics.ContextFactory cf
-      = org.apache.hadoop.metrics.ContextFactory.getFactory();
-    cf.setAttribute("test1.fileName", metricOutFile.getAbsolutePath());
-    cf.setAttribute("test1.period", Integer.toString(updatePeriodSeconds));
-    mc.init("test1", cf);
-
-    assertEquals("test1", mc.getContextName());
-
-    org.apache.hadoop.metrics.MetricsRecord r = mc.createRecord("testRecord");
-    r.setTag("testTag1", "testTagValue1");
-    r.setTag("testTag2", "testTagValue2");
-    r.setMetric("testMetric1", 1);
-    r.setMetric("testMetric2", 33);
-    r.update();
-
-    mc.startMonitoring();
-    assertTrue(mc.isMonitoring());
-
-    // wait 3/2 of the update period:
-    Thread.sleep((1000 * updatePeriodSeconds * 3)/2);
-
-    mc.stopMonitoring();
-    assertFalse(mc.isMonitoring());
-
-    mc.close();
-
-    Map<String, Collection<org.apache.hadoop.metrics.spi.OutputRecord>> m = mc.getAllRecords();
-    assertEquals(1, m.size());
-    Collection<org.apache.hadoop.metrics.spi.OutputRecord> outputRecords = m.get("testRecord");
-    assertNotNull(outputRecords);
-    assertEquals(1, outputRecords.size());
-    org.apache.hadoop.metrics.spi.OutputRecord outputRecord = outputRecords.iterator().next();
-    assertNotNull(outputRecord);
-
-    String outStr = FileUtils.readFileToString(metricOutFile);
-    assertTrue(outStr.length() > 0);
-    int pos = outStr.indexOf(" ");
-    String time = outStr.substring(0, pos);
-
-    DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-    Date date = df.parse(time);
-    assertTrue(date.after(start));
-    assertTrue(date.before(new Date()));
-
-    String reminder = outStr.substring(pos);
-    assertEquals(" test1.testRecord: testTag1=testTagValue1, testTag2=testTagValue2, testMetric1=1,"
-      +" testMetric2=33\n", reminder);
-  }
-
-}

From a473987059e3cec22b519cc856ddbb924395be3b Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 4 Jan 2016 21:45:20 -0800
Subject: [PATCH 1535/1540] pom.xml, CHANGES.txt for 0.94.28RC0

---
 CHANGES.txt | 15 +++++++++++++++
 pom.xml     |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 3d368f11361e..e0b2217443fb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,19 @@
 HBase Change Log
+Release 0.94.28 - 01/04/2016
+Sub-task
+
+    [HBASE-14748] - Update 0.94 apidocs and xref on website
+
+Bug
+
+    [HBASE-12921] - Port HBASE-5356 'region_mover.rb can hang if table region it belongs to is deleted' to 0.94
+    [HBASE-13454] - SecureClient#setupIOStreams should handle all Exceptions
+    [HBASE-13651] - Handle StoreFileScanner FileNotFoundException
+    [HBASE-14799] - Commons-collections object deserialization remote command execution vulnerability
+    [HBASE-14830] - Fix broken links in 0.94 generated docs
+    [HBASE-15054] - Allow 0.94 to compile with JDK8
+    [HBASE-15059] - Allow 0.94 to compile against Hadoop 2.7.x
+
 Release 0.94.27 - 03/18/2015
 Sub-task
 
diff --git a/pom.xml b/pom.xml
index 6507012b3bee..5a5806e5469c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
   <groupId>org.apache.hbase</groupId>
   <artifactId>hbase</artifactId>
   <packaging>jar</packaging>
-  <version>0.94.28-SNAPSHOT</version>
+  <version>0.94.28</version>
   <name>HBase</name>
   <description>
     HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need

From c97c915ec886e1dc47b34a2d5e2356da5c561cab Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Mon, 4 Jan 2016 21:54:15 -0800
Subject: [PATCH 1536/1540] missed some CHANGES.txt for 0.94.28RC0

---
 CHANGES.txt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGES.txt b/CHANGES.txt
index e0b2217443fb..68b6b2eacf66 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,6 +14,14 @@ Bug
     [HBASE-15054] - Allow 0.94 to compile with JDK8
     [HBASE-15059] - Allow 0.94 to compile against Hadoop 2.7.x
 
+Improvement
+
+    [HBASE-13344] - Add enforcer rule that matches our JDK support statement
+
+Task
+
+    [HBASE-14747] - Make it possible to build Javadoc and xref reports for 0.94 again
+
 Release 0.94.27 - 03/18/2015
 Sub-task
 

From 498fd221d37064d146b5af246b494b8ca8cce526 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Thu, 7 Jan 2016 11:15:56 -0800
Subject: [PATCH 1537/1540] HBASE-14213 Ensure ASF policy compliant headers and
 correct LICENSE and NOTICE files in artifacts for 0.94. (Sean Busbey)

---
 LICENSE.txt                                   |  69 ++++++
 NOTICE.txt                                    |  24 +--
 bin/get-active-master.rb                      |   2 -
 bin/graceful_stop.sh                          |   2 -
 bin/hbase                                     |   2 -
 bin/hbase-config.sh                           |   2 -
 bin/hbase-daemon.sh                           |   2 -
 bin/hbase-daemons.sh                          |   2 -
 bin/hirb.rb                                   |   2 -
 bin/local-master-backup.sh                    |   2 -
 bin/local-regionservers.sh                    |   2 -
 bin/master-backup.sh                          |   2 -
 bin/region_mover.rb                           |   2 -
 bin/regionservers.sh                          |   2 -
 bin/replication/copy_tables_desc.rb           |   2 -
 bin/rolling-restart.sh                        |   2 -
 bin/start-hbase.sh                            |   2 -
 bin/stop-hbase.sh                             |   2 -
 bin/zookeepers.sh                             |   2 -
 conf/hadoop-metrics.properties                |  16 ++
 conf/hbase-env.sh                             |   2 -
 conf/hbase-site.xml                           |   2 -
 conf/log4j.properties                         |  16 ++
 .../hbase_jdiff_acrossSingularityTemplate.xml |  27 ++-
 .../hbase_jdiff_afterSingularityTemplate.xml  |  23 +-
 dev-support/hbase_jdiff_template.xml          |  27 ++-
 dev-support/jdiffHBasePublicAPI.sh            |  27 ++-
 dev-support/jdiffHBasePublicAPI_common.sh     |  23 +-
 dev-support/smart-apply-patch.sh              |  23 +-
 dev-support/test-patch.sh                     |  23 +-
 dev-support/test-util.sh                      |   2 -
 pom.xml                                       | 201 +++++++++++++++++-
 .../security/access/AccessController.java     |  23 +-
 .../access/SecureBulkLoadEndpoint.java        |   4 +-
 .../access/SecureBulkLoadProtocol.java        |   4 +-
 security/src/test/resources/hbase-site.xml    |   2 -
 src/assembly/all.xml                          |  27 +++
 .../hadoop/hbase/mapreduce/IndexBuilder.java  |   4 +-
 .../hbase/mapreduce/SampleUploader.java       |   4 +-
 src/examples/thrift/DemoClient.cpp            |   2 -
 src/examples/thrift/DemoClient.java           |   4 +-
 src/examples/thrift/DemoClient.php            |   2 -
 src/examples/thrift/DemoClient.pl             |   2 -
 src/examples/thrift/DemoClient.py             |   3 +-
 src/examples/thrift/DemoClient.rb             |   2 -
 src/examples/thrift/Makefile                  |   2 -
 src/examples/thrift2/DemoClient.java          |   4 +-
 src/examples/thrift2/DemoClient.py            |   2 -
 .../hbase/tmpl/common/TaskMonitorTmpl.jamon   |   2 -
 .../master/AssignmentManagerStatusTmpl.jamon  |   2 -
 .../hbase/tmpl/master/MasterStatusTmpl.jamon  |   2 -
 .../tmpl/regionserver/RSStatusTmpl.jamon      |   2 -
 .../org/apache/hadoop/hbase/Abortable.java    |   4 +-
 .../java/org/apache/hadoop/hbase/Chore.java   |   4 +-
 .../hadoop/hbase/ClockOutOfSyncException.java |   4 +-
 .../apache/hadoop/hbase/ClusterStatus.java    |   4 +-
 .../org/apache/hadoop/hbase/Coprocessor.java  |  25 ++-
 .../hadoop/hbase/CoprocessorEnvironment.java  |  27 +--
 .../hadoop/hbase/DoNotRetryIOException.java   |   4 +-
 .../hbase/DroppedSnapshotException.java       |  29 +--
 .../org/apache/hadoop/hbase/EmptyWatcher.java |   4 +-
 .../hadoop/hbase/HBaseConfiguration.java      |   4 +-
 .../hadoop/hbase/HColumnDescriptor.java       |   4 +-
 .../org/apache/hadoop/hbase/HConstants.java   |   4 +-
 .../hadoop/hbase/HDFSBlocksDistribution.java  |   4 +-
 .../org/apache/hadoop/hbase/HRegionInfo.java  |   4 +-
 .../apache/hadoop/hbase/HRegionLocation.java  |   1 -
 .../apache/hadoop/hbase/HServerAddress.java   |   1 -
 .../org/apache/hadoop/hbase/HServerInfo.java  |   1 -
 .../org/apache/hadoop/hbase/HServerLoad.java  |   1 -
 .../apache/hadoop/hbase/HTableDescriptor.java |   1 -
 .../apache/hadoop/hbase/HealthCheckChore.java |   4 +-
 .../apache/hadoop/hbase/HealthChecker.java    |   4 +-
 .../InvalidFamilyOperationException.java      |   1 -
 .../org/apache/hadoop/hbase/KeyValue.java     |   1 -
 .../hadoop/hbase/LocalHBaseCluster.java       |   1 -
 .../hadoop/hbase/MasterAddressTracker.java    |   1 -
 .../hbase/MasterNotRunningException.java      |   1 -
 .../NotAllMetaRegionsOnlineException.java     |   1 -
 .../hbase/NotServingRegionException.java      |   1 -
 .../hadoop/hbase/PleaseHoldException.java     |   1 -
 .../apache/hadoop/hbase/RegionException.java  |   1 -
 .../hadoop/hbase/RemoteExceptionHandler.java  |   1 -
 .../java/org/apache/hadoop/hbase/Server.java  |   1 -
 .../org/apache/hadoop/hbase/ServerName.java   |   1 -
 .../org/apache/hadoop/hbase/Stoppable.java    |   1 -
 .../hadoop/hbase/TableExistsException.java    |  27 +--
 .../hbase/TableNotDisabledException.java      |   1 -
 .../hbase/TableNotEnabledException.java       |   1 -
 .../hadoop/hbase/TableNotFoundException.java  |   1 -
 .../hadoop/hbase/UnknownRegionException.java  |   1 -
 .../hadoop/hbase/UnknownRowLockException.java |   1 -
 .../hadoop/hbase/UnknownScannerException.java |   1 -
 .../hadoop/hbase/YouAreDeadException.java     |   1 -
 .../hbase/ZooKeeperConnectionException.java   |   1 -
 .../hbase/catalog/RootLocationEditor.java     |   4 +-
 .../apache/hadoop/hbase/client/Action.java    |   4 +-
 .../hadoop/hbase/client/Attributes.java       |   2 -
 .../apache/hadoop/hbase/client/Delete.java    |   2 -
 .../org/apache/hadoop/hbase/client/Get.java   |   4 +-
 .../hadoop/hbase/client/HBaseAdmin.java       |   4 +-
 .../hadoop/hbase/client/HConnection.java      |   4 +-
 .../hbase/client/HConnectionManager.java      |   4 +-
 .../apache/hadoop/hbase/client/HTable.java    |   4 +-
 .../hadoop/hbase/client/HTableFactory.java    |   4 +-
 .../hadoop/hbase/client/HTableInterface.java  |   4 +-
 .../hbase/client/HTableInterfaceFactory.java  |   4 +-
 .../hadoop/hbase/client/HTablePool.java       |   4 +-
 .../hadoop/hbase/client/HTableUtil.java       |   4 +-
 .../apache/hadoop/hbase/client/Increment.java |   4 +-
 .../hadoop/hbase/client/MetaScanner.java      |   4 +-
 .../hadoop/hbase/client/MultiAction.java      |   2 -
 .../apache/hadoop/hbase/client/MultiPut.java  |   2 -
 .../hadoop/hbase/client/MultiPutResponse.java |   2 -
 .../hadoop/hbase/client/MultiResponse.java    |   2 -
 .../apache/hadoop/hbase/client/Mutation.java  |   2 -
 .../client/NoServerForRegionException.java    |   4 +-
 .../apache/hadoop/hbase/client/Operation.java |   2 -
 .../hbase/client/OperationWithAttributes.java |   2 -
 .../org/apache/hadoop/hbase/client/Put.java   |   2 -
 .../hbase/client/RegionOfflineException.java  |   4 +-
 .../apache/hadoop/hbase/client/Result.java    |   2 -
 .../hadoop/hbase/client/ResultScanner.java    |   4 +-
 .../client/RetriesExhaustedException.java     |  27 +--
 .../RetriesExhaustedWithDetailsException.java |   2 -
 .../org/apache/hadoop/hbase/client/Row.java   |   2 -
 .../apache/hadoop/hbase/client/RowLock.java   |   4 +-
 .../org/apache/hadoop/hbase/client/Scan.java  |   2 -
 .../hbase/client/ScannerTimeoutException.java |   4 +-
 .../hadoop/hbase/client/ServerCallable.java   |   4 +-
 .../client/UnmodifyableHColumnDescriptor.java |   4 +-
 .../hbase/client/UnmodifyableHRegionInfo.java |   4 +-
 .../client/UnmodifyableHTableDescriptor.java  |   4 +-
 .../client/coprocessor/AggregationClient.java |   2 -
 .../hbase/client/coprocessor/Batch.java       |   2 -
 .../hadoop/hbase/client/coprocessor/Exec.java |   2 -
 .../hbase/client/coprocessor/ExecResult.java  |   2 -
 .../coprocessor/LongColumnInterpreter.java    |   2 -
 .../client/coprocessor/package-info.java      |   2 -
 .../hadoop/hbase/client/package-info.java     |   2 -
 .../client/replication/ReplicationAdmin.java  |   4 +-
 .../coprocessor/AggregateImplementation.java  |   2 -
 .../hbase/coprocessor/AggregateProtocol.java  |   4 +-
 .../coprocessor/BaseEndpointCoprocessor.java  |   2 -
 .../hbase/coprocessor/BaseMasterObserver.java |   2 -
 .../hbase/coprocessor/BaseRegionObserver.java |  25 ++-
 .../hbase/coprocessor/ColumnInterpreter.java  |   4 +-
 .../coprocessor/CoprocessorException.java     |   4 +-
 .../hbase/coprocessor/CoprocessorHost.java    |   2 -
 .../MasterCoprocessorEnvironment.java         |   2 -
 .../hbase/coprocessor/MasterObserver.java     |   2 -
 .../hbase/coprocessor/ObserverContext.java    |   2 -
 .../RegionCoprocessorEnvironment.java         |   2 -
 .../hbase/coprocessor/RegionObserver.java     |  25 ++-
 .../WALCoprocessorEnvironment.java            |   2 -
 .../hadoop/hbase/coprocessor/WALObserver.java |   2 -
 .../hbase/coprocessor/package-info.java       |   2 -
 .../hadoop/hbase/executor/EventHandler.java   |   4 +-
 .../hbase/executor/ExecutorService.java       |   4 +-
 .../hbase/executor/RegionTransitionData.java  |   4 +-
 .../hadoop/hbase/filter/BinaryComparator.java |   2 -
 .../hbase/filter/BinaryPrefixComparator.java  |   2 -
 .../hadoop/hbase/filter/BitComparator.java    |   2 -
 .../hbase/filter/ColumnCountGetFilter.java    |   2 -
 .../hbase/filter/ColumnPaginationFilter.java  |   2 -
 .../hbase/filter/ColumnPrefixFilter.java      |   2 -
 .../hbase/filter/ColumnRangeFilter.java       |   2 -
 .../hadoop/hbase/filter/CompareFilter.java    |   4 +-
 .../hbase/filter/DependentColumnFilter.java   |   4 +-
 .../hadoop/hbase/filter/FamilyFilter.java     |   4 +-
 .../apache/hadoop/hbase/filter/Filter.java    |   2 -
 .../hadoop/hbase/filter/FilterBase.java       |  18 +-
 .../hadoop/hbase/filter/FilterList.java       |   4 +-
 .../hbase/filter/FirstKeyOnlyFilter.java      |   2 -
 .../hbase/filter/InclusiveStopFilter.java     |   2 -
 .../filter/IncompatibleFilterException.java   |   4 +-
 .../filter/InvalidRowFilterException.java     |   4 +-
 .../hadoop/hbase/filter/KeyOnlyFilter.java    |   2 -
 .../hadoop/hbase/filter/NullComparator.java   |   2 -
 .../hadoop/hbase/filter/PageFilter.java       |   4 +-
 .../hadoop/hbase/filter/ParseConstants.java   |   4 +-
 .../hadoop/hbase/filter/ParseFilter.java      |   4 +-
 .../hadoop/hbase/filter/PrefixFilter.java     |   1 -
 .../hadoop/hbase/filter/QualifierFilter.java  |   4 +-
 .../hadoop/hbase/filter/RandomRowFilter.java  |   4 +-
 .../hbase/filter/RegexStringComparator.java   |   4 +-
 .../apache/hadoop/hbase/filter/RowFilter.java |   4 +-
 .../SingleColumnValueExcludeFilter.java       |   4 +-
 .../hbase/filter/SingleColumnValueFilter.java |   4 +-
 .../hadoop/hbase/filter/SkipFilter.java       |   2 -
 .../hbase/filter/SubstringComparator.java     |   4 +-
 .../hadoop/hbase/filter/ValueFilter.java      |   4 +-
 .../hadoop/hbase/filter/WhileMatchFilter.java |   2 -
 .../filter/WritableByteArrayComparable.java   |   4 +-
 .../hadoop/hbase/filter/package-info.java     |   2 -
 .../hadoop/hbase/io/CodeToClassAndBack.java   |   1 -
 .../hadoop/hbase/io/DoubleOutputStream.java   |   1 -
 .../hadoop/hbase/io/HalfStoreFileReader.java  |   1 -
 .../hadoop/hbase/io/HbaseMapWritable.java     |   1 -
 .../org/apache/hadoop/hbase/io/HeapSize.java  |   1 -
 .../hbase/io/ImmutableBytesWritable.java      |   1 -
 .../org/apache/hadoop/hbase/io/Reference.java |   1 -
 .../org/apache/hadoop/hbase/io/TimeRange.java |   1 -
 .../hadoop/hbase/io/WritableWithSize.java     |   1 -
 .../hbase/io/hfile/AbstractHFileReader.java   |   1 -
 .../hadoop/hbase/io/hfile/BlockCache.java     |   1 -
 .../hfile/BlockCacheColumnFamilySummary.java  |   1 -
 .../hadoop/hbase/io/hfile/BlockType.java      |   1 -
 .../hadoop/hbase/io/hfile/CacheStats.java     |   1 -
 .../hadoop/hbase/io/hfile/Cacheable.java      |   1 -
 .../hadoop/hbase/io/hfile/CachedBlock.java    |   1 -
 .../hbase/io/hfile/CachedBlockQueue.java      |   1 -
 .../hbase/io/hfile/DoubleBlockCache.java      |   1 -
 .../hbase/io/hfile/FixedFileTrailer.java      |   1 -
 .../apache/hadoop/hbase/io/hfile/HFile.java   |   1 -
 .../hbase/io/hfile/HFileBlockIndex.java       |   1 -
 .../hbase/io/hfile/HFilePrettyPrinter.java    |   1 -
 .../hadoop/hbase/io/hfile/HFileReaderV1.java  |   1 -
 .../hadoop/hbase/io/hfile/HFileReaderV2.java  |   1 -
 .../hadoop/hbase/io/hfile/HFileScanner.java   |   1 -
 .../hadoop/hbase/io/hfile/HFileWriterV1.java  |   1 -
 .../hadoop/hbase/io/hfile/HFileWriterV2.java  |   1 -
 .../hbase/io/hfile/InlineBlockWriter.java     |   1 -
 .../hadoop/hbase/io/hfile/LruBlockCache.java  |   1 -
 .../hbase/io/hfile/SimpleBlockCache.java      |   1 -
 .../hbase/io/hfile/slab/SingleSizeCache.java  |   1 -
 .../hadoop/hbase/io/hfile/slab/Slab.java      |   1 -
 .../hadoop/hbase/io/hfile/slab/SlabCache.java |   1 -
 .../io/hfile/slab/SlabItemActionWatcher.java  |   1 -
 .../hadoop/hbase/ipc/ConnectionHeader.java    |   1 -
 .../hadoop/hbase/ipc/CoprocessorProtocol.java |   1 -
 .../apache/hadoop/hbase/ipc/Delayable.java    |   1 -
 .../hadoop/hbase/ipc/ExecRPCInvoker.java      |   1 -
 .../apache/hadoop/hbase/ipc/HBaseClient.java  |   1 -
 .../org/apache/hadoop/hbase/ipc/HBaseRPC.java |   1 -
 .../hbase/ipc/HBaseRPCErrorHandler.java       |   1 -
 .../hadoop/hbase/ipc/HBaseRPCStatistics.java  |   1 -
 .../hadoop/hbase/ipc/HBaseRpcMetrics.java     |   1 -
 .../apache/hadoop/hbase/ipc/HBaseServer.java  |   1 -
 .../hadoop/hbase/ipc/HMasterInterface.java    |   1 -
 .../hbase/ipc/HMasterRegionInterface.java     |   1 -
 .../hadoop/hbase/ipc/HRegionInterface.java    |   1 -
 .../apache/hadoop/hbase/ipc/Invocation.java   |   1 -
 .../apache/hadoop/hbase/ipc/RpcEngine.java    |   1 -
 .../apache/hadoop/hbase/ipc/RpcServer.java    |   1 -
 .../ipc/ServerNotRunningYetException.java     |   1 -
 .../hadoop/hbase/ipc/WritableRpcEngine.java   |   1 -
 .../apache/hadoop/hbase/mapred/Driver.java    |   1 -
 .../hadoop/hbase/mapred/GroupingTableMap.java |   1 -
 .../hbase/mapred/HRegionPartitioner.java      |   1 -
 .../hadoop/hbase/mapred/IdentityTableMap.java |   1 -
 .../hbase/mapred/IdentityTableReduce.java     |   1 -
 .../hadoop/hbase/mapred/RowCounter.java       |   1 -
 .../hadoop/hbase/mapred/TableInputFormat.java |   1 -
 .../hbase/mapred/TableInputFormatBase.java    |   1 -
 .../apache/hadoop/hbase/mapred/TableMap.java  |   1 -
 .../hbase/mapred/TableMapReduceUtil.java      |   1 -
 .../hbase/mapred/TableOutputFormat.java       |   1 -
 .../hbase/mapred/TableRecordReader.java       |   1 -
 .../hbase/mapred/TableRecordReaderImpl.java   |   1 -
 .../hadoop/hbase/mapred/TableReduce.java      |   1 -
 .../hadoop/hbase/mapred/TableSplit.java       |   1 -
 .../hadoop/hbase/mapred/package-info.java     |   1 -
 .../hadoop/hbase/mapreduce/CellCounter.java   |   1 -
 .../hadoop/hbase/mapreduce/CopyTable.java     |   1 -
 .../apache/hadoop/hbase/mapreduce/Driver.java |   1 -
 .../apache/hadoop/hbase/mapreduce/Export.java |   1 -
 .../hbase/mapreduce/GroupingTableMapper.java  |   1 -
 .../hbase/mapreduce/HFileOutputFormat.java    |   1 -
 .../hbase/mapreduce/HRegionPartitioner.java   |   1 -
 .../hbase/mapreduce/IdentityTableMapper.java  |   1 -
 .../hbase/mapreduce/IdentityTableReducer.java |   1 -
 .../apache/hadoop/hbase/mapreduce/Import.java |   1 -
 .../hadoop/hbase/mapreduce/ImportTsv.java     |   1 -
 .../hbase/mapreduce/KeyValueSortReducer.java  |   1 -
 .../mapreduce/LoadIncrementalHFiles.java      |   1 -
 .../mapreduce/MultiTableOutputFormat.java     |   1 -
 .../hbase/mapreduce/PutSortReducer.java       |   1 -
 .../hadoop/hbase/mapreduce/RowCounter.java    |   1 -
 .../SimpleTotalOrderPartitioner.java          |   1 -
 .../hbase/mapreduce/TableInputFormat.java     |   1 -
 .../hbase/mapreduce/TableInputFormatBase.java |   1 -
 .../hbase/mapreduce/TableMapReduceUtil.java   |   1 -
 .../hadoop/hbase/mapreduce/TableMapper.java   |   1 -
 .../hbase/mapreduce/TableOutputCommitter.java |   1 -
 .../hbase/mapreduce/TableOutputFormat.java    |   1 -
 .../hbase/mapreduce/TableRecordReader.java    |   1 -
 .../hadoop/hbase/mapreduce/TableReducer.java  |   1 -
 .../hadoop/hbase/mapreduce/TableSplit.java    |   1 -
 .../mapreduce/hadoopbackport/JarFinder.java   |  23 +-
 .../hadoop/hbase/mapreduce/package-info.java  |   1 -
 .../replication/VerifyReplication.java        |   1 -
 .../hbase/master/ActiveMasterManager.java     |   1 -
 .../hadoop/hbase/master/AssignCallable.java   |   1 -
 .../hbase/master/AssignmentManager.java       |   1 -
 .../hadoop/hbase/master/BulkAssigner.java     |   1 -
 .../hadoop/hbase/master/BulkReOpen.java       |   1 -
 .../hadoop/hbase/master/CatalogJanitor.java   |   1 -
 .../hadoop/hbase/master/DeadServer.java       |   1 -
 .../hbase/master/DefaultLoadBalancer.java     |   1 -
 .../apache/hadoop/hbase/master/HMaster.java   |   1 -
 .../hbase/master/HMasterCommandLine.java      |   1 -
 .../hadoop/hbase/master/LoadBalancer.java     |   1 -
 .../hbase/master/LoadBalancerFactory.java     |   1 -
 .../hbase/master/MasterCoprocessorHost.java   |   1 -
 .../hbase/master/MasterDumpServlet.java       |   1 -
 .../hadoop/hbase/master/MasterFileSystem.java |   1 -
 .../hadoop/hbase/master/MasterServices.java   |   1 -
 .../hbase/master/MasterStatusServlet.java     |   1 -
 .../hadoop/hbase/master/ServerManager.java    |   1 -
 .../hadoop/hbase/master/SplitLogManager.java  |   1 -
 .../hadoop/hbase/master/UnAssignCallable.java |   1 -
 .../master/handler/ClosedRegionHandler.java   |   1 -
 .../master/handler/DeleteTableHandler.java    |   1 -
 .../master/handler/DisableTableHandler.java   |   1 -
 .../master/handler/EnableTableHandler.java    |   1 -
 .../handler/MetaServerShutdownHandler.java    |   1 -
 .../master/handler/ModifyTableHandler.java    |   1 -
 .../master/handler/OpenedRegionHandler.java   |   1 -
 .../master/handler/ServerShutdownHandler.java |   1 -
 .../master/handler/SplitRegionHandler.java    |   1 -
 .../master/handler/TableAddFamilyHandler.java |   1 -
 .../handler/TableDeleteFamilyHandler.java     |   1 -
 .../master/handler/TableEventHandler.java     |   1 -
 .../handler/TableModifyFamilyHandler.java     |   1 -
 .../master/handler/TotesHRegionInfo.java      |   1 -
 .../hadoop/hbase/metrics/HBaseInfo.java       |   1 -
 .../hbase/metrics/MetricsMBeanBase.java       |   1 -
 .../hbase/migration/HRegionInfo090x.java      |   1 -
 .../hbase/monitoring/LogMonitoring.java       |   1 -
 .../MemoryBoundedLogMessageBuffer.java        |   1 -
 .../hbase/monitoring/MonitoredRPCHandler.java |   1 -
 .../monitoring/MonitoredRPCHandlerImpl.java   |   1 -
 .../hbase/monitoring/MonitoredTask.java       |   1 -
 .../hbase/monitoring/MonitoredTaskImpl.java   |   1 -
 .../hbase/monitoring/StateDumpServlet.java    |   1 -
 .../hadoop/hbase/monitoring/TaskMonitor.java  |   1 -
 .../hbase/monitoring/ThreadMonitoring.java    |   1 -
 .../regionserver/ChangedReadersObserver.java  |   1 -
 .../hbase/regionserver/ColumnCount.java       |   1 -
 .../hbase/regionserver/ColumnTracker.java     |   1 -
 .../regionserver/CompactSplitThread.java      |   1 -
 .../regionserver/CompactionRequestor.java     |   1 -
 .../hadoop/hbase/regionserver/DebugPrint.java |   1 -
 .../hbase/regionserver/DeleteTracker.java     |   1 -
 .../regionserver/ExplicitColumnTracker.java   |   1 -
 .../hbase/regionserver/FlushRequester.java    |   1 -
 .../GetClosestRowBeforeTracker.java           |   1 -
 .../hadoop/hbase/regionserver/HRegion.java    |   1 -
 .../hbase/regionserver/HRegionServer.java     |   1 -
 .../HRegionServerCommandLine.java             |   1 -
 .../regionserver/HRegionThriftServer.java     |   1 -
 .../hbase/regionserver/InternalScan.java      |   1 -
 .../hbase/regionserver/InternalScanner.java   |   1 -
 .../hbase/regionserver/KeyValueHeap.java      |   1 -
 .../hbase/regionserver/KeyValueScanner.java   |   1 -
 .../regionserver/KeyValueSkipListSet.java     |   1 -
 .../hbase/regionserver/LeaseException.java    |   1 -
 .../hbase/regionserver/LeaseListener.java     |   1 -
 .../hadoop/hbase/regionserver/Leases.java     |   1 -
 .../hadoop/hbase/regionserver/LogRoller.java  |   1 -
 .../hadoop/hbase/regionserver/LruHashMap.java |   1 -
 .../hadoop/hbase/regionserver/MemStore.java   |   1 -
 .../hbase/regionserver/MemStoreFlusher.java   |   1 -
 .../hbase/regionserver/MemStoreLAB.java       |   1 -
 .../MultiVersionConsistencyControl.java       |   1 -
 .../NoSuchColumnFamilyException.java          |   1 -
 .../regionserver/NonLazyKeyValueScanner.java  |   1 -
 .../hbase/regionserver/OnlineRegions.java     |   1 -
 .../hbase/regionserver/OperationStatus.java   |   1 -
 .../hbase/regionserver/RSDumpServlet.java     |   1 -
 .../hbase/regionserver/RSStatusServlet.java   |   1 -
 .../RegionAlreadyInTransitionException.java   |   1 -
 .../regionserver/RegionCoprocessorHost.java   |   1 -
 .../regionserver/RegionOpeningState.java      |   1 -
 .../hbase/regionserver/RegionScanner.java     |   1 -
 .../regionserver/RegionServerAccounting.java  |   1 -
 .../RegionServerRunningException.java         |   1 -
 .../regionserver/RegionServerServices.java    |   1 -
 .../RegionServerStoppedException.java         |   1 -
 .../regionserver/ReplicationService.java      |   1 -
 .../regionserver/ReplicationSinkService.java  |   1 -
 .../ReplicationSourceService.java             |   1 -
 .../hbase/regionserver/ScanDeleteTracker.java |   1 -
 .../hbase/regionserver/ScanQueryMatcher.java  |   1 -
 .../ScanWildcardColumnTracker.java            |   1 -
 .../hbase/regionserver/ShutdownHook.java      |   1 -
 .../hbase/regionserver/SplitLogWorker.java    |   1 -
 .../hbase/regionserver/SplitRequest.java      |   1 -
 .../hbase/regionserver/SplitTransaction.java  |   1 -
 .../hadoop/hbase/regionserver/Store.java      |   1 -
 .../hadoop/hbase/regionserver/StoreFile.java  |   1 -
 .../hbase/regionserver/StoreFileScanner.java  |   1 -
 .../hbase/regionserver/StoreFlusher.java      |   1 -
 .../hbase/regionserver/StoreScanner.java      |   1 -
 .../hbase/regionserver/TimeRangeTracker.java  |   1 -
 .../regionserver/WrongRegionException.java    |   1 -
 .../compactions/CompactSelection.java         |   1 -
 .../compactions/CompactionProgress.java       |   1 -
 .../compactions/CompactionRequest.java        |   1 -
 .../handler/CloseMetaHandler.java             |   1 -
 .../handler/CloseRegionHandler.java           |   1 -
 .../handler/CloseRootHandler.java             |   1 -
 .../regionserver/handler/OpenMetaHandler.java |   1 -
 .../handler/OpenRegionHandler.java            |   1 -
 .../regionserver/handler/OpenRootHandler.java |   1 -
 .../metrics/RegionServerDynamicMetrics.java   |   1 -
 .../RegionServerDynamicStatistics.java        |   1 -
 .../metrics/RegionServerMetrics.java          |   1 -
 .../metrics/RegionServerStatistics.java       |   1 -
 .../wal/FailedLogCloseException.java          |   1 -
 .../hadoop/hbase/regionserver/wal/HLog.java   |   1 -
 .../hbase/regionserver/wal/HLogKey.java       |   1 -
 .../hbase/regionserver/wal/HLogSplitter.java  |   1 -
 .../wal/OrphanHLogAfterSplitException.java    |   1 -
 .../wal/SequenceFileLogReader.java            |   1 -
 .../wal/SequenceFileLogWriter.java            |   1 -
 .../regionserver/wal/WALActionsListener.java  |   1 -
 .../regionserver/wal/WALCoprocessorHost.java  |   1 -
 .../hbase/regionserver/wal/WALEdit.java       |   1 -
 .../hbase/replication/ReplicationPeer.java    |   1 -
 .../replication/ReplicationZookeeper.java     |   1 -
 .../master/ReplicationLogCleaner.java         |   1 -
 .../replication/regionserver/Replication.java |   1 -
 .../regionserver/ReplicationSink.java         |   1 -
 .../regionserver/ReplicationSinkMetrics.java  |   1 -
 .../regionserver/ReplicationSource.java       |   1 -
 .../ReplicationSourceInterface.java           |   1 -
 .../ReplicationSourceManager.java             |   1 -
 .../ReplicationSourceMetrics.java             |   1 -
 .../regionserver/ReplicationStatistics.java   |   1 -
 .../apache/hadoop/hbase/rest/Constants.java   |   1 -
 .../hadoop/hbase/rest/ExistsResource.java     |   1 -
 .../org/apache/hadoop/hbase/rest/Main.java    |   1 -
 .../hadoop/hbase/rest/MultiRowResource.java   |   1 -
 .../hbase/rest/ProtobufMessageHandler.java    |   1 -
 .../apache/hadoop/hbase/rest/RESTServlet.java |   1 -
 .../hadoop/hbase/rest/RegionsResource.java    |   1 -
 .../hadoop/hbase/rest/ResourceBase.java       |   1 -
 .../hadoop/hbase/rest/ResourceConfig.java     |   1 -
 .../hadoop/hbase/rest/ResultGenerator.java    |   1 -
 .../hadoop/hbase/rest/RootResource.java       |   1 -
 .../apache/hadoop/hbase/rest/RowResource.java |   1 -
 .../hadoop/hbase/rest/RowResultGenerator.java |   1 -
 .../org/apache/hadoop/hbase/rest/RowSpec.java |   1 -
 .../hbase/rest/ScannerInstanceResource.java   |   1 -
 .../hadoop/hbase/rest/ScannerResource.java    |   1 -
 .../hbase/rest/ScannerResultGenerator.java    |   1 -
 .../hadoop/hbase/rest/SchemaResource.java     |   1 -
 .../rest/StorageClusterStatusResource.java    |   1 -
 .../rest/StorageClusterVersionResource.java   |   1 -
 .../hadoop/hbase/rest/TableResource.java      |   1 -
 .../hadoop/hbase/rest/VersionResource.java    |   1 -
 .../hadoop/hbase/rest/client/Client.java      |   1 -
 .../hadoop/hbase/rest/client/Cluster.java     |   1 -
 .../hadoop/hbase/rest/client/RemoteAdmin.java |   1 -
 .../hbase/rest/client/RemoteHTable.java       |   1 -
 .../hadoop/hbase/rest/client/Response.java    |   1 -
 .../hbase/rest/filter/GZIPRequestStream.java  |   1 -
 .../hbase/rest/filter/GZIPRequestWrapper.java |   1 -
 .../hbase/rest/filter/GZIPResponseStream.java |   1 -
 .../rest/filter/GZIPResponseWrapper.java      |   1 -
 .../hadoop/hbase/rest/filter/GzipFilter.java  |   1 -
 .../hbase/rest/metrics/RESTMetrics.java       |   1 -
 .../hbase/rest/metrics/RESTStatistics.java    |   1 -
 .../hadoop/hbase/rest/model/CellModel.java    |   1 -
 .../hadoop/hbase/rest/model/CellSetModel.java |   1 -
 .../hbase/rest/model/ColumnSchemaModel.java   |   1 -
 .../hadoop/hbase/rest/model/RowModel.java     |   1 -
 .../hadoop/hbase/rest/model/ScannerModel.java |   1 -
 .../rest/model/StorageClusterStatusModel.java |   1 -
 .../model/StorageClusterVersionModel.java     |   1 -
 .../hbase/rest/model/TableInfoModel.java      |   1 -
 .../hbase/rest/model/TableListModel.java      |   1 -
 .../hadoop/hbase/rest/model/TableModel.java   |   1 -
 .../hbase/rest/model/TableRegionModel.java    |   1 -
 .../hbase/rest/model/TableSchemaModel.java    |   1 -
 .../hadoop/hbase/rest/model/VersionModel.java |   1 -
 .../org/apache/hadoop/hbase/rest/package.html |   1 -
 .../rest/provider/JAXBContextResolver.java    |   1 -
 .../consumer/ProtobufMessageBodyConsumer.java |   1 -
 .../PlainTextMessageBodyProducer.java         |   1 -
 .../producer/ProtobufMessageBodyProducer.java |   1 -
 .../thrift2/ThriftHBaseServiceHandler.java    |   1 -
 .../hadoop/hbase/thrift2/ThriftServer.java    |   1 -
 .../hadoop/hbase/thrift2/ThriftUtilities.java |   1 -
 .../apache/hadoop/hbase/util/Addressing.java  |   1 -
 .../org/apache/hadoop/hbase/util/Base64.java  |   1 -
 .../apache/hadoop/hbase/util/BloomFilter.java |   1 -
 .../hadoop/hbase/util/BloomFilterBase.java    |   1 -
 .../hadoop/hbase/util/BloomFilterFactory.java |   1 -
 .../hadoop/hbase/util/BloomFilterWriter.java  |   1 -
 .../hadoop/hbase/util/ByteBloomFilter.java    |   1 -
 .../hbase/util/ByteBufferOutputStream.java    |   1 -
 .../org/apache/hadoop/hbase/util/Bytes.java   |   1 -
 .../hbase/util/CancelableProgressable.java    |   1 -
 .../apache/hadoop/hbase/util/ClassSize.java   |   1 -
 .../hbase/util/CollectionBackedScanner.java   |   1 -
 .../hbase/util/CompoundBloomFilter.java       |   1 -
 .../hbase/util/CompoundBloomFilterBase.java   |   1 -
 .../hbase/util/CompoundBloomFilterWriter.java |   1 -
 .../hadoop/hbase/util/CompressionTest.java    |   1 -
 .../hbase/util/DefaultEnvironmentEdge.java    |   1 -
 .../hadoop/hbase/util/DirectMemoryUtils.java  |   1 -
 .../hadoop/hbase/util/EnvironmentEdge.java    |   1 -
 .../hbase/util/EnvironmentEdgeManager.java    |   1 -
 .../org/apache/hadoop/hbase/util/FSUtils.java |   1 -
 .../util/FileSystemVersionException.java      |   1 -
 .../hadoop/hbase/util/GetJavaProperty.java    |   1 -
 .../hadoop/hbase/util/HBaseConfTool.java      |   1 -
 .../hadoop/hbase/util/HBaseFsckRepair.java    |   1 -
 .../org/apache/hadoop/hbase/util/HMerge.java  |   1 -
 .../org/apache/hadoop/hbase/util/Hash.java    |   1 -
 .../apache/hadoop/hbase/util/HashedBytes.java |   1 -
 .../org/apache/hadoop/hbase/util/IdLock.java  |   1 -
 .../util/IncrementingEnvironmentEdge.java     |   1 -
 .../apache/hadoop/hbase/util/InfoServer.java  |   1 -
 .../hadoop/hbase/util/JVMClusterUtil.java     |   1 -
 .../apache/hadoop/hbase/util/JenkinsHash.java |   1 -
 .../apache/hadoop/hbase/util/JvmVersion.java  |   1 -
 .../apache/hadoop/hbase/util/KeyRange.java    |   1 -
 .../org/apache/hadoop/hbase/util/Keying.java  |   1 -
 .../org/apache/hadoop/hbase/util/MD5Hash.java |   1 -
 .../hbase/util/ManualEnvironmentEdge.java     |   1 -
 .../org/apache/hadoop/hbase/util/Merge.java   |   1 -
 .../apache/hadoop/hbase/util/MetaUtils.java   |   1 -
 .../apache/hadoop/hbase/util/MurmurHash.java  |   1 -
 .../org/apache/hadoop/hbase/util/Objects.java |   1 -
 .../org/apache/hadoop/hbase/util/Pair.java    |   1 -
 .../hadoop/hbase/util/PairOfSameType.java     |   1 -
 .../org/apache/hadoop/hbase/util/PoolMap.java |   1 -
 .../apache/hadoop/hbase/util/ProtoUtil.java   |  37 +++-
 .../hbase/util/RegionSplitCalculator.java     |   1 -
 .../hadoop/hbase/util/RegionSplitter.java     |   1 -
 .../hadoop/hbase/util/RetryCounter.java       |   1 -
 .../hbase/util/RetryCounterFactory.java       |   1 -
 .../hadoop/hbase/util/ServerCommandLine.java  |   1 -
 .../org/apache/hadoop/hbase/util/Sleeper.java |   1 -
 .../hadoop/hbase/util/SoftValueSortedMap.java |   1 -
 .../hbase/util/SortedCopyOnWriteSet.java      |   1 -
 .../org/apache/hadoop/hbase/util/Strings.java |   1 -
 .../org/apache/hadoop/hbase/util/Threads.java |   1 -
 .../apache/hadoop/hbase/util/VersionInfo.java |   1 -
 .../apache/hadoop/hbase/util/Writables.java   |   1 -
 .../hadoop/hbase/zookeeper/ClusterId.java     |   1 -
 .../hbase/zookeeper/ClusterStatusTracker.java |   1 -
 .../hadoop/hbase/zookeeper/HQuorumPeer.java   |   1 -
 .../hbase/zookeeper/MetaNodeTracker.java      |   1 -
 .../hbase/zookeeper/MiniZooKeeperCluster.java |   1 -
 .../hbase/zookeeper/RecoverableZooKeeper.java |   1 -
 .../hbase/zookeeper/RegionServerTracker.java  |   1 -
 .../hbase/zookeeper/RootRegionTracker.java    |   1 -
 .../hadoop/hbase/zookeeper/ZKAssign.java      |   1 -
 .../hadoop/hbase/zookeeper/ZKConfig.java      |   1 -
 .../hadoop/hbase/zookeeper/ZKServerTool.java  |   1 -
 .../hadoop/hbase/zookeeper/ZKSplitLog.java    |   1 -
 .../hadoop/hbase/zookeeper/ZKTable.java       |   1 -
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java |   1 -
 .../hbase/zookeeper/ZooKeeperListener.java    |   1 -
 .../zookeeper/ZooKeeperMainServerArg.java     |   1 -
 .../hbase/zookeeper/ZooKeeperNodeTracker.java |   1 -
 .../hbase/zookeeper/ZooKeeperWatcher.java     |   1 -
 .../hadoop/hbase/replication/package.html     |   1 -
 .../hbase/rest/protobuf/CellMessage.proto     |   1 -
 .../hbase/rest/protobuf/CellSetMessage.proto  |   1 -
 .../rest/protobuf/ColumnSchemaMessage.proto   |   1 -
 .../hbase/rest/protobuf/ScannerMessage.proto  |   1 -
 .../StorageClusterStatusMessage.proto         |   1 -
 .../rest/protobuf/TableInfoMessage.proto      |   1 -
 .../rest/protobuf/TableListMessage.proto      |   1 -
 .../rest/protobuf/TableSchemaMessage.proto    |   1 -
 .../hbase/rest/protobuf/VersionMessage.proto  |   1 -
 src/main/ruby/hbase.rb                        |   1 -
 src/main/ruby/hbase/admin.rb                  |   1 -
 src/main/ruby/hbase/hbase.rb                  |   1 -
 src/main/ruby/hbase/replication_admin.rb      |   1 -
 src/main/ruby/hbase/table.rb                  |   1 -
 src/main/ruby/irb/hirb.rb                     |   1 -
 src/main/ruby/shell.rb                        |   1 -
 src/main/ruby/shell/commands.rb               |   1 -
 src/main/ruby/shell/commands/add_peer.rb      |   1 -
 src/main/ruby/shell/commands/alter.rb         |   1 -
 src/main/ruby/shell/commands/alter_async.rb   |   1 -
 src/main/ruby/shell/commands/alter_status.rb  |   1 -
 src/main/ruby/shell/commands/assign.rb        |   1 -
 .../ruby/shell/commands/balance_switch.rb     |   1 -
 src/main/ruby/shell/commands/balancer.rb      |   1 -
 src/main/ruby/shell/commands/close_region.rb  |   1 -
 src/main/ruby/shell/commands/compact.rb       |   1 -
 src/main/ruby/shell/commands/count.rb         |   1 -
 src/main/ruby/shell/commands/create.rb        |   1 -
 src/main/ruby/shell/commands/delete.rb        |   1 -
 src/main/ruby/shell/commands/deleteall.rb     |   1 -
 src/main/ruby/shell/commands/describe.rb      |   1 -
 src/main/ruby/shell/commands/disable.rb       |   1 -
 src/main/ruby/shell/commands/disable_all.rb   |   1 -
 src/main/ruby/shell/commands/disable_peer.rb  |   1 -
 src/main/ruby/shell/commands/drop.rb          |   1 -
 src/main/ruby/shell/commands/drop_all.rb      |   1 -
 src/main/ruby/shell/commands/enable.rb        |   1 -
 src/main/ruby/shell/commands/enable_all.rb    |   1 -
 src/main/ruby/shell/commands/enable_peer.rb   |   1 -
 src/main/ruby/shell/commands/exists.rb        |   1 -
 src/main/ruby/shell/commands/flush.rb         |   1 -
 src/main/ruby/shell/commands/get.rb           |   1 -
 src/main/ruby/shell/commands/get_counter.rb   |   1 -
 src/main/ruby/shell/commands/hlog_roll.rb     |   1 -
 src/main/ruby/shell/commands/incr.rb          |   1 -
 src/main/ruby/shell/commands/is_disabled.rb   |   1 -
 src/main/ruby/shell/commands/is_enabled.rb    |   1 -
 src/main/ruby/shell/commands/list.rb          |   1 -
 src/main/ruby/shell/commands/major_compact.rb |   1 -
 src/main/ruby/shell/commands/move.rb          |   1 -
 src/main/ruby/shell/commands/put.rb           |   1 -
 src/main/ruby/shell/commands/remove_peer.rb   |   1 -
 src/main/ruby/shell/commands/scan.rb          |   1 -
 src/main/ruby/shell/commands/show_filters.rb  |   1 -
 src/main/ruby/shell/commands/split.rb         |   1 -
 .../ruby/shell/commands/start_replication.rb  |   1 -
 src/main/ruby/shell/commands/status.rb        |   1 -
 .../ruby/shell/commands/stop_replication.rb   |   1 -
 src/main/ruby/shell/commands/truncate.rb      |   1 -
 src/main/ruby/shell/commands/unassign.rb      |   1 -
 src/main/ruby/shell/commands/version.rb       |   1 -
 src/main/ruby/shell/commands/zk_dump.rb       |   1 -
 src/main/ruby/shell/formatter.rb              |   1 -
 .../xslt/configuration_to_docbook_section.xsl |   1 -
 src/packages/conf-pseudo/hbase-site.xml       |   1 -
 src/site/xdoc/acid-semantics.xml              |  28 +--
 src/site/xdoc/bulk-loads.xml                  |  24 ++-
 src/site/xdoc/cygwin.xml                      |  24 ++-
 src/site/xdoc/index.xml                       |  24 ++-
 src/site/xdoc/metrics.xml                     |  24 ++-
 src/site/xdoc/old_news.xml                    |  24 ++-
 src/site/xdoc/pseudo-distributed.xml          |  28 +--
 src/site/xdoc/replication.xml                 |  24 ++-
 src/site/xdoc/resources.xml                   |  23 +-
 src/site/xdoc/sponsors.xml                    |  23 +-
 .../apache/hadoop/hbase/HBaseTestCase.java    |   1 -
 .../hadoop/hbase/HBaseTestingUtility.java     |   1 -
 .../hbase/HFilePerformanceEvaluation.java     |   1 -
 .../apache/hadoop/hbase/KeyValueTestUtil.java |   1 -
 .../org/apache/hadoop/hbase/LargeTests.java   |   1 -
 .../hbase/MapFilePerformanceEvaluation.java   |   1 -
 .../org/apache/hadoop/hbase/MediumTests.java  |   1 -
 .../apache/hadoop/hbase/MiniHBaseCluster.java |   1 -
 .../hadoop/hbase/MultithreadedTestUtil.java   |   1 -
 .../hadoop/hbase/PerformanceEvaluation.java   |   1 -
 .../hbase/PerformanceEvaluationCommons.java   |   1 -
 .../apache/hadoop/hbase/ResourceChecker.java  |   1 -
 .../hbase/ResourceCheckerJUnitRule.java       |   1 -
 .../org/apache/hadoop/hbase/SmallTests.java   |   1 -
 .../hadoop/hbase/TestAcidGuarantees.java      |   1 -
 .../TestFSTableDescriptorForceCreation.java   |   1 -
 .../hbase/TestFullLogReconstruction.java      |   1 -
 .../hadoop/hbase/TestGlobalMemStoreSize.java  |   1 -
 .../hadoop/hbase/TestHBaseTestingUtility.java |   1 -
 .../hadoop/hbase/TestHRegionLocation.java     |   1 -
 .../hadoop/hbase/TestHServerAddress.java      |   1 -
 .../apache/hadoop/hbase/TestInfoServers.java  |   1 -
 .../org/apache/hadoop/hbase/TestKeyValue.java |   1 -
 .../hadoop/hbase/TestRegionRebalancing.java   |   1 -
 .../hadoop/hbase/TestSerialization.java       |   1 -
 .../apache/hadoop/hbase/TestZooKeeper.java    |   1 -
 .../hbase/catalog/TestCatalogTracker.java     |   1 -
 .../hbase/catalog/TestMetaReaderEditor.java   |   1 -
 .../apache/hadoop/hbase/client/TestAdmin.java |   1 -
 .../hadoop/hbase/client/TestAttributes.java   |   1 -
 .../hbase/client/TestFromClientSide.java      |   1 -
 .../apache/hadoop/hbase/client/TestGet.java   |   1 -
 .../apache/hadoop/hbase/client/TestHCM.java   |   1 -
 .../hadoop/hbase/client/TestHTablePool.java   |   1 -
 .../hadoop/hbase/client/TestHTableUtil.java   |   1 -
 .../client/TestMetaMigrationRemovingHTD.java  |   1 -
 .../hadoop/hbase/client/TestMetaScanner.java  |   1 -
 .../hbase/client/TestMultiParallel.java       |   1 -
 .../hbase/client/TestMultipleTimestamps.java  |   1 -
 .../hadoop/hbase/client/TestOperation.java    |   1 -
 .../hadoop/hbase/client/TestResult.java       |   1 -
 .../apache/hadoop/hbase/client/TestScan.java  |   1 -
 .../hbase/client/TestScannerTimeout.java      |   1 -
 .../apache/hadoop/hbase/client/TestShell.java |   1 -
 .../hbase/client/TestTimestampsFilter.java    |   1 -
 .../ColumnAggregationEndpoint.java            |   1 -
 .../ColumnAggregationProtocol.java            |   1 -
 .../coprocessor/SampleRegionWALObserver.java  |   1 -
 .../coprocessor/SimpleRegionObserver.java     |   1 -
 .../coprocessor/TestAggregateProtocol.java    |   1 -
 .../hbase/coprocessor/TestClassLoading.java   |   1 -
 .../coprocessor/TestCoprocessorEndpoint.java  |   1 -
 .../coprocessor/TestCoprocessorInterface.java |   1 -
 ...stMasterCoprocessorExceptionWithAbort.java |   1 -
 ...tMasterCoprocessorExceptionWithRemove.java |   1 -
 .../hbase/coprocessor/TestMasterObserver.java |   1 -
 .../TestMultiRowMutationProtocol.java         |   1 -
 .../coprocessor/TestRegionObserverBypass.java |   1 -
 .../TestRegionObserverInterface.java          |   1 -
 .../TestRegionObserverStacking.java           |   1 -
 ...nServerCoprocessorExceptionWithRemove.java |   1 -
 .../hbase/coprocessor/TestWALObserver.java    |   1 -
 .../hbase/executor/TestExecutorService.java   |   1 -
 .../hbase/filter/TestBitComparator.java       |  32 +--
 .../filter/TestColumnPaginationFilter.java    |   1 -
 .../filter/TestDependentColumnFilter.java     |   1 -
 .../hadoop/hbase/filter/TestFilter.java       |   1 -
 .../hadoop/hbase/filter/TestFilterList.java   |   1 -
 .../TestFuzzyRowAndColumnRangeFilter.java     |  27 +--
 .../hbase/filter/TestInclusiveStopFilter.java |   1 -
 .../hbase/filter/TestNullComparator.java      |  31 +--
 .../hadoop/hbase/filter/TestPageFilter.java   |   1 -
 .../hadoop/hbase/filter/TestParseFilter.java  |   1 -
 .../hadoop/hbase/filter/TestPrefixFilter.java |   1 -
 .../hbase/filter/TestRandomRowFilter.java     |   1 -
 .../TestSingleColumnValueExcludeFilter.java   |   1 -
 .../filter/TestSingleColumnValueFilter.java   |   1 -
 .../hbase/io/TestHalfStoreFileReader.java     |   1 -
 .../hbase/io/TestHbaseObjectWritable.java     |   1 -
 .../apache/hadoop/hbase/io/TestHeapSize.java  |   1 -
 .../hbase/io/TestImmutableBytesWritable.java  |   1 -
 .../hadoop/hbase/io/hfile/CacheTestUtils.java |   1 -
 .../hadoop/hbase/io/hfile/RandomSeek.java     |   1 -
 .../TestBlockCacheColumnFamilySummary.java    |   1 -
 .../hbase/io/hfile/TestCacheOnWrite.java      |   1 -
 .../hbase/io/hfile/TestCachedBlockQueue.java  |   1 -
 .../hbase/io/hfile/TestFixedFileTrailer.java  |   1 -
 .../hadoop/hbase/io/hfile/TestHFile.java      |   1 -
 .../hadoop/hbase/io/hfile/TestHFileBlock.java |   1 -
 .../io/hfile/TestHFileBlockCompatibility.java |   1 -
 .../hbase/io/hfile/TestHFileBlockIndex.java   |   1 -
 .../hbase/io/hfile/TestHFileReaderV1.java     |   1 -
 .../hbase/io/hfile/TestHFileWriterV2.java     |   1 -
 .../hbase/io/hfile/TestLruBlockCache.java     |   1 -
 .../hadoop/hbase/io/hfile/TestReseekTo.java   |   1 -
 .../hadoop/hbase/io/hfile/TestSeekTo.java     |   1 -
 .../io/hfile/slab/TestSingleSizeCache.java    |   1 -
 .../hadoop/hbase/io/hfile/slab/TestSlab.java  |   1 -
 .../hbase/io/hfile/slab/TestSlabCache.java    |   1 -
 .../hadoop/hbase/ipc/TestDelayedRpc.java      |   1 -
 .../hbase/mapred/TestTableInputFormat.java    |   1 -
 .../hbase/mapred/TestTableMapReduce.java      |   1 -
 .../hbase/mapreduce/NMapInputFormat.java      |   1 -
 .../mapreduce/TestHFileOutputFormat.java      |   1 -
 .../hadoop/hbase/mapreduce/TestImportTsv.java |   1 -
 .../mapreduce/TestLoadIncrementalHFiles.java  |   1 -
 .../TestSimpleTotalOrderPartitioner.java      |   1 -
 .../hbase/mapreduce/TestTableMapReduce.java   |   1 -
 .../hbase/mapreduce/TestTimeRangeMapRed.java  |   1 -
 .../hbase/master/TestActiveMasterManager.java |   1 -
 .../hbase/master/TestCatalogJanitor.java      |   1 -
 .../hbase/master/TestClockSkewDetection.java  |   1 -
 .../hbase/master/TestDefaultLoadBalancer.java |   1 -
 .../master/TestDistributedLogSplitting.java   |   1 -
 .../hbase/master/TestHMasterRPCException.java |   1 -
 .../hadoop/hbase/master/TestMaster.java       |   1 -
 .../hbase/master/TestMasterFailover.java      |   1 -
 .../TestMasterRestartAfterDisablingTable.java |   1 -
 .../hbase/master/TestMasterStatusServlet.java |   1 -
 .../hbase/master/TestMasterTransitions.java   |   1 -
 .../hbase/master/TestRestartCluster.java      |   1 -
 .../hbase/master/TestRollingRestart.java      |   1 -
 .../hbase/master/TestSplitLogManager.java     |   1 -
 .../master/TestZKBasedOpenCloseRegion.java    |   1 -
 .../hbase/master/cleaner/TestLogsCleaner.java |   1 -
 .../hbase/metrics/TestMetricsMBeanBase.java   |   1 -
 .../TestMemoryBoundedLogMessageBuffer.java    |   1 -
 .../hbase/monitoring/TestTaskMonitor.java     |   1 -
 .../regionserver/CreateRandomStoreFile.java   |   1 -
 .../regionserver/KeyValueScanFixture.java     |   1 -
 .../hbase/regionserver/OOMERegionServer.java  |   1 -
 .../TestBatchHRegionLockingAndWrites.java     |   1 -
 .../hbase/regionserver/TestBlocksRead.java    |   1 -
 .../TestCacheOnWriteInSchema.java             |   1 -
 .../hbase/regionserver/TestColumnSeeking.java |   1 -
 .../regionserver/TestCompactSelection.java    |   1 -
 .../hbase/regionserver/TestCompaction.java    |   1 -
 .../regionserver/TestCompoundBloomFilter.java |   1 -
 .../TestExplicitColumnTracker.java            |   1 -
 .../regionserver/TestFSErrorsExposed.java     |   1 -
 .../TestGetClosestAtOrBefore.java             |   1 -
 .../hbase/regionserver/TestHRegion.java       |   1 -
 .../hbase/regionserver/TestHRegionInfo.java   |   1 -
 .../hbase/regionserver/TestKeyValueHeap.java  |   1 -
 .../regionserver/TestKeyValueScanFixture.java |   1 -
 .../regionserver/TestKeyValueSkipListSet.java |   1 -
 .../TestMasterAddressManager.java             |   1 -
 .../hbase/regionserver/TestMemStore.java      |   1 -
 .../hbase/regionserver/TestMemStoreLAB.java   |   1 -
 .../hbase/regionserver/TestMinVersions.java   |   1 -
 .../regionserver/TestMultiColumnScanner.java  |   1 -
 .../hbase/regionserver/TestQueryMatcher.java  |   1 -
 .../regionserver/TestRSStatusServlet.java     |   1 -
 .../regionserver/TestResettingCounters.java   |   1 -
 .../hbase/regionserver/TestRpcMetrics.java    |   1 -
 .../regionserver/TestScanDeleteTracker.java   |   1 -
 .../TestScanWildcardColumnTracker.java        |   1 -
 .../regionserver/TestScanWithBloomError.java  |   1 -
 .../hbase/regionserver/TestScanner.java       |   1 -
 .../regionserver/TestSeekOptimizations.java   |   1 -
 .../TestServerCustomProtocol.java             |   1 -
 .../regionserver/TestSplitLogWorker.java      |   1 -
 .../regionserver/TestSplitTransaction.java    |   1 -
 .../TestSplitTransactionOnCluster.java        |   1 -
 .../hadoop/hbase/regionserver/TestStore.java  |   1 -
 .../hbase/regionserver/TestStoreFile.java     |   1 -
 .../TestStoreFileBlockCacheSummary.java       |   1 -
 .../hbase/regionserver/TestStoreScanner.java  |   1 -
 .../hbase/regionserver/TestWideScanner.java   |   1 -
 .../handler/TestOpenRegionHandler.java        |   1 -
 .../wal/FaultySequenceFileLogReader.java      |   1 -
 .../regionserver/wal/HLogUtilsForTests.java   |   1 -
 .../InstrumentedSequenceFileLogWriter.java    |   1 -
 .../hbase/regionserver/wal/TestHLog.java      |   1 -
 .../hbase/regionserver/wal/TestHLogBench.java |   1 -
 .../regionserver/wal/TestHLogMethods.java     |   1 -
 .../hbase/regionserver/wal/TestHLogSplit.java |   1 -
 .../regionserver/wal/TestLogRolling.java      |   1 -
 .../wal/TestWALActionsListener.java           |   1 -
 .../hbase/regionserver/wal/TestWALReplay.java |   1 -
 .../replication/ReplicationSourceDummy.java   |   1 -
 .../replication/TestMasterReplication.java    |   1 -
 .../replication/TestReplicationSource.java    |   1 -
 .../regionserver/TestReplicationSink.java     |   1 -
 .../TestReplicationSourceManager.java         |   1 -
 .../hbase/rest/HBaseRESTTestingUtility.java   |   1 -
 .../hbase/rest/PerformanceEvaluation.java     |   1 -
 .../hbase/rest/TestGZIPResponseWrapper.java   |   1 -
 .../hadoop/hbase/rest/TestGzipFilter.java     |   1 -
 .../hbase/rest/TestMultiRowResource.java      |   1 -
 .../hadoop/hbase/rest/TestRESTMetrics.java    |   1 -
 .../hadoop/hbase/rest/TestRowResource.java    |   1 -
 .../hbase/rest/TestScannerResource.java       |   1 -
 .../hbase/rest/TestScannersWithFilters.java   |   1 -
 .../hadoop/hbase/rest/TestSchemaResource.java |   1 -
 .../hadoop/hbase/rest/TestStatusResource.java |   1 -
 .../hadoop/hbase/rest/TestTableResource.java  |   1 -
 .../hbase/rest/TestVersionResource.java       |   1 -
 .../hbase/rest/client/TestRemoteAdmin.java    |   1 -
 .../hbase/rest/client/TestRemoteTable.java    |   1 -
 .../hbase/rest/model/TestCellModel.java       |   1 -
 .../hbase/rest/model/TestCellSetModel.java    |   1 -
 .../rest/model/TestColumnSchemaModel.java     |   1 -
 .../hadoop/hbase/rest/model/TestRowModel.java |   1 -
 .../hbase/rest/model/TestScannerModel.java    |   1 -
 .../model/TestStorageClusterStatusModel.java  |   1 -
 .../model/TestStorageClusterVersionModel.java |   1 -
 .../hbase/rest/model/TestTableInfoModel.java  |   1 -
 .../hbase/rest/model/TestTableListModel.java  |   1 -
 .../rest/model/TestTableRegionModel.java      |   1 -
 .../rest/model/TestTableSchemaModel.java      |   1 -
 .../hbase/rest/model/TestVersionModel.java    |   1 -
 .../hadoop/hbase/security/TestUser.java       |   1 -
 .../hadoop/hbase/thrift/TestThriftServer.java |   1 -
 .../TestThriftHBaseServiceHandler.java        |   1 -
 .../EnvironmentEdgeManagerTestHelper.java     |   1 -
 .../apache/hadoop/hbase/util/TestBase64.java  |   1 -
 .../hbase/util/TestByteBloomFilter.java       |   1 -
 .../apache/hadoop/hbase/util/TestBytes.java   |   1 -
 .../hbase/util/TestCompressionTest.java       |   1 -
 .../util/TestDefaultEnvironmentEdge.java      |   1 -
 .../util/TestEnvironmentEdgeManager.java      |   1 -
 .../apache/hadoop/hbase/util/TestFSUtils.java |   1 -
 .../hadoop/hbase/util/TestHBaseFsck.java      |   1 -
 .../hbase/util/TestHBaseFsckComparator.java   |   1 -
 .../apache/hadoop/hbase/util/TestIdLock.java  |   1 -
 .../util/TestIncrementingEnvironmentEdge.java |   1 -
 .../apache/hadoop/hbase/util/TestKeying.java  |   1 -
 .../hadoop/hbase/util/TestMergeTable.java     |   1 -
 .../hadoop/hbase/util/TestMergeTool.java      |   1 -
 .../apache/hadoop/hbase/util/TestPoolMap.java |   1 -
 .../hbase/util/TestRegionSplitCalculator.java |   1 -
 .../hadoop/hbase/util/TestRegionSplitter.java |   1 -
 .../hadoop/hbase/util/TestRootPath.java       |   1 -
 .../hbase/util/TestSortedCopyOnWriteSet.java  |   1 -
 .../hbase/zookeeper/TestHQuorumPeer.java      |   1 -
 .../hadoop/hbase/zookeeper/TestZKTable.java   |   1 -
 .../zookeeper/TestZooKeeperMainServerArg.java |   1 -
 .../zookeeper/TestZooKeeperNodeTracker.java   |   1 -
 src/test/resources/hbase-site.xml             |   1 -
 .../PerformanceEvaluation_Counter.properties  |   2 -
 src/test/ruby/hbase/admin_test.rb             |   1 -
 src/test/ruby/hbase/hbase_test.rb             |   1 -
 src/test/ruby/hbase/table_test.rb             |   1 -
 src/test/ruby/shell/commands_test.rb          |   1 -
 src/test/ruby/shell/formatter_test.rb         |   1 -
 src/test/ruby/shell/shell_test.rb             |   1 -
 src/test/ruby/tests_runner.rb                 |   1 -
 886 files changed, 876 insertions(+), 1403 deletions(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index d64569567334..b94f0c03adf9 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -200,3 +200,72 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+----
+This project incorporates portions of the 'Protocol Buffers' project avaialble
+under a '3-clause BSD' license.
+
+  Copyright 2008, Google Inc.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+      * Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+      * Neither the name of Google Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  Code generated by the Protocol Buffer compiler is owned by the owner
+  of the input file used when generating it.  This code is not
+  standalone and requires a support library to be linked with it.  This
+  support library is itself covered by the above license.
+
+--
+
+This project incorporates part of the 'FreeBSD Documentation Project'
+available under a BSD-style license.
+
+ * Copyright (c) 2001, 2003, 2010 The FreeBSD Documentation Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: doc/share/misc/docbook.css,v 1.15 2010/03/20 04:15:01 hrs Exp $
diff --git a/NOTICE.txt b/NOTICE.txt
index 337c93beafcd..dcaa169338be 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,20 +1,12 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
+Apache HBase
+Copyright 2007-2015 The Apache Software Foundation
 
-In addition, this product includes software developed by:
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
 
-Jamon (http://www.jamon.org/) is a text template engine for Java used by our
-UI.  It uses the Mozilla Public License (http://www.mozilla.org/MPL/)
-See the tail of http://www.jamon.org/About.html
+--
+This product incorporates portions of the 'Hadoop' project
 
-JUnit (http://www.junit.org/) included under the Common Public License v1.0.  See
-the full text here: http://junit.sourceforge.net/cpl-v10.html
+Copyright 2007-2009 The Apache Software Foundation
 
-JRuby (http://jruby.org) is tri-licensed.  We include it under terms of the
-Common Public License v1.0.
-
-JRuby itself includes libraries variously licensed.  See its COPYING document
-for details: https://github.com/jruby/jruby/blob/master/COPYING
-    
-The JRuby community went out of their way to make JRuby compatible with Apache
-projects: See https://issues.apache.org/jira/browse/HBASE-3374)
+Licensed under the Apache License v2.0
diff --git a/bin/get-active-master.rb b/bin/get-active-master.rb
index 8887a4574c14..6855cfcd7c40 100644
--- a/bin/get-active-master.rb
+++ b/bin/get-active-master.rb
@@ -1,6 +1,4 @@
 #!/usr/bin/env hbase-jruby
-# Copyright 2011 The Apache Software Foundation
-#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements. See the NOTICE file distributed with this
 # work for additional information regarding copyright ownership. The ASF
diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh
index 9a6244006495..80461a593851 100755
--- a/bin/graceful_stop.sh
+++ b/bin/graceful_stop.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2011 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/hbase b/bin/hbase
index 851698507e6e..158c7c8fd475 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -1,8 +1,6 @@
 #! /usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 966744ae52e6..0137db9219dd 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -1,7 +1,5 @@
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/hbase-daemon.sh b/bin/hbase-daemon.sh
index 9c49732bb29c..201548a286eb 100755
--- a/bin/hbase-daemon.sh
+++ b/bin/hbase-daemon.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/hbase-daemons.sh b/bin/hbase-daemons.sh
index 843eaaa74ffe..21ce635777d9 100755
--- a/bin/hbase-daemons.sh
+++ b/bin/hbase-daemons.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 32a51b3bf2f7..c8f13b5703d4 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -1,6 +1,4 @@
 #
-# Copyright 2009 The Apache Software Foundation
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
diff --git a/bin/local-master-backup.sh b/bin/local-master-backup.sh
index fc2efef1ce61..c945e2b39646 100755
--- a/bin/local-master-backup.sh
+++ b/bin/local-master-backup.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/local-regionservers.sh b/bin/local-regionservers.sh
index 31e90cc37084..29adcf396a33 100755
--- a/bin/local-regionservers.sh
+++ b/bin/local-regionservers.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/master-backup.sh b/bin/master-backup.sh
index a79c19e61c5e..feca4ab86572 100755
--- a/bin/master-backup.sh
+++ b/bin/master-backup.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2010 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index dc28331deef8..1ea785d4f1b3 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -1,5 +1,3 @@
-# Copyright 2011 The Apache Software Foundation
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
diff --git a/bin/regionservers.sh b/bin/regionservers.sh
index a86322de9b0c..fc96563b733a 100755
--- a/bin/regionservers.sh
+++ b/bin/regionservers.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/replication/copy_tables_desc.rb b/bin/replication/copy_tables_desc.rb
index 13cfaffb8dfc..6d4e9c8fc452 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -1,6 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 11a28c1fee15..362b29cd0c1c 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/start-hbase.sh b/bin/start-hbase.sh
index ca95f49ee631..aed729a808bf 100755
--- a/bin/start-hbase.sh
+++ b/bin/start-hbase.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/stop-hbase.sh b/bin/stop-hbase.sh
index e9749fee2889..5b2e69d6f21e 100755
--- a/bin/stop-hbase.sh
+++ b/bin/stop-hbase.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/bin/zookeepers.sh b/bin/zookeepers.sh
index 89a214e5a809..97bf41b60528 100755
--- a/bin/zookeepers.sh
+++ b/bin/zookeepers.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2009 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/conf/hadoop-metrics.properties b/conf/hadoop-metrics.properties
index 2060e22df5fb..4eb70a6911d3 100644
--- a/conf/hadoop-metrics.properties
+++ b/conf/hadoop-metrics.properties
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # See http://wiki.apache.org/hadoop/GangliaMetrics
 # Make sure you know whether you are using ganglia 3.0 or 3.1.
 # If 3.1, you will have to patch your hadoop instance with HADOOP-4675
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index 7c740104f761..39f3b74e54bb 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -1,7 +1,5 @@
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml
index af4c30095216..3ecd24c2cf4d 100644
--- a/conf/hbase-site.xml
+++ b/conf/hbase-site.xml
@@ -2,8 +2,6 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 24d7b2b798c1..2c5ed6a3f2c7 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Define some default values that can be overridden by system properties
 hbase.root.logger=INFO,console
 hbase.security.logger=INFO,console
diff --git a/dev-support/hbase_jdiff_acrossSingularityTemplate.xml b/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
index 4c70eae91a4f..e8c586765c68 100644
--- a/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
+++ b/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
@@ -1,16 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
diff --git a/dev-support/hbase_jdiff_afterSingularityTemplate.xml b/dev-support/hbase_jdiff_afterSingularityTemplate.xml
index 81adad017621..6c4cd9355202 100644
--- a/dev-support/hbase_jdiff_afterSingularityTemplate.xml
+++ b/dev-support/hbase_jdiff_afterSingularityTemplate.xml
@@ -1,16 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+  http://www.apache.org/licenses/LICENSE-2.0
 
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
diff --git a/dev-support/hbase_jdiff_template.xml b/dev-support/hbase_jdiff_template.xml
index 48b1ab50a687..21fe8ed1b299 100644
--- a/dev-support/hbase_jdiff_template.xml
+++ b/dev-support/hbase_jdiff_template.xml
@@ -1,16 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
diff --git a/dev-support/jdiffHBasePublicAPI.sh b/dev-support/jdiffHBasePublicAPI.sh
index b6adbbfed5bc..2000d2abd945 100644
--- a/dev-support/jdiffHBasePublicAPI.sh
+++ b/dev-support/jdiffHBasePublicAPI.sh
@@ -1,17 +1,22 @@
 #!/bin/bash
 set -e
 
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 ################################################ ABOUT JDIFF #######################################################
 #
diff --git a/dev-support/jdiffHBasePublicAPI_common.sh b/dev-support/jdiffHBasePublicAPI_common.sh
index 8c7b53ff9fa7..1cc99549b585 100644
--- a/dev-support/jdiffHBasePublicAPI_common.sh
+++ b/dev-support/jdiffHBasePublicAPI_common.sh
@@ -1,15 +1,20 @@
 #!/bin/bash
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 ##########################################################################################################################
 #
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
index cf1a7b06c319..9200e3ba921c 100755
--- a/dev-support/smart-apply-patch.sh
+++ b/dev-support/smart-apply-patch.sh
@@ -1,15 +1,20 @@
 #!/usr/bin/env bash
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index bdef3ab6a4fa..3c728454d035 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1,15 +1,20 @@
 #!/usr/bin/env bash
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 
 #set -x
diff --git a/dev-support/test-util.sh b/dev-support/test-util.sh
index 3101e10d03f0..9219bb96606c 100755
--- a/dev-support/test-util.sh
+++ b/dev-support/test-util.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 #
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/pom.xml b/pom.xml
index 5a5806e5469c..c0859e72ebe2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
     of commodity hardware.
   </description>
   <url>http://hbase.apache.org</url>
-
+  <inceptionYear>2007</inceptionYear>
   <scm>
     <connection>scm:svn:http://svn.apache.org/repos/asf/hbase/trunk</connection>
     <developerConnection>scm:svn:https://svn.apache.org/repos/asf/hbase/trunk</developerConnection>
@@ -324,6 +324,16 @@
          can be overwritten here. -->
     <pluginManagement>
       <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-remote-resources-plugin</artifactId>
+          <version>1.5</version>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <version>2.3</version>
+        </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-release-plugin</artifactId>
@@ -447,6 +457,11 @@
           <artifactId>avro-maven-plugin</artifactId>
           <version>${avro.version}</version>
         </plugin>
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>buildnumber-maven-plugin</artifactId>
+          <version>1.3</version>
+        </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>build-helper-maven-plugin</artifactId>
@@ -503,6 +518,35 @@
                     <execute></execute>
                   </action>
                 </pluginExecution>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-remote-resources-plugin</artifactId>
+                    <versionRange>[1.5,)</versionRange>
+                    <goals>
+                      <goal>process</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <execute></execute>
+                  </action>
+                </pluginExecution>
+                  <pluginExecution>
+                    <pluginExecutionFilter>
+                      <groupId>org.codehaus.mojo</groupId>
+                      <artifactId>buildnumber-maven-plugin</artifactId>
+                      <versionRange>[1.3,)</versionRange>
+                      <goals>
+                        <goal>create-timestamp</goal>
+                      </goals>
+                    </pluginExecutionFilter>
+                    <action>
+                      <execute>
+                        <runOnConfiguration>true</runOnConfiguration>
+                        <runOnIncremental>true</runOnIncremental>
+                      </execute>
+                    </action>
+                  </pluginExecution>
               </pluginExecutions>
             </lifecycleMappingMetadata>
           </configuration>
@@ -523,7 +567,8 @@
               <exclude>**/target/**</exclude>
               <exclude>**/CHANGES.txt</exclude>
               <exclude>**/generated/**</exclude>
-              <exclude>**/conf/*</exclude>
+              <!-- No material contents -->
+              <exclude>conf/regionservers</exclude>
               <exclude>**/*.avpr</exclude>
               <exclude>**/*.svg</exclude> <!-- vector graphics -->
               <exclude>**/*.vm</exclude> <!-- apache doxia generated -->
@@ -532,6 +577,8 @@
 	      <exclude>docs/*</exclude> <!-- auto-gen docs -->
               <!--It don't like freebsd license-->
               <exclude>**/src/site/resources/css/freebsd_docbook.css</exclude>
+              <!-- non-standard notice file from jruby included by reference -->
+              <exclude>**/src/main/resources/META-INF/LEGAL</exclude>
               <!--  exclude source control files -->
               <exclude>.git/**</exclude>
               <exclude>.svn/**</exclude>
@@ -550,6 +597,16 @@
           <include>hbase-default.xml</include>
         </includes>
       </resource>
+<!--
+      <resource>
+        <directory>src/main/resources/</directory>
+        <includes>
+          <include>META-INF/LICENSE</include>
+          <include>META-INF/NOTICE</include>
+        </includes>
+        <filtering>true</filtering>
+      </resource>
+-->
       <resource>
         <directory>${project.build.directory}</directory>
         <includes>
@@ -564,6 +621,15 @@
           <exclude>hbase-site.xml</exclude>
         </excludes>
       </testResource>
+<!--
+      <testResource>
+        <directory>src/main/resources/</directory>
+        <includes>
+          <include>META-INF/NOTICE</include>
+        </includes>
+        <filtering>true</filtering>
+      </testResource>
+-->
     </testResources>
 
     <plugins>
@@ -778,6 +844,16 @@
             <goals>
               <goal>jar-no-fork</goal>
             </goals>
+<!--
+            <configuration>
+              <excludes>
+                <exclude>META-INF/NOTICE</exclude>
+              </excludes>
+              <includes>
+                <include>${project.build.outputDirectory}/META-INF/NOTICE</include>
+              </includes>
+            </configuration>
+-->
           </execution>
         </executions>
       </plugin>
@@ -1068,10 +1144,131 @@
           </additionalConfig>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>buildnumber-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>validate</phase>
+            <goals>
+              <goal>create-timestamp</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <timestampFormat>yyyy</timestampFormat>
+          <timestampPropertyName>build.year</timestampPropertyName>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-remote-resources-plugin</artifactId>
+        <executions>
+          <!--
+            This is different from the default execution because we have
+            to build aggregate information for bundled dependencies.
+          -->
+          <execution>
+            <id>build-legal-for-assembly</id>
+            <!-- has to run before the default -->
+<!--
+            <phase>prepare-package</phase>
+-->
+            <phase>process-sources</phase>
+            <goals>
+              <goal>process</goal>
+            </goals>
+            <configuration>
+              <appendedResourcesDirectory>src/assembly</appendedResourcesDirectory>
+              <outputDirectory>${project.build.directory}/maven-shared-archive-resources-for-assembly</outputDirectory>
+              <resources>
+                <resource>src/assembly/resource</resource>
+              </resources>
+              <supplementalModels>
+                <supplementalModel>src/assembly/resources/supplemental-models.xml</supplementalModel>
+              </supplementalModels>
+              <properties>
+                <bundled-dependencies>true</bundled-dependencies>
+                <debug-print-included-work-info>${license.debug.print.included}</debug-print-included-work-info>
+              </properties>
+              <resourceBundles>
+                <resourceBundle>org.apache:apache-jar-resource-bundle:1.4</resourceBundle>
+              </resourceBundles>
+              <attach>false</attach>
+              <attachToMain>false</attachToMain>
+              <attachToTest>false</attachToTest>
+            </configuration>
+          </execution>
+<!--
+          <execution>
+            <id>default</id>
+          </execution>
+-->
+        </executions>
+      </plugin>
+      <!--
+         Build an aggregation of our templated NOTICE file and the NOTICE files in our dependencies.
+         If MASSEMBLY-382 is fixed we could do this in the assembly
+         Currently relies on env, bash, find, and cat.
+      -->
+      <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <!-- put all of the NOTICE files out of our dependencies -->
+            <id>unpack-dependency-notices</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>unpack-dependencies</goal>
+            </goals>
+            <configuration>
+              <useSubDirectoryPerArtifact>true</useSubDirectoryPerArtifact>
+              <includes>**\/NOTICE,**\/NOTICE.txt</includes>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.4.0</version>
+        <executions>
+          <execution>
+            <id>concat-NOTICE-files</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>env</executable>
+              <arguments>
+                <argument>bash</argument>
+                <argument>-c</argument>
+                <argument>cat maven-shared-archive-resources-for-assembly/META-INF/NOTICE \
+                  `find ${project.build.directory}/dependency -iname NOTICE -or -iname NOTICE.txt` \
+                </argument>
+              </arguments>
+              <outputFile>${project.build.directory}/NOTICE.aggregate</outputFile>
+              <workingDirectory>${project.build.directory}</workingDirectory>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <!-- /end building aggregation of NOTICE files -->
+      <plugin>
+        <groupId>org.apache.felix</groupId>
+        <artifactId>maven-bundle-plugin</artifactId>
+        <version>2.5.3</version>
+        <inherited>true</inherited>
+        <extensions>true</extensions>
+      </plugin>
     </plugins>
   </build>
 
   <properties>
+    <!-- override on command line to have generated LICENSE files include
+         diagnostic info for verifying notice requirements -->
+    <license.debug.print.included>false</license.debug.print.included>
     <maven.build.timestamp.format>
       yyyy-MM-dd'T'HH:mm
     </maven.build.timestamp.format>
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 62faa691784d..97b08498fa30 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -1,15 +1,20 @@
 /*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase.security.access;
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
index 946a97ed2647..1a93c47295b6 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
index 4f58a465954e..63f45fd3f35c 100644
--- a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
+++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/security/src/test/resources/hbase-site.xml b/security/src/test/resources/hbase-site.xml
index cca8832b5a3d..4f1dd5f15d0c 100644
--- a/security/src/test/resources/hbase-site.xml
+++ b/security/src/test/resources/hbase-site.xml
@@ -2,8 +2,6 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
- * Copyright 2007 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/assembly/all.xml b/src/assembly/all.xml
index 11d26f13a650..dc01b7bb4dac 100644
--- a/src/assembly/all.xml
+++ b/src/assembly/all.xml
@@ -33,6 +33,11 @@
       <includes>
         <include>${basedir}/*.txt</include>
       </includes>
+      <!-- version for assmebly generated by build -->
+      <excludes>
+        <exclude>${basedir}/NOTICE.txt</exclude>
+        <exclude>${basedir}/LICENSE.txt</exclude>
+      </excludes>
     </fileSet>
     <fileSet>
       <includes>
@@ -108,4 +113,26 @@
       <directoryMode>0644</directoryMode>
     </dependencySet>
   </dependencySets>
+  <!-- Include the generated LICENSE and NOTICE files -->
+  <files>
+    <file>
+      <source>${project.build.directory}/maven-shared-archive-resources-for-assembly/META-INF/LICENSE</source>
+      <outputDirectory>.</outputDirectory>
+      <destName>LICENSE.txt</destName>
+      <lineEnding>unix</lineEnding>
+    </file>
+    <file>
+      <source>${project.build.directory}/NOTICE.aggregate</source>
+      <outputDirectory>.</outputDirectory>
+      <destName>NOTICE.txt</destName>
+      <lineEnding>unix</lineEnding>
+    </file>
+    <file>
+      <source>${basedir}/src/main/resources/META-INF/LEGAL</source>
+      <outputDirectory>.</outputDirectory>
+      <destName>LEGAL</destName>
+      <lineEnding>unix</lineEnding>
+    </file>
+  </files>
+
 </assembly>
diff --git a/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java
index 31c1b38dfe47..0d092d090c27 100644
--- a/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java
+++ b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2009 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java
index 5629ccac1946..108d65283345 100644
--- a/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java
+++ b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2009 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/examples/thrift/DemoClient.cpp b/src/examples/thrift/DemoClient.cpp
index 06cbc4460d17..e845669cd89d 100644
--- a/src/examples/thrift/DemoClient.cpp
+++ b/src/examples/thrift/DemoClient.cpp
@@ -1,6 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/examples/thrift/DemoClient.java b/src/examples/thrift/DemoClient.java
index bb03fccd4338..036d7fd022ad 100644
--- a/src/examples/thrift/DemoClient.java
+++ b/src/examples/thrift/DemoClient.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2008 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/examples/thrift/DemoClient.php b/src/examples/thrift/DemoClient.php
index 669f2b6fc2fd..93d79d43f509 100644
--- a/src/examples/thrift/DemoClient.php
+++ b/src/examples/thrift/DemoClient.php
@@ -1,7 +1,5 @@
 <?php
 /**
- * Copyright 2008 The Apache Software Foundation
- * 
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/examples/thrift/DemoClient.pl b/src/examples/thrift/DemoClient.pl
index 17d92fad2f0a..a80cca4245a8 100644
--- a/src/examples/thrift/DemoClient.pl
+++ b/src/examples/thrift/DemoClient.pl
@@ -1,6 +1,4 @@
 #!/usr/bin/perl
-#   Copyright 2011 The Apache Software Foundation
-#  
 #   Licensed to the Apache Software Foundation (ASF) under one
 #   or more contributor license agreements.  See the NOTICE file
 #   distributed with this work for additional information
diff --git a/src/examples/thrift/DemoClient.py b/src/examples/thrift/DemoClient.py
index eabbbe84371d..723d7a894c29 100755
--- a/src/examples/thrift/DemoClient.py
+++ b/src/examples/thrift/DemoClient.py
@@ -1,6 +1,5 @@
 #!/usr/bin/python
-'''Copyright 2008 The Apache Software Foundation
- 
+'''
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
   distributed with this work for additional information
diff --git a/src/examples/thrift/DemoClient.rb b/src/examples/thrift/DemoClient.rb
index 2b7b5e7f934d..ea5acad926b2 100644
--- a/src/examples/thrift/DemoClient.rb
+++ b/src/examples/thrift/DemoClient.rb
@@ -1,7 +1,5 @@
 #!/usr/bin/ruby
 
-# Copyright 2008 The Apache Software Foundation
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
diff --git a/src/examples/thrift/Makefile b/src/examples/thrift/Makefile
index 691a1e981ae9..da2f3010a302 100644
--- a/src/examples/thrift/Makefile
+++ b/src/examples/thrift/Makefile
@@ -1,5 +1,3 @@
-# Copyright 2008 The Apache Software Foundation
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
diff --git a/src/examples/thrift2/DemoClient.java b/src/examples/thrift2/DemoClient.java
index d5b805c83e21..828b4c01d357 100644
--- a/src/examples/thrift2/DemoClient.java
+++ b/src/examples/thrift2/DemoClient.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/examples/thrift2/DemoClient.py b/src/examples/thrift2/DemoClient.py
index 67abc5b5be04..3a3ebbbbbc93 100644
--- a/src/examples/thrift2/DemoClient.py
+++ b/src/examples/thrift2/DemoClient.py
@@ -1,6 +1,4 @@
 """
-  Copyright 2011 The Apache Software Foundation
-  
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
   distributed with this work for additional information
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon
index 4379ef59b5cd..0304154c00b6 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon
@@ -1,6 +1,4 @@
 <%doc>
-Copyright 2011 The Apache Software Foundation
-
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
index 0dc0691a894c..e06c5e6124f5 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
@@ -1,6 +1,4 @@
 <%doc>
-Copyright 2011 The Apache Software Foundation
-
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 6af4025ec7f9..c1fffedcc124 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -1,6 +1,4 @@
 <%doc>
-Copyright 2011 The Apache Software Foundation
-
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
index ee17852b1085..15455b566d56 100644
--- a/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
+++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
@@ -1,6 +1,4 @@
 <%doc>
-Copyright 2011 The Apache Software Foundation
-
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/Abortable.java b/src/main/java/org/apache/hadoop/hbase/Abortable.java
index 03249c121718..ec2e0fcf1ca3 100644
--- a/src/main/java/org/apache/hadoop/hbase/Abortable.java
+++ b/src/main/java/org/apache/hadoop/hbase/Abortable.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/Chore.java b/src/main/java/org/apache/hadoop/hbase/Chore.java
index 1581b0c04d2f..924f4eb2c855 100644
--- a/src/main/java/org/apache/hadoop/hbase/Chore.java
+++ b/src/main/java/org/apache/hadoop/hbase/Chore.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java b/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java
index 5c51e4bb8b5a..ef93c5a20a7a 100644
--- a/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java
+++ b/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
index d962594f8e0b..96248a9f98d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
+++ b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2009 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/Coprocessor.java b/src/main/java/org/apache/hadoop/hbase/Coprocessor.java
index c0cb46367154..b29c53114007 100644
--- a/src/main/java/org/apache/hadoop/hbase/Coprocessor.java
+++ b/src/main/java/org/apache/hadoop/hbase/Coprocessor.java
@@ -1,17 +1,20 @@
 /*
- * Copyright 2010 The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase;
diff --git a/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java
index 09bca1153bc1..501ecf9aab7e 100644
--- a/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java
+++ b/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java
@@ -1,17 +1,20 @@
 /*
- * Copyright 2010 The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase;
@@ -57,4 +60,4 @@ public interface CoprocessorEnvironment {
    * @throws IOException
    */
   public HTableInterface getTable(byte[] tableName, ExecutorService service) throws IOException;
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java b/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java
index 5a0c6d2b6957..7b24b16a5dcd 100644
--- a/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java
+++ b/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2008 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java
index 9b1d021c853a..3b344a33a324 100644
--- a/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java
+++ b/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java
@@ -1,17 +1,20 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase;
 import java.io.IOException;
@@ -38,4 +41,4 @@ public DroppedSnapshotException(String msg) {
   public DroppedSnapshotException() {
     super();
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java b/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java
index e0e0a288992d..03d79e2e4193 100644
--- a/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2009 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index 590e774f9c9c..29617def62b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index dca442d049ff..00b50c67ad49 100644
--- a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 6821850820ec..75202cbe9741 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
index d0d1dd0d33fa..27892f447721 100644
--- a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
+++ b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 53f672082e42..641c07d7936d 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
index 85fb91dc7d0a..c0ee5cb5f33c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
index e189aaf1a56f..59b95f3b26e6 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
index 59342462868a..cb2d9c3a2e3e 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
index ffdbc6bf6515..bf41c3bbdc4e 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index edea63152a74..50f39ffdc79f 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java b/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java
index bae41f6a0048..c3797504a777 100644
--- a/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java
+++ b/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/HealthChecker.java b/src/main/java/org/apache/hadoop/hbase/HealthChecker.java
index e77317c81db5..bc291a9a584c 100644
--- a/src/main/java/org/apache/hadoop/hbase/HealthChecker.java
+++ b/src/main/java/org/apache/hadoop/hbase/HealthChecker.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java b/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
index bb2b6669d879..d08da460ffc4 100644
--- a/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
+++ b/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 69412a217c9a..0e8ee9037d80 100644
--- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index 315309f94e67..3d9c6b840d19 100644
--- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java b/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
index 2fdc6ba3c7ca..5277c2835992 100644
--- a/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java b/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java
index 6cf564c7bba6..56d359f4e094 100644
--- a/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java
+++ b/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java b/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java
index 2c275e3586a4..ec3d9262193b 100644
--- a/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java
+++ b/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java b/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java
index 32da8cb7ff07..a2a5c71c0d80 100644
--- a/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
index 309dca413437..15410a7f5eb1 100644
--- a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
+++ b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/RegionException.java b/src/main/java/org/apache/hadoop/hbase/RegionException.java
index 63063a5e95ec..39b4f940be0e 100644
--- a/src/main/java/org/apache/hadoop/hbase/RegionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/RegionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java b/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java
index 485c254cd408..4a1f7bfbeb7b 100644
--- a/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/Server.java b/src/main/java/org/apache/hadoop/hbase/Server.java
index de19e2c2a912..4f5815e8736a 100644
--- a/src/main/java/org/apache/hadoop/hbase/Server.java
+++ b/src/main/java/org/apache/hadoop/hbase/Server.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ServerName.java b/src/main/java/org/apache/hadoop/hbase/ServerName.java
index 2d6fe4e9e87b..c3c80692c3b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/ServerName.java
+++ b/src/main/java/org/apache/hadoop/hbase/ServerName.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/Stoppable.java b/src/main/java/org/apache/hadoop/hbase/Stoppable.java
index 74d4f4a689f2..4e345b2934b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/Stoppable.java
+++ b/src/main/java/org/apache/hadoop/hbase/Stoppable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/TableExistsException.java b/src/main/java/org/apache/hadoop/hbase/TableExistsException.java
index 5fde21920c43..cd8c5788c437 100644
--- a/src/main/java/org/apache/hadoop/hbase/TableExistsException.java
+++ b/src/main/java/org/apache/hadoop/hbase/TableExistsException.java
@@ -1,17 +1,20 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java b/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java
index 42878001a7e1..d8f21d2596f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java
+++ b/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java b/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java
index d0392d52f6e3..40c67c4f7ed9 100644
--- a/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java
+++ b/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java b/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java
index dc6da43e4702..aa93ad4fe9bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java
+++ b/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java
index e87f42a60137..c545959aca46 100644
--- a/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
index 675e6e2b5cdc..9c0ec6b50ec9 100644
--- a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
+++ b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java b/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java
index 13f2f6c3953a..07ba065e3348 100644
--- a/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java
+++ b/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java b/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java
index fcd2ccdd2ed6..6c55619387fd 100644
--- a/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java
+++ b/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java b/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
index ad48b25a8225..7492ffd5752e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
index 1cbf1b6561bd..a0836d89cd77 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Action.java b/src/main/java/org/apache/hadoop/hbase/client/Action.java
index 40b0f2ef54aa..b3ee61c6f687 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Action.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Action.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Attributes.java b/src/main/java/org/apache/hadoop/hbase/client/Attributes.java
index 5999e3a1b166..014db380f987 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Attributes.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Attributes.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index c147f2fbf7bd..ed032cc55c2e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index ad9a058831a5..feb782a2bd26 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index c30b676ad7af..201a5d8279ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index da8078c45c53..38aeed802136 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 7b688698f4c4..7427bc0d90eb 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 309732f4f41a..948a615f16d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java b/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java
index 90f6cb9450fc..ed7d8504808d 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
index e7e296c53879..20cbfb51a0a7 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java
index ab7efcc55294..a1c7e721ee19 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
index 05016e5127f3..e757528ebd8c 100755
--- a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java b/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java
index bc0872af8046..d8286b9f976c 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Increment.java b/src/main/java/org/apache/hadoop/hbase/client/Increment.java
index 52fac0479ed8..659296bb1a66 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Increment.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Increment.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index be48fc6d8e08..9c4a2157e74a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2009 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java b/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java
index 6a864c88be91..5465c45471d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java b/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java
index 9235e2dabad4..cd21455f8f33 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java b/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java
index 7e0311a04f2e..90949e04d20f 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java b/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java
index 290e4c71f8d3..4db8c6b18128 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 32c9e6f98a8e..93883b14d920 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java b/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java
index 4f3391443678..da2454ab1ea3 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Operation.java b/src/main/java/org/apache/hadoop/hbase/client/Operation.java
index dedd2e26ed40..c44c571f9723 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Operation.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Operation.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java b/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
index 97e19b91ede2..777a47501899 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java
index e8ac0a8bdcb0..7630522af363 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Put.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java b/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java
index d223860054df..022bc36bbdee 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 05f528c4787b..9e78fa9e6dde 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java
index 684301893e4c..9b23df57148a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
index b9042f623f63..7df0cff5d784 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
@@ -1,17 +1,20 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase.client;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
index ea14c068649b..c8dd281b5f53 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Row.java b/src/main/java/org/apache/hadoop/hbase/client/Row.java
index cd332bd57f45..7e677ebd40b5 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Row.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Row.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/RowLock.java b/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
index 5888ec844096..93ba647370d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 4ec4a85937d5..7906709f8117 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java
index 5a10b0ed116e..2706bef9dee9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
index 1dd6943e78fc..ea126bddd0b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java
index 301ea12b52bb..9c397995fd9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java
index 412f770ac03b..de14cc1084bd 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java
index 27d1faa4c5fd..a130924b6b67 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
index 4974a298a7b7..3f459428f93e 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java
index d430a3893389..20c5854ca1d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
index cafa8b050404..2652a280ba3b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java
index e4812aff2d9e..8c62d28ed7f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
index c37b5fde8de5..2d5067d59d36 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java
index 8b0fbdf4b309..1549f6c75531 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/package-info.java b/src/main/java/org/apache/hadoop/hbase/client/package-info.java
index b00a55629cd8..d4a8f75c46bf 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/package-info.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index 2a0f0caf2a36..b83935ee0fa9 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
index f6f4b42a3441..65ad1435265a 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java
index e654c0cb9910..9722e999a98a 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java
index 3a787fdf55db..8160fd7489e4 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
index 21fe91e9b23c..5329aacf3b7a 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index ed925f9c1d27..9b51f52f4176 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -1,17 +1,20 @@
 /*
- * Copyright 2011 The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase.coprocessor;
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
index 149fe2134647..f48cd92a38b4 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java
index d4344d0bc6ff..5423f4e29bda 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 24d7519cffd8..0426d30fea08 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
index 75f06536d0f0..c05e4f6331d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index c2aa3bb85009..787d1fa01598 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java
index d5cf6aa0c19f..a8acc56e8e6a 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
index 35a5c19f50e4..4a14e152e47c 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index f11b4edf8d35..baf782e014af 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -1,17 +1,20 @@
 /*
- * Copyright 2010 The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase.coprocessor;
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java
index 2790abddd46b..f6616ceca028 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
index c90189d3e1ea..6ff529cab737 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java
index f92a24581053..6669b24410c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
index 46dafe0d8009..afcd07413297 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
index a34dd691adbd..c0189d144231 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
index 2f5f0924f463..c284e6d0a2b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
+++ b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java
index 1e56948c3c39..99937d71ee20 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java
index 030341a307a1..836441d6435c 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java
index 088a0997e10d..858b65c9e186 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
index 2bca1c4be021..280e596588a1 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
index 1214a1afbd5f..143f83149dc2 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java
index cb89b7d367e4..3ed3e32c5f21 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java
index bd5c57357c0f..4df7ff6bbfc8 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java
index 9089f4b559ee..c54a1b8fa0fe 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
index 2d7dd3e02ec2..3f27eccc1d7a 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java
index 63ec44a6a276..fe47844f4c3b 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
index 3265b380ba05..b5bb0f88232d 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
index 291b5dd9cf4e..b0056d9f2afc 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
@@ -1,20 +1,20 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance                                                                       with the License.  You may obtain a copy of the License at
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase.filter;
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index d676655f33b2..604404510cc7 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java
index 7a068b4d9810..a60373ee93c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
index 0f8740b90094..f028a96c55fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java b/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
index 75edf192cfbd..2dac4e5c27f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java b/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
index 14b8e31cfd5b..0e90ac2a17f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
index 5cf659ce4d26..804873fc1952 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java
index 9a7f770d4b1b..f17de0324b4b 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java
index b00205ab0cd5..8d3f717a1b65 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java b/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java
index 373d7a68fbd9..eebc92d09b5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java
index 409d41ef83a9..4108fe424e76 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
index 4e93925f6890..b494e3233fdd 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java
index cd69277611c6..bb933d44ffc4 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java
index c23ac9b2fcf9..63d54c7558ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
index 98c693e2aee5..86d3945fe472 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java
index 3b00efd2e7e7..882309675e44 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java
index 638629cea638..de62b4661fd2 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
index 4d2d504489b9..a8929350deca 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
index 5fe17dfe176c..94da090af2d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java
index 90bc718a3de6..147b8af612d7 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
index 24521296c6e0..7996e967e864 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
index bed8d5834df8..5924277464d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java b/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java
index c7731e3fde7e..0559df393f80 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java
@@ -1,6 +1,4 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/package-info.java b/src/main/java/org/apache/hadoop/hbase/filter/package-info.java
index 73ccef8ec3ac..b19e7e1603bf 100644
--- a/src/main/java/org/apache/hadoop/hbase/filter/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/filter/package-info.java
@@ -1,6 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java b/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java
index 61a1c5e5b872..0b9ae7ccb2fb 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/DoubleOutputStream.java b/src/main/java/org/apache/hadoop/hbase/io/DoubleOutputStream.java
index cf33cd32e64f..6ec78d073521 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/DoubleOutputStream.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/DoubleOutputStream.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index e07f37056da9..c46b5f34f897 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java
index 45eb49512f65..f252c3b3cabc 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java b/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java
index bd788469ab24..bb2b060bab95 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java b/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
index 1ed73d16df3c..98f5d6c674b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
index 06dc5041ba25..f9da773ed4c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/Reference.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java b/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
index 12a9b6814d3d..250a5f4d2b57 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java b/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java
index f8aefa101252..024bd0066f55 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
index 3c9aab0da1db..0b60031a8b50 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
@@ -1,5 +1,4 @@
   /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
index e3908b321318..717db22dbe22 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
index 34513f1825e6..c171969264e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
index 5797694a8d02..f10c53d3892d 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
index 439d431d3081..ee1ac327f4d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java
index 13d3b4e3b8b7..d1f47b1524c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
index a66ab8512200..e254e8bc9c4d 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
index 1637fbfb9caa..554a524b9976 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
index 0a6c062dffcc..9084d5ee2c92 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
index 693643c74e70..ebcc17787f3f 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 89924640d9ed..fddfc81d50e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index 8611b11ac725..92bc16e1b598 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index fead56cb84c3..d5001e376382 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -1,6 +1,5 @@
 
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
index 2f40db9a7679..b81390adbf7c 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index ad90663d0b51..8cf2ec3ede6f 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
index b06878f68a33..2e04face75df 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
index ed0a1ba28662..be135d77516a 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index ceb474519ec9..c959f830b1a0 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java
index c3840362da14..ead0830a6f16 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 6533c9333886..04e3e8cd4ab0 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
index a14909be5fca..5fc6c0f14d00 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
index d68c2d5f434f..98c5013a78e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java
index ed32980e2b40..645e446f8e19 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
index 9fc60433d9e8..d1f99a2fbd27 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java
index dfd727f8ad73..1de8b730a5c0 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java b/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java
index 632831016300..062aa7457e9d 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java b/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java
index 8211f037fcb1..70942f18aa98 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java b/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java
index 04eb02d9cb8e..8ca5bcd58a46 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java b/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
index 8a3a45f39ac3..9d9f0b0752de 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 5c33cc688efb..48b1acde94a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
index 27884f9d0fd9..005efc032cf3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java
index ad790b55828f..37d72bfafd25 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
index b9fa0dd21d58..042301d9f89e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
index c6cf6b31fd2a..2f0c5d1470b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
index 417e2a248218..2caa2ca50c22 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index 3dc6ba067b6e..037fbfe90988 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
index dfb913353ed1..db33804258a3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index 8eae0ec6fe20..412df105233f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
index 10065709b762..6125d373c0ff 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
index faf725c43ed2..e065d92be46e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 716fa229c409..65772b49e965 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java b/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java
index 7dd9b19541ff..840ce9340206 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
index e8ca2f6887dc..6c73cea1f76b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java b/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java
index d38956c46df4..ddab06904caf 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java b/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
index c368140e6b35..d0e2c1611a54 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
index b58c5c7fcd56..4bd729e684d3 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
index 0f67a9e7626b..e16f6da62953 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
index be0a6bd80893..02e9c6003e49 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java b/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java
index ac50bd796b07..b793484abafb 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
index 395a626b1c03..ea72a6398ba3 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
index a959e5db8969..a7d725594abc 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java
index 597f3efc4a9f..fa02c63a8677 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index 13a304383428..51d4142addf9 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
index 732de9704ecd..494f441a75d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java
index 7133860b2f55..5dec23de7e38 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
index 72e5774afcdd..992b6af17ee7 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java
index 155ce82ca768..fc46f8f2bb21 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java
index 5956ee8e046b..3c85f7960d5f 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java b/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java
index cc5228ae59f6..5a51d6c76b02 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
index 46d8c7179bd7..030d0c2ad9c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 025473b74d18..68a60688621c 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
index dda4241698a8..8e34c08e0e9a 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
index 2c53f6d8161d..81f822e7ce07 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
@@ -1,5 +1,4 @@
 /**
-* Copyright 2009 The Apache Software Foundation
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java
index c38337b2ce95..28fe45a5b33d 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
index f67afedc33e1..ccf51bbef8b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
index 363ba51f9b58..76a55ad95965 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java
index fd5d8fe6f689..a63df8aacff5 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java
index 25f466e86c48..654a90043451 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index d1ceaff45c01..70db074f790e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index 3e0b3b4f0edd..1d5970f992bf 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
index abc356f85f11..e785f523eab1 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 0f476d03d82d..eafbe4aa592e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
index 81d27466cf04..f647dfa10636 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index da621af28d16..b18596141ead 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
index ce2135fb1262..d67f59a2669b 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
index a7524cb95aa2..9a0c0cdc68b6 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
index 58bd78c7928e..57a5df03ec08 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index ad855e215b56..93628cba94d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 5ea583177b62..33cd3a6b7708 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java
index bbceb63dd6ce..6ac4063c4037 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
index 90f066af404b..f167d7676916 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
index 89c9603335ae..775b308802e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java
index a55f82a49737..ffb3490c0422 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java
index d087f854abde..098a57cf8203 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
index bb9fb46cf382..864059ccf289 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
index 14e7949609de..4aecb40677be 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
@@ -1,15 +1,20 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
index 62d429665251..330895b8a42e 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 8ad86d38e778..c5bf89397e40 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 198224aa0d5e..34d87ec5ce98 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java b/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java
index b233d105bab5..b0fc16aebadc 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 31d8014cebdf..b098433ec72c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java b/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java
index 588cf3581775..f36b11ba1e5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java b/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java
index 37e22cc6190a..7839ae9d9870 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index ef6dad0e501a..dd5e440dd60b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
index 26e57144554a..5926cdfb0845 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
index acc7d4e9d053..56b5fa49938d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 82fd1f65d3e3..8febb0dc904c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index e6fcce54cc5e..1b13d9495002 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
index 7d2dd74a569e..8896b07ae76c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java
index 3a22e468dd4a..b02f2cd49d55 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index bfc59831e66e..5e6f50504214 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
index 608979a30188..b0f6109ad01c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index acce4b559da3..05d875291257 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 3eefdfcff2ce..7e4681946b69 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
index 5f37aa455988..316f70510de1 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 1391b6b409f0..e94a527937cf 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 31981e7f5057..50539a6191d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java b/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java
index 2cbe7e08cb5a..e09d9a9598e9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
index 88f207a1af49..d8965c1e7fb9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index 2cb76f3c31c5..813418931dd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
index fcbfb82dc69a..74984d31f711 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
index 42d49175eae9..b554a4395083 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
index a241e2058424..4a369c215ef4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
index a6859f0e6881..affdc5c8145a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
index f171a5a3b8db..bea640a8a2ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 329404528c75..817eb1993a6c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
index 2d544dd155f9..10c3e6e30d5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
index cfad4037b437..5ec0c9742b17 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
index b520762a31a1..141fe4d46ae9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
index b73f25db2d21..213ae736da3b 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
index 34560ec674ce..d0f5a44690be 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
index d08f64982fca..1485e24579bc 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java b/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
index 38ed87256cf9..715f7b076e39 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
index 00d514c57e49..7f98311a2426 100644
--- a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java b/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java
index eeb18e8793b1..36bea094a85e 100644
--- a/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java
+++ b/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java b/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java
index d121ee1f4ef8..4451d538f6ca 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java
index e8b7416843b8..745bde2fec49 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java
index d4f97144d87c..2914603b3425 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
index d68468bc812b..783d5b933dc5 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java
index c59d4153797d..d3a64a4f31e9 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
index 394129c81507..0913ad018f19 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java
index 604f10d9a500..cb714a1ad390 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
index dbdadccd4177..e91ff56dd54f 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java b/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java
index a3fa7064ef5c..4dafd5e184e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
index 82894e2dbb70..a79b0cc62dd5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java
index a617d687fa90..893bb828acb8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
index c192e9e96317..43cd5863bdd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index e1ebe6d8c84a..9ea71c54464a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
index 978854fedc7d..9aef831c3621 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java
index e1d69c7ce5d3..1338dc78aab4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java
index 29630845423f..5ca8ff0253a6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index f953f7ae7943..192b383525cc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
index aecd9c383159..18aa3b5061a5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
index 3a26bbb9f236..30a5f634fe1c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 8f2037644ad0..9727a18a203e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5499135fd468..25e1bb92b34c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
index 71b9985e188a..c22bbf3865f6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
index 7acfe9bd09e2..978b12f4154f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
index 000c109eac18..773519be14f7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
index 6cbed5aeebf2..d8c94e639af7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 3941843b90a3..b659fb3d5cfb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
index 6a7d5c68b964..c272d4f63b8e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
index 20feacf8d6fc..5dd578909497 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
index cafbb28496df..bb1b2aed4f1c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
index a8437364a9b9..f5d9ea30c6f6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
index c5185210d471..f6e951286e6c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index acdd62f043c5..f53761a1bb8d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java
index 161ae189e8c7..d76fceff17f2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 8b8a5b6be111..96784074b948 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index 04f89421b1c3..b7682604d8f2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
index cbb76e8be096..7183d9b6e44d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
index 6b28f03436fa..d5bf897bcfeb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java
index 4881fc0f1a28..405f184918c5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
index 6534e2c754fd..af8b0d43272e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
index 847efb68d3f7..660b8f74ba27 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java b/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
index 1b94ab5540eb..f264ce49e026 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
index 2df824b02922..025161a1ba1e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
index 824c79dcc177..4e4c82043e9f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java
index 1c21825f8a04..91f3f7b6be99 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 6efb8dcd1492..008b6a9ab8d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java
index c5bcb4c8cbbc..0a9fd3b65faa 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
index 7b6762cbd399..6edeb33390b7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index 3fb7c0d28e4a..96977365c2b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java
index ed36ed79abcd..79b348f7676c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 5927af9ee526..1cfe5ba70de7 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java
index 45acb177ba27..f68f9666dd74 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java
index 83ffcc70d9a8..16eea09e8651 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java
index 1d221925e518..7f051109fa6a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
index 9e16d022b9df..6393e5697a2f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
index cc26e3a5a438..acae56aa7509 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index fec739d591a6..e092827b957c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
index 53dd45517a15..3f56a25e3109 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
index 483c4b9e5018..67c49f7d2e65 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
index 0dbc4f9e2a35..ed708c823bf9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
index 11acee5c3238..ac3775add30c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 356110d28f7d..091a27a706ab 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 3bd9f1173842..366a2c1bcb3f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 233f843ac364..540826153ccc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index 1594a0aa9aee..33571f0c9fbb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java
index d2eb697bc8a1..fb0dd2521e90 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index c4ee384ef208..68a2cb8968d2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
index 37358b39b983..81214a29b079 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java
index 52b9a6c05dbe..744f0e7c6a30 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java
index 9b335785db71..b2143a38ad57 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
index f91b7822c4ad..53bb249435c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
index 65f14beb9782..e8a7eee84ebf 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
index 83544e2f0e9b..5b2522055a30 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
index 286a6dae1ef0..68319f84b09f 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java
index 4cfeec6549d0..899ae01f27fe 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
index 66e57069d15e..7c044a89bbf4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index d7edc300c559..0432684ed6d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java
index 9a4f01a838fd..ed48700b1aca 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
index cd76211419d9..43200f1d39a0 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java
index ea96cf50a506..296db55b5260 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index 1a4d978d55c8..70c17dab9679 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java
index 04fe7b10e443..83c832bf4a94 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java
index 393b1d237dc4..f6ab59954e43 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 980e1e56b2ea..ccc4bca7b4d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
index 95114811c366..2992538f7c6a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
index 404caa4808fc..d30060876908 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java
index 1c93def668e7..17c51ed076c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
index 9420dde688dd..f31e5f6cba26 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
index e224658a0a4e..6b7e74fbd3a2 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
index 3cdf1bc03e05..fa146dca30ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java
index b14e190aa7fe..eb051aa25245 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java
@@ -1,6 +1,5 @@
 
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index eb6c10c4176b..39616edacdac 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index d977830efc29..dde755d729aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index c1c2b581a28f..df31542dd6a9 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index edc8bc9a9500..4b65fe780f62 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 13a80bfb7014..dbb58ed3924f 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index b11db60db74b..318e0ba1fad5 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java
index ae14375c602d..42523a01ba98 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 2916ba782126..a639ca44aaef 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index 6778c43d6f69..e9113baedf2c 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index bee69e6116c8..37bb31a32ca8 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
index eee59a0eeaea..fe244a3d2b82 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java
index d496e5fb828a..51e48f7c34af 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
index 21d76feafd7e..edc53dc73231 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
index 62a1b0e29cfc..0344a941c5f0 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
index d17ac14cb7a8..1cf4056fe8ea 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/Main.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index 887e8a1a523d..ce38b84e35b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java b/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
index 405cacea84e2..2eccbf9eda00 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
index 91dca78d21eb..7d397164dc34 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
index 35c0aa6f987a..6d36afbf4d5e 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java b/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java
index 6167ccc594cc..61f62b5c182d 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java b/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java
index 19c99e787e96..f9df8c64640b 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java
index 4e7edf960498..8b52bb174174 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
index e26575a69879..8bfb303ab82c 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index 5bdede75f248..1612fcae0aa8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
index 8d6cfb48513d..dd5e6a742fb0 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java b/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
index 033d685d40cc..e298dfd78fe8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
index 7f58e96c4d7a..e6b55ba50586 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
index 8ebda4f4aea6..95ab21a44999 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
index 4cf2d7beec0e..d407e666c4d5 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
index e6bbfb91f537..7105c7190cf4 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
index 16af87b2ac46..2ad28736df82 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
index 0ea30041ed77..f9c92e45f156 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index 88cc68f02f29..346682d10c94 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
index cdb55dc9bd36..3da0cea4326b 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java
index 1e0dbbebd176..9df5be214879 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java
index 46724470c07f..6881ac4630ae 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index a3d01973ee8e..b8fb4018064d 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
index 527384dea52f..c17110689a4f 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java
index 421065b895ac..e3b5dc547300 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java
index 0bd5f653a982..3b659c2222a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java
index 764576c208df..65e43f7b09aa 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java
index d27b37beaa11..7ef77aeab60c 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java
index 08a976b32fdf..36d1a758c899 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java
index 9c1c3f60c091..56cdbfa40343 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java
index d5abe6fb0a95..4e9c20d5f1f5 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java
index d29d50de8eb8..9337f26f0a08 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
index 3413d00a9b64..2385e1848595 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
index 7e7073c4bda4..9490735b7122 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
index caf53683e95c..7dbbe953058e 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
index a987695554ed..7fdb027800f9 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
index 1f6b1d768116..8081067e507c 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
index c976c31c1494..37de8cefee5c 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java
index 05634795e4f1..58111e2c5e27 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
index ce6fb96fe598..e83f3126a270 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
index 1c276c257060..31b9953284d0 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java
index e1d33cd1b360..3cff036a6857 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
index 67e7a049acec..48eb8391ce07 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
index fa6e3a6654b8..106e06c379b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
index e4b6b0fbf83a..8d027710d7be 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/package.html b/src/main/java/org/apache/hadoop/hbase/rest/package.html
index 9ed0eadfa65c..53dc30a185e4 100755
--- a/src/main/java/org/apache/hadoop/hbase/rest/package.html
+++ b/src/main/java/org/apache/hadoop/hbase/rest/package.html
@@ -2,7 +2,6 @@
 <html>
 
 <!--
-   Copyright 2009 The Apache Software Foundation
 
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java
index 0c2ab3d4a8a5..7d9f4f1c5288 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java
index daccb8d1d6be..1e7d423f6ffe 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java
index 092c69575b4d..85c5dce98cbb 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
index 0117604264c3..9e62a76fda1c 100644
--- a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
+++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index 0fa9830be4ab..573785380031 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index e2831f7ae705..a997b7f714c8 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
index 1e64b4717d34..1a1b7b2429c7 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Addressing.java b/src/main/java/org/apache/hadoop/hbase/util/Addressing.java
index 05c51ea440ca..1516a4bf977d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Addressing.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Addressing.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Base64.java b/src/main/java/org/apache/hadoop/hbase/util/Base64.java
index 892f8083c964..a960fddc7f9e 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Base64.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Base64.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java
index 7dd2f68e43d0..03a4b9de077a 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java
index ab218b77f757..bcc8e7cb1bb8 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
index 418bd16ae9c1..bdae6f6760b2 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java
index 46691fbfb4ab..38d715931830 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
index 7f666ac87193..de84a342f490 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java b/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java
index 599c15b1dcf9..cc54599647eb 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index 3fa1317a6d97..3e339347a7d9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java b/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java
index 6d7c3890c6b3..ffacd08a16b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
index 95313ad6ecb8..ff130d564c99 100755
--- a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java b/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
index 25cec2af143c..2acce5e8e4b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
index 6c4ddcf279f4..a4ae3db1a288 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java
index a096087047c4..7fcd15ae7d15 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java
index 8571d9c841bd..f396b81c02ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java b/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
index f37d262ce435..a3a33331a2fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java
index 66f9192ed786..52189632310d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java b/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
index f9081f20748a..7fa7891307e7 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java
index 16e65d34dc70..4bbcf8914fa3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
index ce4b82b23a9c..c2911cabf417 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 24dde3011db6..53062a32766a 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java b/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java
index 5235121fe380..2e0a96f55642 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java b/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java
index 361334db20f7..b12c5925738c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2012 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java
index 225f92c78837..b4fb90384466 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index 03f8a1cc9271..a871642cba31 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HMerge.java b/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
index a685aa77c397..e2023e098f27 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Hash.java b/src/main/java/org/apache/hadoop/hbase/util/Hash.java
index 8a3dcf5bc9a1..1dfb84decc0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Hash.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Hash.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java b/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java
index e6471cbbfd15..4b24cc2d3972 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/IdLock.java b/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
index e9202dde61cc..b44bbc70bba5 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java
index e105b779c69e..1f03512411b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
index 11e77dc52afd..fc783c489ee9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
index c990178e28e9..2238a98e8bbf 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java b/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java
index f4f61d0210f4..86c5d2fc1bb9 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java b/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java
index b7eb7e508e0e..b71e1682cabe 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java b/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java
index 958edef8d5a4..501e56c4cb0c 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Keying.java b/src/main/java/org/apache/hadoop/hbase/util/Keying.java
index 2e3d02761a16..11e9947a23ec 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Keying.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Keying.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java b/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java
index b2998c9b040e..74dc07a391d8 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
index a56926aab1d8..ff4da3ba0f49 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Merge.java b/src/main/java/org/apache/hadoop/hbase/util/Merge.java
index 1ed4225bc517..09d95700dec7 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Merge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Merge.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java b/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
index af8d734a9f5c..f767b900df38 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java b/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java
index 085bf1e34299..97641aa5c6c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Objects.java b/src/main/java/org/apache/hadoop/hbase/util/Objects.java
index 088b54a3bb79..0f4ffcc7d41f 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Objects.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Objects.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Pair.java b/src/main/java/org/apache/hadoop/hbase/util/Pair.java
index 9b4a567c93d3..b6abcc3e884f 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Pair.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Pair.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java b/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java
index ddee30d23283..2744a1a9f221 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
index 1b6e1c825c91..f6c5b57aa3a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java b/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java
index 92129676256e..1d96761b09fa 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -14,6 +14,39 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * The portion of this file denoted by 'Copied from com.google.protobuf.CodedInputStream'
+ * is from Protocol Buffers v2.4.1 under the following license
+ *
+ * Copyright 2008 Google Inc.  All rights reserved.
+ * http://code.google.com/p/protobuf/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  */
 
 package org.apache.hadoop.hbase.util;
@@ -30,6 +63,7 @@ public abstract class ProtoUtil {
    * @throws IOException if it is malformed or EOF.
    */
   public static int readRawVarint32(DataInput in) throws IOException {
+  // Copied from com.google.protobuf.CodedInputStream v2.4.1 readRawVarint32
     byte tmp = in.readByte();
     if (tmp >= 0) {
       return tmp;
@@ -62,5 +96,6 @@ public static int readRawVarint32(DataInput in) throws IOException {
     }
     return result;
   }
+  // end of copied from protobuf
 
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
index 449e2b002e09..aed0a827a2fc 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index 619a715196c3..0e0b10a75048 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java b/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
index 15e741d1f25a..163e40f47db4 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java b/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
index 445234e29b55..f1c964f02f23 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java
index b2f377056758..7c8fff809fa4 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java b/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
index a8d7d6830730..d82436491625 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java b/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java
index 7229e1465011..75531d55b43d 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java b/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java
index f04f306e812e..f2e1e735b919 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Strings.java b/src/main/java/org/apache/hadoop/hbase/util/Strings.java
index 5e98c597aa72..792e98e8d3a0 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Strings.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Strings.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index 5a51bccbb389..2e61af2085a7 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java b/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java
index b516e16b6a2d..0a2b5aeb4fbb 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Writables.java b/src/main/java/org/apache/hadoop/hbase/util/Writables.java
index 1719e8423687..2068b0f8c1b3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Writables.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Writables.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java
index 0b1b647be49a..5c9cabe82a6c 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
index 7e1a9528650b..f4193221a455 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
index 43a9e974cb0f..2061fb2731f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
index 4da6f9627285..1cefdbcff692 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
index 92ff81b800bc..fadcba8d2b5a 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 93ba93007716..0ad712e6d56d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
index d8f0491869aa..260bc34252af 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
index 48a8b3d3bdec..3e4564ceda22 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
index f14b026a0bbc..142f0966e95c 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
index 9161b8583d38..eeeacdbbe2e2 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
index 500bd3c146e8..c975aeb47855 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
index 8e9aa499c78b..07d9eb888236 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index cd95e0692102..a773ab32c596 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 9e03bd08313f..b24be4c7fe3f 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
index bfd1ce86a6fa..b7d69043fa16 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java
index e6f7f16a773d..d4ea161b06d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
index 4365f78e7172..9c4893f184c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index bca28e81a6c4..d69d4512b7a8 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html b/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html
index d61becb38bde..b97a1a6629e8 100644
--- a/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html
+++ b/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html
@@ -2,7 +2,6 @@
 <html>
 
 <!--
-   Copyright 2010 The Apache Software Foundation
 
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto
index a7bfe83b79ed..8d4515417f08 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto
index dfdf125ed44d..4774a8d0e8f7 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto
index 0a9a9af68657..05e33b648329 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto
index 6ef3191b0a60..f7aca47131ff 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto
index 46e275d9cc56..a0291b4e9e8a 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto
index 5dd91204247a..674499c26874 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto
index 2ce4d25ee8aa..fbd76ea029af 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto
index d8177229d45b..47a4da589b95 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto
index 2404a2ebd35d..cc107b3d9a61 100644
--- a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto
+++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto
@@ -1,4 +1,3 @@
-// Copyright 2010 The Apache Software Foundation
 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/hbase.rb b/src/main/ruby/hbase.rb
index fbcdc006aef8..a90976a726fe 100644
--- a/src/main/ruby/hbase.rb
+++ b/src/main/ruby/hbase.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb
index 367485bd0ff5..29599e071932 100644
--- a/src/main/ruby/hbase/admin.rb
+++ b/src/main/ruby/hbase/admin.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/hbase/hbase.rb b/src/main/ruby/hbase/hbase.rb
index 2c37840dd206..f97d0f95fdfb 100644
--- a/src/main/ruby/hbase/hbase.rb
+++ b/src/main/ruby/hbase/hbase.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/hbase/replication_admin.rb b/src/main/ruby/hbase/replication_admin.rb
index 99f0479b0e6f..2ade317315f5 100644
--- a/src/main/ruby/hbase/replication_admin.rb
+++ b/src/main/ruby/hbase/replication_admin.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb
index ed8437ca88d1..125abd4f0726 100644
--- a/src/main/ruby/hbase/table.rb
+++ b/src/main/ruby/hbase/table.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/irb/hirb.rb b/src/main/ruby/irb/hirb.rb
index 584f70087f44..b32e691cf8c6 100644
--- a/src/main/ruby/irb/hirb.rb
+++ b/src/main/ruby/irb/hirb.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell.rb b/src/main/ruby/shell.rb
index 29b08f6ead02..e8778f4e56e8 100644
--- a/src/main/ruby/shell.rb
+++ b/src/main/ruby/shell.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands.rb b/src/main/ruby/shell/commands.rb
index 20e6aafa6a5a..57b8f1c49462 100644
--- a/src/main/ruby/shell/commands.rb
+++ b/src/main/ruby/shell/commands.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2009 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/add_peer.rb b/src/main/ruby/shell/commands/add_peer.rb
index 7669fb7c5266..deb5c5def51f 100644
--- a/src/main/ruby/shell/commands/add_peer.rb
+++ b/src/main/ruby/shell/commands/add_peer.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/alter.rb b/src/main/ruby/shell/commands/alter.rb
index 3be8f0a42789..dc9e1e18ae0c 100644
--- a/src/main/ruby/shell/commands/alter.rb
+++ b/src/main/ruby/shell/commands/alter.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/alter_async.rb b/src/main/ruby/shell/commands/alter_async.rb
index 01dfd994656a..164f33914915 100644
--- a/src/main/ruby/shell/commands/alter_async.rb
+++ b/src/main/ruby/shell/commands/alter_async.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/alter_status.rb b/src/main/ruby/shell/commands/alter_status.rb
index cc79e1108b6d..d62edacc9c67 100644
--- a/src/main/ruby/shell/commands/alter_status.rb
+++ b/src/main/ruby/shell/commands/alter_status.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/assign.rb b/src/main/ruby/shell/commands/assign.rb
index 4c83d3cf2006..de0071a68d87 100644
--- a/src/main/ruby/shell/commands/assign.rb
+++ b/src/main/ruby/shell/commands/assign.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/balance_switch.rb b/src/main/ruby/shell/commands/balance_switch.rb
index 0eac7654a812..4d7778daa41b 100644
--- a/src/main/ruby/shell/commands/balance_switch.rb
+++ b/src/main/ruby/shell/commands/balance_switch.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/balancer.rb b/src/main/ruby/shell/commands/balancer.rb
index 05ecd3a2bc71..c329eced4b9d 100644
--- a/src/main/ruby/shell/commands/balancer.rb
+++ b/src/main/ruby/shell/commands/balancer.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/close_region.rb b/src/main/ruby/shell/commands/close_region.rb
index f1010ac6ca01..1cd149725a7a 100644
--- a/src/main/ruby/shell/commands/close_region.rb
+++ b/src/main/ruby/shell/commands/close_region.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/compact.rb b/src/main/ruby/shell/commands/compact.rb
index fa0b82eef8cc..47cf0380ac7c 100644
--- a/src/main/ruby/shell/commands/compact.rb
+++ b/src/main/ruby/shell/commands/compact.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/count.rb b/src/main/ruby/shell/commands/count.rb
index 6596441c1c32..fdc49a2b0e6d 100644
--- a/src/main/ruby/shell/commands/count.rb
+++ b/src/main/ruby/shell/commands/count.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/create.rb b/src/main/ruby/shell/commands/create.rb
index 14c1b0ff4f65..219b26338578 100644
--- a/src/main/ruby/shell/commands/create.rb
+++ b/src/main/ruby/shell/commands/create.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/delete.rb b/src/main/ruby/shell/commands/delete.rb
index 12bc40593359..2235d27da29c 100644
--- a/src/main/ruby/shell/commands/delete.rb
+++ b/src/main/ruby/shell/commands/delete.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/deleteall.rb b/src/main/ruby/shell/commands/deleteall.rb
index 5731b606b41e..a8c51c2b4a50 100644
--- a/src/main/ruby/shell/commands/deleteall.rb
+++ b/src/main/ruby/shell/commands/deleteall.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/describe.rb b/src/main/ruby/shell/commands/describe.rb
index 0f355074cda6..cce82c47705e 100644
--- a/src/main/ruby/shell/commands/describe.rb
+++ b/src/main/ruby/shell/commands/describe.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/disable.rb b/src/main/ruby/shell/commands/disable.rb
index 34c5f9ca7d08..b10aa9bf0357 100644
--- a/src/main/ruby/shell/commands/disable.rb
+++ b/src/main/ruby/shell/commands/disable.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/disable_all.rb b/src/main/ruby/shell/commands/disable_all.rb
index b7c9cf5fdd95..0e7c30ee9978 100644
--- a/src/main/ruby/shell/commands/disable_all.rb
+++ b/src/main/ruby/shell/commands/disable_all.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/disable_peer.rb b/src/main/ruby/shell/commands/disable_peer.rb
index da9941a9572d..416545b21fb8 100644
--- a/src/main/ruby/shell/commands/disable_peer.rb
+++ b/src/main/ruby/shell/commands/disable_peer.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/drop.rb b/src/main/ruby/shell/commands/drop.rb
index 40831d2fa06e..6238fe65b8f0 100644
--- a/src/main/ruby/shell/commands/drop.rb
+++ b/src/main/ruby/shell/commands/drop.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/drop_all.rb b/src/main/ruby/shell/commands/drop_all.rb
index dcbefa32346b..e482d6b0c79c 100644
--- a/src/main/ruby/shell/commands/drop_all.rb
+++ b/src/main/ruby/shell/commands/drop_all.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/enable.rb b/src/main/ruby/shell/commands/enable.rb
index a0dc340f5ddd..a4d495a55e9f 100644
--- a/src/main/ruby/shell/commands/enable.rb
+++ b/src/main/ruby/shell/commands/enable.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/enable_all.rb b/src/main/ruby/shell/commands/enable_all.rb
index ee016373f47f..fc0f081f1509 100644
--- a/src/main/ruby/shell/commands/enable_all.rb
+++ b/src/main/ruby/shell/commands/enable_all.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/enable_peer.rb b/src/main/ruby/shell/commands/enable_peer.rb
index de459092b58b..55136ffb7fb5 100644
--- a/src/main/ruby/shell/commands/enable_peer.rb
+++ b/src/main/ruby/shell/commands/enable_peer.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/exists.rb b/src/main/ruby/shell/commands/exists.rb
index f35f197238fb..b7932ced528f 100644
--- a/src/main/ruby/shell/commands/exists.rb
+++ b/src/main/ruby/shell/commands/exists.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/flush.rb b/src/main/ruby/shell/commands/flush.rb
index ba597662c52a..185b4731706c 100644
--- a/src/main/ruby/shell/commands/flush.rb
+++ b/src/main/ruby/shell/commands/flush.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/get.rb b/src/main/ruby/shell/commands/get.rb
index 8456d58a5cb6..8c3161e100c5 100644
--- a/src/main/ruby/shell/commands/get.rb
+++ b/src/main/ruby/shell/commands/get.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/get_counter.rb b/src/main/ruby/shell/commands/get_counter.rb
index 3cbe2268d9ad..5c25f05c91de 100644
--- a/src/main/ruby/shell/commands/get_counter.rb
+++ b/src/main/ruby/shell/commands/get_counter.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/hlog_roll.rb b/src/main/ruby/shell/commands/hlog_roll.rb
index 02c7c592822d..b292791541fd 100644
--- a/src/main/ruby/shell/commands/hlog_roll.rb
+++ b/src/main/ruby/shell/commands/hlog_roll.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2011 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/incr.rb b/src/main/ruby/shell/commands/incr.rb
index 38a2fc517934..c0c60a5fd328 100644
--- a/src/main/ruby/shell/commands/incr.rb
+++ b/src/main/ruby/shell/commands/incr.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/is_disabled.rb b/src/main/ruby/shell/commands/is_disabled.rb
index 9d3c7ee7471e..28918ee42980 100644
--- a/src/main/ruby/shell/commands/is_disabled.rb
+++ b/src/main/ruby/shell/commands/is_disabled.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/is_enabled.rb b/src/main/ruby/shell/commands/is_enabled.rb
index 96b2b152c2e1..6a98a36b71d0 100644
--- a/src/main/ruby/shell/commands/is_enabled.rb
+++ b/src/main/ruby/shell/commands/is_enabled.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/list.rb b/src/main/ruby/shell/commands/list.rb
index 592fb5eec253..16af27b9c097 100644
--- a/src/main/ruby/shell/commands/list.rb
+++ b/src/main/ruby/shell/commands/list.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/major_compact.rb b/src/main/ruby/shell/commands/major_compact.rb
index 325344b5265a..f05cc1d78dc9 100644
--- a/src/main/ruby/shell/commands/major_compact.rb
+++ b/src/main/ruby/shell/commands/major_compact.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/move.rb b/src/main/ruby/shell/commands/move.rb
index 0e3db8fe4729..e6b28288e3fa 100644
--- a/src/main/ruby/shell/commands/move.rb
+++ b/src/main/ruby/shell/commands/move.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/put.rb b/src/main/ruby/shell/commands/put.rb
index dde043370838..6bbe7cddf37b 100644
--- a/src/main/ruby/shell/commands/put.rb
+++ b/src/main/ruby/shell/commands/put.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/remove_peer.rb b/src/main/ruby/shell/commands/remove_peer.rb
index 034434aa5dab..5ae5786fc4a4 100644
--- a/src/main/ruby/shell/commands/remove_peer.rb
+++ b/src/main/ruby/shell/commands/remove_peer.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/scan.rb b/src/main/ruby/shell/commands/scan.rb
index 518e628d67bd..f758fb21f7a2 100644
--- a/src/main/ruby/shell/commands/scan.rb
+++ b/src/main/ruby/shell/commands/scan.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/show_filters.rb b/src/main/ruby/shell/commands/show_filters.rb
index 1581db5b37aa..716859fc0618 100644
--- a/src/main/ruby/shell/commands/show_filters.rb
+++ b/src/main/ruby/shell/commands/show_filters.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2011 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/split.rb b/src/main/ruby/shell/commands/split.rb
index 47e21f86c218..9488ed9ef156 100644
--- a/src/main/ruby/shell/commands/split.rb
+++ b/src/main/ruby/shell/commands/split.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/start_replication.rb b/src/main/ruby/shell/commands/start_replication.rb
index 5d1cd1b5ee1c..3ea97a90c993 100644
--- a/src/main/ruby/shell/commands/start_replication.rb
+++ b/src/main/ruby/shell/commands/start_replication.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/status.rb b/src/main/ruby/shell/commands/status.rb
index 4b22acb4abf1..f72c13caef60 100644
--- a/src/main/ruby/shell/commands/status.rb
+++ b/src/main/ruby/shell/commands/status.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/stop_replication.rb b/src/main/ruby/shell/commands/stop_replication.rb
index f5074d73a870..2e24fa5cfe69 100644
--- a/src/main/ruby/shell/commands/stop_replication.rb
+++ b/src/main/ruby/shell/commands/stop_replication.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/truncate.rb b/src/main/ruby/shell/commands/truncate.rb
index a24e167b9829..b7812fbe3dd5 100644
--- a/src/main/ruby/shell/commands/truncate.rb
+++ b/src/main/ruby/shell/commands/truncate.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/unassign.rb b/src/main/ruby/shell/commands/unassign.rb
index 75242baf4bc7..60cc6ca3fc80 100644
--- a/src/main/ruby/shell/commands/unassign.rb
+++ b/src/main/ruby/shell/commands/unassign.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/version.rb b/src/main/ruby/shell/commands/version.rb
index 372b0dc341fe..63e9712a52d6 100644
--- a/src/main/ruby/shell/commands/version.rb
+++ b/src/main/ruby/shell/commands/version.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/commands/zk_dump.rb b/src/main/ruby/shell/commands/zk_dump.rb
index bb23962df860..c0b509a3c6e8 100644
--- a/src/main/ruby/shell/commands/zk_dump.rb
+++ b/src/main/ruby/shell/commands/zk_dump.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/ruby/shell/formatter.rb b/src/main/ruby/shell/formatter.rb
index ea17882a6ecb..36aaf76aea7e 100644
--- a/src/main/ruby/shell/formatter.rb
+++ b/src/main/ruby/shell/formatter.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/main/xslt/configuration_to_docbook_section.xsl b/src/main/xslt/configuration_to_docbook_section.xsl
index 95f7fd5d8c33..5f412df9b647 100644
--- a/src/main/xslt/configuration_to_docbook_section.xsl
+++ b/src/main/xslt/configuration_to_docbook_section.xsl
@@ -4,7 +4,6 @@
 <xsl:template match="configuration">
 <!--
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/packages/conf-pseudo/hbase-site.xml b/src/packages/conf-pseudo/hbase-site.xml
index b4e8de6f0bb1..42c6ef51c4b9 100644
--- a/src/packages/conf-pseudo/hbase-site.xml
+++ b/src/packages/conf-pseudo/hbase-site.xml
@@ -2,7 +2,6 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/site/xdoc/acid-semantics.xml b/src/site/xdoc/acid-semantics.xml
index 81f7f04fed4a..2696df78bd97 100644
--- a/src/site/xdoc/acid-semantics.xml
+++ b/src/site/xdoc/acid-semantics.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
diff --git a/src/site/xdoc/bulk-loads.xml b/src/site/xdoc/bulk-loads.xml
index f4d3d75f9511..2195003042be 100644
--- a/src/site/xdoc/bulk-loads.xml
+++ b/src/site/xdoc/bulk-loads.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
 
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 <document xmlns="http://maven.apache.org/XDOC/2.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
diff --git a/src/site/xdoc/cygwin.xml b/src/site/xdoc/cygwin.xml
index 40e0f72cf853..406c0a9b6eac 100644
--- a/src/site/xdoc/cygwin.xml
+++ b/src/site/xdoc/cygwin.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
 
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 <document xmlns="http://maven.apache.org/XDOC/2.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml
index b4c1f817defe..3cd0bb0a22e8 100644
--- a/src/site/xdoc/index.xml
+++ b/src/site/xdoc/index.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
 
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 <document xmlns="http://maven.apache.org/XDOC/2.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
diff --git a/src/site/xdoc/metrics.xml b/src/site/xdoc/metrics.xml
index 47902b8df49c..f3ab7d76c149 100644
--- a/src/site/xdoc/metrics.xml
+++ b/src/site/xdoc/metrics.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
 
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 <document xmlns="http://maven.apache.org/XDOC/2.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
diff --git a/src/site/xdoc/old_news.xml b/src/site/xdoc/old_news.xml
index 1dc3eaeb1b1d..0ce32ce93fe0 100644
--- a/src/site/xdoc/old_news.xml
+++ b/src/site/xdoc/old_news.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
 
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
diff --git a/src/site/xdoc/pseudo-distributed.xml b/src/site/xdoc/pseudo-distributed.xml
index a1b38babb997..670f1e7c8528 100644
--- a/src/site/xdoc/pseudo-distributed.xml
+++ b/src/site/xdoc/pseudo-distributed.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
diff --git a/src/site/xdoc/replication.xml b/src/site/xdoc/replication.xml
index 727ca267636c..cf440a56d0a7 100644
--- a/src/site/xdoc/replication.xml
+++ b/src/site/xdoc/replication.xml
@@ -1,17 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
 
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 
 <!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
diff --git a/src/site/xdoc/resources.xml b/src/site/xdoc/resources.xml
index 214fc79aa372..d067c1e1e1e3 100644
--- a/src/site/xdoc/resources.xml
+++ b/src/site/xdoc/resources.xml
@@ -1,16 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-      http://www.apache.org/licenses/LICENSE-2.0
+  http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 <document xmlns="http://maven.apache.org/XDOC/2.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
diff --git a/src/site/xdoc/sponsors.xml b/src/site/xdoc/sponsors.xml
index 639e0c39acbb..2d39344a1432 100644
--- a/src/site/xdoc/sponsors.xml
+++ b/src/site/xdoc/sponsors.xml
@@ -1,16 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
 
-      http://www.apache.org/licenses/LICENSE-2.0
+  http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
 -->
 <document xmlns="http://maven.apache.org/XDOC/2.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index cbbf737c44ce..0abfeeed50fe 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index fc548261b34b..1feea38bdde2 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
index 47a1d7e8e201..8c492d3e451b 100644
--- a/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java b/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java
index 36d768a9fa16..af9032bf96f9 100644
--- a/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/LargeTests.java b/src/test/java/org/apache/hadoop/hbase/LargeTests.java
index da9abdafc7b8..958ffd71c7ef 100644
--- a/src/test/java/org/apache/hadoop/hbase/LargeTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/LargeTests.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java
index 76359498b7fc..ec47318a2d0c 100644
--- a/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/MediumTests.java b/src/test/java/org/apache/hadoop/hbase/MediumTests.java
index 24a44b9523e6..a51a2c9be6e7 100644
--- a/src/test/java/org/apache/hadoop/hbase/MediumTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/MediumTests.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 109d94eeae49..5be589d6ed3a 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java b/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
index f9a00dded996..8758f9eb7098 100644
--- a/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 1c2ae3b49466..6d4a65c0d24a 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java
index eac7207cb0e8..86fe1360e23a 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
index 62a51445ab23..118a0633cf2e 100644
--- a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
+++ b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java b/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
index 56638c95e892..86fe4c11bb22 100644
--- a/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
+++ b/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/SmallTests.java b/src/test/java/org/apache/hadoop/hbase/SmallTests.java
index eba6a948f0b7..6953667d6998 100644
--- a/src/test/java/org/apache/hadoop/hbase/SmallTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/SmallTests.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
index ace876a9e61d..2e43387ccec3 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java b/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java
index 6bc7c32b3439..d7fff04726f8 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
index e725d434577d..4c379b4c7631 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java b/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
index ad77e0a12ace..7615479523a1 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
index 3ed5d52e008d..dad29883a820 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java b/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java
index a9b8f9458be4..98f070e9cfe4 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java b/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java
index 1a6933752cf3..a86a52a4ad44 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java b/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
index f7c90d4993be..f257a605fbd5 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
index fe3dfeb2b291..e92d0a63c52b 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
index ec04f4d8a651..a5eba33f9679 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
index bd7afd08ee98..977d59c28c75 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 1359ac1dee60..89537971ddf7 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
index 39ef3a0e3d2a..a78a893c5047 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
index 1105ec94c903..07af255cbb23 100644
--- a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
+++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index f2495508d7c3..a3f3d1657e3a 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java
index e071b769736f..a65724369d6b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index aceaeedfa07a..0c8829132659 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
index d434ccc4d431..3af07dbd3d62 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index f6343c50ac58..8c9b74f4de11 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
index be033512952b..0c5a0759d84e 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java
index 24f878e7d74a..8da61a5cd655 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
index 6eff27c66545..8dd6949ad540 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
index 3d7912591193..f83148ce34ab 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
index 404ccffc4430..a1e984676923 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
index d479259965e0..e4b29e11b2d2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java b/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
index 43f856f2d41d..550b2e5287af 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
index 685ff2e2308c..685767b971db 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
index cdb40bb526d5..15a6d8da72a7 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
index 4b7be5cec4b6..3fcd3f29b91b 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestShell.java b/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
index 624913bdf87d..f20e3b1b53a6 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java b/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
index 837ce2824fcb..ec482da42e71 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java
index 38c55aaacc7d..5897e5c63af9 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java
index 658c07bb6f08..5b38e0b8f060 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java
index ff9c502f2bea..130443d4cddf 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index f587c2e3cd97..5aba66416347 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java
index 28af3161e3e4..edb641713ea8 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index ba1b5287e99e..0d87f4b398c5 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
index 1c596d5831c2..03d80a78cbe5 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index c03be922d297..e152cb88b265 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
index 0e0b4228d2c8..fe44ab9a50b9 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
index d7e0f65e3bc9..02068c54ddab 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index ff412b38da4e..53445e6a1708 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java
index 8afdb07fa980..f6a44f541a90 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMultiRowMutationProtocol.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
index 7a118dab9dda..ab1b0e0a3308 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index 9d8f0fce2176..75d89042b8a0 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
index ffa5de4a8697..4edfd385876e 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
index a857a82b5c4b..d60ce9c626d2 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
index 36dd28938a78..ff2622bd1d74 100644
--- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
+++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java b/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
index 805421f7a4be..29201e6999bc 100644
--- a/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
+++ b/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java b/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
index 50888af7625f..aff6cf2f5e81 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
@@ -1,18 +1,20 @@
-/**
- *  Copyright 2010 The Apache Software Foundation
- * 
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- * 
- *       http://www.apache.org/licenses/LICENSE-2.0
- * 
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *  under the License.
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase.filter;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java
index 720d882d0fba..0d4208567bda 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
index 09f83da29f60..2d205bfbd1db 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index c09bba00a487..c95bc0aced0b 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 7a25661817c9..110eea422e85 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java
index 991d3904143a..e1ed7b6f8ffb 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java
@@ -1,17 +1,20 @@
-/**
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *       http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *  under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase.filter;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java
index a12bd8eaa16a..50395cf6f01b 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java b/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java
index 62639b8349e9..37dbbe9dccdd 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestNullComparator.java
@@ -1,17 +1,20 @@
-/**
- * 
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- * 
- *       http://www.apache.org/licenses/LICENSE-2.0
- * 
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *  under the License.
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 package org.apache.hadoop.hbase.filter;
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java
index ee9a9611a5dc..8bc03ae7039c 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
index 922a6c1d332b..3323fbd6e22f 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java
index 24a999e51d60..48d2a0654dc5 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java
index 7731cd9ccbc9..9662c9ec8fee 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
index 31f5db453321..4e348ba2a51e 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
index fca32a73d470..2726b347e277 100644
--- a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
index 7d8eaa75aa7c..d655a6be48fb 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
index 892dda1f94f5..239e1484c876 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
index 4ed6aa55b184..cb29cacc69ff 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java b/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java
index 6217dec3da36..39ea85782002 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
index 88b8708376f2..537ecd5700d6 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java
index a48a69f4212b..5bb53a664741 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java
index 76738063721a..e6a957ef748a 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index aa20de3dce5d..7c503c0100e8 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
index 0dbe3e3ef74c..8769ad99ffe5 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
index 7dd63d53591f..7f875828e025 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
index 44dfb7388445..da9b5ac09396 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index 74402244c09d..5d00aa2e074e 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
index 98ea1805be0e..f5013e201ae6 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 6280c21836e6..22c9b696c669 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
index d313ddd435d4..91bb456561f3 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
index 1ddafa68cd25..b1f49bb20f1d 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index 65af0d74ece8..a4415265f154 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
index 6c859fdb1dc4..f4a55e92dc69 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
index 1a2ce15d9ee6..6dbc503ce176 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
index 4f21fbdff638..2df5bff4e982 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java
index 124d131849c7..0f09b063f995 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
index a02f109515e2..577d42339aef 100644
--- a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
+++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
index ee390cd14e54..c6b5d6bc10bc 100644
--- a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
+++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
index 38be7e42f530..5f9351988229 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
index 7fe273bf9b46..fad6678e8d90 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java
index f4b3f65042a7..92888ed5e0f8 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 565670537b70..5c4ebec337fc 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index 77d044b944ca..dd0263788ad2 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 1e390baa9f11..c48963414327 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java
index 7ec759a391c0..ac65c6f3ef77 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index e26825f5100e..8f11e03b8349 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
index 74ea4ec35fb4..15d90ec834d8 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
index 0f5f41c69405..52e555e7cd01 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 13a3c2724477..073345fdffc0 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
index e1a07c8a3cfd..7c6c5694bb03 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java b/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java
index 40721cc0198d..dbb1e286f7e2 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 51e4eba94cf4..73a2d3b28028 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
index 8c4de71bf114..33442910011c 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index 65b4d5962ad7..4bcc544e9ca0 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 6d1492d97e59..962b075cf2cc 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
index ec08b1740c1d..6c65b4e12a15 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
index 885cea6bd3fe..56f4fe8d4397 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
index 762a2b8eeb77..3b4cc0d2ea44 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index 7703be3d5beb..70b2793dd094 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java b/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
index 9462d114a552..e2d334ff8d9b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index a60bc9533bfa..e09e6c398c11 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
index 8ddb3511acb4..b9381bdff040 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index e38309c27870..5ae1235beace 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java
index 96e7772185db..d040e2a2535f 100644
--- a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java
+++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java b/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java
index ac76887569c6..9c46307629a9 100644
--- a/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java
+++ b/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java b/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java
index 98de0247b998..3b35c2be1eaa 100644
--- a/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java
+++ b/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
index f1cd8a02ca5c..bfa9f0508abb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java b/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java
index 08fb91e22926..53c55be8d16b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java b/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
index cac29897efcf..636caf8b1da1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java
index e20aab3121c0..b50e06ecc1c4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBatchHRegionLockingAndWrites.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
index 3813ea4f6a8c..8eed964a4175 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
index 863b97ce7000..2ce15f78c26a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
index e865a8a66205..0eec002bdcd5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
index b37053aeba8c..b6dac3e76b33 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 4971a86852cc..daa2297fcb63 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
index aeaf62550684..9a22c3fd9393 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index 71febf3a0d9d..354f66869e4b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
index 89dfbf74af8e..6f918b5fc983 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
index 5f97167543a6..9478a57fead2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index c459180bd3d0..5f0644ba876d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
index 17538750e717..0098fc94aa13 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
index 69f57ee98361..71d8a9e3d773 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java
index 1cc893a519bb..edc187b4a4a5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java
index d9158bebf9c9..5f0feb50f415 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
index e91d83caef7f..4453f0ea1b73 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
index ba89b7c09d89..fa4a64c202d9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
index d7b01ca72eae..8d0153a9353a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
index 631d7706c117..4e8270e27411 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
index b716c53c0eee..2feb20a3a322 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index f5f427305f9a..206ca46a275c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java
index 84782601f202..0d1dbf8a022f 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
index 96afbd2935df..f51670e43074 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java
index 38e5b3413424..183fedf39ca3 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRpcMetrics.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
index 765f12b8ff06..a7045e6c7267 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
index 2a76c1e93bb9..7188471a43cb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java
index bdb1231d97a8..2779302c9628 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
index 32e8d189fcff..eb26b9d72810 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
index 345b10a6cfb5..104f99b1fb1a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java
index e99d2514a4d3..9af7d76b4bb1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index 5d4b78279d6a..e20ef8171467 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index 4d6eb90aa465..965a32fff31c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index ec0cfb8f6673..fba778720313 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index e39795ceff2c..4b8208968706 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 34116c71c4d5..8cef69b87ec4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java
index 0ffb8113b7af..3cf81fd995e1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
index 01f0731549f2..f0b2688b4c04 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
index 09d3151cd212..82033c12ebac 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
index a2b3fef0453c..1648f4fcfebf 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
index c12cac19ff57..f926952225c7 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
index 0ec54f32b11f..f56c2156e1ed 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
index bf9bfc42d679..1e669a4cf356 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 5496eaff51c4..e59dac9708cb 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java
index 0b0a7cee11ad..c6584bbcf0fc 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
index 13f77ec33558..1da80c576a84 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 1d49cc8d8d9d..95d7009aa55b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
index ce57070ead4d..73842264b53b 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
index de81e9170c7b..ee1789bb5938 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 150c31a168e2..cad123c21f3c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index 2daf643ba368..3a5baa992757 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
index 69a8f2f26511..c90c321a23e2 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
index fbf38f1efc0f..603b0a74d575 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
index 18eb530fcffb..04ff8ba2b4ec 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index f14fed94b15f..891aaca8a009 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
index b4579edb68db..4d4c1007fa5c 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java
index 41893a4d40fd..f41a41c27537 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java b/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java
index 5063a8f643a6..cdd69fd7283e 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java b/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java
index 4db967f13c22..d45980f3900b 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
index e45ed2adb258..d485e2da04d4 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java
index 2ea49b4dc91f..73ac925526b6 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRESTMetrics.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with this
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
index 6cff8510ec05..cc14f75c0ced 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
index 3c09c383cf57..51e767ab9985 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java b/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
index d83e18627b2e..c8af1cc316a7 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
index 838798771521..e6e7ab8825fb 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
index ea61dc46277d..e0bef84edbc3 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
index 4cf4636328e7..de8ffc98af6b 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
index 5e388a9ca953..5f7519cf7a1a 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
index 78c5be6f17f1..77d59f1257bf 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index 376947dff323..4d1f49c1a5e2 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java
index 4e7a56bc6c6c..8eb93a0639aa 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java
index 3d358c065f82..e1f2f8d9c3d8 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java
index 62014a347f23..bdeaa87a0db8 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java
index 80386456cedf..ca2de3ab323e 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java
index dc55f3accfc6..4d244c035387 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java
index c44f720484e2..cc3fec80ec71 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java
index 3a014860f0a5..f15b151db629 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java
index 7fb74a7b79dc..f2455709012e 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java
index a3d1bf987798..b05a802646ee 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java
index b6f0ab5d4ba6..90df8ce335c0 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java
index c9b298973151..851223f2d1a8 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java
index 2ecb7d90d86d..dec09cf24bff 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/security/TestUser.java b/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
index e90de1f5967e..ca088047fe93 100644
--- a/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
+++ b/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 0cd4620a839a..e3477a3a0a43 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 3578d10e4063..64a654b978bb 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java b/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 730f4e3af6a8..47d79cbcf8bb 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java b/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java
index c55c4d4f87b7..979e33980a6f 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java b/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
index 2240018893fe..fd6bd8f0f488 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
index 70e707a4fd92..b49f696f1a04 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java b/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index df783958b799..2e9bcb8d1ea8 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java b/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java
index ea6903e8f48a..2a29c2b1295c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java b/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java
index 59b2c2bc5a24..bce0b94ead98 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
index 21ac5298c002..853ff032078b 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 3e95a5e6fdc1..ea7501a902d4 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
index 00d2440181b6..a5096b933563 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java b/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java
index 403897466b2d..ca2f39ce5f48 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java b/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java
index f4fbce953f4b..a3d7eb8be971 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java b/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java
index 88f5821f9f49..ebf1ccffd6b2 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
index a9948c7fe0ac..ef7bbeaf8301 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index f5f0ac7826c7..ff619b8c3c14 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java b/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java
index bb4304c6afdd..ca639db9c1c5 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
index a2419f997c06..afef0ff9d4f5 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java
index 2a72c715787b..6f616e6dbc4c 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java b/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java
index c596018c331e..6d27552de05f 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2008 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java b/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java
index 29e025abc127..49f3cd6d94db 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java
@@ -1,5 +1,4 @@
 /*
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
index 764ac1984f10..3e1c89412856 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2009 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
index 643c583e60f7..cc5cdf2512da 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java
index ddc8dc4bac79..cd880526c1eb 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java
index 370738930f77..fec4cabc204a 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java
@@ -1,5 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/resources/hbase-site.xml b/src/test/resources/hbase-site.xml
index 67e6cf55a8e6..61dd018c5c3d 100644
--- a/src/test/resources/hbase-site.xml
+++ b/src/test/resources/hbase-site.xml
@@ -2,7 +2,6 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
- * Copyright 2007 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties b/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties
index 28493ff9f913..6fca96ab0e82 100644
--- a/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties
+++ b/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties
@@ -1,8 +1,6 @@
 # ResourceBundle properties file for Map-Reduce counters
 
 #/**
-# * Copyright 2007 The Apache Software Foundation
-# *
 # * Licensed to the Apache Software Foundation (ASF) under one
 # * or more contributor license agreements.  See the NOTICE file
 # * distributed with this work for additional information
diff --git a/src/test/ruby/hbase/admin_test.rb b/src/test/ruby/hbase/admin_test.rb
index ae67ba627bc2..047db0a92420 100644
--- a/src/test/ruby/hbase/admin_test.rb
+++ b/src/test/ruby/hbase/admin_test.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/ruby/hbase/hbase_test.rb b/src/test/ruby/hbase/hbase_test.rb
index 4e3fae3e629f..c2f54050038f 100644
--- a/src/test/ruby/hbase/hbase_test.rb
+++ b/src/test/ruby/hbase/hbase_test.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/ruby/hbase/table_test.rb b/src/test/ruby/hbase/table_test.rb
index 067058d8a01f..dce6c9aba1da 100644
--- a/src/test/ruby/hbase/table_test.rb
+++ b/src/test/ruby/hbase/table_test.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/ruby/shell/commands_test.rb b/src/test/ruby/shell/commands_test.rb
index 1a315a7adc28..1fbe69f0356b 100644
--- a/src/test/ruby/shell/commands_test.rb
+++ b/src/test/ruby/shell/commands_test.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/ruby/shell/formatter_test.rb b/src/test/ruby/shell/formatter_test.rb
index 5b5c6369bc26..701047931926 100644
--- a/src/test/ruby/shell/formatter_test.rb
+++ b/src/test/ruby/shell/formatter_test.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/ruby/shell/shell_test.rb b/src/test/ruby/shell/shell_test.rb
index bc3000c30d19..1060074b34ba 100644
--- a/src/test/ruby/shell/shell_test.rb
+++ b/src/test/ruby/shell/shell_test.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
diff --git a/src/test/ruby/tests_runner.rb b/src/test/ruby/tests_runner.rb
index 251473fd47ad..56d63c877d05 100644
--- a/src/test/ruby/tests_runner.rb
+++ b/src/test/ruby/tests_runner.rb
@@ -1,5 +1,4 @@
 #
-# Copyright 2010 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

From 45a4212580b8666dd1108d5da5e1771afb213242 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 8 Jan 2016 11:49:05 -0800
Subject: [PATCH 1538/1540] HBASE-14213 Addendum, missed new files.

---
 src/assembly/META-INF/LICENSE.vm              | 2052 +++++++++++++++++
 src/assembly/META-INF/NOTICE.vm               |  324 +++
 .../resources/supplemental-models.xml         | 1664 +++++++++++++
 src/main/appended-resources/META-INF/LICENSE  |   37 +
 src/main/appended-resources/META-INF/NOTICE   |    6 +
 src/main/resources/META-INF/LEGAL             |    5 +
 src/test/resources/META-INF/LICENSE           |  202 ++
 7 files changed, 4290 insertions(+)
 create mode 100644 src/assembly/META-INF/LICENSE.vm
 create mode 100644 src/assembly/META-INF/NOTICE.vm
 create mode 100644 src/assembly/resources/supplemental-models.xml
 create mode 100644 src/main/appended-resources/META-INF/LICENSE
 create mode 100644 src/main/appended-resources/META-INF/NOTICE
 create mode 100644 src/main/resources/META-INF/LEGAL
 create mode 100644 src/test/resources/META-INF/LICENSE

diff --git a/src/assembly/META-INF/LICENSE.vm b/src/assembly/META-INF/LICENSE.vm
new file mode 100644
index 000000000000..f712b8159764
--- /dev/null
+++ b/src/assembly/META-INF/LICENSE.vm
@@ -0,0 +1,2052 @@
+#**
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*#
+## Special cases, for e.g. ASL2.0 licensed works that bundle additional third party works
+#set($bundled-dependencies = ${bundled-dependencies.equalsIgnoreCase("true")})
+#if($bundled-dependencies)
+====
+${project.name} contained works
+
+This product contains additional works that are distributed under licenses
+other than ASL v2. Details below.
+
+#end
+## included in source
+----
+This project incorporates portions of the 'Protocol Buffers' project avaialble
+under a '3-clause BSD' license.
+
+  Copyright 2008, Google Inc.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+      * Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+      * Neither the name of Google Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  Code generated by the Protocol Buffer compiler is owned by the owner
+  of the input file used when generating it.  This code is not
+  standalone and requires a support library to be linked with it.  This
+  support library is itself covered by the above license.
+
+--
+
+This project incorporates part of the 'FreeBSD Documentation Project'
+available under a BSD-style license.
+
+ * Copyright (c) 2001, 2003, 2010 The FreeBSD Documentation Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: doc/share/misc/docbook.css,v 1.15 2010/03/20 04:15:01 hrs Exp $
+
+## macros used for including actual text
+## supplemental from httpcomponents:httpcore
+#macro (httpcomponents_httpcore_license)
+====
+This project contains annotations in the package org.apache.http.annotation
+which are derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls.
+See http://www.jcip.net and the Creative Commons Attribution License
+(http://creativecommons.org/licenses/by/2.5)
+Full text: http://creativecommons.org/licenses/by/2.5/legalcode
+
+License
+
+THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE
+("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF
+THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
+
+BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF
+THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE
+OF SUCH TERMS AND CONDITIONS.
+
+1. Definitions
+
+    "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which
+the Work in its entirety in unmodified form, along with a number of other contributions,
+constituting separate and independent works in themselves, are assembled into a collective whole. A
+work that constitutes a Collective Work will not be considered a Derivative Work (as defined below)
+for the purposes of this License.  "Derivative Work" means a work based upon the Work or upon the
+Work and other pre-existing works, such as a translation, musical arrangement, dramatization,
+fictionalization, motion picture version, sound recording, art reproduction, abridgment,
+condensation, or any other form in which the Work may be recast, transformed, or adapted, except
+that a work that constitutes a Collective Work will not be considered a Derivative Work for the
+purpose of this License. For the avoidance of doubt, where the Work is a musical composition or
+sound recording, the synchronization of the Work in timed-relation with a moving image ("synching")
+will be considered a Derivative Work for the purpose of this License.  "Licensor" means the
+individual or entity that offers the Work under the terms of this License.  "Original Author" means
+the individual or entity who created the Work.  "Work" means the copyrightable work of authorship
+offered under the terms of this License.  "You" means an individual or entity exercising rights
+under this License who has not previously violated the terms of this License with respect to the
+Work, or who has received express permission from the Licensor to exercise rights under this License
+despite a previous violation.
+
+2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights
+arising from fair use, first sale or other limitations on the exclusive rights of the copyright
+owner under copyright law or other applicable laws.
+
+3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a
+worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright)
+license to exercise the rights in the Work as stated below:
+
+    to reproduce the Work, to incorporate the Work into one or more Collective Works, and to
+reproduce the Work as incorporated in the Collective Works; to create and reproduce Derivative
+Works; to distribute copies or phonorecords of, display publicly, perform publicly, and perform
+publicly by means of a digital audio transmission the Work including as incorporated in Collective
+Works; to distribute copies or phonorecords of, display publicly, perform publicly, and perform
+publicly by means of a digital audio transmission Derivative Works.
+
+    For the avoidance of doubt, where the work is a musical composition: Performance Royalties Under
+Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a
+performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public
+digital performance (e.g. webcast) of the Work.  Mechanical Rights and Statutory Royalties. Licensor
+waives the exclusive right to collect, whether individually or via a music rights agency or
+designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work
+("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of
+the US Copyright Act (or the equivalent in other jurisdictions).  Webcasting Rights and Statutory
+Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the
+exclusive right to collect, whether individually or via a performance-rights society (e.g.
+SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to
+the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in
+other jurisdictions).
+
+The above rights may be exercised in all media and formats whether now known or hereafter devised.
+The above rights include the right to make such modifications as are technically necessary to
+exercise the rights in other media and formats. All rights not expressly granted by Licensor are
+hereby reserved.
+
+4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by
+the following restrictions:
+
+    You may distribute, publicly display, publicly perform, or publicly digitally perform the Work
+only under the terms of this License, and You must include a copy of, or the Uniform Resource
+Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly
+display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on
+the Work that alter or restrict the terms of this License or the recipients' exercise of the rights
+granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to
+this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly
+perform, or publicly digitally perform the Work with any technological measures that control access
+or use of the Work in a manner inconsistent with the terms of this License Agreement. The above
+applies to the Work as incorporated in a Collective Work, but this does not require the Collective
+Work apart from the Work itself to be made subject to the terms of this License. If You create a
+Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the
+Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative
+Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative
+Work any credit as required by clause 4(b), as requested.  If you distribute, publicly display,
+publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective
+Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium
+or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if
+supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties
+(e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright
+notice, terms of service or by other reasonable means, the name of such party or parties; the title
+of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if
+any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the
+copyright notice or licensing information for the Work; and in the case of a Derivative Work, a
+credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work
+by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be
+implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or
+Collective Work, at a minimum such credit will appear where any other comparable authorship credit
+appears and in a manner at least as prominent as such other comparable authorship credit.
+
+5. Representations, Warranties and Disclaimer
+
+UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND
+MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY
+OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A
+PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE
+PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE
+EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+
+6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL
+LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE
+OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. Termination
+
+    This License and the rights granted hereunder will terminate automatically upon any breach by
+You of the terms of this License. Individuals or entities who have received Derivative Works or
+Collective Works from You under this License, however, will not have their licenses terminated
+provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2,
+5, 6, 7, and 8 will survive any termination of this License.  Subject to the above terms and
+conditions, the license granted here is perpetual (for the duration of the applicable copyright in
+the Work). Notwithstanding the above, Licensor reserves the right to release the Work under
+different license terms or to stop distributing the Work at any time; provided, however that any
+such election will not serve to withdraw this License (or any other license that has been, or is
+required to be, granted under the terms of this License), and this License will continue in full
+force and effect unless terminated as stated above.
+
+8. Miscellaneous
+
+    Each time You distribute or publicly digitally perform the Work or a Collective Work, the
+Licensor offers to the recipient a license to the Work on the same terms and conditions as the
+license granted to You under this License.  Each time You distribute or publicly digitally perform a
+Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms
+and conditions as the license granted to You under this License.  If any provision of this License
+is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability
+of the remainder of the terms of this License, and without further action by the parties to this
+agreement, such provision shall be reformed to the minimum extent necessary to make such provision
+valid and enforceable.  No term or provision of this License shall be deemed waived and no breach
+consented to unless such waiver or consent shall be in writing and signed by the party to be charged
+with such waiver or consent.  This License constitutes the entire agreement between the parties with
+respect to the Work licensed here. There are no understandings, agreements or representations with
+respect to the Work not specified here. Licensor shall not be bound by any additional provisions
+that may appear in any communication from You. This License may not be modified without the mutual
+written agreement of the Licensor and You.
+#end
+## Supplemental from Netty netty-all 4
+#macro(netty_all_4)
+----
+The Netty Project contained works
+
+This product contains the extensions to Java Collections Framework which has
+been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
+
+  * LICENSE (Public Domain):
+The person or persons who have associated work with this document (the
+"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of
+his knowledge, the work of authorship identified is in the public domain of
+the country from which the work is published, or (b) hereby dedicates whatever
+copyright the dedicators holds in the work of authorship identified below (the
+"Work") to the public domain. A certifier, moreover, dedicates any copyright
+interest he may have in the associated work, and for these purposes, is
+described as a "dedicator" below.
+
+A certifier has taken reasonable steps to verify the copyright status of this
+work. Certifier recognizes that his good faith efforts may not shield him from
+liability if in fact the work certified is not in the public domain.
+
+Dedicator makes this dedication for the benefit of the public at large and to
+the detriment of the Dedicator's heirs and successors. Dedicator intends this
+dedication to be an overt act of relinquishment in perpetuity of all present
+and future rights under copyright law, whether vested or contingent, in the
+Work. Dedicator understands that such relinquishment of all rights includes
+the relinquishment of all rights to enforce (by lawsuit or otherwise) those
+copyrights in the Work.
+
+Dedicator recognizes that, once placed in the public domain, the Work may be
+freely reproduced, distributed, transmitted, used, modified, built upon, or
+otherwise exploited by anyone for any purpose, commercial or non-commercial,
+and in any way, including by methods that have not yet been invented or
+conceived.
+  * HOMEPAGE:
+    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
+    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+
+This product contains a modified version of Robert Harder's Public Domain
+Base64 Encoder and Decoder, which can be obtained at:
+
+  * LICENSE (Public Domain):
+The person or persons who have associated work with this document (the
+"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of
+his knowledge, the work of authorship identified is in the public domain of
+the country from which the work is published, or (b) hereby dedicates whatever
+copyright the dedicators holds in the work of authorship identified below (the
+"Work") to the public domain. A certifier, moreover, dedicates any copyright
+interest he may have in the associated work, and for these purposes, is
+described as a "dedicator" below.
+
+A certifier has taken reasonable steps to verify the copyright status of this
+work. Certifier recognizes that his good faith efforts may not shield him from
+liability if in fact the work certified is not in the public domain.
+
+Dedicator makes this dedication for the benefit of the public at large and to
+the detriment of the Dedicator's heirs and successors. Dedicator intends this
+dedication to be an overt act of relinquishment in perpetuate of all present
+and future rights under copyright law, whether vested or contingent, in the
+Work. Dedicator understands that such relinquishment of all rights includes
+the relinquishment of all rights to enforce (by lawsuit or otherwise) those
+copyrights in the Work.
+
+Dedicator recognizes that, once placed in the public domain, the Work may be
+freely reproduced, distributed, transmitted, used, modified, built upon, or
+otherwise exploited by anyone for any purpose, commercial or non-commercial,
+and in any way, including by methods that have not yet been invented or
+conceived.
+  * HOMEPAGE:
+    * http://iharder.sourceforge.net/current/java/base64/
+
+This product contains a modified portion of 'Webbit', an event based
+WebSocket and HTTP server, which can be obtained at:
+
+  * LICENSE (BSD License):
+Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the Webbit nor the names of
+  its contributors may be used to endorse or promote products
+  derived from this software without specific prior written
+  permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+  * HOMEPAGE:
+    * https://github.com/joewalnes/webbit
+
+This product contains a modified portion of 'SLF4J', a simple logging
+facade for Java, which can be obtained at:
+
+  * LICENSE (MIT License):
+/*
+ * Copyright (c) 2004-2007 QOS.ch
+ * All rights reserved.
+ *
+ * Permission is hereby granted, free  of charge, to any person obtaining
+ * a  copy  of this  software  and  associated  documentation files  (the
+ * "Software"), to  deal in  the Software without  restriction, including
+ * without limitation  the rights to  use, copy, modify,  merge, publish,
+ * distribute,  sublicense, and/or sell  copies of  the Software,  and to
+ * permit persons to whom the Software  is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The  above  copyright  notice  and  this permission  notice  shall  be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
+ * EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
+ * MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+  * HOMEPAGE:
+    * http://www.slf4j.org/
+#end
+## Supplemental from commons-math
+## parameter two-and-three true = updated to also cover commons-math 2, just 3 otherwise
+#macro(commons_math_license $two-and-three)
+----
+APACHE COMMONS MATH DERIVATIVE WORKS:
+
+The Apache commons-math library includes a number of subcomponents
+whose implementation is derived from original sources written
+in C or Fortran.  License terms of the original sources
+are reproduced below.
+
+===============================================================================
+For the lmder, lmpar and qrsolv Fortran routine from minpack and translated in
+the LevenbergMarquardtOptimizer class in package
+org.apache.commons.math3.optimization.general
+#if(${two-and-three})
+and package
+org.apache.commons.math.optimization.general
+#end
+Original source copyright and license statement:
+
+Minpack Copyright Notice (1999) University of Chicago.  All rights reserved
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions of source code must retain the above
+copyright notice, this list of conditions and the following
+disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials
+provided with the distribution.
+
+3. The end-user documentation included with the
+redistribution, if any, must include the following
+acknowledgment:
+
+   "This product includes software developed by the
+   University of Chicago, as Operator of Argonne National
+   Laboratory.
+
+Alternately, this acknowledgment may appear in the software
+itself, if and wherever such third-party acknowledgments
+normally appear.
+
+4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS"
+WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE
+UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND
+THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE
+OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY
+OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
+USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF
+THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4)
+DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION
+UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL
+BE CORRECTED.
+
+5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT
+HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF
+ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT,
+INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF
+ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF
+PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
+SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
+(INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
+EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
+POSSIBILITY OF SUCH LOSS OR DAMAGES.
+===============================================================================
+
+Copyright and license statement for the odex Fortran routine developed by
+E. Hairer and G. Wanner and translated in GraggBulirschStoerIntegrator class
+in package org.apache.commons.math3.ode.nonstiff:
+#if(${two-and-three})
+and in package org.apache.commons.math.ode.nonstiff:
+#end
+
+Copyright (c) 2004, Ernst Hairer
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+===============================================================================
+
+Copyright and license statement for the original lapack fortran routines
+translated in EigenDecompositionImpl class in package
+org.apache.commons.math3.linear:
+#if(${two-and-three})
+and package
+org.apache.commons.math.linear:
+#end
+
+Copyright (c) 1992-2008 The University of Tennessee.  All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+===============================================================================
+
+Copyright and license statement for the original Mersenne twister C
+routines translated in MersenneTwister class in package
+org.apache.commons.math3.random:
+#if(${two-and-three})
+and package
+org.apache.commons.math.random:
+#end
+
+   Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+
+     3. The names of its contributors may not be used to endorse or promote
+        products derived from this software without specific prior written
+        permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+===============================================================================
+
+The class "org.apache.commons.math3.exception.util.LocalizedFormatsTest" is
+an adapted version of "OrekitMessagesTest" test class for the Orekit library
+The "org.apache.commons.math3.analysis.interpolation.HermiteInterpolator"
+has been imported from the Orekit space flight dynamics library.
+
+Th Orekit library is described at:
+  https://www.orekit.org/forge/projects/orekit
+The original files are distributed under the terms of the Apache 2 license
+which is: Copyright 2010 CS Communication & Systèmes
+
+#end
+## Supplemental from commons-math if just commons-math 2 is present
+#macro(commons_math_two_license)
+----
+APACHE COMMONS MATH DERIVATIVE WORKS:
+
+The Apache commons-math library includes a number of subcomponents
+whose implementation is derived from original sources written
+in C or Fortran.  License terms of the original sources
+are reproduced below.
+
+===============================================================================
+For the lmder, lmpar and qrsolv Fortran routine from minpack and translated in
+the LevenbergMarquardtOptimizer class in package
+org.apache.commons.math.optimization.general
+Original source copyright and license statement:
+
+Minpack Copyright Notice (1999) University of Chicago.  All rights reserved
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions of source code must retain the above
+copyright notice, this list of conditions and the following
+disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials
+provided with the distribution.
+
+3. The end-user documentation included with the
+redistribution, if any, must include the following
+acknowledgment:
+
+   "This product includes software developed by the
+   University of Chicago, as Operator of Argonne National
+   Laboratory.
+
+Alternately, this acknowledgment may appear in the software
+itself, if and wherever such third-party acknowledgments
+normally appear.
+
+4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS"
+WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE
+UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND
+THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE
+OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY
+OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
+USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF
+THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4)
+DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION
+UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL
+BE CORRECTED.
+
+5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT
+HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF
+ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT,
+INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF
+ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF
+PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
+SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
+(INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
+EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
+POSSIBILITY OF SUCH LOSS OR DAMAGES.
+===============================================================================
+
+Copyright and license statement for the odex Fortran routine developed by
+E. Hairer and G. Wanner and translated in GraggBulirschStoerIntegrator class
+in package org.apache.commons.math.ode.nonstiff:
+
+Copyright (c) 2004, Ernst Hairer
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+===============================================================================
+
+Copyright and license statement for the original lapack fortran routines
+translated in EigenDecompositionImpl class in package
+org.apache.commons.math.linear:
+
+Copyright (c) 1992-2008 The University of Tennessee.  All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+===============================================================================
+
+Copyright and license statement for the original Mersenne twister C
+routines translated in MersenneTwister class in package
+org.apache.commons.math.random:
+
+   Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+
+     3. The names of its contributors may not be used to endorse or promote
+        products derived from this software without specific prior written
+        permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+===============================================================================
+#end
+## Hadoop supplemental
+#macro(hadoop_license)
+--
+APACHE HADOOP SUBCOMPONENTS:
+
+The Apache Hadoop project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+For the org.apache.hadoop.util.bloom.* classes:
+
+/**
+ *
+ * Copyright (c) 2005, European Commission project OneLab under contract
+ * 034819 (http://www.one-lab.org)
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the name of the University Catholique de Louvain - UCL
+ *    nor the names of its contributors may be used to endorse or
+ *    promote products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+## native code not included, skip section on native o.a.h.u bulk_crc32.c
+## lz4 native code not included, skip
+#end
+#macro (htrace_license)
+## HTrace supplemental for htrace-core is a no-op.
+## skip levigo
+## skip kingpin
+## skip units
+## skip D3
+## skip bootstrap
+## skip Gorilla mux
+## skip godep
+## skip underscore
+## skip jquery
+## skip backbone
+## relocated jackson 2.4.0 is ASLv2 with no notice
+## relocated commons-logging 1.1.1 is in NOTICE.vm
+#end
+#macro (thrift_license)
+## Thrift supplemental for libthrift is a no-op.
+## skip erlang
+## skip boost detection
+## skip cpp compiler source
+#end
+## Supplemental for jruby inclusion
+#macro(jruby_license)
+====
+JRuby Incorporated Works
+
+Some additional libraries distributed with JRuby are not covered by
+JRuby's licence.
+
+## skip bytelist and yydebug because they're covered by the jruby copyright and license
+## skip rails benchmark javascript, because it's not in the complete jar
+The "rake" library (http://rake.rubyforge.org/) is distributed under
+ the MIT license (available above), and has the following copyright:
+
+   Copyright (c) 2003, 2004 Jim Weirich
+## jruby bundles asm, but we already ref that in our included deps
+
+## joda-time only in NOTICE because it's ASLv2
+## commons-logging in NOTICE because it's ASLv2
+ build_lib/apt-mirror-api.jar, build_lib/bnd-0.0.249.jar,
+ BSF and ant are distributed under the Apache Software License,
+ Version 1.1 (shown below)
+
+--- ASL v1.1 license
+/* ====================================================================
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2000 The Apache Software Foundation.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The end-user documentation included with the redistribution,
+ *    if any, must include the following acknowledgment:
+ *       "This product includes software developed by the
+ *        Apache Software Foundation (http://www.apache.org/)."
+ *    Alternately, this acknowledgment may appear in the software itself,
+ *    if and wherever such third-party acknowledgments normally appear.
+ *
+ * 4. The names "Apache" and "Apache Software Foundation" must
+ *    not be used to endorse or promote products derived from this
+ *    software without prior written permission. For written
+ *    permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache",
+ *    nor may "Apache" appear in their name, without prior written
+ *    permission of the Apache Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ * Portions of this software are based upon public domain software
+ * originally written at the National Center for Supercomputing Applications,
+ * University of Illinois, Urbana-Champaign.
+ */
+--- End ASL v1.1 license
+
+ build_lib/constantine.jar (http://kenai.com/projects/constantine)
+## jruby bundles jcodings and joni, but we already ref that in our included deps
+ distributed under the MIT license (available above).
+
+## section on ASLv2 incorporated works just in NOTICE
+## skip emma and junit; both aren't in the complete jar
+ build_lib/jline-* (http://jline.sourceforge.net) is distributed
+ under the following BSD-style license:
+
+   Redistribution and use in source and binary forms, with or
+   without modification, are permitted provided that the following
+   conditions are met:
+
+   Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with
+   the distribution.
+
+   Neither the name of JLine nor the names of its contributors
+   may be used to endorse or promote products derived from this
+   software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+   BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+   AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+   EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+   OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+   AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+   LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+   IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+   OF THE POSSIBILITY OF SUCH DAMAGE.
+
+## I couldn't determine what use of bouncycastle this referred to
+## so I'm not sure if it's included in the jruby-complete jar or not.
+ Bouncycastle is released under the MIT license (available above),
+ and is Copyright (c) 2000 - 2006 The Legion Of The Bouncy Castle.
+
+ The "yecht" library is Copyright (c) 2009-2011 Ola Bini, and released
+ under the MIT license (available above).
+
+## jruby bundles jquery and we don't otherwise include it in 0.94
+  JRuby bundles portions of the 'JQuery' project under the terms of the MIT license.
+
+    Copyright (c) 2008 John Resig (jquery.com)
+    http://jquery.com/
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+  most files found in lib/ruby/1.8, lib/ruby/1.9, and lib/ruby/site_ruby/1.8/rubygems*
+  are distributed under The Ruby License's non-GPL provision (available below).
+
+Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
+You can redistribute it and/or modify it under either the terms of the GPL
+(see the file GPL), or the conditions below:
+
+  1. You may make and give away verbatim copies of the source form of the
+     software without restriction, provided that you duplicate all of the
+     original copyright notices and associated disclaimers.
+
+  2. You may modify your copy of the software in any way, provided that
+     you do at least ONE of the following:
+
+       a) place your modifications in the Public Domain or otherwise
+          make them Freely Available, such as by posting said
+	  modifications to Usenet or an equivalent medium, or by allowing
+	  the author to include your modifications in the software.
+
+       b) use the modified software only within your corporation or
+          organization.
+
+       c) give non-standard binaries non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  3. You may distribute the software in object code or binary form,
+     provided that you do at least ONE of the following:
+
+       a) distribute the binaries and library files of the software,
+	  together with instructions (in the manual page or equivalent)
+	  on where to get the original distribution.
+
+       b) accompany the distribution with the machine-readable source of
+	  the software.
+
+       c) give non-standard binaries non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  4. You may modify and include the part of the software into any other
+     software (possibly commercial).  But some files in the distribution
+     are not written by the author, so that they are not under these terms.
+
+     For the list of those files and their copying conditions, see the
+     file LEGAL.
+
+  5. The scripts and library files supplied as input to or produced as
+     output from the software do not automatically fall under the
+     copyright of the software, but belong to whomever generated them,
+     and may be sold commercially, and may be aggregated with this
+     software.
+
+  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+     PURPOSE.
+#end
+## modules with bundled works in source
+## when true, we're in a module that makes a binary dist with
+## bundled works.
+#if(${bundled-dependencies})
+====
+## gather up CDDL licensed works
+#set($cddl_1_0 = [])
+#set($cddl_1_1 = [])
+## gather up MIT licensed works
+#set($mit = [])
+## gather up vanilla BSD 3-clause works
+#set($bsd3 = [])
+## gather up CPL 1.0 works
+#set($cpl = [])
+## track commons-math
+#set($commons-math-two = false)
+#set($commons-math-three = false)
+## track if we need jruby additionals.
+#set($jruby = false)
+## track hadoops
+#set($hadoop = false)
+## include LICENSE sections for anything not under ASL2.0
+#foreach( ${dep} in ${projects} )
+#if(${debug-print-included-work-info.equalsIgnoreCase("true")})
+=====
+Check license for included work
+
+#if(${dep.licenses.size()} == 0)
+No license info included; must update supplemental-models
+#else
+License short name: '${dep.licenses[0].name}'
+#end
+
+<groupId>${dep.groupId}</groupId>
+<artifactId>${dep.artifactId}</artifactId>
+<version>${dep.version}</version>
+
+maven central search
+g:${dep.groupId} AND a:${dep.artifactId} AND v:${dep.version}
+
+project website
+${dep.url}
+project source
+${dep.scm.url}
+#end
+#set($aggregated=false)
+## Check for our set of known dependencies that require manual LICENSE additions.
+#if($dep.artifactId.equals("commons-math"))
+#set($commons-math-two=true)
+#end
+#if($dep.artifactId.equals("commons-math3"))
+#set($commons-math-three=true)
+#end
+#if($dep.artifactId.equals("jruby-complete"))
+#set($jruby=true)
+#end
+#if($dep.artifactId.startsWith("hadoop"))
+#set($hadoop=true)
+#end
+#if($dep.artifactId.equals("hbase-server"))
+#hbase_server_license()
+#end
+#if($dep.artifactId.equals("httpcore") && $dep.groupId.equals("org.apache.httpcomponents"))
+## Server includes httpcomponents:httpcore 4.1.2 (rather than 4.2.5 from client / assembly), and
+## that version has no supplemental material because the JCIP-ANNOTATIONS hadn't been added yet.
+#if(${dep.version.startsWith("4.2")})
+#httpcomponents_httpcore_license()
+#end
+#end
+#if($dep.artifactId.equals("netty-all") && $dep.version.startsWith("4"))
+#netty_all_4()
+#end
+#if($dep.artifactId.equals("htrace-core"))
+#htrace_license()
+#end
+#if($dep.artifactId.equals("libthrift"))
+#thrift_license()
+#end
+## Note that this will fail the build if we don't have a license. update supplemental-models.
+#if( !(${dep.licenses[0].name.contains("Apache Software License, Version 2.0")}) )
+#if( ${dep.licenses[0].name.contains("CDDL")} )
+#if( ${dep.licenses[0].name.contains("1.0")} )
+#set($aggregated = $cddl_1_0.add($dep))
+#elseif( ${dep.licenses[0].name.contains("1.1")} )
+#set($aggregated = $cddl_1_1.add($dep))
+#end
+#end
+#if( ${dep.licenses[0].name.contains("MIT")} )
+#set($aggregated = $mit.add($dep))
+#end
+#if(${dep.licenses[0].name.equals("BSD 3-Clause License")})
+#set($aggregated = $bsd3.add($dep))
+#end
+#if(${dep.licenses[0].name.equals("Common Public License Version 1.0")})
+#set($aggregated = $cpl.add($dep))
+#end
+#if(!${aggregated})
+--
+This product includes ${dep.name} licensed under the ${dep.licenses[0].name}.
+
+${dep.licenses[0].comments}
+#end
+#end
+#end
+#if($hadoop)
+#hadoop_license()
+#end
+#if($commons-math-three)
+#commons_math_license($commons-math-two)
+#elseif($commons-math-two)
+#commons_math_two_license()
+#end
+#if(!(${mit.isEmpty()}))
+====
+## print all the MIT licensed works
+This product includes the following works licensed under the MIT license:
+
+#foreach($dep in $mit)
+#if( !(${dep.licenses[0].comments.isEmpty()}) )
+  * ${dep.name}, ${dep.licenses[0].comments}
+#else
+  * ${dep.name}
+#end
+#end
+
+The MIT License (MIT)
+
+Copyright (c) <year> <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+#end
+#if(!(${bsd3.isEmpty()}))
+====
+## print all the BSD 3 Clause licensed works (only when they used the BSD-3 text directly)
+This product includes the following works licensed under the BSD 3-Clause license:
+
+#foreach($dep in $bsd3)
+#if( !(${dep.licenses[0].comments.isEmpty()}) )
+  * ${dep.name}, ${dep.licenses[0].comments}
+#else
+  * ${dep.name}
+#end
+#end
+
+Copyright (c) <YEAR>, <OWNER>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#end
+#if(!(${cddl_1_0.isEmpty()}))
+====
+## print all the CDDL 1.0 licensed works
+This product includes the following works licensed under the CDDL 1.0 license:
+
+#foreach($dep in $cddl_1_0)
+#if( !(${dep.licenses[0].comments.isEmpty()}) )
+  * ${dep.name}, ${dep.licenses[0].comments}
+#else
+  * ${dep.name}
+#end
+#end
+
+COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 1.
+
+Definitions.
+
+1.1. Contributor means each individual or entity that creates or contributes to the creation of
+Modifications.
+
+1.2. Contributor Version means the combination of the Original Software, prior Modifications used by
+a Contributor (if any), and the Modifications made by that particular Contributor.
+
+1.3. Covered Software means (a) the Original Software, or (b) Modifications, or (c) the combination
+of files containing Original Software with files containing Modifications, in each case including
+portions thereof.
+
+1.4. Executable means the Covered Software in any form other than Source Code.
+
+1.5. Initial Developer means the individual or entity that first makes Original Software available
+under this License.
+
+1.6. Larger Work means a work which combines Covered Software or portions thereof with code not
+governed by the terms of this License.
+
+1.7. License means this document.
+
+1.8. Licensable means having the right to grant, to the maximum extent possible, whether at the time
+of the initial grant or subsequently acquired, any and all of the rights conveyed herein.
+
+1.9. Modifications means the Source Code and Executable form of any of the following: A. Any file
+that results from an addition to, deletion from or modification of the contents of a file containing
+Original Software or previous Modifications; B. Any new file that contains any part of the Original
+Software or previous Modification; or C. Any new file that is contributed or otherwise made
+available under the terms of this License.
+
+1.10. Original Software means the Source Code and Executable form of computer software code that is
+originally released under this License.
+
+1.11. Patent Claims means any patent claim(s), now owned or hereafter acquired, including without
+limitation, method, process, and apparatus claims, in any patent Licensable by grantor.
+
+1.12. Source Code means (a) the common form of computer software code in which modifications are
+made and (b) associated documentation included in or with such code.
+
+1.13. You (or Your) means an individual or a legal entity exercising rights under, and complying
+with all of the terms of, this License. For legal entities, You includes any entity which controls,
+is controlled by, or is under common control with You. For purposes of this definition, control
+means (a) the power, direct or indirect, to cause the direction or management of such entity,
+whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the
+outstanding shares or beneficial ownership of such entity.
+
+2. License Grants.
+
+ 2.1. The Initial Developer Grant. Conditioned upon Your compliance with Section 3.1 below and
+subject to third party intellectual property claims, the Initial Developer hereby grants You a
+world-wide, royalty-free, non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark) Licensable by Initial
+Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original
+Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and
+
+(b) under Patent Claims infringed by the making, using or selling of Original Software, to make,
+have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original
+Software (or portions thereof);
+
+ (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer
+first distributes or otherwise makes the Original Software available to a third party under the
+terms of this License;
+
+ (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You
+delete from the Original Software, or (2) for infringements caused by: (i) the modification of the
+Original Software, or (ii) the combination of the Original Software with other software or devices.
+
+2.2. Contributor Grant. Conditioned upon Your compliance with Section 3.1 below and subject to third
+party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to
+use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by
+such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as
+Covered Software and/or as part of a Larger Work; and
+
+(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that
+Contributor either alone and/or in combination with its Contributor Version (or portions of such
+combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1)
+Modifications made by that Contributor (or portions thereof); and (2) the combination of
+Modifications made by that Contributor with its Contributor Version (or portions of such
+combination).
+
+(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first
+distributes or otherwise makes the Modifications available to a third party.
+
+(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that
+Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third
+party modifications of Contributor Version, or (ii) the combination of Modifications made by that
+Contributor with other software (except as part of the Contributor Version) or other devices; or (3)
+under Patent Claims infringed by Covered Software in the absence of Modifications made by that
+Contributor.
+
+3. Distribution Obligations.
+
+3.1. Availability of Source Code. Any Covered Software that You distribute or otherwise make
+available in Executable form must also be made available in Source Code form and that Source Code
+form must be distributed only under the terms of this License. You must include a copy of this
+License with every copy of the Source Code form of the Covered Software You distribute or otherwise
+make available. You must inform recipients of any such Covered Software in Executable form as to how
+they can obtain such Covered Software in Source Code form in a reasonable manner on or through a
+medium customarily used for software exchange.
+
+3.2. Modifications. The Modifications that You create or to which You contribute are governed by the
+terms of this License. You represent that You believe Your Modifications are Your original
+creation(s) and/or You have sufficient rights to grant the rights conveyed by this License.
+
+3.3. Required Notices. You must include a notice in each of Your Modifications that identifies You
+as the Contributor of the Modification. You may not remove or alter any copyright, patent or
+trademark notices contained within the Covered Software, or any notices of licensing or any
+descriptive text giving attribution to any Contributor or the Initial Developer.
+
+3.4. Application of Additional Terms. You may not offer or impose any terms on any Covered Software
+in Source Code form that alters or restricts the applicable version of this License or the
+recipients rights hereunder. You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered Software. However, you may
+do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You
+must make it absolutely clear that any such warranty, support, indemnity or liability obligation is
+offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor
+for any liability incurred by the Initial Developer or such Contributor as a result of warranty,
+support, indemnity or liability terms You offer.
+
+3.5. Distribution of Executable Versions. You may distribute the Executable form of the Covered
+Software under the terms of this License or under the terms of a license of Your choice, which may
+contain terms different from this License, provided that You are in compliance with the terms of
+this License and that the license for the Executable form does not attempt to limit or alter the
+recipients rights in the Source Code form from the rights set forth in this License. If You
+distribute the Covered Software in Executable form under a different license, You must make it
+absolutely clear that any terms which differ from this License are offered by You alone, not by the
+Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every
+Contributor for any liability incurred by the Initial Developer or such Contributor as a result of
+any such terms You offer.
+
+3.6. Larger Works. You may create a Larger Work by combining Covered Software with other code not
+governed by the terms of this License and distribute the Larger Work as a single product. In such a
+case, You must make sure the requirements of this License are fulfilled for the Covered Software.
+
+4. Versions of the License.
+
+4.1. New Versions. Sun Microsystems, Inc. is the initial license steward and may publish revised
+and/or new versions of this License from time to time. Each version will be given a distinguishing
+version number. Except as provided in Section 4.3, no one other than the license steward has the
+right to modify this License.
+
+4.2. Effect of New Versions. You may always continue to use, distribute or otherwise make the
+Covered Software available under the terms of the version of the License under which You originally
+received the Covered Software. If the Initial Developer includes a notice in the Original Software
+prohibiting it from being distributed or otherwise made available under any subsequent version of
+the License, You must distribute and make the Covered Software available under the terms of the
+version of the License under which You originally received the Covered Software. Otherwise, You may
+also choose to use, distribute or otherwise make the Covered Software available under the terms of
+any subsequent version of the License published by the license steward.
+
+4.3. Modified Versions. When You are an Initial Developer and You want to create a new license for
+Your Original Software, You may create and use a modified version of this License if You: (a) rename
+the license and remove any references to the name of the license steward (except to note that the
+license differs from this License); and (b) otherwise make it clear that the license contains terms
+which differ from this License.
+
+5. DISCLAIMER OF WARRANTY. COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS,
+WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES
+THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR
+NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH
+YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR
+ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS
+DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE
+IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
+
+6. TERMINATION.
+
+6.1. This License and the rights granted hereunder will terminate automatically if You fail to
+comply with terms herein and fail to cure such breach within 30 days of becoming aware of the
+breach. Provisions which, by their nature, must remain in effect beyond the termination of this
+License shall survive.
+
+6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against
+Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert
+such claim is referred to as Participant) alleging that the Participant Software (meaning the
+Contributor Version where the Participant is a Contributor or the Original Software where the
+Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all
+rights granted directly or indirectly to You by such Participant, the Initial Developer (if the
+Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this
+License shall, upon 60 days notice from Participant terminate prospectively and automatically at the
+expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim
+with respect to the Participant Software against such Participant either unilaterally or pursuant to
+a written agreement with Participant.
+
+6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have
+been validly granted by You or any distributor hereunder prior to termination (excluding licenses
+granted to You by any distributor) shall survive termination.
+
+7. LIMITATION OF LIABILITY. UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
+(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER
+CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE
+LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
+CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE,
+COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH
+PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY
+SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO
+THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION
+OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY
+TO YOU.
+
+8. U.S. GOVERNMENT END USERS. The Covered Software is a commercial item, as that term is defined in
+48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as that term is defined at
+48 C.F.R.  252.227-7014(a)(1)) and commercial computer software documentation as such terms are used
+in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through
+227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those
+rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other
+FAR, DFAR, or other clause or provision that addresses Government rights in computer software under
+this License.
+
+9. MISCELLANEOUS. This License represents the complete agreement concerning subject matter hereof.
+If any provision of this License is held to be unenforceable, such provision shall be reformed only
+to the extent necessary to make it enforceable. This License shall be governed by the law of the
+jurisdiction specified in a notice contained within the Original Software (except to the extent
+applicable law, if any, provides otherwise), excluding such jurisdictions conflict-of-law
+provisions. Any litigation relating to this License shall be subject to the jurisdiction of the
+courts located in the jurisdiction and venue specified in a notice contained within the Original
+Software, with the losing party responsible for costs, including, without limitation, court costs
+and reasonable attorneys fees and expenses. The application of the United Nations Convention on
+Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which
+provides that the language of a contract shall be construed against the drafter shall not apply to
+this License. You agree that You alone are responsible for compliance with the United States export
+administration regulations (and the export control laws and regulation of any other countries) when
+You use, distribute or otherwise make available any Covered Software.
+
+10. RESPONSIBILITY FOR CLAIMS. As between Initial Developer and the Contributors, each party is
+responsible for claims and damages arising, directly or indirectly, out of its utilization of rights
+under this License and You agree to work with Initial Developer and Contributors to distribute such
+responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute
+any admission of liability.
+
+NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) The code
+released under the CDDL shall be governed by the laws of the State of California (excluding
+conflict-of-law provisions). Any litigation relating to this License shall be subject to the
+jurisdiction of the Federal Courts of the Northern District of California and the state courts of
+the State of California, with venue lying in Santa Clara County, California.
+
+#end
+#if(!(${cddl_1_1.isEmpty()}))
+====
+## print all the CDDL 1.1 licensed works
+This product includes the following works licensed under the CDDL 1.1 license:
+
+#foreach($dep in $cddl_1_1)
+#if( !(${dep.licenses[0].comments.isEmpty()}) )
+  * ${dep.name}, ${dep.licenses[0].comments}
+#else
+  * ${dep.name}
+#end
+#end
+
+COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1
+
+1. Definitions.
+
+1.1. “Contributor” means each individual or entity that creates or contributes to the creation of
+Modifications.
+
+1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used
+by a Contributor (if any), and the Modifications made by that particular Contributor.
+
+1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the
+combination of files containing Original Software with files containing Modifications, in each case
+including portions thereof.
+
+1.4. “Executable” means the Covered Software in any form other than Source Code.
+
+1.5. “Initial Developer” means the individual or entity that first makes Original Software available
+under this License.
+
+1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not
+governed by the terms of this License.
+
+1.7. “License” means this document.
+
+1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the
+time of the initial grant or subsequently acquired, any and all of the rights conveyed herein.
+
+1.9. “Modifications” means the Source Code and Executable form of any of the following:
+
+A. Any file that results from an addition to, deletion from or modification of the contents of a
+file containing Original Software or previous Modifications;
+
+B. Any new file that contains any part of the Original Software or previous Modification; or
+
+C. Any new file that is contributed or otherwise made available under the terms of this License.
+
+1.10. “Original Software” means the Source Code and Executable form of computer software code that
+is originally released under this License.
+
+1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without
+limitation, method, process, and apparatus claims, in any patent Licensable by grantor.
+
+1.12. “Source Code” means (a) the common form of computer software code in which modifications are
+made and (b) associated documentation included in or with such code.
+
+1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying
+with all of the terms of, this License. For legal entities, “You” includes any entity which
+controls, is controlled by, or is under common control with You. For purposes of this definition,
+“control” means (a) the power, direct or indirect, to cause the direction or management of such
+entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the
+outstanding shares or beneficial ownership of such entity.
+
+2. License Grants.
+
+2.1. The Initial Developer Grant.
+
+Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual
+property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive
+license:
+
+(a) under intellectual property rights (other than patent or trademark) Licensable by Initial
+Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original
+Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and
+
+(b) under Patent Claims infringed by the making, using or selling of Original Software, to make,
+have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original
+Software (or portions thereof).
+
+(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer
+first distributes or otherwise makes the Original Software available to a third party under the
+terms of this License.
+
+(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete
+from the Original Software, or (2) for infringements caused by: (i) the modification of the Original
+Software, or (ii) the combination of the Original Software with other software or devices.
+
+2.2. Contributor Grant.
+
+Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual
+property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive
+license:
+
+(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to
+use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by
+such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as
+Covered Software and/or as part of a Larger Work; and
+
+(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that
+Contributor either alone and/or in combination with its Contributor Version (or portions of such
+combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1)
+Modifications made by that Contributor (or portions thereof); and (2) the combination of
+Modifications made by that Contributor with its Contributor Version (or portions of such
+combination).
+
+(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first
+distributes or otherwise makes the Modifications available to a third party.
+
+(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that
+Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third
+party modifications of Contributor Version, or (ii) the combination of Modifications made by that
+Contributor with other software (except as part of the Contributor Version) or other devices; or (3)
+under Patent Claims infringed by Covered Software in the absence of Modifications made by that
+Contributor.
+
+3. Distribution Obligations.
+
+3.1. Availability of Source Code.
+
+Any Covered Software that You distribute or otherwise make available in Executable form must also be
+made available in Source Code form and that Source Code form must be distributed only under the
+terms of this License. You must include a copy of this License with every copy of the Source Code
+form of the Covered Software You distribute or otherwise make available. You must inform recipients
+of any such Covered Software in Executable form as to how they can obtain such Covered Software in
+Source Code form in a reasonable manner on or through a medium customarily used for software
+exchange.
+
+3.2. Modifications.
+
+The Modifications that You create or to which You contribute are governed by the terms of this
+License. You represent that You believe Your Modifications are Your original creation(s) and/or You
+have sufficient rights to grant the rights conveyed by this License.
+
+3.3. Required Notices.
+
+You must include a notice in each of Your Modifications that identifies You as the Contributor of
+the Modification. You may not remove or alter any copyright, patent or trademark notices contained
+within the Covered Software, or any notices of licensing or any descriptive text giving attribution
+to any Contributor or the Initial Developer.
+
+3.4. Application of Additional Terms.
+
+You may not offer or impose any terms on any Covered Software in Source Code form that alters or
+restricts the applicable version of this License or the recipients' rights hereunder. You may choose
+to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or
+more recipients of Covered Software. However, you may do so only on Your own behalf, and not on
+behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such
+warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree
+to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial
+Developer or such Contributor as a result of warranty, support, indemnity or liability terms You
+offer.
+
+3.5. Distribution of Executable Versions.
+
+You may distribute the Executable form of the Covered Software under the terms of this License or
+under the terms of a license of Your choice, which may contain terms different from this License,
+provided that You are in compliance with the terms of this License and that the license for the
+Executable form does not attempt to limit or alter the recipient's rights in the Source Code form
+from the rights set forth in this License. If You distribute the Covered Software in Executable form
+under a different license, You must make it absolutely clear that any terms which differ from this
+License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to
+indemnify the Initial Developer and every Contributor for any liability incurred by the Initial
+Developer or such Contributor as a result of any such terms You offer.
+
+3.6. Larger Works.
+
+You may create a Larger Work by combining Covered Software with other code not governed by the terms
+of this License and distribute the Larger Work as a single product. In such a case, You must make
+sure the requirements of this License are fulfilled for the Covered Software.
+
+4. Versions of the License.
+
+4.1. New Versions.
+
+Oracle is the initial license steward and may publish revised and/or new versions of this License
+from time to time. Each version will be given a distinguishing version number. Except as provided in
+Section 4.3, no one other than the license steward has the right to modify this License.
+
+4.2. Effect of New Versions.
+
+You may always continue to use, distribute or otherwise make the Covered Software available under
+the terms of the version of the License under which You originally received the Covered Software. If
+the Initial Developer includes a notice in the Original Software prohibiting it from being
+distributed or otherwise made available under any subsequent version of the License, You must
+distribute and make the Covered Software available under the terms of the version of the License
+under which You originally received the Covered Software. Otherwise, You may also choose to use,
+distribute or otherwise make the Covered Software available under the terms of any subsequent
+version of the License published by the license steward.
+
+4.3. Modified Versions.
+
+When You are an Initial Developer and You want to create a new license for Your Original Software,
+You may create and use a modified version of this License if You: (a) rename the license and remove
+any references to the name of the license steward (except to note that the license differs from this
+License); and (b) otherwise make it clear that the license contains terms which differ from this
+License.
+
+5. DISCLAIMER OF WARRANTY.
+
+COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND,
+EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS
+FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO
+THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE
+DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST
+OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN
+ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER
+THIS DISCLAIMER.
+
+6. TERMINATION.
+
+6.1. This License and the rights granted hereunder will terminate automatically if You fail to
+comply with terms herein and fail to cure such breach within 30 days of becoming aware of the
+breach. Provisions which, by their nature, must remain in effect beyond the termination of this
+License shall survive.
+
+6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against
+Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert
+such claim is referred to as “Participant”) alleging that the Participant Software (meaning the
+Contributor Version where the Participant is a Contributor or the Original Software where the
+Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all
+rights granted directly or indirectly to You by such Participant, the Initial Developer (if the
+Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this
+License shall, upon 60 days notice from Participant terminate prospectively and automatically at the
+expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim
+with respect to the Participant Software against such Participant either unilaterally or pursuant to
+a written agreement with Participant.
+
+6.3. If You assert a patent infringement claim against Participant alleging that the Participant
+Software directly or indirectly infringes any patent where such claim is resolved (such as by
+license or settlement) prior to the initiation of patent infringement litigation, then the
+reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be
+taken into account in determining the amount or value of any payment or license.
+
+6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have
+been validly granted by You or any distributor hereunder prior to termination (excluding licenses
+granted to You by any distributor) shall survive termination.
+
+7. LIMITATION OF LIABILITY.
+
+UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR
+OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED
+SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL,
+INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR
+LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL
+DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES.
+THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING
+FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME
+JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
+THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
+
+8. U.S. GOVERNMENT END USERS.
+
+The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995),
+consisting of “commercial computer software” (as that term is defined at 48 C.F.R. §
+252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48
+C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through
+227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those
+rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other
+FAR, DFAR, or other clause or provision that addresses Government rights in computer software under
+this License.
+
+9. MISCELLANEOUS.
+
+This License represents the complete agreement concerning subject matter hereof. If any provision of
+this License is held to be unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. This License shall be governed by the law of the jurisdiction
+specified in a notice contained within the Original Software (except to the extent applicable law,
+if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any
+litigation relating to this License shall be subject to the jurisdiction of the courts located in
+the jurisdiction and venue specified in a notice contained within the Original Software, with the
+losing party responsible for costs, including, without limitation, court costs and reasonable
+attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the
+International Sale of Goods is expressly excluded. Any law or regulation which provides that the
+language of a contract shall be construed against the drafter shall not apply to this License. You
+agree that You alone are responsible for compliance with the United States export administration
+regulations (and the export control laws and regulation of any other countries) when You use,
+distribute or otherwise make available any Covered Software.
+
+10. RESPONSIBILITY FOR CLAIMS.
+
+As between Initial Developer and the Contributors, each party is responsible for claims and damages
+arising, directly or indirectly, out of its utilization of rights under this License and You agree
+to work with Initial Developer and Contributors to distribute such responsibility on an equitable
+basis. Nothing herein is intended or shall be deemed to constitute any admission of liability.
+
+--
+
+NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
+
+The code released under the CDDL shall be governed by the laws of the State of California (excluding
+conflict-of-law provisions). Any litigation relating to this License shall be subject to the
+jurisdiction of the Federal Courts of the Northern District of California and the state courts of
+the State of California, with venue lying in Santa Clara County, California.
+
+
+#end
+#if(!(${cpl.isEmpty()}))
+====
+## print all the CPL 1.0 licensed works
+This product includes the following works licensed under the Common Public License Version 1.0:
+
+#foreach($dep in $cpl)
+#if( !(${dep.licenses[0].comments.isEmpty()}) )
+  * ${dep.name}, ${dep.licenses[0].comments}
+#else
+  * ${dep.name}
+#end
+#end
+
+Common Public License - v 1.0
+
+  THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON
+  PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF
+  THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
+
+  1. DEFINITIONS
+
+  "Contribution" means:
+
+  a) in the case of the initial Contributor, the initial code and
+    documentation distributed under this Agreement, and
+
+  b) in the case of each subsequent Contributor:
+
+  i) changes to the Program, and
+
+  ii) additions to the Program;
+
+  where such changes and/or additions to the Program originate
+  from and are distributed by that particular Contributor. A
+  Contribution 'originates' from a Contributor if it was added to the
+  Program by such Contributor itself or anyone acting on such
+  Contributor's behalf. Contributions do not include additions to the
+  Program which: (i) are separate modules of software distributed in
+  conjunction with the Program under their own license agreement,
+  and (ii) are not derivative works of the Program.
+
+  "Contributor" means any person or entity that distributes the Program.
+
+  "Licensed Patents " mean patent claims licensable by a Contributor
+  which are necessarily infringed by the use or sale of its Contribution
+  alone or when combined with the Program.
+
+  "Program" means the Contributions distributed in accordance with this Agreement.
+
+  "Recipient" means anyone who receives the Program under this
+  Agreement, including all Contributors.
+
+  2. GRANT OF RIGHTS
+
+  a) Subject to the terms of this Agreement, each Contributor
+  hereby grants Recipient a non-exclusive, worldwide, royalty-free
+  copyright license to reproduce, prepare derivative works of, publicly
+  display, publicly perform, distribute and sublicense the Contribution
+  of such Contributor, if any, and such derivative works, in source code
+  and object code form.
+
+  b) Subject to the terms of this Agreement, each Contributor
+  hereby grants Recipient a non-exclusive, worldwide, royalty-free
+  patent license under Licensed Patents to make, use, sell, offer to
+  sell, import and otherwise transfer the Contribution of such
+  Contributor, if any, in source code and object code form. This patent
+  license shall apply to the combination of the Contribution and the
+  Program if, at the time the Contribution is added by the Contributor,
+  such addition of the Contribution causes such combination to be
+  covered by the Licensed Patents. The patent license shall not apply to
+  any other combinations which include the Contribution. No hardware per
+  se is licensed hereunder.
+
+  c) Recipient understands that although each Contributor grants
+  the licenses to its Contributions set forth herein, no assurances are
+  provided by any Contributor that the Program does not infringe the
+  patent or other intellectual property rights of any other entity. Each
+  Contributor disclaims any liability to Recipient for claims brought by
+  any other entity based on infringement of intellectual property rights
+  or otherwise. As a condition to exercising the rights and licenses
+  granted hereunder, each Recipient hereby assumes sole responsibility
+  to secure any other intellectual property rights needed, if any. For
+  example, if a third party patent license is required to allow
+  Recipient to distribute the Program, it is Recipient's responsibility
+  to acquire that license before distributing the Program.
+
+  d) Each Contributor represents that to its knowledge it has
+  sufficient copyright rights in its Contribution, if any, to grant the
+  copyright license set forth in this Agreement.
+
+  3. REQUIREMENTS
+
+  A Contributor may choose to distribute the Program in object code form
+  under its own license agreement, provided that:
+
+  a) it complies with the terms and conditions of this Agreement;
+    and
+
+  b) its license agreement:
+
+  i) effectively disclaims on behalf of all Contributors all
+  warranties and conditions, express and implied, including warranties
+  or conditions of title and non-infringement, and implied warranties or
+  conditions of merchantability and fitness for a particular purpose;
+
+  ii) effectively excludes on behalf of all Contributors all
+  liability for damages, including direct, indirect, special, incidental
+  and consequential damages, such as lost profits;
+
+  iii) states that any provisions which differ from this Agreement
+  are offered by that Contributor alone and not by any other party; and
+
+  iv) states that source code for the Program is available from
+  such Contributor, and informs licensees how to obtain it in a
+  reasonable manner on or through a medium customarily used for software
+  exchange.
+
+  When the Program is made available in source code form:
+
+  a) it must be made available under this Agreement; and
+
+  b) a copy of this Agreement must be included with each copy of
+  the Program.
+
+  Contributors may not remove or alter any copyright notices contained
+  within the Program.
+
+  Each Contributor must identify itself as the originator of its
+  Contribution, if any, in a manner that reasonably allows subsequent
+  Recipients to identify the originator of the Contribution.
+
+  4. COMMERCIAL DISTRIBUTION
+
+  Commercial distributors of software may accept certain
+  responsibilities with respect to end users, business partners and the
+  like. While this license is intended to facilitate the commercial use
+  of the Program, the Contributor who includes the Program in a
+  commercial product offering should do so in a manner which does not
+  create potential liability for other Contributors. Therefore, if a
+  Contributor includes the Program in a commercial product offering,
+  such Contributor ("Commercial Contributor") hereby agrees to defend
+  and indemnify every other Contributor ("Indemnified Contributor")
+  against any losses, damages and costs (collectively "Losses") arising
+  from claims, lawsuits and other legal actions brought by a third party
+  against the Indemnified Contributor to the extent caused by the acts
+  or omissions of such Commercial Contributor in connection with its
+  distribution of the Program in a commercial product offering. The
+  obligations in this section do not apply to any claims or Losses
+  relating to any actual or alleged intellectual property
+  infringement. In order to qualify, an Indemnified Contributor must: a)
+  promptly notify the Commercial Contributor in writing of such claim,
+  and b) allow the Commercial Contributor to control, and cooperate with
+  the Commercial Contributor in, the defense and any related settlement
+  negotiations. The Indemnified Contributor may participate in any such
+  claim at its own expense.
+
+  For example, a Contributor might include the Program in a commercial
+  product offering, Product X. That Contributor is then a Commercial
+  Contributor. If that Commercial Contributor then makes performance
+  claims, or offers warranties related to Product X, those performance
+  claims and warranties are such Commercial Contributor's responsibility
+  alone. Under this section, the Commercial Contributor would have to
+  defend claims against the other Contributors related to those
+  performance claims and warranties, and if a court requires any other
+  Contributor to pay any damages as a result, the Commercial Contributor
+  must pay those damages.
+
+  5. NO WARRANTY
+
+  EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS
+  PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY
+  WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY
+  OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely
+  responsible for determining the appropriateness of using and
+  distributing the Program and assumes all risks associated with its
+  exercise of rights under this Agreement, including but not limited to
+  the risks and costs of program errors, compliance with applicable
+  laws, damage to or loss of data, programs or equipment, and
+  unavailability or interruption of operations.
+
+  6. DISCLAIMER OF LIABILITY
+
+  EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR
+  ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT,
+  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING
+  WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF
+  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR
+  DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
+  HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+  7. GENERAL
+
+  If any provision of this Agreement is invalid or unenforceable under
+  applicable law, it shall not affect the validity or enforceability of
+  the remainder of the terms of this Agreement, and without further
+  action by the parties hereto, such provision shall be reformed to the
+  minimum extent necessary to make such provision valid and enforceable.
+
+  If Recipient institutes patent litigation against a Contributor with
+  respect to a patent applicable to software (including a cross-claim or
+  counterclaim in a lawsuit), then any patent licenses granted by that
+  Contributor to such Recipient under this Agreement shall terminate as
+  of the date such litigation is filed. In addition, if Recipient
+  institutes patent litigation against any entity (including a
+  cross-claim or counterclaim in a lawsuit) alleging that the Program
+  itself (excluding combinations of the Program with other software or
+  hardware) infringes such Recipient's patent(s), then such Recipient's
+  rights granted under Section 2(b) shall terminate as of the date such
+  litigation is filed.
+
+  All Recipient's rights under this Agreement shall terminate if it
+  fails to comply with any of the material terms or conditions of this
+  Agreement and does not cure such failure in a reasonable period of
+  time after becoming aware of such noncompliance. If all Recipient's
+  rights under this Agreement terminate, Recipient agrees to cease use
+  and distribution of the Program as soon as reasonably
+  practicable. However, Recipient's obligations under this Agreement and
+  any licenses granted by Recipient relating to the Program shall
+  continue and survive.
+
+  Everyone is permitted to copy and distribute copies of this Agreement,
+  but in order to avoid inconsistency the Agreement is copyrighted and
+  may only be modified in the following manner. The Agreement Steward
+  reserves the right to publish new versions (including revisions) of
+  this Agreement from time to time. No one other than the Agreement
+  Steward has the right to modify this Agreement. IBM is the initial
+  Agreement Steward. IBM may assign the responsibility to serve as the
+  Agreement Steward to a suitable separate entity. Each new version of
+  the Agreement will be given a distinguishing version number. The
+  Program (including Contributions) may always be distributed subject to
+  the version of the Agreement under which it was received. In addition,
+  after a new version of the Agreement is published, Contributor may
+  elect to distribute the Program (including its Contributions) under
+  the new version. Except as expressly stated in Sections 2(a) and 2(b)
+  above, Recipient receives no rights or licenses to the intellectual
+  property of any Contributor under this Agreement, whether expressly,
+  by implication, estoppel or otherwise. All rights in the Program not
+  expressly granted under this Agreement are reserved.
+
+  This Agreement is governed by the laws of the State of New York and
+  the intellectual property laws of the United States of America. No
+  party to this Agreement will bring a legal action under this Agreement
+  more than one year after the cause of action arose. Each party waives
+ its rights to a jury trial in any resulting litigation.
+#end
+## We put the JRuby incorporated works bits here, since we first reference JRuby in the CPL section.
+#if($jruby)
+#jruby_license()
+#end
+#end
diff --git a/src/assembly/META-INF/NOTICE.vm b/src/assembly/META-INF/NOTICE.vm
new file mode 100644
index 000000000000..2c8c328bcd8d
--- /dev/null
+++ b/src/assembly/META-INF/NOTICE.vm
@@ -0,0 +1,324 @@
+#**
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*#
+## Specially handled included deps. e.g. ASL2.0 works that have additional bundled works but
+#set($bundled-dependencies = ${bundled-dependencies.equalsIgnoreCase("true")})
+#if($bundled-dependencies)
+====
+${project.name} contained works
+
+This product contains additional works that are distributed under licenses
+other than ASL v2. See LICENSE for full details
+
+#end
+## notice of bundled in source
+--
+This product incorporates portions of the 'Hadoop' project
+
+Copyright 2007-2009 The Apache Software Foundation
+
+Licensed under the Apache License v2.0
+## no NOTICE file.
+#macro(findbugs_cleanroom_notice)
+--
+## Include a note about the cleanroom reimplementation because otherwise it looks like the gpl work.
+This product includes the 'Findbugs Annotations under Apache License' project
+(https://github.com/stephenc/findbugs-annotations), version 1.3.9-1
+licensed under the Apache Software License, version 2.0.
+#end
+#macro(netty_all_4_notice)
+--
+## the parts of the NOTICE file from io.nett:netty-all that are relevant
+This product includes portions of 'the Netty Project'
+
+  * http://netty.io/
+
+Copyright 2011 The Netty Project
+
+licensed under the Apache Software License, version 2.0.
+
+This product contains a modified version of Roland Kuhn's ASL2
+AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue.
+It can be obtained at:
+
+  * LICENSE Apache Software License, version 2.0.
+  * HOMEPAGE:
+    * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/
+          src/main/java/akka/dispatch/AbstractNodeQueue.java
+
+The Netty Project includes additional subcomponents under additional licenses,
+noted in the LICENSE file.
+## Leave out all the 'optionally depends on' bits.
+
+#end
+## NOTICE.txt from jetty 6.1.26, which isn't in the org.mortbay.jetty:jetty-util jar
+## Obtained from fedora's mirror of the 6.1.26 source
+## http://pkgs.fedoraproject.org/repo/pkgs/jetty/jetty-6.1.26-src.zip/
+##     4980757517064efbf655e025d005a0b8/jetty-6.1.26-src.zip
+## https://github.com/toddlipcon/jetty-hadoop-fix/tree/6.1.26
+##
+## updated for server side to include jsp bits from 6.1.14 (notice files are identical save
+## copyright span)
+#macro(jetty_notice $jetty-include-unixcrypt)
+--
+This product includes portions of 'The Jetty Web Container'
+
+Copyright 1995-2009 Mort Bay Consulting Pty Ltd
+
+## javax.servlet paragraph is generated via org.mortbay.jetty:servlet-api-2.5
+#if(${jetty-include-unixcrypt})
+## UnixCrypt.java paragraph only in server
+The UnixCrypt.java code ~Implements the one way cryptography used by
+Unix systems for simple password protection.  Copyright 1996 Aki Yoshida,
+modified April 2001  by Iris Van den Broeke, Daniel Deville.
+Permission to use, copy, modify and distribute UnixCrypt
+for non-commercial or commercial purposes and without fee is
+granted provided that the copyright notice appears in all copies.
+
+#end
+## jsp implementation paragraph is handled via org.mortbay.jetty:jsp-2.1
+
+Some portions of the code are Copyright:
+  2006 Tim Vernum
+  1999 Jason Gilbert.
+## jboss integration skipped, not included
+## win32 service wrapper skipped, not included
+
+## This dot is here to stop a bug in the "collapse white space" feature of velocity
+## that otherwise swallows the license line.
+.
+licensed under the Apache Software License, version 2.0.
+
+#end
+#macro(log4j_notice)
+--
+## log4j has a NOTICE file but it isn't included in the jar
+This product includes 'Apache log4j'
+Copyright 2010 The Apache Software Foundation
+
+#end
+#macro(htrace_notice)
+--
+## htrace includes a relocated commons-logging that has a NOTICE file that's missing the copyright
+This product includes 'Apache Commons Logging'
+Copyright The Apache Software Foundation
+
+#end
+#macro(zookeeper_notice)
+--
+## ZooKeeper has a NOTICE file but it isn't included in the jar
+This product includes 'Apache ZooKeeper'
+Copyright 2009-2012 The Apache Software Foundation
+
+#end
+#macro(jets3t_notice)
+--
+## Jets3t has a NOTICE file but it isn't included in the jar
+This product includes 'Jets3t', which includes software developed by:
+
+   The Apache Software Foundation (http://www.apache.org/).
+
+   The ExoLab Project (http://www.exolab.org/)
+
+   Sun Microsystems (http://www.sun.com/)
+
+   Codehaus (http://castor.codehaus.org)
+
+   Tatu Saloranta (http://wiki.fasterxml.com/TatuSaloranta)
+
+#end
+#macro(jruby_notice)
+JRuby Incorporated Works
+
+Some additional libraries distributed with JRuby are not covered by
+JRuby's licence.
+
+## skip bytelist and yydebug because they're covered by the jruby copyright and license
+## skip rails benchmark javascript, because it's not in the complete jar
+## skip rake because MIT
+## jruby bundles asm, but we already ref that in our included deps
+
+ build_lib/apt-mirror-api.jar, build_lib/bnd-0.0.249.jar,
+ BSF and ant are distributed under the Apache Software License,
+ Version 1.1
+
+## skip constantine because MIT
+## jruby bundles jcodings and joni, but we already ref that in our included deps
+
+ build_lib/commons-logging-1.1.1.jar
+ build_lib/jarjar-1.0rc8.jar (http://code.google.com/p/jarjar),
+ build_lib/joda-time-1.6.2.jar (http://joda-time.sourceforge.net),
+ build_lib/jffi*jar (https://github.com/wmeissner/jffi),
+ build_lib/jaffl.jar (https://github.com/wmeissner/jaffl),
+ build_lib/jnr-netdb.jar (http://github.com/wmeissner/jnr-netdb),
+ build_lib/dynalang-0.3.jar (http://dynalang.sourceforge.net),
+ build_lib/nailgun-0.7.1.jar and tool/nailgun/ng.exe
+ (http://martiansoftware.com/nailgun) are distributed under the
+ Apache License version 2.0.
+## skip emma and junit; both aren't in the complete jar
+## skip jline because BSD-3
+## skip bouncy because MIT
+## skip yecht because MIT
+## skip jquery because MIT
+
+  most files found in lib/ruby/1.8, lib/ruby/1.9, and lib/ruby/site_ruby/1.8/rubygems*
+  are distributed under The Ruby License's non-GPL provision, see LICENSE for details and
+  http://jruby.org/ for source.
+
+#end
+## first bundled source
+#if(${bundled-logo})
+#orca_logo_notice()
+#end
+#if(${bundled-bootstrap})
+#bootstrap_notice()
+#end
+#if(${bundled-dependencies})
+#**
+ Note that this will fail the build if we don't have a license. update supplemental-models via
+ setting '-Dlicense.debug.print.included' and looking in the generated LICENSE file
+ * BSD and MIT licensed works are just in LICENSE.
+ * Public Domain dedicated works are just in LICENSE.
+ * ASL2.0 works are skipped. if they have a NOTICE file it'll get merged via the shade plugin.
+ * TODO If a non-ASL2.0 work has a NOTICE file, we'll get a duplicate.
+*#
+## group up by licenses we know we have several of
+#set($cddl_1_0=[])
+#set($cddl_1_1=[])
+#set($cpl_1_0=[])
+#set($mpl_1_1=[])
+## track jettys
+#set($jetty=false)
+#set($jetty-with-crypt=false)
+## track jruby
+#set($jruby=false)
+#foreach( ${dep} in ${projects} )
+#if(${dep.artifactId.equals("findbugs-annotations")})
+#if(${dep.groupId.equals("com.github.stephenc.findbugs")})
+#findbugs_cleanroom_notice()
+#end
+#end
+#if(${dep.artifactId.equals("netty-all")} && ${dep.version.startsWith("4")})
+#netty_all_4_notice()
+#end
+#if(${dep.artifactId.startsWith("jetty")})
+#set($jetty=true)
+#if(${dep.artifactId.equals("jetty")})
+#set($jetty-with-crypt=true)
+#end
+#end
+#if(${dep.artifactId.equals("log4j")})
+#log4j_notice()
+#end
+#if(${dep.artifactId.equals("htrace-core")})
+#htrace_notice()
+#end
+#if(${dep.artifactId.equals("zookeeper")})
+#zookeeper_notice()
+#end
+#if(${dep.artifactId.equals("jets3t")})
+#jets3t_notice()
+#end
+#if(${dep.artifactId.equals("jruby-complete")})
+#set($jruby=true)
+#end
+#if( !(${dep.licenses[0].name.contains("BSD")} || ${dep.licenses[0].name.contains("MIT")}) )
+#if( !(${dep.licenses[0].name.equals("Public Domain")}) )
+#if( !(${dep.licenses[0].name.contains("Apache Software License, Version 2.0")}) )
+#set($aggregated = false)
+#if(${dep.licenses[0].name.contains("CDDL")})
+#if(${dep.licenses[0].name.contains("1.0")})
+#set($aggregated = $cddl_1_0.add($dep))
+#elseif(${dep.licenses[0].name.contains("1.1")})
+#set($aggregated = $cddl_1_1.add($dep))
+#end
+#end
+#if(${dep.licenses[0].name.equalsIgnoreCase("Common Public License Version 1.0")})
+#set($aggregated = $cpl_1_0.add($dep))
+#end
+#if(${dep.licenses[0].name.equalsIgnoreCase("Mozilla Public License Version 1.1")})
+#set($aggregated = $mpl_1_1.add($dep))
+#end
+#if(!${aggregated})
+--
+## For now, we're assuming we will aggregate all Category B licenses
+This product includes ${dep.name}
+
+Licensed under ${dep.licenses[0].name}, see LICENSE for details.
+
+incorporated from ${dep.groupId}:${dep.artifactId}:${dep.packaging}:${dep.version}
+#end
+#end
+#end
+#end
+#end
+## Print out jetty
+#if(${jetty})
+#jetty_notice(${jetty-with-crypt})
+#end
+## Now go through all the lists of Category-B licensed works and make sure we
+## name them and give a URL for the project's home page.
+
+## list all the CPL
+#if(!(${cpl_1_0.isEmpty()}))
+----
+This product includes the following works licensed under the CPL v1.0, see LICENSE for details:
+
+#foreach(${dep} in ${cpl_1_0})
+  * ${dep.name} incorporated from ${dep.groupId}:${dep.artifactId}:${dep.packaging}:${dep.version}
+    For source see: ${dep.url}
+#end
+#end
+----
+## We presume the CPL list above will include the primary JRuby notice
+#if(${jruby})
+#jruby_notice()
+#end
+## list all the MPL
+#if(!(${mpl_1_1.isEmpty()}))
+----
+This product includes the following works licensed under the MPL v1.1, see LICENSE for details:
+
+#foreach(${dep} in ${mpl_1_1})
+  * ${dep.name} incorporated from ${dep.groupId}:${dep.artifactId}:${dep.packaging}:${dep.version}
+    For source see: ${dep.url}
+#end
+#end
+## list all the CDDL
+#if(!(${cddl_1_0.isEmpty()}))
+----
+This product includes the following works licensed under the CDDL v1.0, see LICENSE for details:
+
+#foreach(${dep} in ${cddl_1_0})
+  * ${dep.name} incorporated from ${dep.groupId}:${dep.artifactId}:${dep.packaging}:${dep.version}
+    For source see: ${dep.url}
+#end
+#end
+#if(!(${cddl_1_1.isEmpty()}))
+----
+This product includes the following works licensed under the CDDL v1.1, see LICENSE for details:
+
+#foreach(${dep} in ${cddl_1_1})
+  * ${dep.name} incorporated from ${dep.groupId}:${dep.artifactId}:${dep.packaging}:${dep.version}
+    For source see: ${dep.url}
+#end
+#end
+----
+Incorporated NOTICE files from bundled works below.
+----
+#end
diff --git a/src/assembly/resources/supplemental-models.xml b/src/assembly/resources/supplemental-models.xml
new file mode 100644
index 000000000000..6b727f9c836c
--- /dev/null
+++ b/src/assembly/resources/supplemental-models.xml
@@ -0,0 +1,1664 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<supplementalDataModels xmlns="http://maven.apache.org/supplemental-model/1.0.0"
+                        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                        xsi:schemaLocation="http://maven.apache.org/supplemental-model/1.0.0 http://maven.apache.org/xsd/supplemental-model-1.0.0.xsd">
+<!-- this file is a superset of the contents of the hbase-shaded-client and
+     hbase-shaded-server versions, with the execption of moving the text of
+     the CPL to LICENSE.vm -->
+<!-- All of the below missing / ambiguous / full text license info verified at dependency source -->
+<!-- Artifacts missing license info in POM -->
+  <supplement>
+    <project>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+
+      <organization>
+        <name>The Apache Software Foundation</name>
+        <url>http://www.apache.org/</url>
+      </organization>
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>commons-beanutils</groupId>
+      <artifactId>commons-beanutils</artifactId>
+
+      <organization>
+        <name>The Apache Software Foundation</name>
+        <url>http://www.apache.org/</url>
+      </organization>
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-core</artifactId>
+
+      <organization>
+        <name>The Apache Software Foundation</name>
+        <url>http://www.apache.org/</url>
+      </organization>
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+
+      <organization>
+        <name>The Apache Software Foundation</name>
+        <url>http://www.apache.org/</url>
+      </organization>
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Artifacts with ambiguously named licenses in POM -->
+  <supplement>
+    <project>
+      <groupId>com.github.stephenc.findbugs</groupId>
+      <artifactId>findbugs-annotations</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.github.stephenc.high-scale-lib</groupId>
+      <artifactId>high-scale-lib</artifactId>
+      <name>Highly Scalable Java</name>
+
+      <licenses>
+        <license>
+          <name>Public Domain</name>
+          <distribution>repo</distribution>
+          <url>http://creativecommons.org/licenses/publicdomain/</url>
+          <comments>
+The person or persons who have associated work with this document (the
+"Dedicator" or "Certifier") hereby either (a) certifies that, to the best
+of his knowledge, the work of authorship identified is in the public
+domain of the country from which the work is published, or (b) hereby
+dedicates whatever copyright the dedicators holds in the work of
+authorship identified below (the "Work") to the public domain. A
+certifier, moreover, dedicates any copyright interest he may have in the
+associated work, and for these purposes, is described as a "dedicator"
+below.
+
+A certifier has taken reasonable steps to verify the copyright status of
+this work. Certifier recognizes that his good faith efforts may not
+shield him from liability if in fact the work certified is not in the
+public domain.
+
+Dedicator makes this dedication for the benefit of the public at large and
+to the detriment of the Dedicator's heirs and successors. Dedicator
+intends this dedication to be an overt act of relinquishment in perpetuity
+of all present and future rights under copyright law, whether vested or
+contingent, in the Work. Dedicator understands that such relinquishment of
+all rights includes the relinquishment of all rights to enforce (by
+lawsuit or otherwise) those copyrights in the Work.
+
+Dedicator recognizes that, once placed in the public domain, the Work may
+be freely reproduced, distributed, transmitted, used, modified, built
+upon, or otherwise exploited by anyone for any purpose, commercial or
+non-commercial, and in any way, including by methods that have not yet
+been invented or conceived.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.jboss.netty</groupId>
+      <artifactId>netty</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>io.netty</groupId>
+      <artifactId>netty</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.cloudera.htrace</groupId>
+      <artifactId>htrace-core</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Ambiguous license names in server and not in client -->
+  <supplement>
+    <project>
+      <groupId>net.java.dev.jets3t</groupId>
+      <artifactId>jets3t</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty-sslengine</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jsp-api-2.1</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.yammer.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.codehaus.jettison</groupId>
+      <artifactId>jettison</artifactId>
+
+      <licenses>
+        <license>
+          <name>The Apache Software License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright 2006 Envoi Solutions LLC
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Permissive licenses that need text in LICENSE -->
+  <supplement>
+    <project>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <name>Protocol Buffer Java API</name>
+
+      <licenses>
+        <license>
+<!-- Note that the link is to the BSD 2-Clause, but the text provided
+     is the 3-Clause :(
+-->
+          <name>New BSD license</name>
+          <url>http://www.opensource.org/licenses/bsd-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright 2008, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.jcraft</groupId>
+      <artifactId>jsch</artifactId>
+      <name>JSch</name>
+
+      <licenses>
+<!-- Text is slightly different than the standard 3-Clause, and
+     they expressly linked to this version in their pom, so use
+     as-is instead of grouping. :(
+-->
+        <license>
+          <name>BSD license</name>
+          <url>http://www.jcraft.com/jsch/LICENSE.txt</url>
+          <comments>
+Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in
+     the documentation and/or other materials provided with the distribution.
+
+  3. The names of the authors may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.thoughtworks.paranamer</groupId>
+      <artifactId>paranamer</artifactId>
+      <name>ParaNamer Core</name>
+
+      <licenses>
+        <license>
+          <name>BSD 3-Clause License</name>
+          <url>https://github.com/codehaus/paranamer-git/blob/paranamer-2.3/LICENSE.txt</url>
+          <distribution>repo</distribution>
+          <comments>
+ Copyright (c) 2006 Paul Hammant &amp; ThoughtWorks Inc
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.jruby.jcodings</groupId>
+      <artifactId>jcodings</artifactId>
+      <name>JCodings</name>
+
+      <licenses>
+        <license>
+          <name>MIT License</name>
+          <url>http://www.opensource.org/licenses/mit-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2008-2012 The JCodings Authors
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.jruby.joni</groupId>
+      <artifactId>joni</artifactId>
+      <name>Joni</name>
+
+      <licenses>
+        <license>
+          <name>MIT License</name>
+          <url>http://www.opensource.org/licenses/mit-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2008-2014 The Joni Authors
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <name>SLF4J API Module</name>
+
+      <licenses>
+        <license>
+          <name>MIT License</name>
+          <url>http://www.opensource.org/licenses/mit-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2004-2013 QOS.ch
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <name>SLF4J LOG4J-12 Binding</name>
+
+      <licenses>
+        <license>
+          <name>MIT License</name>
+          <url>http://www.opensource.org/licenses/mit-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2004-2008 QOS.ch
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>xmlenc</groupId>
+      <artifactId>xmlenc</artifactId>
+      <name>xmlenc Library</name>
+
+      <licenses>
+        <license>
+          <name>BSD 3-Clause License</name>
+          <url>http://www.opensource.org/licenses/bsd-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright 2003-2005, Ernst de Haan &lt;wfe.dehaan@gmail.com&gt;
+          </comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.tukaani</groupId>
+      <artifactId>xz</artifactId>
+
+      <licenses>
+        <license>
+          <name>Public Domain</name>
+          <distribution>repo</distribution>
+          <comments>
+Licensing of XZ for Java
+========================
+
+    All the files in this package have been written by Lasse Collin
+    and/or Igor Pavlov. All these files have been put into the
+    public domain. You can do whatever you want with these files.
+
+    This software is provided "as is", without any warranty.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- permissive licenses in server and not in client -->
+  <supplement>
+    <project>
+      <groupId>aopalliance</groupId>
+      <artifactId>aopalliance</artifactId>
+      <name>AOP alliance</name>
+
+      <licenses>
+        <license>
+          <name>Public Domain</name>
+          <distribution>repo</distribution>
+          <comments>
+LICENCE: all the source code provided by AOP Alliance is Public Domain.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>asm</groupId>
+      <artifactId>asm</artifactId>
+      <name>ASM: a very small and fast Java bytecode manipulation framework</name>
+
+      <licenses>
+        <license>
+          <name>BSD 3-Clause License</name>
+          <url>http://cvs.forge.objectweb.org/cgi-bin/viewcvs.cgi/*checkout*/asm/asm/LICENSE.txt?rev=1.3&amp;only_with_tag=ASM_3_1_MVN</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2000-2005 INRIA, France Telecom
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.fusesource.leveldbjni</groupId>
+      <artifactId>leveldbjni-all</artifactId>
+
+      <licenses>
+        <license>
+<!-- Original license text substituted "FuseSource Corp." for "the copyright holder"
+     in the original 3-Clause license. Since their pom linked directly to the standard
+     3-Clause text, we're grouping them with the others.
+-->
+          <name>BSD 3-Clause License</name>
+          <url>http://www.opensource.org/licenses/BSD-3-Clause</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2011 FuseSource Corp. All rights reserved.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Permissive licenses in the 0.94 assembly -->
+  <supplement>
+    <project>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-core</artifactId>
+
+      <licenses>
+        <license>
+          <name>New BSD license</name>
+          <url>http://www.opensource.org/licenses/bsd-license.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2000-2006, www.hamcrest.org
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of
+conditions and the following disclaimer. Redistributions in binary form must reproduce
+the above copyright notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the distribution.
+
+Neither the name of Hamcrest nor the names of its contributors may be used to endorse
+or promote products derived from this software without specific prior written
+permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+
+<!-- Category B licenses that need text in LICENSE and a NOTICE -->
+  <supplement>
+    <project>
+      <groupId>javax.activation</groupId>
+      <artifactId>activation</artifactId>
+      <name>JavaBeans Activation Framework (JAF)</name>
+      <url>http://java.sun.com/products/javabeans/jaf/index.jsp</url>
+
+      <licenses>
+        <license>
+          <name>Common Development and Distribution License (CDDL) v1.0</name>
+          <url>https://glassfish.dev.java.net/public/CDDLv1.0.html</url>
+          <distribution>repo</distribution>
+          <comments>
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+      <name>JAXB API bundle for GlassFish V3</name>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <name>JUnit</name>
+      <url>http://junit.org/</url>
+
+      <licenses>
+        <license>
+          <name>Common Public License Version 1.0</name>
+          <url>http://www.opensource.org/licenses/cpl1.0.txt</url>
+          <distribution>repo</distribution>
+          <comments>
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Category B in server and not in client -->
+  <supplement>
+    <project>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-client</artifactId>
+<!-- The Jersey 1.9 sources are only available at the old svn server. -->
+      <url>https://java.net/projects/jersey/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010-2011 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-core</artifactId>
+      <url>https://java.net/projects/jersey/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010-2011 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-json</artifactId>
+      <url>https://java.net/projects/jersey/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010-2011 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-server</artifactId>
+      <url>https://java.net/projects/jersey/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010-2011 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.sun.jersey.contribs</groupId>
+      <artifactId>jersey-guice</artifactId>
+      <url>https://java.net/projects/jersey/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010-2011 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>com.sun.xml.bind</groupId>
+      <artifactId>jaxb-impl</artifactId>
+      <name>JAXB Reference Implementation for GlassFish</name>
+      <url>https://jaxb.java.net/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+      <name>JAXB API bundle for GlassFish V3</name>
+      <url>https://jaxb.java.net/</url>
+
+      <licenses>
+        <license>
+          <name>CDDL 1.1</name>
+          <url>https://glassfish.java.net/public/CDDL+GPL_1_1.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2010 Oracle and/or its affiliates.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+<!-- Found by seraching through the servlet-api-source-2.5.jar  -->
+    <project>
+      <groupId>javax.servlet</groupId>
+      <artifactId>servlet-api</artifactId>
+      <name>Java Servlet API v2.5</name>
+      <url>http://search.maven.org/#artifactdetails%7Cjavax.servlet%7Cservlet-api%7C2.5%7Cjar</url>
+
+      <licenses>
+        <license>
+          <name>Common Development and Distribution License (CDDL) v1.0</name>
+          <url>https://glassfish.dev.java.net/public/CDDLv1.0.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright 1999-2005 Sun Microsystems, Inc.
+Portions copyright 2002 International Business Machines Corporation
+Portions copyright Apache Software Foundation
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>servlet-api-2.5</artifactId>
+      <name>Servlet Specification 2.5 API</name>
+      <url>http://www.eclipse.org/jetty/</url>
+
+      <licenses>
+        <license>
+          <name>Common Development and Distribution License (CDDL) v1.0</name>
+          <url>https://glassfish.dev.java.net/public/CDDLv1.0.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright 1999-2005 Sun Microsystems, Inc.
+Portions copyright 2002 International Business Machines Corporation
+Portions copyright Apache Software Foundation
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jsp-2.1</artifactId>
+      <name>JSP2.1 Jasper implementation from Glassfish</name>
+      <url>http://www.eclipse.org/jetty/</url>
+
+      <licenses>
+        <license>
+          <name>Common Development and Distribution License (CDDL) v1.0</name>
+          <url>https://glassfish.dev.java.net/public/CDDLv1.0.html</url>
+          <distribution>repo</distribution>
+<!-- Taken from Jetty notice -->
+          <comments>
+Copyright 2005 Sun Microsystems, Inc. and portions Copyright Apache Software Foundation.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+  <supplement>
+    <project>
+      <groupId>org.jamon</groupId>
+      <artifactId>jamon-runtime</artifactId>
+      <name>Jamon runtime support classes</name>
+      <url>http://www.jamon.org/</url>
+
+      <licenses>
+        <license>
+          <url>http://www.mozilla.org/MPL/MPL-1.1.txt</url>
+          <name>Mozilla Public License Version 1.1</name>
+          <distribution>repo</distribution>
+          <comments>
+                          MOZILLA PUBLIC LICENSE
+                                Version 1.1
+
+                              ---------------
+
+1. Definitions.
+
+     1.0.1. "Commercial Use" means distribution or otherwise making the
+     Covered Code available to a third party.
+
+     1.1. "Contributor" means each entity that creates or contributes to
+     the creation of Modifications.
+
+     1.2. "Contributor Version" means the combination of the Original
+     Code, prior Modifications used by a Contributor, and the Modifications
+     made by that particular Contributor.
+
+     1.3. "Covered Code" means the Original Code or Modifications or the
+     combination of the Original Code and Modifications, in each case
+     including portions thereof.
+
+     1.4. "Electronic Distribution Mechanism" means a mechanism generally
+     accepted in the software development community for the electronic
+     transfer of data.
+
+     1.5. "Executable" means Covered Code in any form other than Source
+     Code.
+
+     1.6. "Initial Developer" means the individual or entity identified
+     as the Initial Developer in the Source Code notice required by Exhibit
+     A.
+
+     1.7. "Larger Work" means a work which combines Covered Code or
+     portions thereof with code not governed by the terms of this License.
+
+     1.8. "License" means this document.
+
+     1.8.1. "Licensable" means having the right to grant, to the maximum
+     extent possible, whether at the time of the initial grant or
+     subsequently acquired, any and all of the rights conveyed herein.
+
+     1.9. "Modifications" means any addition to or deletion from the
+     substance or structure of either the Original Code or any previous
+     Modifications. When Covered Code is released as a series of files, a
+     Modification is:
+          A. Any addition to or deletion from the contents of a file
+          containing Original Code or previous Modifications.
+
+          B. Any new file that contains any part of the Original Code or
+          previous Modifications.
+
+     1.10. "Original Code" means Source Code of computer software code
+     which is described in the Source Code notice required by Exhibit A as
+     Original Code, and which, at the time of its release under this
+     License is not already Covered Code governed by this License.
+
+     1.10.1. "Patent Claims" means any patent claim(s), now owned or
+     hereafter acquired, including without limitation,  method, process,
+     and apparatus claims, in any patent Licensable by grantor.
+
+     1.11. "Source Code" means the preferred form of the Covered Code for
+     making modifications to it, including all modules it contains, plus
+     any associated interface definition files, scripts used to control
+     compilation and installation of an Executable, or source code
+     differential comparisons against either the Original Code or another
+     well known, available Covered Code of the Contributor's choice. The
+     Source Code can be in a compressed or archival form, provided the
+     appropriate decompression or de-archiving software is widely available
+     for no charge.
+
+     1.12. "You" (or "Your")  means an individual or a legal entity
+     exercising rights under, and complying with all of the terms of, this
+     License or a future version of this License issued under Section 6.1.
+     For legal entities, "You" includes any entity which controls, is
+     controlled by, or is under common control with You. For purposes of
+     this definition, "control" means (a) the power, direct or indirect,
+     to cause the direction or management of such entity, whether by
+     contract or otherwise, or (b) ownership of more than fifty percent
+     (50%) of the outstanding shares or beneficial ownership of such
+     entity.
+
+2. Source Code License.
+
+     2.1. The Initial Developer Grant.
+     The Initial Developer hereby grants You a world-wide, royalty-free,
+     non-exclusive license, subject to third party intellectual property
+     claims:
+          (a)  under intellectual property rights (other than patent or
+          trademark) Licensable by Initial Developer to use, reproduce,
+          modify, display, perform, sublicense and distribute the Original
+          Code (or portions thereof) with or without Modifications, and/or
+          as part of a Larger Work; and
+
+          (b) under Patents Claims infringed by the making, using or
+          selling of Original Code, to make, have made, use, practice,
+          sell, and offer for sale, and/or otherwise dispose of the
+          Original Code (or portions thereof).
+
+          (c) the licenses granted in this Section 2.1(a) and (b) are
+          effective on the date Initial Developer first distributes
+          Original Code under the terms of this License.
+
+          (d) Notwithstanding Section 2.1(b) above, no patent license is
+          granted: 1) for code that You delete from the Original Code; 2)
+          separate from the Original Code;  or 3) for infringements caused
+          by: i) the modification of the Original Code or ii) the
+          combination of the Original Code with other software or devices.
+
+     2.2. Contributor Grant.
+     Subject to third party intellectual property claims, each Contributor
+     hereby grants You a world-wide, royalty-free, non-exclusive license
+
+          (a)  under intellectual property rights (other than patent or
+          trademark) Licensable by Contributor, to use, reproduce, modify,
+          display, perform, sublicense and distribute the Modifications
+          created by such Contributor (or portions thereof) either on an
+          unmodified basis, with other Modifications, as Covered Code
+          and/or as part of a Larger Work; and
+
+          (b) under Patent Claims infringed by the making, using, or
+          selling of  Modifications made by that Contributor either alone
+          and/or in combination with its Contributor Version (or portions
+          of such combination), to make, use, sell, offer for sale, have
+          made, and/or otherwise dispose of: 1) Modifications made by that
+          Contributor (or portions thereof); and 2) the combination of
+          Modifications made by that Contributor with its Contributor
+          Version (or portions of such combination).
+
+          (c) the licenses granted in Sections 2.2(a) and 2.2(b) are
+          effective on the date Contributor first makes Commercial Use of
+          the Covered Code.
+
+          (d)    Notwithstanding Section 2.2(b) above, no patent license is
+          granted: 1) for any code that Contributor has deleted from the
+          Contributor Version; 2)  separate from the Contributor Version;
+          3)  for infringements caused by: i) third party modifications of
+          Contributor Version or ii)  the combination of Modifications made
+          by that Contributor with other software  (except as part of the
+          Contributor Version) or other devices; or 4) under Patent Claims
+          infringed by Covered Code in the absence of Modifications made by
+          that Contributor.
+
+3. Distribution Obligations.
+
+     3.1. Application of License.
+     The Modifications which You create or to which You contribute are
+     governed by the terms of this License, including without limitation
+     Section 2.2. The Source Code version of Covered Code may be
+     distributed only under the terms of this License or a future version
+     of this License released under Section 6.1, and You must include a
+     copy of this License with every copy of the Source Code You
+     distribute. You may not offer or impose any terms on any Source Code
+     version that alters or restricts the applicable version of this
+     License or the recipients' rights hereunder. However, You may include
+     an additional document offering the additional rights described in
+     Section 3.5.
+
+     3.2. Availability of Source Code.
+     Any Modification which You create or to which You contribute must be
+     made available in Source Code form under the terms of this License
+     either on the same media as an Executable version or via an accepted
+     Electronic Distribution Mechanism to anyone to whom you made an
+     Executable version available; and if made available via Electronic
+     Distribution Mechanism, must remain available for at least twelve (12)
+     months after the date it initially became available, or at least six
+     (6) months after a subsequent version of that particular Modification
+     has been made available to such recipients. You are responsible for
+     ensuring that the Source Code version remains available even if the
+     Electronic Distribution Mechanism is maintained by a third party.
+
+     3.3. Description of Modifications.
+     You must cause all Covered Code to which You contribute to contain a
+     file documenting the changes You made to create that Covered Code and
+     the date of any change. You must include a prominent statement that
+     the Modification is derived, directly or indirectly, from Original
+     Code provided by the Initial Developer and including the name of the
+     Initial Developer in (a) the Source Code, and (b) in any notice in an
+     Executable version or related documentation in which You describe the
+     origin or ownership of the Covered Code.
+
+     3.4. Intellectual Property Matters
+          (a) Third Party Claims.
+          If Contributor has knowledge that a license under a third party's
+          intellectual property rights is required to exercise the rights
+          granted by such Contributor under Sections 2.1 or 2.2,
+          Contributor must include a text file with the Source Code
+          distribution titled "LEGAL" which describes the claim and the
+          party making the claim in sufficient detail that a recipient will
+          know whom to contact. If Contributor obtains such knowledge after
+          the Modification is made available as described in Section 3.2,
+          Contributor shall promptly modify the LEGAL file in all copies
+          Contributor makes available thereafter and shall take other steps
+          (such as notifying appropriate mailing lists or newsgroups)
+          reasonably calculated to inform those who received the Covered
+          Code that new knowledge has been obtained.
+
+          (b) Contributor APIs.
+          If Contributor's Modifications include an application programming
+          interface and Contributor has knowledge of patent licenses which
+          are reasonably necessary to implement that API, Contributor must
+          also include this information in the LEGAL file.
+
+               (c)    Representations.
+          Contributor represents that, except as disclosed pursuant to
+          Section 3.4(a) above, Contributor believes that Contributor's
+          Modifications are Contributor's original creation(s) and/or
+          Contributor has sufficient rights to grant the rights conveyed by
+          this License.
+
+     3.5. Required Notices.
+     You must duplicate the notice in Exhibit A in each file of the Source
+     Code.  If it is not possible to put such notice in a particular Source
+     Code file due to its structure, then You must include such notice in a
+     location (such as a relevant directory) where a user would be likely
+     to look for such a notice.  If You created one or more Modification(s)
+     You may add your name as a Contributor to the notice described in
+     Exhibit A.  You must also duplicate this License in any documentation
+     for the Source Code where You describe recipients' rights or ownership
+     rights relating to Covered Code.  You may choose to offer, and to
+     charge a fee for, warranty, support, indemnity or liability
+     obligations to one or more recipients of Covered Code. However, You
+     may do so only on Your own behalf, and not on behalf of the Initial
+     Developer or any Contributor. You must make it absolutely clear than
+     any such warranty, support, indemnity or liability obligation is
+     offered by You alone, and You hereby agree to indemnify the Initial
+     Developer and every Contributor for any liability incurred by the
+     Initial Developer or such Contributor as a result of warranty,
+     support, indemnity or liability terms You offer.
+
+     3.6. Distribution of Executable Versions.
+     You may distribute Covered Code in Executable form only if the
+     requirements of Section 3.1-3.5 have been met for that Covered Code,
+     and if You include a notice stating that the Source Code version of
+     the Covered Code is available under the terms of this License,
+     including a description of how and where You have fulfilled the
+     obligations of Section 3.2. The notice must be conspicuously included
+     in any notice in an Executable version, related documentation or
+     collateral in which You describe recipients' rights relating to the
+     Covered Code. You may distribute the Executable version of Covered
+     Code or ownership rights under a license of Your choice, which may
+     contain terms different from this License, provided that You are in
+     compliance with the terms of this License and that the license for the
+     Executable version does not attempt to limit or alter the recipient's
+     rights in the Source Code version from the rights set forth in this
+     License. If You distribute the Executable version under a different
+     license You must make it absolutely clear that any terms which differ
+     from this License are offered by You alone, not by the Initial
+     Developer or any Contributor. You hereby agree to indemnify the
+     Initial Developer and every Contributor for any liability incurred by
+     the Initial Developer or such Contributor as a result of any such
+     terms You offer.
+
+     3.7. Larger Works.
+     You may create a Larger Work by combining Covered Code with other code
+     not governed by the terms of this License and distribute the Larger
+     Work as a single product. In such a case, You must make sure the
+     requirements of this License are fulfilled for the Covered Code.
+
+4. Inability to Comply Due to Statute or Regulation.
+
+     If it is impossible for You to comply with any of the terms of this
+     License with respect to some or all of the Covered Code due to
+     statute, judicial order, or regulation then You must: (a) comply with
+     the terms of this License to the maximum extent possible; and (b)
+     describe the limitations and the code they affect. Such description
+     must be included in the LEGAL file described in Section 3.4 and must
+     be included with all distributions of the Source Code. Except to the
+     extent prohibited by statute or regulation, such description must be
+     sufficiently detailed for a recipient of ordinary skill to be able to
+     understand it.
+
+5. Application of this License.
+
+     This License applies to code to which the Initial Developer has
+     attached the notice in Exhibit A and to related Covered Code.
+
+6. Versions of the License.
+
+     6.1. New Versions.
+     Netscape Communications Corporation ("Netscape") may publish revised
+     and/or new versions of the License from time to time. Each version
+     will be given a distinguishing version number.
+
+     6.2. Effect of New Versions.
+     Once Covered Code has been published under a particular version of the
+     License, You may always continue to use it under the terms of that
+     version. You may also choose to use such Covered Code under the terms
+     of any subsequent version of the License published by Netscape. No one
+     other than Netscape has the right to modify the terms applicable to
+     Covered Code created under this License.
+
+     6.3. Derivative Works.
+     If You create or use a modified version of this License (which you may
+     only do in order to apply it to code which is not already Covered Code
+     governed by this License), You must (a) rename Your license so that
+     the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape",
+     "MPL", "NPL" or any confusingly similar phrase do not appear in your
+     license (except to note that your license differs from this License)
+     and (b) otherwise make it clear that Your version of the license
+     contains terms which differ from the Mozilla Public License and
+     Netscape Public License. (Filling in the name of the Initial
+     Developer, Original Code or Contributor in the notice described in
+     Exhibit A shall not of themselves be deemed to be modifications of
+     this License.)
+
+7. DISCLAIMER OF WARRANTY.
+
+     COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
+     WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+     WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
+     DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
+     THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE
+     IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT,
+     YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE
+     COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER
+     OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
+     ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
+
+8. TERMINATION.
+
+     8.1.  This License and the rights granted hereunder will terminate
+     automatically if You fail to comply with terms herein and fail to cure
+     such breach within 30 days of becoming aware of the breach. All
+     sublicenses to the Covered Code which are properly granted shall
+     survive any termination of this License. Provisions which, by their
+     nature, must remain in effect beyond the termination of this License
+     shall survive.
+
+     8.2.  If You initiate litigation by asserting a patent infringement
+     claim (excluding declatory judgment actions) against Initial Developer
+     or a Contributor (the Initial Developer or Contributor against whom
+     You file such action is referred to as "Participant")  alleging that:
+
+     (a)  such Participant's Contributor Version directly or indirectly
+     infringes any patent, then any and all rights granted by such
+     Participant to You under Sections 2.1 and/or 2.2 of this License
+     shall, upon 60 days notice from Participant terminate prospectively,
+     unless if within 60 days after receipt of notice You either: (i)
+     agree in writing to pay Participant a mutually agreeable reasonable
+     royalty for Your past and future use of Modifications made by such
+     Participant, or (ii) withdraw Your litigation claim with respect to
+     the Contributor Version against such Participant.  If within 60 days
+     of notice, a reasonable royalty and payment arrangement are not
+     mutually agreed upon in writing by the parties or the litigation claim
+     is not withdrawn, the rights granted by Participant to You under
+     Sections 2.1 and/or 2.2 automatically terminate at the expiration of
+     the 60 day notice period specified above.
+
+     (b)  any software, hardware, or device, other than such Participant's
+     Contributor Version, directly or indirectly infringes any patent, then
+     any rights granted to You by such Participant under Sections 2.1(b)
+     and 2.2(b) are revoked effective as of the date You first made, used,
+     sold, distributed, or had made, Modifications made by that
+     Participant.
+
+     8.3.  If You assert a patent infringement claim against Participant
+     alleging that such Participant's Contributor Version directly or
+     indirectly infringes any patent where such claim is resolved (such as
+     by license or settlement) prior to the initiation of patent
+     infringement litigation, then the reasonable value of the licenses
+     granted by such Participant under Sections 2.1 or 2.2 shall be taken
+     into account in determining the amount or value of any payment or
+     license.
+
+     8.4.  In the event of termination under Sections 8.1 or 8.2 above,
+     all end user license agreements (excluding distributors and resellers)
+     which have been validly granted by You or any distributor hereunder
+     prior to termination shall survive termination.
+
+9. LIMITATION OF LIABILITY.
+
+     UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
+     (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
+     DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE,
+     OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR
+     ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
+     CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL,
+     WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
+     COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
+     INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
+     LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY
+     RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
+     PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE
+     EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
+     THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
+
+10. U.S. GOVERNMENT END USERS.
+
+     The Covered Code is a "commercial item," as that term is defined in
+     48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
+     software" and "commercial computer software documentation," as such
+     terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48
+     C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995),
+     all U.S. Government End Users acquire Covered Code with only those
+     rights set forth herein.
+
+11. MISCELLANEOUS.
+
+     This License represents the complete agreement concerning subject
+     matter hereof. If any provision of this License is held to be
+     unenforceable, such provision shall be reformed only to the extent
+     necessary to make it enforceable. This License shall be governed by
+     California law provisions (except to the extent applicable law, if
+     any, provides otherwise), excluding its conflict-of-law provisions.
+     With respect to disputes in which at least one party is a citizen of,
+     or an entity chartered or registered to do business in the United
+     States of America, any litigation relating to this License shall be
+     subject to the jurisdiction of the Federal Courts of the Northern
+     District of California, with venue lying in Santa Clara County,
+     California, with the losing party responsible for costs, including
+     without limitation, court costs and reasonable attorneys' fees and
+     expenses. The application of the United Nations Convention on
+     Contracts for the International Sale of Goods is expressly excluded.
+     Any law or regulation which provides that the language of a contract
+     shall be construed against the drafter shall not apply to this
+     License.
+
+12. RESPONSIBILITY FOR CLAIMS.
+
+     As between Initial Developer and the Contributors, each party is
+     responsible for claims and damages arising, directly or indirectly,
+     out of its utilization of rights under this License and You agree to
+     work with Initial Developer and Contributors to distribute such
+     responsibility on an equitable basis. Nothing herein is intended or
+     shall be deemed to constitute any admission of liability.
+
+13. MULTIPLE-LICENSED CODE.
+
+     Initial Developer may designate portions of the Covered Code as
+     "Multiple-Licensed".  "Multiple-Licensed" means that the Initial
+     Developer permits you to utilize portions of the Covered Code under
+     Your choice of the MPL or the alternative licenses, if any, specified
+     by the Initial Developer in the file described in Exhibit A.
+
+EXHIBIT A -Mozilla Public License.
+
+     ``The contents of this file are subject to the Mozilla Public License
+     Version 1.1 (the "License"); you may not use this file except in
+     compliance with the License. You may obtain a copy of the License at
+     http://www.mozilla.org/MPL/
+
+     Software distributed under the License is distributed on an "AS IS"
+     basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+     License for the specific language governing rights and limitations
+     under the License.
+
+     The Original Code is ______________________________________.
+
+     The Initial Developer of the Original Code is ________________________.
+     Portions created by ______________________ are Copyright (C) ______
+     _______________________. All Rights Reserved.
+
+     Contributor(s): ______________________________________.
+
+     Alternatively, the contents of this file may be used under the terms
+     of the _____ license (the  "[___] License"), in which case the
+     provisions of [______] License are applicable instead of those
+     above.  If you wish to allow use of your version of this file only
+     under the terms of the [____] License and not to allow others to use
+     your version of this file under the MPL, indicate your decision by
+     deleting  the provisions above and replace  them with the notice and
+     other provisions required by the [___] License.  If you do not delete
+     the provisions above, a recipient may use your version of this file
+     under either the MPL or the [___] License."
+
+     [NOTE: The text of this Exhibit A may differ slightly from the text of
+     the notices in the Source Code files of the Original Code. You should
+     use the text of this Exhibit A rather than the text found in the
+     Original Code Source Code for Your Modifications.]
+
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Category B in assembly but not server/client -->
+  <supplement>
+    <project>
+      <groupId>org.jruby</groupId>
+      <artifactId>jruby-complete</artifactId>
+      <name>JRuby Complete</name>
+      <url>http://www.jruby.org/</url>
+
+      <licenses>
+        <license>
+          <name>Common Public License Version 1.0</name>
+          <url>http://www-128.ibm.com/developerworks/library/os-cpl.html</url>
+          <distribution>repo</distribution>
+          <comments>
+Copyright (c) 2007-2011 The JRuby project
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+<!-- Category B in 0.94 assembly -->
+  <supplement>
+    <project>
+      <groupId>org.eclipse.jdt</groupId>
+      <artifactId>core</artifactId>
+      <name>Eclipse JDT Core</name>
+      <url>http://www.eclipse.org/jdt/</url>
+
+      <licenses>
+        <license>
+          <name>Eclipse Public License v1.0</name>
+          <url>http://www.eclipse.org/org/documents/epl-v10.php</url>
+          <distribution>repo</distribution>
+          <comments>
+Eclipse Public License - v 1.0
+
+THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC
+LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM
+CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
+
+1. DEFINITIONS
+
+"Contribution" means:
+
+    a) in the case of the initial Contributor, the initial code and
+documentation distributed under this Agreement, and
+
+    b) in the case of each subsequent Contributor:
+
+    i) changes to the Program, and
+
+    ii) additions to the Program;
+
+    where such changes and/or additions to the Program originate from and are
+distributed by that particular Contributor. A Contribution 'originates' from a
+Contributor if it was added to the Program by such Contributor itself or anyone
+acting on such Contributor's behalf. Contributions do not include additions to
+the Program which: (i) are separate modules of software distributed in
+conjunction with the Program under their own license agreement, and (ii) are
+not derivative works of the Program.
+
+"Contributor" means any person or entity that distributes the Program.
+
+"Licensed Patents" mean patent claims licensable by a Contributor which are
+necessarily infringed by the use or sale of its Contribution alone or when
+combined with the Program.
+
+"Program" means the Contributions distributed in accordance with this
+Agreement.
+
+"Recipient" means anyone who receives the Program under this Agreement,
+including all Contributors.
+
+2. GRANT OF RIGHTS
+
+a) Subject to the terms of this Agreement, each Contributor hereby grants
+Recipient a non-exclusive, worldwide, royalty-free copyright license to
+reproduce, prepare derivative works of, publicly display, publicly perform,
+distribute and sublicense the Contribution of such Contributor, if any, and
+such derivative works, in source code and object code form.
+
+b) Subject to the terms of this Agreement, each Contributor hereby grants
+Recipient a non-exclusive, worldwide, royalty-free patent license under
+Licensed Patents to make, use, sell, offer to sell, import and otherwise
+transfer the Contribution of such Contributor, if any, in source code and
+object code form. This patent license shall apply to the combination of the
+Contribution and the Program if, at the time the Contribution is added by the
+Contributor, such addition of the Contribution causes such combination to be
+covered by the Licensed Patents. The patent license shall not apply to any
+other combinations which include the Contribution. No hardware per se is
+licensed hereunder.
+
+c) Recipient understands that although each Contributor grants the licenses to
+its Contributions set forth herein, no assurances are provided by any
+Contributor that the Program does not infringe the patent or other intellectual
+property rights of any other entity. Each Contributor disclaims any liability
+to Recipient for claims brought by any other entity based on infringement of
+intellectual property rights or otherwise. As a condition to exercising the
+rights and licenses granted hereunder, each Recipient hereby assumes sole
+responsibility to secure any other intellectual property rights needed, if any.
+For example, if a third party patent license is required to allow Recipient to
+distribute the Program, it is Recipient's responsibility to acquire that
+license before distributing the Program.
+
+d) Each Contributor represents that to its knowledge it has sufficient
+copyright rights in its Contribution, if any, to grant the copyright license
+set forth in this Agreement.
+
+3. REQUIREMENTS
+
+A Contributor may choose to distribute the Program in object code form under
+its own license agreement, provided that:
+
+    a) it complies with the terms and conditions of this Agreement; and
+
+    b) its license agreement:
+
+    i) effectively disclaims on behalf of all Contributors all warranties and
+conditions, express and implied, including warranties or conditions of title
+and non-infringement, and implied warranties or conditions of merchantability
+and fitness for a particular purpose;
+
+    ii) effectively excludes on behalf of all Contributors all liability for
+damages, including direct, indirect, special, incidental and consequential
+damages, such as lost profits;
+
+    iii) states that any provisions which differ from this Agreement are
+offered by that Contributor alone and not by any other party; and
+
+    iv) states that source code for the Program is available from such
+Contributor, and informs licensees how to obtain it in a reasonable manner on
+or through a medium customarily used for software exchange.
+
+When the Program is made available in source code form:
+
+    a) it must be made available under this Agreement; and
+
+    b) a copy of this Agreement must be included with each copy of the Program.
+
+Contributors may not remove or alter any copyright notices contained within the
+Program.
+
+Each Contributor must identify itself as the originator of its Contribution, if
+any, in a manner that reasonably allows subsequent Recipients to identify the
+originator of the Contribution.
+
+4. COMMERCIAL DISTRIBUTION
+
+Commercial distributors of software may accept certain responsibilities with
+respect to end users, business partners and the like. While this license is
+intended to facilitate the commercial use of the Program, the Contributor who
+includes the Program in a commercial product offering should do so in a manner
+which does not create potential liability for other Contributors. Therefore, if
+a Contributor includes the Program in a commercial product offering, such
+Contributor ("Commercial Contributor") hereby agrees to defend and indemnify
+every other Contributor ("Indemnified Contributor") against any losses, damages
+and costs (collectively "Losses") arising from claims, lawsuits and other legal
+actions brought by a third party against the Indemnified Contributor to the
+extent caused by the acts or omissions of such Commercial Contributor in
+connection with its distribution of the Program in a commercial product
+offering. The obligations in this section do not apply to any claims or Losses
+relating to any actual or alleged intellectual property infringement. In order
+to qualify, an Indemnified Contributor must: a) promptly notify the Commercial
+Contributor in writing of such claim, and b) allow the Commercial Contributor
+to control, and cooperate with the Commercial Contributor in, the defense and
+any related settlement negotiations. The Indemnified Contributor may
+participate in any such claim at its own expense.
+
+For example, a Contributor might include the Program in a commercial product
+offering, Product X. That Contributor is then a Commercial Contributor. If that
+Commercial Contributor then makes performance claims, or offers warranties
+related to Product X, those performance claims and warranties are such
+Commercial Contributor's responsibility alone. Under this section, the
+Commercial Contributor would have to defend claims against the other
+Contributors related to those performance claims and warranties, and if a court
+requires any other Contributor to pay any damages as a result, the Commercial
+Contributor must pay those damages.
+
+5. NO WARRANTY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
+IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each
+Recipient is solely responsible for determining the appropriateness of using
+and distributing the Program and assumes all risks associated with its exercise
+of rights under this Agreement , including but not limited to the risks and
+costs of program errors, compliance with applicable laws, damage to or loss of
+data, programs or equipment, and unavailability or interruption of operations.
+
+6. DISCLAIMER OF LIABILITY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY
+CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
+PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS
+GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. GENERAL
+
+If any provision of this Agreement is invalid or unenforceable under applicable
+law, it shall not affect the validity or enforceability of the remainder of the
+terms of this Agreement, and without further action by the parties hereto, such
+provision shall be reformed to the minimum extent necessary to make such
+provision valid and enforceable.
+
+If Recipient institutes patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Program itself
+(excluding combinations of the Program with other software or hardware)
+infringes such Recipient's patent(s), then such Recipient's rights granted
+under Section 2(b) shall terminate as of the date such litigation is filed.
+
+All Recipient's rights under this Agreement shall terminate if it fails to
+comply with any of the material terms or conditions of this Agreement and does
+not cure such failure in a reasonable period of time after becoming aware of
+such noncompliance. If all Recipient's rights under this Agreement terminate,
+Recipient agrees to cease use and distribution of the Program as soon as
+reasonably practicable. However, Recipient's obligations under this Agreement
+and any licenses granted by Recipient relating to the Program shall continue
+and survive.
+
+Everyone is permitted to copy and distribute copies of this Agreement, but in
+order to avoid inconsistency the Agreement is copyrighted and may only be
+modified in the following manner. The Agreement Steward reserves the right to
+publish new versions (including revisions) of this Agreement from time to time.
+No one other than the Agreement Steward has the right to modify this Agreement.
+The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation
+may assign the responsibility to serve as the Agreement Steward to a suitable
+separate entity. Each new version of the Agreement will be given a
+distinguishing version number. The Program (including Contributions) may always
+be distributed subject to the version of the Agreement under which it was
+received. In addition, after a new version of the Agreement is published,
+Contributor may elect to distribute the Program (including its Contributions)
+under the new version. Except as expressly stated in Sections 2(a) and 2(b)
+above, Recipient receives no rights or licenses to the intellectual property of
+any Contributor under this Agreement, whether expressly, by implication,
+estoppel or otherwise. All rights in the Program not expressly granted under
+this Agreement are reserved.
+
+This Agreement is governed by the laws of the State of New York and the
+intellectual property laws of the United States of America. No party to this
+Agreement will bring a legal action under this Agreement more than one year
+after the cause of action arose. Each party waives its rights to a jury trial
+in any resulting litigation.
+</comments>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
+</supplementalDataModels>
diff --git a/src/main/appended-resources/META-INF/LICENSE b/src/main/appended-resources/META-INF/LICENSE
new file mode 100644
index 000000000000..6ec590ec20e6
--- /dev/null
+++ b/src/main/appended-resources/META-INF/LICENSE
@@ -0,0 +1,37 @@
+----
+This project incorporates portions of the 'Protocol Buffers' project avaialble
+under a '3-clause BSD' license.
+
+  Copyright 2008, Google Inc.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+      * Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+      * Neither the name of Google Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  Code generated by the Protocol Buffer compiler is owned by the owner
+  of the input file used when generating it.  This code is not
+  standalone and requires a support library to be linked with it.  This
+  support library is itself covered by the above license.
diff --git a/src/main/appended-resources/META-INF/NOTICE b/src/main/appended-resources/META-INF/NOTICE
new file mode 100644
index 000000000000..d8f61099f299
--- /dev/null
+++ b/src/main/appended-resources/META-INF/NOTICE
@@ -0,0 +1,6 @@
+--
+This product incorporates portions of the 'Hadoop' project
+
+Copyright 2007-2009 The Apache Software Foundation
+
+Licensed under the Apache License v2.0
diff --git a/src/main/resources/META-INF/LEGAL b/src/main/resources/META-INF/LEGAL
new file mode 100644
index 000000000000..9ac7f5a78b15
--- /dev/null
+++ b/src/main/resources/META-INF/LEGAL
@@ -0,0 +1,5 @@
+In Ruby's source distribution, this would describe a number of C source files
+that have different licenses than Ruby itself. None of those apply to JRuby,
+so we have this file here as a placeholder.
+
+For details of licensing of this collective work, see LICENSE
diff --git a/src/test/resources/META-INF/LICENSE b/src/test/resources/META-INF/LICENSE
new file mode 100644
index 000000000000..d64569567334
--- /dev/null
+++ b/src/test/resources/META-INF/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

From a25e9bebfee584505e994f8ff5354336f2b46f24 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Fri, 8 Jan 2016 13:50:24 -0800
Subject: [PATCH 1539/1540] HBASE-15084 Remove references to
 repository.codehaus.org.

---
 pom.xml | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/pom.xml b/pom.xml
index c0859e72ebe2..9d8314c6791d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -273,17 +273,6 @@
         <enabled>true</enabled>
       </releases>
     </repository>
-    <repository>
-      <id>codehaus</id>
-      <name>Codehaus Public</name>
-      <url>http://repository.codehaus.org/</url>
-      <snapshots>
-        <enabled>false</enabled>
-      </snapshots>
-      <releases>
-        <enabled>true</enabled>
-      </releases>
-    </repository>
     <repository>
       <id>repository.jboss.org</id>
       <url>http://repository.jboss.org/nexus/content/groups/public-jboss/</url>

From 34487ecc6f90f486325b625a6909de888008f4b2 Mon Sep 17 00:00:00 2001
From: Lars Hofhansl <larsh@apache.org>
Date: Tue, 12 Jan 2016 16:55:59 -0800
Subject: [PATCH 1540/1540] HBASE-14747 Addendum, do not generate a complete
 cross reference of all HBase code.

---
 pom.xml | 97 ---------------------------------------------------------
 1 file changed, 97 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9d8314c6791d..ccbcfbbea81d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2907,7 +2907,6 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
             <id>devapi</id>
             <reports>
               <report>aggregate</report>
-              <report>test-aggregate</report>
             </reports>
             <configuration>
               <destDir>devapidocs</destDir>
@@ -2920,50 +2919,6 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
               </sourceFileExcludes>
               <excludePackageNames>*.generated.master:*.generated:org.apache.hadoop.hbase.tmpl.common:com.google.protobuf:org.apache.hadoop.hbase.spark</excludePackageNames>
               <quiet>true</quiet>
-              <linksource>true</linksource>
-              <sourcetab>2</sourcetab>
-              <validateLinks>true</validateLinks>
-              <fixClassComment>true</fixClassComment>
-              <fixFieldComment>true</fixFieldComment>
-              <fixMethodComment>true</fixMethodComment>
-              <fixTags>all</fixTags>
-              <notimestamp>true</notimestamp>
-              <!-- Pass some options straight to the javadoc executable since it is easier -->
-              <additionalJOption>-J-Xmx2G</additionalJOption>
-              <!-- JDK8 javadoc requires test scope transitive dependencies due to our custom doclet -->
-              <additionalDependencies>
-                <additionalDependency>
-                  <groupId>org.mockito</groupId>
-                  <artifactId>mockito-all</artifactId>
-                  <version>${mockito-all.version}</version>
-                </additionalDependency>
-                <additionalDependency>
-                  <groupId>org.hamcrest</groupId>
-                  <artifactId>hamcrest-core</artifactId>
-                  <version>${hamcrest.version}</version>
-                </additionalDependency>
-              </additionalDependencies>
-              <inherited>false</inherited>
-            </configuration>
-          </reportSet>
-          <reportSet>
-            <id>testdevapi</id>
-            <reports>
-              <report>test-aggregate</report>
-            </reports>
-            <configuration>
-              <destDir>testdevapidocs</destDir>
-              <name>Developer API</name>
-              <description>The full HBase API, including private and unstable APIs</description>
-              <sourceFileExcludes>
-                <exclude>**/generated/*</exclude>
-                <exclude>**/protobuf/*</exclude>
-                <exclude>**/*.scala</exclude>
-              </sourceFileExcludes>
-              <excludePackageNames>*.generated.master:*.generated:org.apache.hadoop.hbase.tmpl.common:com.google.protobuf:org.apache.hadoop.hbase.spark</excludePackageNames>
-              <quiet>true</quiet>
-              <linksource>true</linksource>
-              <sourcetab>2</sourcetab>
               <validateLinks>true</validateLinks>
               <fixClassComment>true</fixClassComment>
               <fixFieldComment>true</fixFieldComment>
@@ -3014,58 +2969,6 @@ MAVEN_OPTS=-Xmx2048m mvn clean test -Pclover site -->
               <sourceFilesExclude>**/generated/*</sourceFilesExclude>
               <excludePackageNames>org.apache.hadoop.hbase.generated.master:org.apache.hadoop.hbase.protobuf.generated:org.apache.hadoop.hbase.tmpl.common</excludePackageNames>
               <quiet>true</quiet>
-              <linksource>true</linksource>
-              <sourcetab>2</sourcetab>
-              <validateLinks>true</validateLinks>
-              <fixClassComment>true</fixClassComment>
-              <fixFieldComment>true</fixFieldComment>
-              <fixMethodComment>true</fixMethodComment>
-              <fixTags>all</fixTags>
-              <notimestamp>true</notimestamp>
-              <!-- Pass some options straight to the javadoc executable since it is easier -->
-              <additionalJOption>-J-Xmx2G</additionalJOption>
-              <!-- JDK8 javadoc requires test scope transitive dependencies due to our custom doclet -->
-              <additionalDependencies>
-                <additionalDependency>
-                  <groupId>org.mockito</groupId>
-                  <artifactId>mockito-all</artifactId>
-                  <version>${mockito-all.version}</version>
-                </additionalDependency>
-                <additionalDependency>
-                  <groupId>org.hamcrest</groupId>
-                  <artifactId>hamcrest-core</artifactId>
-                  <version>${hamcrest.version}</version>
-                </additionalDependency>
-              </additionalDependencies>
-              <inherited>false</inherited>
-            </configuration>
-          </reportSet>
-          <reportSet>
-            <id>testuserapi</id>
-            <reports>
-              <report>test-aggregate</report>
-            </reports>
-            <configuration>
-              <destDir>testapidocs</destDir>
-              <name>User API</name>
-              <description>The HBase Application Programmer's API</description>
-              <excludePackageNames>
-                org.apache.hadoop.hbase.backup*:org.apache.hadoop.hbase.catalog:org.apache.hadoop.hbase.client.coprocessor:org.apache.hadoop.hbase.client.metrics:org.apache.hadoop.hbase.codec*:org.apache.hadoop.hbase.constraint:org.apache.hadoop.hbase.coprocessor.*:org.apache.hadoop.hbase.executor:org.apache.hadoop.hbase.fs:*.generated.*:org.apache.hadoop.hbase.io.hfile.*:org.apache.hadoop.hbase.mapreduce.hadoopbackport:org.apache.hadoop.hbase.mapreduce.replication:org.apache.hadoop.hbase.master.*:org.apache.hadoop.hbase.metrics*:org.apache.hadoop.hbase.migration:org.apache.hadoop.hbase.monitoring:org.apache.hadoop.hbase.p*:org.apache.hadoop.hbase.regionserver.compactions:org.apache.hadoop.hbase.regionserver.handler:org.apache.hadoop.hbase.regionserver.snapshot:org.apache.hadoop.hbase.replication.*:org.apache.hadoop.hbase.rest.filter:org.apache.hadoop.hbase.rest.model:org.apache.hadoop.hbase.rest.p*:org.apache.hadoop.hbase.security.*:org.apache.hadoop.hbase.thrift*:org.apache.hadoop.hbase.tmpl.*:org.apache.hadoop.hbase.tool:org.apache.hadoop.hbase.trace:org.apache.hadoop.hbase.util.byterange*:org.apache.hadoop.hbase.util.test:org.apache.hadoop.hbase.util.vint:org.apache.hadoop.hbase.zookeeper.lock:org.apache.hadoop.metrics2*
-              </excludePackageNames>
-              <!-- switch on dependency-driven aggregation -->
-              <includeDependencySources>false</includeDependencySources>
-              <dependencySourceIncludes>
-                <!-- include ONLY dependencies I control -->
-                <dependencySourceInclude>org.apache.hbase:hbase-annotations</dependencySourceInclude>
-              </dependencySourceIncludes>
-              <outputDirectory>${project.reporting.outputDirectory}/devapidocs</outputDirectory>
-              <name>Developer API</name>
-              <description>The full HBase API, including private and unstable APIs</description>
-              <sourceFilesExclude>**/generated/*</sourceFilesExclude>
-              <excludePackageNames>org.apache.hadoop.hbase.generated.master:org.apache.hadoop.hbase.protobuf.generated:org.apache.hadoop.hbase.tmpl.common</excludePackageNames>
-              <quiet>true</quiet>
-              <linksource>true</linksource>
-              <sourcetab>2</sourcetab>
               <validateLinks>true</validateLinks>
               <fixClassComment>true</fixClassComment>
               <fixFieldComment>true</fixFieldComment>